fix: content quality, image format, task monitor, calendar data source, search UI & sort
This commit is contained in:
@@ -321,6 +321,27 @@ def _extract_zhihu_post_id(url: str) -> str:
|
||||
raise ValueError("无法从URL中提取知乎文章/回答ID")
|
||||
|
||||
|
||||
def _fetch_zhihu_page(post_id: str) -> dict:
|
||||
"""多种方式尝试获取知乎文章数据,返回原始 JSON"""
|
||||
UA = _ZHIHU_UA
|
||||
# 方式1:专栏 API
|
||||
urls = [
|
||||
f"https://zhuanlan.zhihu.com/api/posts/{post_id}",
|
||||
f"https://www.zhihu.com/api/v4/posts/{post_id}",
|
||||
f"https://www.zhihu.com/api/v4/answers/{post_id}",
|
||||
]
|
||||
exceptions = []
|
||||
for url in urls:
|
||||
try:
|
||||
resp = http_requests.get(url, headers={"User-Agent": UA}, timeout=10)
|
||||
if resp.status_code == 200:
|
||||
return resp.json()
|
||||
exceptions.append(f"{url} → {resp.status_code}")
|
||||
except Exception as e:
|
||||
exceptions.append(f"{url} → {e}")
|
||||
raise RuntimeError(f"知乎API已封禁,无法获取数据({'; '.join(exceptions)})")
|
||||
|
||||
|
||||
class ZhihuFetchRequest(BaseModel):
|
||||
topic_id: str
|
||||
zhihu_url: str
|
||||
@@ -338,17 +359,9 @@ def fetch_zhihu_metrics(
|
||||
raise HTTPException(status_code=404, detail="选题不存在")
|
||||
|
||||
post_id = _extract_zhihu_post_id(data.zhihu_url)
|
||||
api_url = f"https://zhuanlan.zhihu.com/api/posts/{post_id}"
|
||||
|
||||
try:
|
||||
resp = http_requests.get(api_url, headers={"User-Agent": _ZHIHU_UA}, timeout=15)
|
||||
if resp.status_code == 404:
|
||||
api_url = f"https://www.zhihu.com/api/v4/answers/{post_id}"
|
||||
resp = http_requests.get(api_url, headers={"User-Agent": _ZHIHU_UA}, timeout=15)
|
||||
if resp.status_code != 200:
|
||||
raise HTTPException(status_code=502, detail=f"知乎API返回 {resp.status_code}")
|
||||
|
||||
raw = resp.json()
|
||||
raw = _fetch_zhihu_page(post_id)
|
||||
platform = "zhihu"
|
||||
|
||||
existing = db.query(ContentMetrics).filter(
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"""发布管理 API"""
|
||||
from fastapi import APIRouter, HTTPException, Depends, Request
|
||||
from pydantic import BaseModel
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional, List
|
||||
|
||||
from ..database import get_db
|
||||
@@ -86,8 +86,9 @@ async def create_publish_record(
|
||||
))
|
||||
|
||||
topic.status = '已发布'
|
||||
topic.updated_at = datetime.now()
|
||||
topic.published_at = datetime.now().date()
|
||||
_now = datetime.now(timezone(timedelta(hours=8)))
|
||||
topic.updated_at = _now
|
||||
topic.published_at = _now.date()
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
db.refresh(topic)
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import logging
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import List, Optional
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from ..database import get_db
|
||||
from ..models import SearchProvider
|
||||
from .auth import get_current_user
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/search-providers", tags=["search_providers"])
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_providers(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
||||
providers = db.query(SearchProvider).order_by(SearchProvider.priority).all()
|
||||
return [p.to_dict() for p in providers]
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_provider(data: dict, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
||||
p = SearchProvider(
|
||||
name=data.get("name", ""),
|
||||
provider_type=data.get("provider_type", ""),
|
||||
api_key=data.get("api_key", ""),
|
||||
api_url=data.get("api_url", ""),
|
||||
console_url=data.get("console_url", ""),
|
||||
priority=data.get("priority", 99),
|
||||
enabled=data.get("enabled", True),
|
||||
daily_limit=data.get("daily_limit", 1500),
|
||||
extra_config=data.get("extra_config", {}),
|
||||
)
|
||||
db.add(p)
|
||||
db.commit()
|
||||
db.refresh(p)
|
||||
return p.to_dict()
|
||||
|
||||
|
||||
@router.put("/{provider_id}")
|
||||
def update_provider(provider_id: int, data: dict, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
||||
p = db.query(SearchProvider).filter(SearchProvider.id == provider_id).first()
|
||||
if not p:
|
||||
raise HTTPException(status_code=404, detail="Provider not found")
|
||||
for key in ("name", "provider_type", "api_key", "api_url", "console_url", "priority", "enabled", "daily_limit"):
|
||||
if key in data:
|
||||
setattr(p, key, data[key])
|
||||
if "extra_config" in data:
|
||||
p.extra_config = data["extra_config"]
|
||||
p.updated_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
db.refresh(p)
|
||||
return p.to_dict()
|
||||
|
||||
|
||||
@router.delete("/{provider_id}")
|
||||
def delete_provider(provider_id: int, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
||||
p = db.query(SearchProvider).filter(SearchProvider.id == provider_id).first()
|
||||
if not p:
|
||||
raise HTTPException(status_code=404, detail="Provider not found")
|
||||
db.delete(p)
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/{provider_id}/test")
|
||||
def test_provider(provider_id: int, data: dict = {}, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
||||
p = db.query(SearchProvider).filter(SearchProvider.id == provider_id).first()
|
||||
if not p:
|
||||
raise HTTPException(status_code=404, detail="Provider not found")
|
||||
query = data.get("query", "测试搜索")
|
||||
try:
|
||||
test_key = data.get("api_key") or p.api_key
|
||||
test_url = data.get("api_url") or p.api_url
|
||||
if p.provider_type == "baidu":
|
||||
import requests
|
||||
resp = requests.post(
|
||||
test_url,
|
||||
headers={"Authorization": f"Bearer {test_key}", "Content-Type": "application/json"},
|
||||
json={"messages": [{"role": "user", "content": query}], "search_source": "baidu_search_v2", "resource_type_filter": [{"type": "web", "top_k": 3}]},
|
||||
timeout=15
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return {"ok": False, "error": f"HTTP {resp.status_code}: {resp.text[:200]}"}
|
||||
return {"ok": True, "results": resp.json().get("results", [])[:3]}
|
||||
elif p.provider_type == "qiniu":
|
||||
import requests
|
||||
resp = requests.post(
|
||||
test_url,
|
||||
headers={"Authorization": f"Bearer {test_key}", "Content-Type": "application/json"},
|
||||
json={"query": query, "max_results": 3, "search_type": "web"},
|
||||
timeout=15
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return {"ok": False, "error": f"HTTP {resp.status_code}: {resp.text[:200]}"}
|
||||
return {"ok": True, "results": resp.json().get("results", resp.json().get("data", []))[:3]}
|
||||
elif p.provider_type == "tinyfish":
|
||||
import requests
|
||||
resp = requests.get(
|
||||
test_url,
|
||||
params={"query": query, "max_results": 3},
|
||||
headers={"X-API-Key": test_key},
|
||||
timeout=15
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return {"ok": False, "error": f"HTTP {resp.status_code}: {resp.text[:200]}"}
|
||||
return {"ok": True, "results": resp.json().get("results", resp.json().get("data", []))[:3]}
|
||||
elif p.provider_type == "bing":
|
||||
import requests
|
||||
resp = requests.get(
|
||||
test_url,
|
||||
params={"q": query, "count": 3, "mkt": "zh-CN"},
|
||||
headers={"Ocp-Apim-Subscription-Key": test_key},
|
||||
timeout=15
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return {"ok": False, "error": f"HTTP {resp.status_code}: {resp.text[:200]}"}
|
||||
return {"ok": True, "results": resp.json().get("webPages", {}).get("value", [])[:3]}
|
||||
elif p.provider_type == "mcp":
|
||||
import subprocess, json as _json
|
||||
mcp_script = Path(__file__).resolve().parent.parent.parent.parent.parent / "scripts" / "mcp_search_server.py"
|
||||
r = subprocess.run(
|
||||
[sys.executable, str(mcp_script), "--query", query],
|
||||
capture_output=True, text=True, timeout=90,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
return {"ok": False, "error": f"子进程失败: {r.stderr[:200]}"}
|
||||
return {"ok": True, "results": _json.loads(r.stdout)[:3]}
|
||||
return {"ok": False, "error": f"Unknown provider_type: {p.provider_type}"}
|
||||
except Exception as e:
|
||||
return {"ok": False, "error": str(e)}
|
||||
|
||||
|
||||
@router.post("/reset-usage")
|
||||
def reset_usage(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
||||
db.query(SearchProvider).update({SearchProvider.usage_today: 0})
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
@@ -3,16 +3,17 @@ import subprocess
|
||||
from fastapi import APIRouter, HTTPException, Depends, Body
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func
|
||||
from datetime import datetime, date
|
||||
from datetime import datetime, date, timezone, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Optional
|
||||
import os
|
||||
import os
|
||||
import json
|
||||
from ..database import get_db
|
||||
from ..models import Topic, Article, TaskConfig, TaskLog
|
||||
from ..core.generator import run_creator, get_generator_status
|
||||
from ..core.optimizer import run_optimizer, get_optimizer_status
|
||||
from ..core.collector import run_collector, get_collector_status
|
||||
from ..core.generator import run_creator, get_generator_status, _running_processes as _generator_running
|
||||
from ..core.optimizer import run_optimizer, get_optimizer_status, _running_processes as _optimizer_running
|
||||
from ..core.collector import run_collector, get_collector_status, _running_processes as _collector_running
|
||||
import threading
|
||||
from ..core.sync import sync_all_topics
|
||||
from ..core.scheduler import scheduler
|
||||
@@ -26,6 +27,32 @@ LOGS_DIR = PROJECT_ROOT / "automation" / "logs"
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/system", tags=["system"])
|
||||
|
||||
_active_monitors: Dict[int, dict] = {}
|
||||
|
||||
def _monitor_subprocess(log_id: int, proc, module_id: str, task_name: str, db_session_factory):
|
||||
"""阻塞等待子进程退出(最长 1800s),完成后更新 task_logs"""
|
||||
try:
|
||||
returncode = proc.wait(timeout=1800)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
returncode = -1
|
||||
logger.warning("Subprocess %s (pid=%s) killed after 1800s timeout", module_id, proc.pid)
|
||||
finished_at = datetime.now(timezone.utc)
|
||||
try:
|
||||
db = db_session_factory()
|
||||
log = db.query(TaskLog).filter(TaskLog.id == log_id).first()
|
||||
if log:
|
||||
log.status = "success" if returncode == 0 else "failed"
|
||||
log.finished_at = finished_at
|
||||
if log.started_at:
|
||||
log.duration = int((finished_at - log.started_at).total_seconds())
|
||||
db.commit()
|
||||
db.close()
|
||||
except Exception as e:
|
||||
logger.warning("Failed to update task log %s: %s", log_id, e)
|
||||
finally:
|
||||
_active_monitors.pop(log_id, None)
|
||||
|
||||
def _aggregate_status_counts(q):
|
||||
"""聚合状态计数,兼容中英文状态值"""
|
||||
raw = q.with_entities(Topic.status, func.count()).group_by(Topic.status).all()
|
||||
@@ -60,12 +87,22 @@ def get_status(db: Session = Depends(get_db)):
|
||||
}
|
||||
}
|
||||
|
||||
@router.post("/generate/run", dependencies=[Depends(get_current_user)])
|
||||
@router.post("/generate/run")
|
||||
def trigger_generation(topic_id: Optional[str] = None, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
||||
logger.info(f"Generation triggered by {current_user.username}, topic_id={topic_id}")
|
||||
try:
|
||||
result = run_creator(topic_id)
|
||||
return {"message": "内容创作已后台启动", "pid": result.get("pid")}
|
||||
from ..database import SessionLocal
|
||||
log = TaskLog(module_id="scheduled_generate", task_name="🤖 内容创作", status="running", message="内容创作已启动", triggered_by="manual", started_at=datetime.now(timezone.utc))
|
||||
db.add(log)
|
||||
db.commit()
|
||||
log_id = log.id
|
||||
proc_info = result.get("proc") or result
|
||||
proc = _generator_running.get("generator", {}).get("process") if "pid" in result else None
|
||||
if proc:
|
||||
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_generate", "🤖 内容创作", SessionLocal), daemon=True)
|
||||
t.start()
|
||||
return {"message": "内容创作已后台启动", "pid": result.get("pid"), "log_id": log_id}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@@ -76,12 +113,21 @@ def generation_status():
|
||||
return {"status": "idle", "message": "当前无运行中的创作任务"}
|
||||
return status
|
||||
|
||||
@router.post("/collect/run", dependencies=[Depends(get_current_user)])
|
||||
@router.post("/collect/run")
|
||||
def trigger_collection(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
||||
logger.info(f"Manual collection triggered by {current_user.username}")
|
||||
try:
|
||||
result = run_collector()
|
||||
return {"message": "内容采集已后台启动", "result": result}
|
||||
from ..database import SessionLocal
|
||||
log = TaskLog(module_id="scheduled_collect", task_name="📡 内容采集", status="running", message="内容采集已启动", triggered_by="manual", started_at=datetime.now(timezone.utc))
|
||||
db.add(log)
|
||||
db.commit()
|
||||
log_id = log.id
|
||||
proc = _collector_running.get("collector", {}).get("process") if "pid" in result else None
|
||||
if proc:
|
||||
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_collect", "📡 内容采集", SessionLocal), daemon=True)
|
||||
t.start()
|
||||
return {"message": "内容采集已后台启动", "result": result, "log_id": log_id}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@@ -92,11 +138,20 @@ def collection_status():
|
||||
return {"status": "idle", "message": "当前无运行中的采集任务"}
|
||||
return status
|
||||
|
||||
@router.post("/review/run", dependencies=[Depends(get_current_user)])
|
||||
@router.post("/review/run")
|
||||
def trigger_review(topic_ids: Optional[List[str]] = None, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
||||
try:
|
||||
result = run_optimizer(topic_ids)
|
||||
return {"message": "合规审查已后台启动", "pid": result.get("pid")}
|
||||
from ..database import SessionLocal
|
||||
log = TaskLog(module_id="scheduled_optimize", task_name="🔍 合规审查", status="running", message="合规审查已启动", triggered_by="manual", started_at=datetime.now(timezone.utc))
|
||||
db.add(log)
|
||||
db.commit()
|
||||
log_id = log.id
|
||||
proc = _optimizer_running.get("optimizer", {}).get("process") if "pid" in result else None
|
||||
if proc:
|
||||
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_optimize", "🔍 合规审查", SessionLocal), daemon=True)
|
||||
t.start()
|
||||
return {"message": "合规审查已后台启动", "pid": result.get("pid"), "log_id": log_id}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@@ -177,17 +232,24 @@ def trigger_metrics_sync():
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@router.post("/refresh-search-cache/run")
|
||||
def trigger_refresh_search_cache():
|
||||
def trigger_refresh_search_cache(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
||||
try:
|
||||
import sys as sys_mod
|
||||
scripts_dir = Path(__file__).parent.parent.parent.parent / "scripts"
|
||||
from ..database import SessionLocal as _ss
|
||||
proc = subprocess.Popen(
|
||||
[sys_mod.executable, str(scripts_dir / "opencode_search.py"), "--refresh-cache"],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
||||
cwd=scripts_dir.parent.parent
|
||||
)
|
||||
logger.info("Search cache refresh started (pid=%s)", proc.pid)
|
||||
return {"message": "搜索缓存刷新已后台启动", "pid": proc.pid}
|
||||
log = TaskLog(module_id="scheduled_refresh_search_cache", task_name="🔍 搜索缓存", status="running", message="搜索缓存刷新已启动", triggered_by="manual", started_at=datetime.now(timezone.utc), result_data={"pid": proc.pid})
|
||||
db.add(log)
|
||||
db.commit()
|
||||
log_id = log.id
|
||||
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_refresh_search_cache", "🔍 搜索缓存", _ss), daemon=True)
|
||||
t.start()
|
||||
return {"message": "搜索缓存刷新已后台启动", "pid": proc.pid, "log_id": log_id}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@@ -259,6 +321,7 @@ def get_modules_status(db: Session = Depends(get_db)):
|
||||
"scheduled_optimize": {"name": "🔍 合规审查", "cron": "03:00", "params_desc": {"auto_pass_threshold": "自动通过分数阈值"}},
|
||||
"scheduled_optimize_sources": {"name": "📡 信息源优化", "cron": "05:00", "params_desc": {}},
|
||||
"scheduled_metrics_sync": {"name": "📊 指标同步", "cron": "06:00", "params_desc": {}},
|
||||
"scheduled_task_monitor": {"name": "⏰ 任务监控", "cron": "*", "params_desc": {}},
|
||||
}
|
||||
|
||||
modules = []
|
||||
|
||||
@@ -19,6 +19,7 @@ MODULES = {
|
||||
"scheduled_optimize": "🔍 合规审查",
|
||||
"scheduled_optimize_sources": "📡 信息源优化",
|
||||
"scheduled_metrics_sync": "📊 指标同步",
|
||||
"scheduled_task_monitor": "⏰ 任务监控",
|
||||
}
|
||||
|
||||
@router.get("", response_model=List[TaskLogResponse])
|
||||
@@ -68,6 +69,29 @@ def list_modules(db: Session = Depends(get_db), admin_user=Depends(get_current_a
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/log-types")
|
||||
def list_log_types(db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
|
||||
used = db.query(TaskLog.module_id).distinct().all()
|
||||
used_ids = [r[0] for r in used]
|
||||
result = []
|
||||
for mid, name in MODULES.items():
|
||||
if mid in used_ids or True:
|
||||
log_file_map = {
|
||||
"scheduled_refresh_search_cache": "opencode_search",
|
||||
"scheduled_fetch_trends": "trends",
|
||||
"scheduled_collect": "collector",
|
||||
"scheduled_generate": "creator",
|
||||
"scheduled_optimize": "optimizer",
|
||||
"scheduled_optimize_sources": "collector",
|
||||
"scheduled_metrics_sync": "sync",
|
||||
}
|
||||
result.append({"module_id": mid, "name": name, "log_file": log_file_map.get(mid, mid)})
|
||||
for mid in used_ids:
|
||||
if mid not in MODULES:
|
||||
result.append({"module_id": mid, "name": mid, "log_file": mid})
|
||||
return result
|
||||
|
||||
@router.get("/{log_id}", response_model=TaskLogResponse)
|
||||
def get_task_log(log_id: int, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
|
||||
log = db.query(TaskLog).filter(TaskLog.id == log_id).first()
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import uuid
|
||||
import threading
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import List, Optional
|
||||
@@ -8,6 +9,8 @@ from ..models import ContentTask, Topic
|
||||
from ..schemas import ContentTaskCreate, ContentTaskResponse
|
||||
from .auth import get_current_user, org_filter
|
||||
|
||||
_creator_semaphore = threading.Semaphore(3)
|
||||
|
||||
router = APIRouter(prefix="/api/tasks", tags=["tasks"])
|
||||
|
||||
|
||||
@@ -240,6 +243,7 @@ def _get_module_detail_data(module_id: str, db, ROOT, DATA_DIR, LOGS_DIR, today_
|
||||
"scheduled_optimize": {"name": "🔍 合规审查", "description": "LLM 审查已创作文章,检查合规、打分数、优化建议"},
|
||||
"scheduled_optimize_sources": {"name": "📡 信息源优化", "description": "AI 分析当前类别和信息源的市场匹配度,给出调整建议"},
|
||||
"scheduled_metrics_sync": {"name": "📊 指标同步", "description": "从各平台公开 API 获取已发布文章的互动数据(点赞、阅读、评论等)"},
|
||||
"scheduled_task_monitor": {"name": "⏰ 任务监控", "description": "每小时自动检查卡死/中断任务,标记为失败以便重新执行"},
|
||||
}
|
||||
|
||||
meta = MODULE_META.get(module_id, {"name": module_id, "description": ""})
|
||||
@@ -430,7 +434,6 @@ def run_creator_task(
|
||||
db: Session = Depends(get_db),
|
||||
current_user=Depends(get_current_user)
|
||||
):
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
@@ -448,44 +451,51 @@ def run_creator_task(
|
||||
db.commit()
|
||||
db.refresh(task)
|
||||
|
||||
from ..core.generator import run_creator
|
||||
|
||||
def _run():
|
||||
from ..database import SessionLocal
|
||||
from ..core.generator import run_creator_blocking
|
||||
from datetime import datetime, timezone
|
||||
new_db = SessionLocal()
|
||||
import traceback
|
||||
_creator_semaphore.acquire()
|
||||
try:
|
||||
new_task = new_db.query(ContentTask).filter(ContentTask.task_id == task_id).first()
|
||||
if new_task:
|
||||
new_task.message = "创作脚本运行中..."
|
||||
new_task.progress = 30
|
||||
new_db.commit()
|
||||
result = run_creator(topic_id)
|
||||
new_task = new_db.query(ContentTask).filter(ContentTask.task_id == task_id).first()
|
||||
if new_task:
|
||||
finished = datetime.now(timezone.utc)
|
||||
new_task.status = "completed"
|
||||
new_task.finished_at = finished
|
||||
new_task.progress = 100
|
||||
new_task.message = "创作完成"
|
||||
new_task.result_data = result or {}
|
||||
if new_task.started_at:
|
||||
new_task.duration = int((finished - new_task.started_at).total_seconds())
|
||||
new_db.commit()
|
||||
except Exception as e:
|
||||
new_task = new_db.query(ContentTask).filter(ContentTask.task_id == task_id).first()
|
||||
if new_task:
|
||||
finished = datetime.now(timezone.utc)
|
||||
new_task.status = "failed"
|
||||
new_task.finished_at = finished
|
||||
new_task.error_msg = str(e)
|
||||
if new_task.started_at:
|
||||
new_task.duration = int((finished - new_task.started_at).total_seconds())
|
||||
new_db.commit()
|
||||
finally:
|
||||
new_db.close()
|
||||
new_db = SessionLocal()
|
||||
try:
|
||||
new_task = new_db.query(ContentTask).filter(ContentTask.task_id == task_id).first()
|
||||
if new_task:
|
||||
new_task.message = "创作脚本运行中..."
|
||||
new_task.progress = 30
|
||||
new_db.commit()
|
||||
|
||||
thread = threading.Thread(target=_run)
|
||||
result = run_creator_blocking(topic_id)
|
||||
|
||||
new_task = new_db.query(ContentTask).filter(ContentTask.task_id == task_id).first()
|
||||
if new_task:
|
||||
finished = datetime.now(timezone.utc)
|
||||
new_task.status = "completed"
|
||||
new_task.finished_at = finished
|
||||
new_task.progress = 100
|
||||
new_task.message = "创作完成"
|
||||
new_task.result_data = {"stdout": (result or {}).get("stdout", "")[:2000]} if isinstance(result, dict) else {"raw": str(result)[:2000]}
|
||||
if new_task.started_at:
|
||||
new_task.duration = int((finished - new_task.started_at).total_seconds())
|
||||
new_db.commit()
|
||||
except Exception as e:
|
||||
new_db.rollback()
|
||||
new_task = new_db.query(ContentTask).filter(ContentTask.task_id == task_id).first()
|
||||
if new_task:
|
||||
finished = datetime.now(timezone.utc)
|
||||
new_task.status = "failed"
|
||||
new_task.finished_at = finished
|
||||
new_task.error_msg = f"{type(e).__name__}: {e}\n{traceback.format_exc()}"
|
||||
if new_task.started_at:
|
||||
new_task.duration = int((finished - new_task.started_at).total_seconds())
|
||||
new_db.commit()
|
||||
finally:
|
||||
new_db.close()
|
||||
finally:
|
||||
_creator_semaphore.release()
|
||||
|
||||
thread = threading.Thread(target=_run, daemon=True)
|
||||
thread.start()
|
||||
|
||||
return task
|
||||
@@ -27,6 +27,8 @@ MODULES = {
|
||||
"scheduled_optimize": {"name": "🔍 合规审查", "cron": "03:00"},
|
||||
"scheduled_optimize_sources": {"name": "📡 信息源优化", "cron": "05:00"},
|
||||
"scheduled_metrics_sync": {"name": "📊 指标同步", "cron": "06:00"},
|
||||
"scheduled_reset_search_usage": {"name": "🔁 搜索用量重置", "cron": "00:05"},
|
||||
"scheduled_task_monitor": {"name": "⏰ 任务监控", "cron": "*"},
|
||||
}
|
||||
|
||||
def _log_task(module_id: str, status: str, message: str = None,
|
||||
@@ -107,6 +109,7 @@ class TaskScheduler:
|
||||
("scheduled_optimize", self._run_optimize, "合规审查"),
|
||||
("scheduled_optimize_sources", self._run_optimize_sources, "信息源优化"),
|
||||
("scheduled_metrics_sync", self._run_metrics_sync, "指标同步"),
|
||||
("scheduled_reset_search_usage", self._run_reset_search_usage, "搜索用量重置"),
|
||||
]
|
||||
|
||||
for module_id, fn, name in MODULE_JOBS:
|
||||
@@ -129,6 +132,17 @@ class TaskScheduler:
|
||||
)
|
||||
logger.info(f"调度任务: {module_id} -> {schedule}")
|
||||
|
||||
# 每小时运行的任务监控:检测卡死/中断任务
|
||||
self.scheduler.add_job(
|
||||
self._run_task_monitor,
|
||||
CronTrigger(hour='*/1'),
|
||||
id='scheduled_task_monitor',
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
coalesce=True
|
||||
)
|
||||
logger.info("调度任务: scheduled_task_monitor -> 每小时")
|
||||
|
||||
self.scheduler.start()
|
||||
self._started = True
|
||||
logger.info("Scheduler started with dynamic schedule from TaskConfig")
|
||||
@@ -441,6 +455,95 @@ class TaskScheduler:
|
||||
started_at=started, finished_at=datetime.now(timezone.utc))
|
||||
logger.exception("[Scheduled] Metrics sync failed: %s", e)
|
||||
|
||||
def _run_reset_search_usage(self):
|
||||
"""每日凌晨重置搜索 API 提供商用量计数"""
|
||||
started = datetime.now(timezone.utc)
|
||||
_log_task("scheduled_reset_search_usage", "running", started_at=started)
|
||||
try:
|
||||
from ..database import SessionLocal
|
||||
from ..models import SearchProvider
|
||||
db = SessionLocal()
|
||||
try:
|
||||
total = db.query(SearchProvider).update({SearchProvider.usage_today: 0, SearchProvider.last_used_at: None})
|
||||
db.commit()
|
||||
_log_task("scheduled_reset_search_usage", "success",
|
||||
message=f"已重置 {total} 个提供商用量",
|
||||
result_data={"reset_count": total},
|
||||
started_at=started, finished_at=datetime.now(timezone.utc))
|
||||
logger.info("[Scheduled] Reset %d search providers usage", total)
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as e:
|
||||
_log_task("scheduled_reset_search_usage", "failed",
|
||||
message=str(e),
|
||||
error_trace=traceback.format_exc(),
|
||||
started_at=started, finished_at=datetime.now(timezone.utc))
|
||||
logger.exception("[Scheduled] Reset search usage failed: %s", e)
|
||||
|
||||
def _run_task_monitor(self):
|
||||
"""每小时检查卡死/中断的任务,标记为失败"""
|
||||
started = datetime.now(timezone.utc)
|
||||
_log_task("scheduled_task_monitor", "running", started_at=started)
|
||||
stuck_tasklog_timeout = 7200 # 超过2小时视为卡死
|
||||
stuck_contenttask_timeout = 10800 # 超过3小时视为卡死
|
||||
try:
|
||||
from ..database import SessionLocal
|
||||
from ..models import TaskLog, ContentTask
|
||||
db = SessionLocal()
|
||||
try:
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff_tasklog = now.timestamp() - stuck_tasklog_timeout
|
||||
cutoff_content = now.timestamp() - stuck_contenttask_timeout
|
||||
marked = 0
|
||||
|
||||
# 检查 TaskLog 中卡死的 running 记录
|
||||
stuck_logs = db.query(TaskLog).filter(
|
||||
TaskLog.status == "running",
|
||||
TaskLog.started_at.isnot(None)
|
||||
).all()
|
||||
for log in stuck_logs:
|
||||
if log.started_at.timestamp() < cutoff_tasklog:
|
||||
log.status = "failed"
|
||||
log.finished_at = now
|
||||
log.error_trace = "系统监控:任务运行超时(超过2小时)或进程中断,已自动标记为失败"
|
||||
if log.started_at:
|
||||
log.duration = int((now - log.started_at).total_seconds())
|
||||
marked += 1
|
||||
logger.warning("[TaskMonitor] 标记 TaskLog %d (%s) 为失败(超时)", log.id, log.module_id)
|
||||
|
||||
# 检查 ContentTask 中卡死的 running 记录
|
||||
stuck_tasks = db.query(ContentTask).filter(
|
||||
ContentTask.status == "running",
|
||||
ContentTask.started_at.isnot(None)
|
||||
).all()
|
||||
for task in stuck_tasks:
|
||||
if task.started_at.timestamp() < cutoff_content:
|
||||
task.status = "failed"
|
||||
task.finished_at = now
|
||||
task.error_msg = "系统监控:任务运行超时(超过3小时)或进程中断,已自动标记为失败"
|
||||
if task.started_at:
|
||||
task.duration = int((now - task.started_at).total_seconds())
|
||||
marked += 1
|
||||
logger.warning("[TaskMonitor] 标记 ContentTask %s (%s) 为失败(超时)", task.task_id, task.stage)
|
||||
|
||||
if marked:
|
||||
db.commit()
|
||||
logger.info("[TaskMonitor] 已标记 %d 个卡死任务为失败", marked)
|
||||
|
||||
_log_task("scheduled_task_monitor", "success",
|
||||
message=f"检查完成,标记 {marked} 个卡死任务",
|
||||
result_data={"marked_failed": marked},
|
||||
started_at=started, finished_at=datetime.now(timezone.utc))
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as e:
|
||||
import traceback
|
||||
_log_task("scheduled_task_monitor", "failed",
|
||||
message=str(e),
|
||||
error_trace=traceback.format_exc(),
|
||||
started_at=started, finished_at=datetime.now(timezone.utc))
|
||||
logger.exception("[TaskMonitor] 监控检查失败: %s", e)
|
||||
|
||||
def get_jobs(self):
|
||||
"""返回当前所有定时任务的状态"""
|
||||
jobs = []
|
||||
|
||||
@@ -53,6 +53,7 @@ def init_db():
|
||||
for table, col, typ in [
|
||||
("users", "org_id", "VARCHAR DEFAULT 'default'"),
|
||||
("topics", "org_id", "VARCHAR DEFAULT 'default'"),
|
||||
("topics", "reviewed_at", "TIMESTAMP"),
|
||||
("platform_configs", "requires_image", "BOOLEAN DEFAULT FALSE"),
|
||||
("platform_configs", "image_count_min", "INTEGER DEFAULT 0"),
|
||||
("platform_configs", "image_count_max", "INTEGER DEFAULT 0"),
|
||||
@@ -82,6 +83,7 @@ def init_db():
|
||||
("prompt_configs", "temperature", "FLOAT"),
|
||||
("prompt_configs", "max_tokens", "INTEGER"),
|
||||
("prompt_configs", "created_by", "VARCHAR"),
|
||||
("search_providers", "console_url", "VARCHAR"),
|
||||
("keyword_domain_map", "id", "INTEGER PRIMARY KEY"),
|
||||
("keyword_domain_map", "pattern", "VARCHAR"),
|
||||
("keyword_domain_map", "domain", "VARCHAR"),
|
||||
@@ -124,10 +126,11 @@ def init_db():
|
||||
conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {col} {typ}"))
|
||||
except Exception:
|
||||
pass
|
||||
# Create roles and menus tables if they don't exist
|
||||
# Create roles, menus, and search_providers tables if they don't exist
|
||||
for tbl_sql in [
|
||||
"CREATE TABLE IF NOT EXISTS roles (id SERIAL PRIMARY KEY, name VARCHAR UNIQUE NOT NULL, description VARCHAR DEFAULT '', is_system BOOLEAN DEFAULT FALSE, created_at TIMESTAMP WITH TIME ZONE DEFAULT now())",
|
||||
"CREATE TABLE IF NOT EXISTS menus (id SERIAL PRIMARY KEY, parent_id INTEGER REFERENCES menus(id), name VARCHAR NOT NULL, path VARCHAR NOT NULL, icon VARCHAR DEFAULT '', sort_order INTEGER DEFAULT 0, roles JSON DEFAULT '[]'::json, is_active BOOLEAN DEFAULT TRUE, created_at TIMESTAMP WITH TIME ZONE DEFAULT now())",
|
||||
"CREATE TABLE IF NOT EXISTS search_providers (id SERIAL PRIMARY KEY, name VARCHAR NOT NULL, provider_type VARCHAR NOT NULL, api_key VARCHAR, api_url VARCHAR, priority INTEGER DEFAULT 1, enabled BOOLEAN DEFAULT TRUE, daily_limit INTEGER DEFAULT 1500, usage_today INTEGER DEFAULT 0, extra_config JSON DEFAULT '{}'::json, last_used_at TIMESTAMP, created_at TIMESTAMP WITH TIME ZONE DEFAULT now(), updated_at TIMESTAMP WITH TIME ZONE DEFAULT now())",
|
||||
]:
|
||||
try:
|
||||
conn.execute(text(tbl_sql))
|
||||
|
||||
@@ -6,7 +6,7 @@ from .database import SessionLocal, init_db
|
||||
from .models import (
|
||||
Topic, TopicField, TopicConfigField, TopicStatusConfig,
|
||||
User, Case, LLMConfig, SystemConfig, PlatformConfig,
|
||||
CollectorCategory, CollectorSource, Role, Menu
|
||||
CollectorCategory, CollectorSource, Role, Menu, SearchProvider
|
||||
)
|
||||
import bcrypt
|
||||
|
||||
@@ -71,6 +71,17 @@ def import_initial_data():
|
||||
db.commit()
|
||||
print("✅ 插入默认系统配置")
|
||||
|
||||
# 初始化默认搜索 API 提供商
|
||||
if db.query(SearchProvider).count() == 0:
|
||||
providers = [
|
||||
SearchProvider(name="百度千帆", provider_type="baidu", api_key="", api_url="https://qianfan.baidubce.com/v2/ai_search/web_search", console_url="https://console.bce.baidu.com/qianfan/ais/console/onlineService", priority=1, enabled=True, daily_limit=50),
|
||||
SearchProvider(name="opencode云搜索", provider_type="mcp", api_key="", api_url="", console_url="https://opencode.ai", priority=2, enabled=True, daily_limit=99999),
|
||||
]
|
||||
for p in providers:
|
||||
db.add(p)
|
||||
db.commit()
|
||||
print("✅ 插入默认搜索 API 提供商")
|
||||
|
||||
if db.query(PlatformConfig).count() == 0:
|
||||
platforms = [
|
||||
{
|
||||
|
||||
@@ -9,7 +9,7 @@ from pathlib import Path
|
||||
|
||||
from .database import engine, get_db, init_db
|
||||
from .models import Base
|
||||
from .api import topics, system, articles, publishing, auth, admin, audit, optimizer_logs, cases, task_logs, task_configs, prompt_configs, llm_configs, system_configs, topic_config, calendar, metrics, assets, tasks, platform_config, collector_mgmt, assistant, config_items, role_configs, menu_configs
|
||||
from .api import topics, system, articles, publishing, auth, admin, audit, optimizer_logs, cases, task_logs, task_configs, prompt_configs, llm_configs, system_configs, topic_config, calendar, metrics, assets, tasks, platform_config, collector_mgmt, assistant, config_items, role_configs, menu_configs, search_providers
|
||||
from .initial_data import import_initial_data
|
||||
from .core.scheduler import scheduler
|
||||
|
||||
@@ -103,6 +103,7 @@ app.include_router(config_items.router)
|
||||
app.include_router(role_configs.router)
|
||||
app.include_router(menu_configs.router)
|
||||
app.include_router(menu_configs.public_router)
|
||||
app.include_router(search_providers.router)
|
||||
|
||||
# 挂载自动生成的图片(必须先于前端根挂载)
|
||||
PROJECT_ROOT_DIR = Path(__file__).parent.parent.parent.parent
|
||||
|
||||
@@ -2,7 +2,44 @@ from sqlalchemy import Column, String, Integer, Float, Date, DateTime, Text, Boo
|
||||
from sqlalchemy.sql import func
|
||||
from sqlalchemy.orm import relationship
|
||||
from .database import Base
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
class SearchProvider(Base):
|
||||
__tablename__ = "search_providers"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True, autoincrement=True)
|
||||
name = Column(String, nullable=False, comment="显示名称")
|
||||
provider_type = Column(String, nullable=False, comment="baidu / qiniu / tinyfish")
|
||||
api_key = Column(String, nullable=True, comment="API密钥")
|
||||
api_url = Column(String, nullable=True, comment="API地址")
|
||||
priority = Column(Integer, default=1, comment="优先级,越小越优先")
|
||||
enabled = Column(Boolean, default=True)
|
||||
daily_limit = Column(Integer, default=1500, comment="每日调用上限")
|
||||
usage_today = Column(Integer, default=0, comment="当日已用次数")
|
||||
console_url = Column(String, nullable=True, comment="官网控制台地址")
|
||||
extra_config = Column(JSON, default=dict, comment="额外配置")
|
||||
last_used_at = Column(DateTime(timezone=True), nullable=True)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), onupdate=func.now())
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"id": self.id,
|
||||
"name": self.name,
|
||||
"provider_type": self.provider_type,
|
||||
"api_key": self.api_key,
|
||||
"api_url": self.api_url,
|
||||
"console_url": self.console_url,
|
||||
"priority": self.priority,
|
||||
"enabled": self.enabled,
|
||||
"daily_limit": self.daily_limit,
|
||||
"usage_today": self.usage_today,
|
||||
"extra_config": self.extra_config or {},
|
||||
"last_used_at": self.last_used_at.isoformat() if self.last_used_at else None,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
class AuditLog(Base):
|
||||
@@ -221,6 +258,7 @@ class Topic(Base):
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), onupdate=func.now())
|
||||
generated_at = Column(DateTime(timezone=True), nullable=True)
|
||||
reviewed_at = Column(DateTime(timezone=True), nullable=True)
|
||||
ready_at = Column(Date)
|
||||
published_at = Column(Date)
|
||||
compliance_score = Column(Integer)
|
||||
|
||||
@@ -112,6 +112,7 @@ class TopicResponse(TopicBase):
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
generated_at: Optional[datetime] = None
|
||||
reviewed_at: Optional[datetime] = None
|
||||
ready_at: Optional[date] = None
|
||||
published_at: Optional[date] = None
|
||||
compliance_score: Optional[int] = None
|
||||
|
||||
Reference in New Issue
Block a user