fix: content quality, image format, task monitor, calendar data source, search UI & sort

This commit is contained in:
Yuzhiran Dev
2026-05-26 11:26:10 +08:00
parent b2d043b231
commit ac8644d752
36 changed files with 2294 additions and 873 deletions
+22 -9
View File
@@ -321,6 +321,27 @@ def _extract_zhihu_post_id(url: str) -> str:
raise ValueError("无法从URL中提取知乎文章/回答ID")
def _fetch_zhihu_page(post_id: str) -> dict:
"""多种方式尝试获取知乎文章数据,返回原始 JSON"""
UA = _ZHIHU_UA
# 方式1:专栏 API
urls = [
f"https://zhuanlan.zhihu.com/api/posts/{post_id}",
f"https://www.zhihu.com/api/v4/posts/{post_id}",
f"https://www.zhihu.com/api/v4/answers/{post_id}",
]
exceptions = []
for url in urls:
try:
resp = http_requests.get(url, headers={"User-Agent": UA}, timeout=10)
if resp.status_code == 200:
return resp.json()
exceptions.append(f"{url}{resp.status_code}")
except Exception as e:
exceptions.append(f"{url}{e}")
raise RuntimeError(f"知乎API已封禁,无法获取数据({'; '.join(exceptions)}")
class ZhihuFetchRequest(BaseModel):
topic_id: str
zhihu_url: str
@@ -338,17 +359,9 @@ def fetch_zhihu_metrics(
raise HTTPException(status_code=404, detail="选题不存在")
post_id = _extract_zhihu_post_id(data.zhihu_url)
api_url = f"https://zhuanlan.zhihu.com/api/posts/{post_id}"
try:
resp = http_requests.get(api_url, headers={"User-Agent": _ZHIHU_UA}, timeout=15)
if resp.status_code == 404:
api_url = f"https://www.zhihu.com/api/v4/answers/{post_id}"
resp = http_requests.get(api_url, headers={"User-Agent": _ZHIHU_UA}, timeout=15)
if resp.status_code != 200:
raise HTTPException(status_code=502, detail=f"知乎API返回 {resp.status_code}")
raw = resp.json()
raw = _fetch_zhihu_page(post_id)
platform = "zhihu"
existing = db.query(ContentMetrics).filter(
+4 -3
View File
@@ -2,7 +2,7 @@
"""发布管理 API"""
from fastapi import APIRouter, HTTPException, Depends, Request
from pydantic import BaseModel
from datetime import datetime
from datetime import datetime, timezone, timedelta
from typing import Optional, List
from ..database import get_db
@@ -86,8 +86,9 @@ async def create_publish_record(
))
topic.status = '已发布'
topic.updated_at = datetime.now()
topic.published_at = datetime.now().date()
_now = datetime.now(timezone(timedelta(hours=8)))
topic.updated_at = _now
topic.published_at = _now.date()
db.commit()
db.expire_all()
db.refresh(topic)
@@ -0,0 +1,141 @@
import logging
import sys
import json
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session
from typing import List, Optional
from datetime import datetime, timezone
from ..database import get_db
from ..models import SearchProvider
from .auth import get_current_user
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/search-providers", tags=["search_providers"])
@router.get("")
def list_providers(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
providers = db.query(SearchProvider).order_by(SearchProvider.priority).all()
return [p.to_dict() for p in providers]
@router.post("")
def create_provider(data: dict, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
p = SearchProvider(
name=data.get("name", ""),
provider_type=data.get("provider_type", ""),
api_key=data.get("api_key", ""),
api_url=data.get("api_url", ""),
console_url=data.get("console_url", ""),
priority=data.get("priority", 99),
enabled=data.get("enabled", True),
daily_limit=data.get("daily_limit", 1500),
extra_config=data.get("extra_config", {}),
)
db.add(p)
db.commit()
db.refresh(p)
return p.to_dict()
@router.put("/{provider_id}")
def update_provider(provider_id: int, data: dict, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
p = db.query(SearchProvider).filter(SearchProvider.id == provider_id).first()
if not p:
raise HTTPException(status_code=404, detail="Provider not found")
for key in ("name", "provider_type", "api_key", "api_url", "console_url", "priority", "enabled", "daily_limit"):
if key in data:
setattr(p, key, data[key])
if "extra_config" in data:
p.extra_config = data["extra_config"]
p.updated_at = datetime.now(timezone.utc)
db.commit()
db.refresh(p)
return p.to_dict()
@router.delete("/{provider_id}")
def delete_provider(provider_id: int, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
p = db.query(SearchProvider).filter(SearchProvider.id == provider_id).first()
if not p:
raise HTTPException(status_code=404, detail="Provider not found")
db.delete(p)
db.commit()
return {"ok": True}
@router.post("/{provider_id}/test")
def test_provider(provider_id: int, data: dict = {}, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
p = db.query(SearchProvider).filter(SearchProvider.id == provider_id).first()
if not p:
raise HTTPException(status_code=404, detail="Provider not found")
query = data.get("query", "测试搜索")
try:
test_key = data.get("api_key") or p.api_key
test_url = data.get("api_url") or p.api_url
if p.provider_type == "baidu":
import requests
resp = requests.post(
test_url,
headers={"Authorization": f"Bearer {test_key}", "Content-Type": "application/json"},
json={"messages": [{"role": "user", "content": query}], "search_source": "baidu_search_v2", "resource_type_filter": [{"type": "web", "top_k": 3}]},
timeout=15
)
if resp.status_code != 200:
return {"ok": False, "error": f"HTTP {resp.status_code}: {resp.text[:200]}"}
return {"ok": True, "results": resp.json().get("results", [])[:3]}
elif p.provider_type == "qiniu":
import requests
resp = requests.post(
test_url,
headers={"Authorization": f"Bearer {test_key}", "Content-Type": "application/json"},
json={"query": query, "max_results": 3, "search_type": "web"},
timeout=15
)
if resp.status_code != 200:
return {"ok": False, "error": f"HTTP {resp.status_code}: {resp.text[:200]}"}
return {"ok": True, "results": resp.json().get("results", resp.json().get("data", []))[:3]}
elif p.provider_type == "tinyfish":
import requests
resp = requests.get(
test_url,
params={"query": query, "max_results": 3},
headers={"X-API-Key": test_key},
timeout=15
)
if resp.status_code != 200:
return {"ok": False, "error": f"HTTP {resp.status_code}: {resp.text[:200]}"}
return {"ok": True, "results": resp.json().get("results", resp.json().get("data", []))[:3]}
elif p.provider_type == "bing":
import requests
resp = requests.get(
test_url,
params={"q": query, "count": 3, "mkt": "zh-CN"},
headers={"Ocp-Apim-Subscription-Key": test_key},
timeout=15
)
if resp.status_code != 200:
return {"ok": False, "error": f"HTTP {resp.status_code}: {resp.text[:200]}"}
return {"ok": True, "results": resp.json().get("webPages", {}).get("value", [])[:3]}
elif p.provider_type == "mcp":
import subprocess, json as _json
mcp_script = Path(__file__).resolve().parent.parent.parent.parent.parent / "scripts" / "mcp_search_server.py"
r = subprocess.run(
[sys.executable, str(mcp_script), "--query", query],
capture_output=True, text=True, timeout=90,
)
if r.returncode != 0:
return {"ok": False, "error": f"子进程失败: {r.stderr[:200]}"}
return {"ok": True, "results": _json.loads(r.stdout)[:3]}
return {"ok": False, "error": f"Unknown provider_type: {p.provider_type}"}
except Exception as e:
return {"ok": False, "error": str(e)}
@router.post("/reset-usage")
def reset_usage(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
db.query(SearchProvider).update({SearchProvider.usage_today: 0})
db.commit()
return {"ok": True}
+75 -12
View File
@@ -3,16 +3,17 @@ import subprocess
from fastapi import APIRouter, HTTPException, Depends, Body
from sqlalchemy.orm import Session
from sqlalchemy import func
from datetime import datetime, date
from datetime import datetime, date, timezone, timedelta
from pathlib import Path
from typing import Dict, Any, List, Optional
import os
import os
import json
from ..database import get_db
from ..models import Topic, Article, TaskConfig, TaskLog
from ..core.generator import run_creator, get_generator_status
from ..core.optimizer import run_optimizer, get_optimizer_status
from ..core.collector import run_collector, get_collector_status
from ..core.generator import run_creator, get_generator_status, _running_processes as _generator_running
from ..core.optimizer import run_optimizer, get_optimizer_status, _running_processes as _optimizer_running
from ..core.collector import run_collector, get_collector_status, _running_processes as _collector_running
import threading
from ..core.sync import sync_all_topics
from ..core.scheduler import scheduler
@@ -26,6 +27,32 @@ LOGS_DIR = PROJECT_ROOT / "automation" / "logs"
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/system", tags=["system"])
_active_monitors: Dict[int, dict] = {}
def _monitor_subprocess(log_id: int, proc, module_id: str, task_name: str, db_session_factory):
"""阻塞等待子进程退出(最长 1800s),完成后更新 task_logs"""
try:
returncode = proc.wait(timeout=1800)
except subprocess.TimeoutExpired:
proc.kill()
returncode = -1
logger.warning("Subprocess %s (pid=%s) killed after 1800s timeout", module_id, proc.pid)
finished_at = datetime.now(timezone.utc)
try:
db = db_session_factory()
log = db.query(TaskLog).filter(TaskLog.id == log_id).first()
if log:
log.status = "success" if returncode == 0 else "failed"
log.finished_at = finished_at
if log.started_at:
log.duration = int((finished_at - log.started_at).total_seconds())
db.commit()
db.close()
except Exception as e:
logger.warning("Failed to update task log %s: %s", log_id, e)
finally:
_active_monitors.pop(log_id, None)
def _aggregate_status_counts(q):
"""聚合状态计数,兼容中英文状态值"""
raw = q.with_entities(Topic.status, func.count()).group_by(Topic.status).all()
@@ -60,12 +87,22 @@ def get_status(db: Session = Depends(get_db)):
}
}
@router.post("/generate/run", dependencies=[Depends(get_current_user)])
@router.post("/generate/run")
def trigger_generation(topic_id: Optional[str] = None, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
logger.info(f"Generation triggered by {current_user.username}, topic_id={topic_id}")
try:
result = run_creator(topic_id)
return {"message": "内容创作已后台启动", "pid": result.get("pid")}
from ..database import SessionLocal
log = TaskLog(module_id="scheduled_generate", task_name="🤖 内容创作", status="running", message="内容创作已启动", triggered_by="manual", started_at=datetime.now(timezone.utc))
db.add(log)
db.commit()
log_id = log.id
proc_info = result.get("proc") or result
proc = _generator_running.get("generator", {}).get("process") if "pid" in result else None
if proc:
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_generate", "🤖 内容创作", SessionLocal), daemon=True)
t.start()
return {"message": "内容创作已后台启动", "pid": result.get("pid"), "log_id": log_id}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@@ -76,12 +113,21 @@ def generation_status():
return {"status": "idle", "message": "当前无运行中的创作任务"}
return status
@router.post("/collect/run", dependencies=[Depends(get_current_user)])
@router.post("/collect/run")
def trigger_collection(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
logger.info(f"Manual collection triggered by {current_user.username}")
try:
result = run_collector()
return {"message": "内容采集已后台启动", "result": result}
from ..database import SessionLocal
log = TaskLog(module_id="scheduled_collect", task_name="📡 内容采集", status="running", message="内容采集已启动", triggered_by="manual", started_at=datetime.now(timezone.utc))
db.add(log)
db.commit()
log_id = log.id
proc = _collector_running.get("collector", {}).get("process") if "pid" in result else None
if proc:
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_collect", "📡 内容采集", SessionLocal), daemon=True)
t.start()
return {"message": "内容采集已后台启动", "result": result, "log_id": log_id}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@@ -92,11 +138,20 @@ def collection_status():
return {"status": "idle", "message": "当前无运行中的采集任务"}
return status
@router.post("/review/run", dependencies=[Depends(get_current_user)])
@router.post("/review/run")
def trigger_review(topic_ids: Optional[List[str]] = None, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
try:
result = run_optimizer(topic_ids)
return {"message": "合规审查已后台启动", "pid": result.get("pid")}
from ..database import SessionLocal
log = TaskLog(module_id="scheduled_optimize", task_name="🔍 合规审查", status="running", message="合规审查已启动", triggered_by="manual", started_at=datetime.now(timezone.utc))
db.add(log)
db.commit()
log_id = log.id
proc = _optimizer_running.get("optimizer", {}).get("process") if "pid" in result else None
if proc:
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_optimize", "🔍 合规审查", SessionLocal), daemon=True)
t.start()
return {"message": "合规审查已后台启动", "pid": result.get("pid"), "log_id": log_id}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@@ -177,17 +232,24 @@ def trigger_metrics_sync():
raise HTTPException(status_code=500, detail=str(e))
@router.post("/refresh-search-cache/run")
def trigger_refresh_search_cache():
def trigger_refresh_search_cache(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
try:
import sys as sys_mod
scripts_dir = Path(__file__).parent.parent.parent.parent / "scripts"
from ..database import SessionLocal as _ss
proc = subprocess.Popen(
[sys_mod.executable, str(scripts_dir / "opencode_search.py"), "--refresh-cache"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
cwd=scripts_dir.parent.parent
)
logger.info("Search cache refresh started (pid=%s)", proc.pid)
return {"message": "搜索缓存刷新已后台启动", "pid": proc.pid}
log = TaskLog(module_id="scheduled_refresh_search_cache", task_name="🔍 搜索缓存", status="running", message="搜索缓存刷新已启动", triggered_by="manual", started_at=datetime.now(timezone.utc), result_data={"pid": proc.pid})
db.add(log)
db.commit()
log_id = log.id
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_refresh_search_cache", "🔍 搜索缓存", _ss), daemon=True)
t.start()
return {"message": "搜索缓存刷新已后台启动", "pid": proc.pid, "log_id": log_id}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@@ -259,6 +321,7 @@ def get_modules_status(db: Session = Depends(get_db)):
"scheduled_optimize": {"name": "🔍 合规审查", "cron": "03:00", "params_desc": {"auto_pass_threshold": "自动通过分数阈值"}},
"scheduled_optimize_sources": {"name": "📡 信息源优化", "cron": "05:00", "params_desc": {}},
"scheduled_metrics_sync": {"name": "📊 指标同步", "cron": "06:00", "params_desc": {}},
"scheduled_task_monitor": {"name": "⏰ 任务监控", "cron": "*", "params_desc": {}},
}
modules = []
+24
View File
@@ -19,6 +19,7 @@ MODULES = {
"scheduled_optimize": "🔍 合规审查",
"scheduled_optimize_sources": "📡 信息源优化",
"scheduled_metrics_sync": "📊 指标同步",
"scheduled_task_monitor": "⏰ 任务监控",
}
@router.get("", response_model=List[TaskLogResponse])
@@ -68,6 +69,29 @@ def list_modules(db: Session = Depends(get_db), admin_user=Depends(get_current_a
})
return result
@router.get("/log-types")
def list_log_types(db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
used = db.query(TaskLog.module_id).distinct().all()
used_ids = [r[0] for r in used]
result = []
for mid, name in MODULES.items():
if mid in used_ids or True:
log_file_map = {
"scheduled_refresh_search_cache": "opencode_search",
"scheduled_fetch_trends": "trends",
"scheduled_collect": "collector",
"scheduled_generate": "creator",
"scheduled_optimize": "optimizer",
"scheduled_optimize_sources": "collector",
"scheduled_metrics_sync": "sync",
}
result.append({"module_id": mid, "name": name, "log_file": log_file_map.get(mid, mid)})
for mid in used_ids:
if mid not in MODULES:
result.append({"module_id": mid, "name": mid, "log_file": mid})
return result
@router.get("/{log_id}", response_model=TaskLogResponse)
def get_task_log(log_id: int, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
log = db.query(TaskLog).filter(TaskLog.id == log_id).first()
+44 -34
View File
@@ -1,4 +1,5 @@
import uuid
import threading
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.orm import Session
from typing import List, Optional
@@ -8,6 +9,8 @@ from ..models import ContentTask, Topic
from ..schemas import ContentTaskCreate, ContentTaskResponse
from .auth import get_current_user, org_filter
_creator_semaphore = threading.Semaphore(3)
router = APIRouter(prefix="/api/tasks", tags=["tasks"])
@@ -240,6 +243,7 @@ def _get_module_detail_data(module_id: str, db, ROOT, DATA_DIR, LOGS_DIR, today_
"scheduled_optimize": {"name": "🔍 合规审查", "description": "LLM 审查已创作文章,检查合规、打分数、优化建议"},
"scheduled_optimize_sources": {"name": "📡 信息源优化", "description": "AI 分析当前类别和信息源的市场匹配度,给出调整建议"},
"scheduled_metrics_sync": {"name": "📊 指标同步", "description": "从各平台公开 API 获取已发布文章的互动数据(点赞、阅读、评论等)"},
"scheduled_task_monitor": {"name": "⏰ 任务监控", "description": "每小时自动检查卡死/中断任务,标记为失败以便重新执行"},
}
meta = MODULE_META.get(module_id, {"name": module_id, "description": ""})
@@ -430,7 +434,6 @@ def run_creator_task(
db: Session = Depends(get_db),
current_user=Depends(get_current_user)
):
import threading
from datetime import datetime, timezone
now = datetime.now(timezone.utc)
@@ -448,44 +451,51 @@ def run_creator_task(
db.commit()
db.refresh(task)
from ..core.generator import run_creator
def _run():
from ..database import SessionLocal
from ..core.generator import run_creator_blocking
from datetime import datetime, timezone
new_db = SessionLocal()
import traceback
_creator_semaphore.acquire()
try:
new_task = new_db.query(ContentTask).filter(ContentTask.task_id == task_id).first()
if new_task:
new_task.message = "创作脚本运行中..."
new_task.progress = 30
new_db.commit()
result = run_creator(topic_id)
new_task = new_db.query(ContentTask).filter(ContentTask.task_id == task_id).first()
if new_task:
finished = datetime.now(timezone.utc)
new_task.status = "completed"
new_task.finished_at = finished
new_task.progress = 100
new_task.message = "创作完成"
new_task.result_data = result or {}
if new_task.started_at:
new_task.duration = int((finished - new_task.started_at).total_seconds())
new_db.commit()
except Exception as e:
new_task = new_db.query(ContentTask).filter(ContentTask.task_id == task_id).first()
if new_task:
finished = datetime.now(timezone.utc)
new_task.status = "failed"
new_task.finished_at = finished
new_task.error_msg = str(e)
if new_task.started_at:
new_task.duration = int((finished - new_task.started_at).total_seconds())
new_db.commit()
finally:
new_db.close()
new_db = SessionLocal()
try:
new_task = new_db.query(ContentTask).filter(ContentTask.task_id == task_id).first()
if new_task:
new_task.message = "创作脚本运行中..."
new_task.progress = 30
new_db.commit()
thread = threading.Thread(target=_run)
result = run_creator_blocking(topic_id)
new_task = new_db.query(ContentTask).filter(ContentTask.task_id == task_id).first()
if new_task:
finished = datetime.now(timezone.utc)
new_task.status = "completed"
new_task.finished_at = finished
new_task.progress = 100
new_task.message = "创作完成"
new_task.result_data = {"stdout": (result or {}).get("stdout", "")[:2000]} if isinstance(result, dict) else {"raw": str(result)[:2000]}
if new_task.started_at:
new_task.duration = int((finished - new_task.started_at).total_seconds())
new_db.commit()
except Exception as e:
new_db.rollback()
new_task = new_db.query(ContentTask).filter(ContentTask.task_id == task_id).first()
if new_task:
finished = datetime.now(timezone.utc)
new_task.status = "failed"
new_task.finished_at = finished
new_task.error_msg = f"{type(e).__name__}: {e}\n{traceback.format_exc()}"
if new_task.started_at:
new_task.duration = int((finished - new_task.started_at).total_seconds())
new_db.commit()
finally:
new_db.close()
finally:
_creator_semaphore.release()
thread = threading.Thread(target=_run, daemon=True)
thread.start()
return task
+103
View File
@@ -27,6 +27,8 @@ MODULES = {
"scheduled_optimize": {"name": "🔍 合规审查", "cron": "03:00"},
"scheduled_optimize_sources": {"name": "📡 信息源优化", "cron": "05:00"},
"scheduled_metrics_sync": {"name": "📊 指标同步", "cron": "06:00"},
"scheduled_reset_search_usage": {"name": "🔁 搜索用量重置", "cron": "00:05"},
"scheduled_task_monitor": {"name": "⏰ 任务监控", "cron": "*"},
}
def _log_task(module_id: str, status: str, message: str = None,
@@ -107,6 +109,7 @@ class TaskScheduler:
("scheduled_optimize", self._run_optimize, "合规审查"),
("scheduled_optimize_sources", self._run_optimize_sources, "信息源优化"),
("scheduled_metrics_sync", self._run_metrics_sync, "指标同步"),
("scheduled_reset_search_usage", self._run_reset_search_usage, "搜索用量重置"),
]
for module_id, fn, name in MODULE_JOBS:
@@ -129,6 +132,17 @@ class TaskScheduler:
)
logger.info(f"调度任务: {module_id} -> {schedule}")
# 每小时运行的任务监控:检测卡死/中断任务
self.scheduler.add_job(
self._run_task_monitor,
CronTrigger(hour='*/1'),
id='scheduled_task_monitor',
replace_existing=True,
max_instances=1,
coalesce=True
)
logger.info("调度任务: scheduled_task_monitor -> 每小时")
self.scheduler.start()
self._started = True
logger.info("Scheduler started with dynamic schedule from TaskConfig")
@@ -441,6 +455,95 @@ class TaskScheduler:
started_at=started, finished_at=datetime.now(timezone.utc))
logger.exception("[Scheduled] Metrics sync failed: %s", e)
def _run_reset_search_usage(self):
"""每日凌晨重置搜索 API 提供商用量计数"""
started = datetime.now(timezone.utc)
_log_task("scheduled_reset_search_usage", "running", started_at=started)
try:
from ..database import SessionLocal
from ..models import SearchProvider
db = SessionLocal()
try:
total = db.query(SearchProvider).update({SearchProvider.usage_today: 0, SearchProvider.last_used_at: None})
db.commit()
_log_task("scheduled_reset_search_usage", "success",
message=f"已重置 {total} 个提供商用量",
result_data={"reset_count": total},
started_at=started, finished_at=datetime.now(timezone.utc))
logger.info("[Scheduled] Reset %d search providers usage", total)
finally:
db.close()
except Exception as e:
_log_task("scheduled_reset_search_usage", "failed",
message=str(e),
error_trace=traceback.format_exc(),
started_at=started, finished_at=datetime.now(timezone.utc))
logger.exception("[Scheduled] Reset search usage failed: %s", e)
def _run_task_monitor(self):
"""每小时检查卡死/中断的任务,标记为失败"""
started = datetime.now(timezone.utc)
_log_task("scheduled_task_monitor", "running", started_at=started)
stuck_tasklog_timeout = 7200 # 超过2小时视为卡死
stuck_contenttask_timeout = 10800 # 超过3小时视为卡死
try:
from ..database import SessionLocal
from ..models import TaskLog, ContentTask
db = SessionLocal()
try:
now = datetime.now(timezone.utc)
cutoff_tasklog = now.timestamp() - stuck_tasklog_timeout
cutoff_content = now.timestamp() - stuck_contenttask_timeout
marked = 0
# 检查 TaskLog 中卡死的 running 记录
stuck_logs = db.query(TaskLog).filter(
TaskLog.status == "running",
TaskLog.started_at.isnot(None)
).all()
for log in stuck_logs:
if log.started_at.timestamp() < cutoff_tasklog:
log.status = "failed"
log.finished_at = now
log.error_trace = "系统监控:任务运行超时(超过2小时)或进程中断,已自动标记为失败"
if log.started_at:
log.duration = int((now - log.started_at).total_seconds())
marked += 1
logger.warning("[TaskMonitor] 标记 TaskLog %d (%s) 为失败(超时)", log.id, log.module_id)
# 检查 ContentTask 中卡死的 running 记录
stuck_tasks = db.query(ContentTask).filter(
ContentTask.status == "running",
ContentTask.started_at.isnot(None)
).all()
for task in stuck_tasks:
if task.started_at.timestamp() < cutoff_content:
task.status = "failed"
task.finished_at = now
task.error_msg = "系统监控:任务运行超时(超过3小时)或进程中断,已自动标记为失败"
if task.started_at:
task.duration = int((now - task.started_at).total_seconds())
marked += 1
logger.warning("[TaskMonitor] 标记 ContentTask %s (%s) 为失败(超时)", task.task_id, task.stage)
if marked:
db.commit()
logger.info("[TaskMonitor] 已标记 %d 个卡死任务为失败", marked)
_log_task("scheduled_task_monitor", "success",
message=f"检查完成,标记 {marked} 个卡死任务",
result_data={"marked_failed": marked},
started_at=started, finished_at=datetime.now(timezone.utc))
finally:
db.close()
except Exception as e:
import traceback
_log_task("scheduled_task_monitor", "failed",
message=str(e),
error_trace=traceback.format_exc(),
started_at=started, finished_at=datetime.now(timezone.utc))
logger.exception("[TaskMonitor] 监控检查失败: %s", e)
def get_jobs(self):
"""返回当前所有定时任务的状态"""
jobs = []
+4 -1
View File
@@ -53,6 +53,7 @@ def init_db():
for table, col, typ in [
("users", "org_id", "VARCHAR DEFAULT 'default'"),
("topics", "org_id", "VARCHAR DEFAULT 'default'"),
("topics", "reviewed_at", "TIMESTAMP"),
("platform_configs", "requires_image", "BOOLEAN DEFAULT FALSE"),
("platform_configs", "image_count_min", "INTEGER DEFAULT 0"),
("platform_configs", "image_count_max", "INTEGER DEFAULT 0"),
@@ -82,6 +83,7 @@ def init_db():
("prompt_configs", "temperature", "FLOAT"),
("prompt_configs", "max_tokens", "INTEGER"),
("prompt_configs", "created_by", "VARCHAR"),
("search_providers", "console_url", "VARCHAR"),
("keyword_domain_map", "id", "INTEGER PRIMARY KEY"),
("keyword_domain_map", "pattern", "VARCHAR"),
("keyword_domain_map", "domain", "VARCHAR"),
@@ -124,10 +126,11 @@ def init_db():
conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {col} {typ}"))
except Exception:
pass
# Create roles and menus tables if they don't exist
# Create roles, menus, and search_providers tables if they don't exist
for tbl_sql in [
"CREATE TABLE IF NOT EXISTS roles (id SERIAL PRIMARY KEY, name VARCHAR UNIQUE NOT NULL, description VARCHAR DEFAULT '', is_system BOOLEAN DEFAULT FALSE, created_at TIMESTAMP WITH TIME ZONE DEFAULT now())",
"CREATE TABLE IF NOT EXISTS menus (id SERIAL PRIMARY KEY, parent_id INTEGER REFERENCES menus(id), name VARCHAR NOT NULL, path VARCHAR NOT NULL, icon VARCHAR DEFAULT '', sort_order INTEGER DEFAULT 0, roles JSON DEFAULT '[]'::json, is_active BOOLEAN DEFAULT TRUE, created_at TIMESTAMP WITH TIME ZONE DEFAULT now())",
"CREATE TABLE IF NOT EXISTS search_providers (id SERIAL PRIMARY KEY, name VARCHAR NOT NULL, provider_type VARCHAR NOT NULL, api_key VARCHAR, api_url VARCHAR, priority INTEGER DEFAULT 1, enabled BOOLEAN DEFAULT TRUE, daily_limit INTEGER DEFAULT 1500, usage_today INTEGER DEFAULT 0, extra_config JSON DEFAULT '{}'::json, last_used_at TIMESTAMP, created_at TIMESTAMP WITH TIME ZONE DEFAULT now(), updated_at TIMESTAMP WITH TIME ZONE DEFAULT now())",
]:
try:
conn.execute(text(tbl_sql))
+12 -1
View File
@@ -6,7 +6,7 @@ from .database import SessionLocal, init_db
from .models import (
Topic, TopicField, TopicConfigField, TopicStatusConfig,
User, Case, LLMConfig, SystemConfig, PlatformConfig,
CollectorCategory, CollectorSource, Role, Menu
CollectorCategory, CollectorSource, Role, Menu, SearchProvider
)
import bcrypt
@@ -71,6 +71,17 @@ def import_initial_data():
db.commit()
print("✅ 插入默认系统配置")
# 初始化默认搜索 API 提供商
if db.query(SearchProvider).count() == 0:
providers = [
SearchProvider(name="百度千帆", provider_type="baidu", api_key="", api_url="https://qianfan.baidubce.com/v2/ai_search/web_search", console_url="https://console.bce.baidu.com/qianfan/ais/console/onlineService", priority=1, enabled=True, daily_limit=50),
SearchProvider(name="opencode云搜索", provider_type="mcp", api_key="", api_url="", console_url="https://opencode.ai", priority=2, enabled=True, daily_limit=99999),
]
for p in providers:
db.add(p)
db.commit()
print("✅ 插入默认搜索 API 提供商")
if db.query(PlatformConfig).count() == 0:
platforms = [
{
+2 -1
View File
@@ -9,7 +9,7 @@ from pathlib import Path
from .database import engine, get_db, init_db
from .models import Base
from .api import topics, system, articles, publishing, auth, admin, audit, optimizer_logs, cases, task_logs, task_configs, prompt_configs, llm_configs, system_configs, topic_config, calendar, metrics, assets, tasks, platform_config, collector_mgmt, assistant, config_items, role_configs, menu_configs
from .api import topics, system, articles, publishing, auth, admin, audit, optimizer_logs, cases, task_logs, task_configs, prompt_configs, llm_configs, system_configs, topic_config, calendar, metrics, assets, tasks, platform_config, collector_mgmt, assistant, config_items, role_configs, menu_configs, search_providers
from .initial_data import import_initial_data
from .core.scheduler import scheduler
@@ -103,6 +103,7 @@ app.include_router(config_items.router)
app.include_router(role_configs.router)
app.include_router(menu_configs.router)
app.include_router(menu_configs.public_router)
app.include_router(search_providers.router)
# 挂载自动生成的图片(必须先于前端根挂载)
PROJECT_ROOT_DIR = Path(__file__).parent.parent.parent.parent
+39 -1
View File
@@ -2,7 +2,44 @@ from sqlalchemy import Column, String, Integer, Float, Date, DateTime, Text, Boo
from sqlalchemy.sql import func
from sqlalchemy.orm import relationship
from .database import Base
from datetime import datetime
from datetime import datetime, timezone
class SearchProvider(Base):
__tablename__ = "search_providers"
id = Column(Integer, primary_key=True, index=True, autoincrement=True)
name = Column(String, nullable=False, comment="显示名称")
provider_type = Column(String, nullable=False, comment="baidu / qiniu / tinyfish")
api_key = Column(String, nullable=True, comment="API密钥")
api_url = Column(String, nullable=True, comment="API地址")
priority = Column(Integer, default=1, comment="优先级,越小越优先")
enabled = Column(Boolean, default=True)
daily_limit = Column(Integer, default=1500, comment="每日调用上限")
usage_today = Column(Integer, default=0, comment="当日已用次数")
console_url = Column(String, nullable=True, comment="官网控制台地址")
extra_config = Column(JSON, default=dict, comment="额外配置")
last_used_at = Column(DateTime(timezone=True), nullable=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), onupdate=func.now())
def to_dict(self):
return {
"id": self.id,
"name": self.name,
"provider_type": self.provider_type,
"api_key": self.api_key,
"api_url": self.api_url,
"console_url": self.console_url,
"priority": self.priority,
"enabled": self.enabled,
"daily_limit": self.daily_limit,
"usage_today": self.usage_today,
"extra_config": self.extra_config or {},
"last_used_at": self.last_used_at.isoformat() if self.last_used_at else None,
"created_at": self.created_at.isoformat() if self.created_at else None,
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
}
class AuditLog(Base):
@@ -221,6 +258,7 @@ class Topic(Base):
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), onupdate=func.now())
generated_at = Column(DateTime(timezone=True), nullable=True)
reviewed_at = Column(DateTime(timezone=True), nullable=True)
ready_at = Column(Date)
published_at = Column(Date)
compliance_score = Column(Integer)
+1
View File
@@ -112,6 +112,7 @@ class TopicResponse(TopicBase):
created_at: Optional[datetime] = None
updated_at: Optional[datetime] = None
generated_at: Optional[datetime] = None
reviewed_at: Optional[datetime] = None
ready_at: Optional[date] = None
published_at: Optional[date] = None
compliance_score: Optional[int] = None