Yuzhiran Dev
6f81167691
新增: - scripts/opencode_search.py: 通过 npx opencode run 调用 webfetch 联网搜索 - 搜索缓存每日 02:30 自动刷新 (scheduler) - 管理后台「搜索缓存」模块 + 立即运行按钮 - POST /api/system/refresh-search-cache/run 手动触发端点 - 8个分类搜索词从sources.yaml读取，调用AI联网搜索真实内容 机制: Python脚本 → npx opencode run → AI webfetch → 真实搜索结果 → 写入search_cache.json → 采集器读缓存 → LLM基于真实数据生成选题 不再需要API Key,不依赖任何搜索引擎,搜索结果来自AI的webfetch能力
298 lines
13 KiB
Python
298 lines
13 KiB
Python
import logging
|
|
from fastapi import APIRouter, HTTPException, Depends, Body
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy import func
|
|
from datetime import datetime, date
|
|
from pathlib import Path
|
|
from typing import Dict, Any, List, Optional
|
|
from pathlib import Path
|
|
import os
|
|
import json
|
|
from ..database import get_db
|
|
from ..models import Topic, Article
|
|
from ..core.generator import run_creator
|
|
from ..core.optimizer import run_optimizer
|
|
from ..core.collector import run_collector
|
|
from ..core.sync import sync_all_topics
|
|
from ..core.scheduler import scheduler
|
|
from .auth import get_current_user, org_filter
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[4]
|
|
if os.getenv('PROJECT_ROOT'):
|
|
PROJECT_ROOT = Path(os.getenv('PROJECT_ROOT'))
|
|
LOGS_DIR = PROJECT_ROOT / "automation" / "logs"
|
|
|
|
logger = logging.getLogger(__name__)
|
|
router = APIRouter(prefix="/api/system", tags=["system"])
|
|
|
|
def _aggregate_status_counts(q):
|
|
"""聚合状态计数，兼容中英文状态值"""
|
|
raw = q.with_entities(Topic.status, func.count()).group_by(Topic.status).all()
|
|
mapping = {
|
|
'pending': ['pending', '待处理'],
|
|
'review': ['review', '待审查'],
|
|
'ready': ['ready', '待发布'],
|
|
'published': ['published', '已发布']
|
|
}
|
|
counts = {'pending': 0, 'review': 0, 'ready': 0, 'published': 0}
|
|
for status_val, cnt in raw:
|
|
for key, aliases in mapping.items():
|
|
if status_val in aliases:
|
|
counts[key] += cnt
|
|
break
|
|
return counts
|
|
|
|
@router.get("/status")
|
|
def get_status(db: Session = Depends(get_db)):
|
|
total = db.query(Topic).count()
|
|
counts = _aggregate_status_counts(db.query(Topic))
|
|
today = date.today()
|
|
today_count = db.query(Topic).filter(func.date(Topic.created_at) == today).count()
|
|
return {
|
|
"stats": {
|
|
"total": total,
|
|
"pending": counts['pending'],
|
|
"review": counts['review'],
|
|
"ready": counts['ready'],
|
|
"published": counts['published'],
|
|
"today": today_count
|
|
}
|
|
}
|
|
|
|
@router.post("/generate/run", dependencies=[Depends(get_current_user)])
|
|
def trigger_generation(topic_id: str = Body(None, embed=True), db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
|
logger.info(f"Received topic_id={topic_id}")
|
|
try:
|
|
result = run_creator(topic_id)
|
|
if not result["ok"]:
|
|
raise HTTPException(status_code=500, detail=result["error"])
|
|
sync_all_topics()
|
|
return {"message": "Generation triggered", "result": result}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@router.post("/collect/run", dependencies=[Depends(get_current_user)])
|
|
def trigger_collection(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
|
logger.info(f"Manual collection triggered by {current_user.username}")
|
|
try:
|
|
result = run_collector()
|
|
return {"message": "内容采集已完成", "result": result}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@router.post("/review/run", dependencies=[Depends(get_current_user)])
|
|
def trigger_review(topic_ids: List[str] = Body(None, embed=True), db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
|
try:
|
|
result = run_optimizer(topic_ids)
|
|
if not result["ok"]:
|
|
raise HTTPException(status_code=500, detail=result["error"])
|
|
report = result.get("report")
|
|
if report and report["summary"]["total_articles"] > 0:
|
|
s = report["summary"]
|
|
total = s["total_articles"]
|
|
avg = s["average_score"]
|
|
msg = f"审查完成: {total} 篇全部通过 ({avg:.0f}分)"
|
|
return {"message": msg, "summary": s}
|
|
else:
|
|
if topic_ids:
|
|
updated = 0
|
|
for tid in topic_ids:
|
|
q = db.query(Topic).filter(Topic.id == tid)
|
|
of = org_filter(current_user, Topic)
|
|
if of is not True:
|
|
q = q.filter(of)
|
|
topic = q.first()
|
|
if topic and topic.status in ('review', '待审查'):
|
|
topic.status = 'ready'
|
|
if not topic.generated_at:
|
|
topic.generated_at = datetime.utcnow()
|
|
updated += 1
|
|
db.commit()
|
|
if updated:
|
|
logger.info(f"Review: {updated} topics advanced to 'ready' (no release files)")
|
|
return {"message": "审查完成（未找到 release 文件，仅推进状态）", "stdout": result.get("stdout", "")}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@router.get("/logs/{log_date}", dependencies=[Depends(get_current_user)])
|
|
def get_logs(log_date: str, log_type: str = "creator"):
|
|
log_file = LOGS_DIR / f"{log_type}_{log_date}.log"
|
|
if not log_file.exists():
|
|
raise HTTPException(status_code=404, detail=f"Log file not found: {log_file}")
|
|
content = log_file.read_text(encoding='utf-8')
|
|
lines = content.splitlines()[-100:] if log_type != "collector" else content.splitlines()[-200:]
|
|
return {"log_date": log_date, "log_type": log_type, "content": lines}
|
|
|
|
@router.get("/pipeline/status", dependencies=[Depends(get_current_user)])
|
|
def get_pipeline_status(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
|
topic_base = db.query(Topic)
|
|
of = org_filter(current_user, Topic)
|
|
if of is not True:
|
|
topic_base = topic_base.filter(of)
|
|
total = topic_base.count()
|
|
counts = _aggregate_status_counts(topic_base)
|
|
log_files = {
|
|
"collector": LOGS_DIR / f"collector_{date.today().isoformat()}.log",
|
|
"creator": LOGS_DIR / f"creator_{date.today().isoformat()}.log",
|
|
"optimizer": LOGS_DIR / f"optimizer_{date.today().isoformat()}.log",
|
|
}
|
|
pipeline_status = {}
|
|
for name, log_file in log_files.items():
|
|
if log_file.exists():
|
|
mtime = datetime.fromtimestamp(log_file.stat().st_mtime)
|
|
pipeline_status[name] = {"last_run": mtime.isoformat(), "exists": True}
|
|
else:
|
|
pipeline_status[name] = {"exists": False, "last_run": None}
|
|
return {"topics_count": total, "status_distribution": counts, "pipeline_modules": pipeline_status}
|
|
|
|
@router.post("/sync/run")
|
|
def run_sync():
|
|
try:
|
|
sync_all_topics()
|
|
return {"message": "Sync completed (DB → JSON backup)"}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@router.post("/optimize-sources/run")
|
|
def trigger_optimize_sources():
|
|
try:
|
|
from ..core.scheduler import scheduler
|
|
scheduler._run_optimize_sources()
|
|
log_file = LOGS_DIR / f"optimizer_sources_{date.today().isoformat()}.log"
|
|
log_file.parent.mkdir(parents=True, exist_ok=True)
|
|
log_file.write_text(f"{datetime.now().isoformat()} - 信息源优化完成\n")
|
|
return {"message": "信息源优化已完成"}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@router.post("/metrics-sync/run")
|
|
def trigger_metrics_sync():
|
|
try:
|
|
from ..core.scheduler import scheduler
|
|
scheduler._run_metrics_sync()
|
|
log_file = LOGS_DIR / f"metrics_sync_{date.today().isoformat()}.log"
|
|
log_file.parent.mkdir(parents=True, exist_ok=True)
|
|
log_file.write_text(f"{datetime.now().isoformat()} - 指标同步完成\n")
|
|
return {"message": "指标同步已完成"}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@router.post("/refresh-search-cache/run")
|
|
def trigger_refresh_search_cache():
|
|
"""手动刷新搜索缓存"""
|
|
try:
|
|
import subprocess, sys as sys_mod
|
|
scripts_dir = Path(__file__).parent.parent.parent.parent / "scripts"
|
|
result = subprocess.run(
|
|
[sys_mod.executable, str(scripts_dir / "opencode_search.py"), "--refresh-cache"],
|
|
capture_output=True, text=True, timeout=600
|
|
)
|
|
if result.returncode != 0:
|
|
raise Exception(result.stderr[-500:])
|
|
return {"message": "搜索缓存已刷新", "output": result.stdout.strip()}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@router.post("/trends/run")
|
|
def trigger_trends_refresh():
|
|
"""手动刷新热点趋势数据"""
|
|
try:
|
|
import subprocess, sys as sys_mod
|
|
from pathlib import Path
|
|
scripts_dir = Path(__file__).parent.parent.parent.parent / "scripts"
|
|
result = subprocess.run(
|
|
[sys_mod.executable, str(scripts_dir / "trends.py")],
|
|
capture_output=True, text=True, timeout=120
|
|
)
|
|
if result.returncode != 0:
|
|
raise Exception(result.stderr[-500:])
|
|
return {"message": "热点趋势已刷新", "output": result.stdout.strip()}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@router.get("/automation/topics")
|
|
def list_automation_topics(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
|
try:
|
|
topic_base = db.query(Topic)
|
|
of = org_filter(current_user, Topic)
|
|
if of is not True:
|
|
topic_base = topic_base.filter(of)
|
|
topics = topic_base.order_by(Topic.created_at.desc()).limit(100).all()
|
|
result = []
|
|
for t in topics:
|
|
result.append({
|
|
"id": t.id,
|
|
"title": t.title,
|
|
"field": t.field,
|
|
"status": t.status,
|
|
"priority": t.priority,
|
|
"priority_score": t.priority_score,
|
|
"total_score": t.total_score,
|
|
"created_at": t.created_at.isoformat() if t.created_at else None,
|
|
"updated_at": t.updated_at.isoformat() if t.updated_at else None,
|
|
"ready_at": t.ready_at.isoformat() if t.ready_at else None,
|
|
"compliance_score": t.compliance_score
|
|
})
|
|
return {"count": len(result), "topics": result[:50]}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@router.post("/refresh")
|
|
def refresh_all():
|
|
try:
|
|
sync_all_topics()
|
|
return {"message": "Refresh completed"}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@router.get("/scheduler/status", dependencies=[Depends(get_current_user)])
|
|
def get_scheduler_status():
|
|
return {"running": scheduler._started, "jobs": scheduler.get_jobs()}
|
|
|
|
|
|
@router.get("/modules/status", dependencies=[Depends(get_current_user)])
|
|
def get_modules_status():
|
|
today_str = date.today().isoformat()
|
|
log_based: dict = {
|
|
"scheduled_collect": {"name": "📡 内容采集", "log": LOGS_DIR / f"collector_{today_str}.log"},
|
|
"scheduled_refresh_search_cache": {"name": "🔍 搜索缓存", "log": LOGS_DIR / f"opencode_search_{today_str}.log"},
|
|
"scheduled_fetch_trends": {"name": "🔥 热点趋势", "log": LOGS_DIR / f"trends_{today_str}.log"},
|
|
"scheduled_generate": {"name": "🤖 内容创作", "log": LOGS_DIR / f"creator_{today_str}.log"},
|
|
"scheduled_optimize": {"name": "🔍 合规审查", "log": LOGS_DIR / f"optimizer_{today_str}.log"},
|
|
"scheduled_optimize_sources": {"name": "📡 信息源优化", "log": LOGS_DIR / f"optimizer_sources_{today_str}.log"},
|
|
"scheduled_metrics_sync": {"name": "📊 指标同步", "log": LOGS_DIR / f"metrics_sync_{today_str}.log"},
|
|
}
|
|
jobs = {j['id']: j for j in scheduler.get_jobs()}
|
|
modules = []
|
|
for mod_id, cfg in log_based.items():
|
|
log_file = cfg["log"]
|
|
last_run = None
|
|
task_count = 0
|
|
success_rate = None
|
|
if log_file.exists():
|
|
mtime = datetime.fromtimestamp(log_file.stat().st_mtime)
|
|
last_run = mtime.strftime("%Y-%m-%d %H:%M")
|
|
content = log_file.read_text(encoding="utf-8", errors="ignore")
|
|
task_count = content.count("完成") + content.count("success") + content.count("SUCCESS")
|
|
total = task_count + content.count("失败") + content.count("failed") + content.count("ERROR")
|
|
success_rate = round(task_count / total * 100) if total > 0 else None
|
|
status = "running" if mod_id in jobs else "stopped"
|
|
job = jobs.get(mod_id)
|
|
next_run = None
|
|
if job and job.get("next_run_time"):
|
|
try:
|
|
next_dt = datetime.fromisoformat(job["next_run_time"])
|
|
next_run = next_dt.strftime("%Y-%m-%d %H:%M")
|
|
except Exception:
|
|
next_run = job["next_run_time"]
|
|
modules.append({
|
|
"id": mod_id,
|
|
"title": cfg["name"],
|
|
"status": status,
|
|
"last_run": last_run or "从未运行",
|
|
"next_run": next_run or "待计划",
|
|
"task_count": task_count,
|
|
"success_rate": success_rate if success_rate is not None else 0,
|
|
})
|
|
return {"modules": modules, "scheduler": {"running": scheduler._started, "jobs": scheduler.get_jobs()}}
|