yu-zhi-ran/platform/backend/app/api/system.py

import logging
import subprocess
from fastapi import APIRouter, HTTPException, Depends, Body
from sqlalchemy.orm import Session
from sqlalchemy import func
from datetime import datetime, date, timezone, timedelta
from pathlib import Path
from typing import Dict, Any, List, Optional
import os
import os
import json
from ..database import get_db
from ..models import Topic, Article, TaskConfig, TaskLog
from ..core.generator import run_creator, get_generator_status, _running_processes as _generator_running
from ..core.optimizer import run_optimizer, get_optimizer_status, _running_processes as _optimizer_running
from ..core.collector import run_collector, get_collector_status, _running_processes as _collector_running
import threading
from ..core.sync import sync_all_topics
from ..core.scheduler import scheduler
from .auth import get_current_user, org_filter

PROJECT_ROOT = Path(__file__).resolve().parents[4]
if os.getenv('PROJECT_ROOT'):
    PROJECT_ROOT = Path(os.getenv('PROJECT_ROOT'))
LOGS_DIR = PROJECT_ROOT / "automation" / "logs"

logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/system", tags=["system"])

_active_monitors: Dict[int, dict] = {}

def _monitor_subprocess(log_id: int, proc, module_id: str, task_name: str, db_session_factory):
    """阻塞等待子进程退出（最长 1800s），完成后更新 task_logs"""
    try:
        returncode = proc.wait(timeout=1800)
    except subprocess.TimeoutExpired:
        proc.kill()
        returncode = -1
        logger.warning("Subprocess %s (pid=%s) killed after 1800s timeout", module_id, proc.pid)
    finished_at = datetime.now(timezone.utc)
    try:
        db = db_session_factory()
        log = db.query(TaskLog).filter(TaskLog.id == log_id).first()
        if log:
            log.status = "success" if returncode == 0 else "failed"
            log.finished_at = finished_at
            if log.started_at:
                log.duration = int((finished_at - log.started_at).total_seconds())
            db.commit()
        db.close()
    except Exception as e:
        logger.warning("Failed to update task log %s: %s", log_id, e)
    finally:
        _active_monitors.pop(log_id, None)

def _aggregate_status_counts(q):
    """聚合状态计数，兼容中英文状态值"""
    raw = q.with_entities(Topic.status, func.count()).group_by(Topic.status).all()
    mapping = {
        'pending': ['pending', '待处理'],
        'review': ['review', '待审查'],
        'ready': ['ready', '待发布'],
        'published': ['published', '已发布']
    }
    counts = {'pending': 0, 'review': 0, 'ready': 0, 'published': 0}
    for status_val, cnt in raw:
        for key, aliases in mapping.items():
            if status_val in aliases:
                counts[key] += cnt
                break
    return counts

@router.get("/status")
def get_status(db: Session = Depends(get_db)):
    total = db.query(Topic).count()
    counts = _aggregate_status_counts(db.query(Topic))
    today = date.today()
    today_count = db.query(Topic).filter(func.date(Topic.created_at) == today).count()
    return {
        "stats": {
            "total": total,
            "pending": counts['pending'],
            "review": counts['review'],
            "ready": counts['ready'],
            "published": counts['published'],
            "today": today_count
        }
    }

@router.post("/generate/run")
def trigger_generation(topic_id: Optional[str] = None, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
    logger.info(f"Generation triggered by {current_user.username}, topic_id={topic_id}")
    try:
        result = run_creator(topic_id)
        from ..database import SessionLocal
        log = TaskLog(module_id="scheduled_generate", task_name="🤖 内容创作", status="running", message="内容创作已启动", triggered_by="manual", started_at=datetime.now(timezone.utc))
        db.add(log)
        db.commit()
        log_id = log.id
        proc_info = result.get("proc") or result
        proc = _generator_running.get("generator", {}).get("process") if "pid" in result else None
        if proc:
            t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_generate", "🤖 内容创作", SessionLocal), daemon=True)
            t.start()
        return {"message": "内容创作已后台启动", "pid": result.get("pid"), "log_id": log_id}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@router.get("/generate/status", dependencies=[Depends(get_current_user)])
def generation_status():
    status = get_generator_status()
    if status is None:
        return {"status": "idle", "message": "当前无运行中的创作任务"}
    return status

@router.post("/collect/run")
def trigger_collection(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
    logger.info(f"Manual collection triggered by {current_user.username}")
    try:
        result = run_collector()
        from ..database import SessionLocal
        log = TaskLog(module_id="scheduled_collect", task_name="📡 内容采集", status="running", message="内容采集已启动", triggered_by="manual", started_at=datetime.now(timezone.utc))
        db.add(log)
        db.commit()
        log_id = log.id
        proc = _collector_running.get("collector", {}).get("process") if "pid" in result else None
        if proc:
            t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_collect", "📡 内容采集", SessionLocal), daemon=True)
            t.start()
        return {"message": "内容采集已后台启动", "result": result, "log_id": log_id}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@router.get("/collect/status", dependencies=[Depends(get_current_user)])
def collection_status():
    status = get_collector_status()
    if status is None:
        return {"status": "idle", "message": "当前无运行中的采集任务"}
    return status

@router.post("/review/run")
def trigger_review(topic_ids: Optional[List[str]] = None, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
    try:
        result = run_optimizer(topic_ids)
        from ..database import SessionLocal
        log = TaskLog(module_id="scheduled_optimize", task_name="🔍 合规审查", status="running", message="合规审查已启动", triggered_by="manual", started_at=datetime.now(timezone.utc))
        db.add(log)
        db.commit()
        log_id = log.id
        proc = _optimizer_running.get("optimizer", {}).get("process") if "pid" in result else None
        if proc:
            t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_optimize", "🔍 合规审查", SessionLocal), daemon=True)
            t.start()
        return {"message": "合规审查已后台启动", "pid": result.get("pid"), "log_id": log_id}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@router.get("/review/status", dependencies=[Depends(get_current_user)])
def review_status():
    status = get_optimizer_status()
    if status is None:
        return {"status": "idle", "message": "当前无运行中的审查任务"}
    return status

@router.get("/logs/{log_date}", dependencies=[Depends(get_current_user)])
def get_logs(log_date: str, log_type: str = "creator"):
    log_file = LOGS_DIR / f"{log_type}_{log_date}.log"
    if not log_file.exists():
        raise HTTPException(status_code=404, detail=f"Log file not found: {log_file}")
    content = log_file.read_text(encoding='utf-8')
    lines = content.splitlines()[-100:] if log_type != "collector" else content.splitlines()[-200:]
    return {"log_date": log_date, "log_type": log_type, "content": lines}

@router.get("/pipeline/status", dependencies=[Depends(get_current_user)])
def get_pipeline_status(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
    topic_base = db.query(Topic)
    of = org_filter(current_user, Topic)
    if of is not True:
        topic_base = topic_base.filter(of)
    total = topic_base.count()
    counts = _aggregate_status_counts(topic_base)
    log_files = {
        "collector": LOGS_DIR / f"collector_{date.today().isoformat()}.log",
        "creator": LOGS_DIR / f"creator_{date.today().isoformat()}.log",
        "optimizer": LOGS_DIR / f"optimizer_{date.today().isoformat()}.log",
    }
    pipeline_status = {}
    for name, log_file in log_files.items():
        if log_file.exists():
            mtime = datetime.fromtimestamp(log_file.stat().st_mtime)
            pipeline_status[name] = {"last_run": mtime.isoformat(), "exists": True}
        else:
            pipeline_status[name] = {"exists": False, "last_run": None}
    return {"topics_count": total, "status_distribution": counts, "pipeline_modules": pipeline_status}

@router.post("/sync/run")
def run_sync():
    try:
        sync_all_topics()
        return {"message": "Sync completed (DB → JSON backup)"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@router.post("/optimize-sources/run")
def trigger_optimize_sources():
    try:
        from ..core.scheduler import scheduler
        def _bg():
            try:
                scheduler._run_optimize_sources()
            except Exception as e:
                logger.exception("Background optimize sources failed: %s", e)
        t = threading.Thread(target=_bg, daemon=True)
        t.start()
        return {"message": "信息源优化已后台启动"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@router.post("/metrics-sync/run")
def trigger_metrics_sync():
    try:
        from ..core.scheduler import scheduler
        def _bg():
            try:
                scheduler._run_metrics_sync()
            except Exception as e:
                logger.exception("Background metrics sync failed: %s", e)
        t = threading.Thread(target=_bg, daemon=True)
        t.start()
        return {"message": "指标同步已后台启动"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@router.post("/refresh-search-cache/run")
def trigger_refresh_search_cache(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
    try:
        import sys as sys_mod
        scripts_dir = Path(__file__).parent.parent.parent.parent / "scripts"
        from ..database import SessionLocal as _ss
        proc = subprocess.Popen(
            [sys_mod.executable, str(scripts_dir / "opencode_search.py"), "--refresh-cache"],
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
            cwd=scripts_dir.parent.parent
        )
        logger.info("Search cache refresh started (pid=%s)", proc.pid)
        log = TaskLog(module_id="scheduled_refresh_search_cache", task_name="🔍 搜索缓存", status="running", message="搜索缓存刷新已启动", triggered_by="manual", started_at=datetime.now(timezone.utc), result_data={"pid": proc.pid})
        db.add(log)
        db.commit()
        log_id = log.id
        t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_refresh_search_cache", "🔍 搜索缓存", _ss), daemon=True)
        t.start()
        return {"message": "搜索缓存刷新已后台启动", "pid": proc.pid, "log_id": log_id}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@router.post("/trends/run")
def trigger_trends_refresh():
    try:
        import sys as sys_mod
        scripts_dir = Path(__file__).parent.parent.parent.parent / "scripts"
        proc = subprocess.Popen(
            [sys_mod.executable, str(scripts_dir / "trends.py")],
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
            cwd=scripts_dir.parent.parent
        )
        logger.info("Trends refresh started (pid=%s)", proc.pid)
        return {"message": "热点趋势刷新已后台启动", "pid": proc.pid}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@router.get("/automation/topics")
def list_automation_topics(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
    try:
        topic_base = db.query(Topic)
        of = org_filter(current_user, Topic)
        if of is not True:
            topic_base = topic_base.filter(of)
        topics = topic_base.order_by(Topic.created_at.desc()).limit(100).all()
        result = []
        for t in topics:
            result.append({
                "id": t.id,
                "title": t.title,
                "field": t.field,
                "status": t.status,
                "priority": t.priority,
                "priority_score": t.priority_score,
                "total_score": t.total_score,
                "created_at": t.created_at.isoformat() if t.created_at else None,
                "updated_at": t.updated_at.isoformat() if t.updated_at else None,
                "ready_at": t.ready_at.isoformat() if t.ready_at else None,
                "compliance_score": t.compliance_score
            })
        return {"count": len(result), "topics": result[:50]}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@router.post("/refresh")
def refresh_all():
    try:
        sync_all_topics()
        return {"message": "Refresh completed"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@router.get("/scheduler/status", dependencies=[Depends(get_current_user)])
def get_scheduler_status():
    return {"running": scheduler._started, "jobs": scheduler.get_jobs()}


@router.get("/modules/status", dependencies=[Depends(get_current_user)])
def get_modules_status(db: Session = Depends(get_db)):
    configs = db.query(TaskConfig).all()
    config_map = {c.module_id: c for c in configs}

    MODULE_META = {
        "scheduled_refresh_search_cache": {"name": "🔍 搜索缓存", "cron": "01:00", "params_desc": {"refresh_queries": "搜索关键词列表"}},
        "scheduled_fetch_trends": {"name": "🔥 热点趋势", "cron": "01:10", "params_desc": {}},
        "scheduled_collect": {"name": "📡 内容采集", "cron": "01:30", "params_desc": {"max_topics": "最大选题数", "categories": "采集类别"}},
        "scheduled_generate": {"name": "🤖 内容创作", "cron": "02:00", "params_desc": {"auto_review": "自动合规审查"}},
        "scheduled_optimize": {"name": "🔍 合规审查", "cron": "03:00", "params_desc": {"auto_pass_threshold": "自动通过分数阈值"}},
        "scheduled_optimize_sources": {"name": "📡 信息源优化", "cron": "05:00", "params_desc": {}},
        "scheduled_metrics_sync": {"name": "📊 指标同步", "cron": "06:00", "params_desc": {}},
        "scheduled_task_monitor": {"name": "⏰ 任务监控", "cron": "*", "params_desc": {}},
    }

    modules = []
    for mod_id, meta in MODULE_META.items():
        cfg = config_map.get(mod_id)
        latest = db.query(TaskLog).filter(TaskLog.module_id == mod_id).order_by(TaskLog.started_at.desc()).first()
        next_run = _get_next_run(mod_id)

        total = db.query(TaskLog).filter(TaskLog.module_id == mod_id).count()
        success = db.query(TaskLog).filter(TaskLog.module_id == mod_id, TaskLog.status == "success").count()
        failed = db.query(TaskLog).filter(TaskLog.module_id == mod_id, TaskLog.status == "failed").count()
        running = db.query(TaskLog).filter(TaskLog.module_id == mod_id, TaskLog.status == "running").count()

        modules.append({
            "module_id": mod_id,
            "title": meta["name"],
            "enabled": cfg.enabled if cfg else True,
            "params": cfg.params if cfg else {},
            "params_desc": meta["params_desc"],
            "schedule": cfg.schedule if cfg else meta["cron"],
            "cron_default": meta["cron"],
            "status": "running" if running else ("stopped" if not (cfg and cfg.enabled) else "idle"),
            "last_run": latest.started_at.strftime("%Y-%m-%d %H:%M") if latest and latest.started_at else None,
            "last_status": latest.status if latest else None,
            "last_message": latest.message if latest else None,
            "last_result": latest.result_data if latest else None,
            "next_run": next_run,
            "total_runs": total,
            "success_runs": success,
            "failed_runs": failed,
            "running": running,
        })

    jobs = scheduler.get_jobs()
    return {"modules": modules, "scheduler": {"running": scheduler._started, "jobs": jobs}}


def _get_next_run(mod_id: str) -> Optional[str]:
    for job in scheduler.get_jobs():
        if job["id"] == mod_id and job["next_run_time"]:
            try:
                dt = datetime.fromisoformat(job["next_run_time"])
                return dt.strftime("%Y-%m-%d %H:%M")
            except Exception:
                return job["next_run_time"]
    return None