fix: 合规审查卡死修复 + 小红书复制格式 + today-only过滤
This commit is contained in:
@@ -1,5 +1,6 @@
|
|||||||
import logging
|
import logging
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import re
|
||||||
from fastapi import APIRouter, HTTPException, Depends, Body
|
from fastapi import APIRouter, HTTPException, Depends, Body
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
from sqlalchemy import func
|
from sqlalchemy import func
|
||||||
@@ -7,7 +8,6 @@ from datetime import datetime, date, timezone, timedelta
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Dict, Any, List, Optional
|
from typing import Dict, Any, List, Optional
|
||||||
import os
|
import os
|
||||||
import os
|
|
||||||
import json
|
import json
|
||||||
from ..database import get_db
|
from ..database import get_db
|
||||||
from ..models import Topic, Article, TaskConfig, TaskLog
|
from ..models import Topic, Article, TaskConfig, TaskLog
|
||||||
@@ -29,14 +29,48 @@ router = APIRouter(prefix="/api/system", tags=["system"])
|
|||||||
|
|
||||||
_active_monitors: Dict[int, dict] = {}
|
_active_monitors: Dict[int, dict] = {}
|
||||||
|
|
||||||
|
def _parse_stdout_result(stdout: str, module_id: str) -> dict:
|
||||||
|
"""从脚本 stdout 中提取关键指标存入 result_data"""
|
||||||
|
result = {}
|
||||||
|
if module_id == "scheduled_collect":
|
||||||
|
m = re.search(r'SUCCESS:\s*Collected\s*(\d+)\s*cases?\s*and\s*(\d+)\s*topics?', stdout)
|
||||||
|
if m:
|
||||||
|
result["cases_saved"] = int(m.group(1))
|
||||||
|
result["topics_saved"] = int(m.group(2))
|
||||||
|
m = re.search(r'WARNING:\s*(.*)', stdout)
|
||||||
|
if m:
|
||||||
|
result["warning"] = m.group(1).strip()
|
||||||
|
elif module_id in ("scheduled_generate", "scheduled_optimize"):
|
||||||
|
# 尝试解析 JSON 输出
|
||||||
|
try:
|
||||||
|
data = json.loads(stdout.strip())
|
||||||
|
if isinstance(data, dict):
|
||||||
|
for k, v in data.items():
|
||||||
|
if isinstance(v, (str, int, float, bool)):
|
||||||
|
result[k] = v
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
pass
|
||||||
|
if not result:
|
||||||
|
m = re.search(r'SUCCESS:\s*(.*)', stdout)
|
||||||
|
if m:
|
||||||
|
result["summary"] = m.group(1).strip()
|
||||||
|
return result
|
||||||
|
|
||||||
def _monitor_subprocess(log_id: int, proc, module_id: str, task_name: str, db_session_factory):
|
def _monitor_subprocess(log_id: int, proc, module_id: str, task_name: str, db_session_factory):
|
||||||
"""阻塞等待子进程退出(最长 1800s),完成后更新 task_logs"""
|
"""阻塞等待子进程退出(最长 1800s),完成后更新 task_logs"""
|
||||||
|
stdout, stderr = "", ""
|
||||||
try:
|
try:
|
||||||
returncode = proc.wait(timeout=1800)
|
stdout, stderr = proc.communicate(timeout=1800)
|
||||||
|
returncode = proc.returncode
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
proc.kill()
|
proc.kill()
|
||||||
|
stdout, stderr = proc.communicate(timeout=5)
|
||||||
returncode = -1
|
returncode = -1
|
||||||
logger.warning("Subprocess %s (pid=%s) killed after 1800s timeout", module_id, proc.pid)
|
logger.warning("Subprocess %s (pid=%s) killed after 1800s timeout", module_id, proc.pid)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Subprocess %s monitor error: %s", module_id, e)
|
||||||
|
returncode = -1
|
||||||
|
stdout = stderr = ""
|
||||||
finished_at = datetime.now(timezone.utc)
|
finished_at = datetime.now(timezone.utc)
|
||||||
try:
|
try:
|
||||||
db = db_session_factory()
|
db = db_session_factory()
|
||||||
@@ -44,8 +78,17 @@ def _monitor_subprocess(log_id: int, proc, module_id: str, task_name: str, db_se
|
|||||||
if log:
|
if log:
|
||||||
log.status = "success" if returncode == 0 else "failed"
|
log.status = "success" if returncode == 0 else "failed"
|
||||||
log.finished_at = finished_at
|
log.finished_at = finished_at
|
||||||
|
msg = f"{task_name} {'完成' if returncode == 0 else '失败'}"
|
||||||
|
log.message = msg
|
||||||
|
if returncode != 0:
|
||||||
|
log.error_trace = (stderr or stdout or "")[:2000]
|
||||||
if log.started_at:
|
if log.started_at:
|
||||||
log.duration = int((finished_at - log.started_at).total_seconds())
|
log.duration = int((finished_at - log.started_at).total_seconds())
|
||||||
|
# 解析 stdout 提取结果指标
|
||||||
|
if returncode == 0 and stdout:
|
||||||
|
parsed = _parse_stdout_result(stdout, module_id)
|
||||||
|
if parsed:
|
||||||
|
log.result_data = parsed
|
||||||
db.commit()
|
db.commit()
|
||||||
db.close()
|
db.close()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -97,12 +140,13 @@ def trigger_generation(topic_id: Optional[str] = None, db: Session = Depends(get
|
|||||||
db.add(log)
|
db.add(log)
|
||||||
db.commit()
|
db.commit()
|
||||||
log_id = log.id
|
log_id = log.id
|
||||||
proc_info = result.get("proc") or result
|
proc = result.get("_proc")
|
||||||
proc = _generator_running.get("generator", {}).get("process") if "pid" in result else None
|
|
||||||
if proc:
|
if proc:
|
||||||
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_generate", "🤖 内容创作", SessionLocal), daemon=True)
|
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_generate", "🤖 内容创作", SessionLocal), daemon=True)
|
||||||
t.start()
|
t.start()
|
||||||
return {"message": "内容创作已后台启动", "pid": result.get("pid"), "log_id": log_id}
|
return {"message": "内容创作已后台启动", "pid": result.get("pid"), "log_id": log_id}
|
||||||
|
except RuntimeError as e:
|
||||||
|
raise HTTPException(status_code=409, detail=str(e))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
@@ -123,11 +167,13 @@ def trigger_collection(db: Session = Depends(get_db), current_user=Depends(get_c
|
|||||||
db.add(log)
|
db.add(log)
|
||||||
db.commit()
|
db.commit()
|
||||||
log_id = log.id
|
log_id = log.id
|
||||||
proc = _collector_running.get("collector", {}).get("process") if "pid" in result else None
|
proc = result.get("_proc")
|
||||||
if proc:
|
if proc:
|
||||||
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_collect", "📡 内容采集", SessionLocal), daemon=True)
|
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_collect", "📡 内容采集", SessionLocal), daemon=True)
|
||||||
t.start()
|
t.start()
|
||||||
return {"message": "内容采集已后台启动", "result": result, "log_id": log_id}
|
return {"message": "内容采集已后台启动", "result": result, "log_id": log_id}
|
||||||
|
except RuntimeError as e:
|
||||||
|
raise HTTPException(status_code=409, detail=str(e))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
@@ -139,7 +185,8 @@ def collection_status():
|
|||||||
return status
|
return status
|
||||||
|
|
||||||
@router.post("/review/run")
|
@router.post("/review/run")
|
||||||
def trigger_review(topic_ids: Optional[List[str]] = None, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
def trigger_review(data: Dict = Body({}), db: Session = Depends(get_db), current_user=Depends(get_current_user)):
|
||||||
|
topic_ids = data.get("topic_ids") if isinstance(data, dict) else None
|
||||||
try:
|
try:
|
||||||
result = run_optimizer(topic_ids)
|
result = run_optimizer(topic_ids)
|
||||||
from ..database import SessionLocal
|
from ..database import SessionLocal
|
||||||
@@ -147,11 +194,13 @@ def trigger_review(topic_ids: Optional[List[str]] = None, db: Session = Depends(
|
|||||||
db.add(log)
|
db.add(log)
|
||||||
db.commit()
|
db.commit()
|
||||||
log_id = log.id
|
log_id = log.id
|
||||||
proc = _optimizer_running.get("optimizer", {}).get("process") if "pid" in result else None
|
proc = result.get("_proc")
|
||||||
if proc:
|
if proc:
|
||||||
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_optimize", "🔍 合规审查", SessionLocal), daemon=True)
|
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_optimize", "🔍 合规审查", SessionLocal), daemon=True)
|
||||||
t.start()
|
t.start()
|
||||||
return {"message": "合规审查已后台启动", "pid": result.get("pid"), "log_id": log_id}
|
return {"message": "合规审查已后台启动", "pid": result.get("pid"), "log_id": log_id}
|
||||||
|
except RuntimeError as e:
|
||||||
|
raise HTTPException(status_code=409, detail=str(e))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
@@ -207,7 +256,7 @@ def trigger_optimize_sources():
|
|||||||
from ..core.scheduler import scheduler
|
from ..core.scheduler import scheduler
|
||||||
def _bg():
|
def _bg():
|
||||||
try:
|
try:
|
||||||
scheduler._run_optimize_sources()
|
scheduler._run_optimize_sources(triggered_by="manual")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("Background optimize sources failed: %s", e)
|
logger.exception("Background optimize sources failed: %s", e)
|
||||||
t = threading.Thread(target=_bg, daemon=True)
|
t = threading.Thread(target=_bg, daemon=True)
|
||||||
@@ -222,7 +271,7 @@ def trigger_metrics_sync():
|
|||||||
from ..core.scheduler import scheduler
|
from ..core.scheduler import scheduler
|
||||||
def _bg():
|
def _bg():
|
||||||
try:
|
try:
|
||||||
scheduler._run_metrics_sync()
|
scheduler._run_metrics_sync(triggered_by="manual")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("Background metrics sync failed: %s", e)
|
logger.exception("Background metrics sync failed: %s", e)
|
||||||
t = threading.Thread(target=_bg, daemon=True)
|
t = threading.Thread(target=_bg, daemon=True)
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from fastapi import APIRouter, Depends, HTTPException
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
from sqlalchemy import func as sa_func
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
|
|
||||||
from ..database import get_db
|
from ..database import get_db
|
||||||
@@ -17,15 +18,44 @@ DEFAULT_CONFIGS = {
|
|||||||
"scheduled_optimize": {"name": "🔍 合规审查", "cron": "03:00", "params": {"auto_pass_threshold": 80}},
|
"scheduled_optimize": {"name": "🔍 合规审查", "cron": "03:00", "params": {"auto_pass_threshold": 80}},
|
||||||
"scheduled_optimize_sources": {"name": "📡 信息源优化", "cron": "05:00", "params": {}},
|
"scheduled_optimize_sources": {"name": "📡 信息源优化", "cron": "05:00", "params": {}},
|
||||||
"scheduled_metrics_sync": {"name": "📊 指标同步", "cron": "06:00", "params": {}},
|
"scheduled_metrics_sync": {"name": "📊 指标同步", "cron": "06:00", "params": {}},
|
||||||
|
"scheduled_task_monitor": {"name": "⏰ 任务监控", "cron": "*", "params": {}},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def _attach_last_log(resp: TaskConfigResponse, db: Session, module_id: str) -> TaskConfigResponse:
|
||||||
|
"""从最近一次 TaskLog 中附加 last_run_at 和 result_data"""
|
||||||
|
last_log = db.query(TaskLog).filter(
|
||||||
|
TaskLog.module_id == module_id, TaskLog.status.in_(["success", "failed"])
|
||||||
|
).order_by(TaskLog.started_at.desc()).first()
|
||||||
|
if last_log:
|
||||||
|
resp.last_run_at = last_log.started_at
|
||||||
|
resp.result_data = last_log.result_data or {}
|
||||||
|
return resp
|
||||||
|
|
||||||
@router.get("", response_model=List[TaskConfigResponse])
|
@router.get("", response_model=List[TaskConfigResponse])
|
||||||
def list_configs(db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
|
def list_configs(db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
|
||||||
configs = db.query(TaskConfig).order_by(TaskConfig.id).all()
|
configs = db.query(TaskConfig).order_by(TaskConfig.id).all()
|
||||||
if not configs:
|
if not configs:
|
||||||
_ensure_defaults(db)
|
_ensure_defaults(db)
|
||||||
configs = db.query(TaskConfig).order_by(TaskConfig.id).all()
|
configs = db.query(TaskConfig).order_by(TaskConfig.id).all()
|
||||||
return [TaskConfigResponse.model_validate(c) for c in configs]
|
# 批量查询各模块最近一次运行记录
|
||||||
|
latest_ids = db.query(
|
||||||
|
sa_func.max(TaskLog.id).label('max_id')
|
||||||
|
).filter(
|
||||||
|
TaskLog.status.in_(["success", "failed"])
|
||||||
|
).group_by(TaskLog.module_id).subquery()
|
||||||
|
latest_logs = db.query(TaskLog).filter(
|
||||||
|
TaskLog.id.in_(db.query(latest_ids.c.max_id))
|
||||||
|
).all()
|
||||||
|
log_by_module = {log.module_id: log for log in latest_logs}
|
||||||
|
result = []
|
||||||
|
for cfg in configs:
|
||||||
|
resp = TaskConfigResponse.model_validate(cfg)
|
||||||
|
log = log_by_module.get(cfg.module_id)
|
||||||
|
if log:
|
||||||
|
resp.last_run_at = log.started_at
|
||||||
|
resp.result_data = log.result_data or {}
|
||||||
|
result.append(resp)
|
||||||
|
return result
|
||||||
|
|
||||||
@router.get("/{module_id}", response_model=TaskConfigResponse)
|
@router.get("/{module_id}", response_model=TaskConfigResponse)
|
||||||
def get_config(module_id: str, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
|
def get_config(module_id: str, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
|
||||||
@@ -33,7 +63,14 @@ def get_config(module_id: str, db: Session = Depends(get_db), admin_user=Depends
|
|||||||
if not cfg:
|
if not cfg:
|
||||||
_ensure_defaults(db)
|
_ensure_defaults(db)
|
||||||
cfg = db.query(TaskConfig).filter(TaskConfig.module_id == module_id).first()
|
cfg = db.query(TaskConfig).filter(TaskConfig.module_id == module_id).first()
|
||||||
return cfg
|
if not cfg:
|
||||||
|
default = DEFAULT_CONFIGS.get(module_id, {})
|
||||||
|
cfg = TaskConfig(module_id=module_id, enabled=True, params=default.get("params", {}), schedule=default.get("cron", ""))
|
||||||
|
db.add(cfg)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(cfg)
|
||||||
|
resp = TaskConfigResponse.model_validate(cfg)
|
||||||
|
return _attach_last_log(resp, db, module_id)
|
||||||
|
|
||||||
@router.put("/{module_id}", response_model=TaskConfigResponse)
|
@router.put("/{module_id}", response_model=TaskConfigResponse)
|
||||||
def update_config(module_id: str, data: TaskConfigUpdate, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
|
def update_config(module_id: str, data: TaskConfigUpdate, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ MODULES = {
|
|||||||
"scheduled_optimize_sources": "📡 信息源优化",
|
"scheduled_optimize_sources": "📡 信息源优化",
|
||||||
"scheduled_metrics_sync": "📊 指标同步",
|
"scheduled_metrics_sync": "📊 指标同步",
|
||||||
"scheduled_task_monitor": "⏰ 任务监控",
|
"scheduled_task_monitor": "⏰ 任务监控",
|
||||||
|
"scheduled_reset_search_usage": "🔁 搜索用量重置",
|
||||||
}
|
}
|
||||||
|
|
||||||
@router.get("", response_model=List[TaskLogResponse])
|
@router.get("", response_model=List[TaskLogResponse])
|
||||||
@@ -83,8 +84,10 @@ def list_log_types(db: Session = Depends(get_db), admin_user=Depends(get_current
|
|||||||
"scheduled_collect": "collector",
|
"scheduled_collect": "collector",
|
||||||
"scheduled_generate": "creator",
|
"scheduled_generate": "creator",
|
||||||
"scheduled_optimize": "optimizer",
|
"scheduled_optimize": "optimizer",
|
||||||
"scheduled_optimize_sources": "collector",
|
"scheduled_optimize_sources": "optimizer_sources",
|
||||||
"scheduled_metrics_sync": "sync",
|
"scheduled_metrics_sync": "metrics_sync",
|
||||||
|
"scheduled_reset_search_usage": "reset_search_usage",
|
||||||
|
"scheduled_task_monitor": "task_monitor",
|
||||||
}
|
}
|
||||||
result.append({"module_id": mid, "name": name, "log_file": log_file_map.get(mid, mid)})
|
result.append({"module_id": mid, "name": name, "log_file": log_file_map.get(mid, mid)})
|
||||||
for mid in used_ids:
|
for mid in used_ids:
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from pathlib import Path
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
|
import threading
|
||||||
from typing import Optional, Dict
|
from typing import Optional, Dict
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -17,6 +18,7 @@ if os.getenv('PROJECT_ROOT'):
|
|||||||
PROJECT_ROOT = Path(os.getenv('PROJECT_ROOT'))
|
PROJECT_ROOT = Path(os.getenv('PROJECT_ROOT'))
|
||||||
|
|
||||||
_running_processes: Dict[str, dict] = {}
|
_running_processes: Dict[str, dict] = {}
|
||||||
|
_lock = threading.Lock()
|
||||||
|
|
||||||
def _get_cmd():
|
def _get_cmd():
|
||||||
script_path = PROJECT_ROOT / "scripts" / "collector.py"
|
script_path = PROJECT_ROOT / "scripts" / "collector.py"
|
||||||
@@ -29,6 +31,11 @@ def _get_cmd():
|
|||||||
|
|
||||||
def run_collector():
|
def run_collector():
|
||||||
"""运行选题收集脚本(非阻塞,后台运行)"""
|
"""运行选题收集脚本(非阻塞,后台运行)"""
|
||||||
|
with _lock:
|
||||||
|
existing = _running_processes.get("collector", {})
|
||||||
|
proc = existing.get("process")
|
||||||
|
if proc and proc.poll() is None:
|
||||||
|
raise RuntimeError("已有内容采集任务正在运行")
|
||||||
cmd = _get_cmd()
|
cmd = _get_cmd()
|
||||||
proc = subprocess.Popen(
|
proc = subprocess.Popen(
|
||||||
cmd,
|
cmd,
|
||||||
@@ -40,10 +47,10 @@ def run_collector():
|
|||||||
_running_processes["collector"] = {
|
_running_processes["collector"] = {
|
||||||
"pid": proc.pid,
|
"pid": proc.pid,
|
||||||
"started_at": time.time(),
|
"started_at": time.time(),
|
||||||
"process": proc
|
"process": proc,
|
||||||
}
|
}
|
||||||
logger.info("Collector started in background (pid=%s)", proc.pid)
|
logger.info("Collector started in background (pid=%s)", proc.pid)
|
||||||
return {"ok": True, "pid": proc.pid}
|
return {"ok": True, "pid": proc.pid, "_proc": proc}
|
||||||
|
|
||||||
def run_collector_blocking(timeout: int = 300):
|
def run_collector_blocking(timeout: int = 300):
|
||||||
"""运行选题收集脚本(阻塞,带超时,给定时任务用)"""
|
"""运行选题收集脚本(阻塞,带超时,给定时任务用)"""
|
||||||
@@ -60,22 +67,27 @@ def run_collector_blocking(timeout: int = 300):
|
|||||||
return {"ok": True, "output": result.stdout}
|
return {"ok": True, "output": result.stdout}
|
||||||
|
|
||||||
def get_collector_status() -> Optional[Dict]:
|
def get_collector_status() -> Optional[Dict]:
|
||||||
"""获取当前采集任务状态"""
|
"""获取当前采集任务状态(不消费子进程 stdout,留给 _monitor_subprocess)"""
|
||||||
info = _running_processes.get("collector")
|
info = _running_processes.get("collector")
|
||||||
if not info:
|
if not info:
|
||||||
return None
|
return None
|
||||||
proc: subprocess.Popen = info["process"]
|
proc: subprocess.Popen = info["process"]
|
||||||
|
if "finished" in info:
|
||||||
|
return {
|
||||||
|
"status": "completed" if info.get("returncode") == 0 else "failed",
|
||||||
|
"pid": info["pid"],
|
||||||
|
"elapsed": round(time.time() - info["started_at"], 1),
|
||||||
|
"returncode": info.get("returncode"),
|
||||||
|
}
|
||||||
if proc.poll() is not None:
|
if proc.poll() is not None:
|
||||||
stdout, stderr = proc.communicate()
|
info["returncode"] = proc.returncode
|
||||||
|
info["finished"] = True
|
||||||
elapsed = time.time() - info["started_at"]
|
elapsed = time.time() - info["started_at"]
|
||||||
del _running_processes["collector"]
|
|
||||||
return {
|
return {
|
||||||
"status": "completed" if proc.returncode == 0 else "failed",
|
"status": "completed" if proc.returncode == 0 else "failed",
|
||||||
"pid": info["pid"],
|
"pid": info["pid"],
|
||||||
"elapsed": round(elapsed, 1),
|
"elapsed": round(elapsed, 1),
|
||||||
"returncode": proc.returncode,
|
"returncode": proc.returncode,
|
||||||
"stdout": stdout.strip()[-500:],
|
|
||||||
"stderr": stderr.strip()[-500:],
|
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
"status": "running",
|
"status": "running",
|
||||||
|
|||||||
@@ -2,8 +2,10 @@ import subprocess
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import json
|
||||||
import time
|
import time
|
||||||
from typing import Optional, Dict
|
import threading
|
||||||
|
from typing import Optional, Dict, List
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -12,6 +14,7 @@ if os.getenv('PROJECT_ROOT'):
|
|||||||
PROJECT_ROOT = Path(os.getenv('PROJECT_ROOT'))
|
PROJECT_ROOT = Path(os.getenv('PROJECT_ROOT'))
|
||||||
|
|
||||||
_running_processes: Dict[str, dict] = {}
|
_running_processes: Dict[str, dict] = {}
|
||||||
|
_lock = threading.Lock()
|
||||||
|
|
||||||
def _get_cmd(topic_id: str = None):
|
def _get_cmd(topic_id: str = None):
|
||||||
script_path = PROJECT_ROOT / "scripts" / "creator.py"
|
script_path = PROJECT_ROOT / "scripts" / "creator.py"
|
||||||
@@ -21,15 +24,22 @@ def _get_cmd(topic_id: str = None):
|
|||||||
cmd = [str(venv_python), str(script_path)] if venv_python.exists() else ["python3", str(script_path)]
|
cmd = [str(venv_python), str(script_path)] if venv_python.exists() else ["python3", str(script_path)]
|
||||||
if topic_id:
|
if topic_id:
|
||||||
cmd.extend(["--topic-id", topic_id])
|
cmd.extend(["--topic-id", topic_id])
|
||||||
|
else:
|
||||||
|
cmd.append("--today-only")
|
||||||
return cmd
|
return cmd
|
||||||
|
|
||||||
def run_creator(topic_id: str = None):
|
def run_creator(topic_id: str = None):
|
||||||
"""非阻塞:后台启动创作脚本,立即返回"""
|
"""非阻塞:后台启动创作脚本,立即返回"""
|
||||||
|
with _lock:
|
||||||
|
existing = _running_processes.get("generator", {})
|
||||||
|
proc = existing.get("process")
|
||||||
|
if proc and proc.poll() is None:
|
||||||
|
raise RuntimeError("已有创作任务正在运行")
|
||||||
cmd = _get_cmd(topic_id)
|
cmd = _get_cmd(topic_id)
|
||||||
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=str(PROJECT_ROOT))
|
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=str(PROJECT_ROOT))
|
||||||
_running_processes["generator"] = {"pid": proc.pid, "started_at": time.time(), "process": proc, "topic_id": topic_id}
|
_running_processes["generator"] = {"pid": proc.pid, "started_at": time.time(), "process": proc, "topic_id": topic_id}
|
||||||
logger.info("Creator started in background (pid=%s, topic_id=%s)", proc.pid, topic_id)
|
logger.info("Creator started in background (pid=%s, topic_id=%s)", proc.pid, topic_id)
|
||||||
return {"ok": True, "pid": proc.pid, "topic_id": topic_id}
|
return {"ok": True, "pid": proc.pid, "topic_id": topic_id, "_proc": proc}
|
||||||
|
|
||||||
def run_creator_blocking(topic_id: str = None, timeout: int = 1800):
|
def run_creator_blocking(topic_id: str = None, timeout: int = 1800):
|
||||||
"""阻塞版:带超时,给定时任务使用"""
|
"""阻塞版:带超时,给定时任务使用"""
|
||||||
@@ -93,9 +103,11 @@ def get_generator_status() -> Optional[Dict]:
|
|||||||
if not info:
|
if not info:
|
||||||
return None
|
return None
|
||||||
proc = info["process"]
|
proc = info["process"]
|
||||||
|
if "finished" in info:
|
||||||
|
return {"status": "completed" if info.get("returncode") == 0 else "failed", "pid": info["pid"], "elapsed": round(time.time() - info["started_at"], 1), "returncode": info.get("returncode"), "topic_id": info.get("topic_id")}
|
||||||
if proc.poll() is not None:
|
if proc.poll() is not None:
|
||||||
stdout, stderr = proc.communicate()
|
info["returncode"] = proc.returncode
|
||||||
|
info["finished"] = True
|
||||||
elapsed = time.time() - info["started_at"]
|
elapsed = time.time() - info["started_at"]
|
||||||
del _running_processes["generator"]
|
|
||||||
return {"status": "completed" if proc.returncode == 0 else "failed", "pid": info["pid"], "elapsed": round(elapsed, 1), "returncode": proc.returncode, "topic_id": info["topic_id"]}
|
return {"status": "completed" if proc.returncode == 0 else "failed", "pid": info["pid"], "elapsed": round(elapsed, 1), "returncode": proc.returncode, "topic_id": info["topic_id"]}
|
||||||
return {"status": "running", "pid": info["pid"], "elapsed": round(time.time() - info["started_at"], 1)}
|
return {"status": "running", "pid": info["pid"], "elapsed": round(time.time() - info["started_at"], 1)}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
|
import threading
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import List, Optional, Dict
|
from typing import List, Optional, Dict
|
||||||
|
|
||||||
@@ -14,6 +15,7 @@ if os.getenv('PROJECT_ROOT'):
|
|||||||
PROJECT_ROOT = Path(os.getenv('PROJECT_ROOT'))
|
PROJECT_ROOT = Path(os.getenv('PROJECT_ROOT'))
|
||||||
|
|
||||||
_running_processes: Dict[str, dict] = {}
|
_running_processes: Dict[str, dict] = {}
|
||||||
|
_lock = threading.Lock()
|
||||||
|
|
||||||
def _get_cmd(topic_ids: List[str] = None):
|
def _get_cmd(topic_ids: List[str] = None):
|
||||||
script_path = PROJECT_ROOT / "scripts" / "compliance_optimizer.py"
|
script_path = PROJECT_ROOT / "scripts" / "compliance_optimizer.py"
|
||||||
@@ -23,15 +25,22 @@ def _get_cmd(topic_ids: List[str] = None):
|
|||||||
cmd = [str(venv_python), str(script_path)] if venv_python.exists() else ["python3", str(script_path)]
|
cmd = [str(venv_python), str(script_path)] if venv_python.exists() else ["python3", str(script_path)]
|
||||||
if topic_ids:
|
if topic_ids:
|
||||||
cmd.extend(["--topic-ids", ','.join(topic_ids)])
|
cmd.extend(["--topic-ids", ','.join(topic_ids)])
|
||||||
|
else:
|
||||||
|
cmd.append("--today-only")
|
||||||
return cmd
|
return cmd
|
||||||
|
|
||||||
def run_optimizer(topic_ids: List[str] = None):
|
def run_optimizer(topic_ids: List[str] = None):
|
||||||
"""非阻塞:后台启动合规审查脚本,立即返回"""
|
"""非阻塞:后台启动合规审查脚本,立即返回"""
|
||||||
|
with _lock:
|
||||||
|
existing = _running_processes.get("optimizer", {})
|
||||||
|
proc = existing.get("process")
|
||||||
|
if proc and proc.poll() is None:
|
||||||
|
raise RuntimeError("已有合规审查任务正在运行,请等待完成后再试")
|
||||||
cmd = _get_cmd(topic_ids)
|
cmd = _get_cmd(topic_ids)
|
||||||
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=str(PROJECT_ROOT))
|
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=str(PROJECT_ROOT))
|
||||||
_running_processes["optimizer"] = {"pid": proc.pid, "started_at": time.time(), "process": proc, "topic_ids": topic_ids}
|
_running_processes["optimizer"] = {"pid": proc.pid, "started_at": time.time(), "process": proc, "topic_ids": topic_ids}
|
||||||
logger.info("Optimizer started in background (pid=%s, topic_ids=%s)", proc.pid, topic_ids)
|
logger.info("Optimizer started in background (pid=%s, topic_ids=%s)", proc.pid, topic_ids)
|
||||||
return {"ok": True, "pid": proc.pid, "topic_ids": topic_ids}
|
return {"ok": True, "pid": proc.pid, "topic_ids": topic_ids, "_proc": proc}
|
||||||
|
|
||||||
def run_optimizer_blocking(topic_ids: List[str] = None, timeout: int = 600):
|
def run_optimizer_blocking(topic_ids: List[str] = None, timeout: int = 600):
|
||||||
"""阻塞版:带超时,给定时任务使用"""
|
"""阻塞版:带超时,给定时任务使用"""
|
||||||
@@ -54,9 +63,11 @@ def get_optimizer_status() -> Optional[Dict]:
|
|||||||
if not info:
|
if not info:
|
||||||
return None
|
return None
|
||||||
proc = info["process"]
|
proc = info["process"]
|
||||||
|
if "finished" in info:
|
||||||
|
return {"status": "completed" if info.get("returncode") == 0 else "failed", "pid": info["pid"], "elapsed": round(time.time() - info["started_at"], 1), "returncode": info.get("returncode")}
|
||||||
if proc.poll() is not None:
|
if proc.poll() is not None:
|
||||||
stdout, stderr = proc.communicate()
|
info["returncode"] = proc.returncode
|
||||||
|
info["finished"] = True
|
||||||
elapsed = time.time() - info["started_at"]
|
elapsed = time.time() - info["started_at"]
|
||||||
del _running_processes["optimizer"]
|
return {"status": "completed" if proc.returncode == 0 else "failed", "pid": info["pid"], "elapsed": round(elapsed, 1), "returncode": proc.returncode}
|
||||||
return {"status": "completed" if proc.returncode == 0 else "failed", "pid": info["pid"], "elapsed": round(elapsed, 1), "returncode": proc.returncode, "stdout": stdout.strip()[-300:], "stderr": stderr.strip()[-300:]}
|
|
||||||
return {"status": "running", "pid": info["pid"], "elapsed": round(time.time() - info["started_at"], 1)}
|
return {"status": "running", "pid": info["pid"], "elapsed": round(time.time() - info["started_at"], 1)}
|
||||||
|
|||||||
@@ -31,10 +31,38 @@ MODULES = {
|
|||||||
"scheduled_task_monitor": {"name": "⏰ 任务监控", "cron": "*"},
|
"scheduled_task_monitor": {"name": "⏰ 任务监控", "cron": "*"},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LOG_FILE_MAP = {
|
||||||
|
"scheduled_refresh_search_cache": "opencode_search",
|
||||||
|
"scheduled_fetch_trends": "trends",
|
||||||
|
"scheduled_collect": "collector",
|
||||||
|
"scheduled_generate": "creator",
|
||||||
|
"scheduled_optimize": "optimizer",
|
||||||
|
"scheduled_optimize_sources": "optimizer_sources",
|
||||||
|
"scheduled_metrics_sync": "metrics_sync",
|
||||||
|
"scheduled_reset_search_usage": "reset_search_usage",
|
||||||
|
"scheduled_task_monitor": "task_monitor",
|
||||||
|
}
|
||||||
|
|
||||||
|
def _log_to_file(module_id: str, status: str, message: str = None, error_trace: str = None):
|
||||||
|
log_name = LOG_FILE_MAP.get(module_id, module_id)
|
||||||
|
log_dir = PROJECT_ROOT / "automation" / "logs"
|
||||||
|
log_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
log_file = log_dir / f"{log_name}_{datetime.now().strftime('%Y-%m-%d')}.log"
|
||||||
|
ts = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||||
|
line = f"[{ts}] [{status.upper()}] {message or ''}"
|
||||||
|
if error_trace:
|
||||||
|
line += f"\n{error_trace[:500]}"
|
||||||
|
try:
|
||||||
|
with open(log_file, 'a', encoding='utf-8') as f:
|
||||||
|
f.write(line + "\n")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
def _log_task(module_id: str, status: str, message: str = None,
|
def _log_task(module_id: str, status: str, message: str = None,
|
||||||
error_trace: str = None, result_data: dict = None,
|
error_trace: str = None, result_data: dict = None,
|
||||||
started_at: datetime = None, finished_at: datetime = None,
|
started_at: datetime = None, finished_at: datetime = None,
|
||||||
triggered_by: str = "scheduler", next_run_time: datetime = None):
|
triggered_by: str = "scheduler", next_run_time: datetime = None):
|
||||||
|
_log_to_file(module_id, status, message, error_trace)
|
||||||
try:
|
try:
|
||||||
from ..database import SessionLocal
|
from ..database import SessionLocal
|
||||||
from ..models import TaskLog
|
from ..models import TaskLog
|
||||||
@@ -159,7 +187,7 @@ class TaskScheduler:
|
|||||||
logger.info("[Scheduled] Fetching hot trends...")
|
logger.info("[Scheduled] Fetching hot trends...")
|
||||||
import subprocess
|
import subprocess
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
[sys.executable, str(Path(__file__).parent.parent.parent.parent / "scripts" / "trends.py")],
|
[sys.executable, str(PROJECT_ROOT / "scripts" / "trends.py")],
|
||||||
capture_output=True, text=True, timeout=120
|
capture_output=True, text=True, timeout=120
|
||||||
)
|
)
|
||||||
if result.returncode == 0:
|
if result.returncode == 0:
|
||||||
@@ -190,7 +218,7 @@ class TaskScheduler:
|
|||||||
logger.info("[Scheduled] Refreshing search cache via opencode...")
|
logger.info("[Scheduled] Refreshing search cache via opencode...")
|
||||||
import subprocess
|
import subprocess
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
[sys.executable, str(Path(__file__).parent.parent.parent.parent / "scripts" / "opencode_search.py"), "--refresh-cache"],
|
[sys.executable, str(PROJECT_ROOT / "scripts" / "opencode_search.py"), "--refresh-cache"],
|
||||||
capture_output=True, text=True, timeout=600
|
capture_output=True, text=True, timeout=600
|
||||||
)
|
)
|
||||||
for line in result.stdout.strip().split("\n"):
|
for line in result.stdout.strip().split("\n"):
|
||||||
@@ -285,10 +313,10 @@ class TaskScheduler:
|
|||||||
started_at=started, finished_at=datetime.now(timezone.utc))
|
started_at=started, finished_at=datetime.now(timezone.utc))
|
||||||
logger.exception("[Scheduled] Collection failed: %s", e)
|
logger.exception("[Scheduled] Collection failed: %s", e)
|
||||||
|
|
||||||
def _run_optimize_sources(self):
|
def _run_optimize_sources(self, triggered_by="scheduler"):
|
||||||
"""AI自动优化采集类别与信息源:对比市场热点和当前配置,给出调整建议"""
|
"""AI自动优化采集类别与信息源:对比市场热点和当前配置,给出调整建议"""
|
||||||
started = datetime.now(timezone.utc)
|
started = datetime.now(timezone.utc)
|
||||||
_log_task("scheduled_optimize_sources", "running", started_at=started)
|
_log_task("scheduled_optimize_sources", "running", started_at=started, triggered_by=triggered_by)
|
||||||
try:
|
try:
|
||||||
logger.info("[Scheduled] Starting source optimization with AI...")
|
logger.info("[Scheduled] Starting source optimization with AI...")
|
||||||
from .nvidia_client import call_llm
|
from .nvidia_client import call_llm
|
||||||
@@ -337,19 +365,19 @@ class TaskScheduler:
|
|||||||
"sources_assessed": len(result.get("source_assessment", [])),
|
"sources_assessed": len(result.get("source_assessment", [])),
|
||||||
"suggested_cats": len(result.get("suggested_new_categories", [])),
|
"suggested_cats": len(result.get("suggested_new_categories", [])),
|
||||||
"suggested_srcs": len(result.get("suggested_new_sources", []))},
|
"suggested_srcs": len(result.get("suggested_new_sources", []))},
|
||||||
started_at=started, finished_at=datetime.now(timezone.utc))
|
started_at=started, finished_at=datetime.now(timezone.utc), triggered_by=triggered_by)
|
||||||
db.close()
|
db.close()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log_task("scheduled_optimize_sources", "failed",
|
_log_task("scheduled_optimize_sources", "failed",
|
||||||
message=str(e),
|
message=str(e),
|
||||||
error_trace=traceback.format_exc(),
|
error_trace=traceback.format_exc(),
|
||||||
started_at=started, finished_at=datetime.now(timezone.utc))
|
started_at=started, finished_at=datetime.now(timezone.utc), triggered_by=triggered_by)
|
||||||
logger.exception("[Scheduled] Source AI optimization failed: %s", e)
|
logger.exception("[Scheduled] Source AI optimization failed: %s", e)
|
||||||
|
|
||||||
def _run_metrics_sync(self):
|
def _run_metrics_sync(self, triggered_by="scheduler"):
|
||||||
"""定时从各平台公开API获取发布文章的效果数据(当前仅支持知乎)"""
|
"""定时从各平台公开API获取发布文章的效果数据(当前仅支持知乎)"""
|
||||||
started = datetime.now(timezone.utc)
|
started = datetime.now(timezone.utc)
|
||||||
_log_task("scheduled_metrics_sync", "running", started_at=started)
|
_log_task("scheduled_metrics_sync", "running", started_at=started, triggered_by=triggered_by)
|
||||||
try:
|
try:
|
||||||
logger.info("[Scheduled] Starting metrics sync (zhihu auto-fetch)...")
|
logger.info("[Scheduled] Starting metrics sync (zhihu auto-fetch)...")
|
||||||
from ..database import SessionLocal
|
from ..database import SessionLocal
|
||||||
@@ -411,7 +439,7 @@ class TaskScheduler:
|
|||||||
_log_task("scheduled_metrics_sync", "success",
|
_log_task("scheduled_metrics_sync", "success",
|
||||||
message=f"同步完成,{count} 篇知乎文章",
|
message=f"同步完成,{count} 篇知乎文章",
|
||||||
result_data={"articles_synced": count},
|
result_data={"articles_synced": count},
|
||||||
started_at=started, finished_at=datetime.now(timezone.utc))
|
started_at=started, finished_at=datetime.now(timezone.utc), triggered_by=triggered_by)
|
||||||
# 生成指标反馈:按 field 聚合表现,写入 metrics_feedback.json 供 collector 读取
|
# 生成指标反馈:按 field 聚合表现,写入 metrics_feedback.json 供 collector 读取
|
||||||
try:
|
try:
|
||||||
import json as json_mod
|
import json as json_mod
|
||||||
@@ -436,7 +464,7 @@ class TaskScheduler:
|
|||||||
"top_domains": [(f, s) for f, s, _ in scored[:5]],
|
"top_domains": [(f, s) for f, s, _ in scored[:5]],
|
||||||
"detail": [{"field": f, "score": s, "articles": c} for f, s, c in scored],
|
"detail": [{"field": f, "score": s, "articles": c} for f, s, c in scored],
|
||||||
}
|
}
|
||||||
feedback_file = Path(__file__).parent.parent.parent.parent / "automation" / "data" / "metrics_feedback.json"
|
feedback_file = PROJECT_ROOT / "automation" / "data" / "metrics_feedback.json"
|
||||||
feedback_file.parent.mkdir(parents=True, exist_ok=True)
|
feedback_file.parent.mkdir(parents=True, exist_ok=True)
|
||||||
feedback_file.write_text(json_mod.dumps(feedback_data, ensure_ascii=False, indent=2), encoding='utf-8')
|
feedback_file.write_text(json_mod.dumps(feedback_data, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||||
logger.info("[Scheduled] Metrics feedback written: top domain %s (score %.1f)", scored[0][0], scored[0][1])
|
logger.info("[Scheduled] Metrics feedback written: top domain %s (score %.1f)", scored[0][0], scored[0][1])
|
||||||
@@ -445,14 +473,14 @@ class TaskScheduler:
|
|||||||
else:
|
else:
|
||||||
_log_task("scheduled_metrics_sync", "success",
|
_log_task("scheduled_metrics_sync", "success",
|
||||||
message="无已发布的知乎文章",
|
message="无已发布的知乎文章",
|
||||||
started_at=started, finished_at=datetime.now(timezone.utc))
|
started_at=started, finished_at=datetime.now(timezone.utc), triggered_by=triggered_by)
|
||||||
logger.info("[Scheduled] Metrics sync: no zhihu articles to sync")
|
logger.info("[Scheduled] Metrics sync: no zhihu articles to sync")
|
||||||
db.close()
|
db.close()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_log_task("scheduled_metrics_sync", "failed",
|
_log_task("scheduled_metrics_sync", "failed",
|
||||||
message=str(e),
|
message=str(e),
|
||||||
error_trace=traceback.format_exc(),
|
error_trace=traceback.format_exc(),
|
||||||
started_at=started, finished_at=datetime.now(timezone.utc))
|
started_at=started, finished_at=datetime.now(timezone.utc), triggered_by=triggered_by)
|
||||||
logger.exception("[Scheduled] Metrics sync failed: %s", e)
|
logger.exception("[Scheduled] Metrics sync failed: %s", e)
|
||||||
|
|
||||||
def _run_reset_search_usage(self):
|
def _run_reset_search_usage(self):
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ from sqlalchemy.ext.declarative import declarative_base
|
|||||||
from sqlalchemy.orm import sessionmaker
|
from sqlalchemy.orm import sessionmaker
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
|
||||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent.parent
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent.parent
|
||||||
|
|
||||||
@@ -140,6 +141,20 @@ def init_db():
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass # SQLite 不支持 IF NOT EXISTS,但 create_all 对 SQLite 够用,这里仅为 PostgreSQL 迁移
|
pass # SQLite 不支持 IF NOT EXISTS,但 create_all 对 SQLite 够用,这里仅为 PostgreSQL 迁移
|
||||||
|
|
||||||
|
# 清理卡死的 running 任务(超过 30 分钟无更新的标记为失败)
|
||||||
|
try:
|
||||||
|
with SessionLocal() as sess:
|
||||||
|
from sqlalchemy import text as sa_text
|
||||||
|
cutoff = datetime.now(timezone.utc) - timedelta(minutes=30)
|
||||||
|
sess.execute(
|
||||||
|
sa_text("UPDATE task_logs SET status='failed', message='系统启动时清理卡死任务', finished_at=now() "
|
||||||
|
"WHERE status='running' AND started_at < :cutoff AND module_id != 'scheduled_task_monitor'"),
|
||||||
|
{"cutoff": cutoff}
|
||||||
|
)
|
||||||
|
sess.commit()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
def get_db():
|
def get_db():
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -503,6 +503,8 @@ class TaskConfigUpdate(BaseModel):
|
|||||||
class TaskConfigResponse(TaskConfigBase):
|
class TaskConfigResponse(TaskConfigBase):
|
||||||
id: int
|
id: int
|
||||||
last_modified_by: Optional[str] = None
|
last_modified_by: Optional[str] = None
|
||||||
|
last_run_at: Optional[datetime] = None
|
||||||
|
result_data: Optional[Dict[str, Any]] = None
|
||||||
created_at: Optional[datetime] = None
|
created_at: Optional[datetime] = None
|
||||||
updated_at: Optional[datetime] = None
|
updated_at: Optional[datetime] = None
|
||||||
|
|
||||||
|
|||||||
@@ -363,78 +363,13 @@
|
|||||||
</div>
|
</div>
|
||||||
</el-tab-pane>
|
</el-tab-pane>
|
||||||
<el-tab-pane label="📤 产出结果" name="outputs" style="overflow:auto;flex:1;">
|
<el-tab-pane label="📤 产出结果" name="outputs" style="overflow:auto;flex:1;">
|
||||||
<div v-if="drawerData.last_result && Object.keys(drawerData.last_result).length > 0">
|
<div v-if="drawerData.result_data && Object.keys(drawerData.result_data).length > 0">
|
||||||
<div style="font-size:13px;font-weight:600;margin-bottom:10px;color:#303133;">最近运行结果</div>
|
<div v-for="(val, key) in drawerData.result_data" :key="key" style="margin-bottom:6px;font-size:13px;">
|
||||||
<div v-for="(val, key) in drawerData.last_result" :key="key" style="margin-bottom:6px;font-size:13px;">
|
|
||||||
<span style="color:#909399;">{{ key }}:</span>
|
<span style="color:#909399;">{{ key }}:</span>
|
||||||
<span :style="{color: ['topics_count','articles_synced','processed','passed','topics_found'].includes(key) ? '#409eff' : '#606266', fontWeight: ['topics_count','articles_synced'].includes(key) ? '600' : '400'}">{{ val }}</span>
|
<span :style="{color: ['topics_count','articles_synced','processed','passed','topics_found'].includes(key) ? '#409eff' : '#606266', fontWeight: ['topics_count','articles_synced'].includes(key) ? '600' : '400'}">{{ val }}</span>
|
||||||
</div>
|
</div>
|
||||||
<el-divider v-if="Object.keys(drawerData.outputs || {}).length > 0" />
|
|
||||||
</div>
|
|
||||||
<div v-if="Object.keys(drawerData.outputs || {}).length === 0" style="color:#909399;padding:20px;text-align:center;">暂无产出数据</div>
|
|
||||||
<div v-else>
|
|
||||||
<div v-for="(val, key) in drawerData.outputs" :key="key" style="margin-bottom:16px;border-bottom:1px solid #f0f2f5;padding-bottom:12px;">
|
|
||||||
<div style="font-size:13px;font-weight:600;color:#303133;margin-bottom:6px;">{{ key }}</div>
|
|
||||||
<div v-if="key === '热点列表' && Array.isArray(val)">
|
|
||||||
<el-table :data="val" size="small" max-height="400" style="width:100%;">
|
|
||||||
<el-table-column prop="topic" label="话题" min-width="180"></el-table-column>
|
|
||||||
<el-table-column prop="domain" label="领域" width="100"></el-table-column>
|
|
||||||
<el-table-column prop="platform" label="平台" width="100"></el-table-column>
|
|
||||||
<el-table-column prop="source" label="来源" width="80"></el-table-column>
|
|
||||||
</el-table>
|
|
||||||
</div>
|
|
||||||
<div v-else-if="key === '最新选题' && Array.isArray(val)">
|
|
||||||
<el-table :data="val" size="small" max-height="300" style="width:100%;">
|
|
||||||
<el-table-column prop="id" label="ID" width="80"></el-table-column>
|
|
||||||
<el-table-column prop="title" label="标题" min-width="150"></el-table-column>
|
|
||||||
<el-table-column prop="field" label="领域" width="80"></el-table-column>
|
|
||||||
<el-table-column prop="status" label="状态" width="70"></el-table-column>
|
|
||||||
<el-table-column prop="created" label="创建时间" min-width="150"></el-table-column>
|
|
||||||
</el-table>
|
|
||||||
</div>
|
|
||||||
<div v-else-if="(key === '最新文章' || key === '最新指标') && Array.isArray(val)">
|
|
||||||
<el-table :data="val" size="small" max-height="300" style="width:100%;">
|
|
||||||
<el-table-column v-for="col in Object.keys(val[0]||{})" :key="col" :prop="col" :label="col" min-width="120"></el-table-column>
|
|
||||||
</el-table>
|
|
||||||
</div>
|
|
||||||
<div v-else-if="key === '各分类结果' && Array.isArray(val)">
|
|
||||||
<div v-for="item in val" :key="item.query" style="margin-bottom:8px;padding:8px;background:#f8faff;border-radius:6px;">
|
|
||||||
<div style="font-size:13px;font-weight:500;">{{ item.query }} <el-tag size="mini">{{ item.count }}条</el-tag></div>
|
|
||||||
<div style="font-size:12px;color:#909399;margin-top:4px;" v-for="s in item.samples" :key="s">● {{ s }}</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div v-else-if="key === '文章评分' && Array.isArray(val)">
|
|
||||||
<div v-for="item in val" :key="item.title" style="margin-bottom:6px;display:flex;align-items:center;gap:8px;">
|
|
||||||
<span style="font-size:13px;">{{ item.title }}</span>
|
|
||||||
<el-rate :model-value="parseScore(item.score)" disabled show-score score-template="{value}分" size="small"></el-rate>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div v-else-if="key === 'AI建议摘要'" style="background:#f0f9eb;padding:10px 14px;border-radius:8px;font-size:13px;color:#303133;">{{ val }}</div>
|
|
||||||
<div v-else-if="key === '建议新增类别' && Array.isArray(val)">
|
|
||||||
<div v-for="item in val" :key="item.name" style="padding:8px;background:#f0f9eb;border-radius:6px;margin-bottom:6px;">
|
|
||||||
<div style="font-weight:500;">{{ item.name }}</div>
|
|
||||||
<div style="font-size:12px;color:#606266;">{{ item.reason }}</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div v-else-if="key === '类别评估' && Array.isArray(val)">
|
|
||||||
<div v-for="item in val" :key="item.name" style="margin-bottom:4px;font-size:13px;">
|
|
||||||
<el-tag :type="item.status==='保留'?'success':'warning'" size="mini" style="margin-right:6px;">{{ item.status }}</el-tag>
|
|
||||||
{{ item.name }} — {{ item.reason }}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div v-else-if="key === '来源分布' && typeof val === 'object'">
|
|
||||||
<div v-for="(cnt, src) in val" :key="src" style="margin-bottom:4px;font-size:13px;">
|
|
||||||
{{ src }}: <el-tag size="mini">{{ cnt }}条</el-tag>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div v-else-if="key === '高互动领域' && Array.isArray(val)">
|
|
||||||
<div v-for="item in val" :key="item[0]" style="margin-bottom:4px;font-size:13px;">
|
|
||||||
{{ item[0] }}: <el-tag size="mini" type="success">{{ item[1] }}分</el-tag>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div v-else style="font-size:13px;color:#606266;">{{ val }}</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
|
<div v-else style="color:#909399;padding:20px;text-align:center;">暂无产出数据</div>
|
||||||
</el-tab-pane>
|
</el-tab-pane>
|
||||||
<el-tab-pane label="📋 运行记录" name="history" style="overflow:auto;flex:1;">
|
<el-tab-pane label="📋 运行记录" name="history" style="overflow:auto;flex:1;">
|
||||||
<div v-if="drawerHistory && drawerHistory.length > 0">
|
<div v-if="drawerHistory && drawerHistory.length > 0">
|
||||||
|
|||||||
@@ -459,17 +459,30 @@ const TopicsApp = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (platform === 'xiaohongshu') {
|
if (platform === 'xiaohongshu') {
|
||||||
const lines = [];
|
const container = doc.createElement('div');
|
||||||
|
if (titleEl) {
|
||||||
|
const strong = doc.createElement('strong');
|
||||||
|
strong.textContent = title;
|
||||||
|
container.appendChild(strong);
|
||||||
|
container.appendChild(doc.createElement('br'));
|
||||||
|
container.appendChild(doc.createElement('br'));
|
||||||
|
}
|
||||||
body.querySelectorAll('h1,h2,h3,h4,p,li').forEach(el => {
|
body.querySelectorAll('h1,h2,h3,h4,p,li').forEach(el => {
|
||||||
const text = el.textContent.trim();
|
const text = el.textContent.trim();
|
||||||
if (!text || text.length < 2) return;
|
if (!text || text.length < 2) return;
|
||||||
const tag = el.tagName.toLowerCase();
|
const clone = el.cloneNode(true);
|
||||||
const prefix = tag.startsWith('h') ? '\n### ' : '- ';
|
container.appendChild(clone);
|
||||||
const clean = text.replace(/\n/g, ' ').replace(/^[\s#]+|[\s#]+$/g, '');
|
container.appendChild(doc.createElement('br'));
|
||||||
if (clean) lines.push(prefix + clean);
|
});
|
||||||
|
const cleanHtml = container.innerHTML;
|
||||||
|
const blob = new Blob([cleanHtml], { type: 'text/html' });
|
||||||
|
const richText = new Blob([cleanHtml.replace(/<[^>]+>/g, '').replace(/\n{3,}/g, '\n\n')], { type: 'text/plain' });
|
||||||
|
const item = new ClipboardItem({ 'text/html': blob, 'text/plain': richText });
|
||||||
|
navigator.clipboard.write([item]).then(() => {
|
||||||
|
this.$message.success('✅ 已复制(含格式),Ctrl+V 粘贴到小红书');
|
||||||
|
}).catch(() => {
|
||||||
|
navigator.clipboard.writeText(cleanHtml.replace(/<[^>]+>/g, '').replace(/\n{3,}/g, '\n\n')).then(() => this.$message.success('✅ 已复制')).catch(() => this.$message.error('❌ 复制失败'));
|
||||||
});
|
});
|
||||||
const md = `**${title}**\n\n${lines.join('\n')}`;
|
|
||||||
navigator.clipboard.writeText(md).then(() => this.$message.success('✅ 已复制 Markdown(小红书格式)')).catch(() => this.$message.error('❌ 复制失败'));
|
|
||||||
} else {
|
} else {
|
||||||
const container = doc.createElement('div');
|
const container = doc.createElement('div');
|
||||||
if (titleEl) {
|
if (titleEl) {
|
||||||
@@ -477,7 +490,7 @@ const TopicsApp = {
|
|||||||
h1.textContent = title;
|
h1.textContent = title;
|
||||||
container.appendChild(h1);
|
container.appendChild(h1);
|
||||||
}
|
}
|
||||||
body.querySelectorAll('h2,h3,h4,p,li,blockquote,img,pre,code,table,hr').forEach(el => {
|
body.querySelectorAll('h2,h3,h4,p,li,blockquote,pre,code,table,hr').forEach(el => {
|
||||||
const clone = el.cloneNode(true);
|
const clone = el.cloneNode(true);
|
||||||
container.appendChild(clone);
|
container.appendChild(clone);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from pathlib import Path
|
|||||||
from typing import Dict, List, Optional, Tuple
|
from typing import Dict, List, Optional, Tuple
|
||||||
from dataclasses import dataclass, asdict
|
from dataclasses import dataclass, asdict
|
||||||
|
|
||||||
PROJECT_ROOT = Path('/root/openclaw-workspace/projects/yu-zhi-ran')
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||||
sys.path.insert(0, str(PROJECT_ROOT))
|
sys.path.insert(0, str(PROJECT_ROOT))
|
||||||
sys.path.insert(0, str(PROJECT_ROOT / "platform" / "backend"))
|
sys.path.insert(0, str(PROJECT_ROOT / "platform" / "backend"))
|
||||||
|
|
||||||
@@ -92,7 +92,7 @@ def load_topic_map():
|
|||||||
topics = export_topics_to_json()
|
topics = export_topics_to_json()
|
||||||
return {t['id']: t for t in topics}
|
return {t['id']: t for t in topics}
|
||||||
|
|
||||||
def get_articles_from_db(topic_ids: Optional[List[str]] = None) -> List[Tuple[str, str, str]]:
|
def get_articles_from_db(topic_ids: Optional[List[str]] = None, today_only: bool = False) -> List[Tuple[str, str, str]]:
|
||||||
"""从 articles 表读取 HTML 内容
|
"""从 articles 表读取 HTML 内容
|
||||||
|
|
||||||
Returns: [(html_content, platform, topic_id), ...]
|
Returns: [(html_content, platform, topic_id), ...]
|
||||||
@@ -110,9 +110,14 @@ def get_articles_from_db(topic_ids: Optional[List[str]] = None) -> List[Tuple[st
|
|||||||
else:
|
else:
|
||||||
from app.database import SessionLocal
|
from app.database import SessionLocal
|
||||||
from app.models import Article
|
from app.models import Article
|
||||||
|
from sqlalchemy import func
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
try:
|
try:
|
||||||
all_articles = db.query(Article).filter(Article.html_content.isnot(None)).all()
|
query = db.query(Article).filter(Article.html_content.isnot(None))
|
||||||
|
if today_only:
|
||||||
|
cutoff = datetime.datetime.now() - datetime.timedelta(hours=24)
|
||||||
|
query = query.filter(Article.created_at >= cutoff)
|
||||||
|
all_articles = query.all()
|
||||||
for a in all_articles:
|
for a in all_articles:
|
||||||
results.append((a.html_content, a.platform, a.topic_id))
|
results.append((a.html_content, a.platform, a.topic_id))
|
||||||
finally:
|
finally:
|
||||||
@@ -222,14 +227,17 @@ def _load_platform_configs() -> Dict[str, Dict]:
|
|||||||
finally:
|
finally:
|
||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
def main(topic_ids: List[str] = None):
|
def main(topic_ids: List[str] = None, today_only: bool = False):
|
||||||
logger.info("=== 合规审查与优化开始 ===")
|
logger.info("=== 合规审查与优化开始 ===")
|
||||||
logger.info("LLM 配置: opencode-go (model=deepseek-v4-flash) — 固定用于合规审查")
|
logger.info("LLM 配置: opencode-go (model=deepseek-v4-flash) — 固定用于合规审查")
|
||||||
|
|
||||||
platform_configs = _load_platform_configs()
|
platform_configs = _load_platform_configs()
|
||||||
logger.info(f"已加载 {len(platform_configs)} 个平台配置")
|
logger.info(f"已加载 {len(platform_configs)} 个平台配置")
|
||||||
|
|
||||||
articles = get_articles_from_db(topic_ids)
|
if today_only:
|
||||||
|
logger.info("仅处理当天创建的选题文章")
|
||||||
|
|
||||||
|
articles = get_articles_from_db(topic_ids, today_only)
|
||||||
if not articles:
|
if not articles:
|
||||||
logger.warning("未找到任何文章(可能尚未创作或同步到 DB)")
|
logger.warning("未找到任何文章(可能尚未创作或同步到 DB)")
|
||||||
report_file = DRAFTS_DIR / TODAY / "optimization_report.json"
|
report_file = DRAFTS_DIR / TODAY / "optimization_report.json"
|
||||||
@@ -349,6 +357,7 @@ if __name__ == "__main__":
|
|||||||
import argparse
|
import argparse
|
||||||
parser = argparse.ArgumentParser(description='合规审查与优化任务')
|
parser = argparse.ArgumentParser(description='合规审查与优化任务')
|
||||||
parser.add_argument('--topic-ids', help='逗号分隔的选题ID列表,例如: A01,B02')
|
parser.add_argument('--topic-ids', help='逗号分隔的选题ID列表,例如: A01,B02')
|
||||||
|
parser.add_argument('--today-only', action='store_true', help='仅处理当天创建的选题文章')
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
topic_ids = args.topic_ids.split(',') if args.topic_ids else None
|
topic_ids = args.topic_ids.split(',') if args.topic_ids else None
|
||||||
main(topic_ids)
|
main(topic_ids, today_only=args.today_only)
|
||||||
|
|||||||
+6
-5
@@ -28,7 +28,7 @@ logging.basicConfig(
|
|||||||
)
|
)
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
def select_next_topic(topic_id: str = None) -> Dict:
|
def select_next_topic(topic_id: str = None, today_only: bool = False) -> Dict:
|
||||||
"""选择并锁定要创作的选题(趋势引擎匹配 → 回退优先级)"""
|
"""选择并锁定要创作的选题(趋势引擎匹配 → 回退优先级)"""
|
||||||
if topic_id:
|
if topic_id:
|
||||||
topic = get_topic_by_id(topic_id)
|
topic = get_topic_by_id(topic_id)
|
||||||
@@ -50,7 +50,7 @@ def select_next_topic(topic_id: str = None) -> Dict:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"选题引擎失效,回退简单策略: {e}")
|
logger.warning(f"选题引擎失效,回退简单策略: {e}")
|
||||||
|
|
||||||
topic = get_next_topic(priority='高') or get_next_topic()
|
topic = get_next_topic(priority='高', today_only=today_only) or get_next_topic(today_only=today_only)
|
||||||
if not topic:
|
if not topic:
|
||||||
raise ValueError("No available topics to create (all locked or wrong status)")
|
raise ValueError("No available topics to create (all locked or wrong status)")
|
||||||
update_topic_status(topic['id'], 'review')
|
update_topic_status(topic['id'], 'review')
|
||||||
@@ -80,11 +80,11 @@ def run_optimizer_step(topic_id: str) -> bool:
|
|||||||
logger.info(f"compliance_optimizer 完成: {result.stdout.strip()}")
|
logger.info(f"compliance_optimizer 完成: {result.stdout.strip()}")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def run_pipeline(topic_id: str = None) -> Dict:
|
def run_pipeline(topic_id: str = None, today_only: bool = False) -> Dict:
|
||||||
"""运行完整流水线:研究 → 大纲 → 撰写 → 合规优化"""
|
"""运行完整流水线:研究 → 大纲 → 撰写 → 合规优化"""
|
||||||
tid = None
|
tid = None
|
||||||
try:
|
try:
|
||||||
topic = select_next_topic(topic_id)
|
topic = select_next_topic(topic_id, today_only)
|
||||||
tid = topic['id']
|
tid = topic['id']
|
||||||
logger.info(f"开始创作流水线: topic_id={tid}, title={topic.get('title')}")
|
logger.info(f"开始创作流水线: topic_id={tid}, title={topic.get('title')}")
|
||||||
|
|
||||||
@@ -127,9 +127,10 @@ def main():
|
|||||||
import argparse
|
import argparse
|
||||||
parser = argparse.ArgumentParser(description='内容创作流水线(研究→大纲→撰写→合规优化)')
|
parser = argparse.ArgumentParser(description='内容创作流水线(研究→大纲→撰写→合规优化)')
|
||||||
parser.add_argument('--topic-id', help='指定选题ID,不指定则自动选择待处理选题')
|
parser.add_argument('--topic-id', help='指定选题ID,不指定则自动选择待处理选题')
|
||||||
|
parser.add_argument('--today-only', action='store_true', help='仅处理当天创建的选题')
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
result = run_pipeline(args.topic_id)
|
result = run_pipeline(args.topic_id, today_only=args.today_only)
|
||||||
print(json.dumps(result, ensure_ascii=False))
|
print(json.dumps(result, ensure_ascii=False))
|
||||||
sys.exit(0 if result['ok'] else 1)
|
sys.exit(0 if result['ok'] else 1)
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from datetime import datetime, date
|
from datetime import datetime, date, timedelta
|
||||||
from typing import Optional, Dict, List
|
from typing import Optional, Dict, List
|
||||||
|
|
||||||
# 加载 .env(在 scripts/ 目录下运行时需要)
|
# 加载 .env(在 scripts/ 目录下运行时需要)
|
||||||
@@ -53,7 +53,7 @@ def get_topics_by_status(status: str, db: Optional[Session] = None) -> List[Dict
|
|||||||
if close_db:
|
if close_db:
|
||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
def get_next_topic(priority: Optional[str] = None, db: Optional[Session] = None) -> Optional[Dict]:
|
def get_next_topic(priority: Optional[str] = None, db: Optional[Session] = None, today_only: bool = False) -> Optional[Dict]:
|
||||||
"""获取下一个待处理的选题(状态为 pending/待处理)"""
|
"""获取下一个待处理的选题(状态为 pending/待处理)"""
|
||||||
close_db = False
|
close_db = False
|
||||||
if db is None:
|
if db is None:
|
||||||
@@ -65,6 +65,9 @@ def get_next_topic(priority: Optional[str] = None, db: Optional[Session] = None)
|
|||||||
query = db.query(Topic).filter(Topic.status.in_(status_filter))
|
query = db.query(Topic).filter(Topic.status.in_(status_filter))
|
||||||
if priority:
|
if priority:
|
||||||
query = query.filter(Topic.priority == priority)
|
query = query.filter(Topic.priority == priority)
|
||||||
|
if today_only:
|
||||||
|
cutoff = datetime.now() - timedelta(hours=24)
|
||||||
|
query = query.filter(Topic.created_at >= cutoff)
|
||||||
topic = query.order_by(Topic.priority_score.desc().nullslast(), Topic.created_at.asc()).first()
|
topic = query.order_by(Topic.priority_score.desc().nullslast(), Topic.created_at.asc()).first()
|
||||||
return topic_to_dict(topic) if topic else None
|
return topic_to_dict(topic) if topic else None
|
||||||
finally:
|
finally:
|
||||||
|
|||||||
Reference in New Issue
Block a user