txai-dev/yu-zhi-ran
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: opencode冗余清理 + LLM任务级模型选择 + systemd服务化

Browse Source 
- 删除 opencode_search.py / mcp_search_server.py 及所有 MCP 引用
- 移除搜索缓存定时任务（scheduled_refresh_search_cache）
- 清理前后端所有 opencode/MCP 代码和注释
- LLM 提供商量换：opencode-go→nvidia(默认)+sensenova(合规审查)
- llm_configs 新增 is_default 字段，API 层互斥逻辑
- 所有定时任务支持独立 LLM 模型选择（LLM_TASK_PROVIDER env）
- compliance_optimizer.py 修复：import os / 解硬编码 / 关键词过滤
- Scheduler 日志修复：始终 INSERT，避免僵尸 running 行
- Systemd 服务化：Restart=always / 单 worker / Type=exec
- 搜索提供商：替换 opencode→360/搜狗/微信（免 Key）
- 更新 AGENTS.md / PROGRESS.md
This commit is contained in:
Yuzhiran Dev
2026-06-02 15:38:16 +08:00
parent abbf1468bc
commit 499c511140
25 changed files with 451 additions and 678 deletions
Download Patch File Download Diff File Expand all files Collapse all files
platform/backend/app/api/llm_configs.py
+11
View File
@@ -9,6 +9,14 @@ from .auth import get_current_admin
router = APIRouter(prefix="/api/admin/llmconfigs", tags=["admin"])
def _apply_default_exclusive(config: LLMConfig, db: Session):
"""当 config.is_default=True 时，将其他所有配置的 is_default 置为 False"""
if config.is_default:
db.query(LLMConfig).filter(LLMConfig.id != config.id).update(
{"is_default": False}, synchronize_session=False
)
db.flush()
@router.get("", response_model=List[LLMConfigResponse])
def list_llm_configs(
request: Request,
@@ -42,6 +50,8 @@ def create_llm_config(
"""创建 LLM 配置"""
config = LLMConfig(**config_data.model_dump())
db.add(config)
db.flush()
_apply_default_exclusive(config, db)
db.commit()
db.refresh(config)
return config
@@ -61,6 +71,7 @@ def update_llm_config(
update_data = config_update.model_dump(exclude_unset=True)
for field, value in update_data.items():
setattr(config, field, value)
_apply_default_exclusive(config, db)
db.commit()
db.refresh(config)
return config
platform/backend/app/api/search_providers.py
-10
View File
@@ -119,16 +119,6 @@ def test_provider(provider_id: int, data: dict = {}, db: Session = Depends(get_d
if resp.status_code != 200:
return {"ok": False, "error": f"HTTP {resp.status_code}: {resp.text[:200]}"}
return {"ok": True, "results": resp.json().get("webPages", {}).get("value", [])[:3]}
elif p.provider_type == "mcp":
import subprocess, json as _json
mcp_script = Path(__file__).resolve().parent.parent.parent.parent.parent / "scripts" / "mcp_search_server.py"
r = subprocess.run(
[sys.executable, str(mcp_script), "--query", query],
capture_output=True, text=True, timeout=90,
)
if r.returncode != 0:
return {"ok": False, "error": f"子进程失败: {r.stderr[:200]}"}
return {"ok": True, "results": _json.loads(r.stdout)[:3]}
return {"ok": False, "error": f"Unknown provider_type: {p.provider_type}"}
except Exception as e:
return {"ok": False, "error": str(e)}
platform/backend/app/api/system.py
-23
View File
@@ -280,28 +280,6 @@ def trigger_metrics_sync():
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.post("/refresh-search-cache/run")
def trigger_refresh_search_cache(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
try:
import sys as sys_mod
scripts_dir = PROJECT_ROOT / "scripts"
from ..database import SessionLocal as _ss
proc = subprocess.Popen(
[sys_mod.executable, str(scripts_dir / "opencode_search.py"), "--refresh-cache"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
cwd=str(PROJECT_ROOT)
)
logger.info("Search cache refresh started (pid=%s)", proc.pid)
log = TaskLog(module_id="scheduled_refresh_search_cache", task_name="🔍 搜索缓存", status="running", message="搜索缓存刷新已启动", triggered_by="manual", started_at=datetime.now(timezone.utc), result_data={"pid": proc.pid})
db.add(log)
db.commit()
log_id = log.id
t = threading.Thread(target=_monitor_subprocess, args=(log_id, proc, "scheduled_refresh_search_cache", "🔍 搜索缓存", _ss), daemon=True)
t.start()
return {"message": "搜索缓存刷新已后台启动", "pid": proc.pid, "log_id": log_id}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.post("/trends/run")
def trigger_trends_refresh(db: Session = Depends(get_db), current_user=Depends(get_current_user)):
try:
@@ -370,7 +348,6 @@ def get_modules_status(db: Session = Depends(get_db)):
config_map = {c.module_id: c for c in configs}
MODULE_META = {
"scheduled_refresh_search_cache": {"name": "🔍 搜索缓存", "cron": "01:00", "params_desc": {"refresh_queries": "搜索关键词列表"}},
"scheduled_fetch_trends": {"name": "🔥 热点趋势", "cron": "01:10", "params_desc": {}},
"scheduled_collect": {"name": "📡 内容采集", "cron": "01:30", "params_desc": {"max_topics": "最大选题数", "categories": "采集类别"}},
"scheduled_generate": {"name": "🤖 内容创作", "cron": "02:00", "params_desc": {"auto_review": "自动合规审查"}},
platform/backend/app/api/task_configs.py
+7 -8
View File
@@ -11,14 +11,13 @@ from .auth import get_current_admin
router = APIRouter(prefix="/api/admin/task-configs", tags=["admin"])
DEFAULT_CONFIGS = {
"scheduled_refresh_search_cache": {"name": "🔍 搜索缓存", "cron": "01:00", "params": {}},
"scheduled_fetch_trends": {"name": "🔥 热点趋势", "cron": "01:10", "params": {}},
"scheduled_collect": {"name": "📡 内容采集", "cron": "01:30", "params": {"max_topics": 20}},
"scheduled_generate": {"name": "🤖 内容创作", "cron": "02:00", "params": {"auto_review": True}},
"scheduled_optimize": {"name": "🔍 合规审查", "cron": "03:00", "params": {"auto_pass_threshold": 80}},
"scheduled_optimize_sources": {"name": "📡 信息源优化", "cron": "05:00", "params": {}},
"scheduled_metrics_sync": {"name": "📊 指标同步", "cron": "06:00", "params": {}},
"scheduled_task_monitor": {"name": "⏰ 任务监控", "cron": "*", "params": {}},
"scheduled_fetch_trends": {"name": "🔥 热点趋势", "cron": "01:10", "params": {"llm_provider": "nvidia"}},
"scheduled_collect": {"name": "📡 内容采集", "cron": "01:30", "params": {"max_topics": 20, "llm_provider": "nvidia"}},
"scheduled_generate": {"name": "🤖 内容创作", "cron": "02:00", "params": {"auto_review": True, "llm_provider": "nvidia"}},
"scheduled_optimize": {"name": "🔍 合规审查", "cron": "03:00", "params": {"auto_pass_threshold": 80, "llm_provider": "sensenova"}},
"scheduled_optimize_sources": {"name": "📡 信息源优化", "cron": "05:00", "params": {"llm_provider": "nvidia"}},
"scheduled_metrics_sync": {"name": "📊 指标同步", "cron": "06:00", "params": {"llm_provider": "nvidia"}},
"scheduled_task_monitor": {"name": "⏰ 任务监控", "cron": "*", "params": {"llm_provider": "nvidia"}},
}
def _attach_last_log(resp: TaskConfigResponse, db: Session, module_id: str) -> TaskConfigResponse:
platform/backend/app/api/task_logs.py
-2
View File
@@ -12,7 +12,6 @@ from .auth import get_current_admin
router = APIRouter(prefix="/api/admin/task-logs", tags=["admin"])
MODULES = {
"scheduled_refresh_search_cache": "🔍 搜索缓存",
"scheduled_fetch_trends": "🔥 热点趋势",
"scheduled_collect": "📡 内容采集",
"scheduled_generate": "🤖 内容创作",
@@ -79,7 +78,6 @@ def list_log_types(db: Session = Depends(get_db), admin_user=Depends(get_current
for mid, name in MODULES.items():
if mid in used_ids or True:
log_file_map = {
"scheduled_refresh_search_cache": "opencode_search",
"scheduled_fetch_trends": "trends",
"scheduled_collect": "collector",
"scheduled_generate": "creator",
platform/backend/app/api/tasks.py
+1 -19
View File
@@ -236,7 +236,6 @@ def _get_module_detail_data(module_id: str, db, ROOT, DATA_DIR, LOGS_DIR, today_
import json as json_mod
import re as re_mod
MODULE_META = {
"scheduled_refresh_search_cache": {"name": "🔍 搜索缓存", "description": "通过 opencode webfetch 联网搜索，刷新 8 个分类的搜索缓存，供内容采集器使用"},
"scheduled_fetch_trends": {"name": "🔥 热点趋势", "description": "从百度、微博、知乎实时热搜 API 抓取当天热点，LLM 补充，存入 trends.json"},
"scheduled_collect": {"name": "📡 内容采集", "description": "读取搜索缓存 + 热点趋势 + 历史表现 + AI 建议，经 LLM 分析后生成选题"},
"scheduled_generate": {"name": "🤖 内容创作", "description": "基于选题，LLM 生成三平台文章（知乎、微信、小红书），存入 articles 表"},
@@ -253,23 +252,7 @@ def _get_module_detail_data(module_id: str, db, ROOT, DATA_DIR, LOGS_DIR, today_
outputs = {}
history = []
if module_id == "scheduled_refresh_search_cache":
cache_file = DATA_DIR / "search_cache.json"
if cache_file.exists():
try:
cache = json_mod.loads(cache_file.read_text(encoding="utf-8"))
meta_ = cache.pop("_metadata", {})
for q, results in cache.items():
inputs.setdefault("搜索词", []).append(q)
outputs.setdefault("各分类结果", []).append({
"query": q, "count": len(results),
"samples": [r.get("title","")[:50] for r in results[:3]]
})
outputs["更新时间"] = meta_.get("updated_at", "")
outputs["结果总数"] = sum(len(v) for v in cache.values())
except Exception:
pass
# try reading queries from yaml
# try reading queries from yaml
try:
import yaml
cfg_path = ROOT / "config" / "sources.yaml"
@@ -390,7 +373,6 @@ def _get_module_detail_data(module_id: str, db, ROOT, DATA_DIR, LOGS_DIR, today_
# History from log files
log_map = {
"scheduled_refresh_search_cache": LOGS_DIR / f"opencode_search_{today_str}.log",
"scheduled_fetch_trends": LOGS_DIR / f"trends_{today_str}.log",
"scheduled_collect": LOGS_DIR / f"collector_{today_str}.log",
"scheduled_generate": LOGS_DIR / f"creator_{today_str}.log",
platform/backend/app/core/nvidia_client.py
+14 -3
View File
@@ -35,11 +35,18 @@ _FALLBACK = {
}
def _get_active_provider() -> str:
"""从 DB 读取活跃供应商，DB 不可用时回退环境变量"""
"""从 DB 读取活跃供应商，优先取 is_default=True；DB 不可用时回退环境变量"""
try:
from ..database import SessionLocal
from ..models import LLMConfig
db = SessionLocal()
# 优先取默认
default = db.query(LLMConfig).filter(
LLMConfig.is_default == True, LLMConfig.is_active == True
).first()
if default and default.provider:
db.close()
return default.provider
active = db.query(LLMConfig).filter(LLMConfig.is_active == True).first()
db.close()
if active and active.provider:
@@ -58,7 +65,7 @@ def _get_provider_config(provider: Optional[str] = None) -> dict:
from ..database import SessionLocal
from ..models import LLMConfig
db = SessionLocal()
cfg = db.query(LLMConfig).filter(LLMConfig.provider == p).order_by(LLMConfig.is_active.desc()).first()
cfg = db.query(LLMConfig).filter(LLMConfig.provider == p).order_by(LLMConfig.is_default.desc(), LLMConfig.is_active.desc()).first()
if cfg:
db_model = cfg.model
db_base_url = cfg.base_url
@@ -107,7 +114,7 @@ def _get_provider_fallback_list() -> List[str]:
return providers
except Exception:
pass
return ["opencode-go", "nvidia"]
return ["nvidia", "sensenova", "opencode-go"]
def call_llm(
prompt: str,
@@ -128,6 +135,10 @@ def call_llm(
system_prompt = system_prompt if system_prompt is not None else defaults["system_prompt"]
providers_to_try = [provider] if provider else _get_provider_fallback_list()
# LLM_TASK_PROVIDER 环境变量可覆盖任务级别的模型选择
if not provider and os.getenv("LLM_TASK_PROVIDER"):
task_provider = os.getenv("LLM_TASK_PROVIDER")
providers_to_try = [task_provider] + [p for p in providers_to_try if p != task_provider]
last_error = None
for p in providers_to_try:
try:
platform/backend/app/core/scheduler.py
+24 -44
View File
@@ -19,8 +19,25 @@ from .collector import run_collector_blocking
logger = logging.getLogger(__name__)
def _set_task_llm_provider(module_id: str):
"""从 TaskConfig 读取 llm_provider 并设为环境变量，供子进程和 call_llm 读取"""
try:
from ..database import SessionLocal
from ..models import TaskConfig
db = SessionLocal()
cfg = db.query(TaskConfig).filter(TaskConfig.module_id == module_id).first()
db.close()
if cfg and cfg.params:
provider = cfg.params.get("llm_provider")
if provider:
os.environ["LLM_TASK_PROVIDER"] = provider
logger.debug("[%s] LLM provider set to %s", module_id, provider)
return
except Exception:
pass
os.environ.pop("LLM_TASK_PROVIDER", None)
MODULES = {
"scheduled_refresh_search_cache": {"name": "🔍 搜索缓存", "cron": "01:00"},
"scheduled_fetch_trends": {"name": "🔥 热点趋势", "cron": "01:10"},
"scheduled_collect": {"name": "📡 内容采集", "cron": "01:30"},
"scheduled_generate": {"name": "🤖 内容创作", "cron": "02:00"},
@@ -32,7 +49,6 @@ MODULES = {
}
LOG_FILE_MAP = {
"scheduled_refresh_search_cache": "opencode_search",
"scheduled_fetch_trends": "trends",
"scheduled_collect": "collector",
"scheduled_generate": "creator",
@@ -149,7 +165,6 @@ class TaskScheduler:
db.close()
MODULE_JOBS = [
("scheduled_refresh_search_cache", self._run_refresh_search_cache, "搜索缓存"),
("scheduled_fetch_trends", self._run_fetch_trends, "热点趋势"),
("scheduled_collect", self._run_collect, "内容采集"),
("scheduled_generate", self._run_generate, "内容创作"),
@@ -200,6 +215,7 @@ class TaskScheduler:
def _run_fetch_trends(self):
"""定时刷新热点趋势（百度/微博/知乎实时热搜 + LLM补充）"""
_set_task_llm_provider("scheduled_fetch_trends")
started = datetime.now(timezone.utc)
log_id = _log_task("scheduled_fetch_trends", "running", started_at=started)
try:
@@ -229,48 +245,8 @@ class TaskScheduler:
started_at=started, finished_at=datetime.now(timezone.utc))
logger.exception("[Scheduled] Trends refresh error: %s", e)
def _run_refresh_search_cache(self):
"""定时刷新搜索缓存（通过 opencode webfetch）"""
started = datetime.now(timezone.utc)
log_id = _log_task("scheduled_refresh_search_cache", "running", started_at=started)
try:
logger.info("[Scheduled] Refreshing search cache via opencode...")
import subprocess
result = subprocess.run(
[sys.executable, str(PROJECT_ROOT / "scripts" / "opencode_search.py"), "--refresh-cache"],
capture_output=True, text=True, timeout=600
)
for line in result.stdout.strip().split("\n"):
if line.strip():
logger.info("[SearchCache] %s", line.strip())
for line in result.stderr.strip().split("\n"):
if line.strip():
logger.warning("[SearchCache] %s", line.strip())
if result.returncode == 0:
_log_task("scheduled_refresh_search_cache", "success", log_id=log_id,
message="搜索缓存刷新成功",
result_data={"output_lines": len(result.stdout.splitlines())},
started_at=started, finished_at=datetime.now(timezone.utc))
logger.info("[Scheduled] Search cache refreshed")
else:
_log_task("scheduled_refresh_search_cache", "failed", log_id=log_id,
message="部分失败",
error_trace=result.stderr[-500:],
started_at=started, finished_at=datetime.now(timezone.utc))
logger.warning("[Scheduled] Search cache refresh may have partial failures")
except subprocess.TimeoutExpired:
_log_task("scheduled_refresh_search_cache", "failed", log_id=log_id,
message="超时",
started_at=started, finished_at=datetime.now(timezone.utc))
logger.warning("[Scheduled] Search cache refresh timed out")
except Exception as e:
_log_task("scheduled_refresh_search_cache", "failed", log_id=log_id,
message=str(e),
error_trace=traceback.format_exc(),
started_at=started, finished_at=datetime.now(timezone.utc))
logger.exception("[Scheduled] Search cache refresh error: %s", e)
def _run_generate(self):
_set_task_llm_provider("scheduled_generate")
started = datetime.now(timezone.utc)
log_id = _log_task("scheduled_generate", "running", started_at=started)
try:
@@ -296,6 +272,7 @@ class TaskScheduler:
logger.exception("[Scheduled] Generation pipeline failed: %s", e)
def _run_optimize(self):
_set_task_llm_provider("scheduled_optimize")
started = datetime.now(timezone.utc)
log_id = _log_task("scheduled_optimize", "running", started_at=started)
try:
@@ -314,6 +291,7 @@ class TaskScheduler:
logger.exception("[Scheduled] Review failed: %s", e)
def _run_collect(self):
_set_task_llm_provider("scheduled_collect")
started = datetime.now(timezone.utc)
log_id = _log_task("scheduled_collect", "running", started_at=started)
try:
@@ -334,6 +312,7 @@ class TaskScheduler:
def _run_optimize_sources(self, triggered_by="scheduler"):
"""AI自动优化采集类别与信息源：对比市场热点和当前配置，给出调整建议"""
_set_task_llm_provider("scheduled_optimize_sources")
started = datetime.now(timezone.utc)
log_id = _log_task("scheduled_optimize_sources", "running", started_at=started, triggered_by=triggered_by)
try:
@@ -397,6 +376,7 @@ class TaskScheduler:
def _run_metrics_sync(self, triggered_by="scheduler"):
"""定时从各平台公开API获取发布文章的效果数据（当前仅支持知乎）"""
_set_task_llm_provider("scheduled_metrics_sync")
started = datetime.now(timezone.utc)
log_id = _log_task("scheduled_metrics_sync", "running", started_at=started, triggered_by=triggered_by)
try:
platform/backend/app/database.py
+1
View File
@@ -47,6 +47,7 @@ def init_db():
conn.execute(text("ALTER TABLE llm_configs ADD COLUMN IF NOT EXISTS provider VARCHAR DEFAULT 'opencode-go'"))
conn.execute(text("ALTER TABLE llm_configs ADD COLUMN IF NOT EXISTS base_url VARCHAR"))
conn.execute(text("ALTER TABLE llm_configs ADD COLUMN IF NOT EXISTS api_key VARCHAR"))
conn.execute(text("ALTER TABLE llm_configs ADD COLUMN IF NOT EXISTS is_default BOOLEAN DEFAULT FALSE"))
try:
conn.execute(text("ALTER TABLE articles ADD COLUMN IF NOT EXISTS images JSON DEFAULT '{}'::json"))
except Exception:
platform/backend/app/initial_data.py
+5 -6
View File
@@ -35,7 +35,7 @@ def import_initial_data():
db.commit()
print(f"✅ 创建默认管理员: {DEFAULT_ADMIN_USERNAME}")
# 补充或更新 LLM 供应商配置（opencode-go 为主，nvidia 为备）
# 补充或更新 LLM 供应商配置（nvidia 为主，opencode-go 为备）
expected = {
"opencode-go": dict(provider="opencode-go", model="deepseek-v4-flash",
base_url="https://opencode.ai/zen/go/v1", temperature=0.7, max_tokens=131072, is_active=True,
@@ -74,11 +74,10 @@ def import_initial_data():
# 初始化默认搜索 API 提供商
if db.query(SearchProvider).count() == 0:
providers = [
SearchProvider(name="百度千帆", provider_type="baidu", api_key="", api_url="https://qianfan.baidubce.com/v2/ai_search/web_search", console_url="https://console.bce.baidu.com/qianfan/ais/console/onlineService", priority=1, enabled=True, daily_limit=50),
SearchProvider(name="opencode云搜索", provider_type="mcp", api_key="", api_url="", console_url="https://opencode.ai", priority=2, enabled=True, daily_limit=99999),
SearchProvider(name="360搜索", provider_type="360", api_key="", api_url="", console_url="https://www.so.com", priority=3, enabled=True, daily_limit=200),
SearchProvider(name="搜狗搜索", provider_type="sogou", api_key="", api_url="", console_url="https://sogou.com", priority=4, enabled=True, daily_limit=200),
SearchProvider(name="微信搜一搜", provider_type="wechat", api_key="", api_url="", console_url="https://wx.sogou.com/weixin", priority=5, enabled=True, daily_limit=200),
SearchProvider(name="百度千帆", provider_type="baidu", api_key="", api_url="https://qianfan.baidubce.com/v2/ai_search/web_search", console_url="https://console.bce.baidu.com/qianfan/ais/console/onlineService", priority=1, enabled=True, daily_limit=200),
SearchProvider(name="360搜索", provider_type="360", api_key="", api_url="", console_url="https://www.so.com", priority=0, enabled=True, daily_limit=99999),
SearchProvider(name="搜狗搜索", provider_type="sogou", api_key="", api_url="", console_url="https://sogou.com", priority=1, enabled=True, daily_limit=99999),
SearchProvider(name="微信搜一搜", provider_type="wechat", api_key="", api_url="", console_url="https://wx.sogou.com/weixin", priority=2, enabled=True, daily_limit=99999),
]
for p in providers:
db.add(p)
platform/backend/app/models.py
+2
View File
@@ -627,6 +627,7 @@ class LLMConfig(Base):
base_url = Column(String, nullable=True)
api_key = Column(String, nullable=True)
is_active = Column(Boolean, default=True)
is_default = Column(Boolean, default=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), onupdate=func.now())
@@ -643,6 +644,7 @@ class LLMConfig(Base):
"base_url": self.base_url,
"api_key": f"{self.api_key[:8]}..." if self.api_key else None,
"is_active": self.is_active,
"is_default": self.is_default,
"created_at": self.created_at.isoformat() if self.created_at else None,
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
}
platform/backend/app/schemas.py
+1
View File
@@ -522,6 +522,7 @@ class LLMConfigBase(BaseModel):
base_url: Optional[str] = None
api_key: Optional[str] = None
is_active: bool = True
is_default: bool = False
class LLMConfigResponse(LLMConfigBase):