chore: opencode冗余清理 + LLM任务级模型选择 + systemd服务化

- 删除 opencode_search.py / mcp_search_server.py 及所有 MCP 引用
- 移除搜索缓存定时任务(scheduled_refresh_search_cache)
- 清理前后端所有 opencode/MCP 代码和注释
- LLM 提供商量换:opencode-go→nvidia(默认)+sensenova(合规审查)
- llm_configs 新增 is_default 字段,API 层互斥逻辑
- 所有定时任务支持独立 LLM 模型选择(LLM_TASK_PROVIDER env)
- compliance_optimizer.py 修复:import os / 解硬编码 / 关键词过滤
- Scheduler 日志修复:始终 INSERT,避免僵尸 running 行
- Systemd 服务化:Restart=always / 单 worker / Type=exec
- 搜索提供商:替换 opencode→360/搜狗/微信(免 Key)
- 更新 AGENTS.md / PROGRESS.md
This commit is contained in:
Yuzhiran Dev
2026-06-02 15:38:16 +08:00
parent abbf1468bc
commit 499c511140
25 changed files with 451 additions and 678 deletions
+8 -8
View File
@@ -3,7 +3,7 @@
合规审查:文章合规检查 → LLM迭代修复
从articles表读取待审文章,进行合规评分;不合格文章由LLM修复(最多3次),通过后更新选题状态为待发布
"""
import json, datetime, logging, sys, re
import json, os, datetime, logging, sys, re
from pathlib import Path
from typing import Dict, List, Optional, Tuple
from dataclasses import dataclass, asdict
@@ -165,7 +165,7 @@ def polish_with_llm(html: str, platform: str, remaining_issues: Optional[List[Di
"""用 LLM 优化文章内容,返回 (html, log_message_or_None)
如果指定 remaining_issues,则针对性修复合规问题
LLM 失败时自动重试一次
固定使用 opencode-go (deepseek-v4-flash) — 审查用更好的模型
LLM 提供者由 LLM_TASK_PROVIDER 环境变量决定(scheduler 从 TaskConfig 读取设置)
"""
if not HAVE_LLM:
return html, None
@@ -182,16 +182,15 @@ def polish_with_llm(html: str, platform: str, remaining_issues: Optional[List[Di
prompt = get_prompt("compliance_fix", issues_desc=issues_desc, html=html)
else:
prompt = get_prompt("compliance_polish", html=html)
polished = call_llm(prompt, provider="sensenova", temperature=temperature, max_tokens=max_tokens, system_prompt=system_prompt)
polished = call_llm(prompt, temperature=temperature, max_tokens=max_tokens, system_prompt=system_prompt)
polished = clean_html_content(polished)
polished = strip_ai_preface(polished)
polished = strip_thinking_html(polished)
if '<h2' in polished or '<p>' in polished:
if len(polished) > len(html) * 0.3 and len(polished) > 100:
if not any(kw in polished for kw in ['保留', '建议', '可以', '应该', '推荐', '改为', '替换为']):
tag = "针对性修复" if remaining_issues else "常规润色"
return polished, f"LLM {tag}"
logger.warning(f"LLM 优化输出异常(过短或含建议性文字),保留原文 (len={len(polished)})")
tag = "针对性修复" if remaining_issues else "常规润色"
return polished, f"LLM {tag}"
logger.warning(f"LLM 优化输出过短,保留原文 (len={len(polished)})")
except Exception as e:
logger.warning(f"LLM 优化失败 (尝试 {attempt+1}/2): {e}")
if attempt == 0:
@@ -229,7 +228,8 @@ def _load_platform_configs() -> Dict[str, Dict]:
def main(topic_ids: List[str] = None, today_only: bool = False):
logger.info("=== 合规审查与优化开始 ===")
logger.info("LLM 配置: opencode-go (model=deepseek-v4-flash) — 固定用于合规审查")
llm_provider = os.getenv("LLM_TASK_PROVIDER", "sensenova")
logger.info(f"LLM 配置: {llm_provider} — 合规审查")
platform_configs = _load_platform_configs()
logger.info(f"已加载 {len(platform_configs)} 个平台配置")
-300
View File
@@ -1,300 +0,0 @@
#!/usr/bin/env python3
"""
MCP Search Server — provides web search via opencode infrastructure.
Two search methods (automatic fallback):
1. npx opencode run (rate-limited but returns real web results)
2. opencode-go API + model training data (no rate limit, less fresh)
Usage:
python3 scripts/mcp_search_server.py # MCP server (stdio)
python3 scripts/mcp_search_server.py --query Q # one-shot search
python3 scripts/mcp_search_server.py --url U # one-shot webfetch
"""
import json, os, subprocess, sys, time
from pathlib import Path
from typing import Any, Dict, List, Optional
PROJECT_ROOT = Path(__file__).resolve().parent.parent
CACHE_FILE = PROJECT_ROOT / "automation" / "data" / "mcp_search_cache.json"
SESSION_FILE = PROJECT_ROOT / "automation" / "data" / "mcp_session.txt"
CACHE_TTL = 3600
SESSION_TITLE = "opencode搜索"
API_BASE = "https://opencode.ai/zen/go/v1"
API_KEY = os.environ.get("OPENCODE_API_KEY", "")
if not API_KEY:
try:
from dotenv import load_dotenv
env_path = PROJECT_ROOT / "platform" / "backend" / ".env"
load_dotenv(env_path)
API_KEY = os.environ.get("OPENCODE_API_KEY", "")
except Exception:
pass
# ── session (reuse same session for all MCP searches) ─────────────
def _load_session() -> Optional[str]:
if SESSION_FILE.exists():
try:
return SESSION_FILE.read_text().strip() or None
except Exception:
pass
return None
def _save_session_from_output(stdout: str):
for line in stdout.strip().split("\n"):
try:
ev = json.loads(line)
sid = ev.get("sessionID") or ev.get("part", {}).get("sessionID")
if sid:
SESSION_FILE.parent.mkdir(parents=True, exist_ok=True)
SESSION_FILE.write_text(sid)
return
except Exception:
pass
# ── cache ─────────────────────────────────────────────────────────
def _check_cache(query: str) -> Optional[List[Dict]]:
if not CACHE_FILE.exists():
return None
try:
data = json.loads(CACHE_FILE.read_text())
entry = data.get(query)
if entry and time.time() - entry.get("ts", 0) < CACHE_TTL:
return entry.get("results")
except Exception:
pass
return None
def _write_cache(query: str, results: List[Dict]):
CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
data = {}
if CACHE_FILE.exists():
try:
data = json.loads(CACHE_FILE.read_text())
except Exception:
pass
data[query] = {"ts": time.time(), "results": results}
keys = sorted(data.keys(), key=lambda k: data[k].get("ts", 0), reverse=True)[:200]
CACHE_FILE.write_text(json.dumps({k: data[k] for k in keys}, ensure_ascii=False))
# ── method 1: npx opencode run ────────────────────────────────────
def _search_via_opencode_cli(query: str, max_results: int) -> Optional[List[Dict]]:
"""Use npx opencode run to execute websearch tool (short timeout)."""
sid = _load_session()
args = ["npx", "opencode", "run", f"websearch {query}", "--format", "json", "--title", SESSION_TITLE]
if sid:
args.extend(["--session", sid, "--continue"])
try:
r = subprocess.run(
args, capture_output=True, text=True, timeout=15,
env={**os.environ, "OPENCODE_DISABLE_AUTOUPDATE": "1"}
)
except subprocess.TimeoutExpired:
return None
except Exception:
return None
if r.returncode != 0:
return None
# Save session ID for reuse
_save_session_from_output(r.stdout)
for line in r.stdout.strip().split("\n"):
try:
ev = json.loads(line)
if ev.get("type") == "tool_use":
part = ev.get("part", {})
state = part.get("state", {})
if part.get("tool") == "websearch" and state.get("status") == "completed":
data = json.loads(state["output"])
results = []
for item in (data.get("results") or [])[:max_results]:
url = (item.get("url") or "").strip()
title = (item.get("title") or "").strip()
excerpts = item.get("excerpts") or []
content = (excerpts[0] if excerpts else "")[:500]
if url and title:
results.append({"title": title, "url": url, "content": content, "source": "opencode_cli"})
return results
except Exception:
pass
return None
# ── method 2: opencode-go API + training data ─────────────────────
def _search_via_api(query: str, max_results: int) -> Optional[List[Dict]]:
"""Use opencode-go API to answer query from training data (no rate limit)."""
if not API_KEY:
return None
import requests
try:
resp = requests.post(
f"{API_BASE}/chat/completions",
headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"},
json={
"model": "deepseek-v4-flash",
"messages": [{"role": "user", "content": (
f"你现在是一个网络搜索工具。用户查询: {query[:100]}\n\n"
f"请根据你的训练数据,提供{max_results}条最相关的网页结果,包含标题、URL和摘要。"
f"以JSON格式输出: [{{\"title\":\"...\",\"url\":\"...\",\"content\":\"...\"}}]"
f"仅输出JSON数组,不要其他文字。如果URL不确定,用合理占位。"
)}],
"temperature": 0.3,
"max_tokens": 2000,
},
timeout=30
)
data = resp.json()
content = data.get("choices", [{}])[0].get("message", {}).get("content", "")
# Extract JSON array
import re as _re
m = _re.search(r'\[.*?\]', content, _re.DOTALL)
if m:
items = json.loads(m.group())
if isinstance(items, list):
for item in items:
item["source"] = "opencode_api"
return items[:max_results]
except Exception:
pass
return None
# ── search ─────────────────────────────────────────────────────────
def web_search(query: str, max_results: int = 8) -> List[Dict]:
max_results = min(max_results, 10)
cached = _check_cache(query)
if cached:
return cached[:max_results]
results = _search_via_opencode_cli(query, max_results)
if results:
_write_cache(query, results)
return results
results = _search_via_api(query, max_results)
if results:
_write_cache(query, results)
return results
return []
def webfetch(url: str) -> Optional[str]:
sid = _load_session()
args = ["npx", "opencode", "run", f"webfetch {url}", "--format", "json", "--title", SESSION_TITLE]
if sid:
args.extend(["--session", sid, "--continue"])
try:
r = subprocess.run(
args, capture_output=True, text=True, timeout=60,
env={**os.environ, "OPENCODE_DISABLE_AUTOUPDATE": "1"}
)
if r.returncode == 0:
_save_session_from_output(r.stdout)
for line in r.stdout.strip().split("\n"):
try:
ev = json.loads(line)
if ev.get("type") == "tool_use":
p = ev.get("part", {})
s = p.get("state", {})
if p.get("tool") == "webfetch" and s.get("status") == "completed":
return s.get("output", "")[:10000]
except Exception:
pass
except Exception:
pass
return None
# ── MCP protocol (JSON-RPC 2.0 over stdio) ────────────────────────
def _read_msg() -> Optional[Dict]:
line = sys.stdin.readline()
if not line:
return None
try:
return json.loads(line)
except json.JSONDecodeError:
return None
def _send_msg(msg: Dict):
sys.stdout.write(json.dumps(msg, ensure_ascii=False) + "\n")
sys.stdout.flush()
def _send_error(req_id: Any, code: int, message: str):
_send_msg({"jsonrpc": "2.0", "id": req_id, "error": {"code": code, "message": message}})
def _send_result(req_id: Any, result: Any):
_send_msg({"jsonrpc": "2.0", "id": req_id, "result": result})
def serve():
sys.stdin.reconfigure(encoding="utf-8")
sys.stdout.reconfigure(encoding="utf-8")
while True:
msg = _read_msg()
if msg is None:
break
req_id = msg.get("id")
method = msg.get("method", "")
params = msg.get("params", {})
if method == "initialize":
_send_result(req_id, {
"protocolVersion": "2024-11-05",
"capabilities": {"tools": {"listChanged": False}},
"serverInfo": {"name": "opencode-search-mcp", "version": "1.0.0"}
})
elif method == "notifications/initialized":
pass
elif method == "tools/list":
_send_result(req_id, {"tools": [
{"name": "web_search", "description": "Search the web. Returns up to 10 results with title, url, content.", "inputSchema": {
"type": "object", "properties": {
"query": {"type": "string", "description": "Search query"},
"max_results": {"type": "number", "description": "Max results (1-10)", "default": 8}
}, "required": ["query"]
}},
{"name": "webfetch", "description": "Fetch and extract content from a URL.", "inputSchema": {
"type": "object", "properties": {"url": {"type": "string", "description": "URL to fetch"}},
"required": ["url"]
}}
]})
elif method == "tools/call":
name = params.get("name", "")
args = params.get("arguments", {})
try:
if name == "web_search":
results = web_search(args.get("query", ""), int(args.get("max_results", 8)))
_send_result(req_id, {"content": [{"type": "text", "text": json.dumps(results, ensure_ascii=False)}]})
elif name == "webfetch":
content = webfetch(args.get("url", ""))
_send_result(req_id, {"content": [{"type": "text", "text": content or "Failed to fetch URL"}]})
else:
_send_error(req_id, -32601, f"Unknown tool: {name}")
except Exception as e:
_send_error(req_id, -32603, str(e))
elif method == "shutdown":
_send_result(req_id, {})
break
else:
_send_error(req_id, -32601, f"Unknown method: {method}")
def main():
if "--query" in sys.argv:
idx = sys.argv.index("--query")
q = sys.argv[idx + 1] if idx + 1 < len(sys.argv) else ""
print(json.dumps(web_search(q), ensure_ascii=False, indent=2))
return
if "--url" in sys.argv:
idx = sys.argv.index("--url")
u = sys.argv[idx + 1] if idx + 1 < len(sys.argv) else ""
print(webfetch(u) or "Failed")
return
serve()
if __name__ == "__main__":
main()
-196
View File
@@ -1,196 +0,0 @@
#!/usr/bin/env python3
"""
通过 opencode CLI 执行联网搜索
利用 opencode 的 webfetch 能力(当前 AI 环境可无障碍访问互联网)
用法:
python3 scripts/opencode_search.py --query "可持续生活 趋势 2026"
python3 scripts/opencode_search.py --refresh-cache # 刷新所有分类的缓存
"""
import argparse, datetime, json, logging, os, re, subprocess, sys, time
from pathlib import Path
from typing import Dict, List, Optional
PROJECT_ROOT = Path(__file__).parent.parent
sys.path.insert(0, str(PROJECT_ROOT))
LOGS_DIR = PROJECT_ROOT / "automation" / "logs"
TODAY = datetime.datetime.now().strftime("%Y-%m-%d")
LOG_FILE = LOGS_DIR / f"opencode_search_{TODAY}.log"
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[logging.FileHandler(LOG_FILE, encoding='utf-8'), logging.StreamHandler()])
logger = logging.getLogger(__name__)
SEARCH_CACHE_FILE = PROJECT_ROOT / "automation" / "data" / "search_cache.json"
SESSION_FILE = PROJECT_ROOT / "automation" / "data" / "opencode_session.txt"
def _get_or_create_session() -> Optional[str]:
"""获取或创建持久 session ID"""
if SESSION_FILE.exists():
try:
sid = SESSION_FILE.read_text().strip()
if sid:
result = subprocess.run(
["npx", "opencode", "run", "ping", "--session", sid, "--format", "json"],
capture_output=True, text=True, timeout=10,
cwd=str(PROJECT_ROOT),
env={**os.environ, "OPENCODE_DISABLE_AUTOUPDATE": "1"}
)
if result.returncode == 0:
return sid
except Exception:
pass
result = subprocess.run(
["npx", "opencode", "run", "init", "--format", "json"],
capture_output=True, text=True, timeout=30,
cwd=str(PROJECT_ROOT),
env={**os.environ, "OPENCODE_DISABLE_AUTOUPDATE": "1"}
)
for line in result.stdout.strip().split("\n"):
try:
event = json.loads(line)
sid = event.get("sessionID") or event.get("part", {}).get("sessionID")
if sid:
SESSION_FILE.write_text(sid)
return sid
except Exception:
pass
return None
_session_id = None
def _run_opencode(prompt: str, timeout: int = 60) -> Optional[str]:
"""调用 opencode run 执行任务,返回文本输出"""
global _session_id
if _session_id is None:
_session_id = _get_or_create_session()
args = ["npx", "opencode", "run", prompt, "--format", "json"]
if _session_id:
args.extend(["--session", _session_id, "--continue"])
try:
result = subprocess.run(
args,
capture_output=True, text=True, timeout=timeout,
cwd=str(PROJECT_ROOT),
env={**os.environ, "OPENCODE_DISABLE_AUTOUPDATE": "1"}
)
if result.returncode != 0:
logger.warning(f"opencode run 返回非零: {result.stderr[:200]}")
return None
for line in result.stdout.strip().split("\n"):
try:
event = json.loads(line)
if event.get("type") == "error":
logger.warning(f"opencode 错误: {event}")
return None
except json.JSONDecodeError:
pass
lines = []
for line in result.stdout.strip().split("\n"):
try:
event = json.loads(line)
if event.get("type") == "text":
text = event.get("part", {}).get("text", "")
if text:
lines.append(text)
except json.JSONDecodeError:
pass
output = "\n".join(lines).strip()
return output if output else None
except subprocess.TimeoutExpired:
logger.warning(f"opencode run 超时 ({timeout}s)")
return None
except Exception as e:
logger.warning(f"opencode run 失败: {e}")
return None
def search_via_opencode(query: str, max_results: int = 5) -> List[Dict]:
"""通过 MCP 搜索工具联网搜索(替代脆弱的 npx prompt 方式)"""
try:
from search_utils import search
return search(query, max_results)
except Exception as e:
logger.warning("search_utils 不可用,回退子进程: %s", e)
result = subprocess.run(
[sys.executable, str(PROJECT_ROOT / "scripts" / "mcp_search_server.py"),
"--query", query],
capture_output=True, text=True, timeout=90,
)
if result.returncode == 0:
try:
return json.loads(result.stdout)[:max_results]
except Exception:
pass
return []
def refresh_cache():
"""刷新所有搜索分类的缓存"""
try:
with open(PROJECT_ROOT / "config" / "sources.yaml") as f:
import yaml
cfg = yaml.safe_load(f)
queries = [s["query"] for s in cfg["sustainability_sources"]["web_search"]]
except Exception:
logger.warning("无法读取 sources.yaml,使用默认查询")
queries = [
"以旧换新 二手交易 循环 2026",
"新能源车 绿色通勤 低碳 2026",
"干净饮食 有机食品 2026",
"零浪费 极简生活 可持续时尚 2026",
"绿色家电 一级能效 节能 2026",
"碳账户 碳普惠 个人碳减排 2026",
"环保科技 绿色产品 可持续材料 2026",
"AI工具 人工智能 效率提升 2026",
]
cache = {"_metadata": {"updated_at": datetime.datetime.now().isoformat()}}
if SEARCH_CACHE_FILE.exists():
try:
old = json.loads(SEARCH_CACHE_FILE.read_text(encoding="utf-8"))
for k, v in old.items():
if not k.startswith("_"):
cache.setdefault(k, v)
except Exception:
pass
for i, q in enumerate(queries):
logger.info(f"[{i+1}/{len(queries)}] 搜索: {q}")
results = search_via_opencode(q, max_results=4)
if results:
cache[q] = results
else:
logger.warning(f" {q} 搜索无结果,保留旧缓存")
time.sleep(2)
SEARCH_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
SEARCH_CACHE_FILE.write_text(json.dumps(cache, ensure_ascii=False, indent=2), encoding="utf-8")
logger.info(f"缓存已刷新: {sum(len(v) for v in cache.values())}")
def main():
parser = argparse.ArgumentParser(description="通过 opencode 联网搜索")
parser.add_argument("--query", help="搜索词")
parser.add_argument("--refresh-cache", action="store_true", help="刷新所有分类缓存")
parser.add_argument("--max-results", type=int, default=5)
args = parser.parse_args()
if args.refresh_cache:
refresh_cache()
return
if args.query:
results = search_via_opencode(args.query, args.max_results)
print(json.dumps(results, ensure_ascii=False, indent=2))
return
parser.print_help()
if __name__ == "__main__":
main()
+1 -28
View File
@@ -179,32 +179,6 @@ def _call_bing(api_key: str, api_url: str, query: str, max_results: int) -> List
} for r in items[:max_results]]
def _call_mcp(api_key: str, api_url: str, query: str, max_results: int) -> List[Dict]:
"""Call the MCP search server directly (no API key needed)."""
import subprocess
try:
r = subprocess.run(
[sys.executable, str(PROJECT_ROOT / "scripts" / "mcp_search_server.py"),
"--query", query],
capture_output=True, text=True, timeout=90,
)
if r.returncode != 0:
logger.warning("MCP搜索子进程返回非零: %s", r.stderr[:100])
return []
results = json.loads(r.stdout)
if isinstance(results, list):
for res in results:
res["source"] = "opencode"
return results[:max_results]
except json.JSONDecodeError as e:
logger.warning("MCP搜索JSON解析失败: %s", e)
except subprocess.TimeoutExpired:
logger.warning("MCP搜索超时 (90s)")
except Exception as e:
logger.warning("MCP搜索失败: %s", e)
return []
def _call_360(api_key: str, api_url: str, query: str, max_results: int) -> List[Dict]:
"""360搜索(HTML爬取,无需 API Key"""
from bs4 import BeautifulSoup
@@ -311,7 +285,6 @@ _PROVIDER_CALLS = {
"qiniu": _call_qiniu,
"tinyfish": _call_tinyfish,
"bing": _call_bing,
"mcp": _call_mcp,
"360": _call_360,
"sogou": _call_sogou,
"wechat": _call_wechat,
@@ -325,7 +298,7 @@ def search(query: str, max_results: int = 5) -> List[Dict]:
if (p.get("usage_today") or 0) >= (p.get("daily_limit") or 99999):
logger.info("提供商 %s 已达日限 %s,跳过", p.get("name"), p.get("daily_limit"))
continue
no_key_types = {"mcp", "360", "sogou", "wechat"}
no_key_types = {"360", "sogou", "wechat"}
if not p.get("api_key") and p.get("provider_type") not in no_key_types:
logger.info("提供商 %s 未配置 API Key,跳过", p.get("name"))
continue
+3 -3
View File
@@ -3,7 +3,7 @@
网络搜索模块
三种模式(优先级从高到低):
1. 本地缓存(opencode webfetch 预填充
1. 本地缓存(search_cache.json
2. Bing Web Search API(设 BING_API_KEY
3. Bing 网页抓取(服务器环境常反爬拦截)
"""
@@ -92,7 +92,7 @@ def search_scrape(query: str, max_results: int = 5) -> List[Dict]:
def search_from_cache(query: str, max_results: int = 5) -> List[Dict]:
""" opencode webfetch 预填充的缓存中读取(跳过超过36小时的缓存)"""
"""本地搜索缓存中读取(跳过超过36小时的缓存)"""
if not SEARCH_CACHE_FILE.exists():
return []
try:
@@ -114,7 +114,7 @@ def search_from_cache(query: str, max_results: int = 5) -> List[Dict]:
def save_to_cache(query: str, results: List[Dict]):
"""保存搜索结果到缓存(供 opencode webfetch 填充时使用)"""
"""保存搜索结果到缓存(供填充时使用)"""
cache = {}
if SEARCH_CACHE_FILE.exists():
try: