chore: opencode冗余清理 + LLM任务级模型选择 + systemd服务化
- 删除 opencode_search.py / mcp_search_server.py 及所有 MCP 引用 - 移除搜索缓存定时任务(scheduled_refresh_search_cache) - 清理前后端所有 opencode/MCP 代码和注释 - LLM 提供商量换:opencode-go→nvidia(默认)+sensenova(合规审查) - llm_configs 新增 is_default 字段,API 层互斥逻辑 - 所有定时任务支持独立 LLM 模型选择(LLM_TASK_PROVIDER env) - compliance_optimizer.py 修复:import os / 解硬编码 / 关键词过滤 - Scheduler 日志修复:始终 INSERT,避免僵尸 running 行 - Systemd 服务化:Restart=always / 单 worker / Type=exec - 搜索提供商:替换 opencode→360/搜狗/微信(免 Key) - 更新 AGENTS.md / PROGRESS.md
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
合规审查:文章合规检查 → LLM迭代修复
|
||||
从articles表读取待审文章,进行合规评分;不合格文章由LLM修复(最多3次),通过后更新选题状态为待发布
|
||||
"""
|
||||
import json, datetime, logging, sys, re
|
||||
import json, os, datetime, logging, sys, re
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from dataclasses import dataclass, asdict
|
||||
@@ -165,7 +165,7 @@ def polish_with_llm(html: str, platform: str, remaining_issues: Optional[List[Di
|
||||
"""用 LLM 优化文章内容,返回 (html, log_message_or_None)
|
||||
如果指定 remaining_issues,则针对性修复合规问题
|
||||
LLM 失败时自动重试一次
|
||||
固定使用 opencode-go (deepseek-v4-flash) — 审查用更好的模型
|
||||
LLM 提供者由 LLM_TASK_PROVIDER 环境变量决定(scheduler 从 TaskConfig 读取设置)
|
||||
"""
|
||||
if not HAVE_LLM:
|
||||
return html, None
|
||||
@@ -182,16 +182,15 @@ def polish_with_llm(html: str, platform: str, remaining_issues: Optional[List[Di
|
||||
prompt = get_prompt("compliance_fix", issues_desc=issues_desc, html=html)
|
||||
else:
|
||||
prompt = get_prompt("compliance_polish", html=html)
|
||||
polished = call_llm(prompt, provider="sensenova", temperature=temperature, max_tokens=max_tokens, system_prompt=system_prompt)
|
||||
polished = call_llm(prompt, temperature=temperature, max_tokens=max_tokens, system_prompt=system_prompt)
|
||||
polished = clean_html_content(polished)
|
||||
polished = strip_ai_preface(polished)
|
||||
polished = strip_thinking_html(polished)
|
||||
if '<h2' in polished or '<p>' in polished:
|
||||
if len(polished) > len(html) * 0.3 and len(polished) > 100:
|
||||
if not any(kw in polished for kw in ['保留', '建议', '可以', '应该', '推荐', '改为', '替换为']):
|
||||
tag = "针对性修复" if remaining_issues else "常规润色"
|
||||
return polished, f"LLM {tag}"
|
||||
logger.warning(f"LLM 优化输出异常(过短或含建议性文字),保留原文 (len={len(polished)})")
|
||||
tag = "针对性修复" if remaining_issues else "常规润色"
|
||||
return polished, f"LLM {tag}"
|
||||
logger.warning(f"LLM 优化输出过短,保留原文 (len={len(polished)})")
|
||||
except Exception as e:
|
||||
logger.warning(f"LLM 优化失败 (尝试 {attempt+1}/2): {e}")
|
||||
if attempt == 0:
|
||||
@@ -229,7 +228,8 @@ def _load_platform_configs() -> Dict[str, Dict]:
|
||||
|
||||
def main(topic_ids: List[str] = None, today_only: bool = False):
|
||||
logger.info("=== 合规审查与优化开始 ===")
|
||||
logger.info("LLM 配置: opencode-go (model=deepseek-v4-flash) — 固定用于合规审查")
|
||||
llm_provider = os.getenv("LLM_TASK_PROVIDER", "sensenova")
|
||||
logger.info(f"LLM 配置: {llm_provider} — 合规审查")
|
||||
|
||||
platform_configs = _load_platform_configs()
|
||||
logger.info(f"已加载 {len(platform_configs)} 个平台配置")
|
||||
|
||||
@@ -1,300 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MCP Search Server — provides web search via opencode infrastructure.
|
||||
|
||||
Two search methods (automatic fallback):
|
||||
1. npx opencode run (rate-limited but returns real web results)
|
||||
2. opencode-go API + model training data (no rate limit, less fresh)
|
||||
|
||||
Usage:
|
||||
python3 scripts/mcp_search_server.py # MCP server (stdio)
|
||||
python3 scripts/mcp_search_server.py --query Q # one-shot search
|
||||
python3 scripts/mcp_search_server.py --url U # one-shot webfetch
|
||||
"""
|
||||
import json, os, subprocess, sys, time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
CACHE_FILE = PROJECT_ROOT / "automation" / "data" / "mcp_search_cache.json"
|
||||
SESSION_FILE = PROJECT_ROOT / "automation" / "data" / "mcp_session.txt"
|
||||
CACHE_TTL = 3600
|
||||
SESSION_TITLE = "opencode搜索"
|
||||
|
||||
API_BASE = "https://opencode.ai/zen/go/v1"
|
||||
API_KEY = os.environ.get("OPENCODE_API_KEY", "")
|
||||
if not API_KEY:
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
env_path = PROJECT_ROOT / "platform" / "backend" / ".env"
|
||||
load_dotenv(env_path)
|
||||
API_KEY = os.environ.get("OPENCODE_API_KEY", "")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ── session (reuse same session for all MCP searches) ─────────────
|
||||
def _load_session() -> Optional[str]:
|
||||
if SESSION_FILE.exists():
|
||||
try:
|
||||
return SESSION_FILE.read_text().strip() or None
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
def _save_session_from_output(stdout: str):
|
||||
for line in stdout.strip().split("\n"):
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
sid = ev.get("sessionID") or ev.get("part", {}).get("sessionID")
|
||||
if sid:
|
||||
SESSION_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
SESSION_FILE.write_text(sid)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ── cache ─────────────────────────────────────────────────────────
|
||||
def _check_cache(query: str) -> Optional[List[Dict]]:
|
||||
if not CACHE_FILE.exists():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(CACHE_FILE.read_text())
|
||||
entry = data.get(query)
|
||||
if entry and time.time() - entry.get("ts", 0) < CACHE_TTL:
|
||||
return entry.get("results")
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
def _write_cache(query: str, results: List[Dict]):
|
||||
CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = {}
|
||||
if CACHE_FILE.exists():
|
||||
try:
|
||||
data = json.loads(CACHE_FILE.read_text())
|
||||
except Exception:
|
||||
pass
|
||||
data[query] = {"ts": time.time(), "results": results}
|
||||
keys = sorted(data.keys(), key=lambda k: data[k].get("ts", 0), reverse=True)[:200]
|
||||
CACHE_FILE.write_text(json.dumps({k: data[k] for k in keys}, ensure_ascii=False))
|
||||
|
||||
|
||||
# ── method 1: npx opencode run ────────────────────────────────────
|
||||
def _search_via_opencode_cli(query: str, max_results: int) -> Optional[List[Dict]]:
|
||||
"""Use npx opencode run to execute websearch tool (short timeout)."""
|
||||
sid = _load_session()
|
||||
args = ["npx", "opencode", "run", f"websearch {query}", "--format", "json", "--title", SESSION_TITLE]
|
||||
if sid:
|
||||
args.extend(["--session", sid, "--continue"])
|
||||
try:
|
||||
r = subprocess.run(
|
||||
args, capture_output=True, text=True, timeout=15,
|
||||
env={**os.environ, "OPENCODE_DISABLE_AUTOUPDATE": "1"}
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
if r.returncode != 0:
|
||||
return None
|
||||
# Save session ID for reuse
|
||||
_save_session_from_output(r.stdout)
|
||||
for line in r.stdout.strip().split("\n"):
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
if ev.get("type") == "tool_use":
|
||||
part = ev.get("part", {})
|
||||
state = part.get("state", {})
|
||||
if part.get("tool") == "websearch" and state.get("status") == "completed":
|
||||
data = json.loads(state["output"])
|
||||
results = []
|
||||
for item in (data.get("results") or [])[:max_results]:
|
||||
url = (item.get("url") or "").strip()
|
||||
title = (item.get("title") or "").strip()
|
||||
excerpts = item.get("excerpts") or []
|
||||
content = (excerpts[0] if excerpts else "")[:500]
|
||||
if url and title:
|
||||
results.append({"title": title, "url": url, "content": content, "source": "opencode_cli"})
|
||||
return results
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
# ── method 2: opencode-go API + training data ─────────────────────
|
||||
def _search_via_api(query: str, max_results: int) -> Optional[List[Dict]]:
|
||||
"""Use opencode-go API to answer query from training data (no rate limit)."""
|
||||
if not API_KEY:
|
||||
return None
|
||||
import requests
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{API_BASE}/chat/completions",
|
||||
headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"},
|
||||
json={
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [{"role": "user", "content": (
|
||||
f"你现在是一个网络搜索工具。用户查询: {query[:100]}\n\n"
|
||||
f"请根据你的训练数据,提供{max_results}条最相关的网页结果,包含标题、URL和摘要。"
|
||||
f"以JSON格式输出: [{{\"title\":\"...\",\"url\":\"...\",\"content\":\"...\"}}]"
|
||||
f"仅输出JSON数组,不要其他文字。如果URL不确定,用合理占位。"
|
||||
)}],
|
||||
"temperature": 0.3,
|
||||
"max_tokens": 2000,
|
||||
},
|
||||
timeout=30
|
||||
)
|
||||
data = resp.json()
|
||||
content = data.get("choices", [{}])[0].get("message", {}).get("content", "")
|
||||
# Extract JSON array
|
||||
import re as _re
|
||||
m = _re.search(r'\[.*?\]', content, _re.DOTALL)
|
||||
if m:
|
||||
items = json.loads(m.group())
|
||||
if isinstance(items, list):
|
||||
for item in items:
|
||||
item["source"] = "opencode_api"
|
||||
return items[:max_results]
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
# ── search ─────────────────────────────────────────────────────────
|
||||
def web_search(query: str, max_results: int = 8) -> List[Dict]:
|
||||
max_results = min(max_results, 10)
|
||||
cached = _check_cache(query)
|
||||
if cached:
|
||||
return cached[:max_results]
|
||||
|
||||
results = _search_via_opencode_cli(query, max_results)
|
||||
if results:
|
||||
_write_cache(query, results)
|
||||
return results
|
||||
|
||||
results = _search_via_api(query, max_results)
|
||||
if results:
|
||||
_write_cache(query, results)
|
||||
return results
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def webfetch(url: str) -> Optional[str]:
|
||||
sid = _load_session()
|
||||
args = ["npx", "opencode", "run", f"webfetch {url}", "--format", "json", "--title", SESSION_TITLE]
|
||||
if sid:
|
||||
args.extend(["--session", sid, "--continue"])
|
||||
try:
|
||||
r = subprocess.run(
|
||||
args, capture_output=True, text=True, timeout=60,
|
||||
env={**os.environ, "OPENCODE_DISABLE_AUTOUPDATE": "1"}
|
||||
)
|
||||
if r.returncode == 0:
|
||||
_save_session_from_output(r.stdout)
|
||||
for line in r.stdout.strip().split("\n"):
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
if ev.get("type") == "tool_use":
|
||||
p = ev.get("part", {})
|
||||
s = p.get("state", {})
|
||||
if p.get("tool") == "webfetch" and s.get("status") == "completed":
|
||||
return s.get("output", "")[:10000]
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
# ── MCP protocol (JSON-RPC 2.0 over stdio) ────────────────────────
|
||||
def _read_msg() -> Optional[Dict]:
|
||||
line = sys.stdin.readline()
|
||||
if not line:
|
||||
return None
|
||||
try:
|
||||
return json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
def _send_msg(msg: Dict):
|
||||
sys.stdout.write(json.dumps(msg, ensure_ascii=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
def _send_error(req_id: Any, code: int, message: str):
|
||||
_send_msg({"jsonrpc": "2.0", "id": req_id, "error": {"code": code, "message": message}})
|
||||
|
||||
def _send_result(req_id: Any, result: Any):
|
||||
_send_msg({"jsonrpc": "2.0", "id": req_id, "result": result})
|
||||
|
||||
|
||||
def serve():
|
||||
sys.stdin.reconfigure(encoding="utf-8")
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
while True:
|
||||
msg = _read_msg()
|
||||
if msg is None:
|
||||
break
|
||||
req_id = msg.get("id")
|
||||
method = msg.get("method", "")
|
||||
params = msg.get("params", {})
|
||||
if method == "initialize":
|
||||
_send_result(req_id, {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {"tools": {"listChanged": False}},
|
||||
"serverInfo": {"name": "opencode-search-mcp", "version": "1.0.0"}
|
||||
})
|
||||
elif method == "notifications/initialized":
|
||||
pass
|
||||
elif method == "tools/list":
|
||||
_send_result(req_id, {"tools": [
|
||||
{"name": "web_search", "description": "Search the web. Returns up to 10 results with title, url, content.", "inputSchema": {
|
||||
"type": "object", "properties": {
|
||||
"query": {"type": "string", "description": "Search query"},
|
||||
"max_results": {"type": "number", "description": "Max results (1-10)", "default": 8}
|
||||
}, "required": ["query"]
|
||||
}},
|
||||
{"name": "webfetch", "description": "Fetch and extract content from a URL.", "inputSchema": {
|
||||
"type": "object", "properties": {"url": {"type": "string", "description": "URL to fetch"}},
|
||||
"required": ["url"]
|
||||
}}
|
||||
]})
|
||||
elif method == "tools/call":
|
||||
name = params.get("name", "")
|
||||
args = params.get("arguments", {})
|
||||
try:
|
||||
if name == "web_search":
|
||||
results = web_search(args.get("query", ""), int(args.get("max_results", 8)))
|
||||
_send_result(req_id, {"content": [{"type": "text", "text": json.dumps(results, ensure_ascii=False)}]})
|
||||
elif name == "webfetch":
|
||||
content = webfetch(args.get("url", ""))
|
||||
_send_result(req_id, {"content": [{"type": "text", "text": content or "Failed to fetch URL"}]})
|
||||
else:
|
||||
_send_error(req_id, -32601, f"Unknown tool: {name}")
|
||||
except Exception as e:
|
||||
_send_error(req_id, -32603, str(e))
|
||||
elif method == "shutdown":
|
||||
_send_result(req_id, {})
|
||||
break
|
||||
else:
|
||||
_send_error(req_id, -32601, f"Unknown method: {method}")
|
||||
|
||||
|
||||
def main():
|
||||
if "--query" in sys.argv:
|
||||
idx = sys.argv.index("--query")
|
||||
q = sys.argv[idx + 1] if idx + 1 < len(sys.argv) else ""
|
||||
print(json.dumps(web_search(q), ensure_ascii=False, indent=2))
|
||||
return
|
||||
if "--url" in sys.argv:
|
||||
idx = sys.argv.index("--url")
|
||||
u = sys.argv[idx + 1] if idx + 1 < len(sys.argv) else ""
|
||||
print(webfetch(u) or "Failed")
|
||||
return
|
||||
serve()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,196 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
通过 opencode CLI 执行联网搜索
|
||||
利用 opencode 的 webfetch 能力(当前 AI 环境可无障碍访问互联网)
|
||||
|
||||
用法:
|
||||
python3 scripts/opencode_search.py --query "可持续生活 趋势 2026"
|
||||
python3 scripts/opencode_search.py --refresh-cache # 刷新所有分类的缓存
|
||||
"""
|
||||
import argparse, datetime, json, logging, os, re, subprocess, sys, time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
PROJECT_ROOT = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
LOGS_DIR = PROJECT_ROOT / "automation" / "logs"
|
||||
TODAY = datetime.datetime.now().strftime("%Y-%m-%d")
|
||||
LOG_FILE = LOGS_DIR / f"opencode_search_{TODAY}.log"
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[logging.FileHandler(LOG_FILE, encoding='utf-8'), logging.StreamHandler()])
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SEARCH_CACHE_FILE = PROJECT_ROOT / "automation" / "data" / "search_cache.json"
|
||||
SESSION_FILE = PROJECT_ROOT / "automation" / "data" / "opencode_session.txt"
|
||||
|
||||
|
||||
def _get_or_create_session() -> Optional[str]:
|
||||
"""获取或创建持久 session ID"""
|
||||
if SESSION_FILE.exists():
|
||||
try:
|
||||
sid = SESSION_FILE.read_text().strip()
|
||||
if sid:
|
||||
result = subprocess.run(
|
||||
["npx", "opencode", "run", "ping", "--session", sid, "--format", "json"],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
cwd=str(PROJECT_ROOT),
|
||||
env={**os.environ, "OPENCODE_DISABLE_AUTOUPDATE": "1"}
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return sid
|
||||
except Exception:
|
||||
pass
|
||||
result = subprocess.run(
|
||||
["npx", "opencode", "run", "init", "--format", "json"],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
cwd=str(PROJECT_ROOT),
|
||||
env={**os.environ, "OPENCODE_DISABLE_AUTOUPDATE": "1"}
|
||||
)
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
try:
|
||||
event = json.loads(line)
|
||||
sid = event.get("sessionID") or event.get("part", {}).get("sessionID")
|
||||
if sid:
|
||||
SESSION_FILE.write_text(sid)
|
||||
return sid
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
_session_id = None
|
||||
|
||||
|
||||
def _run_opencode(prompt: str, timeout: int = 60) -> Optional[str]:
|
||||
"""调用 opencode run 执行任务,返回文本输出"""
|
||||
global _session_id
|
||||
if _session_id is None:
|
||||
_session_id = _get_or_create_session()
|
||||
args = ["npx", "opencode", "run", prompt, "--format", "json"]
|
||||
if _session_id:
|
||||
args.extend(["--session", _session_id, "--continue"])
|
||||
try:
|
||||
result = subprocess.run(
|
||||
args,
|
||||
capture_output=True, text=True, timeout=timeout,
|
||||
cwd=str(PROJECT_ROOT),
|
||||
env={**os.environ, "OPENCODE_DISABLE_AUTOUPDATE": "1"}
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.warning(f"opencode run 返回非零: {result.stderr[:200]}")
|
||||
return None
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
try:
|
||||
event = json.loads(line)
|
||||
if event.get("type") == "error":
|
||||
logger.warning(f"opencode 错误: {event}")
|
||||
return None
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
lines = []
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
try:
|
||||
event = json.loads(line)
|
||||
if event.get("type") == "text":
|
||||
text = event.get("part", {}).get("text", "")
|
||||
if text:
|
||||
lines.append(text)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
output = "\n".join(lines).strip()
|
||||
return output if output else None
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning(f"opencode run 超时 ({timeout}s)")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning(f"opencode run 失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def search_via_opencode(query: str, max_results: int = 5) -> List[Dict]:
|
||||
"""通过 MCP 搜索工具联网搜索(替代脆弱的 npx prompt 方式)"""
|
||||
try:
|
||||
from search_utils import search
|
||||
return search(query, max_results)
|
||||
except Exception as e:
|
||||
logger.warning("search_utils 不可用,回退子进程: %s", e)
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(PROJECT_ROOT / "scripts" / "mcp_search_server.py"),
|
||||
"--query", query],
|
||||
capture_output=True, text=True, timeout=90,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
try:
|
||||
return json.loads(result.stdout)[:max_results]
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def refresh_cache():
|
||||
"""刷新所有搜索分类的缓存"""
|
||||
try:
|
||||
with open(PROJECT_ROOT / "config" / "sources.yaml") as f:
|
||||
import yaml
|
||||
cfg = yaml.safe_load(f)
|
||||
queries = [s["query"] for s in cfg["sustainability_sources"]["web_search"]]
|
||||
except Exception:
|
||||
logger.warning("无法读取 sources.yaml,使用默认查询")
|
||||
queries = [
|
||||
"以旧换新 二手交易 循环 2026",
|
||||
"新能源车 绿色通勤 低碳 2026",
|
||||
"干净饮食 有机食品 2026",
|
||||
"零浪费 极简生活 可持续时尚 2026",
|
||||
"绿色家电 一级能效 节能 2026",
|
||||
"碳账户 碳普惠 个人碳减排 2026",
|
||||
"环保科技 绿色产品 可持续材料 2026",
|
||||
"AI工具 人工智能 效率提升 2026",
|
||||
]
|
||||
|
||||
cache = {"_metadata": {"updated_at": datetime.datetime.now().isoformat()}}
|
||||
if SEARCH_CACHE_FILE.exists():
|
||||
try:
|
||||
old = json.loads(SEARCH_CACHE_FILE.read_text(encoding="utf-8"))
|
||||
for k, v in old.items():
|
||||
if not k.startswith("_"):
|
||||
cache.setdefault(k, v)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for i, q in enumerate(queries):
|
||||
logger.info(f"[{i+1}/{len(queries)}] 搜索: {q}")
|
||||
results = search_via_opencode(q, max_results=4)
|
||||
if results:
|
||||
cache[q] = results
|
||||
else:
|
||||
logger.warning(f" {q} 搜索无结果,保留旧缓存")
|
||||
time.sleep(2)
|
||||
|
||||
SEARCH_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
SEARCH_CACHE_FILE.write_text(json.dumps(cache, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
logger.info(f"缓存已刷新: {sum(len(v) for v in cache.values())} 条")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="通过 opencode 联网搜索")
|
||||
parser.add_argument("--query", help="搜索词")
|
||||
parser.add_argument("--refresh-cache", action="store_true", help="刷新所有分类缓存")
|
||||
parser.add_argument("--max-results", type=int, default=5)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.refresh_cache:
|
||||
refresh_cache()
|
||||
return
|
||||
|
||||
if args.query:
|
||||
results = search_via_opencode(args.query, args.max_results)
|
||||
print(json.dumps(results, ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
parser.print_help()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+1
-28
@@ -179,32 +179,6 @@ def _call_bing(api_key: str, api_url: str, query: str, max_results: int) -> List
|
||||
} for r in items[:max_results]]
|
||||
|
||||
|
||||
def _call_mcp(api_key: str, api_url: str, query: str, max_results: int) -> List[Dict]:
|
||||
"""Call the MCP search server directly (no API key needed)."""
|
||||
import subprocess
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[sys.executable, str(PROJECT_ROOT / "scripts" / "mcp_search_server.py"),
|
||||
"--query", query],
|
||||
capture_output=True, text=True, timeout=90,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
logger.warning("MCP搜索子进程返回非零: %s", r.stderr[:100])
|
||||
return []
|
||||
results = json.loads(r.stdout)
|
||||
if isinstance(results, list):
|
||||
for res in results:
|
||||
res["source"] = "opencode"
|
||||
return results[:max_results]
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning("MCP搜索JSON解析失败: %s", e)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("MCP搜索超时 (90s)")
|
||||
except Exception as e:
|
||||
logger.warning("MCP搜索失败: %s", e)
|
||||
return []
|
||||
|
||||
|
||||
def _call_360(api_key: str, api_url: str, query: str, max_results: int) -> List[Dict]:
|
||||
"""360搜索(HTML爬取,无需 API Key)"""
|
||||
from bs4 import BeautifulSoup
|
||||
@@ -311,7 +285,6 @@ _PROVIDER_CALLS = {
|
||||
"qiniu": _call_qiniu,
|
||||
"tinyfish": _call_tinyfish,
|
||||
"bing": _call_bing,
|
||||
"mcp": _call_mcp,
|
||||
"360": _call_360,
|
||||
"sogou": _call_sogou,
|
||||
"wechat": _call_wechat,
|
||||
@@ -325,7 +298,7 @@ def search(query: str, max_results: int = 5) -> List[Dict]:
|
||||
if (p.get("usage_today") or 0) >= (p.get("daily_limit") or 99999):
|
||||
logger.info("提供商 %s 已达日限 %s,跳过", p.get("name"), p.get("daily_limit"))
|
||||
continue
|
||||
no_key_types = {"mcp", "360", "sogou", "wechat"}
|
||||
no_key_types = {"360", "sogou", "wechat"}
|
||||
if not p.get("api_key") and p.get("provider_type") not in no_key_types:
|
||||
logger.info("提供商 %s 未配置 API Key,跳过", p.get("name"))
|
||||
continue
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
网络搜索模块
|
||||
|
||||
三种模式(优先级从高到低):
|
||||
1. 本地缓存(opencode webfetch 预填充)
|
||||
1. 本地缓存(search_cache.json)
|
||||
2. Bing Web Search API(设 BING_API_KEY)
|
||||
3. Bing 网页抓取(服务器环境常反爬拦截)
|
||||
"""
|
||||
@@ -92,7 +92,7 @@ def search_scrape(query: str, max_results: int = 5) -> List[Dict]:
|
||||
|
||||
|
||||
def search_from_cache(query: str, max_results: int = 5) -> List[Dict]:
|
||||
"""从 opencode webfetch 预填充的缓存中读取(跳过超过36小时的缓存)"""
|
||||
"""从本地搜索缓存中读取(跳过超过36小时的缓存)"""
|
||||
if not SEARCH_CACHE_FILE.exists():
|
||||
return []
|
||||
try:
|
||||
@@ -114,7 +114,7 @@ def search_from_cache(query: str, max_results: int = 5) -> List[Dict]:
|
||||
|
||||
|
||||
def save_to_cache(query: str, results: List[Dict]):
|
||||
"""保存搜索结果到缓存(供 opencode webfetch 填充时使用)"""
|
||||
"""保存搜索结果到缓存(供填充时使用)"""
|
||||
cache = {}
|
||||
if SEARCH_CACHE_FILE.exists():
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user