联网搜索能力集成: opencode webfetch → 采集器

新增:
- scripts/opencode_search.py: 通过 npx opencode run 调用 webfetch 联网搜索
- 搜索缓存每日 02:30 自动刷新 (scheduler)
- 管理后台「搜索缓存」模块 + 立即运行按钮
- POST /api/system/refresh-search-cache/run 手动触发端点
- 8个分类搜索词从sources.yaml读取,调用AI联网搜索真实内容

机制:
  Python脚本 → npx opencode run → AI webfetch → 真实搜索结果 → 写入search_cache.json
  → 采集器读缓存 → LLM基于真实数据生成选题

不再需要API Key,不依赖任何搜索引擎,搜索结果来自AI的webfetch能力
This commit is contained in:
Yuzhiran Dev
2026-05-21 09:01:02 +08:00
parent 7d50878c7d
commit 6f81167691
5 changed files with 256 additions and 47 deletions
+18
View File
@@ -3,6 +3,7 @@ from fastapi import APIRouter, HTTPException, Depends, Body
from sqlalchemy.orm import Session
from sqlalchemy import func
from datetime import datetime, date
from pathlib import Path
from typing import Dict, Any, List, Optional
from pathlib import Path
import os
@@ -176,6 +177,22 @@ def trigger_metrics_sync():
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.post("/refresh-search-cache/run")
def trigger_refresh_search_cache():
"""手动刷新搜索缓存"""
try:
import subprocess, sys as sys_mod
scripts_dir = Path(__file__).parent.parent.parent.parent / "scripts"
result = subprocess.run(
[sys_mod.executable, str(scripts_dir / "opencode_search.py"), "--refresh-cache"],
capture_output=True, text=True, timeout=600
)
if result.returncode != 0:
raise Exception(result.stderr[-500:])
return {"message": "搜索缓存已刷新", "output": result.stdout.strip()}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.post("/trends/run")
def trigger_trends_refresh():
"""手动刷新热点趋势数据"""
@@ -238,6 +255,7 @@ def get_modules_status():
today_str = date.today().isoformat()
log_based: dict = {
"scheduled_collect": {"name": "📡 内容采集", "log": LOGS_DIR / f"collector_{today_str}.log"},
"scheduled_refresh_search_cache": {"name": "🔍 搜索缓存", "log": LOGS_DIR / f"opencode_search_{today_str}.log"},
"scheduled_fetch_trends": {"name": "🔥 热点趋势", "log": LOGS_DIR / f"trends_{today_str}.log"},
"scheduled_generate": {"name": "🤖 内容创作", "log": LOGS_DIR / f"creator_{today_str}.log"},
"scheduled_optimize": {"name": "🔍 合规审查", "log": LOGS_DIR / f"optimizer_{today_str}.log"},
+33 -1
View File
@@ -62,6 +62,14 @@ class TaskScheduler:
max_instances=1,
coalesce=True
)
self.scheduler.add_job(
self._run_refresh_search_cache,
CronTrigger(hour=2, minute=30),
id='scheduled_refresh_search_cache',
replace_existing=True,
max_instances=1,
coalesce=True
)
self.scheduler.add_job(
self._run_metrics_sync,
CronTrigger(hour=6, minute=0),
@@ -72,7 +80,7 @@ class TaskScheduler:
)
self.scheduler.start()
self._started = True
logger.info("Scheduler started: 01:30 collect, 03:00 trends, 03:30 generate, 04:30 review, 05:00 optimize_sources, 06:00 metrics_sync")
logger.info("Scheduler started: 01:30 collect, 02:30 refresh_search, 03:00 trends, 03:30 generate, 04:30 review, 05:00 optimize_sources, 06:00 metrics_sync")
def shutdown(self):
if self.scheduler.running:
self.scheduler.shutdown()
@@ -97,6 +105,30 @@ class TaskScheduler:
except Exception as e:
logger.exception("[Scheduled] Trends refresh error: %s", e)
def _run_refresh_search_cache(self):
"""定时刷新搜索缓存(通过 opencode webfetch"""
try:
logger.info("[Scheduled] Refreshing search cache via opencode...")
import subprocess
result = subprocess.run(
[sys.executable, str(Path(__file__).parent.parent.parent.parent / "scripts" / "opencode_search.py"), "--refresh-cache"],
capture_output=True, text=True, timeout=600
)
for line in result.stdout.strip().split("\n"):
if line.strip():
logger.info("[SearchCache] %s", line.strip())
for line in result.stderr.strip().split("\n"):
if line.strip():
logger.warning("[SearchCache] %s", line.strip())
if result.returncode == 0:
logger.info("[Scheduled] Search cache refreshed")
else:
logger.warning("[Scheduled] Search cache refresh may have partial failures")
except subprocess.TimeoutExpired:
logger.warning("[Scheduled] Search cache refresh timed out")
except Exception as e:
logger.exception("[Scheduled] Search cache refresh error: %s", e)
def _run_generate(self):
try:
logger.info("[Scheduled] Starting content generation...")
+1
View File
@@ -285,6 +285,7 @@
this.runningModule = modId;
const endpoints = {
scheduled_collect: '/api/system/collect/run',
scheduled_refresh_search_cache: '/api/system/refresh-search-cache/run',
scheduled_fetch_trends: '/api/system/trends/run',
scheduled_generate: '/api/system/generate/run',
scheduled_optimize: '/api/system/review/run',