联网搜索能力集成: opencode webfetch → 采集器
新增: - scripts/opencode_search.py: 通过 npx opencode run 调用 webfetch 联网搜索 - 搜索缓存每日 02:30 自动刷新 (scheduler) - 管理后台「搜索缓存」模块 + 立即运行按钮 - POST /api/system/refresh-search-cache/run 手动触发端点 - 8个分类搜索词从sources.yaml读取,调用AI联网搜索真实内容 机制: Python脚本 → npx opencode run → AI webfetch → 真实搜索结果 → 写入search_cache.json → 采集器读缓存 → LLM基于真实数据生成选题 不再需要API Key,不依赖任何搜索引擎,搜索结果来自AI的webfetch能力
This commit is contained in:
@@ -3,6 +3,7 @@ from fastapi import APIRouter, HTTPException, Depends, Body
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func
|
||||
from datetime import datetime, date
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Optional
|
||||
from pathlib import Path
|
||||
import os
|
||||
@@ -176,6 +177,22 @@ def trigger_metrics_sync():
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@router.post("/refresh-search-cache/run")
|
||||
def trigger_refresh_search_cache():
|
||||
"""手动刷新搜索缓存"""
|
||||
try:
|
||||
import subprocess, sys as sys_mod
|
||||
scripts_dir = Path(__file__).parent.parent.parent.parent / "scripts"
|
||||
result = subprocess.run(
|
||||
[sys_mod.executable, str(scripts_dir / "opencode_search.py"), "--refresh-cache"],
|
||||
capture_output=True, text=True, timeout=600
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise Exception(result.stderr[-500:])
|
||||
return {"message": "搜索缓存已刷新", "output": result.stdout.strip()}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@router.post("/trends/run")
|
||||
def trigger_trends_refresh():
|
||||
"""手动刷新热点趋势数据"""
|
||||
@@ -238,6 +255,7 @@ def get_modules_status():
|
||||
today_str = date.today().isoformat()
|
||||
log_based: dict = {
|
||||
"scheduled_collect": {"name": "📡 内容采集", "log": LOGS_DIR / f"collector_{today_str}.log"},
|
||||
"scheduled_refresh_search_cache": {"name": "🔍 搜索缓存", "log": LOGS_DIR / f"opencode_search_{today_str}.log"},
|
||||
"scheduled_fetch_trends": {"name": "🔥 热点趋势", "log": LOGS_DIR / f"trends_{today_str}.log"},
|
||||
"scheduled_generate": {"name": "🤖 内容创作", "log": LOGS_DIR / f"creator_{today_str}.log"},
|
||||
"scheduled_optimize": {"name": "🔍 合规审查", "log": LOGS_DIR / f"optimizer_{today_str}.log"},
|
||||
|
||||
@@ -62,6 +62,14 @@ class TaskScheduler:
|
||||
max_instances=1,
|
||||
coalesce=True
|
||||
)
|
||||
self.scheduler.add_job(
|
||||
self._run_refresh_search_cache,
|
||||
CronTrigger(hour=2, minute=30),
|
||||
id='scheduled_refresh_search_cache',
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
coalesce=True
|
||||
)
|
||||
self.scheduler.add_job(
|
||||
self._run_metrics_sync,
|
||||
CronTrigger(hour=6, minute=0),
|
||||
@@ -72,7 +80,7 @@ class TaskScheduler:
|
||||
)
|
||||
self.scheduler.start()
|
||||
self._started = True
|
||||
logger.info("Scheduler started: 01:30 collect, 03:00 trends, 03:30 generate, 04:30 review, 05:00 optimize_sources, 06:00 metrics_sync")
|
||||
logger.info("Scheduler started: 01:30 collect, 02:30 refresh_search, 03:00 trends, 03:30 generate, 04:30 review, 05:00 optimize_sources, 06:00 metrics_sync")
|
||||
def shutdown(self):
|
||||
if self.scheduler.running:
|
||||
self.scheduler.shutdown()
|
||||
@@ -97,6 +105,30 @@ class TaskScheduler:
|
||||
except Exception as e:
|
||||
logger.exception("[Scheduled] Trends refresh error: %s", e)
|
||||
|
||||
def _run_refresh_search_cache(self):
|
||||
"""定时刷新搜索缓存(通过 opencode webfetch)"""
|
||||
try:
|
||||
logger.info("[Scheduled] Refreshing search cache via opencode...")
|
||||
import subprocess
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(Path(__file__).parent.parent.parent.parent / "scripts" / "opencode_search.py"), "--refresh-cache"],
|
||||
capture_output=True, text=True, timeout=600
|
||||
)
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if line.strip():
|
||||
logger.info("[SearchCache] %s", line.strip())
|
||||
for line in result.stderr.strip().split("\n"):
|
||||
if line.strip():
|
||||
logger.warning("[SearchCache] %s", line.strip())
|
||||
if result.returncode == 0:
|
||||
logger.info("[Scheduled] Search cache refreshed")
|
||||
else:
|
||||
logger.warning("[Scheduled] Search cache refresh may have partial failures")
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("[Scheduled] Search cache refresh timed out")
|
||||
except Exception as e:
|
||||
logger.exception("[Scheduled] Search cache refresh error: %s", e)
|
||||
|
||||
def _run_generate(self):
|
||||
try:
|
||||
logger.info("[Scheduled] Starting content generation...")
|
||||
|
||||
@@ -285,6 +285,7 @@
|
||||
this.runningModule = modId;
|
||||
const endpoints = {
|
||||
scheduled_collect: '/api/system/collect/run',
|
||||
scheduled_refresh_search_cache: '/api/system/refresh-search-cache/run',
|
||||
scheduled_fetch_trends: '/api/system/trends/run',
|
||||
scheduled_generate: '/api/system/generate/run',
|
||||
scheduled_optimize: '/api/system/review/run',
|
||||
|
||||
Reference in New Issue
Block a user