From 3fab87ee1174920713f1c4f50ff0d82c60370b51 Mon Sep 17 00:00:00 2001 From: Yuzhiran Dev Date: Fri, 29 May 2026 09:35:15 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E360/=E6=90=9C?= =?UTF-8?q?=E7=8B=97/=E5=BE=AE=E4=BF=A1=E6=90=9C=E7=B4=A2=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E5=95=86=20+=20PG15=E2=86=9216=E5=8D=87=E7=BA=A7=20&?= =?UTF-8?q?=20=E9=A1=B9=E7=9B=AE=E6=96=87=E6=A1=A3=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - search_utils.py: 新增 _call_360/_call_sogou/_call_wechat HTML爬取函数 - initial_data.py: 种子数据新增三个搜索提供商（priority 3/4/5） - models.py: provider_type 注释补充新类型 - admin.html: 搜索提供商类型下拉框新增三个选项 - AGENTS.md/PROGRESS.md/README.md: PostgreSQL 15→16 - README.md: 移除硬编码数据库密码 --- AGENTS.md | 2 +- PROGRESS.md | 4 +- README.md | 11 +-- platform/backend/app/initial_data.py | 3 + platform/backend/app/models.py | 2 +- platform/frontend/admin.html | 3 + scripts/search_utils.py | 107 ++++++++++++++++++++++++++- 7 files changed, 118 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c1cb98d..bc628d9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # AGENTS.md ## Stack -- **Backend**: FastAPI 0.104 + SQLAlchemy 2.0 + PostgreSQL 15 (`yzr_nr`) +- **Backend**: FastAPI 0.104 + SQLAlchemy 2.0 + PostgreSQL 16 (`yzr_nr`) - **Frontend**: Vue 3 (CDN, no build step) + Element Plus — static HTML served by FastAPI - **Auth**: JWT (`python-jose` + bcrypt), default admin `admin/admin123` - **Scheduler**: APScheduler (daily cron: 01:00 searchcache, 01:10 trends, 01:30 collect, 02:00 generate, 03:00 optimize, 05:00 sources, 06:00 metrics) diff --git a/PROGRESS.md b/PROGRESS.md index df89367..84acc2a 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -12,7 +12,7 @@ | 项目 | 内容 | |------|------| | 品牌 | 宇之然 — 科技沟通万物，愿世间百态回归自然 | -| 技术栈 | FastAPI + SQLAlchemy + PostgreSQL 15 + Vue 3 (CDN) + Element Plus | +| 技术栈 | FastAPI + SQLAlchemy + PostgreSQL 16 + Vue 3 (CDN) + Element Plus | | 平台服务 | 运行中 (端口 8001) | | 策略阶段 | 全球-本土对比研究（2026-04-15 升级） | | Git 提交 | 99 commits · 4 tags (v1.0.0~v1.0.4) · main 分支 | @@ -49,7 +49,7 @@ | 后端 API (auth/topics/articles/publishing/calendar/metrics/assets/tasks/platform_config/admin) | ✅ 完成 | 核心 11 个 API 模块，JWT 认证 | | 扩展 API (cases/audit/llm_configs/system_configs/optimizer_logs/task_logs/task_configs) | ✅ 完成 | 新增案例库、审计、LLM配置、任务配置、任务运行记录模块 | | 前端页面 (仪表盘/选题/日历/数据/素材/任务/平台/系统管理/用户/日志/文章) | ✅ 完成 | Vue 3 + Element Plus SPA | -| 数据库 (PostgreSQL 15) | ✅ 运行中 | `yzr_nr` 库 | +| 数据库 (PostgreSQL 16) | ✅ 运行中 | `yzr_nr` 库 | | 服务 | ✅ 运行中 | 端口 8001 | | 数据库迁移 (SQLite→PostgreSQL) | ✅ 完成 | 2026-05-08 | | PWA 离线支持 | ✅ 完成 | manifest.json + Service Worker + 离线页面 | diff --git a/README.md b/README.md index 28afb17..69f3a9f 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ |------|------| | 后端 | FastAPI 0.104+ | | ORM | SQLAlchemy 2.0+ | -| 数据库 | PostgreSQL 15 | +| 数据库 | PostgreSQL 16 | | 前端 | Vue 3 (CDN) + Element Plus | | 认证 | JWT (python-jose) + bcrypt | @@ -24,14 +24,7 @@ ## 数据库配置 -``` -USE_POSTGRES=true -PG_HOST=127.0.0.1 -PG_PORT=5432 -PG_DATABASE=yzr_nr -PG_USER=yzr_nr -PG_PASSWORD=aTX3WKKnPfRnM5PC -``` +详见 `platform/backend/.env` --- diff --git a/platform/backend/app/initial_data.py b/platform/backend/app/initial_data.py index fcfd9e3..ba6bfc3 100644 --- a/platform/backend/app/initial_data.py +++ b/platform/backend/app/initial_data.py @@ -76,6 +76,9 @@ def import_initial_data(): providers = [ SearchProvider(name="百度千帆", provider_type="baidu", api_key="", api_url="https://qianfan.baidubce.com/v2/ai_search/web_search", console_url="https://console.bce.baidu.com/qianfan/ais/console/onlineService", priority=1, enabled=True, daily_limit=50), SearchProvider(name="opencode云搜索", provider_type="mcp", api_key="", api_url="", console_url="https://opencode.ai", priority=2, enabled=True, daily_limit=99999), + SearchProvider(name="360搜索", provider_type="360", api_key="", api_url="", console_url="https://www.so.com", priority=3, enabled=True, daily_limit=200), + SearchProvider(name="搜狗搜索", provider_type="sogou", api_key="", api_url="", console_url="https://sogou.com", priority=4, enabled=True, daily_limit=200), + SearchProvider(name="微信搜一搜", provider_type="wechat", api_key="", api_url="", console_url="https://wx.sogou.com/weixin", priority=5, enabled=True, daily_limit=200), ] for p in providers: db.add(p) diff --git a/platform/backend/app/models.py b/platform/backend/app/models.py index 906dfba..030c13c 100644 --- a/platform/backend/app/models.py +++ b/platform/backend/app/models.py @@ -10,7 +10,7 @@ class SearchProvider(Base): id = Column(Integer, primary_key=True, index=True, autoincrement=True) name = Column(String, nullable=False, comment="显示名称") - provider_type = Column(String, nullable=False, comment="baidu / qiniu / tinyfish") + provider_type = Column(String, nullable=False, comment="baidu / qiniu / tinyfish / bing / mcp / 360 / sogou / wechat") api_key = Column(String, nullable=True, comment="API密钥") api_url = Column(String, nullable=True, comment="API地址") priority = Column(Integer, default=1, comment="优先级，越小越优先") diff --git a/platform/frontend/admin.html b/platform/frontend/admin.html index 0ebe6af..1a13d16 100644 --- a/platform/frontend/admin.html +++ b/platform/frontend/admin.html @@ -557,6 +557,9 @@ + + + diff --git a/scripts/search_utils.py b/scripts/search_utils.py index 9ee9435..5b2b389 100644 --- a/scripts/search_utils.py +++ b/scripts/search_utils.py @@ -205,12 +205,116 @@ def _call_mcp(api_key: str, api_url: str, query: str, max_results: int) -> List[ return [] +def _call_360(api_key: str, api_url: str, query: str, max_results: int) -> List[Dict]: + """360搜索（HTML爬取，无需 API Key）""" + from bs4 import BeautifulSoup + import requests + try: + resp = requests.get( + api_url or "https://www.so.com/s", + params={"q": query}, + headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}, + timeout=15, + ) + if resp.status_code != 200: + logger.warning("360搜索返回 %s", resp.status_code) + return [] + soup = BeautifulSoup(resp.text, "html.parser") + results = [] + for item in soup.select("li.res-list, li[class*=result], .rb"): + title_el = item.select_one("h3.res-title a, h3[class*=title] a, .res-title a") + if not title_el: + continue + title = title_el.get_text(strip=True)[:120] + url = title_el.get("href", "") + snippet_el = item.select_one("p.res-desc, p[class*=desc], .res-desc") + snippet = snippet_el.get_text(strip=True)[:300] if snippet_el else "" + results.append({"title": title, "url": url, "content": snippet, "source": "360"}) + if len(results) >= max_results: + break + return results + except Exception as e: + logger.warning("360搜索失败: %s", e) + return [] + + +def _call_sogou(api_key: str, api_url: str, query: str, max_results: int) -> List[Dict]: + """搜狗搜索（HTML爬取，无需 API Key）""" + from bs4 import BeautifulSoup + import requests + try: + resp = requests.get( + api_url or "https://sogou.com/web", + params={"query": query}, + headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}, + timeout=15, + ) + if resp.status_code != 200: + logger.warning("搜狗搜索返回 %s", resp.status_code) + return [] + soup = BeautifulSoup(resp.text, "html.parser") + results = [] + for item in soup.select("div.vrwrap, div[class*=vr], .rb"): + title_el = item.select_one("h3.vr-title a, h3[class*=title] a, .vr-title a") + if not title_el: + continue + title = title_el.get_text(strip=True)[:120] + url = title_el.get("href", "") + snippet_el = item.select_one("p.str-text, div.str-text, p[class*=str]") + snippet = snippet_el.get_text(strip=True)[:300] if snippet_el else "" + results.append({"title": title, "url": url, "content": snippet, "source": "sogou"}) + if len(results) >= max_results: + break + return results + except Exception as e: + logger.warning("搜狗搜索失败: %s", e) + return [] + + +def _call_wechat(api_key: str, api_url: str, query: str, max_results: int) -> List[Dict]: + """微信搜一搜（通过搜狗抓取，无需 API Key）""" + from bs4 import BeautifulSoup + import requests + try: + resp = requests.get( + api_url or "https://wx.sogou.com/weixin", + params={"type": 2, "query": query}, + headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}, + timeout=15, + ) + if resp.status_code != 200: + logger.warning("微信搜索返回 %s", resp.status_code) + return [] + soup = BeautifulSoup(resp.text, "html.parser") + results = [] + for item in soup.select("div.news-box, li.news-list, div[class*=news]"): + title_el = item.select_one("h3 a, .txt-box h3 a") + if not title_el: + continue + title = title_el.get_text(strip=True)[:120] + url = title_el.get("href", "") + if url and not url.startswith("http"): + url = "https://wx.sogou.com" + url + snippet_el = item.select_one("p.txt-info, div.txt-info, .txt-info") + snippet = snippet_el.get_text(strip=True)[:300] if snippet_el else "" + results.append({"title": title, "url": url, "content": snippet, "source": "wechat"}) + if len(results) >= max_results: + break + return results + except Exception as e: + logger.warning("微信搜索失败: %s", e) + return [] + + _PROVIDER_CALLS = { "baidu": _call_baidu, "qiniu": _call_qiniu, "tinyfish": _call_tinyfish, "bing": _call_bing, "mcp": _call_mcp, + "360": _call_360, + "sogou": _call_sogou, + "wechat": _call_wechat, } @@ -221,7 +325,8 @@ def search(query: str, max_results: int = 5) -> List[Dict]: if (p.get("usage_today") or 0) >= (p.get("daily_limit") or 99999): logger.info("提供商 %s 已达日限 %s，跳过", p.get("name"), p.get("daily_limit")) continue - if not p.get("api_key") and p.get("provider_type") != "mcp": + no_key_types = {"mcp", "360", "sogou", "wechat"} + if not p.get("api_key") and p.get("provider_type") not in no_key_types: logger.info("提供商 %s 未配置 API Key，跳过", p.get("name")) continue call_fn = _PROVIDER_CALLS.get(p.get("provider_type"))