Yuzhiran Dev
eb1c4205fa
- Bing Web Search API支持（设BING_API_KEY环境变量即可） - 抓取回退但诚实面对反爬限制 - 搜索为空时采集器正常运行（LLM直接生成选题） - 降低搜索依赖为可选增强
107 lines
3.7 KiB
Python
107 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
网络搜索模块
|
|
|
|
有两种模式：
|
|
1. Bing Web Search API（优先）：需设置环境变量 BING_API_KEY
|
|
2. 网页搜索回退：从 cn.bing.com 抓取，但服务器环境常反爬拦截
|
|
|
|
当搜索不可用时，返回空列表。采集器已处理此情况——LLM 直接生成选题。
|
|
"""
|
|
import logging, os, re
|
|
from typing import List, Dict, Optional
|
|
from urllib.parse import quote_plus
|
|
import requests
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
|
|
|
BING_API_KEY = os.getenv("BING_API_KEY", "")
|
|
|
|
|
|
def search_api(query: str, max_results: int = 5) -> List[Dict]:
|
|
"""Bing Web Search API（需要 BING_API_KEY 环境变量）"""
|
|
if not BING_API_KEY:
|
|
return []
|
|
try:
|
|
resp = requests.get(
|
|
"https://api.bing.microsoft.com/v7.0/search",
|
|
params={"q": query, "count": max_results, "mkt": "zh-CN"},
|
|
headers={"Ocp-Apim-Subscription-Key": BING_API_KEY},
|
|
timeout=10
|
|
)
|
|
if resp.status_code != 200:
|
|
logger.warning(f"Bing API 返回 {resp.status_code}")
|
|
return []
|
|
data = resp.json()
|
|
results = []
|
|
for item in data.get("webPages", {}).get("value", [])[:max_results]:
|
|
results.append({
|
|
"title": item.get("name", "")[:120],
|
|
"url": item.get("url", ""),
|
|
"content": item.get("snippet", "")[:300],
|
|
"source": "bing_api"
|
|
})
|
|
logger.info(f"Bing API '{query[:20]}': {len(results)} 条")
|
|
return results
|
|
except Exception as e:
|
|
logger.warning(f"Bing API 失败: {e}")
|
|
return []
|
|
|
|
|
|
def search_scrape(query: str, max_results: int = 5) -> List[Dict]:
|
|
"""从 cn.bing.com 抓取搜索结果（服务器环境常遭受反爬，返回空为正常）"""
|
|
try:
|
|
resp = requests.get(
|
|
"https://cn.bing.com/search",
|
|
params={"q": query, "setlang": "zh-cn", "cc": "cn", "count": "15"},
|
|
headers={"User-Agent": UA, "Accept-Language": "zh-CN,zh;q=0.9"},
|
|
timeout=15
|
|
)
|
|
if resp.status_code != 200:
|
|
return []
|
|
|
|
html = resp.text
|
|
results = []
|
|
seen = set()
|
|
|
|
for m in re.finditer(
|
|
r'<li[^>]*class="[^"]*b_algo[^"]*"[^>]*>.*?<a[^>]*href="([^"]*)"[^>]*>(.*?)</a>',
|
|
html, re.DOTALL
|
|
):
|
|
href, title_raw = m.group(1), m.group(2)
|
|
title = re.sub(r'<[^>]+>', '', title_raw).strip()
|
|
if not title or len(title) < 8 or href in seen:
|
|
continue
|
|
if re.search(r'(bing\.com|microsoft\.com|beian\.miit|beian\.mps)', href, re.I):
|
|
continue
|
|
if re.search(r'(zdic|hanyu|hancibao|chengyu|dict\.|iciba|bishun)', href, re.I):
|
|
continue
|
|
if len(title) <= 5:
|
|
continue
|
|
seen.add(href)
|
|
results.append({"title": title[:120], "url": href, "content": "", "source": "bing"})
|
|
if len(results) >= max_results:
|
|
break
|
|
|
|
return results
|
|
except Exception as e:
|
|
logger.warning(f"Bing 抓取失败: {e}")
|
|
return []
|
|
|
|
|
|
def search(query: str, max_results: int = 5) -> List[Dict]:
|
|
"""统一搜索接口：API 优先 → 网页抓取回退"""
|
|
if BING_API_KEY:
|
|
results = search_api(query, max_results)
|
|
if results:
|
|
return results
|
|
|
|
results = search_scrape(query, max_results)
|
|
if results:
|
|
return results
|
|
|
|
logger.info(f"搜索 '{query[:20]}' 无结果（服务器环境限制，不影响采集器正常运行）")
|
|
return []
|