yuzhiran
9fba863ce7
initial_data: field descriptions humanized, collector search queries include long-tail keywords, platform formats updated collector.py: DEFAULT_CHINA_PAINS expanded with specific pain questions (e.g. '35岁学AI来得及吗') topic_selector.py: domain mapping 11->18 entries (Prompt/AI编程/AI写作/数据隐私/AI教育/一人企业) Ultraworked with Sisyphus Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
962 lines
41 KiB
Python
962 lines
41 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
内容采集管道：趋势抓取 → 分析 → 选题生成 → 存入选题库
|
||
聚焦AI与科技人文领域，从RSS搜索和Web搜索采集信息，经LLM分析后生成选题
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
import yaml
|
||
import json
|
||
import datetime
|
||
import logging
|
||
from pathlib import Path
|
||
import feedparser
|
||
import requests
|
||
import re
|
||
from typing import Dict, List, Optional, Tuple
|
||
import hashlib
|
||
from dataclasses import dataclass, asdict
|
||
import subprocess
|
||
|
||
# 项目根目录
|
||
# scripts/collector.py 位于 <project_root>/scripts/,因此向上2级即可
|
||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||
sys.path.insert(0, str(PROJECT_ROOT))
|
||
sys.path.insert(0, str(PROJECT_ROOT / "platform" / "backend"))
|
||
|
||
# 配置路径
|
||
CONFIG_DIR = PROJECT_ROOT / "config"
|
||
DATA_DIR = PROJECT_ROOT / "automation" / "data"
|
||
LOGS_DIR = PROJECT_ROOT / "automation" / "logs"
|
||
TODAY = datetime.datetime.now().strftime("%Y-%m-%d")
|
||
|
||
# 日志配置
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||
handlers=[
|
||
logging.FileHandler(LOGS_DIR / f"collector_{TODAY}.log"),
|
||
logging.StreamHandler()
|
||
]
|
||
)
|
||
logger = logging.getLogger(__name__)
|
||
DEFAULT_CHINA_PAINS = {
|
||
"AI工具实测": "AI工具选择困难、免费版功能太受限、ChatGPT不如国产好用、学习曲线陡峭、不知道哪些AI工具真正值得付费、看了测评还是不知道怎么用",
|
||
"AI前沿资讯": "AI更新太快跟不上、论文太多看不完、不知道哪些是真突破、被营销号带偏、每天刷AI新闻但感觉什么都没学到",
|
||
"未来工作方式": "AI裁员焦虑加剧、远程协作效率打折、职业方向看不清、技能更新压力大、35岁学AI来得及吗、转行AI需要什么技能",
|
||
"科技人文反思": "AI伦理边界模糊、数据隐私被侵蚀、数字依赖越来越重、人机关系困惑、孩子该不该用AI写作业、AI生成的内容能信吗",
|
||
"AI职场应用": "AI替代担心、岗位技能要求变了、职场竞争加剧、缺乏AI实操指导、怎么用AI写周报、怎么用AI做数据分析、AI提示词怎么写才有效",
|
||
}
|
||
|
||
_cached_china_pains = None
|
||
|
||
def _load_china_pains():
|
||
global _cached_china_pains
|
||
if _cached_china_pains is not None:
|
||
return _cached_china_pains
|
||
try:
|
||
from app.database import SessionLocal
|
||
from app.models import CollectorCategory
|
||
db = SessionLocal()
|
||
try:
|
||
cats = db.query(CollectorCategory).filter(
|
||
CollectorCategory.is_active == True,
|
||
CollectorCategory.pain_template.isnot(None),
|
||
CollectorCategory.pain_template != ""
|
||
).all()
|
||
if cats:
|
||
_cached_china_pains = {c.name: c.pain_template for c in cats}
|
||
logger.info(f"从DB加载 {len(_cached_china_pains)} 个类别的pain_template")
|
||
return _cached_china_pains
|
||
finally:
|
||
db.close()
|
||
except Exception as e:
|
||
logger.warning(f"从DB加载 china_pains 失败: {e}")
|
||
_cached_china_pains = DEFAULT_CHINA_PAINS
|
||
return _cached_china_pains
|
||
|
||
|
||
def _get_china_pain(category: str) -> str:
|
||
pains = _load_china_pains()
|
||
return pains.get(category, "中国相关数据不足,需本土化验证")
|
||
|
||
|
||
@dataclass
|
||
class SustainabilitySource:
|
||
"""内容采集源"""
|
||
name: str
|
||
type: str # rss, web_search, web, api, local
|
||
url: Optional[str] = None # RSS URL 或通用链接
|
||
update_frequency: str = "daily"
|
||
credibility: str = "medium"
|
||
focus: str = "AI与科技"
|
||
keywords: Optional[List[str]] = None # 源特定关键词
|
||
query: Optional[str] = None # 搜索查询词(w eb_search类型用)
|
||
|
||
@dataclass
|
||
class SustainabilityCase:
|
||
"""内容案例"""
|
||
id: str
|
||
country: str
|
||
category: str # 类别:AI前沿资讯、AI工具实测等
|
||
title: str
|
||
core_idea: str
|
||
data_facts: str
|
||
global_advantage: str
|
||
china_pain_point: str
|
||
localization_suggestion: str
|
||
mvp_action: str
|
||
source_url: str
|
||
credibility_rating: str # ⭐⭐ ⭐⭐⭐
|
||
china_applicability: str # ⭐ ⭐⭐ ⭐⭐⭐
|
||
collection_date: str
|
||
status: str = "待验证"
|
||
|
||
@dataclass
|
||
class SustainabilityTopic:
|
||
"""AI与科技选题"""
|
||
id: str
|
||
title: str
|
||
cases: List[str] # 关联的案例ID列表
|
||
audience: str # 目标受众
|
||
china_pain_points: str
|
||
localization_solution: str
|
||
mvp_actions: str
|
||
estimated_length: int
|
||
priority_score: float
|
||
field: str = "AI与效率" # 内容领域
|
||
format: str = "趋势洞察 + 实操指南" # 内容形式
|
||
core_concept: str = "" # 核心理念
|
||
audience_pain: str = "" # 受众痛点
|
||
unique_angle: str = "" # 独特角度
|
||
priority: str = "中" # 优先级:高/中/低
|
||
total_score: Optional[float] = None # 总分
|
||
compliance_score: int = 100 # 合规分数
|
||
source_file: str = "automation/data/sustainability_topics.json" # 来源文件
|
||
ready_at: Optional[str] = None # 就绪时间
|
||
published_at: Optional[str] = None # 发布时间
|
||
platform_urls: dict = None # 平台发布链接
|
||
status: str = "待处理" # 待处理/待审查/待发布/已发布
|
||
lock_by: Optional[str] = None # 被哪个任务锁定
|
||
lock_at: Optional[str] = None # 锁定时间
|
||
created_at: Optional[str] = None # 创建时间
|
||
|
||
def __post_init__(self):
|
||
if self.platform_urls is None:
|
||
self.platform_urls = {}
|
||
|
||
class SustainabilityCollector:
|
||
"""AI与科技内容采集器"""
|
||
|
||
def __init__(self):
|
||
self.load_config()
|
||
self.today_dir = DATA_DIR / "sustainability_raw" / TODAY
|
||
self.today_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
# 结果存储
|
||
self.new_cases: List[SustainabilityCase] = []
|
||
self.new_topics: List[SustainabilityTopic] = []
|
||
|
||
def load_config(self):
|
||
"""加载配置：优先从DB读取，DB为空则从YAML fallback再写入DB"""
|
||
self.config = {}
|
||
self.config_path = CONFIG_DIR / "sources.yaml"
|
||
if self.config_path.exists():
|
||
with open(self.config_path, encoding='utf-8') as f:
|
||
self.config = yaml.safe_load(f) or {}
|
||
|
||
with open(CONFIG_DIR / "wecom_config.yaml", encoding='utf-8') as f:
|
||
self.wecom_config = yaml.safe_load(f)
|
||
|
||
# 优先从DB读取类别和源
|
||
self.sources = []
|
||
try:
|
||
from app.database import SessionLocal
|
||
from app.models import CollectorCategory, CollectorSource
|
||
db = SessionLocal()
|
||
try:
|
||
cats = db.query(CollectorCategory).filter(CollectorCategory.is_active == True).order_by(CollectorCategory.sort_order).all()
|
||
if cats:
|
||
# 用DB中的类别覆盖YAML
|
||
self.config["sustainability_categories"] = [c.name for c in cats]
|
||
sources_db = db.query(CollectorSource).filter(CollectorSource.is_active == True).order_by(CollectorSource.sort_order).all()
|
||
for s in sources_db:
|
||
self.sources.append(SustainabilitySource(
|
||
name=s.name,
|
||
type=s.source_type,
|
||
url=s.url or '',
|
||
query=s.query or '',
|
||
credibility=s.credibility or 'medium',
|
||
focus=s.focus or 'AI与科技',
|
||
))
|
||
logger.info(f"从DB加载 {len(cats)} 个类别, {len(self.sources)} 个信息源")
|
||
db.close()
|
||
return
|
||
except Exception as e:
|
||
logger.warning(f"DB读取类别/源失败，回退YAML: {e}")
|
||
db.close()
|
||
except Exception as e:
|
||
logger.warning(f"DB连接失败，回退YAML: {e}")
|
||
|
||
# YAML fallback
|
||
for source_group in self.config.get("sustainability_sources", {}).values():
|
||
for source_info in source_group:
|
||
source_info = source_info.copy()
|
||
if 'base_url' in source_info and 'url' not in source_info:
|
||
source_info['url'] = source_info.pop('base_url')
|
||
source_info.setdefault('update_frequency', 'daily')
|
||
source_info.setdefault('focus', 'AI与科技')
|
||
source_info.setdefault('keywords', None)
|
||
allowed_keys = {'name', 'type', 'url', 'update_frequency', 'credibility', 'focus', 'keywords', 'query'}
|
||
filtered_info = {k: v for k, v in source_info.items() if k in allowed_keys}
|
||
self.sources.append(SustainabilitySource(**filtered_info))
|
||
|
||
logger.info(f"YAML fallback: 加载 {len(self.sources)} 个信息源")
|
||
|
||
def load_local_cases_from_db(self) -> List[SustainabilityCase]:
|
||
"""从本地案例库加载历史案例,用于降级生成选题"""
|
||
local_cases = []
|
||
db_file = DATA_DIR / "sustainability_cases.json"
|
||
if db_file.exists():
|
||
try:
|
||
with open(db_file, 'r', encoding='utf-8') as f:
|
||
cases_data = json.load(f)
|
||
# 取最近50个案例(按日期倒序)
|
||
recent_cases = cases_data[-50:] if len(cases_data) > 50 else cases_data
|
||
for case_dict in recent_cases:
|
||
# 转换为 dataclass
|
||
case = SustainabilityCase(**case_dict)
|
||
local_cases.append(case)
|
||
logger.info(f"从本地数据库加载了 {len(local_cases)} 个历史案例")
|
||
except Exception as e:
|
||
logger.error(f"读取本地案例库失败: {e}")
|
||
return local_cases
|
||
|
||
def load_local_cases_from_markdown(self) -> List[SustainabilityCase]:
|
||
"""从 Markdown 案例文件解析案例(备用)"""
|
||
local_cases = []
|
||
md_file = PROJECT_ROOT / "strategy" / "全球案例数据库-v1.md"
|
||
if not md_file.exists():
|
||
return local_cases
|
||
|
||
try:
|
||
content = md_file.read_text(encoding='utf-8')
|
||
# 简单解析:按 "#### ID:" 分割案例
|
||
import re
|
||
blocks = re.split(r'#### ID:', content)
|
||
for block in blocks[1:]: # 第一个是引言
|
||
case_data = {
|
||
'id': 'LOCAL-UNKNOWN',
|
||
'country': 'Global',
|
||
'category': '未分类',
|
||
'title': '',
|
||
'core_idea': '',
|
||
'data_facts': '',
|
||
'global_advantage': '',
|
||
'china_pain_point': '',
|
||
'localization_suggestion': '',
|
||
'mvp_action': '',
|
||
'source_url': '',
|
||
'credibility_rating': '⭐⭐',
|
||
'china_applicability': '⭐⭐',
|
||
'collection_date': TODAY
|
||
}
|
||
|
||
# 提取字段
|
||
title_match = re.search(r'(?:\*\*)?标题(?:\*\*)?[::]\s*(.+)\n', block)
|
||
if title_match:
|
||
case_data['title'] = title_match.group(1).strip()
|
||
case_data['id'] = f"LOCAL-{hashlib.md5(title_match.group(1).encode()).hexdigest()[:6].upper()}"
|
||
else:
|
||
# 尝试从 ## 标题 格式提取
|
||
title_match2 = re.search(r'\*\*(?:标题|ID[::])\*\*[::]?\s*(.+)\n', block)
|
||
if title_match2:
|
||
case_data['title'] = title_match2.group(1).strip()
|
||
case_data['id'] = f"LOCAL-{hashlib.md5(title_match2.group(1).encode()).hexdigest()[:6].upper()}"
|
||
|
||
country_match = re.search(r'(?:\*\*)?国家(?:\*\*)?[::]\s*(.+)\n', block)
|
||
if country_match:
|
||
case_data['country'] = country_match.group(1).strip()
|
||
|
||
field_match = re.search(r'(?:\*\*)?领域(?:\*\*)?[::]\s*(.+)\n', block)
|
||
if field_match:
|
||
field = field_match.group(1).strip()
|
||
# 映射到子领域(扩展映射表)
|
||
category_map = {
|
||
'远程工作方式': '循环消费',
|
||
'数字游民政策': '低碳出行',
|
||
'AI副业服务': '环保科技产品',
|
||
'一人公司模式': '循环消费',
|
||
'未来技能趋势': '干净饮食',
|
||
'可持续生活': '零浪费生活',
|
||
'零浪费生活': '零浪费生活',
|
||
'低碳出行': '低碳出行',
|
||
'循环消费': '循环消费',
|
||
'环保科技': '环保科技产品',
|
||
}
|
||
case_data['category'] = category_map.get(field, field[:4] if len(field) > 4 else field)
|
||
|
||
# 兼容新旧格式:字段可能带 **粗体**
|
||
core_match = re.search(r'(?:\*\*)?核心观点(?:\*\*)?[::]([\s\S]*?)(?=数据/事实|$)', block)
|
||
if core_match:
|
||
case_data['core_idea'] = core_match.group(1).strip()[:500]
|
||
else:
|
||
# 尝试从 - **核心观点** 格式
|
||
core_match2 = re.search(r'- \*\*核心观点\*\*[::]?\s*([\s\S]*?)(?=\n- |$)', block)
|
||
if core_match2:
|
||
case_data['core_idea'] = core_match2.group(1).strip()[:500]
|
||
|
||
data_match = re.search(r'(?:\*\*)?数据/事实(?:\*\*)?[::]([\s\S]*?)(?=全球优势|$)', block)
|
||
if data_match:
|
||
case_data['data_facts'] = data_match.group(1).strip()[:200]
|
||
|
||
global_match = re.search(r'(?:\*\*)?全球优势(?:\*\*)?[::]([\s\S]*?)(?=中国痛点|$)', block)
|
||
if global_match:
|
||
case_data['global_advantage'] = global_match.group(1).strip()[:200]
|
||
|
||
pain_match = re.search(r'(?:\*\*)?中国痛点(?:\*\*)?[::]([\s\S]*?)(?=本土化建议|$)', block)
|
||
if pain_match:
|
||
case_data['china_pain_point'] = pain_match.group(1).strip()[:200]
|
||
|
||
local_match = re.search(r'(?:\*\*)?本土化建议(?:\*\*)?[::]([\s\S]*?)(?=MVP行动|$)', block)
|
||
if local_match:
|
||
case_data['localization_suggestion'] = local_match.group(1).strip()[:200]
|
||
|
||
mvp_match = re.search(r'(?:\*\*)?MVP行动(?:\*\*)?[::]([\s\S]*?)(?=来源URL|$)', block)
|
||
if mvp_match:
|
||
case_data['mvp_action'] = mvp_match.group(1).strip()[:200]
|
||
|
||
url_match = re.search(r'(?:\*\*)?来源URL(?:\*\*)?[::]\s*(.+)\n', block)
|
||
if url_match:
|
||
case_data['source_url'] = url_match.group(1).strip()
|
||
else:
|
||
# 尝试从 URL: 或 来源URL 格式
|
||
url_match2 = re.search(r'[\*\s]*URL[::]?\s*(https?://[^\s]+)\n', block)
|
||
if url_match2:
|
||
case_data['source_url'] = url_match2.group(1).strip()
|
||
|
||
case = SustainabilityCase(**case_data)
|
||
local_cases.append(case)
|
||
|
||
logger.info(f"从 Markdown 案例库解析了 {len(local_cases)} 个案例")
|
||
except Exception as e:
|
||
logger.error(f"解析 Markdown 案例库失败: {e}")
|
||
return local_cases
|
||
|
||
def fetch_rss_feed(self, source: SustainabilitySource) -> List[Dict]:
|
||
"""获取RSS订阅内容"""
|
||
try:
|
||
feed = feedparser.parse(source.url)
|
||
articles = []
|
||
|
||
# 获取源配置的关键词(如果有)
|
||
source_keywords = getattr(source, 'keywords', None) or \
|
||
self.config.get('sustainability_sources', {}).get('global_keywords', [])
|
||
|
||
for entry in feed.entries[:15]: # 增加数量到15
|
||
title = entry.get('title', '')
|
||
content = entry.get('summary', entry.get('description', ''))
|
||
|
||
# 确保 content 不为空
|
||
if not content:
|
||
content = title
|
||
|
||
search_text = (title + content).lower()
|
||
if source_keywords:
|
||
keyword_match = any(keyword.lower() in search_text for keyword in source_keywords)
|
||
else:
|
||
keyword_match = True
|
||
if keyword_match:
|
||
articles.append({
|
||
'title': title,
|
||
'url': entry.get('link', ''),
|
||
'content': content,
|
||
'published': entry.get('published', ''),
|
||
'source_name': source.name
|
||
})
|
||
|
||
logger.info(f"从 {source.name} 获取到 {len(articles)} 篇文章")
|
||
return articles
|
||
|
||
except Exception as e:
|
||
logger.error(f"获取RSS失败 {source.name}: {e}")
|
||
return []
|
||
|
||
def fetch_web_content(self, source: SustainabilitySource) -> List[Dict]:
|
||
"""获取网页内容(简化版,实际需要更复杂的抓取)"""
|
||
logger.info(f"网页信息源 {source.name} 需要手动处理")
|
||
return []
|
||
|
||
def fetch_web_search(self, source: SustainabilitySource) -> List[Dict]:
|
||
"""通过Bing中文搜索获取实时内容"""
|
||
try:
|
||
from web_search import search
|
||
query = source.query or source.url or ''
|
||
query = query.strip()
|
||
if not query:
|
||
logger.warning(f"web_search源 {source.name} 未配置查询词")
|
||
return []
|
||
results = search(query, max_results=8)
|
||
articles = []
|
||
for r in results:
|
||
articles.append({
|
||
'title': r.get('title', ''),
|
||
'url': r.get('url', ''),
|
||
'content': r.get('snippet', ''),
|
||
'published': TODAY,
|
||
'source_name': source.name,
|
||
'search_query': query,
|
||
})
|
||
logger.info(f"搜索 [{query}] 获得 {len(articles)} 条结果")
|
||
return articles
|
||
except Exception as e:
|
||
logger.warning(f"web_search失败 {source.name}: {e}")
|
||
return []
|
||
|
||
def _get_trend_context(self) -> str:
|
||
"""读取热点趋势数据和指标反馈, 返回markdown上下文"""
|
||
parts = []
|
||
try:
|
||
from trends import load_trends
|
||
trends = load_trends()
|
||
if trends:
|
||
lines = ["## 当前热点趋势", ""]
|
||
for t in trends[:5]:
|
||
src = {"weibo": "🔥", "zhihu": "📖", "baidu": "🔍", "llm": "🤖"}.get(t.get("source", ""), "")
|
||
lines.append(f"- {src} **{t['topic']}**（{t.get('platform','')}）：{t.get('reason','')[:80]}")
|
||
kw = t.get("hot_keywords", [])
|
||
if kw:
|
||
lines.append(f" 搜索热词：{' '.join(kw[:3])}")
|
||
parts.append("\n".join(lines))
|
||
except Exception:
|
||
pass
|
||
|
||
metrics_file = DATA_DIR / "metrics_feedback.json"
|
||
if metrics_file.exists():
|
||
try:
|
||
mtime = datetime.datetime.fromtimestamp(metrics_file.stat().st_mtime)
|
||
age = (datetime.datetime.now() - mtime).total_seconds()
|
||
if age > 86400: # 超过24h的数据不采用
|
||
logger.debug("metrics_feedback 过时(%.0fh)，跳过", age / 3600)
|
||
else:
|
||
feedback = json.loads(metrics_file.read_text(encoding='utf-8'))
|
||
top_domains = feedback.get("top_domains", [])
|
||
if top_domains:
|
||
lines = ["## 历史表现反馈（高互动领域优先", ""]
|
||
for d, s in top_domains[:3]:
|
||
lines.append(f"- {d}：平均分 {s}")
|
||
parts.append("\n".join(lines))
|
||
except Exception:
|
||
pass
|
||
|
||
try:
|
||
from app.database import SessionLocal
|
||
from app.models import SystemConfig
|
||
db = SessionLocal()
|
||
try:
|
||
sc = db.query(SystemConfig).filter(SystemConfig.key == "collector_ai_advice").first()
|
||
if sc and sc.value:
|
||
advice = json.loads(sc.value)
|
||
summary = advice.get("summary", "")
|
||
new_cats = advice.get("suggested_new_categories", [])
|
||
if summary:
|
||
parts.append(f"## AI策略建议\n{summary}")
|
||
if new_cats:
|
||
suggested = [f"- {c['name']}（{c.get('reason','')[:50]}）" for c in new_cats[:2]]
|
||
parts.append("建议关注的新方向：\n" + "\n".join(suggested))
|
||
except Exception:
|
||
pass
|
||
finally:
|
||
db.close()
|
||
except Exception:
|
||
pass
|
||
|
||
return "\n\n".join(parts)
|
||
|
||
def _generate_topics_with_llm(self, cases: List[SustainabilityCase] = None, search_results: List[Dict] = None) -> List[SustainabilityTopic]:
|
||
"""用LLM基于采集数据生成选题（数据充分时精确生成，无数据时凭知识生成）"""
|
||
try:
|
||
from app.core.nvidia_client import call_llm
|
||
except ImportError:
|
||
logger.warning("LLM不可用，跳过AI选题生成")
|
||
return []
|
||
|
||
from prompt_loader import get_prompt, get_prompt_params
|
||
|
||
existing = self._get_existing_titles()
|
||
existing_hint = ""
|
||
if existing:
|
||
existing_hint = "\n已存在选题（避免重复）：" + "、".join(t[:20] for t in existing[-8:])
|
||
|
||
categories = self.config.get("sustainability_categories", ["可持续生活"])
|
||
day_idx = datetime.datetime.now().timetuple().tm_yday % len(categories)
|
||
target_category = categories[day_idx]
|
||
|
||
data_section = ""
|
||
if search_results:
|
||
summaries = [f"- {r.get('title','')}: {r.get('content','')[:100]}" for r in search_results[:5]]
|
||
data_section += "搜索结果：\n" + "\n".join(summaries) + "\n"
|
||
if cases:
|
||
case_lines = [f"- {c.title[:40]}({c.category})" for c in cases[:5]]
|
||
data_section += "\n采集案例：\n" + "\n".join(case_lines) + "\n"
|
||
if not data_section:
|
||
data_section = "（当前无实时采集数据，请基于你对中文互联网趋势的了解直接生成）"
|
||
|
||
trend_context = self._get_trend_context()
|
||
|
||
prompt = get_prompt("topic_generate",
|
||
target_category=target_category,
|
||
data_section=data_section,
|
||
existing_hint=existing_hint,
|
||
trend_context=trend_context,
|
||
)
|
||
|
||
try:
|
||
params = get_prompt_params("topic_generate")
|
||
resp = call_llm(prompt, temperature=params.get("temperature", 0.6), max_tokens=params.get("max_tokens", 2000))
|
||
resp = resp.strip()
|
||
if resp.startswith("```"):
|
||
resp = resp.split("\n", 1)[1].rsplit("\n", 1)[0]
|
||
data = json.loads(resp)
|
||
|
||
topic_id = f"TOPIC-{hashlib.md5((target_category + data.get('title','')[:10]).encode()).hexdigest()[:6].upper()}"
|
||
|
||
topic = SustainabilityTopic(
|
||
id=topic_id,
|
||
title=data.get("title", f"{target_category}新观察"),
|
||
cases=[c.id for c in (cases or [])[:3]],
|
||
audience="城市焦虑青年(26-35岁)",
|
||
china_pain_points=data.get("audience_pain", ""),
|
||
localization_solution="文章中将提供具体可执行的建议",
|
||
mvp_actions="读者可立即尝试的3个行动",
|
||
estimated_length=2000,
|
||
priority_score=7.0,
|
||
field=self.map_category_to_field(target_category),
|
||
format=data.get("format", "趋势洞察 + 实操指南"),
|
||
core_concept=data.get("core_concept", ""),
|
||
audience_pain=data.get("audience_pain", ""),
|
||
unique_angle=data.get("unique_angle", ""),
|
||
priority="中",
|
||
total_score=70.0,
|
||
compliance_score=100,
|
||
source_file="automation/data/sustainability_topics.json",
|
||
status="待处理",
|
||
lock_by=None,
|
||
lock_at=None,
|
||
created_at=datetime.datetime.now().isoformat(),
|
||
ready_at=None,
|
||
published_at=None,
|
||
platform_urls={}
|
||
)
|
||
logger.info(f"LLM生成选题: {topic.title}")
|
||
return [topic]
|
||
except Exception as e:
|
||
logger.warning(f"LLM选题生成失败: {e}")
|
||
return []
|
||
|
||
def analyze_article(self, article: Dict) -> Optional[SustainabilityCase]:
|
||
"""分析文章内容,提炼案例"""
|
||
try:
|
||
content = article['content']
|
||
title = article['title']
|
||
|
||
# 提取关键数据(简化版,实际可用NLP)
|
||
data_patterns = [
|
||
r'(\d+\.?\d*)\s*(?:percent|%|百分比)',
|
||
r'(\d+\.?\d*)\s*(?:million|billion|万|亿)',
|
||
r'(\d+\.?\d*)\s*(?:tons|tonnes|吨)',
|
||
r'(\d+\.?\d*)\s*(?:reduction|increase|减少|增加)'
|
||
]
|
||
|
||
data_points = []
|
||
for pattern in data_patterns:
|
||
matches = re.findall(pattern, content, re.IGNORECASE)
|
||
if matches:
|
||
data_points.extend(matches[:3]) # 限制数量
|
||
|
||
if len(data_points) < 2:
|
||
logger.info(f"文章数据不足: {title}")
|
||
return None
|
||
|
||
# 确定国家(简化判断)
|
||
countries = ['China', 'Japan', 'Germany', 'US', 'UK', 'Sweden', 'Netherlands']
|
||
country = 'Global' # 默认
|
||
for c in countries:
|
||
if c.lower() in content.lower():
|
||
country = c
|
||
break
|
||
|
||
# 确定子领域
|
||
categories = self.config["sustainability_categories"]
|
||
category = categories[0] # 默认第一个
|
||
for cat in categories:
|
||
if any(keyword in content.lower() for keyword in [cat.lower(), cat[:4].lower()]):
|
||
category = cat
|
||
break
|
||
|
||
# 生成案例ID
|
||
case_id = f"SUS-{hashlib.md5(title.encode()).hexdigest()[:8].upper()}"
|
||
|
||
# 提取核心观点(简化版)
|
||
# 实际应用中可用AI提取,这里用前100字符
|
||
core_idea = content[:200] if len(content) > 200 else content
|
||
|
||
# 生成中国痛点(基于类别模板，从DB读取pain_template)
|
||
china_pain = _get_china_pain(category)
|
||
|
||
# 生成案例
|
||
case = SustainabilityCase(
|
||
id=case_id,
|
||
country=country,
|
||
category=category,
|
||
title=title,
|
||
core_idea=core_idea,
|
||
data_facts=f"数据点: {', '.join(data_points[:3])}",
|
||
global_advantage="需进一步分析全球优势",
|
||
china_pain_point=china_pain,
|
||
localization_suggestion="需基于中国现实调整实施",
|
||
mvp_action="建议先小规模试点验证",
|
||
source_url=article['url'],
|
||
credibility_rating="⭐⭐" if article['source_name'] in ['GreenBiz', 'Sustainable Brands'] else "⭐",
|
||
china_applicability="⭐⭐",
|
||
collection_date=TODAY
|
||
)
|
||
|
||
return case
|
||
|
||
except Exception as e:
|
||
logger.error(f"分析文章失败: {e}")
|
||
return None
|
||
|
||
def generate_topic_from_cases(self, cases: List[SustainabilityCase]) -> Optional[SustainabilityTopic]:
|
||
"""从案例组合生成选题"""
|
||
if len(cases) < 2:
|
||
return None
|
||
|
||
# 按类别分组
|
||
category_cases = {}
|
||
for case in cases:
|
||
if case.category not in category_cases:
|
||
category_cases[case.category] = []
|
||
category_cases[case.category].append(case)
|
||
|
||
# 选择案例数最多的类别
|
||
main_category = max(category_cases, key=lambda k: len(category_cases[k]))
|
||
main_cases = category_cases[main_category]
|
||
|
||
if len(main_cases) < 2:
|
||
return None
|
||
|
||
# 组合标题（多种模板轮换，避免天天同款）
|
||
case_titles = [case.title[:30] for case in main_cases[:2]]
|
||
day_of_year = datetime.datetime.now().timetuple().tm_yday
|
||
title_templates = [
|
||
f"{main_category}新趋势: {case_titles[0]}与{case_titles[1]}的中国落地路径",
|
||
f"从{case_titles[0][:15]}到{case_titles[1][:15]}: {main_category}的中国实践指南",
|
||
f"2026{main_category}观察: {case_titles[0]}给中国什么启示",
|
||
f"实战对比: {case_titles[0][:10]}vs{case_titles[1][:10]},中国读者该学谁",
|
||
f"为什么{case_titles[0][:15]}在中国行不通(或更行)? — {main_category}深度拆解",
|
||
]
|
||
topic_title = title_templates[day_of_year % len(title_templates)]
|
||
|
||
# 生成选题ID（案例内容hash保证同一批案例产出相同ID，避免重复入库）
|
||
content_seed = main_category + case_titles[0][:10] + case_titles[1][:10]
|
||
topic_id = f"TOPIC-{hashlib.md5(content_seed.encode()).hexdigest()[:6].upper()}"
|
||
|
||
# 计算优先级分数
|
||
priority_weights = self.config["topic_priority"]
|
||
priority_score = (
|
||
priority_weights["audience_match"] * 0.8 + # 受众匹配度预估
|
||
priority_weights["data_availability"] * 0.9 + # 数据可得性
|
||
priority_weights["uniqueness"] * 0.7 + # 独特性
|
||
priority_weights["executability"] * 0.6 + # 可执行性
|
||
priority_weights["brand_fit"] * 0.9 # 品牌契合度
|
||
)
|
||
|
||
# 计算优先级分数(转换为1-10整数)
|
||
priority_weights = self.config["topic_priority"]
|
||
score_float = (
|
||
priority_weights["audience_match"] * 0.8 +
|
||
priority_weights["data_availability"] * 0.9 +
|
||
priority_weights["uniqueness"] * 0.7 +
|
||
priority_weights["executability"] * 0.6 +
|
||
priority_weights["brand_fit"] * 0.9
|
||
)
|
||
# 转换为 1-10 的整数
|
||
score = round(score_float * 10)
|
||
|
||
topic = SustainabilityTopic(
|
||
id=topic_id,
|
||
title=topic_title,
|
||
cases=[case.id for case in main_cases],
|
||
audience="城市焦虑青年(26-35岁)",
|
||
china_pain_points=f"{main_category}在中国面临的主要问题",
|
||
localization_solution="国际案例中国化适配方案",
|
||
mvp_actions="读者可立即尝试的3个行动",
|
||
estimated_length=2500,
|
||
priority_score=round(score, 2),
|
||
field=self.map_category_to_field(main_category),
|
||
format="趋势洞察 + 实操指南",
|
||
core_concept=f"基于{len(main_cases)}个{main_category}案例,提炼本土化落地策略",
|
||
audience_pain=f"{main_category}领域常见的痛点与困惑",
|
||
unique_angle=f"国际案例本土化:{case_titles[0]}与{case_titles[1]}的中国实践",
|
||
priority="中",
|
||
total_score=round(score * 10, 1) if score is not None else None,
|
||
compliance_score=100,
|
||
source_file="automation/data/sustainability_topics.json",
|
||
status="待处理",
|
||
lock_by=None,
|
||
lock_at=None,
|
||
created_at=datetime.datetime.now().isoformat(),
|
||
ready_at=None,
|
||
published_at=None,
|
||
platform_urls={}
|
||
)
|
||
|
||
return topic
|
||
|
||
def map_category_to_field(self, category: str) -> str:
|
||
"""将采集类别映射到内容领域"""
|
||
mapping = {
|
||
"AI前沿资讯": "AI与效率",
|
||
"AI工具实测": "AI与效率",
|
||
"AI趋势分析": "AI与效率",
|
||
"AI职场应用": "未来工作方式",
|
||
"科技人文反思": "科技人文",
|
||
"未来工作方式": "未来工作方式",
|
||
}
|
||
return mapping.get(category, "AI与效率")
|
||
|
||
def save_results(self):
|
||
"""保存收集结果"""
|
||
# 保存案例
|
||
cases_file = self.today_dir / "new_cases.json"
|
||
with open(cases_file, 'w', encoding='utf-8') as f:
|
||
json.dump([asdict(case) for case in self.new_cases], f, ensure_ascii=False, indent=2)
|
||
|
||
# 保存选题
|
||
topics_file = self.today_dir / "new_topics.json"
|
||
with open(topics_file, 'w', encoding='utf-8') as f:
|
||
json.dump([asdict(topic) for topic in self.new_topics], f, ensure_ascii=False, indent=2)
|
||
|
||
# 更新主数据库
|
||
self.update_main_database()
|
||
|
||
logger.info(f"保存了 {len(self.new_cases)} 个案例和 {len(self.new_topics)} 个选题")
|
||
|
||
def update_main_database(self):
|
||
"""更新主数据库和JSON备份"""
|
||
# 1. 更新案例库 (sustainability_cases.json)
|
||
main_cases_file = DATA_DIR / "sustainability_cases.json"
|
||
existing_cases = []
|
||
if main_cases_file.exists():
|
||
try:
|
||
with open(main_cases_file, 'r', encoding='utf-8') as f:
|
||
existing_cases = json.load(f)
|
||
except Exception:
|
||
existing_cases = []
|
||
all_cases = existing_cases + [asdict(case) for case in self.new_cases]
|
||
# 去重
|
||
seen = set()
|
||
unique_cases = []
|
||
for c in all_cases:
|
||
title = c.get('title', '').strip()
|
||
if title and title not in seen:
|
||
seen.add(title)
|
||
unique_cases.append(c)
|
||
unique_cases.sort(key=lambda x: x.get('collection_date', ''), reverse=True)
|
||
with open(main_cases_file, 'w', encoding='utf-8') as f:
|
||
json.dump(unique_cases[:200], f, ensure_ascii=False, indent=2)
|
||
logger.info(f"案例库更新: 总计 {len(unique_cases)} 个案例 (新增 {len(self.new_cases)})")
|
||
|
||
# 2. 更新选题数据库 (主数据源)
|
||
try:
|
||
from db_helper import save_topics_to_db
|
||
save_topics_to_db([asdict(topic) for topic in self.new_topics])
|
||
logger.info(f"选题数据库更新: 处理了 {len(self.new_topics)} 个选题")
|
||
except Exception as e:
|
||
logger.error(f"选题数据库保存失败: {e}")
|
||
|
||
# DB 是唯一数据源，不再写 JSON
|
||
|
||
def send_wecom_notification(self):
|
||
"""发送企业微信通知"""
|
||
try:
|
||
# 调用通知脚本
|
||
notification_script = PROJECT_ROOT / "scripts" / "wecom_notifier.py"
|
||
if not notification_script.exists():
|
||
logger.warning("企业微信通知脚本不存在")
|
||
return
|
||
|
||
# 准备通知数据
|
||
notification_data = {
|
||
"task": "sustainability_collection",
|
||
"time": datetime.datetime.now().strftime("%Y-%m-%d %H:%M"),
|
||
"topic_count": len(self.new_topics),
|
||
"case_count": len(self.new_cases),
|
||
"source_count": len(self.sources),
|
||
"details_link": str(self.today_dir.relative_to(PROJECT_ROOT))
|
||
}
|
||
|
||
data_file = self.today_dir / "notification_data.json"
|
||
with open(data_file, 'w', encoding='utf-8') as f:
|
||
json.dump(notification_data, f, ensure_ascii=False)
|
||
|
||
# 运行通知脚本
|
||
result = subprocess.run(
|
||
[sys.executable, str(notification_script), str(data_file)],
|
||
capture_output=True,
|
||
text=True,
|
||
cwd=PROJECT_ROOT
|
||
)
|
||
|
||
if result.returncode == 0:
|
||
logger.info("企业微信通知发送成功")
|
||
else:
|
||
logger.error(f"通知发送失败: {result.stderr}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"发送通知失败: {e}")
|
||
|
||
def _get_existing_titles(self) -> List[str]:
|
||
"""从DB获取已有的选题标题列表用于去重"""
|
||
try:
|
||
from db_helper import export_topics_to_json
|
||
topics = export_topics_to_json()
|
||
return [t.get('title', '') for t in topics]
|
||
except Exception as e:
|
||
logger.warning(f"读取已有选题失败: {e}")
|
||
return []
|
||
|
||
def _is_duplicate_topic(self, title: str, existing_titles: List[str]) -> bool:
|
||
"""检查选题是否与已有选题重复（前10字重叠即为重复）"""
|
||
prefix = title[:10].strip()
|
||
for et in existing_titles:
|
||
if prefix in et or et[:10] in title:
|
||
return True
|
||
return False
|
||
|
||
def _rotate_category(self, local_cases: List[SustainabilityCase]) -> Tuple[str, List[SustainabilityCase]]:
|
||
"""按日期轮换类别，避免天天选中同一类"""
|
||
category_cases = {}
|
||
for case in local_cases:
|
||
category_cases.setdefault(case.category, []).append(case)
|
||
|
||
if not category_cases:
|
||
return None, []
|
||
|
||
# 按类别名排序固定顺序
|
||
sorted_cats = sorted(category_cases.keys())
|
||
# 用一年中的第几天选类别，保证每天不重样
|
||
day_of_year = datetime.datetime.now().timetuple().tm_yday
|
||
idx = day_of_year % len(sorted_cats)
|
||
main_cat = sorted_cats[idx]
|
||
return main_cat, category_cases[main_cat]
|
||
|
||
def run(self):
|
||
"""主运行流程"""
|
||
logger.info("开始AI与科技内容收集")
|
||
|
||
existing_titles = self._get_existing_titles()
|
||
|
||
# ---------------------- 第一阶段：多源采集 ----------------------
|
||
all_articles = []
|
||
web_search_results = [] # 留给LLM选题用的搜索结果
|
||
|
||
for source in self.sources:
|
||
if source.type == 'rss':
|
||
articles = self.fetch_rss_feed(source)
|
||
all_articles.extend(articles)
|
||
elif source.type == 'web_search':
|
||
articles = self.fetch_web_search(source)
|
||
web_search_results.extend(articles)
|
||
all_articles.extend(articles)
|
||
elif source.type == 'local':
|
||
pass
|
||
|
||
logger.info(f"RSS采集 {sum(1 for a in all_articles if a.get('source_name','') not in [s.name for s in self.sources if s.type=='web_search'])} 篇, "
|
||
f"搜索采集 {len(web_search_results)} 篇")
|
||
|
||
# ---------------------- 第二阶段：RSS文章提炼案例 ----------------------
|
||
rss_articles = [a for a in all_articles if a not in web_search_results]
|
||
for article in rss_articles[:15]:
|
||
case = self.analyze_article(article)
|
||
if case:
|
||
self.new_cases.append(case)
|
||
|
||
# ---------------------- 第三阶段：LLM基于采集数据生成选题 ----------------------
|
||
llm_topics = self._generate_topics_with_llm(cases=self.new_cases, search_results=web_search_results)
|
||
for topic in llm_topics:
|
||
if not self._is_duplicate_topic(topic.title, existing_titles):
|
||
topic.created_at = datetime.datetime.now().isoformat()
|
||
topic.lock_by = None
|
||
topic.lock_at = None
|
||
topic.status = "待处理"
|
||
self.new_topics.append(topic)
|
||
logger.info(f"✅ LLM生成选题: {topic.title}")
|
||
|
||
# ---------------------- 第四阶段：降级策略 ----------------------
|
||
if not self.new_topics and len(self.new_cases) < 2:
|
||
logger.warning(f"LLM选题和RSS案例均不足,启动本地案例降级")
|
||
local_cases = self.load_local_cases_from_db() or self.load_local_cases_from_markdown()
|
||
|
||
if local_cases and len(local_cases) >= 2:
|
||
cat, cat_cases = self._rotate_category(local_cases)
|
||
if cat and len(cat_cases) >= 2:
|
||
selected = cat_cases[:min(4, len(cat_cases))]
|
||
self.new_cases.extend(selected)
|
||
logger.info(f"降级:从类别'{cat}'选取 {len(selected)} 个案例 (day-of-year轮换)")
|
||
else:
|
||
import random
|
||
selected = random.sample(local_cases, min(3, len(local_cases)))
|
||
self.new_cases.extend(selected)
|
||
logger.info(f"降级:随机选取 {len(selected)} 个本地案例")
|
||
|
||
# 用本地案例生成选题
|
||
if not self.new_topics and self.new_cases:
|
||
topic = self.generate_topic_from_cases(self.new_cases)
|
||
if topic:
|
||
if self._is_duplicate_topic(topic.title, existing_titles):
|
||
logger.warning(f"选题重复,跳过: {topic.title}")
|
||
else:
|
||
topic.created_at = datetime.datetime.now().isoformat()
|
||
topic.lock_by = None
|
||
topic.lock_at = None
|
||
topic.status = "待处理"
|
||
self.new_topics.append(topic)
|
||
logger.info(f"生成新选题: {topic.title}")
|
||
else:
|
||
logger.error("降级失败:本地案例库为空")
|
||
|
||
# 5. 保存结果
|
||
self.save_results()
|
||
|
||
# NOTE: 推送通知已禁用，由 publisher 统一发送最终日报
|
||
# self.send_wecom_notification()
|
||
|
||
logger.info(f"收集完成: {len(self.new_cases)} 案例, {len(self.new_topics)} 选题")
|
||
return len(self.new_cases), len(self.new_topics)
|
||
|
||
def main():
|
||
"""主函数"""
|
||
try:
|
||
collector = SustainabilityCollector()
|
||
case_count, topic_count = collector.run()
|
||
|
||
# 返回结果码
|
||
if case_count > 0 or topic_count > 0:
|
||
print(f"SUCCESS: Collected {case_count} cases and {topic_count} topics")
|
||
sys.exit(0)
|
||
else:
|
||
print("WARNING: No new content found")
|
||
sys.exit(1)
|
||
|
||
except Exception as e:
|
||
try: logger.error(f"收集任务失败: {e}")
|
||
except Exception: pass
|
||
print(f"ERROR: {e}")
|
||
sys.exit(1)
|
||
|
||
if __name__ == "__main__":
|
||
main() |