lt
580 lines
24 KiB
Python
580 lines
24 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
可持续性内容收集脚本
|
||
每天凌晨5:00运行，收集全球可持续性趋势信息，提炼选题和案例
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
import yaml
|
||
import json
|
||
import datetime
|
||
import logging
|
||
from pathlib import Path
|
||
import feedparser
|
||
import requests
|
||
import re
|
||
from typing import Dict, List, Optional, Tuple
|
||
import hashlib
|
||
from dataclasses import dataclass, asdict
|
||
import subprocess
|
||
|
||
# 项目根目录
|
||
PROJECT_ROOT = Path(__file__).parent.parent
|
||
sys.path.insert(0, str(PROJECT_ROOT))
|
||
|
||
# 配置路径
|
||
CONFIG_DIR = PROJECT_ROOT / "config"
|
||
DATA_DIR = PROJECT_ROOT / "automation" / "data"
|
||
LOGS_DIR = PROJECT_ROOT / "automation" / "logs"
|
||
TODAY = datetime.datetime.now().strftime("%Y-%m-%d")
|
||
|
||
# 日志配置
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||
handlers=[
|
||
logging.FileHandler(LOGS_DIR / f"collector_{TODAY}.log"),
|
||
logging.StreamHandler()
|
||
]
|
||
)
|
||
logger = logging.getLogger(__name__)
|
||
|
||
@dataclass
|
||
class SustainabilitySource:
|
||
"""可持续性信息源"""
|
||
name: str
|
||
type: str # rss, web, api, report, local
|
||
url: Optional[str] = None # 可为空（如本地源）
|
||
update_frequency: str = "daily"
|
||
credibility: str = "medium"
|
||
focus: str = "可持续性"
|
||
|
||
@dataclass
|
||
class SustainabilityCase:
|
||
"""可持续性案例"""
|
||
id: str
|
||
country: str
|
||
category: str # 子领域：城市农业、零浪费生活等
|
||
title: str
|
||
core_idea: str
|
||
data_facts: str
|
||
global_advantage: str
|
||
china_pain_point: str
|
||
localization_suggestion: str
|
||
mvp_action: str
|
||
source_url: str
|
||
credibility_rating: str # ⭐⭐ ⭐⭐⭐
|
||
china_applicability: str # ⭐ ⭐⭐ ⭐⭐⭐
|
||
collection_date: str
|
||
status: str = "待验证"
|
||
|
||
@dataclass
|
||
class SustainabilityTopic:
|
||
"""可持续性选题"""
|
||
id: str
|
||
title: str
|
||
cases: List[str] # 关联的案例ID列表
|
||
audience: str # 目标受众
|
||
china_pain_points: str
|
||
localization_solution: str
|
||
mvp_actions: str
|
||
estimated_length: int
|
||
priority_score: float
|
||
status: str = "待处理" # 待处理/待审查/待发布/已发布
|
||
lock_by: Optional[str] = None # 被哪个任务锁定
|
||
lock_at: Optional[str] = None # 锁定时间
|
||
created_at: Optional[str] = None # 创建时间
|
||
|
||
class SustainabilityCollector:
|
||
"""可持续性内容收集器"""
|
||
|
||
def __init__(self):
|
||
self.load_config()
|
||
self.today_dir = DATA_DIR / "sustainability_raw" / TODAY
|
||
self.today_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
# 结果存储
|
||
self.new_cases: List[SustainabilityCase] = []
|
||
self.new_topics: List[SustainabilityTopic] = []
|
||
|
||
def load_config(self):
|
||
"""加载配置文件"""
|
||
with open(CONFIG_DIR / "sources.yaml", "r", encoding='utf-8') as f:
|
||
self.config = yaml.safe_load(f)
|
||
|
||
with open(CONFIG_DIR / "wecom_config.yaml", "r", encoding='utf-8') as f:
|
||
self.wecom_config = yaml.safe_load(f)
|
||
|
||
self.sources = []
|
||
for source_group in self.config["sustainability_sources"].values():
|
||
for source_info in source_group:
|
||
# Handle both 'url' and 'base_url' in config
|
||
source_info = source_info.copy()
|
||
if 'base_url' in source_info and 'url' not in source_info:
|
||
source_info['url'] = source_info.pop('base_url')
|
||
# Provide defaults for missing optional fields
|
||
source_info.setdefault('update_frequency', 'daily')
|
||
source_info.setdefault('focus', '可持续性')
|
||
# Filter to only fields accepted by SustainabilitySource
|
||
allowed_keys = {'name', 'type', 'url', 'update_frequency', 'credibility', 'focus'}
|
||
filtered_info = {k: v for k, v in source_info.items() if k in allowed_keys}
|
||
self.sources.append(SustainabilitySource(**filtered_info))
|
||
|
||
logger.info(f"加载了 {len(self.sources)} 个信息源")
|
||
|
||
def load_local_cases_from_db(self) -> List[SustainabilityCase]:
|
||
"""从本地案例库加载历史案例，用于降级生成选题"""
|
||
local_cases = []
|
||
db_file = DATA_DIR / "sustainability_cases.json"
|
||
if db_file.exists():
|
||
try:
|
||
with open(db_file, 'r', encoding='utf-8') as f:
|
||
cases_data = json.load(f)
|
||
# 取最近50个案例（按日期倒序）
|
||
recent_cases = cases_data[-50:] if len(cases_data) > 50 else cases_data
|
||
for case_dict in recent_cases:
|
||
# 转换为 dataclass
|
||
case = SustainabilityCase(**case_dict)
|
||
local_cases.append(case)
|
||
logger.info(f"从本地数据库加载了 {len(local_cases)} 个历史案例")
|
||
except Exception as e:
|
||
logger.error(f"读取本地案例库失败: {e}")
|
||
return local_cases
|
||
|
||
def load_local_cases_from_markdown(self) -> List[SustainabilityCase]:
|
||
"""从 Markdown 案例文件解析案例（备用）"""
|
||
local_cases = []
|
||
md_file = PROJECT_ROOT / "strategy" / "全球案例数据库-v1.md"
|
||
if not md_file.exists():
|
||
return local_cases
|
||
|
||
try:
|
||
content = md_file.read_text(encoding='utf-8')
|
||
# 简单解析：按 "#### ID:" 分割案例
|
||
import re
|
||
blocks = re.split(r'#### ID:', content)
|
||
for block in blocks[1:]: # 第一个是引言
|
||
case_data = {
|
||
'id': 'LOCAL-UNKNOWN',
|
||
'country': 'Global',
|
||
'category': '未分类',
|
||
'title': '',
|
||
'core_idea': '',
|
||
'data_facts': '',
|
||
'global_advantage': '',
|
||
'china_pain_point': '',
|
||
'localization_suggestion': '',
|
||
'mvp_action': '',
|
||
'source_url': '',
|
||
'credibility_rating': '⭐⭐',
|
||
'china_applicability': '⭐⭐',
|
||
'collection_date': TODAY
|
||
}
|
||
|
||
# 提取字段
|
||
title_match = re.search(r'标题[：:]\s*(.+)\n', block)
|
||
if title_match:
|
||
case_data['title'] = title_match.group(1).strip()
|
||
case_data['id'] = f"LOCAL-{hashlib.md5(title_match.group(1).encode()).hexdigest()[:6].upper()}"
|
||
|
||
country_match = re.search(r'国家[：:]\s*(.+)\n', block)
|
||
if country_match:
|
||
case_data['country'] = country_match.group(1).strip()
|
||
|
||
field_match = re.search(r'领域[：:]\s*(.+)\n', block)
|
||
if field_match:
|
||
field = field_match.group(1).strip()
|
||
# 映射到子领域
|
||
category_map = {
|
||
'远程工作方式': '城市农业',
|
||
'数字游民政策': '低碳出行',
|
||
'AI副业服务': '环保科技产品',
|
||
'一人公司模式': '循环消费',
|
||
'未来技能趋势': '可持续饮食'
|
||
}
|
||
case_data['category'] = category_map.get(field, field[:4] if len(field) > 4 else field)
|
||
|
||
core_match = re.search(r'核心观点[：:]([\s\S]*?)(?=数据/事实|$)', block)
|
||
if core_match:
|
||
case_data['core_idea'] = core_match.group(1).strip()[:500]
|
||
|
||
data_match = re.search(r'数据/事实[：:]([\s\S]*?)(?=全球优势|$)', block)
|
||
if data_match:
|
||
case_data['data_facts'] = data_match.group(1).strip()[:200]
|
||
|
||
global_match = re.search(r'全球优势[：:]([\s\S]*?)(?=中国痛点|$)', block)
|
||
if global_match:
|
||
case_data['global_advantage'] = global_match.group(1).strip()[:200]
|
||
|
||
pain_match = re.search(r'中国痛点[：:]([\s\S]*?)(?=本土化建议|$)', block)
|
||
if pain_match:
|
||
case_data['china_pain_point'] = pain_match.group(1).strip()[:200]
|
||
|
||
local_match = re.search(r'本土化建议[：:]([\s\S]*?)(?=MVP行动|$)', block)
|
||
if local_match:
|
||
case_data['localization_suggestion'] = local_match.group(1).strip()[:200]
|
||
|
||
mvp_match = re.search(r'MVP行动[：:]([\s\S]*?)(?=来源URL|$)', block)
|
||
if mvp_match:
|
||
case_data['mvp_action'] = mvp_match.group(1).strip()[:200]
|
||
|
||
url_match = re.search(r'来源URL[：:]\s*(.+)\n', block)
|
||
if url_match:
|
||
case_data['source_url'] = url_match.group(1).strip()
|
||
|
||
case = SustainabilityCase(**case_data)
|
||
local_cases.append(case)
|
||
|
||
logger.info(f"从 Markdown 案例库解析了 {len(local_cases)} 个案例")
|
||
except Exception as e:
|
||
logger.error(f"解析 Markdown 案例库失败: {e}")
|
||
return local_cases
|
||
|
||
def fetch_rss_feed(self, source: SustainabilitySource) -> List[Dict]:
|
||
"""获取RSS订阅内容"""
|
||
try:
|
||
feed = feedparser.parse(source.url)
|
||
articles = []
|
||
|
||
for entry in feed.entries[:10]: # 限制数量
|
||
# 检查是否包含可持续性关键词
|
||
content = entry.get('summary', entry.get('description', ''))
|
||
title = entry.get('title', '')
|
||
|
||
# 可持续性关键词匹配
|
||
sustainability_keywords = [
|
||
'sustainable', 'green', 'eco', 'circular', 'climate',
|
||
'carbon', 'zero waste', 'renewable', 'recycle',
|
||
'环保', '可持续', '碳中和', '循环经济', '零浪费'
|
||
]
|
||
|
||
if any(keyword.lower() in (title + content).lower() for keyword in sustainability_keywords):
|
||
articles.append({
|
||
'title': title,
|
||
'url': entry.get('link', ''),
|
||
'content': content,
|
||
'published': entry.get('published', ''),
|
||
'source_name': source.name
|
||
})
|
||
|
||
logger.info(f"从 {source.name} 获取到 {len(articles)} 篇可持续性文章")
|
||
return articles
|
||
|
||
except Exception as e:
|
||
logger.error(f"获取RSS失败 {source.name}: {e}")
|
||
return []
|
||
|
||
def fetch_web_content(self, source: SustainabilitySource) -> List[Dict]:
|
||
"""获取网页内容（简化版，实际需要更复杂的抓取）"""
|
||
# 简化实现：只记录，不实际抓取
|
||
logger.info(f"网页信息源 {source.name} 需要手动处理")
|
||
return []
|
||
|
||
def analyze_article(self, article: Dict) -> Optional[SustainabilityCase]:
|
||
"""分析文章内容，提炼案例"""
|
||
try:
|
||
content = article['content']
|
||
title = article['title']
|
||
|
||
# 提取关键数据（简化版，实际可用NLP）
|
||
data_patterns = [
|
||
r'(\d+\.?\d*)\s*(?:percent|%|百分比)',
|
||
r'(\d+\.?\d*)\s*(?:million|billion|万|亿)',
|
||
r'(\d+\.?\d*)\s*(?:tons|tonnes|吨)',
|
||
r'(\d+\.?\d*)\s*(?:reduction|increase|减少|增加)'
|
||
]
|
||
|
||
data_points = []
|
||
for pattern in data_patterns:
|
||
matches = re.findall(pattern, content, re.IGNORECASE)
|
||
if matches:
|
||
data_points.extend(matches[:3]) # 限制数量
|
||
|
||
if len(data_points) < 2:
|
||
logger.info(f"文章数据不足: {title}")
|
||
return None
|
||
|
||
# 确定国家（简化判断）
|
||
countries = ['China', 'Japan', 'Germany', 'US', 'UK', 'Sweden', 'Netherlands']
|
||
country = 'Global' # 默认
|
||
for c in countries:
|
||
if c.lower() in content.lower():
|
||
country = c
|
||
break
|
||
|
||
# 确定子领域
|
||
categories = self.config["sustainability_categories"]
|
||
category = categories[0] # 默认第一个
|
||
for cat in categories:
|
||
if any(keyword in content.lower() for keyword in [cat.lower(), cat[:4].lower()]):
|
||
category = cat
|
||
break
|
||
|
||
# 生成案例ID
|
||
case_id = f"SUS-{hashlib.md5(title.encode()).hexdigest()[:8].upper()}"
|
||
|
||
# 提取核心观点（简化版）
|
||
# 实际应用中可用AI提取，这里用前100字符
|
||
core_idea = content[:200] if len(content) > 200 else content
|
||
|
||
# 生成中国痛点（基于类别模板）
|
||
china_pains = {
|
||
"城市农业": "中国城市空间小、光照不足、怕邻居投诉",
|
||
"零浪费生活": "中国垃圾分类执行难、环保产品溢价高",
|
||
"低碳出行": "中国电动车充电难、城市规划不支持",
|
||
"循环消费": "中国二手文化不成熟、维修成本高",
|
||
"能源效率": "中国能源价格波动、设备更换成本高",
|
||
"可持续饮食": "中国预制菜泛滥、有机食品价格高",
|
||
"环保科技产品": "中国消费者关注价格多于环保"
|
||
}
|
||
china_pain = china_pains.get(category, "中国相关数据不足，需本土化验证")
|
||
|
||
# 生成案例
|
||
case = SustainabilityCase(
|
||
id=case_id,
|
||
country=country,
|
||
category=category,
|
||
title=title,
|
||
core_idea=core_idea,
|
||
data_facts=f"数据点: {', '.join(data_points[:3])}",
|
||
global_advantage="需进一步分析全球优势",
|
||
china_pain_point=china_pain,
|
||
localization_suggestion="需基于中国现实调整实施",
|
||
mvp_action="建议先小规模试点验证",
|
||
source_url=article['url'],
|
||
credibility_rating="⭐⭐" if article['source_name'] in ['GreenBiz', 'Sustainable Brands'] else "⭐",
|
||
china_applicability="⭐⭐",
|
||
collection_date=TODAY
|
||
)
|
||
|
||
return case
|
||
|
||
except Exception as e:
|
||
logger.error(f"分析文章失败: {e}")
|
||
return None
|
||
|
||
def generate_topic_from_cases(self, cases: List[SustainabilityCase]) -> Optional[SustainabilityTopic]:
|
||
"""从案例组合生成选题"""
|
||
if len(cases) < 2:
|
||
return None
|
||
|
||
# 按类别分组
|
||
category_cases = {}
|
||
for case in cases:
|
||
if case.category not in category_cases:
|
||
category_cases[case.category] = []
|
||
category_cases[case.category].append(case)
|
||
|
||
# 选择案例数最多的类别
|
||
main_category = max(category_cases, key=lambda k: len(category_cases[k]))
|
||
main_cases = category_cases[main_category]
|
||
|
||
if len(main_cases) < 2:
|
||
return None
|
||
|
||
# 生成选题ID
|
||
topic_id = f"TOPIC-.{hashlib.md5((main_category + TODAY).encode()).hexdigest()[:6].upper()}"
|
||
|
||
# 组合标题
|
||
case_titles = [case.title[:30] for case in main_cases[:2]]
|
||
topic_title = f"{main_category}新趋势: {case_titles[0]}与{case_titles[1]}的中国落地路径"
|
||
|
||
# 计算优先级分数
|
||
priority_weights = self.config["topic_priority"]
|
||
priority_score = (
|
||
priority_weights["audience_match"] * 0.8 + # 受众匹配度预估
|
||
priority_weights["data_availability"] * 0.9 + # 数据可得性
|
||
priority_weights["uniqueness"] * 0.7 + # 独特性
|
||
priority_weights["executability"] * 0.6 + # 可执行性
|
||
priority_weights["brand_fit"] * 0.9 # 品牌契合度
|
||
)
|
||
|
||
topic = SustainabilityTopic(
|
||
id=topic_id,
|
||
title=topic_title,
|
||
cases=[case.id for case in main_cases],
|
||
audience="城市焦虑青年（26-35岁）",
|
||
china_pain_points=f"{main_category}在中国面临的主要问题",
|
||
localization_solution="国际案例中国化适配方案",
|
||
mvp_actions="读者可立即尝试的3个行动",
|
||
estimated_length=2500,
|
||
priority_score=round(priority_score, 2)
|
||
)
|
||
|
||
return topic
|
||
|
||
def save_results(self):
|
||
"""保存收集结果"""
|
||
# 保存案例
|
||
cases_file = self.today_dir / "new_cases.json"
|
||
with open(cases_file, 'w', encoding='utf-8') as f:
|
||
json.dump([asdict(case) for case in self.new_cases], f, ensure_ascii=False, indent=2)
|
||
|
||
# 保存选题
|
||
topics_file = self.today_dir / "new_topics.json"
|
||
with open(topics_file, 'w', encoding='utf-8') as f:
|
||
json.dump([asdict(topic) for topic in self.new_topics], f, ensure_ascii=False, indent=2)
|
||
|
||
# 更新主数据库
|
||
self.update_main_database()
|
||
|
||
logger.info(f"保存了 {len(self.new_cases)} 个案例和 {len(self.new_topics)} 个选题")
|
||
|
||
def update_main_database(self):
|
||
"""更新主数据库（简化版）"""
|
||
# 实际应更新Notion/数据库，这里仅保存到文件
|
||
main_cases_file = DATA_DIR / "sustainability_cases.json"
|
||
main_topics_file = DATA_DIR / "sustainability_topics.json"
|
||
|
||
# 读取现有数据
|
||
existing_cases = []
|
||
existing_topics = []
|
||
|
||
if main_cases_file.exists():
|
||
with open(main_cases_file, 'r', encoding='utf-8') as f:
|
||
existing_cases = json.load(f)
|
||
|
||
if main_topics_file.exists():
|
||
with open(main_topics_file, 'r', encoding='utf-8') as f:
|
||
existing_topics = json.load(f)
|
||
|
||
# 合并新数据
|
||
all_cases = existing_cases + [asdict(case) for case in self.new_cases]
|
||
all_topics = existing_topics + [asdict(topic) for topic in self.new_topics]
|
||
|
||
# 保存（限制总数）
|
||
with open(main_cases_file, 'w', encoding='utf-8') as f:
|
||
json.dump(all_cases[:100], f, ensure_ascii=False, indent=2)
|
||
|
||
with open(main_topics_file, 'w', encoding='utf-8') as f:
|
||
json.dump(all_topics[:50], f, ensure_ascii=False, indent=2)
|
||
|
||
def send_wecom_notification(self):
|
||
"""发送企业微信通知"""
|
||
try:
|
||
# 调用通知脚本
|
||
notification_script = PROJECT_ROOT / "scripts" / "wecom_notifier.py"
|
||
if not notification_script.exists():
|
||
logger.warning("企业微信通知脚本不存在")
|
||
return
|
||
|
||
# 准备通知数据
|
||
notification_data = {
|
||
"task": "sustainability_collection",
|
||
"time": datetime.datetime.now().strftime("%Y-%m-%d %H:%M"),
|
||
"topic_count": len(self.new_topics),
|
||
"case_count": len(self.new_cases),
|
||
"source_count": len(self.sources),
|
||
"details_link": str(self.today_dir.relative_to(PROJECT_ROOT))
|
||
}
|
||
|
||
data_file = self.today_dir / "notification_data.json"
|
||
with open(data_file, 'w', encoding='utf-8') as f:
|
||
json.dump(notification_data, f, ensure_ascii=False)
|
||
|
||
# 运行通知脚本
|
||
result = subprocess.run(
|
||
[sys.executable, str(notification_script), str(data_file)],
|
||
capture_output=True,
|
||
text=True,
|
||
cwd=PROJECT_ROOT
|
||
)
|
||
|
||
if result.returncode == 0:
|
||
logger.info("企业微信通知发送成功")
|
||
else:
|
||
logger.error(f"通知发送失败: {result.stderr}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"发送通知失败: {e}")
|
||
|
||
def run(self):
|
||
"""主运行流程"""
|
||
logger.info("开始可持续性内容收集")
|
||
|
||
# 1. 从所有信息源收集
|
||
all_articles = []
|
||
for source in self.sources:
|
||
if source.type == 'rss':
|
||
articles = self.fetch_rss_feed(source)
|
||
all_articles.extend(articles)
|
||
elif source.type == 'web':
|
||
articles = self.fetch_web_content(source)
|
||
all_articles.extend(articles)
|
||
elif source.type == 'api':
|
||
# TODO: 实现API抓取
|
||
pass
|
||
elif source.type == 'local':
|
||
# 本地源不产生新文章，后续降级处理
|
||
pass
|
||
|
||
logger.info(f"总共收集到 {len(all_articles)} 篇可持续性文章")
|
||
|
||
# 2. 分析文章，提炼案例
|
||
for article in all_articles[:20]: # 限制分析数量
|
||
case = self.analyze_article(article)
|
||
if case:
|
||
self.new_cases.append(case)
|
||
|
||
# 3. 降级策略：如果外部源没有收集到足够案例，使用本地案例库
|
||
if len(self.new_cases) < 2:
|
||
logger.warning(f"外部源案例不足 ({len(self.new_cases)} < 2)，启动降级策略")
|
||
|
||
# 优先：从本地JSON数据库加载最近案例
|
||
local_cases = self.load_local_cases_from_db()
|
||
if len(local_cases) < 2:
|
||
# 备用：从Markdown案例库解析
|
||
local_cases = self.load_local_cases_from_markdown()
|
||
|
||
if local_cases:
|
||
# 随机选取2-3个本地案例作为本次选题的案例基础
|
||
import random
|
||
selected = random.sample(local_cases, min(3, len(local_cases)))
|
||
self.new_cases.extend(selected)
|
||
logger.info(f"降级：使用了 {len(selected)} 个本地案例")
|
||
else:
|
||
logger.error("降级失败：本地案例库为空")
|
||
|
||
# 4. 生成选题
|
||
if self.new_cases:
|
||
topic = self.generate_topic_from_cases(self.new_cases)
|
||
if topic:
|
||
# 标记为今日创建，并添加锁字段（表示未被占用）
|
||
topic.created_at = datetime.datetime.now().isoformat()
|
||
topic.lock_by = None
|
||
topic.lock_at = None
|
||
# 确保状态为「待处理」
|
||
topic.status = "待处理"
|
||
self.new_topics.append(topic)
|
||
|
||
# 5. 保存结果
|
||
self.save_results()
|
||
|
||
# 6. 发送通知
|
||
self.send_wecom_notification()
|
||
|
||
logger.info(f"收集完成: {len(self.new_cases)} 案例, {len(self.new_topics)} 选题")
|
||
return len(self.new_cases), len(self.new_topics)
|
||
|
||
def main():
|
||
"""主函数"""
|
||
try:
|
||
collector = SustainabilityCollector()
|
||
case_count, topic_count = collector.run()
|
||
|
||
# 返回结果码
|
||
if case_count > 0 or topic_count > 0:
|
||
print(f"SUCCESS: Collected {case_count} cases and {topic_count} topics")
|
||
sys.exit(0)
|
||
else:
|
||
print("WARNING: No new content found")
|
||
sys.exit(1)
|
||
|
||
except Exception as e:
|
||
logger.error(f"收集任务失败: {e}")
|
||
print(f"ERROR: {e}")
|
||
sys.exit(1)
|
||
|
||
if __name__ == "__main__":
|
||
main() |