lt
438 lines
17 KiB
Python
438 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
可持续性内容收集脚本
|
|
每天凌晨5:00运行，收集全球可持续性趋势信息，提炼选题和案例
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import yaml
|
|
import json
|
|
import datetime
|
|
import logging
|
|
from pathlib import Path
|
|
import feedparser
|
|
import requests
|
|
import re
|
|
from typing import Dict, List, Optional, Tuple
|
|
import hashlib
|
|
from dataclasses import dataclass, asdict
|
|
import subprocess
|
|
|
|
# 项目根目录
|
|
PROJECT_ROOT = Path(__file__).parent.parent
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
# 配置路径
|
|
CONFIG_DIR = PROJECT_ROOT / "config"
|
|
DATA_DIR = PROJECT_ROOT / "automation" / "data"
|
|
LOGS_DIR = PROJECT_ROOT / "automation" / "logs"
|
|
TODAY = datetime.datetime.now().strftime("%Y-%m-%d")
|
|
|
|
# 日志配置
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
handlers=[
|
|
logging.FileHandler(LOGS_DIR / f"collector_{TODAY}.log"),
|
|
logging.StreamHandler()
|
|
]
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
@dataclass
|
|
class SustainabilitySource:
|
|
"""可持续性信息源"""
|
|
name: str
|
|
type: str # rss, web, api, report
|
|
url: str
|
|
update_frequency: str
|
|
credibility: str # high, medium, low
|
|
focus: str
|
|
|
|
@dataclass
|
|
class SustainabilityCase:
|
|
"""可持续性案例"""
|
|
id: str
|
|
country: str
|
|
category: str # 子领域：城市农业、零浪费生活等
|
|
title: str
|
|
core_idea: str
|
|
data_facts: str
|
|
global_advantage: str
|
|
china_pain_point: str
|
|
localization_suggestion: str
|
|
mvp_action: str
|
|
source_url: str
|
|
credibility_rating: str # ⭐⭐ ⭐⭐⭐
|
|
china_applicability: str # ⭐ ⭐⭐ ⭐⭐⭐
|
|
collection_date: str
|
|
status: str = "待验证"
|
|
|
|
@dataclass
|
|
class SustainabilityTopic:
|
|
"""可持续性选题"""
|
|
id: str
|
|
title: str
|
|
cases: List[str] # 关联的案例ID列表
|
|
audience: str # 目标受众
|
|
china_pain_points: str
|
|
localization_solution: str
|
|
mvp_actions: str
|
|
estimated_length: int
|
|
priority_score: float
|
|
status: str = "待创作"
|
|
|
|
class SustainabilityCollector:
|
|
"""可持续性内容收集器"""
|
|
|
|
def __init__(self):
|
|
self.load_config()
|
|
self.today_dir = DATA_DIR / "sustainability_raw" / TODAY
|
|
self.today_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# 结果存储
|
|
self.new_cases: List[SustainabilityCase] = []
|
|
self.new_topics: List[SustainabilityTopic] = []
|
|
|
|
def load_config(self):
|
|
"""加载配置文件"""
|
|
with open(CONFIG_DIR / "sources.yaml", "r", encoding='utf-8') as f:
|
|
self.config = yaml.safe_load(f)
|
|
|
|
with open(CONFIG_DIR / "wecom_config.yaml", "r", encoding='utf-8') as f:
|
|
self.wecom_config = yaml.safe_load(f)
|
|
|
|
self.sources = []
|
|
for source_group in self.config["sustainability_sources"].values():
|
|
for source_info in source_group:
|
|
# Handle both 'url' and 'base_url' in config
|
|
source_info = source_info.copy()
|
|
if 'base_url' in source_info and 'url' not in source_info:
|
|
source_info['url'] = source_info.pop('base_url')
|
|
# Provide defaults for missing optional fields
|
|
source_info.setdefault('update_frequency', 'daily')
|
|
source_info.setdefault('focus', '可持续性')
|
|
# Filter to only fields accepted by SustainabilitySource
|
|
allowed_keys = {'name', 'type', 'url', 'update_frequency', 'credibility', 'focus'}
|
|
filtered_info = {k: v for k, v in source_info.items() if k in allowed_keys}
|
|
self.sources.append(SustainabilitySource(**filtered_info))
|
|
|
|
logger.info(f"加载了 {len(self.sources)} 个信息源")
|
|
|
|
def fetch_rss_feed(self, source: SustainabilitySource) -> List[Dict]:
|
|
"""获取RSS订阅内容"""
|
|
try:
|
|
feed = feedparser.parse(source.url)
|
|
articles = []
|
|
|
|
for entry in feed.entries[:10]: # 限制数量
|
|
# 检查是否包含可持续性关键词
|
|
content = entry.get('summary', entry.get('description', ''))
|
|
title = entry.get('title', '')
|
|
|
|
# 可持续性关键词匹配
|
|
sustainability_keywords = [
|
|
'sustainable', 'green', 'eco', 'circular', 'climate',
|
|
'carbon', 'zero waste', 'renewable', 'recycle',
|
|
'环保', '可持续', '碳中和', '循环经济', '零浪费'
|
|
]
|
|
|
|
if any(keyword.lower() in (title + content).lower() for keyword in sustainability_keywords):
|
|
articles.append({
|
|
'title': title,
|
|
'url': entry.get('link', ''),
|
|
'content': content,
|
|
'published': entry.get('published', ''),
|
|
'source_name': source.name
|
|
})
|
|
|
|
logger.info(f"从 {source.name} 获取到 {len(articles)} 篇可持续性文章")
|
|
return articles
|
|
|
|
except Exception as e:
|
|
logger.error(f"获取RSS失败 {source.name}: {e}")
|
|
return []
|
|
|
|
def fetch_web_content(self, source: SustainabilitySource) -> List[Dict]:
|
|
"""获取网页内容（简化版，实际需要更复杂的抓取）"""
|
|
# 简化实现：只记录，不实际抓取
|
|
logger.info(f"网页信息源 {source.name} 需要手动处理")
|
|
return []
|
|
|
|
def analyze_article(self, article: Dict) -> Optional[SustainabilityCase]:
|
|
"""分析文章内容，提炼案例"""
|
|
try:
|
|
content = article['content']
|
|
title = article['title']
|
|
|
|
# 提取关键数据（简化版，实际可用NLP）
|
|
data_patterns = [
|
|
r'(\d+\.?\d*)\s*(?:percent|%|百分比)',
|
|
r'(\d+\.?\d*)\s*(?:million|billion|万|亿)',
|
|
r'(\d+\.?\d*)\s*(?:tons|tonnes|吨)',
|
|
r'(\d+\.?\d*)\s*(?:reduction|increase|减少|增加)'
|
|
]
|
|
|
|
data_points = []
|
|
for pattern in data_patterns:
|
|
matches = re.findall(pattern, content, re.IGNORECASE)
|
|
if matches:
|
|
data_points.extend(matches[:3]) # 限制数量
|
|
|
|
if len(data_points) < 2:
|
|
logger.info(f"文章数据不足: {title}")
|
|
return None
|
|
|
|
# 确定国家（简化判断）
|
|
countries = ['China', 'Japan', 'Germany', 'US', 'UK', 'Sweden', 'Netherlands']
|
|
country = 'Global' # 默认
|
|
for c in countries:
|
|
if c.lower() in content.lower():
|
|
country = c
|
|
break
|
|
|
|
# 确定子领域
|
|
categories = self.config["sustainability_categories"]
|
|
category = categories[0] # 默认第一个
|
|
for cat in categories:
|
|
if any(keyword in content.lower() for keyword in [cat.lower(), cat[:4].lower()]):
|
|
category = cat
|
|
break
|
|
|
|
# 生成案例ID
|
|
case_id = f"SUS-{hashlib.md5(title.encode()).hexdigest()[:8].upper()}"
|
|
|
|
# 提取核心观点（简化版）
|
|
# 实际应用中可用AI提取，这里用前100字符
|
|
core_idea = content[:200] if len(content) > 200 else content
|
|
|
|
# 生成中国痛点（基于类别模板）
|
|
china_pains = {
|
|
"城市农业": "中国城市空间小、光照不足、怕邻居投诉",
|
|
"零浪费生活": "中国垃圾分类执行难、环保产品溢价高",
|
|
"低碳出行": "中国电动车充电难、城市规划不支持",
|
|
"循环消费": "中国二手文化不成熟、维修成本高",
|
|
"能源效率": "中国能源价格波动、设备更换成本高",
|
|
"可持续饮食": "中国预制菜泛滥、有机食品价格高",
|
|
"环保科技产品": "中国消费者关注价格多于环保"
|
|
}
|
|
china_pain = china_pains.get(category, "中国相关数据不足，需本土化验证")
|
|
|
|
# 生成案例
|
|
case = SustainabilityCase(
|
|
id=case_id,
|
|
country=country,
|
|
category=category,
|
|
title=title,
|
|
core_idea=core_idea,
|
|
data_facts=f"数据点: {', '.join(data_points[:3])}",
|
|
global_advantage="需进一步分析全球优势",
|
|
china_pain_point=china_pain,
|
|
localization_suggestion="需基于中国现实调整实施",
|
|
mvp_action="建议先小规模试点验证",
|
|
source_url=article['url'],
|
|
credibility_rating="⭐⭐" if article['source_name'] in ['GreenBiz', 'Sustainable Brands'] else "⭐",
|
|
china_applicability="⭐⭐",
|
|
collection_date=TODAY
|
|
)
|
|
|
|
return case
|
|
|
|
except Exception as e:
|
|
logger.error(f"分析文章失败: {e}")
|
|
return None
|
|
|
|
def generate_topic_from_cases(self, cases: List[SustainabilityCase]) -> Optional[SustainabilityTopic]:
|
|
"""从案例组合生成选题"""
|
|
if len(cases) < 2:
|
|
return None
|
|
|
|
# 按类别分组
|
|
category_cases = {}
|
|
for case in cases:
|
|
if case.category not in category_cases:
|
|
category_cases[case.category] = []
|
|
category_cases[case.category].append(case)
|
|
|
|
# 选择案例数最多的类别
|
|
main_category = max(category_cases, key=lambda k: len(category_cases[k]))
|
|
main_cases = category_cases[main_category]
|
|
|
|
if len(main_cases) < 2:
|
|
return None
|
|
|
|
# 生成选题ID
|
|
topic_id = f"TOPIC-.{hashlib.md5((main_category + TODAY).encode()).hexdigest()[:6].upper()}"
|
|
|
|
# 组合标题
|
|
case_titles = [case.title[:30] for case in main_cases[:2]]
|
|
topic_title = f"{main_category}新趋势: {case_titles[0]}与{case_titles[1]}的中国落地路径"
|
|
|
|
# 计算优先级分数
|
|
priority_weights = self.config["topic_priority"]
|
|
priority_score = (
|
|
priority_weights["audience_match"] * 0.8 + # 受众匹配度预估
|
|
priority_weights["data_availability"] * 0.9 + # 数据可得性
|
|
priority_weights["uniqueness"] * 0.7 + # 独特性
|
|
priority_weights["executability"] * 0.6 + # 可执行性
|
|
priority_weights["brand_fit"] * 0.9 # 品牌契合度
|
|
)
|
|
|
|
topic = SustainabilityTopic(
|
|
id=topic_id,
|
|
title=topic_title,
|
|
cases=[case.id for case in main_cases],
|
|
audience="城市焦虑青年（26-35岁）",
|
|
china_pain_points=f"{main_category}在中国面临的主要问题",
|
|
localization_solution="国际案例中国化适配方案",
|
|
mvp_actions="读者可立即尝试的3个行动",
|
|
estimated_length=2500,
|
|
priority_score=round(priority_score, 2)
|
|
)
|
|
|
|
return topic
|
|
|
|
def save_results(self):
|
|
"""保存收集结果"""
|
|
# 保存案例
|
|
cases_file = self.today_dir / "new_cases.json"
|
|
with open(cases_file, 'w', encoding='utf-8') as f:
|
|
json.dump([asdict(case) for case in self.new_cases], f, ensure_ascii=False, indent=2)
|
|
|
|
# 保存选题
|
|
topics_file = self.today_dir / "new_topics.json"
|
|
with open(topics_file, 'w', encoding='utf-8') as f:
|
|
json.dump([asdict(topic) for topic in self.new_topics], f, ensure_ascii=False, indent=2)
|
|
|
|
# 更新主数据库
|
|
self.update_main_database()
|
|
|
|
logger.info(f"保存了 {len(self.new_cases)} 个案例和 {len(self.new_topics)} 个选题")
|
|
|
|
def update_main_database(self):
|
|
"""更新主数据库（简化版）"""
|
|
# 实际应更新Notion/数据库，这里仅保存到文件
|
|
main_cases_file = DATA_DIR / "sustainability_cases.json"
|
|
main_topics_file = DATA_DIR / "sustainability_topics.json"
|
|
|
|
# 读取现有数据
|
|
existing_cases = []
|
|
existing_topics = []
|
|
|
|
if main_cases_file.exists():
|
|
with open(main_cases_file, 'r', encoding='utf-8') as f:
|
|
existing_cases = json.load(f)
|
|
|
|
if main_topics_file.exists():
|
|
with open(main_topics_file, 'r', encoding='utf-8') as f:
|
|
existing_topics = json.load(f)
|
|
|
|
# 合并新数据
|
|
all_cases = existing_cases + [asdict(case) for case in self.new_cases]
|
|
all_topics = existing_topics + [asdict(topic) for topic in self.new_topics]
|
|
|
|
# 保存（限制总数）
|
|
with open(main_cases_file, 'w', encoding='utf-8') as f:
|
|
json.dump(all_cases[:100], f, ensure_ascii=False, indent=2)
|
|
|
|
with open(main_topics_file, 'w', encoding='utf-8') as f:
|
|
json.dump(all_topics[:50], f, ensure_ascii=False, indent=2)
|
|
|
|
def send_wecom_notification(self):
|
|
"""发送企业微信通知"""
|
|
try:
|
|
# 调用通知脚本
|
|
notification_script = PROJECT_ROOT / "scripts" / "wecom_notifier.py"
|
|
if not notification_script.exists():
|
|
logger.warning("企业微信通知脚本不存在")
|
|
return
|
|
|
|
# 准备通知数据
|
|
notification_data = {
|
|
"task": "sustainability_collection",
|
|
"time": datetime.datetime.now().strftime("%Y-%m-%d %H:%M"),
|
|
"topic_count": len(self.new_topics),
|
|
"case_count": len(self.new_cases),
|
|
"source_count": len(self.sources),
|
|
"details_link": str(self.today_dir.relative_to(PROJECT_ROOT))
|
|
}
|
|
|
|
data_file = self.today_dir / "notification_data.json"
|
|
with open(data_file, 'w', encoding='utf-8') as f:
|
|
json.dump(notification_data, f, ensure_ascii=False)
|
|
|
|
# 运行通知脚本
|
|
result = subprocess.run(
|
|
[sys.executable, str(notification_script), str(data_file)],
|
|
capture_output=True,
|
|
text=True,
|
|
cwd=PROJECT_ROOT
|
|
)
|
|
|
|
if result.returncode == 0:
|
|
logger.info("企业微信通知发送成功")
|
|
else:
|
|
logger.error(f"通知发送失败: {result.stderr}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"发送通知失败: {e}")
|
|
|
|
def run(self):
|
|
"""主运行流程"""
|
|
logger.info("开始可持续性内容收集")
|
|
|
|
# 1. 从所有信息源收集
|
|
all_articles = []
|
|
for source in self.sources:
|
|
if source.type == 'rss':
|
|
articles = self.fetch_rss_feed(source)
|
|
all_articles.extend(articles)
|
|
elif source.type == 'web':
|
|
articles = self.fetch_web_content(source)
|
|
all_articles.extend(articles)
|
|
|
|
logger.info(f"总共收集到 {len(all_articles)} 篇可持续性文章")
|
|
|
|
# 2. 分析文章，提炼案例
|
|
for article in all_articles[:20]: # 限制分析数量
|
|
case = self.analyze_article(article)
|
|
if case:
|
|
self.new_cases.append(case)
|
|
|
|
# 3. 生成选题
|
|
if self.new_cases:
|
|
topic = self.generate_topic_from_cases(self.new_cases)
|
|
if topic:
|
|
self.new_topics.append(topic)
|
|
|
|
# 4. 保存结果
|
|
self.save_results()
|
|
|
|
# 5. 发送通知
|
|
self.send_wecom_notification()
|
|
|
|
logger.info(f"收集完成: {len(self.new_cases)} 案例, {len(self.new_topics)} 选题")
|
|
return len(self.new_cases), len(self.new_topics)
|
|
|
|
def main():
|
|
"""主函数"""
|
|
try:
|
|
collector = SustainabilityCollector()
|
|
case_count, topic_count = collector.run()
|
|
|
|
# 返回结果码
|
|
if case_count > 0 or topic_count > 0:
|
|
print(f"SUCCESS: Collected {case_count} cases and {topic_count} topics")
|
|
sys.exit(0)
|
|
else:
|
|
print("WARNING: No new content found")
|
|
sys.exit(1)
|
|
|
|
except Exception as e:
|
|
logger.error(f"收集任务失败: {e}")
|
|
print(f"ERROR: {e}")
|
|
sys.exit(1)
|
|
|
|
if __name__ == "__main__":
|
|
main() |