txai-dev/yu-zhi-ran
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Phase 4 多租户隔离 + 四阶段升级测试 + CSS 统一化

Browse Source 
Phase 4: org_id 注入 JWT/API 过滤/组织管理 CRUD/前端组织列
测试: tests/test_phase_upgrades.py 97项全覆盖
CSS: theme-modern.css 共享 mobile-card-list/status-dot/search-bar 等模式
修复: initial_data.py LLM配置 NOT NULL 约束, TopicResponse 含 org_id
This commit is contained in:
Yuzhiran Dev
2026-05-17 06:56:53 +08:00
parent 301dc3e438
commit 9c37c9a574
45 changed files with 3707 additions and 1366 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/collector.py
+276 -93
View File
@@ -46,12 +46,13 @@ logger = logging.getLogger(__name__)
class SustainabilitySource:
"""可持续性信息源"""
name: str
type: str # rss, web, api, report, local
url: Optional[str] = None # 可为空(如本地源)
type: str # rss, web_search, web, api, local
url: Optional[str] = None # RSS URL 或通用链接
update_frequency: str = "daily"
credibility: str = "medium"
focus: str = "可持续性"
keywords: Optional[List[str]] = None # 源特定关键词
query: Optional[str] = None # 搜索查询词(w eb_search类型用)
@dataclass
class SustainabilityCase:
@@ -118,30 +119,60 @@ class SustainabilityCollector:
self.new_topics: List[SustainabilityTopic] = []
def load_config(self):
"""加载配置文件"""
with open(CONFIG_DIR / "sources.yaml", "r", encoding='utf-8') as f:
self.config = yaml.safe_load(f)
"""加载配置：优先从DB读取，DB为空则从YAML fallback再写入DB"""
self.config = {}
self.config_path = CONFIG_DIR / "sources.yaml"
if self.config_path.exists():
with open(self.config_path, encoding='utf-8') as f:
self.config = yaml.safe_load(f) or {}
with open(CONFIG_DIR / "wecom_config.yaml", "r", encoding='utf-8') as f:
with open(CONFIG_DIR / "wecom_config.yaml", encoding='utf-8') as f:
self.wecom_config = yaml.safe_load(f)
# 优先从DB读取类别和源
self.sources = []
for source_group in self.config["sustainability_sources"].values():
try:
from app.database import SessionLocal
from app.models import CollectorCategory, CollectorSource
db = SessionLocal()
try:
cats = db.query(CollectorCategory).filter(CollectorCategory.is_active == True).order_by(CollectorCategory.sort_order).all()
if cats:
# 用DB中的类别覆盖YAML
self.config["sustainability_categories"] = [c.name for c in cats]
sources_db = db.query(CollectorSource).filter(CollectorSource.is_active == True).order_by(CollectorSource.sort_order).all()
for s in sources_db:
self.sources.append(SustainabilitySource(
name=s.name,
type=s.source_type,
url=s.url or '',
query=s.query or '',
credibility=s.credibility or 'medium',
focus=s.focus or '可持续性',
))
logger.info(f"从DB加载 {len(cats)} 个类别, {len(self.sources)} 个信息源")
db.close()
return
except Exception as e:
logger.warning(f"DB读取类别/源失败，回退YAML: {e}")
db.close()
except Exception as e:
logger.warning(f"DB连接失败，回退YAML: {e}")
# YAML fallback
for source_group in self.config.get("sustainability_sources", {}).values():
for source_info in source_group:
# Handle both 'url' and 'base_url' in config
source_info = source_info.copy()
if 'base_url' in source_info and 'url' not in source_info:
source_info['url'] = source_info.pop('base_url')
# Provide defaults for missing optional fields
source_info.setdefault('update_frequency', 'daily')
source_info.setdefault('focus', '可持续性')
source_info.setdefault('keywords', None)
# Filter to only fields accepted by SustainabilitySource
allowed_keys = {'name', 'type', 'url', 'update_frequency', 'credibility', 'focus', 'keywords'}
allowed_keys = {'name', 'type', 'url', 'update_frequency', 'credibility', 'focus', 'keywords', 'query'}
filtered_info = {k: v for k, v in source_info.items() if k in allowed_keys}
self.sources.append(SustainabilitySource(**filtered_info))
logger.info(f"加载了 {len(self.sources)} 个信息源")
logger.info(f"YAML fallback: 加载 {len(self.sources)} 个信息源")
def load_local_cases_from_db(self) -> List[SustainabilityCase]:
"""从本地案例库加载历史案例,用于降级生成选题"""
@@ -213,16 +244,17 @@ class SustainabilityCollector:
field = field_match.group(1).strip()
# 映射到子领域(扩展映射表)
category_map = {
'远程工作方式': '城市农业',
'远程工作方式': '循环消费',
'数字游民政策': '低碳出行',
'AI副业服务': '环保科技产品',
'一人公司模式': '循环消费',
'未来技能趋势': '可持续饮食',
'可持续生活': '可持续饮食',
'未来技能趋势': '干净饮食',
'可持续生活': '零浪费生活',
'零浪费生活': '零浪费生活',
'低碳出行': '低碳出行',
'循环消费': '循环消费',
'环保科技': '环保科技产品'
'环保科技': '环保科技产品',
'城市农业': '循环消费',
}
case_data['category'] = category_map.get(field, field[:4] if len(field) > 4 else field)
@@ -317,10 +349,129 @@ class SustainabilityCollector:
def fetch_web_content(self, source: SustainabilitySource) -> List[Dict]:
"""获取网页内容(简化版,实际需要更复杂的抓取)"""
# 简化实现:只记录,不实际抓取
logger.info(f"网页信息源 {source.name} 需要手动处理")
return []
def fetch_web_search(self, source: SustainabilitySource) -> List[Dict]:
"""通过Bing中文搜索获取实时内容"""
try:
from web_search import search
query = source.query or source.url or ''
query = query.strip()
if not query:
logger.warning(f"web_search源 {source.name} 未配置查询词")
return []
results = search(query, max_results=8, use_cache=False)
articles = []
for r in results:
articles.append({
'title': r.get('title', ''),
'url': r.get('url', ''),
'content': r.get('snippet', ''),
'published': TODAY,
'source_name': source.name,
'search_query': query,
})
logger.info(f"搜索 [{query}] 获得 {len(articles)} 条结果")
return articles
except Exception as e:
logger.warning(f"web_search失败 {source.name}: {e}")
return []
def _generate_topic_with_llm(self, search_results: List[Dict]) -> Optional[SustainabilityTopic]:
"""用LLM从搜索结果中生成选题"""
try:
from app.core.nvidia_client import call_llm
except ImportError:
logger.warning("LLM不可用，跳过AI选题生成")
return None
if not search_results:
return None
# 整理搜索结果摘要
summaries = []
for r in search_results[:6]:
summaries.append(f"- {r.get('title','')}: {r.get('content','')[:150]}")
search_text = "\n".join(summaries)
# 获取已有选题做去重参考
existing = self._get_existing_titles()
existing_hint = ""
if existing:
existing_hint = f"\n以下选题已存在，请避免重复：\n" + "\n".join(f"- {t[:30]}" for t in existing[-10:])
# 按日期选不同类别
categories = self.config.get("sustainability_categories", ["可持续生活"])
day_idx = datetime.datetime.now().timetuple().tm_yday % len(categories)
target_category = categories[day_idx]
prompt = f"""你是一个内容策略师。基于以下搜索结果，生成一个有价值、适合中文互联网传播的选题。
目标类别：{target_category}
搜索结果：
{search_text}
{existing_hint}
请生成一个选题，输出JSON格式：
{{
"title": "标题（20字内，有吸引力，含核心关键词）",
"core_concept": "核心观点（一句话说清独特价值）",
"audience_pain": "受众痛点（真实用户的困惑或需求）",
"unique_angle": "独特视角（差异化切入点）",
"format": "内容形式（趋势洞察/实操指南/对比分析/案例解读）"
}}
要求：
- 标题要像人会搜索的，带领域关键词
- 避免「新趋势」「指南」「攻略」这类同质化结尾
- 切入点要具体，不要泛泛而谈
- 优先考虑中国读者能实操的内容
只输出JSON，不要其他文字。"""
try:
resp = call_llm(prompt, temperature=0.7, max_tokens=800)
resp = resp.strip()
if resp.startswith("```"):
resp = resp.split("\n", 1)[1].rsplit("\n", 1)[0]
data = json.loads(resp)
topic_id = f"TOPIC-{hashlib.md5((target_category + data.get('title','')[:10]).encode()).hexdigest()[:6].upper()}"
topic = SustainabilityTopic(
id=topic_id,
title=data.get("title", f"{target_category}新观察"),
cases=[],
audience="城市焦虑青年(26-35岁)",
china_pain_points=data.get("audience_pain", ""),
localization_solution="文章中将提供具体可执行的建议",
mvp_actions="读者可立即尝试的3个行动",
estimated_length=2000,
priority_score=7.0,
field=self.map_category_to_field(target_category),
format=data.get("format", "趋势洞察 + 实操指南"),
core_concept=data.get("core_concept", ""),
audience_pain=data.get("audience_pain", ""),
unique_angle=data.get("unique_angle", ""),
priority="中",
total_score=70.0,
compliance_score=100,
source_file="automation/data/sustainability_topics.json",
status="待处理",
lock_by=None,
lock_at=None,
created_at=datetime.datetime.now().isoformat(),
ready_at=None,
published_at=None,
platform_urls={}
)
logger.info(f"LLM生成选题: {topic.title}")
return topic
except Exception as e:
logger.warning(f"LLM选题生成失败: {e}")
return None
def analyze_article(self, article: Dict) -> Optional[SustainabilityCase]:
"""分析文章内容,提炼案例"""
try:
@@ -370,13 +521,14 @@ class SustainabilityCollector:
# 生成中国痛点(基于类别模板)
china_pains = {
"城市农业": "中国城市空间小、光照不足、怕邻居投诉",
"零浪费生活": "中国垃圾分类执行难、环保产品溢价高",
"低碳出行": "中国电动车充电难、城市规划不支持",
"循环消费": "中国二手文化不成熟、维修成本高",
"能源效率": "中国能源价格波动、设备更换成本高",
"可持续饮食": "中国预制菜泛滥、有机食品价格高",
"环保科技产品": "中国消费者关注价格多于环保"
"循环消费": "以旧换新流程繁琐、二手商品信任缺失、租赁市场不规范",
"低碳出行": "新能源车充电设施不足、城市规划不支持骑行、通勤距离长",
"干净饮食": "有机食品价格高、真伪难辨、外卖为主的生活方式难以改变",
"零浪费生活": "环保产品溢价高、可持续选择不便、漂绿营销难以分辨",
"绿色家电与节能": "绿色家电初期投入高、节能效果难量化、老旧小区改造难",
"碳普惠": "碳账户普及率低、减排量兑换吸引力不足、公众认知有限",
"环保科技产品": "绿色产品溢价68%难以承受、缺乏统一认证标准、担心漂绿",
"AI与效率": "AI工具选择困难、数据隐私担忧、学习成本高、实际效果难验证"
}
china_pain = china_pains.get(category, "中国相关数据不足,需本土化验证")
@@ -423,12 +575,21 @@ class SustainabilityCollector:
if len(main_cases) < 2:
return None
# 生成选题ID
topic_id = f"TOPIC-{hashlib.md5((main_category + TODAY).encode()).hexdigest()[:6].upper()}"
# 组合标题
# 组合标题（多种模板轮换，避免天天同款）
case_titles = [case.title[:30] for case in main_cases[:2]]
topic_title = f"{main_category}新趋势: {case_titles[0]}与{case_titles[1]}的中国落地路径"
day_of_year = datetime.datetime.now().timetuple().tm_yday
title_templates = [
f"{main_category}新趋势: {case_titles[0]}与{case_titles[1]}的中国落地路径",
f"从{case_titles[0][:15]}到{case_titles[1][:15]}: {main_category}的中国实践指南",
f"2026{main_category}观察: {case_titles[0]}给中国什么启示",
f"实战对比: {case_titles[0][:10]}vs{case_titles[1][:10]},中国读者该学谁",
f"为什么{case_titles[0][:15]}在中国行不通(或更行)? — {main_category}深度拆解",
]
topic_title = title_templates[day_of_year % len(title_templates)]
# 生成选题ID（案例内容hash保证同一批案例产出相同ID，避免重复入库）
content_seed = main_category + case_titles[0][:10] + case_titles[1][:10]
topic_id = f"TOPIC-{hashlib.md5(content_seed.encode()).hexdigest()[:6].upper()}"
# 计算优先级分数
priority_weights = self.config["topic_priority"]
@@ -484,15 +645,7 @@ class SustainabilityCollector:
def map_category_to_field(self, category: str) -> str:
"""将案例类别映射到内容领域的字段"""
category_map = {
"城市农业": "可持续生活系统",
"零浪费生活": "可持续生活系统",
"低碳出行": "可持续生活系统",
"循环消费": "可持续生活系统",
"能源效率": "可持续生活系统",
"环保科技产品": "可持续生活系统"
}
return category_map.get(category, "可持续生活系统")
return "可持续生活系统"
def save_results(self):
"""保存收集结果"""
@@ -585,87 +738,117 @@ class SustainabilityCollector:
except Exception as e:
logger.error(f"发送通知失败: {e}")
def _get_existing_titles(self) -> List[str]:
"""从DB获取已有的选题标题列表用于去重"""
try:
from db_helper import export_topics_to_json
topics = export_topics_to_json()
return [t.get('title', '') for t in topics]
except Exception as e:
logger.warning(f"读取已有选题失败: {e}")
return []
def _is_duplicate_topic(self, title: str, existing_titles: List[str]) -> bool:
"""检查选题是否与已有选题重复（前10字重叠即为重复）"""
prefix = title[:10].strip()
for et in existing_titles:
if prefix in et or et[:10] in title:
return True
return False
def _rotate_category(self, local_cases: List[SustainabilityCase]) -> Tuple[str, List[SustainabilityCase]]:
"""按日期轮换类别，避免天天选中同一类"""
category_cases = {}
for case in local_cases:
category_cases.setdefault(case.category, []).append(case)
if not category_cases:
return None, []
# 按类别名排序固定顺序
sorted_cats = sorted(category_cases.keys())
# 用一年中的第几天选类别，保证每天不重样
day_of_year = datetime.datetime.now().timetuple().tm_yday
idx = day_of_year % len(sorted_cats)
main_cat = sorted_cats[idx]
return main_cat, category_cases[main_cat]
def run(self):
"""主运行流程"""
logger.info("开始可持续性内容收集")
# 1. 从所有信息源收集
existing_titles = self._get_existing_titles()
# ---------------------- 第一阶段：多源采集 ----------------------
all_articles = []
web_search_results = [] # 留给LLM选题用的搜索结果
for source in self.sources:
if source.type == 'rss':
articles = self.fetch_rss_feed(source)
all_articles.extend(articles)
elif source.type == 'web':
articles = self.fetch_web_content(source)
elif source.type == 'web_search':
articles = self.fetch_web_search(source)
web_search_results.extend(articles)
all_articles.extend(articles)
elif source.type == 'api':
# TODO: 实现API抓取
pass
elif source.type == 'local':
# 本地源不产生新文章,后续降级处理
pass
logger.info(f"总共收集到 {len(all_articles)} 篇可持续性文章")
logger.info(f"RSS采集 {sum(1 for a in all_articles if a.get('source_name','') not in [s.name for s in self.sources if s.type=='web_search'])} 篇, "
f"搜索采集 {len(web_search_results)} 篇")
# 2. 分析文章,提炼案例
for article in all_articles[:20]: # 限制分析数量
# ---------------------- 第二阶段：尝试LLM选题生成 ----------------------
llm_topic = None
if web_search_results:
llm_topic = self._generate_topic_with_llm(web_search_results)
if llm_topic and not self._is_duplicate_topic(llm_topic.title, existing_titles):
llm_topic.created_at = datetime.datetime.now().isoformat()
llm_topic.lock_by = None
llm_topic.lock_at = None
llm_topic.status = "待处理"
self.new_topics.append(llm_topic)
logger.info(f"✅ LLM生成选题: {llm_topic.title}")
# ---------------------- 第三阶段：RSS文章提炼案例 ----------------------
rss_articles = [a for a in all_articles if a not in web_search_results]
for article in rss_articles[:15]:
case = self.analyze_article(article)
if case:
self.new_cases.append(case)
# 3. 降级策略:如果外部源没有收集到足够案例,使用本地案例库
if len(self.new_cases) < 2:
logger.warning(f"外部源案例不足 ({len(self.new_cases)} < 2),启动降级策略")
# 从本地JSON数据库加载案例(按类别分组,选择案例最多的类别)
local_cases = self.load_local_cases_from_db()
# ---------------------- 第四阶段：降级策略 ----------------------
if not self.new_topics and len(self.new_cases) < 2:
logger.warning(f"LLM选题和RSS案例均不足,启动本地案例降级")
local_cases = self.load_local_cases_from_db() or self.load_local_cases_from_markdown()
if local_cases and len(local_cases) >= 2:
# 按类别分组,选择案例最多的类别
category_cases = {}
for case in local_cases:
cat = case.category
if cat not in category_cases:
category_cases[cat] = []
category_cases[cat].append(case)
# 找出案例最多的类别
main_category = max(category_cases, key=lambda k: len(category_cases[k]))
main_cases = category_cases[main_category]
# 确保至少有2个案例
if len(main_cases) >= 2:
selected = main_cases[:min(3, len(main_cases))]
cat, cat_cases = self._rotate_category(local_cases)
if cat and len(cat_cases) >= 2:
selected = cat_cases[:min(4, len(cat_cases))]
self.new_cases.extend(selected)
logger.info(f"降级:从类别'{main_category}'选取了 {len(selected)} 个案例")
logger.info(f"降级:从类别'{cat}'选取 {len(selected)} 个案例 (day-of-year轮换)")
else:
# 如果每个类别都少于2个,则随机选2个(可能类别不同,generate_topic_from_cases会合并)
import random
selected = random.sample(local_cases, min(3, len(local_cases)))
self.new_cases.extend(selected)
logger.info(f"降级:随机选取了 {len(selected)} 个本地案例")
logger.info(f"降级:随机选取 {len(selected)} 个本地案例")
# 用本地案例生成选题
if not self.new_topics and self.new_cases:
topic = self.generate_topic_from_cases(self.new_cases)
if topic:
if self._is_duplicate_topic(topic.title, existing_titles):
logger.warning(f"选题重复,跳过: {topic.title}")
else:
topic.created_at = datetime.datetime.now().isoformat()
topic.lock_by = None
topic.lock_at = None
topic.status = "待处理"
self.new_topics.append(topic)
logger.info(f"生成新选题: {topic.title}")
else:
# 备用:从Markdown案例库解析
local_cases = self.load_local_cases_from_markdown()
if local_cases:
import random
selected = random.sample(local_cases, min(3, len(local_cases)))
self.new_cases.extend(selected)
logger.info(f"降级(Markdown):使用了 {len(selected)} 个案例")
else:
logger.error("降级失败:本地案例库为空")
# 4. 生成选题
if self.new_cases:
topic = self.generate_topic_from_cases(self.new_cases)
if topic:
# 标记为今日创建,并添加锁字段(表示未被占用)
topic.created_at = datetime.datetime.now().isoformat()
topic.lock_by = None
topic.lock_at = None
# 确保状态为「待处理」
topic.status = "待处理"
self.new_topics.append(topic)
logger.error("降级失败:本地案例库为空")
# 5. 保存结果
self.save_results()