配置全面迁移数据库：PromptConfig、TaskConfig动态调度、敏感词/清洗规则/趋势映射/平台标签/痛点模板全部可编辑
- 新增 PromptConfig 模型 + API，支持提示词在线编辑（16条默认） - 调度器动态读取 TaskConfig.schedule，admin 可调执行时间 - 新增 KeywordDomainMap、SensitiveWord、ContentCleanRule、TrendFieldMapping 表 - DOMAINS、TREND_DOMAIN_MAP、PLATFORM_TAGS、china_pains、RSS关键词、priority_weights 全部迁移到 DB - tasks.html 重构：卡片网格+配置/产出/历史/提示词四个Tab，折叠显示 - 清理冗余代码：DEFAULT_PROMPTS死代码、collector.py unreachable代码、compliance_checker bug - strip_thinking_html 改用 DB 规则优先
This commit is contained in:
Yuzhiran Dev
+57
-37
@@ -39,7 +39,48 @@ logging.basicConfig(
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
DEFAULT_CHINA_PAINS = {
|
||||
"循环消费": "以旧换新流程繁琐、二手商品信任缺失、租赁市场不规范",
|
||||
"低碳出行": "新能源车充电设施不足、城市规划不支持骑行、通勤距离长",
|
||||
"干净饮食": "有机食品价格高、真伪难辨、外卖为主的生活方式难以改变",
|
||||
"零浪费生活": "环保产品溢价高、可持续选择不便、漂绿营销难以分辨",
|
||||
"绿色家电与节能": "绿色家电初期投入高、节能效果难量化、老旧小区改造难",
|
||||
"碳普惠": "碳账户普及率低、减排量兑换吸引力不足、公众认知有限",
|
||||
"环保科技产品": "绿色产品溢价68%难以承受、缺乏统一认证标准、担心漂绿",
|
||||
"AI与效率": "AI工具选择困难、数据隐私担忧、学习成本高、实际效果难验证"
|
||||
}
|
||||
|
||||
_cached_china_pains = None
|
||||
|
||||
def _load_china_pains():
|
||||
global _cached_china_pains
|
||||
if _cached_china_pains is not None:
|
||||
return _cached_china_pains
|
||||
try:
|
||||
from app.database import SessionLocal
|
||||
from app.models import CollectorCategory
|
||||
db = SessionLocal()
|
||||
try:
|
||||
cats = db.query(CollectorCategory).filter(
|
||||
CollectorCategory.is_active == True,
|
||||
CollectorCategory.pain_template.isnot(None),
|
||||
CollectorCategory.pain_template != ""
|
||||
).all()
|
||||
if cats:
|
||||
_cached_china_pains = {c.name: c.pain_template for c in cats}
|
||||
logger.info(f"从DB加载 {len(_cached_china_pains)} 个类别的pain_template")
|
||||
return _cached_china_pains
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"从DB加载 china_pains 失败: {e}")
|
||||
_cached_china_pains = DEFAULT_CHINA_PAINS
|
||||
return _cached_china_pains
|
||||
|
||||
|
||||
def _get_china_pain(category: str) -> str:
|
||||
pains = _load_china_pains()
|
||||
return pains.get(category, "中国相关数据不足,需本土化验证")
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -322,14 +363,8 @@ class SustainabilityCollector:
|
||||
if not content:
|
||||
content = title
|
||||
|
||||
# 关键词匹配(来源特定或全局)
|
||||
keywords = source_keywords if source_keywords else [
|
||||
'sustainable', 'green', 'eco', 'circular', 'climate', 'carbon',
|
||||
'zero waste', 'renewable', 'recycle', '环保', '可持续', '碳中和',
|
||||
'循环经济', '零浪费', '低碳', '生态'
|
||||
]
|
||||
|
||||
search_text = (title + content).lower()
|
||||
keywords = source_keywords if source_keywords else _load_rss_keywords()
|
||||
if any(keyword.lower() in search_text for keyword in keywords):
|
||||
articles.append({
|
||||
'title': title,
|
||||
@@ -445,6 +480,8 @@ class SustainabilityCollector:
|
||||
logger.warning("LLM不可用，跳过AI选题生成")
|
||||
return []
|
||||
|
||||
from prompt_loader import get_prompt, get_prompt_params
|
||||
|
||||
existing = self._get_existing_titles()
|
||||
existing_hint = ""
|
||||
if existing:
|
||||
@@ -461,28 +498,21 @@ class SustainabilityCollector:
|
||||
if cases:
|
||||
case_lines = [f"- {c.title[:40]}({c.category})" for c in cases[:5]]
|
||||
data_section += "\n采集案例：\n" + "\n".join(case_lines) + "\n"
|
||||
if not data_section:
|
||||
data_section = "（当前无实时采集数据，请基于你对中文互联网趋势的了解直接生成）"
|
||||
|
||||
trend_context = self._get_trend_context()
|
||||
|
||||
prompt = f"""你是一个内容策略师。基于以下信息，为「{target_category}」类别生成一个高质量选题。
|
||||
|
||||
{data_section if data_section else "（当前无实时采集数据，请基于你对中文互联网趋势的了解直接生成）"}
|
||||
{existing_hint}
|
||||
|
||||
{trend_context}
|
||||
|
||||
输出一个选题，格式JSON：
|
||||
{{{{
|
||||
"title": "标题（20字内，含核心关键词）",
|
||||
"core_concept": "核心观点（一句话）",
|
||||
"audience_pain": "受众痛点",
|
||||
"unique_angle": "差异化切入点",
|
||||
"format": "内容形式（趋势洞察/实操指南/对比分析/案例解读）"
|
||||
}}}}
|
||||
只输出JSON。"""
|
||||
prompt = get_prompt("topic_generate",
|
||||
target_category=target_category,
|
||||
data_section=data_section,
|
||||
existing_hint=existing_hint,
|
||||
trend_context=trend_context,
|
||||
)
|
||||
|
||||
try:
|
||||
resp = call_llm(prompt, temperature=0.7)
|
||||
params = get_prompt_params("topic_generate")
|
||||
resp = call_llm(prompt, temperature=params.get("temperature", 0.6), max_tokens=params.get("max_tokens", 2000))
|
||||
resp = resp.strip()
|
||||
if resp.startswith("```"):
|
||||
resp = resp.split("\n", 1)[1].rsplit("\n", 1)[0]
|
||||
@@ -570,18 +600,8 @@ class SustainabilityCollector:
|
||||
# 实际应用中可用AI提取,这里用前100字符
|
||||
core_idea = content[:200] if len(content) > 200 else content
|
||||
|
||||
# 生成中国痛点(基于类别模板)
|
||||
china_pains = {
|
||||
"循环消费": "以旧换新流程繁琐、二手商品信任缺失、租赁市场不规范",
|
||||
"低碳出行": "新能源车充电设施不足、城市规划不支持骑行、通勤距离长",
|
||||
"干净饮食": "有机食品价格高、真伪难辨、外卖为主的生活方式难以改变",
|
||||
"零浪费生活": "环保产品溢价高、可持续选择不便、漂绿营销难以分辨",
|
||||
"绿色家电与节能": "绿色家电初期投入高、节能效果难量化、老旧小区改造难",
|
||||
"碳普惠": "碳账户普及率低、减排量兑换吸引力不足、公众认知有限",
|
||||
"环保科技产品": "绿色产品溢价68%难以承受、缺乏统一认证标准、担心漂绿",
|
||||
"AI与效率": "AI工具选择困难、数据隐私担忧、学习成本高、实际效果难验证"
|
||||
}
|
||||
china_pain = china_pains.get(category, "中国相关数据不足,需本土化验证")
|
||||
# 生成中国痛点(基于类别模板，从DB读取pain_template)
|
||||
china_pain = _get_china_pain(category)
|
||||
|
||||
# 生成案例
|
||||
case = SustainabilityCase(
|
||||
|
||||
@@ -7,15 +7,13 @@
|
||||
import re
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
# 敏感词库（示例，需要持续更新）
|
||||
SENSITIVE_WORDS = {
|
||||
"政治敏感": ["国家主席", "政治局", "常委", "军委", "统战部", "颠覆国家", "分裂主义", "台独", "疆独", "藏独"],
|
||||
"违禁内容": ["赌博", "毒品", "迷药", "枪支", "炸药", "色情", "低俗", "反动", "邪教"],
|
||||
"不实信息": [" guaranteed 赚钱", "一夜暴富", "100%有效", "包治百病", "绝对正确"],
|
||||
"领导人相关": ["主席", "总理", "总书记", "国家领导人"] # 需上下文判断
|
||||
"领导人相关": ["主席", "总理", "总书记", "国家领导人"]
|
||||
}
|
||||
|
||||
# 平台规则限制
|
||||
PLATFORM_RULES = {
|
||||
"zhihu": {
|
||||
"max_title_len": 100,
|
||||
@@ -37,6 +35,70 @@ PLATFORM_RULES = {
|
||||
}
|
||||
}
|
||||
|
||||
_cached_sensitive_words = None
|
||||
_cached_platform_rules = None
|
||||
|
||||
def _load_sensitive_words():
|
||||
global _cached_sensitive_words
|
||||
if _cached_sensitive_words is not None:
|
||||
return _cached_sensitive_words
|
||||
|
||||
try:
|
||||
from app.core.prompt_loader import _get_session
|
||||
from app.models import SensitiveWord
|
||||
session = _get_session()
|
||||
try:
|
||||
rows = session.query(SensitiveWord).filter(SensitiveWord.is_active == True).all()
|
||||
if rows:
|
||||
result = {}
|
||||
for r in rows:
|
||||
cat = r.category or "general"
|
||||
if cat not in result:
|
||||
result[cat] = []
|
||||
result[cat].append(r.word)
|
||||
_cached_sensitive_words = result
|
||||
return _cached_sensitive_words
|
||||
finally:
|
||||
session.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
_cached_sensitive_words = SENSITIVE_WORDS
|
||||
return _cached_sensitive_words
|
||||
|
||||
|
||||
def _load_platform_rules():
|
||||
global _cached_platform_rules
|
||||
if _cached_platform_rules is not None:
|
||||
return _cached_platform_rules
|
||||
|
||||
try:
|
||||
from app.core.prompt_loader import _get_session
|
||||
from app.models import PlatformConfig
|
||||
import json
|
||||
session = _get_session()
|
||||
try:
|
||||
rows = session.query(PlatformConfig).all()
|
||||
if rows:
|
||||
result = {}
|
||||
for r in rows:
|
||||
try:
|
||||
cfg = json.loads(r.config_data) if r.config_data else {}
|
||||
except:
|
||||
cfg = {}
|
||||
if cfg:
|
||||
result[r.platform] = cfg
|
||||
if result:
|
||||
_cached_platform_rules = result
|
||||
return _cached_platform_rules
|
||||
finally:
|
||||
session.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
_cached_platform_rules = PLATFORM_RULES
|
||||
return _cached_platform_rules
|
||||
|
||||
class ComplianceChecker:
|
||||
"""合规审查器"""
|
||||
|
||||
@@ -90,7 +152,8 @@ class ComplianceChecker:
|
||||
|
||||
def _check_sensitive_words(self, text: str):
|
||||
"""检查敏感词"""
|
||||
for category, words in SENSITIVE_WORDS.items():
|
||||
words_map = _load_sensitive_words()
|
||||
for category, words in words_map.items():
|
||||
for word in words:
|
||||
if word in text:
|
||||
self.issues.append({
|
||||
@@ -102,7 +165,8 @@ class ComplianceChecker:
|
||||
|
||||
def _check_platform_rules(self, text: str, platform: str):
|
||||
"""检查平台特定规则"""
|
||||
rules = PLATFORM_RULES.get(platform, {})
|
||||
rules_map = _load_platform_rules()
|
||||
rules = rules_map.get(platform, {})
|
||||
|
||||
# 标题长度（从HTML中提取）
|
||||
max_title_len = self._get_platform_rule('max_title_len', rules.get("max_title_len"))
|
||||
@@ -244,7 +308,7 @@ class ComplianceChecker:
|
||||
"""检查文章最小字数（去除HTML标签）"""
|
||||
plain = re.sub(r'<[^>]+>', '', text)
|
||||
word_count = len(plain.strip())
|
||||
min_words = self._get_platform_rule('min_word_count', PLATFORM_RULES.get(platform, {}).get("min_word_count", 1000))
|
||||
min_words = self._get_platform_rule('min_word_count', 1000)
|
||||
if word_count < min_words:
|
||||
self.issues.append({
|
||||
"type": "内容完整度",
|
||||
|
||||
@@ -20,6 +20,8 @@ except ImportError:
|
||||
HAVE_LLM = False
|
||||
|
||||
from db_helper import get_topic_by_id, update_topic_status, get_active_llm_config, get_articles_by_topic, save_article
|
||||
from content_cleaner import strip_thinking, strip_ai_preface, strip_thinking_html, clean_html_content
|
||||
from prompt_loader import get_prompt, get_prompt_params
|
||||
|
||||
DATA_DIR = PROJECT_ROOT / "automation" / "data"
|
||||
DRAFTS_DIR = DATA_DIR / "drafts"
|
||||
@@ -30,11 +32,42 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
|
||||
handlers=[logging.FileHandler(LOGS_DIR / f"optimizer_{TODAY}.log"), logging.StreamHandler()])
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PLATFORM_TAGS = {
|
||||
DEFAULT_PLATFORM_TAGS = {
|
||||
"zhihu": ["科技", "职场"],
|
||||
"xiaohongshu": ["AI", "可持续", "生活方式"]
|
||||
}
|
||||
|
||||
_cached_platform_tags = None
|
||||
|
||||
def _load_platform_tags():
|
||||
global _cached_platform_tags
|
||||
if _cached_platform_tags is not None:
|
||||
return _cached_platform_tags
|
||||
try:
|
||||
from app.database import SessionLocal
|
||||
from app.models import PlatformConfig
|
||||
db = SessionLocal()
|
||||
try:
|
||||
configs = db.query(PlatformConfig).filter(PlatformConfig.is_active == True).all()
|
||||
if configs:
|
||||
_cached_platform_tags = {}
|
||||
for c in configs:
|
||||
tags = c.to_dict().get("allowed_tags", [])
|
||||
if tags:
|
||||
_cached_platform_tags[c.platform] = tags
|
||||
if _cached_platform_tags:
|
||||
logger.info(f"从DB加载 {len(_cached_platform_tags)} 个平台的标签")
|
||||
return _cached_platform_tags
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"从DB加载 platform_tags 失败: {e}")
|
||||
_cached_platform_tags = DEFAULT_PLATFORM_TAGS
|
||||
return _cached_platform_tags
|
||||
|
||||
def get_platform_tags():
|
||||
return _load_platform_tags()
|
||||
|
||||
_llm_config_cache = None
|
||||
|
||||
def get_llm_config():
|
||||
@@ -110,13 +143,14 @@ def fix_wechat_title(html: str, title: str) -> str:
|
||||
return html
|
||||
|
||||
def fix_tags(html: str, platform: str) -> str:
|
||||
tags_map = get_platform_tags()
|
||||
if platform == "zhihu":
|
||||
tags_str = " ".join(f"#{t}" for t in PLATFORM_TAGS["zhihu"])
|
||||
tags_str = " ".join(f"#{t}" for t in tags_map.get("zhihu", []))
|
||||
if '<div class="tags">' in html:
|
||||
old = html.split('<div class="tags">')[1].split('</div>')[0]
|
||||
html = html.replace(f'<div class="tags">{old}</div>', f'<div class="tags">{tags_str}</div>')
|
||||
elif platform == "xiaohongshu":
|
||||
tags_str = " ".join(f"#{t}" for t in PLATFORM_TAGS["xiaohongshu"])
|
||||
tags_str = " ".join(f"#{t}" for t in tags_map.get("xiaohongshu", []))
|
||||
if '<div class="hashtags">' in html:
|
||||
old = html.split('<div class="hashtags">')[1].split('</div>')[0]
|
||||
html = html.replace(f'<div class="hashtags">{old}</div>', f'<div class="hashtags">{tags_str}</div>')
|
||||
@@ -140,32 +174,13 @@ def polish_with_llm(html: str, platform: str, remaining_issues: Optional[List[Di
|
||||
f"- [{i['type']}] {i.get('category','')}: {i.get('detail','')} (建议: {i.get('suggestion','')})"
|
||||
for i in remaining_issues
|
||||
)
|
||||
polish_prompt = f"""你是一个专业的内容合规优化助手。以下文章存在合规问题，请逐一修复并输出完整HTML。
|
||||
|
||||
需修复的问题：
|
||||
{issues_desc}
|
||||
|
||||
原文：
|
||||
{html}
|
||||
|
||||
要求：
|
||||
- 只修复上述问题，不改变文章结构和核心内容
|
||||
- 保持<h2>, <h3>, <p>等标签结构不变
|
||||
- 修复后内容依然保持可读性和自然语感（不要因为合规变成生硬的表达）
|
||||
- 替换敏感词时选择意思相近的替代词，不删节重要信息"""
|
||||
prompt = get_prompt("compliance_fix", issues_desc=issues_desc, html=html)
|
||||
else:
|
||||
polish_prompt = f"""你是一个专业的内容润色助手。请润色以下文章，提升表达的自然感和可读性。
|
||||
|
||||
原文：
|
||||
{html}
|
||||
|
||||
要求：
|
||||
- 保持原文事实、数据、章节结构不变
|
||||
- 输出相同的HTML格式（保留<h2>, <h3>, <p>标签）
|
||||
- 提升表达的自然感，让它更像是人写的
|
||||
- 避免AI常见表达模式（「首先其次最后」「总的来说」「值得注意的是」等）
|
||||
- 短句化，读起来更流畅"""
|
||||
polished = call_llm(polish_prompt, model=model, temperature=temperature, max_tokens=max_tokens, system_prompt=system_prompt)
|
||||
prompt = get_prompt("compliance_polish", html=html)
|
||||
polished = call_llm(prompt, model=model, temperature=temperature, max_tokens=max_tokens, system_prompt=system_prompt)
|
||||
polished = clean_html_content(polished)
|
||||
polished = strip_ai_preface(polished)
|
||||
polished = strip_thinking_html(polished)
|
||||
if '<h2' in polished or '<p>' in polished:
|
||||
if len(polished) > len(html) * 0.3 and len(polished) > 100:
|
||||
if not any(kw in polished[:100] for kw in ['保留', '建议', '可以', '应该', '推荐']):
|
||||
@@ -178,6 +193,7 @@ def polish_with_llm(html: str, platform: str, remaining_issues: Optional[List[Di
|
||||
|
||||
def optimize_article(html: str, platform: str, topic_data: Dict, remaining_issues: Optional[List[Dict]] = None) -> Tuple[str, List[str]]:
|
||||
logs = []
|
||||
html = strip_thinking_html(html)
|
||||
if platform == "wechat":
|
||||
html = fix_wechat_title(html, topic_data.get("title", ""))
|
||||
logs.append("标题截断（含后缀）")
|
||||
@@ -185,7 +201,8 @@ def optimize_article(html: str, platform: str, topic_data: Dict, remaining_issue
|
||||
before = html
|
||||
html = fix_tags(html, platform)
|
||||
if html != before:
|
||||
logs.append(f"标签标准化为{PLATFORM_TAGS[platform]}")
|
||||
tags_map = get_platform_tags()
|
||||
logs.append(f"标签标准化为{tags_map.get(platform, [])}")
|
||||
polished, pol_log = polish_with_llm(html, platform, remaining_issues)
|
||||
if pol_log:
|
||||
html = polished
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
内容清洗工具集：所有 AI 思考内容/噪音段落的清洗逻辑集中管理
|
||||
各脚本（writer/outline/compliance_optimizer）统一引用此模块
|
||||
"""
|
||||
import re
|
||||
from typing import List, Tuple
|
||||
|
||||
THINKING_PATTERNS: List[str] = [
|
||||
r'^(好的|好的，|好[之，]|我来|让我|我将|我这就).*?(?=\n|$)',
|
||||
r'^(以下|下面是|这是|为您|根据).*?(?=\n|$)',
|
||||
r'^基于.*?(?=\n|$)',
|
||||
r'^【.*?】',
|
||||
r'^这里.*?(?=\n|$)',
|
||||
r'\n+希望[这以].*?$',
|
||||
r'\n+如果.*?$',
|
||||
r'\n+若有.*?$',
|
||||
r'\n+如有.*?$',
|
||||
r'\n+\*\*免责.*?$',
|
||||
r'^(这是按照要求|我已按|根据您的要求|^首先|^其次|^最后|^补充|^完成后).*?(?=\n|$)',
|
||||
r'^(以下是|下面为|这是完整|已按要求|已完成|处理完成).*?(?=\n|$)',
|
||||
]
|
||||
|
||||
AI_PREFACE_PATTERNS: List[str] = [
|
||||
r'^(好的[，,]?|好的 |我来|让我|我将|我这就|以下|下面|这是|为您|基于)',
|
||||
r'^(这是按照要求|我已按|根据您的要求|^首先|^其次|^最后|^补充|^完成后|以下是|下面为|这是完整|已按要求|已完成)',
|
||||
]
|
||||
|
||||
AI_VERBAL_PATTERNS: List[str] = [
|
||||
r'^(首先|其次|最后)(，|,)?',
|
||||
r'^总的来说',
|
||||
r'^值得注意的是',
|
||||
r'^换句话说',
|
||||
r'^总而言之',
|
||||
r'^简而言之',
|
||||
r'^一言以蔽之',
|
||||
r'^可以说',
|
||||
r'^不难发现',
|
||||
r'^由此可见',
|
||||
r'^综上所述',
|
||||
r'^通过以上',
|
||||
]
|
||||
|
||||
_cached_clean_rules = None
|
||||
|
||||
def _load_clean_rules():
|
||||
global _cached_clean_rules
|
||||
if _cached_clean_rules is not None:
|
||||
return _cached_clean_rules
|
||||
|
||||
try:
|
||||
from app.core.prompt_loader import _get_session
|
||||
from app.models import ContentCleanRule
|
||||
session = _get_session()
|
||||
try:
|
||||
rows = session.query(ContentCleanRule).filter(ContentCleanRule.is_active == True).order_by(ContentCleanRule.sort_order).all()
|
||||
if rows:
|
||||
result = {"thinking": [], "preface": [], "verbosity": [], "html_thinking": []}
|
||||
for r in rows:
|
||||
rule_type = r.rule_type or "thinking"
|
||||
if rule_type in result:
|
||||
result[rule_type].append(r.pattern)
|
||||
_cached_clean_rules = result
|
||||
return _cached_clean_rules
|
||||
finally:
|
||||
session.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
_cached_clean_rules = {
|
||||
"thinking": THINKING_PATTERNS,
|
||||
"preface": AI_PREFACE_PATTERNS,
|
||||
"verbosity": AI_VERBAL_PATTERNS,
|
||||
"html_thinking": [],
|
||||
}
|
||||
return _cached_clean_rules
|
||||
|
||||
|
||||
def _get_thinking_patterns() -> List[str]:
|
||||
rules = _load_clean_rules()
|
||||
return rules.get("thinking", THINKING_PATTERNS)
|
||||
|
||||
|
||||
def _get_preface_patterns() -> List[str]:
|
||||
rules = _load_clean_rules()
|
||||
return rules.get("preface", AI_PREFACE_PATTERNS)
|
||||
|
||||
|
||||
def _get_verbal_patterns() -> List[str]:
|
||||
rules = _load_clean_rules()
|
||||
return rules.get("verbosity", AI_VERBAL_PATTERNS)
|
||||
|
||||
|
||||
def strip_thinking(text: str) -> str:
|
||||
"""清洗 AI 思考前缀/后缀（正则替换，支持纯文本和 HTML 内联）"""
|
||||
for pat in _get_thinking_patterns():
|
||||
text = re.sub(pat, '', text, flags=re.MULTILINE)
|
||||
return text.strip()
|
||||
|
||||
def strip_thinking_html(html: str) -> str:
|
||||
"""清洗 HTML 中的 AI 思考段落（处理 <p>/<div> 包裹的情况）"""
|
||||
rules = _load_clean_rules()
|
||||
patterns = rules.get("html_thinking", [])
|
||||
if not patterns:
|
||||
patterns = [
|
||||
r'<p[^>]*>(好的|好的，|好[的，]|我来|让我|我将|我这就|以下|下面|这是|为您|基于|这是按照要求|我已按|根据您的要求|^首先|^其次|^最后|^补充|^完成后|以下是|下面为|这是完整|已按要求|已完成).*?</p>',
|
||||
r'<div[^>]*>(好的|好的，|好[的，]|我来|让我|我将|我这就|以下|下面|这是|为您|基于|这是按照要求|我已按|根据您的要求|^首先|^其次|^最后|^补充|^完成后|以下是|下面为|这是完整|已按要求|已完成).*?</div>',
|
||||
r'<p[^>]*>首先.*?</p>',
|
||||
r'<p[^>]*>其次.*?</p>',
|
||||
r'<p[^>]*>最后.*?</p>',
|
||||
r'<p[^>]*>(总的来说|值得注意的是|换句话说|总而言之|简而言之|一言以蔽之|可以说|不难发现|由此可见|综上所述).*?</p>',
|
||||
r'<div[^>]*>(总的来说|值得注意的是|换句话说|总而言之|简而言之|一言以蔽之|可以说|不难发现|由此可见|综上所述).*?</div>',
|
||||
]
|
||||
for pat in patterns:
|
||||
html = re.sub(pat, '', html, flags=re.IGNORECASE)
|
||||
return html
|
||||
|
||||
def strip_ai_preface(text: str) -> str:
|
||||
"""清洗以 AI 自述开头的整段说明文字（含代码围栏块）"""
|
||||
lines = text.split('\n')
|
||||
result = []
|
||||
skip_mode = False
|
||||
code_start = re.compile(r'^```')
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if skip_mode:
|
||||
if code_start.match(stripped):
|
||||
skip_mode = False
|
||||
continue
|
||||
should_skip = False
|
||||
for pat in _get_preface_patterns():
|
||||
if re.match(pat, stripped):
|
||||
should_skip = True
|
||||
break
|
||||
if should_skip:
|
||||
if code_start.match(stripped) or '```' in stripped:
|
||||
skip_mode = True
|
||||
continue
|
||||
result.append(line)
|
||||
return '\n'.join(result).strip()
|
||||
|
||||
def strip_ai_verbosity(text: str) -> str:
|
||||
"""清洗正文中常见的 AI 套话段落"""
|
||||
lines = text.split('\n')
|
||||
result = []
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
skip = False
|
||||
for pat in _get_verbal_patterns():
|
||||
if re.match(pat, stripped):
|
||||
skip = True
|
||||
break
|
||||
if not skip:
|
||||
result.append(line)
|
||||
return '\n'.join(result).strip()
|
||||
|
||||
def clean_markdown_content(text: str) -> str:
|
||||
"""清洗 markdown 正文：去思考内容 + 去 AI 套话 + 去格式噪音"""
|
||||
text = strip_thinking(text)
|
||||
text = strip_ai_preface(text)
|
||||
text = strip_ai_verbosity(text)
|
||||
lines = text.split('\n')
|
||||
cleaned = []
|
||||
in_code = False
|
||||
for line in lines:
|
||||
if line.strip().startswith('```'):
|
||||
in_code = not in_code
|
||||
continue
|
||||
if in_code:
|
||||
continue
|
||||
line = re.sub(r'^#{1,6}\s+', '', line)
|
||||
line = re.sub(r'^[\-\*\+]\s+', '', line)
|
||||
line = re.sub(r'^\d+[\.\)]\s+', '', line)
|
||||
line = re.sub(r'\*{1,3}([^*]+)\*{1,3}', r'\1', line)
|
||||
cleaned.append(line)
|
||||
return '\n'.join(cleaned).strip()
|
||||
|
||||
def clean_html_content(html: str) -> str:
|
||||
"""清洗 HTML 输出：去 markdown 代码围栏头尾 + 去 AI 思考注释"""
|
||||
html = re.sub(r'^```+\w*\s*\n?', '', html)
|
||||
html = html.strip()
|
||||
html = re.sub(r'\n?```+\s*$', '', html)
|
||||
html = strip_thinking_html(html)
|
||||
return html
|
||||
|
||||
def clean_full_pipeline(text: str, output_format: str = 'markdown') -> str:
|
||||
"""
|
||||
完整清洗流程：
|
||||
- markdown 输入：先去思考前缀 → 再去格式噪音 → 再转 HTML
|
||||
- html 输入：直接去代码围栏 + 思考注释
|
||||
"""
|
||||
if output_format == 'html':
|
||||
return clean_html_content(text)
|
||||
return clean_markdown_content(text)
|
||||
|
||||
def get_statistics(text: str) -> dict:
|
||||
"""返回清洗前后的行数/字数统计（用于日志）"""
|
||||
original_lines = len(text.split('\n'))
|
||||
original_chars = len(text)
|
||||
cleaned = strip_thinking(text)
|
||||
cleaned = strip_ai_preface(cleaned)
|
||||
cleaned = strip_ai_verbosity(cleaned)
|
||||
cleaned_lines = len(cleaned.split('\n'))
|
||||
cleaned_chars = len(cleaned)
|
||||
return {
|
||||
'original_lines': original_lines,
|
||||
'cleaned_lines': cleaned_lines,
|
||||
'original_chars': original_chars,
|
||||
'cleaned_chars': cleaned_chars,
|
||||
'dropped_lines': original_lines - cleaned_lines,
|
||||
'dropped_chars': original_chars - cleaned_chars,
|
||||
}
|
||||
@@ -16,8 +16,10 @@ sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
LOGS_DIR = PROJECT_ROOT / "automation" / "logs"
|
||||
TODAY = datetime.datetime.now().strftime("%Y-%m-%d")
|
||||
LOG_FILE = LOGS_DIR / f"opencode_search_{TODAY}.log"
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[logging.FileHandler(LOG_FILE, encoding='utf-8'), logging.StreamHandler()])
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SEARCH_CACHE_FILE = PROJECT_ROOT / "automation" / "data" / "search_cache.json"
|
||||
|
||||
+13
-50
@@ -12,6 +12,7 @@ sys.path.insert(0, str(PROJECT_ROOT))
|
||||
sys.path.insert(0, str(PROJECT_ROOT / "platform" / "backend"))
|
||||
|
||||
from db_helper import get_topic_by_id
|
||||
from prompt_loader import get_prompt, get_prompt_params
|
||||
try:
|
||||
from app.core.nvidia_client import call_llm
|
||||
HAVE_LLM = True
|
||||
@@ -52,57 +53,19 @@ class Outliner:
|
||||
|
||||
if HAVE_LLM:
|
||||
_now = datetime.datetime.now()
|
||||
prompt = f"""你是一个资深内容编辑，擅长设计读者爱看+搜索引擎友好+平台愿意推荐+有市场传播力的文章结构。
|
||||
|
||||
⚠️ 今天日期：{_now.strftime('%Y年%m月%d日')}。当前年份：{_now.year}年。
|
||||
|
||||
## 选题信息
|
||||
标题：{title}
|
||||
领域：{field}
|
||||
核心观点：{core}
|
||||
受众痛点：{pain}
|
||||
独特视角：{angle}
|
||||
|
||||
## 研究笔记
|
||||
{cases_summary}
|
||||
|
||||
## 大纲设计要求
|
||||
### 结构
|
||||
- 5-8章，每章2-4个要点
|
||||
- 结构要有递进：要么认知升级型，要么问题解决型
|
||||
- 把独特视角和受众痛点融入各章，不单独列
|
||||
- 每章标题自带信息量+好奇心，不要「引言」「总结」这类通用标题
|
||||
- 开头要有"钩子"（hook）抓住读者，结尾要有可转发/收藏的总结
|
||||
|
||||
### 数据与热点
|
||||
- **全文必须使用{_now.year-1}-{_now.year}年最新数据**，禁用一切过时数据
|
||||
- 每个观点必须配最新的国内外热点事件/数据/政策来佐证
|
||||
- 体现当前行业正在讨论的核心议题，拒绝泛泛而谈
|
||||
|
||||
### 独特风格
|
||||
- 有自己的判断和立场，不是搬运观点
|
||||
- 每章至少一个"反常识"或"很少有人提"的洞察
|
||||
- 避免同质化表达、老生常谈
|
||||
|
||||
### SEO
|
||||
- H2/H3自然包含用户搜索时会用的短语
|
||||
- 确保大纲覆盖2-3个高价值搜索词，包含1个长尾词
|
||||
- 每章标题对用户搜索意图有回应
|
||||
|
||||
### 市场价值
|
||||
- 读完每章读者能拿走一个实际有用的东西（方法/清单/思维框架/判断标准）
|
||||
- 避免「信息增量为零」的空洞章节
|
||||
- 思考：这篇文章对读者职业/生活/认知有什么用？
|
||||
|
||||
### 平台推荐优化
|
||||
- 知乎：偏硬核数据分析和深度逻辑，结构要有论证链
|
||||
- 小红书：偏实操步骤/清单/对比，结构要一目了然
|
||||
- 公众号：偏故事化开头+情感共鸣+金句结尾，段落节奏快
|
||||
- 一个框架适应三平台，各平台可裁剪侧重点
|
||||
|
||||
直接输出大纲，不要输出思考过程。"""
|
||||
prompt = get_prompt("outline_generation",
|
||||
date=_now.strftime('%Y年%m月%d日'),
|
||||
year=_now.year,
|
||||
title=title,
|
||||
field=field,
|
||||
core=core,
|
||||
pain=pain,
|
||||
angle=angle,
|
||||
cases_summary=cases_summary,
|
||||
)
|
||||
try:
|
||||
outline = call_llm(prompt, temperature=0.6, system_prompt="你是一个有经验的内容编辑，擅长为不同选题设计差异化的文章结构。")
|
||||
params = get_prompt_params("outline_generation")
|
||||
outline = call_llm(prompt, temperature=params.get("temperature", 0.7), max_tokens=params.get("max_tokens", 4000), system_prompt="你是一个有经验的内容编辑，擅长为不同选题设计差异化的文章结构。")
|
||||
logger.info(f"LLM 大纲生成成功，长度：{len(outline)}")
|
||||
return f"# 文章大纲：{title}\n\n{outline}\n\n---\n*大纲生成时间：{TODAY}*"
|
||||
except Exception as e:
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import os, sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
PROJECT_ROOT = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
sys.path.insert(0, str(PROJECT_ROOT / 'platform' / 'backend'))
|
||||
|
||||
_PROMPT_DEFAULTS = {
|
||||
"topics_trends": {
|
||||
"content": "你是中文互联网趋势分析师。请列出今天（{date}）中文互联网上最值得创作的10个话题。\n\n要求：\n1. 覆盖领域：{domains}\n2. 从真实用户角度出发\n3. 每个话题需包含：\n - \"domain\": 领域\n - \"topic\": 话题名称\n - \"reason\": 为什么现在讨论这个（1句话，有具体事件/数据支撑）\n - \"hot_keywords\": 3-5个搜索词（含1-2个长尾词）\n - \"platform\": 最适合分发的平台（知乎/小红书/微信/多平台）\n - \"seo_angle\": 从什么角度切入能获得搜索流量（1句话）\n - \"engagement\": 高/中/低\n\n输出 JSON 数组。只输出 JSON，不要其他文字。",
|
||||
"temperature": 0.7, "max_tokens": 3000,
|
||||
"variables": ["date", "domains"],
|
||||
},
|
||||
"topic_generate": {
|
||||
"content": "你是一个内容策略师。基于以下信息，为「{target_category}」类别生成一个高质量选题。\n\n{data_section}\n{existing_hint}\n{trend_context}\n\n输出一个选题，格式JSON：\n{\n \"title\": \"标题（20字内，含核心关键词）\",\n \"core_concept\": \"核心观点（一句话）\",\n \"audience_pain\": \"受众痛点\",\n \"unique_angle\": \"差异化切入点\",\n \"format\": \"内容形式（趋势洞察/实操指南/对比分析/案例解读）\"\n}\n只输出JSON。",
|
||||
"temperature": 0.6, "max_tokens": 2000,
|
||||
"variables": ["target_category", "data_section", "existing_hint", "trend_context"],
|
||||
},
|
||||
"topic_selector_gaps": {
|
||||
"content": "你是一个敏锐的内容策略师，擅长将热点转化为有价值、有传播力的选题。以下热点当前未覆盖，请为每个热点生成选题建议。\n\n{gaps}\n\n每个选题需包含：\n- \"title\": 标题（20字内，包含核心关键词，有吸引力）\n- \"field\": 所属领域\n- \"core_concept\": 核心观点（一句话说清独特价值）\n- \"audience_pain\": 受众痛点（真实用户的困惑/焦虑/需求）\n- \"unique_angle\": 独特视角（差异化切入点，含SEO关键词潜力）\n- \"target_platform\": 最适合发布平台（知乎/小红书/微信/多平台）\n- \"estimated_search_volume\": 预估搜索热度（高/中/低）\n\n只输出 JSON 数组，不要其他文字。",
|
||||
"temperature": 0.6, "max_tokens": 2000,
|
||||
"variables": ["gaps"],
|
||||
},
|
||||
"section_expansion": {
|
||||
"content": "你是一个资深作者，正在写一篇关于「{topic_title}」的文章。请写「{section_title}」这一节。\n\n今天日期：{date}。\n\n笔记要点：\n{content}\n\n【输出要求】\n输出3-6段纯粹、流畅的段落文字，每节内容根据平台需求控制在200-800字之间。\n\n格式：\n- 禁止任何标题/列表/格式标记（#、-、*、1.、**等）\n- 每段3-5句，段间空行分隔\n- 用「你」或「我们」视角，自然口语化\n\n内容要求（让文章在各平台能被推荐）：\n- 开头直接切入痛点或反常识观点，抓住注意力\n- 每个观点配具体案例或数据（用「据统计」「调研显示」等），不要空泛说理\n- 有独特判断和立场，避免正确废话\n- 回答「所以呢」——读者看完能带走什么\n- 结尾有情绪感召力，让人想点赞/收藏/转发\n\n直接输出段落正文，不要任何附加说明。",
|
||||
"temperature": 0.75, "max_tokens": 3000,
|
||||
"variables": ["topic_title", "section_title", "date", "content"],
|
||||
},
|
||||
"title_optimize_zhihu": {
|
||||
"content": "你是一个知乎内容专家。为以下文章起3个高点击率标题。\n\n标题：{title}\n核心观点：{core}\n受众痛点：{pain}\n领域：{field}\n\n要求：\n- 信息密度高，SEO关键词靠前\n- 偏好数字、对比、悬念、痛点类标题\n- 20字以内\n- 不要「如何...」开头\n- 有独特视角和差异化\n- 能引发讨论\n\n输出3个选项，格式：\n1. 标题A\n2. 标题B\n3. 标题C\n只输出标题，不要其他文字。",
|
||||
"temperature": 0.8, "max_tokens": 1500,
|
||||
"variables": ["title", "core", "pain", "field"],
|
||||
},
|
||||
"title_optimize_wechat": {
|
||||
"content": "你是一个公众号资深作者。为以下文章起3个10万+潜力标题。\n\n标题：{title}\n核心观点：{core}\n\n要求：\n- 制造好奇心和话题感\n- 包含微信SEO关键词\n- 口语化，避免感叹号堆砌\n- 15-25字\n- 有情感共鸣或争议性\n\n输出3个选项，格式：\n1. 标题A\n2. 标题B\n3. 标题C\n只输出标题，不要其他文字。",
|
||||
"temperature": 0.8, "max_tokens": 1500,
|
||||
"variables": ["title", "core"],
|
||||
},
|
||||
"title_optimize_xhs": {
|
||||
"content": "你是一个小红书爆款专家。为以下文章起3个热门标题。\n\n标题：{title}\n核心观点：{core}\n\n要求：\n- 20字以内\n- 爆款模式：数字+结果 / 痛点+方案 / 反常识\n- 包含小红书SEO关键词\n- 1个精确emoji\n- 有场景感、结果感、满足感\n- 不要「必看/收藏/码住」\n\n输出3个选项，格式：\n1. 标题A\n2. 标题B\n3. 标题C\n只输出标题，不要其他文字。",
|
||||
"temperature": 0.8, "max_tokens": 1500,
|
||||
"variables": ["title", "core"],
|
||||
},
|
||||
"research_summary": {
|
||||
"content": "你是一个行业研究员+内容策略师，擅长从案例中发现真洞察+抢占热点的敏锐嗅觉，能判断什么内容对真实读者最有价值且正被市场热议。\n\n今天是{now}。当前年份：{year}年。\n\n基于以下选题和相关案例，写出能支撑文章核心观点、对读者真正有用的研究发现。\n\n## 选题\n标题：{title}\n领域：{field}\n核心观点：{core}\n受众痛点：{pain}\n独特视角：{angle}\n{search_section}\n## 相关案例（{n}个）\n{cases_text}\n\n## 输出要求（按顺序）：\n1. **国内外最新热点关联**：... **所有数据必须是{year-1}-{year}年最新数据，禁用一切过时数据**\n2. **核心发现**：2-3个真正有价值的洞察。每条需包含这个发现对读者意味着什么，以及支撑数据（附数据来源）\n3. **独特观点储备**：哪些角度别人没写过、可以讲出差异化？提供至少一个反向/冷门视角\n4. **SEO关键词建议**：...\n5. **讨论点**：哪个观点最有争议或最可能引发讨论/转发/评论？\n6. **市场价值判断**：...\n\n风格：说人话，直击要点，像资深编辑在给作者做 briefing。避免「首先其次最后」「综上所述」。直接输出内容，不要输出思考过程。",
|
||||
"temperature": 0.7, "max_tokens": 4000,
|
||||
"variables": ["now", "year", "title", "field", "core", "pain", "angle", "search_section", "n", "cases_text"],
|
||||
},
|
||||
"outline_generation": {
|
||||
"content": "你是一个资深内容编辑，擅长设计读者爱看+搜索引擎友好+平台愿意推荐+有市场传播力的文章结构。\n\n今天是{date}。当前年份：{year}年。\n\n## 选题信息\n标题：{title}\n领域：{field}\n核心观点：{core}\n受众痛点：{pain}\n独特视角：{angle}\n\n## 研究笔记\n{cases_summary}\n\n## 大纲设计要求\n### 结构\n- 5-8章，每章2-4个要点\n- 结构要有递进：要么认知升级型，要么问题解决型\n- 把独特视角和受众痛点融入各章，不单独列\n- 每章标题自带信息量+好奇心，不要「引言」「总结」这类通用标题\n- 开头要有\"钩子\"（hook）抓住读者，结尾要有可转发/收藏的总结\n\n### 数据与热点\n- **全文必须使用{year-1}-{year}年最新数据**，禁用一切过时数据\n- 每个观点必须配最新的国内外热点事件/数据/政策来佐证\n- 体现当前行业正在讨论的核心议题，拒绝泛泛而谈\n\n### 独特风格\n- 有自己的判断和立场，不是搬运观点\n- 每章至少一个\"反常识\"或\"很少有人提\"的洞察\n- 避免同质化表达、老生常谈\n\n### SEO\n- H2/H3自然包含用户搜索时会用的短语\n- 确保大纲覆盖2-3个高价值搜索词，包含1个长尾词\n- 每章标题对用户搜索意图有回应\n\n### 市场价值\n- 读完每章读者能拿走一个实际有用的东西（方法/清单/思维框架/判断标准）\n- 避免「信息增量为零」的空洞章节\n\n直接输出大纲，不要输出思考过程。",
|
||||
"temperature": 0.7, "max_tokens": 4000,
|
||||
"variables": ["date", "year", "title", "field", "core", "pain", "angle", "cases_summary"],
|
||||
},
|
||||
"compliance_fix": {
|
||||
"content": "你是一个专业的内容合规优化助手。以下文章存在合规问题，请逐一修复并输出完整HTML。\n\n需修复的问题：\n{issues_desc}\n\n原文：\n{html}\n\n要求：\n- 只修复上述问题，不改变文章结构和核心内容\n- 保持<h2>, <h3>, <p>等标签结构不变\n- 修复后内容依然保持可读性和自然语感（不要因为合规变成生硬的表达）\n- 替换敏感词时选择意思相近的替代词，不删节重要信息",
|
||||
"temperature": 0.3, "max_tokens": 8000,
|
||||
"variables": ["issues_desc", "html"],
|
||||
},
|
||||
"compliance_polish": {
|
||||
"content": "你是一个专业的内容润色助手。请润色以下文章，提升表达的自然感和可读性。\n\n原文：\n{html}\n\n要求：\n- 保持原文事实、数据、章节结构不变\n- 输出相同的HTML格式（保留<h2>, <h3>, <p>标签）\n- 提升表达的自然感，让它更像是人写的\n- 避免AI常见表达模式（「首先其次最后」「总的来说」「值得注意的是」等）\n- 短句化，读起来更流畅",
|
||||
"temperature": 0.4, "max_tokens": 8000,
|
||||
"variables": ["html"],
|
||||
},
|
||||
"sources_optimization": {
|
||||
"content": "你是一个内容策略分析师。分析当前中文互联网可持续生活领域的真实热点，与以下配置进行对比。\n\n当前配置的类别（{n}个）：\n<cat_names>\n\n当前配置的信息源（{n2}个）：\n<src_summary>\n\n请完成以下任务：\n1. 评估每个类别是否仍符合{year}年中国市场真实热点\n2. 评估每个信息源是否可能在中国正常访问\n3. 建议新增或删除的类别（最多2条）\n4. 建议新增的信息源搜索词（最多3条，包含具体搜索词）\n\n输出 JSON 格式：\n{\n \"category_assessment\": [{\"name\": \"...\", \"status\": \"保留/淘汰/合并\", \"reason\": \"...\"}],\n \"source_assessment\": [{\"name\": \"...\", \"status\": \"保留/淘汰/替换\", \"reason\": \"...\"}],\n \"suggested_new_categories\": [{\"name\": \"...\", \"search_query\": \"...\", \"reason\": \"...\"}],\n \"suggested_new_sources\": [{\"name\": \"...\", \"type\": \"web_search\", \"query\": \"...\", \"focus\": \"...\"}],\n \"summary\": \"一句话总结本次优化建议\"\n}\n\n只输出json，不要其他文字。",
|
||||
"temperature": 0.5, "max_tokens": 3000,
|
||||
"variables": ["n", "cat_names", "n2", "src_summary", "year"],
|
||||
},
|
||||
"tags_generation": {
|
||||
"content": "为以下文章生成{platform}标签（3-5个）。\n\n标题：{title}\n领域：{field}\n核心观点：{core}\n\n要求：每个2-4字。直接输出标签，空格分隔。不要输出思考过程。",
|
||||
"temperature": 0.3, "max_tokens": 500,
|
||||
"variables": ["platform", "title", "field", "core"],
|
||||
},
|
||||
}
|
||||
|
||||
_DB_CACHE: Dict[str, Dict[str, Any]] = {}
|
||||
_CACHE_LOADED = False
|
||||
|
||||
|
||||
def _ensure_db_loaded():
|
||||
global _DB_CACHE, _CACHE_LOADED
|
||||
if _CACHE_LOADED:
|
||||
return
|
||||
try:
|
||||
if os.getenv('USE_POSTGRES', 'true') == 'true':
|
||||
from app.database import SessionLocal
|
||||
from app.models import PromptConfig
|
||||
db = SessionLocal()
|
||||
try:
|
||||
for p in db.query(PromptConfig).filter(PromptConfig.enabled == True).all():
|
||||
_DB_CACHE[p.key] = {
|
||||
"content": p.content,
|
||||
"temperature": p.temperature,
|
||||
"max_tokens": p.max_tokens,
|
||||
}
|
||||
finally:
|
||||
db.close()
|
||||
except Exception:
|
||||
pass
|
||||
_CACHE_LOADED = True
|
||||
|
||||
|
||||
def get_prompt(key: str, **kwargs) -> str:
|
||||
_ensure_db_loaded()
|
||||
if key in _DB_CACHE:
|
||||
content = _DB_CACHE[key]["content"]
|
||||
elif key in _PROMPT_DEFAULTS:
|
||||
content = _PROMPT_DEFAULTS[key]["content"]
|
||||
else:
|
||||
return ""
|
||||
for k, v in kwargs.items():
|
||||
content = content.replace("{" + k + "}", str(v))
|
||||
return content
|
||||
|
||||
|
||||
def get_prompt_params(key: str) -> Dict[str, Any]:
|
||||
_ensure_db_loaded()
|
||||
if key in _DB_CACHE:
|
||||
return {
|
||||
"temperature": _DB_CACHE[key].get("temperature"),
|
||||
"max_tokens": _DB_CACHE[key].get("max_tokens"),
|
||||
}
|
||||
if key in _PROMPT_DEFAULTS:
|
||||
return {
|
||||
"temperature": _PROMPT_DEFAULTS[key].get("temperature"),
|
||||
"max_tokens": _PROMPT_DEFAULTS[key].get("max_tokens"),
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
def reload_prompts():
|
||||
global _CACHE_LOADED, _DB_CACHE
|
||||
_CACHE_LOADED = False
|
||||
_DB_CACHE = {}
|
||||
_ensure_db_loaded()
|
||||
@@ -22,7 +22,7 @@ TODAY = __import__('datetime').datetime.now().strftime("%Y-%m-%d")
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TREND_DOMAIN_MAP = {
|
||||
DEFAULT_TREND_DOMAIN_MAP = {
|
||||
"远程工作": "未来工作方式",
|
||||
"AI工具": "AI与效率",
|
||||
"可持续生活": "可持续生活系统",
|
||||
@@ -37,11 +37,38 @@ TREND_DOMAIN_MAP = {
|
||||
"家庭教育": "科技人文交叉",
|
||||
}
|
||||
|
||||
_cached_trend_domain_map = None
|
||||
|
||||
def _load_trend_domain_map():
|
||||
global _cached_trend_domain_map
|
||||
if _cached_trend_domain_map is not None:
|
||||
return _cached_trend_domain_map
|
||||
try:
|
||||
from app.core.prompt_loader import _get_session
|
||||
from app.models import TrendFieldMapping
|
||||
session = _get_session()
|
||||
try:
|
||||
rows = session.query(TrendFieldMapping).filter(TrendFieldMapping.is_active == True).order_by(TrendFieldMapping.sort_order).all()
|
||||
if rows:
|
||||
_cached_trend_domain_map = {r.trend_keyword: r.field_name for r in rows}
|
||||
logger.info(f"从DB加载 {len(_cached_trend_domain_map)} 条 trend_domain_map")
|
||||
return _cached_trend_domain_map
|
||||
finally:
|
||||
session.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"从DB加载 trend_domain_map 失败: {e}")
|
||||
_cached_trend_domain_map = DEFAULT_TREND_DOMAIN_MAP
|
||||
return _cached_trend_domain_map
|
||||
|
||||
def get_trend_domain_map():
|
||||
return _load_trend_domain_map()
|
||||
|
||||
def _topic_trend_score(topic: Dict, trend: Dict) -> float:
|
||||
field = (topic.get("field") or "").lower()
|
||||
title = (topic.get("title") or "").lower()
|
||||
core = (topic.get("core_concept") or "").lower()
|
||||
trend_domain = TREND_DOMAIN_MAP.get(trend.get("domain", ""), "")
|
||||
trend_domain_map = get_trend_domain_map()
|
||||
trend_domain = trend_domain_map.get(trend.get("domain", ""), "")
|
||||
keywords = [trend.get("topic", "")] + trend.get("hot_keywords", [])
|
||||
score = 0.0
|
||||
if trend_domain and trend_domain in field:
|
||||
|
||||
+59
-21
@@ -26,7 +26,34 @@ logging.basicConfig(
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DOMAINS = ["远程工作", "AI工具", "可持续生活", "知识管理", "数字生活", "科技人文"]
|
||||
DEFAULT_DOMAINS = ["远程工作", "AI工具", "可持续生活", "知识管理", "数字生活", "科技人文"]
|
||||
_cached_domains = None
|
||||
|
||||
def _load_domains():
|
||||
global _cached_domains
|
||||
if _cached_domains is not None:
|
||||
return _cached_domains
|
||||
try:
|
||||
from app.core.prompt_loader import _get_session
|
||||
from app.models import SystemConfig
|
||||
session = _get_session()
|
||||
try:
|
||||
sc = session.query(SystemConfig).filter(SystemConfig.key == "trend_domains").first()
|
||||
if sc and sc.value:
|
||||
parsed = json.loads(sc.value)
|
||||
if isinstance(parsed, list) and parsed:
|
||||
_cached_domains = parsed
|
||||
logger.info(f"从DB加载 {len(_cached_domains)} 个trend_domains")
|
||||
return _cached_domains
|
||||
finally:
|
||||
session.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"从DB加载 trend_domains 失败: {e}")
|
||||
_cached_domains = DEFAULT_DOMAINS
|
||||
return _cached_domains
|
||||
|
||||
def get_domains():
|
||||
return _load_domains()
|
||||
|
||||
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
|
||||
@@ -39,10 +66,35 @@ _KEYWORD_DOMAIN_MAP = [
|
||||
(r"科技|人文|教育|心理|哲学|社会学", "科技人文"),
|
||||
]
|
||||
|
||||
_cached_keyword_domain_map = None
|
||||
|
||||
def _load_keyword_domain_map():
|
||||
global _cached_keyword_domain_map
|
||||
if _cached_keyword_domain_map is not None:
|
||||
return _cached_keyword_domain_map
|
||||
|
||||
try:
|
||||
from app.core.prompt_loader import _get_session
|
||||
from app.models import KeywordDomainMap
|
||||
session = _get_session()
|
||||
try:
|
||||
rows = session.query(KeywordDomainMap).filter(KeywordDomainMap.is_active == True).order_by(KeywordDomainMap.sort_order).all()
|
||||
if rows:
|
||||
_cached_keyword_domain_map = [(r.pattern, r.domain) for r in rows]
|
||||
logger.info(f"从DB加载 {len(rows)} 条 keyword_domain_map 规则")
|
||||
return _cached_keyword_domain_map
|
||||
finally:
|
||||
session.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"从DB加载 keyword_domain_map 失败: {e}")
|
||||
|
||||
_cached_keyword_domain_map = _KEYWORD_DOMAIN_MAP
|
||||
return _cached_keyword_domain_map
|
||||
|
||||
|
||||
def _guess_domain(topic: str, reason: str = "") -> str:
|
||||
text = (topic + " " + reason).lower()
|
||||
for pattern, domain in _KEYWORD_DOMAIN_MAP:
|
||||
for pattern, domain in _load_keyword_domain_map():
|
||||
if re.search(pattern, text, re.IGNORECASE):
|
||||
return domain
|
||||
return "科技人文"
|
||||
@@ -186,26 +238,12 @@ def fetch_baidu_hot() -> List[Dict]:
|
||||
|
||||
|
||||
def fetch_llm_trends() -> List[Dict]:
|
||||
prompt = f"""你是中文互联网趋势分析师。请列出今天（2026年5月）中文互联网上最值得创作的10个话题。
|
||||
|
||||
要求：
|
||||
1. 覆盖领域：{', '.join(DOMAINS)}
|
||||
2. 从真实用户角度出发
|
||||
3. 每个话题需包含：
|
||||
- "domain": 领域
|
||||
- "topic": 话题名称
|
||||
- "reason": 为什么现在讨论这个（1句话，有具体事件/数据支撑）
|
||||
- "hot_keywords": 3-5个搜索词（含1-2个长尾词）
|
||||
- "platform": 最适合分发的平台（知乎/小红书/微信/多平台）
|
||||
- "seo_angle": 从什么角度切入能获得搜索流量（1句话）
|
||||
- "engagement": 高/中/低
|
||||
|
||||
输出 JSON 数组：
|
||||
[{{"domain": "...", "topic": "...", "reason": "...", "hot_keywords": ["..."], "platform": "...", "seo_angle": "...", "engagement": "..."}}]
|
||||
|
||||
只输出 JSON，不要其他文字。"""
|
||||
try:
|
||||
resp = call_llm(prompt, temperature=0.4)
|
||||
from prompt_loader import get_prompt, get_prompt_params
|
||||
prompt = get_prompt("topics_trends", date=datetime.datetime.now().strftime("%Y年%m月%d"),
|
||||
domains=", ".join(get_domains()))
|
||||
params = get_prompt_params("topics_trends")
|
||||
resp = call_llm(prompt, temperature=params.get("temperature", 0.4), max_tokens=params.get("max_tokens", 2000))
|
||||
resp = resp.strip()
|
||||
if resp.startswith("```"):
|
||||
resp = resp.split("\n", 1)[1].rsplit("\n", 1)[0]
|
||||
|
||||
+44
-100
@@ -12,6 +12,8 @@ sys.path.insert(0, str(PROJECT_ROOT))
|
||||
sys.path.insert(0, str(PROJECT_ROOT / 'platform' / 'backend'))
|
||||
|
||||
from db_helper import get_topic_by_id, update_topic_status, save_article
|
||||
from content_cleaner import strip_thinking, strip_ai_preface, clean_markdown_content, clean_html_content
|
||||
from prompt_loader import get_prompt, get_prompt_params
|
||||
try:
|
||||
from app.core.nvidia_client import call_llm
|
||||
HAVE_LLM = True
|
||||
@@ -116,21 +118,7 @@ class Writer:
|
||||
|
||||
@staticmethod
|
||||
def _clean_markdown(text: str) -> str:
|
||||
lines = text.split('\n')
|
||||
cleaned = []
|
||||
in_code_fence = False
|
||||
for line in lines:
|
||||
if line.strip().startswith('```'):
|
||||
in_code_fence = not in_code_fence
|
||||
continue
|
||||
if in_code_fence:
|
||||
continue
|
||||
line = re.sub(r'^#{1,6}\s+', '', line)
|
||||
line = re.sub(r'^[\-\*\+]\s+', '', line)
|
||||
line = re.sub(r'^\d+[\.\)]\s+', '', line)
|
||||
line = re.sub(r'\*{1,3}([^*]+)\*{1,3}', r'\1', line)
|
||||
cleaned.append(line)
|
||||
return '\n'.join(cleaned).strip()
|
||||
return clean_markdown_content(text)
|
||||
|
||||
@staticmethod
|
||||
def _is_outline_noise(line: str) -> bool:
|
||||
@@ -160,31 +148,15 @@ class Writer:
|
||||
# 大纲要点格式（>40% 行以 -/*/** 开头）应始终由 LLM 展开为连贯段落
|
||||
if HAVE_LLM and self._is_bullet_only(content):
|
||||
logger.info(f"使用 LLM 扩写章节(要点→段落): {section['title']}")
|
||||
prompt = f"""你是一个资深作者，正在写一篇关于「{self.topic['title']}」的文章。请写「{section['title']}」这一节。
|
||||
|
||||
今天日期：{datetime.datetime.now().strftime('%Y年%m月%d日')}。
|
||||
|
||||
笔记要点：
|
||||
{content}
|
||||
|
||||
【输出要求】
|
||||
输出3-6段纯粹、流畅的段落文字，每节内容根据平台需求控制在200-800字之间。
|
||||
|
||||
格式：
|
||||
- 禁止任何标题/列表/格式标记（#、-、*、1.、**等）
|
||||
- 每段3-5句，段间空行分隔
|
||||
- 用「你」或「我们」视角，自然口语化
|
||||
|
||||
内容要求（让文章在各平台能被推荐）：
|
||||
- 开头直接切入痛点或反常识观点，抓住注意力
|
||||
- 每个观点配具体案例或数据（用「据统计」「调研显示」等），不要空泛说理
|
||||
- 有独特判断和立场，避免正确废话
|
||||
- 回答「所以呢」——读者看完能带走什么
|
||||
- 结尾有情绪感召力，让人想点赞/收藏/转发
|
||||
|
||||
直接输出段落正文，不要任何附加说明。"""
|
||||
prompt = get_prompt("section_expansion",
|
||||
topic_title=self.topic['title'],
|
||||
section_title=section['title'],
|
||||
date=datetime.datetime.now().strftime('%Y年%m月%d日'),
|
||||
content=content,
|
||||
)
|
||||
try:
|
||||
expanded = call_llm(prompt, temperature=0.6)
|
||||
params = get_prompt_params("section_expansion")
|
||||
expanded = call_llm(prompt, temperature=params.get("temperature", 0.75), max_tokens=params.get("max_tokens", 3000))
|
||||
if expanded:
|
||||
cleaned = self._clean_markdown(expanded.strip())
|
||||
if cleaned:
|
||||
@@ -284,15 +256,17 @@ class Writer:
|
||||
core = self.topic.get('core_concept', '')
|
||||
|
||||
tag_prompts = {
|
||||
"zhihu": f"为以下文章生成知乎标签（3-5个）。标题：{title} 领域：{field} 核心观点：{core} 每个2-4字。直接输出标签，空格分隔。不要输出思考过程。",
|
||||
"wechat": f"为以下文章生成公众号标签（3-5个）。标题：{title} 领域：{field} 核心观点：{core} 每个2-4字。直接输出标签，空格分隔。不要输出思考过程。",
|
||||
"xiaohongshu": f"为以下文章生成小红书标签（3-5个）。标题：{title} 领域：{field} 核心观点：{core} 每个2-4字。直接输出标签，空格分隔。不要输出思考过程。",
|
||||
"zhihu": get_prompt("tags_generation", platform="知乎", title=title, field=field, core=core),
|
||||
"wechat": get_prompt("tags_generation", platform="公众号", title=title, field=field, core=core),
|
||||
"xiaohongshu": get_prompt("tags_generation", platform="小红书", title=title, field=field, core=core),
|
||||
}
|
||||
|
||||
if HAVE_LLM:
|
||||
prompt = tag_prompts.get(platform, f"根据文章信息生成适合{platform}的标签。标题：{title} 领域：{field} 核心观点：{core} 直接输出标签，空格分隔。")
|
||||
prompt = tag_prompts.get(platform, get_prompt("tags_generation", platform=platform, title=title, field=field, core=core))
|
||||
try:
|
||||
tags_text = call_llm(prompt, temperature=0.2)
|
||||
params = get_prompt_params("tags_generation")
|
||||
tags_text = call_llm(prompt, temperature=params.get("temperature", 0.3), max_tokens=params.get("max_tokens", 500))
|
||||
tags_text = strip_thinking(tags_text)
|
||||
if tags_text:
|
||||
tags = [t.strip('#') for t in tags_text.strip().split() if t.strip('#')]
|
||||
if tags:
|
||||
@@ -331,64 +305,33 @@ class Writer:
|
||||
if not HAVE_LLM:
|
||||
return original
|
||||
|
||||
title_templates = {
|
||||
"zhihu": f"""你是一个知乎用户，在给自己的深度回答起高点击率标题。
|
||||
if platform == "zhihu":
|
||||
prompt = get_prompt("title_optimize_zhihu",
|
||||
title=original,
|
||||
core=self.topic.get('core_concept', ''),
|
||||
pain=self.topic.get('audience_pain', ''),
|
||||
field=self.topic.get('field', ''),
|
||||
)
|
||||
elif platform == "wechat":
|
||||
prompt = get_prompt("title_optimize_wechat",
|
||||
title=original,
|
||||
core=self.topic.get('core_concept', ''),
|
||||
)
|
||||
elif platform == "xiaohongshu":
|
||||
prompt = get_prompt("title_optimize_xhs",
|
||||
title=original,
|
||||
core=self.topic.get('core_concept', ''),
|
||||
)
|
||||
else:
|
||||
prompt = f"给以下文章改个吸引人的{platform}标题：{original}"
|
||||
|
||||
原文标题：{original}
|
||||
领域：{self.topic.get('field', '')}
|
||||
|
||||
要求：
|
||||
- 有信息量：一看就知道能解决什么问题
|
||||
- 含知乎搜索关键词（SEO），利用知乎搜索联想热词
|
||||
- 带数字或对比最好（「3个方法」「从…到…」）
|
||||
- 20字以内
|
||||
- 参考知乎真实高赞标题风格，不要套路句式
|
||||
- 避免「如何…」废句式、「XXX指南/手册/全攻略」
|
||||
- 有观点、有态度，不是中性描述
|
||||
- 直击目标读者痛点或好奇心
|
||||
- 直接输出3个标题选项，每行一个，不要输出思考过程
|
||||
|
||||
生成 3 个选项，每行一个。""",
|
||||
|
||||
"wechat": f"""你是一个公众号作者，在给可能10万+的文章起标题。
|
||||
|
||||
原文标题：{original}
|
||||
领域：{self.topic.get('field', '')}
|
||||
|
||||
要求：
|
||||
- 制造好奇心和点击欲，让人觉得不点开会错过
|
||||
- 包含微信搜索关键词（微信SEO），利用搜一搜热门词
|
||||
- 口语化，不要书面腔
|
||||
- 不要感叹号堆砌，不要「重磅/震惊/紧急」
|
||||
- 字数15-25字最佳
|
||||
- 有情绪感召力：共鸣/好奇/焦虑/期待
|
||||
- 参考近期10万+标题的语气节奏
|
||||
- 直接输出3个标题选项，每行一个，不要输出思考过程
|
||||
|
||||
生成 3 个选项，每行一个。""",
|
||||
|
||||
"xiaohongshu": f"""你是一个小红书用户，在给笔记起能上热门推荐的标题。
|
||||
|
||||
原文标题：{original}
|
||||
领域：{self.topic.get('field', '')}
|
||||
|
||||
要求：
|
||||
- 20字以内
|
||||
- 采用爆款模式：数字+结果/痛点+方案/反常识观点/对比式
|
||||
- 包含小红书搜索关键词（SEO），利用搜索下拉热词
|
||||
- 带1个精准emoji点缀，不要三个起堆
|
||||
- 有场景感/结果感/获得感
|
||||
- 不要「必看/收藏/码住」
|
||||
- 像真实用户写的，不是运营写的
|
||||
- 参考小红书搜索热榜标题风格
|
||||
- 直接输出3个标题选项，每行一个，不要输出思考过程
|
||||
|
||||
生成 3 个选项，每行一个。""",
|
||||
}
|
||||
|
||||
prompt = title_templates.get(platform, f"给以下文章改个吸引人的{platform}标题：{original}")
|
||||
try:
|
||||
resp = call_llm(prompt, temperature=0.7)
|
||||
if platform in ("zhihu", "wechat", "xiaohongshu"):
|
||||
params = get_prompt_params(f"title_optimize_{platform}")
|
||||
resp = call_llm(prompt, temperature=params.get("temperature", 0.8), max_tokens=params.get("max_tokens", 1500))
|
||||
else:
|
||||
resp = call_llm(prompt, temperature=0.7)
|
||||
resp = strip_thinking(resp)
|
||||
titles = []
|
||||
for line in resp.strip().split('\n'):
|
||||
line = line.strip()
|
||||
@@ -414,6 +357,7 @@ class Writer:
|
||||
else:
|
||||
template = "<!DOCTYPE html><html><head><meta charset='UTF-8'><title>{{TITLE}}</title><meta name='viewport' content='width=device-width'><style>body{max-width:800px;margin:0 auto;padding:20px;font-family:-apple-system,sans-serif;line-height:1.8}</style></head><body><h1>{{TITLE}}</h1><!-- CONTENT --></body></html>"
|
||||
|
||||
adapted = strip_ai_preface(adapted)
|
||||
html = template.replace("{{TITLE}}", title).replace("{{DATE}}", TODAY).replace("{{GEN_TIME}}", GEN_TIME)
|
||||
html_content = _md_parser(adapted)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user