feat: 内容数据迁移至数据库，合规审查全链路打通
- 文章 HTML 存储从文件系统迁移至 articles 表，删除 releases 目录 - 合规审查从 DB 读取 HTML，审查结果写回 DB，通过后自动推进至待发布 - 新增 todayCount 筛选按钮，与系统概览统计数据一致 - 全屏预览修复：提升 z-index 超过侧边栏，添加退出全屏/关闭按钮 - 统一 '优化' → '审查' 命名，消除前后端术语不一致 - 调度器创作完成后自动触发审查（生成 → 审查 → 待发布） - 清理旧备份/调试文件、过期大纲和研究笔记
This commit is contained in:
Yuzhiran Dev
+138
-131
@@ -1,36 +1,23 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
合规审查与优化任务（数据库版）
|
||||
每天 05:45 运行，处理当天所有 draft 文章：
|
||||
1. 执行合规检查（compliance_checker）
|
||||
2. 自动修复已知问题（标题、标签）
|
||||
3. 重写合规版本
|
||||
4. 更新选题状态为「待发布」
|
||||
5. 生成优化报告通知
|
||||
"""
|
||||
|
||||
import json, datetime, logging, sys, re
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from dataclasses import dataclass, asdict
|
||||
|
||||
PROJECT_ROOT = Path('/root/openclaw-workspace/projects/yu-zhi-ran')
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
from scripts.compliance_checker import check_article
|
||||
|
||||
# 导入 LLM 客户端（合规优化使用 NVIDIA）
|
||||
sys.path.insert(0, str(PROJECT_ROOT / "platform" / "backend"))
|
||||
|
||||
from scripts.compliance_checker import check_article
|
||||
try:
|
||||
from app.core.modelscope_client import call_llm
|
||||
from app.core.nvidia_client import call_llm
|
||||
HAVE_LLM = True
|
||||
except ImportError:
|
||||
HAVE_LLM = False
|
||||
|
||||
# 导入数据库辅助模块
|
||||
from db_helper import get_topic_by_id, update_topic_status
|
||||
from db_helper import get_topic_by_id, update_topic_status, get_active_llm_config, get_articles_by_topic, save_article
|
||||
|
||||
DATA_DIR = PROJECT_ROOT / "automation" / "data"
|
||||
RELEASES_DIR = DATA_DIR / "releases"
|
||||
DRAFTS_DIR = DATA_DIR / "drafts"
|
||||
LOGS_DIR = PROJECT_ROOT / "automation" / "logs"
|
||||
TODAY = datetime.datetime.now().strftime("%Y-%m-%d")
|
||||
@@ -39,12 +26,19 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
|
||||
handlers=[logging.FileHandler(LOGS_DIR / f"optimizer_{TODAY}.log"), logging.StreamHandler()])
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 平台白名单标签
|
||||
PLATFORM_TAGS = {
|
||||
"zhihu": ["科技", "职场"],
|
||||
"xiaohongshu": ["AI", "可持续", "生活方式"]
|
||||
}
|
||||
|
||||
_llm_config_cache = None
|
||||
|
||||
def get_llm_config():
|
||||
global _llm_config_cache
|
||||
if _llm_config_cache is None:
|
||||
_llm_config_cache = get_active_llm_config()
|
||||
return _llm_config_cache
|
||||
|
||||
@dataclass
|
||||
class OptimizationResult:
|
||||
file: str
|
||||
@@ -57,22 +51,40 @@ class OptimizationResult:
|
||||
status: str
|
||||
|
||||
def load_topic_map():
|
||||
"""从数据库加载所有选题数据"""
|
||||
from db_helper import export_topics_to_json
|
||||
topics = export_topics_to_json()
|
||||
return {t['id']: t for t in topics}
|
||||
|
||||
def update_topic_status_db_only(topic_id: str, status: str):
|
||||
"""仅更新数据库状态（不更新JSON）"""
|
||||
from db_helper import update_topic_status
|
||||
update_topic_status(topic_id, status)
|
||||
def get_articles_from_db(topic_ids: Optional[List[str]] = None) -> List[Tuple[str, str, str]]:
|
||||
"""从 articles 表读取 HTML 内容
|
||||
|
||||
Returns: [(html_content, platform, topic_id), ...]
|
||||
"""
|
||||
from db_helper import get_articles_by_topic
|
||||
results = []
|
||||
seen_topics = set()
|
||||
if topic_ids:
|
||||
for tid in topic_ids:
|
||||
articles = get_articles_by_topic(tid)
|
||||
for a in articles:
|
||||
if a.get("html_content"):
|
||||
results.append((a["html_content"], a["platform"], a["topic_id"]))
|
||||
seen_topics.add(a["topic_id"])
|
||||
else:
|
||||
from app.database import SessionLocal
|
||||
from app.models import Article
|
||||
db = SessionLocal()
|
||||
try:
|
||||
all_articles = db.query(Article).filter(Article.html_content.isnot(None)).all()
|
||||
for a in all_articles:
|
||||
results.append((a.html_content, a.platform, a.topic_id))
|
||||
finally:
|
||||
db.close()
|
||||
return results
|
||||
|
||||
def fix_wechat_title(html: str, title: str) -> str:
|
||||
"""微信标题优化：<title>和<h1>都控制长度（考虑后缀）"""
|
||||
suffix = f" - {TODAY} - 微信公众号"
|
||||
max_base_len = 32 - len(suffix) # <title> 中 base 部分允许的最大长度
|
||||
|
||||
# 处理 <title>...</title>
|
||||
max_base_len = 32 - len(suffix)
|
||||
title_tag = re.search(r'<title>([^<]+)</title>', html)
|
||||
if title_tag:
|
||||
full_title = title_tag.group(1)
|
||||
@@ -84,8 +96,6 @@ def fix_wechat_title(html: str, title: str) -> str:
|
||||
base = base[:max_base_len-3] + "..."
|
||||
new_full = base + suffix
|
||||
html = html.replace(full_title, new_full)
|
||||
|
||||
# 处理 <h1>...</h1>
|
||||
h1_match = re.search(r'<h1[^>]*>([^<]+)</h1>', html)
|
||||
if h1_match:
|
||||
current_h1 = h1_match.group(1)
|
||||
@@ -93,11 +103,9 @@ def fix_wechat_title(html: str, title: str) -> str:
|
||||
if len(base_h1) > 32:
|
||||
base_h1 = base_h1[:29] + "..."
|
||||
html = html.replace(current_h1, base_h1)
|
||||
|
||||
return html
|
||||
|
||||
def fix_tags(html: str, platform: str) -> str:
|
||||
"""强制替换标签为平台白名单"""
|
||||
if platform == "zhihu":
|
||||
tags_str = " ".join(f"#{t}" for t in PLATFORM_TAGS["zhihu"])
|
||||
if '<div class="tags">' in html:
|
||||
@@ -110,7 +118,31 @@ def fix_tags(html: str, platform: str) -> str:
|
||||
html = html.replace(f'<div class="hashtags">{old}</div>', f'<div class="hashtags">{tags_str}</div>')
|
||||
return html
|
||||
|
||||
def optimize_article(html: str, platform: str, topic_data: Dict) -> (str, List[str]):
|
||||
def polish_with_llm(html: str, platform: str) -> Tuple[str, Optional[str]]:
|
||||
"""用 LLM 优化文章内容，返回 (html, log_message_or_None)"""
|
||||
if not HAVE_LLM:
|
||||
return html, None
|
||||
llm_cfg = get_llm_config()
|
||||
try:
|
||||
model = llm_cfg.get('model') if llm_cfg else None
|
||||
temperature = llm_cfg.get('temperature', 0.5) if llm_cfg else 0.5
|
||||
max_tokens = llm_cfg.get('max_tokens', 4000) if llm_cfg else 4000
|
||||
system_prompt = llm_cfg.get('system_prompt') if llm_cfg else "你是一个专业的内容创作助手。"
|
||||
|
||||
polish_prompt = f"""你是一个专业的内容润色助手。请优化以下文章内容，提升表达的专业性和可读性，保持原文事实、数据、章节结构不变，输出相同的HTML格式（保留<h2>, <h3>, <p>标签）。
|
||||
|
||||
原文：
|
||||
{html}
|
||||
|
||||
优化后："""
|
||||
polished = call_llm(polish_prompt, model=model, temperature=temperature, max_tokens=max_tokens, system_prompt=system_prompt)
|
||||
if '<h2' in polished or '<p>' in polished:
|
||||
return polished, "LLM 内容优化"
|
||||
except Exception as e:
|
||||
logger.warning(f"LLM 优化失败: {e}")
|
||||
return html, None
|
||||
|
||||
def optimize_article(html: str, platform: str, topic_data: Dict) -> Tuple[str, List[str]]:
|
||||
logs = []
|
||||
if platform == "wechat":
|
||||
html = fix_wechat_title(html, topic_data.get("title", ""))
|
||||
@@ -127,124 +159,99 @@ def optimize_article(html: str, platform: str, topic_data: Dict) -> (str, List[s
|
||||
src = m.group(1)
|
||||
if not src.startswith('data:image/'):
|
||||
logs.append(f"图片未内联: {src[:50]}... 需手动修复")
|
||||
|
||||
if HAVE_LLM:
|
||||
try:
|
||||
polish_prompt = f"""你是一个专业的内容润色助手。请优化以下文章内容，提升表达的专业性和可读性，保持原文事实、数据、章节结构不变，输出相同的HTML格式（保留<h2>, <h3>, <p>标签）。
|
||||
|
||||
原文：
|
||||
{html}
|
||||
|
||||
优化后："""
|
||||
polished = call_llm(polish_prompt, temperature=0.5, max_tokens=4000)
|
||||
if '<h2' in polished or '<p>' in polished:
|
||||
html = polished
|
||||
logs.append("LLM 内容优化（NVIDIA）")
|
||||
except Exception as e:
|
||||
logger.warning(f"LLM 优化失败: {e}")
|
||||
polished, pol_log = polish_with_llm(html, platform)
|
||||
if pol_log:
|
||||
html = polished
|
||||
logs.append(pol_log)
|
||||
return html, logs
|
||||
|
||||
def main(topic_ids: List[str] = None):
|
||||
logger.info("=== 合规审查与优化开始 ===")
|
||||
release_dir = RELEASES_DIR / TODAY
|
||||
if not release_dir.exists():
|
||||
logger.warning(f"今日发布目录不存在: {release_dir}")
|
||||
return
|
||||
llm_cfg = get_llm_config()
|
||||
if llm_cfg:
|
||||
logger.info(f"LLM 配置: {llm_cfg['name']} (model={llm_cfg['model']})")
|
||||
else:
|
||||
logger.info("LLM 配置: 使用环境变量默认值")
|
||||
|
||||
articles = get_articles_from_db(topic_ids)
|
||||
if not articles:
|
||||
logger.warning("未找到任何文章（可能尚未创作或同步到 DB）")
|
||||
report_file = DRAFTS_DIR / TODAY / "optimization_report.json"
|
||||
report_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(report_file, 'w', encoding='utf-8') as f:
|
||||
json.dump({
|
||||
"date": TODAY,
|
||||
"summary": {"total_articles": 0, "passed_auto": 0, "need_manual": 0, "average_score": 0},
|
||||
"details": [], "all_passed": True
|
||||
}, f, ensure_ascii=False, indent=2)
|
||||
print("OPTIMIZATION_COMPLETE: 0 articles found")
|
||||
sys.exit(0)
|
||||
|
||||
topic_map = load_topic_map()
|
||||
results = []
|
||||
all_passed = True
|
||||
|
||||
for platform_dir in ["zhihu", "wechat", "xiaohongshu"]:
|
||||
platform_path = release_dir / platform_dir
|
||||
if not platform_path.exists():
|
||||
for html, platform_dir, topic_id in articles:
|
||||
topic_data = topic_map.get(topic_id)
|
||||
if not topic_data:
|
||||
logger.warning(f"未找到选题: {topic_id}")
|
||||
continue
|
||||
for html_file in platform_path.glob("*.html"):
|
||||
stem = html_file.stem
|
||||
parts = stem.split('_')
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
topic_id = parts[1]
|
||||
if topic_ids is not None and topic_id not in topic_ids:
|
||||
continue
|
||||
topic_data = topic_map.get(topic_id)
|
||||
if not topic_data:
|
||||
logger.warning(f"未找到选题: {topic_id}")
|
||||
continue
|
||||
|
||||
html = html_file.read_text(encoding='utf-8')
|
||||
check_result = check_article(html, platform_dir, topic_data)
|
||||
issues = check_result['issues']
|
||||
score = check_result['score']
|
||||
check_result = check_article(html, platform_dir, topic_data)
|
||||
issues = check_result['issues']
|
||||
score = check_result['score']
|
||||
label = f"{platform_dir}/{topic_id}"
|
||||
|
||||
if any(issue['type'] == '平台规则' for issue in issues):
|
||||
optimized_html, opt_logs = optimize_article(html, platform_dir, topic_data)
|
||||
recheck = check_article(optimized_html, platform_dir, topic_data)
|
||||
if recheck['passed']:
|
||||
html_file.write_text(optimized_html, encoding='utf-8')
|
||||
logger.info(f"✅ {html_file.name} 已优化并通过合规检查")
|
||||
results.append(OptimizationResult(
|
||||
file=str(html_file.relative_to(PROJECT_ROOT)),
|
||||
platform=platform_dir,
|
||||
topic_id=topic_id,
|
||||
title=topic_data.get('title',''),
|
||||
original_issues=len(issues),
|
||||
fixed_issues=len(issues) - len(recheck['issues']),
|
||||
final_score=recheck['score'],
|
||||
status="passed"
|
||||
))
|
||||
update_topic_status_db_only(topic_id, 'pending')
|
||||
else:
|
||||
logger.warning(f"⚠️ {html_file.name} 优化后仍有问题，需人工审核")
|
||||
results.append(OptimizationResult(
|
||||
file=str(html_file.relative_to(PROJECT_ROOT)),
|
||||
platform=platform_dir,
|
||||
topic_id=topic_id,
|
||||
title=topic_data.get('title',''),
|
||||
original_issues=len(issues),
|
||||
fixed_issues=len(issues) - len(recheck['issues']),
|
||||
final_score=recheck['score'],
|
||||
status="manual_review"
|
||||
))
|
||||
all_passed = False
|
||||
else:
|
||||
if issues:
|
||||
optimized_html, opt_logs = optimize_article(html, platform_dir, topic_data)
|
||||
recheck = check_article(optimized_html, platform_dir, topic_data)
|
||||
if recheck['passed']:
|
||||
save_article(topic_id, platform_dir, optimized_html)
|
||||
logger.info(f"✅ {label} 已修复并通过审查 ({len(issues)} issues fixed)")
|
||||
results.append(OptimizationResult(
|
||||
file=str(html_file.relative_to(PROJECT_ROOT)),
|
||||
file=f"db:{platform_dir}_{topic_id}",
|
||||
platform=platform_dir,
|
||||
topic_id=topic_id,
|
||||
title=topic_data.get('title',''),
|
||||
original_issues=0,
|
||||
fixed_issues=0,
|
||||
final_score=score,
|
||||
status="passed" if check_result['passed'] else "manual_review"
|
||||
original_issues=len(issues),
|
||||
fixed_issues=len(issues) - len(recheck['issues']),
|
||||
final_score=recheck['score'],
|
||||
status="passed"
|
||||
))
|
||||
if not check_result['passed']:
|
||||
all_passed = False
|
||||
else:
|
||||
logger.warning(f"⚠️ {label} 仍有 {len(recheck['issues'])} 个问题需人工处理")
|
||||
results.append(OptimizationResult(
|
||||
file=f"db:{platform_dir}_{topic_id}",
|
||||
platform=platform_dir,
|
||||
topic_id=topic_id,
|
||||
title=topic_data.get('title',''),
|
||||
original_issues=len(issues),
|
||||
fixed_issues=len(issues) - len(recheck['issues']),
|
||||
final_score=recheck['score'],
|
||||
status="manual_review"
|
||||
))
|
||||
all_passed = False
|
||||
else:
|
||||
results.append(OptimizationResult(
|
||||
file=f"db:{platform_dir}_{topic_id}",
|
||||
platform=platform_dir,
|
||||
topic_id=topic_id,
|
||||
title=topic_data.get('title',''),
|
||||
original_issues=0,
|
||||
fixed_issues=0,
|
||||
final_score=score,
|
||||
status="passed"
|
||||
))
|
||||
logger.info(f"✅ {label} 合规检查通过 ({score}分)")
|
||||
|
||||
# 更新选题状态 (JSON + 数据库)
|
||||
passed_ids = set()
|
||||
for res in results:
|
||||
if res.status == "passed":
|
||||
tid = res.topic_id
|
||||
# 更新数据库状态为 'pending'（待发布）
|
||||
update_topic_status(tid, 'ready')
|
||||
# 可选：同时更新 JSON 以保持兼容
|
||||
# (已废弃，但保留更新，避免其他组件出错)
|
||||
try:
|
||||
json_path = DATA_DIR / "sustainability_topics.json"
|
||||
with open(json_path, 'r', encoding='utf-8') as f:
|
||||
topics = json.load(f)
|
||||
for t in topics:
|
||||
if t.get('id') == tid:
|
||||
t['status'] = 'ready'
|
||||
t['ready_at'] = TODAY
|
||||
t['compliance_score'] = res.final_score
|
||||
break
|
||||
with open(json_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(topics, f, ensure_ascii=False, indent=2)
|
||||
except Exception as e:
|
||||
logger.warning(f"更新 JSON 失败: {e}")
|
||||
passed_ids.add(res.topic_id)
|
||||
for tid in passed_ids:
|
||||
update_topic_status(tid, 'ready')
|
||||
logger.info(f"选题 {tid} 状态 → ready（待发布）")
|
||||
|
||||
# 生成报告
|
||||
report_file = DRAFTS_DIR / TODAY / "optimization_report.json"
|
||||
report_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
report = {
|
||||
@@ -261,7 +268,7 @@ def main(topic_ids: List[str] = None):
|
||||
with open(report_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||||
|
||||
logger.info(f"✅ 合规优化完成: {len(results)} 篇文章, {sum(1 for r in results if r.status=='passed')} 篇自动通过")
|
||||
logger.info(f"✅ 合规审查完成: {len(results)} 篇文章, {sum(1 for r in results if r.status=='passed')} 篇通过")
|
||||
print(f"OPTIMIZATION_COMPLETE: {len(results)} articles, {sum(1 for r in results if r.status=='passed')} passed, {sum(1 for r in results if r.status=='manual_review')} need manual review")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user