feat: 内容数据迁移至数据库,合规审查全链路打通

- 文章 HTML 存储从文件系统迁移至 articles 表,删除 releases 目录
- 合规审查从 DB 读取 HTML,审查结果写回 DB,通过后自动推进至待发布
- 新增 todayCount 筛选按钮,与系统概览统计数据一致
- 全屏预览修复:提升 z-index 超过侧边栏,添加退出全屏/关闭按钮
- 统一 '优化' → '审查' 命名,消除前后端术语不一致
- 调度器创作完成后自动触发审查(生成 → 审查 → 待发布)
- 清理旧备份/调试文件、过期大纲和研究笔记
This commit is contained in:
Yuzhiran Dev
2026-05-13 17:33:56 +08:00
parent bc6a302e59
commit 233e23016c
234 changed files with 5670 additions and 10651 deletions
+62 -23
View File
@@ -17,6 +17,7 @@ def run_creator(topic_id: str = None):
Args:
topic_id: 可选,指定要创作的选题ID。不指定则创作优先级最高的选题。
"""
from datetime import datetime, timezone
script_path = PROJECT_ROOT / "scripts" / "creator.py"
venv_python = PROJECT_ROOT / "platform" / "backend" / "venv" / "bin" / "python"
if venv_python.exists():
@@ -30,32 +31,70 @@ def run_creator(topic_id: str = None):
cwd=str(PROJECT_ROOT),
capture_output=True,
text=True,
timeout=1800 # 30分钟超时,避免AI撰写超时
timeout=1800
)
if result.returncode != 0:
logger.error(f"Creator failed: {result.stderr}")
if topic_id:
try:
sync_topic_to_db(topic_id)
except Exception as e:
logger.warning(f"Sync after creation failed: {e}")
return {"ok": False, "error": result.stderr}
# 解析日志,找出选择了哪个选题
topic_id = None
for line in result.stdout.splitlines():
if "选择了选题:" in line:
# 格式: 2026-04-16 ... INFO - 选择了选题: 标题 (优先级: X)
# 标题可能在行内,但ID不一定有。我们稍后用文件同步。
logger.info(line.strip())
if "选题" in line and "已标记为「待发布」" in line:
# 如: 2026-04-16 ... INFO - 选题 A01 已标记为「待发布」
import re
from .sync import sync_topic_to_db
m = re.search(r'选题\s+([A-Za-z0-9]+)', line)
if m:
topic_id = m.group(1)
if not topic_id:
for line in result.stdout.splitlines():
if "选题" in line and "已标记为「待发布」" in line:
import re
m = re.search(r'选题\s+([A-Za-z0-9]+)', line)
if m:
topic_id = m.group(1)
if topic_id:
try:
from ..database import SessionLocal
from ..models import Topic, Article
db = SessionLocal()
topic = db.query(Topic).filter(Topic.id == topic_id).first()
if topic:
topic.generated_at = datetime.now(timezone.utc)
if topic.status in ('pending', '待处理'):
topic.status = 'review'
db.commit()
logger.info(f"Topic {topic_id} updated: generated_at set, status→{topic.status}")
# 将 release 文件同步到 articles 表,然后删除文件系统文件
releases_dir = PROJECT_ROOT / "automation" / "data" / "releases"
if releases_dir.exists():
for dd in sorted(releases_dir.iterdir(), reverse=True):
if not dd.is_dir() or not dd.name[:4].isdigit():
continue
for platform_dir in ["zhihu", "wechat", "xiaohongshu"]:
pdir = dd / platform_dir
if not pdir.exists():
continue
for hf in pdir.glob(f"*{topic_id}.html"):
html = hf.read_text(encoding='utf-8')
article_id = f"{platform_dir}_{topic_id}"
existing = db.query(Article).filter(Article.id == article_id).first()
if existing:
existing.html_content = html
else:
db.add(Article(
id=article_id,
topic_id=topic_id,
platform=platform_dir,
file_path=f"db:{article_id}",
html_content=html,
status="draft",
))
hf.unlink()
logger.info(f"Synced {hf.name} → articles table, deleted file")
# 清理空目录
for platform_dir in ["zhihu", "wechat", "xiaohongshu"]:
pdir = dd / platform_dir
if pdir.exists() and not any(pdir.iterdir()):
pdir.rmdir()
if dd.exists() and not any(dd.iterdir()):
dd.rmdir()
db.commit()
except Exception as e:
logger.warning(f"DB sync after creation failed: {e}")
return {
"ok": True,
"topic_id": topic_id,
@@ -1,118 +0,0 @@
""" ModelScope 专用 LLM 客户端 """
import requests
import json
from typing import Optional
class LLMError(Exception):
pass
# 临时使用 NVIDIA 端点(ModelScope Key 已失效)
CONFIG = {
"base_url": "https://integrate.api.nvidia.com/v1",
"api_key": "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",
"model": "stepfun-ai/step-3.5-flash",
}
def call_llm(
prompt: str,
system_prompt: str = "你是一个专业的内容创作助手。",
temperature: float = 0.7,
max_tokens: int = 2000,
stream: bool = False,
) -> str:
"""调用 ModelScope LLM 生成文本"""
endpoint = f"{CONFIG['base_url'].rstrip('/')}/chat/completions"
headers = {
"Authorization": f"Bearer {CONFIG['api_key']}",
"Content-Type": "application/json"
}
payload = {
"model": CONFIG["model"],
"messages": [
{"role": "system", "content": system_prompt},
{"role": "user", "content": prompt}
],
"temperature": temperature,
"max_tokens": max_tokens,
"stream": stream,
}
try:
resp = requests.post(endpoint, json=payload, headers=headers, timeout=120, stream=stream)
if resp.status_code != 200:
raise LLMError(f"HTTP {resp.status_code}: {resp.text[:200]}")
if stream:
full = []
for line in resp.iter_lines():
if not line:
continue
if line.startswith(b'data: '):
data = line[6:]
if data == b'[DONE]':
break
try:
chunk = json.loads(data)
delta = chunk['choices'][0]['delta']
if 'reasoning_content' in delta and delta['reasoning_content']:
full.append(delta['reasoning_content'])
if 'content' in delta and delta['content']:
full.append(delta['content'])
except Exception:
continue
return "".join(full)
else:
data = resp.json()
msg = data["choices"][0]["message"]
content = msg.get('content') or msg.get('reasoning') or msg.get('reasoning_content')
return content.strip() if content else ''
except requests.RequestException as e:
raise LLMError(f"Request failed: {e}")
def expand_content_with_llm(
topic: dict,
section_title: str,
section_content: str,
context: str = ""
) -> str:
"""扩写大纲章节,返回包含 ## 标题的完整 Markdown"""
prompt = f"""你是一个专业的内容创作者,风格精炼、直接、切中要点。请将以下大纲扩展为完整的文章章节,要求如下:
### 选题信息
标题:{topic.get('title')}
领域:{topic.get('field')}
核心观点:{topic.get('core_concept', '')}
受众痛点:{topic.get('audience_pain', '')}
独特视角:{topic.get('unique_angle', '')}
### 当前章节
## {section_title}
{section_content}
### 输出要求
- 以 "## {section_title}" 开始
- 字数:200-300 字(精炼为主)
- 语言:直白、有冲击力,避免空洞套话
- 使用 Markdown 格式
- 每个论点配具体案例或数据支撑
- 确保与整体文章调性一致
直接输出完整 Markdown 章节(包括标题和正文)。"""
if context:
prompt = f"# 参考资料\n{context}\n\n{prompt}"
try:
result = call_llm(prompt, temperature=0.8, max_tokens=1000)
return result.strip()
except Exception as e:
return f"## {section_title}\n\nLLM 调用失败:{e},请手动补充)"
# 测试
if __name__ == "__main__":
try:
print(f"[modelscope_client] 使用模型:{CONFIG['model']}")
resp = call_llm("你好,请用一句话介绍你自己。", max_tokens=50)
print(f"[modelscope_client] 响应:{resp}")
except Exception as e:
print(f"[modelscope_client] 错误:{e}")
+36 -54
View File
@@ -1,57 +1,50 @@
"""
NVIDIA 专用 LLM 客户端(优化配置)
支持 Google Gemma 和其他 NVIDIA 模型
Unified LLM Client
支持 NVIDIA / 兼容 OpenAI 格式的 API,配置从环境变量读取
"""
import os
import requests
import json
from typing import Optional, Dict, Any
from pathlib import Path
from dotenv import load_dotenv
env_path = Path(__file__).resolve().parents[2] / ".env"
load_dotenv(env_path)
class LLMError(Exception):
pass
# 固定配置(你的可用 key
CONFIG = {
"base_url": "https://integrate.api.nvidia.com/v1",
"api_key": "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",
"base_url": os.getenv("LLM_BASE_URL", "https://integrate.api.nvidia.com/v1"),
"api_key": os.getenv("LLM_API_KEY", ""),
"model": os.getenv("LLM_MODEL", "google/gemma-3n-e4b-it"),
}
def call_llm(
prompt: str,
model: str = "google/gemma-3n-e4b-it",
model: Optional[str] = None,
system_prompt: str = "你是一个专业的内容创作助手。",
temperature: float = 0.20,
max_tokens: int = 512,
max_tokens: int = 2048,
top_p: float = 0.70,
frequency_penalty: float = 0.00,
presence_penalty: float = 0.00,
stream: bool = False,
additional_params: Optional[Dict[str, Any]] = None,
) -> str:
"""
调用 NVIDIA LLM 生成文本
参数:
prompt: 用户提示词
model: 模型名称,默认 google/gemma-3n-e4b-it
system_prompt: 系统提示词
temperature: 温度参数 (0.0-2.0),默认 0.20
max_tokens: 最大生成 token 数,默认 512
top_p: 核采样参数 (0.0-1.0),默认 0.70
frequency_penalty: 频率惩罚 (0.0-2.0),默认 0.00
presence_penalty: 存在惩罚 (0.0-2.0),默认 0.00
stream: 是否流式输出,默认 False
additional_params: 额外参数(如 reasoning_effort
"""
if not CONFIG["api_key"]:
raise LLMError("LLM_API_KEY 未配置,请在 backend/.env 中设置")
endpoint = f"{CONFIG['base_url'].rstrip('/')}/chat/completions"
headers = {
"Authorization": f"Bearer {CONFIG['api_key']}",
"Content-Type": "application/json"
}
# 基础参数
payload = {
"model": model,
"model": model or CONFIG["model"],
"messages": [
{"role": "system", "content": system_prompt},
{"role": "user", "content": prompt}
@@ -63,11 +56,9 @@ def call_llm(
"presence_penalty": presence_penalty,
"stream": stream,
}
# 添加额外参数(如 reasoning_effort
if additional_params:
payload.update(additional_params)
try:
resp = requests.post(endpoint, json=payload, headers=headers, timeout=120, stream=stream)
if resp.status_code != 200:
@@ -75,22 +66,18 @@ def call_llm(
if stream:
full = []
for line in resp.iter_lines():
if not line:
continue
if not line: continue
if line.startswith(b'data: '):
data = line[6:]
if data == b'[DONE]':
break
if data == b'[DONE]': break
try:
chunk = json.loads(data)
delta = chunk['choices'][0]['delta']
# 支持 reasoning_content 或 reasoning 字段
if 'reasoning_content' in delta and delta['reasoning_content']:
if delta.get('reasoning_content'):
full.append(delta['reasoning_content'])
if 'content' in delta and delta['content']:
if delta.get('content'):
full.append(delta['content'])
except Exception:
continue
except Exception: continue
return "".join(full)
else:
data = resp.json()
@@ -100,9 +87,13 @@ def call_llm(
except requests.RequestException as e:
raise LLMError(f"Request failed: {e}")
def expand_content_with_llm(topic: dict, section_title: str, section_content: str, context: str = "") -> str:
"""扩写大纲章节,返回包含 ## 标题的完整 Markdown"""
prompt = f"""你是一个专业的内容创作者,风格精炼、直接、切中要点。请将以下大纲扩展为完整的文章章节,要求如下:
def expand_content_with_llm(
topic: dict,
section_title: str,
section_content: str,
context: str = ""
) -> str:
prompt = f"""你是一个专业的内容创作者,风格精炼、直接、切中要点。请将以下大纲扩展为完整的文章章节:
### 选题信息
标题:{topic.get('title')}
@@ -117,7 +108,7 @@ def expand_content_with_llm(topic: dict, section_title: str, section_content: st
### 输出要求
- 以 "## {section_title}" 开始
- 字数:200-300 字(精炼为主)
- 字数:200-300 字
- 语言:直白、有冲击力,避免空洞套话
- 使用 Markdown 格式
- 每个论点配具体案例或数据支撑
@@ -128,24 +119,15 @@ def expand_content_with_llm(topic: dict, section_title: str, section_content: st
prompt = f"# 参考资料\n{context}\n\n{prompt}"
try:
result = call_llm(
prompt,
model="google/gemma-3n-e4b-it",
temperature=0.20,
max_tokens=1000,
top_p=0.70,
frequency_penalty=0.00,
presence_penalty=0.00
)
result = call_llm(prompt, temperature=0.3, max_tokens=1500)
return result.strip()
except Exception as e:
return f"## {section_title}\n\nLLM 调用失败:{e},请手动补充)"
# 测试
if __name__ == "__main__":
try:
print(f"[nvidia_client] 使用模型: google/gemma-3n-e4b-it")
print(f"[nvidia_client] 模型:{CONFIG['model']}")
resp = call_llm("你好,请用一句话介绍你自己。", max_tokens=50)
print(f"[nvidia_client] 响应: {resp}")
print(f"[nvidia_client] 响应{resp}")
except Exception as e:
print(f"[nvidia_client] 错误: {e}")
print(f"[nvidia_client] 错误{e}")
+6 -2
View File
@@ -9,7 +9,7 @@ from typing import List
logger = logging.getLogger(__name__)
# 计算项目根目录(从本文件位置上升4层)
PROJECT_ROOT = Path(__file__).resolve().parents[1]
PROJECT_ROOT = Path(__file__).resolve().parents[4]
if os.getenv('PROJECT_ROOT'):
PROJECT_ROOT = Path(os.getenv('PROJECT_ROOT'))
@@ -20,7 +20,11 @@ def run_optimizer(topic_ids: List[str] = None):
topic_ids: 可选,指定要优化的选题ID列表。不指定则优化所有 draft 文章。
"""
script_path = PROJECT_ROOT / "scripts" / "compliance_optimizer.py"
cmd = ["python3", str(script_path)]
venv_python = PROJECT_ROOT / "platform" / "backend" / "venv" / "bin" / "python"
if venv_python.exists():
cmd = [str(venv_python), str(script_path)]
else:
cmd = ["python3", str(script_path)]
if topic_ids:
cmd.extend(["--topic-ids", ','.join(topic_ids)])
logger.info(f"[DEBUG] Running optimizer with topic_ids={topic_ids}, cmd={' '.join(cmd)}")
-112
View File
@@ -1,112 +0,0 @@
"""
qnaigc 专用 LLM 客户端
模型:arcee-ai/trinity-large-preview
"""
import requests
import json
from typing import Optional
class LLMError(Exception):
pass
CONFIG = {
"base_url": "https://api.qnaigc.com/v1",
"api_key": "sk-2cb9561a18351015d3120ffac4abae0480fa17e0d28469bdce5fc905d1a42e0d",
"model": "arcee-ai/trinity-large-preview",
}
def call_llm(
prompt: str,
system_prompt: str = "你是一个专业的内容创作助手。",
temperature: float = 0.7,
max_tokens: int = 2000,
stream: bool = False,
) -> str:
endpoint = f"{CONFIG['base_url'].rstrip('/')}/chat/completions"
headers = {
"Authorization": f"Bearer {CONFIG['api_key']}",
"Content-Type": "application/json"
}
payload = {
"model": CONFIG["model"],
"messages": [
{"role": "system", "content": system_prompt},
{"role": "user", "content": prompt}
],
"temperature": temperature,
"max_tokens": max_tokens,
"stream": stream,
}
try:
resp = requests.post(endpoint, json=payload, headers=headers, timeout=120, stream=stream)
if resp.status_code != 200:
raise LLMError(f"HTTP {resp.status_code}: {resp.text[:200]}")
if stream:
full = []
for line in resp.iter_lines():
if not line:
continue
if line.startswith(b'data: '):
data = line[6:]
if data == b'[DONE]':
break
try:
chunk = json.loads(data)
delta = chunk['choices'][0]['delta']
if 'reasoning_content' in delta and delta['reasoning_content']:
full.append(delta['reasoning_content'])
if 'content' in delta and delta['content']:
full.append(delta['content'])
except Exception:
continue
return "".join(full)
else:
data = resp.json()
msg = data["choices"][0]["message"]
content = msg.get('content') or msg.get('reasoning') or msg.get('reasoning_content')
return content.strip() if content else ''
except requests.RequestException as e:
raise LLMError(f"Request failed: {e}")
def expand_content_with_llm(topic: dict, section_title: str, section_content: str, context: str = "") -> str:
"""扩写大纲章节,返回包含 ## 标题的完整 Markdown"""
prompt = f"""你是一个专业的内容创作者,风格精炼、直接、切中要点。请将以下大纲扩展为完整的文章章节,要求如下:
### 选题信息
标题:{topic.get('title')}
领域:{topic.get('field')}
核心观点:{topic.get('core_concept', '')}
受众痛点:{topic.get('audience_pain', '')}
独特视角:{topic.get('unique_angle', '')}
### 当前章节
## {section_title}
{section_content}
### 输出要求
- 以 "## {section_title}" 开始
- 字数:200-300 字(精炼为主)
- 语言:直白、有冲击力,避免空洞套话
- 使用 Markdown 格式
- 每个论点配具体案例或数据支撑
- 确保与整体文章调性一致
直接输出完整 Markdown 章节(包括标题和正文)。"""
if context:
prompt = f"# 参考资料\n{context}\n\n{prompt}"
try:
result = call_llm(prompt, temperature=0.8, max_tokens=1000)
return result.strip()
except Exception as e:
return f"## {section_title}\n\nLLM 调用失败:{e},请手动补充)"
# 测试
if __name__ == "__main__":
try:
print(f"[qnaigc_client] 使用模型: {CONFIG['model']}")
resp = call_llm("你好,请用一句话介绍你自己。", max_tokens=50)
print(f"[qnaigc_client] 响应: {resp}")
except Exception as e:
print(f"[qnaigc_client] 错误: {e}")
+9 -1
View File
@@ -60,8 +60,16 @@ class TaskScheduler:
logger.info("[Scheduled] Starting content generation...")
result = run_creator()
logger.info("[Scheduled] Generation completed: %s", result)
created_id = result.get("topic_id") if isinstance(result, dict) else None
if created_id:
logger.info("[Scheduled] Running compliance review on %s...", created_id)
review_result = run_optimizer([created_id])
if review_result.get("ok"):
logger.info("[Scheduled] Review completed for %s", created_id)
else:
logger.warning("[Scheduled] Review failed: %s", review_result.get("error"))
except Exception as e:
logger.exception("[Scheduled] Generation failed: %s", e)
logger.exception("[Scheduled] Generation pipeline failed: %s", e)
def _run_optimize(self):
try:
+10 -84
View File
@@ -1,20 +1,10 @@
import json
from datetime import datetime, date
from pathlib import Path
from sqlalchemy.orm import Session
from ..database import SessionLocal
from ..models import Topic
import os
# 计算项目根目录(从本文件位置上升4层)
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if os.getenv('PROJECT_ROOT'):
PROJECT_ROOT = Path(os.getenv('PROJECT_ROOT'))
TOPICS_FILE = PROJECT_ROOT / "automation" / "data" / "sustainability_topics.json"
def sync_topic_to_db(topic_id: str, db: Session = None) -> Topic:
"""注意:此函数原用于将JSON单个选题同步到数据库。现已不需要,保留用于兼容。当前方向相反(DB为主),此处仅从数据库导出到JSON(如果需要)"""
# 为了不破坏旧调用,我们改为从数据库读取并写入 JSON 文件(单条更新)
close_db = False
if db is None:
db = SessionLocal()
@@ -23,88 +13,24 @@ def sync_topic_to_db(topic_id: str, db: Session = None) -> Topic:
topic = db.query(Topic).filter(Topic.id == topic_id).first()
if not topic:
raise ValueError(f"Topic {topic_id} not found in DB")
# 写入 JSON 文件(作为备份)
try:
if TOPICS_FILE.exists():
with open(TOPICS_FILE, 'r', encoding='utf-8') as f:
topics = json.load(f)
else:
topics = []
# 转为字典
tdict = {
'id': topic.id,
'title': topic.title,
'field': topic.field,
'format': topic.format,
'core_concept': topic.core_concept,
'audience_pain': topic.audience_pain,
'unique_angle': topic.unique_angle,
'priority': topic.priority,
'priority_score': topic.priority_score,
'total_score': topic.total_score,
'status': topic.status,
'cases': topic.cases or [],
'source_file': topic.source_file,
'created_at': topic.created_at.isoformat() if topic.created_at else None,
'updated_at': topic.updated_at.isoformat() if topic.updated_at else None,
'ready_at': topic.ready_at.isoformat() if topic.ready_at else None,
'published_at': topic.published_at.isoformat() if topic.published_at else None,
'compliance_score': topic.compliance_score,
'platform_urls': topic.platform_urls or {}
}
# 更新或追加
found = False
for i, t in enumerate(topics):
if t['id'] == topic_id:
topics[i] = tdict
found = True
break
if not found:
topics.append(tdict)
with open(TOPICS_FILE, 'w', encoding='utf-8') as f:
json.dump(topics, f, ensure_ascii=False, indent=2)
except Exception as e:
print(f"[Warning] JSON backup failed: {e}")
return topic
finally:
if close_db:
db.close()
def sync_all_topics():
"""导出所有选题到 JSON 文件(用于备份或兼容)"""
db = SessionLocal()
logger = __import__('logging').getLogger(__name__)
try:
topics = db.query(Topic).order_by(Topic.created_at).all()
topic_list = []
for t in topics:
tdict = {
'id': t.id,
'title': t.title,
'field': t.field,
'format': t.format,
'core_concept': t.core_concept,
'audience_pain': t.audience_pain,
'unique_angle': t.unique_angle,
'priority': t.priority,
'priority_score': t.priority_score,
'total_score': t.total_score,
'status': t.status,
'cases': t.cases or [],
'source_file': t.source_file,
'created_at': t.created_at.isoformat() if t.created_at else None,
'updated_at': t.updated_at.isoformat() if t.updated_at else None,
'ready_at': t.ready_at.isoformat() if t.ready_at else None,
'published_at': t.published_at.isoformat() if t.published_at else None,
'compliance_score': t.compliance_score,
'platform_urls': t.platform_urls or {}
}
topic_list.append(tdict)
TOPICS_FILE.parent.mkdir(parents=True, exist_ok=True)
with open(TOPICS_FILE, 'w', encoding='utf-8') as f:
json.dump(topic_list, f, ensure_ascii=False, indent=2)
print(f"✅ 导出 {len(topic_list)} 个选题到 JSON (兼容模式)")
finally:
from sqlalchemy import func
db = SessionLocal()
total = db.query(Topic).count()
rows = db.query(Topic.status, func.count(Topic.id)).group_by(Topic.status).all()
by_status = {s: int(c) for s, c in rows}
logger.info(f"sync_all_topics: DB verified — {total} topics total, statuses: {by_status}")
db.close()
except Exception as e:
logger.error(f"sync_all_topics: DB connection failed — {e}")
raise
if __name__ == "__main__":
sync_all_topics()