配置全面迁移数据库:PromptConfig、TaskConfig动态调度、敏感词/清洗规则/趋势映射/平台标签/痛点模板全部可编辑

- 新增 PromptConfig 模型 + API,支持提示词在线编辑(16条默认)
- 调度器动态读取 TaskConfig.schedule,admin 可调执行时间
- 新增 KeywordDomainMap、SensitiveWord、ContentCleanRule、TrendFieldMapping 表
- DOMAINS、TREND_DOMAIN_MAP、PLATFORM_TAGS、china_pains、RSS关键词、priority_weights 全部迁移到 DB
- tasks.html 重构:卡片网格+配置/产出/历史/提示词四个Tab,折叠显示
- 清理冗余代码:DEFAULT_PROMPTS死代码、collector.py unreachable代码、compliance_checker bug
- strip_thinking_html 改用 DB 规则优先
This commit is contained in:
Yuzhiran Dev
2026-05-22 11:18:23 +08:00
parent a8e0a76e07
commit 1855f190f5
31 changed files with 2927 additions and 1127 deletions
+390
View File
@@ -0,0 +1,390 @@
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session
from typing import List, Optional
from pydantic import BaseModel, ConfigDict
from datetime import datetime
import json
from ..database import get_db
from ..models import KeywordDomainMap, SensitiveWord, ContentCleanRule, TrendFieldMapping, CollectorCategory
from .auth import get_current_admin
router = APIRouter(prefix="/api/admin/config", tags=["admin"])
DEFAULT_KEYWORD_DOMAIN_MAP = [
{"pattern": r"AI|人工智能|大模型|GPT|机器学习|深度学习|聊天机器人|LLM", "domain": "AI工具", "sort_order": 1},
{"pattern": r"远程|居家办公|自由职业|数字游民|远程协作", "domain": "远程工作", "sort_order": 2},
{"pattern": r"可持续|环保|低碳|绿色|碳中和|循环|零浪费|垃圾分类|节能", "domain": "可持续生活", "sort_order": 3},
{"pattern": r"知识管理|笔记|Obsidian|Notion|第二大脑|读书|阅读", "domain": "知识管理", "sort_order": 4},
{"pattern": r"数字生活|数码|手机|电脑|智能|APP|应用、软件", "domain": "数字生活", "sort_order": 5},
{"pattern": r"科技|人文|教育|心理|哲学|社会学", "domain": "科技人文", "sort_order": 6},
]
DEFAULT_TREND_FIELD_MAP = [
{"trend_keyword": "远程工作", "field_name": "未来工作方式", "sort_order": 1},
{"trend_keyword": "AI工具", "field_name": "AI与效率", "sort_order": 2},
{"trend_keyword": "可持续生活", "field_name": "可持续生活系统", "sort_order": 3},
{"trend_keyword": "知识管理", "field_name": "个人知识工厂", "sort_order": 4},
{"trend_keyword": "数字生活", "field_name": "科技人文交叉", "sort_order": 5},
{"trend_keyword": "科技人文", "field_name": "科技人文交叉", "sort_order": 6},
{"trend_keyword": "个人成长", "field_name": "个人成长", "sort_order": 7},
{"trend_keyword": "副业", "field_name": "个人成长", "sort_order": 8},
{"trend_keyword": "AI创作", "field_name": "AI与效率", "sort_order": 9},
{"trend_keyword": "未来工作", "field_name": "未来工作方式", "sort_order": 10},
{"trend_keyword": "效率工具", "field_name": "AI与效率", "sort_order": 11},
{"trend_keyword": "家庭教育", "field_name": "科技人文交叉", "sort_order": 12},
]
DEFAULT_CHINA_PAIN_TEMPLATES = {
"循环消费": "以旧换新流程繁琐、二手商品信任缺失、租赁市场不规范",
"低碳出行": "新能源车充电设施不足、城市规划不支持骑行、通勤距离长",
"干净饮食": "有机食品价格高、真伪难辨、外卖为主的生活方式难以改变",
"零浪费生活": "环保产品溢价高、可持续选择不便、漂绿营销难以分辨",
"绿色家电与节能": "绿色家电初期投入高、节能效果难量化、老旧小区改造难",
"碳普惠": "碳账户普及率低、减排量兑换吸引力不足、公众认知有限",
"环保科技产品": "绿色产品溢价68%难以承受、缺乏统一认证标准、担心漂绿",
"AI与效率": "AI工具选择困难、数据隐私担忧、学习成本高、实际效果难验证",
}
DEFAULT_GLOBAL_RSS_KEYWORDS = ['sustainable', 'green', 'eco', 'circular', 'climate', 'carbon', 'zero waste', 'renewable', 'recycle', '环保', '可持续', '碳中和', '循环经济', '零浪费', '低碳', '生态']
DEFAULT_PRIORITY_WEIGHTS = {"audience_match": 0.3, "data_availability": 0.25, "uniqueness": 0.2, "executability": 0.15, "brand_fit": 0.1}
DEFAULT_DOMAINS = ["远程工作", "AI工具", "可持续生活", "知识管理", "数字生活", "科技人文"]
DEFAULT_SENSITIVE_WORDS = [
{"word": "国家主席", "category": "political"},
{"word": "政治局", "category": "political"},
{"word": "常委", "category": "political"},
{"word": "军委", "category": "political"},
{"word": "统战部", "category": "political"},
{"word": "颠覆国家", "category": "political"},
{"word": "分裂主义", "category": "political"},
{"word": "台独", "category": "political"},
{"word": "疆独", "category": "political"},
{"word": "藏独", "category": "political"},
{"word": "赌博", "category": "prohibited"},
{"word": "毒品", "category": "prohibited"},
{"word": "迷药", "category": "prohibited"},
{"word": "枪支", "category": "prohibited"},
{"word": "炸药", "category": "prohibited"},
{"word": "色情", "category": "prohibited"},
{"word": "低俗", "category": "prohibited"},
{"word": "反动", "category": "prohibited"},
{"word": "邪教", "category": "prohibited"},
{"word": "保证赚钱", "category": "misleading"},
{"word": "一夜暴富", "category": "misleading"},
{"word": "100%有效", "category": "misleading"},
{"word": "包治百病", "category": "misleading"},
{"word": "绝对正确", "category": "misleading"},
{"word": "国家机密", "category": "legal"},
{"word": "军事秘密", "category": "legal"},
{"word": "绝密", "category": "legal"},
{"word": "迷信", "category": "legal"},
]
DEFAULT_CONTENT_CLEAN_RULES = [
{"rule_type": "thinking", "pattern": r"^(好的|好的,|好[的,]|我来|让我|我将|我这就).*?(?=\n|$)", "description": "AI思考模式1", "sort_order": 1},
{"rule_type": "thinking", "pattern": r"^(以下|下面是|这是|为您|根据).*?(?=\n|$)", "description": "AI思考模式2", "sort_order": 2},
{"rule_type": "preface", "pattern": r"^(基于|\u3010.*?\u3011|这里.*)", "description": "AI前缀模式", "sort_order": 3},
{"rule_type": "verbosity", "pattern": r"^首先|^其次|^最后", "description": "AI废话-首先其次", "sort_order": 4},
{"rule_type": "verbosity", "pattern": r"^总的来说$", "description": "AI废话-总的来说", "sort_order": 5},
{"rule_type": "verbosity", "pattern": r"^值得注意的是$", "description": "AI废话-值得注意的是", "sort_order": 6},
{"rule_type": "verbosity", "pattern": r"^换句话说$", "description": "AI废话-换句话说", "sort_order": 7},
{"rule_type": "verbosity", "pattern": r"^总而言之$", "description": "AI废话-总而言之", "sort_order": 8},
{"rule_type": "verbosity", "pattern": r"^简而言之$", "description": "AI废话-简而言之", "sort_order": 9},
{"rule_type": "verbosity", "pattern": r"^一言以蔽之$", "description": "AI废话-一言以蔽之", "sort_order": 10},
{"rule_type": "verbosity", "pattern": r"^可以说$", "description": "AI废话-可以说", "sort_order": 11},
{"rule_type": "verbosity", "pattern": r"^不难发现$", "description": "AI废话-不难发现", "sort_order": 12},
{"rule_type": "verbosity", "pattern": r"^由此可见$", "description": "AI废话-由此可见", "sort_order": 13},
{"rule_type": "verbosity", "pattern": r"^综上所述$", "description": "AI废话-综上所述", "sort_order": 14},
{"rule_type": "verbosity", "pattern": r"^通过以上", "description": "AI废话-通过以上", "sort_order": 15},
{"rule_type": "html_thinking", "pattern": r"<p[^>]*>(好的|好的,|好[的,]|我来|让我|我将|我这就)", "description": "AI思考-HTML模式", "sort_order": 16},
]
class KeywordDomainMapResponse(BaseModel):
id: int
pattern: str
domain: str
sort_order: int
is_active: bool
created_at: Optional[datetime] = None
updated_at: Optional[datetime] = None
model_config = ConfigDict(from_attributes=True)
class KeywordDomainMapCreate(BaseModel):
pattern: str
domain: str
sort_order: int = 0
class KeywordDomainMapUpdate(BaseModel):
pattern: Optional[str] = None
domain: Optional[str] = None
sort_order: Optional[int] = None
is_active: Optional[bool] = None
class SensitiveWordResponse(BaseModel):
id: int
word: str
category: str
is_active: bool
added_by: Optional[str] = None
created_at: Optional[datetime] = None
model_config = ConfigDict(from_attributes=True)
class SensitiveWordCreate(BaseModel):
word: str
category: str = "general"
class ContentCleanRuleResponse(BaseModel):
id: int
rule_type: str
pattern: str
description: Optional[str] = None
is_active: bool
sort_order: int
created_at: Optional[datetime] = None
model_config = ConfigDict(from_attributes=True)
class ContentCleanRuleCreate(BaseModel):
rule_type: str
pattern: str
description: Optional[str] = None
sort_order: int = 0
class ContentCleanRuleUpdate(BaseModel):
pattern: Optional[str] = None
description: Optional[str] = None
is_active: Optional[bool] = None
sort_order: Optional[int] = None
class TrendFieldMappingResponse(BaseModel):
id: int
trend_keyword: str
field_name: str
sort_order: int
is_active: bool
created_at: Optional[datetime] = None
model_config = ConfigDict(from_attributes=True)
class TrendFieldMappingCreate(BaseModel):
trend_keyword: str
field_name: str
sort_order: int = 0
class TrendFieldMappingUpdate(BaseModel):
trend_keyword: Optional[str] = None
field_name: Optional[str] = None
sort_order: Optional[int] = None
is_active: Optional[bool] = None
class SystemConfigValueResponse(BaseModel):
key: str
value: Optional[str] = None
description: Optional[str] = None
def _ensure_defaults(db: Session):
if db.query(KeywordDomainMap).count() == 0:
for item in DEFAULT_KEYWORD_DOMAIN_MAP:
db.add(KeywordDomainMap(**item))
if db.query(SensitiveWord).count() == 0:
for item in DEFAULT_SENSITIVE_WORDS:
db.add(SensitiveWord(**item))
if db.query(ContentCleanRule).count() == 0:
for item in DEFAULT_CONTENT_CLEAN_RULES:
db.add(ContentCleanRule(**item))
if db.query(TrendFieldMapping).count() == 0:
for item in DEFAULT_TREND_FIELD_MAP:
db.add(TrendFieldMapping(**item))
for name, pain in DEFAULT_CHINA_PAIN_TEMPLATES.items():
cat = db.query(CollectorCategory).filter(CollectorCategory.name == name).first()
if cat and not cat.pain_template:
cat.pain_template = pain
db.commit()
@router.get("/keyword-domain-map", response_model=List[KeywordDomainMapResponse])
def list_keyword_domain_map(db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
_ensure_defaults(db)
return db.query(KeywordDomainMap).order_by(KeywordDomainMap.sort_order).all()
@router.post("/keyword-domain-map", response_model=KeywordDomainMapResponse)
def create_keyword_domain_map(data: KeywordDomainMapCreate, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
item = KeywordDomainMap(**data.model_dump())
db.add(item)
db.commit()
db.refresh(item)
return item
@router.put("/keyword-domain-map/{item_id}", response_model=KeywordDomainMapResponse)
def update_keyword_domain_map(item_id: int, data: KeywordDomainMapUpdate, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
item = db.query(KeywordDomainMap).filter(KeywordDomainMap.id == item_id).first()
if not item:
raise HTTPException(status_code=404, detail="未找到")
for k, v in data.model_dump(exclude_unset=True).items():
if v is not None:
setattr(item, k, v)
db.commit()
db.refresh(item)
return item
@router.delete("/keyword-domain-map/{item_id}")
def delete_keyword_domain_map(item_id: int, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
item = db.query(KeywordDomainMap).filter(KeywordDomainMap.id == item_id).first()
if not item:
raise HTTPException(status_code=404, detail="未找到")
db.delete(item)
db.commit()
return {"message": "删除成功"}
@router.get("/sensitive-words", response_model=List[SensitiveWordResponse])
def list_sensitive_words(category: Optional[str] = None, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
_ensure_defaults(db)
q = db.query(SensitiveWord)
if category:
q = q.filter(SensitiveWord.category == category)
return q.order_by(SensitiveWord.id).all()
@router.post("/sensitive-words", response_model=SensitiveWordResponse)
def create_sensitive_word(data: SensitiveWordCreate, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
item = SensitiveWord(**data.model_dump(), added_by=admin_user.username)
db.add(item)
db.commit()
db.refresh(item)
return item
@router.put("/sensitive-words/{item_id}", response_model=SensitiveWordResponse)
def update_sensitive_word(item_id: int, enabled: bool = None, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
item = db.query(SensitiveWord).filter(SensitiveWord.id == item_id).first()
if not item:
raise HTTPException(status_code=404, detail="未找到")
if enabled is not None:
item.is_active = enabled
db.commit()
db.refresh(item)
return item
@router.delete("/sensitive-words/{item_id}")
def delete_sensitive_word(item_id: int, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
item = db.query(SensitiveWord).filter(SensitiveWord.id == item_id).first()
if not item:
raise HTTPException(status_code=404, detail="未找到")
db.delete(item)
db.commit()
return {"message": "删除成功"}
@router.get("/content-clean-rules", response_model=List[ContentCleanRuleResponse])
def list_content_clean_rules(rule_type: Optional[str] = None, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
_ensure_defaults(db)
q = db.query(ContentCleanRule)
if rule_type:
q = q.filter(ContentCleanRule.rule_type == rule_type)
return q.order_by(ContentCleanRule.sort_order).all()
@router.post("/content-clean-rules", response_model=ContentCleanRuleResponse)
def create_content_clean_rule(data: ContentCleanRuleCreate, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
item = ContentCleanRule(**data.model_dump())
db.add(item)
db.commit()
db.refresh(item)
return item
@router.put("/content-clean-rules/{item_id}", response_model=ContentCleanRuleResponse)
def update_content_clean_rule(item_id: int, data: ContentCleanRuleUpdate, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
item = db.query(ContentCleanRule).filter(ContentCleanRule.id == item_id).first()
if not item:
raise HTTPException(status_code=404, detail="未找到")
for k, v in data.model_dump(exclude_unset=True).items():
if v is not None:
setattr(item, k, v)
db.commit()
db.refresh(item)
return item
@router.delete("/content-clean-rules/{item_id}")
def delete_content_clean_rule(item_id: int, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
item = db.query(ContentCleanRule).filter(ContentCleanRule.id == item_id).first()
if not item:
raise HTTPException(status_code=404, detail="未找到")
db.delete(item)
db.commit()
return {"message": "删除成功"}
@router.get("/trend-field-mapping", response_model=List[TrendFieldMappingResponse])
def list_trend_field_mapping(db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
_ensure_defaults(db)
return db.query(TrendFieldMapping).filter(TrendFieldMapping.is_active == True).order_by(TrendFieldMapping.sort_order).all()
@router.post("/trend-field-mapping", response_model=TrendFieldMappingResponse)
def create_trend_field_mapping(data: TrendFieldMappingCreate, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
item = TrendFieldMapping(**data.model_dump())
db.add(item)
db.commit()
db.refresh(item)
return item
@router.put("/trend-field-mapping/{item_id}", response_model=TrendFieldMappingResponse)
def update_trend_field_mapping(item_id: int, data: TrendFieldMappingUpdate, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
item = db.query(TrendFieldMapping).filter(TrendFieldMapping.id == item_id).first()
if not item:
raise HTTPException(status_code=404, detail="未找到")
for k, v in data.model_dump(exclude_unset=True).items():
if v is not None:
setattr(item, k, v)
db.commit()
db.refresh(item)
return item
@router.delete("/trend-field-mapping/{item_id}")
def delete_trend_field_mapping(item_id: int, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
item = db.query(TrendFieldMapping).filter(TrendFieldMapping.id == item_id).first()
if not item:
raise HTTPException(status_code=404, detail="未找到")
db.delete(item)
db.commit()
return {"message": "删除成功"}
@router.get("/system-config/{key}", response_model=SystemConfigValueResponse)
def get_system_config(key: str, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
from ..models import SystemConfig
cfg = db.query(SystemConfig).filter(SystemConfig.key == key).first()
if not cfg:
default_map = {
"trend_domains": json.dumps(DEFAULT_DOMAINS),
"rss_default_keywords": json.dumps(DEFAULT_GLOBAL_RSS_KEYWORDS),
"priority_weights": json.dumps(DEFAULT_PRIORITY_WEIGHTS),
}
if key in default_map:
return SystemConfigValueResponse(key=key, value=default_map[key], description=f"系统默认配置 - {key}")
raise HTTPException(status_code=404, detail="未找到")
return SystemConfigValueResponse(key=cfg.key, value=cfg.value, description=cfg.description)
@router.put("/system-config/{key}")
def update_system_config(key: str, value: str, description: Optional[str] = None, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
from ..models import SystemConfig
cfg = db.query(SystemConfig).filter(SystemConfig.key == key).first()
if cfg:
cfg.value = value
if description is not None:
cfg.description = description
else:
cfg = SystemConfig(key=key, value=value, description=description or key)
db.add(cfg)
db.commit()
return {"message": "保存成功", "key": key, "value": value}
@router.get("/system-configs")
def list_system_configs(db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
from ..models import SystemConfig
configs = db.query(SystemConfig).order_by(SystemConfig.key).all()
defaults = {
"trend_domains": json.dumps(DEFAULT_DOMAINS),
"rss_default_keywords": json.dumps(DEFAULT_GLOBAL_RSS_KEYWORDS),
"priority_weights": json.dumps(DEFAULT_PRIORITY_WEIGHTS),
}
result = []
for c in configs:
result.append({"key": c.key, "value": c.value, "description": c.description, "is_default": False})
for k, v in defaults.items():
if not any(x["key"] == k for x in result):
result.append({"key": k, "value": v, "description": f"系统默认 - {k}", "is_default": True})
return result
+117
View File
@@ -0,0 +1,117 @@
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session
from typing import List, Optional
from pydantic import BaseModel, ConfigDict
from datetime import datetime
from ..database import get_db
from ..models import PromptConfig
from .auth import get_current_admin
router = APIRouter(prefix="/api/admin/prompt-configs", tags=["admin"])
class PromptConfigBase(BaseModel):
key: str
module_id: Optional[str] = None
category: str = "prompt"
version: str = "v1"
content: str
variables: List[dict] = []
description: Optional[str] = None
enabled: bool = True
temperature: Optional[float] = None
max_tokens: Optional[int] = None
created_by: Optional[str] = None
class PromptConfigCreate(PromptConfigBase):
pass
class PromptConfigUpdate(BaseModel):
content: Optional[str] = None
variables: Optional[List[dict]] = None
description: Optional[str] = None
enabled: Optional[bool] = None
temperature: Optional[float] = None
max_tokens: Optional[int] = None
class PromptConfigResponse(PromptConfigBase):
id: int
created_at: Optional[datetime] = None
updated_at: Optional[datetime] = None
model_config = ConfigDict(from_attributes=True)
@router.get("", response_model=List[PromptConfigResponse])
def list_prompts(
module_id: Optional[str] = None,
category: Optional[str] = None,
db: Session = Depends(get_db),
admin_user=Depends(get_current_admin),
):
q = db.query(PromptConfig)
if module_id:
q = q.filter(PromptConfig.module_id.in_([module_id, "all"]))
if category:
q = q.filter(PromptConfig.category == category)
prompts = q.order_by(PromptConfig.module_id, PromptConfig.key).all()
if not prompts:
_ensure_defaults(db)
prompts = q.order_by(PromptConfig.module_id, PromptConfig.key).all()
return [PromptConfigResponse.model_validate(p) for p in prompts]
@router.get("/{prompt_id}", response_model=PromptConfigResponse)
def get_prompt(prompt_id: int, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
prompt = db.query(PromptConfig).filter(PromptConfig.id == prompt_id).first()
if not prompt:
raise HTTPException(status_code=404, detail="未找到该配置")
return prompt
@router.post("", response_model=PromptConfigResponse)
def create_prompt(data: PromptConfigCreate, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
existing = db.query(PromptConfig).filter(PromptConfig.key == data.key).first()
if existing:
raise HTTPException(status_code=400, detail=f"key '{data.key}' 已存在")
prompt = PromptConfig(**data.model_dump())
db.add(prompt)
db.commit()
db.refresh(prompt)
return prompt
@router.put("/{prompt_id}", response_model=PromptConfigResponse)
def update_prompt(prompt_id: int, data: PromptConfigUpdate, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
prompt = db.query(PromptConfig).filter(PromptConfig.id == prompt_id).first()
if not prompt:
raise HTTPException(status_code=404, detail="未找到该配置")
for field, value in data.model_dump(exclude_unset=True).items():
if value is not None:
setattr(prompt, field, value)
db.commit()
db.refresh(prompt)
return prompt
@router.delete("/{prompt_id}")
def delete_prompt(prompt_id: int, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
prompt = db.query(PromptConfig).filter(PromptConfig.id == prompt_id).first()
if not prompt:
raise HTTPException(status_code=404, detail="未找到该配置")
db.delete(prompt)
db.commit()
return {"message": "删除成功"}
def _ensure_defaults(db: Session):
existing_keys = {p.key for p in db.query(PromptConfig).all()}
from .task_configs import DEFAULT_PROMPTS
for p in DEFAULT_PROMPTS:
if p["key"] not in existing_keys:
db.add(PromptConfig(**p))
db.commit()
+57 -43
View File
@@ -9,7 +9,7 @@ from typing import Dict, Any, List, Optional
import os
import json
from ..database import get_db
from ..models import Topic, Article
from ..models import Topic, Article, TaskConfig, TaskLog
from ..core.generator import run_creator, get_generator_status
from ..core.optimizer import run_optimizer, get_optimizer_status
from ..core.collector import run_collector, get_collector_status
@@ -61,7 +61,7 @@ def get_status(db: Session = Depends(get_db)):
}
@router.post("/generate/run", dependencies=[Depends(get_current_user)])
def trigger_generation(topic_id: str = Body(None, embed=True), db: Session = Depends(get_db), current_user=Depends(get_current_user)):
def trigger_generation(topic_id: Optional[str] = None, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
logger.info(f"Generation triggered by {current_user.username}, topic_id={topic_id}")
try:
result = run_creator(topic_id)
@@ -93,7 +93,7 @@ def collection_status():
return status
@router.post("/review/run", dependencies=[Depends(get_current_user)])
def trigger_review(topic_ids: List[str] = Body(None, embed=True), db: Session = Depends(get_db), current_user=Depends(get_current_user)):
def trigger_review(topic_ids: Optional[List[str]] = None, db: Session = Depends(get_db), current_user=Depends(get_current_user)):
try:
result = run_optimizer(topic_ids)
return {"message": "合规审查已后台启动", "pid": result.get("pid")}
@@ -247,47 +247,61 @@ def get_scheduler_status():
@router.get("/modules/status", dependencies=[Depends(get_current_user)])
def get_modules_status():
today_str = date.today().isoformat()
log_based: dict = {
"scheduled_collect": {"name": "📡 内容采集", "log": LOGS_DIR / f"collector_{today_str}.log"},
"scheduled_refresh_search_cache": {"name": "🔍 搜索缓存", "log": LOGS_DIR / f"opencode_search_{today_str}.log"},
"scheduled_fetch_trends": {"name": "🔥 热点趋势", "log": LOGS_DIR / f"trends_{today_str}.log"},
"scheduled_generate": {"name": "🤖 内容创作", "log": LOGS_DIR / f"creator_{today_str}.log"},
"scheduled_optimize": {"name": "🔍 合规审查", "log": LOGS_DIR / f"optimizer_{today_str}.log"},
"scheduled_optimize_sources": {"name": "📡 信息源优化", "log": LOGS_DIR / f"optimizer_sources_{today_str}.log"},
"scheduled_metrics_sync": {"name": "📊 指标同步", "log": LOGS_DIR / f"metrics_sync_{today_str}.log"},
def get_modules_status(db: Session = Depends(get_db)):
configs = db.query(TaskConfig).all()
config_map = {c.module_id: c for c in configs}
MODULE_META = {
"scheduled_refresh_search_cache": {"name": "🔍 搜索缓存", "cron": "01:00", "params_desc": {"refresh_queries": "搜索关键词列表"}},
"scheduled_fetch_trends": {"name": "🔥 热点趋势", "cron": "01:10", "params_desc": {}},
"scheduled_collect": {"name": "📡 内容采集", "cron": "01:30", "params_desc": {"max_topics": "最大选题数", "categories": "采集类别"}},
"scheduled_generate": {"name": "🤖 内容创作", "cron": "02:00", "params_desc": {"auto_review": "自动合规审查"}},
"scheduled_optimize": {"name": "🔍 合规审查", "cron": "03:00", "params_desc": {"auto_pass_threshold": "自动通过分数阈值"}},
"scheduled_optimize_sources": {"name": "📡 信息源优化", "cron": "05:00", "params_desc": {}},
"scheduled_metrics_sync": {"name": "📊 指标同步", "cron": "06:00", "params_desc": {}},
}
jobs = {j['id']: j for j in scheduler.get_jobs()}
modules = []
for mod_id, cfg in log_based.items():
log_file = cfg["log"]
last_run = None
task_count = 0
success_rate = None
if log_file.exists():
mtime = datetime.fromtimestamp(log_file.stat().st_mtime)
last_run = mtime.strftime("%Y-%m-%d %H:%M")
content = log_file.read_text(encoding="utf-8", errors="ignore")
task_count = content.count("完成") + content.count("success") + content.count("SUCCESS")
total = task_count + content.count("失败") + content.count("failed") + content.count("ERROR")
success_rate = round(task_count / total * 100) if total > 0 else None
status = "running" if mod_id in jobs else "stopped"
job = jobs.get(mod_id)
next_run = None
if job and job.get("next_run_time"):
try:
next_dt = datetime.fromisoformat(job["next_run_time"])
next_run = next_dt.strftime("%Y-%m-%d %H:%M")
except Exception:
next_run = job["next_run_time"]
for mod_id, meta in MODULE_META.items():
cfg = config_map.get(mod_id)
latest = db.query(TaskLog).filter(TaskLog.module_id == mod_id).order_by(TaskLog.started_at.desc()).first()
next_run = _get_next_run(mod_id)
total = db.query(TaskLog).filter(TaskLog.module_id == mod_id).count()
success = db.query(TaskLog).filter(TaskLog.module_id == mod_id, TaskLog.status == "success").count()
failed = db.query(TaskLog).filter(TaskLog.module_id == mod_id, TaskLog.status == "failed").count()
running = db.query(TaskLog).filter(TaskLog.module_id == mod_id, TaskLog.status == "running").count()
modules.append({
"id": mod_id,
"title": cfg["name"],
"status": status,
"last_run": last_run or "从未运行",
"next_run": next_run or "待计划",
"task_count": task_count,
"success_rate": success_rate if success_rate is not None else 0,
"module_id": mod_id,
"title": meta["name"],
"enabled": cfg.enabled if cfg else True,
"params": cfg.params if cfg else {},
"params_desc": meta["params_desc"],
"schedule": cfg.schedule if cfg else meta["cron"],
"cron_default": meta["cron"],
"status": "running" if running else ("stopped" if not (cfg and cfg.enabled) else "idle"),
"last_run": latest.started_at.strftime("%Y-%m-%d %H:%M") if latest and latest.started_at else None,
"last_status": latest.status if latest else None,
"last_message": latest.message if latest else None,
"last_result": latest.result_data if latest else None,
"next_run": next_run,
"total_runs": total,
"success_runs": success,
"failed_runs": failed,
"running": running,
})
return {"modules": modules, "scheduler": {"running": scheduler._started, "jobs": scheduler.get_jobs()}}
jobs = scheduler.get_jobs()
return {"modules": modules, "scheduler": {"running": scheduler._started, "jobs": jobs}}
def _get_next_run(mod_id: str) -> Optional[str]:
for job in scheduler.get_jobs():
if job["id"] == mod_id and job["next_run_time"]:
try:
dt = datetime.fromisoformat(job["next_run_time"])
return dt.strftime("%Y-%m-%d %H:%M")
except Exception:
return job["next_run_time"]
return None
+66
View File
@@ -0,0 +1,66 @@
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session
from typing import List, Optional
from ..database import get_db
from ..models import TaskConfig, TaskLog
from ..schemas import TaskConfigBase, TaskConfigUpdate, TaskConfigResponse, TaskLogResponse
from .auth import get_current_admin
router = APIRouter(prefix="/api/admin/task-configs", tags=["admin"])
DEFAULT_CONFIGS = {
"scheduled_refresh_search_cache": {"name": "🔍 搜索缓存", "cron": "01:00", "params": {}},
"scheduled_fetch_trends": {"name": "🔥 热点趋势", "cron": "01:10", "params": {}},
"scheduled_collect": {"name": "📡 内容采集", "cron": "01:30", "params": {"max_topics": 20}},
"scheduled_generate": {"name": "🤖 内容创作", "cron": "02:00", "params": {"auto_review": True}},
"scheduled_optimize": {"name": "🔍 合规审查", "cron": "03:00", "params": {"auto_pass_threshold": 80}},
"scheduled_optimize_sources": {"name": "📡 信息源优化", "cron": "05:00", "params": {}},
"scheduled_metrics_sync": {"name": "📊 指标同步", "cron": "06:00", "params": {}},
}
@router.get("", response_model=List[TaskConfigResponse])
def list_configs(db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
configs = db.query(TaskConfig).order_by(TaskConfig.id).all()
if not configs:
_ensure_defaults(db)
configs = db.query(TaskConfig).order_by(TaskConfig.id).all()
return [TaskConfigResponse.model_validate(c) for c in configs]
@router.get("/{module_id}", response_model=TaskConfigResponse)
def get_config(module_id: str, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
cfg = db.query(TaskConfig).filter(TaskConfig.module_id == module_id).first()
if not cfg:
_ensure_defaults(db)
cfg = db.query(TaskConfig).filter(TaskConfig.module_id == module_id).first()
return cfg
@router.put("/{module_id}", response_model=TaskConfigResponse)
def update_config(module_id: str, data: TaskConfigUpdate, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
cfg = db.query(TaskConfig).filter(TaskConfig.module_id == module_id).first()
if not cfg:
_ensure_defaults(db)
cfg = db.query(TaskConfig).filter(TaskConfig.module_id == module_id).first()
if data.enabled is not None:
cfg.enabled = data.enabled
if data.params is not None:
cfg.params = data.params
if data.schedule is not None:
cfg.schedule = data.schedule
if data.last_modified_by:
cfg.last_modified_by = data.last_modified_by
db.commit()
db.refresh(cfg)
return cfg
@router.get("/history/{module_id}", response_model=List[TaskLogResponse])
def get_module_history(module_id: str, db: Session = Depends(get_db), admin_user=Depends(get_current_admin), limit: int = 20):
logs = db.query(TaskLog).filter(TaskLog.module_id == module_id).order_by(TaskLog.started_at.desc()).limit(limit).all()
return [TaskLogResponse.model_validate(l) for l in logs]
def _ensure_defaults(db: Session):
existing = {c.module_id for c in db.query(TaskConfig).all()}
for mid, info in DEFAULT_CONFIGS.items():
if mid not in existing:
db.add(TaskConfig(module_id=mid, enabled=True, params=info.get("params", {}), schedule=info.get("cron", "")))
db.commit()
+62 -48
View File
@@ -1,55 +1,82 @@
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session
from sqlalchemy import or_
from typing import List, Optional
from datetime import datetime, timezone
from ..database import get_db
from ..models import TaskLog
from ..schemas import TaskLogBase, TaskLogResponse
from .auth import get_current_admin
router = APIRouter(prefix="/api/admin/tasklogs", tags=["admin"])
router = APIRouter(prefix="/api/admin/task-logs", tags=["admin"])
MODULES = {
"scheduled_refresh_search_cache": "🔍 搜索缓存",
"scheduled_fetch_trends": "🔥 热点趋势",
"scheduled_collect": "📡 内容采集",
"scheduled_generate": "🤖 内容创作",
"scheduled_optimize": "🔍 合规审查",
"scheduled_optimize_sources": "📡 信息源优化",
"scheduled_metrics_sync": "📊 指标同步",
}
@router.get("", response_model=List[TaskLogResponse])
def list_task_logs(
request: Request,
db: Session = Depends(get_db),
admin_user = Depends(get_current_admin),
topic_id: Optional[str] = None,
task_name: Optional[str] = None,
status: Optional[str] = None
admin_user=Depends(get_current_admin),
module_id: Optional[str] = None,
status: Optional[str] = None,
date: Optional[str] = None,
limit: int = 50,
):
"""获取任务日志列表(可过滤)"""
query = db.query(TaskLog)
if topic_id:
query = query.filter(TaskLog.topic_id == topic_id)
if task_name:
query = query.filter(TaskLog.task_name == task_name)
if module_id:
query = query.filter(TaskLog.module_id == module_id)
if status:
query = query.filter(TaskLog.status == status)
logs = query.order_by(TaskLog.started_at.desc()).all()
if date:
try:
dt = datetime.strptime(date, "%Y-%m-%d").replace(tzinfo=timezone.utc)
next_day = datetime(dt.year, dt.month, dt.day + 1, tzinfo=timezone.utc)
query = query.filter(TaskLog.started_at >= dt, TaskLog.started_at < next_day)
except ValueError:
pass
logs = query.order_by(TaskLog.started_at.desc()).limit(limit).all()
return [TaskLogResponse.model_validate(l) for l in logs]
@router.get("/modules")
def list_modules(db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
today = datetime.now(timezone.utc).date().isoformat()
result = []
for mid, name in MODULES.items():
latest = db.query(TaskLog).filter(TaskLog.module_id == mid).order_by(TaskLog.started_at.desc()).first()
total = db.query(TaskLog).filter(TaskLog.module_id == mid).count()
success = db.query(TaskLog).filter(TaskLog.module_id == mid, TaskLog.status == "success").count()
failed = db.query(TaskLog).filter(TaskLog.module_id == mid, TaskLog.status == "failed").count()
running = db.query(TaskLog).filter(TaskLog.module_id == mid, TaskLog.status == "running").count()
result.append({
"module_id": mid,
"name": name,
"last_run": latest.started_at.isoformat() if latest and latest.started_at else None,
"last_status": latest.status if latest else None,
"last_message": latest.message if latest else None,
"total_runs": total,
"success_runs": success,
"failed_runs": failed,
"running": running,
})
return result
@router.get("/{log_id}", response_model=TaskLogResponse)
def get_task_log(
log_id: int,
request: Request,
db: Session = Depends(get_db),
admin_user = Depends(get_current_admin)
):
"""获取单个任务日志详情"""
def get_task_log(log_id: int, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
log = db.query(TaskLog).filter(TaskLog.id == log_id).first()
if not log:
raise HTTPException(status_code=404, detail="日志不存在")
raise HTTPException(status_code=404, detail="记录不存在")
return log
@router.post("", response_model=TaskLogResponse)
def create_task_log(
log_data: TaskLogBase,
request: Request,
db: Session = Depends(get_db),
admin_user = Depends(get_current_admin)
):
"""创建任务日志(用于手动记录)"""
def create_task_log(log_data: TaskLogBase, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
log = TaskLog(**log_data.model_dump())
db.add(log)
db.commit()
@@ -57,35 +84,22 @@ def create_task_log(
return log
@router.put("/{log_id}", response_model=TaskLogResponse)
def update_task_log(
log_id: int,
log_update: TaskLogBase,
request: Request,
db: Session = Depends(get_db),
admin_user = Depends(get_current_admin)
):
"""更新任务日志"""
def update_task_log(log_id: int, log_update: TaskLogBase, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
log = db.query(TaskLog).filter(TaskLog.id == log_id).first()
if not log:
raise HTTPException(status_code=404, detail="日志不存在")
update_data = log_update.model_dump(exclude_unset=True)
for field, value in update_data.items():
raise HTTPException(status_code=404, detail="记录不存在")
data = log_update.model_dump(exclude_unset=True)
for field, value in data.items():
setattr(log, field, value)
db.commit()
db.refresh(log)
return log
@router.delete("/{log_id}")
def delete_task_log(
log_id: int,
request: Request,
db: Session = Depends(get_db),
admin_user = Depends(get_current_admin)
):
"""删除任务日志"""
def delete_task_log(log_id: int, db: Session = Depends(get_db), admin_user=Depends(get_current_admin)):
log = db.query(TaskLog).filter(TaskLog.id == log_id).first()
if not log:
raise HTTPException(status_code=404, detail="日志不存在")
raise HTTPException(status_code=404, detail="记录不存在")
db.delete(log)
db.commit()
return {"message": "删除成功"}
return {"message": "删除成功"}
+33 -11
View File
@@ -213,6 +213,25 @@ def get_module_detail(module_id: str, db: Session = Depends(get_db), current_use
LOGS_DIR = ROOT / "automation" / "logs"
today_str = date_mod.today().isoformat()
try:
return _get_module_detail_data(module_id, db, ROOT, DATA_DIR, LOGS_DIR, today_str)
except Exception as e:
return {
"module_id": module_id,
"title": module_id,
"description": "",
"status": "stopped",
"inputs": {},
"outputs": {"error": str(e)},
"history": [],
"log_excerpt": "",
}
def _get_module_detail_data(module_id: str, db, ROOT, DATA_DIR, LOGS_DIR, today_str):
from datetime import datetime as dt_mod, date as date_mod
from pathlib import Path as PathMod
import json as json_mod
import re as re_mod
MODULE_META = {
"scheduled_refresh_search_cache": {"name": "🔍 搜索缓存", "description": "通过 opencode webfetch 联网搜索,刷新 8 个分类的搜索缓存,供内容采集器使用"},
"scheduled_fetch_trends": {"name": "🔥 热点趋势", "description": "从百度、微博、知乎实时热搜 API 抓取当天热点,LLM 补充,存入 trends.json"},
@@ -279,7 +298,7 @@ def get_module_detail(module_id: str, db: Session = Depends(get_db), current_use
pass
elif module_id == "scheduled_collect":
from ..models import Topic, CollectorCategory
from ..models import CollectorCategory
pending = db.query(Topic).filter(Topic.status.in_(["pending", "待处理"])).count()
total_topics = db.query(Topic).count()
cats = db.query(CollectorCategory).filter(CollectorCategory.is_active == True).all()
@@ -296,16 +315,19 @@ def get_module_detail(module_id: str, db: Session = Depends(get_db), current_use
pending_t = db.query(Topic).filter(Topic.status.in_(["pending", "待处理"])).count()
inputs["待创作选题"] = pending_t
outputs["待审查"] = review
from ..models import Article
recent_articles = db.query(Article, Topic.title.label("topic_title")).join(Topic, Article.topic_id == Topic.id, isouter=True).order_by(Article.created_at.desc()).limit(5).all()
outputs["最新文章"] = []
seen_articles = set()
for a in recent_articles:
art = a.Article if hasattr(a, 'Article') else a[0]
tid = a.topic_title if hasattr(a, 'topic_title') else (a[1] if len(a) > 1 else "")
if art.id not in seen_articles:
seen_articles.add(art.id)
outputs["最新文章"].append({"id": art.id, "platform": art.platform, "topic": tid, "status": art.status, "created": art.created_at.isoformat() if art.created_at else ""})
try:
from ..models import Article
recent_articles = db.query(Article, Topic.title.label("topic_title")).join(Topic, Article.topic_id == Topic.id, isouter=True).order_by(Article.created_at.desc()).limit(5).all()
outputs["最新文章"] = []
seen_articles = set()
for a in recent_articles:
art = a[0]
tid = a[1] if len(a) > 1 else ""
if art.id not in seen_articles:
seen_articles.add(art.id)
outputs["最新文章"].append({"id": art.id, "platform": art.platform, "topic": tid, "status": art.status, "created": art.created_at.isoformat() if art.created_at else ""})
except Exception as e:
outputs["最新文章_错误"] = str(e)
elif module_id == "scheduled_optimize":
outputs["待审查选题"] = db.query(Topic).filter(Topic.status.in_(["review", "待审查"])).count()