feat(script): SEO auditor + PROGRESS.md v24 update

Add seo_auditor.py - crawls external websites, scores 6 SEO dimensions (meta/heading/content/performance/links/mobile), generates optimization tasks. Update PROGRESS.md with v24 external promotion feature. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-06-21 16:07:43 +08:00
parent 8a04a23e86
commit e9cd0374f4
2 changed files with 476 additions and 1 deletions
@@ -0,0 +1,416 @@
+"""
+SEO Auditor — 外部网站 SEO 审计脚本
+
+功能：
+- 抓取目标网站首页和关键页面
+- 分析 Meta 标签、标题结构、内容质量、性能、链接、移动端适配
+- 生成评分报告（0-100 分/维度）
+- 自动生成优化建议
+- 结果写入 DB（SEOAudit + OptimizationTask）
+
+用法：
+  python3 scripts/seo_auditor.py --product-id 1
+  python3 scripts/seo_auditor.py --url https://example.com --name "我的网站"
+"""
+import sys
+import os
+import re
+import json
+import logging
+from datetime import datetime, timezone
+from urllib.parse import urlparse
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'platform', 'backend'))
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
+logger = logging.getLogger(__name__)
+
+
+def extract_meta(html):
+    """分析 Meta 标签"""
+    score = 100
+    issues = []
+
+    title = re.search(r'<title[^>]*>(.*?)</title>', html, re.I | re.S)
+    if not title:
+        score -= 30
+        issues.append({"category": "meta", "severity": "high", "issue": "缺少 <title> 标签", "recommendation": "添加包含目标关键词的 title 标签，建议 50-60 字符"})
+    else:
+        t = title.group(1).strip()
+        if len(t) < 10:
+            score -= 15
+            issues.append({"category": "meta", "severity": "medium", "issue": f"title 过短 ({len(t)}字符)", "recommendation": "title 建议 50-60 字符"})
+        if len(t) > 80:
+            score -= 10
+            issues.append({"category": "meta", "severity": "low", "issue": f"title 过长 ({len(t)}字符)", "recommendation": "title 超过 80 字符可能在搜索结果中被截断"})
+
+    desc = re.search(r'<meta\s+name=["\']description["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
+    if not desc:
+        score -= 25
+        issues.append({"category": "meta", "severity": "high", "issue": "缺少 meta description", "recommendation": "添加包含关键词的 meta description，建议 120-160 字符"})
+    else:
+        d = desc.group(1).strip()
+        if len(d) < 50:
+            score -= 10
+            issues.append({"category": "meta", "severity": "medium", "issue": f"description 过短 ({len(d)}字符)", "recommendation": "description 建议 120-160 字符"})
+
+    keywords = re.search(r'<meta\s+name=["\']keywords["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
+    if not keywords:
+        score -= 5
+        issues.append({"category": "meta", "severity": "low", "issue": "缺少 meta keywords", "recommendation": "添加 meta keywords（虽然不是排名因素，但用于内容相关性提示）"})
+
+    og_title = re.search(r'<meta\s+property=["\']og:title["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
+    og_desc = re.search(r'<meta\s+property=["\']og:description["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
+    og_image = re.search(r'<meta\s+property=["\']og:image["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
+    if not og_title:
+        score -= 10
+        issues.append({"category": "meta", "severity": "medium", "issue": "缺少 og:title", "recommendation": "添加 Open Graph title 改善社交分享展示"})
+    if not og_desc:
+        score -= 5
+        issues.append({"category": "meta", "severity": "low", "issue": "缺少 og:description", "recommendation": "添加 og:description 改善社交分享展示"})
+    if not og_image:
+        score -= 5
+        issues.append({"category": "meta", "severity": "low", "issue": "缺少 og:image", "recommendation": "添加 og:image 让分享链接显示缩略图"})
+
+    return max(0, score), issues
+
+
+def extract_headings(html):
+    """分析标题结构"""
+    score = 100
+    issues = []
+
+    h1s = re.findall(r'<h1[^>]*>(.*?)</h1>', html, re.I | re.S)
+    if len(h1s) == 0:
+        score -= 30
+        issues.append({"category": "heading", "severity": "high", "issue": "页面缺少 H1 标签", "recommendation": "每个页面应当只有一个 H1，包含主要关键词"})
+    elif len(h1s) > 1:
+        score -= 15
+        issues.append({"category": "heading", "severity": "medium", "issue": f"存在 {len(h1s)} 个 H1 标签", "recommendation": "每个页面应当只有一个 H1 标签"})
+    else:
+        h1_text = re.sub(r'<[^>]+>', '', h1s[0]).strip()
+        if len(h1_text) < 5:
+            score -= 5
+            issues.append({"category": "heading", "severity": "low", "issue": "H1 内容过短", "recommendation": "H1 应清晰描述页面主题"})
+
+    h2s = re.findall(r'<h2[^>]*>(.*?)</h2>', html, re.I | re.S)
+    if len(h2s) == 0 and len(re.findall(r'<(h[2-6])', html, re.I)) > 0:
+        pass
+    elif len(h2s) == 0:
+        score -= 10
+        issues.append({"category": "heading", "severity": "medium", "issue": "缺少 H2 子标题", "recommendation": "使用 H2 组织内容结构，提升可读性和 SEO"})
+
+    # Check heading hierarchy
+    all_headings = re.findall(r'<h([1-6])[^>]*>', html, re.I)
+    if all_headings:
+        prev_level = int(all_headings[0])
+        for level in all_headings[1:]:
+            l = int(level)
+            if l > prev_level + 1:
+                score -= 5
+                issues.append({"category": "heading", "severity": "medium", "issue": f"标题层级跳级：H{prev_level} → H{l}", "recommendation": "标题层级应连续，不要跳级"})
+                break
+            prev_level = l
+
+    return max(0, score), issues
+
+
+def extract_content(html):
+    """分析内容质量"""
+    score = 100
+    issues = []
+
+    text = re.sub(r'<[^>]+>', ' ', html)
+    text = re.sub(r'\s+', ' ', text).strip()
+
+    word_count = len(text)
+    if word_count < 300:
+        score -= 30
+        issues.append({"category": "content", "severity": "high", "issue": f"内容过短（{word_count}字符）", "recommendation": "正文建议至少 300 字符，优质内容建议 1000+ 字符"})
+    elif word_count < 1000:
+        score -= 15
+        issues.append({"category": "content", "severity": "medium", "issue": f"内容偏短（{word_count}字符）", "recommendation": "增加内容深度，建议达到 1000+ 字符"})
+
+    # Check image alt
+    imgs = re.findall(r'<img[^>]+>', html, re.I)
+    no_alt = sum(1 for img in imgs if 'alt=' not in img.lower())
+    if imgs and no_alt == len(imgs):
+        score -= 10
+        issues.append({"category": "content", "severity": "medium", "issue": "所有图片缺少 alt 属性", "recommendation": "为图片添加描述性的 alt 文本"})
+    elif no_alt > 0:
+        score -= 5
+        issues.append({"category": "content", "severity": "low", "issue": f"{no_alt}/{len(imgs)} 张图片缺少 alt", "recommendation": "为剩余图片补全 alt 属性"})
+
+    return max(0, score), issues, {"word_count": word_count, "image_count": len(imgs)}
+
+
+def extract_links(html, base_url):
+    """分析链接"""
+    score = 100
+    issues = []
+    domain = urlparse(base_url).netloc
+
+    internal_links = re.findall(r'href=["\'](https?://[^"\']+)["\']', html, re.I)
+    external_links = [url for url in internal_links if urlparse(url).netloc != domain]
+    internal_links = [url for url in internal_links if urlparse(url).netloc == domain]
+    broken_keywords = re.findall(r'href=["\']([^"\']*(?:404|broken|dead)[^"\']*)["\']', html, re.I)
+
+    if len(internal_links) == 0:
+        score -= 15
+        issues.append({"category": "links", "severity": "medium", "issue": "页面没有内部链接", "recommendation": "添加指向站内其他页面的链接，改善爬虫抓取和用户导航"})
+    if len(external_links) == 0:
+        pass
+    ext_no_nofollow = re.findall(r'<a\s+[^>]*href=["\']https?://(?!' + re.escape(domain) + r')["\'][^>]*>', html, re.I)
+    if ext_no_nofollow:
+        score -= 5
+        issues.append({"category": "links", "severity": "low", "issue": "外部链接缺少 nofollow", "recommendation": "对外部链接添加 rel=\"nofollow noopener\" 属性"})
+
+    # Check for broken link patterns
+    broken = re.findall(r'href=["\'](?:https?://[^"\']*?(?:404|error|not-found)[^"\']*)["\']', html, re.I)
+    if broken:
+        score -= 10
+        issues.append({"category": "links", "severity": "high", "issue": f"发现 {len(broken)} 个疑似断链", "recommendation": "检查并修复或删除失效链接"})
+
+    return max(0, score), issues, {"internal": len(internal_links), "external": len(external_links)}
+
+
+def extract_performance(html):
+    """分析性能(基础版 — 仅分析可前端检测的指标)"""
+    score = 100
+    issues = []
+
+    # Check for render-blocking resources
+    css_links = re.findall(r'<link[^>]*href=["\'].*?\.css["\']', html, re.I)
+    js_scripts = re.findall(r'<script[^>]*src=["\']([^"\']+)["\']', html, re.I)
+    render_blocking = len(css_links) + len(js_scripts)
+    if render_blocking > 10:
+        score -= 10
+        issues.append({"category": "performance", "severity": "medium", "issue": f"存在 {render_blocking} 个渲染阻塞资源", "recommendation": "考虑异步加载非关键 CSS/JS，使用 defer 或 async"})
+
+    # Check image without dimensions
+    imgs_no_dim = re.findall(r'<img(?!\s*(?:width|height)=)', html, re.I)
+    if imgs_no_dim:
+        score -= 5
+        issues.append({"category": "performance", "severity": "low", "issue": f"存在 {len(imgs_no_dim)} 张无宽高属性的图片", "recommendation": "为图片添加 width/height 属性，减少布局偏移(CLS)"})
+
+    return max(0, score), issues
+
+
+def extract_mobile(html):
+    """分析移动端适配"""
+    score = 100
+    issues = []
+
+    viewport = re.search(r'<meta\s+name=["\']viewport["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
+    if not viewport:
+        score -= 40
+        issues.append({"category": "mobile", "severity": "high", "issue": "缺少 viewport meta 标签", "recommendation": "添加 <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"> 确保移动端正确渲染"})
+
+    # Check for responsive CSS
+    media_queries = re.findall(r'@media\s', html, re.I)
+    if not media_queries:
+        score -= 15
+        issues.append({"category": "mobile", "severity": "medium", "issue": "未检测到响应式 CSS", "recommendation": "实现响应式设计，使用媒体查询适配不同屏幕尺寸"})
+
+    # Check font size
+    font_small = re.findall(r'font-size\s*:\s*(?:10|11|12)px', html, re.I)
+    if font_small:
+        score -= 5
+        issues.append({"category": "mobile", "severity": "low", "issue": "检测到小字体 (≤12px)", "recommendation": "移动端正文字号建议至少 16px 防止 iOS 自动缩放"})
+
+    return max(0, score), issues
+
+
+def audit_url(url, name=None):
+    """对单个 URL 执行完整 SEO 审计"""
+    import urllib.request
+    import ssl
+
+    parsed = urlparse(url)
+    domain = parsed.netloc
+    page_name = name or domain
+
+    logger.info(f"审计: {page_name} ({url})")
+
+    try:
+        ctx = ssl.create_default_context()
+        ctx.check_hostname = False
+        ctx.verify_mode = ssl.CERT_NONE
+        req = urllib.request.Request(url, headers={
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml',
+            'Accept-Language': 'zh-CN,zh;q=0.9',
+        })
+        resp = urllib.request.urlopen(req, timeout=15, context=ctx)
+        html = resp.read().decode('utf-8', errors='ignore')
+        logger.info(f"  获取成功: {len(html)} bytes")
+    except Exception as e:
+        logger.error(f"  获取失败: {e}")
+        return None
+
+    meta_score, meta_issues = extract_meta(html)
+    heading_score, heading_issues = extract_headings(html)
+    content_score, content_issues, content_info = extract_content(html)
+    perf_score, perf_issues = extract_performance(html)
+    links_score, links_issues, links_info = extract_links(html, url)
+    mobile_score, mobile_issues = extract_mobile(html)
+
+    all_issues = meta_issues + heading_issues + content_issues + perf_issues + links_issues + mobile_issues
+    overall_score = round(
+        meta_score * 0.20 +
+        heading_score * 0.15 +
+        content_score * 0.25 +
+        perf_score * 0.10 +
+        links_score * 0.10 +
+        mobile_score * 0.20,
+        1
+    )
+
+    raw_data = {
+        "url": url,
+        "title": re.search(r'<title[^>]*>(.*?)</title>', html, re.I | re.S).group(1).strip() if re.search(r'<title[^>]*>(.*?)</title>', html, re.I | re.S) else None,
+        "content_info": content_info,
+        "links_info": links_info,
+        "issues_count": len(all_issues),
+        "issues_by_severity": {
+            "high": sum(1 for i in all_issues if i.get("severity") == "high"),
+            "medium": sum(1 for i in all_issues if i.get("severity") == "medium"),
+            "low": sum(1 for i in all_issues if i.get("severity") == "low"),
+        }
+    }
+
+    result = {
+        "url": url,
+        "page_name": page_name,
+        "overall_score": overall_score,
+        "meta_score": meta_score,
+        "heading_score": heading_score,
+        "content_score": content_score,
+        "perf_score": perf_score,
+        "links_score": links_score,
+        "mobile_score": mobile_score,
+        "issues_found": len(all_issues),
+        "raw_data": raw_data,
+        "issues": all_issues,
+    }
+
+    logger.info(f"  总分: {overall_score}/100 (Meta:{meta_score} 标题:{heading_score} 内容:{content_score} 性能:{perf_score} 链接:{links_score} 移动端:{mobile_score})")
+    logger.info(f"  发现问题: {len(all_issues)} ({raw_data['issues_by_severity']['high']}高/{raw_data['issues_by_severity']['medium']}中/{raw_data['issues_by_severity']['low']}低)")
+
+    return result
+
+
+def save_audit_to_db(product_id, result):
+    """将审计结果写入 DB"""
+    try:
+        from app.database import SessionLocal
+        from app.models import ExternalProduct, SEOAudit, OptimizationTask
+
+        db = SessionLocal()
+        try:
+            product = db.query(ExternalProduct).filter(ExternalProduct.id == product_id).first()
+            if not product:
+                logger.error(f"产品 {product_id} 不存在")
+                return False
+
+            audit = SEOAudit(
+                product_id=product_id,
+                audit_type="full",
+                overall_score=result["overall_score"],
+                meta_score=result["meta_score"],
+                heading_score=result["heading_score"],
+                content_score=result["content_score"],
+                perf_score=result["perf_score"],
+                links_score=result["links_score"],
+                mobile_score=result["mobile_score"],
+                raw_data=result.get("raw_data"),
+                page_count=1,
+                issues_found=result["issues_found"],
+                org_id=product.org_id,
+            )
+            db.add(audit)
+            db.flush()
+
+            for issue in result.get("issues", []):
+                task = OptimizationTask(
+                    audit_id=audit.id,
+                    product_id=product_id,
+                    category=issue.get("category", "other"),
+                    severity=issue.get("severity", "medium"),
+                    issue=issue.get("issue", ""),
+                    recommendation=issue.get("recommendation"),
+                    status="open",
+                    org_id=product.org_id,
+                )
+                db.add(task)
+
+            db.commit()
+            logger.info(f"  审计结果已保存: audit_id={audit.id}, tasks={len(result.get('issues', []))}")
+            return True
+        except Exception as e:
+            db.rollback()
+            logger.error(f"  DB 保存失败: {e}")
+            return False
+        finally:
+            db.close()
+    except ImportError as e:
+        logger.error(f"  无法导入应用模块: {e}")
+        logger.info("  (审计结果仅输出到日志)")
+        return False
+
+
+def run_for_product(product_id):
+    """为指定产品执行 SEO 审计"""
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'platform', 'backend'))
+    try:
+        from app.database import SessionLocal
+        from app.models import ExternalProduct
+        db = SessionLocal()
+        try:
+            product = db.query(ExternalProduct).filter(ExternalProduct.id == product_id).first()
+            if not product:
+                logger.error(f"产品 {product_id} 不存在")
+                return
+            url = product.url
+            name = product.name
+        finally:
+            db.close()
+    except ImportError:
+        logger.error("无法连接到数据库，请确保在项目根目录运行")
+        return
+
+    if not url:
+        logger.error(f"产品 {name} 没有配置 URL")
+        return
+
+    result = audit_url(url, name)
+    if not result:
+        logger.error("审计失败")
+        return
+
+    save_audit_to_db(product_id, result)
+    return result
+
+
+def run_for_url(url, name=None):
+    """为指定 URL 执行审计（不保存到 DB）"""
+    result = audit_url(url, name)
+    return result
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description="SEO Auditor - 外部网站 SEO 审计")
+    parser.add_argument("--product-id", type=int, help="产品 ID（从 DB 读取 URL）")
+    parser.add_argument("--url", help="直接指定 URL")
+    parser.add_argument("--name", help="页面名称")
+    args = parser.parse_args()
+
+    if args.product_id:
+        run_for_product(args.product_id)
+    elif args.url:
+        result = run_for_url(args.url, args.name)
+        if result:
+            print(json.dumps(result, ensure_ascii=False, indent=2))
+    else:
+        parser.print_help()