yuzhiran
e9cd0374f4
Add seo_auditor.py - crawls external websites, scores 6 SEO dimensions (meta/heading/content/performance/links/mobile), generates optimization tasks. Update PROGRESS.md with v24 external promotion feature. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
417 lines
17 KiB
Python
417 lines
17 KiB
Python
"""
|
||
SEO Auditor — 外部网站 SEO 审计脚本
|
||
|
||
功能：
|
||
- 抓取目标网站首页和关键页面
|
||
- 分析 Meta 标签、标题结构、内容质量、性能、链接、移动端适配
|
||
- 生成评分报告（0-100 分/维度）
|
||
- 自动生成优化建议
|
||
- 结果写入 DB（SEOAudit + OptimizationTask）
|
||
|
||
用法：
|
||
python3 scripts/seo_auditor.py --product-id 1
|
||
python3 scripts/seo_auditor.py --url https://example.com --name "我的网站"
|
||
"""
|
||
import sys
|
||
import os
|
||
import re
|
||
import json
|
||
import logging
|
||
from datetime import datetime, timezone
|
||
from urllib.parse import urlparse
|
||
|
||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'platform', 'backend'))
|
||
|
||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
def extract_meta(html):
|
||
"""分析 Meta 标签"""
|
||
score = 100
|
||
issues = []
|
||
|
||
title = re.search(r'<title[^>]*>(.*?)</title>', html, re.I | re.S)
|
||
if not title:
|
||
score -= 30
|
||
issues.append({"category": "meta", "severity": "high", "issue": "缺少 <title> 标签", "recommendation": "添加包含目标关键词的 title 标签，建议 50-60 字符"})
|
||
else:
|
||
t = title.group(1).strip()
|
||
if len(t) < 10:
|
||
score -= 15
|
||
issues.append({"category": "meta", "severity": "medium", "issue": f"title 过短 ({len(t)}字符)", "recommendation": "title 建议 50-60 字符"})
|
||
if len(t) > 80:
|
||
score -= 10
|
||
issues.append({"category": "meta", "severity": "low", "issue": f"title 过长 ({len(t)}字符)", "recommendation": "title 超过 80 字符可能在搜索结果中被截断"})
|
||
|
||
desc = re.search(r'<meta\s+name=["\']description["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
|
||
if not desc:
|
||
score -= 25
|
||
issues.append({"category": "meta", "severity": "high", "issue": "缺少 meta description", "recommendation": "添加包含关键词的 meta description，建议 120-160 字符"})
|
||
else:
|
||
d = desc.group(1).strip()
|
||
if len(d) < 50:
|
||
score -= 10
|
||
issues.append({"category": "meta", "severity": "medium", "issue": f"description 过短 ({len(d)}字符)", "recommendation": "description 建议 120-160 字符"})
|
||
|
||
keywords = re.search(r'<meta\s+name=["\']keywords["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
|
||
if not keywords:
|
||
score -= 5
|
||
issues.append({"category": "meta", "severity": "low", "issue": "缺少 meta keywords", "recommendation": "添加 meta keywords（虽然不是排名因素，但用于内容相关性提示）"})
|
||
|
||
og_title = re.search(r'<meta\s+property=["\']og:title["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
|
||
og_desc = re.search(r'<meta\s+property=["\']og:description["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
|
||
og_image = re.search(r'<meta\s+property=["\']og:image["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
|
||
if not og_title:
|
||
score -= 10
|
||
issues.append({"category": "meta", "severity": "medium", "issue": "缺少 og:title", "recommendation": "添加 Open Graph title 改善社交分享展示"})
|
||
if not og_desc:
|
||
score -= 5
|
||
issues.append({"category": "meta", "severity": "low", "issue": "缺少 og:description", "recommendation": "添加 og:description 改善社交分享展示"})
|
||
if not og_image:
|
||
score -= 5
|
||
issues.append({"category": "meta", "severity": "low", "issue": "缺少 og:image", "recommendation": "添加 og:image 让分享链接显示缩略图"})
|
||
|
||
return max(0, score), issues
|
||
|
||
|
||
def extract_headings(html):
|
||
"""分析标题结构"""
|
||
score = 100
|
||
issues = []
|
||
|
||
h1s = re.findall(r'<h1[^>]*>(.*?)</h1>', html, re.I | re.S)
|
||
if len(h1s) == 0:
|
||
score -= 30
|
||
issues.append({"category": "heading", "severity": "high", "issue": "页面缺少 H1 标签", "recommendation": "每个页面应当只有一个 H1，包含主要关键词"})
|
||
elif len(h1s) > 1:
|
||
score -= 15
|
||
issues.append({"category": "heading", "severity": "medium", "issue": f"存在 {len(h1s)} 个 H1 标签", "recommendation": "每个页面应当只有一个 H1 标签"})
|
||
else:
|
||
h1_text = re.sub(r'<[^>]+>', '', h1s[0]).strip()
|
||
if len(h1_text) < 5:
|
||
score -= 5
|
||
issues.append({"category": "heading", "severity": "low", "issue": "H1 内容过短", "recommendation": "H1 应清晰描述页面主题"})
|
||
|
||
h2s = re.findall(r'<h2[^>]*>(.*?)</h2>', html, re.I | re.S)
|
||
if len(h2s) == 0 and len(re.findall(r'<(h[2-6])', html, re.I)) > 0:
|
||
pass
|
||
elif len(h2s) == 0:
|
||
score -= 10
|
||
issues.append({"category": "heading", "severity": "medium", "issue": "缺少 H2 子标题", "recommendation": "使用 H2 组织内容结构，提升可读性和 SEO"})
|
||
|
||
# Check heading hierarchy
|
||
all_headings = re.findall(r'<h([1-6])[^>]*>', html, re.I)
|
||
if all_headings:
|
||
prev_level = int(all_headings[0])
|
||
for level in all_headings[1:]:
|
||
l = int(level)
|
||
if l > prev_level + 1:
|
||
score -= 5
|
||
issues.append({"category": "heading", "severity": "medium", "issue": f"标题层级跳级：H{prev_level} → H{l}", "recommendation": "标题层级应连续，不要跳级"})
|
||
break
|
||
prev_level = l
|
||
|
||
return max(0, score), issues
|
||
|
||
|
||
def extract_content(html):
|
||
"""分析内容质量"""
|
||
score = 100
|
||
issues = []
|
||
|
||
text = re.sub(r'<[^>]+>', ' ', html)
|
||
text = re.sub(r'\s+', ' ', text).strip()
|
||
|
||
word_count = len(text)
|
||
if word_count < 300:
|
||
score -= 30
|
||
issues.append({"category": "content", "severity": "high", "issue": f"内容过短（{word_count}字符）", "recommendation": "正文建议至少 300 字符，优质内容建议 1000+ 字符"})
|
||
elif word_count < 1000:
|
||
score -= 15
|
||
issues.append({"category": "content", "severity": "medium", "issue": f"内容偏短（{word_count}字符）", "recommendation": "增加内容深度，建议达到 1000+ 字符"})
|
||
|
||
# Check image alt
|
||
imgs = re.findall(r'<img[^>]+>', html, re.I)
|
||
no_alt = sum(1 for img in imgs if 'alt=' not in img.lower())
|
||
if imgs and no_alt == len(imgs):
|
||
score -= 10
|
||
issues.append({"category": "content", "severity": "medium", "issue": "所有图片缺少 alt 属性", "recommendation": "为图片添加描述性的 alt 文本"})
|
||
elif no_alt > 0:
|
||
score -= 5
|
||
issues.append({"category": "content", "severity": "low", "issue": f"{no_alt}/{len(imgs)} 张图片缺少 alt", "recommendation": "为剩余图片补全 alt 属性"})
|
||
|
||
return max(0, score), issues, {"word_count": word_count, "image_count": len(imgs)}
|
||
|
||
|
||
def extract_links(html, base_url):
|
||
"""分析链接"""
|
||
score = 100
|
||
issues = []
|
||
domain = urlparse(base_url).netloc
|
||
|
||
internal_links = re.findall(r'href=["\'](https?://[^"\']+)["\']', html, re.I)
|
||
external_links = [url for url in internal_links if urlparse(url).netloc != domain]
|
||
internal_links = [url for url in internal_links if urlparse(url).netloc == domain]
|
||
broken_keywords = re.findall(r'href=["\']([^"\']*(?:404|broken|dead)[^"\']*)["\']', html, re.I)
|
||
|
||
if len(internal_links) == 0:
|
||
score -= 15
|
||
issues.append({"category": "links", "severity": "medium", "issue": "页面没有内部链接", "recommendation": "添加指向站内其他页面的链接，改善爬虫抓取和用户导航"})
|
||
if len(external_links) == 0:
|
||
pass
|
||
ext_no_nofollow = re.findall(r'<a\s+[^>]*href=["\']https?://(?!' + re.escape(domain) + r')["\'][^>]*>', html, re.I)
|
||
if ext_no_nofollow:
|
||
score -= 5
|
||
issues.append({"category": "links", "severity": "low", "issue": "外部链接缺少 nofollow", "recommendation": "对外部链接添加 rel=\"nofollow noopener\" 属性"})
|
||
|
||
# Check for broken link patterns
|
||
broken = re.findall(r'href=["\'](?:https?://[^"\']*?(?:404|error|not-found)[^"\']*)["\']', html, re.I)
|
||
if broken:
|
||
score -= 10
|
||
issues.append({"category": "links", "severity": "high", "issue": f"发现 {len(broken)} 个疑似断链", "recommendation": "检查并修复或删除失效链接"})
|
||
|
||
return max(0, score), issues, {"internal": len(internal_links), "external": len(external_links)}
|
||
|
||
|
||
def extract_performance(html):
|
||
"""分析性能(基础版 — 仅分析可前端检测的指标)"""
|
||
score = 100
|
||
issues = []
|
||
|
||
# Check for render-blocking resources
|
||
css_links = re.findall(r'<link[^>]*href=["\'].*?\.css["\']', html, re.I)
|
||
js_scripts = re.findall(r'<script[^>]*src=["\']([^"\']+)["\']', html, re.I)
|
||
render_blocking = len(css_links) + len(js_scripts)
|
||
if render_blocking > 10:
|
||
score -= 10
|
||
issues.append({"category": "performance", "severity": "medium", "issue": f"存在 {render_blocking} 个渲染阻塞资源", "recommendation": "考虑异步加载非关键 CSS/JS，使用 defer 或 async"})
|
||
|
||
# Check image without dimensions
|
||
imgs_no_dim = re.findall(r'<img(?!\s*(?:width|height)=)', html, re.I)
|
||
if imgs_no_dim:
|
||
score -= 5
|
||
issues.append({"category": "performance", "severity": "low", "issue": f"存在 {len(imgs_no_dim)} 张无宽高属性的图片", "recommendation": "为图片添加 width/height 属性，减少布局偏移(CLS)"})
|
||
|
||
return max(0, score), issues
|
||
|
||
|
||
def extract_mobile(html):
|
||
"""分析移动端适配"""
|
||
score = 100
|
||
issues = []
|
||
|
||
viewport = re.search(r'<meta\s+name=["\']viewport["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
|
||
if not viewport:
|
||
score -= 40
|
||
issues.append({"category": "mobile", "severity": "high", "issue": "缺少 viewport meta 标签", "recommendation": "添加 <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"> 确保移动端正确渲染"})
|
||
|
||
# Check for responsive CSS
|
||
media_queries = re.findall(r'@media\s', html, re.I)
|
||
if not media_queries:
|
||
score -= 15
|
||
issues.append({"category": "mobile", "severity": "medium", "issue": "未检测到响应式 CSS", "recommendation": "实现响应式设计，使用媒体查询适配不同屏幕尺寸"})
|
||
|
||
# Check font size
|
||
font_small = re.findall(r'font-size\s*:\s*(?:10|11|12)px', html, re.I)
|
||
if font_small:
|
||
score -= 5
|
||
issues.append({"category": "mobile", "severity": "low", "issue": "检测到小字体 (≤12px)", "recommendation": "移动端正文字号建议至少 16px 防止 iOS 自动缩放"})
|
||
|
||
return max(0, score), issues
|
||
|
||
|
||
def audit_url(url, name=None):
|
||
"""对单个 URL 执行完整 SEO 审计"""
|
||
import urllib.request
|
||
import ssl
|
||
|
||
parsed = urlparse(url)
|
||
domain = parsed.netloc
|
||
page_name = name or domain
|
||
|
||
logger.info(f"审计: {page_name} ({url})")
|
||
|
||
try:
|
||
ctx = ssl.create_default_context()
|
||
ctx.check_hostname = False
|
||
ctx.verify_mode = ssl.CERT_NONE
|
||
req = urllib.request.Request(url, headers={
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||
'Accept': 'text/html,application/xhtml+xml',
|
||
'Accept-Language': 'zh-CN,zh;q=0.9',
|
||
})
|
||
resp = urllib.request.urlopen(req, timeout=15, context=ctx)
|
||
html = resp.read().decode('utf-8', errors='ignore')
|
||
logger.info(f" 获取成功: {len(html)} bytes")
|
||
except Exception as e:
|
||
logger.error(f" 获取失败: {e}")
|
||
return None
|
||
|
||
meta_score, meta_issues = extract_meta(html)
|
||
heading_score, heading_issues = extract_headings(html)
|
||
content_score, content_issues, content_info = extract_content(html)
|
||
perf_score, perf_issues = extract_performance(html)
|
||
links_score, links_issues, links_info = extract_links(html, url)
|
||
mobile_score, mobile_issues = extract_mobile(html)
|
||
|
||
all_issues = meta_issues + heading_issues + content_issues + perf_issues + links_issues + mobile_issues
|
||
overall_score = round(
|
||
meta_score * 0.20 +
|
||
heading_score * 0.15 +
|
||
content_score * 0.25 +
|
||
perf_score * 0.10 +
|
||
links_score * 0.10 +
|
||
mobile_score * 0.20,
|
||
1
|
||
)
|
||
|
||
raw_data = {
|
||
"url": url,
|
||
"title": re.search(r'<title[^>]*>(.*?)</title>', html, re.I | re.S).group(1).strip() if re.search(r'<title[^>]*>(.*?)</title>', html, re.I | re.S) else None,
|
||
"content_info": content_info,
|
||
"links_info": links_info,
|
||
"issues_count": len(all_issues),
|
||
"issues_by_severity": {
|
||
"high": sum(1 for i in all_issues if i.get("severity") == "high"),
|
||
"medium": sum(1 for i in all_issues if i.get("severity") == "medium"),
|
||
"low": sum(1 for i in all_issues if i.get("severity") == "low"),
|
||
}
|
||
}
|
||
|
||
result = {
|
||
"url": url,
|
||
"page_name": page_name,
|
||
"overall_score": overall_score,
|
||
"meta_score": meta_score,
|
||
"heading_score": heading_score,
|
||
"content_score": content_score,
|
||
"perf_score": perf_score,
|
||
"links_score": links_score,
|
||
"mobile_score": mobile_score,
|
||
"issues_found": len(all_issues),
|
||
"raw_data": raw_data,
|
||
"issues": all_issues,
|
||
}
|
||
|
||
logger.info(f" 总分: {overall_score}/100 (Meta:{meta_score} 标题:{heading_score} 内容:{content_score} 性能:{perf_score} 链接:{links_score} 移动端:{mobile_score})")
|
||
logger.info(f" 发现问题: {len(all_issues)} ({raw_data['issues_by_severity']['high']}高/{raw_data['issues_by_severity']['medium']}中/{raw_data['issues_by_severity']['low']}低)")
|
||
|
||
return result
|
||
|
||
|
||
def save_audit_to_db(product_id, result):
|
||
"""将审计结果写入 DB"""
|
||
try:
|
||
from app.database import SessionLocal
|
||
from app.models import ExternalProduct, SEOAudit, OptimizationTask
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
product = db.query(ExternalProduct).filter(ExternalProduct.id == product_id).first()
|
||
if not product:
|
||
logger.error(f"产品 {product_id} 不存在")
|
||
return False
|
||
|
||
audit = SEOAudit(
|
||
product_id=product_id,
|
||
audit_type="full",
|
||
overall_score=result["overall_score"],
|
||
meta_score=result["meta_score"],
|
||
heading_score=result["heading_score"],
|
||
content_score=result["content_score"],
|
||
perf_score=result["perf_score"],
|
||
links_score=result["links_score"],
|
||
mobile_score=result["mobile_score"],
|
||
raw_data=result.get("raw_data"),
|
||
page_count=1,
|
||
issues_found=result["issues_found"],
|
||
org_id=product.org_id,
|
||
)
|
||
db.add(audit)
|
||
db.flush()
|
||
|
||
for issue in result.get("issues", []):
|
||
task = OptimizationTask(
|
||
audit_id=audit.id,
|
||
product_id=product_id,
|
||
category=issue.get("category", "other"),
|
||
severity=issue.get("severity", "medium"),
|
||
issue=issue.get("issue", ""),
|
||
recommendation=issue.get("recommendation"),
|
||
status="open",
|
||
org_id=product.org_id,
|
||
)
|
||
db.add(task)
|
||
|
||
db.commit()
|
||
logger.info(f" 审计结果已保存: audit_id={audit.id}, tasks={len(result.get('issues', []))}")
|
||
return True
|
||
except Exception as e:
|
||
db.rollback()
|
||
logger.error(f" DB 保存失败: {e}")
|
||
return False
|
||
finally:
|
||
db.close()
|
||
except ImportError as e:
|
||
logger.error(f" 无法导入应用模块: {e}")
|
||
logger.info(" (审计结果仅输出到日志)")
|
||
return False
|
||
|
||
|
||
def run_for_product(product_id):
|
||
"""为指定产品执行 SEO 审计"""
|
||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'platform', 'backend'))
|
||
try:
|
||
from app.database import SessionLocal
|
||
from app.models import ExternalProduct
|
||
db = SessionLocal()
|
||
try:
|
||
product = db.query(ExternalProduct).filter(ExternalProduct.id == product_id).first()
|
||
if not product:
|
||
logger.error(f"产品 {product_id} 不存在")
|
||
return
|
||
url = product.url
|
||
name = product.name
|
||
finally:
|
||
db.close()
|
||
except ImportError:
|
||
logger.error("无法连接到数据库，请确保在项目根目录运行")
|
||
return
|
||
|
||
if not url:
|
||
logger.error(f"产品 {name} 没有配置 URL")
|
||
return
|
||
|
||
result = audit_url(url, name)
|
||
if not result:
|
||
logger.error("审计失败")
|
||
return
|
||
|
||
save_audit_to_db(product_id, result)
|
||
return result
|
||
|
||
|
||
def run_for_url(url, name=None):
|
||
"""为指定 URL 执行审计（不保存到 DB）"""
|
||
result = audit_url(url, name)
|
||
return result
|
||
|
||
|
||
if __name__ == "__main__":
|
||
import argparse
|
||
parser = argparse.ArgumentParser(description="SEO Auditor - 外部网站 SEO 审计")
|
||
parser.add_argument("--product-id", type=int, help="产品 ID（从 DB 读取 URL）")
|
||
parser.add_argument("--url", help="直接指定 URL")
|
||
parser.add_argument("--name", help="页面名称")
|
||
args = parser.parse_args()
|
||
|
||
if args.product_id:
|
||
run_for_product(args.product_id)
|
||
elif args.url:
|
||
result = run_for_url(args.url, args.name)
|
||
if result:
|
||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||
else:
|
||
parser.print_help()
|