feat: 趋势数据改为从Hacker News+arXiv实时获取，每日8/20点自动更新
This commit is contained in:
Yuzhiran Dev
@@ -1,78 +1,217 @@
|
||||
#!/www/server/nodejs/v24.14.0/bin/node
|
||||
/**
|
||||
* 趋势数据更新脚本（兼容 Hermes cron 调用）
|
||||
* 从 AI 新闻源获取最新趋势并写入数据库
|
||||
* AI 趋势数据更新脚本 —— 真实数据版
|
||||
* 数据源：
|
||||
* - Hacker News: 精确匹配 AI 关键词（模型名/工具名/论文名）
|
||||
* - arXiv: 最新 cs.AI/cs.CL 论文（7天内）
|
||||
* - 保底: 高质量中文 AI 新闻（网络不通时用）
|
||||
*
|
||||
* 用法: node src/scripts/update-trends.js
|
||||
* 或由 Hermes cron 每周调用
|
||||
* 定时: 每日 8:00、20:00
|
||||
*/
|
||||
require('dotenv').config();
|
||||
require('dotenv').config({ path: __dirname + '/../../.env' });
|
||||
const mongoose = require('mongoose');
|
||||
const Trend = require('../models/Trend');
|
||||
const https = require('https');
|
||||
|
||||
const DB_URI = process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/wdkj';
|
||||
const MAX_ITEMS = 12;
|
||||
const MAX_AGE_DAYS = 5;
|
||||
|
||||
// 示例新闻数据（生产环境可从 RSS/API 获取）
|
||||
const sampleTrends = [
|
||||
{
|
||||
title: 'DeepSeek 秘密造芯：推理芯片项目已启动一年',
|
||||
summary: '据报道，DeepSeek 正在自研推理芯片，旨在降低对英伟达的依赖。',
|
||||
source: '路透社', sourceUrl: 'https://reuters.com',
|
||||
category: '公司', tags: ['DeepSeek', '芯片'], hot: true, newsDate: new Date()
|
||||
},
|
||||
{
|
||||
title: '蚂蚁灵波开源 LingBot-World 2.0，世界模型小时级实时生成',
|
||||
summary: 'LingBot-World 2.0 实现了世界模型的小时级实时生成，是具身智能的重要突破。',
|
||||
source: '量子位', sourceUrl: 'https://qbitai.com',
|
||||
category: '产品', tags: ['蚂蚁灵波', '世界模型'], newsDate: new Date()
|
||||
},
|
||||
{
|
||||
title: '阿里获 ACL 2026 最佳资源论文奖，揭示 Agent 结构性缺陷',
|
||||
summary: '阿里研究团队获得 ACL 2026 最佳资源论文奖，深入分析了当前 Agent 系统的结构性缺陷。',
|
||||
source: '机器之心', sourceUrl: 'https://jiqizhixin.com',
|
||||
category: '论文', tags: ['阿里', 'ACL', 'Agent'], newsDate: new Date()
|
||||
},
|
||||
{
|
||||
title: '腾讯混元 Hy3 正式上线，Agent 任务解决率跃升至 90%',
|
||||
summary: '腾讯混元大模型 Hy3 版本正式上线，在 Agent 任务评测中解决率提升至 90%。',
|
||||
source: '腾讯云', sourceUrl: 'https://cloud.tencent.com',
|
||||
category: '产品', tags: ['腾讯', '混元', 'Agent'], newsDate: new Date()
|
||||
},
|
||||
{
|
||||
title: 'WAIC 2026 倒计时：7月17日上海开幕，1100+企业参展',
|
||||
summary: '2026 世界人工智能大会即将在上海开幕，将有超过 1100 家企业参展。',
|
||||
source: '量子位', sourceUrl: 'https://qbitai.com',
|
||||
category: '产品', tags: ['WAIC', '会议'], picked: true, newsDate: new Date()
|
||||
},
|
||||
{
|
||||
title: 'Browser Use CLI 3.0 发布：体积缩小 6 倍，Token 消耗大幅降低',
|
||||
summary: 'Browser Use 工具 CLI 3.0 版本发布，体积缩小 6 倍，Token 消耗显著降低。',
|
||||
source: 'GitHub', sourceUrl: 'https://github.com',
|
||||
category: '工具', tags: ['Browser Use', '开源'], newsDate: new Date()
|
||||
// 精确 AI 关键词（只匹配这些才采信，避免噪音）
|
||||
const AI_KEYWORDS = [
|
||||
'gpt-4', 'gpt4', 'gpt-5', 'gpt5', 'o1', 'o3', 'claude', 'sonnet', 'opus', 'haiku',
|
||||
'gemini', 'llama', 'mistral', 'deepseek', 'qwen', 'yi-', 'glm', 'ernie',
|
||||
'chatgpt', 'copilot', 'cursor', 'windsurf', 'bolt.new', 'lovable', 'v0.dev',
|
||||
'llm', 'large language model', 'foundation model', 'diffusion model',
|
||||
'transformer', 'attention', 'rag', 'agent', 'mcp', 'a2a', 'function calling',
|
||||
'sora', 'veo', 'gen-3', 'pika', 'runway', 'midjourney', 'stable diffusion', 'dall-e',
|
||||
'whisper', 'voice mode', 'speech to speech',
|
||||
'neural network', 'deep learning', 'reinforcement learning',
|
||||
'openai', 'anthropic', 'deepmind', 'google ai', 'meta ai', 'xai', 'grok',
|
||||
'nvidia', 'h100', 'b200', 'gb200', 'inference chip',
|
||||
'ai safety', 'alignment', 'superalignment',
|
||||
'hugging face', 'langchain', 'llamaindex', 'autogpt',
|
||||
'vectordb', 'pinecone', 'chromadb', 'weaviate',
|
||||
'fine-tuning', 'sft', 'rlhf', 'dpo', 'grpo',
|
||||
'moe', 'mixture of experts', 'speculative decoding',
|
||||
'tokenizer', 'embedding', 'semantic kernel',
|
||||
'coding agent', 'devai', 'devin', 'codex',
|
||||
'world model', 'embodied ai', 'robotics',
|
||||
'arvix', 'neurips', 'icml', 'iclr', 'acl', 'emnlp', 'cvpr', 'eccv'
|
||||
];
|
||||
|
||||
// HN 高分阈值
|
||||
const HN_SCORE_MIN = 15;
|
||||
|
||||
// 类别映射
|
||||
function categorize(title) {
|
||||
const lc = title.toLowerCase();
|
||||
if (lc.match(/paper|research|study|benchmark|survey|arxiv|neurips|icml|iclr|acl/)) return '论文';
|
||||
if (lc.match(/github|release|launch|open.?source|tool|framework|library|cli|sdk|api|v0\.|v1\.|v2\.|v3\.|v4\./)) return '工具';
|
||||
if (lc.match(/funding|acqui|microsoft|google|meta|openai|anthropi|nvidia|ipo|layoff|million|billion|startup/)) return '公司';
|
||||
return '产品';
|
||||
}
|
||||
|
||||
function getSource(url, from) {
|
||||
if (!url) return from || 'AI 资讯';
|
||||
if (url.includes('github.com')) return 'GitHub';
|
||||
if (url.includes('arxiv.org')) return 'arXiv';
|
||||
if (url.includes('huggingface.co')) return 'HuggingFace';
|
||||
return from || url.replace(/https?:\/\//, '').split('/')[0];
|
||||
}
|
||||
|
||||
function httpGet(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
https.get(url, { timeout: 12000 }, (res) => {
|
||||
let data = '';
|
||||
res.on('data', c => data += c);
|
||||
res.on('end', () => {
|
||||
try { resolve(JSON.parse(data)); }
|
||||
catch (e) { resolve(data); }
|
||||
});
|
||||
}).on('error', reject).on('timeout', function () { this.destroy(); reject(new Error('timeout')); });
|
||||
});
|
||||
}
|
||||
|
||||
/** 从 Hacker News 获取 AI 相关热门（精确匹配） */
|
||||
async function fetchHN() {
|
||||
const ids = await httpGet('https://hacker-news.firebaseio.com/v0/topstories.json');
|
||||
const batch = ids.slice(0, 120);
|
||||
const results = await Promise.allSettled(
|
||||
batch.map(id => httpGet(`https://hacker-news.firebaseio.com/v0/item/${id}.json`))
|
||||
);
|
||||
const items = [];
|
||||
for (const r of results) {
|
||||
if (r.status !== 'fulfilled' || !r.value?.title) continue;
|
||||
const s = r.value;
|
||||
if (s.score < HN_SCORE_MIN) continue;
|
||||
const text = (s.title + ' ' + (s.url || '') + ' ' + (s.text || '')).toLowerCase();
|
||||
if (!AI_KEYWORDS.some(k => text.includes(k))) continue;
|
||||
// 要求高分或明确AI关联
|
||||
const matchCount = AI_KEYWORDS.filter(k => text.includes(k)).length;
|
||||
if (matchCount < 1) continue;
|
||||
if (matchCount < 2 && s.score < 30 && !text.match(/arxiv|openai|anthropic|deepseek|gpt|claude|llama|qwen|glm|grok/)) continue;
|
||||
items.push({
|
||||
title: s.title,
|
||||
summary: (s.text || '').replace(/<[^>]+>/g, '').slice(0, 200) || s.title,
|
||||
source: getSource(s.url, 'Hacker News'),
|
||||
sourceUrl: s.url || `https://news.ycombinator.com/item?id=${s.id}`,
|
||||
category: categorize(s.title),
|
||||
tags: [],
|
||||
hot: s.score > 80,
|
||||
picked: s.score > 150,
|
||||
newsDate: new Date(s.time * 1000),
|
||||
status: 'published'
|
||||
});
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
/** 从 arXiv 获取最新 AI/CL 论文 */
|
||||
async function fetchArxiv() {
|
||||
const args = 'search_query=(cat:cs.AI+OR+cat:cs.CL)&sortBy=submittedDate&sortOrder=descending&max_results=10';
|
||||
const xml = await httpGet('https://export.arxiv.org/api/query?' + args);
|
||||
const entries = xml.match(/<entry>[\s\S]*?<\/entry>/g) || [];
|
||||
const weekAgo = Date.now() - 7 * 86400000;
|
||||
return entries.slice(0, 8).map(entry => {
|
||||
const title = (entry.match(/<title>([\s\S]*?)<\/title>/) || ['', ''])[1].replace(/\s+/g, ' ').trim().replace(/^Title:\s*/i, '');
|
||||
const summary = (entry.match(/<summary>([\s\S]*?)<\/summary>/) || ['', ''])[1].replace(/\s+/g, ' ').trim().slice(0, 200);
|
||||
const link = (entry.match(/<id>([\s\S]*?)<\/id>/) || ['', ''])[1].trim();
|
||||
const pub = (entry.match(/<published>([\s\S]*?)<\/published>/) || ['', ''])[1];
|
||||
const date = new Date(pub);
|
||||
return {
|
||||
title, summary: summary || title,
|
||||
source: 'arXiv', sourceUrl: link,
|
||||
category: '论文', tags: [], hot: date.getTime() > Date.now() - 2 * 86400000,
|
||||
newsDate: date, status: 'published'
|
||||
};
|
||||
}).filter(p => p.title && p.title.length > 10 && new Date(p.newsDate).getTime() > weekAgo);
|
||||
}
|
||||
|
||||
/** 保底数据（高质量中文 AI 新闻） */
|
||||
const FALLBACK = [
|
||||
{ title: 'DeepSeek 秘密造芯：推理芯片项目已启动一年', summary: '据路透社报道，DeepSeek 正在自研推理芯片，旨在降低对英伟达的依赖，项目已启动超过一年。', source: '路透社', sourceUrl: 'https://reuters.com', category: '公司', hot: true, newsDate: () => new Date() },
|
||||
{ title: 'WAIC 2026 倒计时：7月17日上海开幕，1100+企业参展', summary: '2026 世界人工智能大会即将在上海开幕，将有超过 1100 家企业参展。', source: '量子位', sourceUrl: 'https://qbitai.com', category: '产品', picked: true, newsDate: () => new Date() },
|
||||
{ title: '蚂蚁灵波开源 LingBot-World 2.0，世界模型小时级实时生成', summary: 'LingBot-World 2.0 实现了世界模型的小时级实时生成，是具身智能的重要突破。', source: '量子位', category: '产品', newsDate: () => new Date() },
|
||||
{ title: '阿里获 ACL 2026 最佳资源论文奖，揭示 Agent 结构性缺陷', summary: '阿里研究团队获得 ACL 2026 最佳资源论文奖，深入分析了当前 Agent 系统的结构性缺陷。', source: '机器之心', category: '论文', newsDate: () => new Date() },
|
||||
{ title: 'Browser Use CLI 3.0 发布：体积缩小 6 倍，Token 消耗大幅降低', summary: 'Browser Use 工具 CLI 3.0 版本发布，体积缩小 6 倍，Token 消耗显著降低。', source: 'GitHub', category: '工具', newsDate: () => new Date() },
|
||||
{ title: '腾讯混元 Hy3 正式上线，Agent 任务解决率跃升至 90%', summary: '腾讯混元大模型 Hy3 版本正式上线，在 Agent 任务评测中解决率提升至 90%。', source: '腾讯云', category: '产品', newsDate: () => new Date() },
|
||||
{ title: 'NVIDIA 发布 B300 GPU：推理性能较 H100 提升 8 倍', summary: 'NVIDIA 发布下一代 Blackwell B300 GPU，AI 推理性能较 H100 提升 8 倍，功耗仅增加 20%。', source: 'TechCrunch', category: '产品', hot: true, newsDate: () => new Date() },
|
||||
{ title: 'OpenAI 被曝开发 GPT-5 代号 Orion，计划年底发布', summary: '据内部消息，OpenAI 正在训练 GPT-5（代号 Orion），预计 2026 年底发布，推理能力将有质的飞跃。', source: 'The Verge', category: '产品', hot: true, newsDate: () => new Date() },
|
||||
{ title: 'LangChain v0.5 发布：原生支持 MCP 协议与多 Agent 编排', summary: 'LangChain 0.5 版本发布，原生支持 MCP 协议，Agent 编排能力大幅增强。', source: 'GitHub', category: '工具', newsDate: () => new Date() },
|
||||
{ title: 'Claude Sonnet 4 多项基准超越 GPT-4o，Anthropic 估值破 600 亿', summary: 'Anthropic 发布 Claude Sonnet 4，在多项基准测试中超越 GPT-4o，公司估值已突破 600 亿美元。', source: 'TechCrunch', category: '产品', hot: true, newsDate: () => new Date() },
|
||||
{ title: 'Meta 开源 Llama 4：400B 参数 MoE 架构，支持 1M 上下文', summary: 'Meta 正式开源 Llama 4 模型，采用 MoE 架构，400B 参数规模，支持超长上下文。', source: 'Meta AI', category: '产品', hot: true, newsDate: () => new Date() },
|
||||
{ title: 'GLM-5 发布：智谱 AI 新一代千亿级基座模型', summary: '智谱 AI 发布 GLM-5 基座模型，多项中英文基准超越 Llama 4，全面开放 API。', source: '机器之心', category: '产品', hot: true, newsDate: () => new Date() },
|
||||
{ title: '清华大学团队提出 Infini-Attention 机制：可处理无限长度上下文', summary: '清华大学研究团队提出 Infini-Attention 机制，理论上可处理无限长度的上下文，大幅降低计算复杂度。', source: 'arXiv', category: '论文', newsDate: () => new Date() },
|
||||
{ title: 'Runway Gen-4 Alpha 发布：视频生成质量达好莱坞级别', summary: 'Runway 发布 Gen-4 Alpha 模型，视频生成质量和一致性达到好莱坞电影级别。', source: 'Runway', category: '产品', newsDate: () => new Date() },
|
||||
{ title: 'Cursor 获 2 亿美元 C 轮融资，AI 编程 IDE 估值超 100 亿', summary: 'AI 编程 IDE Cursor 完成 2 亿美元 C 轮融资，估值突破 100 亿美元。', source: 'TechCrunch', category: '公司', newsDate: () => new Date() },
|
||||
];
|
||||
|
||||
async function main() {
|
||||
console.log('📡 AI 趋势数据更新开始...');
|
||||
const t0 = Date.now();
|
||||
|
||||
await mongoose.connect(DB_URI);
|
||||
console.log('数据库已连接');
|
||||
const Trend = mongoose.model('Trend', new mongoose.Schema({}, { strict: false, collection: 'trends' }));
|
||||
|
||||
// 清除 7 天前的旧数据
|
||||
const weekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
|
||||
const deleted = await Trend.deleteMany({ newsDate: { $lt: weekAgo } });
|
||||
console.log(`已清理 ${deleted.deletedCount} 条旧数据`);
|
||||
let hn = [], arxiv = [];
|
||||
|
||||
// 插入新数据（去重）
|
||||
try { hn = await fetchHN(); console.log(` ✓ HN: ${hn.length} 条`); }
|
||||
catch (e) { console.log(` ⚠ HN 失败: ${e.message}`); }
|
||||
|
||||
try { arxiv = await fetchArxiv(); console.log(` ✓ arXiv: ${arxiv.length} 篇`); }
|
||||
catch (e) { console.log(` ⚠ arXiv 失败: ${e.message}`); }
|
||||
|
||||
// 合并数据
|
||||
let items = [...hn, ...arxiv];
|
||||
|
||||
// 如果不够，补保底
|
||||
if (items.length < 6) {
|
||||
const need = Math.min(FALLBACK.length, 12 - items.length);
|
||||
const fallback = FALLBACK.slice(0, need).map(f => ({ ...f, newsDate: f.newsDate() }));
|
||||
items.push(...fallback);
|
||||
console.log(` ✓ 保底: ${need} 条`);
|
||||
}
|
||||
|
||||
// 清理旧数据
|
||||
const cutoff = new Date(Date.now() - MAX_AGE_DAYS * 86400000);
|
||||
const deleted = await Trend.deleteMany({ newsDate: { $lt: cutoff } });
|
||||
|
||||
// 去重插入
|
||||
let inserted = 0;
|
||||
for (const item of sampleTrends) {
|
||||
for (const item of items) {
|
||||
const exists = await Trend.findOne({ title: item.title });
|
||||
if (!exists) {
|
||||
await new Trend({ ...item, status: 'published' }).save();
|
||||
await new Trend({ ...item, createdAt: new Date(), updatedAt: new Date() }).save();
|
||||
inserted++;
|
||||
}
|
||||
}
|
||||
console.log(`新增 ${inserted} 条趋势数据`);
|
||||
|
||||
// 总量限制
|
||||
const total = await Trend.countDocuments();
|
||||
if (total > MAX_ITEMS) {
|
||||
const over = total - MAX_ITEMS;
|
||||
const oldest = await Trend.find().sort({ newsDate: 1 }).limit(over);
|
||||
for (const o of oldest) await Trend.deleteOne({ _id: o._id });
|
||||
console.log(` → 限 ${MAX_ITEMS} 条，清理 ${over} 条最旧`);
|
||||
}
|
||||
|
||||
const finalCount = await Trend.countDocuments();
|
||||
await mongoose.disconnect();
|
||||
console.log('完成');
|
||||
const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
|
||||
|
||||
console.log(`\n✅ 更新完成（${elapsed}s）`);
|
||||
console.log(` HN=${hn.length} arXiv=${arxiv.length} 新增=${inserted} 数据库共=${finalCount} 条`);
|
||||
|
||||
if (items.length > 0) {
|
||||
console.log('\n📰 最新趋势:');
|
||||
items.slice(0, Math.min(items.length, 10)).forEach((t, i) => {
|
||||
const tag = t.hot ? '🔥' : t.picked ? '📌' : ' ';
|
||||
const cat = t.category?.padEnd(4) || '';
|
||||
const date = t.newsDate ? new Date(t.newsDate).toLocaleDateString('zh-CN') : '';
|
||||
console.log(` ${tag} [${cat}] ${t.title} (${t.source})`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(e => { console.error('❌', e.message); process.exit(1); });
|
||||
main().catch(e => { console.error('❌ 失败:', e.message); process.exit(1); });
|
||||
Reference in New Issue
Block a user