#!/usr/bin/env node /** * PERLER LAB — 竞品变更监控爬虫 (Competitor Change Monitor) * ------------------------------------------------------------ * 监控 xylpindou.com 的前端代码变更,通过追踪 Next.js chunk URL 和 build ID 的指纹。 * * 用法: * node monitor.js 单次检查 * node monitor.js --watch 3600 每 3600 秒循环检查 * * 输出: * 变更时: 写入 changes-.log + 控制台警报 * 无变更: 打印 "无变更" + 时间戳 */ const https = require('https'); const http = require('http'); const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); // === 配置 === const TARGET_URL = 'https://xylpindou.com/'; const STATE_FILE = path.join(__dirname, 'state.json'); const CHUNK_REGEX = /src="(\/_next\/static\/chunks\/[^"]+\.js)"/g; const BUILD_ID_REGEX = /"buildId":"([^"]+)"/; // Turbopack RSC 负载中的 buildId 模式: "b":"XXXX" const BUILD_ID_RSC_REGEX = /"b":"([^"]+)"/; // === 辅助函数 === /** * 从 URL 提取文件名部分(最后一个路径段) */ function chunkFilename(url) { return url.split('/').pop(); } /** * 获取当前时间戳字符串 (YYYY-MM-DD HH:MM:SS) */ function timestamp() { const now = new Date(); const pad = (n) => String(n).padStart(2, '0'); return `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())} ${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`; } /** * 获取 ISO 时间戳用于文件名 */ function fileTimestamp() { const now = new Date(); const pad = (n) => String(n).padStart(2, '0'); return `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`; } /** * 获取 HTML 内容(支持 https 和 http) */ function fetchHTML(url) { return new Promise((resolve, reject) => { const client = url.startsWith('https') ? https : http; const req = client.get(url, { timeout: 30000 }, (res) => { // 处理重定向 if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { const redirectUrl = new URL(res.headers.location, url).href; return fetchHTML(redirectUrl).then(resolve).catch(reject); } if (res.statusCode !== 200) { return reject(new Error(`HTTP ${res.statusCode} ${res.statusMessage}`)); } let data = ''; res.on('data', (chunk) => { data += chunk; }); res.on('end', () => resolve(data)); }); req.on('error', (err) => reject(new Error(`网络错误: ${err.message}`))); req.on('timeout', () => { req.destroy(); reject(new Error('请求超时')); }); }); } /** * 从 HTML 中提取所有 Next.js chunk URL */ function extractChunks(html) { const chunks = []; let match; while ((match = CHUNK_REGEX.exec(html)) !== null) { chunks.push(match[1]); } return [...new Set(chunks)].sort(); // 去重 + 排序 } /** * 从 HTML 中提取 build ID */ function extractBuildId(html) { // 先尝试标准 buildId 模式 const matchStandard = html.match(BUILD_ID_REGEX); if (matchStandard) return matchStandard[1]; // 再尝试 RSC 负载中的 "b":"XXXX" 模式 const matchRsc = html.match(BUILD_ID_RSC_REGEX); if (matchRsc) return matchRsc[1]; return null; } /** * 计算指纹: SHA-256(排序后的 chunk URL 列表 + build ID) */ function computeFingerprint(chunks, buildId) { const hash = crypto.createHash('sha256'); hash.update(chunks.join('\n')); hash.update('\n'); hash.update(buildId || ''); return hash.digest('hex'); } /** * 读取之前的状态 */ function readState() { try { if (fs.existsSync(STATE_FILE)) { return JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')); } } catch (err) { console.error(`[${timestamp()}] 读取状态文件失败: ${err.message}`); } return null; } /** * 写入当前状态 */ function writeState(state) { const dir = path.dirname(STATE_FILE); if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2), 'utf-8'); } /** * 写入变更日志 */ function writeChangeLog(oldState, newState) { const oldChunks = oldState ? oldState.chunks : []; const newChunks = newState.chunks; const oldFilenames = new Set(oldChunks.map(chunkFilename)); const newFilenames = new Set(newChunks.map(chunkFilename)); const added = newChunks.filter(c => !oldFilenames.has(chunkFilename(c))); const removed = oldChunks.filter(c => !newFilenames.has(chunkFilename(c))); const logFile = path.join(__dirname, `changes-${fileTimestamp()}.log`); const lines = []; lines.push(`=== 竞品变更检测报告 ===`); lines.push(`检测时间: ${timestamp()}`); lines.push(`目标网址: ${TARGET_URL}`); lines.push(''); lines.push(`旧 Build ID: ${oldState ? oldState.buildId : '无'}`); lines.push(`新 Build ID: ${newState.buildId}`); lines.push(''); lines.push(`--- 新增的 chunk (${added.length}) ---`); added.forEach(c => lines.push(` + ${c}`)); lines.push(''); lines.push(`--- 移除的 chunk (${removed.length}) ---`); removed.forEach(c => lines.push(` - ${c}`)); lines.push(''); lines.push(`--- 不变的 chunk (${newChunks.length - added.length}) ---`); lines.push(`(共 ${newChunks.length} 个 chunk,其中 ${newChunks.length - added.length} 个未变)`); fs.writeFileSync(logFile, lines.join('\n'), 'utf-8'); return logFile; } /** * 单次检查逻辑 */ async function runCheck() { console.log(`[${timestamp()}] 正在检查 ${TARGET_URL} ...`); let html; try { html = await fetchHTML(TARGET_URL); } catch (err) { console.error(`[${timestamp()}] 获取页面失败: ${err.message}`); // 网络失败时,保持旧状态,退出码 0 const oldState = readState(); if (oldState) { console.log(`[${timestamp()}] 保留上次检测状态 (指纹: ${oldState.fingerprint})`); } else { console.log(`[${timestamp()}] 尚无历史状态,跳过`); } return; } const chunks = extractChunks(html); const buildId = extractBuildId(html); if (chunks.length === 0) { console.error(`[${timestamp()}] 警告: 未找到任何 Next.js chunk URL,页面结构可能已变更`); } const fingerprint = computeFingerprint(chunks, buildId); const oldState = readState(); const newState = { fingerprint, chunks, buildId: buildId || 'unknown', lastChecked: new Date().toISOString(), }; if (!oldState) { // 首次运行:记录状态 writeState(newState); console.log(`[${timestamp()}] 首次检测完成,已记录初始状态`); console.log(` Build ID: ${buildId || '未找到'}`); console.log(` Chunk 数量: ${chunks.length}`); console.log(` 指纹: ${fingerprint}`); return; } if (oldState.fingerprint === fingerprint) { console.log(`[${timestamp()}] 无变更 (指纹: ${fingerprint})`); // 更新 lastChecked 但不改变指纹 writeState(newState); return; } // 指纹变更! console.log(`[${timestamp()}] ⚠ 检测到变更!`); console.log(` 旧指纹: ${oldState.fingerprint}`); console.log(` 新指纹: ${fingerprint}`); console.log(` 旧 Build ID: ${oldState.buildId}`); console.log(` 新 Build ID: ${buildId || '未找到'}`); console.log(` 旧 Chunk 数: ${oldState.chunks.length} → 新 Chunk 数: ${chunks.length}`); const logFile = writeChangeLog(oldState, newState); writeState(newState); console.log(`[${timestamp()}] 变更日志已写入: ${logFile}`); console.log(`[${timestamp()}] 请执行爬虫更新: cd competitor-analysis && node crawl.js`); } /** * 主入口 */ async function main() { const args = process.argv.slice(2); const watchIndex = args.indexOf('--watch'); if (watchIndex !== -1 && args[watchIndex + 1]) { const interval = parseInt(args[watchIndex + 1], 10); if (isNaN(interval) || interval < 1) { console.error('错误: --watch 参数必须是正整数(秒数)'); process.exit(1); } console.log(`[${timestamp()}] 监控模式已启动,每 ${interval} 秒检查一次...`); console.log(`[${timestamp()}] 按 Ctrl+C 停止\n`); // 立即执行第一次 await runCheck(); // 定时循环 setInterval(() => { runCheck().catch((err) => { console.error(`[${timestamp()}] 检测过程出错: ${err.message}`); }); }, interval * 1000); } else { // 单次检查 await runCheck(); } } main().catch((err) => { console.error(`[${timestamp()}] 致命错误: ${err.message}`); process.exit(0); // 即使出错也退出码 0 });