#!/usr/bin/env node /** * PERLER LAB — 竞品变更监控爬虫 (Competitor Change Monitor) * ------------------------------------------------------------ * 监控 xylpindou.com 的前端代码变更,通过追踪 Next.js chunk URL 和 build ID 的指纹。 * * 双指纹机制 (dual-fingerprint): * - 首页 (/): fingerprint + chunks + buildId (向后兼容原有字段) * - chibi 页 (/chibi): chibiFingerprint + chibiChunks + chibiBuildId * - 任一页面指纹变化 => 告警 + 写入 changes-.log * * 用法: * node monitor.js 单次检查 * node monitor.js --watch 3600 每 3600 秒循环检查 * * 输出: * 变更时: 写入 changes-.log + 控制台警报 * 无变更: 打印 "无变更" + 时间戳 */ const https = require('https'); const http = require('http'); const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); // === 配置 === const TARGET_URL = 'https://xylpindou.com/'; const CHIBI_URL = 'https://xylpindou.com/chibi'; const STATE_FILE = path.join(__dirname, 'state.json'); const CHUNK_REGEX = /src="(\/_next\/static\/chunks\/[^"]+\.js)"/g; const BUILD_ID_REGEX = /"buildId":"([^"]+)"/; // Turbopack RSC 负载中的 buildId 模式: "b":"XXXX" const BUILD_ID_RSC_REGEX = /"b":"([^"]+)"/; // === 辅助函数 === /** * 从 URL 提取文件名部分(最后一个路径段) */ function chunkFilename(url) { return url.split('/').pop(); } /** * 获取当前时间戳字符串 (YYYY-MM-DD HH:MM:SS) */ function timestamp() { const now = new Date(); const pad = (n) => String(n).padStart(2, '0'); return `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())} ${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`; } /** * 获取 ISO 时间戳用于文件名 */ function fileTimestamp() { const now = new Date(); const pad = (n) => String(n).padStart(2, '0'); return `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`; } /** * 获取 HTML 内容(支持 https 和 http) */ function fetchHTML(url) { return new Promise((resolve, reject) => { const client = url.startsWith('https') ? https : http; const req = client.get(url, { timeout: 30000 }, (res) => { // 处理重定向 if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { const redirectUrl = new URL(res.headers.location, url).href; return fetchHTML(redirectUrl).then(resolve).catch(reject); } if (res.statusCode !== 200) { return reject(new Error(`HTTP ${res.statusCode} ${res.statusMessage}`)); } let data = ''; res.on('data', (chunk) => { data += chunk; }); res.on('end', () => resolve(data)); }); req.on('error', (err) => reject(new Error(`网络错误: ${err.message}`))); req.on('timeout', () => { req.destroy(); reject(new Error('请求超时')); }); }); } /** * 从 HTML 中提取所有 Next.js chunk URL */ function extractChunks(html) { const chunks = []; let match; while ((match = CHUNK_REGEX.exec(html)) !== null) { chunks.push(match[1]); } return [...new Set(chunks)].sort(); // 去重 + 排序 } /** * 从 HTML 中提取 build ID */ function extractBuildId(html) { // 先尝试标准 buildId 模式 const matchStandard = html.match(BUILD_ID_REGEX); if (matchStandard) return matchStandard[1]; // 再尝试 RSC 负载中的 "b":"XXXX" 模式 const matchRsc = html.match(BUILD_ID_RSC_REGEX); if (matchRsc) return matchRsc[1]; return null; } /** * 计算指纹: SHA-256(排序后的 chunk URL 列表 + build ID) */ function computeFingerprint(chunks, buildId) { const hash = crypto.createHash('sha256'); hash.update(chunks.join('\n')); hash.update('\n'); hash.update(buildId || ''); return hash.digest('hex'); } /** * 计算组合指纹: SHA-256(home指纹 + chibi指纹) */ function computeCombinedFingerprint(homeFingerprint, chibiFingerprint) { const hash = crypto.createHash('sha256'); hash.update(homeFingerprint); hash.update('\n'); hash.update(chibiFingerprint); return hash.digest('hex'); } /** * 对比两页 chunk 列表,返回新增/移除(按文件名比较) */ function diffChunks(oldChunks, newChunks) { const oldFilenames = new Set(oldChunks.map(chunkFilename)); const newFilenames = new Set(newChunks.map(chunkFilename)); const added = newChunks.filter(c => !oldFilenames.has(chunkFilename(c))); const removed = oldChunks.filter(c => !newFilenames.has(chunkFilename(c))); return { added, removed }; } /** * 读取之前的状态 */ function readState() { try { if (fs.existsSync(STATE_FILE)) { return JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8')); } } catch (err) { console.error(`[${timestamp()}] 读取状态文件失败: ${err.message}`); } return null; } /** * 写入当前状态 */ function writeState(state) { const dir = path.dirname(STATE_FILE); if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2), 'utf-8'); } /** * 写入变更日志(含首页 + chibi 两页的 chunk 明细) */ function writeChangeLog(oldState, newState) { const oldChunks = oldState ? oldState.chunks || [] : []; const newChunks = newState.chunks || []; const oldChibiChunks = oldState ? oldState.chibiChunks || [] : []; const newChibiChunks = newState.chibiChunks || []; const homeDiff = diffChunks(oldChunks, newChunks); const chibiDiff = diffChunks(oldChibiChunks, newChibiChunks); const logFile = path.join(__dirname, `changes-${fileTimestamp()}.log`); const lines = []; lines.push(`=== 竞品变更检测报告 ===`); lines.push(`检测时间: ${timestamp()}`); lines.push(`目标网址: ${TARGET_URL} (首页) / ${CHIBI_URL} (chibi)`); lines.push(''); lines.push(`组合指纹: ${oldState && oldState.combinedFingerprint ? oldState.combinedFingerprint : '无'} -> ${newState.combinedFingerprint}`); lines.push(''); lines.push(`--- 首页 ---`); lines.push(`旧 Build ID: ${oldState ? oldState.buildId : '无'}`); lines.push(`新 Build ID: ${newState.buildId}`); lines.push(`旧指纹: ${oldState ? oldState.fingerprint : '无'}`); lines.push(`新指纹: ${newState.fingerprint}`); lines.push(''); lines.push(` -- 首页 新增的 chunk (${homeDiff.added.length}) --`); homeDiff.added.forEach(c => lines.push(` + ${c}`)); lines.push(` -- 首页 移除的 chunk (${homeDiff.removed.length}) --`); homeDiff.removed.forEach(c => lines.push(` - ${c}`)); lines.push(` -- 首页 不变的 chunk (${newChunks.length - homeDiff.added.length}) --`); lines.push(` (首页共 ${newChunks.length} 个 chunk)`); lines.push(''); lines.push(`--- chibi 页 ---`); lines.push(`旧 Build ID: ${oldState ? oldState.chibiBuildId : '无'}`); lines.push(`新 Build ID: ${newState.chibiBuildId}`); lines.push(`旧指纹: ${oldState && oldState.chibiFingerprint ? oldState.chibiFingerprint : '无'}`); lines.push(`新指纹: ${newState.chibiFingerprint}`); lines.push(''); lines.push(` -- chibi 新增的 chunk (${chibiDiff.added.length}) --`); chibiDiff.added.forEach(c => lines.push(` + ${c}`)); lines.push(` -- chibi 移除的 chunk (${chibiDiff.removed.length}) --`); chibiDiff.removed.forEach(c => lines.push(` - ${c}`)); lines.push(` -- chibi 不变的 chunk (${newChibiChunks.length - chibiDiff.added.length}) --`); lines.push(` (chibi 共 ${newChibiChunks.length} 个 chunk)`); fs.writeFileSync(logFile, lines.join('\n'), 'utf-8'); return logFile; } /** * 单次检查逻辑 */ async function runCheck() { console.log(`[${timestamp()}] 正在检查 ${TARGET_URL} 和 ${CHIBI_URL} ...`); // 并行抓取两页(每轮检查最多 2 次请求) const [homeResult, chibiResult] = await Promise.all([ fetchHTML(TARGET_URL).then(html => ({ ok: true, html })).catch(err => ({ ok: false, err })), fetchHTML(CHIBI_URL).then(html => ({ ok: true, html })).catch(err => ({ ok: false, err })), ]); if (!homeResult.ok || !chibiResult.ok) { if (!homeResult.ok) { console.error(`[${timestamp()}] 获取首页失败: ${homeResult.err.message}`); } if (!chibiResult.ok) { console.error(`[${timestamp()}] 获取 chibi 页失败: ${chibiResult.err.message}`); } // 网络失败时,保持旧状态,不告警、不覆盖 const oldState = readState(); if (oldState) { console.log(`[${timestamp()}] 保留上次检测状态 (首页指纹: ${oldState.fingerprint})`); } else { console.log(`[${timestamp()}] 尚无历史状态,跳过`); } return; } const homeChunks = extractChunks(homeResult.html); const homeBuildId = extractBuildId(homeResult.html); const homeFingerprint = computeFingerprint(homeChunks, homeBuildId); const chibiChunks = extractChunks(chibiResult.html); const chibiBuildId = extractBuildId(chibiResult.html); const chibiFingerprint = computeFingerprint(chibiChunks, chibiBuildId); if (homeChunks.length === 0) { console.error(`[${timestamp()}] 警告: 首页未找到任何 Next.js chunk URL,页面结构可能已变更`); } if (chibiChunks.length === 0) { console.error(`[${timestamp()}] 警告: chibi 页未找到任何 Next.js chunk URL,页面结构可能已变更`); } const combinedFingerprint = computeCombinedFingerprint(homeFingerprint, chibiFingerprint); const oldState = readState(); const newState = { fingerprint: homeFingerprint, chunks: homeChunks, buildId: homeBuildId || 'unknown', chibiFingerprint, chibiChunks, chibiBuildId: chibiBuildId || 'unknown', combinedFingerprint, lastChecked: new Date().toISOString(), }; if (!oldState) { // 首次运行:记录状态 writeState(newState); console.log(`[${timestamp()}] 首次检测完成,已记录初始状态`); console.log(` 首页 Build ID: ${homeBuildId || '未找到'}`); console.log(` 首页 Chunk 数量: ${homeChunks.length}`); console.log(` 首页指纹: ${homeFingerprint}`); console.log(` chibi Build ID: ${chibiBuildId || '未找到'}`); console.log(` chibi Chunk 数量: ${chibiChunks.length}`); console.log(` chibi指纹: ${chibiFingerprint}`); console.log(` 组合指纹: ${combinedFingerprint}`); return; } // 向后兼容:旧状态只有首页指纹(无 chibi 字段)时,chibi 属于首次初始化,不视为变更 const isChibiMigration = !oldState.chibiFingerprint; const homeChanged = oldState.fingerprint !== homeFingerprint; const chibiChanged = !isChibiMigration && oldState.chibiFingerprint !== chibiFingerprint; if (!homeChanged && !chibiChanged) { console.log(`[${timestamp()}] 无变更 (首页指纹: ${homeFingerprint})`); if (isChibiMigration) { console.log(` chibi 页指纹已初始化 (${chibiFingerprint})`); } // 更新 lastChecked 但不改变指纹 writeState(newState); return; } // 指纹变更! console.log(`[${timestamp()}] ⚠ 检测到变更!`); console.log(` 首页指纹: ${oldState.fingerprint} -> ${homeFingerprint}`); console.log(` chibi指纹: ${isChibiMigration ? '(初始化)' : `${oldState.chibiFingerprint} -> ${chibiFingerprint}`}`); console.log(` 组合指纹: ${oldState.combinedFingerprint || '无'} -> ${combinedFingerprint}`); if (homeChanged) { console.log(` 首页 Build ID: ${oldState.buildId} -> ${homeBuildId || '未找到'}`); console.log(` 首页 Chunk 数: ${oldState.chunks.length} -> ${homeChunks.length}`); } if (chibiChanged) { console.log(` chibi Build ID: ${oldState.chibiBuildId || '无'} -> ${chibiBuildId || '未找到'}`); console.log(` chibi Chunk 数: ${(oldState.chibiChunks || []).length} -> ${chibiChunks.length}`); } const logFile = writeChangeLog(oldState, newState); writeState(newState); console.log(`[${timestamp()}] 变更日志已写入: ${logFile}`); console.log(`[${timestamp()}] 请执行爬虫更新: cd competitor-analysis && node crawl.js`); } /** * 主入口 */ async function main() { const args = process.argv.slice(2); const watchIndex = args.indexOf('--watch'); if (watchIndex !== -1 && args[watchIndex + 1]) { const interval = parseInt(args[watchIndex + 1], 10); if (isNaN(interval) || interval < 1) { console.error('错误: --watch 参数必须是正整数(秒数)'); process.exit(1); } console.log(`[${timestamp()}] 监控模式已启动,每 ${interval} 秒检查一次...`); console.log(`[${timestamp()}] 按 Ctrl+C 停止\n`); // 立即执行第一次 await runCheck(); // 定时循环 setInterval(() => { runCheck().catch((err) => { console.error(`[${timestamp()}] 检测过程出错: ${err.message}`); }); }, interval * 1000); } else { // 单次检查 await runCheck(); } } main().catch((err) => { console.error(`[${timestamp()}] 致命错误: ${err.message}`); process.exit(0); // 即使出错也退出码 0 });