370 lines
13 KiB
JavaScript
370 lines
13 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* PERLER LAB — 竞品变更监控爬虫 (Competitor Change Monitor)
|
||
* ------------------------------------------------------------
|
||
* 监控 xylpindou.com 的前端代码变更,通过追踪 Next.js chunk URL 和 build ID 的指纹。
|
||
*
|
||
* 双指纹机制 (dual-fingerprint):
|
||
* - 首页 (/): fingerprint + chunks + buildId (向后兼容原有字段)
|
||
* - chibi 页 (/chibi): chibiFingerprint + chibiChunks + chibiBuildId
|
||
* - 任一页面指纹变化 => 告警 + 写入 changes-<timestamp>.log
|
||
*
|
||
* 用法:
|
||
* node monitor.js 单次检查
|
||
* node monitor.js --watch 3600 每 3600 秒循环检查
|
||
*
|
||
* 输出:
|
||
* 变更时: 写入 changes-<timestamp>.log + 控制台警报
|
||
* 无变更: 打印 "无变更" + 时间戳
|
||
*/
|
||
|
||
const https = require('https');
|
||
const http = require('http');
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const crypto = require('crypto');
|
||
|
||
// === 配置 ===
|
||
const TARGET_URL = 'https://xylpindou.com/';
|
||
const CHIBI_URL = 'https://xylpindou.com/chibi';
|
||
const STATE_FILE = path.join(__dirname, 'state.json');
|
||
const CHUNK_REGEX = /src="(\/_next\/static\/chunks\/[^"]+\.js)"/g;
|
||
const BUILD_ID_REGEX = /"buildId":"([^"]+)"/;
|
||
// Turbopack RSC 负载中的 buildId 模式: "b":"XXXX"
|
||
const BUILD_ID_RSC_REGEX = /"b":"([^"]+)"/;
|
||
|
||
// === 辅助函数 ===
|
||
|
||
/**
|
||
* 从 URL 提取文件名部分(最后一个路径段)
|
||
*/
|
||
function chunkFilename(url) {
|
||
return url.split('/').pop();
|
||
}
|
||
|
||
/**
|
||
* 获取当前时间戳字符串 (YYYY-MM-DD HH:MM:SS)
|
||
*/
|
||
function timestamp() {
|
||
const now = new Date();
|
||
const pad = (n) => String(n).padStart(2, '0');
|
||
return `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())} ${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`;
|
||
}
|
||
|
||
/**
|
||
* 获取 ISO 时间戳用于文件名
|
||
*/
|
||
function fileTimestamp() {
|
||
const now = new Date();
|
||
const pad = (n) => String(n).padStart(2, '0');
|
||
return `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
|
||
}
|
||
|
||
/**
|
||
* 获取 HTML 内容(支持 https 和 http)
|
||
*/
|
||
function fetchHTML(url) {
|
||
return new Promise((resolve, reject) => {
|
||
const client = url.startsWith('https') ? https : http;
|
||
const req = client.get(url, { timeout: 30000 }, (res) => {
|
||
// 处理重定向
|
||
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
||
const redirectUrl = new URL(res.headers.location, url).href;
|
||
return fetchHTML(redirectUrl).then(resolve).catch(reject);
|
||
}
|
||
|
||
if (res.statusCode !== 200) {
|
||
return reject(new Error(`HTTP ${res.statusCode} ${res.statusMessage}`));
|
||
}
|
||
|
||
let data = '';
|
||
res.on('data', (chunk) => { data += chunk; });
|
||
res.on('end', () => resolve(data));
|
||
});
|
||
|
||
req.on('error', (err) => reject(new Error(`网络错误: ${err.message}`)));
|
||
req.on('timeout', () => { req.destroy(); reject(new Error('请求超时')); });
|
||
});
|
||
}
|
||
|
||
/**
|
||
* 从 HTML 中提取所有 Next.js chunk URL
|
||
*/
|
||
function extractChunks(html) {
|
||
const chunks = [];
|
||
let match;
|
||
while ((match = CHUNK_REGEX.exec(html)) !== null) {
|
||
chunks.push(match[1]);
|
||
}
|
||
return [...new Set(chunks)].sort(); // 去重 + 排序
|
||
}
|
||
|
||
/**
|
||
* 从 HTML 中提取 build ID
|
||
*/
|
||
function extractBuildId(html) {
|
||
// 先尝试标准 buildId 模式
|
||
const matchStandard = html.match(BUILD_ID_REGEX);
|
||
if (matchStandard) return matchStandard[1];
|
||
|
||
// 再尝试 RSC 负载中的 "b":"XXXX" 模式
|
||
const matchRsc = html.match(BUILD_ID_RSC_REGEX);
|
||
if (matchRsc) return matchRsc[1];
|
||
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* 计算指纹: SHA-256(排序后的 chunk URL 列表 + build ID)
|
||
*/
|
||
function computeFingerprint(chunks, buildId) {
|
||
const hash = crypto.createHash('sha256');
|
||
hash.update(chunks.join('\n'));
|
||
hash.update('\n');
|
||
hash.update(buildId || '');
|
||
return hash.digest('hex');
|
||
}
|
||
|
||
/**
|
||
* 计算组合指纹: SHA-256(home指纹 + chibi指纹)
|
||
*/
|
||
function computeCombinedFingerprint(homeFingerprint, chibiFingerprint) {
|
||
const hash = crypto.createHash('sha256');
|
||
hash.update(homeFingerprint);
|
||
hash.update('\n');
|
||
hash.update(chibiFingerprint);
|
||
return hash.digest('hex');
|
||
}
|
||
|
||
/**
|
||
* 对比两页 chunk 列表,返回新增/移除(按文件名比较)
|
||
*/
|
||
function diffChunks(oldChunks, newChunks) {
|
||
const oldFilenames = new Set(oldChunks.map(chunkFilename));
|
||
const newFilenames = new Set(newChunks.map(chunkFilename));
|
||
const added = newChunks.filter(c => !oldFilenames.has(chunkFilename(c)));
|
||
const removed = oldChunks.filter(c => !newFilenames.has(chunkFilename(c)));
|
||
return { added, removed };
|
||
}
|
||
|
||
/**
|
||
* 读取之前的状态
|
||
*/
|
||
function readState() {
|
||
try {
|
||
if (fs.existsSync(STATE_FILE)) {
|
||
return JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
|
||
}
|
||
} catch (err) {
|
||
console.error(`[${timestamp()}] 读取状态文件失败: ${err.message}`);
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* 写入当前状态
|
||
*/
|
||
function writeState(state) {
|
||
const dir = path.dirname(STATE_FILE);
|
||
if (!fs.existsSync(dir)) {
|
||
fs.mkdirSync(dir, { recursive: true });
|
||
}
|
||
fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2), 'utf-8');
|
||
}
|
||
|
||
/**
|
||
* 写入变更日志(含首页 + chibi 两页的 chunk 明细)
|
||
*/
|
||
function writeChangeLog(oldState, newState) {
|
||
const oldChunks = oldState ? oldState.chunks || [] : [];
|
||
const newChunks = newState.chunks || [];
|
||
const oldChibiChunks = oldState ? oldState.chibiChunks || [] : [];
|
||
const newChibiChunks = newState.chibiChunks || [];
|
||
|
||
const homeDiff = diffChunks(oldChunks, newChunks);
|
||
const chibiDiff = diffChunks(oldChibiChunks, newChibiChunks);
|
||
|
||
const logFile = path.join(__dirname, `changes-${fileTimestamp()}.log`);
|
||
const lines = [];
|
||
lines.push(`=== 竞品变更检测报告 ===`);
|
||
lines.push(`检测时间: ${timestamp()}`);
|
||
lines.push(`目标网址: ${TARGET_URL} (首页) / ${CHIBI_URL} (chibi)`);
|
||
lines.push('');
|
||
lines.push(`组合指纹: ${oldState && oldState.combinedFingerprint ? oldState.combinedFingerprint : '无'} -> ${newState.combinedFingerprint}`);
|
||
lines.push('');
|
||
lines.push(`--- 首页 ---`);
|
||
lines.push(`旧 Build ID: ${oldState ? oldState.buildId : '无'}`);
|
||
lines.push(`新 Build ID: ${newState.buildId}`);
|
||
lines.push(`旧指纹: ${oldState ? oldState.fingerprint : '无'}`);
|
||
lines.push(`新指纹: ${newState.fingerprint}`);
|
||
lines.push('');
|
||
lines.push(` -- 首页 新增的 chunk (${homeDiff.added.length}) --`);
|
||
homeDiff.added.forEach(c => lines.push(` + ${c}`));
|
||
lines.push(` -- 首页 移除的 chunk (${homeDiff.removed.length}) --`);
|
||
homeDiff.removed.forEach(c => lines.push(` - ${c}`));
|
||
lines.push(` -- 首页 不变的 chunk (${newChunks.length - homeDiff.added.length}) --`);
|
||
lines.push(` (首页共 ${newChunks.length} 个 chunk)`);
|
||
lines.push('');
|
||
lines.push(`--- chibi 页 ---`);
|
||
lines.push(`旧 Build ID: ${oldState ? oldState.chibiBuildId : '无'}`);
|
||
lines.push(`新 Build ID: ${newState.chibiBuildId}`);
|
||
lines.push(`旧指纹: ${oldState && oldState.chibiFingerprint ? oldState.chibiFingerprint : '无'}`);
|
||
lines.push(`新指纹: ${newState.chibiFingerprint}`);
|
||
lines.push('');
|
||
lines.push(` -- chibi 新增的 chunk (${chibiDiff.added.length}) --`);
|
||
chibiDiff.added.forEach(c => lines.push(` + ${c}`));
|
||
lines.push(` -- chibi 移除的 chunk (${chibiDiff.removed.length}) --`);
|
||
chibiDiff.removed.forEach(c => lines.push(` - ${c}`));
|
||
lines.push(` -- chibi 不变的 chunk (${newChibiChunks.length - chibiDiff.added.length}) --`);
|
||
lines.push(` (chibi 共 ${newChibiChunks.length} 个 chunk)`);
|
||
|
||
fs.writeFileSync(logFile, lines.join('\n'), 'utf-8');
|
||
return logFile;
|
||
}
|
||
|
||
/**
|
||
* 单次检查逻辑
|
||
*/
|
||
async function runCheck() {
|
||
console.log(`[${timestamp()}] 正在检查 ${TARGET_URL} 和 ${CHIBI_URL} ...`);
|
||
|
||
// 并行抓取两页(每轮检查最多 2 次请求)
|
||
const [homeResult, chibiResult] = await Promise.all([
|
||
fetchHTML(TARGET_URL).then(html => ({ ok: true, html })).catch(err => ({ ok: false, err })),
|
||
fetchHTML(CHIBI_URL).then(html => ({ ok: true, html })).catch(err => ({ ok: false, err })),
|
||
]);
|
||
|
||
if (!homeResult.ok || !chibiResult.ok) {
|
||
if (!homeResult.ok) {
|
||
console.error(`[${timestamp()}] 获取首页失败: ${homeResult.err.message}`);
|
||
}
|
||
if (!chibiResult.ok) {
|
||
console.error(`[${timestamp()}] 获取 chibi 页失败: ${chibiResult.err.message}`);
|
||
}
|
||
// 网络失败时,保持旧状态,不告警、不覆盖
|
||
const oldState = readState();
|
||
if (oldState) {
|
||
console.log(`[${timestamp()}] 保留上次检测状态 (首页指纹: ${oldState.fingerprint})`);
|
||
} else {
|
||
console.log(`[${timestamp()}] 尚无历史状态,跳过`);
|
||
}
|
||
return;
|
||
}
|
||
|
||
const homeChunks = extractChunks(homeResult.html);
|
||
const homeBuildId = extractBuildId(homeResult.html);
|
||
const homeFingerprint = computeFingerprint(homeChunks, homeBuildId);
|
||
|
||
const chibiChunks = extractChunks(chibiResult.html);
|
||
const chibiBuildId = extractBuildId(chibiResult.html);
|
||
const chibiFingerprint = computeFingerprint(chibiChunks, chibiBuildId);
|
||
|
||
if (homeChunks.length === 0) {
|
||
console.error(`[${timestamp()}] 警告: 首页未找到任何 Next.js chunk URL,页面结构可能已变更`);
|
||
}
|
||
if (chibiChunks.length === 0) {
|
||
console.error(`[${timestamp()}] 警告: chibi 页未找到任何 Next.js chunk URL,页面结构可能已变更`);
|
||
}
|
||
|
||
const combinedFingerprint = computeCombinedFingerprint(homeFingerprint, chibiFingerprint);
|
||
const oldState = readState();
|
||
|
||
const newState = {
|
||
fingerprint: homeFingerprint,
|
||
chunks: homeChunks,
|
||
buildId: homeBuildId || 'unknown',
|
||
chibiFingerprint,
|
||
chibiChunks,
|
||
chibiBuildId: chibiBuildId || 'unknown',
|
||
combinedFingerprint,
|
||
lastChecked: new Date().toISOString(),
|
||
};
|
||
|
||
if (!oldState) {
|
||
// 首次运行:记录状态
|
||
writeState(newState);
|
||
console.log(`[${timestamp()}] 首次检测完成,已记录初始状态`);
|
||
console.log(` 首页 Build ID: ${homeBuildId || '未找到'}`);
|
||
console.log(` 首页 Chunk 数量: ${homeChunks.length}`);
|
||
console.log(` 首页指纹: ${homeFingerprint}`);
|
||
console.log(` chibi Build ID: ${chibiBuildId || '未找到'}`);
|
||
console.log(` chibi Chunk 数量: ${chibiChunks.length}`);
|
||
console.log(` chibi指纹: ${chibiFingerprint}`);
|
||
console.log(` 组合指纹: ${combinedFingerprint}`);
|
||
return;
|
||
}
|
||
|
||
// 向后兼容:旧状态只有首页指纹(无 chibi 字段)时,chibi 属于首次初始化,不视为变更
|
||
const isChibiMigration = !oldState.chibiFingerprint;
|
||
|
||
const homeChanged = oldState.fingerprint !== homeFingerprint;
|
||
const chibiChanged = !isChibiMigration && oldState.chibiFingerprint !== chibiFingerprint;
|
||
|
||
if (!homeChanged && !chibiChanged) {
|
||
console.log(`[${timestamp()}] 无变更 (首页指纹: ${homeFingerprint})`);
|
||
if (isChibiMigration) {
|
||
console.log(` chibi 页指纹已初始化 (${chibiFingerprint})`);
|
||
}
|
||
// 更新 lastChecked 但不改变指纹
|
||
writeState(newState);
|
||
return;
|
||
}
|
||
|
||
// 指纹变更!
|
||
console.log(`[${timestamp()}] ⚠ 检测到变更!`);
|
||
console.log(` 首页指纹: ${oldState.fingerprint} -> ${homeFingerprint}`);
|
||
console.log(` chibi指纹: ${isChibiMigration ? '(初始化)' : `${oldState.chibiFingerprint} -> ${chibiFingerprint}`}`);
|
||
console.log(` 组合指纹: ${oldState.combinedFingerprint || '无'} -> ${combinedFingerprint}`);
|
||
if (homeChanged) {
|
||
console.log(` 首页 Build ID: ${oldState.buildId} -> ${homeBuildId || '未找到'}`);
|
||
console.log(` 首页 Chunk 数: ${oldState.chunks.length} -> ${homeChunks.length}`);
|
||
}
|
||
if (chibiChanged) {
|
||
console.log(` chibi Build ID: ${oldState.chibiBuildId || '无'} -> ${chibiBuildId || '未找到'}`);
|
||
console.log(` chibi Chunk 数: ${(oldState.chibiChunks || []).length} -> ${chibiChunks.length}`);
|
||
}
|
||
|
||
const logFile = writeChangeLog(oldState, newState);
|
||
writeState(newState);
|
||
|
||
console.log(`[${timestamp()}] 变更日志已写入: ${logFile}`);
|
||
console.log(`[${timestamp()}] 请执行爬虫更新: cd competitor-analysis && node crawl.js`);
|
||
}
|
||
|
||
/**
|
||
* 主入口
|
||
*/
|
||
async function main() {
|
||
const args = process.argv.slice(2);
|
||
const watchIndex = args.indexOf('--watch');
|
||
|
||
if (watchIndex !== -1 && args[watchIndex + 1]) {
|
||
const interval = parseInt(args[watchIndex + 1], 10);
|
||
if (isNaN(interval) || interval < 1) {
|
||
console.error('错误: --watch 参数必须是正整数(秒数)');
|
||
process.exit(1);
|
||
}
|
||
|
||
console.log(`[${timestamp()}] 监控模式已启动,每 ${interval} 秒检查一次...`);
|
||
console.log(`[${timestamp()}] 按 Ctrl+C 停止\n`);
|
||
|
||
// 立即执行第一次
|
||
await runCheck();
|
||
|
||
// 定时循环
|
||
setInterval(() => {
|
||
runCheck().catch((err) => {
|
||
console.error(`[${timestamp()}] 检测过程出错: ${err.message}`);
|
||
});
|
||
}, interval * 1000);
|
||
} else {
|
||
// 单次检查
|
||
await runCheck();
|
||
}
|
||
}
|
||
|
||
main().catch((err) => {
|
||
console.error(`[${timestamp()}] 致命错误: ${err.message}`);
|
||
process.exit(0); // 即使出错也退出码 0
|
||
});
|