Files

370 lines
13 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env node
/**
* PERLER LAB — 竞品变更监控爬虫 (Competitor Change Monitor)
* ------------------------------------------------------------
* 监控 xylpindou.com 的前端代码变更,通过追踪 Next.js chunk URL 和 build ID 的指纹。
*
* 双指纹机制 (dual-fingerprint):
* - 首页 (/): fingerprint + chunks + buildId (向后兼容原有字段)
* - chibi 页 (/chibi): chibiFingerprint + chibiChunks + chibiBuildId
* - 任一页面指纹变化 => 告警 + 写入 changes-<timestamp>.log
*
* 用法:
* node monitor.js 单次检查
* node monitor.js --watch 3600 每 3600 秒循环检查
*
* 输出:
* 变更时: 写入 changes-<timestamp>.log + 控制台警报
* 无变更: 打印 "无变更" + 时间戳
*/
const https = require('https');
const http = require('http');
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
// === 配置 ===
const TARGET_URL = 'https://xylpindou.com/';
const CHIBI_URL = 'https://xylpindou.com/chibi';
const STATE_FILE = path.join(__dirname, 'state.json');
const CHUNK_REGEX = /src="(\/_next\/static\/chunks\/[^"]+\.js)"/g;
const BUILD_ID_REGEX = /"buildId":"([^"]+)"/;
// Turbopack RSC 负载中的 buildId 模式: "b":"XXXX"
const BUILD_ID_RSC_REGEX = /"b":"([^"]+)"/;
// === 辅助函数 ===
/**
* 从 URL 提取文件名部分(最后一个路径段)
*/
function chunkFilename(url) {
return url.split('/').pop();
}
/**
* 获取当前时间戳字符串 (YYYY-MM-DD HH:MM:SS)
*/
function timestamp() {
const now = new Date();
const pad = (n) => String(n).padStart(2, '0');
return `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())} ${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`;
}
/**
* 获取 ISO 时间戳用于文件名
*/
function fileTimestamp() {
const now = new Date();
const pad = (n) => String(n).padStart(2, '0');
return `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
}
/**
* 获取 HTML 内容(支持 https 和 http)
*/
function fetchHTML(url) {
return new Promise((resolve, reject) => {
const client = url.startsWith('https') ? https : http;
const req = client.get(url, { timeout: 30000 }, (res) => {
// 处理重定向
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
const redirectUrl = new URL(res.headers.location, url).href;
return fetchHTML(redirectUrl).then(resolve).catch(reject);
}
if (res.statusCode !== 200) {
return reject(new Error(`HTTP ${res.statusCode} ${res.statusMessage}`));
}
let data = '';
res.on('data', (chunk) => { data += chunk; });
res.on('end', () => resolve(data));
});
req.on('error', (err) => reject(new Error(`网络错误: ${err.message}`)));
req.on('timeout', () => { req.destroy(); reject(new Error('请求超时')); });
});
}
/**
* 从 HTML 中提取所有 Next.js chunk URL
*/
function extractChunks(html) {
const chunks = [];
let match;
while ((match = CHUNK_REGEX.exec(html)) !== null) {
chunks.push(match[1]);
}
return [...new Set(chunks)].sort(); // 去重 + 排序
}
/**
* 从 HTML 中提取 build ID
*/
function extractBuildId(html) {
// 先尝试标准 buildId 模式
const matchStandard = html.match(BUILD_ID_REGEX);
if (matchStandard) return matchStandard[1];
// 再尝试 RSC 负载中的 "b":"XXXX" 模式
const matchRsc = html.match(BUILD_ID_RSC_REGEX);
if (matchRsc) return matchRsc[1];
return null;
}
/**
* 计算指纹: SHA-256(排序后的 chunk URL 列表 + build ID)
*/
function computeFingerprint(chunks, buildId) {
const hash = crypto.createHash('sha256');
hash.update(chunks.join('\n'));
hash.update('\n');
hash.update(buildId || '');
return hash.digest('hex');
}
/**
* 计算组合指纹: SHA-256(home指纹 + chibi指纹)
*/
function computeCombinedFingerprint(homeFingerprint, chibiFingerprint) {
const hash = crypto.createHash('sha256');
hash.update(homeFingerprint);
hash.update('\n');
hash.update(chibiFingerprint);
return hash.digest('hex');
}
/**
* 对比两页 chunk 列表,返回新增/移除(按文件名比较)
*/
function diffChunks(oldChunks, newChunks) {
const oldFilenames = new Set(oldChunks.map(chunkFilename));
const newFilenames = new Set(newChunks.map(chunkFilename));
const added = newChunks.filter(c => !oldFilenames.has(chunkFilename(c)));
const removed = oldChunks.filter(c => !newFilenames.has(chunkFilename(c)));
return { added, removed };
}
/**
* 读取之前的状态
*/
function readState() {
try {
if (fs.existsSync(STATE_FILE)) {
return JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
}
} catch (err) {
console.error(`[${timestamp()}] 读取状态文件失败: ${err.message}`);
}
return null;
}
/**
* 写入当前状态
*/
function writeState(state) {
const dir = path.dirname(STATE_FILE);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2), 'utf-8');
}
/**
* 写入变更日志(含首页 + chibi 两页的 chunk 明细)
*/
function writeChangeLog(oldState, newState) {
const oldChunks = oldState ? oldState.chunks || [] : [];
const newChunks = newState.chunks || [];
const oldChibiChunks = oldState ? oldState.chibiChunks || [] : [];
const newChibiChunks = newState.chibiChunks || [];
const homeDiff = diffChunks(oldChunks, newChunks);
const chibiDiff = diffChunks(oldChibiChunks, newChibiChunks);
const logFile = path.join(__dirname, `changes-${fileTimestamp()}.log`);
const lines = [];
lines.push(`=== 竞品变更检测报告 ===`);
lines.push(`检测时间: ${timestamp()}`);
lines.push(`目标网址: ${TARGET_URL} (首页) / ${CHIBI_URL} (chibi)`);
lines.push('');
lines.push(`组合指纹: ${oldState && oldState.combinedFingerprint ? oldState.combinedFingerprint : '无'} -> ${newState.combinedFingerprint}`);
lines.push('');
lines.push(`--- 首页 ---`);
lines.push(`旧 Build ID: ${oldState ? oldState.buildId : '无'}`);
lines.push(`新 Build ID: ${newState.buildId}`);
lines.push(`旧指纹: ${oldState ? oldState.fingerprint : '无'}`);
lines.push(`新指纹: ${newState.fingerprint}`);
lines.push('');
lines.push(` -- 首页 新增的 chunk (${homeDiff.added.length}) --`);
homeDiff.added.forEach(c => lines.push(` + ${c}`));
lines.push(` -- 首页 移除的 chunk (${homeDiff.removed.length}) --`);
homeDiff.removed.forEach(c => lines.push(` - ${c}`));
lines.push(` -- 首页 不变的 chunk (${newChunks.length - homeDiff.added.length}) --`);
lines.push(` (首页共 ${newChunks.length} 个 chunk)`);
lines.push('');
lines.push(`--- chibi 页 ---`);
lines.push(`旧 Build ID: ${oldState ? oldState.chibiBuildId : '无'}`);
lines.push(`新 Build ID: ${newState.chibiBuildId}`);
lines.push(`旧指纹: ${oldState && oldState.chibiFingerprint ? oldState.chibiFingerprint : '无'}`);
lines.push(`新指纹: ${newState.chibiFingerprint}`);
lines.push('');
lines.push(` -- chibi 新增的 chunk (${chibiDiff.added.length}) --`);
chibiDiff.added.forEach(c => lines.push(` + ${c}`));
lines.push(` -- chibi 移除的 chunk (${chibiDiff.removed.length}) --`);
chibiDiff.removed.forEach(c => lines.push(` - ${c}`));
lines.push(` -- chibi 不变的 chunk (${newChibiChunks.length - chibiDiff.added.length}) --`);
lines.push(` (chibi 共 ${newChibiChunks.length} 个 chunk)`);
fs.writeFileSync(logFile, lines.join('\n'), 'utf-8');
return logFile;
}
/**
* 单次检查逻辑
*/
async function runCheck() {
console.log(`[${timestamp()}] 正在检查 ${TARGET_URL} 和 ${CHIBI_URL} ...`);
// 并行抓取两页(每轮检查最多 2 次请求)
const [homeResult, chibiResult] = await Promise.all([
fetchHTML(TARGET_URL).then(html => ({ ok: true, html })).catch(err => ({ ok: false, err })),
fetchHTML(CHIBI_URL).then(html => ({ ok: true, html })).catch(err => ({ ok: false, err })),
]);
if (!homeResult.ok || !chibiResult.ok) {
if (!homeResult.ok) {
console.error(`[${timestamp()}] 获取首页失败: ${homeResult.err.message}`);
}
if (!chibiResult.ok) {
console.error(`[${timestamp()}] 获取 chibi 页失败: ${chibiResult.err.message}`);
}
// 网络失败时,保持旧状态,不告警、不覆盖
const oldState = readState();
if (oldState) {
console.log(`[${timestamp()}] 保留上次检测状态 (首页指纹: ${oldState.fingerprint})`);
} else {
console.log(`[${timestamp()}] 尚无历史状态,跳过`);
}
return;
}
const homeChunks = extractChunks(homeResult.html);
const homeBuildId = extractBuildId(homeResult.html);
const homeFingerprint = computeFingerprint(homeChunks, homeBuildId);
const chibiChunks = extractChunks(chibiResult.html);
const chibiBuildId = extractBuildId(chibiResult.html);
const chibiFingerprint = computeFingerprint(chibiChunks, chibiBuildId);
if (homeChunks.length === 0) {
console.error(`[${timestamp()}] 警告: 首页未找到任何 Next.js chunk URL,页面结构可能已变更`);
}
if (chibiChunks.length === 0) {
console.error(`[${timestamp()}] 警告: chibi 页未找到任何 Next.js chunk URL,页面结构可能已变更`);
}
const combinedFingerprint = computeCombinedFingerprint(homeFingerprint, chibiFingerprint);
const oldState = readState();
const newState = {
fingerprint: homeFingerprint,
chunks: homeChunks,
buildId: homeBuildId || 'unknown',
chibiFingerprint,
chibiChunks,
chibiBuildId: chibiBuildId || 'unknown',
combinedFingerprint,
lastChecked: new Date().toISOString(),
};
if (!oldState) {
// 首次运行:记录状态
writeState(newState);
console.log(`[${timestamp()}] 首次检测完成,已记录初始状态`);
console.log(` 首页 Build ID: ${homeBuildId || '未找到'}`);
console.log(` 首页 Chunk 数量: ${homeChunks.length}`);
console.log(` 首页指纹: ${homeFingerprint}`);
console.log(` chibi Build ID: ${chibiBuildId || '未找到'}`);
console.log(` chibi Chunk 数量: ${chibiChunks.length}`);
console.log(` chibi指纹: ${chibiFingerprint}`);
console.log(` 组合指纹: ${combinedFingerprint}`);
return;
}
// 向后兼容:旧状态只有首页指纹(无 chibi 字段)时,chibi 属于首次初始化,不视为变更
const isChibiMigration = !oldState.chibiFingerprint;
const homeChanged = oldState.fingerprint !== homeFingerprint;
const chibiChanged = !isChibiMigration && oldState.chibiFingerprint !== chibiFingerprint;
if (!homeChanged && !chibiChanged) {
console.log(`[${timestamp()}] 无变更 (首页指纹: ${homeFingerprint})`);
if (isChibiMigration) {
console.log(` chibi 页指纹已初始化 (${chibiFingerprint})`);
}
// 更新 lastChecked 但不改变指纹
writeState(newState);
return;
}
// 指纹变更!
console.log(`[${timestamp()}] ⚠ 检测到变更!`);
console.log(` 首页指纹: ${oldState.fingerprint} -> ${homeFingerprint}`);
console.log(` chibi指纹: ${isChibiMigration ? '(初始化)' : `${oldState.chibiFingerprint} -> ${chibiFingerprint}`}`);
console.log(` 组合指纹: ${oldState.combinedFingerprint || '无'} -> ${combinedFingerprint}`);
if (homeChanged) {
console.log(` 首页 Build ID: ${oldState.buildId} -> ${homeBuildId || '未找到'}`);
console.log(` 首页 Chunk 数: ${oldState.chunks.length} -> ${homeChunks.length}`);
}
if (chibiChanged) {
console.log(` chibi Build ID: ${oldState.chibiBuildId || '无'} -> ${chibiBuildId || '未找到'}`);
console.log(` chibi Chunk 数: ${(oldState.chibiChunks || []).length} -> ${chibiChunks.length}`);
}
const logFile = writeChangeLog(oldState, newState);
writeState(newState);
console.log(`[${timestamp()}] 变更日志已写入: ${logFile}`);
console.log(`[${timestamp()}] 请执行爬虫更新: cd competitor-analysis && node crawl.js`);
}
/**
* 主入口
*/
async function main() {
const args = process.argv.slice(2);
const watchIndex = args.indexOf('--watch');
if (watchIndex !== -1 && args[watchIndex + 1]) {
const interval = parseInt(args[watchIndex + 1], 10);
if (isNaN(interval) || interval < 1) {
console.error('错误: --watch 参数必须是正整数(秒数)');
process.exit(1);
}
console.log(`[${timestamp()}] 监控模式已启动,每 ${interval} 秒检查一次...`);
console.log(`[${timestamp()}] 按 Ctrl+C 停止\n`);
// 立即执行第一次
await runCheck();
// 定时循环
setInterval(() => {
runCheck().catch((err) => {
console.error(`[${timestamp()}] 检测过程出错: ${err.message}`);
});
}, interval * 1000);
} else {
// 单次检查
await runCheck();
}
}
main().catch((err) => {
console.error(`[${timestamp()}] 致命错误: ${err.message}`);
process.exit(0); // 即使出错也退出码 0
});