365 lines
12 KiB
JavaScript
365 lines
12 KiB
JavaScript
#!/usr/bin/env node
|
|
/*
|
|
* Read-only PixelRAG-style visual evidence capture worker.
|
|
*
|
|
* The API/service builds manifests; this worker consumes one manifest and writes
|
|
* screenshot/tile artifacts only when explicitly invoked.
|
|
*/
|
|
|
|
const fs = require('fs');
|
|
const os = require('os');
|
|
const path = require('path');
|
|
const Module = require('module');
|
|
|
|
const CAPTURE_POLICY = 'read_only_pixelrag_visual_capture_artifact_v1';
|
|
const MANIFEST_POLICY = 'read_only_pixelrag_visual_evidence_manifest_v1';
|
|
const DEFAULT_OUTPUT_DIR = 'runtime_artifacts/pixelrag_visual_evidence';
|
|
const DEFAULT_TIMEOUT_MS = 30000;
|
|
const DEFAULT_SETTLE_MS = 500;
|
|
const DEFAULT_MAX_TILES = 80;
|
|
|
|
const ALLOWED_PLATFORM_HOSTS = {
|
|
momo: new Set(['m.momoshop.com.tw', 'www.momoshop.com.tw']),
|
|
pchome: new Set(['24h.pchome.com.tw', 'ecshweb.pchome.com.tw', 'ecapi-cdn.pchome.com.tw']),
|
|
market_intel: new Set(),
|
|
external_market: new Set(),
|
|
};
|
|
|
|
function parseArgs(argv) {
|
|
const options = {
|
|
manifestFile: '',
|
|
manifestJson: '',
|
|
outputDir: DEFAULT_OUTPUT_DIR,
|
|
dryRun: false,
|
|
timeoutMs: DEFAULT_TIMEOUT_MS,
|
|
settleMs: DEFAULT_SETTLE_MS,
|
|
maxTiles: DEFAULT_MAX_TILES,
|
|
waitUntil: 'domcontentloaded',
|
|
json: false,
|
|
requireAllowedHost: true,
|
|
};
|
|
|
|
for (let i = 0; i < argv.length; i += 1) {
|
|
const arg = argv[i];
|
|
if (arg === '--manifest-file') {
|
|
options.manifestFile = argv[++i];
|
|
} else if (arg === '--manifest-json') {
|
|
options.manifestJson = argv[++i];
|
|
} else if (arg === '--output-dir') {
|
|
options.outputDir = argv[++i];
|
|
} else if (arg === '--dry-run') {
|
|
options.dryRun = true;
|
|
} else if (arg === '--timeout') {
|
|
options.timeoutMs = parseInt(argv[++i], 10) * 1000;
|
|
} else if (arg === '--settle-ms') {
|
|
options.settleMs = parseInt(argv[++i], 10);
|
|
} else if (arg === '--max-tiles') {
|
|
options.maxTiles = parseInt(argv[++i], 10);
|
|
} else if (arg === '--wait-until') {
|
|
options.waitUntil = argv[++i];
|
|
} else if (arg === '--json') {
|
|
options.json = true;
|
|
} else if (arg === '--allow-any-host') {
|
|
options.requireAllowedHost = false;
|
|
} else if (arg === '--help' || arg === '-h') {
|
|
printHelp();
|
|
process.exit(0);
|
|
} else {
|
|
throw new Error(`Unknown argument: ${arg}`);
|
|
}
|
|
}
|
|
|
|
if (!options.manifestFile && !options.manifestJson) {
|
|
throw new Error('Pass --manifest-file or --manifest-json.');
|
|
}
|
|
return options;
|
|
}
|
|
|
|
function printHelp() {
|
|
console.log(`PixelRAG visual evidence capture worker
|
|
|
|
Options:
|
|
--manifest-file PATH Read manifest JSON from file
|
|
--manifest-json JSON Read manifest JSON from CLI arg
|
|
--output-dir DIR Artifact root, default ${DEFAULT_OUTPUT_DIR}
|
|
--dry-run Validate and print planned outputs without browser/network/files
|
|
--timeout SEC Navigation timeout, default 30
|
|
--settle-ms MS Fixed post-DOM settle wait, default 500
|
|
--max-tiles N Tile cap, default 80
|
|
--wait-until EVENT Playwright waitUntil, default domcontentloaded
|
|
--allow-any-host Disable platform host allowlist for local research only
|
|
--json Print compact JSON only
|
|
`);
|
|
}
|
|
|
|
function requirePlaywright() {
|
|
try {
|
|
return require('playwright');
|
|
} catch (error) {
|
|
const candidates = [
|
|
process.env.PLAYWRIGHT_NODE_MODULE_DIR,
|
|
process.env.NODE_PATH,
|
|
path.join(os.homedir(), '.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules'),
|
|
].filter(Boolean);
|
|
|
|
for (const candidate of candidates) {
|
|
const packageJson = path.join(candidate, 'playwright', 'package.json');
|
|
if (fs.existsSync(packageJson)) {
|
|
return Module.createRequire(packageJson)('playwright');
|
|
}
|
|
}
|
|
|
|
throw new Error(
|
|
'Cannot load playwright. Set NODE_PATH or PLAYWRIGHT_NODE_MODULE_DIR to a node_modules directory containing playwright.'
|
|
);
|
|
}
|
|
}
|
|
|
|
function findChromeExecutable() {
|
|
const candidates = [
|
|
process.env.PIXELRAG_CHROME_PATH,
|
|
process.env.RESPONSIVE_CHROME_PATH,
|
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
|
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
|
|
'/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
|
|
].filter(Boolean);
|
|
return candidates.find((candidate) => fs.existsSync(candidate)) || '';
|
|
}
|
|
|
|
function safeName(input) {
|
|
return String(input || '').replace(/[^a-z0-9]+/gi, '_').replace(/^_+|_+$/g, '').toLowerCase() || 'unknown';
|
|
}
|
|
|
|
function loadManifest(options) {
|
|
const raw = options.manifestFile
|
|
? fs.readFileSync(options.manifestFile, 'utf-8')
|
|
: options.manifestJson;
|
|
return JSON.parse(raw);
|
|
}
|
|
|
|
function validateManifest(manifest, options) {
|
|
const errors = [];
|
|
if (!manifest || typeof manifest !== 'object') {
|
|
errors.push('manifest must be an object');
|
|
return errors;
|
|
}
|
|
if (manifest.policy !== MANIFEST_POLICY) {
|
|
errors.push(`manifest policy must be ${MANIFEST_POLICY}`);
|
|
}
|
|
const target = manifest.capture_target || {};
|
|
const url = String(target.url || '');
|
|
let parsedUrl = null;
|
|
try {
|
|
parsedUrl = new URL(url);
|
|
} catch (error) {
|
|
errors.push('capture_target.url must be an absolute URL');
|
|
}
|
|
if (parsedUrl) {
|
|
if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
|
|
errors.push('capture_target.url must use http(s)');
|
|
}
|
|
if (parsedUrl.username || parsedUrl.password) {
|
|
errors.push('URL credentials are not allowed');
|
|
}
|
|
const platform = String(target.platform || '');
|
|
const allowed = ALLOWED_PLATFORM_HOSTS[platform];
|
|
if (options.requireAllowedHost && allowed && allowed.size > 0 && !allowed.has(parsedUrl.hostname)) {
|
|
errors.push(`host ${parsedUrl.hostname} is not allowed for platform ${platform}`);
|
|
}
|
|
}
|
|
if (!target.platform) errors.push('capture_target.platform is required');
|
|
if (!target.crawler) errors.push('capture_target.crawler is required');
|
|
if (!target.trigger_reason) errors.push('capture_target.trigger_reason is required');
|
|
return errors;
|
|
}
|
|
|
|
function positiveInt(value, fallback) {
|
|
const parsed = parseInt(value, 10);
|
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
}
|
|
|
|
function buildTilePlan({ width, height, tileWidth, tileHeight, maxTiles }) {
|
|
const tiles = [];
|
|
const tilesX = Math.max(1, Math.ceil(width / tileWidth));
|
|
const tilesY = Math.max(1, Math.ceil(height / tileHeight));
|
|
for (let row = 0; row < tilesY; row += 1) {
|
|
for (let col = 0; col < tilesX; col += 1) {
|
|
if (tiles.length >= maxTiles) break;
|
|
const x = col * tileWidth;
|
|
const y = row * tileHeight;
|
|
tiles.push({
|
|
index: tiles.length,
|
|
row,
|
|
col,
|
|
x,
|
|
y,
|
|
width: Math.min(tileWidth, Math.max(1, width - x)),
|
|
height: Math.min(tileHeight, Math.max(1, height - y)),
|
|
});
|
|
}
|
|
}
|
|
return {
|
|
tiles_x: tilesX,
|
|
tiles_y: tilesY,
|
|
planned_tile_count: tilesX * tilesY,
|
|
emitted_tile_count: tiles.length,
|
|
capped: tiles.length < tilesX * tilesY,
|
|
tiles,
|
|
};
|
|
}
|
|
|
|
function outputPaths(manifest, options) {
|
|
const target = manifest.capture_target || {};
|
|
const manifestId = manifest.manifest_id || safeName(`${target.platform}-${target.crawler}-${target.url}`).slice(0, 20);
|
|
const root = path.resolve(options.outputDir, safeName(target.platform || 'unknown'), manifestId);
|
|
return {
|
|
root,
|
|
receipt: path.join(root, 'capture_receipt.json'),
|
|
screenshot: path.join(root, 'fullpage.png'),
|
|
tilesDir: path.join(root, 'tiles'),
|
|
};
|
|
}
|
|
|
|
function buildBaseArtifact(manifest, options, errors = []) {
|
|
const viewport = manifest.viewport || {};
|
|
const tileSize = manifest.tile_size || {};
|
|
const pageSize = manifest.page_size || {};
|
|
const width = positiveInt(pageSize.width, positiveInt(viewport.width, 1440));
|
|
const height = positiveInt(pageSize.height, positiveInt(viewport.height, 950));
|
|
const tileWidth = positiveInt(tileSize.width, 512);
|
|
const tileHeight = positiveInt(tileSize.height, 512);
|
|
const paths = outputPaths(manifest, options);
|
|
const tilePlan = buildTilePlan({
|
|
width,
|
|
height,
|
|
tileWidth,
|
|
tileHeight,
|
|
maxTiles: positiveInt(options.maxTiles, DEFAULT_MAX_TILES),
|
|
});
|
|
return {
|
|
success: errors.length === 0,
|
|
policy: CAPTURE_POLICY,
|
|
status: errors.length ? 'rejected' : (options.dryRun ? 'dry_run_ready' : 'capture_pending'),
|
|
generated_at: new Date().toISOString(),
|
|
errors,
|
|
manifest_id: manifest.manifest_id || '',
|
|
manifest_policy: manifest.policy || '',
|
|
capture_target: manifest.capture_target || {},
|
|
controlled_apply: {
|
|
db_write: false,
|
|
secret_read: false,
|
|
github_dependency: false,
|
|
production_price_write: false,
|
|
artifact_write: !options.dryRun && errors.length === 0,
|
|
},
|
|
planned_output: {
|
|
root: paths.root,
|
|
receipt: paths.receipt,
|
|
screenshot: paths.screenshot,
|
|
tiles_dir: paths.tilesDir,
|
|
},
|
|
viewport: {
|
|
width: positiveInt(viewport.width, 1440),
|
|
height: positiveInt(viewport.height, 950),
|
|
name: viewport.name || 'desktop-1440',
|
|
},
|
|
page_size: { width, height },
|
|
tile_size: { width: tileWidth, height: tileHeight },
|
|
tile_plan: tilePlan,
|
|
files: [],
|
|
};
|
|
}
|
|
|
|
async function capture(manifest, options, artifact) {
|
|
const { chromium } = requirePlaywright();
|
|
const paths = outputPaths(manifest, options);
|
|
const viewport = artifact.viewport;
|
|
fs.mkdirSync(paths.tilesDir, { recursive: true });
|
|
|
|
const launchOptions = { headless: true };
|
|
const chromePath = findChromeExecutable();
|
|
if (chromePath) {
|
|
launchOptions.executablePath = chromePath;
|
|
}
|
|
|
|
const browser = await chromium.launch(launchOptions);
|
|
const context = await browser.newContext({ ignoreHTTPSErrors: true, viewport });
|
|
const page = await context.newPage();
|
|
try {
|
|
const response = await page.goto(manifest.capture_target.url, {
|
|
waitUntil: options.waitUntil,
|
|
timeout: options.timeoutMs,
|
|
});
|
|
await page.waitForSelector('body', { timeout: Math.min(options.timeoutMs, 5000) }).catch(() => {});
|
|
await page.evaluate(() => document.fonts && document.fonts.ready).catch(() => {});
|
|
if (options.settleMs > 0) {
|
|
await page.waitForTimeout(options.settleMs);
|
|
}
|
|
const metrics = await page.evaluate(() => {
|
|
const root = document.documentElement;
|
|
const body = document.body;
|
|
return {
|
|
final_url: location.href,
|
|
title: document.title,
|
|
scroll_width: Math.max(root.scrollWidth, body.scrollWidth, window.innerWidth),
|
|
scroll_height: Math.max(root.scrollHeight, body.scrollHeight, window.innerHeight),
|
|
};
|
|
});
|
|
|
|
await page.screenshot({ path: paths.screenshot, fullPage: true });
|
|
const files = [{ kind: 'fullpage_screenshot', path: paths.screenshot }];
|
|
const tilePlan = buildTilePlan({
|
|
width: metrics.scroll_width,
|
|
height: metrics.scroll_height,
|
|
tileWidth: artifact.tile_size.width,
|
|
tileHeight: artifact.tile_size.height,
|
|
maxTiles: positiveInt(options.maxTiles, DEFAULT_MAX_TILES),
|
|
});
|
|
for (const tile of tilePlan.tiles) {
|
|
const filePath = path.join(paths.tilesDir, `tile_${String(tile.index).padStart(3, '0')}.png`);
|
|
await page.screenshot({
|
|
path: filePath,
|
|
clip: { x: tile.x, y: tile.y, width: tile.width, height: tile.height },
|
|
});
|
|
files.push({ kind: 'tile', path: filePath, tile });
|
|
}
|
|
|
|
const captured = {
|
|
...artifact,
|
|
success: true,
|
|
status: 'captured',
|
|
http_status: response ? response.status() : 0,
|
|
page_metrics: metrics,
|
|
page_size: { width: metrics.scroll_width, height: metrics.scroll_height },
|
|
tile_plan: tilePlan,
|
|
files,
|
|
};
|
|
fs.writeFileSync(paths.receipt, `${JSON.stringify(captured, null, 2)}\n`, 'utf-8');
|
|
captured.files.push({ kind: 'receipt', path: paths.receipt });
|
|
return captured;
|
|
} finally {
|
|
await page.close().catch(() => {});
|
|
await context.close().catch(() => {});
|
|
await browser.close().catch(() => {});
|
|
}
|
|
}
|
|
|
|
async function main() {
|
|
const options = parseArgs(process.argv.slice(2));
|
|
const manifest = loadManifest(options);
|
|
const errors = validateManifest(manifest, options);
|
|
const artifact = buildBaseArtifact(manifest, options, errors);
|
|
if (errors.length || options.dryRun) {
|
|
console.log(JSON.stringify(artifact, null, 2));
|
|
process.exitCode = errors.length ? 1 : 0;
|
|
return;
|
|
}
|
|
const captured = await capture(manifest, options, artifact);
|
|
console.log(JSON.stringify(captured, null, 2));
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(error.message || error);
|
|
process.exit(1);
|
|
});
|