diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index 6dcd39c..9895afa 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -87,6 +87,28 @@ python scripts/ops/report_pixelrag_crawler_integration.py \ /api/ai-automation/pixelrag-crawler-integration?platform=pchome&manifest_url=https://24h.pchome.com.tw/prod/TEST-000000001&crawler=PChomeCrawler.search_products&trigger_reason=parser_empty ``` +視覺證據 capture worker： + +```bash +python3 scripts/ops/report_pixelrag_crawler_integration.py \ + --platform momo \ + --manifest-url "https://m.momoshop.com.tw/search.momo?searchKeyword=test" \ + --crawler MomoCrawler.search_products \ + --trigger-reason parser_empty \ + --page-size page=1440x1900 \ + --tile-size tile=512x512 \ + > /tmp/pixelrag_manifest.json + +node scripts/ops/capture_pixelrag_visual_evidence.js \ + --manifest-file /tmp/pixelrag_manifest.json \ + --output-dir data/ai_automation/pixelrag_visual_evidence \ + --dry-run + +node scripts/ops/capture_pixelrag_visual_evidence.js \ + --manifest-file /tmp/pixelrag_manifest.json \ + --output-dir data/ai_automation/pixelrag_visual_evidence +``` + 安全邊界： - read-only；不登入、不下單、不加入購物車、不寫第三方狀態。 diff --git a/scripts/ops/capture_pixelrag_visual_evidence.js b/scripts/ops/capture_pixelrag_visual_evidence.js new file mode 100644 index 0000000..3ff6fca --- /dev/null +++ b/scripts/ops/capture_pixelrag_visual_evidence.js @@ -0,0 +1,364 @@ +#!/usr/bin/env node +/* + * Read-only PixelRAG-style visual evidence capture worker. + * + * The API/service builds manifests; this worker consumes one manifest and writes + * screenshot/tile artifacts only when explicitly invoked. + */ + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const Module = require('module'); + +const CAPTURE_POLICY = 'read_only_pixelrag_visual_capture_artifact_v1'; +const MANIFEST_POLICY = 'read_only_pixelrag_visual_evidence_manifest_v1'; +const DEFAULT_OUTPUT_DIR = 'data/ai_automation/pixelrag_visual_evidence'; +const DEFAULT_TIMEOUT_MS = 30000; +const DEFAULT_SETTLE_MS = 500; +const DEFAULT_MAX_TILES = 80; + +const ALLOWED_PLATFORM_HOSTS = { + momo: new Set(['m.momoshop.com.tw', 'www.momoshop.com.tw']), + pchome: new Set(['24h.pchome.com.tw', 'ecshweb.pchome.com.tw', 'ecapi-cdn.pchome.com.tw']), + market_intel: new Set(), + external_market: new Set(), +}; + +function parseArgs(argv) { + const options = { + manifestFile: '', + manifestJson: '', + outputDir: DEFAULT_OUTPUT_DIR, + dryRun: false, + timeoutMs: DEFAULT_TIMEOUT_MS, + settleMs: DEFAULT_SETTLE_MS, + maxTiles: DEFAULT_MAX_TILES, + waitUntil: 'domcontentloaded', + json: false, + requireAllowedHost: true, + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--manifest-file') { + options.manifestFile = argv[++i]; + } else if (arg === '--manifest-json') { + options.manifestJson = argv[++i]; + } else if (arg === '--output-dir') { + options.outputDir = argv[++i]; + } else if (arg === '--dry-run') { + options.dryRun = true; + } else if (arg === '--timeout') { + options.timeoutMs = parseInt(argv[++i], 10) * 1000; + } else if (arg === '--settle-ms') { + options.settleMs = parseInt(argv[++i], 10); + } else if (arg === '--max-tiles') { + options.maxTiles = parseInt(argv[++i], 10); + } else if (arg === '--wait-until') { + options.waitUntil = argv[++i]; + } else if (arg === '--json') { + options.json = true; + } else if (arg === '--allow-any-host') { + options.requireAllowedHost = false; + } else if (arg === '--help' || arg === '-h') { + printHelp(); + process.exit(0); + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + + if (!options.manifestFile && !options.manifestJson) { + throw new Error('Pass --manifest-file or --manifest-json.'); + } + return options; +} + +function printHelp() { + console.log(`PixelRAG visual evidence capture worker + +Options: + --manifest-file PATH Read manifest JSON from file + --manifest-json JSON Read manifest JSON from CLI arg + --output-dir DIR Artifact root, default ${DEFAULT_OUTPUT_DIR} + --dry-run Validate and print planned outputs without browser/network/files + --timeout SEC Navigation timeout, default 30 + --settle-ms MS Fixed post-DOM settle wait, default 500 + --max-tiles N Tile cap, default 80 + --wait-until EVENT Playwright waitUntil, default domcontentloaded + --allow-any-host Disable platform host allowlist for local research only + --json Print compact JSON only +`); +} + +function requirePlaywright() { + try { + return require('playwright'); + } catch (error) { + const candidates = [ + process.env.PLAYWRIGHT_NODE_MODULE_DIR, + process.env.NODE_PATH, + path.join(os.homedir(), '.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules'), + ].filter(Boolean); + + for (const candidate of candidates) { + const packageJson = path.join(candidate, 'playwright', 'package.json'); + if (fs.existsSync(packageJson)) { + return Module.createRequire(packageJson)('playwright'); + } + } + + throw new Error( + 'Cannot load playwright. Set NODE_PATH or PLAYWRIGHT_NODE_MODULE_DIR to a node_modules directory containing playwright.' + ); + } +} + +function findChromeExecutable() { + const candidates = [ + process.env.PIXELRAG_CHROME_PATH, + process.env.RESPONSIVE_CHROME_PATH, + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary', + '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge', + ].filter(Boolean); + return candidates.find((candidate) => fs.existsSync(candidate)) || ''; +} + +function safeName(input) { + return String(input || '').replace(/[^a-z0-9]+/gi, '_').replace(/^_+|_+$/g, '').toLowerCase() || 'unknown'; +} + +function loadManifest(options) { + const raw = options.manifestFile + ? fs.readFileSync(options.manifestFile, 'utf-8') + : options.manifestJson; + return JSON.parse(raw); +} + +function validateManifest(manifest, options) { + const errors = []; + if (!manifest || typeof manifest !== 'object') { + errors.push('manifest must be an object'); + return errors; + } + if (manifest.policy !== MANIFEST_POLICY) { + errors.push(`manifest policy must be ${MANIFEST_POLICY}`); + } + const target = manifest.capture_target || {}; + const url = String(target.url || ''); + let parsedUrl = null; + try { + parsedUrl = new URL(url); + } catch (error) { + errors.push('capture_target.url must be an absolute URL'); + } + if (parsedUrl) { + if (!['http:', 'https:'].includes(parsedUrl.protocol)) { + errors.push('capture_target.url must use http(s)'); + } + if (parsedUrl.username || parsedUrl.password) { + errors.push('URL credentials are not allowed'); + } + const platform = String(target.platform || ''); + const allowed = ALLOWED_PLATFORM_HOSTS[platform]; + if (options.requireAllowedHost && allowed && allowed.size > 0 && !allowed.has(parsedUrl.hostname)) { + errors.push(`host ${parsedUrl.hostname} is not allowed for platform ${platform}`); + } + } + if (!target.platform) errors.push('capture_target.platform is required'); + if (!target.crawler) errors.push('capture_target.crawler is required'); + if (!target.trigger_reason) errors.push('capture_target.trigger_reason is required'); + return errors; +} + +function positiveInt(value, fallback) { + const parsed = parseInt(value, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function buildTilePlan({ width, height, tileWidth, tileHeight, maxTiles }) { + const tiles = []; + const tilesX = Math.max(1, Math.ceil(width / tileWidth)); + const tilesY = Math.max(1, Math.ceil(height / tileHeight)); + for (let row = 0; row < tilesY; row += 1) { + for (let col = 0; col < tilesX; col += 1) { + if (tiles.length >= maxTiles) break; + const x = col * tileWidth; + const y = row * tileHeight; + tiles.push({ + index: tiles.length, + row, + col, + x, + y, + width: Math.min(tileWidth, Math.max(1, width - x)), + height: Math.min(tileHeight, Math.max(1, height - y)), + }); + } + } + return { + tiles_x: tilesX, + tiles_y: tilesY, + planned_tile_count: tilesX * tilesY, + emitted_tile_count: tiles.length, + capped: tiles.length < tilesX * tilesY, + tiles, + }; +} + +function outputPaths(manifest, options) { + const target = manifest.capture_target || {}; + const manifestId = manifest.manifest_id || safeName(`${target.platform}-${target.crawler}-${target.url}`).slice(0, 20); + const root = path.resolve(options.outputDir, safeName(target.platform || 'unknown'), manifestId); + return { + root, + receipt: path.join(root, 'capture_receipt.json'), + screenshot: path.join(root, 'fullpage.png'), + tilesDir: path.join(root, 'tiles'), + }; +} + +function buildBaseArtifact(manifest, options, errors = []) { + const viewport = manifest.viewport || {}; + const tileSize = manifest.tile_size || {}; + const pageSize = manifest.page_size || {}; + const width = positiveInt(pageSize.width, positiveInt(viewport.width, 1440)); + const height = positiveInt(pageSize.height, positiveInt(viewport.height, 950)); + const tileWidth = positiveInt(tileSize.width, 512); + const tileHeight = positiveInt(tileSize.height, 512); + const paths = outputPaths(manifest, options); + const tilePlan = buildTilePlan({ + width, + height, + tileWidth, + tileHeight, + maxTiles: positiveInt(options.maxTiles, DEFAULT_MAX_TILES), + }); + return { + success: errors.length === 0, + policy: CAPTURE_POLICY, + status: errors.length ? 'rejected' : (options.dryRun ? 'dry_run_ready' : 'capture_pending'), + generated_at: new Date().toISOString(), + errors, + manifest_id: manifest.manifest_id || '', + manifest_policy: manifest.policy || '', + capture_target: manifest.capture_target || {}, + controlled_apply: { + db_write: false, + secret_read: false, + github_dependency: false, + production_price_write: false, + artifact_write: !options.dryRun && errors.length === 0, + }, + planned_output: { + root: paths.root, + receipt: paths.receipt, + screenshot: paths.screenshot, + tiles_dir: paths.tilesDir, + }, + viewport: { + width: positiveInt(viewport.width, 1440), + height: positiveInt(viewport.height, 950), + name: viewport.name || 'desktop-1440', + }, + page_size: { width, height }, + tile_size: { width: tileWidth, height: tileHeight }, + tile_plan: tilePlan, + files: [], + }; +} + +async function capture(manifest, options, artifact) { + const { chromium } = requirePlaywright(); + const paths = outputPaths(manifest, options); + const viewport = artifact.viewport; + fs.mkdirSync(paths.tilesDir, { recursive: true }); + + const launchOptions = { headless: true }; + const chromePath = findChromeExecutable(); + if (chromePath) { + launchOptions.executablePath = chromePath; + } + + const browser = await chromium.launch(launchOptions); + const context = await browser.newContext({ ignoreHTTPSErrors: true, viewport }); + const page = await context.newPage(); + try { + const response = await page.goto(manifest.capture_target.url, { + waitUntil: options.waitUntil, + timeout: options.timeoutMs, + }); + await page.waitForSelector('body', { timeout: Math.min(options.timeoutMs, 5000) }).catch(() => {}); + await page.evaluate(() => document.fonts && document.fonts.ready).catch(() => {}); + if (options.settleMs > 0) { + await page.waitForTimeout(options.settleMs); + } + const metrics = await page.evaluate(() => { + const root = document.documentElement; + const body = document.body; + return { + final_url: location.href, + title: document.title, + scroll_width: Math.max(root.scrollWidth, body.scrollWidth, window.innerWidth), + scroll_height: Math.max(root.scrollHeight, body.scrollHeight, window.innerHeight), + }; + }); + + await page.screenshot({ path: paths.screenshot, fullPage: true }); + const files = [{ kind: 'fullpage_screenshot', path: paths.screenshot }]; + const tilePlan = buildTilePlan({ + width: metrics.scroll_width, + height: metrics.scroll_height, + tileWidth: artifact.tile_size.width, + tileHeight: artifact.tile_size.height, + maxTiles: positiveInt(options.maxTiles, DEFAULT_MAX_TILES), + }); + for (const tile of tilePlan.tiles) { + const filePath = path.join(paths.tilesDir, `tile_${String(tile.index).padStart(3, '0')}.png`); + await page.screenshot({ + path: filePath, + clip: { x: tile.x, y: tile.y, width: tile.width, height: tile.height }, + }); + files.push({ kind: 'tile', path: filePath, tile }); + } + + const captured = { + ...artifact, + success: true, + status: 'captured', + http_status: response ? response.status() : 0, + page_metrics: metrics, + page_size: { width: metrics.scroll_width, height: metrics.scroll_height }, + tile_plan: tilePlan, + files, + }; + fs.writeFileSync(paths.receipt, `${JSON.stringify(captured, null, 2)}\n`, 'utf-8'); + captured.files.push({ kind: 'receipt', path: paths.receipt }); + return captured; + } finally { + await page.close().catch(() => {}); + await context.close().catch(() => {}); + await browser.close().catch(() => {}); + } +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const manifest = loadManifest(options); + const errors = validateManifest(manifest, options); + const artifact = buildBaseArtifact(manifest, options, errors); + if (errors.length || options.dryRun) { + console.log(JSON.stringify(artifact, null, 2)); + process.exitCode = errors.length ? 1 : 0; + return; + } + const captured = await capture(manifest, options, artifact); + console.log(JSON.stringify(captured, null, 2)); +} + +main().catch((error) => { + console.error(error.message || error); + process.exit(1); +}); diff --git a/tests/test_pixelrag_crawler_integration_service.py b/tests/test_pixelrag_crawler_integration_service.py index 4d7f5ad..38bd30b 100644 --- a/tests/test_pixelrag_crawler_integration_service.py +++ b/tests/test_pixelrag_crawler_integration_service.py @@ -1,7 +1,10 @@ import json +import shutil import subprocess import sys +import pytest + def test_pixelrag_assessment_starts_visual_evidence_fallback_not_price_write(): from services.pixelrag_crawler_integration_service import ( @@ -226,3 +229,76 @@ def test_pixelrag_ai_automation_route_returns_assessment_and_manifest(): assert manifest["policy"] == "read_only_pixelrag_visual_evidence_manifest_v1" assert manifest["result"] == "PIXELRAG_VISUAL_EVIDENCE_MANIFEST_READY" assert manifest["capture_target"]["platform"] == "pchome" + + +def test_pixelrag_capture_worker_dry_run_plans_artifact_outputs(tmp_path): + if not shutil.which("node"): + pytest.skip("node is required for the dry-run capture worker test") + from services.pixelrag_crawler_integration_service import ( + build_pixelrag_visual_evidence_manifest, + ) + + manifest = build_pixelrag_visual_evidence_manifest( + url="https://m.momoshop.com.tw/search.momo?searchKeyword=test", + platform="momo", + crawler="MomoCrawler.search_products", + trigger_reason="parser_empty", + page_size={"width": 1024, "height": 1024}, + tile_size={"width": 512, "height": 512}, + ) + completed = subprocess.run( + [ + "node", + "scripts/ops/capture_pixelrag_visual_evidence.js", + "--manifest-json", + json.dumps(manifest), + "--output-dir", + str(tmp_path), + "--dry-run", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["policy"] == "read_only_pixelrag_visual_capture_artifact_v1" + assert payload["status"] == "dry_run_ready" + assert payload["controlled_apply"]["artifact_write"] is False + assert payload["tile_plan"]["planned_tile_count"] == 4 + assert payload["planned_output"]["screenshot"].endswith("fullpage.png") + + +def test_pixelrag_capture_worker_rejects_wrong_platform_host(tmp_path): + if not shutil.which("node"): + pytest.skip("node is required for the dry-run capture worker test") + from services.pixelrag_crawler_integration_service import ( + build_pixelrag_visual_evidence_manifest, + ) + + manifest = build_pixelrag_visual_evidence_manifest( + url="https://example.test/product", + platform="momo", + crawler="MomoCrawler.search_products", + trigger_reason="parser_empty", + ) + completed = subprocess.run( + [ + "node", + "scripts/ops/capture_pixelrag_visual_evidence.js", + "--manifest-json", + json.dumps(manifest), + "--output-dir", + str(tmp_path), + "--dry-run", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 1 + payload = json.loads(completed.stdout) + assert payload["status"] == "rejected" + assert any("not allowed for platform momo" in error for error in payload["errors"])