feat(crawler): capture pixelrag visual evidence artifacts

docs/guides/browse_sh_crawler_playbook.md

@@ -87,6 +87,28 @@ python scripts/ops/report_pixelrag_crawler_integration.py \
 /api/ai-automation/pixelrag-crawler-integration?platform=pchome&manifest_url=https://24h.pchome.com.tw/prod/TEST-000000001&crawler=PChomeCrawler.search_products&trigger_reason=parser_empty
 ```
 
+視覺證據 capture worker：
+
+```bash
+python3 scripts/ops/report_pixelrag_crawler_integration.py \
+  --platform momo \
+  --manifest-url "https://m.momoshop.com.tw/search.momo?searchKeyword=test" \
+  --crawler MomoCrawler.search_products \
+  --trigger-reason parser_empty \
+  --page-size page=1440x1900 \
+  --tile-size tile=512x512 \
+  > /tmp/pixelrag_manifest.json
+
+node scripts/ops/capture_pixelrag_visual_evidence.js \
+  --manifest-file /tmp/pixelrag_manifest.json \
+  --output-dir data/ai_automation/pixelrag_visual_evidence \
+  --dry-run
+
+node scripts/ops/capture_pixelrag_visual_evidence.js \
+  --manifest-file /tmp/pixelrag_manifest.json \
+  --output-dir data/ai_automation/pixelrag_visual_evidence
+```
+
 安全邊界：
 
 - read-only；不登入、不下單、不加入購物車、不寫第三方狀態。

scripts/ops/capture_pixelrag_visual_evidence.js

@@ -0,0 +1,364 @@
+#!/usr/bin/env node
+/*
+ * Read-only PixelRAG-style visual evidence capture worker.
+ *
+ * The API/service builds manifests; this worker consumes one manifest and writes
+ * screenshot/tile artifacts only when explicitly invoked.
+ */
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const Module = require('module');
+
+const CAPTURE_POLICY = 'read_only_pixelrag_visual_capture_artifact_v1';
+const MANIFEST_POLICY = 'read_only_pixelrag_visual_evidence_manifest_v1';
+const DEFAULT_OUTPUT_DIR = 'data/ai_automation/pixelrag_visual_evidence';
+const DEFAULT_TIMEOUT_MS = 30000;
+const DEFAULT_SETTLE_MS = 500;
+const DEFAULT_MAX_TILES = 80;
+
+const ALLOWED_PLATFORM_HOSTS = {
+  momo: new Set(['m.momoshop.com.tw', 'www.momoshop.com.tw']),
+  pchome: new Set(['24h.pchome.com.tw', 'ecshweb.pchome.com.tw', 'ecapi-cdn.pchome.com.tw']),
+  market_intel: new Set(),
+  external_market: new Set(),
+};
+
+function parseArgs(argv) {
+  const options = {
+    manifestFile: '',
+    manifestJson: '',
+    outputDir: DEFAULT_OUTPUT_DIR,
+    dryRun: false,
+    timeoutMs: DEFAULT_TIMEOUT_MS,
+    settleMs: DEFAULT_SETTLE_MS,
+    maxTiles: DEFAULT_MAX_TILES,
+    waitUntil: 'domcontentloaded',
+    json: false,
+    requireAllowedHost: true,
+  };
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === '--manifest-file') {
+      options.manifestFile = argv[++i];
+    } else if (arg === '--manifest-json') {
+      options.manifestJson = argv[++i];
+    } else if (arg === '--output-dir') {
+      options.outputDir = argv[++i];
+    } else if (arg === '--dry-run') {
+      options.dryRun = true;
+    } else if (arg === '--timeout') {
+      options.timeoutMs = parseInt(argv[++i], 10) * 1000;
+    } else if (arg === '--settle-ms') {
+      options.settleMs = parseInt(argv[++i], 10);
+    } else if (arg === '--max-tiles') {
+      options.maxTiles = parseInt(argv[++i], 10);
+    } else if (arg === '--wait-until') {
+      options.waitUntil = argv[++i];
+    } else if (arg === '--json') {
+      options.json = true;
+    } else if (arg === '--allow-any-host') {
+      options.requireAllowedHost = false;
+    } else if (arg === '--help' || arg === '-h') {
+      printHelp();
+      process.exit(0);
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+
+  if (!options.manifestFile && !options.manifestJson) {
+    throw new Error('Pass --manifest-file or --manifest-json.');
+  }
+  return options;
+}
+
+function printHelp() {
+  console.log(`PixelRAG visual evidence capture worker
+
+Options:
+  --manifest-file PATH       Read manifest JSON from file
+  --manifest-json JSON       Read manifest JSON from CLI arg
+  --output-dir DIR           Artifact root, default ${DEFAULT_OUTPUT_DIR}
+  --dry-run                  Validate and print planned outputs without browser/network/files
+  --timeout SEC              Navigation timeout, default 30
+  --settle-ms MS             Fixed post-DOM settle wait, default 500
+  --max-tiles N              Tile cap, default 80
+  --wait-until EVENT         Playwright waitUntil, default domcontentloaded
+  --allow-any-host           Disable platform host allowlist for local research only
+  --json                     Print compact JSON only
+`);
+}
+
+function requirePlaywright() {
+  try {
+    return require('playwright');
+  } catch (error) {
+    const candidates = [
+      process.env.PLAYWRIGHT_NODE_MODULE_DIR,
+      process.env.NODE_PATH,
+      path.join(os.homedir(), '.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules'),
+    ].filter(Boolean);
+
+    for (const candidate of candidates) {
+      const packageJson = path.join(candidate, 'playwright', 'package.json');
+      if (fs.existsSync(packageJson)) {
+        return Module.createRequire(packageJson)('playwright');
+      }
+    }
+
+    throw new Error(
+      'Cannot load playwright. Set NODE_PATH or PLAYWRIGHT_NODE_MODULE_DIR to a node_modules directory containing playwright.'
+    );
+  }
+}
+
+function findChromeExecutable() {
+  const candidates = [
+    process.env.PIXELRAG_CHROME_PATH,
+    process.env.RESPONSIVE_CHROME_PATH,
+    '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+    '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
+    '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
+  ].filter(Boolean);
+  return candidates.find((candidate) => fs.existsSync(candidate)) || '';
+}
+
+function safeName(input) {
+  return String(input || '').replace(/[^a-z0-9]+/gi, '_').replace(/^_+|_+$/g, '').toLowerCase() || 'unknown';
+}
+
+function loadManifest(options) {
+  const raw = options.manifestFile
+    ? fs.readFileSync(options.manifestFile, 'utf-8')
+    : options.manifestJson;
+  return JSON.parse(raw);
+}
+
+function validateManifest(manifest, options) {
+  const errors = [];
+  if (!manifest || typeof manifest !== 'object') {
+    errors.push('manifest must be an object');
+    return errors;
+  }
+  if (manifest.policy !== MANIFEST_POLICY) {
+    errors.push(`manifest policy must be ${MANIFEST_POLICY}`);
+  }
+  const target = manifest.capture_target || {};
+  const url = String(target.url || '');
+  let parsedUrl = null;
+  try {
+    parsedUrl = new URL(url);
+  } catch (error) {
+    errors.push('capture_target.url must be an absolute URL');
+  }
+  if (parsedUrl) {
+    if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
+      errors.push('capture_target.url must use http(s)');
+    }
+    if (parsedUrl.username || parsedUrl.password) {
+      errors.push('URL credentials are not allowed');
+    }
+    const platform = String(target.platform || '');
+    const allowed = ALLOWED_PLATFORM_HOSTS[platform];
+    if (options.requireAllowedHost && allowed && allowed.size > 0 && !allowed.has(parsedUrl.hostname)) {
+      errors.push(`host ${parsedUrl.hostname} is not allowed for platform ${platform}`);
+    }
+  }
+  if (!target.platform) errors.push('capture_target.platform is required');
+  if (!target.crawler) errors.push('capture_target.crawler is required');
+  if (!target.trigger_reason) errors.push('capture_target.trigger_reason is required');
+  return errors;
+}
+
+function positiveInt(value, fallback) {
+  const parsed = parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function buildTilePlan({ width, height, tileWidth, tileHeight, maxTiles }) {
+  const tiles = [];
+  const tilesX = Math.max(1, Math.ceil(width / tileWidth));
+  const tilesY = Math.max(1, Math.ceil(height / tileHeight));
+  for (let row = 0; row < tilesY; row += 1) {
+    for (let col = 0; col < tilesX; col += 1) {
+      if (tiles.length >= maxTiles) break;
+      const x = col * tileWidth;
+      const y = row * tileHeight;
+      tiles.push({
+        index: tiles.length,
+        row,
+        col,
+        x,
+        y,
+        width: Math.min(tileWidth, Math.max(1, width - x)),
+        height: Math.min(tileHeight, Math.max(1, height - y)),
+      });
+    }
+  }
+  return {
+    tiles_x: tilesX,
+    tiles_y: tilesY,
+    planned_tile_count: tilesX * tilesY,
+    emitted_tile_count: tiles.length,
+    capped: tiles.length < tilesX * tilesY,
+    tiles,
+  };
+}
+
+function outputPaths(manifest, options) {
+  const target = manifest.capture_target || {};
+  const manifestId = manifest.manifest_id || safeName(`${target.platform}-${target.crawler}-${target.url}`).slice(0, 20);
+  const root = path.resolve(options.outputDir, safeName(target.platform || 'unknown'), manifestId);
+  return {
+    root,
+    receipt: path.join(root, 'capture_receipt.json'),
+    screenshot: path.join(root, 'fullpage.png'),
+    tilesDir: path.join(root, 'tiles'),
+  };
+}
+
+function buildBaseArtifact(manifest, options, errors = []) {
+  const viewport = manifest.viewport || {};
+  const tileSize = manifest.tile_size || {};
+  const pageSize = manifest.page_size || {};
+  const width = positiveInt(pageSize.width, positiveInt(viewport.width, 1440));
+  const height = positiveInt(pageSize.height, positiveInt(viewport.height, 950));
+  const tileWidth = positiveInt(tileSize.width, 512);
+  const tileHeight = positiveInt(tileSize.height, 512);
+  const paths = outputPaths(manifest, options);
+  const tilePlan = buildTilePlan({
+    width,
+    height,
+    tileWidth,
+    tileHeight,
+    maxTiles: positiveInt(options.maxTiles, DEFAULT_MAX_TILES),
+  });
+  return {
+    success: errors.length === 0,
+    policy: CAPTURE_POLICY,
+    status: errors.length ? 'rejected' : (options.dryRun ? 'dry_run_ready' : 'capture_pending'),
+    generated_at: new Date().toISOString(),
+    errors,
+    manifest_id: manifest.manifest_id || '',
+    manifest_policy: manifest.policy || '',
+    capture_target: manifest.capture_target || {},
+    controlled_apply: {
+      db_write: false,
+      secret_read: false,
+      github_dependency: false,
+      production_price_write: false,
+      artifact_write: !options.dryRun && errors.length === 0,
+    },
+    planned_output: {
+      root: paths.root,
+      receipt: paths.receipt,
+      screenshot: paths.screenshot,
+      tiles_dir: paths.tilesDir,
+    },
+    viewport: {
+      width: positiveInt(viewport.width, 1440),
+      height: positiveInt(viewport.height, 950),
+      name: viewport.name || 'desktop-1440',
+    },
+    page_size: { width, height },
+    tile_size: { width: tileWidth, height: tileHeight },
+    tile_plan: tilePlan,
+    files: [],
+  };
+}
+
+async function capture(manifest, options, artifact) {
+  const { chromium } = requirePlaywright();
+  const paths = outputPaths(manifest, options);
+  const viewport = artifact.viewport;
+  fs.mkdirSync(paths.tilesDir, { recursive: true });
+
+  const launchOptions = { headless: true };
+  const chromePath = findChromeExecutable();
+  if (chromePath) {
+    launchOptions.executablePath = chromePath;
+  }
+
+  const browser = await chromium.launch(launchOptions);
+  const context = await browser.newContext({ ignoreHTTPSErrors: true, viewport });
+  const page = await context.newPage();
+  try {
+    const response = await page.goto(manifest.capture_target.url, {
+      waitUntil: options.waitUntil,
+      timeout: options.timeoutMs,
+    });
+    await page.waitForSelector('body', { timeout: Math.min(options.timeoutMs, 5000) }).catch(() => {});
+    await page.evaluate(() => document.fonts && document.fonts.ready).catch(() => {});
+    if (options.settleMs > 0) {
+      await page.waitForTimeout(options.settleMs);
+    }
+    const metrics = await page.evaluate(() => {
+      const root = document.documentElement;
+      const body = document.body;
+      return {
+        final_url: location.href,
+        title: document.title,
+        scroll_width: Math.max(root.scrollWidth, body.scrollWidth, window.innerWidth),
+        scroll_height: Math.max(root.scrollHeight, body.scrollHeight, window.innerHeight),
+      };
+    });
+
+    await page.screenshot({ path: paths.screenshot, fullPage: true });
+    const files = [{ kind: 'fullpage_screenshot', path: paths.screenshot }];
+    const tilePlan = buildTilePlan({
+      width: metrics.scroll_width,
+      height: metrics.scroll_height,
+      tileWidth: artifact.tile_size.width,
+      tileHeight: artifact.tile_size.height,
+      maxTiles: positiveInt(options.maxTiles, DEFAULT_MAX_TILES),
+    });
+    for (const tile of tilePlan.tiles) {
+      const filePath = path.join(paths.tilesDir, `tile_${String(tile.index).padStart(3, '0')}.png`);
+      await page.screenshot({
+        path: filePath,
+        clip: { x: tile.x, y: tile.y, width: tile.width, height: tile.height },
+      });
+      files.push({ kind: 'tile', path: filePath, tile });
+    }
+
+    const captured = {
+      ...artifact,
+      success: true,
+      status: 'captured',
+      http_status: response ? response.status() : 0,
+      page_metrics: metrics,
+      page_size: { width: metrics.scroll_width, height: metrics.scroll_height },
+      tile_plan: tilePlan,
+      files,
+    };
+    fs.writeFileSync(paths.receipt, `${JSON.stringify(captured, null, 2)}\n`, 'utf-8');
+    captured.files.push({ kind: 'receipt', path: paths.receipt });
+    return captured;
+  } finally {
+    await page.close().catch(() => {});
+    await context.close().catch(() => {});
+    await browser.close().catch(() => {});
+  }
+}
+
+async function main() {
+  const options = parseArgs(process.argv.slice(2));
+  const manifest = loadManifest(options);
+  const errors = validateManifest(manifest, options);
+  const artifact = buildBaseArtifact(manifest, options, errors);
+  if (errors.length || options.dryRun) {
+    console.log(JSON.stringify(artifact, null, 2));
+    process.exitCode = errors.length ? 1 : 0;
+    return;
+  }
+  const captured = await capture(manifest, options, artifact);
+  console.log(JSON.stringify(captured, null, 2));
+}
+
+main().catch((error) => {
+  console.error(error.message || error);
+  process.exit(1);
+});

tests/test_pixelrag_crawler_integration_service.py

@@ -1,7 +1,10 @@
 import json
+import shutil
 import subprocess
 import sys
 
+import pytest
+
 
 def test_pixelrag_assessment_starts_visual_evidence_fallback_not_price_write():
     from services.pixelrag_crawler_integration_service import (
@@ -226,3 +229,76 @@ def test_pixelrag_ai_automation_route_returns_assessment_and_manifest():
     assert manifest["policy"] == "read_only_pixelrag_visual_evidence_manifest_v1"
     assert manifest["result"] == "PIXELRAG_VISUAL_EVIDENCE_MANIFEST_READY"
     assert manifest["capture_target"]["platform"] == "pchome"
+
+
+def test_pixelrag_capture_worker_dry_run_plans_artifact_outputs(tmp_path):
+    if not shutil.which("node"):
+        pytest.skip("node is required for the dry-run capture worker test")
+    from services.pixelrag_crawler_integration_service import (
+        build_pixelrag_visual_evidence_manifest,
+    )
+
+    manifest = build_pixelrag_visual_evidence_manifest(
+        url="https://m.momoshop.com.tw/search.momo?searchKeyword=test",
+        platform="momo",
+        crawler="MomoCrawler.search_products",
+        trigger_reason="parser_empty",
+        page_size={"width": 1024, "height": 1024},
+        tile_size={"width": 512, "height": 512},
+    )
+    completed = subprocess.run(
+        [
+            "node",
+            "scripts/ops/capture_pixelrag_visual_evidence.js",
+            "--manifest-json",
+            json.dumps(manifest),
+            "--output-dir",
+            str(tmp_path),
+            "--dry-run",
+        ],
+        capture_output=True,
+        check=False,
+        text=True,
+    )
+
+    assert completed.returncode == 0
+    payload = json.loads(completed.stdout)
+    assert payload["policy"] == "read_only_pixelrag_visual_capture_artifact_v1"
+    assert payload["status"] == "dry_run_ready"
+    assert payload["controlled_apply"]["artifact_write"] is False
+    assert payload["tile_plan"]["planned_tile_count"] == 4
+    assert payload["planned_output"]["screenshot"].endswith("fullpage.png")
+
+
+def test_pixelrag_capture_worker_rejects_wrong_platform_host(tmp_path):
+    if not shutil.which("node"):
+        pytest.skip("node is required for the dry-run capture worker test")
+    from services.pixelrag_crawler_integration_service import (
+        build_pixelrag_visual_evidence_manifest,
+    )
+
+    manifest = build_pixelrag_visual_evidence_manifest(
+        url="https://example.test/product",
+        platform="momo",
+        crawler="MomoCrawler.search_products",
+        trigger_reason="parser_empty",
+    )
+    completed = subprocess.run(
+        [
+            "node",
+            "scripts/ops/capture_pixelrag_visual_evidence.js",
+            "--manifest-json",
+            json.dumps(manifest),
+            "--output-dir",
+            str(tmp_path),
+            "--dry-run",
+        ],
+        capture_output=True,
+        check=False,
+        text=True,
+    )
+
+    assert completed.returncode == 1
+    payload = json.loads(completed.stdout)
+    assert payload["status"] == "rejected"
+    assert any("not allowed for platform momo" in error for error in payload["errors"])