ewoooc/scripts/ops/capture_pixelrag_visual_evidence.js

#!/usr/bin/env node
/*
 * Read-only PixelRAG-style visual evidence capture worker.
 *
 * The API/service builds manifests; this worker consumes one manifest and writes
 * screenshot/tile artifacts only when explicitly invoked.
 */

const fs = require('fs');
const os = require('os');
const path = require('path');
const Module = require('module');

const CAPTURE_POLICY = 'read_only_pixelrag_visual_capture_artifact_v1';
const MANIFEST_POLICY = 'read_only_pixelrag_visual_evidence_manifest_v1';
const DEFAULT_OUTPUT_DIR = 'runtime_artifacts/pixelrag_visual_evidence';
const DEFAULT_TIMEOUT_MS = 30000;
const DEFAULT_SETTLE_MS = 500;
const DEFAULT_MAX_TILES = 80;
const FORBIDDEN_CONTEXT_HEADER_TOKENS = ['cookie', 'authorization', 'token', 'secret', 'key'];

const ALLOWED_PLATFORM_HOSTS = {
  momo: new Set(['m.momoshop.com.tw', 'www.momoshop.com.tw']),
  pchome: new Set(['24h.pchome.com.tw', 'ecshweb.pchome.com.tw', 'ecapi-cdn.pchome.com.tw']),
  shopee_tw: new Set(['shopee.tw', 'mall.shopee.tw']),
  coupang_tw: new Set(['www.tw.coupang.com', 'tw.coupang.com']),
  yahoo_shopping_tw: new Set(['tw.buy.yahoo.com']),
  etmall_tw: new Set(['www.etmall.com.tw', 'etmall.com.tw']),
  friday_tw: new Set(['shopping.friday.tw', 'ec-m.shopping.friday.tw', 'ec-w.shopping.friday.tw']),
  rakuten_tw: new Set(['www.rakuten.com.tw', 'rakuten.com.tw']),
  market_intel: new Set(),
  external_market: new Set(),
};

function parseArgs(argv) {
  const options = {
    manifestFile: '',
    manifestJson: '',
    outputDir: DEFAULT_OUTPUT_DIR,
    dryRun: false,
    timeoutMs: DEFAULT_TIMEOUT_MS,
    settleMs: DEFAULT_SETTLE_MS,
    maxTiles: DEFAULT_MAX_TILES,
    waitUntil: 'domcontentloaded',
    json: false,
    requireAllowedHost: true,
  };

  for (let i = 0; i < argv.length; i += 1) {
    const arg = argv[i];
    if (arg === '--manifest-file') {
      options.manifestFile = argv[++i];
    } else if (arg === '--manifest-json') {
      options.manifestJson = argv[++i];
    } else if (arg === '--output-dir') {
      options.outputDir = argv[++i];
    } else if (arg === '--dry-run') {
      options.dryRun = true;
    } else if (arg === '--timeout') {
      options.timeoutMs = parseInt(argv[++i], 10) * 1000;
    } else if (arg === '--settle-ms') {
      options.settleMs = parseInt(argv[++i], 10);
    } else if (arg === '--max-tiles') {
      options.maxTiles = parseInt(argv[++i], 10);
    } else if (arg === '--wait-until') {
      options.waitUntil = argv[++i];
    } else if (arg === '--json') {
      options.json = true;
    } else if (arg === '--allow-any-host') {
      options.requireAllowedHost = false;
    } else if (arg === '--help' || arg === '-h') {
      printHelp();
      process.exit(0);
    } else {
      throw new Error(`Unknown argument: ${arg}`);
    }
  }

  if (!options.manifestFile && !options.manifestJson) {
    throw new Error('Pass --manifest-file or --manifest-json.');
  }
  return options;
}

function printHelp() {
  console.log(`PixelRAG visual evidence capture worker

Options:
  --manifest-file PATH       Read manifest JSON from file
  --manifest-json JSON       Read manifest JSON from CLI arg
  --output-dir DIR           Artifact root, default ${DEFAULT_OUTPUT_DIR}
  --dry-run                  Validate and print planned outputs without browser/network/files
  --timeout SEC              Navigation timeout, default 30
  --settle-ms MS             Fixed post-DOM settle wait, default 500
  --max-tiles N              Tile cap, default 80
  --wait-until EVENT         Playwright waitUntil, default domcontentloaded
  --allow-any-host           Disable platform host allowlist for local research only
  --json                     Print compact JSON only
`);
}

function requirePlaywright() {
  try {
    return require('playwright');
  } catch (error) {
    const candidates = [
      process.env.PLAYWRIGHT_NODE_MODULE_DIR,
      process.env.NODE_PATH,
      path.join(os.homedir(), '.cache/codex-runtimes/codex-primary-runtime/dependencies/node/node_modules'),
    ].filter(Boolean);

    for (const candidate of candidates) {
      const packageJson = path.join(candidate, 'playwright', 'package.json');
      if (fs.existsSync(packageJson)) {
        return Module.createRequire(packageJson)('playwright');
      }
    }

    throw new Error(
      'Cannot load playwright. Set NODE_PATH or PLAYWRIGHT_NODE_MODULE_DIR to a node_modules directory containing playwright.'
    );
  }
}

function findChromeExecutable() {
  const candidates = [
    process.env.PIXELRAG_CHROME_PATH,
    process.env.RESPONSIVE_CHROME_PATH,
    '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
    '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
    '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
  ].filter(Boolean);
  return candidates.find((candidate) => fs.existsSync(candidate)) || '';
}

function safeName(input) {
  return String(input || '').replace(/[^a-z0-9]+/gi, '_').replace(/^_+|_+$/g, '').toLowerCase() || 'unknown';
}

function loadManifest(options) {
  const raw = options.manifestFile
    ? fs.readFileSync(options.manifestFile, 'utf-8')
    : options.manifestJson;
  return JSON.parse(raw);
}

function validateManifest(manifest, options) {
  const errors = [];
  if (!manifest || typeof manifest !== 'object') {
    errors.push('manifest must be an object');
    return errors;
  }
  if (manifest.policy !== MANIFEST_POLICY) {
    errors.push(`manifest policy must be ${MANIFEST_POLICY}`);
  }
  const target = manifest.capture_target || {};
  const url = String(target.url || '');
  let parsedUrl = null;
  try {
    parsedUrl = new URL(url);
  } catch (error) {
    errors.push('capture_target.url must be an absolute URL');
  }
  if (parsedUrl) {
    if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
      errors.push('capture_target.url must use http(s)');
    }
    if (parsedUrl.username || parsedUrl.password) {
      errors.push('URL credentials are not allowed');
    }
    const platform = String(target.platform || '');
    const allowed = ALLOWED_PLATFORM_HOSTS[platform];
    if (options.requireAllowedHost && allowed && allowed.size > 0 && !allowed.has(parsedUrl.hostname)) {
      errors.push(`host ${parsedUrl.hostname} is not allowed for platform ${platform}`);
    }
  }
  if (!target.platform) errors.push('capture_target.platform is required');
  if (!target.crawler) errors.push('capture_target.crawler is required');
  if (!target.trigger_reason) errors.push('capture_target.trigger_reason is required');
  return errors;
}

function positiveInt(value, fallback) {
  const parsed = parseInt(value, 10);
  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}

function safePublicBrowserContext(manifest) {
  const input = manifest.public_browser_context || {};
  const headers = {};
  const rawHeaders = input.extra_http_headers || {};
  for (const [key, value] of Object.entries(rawHeaders)) {
    const cleanKey = String(key || '').trim();
    const lowerKey = cleanKey.toLowerCase();
    if (!cleanKey || FORBIDDEN_CONTEXT_HEADER_TOKENS.some((token) => lowerKey.includes(token))) {
      continue;
    }
    headers[cleanKey] = String(value || '');
  }
  return {
    context_policy: 'public_empty_browser_context_no_login',
    locale: String(input.locale || 'zh-TW'),
    timezone_id: String(input.timezone_id || input.timezoneId || 'Asia/Taipei'),
    extra_http_headers: headers,
    credentialed_session_allowed: false,
    storage_state_allowed: false,
    raw_cookie_or_session_read_allowed: false,
    login_allowed: false,
  };
}

function buildTilePlan({ width, height, tileWidth, tileHeight, maxTiles }) {
  const tiles = [];
  const tilesX = Math.max(1, Math.ceil(width / tileWidth));
  const tilesY = Math.max(1, Math.ceil(height / tileHeight));
  for (let row = 0; row < tilesY; row += 1) {
    for (let col = 0; col < tilesX; col += 1) {
      if (tiles.length >= maxTiles) break;
      const x = col * tileWidth;
      const y = row * tileHeight;
      tiles.push({
        index: tiles.length,
        row,
        col,
        x,
        y,
        width: Math.min(tileWidth, Math.max(1, width - x)),
        height: Math.min(tileHeight, Math.max(1, height - y)),
      });
    }
  }
  return {
    tiles_x: tilesX,
    tiles_y: tilesY,
    planned_tile_count: tilesX * tilesY,
    emitted_tile_count: tiles.length,
    capped: tiles.length < tilesX * tilesY,
    tiles,
  };
}

function outputPaths(manifest, options) {
  const target = manifest.capture_target || {};
  const manifestId = manifest.manifest_id || safeName(`${target.platform}-${target.crawler}-${target.url}`).slice(0, 20);
  const root = path.resolve(options.outputDir, safeName(target.platform || 'unknown'), manifestId);
  return {
    root,
    receipt: path.join(root, 'capture_receipt.json'),
    screenshot: path.join(root, 'fullpage.png'),
    tilesDir: path.join(root, 'tiles'),
  };
}

function buildBaseArtifact(manifest, options, errors = []) {
  const viewport = manifest.viewport || {};
  const tileSize = manifest.tile_size || {};
  const pageSize = manifest.page_size || {};
  const width = positiveInt(pageSize.width, positiveInt(viewport.width, 1440));
  const height = positiveInt(pageSize.height, positiveInt(viewport.height, 950));
  const tileWidth = positiveInt(tileSize.width, 512);
  const tileHeight = positiveInt(tileSize.height, 512);
  const paths = outputPaths(manifest, options);
  const tilePlan = buildTilePlan({
    width,
    height,
    tileWidth,
    tileHeight,
    maxTiles: positiveInt(options.maxTiles, DEFAULT_MAX_TILES),
  });
  return {
    success: errors.length === 0,
    policy: CAPTURE_POLICY,
    status: errors.length ? 'rejected' : (options.dryRun ? 'dry_run_ready' : 'capture_pending'),
    generated_at: new Date().toISOString(),
    errors,
    manifest_id: manifest.manifest_id || '',
    manifest_policy: manifest.policy || '',
    capture_target: manifest.capture_target || {},
    controlled_apply: {
      db_write: false,
      secret_read: false,
      github_dependency: false,
      production_price_write: false,
      artifact_write: !options.dryRun && errors.length === 0,
    },
    planned_output: {
      root: paths.root,
      receipt: paths.receipt,
      screenshot: paths.screenshot,
      tiles_dir: paths.tilesDir,
    },
    viewport: {
      width: positiveInt(viewport.width, 1440),
      height: positiveInt(viewport.height, 950),
      name: viewport.name || 'desktop-1440',
    },
    page_size: { width, height },
    tile_size: { width: tileWidth, height: tileHeight },
    tile_plan: tilePlan,
    files: [],
    public_browser_context: safePublicBrowserContext(manifest),
  };
}

async function capture(manifest, options, artifact) {
  const { chromium } = requirePlaywright();
  const paths = outputPaths(manifest, options);
  const viewport = artifact.viewport;
  fs.mkdirSync(paths.tilesDir, { recursive: true });

  const launchOptions = { headless: true };
  const chromePath = findChromeExecutable();
  if (chromePath) {
    launchOptions.executablePath = chromePath;
  }

  const publicContext = safePublicBrowserContext(manifest);
  const contextOptions = {
    ignoreHTTPSErrors: true,
    viewport,
    locale: publicContext.locale,
    timezoneId: publicContext.timezone_id,
  };
  if (Object.keys(publicContext.extra_http_headers).length) {
    contextOptions.extraHTTPHeaders = publicContext.extra_http_headers;
  }

  const browser = await chromium.launch(launchOptions);
  const context = await browser.newContext(contextOptions);
  const page = await context.newPage();
  try {
    const response = await page.goto(manifest.capture_target.url, {
      waitUntil: options.waitUntil,
      timeout: options.timeoutMs,
    });
    await page.waitForSelector('body', { timeout: Math.min(options.timeoutMs, 5000) }).catch(() => {});
    await page.evaluate(() => document.fonts && document.fonts.ready).catch(() => {});
    if (options.settleMs > 0) {
      await page.waitForTimeout(options.settleMs);
    }
    const metrics = await page.evaluate(() => {
      const root = document.documentElement;
      const body = document.body;
      return {
        final_url: location.href,
        title: document.title,
        scroll_width: Math.max(root.scrollWidth, body.scrollWidth, window.innerWidth),
        scroll_height: Math.max(root.scrollHeight, body.scrollHeight, window.innerHeight),
      };
    });

    await page.screenshot({ path: paths.screenshot, fullPage: true });
    const files = [{ kind: 'fullpage_screenshot', path: paths.screenshot }];
    const tilePlan = buildTilePlan({
      width: metrics.scroll_width,
      height: metrics.scroll_height,
      tileWidth: artifact.tile_size.width,
      tileHeight: artifact.tile_size.height,
      maxTiles: positiveInt(options.maxTiles, DEFAULT_MAX_TILES),
    });
    for (const tile of tilePlan.tiles) {
      const filePath = path.join(paths.tilesDir, `tile_${String(tile.index).padStart(3, '0')}.png`);
      await page.screenshot({
        path: filePath,
        clip: { x: tile.x, y: tile.y, width: tile.width, height: tile.height },
      });
      files.push({ kind: 'tile', path: filePath, tile });
    }

    const captured = {
      ...artifact,
      success: true,
      status: 'captured',
      http_status: response ? response.status() : 0,
      page_metrics: metrics,
      page_size: { width: metrics.scroll_width, height: metrics.scroll_height },
      tile_plan: tilePlan,
      files,
    };
    fs.writeFileSync(paths.receipt, `${JSON.stringify(captured, null, 2)}\n`, 'utf-8');
    captured.files.push({ kind: 'receipt', path: paths.receipt });
    return captured;
  } finally {
    await page.close().catch(() => {});
    await context.close().catch(() => {});
    await browser.close().catch(() => {});
  }
}

async function main() {
  const options = parseArgs(process.argv.slice(2));
  const manifest = loadManifest(options);
  const errors = validateManifest(manifest, options);
  const artifact = buildBaseArtifact(manifest, options, errors);
  if (errors.length || options.dryRun) {
    console.log(JSON.stringify(artifact, null, 2));
    process.exitCode = errors.length ? 1 : 0;
    return;
  }
  const captured = await capture(manifest, options, artifact);
  console.log(JSON.stringify(captured, null, 2));
}

main().catch((error) => {
  console.error(error.message || error);
  process.exit(1);
});