ewoooc/scripts/ops/capture_pixelrag_visual_evidence.py

#!/usr/bin/env python3
"""Read-only PixelRAG-style visual evidence capture worker.

This Python worker is the production-friendly counterpart of the Node worker:
it can dry-run without Playwright and only imports Playwright when actual capture
is requested.
"""

from __future__ import annotations

import argparse
import json
import math
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from urllib.parse import urlparse


CAPTURE_POLICY = "read_only_pixelrag_visual_capture_artifact_v1"
MANIFEST_POLICY = "read_only_pixelrag_visual_evidence_manifest_v1"
DEFAULT_OUTPUT_DIR = "data/ai_automation/pixelrag_visual_evidence"
DEFAULT_TIMEOUT_MS = 30000
DEFAULT_SETTLE_MS = 500
DEFAULT_MAX_TILES = 80
ALLOWED_PLATFORM_HOSTS = {
    "momo": {"m.momoshop.com.tw", "www.momoshop.com.tw"},
    "pchome": {"24h.pchome.com.tw", "ecshweb.pchome.com.tw", "ecapi-cdn.pchome.com.tw"},
    "market_intel": set(),
    "external_market": set(),
}


def _positive_int(value: Any, fallback: int) -> int:
    try:
        parsed = int(value)
    except (TypeError, ValueError):
        return fallback
    return parsed if parsed > 0 else fallback


def _safe_name(value: Any) -> str:
    text = "".join(ch if ch.isalnum() else "_" for ch in str(value or "").lower())
    text = "_".join(part for part in text.split("_") if part)
    return text or "unknown"


def _load_manifest(args: argparse.Namespace) -> dict[str, Any]:
    if args.manifest_file:
        return json.loads(Path(args.manifest_file).read_text(encoding="utf-8"))
    return json.loads(args.manifest_json)


def _validate_manifest(manifest: dict[str, Any], *, require_allowed_host: bool) -> list[str]:
    errors: list[str] = []
    if manifest.get("policy") != MANIFEST_POLICY:
        errors.append(f"manifest policy must be {MANIFEST_POLICY}")
    target = manifest.get("capture_target") or {}
    raw_url = str(target.get("url") or "")
    parsed = urlparse(raw_url)
    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
        errors.append("capture_target.url must be an absolute http(s) URL")
    if parsed.username or parsed.password:
        errors.append("URL credentials are not allowed")
    platform = str(target.get("platform") or "")
    allowed_hosts = ALLOWED_PLATFORM_HOSTS.get(platform)
    if (
        require_allowed_host
        and parsed.netloc
        and allowed_hosts
        and parsed.hostname not in allowed_hosts
    ):
        errors.append(f"host {parsed.hostname} is not allowed for platform {platform}")
    if not platform:
        errors.append("capture_target.platform is required")
    if not target.get("crawler"):
        errors.append("capture_target.crawler is required")
    if not target.get("trigger_reason"):
        errors.append("capture_target.trigger_reason is required")
    return errors


def _build_tile_plan(
    *,
    width: int,
    height: int,
    tile_width: int,
    tile_height: int,
    max_tiles: int,
) -> dict[str, Any]:
    tiles_x = max(1, math.ceil(width / tile_width))
    tiles_y = max(1, math.ceil(height / tile_height))
    tiles: list[dict[str, int]] = []
    for row in range(tiles_y):
        for col in range(tiles_x):
            if len(tiles) >= max_tiles:
                break
            x = col * tile_width
            y = row * tile_height
            tiles.append({
                "index": len(tiles),
                "row": row,
                "col": col,
                "x": x,
                "y": y,
                "width": min(tile_width, max(1, width - x)),
                "height": min(tile_height, max(1, height - y)),
            })
    planned = tiles_x * tiles_y
    return {
        "tiles_x": tiles_x,
        "tiles_y": tiles_y,
        "planned_tile_count": planned,
        "emitted_tile_count": len(tiles),
        "capped": len(tiles) < planned,
        "tiles": tiles,
    }


def _output_paths(manifest: dict[str, Any], output_dir: str) -> dict[str, Path]:
    target = manifest.get("capture_target") or {}
    manifest_id = manifest.get("manifest_id") or _safe_name(
        f"{target.get('platform')}-{target.get('crawler')}-{target.get('url')}"
    )[:20]
    root = Path(output_dir).resolve() / _safe_name(target.get("platform")) / manifest_id
    return {
        "root": root,
        "receipt": root / "capture_receipt.json",
        "screenshot": root / "fullpage.png",
        "tiles_dir": root / "tiles",
    }


def _base_artifact(
    manifest: dict[str, Any],
    args: argparse.Namespace,
    errors: list[str],
) -> dict[str, Any]:
    viewport = manifest.get("viewport") or {}
    page_size = manifest.get("page_size") or {}
    tile_size = manifest.get("tile_size") or {}
    width = _positive_int(page_size.get("width"), _positive_int(viewport.get("width"), 1440))
    height = _positive_int(page_size.get("height"), _positive_int(viewport.get("height"), 950))
    tile_width = _positive_int(tile_size.get("width"), 512)
    tile_height = _positive_int(tile_size.get("height"), 512)
    paths = _output_paths(manifest, args.output_dir)
    return {
        "success": not errors,
        "policy": CAPTURE_POLICY,
        "status": "rejected" if errors else ("dry_run_ready" if args.dry_run else "capture_pending"),
        "generated_at": datetime.now(timezone.utc).isoformat(),
        "errors": errors,
        "manifest_id": manifest.get("manifest_id") or "",
        "manifest_policy": manifest.get("policy") or "",
        "capture_target": manifest.get("capture_target") or {},
        "controlled_apply": {
            "db_write": False,
            "secret_read": False,
            "github_dependency": False,
            "production_price_write": False,
            "artifact_write": bool(not args.dry_run and not errors),
        },
        "planned_output": {
            "root": str(paths["root"]),
            "receipt": str(paths["receipt"]),
            "screenshot": str(paths["screenshot"]),
            "tiles_dir": str(paths["tiles_dir"]),
        },
        "viewport": {
            "name": viewport.get("name") or "desktop-1440",
            "width": _positive_int(viewport.get("width"), 1440),
            "height": _positive_int(viewport.get("height"), 950),
        },
        "page_size": {"width": width, "height": height},
        "tile_size": {"width": tile_width, "height": tile_height},
        "tile_plan": _build_tile_plan(
            width=width,
            height=height,
            tile_width=tile_width,
            tile_height=tile_height,
            max_tiles=_positive_int(args.max_tiles, DEFAULT_MAX_TILES),
        ),
        "files": [],
    }


def _capture(
    manifest: dict[str, Any],
    args: argparse.Namespace,
    artifact: dict[str, Any],
) -> dict[str, Any]:
    from playwright.sync_api import sync_playwright

    paths = _output_paths(manifest, args.output_dir)
    paths["tiles_dir"].mkdir(parents=True, exist_ok=True)
    target = manifest.get("capture_target") or {}
    viewport = artifact["viewport"]
    timeout_ms = _positive_int(args.timeout * 1000, DEFAULT_TIMEOUT_MS)

    with sync_playwright() as playwright:
        browser = playwright.chromium.launch(headless=True)
        context = browser.new_context(ignore_https_errors=True, viewport={
            "width": viewport["width"],
            "height": viewport["height"],
        })
        page = context.new_page()
        try:
            response = page.goto(target["url"], wait_until=args.wait_until, timeout=timeout_ms)
            try:
                page.wait_for_selector("body", timeout=min(timeout_ms, 5000))
            except Exception:
                pass
            if args.settle_ms > 0:
                page.wait_for_timeout(args.settle_ms)
            metrics = page.evaluate(
                """() => {
                    const root = document.documentElement;
                    const body = document.body;
                    return {
                        final_url: location.href,
                        title: document.title,
                        scroll_width: Math.max(root.scrollWidth, body.scrollWidth, window.innerWidth),
                        scroll_height: Math.max(root.scrollHeight, body.scrollHeight, window.innerHeight)
                    };
                }"""
            )
            page.screenshot(path=str(paths["screenshot"]), full_page=True)
            tile_plan = _build_tile_plan(
                width=_positive_int(metrics.get("scroll_width"), artifact["page_size"]["width"]),
                height=_positive_int(metrics.get("scroll_height"), artifact["page_size"]["height"]),
                tile_width=artifact["tile_size"]["width"],
                tile_height=artifact["tile_size"]["height"],
                max_tiles=_positive_int(args.max_tiles, DEFAULT_MAX_TILES),
            )
            files: list[dict[str, Any]] = [
                {"kind": "fullpage_screenshot", "path": str(paths["screenshot"])},
            ]
            for tile in tile_plan["tiles"]:
                file_path = paths["tiles_dir"] / f"tile_{tile['index']:03d}.png"
                page.screenshot(path=str(file_path), clip={
                    "x": tile["x"],
                    "y": tile["y"],
                    "width": tile["width"],
                    "height": tile["height"],
                })
                files.append({"kind": "tile", "path": str(file_path), "tile": tile})
            captured = {
                **artifact,
                "success": True,
                "status": "captured",
                "http_status": response.status if response else 0,
                "page_metrics": metrics,
                "page_size": {
                    "width": _positive_int(metrics.get("scroll_width"), artifact["page_size"]["width"]),
                    "height": _positive_int(metrics.get("scroll_height"), artifact["page_size"]["height"]),
                },
                "tile_plan": tile_plan,
                "files": files,
            }
            paths["receipt"].write_text(json.dumps(captured, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
            captured["files"].append({"kind": "receipt", "path": str(paths["receipt"])})
            return captured
        finally:
            context.close()
            browser.close()


def main() -> int:
    parser = argparse.ArgumentParser(description="Capture PixelRAG visual evidence from a manifest.")
    parser.add_argument("--manifest-file")
    parser.add_argument("--manifest-json")
    parser.add_argument("--output-dir", default=DEFAULT_OUTPUT_DIR)
    parser.add_argument("--dry-run", action="store_true")
    parser.add_argument("--timeout", type=int, default=30)
    parser.add_argument("--settle-ms", type=int, default=DEFAULT_SETTLE_MS)
    parser.add_argument("--max-tiles", type=int, default=DEFAULT_MAX_TILES)
    parser.add_argument("--wait-until", default="domcontentloaded")
    parser.add_argument("--allow-any-host", action="store_true")
    args = parser.parse_args()
    if not args.manifest_file and not args.manifest_json:
        parser.error("Pass --manifest-file or --manifest-json.")

    manifest = _load_manifest(args)
    errors = _validate_manifest(manifest, require_allowed_host=not args.allow_any_host)
    artifact = _base_artifact(manifest, args, errors)
    if errors or args.dry_run:
        print(json.dumps(artifact, ensure_ascii=False, indent=2))
        return 1 if errors else 0
    captured = _capture(manifest, args, artifact)
    print(json.dumps(captured, ensure_ascii=False, indent=2))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())