"""Read-only PixelRAG platform probe readiness. This module turns PixelRAG capture/VLM barrier evidence into concrete machine actions for public marketplace probing. It does not read sessions/cookies, perform network calls, write DB rows, or promote price truth. """ from __future__ import annotations import json import os from datetime import datetime, timezone from pathlib import Path from typing import Any, Mapping from urllib.parse import parse_qs, unquote_plus, urlparse from services.market_intel.adapters.registry import get_adapter from services.pixelrag_crawler_integration_service import ( DEFAULT_ARTIFACT_MAX_AGE_HOURS, DEFAULT_ARTIFACT_ROOT, build_pixelrag_marketplace_search_manifest, _parse_iso_datetime, ) from services.pixelrag_rag_candidate_replay_service import ( build_pixelrag_rag_candidate_replay_readback, ) POLICY = "read_only_pixelrag_platform_probe_readiness_v1" DEFAULT_LIMIT = 50 DEFAULT_ACCEPT_LANGUAGE = "zh-TW,zh-Hant;q=0.95,zh;q=0.9,en;q=0.7" DEFAULT_VLM_RECEIPT_ROOT = os.getenv( "PIXELRAG_VLM_REPLAY_RECEIPT_ROOT", "/app/data/ai_automation/pixelrag_vlm_replay_receipts" if Path("/app/data").exists() else "runtime_artifacts/pixelrag_vlm_replay_receipts", ) PLATFORM_ADAPTER_ALIASES = { "shopee_tw": "shopee", "coupang_tw": "coupang", "yahoo_shopping_tw": "yahoo", } def _normalise_platforms( platform: str | tuple[str, ...] | list[str] | None, ) -> tuple[str, ...]: if isinstance(platform, str): value = platform.strip().lower() return (value,) if value else () return tuple( str(item or "").strip().lower() for item in (platform or ()) if str(item or "").strip() ) def _read_json(path: Path) -> tuple[dict[str, Any], str]: try: parsed = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: return {}, str(exc)[:300] return parsed if isinstance(parsed, dict) else {}, "" def _safe_int(value: Any, default: int = 0) -> int: try: return int(value or default) except (TypeError, ValueError): return default def _safe_text(value: Any) -> str: if value is None: return "" if isinstance(value, str): return value if isinstance(value, Mapping): return " ".join(_safe_text(item) for item in value.values()) if isinstance(value, list): return " ".join(_safe_text(item) for item in value) return str(value) def _extract_keyword(url: str) -> str: parsed = urlparse(str(url or "")) query = parse_qs(parsed.query) for key in ("keyword", "q", "query", "p", "search", "searchKeyword"): values = query.get(key) or [] for value in values: text = unquote_plus(str(value or "")).strip() if text: return text return "" def _adapter_code(platform: str) -> str: clean = str(platform or "").strip().lower() return PLATFORM_ADAPTER_ALIASES.get(clean, clean) def _structured_source_plan(platform: str) -> dict[str, Any]: adapter = get_adapter(_adapter_code(platform)) if not adapter: return { "available": False, "adapter_code": _adapter_code(platform), "network_request_allowed": False, "database_write_allowed": False, "sources": [], } plan = adapter.build_discovery_plan() return { "available": True, "adapter_code": plan.get("platform_code"), "platform_name": plan.get("platform_name"), "base_url": plan.get("base_url"), "safety_policy": plan.get("safety_policy"), "network_request_allowed": bool(plan.get("network_request_allowed")), "database_write_allowed": bool(plan.get("database_write_allowed")), "sources": list(plan.get("sources") or []), } def _public_browser_context(platform: str) -> dict[str, Any]: return { "context_policy": "public_empty_browser_context_no_login", "locale": "zh-TW", "timezone_id": "Asia/Taipei", "viewport": {"name": "desktop-1440", "width": 1440, "height": 950}, "extra_http_headers": { "Accept-Language": DEFAULT_ACCEPT_LANGUAGE, }, "credentialed_session_allowed": False, "storage_state_allowed": False, "raw_cookie_or_session_read_allowed": False, "login_allowed": False, "cart_or_checkout_allowed": False, "platform": platform, } def _barrier_type_from_signals( *, platform: str, http_status: int, visual_barrier_reason: str, url: str, title: str, validation: Mapping[str, Any] | None = None, parsed_output: Mapping[str, Any] | None = None, ) -> str: validation = validation or {} parsed_output = parsed_output or {} haystack = " ".join( [ visual_barrier_reason, url, title, _safe_text(parsed_output.get("notes")), _safe_text((parsed_output.get("fields") or {}).get("title")), ] ).lower() if http_status in {401, 403} or "access denied" in haystack: return "access_denied" if "verify/traffic" in haystack or "traffic" in haystack: return "traffic_verification_interstitial" if ( validation.get("interstitial_signal_detected") or "language selection" in haystack or "select language" in haystack or "choose language" in haystack or "region selection" in haystack or "select region" in haystack or "language" in haystack or "語言" in haystack ): return "language_or_region_interstitial" if validation.get("generic_marketplace_title_detected") or ( platform == "shopee_tw" and "花得更少買得更好" in haystack ): return "generic_marketplace_landing" if visual_barrier_reason: return "platform_visual_barrier" if validation.get("non_product_or_interstitial_detected"): return "non_product_or_interstitial" return "unknown_platform_barrier" def _probe_status(platform: str, barrier_type: str) -> str: if platform == "shopee_tw" and barrier_type in { "language_or_region_interstitial", "generic_marketplace_landing", "traffic_verification_interstitial", }: return "ready_for_public_context_probe" if barrier_type in {"access_denied", "traffic_verification_interstitial"}: return "structured_source_or_backoff_required" return "ready_for_platform_probe" def _next_machine_action(platform: str, barrier_type: str, status: str) -> str: if platform == "shopee_tw" and status == "ready_for_public_context_probe": return "run_shopee_public_context_probe_then_structured_source_fallback" if status == "structured_source_or_backoff_required": return "use_structured_source_or_platform_backoff_policy" return "run_public_platform_context_probe" def _recommended_actions(platform: str, barrier_type: str, status: str) -> list[dict[str, Any]]: actions: list[dict[str, Any]] = [] if status == "ready_for_public_context_probe": actions.append({ "order": 1, "action": "rerun_visual_capture_with_public_browser_context", "machine_runnable": True, "context_keys": ["locale", "timezone_id", "extra_http_headers", "viewport"], }) actions.append({ "order": len(actions) + 1, "action": "read_structured_market_intel_adapter_sources", "machine_runnable": True, "adapter_code": _adapter_code(platform), }) if barrier_type in {"access_denied", "traffic_verification_interstitial"}: actions.append({ "order": len(actions) + 1, "action": "apply_platform_backoff_and_do_not_treat_barrier_as_product_data", "machine_runnable": True, }) actions.append({ "order": len(actions) + 1, "action": "keep_visual_fields_out_of_formal_price_tables", "machine_runnable": True, }) return actions def _manifest_preview(platform: str, url: str) -> dict[str, Any] | None: keyword = _extract_keyword(url) if not keyword: return None manifest = build_pixelrag_marketplace_search_manifest( platform=platform, keyword=keyword, crawler="PixelRAGPlatformProbe.public_context_visual_fallback", trigger_reason="platform_interstitial_or_blocked_page_probe", evidence_intent="collect_public_marketplace_offer_cards_after_platform_probe", ) if manifest.get("success"): manifest["public_browser_context"] = _public_browser_context(platform) return manifest def _vlm_receipt_candidates( root: Path, *, platforms: tuple[str, ...], limit: int, ) -> list[Path]: if not root.exists(): return [] candidates: list[Path] = [] if platforms: for platform in platforms: candidates.extend((root / platform).glob("*/vlm_replay_receipt.json")) else: candidates.extend(root.glob("*/*/vlm_replay_receipt.json")) return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] def _candidate_from_capture(candidate: Mapping[str, Any]) -> dict[str, Any] | None: next_action = str(candidate.get("next_machine_action") or "") visual_barrier_reason = str(candidate.get("visual_barrier_reason") or "") http_status = _safe_int(candidate.get("http_status")) if ( "platform_probe" not in next_action and not visual_barrier_reason and http_status < 400 ): return None platform = str(candidate.get("platform") or "unknown").strip().lower() manifest_id = str(candidate.get("manifest_id") or "").strip() url = str(candidate.get("url") or "").strip() title = str(candidate.get("title") or "").strip() barrier_type = _barrier_type_from_signals( platform=platform, http_status=http_status, visual_barrier_reason=visual_barrier_reason, url=url, title=title, ) status = _probe_status(platform, barrier_type) return { "platform": platform, "manifest_id": manifest_id, "source_type": "capture_receipt", "source_receipt_path": candidate.get("receipt_path"), "generated_at": candidate.get("generated_at"), "age_hours": candidate.get("age_hours"), "url": url, "title": title, "http_status": http_status, "barrier_type": barrier_type, "visual_barrier_reason": visual_barrier_reason, "probe_status": status, "probe_ready": True, "next_machine_action": _next_machine_action(platform, barrier_type, status), "public_browser_context": _public_browser_context(platform), "capture_manifest_preview": _manifest_preview(platform, url), "structured_source_fallback": _structured_source_plan(platform), "recommended_probe_actions": _recommended_actions(platform, barrier_type, status), "source_next_machine_action": candidate.get("next_machine_action"), "writes_database": False, "primary_human_gate_count": 0, } def _candidate_from_vlm(path: Path, *, now: datetime, max_age_hours: int) -> dict[str, Any] | None: receipt, error = _read_json(path) if error: return { "platform": path.parent.parent.name, "manifest_id": path.parent.name, "source_type": "vlm_replay_receipt", "source_receipt_path": str(path), "probe_status": "invalid_vlm_receipt", "probe_ready": False, "barrier_type": "invalid_receipt", "errors": [error], "next_machine_action": "fix_invalid_pixelrag_vlm_receipt", "writes_database": False, "primary_human_gate_count": 0, } validation = receipt.get("validation") if isinstance(receipt.get("validation"), Mapping) else {} parsed_output = receipt.get("parsed_output") if isinstance(receipt.get("parsed_output"), Mapping) else {} next_action = str(receipt.get("next_machine_action") or "") if ( "platform_probe" not in next_action and not validation.get("non_product_or_interstitial_detected") and not validation.get("blocked_page_detected") ): return None platform = str(receipt.get("platform") or path.parent.parent.name).strip().lower() manifest_id = str(receipt.get("manifest_id") or path.parent.name).strip() generated = _parse_iso_datetime(receipt.get("generated_at")) age_hours = ((now - generated).total_seconds() / 3600) if generated else None source_url = "" source_title = "" source_receipt = str(receipt.get("source_receipt_path") or "") if source_receipt: capture, _ = _read_json(Path(source_receipt)) capture_target = capture.get("capture_target") or {} page_metrics = capture.get("page_metrics") or {} source_url = str(capture_target.get("url") or page_metrics.get("final_url") or "") source_title = str(page_metrics.get("title") or "") barrier_type = _barrier_type_from_signals( platform=platform, http_status=0, visual_barrier_reason="", url=source_url, title=source_title, validation=validation, parsed_output=parsed_output, ) status = _probe_status(platform, barrier_type) return { "platform": platform, "manifest_id": manifest_id, "source_type": "vlm_replay_receipt", "source_receipt_path": str(path), "source_capture_receipt_path": source_receipt, "generated_at": receipt.get("generated_at"), "age_hours": round(age_hours, 3) if age_hours is not None else None, "stale": age_hours is None or age_hours > max_age_hours, "url": source_url, "title": source_title, "http_status": 0, "barrier_type": barrier_type, "probe_status": status, "probe_ready": True, "validation": { "blocked_page_detected": bool(validation.get("blocked_page_detected")), "non_product_or_interstitial_detected": bool( validation.get("non_product_or_interstitial_detected") ), "interstitial_signal_detected": bool( validation.get("interstitial_signal_detected") ), "generic_marketplace_title_detected": bool( validation.get("generic_marketplace_title_detected") ), "present_field_count": _safe_int(validation.get("present_field_count")), "missing_required_fields": list(validation.get("missing_required_fields") or []), }, "next_machine_action": _next_machine_action(platform, barrier_type, status), "public_browser_context": _public_browser_context(platform), "capture_manifest_preview": _manifest_preview(platform, source_url), "structured_source_fallback": _structured_source_plan(platform), "recommended_probe_actions": _recommended_actions(platform, barrier_type, status), "source_next_machine_action": receipt.get("next_machine_action"), "writes_database": False, "primary_human_gate_count": 0, } def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]: by_key: dict[tuple[str, str], dict[str, Any]] = {} source_rank = {"vlm_replay_receipt": 2, "capture_receipt": 1} for item in items: key = (str(item.get("platform") or ""), str(item.get("manifest_id") or "")) current = by_key.get(key) if not current or source_rank.get(str(item.get("source_type")), 0) >= source_rank.get( str(current.get("source_type")), 0 ): by_key[key] = item return list(by_key.values()) def build_pixelrag_platform_probe_readiness( *, artifact_root: str | Path | None = None, vlm_receipt_root: str | Path | None = None, platform: str | tuple[str, ...] | list[str] | None = None, max_age_hours: int | None = None, limit: int | None = None, ) -> dict[str, Any]: """Build a no-write platform probe plan from PixelRAG barrier receipts.""" capture_root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT) vlm_root = Path(vlm_receipt_root or DEFAULT_VLM_RECEIPT_ROOT) platforms = _normalise_platforms(platform) max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) now = datetime.now(timezone.utc) capture_readback = build_pixelrag_rag_candidate_replay_readback( artifact_root=capture_root, platform=platforms, max_age_hours=max_age, limit=item_limit, ) capture_items = [ item for item in ( _candidate_from_capture(candidate) for candidate in list(capture_readback.get("candidates") or []) ) if item ] vlm_items = [ item for item in ( _candidate_from_vlm(path, now=now, max_age_hours=max_age) for path in _vlm_receipt_candidates(vlm_root, platforms=platforms, limit=item_limit) ) if item ] probe_items = _dedupe_items(capture_items + vlm_items) invalid_count = sum(1 for item in probe_items if not item.get("probe_ready")) ready_count = sum(1 for item in probe_items if item.get("probe_ready")) shopee_public_context_count = sum( 1 for item in probe_items if item.get("platform") == "shopee_tw" and item.get("probe_status") == "ready_for_public_context_probe" ) access_denied_count = sum(1 for item in probe_items if item.get("barrier_type") == "access_denied") traffic_count = sum( 1 for item in probe_items if item.get("barrier_type") == "traffic_verification_interstitial" ) language_count = sum( 1 for item in probe_items if item.get("barrier_type") == "language_or_region_interstitial" ) structured_fallback_count = sum( 1 for item in probe_items if (item.get("structured_source_fallback") or {}).get("available") ) if invalid_count and invalid_count == len(probe_items): status = "critical" elif probe_items and ready_count: status = "ok" else: status = "warning" next_action = ( "fix_invalid_pixelrag_platform_probe_receipts" if invalid_count and not ready_count else ( "run_platform_probe_or_structured_source_fallback" if ready_count else "run_pixelrag_visual_capture_worker" ) ) return { "success": status != "critical", "policy": POLICY, "status": status, "generated_at": now.isoformat(), "artifact_root": str(capture_root), "vlm_receipt_root": str(vlm_root), "platform_filter": list(platforms), "max_age_hours": max_age, "limit": item_limit, "summary": { "probe_candidate_count": len(probe_items), "ready_for_probe_count": ready_count, "invalid_count": invalid_count, "capture_source_count": len(capture_items), "vlm_source_count": len(vlm_items), "shopee_public_context_probe_count": shopee_public_context_count, "language_or_region_interstitial_count": language_count, "traffic_verification_count": traffic_count, "access_denied_count": access_denied_count, "structured_source_fallback_count": structured_fallback_count, "writes_database_count": 0, "primary_human_gate_count": 0, "platforms": sorted({str(item.get("platform") or "unknown") for item in probe_items}), }, "probe_items": probe_items, "source_capture_replay": { "policy": capture_readback.get("policy"), "status": capture_readback.get("status"), "summary": capture_readback.get("summary"), "next_machine_action": capture_readback.get("next_machine_action"), }, "probe_contract": { "automation_mode": "platform_probe_plan_no_write", "network_call": False, "db_write": False, "writes_database": False, "writes_ai_insights": False, "writes_price_tables": False, "secret_read": False, "raw_cookie_or_session_read": False, "credentialed_session_allowed": False, "login_allowed": False, "blocked_pages_are_not_product_data": True, "visual_fields_are_candidate_evidence_only": True, "primary_human_gate_count": 0, }, "controlled_apply": { "network_call": False, "db_write": False, "writes_database": False, "writes_database_count": 0, "secret_read": False, "raw_cookie_or_session_read": False, "production_price_write": False, "artifact_write": False, "primary_human_gate_count": 0, }, "next_machine_action": next_action, } __all__ = [ "POLICY", "build_pixelrag_platform_probe_readiness", ]