"""Read-only PixelRAG receipt to internal RAG candidate replay bridge. The bridge only converts saved visual evidence receipts into machine-readable candidate payloads. It does not OCR, call a model, write ai_insights, or promote anything into the production price tables. """ from __future__ import annotations import os import json from datetime import datetime, timezone from pathlib import Path from typing import Any, Mapping from services.pixelrag_crawler_integration_service import ( DEFAULT_ARTIFACT_MAX_AGE_HOURS, DEFAULT_ARTIFACT_ROOT, ECOMMERCE_PLATFORM_PROFILES, _parse_iso_datetime, _portable_receipt_file_path, _receipt_candidates, _safe_int, _visual_barrier_reason, ) POLICY = "read_only_pixelrag_rag_candidate_replay_v1" INTERNAL_RAG_TARGET = "rag_service.learning_episode_candidate_preview" DEFAULT_LIMIT = 50 DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT = os.getenv( "PIXELRAG_PLATFORM_PROBE_WORKER_RECEIPT_ROOT", "/app/data/ai_automation/pixelrag_platform_probe_worker_receipts" if Path("/app/data").exists() else "runtime_artifacts/pixelrag_platform_probe_worker_receipts", ) def _status_counts(items: list[dict[str, Any]]) -> dict[str, int]: counts: dict[str, int] = {} for item in items: status = str(item.get("candidate_status") or "unknown") counts[status] = counts.get(status, 0) + 1 return counts def _platform_from_receipt(receipt_path: Path, receipt: Mapping[str, Any]) -> str: capture_target = receipt.get("capture_target") or {} platform = str(capture_target.get("platform") or "").strip().lower() if platform: return platform try: return receipt_path.parent.parent.name except IndexError: return "unknown" def _manifest_id_from_receipt(receipt_path: Path, receipt: Mapping[str, Any]) -> str: return str(receipt.get("manifest_id") or receipt_path.parent.name or "").strip() def _count_receipt_files(receipt_path: Path, receipt: Mapping[str, Any]) -> dict[str, Any]: tile_file_count = 0 missing_file_count = 0 file_statuses: list[dict[str, Any]] = [] for item in list(receipt.get("files") or []): kind = item.get("kind") path = _portable_receipt_file_path(receipt_path, item.get("path"), kind) exists = path.exists() if kind == "tile": tile_file_count += 1 if not exists: missing_file_count += 1 file_statuses.append({ "kind": kind, "path": str(path), "exists": exists, }) return { "tile_file_count": tile_file_count, "missing_file_count": missing_file_count, "files": file_statuses, } def _candidate_text( *, platform: str, manifest_id: str, receipt: Mapping[str, Any], file_summary: Mapping[str, Any], barrier_reason: str, ) -> str: capture_target = receipt.get("capture_target") or {} page_metrics = receipt.get("page_metrics") or {} profile = ECOMMERCE_PLATFORM_PROFILES.get(platform) or {} url = str(capture_target.get("url") or page_metrics.get("final_url") or "").strip() title = str(page_metrics.get("title") or "").strip() data_targets = ", ".join(str(item) for item in list(profile.get("data_targets") or [])[:8]) parts = [ f"platform={platform}", f"manifest_id={manifest_id}", f"title={title or 'unknown'}", f"url={url or 'unknown'}", f"http_status={_safe_int(receipt.get('http_status'))}", f"capture_status={receipt.get('status') or 'unknown'}", f"tile_file_count={int(file_summary.get('tile_file_count') or 0)}", f"missing_file_count={int(file_summary.get('missing_file_count') or 0)}", ] if data_targets: parts.append(f"intended_fields={data_targets}") if barrier_reason: parts.append(f"visual_barrier={barrier_reason}") return " | ".join(parts) def _candidate_from_receipt_path( receipt_path: Path, *, now: datetime, max_age_hours: int, ) -> dict[str, Any]: errors: list[str] = [] try: receipt = json.loads(receipt_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: receipt = {} errors.append(str(exc)[:300]) platform = _platform_from_receipt(receipt_path, receipt) manifest_id = _manifest_id_from_receipt(receipt_path, receipt) generated_at = _parse_iso_datetime(receipt.get("generated_at")) age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None stale = age_hours is None or age_hours > max_age_hours file_summary = _count_receipt_files(receipt_path, receipt) missing_file_count = int(file_summary.get("missing_file_count") or 0) barrier_reason = _visual_barrier_reason(receipt) capture_status = str(receipt.get("status") or "").strip().lower() http_status = _safe_int(receipt.get("http_status")) blocked_reasons: list[str] = [] if errors: blocked_reasons.append("receipt_parse_error") if capture_status != "captured": blocked_reasons.append("receipt_not_captured") if stale: blocked_reasons.append("receipt_stale_or_missing_timestamp") if missing_file_count: blocked_reasons.append("receipt_missing_files") if barrier_reason: blocked_reasons.append(f"visual_barrier:{barrier_reason}") if http_status >= 400 and not barrier_reason: blocked_reasons.append(f"http_status_{http_status}") eligible = not blocked_reasons candidate_status = "eligible" if eligible else ("invalid" if errors else "blocked") capture_target = receipt.get("capture_target") or {} page_metrics = receipt.get("page_metrics") or {} return { "candidate_status": candidate_status, "eligible_for_rag_candidate_replay": eligible, "blocked_reasons": blocked_reasons, "platform": platform, "manifest_id": manifest_id, "receipt_path": str(receipt_path), "generated_at": receipt.get("generated_at"), "age_hours": round(age_hours, 3) if age_hours is not None else None, "stale": stale, "http_status": http_status, "capture_status": receipt.get("status"), "url": capture_target.get("url") or page_metrics.get("final_url"), "title": page_metrics.get("title"), "visual_barrier_detected": bool(barrier_reason), "visual_barrier_reason": barrier_reason, "tile_file_count": int(file_summary.get("tile_file_count") or 0), "missing_file_count": missing_file_count, "rag_candidate_text": _candidate_text( platform=platform, manifest_id=manifest_id, receipt=receipt, file_summary=file_summary, barrier_reason=barrier_reason, ), "internal_rag_target": INTERNAL_RAG_TARGET, "promotion_boundary": { "direct_ai_insights_write_allowed": False, "direct_price_write_allowed": False, "requires_ocr_or_vlm_replay": True, "requires_identity_matcher_replay": True, "requires_promotion_gate": True, "requires_embedding_signature_guard": True, }, "next_machine_action": ( "run_ocr_vlm_replay_then_promotion_gate" if eligible else ( "run_platform_probe_or_use_structured_api" if barrier_reason else "refresh_pixelrag_visual_capture_receipt" ) ), "file_summary": file_summary, } def _platform_probe_worker_receipt_candidates( root: Path, *, platforms: tuple[str, ...] | list[str] | None, limit: int, ) -> list[Path]: if not root.exists(): return [] selected_platforms = [ str(platform or "").strip().lower() for platform in (platforms or []) if str(platform or "").strip() ] candidates: list[Path] = [] if selected_platforms: for platform in selected_platforms: candidates.extend((root / platform).glob("*/platform_probe_worker_receipt.json")) else: candidates.extend(root.glob("*/*/platform_probe_worker_receipt.json")) return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] def _candidate_from_platform_probe_worker_receipt_path( receipt_path: Path, *, now: datetime, max_age_hours: int, ) -> dict[str, Any]: errors: list[str] = [] try: receipt = json.loads(receipt_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: receipt = {} errors.append(str(exc)[:300]) try: file_generated_at = datetime.fromtimestamp(receipt_path.stat().st_mtime, tz=timezone.utc) except OSError: file_generated_at = None generated_at = _parse_iso_datetime(receipt.get("generated_at")) or file_generated_at age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None stale = age_hours is None or age_hours > max_age_hours platform = str(receipt.get("platform") or receipt_path.parent.parent.name).strip().lower() manifest_id = str(receipt.get("manifest_id") or receipt_path.parent.name).strip() worker_status = str(receipt.get("worker_status") or "").strip() structured_package = ( receipt.get("structured_source_package") if isinstance(receipt.get("structured_source_package"), Mapping) else {} ) structured_available = bool(structured_package.get("available")) capture_runtime_gap = bool(structured_package.get("capture_runtime_unavailable")) or ( worker_status == "capture_runtime_unavailable_structured_fallback_package" ) capture_receipt_path = str(receipt.get("capture_receipt_path") or "").strip() source_receipt_path = str(receipt.get("source_receipt_path") or "").strip() blocked_reasons: list[str] = [] if errors: blocked_reasons.append("platform_probe_worker_receipt_parse_error") if stale: blocked_reasons.append("platform_probe_worker_receipt_stale_or_missing_timestamp") if capture_runtime_gap: blocked_reasons.append("capture_runtime_unavailable") barrier_type = str(receipt.get("barrier_type") or "").strip() if barrier_type: blocked_reasons.append(f"platform_barrier:{barrier_type}") if errors: candidate_status = "invalid" elif structured_available: candidate_status = "source_contract_fallback" elif worker_status == "executed_capture_ok" and capture_receipt_path: candidate_status = "probe_capture_ready" else: candidate_status = "worker_handoff" rag_text_parts = [ f"platform={platform}", f"manifest_id={manifest_id}", f"worker_status={worker_status or 'unknown'}", f"probe_status={receipt.get('probe_status') or 'unknown'}", f"barrier_type={barrier_type or 'none'}", f"source_receipt_path={source_receipt_path or 'unknown'}", f"capture_receipt_path={capture_receipt_path or 'none'}", f"structured_adapter={structured_package.get('adapter_code') or 'none'}", f"structured_source_count={int(structured_package.get('source_count') or 0)}", f"capture_runtime_unavailable={capture_runtime_gap}", ] return { "candidate_status": candidate_status, "source_type": "platform_probe_worker_receipt", "eligible_for_rag_candidate_replay": False, "eligible_for_source_contract_replay": bool(structured_available), "blocked_reasons": blocked_reasons, "platform": platform, "manifest_id": manifest_id, "receipt_path": str(receipt_path), "generated_at": generated_at.isoformat() if generated_at else None, "age_hours": round(age_hours, 3) if age_hours is not None else None, "stale": stale, "http_status": 0, "capture_status": worker_status, "url": receipt.get("url"), "title": receipt.get("title"), "visual_barrier_detected": bool(barrier_type), "visual_barrier_reason": barrier_type, "tile_file_count": 0, "missing_file_count": 0, "platform_probe_worker": { "worker_status": worker_status, "probe_status": receipt.get("probe_status"), "barrier_type": barrier_type, "source_receipt_path": source_receipt_path, "capture_receipt_path": capture_receipt_path, "network_call_performed": bool(receipt.get("network_call_performed")), "artifact_write_performed": bool(receipt.get("artifact_write_performed")), "writes_database": bool(receipt.get("writes_database")), "next_machine_action": receipt.get("next_machine_action"), "structured_source_package": structured_package, }, "rag_candidate_text": " | ".join(rag_text_parts), "internal_rag_target": INTERNAL_RAG_TARGET, "promotion_boundary": { "direct_ai_insights_write_allowed": False, "direct_price_write_allowed": False, "requires_ocr_or_vlm_replay": False, "requires_identity_matcher_replay": True, "requires_promotion_gate": True, "requires_embedding_signature_guard": True, "requires_source_contract_before_knowledge_write": True, }, "next_machine_action": ( "run_source_contract_replay_for_platform_probe_worker_receipt" if structured_available else ( "run_pixelrag_rag_candidate_replay_after_probe_capture" if capture_receipt_path else "refresh_pixelrag_platform_probe_worker_receipt" ) ), "file_summary": {"tile_file_count": 0, "missing_file_count": 0, "files": []}, } def _unique_receipt_candidates( root: Path, *, platforms: tuple[str, ...] | list[str] | None, limit: int, ) -> list[Path]: seen: set[Path] = set() candidates: list[Path] = [] selected_platforms = [ str(platform or "").strip().lower() for platform in (platforms or []) if str(platform or "").strip() ] if selected_platforms: for platform in selected_platforms: for path in _receipt_candidates(root, platform=platform): resolved = path.resolve() if resolved not in seen: seen.add(resolved) candidates.append(path) else: for path in _receipt_candidates(root): resolved = path.resolve() if resolved not in seen: seen.add(resolved) candidates.append(path) return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] def build_pixelrag_rag_candidate_replay_readback( *, artifact_root: str | Path | None = None, platform_probe_worker_receipt_root: str | Path | None = None, platform: str | tuple[str, ...] | list[str] | None = None, max_age_hours: int | None = None, limit: int | None = None, ) -> dict[str, Any]: """Build a replayable RAG candidate readback from PixelRAG receipts.""" root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT) worker_root = Path(platform_probe_worker_receipt_root or DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT) max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) if isinstance(platform, str): platforms = (platform.strip().lower(),) if platform.strip() else None else: platforms = tuple( str(item or "").strip().lower() for item in (platform or []) if str(item or "").strip() ) or None now = datetime.now(timezone.utc) candidates = _unique_receipt_candidates(root, platforms=platforms, limit=item_limit) replay_candidates = [ _candidate_from_receipt_path(path, now=now, max_age_hours=max_age) for path in candidates ] worker_receipts = _platform_probe_worker_receipt_candidates( worker_root, platforms=platforms, limit=item_limit, ) worker_candidates = [ _candidate_from_platform_probe_worker_receipt_path( path, now=now, max_age_hours=max_age, ) for path in worker_receipts ] all_candidates = replay_candidates + worker_candidates eligible_count = sum(1 for item in replay_candidates if item["candidate_status"] == "eligible") blocked_count = sum(1 for item in replay_candidates if item["candidate_status"] == "blocked") invalid_count = sum(1 for item in all_candidates if item["candidate_status"] == "invalid") visual_barrier_count = sum(1 for item in replay_candidates if item["visual_barrier_detected"]) missing_file_count = sum(int(item.get("missing_file_count") or 0) for item in replay_candidates) tile_file_count = sum(int(item.get("tile_file_count") or 0) for item in replay_candidates) stale_count = sum(1 for item in all_candidates if item.get("stale")) platform_probe_worker_receipt_count = len(worker_candidates) source_contract_fallback_count = sum( 1 for item in worker_candidates if item.get("candidate_status") == "source_contract_fallback" ) capture_runtime_gap_count = sum( 1 for item in worker_candidates if (item.get("platform_probe_worker") or {}).get("structured_source_package", {}).get( "capture_runtime_unavailable" ) ) if invalid_count and invalid_count == len(all_candidates): status = "critical" elif eligible_count == 0 or blocked_count or invalid_count or visual_barrier_count: status = "warning" else: status = "ok" if worker_candidates and not replay_candidates and not invalid_count: status = "ok" if not all_candidates: status = "warning" summary = { "receipt_count": len(all_candidates), "visual_receipt_count": len(replay_candidates), "platform_probe_worker_receipt_count": platform_probe_worker_receipt_count, "eligible_count": eligible_count, "blocked_count": blocked_count, "invalid_count": invalid_count, "visual_barrier_count": visual_barrier_count, "source_contract_fallback_count": source_contract_fallback_count, "capture_runtime_gap_count": capture_runtime_gap_count, "tile_file_count": tile_file_count, "missing_file_count": missing_file_count, "stale_count": stale_count, "status_counts": _status_counts(all_candidates), "platforms": sorted({str(item.get("platform") or "unknown") for item in all_candidates}), } if not all_candidates: next_machine_action = "run_pixelrag_visual_capture_worker" elif invalid_count and not eligible_count: next_machine_action = "fix_invalid_pixelrag_receipts" elif eligible_count and source_contract_fallback_count: next_machine_action = "run_ocr_vlm_replay_and_source_contract_replay" elif source_contract_fallback_count: next_machine_action = "run_source_contract_replay_for_platform_probe_worker_receipts" elif visual_barrier_count and not eligible_count: next_machine_action = "run_platform_probe_or_use_structured_api" elif eligible_count: next_machine_action = "run_ocr_vlm_replay_then_promotion_gate" else: next_machine_action = "refresh_pixelrag_visual_capture_receipt" return { "success": invalid_count == 0, "policy": POLICY, "status": status, "generated_at": now.isoformat(), "artifact_root": str(root), "platform_probe_worker_receipt_root": str(worker_root), "max_age_hours": max_age, "platform_filter": list(platforms or []), "summary": summary, "candidates": all_candidates, "replay_contract": { "internal_rag_target": INTERNAL_RAG_TARGET, "writes_database": False, "writes_ai_insights": False, "writes_price_tables": False, "network_call": False, "secret_read": False, "promotion_gate_required_before_knowledge_write": True, "ocr_or_vlm_required_before_field_extraction": True, "blocked_pages_are_not_product_data": True, "platform_probe_worker_receipts_are_source_contract_candidates": True, }, "next_machine_action": next_machine_action, "controlled_apply": { "network_call": False, "db_write": False, "secret_read": False, "production_price_write": False, "artifact_write": False, }, } __all__ = [ "INTERNAL_RAG_TARGET", "POLICY", "build_pixelrag_rag_candidate_replay_readback", ]