524 lines
20 KiB
Python
524 lines
20 KiB
Python
"""Read-only PixelRAG receipt to internal RAG candidate replay bridge.
|
|
|
|
The bridge only converts saved visual evidence receipts into machine-readable
|
|
candidate payloads. It does not OCR, call a model, write ai_insights, or promote
|
|
anything into the production price tables.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Mapping
|
|
|
|
from services.pixelrag_crawler_integration_service import (
|
|
DEFAULT_ARTIFACT_MAX_AGE_HOURS,
|
|
DEFAULT_ARTIFACT_ROOT,
|
|
ECOMMERCE_PLATFORM_PROFILES,
|
|
_parse_iso_datetime,
|
|
_portable_receipt_file_path,
|
|
_receipt_candidates,
|
|
_safe_int,
|
|
_visual_barrier_reason,
|
|
)
|
|
|
|
|
|
POLICY = "read_only_pixelrag_rag_candidate_replay_v1"
|
|
INTERNAL_RAG_TARGET = "rag_service.learning_episode_candidate_preview"
|
|
DEFAULT_LIMIT = 50
|
|
DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT = os.getenv(
|
|
"PIXELRAG_PLATFORM_PROBE_WORKER_RECEIPT_ROOT",
|
|
"/app/data/ai_automation/pixelrag_platform_probe_worker_receipts"
|
|
if Path("/app/data").exists()
|
|
else "runtime_artifacts/pixelrag_platform_probe_worker_receipts",
|
|
)
|
|
|
|
|
|
def _status_counts(items: list[dict[str, Any]]) -> dict[str, int]:
|
|
counts: dict[str, int] = {}
|
|
for item in items:
|
|
status = str(item.get("candidate_status") or "unknown")
|
|
counts[status] = counts.get(status, 0) + 1
|
|
return counts
|
|
|
|
|
|
def _platform_from_receipt(receipt_path: Path, receipt: Mapping[str, Any]) -> str:
|
|
capture_target = receipt.get("capture_target") or {}
|
|
platform = str(capture_target.get("platform") or "").strip().lower()
|
|
if platform:
|
|
return platform
|
|
try:
|
|
return receipt_path.parent.parent.name
|
|
except IndexError:
|
|
return "unknown"
|
|
|
|
|
|
def _manifest_id_from_receipt(receipt_path: Path, receipt: Mapping[str, Any]) -> str:
|
|
return str(receipt.get("manifest_id") or receipt_path.parent.name or "").strip()
|
|
|
|
|
|
def _count_receipt_files(receipt_path: Path, receipt: Mapping[str, Any]) -> dict[str, Any]:
|
|
tile_file_count = 0
|
|
missing_file_count = 0
|
|
file_statuses: list[dict[str, Any]] = []
|
|
for item in list(receipt.get("files") or []):
|
|
kind = item.get("kind")
|
|
path = _portable_receipt_file_path(receipt_path, item.get("path"), kind)
|
|
exists = path.exists()
|
|
if kind == "tile":
|
|
tile_file_count += 1
|
|
if not exists:
|
|
missing_file_count += 1
|
|
file_statuses.append({
|
|
"kind": kind,
|
|
"path": str(path),
|
|
"exists": exists,
|
|
})
|
|
return {
|
|
"tile_file_count": tile_file_count,
|
|
"missing_file_count": missing_file_count,
|
|
"files": file_statuses,
|
|
}
|
|
|
|
|
|
def _candidate_text(
|
|
*,
|
|
platform: str,
|
|
manifest_id: str,
|
|
receipt: Mapping[str, Any],
|
|
file_summary: Mapping[str, Any],
|
|
barrier_reason: str,
|
|
) -> str:
|
|
capture_target = receipt.get("capture_target") or {}
|
|
page_metrics = receipt.get("page_metrics") or {}
|
|
profile = ECOMMERCE_PLATFORM_PROFILES.get(platform) or {}
|
|
url = str(capture_target.get("url") or page_metrics.get("final_url") or "").strip()
|
|
title = str(page_metrics.get("title") or "").strip()
|
|
data_targets = ", ".join(str(item) for item in list(profile.get("data_targets") or [])[:8])
|
|
parts = [
|
|
f"platform={platform}",
|
|
f"manifest_id={manifest_id}",
|
|
f"title={title or 'unknown'}",
|
|
f"url={url or 'unknown'}",
|
|
f"http_status={_safe_int(receipt.get('http_status'))}",
|
|
f"capture_status={receipt.get('status') or 'unknown'}",
|
|
f"tile_file_count={int(file_summary.get('tile_file_count') or 0)}",
|
|
f"missing_file_count={int(file_summary.get('missing_file_count') or 0)}",
|
|
]
|
|
if data_targets:
|
|
parts.append(f"intended_fields={data_targets}")
|
|
if barrier_reason:
|
|
parts.append(f"visual_barrier={barrier_reason}")
|
|
return " | ".join(parts)
|
|
|
|
|
|
def _candidate_from_receipt_path(
|
|
receipt_path: Path,
|
|
*,
|
|
now: datetime,
|
|
max_age_hours: int,
|
|
) -> dict[str, Any]:
|
|
errors: list[str] = []
|
|
try:
|
|
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
receipt = {}
|
|
errors.append(str(exc)[:300])
|
|
|
|
platform = _platform_from_receipt(receipt_path, receipt)
|
|
manifest_id = _manifest_id_from_receipt(receipt_path, receipt)
|
|
generated_at = _parse_iso_datetime(receipt.get("generated_at"))
|
|
age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None
|
|
stale = age_hours is None or age_hours > max_age_hours
|
|
file_summary = _count_receipt_files(receipt_path, receipt)
|
|
missing_file_count = int(file_summary.get("missing_file_count") or 0)
|
|
barrier_reason = _visual_barrier_reason(receipt)
|
|
capture_status = str(receipt.get("status") or "").strip().lower()
|
|
http_status = _safe_int(receipt.get("http_status"))
|
|
|
|
blocked_reasons: list[str] = []
|
|
if errors:
|
|
blocked_reasons.append("receipt_parse_error")
|
|
if capture_status != "captured":
|
|
blocked_reasons.append("receipt_not_captured")
|
|
if stale:
|
|
blocked_reasons.append("receipt_stale_or_missing_timestamp")
|
|
if missing_file_count:
|
|
blocked_reasons.append("receipt_missing_files")
|
|
if barrier_reason:
|
|
blocked_reasons.append(f"visual_barrier:{barrier_reason}")
|
|
if http_status >= 400 and not barrier_reason:
|
|
blocked_reasons.append(f"http_status_{http_status}")
|
|
|
|
eligible = not blocked_reasons
|
|
candidate_status = "eligible" if eligible else ("invalid" if errors else "blocked")
|
|
capture_target = receipt.get("capture_target") or {}
|
|
page_metrics = receipt.get("page_metrics") or {}
|
|
|
|
return {
|
|
"candidate_status": candidate_status,
|
|
"eligible_for_rag_candidate_replay": eligible,
|
|
"blocked_reasons": blocked_reasons,
|
|
"platform": platform,
|
|
"manifest_id": manifest_id,
|
|
"receipt_path": str(receipt_path),
|
|
"generated_at": receipt.get("generated_at"),
|
|
"age_hours": round(age_hours, 3) if age_hours is not None else None,
|
|
"stale": stale,
|
|
"http_status": http_status,
|
|
"capture_status": receipt.get("status"),
|
|
"url": capture_target.get("url") or page_metrics.get("final_url"),
|
|
"title": page_metrics.get("title"),
|
|
"visual_barrier_detected": bool(barrier_reason),
|
|
"visual_barrier_reason": barrier_reason,
|
|
"tile_file_count": int(file_summary.get("tile_file_count") or 0),
|
|
"missing_file_count": missing_file_count,
|
|
"rag_candidate_text": _candidate_text(
|
|
platform=platform,
|
|
manifest_id=manifest_id,
|
|
receipt=receipt,
|
|
file_summary=file_summary,
|
|
barrier_reason=barrier_reason,
|
|
),
|
|
"internal_rag_target": INTERNAL_RAG_TARGET,
|
|
"promotion_boundary": {
|
|
"direct_ai_insights_write_allowed": False,
|
|
"direct_price_write_allowed": False,
|
|
"requires_ocr_or_vlm_replay": True,
|
|
"requires_identity_matcher_replay": True,
|
|
"requires_promotion_gate": True,
|
|
"requires_embedding_signature_guard": True,
|
|
},
|
|
"next_machine_action": (
|
|
"run_ocr_vlm_replay_then_promotion_gate"
|
|
if eligible
|
|
else (
|
|
"run_platform_probe_or_use_structured_api"
|
|
if barrier_reason
|
|
else "refresh_pixelrag_visual_capture_receipt"
|
|
)
|
|
),
|
|
"file_summary": file_summary,
|
|
}
|
|
|
|
|
|
def _platform_probe_worker_receipt_candidates(
|
|
root: Path,
|
|
*,
|
|
platforms: tuple[str, ...] | list[str] | None,
|
|
limit: int,
|
|
) -> list[Path]:
|
|
if not root.exists():
|
|
return []
|
|
selected_platforms = [
|
|
str(platform or "").strip().lower()
|
|
for platform in (platforms or [])
|
|
if str(platform or "").strip()
|
|
]
|
|
candidates: list[Path] = []
|
|
if selected_platforms:
|
|
for platform in selected_platforms:
|
|
candidates.extend((root / platform).glob("*/platform_probe_worker_receipt.json"))
|
|
else:
|
|
candidates.extend(root.glob("*/*/platform_probe_worker_receipt.json"))
|
|
return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit]
|
|
|
|
|
|
def _candidate_from_platform_probe_worker_receipt_path(
|
|
receipt_path: Path,
|
|
*,
|
|
now: datetime,
|
|
max_age_hours: int,
|
|
) -> dict[str, Any]:
|
|
errors: list[str] = []
|
|
try:
|
|
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
receipt = {}
|
|
errors.append(str(exc)[:300])
|
|
|
|
try:
|
|
file_generated_at = datetime.fromtimestamp(receipt_path.stat().st_mtime, tz=timezone.utc)
|
|
except OSError:
|
|
file_generated_at = None
|
|
generated_at = _parse_iso_datetime(receipt.get("generated_at")) or file_generated_at
|
|
age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None
|
|
stale = age_hours is None or age_hours > max_age_hours
|
|
platform = str(receipt.get("platform") or receipt_path.parent.parent.name).strip().lower()
|
|
manifest_id = str(receipt.get("manifest_id") or receipt_path.parent.name).strip()
|
|
worker_status = str(receipt.get("worker_status") or "").strip()
|
|
structured_package = (
|
|
receipt.get("structured_source_package")
|
|
if isinstance(receipt.get("structured_source_package"), Mapping)
|
|
else {}
|
|
)
|
|
structured_available = bool(structured_package.get("available"))
|
|
capture_runtime_gap = bool(structured_package.get("capture_runtime_unavailable")) or (
|
|
worker_status == "capture_runtime_unavailable_structured_fallback_package"
|
|
)
|
|
capture_receipt_path = str(receipt.get("capture_receipt_path") or "").strip()
|
|
source_receipt_path = str(receipt.get("source_receipt_path") or "").strip()
|
|
blocked_reasons: list[str] = []
|
|
if errors:
|
|
blocked_reasons.append("platform_probe_worker_receipt_parse_error")
|
|
if stale:
|
|
blocked_reasons.append("platform_probe_worker_receipt_stale_or_missing_timestamp")
|
|
if capture_runtime_gap:
|
|
blocked_reasons.append("capture_runtime_unavailable")
|
|
barrier_type = str(receipt.get("barrier_type") or "").strip()
|
|
if barrier_type:
|
|
blocked_reasons.append(f"platform_barrier:{barrier_type}")
|
|
|
|
if errors:
|
|
candidate_status = "invalid"
|
|
elif structured_available:
|
|
candidate_status = "source_contract_fallback"
|
|
elif worker_status == "executed_capture_ok" and capture_receipt_path:
|
|
candidate_status = "probe_capture_ready"
|
|
else:
|
|
candidate_status = "worker_handoff"
|
|
|
|
rag_text_parts = [
|
|
f"platform={platform}",
|
|
f"manifest_id={manifest_id}",
|
|
f"worker_status={worker_status or 'unknown'}",
|
|
f"probe_status={receipt.get('probe_status') or 'unknown'}",
|
|
f"barrier_type={barrier_type or 'none'}",
|
|
f"source_receipt_path={source_receipt_path or 'unknown'}",
|
|
f"capture_receipt_path={capture_receipt_path or 'none'}",
|
|
f"structured_adapter={structured_package.get('adapter_code') or 'none'}",
|
|
f"structured_source_count={int(structured_package.get('source_count') or 0)}",
|
|
f"capture_runtime_unavailable={capture_runtime_gap}",
|
|
]
|
|
return {
|
|
"candidate_status": candidate_status,
|
|
"source_type": "platform_probe_worker_receipt",
|
|
"eligible_for_rag_candidate_replay": False,
|
|
"eligible_for_source_contract_replay": bool(structured_available),
|
|
"blocked_reasons": blocked_reasons,
|
|
"platform": platform,
|
|
"manifest_id": manifest_id,
|
|
"receipt_path": str(receipt_path),
|
|
"generated_at": generated_at.isoformat() if generated_at else None,
|
|
"age_hours": round(age_hours, 3) if age_hours is not None else None,
|
|
"stale": stale,
|
|
"http_status": 0,
|
|
"capture_status": worker_status,
|
|
"url": receipt.get("url"),
|
|
"title": receipt.get("title"),
|
|
"visual_barrier_detected": bool(barrier_type),
|
|
"visual_barrier_reason": barrier_type,
|
|
"tile_file_count": 0,
|
|
"missing_file_count": 0,
|
|
"platform_probe_worker": {
|
|
"worker_status": worker_status,
|
|
"probe_status": receipt.get("probe_status"),
|
|
"barrier_type": barrier_type,
|
|
"source_receipt_path": source_receipt_path,
|
|
"capture_receipt_path": capture_receipt_path,
|
|
"network_call_performed": bool(receipt.get("network_call_performed")),
|
|
"artifact_write_performed": bool(receipt.get("artifact_write_performed")),
|
|
"writes_database": bool(receipt.get("writes_database")),
|
|
"next_machine_action": receipt.get("next_machine_action"),
|
|
"structured_source_package": structured_package,
|
|
},
|
|
"rag_candidate_text": " | ".join(rag_text_parts),
|
|
"internal_rag_target": INTERNAL_RAG_TARGET,
|
|
"promotion_boundary": {
|
|
"direct_ai_insights_write_allowed": False,
|
|
"direct_price_write_allowed": False,
|
|
"requires_ocr_or_vlm_replay": False,
|
|
"requires_identity_matcher_replay": True,
|
|
"requires_promotion_gate": True,
|
|
"requires_embedding_signature_guard": True,
|
|
"requires_source_contract_before_knowledge_write": True,
|
|
},
|
|
"next_machine_action": (
|
|
"run_source_contract_replay_for_platform_probe_worker_receipt"
|
|
if structured_available
|
|
else (
|
|
"run_pixelrag_rag_candidate_replay_after_probe_capture"
|
|
if capture_receipt_path
|
|
else "refresh_pixelrag_platform_probe_worker_receipt"
|
|
)
|
|
),
|
|
"file_summary": {"tile_file_count": 0, "missing_file_count": 0, "files": []},
|
|
}
|
|
|
|
|
|
def _unique_receipt_candidates(
|
|
root: Path,
|
|
*,
|
|
platforms: tuple[str, ...] | list[str] | None,
|
|
limit: int,
|
|
) -> list[Path]:
|
|
seen: set[Path] = set()
|
|
candidates: list[Path] = []
|
|
selected_platforms = [
|
|
str(platform or "").strip().lower()
|
|
for platform in (platforms or [])
|
|
if str(platform or "").strip()
|
|
]
|
|
if selected_platforms:
|
|
for platform in selected_platforms:
|
|
for path in _receipt_candidates(root, platform=platform):
|
|
resolved = path.resolve()
|
|
if resolved not in seen:
|
|
seen.add(resolved)
|
|
candidates.append(path)
|
|
else:
|
|
for path in _receipt_candidates(root):
|
|
resolved = path.resolve()
|
|
if resolved not in seen:
|
|
seen.add(resolved)
|
|
candidates.append(path)
|
|
return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit]
|
|
|
|
|
|
def build_pixelrag_rag_candidate_replay_readback(
|
|
*,
|
|
artifact_root: str | Path | None = None,
|
|
platform_probe_worker_receipt_root: str | Path | None = None,
|
|
platform: str | tuple[str, ...] | list[str] | None = None,
|
|
max_age_hours: int | None = None,
|
|
limit: int | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Build a replayable RAG candidate readback from PixelRAG receipts."""
|
|
root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT)
|
|
worker_root = Path(platform_probe_worker_receipt_root or DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT)
|
|
max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS))
|
|
item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250))
|
|
if isinstance(platform, str):
|
|
platforms = (platform.strip().lower(),) if platform.strip() else None
|
|
else:
|
|
platforms = tuple(
|
|
str(item or "").strip().lower()
|
|
for item in (platform or [])
|
|
if str(item or "").strip()
|
|
) or None
|
|
now = datetime.now(timezone.utc)
|
|
|
|
candidates = _unique_receipt_candidates(root, platforms=platforms, limit=item_limit)
|
|
replay_candidates = [
|
|
_candidate_from_receipt_path(path, now=now, max_age_hours=max_age)
|
|
for path in candidates
|
|
]
|
|
worker_receipts = _platform_probe_worker_receipt_candidates(
|
|
worker_root,
|
|
platforms=platforms,
|
|
limit=item_limit,
|
|
)
|
|
worker_candidates = [
|
|
_candidate_from_platform_probe_worker_receipt_path(
|
|
path,
|
|
now=now,
|
|
max_age_hours=max_age,
|
|
)
|
|
for path in worker_receipts
|
|
]
|
|
all_candidates = replay_candidates + worker_candidates
|
|
eligible_count = sum(1 for item in replay_candidates if item["candidate_status"] == "eligible")
|
|
blocked_count = sum(1 for item in replay_candidates if item["candidate_status"] == "blocked")
|
|
invalid_count = sum(1 for item in all_candidates if item["candidate_status"] == "invalid")
|
|
visual_barrier_count = sum(1 for item in replay_candidates if item["visual_barrier_detected"])
|
|
missing_file_count = sum(int(item.get("missing_file_count") or 0) for item in replay_candidates)
|
|
tile_file_count = sum(int(item.get("tile_file_count") or 0) for item in replay_candidates)
|
|
stale_count = sum(1 for item in all_candidates if item.get("stale"))
|
|
platform_probe_worker_receipt_count = len(worker_candidates)
|
|
source_contract_fallback_count = sum(
|
|
1 for item in worker_candidates if item.get("candidate_status") == "source_contract_fallback"
|
|
)
|
|
capture_runtime_gap_count = sum(
|
|
1
|
|
for item in worker_candidates
|
|
if (item.get("platform_probe_worker") or {}).get("structured_source_package", {}).get(
|
|
"capture_runtime_unavailable"
|
|
)
|
|
)
|
|
|
|
if invalid_count and invalid_count == len(all_candidates):
|
|
status = "critical"
|
|
elif eligible_count == 0 or blocked_count or invalid_count or visual_barrier_count:
|
|
status = "warning"
|
|
else:
|
|
status = "ok"
|
|
|
|
if worker_candidates and not replay_candidates and not invalid_count:
|
|
status = "ok"
|
|
if not all_candidates:
|
|
status = "warning"
|
|
|
|
summary = {
|
|
"receipt_count": len(all_candidates),
|
|
"visual_receipt_count": len(replay_candidates),
|
|
"platform_probe_worker_receipt_count": platform_probe_worker_receipt_count,
|
|
"eligible_count": eligible_count,
|
|
"blocked_count": blocked_count,
|
|
"invalid_count": invalid_count,
|
|
"visual_barrier_count": visual_barrier_count,
|
|
"source_contract_fallback_count": source_contract_fallback_count,
|
|
"capture_runtime_gap_count": capture_runtime_gap_count,
|
|
"tile_file_count": tile_file_count,
|
|
"missing_file_count": missing_file_count,
|
|
"stale_count": stale_count,
|
|
"status_counts": _status_counts(all_candidates),
|
|
"platforms": sorted({str(item.get("platform") or "unknown") for item in all_candidates}),
|
|
}
|
|
|
|
if not all_candidates:
|
|
next_machine_action = "run_pixelrag_visual_capture_worker"
|
|
elif invalid_count and not eligible_count:
|
|
next_machine_action = "fix_invalid_pixelrag_receipts"
|
|
elif eligible_count and source_contract_fallback_count:
|
|
next_machine_action = "run_ocr_vlm_replay_and_source_contract_replay"
|
|
elif source_contract_fallback_count:
|
|
next_machine_action = "run_source_contract_replay_for_platform_probe_worker_receipts"
|
|
elif visual_barrier_count and not eligible_count:
|
|
next_machine_action = "run_platform_probe_or_use_structured_api"
|
|
elif eligible_count:
|
|
next_machine_action = "run_ocr_vlm_replay_then_promotion_gate"
|
|
else:
|
|
next_machine_action = "refresh_pixelrag_visual_capture_receipt"
|
|
|
|
return {
|
|
"success": invalid_count == 0,
|
|
"policy": POLICY,
|
|
"status": status,
|
|
"generated_at": now.isoformat(),
|
|
"artifact_root": str(root),
|
|
"platform_probe_worker_receipt_root": str(worker_root),
|
|
"max_age_hours": max_age,
|
|
"platform_filter": list(platforms or []),
|
|
"summary": summary,
|
|
"candidates": all_candidates,
|
|
"replay_contract": {
|
|
"internal_rag_target": INTERNAL_RAG_TARGET,
|
|
"writes_database": False,
|
|
"writes_ai_insights": False,
|
|
"writes_price_tables": False,
|
|
"network_call": False,
|
|
"secret_read": False,
|
|
"promotion_gate_required_before_knowledge_write": True,
|
|
"ocr_or_vlm_required_before_field_extraction": True,
|
|
"blocked_pages_are_not_product_data": True,
|
|
"platform_probe_worker_receipts_are_source_contract_candidates": True,
|
|
},
|
|
"next_machine_action": next_machine_action,
|
|
"controlled_apply": {
|
|
"network_call": False,
|
|
"db_write": False,
|
|
"secret_read": False,
|
|
"production_price_write": False,
|
|
"artifact_write": False,
|
|
},
|
|
}
|
|
|
|
|
|
__all__ = [
|
|
"INTERNAL_RAG_TARGET",
|
|
"POLICY",
|
|
"build_pixelrag_rag_candidate_replay_readback",
|
|
]
|