531 lines
20 KiB
Python
531 lines
20 KiB
Python
"""Controlled PixelRAG platform probe worker.
|
|
|
|
This worker executes the machine actions emitted by platform-probe readiness:
|
|
public empty-context visual capture for recoverable marketplace interstitials,
|
|
or structured-source/backoff receipts for hard blocked platforms. It never
|
|
reads sessions/cookies, logs in, writes DB rows, or promotes price truth.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Mapping
|
|
|
|
from services.pixelrag_crawler_integration_service import (
|
|
DEFAULT_ARTIFACT_MAX_AGE_HOURS,
|
|
DEFAULT_ARTIFACT_ROOT,
|
|
)
|
|
from services.pixelrag_platform_probe_service import (
|
|
build_pixelrag_platform_probe_readiness,
|
|
)
|
|
|
|
|
|
POLICY = "controlled_pixelrag_platform_probe_worker_v1"
|
|
DEFAULT_LIMIT = 25
|
|
DEFAULT_TIMEOUT_SECONDS = 30
|
|
DEFAULT_SETTLE_MS = 800
|
|
DEFAULT_MAX_TILES = 12
|
|
DEFAULT_WORKER_RECEIPT_ROOT = os.getenv(
|
|
"PIXELRAG_PLATFORM_PROBE_WORKER_RECEIPT_ROOT",
|
|
"/app/data/ai_automation/pixelrag_platform_probe_worker_receipts"
|
|
if Path("/app/data").exists()
|
|
else "runtime_artifacts/pixelrag_platform_probe_worker_receipts",
|
|
)
|
|
DEFAULT_CAPTURE_OUTPUT_ROOT = os.getenv(
|
|
"PIXELRAG_PLATFORM_PROBE_CAPTURE_OUTPUT_ROOT",
|
|
str(DEFAULT_ARTIFACT_ROOT),
|
|
)
|
|
CAPTURE_READY_STATUSES = {
|
|
"ready_for_public_context_probe",
|
|
"ready_for_platform_probe",
|
|
}
|
|
|
|
|
|
def _normalise_platforms(
|
|
platform: str | tuple[str, ...] | list[str] | None,
|
|
) -> tuple[str, ...]:
|
|
if isinstance(platform, str):
|
|
value = platform.strip().lower()
|
|
return (value,) if value else ()
|
|
return tuple(
|
|
str(item or "").strip().lower()
|
|
for item in (platform or ())
|
|
if str(item or "").strip()
|
|
)
|
|
|
|
|
|
def _safe_segment(value: Any) -> str:
|
|
text = str(value or "unknown").strip().lower()
|
|
text = re.sub(r"[^a-z0-9._-]+", "-", text)
|
|
return text.strip("-") or "unknown"
|
|
|
|
|
|
def _capture_script_path() -> Path:
|
|
return Path(__file__).resolve().parents[1] / "scripts" / "ops" / "capture_pixelrag_visual_evidence.py"
|
|
|
|
|
|
def _base_item(item: Mapping[str, Any]) -> dict[str, Any]:
|
|
fallback = item.get("structured_source_fallback")
|
|
if not isinstance(fallback, Mapping):
|
|
fallback = {}
|
|
return {
|
|
"platform": item.get("platform"),
|
|
"manifest_id": item.get("manifest_id"),
|
|
"source_type": item.get("source_type"),
|
|
"source_receipt_path": item.get("source_receipt_path"),
|
|
"source_capture_receipt_path": item.get("source_capture_receipt_path"),
|
|
"probe_status": item.get("probe_status"),
|
|
"barrier_type": item.get("barrier_type"),
|
|
"url": item.get("url"),
|
|
"title": item.get("title"),
|
|
"structured_source_fallback_available": bool(fallback.get("available")),
|
|
"structured_source_count": len(list(fallback.get("sources") or [])),
|
|
"writes_database": False,
|
|
"model_call_performed": False,
|
|
"primary_human_gate_count": 0,
|
|
}
|
|
|
|
|
|
def _write_worker_receipt(
|
|
*,
|
|
output_root: Path,
|
|
item: Mapping[str, Any],
|
|
worker_item: Mapping[str, Any],
|
|
) -> str:
|
|
target = (
|
|
output_root
|
|
/ _safe_segment(item.get("platform"))
|
|
/ _safe_segment(item.get("manifest_id"))
|
|
/ "platform_probe_worker_receipt.json"
|
|
)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
receipt = dict(worker_item)
|
|
receipt["artifact_write_performed"] = True
|
|
receipt["receipt_path"] = str(target)
|
|
target.write_text(
|
|
json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True),
|
|
encoding="utf-8",
|
|
)
|
|
return str(target)
|
|
|
|
|
|
def _dry_run_item(item: Mapping[str, Any]) -> dict[str, Any]:
|
|
base = _base_item(item)
|
|
probe_status = str(item.get("probe_status") or "")
|
|
manifest = item.get("capture_manifest_preview")
|
|
capture_ready = (
|
|
probe_status in CAPTURE_READY_STATUSES
|
|
and isinstance(manifest, Mapping)
|
|
and bool(manifest.get("success"))
|
|
)
|
|
if capture_ready:
|
|
base.update({
|
|
"worker_status": "dry_run_ready_for_public_context_capture",
|
|
"ready_for_execution": True,
|
|
"capture_execution_ready": True,
|
|
"capture_manifest_id": manifest.get("manifest_id"),
|
|
"public_browser_context_policy": (
|
|
(manifest.get("public_browser_context") or {}).get("context_policy")
|
|
),
|
|
"network_call_performed": False,
|
|
"artifact_write_performed": False,
|
|
"next_machine_action": "run_pixelrag_platform_probe_worker_execute",
|
|
})
|
|
return base
|
|
base.update({
|
|
"worker_status": "dry_run_structured_source_fallback_ready",
|
|
"ready_for_execution": True,
|
|
"capture_execution_ready": False,
|
|
"network_call_performed": False,
|
|
"artifact_write_performed": False,
|
|
"structured_source_fallback": item.get("structured_source_fallback") or {},
|
|
"next_machine_action": "run_structured_source_or_platform_backoff_policy",
|
|
})
|
|
return base
|
|
|
|
|
|
def _skipped_item(item: Mapping[str, Any]) -> dict[str, Any]:
|
|
base = _base_item(item)
|
|
base.update({
|
|
"worker_status": "skipped_not_probe_ready",
|
|
"ready_for_execution": False,
|
|
"capture_execution_ready": False,
|
|
"network_call_performed": False,
|
|
"artifact_write_performed": False,
|
|
"next_machine_action": item.get("next_machine_action")
|
|
or "refresh_pixelrag_platform_probe_readiness",
|
|
})
|
|
return base
|
|
|
|
|
|
def _structured_source_item(
|
|
item: Mapping[str, Any],
|
|
*,
|
|
output_root: Path,
|
|
execute: bool,
|
|
write_receipt: bool,
|
|
) -> dict[str, Any]:
|
|
base = _base_item(item)
|
|
fallback = item.get("structured_source_fallback") or {}
|
|
base.update({
|
|
"worker_status": (
|
|
"executed_structured_source_fallback_package"
|
|
if execute
|
|
else "dry_run_structured_source_fallback_ready"
|
|
),
|
|
"ready_for_execution": True,
|
|
"capture_execution_ready": False,
|
|
"network_call_performed": False,
|
|
"artifact_write_performed": False,
|
|
"structured_source_fallback": fallback,
|
|
"structured_source_package": {
|
|
"adapter_code": fallback.get("adapter_code"),
|
|
"available": bool(fallback.get("available")),
|
|
"source_count": len(list(fallback.get("sources") or [])),
|
|
"network_request_allowed": bool(fallback.get("network_request_allowed")),
|
|
"database_write_allowed": False,
|
|
"dry_run_only": True,
|
|
"blocked_page_not_product_data": True,
|
|
},
|
|
"next_machine_action": "run_structured_source_or_platform_backoff_policy",
|
|
})
|
|
if execute and write_receipt:
|
|
base["receipt_path"] = _write_worker_receipt(
|
|
output_root=output_root,
|
|
item=item,
|
|
worker_item=base,
|
|
)
|
|
base["artifact_write_performed"] = True
|
|
return base
|
|
|
|
|
|
def _run_capture_subprocess(
|
|
*,
|
|
manifest: Mapping[str, Any],
|
|
capture_output_root: Path,
|
|
timeout_seconds: int,
|
|
settle_ms: int,
|
|
max_tiles: int,
|
|
capture_script: Path | None = None,
|
|
) -> dict[str, Any]:
|
|
script = capture_script or _capture_script_path()
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(script),
|
|
"--manifest-json",
|
|
json.dumps(dict(manifest), ensure_ascii=False),
|
|
"--output-dir",
|
|
str(capture_output_root),
|
|
"--timeout",
|
|
str(max(1, int(timeout_seconds or DEFAULT_TIMEOUT_SECONDS))),
|
|
"--settle-ms",
|
|
str(max(0, int(settle_ms or DEFAULT_SETTLE_MS))),
|
|
"--max-tiles",
|
|
str(max(1, int(max_tiles or DEFAULT_MAX_TILES))),
|
|
],
|
|
capture_output=True,
|
|
check=False,
|
|
text=True,
|
|
)
|
|
try:
|
|
payload = json.loads(completed.stdout)
|
|
except json.JSONDecodeError:
|
|
payload = {}
|
|
return {
|
|
"returncode": completed.returncode,
|
|
"stdout_json": payload,
|
|
"stdout_excerpt": completed.stdout[:500],
|
|
"stderr_excerpt": completed.stderr[:500],
|
|
}
|
|
|
|
|
|
def _capture_item(
|
|
item: Mapping[str, Any],
|
|
*,
|
|
output_root: Path,
|
|
capture_output_root: Path,
|
|
timeout_seconds: int,
|
|
settle_ms: int,
|
|
max_tiles: int,
|
|
write_receipt: bool,
|
|
capture_script: Path | None,
|
|
) -> dict[str, Any]:
|
|
base = _base_item(item)
|
|
manifest = item.get("capture_manifest_preview")
|
|
if not isinstance(manifest, Mapping) or not manifest.get("success"):
|
|
base.update({
|
|
"worker_status": "skipped_missing_capture_manifest",
|
|
"ready_for_execution": False,
|
|
"capture_execution_ready": False,
|
|
"network_call_performed": False,
|
|
"artifact_write_performed": False,
|
|
"next_machine_action": "refresh_pixelrag_platform_probe_manifest_or_use_structured_source",
|
|
})
|
|
if write_receipt:
|
|
base["receipt_path"] = _write_worker_receipt(
|
|
output_root=output_root,
|
|
item=item,
|
|
worker_item=base,
|
|
)
|
|
base["artifact_write_performed"] = True
|
|
return base
|
|
|
|
result = _run_capture_subprocess(
|
|
manifest=manifest,
|
|
capture_output_root=capture_output_root,
|
|
timeout_seconds=timeout_seconds,
|
|
settle_ms=settle_ms,
|
|
max_tiles=max_tiles,
|
|
capture_script=capture_script,
|
|
)
|
|
capture_payload = result["stdout_json"]
|
|
files = list(capture_payload.get("files") or []) if isinstance(capture_payload, Mapping) else []
|
|
tile_file_count = sum(1 for file_item in files if file_item.get("kind") == "tile")
|
|
receipt_path = ""
|
|
planned_output = capture_payload.get("planned_output") if isinstance(capture_payload, Mapping) else {}
|
|
if isinstance(planned_output, Mapping):
|
|
receipt_path = str(planned_output.get("receipt") or "")
|
|
for file_item in files:
|
|
if file_item.get("kind") == "receipt":
|
|
receipt_path = str(file_item.get("path") or receipt_path)
|
|
|
|
captured_ok = (
|
|
result["returncode"] == 0
|
|
and isinstance(capture_payload, Mapping)
|
|
and capture_payload.get("success") is True
|
|
and capture_payload.get("status") == "captured"
|
|
)
|
|
base.update({
|
|
"worker_status": "executed_capture_ok" if captured_ok else "capture_worker_error",
|
|
"ready_for_execution": True,
|
|
"capture_execution_ready": True,
|
|
"capture_manifest_id": manifest.get("manifest_id"),
|
|
"capture_returncode": result["returncode"],
|
|
"capture_status": capture_payload.get("status") if isinstance(capture_payload, Mapping) else "",
|
|
"capture_http_status": int(capture_payload.get("http_status") or 0)
|
|
if isinstance(capture_payload, Mapping)
|
|
else 0,
|
|
"capture_tile_file_count": tile_file_count,
|
|
"capture_receipt_path": receipt_path,
|
|
"network_call_performed": True,
|
|
"artifact_write_performed": bool(captured_ok),
|
|
"capture_stdout_excerpt": result["stdout_excerpt"] if not captured_ok else "",
|
|
"capture_stderr_excerpt": result["stderr_excerpt"] if not captured_ok else "",
|
|
"next_machine_action": (
|
|
"run_pixelrag_rag_candidate_replay_after_probe_capture"
|
|
if captured_ok
|
|
else "repair_pixelrag_platform_probe_worker_capture_runtime_or_use_structured_source"
|
|
),
|
|
})
|
|
if write_receipt:
|
|
base["receipt_path"] = _write_worker_receipt(
|
|
output_root=output_root,
|
|
item=item,
|
|
worker_item=base,
|
|
)
|
|
base["artifact_write_performed"] = True
|
|
return base
|
|
|
|
|
|
def run_pixelrag_platform_probe_worker(
|
|
*,
|
|
artifact_root: str | Path | None = None,
|
|
vlm_receipt_root: str | Path | None = None,
|
|
output_root: str | Path | None = None,
|
|
capture_output_root: str | Path | None = None,
|
|
platform: str | tuple[str, ...] | list[str] | None = None,
|
|
max_age_hours: int | None = None,
|
|
limit: int | None = None,
|
|
timeout: int | None = None,
|
|
settle_ms: int | None = None,
|
|
max_tiles: int | None = None,
|
|
execute: bool = False,
|
|
write_receipt: bool = False,
|
|
capture_script: str | Path | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Run or dry-run public platform probe machine actions."""
|
|
output = Path(output_root or DEFAULT_WORKER_RECEIPT_ROOT)
|
|
capture_output = Path(capture_output_root or DEFAULT_CAPTURE_OUTPUT_ROOT)
|
|
platforms = _normalise_platforms(platform)
|
|
max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS))
|
|
item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250))
|
|
timeout_seconds = max(1, min(int(timeout or DEFAULT_TIMEOUT_SECONDS), 120))
|
|
settle = max(0, min(int(settle_ms or DEFAULT_SETTLE_MS), 5000))
|
|
tiles = max(1, min(int(max_tiles or DEFAULT_MAX_TILES), 80))
|
|
generated_at = datetime.now(timezone.utc).isoformat()
|
|
capture_script_path = Path(capture_script) if capture_script else None
|
|
readiness_artifact_root = artifact_root or DEFAULT_ARTIFACT_ROOT
|
|
|
|
readiness = build_pixelrag_platform_probe_readiness(
|
|
artifact_root=readiness_artifact_root,
|
|
vlm_receipt_root=vlm_receipt_root,
|
|
platform=platforms,
|
|
max_age_hours=max_age,
|
|
limit=item_limit,
|
|
)
|
|
probe_items = list(readiness.get("probe_items") or [])
|
|
worker_items: list[dict[str, Any]] = []
|
|
for item in probe_items:
|
|
if not item.get("probe_ready"):
|
|
worker_items.append(_skipped_item(item))
|
|
continue
|
|
probe_status = str(item.get("probe_status") or "")
|
|
manifest = item.get("capture_manifest_preview")
|
|
can_capture = (
|
|
probe_status in CAPTURE_READY_STATUSES
|
|
and isinstance(manifest, Mapping)
|
|
and bool(manifest.get("success"))
|
|
)
|
|
if not execute:
|
|
worker_items.append(_dry_run_item(item))
|
|
continue
|
|
if can_capture:
|
|
worker_items.append(_capture_item(
|
|
item,
|
|
output_root=output,
|
|
capture_output_root=capture_output,
|
|
timeout_seconds=timeout_seconds,
|
|
settle_ms=settle,
|
|
max_tiles=tiles,
|
|
write_receipt=write_receipt,
|
|
capture_script=capture_script_path,
|
|
))
|
|
else:
|
|
worker_items.append(_structured_source_item(
|
|
item,
|
|
output_root=output,
|
|
execute=True,
|
|
write_receipt=write_receipt,
|
|
))
|
|
|
|
ready_count = sum(1 for item in probe_items if item.get("probe_ready"))
|
|
dry_run_count = sum(1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_"))
|
|
structured_count = sum(
|
|
1 for item in worker_items if "structured_source_fallback" in str(item.get("worker_status") or "")
|
|
)
|
|
capture_ready_count = sum(1 for item in worker_items if item.get("capture_execution_ready"))
|
|
capture_execute_count = sum(
|
|
1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_capture")
|
|
)
|
|
capture_ok_count = sum(1 for item in worker_items if item.get("worker_status") == "executed_capture_ok")
|
|
capture_error_count = sum(1 for item in worker_items if item.get("worker_status") == "capture_worker_error")
|
|
executed_structured_count = sum(
|
|
1 for item in worker_items if item.get("worker_status") == "executed_structured_source_fallback_package"
|
|
)
|
|
executed_count = capture_execute_count + executed_structured_count
|
|
skipped_count = sum(1 for item in worker_items if str(item.get("worker_status") or "").startswith("skipped_"))
|
|
receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path"))
|
|
network_call_performed = any(bool(item.get("network_call_performed")) for item in worker_items)
|
|
artifact_write_performed = any(bool(item.get("artifact_write_performed")) for item in worker_items)
|
|
|
|
if capture_error_count:
|
|
status = "critical" if execute and capture_ok_count == 0 and capture_ready_count else "warning"
|
|
elif probe_items and ready_count:
|
|
status = "ok"
|
|
elif probe_items:
|
|
status = "warning"
|
|
else:
|
|
status = "warning"
|
|
|
|
if not probe_items:
|
|
next_action = "run_pixelrag_visual_capture_worker"
|
|
elif capture_error_count:
|
|
next_action = "repair_pixelrag_platform_probe_worker_capture_runtime_or_use_structured_source"
|
|
elif not execute and capture_ready_count:
|
|
next_action = "run_pixelrag_platform_probe_worker_execute"
|
|
elif execute and capture_ok_count:
|
|
next_action = "run_pixelrag_rag_candidate_replay_after_probe_capture"
|
|
elif structured_count:
|
|
next_action = "run_structured_source_or_platform_backoff_policy"
|
|
else:
|
|
next_action = "refresh_pixelrag_platform_probe_readiness"
|
|
|
|
summary = {
|
|
"probe_candidate_count": len(probe_items),
|
|
"ready_count": ready_count,
|
|
"skipped_count": skipped_count,
|
|
"dry_run_count": dry_run_count,
|
|
"capture_ready_count": capture_ready_count,
|
|
"capture_execute_count": capture_execute_count,
|
|
"capture_ok_count": capture_ok_count,
|
|
"capture_error_count": capture_error_count,
|
|
"structured_fallback_count": structured_count,
|
|
"executed_structured_count": executed_structured_count,
|
|
"executed_count": executed_count,
|
|
"receipt_written_count": receipt_written_count,
|
|
"network_call_performed": network_call_performed,
|
|
"artifact_write_performed": artifact_write_performed,
|
|
"writes_database_count": 0,
|
|
"primary_human_gate_count": 0,
|
|
"platforms": sorted({str(item.get("platform") or "unknown") for item in probe_items}),
|
|
}
|
|
return {
|
|
"success": status != "critical",
|
|
"policy": POLICY,
|
|
"status": status,
|
|
"generated_at": generated_at,
|
|
"artifact_root": str(readiness_artifact_root),
|
|
"vlm_receipt_root": str(
|
|
vlm_receipt_root
|
|
or readiness.get("vlm_receipt_root")
|
|
or ""
|
|
),
|
|
"output_root": str(output),
|
|
"capture_output_root": str(capture_output),
|
|
"platform_filter": list(platforms),
|
|
"max_age_hours": max_age,
|
|
"limit": item_limit,
|
|
"timeout_seconds": timeout_seconds,
|
|
"settle_ms": settle,
|
|
"max_tiles": tiles,
|
|
"execute": bool(execute),
|
|
"write_receipt": bool(write_receipt),
|
|
"summary": summary,
|
|
"worker_items": worker_items,
|
|
"source_readiness": {
|
|
"policy": readiness.get("policy"),
|
|
"status": readiness.get("status"),
|
|
"summary": readiness.get("summary"),
|
|
"next_machine_action": readiness.get("next_machine_action"),
|
|
},
|
|
"controlled_apply": {
|
|
"network_call": bool(execute and network_call_performed),
|
|
"model_call": False,
|
|
"artifact_write": artifact_write_performed,
|
|
"db_write": False,
|
|
"writes_database": False,
|
|
"writes_database_count": 0,
|
|
"secret_read": False,
|
|
"raw_cookie_or_session_read": False,
|
|
"credentialed_session_allowed": False,
|
|
"login_allowed": False,
|
|
"production_price_write": False,
|
|
"primary_human_gate_count": 0,
|
|
},
|
|
"promotion_boundary": {
|
|
"writes_ai_insights": False,
|
|
"writes_price_tables": False,
|
|
"blocked_pages_are_not_product_data": True,
|
|
"visual_fields_are_candidate_evidence_only": True,
|
|
"requires_rag_candidate_replay_after_capture": True,
|
|
"requires_identity_matcher_replay": True,
|
|
"requires_promotion_gate": True,
|
|
},
|
|
"next_machine_action": next_action,
|
|
}
|
|
|
|
|
|
__all__ = [
|
|
"DEFAULT_CAPTURE_OUTPUT_ROOT",
|
|
"DEFAULT_WORKER_RECEIPT_ROOT",
|
|
"POLICY",
|
|
"run_pixelrag_platform_probe_worker",
|
|
]
|