Files
ewoooc/services/pixelrag_platform_probe_worker_service.py

602 lines
22 KiB
Python

"""Controlled PixelRAG platform probe worker.
This worker executes the machine actions emitted by platform-probe readiness:
public empty-context visual capture for recoverable marketplace interstitials,
or structured-source/backoff receipts for hard blocked platforms. It never
reads sessions/cookies, logs in, writes DB rows, or promotes price truth.
"""
from __future__ import annotations
import json
import os
import re
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Mapping
from services.pixelrag_crawler_integration_service import (
DEFAULT_ARTIFACT_MAX_AGE_HOURS,
DEFAULT_ARTIFACT_ROOT,
)
from services.pixelrag_platform_probe_service import (
build_pixelrag_platform_probe_readiness,
)
POLICY = "controlled_pixelrag_platform_probe_worker_v1"
DEFAULT_LIMIT = 25
DEFAULT_TIMEOUT_SECONDS = 30
DEFAULT_SETTLE_MS = 800
DEFAULT_MAX_TILES = 12
DEFAULT_WORKER_RECEIPT_ROOT = os.getenv(
"PIXELRAG_PLATFORM_PROBE_WORKER_RECEIPT_ROOT",
"/app/data/ai_automation/pixelrag_platform_probe_worker_receipts"
if Path("/app/data").exists()
else "runtime_artifacts/pixelrag_platform_probe_worker_receipts",
)
DEFAULT_CAPTURE_OUTPUT_ROOT = os.getenv(
"PIXELRAG_PLATFORM_PROBE_CAPTURE_OUTPUT_ROOT",
str(DEFAULT_ARTIFACT_ROOT),
)
CAPTURE_READY_STATUSES = {
"ready_for_public_context_probe",
"ready_for_platform_probe",
}
def _normalise_platforms(
platform: str | tuple[str, ...] | list[str] | None,
) -> tuple[str, ...]:
if isinstance(platform, str):
value = platform.strip().lower()
return (value,) if value else ()
return tuple(
str(item or "").strip().lower()
for item in (platform or ())
if str(item or "").strip()
)
def _safe_segment(value: Any) -> str:
text = str(value or "unknown").strip().lower()
text = re.sub(r"[^a-z0-9._-]+", "-", text)
return text.strip("-") or "unknown"
def _capture_script_path() -> Path:
return Path(__file__).resolve().parents[1] / "scripts" / "ops" / "capture_pixelrag_visual_evidence.py"
def _base_item(item: Mapping[str, Any]) -> dict[str, Any]:
fallback = item.get("structured_source_fallback")
if not isinstance(fallback, Mapping):
fallback = {}
return {
"platform": item.get("platform"),
"manifest_id": item.get("manifest_id"),
"source_type": item.get("source_type"),
"source_receipt_path": item.get("source_receipt_path"),
"source_capture_receipt_path": item.get("source_capture_receipt_path"),
"probe_status": item.get("probe_status"),
"barrier_type": item.get("barrier_type"),
"url": item.get("url"),
"title": item.get("title"),
"structured_source_fallback_available": bool(fallback.get("available")),
"structured_source_count": len(list(fallback.get("sources") or [])),
"writes_database": False,
"model_call_performed": False,
"primary_human_gate_count": 0,
}
def _write_worker_receipt(
*,
output_root: Path,
item: Mapping[str, Any],
worker_item: Mapping[str, Any],
) -> str:
target = (
output_root
/ _safe_segment(item.get("platform"))
/ _safe_segment(item.get("manifest_id"))
/ "platform_probe_worker_receipt.json"
)
target.parent.mkdir(parents=True, exist_ok=True)
receipt = dict(worker_item)
receipt["artifact_write_performed"] = True
receipt["receipt_path"] = str(target)
target.write_text(
json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True),
encoding="utf-8",
)
return str(target)
def _dry_run_item(item: Mapping[str, Any]) -> dict[str, Any]:
base = _base_item(item)
probe_status = str(item.get("probe_status") or "")
manifest = item.get("capture_manifest_preview")
capture_ready = (
probe_status in CAPTURE_READY_STATUSES
and isinstance(manifest, Mapping)
and bool(manifest.get("success"))
)
if capture_ready:
base.update({
"worker_status": "dry_run_ready_for_public_context_capture",
"ready_for_execution": True,
"capture_execution_ready": True,
"capture_manifest_id": manifest.get("manifest_id"),
"public_browser_context_policy": (
(manifest.get("public_browser_context") or {}).get("context_policy")
),
"network_call_performed": False,
"artifact_write_performed": False,
"next_machine_action": "run_pixelrag_platform_probe_worker_execute",
})
return base
base.update({
"worker_status": "dry_run_structured_source_fallback_ready",
"ready_for_execution": True,
"capture_execution_ready": False,
"network_call_performed": False,
"artifact_write_performed": False,
"structured_source_fallback": item.get("structured_source_fallback") or {},
"next_machine_action": "run_structured_source_or_platform_backoff_policy",
})
return base
def _skipped_item(item: Mapping[str, Any]) -> dict[str, Any]:
base = _base_item(item)
base.update({
"worker_status": "skipped_not_probe_ready",
"ready_for_execution": False,
"capture_execution_ready": False,
"network_call_performed": False,
"artifact_write_performed": False,
"next_machine_action": item.get("next_machine_action")
or "refresh_pixelrag_platform_probe_readiness",
})
return base
def _structured_source_item(
item: Mapping[str, Any],
*,
output_root: Path,
execute: bool,
write_receipt: bool,
) -> dict[str, Any]:
base = _base_item(item)
fallback = item.get("structured_source_fallback") or {}
base.update({
"worker_status": (
"executed_structured_source_fallback_package"
if execute
else "dry_run_structured_source_fallback_ready"
),
"ready_for_execution": True,
"capture_execution_ready": False,
"network_call_performed": False,
"artifact_write_performed": False,
"structured_source_fallback": fallback,
"structured_source_package": {
"adapter_code": fallback.get("adapter_code"),
"available": bool(fallback.get("available")),
"source_count": len(list(fallback.get("sources") or [])),
"network_request_allowed": bool(fallback.get("network_request_allowed")),
"database_write_allowed": False,
"dry_run_only": True,
"blocked_page_not_product_data": True,
},
"next_machine_action": "run_structured_source_or_platform_backoff_policy",
})
if execute and write_receipt:
base["receipt_path"] = _write_worker_receipt(
output_root=output_root,
item=item,
worker_item=base,
)
base["artifact_write_performed"] = True
return base
def _run_capture_subprocess(
*,
manifest: Mapping[str, Any],
capture_output_root: Path,
timeout_seconds: int,
settle_ms: int,
max_tiles: int,
capture_script: Path | None = None,
) -> dict[str, Any]:
script = capture_script or _capture_script_path()
completed = subprocess.run(
[
sys.executable,
str(script),
"--manifest-json",
json.dumps(dict(manifest), ensure_ascii=False),
"--output-dir",
str(capture_output_root),
"--timeout",
str(max(1, int(timeout_seconds or DEFAULT_TIMEOUT_SECONDS))),
"--settle-ms",
str(max(0, int(settle_ms or DEFAULT_SETTLE_MS))),
"--max-tiles",
str(max(1, int(max_tiles or DEFAULT_MAX_TILES))),
],
capture_output=True,
check=False,
text=True,
)
try:
payload = json.loads(completed.stdout)
except json.JSONDecodeError:
payload = {}
stderr_excerpt = completed.stderr[:500]
if len(completed.stderr) > 500:
stderr_excerpt = f"{completed.stderr[:250]}...{completed.stderr[-250:]}"
return {
"returncode": completed.returncode,
"stdout_json": payload,
"stdout_excerpt": completed.stdout[:500],
"stderr_excerpt": stderr_excerpt,
}
def _capture_runtime_unavailable(result: Mapping[str, Any]) -> bool:
haystack = " ".join(
[
str(result.get("stdout_excerpt") or ""),
str(result.get("stderr_excerpt") or ""),
]
).lower()
return (
"modulenotfounderror" in haystack
and "playwright" in haystack
) or "no module named 'playwright'" in haystack or (
"from playwright.sync_api" in haystack
and "traceback" in haystack
) or (
"playwright" in haystack
and (
"executable doesn't exist" in haystack
or "please run" in haystack
or "browsertype.launch" in haystack
)
)
def _capture_item(
item: Mapping[str, Any],
*,
output_root: Path,
capture_output_root: Path,
timeout_seconds: int,
settle_ms: int,
max_tiles: int,
write_receipt: bool,
capture_script: Path | None,
) -> dict[str, Any]:
base = _base_item(item)
manifest = item.get("capture_manifest_preview")
if not isinstance(manifest, Mapping) or not manifest.get("success"):
base.update({
"worker_status": "skipped_missing_capture_manifest",
"ready_for_execution": False,
"capture_execution_ready": False,
"network_call_performed": False,
"artifact_write_performed": False,
"next_machine_action": "refresh_pixelrag_platform_probe_manifest_or_use_structured_source",
})
if write_receipt:
base["receipt_path"] = _write_worker_receipt(
output_root=output_root,
item=item,
worker_item=base,
)
base["artifact_write_performed"] = True
return base
result = _run_capture_subprocess(
manifest=manifest,
capture_output_root=capture_output_root,
timeout_seconds=timeout_seconds,
settle_ms=settle_ms,
max_tiles=max_tiles,
capture_script=capture_script,
)
capture_payload = result["stdout_json"]
files = list(capture_payload.get("files") or []) if isinstance(capture_payload, Mapping) else []
tile_file_count = sum(1 for file_item in files if file_item.get("kind") == "tile")
receipt_path = ""
planned_output = capture_payload.get("planned_output") if isinstance(capture_payload, Mapping) else {}
if isinstance(planned_output, Mapping):
receipt_path = str(planned_output.get("receipt") or "")
for file_item in files:
if file_item.get("kind") == "receipt":
receipt_path = str(file_item.get("path") or receipt_path)
captured_ok = (
result["returncode"] == 0
and isinstance(capture_payload, Mapping)
and capture_payload.get("success") is True
and capture_payload.get("status") == "captured"
)
runtime_missing = not captured_ok and _capture_runtime_unavailable(result)
structured_fallback_available = bool(
(item.get("structured_source_fallback") or {}).get("available")
)
capture_fallback = not captured_ok and structured_fallback_available
base.update({
"worker_status": (
"executed_capture_ok"
if captured_ok
else (
(
"capture_runtime_unavailable_structured_fallback_package"
if runtime_missing
else "capture_failed_structured_fallback_package"
)
if capture_fallback
else "capture_worker_error"
)
),
"ready_for_execution": True,
"capture_execution_ready": True,
"capture_manifest_id": manifest.get("manifest_id"),
"capture_returncode": result["returncode"],
"capture_status": capture_payload.get("status") if isinstance(capture_payload, Mapping) else "",
"capture_http_status": int(capture_payload.get("http_status") or 0)
if isinstance(capture_payload, Mapping)
else 0,
"capture_tile_file_count": tile_file_count,
"capture_receipt_path": receipt_path,
"network_call_performed": bool(not runtime_missing),
"artifact_write_performed": bool(captured_ok),
"structured_source_fallback": item.get("structured_source_fallback") or {},
"structured_source_package": (
{
"adapter_code": (item.get("structured_source_fallback") or {}).get("adapter_code"),
"available": True,
"source_count": len(list((item.get("structured_source_fallback") or {}).get("sources") or [])),
"network_request_allowed": bool(
(item.get("structured_source_fallback") or {}).get("network_request_allowed")
),
"database_write_allowed": False,
"dry_run_only": True,
"blocked_page_not_product_data": True,
"capture_runtime_unavailable": runtime_missing,
}
if capture_fallback
else None
),
"capture_stdout_excerpt": result["stdout_excerpt"] if not captured_ok else "",
"capture_stderr_excerpt": result["stderr_excerpt"] if not captured_ok else "",
"next_machine_action": (
"run_pixelrag_rag_candidate_replay_after_probe_capture"
if captured_ok
else (
(
"run_structured_source_or_install_pixelrag_capture_runtime"
if runtime_missing
else "run_structured_source_or_retry_platform_capture"
)
if capture_fallback
else "repair_pixelrag_platform_probe_worker_capture_runtime_or_use_structured_source"
)
),
})
if write_receipt:
base["receipt_path"] = _write_worker_receipt(
output_root=output_root,
item=item,
worker_item=base,
)
base["artifact_write_performed"] = True
return base
def run_pixelrag_platform_probe_worker(
*,
artifact_root: str | Path | None = None,
vlm_receipt_root: str | Path | None = None,
output_root: str | Path | None = None,
capture_output_root: str | Path | None = None,
platform: str | tuple[str, ...] | list[str] | None = None,
max_age_hours: int | None = None,
limit: int | None = None,
timeout: int | None = None,
settle_ms: int | None = None,
max_tiles: int | None = None,
execute: bool = False,
write_receipt: bool = False,
capture_script: str | Path | None = None,
) -> dict[str, Any]:
"""Run or dry-run public platform probe machine actions."""
output = Path(output_root or DEFAULT_WORKER_RECEIPT_ROOT)
capture_output = Path(capture_output_root or DEFAULT_CAPTURE_OUTPUT_ROOT)
platforms = _normalise_platforms(platform)
max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS))
item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250))
timeout_seconds = max(1, min(int(timeout or DEFAULT_TIMEOUT_SECONDS), 120))
settle = max(0, min(int(settle_ms or DEFAULT_SETTLE_MS), 5000))
tiles = max(1, min(int(max_tiles or DEFAULT_MAX_TILES), 80))
generated_at = datetime.now(timezone.utc).isoformat()
capture_script_path = Path(capture_script) if capture_script else None
readiness_artifact_root = artifact_root or DEFAULT_ARTIFACT_ROOT
readiness = build_pixelrag_platform_probe_readiness(
artifact_root=readiness_artifact_root,
vlm_receipt_root=vlm_receipt_root,
platform=platforms,
max_age_hours=max_age,
limit=item_limit,
)
probe_items = list(readiness.get("probe_items") or [])
worker_items: list[dict[str, Any]] = []
for item in probe_items:
if not item.get("probe_ready"):
worker_items.append(_skipped_item(item))
continue
probe_status = str(item.get("probe_status") or "")
manifest = item.get("capture_manifest_preview")
can_capture = (
probe_status in CAPTURE_READY_STATUSES
and isinstance(manifest, Mapping)
and bool(manifest.get("success"))
)
if not execute:
worker_items.append(_dry_run_item(item))
continue
if can_capture:
worker_items.append(_capture_item(
item,
output_root=output,
capture_output_root=capture_output,
timeout_seconds=timeout_seconds,
settle_ms=settle,
max_tiles=tiles,
write_receipt=write_receipt,
capture_script=capture_script_path,
))
else:
worker_items.append(_structured_source_item(
item,
output_root=output,
execute=True,
write_receipt=write_receipt,
))
ready_count = sum(1 for item in probe_items if item.get("probe_ready"))
dry_run_count = sum(1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_"))
structured_count = sum(
1 for item in worker_items if "structured" in str(item.get("worker_status") or "")
)
capture_ready_count = sum(1 for item in worker_items if item.get("capture_execution_ready"))
capture_execute_count = sum(
1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_capture")
)
capture_ok_count = sum(1 for item in worker_items if item.get("worker_status") == "executed_capture_ok")
capture_error_count = sum(1 for item in worker_items if item.get("worker_status") == "capture_worker_error")
executed_structured_count = sum(
1
for item in worker_items
if "structured" in str(item.get("worker_status") or "")
and not str(item.get("worker_status") or "").startswith("dry_run_")
)
executed_count = capture_execute_count + executed_structured_count
skipped_count = sum(1 for item in worker_items if str(item.get("worker_status") or "").startswith("skipped_"))
receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path"))
network_call_performed = any(bool(item.get("network_call_performed")) for item in worker_items)
artifact_write_performed = any(bool(item.get("artifact_write_performed")) for item in worker_items)
if capture_error_count:
status = "critical" if execute and capture_ok_count == 0 and capture_ready_count else "warning"
elif probe_items and ready_count:
status = "ok"
elif probe_items:
status = "warning"
else:
status = "warning"
if not probe_items:
next_action = "run_pixelrag_visual_capture_worker"
elif capture_error_count:
next_action = "repair_pixelrag_platform_probe_worker_capture_runtime_or_use_structured_source"
elif not execute and capture_ready_count:
next_action = "run_pixelrag_platform_probe_worker_execute"
elif execute and capture_ok_count:
next_action = "run_pixelrag_rag_candidate_replay_after_probe_capture"
elif structured_count:
next_action = "run_structured_source_or_platform_backoff_policy"
else:
next_action = "refresh_pixelrag_platform_probe_readiness"
summary = {
"probe_candidate_count": len(probe_items),
"ready_count": ready_count,
"skipped_count": skipped_count,
"dry_run_count": dry_run_count,
"capture_ready_count": capture_ready_count,
"capture_execute_count": capture_execute_count,
"capture_ok_count": capture_ok_count,
"capture_error_count": capture_error_count,
"structured_fallback_count": structured_count,
"executed_structured_count": executed_structured_count,
"executed_count": executed_count,
"receipt_written_count": receipt_written_count,
"network_call_performed": network_call_performed,
"artifact_write_performed": artifact_write_performed,
"writes_database_count": 0,
"primary_human_gate_count": 0,
"platforms": sorted({str(item.get("platform") or "unknown") for item in probe_items}),
}
return {
"success": status != "critical",
"policy": POLICY,
"status": status,
"generated_at": generated_at,
"artifact_root": str(readiness_artifact_root),
"vlm_receipt_root": str(
vlm_receipt_root
or readiness.get("vlm_receipt_root")
or ""
),
"output_root": str(output),
"capture_output_root": str(capture_output),
"platform_filter": list(platforms),
"max_age_hours": max_age,
"limit": item_limit,
"timeout_seconds": timeout_seconds,
"settle_ms": settle,
"max_tiles": tiles,
"execute": bool(execute),
"write_receipt": bool(write_receipt),
"summary": summary,
"worker_items": worker_items,
"source_readiness": {
"policy": readiness.get("policy"),
"status": readiness.get("status"),
"summary": readiness.get("summary"),
"next_machine_action": readiness.get("next_machine_action"),
},
"controlled_apply": {
"network_call": bool(execute and network_call_performed),
"model_call": False,
"artifact_write": artifact_write_performed,
"db_write": False,
"writes_database": False,
"writes_database_count": 0,
"secret_read": False,
"raw_cookie_or_session_read": False,
"credentialed_session_allowed": False,
"login_allowed": False,
"production_price_write": False,
"primary_human_gate_count": 0,
},
"promotion_boundary": {
"writes_ai_insights": False,
"writes_price_tables": False,
"blocked_pages_are_not_product_data": True,
"visual_fields_are_candidate_evidence_only": True,
"requires_rag_candidate_replay_after_capture": True,
"requires_identity_matcher_replay": True,
"requires_promotion_gate": True,
},
"next_machine_action": next_action,
}
__all__ = [
"DEFAULT_CAPTURE_OUTPUT_ROOT",
"DEFAULT_WORKER_RECEIPT_ROOT",
"POLICY",
"run_pixelrag_platform_probe_worker",
]