566 lines
21 KiB
Python
566 lines
21 KiB
Python
"""Read-only PixelRAG platform probe readiness.
|
|
|
|
This module turns PixelRAG capture/VLM barrier evidence into concrete machine
|
|
actions for public marketplace probing. It does not read sessions/cookies,
|
|
perform network calls, write DB rows, or promote price truth.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Mapping
|
|
from urllib.parse import parse_qs, unquote_plus, urlparse
|
|
|
|
from services.market_intel.adapters.registry import get_adapter
|
|
from services.pixelrag_crawler_integration_service import (
|
|
DEFAULT_ARTIFACT_MAX_AGE_HOURS,
|
|
DEFAULT_ARTIFACT_ROOT,
|
|
build_pixelrag_marketplace_search_manifest,
|
|
_parse_iso_datetime,
|
|
)
|
|
from services.pixelrag_rag_candidate_replay_service import (
|
|
build_pixelrag_rag_candidate_replay_readback,
|
|
)
|
|
|
|
|
|
POLICY = "read_only_pixelrag_platform_probe_readiness_v1"
|
|
DEFAULT_LIMIT = 50
|
|
DEFAULT_ACCEPT_LANGUAGE = "zh-TW,zh-Hant;q=0.95,zh;q=0.9,en;q=0.7"
|
|
DEFAULT_VLM_RECEIPT_ROOT = os.getenv(
|
|
"PIXELRAG_VLM_REPLAY_RECEIPT_ROOT",
|
|
"/app/data/ai_automation/pixelrag_vlm_replay_receipts"
|
|
if Path("/app/data").exists()
|
|
else "runtime_artifacts/pixelrag_vlm_replay_receipts",
|
|
)
|
|
PLATFORM_ADAPTER_ALIASES = {
|
|
"shopee_tw": "shopee",
|
|
"coupang_tw": "coupang",
|
|
"yahoo_shopping_tw": "yahoo",
|
|
}
|
|
|
|
|
|
def _normalise_platforms(
|
|
platform: str | tuple[str, ...] | list[str] | None,
|
|
) -> tuple[str, ...]:
|
|
if isinstance(platform, str):
|
|
value = platform.strip().lower()
|
|
return (value,) if value else ()
|
|
return tuple(
|
|
str(item or "").strip().lower()
|
|
for item in (platform or ())
|
|
if str(item or "").strip()
|
|
)
|
|
|
|
|
|
def _read_json(path: Path) -> tuple[dict[str, Any], str]:
|
|
try:
|
|
parsed = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
return {}, str(exc)[:300]
|
|
return parsed if isinstance(parsed, dict) else {}, ""
|
|
|
|
|
|
def _safe_int(value: Any, default: int = 0) -> int:
|
|
try:
|
|
return int(value or default)
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
def _safe_text(value: Any) -> str:
|
|
if value is None:
|
|
return ""
|
|
if isinstance(value, str):
|
|
return value
|
|
if isinstance(value, Mapping):
|
|
return " ".join(_safe_text(item) for item in value.values())
|
|
if isinstance(value, list):
|
|
return " ".join(_safe_text(item) for item in value)
|
|
return str(value)
|
|
|
|
|
|
def _extract_keyword(url: str) -> str:
|
|
parsed = urlparse(str(url or ""))
|
|
query = parse_qs(parsed.query)
|
|
for key in ("keyword", "q", "query", "p", "search", "searchKeyword"):
|
|
values = query.get(key) or []
|
|
for value in values:
|
|
text = unquote_plus(str(value or "")).strip()
|
|
if text:
|
|
return text
|
|
return ""
|
|
|
|
|
|
def _adapter_code(platform: str) -> str:
|
|
clean = str(platform or "").strip().lower()
|
|
return PLATFORM_ADAPTER_ALIASES.get(clean, clean)
|
|
|
|
|
|
def _structured_source_plan(platform: str) -> dict[str, Any]:
|
|
adapter = get_adapter(_adapter_code(platform))
|
|
if not adapter:
|
|
return {
|
|
"available": False,
|
|
"adapter_code": _adapter_code(platform),
|
|
"network_request_allowed": False,
|
|
"database_write_allowed": False,
|
|
"sources": [],
|
|
}
|
|
plan = adapter.build_discovery_plan()
|
|
return {
|
|
"available": True,
|
|
"adapter_code": plan.get("platform_code"),
|
|
"platform_name": plan.get("platform_name"),
|
|
"base_url": plan.get("base_url"),
|
|
"safety_policy": plan.get("safety_policy"),
|
|
"network_request_allowed": bool(plan.get("network_request_allowed")),
|
|
"database_write_allowed": bool(plan.get("database_write_allowed")),
|
|
"sources": list(plan.get("sources") or []),
|
|
}
|
|
|
|
|
|
def _public_browser_context(platform: str) -> dict[str, Any]:
|
|
return {
|
|
"context_policy": "public_empty_browser_context_no_login",
|
|
"locale": "zh-TW",
|
|
"timezone_id": "Asia/Taipei",
|
|
"viewport": {"name": "desktop-1440", "width": 1440, "height": 950},
|
|
"extra_http_headers": {
|
|
"Accept-Language": DEFAULT_ACCEPT_LANGUAGE,
|
|
},
|
|
"credentialed_session_allowed": False,
|
|
"storage_state_allowed": False,
|
|
"raw_cookie_or_session_read_allowed": False,
|
|
"login_allowed": False,
|
|
"cart_or_checkout_allowed": False,
|
|
"platform": platform,
|
|
}
|
|
|
|
|
|
def _barrier_type_from_signals(
|
|
*,
|
|
platform: str,
|
|
http_status: int,
|
|
visual_barrier_reason: str,
|
|
url: str,
|
|
title: str,
|
|
validation: Mapping[str, Any] | None = None,
|
|
parsed_output: Mapping[str, Any] | None = None,
|
|
) -> str:
|
|
validation = validation or {}
|
|
parsed_output = parsed_output or {}
|
|
haystack = " ".join(
|
|
[
|
|
visual_barrier_reason,
|
|
url,
|
|
title,
|
|
_safe_text(parsed_output.get("notes")),
|
|
_safe_text((parsed_output.get("fields") or {}).get("title")),
|
|
]
|
|
).lower()
|
|
if http_status in {401, 403} or "access denied" in haystack:
|
|
return "access_denied"
|
|
if "verify/traffic" in haystack or "traffic" in haystack:
|
|
return "traffic_verification_interstitial"
|
|
if (
|
|
validation.get("interstitial_signal_detected")
|
|
or "language selection" in haystack
|
|
or "select language" in haystack
|
|
or "choose language" in haystack
|
|
or "region selection" in haystack
|
|
or "select region" in haystack
|
|
or "language" in haystack
|
|
or "語言" in haystack
|
|
):
|
|
return "language_or_region_interstitial"
|
|
if validation.get("generic_marketplace_title_detected") or (
|
|
platform == "shopee_tw" and "花得更少買得更好" in haystack
|
|
):
|
|
return "generic_marketplace_landing"
|
|
if visual_barrier_reason:
|
|
return "platform_visual_barrier"
|
|
if validation.get("non_product_or_interstitial_detected"):
|
|
return "non_product_or_interstitial"
|
|
return "unknown_platform_barrier"
|
|
|
|
|
|
def _probe_status(platform: str, barrier_type: str) -> str:
|
|
if platform == "shopee_tw" and barrier_type in {
|
|
"language_or_region_interstitial",
|
|
"generic_marketplace_landing",
|
|
"traffic_verification_interstitial",
|
|
}:
|
|
return "ready_for_public_context_probe"
|
|
if barrier_type in {"access_denied", "traffic_verification_interstitial"}:
|
|
return "structured_source_or_backoff_required"
|
|
return "ready_for_platform_probe"
|
|
|
|
|
|
def _next_machine_action(platform: str, barrier_type: str, status: str) -> str:
|
|
if platform == "shopee_tw" and status == "ready_for_public_context_probe":
|
|
return "run_shopee_public_context_probe_then_structured_source_fallback"
|
|
if status == "structured_source_or_backoff_required":
|
|
return "use_structured_source_or_platform_backoff_policy"
|
|
return "run_public_platform_context_probe"
|
|
|
|
|
|
def _recommended_actions(platform: str, barrier_type: str, status: str) -> list[dict[str, Any]]:
|
|
actions: list[dict[str, Any]] = []
|
|
if status == "ready_for_public_context_probe":
|
|
actions.append({
|
|
"order": 1,
|
|
"action": "rerun_visual_capture_with_public_browser_context",
|
|
"machine_runnable": True,
|
|
"context_keys": ["locale", "timezone_id", "extra_http_headers", "viewport"],
|
|
})
|
|
actions.append({
|
|
"order": len(actions) + 1,
|
|
"action": "read_structured_market_intel_adapter_sources",
|
|
"machine_runnable": True,
|
|
"adapter_code": _adapter_code(platform),
|
|
})
|
|
if barrier_type in {"access_denied", "traffic_verification_interstitial"}:
|
|
actions.append({
|
|
"order": len(actions) + 1,
|
|
"action": "apply_platform_backoff_and_do_not_treat_barrier_as_product_data",
|
|
"machine_runnable": True,
|
|
})
|
|
actions.append({
|
|
"order": len(actions) + 1,
|
|
"action": "keep_visual_fields_out_of_formal_price_tables",
|
|
"machine_runnable": True,
|
|
})
|
|
return actions
|
|
|
|
|
|
def _manifest_preview(platform: str, url: str) -> dict[str, Any] | None:
|
|
keyword = _extract_keyword(url)
|
|
if not keyword:
|
|
return None
|
|
manifest = build_pixelrag_marketplace_search_manifest(
|
|
platform=platform,
|
|
keyword=keyword,
|
|
crawler="PixelRAGPlatformProbe.public_context_visual_fallback",
|
|
trigger_reason="platform_interstitial_or_blocked_page_probe",
|
|
evidence_intent="collect_public_marketplace_offer_cards_after_platform_probe",
|
|
)
|
|
if manifest.get("success"):
|
|
manifest["public_browser_context"] = _public_browser_context(platform)
|
|
return manifest
|
|
|
|
|
|
def _vlm_receipt_candidates(
|
|
root: Path,
|
|
*,
|
|
platforms: tuple[str, ...],
|
|
limit: int,
|
|
) -> list[Path]:
|
|
if not root.exists():
|
|
return []
|
|
candidates: list[Path] = []
|
|
if platforms:
|
|
for platform in platforms:
|
|
candidates.extend((root / platform).glob("*/vlm_replay_receipt.json"))
|
|
else:
|
|
candidates.extend(root.glob("*/*/vlm_replay_receipt.json"))
|
|
return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit]
|
|
|
|
|
|
def _candidate_from_capture(candidate: Mapping[str, Any]) -> dict[str, Any] | None:
|
|
next_action = str(candidate.get("next_machine_action") or "")
|
|
visual_barrier_reason = str(candidate.get("visual_barrier_reason") or "")
|
|
http_status = _safe_int(candidate.get("http_status"))
|
|
if (
|
|
"platform_probe" not in next_action
|
|
and not visual_barrier_reason
|
|
and http_status < 400
|
|
):
|
|
return None
|
|
|
|
platform = str(candidate.get("platform") or "unknown").strip().lower()
|
|
manifest_id = str(candidate.get("manifest_id") or "").strip()
|
|
url = str(candidate.get("url") or "").strip()
|
|
title = str(candidate.get("title") or "").strip()
|
|
barrier_type = _barrier_type_from_signals(
|
|
platform=platform,
|
|
http_status=http_status,
|
|
visual_barrier_reason=visual_barrier_reason,
|
|
url=url,
|
|
title=title,
|
|
)
|
|
status = _probe_status(platform, barrier_type)
|
|
return {
|
|
"platform": platform,
|
|
"manifest_id": manifest_id,
|
|
"source_type": "capture_receipt",
|
|
"source_receipt_path": candidate.get("receipt_path"),
|
|
"generated_at": candidate.get("generated_at"),
|
|
"age_hours": candidate.get("age_hours"),
|
|
"url": url,
|
|
"title": title,
|
|
"http_status": http_status,
|
|
"barrier_type": barrier_type,
|
|
"visual_barrier_reason": visual_barrier_reason,
|
|
"probe_status": status,
|
|
"probe_ready": True,
|
|
"next_machine_action": _next_machine_action(platform, barrier_type, status),
|
|
"public_browser_context": _public_browser_context(platform),
|
|
"capture_manifest_preview": _manifest_preview(platform, url),
|
|
"structured_source_fallback": _structured_source_plan(platform),
|
|
"recommended_probe_actions": _recommended_actions(platform, barrier_type, status),
|
|
"source_next_machine_action": candidate.get("next_machine_action"),
|
|
"writes_database": False,
|
|
"primary_human_gate_count": 0,
|
|
}
|
|
|
|
|
|
def _candidate_from_vlm(path: Path, *, now: datetime, max_age_hours: int) -> dict[str, Any] | None:
|
|
receipt, error = _read_json(path)
|
|
if error:
|
|
return {
|
|
"platform": path.parent.parent.name,
|
|
"manifest_id": path.parent.name,
|
|
"source_type": "vlm_replay_receipt",
|
|
"source_receipt_path": str(path),
|
|
"probe_status": "invalid_vlm_receipt",
|
|
"probe_ready": False,
|
|
"barrier_type": "invalid_receipt",
|
|
"errors": [error],
|
|
"next_machine_action": "fix_invalid_pixelrag_vlm_receipt",
|
|
"writes_database": False,
|
|
"primary_human_gate_count": 0,
|
|
}
|
|
|
|
validation = receipt.get("validation") if isinstance(receipt.get("validation"), Mapping) else {}
|
|
parsed_output = receipt.get("parsed_output") if isinstance(receipt.get("parsed_output"), Mapping) else {}
|
|
next_action = str(receipt.get("next_machine_action") or "")
|
|
if (
|
|
"platform_probe" not in next_action
|
|
and not validation.get("non_product_or_interstitial_detected")
|
|
and not validation.get("blocked_page_detected")
|
|
):
|
|
return None
|
|
|
|
platform = str(receipt.get("platform") or path.parent.parent.name).strip().lower()
|
|
manifest_id = str(receipt.get("manifest_id") or path.parent.name).strip()
|
|
generated = _parse_iso_datetime(receipt.get("generated_at"))
|
|
age_hours = ((now - generated).total_seconds() / 3600) if generated else None
|
|
source_url = ""
|
|
source_title = ""
|
|
source_receipt = str(receipt.get("source_receipt_path") or "")
|
|
if source_receipt:
|
|
capture, _ = _read_json(Path(source_receipt))
|
|
capture_target = capture.get("capture_target") or {}
|
|
page_metrics = capture.get("page_metrics") or {}
|
|
source_url = str(capture_target.get("url") or page_metrics.get("final_url") or "")
|
|
source_title = str(page_metrics.get("title") or "")
|
|
barrier_type = _barrier_type_from_signals(
|
|
platform=platform,
|
|
http_status=0,
|
|
visual_barrier_reason="",
|
|
url=source_url,
|
|
title=source_title,
|
|
validation=validation,
|
|
parsed_output=parsed_output,
|
|
)
|
|
status = _probe_status(platform, barrier_type)
|
|
return {
|
|
"platform": platform,
|
|
"manifest_id": manifest_id,
|
|
"source_type": "vlm_replay_receipt",
|
|
"source_receipt_path": str(path),
|
|
"source_capture_receipt_path": source_receipt,
|
|
"generated_at": receipt.get("generated_at"),
|
|
"age_hours": round(age_hours, 3) if age_hours is not None else None,
|
|
"stale": age_hours is None or age_hours > max_age_hours,
|
|
"url": source_url,
|
|
"title": source_title,
|
|
"http_status": 0,
|
|
"barrier_type": barrier_type,
|
|
"probe_status": status,
|
|
"probe_ready": True,
|
|
"validation": {
|
|
"blocked_page_detected": bool(validation.get("blocked_page_detected")),
|
|
"non_product_or_interstitial_detected": bool(
|
|
validation.get("non_product_or_interstitial_detected")
|
|
),
|
|
"interstitial_signal_detected": bool(
|
|
validation.get("interstitial_signal_detected")
|
|
),
|
|
"generic_marketplace_title_detected": bool(
|
|
validation.get("generic_marketplace_title_detected")
|
|
),
|
|
"present_field_count": _safe_int(validation.get("present_field_count")),
|
|
"missing_required_fields": list(validation.get("missing_required_fields") or []),
|
|
},
|
|
"next_machine_action": _next_machine_action(platform, barrier_type, status),
|
|
"public_browser_context": _public_browser_context(platform),
|
|
"capture_manifest_preview": _manifest_preview(platform, source_url),
|
|
"structured_source_fallback": _structured_source_plan(platform),
|
|
"recommended_probe_actions": _recommended_actions(platform, barrier_type, status),
|
|
"source_next_machine_action": receipt.get("next_machine_action"),
|
|
"writes_database": False,
|
|
"primary_human_gate_count": 0,
|
|
}
|
|
|
|
|
|
def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
by_key: dict[tuple[str, str], dict[str, Any]] = {}
|
|
source_rank = {"vlm_replay_receipt": 2, "capture_receipt": 1}
|
|
for item in items:
|
|
key = (str(item.get("platform") or ""), str(item.get("manifest_id") or ""))
|
|
current = by_key.get(key)
|
|
if not current or source_rank.get(str(item.get("source_type")), 0) >= source_rank.get(
|
|
str(current.get("source_type")), 0
|
|
):
|
|
by_key[key] = item
|
|
return list(by_key.values())
|
|
|
|
|
|
def build_pixelrag_platform_probe_readiness(
|
|
*,
|
|
artifact_root: str | Path | None = None,
|
|
vlm_receipt_root: str | Path | None = None,
|
|
platform: str | tuple[str, ...] | list[str] | None = None,
|
|
max_age_hours: int | None = None,
|
|
limit: int | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Build a no-write platform probe plan from PixelRAG barrier receipts."""
|
|
capture_root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT)
|
|
vlm_root = Path(vlm_receipt_root or DEFAULT_VLM_RECEIPT_ROOT)
|
|
platforms = _normalise_platforms(platform)
|
|
max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS))
|
|
item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250))
|
|
now = datetime.now(timezone.utc)
|
|
|
|
capture_readback = build_pixelrag_rag_candidate_replay_readback(
|
|
artifact_root=capture_root,
|
|
platform=platforms,
|
|
max_age_hours=max_age,
|
|
limit=item_limit,
|
|
)
|
|
capture_items = [
|
|
item
|
|
for item in (
|
|
_candidate_from_capture(candidate)
|
|
for candidate in list(capture_readback.get("candidates") or [])
|
|
)
|
|
if item
|
|
]
|
|
vlm_items = [
|
|
item
|
|
for item in (
|
|
_candidate_from_vlm(path, now=now, max_age_hours=max_age)
|
|
for path in _vlm_receipt_candidates(vlm_root, platforms=platforms, limit=item_limit)
|
|
)
|
|
if item
|
|
]
|
|
probe_items = _dedupe_items(capture_items + vlm_items)
|
|
invalid_count = sum(1 for item in probe_items if not item.get("probe_ready"))
|
|
ready_count = sum(1 for item in probe_items if item.get("probe_ready"))
|
|
shopee_public_context_count = sum(
|
|
1
|
|
for item in probe_items
|
|
if item.get("platform") == "shopee_tw"
|
|
and item.get("probe_status") == "ready_for_public_context_probe"
|
|
)
|
|
access_denied_count = sum(1 for item in probe_items if item.get("barrier_type") == "access_denied")
|
|
traffic_count = sum(
|
|
1 for item in probe_items if item.get("barrier_type") == "traffic_verification_interstitial"
|
|
)
|
|
language_count = sum(
|
|
1 for item in probe_items if item.get("barrier_type") == "language_or_region_interstitial"
|
|
)
|
|
structured_fallback_count = sum(
|
|
1
|
|
for item in probe_items
|
|
if (item.get("structured_source_fallback") or {}).get("available")
|
|
)
|
|
|
|
if invalid_count and invalid_count == len(probe_items):
|
|
status = "critical"
|
|
elif probe_items and ready_count:
|
|
status = "ok"
|
|
else:
|
|
status = "warning"
|
|
|
|
next_action = (
|
|
"fix_invalid_pixelrag_platform_probe_receipts"
|
|
if invalid_count and not ready_count
|
|
else (
|
|
"run_platform_probe_or_structured_source_fallback"
|
|
if ready_count
|
|
else "run_pixelrag_visual_capture_worker"
|
|
)
|
|
)
|
|
|
|
return {
|
|
"success": status != "critical",
|
|
"policy": POLICY,
|
|
"status": status,
|
|
"generated_at": now.isoformat(),
|
|
"artifact_root": str(capture_root),
|
|
"vlm_receipt_root": str(vlm_root),
|
|
"platform_filter": list(platforms),
|
|
"max_age_hours": max_age,
|
|
"limit": item_limit,
|
|
"summary": {
|
|
"probe_candidate_count": len(probe_items),
|
|
"ready_for_probe_count": ready_count,
|
|
"invalid_count": invalid_count,
|
|
"capture_source_count": len(capture_items),
|
|
"vlm_source_count": len(vlm_items),
|
|
"shopee_public_context_probe_count": shopee_public_context_count,
|
|
"language_or_region_interstitial_count": language_count,
|
|
"traffic_verification_count": traffic_count,
|
|
"access_denied_count": access_denied_count,
|
|
"structured_source_fallback_count": structured_fallback_count,
|
|
"writes_database_count": 0,
|
|
"primary_human_gate_count": 0,
|
|
"platforms": sorted({str(item.get("platform") or "unknown") for item in probe_items}),
|
|
},
|
|
"probe_items": probe_items,
|
|
"source_capture_replay": {
|
|
"policy": capture_readback.get("policy"),
|
|
"status": capture_readback.get("status"),
|
|
"summary": capture_readback.get("summary"),
|
|
"next_machine_action": capture_readback.get("next_machine_action"),
|
|
},
|
|
"probe_contract": {
|
|
"automation_mode": "platform_probe_plan_no_write",
|
|
"network_call": False,
|
|
"db_write": False,
|
|
"writes_database": False,
|
|
"writes_ai_insights": False,
|
|
"writes_price_tables": False,
|
|
"secret_read": False,
|
|
"raw_cookie_or_session_read": False,
|
|
"credentialed_session_allowed": False,
|
|
"login_allowed": False,
|
|
"blocked_pages_are_not_product_data": True,
|
|
"visual_fields_are_candidate_evidence_only": True,
|
|
"primary_human_gate_count": 0,
|
|
},
|
|
"controlled_apply": {
|
|
"network_call": False,
|
|
"db_write": False,
|
|
"writes_database": False,
|
|
"writes_database_count": 0,
|
|
"secret_read": False,
|
|
"raw_cookie_or_session_read": False,
|
|
"production_price_write": False,
|
|
"artifact_write": False,
|
|
"primary_human_gate_count": 0,
|
|
},
|
|
"next_machine_action": next_action,
|
|
}
|
|
|
|
|
|
__all__ = [
|
|
"POLICY",
|
|
"build_pixelrag_platform_probe_readiness",
|
|
]
|