fix(ai): detect PixelRAG marketplace interstitials

services/pixelrag_vlm_replay_worker_service.py

@@ -55,6 +55,25 @@ DEFAULT_MODEL = (
     or "minicpm-v:latest"
 )
 RAW_EXCERPT_LIMIT = 500
+INTERSTITIAL_SIGNAL_TOKENS = (
+    "language selection",
+    "select language",
+    "choose language",
+    "region selection",
+    "select region",
+    "app-download",
+    "app download",
+    "landing page",
+    "loading page",
+    "logo-only",
+    "cookie consent",
+    "選擇語言",
+    "選擇地區",
+    "語言",
+)
+GENERIC_MARKETPLACE_TITLE_TOKENS = (
+    "蝦皮購物 | 花得更少買得更好",
+)
 
 
 def _normalise_platforms(
@@ -176,6 +195,23 @@ def _field_value_present(value: Any) -> bool:
     return True
 
 
+def _stringify_signal(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, Mapping):
+        return " ".join(_stringify_signal(item) for item in value.values())
+    if isinstance(value, list):
+        return " ".join(_stringify_signal(item) for item in value)
+    return str(value)
+
+
+def _has_interstitial_signal(*values: Any) -> bool:
+    haystack = " ".join(_stringify_signal(value) for value in values).lower()
+    return any(token.lower() in haystack for token in INTERSTITIAL_SIGNAL_TOKENS)
+
+
 def _validate_model_payload(
     parsed: Mapping[str, Any],
     item: Mapping[str, Any],
@@ -187,6 +223,7 @@ def _validate_model_payload(
     low_confidence_fields: list[str] = []
     present_field_count = 0
     blocked_detected = bool(parsed.get("blocked_page_detected"))
+    title_value = None
 
     for contract in list(item.get("field_contract") or []):
         field_name = str(contract.get("field") or "")
@@ -194,6 +231,8 @@ def _validate_model_payload(
         if not isinstance(field_payload, Mapping):
             field_payload = {}
         value = field_payload.get("value")
+        if field_name == "title":
+            title_value = value
         evidence_refs = list(field_payload.get("evidence_refs") or [])
         try:
             confidence = float(field_payload.get("confidence") or 0)
@@ -218,15 +257,31 @@ def _validate_model_payload(
     for field in declared_missing:
         if field not in missing_required:
             missing_required.append(field)
+    notes_payload = parsed.get("notes")
+    generic_marketplace_title_detected = (
+        isinstance(title_value, str)
+        and any(token in title_value for token in GENERIC_MARKETPLACE_TITLE_TOKENS)
+    )
+    interstitial_signal_detected = _has_interstitial_signal(
+        notes_payload,
+        title_value,
+        item.get("title_hint"),
+    )
     non_product_or_interstitial_detected = (
         not blocked_detected
-        and present_field_count == 0
+        and (
+            present_field_count == 0
+            or interstitial_signal_detected
+            or generic_marketplace_title_detected
+        )
         and bool(missing_required)
     )
 
     return {
         "blocked_page_detected": blocked_detected,
         "non_product_or_interstitial_detected": non_product_or_interstitial_detected,
+        "interstitial_signal_detected": interstitial_signal_detected,
+        "generic_marketplace_title_detected": generic_marketplace_title_detected,
         "present_field_count": present_field_count,
         "missing_required_fields": missing_required,
         "field_evidence_missing": field_evidence_missing,