fix(ai): detect PixelRAG marketplace interstitials

2026-07-10 00:49:54 +08:00
parent c491723961
commit 83f6d2e255
5 changed files with 124 additions and 3 deletions
--- a/TODO_NEXT_STEPS.txt
+++ b/TODO_NEXT_STEPS.txt
@@ -9,7 +9,7 @@
    python scripts/ops/check_production_version_truth.py

 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。
-本輪 source target 為 `V10.757`；部署完成前不得宣稱正式環境已是 `V10.757`。
+本輪 source target 為 `V10.758`；部署完成前不得宣稱正式環境已是 `V10.758`。
 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。

 ================================================================================
--- a/config.py
+++ b/config.py
@@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '')
 # ==========================================
 # 系統版本與路徑
 # ==========================================
-SYSTEM_VERSION = "V10.757"
+SYSTEM_VERSION = "V10.758"
 LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log')
 public_url = PUBLIC_URL  # 用於模板顯示

--- a/docs/AI_INTELLIGENCE_MODULE_SOT.md
+++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md
@@ -887,6 +887,7 @@ POSTGRES_HOST=momo-db
 | 2026-07-10 | PixelRAG VLM no-probe 監控不得把 unknown 顯示成 failed | V10.755 起 smoke / scheduled-health 的 `pixelrag_vlm_route_readiness.details.generate_route_ready` 在 `generate_probe_performed=false` 時必須保留為 `null`，表示尚未做 `/api/generate` preflight；不得把未探測狀態轉成 `false`，避免 UI 或監控把 tags-only readiness 誤讀為 generate failure。 |
 | 2026-07-10 | PixelRAG VLM 空欄位與 Shopee language interstitial 必須自動轉 platform probe | V10.756 起 VLM prompt 明確把 language / region / app-download / landing / loading / logo-only / cookie consent pages 視為非商品卡；若 VLM 回傳所有 offer fields 皆空且 required title / price 缺失，worker 會標記 `non_product_or_interstitial_detected=true`，receipt 下一步改為 `run_platform_probe_or_use_structured_api`，不得反覆要求人工審核或無限 rerun VLM。 |
 | 2026-07-10 | PixelRAG VLM worker preflight 必須容納 111 VLM 冷啟延遲 | V10.757 起 `run_pixelrag_vlm_replay_worker.py --execute` 與對應 API 的 `route_generate_probe_timeout` 預設為 20 秒；111 `minicpm-v` 在正式環境可於約 4 到 20 秒間完成小型 `/api/generate` probe，8 秒容易造成假陰性。若 VLM receipt 已判定 `run_platform_probe_or_use_structured_api`，run-level `next_machine_action` 也必須同樣輸出 platform probe，不得停留在 rerun VLM 摘要。 |
+| 2026-07-10 | PixelRAG VLM language note / generic marketplace slogan 必須視為 interstitial | V10.758 起若 VLM notes 或 title 顯示 `Language selection page`、語言/地區選擇，或只抽到 Shopee `蝦皮購物 | 花得更少買得更好` 這類網站 slogan 且 price/shop 缺失，worker validation 會標記 `interstitial_signal_detected` / `generic_marketplace_title_detected` 與 `non_product_or_interstitial_detected=true`，run-level 與 item-level 下一步都輸出 `run_platform_probe_or_use_structured_api`。 |
 | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 |
 | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 |
 | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 |
--- a/services/pixelrag_vlm_replay_worker_service.py
+++ b/services/pixelrag_vlm_replay_worker_service.py
@@ -55,6 +55,25 @@ DEFAULT_MODEL = (
    or "minicpm-v:latest"
 )
 RAW_EXCERPT_LIMIT = 500
+INTERSTITIAL_SIGNAL_TOKENS = (
+    "language selection",
+    "select language",
+    "choose language",
+    "region selection",
+    "select region",
+    "app-download",
+    "app download",
+    "landing page",
+    "loading page",
+    "logo-only",
+    "cookie consent",
+    "選擇語言",
+    "選擇地區",
+    "語言",
+)
+GENERIC_MARKETPLACE_TITLE_TOKENS = (
+    "蝦皮購物 | 花得更少買得更好",
+)


 def _normalise_platforms(
@@ -176,6 +195,23 @@ def _field_value_present(value: Any) -> bool:
    return True


+def _stringify_signal(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, Mapping):
+        return " ".join(_stringify_signal(item) for item in value.values())
+    if isinstance(value, list):
+        return " ".join(_stringify_signal(item) for item in value)
+    return str(value)
+
+
+def _has_interstitial_signal(*values: Any) -> bool:
+    haystack = " ".join(_stringify_signal(value) for value in values).lower()
+    return any(token.lower() in haystack for token in INTERSTITIAL_SIGNAL_TOKENS)
+
+
 def _validate_model_payload(
    parsed: Mapping[str, Any],
    item: Mapping[str, Any],
@@ -187,6 +223,7 @@ def _validate_model_payload(
    low_confidence_fields: list[str] = []
    present_field_count = 0
    blocked_detected = bool(parsed.get("blocked_page_detected"))
+    title_value = None

    for contract in list(item.get("field_contract") or []):
        field_name = str(contract.get("field") or "")
@@ -194,6 +231,8 @@ def _validate_model_payload(
        if not isinstance(field_payload, Mapping):
            field_payload = {}
        value = field_payload.get("value")
+        if field_name == "title":
+            title_value = value
        evidence_refs = list(field_payload.get("evidence_refs") or [])
        try:
            confidence = float(field_payload.get("confidence") or 0)
@@ -218,15 +257,31 @@ def _validate_model_payload(
    for field in declared_missing:
        if field not in missing_required:
            missing_required.append(field)
+    notes_payload = parsed.get("notes")
+    generic_marketplace_title_detected = (
+        isinstance(title_value, str)
+        and any(token in title_value for token in GENERIC_MARKETPLACE_TITLE_TOKENS)
+    )
+    interstitial_signal_detected = _has_interstitial_signal(
+        notes_payload,
+        title_value,
+        item.get("title_hint"),
+    )
    non_product_or_interstitial_detected = (
        not blocked_detected
-        and present_field_count == 0
+        and (
+            present_field_count == 0
+            or interstitial_signal_detected
+            or generic_marketplace_title_detected
+        )
        and bool(missing_required)
    )

    return {
        "blocked_page_detected": blocked_detected,
        "non_product_or_interstitial_detected": non_product_or_interstitial_detected,
+        "interstitial_signal_detected": interstitial_signal_detected,
+        "generic_marketplace_title_detected": generic_marketplace_title_detected,
        "present_field_count": present_field_count,
        "missing_required_fields": missing_required,
        "field_evidence_missing": field_evidence_missing,
--- a/tests/test_pixelrag_vlm_replay_worker_service.py
+++ b/tests/test_pixelrag_vlm_replay_worker_service.py
@@ -244,6 +244,71 @@ def test_pixelrag_vlm_replay_worker_empty_fields_routes_to_platform_probe(tmp_pa
    assert receipt["next_machine_action"] == "run_platform_probe_or_use_structured_api"


+def test_pixelrag_vlm_replay_worker_language_note_routes_to_platform_probe(tmp_path, monkeypatch):
+    from services import pixelrag_vlm_replay_worker_service as service
+
+    _write_receipt(
+        tmp_path,
+        platform="shopee_tw",
+        manifest_id="shopee-language-note",
+        title="蝦皮購物 | 花得更少買得更好",
+        url="https://shopee.tw/search?keyword=sunscreen",
+    )
+
+    class FakeOllama:
+        def __init__(self, model):
+            self.model = model
+
+        def generate(self, *args, **kwargs):
+            return SimpleNamespace(
+                success=True,
+                content=json.dumps({
+                    "blocked_page_detected": False,
+                    "fields": {
+                        "title": {
+                            "value": "蝦皮購物 | 花得更少買得更好",
+                            "confidence": 0.95,
+                            "evidence_refs": ["tile:1"],
+                        },
+                        "price": {"value": None, "confidence": 0.0, "evidence_refs": []},
+                        "shop": {"value": None, "confidence": 0.0, "evidence_refs": []},
+                    },
+                    "quality": {
+                        "overall_confidence": 0.5,
+                        "missing_required_fields": ["title", "price", "shop"],
+                        "requires_identity_matcher_replay": True,
+                        "requires_promotion_gate": False,
+                    },
+                    "notes": [{"note": "Language selection page"}],
+                }),
+                model="minicpm-v:latest",
+                error=None,
+                total_duration=2.0,
+                host="http://192.168.0.111:11434",
+                input_tokens=10,
+                output_tokens=40,
+            )
+
+    monkeypatch.setattr(service, "OllamaService", FakeOllama)
+    payload = service.run_pixelrag_ollama_vlm_replay_worker(
+        artifact_root=tmp_path,
+        output_root=tmp_path / "receipts",
+        platform="shopee_tw",
+        execute=True,
+        write_receipt=True,
+        tile_limit=4,
+        auto_select_model=False,
+    )
+
+    assert payload["next_machine_action"] == "run_platform_probe_or_use_structured_api"
+    item = payload["worker_items"][0]
+    assert item["next_machine_action"] == "run_platform_probe_or_use_structured_api"
+    assert item["validation"]["present_field_count"] == 1
+    assert item["validation"]["interstitial_signal_detected"] is True
+    assert item["validation"]["generic_marketplace_title_detected"] is True
+    assert item["validation"]["non_product_or_interstitial_detected"] is True
+
+
 def test_pixelrag_vlm_replay_worker_auto_selects_installed_candidate(tmp_path, monkeypatch):
    from services import pixelrag_vlm_replay_worker_service as service