diff --git a/config.py b/config.py index fc4b4b1..2488ffa 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.761" +SYSTEM_VERSION = "V10.762" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 5bd159b..9fc5563 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,8 +1,8 @@ # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth -> **最後更新**: 2026-07-09 (台北時間) +> **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.748 +> **適用版本**: V10.762 --- @@ -894,6 +894,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG platform probe 必須自動消化 marketplace interstitial / blocked receipt | V10.759 起 `/api/ai-automation/pixelrag-platform-probe`、`scripts/ops/report_pixelrag_platform_probe.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 platform probe / `pixelrag_platform_probe` family；Shopee traffic / language / generic landing 會轉成 `run_shopee_public_context_probe_then_structured_source_fallback`，Coupang 403 / access denied 會轉成 `use_structured_source_or_platform_backoff_policy`。probe plan 只產生 public empty browser context、structured adapter fallback 與 no-write contract，不讀或注入 cookie/session/login，不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture worker 只允許 locale/timezone/非敏感 header 並濾掉 Cookie / Authorization / token / secret / key 類 header。 | | 2026-07-10 | PixelRAG platform probe worker 必須把 probe plan 變成 controlled automation | V10.760 起 `/api/ai-automation/pixelrag-platform-probe-worker`、`scripts/ops/run_pixelrag_platform_probe_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled platform probe worker / `pixelrag_platform_probe_worker` family；dry-run 只規劃 capture 或 structured fallback，execute 模式才執行 public empty-context capture 或寫 structured-source/backoff artifact receipt。此 worker 明確標記 `network_call` 只在 capture execute 時為 true、`model_call=false`、`writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`raw_cookie_or_session_read=false`、`primary_human_gate_count=0`；capture 成功後必須回到 PixelRAG RAG candidate replay，不得直接寫正式價格或把 blocked page 當商品資料。 | | 2026-07-10 | PixelRAG platform probe worker 缺 capture runtime 時必須自動降級 | V10.761 起若 `run_pixelrag_platform_probe_worker.py --execute --write-receipt` 在 app 容器偵測到 `ModuleNotFoundError: playwright`，worker 會自動產生 `capture_runtime_unavailable_structured_fallback_package` receipt，`network_call=false`、`artifact_write=true`、`writes_database=false`、`primary_human_gate_count=0`，不再把 Shopee/Coupang platform probe runtime gap 變成人工主流程。 | +| 2026-07-10 | PixelRAG platform probe worker receipts 必須回到 RAG/source-contract replay | V10.762 起 `/api/ai-automation/pixelrag-rag-candidate-replay` 與 `scripts/ops/report_pixelrag_rag_candidate_replay.py` 必須同時讀 visual capture receipts 與 `platform_probe_worker_receipt.json`；worker receipts 只能成為 `source_contract_fallback` / `eligible_for_source_contract_replay` 候選，並輸出 `platform_probe_worker_receipt_count`、`source_contract_fallback_count`、`capture_runtime_gap_count` 與下一步 `run_source_contract_replay_for_platform_probe_worker_receipts`。此路徑不寫 DB、不寫 `ai_insights`、不寫正式價格表、不把 blocked page 當商品資料，也不得退回人工審核主流程。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index bb5f192..e814f88 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -16,7 +16,7 @@ |---|---|---| | Completed | Multi-commerce PixelRAG visual evidence lane for momo, pchome, shopee_tw, coupang_tw, yahoo_shopping_tw, etmall_tw, friday_tw, rakuten_tw | API and CLI can emit expansion plans and marketplace manifests; PChome parser failures can queue manifests. | | Completed | External MCP/RAG capability inventory absorbed into internal governance readback | `/api/ai-automation/external-mcp-rag-integration` and `scripts/ops/report_external_mcp_rag_integration.py` expose 9 capabilities, absorbed/unresolved counts, and runtime flags. | -| Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read receipts, split eligible vs blocked, and require OCR/VLM replay plus PromotionGate before knowledge writes. | +| Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read visual capture receipts and platform probe worker receipts, split eligible / blocked / source-contract-fallback candidates, and require OCR/VLM replay, source contract replay, identity matching, plus PromotionGate before knowledge writes. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | @@ -33,7 +33,7 @@ | Not started | Ollama-first multimodal embedding benchmark | Verify local Qwen3-VL or equivalent visual embedding on GCP-A -> GCP-B -> 111 before any visual vector retrieval. | | Not started | pgvector-compatible visual evidence metadata | Design metadata-first retrieval without FAISS in production unless ADR approves a different store. | | In progress | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; `run_pixelrag_platform_probe_worker.py --execute --write-receipt` now emits structured-source/backoff receipts without network, DB writes, or product-data promotion. | -| Not started | Marketplace source contracts beyond MOMO/PChome | Promote stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten receipts into source contracts only after provenance, rate-limit, public boundary, and replay evidence. | +| In progress | Marketplace source contracts beyond MOMO/PChome | Platform probe worker receipts now enter RAG candidate replay as source-contract fallback candidates; next machine action is to replay stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten source contracts only after provenance, rate-limit, public boundary, and replay evidence. | | In progress | Professional product website UI/UX text reduction | Continue compact AI surfaces, visual QA, and user-facing copy guardrails; avoid engineering logs or work-session prose in the UI. | ## Always Enforced diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 5974c5a..9f31159 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -705,6 +705,9 @@ def ai_automation_pixelrag_rag_candidate_replay_api(): max_age_hours = request.args.get('max_age_hours', 168, type=int) limit = request.args.get('limit', 50, type=int) return jsonify(build_pixelrag_rag_candidate_replay_readback( + platform_probe_worker_receipt_root=str( + request.args.get('platform_probe_worker_receipt_root') or '' + ).strip() or None, platform=platforms, max_age_hours=max(1, min(max_age_hours or 168, 720)), limit=max(1, min(limit or 50, 250)), diff --git a/scripts/ops/report_pixelrag_rag_candidate_replay.py b/scripts/ops/report_pixelrag_rag_candidate_replay.py index 9520f8c..50befd6 100755 --- a/scripts/ops/report_pixelrag_rag_candidate_replay.py +++ b/scripts/ops/report_pixelrag_rag_candidate_replay.py @@ -26,6 +26,10 @@ def main() -> int: "--artifact-root", help="PixelRAG visual evidence artifact root；預設使用 production/container 設定。", ) + parser.add_argument( + "--platform-probe-worker-receipt-root", + help="PixelRAG platform probe worker receipt root；預設使用 production/container 設定。", + ) parser.add_argument( "--platform", action="append", @@ -48,6 +52,7 @@ def main() -> int: payload = build_pixelrag_rag_candidate_replay_readback( artifact_root=args.artifact_root, + platform_probe_worker_receipt_root=args.platform_probe_worker_receipt_root, platform=tuple(args.platforms or ()), max_age_hours=args.max_age_hours, limit=args.limit, diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index a2283b4..b6ae444 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -4556,6 +4556,18 @@ def build_scheduled_automation_health_summary( "eligible_count": int(pixelrag_rag_replay_details.get("eligible_count") or 0), "blocked_count": int(pixelrag_rag_replay_details.get("blocked_count") or 0), "invalid_count": int(pixelrag_rag_replay_details.get("invalid_count") or 0), + "visual_receipt_count": int( + pixelrag_rag_replay_details.get("visual_receipt_count") or 0 + ), + "platform_probe_worker_receipt_count": int( + pixelrag_rag_replay_details.get("platform_probe_worker_receipt_count") or 0 + ), + "source_contract_fallback_count": int( + pixelrag_rag_replay_details.get("source_contract_fallback_count") or 0 + ), + "capture_runtime_gap_count": int( + pixelrag_rag_replay_details.get("capture_runtime_gap_count") or 0 + ), "visual_barrier_count": int( pixelrag_rag_replay_details.get("visual_barrier_count") or 0 ), @@ -12750,11 +12762,15 @@ def _pixelrag_rag_candidate_replay_check() -> Dict[str, Any]: blocked_count = int(summary.get("blocked_count") or 0) invalid_count = int(summary.get("invalid_count") or 0) visual_barrier_count = int(summary.get("visual_barrier_count") or 0) + worker_receipt_count = int(summary.get("platform_probe_worker_receipt_count") or 0) + source_contract_fallback_count = int(summary.get("source_contract_fallback_count") or 0) + capture_runtime_gap_count = int(summary.get("capture_runtime_gap_count") or 0) status = readback.get("status") or "warning" summary_text = ( f"PixelRAG RAG candidate replay receipts={receipt_count}, " f"eligible={eligible_count}, blocked={blocked_count}, invalid={invalid_count}, " - f"barriers={visual_barrier_count}" + f"barriers={visual_barrier_count}, worker_receipts={worker_receipt_count}, " + f"source_contract_fallback={source_contract_fallback_count}" ) return _check( "PixelRAG RAG candidate replay", @@ -12767,6 +12783,10 @@ def _pixelrag_rag_candidate_replay_check() -> Dict[str, Any]: "blocked_count": blocked_count, "invalid_count": invalid_count, "visual_barrier_count": visual_barrier_count, + "visual_receipt_count": int(summary.get("visual_receipt_count") or 0), + "platform_probe_worker_receipt_count": worker_receipt_count, + "source_contract_fallback_count": source_contract_fallback_count, + "capture_runtime_gap_count": capture_runtime_gap_count, "tile_file_count": int(summary.get("tile_file_count") or 0), "missing_file_count": int(summary.get("missing_file_count") or 0), "stale_count": int(summary.get("stale_count") or 0), diff --git a/services/pixelrag_rag_candidate_replay_service.py b/services/pixelrag_rag_candidate_replay_service.py index c390e21..9a33d6f 100644 --- a/services/pixelrag_rag_candidate_replay_service.py +++ b/services/pixelrag_rag_candidate_replay_service.py @@ -7,6 +7,7 @@ anything into the production price tables. from __future__ import annotations +import os import json from datetime import datetime, timezone from pathlib import Path @@ -27,6 +28,12 @@ from services.pixelrag_crawler_integration_service import ( POLICY = "read_only_pixelrag_rag_candidate_replay_v1" INTERNAL_RAG_TARGET = "rag_service.learning_episode_candidate_preview" DEFAULT_LIMIT = 50 +DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT = os.getenv( + "PIXELRAG_PLATFORM_PROBE_WORKER_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_platform_probe_worker_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_platform_probe_worker_receipts", +) def _status_counts(items: list[dict[str, Any]]) -> dict[str, int]: @@ -197,6 +204,150 @@ def _candidate_from_receipt_path( } +def _platform_probe_worker_receipt_candidates( + root: Path, + *, + platforms: tuple[str, ...] | list[str] | None, + limit: int, +) -> list[Path]: + if not root.exists(): + return [] + selected_platforms = [ + str(platform or "").strip().lower() + for platform in (platforms or []) + if str(platform or "").strip() + ] + candidates: list[Path] = [] + if selected_platforms: + for platform in selected_platforms: + candidates.extend((root / platform).glob("*/platform_probe_worker_receipt.json")) + else: + candidates.extend(root.glob("*/*/platform_probe_worker_receipt.json")) + return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] + + +def _candidate_from_platform_probe_worker_receipt_path( + receipt_path: Path, + *, + now: datetime, + max_age_hours: int, +) -> dict[str, Any]: + errors: list[str] = [] + try: + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + receipt = {} + errors.append(str(exc)[:300]) + + try: + file_generated_at = datetime.fromtimestamp(receipt_path.stat().st_mtime, tz=timezone.utc) + except OSError: + file_generated_at = None + generated_at = _parse_iso_datetime(receipt.get("generated_at")) or file_generated_at + age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None + stale = age_hours is None or age_hours > max_age_hours + platform = str(receipt.get("platform") or receipt_path.parent.parent.name).strip().lower() + manifest_id = str(receipt.get("manifest_id") or receipt_path.parent.name).strip() + worker_status = str(receipt.get("worker_status") or "").strip() + structured_package = ( + receipt.get("structured_source_package") + if isinstance(receipt.get("structured_source_package"), Mapping) + else {} + ) + structured_available = bool(structured_package.get("available")) + capture_runtime_gap = bool(structured_package.get("capture_runtime_unavailable")) or ( + worker_status == "capture_runtime_unavailable_structured_fallback_package" + ) + capture_receipt_path = str(receipt.get("capture_receipt_path") or "").strip() + source_receipt_path = str(receipt.get("source_receipt_path") or "").strip() + blocked_reasons: list[str] = [] + if errors: + blocked_reasons.append("platform_probe_worker_receipt_parse_error") + if stale: + blocked_reasons.append("platform_probe_worker_receipt_stale_or_missing_timestamp") + if capture_runtime_gap: + blocked_reasons.append("capture_runtime_unavailable") + barrier_type = str(receipt.get("barrier_type") or "").strip() + if barrier_type: + blocked_reasons.append(f"platform_barrier:{barrier_type}") + + if errors: + candidate_status = "invalid" + elif structured_available: + candidate_status = "source_contract_fallback" + elif worker_status == "executed_capture_ok" and capture_receipt_path: + candidate_status = "probe_capture_ready" + else: + candidate_status = "worker_handoff" + + rag_text_parts = [ + f"platform={platform}", + f"manifest_id={manifest_id}", + f"worker_status={worker_status or 'unknown'}", + f"probe_status={receipt.get('probe_status') or 'unknown'}", + f"barrier_type={barrier_type or 'none'}", + f"source_receipt_path={source_receipt_path or 'unknown'}", + f"capture_receipt_path={capture_receipt_path or 'none'}", + f"structured_adapter={structured_package.get('adapter_code') or 'none'}", + f"structured_source_count={int(structured_package.get('source_count') or 0)}", + f"capture_runtime_unavailable={capture_runtime_gap}", + ] + return { + "candidate_status": candidate_status, + "source_type": "platform_probe_worker_receipt", + "eligible_for_rag_candidate_replay": False, + "eligible_for_source_contract_replay": bool(structured_available), + "blocked_reasons": blocked_reasons, + "platform": platform, + "manifest_id": manifest_id, + "receipt_path": str(receipt_path), + "generated_at": generated_at.isoformat() if generated_at else None, + "age_hours": round(age_hours, 3) if age_hours is not None else None, + "stale": stale, + "http_status": 0, + "capture_status": worker_status, + "url": receipt.get("url"), + "title": receipt.get("title"), + "visual_barrier_detected": bool(barrier_type), + "visual_barrier_reason": barrier_type, + "tile_file_count": 0, + "missing_file_count": 0, + "platform_probe_worker": { + "worker_status": worker_status, + "probe_status": receipt.get("probe_status"), + "barrier_type": barrier_type, + "source_receipt_path": source_receipt_path, + "capture_receipt_path": capture_receipt_path, + "network_call_performed": bool(receipt.get("network_call_performed")), + "artifact_write_performed": bool(receipt.get("artifact_write_performed")), + "writes_database": bool(receipt.get("writes_database")), + "next_machine_action": receipt.get("next_machine_action"), + "structured_source_package": structured_package, + }, + "rag_candidate_text": " | ".join(rag_text_parts), + "internal_rag_target": INTERNAL_RAG_TARGET, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "requires_ocr_or_vlm_replay": False, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + "requires_source_contract_before_knowledge_write": True, + }, + "next_machine_action": ( + "run_source_contract_replay_for_platform_probe_worker_receipt" + if structured_available + else ( + "run_pixelrag_rag_candidate_replay_after_probe_capture" + if capture_receipt_path + else "refresh_pixelrag_platform_probe_worker_receipt" + ) + ), + "file_summary": {"tile_file_count": 0, "missing_file_count": 0, "files": []}, + } + + def _unique_receipt_candidates( root: Path, *, @@ -229,12 +380,14 @@ def _unique_receipt_candidates( def build_pixelrag_rag_candidate_replay_readback( *, artifact_root: str | Path | None = None, + platform_probe_worker_receipt_root: str | Path | None = None, platform: str | tuple[str, ...] | list[str] | None = None, max_age_hours: int | None = None, limit: int | None = None, ) -> dict[str, Any]: """Build a replayable RAG candidate readback from PixelRAG receipts.""" root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT) + worker_root = Path(platform_probe_worker_receipt_root or DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT) max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) if isinstance(platform, str): @@ -252,35 +405,66 @@ def build_pixelrag_rag_candidate_replay_readback( _candidate_from_receipt_path(path, now=now, max_age_hours=max_age) for path in candidates ] + worker_receipts = _platform_probe_worker_receipt_candidates( + worker_root, + platforms=platforms, + limit=item_limit, + ) + worker_candidates = [ + _candidate_from_platform_probe_worker_receipt_path( + path, + now=now, + max_age_hours=max_age, + ) + for path in worker_receipts + ] + all_candidates = replay_candidates + worker_candidates eligible_count = sum(1 for item in replay_candidates if item["candidate_status"] == "eligible") blocked_count = sum(1 for item in replay_candidates if item["candidate_status"] == "blocked") - invalid_count = sum(1 for item in replay_candidates if item["candidate_status"] == "invalid") + invalid_count = sum(1 for item in all_candidates if item["candidate_status"] == "invalid") visual_barrier_count = sum(1 for item in replay_candidates if item["visual_barrier_detected"]) missing_file_count = sum(int(item.get("missing_file_count") or 0) for item in replay_candidates) tile_file_count = sum(int(item.get("tile_file_count") or 0) for item in replay_candidates) - stale_count = sum(1 for item in replay_candidates if item.get("stale")) + stale_count = sum(1 for item in all_candidates if item.get("stale")) + platform_probe_worker_receipt_count = len(worker_candidates) + source_contract_fallback_count = sum( + 1 for item in worker_candidates if item.get("candidate_status") == "source_contract_fallback" + ) + capture_runtime_gap_count = sum( + 1 + for item in worker_candidates + if (item.get("platform_probe_worker") or {}).get("structured_source_package", {}).get( + "capture_runtime_unavailable" + ) + ) - if invalid_count and invalid_count == len(replay_candidates): + if invalid_count and invalid_count == len(all_candidates): status = "critical" elif eligible_count == 0 or blocked_count or invalid_count or visual_barrier_count: status = "warning" else: status = "ok" - if not replay_candidates: + if worker_candidates and not replay_candidates and not invalid_count: + status = "ok" + if not all_candidates: status = "warning" summary = { - "receipt_count": len(replay_candidates), + "receipt_count": len(all_candidates), + "visual_receipt_count": len(replay_candidates), + "platform_probe_worker_receipt_count": platform_probe_worker_receipt_count, "eligible_count": eligible_count, "blocked_count": blocked_count, "invalid_count": invalid_count, "visual_barrier_count": visual_barrier_count, + "source_contract_fallback_count": source_contract_fallback_count, + "capture_runtime_gap_count": capture_runtime_gap_count, "tile_file_count": tile_file_count, "missing_file_count": missing_file_count, "stale_count": stale_count, - "status_counts": _status_counts(replay_candidates), - "platforms": sorted({str(item.get("platform") or "unknown") for item in replay_candidates}), + "status_counts": _status_counts(all_candidates), + "platforms": sorted({str(item.get("platform") or "unknown") for item in all_candidates}), } return { @@ -289,10 +473,11 @@ def build_pixelrag_rag_candidate_replay_readback( "status": status, "generated_at": now.isoformat(), "artifact_root": str(root), + "platform_probe_worker_receipt_root": str(worker_root), "max_age_hours": max_age, "platform_filter": list(platforms or []), "summary": summary, - "candidates": replay_candidates, + "candidates": all_candidates, "replay_contract": { "internal_rag_target": INTERNAL_RAG_TARGET, "writes_database": False, @@ -303,20 +488,25 @@ def build_pixelrag_rag_candidate_replay_readback( "promotion_gate_required_before_knowledge_write": True, "ocr_or_vlm_required_before_field_extraction": True, "blocked_pages_are_not_product_data": True, + "platform_probe_worker_receipts_are_source_contract_candidates": True, }, "next_machine_action": ( "run_pixelrag_visual_capture_worker" - if not replay_candidates + if not all_candidates else ( "fix_invalid_pixelrag_receipts" if invalid_count and not eligible_count else ( - "run_platform_probe_or_use_structured_api" - if visual_barrier_count and not eligible_count + "run_source_contract_replay_for_platform_probe_worker_receipts" + if source_contract_fallback_count and not eligible_count else ( - "run_ocr_vlm_replay_then_promotion_gate" - if eligible_count - else "refresh_pixelrag_visual_capture_receipt" + "run_platform_probe_or_use_structured_api" + if visual_barrier_count and not eligible_count + else ( + "run_ocr_vlm_replay_then_promotion_gate" + if eligible_count + else "refresh_pixelrag_visual_capture_receipt" + ) ) ) ) diff --git a/tests/test_pixelrag_rag_candidate_replay_service.py b/tests/test_pixelrag_rag_candidate_replay_service.py index de9628a..35c89f4 100644 --- a/tests/test_pixelrag_rag_candidate_replay_service.py +++ b/tests/test_pixelrag_rag_candidate_replay_service.py @@ -40,6 +40,45 @@ def _write_receipt(root, *, platform, manifest_id, title, url, http_status=200): return path +def _write_platform_probe_worker_receipt( + root, + *, + platform, + manifest_id, + worker_status="executed_structured_source_fallback_package", + barrier_type="access_denied", + capture_runtime_unavailable=False, +): + receipt_dir = root / platform / manifest_id + receipt_dir.mkdir(parents=True) + payload = { + "platform": platform, + "manifest_id": manifest_id, + "source_type": "capture_receipt", + "source_receipt_path": f"runtime_artifacts/pixelrag_visual_evidence/{platform}/{manifest_id}/capture_receipt.json", + "worker_status": worker_status, + "probe_status": "structured_source_or_backoff_required", + "barrier_type": barrier_type, + "network_call_performed": False, + "artifact_write_performed": True, + "writes_database": False, + "next_machine_action": "run_structured_source_or_platform_backoff_policy", + "structured_source_package": { + "adapter_code": platform.replace("_tw", ""), + "available": True, + "source_count": 2, + "network_request_allowed": False, + "database_write_allowed": False, + "dry_run_only": True, + "blocked_page_not_product_data": True, + "capture_runtime_unavailable": capture_runtime_unavailable, + }, + } + path = receipt_dir / "platform_probe_worker_receipt.json" + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return path + + def test_pixelrag_rag_candidate_replay_splits_eligible_and_blocked_receipts(tmp_path): from services.pixelrag_rag_candidate_replay_service import ( POLICY, @@ -62,7 +101,10 @@ def test_pixelrag_rag_candidate_replay_splits_eligible_and_blocked_receipts(tmp_ http_status=403, ) - payload = build_pixelrag_rag_candidate_replay_readback(artifact_root=tmp_path) + payload = build_pixelrag_rag_candidate_replay_readback( + artifact_root=tmp_path, + platform_probe_worker_receipt_root=tmp_path / "worker_empty", + ) by_platform = {item["platform"]: item for item in payload["candidates"]} assert payload["policy"] == POLICY @@ -81,6 +123,93 @@ def test_pixelrag_rag_candidate_replay_splits_eligible_and_blocked_receipts(tmp_ assert payload["controlled_apply"]["db_write"] is False +def test_pixelrag_rag_candidate_replay_reads_platform_probe_worker_receipts(tmp_path): + from services.pixelrag_rag_candidate_replay_service import ( + build_pixelrag_rag_candidate_replay_readback, + ) + + worker_root = tmp_path / "worker" + _write_platform_probe_worker_receipt( + worker_root, + platform="coupang_tw", + manifest_id="coupang-403", + barrier_type="access_denied", + ) + _write_platform_probe_worker_receipt( + worker_root, + platform="shopee_tw", + manifest_id="shopee-runtime-gap", + worker_status="capture_runtime_unavailable_structured_fallback_package", + barrier_type="traffic_verification_interstitial", + capture_runtime_unavailable=True, + ) + + payload = build_pixelrag_rag_candidate_replay_readback( + artifact_root=tmp_path / "visual_empty", + platform_probe_worker_receipt_root=worker_root, + platform=("coupang_tw", "shopee_tw"), + ) + by_platform = {item["platform"]: item for item in payload["candidates"]} + + assert payload["status"] == "ok" + assert payload["summary"]["receipt_count"] == 2 + assert payload["summary"]["visual_receipt_count"] == 0 + assert payload["summary"]["platform_probe_worker_receipt_count"] == 2 + assert payload["summary"]["source_contract_fallback_count"] == 2 + assert payload["summary"]["capture_runtime_gap_count"] == 1 + assert payload["next_machine_action"] == ( + "run_source_contract_replay_for_platform_probe_worker_receipts" + ) + assert by_platform["coupang_tw"]["candidate_status"] == "source_contract_fallback" + assert by_platform["coupang_tw"]["eligible_for_source_contract_replay"] is True + assert by_platform["coupang_tw"]["eligible_for_rag_candidate_replay"] is False + assert by_platform["coupang_tw"]["promotion_boundary"][ + "requires_source_contract_before_knowledge_write" + ] is True + assert by_platform["shopee_tw"]["platform_probe_worker"][ + "structured_source_package" + ]["capture_runtime_unavailable"] is True + assert payload["replay_contract"][ + "platform_probe_worker_receipts_are_source_contract_candidates" + ] is True + assert payload["controlled_apply"]["db_write"] is False + + +def test_pixelrag_rag_candidate_replay_prefers_worker_source_contract_handoff(tmp_path): + from services.pixelrag_rag_candidate_replay_service import ( + build_pixelrag_rag_candidate_replay_readback, + ) + + visual_root = tmp_path / "visual" + worker_root = tmp_path / "worker" + _write_receipt( + visual_root, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + _write_platform_probe_worker_receipt( + worker_root, + platform="coupang_tw", + manifest_id="coupang-403", + barrier_type="access_denied", + ) + + payload = build_pixelrag_rag_candidate_replay_readback( + artifact_root=visual_root, + platform_probe_worker_receipt_root=worker_root, + platform="coupang_tw", + ) + + assert payload["summary"]["visual_barrier_count"] == 1 + assert payload["summary"]["source_contract_fallback_count"] == 1 + assert payload["next_machine_action"] == ( + "run_source_contract_replay_for_platform_probe_worker_receipts" + ) + + def test_pixelrag_rag_candidate_replay_cli_outputs_machine_readable_json(tmp_path): _write_receipt( tmp_path, @@ -98,6 +227,8 @@ def test_pixelrag_rag_candidate_replay_cli_outputs_machine_readable_json(tmp_pat str(tmp_path), "--platform", "shopee_tw", + "--platform-probe-worker-receipt-root", + str(tmp_path / "worker_empty"), ], capture_output=True, check=False, @@ -124,6 +255,11 @@ def test_pixelrag_rag_candidate_replay_route_returns_readback(tmp_path, monkeypa url="https://shopee.tw/search?keyword=sunscreen", ) monkeypatch.setattr(service, "DEFAULT_ARTIFACT_ROOT", str(tmp_path)) + monkeypatch.setattr( + service, + "DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT", + str(tmp_path / "worker_empty"), + ) app = Flask(__name__) with app.test_request_context(