diff --git a/config.py b/config.py index 6490c65..9aef9ac 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.763" +SYSTEM_VERSION = "V10.764" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 9565795..d63f856 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.763 +> **適用版本**: V10.764 --- @@ -896,6 +896,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG platform probe worker 缺 capture runtime 時必須自動降級 | V10.761 起若 `run_pixelrag_platform_probe_worker.py --execute --write-receipt` 在 app 容器偵測到 `ModuleNotFoundError: playwright`，worker 會自動產生 `capture_runtime_unavailable_structured_fallback_package` receipt，`network_call=false`、`artifact_write=true`、`writes_database=false`、`primary_human_gate_count=0`，不再把 Shopee/Coupang platform probe runtime gap 變成人工主流程。 | | 2026-07-10 | PixelRAG platform probe worker receipts 必須回到 RAG/source-contract replay | V10.762 起 `/api/ai-automation/pixelrag-rag-candidate-replay` 與 `scripts/ops/report_pixelrag_rag_candidate_replay.py` 必須同時讀 visual capture receipts 與 `platform_probe_worker_receipt.json`；worker receipts 只能成為 `source_contract_fallback` / `eligible_for_source_contract_replay` 候選，並輸出 `platform_probe_worker_receipt_count`、`source_contract_fallback_count`、`capture_runtime_gap_count` 與下一步 `run_source_contract_replay_for_platform_probe_worker_receipts`。此路徑不寫 DB、不寫 `ai_insights`、不寫正式價格表、不把 blocked page 當商品資料，也不得退回人工審核主流程。 | | 2026-07-10 | PixelRAG RAG replay 下一步不得隱藏並行主線 | V10.763 起若同一批 PixelRAG replay 同時存在 ready visual receipt 與 platform probe worker `source_contract_fallback` receipt，`next_machine_action` 必須輸出 `run_ocr_vlm_replay_and_source_contract_replay`，避免只顯示 OCR/VLM 而把 marketplace source-contract 自動化 lane 藏在 summary 裡。 | +| 2026-07-10 | PixelRAG source-contract fallback 必須有 controlled replay worker | V10.764 起 `/api/ai-automation/pixelrag-source-contract-replay-worker`、`scripts/ops/run_pixelrag_source_contract_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG source-contract replay worker / `pixelrag_source_contract_replay_worker` family；worker 只讀 platform probe worker `source_contract_fallback` receipts，驗證 structured package、source count、no-DB、network-deferred、blocked-page-not-product-data、source receipt provenance，execute 模式只寫 `source_contract_replay_receipt.json` artifact。此路徑不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 marketplace source-contract adapter replay preflight。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index bc9885c..181bfd4 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -17,6 +17,7 @@ | Completed | Multi-commerce PixelRAG visual evidence lane for momo, pchome, shopee_tw, coupang_tw, yahoo_shopping_tw, etmall_tw, friday_tw, rakuten_tw | API and CLI can emit expansion plans and marketplace manifests; PChome parser failures can queue manifests. | | Completed | External MCP/RAG capability inventory absorbed into internal governance readback | `/api/ai-automation/external-mcp-rag-integration` and `scripts/ops/report_external_mcp_rag_integration.py` expose 9 capabilities, absorbed/unresolved counts, and runtime flags. | | Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read visual capture receipts and platform probe worker receipts, split eligible / blocked / source-contract-fallback candidates, and expose combined OCR/VLM + source-contract next actions before identity matching and PromotionGate knowledge writes. | +| Completed | PixelRAG source-contract replay worker | `/api/ai-automation/pixelrag-source-contract-replay-worker` and `scripts/ops/run_pixelrag_source_contract_replay_worker.py` turn platform probe worker `source_contract_fallback` receipts into replayable source-contract artifact receipts with provenance, public-boundary, no-DB, and blocked-page-not-product-data guards. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | @@ -33,7 +34,7 @@ | Not started | Ollama-first multimodal embedding benchmark | Verify local Qwen3-VL or equivalent visual embedding on GCP-A -> GCP-B -> 111 before any visual vector retrieval. | | Not started | pgvector-compatible visual evidence metadata | Design metadata-first retrieval without FAISS in production unless ADR approves a different store. | | In progress | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; `run_pixelrag_platform_probe_worker.py --execute --write-receipt` now emits structured-source/backoff receipts without network, DB writes, or product-data promotion. | -| In progress | Marketplace source contracts beyond MOMO/PChome | Platform probe worker receipts now enter RAG candidate replay as source-contract fallback candidates; next machine action is to replay stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten source contracts only after provenance, rate-limit, public boundary, and replay evidence. | +| In progress | Marketplace source contracts beyond MOMO/PChome | Platform probe worker receipts now enter source-contract replay as artifact receipts; next machine action is marketplace source-contract adapter replay preflight for stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten contracts after provenance, rate-limit, public boundary, and replay evidence. | | In progress | Professional product website UI/UX text reduction | Continue compact AI surfaces, visual QA, and user-facing copy guardrails; avoid engineering logs or work-session prose in the UI. | ## Always Enforced diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index 12993ae..dd9985c 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -196,12 +196,16 @@ python scripts/ops/report_pixelrag_platform_probe.py --platform shopee_tw --plat python scripts/ops/run_pixelrag_platform_probe_worker.py --platform shopee_tw --platform coupang_tw python scripts/ops/run_pixelrag_platform_probe_worker.py --platform shopee_tw --execute --write-receipt --limit 1 python scripts/ops/run_pixelrag_platform_probe_worker.py --platform coupang_tw --execute --write-receipt --limit 1 +python scripts/ops/run_pixelrag_source_contract_replay_worker.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_source_contract_replay_worker.py --platform coupang_tw --execute --write-receipt --limit 1 ``` API readback: `/api/ai-automation/pixelrag-vlm-route-readiness` 與 `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。worker 預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。execute 前會自動讀 approved Ollama `/api/tags` 選擇已安裝候選模型；若完全沒有候選，會寫 `model_route_not_ready` artifact receipt，不盲打 generate。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 Platform probe worker API readback: `/api/ai-automation/pixelrag-platform-probe-worker?platform=shopee_tw`。worker 預設為 dry-run，不抓外站、不寫 artifact；`execute=true&write_receipt=true` 才對 Shopee language / traffic / generic landing 執行 public empty-context capture，或對 Coupang 403 / access denied 寫 structured-source/backoff receipt。capture 成功後仍回到 RAG candidate replay，不得直接寫 `ai_insights`、正式價格表或競品價格歷史。 +Source-contract replay worker API readback: `/api/ai-automation/pixelrag-source-contract-replay-worker?platform=coupang_tw`。worker 預設為 dry-run，不抓外站、不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才把 platform probe worker 的 `source_contract_fallback` receipt 轉成 source-contract replay artifact。artifact 仍只是候選合約證據，不得直接寫 `ai_insights`、正式價格表或競品價格歷史。 + Application portfolio： ```bash diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 9f31159..8574ebc 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -905,6 +905,43 @@ def ai_automation_pixelrag_platform_probe_worker_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-source-contract-replay-worker') +@login_required +def ai_automation_pixelrag_source_contract_replay_worker_api(): + """Controlled PixelRAG source-contract replay worker dry-run/execute readback.""" + from services.pixelrag_source_contract_replay_worker_service import ( + run_pixelrag_source_contract_replay_worker, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + execute = str(request.args.get('execute') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + write_receipt = str(request.args.get('write_receipt') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 25, type=int) + return jsonify(run_pixelrag_source_contract_replay_worker( + platform_probe_worker_receipt_root=str( + request.args.get('platform_probe_worker_receipt_root') or '' + ).strip() or None, + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 25, 250)), + execute=execute, + write_receipt=bool(execute and write_receipt), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/run_pixelrag_source_contract_replay_worker.py b/scripts/ops/run_pixelrag_source_contract_replay_worker.py new file mode 100755 index 0000000..eb385af --- /dev/null +++ b/scripts/ops/run_pixelrag_source_contract_replay_worker.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Run or dry-run the PixelRAG source-contract replay worker.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_source_contract_replay_worker_service import ( # noqa: E402 + run_pixelrag_source_contract_replay_worker, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="執行或 dry-run PixelRAG source-contract replay worker。" + ) + parser.add_argument( + "--artifact-root", + help="PixelRAG visual evidence artifact root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--platform-probe-worker-receipt-root", + help="PixelRAG platform probe worker receipt root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--output-root", + help="Source-contract replay artifact receipt output root。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=25, + help="最多處理 source-contract fallback receipt 數。", + ) + parser.add_argument( + "--execute", + action="store_true", + help="執行 source-contract replay；未指定時只做 no-write dry-run。", + ) + parser.add_argument( + "--write-receipt", + action="store_true", + help="execute 後寫入 source-contract replay artifact receipt；不寫 DB。", + ) + args = parser.parse_args() + + payload = run_pixelrag_source_contract_replay_worker( + artifact_root=args.artifact_root, + platform_probe_worker_receipt_root=args.platform_probe_worker_receipt_root, + output_root=args.output_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + execute=args.execute, + write_receipt=bool(args.write_receipt and args.execute), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index b6ae444..1671bff 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -520,6 +520,21 @@ def build_scheduled_automation_health_summary( if not pixelrag_rag_replay or not pixelrag_rag_replay_details: pixelrag_rag_replay = _pixelrag_rag_candidate_replay_check() pixelrag_rag_replay_details = pixelrag_rag_replay.get("details") or {} + pixelrag_source_contract_worker = _find_check( + source_result, + "PixelRAG source-contract replay worker", + ) + pixelrag_source_contract_worker_details = ( + pixelrag_source_contract_worker.get("details") or {} + ) + if ( + not pixelrag_source_contract_worker + or not pixelrag_source_contract_worker_details + ): + pixelrag_source_contract_worker = _pixelrag_source_contract_replay_worker_check() + pixelrag_source_contract_worker_details = ( + pixelrag_source_contract_worker.get("details") or {} + ) pixelrag_ocr_vlm_replay = _find_check(source_result, "PixelRAG OCR/VLM replay contract") pixelrag_ocr_vlm_replay_details = pixelrag_ocr_vlm_replay.get("details") or {} if not pixelrag_ocr_vlm_replay or not pixelrag_ocr_vlm_replay_details: @@ -4582,6 +4597,55 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_source_contract_replay_worker", + "label": "PixelRAG source-contract replay worker", + "status": pixelrag_source_contract_worker.get("status") or "warning", + "summary": ( + pixelrag_source_contract_worker.get("summary") + or "PixelRAG source-contract replay worker has no latest readback." + ), + "next_machine_action": pixelrag_source_contract_worker_details.get( + "next_machine_action" + ) + or "run_pixelrag_source_contract_replay_worker_dry_run", + "details": { + "policy": pixelrag_source_contract_worker_details.get("policy"), + "candidate_count": int( + pixelrag_source_contract_worker_details.get("candidate_count") or 0 + ), + "ready_count": int( + pixelrag_source_contract_worker_details.get("ready_count") or 0 + ), + "blocked_count": int( + pixelrag_source_contract_worker_details.get("blocked_count") or 0 + ), + "dry_run_count": int( + pixelrag_source_contract_worker_details.get("dry_run_count") or 0 + ), + "executed_count": int( + pixelrag_source_contract_worker_details.get("executed_count") or 0 + ), + "receipt_written_count": int( + pixelrag_source_contract_worker_details.get("receipt_written_count") or 0 + ), + "capture_runtime_gap_count": int( + pixelrag_source_contract_worker_details.get("capture_runtime_gap_count") or 0 + ), + "network_call_performed": bool( + pixelrag_source_contract_worker_details.get("network_call_performed") + ), + "model_call_performed": bool( + pixelrag_source_contract_worker_details.get("model_call_performed") + ), + "artifact_write_performed": bool( + pixelrag_source_contract_worker_details.get("artifact_write_performed") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "key": "pixelrag_ocr_vlm_replay", "label": "PixelRAG OCR/VLM replay contract", @@ -12812,6 +12876,67 @@ def _pixelrag_rag_candidate_replay_check() -> Dict[str, Any]: ) +def _pixelrag_source_contract_replay_worker_check() -> Dict[str, Any]: + """Dry-run sentinel for PixelRAG source-contract replay automation.""" + try: + from services.pixelrag_source_contract_replay_worker_service import ( + run_pixelrag_source_contract_replay_worker, + ) + + readback = run_pixelrag_source_contract_replay_worker(execute=False) + summary = readback.get("summary") or {} + candidate_count = int(summary.get("candidate_count") or 0) + ready_count = int(summary.get("ready_count") or 0) + blocked_count = int(summary.get("blocked_count") or 0) + dry_run_count = int(summary.get("dry_run_count") or 0) + executed_count = int(summary.get("executed_count") or 0) + receipt_written_count = int(summary.get("receipt_written_count") or 0) + status = readback.get("status") or "warning" + summary_text = ( + f"PixelRAG source-contract replay worker candidates={candidate_count}, " + f"ready={ready_count}, blocked={blocked_count}, dry_run={dry_run_count}, " + f"executed={executed_count}, receipts={receipt_written_count}" + ) + return _check( + "PixelRAG source-contract replay worker", + status, + summary_text, + { + "policy": readback.get("policy"), + "candidate_count": candidate_count, + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "capture_runtime_gap_count": int( + summary.get("capture_runtime_gap_count") or 0 + ), + "guard_failed_count": int(summary.get("guard_failed_count") or 0), + "network_call_performed": bool(summary.get("network_call_performed")), + "model_call_performed": bool(summary.get("model_call_performed")), + "artifact_write_performed": bool( + summary.get("artifact_write_performed") + ), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG source-contract replay worker", + "critical", + f"PixelRAG source-contract replay worker 無法執行 dry-run：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def _pixelrag_ocr_vlm_replay_check() -> Dict[str, Any]: """Read-only sentinel for PixelRAG receipts becoming OCR/VLM replay contracts.""" try: @@ -13143,6 +13268,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _sitewide_visual_qa_check(), _external_mcp_rag_integration_check(), _pixelrag_rag_candidate_replay_check(), + _pixelrag_source_contract_replay_worker_check(), _pixelrag_ocr_vlm_replay_check(), _pixelrag_vlm_route_readiness_check(), _pixelrag_vlm_replay_worker_check(), diff --git a/services/pixelrag_source_contract_replay_worker_service.py b/services/pixelrag_source_contract_replay_worker_service.py new file mode 100644 index 0000000..19fd468 --- /dev/null +++ b/services/pixelrag_source_contract_replay_worker_service.py @@ -0,0 +1,364 @@ +"""Controlled PixelRAG source-contract replay worker. + +This worker turns platform-probe structured fallback receipts into replayable +source-contract receipts. It does not fetch marketplace pages, call models, +write databases, or promote price truth. +""" + +from __future__ import annotations + +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.pixelrag_crawler_integration_service import DEFAULT_ARTIFACT_MAX_AGE_HOURS +from services.pixelrag_rag_candidate_replay_service import ( + DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT, + build_pixelrag_rag_candidate_replay_readback, +) + + +POLICY = "controlled_pixelrag_source_contract_replay_worker_v1" +SOURCE_CONTRACT_VERSION = "pixelrag_marketplace_source_contract_v1" +DEFAULT_LIMIT = 25 +DEFAULT_OUTPUT_ROOT = os.getenv( + "PIXELRAG_SOURCE_CONTRACT_REPLAY_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_source_contract_replay_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_source_contract_replay_receipts", +) + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...]: + if isinstance(platform, str): + value = platform.strip().lower() + return (value,) if value else () + return tuple( + str(item or "").strip().lower() + for item in (platform or ()) + if str(item or "").strip() + ) + + +def _safe_segment(value: Any) -> str: + text = str(value or "unknown").strip().lower() + text = re.sub(r"[^a-z0-9._-]+", "-", text) + return text.strip("-") or "unknown" + + +def _source_contract_candidates(readback: Mapping[str, Any]) -> list[dict[str, Any]]: + candidates: list[dict[str, Any]] = [] + for item in list(readback.get("candidates") or []): + if not isinstance(item, Mapping): + continue + if item.get("candidate_status") != "source_contract_fallback": + continue + if not item.get("eligible_for_source_contract_replay"): + continue + candidates.append(dict(item)) + return candidates + + +def _contract_checks(candidate: Mapping[str, Any]) -> dict[str, bool]: + worker = candidate.get("platform_probe_worker") + if not isinstance(worker, Mapping): + worker = {} + package = worker.get("structured_source_package") + if not isinstance(package, Mapping): + package = {} + return { + "structured_source_package_available": bool(package.get("available")), + "source_count_positive": int(package.get("source_count") or 0) > 0, + "database_write_disallowed": not bool(package.get("database_write_allowed")), + "network_request_deferred": not bool(package.get("network_request_allowed")), + "dry_run_only_contract": bool(package.get("dry_run_only")), + "blocked_page_not_product_data": bool(package.get("blocked_page_not_product_data")), + "source_worker_receipt_path_present": bool(candidate.get("receipt_path")), + "source_receipt_path_present": bool(worker.get("source_receipt_path")), + } + + +def _source_contract_preview(candidate: Mapping[str, Any]) -> dict[str, Any]: + worker = candidate.get("platform_probe_worker") + if not isinstance(worker, Mapping): + worker = {} + package = worker.get("structured_source_package") + if not isinstance(package, Mapping): + package = {} + platform = str(candidate.get("platform") or "unknown") + manifest_id = str(candidate.get("manifest_id") or "unknown") + adapter_code = str(package.get("adapter_code") or platform) + return { + "source_contract_version": SOURCE_CONTRACT_VERSION, + "contract_id": f"{platform}:{manifest_id}:{adapter_code}", + "adapter_code": adapter_code, + "source_count": int(package.get("source_count") or 0), + "capture_runtime_unavailable": bool(package.get("capture_runtime_unavailable")), + "barrier_type": worker.get("barrier_type"), + "expected_offer_fields": [ + "platform", + "platform_product_id", + "title", + "url", + "image_url", + "price", + "currency", + "availability", + "seller", + "promotion_badges", + "shipping_badges", + "evidence_time", + "source_method", + "confidence", + ], + "required_before_data_promotion": [ + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", + ], + } + + +def _worker_item( + candidate: Mapping[str, Any], + *, + execute: bool, +) -> dict[str, Any]: + checks = _contract_checks(candidate) + check_count = len(checks) + pass_count = sum(1 for passed in checks.values() if passed) + ready = pass_count == check_count + worker = candidate.get("platform_probe_worker") + if not isinstance(worker, Mapping): + worker = {} + package = worker.get("structured_source_package") + if not isinstance(package, Mapping): + package = {} + status = ( + "executed_source_contract_replay_ready" + if execute and ready + else ( + "dry_run_ready_for_source_contract_replay" + if ready + else "skipped_source_contract_guard_failed" + ) + ) + return { + "worker_status": status, + "platform": candidate.get("platform"), + "manifest_id": candidate.get("manifest_id"), + "source_type": "platform_probe_worker_receipt", + "source_worker_receipt_path": candidate.get("receipt_path"), + "source_receipt_path": worker.get("source_receipt_path"), + "source_contract_status": "ready" if ready else "blocked", + "ready_for_execution": ready, + "execute": bool(execute), + "adapter_code": package.get("adapter_code"), + "structured_source_count": int(package.get("source_count") or 0), + "capture_runtime_unavailable": bool(package.get("capture_runtime_unavailable")), + "barrier_type": worker.get("barrier_type"), + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": False, + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + "source_contract_checks": checks, + "source_contract_check_count": check_count, + "source_contract_check_pass_count": pass_count, + "source_contract_preview": _source_contract_preview(candidate), + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "source_contract_is_candidate_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": ( + "run_marketplace_source_contract_adapter_replay_preflight" + if execute and ready + else ( + "run_pixelrag_source_contract_replay_worker_execute" + if ready + else "repair_source_contract_fallback_package" + ) + ), + } + + +def _write_replay_receipt(*, output_root: Path, item: Mapping[str, Any]) -> str: + target = ( + output_root + / _safe_segment(item.get("platform")) + / _safe_segment(item.get("manifest_id")) + / "source_contract_replay_receipt.json" + ) + target.parent.mkdir(parents=True, exist_ok=True) + receipt = dict(item) + receipt["artifact_write_performed"] = True + receipt["receipt_path"] = str(target) + receipt["generated_at"] = datetime.now(timezone.utc).isoformat() + receipt["policy"] = POLICY + target.write_text( + json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + return str(target) + + +def run_pixelrag_source_contract_replay_worker( + *, + artifact_root: str | Path | None = None, + platform_probe_worker_receipt_root: str | Path | None = None, + output_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, + execute: bool = False, + write_receipt: bool = False, +) -> dict[str, Any]: + """Run or dry-run source-contract replay for platform probe worker receipts.""" + output = Path(output_root or DEFAULT_OUTPUT_ROOT) + worker_root = Path( + platform_probe_worker_receipt_root + or DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT + ) + platforms = _normalise_platforms(platform) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + readback = build_pixelrag_rag_candidate_replay_readback( + artifact_root=artifact_root, + platform_probe_worker_receipt_root=worker_root, + platform=platforms, + max_age_hours=max_age, + limit=item_limit, + ) + candidates = _source_contract_candidates(readback) + worker_items: list[dict[str, Any]] = [] + for candidate in candidates: + item = _worker_item(candidate, execute=execute) + if execute and write_receipt and item.get("ready_for_execution"): + item["receipt_path"] = _write_replay_receipt(output_root=output, item=item) + item["artifact_write_performed"] = True + worker_items.append(item) + + ready_count = sum(1 for item in worker_items if item.get("ready_for_execution")) + blocked_count = len(worker_items) - ready_count + dry_run_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_") + ) + executed_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_") + ) + receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path")) + capture_runtime_gap_count = sum( + 1 for item in worker_items if item.get("capture_runtime_unavailable") + ) + guard_failed_count = sum( + 1 for item in worker_items if item.get("source_contract_status") != "ready" + ) + + if not worker_items: + status = "warning" + elif guard_failed_count: + status = "warning" + else: + status = "ok" + + if not worker_items: + next_action = "run_pixelrag_platform_probe_worker_or_rag_candidate_replay" + elif not execute and ready_count: + next_action = "run_pixelrag_source_contract_replay_worker_execute" + elif execute and receipt_written_count: + next_action = "run_marketplace_source_contract_adapter_replay_preflight" + elif guard_failed_count: + next_action = "repair_source_contract_fallback_package" + else: + next_action = "refresh_pixelrag_source_contract_replay_candidates" + + summary = { + "candidate_count": len(worker_items), + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "capture_runtime_gap_count": capture_runtime_gap_count, + "guard_failed_count": guard_failed_count, + "platforms": sorted({str(item.get("platform") or "unknown") for item in worker_items}), + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": bool(receipt_written_count), + "writes_database_count": 0, + "primary_human_gate_count": 0, + } + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": datetime.now(timezone.utc).isoformat(), + "source_contract_version": SOURCE_CONTRACT_VERSION, + "artifact_root": str(artifact_root or readback.get("artifact_root") or ""), + "platform_probe_worker_receipt_root": str(worker_root), + "output_root": str(output), + "platform_filter": list(platforms), + "max_age_hours": max_age, + "limit": item_limit, + "execute": bool(execute), + "write_receipt": bool(write_receipt and execute), + "summary": summary, + "worker_items": worker_items, + "source_readback": { + "policy": readback.get("policy"), + "status": readback.get("status"), + "summary": readback.get("summary"), + "next_machine_action": readback.get("next_machine_action"), + }, + "controlled_apply": { + "network_call": False, + "model_call": False, + "artifact_write": bool(receipt_written_count), + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "secret_read": False, + "raw_cookie_or_session_read": False, + "credentialed_session_allowed": False, + "login_allowed": False, + "production_price_write": False, + "primary_human_gate_count": 0, + }, + "promotion_boundary": { + "writes_ai_insights": False, + "writes_price_tables": False, + "source_contract_is_candidate_only": True, + "blocked_pages_are_not_product_data": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "DEFAULT_OUTPUT_ROOT", + "POLICY", + "SOURCE_CONTRACT_VERSION", + "run_pixelrag_source_contract_replay_worker", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index 990f125..34df2c8 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1310,6 +1310,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -1319,7 +1320,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 35, "warning": 1, "critical": 1, "total": 37} + assert result["summary"] == {"ok": 36, "warning": 1, "critical": 1, "total": 38} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3848,6 +3849,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -3908,7 +3910,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 37, "warning": 0, "critical": 0, "total": 37}, + "summary": {"ok": 38, "warning": 0, "critical": 0, "total": 38}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4017,6 +4019,27 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "next_machine_action": "run_ocr_vlm_replay_then_promotion_gate", }, }, + { + "name": "PixelRAG source-contract replay worker", + "status": "ok", + "summary": "PixelRAG source-contract replay worker candidates=1, ready=1, blocked=0, dry_run=1, executed=0, receipts=0", + "details": { + "policy": "controlled_pixelrag_source_contract_replay_worker_v1", + "candidate_count": 1, + "ready_count": 1, + "blocked_count": 0, + "dry_run_count": 1, + "executed_count": 0, + "receipt_written_count": 0, + "capture_runtime_gap_count": 0, + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": "run_pixelrag_source_contract_replay_worker_execute", + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "name": "PixelRAG OCR/VLM replay contract", "status": "ok", @@ -4131,7 +4154,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 34 + assert summary["summary"]["total"] == 35 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -6463,6 +6486,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag replay ok", )) + monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check( + "PixelRAG source-contract replay worker", + "ok", + "pixelrag source contract ok", + )) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check( "PixelRAG OCR/VLM replay contract", "ok", @@ -6503,7 +6531,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 37 + assert result["summary"]["total"] == 38 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_source_contract_replay_worker_service.py b/tests/test_pixelrag_source_contract_replay_worker_service.py new file mode 100644 index 0000000..be19278 --- /dev/null +++ b/tests/test_pixelrag_source_contract_replay_worker_service.py @@ -0,0 +1,169 @@ +import json +import subprocess +import sys + +from tests.test_pixelrag_rag_candidate_replay_service import ( + _write_platform_probe_worker_receipt, +) + + +def test_pixelrag_source_contract_replay_worker_dry_run_reads_worker_receipts(tmp_path): + from services.pixelrag_source_contract_replay_worker_service import ( + POLICY, + run_pixelrag_source_contract_replay_worker, + ) + + worker_root = tmp_path / "worker" + _write_platform_probe_worker_receipt( + worker_root, + platform="coupang_tw", + manifest_id="coupang-403", + barrier_type="access_denied", + ) + _write_platform_probe_worker_receipt( + worker_root, + platform="shopee_tw", + manifest_id="shopee-runtime-gap", + worker_status="capture_runtime_unavailable_structured_fallback_package", + barrier_type="traffic_verification_interstitial", + capture_runtime_unavailable=True, + ) + + payload = run_pixelrag_source_contract_replay_worker( + artifact_root=tmp_path / "visual_empty", + platform_probe_worker_receipt_root=worker_root, + platform=("coupang_tw", "shopee_tw"), + ) + by_platform = {item["platform"]: item for item in payload["worker_items"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "ok" + assert payload["execute"] is False + assert payload["summary"]["candidate_count"] == 2 + assert payload["summary"]["ready_count"] == 2 + assert payload["summary"]["dry_run_count"] == 2 + assert payload["summary"]["capture_runtime_gap_count"] == 1 + assert payload["summary"]["writes_database_count"] == 0 + assert payload["controlled_apply"]["network_call"] is False + assert payload["controlled_apply"]["writes_database"] is False + assert payload["controlled_apply"]["primary_human_gate_count"] == 0 + assert payload["next_machine_action"] == ( + "run_pixelrag_source_contract_replay_worker_execute" + ) + assert by_platform["coupang_tw"]["worker_status"] == ( + "dry_run_ready_for_source_contract_replay" + ) + assert by_platform["coupang_tw"]["source_contract_checks"][ + "blocked_page_not_product_data" + ] is True + assert by_platform["shopee_tw"]["capture_runtime_unavailable"] is True + + +def test_pixelrag_source_contract_replay_worker_execute_writes_receipts(tmp_path): + from services.pixelrag_source_contract_replay_worker_service import ( + run_pixelrag_source_contract_replay_worker, + ) + + worker_root = tmp_path / "worker" + output_root = tmp_path / "source_contract_receipts" + _write_platform_probe_worker_receipt( + worker_root, + platform="coupang_tw", + manifest_id="coupang-403", + barrier_type="access_denied", + ) + + payload = run_pixelrag_source_contract_replay_worker( + artifact_root=tmp_path / "visual_empty", + platform_probe_worker_receipt_root=worker_root, + output_root=output_root, + platform="coupang_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["executed_count"] == 1 + assert payload["summary"]["receipt_written_count"] == 1 + assert payload["controlled_apply"]["artifact_write"] is True + assert payload["controlled_apply"]["writes_database"] is False + assert payload["next_machine_action"] == ( + "run_marketplace_source_contract_adapter_replay_preflight" + ) + receipt_path = ( + output_root + / "coupang_tw" + / "coupang-403" + / "source_contract_replay_receipt.json" + ) + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "executed_source_contract_replay_ready" + assert receipt["artifact_write_performed"] is True + assert receipt["writes_database"] is False + assert receipt["source_contract_preview"]["source_contract_version"] == ( + "pixelrag_marketplace_source_contract_v1" + ) + + +def test_pixelrag_source_contract_replay_worker_cli_outputs_json(tmp_path): + worker_root = tmp_path / "worker" + _write_platform_probe_worker_receipt( + worker_root, + platform="coupang_tw", + manifest_id="coupang-403", + barrier_type="access_denied", + ) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/run_pixelrag_source_contract_replay_worker.py", + "--artifact-root", + str(tmp_path / "visual_empty"), + "--platform-probe-worker-receipt-root", + str(worker_root), + "--platform", + "coupang_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False + + +def test_pixelrag_source_contract_replay_worker_route_returns_readback(tmp_path, monkeypatch): + from flask import Flask + from routes import system_public_routes as routes + from services import pixelrag_source_contract_replay_worker_service as service + + worker_root = tmp_path / "worker" + _write_platform_probe_worker_receipt( + worker_root, + platform="coupang_tw", + manifest_id="coupang-403", + barrier_type="access_denied", + ) + monkeypatch.setattr( + service, + "DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT", + str(worker_root), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-source-contract-replay-worker?platform=coupang_tw" + ): + response = routes.ai_automation_pixelrag_source_contract_replay_worker_api.__wrapped__() + payload = response.get_json() + + assert payload["policy"] == "controlled_pixelrag_source_contract_replay_worker_v1" + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False