From c62ec7b504c7f74ce8a6a3b5312e30c1e6947513 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 12:51:56 +0800 Subject: [PATCH] feat(ai): add PixelRAG marketplace candidate knowledge replay --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 4 +- .../ai_automation_mainline_work_items.md | 3 +- docs/guides/browse_sh_crawler_playbook.md | 4 + routes/system_public_routes.py | 37 + ..._marketplace_candidate_knowledge_replay.py | 79 ++ services/ai_automation_smoke_service.py | 174 +++++ ...lace_candidate_knowledge_replay_service.py | 729 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 42 +- ...lace_candidate_knowledge_replay_service.py | 325 ++++++++ 10 files changed, 1392 insertions(+), 7 deletions(-) create mode 100644 scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py create mode 100644 services/pixelrag_marketplace_candidate_knowledge_replay_service.py create mode 100644 tests/test_pixelrag_marketplace_candidate_knowledge_replay_service.py diff --git a/config.py b/config.py index 387623e..3fa492f 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.769" +SYSTEM_VERSION = "V10.770" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index ffdd70c..9d2b355 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.769 +> **適用版本**: V10.770 --- @@ -125,6 +125,7 @@ - 2026-07-10 起 PixelRAG marketplace identity matcher replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-identity-matcher-replay` 與 `scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py` 預設只讀 adapter dry-run receipts 並產生 deterministic identity key / match contract；`execute=true&write_receipt=true` 只寫 `marketplace_identity_matcher_replay_receipt.json` artifact，輸出 identity candidate contracts、source provenance、PromotionGate readiness 與 no-write 邊界。此 worker 不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；identity replay receipt 只允許進 PromotionGate replay，不得直接進價格表或候選知識層。 - 2026-07-10 起 PixelRAG marketplace PromotionGate replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-promotion-gate-replay` 與 `scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py` 預設只讀 identity matcher receipts 並產生 deterministic PromotionGate contract；`execute=true&write_receipt=true` 只寫 `marketplace_promotion_gate_replay_receipt.json` artifact，輸出 promotion candidate contracts、embedding signature guard readiness、source provenance 與 no-write 邊界。此 worker 不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；PromotionGate receipt 只允許進 embedding signature guard replay，不得直接進價格表或候選知識層。 - 2026-07-10 起 PixelRAG marketplace embedding signature guard replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay` 與 `scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py` 預設只讀 PromotionGate receipts 並產生 deterministic embedding signature guard contract；`execute=true&write_receipt=true` 只寫 `marketplace_embedding_signature_guard_replay_receipt.json` artifact，輸出 expected embedding signature、signature guard contracts、candidate knowledge replay readiness 與 no-write 邊界。此 worker 不產生 embedding、不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；embedding signature guard receipt 只允許進 candidate knowledge replay，不得直接進價格表或正式知識層。 +- 2026-07-10 起 PixelRAG marketplace candidate knowledge replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay` 與 `scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py` 預設只讀 embedding signature guard receipts 並產生 deterministic internal RAG candidate knowledge contract；`execute=true&write_receipt=true` 只寫 `marketplace_candidate_knowledge_replay_receipt.json` artifact，輸出 candidate knowledge text、internal RAG preview readiness、source provenance 與 no-write 邊界。此 worker 不產生 embedding、不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；candidate knowledge receipt 只允許進 internal RAG candidate canary，不得直接進正式知識層或價格表。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -906,6 +907,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG marketplace identity matcher replay 必須先產生 no-write match receipt | V10.767 起 `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay`、`scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace identity matcher replay / `pixelrag_marketplace_identity_matcher_replay` family；replay 只讀 `marketplace_adapter_dry_run_receipt.json`，驗證 adapter dry-run ready、identity key fields、candidate contracts、required gates、no-DB/no-network/no-model、source provenance 與 price-write blocked。execute 模式只寫 `marketplace_identity_matcher_replay_receipt.json` artifact，產生 deterministic identity candidate contracts，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 PromotionGate replay。 | | 2026-07-10 | PixelRAG marketplace PromotionGate replay 必須先產生 no-write promotion receipt | V10.768 起 `/api/ai-automation/pixelrag-marketplace-promotion-gate-replay`、`scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace PromotionGate replay / `pixelrag_marketplace_promotion_gate_replay` family；replay 只讀 `marketplace_identity_matcher_replay_receipt.json`，驗證 identity matcher ready、identity checks all passed、required promotion gates、embedding signature guard readiness、no-DB/no-network/no-model、source provenance 與 price-write blocked。execute 模式只寫 `marketplace_promotion_gate_replay_receipt.json` artifact，產生 deterministic promotion candidate contracts，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 embedding signature guard replay。 | | 2026-07-10 | PixelRAG marketplace embedding signature guard replay 必須先產生 no-write signature receipt | V10.769 起 `/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay`、`scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace embedding signature guard replay / `pixelrag_marketplace_embedding_signature_guard_replay` family；replay 只讀 `marketplace_promotion_gate_replay_receipt.json`，驗證 PromotionGate ready、PromotionGate checks all passed、expected embedding signature、signature guard contracts、candidate knowledge readiness、no-DB/no-network/no-model、no embedding generation、source provenance 與 price-write blocked。execute 模式只寫 `marketplace_embedding_signature_guard_replay_receipt.json` artifact，產生 deterministic signature guard contracts，不產生 embedding、不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 candidate knowledge replay。 | +| 2026-07-10 | PixelRAG marketplace candidate knowledge replay 必須先產生 no-write internal RAG candidate receipt | V10.770 起 `/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay`、`scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace candidate knowledge replay / `pixelrag_marketplace_candidate_knowledge_replay` family；replay 只讀 `marketplace_embedding_signature_guard_replay_receipt.json`，驗證 signature guard ready、signature checks all passed、expected embedding signature、knowledge candidate text、internal RAG preview readiness、no-DB/no-network/no-model、no embedding generation、source provenance、AI insights write blocked 與 price-write blocked。execute 模式只寫 `marketplace_candidate_knowledge_replay_receipt.json` artifact，產生 deterministic internal RAG candidate preview contracts，不產生 embedding、不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 internal RAG candidate canary。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 219e3d9..4ed8ca6 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -23,11 +23,12 @@ | Completed | PixelRAG marketplace identity matcher replay | `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay` and `scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py` turn adapter dry-run receipts into deterministic identity candidate contracts and source provenance checks, with price writes blocked until PromotionGate replay. | | Completed | PixelRAG marketplace PromotionGate replay | `/api/ai-automation/pixelrag-marketplace-promotion-gate-replay` and `scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py` turn identity matcher receipts into deterministic PromotionGate contracts and embedding-signature readiness, with price writes still blocked. | | Completed | PixelRAG marketplace embedding signature guard replay | `/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay` and `scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py` turn PromotionGate receipts into deterministic embedding-signature guard contracts and candidate-knowledge readiness, with DB and price writes still blocked. | +| Completed | PixelRAG marketplace candidate knowledge replay | `/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay` and `scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py` turn embedding-signature guard receipts into deterministic internal RAG candidate preview contracts, with DB, `ai_insights`, and price writes still blocked until canary. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | | Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture when runtime is available, missing Playwright auto-falls back to structured-source/backoff receipt, and Coupang 403/access denied goes to structured-source/backoff artifact receipts. | -| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, PixelRAG platform probe worker, source-contract replay, adapter preflight, adapter dry-run, identity matcher replay, PromotionGate replay, and embedding signature guard replay families. | +| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, PixelRAG platform probe worker, source-contract replay, adapter preflight, adapter dry-run, identity matcher replay, PromotionGate replay, embedding signature guard replay, and candidate knowledge replay families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | ## P1 diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index ac79f76..123bd27 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -208,6 +208,8 @@ python scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py --platform python scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py --platform coupang_tw --execute --write-receipt --limit 1 python scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py --platform shopee_tw --platform coupang_tw python scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py --platform coupang_tw --execute --write-receipt --limit 1 +python scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py --platform coupang_tw --execute --write-receipt --limit 1 ``` API readback: `/api/ai-automation/pixelrag-vlm-route-readiness` 與 `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。worker 預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。execute 前會自動讀 approved Ollama `/api/tags` 選擇已安裝候選模型；若完全沒有候選，會寫 `model_route_not_ready` artifact receipt，不盲打 generate。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 @@ -226,6 +228,8 @@ Marketplace PromotionGate replay API readback: `/api/ai-automation/pixelrag-mark Marketplace embedding signature guard replay API readback: `/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay?platform=coupang_tw`。embedding signature guard replay 預設只讀 PromotionGate receipts 並建立 deterministic signature guard contracts；`execute=true&write_receipt=true` 只寫 embedding signature guard artifact receipt，不產生 embedding、不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表。signature guard receipt 下一步只能進 candidate knowledge replay，不得直接變成價格資料。 +Marketplace candidate knowledge replay API readback: `/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay?platform=coupang_tw`。candidate knowledge replay 預設只讀 embedding signature guard receipts 並建立 deterministic internal RAG candidate preview contracts；`execute=true&write_receipt=true` 只寫 candidate knowledge artifact receipt，不產生 embedding、不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表。candidate knowledge receipt 下一步只能進 internal RAG candidate canary，不得直接變成正式知識庫或價格資料。 + Application portfolio： ```bash diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index ef55f49..7f1ab0e 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -1127,6 +1127,43 @@ def ai_automation_pixelrag_marketplace_embedding_signature_guard_replay_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay') +@login_required +def ai_automation_pixelrag_marketplace_candidate_knowledge_replay_api(): + """Controlled PixelRAG marketplace candidate knowledge readback.""" + from services.pixelrag_marketplace_candidate_knowledge_replay_service import ( + run_pixelrag_marketplace_candidate_knowledge_replay, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + execute = str(request.args.get('execute') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + write_receipt = str(request.args.get('write_receipt') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 25, type=int) + return jsonify(run_pixelrag_marketplace_candidate_knowledge_replay( + embedding_signature_guard_receipt_root=str( + request.args.get('embedding_signature_guard_receipt_root') or '' + ).strip() or None, + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 25, 250)), + execute=execute, + write_receipt=bool(execute and write_receipt), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py b/scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py new file mode 100644 index 0000000..a532303 --- /dev/null +++ b/scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Run or dry-run PixelRAG marketplace candidate knowledge replay.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_marketplace_candidate_knowledge_replay_service import ( # noqa: E402 + run_pixelrag_marketplace_candidate_knowledge_replay, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="執行或 dry-run PixelRAG marketplace candidate knowledge replay。" + ) + parser.add_argument( + "--embedding-signature-guard-receipt-root", + help="Marketplace embedding signature guard receipt root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--output-root", + help="Marketplace candidate knowledge replay artifact receipt output root。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=25, + help="最多處理 embedding signature guard receipt 數。", + ) + parser.add_argument( + "--execute", + action="store_true", + help="執行 candidate knowledge replay；未指定時只做 no-write dry-run readiness。", + ) + parser.add_argument( + "--write-receipt", + action="store_true", + help="execute 後寫入 candidate knowledge replay artifact receipt；不寫 DB。", + ) + args = parser.parse_args() + + payload = run_pixelrag_marketplace_candidate_knowledge_replay( + embedding_signature_guard_receipt_root=( + args.embedding_signature_guard_receipt_root + ), + output_root=args.output_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + execute=args.execute, + write_receipt=bool(args.write_receipt and args.execute), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index 2f7a83f..ff37fac 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -620,6 +620,23 @@ def build_scheduled_automation_health_summary( pixelrag_marketplace_embedding_signature_details = ( pixelrag_marketplace_embedding_signature.get("details") or {} ) + pixelrag_marketplace_candidate_knowledge = _find_check( + source_result, + "PixelRAG marketplace candidate knowledge replay", + ) + pixelrag_marketplace_candidate_knowledge_details = ( + pixelrag_marketplace_candidate_knowledge.get("details") or {} + ) + if ( + not pixelrag_marketplace_candidate_knowledge + or not pixelrag_marketplace_candidate_knowledge_details + ): + pixelrag_marketplace_candidate_knowledge = ( + _pixelrag_marketplace_candidate_knowledge_replay_check() + ) + pixelrag_marketplace_candidate_knowledge_details = ( + pixelrag_marketplace_candidate_knowledge.get("details") or {} + ) pixelrag_ocr_vlm_replay = _find_check(source_result, "PixelRAG OCR/VLM replay contract") pixelrag_ocr_vlm_replay_details = pixelrag_ocr_vlm_replay.get("details") or {} if not pixelrag_ocr_vlm_replay or not pixelrag_ocr_vlm_replay_details: @@ -5010,6 +5027,80 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_marketplace_candidate_knowledge_replay", + "label": "PixelRAG marketplace candidate knowledge replay", + "status": ( + pixelrag_marketplace_candidate_knowledge.get("status") or "warning" + ), + "summary": ( + pixelrag_marketplace_candidate_knowledge.get("summary") + or "PixelRAG marketplace candidate knowledge replay has no latest readback." + ), + "next_machine_action": pixelrag_marketplace_candidate_knowledge_details.get( + "next_machine_action" + ) + or "run_marketplace_candidate_knowledge_replay", + "details": { + "policy": pixelrag_marketplace_candidate_knowledge_details.get("policy"), + "candidate_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("candidate_count") or 0 + ), + "ready_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("ready_count") or 0 + ), + "blocked_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("blocked_count") or 0 + ), + "dry_run_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("dry_run_count") or 0 + ), + "executed_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("executed_count") or 0 + ), + "receipt_written_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("receipt_written_count") or 0 + ), + "capture_runtime_gap_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("capture_runtime_gap_count") or 0 + ), + "knowledge_candidate_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("knowledge_candidate_count") or 0 + ), + "internal_rag_ready_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("internal_rag_ready_count") or 0 + ), + "expected_embedding_signatures": list( + pixelrag_marketplace_candidate_knowledge_details.get( + "expected_embedding_signatures" + ) + or [] + ), + "network_call_performed": bool( + pixelrag_marketplace_candidate_knowledge_details.get("network_call_performed") + ), + "model_call_performed": bool( + pixelrag_marketplace_candidate_knowledge_details.get("model_call_performed") + ), + "embedding_generation_performed": bool( + pixelrag_marketplace_candidate_knowledge_details.get( + "embedding_generation_performed" + ) + ), + "artifact_write_performed": bool( + pixelrag_marketplace_candidate_knowledge_details.get("artifact_write_performed") + ), + "writes_ai_insights": bool( + pixelrag_marketplace_candidate_knowledge_details.get("writes_ai_insights") + ), + "writes_price_tables": bool( + pixelrag_marketplace_candidate_knowledge_details.get("writes_price_tables") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "key": "pixelrag_ocr_vlm_replay", "label": "PixelRAG OCR/VLM replay contract", @@ -13649,6 +13740,88 @@ def _pixelrag_marketplace_embedding_signature_guard_replay_check() -> Dict[str, ) +def _pixelrag_marketplace_candidate_knowledge_replay_check() -> Dict[str, Any]: + """Dry-run sentinel for PixelRAG marketplace candidate knowledge replay.""" + try: + from services.pixelrag_marketplace_candidate_knowledge_replay_service import ( + run_pixelrag_marketplace_candidate_knowledge_replay, + ) + + readback = run_pixelrag_marketplace_candidate_knowledge_replay( + execute=False, + ) + summary = readback.get("summary") or {} + candidate_count = int(summary.get("candidate_count") or 0) + ready_count = int(summary.get("ready_count") or 0) + blocked_count = int(summary.get("blocked_count") or 0) + dry_run_count = int(summary.get("dry_run_count") or 0) + executed_count = int(summary.get("executed_count") or 0) + receipt_written_count = int(summary.get("receipt_written_count") or 0) + knowledge_candidate_count = int( + summary.get("knowledge_candidate_count") or 0 + ) + internal_rag_ready_count = int( + summary.get("internal_rag_ready_count") or 0 + ) + signatures = list(summary.get("expected_embedding_signatures") or []) + status = readback.get("status") or "warning" + signature_text = ",".join(str(item) for item in signatures) or "none" + summary_text = ( + f"PixelRAG marketplace candidate knowledge replay " + f"candidates={candidate_count}, ready={ready_count}, " + f"blocked={blocked_count}, dry_run={dry_run_count}, " + f"executed={executed_count}, receipts={receipt_written_count}, " + f"knowledge_candidates={knowledge_candidate_count}, " + f"internal_rag_ready={internal_rag_ready_count}, signatures={signature_text}" + ) + return _check( + "PixelRAG marketplace candidate knowledge replay", + status, + summary_text, + { + "policy": readback.get("policy"), + "candidate_count": candidate_count, + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "capture_runtime_gap_count": int( + summary.get("capture_runtime_gap_count") or 0 + ), + "guard_failed_count": int(summary.get("guard_failed_count") or 0), + "knowledge_candidate_count": knowledge_candidate_count, + "internal_rag_ready_count": internal_rag_ready_count, + "expected_embedding_signatures": signatures, + "network_call_performed": bool(summary.get("network_call_performed")), + "model_call_performed": bool(summary.get("model_call_performed")), + "embedding_generation_performed": bool( + summary.get("embedding_generation_performed") + ), + "artifact_write_performed": bool( + summary.get("artifact_write_performed") + ), + "writes_ai_insights": bool(summary.get("writes_ai_insights")), + "writes_price_tables": bool(summary.get("writes_price_tables")), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG marketplace candidate knowledge replay", + "critical", + f"PixelRAG marketplace candidate knowledge replay 無法執行 dry-run：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def _pixelrag_ocr_vlm_replay_check() -> Dict[str, Any]: """Read-only sentinel for PixelRAG receipts becoming OCR/VLM replay contracts.""" try: @@ -13986,6 +14159,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _pixelrag_marketplace_identity_matcher_replay_check(), _pixelrag_marketplace_promotion_gate_replay_check(), _pixelrag_marketplace_embedding_signature_guard_replay_check(), + _pixelrag_marketplace_candidate_knowledge_replay_check(), _pixelrag_ocr_vlm_replay_check(), _pixelrag_vlm_route_readiness_check(), _pixelrag_vlm_replay_worker_check(), diff --git a/services/pixelrag_marketplace_candidate_knowledge_replay_service.py b/services/pixelrag_marketplace_candidate_knowledge_replay_service.py new file mode 100644 index 0000000..fc94a9e --- /dev/null +++ b/services/pixelrag_marketplace_candidate_knowledge_replay_service.py @@ -0,0 +1,729 @@ +"""Controlled PixelRAG marketplace candidate knowledge replay worker. + +This worker turns marketplace embedding signature guard receipts into +deterministic internal RAG candidate-knowledge receipts. It does not generate +embeddings, call models, call networks, write databases, write AI insights, or +promote candidate prices. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.pixelrag_crawler_integration_service import DEFAULT_ARTIFACT_MAX_AGE_HOURS +from services.pixelrag_marketplace_embedding_signature_guard_replay_service import ( + DEFAULT_OUTPUT_ROOT as DEFAULT_EMBEDDING_SIGNATURE_GUARD_RECEIPT_ROOT, + EMBEDDING_SIGNATURE_GUARD_REPLAY_VERSION, +) +from services.rag_service import get_embedding_signature + + +POLICY = "controlled_pixelrag_marketplace_candidate_knowledge_replay_v1" +CANDIDATE_KNOWLEDGE_REPLAY_VERSION = ( + "pixelrag_marketplace_candidate_knowledge_replay_v1" +) +INTERNAL_RAG_TARGET = "rag_service.learning_episode_candidate_preview" +DEFAULT_LIMIT = 25 +DEFAULT_OUTPUT_ROOT = os.getenv( + "PIXELRAG_MARKETPLACE_CANDIDATE_KNOWLEDGE_REPLAY_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_marketplace_candidate_knowledge_replay_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_marketplace_candidate_knowledge_replay_receipts", +) + +REQUIRED_SIGNATURE_GATES = { + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", +} + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...]: + if isinstance(platform, str): + value = platform.strip().lower() + return (value,) if value else () + return tuple( + str(item or "").strip().lower() + for item in (platform or ()) + if str(item or "").strip() + ) + + +def _safe_segment(value: Any) -> str: + text = str(value or "unknown").strip().lower() + text = re.sub(r"[^a-z0-9._-]+", "-", text) + return text.strip("-") or "unknown" + + +def _parse_iso_datetime(value: Any) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + + +def _receipt_candidates( + root: Path, + *, + platforms: tuple[str, ...], + limit: int, +) -> list[Path]: + if not root.exists(): + return [] + candidates: list[Path] = [] + if platforms: + for platform in platforms: + candidates.extend( + (root / platform).glob( + "*/marketplace_embedding_signature_guard_replay_receipt.json" + ) + ) + else: + candidates.extend( + root.glob("*/*/marketplace_embedding_signature_guard_replay_receipt.json") + ) + return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] + + +def _load_receipt(path: Path) -> tuple[dict[str, Any], list[str]]: + try: + return json.loads(path.read_text(encoding="utf-8")), [] + except (OSError, json.JSONDecodeError) as exc: + return {}, [str(exc)[:300]] + + +def _as_mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _as_list(value: Any) -> list[Any]: + return list(value) if isinstance(value, list) else [] + + +def _required_before(signature_payload: Mapping[str, Any]) -> set[str]: + return set(str(item) for item in _as_list(signature_payload.get("required_before_data_promotion"))) + + +def _knowledge_fingerprint( + platform: str, + manifest_id: str, + candidate_id: str, + signature_guard_fingerprint: str, + embedding_signature: str, +) -> str: + digest = hashlib.sha256( + ( + f"{platform}:{manifest_id}:{candidate_id}:" + f"{signature_guard_fingerprint}:{embedding_signature}" + ).encode("utf-8") + ).hexdigest() + return digest[:24] + + +def _knowledge_text( + *, + platform: str, + manifest_id: str, + adapter_code: str, + barrier_type: str, + candidate: Mapping[str, Any], + embedding_signature: str, +) -> str: + parts = [ + f"platform={platform}", + f"manifest_id={manifest_id}", + f"adapter_code={adapter_code or 'unknown'}", + f"barrier_type={barrier_type or 'none'}", + f"candidate_id={candidate.get('candidate_id') or 'unknown'}", + f"candidate_index={int(candidate.get('candidate_index') or 0)}", + f"identity_fingerprint={candidate.get('identity_fingerprint') or 'unknown'}", + f"promotion_fingerprint={candidate.get('promotion_fingerprint') or 'unknown'}", + ( + "embedding_signature_guard_fingerprint=" + f"{candidate.get('embedding_signature_guard_fingerprint') or 'unknown'}" + ), + f"embedding_signature={embedding_signature}", + "knowledge_source=pixelrag_marketplace_signature_guard_replay", + "write_target=internal_rag_candidate_preview", + ] + return " | ".join(parts) + + +def _candidate_knowledge_contracts( + receipt: Mapping[str, Any], + *, + signature_payload: Mapping[str, Any], +) -> list[dict[str, Any]]: + platform = str(receipt.get("platform") or "unknown") + manifest_id = str(receipt.get("manifest_id") or "unknown") + adapter_code = str(signature_payload.get("adapter_code") or receipt.get("adapter_code") or "") + barrier_type = str(signature_payload.get("barrier_type") or "") + embedding_contract = _as_mapping(signature_payload.get("embedding_signature_contract")) + embedding_signature = str(embedding_contract.get("embedding_signature") or "") + contracts: list[dict[str, Any]] = [] + for candidate in _as_list(signature_payload.get("signature_guard_candidate_contracts")): + if not isinstance(candidate, Mapping): + continue + candidate_id = str(candidate.get("candidate_id") or f"{platform}:{manifest_id}:unknown") + signature_guard_fingerprint = str( + candidate.get("embedding_signature_guard_fingerprint") or "missing" + ) + knowledge_fingerprint = _knowledge_fingerprint( + platform, + manifest_id, + candidate_id, + signature_guard_fingerprint, + embedding_signature, + ) + contracts.append( + { + "candidate_id": candidate_id, + "candidate_index": int(candidate.get("candidate_index") or 0), + "knowledge_candidate_id": ( + f"{platform}:{manifest_id}:{knowledge_fingerprint}" + ), + "identity_fingerprint": candidate.get("identity_fingerprint"), + "promotion_fingerprint": candidate.get("promotion_fingerprint"), + "embedding_signature_guard_fingerprint": signature_guard_fingerprint, + "candidate_knowledge_fingerprint": knowledge_fingerprint, + "embedding_signature": embedding_signature, + "expected_embedding_signature": ( + candidate.get("expected_embedding_signature") or embedding_signature + ), + "candidate_knowledge_text": _knowledge_text( + platform=platform, + manifest_id=manifest_id, + adapter_code=adapter_code, + barrier_type=barrier_type, + candidate=candidate, + embedding_signature=embedding_signature, + ), + "internal_rag_target": INTERNAL_RAG_TARGET, + "knowledge_stage": "pre_internal_rag_candidate_canary", + "knowledge_strategy": "deterministic_candidate_knowledge_replay_no_db_write", + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + "requires_candidate_knowledge_replay": True, + "requires_internal_rag_candidate_canary": True, + "embedding_generation_performed": False, + "model_call_performed": False, + "ready_for_internal_rag_candidate_replay": True, + "ready_for_rag_candidate_preview": True, + "ready_for_ai_insights_write": False, + "ready_for_price_table_write": False, + "ai_insights_write_blocked_until_canary": True, + "price_write_blocked_until_candidate_canary": True, + } + ) + return contracts + + +def _candidate_knowledge_payload( + receipt: Mapping[str, Any], + *, + receipt_path: Path, +) -> dict[str, Any]: + signature_payload = _as_mapping(receipt.get("embedding_signature_guard_replay")) + knowledge_contracts = _candidate_knowledge_contracts( + receipt, + signature_payload=signature_payload, + ) + return { + "candidate_knowledge_replay_version": CANDIDATE_KNOWLEDGE_REPLAY_VERSION, + "embedding_signature_guard_replay_version": signature_payload.get( + "embedding_signature_guard_replay_version" + ), + "promotion_gate_replay_version": signature_payload.get( + "promotion_gate_replay_version" + ), + "identity_matcher_replay_version": signature_payload.get( + "identity_matcher_replay_version" + ), + "adapter_dry_run_version": signature_payload.get("adapter_dry_run_version"), + "adapter_preflight_version": signature_payload.get("adapter_preflight_version"), + "source_contract_version": signature_payload.get("source_contract_version"), + "adapter_code": signature_payload.get("adapter_code") or receipt.get("adapter_code"), + "contract_id": signature_payload.get("contract_id"), + "barrier_type": signature_payload.get("barrier_type"), + "capture_runtime_unavailable": bool( + signature_payload.get("capture_runtime_unavailable") + ), + "input_embedding_signature_guard_receipt_path": str(receipt_path), + "input_promotion_gate_receipt_path": signature_payload.get( + "input_promotion_gate_receipt_path" + ), + "input_identity_matcher_receipt_path": signature_payload.get( + "input_identity_matcher_receipt_path" + ), + "input_adapter_dry_run_receipt_path": signature_payload.get( + "input_adapter_dry_run_receipt_path" + ), + "adapter_preflight_receipt_path": signature_payload.get( + "adapter_preflight_receipt_path" + ), + "source_contract_replay_receipt_path": signature_payload.get( + "source_contract_replay_receipt_path" + ), + "source_worker_receipt_path": signature_payload.get("source_worker_receipt_path"), + "source_receipt_path": signature_payload.get("source_receipt_path"), + "embedding_signature_contract": signature_payload.get( + "embedding_signature_contract" + ) + or {}, + "candidate_knowledge_execution_mode": ( + "deterministic_candidate_knowledge_replay_no_embedding_no_network_no_db" + ), + "candidate_knowledge_count": len(knowledge_contracts), + "candidate_knowledge_contracts": knowledge_contracts, + "required_before_data_promotion": list( + signature_payload.get("required_before_data_promotion") or [] + ), + "allowed_next_step": "run_internal_rag_candidate_canary", + } + + +def _candidate_knowledge_checks( + receipt: Mapping[str, Any], + *, + stale: bool, + errors: list[str], + knowledge_payload: Mapping[str, Any], +) -> dict[str, bool]: + signature_payload = _as_mapping(receipt.get("embedding_signature_guard_replay")) + boundary = _as_mapping(receipt.get("promotion_boundary")) + signature_checks = _as_mapping( + receipt.get("embedding_signature_guard_replay_checks") + ) + signature_contracts = _as_list( + signature_payload.get("signature_guard_candidate_contracts") + ) + knowledge_contracts = _as_list( + knowledge_payload.get("candidate_knowledge_contracts") + ) + embedding_contract = _as_mapping( + knowledge_payload.get("embedding_signature_contract") + ) + expected_signature = str(embedding_contract.get("embedding_signature") or "") + current_signature = get_embedding_signature() + required_before = _required_before(signature_payload) + return { + "receipt_parse_ok": not errors, + "receipt_fresh": not stale, + "embedding_signature_guard_replay_ready": ( + receipt.get("worker_status") + == "executed_marketplace_embedding_signature_guard_replay_ready" + ), + "embedding_signature_guard_version_supported": ( + signature_payload.get("embedding_signature_guard_replay_version") + == EMBEDDING_SIGNATURE_GUARD_REPLAY_VERSION + ), + "embedding_signature_guard_checks_all_passed": ( + int(receipt.get("embedding_signature_guard_replay_check_pass_count") or 0) + == int(receipt.get("embedding_signature_guard_replay_check_count") or -1) + and int(receipt.get("embedding_signature_guard_replay_check_count") or 0) > 0 + ), + "blocked_page_not_product_data": bool( + signature_checks.get("blocked_page_not_product_data") + ), + "signature_guard_candidate_contracts_present": bool(signature_contracts), + "candidate_knowledge_contracts_generated": bool(knowledge_contracts), + "candidate_knowledge_count_matches_signature_count": ( + len(knowledge_contracts) == len(signature_contracts) + ), + "signature_candidates_ready_for_candidate_knowledge": all( + bool(item.get("ready_for_candidate_knowledge_replay")) + for item in signature_contracts + ), + "required_signature_gates_present": REQUIRED_SIGNATURE_GATES.issubset( + required_before + ), + "candidate_knowledge_replay_required": all( + bool(item.get("requires_candidate_knowledge_replay")) + for item in signature_contracts + ), + "embedding_signature_present": ( + len(expected_signature) == 12 + and all(ch in "0123456789abcdef" for ch in expected_signature) + ), + "embedding_signature_matches_current_rag_signature": ( + expected_signature == current_signature + ), + "embedding_generation_not_performed": ( + not bool(embedding_contract.get("embedding_generation_performed")) + and not bool(embedding_contract.get("model_call_performed")) + and not bool(receipt.get("embedding_generation_performed")) + ), + "knowledge_text_generated": all( + bool(item.get("candidate_knowledge_text")) for item in knowledge_contracts + ), + "internal_rag_preview_ready": all( + bool(item.get("ready_for_internal_rag_candidate_replay")) + and item.get("internal_rag_target") == INTERNAL_RAG_TARGET + for item in knowledge_contracts + ), + "ai_insights_write_disallowed": all( + not bool(item.get("ready_for_ai_insights_write")) + and bool(item.get("ai_insights_write_blocked_until_canary")) + for item in knowledge_contracts + ), + "price_write_blocked_until_candidate_canary": all( + bool(item.get("price_write_blocked_until_candidate_canary")) + and not bool(item.get("ready_for_price_table_write")) + for item in knowledge_contracts + ), + "direct_price_write_disallowed": not bool( + boundary.get("direct_price_write_allowed") + ), + "direct_ai_insights_write_disallowed": not bool( + boundary.get("direct_ai_insights_write_allowed") + ), + "database_write_disallowed": not bool(receipt.get("writes_database")) + and int(receipt.get("writes_database_count") or 0) == 0, + "network_call_disallowed": not bool(receipt.get("network_call_performed")), + "model_call_disallowed": not bool(receipt.get("model_call_performed")), + "source_provenance_paths_present": bool( + knowledge_payload.get("input_embedding_signature_guard_receipt_path") + and knowledge_payload.get("input_promotion_gate_receipt_path") + and knowledge_payload.get("input_identity_matcher_receipt_path") + and knowledge_payload.get("input_adapter_dry_run_receipt_path") + and knowledge_payload.get("adapter_preflight_receipt_path") + and knowledge_payload.get("source_contract_replay_receipt_path") + ), + } + + +def _worker_item( + receipt_path: Path, + *, + now: datetime, + max_age_hours: int, + execute: bool, +) -> dict[str, Any]: + receipt, errors = _load_receipt(receipt_path) + generated_at = _parse_iso_datetime(receipt.get("generated_at")) + if generated_at is None: + try: + generated_at = datetime.fromtimestamp(receipt_path.stat().st_mtime, tz=timezone.utc) + except OSError: + generated_at = None + age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None + stale = age_hours is None or age_hours > max_age_hours + knowledge_payload = _candidate_knowledge_payload(receipt, receipt_path=receipt_path) + checks = _candidate_knowledge_checks( + receipt, + stale=stale, + errors=errors, + knowledge_payload=knowledge_payload, + ) + check_count = len(checks) + pass_count = sum(1 for passed in checks.values() if passed) + ready = pass_count == check_count + platform = str(receipt.get("platform") or receipt_path.parent.parent.name).strip().lower() + manifest_id = str(receipt.get("manifest_id") or receipt_path.parent.name).strip() + status = ( + "executed_marketplace_candidate_knowledge_replay_ready" + if execute and ready + else ( + "dry_run_ready_for_marketplace_candidate_knowledge_replay" + if ready + else "skipped_marketplace_candidate_knowledge_replay_guard_failed" + ) + ) + return { + "worker_status": status, + "platform": platform, + "manifest_id": manifest_id, + "source_type": "marketplace_embedding_signature_guard_replay_receipt", + "embedding_signature_guard_receipt_path": str(receipt_path), + "promotion_gate_receipt_path": knowledge_payload.get( + "input_promotion_gate_receipt_path" + ), + "identity_matcher_receipt_path": knowledge_payload.get( + "input_identity_matcher_receipt_path" + ), + "adapter_dry_run_receipt_path": knowledge_payload.get( + "input_adapter_dry_run_receipt_path" + ), + "adapter_preflight_receipt_path": knowledge_payload.get( + "adapter_preflight_receipt_path" + ), + "source_contract_replay_receipt_path": knowledge_payload.get( + "source_contract_replay_receipt_path" + ), + "source_worker_receipt_path": knowledge_payload.get("source_worker_receipt_path"), + "source_receipt_path": knowledge_payload.get("source_receipt_path"), + "adapter_code": knowledge_payload.get("adapter_code"), + "candidate_knowledge_replay_status": "ready" if ready else "blocked", + "ready_for_execution": ready, + "execute": bool(execute), + "generated_at": generated_at.isoformat() if generated_at else None, + "age_hours": round(age_hours, 3) if age_hours is not None else None, + "stale": stale, + "network_call_performed": False, + "model_call_performed": False, + "embedding_generation_performed": False, + "artifact_write_performed": False, + "writes_database": False, + "writes_database_count": 0, + "writes_ai_insights": False, + "writes_price_tables": False, + "primary_human_gate_count": 0, + "candidate_knowledge_replay_checks": checks, + "candidate_knowledge_replay_check_count": check_count, + "candidate_knowledge_replay_check_pass_count": pass_count, + "candidate_knowledge_replay": knowledge_payload, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "candidate_knowledge_replay_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + "requires_internal_rag_candidate_canary": True, + }, + "next_machine_action": ( + "run_internal_rag_candidate_canary" + if execute and ready + else ( + "run_marketplace_candidate_knowledge_replay_execute" + if ready + else "repair_marketplace_candidate_knowledge_replay_inputs" + ) + ), + } + + +def _write_candidate_knowledge_receipt( + *, + output_root: Path, + item: Mapping[str, Any], +) -> str: + target = ( + output_root + / _safe_segment(item.get("platform")) + / _safe_segment(item.get("manifest_id")) + / "marketplace_candidate_knowledge_replay_receipt.json" + ) + target.parent.mkdir(parents=True, exist_ok=True) + receipt = dict(item) + receipt["artifact_write_performed"] = True + receipt["receipt_path"] = str(target) + receipt["generated_at"] = datetime.now(timezone.utc).isoformat() + receipt["policy"] = POLICY + target.write_text( + json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + return str(target) + + +def run_pixelrag_marketplace_candidate_knowledge_replay( + *, + embedding_signature_guard_receipt_root: str | Path | None = None, + output_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, + execute: bool = False, + write_receipt: bool = False, +) -> dict[str, Any]: + """Run or dry-run marketplace candidate knowledge replay.""" + source_root = Path( + embedding_signature_guard_receipt_root + or DEFAULT_EMBEDDING_SIGNATURE_GUARD_RECEIPT_ROOT + ) + output = Path(output_root or DEFAULT_OUTPUT_ROOT) + platforms = _normalise_platforms(platform) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + now = datetime.now(timezone.utc) + receipt_paths = _receipt_candidates(source_root, platforms=platforms, limit=item_limit) + worker_items: list[dict[str, Any]] = [] + for receipt_path in receipt_paths: + item = _worker_item( + receipt_path, + now=now, + max_age_hours=max_age, + execute=execute, + ) + if execute and write_receipt and item.get("ready_for_execution"): + item["receipt_path"] = _write_candidate_knowledge_receipt( + output_root=output, + item=item, + ) + item["artifact_write_performed"] = True + worker_items.append(item) + + ready_count = sum(1 for item in worker_items if item.get("ready_for_execution")) + blocked_count = len(worker_items) - ready_count + dry_run_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_") + ) + executed_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_") + ) + receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path")) + guard_failed_count = sum( + 1 + for item in worker_items + if item.get("candidate_knowledge_replay_status") != "ready" + ) + capture_runtime_gap_count = sum( + 1 + for item in worker_items + if (item.get("candidate_knowledge_replay") or {}).get( + "capture_runtime_unavailable" + ) + ) + knowledge_candidate_count = sum( + int( + (item.get("candidate_knowledge_replay") or {}).get( + "candidate_knowledge_count" + ) + or 0 + ) + for item in worker_items + ) + internal_rag_ready_count = sum( + sum( + 1 + for candidate in _as_list( + (item.get("candidate_knowledge_replay") or {}).get( + "candidate_knowledge_contracts" + ) + ) + if isinstance(candidate, Mapping) + and candidate.get("ready_for_internal_rag_candidate_replay") + ) + for item in worker_items + ) + expected_signatures = sorted( + { + str( + ( + (item.get("candidate_knowledge_replay") or {}) + .get("embedding_signature_contract") + or {} + ).get("embedding_signature") + or "" + ) + for item in worker_items + } + - {""} + ) + + if not worker_items: + status = "warning" + elif guard_failed_count: + status = "warning" + else: + status = "ok" + + if not worker_items: + next_action = "run_marketplace_embedding_signature_guard_replay" + elif not execute and ready_count: + next_action = "run_marketplace_candidate_knowledge_replay_execute" + elif execute and receipt_written_count: + next_action = "run_internal_rag_candidate_canary" + elif guard_failed_count: + next_action = "repair_marketplace_candidate_knowledge_replay_inputs" + else: + next_action = "refresh_marketplace_candidate_knowledge_replay_candidates" + + summary = { + "candidate_count": len(worker_items), + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "guard_failed_count": guard_failed_count, + "capture_runtime_gap_count": capture_runtime_gap_count, + "knowledge_candidate_count": knowledge_candidate_count, + "internal_rag_ready_count": internal_rag_ready_count, + "expected_embedding_signatures": expected_signatures, + "platforms": sorted({str(item.get("platform") or "unknown") for item in worker_items}), + "network_call_performed": False, + "model_call_performed": False, + "embedding_generation_performed": False, + "artifact_write_performed": bool(receipt_written_count), + "writes_database_count": 0, + "writes_ai_insights": False, + "writes_price_tables": False, + "primary_human_gate_count": 0, + } + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": now.isoformat(), + "candidate_knowledge_replay_version": CANDIDATE_KNOWLEDGE_REPLAY_VERSION, + "embedding_signature_guard_receipt_root": str(source_root), + "output_root": str(output), + "platform_filter": list(platforms), + "max_age_hours": max_age, + "limit": item_limit, + "execute": bool(execute), + "write_receipt": bool(write_receipt and execute), + "summary": summary, + "worker_items": worker_items, + "controlled_apply": { + "network_call": False, + "model_call": False, + "embedding_generation": False, + "artifact_write": bool(receipt_written_count), + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "writes_ai_insights": False, + "writes_price_tables": False, + "secret_read": False, + "raw_cookie_or_session_read": False, + "credentialed_session_allowed": False, + "login_allowed": False, + "production_price_write": False, + "primary_human_gate_count": 0, + }, + "promotion_boundary": { + "writes_ai_insights": False, + "writes_price_tables": False, + "candidate_knowledge_replay_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + "requires_internal_rag_candidate_canary": True, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "CANDIDATE_KNOWLEDGE_REPLAY_VERSION", + "DEFAULT_OUTPUT_ROOT", + "INTERNAL_RAG_TARGET", + "POLICY", + "run_pixelrag_marketplace_candidate_knowledge_replay", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index d350b61..a2eb71b 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1316,6 +1316,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_pixelrag_marketplace_identity_matcher_replay_check", lambda: smoke._check("pixelrag marketplace identity matcher replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_promotion_gate_replay_check", lambda: smoke._check("pixelrag marketplace promotion gate replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_embedding_signature_guard_replay_check", lambda: smoke._check("pixelrag marketplace embedding signature guard replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_candidate_knowledge_replay_check", lambda: smoke._check("pixelrag marketplace candidate knowledge replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -1325,7 +1326,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 41, "warning": 1, "critical": 1, "total": 43} + assert result["summary"] == {"ok": 42, "warning": 1, "critical": 1, "total": 44} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3860,6 +3861,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_pixelrag_marketplace_identity_matcher_replay_check", lambda: smoke._check("pixelrag marketplace identity matcher replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_promotion_gate_replay_check", lambda: smoke._check("pixelrag marketplace promotion gate replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_embedding_signature_guard_replay_check", lambda: smoke._check("pixelrag marketplace embedding signature guard replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_candidate_knowledge_replay_check", lambda: smoke._check("pixelrag marketplace candidate knowledge replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -3920,7 +3922,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 43, "warning": 0, "critical": 0, "total": 43}, + "summary": {"ok": 44, "warning": 0, "critical": 0, "total": 44}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4164,6 +4166,33 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "primary_human_gate_count": 0, }, }, + { + "name": "PixelRAG marketplace candidate knowledge replay", + "status": "ok", + "summary": "PixelRAG marketplace candidate knowledge replay candidates=1, ready=1, blocked=0, dry_run=1, executed=0, receipts=0, knowledge_candidates=2, internal_rag_ready=2, signatures=8c8063c66dbe", + "details": { + "policy": "controlled_pixelrag_marketplace_candidate_knowledge_replay_v1", + "candidate_count": 1, + "ready_count": 1, + "blocked_count": 0, + "dry_run_count": 1, + "executed_count": 0, + "receipt_written_count": 0, + "capture_runtime_gap_count": 0, + "knowledge_candidate_count": 2, + "internal_rag_ready_count": 2, + "expected_embedding_signatures": ["8c8063c66dbe"], + "network_call_performed": False, + "model_call_performed": False, + "embedding_generation_performed": False, + "artifact_write_performed": False, + "writes_ai_insights": False, + "writes_price_tables": False, + "next_machine_action": "run_marketplace_candidate_knowledge_replay_execute", + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "name": "PixelRAG OCR/VLM replay contract", "status": "ok", @@ -4278,7 +4307,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 40 + assert summary["summary"]["total"] == 41 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -6640,6 +6669,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag marketplace embedding signature guard replay ok", )) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_candidate_knowledge_replay_check", lambda: smoke._check( + "PixelRAG marketplace candidate knowledge replay", + "ok", + "pixelrag marketplace candidate knowledge replay ok", + )) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check( "PixelRAG OCR/VLM replay contract", "ok", @@ -6680,7 +6714,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 43 + assert result["summary"]["total"] == 44 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_marketplace_candidate_knowledge_replay_service.py b/tests/test_pixelrag_marketplace_candidate_knowledge_replay_service.py new file mode 100644 index 0000000..e615cf5 --- /dev/null +++ b/tests/test_pixelrag_marketplace_candidate_knowledge_replay_service.py @@ -0,0 +1,325 @@ +import json +import subprocess +import sys +from datetime import datetime, timezone + + +def _signature_candidate(platform, manifest_id, index, embedding_signature): + return { + "candidate_id": f"{platform}:{manifest_id}:{index}", + "candidate_index": index, + "identity_fingerprint": f"identity-{platform}-{index}", + "promotion_fingerprint": f"promotion-{platform}-{index}", + "embedding_signature_guard_fingerprint": f"signature-guard-{platform}-{index}", + "expected_embedding_signature": embedding_signature, + "embedding_signature": embedding_signature, + "guard_stage": "pre_candidate_knowledge_replay", + "ready_for_candidate_knowledge_replay": True, + "ready_for_ai_insights_write": False, + "ready_for_price_table_write": False, + "price_write_blocked_until_candidate_canary": True, + "requires_candidate_knowledge_replay": True, + "requires_embedding_signature_guard": True, + "requires_source_provenance": True, + "embedding_generation_performed": False, + } + + +def _write_embedding_signature_guard_receipt( + root, + *, + platform="coupang_tw", + manifest_id="coupang-403", + adapter_code="coupang", + barrier_type="access_denied", + capture_runtime_unavailable=False, +): + from services.rag_service import get_embedding_signature + + receipt_dir = root / platform / manifest_id + receipt_dir.mkdir(parents=True) + source_root = "runtime_artifacts" + embedding_signature = get_embedding_signature() + payload = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "policy": "controlled_pixelrag_marketplace_embedding_signature_guard_replay_v1", + "worker_status": "executed_marketplace_embedding_signature_guard_replay_ready", + "platform": platform, + "manifest_id": manifest_id, + "adapter_code": adapter_code, + "embedding_signature_guard_replay_check_count": 24, + "embedding_signature_guard_replay_check_pass_count": 24, + "embedding_signature_guard_replay_checks": { + "blocked_page_not_product_data": True, + "candidate_knowledge_replay_ready": True, + "database_write_disallowed": True, + "direct_ai_insights_write_disallowed": True, + "direct_price_write_disallowed": True, + "embedding_generation_not_performed": True, + "embedding_signature_guard_required": True, + "embedding_signature_present": True, + "model_call_disallowed": True, + "network_call_disallowed": True, + "price_write_blocked_until_candidate_canary": True, + "receipt_fresh": True, + "receipt_parse_ok": True, + "required_promotion_gates_present": True, + "signature_guard_contracts_generated": True, + "source_provenance_paths_present": True, + }, + "embedding_signature_guard_replay": { + "embedding_signature_guard_replay_version": "pixelrag_marketplace_embedding_signature_guard_replay_v1", + "promotion_gate_replay_version": "pixelrag_marketplace_promotion_gate_replay_v1", + "identity_matcher_replay_version": "pixelrag_marketplace_identity_matcher_replay_v1", + "adapter_dry_run_version": "pixelrag_marketplace_adapter_dry_run_v1", + "adapter_preflight_version": "pixelrag_marketplace_adapter_preflight_v1", + "source_contract_version": "pixelrag_marketplace_source_contract_v1", + "contract_id": f"{platform}:{manifest_id}:{adapter_code}", + "adapter_code": adapter_code, + "barrier_type": barrier_type, + "capture_runtime_unavailable": capture_runtime_unavailable, + "input_embedding_signature_guard_receipt_path": ( + f"{source_root}/pixelrag_marketplace_embedding_signature_guard_replay_receipts/" + f"{platform}/{manifest_id}/marketplace_embedding_signature_guard_replay_receipt.json" + ), + "input_promotion_gate_receipt_path": ( + f"{source_root}/pixelrag_marketplace_promotion_gate_replay_receipts/" + f"{platform}/{manifest_id}/marketplace_promotion_gate_replay_receipt.json" + ), + "input_identity_matcher_receipt_path": ( + f"{source_root}/pixelrag_marketplace_identity_matcher_replay_receipts/" + f"{platform}/{manifest_id}/marketplace_identity_matcher_replay_receipt.json" + ), + "input_adapter_dry_run_receipt_path": ( + f"{source_root}/pixelrag_marketplace_adapter_dry_run_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_dry_run_receipt.json" + ), + "adapter_preflight_receipt_path": ( + f"{source_root}/pixelrag_marketplace_adapter_preflight_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_preflight_receipt.json" + ), + "source_contract_replay_receipt_path": ( + f"{source_root}/pixelrag_source_contract_replay_receipts/" + f"{platform}/{manifest_id}/source_contract_replay_receipt.json" + ), + "source_worker_receipt_path": ( + f"{source_root}/pixelrag_platform_probe_worker_receipts/" + f"{platform}/{manifest_id}/platform_probe_worker_receipt.json" + ), + "source_receipt_path": ( + f"{source_root}/pixelrag_visual_evidence/" + f"{platform}/{manifest_id}/capture_receipt.json" + ), + "embedding_signature_contract": { + "embedding_signature": embedding_signature, + "embedding_model": "bge-m3:latest", + "embedding_dim": 1024, + "embedding_normalize": True, + "embedding_generation_performed": False, + "model_call_performed": False, + }, + "embedding_signature_guard_execution_mode": ( + "deterministic_signature_guard_only_no_embedding_no_network_no_db" + ), + "signature_guard_candidate_count": 2, + "signature_guard_candidate_contracts": [ + _signature_candidate(platform, manifest_id, 0, embedding_signature), + _signature_candidate(platform, manifest_id, 1, embedding_signature), + ], + "required_before_data_promotion": [ + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", + ], + "allowed_next_step": "run_marketplace_candidate_knowledge_replay", + }, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "requires_candidate_knowledge_replay": True, + }, + "writes_database": False, + "writes_database_count": 0, + "network_call_performed": False, + "model_call_performed": False, + "embedding_generation_performed": False, + } + path = receipt_dir / "marketplace_embedding_signature_guard_replay_receipt.json" + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return path + + +def test_marketplace_candidate_knowledge_reads_signature_guard_receipts(tmp_path): + from services.pixelrag_marketplace_candidate_knowledge_replay_service import ( + INTERNAL_RAG_TARGET, + POLICY, + run_pixelrag_marketplace_candidate_knowledge_replay, + ) + from services.rag_service import get_embedding_signature + + signature_root = tmp_path / "signature" + _write_embedding_signature_guard_receipt(signature_root) + _write_embedding_signature_guard_receipt( + signature_root, + platform="shopee_tw", + manifest_id="shopee-runtime-gap", + adapter_code="shopee", + barrier_type="language_or_region_interstitial", + capture_runtime_unavailable=True, + ) + + payload = run_pixelrag_marketplace_candidate_knowledge_replay( + embedding_signature_guard_receipt_root=signature_root, + platform=("coupang_tw", "shopee_tw"), + ) + by_platform = {item["platform"]: item for item in payload["worker_items"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "ok" + assert payload["summary"]["candidate_count"] == 2 + assert payload["summary"]["ready_count"] == 2 + assert payload["summary"]["dry_run_count"] == 2 + assert payload["summary"]["knowledge_candidate_count"] == 4 + assert payload["summary"]["internal_rag_ready_count"] == 4 + assert payload["summary"]["capture_runtime_gap_count"] == 1 + assert payload["summary"]["expected_embedding_signatures"] == [ + get_embedding_signature() + ] + assert payload["summary"]["writes_database_count"] == 0 + assert payload["summary"]["writes_ai_insights"] is False + assert payload["summary"]["writes_price_tables"] is False + assert payload["controlled_apply"]["embedding_generation"] is False + assert payload["controlled_apply"]["writes_ai_insights"] is False + assert payload["promotion_boundary"]["writes_price_tables"] is False + assert payload["next_machine_action"] == ( + "run_marketplace_candidate_knowledge_replay_execute" + ) + assert by_platform["coupang_tw"]["worker_status"] == ( + "dry_run_ready_for_marketplace_candidate_knowledge_replay" + ) + first_knowledge = by_platform["coupang_tw"]["candidate_knowledge_replay"][ + "candidate_knowledge_contracts" + ][0] + assert first_knowledge["embedding_signature"] == get_embedding_signature() + assert first_knowledge["internal_rag_target"] == INTERNAL_RAG_TARGET + assert first_knowledge["ready_for_internal_rag_candidate_replay"] is True + assert first_knowledge["ready_for_ai_insights_write"] is False + assert first_knowledge["ready_for_price_table_write"] is False + assert first_knowledge["price_write_blocked_until_candidate_canary"] is True + assert by_platform["shopee_tw"]["candidate_knowledge_replay"][ + "capture_runtime_unavailable" + ] is True + + +def test_marketplace_candidate_knowledge_execute_writes_receipt(tmp_path): + from services.pixelrag_marketplace_candidate_knowledge_replay_service import ( + run_pixelrag_marketplace_candidate_knowledge_replay, + ) + + signature_root = tmp_path / "signature" + output_root = tmp_path / "knowledge" + _write_embedding_signature_guard_receipt(signature_root) + + payload = run_pixelrag_marketplace_candidate_knowledge_replay( + embedding_signature_guard_receipt_root=signature_root, + output_root=output_root, + platform="coupang_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["executed_count"] == 1 + assert payload["summary"]["receipt_written_count"] == 1 + assert payload["controlled_apply"]["artifact_write"] is True + assert payload["controlled_apply"]["writes_database"] is False + assert payload["next_machine_action"] == "run_internal_rag_candidate_canary" + receipt_path = ( + output_root + / "coupang_tw" + / "coupang-403" + / "marketplace_candidate_knowledge_replay_receipt.json" + ) + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == ( + "executed_marketplace_candidate_knowledge_replay_ready" + ) + assert receipt["artifact_write_performed"] is True + assert receipt["writes_database"] is False + assert receipt["writes_ai_insights"] is False + assert receipt["writes_price_tables"] is False + assert receipt["candidate_knowledge_replay_check_pass_count"] == ( + receipt["candidate_knowledge_replay_check_count"] + ) + assert receipt["candidate_knowledge_replay"]["allowed_next_step"] == ( + "run_internal_rag_candidate_canary" + ) + + +def test_marketplace_candidate_knowledge_cli_outputs_json(tmp_path): + signature_root = tmp_path / "signature" + _write_embedding_signature_guard_receipt(signature_root) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py", + "--embedding-signature-guard-receipt-root", + str(signature_root), + "--platform", + "coupang_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["summary"]["knowledge_candidate_count"] == 2 + assert payload["controlled_apply"]["writes_database"] is False + assert payload["controlled_apply"]["writes_ai_insights"] is False + + +def test_marketplace_candidate_knowledge_route_returns_readback( + tmp_path, + monkeypatch, +): + from flask import Flask + from routes import system_public_routes as routes + from services import ( + pixelrag_marketplace_candidate_knowledge_replay_service as service, + ) + + signature_root = tmp_path / "signature" + _write_embedding_signature_guard_receipt(signature_root) + monkeypatch.setattr( + service, + "DEFAULT_EMBEDDING_SIGNATURE_GUARD_RECEIPT_ROOT", + str(signature_root), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay?platform=coupang_tw" + ): + response = ( + routes + .ai_automation_pixelrag_marketplace_candidate_knowledge_replay_api + .__wrapped__() + ) + payload = response.get_json() + + assert payload["policy"] == ( + "controlled_pixelrag_marketplace_candidate_knowledge_replay_v1" + ) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False + assert payload["controlled_apply"]["writes_ai_insights"] is False