From 911369c11969e3ee029d9954494793aec3ca9b06 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 09:50:30 +0800 Subject: [PATCH] feat(ai): add PixelRAG platform probe worker --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 2 + .../ai_automation_mainline_work_items.md | 7 +- docs/guides/ai_automation_session_sop.md | 4 +- docs/guides/browse_sh_crawler_playbook.md | 6 + routes/system_public_routes.py | 40 ++ .../ops/run_pixelrag_platform_probe_worker.py | 108 ++++ services/ai_automation_smoke_service.py | 114 ++++ .../pixelrag_platform_probe_worker_service.py | 530 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 46 +- ..._pixelrag_platform_probe_worker_service.py | 248 ++++++++ 11 files changed, 1098 insertions(+), 9 deletions(-) create mode 100644 scripts/ops/run_pixelrag_platform_probe_worker.py create mode 100644 services/pixelrag_platform_probe_worker_service.py create mode 100644 tests/test_pixelrag_platform_probe_worker_service.py diff --git a/config.py b/config.py index 7e287c8..b69ad11 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.759" +SYSTEM_VERSION = "V10.760" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index d239c2c..f6f9d50 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -119,6 +119,7 @@ - 2026-07-09 起 PixelRAG ready receipts 必須有 Ollama-first VLM replay worker：`/api/ai-automation/pixelrag-vlm-replay-worker` 與 `scripts/ops/run_pixelrag_vlm_replay_worker.py` 預設 dry-run，`execute=true&write_receipt=true` 才呼叫 approved Ollama VLM route 並寫 artifact receipt；`/api/ai-automation/smoke` 需包含 `PixelRAG VLM replay worker`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_vlm_replay_worker` family。此 worker 不讀 secret、不寫 DB、不寫 `ai_insights`、不寫正式價格表；blocked / 403 / captcha / access denied receipt 自動跳 platform probe 或 structured API，ready receipt 的 VLM 結果仍須 identity matcher replay 與 PromotionGate 才能進候選知識層。 - 2026-07-09 起 PixelRAG VLM route readiness 必須在 execute 前可讀回：`/api/ai-automation/pixelrag-vlm-route-readiness` 與 `scripts/ops/report_pixelrag_vlm_route_readiness.py` 只讀 approved Ollama `/api/tags`，輸出 configured model、candidate model、reachable host、model_route_ready 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG VLM route readiness`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_vlm_route_readiness` family。此 readback 不呼叫 `/api/generate`、不讀 secret、不寫 DB；worker execute 必須使用它自動避開缺模型路由，完全缺候選時寫 `model_route_not_ready` artifact receipt。 - 2026-07-10 起 PixelRAG platform probe 必須把 blocked / interstitial receipt 轉成機器可執行下一步：`/api/ai-automation/pixelrag-platform-probe` 與 `scripts/ops/report_pixelrag_platform_probe.py` 需讀取 capture receipts 與 VLM replay receipts，辨識 Shopee language / region / generic marketplace slogan / traffic verification 與 Coupang 403 / access denied，輸出 public browser context probe、structured source fallback、platform backoff、blocked page not product data 與 no-write 邊界；`/api/ai-automation/smoke` 需包含 `PixelRAG platform probe readiness`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_platform_probe` family。此 readback 不抓外站、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture worker 可消化 `public_browser_context` 的 locale/timezone/Accept-Language，但必須濾掉 Cookie、Authorization、token/secret/key 類 header。 +- 2026-07-10 起 PixelRAG platform probe 必須有 controlled worker：`/api/ai-automation/pixelrag-platform-probe-worker` 與 `scripts/ops/run_pixelrag_platform_probe_worker.py` 預設 dry-run，`execute=true&write_receipt=true` 才會對 Shopee language / traffic / generic landing 執行 public empty-context capture，並對 Coupang 403 / access denied 自動產生 structured-source/backoff artifact receipt；`/api/ai-automation/smoke` 需包含 `PixelRAG platform probe worker`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_platform_probe_worker` family。此 worker 不登入、不讀 cookie/session/secret、不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture 成功後下一步固定為 RAG candidate replay，structured fallback 只產生機器可讀 package，不把 blocked page 當商品資料。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -890,6 +891,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG VLM worker preflight 必須容納 111 VLM 冷啟延遲 | V10.757 起 `run_pixelrag_vlm_replay_worker.py --execute` 與對應 API 的 `route_generate_probe_timeout` 預設為 20 秒；111 `minicpm-v` 在正式環境可於約 4 到 20 秒間完成小型 `/api/generate` probe，8 秒容易造成假陰性。若 VLM receipt 已判定 `run_platform_probe_or_use_structured_api`，run-level `next_machine_action` 也必須同樣輸出 platform probe，不得停留在 rerun VLM 摘要。 | | 2026-07-10 | PixelRAG VLM language note / generic marketplace slogan 必須視為 interstitial | V10.758 起若 VLM notes 或 title 顯示 `Language selection page`、語言/地區選擇，或只抽到 Shopee `蝦皮購物 | 花得更少買得更好` 這類網站 slogan 且 price/shop 缺失，worker validation 會標記 `interstitial_signal_detected` / `generic_marketplace_title_detected` 與 `non_product_or_interstitial_detected=true`，run-level 與 item-level 下一步都輸出 `run_platform_probe_or_use_structured_api`。 | | 2026-07-10 | PixelRAG platform probe 必須自動消化 marketplace interstitial / blocked receipt | V10.759 起 `/api/ai-automation/pixelrag-platform-probe`、`scripts/ops/report_pixelrag_platform_probe.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 platform probe / `pixelrag_platform_probe` family；Shopee traffic / language / generic landing 會轉成 `run_shopee_public_context_probe_then_structured_source_fallback`，Coupang 403 / access denied 會轉成 `use_structured_source_or_platform_backoff_policy`。probe plan 只產生 public empty browser context、structured adapter fallback 與 no-write contract，不讀或注入 cookie/session/login，不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture worker 只允許 locale/timezone/非敏感 header 並濾掉 Cookie / Authorization / token / secret / key 類 header。 | +| 2026-07-10 | PixelRAG platform probe worker 必須把 probe plan 變成 controlled automation | V10.760 起 `/api/ai-automation/pixelrag-platform-probe-worker`、`scripts/ops/run_pixelrag_platform_probe_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled platform probe worker / `pixelrag_platform_probe_worker` family；dry-run 只規劃 capture 或 structured fallback，execute 模式才執行 public empty-context capture 或寫 structured-source/backoff artifact receipt。此 worker 明確標記 `network_call` 只在 capture execute 時為 true、`model_call=false`、`writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`raw_cookie_or_session_read=false`、`primary_human_gate_count=0`；capture 成功後必須回到 PixelRAG RAG candidate replay，不得直接寫正式價格或把 blocked page 當商品資料。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 63e406a..e146883 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -1,6 +1,6 @@ # AI Automation Mainline Work Items -> Updated: 2026-07-09 +> Updated: 2026-07-10 > Scope: EwoooC / MOMO Pro System AI automation, PixelRAG, MCP/RAG, production UX. ## Source Of Truth @@ -20,7 +20,8 @@ | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | -| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, and PixelRAG VLM replay worker families. | +| Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture, Coupang 403/access denied goes to structured-source/backoff artifact receipts. | +| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, and PixelRAG platform probe worker families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | ## P1 @@ -31,7 +32,7 @@ | Completed | Ollama-first OCR/VLM extraction worker | `run_pixelrag_vlm_replay_worker.py --execute --write-receipt` executes the replay contract against approved Ollama multimodal routes and emits confidence/evidence artifact receipts without writing formal price truth. | | Not started | Ollama-first multimodal embedding benchmark | Verify local Qwen3-VL or equivalent visual embedding on GCP-A -> GCP-B -> 111 before any visual vector retrieval. | | Not started | pgvector-compatible visual evidence metadata | Design metadata-first retrieval without FAISS in production unless ADR approves a different store. | -| Not started | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; prefer public structured data or approved probe, never count blocked pages as product data. | +| In progress | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; `run_pixelrag_platform_probe_worker.py --execute --write-receipt` now emits structured-source/backoff receipts without network, DB writes, or product-data promotion. | | Not started | Marketplace source contracts beyond MOMO/PChome | Promote stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten receipts into source contracts only after provenance, rate-limit, public boundary, and replay evidence. | | In progress | Professional product website UI/UX text reduction | Continue compact AI surfaces, visual QA, and user-facing copy guardrails; avoid engineering logs or work-session prose in the UI. | diff --git a/docs/guides/ai_automation_session_sop.md b/docs/guides/ai_automation_session_sop.md index eeb8430..d23d821 100644 --- a/docs/guides/ai_automation_session_sop.md +++ b/docs/guides/ai_automation_session_sop.md @@ -38,7 +38,9 @@ 或 `python scripts/ops/run_pixelrag_vlm_replay_worker.py` 可讀回 dry-run/execute、model_call、artifact receipt、blocked/ready 分流、DB write=0 與 primary human gate=0；execute 結果仍只是 candidate evidence。 - PixelRAG VLM route readiness 必須確認 `/api/ai-automation/pixelrag-vlm-route-readiness` 或 `python scripts/ops/report_pixelrag_vlm_route_readiness.py` 可讀回 approved Ollama hosts、configured model 是否安裝、candidate model、自動選模下一步、model_call=false、DB write=0；缺模型不得等 execute 才暴露。 -- AI automation smoke 必須包含 external MCP/RAG integration、PixelRAG RAG candidate replay、PixelRAG OCR/VLM replay contract、PixelRAG VLM route readiness 與 PixelRAG VLM replay worker family，避免 registry 已完成但 runtime flag / receipt replay / VLM route / VLM worker 未完成時被誤報為全自動閉環。 +- PixelRAG platform probe worker 必須確認 `/api/ai-automation/pixelrag-platform-probe-worker` + 或 `python scripts/ops/run_pixelrag_platform_probe_worker.py` 可讀回 dry-run/execute、public-context capture、structured-source/backoff receipt、network_call、artifact receipt、DB write=0 與 primary human gate=0；Shopee interstitial 可 execute capture，Coupang 403 必須自動落 structured fallback，不得要求人工主流程。 +- AI automation smoke 必須包含 external MCP/RAG integration、PixelRAG RAG candidate replay、PixelRAG OCR/VLM replay contract、PixelRAG VLM route readiness、PixelRAG VLM replay worker、PixelRAG platform probe readiness 與 PixelRAG platform probe worker family，避免 registry 已完成但 runtime flag / receipt replay / VLM route / VLM worker / platform probe worker 未完成時被誤報為全自動閉環。 - AI 自動化 Prometheus 指標變更必須同步檢查 `docker/grafana/provisioning/dashboards/json/ai-automation-overview.json` 是否需要新增 panel 或查詢。 - 188 線上 active monitoring stack 以 `monitoring/prometheus.yml` 為準；110 gateway 另有 `/home/wooo/monitoring/prometheus.yml`。若 dashboard 無資料，先確認 Prometheus `momo-app` target 與 `momo-network` 連線；所有 Blackbox HTTP target 必須打 `/health`，不可打 Dashboard 首頁 `/`。 - Smoke dashboard 會保存 JSONL 趨勢；若新增檢查項目，要確保 history compact record 仍保持小而可讀。 diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index 1b50cc1..12993ae 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -192,10 +192,16 @@ python scripts/ops/report_pixelrag_vlm_route_readiness.py --model minicpm-v:late python scripts/ops/run_pixelrag_vlm_replay_worker.py python scripts/ops/run_pixelrag_vlm_replay_worker.py --platform shopee_tw --platform coupang_tw python scripts/ops/run_pixelrag_vlm_replay_worker.py --platform shopee_tw --execute --write-receipt --limit 1 +python scripts/ops/report_pixelrag_platform_probe.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_platform_probe_worker.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_platform_probe_worker.py --platform shopee_tw --execute --write-receipt --limit 1 +python scripts/ops/run_pixelrag_platform_probe_worker.py --platform coupang_tw --execute --write-receipt --limit 1 ``` API readback: `/api/ai-automation/pixelrag-vlm-route-readiness` 與 `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。worker 預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。execute 前會自動讀 approved Ollama `/api/tags` 選擇已安裝候選模型；若完全沒有候選，會寫 `model_route_not_ready` artifact receipt，不盲打 generate。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 +Platform probe worker API readback: `/api/ai-automation/pixelrag-platform-probe-worker?platform=shopee_tw`。worker 預設為 dry-run，不抓外站、不寫 artifact；`execute=true&write_receipt=true` 才對 Shopee language / traffic / generic landing 執行 public empty-context capture，或對 Coupang 403 / access denied 寫 structured-source/backoff receipt。capture 成功後仍回到 RAG candidate replay，不得直接寫 `ai_insights`、正式價格表或競品價格歷史。 + Application portfolio： ```bash diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 86b3259..5974c5a 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -862,6 +862,46 @@ def ai_automation_pixelrag_platform_probe_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-platform-probe-worker') +@login_required +def ai_automation_pixelrag_platform_probe_worker_api(): + """Controlled PixelRAG platform probe worker dry-run/execute readback.""" + from services.pixelrag_platform_probe_worker_service import ( + run_pixelrag_platform_probe_worker, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + execute = str(request.args.get('execute') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + write_receipt = str(request.args.get('write_receipt') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 25, type=int) + timeout = request.args.get('timeout', 30, type=int) + settle_ms = request.args.get('settle_ms', 800, type=int) + max_tiles = request.args.get('max_tiles', 12, type=int) + return jsonify(run_pixelrag_platform_probe_worker( + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 25, 250)), + timeout=max(1, min(timeout or 30, 120)), + settle_ms=max(0, min(settle_ms or 800, 5000)), + max_tiles=max(1, min(max_tiles or 12, 80)), + execute=execute, + write_receipt=bool(execute and write_receipt), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/run_pixelrag_platform_probe_worker.py b/scripts/ops/run_pixelrag_platform_probe_worker.py new file mode 100644 index 0000000..9c6fdea --- /dev/null +++ b/scripts/ops/run_pixelrag_platform_probe_worker.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Run or dry-run the PixelRAG platform probe worker.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_platform_probe_worker_service import ( # noqa: E402 + run_pixelrag_platform_probe_worker, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="執行或 dry-run PixelRAG platform probe worker。" + ) + parser.add_argument( + "--artifact-root", + help="PixelRAG visual evidence artifact root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--vlm-receipt-root", + help="PixelRAG VLM replay receipt root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--output-root", + help="Platform probe worker receipt output root。", + ) + parser.add_argument( + "--capture-output-root", + help="Public-context capture artifact output root；預設寫回 PixelRAG visual evidence root。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=25, + help="最多處理 probe candidate 數。", + ) + parser.add_argument( + "--timeout", + type=int, + default=30, + help="public-context capture timeout 秒數。", + ) + parser.add_argument( + "--settle-ms", + type=int, + default=800, + help="頁面載入後等待毫秒數。", + ) + parser.add_argument( + "--max-tiles", + type=int, + default=12, + help="每次 public-context capture 最多切出的 tiles。", + ) + parser.add_argument( + "--execute", + action="store_true", + help="真的執行 public-context capture 或 structured fallback package；未指定時只做 no-write dry-run。", + ) + parser.add_argument( + "--write-receipt", + action="store_true", + help="execute 後寫入 platform probe worker artifact receipt；不寫 DB。", + ) + args = parser.parse_args() + + payload = run_pixelrag_platform_probe_worker( + artifact_root=args.artifact_root, + vlm_receipt_root=args.vlm_receipt_root, + output_root=args.output_root, + capture_output_root=args.capture_output_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + timeout=args.timeout, + settle_ms=args.settle_ms, + max_tiles=args.max_tiles, + execute=args.execute, + write_receipt=bool(args.write_receipt and args.execute), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index c62d893..a2283b4 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -540,6 +540,11 @@ def build_scheduled_automation_health_summary( if not pixelrag_platform_probe or not pixelrag_platform_probe_details: pixelrag_platform_probe = _pixelrag_platform_probe_check() pixelrag_platform_probe_details = pixelrag_platform_probe.get("details") or {} + pixelrag_platform_probe_worker = _find_check(source_result, "PixelRAG platform probe worker") + pixelrag_platform_probe_worker_details = pixelrag_platform_probe_worker.get("details") or {} + if not pixelrag_platform_probe_worker or not pixelrag_platform_probe_worker_details: + pixelrag_platform_probe_worker = _pixelrag_platform_probe_worker_check() + pixelrag_platform_probe_worker_details = pixelrag_platform_probe_worker.get("details") or {} smoke_status = source_result.get("status") or ("warning" if latest_history else "warning") freshness_family = _history_freshness_family( latest_history, @@ -4728,6 +4733,56 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_platform_probe_worker", + "label": "PixelRAG platform probe worker", + "status": pixelrag_platform_probe_worker.get("status") or "warning", + "summary": ( + pixelrag_platform_probe_worker.get("summary") + or "PixelRAG platform probe worker has no latest readback." + ), + "next_machine_action": pixelrag_platform_probe_worker_details.get("next_machine_action") + or "run_pixelrag_platform_probe_worker_dry_run", + "details": { + "policy": pixelrag_platform_probe_worker_details.get("policy"), + "probe_candidate_count": int( + pixelrag_platform_probe_worker_details.get("probe_candidate_count") or 0 + ), + "ready_count": int( + pixelrag_platform_probe_worker_details.get("ready_count") or 0 + ), + "dry_run_count": int( + pixelrag_platform_probe_worker_details.get("dry_run_count") or 0 + ), + "capture_ready_count": int( + pixelrag_platform_probe_worker_details.get("capture_ready_count") or 0 + ), + "capture_execute_count": int( + pixelrag_platform_probe_worker_details.get("capture_execute_count") or 0 + ), + "capture_ok_count": int( + pixelrag_platform_probe_worker_details.get("capture_ok_count") or 0 + ), + "capture_error_count": int( + pixelrag_platform_probe_worker_details.get("capture_error_count") or 0 + ), + "structured_fallback_count": int( + pixelrag_platform_probe_worker_details.get("structured_fallback_count") or 0 + ), + "receipt_written_count": int( + pixelrag_platform_probe_worker_details.get("receipt_written_count") or 0 + ), + "network_call_performed": bool( + pixelrag_platform_probe_worker_details.get("network_call_performed") + ), + "artifact_write_performed": bool( + pixelrag_platform_probe_worker_details.get("artifact_write_performed") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, freshness_family, { "key": "daily_summary_delivery", @@ -12976,6 +13031,64 @@ def _pixelrag_platform_probe_check() -> Dict[str, Any]: ) +def _pixelrag_platform_probe_worker_check() -> Dict[str, Any]: + """Dry-run sentinel for PixelRAG platform probe worker automation.""" + try: + from services.pixelrag_platform_probe_worker_service import ( + run_pixelrag_platform_probe_worker, + ) + + readback = run_pixelrag_platform_probe_worker() + summary = readback.get("summary") or {} + candidate_count = int(summary.get("probe_candidate_count") or 0) + ready_count = int(summary.get("ready_count") or 0) + capture_ready_count = int(summary.get("capture_ready_count") or 0) + structured_count = int(summary.get("structured_fallback_count") or 0) + status = readback.get("status") or "warning" + summary_text = ( + f"PixelRAG platform probe worker candidates={candidate_count}, " + f"ready={ready_count}, capture_ready={capture_ready_count}, " + f"structured_fallback={structured_count}, execute=false" + ) + return _check( + "PixelRAG platform probe worker", + status, + summary_text, + { + "policy": readback.get("policy"), + "probe_candidate_count": candidate_count, + "ready_count": ready_count, + "skipped_count": int(summary.get("skipped_count") or 0), + "dry_run_count": int(summary.get("dry_run_count") or 0), + "capture_ready_count": capture_ready_count, + "capture_execute_count": int(summary.get("capture_execute_count") or 0), + "capture_ok_count": int(summary.get("capture_ok_count") or 0), + "capture_error_count": int(summary.get("capture_error_count") or 0), + "structured_fallback_count": structured_count, + "executed_structured_count": int(summary.get("executed_structured_count") or 0), + "executed_count": int(summary.get("executed_count") or 0), + "receipt_written_count": int(summary.get("receipt_written_count") or 0), + "network_call_performed": bool(summary.get("network_call_performed")), + "artifact_write_performed": bool(summary.get("artifact_write_performed")), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG platform probe worker", + "critical", + f"PixelRAG platform probe worker 無法執行：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: int = 20) -> Dict[str, Any]: checks: List[Dict[str, Any]] = [ _event_router_check(), @@ -13014,6 +13127,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _pixelrag_vlm_route_readiness_check(), _pixelrag_vlm_replay_worker_check(), _pixelrag_platform_probe_check(), + _pixelrag_platform_probe_worker_check(), ] worst = max(checks, key=lambda item: STATUS_RANK.get(item["status"], 2))["status"] result = { diff --git a/services/pixelrag_platform_probe_worker_service.py b/services/pixelrag_platform_probe_worker_service.py new file mode 100644 index 0000000..5d3b76c --- /dev/null +++ b/services/pixelrag_platform_probe_worker_service.py @@ -0,0 +1,530 @@ +"""Controlled PixelRAG platform probe worker. + +This worker executes the machine actions emitted by platform-probe readiness: +public empty-context visual capture for recoverable marketplace interstitials, +or structured-source/backoff receipts for hard blocked platforms. It never +reads sessions/cookies, logs in, writes DB rows, or promotes price truth. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.pixelrag_crawler_integration_service import ( + DEFAULT_ARTIFACT_MAX_AGE_HOURS, + DEFAULT_ARTIFACT_ROOT, +) +from services.pixelrag_platform_probe_service import ( + build_pixelrag_platform_probe_readiness, +) + + +POLICY = "controlled_pixelrag_platform_probe_worker_v1" +DEFAULT_LIMIT = 25 +DEFAULT_TIMEOUT_SECONDS = 30 +DEFAULT_SETTLE_MS = 800 +DEFAULT_MAX_TILES = 12 +DEFAULT_WORKER_RECEIPT_ROOT = os.getenv( + "PIXELRAG_PLATFORM_PROBE_WORKER_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_platform_probe_worker_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_platform_probe_worker_receipts", +) +DEFAULT_CAPTURE_OUTPUT_ROOT = os.getenv( + "PIXELRAG_PLATFORM_PROBE_CAPTURE_OUTPUT_ROOT", + str(DEFAULT_ARTIFACT_ROOT), +) +CAPTURE_READY_STATUSES = { + "ready_for_public_context_probe", + "ready_for_platform_probe", +} + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...]: + if isinstance(platform, str): + value = platform.strip().lower() + return (value,) if value else () + return tuple( + str(item or "").strip().lower() + for item in (platform or ()) + if str(item or "").strip() + ) + + +def _safe_segment(value: Any) -> str: + text = str(value or "unknown").strip().lower() + text = re.sub(r"[^a-z0-9._-]+", "-", text) + return text.strip("-") or "unknown" + + +def _capture_script_path() -> Path: + return Path(__file__).resolve().parents[1] / "scripts" / "ops" / "capture_pixelrag_visual_evidence.py" + + +def _base_item(item: Mapping[str, Any]) -> dict[str, Any]: + fallback = item.get("structured_source_fallback") + if not isinstance(fallback, Mapping): + fallback = {} + return { + "platform": item.get("platform"), + "manifest_id": item.get("manifest_id"), + "source_type": item.get("source_type"), + "source_receipt_path": item.get("source_receipt_path"), + "source_capture_receipt_path": item.get("source_capture_receipt_path"), + "probe_status": item.get("probe_status"), + "barrier_type": item.get("barrier_type"), + "url": item.get("url"), + "title": item.get("title"), + "structured_source_fallback_available": bool(fallback.get("available")), + "structured_source_count": len(list(fallback.get("sources") or [])), + "writes_database": False, + "model_call_performed": False, + "primary_human_gate_count": 0, + } + + +def _write_worker_receipt( + *, + output_root: Path, + item: Mapping[str, Any], + worker_item: Mapping[str, Any], +) -> str: + target = ( + output_root + / _safe_segment(item.get("platform")) + / _safe_segment(item.get("manifest_id")) + / "platform_probe_worker_receipt.json" + ) + target.parent.mkdir(parents=True, exist_ok=True) + receipt = dict(worker_item) + receipt["artifact_write_performed"] = True + receipt["receipt_path"] = str(target) + target.write_text( + json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + return str(target) + + +def _dry_run_item(item: Mapping[str, Any]) -> dict[str, Any]: + base = _base_item(item) + probe_status = str(item.get("probe_status") or "") + manifest = item.get("capture_manifest_preview") + capture_ready = ( + probe_status in CAPTURE_READY_STATUSES + and isinstance(manifest, Mapping) + and bool(manifest.get("success")) + ) + if capture_ready: + base.update({ + "worker_status": "dry_run_ready_for_public_context_capture", + "ready_for_execution": True, + "capture_execution_ready": True, + "capture_manifest_id": manifest.get("manifest_id"), + "public_browser_context_policy": ( + (manifest.get("public_browser_context") or {}).get("context_policy") + ), + "network_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": "run_pixelrag_platform_probe_worker_execute", + }) + return base + base.update({ + "worker_status": "dry_run_structured_source_fallback_ready", + "ready_for_execution": True, + "capture_execution_ready": False, + "network_call_performed": False, + "artifact_write_performed": False, + "structured_source_fallback": item.get("structured_source_fallback") or {}, + "next_machine_action": "run_structured_source_or_platform_backoff_policy", + }) + return base + + +def _skipped_item(item: Mapping[str, Any]) -> dict[str, Any]: + base = _base_item(item) + base.update({ + "worker_status": "skipped_not_probe_ready", + "ready_for_execution": False, + "capture_execution_ready": False, + "network_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": item.get("next_machine_action") + or "refresh_pixelrag_platform_probe_readiness", + }) + return base + + +def _structured_source_item( + item: Mapping[str, Any], + *, + output_root: Path, + execute: bool, + write_receipt: bool, +) -> dict[str, Any]: + base = _base_item(item) + fallback = item.get("structured_source_fallback") or {} + base.update({ + "worker_status": ( + "executed_structured_source_fallback_package" + if execute + else "dry_run_structured_source_fallback_ready" + ), + "ready_for_execution": True, + "capture_execution_ready": False, + "network_call_performed": False, + "artifact_write_performed": False, + "structured_source_fallback": fallback, + "structured_source_package": { + "adapter_code": fallback.get("adapter_code"), + "available": bool(fallback.get("available")), + "source_count": len(list(fallback.get("sources") or [])), + "network_request_allowed": bool(fallback.get("network_request_allowed")), + "database_write_allowed": False, + "dry_run_only": True, + "blocked_page_not_product_data": True, + }, + "next_machine_action": "run_structured_source_or_platform_backoff_policy", + }) + if execute and write_receipt: + base["receipt_path"] = _write_worker_receipt( + output_root=output_root, + item=item, + worker_item=base, + ) + base["artifact_write_performed"] = True + return base + + +def _run_capture_subprocess( + *, + manifest: Mapping[str, Any], + capture_output_root: Path, + timeout_seconds: int, + settle_ms: int, + max_tiles: int, + capture_script: Path | None = None, +) -> dict[str, Any]: + script = capture_script or _capture_script_path() + completed = subprocess.run( + [ + sys.executable, + str(script), + "--manifest-json", + json.dumps(dict(manifest), ensure_ascii=False), + "--output-dir", + str(capture_output_root), + "--timeout", + str(max(1, int(timeout_seconds or DEFAULT_TIMEOUT_SECONDS))), + "--settle-ms", + str(max(0, int(settle_ms or DEFAULT_SETTLE_MS))), + "--max-tiles", + str(max(1, int(max_tiles or DEFAULT_MAX_TILES))), + ], + capture_output=True, + check=False, + text=True, + ) + try: + payload = json.loads(completed.stdout) + except json.JSONDecodeError: + payload = {} + return { + "returncode": completed.returncode, + "stdout_json": payload, + "stdout_excerpt": completed.stdout[:500], + "stderr_excerpt": completed.stderr[:500], + } + + +def _capture_item( + item: Mapping[str, Any], + *, + output_root: Path, + capture_output_root: Path, + timeout_seconds: int, + settle_ms: int, + max_tiles: int, + write_receipt: bool, + capture_script: Path | None, +) -> dict[str, Any]: + base = _base_item(item) + manifest = item.get("capture_manifest_preview") + if not isinstance(manifest, Mapping) or not manifest.get("success"): + base.update({ + "worker_status": "skipped_missing_capture_manifest", + "ready_for_execution": False, + "capture_execution_ready": False, + "network_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": "refresh_pixelrag_platform_probe_manifest_or_use_structured_source", + }) + if write_receipt: + base["receipt_path"] = _write_worker_receipt( + output_root=output_root, + item=item, + worker_item=base, + ) + base["artifact_write_performed"] = True + return base + + result = _run_capture_subprocess( + manifest=manifest, + capture_output_root=capture_output_root, + timeout_seconds=timeout_seconds, + settle_ms=settle_ms, + max_tiles=max_tiles, + capture_script=capture_script, + ) + capture_payload = result["stdout_json"] + files = list(capture_payload.get("files") or []) if isinstance(capture_payload, Mapping) else [] + tile_file_count = sum(1 for file_item in files if file_item.get("kind") == "tile") + receipt_path = "" + planned_output = capture_payload.get("planned_output") if isinstance(capture_payload, Mapping) else {} + if isinstance(planned_output, Mapping): + receipt_path = str(planned_output.get("receipt") or "") + for file_item in files: + if file_item.get("kind") == "receipt": + receipt_path = str(file_item.get("path") or receipt_path) + + captured_ok = ( + result["returncode"] == 0 + and isinstance(capture_payload, Mapping) + and capture_payload.get("success") is True + and capture_payload.get("status") == "captured" + ) + base.update({ + "worker_status": "executed_capture_ok" if captured_ok else "capture_worker_error", + "ready_for_execution": True, + "capture_execution_ready": True, + "capture_manifest_id": manifest.get("manifest_id"), + "capture_returncode": result["returncode"], + "capture_status": capture_payload.get("status") if isinstance(capture_payload, Mapping) else "", + "capture_http_status": int(capture_payload.get("http_status") or 0) + if isinstance(capture_payload, Mapping) + else 0, + "capture_tile_file_count": tile_file_count, + "capture_receipt_path": receipt_path, + "network_call_performed": True, + "artifact_write_performed": bool(captured_ok), + "capture_stdout_excerpt": result["stdout_excerpt"] if not captured_ok else "", + "capture_stderr_excerpt": result["stderr_excerpt"] if not captured_ok else "", + "next_machine_action": ( + "run_pixelrag_rag_candidate_replay_after_probe_capture" + if captured_ok + else "repair_pixelrag_platform_probe_worker_capture_runtime_or_use_structured_source" + ), + }) + if write_receipt: + base["receipt_path"] = _write_worker_receipt( + output_root=output_root, + item=item, + worker_item=base, + ) + base["artifact_write_performed"] = True + return base + + +def run_pixelrag_platform_probe_worker( + *, + artifact_root: str | Path | None = None, + vlm_receipt_root: str | Path | None = None, + output_root: str | Path | None = None, + capture_output_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, + timeout: int | None = None, + settle_ms: int | None = None, + max_tiles: int | None = None, + execute: bool = False, + write_receipt: bool = False, + capture_script: str | Path | None = None, +) -> dict[str, Any]: + """Run or dry-run public platform probe machine actions.""" + output = Path(output_root or DEFAULT_WORKER_RECEIPT_ROOT) + capture_output = Path(capture_output_root or DEFAULT_CAPTURE_OUTPUT_ROOT) + platforms = _normalise_platforms(platform) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + timeout_seconds = max(1, min(int(timeout or DEFAULT_TIMEOUT_SECONDS), 120)) + settle = max(0, min(int(settle_ms or DEFAULT_SETTLE_MS), 5000)) + tiles = max(1, min(int(max_tiles or DEFAULT_MAX_TILES), 80)) + generated_at = datetime.now(timezone.utc).isoformat() + capture_script_path = Path(capture_script) if capture_script else None + readiness_artifact_root = artifact_root or DEFAULT_ARTIFACT_ROOT + + readiness = build_pixelrag_platform_probe_readiness( + artifact_root=readiness_artifact_root, + vlm_receipt_root=vlm_receipt_root, + platform=platforms, + max_age_hours=max_age, + limit=item_limit, + ) + probe_items = list(readiness.get("probe_items") or []) + worker_items: list[dict[str, Any]] = [] + for item in probe_items: + if not item.get("probe_ready"): + worker_items.append(_skipped_item(item)) + continue + probe_status = str(item.get("probe_status") or "") + manifest = item.get("capture_manifest_preview") + can_capture = ( + probe_status in CAPTURE_READY_STATUSES + and isinstance(manifest, Mapping) + and bool(manifest.get("success")) + ) + if not execute: + worker_items.append(_dry_run_item(item)) + continue + if can_capture: + worker_items.append(_capture_item( + item, + output_root=output, + capture_output_root=capture_output, + timeout_seconds=timeout_seconds, + settle_ms=settle, + max_tiles=tiles, + write_receipt=write_receipt, + capture_script=capture_script_path, + )) + else: + worker_items.append(_structured_source_item( + item, + output_root=output, + execute=True, + write_receipt=write_receipt, + )) + + ready_count = sum(1 for item in probe_items if item.get("probe_ready")) + dry_run_count = sum(1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_")) + structured_count = sum( + 1 for item in worker_items if "structured_source_fallback" in str(item.get("worker_status") or "") + ) + capture_ready_count = sum(1 for item in worker_items if item.get("capture_execution_ready")) + capture_execute_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_capture") + ) + capture_ok_count = sum(1 for item in worker_items if item.get("worker_status") == "executed_capture_ok") + capture_error_count = sum(1 for item in worker_items if item.get("worker_status") == "capture_worker_error") + executed_structured_count = sum( + 1 for item in worker_items if item.get("worker_status") == "executed_structured_source_fallback_package" + ) + executed_count = capture_execute_count + executed_structured_count + skipped_count = sum(1 for item in worker_items if str(item.get("worker_status") or "").startswith("skipped_")) + receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path")) + network_call_performed = any(bool(item.get("network_call_performed")) for item in worker_items) + artifact_write_performed = any(bool(item.get("artifact_write_performed")) for item in worker_items) + + if capture_error_count: + status = "critical" if execute and capture_ok_count == 0 and capture_ready_count else "warning" + elif probe_items and ready_count: + status = "ok" + elif probe_items: + status = "warning" + else: + status = "warning" + + if not probe_items: + next_action = "run_pixelrag_visual_capture_worker" + elif capture_error_count: + next_action = "repair_pixelrag_platform_probe_worker_capture_runtime_or_use_structured_source" + elif not execute and capture_ready_count: + next_action = "run_pixelrag_platform_probe_worker_execute" + elif execute and capture_ok_count: + next_action = "run_pixelrag_rag_candidate_replay_after_probe_capture" + elif structured_count: + next_action = "run_structured_source_or_platform_backoff_policy" + else: + next_action = "refresh_pixelrag_platform_probe_readiness" + + summary = { + "probe_candidate_count": len(probe_items), + "ready_count": ready_count, + "skipped_count": skipped_count, + "dry_run_count": dry_run_count, + "capture_ready_count": capture_ready_count, + "capture_execute_count": capture_execute_count, + "capture_ok_count": capture_ok_count, + "capture_error_count": capture_error_count, + "structured_fallback_count": structured_count, + "executed_structured_count": executed_structured_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "network_call_performed": network_call_performed, + "artifact_write_performed": artifact_write_performed, + "writes_database_count": 0, + "primary_human_gate_count": 0, + "platforms": sorted({str(item.get("platform") or "unknown") for item in probe_items}), + } + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": generated_at, + "artifact_root": str(readiness_artifact_root), + "vlm_receipt_root": str( + vlm_receipt_root + or readiness.get("vlm_receipt_root") + or "" + ), + "output_root": str(output), + "capture_output_root": str(capture_output), + "platform_filter": list(platforms), + "max_age_hours": max_age, + "limit": item_limit, + "timeout_seconds": timeout_seconds, + "settle_ms": settle, + "max_tiles": tiles, + "execute": bool(execute), + "write_receipt": bool(write_receipt), + "summary": summary, + "worker_items": worker_items, + "source_readiness": { + "policy": readiness.get("policy"), + "status": readiness.get("status"), + "summary": readiness.get("summary"), + "next_machine_action": readiness.get("next_machine_action"), + }, + "controlled_apply": { + "network_call": bool(execute and network_call_performed), + "model_call": False, + "artifact_write": artifact_write_performed, + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "secret_read": False, + "raw_cookie_or_session_read": False, + "credentialed_session_allowed": False, + "login_allowed": False, + "production_price_write": False, + "primary_human_gate_count": 0, + }, + "promotion_boundary": { + "writes_ai_insights": False, + "writes_price_tables": False, + "blocked_pages_are_not_product_data": True, + "visual_fields_are_candidate_evidence_only": True, + "requires_rag_candidate_replay_after_capture": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "DEFAULT_CAPTURE_OUTPUT_ROOT", + "DEFAULT_WORKER_RECEIPT_ROOT", + "POLICY", + "run_pixelrag_platform_probe_worker", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index 7caf4bb..990f125 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1314,11 +1314,12 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_platform_probe_check", lambda: smoke._check("pixelrag platform probe", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_platform_probe_worker_check", lambda: smoke._check("pixelrag platform probe worker", "ok", "ok")) result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 34, "warning": 1, "critical": 1, "total": 36} + assert result["summary"] == {"ok": 35, "warning": 1, "critical": 1, "total": 37} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3851,6 +3852,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_platform_probe_check", lambda: smoke._check("pixelrag platform probe", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_platform_probe_worker_check", lambda: smoke._check("pixelrag platform probe worker", "ok", "ok")) first = smoke.collect_ai_automation_smoke(history_limit=5) second = smoke.collect_ai_automation_smoke(history_limit=5) @@ -3906,7 +3908,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 36, "warning": 0, "critical": 0, "total": 36}, + "summary": {"ok": 37, "warning": 0, "critical": 0, "total": 37}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4089,6 +4091,28 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "writes_database_count": 0, "primary_human_gate_count": 0, }, + }, + { + "name": "PixelRAG platform probe worker", + "status": "ok", + "summary": "PixelRAG platform probe worker candidates=1, ready=1, capture_ready=1, structured_fallback=0, execute=false", + "details": { + "policy": "controlled_pixelrag_platform_probe_worker_v1", + "probe_candidate_count": 1, + "ready_count": 1, + "dry_run_count": 1, + "capture_ready_count": 1, + "capture_execute_count": 0, + "capture_ok_count": 0, + "capture_error_count": 0, + "structured_fallback_count": 0, + "receipt_written_count": 0, + "network_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": "run_pixelrag_platform_probe_worker_execute", + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, } ], }, ensure_ascii=False) + "\n", @@ -4107,7 +4131,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 33 + assert summary["summary"]["total"] == 34 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -5814,6 +5838,15 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( assert pixelrag_platform_probe_family["status"] == "ok" assert pixelrag_platform_probe_family["details"]["ready_for_probe_count"] == 1 assert pixelrag_platform_probe_family["details"]["primary_human_gate_count"] == 0 + pixelrag_platform_probe_worker_family = next( + item for item in summary["families"] + if item["key"] == "pixelrag_platform_probe_worker" + ) + assert pixelrag_platform_probe_worker_family["status"] == "ok" + assert pixelrag_platform_probe_worker_family["details"]["ready_count"] == 1 + assert pixelrag_platform_probe_worker_family["details"]["dry_run_count"] == 1 + assert pixelrag_platform_probe_worker_family["details"]["network_call_performed"] is False + assert pixelrag_platform_probe_worker_family["details"]["writes_database_count"] == 0 assert summary["scheduled_outputs"]["telegram_send_in_preview"] is False assert summary["scheduled_outputs"]["writes_database"] is False assert summary["automation_policy"]["primary_flow"] == "ai_controlled" @@ -6450,6 +6483,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag platform probe ok", )) + monkeypatch.setattr(smoke, "_pixelrag_platform_probe_worker_check", lambda: smoke._check( + "PixelRAG platform probe worker", + "ok", + "pixelrag platform probe worker ok", + )) result = smoke.collect_ai_automation_smoke(record_history=False) @@ -6465,7 +6503,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 36 + assert result["summary"]["total"] == 37 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_platform_probe_worker_service.py b/tests/test_pixelrag_platform_probe_worker_service.py new file mode 100644 index 0000000..8236375 --- /dev/null +++ b/tests/test_pixelrag_platform_probe_worker_service.py @@ -0,0 +1,248 @@ +import json +import subprocess +import sys +from pathlib import Path + +from tests.test_pixelrag_rag_candidate_replay_service import _write_receipt + + +def _patch_receipt(path, **updates): + payload = json.loads(path.read_text(encoding="utf-8")) + for key, value in updates.items(): + if key == "page_metrics": + payload.setdefault("page_metrics", {}).update(value) + else: + payload[key] = value + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return payload + + +def test_pixelrag_platform_probe_worker_dry_run_splits_capture_and_structured(tmp_path): + from services.pixelrag_platform_probe_worker_service import ( + POLICY, + run_pixelrag_platform_probe_worker, + ) + + shopee_receipt = _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-traffic", + title="蝦皮購物 | 花得更少買得更好", + url="https://shopee.tw/search?keyword=sunscreen", + ) + _patch_receipt( + shopee_receipt, + page_metrics={ + "final_url": "https://shopee.tw/verify/traffic/error?home_url=https%3A%2F%2Fshopee.tw", + "title": "蝦皮購物 | 花得更少買得更好", + }, + ) + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + + payload = run_pixelrag_platform_probe_worker( + artifact_root=tmp_path, + platform=("shopee_tw", "coupang_tw"), + ) + by_platform = {item["platform"]: item for item in payload["worker_items"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "ok" + assert payload["execute"] is False + assert payload["summary"]["probe_candidate_count"] == 2 + assert payload["summary"]["ready_count"] == 2 + assert payload["summary"]["dry_run_count"] == 2 + assert payload["summary"]["capture_ready_count"] == 1 + assert payload["summary"]["structured_fallback_count"] == 1 + assert payload["summary"]["network_call_performed"] is False + assert payload["summary"]["writes_database_count"] == 0 + assert payload["controlled_apply"]["primary_human_gate_count"] == 0 + assert payload["next_machine_action"] == "run_pixelrag_platform_probe_worker_execute" + assert by_platform["shopee_tw"]["worker_status"] == "dry_run_ready_for_public_context_capture" + assert by_platform["shopee_tw"]["public_browser_context_policy"] == ( + "public_empty_browser_context_no_login" + ) + assert by_platform["coupang_tw"]["worker_status"] == ( + "dry_run_structured_source_fallback_ready" + ) + assert by_platform["coupang_tw"]["structured_source_fallback_available"] is True + + +def test_pixelrag_platform_probe_worker_execute_capture_writes_receipt(tmp_path, monkeypatch): + from services import pixelrag_platform_probe_worker_service as service + + shopee_receipt = _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-traffic", + title="蝦皮購物 | 花得更少買得更好", + url="https://shopee.tw/search?keyword=sunscreen", + ) + _patch_receipt( + shopee_receipt, + page_metrics={ + "final_url": "https://shopee.tw/verify/traffic/error?home_url=https%3A%2F%2Fshopee.tw", + "title": "蝦皮購物 | 花得更少買得更好", + }, + ) + + def fake_capture_subprocess(**kwargs): + manifest = kwargs["manifest"] + capture_root = Path(kwargs["capture_output_root"]) + receipt_path = capture_root / "shopee_tw" / manifest["manifest_id"] / "capture_receipt.json" + return { + "returncode": 0, + "stdout_json": { + "success": True, + "status": "captured", + "http_status": 200, + "planned_output": {"receipt": str(receipt_path)}, + "files": [ + {"kind": "fullpage_screenshot", "path": str(receipt_path.parent / "fullpage.png")}, + {"kind": "tile", "path": str(receipt_path.parent / "tiles" / "tile_000.png")}, + {"kind": "receipt", "path": str(receipt_path)}, + ], + }, + "stdout_excerpt": "", + "stderr_excerpt": "", + } + + monkeypatch.setattr(service, "_run_capture_subprocess", fake_capture_subprocess) + payload = service.run_pixelrag_platform_probe_worker( + artifact_root=tmp_path, + output_root=tmp_path / "worker_receipts", + capture_output_root=tmp_path / "capture_output", + platform="shopee_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["capture_execute_count"] == 1 + assert payload["summary"]["capture_ok_count"] == 1 + assert payload["summary"]["receipt_written_count"] == 1 + assert payload["summary"]["network_call_performed"] is True + assert payload["controlled_apply"]["network_call"] is True + assert payload["controlled_apply"]["writes_database"] is False + item = payload["worker_items"][0] + assert item["worker_status"] == "executed_capture_ok" + assert item["next_machine_action"] == "run_pixelrag_rag_candidate_replay_after_probe_capture" + receipt_path = ( + tmp_path + / "worker_receipts" + / "shopee_tw" + / "shopee-traffic" + / "platform_probe_worker_receipt.json" + ) + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "executed_capture_ok" + assert receipt["artifact_write_performed"] is True + assert receipt["receipt_path"] == str(receipt_path) + + +def test_pixelrag_platform_probe_worker_execute_structured_fallback_writes_receipt(tmp_path): + from services.pixelrag_platform_probe_worker_service import run_pixelrag_platform_probe_worker + + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + + payload = run_pixelrag_platform_probe_worker( + artifact_root=tmp_path, + output_root=tmp_path / "worker_receipts", + platform="coupang_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["executed_structured_count"] == 1 + assert payload["summary"]["capture_execute_count"] == 0 + assert payload["summary"]["network_call_performed"] is False + assert payload["summary"]["receipt_written_count"] == 1 + item = payload["worker_items"][0] + assert item["worker_status"] == "executed_structured_source_fallback_package" + assert item["structured_source_package"]["adapter_code"] == "coupang" + assert item["structured_source_package"]["database_write_allowed"] is False + receipt_path = ( + tmp_path + / "worker_receipts" + / "coupang_tw" + / "coupang-403" + / "platform_probe_worker_receipt.json" + ) + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "executed_structured_source_fallback_package" + assert receipt["artifact_write_performed"] is True + + +def test_pixelrag_platform_probe_worker_cli_outputs_machine_readable_json(tmp_path): + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/run_pixelrag_platform_probe_worker.py", + "--artifact-root", + str(tmp_path), + "--platform", + "coupang_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["probe_candidate_count"] == 1 + assert payload["summary"]["structured_fallback_count"] == 1 + assert payload["controlled_apply"]["network_call"] is False + assert payload["controlled_apply"]["writes_database"] is False + + +def test_pixelrag_platform_probe_worker_route_returns_readback(tmp_path, monkeypatch): + from flask import Flask + from routes import system_public_routes as routes + from services import pixelrag_platform_probe_worker_service as service + + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + monkeypatch.setattr(service, "DEFAULT_ARTIFACT_ROOT", str(tmp_path)) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-platform-probe-worker?platform=coupang_tw" + ): + response = routes.ai_automation_pixelrag_platform_probe_worker_api.__wrapped__() + payload = response.get_json() + + assert payload["policy"] == "controlled_pixelrag_platform_probe_worker_v1" + assert payload["summary"]["probe_candidate_count"] == 1 + assert payload["summary"]["structured_fallback_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False