From 079832f2a46e14ed87d57bb8807c25d8b722c475 Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 21:55:45 +0800 Subject: [PATCH 01/75] chore(dev): sync ewoooc development workbench state --- .env.example | 37 + docs/memory/README.md | 2 + .../code_modularization_inventory_20260430.md | 74 +- .../memory/pchome_mapping_backlog_20260628.md | 117 +++ .../production_version_truth_20260627.md | 29 + scripts/ops/report_pchome_mapping_backlog.py | 153 ++++ services/ai_automation_smoke_service.py | 699 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 123 +++ tests/test_auto_import_failure_boundaries.py | 1 + tests/test_competitor_identity_revalidator.py | 18 +- tests/test_competitor_intel_cache.py | 30 +- tests/test_elephant_alpha_engine.py | 10 +- tests/test_event_router.py | 19 +- tests/test_external_market_offer_service.py | 38 +- tests/test_frontend_v2_assets.py | 76 +- tests/test_market_intel_skeleton.py | 2 +- tests/test_migration_metadata_coverage.py | 15 + tests/test_momo_crawler_targeted_search.py | 2 +- tests/test_nemotron_decision_envelope.py | 3 +- tests/test_openclaw_daily_template.py | 2 +- tests/test_pchome_revenue_growth_service.py | 25 +- tests/test_telegram_triaged_alert_format.py | 38 +- tests/test_webcrumbs_host_data_service.py | 17 +- web/static/css/page-dashboard-v2.css | 208 ++++++ web/static/css/page-growth-bem.css | 18 + 25 files changed, 1649 insertions(+), 107 deletions(-) create mode 100644 docs/memory/pchome_mapping_backlog_20260628.md create mode 100644 docs/memory/production_version_truth_20260627.md create mode 100755 scripts/ops/report_pchome_mapping_backlog.py diff --git a/.env.example b/.env.example index 151edbf..ac91b08 100644 --- a/.env.example +++ b/.env.example @@ -46,6 +46,10 @@ EMAIL_RECEIVER=receiver_email@gmail.com # ========================================== PUBLIC_URL=http://your_server_ip:port NGROK_AUTH_TOKEN=your_ngrok_auth_token +FLASK_ENV=production +KUBERNETES_SERVICE_HOST= +CICD_PROD_BASE_URL= +CICD_UAT_BASE_URL= # ========================================== # Webcrumbs 共用 UI Runtime @@ -80,6 +84,9 @@ FRONTEND_VERSION=stable MARKET_INTEL_ENABLED=false MARKET_INTEL_CRAWLER_ENABLED=false MARKET_INTEL_WRITE_ENABLED=false +MCP_FAST_STATIC_FALLBACK=false +MCP_OLLAMA_TIMEOUT=30 +MCP_OLLAMA_NUM_PREDICT=512 # ========================================== # 通訊模組設定（從環境變數讀取） @@ -156,6 +163,33 @@ REDIS_PORT=6379 # Token 檔案位置：config/google_token.pickle（首次認證後自動產生） GDRIVE_FOLDER_PATH=業績報表/當日業績 GDRIVE_FILE_PATTERN=即時業績_當日 +GOOGLE_DRIVE_CREDENTIALS_FILE=config/google_credentials.json +GOOGLE_DRIVE_TOKEN_FILE=config/google_token.json +GOOGLE_DRIVE_LEGACY_PICKLE_FILE=config/google_token.pickle + +# ========================================== +# 匯入 / PChome / 外部商品資料控制 +# ========================================== +DAILY_SALES_IMPORT_STALENESS_GUARD_ENABLED=true +DAILY_SALES_IMPORT_MAX_STALE_DAYS=3 +EXTERNAL_OFFER_SYNC_LIMIT=200 +PCHOME_PUBLIC_API_TIMEOUT_SECONDS=8 +PCHOME_STALE_RECOVERY_ENABLED=false +PCHOME_SALES_JOIN_BY_MOMO_SKU_ENABLED=false +PCHOME_GROWTH_MOMO_BACKFILL_LIMIT=20 +PCHOME_BACKFILL_EXPIRED_REFRESH_LIMIT=20 +PCHOME_BACKFILL_RETRYABLE_LIMIT=20 +PCHOME_BACKFILL_REVALIDATION_LIMIT=20 +PCHOME_BACKFILL_UNMATCHED_LIMIT=20 +PCHOME_FEEDER_TTL_HOURS=24 +PCHOME_FEEDER_BACKFILL_TIMEOUT=8 +PCHOME_FEEDER_BACKFILL_SEARCH_MAX_PAGES=1 +PCHOME_FEEDER_BACKFILL_MAX_SEARCH_TERMS=3 +PCHOME_FEEDER_BACKFILL_MAX_SECONDS_PER_SKU=15 +MOMO_TARGETED_SEARCH_LIMIT_PER_TERM=3 +MOMO_TARGETED_SEARCH_MAX_PRODUCTS=10 +MOMO_TARGETED_SEARCH_MAX_TERMS=3 +MOMO_TARGETED_SEARCH_MIN_SCORE=0.5 # ========================================== # Hermes 3 競價情報分析（Module 2 / ADR-012） @@ -184,6 +218,8 @@ OLLAMA_EMBED_GCP_FAILURE_NOTICE_SEC=30 # [預設 true] 背景 embedding 會讀 host_health_probes，跳過最近 runtime unhealthy 的 GCP 節點 OLLAMA_EMBED_HOST_HEALTH_SKIP_ENABLED=true OLLAMA_EMBED_HOST_HEALTH_SKIP_WINDOW_MINUTES=20 +OLLAMA_SECONDARY_MODEL_FALLBACK=gemma3:4b +OLLAMA_SECONDARY_MODEL_FALLBACK_PATTERNS= # 111 Mac final fallback guardrail and allowlist proxy OLLAMA_111_CIRCUIT_BREAKER_ENABLED=true @@ -236,6 +272,7 @@ ELEPHANT_ALPHA_HERMES_MODEL=hermes3:latest ELEPHANT_ALPHA_NEMOTRON_NIM_ENDPOINT=https://integrate.api.nvidia.com/v1 ELEPHANT_ALPHA_URL=https://integrate.api.nvidia.com/v1/chat/completions ELEPHANT_ALPHA_OPENCLAW_GEMINI_ENDPOINT=https://generativelanguage.googleapis.com/v1beta +PPT_SCHEDULED_FAST_FALLBACK=false # ── Google Gemini API（僅備援 / 鎖定場景）──────────────────────────────────── # Gemini 只能作為 Ollama 失敗備援或 ADR-028 鎖定場景，不可設為通用預設 provider diff --git a/docs/memory/README.md b/docs/memory/README.md index 9d9a54f..a9943c0 100644 --- a/docs/memory/README.md +++ b/docs/memory/README.md @@ -25,6 +25,8 @@ | `schema_inventory_baseline.md` | DB 表分類與 drift 基線 | 要收斂 migration / ORM / raw SQL 真相時 | | `frontend_v3_handoff_20260512.md` | 前端 V3 守門落地、正式部署、UI/UX 響應式缺口與接續順序 | 新 session 接續前端 V3 全站 UI/UX、手機版確認、`/daily_sales` 或 `/edm` 修正與部署時 | | `claude_inventory_validation_20260513.md` | Claude Code V1/V2 盤點逐項驗證、已修項與不可盲動清單 | 接續盤點整改、判斷某項是否仍是阻擋、避免重複清已修項時 | +| `production_version_truth_20260627.md` | 正式環境 `/health` 作為最新版本真相，避免 local/Gitea/iCloud drift 誤判 | 開工、部署前、handoff 或任何版本宣稱前 | +| `pchome_mapping_backlog_20260628.md` | PChome growth API 的商品對應缺口只讀盤點、mapping rate 與下一步順序 | 接續 PChome 商品對應率、正式 API readback、mapping backlog 優化時 | ## 關聯 Guide diff --git a/docs/memory/code_modularization_inventory_20260430.md b/docs/memory/code_modularization_inventory_20260430.md index ce423cd..a9a7900 100644 --- a/docs/memory/code_modularization_inventory_20260430.md +++ b/docs/memory/code_modularization_inventory_20260430.md @@ -7,7 +7,7 @@ - Python 總量：約 139,476 行（排除 `venv/`、`backups/`、`__pycache__/`、`.claude/worktrees/`）。 - 最大壓力區：`services/` 約 84,159 行、`routes/` 約 36,245 行。 - `app.py` 目前約 1,232 行，功能定位應固定為 bootstrap / Blueprint registration / startup guard，不再承接新 route。 -- 目前工作樹仍有 33 個 Python 檔案達到或超過 800 行；這些不是禁止修 bug，而是禁止繼續塞新功能。 +- 目前工作樹仍有 42 個 Python 檔案達到或超過 800 行；這些不是禁止修 bug，而是禁止繼續塞新功能。 - 2026-05-05 追記：Phase 38→56 觀測台戰役讓 `routes/admin_observability_routes.py` 與 `run_scheduler.py` 進入大檔治理清單；後續觀測台功能應先抽 query/action service，不再把新 SQL 與 L2 mutation 直接塞回 route。 - 2026-05-06 追記：跨平台市場情報模組啟動前，必須先把新增爬蟲、排程、DB schema、UI route 全部隔離在 `market_*` / `services/market_intel/` / `routes/market_intel_routes.py`，不可塞回既有大檔。 - 2026-05-18 追記：Phase 42 市場情報只在 `app.py` 的 `EXPECTED_METADATA_TABLES` 補上 `market_alert_review_queue` 名稱，未新增 route / bootstrap 邏輯；後續仍應把 metadata verification 抽到 app factory 或 startup guard module，避免 `app.py` 繼續承接功能。 @@ -71,46 +71,56 @@ - 2026-05-24 追記：同步 PChome review queue 決策信封合併後的 `services/competitor_intel_repository.py` 行數；此處只更新 inventory，不變更拆分策略。 - 2026-05-25 追記：同步背景 embedding 讀取 `host_health_probes` skip guard 後的 `services/ollama_service.py` 行數；此處只更新 inventory，不變更 Ollama 路由決策。 - 2026-05-29 追記：同步 PChome near-threshold / focused identity 回收系列後的 `services/marketplace_product_matcher.py` 行數；此處只更新 inventory，不變更拆分策略。 +- 2026-06-29 追記：同步 AI 自動化例外決策、PChome mapping backlog 與 market intel alias work 後的 >800 行檔案清單；此處只更新 inventory，不變更拆分策略。 +- 2026-06-29 追記：同步 market intel AI-controlled canonical route aliases 後的 route extension 行數；`routes/market_intel_mcp_run_routes.py` 與 `routes/market_intel_review_post_routes.py` 進入 800 行治理清單，下一步新增 route alias 應優先抽 route registration helper，避免 extension 檔繼續膨脹。 +- 2026-06-29 追記：同步 market intel review/report endpoint compatibility migration 後的 route extension 行數；`routes/market_intel_review_routes.py` 進入 800 行治理清單，下一步應抽 `market_intel_review_route_registration.py` 或 declarative endpoint map，避免 decorator 重複擴張。 ## 達到或超過 800 行檔案清單 | 行數 | 檔案 | 分類 | 拆分方向 | |---:|---|---|---| -| 9225 | `routes/openclaw_bot_routes.py` | P0 巨型 Blueprint | route / bot command service / report service / scheduler hook；禁止再新增市場情報入口 | -| 5499 | `services/ppt_generator.py` | P0 報表生成巨型 service | deck orchestration / slide builders / chart builders / report type registry | -| 3186 | `routes/sales_routes.py` | P0 巨型 Blueprint | page routes / API routes / chart query service / calendar service；分析頁新增功能先抽 `services/sales/` | -| 2973 | `scheduler.py` | P0 排程總管 | task registry / crawler jobs / report jobs / notification jobs；市場情報只能透過獨立 job module 掛入 | -| 2731 | `services/openclaw_strategist_service.py` | P0 OpenClaw service | prompt builders / report composer / strategy rules | -| 3681 | `routes/admin_observability_routes.py` | P0 觀測台巨型 Blueprint | `services/observability_query_service.py` / `services/observability_action_service.py` / route glue | -| 1796 | `routes/ai_routes.py` | P1 AI Blueprint | route glue / AI orchestration service / prompt builders | -| 2154 | `services/nemoton_dispatcher_service.py` | P1 NemoTron service | NIM client / tool-call parser / action dispatcher | -| 2535 | `routes/dashboard_routes.py` | P1 Dashboard Blueprint | competitor decision overview / dashboard query service；首頁資料整併需抽 service | -| 1485 | `routes/vendor_routes.py` | P1 Vendor Blueprint | route glue / stockout mutation/email；V2 page query、stockout list/batches API query、vendor list/detail query 已抽到 `services/vendor_stockout_query_service.py` | -| 1390 | `services/telegram_bot_service.py` | P1 Telegram service | command handlers / message formatters / bot client | -| 1237 | `app.py` | P1 bootstrap | 保持只做 app setup；繼續往 app_factory / extension setup 抽；Phase 42 只做 metadata table name 對齊 | -| 2149 | `services/elephant_alpha_autonomous_engine.py` | P1 ElephantAlpha engine | HITL / executor / planning policy | -| 970 | `routes/cicd_routes.py` | P2 CI/CD Blueprint | route glue / CI query service / deployment action service | -| 1250 | `run_scheduler.py` | P2 scheduler entrypoint | observability jobs / token report jobs / task registration 分離 | -| 916 | `services/ppt_auto_generation_service.py` | P2 PPT 自動產線 service | schedule resolver / generation queue / missing report planner | +| 9424 | `routes/openclaw_bot_routes.py` | P0 巨型 Blueprint | route / bot command service / report service / scheduler hook；禁止再新增市場情報入口 | +| 8689 | `services/pchome_mapping_backlog_service.py` | P0 PChome mapping backlog service | mapping backlog / evidence receipt / controlled apply package 分拆到 `services/pchome_mapping/` | +| 5865 | `services/marketplace_product_matcher.py` | P2 marketplace matcher | identity parsing / unit-comparable scoring / search term quality / persistence normalization | +| 5513 | `services/ppt_generator.py` | P0 報表生成巨型 service | deck orchestration / slide builders / chart builders / report type registry | +| 4398 | `routes/ai_routes.py` | P1 AI Blueprint | route glue / AI orchestration service / prompt builders | +| 4152 | `routes/admin_observability_routes.py` | P0 觀測台巨型 Blueprint | `services/observability_query_service.py` / `services/observability_action_service.py` / route glue | +| 3763 | `services/competitor_price_feeder.py` | P2 competitor price feeder | crawler scheduling / price normalization / retryable candidate recovery / cache strategy | +| 3290 | `routes/sales_routes.py` | P0 巨型 Blueprint | page routes / API routes / chart query service / calendar service；分析頁新增功能先抽 `services/sales/` | +| 3125 | `routes/dashboard_routes.py` | P1 Dashboard Blueprint | competitor decision overview / dashboard query service；首頁資料整併需抽 service | +| 3107 | `scheduler.py` | P0 排程總管 | task registry / crawler jobs / report jobs / notification jobs；市場情報只能透過獨立 job module 掛入 | +| 2961 | `services/openclaw_strategist_service.py` | P0 OpenClaw service | prompt builders / report composer / strategy rules | +| 2383 | `services/competitor_intel_repository.py` | P2 competitor intel repository | review queue query / cache shaping / formatting helpers | +| 2332 | `services/elephant_alpha_autonomous_engine.py` | P1 ElephantAlpha engine | AI exception policy / executor / planning policy | +| 2166 | `services/nemoton_dispatcher_service.py` | P1 NemoTron service | NIM client / tool-call parser / action dispatcher | +| 2017 | `services/market_intel/deployment_readiness.py` | P2 market intel deployment readiness | preflight gates / readiness payload / route contract helpers | +| 1956 | `services/external_market_offer_service.py` | P2 external offer service | offer ingestion / freshness / quality normalization 分離 | +| 1658 | `routes/market_intel_review_report_routes.py` | P2 market intel review report Blueprint | review report route glue / export payload / phase handoff orchestration | +| 1494 | `routes/vendor_routes.py` | P1 Vendor Blueprint | route glue / stockout mutation/email；V2 page query、stockout list/batches API query、vendor list/detail query 已抽到 `services/vendor_stockout_query_service.py` | +| 1456 | `services/ollama_service.py` | P2 Ollama client | host health / request client / fallback policy / response parsing | +| 1417 | `services/telegram_templates.py` | P2 Telegram templates | alert template groups / channel-specific formatting / reusable render helpers | +| 1394 | `services/telegram_bot_service.py` | P1 Telegram service | command handlers / message formatters / bot client | +| 1385 | `services/import_service.py` | P2 import service | validators / import writers / report builders | +| 1306 | `run_scheduler.py` | P2 scheduler entrypoint | observability jobs / token report jobs / task registration 分離 | +| 1271 | `services/code_review_pipeline_service.py` | P2 Code review pipeline service | scan orchestration / finding normalization / persistence adapter | +| 1255 | `app.py` | P1 bootstrap | 保持只做 app setup；繼續往 app_factory / extension setup 抽；Phase 42 只做 metadata table name 對齊 | +| 1188 | `services/ppt_auto_generation_service.py` | P2 PPT 自動產線 service | schedule resolver / generation queue / missing report planner | +| 1023 | `routes/cicd_routes.py` | P2 CI/CD Blueprint | route glue / CI query service / deployment action service | +| 1014 | `routes/daily_sales_routes.py` | P2 Daily Sales Blueprint | route glue / export helpers / daily query and formatting service | +| 1013 | `routes/export_routes.py` | P2 Export flow | export command/router glue / file path / download orchestration | | 966 | `services/trend_crawler.py` | P2 crawler service | source adapters / parser / persistence | | 942 | `services/learning_pipeline.py` | P2 RAG learning pipeline | distiller / promotion gate / persistence / telemetry | -| 940 | `services/import_service.py` | P2 import service | validators / import writers / report builders | -| 1071 | `services/telegram_templates.py` | P2 Telegram templates | alert template groups / channel-specific formatting / reusable render helpers | +| 941 | `services/pchome_revenue_growth_service.py` | P2 PChome revenue growth service | growth aggregation / mapping backlog handoff / source adapter 分離 | +| 931 | `services/pchome_crawler.py` | P2 PChome crawler | search fetch / parsing / fallback source handling / rate limit policy | +| 920 | `routes/market_intel_routes.py` | P2 market intel Blueprint | page route / API route glue / MCP gate route registration helper | +| 906 | `services/competitor_match_attempt_rescore_audit.py` | P2 rescore audit service | audit query / matcher replay / summary payload 分離 | +| 895 | `routes/market_intel_review_routes.py` | P2 market intel review route extension | review decision route glue / canonical alias decorators / legacy endpoint compatibility | | 867 | `services/token_report_service.py` | P2 token report service | query / aggregation / chart payload / notification formatting | -| 4865 | `services/marketplace_product_matcher.py` | P2 marketplace matcher | identity parsing / unit-comparable scoring / search term quality / persistence normalization | -| 865 | `routes/daily_sales_routes.py` | P2 Daily Sales Blueprint | route glue / export helpers / daily query and formatting service | -| 1266 | `services/ollama_service.py` | P2 Ollama client | host health / request client / fallback policy / response parsing | -| 849 | `services/pchome_crawler.py` | P2 PChome crawler | search fetch / parsing / fallback source handling / rate limit policy | -| 1100 | `services/code_review_pipeline_service.py` | P2 Code review pipeline service | scan orchestration / finding normalization / persistence adapter | -| 953 | `routes/export_routes.py` | P2 Export flow | export command/router glue / file path / download orchestration | -| 816 | `services/ppt_vision_service.py` | P2 PPT vision QA service | runtime state / queue status / model probe / audit execution 分離 | -| 2149 | `services/competitor_price_feeder.py` | P2 competitor price feeder | crawler scheduling / price normalization / retryable candidate recovery / cache strategy | -| 1535 | `services/competitor_intel_repository.py` | P2 competitor intel repository | review queue query / cache shaping / formatting helpers | -| 805 | `routes/bot_api_routes.py` | P2 Bot API Blueprint | route glue / bot action service | -| 1319 | `routes/market_intel_review_report_routes.py` | P2 market intel review report Blueprint | review report route glue / export payload / phase handoff orchestration | -| 917 | `routes/market_intel_routes.py` | P2 market intel Blueprint | page route / API route glue / MCP gate route registration helper | -| 1965 | `services/market_intel/deployment_readiness.py` | P2 market intel deployment readiness | preflight gates / readiness payload / route contract helpers | +| 864 | `services/ppt_vision_service.py` | P2 PPT vision QA service | runtime state / queue status / model probe / audit execution 分離 | | 846 | `services/market_intel/candidate_queue_review_ai_summary_persistence_telegram_dispatch_report_catalog_record_run_receipt.py` | P2 market intel review receipt pipeline | AI summary / persistence / Telegram dispatch / report catalog run receipt orchestration | +| 826 | `routes/market_intel_mcp_run_routes.py` | P2 market intel MCP run route extension | route glue / canonical alias decorators / MCP run preview handoff | +| 861 | `routes/market_intel_review_post_routes.py` | P2 market intel review post route extension | review post route glue / canonical alias decorators / AI summary persistence preview handoff | +| 813 | `services/momo_crawler.py` | P2 MOMO crawler | targeted search / browser fallback / parsing 分離 | ## 市場情報開發前置禁區 diff --git a/docs/memory/pchome_mapping_backlog_20260628.md b/docs/memory/pchome_mapping_backlog_20260628.md new file mode 100644 index 0000000..d5e04b6 --- /dev/null +++ b/docs/memory/pchome_mapping_backlog_20260628.md @@ -0,0 +1,117 @@ +# PChome Mapping Backlog Read-Only Report (2026-06-28) + +正式環境目前以 `/api/ai/pchome-growth/opportunities` 作為 PChome 業績成長作戰清單來源。接續商品對應率優化時，先用只讀 reporter 取得 mapping backlog，不直接碰正式 DB。 + +## 產品優先順序 + +1. 先守住正式版本真相：production `/health` 必須是最新版本來源，本機只當 source candidate。 +2. AI 自動化營收閉環優先：正式 `needs_mapping_count=16` 先處理 direct mapping，再處理 review candidate，不退回人工逐筆找商品。 +3. AI Agent 決策必須 machine-verifiable：每個候選、證據補齊、單位價判斷、待確認項都要有 confidence、guardrails、trace、資料品質與可回滾下一步。 +4. 主流專業產品網站化必須同步推進：首頁 / Dashboard 要直接呈現缺口、價值、下一步與可執行 package，不得只顯示工程 API 或 raw endpoint。 +5. 外部來源治理要跟上：MOMO 是已接入來源，其他主流平台至少要有來源契約或待接入狀態；所有資料要正規化成 offer evidence。 +6. 再做小批次 no-write operator package / persistence dry-run / verifier，確認可回滾、可重跑、可讀回。 +7. 最後才推 DB apply authorization / signing / closeout 鏈；這些是底層 apply 安全工程，不得搶在產品缺口、AI 自動化與專業網站化前面。 + +## 技術入口順序（按需使用，不代表產品優先順序） + +1. 先跑 `python scripts/ops/check_production_version_truth.py`，確認 production `/health` 仍是版本真相。 +2. 再跑 `python scripts/ops/report_pchome_mapping_backlog.py --json`。 +3. 或使用登入後只讀 API `/api/ai/pchome-growth/mapping-backlog` 讀同一套 service summary。 +4. 需要小批次操作前，先讀只讀 API `/api/ai/pchome-growth/mapping-backlog/operator-preview`。 +5. 需要 P1 direct mapping 自動搜尋候選包時，先讀 `/api/ai/pchome-growth/mapping-backlog/direct-mapping-auto-search-package`；預設只產生搜尋詞、身份錨點與接受門檻，不搜尋、不寫 DB。 +6. 需要 P2 候選機器決策包時，讀 `/api/ai/pchome-growth/mapping-backlog/direct-mapping-candidate-decision-package`；預設不搜尋、不寫 DB，若有候選則輸出 decision envelope、confidence、guardrails、failure reasons 與 exception-only machine review 分流。 +7. 需要看整體 AI 自動化進度時，讀 `/api/ai/pchome-growth/ai-automation-readiness`；它聚合缺口偵測、同款搜尋包、候選決策包、證據收據與受控落地，並回傳 `primary_human_gate_count=0`、`automation_policy.primary_flow=ai_controlled`。 +8. 需要補商品證據前，先讀只讀 API `/api/ai/pchome-growth/mapping-backlog/evidence-enrichment-preview`。 +9. 需要檢查 image / availability / price 要從哪裡補時，先讀只讀 API `/api/ai/pchome-growth/mapping-backlog/evidence-source-preview`。 +10. 需要 controlled GET smoke 時，先讀 `/api/ai/pchome-growth/mapping-backlog/evidence-fetch-gate`；預設不抓外站，只有 `execute_fetch=1` 才執行小批次 GET。 +11. 需要檢查 fetch receipt 能補掉哪些 evidence 缺口時，先讀 `/api/ai/pchome-growth/mapping-backlog/evidence-merge-preview`；預設不抓外站，只有 `execute_fetch=1` 才走 controlled fetch gate。 +12. 需要自動化收據時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-receipt-gate`；預設不抓外站、不 persistence，只有 `execute_fetch=1` 才走 controlled fetch gate 產生 ready receipt。 +13. 需要自動化 persistence dry-run contract 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-persistence-gate`；只接受 ready receipts，輸出 idempotency key、payload hash、schema migration contract、transaction preview、rollback plan 與 post-write verifier，但不寫 DB。 +14. 需要 schema migration / future apply verifier contract 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-schema-migration-preview`；只輸出 DDL preview、rollback preview、prewrite snapshot contract 與 post-write verifier，不執行 migration、不寫 DB。 +15. 需要 migration file / future apply endpoint contract 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-migration-file-preview`；只輸出 `migrations/045_pchome_auto_policy_evidence_receipts.sql` 內容預覽、hash、request contract、abort conditions 與 rollback contract，不寫檔、不執行 endpoint。 +16. 需要真正 migration file 生成前的 readiness closeout 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-apply-readiness-closeout`；只輸出 readiness checks、future-apply blockers 與 closeout decision，不寫檔、不執行 endpoint、不寫 DB。 +17. 需要 migration file generation request contract 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-migration-file-generation-request`；只輸出 request id、target file、expected hash、required artifacts 與 file generation steps，不寫檔、不執行 endpoint、不寫 DB。 +18. 需要 migration apply gate preview 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-migration-apply-gate-preview`；只讀本地 `045` migration 檔、hash、apply endpoint contract、prewrite snapshot requirement 與 post-apply verifier，不執行 SQL、不寫 DB。 +19. 需要 explicit DB apply request gate preview 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-request-gate-preview`；只輸出 command preview、required artifacts、abort conditions、runtime readback 與 rollback gate preview，不讀 secret、不執行 SQL、不寫 DB。 +20. 需要 DB apply execution preflight / snapshot artifact preview 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-execution-preflight`；只輸出 prewrite snapshot plan、post-apply readback plan、rollback artifact plan 與 abort conditions，不讀 secret、不執行 SQL、不寫 DB。 +21. 需要 fresh production apply authorization package 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-package`；只輸出 authorization checks、freshness requirements、machine apply manifest 與 verifier bundle，不讀 secret、不執行 SQL、不寫 DB。 +22. 需要 fresh-run verifier artifact generator preview 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-verifier-artifact-preview`；只輸出 prewrite snapshot / post-apply readback / rollback artifact schema、generation plan 與 verifier manifest，不寫檔、不讀 secret、不執行 SQL、不寫 DB。 +23. 需要 final explicit DB apply handoff package 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-final-handoff-package`；只輸出 final handoff sections、future runbook、command previews、abort gates 與 source proof manifest，不寫檔、不讀 secret、不執行 SQL、不寫 DB。 +24. 需要 controlled apply dry-run shell preview 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-controlled-dry-run-shell-preview`；只輸出 shell phases、script preview、check-mode contract 與 rollback hooks，不寫 script、不讀 secret、不執行 shell、不執行 SQL、不寫 DB。 +25. 需要 controlled dry-run shell closeout / explicit apply authorization boundary 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-controlled-dry-run-shell-closeout`；只輸出 closeout checks、future apply boundaries 與 explicit authorization boundary，不寫 script、不讀 secret、不執行 shell、不執行 SQL、不寫 DB。 +26. 需要 explicit DB apply authorization request intake 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-request-intake`；只輸出 authorization request intake、request payload schema、acceptance gates 與 rejection reasons，不讀 secret、不執行 shell、不執行 SQL、不寫 DB，且不簽發 database apply authorization。 +27. 需要 authorization request intake closeout / final exact request package 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-request-closeout`；只輸出 final exact request package、machine request manifest 與 closeout checks，不讀 secret、不執行 shell、不執行 SQL、不寫 DB，且不簽發 database apply authorization。 +28. 需要 future DB apply authorization lane guard 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`；只輸出 future authorization lane guard、lane transfer contract 與 lane entry requirements，不讀 secret、不執行 shell、不執行 SQL、不寫 DB，且不簽發 database apply authorization。 +29. 需要 future DB apply authorization decision preflight 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`；只輸出 future authorization decision preflight、decision input requirements、decision rejection policy 與 preflight checks，不讀 secret、不執行 shell、不執行 SQL、不寫 DB，且不簽發 database apply authorization。 +30. 需要 future DB apply authorization decision closeout 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`；只輸出 future authorization decision closeout、decision package、decision closeout contract 與 closeout checks，不讀 secret、不執行 shell、不執行 SQL、不寫 DB，且不簽發 database apply authorization。 +31. 需要 future DB apply authorization issuer gate / final nonsecret authorization envelope 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`；只輸出 future authorization issuer gate、final nonsecret authorization envelope、issuer gate contract 與 issuer gate checks，不讀 secret、不執行 shell、不執行 SQL、不寫 DB，且不簽發 database apply authorization。 +32. 需要 future DB apply authorization signing-decision preflight 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`；只輸出 future authorization signing decision preflight、signing decision input requirements、rejection policy 與 preflight checks，不讀 secret、不執行 shell、不執行 SQL、不寫 DB，且不簽發 database apply authorization。 +33. 需要 future DB apply authorization signing-decision closeout / unsigned signing decision package 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout`；只輸出 future authorization signing decision closeout、unsigned signing decision package、closeout contract 與 closeout checks，不讀 secret、不執行 shell、不執行 SQL、不寫 DB，且不簽發 database apply authorization。 +34. 需要 future DB apply authorization signing-issuer guard / signable request boundary 時，讀 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；只輸出 future authorization signing issuer guard、signable request boundary、signing issuer guard contract 與 guard checks，不讀 secret、不執行 shell、不執行 SQL、不寫 DB，且不簽發 database apply authorization。 +35. 若 `result=NEEDS_MAPPING`，先處理 `direct_mapping_count`，再處理 `review_candidate_count`。 + +## 目前讀到的正式缺口 + +- production `/health`: `V10.725` +- PChome growth candidate_count: `20` +- mapped_count: `4` +- mapping_rate: `20.0%` +- needs_mapping_count: `16` +- action_counts: `先補商品對應=14`、`確認候選=2`、`放大價格優勢=4` + +## 邊界 + +- reporter 只做 GET readback。 +- service truth source: `services/pchome_mapping_backlog_service.py` +- operator preview 只產生 run package 與 write gate，不執行搜尋或寫入。 +- direct mapping auto-search package 已接上 `/api/ai/pchome-growth/mapping-backlog/direct-mapping-auto-search-package`；預設只產生搜尋詞、身份錨點與接受門檻，`execute_search=1` 才執行 read-only candidate search，仍不寫 DB。 +- direct mapping candidate decision package 已接上 `/api/ai/pchome-growth/mapping-backlog/direct-mapping-candidate-decision-package`；把候選轉成 P2 decision envelope，分流到 no-write receipt 或 exception-only machine review，仍不寫 DB、不 persistence。 +- AI automation readiness 已接上 `/api/ai/pchome-growth/ai-automation-readiness` 與 Dashboard status strip；它用單一摘要顯示缺口偵測、同款搜尋包、候選決策包、證據收據與受控落地，且 `primary_human_gate_count=0`、`automation_policy.primary_flow=ai_controlled`，例外也走 AI machine-verifiable auto-resolution。 +- Dashboard command center 已加入 AI 自動化作戰流水線，將 P1 搜尋包、P2 決策包、證據收據與受控落地轉成營運語言與數字，不外露 raw endpoint。 +- 外部來源治理已改成由 source readiness 產生完整 source_scope；除 MOMO 已接入外，Shopee、Lazada、Amazon、Google Merchant / Shopping、TikTok Shop、LINE 購物、Rakuten、Yahoo、露天、品牌官網 / Shopify、Meta Commerce、Coupang 都列為待接入且不進告警，並提供 offer evidence 欄位契約。 +- operator preview 參考外部主流規範：Google Merchant Center product data、Google Product / Merchant structured data、Baymard ecommerce product/search UX。 +- 每個 target 都應帶 `evidence_completeness`，至少列出 product URL、present/missing fields、blocking missing fields、auto_accept_ready 與 human_review_required。 +- deterministic `unit_package_basis` parser preview 已接入 evidence completeness，可從商品標題解析 `30ml`、`40ml x2`、`3.2g 多款任選`、`42張` 等單位/入數訊號；若遇到任選、色號、贈品、即期等風險訊號，仍要求人工覆核。 +- evidence enrichment preview 只產生 image / availability / unit basis 的補證據任務與來源建議，不抓外站、不寫 DB；unit basis 可由本地 parser 補足時，不再列為缺口。 +- evidence source preview 已接上 `/api/ai/pchome-growth/mapping-backlog/evidence-source-preview`，只列出 image / availability 的 future GET fetch gate、payload keys checked，以及 price payload mapping probe；preview 本身不抓外站、不寫 DB、不呼叫 LLM。 +- check-mode product-page evidence parser 已完成，可用 HTML fixture 解析 JSON-LD `Product.image`、`Offer.availability`、`og:image` 與 `product:availability` fallback；parser 本身不抓外站、不寫 DB、不呼叫 LLM。 +- controlled read-only fetch gate 已接上 `/api/ai/pchome-growth/mapping-backlog/evidence-fetch-gate`；預設 `execute_fetch=false` 只產生 planned receipts，`execute_fetch=1` 時才對 allowlist domain `24h.pchome.com.tw` 執行最多 12 筆 GET，含 timeout、512KB HTML cap、no-write receipt。 +- production-derived 1 筆 GET smoke：`DDBH3K-A900K0KWV-002` 回 `HTTP 200`、`content_bytes=215631`，解析到 image URL 與 `Offer.availability=InStock`；receipt 仍是 `writes_database=false`、`dispatches_telegram=false`、`llm_calls=false`。 +- evidence merge preview 已接上 `/api/ai/pchome-growth/mapping-backlog/evidence-merge-preview`；production-derived 1 筆 smoke 讓 `DDBH3K-A900K0KWV-002` 的 `image/availability` 從 missing fields 移除，`merge_ready_count=1`、`writes_database_count=0`。 +- automation-first merge policy 已接上：review candidate 的 `pchome_price` 會自動納入 price evidence；production-derived batch=12 已達 `AUTO_ACCEPT_EVIDENCE_MERGE=10/10`、`manual_review_required_count=0`、`remaining_blocker_count=0`、`writes_database_count=0`。 +- auto-policy receipt gate 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-receipt-gate`；production-derived batch=12 已達 `READY_FOR_AUTO_PERSISTENCE=10/10`、`persists_receipt_count=0`、`writes_database_count=0`、`manual_review_required_count=0`。 +- auto-policy persistence gate 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-persistence-gate`；只接受 ready receipts，輸出 idempotency key、payload hash、future target table、transaction preview、schema contract、rollback plan 與 post-write verifier；production-derived batch=12 smoke 已達 `PERSISTENCE_DRY_RUN_READY`、`dry_run_ready_count=10/10`、`manual_review_required_count=0`、`writes_database_count=0`、`persists_receipt_count=0`，目前仍是 no-write / dry-run-only。 +- auto-policy schema migration preview 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-schema-migration-preview`；輸出 `external_offer_evidence_receipts` DDL preview、rollback preview、prewrite snapshot contract、future apply verifier 與 exception-only failure routing；production-derived batch=12 smoke 已達 `SCHEMA_MIGRATION_PREVIEW_READY`、`dry_run_ready_count=10/10`、`schema_statement_count=13`、`future_verifier_count=5`、`current_preview_apply_allowed=false`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 no-write / no-SQL-execute preview。 +- auto-policy migration file preview 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-migration-file-preview`；輸出 `migrations/045_pchome_auto_policy_evidence_receipts.sql` forward SQL preview、hash、rollback preview、future apply endpoint request contract、preflight sequence、abort conditions 與 rollback contract；production-derived batch=12 smoke 已達 `MIGRATION_FILE_PREVIEW_READY`、`dry_run_ready_count=10/10`、`migration_file_line_count=31`、`apply_endpoint_contract_ready_count=1`、`forbidden_forward_tokens_absent=true`、`writes_file_count=0`、`executes_endpoint_count=0`、`writes_database_count=0`，目前仍是 no-file-write / no-endpoint-execute / no-SQL-execute preview。 +- auto-policy apply-readiness closeout 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-apply-readiness-closeout`；輸出 readiness checks、future-apply blockers、migration file hash summary 與 apply endpoint summary；production-derived batch=12 smoke 已達 `APPLY_READINESS_CLOSEOUT_READY`、`readiness_pass_count=9/9`、`current_preview_ready_count=1`、`ready_for_migration_file_generation_request=true`、`ready_for_database_apply=false`、`future_apply_blocker_count=4`、`writes_file_count=0`、`executes_endpoint_count=0`、`writes_database_count=0`；closeout ready 只代表可提出 migration file generation request，不代表 DB apply。 +- auto-policy migration file generation request 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-migration-file-generation-request`；輸出 request id、target file、expected SHA256、required artifacts 與 file generation steps；`migrations/045_pchome_auto_policy_evidence_receipts.sql` 已在本地 repo 生成，hash 與 request expected SHA256 對齊；production-derived batch=12 smoke 已達 `FILE_GENERATION_REQUEST_READY`、`request_ready_count=1`、`dry_run_ready_count=10/10`、`required_artifact_count=4`、`file_generation_step_count=3`、`ready_to_generate_file=true`、`ready_for_database_apply=false`、`generated_file_exists=true`、`generated_file_hash_matches_request=true`、`writes_file_count=0`、`executes_endpoint_count=0`、`writes_database_count=0`，request API 本身仍是 no-file-write / no-endpoint-execute / no-DB-write。 +- auto-policy migration apply gate preview 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-migration-apply-gate-preview`；讀本地 `045` migration 檔並比對 request hash、additive-only、future apply endpoint contract、prewrite snapshot requirement 與 post-apply verifier requirement；production-derived batch=12 smoke 已達 `MIGRATION_APPLY_GATE_PREVIEW_READY`、`apply_gate_pass_count=9/9`、`apply_preview_ready_count=1`、`generated_file_hash_matches_count=1`、`ready_for_explicit_db_apply_request=true`、`ready_for_database_apply_now=false`、`future_apply_blocker_count=3`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 no-SQL-execute / no-DB-write preview。 +- auto-policy explicit DB apply request gate preview 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-request-gate-preview`；輸出 DB apply request id、future command preview、required artifacts、apply sequence、abort conditions、runtime readback 與 rollback gate preview；production-derived batch=12 smoke 已達 `DB_APPLY_REQUEST_GATE_READY`、`request_ready_count=1`、`required_artifact_count=5`、`apply_sequence_step_count=5`、`abort_condition_count=6`、`generated_file_hash_matches_count=1`、`ready_for_explicit_db_apply_request=true`、`ready_for_database_apply_now=false`、`reads_secret_in_preview=false`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 no-secret-read / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply execution preflight 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-execution-preflight`；輸出 prewrite snapshot plan、post-apply readback plan、rollback artifact plan、required artifacts 與 abort conditions；production-derived batch=12 smoke 已達 `DB_APPLY_EXECUTION_PREFLIGHT_READY`、`preflight_ready_count=1`、`request_ready_count=1`、`required_artifact_count=6`、`snapshot_plan_count=5`、`readback_plan_count=6`、`rollback_artifact_count=1`、`generated_file_hash_matches_count=1`、`ready_for_preflight_artifact_generation=true`、`ready_for_database_apply_now=false`、`reads_secret_in_preview=false`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 no-secret-read / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply authorization package 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-package`；輸出 authorization checks、freshness requirements、machine apply manifest 與 verifier bundle；production-derived batch=12 smoke 已達 `DB_APPLY_AUTHORIZATION_PACKAGE_READY`、`authorization_check_count=11`、`authorization_pass_count=11`、`authorization_waiting_count=0`、`authorization_package_ready_count=1`、`freshness_requirement_count=5`、`manifest_step_count=6`、`verifier_bundle_count=3`、`ready_for_explicit_apply_authorization_request=true`、`ready_for_database_apply_now=false`、`reads_secret_in_preview=false`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 no-secret-read / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply verifier artifact preview 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-verifier-artifact-preview`；輸出 prewrite snapshot / post-apply readback / rollback artifact schema、generation plan 與 verifier manifest；production-derived batch=12 smoke 已達 `DB_APPLY_VERIFIER_ARTIFACT_PREVIEW_READY`、`artifact_preview_ready_count=1`、`authorization_package_ready_count=1`、`artifact_schema_count=3`、`artifact_generation_step_count=5`、`verifier_check_count=15`、`writes_artifact_count=0`、`ready_for_future_artifact_generation=true`、`ready_to_write_artifacts_now=false`、`ready_for_database_apply_now=false`、`reads_secret_in_preview=false`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 no-file-write / no-secret-read / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply final handoff package 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-final-handoff-package`；輸出 final handoff sections、future runbook、command previews、abort gates 與 source proof manifest；production-derived batch=12 smoke 已達 `DB_APPLY_FINAL_HANDOFF_PACKAGE_READY`、`final_handoff_ready_count=1`、`artifact_preview_ready_count=1`、`handoff_section_count=6`、`final_runbook_step_count=7`、`command_preview_count=3`、`abort_gate_count=10`、`source_endpoint_count=4`、`ready_for_explicit_db_apply_handoff=true`、`ready_for_database_apply_now=false`、`writes_artifact_count=0`、`reads_secret_in_preview=false`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 no-file-write / no-secret-read / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply controlled dry-run shell preview 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-controlled-dry-run-shell-preview`；輸出 shell phases、script preview、check-mode contract 與 rollback hooks；production-derived batch=12 smoke 已達 `DB_APPLY_CONTROLLED_DRY_RUN_SHELL_PREVIEW_READY`、`dry_run_shell_preview_ready_count=1`、`final_handoff_ready_count=1`、`shell_phase_count=9`、`shell_script_line_count=10`、`check_mode_required_check_count=6`、`rollback_hook_count=3`、`writes_script_count=0`、`executes_script_count=0`、`ready_for_future_shell_script_generation=true`、`ready_to_write_script_now=false`、`ready_to_execute_shell_now=false`、`ready_for_database_apply_now=false`、`reads_secret_in_preview=false`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 no-script-write / no-secret-read / no-shell-execute / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply controlled dry-run shell closeout 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-controlled-dry-run-shell-closeout`；輸出 closeout checks、future apply boundaries 與 explicit authorization boundary；production-derived batch=12 smoke 已達 `DB_APPLY_CONTROLLED_DRY_RUN_SHELL_CLOSEOUT_READY`、`closeout_ready_count=1`、`closeout_check_count=13`、`closeout_pass_count=13`、`closeout_waiting_count=0`、`dry_run_shell_preview_ready_count=1`、`future_apply_boundary_count=6`、`ready_for_explicit_apply_authorization_boundary=true`、`ready_for_database_apply_now=false`、`writes_script_count=0`、`executes_script_count=0`、`reads_secret_in_preview=false`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 no-script-write / no-secret-read / no-shell-execute / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply authorization request intake 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-request-intake`；輸出 authorization request intake、request payload schema、acceptance gates、required request evidence 與 rejection reasons；production-derived batch=12 smoke 已達 `DB_APPLY_AUTHORIZATION_REQUEST_INTAKE_READY`、`authorization_request_intake_ready_count=1`、`required_request_evidence_count=7`、`request_payload_required_field_count=10`、`authorization_acceptance_gate_count=11`、`authorization_acceptance_pass_count=11`、`authorization_acceptance_waiting_count=0`、`rejection_reason_count=10`、`closeout_ready_count=1`、`future_apply_boundary_count=6`、`accepts_authorization_request=true`、`issues_database_apply_authorization=false`、`ready_for_database_apply_now=false`、`reads_secret_in_preview=false`、`executes_script_count=0`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 request-intake-only / no-secret-read / no-shell-execute / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply authorization request closeout 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-request-closeout`；輸出 final exact request package、machine request manifest 與 closeout checks；production-derived batch=12 smoke 已達 `DB_APPLY_AUTHORIZATION_REQUEST_CLOSEOUT_READY`、`authorization_request_closeout_ready_count=1`、`closeout_check_count=12`、`closeout_pass_count=12`、`closeout_waiting_count=0`、`authorization_request_intake_ready_count=1`、`exact_request_payload_field_count=10`、`machine_request_manifest_step_count=6`、`required_request_evidence_count=7`、`authorization_acceptance_gate_count=11`、`rejection_reason_count=10`、`ready_for_exact_authorization_request_package=true`、`issues_database_apply_authorization=false`、`ready_for_database_apply_now=false`、`reads_secret_in_preview=false`、`executes_script_count=0`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 final-request-package-only / no-secret-read / no-shell-execute / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply authorization lane guard 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`；輸出 future authorization lane guard、lane transfer contract、lane entry requirements 與 lane guard checks；production-derived batch=12 smoke 已達 `DB_APPLY_AUTHORIZATION_LANE_GUARD_READY`、`authorization_lane_guard_ready_count=1`、`lane_guard_check_count=12`、`lane_guard_pass_count=12`、`lane_guard_waiting_count=0`、`authorization_request_closeout_ready_count=1`、`exact_request_payload_field_count=10`、`machine_request_manifest_step_count=6`、`lane_entry_requirement_count=6`、`required_request_evidence_count=7`、`authorization_acceptance_gate_count=11`、`rejection_reason_count=10`、`ready_for_future_authorization_lane_entry=true`、`issues_database_apply_authorization=false`、`ready_for_database_apply_now=false`、`reads_secret_in_preview=false`、`executes_script_count=0`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 lane-guard-only / no-secret-read / no-shell-execute / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply authorization decision preflight 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`；輸出 future authorization decision preflight、decision input requirements、decision rejection policy 與 preflight checks；production-derived batch=12 smoke 已達 `DB_APPLY_AUTHORIZATION_DECISION_PREFLIGHT_READY`、`authorization_decision_preflight_ready_count=1`、`decision_preflight_check_count=12`、`decision_preflight_pass_count=12`、`decision_preflight_waiting_count=0`、`authorization_lane_guard_ready_count=1`、`decision_input_requirement_count=8`、`decision_rejection_reason_count=10`、`lane_entry_requirement_count=6`、`exact_request_payload_field_count=10`、`machine_request_manifest_step_count=6`、`allows_authorization_decision_in_future_lane=true`、`issues_database_apply_authorization=false`、`ready_for_database_apply_now=false`、`reads_secret_in_preview=false`、`executes_script_count=0`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 decision-preflight-only / no-secret-read / no-shell-execute / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply authorization decision closeout 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`；輸出 future authorization decision closeout、future authorization decision package、decision closeout contract 與 closeout checks；production-derived batch=12 smoke 已達 `DB_APPLY_AUTHORIZATION_DECISION_CLOSEOUT_READY`、`authorization_decision_closeout_ready_count=1`、`decision_closeout_check_count=12`、`decision_closeout_pass_count=12`、`decision_closeout_waiting_count=0`、`authorization_decision_preflight_ready_count=1`、`decision_input_requirement_count=8`、`decision_rejection_reason_count=10`、`post_apply_verifier_required_count=1`、`same_run_truth_required_count=1`、`permits_future_authorization_decision_lane=true`、`issues_database_apply_authorization=false`、`ready_for_database_apply_now=false`、`reads_secret_in_preview=false`、`executes_script_count=0`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 decision-closeout-only / no-secret-read / no-shell-execute / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply authorization issuer gate 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`；輸出 future authorization issuer gate、final nonsecret authorization envelope、issuer gate contract 與 issuer gate checks；production-derived batch=12 smoke 已達 `DB_APPLY_AUTHORIZATION_ISSUER_GATE_READY`、`authorization_issuer_gate_ready_count=1`、`issuer_gate_check_count=12`、`issuer_gate_pass_count=12`、`issuer_gate_waiting_count=0`、`authorization_decision_closeout_ready_count=1`、`decision_closeout_check_count=12`、`required_issuer_evidence_count=9`、`nonsecret_authorization_claim_count=8`、`post_apply_verifier_required_count=1`、`same_run_truth_required_count=1`、`permits_future_authorization_issuer_lane=true`、`issues_database_apply_authorization=false`、`signs_database_apply_authorization=false`、`ready_for_database_apply_now=false`、`secret_material_included=false`、`reads_secret_in_preview=false`、`executes_script_count=0`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 issuer-gate-only / nonsecret-envelope-only / no-secret-read / no-shell-execute / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply authorization signing-decision preflight 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`；輸出 future authorization signing decision preflight、signing decision preflight envelope、signing decision input requirements、rejection policy 與 preflight checks；production-derived batch=12 smoke 已達 `DB_APPLY_AUTHORIZATION_SIGNING_DECISION_PREFLIGHT_READY`、`authorization_signing_decision_preflight_ready_count=1`、`signing_decision_preflight_check_count=12`、`signing_decision_preflight_pass_count=12`、`signing_decision_preflight_waiting_count=0`、`authorization_issuer_gate_ready_count=1`、`issuer_gate_check_count=12`、`required_issuer_evidence_count=9`、`nonsecret_authorization_claim_count=8`、`signing_decision_input_requirement_count=10`、`signing_decision_rejection_reason_count=11`、`allows_future_authorization_signing_decision_lane=true`、`issues_database_apply_authorization=false`、`signs_database_apply_authorization=false`、`ready_for_database_apply_now=false`、`secret_material_required_in_preview=false`、`reads_secret_in_preview=false`、`executes_script_count=0`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 signing-decision-preflight-only / no-secret-read / no-signing / no-shell-execute / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply authorization signing-decision closeout 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout`；輸出 future authorization signing decision closeout、unsigned signing decision package、signing decision closeout contract 與 closeout checks；production-derived batch=12 smoke 已達 `DB_APPLY_AUTHORIZATION_SIGNING_DECISION_CLOSEOUT_READY`、`authorization_signing_decision_closeout_ready_count=1`、`signing_decision_closeout_check_count=12`、`signing_decision_closeout_pass_count=12`、`signing_decision_closeout_waiting_count=0`、`authorization_signing_decision_preflight_ready_count=1`、`signing_decision_input_requirement_count=10`、`signing_decision_rejection_reason_count=11`、`required_issuer_evidence_count=9`、`nonsecret_authorization_claim_count=8`、`permits_future_unsigned_signing_decision_package_lane=true`、`issues_database_apply_authorization=false`、`signs_database_apply_authorization=false`、`ready_for_database_apply_now=false`、`secret_material_included=false`、`secret_material_required_in_preview=false`、`reads_secret_in_preview=false`、`executes_script_count=0`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 signing-decision-closeout-only / unsigned-package-only / no-secret-read / no-signing / no-shell-execute / no-SQL-execute / no-DB-write preview。 +- auto-policy DB apply authorization signing-issuer guard 已接上 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；輸出 future authorization signing issuer guard、signable request boundary、signing issuer guard contract 與 guard checks；production-derived batch=12 smoke 已達 `DB_APPLY_AUTHORIZATION_SIGNING_ISSUER_GUARD_READY`、`authorization_signing_issuer_guard_ready_count=1`、`signing_issuer_guard_check_count=12`、`signing_issuer_guard_pass_count=12`、`signing_issuer_guard_waiting_count=0`、`authorization_signing_decision_closeout_ready_count=1`、`signing_decision_closeout_check_count=12`、`signing_decision_input_requirement_count=10`、`signing_decision_rejection_reason_count=11`、`required_issuer_evidence_count=9`、`nonsecret_authorization_claim_count=8`、`permits_future_authorization_signing_issuer_lane=true`、`issues_database_apply_authorization=false`、`signs_database_apply_authorization=false`、`ready_for_database_apply_now=false`、`secret_material_included=false`、`secret_material_required_in_preview=false`、`reads_secret_in_preview=false`、`executes_script_count=0`、`executes_sql_count=0`、`executes_migration_count=0`、`writes_database_count=0`，目前仍是 signing-issuer-guard-only / signable-boundary-only / no-secret-read / no-signing / no-shell-execute / no-SQL-execute / no-DB-write preview。 +- Dashboard no-write UI 已顯示 `auto_receipt_count` 與 `manual_review_required_count`，首頁不觸發外站 GET、不 persistence。 +- production-derived evidence/source preview 目前 `task_count=6`、`tasks_with_blockers=6`、`missing_field_counts=image=6, availability=6, price=1, unit_price_or_package_basis=0`、`fetch_gate_candidate_count=6`；price 缺口樣本是 review candidate `DDABN0-A900IH795-000`。 +- AI automation 只先產生 Ollama-first read-only assist plan；preview 不呼叫 LLM、不允許 Gemini、不寫 DB。 +- 不執行 backfill POST。 +- 不寫 DB、不掛 scheduler、不派 Telegram。 +- 若版本 guard blocked，不應宣稱本地或 Gitea 是最新正式版本。 + +## 下一步 + +1. 進入 future DB apply authorization signing-issuer closeout / final signable request package：只收斂 signing issuer guard 的 signable boundary、source chain、same-run production truth、post-apply verifier、secret rejection 與 no-signing/no-write guarantees；仍不讀 secret、不在此步簽發 database apply authorization，也不直接 DB apply。 +2. 以 Google Product / Merchant listing structured data 的 `Product.image`、`Offer.availability` 為解析目標。 +3. 順手追 review candidate price payload 邊界，確認缺口是來源 payload 欄位缺失還是 summary 映射缺失。 +4. image / availability / price payload 邊界都補齊並通過 production readback 後，才進入 operator write approval / backfill POST 小批次。 diff --git a/docs/memory/production_version_truth_20260627.md b/docs/memory/production_version_truth_20260627.md new file mode 100644 index 0000000..c2b0058 --- /dev/null +++ b/docs/memory/production_version_truth_20260627.md @@ -0,0 +1,29 @@ +# Production Version Truth 2026-06-27 + +## Rule + +正式環境 `/health` 是 EwoooC 的最新版本真相。 + +Local workspace、Gitea `main`、branch、handoff 或文件只能作為 source candidate；除非 production readback 回同一個版本，否則不能宣稱已上線或最新。 + +## Current Protected Baseline + +- Production URL: `https://mo.wooo.work/health` +- Production readback at setup time: `status=healthy`, `database=postgresql`, `version=V10.725` +- Clean local workspace: `/Users/ogt/codex-workspaces/ewoooc-dev` +- Local branch at setup time: `main` +- Local / origin HEAD at setup time: `f3e412c` +- Clean local `config.py`: `SYSTEM_VERSION = "V10.725"` +- Old iCloud checkout had a local-only `config.py` drift to `V10.726`; that drift is not production truth. + +## Required Check + +Before claiming a version, run: + +```bash +python scripts/ops/check_production_version_truth.py +``` + +If the script reports `BLOCKED`, treat local work as a candidate and do not deploy, report completion, or write handoff text that says the local version is latest. + +For an explicit release-prep branch only, `--allow-local-version-drift` may be used to inspect a local `config.py` version bump, but production `/health` still remains the latest runtime truth until deploy readback matches it. diff --git a/scripts/ops/report_pchome_mapping_backlog.py b/scripts/ops/report_pchome_mapping_backlog.py new file mode 100755 index 0000000..ef39f49 --- /dev/null +++ b/scripts/ops/report_pchome_mapping_backlog.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Read-only report for the production PChome growth mapping backlog.""" + +from __future__ import annotations + +import argparse +import json +import urllib.error +import urllib.parse +import urllib.request +from typing import Any + +from scripts.ops import check_production_version_truth as version_guard +from services.pchome_mapping_backlog_service import summarize_pchome_mapping_backlog + + +DEFAULT_API_URL = "https://mo.wooo.work/api/ai/pchome-growth/opportunities" +DEFAULT_LIMIT = 20 + + +def with_limit(url: str, limit: int) -> str: + parsed = urllib.parse.urlparse(url) + query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True) + query["limit"] = [str(max(5, min(int(limit or DEFAULT_LIMIT), 50)))] + return urllib.parse.urlunparse(parsed._replace(query=urllib.parse.urlencode(query, doseq=True))) + + +def fetch_json(url: str, timeout: float) -> dict[str, Any]: + with urllib.request.urlopen(url, timeout=timeout) as response: + payload = response.read().decode("utf-8") + data = json.loads(payload) + if not isinstance(data, dict): + raise ValueError("API payload must be a JSON object") + return data + + +def summarize_payload(payload: dict[str, Any]) -> dict[str, Any]: + return summarize_pchome_mapping_backlog(payload) + + +def build_report( + api_url: str, + limit: int, + timeout: float, + health_url: str, + skip_version_truth: bool, +) -> dict[str, Any]: + version_report = None + version_errors: list[str] = [] + if not skip_version_truth: + version_report = version_guard.build_report(health_url, timeout) + version_ok, version_errors = version_guard.evaluate(version_report, allow_local_version_drift=False) + if not version_ok: + return { + "policy": "read_only_production_pchome_mapping_backlog", + "result": "BLOCKED", + "api_url": with_limit(api_url, limit), + "version_truth": version_report, + "errors": version_errors, + } + + final_url = with_limit(api_url, limit) + payload = fetch_json(final_url, timeout) + summary = summarize_payload(payload) + return { + **summary, + "policy": "read_only_production_pchome_mapping_backlog", + "api_url": final_url, + "version_truth": version_report, + "errors": version_errors, + } + + +def format_text(report: dict[str, Any]) -> str: + stats = report.get("stats") or {} + backlog = report.get("backlog") or {} + version = ((report.get("version_truth") or {}).get("production") or {}).get("version") + lines = [ + "pchome_mapping_backlog:", + f"- policy: {report.get('policy')}", + f"- result: {report.get('result')}", + f"- production_version: {version or 'not_checked'}", + f"- api_url: {report.get('api_url')}", + f"- generated_at: {report.get('generated_at')}", + f"- cache_state: {report.get('cache_state') or 'unknown'}", + f"- candidate_count: {stats.get('candidate_count')}", + f"- mapped_count: {stats.get('mapped_count')}", + f"- mapping_rate: {stats.get('mapping_rate')}%", + f"- needs_mapping_count: {stats.get('needs_mapping_count')}", + f"- review_candidate_count: {stats.get('review_candidate_count')}", + f"- overall_sales_7d: {stats.get('overall_sales_7d')}", + f"- action_counts: {json.dumps(stats.get('action_counts') or {}, ensure_ascii=False, sort_keys=True)}", + f"- direct_mapping_count: {backlog.get('direct_mapping_count')}", + f"- mapped_opportunity_count: {backlog.get('mapped_opportunity_count')}", + ] + for error in report.get("errors") or []: + lines.append(f"- blocker: {error}") + + top_items = list((backlog.get("top_needs_mapping") or [])[:5]) + if top_items: + lines.append("- top_needs_mapping:") + for item in top_items: + lines.append( + " - " + f"{item.get('pchome_product_id')} | " + f"sales_7d={item.get('sales_7d')} | " + f"score={item.get('priority_score')} | " + f"action={item.get('action_label')}" + ) + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--api-url", default=DEFAULT_API_URL) + parser.add_argument("--health-url", default=version_guard.DEFAULT_HEALTH_URL) + parser.add_argument("--limit", type=int, default=DEFAULT_LIMIT) + parser.add_argument("--timeout", type=float, default=12.0) + parser.add_argument("--json", action="store_true", help="Print machine-readable report") + parser.add_argument( + "--skip-version-truth", + action="store_true", + help="Skip production version guard; intended only for offline parser tests.", + ) + args = parser.parse_args(argv) + + try: + report = build_report( + api_url=args.api_url, + limit=args.limit, + timeout=args.timeout, + health_url=args.health_url, + skip_version_truth=args.skip_version_truth, + ) + except (OSError, ValueError, urllib.error.URLError) as exc: + report = { + "policy": "read_only_production_pchome_mapping_backlog", + "result": "BLOCKED", + "api_url": with_limit(args.api_url, args.limit), + "errors": [str(exc)], + } + print(json.dumps(report, ensure_ascii=False, indent=2) if args.json else format_text(report)) + return 2 + + if args.json: + print(json.dumps(report, ensure_ascii=False, indent=2)) + else: + print(format_text(report)) + return 0 if report.get("result") != "BLOCKED" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index 37551ea..4a3a33f 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -501,6 +501,23 @@ def build_scheduled_automation_health_summary( auto_policy_controlled_dry_run_receipt_closeout_details = ( auto_policy_controlled_dry_run_receipt_closeout.get("details") or {} ) + auto_policy_controlled_dry_run_runner_readiness = _find_check( + source_result, + "PChome auto-policy controlled dry-run runner readiness", + ) + auto_policy_controlled_dry_run_runner_readiness_details = ( + auto_policy_controlled_dry_run_runner_readiness.get("details") or {} + ) + if ( + not auto_policy_controlled_dry_run_runner_readiness + or not auto_policy_controlled_dry_run_runner_readiness_details + ): + auto_policy_controlled_dry_run_runner_readiness = ( + _pchome_auto_policy_controlled_dry_run_runner_readiness_check() + ) + auto_policy_controlled_dry_run_runner_readiness_details = ( + auto_policy_controlled_dry_run_runner_readiness.get("details") or {} + ) surface_readback = _find_check(source_result, "AI surface HTML readback") surface_details = surface_readback.get("details") or {} sitewide_readback = _find_check(source_result, "Sitewide UI/UX Agent readback") @@ -4408,6 +4425,36 @@ def build_scheduled_automation_health_summary( "writes_database": False, }, }, + { + "key": "pchome_auto_policy_controlled_dry_run_runner_readiness", + "label": "PChome auto-policy controlled dry-run runner readiness", + "status": ( + auto_policy_controlled_dry_run_runner_readiness.get("status") + or "warning" + ), + "summary": ( + auto_policy_controlled_dry_run_runner_readiness.get("summary") + or ( + "PChome auto-policy controlled dry-run runner readiness " + "has no latest check." + ) + ), + "next_machine_action": ( + auto_policy_controlled_dry_run_runner_readiness_details.get( + "next_machine_action" + ) + or ( + "continue_pchome_auto_policy_controlled_dry_run_runner_readiness_monitoring" + if auto_policy_controlled_dry_run_runner_readiness.get("status") + == "ok" + else "refresh_pchome_auto_policy_controlled_dry_run_runner_readiness" + ) + ), + "details": { + **auto_policy_controlled_dry_run_runner_readiness_details, + "writes_database": False, + }, + }, { "key": "ai_surface_html_readback", "label": "AI surface HTML readback", @@ -12146,6 +12193,657 @@ def _pchome_auto_policy_controlled_dry_run_receipt_closeout_check() -> Dict[str, ) +def _pchome_auto_policy_controlled_dry_run_runner_readiness_check() -> Dict[str, Any]: + """Read-only monitor for the controlled dry-run runner readiness binding.""" + try: + from services import pchome_mapping_backlog_service as backlog + from services.ai_exception_contract import ( + LEGACY_REVIEW_REQUIRED_COUNT_KEY, + PRIMARY_HUMAN_GATE_COUNT_KEY, + ) + + payload = _pchome_auto_policy_decision_preflight_contract_payload() + readiness = ( + backlog.build_pchome_auto_policy_db_apply_controlled_dry_run_runner_readiness( + payload, + batch_size=max( + 1, + min(_PCHOME_AUTO_POLICY_DECISION_PREFLIGHT_BATCH_SIZE, 50), + ), + execute_fetch=True, + timeout_seconds=max( + 1, + min(_PCHOME_AUTO_POLICY_DECISION_PREFLIGHT_TIMEOUT_SECONDS, 30), + ), + http_get=_pchome_auto_policy_machine_fetch_evidence_get, + ) + ) + summary = readiness.get("summary") or {} + future_plan = ( + readiness.get( + "future_database_apply_controlled_dry_run_execution_plan_binding" + ) + or {} + ) + runner = readiness.get("controlled_dry_run_runner_readiness") or {} + contract = readiness.get("controlled_dry_run_runner_readiness_contract") or {} + safety = readiness.get("safety") or {} + execution_plan = runner.get("execution_plan_binding") or {} + result_parser = runner.get("dry_run_result_parser") or {} + validation = runner.get("receipt_validation_report") or {} + source_closeout = ( + readiness.get("source_controlled_dry_run_receipt_closeout") or {} + ) + source_future_verification = ( + readiness.get( + "source_database_apply_controlled_dry_run_result_parser_verification" + ) + or {} + ) + source_closeout_contract = ( + readiness.get("source_controlled_dry_run_receipt_closeout_contract") + or {} + ) + result = str(readiness.get("result") or "UNKNOWN") + + def _count(key: str) -> int: + return int(summary.get(key) or 0) + + ready_count = _count("controlled_dry_run_runner_readiness_ready_count") + check_count = _count("controlled_dry_run_runner_readiness_check_count") + pass_count = _count("controlled_dry_run_runner_readiness_pass_count") + waiting_count = _count("controlled_dry_run_runner_readiness_waiting_count") + receipt_closeout_ready_count = _count( + "controlled_dry_run_receipt_closeout_ready_count" + ) + receipt_closeout_check_count = _count( + "controlled_dry_run_receipt_closeout_check_count" + ) + package_ready_count = _count("controlled_dry_run_package_ready_count") + package_check_count = _count("controlled_dry_run_package_check_count") + controlled_apply_ready_count = _count( + "controlled_apply_final_preflight_ready_count" + ) + controlled_apply_check_count = _count( + "controlled_apply_final_preflight_check_count" + ) + evidence_closeout_ready_count = _count( + "authorization_evidence_execution_closeout_ready_count" + ) + evidence_closeout_check_count = _count( + "authorization_evidence_execution_closeout_check_count" + ) + evidence_preflight_ready_count = _count( + "authorization_evidence_execution_preflight_ready_count" + ) + evidence_preflight_check_count = _count( + "authorization_evidence_execution_preflight_check_count" + ) + final_verifier_gate_count = _count("database_apply_final_verifier_gate_count") + final_verifier_gate_ready_count = _count( + "database_apply_authorization_final_verifier_gate_ready_count" + ) + runner_count = _count("controlled_dry_run_runner_readiness_count") + runner_field_count = _count("controlled_dry_run_runner_readiness_field_count") + runner_gate_count = _count( + "controlled_dry_run_runner_readiness_acceptance_gate_count" + ) + execution_plan_count = _count("execution_plan_binding_count") + execution_plan_field_count = _count("execution_plan_binding_field_count") + parser_count = _count("dry_run_result_parser_count") + parser_field_count = _count("dry_run_result_parser_field_count") + validation_report_count = _count("receipt_validation_report_count") + validation_field_count = _count("receipt_validation_field_count") + receipt_closeout_count = _count("controlled_dry_run_receipt_closeout_count") + receipt_closeout_field_count = _count( + "controlled_dry_run_receipt_closeout_field_count" + ) + receipt_closeout_gate_count = _count( + "controlled_dry_run_receipt_closeout_acceptance_gate_count" + ) + rollback_binding_count = _count("rollback_binding_count") + verifier_binding_count = _count("post_apply_verifier_binding_count") + verifier_required_count = _count("post_apply_verifier_required_count") + same_run_truth_count = _count("same_run_truth_required_count") + writes_script_count = _count("writes_script_count") + writes_artifact_count = _count("writes_artifact_count") + reads_secret_count = _count("reads_secret_count") + executes_script_count = _count("executes_script_count") + executes_migration_count = _count("executes_migration_count") + executes_endpoint_count = _count("executes_endpoint_count") + executes_sql_count = _count("executes_sql_count") + writes_database_count = _count("writes_database_count") + signs_database_apply_authorization_count = _count( + "signs_database_apply_authorization_count" + ) + primary_human_gate_count = _count(PRIMARY_HUMAN_GATE_COUNT_KEY) + manual_review_required_count = _count(LEGACY_REVIEW_REQUIRED_COUNT_KEY) + side_effect_count = ( + writes_script_count + + writes_artifact_count + + reads_secret_count + + executes_script_count + + executes_migration_count + + executes_endpoint_count + + executes_sql_count + + writes_database_count + + signs_database_apply_authorization_count + ) + risk_detected = any( + [ + side_effect_count, + primary_human_gate_count, + manual_review_required_count, + safety.get("reads_secret_in_preview") is not False, + safety.get("writes_file") is not False, + safety.get("writes_script_in_preview") is not False, + safety.get("writes_artifact_in_preview") is not False, + safety.get("executes_script") is not False, + safety.get("executes_endpoint") is not False, + safety.get("executes_migration") is not False, + safety.get("executes_sql") is not False, + safety.get("writes_database") is not False, + safety.get("signs_database_apply_authorization") is not False, + safety.get("performs_detached_signature_verification") is not False, + safety.get("persists_verifier_receipt") is not False, + safety.get("executes_authorization_evidence") is not False, + safety.get("executes_database_apply") is not False, + future_plan.get("dry_run_execution_performed") is not False, + future_plan.get("runner_execution_authorized") is not False, + future_plan.get("dry_run_execution_authorized") is not False, + future_plan.get("ready_for_database_apply_now") is not False, + future_plan.get("database_apply_authorized") is not False, + future_plan.get("issues_database_apply_authorization") is not False, + future_plan.get("signs_database_apply_authorization") is not False, + future_plan.get("executes_authorization_evidence") is not False, + future_plan.get("executes_database_apply") is not False, + future_plan.get("executes_endpoint") is not False, + future_plan.get("executes_sql") is not False, + future_plan.get("writes_database") is not False, + future_plan.get("reads_secret_in_preview") is not False, + runner.get("runner_execution_authorized") is not False, + runner.get("dry_run_execution_authorized") is not False, + runner.get("ready_for_database_apply_now") is not False, + runner.get("database_apply_authorized") is not False, + runner.get("issues_database_apply_authorization") is not False, + runner.get("signs_database_apply_authorization") is not False, + runner.get("accepts_plaintext_secret") is not False, + runner.get("reads_secret_in_preview") is not False, + runner.get("signature_material_included") is not False, + runner.get("secret_material_included") is not False, + runner.get("executes_authorization_evidence") is not False, + runner.get("executes_database_apply") is not False, + runner.get("executes_endpoint_in_preview") is not False, + runner.get("executes_sql_in_preview") is not False, + runner.get("writes_database_in_preview") is not False, + execution_plan.get("execution_authorized") is not False, + execution_plan.get("dry_run_execution_authorized") is not False, + execution_plan.get("runner_execution_authorized") is not False, + execution_plan.get("shell_execution_included") is not False, + execution_plan.get("endpoint_execution_included") is not False, + execution_plan.get("sql_execution_included") is not False, + execution_plan.get("database_write_included") is not False, + execution_plan.get("stdout_capture_allowed") is not False, + execution_plan.get("stderr_capture_allowed") is not False, + execution_plan.get("database_apply_authorized") is not False, + execution_plan.get("ready_for_database_apply_now") is not False, + validation.get("execution_performed") is not False, + validation.get("stdout_included") is not False, + validation.get("stderr_included") is not False, + validation.get("database_apply_authorized") is not False, + validation.get("executes_shell") is not False, + validation.get("executes_endpoint") is not False, + validation.get("executes_sql") is not False, + validation.get("writes_database") is not False, + validation.get("reads_secret") is not False, + result_parser.get("execution_required") is not False, + result_parser.get("stdout_allowed") is not False, + result_parser.get("stderr_allowed") is not False, + result_parser.get("database_apply_authorized") is not False, + source_closeout_contract.get( + "permits_future_database_apply_controlled_dry_run_runner_readiness" + ) + is not True, + contract.get("accepts_plaintext_secret") is not False, + contract.get("performs_detached_signature_verification") is not False, + contract.get("persists_verifier_receipt") is not False, + contract.get("executes_authorization_evidence") is not False, + contract.get("executes_database_apply") is not False, + contract.get("executes_endpoint") is not False, + contract.get("executes_sql") is not False, + contract.get("issues_database_apply_authorization") is not False, + contract.get("database_apply_authorized") is not False, + contract.get("ready_for_database_apply_now") is not False, + contract.get("signs_database_apply_authorization") is not False, + contract.get("writes_database") is not False, + contract.get("executes_in_preview") is not False, + contract.get("secret_material_required_in_preview") is not False, + ] + ) + runner_contract_present = ( + result == "DB_APPLY_CONTROLLED_DRY_RUN_RUNNER_READINESS_READY" + and ready_count == 1 + and check_count >= 12 + and pass_count == check_count + and waiting_count == 0 + and receipt_closeout_ready_count == 1 + and receipt_closeout_check_count == 12 + and package_ready_count == 1 + and package_check_count == 12 + and controlled_apply_ready_count == 1 + and controlled_apply_check_count == 12 + and evidence_closeout_ready_count == 1 + and evidence_closeout_check_count == 12 + and evidence_preflight_ready_count == 1 + and evidence_preflight_check_count == 12 + and final_verifier_gate_count == 1 + and final_verifier_gate_ready_count == 1 + and runner_count == 1 + and runner_field_count == 12 + and runner_gate_count == 10 + and execution_plan_count == 1 + and execution_plan_field_count == 12 + and parser_count == 1 + and parser_field_count == 10 + and validation_report_count == 1 + and validation_field_count == 8 + and receipt_closeout_count == 1 + and receipt_closeout_field_count == 12 + and receipt_closeout_gate_count == 10 + and rollback_binding_count == 1 + and verifier_binding_count == 1 + and verifier_required_count == 1 + and same_run_truth_count == 1 + and future_plan.get( + "ready_for_future_database_apply_controlled_dry_run_execution_plan_binding" + ) + is True + and future_plan.get( + "can_enter_future_database_apply_controlled_dry_run_execution_plan_closeout" + ) + is True + and future_plan.get("controlled_dry_run_runner_readiness_ready") is True + and future_plan.get("execution_plan_bound") is True + and future_plan.get("dry_run_execution_performed") is False + and future_plan.get("runner_execution_authorized") is False + and future_plan.get("dry_run_execution_authorized") is False + and source_future_verification.get( + "can_enter_future_database_apply_controlled_dry_run_runner_readiness" + ) + is True + and source_future_verification.get("dry_run_execution_performed") + is False + and runner.get("authorization_material_type") + == "controlled_dry_run_runner_readiness" + and runner.get( + "ready_for_future_database_apply_controlled_dry_run_runner_readiness" + ) + is True + and runner.get("target_migration_hash_locked") is True + and runner.get("dry_run_only") is True + and runner.get("check_mode_only") is True + and runner.get("runner_readiness_only") is True + and runner.get("execution_plan_preview_only") is True + and runner.get("requires_fresh_production_truth_in_same_run") is True + and runner.get("requires_post_apply_verifier") is True + and execution_plan.get("runner_mode") + == "future_controlled_dry_run_runner_readiness_only" + and execution_plan.get("plan_status") + == "plan_binding_preview_not_executable" + and execution_plan.get("dry_run_only") is True + and execution_plan.get("check_mode_only") is True + and execution_plan.get("execution_authorized") is False + and execution_plan.get("dry_run_execution_authorized") is False + and execution_plan.get("runner_execution_authorized") is False + and execution_plan.get("shell_execution_included") is False + and execution_plan.get("endpoint_execution_included") is False + and execution_plan.get("sql_execution_included") is False + and execution_plan.get("database_write_included") is False + and execution_plan.get("stdout_capture_allowed") is False + and execution_plan.get("stderr_capture_allowed") is False + and execution_plan.get("database_apply_authorized") is False + and execution_plan.get("ready_for_database_apply_now") is False + and validation.get("receipt_validation_status") + == "preview_validated_not_executed" + and validation.get("execution_performed") is False + and validation.get("stdout_included") is False + and validation.get("stderr_included") is False + and validation.get("database_apply_authorized") is False + and validation.get("executes_shell") is False + and validation.get("executes_endpoint") is False + and validation.get("executes_sql") is False + and validation.get("writes_database") is False + and validation.get("reads_secret") is False + and result_parser.get("expected_receipt_status") + == "preview_only_not_executed" + and result_parser.get("execution_required") is False + and result_parser.get("parser_verification_status") + == "schema_preview_ready" + and contract.get("machine_verifiable") is True + and contract.get( + "permits_future_database_apply_controlled_dry_run_execution_plan_binding" + ) + is True + and contract.get("manual_review_mode") == "exception_only" + and safety.get("manual_review_mode") == "exception_only" + ) + + if risk_detected: + status = "critical" + summary_text = ( + "PChome auto-policy controlled dry-run runner readiness " + "偵測到 secret、runner execution、stdout/stderr、SQL、DB write、" + "database apply、簽發授權或人工主 gate 風險" + ) + next_machine_action = ( + "halt_pchome_auto_policy_controlled_dry_run_runner_readiness_and_inspect_risk" + ) + elif runner_contract_present: + status = "ok" + summary_text = ( + "PChome auto-policy controlled dry-run runner readiness " + f"已自動完成 {pass_count}/{check_count} checks，execution plan 已綁定且不可執行，runner/write/apply=0" + ) + next_machine_action = ( + "continue_to_pchome_auto_policy_controlled_dry_run_execution_plan_closeout_lane" + ) + else: + status = "warning" + summary_text = ( + "PChome auto-policy controlled dry-run runner readiness " + "尚未達自動監控契約" + ) + next_machine_action = ( + "refresh_pchome_auto_policy_controlled_dry_run_runner_readiness" + ) + + return _check( + "PChome auto-policy controlled dry-run runner readiness", + status, + summary_text, + { + "policy": readiness.get("policy"), + "result": result, + "source_policy": readiness.get("source_policy"), + "controlled_dry_run_runner_readiness_ready_count": ready_count, + "controlled_dry_run_runner_readiness_check_count": check_count, + "controlled_dry_run_runner_readiness_pass_count": pass_count, + "controlled_dry_run_runner_readiness_waiting_count": waiting_count, + "controlled_dry_run_receipt_closeout_ready_count": ( + receipt_closeout_ready_count + ), + "controlled_dry_run_receipt_closeout_check_count": ( + receipt_closeout_check_count + ), + "controlled_dry_run_package_ready_count": package_ready_count, + "controlled_dry_run_package_check_count": package_check_count, + "controlled_apply_final_preflight_ready_count": ( + controlled_apply_ready_count + ), + "controlled_apply_final_preflight_check_count": ( + controlled_apply_check_count + ), + "authorization_evidence_execution_closeout_ready_count": ( + evidence_closeout_ready_count + ), + "authorization_evidence_execution_closeout_check_count": ( + evidence_closeout_check_count + ), + "authorization_evidence_execution_preflight_ready_count": ( + evidence_preflight_ready_count + ), + "authorization_evidence_execution_preflight_check_count": ( + evidence_preflight_check_count + ), + "database_apply_final_verifier_gate_count": final_verifier_gate_count, + "database_apply_authorization_final_verifier_gate_ready_count": ( + final_verifier_gate_ready_count + ), + "controlled_dry_run_runner_readiness_count": runner_count, + "controlled_dry_run_runner_readiness_field_count": runner_field_count, + "controlled_dry_run_runner_readiness_acceptance_gate_count": ( + runner_gate_count + ), + "execution_plan_binding_count": execution_plan_count, + "execution_plan_binding_field_count": execution_plan_field_count, + "dry_run_result_parser_count": parser_count, + "dry_run_result_parser_field_count": parser_field_count, + "receipt_validation_report_count": validation_report_count, + "receipt_validation_field_count": validation_field_count, + "controlled_dry_run_receipt_closeout_count": receipt_closeout_count, + "controlled_dry_run_receipt_closeout_field_count": ( + receipt_closeout_field_count + ), + "controlled_dry_run_receipt_closeout_acceptance_gate_count": ( + receipt_closeout_gate_count + ), + "rollback_binding_count": rollback_binding_count, + "post_apply_verifier_binding_count": verifier_binding_count, + "post_apply_verifier_required_count": verifier_required_count, + "same_run_truth_required_count": same_run_truth_count, + "writes_script_count": writes_script_count, + "writes_artifact_count": writes_artifact_count, + "reads_secret_count": reads_secret_count, + "executes_script_count": executes_script_count, + "executes_migration_count": executes_migration_count, + "executes_endpoint_count": executes_endpoint_count, + "executes_sql_count": executes_sql_count, + "writes_database_count": writes_database_count, + "signs_database_apply_authorization_count": ( + signs_database_apply_authorization_count + ), + "primary_human_gate_count": primary_human_gate_count, + "manual_review_required_count": manual_review_required_count, + "manual_review_mode": safety.get("manual_review_mode"), + "payload_source": "local_contract_fixture", + "machine_fetch_evidence_source": "local_schema_fixture", + "http_get_adapter": "_pchome_auto_policy_machine_fetch_evidence_get", + "execute_fetch": True, + "outbound_network": False, + "business_data_source": False, + "ready_for_future_database_apply_controlled_dry_run_execution_plan_binding": bool( + future_plan.get( + "ready_for_future_database_apply_controlled_dry_run_execution_plan_binding" + ) + ), + "can_enter_future_database_apply_controlled_dry_run_execution_plan_closeout": bool( + future_plan.get( + "can_enter_future_database_apply_controlled_dry_run_execution_plan_closeout" + ) + ), + "controlled_dry_run_runner_readiness_ready": bool( + future_plan.get("controlled_dry_run_runner_readiness_ready") + ), + "execution_plan_bound": bool(future_plan.get("execution_plan_bound")), + "source_controlled_dry_run_receipt_closeout_ready": bool( + source_future_verification.get( + "controlled_dry_run_receipt_closeout_ready" + ) + ), + "dry_run_execution_performed": bool( + future_plan.get("dry_run_execution_performed") + or source_future_verification.get("dry_run_execution_performed") + ), + "ready_for_future_database_apply_controlled_dry_run_runner_readiness": bool( + runner.get( + "ready_for_future_database_apply_controlled_dry_run_runner_readiness" + ) + ), + "target_migration_hash_locked": bool( + runner.get("target_migration_hash_locked") + or source_closeout.get("target_migration_hash_locked") + ), + "dry_run_only": bool(runner.get("dry_run_only")), + "check_mode_only": bool(runner.get("check_mode_only")), + "runner_readiness_only": bool(runner.get("runner_readiness_only")), + "execution_plan_preview_only": bool( + runner.get("execution_plan_preview_only") + ), + "requires_fresh_production_truth": bool( + runner.get("requires_fresh_production_truth_in_same_run") + ), + "requires_post_apply_verifier": bool( + runner.get("requires_post_apply_verifier") + ), + "execution_plan_runner_mode": execution_plan.get("runner_mode"), + "execution_plan_status": execution_plan.get("plan_status"), + "execution_authorized": bool( + execution_plan.get("execution_authorized") + ), + "dry_run_execution_authorized": bool( + execution_plan.get("dry_run_execution_authorized") + or future_plan.get("dry_run_execution_authorized") + or runner.get("dry_run_execution_authorized") + ), + "runner_execution_authorized": bool( + execution_plan.get("runner_execution_authorized") + or future_plan.get("runner_execution_authorized") + or runner.get("runner_execution_authorized") + ), + "shell_execution_included": bool( + execution_plan.get("shell_execution_included") + ), + "endpoint_execution_included": bool( + execution_plan.get("endpoint_execution_included") + ), + "sql_execution_included": bool( + execution_plan.get("sql_execution_included") + ), + "database_write_included": bool( + execution_plan.get("database_write_included") + ), + "stdout_capture_allowed": bool( + execution_plan.get("stdout_capture_allowed") + ), + "stderr_capture_allowed": bool( + execution_plan.get("stderr_capture_allowed") + ), + "receipt_validation_status": validation.get( + "receipt_validation_status" + ), + "validation_execution_performed": bool( + validation.get("execution_performed") + ), + "validation_stdout_included": bool(validation.get("stdout_included")), + "validation_stderr_included": bool(validation.get("stderr_included")), + "validation_database_apply_authorized": bool( + validation.get("database_apply_authorized") + ), + "validation_executes_shell": bool(validation.get("executes_shell")), + "validation_executes_endpoint": bool( + validation.get("executes_endpoint") + ), + "validation_executes_sql": bool(validation.get("executes_sql")), + "validation_writes_database": bool(validation.get("writes_database")), + "validation_reads_secret": bool(validation.get("reads_secret")), + "parser_expected_receipt_status": result_parser.get( + "expected_receipt_status" + ), + "parser_execution_required": bool( + result_parser.get("execution_required") + ), + "parser_stdout_allowed": bool(result_parser.get("stdout_allowed")), + "parser_stderr_allowed": bool(result_parser.get("stderr_allowed")), + "parser_database_apply_authorized": bool( + result_parser.get("database_apply_authorized") + ), + "parser_verification_status": result_parser.get( + "parser_verification_status" + ), + "permits_future_database_apply_controlled_dry_run_execution_plan_binding": bool( + contract.get( + "permits_future_database_apply_controlled_dry_run_execution_plan_binding" + ) + ), + "accepts_plaintext_secret": bool( + runner.get("accepts_plaintext_secret") + or contract.get("accepts_plaintext_secret") + ), + "reads_secret_in_preview": bool( + runner.get("reads_secret_in_preview") + or safety.get("reads_secret_in_preview") + ), + "signature_material_included": bool( + runner.get("signature_material_included") + ), + "secret_material_included": bool( + runner.get("secret_material_included") + ), + "secret_material_required_in_preview": bool( + contract.get("secret_material_required_in_preview") + ), + "executes_authorization_evidence": bool( + future_plan.get("executes_authorization_evidence") + or runner.get("executes_authorization_evidence") + or contract.get("executes_authorization_evidence") + or safety.get("executes_authorization_evidence") + ), + "executes_database_apply": bool( + future_plan.get("executes_database_apply") + or runner.get("executes_database_apply") + or contract.get("executes_database_apply") + or safety.get("executes_database_apply") + ), + "executes_endpoint": bool( + future_plan.get("executes_endpoint") + or contract.get("executes_endpoint") + or safety.get("executes_endpoint") + ), + "executes_sql": bool( + future_plan.get("executes_sql") + or contract.get("executes_sql") + or safety.get("executes_sql") + ), + "ready_for_database_apply_now": bool( + future_plan.get("ready_for_database_apply_now") + or runner.get("ready_for_database_apply_now") + or execution_plan.get("ready_for_database_apply_now") + or contract.get("ready_for_database_apply_now") + ), + "database_apply_authorized": bool( + future_plan.get("database_apply_authorized") + or runner.get("database_apply_authorized") + or execution_plan.get("database_apply_authorized") + or contract.get("database_apply_authorized") + ), + "issues_database_apply_authorization": bool( + future_plan.get("issues_database_apply_authorization") + or runner.get("issues_database_apply_authorization") + or contract.get("issues_database_apply_authorization") + ), + "signs_database_apply_authorization": bool( + future_plan.get("signs_database_apply_authorization") + or runner.get("signs_database_apply_authorization") + or contract.get("signs_database_apply_authorization") + ), + "writes_database": False, + "next_machine_action": next_machine_action, + }, + ) + except Exception as exc: + return _check( + "PChome auto-policy controlled dry-run runner readiness", + "warning", + ( + "PChome auto-policy controlled dry-run runner readiness " + f"例行監控暫時無法讀取：{exc}" + ), + { + "writes_database": False, + "writes_database_count": 0, + "signs_database_apply_authorization_count": 0, + "primary_human_gate_count": 0, + "execute_fetch": True, + "outbound_network": False, + "business_data_source": False, + "payload_source": "local_contract_fixture", + "next_machine_action": ( + "refresh_pchome_auto_policy_controlled_dry_run_runner_readiness" + ), + }, + ) + + def _ai_surface_html_readback_check() -> Dict[str, Any]: """Read-only AI product surface guardrail readback.""" try: @@ -12393,6 +13091,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _pchome_auto_policy_controlled_apply_final_preflight_check(), _pchome_auto_policy_controlled_dry_run_package_check(), _pchome_auto_policy_controlled_dry_run_receipt_closeout_check(), + _pchome_auto_policy_controlled_dry_run_runner_readiness_check(), _ai_surface_html_readback_check(), _sitewide_ui_ux_agent_check(), _sitewide_visual_qa_check(), diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index d30bf39..3031bc7 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1250,6 +1250,129 @@ def _auto_policy_controlled_dry_run_receipt_closeout_history_check(status="ok"): } +def _auto_policy_controlled_dry_run_runner_readiness_history_check(status="ok"): + return { + "name": "PChome auto-policy controlled dry-run runner readiness", + "status": status, + "summary": ( + "PChome auto-policy controlled dry-run runner readiness 已自動完成" + ), + "details": { + "policy": ( + "read_only_pchome_growth_auto_policy_db_apply_controlled_" + "dry_run_runner_readiness" + ), + "result": "DB_APPLY_CONTROLLED_DRY_RUN_RUNNER_READINESS_READY", + "controlled_dry_run_runner_readiness_ready_count": 1, + "controlled_dry_run_runner_readiness_check_count": 12, + "controlled_dry_run_runner_readiness_pass_count": 12, + "controlled_dry_run_runner_readiness_waiting_count": 0, + "controlled_dry_run_receipt_closeout_ready_count": 1, + "controlled_dry_run_receipt_closeout_check_count": 12, + "controlled_dry_run_package_ready_count": 1, + "controlled_dry_run_package_check_count": 12, + "controlled_apply_final_preflight_ready_count": 1, + "controlled_apply_final_preflight_check_count": 12, + "authorization_evidence_execution_closeout_ready_count": 1, + "authorization_evidence_execution_closeout_check_count": 12, + "authorization_evidence_execution_preflight_ready_count": 1, + "authorization_evidence_execution_preflight_check_count": 12, + "database_apply_final_verifier_gate_count": 1, + "database_apply_authorization_final_verifier_gate_ready_count": 1, + "controlled_dry_run_runner_readiness_count": 1, + "controlled_dry_run_runner_readiness_field_count": 12, + "controlled_dry_run_runner_readiness_acceptance_gate_count": 10, + "execution_plan_binding_count": 1, + "execution_plan_binding_field_count": 12, + "dry_run_result_parser_count": 1, + "dry_run_result_parser_field_count": 10, + "receipt_validation_report_count": 1, + "receipt_validation_field_count": 8, + "controlled_dry_run_receipt_closeout_count": 1, + "controlled_dry_run_receipt_closeout_field_count": 12, + "controlled_dry_run_receipt_closeout_acceptance_gate_count": 10, + "rollback_binding_count": 1, + "post_apply_verifier_binding_count": 1, + "post_apply_verifier_required_count": 1, + "same_run_truth_required_count": 1, + "writes_script_count": 0, + "writes_artifact_count": 0, + "reads_secret_count": 0, + "executes_script_count": 0, + "executes_migration_count": 0, + "executes_endpoint_count": 0, + "executes_sql_count": 0, + "writes_database_count": 0, + "signs_database_apply_authorization_count": 0, + "primary_human_gate_count": 0, + "manual_review_required_count": 0, + "manual_review_mode": "exception_only", + "payload_source": "local_contract_fixture", + "execute_fetch": True, + "outbound_network": False, + "business_data_source": False, + "ready_for_future_database_apply_controlled_dry_run_execution_plan_binding": True, + "can_enter_future_database_apply_controlled_dry_run_execution_plan_closeout": True, + "controlled_dry_run_runner_readiness_ready": True, + "execution_plan_bound": True, + "source_controlled_dry_run_receipt_closeout_ready": True, + "dry_run_execution_performed": False, + "ready_for_future_database_apply_controlled_dry_run_runner_readiness": True, + "target_migration_hash_locked": True, + "dry_run_only": True, + "check_mode_only": True, + "runner_readiness_only": True, + "execution_plan_preview_only": True, + "requires_fresh_production_truth": True, + "requires_post_apply_verifier": True, + "execution_plan_runner_mode": "future_controlled_dry_run_runner_readiness_only", + "execution_plan_status": "plan_binding_preview_not_executable", + "execution_authorized": False, + "dry_run_execution_authorized": False, + "runner_execution_authorized": False, + "shell_execution_included": False, + "endpoint_execution_included": False, + "sql_execution_included": False, + "database_write_included": False, + "stdout_capture_allowed": False, + "stderr_capture_allowed": False, + "receipt_validation_status": "preview_validated_not_executed", + "validation_execution_performed": False, + "validation_stdout_included": False, + "validation_stderr_included": False, + "validation_database_apply_authorized": False, + "validation_executes_shell": False, + "validation_executes_endpoint": False, + "validation_executes_sql": False, + "validation_writes_database": False, + "validation_reads_secret": False, + "parser_expected_receipt_status": "preview_only_not_executed", + "parser_execution_required": False, + "parser_stdout_allowed": False, + "parser_stderr_allowed": False, + "parser_database_apply_authorized": False, + "parser_verification_status": "schema_preview_ready", + "permits_future_database_apply_controlled_dry_run_execution_plan_binding": True, + "accepts_plaintext_secret": False, + "reads_secret_in_preview": False, + "signature_material_included": False, + "secret_material_included": False, + "secret_material_required_in_preview": False, + "executes_authorization_evidence": False, + "executes_database_apply": False, + "executes_endpoint": False, + "executes_sql": False, + "ready_for_database_apply_now": False, + "database_apply_authorized": False, + "issues_database_apply_authorization": False, + "signs_database_apply_authorization": False, + "next_machine_action": ( + "continue_to_pchome_auto_policy_controlled_dry_run_execution_plan_closeout_lane" + ), + }, + } + + def test_event_router_smoke_reports_queued_deliveries(tmp_path, monkeypatch): from services import ai_automation_metrics as metrics from services import ai_automation_smoke_service as smoke diff --git a/tests/test_auto_import_failure_boundaries.py b/tests/test_auto_import_failure_boundaries.py index fc7b770..44eec9d 100644 --- a/tests/test_auto_import_failure_boundaries.py +++ b/tests/test_auto_import_failure_boundaries.py @@ -43,6 +43,7 @@ def _prepare_daily_sales_tables(import_service): def test_daily_sales_import_fails_when_monthly_sync_fails(monkeypatch, tmp_path): + monkeypatch.setenv("DAILY_SALES_IMPORT_MAX_STALE_DAYS", "30") import_service = _load_import_service(monkeypatch, f"sqlite:///{tmp_path / 'momo.db'}") _prepare_daily_sales_tables(import_service) diff --git a/tests/test_competitor_identity_revalidator.py b/tests/test_competitor_identity_revalidator.py index ce9234a..0949888 100644 --- a/tests/test_competitor_identity_revalidator.py +++ b/tests/test_competitor_identity_revalidator.py @@ -117,7 +117,7 @@ def test_dashboard_match_status_explains_unit_comparable_bundle(): assert "已換算單位價" in status["summary"] -def test_dashboard_match_status_shows_manual_review_closure_states(): +def test_dashboard_match_status_shows_ai_exception_closure_states(): from routes.dashboard_routes import _build_competitor_decision, _build_pchome_match_status rejected = _build_pchome_match_status({ @@ -133,15 +133,15 @@ def test_dashboard_match_status_shows_manual_review_closure_states(): "best_match_score": 0.68, }) - assert rejected["label"] == "人工已否決" + assert rejected["label"] == "AI 已排除" assert "跳過正式價差寫入" in rejected["summary"] - assert unit_price["label"] == "人工標記單位價" + assert unit_price["label"] == "AI 標記單位價" assert "總價不可直接比較" in unit_price["summary"] - assert needs_research["label"] == "人工要求補搜尋" + assert needs_research["label"] == "AI 要求補搜尋" assert "重新抓取" in needs_research["summary"] decision = _build_competitor_decision(980, 899, match_status=rejected) - assert decision["label"] == "人工已否決" + assert decision["label"] == "AI 已排除" assert decision["gap_amount"] is None @@ -154,9 +154,9 @@ def test_dashboard_match_status_shows_rescore_accepted_review_state(): "error_message": "matcher_rescore=accepted_current; reasons=strong_exact_spec_match", }) - assert status["label"] == "重算待人工覆核" + assert status["label"] == "重算待 AI 決策" assert status["tone"] == "watch" - assert "人工確認身份後才可寫入正式 PChome 價差" in status["summary"] + assert "AI 決策信封確認身份" in status["summary"] def test_dashboard_match_status_uses_specific_matcher_reason_labels(): @@ -175,8 +175,8 @@ def test_dashboard_match_status_uses_specific_matcher_reason_labels(): assert finish_gap["label"] == "妝效質地不同" assert "妝效質地不同" in finish_gap["summary"] - assert variant_review["label"] == "多款任選待確認" - assert "需人工確認" in variant_review["summary"] + assert variant_review["label"] == "多款任選待決策" + assert "需 AI 例外決策" in variant_review["summary"] def test_dashboard_match_status_uses_focused_marketplace_reason_labels(): diff --git a/tests/test_competitor_intel_cache.py b/tests/test_competitor_intel_cache.py index 6f2853d..beabbfb 100644 --- a/tests/test_competitor_intel_cache.py +++ b/tests/test_competitor_intel_cache.py @@ -184,12 +184,12 @@ def test_competitor_review_queue_is_canonical_unit_price_handoff(): assert "\"action_label\"" in source assert "build_unit_price_comparison" in source assert "需單位價覆核" in daily_template - assert "人工採用" in daily_template - assert "人工否決" in daily_template - assert "人工單位價" in daily_template + assert "AI 採用" in daily_template + assert "AI 排除" in daily_template + assert "AI 單位價" in daily_template assert "competitor_intel.review_queue" in daily_template - assert "coverage.fresh_matches" in growth_template - assert "coverage.fresh_match_rate" in growth_template + assert "chart_data.competitor_coverage" in growth_template + assert "coverage.decision_support_rate" in growth_template assert "coverage.decision_ready_rate" in growth_template assert "coverage.decision_support_rate" in growth_template assert "coverage.catalog_comparable_count" in growth_template @@ -199,12 +199,12 @@ def test_competitor_review_queue_is_canonical_unit_price_handoff(): assert "未形成有效身份配對" in growth_template assert "coverage.unit_comparable_count" in growth_template assert "coverage.rescore_accepted_count" in growth_template - assert "重算待人工覆核" in growth_template + assert "重算待 AI 決策" in growth_template assert "coverage.manual_accept_count" in growth_template assert "coverage.manual_reject_count" in growth_template assert "coverage.manual_unit_price_count" in growth_template assert "comp_coverage.rescore_accepted_count" in daily_template - assert "重算待人工覆核" in daily_template + assert "重算待 AI 決策" in daily_template assert "comp_coverage.decision_support_rate" in daily_template assert "comp_coverage.catalog_comparable_count" in daily_template assert "精準可告警覆蓋" in daily_template @@ -364,7 +364,9 @@ def test_competitor_review_reasons_prefer_json_payload_labels(): assert envelope["identity_evidence"]["brand"]["shared"] == ["mac"] assert envelope["offer_evidence"]["gap_pct"] == 12.5 assert envelope["difference_highlights"][0]["dimension"] == "妝效/質地不同" - assert envelope["recommended_action"]["requires_hitl"] is True + assert envelope["recommended_action"]["requires_hitl"] is False + assert envelope["recommended_action"]["owner"] == "AI Agent" + assert envelope["recommended_action"]["ai_exception_mode"] == "machine_verifiable_auto_resolution" assert envelope["recommended_action"]["action"] == "verify_or_reject_identity" assert any(evidence["metric"] == "reasons" for evidence in envelope["evidence"]) assert any(evidence["metric"] == "identity_evidence" for evidence in envelope["evidence"]) @@ -502,12 +504,12 @@ def test_review_decision_brief_is_shared_by_openclaw_and_ppt(): brief = summarize_review_decision_envelopes([item], limit=5) - assert brief["hitl_count"] == 1 + assert brief["hitl_count"] == 0 assert brief["auto_execute_blocked_count"] == 1 assert brief["severity_counts"] assert brief["data_quality_counts"] == {"complete": 1} - assert "人工確認身份後採用同款" in brief["text"] - assert "HITL" in brief["text"] + assert "AI 確認身份後採用同款" in brief["text"] + assert "HITL" not in brief["text"] assert "DDAO4C-A79050612" in brief["text"] repo_source = (ROOT / "services" / "competitor_intel_repository.py").read_text(encoding="utf-8") @@ -518,11 +520,11 @@ def test_review_decision_brief_is_shared_by_openclaw_and_ppt(): assert "\"review_decision_brief\": summarize_review_decision_envelopes" in repo_source assert "summarize_review_decision_envelopes" in openclaw_source assert "review_decision_text" in openclaw_source - assert "PChome 覆核決策信封（HITL，不可自動寫正式價差）" in openclaw_source + assert "PChome AI 決策信封（no-write receipt 前不寫正式價差）" in openclaw_source assert "SUM(CASE WHEN attempt_status IN ('unit_comparable'" not in openclaw_source assert "review_decision_brief" in bot_source - assert "覆核決策信封（HITL，不可自動寫正式價差）" in bot_source - assert "覆核決策信封（HITL）" in ppt_source + assert "AI 決策信封（no-write receipt 前不寫正式價差）" in bot_source + assert "AI 決策信封" in ppt_source def test_competitor_ppt_prompt_uses_neutral_ewooc_viewpoint(): diff --git a/tests/test_elephant_alpha_engine.py b/tests/test_elephant_alpha_engine.py index f55327f..4d5bdde 100644 --- a/tests/test_elephant_alpha_engine.py +++ b/tests/test_elephant_alpha_engine.py @@ -146,7 +146,7 @@ def test_competitor_db_evidence_actions_are_concrete(): ) assert actions == [ - "[SKU-1] 測試商品｜MOMO $1,200 vs PChome $990 (+17.5%)｜每件價差 NT$ 210｜證據：高信心同款 / 總價可比 / 可直接價格告警 / score 0.84｜建議人工確認 PChome identity_v2 後評估跟價或促銷｜PChome D123456" + "[SKU-1] 測試商品｜MOMO $1,200 vs PChome $990 (+17.5%)｜每件價差 NT$ 210｜證據：高信心同款 / 總價可比 / 可直接價格告警 / score 0.84｜建議 AI 自動驗證確認 PChome identity_v2 後評估跟價或促銷｜PChome D123456" ] @@ -279,9 +279,10 @@ def test_price_review_decision_envelope_blocks_auto_execution(): assert envelope["decision_type"] == "price_decision_review" assert envelope["source_agent"] == "elephant_alpha" - assert envelope["recommended_action"]["requires_hitl"] is True + assert envelope["recommended_action"]["requires_hitl"] is False + assert envelope["recommended_action"]["requires_ai_exception"] is True assert envelope["guardrails"]["can_auto_execute"] is False - assert envelope["guardrails"]["blocked_reason"] == "price decisions require HITL; execution_plan skipped" + assert envelope["guardrails"]["blocked_reason"] == "price decisions require AI 例外決策; execution_plan skipped" assert envelope["guardrails"]["data_quality"] == "complete" @@ -701,7 +702,8 @@ def test_human_escalation_decision_envelope_blocks_auto_execution(): assert envelope["decision_type"] == "ea_escalation" assert envelope["guardrails"]["can_auto_execute"] is False assert envelope["guardrails"]["data_quality"] == "partial" - assert envelope["recommended_action"]["requires_hitl"] is True + assert envelope["recommended_action"]["requires_hitl"] is False + assert envelope["recommended_action"]["requires_ai_exception"] is True assert envelope["trace"]["provider"] == "elephant_alpha" diff --git a/tests/test_event_router.py b/tests/test_event_router.py index e24d4e6..8a43dcf 100644 --- a/tests/test_event_router.py +++ b/tests/test_event_router.py @@ -163,11 +163,16 @@ def test_dispatch_decision_envelope_skips_ai_handler(monkeypatch): "confidence": 0.93, } ], - "recommended_action": {"action": "price_follow_review", "owner": "營運", "requires_hitl": True}, + "recommended_action": { + "action": "price_follow_review", + "owner": "營運", + "requires_hitl": False, + "requires_ai_exception": True, + }, "guardrails": { "can_auto_execute": False, "data_quality": "complete", - "blocked_reason": "價格調整需人工覆核", + "blocked_reason": "價格調整需 AI 例外決策", }, "expected_impact": { "gap_amount": 20, @@ -193,7 +198,8 @@ def test_dispatch_decision_envelope_skips_ai_handler(monkeypatch): assert "🎯 標的" in message assert "📊 價格證據" in message assert "🧩 比對證據" in message - assert "✅ 人工下一步" in message + assert "✅ AI 例外下一步" in message + assert "AI 例外：需要" in message assert "🚦 通知分級" in message assert "直接價格威脅" in message assert "高信心同款 / 總價可比 / 可直接價格告警" in message @@ -231,7 +237,12 @@ def test_price_decision_template_marks_unit_price_review_as_non_direct_alert(): "basis": "same_product_different_pack/unit_price/unit_price_review", } ], - "recommended_action": {"action": "unit_price_required", "owner": "營運", "requires_hitl": True}, + "recommended_action": { + "action": "unit_price_required", + "owner": "營運", + "requires_hitl": False, + "requires_ai_exception": True, + }, "guardrails": { "can_auto_execute": False, "data_quality": "partial", diff --git a/tests/test_external_market_offer_service.py b/tests/test_external_market_offer_service.py index 9677e3c..bcf3763 100644 --- a/tests/test_external_market_offer_service.py +++ b/tests/test_external_market_offer_service.py @@ -38,6 +38,23 @@ def test_connector_contract_lists_mainstream_sources_without_auto_alerts(): assert sources["tiktok_shop"]["plain_note"].startswith("保留短影音電商") assert sources["ruten"]["display_name"] == "露天" assert "price" in payload["manual_csv"]["required_headers"] + assert "product_url" in payload["manual_csv"]["optional_headers"] + assert "image_url" in payload["manual_csv"]["optional_headers"] + assert "stock_status" in payload["manual_csv"]["optional_headers"] + evidence_fields = { + field["name"] + for field in payload["offer_evidence_contract"]["professional_evidence_fields"] + } + assert { + "source_product_id", + "title", + "product_url", + "image_url", + "price", + "stock_status", + "observed_at", + "quality_score", + }.issubset(evidence_fields) def test_normalized_offer_payload_validates_plain_required_fields(): @@ -50,6 +67,9 @@ def test_normalized_offer_payload_validates_plain_required_fields(): "price": "899", "observed_at": "2026-06-15T10:00:00", "ingestion_method": "manual_csv", + "product_url": "https://momo.test/MOMO-1", + "image_url": "https://img.test/MOMO-1.jpg", + "stock_status": "in_stock", "quality_score": 82, "quality_note": "人工確認同款", }) @@ -57,6 +77,9 @@ def test_normalized_offer_payload_validates_plain_required_fields(): assert errors == [] assert record is not None assert record.to_record()["source_offer_key"] == "momo_reference:MOMO-1" + assert record.to_record()["product_url"] == "https://momo.test/MOMO-1" + assert record.to_record()["image_url"] == "https://img.test/MOMO-1.jpg" + assert record.to_record()["stock_status"] == "in_stock" assert record.to_record()["data_quality_status"] == "needs_review" missing_record, missing_errors = normalize_external_offer_payload({ @@ -91,7 +114,7 @@ def test_external_offer_csv_dry_run_accepts_chinese_headers_and_classifies_rows( } rows = payload["rows"] assert rows[0]["status_label"] == "可使用" - assert rows[1]["status_label"] == "需人工確認" + assert rows[1]["status_label"] == "需 AI 自動驗證確認" assert "這個來源目前先暫停，不進告警" in rows[1]["reasons"] assert rows[2]["status_label"] == "不能使用" assert "缺少售價" in rows[2]["reasons"] @@ -117,7 +140,7 @@ def test_external_offer_csv_dry_run_accepts_plain_business_headers_and_values(): assert rows[0]["source_code"] == "momo_reference" assert rows[0]["status_label"] == "可使用" assert rows[1]["source_code"] == "momo_reference" - assert rows[1]["status_label"] == "需人工確認" + assert rows[1]["status_label"] == "需 AI 自動驗證確認" assert "尚未確認同款" in rows[1]["reasons"] assert "缺少 PChome 商品 ID，無法連到業績" in rows[1]["reasons"] @@ -377,7 +400,7 @@ def test_sync_targeted_momo_candidates_skips_total_price_identity_review(monkeyp assert payload["success"] is True assert payload["candidate_count"] == 1 assert payload["written_count"] == 0 - assert payload["skipped_reasons"] == {"候選仍需人工確認": 1} + assert payload["skipped_reasons"] == {"候選仍需 AI 自動驗證確認": 1} assert stale_marks == [] with engine.connect() as conn: count = conn.execute(text("SELECT COUNT(*) FROM external_offers")).scalar() @@ -424,7 +447,7 @@ def test_sync_targeted_momo_review_candidates_writes_needs_review_offer(monkeypa assert payload["status"] == "synced" assert payload["candidate_count"] == 2 assert payload["written_count"] == 1 - assert payload["skipped_reasons"] == {"人工確認候選分數過低": 1} + assert payload["skipped_reasons"] == {"AI 自動驗證確認候選分數過低": 1} with engine.connect() as conn: row = conn.execute(text(""" @@ -608,6 +631,13 @@ def test_external_source_readiness_uses_legacy_momo_reference_cache(): assert sources["tiktok_shop"]["plain_state"] == "先保留接口，不進告警" assert sources["line_shopping"]["plain_state"] == "先保留接口，不進告警" assert sources["shopify_brand_store"]["plain_state"] == "先保留接口，不進告警" + evidence_fields = { + field["name"] + for field in payload["offer_evidence_contract"]["professional_evidence_fields"] + } + assert "image_url" in evidence_fields + assert "stock_status" in evidence_fields + assert payload["offer_evidence_contract"]["quality_gate"]["unverified_data_route"] == "待確認候選，不進告警" assert payload["plain_summary"] == "MOMO 先用；其他主流平台已列管，未接合法穩定來源前不進告警。" diff --git a/tests/test_frontend_v2_assets.py b/tests/test_frontend_v2_assets.py index 8b22400..4aad45d 100644 --- a/tests/test_frontend_v2_assets.py +++ b/tests/test_frontend_v2_assets.py @@ -338,12 +338,16 @@ def test_dashboard_v2_is_production_default_and_uses_real_dashboard_data(): assert "def _load_competitor_decision_overview(session, latest_items=None)" in route_source assert "def _load_pchome_growth_command_center(session)" in route_source assert "build_pchome_growth_opportunities(engine, limit=16)" in route_source + assert "build_pchome_direct_mapping_candidate_decision_package(" in route_source assert "build_pchome_direct_mapping_retry_candidate_exception_controlled_apply_receipt_replay_package(" in route_source assert "build_pchome_direct_mapping_retry_candidate_exception_controlled_apply_compact_readback_package(" in route_source assert "build_pchome_direct_mapping_retry_candidate_exception_controlled_apply_artifact_retention_package(" in route_source + assert "build_pchome_growth_ai_automation_readiness(" in route_source + assert "'ai_automation_readiness': ai_automation_readiness" in route_source assert "'controlled_apply_compact_readback': controlled_apply_compact_readback" in route_source assert "'artifact_retention_policy': artifact_retention_policy" in route_source assert "'ai_automation_first_viewport': {" in route_source + assert "'automation_pipeline': automation_pipeline" in route_source assert "pchome_growth_command_center=pchome_growth_command_center" in route_source assert "fetch_competitor_review_queue" in route_source assert "fetch_competitor_review_queue_page" in route_source @@ -393,7 +397,7 @@ def test_dashboard_v2_is_production_default_and_uses_real_dashboard_data(): assert "clear_competitor_intel_cache()" in route_source assert "_extract_match_diagnostic_reasons" in route_source assert "妝效質地不同" in route_source - assert "多款任選待確認" in route_source + assert "多款任選待決策" in route_source assert "MockRecord" not in route_source assert "{% for item in items %}" in dashboard assert "PChome 業績成長作戰台" in dashboard @@ -403,10 +407,21 @@ def test_dashboard_v2_is_production_default_and_uses_real_dashboard_data(): assert "高業績商品作戰清單" in dashboard assert "業績 × MOMO 價格 × 下一步" in dashboard assert "growth.mapping_rate" in dashboard + assert "growth.mapping_backlog" in dashboard + assert "商品對應缺口" in dashboard + assert "候選待確認" in dashboard + assert "自動收據" in dashboard + assert "mapping_backlog.auto_receipt_count" in dashboard + assert "AI 自動化作戰流水線" in dashboard + assert "growth.automation_pipeline" in dashboard assert "找同款 · 決策 · 證據 · 落地 · 回讀 · 留存" in dashboard + assert "AI 主流程" in dashboard assert "growth.ai_automation_first_viewport" in dashboard assert "ai_first_viewport.title" in dashboard assert "ai_first_viewport.metric" in dashboard + assert "primary_human_gate_count" in dashboard + assert "ai_exception_count" in dashboard + assert "主流程人工" not in dashboard assert "growth.top_opportunities" in dashboard assert "比價監控總覽" in dashboard assert "決策支援覆蓋率" in dashboard @@ -427,6 +442,12 @@ def test_dashboard_v2_is_production_default_and_uses_real_dashboard_data(): assert "身份採用待核" in dashboard assert "grid-template-columns: repeat(5, minmax(0, 1fr))" in dashboard_css assert ".growth-command-center" in dashboard_css + assert ".growth-mapping-backlog" in dashboard_css + assert ".growth-mapping-metric.is-needs" in dashboard_css + assert ".growth-mapping-metric.is-auto" in dashboard_css + assert ".growth-automation-panel" in dashboard_css + assert ".growth-automation-status" in dashboard_css + assert ".growth-automation-step.is-warning" in dashboard_css assert ".growth-strategy-grid" in dashboard_css assert ".growth-opportunity-table" in dashboard_css assert "{% if review_total_is_estimated %}約 {% endif %}" in dashboard @@ -440,14 +461,14 @@ def test_dashboard_v2_is_production_default_and_uses_real_dashboard_data(): assert "'legacy_low_score'" in route_source assert "dashboard-review-segments" in dashboard assert "data-pchome-review-action" in dashboard - assert "採用同款" in dashboard - assert "否決候選" in dashboard - assert "標記單位價" in dashboard - assert "補搜尋" in dashboard - assert "覆核建議：" in dashboard + assert "AI 採用同款" in dashboard + assert "AI 排除候選" in dashboard + assert "AI 標記單位價" in dashboard + assert "AI 補搜尋" in dashboard + assert "AI 建議：" in dashboard assert "review.catalog_review_guidance.action_hint" in dashboard assert 'data-review-action="needs_research"' in dashboard - assert "人工閉環" in route_source + assert "AI 例外閉環" in route_source assert "AI 挑品清單" in dashboard assert "比價覆核隊列" in dashboard assert "下一步" in dashboard @@ -534,7 +555,7 @@ def test_pchome_review_export_and_diagnostics_use_real_queue_data(): assert "商品 / MOMO" in dashboard assert "PChome 待確認商品" in dashboard assert "開 PChome 待確認商品" in dashboard - assert "確認採用這筆 PChome 商品為正式同款配對？" in dashboard + assert "執行 AI 決策：採用這筆 PChome 商品為正式同款配對？" in dashboard assert "PChome 候選" not in dashboard assert "覆核判讀" in dashboard assert "dashboard-review-workbench-row" in dashboard @@ -551,7 +572,9 @@ def test_pchome_review_export_and_diagnostics_use_real_queue_data(): assert 'title="決策追蹤"' in dashboard assert "優先 {{ envelope.severity" in dashboard assert "證據完整" in dashboard - assert "需人工" in dashboard + assert "AI 待決策" in dashboard + assert "需人工" not in dashboard + assert "人工覆核" not in dashboard assert "review.decision_envelope" in dashboard assert "dashboard-review-envelope" in dashboard assert "review.diagnostic_reasons" in dashboard @@ -627,6 +650,41 @@ def test_ai_intelligence_uses_v2_shell_and_real_runtime_apis(): assert "price_basis_label" in template assert "商品總價" in template assert "/api/ai/pchome-growth/opportunities" in template + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/operator-preview')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/direct-mapping-auto-search-package')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/direct-mapping-candidate-decision-package')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/ai-automation-readiness')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/evidence-enrichment-preview')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/evidence-source-preview')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/evidence-fetch-gate')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/evidence-merge-preview')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-receipt-gate')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-persistence-gate')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-schema-migration-preview')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-migration-file-preview')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-apply-readiness-closeout')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-migration-file-generation-request')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-migration-apply-gate-preview')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-request-gate-preview')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-execution-preflight')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-package')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-verifier-artifact-preview')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-final-handoff-package')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-controlled-dry-run-shell-preview')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-controlled-dry-run-shell-closeout')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-request-intake')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-request-closeout')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout')" in route_source + assert "@ai_bp.route('/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard')" in route_source + assert "summarize_pchome_mapping_backlog(payload)" in route_source + assert "build_pchome_mapping_operator_preview(payload, batch_size=batch_size)" in route_source + assert "build_pchome_evidence_enrichment_preview(payload, batch_size=batch_size)" in route_source assert "最近處理紀錄" in template assert "處理紀錄" in template assert "作戰建議紀錄" not in template diff --git a/tests/test_market_intel_skeleton.py b/tests/test_market_intel_skeleton.py index d2b9db9..c019fc9 100644 --- a/tests/test_market_intel_skeleton.py +++ b/tests/test_market_intel_skeleton.py @@ -882,7 +882,7 @@ def test_market_intel_preview_template_uses_safe_fetch_false_endpoint(): assert "市場情報尚未進入正式決策" in template assert "資料狀態" in template assert "來源規格" in template - assert "手動整理" in template + assert "AI 受控整理" in template assert "data-market-intel-preview" not in template assert "data-market-intel-writer" not in template assert "/api/market_intel/" not in template diff --git a/tests/test_migration_metadata_coverage.py b/tests/test_migration_metadata_coverage.py index c9e2a95..64cbb6e 100644 --- a/tests/test_migration_metadata_coverage.py +++ b/tests/test_migration_metadata_coverage.py @@ -74,6 +74,21 @@ def test_ai_calls_provider_check_accepts_ollama_other_telemetry_bucket(): assert "NOT VALID" in migration +def test_pchome_auto_policy_receipt_migration_is_additive_only(): + path = ROOT / "migrations" / "045_pchome_auto_policy_evidence_receipts.sql" + + assert path.exists() + migration = path.read_text(encoding="utf-8") + assert "Migration 045: PChome auto-policy evidence receipts" in migration + assert "CREATE TABLE IF NOT EXISTS external_offer_evidence_receipts" in migration + assert "evidence_delta_json JSONB NOT NULL DEFAULT '{}'::jsonb" in migration + assert "GRANT ALL PRIVILEGES ON external_offer_evidence_receipts TO momo" in migration + assert "DROP " not in migration.upper() + assert "TRUNCATE " not in migration.upper() + assert "DELETE " not in migration.upper() + assert "ALTER TABLE EXTERNAL_OFFERS" not in migration.upper() + + def test_rag_embedding_signature_migration_covers_query_and_learning_tables(): migration = (ROOT / "migrations" / "034_add_embedding_signature_to_rag_tables.sql").read_text(encoding="utf-8") diff --git a/tests/test_momo_crawler_targeted_search.py b/tests/test_momo_crawler_targeted_search.py index 5056095..5425086 100644 --- a/tests/test_momo_crawler_targeted_search.py +++ b/tests/test_momo_crawler_targeted_search.py @@ -178,7 +178,7 @@ def test_targeted_momo_search_keeps_identity_review_candidates_manual(monkeypatc ) assert success is True - assert "需人工確認 1 筆" in message + assert "需 AI 自動驗證確認 1 筆" in message assert products[0]["auto_compare_type"] == "manual_review" assert products[0]["can_auto_compare"] is False assert products[0]["target_price_basis"] == "none" diff --git a/tests/test_nemotron_decision_envelope.py b/tests/test_nemotron_decision_envelope.py index 90f59a8..f4103de 100644 --- a/tests/test_nemotron_decision_envelope.py +++ b/tests/test_nemotron_decision_envelope.py @@ -37,7 +37,8 @@ def test_price_decision_envelope_has_hilt_guardrails(): assert envelope["guardrails"]["match_type"] == "exact" assert envelope["guardrails"]["price_basis"] == "total_price" assert envelope["guardrails"]["alert_tier"] == "price_alert_exact" - assert envelope["recommended_action"]["requires_hitl"] is True + assert envelope["recommended_action"]["requires_hitl"] is False + assert envelope["recommended_action"]["requires_ai_exception"] is True assert envelope["expected_impact"]["revenue_loss_7d"] == 65000 assert envelope["expected_impact"]["gap_amount"] == 201 assert envelope["expected_impact"]["momo_price"] == 1200 diff --git a/tests/test_openclaw_daily_template.py b/tests/test_openclaw_daily_template.py index 8e2122b..bdad619 100644 --- a/tests/test_openclaw_daily_template.py +++ b/tests/test_openclaw_daily_template.py @@ -158,7 +158,7 @@ class TestKPIComputation: assert "fetch_competitor_coverage" in source assert "summarize_review_decision_envelopes" in source assert "review_decision_text" in source - assert "重算待人工覆核" in source + assert "重算待 AI 決策" in source # ═══════════════════════════════════════════════════════════════════════════ diff --git a/tests/test_pchome_revenue_growth_service.py b/tests/test_pchome_revenue_growth_service.py index 59ad78b..67fcc57 100644 --- a/tests/test_pchome_revenue_growth_service.py +++ b/tests/test_pchome_revenue_growth_service.py @@ -185,7 +185,26 @@ def test_pchome_growth_opportunities_use_plain_language_and_pause_shopee_coupang assert payload["success"] is True assert payload["system_name"] == "PChome 業績成長自動化作戰系統" assert payload["source_scope"]["active_external_sources"] == ["MOMO 外部價格參考"] - assert payload["source_scope"]["paused_external_sources"] == ["蝦皮", "酷澎"] + assert { + "蝦皮", + "Lazada", + "Amazon", + "Google Merchant / Shopping", + "TikTok Shop", + "LINE 購物", + "Rakuten", + "Yahoo 購物", + "露天", + "品牌官網 / Shopify", + "Meta Commerce", + "酷澎", + }.issubset(set(payload["source_scope"]["paused_external_sources"])) + assert payload["source_scope"]["plain_note"] == "MOMO 先用；其他主流平台待接入，不進作戰清單，也不發告警。" + evidence_fields = { + field["name"] + for field in payload["source_scope"]["offer_evidence_contract"]["professional_evidence_fields"] + } + assert {"source_product_id", "product_url", "image_url", "stock_status", "quality_score"}.issubset(evidence_fields) readiness = payload["source_scope"]["source_readiness"] sources = {source["code"]: source for source in readiness["sources"]} assert sources["momo_reference"]["status_label"] == "正在使用" @@ -542,10 +561,12 @@ def test_growth_analysis_uses_actionable_price_command_panel(): assert "ga-chart-card__body--command" in template assert "ga-competitor-command" in template assert "可直接決策" in template - assert "待補 / 待確認" in template + assert "待補 / AI 例外" in template + assert "ga-competitor-evidence" in template assert "下一步" in template assert "前往今日作戰" in template assert "ga-competitor-signal" in css + assert "ga-competitor-evidence" in css assert "ga-competitor-next" in css assert ".growth-analysis-page .ga-chart-card__body--command" in css assert "比價資料品質" not in template diff --git a/tests/test_telegram_triaged_alert_format.py b/tests/test_telegram_triaged_alert_format.py index 73a50cc..4486cc0 100644 --- a/tests/test_telegram_triaged_alert_format.py +++ b/tests/test_telegram_triaged_alert_format.py @@ -77,10 +77,10 @@ def test_ea_escalation_uses_structured_incident_brief(): base_event={ "event_type": "ea_escalation", "title": "🐘 EA 升級審核 · 價格下滑警報", - "summary": "自主決策信心度 0.82 低於門檻，需人工批准", + "summary": "自主決策信心度 0.82 低於門檻，需 AI 例外決策", "id": "ea_review_test", }, - tier_label="🐘 Elephant Alpha · L3 HITL", + tier_label="🐘 Elephant Alpha · L3 AI 例外決策", ai_summary=( "分析顯示 5 個代表性 SKU 的價格差異分別為 16.7%~38.3%，" "且每件價差至多 370 元。" @@ -91,13 +91,13 @@ def test_ea_escalation_uses_structured_incident_brief(): "MOMO $300 vs PChome $250 (+16.7%)｜" "每件價差 NT$ 50｜" "證據：高信心同款 / 總價可比 / 可直接價格告警 / score 0.86｜" - "建議人工確認 PChome identity_v2 後評估跟價或促銷｜" + "建議 AI 自動驗證確認 PChome identity_v2 後評估跟價或促銷｜" "PChome DABC53-A9009OEF", "[3518670] L'Occitane 歐舒丹 官方直營 乳油木｜" "MOMO $1,220 vs PChome $850 (+30.3%)｜" "每件價差 NT$ 370｜" "證據：高信心同款 / 總價可比 / 可直接價格告警 / score 0.91｜" - "建議人工確認 PChome identity_v2 後評估跟價或促銷｜" + "建議 AI 自動驗證確認 PChome identity_v2 後評估跟價或促銷｜" "PChome DDADKS-A900HIG5Y", ], ) @@ -125,7 +125,7 @@ def test_ea_escalation_generic_actions_do_not_render_as_sku_cards(): "summary": "低信心且缺少可格式化的具體行動", "id": "ea_review_generic", }, - tier_label="🐘 Elephant Alpha · L3 HITL", + tier_label="🐘 Elephant Alpha · L3 AI 例外決策", ai_summary="已隱藏 LLM plan 文字，避免把推測當成事實。", ai_cause="觸發類型：程式碼異常偵測 | 信心度：0.62 | 缺少可直接審核的實證資料", ai_actions=[ @@ -145,7 +145,7 @@ def test_ea_escalation_renders_decision_envelope_and_uses_decision_id_callback() base_event={ "event_type": "ea_escalation", "title": "🐘 EA 升級審核 · 資源壓力治理", - "summary": "自主決策信心度 0.82 低於門檻，需人工批准", + "summary": "自主決策信心度 0.82 低於門檻，需 AI 例外決策", "decision_envelope": { "decision_id": "ea_resource_pressure_1942", "decision_type": "resource_optimization", @@ -166,18 +166,19 @@ def test_ea_escalation_renders_decision_envelope_and_uses_decision_id_callback() }, ], "recommended_action": { - "action": "human_review_backlog_triage", + "action": "ai_exception_backlog_triage", "owner": "ops", - "requires_hitl": True, + "requires_hitl": False, + "requires_ai_exception": True, }, "guardrails": { "can_auto_execute": False, - "blocked_reason": "resource_optimization requires HITL", + "blocked_reason": "resource_optimization requires AI exception decision", "data_quality": "complete", }, }, }, - tier_label="🐘 Elephant Alpha · L3 HITL", + tier_label="🐘 Elephant Alpha · L3 AI 例外決策", ai_summary="只採用 action_plans 與 CPU 實測值。", ai_cause="觸發類型：資源壓力治理 | 信心度：0.86", ai_actions=["檢查 priority <= 2 的 action_plans"], @@ -187,7 +188,7 @@ def test_ea_escalation_renders_decision_envelope_and_uses_decision_id_callback() assert "類型：resource_optimization" in msg assert "資料品質：complete　自動執行：不允許" in msg assert "SKU：action_plans" in msg - assert "human_review_backlog_triage" in msg + assert "ai_exception_backlog_triage" in msg assert keyboard["inline_keyboard"][0][0]["callback_data"] == "momo:eig:ea_resource_pressure_1942" @@ -225,10 +226,11 @@ def test_triaged_alert_renders_decision_envelope_contract(): }, ], "recommended_action": { - "action": "human_review", + "action": "ai_exception_decision", "owner": "ops", "deadline": "2026-05-24T18:00:00+08:00", - "requires_hitl": True, + "requires_hitl": False, + "requires_ai_exception": True, }, "expected_impact": { "revenue_loss_7d": 42000, @@ -237,7 +239,7 @@ def test_triaged_alert_renders_decision_envelope_contract(): }, "guardrails": { "can_auto_execute": False, - "blocked_reason": "price adjustment requires HITL", + "blocked_reason": "price adjustment requires AI exception decision", "data_quality": "complete", }, "trace": { @@ -249,7 +251,7 @@ def test_triaged_alert_renders_decision_envelope_contract(): }, }, tier_label="Hermes · P1", - ai_summary="建議進人工價格審核。", + ai_summary="建議進 AI 例外價格決策。", ) assert "🧭 決策信封" in msg @@ -258,7 +260,7 @@ def test_triaged_alert_renders_decision_envelope_contract(): assert "信心度：86%" in msg assert "資料品質：complete" in msg assert "自動執行：不允許" in msg - assert "邊界：price adjustment requires HITL" in msg + assert "邊界：price adjustment requires AI exception decision" in msg assert "標的" in msg assert "PChome" in msg assert "📊 價格證據" in msg @@ -266,8 +268,8 @@ def test_triaged_alert_renders_decision_envelope_contract(): assert "🧩 比對證據" in msg assert "Match：0.91" in msg assert "identity_v2 + price_alert_exact" in msg - assert "✅ 人工下一步" in msg - assert "動作：human_review" in msg + assert "✅ AI 例外下一步" in msg + assert "動作：ai_exception_decision" in msg assert "ai_call_id=123" in msg assert keyboard["inline_keyboard"][0][0]["callback_data"] == "momo:eig:decision_env_001" diff --git a/tests/test_webcrumbs_host_data_service.py b/tests/test_webcrumbs_host_data_service.py index ff29f57..8640cd3 100644 --- a/tests/test_webcrumbs_host_data_service.py +++ b/tests/test_webcrumbs_host_data_service.py @@ -10,7 +10,7 @@ def test_webcrumbs_host_data_maps_price_alert_exact_rows(monkeypatch): lambda passed_engine, limit: [ { "sku": "SKU-REVIEW", - "name": "需人工覆核的非直接告警候選", + "name": "需 AI 例外決策的非直接告警候選", "momo_price": 999, "pchome_price": 500, "gap_pct": 99.8, @@ -70,20 +70,21 @@ def test_webcrumbs_host_data_maps_price_alert_exact_rows(monkeypatch): lambda passed_engine, limit: [ { "sku": "SKU-REVIEW", - "name": "人工覆核精華液", + "name": "AI 例外決策精華液", "decision_envelope": { "decision_id": "review_queue:SKU-REVIEW", "severity": "P2", "subject": { "sku": "SKU-REVIEW", - "name": "人工覆核精華液", + "name": "AI 例外決策精華液", "momo_price": 399, "competitor_price": 329, "competitor_product_id": "PCH-REVIEW", }, "recommended_action": { "action": "review_accept_identity", - "requires_hitl": True, + "requires_hitl": False, + "owner": "AI Agent", }, "expected_impact": {"candidate_gap_pct": 21.3}, "guardrails": { @@ -136,7 +137,9 @@ def test_webcrumbs_host_data_maps_price_alert_exact_rows(monkeypatch): assert payload["metadata"]["stale_match_count"] == 18 assert payload["metadata"]["pending_match_count"] == 612 assert payload["metadata"]["review_queue_count"] == 1 - assert payload["metadata"]["hitl_count"] == 1 + assert payload["metadata"]["hitl_count"] == 0 + assert payload["metadata"]["primary_human_gate_count"] == 0 + assert payload["metadata"]["ai_exception_count"] == 1 assert payload["metadata"]["auto_execute_blocked_count"] == 1 assert payload["metadata"]["decision_envelope_source"] == "competitor_intel_repository" assert payload["reviewDecisionBrief"]["items"][0]["sku"] == "SKU-REVIEW" @@ -188,8 +191,8 @@ def test_webcrumbs_host_data_uses_empty_state_without_risks(monkeypatch): assert payload["metadata"]["stale_match_count"] == 2 assert payload["metadata"]["pending_match_count"] == 9 assert payload["metadata"]["review_queue_count"] == 0 - assert payload["reviewDecisionBrief"]["text"] == "（目前沒有待覆核決策信封）" - assert "非同款、單位價或變體候選" in payload["aiCandidate"]["thesis"] + assert payload["reviewDecisionBrief"]["text"] == "（目前沒有待 AI 決策信封）" + assert "非同款、單位價或變體候選會進入 AI 例外決策隊列" in payload["aiCandidate"]["thesis"] def test_webcrumbs_seed_data_fallback_does_not_expose_demo_values(monkeypatch): diff --git a/web/static/css/page-dashboard-v2.css b/web/static/css/page-dashboard-v2.css index a9aef71..cdb655c 100644 --- a/web/static/css/page-dashboard-v2.css +++ b/web/static/css/page-dashboard-v2.css @@ -84,6 +84,7 @@ .growth-command-kpi, .growth-priority-panel, .growth-strategy-panel, + .growth-automation-panel, .growth-opportunity-panel { min-width: 0; background: var(--momo-bg-surface); @@ -170,12 +171,193 @@ min-width: 0; } + .growth-mapping-backlog { + display: grid; + grid-template-columns: minmax(180px, 0.62fr) minmax(0, 1fr); + gap: 10px; + min-width: 0; + padding: 12px; + background: var(--momo-bg-surface); + border: 1px solid var(--momo-border-light); + border-radius: 8px; + box-shadow: var(--momo-shadow-soft); + } + + .growth-mapping-backlog-head { + display: grid; + align-content: center; + gap: 6px; + min-width: 0; + padding-right: 10px; + border-right: 1px solid var(--momo-border-light); + } + + .growth-mapping-backlog-head strong { + color: var(--momo-text-primary); + font-size: 2rem; + font-weight: 900; + line-height: 1; + } + + .growth-mapping-backlog-head em, + .growth-mapping-metric em { + color: var(--momo-text-secondary); + font-size: 11px; + font-style: normal; + font-weight: 800; + line-height: 1.35; + } + + .growth-mapping-backlog-grid { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 8px; + min-width: 0; + } + + .growth-mapping-metric { + display: grid; + align-content: center; + gap: 5px; + min-width: 0; + min-height: 82px; + padding: 10px 12px; + background: color-mix(in srgb, var(--momo-bg-paper) 78%, transparent); + border: 1px solid var(--momo-border-light); + border-radius: 8px; + } + + .growth-mapping-metric span { + color: var(--momo-text-secondary); + font-size: 12px; + font-weight: 900; + } + + .growth-mapping-metric strong { + color: var(--momo-text-primary); + font-size: 1.7rem; + font-weight: 900; + line-height: 1; + } + + .growth-mapping-metric.is-needs { + border-color: rgba(188, 75, 49, 0.28); + background: rgba(255, 244, 239, 0.72); + } + + .growth-mapping-metric.is-review { + border-color: rgba(210, 158, 58, 0.34); + background: rgba(255, 248, 231, 0.72); + } + + .growth-mapping-metric.is-ready { + border-color: rgba(48, 133, 94, 0.24); + background: rgba(235, 248, 241, 0.72); + } + + .growth-mapping-metric.is-auto { + border-color: rgba(62, 112, 174, 0.26); + background: rgba(236, 244, 255, 0.78); + } + .growth-priority-panel, .growth-strategy-panel, + .growth-automation-panel, .growth-opportunity-panel { padding: 14px; } + .growth-automation-steps { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 8px; + min-width: 0; + } + + .growth-automation-status { + display: grid; + grid-template-columns: auto minmax(0, 1fr) auto; + gap: 10px; + align-items: center; + min-width: 0; + margin-bottom: 8px; + padding: 10px 12px; + background: rgba(235, 248, 241, 0.72); + border: 1px solid rgba(48, 133, 94, 0.24); + border-radius: 8px; + } + + .growth-automation-status span { + color: var(--momo-text-tertiary); + font-size: 10px; + font-weight: 900; + letter-spacing: 0.06em; + } + + .growth-automation-status strong { + overflow-wrap: anywhere; + color: var(--momo-success); + font-size: 14px; + font-weight: 900; + } + + .growth-automation-status em { + color: var(--momo-text-secondary); + font-size: 11px; + font-style: normal; + font-weight: 800; + text-align: right; + } + + .growth-automation-step { + display: grid; + align-content: start; + gap: 6px; + min-width: 0; + min-height: 96px; + padding: 12px; + background: color-mix(in srgb, var(--momo-bg-paper) 78%, transparent); + border: 1px solid var(--momo-border-light); + border-radius: 8px; + } + + .growth-automation-step span { + color: var(--momo-text-secondary); + font-size: 12px; + font-weight: 900; + } + + .growth-automation-step strong { + overflow-wrap: anywhere; + color: var(--momo-text-primary); + font-size: 1.55rem; + font-weight: 900; + line-height: 1; + } + + .growth-automation-step em { + color: var(--momo-text-secondary); + font-size: 11px; + font-style: normal; + font-weight: 800; + line-height: 1.35; + } + + .growth-automation-step.is-danger { + border-color: rgba(188, 75, 49, 0.28); + background: rgba(255, 244, 239, 0.72); + } + + .growth-automation-step.is-warning { + border-color: rgba(210, 158, 58, 0.34); + background: rgba(255, 248, 231, 0.72); + } + + .growth-automation-step.is-success { + border-color: rgba(48, 133, 94, 0.24); + background: rgba(235, 248, 241, 0.72); + } + .growth-panel-head { display: flex; justify-content: space-between; @@ -2114,10 +2296,25 @@ grid-template-columns: 1fr; } + .growth-mapping-backlog { + grid-template-columns: 1fr; + } + + .growth-mapping-backlog-head { + padding-right: 0; + padding-bottom: 10px; + border-right: 0; + border-bottom: 1px solid var(--momo-border-light); + } + .growth-strategy-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .growth-automation-steps { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + .dashboard-kpi-grid { grid-template-columns: repeat(3, minmax(0, 1fr)); } @@ -2150,6 +2347,8 @@ @media (max-width: 640px) { .growth-command-kpis, + .growth-mapping-backlog-grid, + .growth-automation-steps, .growth-strategy-grid, .dashboard-kpi-grid, .dashboard-ai-summary-grid { @@ -2159,6 +2358,7 @@ .growth-command-head, .growth-priority-panel, .growth-strategy-panel, + .growth-automation-panel, .growth-opportunity-panel { padding: 12px; } @@ -2181,6 +2381,14 @@ white-space: normal; } + .growth-automation-status { + grid-template-columns: 1fr; + } + + .growth-automation-status em { + text-align: left; + } + .dashboard-focus-grid { grid-template-columns: 1fr; } diff --git a/web/static/css/page-growth-bem.css b/web/static/css/page-growth-bem.css index 5b6168c..1e3fa9e 100644 --- a/web/static/css/page-growth-bem.css +++ b/web/static/css/page-growth-bem.css @@ -420,6 +420,24 @@ overflow-wrap: anywhere; } +.growth-analysis-page .ga-competitor-evidence { + grid-column: 1 / -1; + display: flex; + flex-wrap: wrap; + gap: 6px; + min-width: 0; +} + +.growth-analysis-page .ga-competitor-evidence span { + border: 1px solid var(--momo-border-muted, rgba(31, 41, 55, 0.12)); + border-radius: 999px; + padding: 4px 8px; + background: var(--momo-surface-soft, rgba(255, 255, 255, 0.72)); + color: var(--momo-text-secondary); + font-size: var(--momo-text-caption, 12px); + font-weight: 800; +} + .growth-analysis-page .ga-competitor-next { grid-column: 1 / -1; min-width: 0; From df5593c1fb9d16245fdd185a0ee3dbfcaafb2238 Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 22:09:00 +0800 Subject: [PATCH 02/75] feat(crawler): expand pixelrag marketplace automation --- .gitignore | 1 + docs/guides/browse_sh_crawler_playbook.md | 38 +- routes/system_public_routes.py | 16 + .../ops/capture_pixelrag_visual_evidence.js | 6 + .../ops/capture_pixelrag_visual_evidence.py | 6 + .../report_pixelrag_crawler_integration.py | 45 ++ services/pchome_crawler.py | 83 ++++ .../pixelrag_crawler_integration_service.py | 439 +++++++++++++++++- tests/test_pchome_crawler_search.py | 26 ++ ...st_pixelrag_crawler_integration_service.py | 131 ++++++ 10 files changed, 784 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 8a8d457..5126dbc 100644 --- a/.gitignore +++ b/.gitignore @@ -91,6 +91,7 @@ data/excel_exports/ data/*_cache/ data/ai_automation_smoke_history.jsonl data/ai_automation/sitewide_visual_qa_latest.json +runtime_artifacts/ # 上傳檔案 web/static/uploads/ diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index ba291ea..39f1bc6 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -54,7 +54,7 @@ python scripts/tools/browse_sh_probe.py -- screenshot ## PixelRAG-style 視覺證據 fallback -2026-07-09 評估 PixelRAG 後，結論是「可導入，但不可直接取代正式爬蟲」。PixelRAG 的核心價值是把渲染後頁面截圖切成 tiles，讓 AI 讀到 HTML parser 可能丟失的視覺結構；本專案第一階段只採用這個視覺證據思路，不直接拉外部 runtime、不用外部 embedding API、不把像素結果寫入正式價格表。 +2026-07-09 評估 PixelRAG 後，結論是「可導入，而且不應只限 MOMO/PChome」。PixelRAG 的核心價值是把渲染後頁面截圖切成 tiles，讓 AI 讀到 HTML parser 可能丟失的視覺結構；本專案第一階段採用這個視覺證據思路，擴充成多電商 public marketplace visual evidence layer，不直接拉外部 runtime、不用外部 embedding API、不把像素結果寫入正式價格表。 導入順序： @@ -65,11 +65,44 @@ python scripts/tools/browse_sh_probe.py -- screenshot 5. Phase 4 若需要索引，優先設計 pgvector-compatible evidence metadata；FAISS 只能先當本地研究/ADR 候選，不直接進 production。 6. Phase 5 才談 crawler fusion；正式 `competitor_prices` / `competitor_price_history` 寫入仍需 matcher replay/canary 證據。 +多電商優先順序： + +| Priority | Platform | PixelRAG 用途 | 邊界 | +|---|---|---|---| +| P0 | MOMO / PChome | 既有 structured crawler 失敗時自動產生 visual manifest queue | 正式價格仍以既有 parser/API/matcher 為準 | +| P1 | Shopee Taiwan / Coupang Taiwan | 抓公共搜尋頁商品卡、促銷 badge、店家/配送/評分視覺證據 | 不登入、不碰購物車、不繞過 anti-bot | +| P2 | Yahoo Shopping / ETMall | 補促銷券、快速到貨、活動價格等視覺訊號 | 先做 controlled probe，再決定 parser | +| P3 | friDay / Rakuten Taiwan | 先建立 probe profile 與 search manifest | URL pattern 需 runtime receipt 驗證後再排程 | + +已支援的 platform code： + +```text +momo +pchome +shopee_tw +coupang_tw +yahoo_shopping_tw +etmall_tw +friday_tw +rakuten_tw +``` + +自動觸發： + +- `PChomeCrawler.fetch_region_page` 找不到商品 ID 時會 queue PixelRAG manifest。 +- `PChomeCrawler.fetch_product_details` API 回傳但無可解析商品時會 queue 商品頁 manifest。 +- `PChomeCrawler.search_products` 搜尋無商品 ID 或 request error 時會 queue 公開搜尋頁 manifest。 +- queue 預設路徑：正式 app 容器 `/app/data/ai_automation/pixelrag_manifest_queue`，本機為 `runtime_artifacts/pixelrag_manifest_queue`。 +- queue 只寫 artifact，不寫 DB、不寫正式價格、不讀 secret；capture worker 後續消費 manifest 產生 screenshot/tile receipt。 + 機器可讀評估： ```bash python scripts/ops/report_pixelrag_crawler_integration.py python scripts/ops/report_pixelrag_crawler_integration.py --platform momo +python scripts/ops/report_pixelrag_crawler_integration.py --ecommerce-expansion-plan +python scripts/ops/report_pixelrag_crawler_integration.py --platform shopee_tw --marketplace-keyword "防曬乳" +python scripts/ops/report_pixelrag_crawler_integration.py --platform coupang_tw --marketplace-keyword "iphone" python scripts/ops/report_pixelrag_crawler_integration.py --capability ollama_multimodal_ready --capability pgvector_visual_ready python scripts/ops/report_pixelrag_crawler_integration.py \ --platform momo \ @@ -84,6 +117,9 @@ python scripts/ops/report_pixelrag_crawler_integration.py \ ```text /api/ai-automation/pixelrag-crawler-integration?platform=momo +/api/ai-automation/pixelrag-crawler-integration?expansion_plan=true +/api/ai-automation/pixelrag-crawler-integration?platform=shopee_tw&marketplace_keyword=%E9%98%B2%E6%9B%AC%E4%B9%B3 +/api/ai-automation/pixelrag-crawler-integration?platform=coupang_tw&marketplace_keyword=iphone /api/ai-automation/pixelrag-crawler-integration?platform=pchome&manifest_url=https://24h.pchome.com.tw/prod/TEST-000000001&crawler=PChomeCrawler.search_products&trigger_reason=parser_empty /api/ai-automation/pixelrag-visual-evidence-readback?platform=pchome&manifest_id=4a93e95e5afb414bc8c3 ``` diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 82d4fe9..030f885 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -640,10 +640,26 @@ def ai_automation_pixelrag_crawler_integration_api(): """Read-only PixelRAG-style crawler visual evidence assessment.""" from services.pixelrag_crawler_integration_service import ( build_pixelrag_crawler_integration_assessment, + build_pixelrag_ecommerce_expansion_plan, + build_pixelrag_marketplace_search_manifest, build_pixelrag_visual_evidence_manifest, ) platforms = request.args.getlist('platform') or None + if str(request.args.get('expansion_plan') or '').lower() in {'1', 'true', 'yes'}: + return jsonify(build_pixelrag_ecommerce_expansion_plan( + target_platforms=tuple(platforms) if platforms else None, + )) + marketplace_keyword = str(request.args.get('marketplace_keyword') or '').strip() + if marketplace_keyword: + platform = (platforms or ['shopee_tw'])[0] + return jsonify(build_pixelrag_marketplace_search_manifest( + platform=platform, + keyword=marketplace_keyword, + crawler=str(request.args.get('crawler') or 'PixelRAGMarketplaceSearch.visual_fallback'), + trigger_reason=str(request.args.get('trigger_reason') or 'marketplace_search_visual_scan'), + evidence_intent=str(request.args.get('evidence_intent') or 'collect_public_marketplace_offer_cards'), + )) manifest_url = str(request.args.get('manifest_url') or '').strip() if manifest_url: platform = (platforms or ['momo'])[0] diff --git a/scripts/ops/capture_pixelrag_visual_evidence.js b/scripts/ops/capture_pixelrag_visual_evidence.js index 8973a57..beea5ae 100644 --- a/scripts/ops/capture_pixelrag_visual_evidence.js +++ b/scripts/ops/capture_pixelrag_visual_evidence.js @@ -21,6 +21,12 @@ const DEFAULT_MAX_TILES = 80; const ALLOWED_PLATFORM_HOSTS = { momo: new Set(['m.momoshop.com.tw', 'www.momoshop.com.tw']), pchome: new Set(['24h.pchome.com.tw', 'ecshweb.pchome.com.tw', 'ecapi-cdn.pchome.com.tw']), + shopee_tw: new Set(['shopee.tw', 'mall.shopee.tw']), + coupang_tw: new Set(['www.tw.coupang.com', 'tw.coupang.com']), + yahoo_shopping_tw: new Set(['tw.buy.yahoo.com']), + etmall_tw: new Set(['www.etmall.com.tw', 'etmall.com.tw']), + friday_tw: new Set(['shopping.friday.tw', 'ec-m.shopping.friday.tw', 'ec-w.shopping.friday.tw']), + rakuten_tw: new Set(['www.rakuten.com.tw', 'rakuten.com.tw']), market_intel: new Set(), external_market: new Set(), }; diff --git a/scripts/ops/capture_pixelrag_visual_evidence.py b/scripts/ops/capture_pixelrag_visual_evidence.py index 6263cd2..f9c2a1f 100644 --- a/scripts/ops/capture_pixelrag_visual_evidence.py +++ b/scripts/ops/capture_pixelrag_visual_evidence.py @@ -26,6 +26,12 @@ DEFAULT_MAX_TILES = 80 ALLOWED_PLATFORM_HOSTS = { "momo": {"m.momoshop.com.tw", "www.momoshop.com.tw"}, "pchome": {"24h.pchome.com.tw", "ecshweb.pchome.com.tw", "ecapi-cdn.pchome.com.tw"}, + "shopee_tw": {"shopee.tw", "mall.shopee.tw"}, + "coupang_tw": {"www.tw.coupang.com", "tw.coupang.com"}, + "yahoo_shopping_tw": {"tw.buy.yahoo.com"}, + "etmall_tw": {"www.etmall.com.tw", "etmall.com.tw"}, + "friday_tw": {"shopping.friday.tw", "ec-m.shopping.friday.tw", "ec-w.shopping.friday.tw"}, + "rakuten_tw": {"www.rakuten.com.tw", "rakuten.com.tw"}, "market_intel": set(), "external_market": set(), } diff --git a/scripts/ops/report_pixelrag_crawler_integration.py b/scripts/ops/report_pixelrag_crawler_integration.py index 0a9f226..ef95edd 100644 --- a/scripts/ops/report_pixelrag_crawler_integration.py +++ b/scripts/ops/report_pixelrag_crawler_integration.py @@ -15,6 +15,8 @@ if str(ROOT) not in sys.path: from services.pixelrag_crawler_integration_service import ( # noqa: E402 build_pixelrag_crawler_integration_assessment, + build_pixelrag_ecommerce_expansion_plan, + build_pixelrag_marketplace_search_manifest, build_pixelrag_visual_evidence_manifest, ) @@ -62,6 +64,12 @@ def main() -> int: help="覆寫 readiness 假設，用於模擬下一階段條件。", ) parser.add_argument("--manifest-url", help="輸出單一 URL 的視覺證據 manifest。") + parser.add_argument("--marketplace-keyword", help="依平台與關鍵字輸出多電商視覺搜尋 manifest。") + parser.add_argument( + "--ecommerce-expansion-plan", + action="store_true", + help="輸出 MOMO/PChome 以外的 PixelRAG 多電商擴充計畫。", + ) parser.add_argument("--crawler", default="crawler_diagnostics", help="manifest 來源 crawler 名稱。") parser.add_argument( "--trigger-reason", @@ -90,6 +98,43 @@ def main() -> int: ) args = parser.parse_args() + if args.ecommerce_expansion_plan: + payload = build_pixelrag_ecommerce_expansion_plan( + target_platforms=tuple(args.platforms) if args.platforms else None, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + if args.marketplace_keyword: + platform = (args.platforms or ["shopee_tw"])[0] + crawler = ( + "PixelRAGMarketplaceSearch.visual_fallback" + if args.crawler == "crawler_diagnostics" + else args.crawler + ) + trigger_reason = ( + "marketplace_search_visual_scan" + if args.trigger_reason == "parser_empty_or_low_confidence" + else args.trigger_reason + ) + evidence_intent = ( + "collect_public_marketplace_offer_cards" + if args.evidence_intent == "recover_visual_offer_evidence" + else args.evidence_intent + ) + payload = build_pixelrag_marketplace_search_manifest( + platform=platform, + keyword=args.marketplace_keyword, + crawler=crawler, + trigger_reason=trigger_reason, + evidence_intent=evidence_intent, + viewport=args.viewport, + page_size=args.page_size, + tile_size=args.tile_size, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + if args.manifest_url: platform = (args.platforms or ["momo"])[0] payload = build_pixelrag_visual_evidence_manifest( diff --git a/services/pchome_crawler.py b/services/pchome_crawler.py index bcd4f8a..8b2158f 100644 --- a/services/pchome_crawler.py +++ b/services/pchome_crawler.py @@ -18,6 +18,7 @@ import logging from typing import List, Dict, Optional, Tuple from dataclasses import dataclass, asdict from datetime import datetime +from urllib.parse import urlencode import requests @@ -139,6 +140,7 @@ class PChomeCrawler: self.session = requests.Session() self.session.headers.update(self.DEFAULT_HEADERS) self._last_request_time = 0 + self.last_visual_evidence_action = None def _rate_limit(self): """速率限制""" @@ -188,6 +190,58 @@ class PChomeCrawler: raise last_error raise requests.RequestException(f"GET failed: {url}") + def _public_search_url(self, keyword: str) -> str: + return f"{self.BASE_URL}/search/?{urlencode({'q': str(keyword or '').strip()})}" + + def _emit_visual_evidence_action( + self, + *, + url: str, + crawler: str, + failure_reason: str, + parsed_item_count: int = 0, + missing_fields: Tuple[str, ...] = ("product_id", "title", "price"), + ) -> Optional[dict]: + """Queue a PixelRAG visual manifest for parser-empty crawler cases.""" + try: + from services.pixelrag_crawler_integration_service import ( + build_pixelrag_crawler_failure_action, + ) + + action = build_pixelrag_crawler_failure_action( + url=url, + platform="pchome", + crawler=crawler, + parser_success=False, + parsed_item_count=parsed_item_count, + missing_fields=missing_fields, + failure_reason=failure_reason, + persist_manifest=True, + ) + self.last_visual_evidence_action = action + if action.get("success"): + logger.info( + "[PChome] PixelRAG visual evidence manifest queued | crawler=%s status=%s path=%s", + crawler, + action.get("status"), + ((action.get("persistence") or {}).get("path") or ""), + ) + else: + logger.warning( + "[PChome] PixelRAG visual evidence action failed | crawler=%s status=%s", + crawler, + action.get("status"), + ) + return action + except Exception as exc: + logger.warning("[PChome] PixelRAG visual evidence action skipped: %s", exc) + self.last_visual_evidence_action = { + "success": False, + "status": "exception", + "error": str(exc)[:300], + } + return self.last_visual_evidence_action + def _normalize_product_id(self, product_id: str) -> str: """ 正規化商品 ID 格式 @@ -248,11 +302,22 @@ class PChomeCrawler: product_ids = self._extract_product_ids_from_html(response.text) logger.info(f"從 {url} 取得 {len(product_ids)} 個商品 ID") + if not product_ids: + self._emit_visual_evidence_action( + url=url, + crawler="PChomeCrawler.fetch_region_page", + failure_reason="HTML parser found no product IDs in region page", + ) return True, f"成功取得 {len(product_ids)} 個商品", product_ids except requests.RequestException as e: logger.error(f"爬取 {url} 失敗: {e}") + self._emit_visual_evidence_action( + url=url, + crawler="PChomeCrawler.fetch_region_page", + failure_reason=f"request_error: {e}", + ) return False, f"請求失敗: {str(e)}", [] def fetch_product_details(self, product_ids: List[str], batch_size: int = 20) -> Tuple[bool, str, List[PChomeProduct]]: @@ -327,6 +392,14 @@ class PChomeCrawler: message = f"成功取得 {len(all_products)} 個商品資料" if failed_count > 0: message += f"，{failed_count} 個失敗" + if product_ids and not all_products: + first_id = self._normalize_product_id(str(product_ids[0])) + self._emit_visual_evidence_action( + url=f"{self.BASE_URL}/prod/{first_id}", + crawler="PChomeCrawler.fetch_product_details", + failure_reason=message, + missing_fields=("title", "price", "product_id", "image_url"), + ) return len(all_products) > 0, message, all_products @@ -487,6 +560,11 @@ class PChomeCrawler: break if not product_ids: + self._emit_visual_evidence_action( + url=self._public_search_url(keyword), + crawler="PChomeCrawler.search_products", + failure_reason="search API returned no product IDs", + ) return False, "沒有找到符合的商品", [] # 取得詳細資料 @@ -497,6 +575,11 @@ class PChomeCrawler: except requests.RequestException as e: logger.error(f"搜尋失敗: {e}") + self._emit_visual_evidence_action( + url=self._public_search_url(keyword), + crawler="PChomeCrawler.search_products", + failure_reason=f"request_error: {e}", + ) return False, f"搜尋失敗: {str(e)}", [] diff --git a/services/pixelrag_crawler_integration_service.py b/services/pixelrag_crawler_integration_service.py index 647e284..4f254cd 100644 --- a/services/pixelrag_crawler_integration_service.py +++ b/services/pixelrag_crawler_integration_service.py @@ -15,20 +15,116 @@ from copy import deepcopy from datetime import datetime, timezone from pathlib import Path from typing import Any, Mapping -from urllib.parse import urlparse +from urllib.parse import quote_plus, urlparse POLICY = "read_only_pixelrag_crawler_integration_assessment_v1" MANIFEST_POLICY = "read_only_pixelrag_visual_evidence_manifest_v1" READBACK_POLICY = "read_only_pixelrag_visual_evidence_readback_v1" +ECOMMERCE_EXPANSION_POLICY = "read_only_pixelrag_ecommerce_expansion_plan_v1" +CRAWLER_FAILURE_ACTION_POLICY = "read_only_pixelrag_crawler_failure_action_v1" RESULT_PHASE1_READY = "PIXELRAG_VISUAL_EVIDENCE_PHASE1_READY" RESULT_BLOCKED_NO_CAPTURE = "PIXELRAG_BLOCKED_NO_VISUAL_CAPTURE" RESULT_MANIFEST_READY = "PIXELRAG_VISUAL_EVIDENCE_MANIFEST_READY" RESULT_MANIFEST_REJECTED = "PIXELRAG_VISUAL_EVIDENCE_MANIFEST_REJECTED" -ALLOWED_PLATFORMS = ("momo", "pchome", "market_intel", "external_market") DEFAULT_VIEWPORT = {"name": "desktop-1440", "width": 1440, "height": 950} DEFAULT_TILE_SIZE = {"width": 512, "height": 512} + +ECOMMERCE_PLATFORM_PROFILES: dict[str, dict[str, Any]] = { + "momo": { + "display_name": "MOMO", + "hosts": ["m.momoshop.com.tw", "www.momoshop.com.tw"], + "search_url_template": "https://m.momoshop.com.tw/search.momo?searchKeyword={query}", + "priority": "incumbent_structured_parser_plus_visual_fallback", + "best_for": ["rendered promo blocks", "bundle/spec cards", "parser-empty search pages"], + "data_targets": ["title", "price", "bundle_spec", "promo_badge", "image"], + }, + "pchome": { + "display_name": "PChome 24h", + "hosts": ["24h.pchome.com.tw", "ecshweb.pchome.com.tw", "ecapi-cdn.pchome.com.tw"], + "search_url_template": "https://24h.pchome.com.tw/search/?q={query}", + "priority": "incumbent_api_parser_plus_visual_fallback", + "best_for": ["search result cards", "campaign badges", "rendered price cards"], + "data_targets": ["title", "price", "product_id", "campaign_badge", "stock_hint"], + }, + "shopee_tw": { + "display_name": "Shopee Taiwan", + "hosts": ["shopee.tw", "mall.shopee.tw"], + "search_url_template": "https://shopee.tw/search?keyword={query}", + "priority": "P1_marketplace_expansion", + "best_for": ["JS-heavy marketplace cards", "seller badges", "shipping/promo chips"], + "data_targets": ["title", "price", "shop", "sold_count", "rating", "shipping_badge"], + }, + "coupang_tw": { + "display_name": "Coupang Taiwan", + "hosts": ["www.tw.coupang.com", "tw.coupang.com"], + "search_url_template": "https://www.tw.coupang.com/search?q={query}", + "priority": "P1_marketplace_expansion", + "best_for": ["Rocket delivery badges", "discount panels", "spec tables"], + "data_targets": ["title", "price", "discount", "delivery_badge", "spec", "review_count"], + }, + "yahoo_shopping_tw": { + "display_name": "Yahoo Shopping Taiwan", + "hosts": ["tw.buy.yahoo.com"], + "search_url_template": "https://tw.buy.yahoo.com/search/product?p={query}", + "priority": "P2_marketplace_expansion", + "best_for": ["search cards", "campaign coupons", "rating and sales badges"], + "data_targets": ["title", "price", "coupon", "rating", "sales_hint"], + }, + "etmall_tw": { + "display_name": "ETMall", + "hosts": ["www.etmall.com.tw", "etmall.com.tw"], + "search_url_template": "https://www.etmall.com.tw/Search?keyword={query}", + "priority": "P2_marketplace_expansion", + "best_for": ["promotion blocks", "coupon and member badges"], + "data_targets": ["title", "price", "promo_badge", "member_price", "shipping_hint"], + }, + "friday_tw": { + "display_name": "friDay Shopping", + "hosts": ["shopping.friday.tw", "ec-m.shopping.friday.tw", "ec-w.shopping.friday.tw"], + "search_url_template": "https://ec-m.shopping.friday.tw/search?keyword={query}", + "priority": "P3_probe_required", + "best_for": ["campaign cards", "member reward hints", "mobile rendered offers"], + "data_targets": ["title", "price", "reward_badge", "campaign_badge"], + }, + "rakuten_tw": { + "display_name": "Rakuten Taiwan", + "hosts": ["www.rakuten.com.tw", "rakuten.com.tw"], + "search_url_template": "https://www.rakuten.com.tw/search/{query}/", + "priority": "P3_probe_required", + "best_for": ["storefront cards", "cross-store price comparisons", "coupon badges"], + "data_targets": ["title", "price", "shop", "coupon", "point_reward"], + }, + "market_intel": { + "display_name": "Market intelligence URL", + "hosts": [], + "search_url_template": "", + "priority": "custom_url_only", + "best_for": ["controlled external market diagnostics"], + "data_targets": ["public_page_visual_evidence"], + }, + "external_market": { + "display_name": "External market URL", + "hosts": [], + "search_url_template": "", + "priority": "custom_url_only", + "best_for": ["controlled public external URL diagnostics"], + "data_targets": ["public_page_visual_evidence"], + }, +} + +ALLOWED_PLATFORMS = tuple(ECOMMERCE_PLATFORM_PROFILES) +MARKETPLACE_EXPANSION_ORDER = ( + "momo", + "pchome", + "shopee_tw", + "coupang_tw", + "yahoo_shopping_tw", + "etmall_tw", + "friday_tw", + "rakuten_tw", +) def _default_artifact_root() -> str: container_root = Path("/app/data/ai_automation") if container_root.exists(): @@ -36,7 +132,18 @@ def _default_artifact_root() -> str: return "runtime_artifacts/pixelrag_visual_evidence" +def _default_manifest_queue_root() -> str: + container_root = Path("/app/data/ai_automation") + if container_root.exists(): + return str(container_root / "pixelrag_manifest_queue") + return "runtime_artifacts/pixelrag_manifest_queue" + + DEFAULT_ARTIFACT_ROOT = os.getenv("PIXELRAG_VISUAL_EVIDENCE_ROOT", _default_artifact_root()) +DEFAULT_MANIFEST_QUEUE_ROOT = os.getenv( + "PIXELRAG_VISUAL_EVIDENCE_MANIFEST_QUEUE_ROOT", + _default_manifest_queue_root(), +) DEFAULT_ARTIFACT_MAX_AGE_HOURS = int(os.getenv("PIXELRAG_VISUAL_EVIDENCE_MAX_AGE_HOURS", "168")) VISUAL_FALLBACK_CONFIDENCE_TRIGGERS = { "low", @@ -188,6 +295,38 @@ def _safe_int(value: Any, default: int = 0) -> int: return default +def _visual_barrier_reason(receipt: Mapping[str, Any]) -> str: + http_status = _safe_int(receipt.get("http_status")) + page_metrics = receipt.get("page_metrics") or {} + final_url = str(page_metrics.get("final_url") or "").lower() + title = str(page_metrics.get("title") or "").lower() + if http_status >= 400: + return f"http_status_{http_status}" + barrier_tokens = ( + "access denied", + "verify/traffic/error", + "captcha", + "blocked", + "robot", + ) + haystack = f"{final_url} {title}" + for token in barrier_tokens: + if token in haystack: + return token.replace("/", "_").replace(" ", "_") + return "" + + +def _platform_profile(platform: str) -> dict[str, Any] | None: + return ECOMMERCE_PLATFORM_PROFILES.get(str(platform or "").strip().lower()) + + +def _search_url_for_platform(platform: str, keyword: str) -> str: + profile = _platform_profile(platform) or {} + template = str(profile.get("search_url_template") or "") + query = quote_plus(str(keyword or "").strip()) + return template.format(query=query) if template and query else "" + + def _build_phases(capabilities: Mapping[str, bool]) -> list[dict[str, Any]]: capture_ready = bool(capabilities["playwright_artifact_pipeline"]) structured_ready = bool(capabilities["structured_crawler_api"]) @@ -316,6 +455,131 @@ def should_emit_visual_evidence_fallback( } +def build_pixelrag_ecommerce_expansion_plan( + *, + target_platforms: tuple[str, ...] | list[str] | None = None, +) -> dict[str, Any]: + """Return a machine-readable PixelRAG rollout plan for public commerce sites.""" + requested = [ + str(platform or "").strip().lower() + for platform in (target_platforms or MARKETPLACE_EXPANSION_ORDER) + if str(platform or "").strip() + ] + selected = [ + platform + for platform in requested + if platform in ECOMMERCE_PLATFORM_PROFILES + ] + unknown = [ + platform + for platform in requested + if platform not in ECOMMERCE_PLATFORM_PROFILES + ] + profiles = [] + for platform in selected: + profile = deepcopy(ECOMMERCE_PLATFORM_PROFILES[platform]) + profile["platform"] = platform + profile["manifest_ready"] = bool(profile.get("search_url_template")) + profile["capture_scope"] = "public_read_only_no_login_no_cart" + profiles.append(profile) + + return { + "success": True, + "policy": ECOMMERCE_EXPANSION_POLICY, + "generated_at": datetime.now(timezone.utc).isoformat(), + "status": "marketplace_profiles_ready", + "platform_count": len(profiles), + "unknown_platforms": unknown, + "platforms": profiles, + "advantages": [ + "Covers JS-heavy cards and visual promotion badges that HTML parsers often miss.", + "Creates replayable screenshots and tiles before adding platform-specific parsers.", + "Lets AI compare public offer presentation across marketplaces without writing price truth.", + "Keeps each platform behind host allowlists and read-only artifact boundaries.", + ], + "limitations": [ + "Screenshots are slower and heavier than structured APIs.", + "Visual evidence is not a formal price source until OCR/VLM replay is verified.", + "Login-only, cart-only, coupon-personalized, or anti-bot-gated states are out of scope.", + "Search URL patterns still need runtime probes before production scheduling per platform.", + ], + "next_actions": [ + { + "priority": "P0", + "action": "capture_public_search_visual_receipts", + "target_platforms": ["shopee_tw", "coupang_tw"], + "status": "ready_for_controlled_probe", + }, + { + "priority": "P1", + "action": "add_visual_card_extraction_replay", + "target_fields": ["title", "price", "promo_badge", "shop", "rating"], + "status": "requires_receipt_samples", + }, + { + "priority": "P2", + "action": "promote_stable_platform_parser", + "status": "only_after_replay_beats_visual_only_diagnostics", + }, + ], + "controlled_apply": _controlled_apply_boundary(), + } + + +def build_pixelrag_marketplace_search_manifest( + *, + platform: str, + keyword: str, + crawler: str = "PixelRAGMarketplaceSearch.visual_fallback", + trigger_reason: str = "marketplace_search_visual_scan", + evidence_intent: str = "collect_public_marketplace_offer_cards", + viewport: Mapping[str, Any] | None = None, + page_size: Mapping[str, Any] | None = None, + tile_size: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Build a visual evidence manifest from a public marketplace keyword.""" + platform_code = str(platform or "").strip().lower() + keyword_text = str(keyword or "").strip() + profile = _platform_profile(platform_code) + errors: list[str] = [] + if not profile: + errors.append(f"platform must be one of: {', '.join(ALLOWED_PLATFORMS)}.") + if not keyword_text: + errors.append("keyword is required.") + search_url = _search_url_for_platform(platform_code, keyword_text) + if profile and not search_url: + errors.append("platform does not define a public search URL template.") + if errors: + return { + "success": False, + "policy": MANIFEST_POLICY, + "result": RESULT_MANIFEST_REJECTED, + "errors": errors, + "controlled_apply": _controlled_apply_boundary(), + } + + manifest = build_pixelrag_visual_evidence_manifest( + url=search_url, + platform=platform_code, + crawler=crawler, + trigger_reason=trigger_reason, + evidence_intent=evidence_intent, + viewport=viewport, + page_size=page_size, + tile_size=tile_size, + ) + if manifest.get("success"): + manifest["marketplace_profile"] = { + "platform": platform_code, + "display_name": profile.get("display_name"), + "hosts": profile.get("hosts"), + "priority": profile.get("priority"), + "data_targets": profile.get("data_targets"), + "keyword": keyword_text, + } + return manifest + + def build_pixelrag_visual_evidence_manifest( *, url: str, @@ -425,6 +689,144 @@ def build_pixelrag_visual_evidence_manifest( } +def persist_pixelrag_visual_evidence_manifest( + manifest: Mapping[str, Any], + *, + queue_root: str | Path | None = None, +) -> dict[str, Any]: + """Persist a capture manifest to the local artifact queue.""" + root = Path(queue_root or DEFAULT_MANIFEST_QUEUE_ROOT) + platform = str((manifest.get("capture_target") or {}).get("platform") or "unknown").strip().lower() + manifest_id = str(manifest.get("manifest_id") or "").strip() + errors: list[str] = [] + if manifest.get("policy") != MANIFEST_POLICY: + errors.append(f"manifest policy must be {MANIFEST_POLICY}") + if not manifest.get("success"): + errors.append("manifest must be successful before persistence") + if not manifest_id: + errors.append("manifest_id is required") + if platform not in ALLOWED_PLATFORMS: + errors.append(f"platform must be one of: {', '.join(ALLOWED_PLATFORMS)}.") + if errors: + return { + "success": False, + "status": "rejected", + "errors": errors, + "queue_root": str(root), + "controlled_apply": { + "artifact_write": False, + "db_write": False, + "secret_read": False, + "production_price_write": False, + }, + } + + target_dir = root / platform + target_dir.mkdir(parents=True, exist_ok=True) + target_path = target_dir / f"{manifest_id}.json" + temp_path = target_path.with_suffix(".json.tmp") + payload = dict(manifest) + payload["queued_at"] = datetime.now(timezone.utc).isoformat() + payload["queue_policy"] = CRAWLER_FAILURE_ACTION_POLICY + temp_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + temp_path.replace(target_path) + return { + "success": True, + "status": "queued", + "path": str(target_path), + "queue_root": str(root), + "controlled_apply": { + "artifact_write": True, + "db_write": False, + "secret_read": False, + "production_price_write": False, + }, + } + + +def build_pixelrag_crawler_failure_action( + *, + url: str, + platform: str, + crawler: str, + parser_success: bool, + parsed_item_count: int = 0, + confidence_band: str = "", + missing_fields: tuple[str, ...] | list[str] | None = None, + failure_reason: str = "", + evidence_intent: str = "recover_visual_offer_evidence", + persist_manifest: bool = False, + queue_root: str | Path | None = None, + viewport: Mapping[str, Any] | None = None, + page_size: Mapping[str, Any] | None = None, + tile_size: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Turn a crawler parse failure into a queued PixelRAG capture action.""" + selector = should_emit_visual_evidence_fallback( + parser_success=parser_success, + parsed_item_count=parsed_item_count, + confidence_band=confidence_band, + missing_fields=missing_fields, + failure_reason=failure_reason, + ) + if not selector["should_emit"]: + return { + "success": True, + "policy": CRAWLER_FAILURE_ACTION_POLICY, + "status": "skipped_structured_evidence_sufficient", + "selector": selector, + "manifest": None, + "persistence": None, + "next_machine_action": "keep_structured_crawler_result", + "controlled_apply": _controlled_apply_boundary(), + } + + manifest = build_pixelrag_visual_evidence_manifest( + url=url, + platform=platform, + crawler=crawler, + trigger_reason=selector["fallback_reason"], + evidence_intent=evidence_intent, + viewport=viewport, + page_size=page_size, + tile_size=tile_size, + ) + persistence = None + if persist_manifest and manifest.get("success"): + persistence = persist_pixelrag_visual_evidence_manifest( + manifest, + queue_root=queue_root, + ) + + manifest_ready = bool(manifest.get("success")) + queued = bool((persistence or {}).get("success")) + return { + "success": manifest_ready and (not persist_manifest or queued), + "policy": CRAWLER_FAILURE_ACTION_POLICY, + "generated_at": datetime.now(timezone.utc).isoformat(), + "status": ( + "manifest_persisted" + if queued + else ("manifest_ready" if manifest_ready else "manifest_rejected") + ), + "selector": selector, + "manifest": manifest, + "persistence": persistence, + "next_machine_action": ( + "run_pixelrag_visual_capture_worker" + if manifest_ready + else "fix_manifest_input" + ), + "controlled_apply": { + "network_call": False, + "artifact_write": bool(queued), + "db_write": False, + "secret_read": False, + "production_price_write": False, + }, + } + + def build_pixelrag_crawler_integration_assessment( *, capabilities: Mapping[str, Any] | None = None, @@ -634,7 +1036,19 @@ def build_pixelrag_visual_evidence_readback( capture_target = receipt.get("capture_target") or {} tile_plan = receipt.get("tile_plan") or {} - status = "ok" if receipt.get("status") == "captured" and missing_file_count == 0 and not stale else "warning" + page_metrics = receipt.get("page_metrics") or {} + http_status = _safe_int(receipt.get("http_status")) + barrier_reason = _visual_barrier_reason(receipt) + status = ( + "ok" + if ( + receipt.get("status") == "captured" + and missing_file_count == 0 + and not stale + and not barrier_reason + ) + else "warning" + ) if errors: status = "critical" @@ -651,9 +1065,11 @@ def build_pixelrag_visual_evidence_readback( "missing_file_count": missing_file_count, "planned_tile_count": _safe_int(tile_plan.get("planned_tile_count")), "emitted_tile_count": _safe_int(tile_plan.get("emitted_tile_count")), - "http_status": _safe_int(receipt.get("http_status")), + "http_status": http_status, "age_hours": round(age_hours, 3) if age_hours is not None else None, "stale": stale, + "visual_barrier_detected": bool(barrier_reason), + "visual_barrier_reason": barrier_reason, "errors": errors, }, "latest_receipt": { @@ -661,14 +1077,18 @@ def build_pixelrag_visual_evidence_readback( "generated_at": receipt.get("generated_at"), "capture_status": receipt.get("status"), "capture_target": capture_target, - "page_metrics": receipt.get("page_metrics") or {}, + "page_metrics": page_metrics, "tile_plan": tile_plan, "files": file_statuses, }, "next_machine_action": ( "keep_pixelrag_visual_evidence_receipt" if status == "ok" - else "run_pixelrag_visual_capture_worker" + else ( + "run_platform_probe_or_use_structured_api" + if barrier_reason + else "run_pixelrag_visual_capture_worker" + ) ), "controlled_apply": { "network_call": False, @@ -680,6 +1100,9 @@ def build_pixelrag_visual_evidence_readback( __all__ = [ + "CRAWLER_FAILURE_ACTION_POLICY", + "ECOMMERCE_EXPANSION_POLICY", + "ECOMMERCE_PLATFORM_PROFILES", "MANIFEST_POLICY", "POLICY", "READBACK_POLICY", @@ -688,7 +1111,11 @@ __all__ = [ "RESULT_MANIFEST_READY", "RESULT_MANIFEST_REJECTED", "build_pixelrag_crawler_integration_assessment", + "build_pixelrag_crawler_failure_action", + "build_pixelrag_ecommerce_expansion_plan", + "build_pixelrag_marketplace_search_manifest", "build_pixelrag_visual_evidence_readback", "build_pixelrag_visual_evidence_manifest", + "persist_pixelrag_visual_evidence_manifest", "should_emit_visual_evidence_fallback", ] diff --git a/tests/test_pchome_crawler_search.py b/tests/test_pchome_crawler_search.py index 6b859b6..6596d74 100644 --- a/tests/test_pchome_crawler_search.py +++ b/tests/test_pchome_crawler_search.py @@ -69,6 +69,32 @@ def test_pchome_search_scans_multiple_pages_until_limit(monkeypatch): assert [product.product_id for product in products] == ["A001", "A002", "A003"] +def test_pchome_search_auto_queues_pixelrag_manifest_on_empty_result(monkeypatch, tmp_path): + from services import pixelrag_crawler_integration_service as pixelrag + from services.pchome_crawler import PChomeCrawler + + monkeypatch.setattr(pixelrag, "DEFAULT_MANIFEST_QUEUE_ROOT", str(tmp_path)) + crawler = PChomeCrawler(timeout=1, delay=0, max_retries=0) + + class FakeSession: + headers = {} + + def get(self, url, params=None, timeout=None): + return _FakeResponse({"Prods": []}) + + crawler.session = FakeSession() + + success, message, products = crawler.search_products("找不到的商品", limit=3, max_pages=1) + + assert success is False + assert message == "沒有找到符合的商品" + assert products == [] + action = crawler.last_visual_evidence_action + assert action["status"] == "manifest_persisted" + assert action["manifest"]["capture_target"]["crawler"] == "PChomeCrawler.search_products" + assert action["persistence"]["path"].startswith(str(tmp_path)) + + def test_pchome_get_retries_transient_timeout(): from services.pchome_crawler import PChomeCrawler diff --git a/tests/test_pixelrag_crawler_integration_service.py b/tests/test_pixelrag_crawler_integration_service.py index f2a386c..263828d 100644 --- a/tests/test_pixelrag_crawler_integration_service.py +++ b/tests/test_pixelrag_crawler_integration_service.py @@ -124,6 +124,67 @@ def test_visual_fallback_selector_routes_parser_empty_and_low_confidence_cases() assert clean_parser["fallback_reason"] == "structured_evidence_sufficient" +def test_pixelrag_ecommerce_expansion_plan_includes_shopee_and_coupang(): + from services.pixelrag_crawler_integration_service import ( + ECOMMERCE_EXPANSION_POLICY, + build_pixelrag_ecommerce_expansion_plan, + ) + + payload = build_pixelrag_ecommerce_expansion_plan() + platforms = {item["platform"]: item for item in payload["platforms"]} + + assert payload["policy"] == ECOMMERCE_EXPANSION_POLICY + assert payload["status"] == "marketplace_profiles_ready" + assert platforms["shopee_tw"]["manifest_ready"] is True + assert platforms["coupang_tw"]["manifest_ready"] is True + assert "price" in platforms["shopee_tw"]["data_targets"] + assert payload["next_actions"][0]["target_platforms"] == ["shopee_tw", "coupang_tw"] + assert payload["controlled_apply"]["production_price_write"] is False + + +def test_marketplace_search_manifest_builds_shopee_visual_capture_target(): + from services.pixelrag_crawler_integration_service import build_pixelrag_marketplace_search_manifest + + payload = build_pixelrag_marketplace_search_manifest( + platform="shopee_tw", + keyword="防曬乳", + page_size={"width": 1440, "height": 1024}, + tile_size={"width": 512, "height": 512}, + ) + + assert payload["success"] is True + assert payload["capture_target"]["platform"] == "shopee_tw" + assert payload["capture_target"]["url"].startswith("https://shopee.tw/search?keyword=") + assert "%E9%98%B2%E6%9B%AC%E4%B9%B3" in payload["capture_target"]["url"] + assert payload["marketplace_profile"]["display_name"] == "Shopee Taiwan" + assert payload["tile_plan"]["tile_count"] == 6 + + +def test_crawler_failure_action_persists_manifest_queue(tmp_path): + from services.pixelrag_crawler_integration_service import build_pixelrag_crawler_failure_action + + payload = build_pixelrag_crawler_failure_action( + url="https://24h.pchome.com.tw/search/?q=test", + platform="pchome", + crawler="PChomeCrawler.search_products", + parser_success=False, + parsed_item_count=0, + missing_fields=["price", "product_id"], + failure_reason="search API returned no product IDs", + persist_manifest=True, + queue_root=tmp_path, + ) + + assert payload["success"] is True + assert payload["status"] == "manifest_persisted" + assert payload["next_machine_action"] == "run_pixelrag_visual_capture_worker" + assert payload["controlled_apply"]["artifact_write"] is True + manifest_path = payload["persistence"]["path"] + queued = json.loads(open(manifest_path, encoding="utf-8").read()) + assert queued["capture_target"]["platform"] == "pchome" + assert queued["queue_policy"] == "read_only_pixelrag_crawler_failure_action_v1" + + def test_visual_evidence_manifest_builds_tile_plan_without_runtime_writes(): from services.pixelrag_crawler_integration_service import ( RESULT_MANIFEST_READY, @@ -226,6 +287,17 @@ def test_pixelrag_ai_automation_route_returns_assessment_and_manifest(): ): manifest_response = routes.ai_automation_pixelrag_crawler_integration_api.__wrapped__() manifest = manifest_response.get_json() + with app.test_request_context( + "/api/ai-automation/pixelrag-crawler-integration?expansion_plan=true" + ): + expansion_response = routes.ai_automation_pixelrag_crawler_integration_api.__wrapped__() + expansion = expansion_response.get_json() + with app.test_request_context( + "/api/ai-automation/pixelrag-crawler-integration" + "?platform=shopee_tw&marketplace_keyword=防曬乳" + ): + marketplace_response = routes.ai_automation_pixelrag_crawler_integration_api.__wrapped__() + marketplace = marketplace_response.get_json() assert assessment["policy"] == "read_only_pixelrag_crawler_integration_assessment_v1" assert assessment["target_platforms"] == ["momo"] @@ -233,6 +305,10 @@ def test_pixelrag_ai_automation_route_returns_assessment_and_manifest(): assert manifest["policy"] == "read_only_pixelrag_visual_evidence_manifest_v1" assert manifest["result"] == "PIXELRAG_VISUAL_EVIDENCE_MANIFEST_READY" assert manifest["capture_target"]["platform"] == "pchome" + assert expansion["policy"] == "read_only_pixelrag_ecommerce_expansion_plan_v1" + assert expansion["next_actions"][0]["target_platforms"] == ["shopee_tw", "coupang_tw"] + assert marketplace["capture_target"]["platform"] == "shopee_tw" + assert marketplace["marketplace_profile"]["keyword"] == "防曬乳" def _write_capture_receipt(root, *, platform="pchome", manifest_id="manifest-1"): @@ -303,6 +379,30 @@ def test_pixelrag_visual_evidence_readback_reports_latest_receipt(tmp_path): assert payload["controlled_apply"]["db_write"] is False +def test_pixelrag_visual_evidence_readback_warns_on_access_denied_page(tmp_path): + from services.pixelrag_crawler_integration_service import build_pixelrag_visual_evidence_readback + + receipt_path = _write_capture_receipt(tmp_path, platform="coupang_tw", manifest_id="denied") + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + receipt["http_status"] = 403 + receipt["page_metrics"] = { + "final_url": "https://www.tw.coupang.com/search?q=iphone", + "title": "Access Denied", + } + receipt_path.write_text(json.dumps(receipt), encoding="utf-8") + + payload = build_pixelrag_visual_evidence_readback( + artifact_root=tmp_path, + platform="coupang_tw", + manifest_id="denied", + ) + + assert payload["status"] == "warning" + assert payload["summary"]["visual_barrier_detected"] is True + assert payload["summary"]["visual_barrier_reason"] == "http_status_403" + assert payload["next_machine_action"] == "run_platform_probe_or_use_structured_api" + + def test_pixelrag_visual_evidence_readback_warns_when_missing(tmp_path): from services.pixelrag_crawler_integration_service import build_pixelrag_visual_evidence_readback @@ -467,3 +567,34 @@ def test_pixelrag_python_capture_worker_dry_run_plans_artifact_outputs(tmp_path) assert payload["controlled_apply"]["artifact_write"] is False assert payload["tile_plan"]["planned_tile_count"] == 4 assert payload["capture_target"]["platform"] == "pchome" + + +def test_pixelrag_python_capture_worker_accepts_shopee_manifest(tmp_path): + from services.pixelrag_crawler_integration_service import build_pixelrag_marketplace_search_manifest + + manifest = build_pixelrag_marketplace_search_manifest( + platform="shopee_tw", + keyword="防曬乳", + page_size={"width": 1024, "height": 1024}, + tile_size={"width": 512, "height": 512}, + ) + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/capture_pixelrag_visual_evidence.py", + "--manifest-json", + json.dumps(manifest), + "--output-dir", + str(tmp_path), + "--dry-run", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["status"] == "dry_run_ready" + assert payload["capture_target"]["platform"] == "shopee_tw" + assert payload["tile_plan"]["planned_tile_count"] == 4 From cc3c0697a70300ed35a431cb24eacad86d951a4b Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 22:18:34 +0800 Subject: [PATCH 03/75] feat(ai): add external mcp rag integration readback --- docs/guides/ai_automation_session_sop.md | 2 + routes/system_public_routes.py | 13 + .../report_external_mcp_rag_integration.py | 40 +++ .../external_mcp_rag_integration_service.py | 308 ++++++++++++++++++ ...st_external_mcp_rag_integration_service.py | 73 +++++ 5 files changed, 436 insertions(+) create mode 100755 scripts/ops/report_external_mcp_rag_integration.py create mode 100644 services/external_mcp_rag_integration_service.py create mode 100644 tests/test_external_mcp_rag_integration_service.py diff --git a/docs/guides/ai_automation_session_sop.md b/docs/guides/ai_automation_session_sop.md index 3027afb..c552759 100644 --- a/docs/guides/ai_automation_session_sop.md +++ b/docs/guides/ai_automation_session_sop.md @@ -26,6 +26,8 @@ - Telegram 失敗必須可暫存與 replay。 - EventRouter / AutoHeal 變更必須更新 `services/ai_automation_metrics.py` 指標或確認既有指標已覆蓋。 - AI 自動化閉環變更必須確認 `/api/ai-automation/smoke` 與 `/ai_automation_smoke` 仍能反映新狀態。 +- 外部 MCP / RAG 能力導入內部治理時，必須確認 `/api/ai-automation/external-mcp-rag-integration` + 或 `python scripts/ops/report_external_mcp_rag_integration.py` 可讀回每個能力的內部落點、狀態、資料邊界與下一個機器動作。 - AI 自動化 Prometheus 指標變更必須同步檢查 `docker/grafana/provisioning/dashboards/json/ai-automation-overview.json` 是否需要新增 panel 或查詢。 - 188 線上 active monitoring stack 以 `monitoring/prometheus.yml` 為準；110 gateway 另有 `/home/wooo/monitoring/prometheus.yml`。若 dashboard 無資料，先確認 Prometheus `momo-app` target 與 `momo-network` 連線；所有 Blackbox HTTP target 必須打 `/health`，不可打 Dashboard 首頁 `/`。 - Smoke dashboard 會保存 JSONL 趨勢；若新增檢查項目，要確保 history compact record 仍保持小而可讀。 diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 030f885..d2813bf 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -689,6 +689,19 @@ def ai_automation_pixelrag_visual_evidence_readback_api(): )) +@system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') +@login_required +def ai_automation_external_mcp_rag_integration_api(): + """Read-only external MCP/RAG absorption status for internal MCP/RAG planes.""" + from services.external_mcp_rag_integration_service import ( + build_external_mcp_rag_integration_readback, + ) + + return jsonify(build_external_mcp_rag_integration_readback( + target_family=str(request.args.get('family') or '').strip() or None, + )) + + @system_public_bp.route('/api/ai-automation/smoke/history/export') @login_required def ai_automation_smoke_history_export(): diff --git a/scripts/ops/report_external_mcp_rag_integration.py b/scripts/ops/report_external_mcp_rag_integration.py new file mode 100755 index 0000000..4b719f5 --- /dev/null +++ b/scripts/ops/report_external_mcp_rag_integration.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Report external MCP/RAG absorption into internal MCP/RAG planes.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.external_mcp_rag_integration_service import ( # noqa: E402 + build_external_mcp_rag_integration_readback, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="輸出外部 MCP/RAG 導入內部 MCP/RAG 治理面的機器可讀狀態。" + ) + parser.add_argument( + "--family", + choices=["external_mcp", "internal_mcp", "external_rag", "internal_rag"], + help="只輸出指定 capability family。", + ) + args = parser.parse_args() + + payload = build_external_mcp_rag_integration_readback( + target_family=args.family, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/external_mcp_rag_integration_service.py b/services/external_mcp_rag_integration_service.py new file mode 100644 index 0000000..282bb05 --- /dev/null +++ b/services/external_mcp_rag_integration_service.py @@ -0,0 +1,308 @@ +"""Read-only inventory for external MCP/RAG absorption into internal control planes.""" + +from __future__ import annotations + +import os +from copy import deepcopy +from datetime import datetime, timezone +from typing import Any + + +POLICY = "read_only_external_mcp_rag_integration_readback_v1" + + +def _enabled(value: str | None) -> bool: + return str(value or "").strip().lower() in {"1", "true", "yes", "on"} + + +def _controlled_apply_boundary() -> dict[str, bool]: + return { + "network_call": False, + "db_write": False, + "secret_read": False, + "production_price_write": False, + "external_side_effect": False, + } + + +def _status_counts(items: list[dict[str, Any]]) -> dict[str, int]: + counts: dict[str, int] = {} + for item in items: + status = str(item.get("status") or "unknown") + counts[status] = counts.get(status, 0) + 1 + return counts + + +def _mcp_runtime_snapshot() -> dict[str, Any]: + try: + from services.mcp_router import MCP_BASE_HOSTS, TOOL_REGISTRY, is_mcp_router_enabled + + return { + "enabled": is_mcp_router_enabled(), + "servers": deepcopy(MCP_BASE_HOSTS), + "caller_count": len(TOOL_REGISTRY), + "tool_registry": { + caller: { + server: list(tools) + for server, tools in servers.items() + } + for caller, servers in TOOL_REGISTRY.items() + }, + } + except Exception as exc: + return { + "enabled": False, + "servers": {}, + "caller_count": 0, + "tool_registry": {}, + "error": str(exc)[:300], + } + + +def _rag_runtime_snapshot() -> dict[str, Any]: + try: + from services.rag_service import ( + RAG_DEFAULT_THRESHOLD, + RAG_DEFAULT_TOP_K, + RAG_EMBED_DIM, + RAG_EMBED_MODEL, + get_embedding_signature, + is_rag_enabled, + ) + + return { + "enabled": is_rag_enabled(), + "vector_store": "pgvector", + "embedding_model": RAG_EMBED_MODEL, + "embedding_dim": RAG_EMBED_DIM, + "embedding_signature": get_embedding_signature(), + "default_top_k": RAG_DEFAULT_TOP_K, + "default_threshold": RAG_DEFAULT_THRESHOLD, + } + except Exception as exc: + return { + "enabled": False, + "vector_store": "pgvector", + "embedding_model": "", + "embedding_dim": 0, + "embedding_signature": "", + "error": str(exc)[:300], + } + + +def _pixelrag_snapshot() -> dict[str, Any]: + try: + from services.pixelrag_crawler_integration_service import ECOMMERCE_PLATFORM_PROFILES + + platforms = sorted( + platform + for platform in ECOMMERCE_PLATFORM_PROFILES + if platform not in {"market_intel", "external_market"} + ) + return { + "enabled": True, + "platform_count": len(platforms), + "platforms": platforms, + "visual_rag_stage": "phase1_visual_evidence_receipts", + } + except Exception as exc: + return { + "enabled": False, + "platform_count": 0, + "platforms": [], + "visual_rag_stage": "unavailable", + "error": str(exc)[:300], + } + + +def _capability_inventory() -> list[dict[str, Any]]: + return [ + { + "id": "mcp.omnisearch.tavily_exa", + "family": "external_mcp", + "external_capability": "Tavily / Exa style public web search through omnisearch MCP", + "internal_plane": "mcp_router", + "internal_entrypoint": "services.mcp_router.TOOL_REGISTRY[mcp_collector][omnisearch]", + "status": "integrated_self_hosted_gateway", + "writes": ["mcp_calls_log_async"], + "data_boundary": "public_search_summary_only", + "next_machine_action": "keep_mcp_router_health_and_cache_readback", + }, + { + "id": "mcp.firecrawl.scrape", + "family": "external_mcp", + "external_capability": "Firecrawl style public page scrape", + "internal_plane": "mcp_router", + "internal_entrypoint": "services.mcp_router.TOOL_REGISTRY[*][firecrawl].scrape_url", + "status": "integrated_self_hosted_gateway", + "writes": ["mcp_calls_log_async"], + "data_boundary": "approved_public_url_only", + "next_machine_action": "enforce_source_contract_before_scheduler_attach", + }, + { + "id": "mcp.postgres.query", + "family": "internal_mcp", + "external_capability": "SQL read interface exposed as MCP tool", + "internal_plane": "mcp_router", + "internal_entrypoint": "postgres.query for approved callers", + "status": "integrated_read_only_contract", + "writes": ["mcp_calls_log_async"], + "data_boundary": "read_only_query_allowlist", + "next_machine_action": "keep_disallowing_unknown_callers_and_write_tools", + }, + { + "id": "mcp.filesystem.readonly", + "family": "internal_mcp", + "external_capability": "Filesystem MCP read diagnostics", + "internal_plane": "mcp_router", + "internal_entrypoint": "ops_diagnostics filesystem read-only tools", + "status": "integrated_read_only_contract", + "writes": ["mcp_calls_log_async"], + "data_boundary": "allowed_directories_read_only", + "next_machine_action": "keep_write_file_and_mutation_tools_rejected", + }, + { + "id": "rag.pixelrag.visual_evidence", + "family": "external_rag", + "external_capability": "PixelRAG page screenshot and tile retrieval pattern", + "internal_plane": "pixelrag_visual_evidence + internal RAG candidate lane", + "internal_entrypoint": "services.pixelrag_crawler_integration_service", + "status": "integrated_phase1_visual_receipts", + "writes": ["artifact_file_only"], + "data_boundary": "public_page_screenshot_tiles_no_price_write", + "next_machine_action": "add_ocr_vlm_replay_before_promoting_to_pgvector_rag", + }, + { + "id": "rag.pgvector.bge_m3", + "family": "internal_rag", + "external_capability": "General text RAG retrieval pattern", + "internal_plane": "rag_service", + "internal_entrypoint": "services.rag_service.RAGService.query", + "status": "internal_primary_ready", + "writes": ["rag_query_log_async"], + "data_boundary": "ai_insights_pgvector_embedding_signature_guard", + "next_machine_action": "keep_embedding_signature_and_feedback_guardrails", + }, + { + "id": "rag.qwen3_vl_embedding", + "family": "external_rag", + "external_capability": "Multimodal visual embedding model for tile retrieval", + "internal_plane": "ollama_first_visual_embedding_benchmark", + "internal_entrypoint": "not_enabled_in_production", + "status": "deferred_until_ollama_first_verified", + "writes": [], + "data_boundary": "no_hosted_visual_embedding_api", + "next_machine_action": "benchmark_local_multimodal_embedding_then_design_pgvector_metadata", + }, + { + "id": "rag.faiss_visual_index", + "family": "external_rag", + "external_capability": "FAISS visual retrieval index used by some PixelRAG pipelines", + "internal_plane": "pgvector_first_policy", + "internal_entrypoint": "not_enabled_in_production", + "status": "rejected_for_production_without_adr", + "writes": [], + "data_boundary": "pgvector_is_the_only_production_vector_store", + "next_machine_action": "use_pgvector_compatible_metadata_or_write_adr_before_any_faiss_trial", + }, + { + "id": "mcp.gemini_grounding", + "family": "external_mcp", + "external_capability": "Gemini grounding as hosted external search fallback", + "internal_plane": "mcp_collector_final_fallback", + "internal_entrypoint": "services.mcp_collector_service guarded fallback", + "status": "disabled_by_default_fallback_only", + "writes": ["mcp_cache_only_when_explicitly_enabled"], + "data_boundary": "gemini_api_hard_disabled_by_default", + "next_machine_action": "prefer_self_hosted_mcp_or_ollama_static_fallback", + }, + ] + + +def build_external_mcp_rag_integration_readback( + *, + target_family: str | None = None, +) -> dict[str, Any]: + """Describe how external MCP/RAG capabilities are absorbed internally.""" + family_filter = str(target_family or "").strip().lower() + inventory = _capability_inventory() + if family_filter: + inventory = [ + item + for item in inventory + if str(item.get("family") or "").lower() == family_filter + ] + + counts = _status_counts(inventory) + unresolved = [ + item + for item in inventory + if str(item.get("status") or "").startswith(("deferred", "rejected", "disabled")) + ] + absorbed = [ + item + for item in inventory + if item not in unresolved + ] + all_absorbed = len(unresolved) == 0 + + return { + "success": True, + "policy": POLICY, + "generated_at": datetime.now(timezone.utc).isoformat(), + "status": "partially_integrated" if not all_absorbed else "fully_integrated", + "answer_to_owner": ( + "不是全部完成；已把主要外部 MCP/RAG 放進內部可治理 registry，" + "其中 self-hosted MCP、pgvector RAG、PixelRAG phase1 已整合，" + "多模態 embedding / FAISS / Gemini grounding 仍需條件或維持停用。" + if not all_absorbed + else "目前 registry 內的外部 MCP/RAG 都已有內部治理落點。" + ), + "completion": { + "total_capabilities": len(inventory), + "absorbed_count": len(absorbed), + "unresolved_count": len(unresolved), + "all_absorbed": all_absorbed, + "status_counts": counts, + }, + "runtime": { + "mcp": _mcp_runtime_snapshot(), + "rag": _rag_runtime_snapshot(), + "pixelrag": _pixelrag_snapshot(), + "env_flags": { + "MCP_ROUTER_ENABLED": _enabled(os.getenv("MCP_ROUTER_ENABLED")), + "RAG_ENABLED": _enabled(os.getenv("RAG_ENABLED")), + "GEMINI_API_HARD_DISABLED": _enabled(os.getenv("GEMINI_API_HARD_DISABLED", "true")), + }, + }, + "inventory": inventory, + "next_machine_actions": [ + { + "priority": "P0", + "action": "wire_pixelrag_receipts_to_internal_rag_candidate_replay", + "status": "ready_after_ocr_vlm_replay_contract", + }, + { + "priority": "P0", + "action": "add_mcp_router_health_readback_to_ai_automation_smoke", + "status": "ready", + }, + { + "priority": "P1", + "action": "benchmark_ollama_first_visual_embedding", + "status": "blocked_until_model_runtime_verified", + }, + { + "priority": "P1", + "action": "promote_successful_marketplace_visual_receipts_to_source_contracts", + "status": "ready_for_shopee_warning_for_coupang", + }, + ], + "controlled_apply": _controlled_apply_boundary(), + } + + +__all__ = [ + "POLICY", + "build_external_mcp_rag_integration_readback", +] diff --git a/tests/test_external_mcp_rag_integration_service.py b/tests/test_external_mcp_rag_integration_service.py new file mode 100644 index 0000000..3f54d55 --- /dev/null +++ b/tests/test_external_mcp_rag_integration_service.py @@ -0,0 +1,73 @@ +import json +import subprocess +import sys + + +def test_external_mcp_rag_readback_reports_partial_truth(): + from services.external_mcp_rag_integration_service import ( + POLICY, + build_external_mcp_rag_integration_readback, + ) + + payload = build_external_mcp_rag_integration_readback() + inventory = {item["id"]: item for item in payload["inventory"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "partially_integrated" + assert payload["completion"]["all_absorbed"] is False + assert payload["completion"]["unresolved_count"] >= 1 + assert "不是全部完成" in payload["answer_to_owner"] + assert inventory["rag.pixelrag.visual_evidence"]["status"] == "integrated_phase1_visual_receipts" + assert inventory["rag.qwen3_vl_embedding"]["status"] == "deferred_until_ollama_first_verified" + assert inventory["rag.faiss_visual_index"]["status"] == "rejected_for_production_without_adr" + assert inventory["mcp.gemini_grounding"]["status"] == "disabled_by_default_fallback_only" + assert payload["controlled_apply"]["db_write"] is False + assert payload["controlled_apply"]["network_call"] is False + + +def test_external_mcp_rag_readback_can_filter_family(): + from services.external_mcp_rag_integration_service import ( + build_external_mcp_rag_integration_readback, + ) + + payload = build_external_mcp_rag_integration_readback(target_family="external_rag") + + assert payload["completion"]["total_capabilities"] == 3 + assert {item["family"] for item in payload["inventory"]} == {"external_rag"} + + +def test_external_mcp_rag_cli_outputs_machine_readable_json(): + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/report_external_mcp_rag_integration.py", + "--family", + "external_rag", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["success"] is True + assert payload["completion"]["total_capabilities"] == 3 + assert payload["inventory"][0]["family"] == "external_rag" + + +def test_external_mcp_rag_ai_automation_route_returns_readback(): + from flask import Flask + from routes import system_public_routes as routes + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/external-mcp-rag-integration?family=external_mcp" + ): + response = routes.ai_automation_external_mcp_rag_integration_api.__wrapped__() + payload = response.get_json() + + assert payload["policy"] == "read_only_external_mcp_rag_integration_readback_v1" + assert payload["success"] is True + assert {item["family"] for item in payload["inventory"]} == {"external_mcp"} + assert payload["runtime"]["mcp"]["caller_count"] >= 1 From 1bbf73a1762422a4a2eca8e5f788b508d1ce2857 Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 22:40:51 +0800 Subject: [PATCH 04/75] =?UTF-8?q?feat(ai):=20=E6=8E=A5=E5=85=A5=20PixelRAG?= =?UTF-8?q?=20RAG=20replay=20smoke?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 1 + TODO_NEXT_STEPS.txt | 12 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 7 +- .../ai_automation_mainline_work_items.md | 38 ++ docs/guides/ai_automation_session_sop.md | 3 + docs/guides/browse_sh_crawler_playbook.md | 10 + routes/system_public_routes.py | 22 ++ .../report_pixelrag_rag_candidate_replay.py | 60 ++++ services/ai_automation_smoke_service.py | 192 ++++++++++ .../pixelrag_rag_candidate_replay_service.py | 338 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 81 ++++- ...t_pixelrag_rag_candidate_replay_service.py | 138 +++++++ 13 files changed, 886 insertions(+), 18 deletions(-) create mode 100644 docs/guides/ai_automation_mainline_work_items.md create mode 100755 scripts/ops/report_pixelrag_rag_candidate_replay.py create mode 100644 services/pixelrag_rag_candidate_replay_service.py create mode 100644 tests/test_pixelrag_rag_candidate_replay_service.py diff --git a/AGENTS.md b/AGENTS.md index 2dc76d3..2a3312a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -139,6 +139,7 @@ - 前端更版路線圖: `docs/guides/frontend_upgrade_roadmap.md` - AI 觀測台 UI 治理: `docs/guides/observability_ui_governance.md` - AI 自動化 Session SOP: `docs/guides/ai_automation_session_sop.md` +- AI 自動化主線工作項目: `docs/guides/ai_automation_mainline_work_items.md` - Browse.sh 爬蟲診斷手冊: `docs/guides/browse_sh_crawler_playbook.md` - Webcrumbs 共用 UI Runtime: `docs/guides/webcrumbs_shared_runtime.md` - PChome 業績成長 UI/UX 守門: `docs/guides/pchome_growth_ui_ux_guardrails.md` diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 3764722..198203b 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -8,9 +8,9 @@ python scripts/ops/check_production_version_truth.py -目前 protected baseline：production `https://mo.wooo.work/health` = `V10.725`、 -clean workspace `/Users/ogt/codex-workspaces/ewoooc-dev` = `V10.725`、 -origin/main = `f3e412c`。舊 iCloud checkout 的 local-only `V10.726` 不是正式版本。 +目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 +本輪 source target 為 `V10.746`；部署完成前不得宣稱正式環境已是 `V10.746`。 +舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ 產品完整工作優先順序鎖 (2026-06-29) [ACTIVE] @@ -56,6 +56,12 @@ P6. DB apply authorization / signing / closeout - 這是最後的受控 apply 安全鏈，不得搶在 P1-P4 前面。 - 預設不讀 secret、不簽發 authorization、不執行 SQL、不寫 DB，直到 controlled apply 條件完整。 +P0-2026-07-09. PixelRAG / MCP / RAG 全自動主線 + - 已完成：多電商 PixelRAG visual evidence lane 與 external MCP/RAG integration readback。 + - 進行中：PixelRAG receipt → internal RAG candidate replay，以及 MCP/RAG runtime health → AI automation smoke。 + - 未開始：OCR/VLM replay、Ollama-first visual embedding benchmark、pgvector-compatible visual metadata、Coupang platform probe / structured API、跨平台 source contracts。 + - 主線文件：`docs/guides/ai_automation_mainline_work_items.md`。 + ================================================================================ PChome Mapping Backlog Read-Only Report (2026-06-28) [ACTIVE] ================================================================================ diff --git a/config.py b/config.py index 7503644..fc12f72 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.745" +SYSTEM_VERSION = "V10.746" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index ab8849d..a34c03d 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,8 +1,8 @@ # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth > **最後更新**: 2026-07-09 (台北時間) -> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.745 +> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 +> **適用版本**: V10.746 --- @@ -112,6 +112,8 @@ - 2026-07-03 起整站 UI/UX 專業化必須由 Sitewide UI/UX Agent 管理：`/api/ai-automation/sitewide-ui-ux-agent` 會掃描真正頁面模板、排除 partial/components/base layouts，輸出高優先頁、compact guardrail、raw engineering issue 與 next machine action；`/api/ai-automation/sitewide-ui-ux-repair-package` 會產生 no-write controlled repair package。當前 Agent 判定為 55 個頁面模板、31 個高優先模板、29 個已具 compact guardrail、0 個待專業化、0 個 raw engineering leak，Sitewide Agent 狀態為 `ok`，repair package 為 `no_op`。四批 controlled UI/UX repair 已完成 AI observability、AI 建議、PPT/CI/CD/業績/比價/成長/供貨風險高優先 surface；下一個機器動作為 `keep_sitewide_ui_ux_agent_monitoring`，後續只接受 Agent 新增 issue 或更高階 visual/mobile QA 的證據驅動修復。 - 2026-07-03 起 Sitewide UI/UX Agent 歸零狀態必須納入 AI automation runtime monitoring：`/api/ai-automation/smoke` 需包含 `Sitewide UI/UX Agent readback` 檢查，要求 `issue_surface_count=0`、`repair_package_status=no_op`、`repair_action_count=0`；`/api/ai-automation/scheduled-health-summary` 需輸出 `sitewide_ui_ux_agent` family；`/metrics` 需以 `momo_ai_automation_scheduled_health_family_status{family="sitewide_ui_ux_agent",...}` 暴露 scheduled family。此監控不得寫 DB、不得送通知、不得要求人工審核。 - 2026-07-03 起整站 desktop/tablet/mobile visual QA 必須進入 AI automation runtime monitoring：`scripts/check_responsive_overflow.js --artifact-output data/ai_automation/sitewide_visual_qa_latest.json` 會輸出機器可讀 artifact；`/api/ai-automation/sitewide-visual-qa-readback` 只讀該 artifact 並回報 route_count、viewport_count、failed_count、overflow_issue_count、stale 與 next_machine_action；`/api/ai-automation/smoke` 需包含 `Sitewide visual QA readback`，`/api/ai-automation/scheduled-health-summary` 需輸出 `sitewide_visual_qa` family，`/metrics` 需暴露 `momo_ai_automation_scheduled_health_family_status{family="sitewide_visual_qa",...}`。readback 不啟動 browser、不寫 DB、不送通知；artifact 生成才需要 Playwright/瀏覽器。 +- 2026-07-09 起 external MCP/RAG integration 必須進入 AI automation runtime monitoring：`/api/ai-automation/external-mcp-rag-integration` 與 `scripts/ops/report_external_mcp_rag_integration.py` 需輸出 9 個 capability 的 absorbed / unresolved 狀態、內部治理落點與 runtime flags；`/api/ai-automation/smoke` 需包含 `External MCP/RAG integration readback`，`/api/ai-automation/scheduled-health-summary` 需輸出 `external_mcp_rag_integration` family。`MCP_ROUTER_ENABLED=false` 或 `RAG_ENABLED=false` 必須被視為 runtime gap，不得因 registry 已完成就宣稱 MCP/RAG runtime 已全面啟用。 +- 2026-07-09 起 PixelRAG visual receipts 必須可轉為內部 RAG candidate replay readback：`/api/ai-automation/pixelrag-rag-candidate-replay` 與 `scripts/ops/report_pixelrag_rag_candidate_replay.py` 只讀 `capture_receipt.json`，拆分 eligible / blocked / invalid receipts，輸出 tile/missing/barrier count、candidate text、promotion boundary 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG RAG candidate replay`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_rag_candidate_replay` family。blocked / 403 / captcha / access denied / verify traffic page 只能進 platform probe 或 structured API 策略，不得當作商品資料、不得寫正式價格表、不得直接寫 `ai_insights`。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -871,6 +873,7 @@ POSTGRES_HOST=momo-db | 2026-07-03 | PChome auto-policy controlled apply final preflight 必須有 runtime monitoring | V10.743 起 `/api/ai-automation/smoke`、`/api/ai-automation/scheduled-health-summary` 與 `/metrics` 必須輸出 `PChome auto-policy controlled apply final preflight` / `pchome_auto_policy_controlled_apply_final_preflight`；此 runtime check 自動驗證 authorization evidence execution closeout carry-forward、database apply authorization final verifier gate、controlled apply final preflight package、rollback binding、post-apply verifier binding、12 個 final preflight checks、12 個 preflight fields、10 個 acceptance gates、dry-run-only、check-mode-only、same-run production truth requirement 與 future controlled dry-run package readiness，但明確標記 `reads_secret_count=0`、`executes_script_count=0`、`executes_sql_count=0`、`writes_database_count=0`、`signs_database_apply_authorization_count=0`、`primary_human_gate_count=0`、`rollback_execution_authorized=false`、`rollback_executes_sql=false`、`rollback_writes_database=false`、`post_apply_verifier_execution_authorized_in_preview=false`、`executes_database_apply=false`、`ready_for_database_apply_now=false`、`database_apply_authorized=false`、`issues_database_apply_authorization=false`、`signs_database_apply_authorization=false`，不讀 secret、不執行 authorization evidence、不執行 database apply、不執行 endpoint/SQL、不寫 DB、不簽發 apply authorization，只允許進入下一段 controlled dry-run package lane。 | | 2026-07-03 | PChome auto-policy controlled dry-run package 必須有 runtime monitoring | V10.744 起 `/api/ai-automation/smoke`、`/api/ai-automation/scheduled-health-summary` 與 `/metrics` 必須輸出 `PChome auto-policy controlled dry-run package` / `pchome_auto_policy_controlled_dry_run_package`；此 runtime check 自動驗證 controlled apply final preflight carry-forward、rollback binding、post-apply verifier binding、12 個 dry-run package checks、12 個 package fields、10 個 acceptance gates、receipt preview、result parser、non-executable command shape、same-run production truth 與 future dry-run receipt closeout readiness，但明確標記 `receipt_execution_performed=false`、`receipt_stdout_included=false`、`receipt_stderr_included=false`、`command_shape_execution_allowed=false`、`database_apply_authorized=false`、`reads_secret_count=0`、`executes_script_count=0`、`executes_sql_count=0`、`writes_database_count=0`、`signs_database_apply_authorization_count=0`、`primary_human_gate_count=0`、`executes_authorization_evidence=false`、`executes_database_apply=false`、`ready_for_database_apply_now=false`、`issues_database_apply_authorization=false`、`signs_database_apply_authorization=false`，不讀 secret、不執行 dry-run、不捕捉 stdout/stderr、不執行 authorization evidence、不執行 database apply、不執行 endpoint/SQL、不寫 DB、不簽發 apply authorization，只允許進入下一段 dry-run receipt closeout lane。 | | 2026-07-09 | PChome auto-policy controlled dry-run receipt closeout 必須有 runtime monitoring | V10.745 起 `/api/ai-automation/smoke`、`/api/ai-automation/scheduled-health-summary` 與 `/metrics` 必須輸出 `PChome auto-policy controlled dry-run receipt closeout` / `pchome_auto_policy_controlled_dry_run_receipt_closeout`；此 runtime check 自動驗證 controlled dry-run package carry-forward、dry-run result parser、receipt validation report、command-shape hash、rollback binding、post-apply verifier binding、12 個 receipt closeout checks、12 個 closeout fields、10 個 acceptance gates、same-run production truth 與 future runner readiness，但明確標記 `receipt_validation_status=preview_validated_not_executed`、`validation_execution_performed=false`、`validation_stdout_included=false`、`validation_stderr_included=false`、`validation_database_apply_authorized=false`、`reads_secret_count=0`、`executes_script_count=0`、`executes_sql_count=0`、`writes_database_count=0`、`signs_database_apply_authorization_count=0`、`primary_human_gate_count=0`、`executes_authorization_evidence=false`、`executes_database_apply=false`、`ready_for_database_apply_now=false`、`database_apply_authorized=false`、`issues_database_apply_authorization=false`、`signs_database_apply_authorization=false`，不讀 secret、不執行 dry-run、不捕捉 stdout/stderr、不執行 authorization evidence、不執行 database apply、不執行 endpoint/SQL、不寫 DB、不簽發 apply authorization，只允許進入下一段 controlled dry-run runner readiness lane。 | +| 2026-07-09 | PixelRAG / external MCP/RAG 必須有 runtime monitoring 與 candidate replay | V10.746 起 `/api/ai-automation/smoke`、`/api/ai-automation/scheduled-health-summary` 必須輸出 `External MCP/RAG integration readback` / `external_mcp_rag_integration` 與 `PixelRAG RAG candidate replay` / `pixelrag_rag_candidate_replay`；external MCP/RAG readback 回報 9 個 capability 的 absorbed / unresolved 與 `MCP_ROUTER_ENABLED`、`RAG_ENABLED` runtime flags，PixelRAG replay 只讀 visual receipt，拆分 eligible / blocked / invalid，並明確標記 blocked page 不是商品資料。此路徑不讀 secret、不呼叫外部網路、不寫 DB、不寫 `ai_insights`、不寫正式價格表；eligible receipt 仍需 OCR/VLM replay、identity matcher replay、PromotionGate 與 embedding signature guard。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md new file mode 100644 index 0000000..d93ca16 --- /dev/null +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -0,0 +1,38 @@ +# AI Automation Mainline Work Items + +> Updated: 2026-07-09 +> Scope: EwoooC / MOMO Pro System AI automation, PixelRAG, MCP/RAG, production UX. + +## Source Of Truth + +- Production readback stays authoritative after deploy: `/health`, container health, AI automation API readback, and smoke. +- Gitea is the only source-control truth. GitHub remains frozen and must not be used. +- Runtime truth must stay separate from source truth: a green test or registry does not mean MCP/RAG runtime is enabled. +- PixelRAG visual receipts are evidence, not formal price truth. They cannot write `competitor_prices`, `competitor_price_history`, or `ai_insights` without replay and promotion gates. + +## P0 + +| Status | Work item | Evidence / next machine action | +|---|---|---| +| Completed | Multi-commerce PixelRAG visual evidence lane for momo, pchome, shopee_tw, coupang_tw, yahoo_shopping_tw, etmall_tw, friday_tw, rakuten_tw | API and CLI can emit expansion plans and marketplace manifests; PChome parser failures can queue manifests. | +| Completed | External MCP/RAG capability inventory absorbed into internal governance readback | `/api/ai-automation/external-mcp-rag-integration` and `scripts/ops/report_external_mcp_rag_integration.py` expose 9 capabilities, absorbed/unresolved counts, and runtime flags. | +| In progress | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read receipts, split eligible vs blocked, and require OCR/VLM replay plus PromotionGate before knowledge writes. | +| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration and PixelRAG RAG candidate replay families. | +| In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | + +## P1 + +| Status | Work item | Evidence / next machine action | +|---|---|---| +| Not started | OCR/VLM replay contract for visual fields | Build a no-write replay that extracts title, price, promo badge, seller, rating, delivery badge from saved tiles and emits confidence/evidence only. | +| Not started | Ollama-first multimodal embedding benchmark | Verify local Qwen3-VL or equivalent visual embedding on GCP-A -> GCP-B -> 111 before any visual vector retrieval. | +| Not started | pgvector-compatible visual evidence metadata | Design metadata-first retrieval without FAISS in production unless ADR approves a different store. | +| Not started | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; prefer public structured data or approved probe, never count blocked pages as product data. | +| Not started | Marketplace source contracts beyond MOMO/PChome | Promote stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten receipts into source contracts only after provenance, rate-limit, public boundary, and replay evidence. | +| In progress | Professional product website UI/UX text reduction | Continue compact AI surfaces, visual QA, and user-facing copy guardrails; avoid engineering logs or work-session prose in the UI. | + +## Always Enforced + +- Manual review is exception-only for low-risk, machine-verifiable controlled apply. +- Break-glass still applies to secrets, destructive DB actions, force pushes, production provider switchovers, raw runtime volumes, and unverified core model replacement. +- AI automation smoke must expose health honestly: `MCP_ROUTER_ENABLED=false` or `RAG_ENABLED=false` is a runtime gap, not a source-code failure. diff --git a/docs/guides/ai_automation_session_sop.md b/docs/guides/ai_automation_session_sop.md index c552759..69733c7 100644 --- a/docs/guides/ai_automation_session_sop.md +++ b/docs/guides/ai_automation_session_sop.md @@ -28,6 +28,9 @@ - AI 自動化閉環變更必須確認 `/api/ai-automation/smoke` 與 `/ai_automation_smoke` 仍能反映新狀態。 - 外部 MCP / RAG 能力導入內部治理時，必須確認 `/api/ai-automation/external-mcp-rag-integration` 或 `python scripts/ops/report_external_mcp_rag_integration.py` 可讀回每個能力的內部落點、狀態、資料邊界與下一個機器動作。 +- PixelRAG 視覺 receipt 導入內部 RAG 時，必須確認 `/api/ai-automation/pixelrag-rag-candidate-replay` + 或 `python scripts/ops/report_pixelrag_rag_candidate_replay.py` 可讀回 eligible / blocked / invalid receipts，且 blocked page 不得被視為商品資料。 +- AI automation smoke 必須包含 external MCP/RAG integration 與 PixelRAG RAG candidate replay family，避免 registry 已完成但 runtime flag / receipt replay 未完成時被誤報為全自動閉環。 - AI 自動化 Prometheus 指標變更必須同步檢查 `docker/grafana/provisioning/dashboards/json/ai-automation-overview.json` 是否需要新增 panel 或查詢。 - 188 線上 active monitoring stack 以 `monitoring/prometheus.yml` 為準；110 gateway 另有 `/home/wooo/monitoring/prometheus.yml`。若 dashboard 無資料，先確認 Prometheus `momo-app` target 與 `momo-network` 連線；所有 Blackbox HTTP target 必須打 `/health`，不可打 Dashboard 首頁 `/`。 - Smoke dashboard 會保存 JSONL 趨勢；若新增檢查項目，要確保 history compact record 仍保持小而可讀。 diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index 39f1bc6..0e8f8f0 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -122,6 +122,7 @@ python scripts/ops/report_pixelrag_crawler_integration.py \ /api/ai-automation/pixelrag-crawler-integration?platform=coupang_tw&marketplace_keyword=iphone /api/ai-automation/pixelrag-crawler-integration?platform=pchome&manifest_url=https://24h.pchome.com.tw/prod/TEST-000000001&crawler=PChomeCrawler.search_products&trigger_reason=parser_empty /api/ai-automation/pixelrag-visual-evidence-readback?platform=pchome&manifest_id=4a93e95e5afb414bc8c3 +/api/ai-automation/pixelrag-rag-candidate-replay?platform=shopee_tw&platform=coupang_tw ``` 視覺證據 capture worker： @@ -161,6 +162,15 @@ runtime_artifacts/pixelrag_visual_evidence///capture_rece API readback 只讀 receipt 和檔案存在狀態，不重新抓外站、不寫資料庫。 正式 app 容器內的 readback 預設讀 `/app/data/ai_automation/pixelrag_visual_evidence`；host worker 若直接在 188 執行，預設仍可用 `runtime_artifacts/pixelrag_visual_evidence`，需要給 API 回讀時再同步到容器掛載的 `data/ai_automation`。 +RAG candidate replay： + +```bash +python scripts/ops/report_pixelrag_rag_candidate_replay.py +python scripts/ops/report_pixelrag_rag_candidate_replay.py --platform shopee_tw --platform coupang_tw +``` + +此 replay 只把 receipt 轉成內部 RAG candidate preview；eligible receipt 仍需 OCR/VLM replay、identity matcher replay、PromotionGate 與 embedding signature guard，blocked / 403 / captcha / access denied receipt 只能進 platform probe 或 structured API 策略，不得當成商品資料。 + 安全邊界： - read-only；不登入、不下單、不加入購物車、不寫第三方狀態。 diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index d2813bf..5f61ccb 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -689,6 +689,28 @@ def ai_automation_pixelrag_visual_evidence_readback_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-rag-candidate-replay') +@login_required +def ai_automation_pixelrag_rag_candidate_replay_api(): + """Read-only PixelRAG receipt to internal RAG candidate replay readback.""" + from services.pixelrag_rag_candidate_replay_service import ( + build_pixelrag_rag_candidate_replay_readback, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 50, type=int) + return jsonify(build_pixelrag_rag_candidate_replay_readback( + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 50, 250)), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/report_pixelrag_rag_candidate_replay.py b/scripts/ops/report_pixelrag_rag_candidate_replay.py new file mode 100755 index 0000000..9520f8c --- /dev/null +++ b/scripts/ops/report_pixelrag_rag_candidate_replay.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Report PixelRAG visual receipts as internal RAG candidate replay payloads.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_rag_candidate_replay_service import ( # noqa: E402 + build_pixelrag_rag_candidate_replay_readback, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="輸出 PixelRAG receipt 對內部 RAG candidate replay 的機器可讀讀回。" + ) + parser.add_argument( + "--artifact-root", + help="PixelRAG visual evidence artifact root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=50, + help="最多輸出 receipt candidate 數。", + ) + args = parser.parse_args() + + payload = build_pixelrag_rag_candidate_replay_readback( + artifact_root=args.artifact_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index 4a3a33f..31d5715 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -527,6 +527,16 @@ def build_scheduled_automation_health_summary( if not visual_qa_readback or not visual_qa_details: visual_qa_readback = _latest_sitewide_visual_qa_readback_check() visual_qa_details = visual_qa_readback.get("details") or {} + external_mcp_rag = _find_check(source_result, "External MCP/RAG integration readback") + external_mcp_rag_details = external_mcp_rag.get("details") or {} + if not external_mcp_rag or not external_mcp_rag_details: + external_mcp_rag = _external_mcp_rag_integration_check() + external_mcp_rag_details = external_mcp_rag.get("details") or {} + pixelrag_rag_replay = _find_check(source_result, "PixelRAG RAG candidate replay") + pixelrag_rag_replay_details = pixelrag_rag_replay.get("details") or {} + if not pixelrag_rag_replay or not pixelrag_rag_replay_details: + pixelrag_rag_replay = _pixelrag_rag_candidate_replay_check() + pixelrag_rag_replay_details = pixelrag_rag_replay.get("details") or {} smoke_status = source_result.get("status") or ("warning" if latest_history else "warning") freshness_family = _history_freshness_family( latest_history, @@ -4520,6 +4530,68 @@ def build_scheduled_automation_health_summary( "writes_database": False, }, }, + { + "key": "external_mcp_rag_integration", + "label": "External MCP/RAG integration", + "status": external_mcp_rag.get("status") or "warning", + "summary": ( + external_mcp_rag.get("summary") + or "External MCP/RAG integration has no latest readback." + ), + "next_machine_action": external_mcp_rag_details.get("next_machine_action") + or "refresh_external_mcp_rag_integration_readback", + "details": { + "policy": external_mcp_rag_details.get("policy"), + "integration_status": external_mcp_rag_details.get("integration_status"), + "total_capabilities": int( + external_mcp_rag_details.get("total_capabilities") or 0 + ), + "absorbed_count": int(external_mcp_rag_details.get("absorbed_count") or 0), + "unresolved_count": int( + external_mcp_rag_details.get("unresolved_count") or 0 + ), + "mcp_router_enabled": bool( + external_mcp_rag_details.get("mcp_router_enabled") + ), + "rag_enabled": bool(external_mcp_rag_details.get("rag_enabled")), + "gemini_hard_disabled": bool( + external_mcp_rag_details.get("gemini_hard_disabled") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, + { + "key": "pixelrag_rag_candidate_replay", + "label": "PixelRAG RAG candidate replay", + "status": pixelrag_rag_replay.get("status") or "warning", + "summary": ( + pixelrag_rag_replay.get("summary") + or "PixelRAG RAG candidate replay has no latest readback." + ), + "next_machine_action": pixelrag_rag_replay_details.get("next_machine_action") + or "run_pixelrag_rag_candidate_replay_readback", + "details": { + "policy": pixelrag_rag_replay_details.get("policy"), + "receipt_count": int(pixelrag_rag_replay_details.get("receipt_count") or 0), + "eligible_count": int(pixelrag_rag_replay_details.get("eligible_count") or 0), + "blocked_count": int(pixelrag_rag_replay_details.get("blocked_count") or 0), + "invalid_count": int(pixelrag_rag_replay_details.get("invalid_count") or 0), + "visual_barrier_count": int( + pixelrag_rag_replay_details.get("visual_barrier_count") or 0 + ), + "missing_file_count": int( + pixelrag_rag_replay_details.get("missing_file_count") or 0 + ), + "blocked_pages_are_not_product_data": bool( + pixelrag_rag_replay_details.get("blocked_pages_are_not_product_data") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, freshness_family, { "key": "daily_summary_delivery", @@ -13062,6 +13134,124 @@ def _latest_sitewide_visual_qa_readback_check() -> Dict[str, Any]: ) +def _external_mcp_rag_integration_check() -> Dict[str, Any]: + """Read-only sentinel for external MCP/RAG registry and runtime flags.""" + try: + from services.external_mcp_rag_integration_service import ( + build_external_mcp_rag_integration_readback, + ) + + readback = build_external_mcp_rag_integration_readback() + completion = readback.get("completion") or {} + runtime = readback.get("runtime") or {} + env_flags = runtime.get("env_flags") or {} + mcp_runtime = runtime.get("mcp") or {} + rag_runtime = runtime.get("rag") or {} + unresolved_count = int(completion.get("unresolved_count") or 0) + absorbed_count = int(completion.get("absorbed_count") or 0) + total_capabilities = int(completion.get("total_capabilities") or 0) + mcp_enabled = bool(env_flags.get("MCP_ROUTER_ENABLED") or mcp_runtime.get("enabled")) + rag_enabled = bool(env_flags.get("RAG_ENABLED") or rag_runtime.get("enabled")) + fully_integrated = readback.get("status") == "fully_integrated" + status = "ok" if fully_integrated and mcp_enabled and rag_enabled else "warning" + summary = ( + f"External MCP/RAG registry {absorbed_count}/{total_capabilities} absorbed; " + f"unresolved={unresolved_count}; runtime mcp={mcp_enabled} rag={rag_enabled}" + ) + return _check( + "External MCP/RAG integration readback", + status, + summary, + { + "policy": readback.get("policy"), + "integration_status": readback.get("status"), + "total_capabilities": total_capabilities, + "absorbed_count": absorbed_count, + "unresolved_count": unresolved_count, + "mcp_router_enabled": mcp_enabled, + "rag_enabled": rag_enabled, + "gemini_hard_disabled": bool(env_flags.get("GEMINI_API_HARD_DISABLED")), + "mcp_caller_count": int(mcp_runtime.get("caller_count") or 0), + "rag_vector_store": rag_runtime.get("vector_store"), + "next_machine_action": ( + "keep_external_mcp_rag_runtime_monitoring" + if status == "ok" + else "complete_mcp_router_health_and_rag_runtime_enablement" + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "External MCP/RAG integration readback", + "critical", + f"External MCP/RAG integration readback 無法執行：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + +def _pixelrag_rag_candidate_replay_check() -> Dict[str, Any]: + """Read-only sentinel for PixelRAG receipts becoming internal RAG candidates.""" + try: + from services.pixelrag_rag_candidate_replay_service import ( + build_pixelrag_rag_candidate_replay_readback, + ) + + readback = build_pixelrag_rag_candidate_replay_readback() + summary = readback.get("summary") or {} + receipt_count = int(summary.get("receipt_count") or 0) + eligible_count = int(summary.get("eligible_count") or 0) + blocked_count = int(summary.get("blocked_count") or 0) + invalid_count = int(summary.get("invalid_count") or 0) + visual_barrier_count = int(summary.get("visual_barrier_count") or 0) + status = readback.get("status") or "warning" + summary_text = ( + f"PixelRAG RAG candidate replay receipts={receipt_count}, " + f"eligible={eligible_count}, blocked={blocked_count}, invalid={invalid_count}, " + f"barriers={visual_barrier_count}" + ) + return _check( + "PixelRAG RAG candidate replay", + status, + summary_text, + { + "policy": readback.get("policy"), + "receipt_count": receipt_count, + "eligible_count": eligible_count, + "blocked_count": blocked_count, + "invalid_count": invalid_count, + "visual_barrier_count": visual_barrier_count, + "tile_file_count": int(summary.get("tile_file_count") or 0), + "missing_file_count": int(summary.get("missing_file_count") or 0), + "stale_count": int(summary.get("stale_count") or 0), + "blocked_pages_are_not_product_data": bool( + (readback.get("replay_contract") or {}).get("blocked_pages_are_not_product_data") + ), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG RAG candidate replay", + "critical", + f"PixelRAG RAG candidate replay 無法執行：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: int = 20) -> Dict[str, Any]: checks: List[Dict[str, Any]] = [ _event_router_check(), @@ -13095,6 +13285,8 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _ai_surface_html_readback_check(), _sitewide_ui_ux_agent_check(), _sitewide_visual_qa_check(), + _external_mcp_rag_integration_check(), + _pixelrag_rag_candidate_replay_check(), ] worst = max(checks, key=lambda item: STATUS_RANK.get(item["status"], 2))["status"] result = { diff --git a/services/pixelrag_rag_candidate_replay_service.py b/services/pixelrag_rag_candidate_replay_service.py new file mode 100644 index 0000000..c390e21 --- /dev/null +++ b/services/pixelrag_rag_candidate_replay_service.py @@ -0,0 +1,338 @@ +"""Read-only PixelRAG receipt to internal RAG candidate replay bridge. + +The bridge only converts saved visual evidence receipts into machine-readable +candidate payloads. It does not OCR, call a model, write ai_insights, or promote +anything into the production price tables. +""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.pixelrag_crawler_integration_service import ( + DEFAULT_ARTIFACT_MAX_AGE_HOURS, + DEFAULT_ARTIFACT_ROOT, + ECOMMERCE_PLATFORM_PROFILES, + _parse_iso_datetime, + _portable_receipt_file_path, + _receipt_candidates, + _safe_int, + _visual_barrier_reason, +) + + +POLICY = "read_only_pixelrag_rag_candidate_replay_v1" +INTERNAL_RAG_TARGET = "rag_service.learning_episode_candidate_preview" +DEFAULT_LIMIT = 50 + + +def _status_counts(items: list[dict[str, Any]]) -> dict[str, int]: + counts: dict[str, int] = {} + for item in items: + status = str(item.get("candidate_status") or "unknown") + counts[status] = counts.get(status, 0) + 1 + return counts + + +def _platform_from_receipt(receipt_path: Path, receipt: Mapping[str, Any]) -> str: + capture_target = receipt.get("capture_target") or {} + platform = str(capture_target.get("platform") or "").strip().lower() + if platform: + return platform + try: + return receipt_path.parent.parent.name + except IndexError: + return "unknown" + + +def _manifest_id_from_receipt(receipt_path: Path, receipt: Mapping[str, Any]) -> str: + return str(receipt.get("manifest_id") or receipt_path.parent.name or "").strip() + + +def _count_receipt_files(receipt_path: Path, receipt: Mapping[str, Any]) -> dict[str, Any]: + tile_file_count = 0 + missing_file_count = 0 + file_statuses: list[dict[str, Any]] = [] + for item in list(receipt.get("files") or []): + kind = item.get("kind") + path = _portable_receipt_file_path(receipt_path, item.get("path"), kind) + exists = path.exists() + if kind == "tile": + tile_file_count += 1 + if not exists: + missing_file_count += 1 + file_statuses.append({ + "kind": kind, + "path": str(path), + "exists": exists, + }) + return { + "tile_file_count": tile_file_count, + "missing_file_count": missing_file_count, + "files": file_statuses, + } + + +def _candidate_text( + *, + platform: str, + manifest_id: str, + receipt: Mapping[str, Any], + file_summary: Mapping[str, Any], + barrier_reason: str, +) -> str: + capture_target = receipt.get("capture_target") or {} + page_metrics = receipt.get("page_metrics") or {} + profile = ECOMMERCE_PLATFORM_PROFILES.get(platform) or {} + url = str(capture_target.get("url") or page_metrics.get("final_url") or "").strip() + title = str(page_metrics.get("title") or "").strip() + data_targets = ", ".join(str(item) for item in list(profile.get("data_targets") or [])[:8]) + parts = [ + f"platform={platform}", + f"manifest_id={manifest_id}", + f"title={title or 'unknown'}", + f"url={url or 'unknown'}", + f"http_status={_safe_int(receipt.get('http_status'))}", + f"capture_status={receipt.get('status') or 'unknown'}", + f"tile_file_count={int(file_summary.get('tile_file_count') or 0)}", + f"missing_file_count={int(file_summary.get('missing_file_count') or 0)}", + ] + if data_targets: + parts.append(f"intended_fields={data_targets}") + if barrier_reason: + parts.append(f"visual_barrier={barrier_reason}") + return " | ".join(parts) + + +def _candidate_from_receipt_path( + receipt_path: Path, + *, + now: datetime, + max_age_hours: int, +) -> dict[str, Any]: + errors: list[str] = [] + try: + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + receipt = {} + errors.append(str(exc)[:300]) + + platform = _platform_from_receipt(receipt_path, receipt) + manifest_id = _manifest_id_from_receipt(receipt_path, receipt) + generated_at = _parse_iso_datetime(receipt.get("generated_at")) + age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None + stale = age_hours is None or age_hours > max_age_hours + file_summary = _count_receipt_files(receipt_path, receipt) + missing_file_count = int(file_summary.get("missing_file_count") or 0) + barrier_reason = _visual_barrier_reason(receipt) + capture_status = str(receipt.get("status") or "").strip().lower() + http_status = _safe_int(receipt.get("http_status")) + + blocked_reasons: list[str] = [] + if errors: + blocked_reasons.append("receipt_parse_error") + if capture_status != "captured": + blocked_reasons.append("receipt_not_captured") + if stale: + blocked_reasons.append("receipt_stale_or_missing_timestamp") + if missing_file_count: + blocked_reasons.append("receipt_missing_files") + if barrier_reason: + blocked_reasons.append(f"visual_barrier:{barrier_reason}") + if http_status >= 400 and not barrier_reason: + blocked_reasons.append(f"http_status_{http_status}") + + eligible = not blocked_reasons + candidate_status = "eligible" if eligible else ("invalid" if errors else "blocked") + capture_target = receipt.get("capture_target") or {} + page_metrics = receipt.get("page_metrics") or {} + + return { + "candidate_status": candidate_status, + "eligible_for_rag_candidate_replay": eligible, + "blocked_reasons": blocked_reasons, + "platform": platform, + "manifest_id": manifest_id, + "receipt_path": str(receipt_path), + "generated_at": receipt.get("generated_at"), + "age_hours": round(age_hours, 3) if age_hours is not None else None, + "stale": stale, + "http_status": http_status, + "capture_status": receipt.get("status"), + "url": capture_target.get("url") or page_metrics.get("final_url"), + "title": page_metrics.get("title"), + "visual_barrier_detected": bool(barrier_reason), + "visual_barrier_reason": barrier_reason, + "tile_file_count": int(file_summary.get("tile_file_count") or 0), + "missing_file_count": missing_file_count, + "rag_candidate_text": _candidate_text( + platform=platform, + manifest_id=manifest_id, + receipt=receipt, + file_summary=file_summary, + barrier_reason=barrier_reason, + ), + "internal_rag_target": INTERNAL_RAG_TARGET, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "requires_ocr_or_vlm_replay": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": ( + "run_ocr_vlm_replay_then_promotion_gate" + if eligible + else ( + "run_platform_probe_or_use_structured_api" + if barrier_reason + else "refresh_pixelrag_visual_capture_receipt" + ) + ), + "file_summary": file_summary, + } + + +def _unique_receipt_candidates( + root: Path, + *, + platforms: tuple[str, ...] | list[str] | None, + limit: int, +) -> list[Path]: + seen: set[Path] = set() + candidates: list[Path] = [] + selected_platforms = [ + str(platform or "").strip().lower() + for platform in (platforms or []) + if str(platform or "").strip() + ] + if selected_platforms: + for platform in selected_platforms: + for path in _receipt_candidates(root, platform=platform): + resolved = path.resolve() + if resolved not in seen: + seen.add(resolved) + candidates.append(path) + else: + for path in _receipt_candidates(root): + resolved = path.resolve() + if resolved not in seen: + seen.add(resolved) + candidates.append(path) + return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] + + +def build_pixelrag_rag_candidate_replay_readback( + *, + artifact_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, +) -> dict[str, Any]: + """Build a replayable RAG candidate readback from PixelRAG receipts.""" + root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + if isinstance(platform, str): + platforms = (platform.strip().lower(),) if platform.strip() else None + else: + platforms = tuple( + str(item or "").strip().lower() + for item in (platform or []) + if str(item or "").strip() + ) or None + now = datetime.now(timezone.utc) + + candidates = _unique_receipt_candidates(root, platforms=platforms, limit=item_limit) + replay_candidates = [ + _candidate_from_receipt_path(path, now=now, max_age_hours=max_age) + for path in candidates + ] + eligible_count = sum(1 for item in replay_candidates if item["candidate_status"] == "eligible") + blocked_count = sum(1 for item in replay_candidates if item["candidate_status"] == "blocked") + invalid_count = sum(1 for item in replay_candidates if item["candidate_status"] == "invalid") + visual_barrier_count = sum(1 for item in replay_candidates if item["visual_barrier_detected"]) + missing_file_count = sum(int(item.get("missing_file_count") or 0) for item in replay_candidates) + tile_file_count = sum(int(item.get("tile_file_count") or 0) for item in replay_candidates) + stale_count = sum(1 for item in replay_candidates if item.get("stale")) + + if invalid_count and invalid_count == len(replay_candidates): + status = "critical" + elif eligible_count == 0 or blocked_count or invalid_count or visual_barrier_count: + status = "warning" + else: + status = "ok" + + if not replay_candidates: + status = "warning" + + summary = { + "receipt_count": len(replay_candidates), + "eligible_count": eligible_count, + "blocked_count": blocked_count, + "invalid_count": invalid_count, + "visual_barrier_count": visual_barrier_count, + "tile_file_count": tile_file_count, + "missing_file_count": missing_file_count, + "stale_count": stale_count, + "status_counts": _status_counts(replay_candidates), + "platforms": sorted({str(item.get("platform") or "unknown") for item in replay_candidates}), + } + + return { + "success": invalid_count == 0, + "policy": POLICY, + "status": status, + "generated_at": now.isoformat(), + "artifact_root": str(root), + "max_age_hours": max_age, + "platform_filter": list(platforms or []), + "summary": summary, + "candidates": replay_candidates, + "replay_contract": { + "internal_rag_target": INTERNAL_RAG_TARGET, + "writes_database": False, + "writes_ai_insights": False, + "writes_price_tables": False, + "network_call": False, + "secret_read": False, + "promotion_gate_required_before_knowledge_write": True, + "ocr_or_vlm_required_before_field_extraction": True, + "blocked_pages_are_not_product_data": True, + }, + "next_machine_action": ( + "run_pixelrag_visual_capture_worker" + if not replay_candidates + else ( + "fix_invalid_pixelrag_receipts" + if invalid_count and not eligible_count + else ( + "run_platform_probe_or_use_structured_api" + if visual_barrier_count and not eligible_count + else ( + "run_ocr_vlm_replay_then_promotion_gate" + if eligible_count + else "refresh_pixelrag_visual_capture_receipt" + ) + ) + ) + ), + "controlled_apply": { + "network_call": False, + "db_write": False, + "secret_read": False, + "production_price_write": False, + "artifact_write": False, + }, + } + + +__all__ = [ + "INTERNAL_RAG_TARGET", + "POLICY", + "build_pixelrag_rag_candidate_replay_readback", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index 3031bc7..3a7bc6b 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1431,11 +1431,13 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_ai_surface_html_readback_check", lambda: smoke._check("surface", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_ui_ux_agent_check", lambda: smoke._check("sitewide", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) + monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 28, "warning": 1, "critical": 1, "total": 30} + assert result["summary"] == {"ok": 31, "warning": 1, "critical": 1, "total": 33} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3962,6 +3964,8 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_ai_surface_html_readback_check", lambda: smoke._check("surface", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_ui_ux_agent_check", lambda: smoke._check("sitewide", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) + monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) first = smoke.collect_ai_automation_smoke(history_limit=5) second = smoke.collect_ai_automation_smoke(history_limit=5) @@ -4017,7 +4021,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 30, "warning": 0, "critical": 0, "total": 30}, + "summary": {"ok": 33, "warning": 0, "critical": 0, "total": 33}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4078,10 +4082,10 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "repair_action_count": 0, }, }, - { - "name": "Sitewide visual QA readback", - "status": "ok", - "summary": "Sitewide visual QA ok，12 routes x 3 viewports 無 overflow 退化", + { + "name": "Sitewide visual QA readback", + "status": "ok", + "summary": "Sitewide visual QA ok，12 routes x 3 viewports 無 overflow 退化", "details": { "policy": "read_only_sitewide_visual_qa_readback_v1", "result_count": 36, @@ -4092,11 +4096,42 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "visual_offender_count": 0, "artifact_path": "data/ai_automation/sitewide_visual_qa_latest.json", "artifact_generated_at": datetime.now().isoformat(timespec="seconds"), - "stale": False, + "stale": False, + }, }, - } - ], - }, ensure_ascii=False) + "\n", + { + "name": "External MCP/RAG integration readback", + "status": "ok", + "summary": "External MCP/RAG registry 9/9 absorbed; unresolved=0; runtime mcp=True rag=True", + "details": { + "policy": "read_only_external_mcp_rag_integration_readback_v1", + "integration_status": "fully_integrated", + "total_capabilities": 9, + "absorbed_count": 9, + "unresolved_count": 0, + "mcp_router_enabled": True, + "rag_enabled": True, + "gemini_hard_disabled": True, + "next_machine_action": "keep_external_mcp_rag_runtime_monitoring", + }, + }, + { + "name": "PixelRAG RAG candidate replay", + "status": "ok", + "summary": "PixelRAG RAG candidate replay receipts=1, eligible=1, blocked=0, invalid=0, barriers=0", + "details": { + "policy": "read_only_pixelrag_rag_candidate_replay_v1", + "receipt_count": 1, + "eligible_count": 1, + "blocked_count": 0, + "invalid_count": 0, + "visual_barrier_count": 0, + "blocked_pages_are_not_product_data": True, + "next_machine_action": "run_ocr_vlm_replay_then_promotion_gate", + }, + } + ], + }, ensure_ascii=False) + "\n", encoding="utf-8", ) monkeypatch.setattr(smoke, "_HISTORY_PATH", str(history_path)) @@ -4112,7 +4147,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 27 + assert summary["summary"]["total"] == 30 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -5776,6 +5811,18 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( assert visual_family["status"] == "ok" assert visual_family["details"]["route_count"] == 12 assert visual_family["details"]["failed_count"] == 0 + external_mcp_rag_family = next( + item for item in summary["families"] + if item["key"] == "external_mcp_rag_integration" + ) + assert external_mcp_rag_family["status"] == "ok" + assert external_mcp_rag_family["details"]["unresolved_count"] == 0 + pixelrag_rag_replay_family = next( + item for item in summary["families"] + if item["key"] == "pixelrag_rag_candidate_replay" + ) + assert pixelrag_rag_replay_family["status"] == "ok" + assert pixelrag_rag_replay_family["details"]["eligible_count"] == 1 assert summary["scheduled_outputs"]["telegram_send_in_preview"] is False assert summary["scheduled_outputs"]["writes_database"] is False assert summary["automation_policy"]["primary_flow"] == "ai_controlled" @@ -6382,6 +6429,16 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "visual ok", {"route_count": 12, "viewport_count": 3, "failed_count": 0}, )) + monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check( + "External MCP/RAG integration readback", + "ok", + "external mcp rag ok", + )) + monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check( + "PixelRAG RAG candidate replay", + "ok", + "pixelrag replay ok", + )) result = smoke.collect_ai_automation_smoke(record_history=False) @@ -6397,7 +6454,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 30 + assert result["summary"]["total"] == 33 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_rag_candidate_replay_service.py b/tests/test_pixelrag_rag_candidate_replay_service.py new file mode 100644 index 0000000..de9628a --- /dev/null +++ b/tests/test_pixelrag_rag_candidate_replay_service.py @@ -0,0 +1,138 @@ +import json +import subprocess +import sys +from datetime import datetime, timezone + + +def _write_receipt(root, *, platform, manifest_id, title, url, http_status=200): + receipt_dir = root / platform / manifest_id + tiles_dir = receipt_dir / "tiles" + tiles_dir.mkdir(parents=True) + screenshot = receipt_dir / "screenshot.png" + screenshot.write_bytes(b"png") + files = [{"kind": "screenshot", "path": str(screenshot)}] + for index in range(4): + tile = tiles_dir / f"tile-{index}.png" + tile.write_bytes(b"tile") + files.append({"kind": "tile", "path": str(tile)}) + receipt = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "status": "captured", + "manifest_id": manifest_id, + "http_status": http_status, + "capture_target": { + "platform": platform, + "url": url, + "crawler": "PixelRAGMarketplaceSearch.visual_fallback", + }, + "page_metrics": { + "title": title, + "final_url": url, + }, + "tile_plan": { + "planned_tile_count": 4, + "emitted_tile_count": 4, + }, + "files": files, + } + path = receipt_dir / "capture_receipt.json" + path.write_text(json.dumps(receipt, ensure_ascii=False), encoding="utf-8") + return path + + +def test_pixelrag_rag_candidate_replay_splits_eligible_and_blocked_receipts(tmp_path): + from services.pixelrag_rag_candidate_replay_service import ( + POLICY, + build_pixelrag_rag_candidate_replay_readback, + ) + + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=%E9%98%B2%E6%9B%AC%E4%B9%B3", + ) + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + + payload = build_pixelrag_rag_candidate_replay_readback(artifact_root=tmp_path) + by_platform = {item["platform"]: item for item in payload["candidates"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "warning" + assert payload["summary"]["receipt_count"] == 2 + assert payload["summary"]["eligible_count"] == 1 + assert payload["summary"]["blocked_count"] == 1 + assert payload["summary"]["visual_barrier_count"] == 1 + assert by_platform["shopee_tw"]["eligible_for_rag_candidate_replay"] is True + assert by_platform["shopee_tw"]["promotion_boundary"]["direct_ai_insights_write_allowed"] is False + assert "Shopee 防曬乳" in by_platform["shopee_tw"]["rag_candidate_text"] + assert by_platform["coupang_tw"]["eligible_for_rag_candidate_replay"] is False + assert by_platform["coupang_tw"]["visual_barrier_reason"] == "http_status_403" + assert by_platform["coupang_tw"]["next_machine_action"] == "run_platform_probe_or_use_structured_api" + assert payload["replay_contract"]["blocked_pages_are_not_product_data"] is True + assert payload["controlled_apply"]["db_write"] is False + + +def test_pixelrag_rag_candidate_replay_cli_outputs_machine_readable_json(tmp_path): + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=sunscreen", + ) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/report_pixelrag_rag_candidate_replay.py", + "--artifact-root", + str(tmp_path), + "--platform", + "shopee_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["receipt_count"] == 1 + assert payload["summary"]["eligible_count"] == 1 + assert payload["candidates"][0]["platform"] == "shopee_tw" + + +def test_pixelrag_rag_candidate_replay_route_returns_readback(tmp_path, monkeypatch): + from flask import Flask + from routes import system_public_routes as routes + from services import pixelrag_rag_candidate_replay_service as service + + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=sunscreen", + ) + monkeypatch.setattr(service, "DEFAULT_ARTIFACT_ROOT", str(tmp_path)) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-rag-candidate-replay?platform=shopee_tw" + ): + response = routes.ai_automation_pixelrag_rag_candidate_replay_api.__wrapped__() + payload = response.get_json() + + assert payload["policy"] == "read_only_pixelrag_rag_candidate_replay_v1" + assert payload["summary"]["receipt_count"] == 1 + assert payload["summary"]["eligible_count"] == 1 + assert payload["replay_contract"]["writes_database"] is False From a72713af1e9dab3a6d7232b7d0403b3c58d24772 Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 23:01:54 +0800 Subject: [PATCH 05/75] =?UTF-8?q?feat(ai):=20=E5=BB=BA=E7=AB=8B=20PixelRAG?= =?UTF-8?q?=20OCR=20VLM=20replay=20contract?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO_NEXT_STEPS.txt | 7 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 6 +- .../ai_automation_mainline_work_items.md | 7 +- docs/guides/ai_automation_session_sop.md | 4 +- docs/guides/browse_sh_crawler_playbook.md | 9 + routes/system_public_routes.py | 22 ++ scripts/ops/report_pixelrag_ocr_vlm_replay.py | 60 +++ services/ai_automation_smoke_service.py | 99 +++++ services/pixelrag_ocr_vlm_replay_service.py | 354 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 39 +- tests/test_pixelrag_ocr_vlm_replay_service.py | 144 +++++++ 12 files changed, 739 insertions(+), 14 deletions(-) create mode 100755 scripts/ops/report_pixelrag_ocr_vlm_replay.py create mode 100644 services/pixelrag_ocr_vlm_replay_service.py create mode 100644 tests/test_pixelrag_ocr_vlm_replay_service.py diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 198203b..94ef45c 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -9,7 +9,7 @@ python scripts/ops/check_production_version_truth.py 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 -本輪 source target 為 `V10.746`；部署完成前不得宣稱正式環境已是 `V10.746`。 +本輪 source target 為 `V10.747`；部署完成前不得宣稱正式環境已是 `V10.747`。 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ @@ -58,8 +58,9 @@ P6. DB apply authorization / signing / closeout P0-2026-07-09. PixelRAG / MCP / RAG 全自動主線 - 已完成：多電商 PixelRAG visual evidence lane 與 external MCP/RAG integration readback。 - - 進行中：PixelRAG receipt → internal RAG candidate replay，以及 MCP/RAG runtime health → AI automation smoke。 - - 未開始：OCR/VLM replay、Ollama-first visual embedding benchmark、pgvector-compatible visual metadata、Coupang platform probe / structured API、跨平台 source contracts。 + - 已完成：PixelRAG receipt → internal RAG candidate replay，以及 OCR/VLM replay contract no-write readback。 + - 進行中：MCP/RAG runtime health → AI automation smoke。 + - 未開始：Ollama-first OCR/VLM extraction worker、Ollama-first visual embedding benchmark、pgvector-compatible visual metadata、Coupang platform probe / structured API、跨平台 source contracts。 - 主線文件：`docs/guides/ai_automation_mainline_work_items.md`。 ================================================================================ diff --git a/config.py b/config.py index fc12f72..5c407ff 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.746" +SYSTEM_VERSION = "V10.747" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index a34c03d..0e87108 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,8 +1,8 @@ # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth > **最後更新**: 2026-07-09 (台北時間) -> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.746 +> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 +> **適用版本**: V10.747 --- @@ -114,6 +114,7 @@ - 2026-07-03 起整站 desktop/tablet/mobile visual QA 必須進入 AI automation runtime monitoring：`scripts/check_responsive_overflow.js --artifact-output data/ai_automation/sitewide_visual_qa_latest.json` 會輸出機器可讀 artifact；`/api/ai-automation/sitewide-visual-qa-readback` 只讀該 artifact 並回報 route_count、viewport_count、failed_count、overflow_issue_count、stale 與 next_machine_action；`/api/ai-automation/smoke` 需包含 `Sitewide visual QA readback`，`/api/ai-automation/scheduled-health-summary` 需輸出 `sitewide_visual_qa` family，`/metrics` 需暴露 `momo_ai_automation_scheduled_health_family_status{family="sitewide_visual_qa",...}`。readback 不啟動 browser、不寫 DB、不送通知；artifact 生成才需要 Playwright/瀏覽器。 - 2026-07-09 起 external MCP/RAG integration 必須進入 AI automation runtime monitoring：`/api/ai-automation/external-mcp-rag-integration` 與 `scripts/ops/report_external_mcp_rag_integration.py` 需輸出 9 個 capability 的 absorbed / unresolved 狀態、內部治理落點與 runtime flags；`/api/ai-automation/smoke` 需包含 `External MCP/RAG integration readback`，`/api/ai-automation/scheduled-health-summary` 需輸出 `external_mcp_rag_integration` family。`MCP_ROUTER_ENABLED=false` 或 `RAG_ENABLED=false` 必須被視為 runtime gap，不得因 registry 已完成就宣稱 MCP/RAG runtime 已全面啟用。 - 2026-07-09 起 PixelRAG visual receipts 必須可轉為內部 RAG candidate replay readback：`/api/ai-automation/pixelrag-rag-candidate-replay` 與 `scripts/ops/report_pixelrag_rag_candidate_replay.py` 只讀 `capture_receipt.json`，拆分 eligible / blocked / invalid receipts，輸出 tile/missing/barrier count、candidate text、promotion boundary 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG RAG candidate replay`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_rag_candidate_replay` family。blocked / 403 / captcha / access denied / verify traffic page 只能進 platform probe 或 structured API 策略，不得當作商品資料、不得寫正式價格表、不得直接寫 `ai_insights`。 +- 2026-07-09 起 PixelRAG visual receipts 進入 OCR/VLM 前必須先輸出 no-write replay contract：`/api/ai-automation/pixelrag-ocr-vlm-replay` 與 `scripts/ops/report_pixelrag_ocr_vlm_replay.py` 只讀 saved tiles，輸出欄位 schema、輸出 schema、confidence/evidence validation rules、Ollama-first route contract 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG OCR/VLM replay contract`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_ocr_vlm_replay` family。此 contract 不執行 OCR/VLM、不呼叫模型、不讀 secret、不連外、不寫 DB、不寫 `ai_insights`、不寫正式價格表；blocked / 403 / captcha / access denied / verify traffic page 只能進 platform probe 或 structured API 策略。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -874,6 +875,7 @@ POSTGRES_HOST=momo-db | 2026-07-03 | PChome auto-policy controlled dry-run package 必須有 runtime monitoring | V10.744 起 `/api/ai-automation/smoke`、`/api/ai-automation/scheduled-health-summary` 與 `/metrics` 必須輸出 `PChome auto-policy controlled dry-run package` / `pchome_auto_policy_controlled_dry_run_package`；此 runtime check 自動驗證 controlled apply final preflight carry-forward、rollback binding、post-apply verifier binding、12 個 dry-run package checks、12 個 package fields、10 個 acceptance gates、receipt preview、result parser、non-executable command shape、same-run production truth 與 future dry-run receipt closeout readiness，但明確標記 `receipt_execution_performed=false`、`receipt_stdout_included=false`、`receipt_stderr_included=false`、`command_shape_execution_allowed=false`、`database_apply_authorized=false`、`reads_secret_count=0`、`executes_script_count=0`、`executes_sql_count=0`、`writes_database_count=0`、`signs_database_apply_authorization_count=0`、`primary_human_gate_count=0`、`executes_authorization_evidence=false`、`executes_database_apply=false`、`ready_for_database_apply_now=false`、`issues_database_apply_authorization=false`、`signs_database_apply_authorization=false`，不讀 secret、不執行 dry-run、不捕捉 stdout/stderr、不執行 authorization evidence、不執行 database apply、不執行 endpoint/SQL、不寫 DB、不簽發 apply authorization，只允許進入下一段 dry-run receipt closeout lane。 | | 2026-07-09 | PChome auto-policy controlled dry-run receipt closeout 必須有 runtime monitoring | V10.745 起 `/api/ai-automation/smoke`、`/api/ai-automation/scheduled-health-summary` 與 `/metrics` 必須輸出 `PChome auto-policy controlled dry-run receipt closeout` / `pchome_auto_policy_controlled_dry_run_receipt_closeout`；此 runtime check 自動驗證 controlled dry-run package carry-forward、dry-run result parser、receipt validation report、command-shape hash、rollback binding、post-apply verifier binding、12 個 receipt closeout checks、12 個 closeout fields、10 個 acceptance gates、same-run production truth 與 future runner readiness，但明確標記 `receipt_validation_status=preview_validated_not_executed`、`validation_execution_performed=false`、`validation_stdout_included=false`、`validation_stderr_included=false`、`validation_database_apply_authorized=false`、`reads_secret_count=0`、`executes_script_count=0`、`executes_sql_count=0`、`writes_database_count=0`、`signs_database_apply_authorization_count=0`、`primary_human_gate_count=0`、`executes_authorization_evidence=false`、`executes_database_apply=false`、`ready_for_database_apply_now=false`、`database_apply_authorized=false`、`issues_database_apply_authorization=false`、`signs_database_apply_authorization=false`，不讀 secret、不執行 dry-run、不捕捉 stdout/stderr、不執行 authorization evidence、不執行 database apply、不執行 endpoint/SQL、不寫 DB、不簽發 apply authorization，只允許進入下一段 controlled dry-run runner readiness lane。 | | 2026-07-09 | PixelRAG / external MCP/RAG 必須有 runtime monitoring 與 candidate replay | V10.746 起 `/api/ai-automation/smoke`、`/api/ai-automation/scheduled-health-summary` 必須輸出 `External MCP/RAG integration readback` / `external_mcp_rag_integration` 與 `PixelRAG RAG candidate replay` / `pixelrag_rag_candidate_replay`；external MCP/RAG readback 回報 9 個 capability 的 absorbed / unresolved 與 `MCP_ROUTER_ENABLED`、`RAG_ENABLED` runtime flags，PixelRAG replay 只讀 visual receipt，拆分 eligible / blocked / invalid，並明確標記 blocked page 不是商品資料。此路徑不讀 secret、不呼叫外部網路、不寫 DB、不寫 `ai_insights`、不寫正式價格表；eligible receipt 仍需 OCR/VLM replay、identity matcher replay、PromotionGate 與 embedding signature guard。 | +| 2026-07-09 | PixelRAG OCR/VLM replay contract 必須有 runtime monitoring | V10.747 起 `/api/ai-automation/pixelrag-ocr-vlm-replay`、`scripts/ops/report_pixelrag_ocr_vlm_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 no-write OCR/VLM replay contract / `pixelrag_ocr_vlm_replay` family；readback 只讀 saved tiles 與 RAG candidate replay，輸出 ready / blocked / invalid contracts、field schema、output schema、validation rules、Ollama-first route contract、blocked page guard 與 next machine action。此階段明確標記 `extraction_execution_performed=false`、`ocr_execution_performed=false`、`vlm_execution_performed=false`、`writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`network_call=false`、`secret_read=false`、`primary_human_gate_count=0`；ready receipt 才能進下一段 `run_ollama_first_vlm_replay_worker`，blocked receipt 只能進 platform probe 或 structured API 策略。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index d93ca16..4797b81 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -16,15 +16,16 @@ |---|---|---| | Completed | Multi-commerce PixelRAG visual evidence lane for momo, pchome, shopee_tw, coupang_tw, yahoo_shopping_tw, etmall_tw, friday_tw, rakuten_tw | API and CLI can emit expansion plans and marketplace manifests; PChome parser failures can queue manifests. | | Completed | External MCP/RAG capability inventory absorbed into internal governance readback | `/api/ai-automation/external-mcp-rag-integration` and `scripts/ops/report_external_mcp_rag_integration.py` expose 9 capabilities, absorbed/unresolved counts, and runtime flags. | -| In progress | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read receipts, split eligible vs blocked, and require OCR/VLM replay plus PromotionGate before knowledge writes. | -| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration and PixelRAG RAG candidate replay families. | +| Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read receipts, split eligible vs blocked, and require OCR/VLM replay plus PromotionGate before knowledge writes. | +| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, and PixelRAG OCR/VLM replay contract families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | ## P1 | Status | Work item | Evidence / next machine action | |---|---|---| -| Not started | OCR/VLM replay contract for visual fields | Build a no-write replay that extracts title, price, promo badge, seller, rating, delivery badge from saved tiles and emits confidence/evidence only. | +| Completed | OCR/VLM replay contract for visual fields | `/api/ai-automation/pixelrag-ocr-vlm-replay` and `scripts/ops/report_pixelrag_ocr_vlm_replay.py` turn saved tiles into no-write field contracts, output schemas, validation rules, and Ollama-first worker actions. | +| Not started | Ollama-first OCR/VLM extraction worker | Execute the replay contract against approved local multimodal/OCR models and emit confidence/evidence receipts without writing formal price truth. | | Not started | Ollama-first multimodal embedding benchmark | Verify local Qwen3-VL or equivalent visual embedding on GCP-A -> GCP-B -> 111 before any visual vector retrieval. | | Not started | pgvector-compatible visual evidence metadata | Design metadata-first retrieval without FAISS in production unless ADR approves a different store. | | Not started | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; prefer public structured data or approved probe, never count blocked pages as product data. | diff --git a/docs/guides/ai_automation_session_sop.md b/docs/guides/ai_automation_session_sop.md index 69733c7..5454311 100644 --- a/docs/guides/ai_automation_session_sop.md +++ b/docs/guides/ai_automation_session_sop.md @@ -30,7 +30,9 @@ 或 `python scripts/ops/report_external_mcp_rag_integration.py` 可讀回每個能力的內部落點、狀態、資料邊界與下一個機器動作。 - PixelRAG 視覺 receipt 導入內部 RAG 時，必須確認 `/api/ai-automation/pixelrag-rag-candidate-replay` 或 `python scripts/ops/report_pixelrag_rag_candidate_replay.py` 可讀回 eligible / blocked / invalid receipts，且 blocked page 不得被視為商品資料。 -- AI automation smoke 必須包含 external MCP/RAG integration 與 PixelRAG RAG candidate replay family，避免 registry 已完成但 runtime flag / receipt replay 未完成時被誤報為全自動閉環。 +- PixelRAG 視覺 receipt 進入 OCR/VLM 自動化前，必須確認 `/api/ai-automation/pixelrag-ocr-vlm-replay` + 或 `python scripts/ops/report_pixelrag_ocr_vlm_replay.py` 可讀回 ready / blocked / invalid replay contracts、field schema、validation rules 與 Ollama-first 下一步，且目前不執行 OCR/VLM、不寫正式價格。 +- AI automation smoke 必須包含 external MCP/RAG integration、PixelRAG RAG candidate replay 與 PixelRAG OCR/VLM replay contract family，避免 registry 已完成但 runtime flag / receipt replay / VLM contract 未完成時被誤報為全自動閉環。 - AI 自動化 Prometheus 指標變更必須同步檢查 `docker/grafana/provisioning/dashboards/json/ai-automation-overview.json` 是否需要新增 panel 或查詢。 - 188 線上 active monitoring stack 以 `monitoring/prometheus.yml` 為準；110 gateway 另有 `/home/wooo/monitoring/prometheus.yml`。若 dashboard 無資料，先確認 Prometheus `momo-app` target 與 `momo-network` 連線；所有 Blackbox HTTP target 必須打 `/health`，不可打 Dashboard 首頁 `/`。 - Smoke dashboard 會保存 JSONL 趨勢；若新增檢查項目，要確保 history compact record 仍保持小而可讀。 diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index 0e8f8f0..7f914df 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -171,6 +171,15 @@ python scripts/ops/report_pixelrag_rag_candidate_replay.py --platform shopee_tw 此 replay 只把 receipt 轉成內部 RAG candidate preview；eligible receipt 仍需 OCR/VLM replay、identity matcher replay、PromotionGate 與 embedding signature guard，blocked / 403 / captcha / access denied receipt 只能進 platform probe 或 structured API 策略，不得當成商品資料。 +OCR/VLM replay contract： + +```bash +python scripts/ops/report_pixelrag_ocr_vlm_replay.py +python scripts/ops/report_pixelrag_ocr_vlm_replay.py --platform shopee_tw --platform coupang_tw +``` + +此 contract 只把 saved tiles、欄位 schema、輸出 schema、confidence/evidence 規則與 Ollama-first worker 下一步打包；目前不執行 OCR/VLM、不呼叫模型、不寫 RAG、不寫 `ai_insights`、不寫正式價格表。ready receipt 進 `run_ollama_first_vlm_replay_worker`，blocked / 403 / captcha / access denied receipt 進 platform probe 或 structured API 策略。 + 安全邊界： - read-only；不登入、不下單、不加入購物車、不寫第三方狀態。 diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 5f61ccb..5eb73e7 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -711,6 +711,28 @@ def ai_automation_pixelrag_rag_candidate_replay_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-ocr-vlm-replay') +@login_required +def ai_automation_pixelrag_ocr_vlm_replay_api(): + """Read-only PixelRAG OCR/VLM replay contract readback.""" + from services.pixelrag_ocr_vlm_replay_service import ( + build_pixelrag_ocr_vlm_replay_contract, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 50, type=int) + return jsonify(build_pixelrag_ocr_vlm_replay_contract( + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 50, 250)), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/report_pixelrag_ocr_vlm_replay.py b/scripts/ops/report_pixelrag_ocr_vlm_replay.py new file mode 100755 index 0000000..a706e52 --- /dev/null +++ b/scripts/ops/report_pixelrag_ocr_vlm_replay.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Report PixelRAG receipts as OCR/VLM replay contracts.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_ocr_vlm_replay_service import ( # noqa: E402 + build_pixelrag_ocr_vlm_replay_contract, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="輸出 PixelRAG receipt 的 no-write OCR/VLM replay contract。" + ) + parser.add_argument( + "--artifact-root", + help="PixelRAG visual evidence artifact root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=50, + help="最多輸出 receipt contract 數。", + ) + args = parser.parse_args() + + payload = build_pixelrag_ocr_vlm_replay_contract( + artifact_root=args.artifact_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index 31d5715..88990f8 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -537,6 +537,11 @@ def build_scheduled_automation_health_summary( if not pixelrag_rag_replay or not pixelrag_rag_replay_details: pixelrag_rag_replay = _pixelrag_rag_candidate_replay_check() pixelrag_rag_replay_details = pixelrag_rag_replay.get("details") or {} + pixelrag_ocr_vlm_replay = _find_check(source_result, "PixelRAG OCR/VLM replay contract") + pixelrag_ocr_vlm_replay_details = pixelrag_ocr_vlm_replay.get("details") or {} + if not pixelrag_ocr_vlm_replay or not pixelrag_ocr_vlm_replay_details: + pixelrag_ocr_vlm_replay = _pixelrag_ocr_vlm_replay_check() + pixelrag_ocr_vlm_replay_details = pixelrag_ocr_vlm_replay.get("details") or {} smoke_status = source_result.get("status") or ("warning" if latest_history else "warning") freshness_family = _history_freshness_family( latest_history, @@ -4592,6 +4597,41 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_ocr_vlm_replay", + "label": "PixelRAG OCR/VLM replay contract", + "status": pixelrag_ocr_vlm_replay.get("status") or "warning", + "summary": ( + pixelrag_ocr_vlm_replay.get("summary") + or "PixelRAG OCR/VLM replay contract has no latest readback." + ), + "next_machine_action": pixelrag_ocr_vlm_replay_details.get("next_machine_action") + or "run_pixelrag_ocr_vlm_replay_contract_readback", + "details": { + "policy": pixelrag_ocr_vlm_replay_details.get("policy"), + "receipt_count": int(pixelrag_ocr_vlm_replay_details.get("receipt_count") or 0), + "replay_ready_count": int( + pixelrag_ocr_vlm_replay_details.get("replay_ready_count") or 0 + ), + "blocked_count": int(pixelrag_ocr_vlm_replay_details.get("blocked_count") or 0), + "invalid_count": int(pixelrag_ocr_vlm_replay_details.get("invalid_count") or 0), + "tile_input_count": int( + pixelrag_ocr_vlm_replay_details.get("tile_input_count") or 0 + ), + "field_contract_count": int( + pixelrag_ocr_vlm_replay_details.get("field_contract_count") or 0 + ), + "extraction_execution_performed": bool( + pixelrag_ocr_vlm_replay_details.get("extraction_execution_performed") + ), + "blocked_pages_are_not_product_data": bool( + pixelrag_ocr_vlm_replay_details.get("blocked_pages_are_not_product_data") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, freshness_family, { "key": "daily_summary_delivery", @@ -13252,6 +13292,64 @@ def _pixelrag_rag_candidate_replay_check() -> Dict[str, Any]: ) +def _pixelrag_ocr_vlm_replay_check() -> Dict[str, Any]: + """Read-only sentinel for PixelRAG receipts becoming OCR/VLM replay contracts.""" + try: + from services.pixelrag_ocr_vlm_replay_service import ( + build_pixelrag_ocr_vlm_replay_contract, + ) + + readback = build_pixelrag_ocr_vlm_replay_contract() + summary = readback.get("summary") or {} + receipt_count = int(summary.get("receipt_count") or 0) + replay_ready_count = int(summary.get("replay_ready_count") or 0) + blocked_count = int(summary.get("blocked_count") or 0) + invalid_count = int(summary.get("invalid_count") or 0) + field_contract_count = int(summary.get("field_contract_count") or 0) + status = readback.get("status") or "warning" + summary_text = ( + f"PixelRAG OCR/VLM replay contract receipts={receipt_count}, " + f"ready={replay_ready_count}, blocked={blocked_count}, invalid={invalid_count}, " + f"field_contracts={field_contract_count}" + ) + return _check( + "PixelRAG OCR/VLM replay contract", + status, + summary_text, + { + "policy": readback.get("policy"), + "receipt_count": receipt_count, + "replay_ready_count": replay_ready_count, + "blocked_count": blocked_count, + "invalid_count": invalid_count, + "tile_input_count": int(summary.get("tile_input_count") or 0), + "missing_tile_count": int(summary.get("missing_tile_count") or 0), + "field_contract_count": field_contract_count, + "extraction_execution_performed": bool( + summary.get("extraction_execution_performed") + ), + "blocked_pages_are_not_product_data": bool( + (readback.get("replay_contract") or {}).get("blocked_pages_are_not_product_data") + ), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG OCR/VLM replay contract", + "critical", + f"PixelRAG OCR/VLM replay contract 無法執行：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: int = 20) -> Dict[str, Any]: checks: List[Dict[str, Any]] = [ _event_router_check(), @@ -13287,6 +13385,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _sitewide_visual_qa_check(), _external_mcp_rag_integration_check(), _pixelrag_rag_candidate_replay_check(), + _pixelrag_ocr_vlm_replay_check(), ] worst = max(checks, key=lambda item: STATUS_RANK.get(item["status"], 2))["status"] result = { diff --git a/services/pixelrag_ocr_vlm_replay_service.py b/services/pixelrag_ocr_vlm_replay_service.py new file mode 100644 index 0000000..16891bb --- /dev/null +++ b/services/pixelrag_ocr_vlm_replay_service.py @@ -0,0 +1,354 @@ +"""Read-only PixelRAG OCR/VLM replay contract builder. + +This module turns PixelRAG visual evidence receipts into a machine-readable +contract for an Ollama-first OCR/VLM worker. It does not execute OCR, call a +model, write RAG data, write price tables, or promote visual fields. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.pixelrag_crawler_integration_service import ( + DEFAULT_ARTIFACT_MAX_AGE_HOURS, + DEFAULT_ARTIFACT_ROOT, + ECOMMERCE_PLATFORM_PROFILES, +) +from services.pixelrag_rag_candidate_replay_service import ( + build_pixelrag_rag_candidate_replay_readback, +) + + +POLICY = "read_only_pixelrag_ocr_vlm_replay_contract_v1" +DEFAULT_LIMIT = 50 +DEFAULT_CONFIDENCE_THRESHOLD = 0.72 + +FIELD_ALIASES: dict[str, dict[str, str]] = { + "title": {"type": "string", "required": "true", "evidence": "tile_text_or_card_region"}, + "price": {"type": "money", "required": "true", "evidence": "price_region"}, + "currency": {"type": "string", "required": "false", "evidence": "price_region"}, + "bundle_spec": {"type": "string", "required": "false", "evidence": "spec_or_bundle_region"}, + "promo_badge": {"type": "string", "required": "false", "evidence": "badge_region"}, + "campaign_badge": {"type": "string", "required": "false", "evidence": "badge_region"}, + "stock_hint": {"type": "string", "required": "false", "evidence": "availability_region"}, + "image": {"type": "asset_ref", "required": "false", "evidence": "screenshot_or_tile"}, + "product_id": {"type": "string", "required": "false", "evidence": "url_or_card_region"}, + "shop": {"type": "string", "required": "false", "evidence": "seller_region"}, + "sold_count": {"type": "number_or_label", "required": "false", "evidence": "social_proof_region"}, + "rating": {"type": "number_or_label", "required": "false", "evidence": "rating_region"}, + "shipping_badge": {"type": "string", "required": "false", "evidence": "shipping_region"}, + "discount": {"type": "string", "required": "false", "evidence": "discount_region"}, + "delivery_badge": {"type": "string", "required": "false", "evidence": "delivery_region"}, + "spec": {"type": "string", "required": "false", "evidence": "spec_region"}, + "review_count": {"type": "number_or_label", "required": "false", "evidence": "review_region"}, + "coupon": {"type": "string", "required": "false", "evidence": "coupon_region"}, + "sales_hint": {"type": "string", "required": "false", "evidence": "social_proof_region"}, + "member_price": {"type": "money", "required": "false", "evidence": "member_price_region"}, + "shipping_hint": {"type": "string", "required": "false", "evidence": "shipping_region"}, + "reward_badge": {"type": "string", "required": "false", "evidence": "reward_region"}, + "point_reward": {"type": "string", "required": "false", "evidence": "reward_region"}, + "public_page_visual_evidence": { + "type": "evidence_summary", + "required": "true", + "evidence": "visible_page_regions", + }, +} + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...] | None: + if isinstance(platform, str): + return (platform.strip().lower(),) if platform.strip() else None + selected = tuple( + str(item or "").strip().lower() + for item in (platform or []) + if str(item or "").strip() + ) + return selected or None + + +def _tile_inputs(candidate: Mapping[str, Any]) -> list[dict[str, Any]]: + files = ((candidate.get("file_summary") or {}).get("files") or []) + tiles: list[dict[str, Any]] = [] + for index, item in enumerate(files): + if item.get("kind") != "tile": + continue + path = str(item.get("path") or "").strip() + tiles.append({ + "index": index, + "kind": "tile", + "path": path, + "exists": bool(item.get("exists")), + "evidence_ref": f"tile:{index}", + }) + return tiles + + +def _field_contract(platform: str) -> list[dict[str, Any]]: + profile = ECOMMERCE_PLATFORM_PROFILES.get(platform) or {} + fields = [str(item) for item in list(profile.get("data_targets") or []) if str(item)] + if "price" in fields and "currency" not in fields: + fields.insert(fields.index("price") + 1, "currency") + if not fields: + fields = ["public_page_visual_evidence"] + contracts: list[dict[str, Any]] = [] + for field in fields: + base = FIELD_ALIASES.get(field) or { + "type": "string", + "required": "false", + "evidence": "visible_region", + } + contracts.append({ + "field": field, + "type": base["type"], + "required": base["required"] == "true", + "evidence_requirement": base["evidence"], + "min_confidence": DEFAULT_CONFIDENCE_THRESHOLD if field in {"title", "price"} else 0.62, + "allow_empty": base["required"] != "true", + }) + return contracts + + +def _output_schema(fields: list[dict[str, Any]]) -> dict[str, Any]: + return { + "platform": "string", + "manifest_id": "string", + "source_receipt_path": "string", + "blocked_page_detected": "boolean", + "quality": { + "overall_confidence": "float_0_1", + "field_confidence_min": "float_0_1", + "missing_required_fields": "list[string]", + "requires_identity_matcher_replay": "boolean", + "requires_promotion_gate": "boolean", + }, + "fields": { + item["field"]: { + "value": item["type"], + "confidence": "float_0_1", + "evidence_refs": "list[string]", + } + for item in fields + }, + "next_machine_action": "string", + } + + +def _validation_rules() -> list[str]: + return [ + "blocked_or_access_denied_page_must_not_emit_product_fields", + "every_field_value_requires_at_least_one_tile_evidence_ref", + "required_title_and_price_must_meet_min_confidence_before_promotion_gate", + "price_must_parse_to_numeric_money_before_any_price_candidate", + "currency_defaults_to_TWD_only_when_page_locale_or_symbol_supports_it", + "identity_matcher_replay_required_before_offer_matching", + "promotion_gate_required_before_internal_rag_or_ai_insights_write", + "direct_competitor_prices_write_forbidden", + "direct_competitor_price_history_write_forbidden", + "direct_ai_insights_write_forbidden", + ] + + +def _model_route_contract() -> dict[str, Any]: + return { + "runtime_policy": "ollama_first_no_hosted_vlm_default", + "preferred_chain": [ + {"host": "34.87.90.216:11434", "role": "gcp_a_primary"}, + {"host": "34.21.145.224:11434", "role": "gcp_b_secondary"}, + {"host": "192.168.0.111:11434", "role": "lan_final_fallback"}, + ], + "production_app_host_is_not_ollama_node": "192.168.0.188", + "hosted_vlm_api_allowed": False, + "gemini_default_allowed": False, + "secret_read_allowed": False, + "network_call_performed_by_this_contract": False, + } + + +def _candidate_contract(candidate: Mapping[str, Any]) -> dict[str, Any]: + platform = str(candidate.get("platform") or "unknown") + fields = _field_contract(platform) + tiles = _tile_inputs(candidate) + blocked_reasons = list(candidate.get("blocked_reasons") or []) + candidate_status = str(candidate.get("candidate_status") or "unknown") + ready = candidate_status == "eligible" and bool(tiles) + missing_tile_count = sum(1 for item in tiles if not item["exists"]) + if candidate_status == "eligible" and not tiles: + blocked_reasons.append("no_tile_inputs_available") + if missing_tile_count: + blocked_reasons.append("tile_input_missing_files") + if candidate_status == "invalid": + replay_status = "invalid" + elif ready and not missing_tile_count: + replay_status = "ready_for_ocr_vlm_replay" + else: + replay_status = "blocked" + + return { + "replay_status": replay_status, + "ready_for_ollama_vlm_worker": replay_status == "ready_for_ocr_vlm_replay", + "candidate_status": candidate_status, + "blocked_reasons": blocked_reasons, + "platform": platform, + "manifest_id": candidate.get("manifest_id"), + "source_receipt_path": candidate.get("receipt_path"), + "url": candidate.get("url"), + "title_hint": candidate.get("title"), + "http_status": candidate.get("http_status"), + "visual_barrier_detected": bool(candidate.get("visual_barrier_detected")), + "visual_barrier_reason": candidate.get("visual_barrier_reason"), + "input_tiles": tiles, + "field_contract": fields, + "field_contract_count": len(fields), + "output_schema": _output_schema(fields), + "validation_rules": _validation_rules(), + "prompt_contract": { + "task": "extract_public_marketplace_offer_card_fields_from_tiles", + "language": "zh-TW", + "allowed_inputs": ["tile images", "receipt metadata", "field contract"], + "forbidden_inputs": ["secrets", "cookies", "sessions", "private account pages"], + "instruction": ( + "Return only fields supported by visible tile evidence; mark unknown " + "instead of guessing, and set blocked_page_detected=true for access " + "denied, captcha, login, or traffic verification pages." + ), + }, + "automation_boundary": { + "ocr_execution_performed": False, + "vlm_execution_performed": False, + "writes_database": False, + "writes_ai_insights": False, + "writes_price_tables": False, + "network_call": False, + "secret_read": False, + }, + "next_machine_action": ( + "run_ollama_first_vlm_replay_worker" + if replay_status == "ready_for_ocr_vlm_replay" + else ( + "run_platform_probe_or_use_structured_api" + if candidate.get("visual_barrier_detected") or candidate.get("http_status", 0) >= 400 + else "refresh_pixelrag_visual_capture_receipt" + ) + ), + } + + +def build_pixelrag_ocr_vlm_replay_contract( + *, + artifact_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, +) -> dict[str, Any]: + """Build a no-write OCR/VLM replay contract from PixelRAG receipts.""" + root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + platforms = _normalise_platforms(platform) + generated_at = datetime.now(timezone.utc).isoformat() + + candidate_readback = build_pixelrag_rag_candidate_replay_readback( + artifact_root=root, + platform=platforms, + max_age_hours=max_age, + limit=item_limit, + ) + replay_items = [ + _candidate_contract(candidate) + for candidate in list(candidate_readback.get("candidates") or []) + ] + ready_count = sum(1 for item in replay_items if item["replay_status"] == "ready_for_ocr_vlm_replay") + blocked_count = sum(1 for item in replay_items if item["replay_status"] == "blocked") + invalid_count = sum(1 for item in replay_items if item["replay_status"] == "invalid") + tile_input_count = sum(len(item.get("input_tiles") or []) for item in replay_items) + field_contract_count = sum(int(item.get("field_contract_count") or 0) for item in replay_items) + missing_tile_count = sum( + 1 + for item in replay_items + for tile in list(item.get("input_tiles") or []) + if not tile.get("exists") + ) + + if invalid_count and invalid_count == len(replay_items): + status = "critical" + elif not replay_items or ready_count == 0 or blocked_count or invalid_count or missing_tile_count: + status = "warning" + else: + status = "ok" + + summary = { + "receipt_count": len(replay_items), + "replay_ready_count": ready_count, + "blocked_count": blocked_count, + "invalid_count": invalid_count, + "tile_input_count": tile_input_count, + "missing_tile_count": missing_tile_count, + "field_contract_count": field_contract_count, + "extraction_execution_performed": False, + "ocr_execution_performed": False, + "vlm_execution_performed": False, + "platforms": sorted({str(item.get("platform") or "unknown") for item in replay_items}), + } + + return { + "success": invalid_count == 0, + "policy": POLICY, + "status": status, + "generated_at": generated_at, + "artifact_root": str(root), + "max_age_hours": max_age, + "platform_filter": list(platforms or []), + "summary": summary, + "replay_items": replay_items, + "source_candidate_replay": { + "policy": candidate_readback.get("policy"), + "status": candidate_readback.get("status"), + "summary": candidate_readback.get("summary"), + }, + "model_route_contract": _model_route_contract(), + "replay_contract": { + "automation_mode": "no_write_ocr_vlm_replay_package", + "writes_database": False, + "writes_ai_insights": False, + "writes_price_tables": False, + "network_call": False, + "secret_read": False, + "extraction_execution_performed": False, + "blocked_pages_are_not_product_data": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": ( + "run_pixelrag_visual_capture_worker" + if not replay_items + else ( + "fix_invalid_pixelrag_receipts" + if invalid_count and not ready_count + else ( + "run_ollama_first_vlm_replay_worker" + if ready_count + else "run_platform_probe_or_use_structured_api" + ) + ) + ), + "controlled_apply": { + "network_call": False, + "db_write": False, + "secret_read": False, + "production_price_write": False, + "artifact_write": False, + "primary_human_gate_count": 0, + }, + } + + +__all__ = [ + "DEFAULT_CONFIDENCE_THRESHOLD", + "POLICY", + "build_pixelrag_ocr_vlm_replay_contract", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index 3a7bc6b..c3fcc76 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1433,11 +1433,12 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 31, "warning": 1, "critical": 1, "total": 33} + assert result["summary"] == {"ok": 32, "warning": 1, "critical": 1, "total": 34} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3966,6 +3967,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) first = smoke.collect_ai_automation_smoke(history_limit=5) second = smoke.collect_ai_automation_smoke(history_limit=5) @@ -4021,7 +4023,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 33, "warning": 0, "critical": 0, "total": 33}, + "summary": {"ok": 34, "warning": 0, "critical": 0, "total": 34}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4129,6 +4131,23 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "blocked_pages_are_not_product_data": True, "next_machine_action": "run_ocr_vlm_replay_then_promotion_gate", }, + }, + { + "name": "PixelRAG OCR/VLM replay contract", + "status": "ok", + "summary": "PixelRAG OCR/VLM replay contract receipts=1, ready=1, blocked=0, invalid=0, field_contracts=7", + "details": { + "policy": "read_only_pixelrag_ocr_vlm_replay_contract_v1", + "receipt_count": 1, + "replay_ready_count": 1, + "blocked_count": 0, + "invalid_count": 0, + "tile_input_count": 4, + "field_contract_count": 7, + "extraction_execution_performed": False, + "blocked_pages_are_not_product_data": True, + "next_machine_action": "run_ollama_first_vlm_replay_worker", + }, } ], }, ensure_ascii=False) + "\n", @@ -4147,7 +4166,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 30 + assert summary["summary"]["total"] == 31 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -5823,6 +5842,13 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert pixelrag_rag_replay_family["status"] == "ok" assert pixelrag_rag_replay_family["details"]["eligible_count"] == 1 + pixelrag_ocr_vlm_replay_family = next( + item for item in summary["families"] + if item["key"] == "pixelrag_ocr_vlm_replay" + ) + assert pixelrag_ocr_vlm_replay_family["status"] == "ok" + assert pixelrag_ocr_vlm_replay_family["details"]["replay_ready_count"] == 1 + assert pixelrag_ocr_vlm_replay_family["details"]["extraction_execution_performed"] is False assert summary["scheduled_outputs"]["telegram_send_in_preview"] is False assert summary["scheduled_outputs"]["writes_database"] is False assert summary["automation_policy"]["primary_flow"] == "ai_controlled" @@ -6439,6 +6465,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag replay ok", )) + monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check( + "PixelRAG OCR/VLM replay contract", + "ok", + "pixelrag ocr vlm ok", + )) result = smoke.collect_ai_automation_smoke(record_history=False) @@ -6454,7 +6485,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 33 + assert result["summary"]["total"] == 34 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_ocr_vlm_replay_service.py b/tests/test_pixelrag_ocr_vlm_replay_service.py new file mode 100644 index 0000000..8889934 --- /dev/null +++ b/tests/test_pixelrag_ocr_vlm_replay_service.py @@ -0,0 +1,144 @@ +import json +import subprocess +import sys +from datetime import datetime, timezone + + +def _write_receipt(root, *, platform, manifest_id, title, url, http_status=200): + receipt_dir = root / platform / manifest_id + tiles_dir = receipt_dir / "tiles" + tiles_dir.mkdir(parents=True) + screenshot = receipt_dir / "screenshot.png" + screenshot.write_bytes(b"png") + files = [{"kind": "screenshot", "path": str(screenshot)}] + for index in range(4): + tile = tiles_dir / f"tile-{index}.png" + tile.write_bytes(b"tile") + files.append({"kind": "tile", "path": str(tile)}) + receipt = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "status": "captured", + "manifest_id": manifest_id, + "http_status": http_status, + "capture_target": { + "platform": platform, + "url": url, + "crawler": "PixelRAGMarketplaceSearch.visual_fallback", + }, + "page_metrics": { + "title": title, + "final_url": url, + }, + "tile_plan": { + "planned_tile_count": 4, + "emitted_tile_count": 4, + }, + "files": files, + } + path = receipt_dir / "capture_receipt.json" + path.write_text(json.dumps(receipt, ensure_ascii=False), encoding="utf-8") + return path + + +def test_pixelrag_ocr_vlm_replay_contract_splits_ready_and_blocked_receipts(tmp_path): + from services.pixelrag_ocr_vlm_replay_service import ( + POLICY, + build_pixelrag_ocr_vlm_replay_contract, + ) + + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=%E9%98%B2%E6%9B%AC%E4%B9%B3", + ) + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + + payload = build_pixelrag_ocr_vlm_replay_contract(artifact_root=tmp_path) + by_platform = {item["platform"]: item for item in payload["replay_items"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "warning" + assert payload["summary"]["receipt_count"] == 2 + assert payload["summary"]["replay_ready_count"] == 1 + assert payload["summary"]["blocked_count"] == 1 + assert payload["summary"]["extraction_execution_performed"] is False + assert by_platform["shopee_tw"]["replay_status"] == "ready_for_ocr_vlm_replay" + assert by_platform["shopee_tw"]["ready_for_ollama_vlm_worker"] is True + assert len(by_platform["shopee_tw"]["input_tiles"]) == 4 + assert "price" in { + field["field"] for field in by_platform["shopee_tw"]["field_contract"] + } + assert by_platform["coupang_tw"]["replay_status"] == "blocked" + assert by_platform["coupang_tw"]["next_machine_action"] == ( + "run_platform_probe_or_use_structured_api" + ) + assert payload["model_route_contract"]["hosted_vlm_api_allowed"] is False + assert payload["replay_contract"]["writes_price_tables"] is False + assert payload["controlled_apply"]["primary_human_gate_count"] == 0 + + +def test_pixelrag_ocr_vlm_replay_cli_outputs_machine_readable_json(tmp_path): + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=sunscreen", + ) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/report_pixelrag_ocr_vlm_replay.py", + "--artifact-root", + str(tmp_path), + "--platform", + "shopee_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["receipt_count"] == 1 + assert payload["summary"]["replay_ready_count"] == 1 + assert payload["replay_items"][0]["platform"] == "shopee_tw" + assert payload["replay_items"][0]["automation_boundary"]["vlm_execution_performed"] is False + + +def test_pixelrag_ocr_vlm_replay_route_returns_readback(tmp_path, monkeypatch): + from flask import Flask + from routes import system_public_routes as routes + from services import pixelrag_ocr_vlm_replay_service as service + + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=sunscreen", + ) + monkeypatch.setattr(service, "DEFAULT_ARTIFACT_ROOT", str(tmp_path)) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-ocr-vlm-replay?platform=shopee_tw" + ): + response = routes.ai_automation_pixelrag_ocr_vlm_replay_api.__wrapped__() + payload = response.get_json() + + assert payload["policy"] == "read_only_pixelrag_ocr_vlm_replay_contract_v1" + assert payload["summary"]["receipt_count"] == 1 + assert payload["summary"]["replay_ready_count"] == 1 + assert payload["replay_contract"]["writes_database"] is False From e34c9d777b61cbd035eb13e8bfd80cd8d880b5a7 Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 23:15:07 +0800 Subject: [PATCH 06/75] =?UTF-8?q?feat(ai):=20=E5=BB=BA=E7=AB=8B=20PixelRAG?= =?UTF-8?q?=20application=20portfolio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO_NEXT_STEPS.txt | 3 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 6 +- .../ai_automation_mainline_work_items.md | 1 + docs/guides/ai_automation_session_sop.md | 2 + docs/guides/browse_sh_crawler_playbook.md | 14 + routes/system_public_routes.py | 20 + .../report_pixelrag_application_portfolio.py | 50 ++ .../pixelrag_application_portfolio_service.py | 446 ++++++++++++++++++ ..._pixelrag_application_portfolio_service.py | 86 ++++ 10 files changed, 626 insertions(+), 4 deletions(-) create mode 100755 scripts/ops/report_pixelrag_application_portfolio.py create mode 100644 services/pixelrag_application_portfolio_service.py create mode 100644 tests/test_pixelrag_application_portfolio_service.py diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 94ef45c..566b88d 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -9,7 +9,7 @@ python scripts/ops/check_production_version_truth.py 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 -本輪 source target 為 `V10.747`；部署完成前不得宣稱正式環境已是 `V10.747`。 +本輪 source target 為 `V10.748`；部署完成前不得宣稱正式環境已是 `V10.748`。 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ @@ -59,6 +59,7 @@ P6. DB apply authorization / signing / closeout P0-2026-07-09. PixelRAG / MCP / RAG 全自動主線 - 已完成：多電商 PixelRAG visual evidence lane 與 external MCP/RAG integration readback。 - 已完成：PixelRAG receipt → internal RAG candidate replay，以及 OCR/VLM replay contract no-write readback。 + - 已完成：PixelRAG application portfolio，把可整合/可運用場景整理成 API/CLI 可讀的 priority lane、status、next machine action 與 forbidden guardrail。 - 進行中：MCP/RAG runtime health → AI automation smoke。 - 未開始：Ollama-first OCR/VLM extraction worker、Ollama-first visual embedding benchmark、pgvector-compatible visual metadata、Coupang platform probe / structured API、跨平台 source contracts。 - 主線文件：`docs/guides/ai_automation_mainline_work_items.md`。 diff --git a/config.py b/config.py index 5c407ff..05c9347 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.747" +SYSTEM_VERSION = "V10.748" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 0e87108..81ba400 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,8 +1,8 @@ # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth > **最後更新**: 2026-07-09 (台北時間) -> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.747 +> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 +> **適用版本**: V10.748 --- @@ -115,6 +115,7 @@ - 2026-07-09 起 external MCP/RAG integration 必須進入 AI automation runtime monitoring：`/api/ai-automation/external-mcp-rag-integration` 與 `scripts/ops/report_external_mcp_rag_integration.py` 需輸出 9 個 capability 的 absorbed / unresolved 狀態、內部治理落點與 runtime flags；`/api/ai-automation/smoke` 需包含 `External MCP/RAG integration readback`，`/api/ai-automation/scheduled-health-summary` 需輸出 `external_mcp_rag_integration` family。`MCP_ROUTER_ENABLED=false` 或 `RAG_ENABLED=false` 必須被視為 runtime gap，不得因 registry 已完成就宣稱 MCP/RAG runtime 已全面啟用。 - 2026-07-09 起 PixelRAG visual receipts 必須可轉為內部 RAG candidate replay readback：`/api/ai-automation/pixelrag-rag-candidate-replay` 與 `scripts/ops/report_pixelrag_rag_candidate_replay.py` 只讀 `capture_receipt.json`，拆分 eligible / blocked / invalid receipts，輸出 tile/missing/barrier count、candidate text、promotion boundary 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG RAG candidate replay`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_rag_candidate_replay` family。blocked / 403 / captcha / access denied / verify traffic page 只能進 platform probe 或 structured API 策略，不得當作商品資料、不得寫正式價格表、不得直接寫 `ai_insights`。 - 2026-07-09 起 PixelRAG visual receipts 進入 OCR/VLM 前必須先輸出 no-write replay contract：`/api/ai-automation/pixelrag-ocr-vlm-replay` 與 `scripts/ops/report_pixelrag_ocr_vlm_replay.py` 只讀 saved tiles，輸出欄位 schema、輸出 schema、confidence/evidence validation rules、Ollama-first route contract 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG OCR/VLM replay contract`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_ocr_vlm_replay` family。此 contract 不執行 OCR/VLM、不呼叫模型、不讀 secret、不連外、不寫 DB、不寫 `ai_insights`、不寫正式價格表；blocked / 403 / captcha / access denied / verify traffic page 只能進 platform probe 或 structured API 策略。 +- 2026-07-09 起 PixelRAG 可整合/可運用範圍必須有機器可讀 application portfolio：`/api/ai-automation/pixelrag-application-portfolio` 與 `scripts/ops/report_pixelrag_application_portfolio.py` 需輸出 commerce、RAG、UX、ops、marketing、governance lanes，每條 lane 必須包含 priority、status、integrates_with、use_cases、current_capability、next_machine_action、no-write 邊界與 forbidden guardrails。此 portfolio 不抓外站、不呼叫模型、不讀 secret、不寫 DB；其用途是把「還可以整合哪些」變成可排程、可驗證、可拒絕違規場景的主線工作項目。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -876,6 +877,7 @@ POSTGRES_HOST=momo-db | 2026-07-09 | PChome auto-policy controlled dry-run receipt closeout 必須有 runtime monitoring | V10.745 起 `/api/ai-automation/smoke`、`/api/ai-automation/scheduled-health-summary` 與 `/metrics` 必須輸出 `PChome auto-policy controlled dry-run receipt closeout` / `pchome_auto_policy_controlled_dry_run_receipt_closeout`；此 runtime check 自動驗證 controlled dry-run package carry-forward、dry-run result parser、receipt validation report、command-shape hash、rollback binding、post-apply verifier binding、12 個 receipt closeout checks、12 個 closeout fields、10 個 acceptance gates、same-run production truth 與 future runner readiness，但明確標記 `receipt_validation_status=preview_validated_not_executed`、`validation_execution_performed=false`、`validation_stdout_included=false`、`validation_stderr_included=false`、`validation_database_apply_authorized=false`、`reads_secret_count=0`、`executes_script_count=0`、`executes_sql_count=0`、`writes_database_count=0`、`signs_database_apply_authorization_count=0`、`primary_human_gate_count=0`、`executes_authorization_evidence=false`、`executes_database_apply=false`、`ready_for_database_apply_now=false`、`database_apply_authorized=false`、`issues_database_apply_authorization=false`、`signs_database_apply_authorization=false`，不讀 secret、不執行 dry-run、不捕捉 stdout/stderr、不執行 authorization evidence、不執行 database apply、不執行 endpoint/SQL、不寫 DB、不簽發 apply authorization，只允許進入下一段 controlled dry-run runner readiness lane。 | | 2026-07-09 | PixelRAG / external MCP/RAG 必須有 runtime monitoring 與 candidate replay | V10.746 起 `/api/ai-automation/smoke`、`/api/ai-automation/scheduled-health-summary` 必須輸出 `External MCP/RAG integration readback` / `external_mcp_rag_integration` 與 `PixelRAG RAG candidate replay` / `pixelrag_rag_candidate_replay`；external MCP/RAG readback 回報 9 個 capability 的 absorbed / unresolved 與 `MCP_ROUTER_ENABLED`、`RAG_ENABLED` runtime flags，PixelRAG replay 只讀 visual receipt，拆分 eligible / blocked / invalid，並明確標記 blocked page 不是商品資料。此路徑不讀 secret、不呼叫外部網路、不寫 DB、不寫 `ai_insights`、不寫正式價格表；eligible receipt 仍需 OCR/VLM replay、identity matcher replay、PromotionGate 與 embedding signature guard。 | | 2026-07-09 | PixelRAG OCR/VLM replay contract 必須有 runtime monitoring | V10.747 起 `/api/ai-automation/pixelrag-ocr-vlm-replay`、`scripts/ops/report_pixelrag_ocr_vlm_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 no-write OCR/VLM replay contract / `pixelrag_ocr_vlm_replay` family；readback 只讀 saved tiles 與 RAG candidate replay，輸出 ready / blocked / invalid contracts、field schema、output schema、validation rules、Ollama-first route contract、blocked page guard 與 next machine action。此階段明確標記 `extraction_execution_performed=false`、`ocr_execution_performed=false`、`vlm_execution_performed=false`、`writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`network_call=false`、`secret_read=false`、`primary_human_gate_count=0`；ready receipt 才能進下一段 `run_ollama_first_vlm_replay_worker`，blocked receipt 只能進 platform probe 或 structured API 策略。 | +| 2026-07-09 | PixelRAG application portfolio 必須把可整合場景轉成主線工作項目 | V10.748 起 `/api/ai-automation/pixelrag-application-portfolio` 與 `scripts/ops/report_pixelrag_application_portfolio.py` 必須輸出 PixelRAG 在 commerce、RAG、UX、ops、marketing、governance 的可整合/可運用 lanes；每條 lane 需有 priority、status、integrates_with、use_cases、current_capability、next_machine_action、no-write 邊界與 forbidden guardrails。此 readback 依據 PixelRAG visual-RAG pattern、Google Merchant product data、Google Product structured data 與 Baymard product list UX 轉成內部工作項目；它不抓外站、不呼叫模型、不讀 secret、不寫 DB、不把像素結果當正式價格。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 4797b81..b24a1dd 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -17,6 +17,7 @@ | Completed | Multi-commerce PixelRAG visual evidence lane for momo, pchome, shopee_tw, coupang_tw, yahoo_shopping_tw, etmall_tw, friday_tw, rakuten_tw | API and CLI can emit expansion plans and marketplace manifests; PChome parser failures can queue manifests. | | Completed | External MCP/RAG capability inventory absorbed into internal governance readback | `/api/ai-automation/external-mcp-rag-integration` and `scripts/ops/report_external_mcp_rag_integration.py` expose 9 capabilities, absorbed/unresolved counts, and runtime flags. | | Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read receipts, split eligible vs blocked, and require OCR/VLM replay plus PromotionGate before knowledge writes. | +| Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, and PixelRAG OCR/VLM replay contract families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | diff --git a/docs/guides/ai_automation_session_sop.md b/docs/guides/ai_automation_session_sop.md index 5454311..459d619 100644 --- a/docs/guides/ai_automation_session_sop.md +++ b/docs/guides/ai_automation_session_sop.md @@ -32,6 +32,8 @@ 或 `python scripts/ops/report_pixelrag_rag_candidate_replay.py` 可讀回 eligible / blocked / invalid receipts，且 blocked page 不得被視為商品資料。 - PixelRAG 視覺 receipt 進入 OCR/VLM 自動化前，必須確認 `/api/ai-automation/pixelrag-ocr-vlm-replay` 或 `python scripts/ops/report_pixelrag_ocr_vlm_replay.py` 可讀回 ready / blocked / invalid replay contracts、field schema、validation rules 與 Ollama-first 下一步，且目前不執行 OCR/VLM、不寫正式價格。 +- PixelRAG 可整合/可運用場景盤點必須確認 `/api/ai-automation/pixelrag-application-portfolio` + 或 `python scripts/ops/report_pixelrag_application_portfolio.py` 可讀回 area、priority、status、use cases、next machine action 與 forbidden guardrails；不得只存在聊天結論。 - AI automation smoke 必須包含 external MCP/RAG integration、PixelRAG RAG candidate replay 與 PixelRAG OCR/VLM replay contract family，避免 registry 已完成但 runtime flag / receipt replay / VLM contract 未完成時被誤報為全自動閉環。 - AI 自動化 Prometheus 指標變更必須同步檢查 `docker/grafana/provisioning/dashboards/json/ai-automation-overview.json` 是否需要新增 panel 或查詢。 - 188 線上 active monitoring stack 以 `monitoring/prometheus.yml` 為準；110 gateway 另有 `/home/wooo/monitoring/prometheus.yml`。若 dashboard 無資料，先確認 Prometheus `momo-app` target 與 `momo-network` 連線；所有 Blackbox HTTP target 必須打 `/health`，不可打 Dashboard 首頁 `/`。 diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index 7f914df..1949117 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -101,6 +101,8 @@ rakuten_tw python scripts/ops/report_pixelrag_crawler_integration.py python scripts/ops/report_pixelrag_crawler_integration.py --platform momo python scripts/ops/report_pixelrag_crawler_integration.py --ecommerce-expansion-plan +python scripts/ops/report_pixelrag_application_portfolio.py +python scripts/ops/report_pixelrag_application_portfolio.py --area commerce python scripts/ops/report_pixelrag_crawler_integration.py --platform shopee_tw --marketplace-keyword "防曬乳" python scripts/ops/report_pixelrag_crawler_integration.py --platform coupang_tw --marketplace-keyword "iphone" python scripts/ops/report_pixelrag_crawler_integration.py --capability ollama_multimodal_ready --capability pgvector_visual_ready @@ -118,6 +120,8 @@ python scripts/ops/report_pixelrag_crawler_integration.py \ ```text /api/ai-automation/pixelrag-crawler-integration?platform=momo /api/ai-automation/pixelrag-crawler-integration?expansion_plan=true +/api/ai-automation/pixelrag-application-portfolio +/api/ai-automation/pixelrag-application-portfolio?area=commerce /api/ai-automation/pixelrag-crawler-integration?platform=shopee_tw&marketplace_keyword=%E9%98%B2%E6%9B%AC%E4%B9%B3 /api/ai-automation/pixelrag-crawler-integration?platform=coupang_tw&marketplace_keyword=iphone /api/ai-automation/pixelrag-crawler-integration?platform=pchome&manifest_url=https://24h.pchome.com.tw/prod/TEST-000000001&crawler=PChomeCrawler.search_products&trigger_reason=parser_empty @@ -180,6 +184,16 @@ python scripts/ops/report_pixelrag_ocr_vlm_replay.py --platform shopee_tw --plat 此 contract 只把 saved tiles、欄位 schema、輸出 schema、confidence/evidence 規則與 Ollama-first worker 下一步打包；目前不執行 OCR/VLM、不呼叫模型、不寫 RAG、不寫 `ai_insights`、不寫正式價格表。ready receipt 進 `run_ollama_first_vlm_replay_worker`，blocked / 403 / captcha / access denied receipt 進 platform probe 或 structured API 策略。 +Application portfolio： + +```bash +python scripts/ops/report_pixelrag_application_portfolio.py +python scripts/ops/report_pixelrag_application_portfolio.py --area commerce +python scripts/ops/report_pixelrag_application_portfolio.py --area rag --include-forbidden +``` + +此 portfolio 把 PixelRAG 可整合範圍正式分成 commerce、RAG、UX、ops、marketing、governance lanes。每個 lane 都必須有 priority、status、可整合對象、use cases、next machine action 與 no-write 邊界；forbidden lane 明確拒絕 login-only、cart、checkout、private coupon、session 與 anti-bot bypass。 + 安全邊界： - read-only；不登入、不下單、不加入購物車、不寫第三方狀態。 diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 5eb73e7..f0cf283 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -733,6 +733,26 @@ def ai_automation_pixelrag_ocr_vlm_replay_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-application-portfolio') +@login_required +def ai_automation_pixelrag_application_portfolio_api(): + """Read-only PixelRAG application and integration portfolio.""" + from services.pixelrag_application_portfolio_service import ( + build_pixelrag_application_portfolio, + ) + + include_forbidden = str(request.args.get('include_forbidden') or '').lower() in { + '1', + 'true', + 'yes', + } + return jsonify(build_pixelrag_application_portfolio( + area=str(request.args.get('area') or '').strip() or None, + priority=str(request.args.get('priority') or '').strip() or None, + include_forbidden=include_forbidden, + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/report_pixelrag_application_portfolio.py b/scripts/ops/report_pixelrag_application_portfolio.py new file mode 100755 index 0000000..45f979f --- /dev/null +++ b/scripts/ops/report_pixelrag_application_portfolio.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Report the PixelRAG application portfolio as machine-readable JSON.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_application_portfolio_service import ( # noqa: E402 + build_pixelrag_application_portfolio, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="輸出 PixelRAG 還可整合/運用場景的機器可讀 portfolio。" + ) + parser.add_argument( + "--area", + help="限制 area，例如 commerce / rag / ux / ops / marketing / governance。", + ) + parser.add_argument( + "--priority", + help="限制 priority，例如 P0 / P1 / P2 / P3。", + ) + parser.add_argument( + "--include-forbidden", + action="store_true", + help="包含 forbidden guardrail lane。", + ) + args = parser.parse_args() + + payload = build_pixelrag_application_portfolio( + area=args.area, + priority=args.priority, + include_forbidden=args.include_forbidden, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/pixelrag_application_portfolio_service.py b/services/pixelrag_application_portfolio_service.py new file mode 100644 index 0000000..e68ee35 --- /dev/null +++ b/services/pixelrag_application_portfolio_service.py @@ -0,0 +1,446 @@ +"""Read-only PixelRAG application portfolio and integration inventory. + +This module converts the product strategy for PixelRAG into machine-readable +lanes. It is intentionally no-write: it does not crawl, call models, read +secrets, or promote visual evidence into price truth. +""" + +from __future__ import annotations + +from collections import Counter +from copy import deepcopy +from datetime import datetime, timezone +from typing import Any, Mapping + + +POLICY = "read_only_pixelrag_application_portfolio_v1" + +SOURCE_BASIS = [ + { + "id": "pixelrag_visual_rag_pattern", + "basis": "render_public_pages_as_screenshot_tiles_for_visual_retrieval_and_vlm_reading", + "url": "https://medium.com/open-intelligence/pixelrag-proves-that-the-biggest-bottleneck-in-web-rag-was-the-text-parser-all-along-d6ce53c78b02", + }, + { + "id": "google_merchant_product_data", + "basis": "normalize_offer_evidence_around_product_title_price_availability_image_link_and_related_attributes", + "url": "https://support.google.com/merchants/answer/7052112", + }, + { + "id": "google_product_structured_data", + "basis": "use_product_price_availability_reviews_shipping_and_visual_rich-result_fields_as_validation_targets", + "url": "https://developers.google.com/search/docs/appearance/structured-data/product", + }, + { + "id": "baymard_product_list_ux", + "basis": "product_list_cards_need_scannable_visuals_and_key_attributes_for_comparison", + "url": "https://baymard.com/research/ecommerce-product-lists", + }, +] + + +LANES: list[dict[str, Any]] = [ + { + "id": "commerce.marketplace_visual_offer_cards", + "priority": "P0", + "status": "production_contract_ready", + "area": "commerce", + "title": "Multi-commerce public offer card visual evidence", + "integrates_with": [ + "momo", + "pchome", + "shopee_tw", + "coupang_tw", + "yahoo_shopping_tw", + "etmall_tw", + "friday_tw", + "rakuten_tw", + ], + "use_cases": [ + "Rendered product cards", + "Promotion badges", + "Seller and delivery badges", + "Rating and sold-count signals", + "Parser-empty or parser-low-confidence fallback", + ], + "current_capability": [ + "visual_manifest_generation", + "visual_capture_receipts", + "rag_candidate_replay", + "ocr_vlm_replay_contract", + ], + "next_machine_action": "run_ollama_first_vlm_replay_worker", + "writes_database": False, + "promotion_required_before_price_truth": True, + }, + { + "id": "commerce.product_feed_and_structured_data_gap_check", + "priority": "P0", + "status": "source_contract_design_ready", + "area": "commerce", + "title": "Product feed / structured data gap detection", + "integrates_with": [ + "Google Merchant product data", + "Google Product structured data", + "brand sites", + "Shopify storefronts", + "marketplace offer pages", + ], + "use_cases": [ + "Compare visible product card fields against feed/structured-data fields", + "Detect missing image, price, availability, review, shipping, or product URL evidence", + "Build source contracts before platform data enters growth opportunities", + ], + "current_capability": [ + "source_readiness_contract", + "external_offer_evidence_contract", + "pixelrag_field_contract", + ], + "next_machine_action": "build_pixelrag_source_contract_gap_readback", + "writes_database": False, + "promotion_required_before_price_truth": True, + }, + { + "id": "commerce.promotion_coupon_delivery_badge_extraction", + "priority": "P1", + "status": "contract_ready_worker_not_started", + "area": "commerce", + "title": "Promotion, coupon, discount, and delivery badge extraction", + "integrates_with": [ + "Shopee promotion chips", + "Coupang delivery badges", + "MOMO campaign blocks", + "PChome campaign badges", + "Yahoo/ETMall coupon panels", + ], + "use_cases": [ + "Detect visible discounts that structured parsers miss", + "Separate base price from coupon/member/delivery incentives", + "Feed promotion evidence into no-write candidate receipts", + ], + "current_capability": ["ocr_vlm_replay_contract"], + "next_machine_action": "run_ollama_first_vlm_replay_worker_then_promotion_gate", + "writes_database": False, + "promotion_required_before_price_truth": True, + }, + { + "id": "commerce.product_identity_visual_matching", + "priority": "P1", + "status": "requires_worker_and_identity_replay", + "area": "commerce", + "title": "Product identity visual matching", + "integrates_with": [ + "product images", + "package size/spec text", + "brand/logo regions", + "variant cards", + "unit price evidence", + ], + "use_cases": [ + "Reduce false same-product matches", + "Use package/spec visual evidence before accepting competitor offer candidates", + "Keep unit basis evidence attached to each candidate", + ], + "current_capability": ["field_contract", "identity_matcher_required_boundary"], + "next_machine_action": "build_visual_identity_matcher_replay_contract", + "writes_database": False, + "promotion_required_before_price_truth": True, + }, + { + "id": "commerce.availability_delivery_stock_signal", + "priority": "P1", + "status": "source_contract_design_ready", + "area": "commerce", + "title": "Availability, stock, delivery, and shipping signal evidence", + "integrates_with": [ + "PChome stock hints", + "MOMO availability text", + "Coupang rocket delivery badge", + "Shopee shipping badge", + "merchant listing structured data", + ], + "use_cases": [ + "Avoid recommending unavailable or slow-delivery products for promotion", + "Detect delivery advantage as a sales-action signal", + "Flag missing availability in source contracts", + ], + "current_capability": ["ocr_vlm_replay_contract", "source_readiness_contract"], + "next_machine_action": "build_availability_delivery_evidence_replay", + "writes_database": False, + "promotion_required_before_price_truth": True, + }, + { + "id": "commerce.platform_barrier_classifier", + "priority": "P0", + "status": "production_contract_ready", + "area": "governance", + "title": "Blocked page / captcha / login barrier classifier", + "integrates_with": [ + "PixelRAG capture receipts", + "Shopee/Coupang probes", + "structured API fallback strategy", + "source contract status", + ], + "use_cases": [ + "Prevent AI from treating 403/captcha/login pages as product data", + "Route blocked pages to platform probe or structured API strategy", + "Keep anti-bot boundaries explicit", + ], + "current_capability": ["visual_barrier_reason", "blocked_pages_are_not_product_data"], + "next_machine_action": "build_platform_probe_or_structured_api_strategy", + "writes_database": False, + "promotion_required_before_price_truth": True, + }, + { + "id": "rag.visual_evidence_to_internal_knowledge", + "priority": "P1", + "status": "contract_ready_worker_not_started", + "area": "rag", + "title": "Visual evidence to internal RAG learning candidates", + "integrates_with": [ + "rag_service", + "learning_episode_candidate_preview", + "PromotionGate", + "embedding signature guard", + "pgvector metadata lane", + ], + "use_cases": [ + "Turn accepted visual receipts into replayable learning candidates", + "Attach screenshot/tile evidence to every extracted field", + "Only promote after OCR/VLM, identity replay, PromotionGate, and signature guard pass", + ], + "current_capability": ["rag_candidate_replay", "ocr_vlm_replay_contract"], + "next_machine_action": "run_vlm_extraction_then_rag_candidate_promotion_preflight", + "writes_database": False, + "promotion_required_before_price_truth": True, + }, + { + "id": "rag.pgvector_visual_metadata", + "priority": "P2", + "status": "not_started", + "area": "rag", + "title": "pgvector-compatible visual evidence metadata", + "integrates_with": [ + "pgvector", + "BGE-M3 text metadata", + "Qwen3-VL or approved local visual model", + "visual receipt file refs", + ], + "use_cases": [ + "Search visual evidence by platform, field, tile, and confidence", + "Avoid FAISS in production until ADR approves a separate store", + "Keep retrieval metadata in the same governance boundary as text RAG", + ], + "current_capability": ["deferred_until_ollama_first_visual_embedding_benchmark"], + "next_machine_action": "run_ollama_first_visual_embedding_benchmark", + "writes_database": False, + "promotion_required_before_price_truth": True, + }, + { + "id": "ux.product_listing_quality_benchmark", + "priority": "P2", + "status": "source_contract_design_ready", + "area": "ux", + "title": "Product listing/card UX quality benchmark", + "integrates_with": [ + "Baymard product list heuristics", + "PChome growth dashboard", + "AI surface visual QA", + "external marketplace screenshots", + ], + "use_cases": [ + "Benchmark whether our product cards show enough scannable evidence", + "Detect text-heavy or missing-attribute cards", + "Turn external marketplace patterns into product UX guardrails", + ], + "current_capability": ["sitewide_visual_qa_readback", "pixelrag_visual_receipts"], + "next_machine_action": "build_product_card_visual_ux_benchmark_readback", + "writes_database": False, + "promotion_required_before_price_truth": False, + }, + { + "id": "ops.visual_regression_for_public_pages", + "priority": "P2", + "status": "partially_integrated", + "area": "ops", + "title": "Public page visual regression and evidence capture", + "integrates_with": [ + "sitewide visual QA", + "AI automation smoke", + "product landing pages", + "crawler diagnostics pages", + ], + "use_cases": [ + "Catch layout breaks that HTML tests do not see", + "Keep evidence screenshots attached to smoke history", + "Detect content overlap and text-heavy regressions", + ], + "current_capability": ["sitewide_visual_qa_readback"], + "next_machine_action": "merge_pixelrag_tile_receipts_with_sitewide_visual_qa_artifacts", + "writes_database": False, + "promotion_required_before_price_truth": False, + }, + { + "id": "marketing.ad_landing_and_creative_evidence", + "priority": "P3", + "status": "not_started", + "area": "marketing", + "title": "Ad landing page and creative evidence checks", + "integrates_with": [ + "Google Shopping landing pages", + "Meta commerce public landing pages", + "public brand campaign pages", + "product feed destination URLs", + ], + "use_cases": [ + "Verify landing page visible price/image availability against feed contract", + "Detect promotional creative mismatch before campaign decisions", + "Keep evidence read-only and public-page-only", + ], + "current_capability": ["manifest_contract_possible"], + "next_machine_action": "build_public_landing_visual_evidence_contract", + "writes_database": False, + "promotion_required_before_price_truth": True, + }, + { + "id": "forbidden.personalized_or_private_state", + "priority": "P0_guardrail", + "status": "forbidden", + "area": "governance", + "title": "Personalized, cart, checkout, account, or anti-bot bypass states", + "integrates_with": [], + "use_cases": [ + "Explicitly reject login-only prices, carts, checkout pages, private coupons, sessions, and anti-bot bypass", + "Prevent hidden manual or credentialed scraping from entering the automation plan", + ], + "current_capability": ["guardrail_only"], + "next_machine_action": "reject_and_route_to_official_api_or_manual_csv_contract", + "writes_database": False, + "promotion_required_before_price_truth": True, + }, +] + + +def _selected_lanes( + *, + area: str | None = None, + priority: str | None = None, + include_forbidden: bool = False, +) -> list[dict[str, Any]]: + area_key = str(area or "").strip().lower() + priority_key = str(priority or "").strip().upper() + selected: list[dict[str, Any]] = [] + for lane in LANES: + if not include_forbidden and lane["status"] == "forbidden": + continue + if area_key and lane["area"] != area_key: + continue + if priority_key and lane["priority"] != priority_key: + continue + selected.append(deepcopy(lane)) + return selected + + +def _counts(lanes: list[Mapping[str, Any]], key: str) -> dict[str, int]: + return dict(sorted(Counter(str(item.get(key) or "unknown") for item in lanes).items())) + + +def _recommended_order(lanes: list[Mapping[str, Any]]) -> list[dict[str, Any]]: + order = {"P0": 0, "P0_guardrail": 1, "P1": 2, "P2": 3, "P3": 4} + actionable = [ + item for item in lanes + if str(item.get("status")) not in {"forbidden"} + ] + ranked = sorted( + actionable, + key=lambda item: ( + order.get(str(item.get("priority")), 99), + str(item.get("status")), + str(item.get("id")), + ), + ) + return [ + { + "rank": index + 1, + "id": item["id"], + "priority": item["priority"], + "status": item["status"], + "next_machine_action": item["next_machine_action"], + } + for index, item in enumerate(ranked[:8]) + ] + + +def build_pixelrag_application_portfolio( + *, + area: str | None = None, + priority: str | None = None, + include_forbidden: bool = False, +) -> dict[str, Any]: + """Return the product-level PixelRAG integration portfolio.""" + lanes = _selected_lanes(area=area, priority=priority, include_forbidden=include_forbidden) + ready_count = sum( + 1 for item in lanes + if str(item.get("status")) in {"production_contract_ready", "source_contract_design_ready", "partially_integrated"} + ) + blocked_or_forbidden_count = sum( + 1 for item in lanes + if str(item.get("status")) in {"forbidden"} + ) + worker_required_count = sum( + 1 for item in lanes + if "worker" in str(item.get("status")) or "worker" in str(item.get("next_machine_action")) + ) + + return { + "success": True, + "policy": POLICY, + "generated_at": datetime.now(timezone.utc).isoformat(), + "status": "portfolio_ready", + "filters": { + "area": str(area or "").strip().lower() or None, + "priority": str(priority or "").strip().upper() or None, + "include_forbidden": include_forbidden, + }, + "summary": { + "lane_count": len(lanes), + "ready_or_partially_integrated_count": ready_count, + "worker_required_count": worker_required_count, + "blocked_or_forbidden_count": blocked_or_forbidden_count, + "area_counts": _counts(lanes, "area"), + "priority_counts": _counts(lanes, "priority"), + "status_counts": _counts(lanes, "status"), + }, + "source_basis": SOURCE_BASIS, + "lanes": lanes, + "recommended_order": _recommended_order(lanes), + "answer_to_owner": ( + "PixelRAG should become the visual evidence layer for public product cards, " + "feed/structured-data gap detection, promotion/delivery badge extraction, " + "identity matching evidence, internal RAG candidate promotion, and product UX " + "benchmarking. It must stay no-write until OCR/VLM replay, identity replay, " + "PromotionGate, and source contracts pass." + ), + "not_recommended": [ + "Do not use PixelRAG for login-only prices, cart/checkout state, private coupons, sessions, or anti-bot bypass.", + "Do not write competitor_prices, competitor_price_history, ai_insights, or RAG truth directly from pixels.", + "Do not use hosted VLM/embedding APIs as the default path; use Ollama-first verification.", + "Do not put FAISS into production visual retrieval without a separate ADR; prefer pgvector-compatible metadata first.", + ], + "controlled_apply": { + "network_call": False, + "db_write": False, + "secret_read": False, + "production_price_write": False, + "artifact_write": False, + "primary_human_gate_count": 0, + }, + "next_machine_action": ( + "run_ollama_first_vlm_replay_worker" + if worker_required_count + else "keep_pixelrag_application_portfolio_monitoring" + ), + } + + +__all__ = ["POLICY", "build_pixelrag_application_portfolio"] diff --git a/tests/test_pixelrag_application_portfolio_service.py b/tests/test_pixelrag_application_portfolio_service.py new file mode 100644 index 0000000..707af38 --- /dev/null +++ b/tests/test_pixelrag_application_portfolio_service.py @@ -0,0 +1,86 @@ +import json +import subprocess +import sys + + +def test_pixelrag_application_portfolio_prioritizes_commerce_and_guardrails(): + from services.pixelrag_application_portfolio_service import ( + POLICY, + build_pixelrag_application_portfolio, + ) + + payload = build_pixelrag_application_portfolio(include_forbidden=True) + lanes = {item["id"]: item for item in payload["lanes"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "portfolio_ready" + assert payload["summary"]["lane_count"] >= 10 + assert payload["summary"]["worker_required_count"] >= 1 + assert "commerce.marketplace_visual_offer_cards" in lanes + assert lanes["commerce.marketplace_visual_offer_cards"]["priority"] == "P0" + assert lanes["commerce.marketplace_visual_offer_cards"]["next_machine_action"] == ( + "run_ollama_first_vlm_replay_worker" + ) + assert "Google Merchant product data" in lanes[ + "commerce.product_feed_and_structured_data_gap_check" + ]["integrates_with"] + assert lanes["commerce.platform_barrier_classifier"]["status"] == ( + "production_contract_ready" + ) + assert lanes["forbidden.personalized_or_private_state"]["status"] == "forbidden" + assert payload["controlled_apply"]["db_write"] is False + assert payload["controlled_apply"]["primary_human_gate_count"] == 0 + assert any(item["id"] == "pixelrag_visual_rag_pattern" for item in payload["source_basis"]) + assert payload["recommended_order"][0]["priority"] == "P0" + + +def test_pixelrag_application_portfolio_filters_area_without_forbidden_by_default(): + from services.pixelrag_application_portfolio_service import build_pixelrag_application_portfolio + + payload = build_pixelrag_application_portfolio(area="governance") + + assert {item["area"] for item in payload["lanes"]} == {"governance"} + assert all(item["status"] != "forbidden" for item in payload["lanes"]) + assert payload["summary"]["lane_count"] == 1 + assert payload["lanes"][0]["id"] == "commerce.platform_barrier_classifier" + + +def test_pixelrag_application_portfolio_cli_outputs_machine_readable_json(): + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/report_pixelrag_application_portfolio.py", + "--area", + "commerce", + "--priority", + "P1", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["success"] is True + assert payload["filters"]["area"] == "commerce" + assert payload["filters"]["priority"] == "P1" + assert {item["priority"] for item in payload["lanes"]} == {"P1"} + assert payload["controlled_apply"]["network_call"] is False + + +def test_pixelrag_application_portfolio_route_returns_readback(): + from flask import Flask + from routes import system_public_routes as routes + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-application-portfolio?area=rag" + ): + response = routes.ai_automation_pixelrag_application_portfolio_api.__wrapped__() + payload = response.get_json() + + assert payload["policy"] == "read_only_pixelrag_application_portfolio_v1" + assert payload["filters"]["area"] == "rag" + assert {item["area"] for item in payload["lanes"]} == {"rag"} + assert payload["next_machine_action"] == "run_ollama_first_vlm_replay_worker" From 55ce12e6b030fb28ae517322a456734e887e29b4 Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 23:30:43 +0800 Subject: [PATCH 07/75] =?UTF-8?q?feat(ai):=20=E5=BB=BA=E7=AB=8B=20PixelRAG?= =?UTF-8?q?=20VLM=20replay=20worker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO_NEXT_STEPS.txt | 5 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 2 + .../ai_automation_mainline_work_items.md | 5 +- docs/guides/ai_automation_session_sop.md | 4 +- docs/guides/browse_sh_crawler_playbook.md | 10 + routes/system_public_routes.py | 39 ++ scripts/ops/run_pixelrag_vlm_replay_worker.py | 98 ++++ services/ai_automation_smoke_service.py | 99 ++++ .../pixelrag_vlm_replay_worker_service.py | 532 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 45 +- ...test_pixelrag_vlm_replay_worker_service.py | 181 ++++++ 12 files changed, 1012 insertions(+), 10 deletions(-) create mode 100755 scripts/ops/run_pixelrag_vlm_replay_worker.py create mode 100644 services/pixelrag_vlm_replay_worker_service.py create mode 100644 tests/test_pixelrag_vlm_replay_worker_service.py diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 566b88d..1fd171e 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -9,7 +9,7 @@ python scripts/ops/check_production_version_truth.py 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 -本輪 source target 為 `V10.748`；部署完成前不得宣稱正式環境已是 `V10.748`。 +本輪 source target 為 `V10.749`；部署完成前不得宣稱正式環境已是 `V10.749`。 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ @@ -60,8 +60,9 @@ P0-2026-07-09. PixelRAG / MCP / RAG 全自動主線 - 已完成：多電商 PixelRAG visual evidence lane 與 external MCP/RAG integration readback。 - 已完成：PixelRAG receipt → internal RAG candidate replay，以及 OCR/VLM replay contract no-write readback。 - 已完成：PixelRAG application portfolio，把可整合/可運用場景整理成 API/CLI 可讀的 priority lane、status、next machine action 與 forbidden guardrail。 + - 已完成：PixelRAG Ollama-first VLM replay worker，提供 dry-run/execute、artifact receipt、blocked page guard、confidence/evidence validation，且 DB write=0、primary human gate=0。 - 進行中：MCP/RAG runtime health → AI automation smoke。 - - 未開始：Ollama-first OCR/VLM extraction worker、Ollama-first visual embedding benchmark、pgvector-compatible visual metadata、Coupang platform probe / structured API、跨平台 source contracts。 + - 未開始：Ollama-first visual embedding benchmark、pgvector-compatible visual metadata、Coupang platform probe / structured API、跨平台 source contracts。 - 主線文件：`docs/guides/ai_automation_mainline_work_items.md`。 ================================================================================ diff --git a/config.py b/config.py index 05c9347..982dab1 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.748" +SYSTEM_VERSION = "V10.749" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 81ba400..80aee45 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -116,6 +116,7 @@ - 2026-07-09 起 PixelRAG visual receipts 必須可轉為內部 RAG candidate replay readback：`/api/ai-automation/pixelrag-rag-candidate-replay` 與 `scripts/ops/report_pixelrag_rag_candidate_replay.py` 只讀 `capture_receipt.json`，拆分 eligible / blocked / invalid receipts，輸出 tile/missing/barrier count、candidate text、promotion boundary 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG RAG candidate replay`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_rag_candidate_replay` family。blocked / 403 / captcha / access denied / verify traffic page 只能進 platform probe 或 structured API 策略，不得當作商品資料、不得寫正式價格表、不得直接寫 `ai_insights`。 - 2026-07-09 起 PixelRAG visual receipts 進入 OCR/VLM 前必須先輸出 no-write replay contract：`/api/ai-automation/pixelrag-ocr-vlm-replay` 與 `scripts/ops/report_pixelrag_ocr_vlm_replay.py` 只讀 saved tiles，輸出欄位 schema、輸出 schema、confidence/evidence validation rules、Ollama-first route contract 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG OCR/VLM replay contract`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_ocr_vlm_replay` family。此 contract 不執行 OCR/VLM、不呼叫模型、不讀 secret、不連外、不寫 DB、不寫 `ai_insights`、不寫正式價格表；blocked / 403 / captcha / access denied / verify traffic page 只能進 platform probe 或 structured API 策略。 - 2026-07-09 起 PixelRAG 可整合/可運用範圍必須有機器可讀 application portfolio：`/api/ai-automation/pixelrag-application-portfolio` 與 `scripts/ops/report_pixelrag_application_portfolio.py` 需輸出 commerce、RAG、UX、ops、marketing、governance lanes，每條 lane 必須包含 priority、status、integrates_with、use_cases、current_capability、next_machine_action、no-write 邊界與 forbidden guardrails。此 portfolio 不抓外站、不呼叫模型、不讀 secret、不寫 DB；其用途是把「還可以整合哪些」變成可排程、可驗證、可拒絕違規場景的主線工作項目。 +- 2026-07-09 起 PixelRAG ready receipts 必須有 Ollama-first VLM replay worker：`/api/ai-automation/pixelrag-vlm-replay-worker` 與 `scripts/ops/run_pixelrag_vlm_replay_worker.py` 預設 dry-run，`execute=true&write_receipt=true` 才呼叫 approved Ollama VLM route 並寫 artifact receipt；`/api/ai-automation/smoke` 需包含 `PixelRAG VLM replay worker`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_vlm_replay_worker` family。此 worker 不讀 secret、不寫 DB、不寫 `ai_insights`、不寫正式價格表；blocked / 403 / captcha / access denied receipt 自動跳 platform probe 或 structured API，ready receipt 的 VLM 結果仍須 identity matcher replay 與 PromotionGate 才能進候選知識層。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -878,6 +879,7 @@ POSTGRES_HOST=momo-db | 2026-07-09 | PixelRAG / external MCP/RAG 必須有 runtime monitoring 與 candidate replay | V10.746 起 `/api/ai-automation/smoke`、`/api/ai-automation/scheduled-health-summary` 必須輸出 `External MCP/RAG integration readback` / `external_mcp_rag_integration` 與 `PixelRAG RAG candidate replay` / `pixelrag_rag_candidate_replay`；external MCP/RAG readback 回報 9 個 capability 的 absorbed / unresolved 與 `MCP_ROUTER_ENABLED`、`RAG_ENABLED` runtime flags，PixelRAG replay 只讀 visual receipt，拆分 eligible / blocked / invalid，並明確標記 blocked page 不是商品資料。此路徑不讀 secret、不呼叫外部網路、不寫 DB、不寫 `ai_insights`、不寫正式價格表；eligible receipt 仍需 OCR/VLM replay、identity matcher replay、PromotionGate 與 embedding signature guard。 | | 2026-07-09 | PixelRAG OCR/VLM replay contract 必須有 runtime monitoring | V10.747 起 `/api/ai-automation/pixelrag-ocr-vlm-replay`、`scripts/ops/report_pixelrag_ocr_vlm_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 no-write OCR/VLM replay contract / `pixelrag_ocr_vlm_replay` family；readback 只讀 saved tiles 與 RAG candidate replay，輸出 ready / blocked / invalid contracts、field schema、output schema、validation rules、Ollama-first route contract、blocked page guard 與 next machine action。此階段明確標記 `extraction_execution_performed=false`、`ocr_execution_performed=false`、`vlm_execution_performed=false`、`writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`network_call=false`、`secret_read=false`、`primary_human_gate_count=0`；ready receipt 才能進下一段 `run_ollama_first_vlm_replay_worker`，blocked receipt 只能進 platform probe 或 structured API 策略。 | | 2026-07-09 | PixelRAG application portfolio 必須把可整合場景轉成主線工作項目 | V10.748 起 `/api/ai-automation/pixelrag-application-portfolio` 與 `scripts/ops/report_pixelrag_application_portfolio.py` 必須輸出 PixelRAG 在 commerce、RAG、UX、ops、marketing、governance 的可整合/可運用 lanes；每條 lane 需有 priority、status、integrates_with、use_cases、current_capability、next_machine_action、no-write 邊界與 forbidden guardrails。此 readback 依據 PixelRAG visual-RAG pattern、Google Merchant product data、Google Product structured data 與 Baymard product list UX 轉成內部工作項目；它不抓外站、不呼叫模型、不讀 secret、不寫 DB、不把像素結果當正式價格。 | +| 2026-07-09 | PixelRAG Ollama-first VLM replay worker 必須有 runtime monitoring | V10.749 起 `/api/ai-automation/pixelrag-vlm-replay-worker`、`scripts/ops/run_pixelrag_vlm_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled VLM replay worker / `pixelrag_vlm_replay_worker` family；readback 預設 dry-run，不呼叫模型、不寫 artifact，execute 模式只讀 saved tiles、呼叫 approved Ollama VLM route、驗證 JSON field confidence/evidence refs，並只寫 artifact receipt。此 worker 明確標記 `writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`primary_human_gate_count=0`；blocked page 不得輸出商品欄位，ready VLM 結果仍需 identity matcher replay 與 PromotionGate。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index b24a1dd..2292dbe 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -18,7 +18,8 @@ | Completed | External MCP/RAG capability inventory absorbed into internal governance readback | `/api/ai-automation/external-mcp-rag-integration` and `scripts/ops/report_external_mcp_rag_integration.py` expose 9 capabilities, absorbed/unresolved counts, and runtime flags. | | Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read receipts, split eligible vs blocked, and require OCR/VLM replay plus PromotionGate before knowledge writes. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | -| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, and PixelRAG OCR/VLM replay contract families. | +| Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | +| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, and PixelRAG VLM replay worker families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | ## P1 @@ -26,7 +27,7 @@ | Status | Work item | Evidence / next machine action | |---|---|---| | Completed | OCR/VLM replay contract for visual fields | `/api/ai-automation/pixelrag-ocr-vlm-replay` and `scripts/ops/report_pixelrag_ocr_vlm_replay.py` turn saved tiles into no-write field contracts, output schemas, validation rules, and Ollama-first worker actions. | -| Not started | Ollama-first OCR/VLM extraction worker | Execute the replay contract against approved local multimodal/OCR models and emit confidence/evidence receipts without writing formal price truth. | +| Completed | Ollama-first OCR/VLM extraction worker | `run_pixelrag_vlm_replay_worker.py --execute --write-receipt` executes the replay contract against approved Ollama multimodal routes and emits confidence/evidence artifact receipts without writing formal price truth. | | Not started | Ollama-first multimodal embedding benchmark | Verify local Qwen3-VL or equivalent visual embedding on GCP-A -> GCP-B -> 111 before any visual vector retrieval. | | Not started | pgvector-compatible visual evidence metadata | Design metadata-first retrieval without FAISS in production unless ADR approves a different store. | | Not started | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; prefer public structured data or approved probe, never count blocked pages as product data. | diff --git a/docs/guides/ai_automation_session_sop.md b/docs/guides/ai_automation_session_sop.md index 459d619..4c30080 100644 --- a/docs/guides/ai_automation_session_sop.md +++ b/docs/guides/ai_automation_session_sop.md @@ -34,7 +34,9 @@ 或 `python scripts/ops/report_pixelrag_ocr_vlm_replay.py` 可讀回 ready / blocked / invalid replay contracts、field schema、validation rules 與 Ollama-first 下一步，且目前不執行 OCR/VLM、不寫正式價格。 - PixelRAG 可整合/可運用場景盤點必須確認 `/api/ai-automation/pixelrag-application-portfolio` 或 `python scripts/ops/report_pixelrag_application_portfolio.py` 可讀回 area、priority、status、use cases、next machine action 與 forbidden guardrails；不得只存在聊天結論。 -- AI automation smoke 必須包含 external MCP/RAG integration、PixelRAG RAG candidate replay 與 PixelRAG OCR/VLM replay contract family，避免 registry 已完成但 runtime flag / receipt replay / VLM contract 未完成時被誤報為全自動閉環。 +- PixelRAG VLM replay worker 必須確認 `/api/ai-automation/pixelrag-vlm-replay-worker` + 或 `python scripts/ops/run_pixelrag_vlm_replay_worker.py` 可讀回 dry-run/execute、model_call、artifact receipt、blocked/ready 分流、DB write=0 與 primary human gate=0；execute 結果仍只是 candidate evidence。 +- AI automation smoke 必須包含 external MCP/RAG integration、PixelRAG RAG candidate replay、PixelRAG OCR/VLM replay contract 與 PixelRAG VLM replay worker family，避免 registry 已完成但 runtime flag / receipt replay / VLM worker 未完成時被誤報為全自動閉環。 - AI 自動化 Prometheus 指標變更必須同步檢查 `docker/grafana/provisioning/dashboards/json/ai-automation-overview.json` 是否需要新增 panel 或查詢。 - 188 線上 active monitoring stack 以 `monitoring/prometheus.yml` 為準；110 gateway 另有 `/home/wooo/monitoring/prometheus.yml`。若 dashboard 無資料，先確認 Prometheus `momo-app` target 與 `momo-network` 連線；所有 Blackbox HTTP target 必須打 `/health`，不可打 Dashboard 首頁 `/`。 - Smoke dashboard 會保存 JSONL 趨勢；若新增檢查項目，要確保 history compact record 仍保持小而可讀。 diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index 1949117..f21b4c2 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -184,6 +184,16 @@ python scripts/ops/report_pixelrag_ocr_vlm_replay.py --platform shopee_tw --plat 此 contract 只把 saved tiles、欄位 schema、輸出 schema、confidence/evidence 規則與 Ollama-first worker 下一步打包；目前不執行 OCR/VLM、不呼叫模型、不寫 RAG、不寫 `ai_insights`、不寫正式價格表。ready receipt 進 `run_ollama_first_vlm_replay_worker`，blocked / 403 / captcha / access denied receipt 進 platform probe 或 structured API 策略。 +Ollama-first VLM replay worker： + +```bash +python scripts/ops/run_pixelrag_vlm_replay_worker.py +python scripts/ops/run_pixelrag_vlm_replay_worker.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_vlm_replay_worker.py --platform shopee_tw --execute --write-receipt --limit 1 +``` + +API readback: `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 + Application portfolio： ```bash diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index f0cf283..acf99d2 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -753,6 +753,45 @@ def ai_automation_pixelrag_application_portfolio_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-vlm-replay-worker') +@login_required +def ai_automation_pixelrag_vlm_replay_worker_api(): + """Dry-run or execute the PixelRAG Ollama-first VLM replay worker.""" + from services.pixelrag_vlm_replay_worker_service import ( + run_pixelrag_ollama_vlm_replay_worker, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + execute = str(request.args.get('execute') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + write_receipt = str(request.args.get('write_receipt') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 25, type=int) + tile_limit = request.args.get('tile_limit', 4, type=int) + timeout = request.args.get('timeout', 90, type=int) + return jsonify(run_pixelrag_ollama_vlm_replay_worker( + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 25, 250)), + tile_limit=max(1, min(tile_limit or 4, 12)), + model=str(request.args.get('model') or '').strip() or None, + timeout=max(10, min(timeout or 90, 240)), + execute=execute, + write_receipt=bool(execute and write_receipt), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/run_pixelrag_vlm_replay_worker.py b/scripts/ops/run_pixelrag_vlm_replay_worker.py new file mode 100755 index 0000000..5416d80 --- /dev/null +++ b/scripts/ops/run_pixelrag_vlm_replay_worker.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Run or dry-run the PixelRAG Ollama VLM replay worker.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_vlm_replay_worker_service import ( # noqa: E402 + DEFAULT_MODEL, + run_pixelrag_ollama_vlm_replay_worker, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="執行或 dry-run PixelRAG Ollama-first VLM replay worker。" + ) + parser.add_argument( + "--artifact-root", + help="PixelRAG visual evidence artifact root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--output-root", + help="VLM replay artifact receipt output root。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=25, + help="最多處理 receipt 數。", + ) + parser.add_argument( + "--tile-limit", + type=int, + default=4, + help="每個 receipt 最多送入 VLM 的 tile 數。", + ) + parser.add_argument( + "--model", + default=DEFAULT_MODEL, + help="Ollama VLM model。", + ) + parser.add_argument( + "--timeout", + type=int, + default=90, + help="單次 Ollama generate timeout 秒數。", + ) + parser.add_argument( + "--execute", + action="store_true", + help="真的呼叫 Ollama VLM；未指定時只做 no-write dry-run。", + ) + parser.add_argument( + "--write-receipt", + action="store_true", + help="execute 後寫入 VLM replay artifact receipt；不寫 DB。", + ) + args = parser.parse_args() + + payload = run_pixelrag_ollama_vlm_replay_worker( + artifact_root=args.artifact_root, + output_root=args.output_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + tile_limit=args.tile_limit, + model=args.model, + timeout=args.timeout, + execute=args.execute, + write_receipt=bool(args.write_receipt and args.execute), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index 88990f8..63db710 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -542,6 +542,11 @@ def build_scheduled_automation_health_summary( if not pixelrag_ocr_vlm_replay or not pixelrag_ocr_vlm_replay_details: pixelrag_ocr_vlm_replay = _pixelrag_ocr_vlm_replay_check() pixelrag_ocr_vlm_replay_details = pixelrag_ocr_vlm_replay.get("details") or {} + pixelrag_vlm_replay_worker = _find_check(source_result, "PixelRAG VLM replay worker") + pixelrag_vlm_replay_worker_details = pixelrag_vlm_replay_worker.get("details") or {} + if not pixelrag_vlm_replay_worker or not pixelrag_vlm_replay_worker_details: + pixelrag_vlm_replay_worker = _pixelrag_vlm_replay_worker_check() + pixelrag_vlm_replay_worker_details = pixelrag_vlm_replay_worker.get("details") or {} smoke_status = source_result.get("status") or ("warning" if latest_history else "warning") freshness_family = _history_freshness_family( latest_history, @@ -4632,6 +4637,42 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_vlm_replay_worker", + "label": "PixelRAG VLM replay worker", + "status": pixelrag_vlm_replay_worker.get("status") or "warning", + "summary": ( + pixelrag_vlm_replay_worker.get("summary") + or "PixelRAG VLM replay worker has no latest readback." + ), + "next_machine_action": pixelrag_vlm_replay_worker_details.get("next_machine_action") + or "run_pixelrag_vlm_replay_worker_dry_run", + "details": { + "policy": pixelrag_vlm_replay_worker_details.get("policy"), + "receipt_count": int(pixelrag_vlm_replay_worker_details.get("receipt_count") or 0), + "ready_count": int(pixelrag_vlm_replay_worker_details.get("ready_count") or 0), + "dry_run_count": int(pixelrag_vlm_replay_worker_details.get("dry_run_count") or 0), + "executed_count": int(pixelrag_vlm_replay_worker_details.get("executed_count") or 0), + "executed_ok_count": int(pixelrag_vlm_replay_worker_details.get("executed_ok_count") or 0), + "executed_warning_count": int( + pixelrag_vlm_replay_worker_details.get("executed_warning_count") or 0 + ), + "model_error_count": int(pixelrag_vlm_replay_worker_details.get("model_error_count") or 0), + "parse_error_count": int(pixelrag_vlm_replay_worker_details.get("parse_error_count") or 0), + "receipt_written_count": int( + pixelrag_vlm_replay_worker_details.get("receipt_written_count") or 0 + ), + "model_call_performed": bool( + pixelrag_vlm_replay_worker_details.get("model_call_performed") + ), + "artifact_write_performed": bool( + pixelrag_vlm_replay_worker_details.get("artifact_write_performed") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, freshness_family, { "key": "daily_summary_delivery", @@ -13350,6 +13391,63 @@ def _pixelrag_ocr_vlm_replay_check() -> Dict[str, Any]: ) +def _pixelrag_vlm_replay_worker_check() -> Dict[str, Any]: + """Dry-run sentinel for the Ollama-first PixelRAG VLM replay worker.""" + try: + from services.pixelrag_vlm_replay_worker_service import ( + run_pixelrag_ollama_vlm_replay_worker, + ) + + readback = run_pixelrag_ollama_vlm_replay_worker(execute=False) + summary = readback.get("summary") or {} + receipt_count = int(summary.get("receipt_count") or 0) + ready_count = int(summary.get("ready_count") or 0) + dry_run_count = int(summary.get("dry_run_count") or 0) + skipped_count = int(summary.get("skipped_count") or 0) + executed_count = int(summary.get("executed_count") or 0) + status = readback.get("status") or "warning" + summary_text = ( + f"PixelRAG VLM replay worker receipts={receipt_count}, " + f"ready={ready_count}, dry_run={dry_run_count}, skipped={skipped_count}, " + f"executed={executed_count}" + ) + return _check( + "PixelRAG VLM replay worker", + status, + summary_text, + { + "policy": readback.get("policy"), + "receipt_count": receipt_count, + "ready_count": ready_count, + "skipped_count": skipped_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "executed_ok_count": int(summary.get("executed_ok_count") or 0), + "executed_warning_count": int(summary.get("executed_warning_count") or 0), + "model_error_count": int(summary.get("model_error_count") or 0), + "parse_error_count": int(summary.get("parse_error_count") or 0), + "receipt_written_count": int(summary.get("receipt_written_count") or 0), + "model_call_performed": bool(summary.get("model_call_performed")), + "artifact_write_performed": bool(summary.get("artifact_write_performed")), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG VLM replay worker", + "critical", + f"PixelRAG VLM replay worker 無法執行 dry-run：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: int = 20) -> Dict[str, Any]: checks: List[Dict[str, Any]] = [ _event_router_check(), @@ -13386,6 +13484,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _external_mcp_rag_integration_check(), _pixelrag_rag_candidate_replay_check(), _pixelrag_ocr_vlm_replay_check(), + _pixelrag_vlm_replay_worker_check(), ] worst = max(checks, key=lambda item: STATUS_RANK.get(item["status"], 2))["status"] result = { diff --git a/services/pixelrag_vlm_replay_worker_service.py b/services/pixelrag_vlm_replay_worker_service.py new file mode 100644 index 0000000..d6704b0 --- /dev/null +++ b/services/pixelrag_vlm_replay_worker_service.py @@ -0,0 +1,532 @@ +"""Ollama-first PixelRAG VLM replay worker. + +This worker executes the next machine action emitted by the PixelRAG +OCR/VLM replay contract. It reads saved screenshot tiles, calls approved +Ollama hosts, validates evidence-bound JSON fields, and optionally writes an +artifact receipt. It never writes DB rows, AI insights, or price truth. +""" + +from __future__ import annotations + +import base64 +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.ollama_service import OllamaService, get_host_label, get_provider_tag +from services.pixelrag_crawler_integration_service import ( + DEFAULT_ARTIFACT_MAX_AGE_HOURS, + DEFAULT_ARTIFACT_ROOT, +) +from services.pixelrag_ocr_vlm_replay_service import ( + DEFAULT_CONFIDENCE_THRESHOLD, + build_pixelrag_ocr_vlm_replay_contract, +) + + +POLICY = "controlled_pixelrag_ollama_vlm_replay_worker_v1" +DEFAULT_LIMIT = 25 +DEFAULT_TILE_LIMIT = 4 +DEFAULT_TIMEOUT_SECONDS = 90 +DEFAULT_OUTPUT_ROOT = os.getenv( + "PIXELRAG_VLM_REPLAY_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_vlm_replay_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_vlm_replay_receipts", +) +DEFAULT_MODEL = ( + os.getenv("PIXELRAG_VLM_MODEL") + or os.getenv("PPT_VISION_MODEL") + or "minicpm-v:latest" +) +RAW_EXCERPT_LIMIT = 500 + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...]: + if isinstance(platform, str): + value = platform.strip().lower() + return (value,) if value else () + return tuple( + str(item or "").strip().lower() + for item in (platform or ()) + if str(item or "").strip() + ) + + +def _safe_segment(value: Any) -> str: + text = str(value or "unknown").strip().lower() + text = re.sub(r"[^a-z0-9._-]+", "-", text) + return text.strip("-") or "unknown" + + +def _resolve_tile_path(path: str, root: Path) -> Path: + tile_path = Path(str(path or "").strip()) + if tile_path.is_absolute(): + return tile_path + return root / tile_path + + +def _tile_images(item: Mapping[str, Any], *, root: Path, tile_limit: int) -> tuple[list[str], list[dict[str, Any]]]: + images: list[str] = [] + evidence: list[dict[str, Any]] = [] + for tile in list(item.get("input_tiles") or [])[:tile_limit]: + evidence_ref = str(tile.get("evidence_ref") or "") + path = _resolve_tile_path(str(tile.get("path") or ""), root) + tile_evidence = { + "evidence_ref": evidence_ref, + "path": str(path), + "exists": path.exists(), + "loaded": False, + } + if path.exists(): + raw = path.read_bytes() + images.append(base64.b64encode(raw).decode("ascii")) + tile_evidence["loaded"] = True + tile_evidence["byte_size"] = len(raw) + evidence.append(tile_evidence) + return images, evidence + + +def _extract_json_object(content: str) -> dict[str, Any]: + text = str(content or "").strip() + if not text: + raise ValueError("empty_model_output") + if text.startswith("```"): + text = re.sub(r"^```(?:json)?\s*", "", text, flags=re.IGNORECASE) + text = re.sub(r"\s*```$", "", text) + try: + parsed = json.loads(text) + except json.JSONDecodeError: + start = text.find("{") + end = text.rfind("}") + if start < 0 or end <= start: + raise + parsed = json.loads(text[start:end + 1]) + if not isinstance(parsed, dict): + raise ValueError("model_output_not_json_object") + return parsed + + +def _prompt_for_item(item: Mapping[str, Any]) -> str: + field_contract = list(item.get("field_contract") or []) + compact_contract = [ + { + "field": field.get("field"), + "type": field.get("type"), + "required": bool(field.get("required")), + "min_confidence": field.get("min_confidence"), + "evidence_requirement": field.get("evidence_requirement"), + } + for field in field_contract + ] + metadata = { + "platform": item.get("platform"), + "manifest_id": item.get("manifest_id"), + "url": item.get("url"), + "title_hint": item.get("title_hint"), + "http_status": item.get("http_status"), + "field_contract": compact_contract, + "input_evidence_refs": [ + tile.get("evidence_ref") for tile in list(item.get("input_tiles") or []) + ], + } + return ( + "You are a strict public marketplace offer-card VLM extractor.\n" + "Return only valid JSON. Do not use markdown. Do not guess.\n" + "Use only visible tile evidence and cite evidence_refs like tile:1.\n" + "If the tile is access denied, captcha, login, traffic verification, or not a product/search card, " + "set blocked_page_detected=true and leave product fields empty.\n" + "Required JSON schema:\n" + "{\n" + ' "blocked_page_detected": false,\n' + ' "fields": {"field_name": {"value": null, "confidence": 0.0, "evidence_refs": []}},\n' + ' "quality": {"overall_confidence": 0.0, "missing_required_fields": [], ' + '"requires_identity_matcher_replay": true, "requires_promotion_gate": true},\n' + ' "notes": []\n' + "}\n" + "Metadata and field contract:\n" + f"{json.dumps(metadata, ensure_ascii=False, sort_keys=True)}" + ) + + +def _field_value_present(value: Any) -> bool: + if value is None: + return False + if isinstance(value, str): + return bool(value.strip()) + return True + + +def _validate_model_payload( + parsed: Mapping[str, Any], + item: Mapping[str, Any], +) -> dict[str, Any]: + fields = parsed.get("fields") if isinstance(parsed.get("fields"), Mapping) else {} + quality = parsed.get("quality") if isinstance(parsed.get("quality"), Mapping) else {} + missing_required: list[str] = [] + field_evidence_missing: list[str] = [] + low_confidence_fields: list[str] = [] + blocked_detected = bool(parsed.get("blocked_page_detected")) + + for contract in list(item.get("field_contract") or []): + field_name = str(contract.get("field") or "") + field_payload = fields.get(field_name) if isinstance(fields, Mapping) else {} + if not isinstance(field_payload, Mapping): + field_payload = {} + value = field_payload.get("value") + evidence_refs = list(field_payload.get("evidence_refs") or []) + try: + confidence = float(field_payload.get("confidence") or 0) + except (TypeError, ValueError): + confidence = 0.0 + min_confidence = float(contract.get("min_confidence") or DEFAULT_CONFIDENCE_THRESHOLD) + present = _field_value_present(value) + if present and not evidence_refs: + field_evidence_missing.append(field_name) + if present and confidence < min_confidence: + low_confidence_fields.append(field_name) + if contract.get("required") and (blocked_detected or not present or confidence < min_confidence): + missing_required.append(field_name) + + declared_missing = [ + str(field) + for field in list(quality.get("missing_required_fields") or []) + if str(field).strip() + ] + for field in declared_missing: + if field not in missing_required: + missing_required.append(field) + + return { + "blocked_page_detected": blocked_detected, + "missing_required_fields": missing_required, + "field_evidence_missing": field_evidence_missing, + "low_confidence_fields": low_confidence_fields, + "valid_for_identity_matcher_replay": ( + not blocked_detected + and not missing_required + and not field_evidence_missing + ), + "requires_identity_matcher_replay": bool( + quality.get("requires_identity_matcher_replay", True) + ), + "requires_promotion_gate": bool(quality.get("requires_promotion_gate", True)), + } + + +def _write_replay_receipt( + *, + output_root: Path, + item: Mapping[str, Any], + worker_item: Mapping[str, Any], +) -> str: + target = ( + output_root + / _safe_segment(item.get("platform")) + / _safe_segment(item.get("manifest_id")) + / "vlm_replay_receipt.json" + ) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + json.dumps(worker_item, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + return str(target) + + +def _skipped_item(item: Mapping[str, Any]) -> dict[str, Any]: + return { + "platform": item.get("platform"), + "manifest_id": item.get("manifest_id"), + "source_receipt_path": item.get("source_receipt_path"), + "worker_status": "skipped_blocked_or_not_ready", + "replay_status": item.get("replay_status"), + "blocked_reasons": list(item.get("blocked_reasons") or []), + "model_call_performed": False, + "artifact_write_performed": False, + "writes_database": False, + "next_machine_action": item.get("next_machine_action") + or "run_platform_probe_or_use_structured_api", + } + + +def _dry_run_item(item: Mapping[str, Any]) -> dict[str, Any]: + return { + "platform": item.get("platform"), + "manifest_id": item.get("manifest_id"), + "source_receipt_path": item.get("source_receipt_path"), + "worker_status": "dry_run_ready", + "ready_for_execution": True, + "tile_input_count": len(list(item.get("input_tiles") or [])), + "field_contract_count": int(item.get("field_contract_count") or 0), + "model_call_performed": False, + "artifact_write_performed": False, + "writes_database": False, + "next_machine_action": "run_pixelrag_vlm_replay_worker_execute", + } + + +def _execute_item( + item: Mapping[str, Any], + *, + root: Path, + output_root: Path, + model: str, + timeout: int, + tile_limit: int, + write_receipt: bool, +) -> dict[str, Any]: + images, tile_evidence = _tile_images(item, root=root, tile_limit=tile_limit) + base: dict[str, Any] = { + "platform": item.get("platform"), + "manifest_id": item.get("manifest_id"), + "source_receipt_path": item.get("source_receipt_path"), + "worker_status": "executing", + "model": model, + "tile_evidence": tile_evidence, + "tile_image_count": len(images), + "model_call_performed": bool(images), + "artifact_write_performed": False, + "writes_database": False, + } + if not images: + base.update({ + "worker_status": "skipped_no_loadable_tiles", + "next_machine_action": "refresh_pixelrag_visual_capture_receipt", + }) + return base + + response = OllamaService(model=model).generate( + _prompt_for_item(item), + model=model, + temperature=0.1, + timeout=max(10, int(timeout or DEFAULT_TIMEOUT_SECONDS)), + options={"num_predict": 700, "num_ctx": 4096}, + images=images, + ) + base.update({ + "host": response.host, + "host_label": get_host_label(response.host or ""), + "provider": get_provider_tag(response.host or ""), + "actual_model": response.model, + "input_tokens": int(response.input_tokens or 0), + "output_tokens": int(response.output_tokens or 0), + "total_duration": response.total_duration, + }) + if not response.success: + base.update({ + "worker_status": "model_error", + "model_error": str(response.error or "")[:RAW_EXCERPT_LIMIT], + "next_machine_action": "repair_ollama_vlm_runtime_or_model_route", + }) + return base + + try: + parsed = _extract_json_object(response.content) + except Exception as exc: + base.update({ + "worker_status": "model_output_parse_error", + "parse_error": str(exc)[:RAW_EXCERPT_LIMIT], + "raw_model_output_excerpt": str(response.content or "")[:RAW_EXCERPT_LIMIT], + "next_machine_action": "tighten_pixelrag_vlm_prompt_or_model", + }) + if write_receipt: + base["receipt_path"] = _write_replay_receipt( + output_root=output_root, + item=item, + worker_item=base, + ) + base["artifact_write_performed"] = True + return base + + validation = _validate_model_payload(parsed, item) + missing_required = list(validation.get("missing_required_fields") or []) + evidence_missing = list(validation.get("field_evidence_missing") or []) + blocked_detected = bool(validation.get("blocked_page_detected")) + status = "executed_ok" + next_action = "run_identity_matcher_replay_then_promotion_gate" + if blocked_detected: + status = "executed_warning" + next_action = "run_platform_probe_or_use_structured_api" + elif missing_required or evidence_missing: + status = "executed_warning" + next_action = "rerun_vlm_replay_with_more_tiles_or_ocr" + + base.update({ + "worker_status": status, + "parsed_output": parsed, + "validation": validation, + "required_field_missing_count": len(missing_required), + "field_evidence_missing_count": len(evidence_missing), + "next_machine_action": next_action, + }) + if write_receipt: + base["receipt_path"] = _write_replay_receipt( + output_root=output_root, + item=item, + worker_item=base, + ) + base["artifact_write_performed"] = True + return base + + +def run_pixelrag_ollama_vlm_replay_worker( + *, + artifact_root: str | Path | None = None, + output_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, + tile_limit: int | None = None, + model: str | None = None, + timeout: int | None = None, + execute: bool = False, + write_receipt: bool = False, +) -> dict[str, Any]: + """Run or dry-run the PixelRAG VLM replay worker.""" + root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT) + output = Path(output_root or DEFAULT_OUTPUT_ROOT) + platforms = _normalise_platforms(platform) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + tiles = max(1, min(int(tile_limit or DEFAULT_TILE_LIMIT), 12)) + selected_model = str(model or DEFAULT_MODEL) + selected_timeout = max(10, int(timeout or DEFAULT_TIMEOUT_SECONDS)) + generated_at = datetime.now(timezone.utc).isoformat() + + contract = build_pixelrag_ocr_vlm_replay_contract( + artifact_root=root, + platform=platforms, + max_age_hours=max_age, + limit=item_limit, + ) + replay_items = list(contract.get("replay_items") or []) + worker_items: list[dict[str, Any]] = [] + for item in replay_items: + if not item.get("ready_for_ollama_vlm_worker"): + worker_items.append(_skipped_item(item)) + continue + if not execute: + worker_items.append(_dry_run_item(item)) + continue + worker_items.append(_execute_item( + item, + root=root, + output_root=output, + model=selected_model, + timeout=selected_timeout, + tile_limit=tiles, + write_receipt=write_receipt, + )) + + ready_count = sum(1 for item in replay_items if item.get("ready_for_ollama_vlm_worker")) + skipped_count = sum(1 for item in worker_items if item.get("worker_status") == "skipped_blocked_or_not_ready") + dry_run_count = sum(1 for item in worker_items if item.get("worker_status") == "dry_run_ready") + executed_count = sum(1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_")) + executed_ok_count = sum(1 for item in worker_items if item.get("worker_status") == "executed_ok") + executed_warning_count = sum(1 for item in worker_items if item.get("worker_status") == "executed_warning") + model_error_count = sum(1 for item in worker_items if item.get("worker_status") == "model_error") + parse_error_count = sum(1 for item in worker_items if item.get("worker_status") == "model_output_parse_error") + no_tile_count = sum(1 for item in worker_items if item.get("worker_status") == "skipped_no_loadable_tiles") + receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path")) + required_missing_count = sum( + int(item.get("required_field_missing_count") or 0) + for item in worker_items + ) + model_call_performed = any(bool(item.get("model_call_performed")) for item in worker_items) + artifact_write_performed = any(bool(item.get("artifact_write_performed")) for item in worker_items) + + if parse_error_count or model_error_count or no_tile_count: + status = "critical" if ready_count and executed_ok_count == 0 and execute else "warning" + elif executed_warning_count or skipped_count or dry_run_count or (not replay_items): + status = "warning" + else: + status = "ok" + + if not replay_items: + next_action = "run_pixelrag_visual_capture_worker" + elif not execute and ready_count: + next_action = "run_pixelrag_vlm_replay_worker_execute" + elif model_error_count or parse_error_count: + next_action = "repair_ollama_vlm_runtime_or_model_route" + elif executed_warning_count: + next_action = "rerun_vlm_replay_with_more_tiles_or_platform_probe" + elif executed_ok_count: + next_action = "run_identity_matcher_replay_then_promotion_gate" + else: + next_action = "run_platform_probe_or_use_structured_api" + + summary = { + "receipt_count": len(replay_items), + "ready_count": ready_count, + "skipped_count": skipped_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "executed_ok_count": executed_ok_count, + "executed_warning_count": executed_warning_count, + "model_error_count": model_error_count, + "parse_error_count": parse_error_count, + "no_tile_count": no_tile_count, + "receipt_written_count": receipt_written_count, + "required_field_missing_count": required_missing_count, + "model_call_performed": model_call_performed, + "artifact_write_performed": artifact_write_performed, + "writes_database_count": 0, + "primary_human_gate_count": 0, + "platforms": sorted({str(item.get("platform") or "unknown") for item in replay_items}), + } + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": generated_at, + "artifact_root": str(root), + "output_root": str(output), + "platform_filter": list(platforms), + "max_age_hours": max_age, + "limit": item_limit, + "tile_limit": tiles, + "model": selected_model, + "timeout_seconds": selected_timeout, + "execute": bool(execute), + "write_receipt": bool(write_receipt), + "summary": summary, + "worker_items": worker_items, + "source_contract": { + "policy": contract.get("policy"), + "status": contract.get("status"), + "summary": contract.get("summary"), + "next_machine_action": contract.get("next_machine_action"), + }, + "controlled_apply": { + "network_call": bool(execute and model_call_performed), + "model_call": bool(execute and model_call_performed), + "artifact_write": artifact_write_performed, + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "secret_read": False, + "production_price_write": False, + "primary_human_gate_count": 0, + }, + "promotion_boundary": { + "writes_ai_insights": False, + "writes_price_tables": False, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "visual_fields_are_candidate_evidence_only": True, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "DEFAULT_MODEL", + "POLICY", + "run_pixelrag_ollama_vlm_replay_worker", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index c3fcc76..e203e27 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1434,11 +1434,12 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 32, "warning": 1, "critical": 1, "total": 34} + assert result["summary"] == {"ok": 33, "warning": 1, "critical": 1, "total": 35} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3968,6 +3969,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) first = smoke.collect_ai_automation_smoke(history_limit=5) second = smoke.collect_ai_automation_smoke(history_limit=5) @@ -4023,7 +4025,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 34, "warning": 0, "critical": 0, "total": 34}, + "summary": {"ok": 35, "warning": 0, "critical": 0, "total": 35}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4148,6 +4150,27 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "blocked_pages_are_not_product_data": True, "next_machine_action": "run_ollama_first_vlm_replay_worker", }, + }, + { + "name": "PixelRAG VLM replay worker", + "status": "ok", + "summary": "PixelRAG VLM replay worker receipts=1, ready=1, dry_run=1, skipped=0, executed=0", + "details": { + "policy": "controlled_pixelrag_ollama_vlm_replay_worker_v1", + "receipt_count": 1, + "ready_count": 1, + "skipped_count": 0, + "dry_run_count": 1, + "executed_count": 0, + "executed_ok_count": 0, + "executed_warning_count": 0, + "model_error_count": 0, + "parse_error_count": 0, + "receipt_written_count": 0, + "model_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": "run_pixelrag_vlm_replay_worker_execute", + }, } ], }, ensure_ascii=False) + "\n", @@ -4166,7 +4189,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 31 + assert summary["summary"]["total"] == 32 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -5849,6 +5872,15 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( assert pixelrag_ocr_vlm_replay_family["status"] == "ok" assert pixelrag_ocr_vlm_replay_family["details"]["replay_ready_count"] == 1 assert pixelrag_ocr_vlm_replay_family["details"]["extraction_execution_performed"] is False + pixelrag_vlm_replay_worker_family = next( + item for item in summary["families"] + if item["key"] == "pixelrag_vlm_replay_worker" + ) + assert pixelrag_vlm_replay_worker_family["status"] == "ok" + assert pixelrag_vlm_replay_worker_family["details"]["ready_count"] == 1 + assert pixelrag_vlm_replay_worker_family["details"]["dry_run_count"] == 1 + assert pixelrag_vlm_replay_worker_family["details"]["model_call_performed"] is False + assert pixelrag_vlm_replay_worker_family["details"]["writes_database_count"] == 0 assert summary["scheduled_outputs"]["telegram_send_in_preview"] is False assert summary["scheduled_outputs"]["writes_database"] is False assert summary["automation_policy"]["primary_flow"] == "ai_controlled" @@ -6470,6 +6502,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag ocr vlm ok", )) + monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check( + "PixelRAG VLM replay worker", + "ok", + "pixelrag vlm worker ok", + )) result = smoke.collect_ai_automation_smoke(record_history=False) @@ -6485,7 +6522,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 34 + assert result["summary"]["total"] == 35 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_vlm_replay_worker_service.py b/tests/test_pixelrag_vlm_replay_worker_service.py new file mode 100644 index 0000000..fdc0df0 --- /dev/null +++ b/tests/test_pixelrag_vlm_replay_worker_service.py @@ -0,0 +1,181 @@ +import json +import subprocess +import sys +from types import SimpleNamespace + +from tests.test_pixelrag_ocr_vlm_replay_service import _write_receipt + + +def test_pixelrag_vlm_replay_worker_dry_run_keeps_blocked_pages_out(tmp_path): + from services.pixelrag_vlm_replay_worker_service import ( + POLICY, + run_pixelrag_ollama_vlm_replay_worker, + ) + + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=sunscreen", + ) + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + + payload = run_pixelrag_ollama_vlm_replay_worker( + artifact_root=tmp_path, + platform=("shopee_tw", "coupang_tw"), + ) + + assert payload["policy"] == POLICY + assert payload["status"] == "warning" + assert payload["execute"] is False + assert payload["summary"]["receipt_count"] == 2 + assert payload["summary"]["ready_count"] == 1 + assert payload["summary"]["dry_run_count"] == 1 + assert payload["summary"]["skipped_count"] == 1 + assert payload["summary"]["model_call_performed"] is False + assert payload["summary"]["artifact_write_performed"] is False + assert payload["summary"]["writes_database_count"] == 0 + assert payload["controlled_apply"]["primary_human_gate_count"] == 0 + assert payload["next_machine_action"] == "run_pixelrag_vlm_replay_worker_execute" + by_platform = {item["platform"]: item for item in payload["worker_items"]} + assert by_platform["shopee_tw"]["worker_status"] == "dry_run_ready" + assert by_platform["coupang_tw"]["worker_status"] == "skipped_blocked_or_not_ready" + + +def test_pixelrag_vlm_replay_worker_execute_writes_artifact_receipt(tmp_path, monkeypatch): + from services import pixelrag_vlm_replay_worker_service as service + + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=sunscreen", + ) + + class FakeOllama: + def __init__(self, model): + self.model = model + + def generate(self, *args, **kwargs): + return SimpleNamespace( + success=True, + content=json.dumps({ + "blocked_page_detected": False, + "fields": { + "title": { + "value": "防曬乳 SPF50", + "confidence": 0.92, + "evidence_refs": ["tile:1"], + }, + "price": { + "value": "399", + "confidence": 0.88, + "evidence_refs": ["tile:2"], + }, + }, + "quality": { + "overall_confidence": 0.89, + "missing_required_fields": [], + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + }, + "notes": [], + }), + model="minicpm-v:latest", + error=None, + total_duration=1.5, + host="http://34.87.90.216:11434", + input_tokens=12, + output_tokens=80, + ) + + monkeypatch.setattr(service, "OllamaService", FakeOllama) + payload = service.run_pixelrag_ollama_vlm_replay_worker( + artifact_root=tmp_path, + output_root=tmp_path / "receipts", + platform="shopee_tw", + execute=True, + write_receipt=True, + tile_limit=1, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["executed_count"] == 1 + assert payload["summary"]["executed_ok_count"] == 1 + assert payload["summary"]["receipt_written_count"] == 1 + assert payload["summary"]["writes_database_count"] == 0 + item = payload["worker_items"][0] + assert item["worker_status"] == "executed_ok" + assert item["provider"] == "gcp_ollama" + assert item["validation"]["valid_for_identity_matcher_replay"] is True + receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-ok" / "vlm_replay_receipt.json" + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["parsed_output"]["fields"]["title"]["value"] == "防曬乳 SPF50" + + +def test_pixelrag_vlm_replay_worker_cli_outputs_machine_readable_json(tmp_path): + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=sunscreen", + ) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/run_pixelrag_vlm_replay_worker.py", + "--artifact-root", + str(tmp_path), + "--platform", + "shopee_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["receipt_count"] == 1 + assert payload["summary"]["dry_run_count"] == 1 + assert payload["controlled_apply"]["model_call"] is False + assert payload["controlled_apply"]["writes_database"] is False + + +def test_pixelrag_vlm_replay_worker_route_returns_readback(tmp_path, monkeypatch): + from flask import Flask + from routes import system_public_routes as routes + from services import pixelrag_vlm_replay_worker_service as service + + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=sunscreen", + ) + monkeypatch.setattr(service, "DEFAULT_ARTIFACT_ROOT", str(tmp_path)) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw" + ): + response = routes.ai_automation_pixelrag_vlm_replay_worker_api.__wrapped__() + payload = response.get_json() + + assert payload["policy"] == "controlled_pixelrag_ollama_vlm_replay_worker_v1" + assert payload["summary"]["receipt_count"] == 1 + assert payload["summary"]["dry_run_count"] == 1 + assert payload["controlled_apply"]["model_call"] is False From ab07eb903d4aad393b417beed132ff3dc5ecb38a Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 23:43:07 +0800 Subject: [PATCH 08/75] =?UTF-8?q?fix(ai):=20=E4=BF=9D=E7=95=99=20PixelRAG?= =?UTF-8?q?=20VLM=20model=20error=20receipt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO_NEXT_STEPS.txt | 4 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 2 +- .../pixelrag_vlm_replay_worker_service.py | 7 +++ ...test_pixelrag_vlm_replay_worker_service.py | 48 +++++++++++++++++++ 5 files changed, 59 insertions(+), 4 deletions(-) diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 1fd171e..dd1cbf7 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -9,7 +9,7 @@ python scripts/ops/check_production_version_truth.py 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 -本輪 source target 為 `V10.749`；部署完成前不得宣稱正式環境已是 `V10.749`。 +本輪 source target 為 `V10.750`；部署完成前不得宣稱正式環境已是 `V10.750`。 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ @@ -60,7 +60,7 @@ P0-2026-07-09. PixelRAG / MCP / RAG 全自動主線 - 已完成：多電商 PixelRAG visual evidence lane 與 external MCP/RAG integration readback。 - 已完成：PixelRAG receipt → internal RAG candidate replay，以及 OCR/VLM replay contract no-write readback。 - 已完成：PixelRAG application portfolio，把可整合/可運用場景整理成 API/CLI 可讀的 priority lane、status、next machine action 與 forbidden guardrail。 - - 已完成：PixelRAG Ollama-first VLM replay worker，提供 dry-run/execute、artifact receipt、blocked page guard、confidence/evidence validation，且 DB write=0、primary human gate=0。 + - 已完成：PixelRAG Ollama-first VLM replay worker，提供 dry-run/execute、artifact receipt、model_error receipt、blocked page guard、confidence/evidence validation，且 DB write=0、primary human gate=0。 - 進行中：MCP/RAG runtime health → AI automation smoke。 - 未開始：Ollama-first visual embedding benchmark、pgvector-compatible visual metadata、Coupang platform probe / structured API、跨平台 source contracts。 - 主線文件：`docs/guides/ai_automation_mainline_work_items.md`。 diff --git a/config.py b/config.py index 982dab1..881ed22 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.749" +SYSTEM_VERSION = "V10.750" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 80aee45..0e7c092 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -879,7 +879,7 @@ POSTGRES_HOST=momo-db | 2026-07-09 | PixelRAG / external MCP/RAG 必須有 runtime monitoring 與 candidate replay | V10.746 起 `/api/ai-automation/smoke`、`/api/ai-automation/scheduled-health-summary` 必須輸出 `External MCP/RAG integration readback` / `external_mcp_rag_integration` 與 `PixelRAG RAG candidate replay` / `pixelrag_rag_candidate_replay`；external MCP/RAG readback 回報 9 個 capability 的 absorbed / unresolved 與 `MCP_ROUTER_ENABLED`、`RAG_ENABLED` runtime flags，PixelRAG replay 只讀 visual receipt，拆分 eligible / blocked / invalid，並明確標記 blocked page 不是商品資料。此路徑不讀 secret、不呼叫外部網路、不寫 DB、不寫 `ai_insights`、不寫正式價格表；eligible receipt 仍需 OCR/VLM replay、identity matcher replay、PromotionGate 與 embedding signature guard。 | | 2026-07-09 | PixelRAG OCR/VLM replay contract 必須有 runtime monitoring | V10.747 起 `/api/ai-automation/pixelrag-ocr-vlm-replay`、`scripts/ops/report_pixelrag_ocr_vlm_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 no-write OCR/VLM replay contract / `pixelrag_ocr_vlm_replay` family；readback 只讀 saved tiles 與 RAG candidate replay，輸出 ready / blocked / invalid contracts、field schema、output schema、validation rules、Ollama-first route contract、blocked page guard 與 next machine action。此階段明確標記 `extraction_execution_performed=false`、`ocr_execution_performed=false`、`vlm_execution_performed=false`、`writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`network_call=false`、`secret_read=false`、`primary_human_gate_count=0`；ready receipt 才能進下一段 `run_ollama_first_vlm_replay_worker`，blocked receipt 只能進 platform probe 或 structured API 策略。 | | 2026-07-09 | PixelRAG application portfolio 必須把可整合場景轉成主線工作項目 | V10.748 起 `/api/ai-automation/pixelrag-application-portfolio` 與 `scripts/ops/report_pixelrag_application_portfolio.py` 必須輸出 PixelRAG 在 commerce、RAG、UX、ops、marketing、governance 的可整合/可運用 lanes；每條 lane 需有 priority、status、integrates_with、use_cases、current_capability、next_machine_action、no-write 邊界與 forbidden guardrails。此 readback 依據 PixelRAG visual-RAG pattern、Google Merchant product data、Google Product structured data 與 Baymard product list UX 轉成內部工作項目；它不抓外站、不呼叫模型、不讀 secret、不寫 DB、不把像素結果當正式價格。 | -| 2026-07-09 | PixelRAG Ollama-first VLM replay worker 必須有 runtime monitoring | V10.749 起 `/api/ai-automation/pixelrag-vlm-replay-worker`、`scripts/ops/run_pixelrag_vlm_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled VLM replay worker / `pixelrag_vlm_replay_worker` family；readback 預設 dry-run，不呼叫模型、不寫 artifact，execute 模式只讀 saved tiles、呼叫 approved Ollama VLM route、驗證 JSON field confidence/evidence refs，並只寫 artifact receipt。此 worker 明確標記 `writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`primary_human_gate_count=0`；blocked page 不得輸出商品欄位，ready VLM 結果仍需 identity matcher replay 與 PromotionGate。 | +| 2026-07-09 | PixelRAG Ollama-first VLM replay worker 必須有 runtime monitoring | V10.750 起 `/api/ai-automation/pixelrag-vlm-replay-worker`、`scripts/ops/run_pixelrag_vlm_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled VLM replay worker / `pixelrag_vlm_replay_worker` family；readback 預設 dry-run，不呼叫模型、不寫 artifact，execute 模式只讀 saved tiles、呼叫 approved Ollama VLM route、驗證 JSON field confidence/evidence refs，並只寫 artifact receipt；model_error 也必須寫 failure artifact receipt。此 worker 明確標記 `writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`primary_human_gate_count=0`；blocked page 不得輸出商品欄位，ready VLM 結果仍需 identity matcher replay 與 PromotionGate。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/services/pixelrag_vlm_replay_worker_service.py b/services/pixelrag_vlm_replay_worker_service.py index d6704b0..a02837f 100644 --- a/services/pixelrag_vlm_replay_worker_service.py +++ b/services/pixelrag_vlm_replay_worker_service.py @@ -324,6 +324,13 @@ def _execute_item( "model_error": str(response.error or "")[:RAW_EXCERPT_LIMIT], "next_machine_action": "repair_ollama_vlm_runtime_or_model_route", }) + if write_receipt: + base["receipt_path"] = _write_replay_receipt( + output_root=output_root, + item=item, + worker_item=base, + ) + base["artifact_write_performed"] = True return base try: diff --git a/tests/test_pixelrag_vlm_replay_worker_service.py b/tests/test_pixelrag_vlm_replay_worker_service.py index fdc0df0..acc69c2 100644 --- a/tests/test_pixelrag_vlm_replay_worker_service.py +++ b/tests/test_pixelrag_vlm_replay_worker_service.py @@ -123,6 +123,54 @@ def test_pixelrag_vlm_replay_worker_execute_writes_artifact_receipt(tmp_path, mo assert receipt["parsed_output"]["fields"]["title"]["value"] == "防曬乳 SPF50" +def test_pixelrag_vlm_replay_worker_writes_model_error_receipt(tmp_path, monkeypatch): + from services import pixelrag_vlm_replay_worker_service as service + + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=sunscreen", + ) + + class FakeOllama: + def __init__(self, model): + self.model = model + + def generate(self, *args, **kwargs): + return SimpleNamespace( + success=False, + content="", + model="minicpm-v:latest", + error="all hosts failed", + total_duration=None, + host="http://34.21.145.224:11434", + input_tokens=0, + output_tokens=0, + ) + + monkeypatch.setattr(service, "OllamaService", FakeOllama) + payload = service.run_pixelrag_ollama_vlm_replay_worker( + artifact_root=tmp_path, + output_root=tmp_path / "receipts", + platform="shopee_tw", + execute=True, + write_receipt=True, + tile_limit=1, + ) + + assert payload["status"] == "critical" + assert payload["summary"]["model_error_count"] == 1 + assert payload["summary"]["receipt_written_count"] == 1 + assert payload["summary"]["artifact_write_performed"] is True + assert payload["controlled_apply"]["writes_database"] is False + receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-ok" / "vlm_replay_receipt.json" + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "model_error" + assert receipt["next_machine_action"] == "repair_ollama_vlm_runtime_or_model_route" + + def test_pixelrag_vlm_replay_worker_cli_outputs_machine_readable_json(tmp_path): _write_receipt( tmp_path, From c82bfc1835bc091f48f03b2d31308c7aad0755fe Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 23:48:30 +0800 Subject: [PATCH 09/75] =?UTF-8?q?fix(ai):=20=E8=AE=93=20PixelRAG=20VLM=20r?= =?UTF-8?q?eceipt=20=E8=87=AA=E8=AD=89=20artifact=20write?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO_NEXT_STEPS.txt | 4 ++-- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 2 +- services/pixelrag_vlm_replay_worker_service.py | 5 ++++- tests/test_pixelrag_vlm_replay_worker_service.py | 4 ++++ 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index dd1cbf7..3363716 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -9,7 +9,7 @@ python scripts/ops/check_production_version_truth.py 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 -本輪 source target 為 `V10.750`；部署完成前不得宣稱正式環境已是 `V10.750`。 +本輪 source target 為 `V10.751`；部署完成前不得宣稱正式環境已是 `V10.751`。 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ @@ -60,7 +60,7 @@ P0-2026-07-09. PixelRAG / MCP / RAG 全自動主線 - 已完成：多電商 PixelRAG visual evidence lane 與 external MCP/RAG integration readback。 - 已完成：PixelRAG receipt → internal RAG candidate replay，以及 OCR/VLM replay contract no-write readback。 - 已完成：PixelRAG application portfolio，把可整合/可運用場景整理成 API/CLI 可讀的 priority lane、status、next machine action 與 forbidden guardrail。 - - 已完成：PixelRAG Ollama-first VLM replay worker，提供 dry-run/execute、artifact receipt、model_error receipt、blocked page guard、confidence/evidence validation，且 DB write=0、primary human gate=0。 + - 已完成：PixelRAG Ollama-first VLM replay worker，提供 dry-run/execute、自證 artifact receipt、model_error receipt、blocked page guard、confidence/evidence validation，且 DB write=0、primary human gate=0。 - 進行中：MCP/RAG runtime health → AI automation smoke。 - 未開始：Ollama-first visual embedding benchmark、pgvector-compatible visual metadata、Coupang platform probe / structured API、跨平台 source contracts。 - 主線文件：`docs/guides/ai_automation_mainline_work_items.md`。 diff --git a/config.py b/config.py index 881ed22..6123d34 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.750" +SYSTEM_VERSION = "V10.751" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 0e7c092..3fcb42f 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -879,7 +879,7 @@ POSTGRES_HOST=momo-db | 2026-07-09 | PixelRAG / external MCP/RAG 必須有 runtime monitoring 與 candidate replay | V10.746 起 `/api/ai-automation/smoke`、`/api/ai-automation/scheduled-health-summary` 必須輸出 `External MCP/RAG integration readback` / `external_mcp_rag_integration` 與 `PixelRAG RAG candidate replay` / `pixelrag_rag_candidate_replay`；external MCP/RAG readback 回報 9 個 capability 的 absorbed / unresolved 與 `MCP_ROUTER_ENABLED`、`RAG_ENABLED` runtime flags，PixelRAG replay 只讀 visual receipt，拆分 eligible / blocked / invalid，並明確標記 blocked page 不是商品資料。此路徑不讀 secret、不呼叫外部網路、不寫 DB、不寫 `ai_insights`、不寫正式價格表；eligible receipt 仍需 OCR/VLM replay、identity matcher replay、PromotionGate 與 embedding signature guard。 | | 2026-07-09 | PixelRAG OCR/VLM replay contract 必須有 runtime monitoring | V10.747 起 `/api/ai-automation/pixelrag-ocr-vlm-replay`、`scripts/ops/report_pixelrag_ocr_vlm_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 no-write OCR/VLM replay contract / `pixelrag_ocr_vlm_replay` family；readback 只讀 saved tiles 與 RAG candidate replay，輸出 ready / blocked / invalid contracts、field schema、output schema、validation rules、Ollama-first route contract、blocked page guard 與 next machine action。此階段明確標記 `extraction_execution_performed=false`、`ocr_execution_performed=false`、`vlm_execution_performed=false`、`writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`network_call=false`、`secret_read=false`、`primary_human_gate_count=0`；ready receipt 才能進下一段 `run_ollama_first_vlm_replay_worker`，blocked receipt 只能進 platform probe 或 structured API 策略。 | | 2026-07-09 | PixelRAG application portfolio 必須把可整合場景轉成主線工作項目 | V10.748 起 `/api/ai-automation/pixelrag-application-portfolio` 與 `scripts/ops/report_pixelrag_application_portfolio.py` 必須輸出 PixelRAG 在 commerce、RAG、UX、ops、marketing、governance 的可整合/可運用 lanes；每條 lane 需有 priority、status、integrates_with、use_cases、current_capability、next_machine_action、no-write 邊界與 forbidden guardrails。此 readback 依據 PixelRAG visual-RAG pattern、Google Merchant product data、Google Product structured data 與 Baymard product list UX 轉成內部工作項目；它不抓外站、不呼叫模型、不讀 secret、不寫 DB、不把像素結果當正式價格。 | -| 2026-07-09 | PixelRAG Ollama-first VLM replay worker 必須有 runtime monitoring | V10.750 起 `/api/ai-automation/pixelrag-vlm-replay-worker`、`scripts/ops/run_pixelrag_vlm_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled VLM replay worker / `pixelrag_vlm_replay_worker` family；readback 預設 dry-run，不呼叫模型、不寫 artifact，execute 模式只讀 saved tiles、呼叫 approved Ollama VLM route、驗證 JSON field confidence/evidence refs，並只寫 artifact receipt；model_error 也必須寫 failure artifact receipt。此 worker 明確標記 `writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`primary_human_gate_count=0`；blocked page 不得輸出商品欄位，ready VLM 結果仍需 identity matcher replay 與 PromotionGate。 | +| 2026-07-09 | PixelRAG Ollama-first VLM replay worker 必須有 runtime monitoring | V10.751 起 `/api/ai-automation/pixelrag-vlm-replay-worker`、`scripts/ops/run_pixelrag_vlm_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled VLM replay worker / `pixelrag_vlm_replay_worker` family；readback 預設 dry-run，不呼叫模型、不寫 artifact，execute 模式只讀 saved tiles、呼叫 approved Ollama VLM route、驗證 JSON field confidence/evidence refs，並只寫 artifact receipt；model_error 也必須寫 failure artifact receipt，receipt 檔內需自證 `artifact_write_performed=true` 與 `receipt_path`。此 worker 明確標記 `writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`primary_human_gate_count=0`；blocked page 不得輸出商品欄位，ready VLM 結果仍需 identity matcher replay 與 PromotionGate。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/services/pixelrag_vlm_replay_worker_service.py b/services/pixelrag_vlm_replay_worker_service.py index a02837f..a249960 100644 --- a/services/pixelrag_vlm_replay_worker_service.py +++ b/services/pixelrag_vlm_replay_worker_service.py @@ -232,8 +232,11 @@ def _write_replay_receipt( / "vlm_replay_receipt.json" ) target.parent.mkdir(parents=True, exist_ok=True) + receipt_payload = dict(worker_item) + receipt_payload["artifact_write_performed"] = True + receipt_payload["receipt_path"] = str(target) target.write_text( - json.dumps(worker_item, ensure_ascii=False, indent=2, sort_keys=True), + json.dumps(receipt_payload, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8", ) return str(target) diff --git a/tests/test_pixelrag_vlm_replay_worker_service.py b/tests/test_pixelrag_vlm_replay_worker_service.py index acc69c2..0d6aaf0 100644 --- a/tests/test_pixelrag_vlm_replay_worker_service.py +++ b/tests/test_pixelrag_vlm_replay_worker_service.py @@ -121,6 +121,8 @@ def test_pixelrag_vlm_replay_worker_execute_writes_artifact_receipt(tmp_path, mo assert receipt_path.exists() receipt = json.loads(receipt_path.read_text(encoding="utf-8")) assert receipt["parsed_output"]["fields"]["title"]["value"] == "防曬乳 SPF50" + assert receipt["artifact_write_performed"] is True + assert receipt["receipt_path"] == str(receipt_path) def test_pixelrag_vlm_replay_worker_writes_model_error_receipt(tmp_path, monkeypatch): @@ -168,6 +170,8 @@ def test_pixelrag_vlm_replay_worker_writes_model_error_receipt(tmp_path, monkeyp receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-ok" / "vlm_replay_receipt.json" receipt = json.loads(receipt_path.read_text(encoding="utf-8")) assert receipt["worker_status"] == "model_error" + assert receipt["artifact_write_performed"] is True + assert receipt["receipt_path"] == str(receipt_path) assert receipt["next_machine_action"] == "repair_ollama_vlm_runtime_or_model_route" From d1ca0670544a8c8e29d1d11f44353027345da990 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 00:02:22 +0800 Subject: [PATCH 10/75] =?UTF-8?q?feat(ai):=20=E5=BB=BA=E7=AB=8B=20PixelRAG?= =?UTF-8?q?=20VLM=20route=20readiness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO_NEXT_STEPS.txt | 3 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 2 + .../ai_automation_mainline_work_items.md | 3 +- docs/guides/ai_automation_session_sop.md | 4 +- docs/guides/browse_sh_crawler_playbook.md | 4 +- routes/system_public_routes.py | 29 +++ .../report_pixelrag_vlm_route_readiness.py | 49 ++++ scripts/ops/run_pixelrag_vlm_replay_worker.py | 13 ++ services/ai_automation_smoke_service.py | 98 ++++++++ .../pixelrag_vlm_replay_worker_service.py | 79 ++++++- .../pixelrag_vlm_route_readiness_service.py | 212 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 42 +++- ...test_pixelrag_vlm_replay_worker_service.py | 82 +++++++ ...st_pixelrag_vlm_route_readiness_service.py | 110 +++++++++ 15 files changed, 722 insertions(+), 10 deletions(-) create mode 100644 scripts/ops/report_pixelrag_vlm_route_readiness.py create mode 100644 services/pixelrag_vlm_route_readiness_service.py create mode 100644 tests/test_pixelrag_vlm_route_readiness_service.py diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 3363716..3853591 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -9,7 +9,7 @@ python scripts/ops/check_production_version_truth.py 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 -本輪 source target 為 `V10.751`；部署完成前不得宣稱正式環境已是 `V10.751`。 +本輪 source target 為 `V10.752`；部署完成前不得宣稱正式環境已是 `V10.752`。 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ @@ -61,6 +61,7 @@ P0-2026-07-09. PixelRAG / MCP / RAG 全自動主線 - 已完成：PixelRAG receipt → internal RAG candidate replay，以及 OCR/VLM replay contract no-write readback。 - 已完成：PixelRAG application portfolio，把可整合/可運用場景整理成 API/CLI 可讀的 priority lane、status、next machine action 與 forbidden guardrail。 - 已完成：PixelRAG Ollama-first VLM replay worker，提供 dry-run/execute、自證 artifact receipt、model_error receipt、blocked page guard、confidence/evidence validation，且 DB write=0、primary human gate=0。 + - 已完成：PixelRAG VLM route readiness 與 auto-select model，execute 前自動讀 approved Ollama `/api/tags`，避免 configured model 缺失時盲打 generate；完全缺候選時寫 `model_route_not_ready` receipt。 - 進行中：MCP/RAG runtime health → AI automation smoke。 - 未開始：Ollama-first visual embedding benchmark、pgvector-compatible visual metadata、Coupang platform probe / structured API、跨平台 source contracts。 - 主線文件：`docs/guides/ai_automation_mainline_work_items.md`。 diff --git a/config.py b/config.py index 6123d34..bdada6f 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.751" +SYSTEM_VERSION = "V10.752" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 3fcb42f..62cb6e5 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -117,6 +117,7 @@ - 2026-07-09 起 PixelRAG visual receipts 進入 OCR/VLM 前必須先輸出 no-write replay contract：`/api/ai-automation/pixelrag-ocr-vlm-replay` 與 `scripts/ops/report_pixelrag_ocr_vlm_replay.py` 只讀 saved tiles，輸出欄位 schema、輸出 schema、confidence/evidence validation rules、Ollama-first route contract 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG OCR/VLM replay contract`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_ocr_vlm_replay` family。此 contract 不執行 OCR/VLM、不呼叫模型、不讀 secret、不連外、不寫 DB、不寫 `ai_insights`、不寫正式價格表；blocked / 403 / captcha / access denied / verify traffic page 只能進 platform probe 或 structured API 策略。 - 2026-07-09 起 PixelRAG 可整合/可運用範圍必須有機器可讀 application portfolio：`/api/ai-automation/pixelrag-application-portfolio` 與 `scripts/ops/report_pixelrag_application_portfolio.py` 需輸出 commerce、RAG、UX、ops、marketing、governance lanes，每條 lane 必須包含 priority、status、integrates_with、use_cases、current_capability、next_machine_action、no-write 邊界與 forbidden guardrails。此 portfolio 不抓外站、不呼叫模型、不讀 secret、不寫 DB；其用途是把「還可以整合哪些」變成可排程、可驗證、可拒絕違規場景的主線工作項目。 - 2026-07-09 起 PixelRAG ready receipts 必須有 Ollama-first VLM replay worker：`/api/ai-automation/pixelrag-vlm-replay-worker` 與 `scripts/ops/run_pixelrag_vlm_replay_worker.py` 預設 dry-run，`execute=true&write_receipt=true` 才呼叫 approved Ollama VLM route 並寫 artifact receipt；`/api/ai-automation/smoke` 需包含 `PixelRAG VLM replay worker`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_vlm_replay_worker` family。此 worker 不讀 secret、不寫 DB、不寫 `ai_insights`、不寫正式價格表；blocked / 403 / captcha / access denied receipt 自動跳 platform probe 或 structured API，ready receipt 的 VLM 結果仍須 identity matcher replay 與 PromotionGate 才能進候選知識層。 +- 2026-07-09 起 PixelRAG VLM route readiness 必須在 execute 前可讀回：`/api/ai-automation/pixelrag-vlm-route-readiness` 與 `scripts/ops/report_pixelrag_vlm_route_readiness.py` 只讀 approved Ollama `/api/tags`，輸出 configured model、candidate model、reachable host、model_route_ready 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG VLM route readiness`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_vlm_route_readiness` family。此 readback 不呼叫 `/api/generate`、不讀 secret、不寫 DB；worker execute 必須使用它自動避開缺模型路由，完全缺候選時寫 `model_route_not_ready` artifact receipt。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -880,6 +881,7 @@ POSTGRES_HOST=momo-db | 2026-07-09 | PixelRAG OCR/VLM replay contract 必須有 runtime monitoring | V10.747 起 `/api/ai-automation/pixelrag-ocr-vlm-replay`、`scripts/ops/report_pixelrag_ocr_vlm_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 no-write OCR/VLM replay contract / `pixelrag_ocr_vlm_replay` family；readback 只讀 saved tiles 與 RAG candidate replay，輸出 ready / blocked / invalid contracts、field schema、output schema、validation rules、Ollama-first route contract、blocked page guard 與 next machine action。此階段明確標記 `extraction_execution_performed=false`、`ocr_execution_performed=false`、`vlm_execution_performed=false`、`writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`network_call=false`、`secret_read=false`、`primary_human_gate_count=0`；ready receipt 才能進下一段 `run_ollama_first_vlm_replay_worker`，blocked receipt 只能進 platform probe 或 structured API 策略。 | | 2026-07-09 | PixelRAG application portfolio 必須把可整合場景轉成主線工作項目 | V10.748 起 `/api/ai-automation/pixelrag-application-portfolio` 與 `scripts/ops/report_pixelrag_application_portfolio.py` 必須輸出 PixelRAG 在 commerce、RAG、UX、ops、marketing、governance 的可整合/可運用 lanes；每條 lane 需有 priority、status、integrates_with、use_cases、current_capability、next_machine_action、no-write 邊界與 forbidden guardrails。此 readback 依據 PixelRAG visual-RAG pattern、Google Merchant product data、Google Product structured data 與 Baymard product list UX 轉成內部工作項目；它不抓外站、不呼叫模型、不讀 secret、不寫 DB、不把像素結果當正式價格。 | | 2026-07-09 | PixelRAG Ollama-first VLM replay worker 必須有 runtime monitoring | V10.751 起 `/api/ai-automation/pixelrag-vlm-replay-worker`、`scripts/ops/run_pixelrag_vlm_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled VLM replay worker / `pixelrag_vlm_replay_worker` family；readback 預設 dry-run，不呼叫模型、不寫 artifact，execute 模式只讀 saved tiles、呼叫 approved Ollama VLM route、驗證 JSON field confidence/evidence refs，並只寫 artifact receipt；model_error 也必須寫 failure artifact receipt，receipt 檔內需自證 `artifact_write_performed=true` 與 `receipt_path`。此 worker 明確標記 `writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`primary_human_gate_count=0`；blocked page 不得輸出商品欄位，ready VLM 結果仍需 identity matcher replay 與 PromotionGate。 | +| 2026-07-09 | PixelRAG VLM route readiness 必須有 runtime monitoring 與 execute 前自動選模 | V10.752 起 `/api/ai-automation/pixelrag-vlm-route-readiness`、`scripts/ops/report_pixelrag_vlm_route_readiness.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 read-only approved Ollama route/model readiness / `pixelrag_vlm_route_readiness` family；readback 只打 `/api/tags`，不呼叫 `/api/generate`，輸出 reachable host、configured model available count、candidate model、candidate host、model_route_ready 與 next machine action。`run_pixelrag_vlm_replay_worker.py --execute` 預設必須使用此 readback 自動選擇已安裝候選模型；完全沒有候選時寫 `model_route_not_ready` artifact receipt，不得盲打 missing model。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 2292dbe..63e406a 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -19,7 +19,8 @@ | Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read receipts, split eligible vs blocked, and require OCR/VLM replay plus PromotionGate before knowledge writes. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | -| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, and PixelRAG VLM replay worker families. | +| Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | +| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, and PixelRAG VLM replay worker families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | ## P1 diff --git a/docs/guides/ai_automation_session_sop.md b/docs/guides/ai_automation_session_sop.md index 4c30080..eeb8430 100644 --- a/docs/guides/ai_automation_session_sop.md +++ b/docs/guides/ai_automation_session_sop.md @@ -36,7 +36,9 @@ 或 `python scripts/ops/report_pixelrag_application_portfolio.py` 可讀回 area、priority、status、use cases、next machine action 與 forbidden guardrails；不得只存在聊天結論。 - PixelRAG VLM replay worker 必須確認 `/api/ai-automation/pixelrag-vlm-replay-worker` 或 `python scripts/ops/run_pixelrag_vlm_replay_worker.py` 可讀回 dry-run/execute、model_call、artifact receipt、blocked/ready 分流、DB write=0 與 primary human gate=0；execute 結果仍只是 candidate evidence。 -- AI automation smoke 必須包含 external MCP/RAG integration、PixelRAG RAG candidate replay、PixelRAG OCR/VLM replay contract 與 PixelRAG VLM replay worker family，避免 registry 已完成但 runtime flag / receipt replay / VLM worker 未完成時被誤報為全自動閉環。 +- PixelRAG VLM route readiness 必須確認 `/api/ai-automation/pixelrag-vlm-route-readiness` + 或 `python scripts/ops/report_pixelrag_vlm_route_readiness.py` 可讀回 approved Ollama hosts、configured model 是否安裝、candidate model、自動選模下一步、model_call=false、DB write=0；缺模型不得等 execute 才暴露。 +- AI automation smoke 必須包含 external MCP/RAG integration、PixelRAG RAG candidate replay、PixelRAG OCR/VLM replay contract、PixelRAG VLM route readiness 與 PixelRAG VLM replay worker family，避免 registry 已完成但 runtime flag / receipt replay / VLM route / VLM worker 未完成時被誤報為全自動閉環。 - AI 自動化 Prometheus 指標變更必須同步檢查 `docker/grafana/provisioning/dashboards/json/ai-automation-overview.json` 是否需要新增 panel 或查詢。 - 188 線上 active monitoring stack 以 `monitoring/prometheus.yml` 為準；110 gateway 另有 `/home/wooo/monitoring/prometheus.yml`。若 dashboard 無資料，先確認 Prometheus `momo-app` target 與 `momo-network` 連線；所有 Blackbox HTTP target 必須打 `/health`，不可打 Dashboard 首頁 `/`。 - Smoke dashboard 會保存 JSONL 趨勢；若新增檢查項目，要確保 history compact record 仍保持小而可讀。 diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index f21b4c2..1b50cc1 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -187,12 +187,14 @@ python scripts/ops/report_pixelrag_ocr_vlm_replay.py --platform shopee_tw --plat Ollama-first VLM replay worker： ```bash +python scripts/ops/report_pixelrag_vlm_route_readiness.py +python scripts/ops/report_pixelrag_vlm_route_readiness.py --model minicpm-v:latest --include-models python scripts/ops/run_pixelrag_vlm_replay_worker.py python scripts/ops/run_pixelrag_vlm_replay_worker.py --platform shopee_tw --platform coupang_tw python scripts/ops/run_pixelrag_vlm_replay_worker.py --platform shopee_tw --execute --write-receipt --limit 1 ``` -API readback: `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 +API readback: `/api/ai-automation/pixelrag-vlm-route-readiness` 與 `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。worker 預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。execute 前會自動讀 approved Ollama `/api/tags` 選擇已安裝候選模型；若完全沒有候選，會寫 `model_route_not_ready` artifact receipt，不盲打 generate。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 Application portfolio： diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index acf99d2..a529702 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -780,6 +780,12 @@ def ai_automation_pixelrag_vlm_replay_worker_api(): limit = request.args.get('limit', 25, type=int) tile_limit = request.args.get('tile_limit', 4, type=int) timeout = request.args.get('timeout', 90, type=int) + auto_select_model = str(request.args.get('auto_select_model', 'true')).strip().lower() not in { + '0', + 'false', + 'no', + } + route_readiness_timeout = request.args.get('route_readiness_timeout', 3, type=int) return jsonify(run_pixelrag_ollama_vlm_replay_worker( platform=platforms, max_age_hours=max(1, min(max_age_hours or 168, 720)), @@ -789,6 +795,29 @@ def ai_automation_pixelrag_vlm_replay_worker_api(): timeout=max(10, min(timeout or 90, 240)), execute=execute, write_receipt=bool(execute and write_receipt), + auto_select_model=auto_select_model, + route_readiness_timeout=max(1, min(route_readiness_timeout or 3, 20)), + )) + + +@system_public_bp.route('/api/ai-automation/pixelrag-vlm-route-readiness') +@login_required +def ai_automation_pixelrag_vlm_route_readiness_api(): + """Read-only PixelRAG VLM approved route/model readiness.""" + from services.pixelrag_vlm_route_readiness_service import ( + build_pixelrag_vlm_route_readiness, + ) + + include_models = str(request.args.get('include_models') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + timeout = request.args.get('timeout', 3, type=int) + return jsonify(build_pixelrag_vlm_route_readiness( + model=str(request.args.get('model') or '').strip() or None, + timeout_seconds=max(1, min(timeout or 3, 20)), + include_models=include_models, )) diff --git a/scripts/ops/report_pixelrag_vlm_route_readiness.py b/scripts/ops/report_pixelrag_vlm_route_readiness.py new file mode 100644 index 0000000..c7b9b10 --- /dev/null +++ b/scripts/ops/report_pixelrag_vlm_route_readiness.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Report PixelRAG VLM route/model readiness.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_vlm_route_readiness_service import ( # noqa: E402 + build_pixelrag_vlm_route_readiness, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="讀回 PixelRAG VLM approved Ollama route / model readiness。" + ) + parser.add_argument("--model", help="要檢查的 configured VLM model。") + parser.add_argument( + "--timeout", + type=int, + default=3, + help="/api/tags probe timeout 秒數。", + ) + parser.add_argument( + "--include-models", + action="store_true", + help="輸出每個 host 的 model list。", + ) + args = parser.parse_args() + + payload = build_pixelrag_vlm_route_readiness( + model=args.model, + timeout_seconds=args.timeout, + include_models=args.include_models, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/run_pixelrag_vlm_replay_worker.py b/scripts/ops/run_pixelrag_vlm_replay_worker.py index 5416d80..ab78f94 100755 --- a/scripts/ops/run_pixelrag_vlm_replay_worker.py +++ b/scripts/ops/run_pixelrag_vlm_replay_worker.py @@ -76,6 +76,17 @@ def main() -> int: action="store_true", help="execute 後寫入 VLM replay artifact receipt；不寫 DB。", ) + parser.add_argument( + "--no-auto-select-model", + action="store_true", + help="停用 execute 前的 approved route / installed model 自動選擇。", + ) + parser.add_argument( + "--route-readiness-timeout", + type=int, + default=3, + help="auto-select model 的 /api/tags readiness timeout 秒數。", + ) args = parser.parse_args() payload = run_pixelrag_ollama_vlm_replay_worker( @@ -89,6 +100,8 @@ def main() -> int: timeout=args.timeout, execute=args.execute, write_receipt=bool(args.write_receipt and args.execute), + auto_select_model=not args.no_auto_select_model, + route_readiness_timeout=args.route_readiness_timeout, ) print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) return 0 if payload.get("success") else 1 diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index 63db710..c8ffcb4 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -542,6 +542,11 @@ def build_scheduled_automation_health_summary( if not pixelrag_ocr_vlm_replay or not pixelrag_ocr_vlm_replay_details: pixelrag_ocr_vlm_replay = _pixelrag_ocr_vlm_replay_check() pixelrag_ocr_vlm_replay_details = pixelrag_ocr_vlm_replay.get("details") or {} + pixelrag_vlm_route_readiness = _find_check(source_result, "PixelRAG VLM route readiness") + pixelrag_vlm_route_readiness_details = pixelrag_vlm_route_readiness.get("details") or {} + if not pixelrag_vlm_route_readiness or not pixelrag_vlm_route_readiness_details: + pixelrag_vlm_route_readiness = _pixelrag_vlm_route_readiness_check() + pixelrag_vlm_route_readiness_details = pixelrag_vlm_route_readiness.get("details") or {} pixelrag_vlm_replay_worker = _find_check(source_result, "PixelRAG VLM replay worker") pixelrag_vlm_replay_worker_details = pixelrag_vlm_replay_worker.get("details") or {} if not pixelrag_vlm_replay_worker or not pixelrag_vlm_replay_worker_details: @@ -4637,6 +4642,40 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_vlm_route_readiness", + "label": "PixelRAG VLM route readiness", + "status": pixelrag_vlm_route_readiness.get("status") or "warning", + "summary": ( + pixelrag_vlm_route_readiness.get("summary") + or "PixelRAG VLM route readiness has no latest readback." + ), + "next_machine_action": pixelrag_vlm_route_readiness_details.get("next_machine_action") + or "run_pixelrag_vlm_route_readiness_readback", + "details": { + "policy": pixelrag_vlm_route_readiness_details.get("policy"), + "host_count": int(pixelrag_vlm_route_readiness_details.get("host_count") or 0), + "reachable_host_count": int( + pixelrag_vlm_route_readiness_details.get("reachable_host_count") or 0 + ), + "configured_model": pixelrag_vlm_route_readiness_details.get("configured_model"), + "configured_model_available_count": int( + pixelrag_vlm_route_readiness_details.get("configured_model_available_count") or 0 + ), + "candidate_model": pixelrag_vlm_route_readiness_details.get("candidate_model"), + "candidate_host": pixelrag_vlm_route_readiness_details.get("candidate_host"), + "candidate_provider": pixelrag_vlm_route_readiness_details.get("candidate_provider"), + "model_route_ready": bool( + pixelrag_vlm_route_readiness_details.get("model_route_ready") + ), + "model_call_performed": bool( + pixelrag_vlm_route_readiness_details.get("model_call_performed") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "key": "pixelrag_vlm_replay_worker", "label": "PixelRAG VLM replay worker", @@ -4658,6 +4697,9 @@ def build_scheduled_automation_health_summary( pixelrag_vlm_replay_worker_details.get("executed_warning_count") or 0 ), "model_error_count": int(pixelrag_vlm_replay_worker_details.get("model_error_count") or 0), + "model_route_not_ready_count": int( + pixelrag_vlm_replay_worker_details.get("model_route_not_ready_count") or 0 + ), "parse_error_count": int(pixelrag_vlm_replay_worker_details.get("parse_error_count") or 0), "receipt_written_count": int( pixelrag_vlm_replay_worker_details.get("receipt_written_count") or 0 @@ -13425,6 +13467,7 @@ def _pixelrag_vlm_replay_worker_check() -> Dict[str, Any]: "executed_ok_count": int(summary.get("executed_ok_count") or 0), "executed_warning_count": int(summary.get("executed_warning_count") or 0), "model_error_count": int(summary.get("model_error_count") or 0), + "model_route_not_ready_count": int(summary.get("model_route_not_ready_count") or 0), "parse_error_count": int(summary.get("parse_error_count") or 0), "receipt_written_count": int(summary.get("receipt_written_count") or 0), "model_call_performed": bool(summary.get("model_call_performed")), @@ -13448,6 +13491,60 @@ def _pixelrag_vlm_replay_worker_check() -> Dict[str, Any]: ) +def _pixelrag_vlm_route_readiness_check() -> Dict[str, Any]: + """Read-only sentinel for approved Ollama VLM route/model readiness.""" + try: + from services.pixelrag_vlm_route_readiness_service import ( + build_pixelrag_vlm_route_readiness, + ) + + readback = build_pixelrag_vlm_route_readiness(timeout_seconds=3) + summary = readback.get("summary") or {} + host_count = int(summary.get("host_count") or 0) + reachable_host_count = int(summary.get("reachable_host_count") or 0) + configured_model = summary.get("configured_model") + configured_available = int(summary.get("configured_model_available_count") or 0) + candidate_model = summary.get("candidate_model") + status = readback.get("status") or "warning" + summary_text = ( + f"PixelRAG VLM route readiness hosts={reachable_host_count}/{host_count}, " + f"configured={configured_model}, configured_available={configured_available}, " + f"candidate={candidate_model or 'none'}" + ) + return _check( + "PixelRAG VLM route readiness", + status, + summary_text, + { + "policy": readback.get("policy"), + "host_count": host_count, + "reachable_host_count": reachable_host_count, + "configured_model": configured_model, + "configured_model_available_count": configured_available, + "candidate_model": candidate_model, + "candidate_host": summary.get("candidate_host"), + "candidate_provider": summary.get("candidate_provider"), + "model_route_ready": bool(summary.get("model_route_ready")), + "model_call_performed": bool(summary.get("model_call_performed")), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG VLM route readiness", + "critical", + f"PixelRAG VLM route readiness 無法執行：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: int = 20) -> Dict[str, Any]: checks: List[Dict[str, Any]] = [ _event_router_check(), @@ -13484,6 +13581,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _external_mcp_rag_integration_check(), _pixelrag_rag_candidate_replay_check(), _pixelrag_ocr_vlm_replay_check(), + _pixelrag_vlm_route_readiness_check(), _pixelrag_vlm_replay_worker_check(), ] worst = max(checks, key=lambda item: STATUS_RANK.get(item["status"], 2))["status"] diff --git a/services/pixelrag_vlm_replay_worker_service.py b/services/pixelrag_vlm_replay_worker_service.py index a249960..853e765 100644 --- a/services/pixelrag_vlm_replay_worker_service.py +++ b/services/pixelrag_vlm_replay_worker_service.py @@ -25,6 +25,9 @@ from services.pixelrag_ocr_vlm_replay_service import ( DEFAULT_CONFIDENCE_THRESHOLD, build_pixelrag_ocr_vlm_replay_contract, ) +from services.pixelrag_vlm_route_readiness_service import ( + build_pixelrag_vlm_route_readiness, +) POLICY = "controlled_pixelrag_ollama_vlm_replay_worker_v1" @@ -274,6 +277,39 @@ def _dry_run_item(item: Mapping[str, Any]) -> dict[str, Any]: } +def _model_route_not_ready_item( + item: Mapping[str, Any], + *, + output_root: Path, + route_readiness: Mapping[str, Any], + write_receipt: bool, +) -> dict[str, Any]: + summary = route_readiness.get("summary") or {} + worker_item = { + "platform": item.get("platform"), + "manifest_id": item.get("manifest_id"), + "source_receipt_path": item.get("source_receipt_path"), + "worker_status": "model_route_not_ready", + "model": summary.get("configured_model"), + "candidate_model": summary.get("candidate_model"), + "candidate_host": summary.get("candidate_host"), + "model_call_performed": False, + "artifact_write_performed": False, + "writes_database": False, + "route_readiness_status": route_readiness.get("status"), + "next_machine_action": route_readiness.get("next_machine_action") + or "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host", + } + if write_receipt: + worker_item["receipt_path"] = _write_replay_receipt( + output_root=output_root, + item=item, + worker_item=worker_item, + ) + worker_item["artifact_write_performed"] = True + return worker_item + + def _execute_item( item: Mapping[str, Any], *, @@ -397,6 +433,8 @@ def run_pixelrag_ollama_vlm_replay_worker( timeout: int | None = None, execute: bool = False, write_receipt: bool = False, + auto_select_model: bool = True, + route_readiness_timeout: int | None = None, ) -> dict[str, Any]: """Run or dry-run the PixelRAG VLM replay worker.""" root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT) @@ -407,7 +445,10 @@ def run_pixelrag_ollama_vlm_replay_worker( tiles = max(1, min(int(tile_limit or DEFAULT_TILE_LIMIT), 12)) selected_model = str(model or DEFAULT_MODEL) selected_timeout = max(10, int(timeout or DEFAULT_TIMEOUT_SECONDS)) + readiness_timeout = max(1, min(int(route_readiness_timeout or 3), 20)) generated_at = datetime.now(timezone.utc).isoformat() + route_readiness: dict[str, Any] | None = None + model_route_ready = True contract = build_pixelrag_ocr_vlm_replay_contract( artifact_root=root, @@ -416,6 +457,17 @@ def run_pixelrag_ollama_vlm_replay_worker( limit=item_limit, ) replay_items = list(contract.get("replay_items") or []) + if execute and auto_select_model: + route_readiness = build_pixelrag_vlm_route_readiness( + model=selected_model, + timeout_seconds=readiness_timeout, + ) + route_summary = route_readiness.get("summary") or {} + candidate_model = str(route_summary.get("candidate_model") or "").strip() + model_route_ready = bool(route_summary.get("model_route_ready")) + if candidate_model: + selected_model = candidate_model + worker_items: list[dict[str, Any]] = [] for item in replay_items: if not item.get("ready_for_ollama_vlm_worker"): @@ -424,6 +476,14 @@ def run_pixelrag_ollama_vlm_replay_worker( if not execute: worker_items.append(_dry_run_item(item)) continue + if not model_route_ready and route_readiness is not None: + worker_items.append(_model_route_not_ready_item( + item, + output_root=output, + route_readiness=route_readiness, + write_receipt=write_receipt, + )) + continue worker_items.append(_execute_item( item, root=root, @@ -441,6 +501,7 @@ def run_pixelrag_ollama_vlm_replay_worker( executed_ok_count = sum(1 for item in worker_items if item.get("worker_status") == "executed_ok") executed_warning_count = sum(1 for item in worker_items if item.get("worker_status") == "executed_warning") model_error_count = sum(1 for item in worker_items if item.get("worker_status") == "model_error") + route_not_ready_count = sum(1 for item in worker_items if item.get("worker_status") == "model_route_not_ready") parse_error_count = sum(1 for item in worker_items if item.get("worker_status") == "model_output_parse_error") no_tile_count = sum(1 for item in worker_items if item.get("worker_status") == "skipped_no_loadable_tiles") receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path")) @@ -451,7 +512,7 @@ def run_pixelrag_ollama_vlm_replay_worker( model_call_performed = any(bool(item.get("model_call_performed")) for item in worker_items) artifact_write_performed = any(bool(item.get("artifact_write_performed")) for item in worker_items) - if parse_error_count or model_error_count or no_tile_count: + if parse_error_count or model_error_count or route_not_ready_count or no_tile_count: status = "critical" if ready_count and executed_ok_count == 0 and execute else "warning" elif executed_warning_count or skipped_count or dry_run_count or (not replay_items): status = "warning" @@ -462,6 +523,8 @@ def run_pixelrag_ollama_vlm_replay_worker( next_action = "run_pixelrag_visual_capture_worker" elif not execute and ready_count: next_action = "run_pixelrag_vlm_replay_worker_execute" + elif route_not_ready_count: + next_action = "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host" elif model_error_count or parse_error_count: next_action = "repair_ollama_vlm_runtime_or_model_route" elif executed_warning_count: @@ -480,6 +543,7 @@ def run_pixelrag_ollama_vlm_replay_worker( "executed_ok_count": executed_ok_count, "executed_warning_count": executed_warning_count, "model_error_count": model_error_count, + "model_route_not_ready_count": route_not_ready_count, "parse_error_count": parse_error_count, "no_tile_count": no_tile_count, "receipt_written_count": receipt_written_count, @@ -502,11 +566,24 @@ def run_pixelrag_ollama_vlm_replay_worker( "limit": item_limit, "tile_limit": tiles, "model": selected_model, + "configured_model": str(model or DEFAULT_MODEL), "timeout_seconds": selected_timeout, "execute": bool(execute), "write_receipt": bool(write_receipt), + "auto_select_model": bool(auto_select_model), + "route_readiness_timeout_seconds": readiness_timeout, "summary": summary, "worker_items": worker_items, + "route_readiness": ( + { + "policy": route_readiness.get("policy"), + "status": route_readiness.get("status"), + "summary": route_readiness.get("summary"), + "next_machine_action": route_readiness.get("next_machine_action"), + } + if route_readiness + else None + ), "source_contract": { "policy": contract.get("policy"), "status": contract.get("status"), diff --git a/services/pixelrag_vlm_route_readiness_service.py b/services/pixelrag_vlm_route_readiness_service.py new file mode 100644 index 0000000..9cc4acf --- /dev/null +++ b/services/pixelrag_vlm_route_readiness_service.py @@ -0,0 +1,212 @@ +"""Read-only PixelRAG VLM route readiness. + +This module checks approved Ollama routes for installed VLM candidate models. +It does not call /api/generate, read secrets, write DB data, or change runtime +configuration. The worker can use the result to avoid blindly executing a +missing configured model. +""" + +from __future__ import annotations + +import os +from datetime import datetime, timezone +from typing import Any + +import requests + +from services.ollama_service import ( + OLLAMA_HOST_FALLBACK, + OLLAMA_HOST_PRIMARY, + OLLAMA_HOST_PRIMARY_PROXY, + OLLAMA_HOST_SECONDARY, + OLLAMA_HOST_SECONDARY_PROXY, + get_host_label, + get_provider_tag, +) + + +POLICY = "read_only_pixelrag_vlm_route_readiness_v1" +DEFAULT_TIMEOUT_SECONDS = 3 +DEFAULT_MODEL = ( + os.getenv("PIXELRAG_VLM_MODEL") + or os.getenv("PPT_VISION_MODEL") + or "minicpm-v:latest" +) +VISION_MODEL_CANDIDATES = ( + "minicpm-v:latest", + "gemma3:4b", + "llava:latest", +) + + +def _unique(values: list[str] | tuple[str, ...]) -> list[str]: + seen: set[str] = set() + result: list[str] = [] + for value in values: + clean = str(value or "").strip() + if not clean or clean in seen: + continue + seen.add(clean) + result.append(clean) + return result + + +def _approved_hosts() -> list[str]: + return _unique(( + OLLAMA_HOST_PRIMARY, + OLLAMA_HOST_PRIMARY_PROXY, + OLLAMA_HOST_SECONDARY, + OLLAMA_HOST_SECONDARY_PROXY, + OLLAMA_HOST_FALLBACK, + )) + + +def _candidate_models(configured_model: str) -> list[str]: + env_candidates = [ + item.strip() + for item in os.getenv("PIXELRAG_VLM_MODEL_CANDIDATES", "").split(",") + if item.strip() + ] + return _unique((configured_model, *env_candidates, *VISION_MODEL_CANDIDATES)) + + +def _fetch_models(host: str, timeout: int) -> dict[str, Any]: + clean_host = str(host or "").rstrip("/") + item: dict[str, Any] = { + "host": clean_host, + "host_label": get_host_label(clean_host), + "provider": get_provider_tag(clean_host), + "reachable": False, + "model_count": 0, + "models": [], + "error": "", + } + try: + response = requests.get(f"{clean_host}/api/tags", timeout=max(1, timeout)) + item["http_status"] = response.status_code + if response.status_code != 200: + item["error"] = f"HTTP {response.status_code}: {response.text[:180]}" + return item + payload = response.json() + models = [ + str(model.get("name") or "").strip() + for model in list(payload.get("models") or []) + if str(model.get("name") or "").strip() + ] + item.update({ + "reachable": True, + "model_count": len(models), + "models": models[:80], + }) + except Exception as exc: + item["error"] = f"{type(exc).__name__}: {str(exc)[:180]}" + return item + + +def build_pixelrag_vlm_route_readiness( + *, + model: str | None = None, + timeout_seconds: int | None = None, + include_models: bool = False, +) -> dict[str, Any]: + """Read approved Ollama tags and recommend an installed VLM model.""" + configured_model = str(model or DEFAULT_MODEL).strip() or DEFAULT_MODEL + timeout = max(1, min(int(timeout_seconds or DEFAULT_TIMEOUT_SECONDS), 20)) + generated_at = datetime.now(timezone.utc).isoformat() + candidates = _candidate_models(configured_model) + host_results = [_fetch_models(host, timeout) for host in _approved_hosts()] + + reachable_hosts = [item for item in host_results if item.get("reachable")] + configured_hosts = [ + item for item in reachable_hosts + if configured_model in set(item.get("models") or []) + ] + selected_model = "" + selected_host = "" + selected_provider = "" + selected_reason = "" + for candidate in candidates: + for host in reachable_hosts: + if candidate not in set(host.get("models") or []): + continue + selected_model = candidate + selected_host = str(host.get("host") or "") + selected_provider = str(host.get("provider") or "") + selected_reason = ( + "configured_model_available" + if candidate == configured_model + else "installed_candidate_fallback" + ) + break + if selected_model: + break + + model_route_ready = bool(selected_model and selected_host) + if not reachable_hosts: + status = "critical" + next_action = "repair_approved_ollama_host_connectivity" + elif not model_route_ready: + status = "critical" + next_action = "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host" + elif selected_model != configured_model: + status = "warning" + next_action = "run_pixelrag_vlm_replay_worker_execute_with_selected_model" + else: + status = "ok" + next_action = "run_pixelrag_vlm_replay_worker_execute" + + public_host_results: list[dict[str, Any]] = [] + for item in host_results: + result = dict(item) + result["configured_model_available"] = configured_model in set(item.get("models") or []) + result["candidate_models_available"] = [ + candidate for candidate in candidates + if candidate in set(item.get("models") or []) + ] + if not include_models: + result.pop("models", None) + public_host_results.append(result) + + summary = { + "host_count": len(host_results), + "reachable_host_count": len(reachable_hosts), + "configured_model": configured_model, + "configured_model_available_count": len(configured_hosts), + "candidate_model": selected_model, + "candidate_host": selected_host, + "candidate_provider": selected_provider, + "candidate_selection_reason": selected_reason, + "model_route_ready": model_route_ready, + "generate_probe_performed": False, + "model_call_performed": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + } + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": generated_at, + "timeout_seconds": timeout, + "configured_model": configured_model, + "candidate_models": candidates, + "summary": summary, + "hosts": public_host_results, + "controlled_apply": { + "network_call": True, + "model_call": False, + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "secret_read": False, + "production_price_write": False, + "primary_human_gate_count": 0, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "POLICY", + "build_pixelrag_vlm_route_readiness", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index e203e27..8f57ce3 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1434,12 +1434,13 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 33, "warning": 1, "critical": 1, "total": 35} + assert result["summary"] == {"ok": 34, "warning": 1, "critical": 1, "total": 36} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3969,6 +3970,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) first = smoke.collect_ai_automation_smoke(history_limit=5) @@ -4025,7 +4027,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 35, "warning": 0, "critical": 0, "total": 35}, + "summary": {"ok": 36, "warning": 0, "critical": 0, "total": 36}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4151,6 +4153,24 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "next_machine_action": "run_ollama_first_vlm_replay_worker", }, }, + { + "name": "PixelRAG VLM route readiness", + "status": "ok", + "summary": "PixelRAG VLM route readiness hosts=1/5, configured=minicpm-v:latest, configured_available=1, candidate=minicpm-v:latest", + "details": { + "policy": "read_only_pixelrag_vlm_route_readiness_v1", + "host_count": 5, + "reachable_host_count": 1, + "configured_model": "minicpm-v:latest", + "configured_model_available_count": 1, + "candidate_model": "minicpm-v:latest", + "candidate_host": "http://34.21.145.224:11434", + "candidate_provider": "ollama_secondary", + "model_route_ready": True, + "model_call_performed": False, + "next_machine_action": "run_pixelrag_vlm_replay_worker_execute", + }, + }, { "name": "PixelRAG VLM replay worker", "status": "ok", @@ -4165,6 +4185,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "executed_ok_count": 0, "executed_warning_count": 0, "model_error_count": 0, + "model_route_not_ready_count": 0, "parse_error_count": 0, "receipt_written_count": 0, "model_call_performed": False, @@ -4189,7 +4210,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 32 + assert summary["summary"]["total"] == 33 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -5872,6 +5893,14 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( assert pixelrag_ocr_vlm_replay_family["status"] == "ok" assert pixelrag_ocr_vlm_replay_family["details"]["replay_ready_count"] == 1 assert pixelrag_ocr_vlm_replay_family["details"]["extraction_execution_performed"] is False + pixelrag_vlm_route_readiness_family = next( + item for item in summary["families"] + if item["key"] == "pixelrag_vlm_route_readiness" + ) + assert pixelrag_vlm_route_readiness_family["status"] == "ok" + assert pixelrag_vlm_route_readiness_family["details"]["model_route_ready"] is True + assert pixelrag_vlm_route_readiness_family["details"]["model_call_performed"] is False + assert pixelrag_vlm_route_readiness_family["details"]["writes_database_count"] == 0 pixelrag_vlm_replay_worker_family = next( item for item in summary["families"] if item["key"] == "pixelrag_vlm_replay_worker" @@ -6502,6 +6531,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag ocr vlm ok", )) + monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check( + "PixelRAG VLM route readiness", + "ok", + "pixelrag vlm route ok", + )) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check( "PixelRAG VLM replay worker", "ok", @@ -6522,7 +6556,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 35 + assert result["summary"]["total"] == 36 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_vlm_replay_worker_service.py b/tests/test_pixelrag_vlm_replay_worker_service.py index 0d6aaf0..ea0f66c 100644 --- a/tests/test_pixelrag_vlm_replay_worker_service.py +++ b/tests/test_pixelrag_vlm_replay_worker_service.py @@ -106,6 +106,7 @@ def test_pixelrag_vlm_replay_worker_execute_writes_artifact_receipt(tmp_path, mo execute=True, write_receipt=True, tile_limit=1, + auto_select_model=False, ) assert payload["status"] == "ok" @@ -160,6 +161,7 @@ def test_pixelrag_vlm_replay_worker_writes_model_error_receipt(tmp_path, monkeyp execute=True, write_receipt=True, tile_limit=1, + auto_select_model=False, ) assert payload["status"] == "critical" @@ -175,6 +177,86 @@ def test_pixelrag_vlm_replay_worker_writes_model_error_receipt(tmp_path, monkeyp assert receipt["next_machine_action"] == "repair_ollama_vlm_runtime_or_model_route" +def test_pixelrag_vlm_replay_worker_auto_selects_installed_candidate(tmp_path, monkeypatch): + from services import pixelrag_vlm_replay_worker_service as service + + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=sunscreen", + ) + + monkeypatch.setattr( + service, + "build_pixelrag_vlm_route_readiness", + lambda **kwargs: { + "policy": "read_only_pixelrag_vlm_route_readiness_v1", + "status": "warning", + "summary": { + "configured_model": "minicpm-v:latest", + "candidate_model": "gemma3:4b", + "candidate_host": "http://34.21.145.224:11434", + "model_route_ready": True, + }, + "next_machine_action": "run_pixelrag_vlm_replay_worker_execute_with_selected_model", + }, + ) + + class FakeOllama: + def __init__(self, model): + self.model = model + + def generate(self, *args, **kwargs): + assert kwargs["model"] == "gemma3:4b" + return SimpleNamespace( + success=True, + content=json.dumps({ + "blocked_page_detected": False, + "fields": { + "title": { + "value": "防曬乳 SPF50", + "confidence": 0.92, + "evidence_refs": ["tile:1"], + }, + "price": { + "value": "399", + "confidence": 0.88, + "evidence_refs": ["tile:1"], + }, + }, + "quality": { + "overall_confidence": 0.90, + "missing_required_fields": [], + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + }, + }), + model="gemma3:4b", + error=None, + total_duration=1.0, + host="http://34.21.145.224:11434", + input_tokens=10, + output_tokens=50, + ) + + monkeypatch.setattr(service, "OllamaService", FakeOllama) + payload = service.run_pixelrag_ollama_vlm_replay_worker( + artifact_root=tmp_path, + platform="shopee_tw", + model="minicpm-v:latest", + execute=True, + tile_limit=1, + auto_select_model=True, + ) + + assert payload["model"] == "gemma3:4b" + assert payload["configured_model"] == "minicpm-v:latest" + assert payload["route_readiness"]["summary"]["candidate_model"] == "gemma3:4b" + assert payload["summary"]["executed_ok_count"] == 1 + + def test_pixelrag_vlm_replay_worker_cli_outputs_machine_readable_json(tmp_path): _write_receipt( tmp_path, diff --git a/tests/test_pixelrag_vlm_route_readiness_service.py b/tests/test_pixelrag_vlm_route_readiness_service.py new file mode 100644 index 0000000..c24cfd9 --- /dev/null +++ b/tests/test_pixelrag_vlm_route_readiness_service.py @@ -0,0 +1,110 @@ +import json + + +class FakeResponse: + def __init__(self, status_code=200, payload=None, text=""): + self.status_code = status_code + self._payload = payload or {} + self.text = text + + def json(self): + return self._payload + + +def test_pixelrag_vlm_route_readiness_selects_installed_candidate(monkeypatch): + from services import pixelrag_vlm_route_readiness_service as service + + hosts = [ + "http://primary:11434", + "http://secondary:11434", + ] + monkeypatch.setattr(service, "_approved_hosts", lambda: hosts) + + def fake_get(url, timeout): + if url.startswith("http://primary"): + raise TimeoutError("primary timeout") + return FakeResponse( + payload={ + "models": [ + {"name": "gemma3:4b"}, + {"name": "bge-m3:latest"}, + ] + } + ) + + monkeypatch.setattr(service.requests, "get", fake_get) + payload = service.build_pixelrag_vlm_route_readiness( + model="minicpm-v:latest", + timeout_seconds=1, + ) + + assert payload["policy"] == "read_only_pixelrag_vlm_route_readiness_v1" + assert payload["status"] == "warning" + assert payload["summary"]["reachable_host_count"] == 1 + assert payload["summary"]["configured_model_available_count"] == 0 + assert payload["summary"]["candidate_model"] == "gemma3:4b" + assert payload["summary"]["model_route_ready"] is True + assert payload["controlled_apply"]["model_call"] is False + assert payload["controlled_apply"]["writes_database"] is False + assert payload["next_machine_action"] == "run_pixelrag_vlm_replay_worker_execute_with_selected_model" + + +def test_pixelrag_vlm_route_readiness_critical_when_no_candidate(monkeypatch): + from services import pixelrag_vlm_route_readiness_service as service + + monkeypatch.setattr(service, "_approved_hosts", lambda: ["http://secondary:11434"]) + monkeypatch.setattr( + service.requests, + "get", + lambda url, timeout: FakeResponse(payload={"models": [{"name": "bge-m3:latest"}]}), + ) + + payload = service.build_pixelrag_vlm_route_readiness(model="minicpm-v:latest") + + assert payload["status"] == "critical" + assert payload["success"] is False + assert payload["summary"]["model_route_ready"] is False + assert payload["next_machine_action"] == "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host" + + +def test_pixelrag_vlm_route_readiness_route_returns_readback(monkeypatch): + from flask import Flask + from routes import system_public_routes as routes + from services import pixelrag_vlm_route_readiness_service as service + + monkeypatch.setattr( + service, + "build_pixelrag_vlm_route_readiness", + lambda **kwargs: { + "success": True, + "policy": "read_only_pixelrag_vlm_route_readiness_v1", + "status": "ok", + "summary": {"model_route_ready": True, "candidate_model": "gemma3:4b"}, + "next_machine_action": "run_pixelrag_vlm_replay_worker_execute", + }, + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-vlm-route-readiness?model=minicpm-v:latest" + ): + response = routes.ai_automation_pixelrag_vlm_route_readiness_api.__wrapped__() + payload = response.get_json() + + assert payload["policy"] == "read_only_pixelrag_vlm_route_readiness_v1" + assert payload["summary"]["candidate_model"] == "gemma3:4b" + + +def test_pixelrag_vlm_route_readiness_cli_outputs_json(monkeypatch, tmp_path): + from services import pixelrag_vlm_route_readiness_service as service + + monkeypatch.setattr(service, "_approved_hosts", lambda: ["http://secondary:11434"]) + monkeypatch.setattr( + service.requests, + "get", + lambda url, timeout: FakeResponse(payload={"models": [{"name": "gemma3:4b"}]}), + ) + + payload = service.build_pixelrag_vlm_route_readiness(model="minicpm-v:latest") + assert json.dumps(payload, ensure_ascii=False) + assert payload["summary"]["candidate_model"] == "gemma3:4b" From bed3a8a8ee25fea385a23fa11854852247dc39a4 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 00:17:02 +0800 Subject: [PATCH 11/75] feat(ai): add PixelRAG VLM generate preflight --- TODO_NEXT_STEPS.txt | 2 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 1 + routes/system_public_routes.py | 19 +++ .../report_pixelrag_vlm_route_readiness.py | 13 ++ scripts/ops/run_pixelrag_vlm_replay_worker.py | 13 ++ services/ai_automation_smoke_service.py | 22 +++ .../pixelrag_vlm_replay_worker_service.py | 48 ++++++- .../pixelrag_vlm_route_readiness_service.py | 136 ++++++++++++++++-- ...test_pixelrag_vlm_replay_worker_service.py | 77 ++++++++++ ...st_pixelrag_vlm_route_readiness_service.py | 65 +++++++++ 11 files changed, 384 insertions(+), 14 deletions(-) diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 3853591..478de4e 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -9,7 +9,7 @@ python scripts/ops/check_production_version_truth.py 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 -本輪 source target 為 `V10.752`；部署完成前不得宣稱正式環境已是 `V10.752`。 +本輪 source target 為 `V10.753`；部署完成前不得宣稱正式環境已是 `V10.753`。 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ diff --git a/config.py b/config.py index bdada6f..4a2c7ff 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.752" +SYSTEM_VERSION = "V10.753" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 62cb6e5..bc75415 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -882,6 +882,7 @@ POSTGRES_HOST=momo-db | 2026-07-09 | PixelRAG application portfolio 必須把可整合場景轉成主線工作項目 | V10.748 起 `/api/ai-automation/pixelrag-application-portfolio` 與 `scripts/ops/report_pixelrag_application_portfolio.py` 必須輸出 PixelRAG 在 commerce、RAG、UX、ops、marketing、governance 的可整合/可運用 lanes；每條 lane 需有 priority、status、integrates_with、use_cases、current_capability、next_machine_action、no-write 邊界與 forbidden guardrails。此 readback 依據 PixelRAG visual-RAG pattern、Google Merchant product data、Google Product structured data 與 Baymard product list UX 轉成內部工作項目；它不抓外站、不呼叫模型、不讀 secret、不寫 DB、不把像素結果當正式價格。 | | 2026-07-09 | PixelRAG Ollama-first VLM replay worker 必須有 runtime monitoring | V10.751 起 `/api/ai-automation/pixelrag-vlm-replay-worker`、`scripts/ops/run_pixelrag_vlm_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled VLM replay worker / `pixelrag_vlm_replay_worker` family；readback 預設 dry-run，不呼叫模型、不寫 artifact，execute 模式只讀 saved tiles、呼叫 approved Ollama VLM route、驗證 JSON field confidence/evidence refs，並只寫 artifact receipt；model_error 也必須寫 failure artifact receipt，receipt 檔內需自證 `artifact_write_performed=true` 與 `receipt_path`。此 worker 明確標記 `writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`primary_human_gate_count=0`；blocked page 不得輸出商品欄位，ready VLM 結果仍需 identity matcher replay 與 PromotionGate。 | | 2026-07-09 | PixelRAG VLM route readiness 必須有 runtime monitoring 與 execute 前自動選模 | V10.752 起 `/api/ai-automation/pixelrag-vlm-route-readiness`、`scripts/ops/report_pixelrag_vlm_route_readiness.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 read-only approved Ollama route/model readiness / `pixelrag_vlm_route_readiness` family；readback 只打 `/api/tags`，不呼叫 `/api/generate`，輸出 reachable host、configured model available count、candidate model、candidate host、model_route_ready 與 next machine action。`run_pixelrag_vlm_replay_worker.py --execute` 預設必須使用此 readback 自動選擇已安裝候選模型；完全沒有候選時寫 `model_route_not_ready` artifact receipt，不得盲打 missing model。 | +| 2026-07-10 | PixelRAG VLM execute preflight 必須驗證 generate route | V10.753 起 `/api/ai-automation/pixelrag-vlm-route-readiness?probe_generate=true` 與 `scripts/ops/report_pixelrag_vlm_route_readiness.py --probe-generate` 可對已安裝候選模型執行極小 `/api/generate` preflight；smoke 預設仍不呼叫模型。`run_pixelrag_vlm_replay_worker.py --execute` 預設先執行 generate preflight，若 GCP-A direct / 110 proxy timeout、GCP-B candidate 雖安裝但 generate 不健康，worker 必須在送入 screenshot tiles 前寫 `model_route_not_ready` artifact receipt，輸出 `tag_model_route_ready`、`generate_route_ready`、`route_model_call_performed` 與 `tile_model_call_performed=false`，下一步固定為 `repair_ollama_vlm_generate_runtime_or_proxy_timeout`；不得把 tags 可見誤當 VLM 可執行。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index a529702..1a90750 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -785,7 +785,16 @@ def ai_automation_pixelrag_vlm_replay_worker_api(): 'false', 'no', } + probe_generate_before_execute = ( + str(request.args.get('probe_generate_before_execute', 'true')).strip().lower() + not in { + '0', + 'false', + 'no', + } + ) route_readiness_timeout = request.args.get('route_readiness_timeout', 3, type=int) + route_generate_probe_timeout = request.args.get('route_generate_probe_timeout', 8, type=int) return jsonify(run_pixelrag_ollama_vlm_replay_worker( platform=platforms, max_age_hours=max(1, min(max_age_hours or 168, 720)), @@ -797,6 +806,8 @@ def ai_automation_pixelrag_vlm_replay_worker_api(): write_receipt=bool(execute and write_receipt), auto_select_model=auto_select_model, route_readiness_timeout=max(1, min(route_readiness_timeout or 3, 20)), + probe_generate_before_execute=probe_generate_before_execute, + route_generate_probe_timeout=max(1, min(route_generate_probe_timeout or 8, 30)), )) @@ -813,11 +824,19 @@ def ai_automation_pixelrag_vlm_route_readiness_api(): 'true', 'yes', } + probe_generate = str(request.args.get('probe_generate') or '').strip().lower() in { + '1', + 'true', + 'yes', + } timeout = request.args.get('timeout', 3, type=int) + probe_timeout = request.args.get('probe_timeout', 8, type=int) return jsonify(build_pixelrag_vlm_route_readiness( model=str(request.args.get('model') or '').strip() or None, timeout_seconds=max(1, min(timeout or 3, 20)), include_models=include_models, + probe_generate=probe_generate, + probe_timeout_seconds=max(1, min(probe_timeout or 8, 30)), )) diff --git a/scripts/ops/report_pixelrag_vlm_route_readiness.py b/scripts/ops/report_pixelrag_vlm_route_readiness.py index c7b9b10..fa7b6fa 100644 --- a/scripts/ops/report_pixelrag_vlm_route_readiness.py +++ b/scripts/ops/report_pixelrag_vlm_route_readiness.py @@ -34,12 +34,25 @@ def main() -> int: action="store_true", help="輸出每個 host 的 model list。", ) + parser.add_argument( + "--probe-generate", + action="store_true", + help="執行極小 /api/generate preflight；預設不呼叫模型。", + ) + parser.add_argument( + "--probe-timeout", + type=int, + default=8, + help="/api/generate preflight timeout 秒數。", + ) args = parser.parse_args() payload = build_pixelrag_vlm_route_readiness( model=args.model, timeout_seconds=args.timeout, include_models=args.include_models, + probe_generate=args.probe_generate, + probe_timeout_seconds=args.probe_timeout, ) print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) return 0 if payload.get("success") else 1 diff --git a/scripts/ops/run_pixelrag_vlm_replay_worker.py b/scripts/ops/run_pixelrag_vlm_replay_worker.py index ab78f94..a412bd5 100755 --- a/scripts/ops/run_pixelrag_vlm_replay_worker.py +++ b/scripts/ops/run_pixelrag_vlm_replay_worker.py @@ -87,6 +87,17 @@ def main() -> int: default=3, help="auto-select model 的 /api/tags readiness timeout 秒數。", ) + parser.add_argument( + "--no-probe-generate-before-execute", + action="store_true", + help="停用 execute 前的極小 /api/generate preflight。", + ) + parser.add_argument( + "--route-generate-probe-timeout", + type=int, + default=8, + help="execute 前 /api/generate preflight timeout 秒數。", + ) args = parser.parse_args() payload = run_pixelrag_ollama_vlm_replay_worker( @@ -102,6 +113,8 @@ def main() -> int: write_receipt=bool(args.write_receipt and args.execute), auto_select_model=not args.no_auto_select_model, route_readiness_timeout=args.route_readiness_timeout, + probe_generate_before_execute=not args.no_probe_generate_before_execute, + route_generate_probe_timeout=args.route_generate_probe_timeout, ) print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) return 0 if payload.get("success") else 1 diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index c8ffcb4..daa4980 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -4665,9 +4665,24 @@ def build_scheduled_automation_health_summary( "candidate_model": pixelrag_vlm_route_readiness_details.get("candidate_model"), "candidate_host": pixelrag_vlm_route_readiness_details.get("candidate_host"), "candidate_provider": pixelrag_vlm_route_readiness_details.get("candidate_provider"), + "tag_model_route_ready": bool( + pixelrag_vlm_route_readiness_details.get("tag_model_route_ready") + ), "model_route_ready": bool( pixelrag_vlm_route_readiness_details.get("model_route_ready") ), + "generate_probe_performed": bool( + pixelrag_vlm_route_readiness_details.get("generate_probe_performed") + ), + "generate_probe_ok_count": int( + pixelrag_vlm_route_readiness_details.get("generate_probe_ok_count") or 0 + ), + "generate_route_ready": bool( + pixelrag_vlm_route_readiness_details.get("generate_route_ready") + ), + "generate_ready_model": pixelrag_vlm_route_readiness_details.get("generate_ready_model"), + "generate_ready_host": pixelrag_vlm_route_readiness_details.get("generate_ready_host"), + "generate_ready_provider": pixelrag_vlm_route_readiness_details.get("generate_ready_provider"), "model_call_performed": bool( pixelrag_vlm_route_readiness_details.get("model_call_performed") ), @@ -13524,7 +13539,14 @@ def _pixelrag_vlm_route_readiness_check() -> Dict[str, Any]: "candidate_model": candidate_model, "candidate_host": summary.get("candidate_host"), "candidate_provider": summary.get("candidate_provider"), + "tag_model_route_ready": bool(summary.get("tag_model_route_ready")), "model_route_ready": bool(summary.get("model_route_ready")), + "generate_probe_performed": bool(summary.get("generate_probe_performed")), + "generate_probe_ok_count": int(summary.get("generate_probe_ok_count") or 0), + "generate_route_ready": bool(summary.get("generate_route_ready")), + "generate_ready_model": summary.get("generate_ready_model"), + "generate_ready_host": summary.get("generate_ready_host"), + "generate_ready_provider": summary.get("generate_ready_provider"), "model_call_performed": bool(summary.get("model_call_performed")), "next_machine_action": readback.get("next_machine_action"), "writes_database": False, diff --git a/services/pixelrag_vlm_replay_worker_service.py b/services/pixelrag_vlm_replay_worker_service.py index 853e765..aee70ea 100644 --- a/services/pixelrag_vlm_replay_worker_service.py +++ b/services/pixelrag_vlm_replay_worker_service.py @@ -34,6 +34,7 @@ POLICY = "controlled_pixelrag_ollama_vlm_replay_worker_v1" DEFAULT_LIMIT = 25 DEFAULT_TILE_LIMIT = 4 DEFAULT_TIMEOUT_SECONDS = 90 +DEFAULT_ROUTE_GENERATE_PROBE_TIMEOUT_SECONDS = 8 DEFAULT_OUTPUT_ROOT = os.getenv( "PIXELRAG_VLM_REPLAY_RECEIPT_ROOT", "/app/data/ai_automation/pixelrag_vlm_replay_receipts" @@ -293,6 +294,13 @@ def _model_route_not_ready_item( "model": summary.get("configured_model"), "candidate_model": summary.get("candidate_model"), "candidate_host": summary.get("candidate_host"), + "tag_model_route_ready": bool(summary.get("tag_model_route_ready")), + "generate_probe_performed": bool(summary.get("generate_probe_performed")), + "generate_probe_ok_count": int(summary.get("generate_probe_ok_count") or 0), + "generate_route_ready": bool(summary.get("generate_route_ready")), + "generate_ready_model": summary.get("generate_ready_model"), + "generate_ready_host": summary.get("generate_ready_host"), + "generate_ready_provider": summary.get("generate_ready_provider"), "model_call_performed": False, "artifact_write_performed": False, "writes_database": False, @@ -435,6 +443,8 @@ def run_pixelrag_ollama_vlm_replay_worker( write_receipt: bool = False, auto_select_model: bool = True, route_readiness_timeout: int | None = None, + probe_generate_before_execute: bool = True, + route_generate_probe_timeout: int | None = None, ) -> dict[str, Any]: """Run or dry-run the PixelRAG VLM replay worker.""" root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT) @@ -446,6 +456,16 @@ def run_pixelrag_ollama_vlm_replay_worker( selected_model = str(model or DEFAULT_MODEL) selected_timeout = max(10, int(timeout or DEFAULT_TIMEOUT_SECONDS)) readiness_timeout = max(1, min(int(route_readiness_timeout or 3), 20)) + generate_probe_timeout = max( + 1, + min( + int( + route_generate_probe_timeout + or DEFAULT_ROUTE_GENERATE_PROBE_TIMEOUT_SECONDS + ), + 30, + ), + ) generated_at = datetime.now(timezone.utc).isoformat() route_readiness: dict[str, Any] | None = None model_route_ready = True @@ -461,6 +481,8 @@ def run_pixelrag_ollama_vlm_replay_worker( route_readiness = build_pixelrag_vlm_route_readiness( model=selected_model, timeout_seconds=readiness_timeout, + probe_generate=bool(probe_generate_before_execute), + probe_timeout_seconds=generate_probe_timeout, ) route_summary = route_readiness.get("summary") or {} candidate_model = str(route_summary.get("candidate_model") or "").strip() @@ -509,7 +531,19 @@ def run_pixelrag_ollama_vlm_replay_worker( int(item.get("required_field_missing_count") or 0) for item in worker_items ) - model_call_performed = any(bool(item.get("model_call_performed")) for item in worker_items) + tile_model_call_performed = any( + bool(item.get("model_call_performed")) for item in worker_items + ) + route_model_call_performed = bool( + route_readiness + and ( + (route_readiness.get("controlled_apply") or {}).get("model_call") + or (route_readiness.get("summary") or {}).get("model_call_performed") + ) + ) + model_call_performed = bool( + tile_model_call_performed or route_model_call_performed + ) artifact_write_performed = any(bool(item.get("artifact_write_performed")) for item in worker_items) if parse_error_count or model_error_count or route_not_ready_count or no_tile_count: @@ -524,7 +558,11 @@ def run_pixelrag_ollama_vlm_replay_worker( elif not execute and ready_count: next_action = "run_pixelrag_vlm_replay_worker_execute" elif route_not_ready_count: - next_action = "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host" + next_action = ( + route_readiness.get("next_machine_action") + if route_readiness + else None + ) or "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host" elif model_error_count or parse_error_count: next_action = "repair_ollama_vlm_runtime_or_model_route" elif executed_warning_count: @@ -548,6 +586,8 @@ def run_pixelrag_ollama_vlm_replay_worker( "no_tile_count": no_tile_count, "receipt_written_count": receipt_written_count, "required_field_missing_count": required_missing_count, + "route_model_call_performed": route_model_call_performed, + "tile_model_call_performed": tile_model_call_performed, "model_call_performed": model_call_performed, "artifact_write_performed": artifact_write_performed, "writes_database_count": 0, @@ -572,6 +612,8 @@ def run_pixelrag_ollama_vlm_replay_worker( "write_receipt": bool(write_receipt), "auto_select_model": bool(auto_select_model), "route_readiness_timeout_seconds": readiness_timeout, + "probe_generate_before_execute": bool(probe_generate_before_execute), + "route_generate_probe_timeout_seconds": generate_probe_timeout, "summary": summary, "worker_items": worker_items, "route_readiness": ( @@ -591,7 +633,7 @@ def run_pixelrag_ollama_vlm_replay_worker( "next_machine_action": contract.get("next_machine_action"), }, "controlled_apply": { - "network_call": bool(execute and model_call_performed), + "network_call": bool(execute and (route_readiness or model_call_performed)), "model_call": bool(execute and model_call_performed), "artifact_write": artifact_write_performed, "db_write": False, diff --git a/services/pixelrag_vlm_route_readiness_service.py b/services/pixelrag_vlm_route_readiness_service.py index 9cc4acf..aeec0fe 100644 --- a/services/pixelrag_vlm_route_readiness_service.py +++ b/services/pixelrag_vlm_route_readiness_service.py @@ -1,14 +1,15 @@ -"""Read-only PixelRAG VLM route readiness. +"""PixelRAG VLM route readiness. This module checks approved Ollama routes for installed VLM candidate models. -It does not call /api/generate, read secrets, write DB data, or change runtime -configuration. The worker can use the result to avoid blindly executing a -missing configured model. +By default it only reads /api/tags. Execute preflight can opt into a tiny +/api/generate probe to avoid blindly sending screenshot tiles to a route that +is installed but cannot generate. """ from __future__ import annotations import os +import time from datetime import datetime, timezone from typing import Any @@ -27,6 +28,8 @@ from services.ollama_service import ( POLICY = "read_only_pixelrag_vlm_route_readiness_v1" DEFAULT_TIMEOUT_SECONDS = 3 +DEFAULT_PROBE_TIMEOUT_SECONDS = 8 +DEFAULT_PROBE_PROMPT = "Return exactly OK." DEFAULT_MODEL = ( os.getenv("PIXELRAG_VLM_MODEL") or os.getenv("PPT_VISION_MODEL") @@ -103,15 +106,83 @@ def _fetch_models(host: str, timeout: int) -> dict[str, Any]: return item +def _probe_generate( + host: str, + *, + model: str, + timeout: int, + prompt: str, +) -> dict[str, Any]: + clean_host = str(host or "").rstrip("/") + started = time.monotonic() + result: dict[str, Any] = { + "generate_probe_performed": True, + "generate_probe_model": str(model or "").strip(), + "generate_probe_ok": False, + "generate_probe_error": "", + "generate_probe_duration_ms": 0, + } + payload = { + "model": str(model or "").strip(), + "prompt": str(prompt or DEFAULT_PROBE_PROMPT), + "stream": False, + "options": { + "temperature": 0, + "num_predict": 8, + "num_ctx": 512, + }, + "keep_alive": "1m", + } + try: + response = requests.post( + f"{clean_host}/api/generate", + json=payload, + timeout=max(1, timeout), + ) + result["generate_probe_http_status"] = response.status_code + if response.status_code != 200: + result["generate_probe_error"] = ( + f"HTTP {response.status_code}: {response.text[:180]}" + ) + return result + try: + body = response.json() + except Exception as exc: + result["generate_probe_error"] = f"invalid_json: {type(exc).__name__}" + return result + result["generate_probe_ok"] = ( + isinstance(body, dict) + and ( + "response" in body + or bool(body.get("done")) + or bool(body.get("context")) + ) + ) + if not result["generate_probe_ok"]: + result["generate_probe_error"] = "generate_response_missing_output_fields" + except Exception as exc: + result["generate_probe_error"] = f"{type(exc).__name__}: {str(exc)[:180]}" + finally: + result["generate_probe_duration_ms"] = int((time.monotonic() - started) * 1000) + return result + + def build_pixelrag_vlm_route_readiness( *, model: str | None = None, timeout_seconds: int | None = None, include_models: bool = False, + probe_generate: bool = False, + probe_timeout_seconds: int | None = None, + probe_prompt: str | None = None, ) -> dict[str, Any]: """Read approved Ollama tags and recommend an installed VLM model.""" configured_model = str(model or DEFAULT_MODEL).strip() or DEFAULT_MODEL timeout = max(1, min(int(timeout_seconds or DEFAULT_TIMEOUT_SECONDS), 20)) + probe_timeout = max( + 1, + min(int(probe_timeout_seconds or DEFAULT_PROBE_TIMEOUT_SECONDS), 30), + ) generated_at = datetime.now(timezone.utc).isoformat() candidates = _candidate_models(configured_model) host_results = [_fetch_models(host, timeout) for host in _approved_hosts()] @@ -141,13 +212,48 @@ def build_pixelrag_vlm_route_readiness( if selected_model: break - model_route_ready = bool(selected_model and selected_host) + tag_model_route_ready = bool(selected_model and selected_host) + generate_probe_performed = bool(probe_generate and tag_model_route_ready) + generate_probe_ok_count = 0 + generate_ready_model = "" + generate_ready_host = "" + generate_ready_provider = "" + generate_route_ready: bool | None = None + if generate_probe_performed: + generate_route_ready = False + for host in reachable_hosts: + if selected_model not in set(host.get("models") or []): + continue + probe_result = _probe_generate( + str(host.get("host") or ""), + model=selected_model, + timeout=probe_timeout, + prompt=probe_prompt or DEFAULT_PROBE_PROMPT, + ) + host.update(probe_result) + if probe_result.get("generate_probe_ok"): + generate_probe_ok_count += 1 + if not generate_ready_host: + generate_ready_model = selected_model + generate_ready_host = str(host.get("host") or "") + generate_ready_provider = str(host.get("provider") or "") + if generate_ready_host: + selected_host = generate_ready_host + selected_provider = generate_ready_provider + generate_route_ready = True + + model_route_ready = tag_model_route_ready and ( + bool(generate_route_ready) if generate_probe_performed else True + ) if not reachable_hosts: status = "critical" next_action = "repair_approved_ollama_host_connectivity" - elif not model_route_ready: + elif not tag_model_route_ready: status = "critical" next_action = "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host" + elif generate_probe_performed and not generate_route_ready: + status = "critical" + next_action = "repair_ollama_vlm_generate_runtime_or_proxy_timeout" elif selected_model != configured_model: status = "warning" next_action = "run_pixelrag_vlm_replay_worker_execute_with_selected_model" @@ -163,6 +269,11 @@ def build_pixelrag_vlm_route_readiness( candidate for candidate in candidates if candidate in set(item.get("models") or []) ] + result.setdefault("generate_probe_performed", False) + result.setdefault("generate_probe_model", "") + result.setdefault("generate_probe_ok", False) + result.setdefault("generate_probe_error", "") + result.setdefault("generate_probe_duration_ms", 0) if not include_models: result.pop("models", None) public_host_results.append(result) @@ -176,9 +287,15 @@ def build_pixelrag_vlm_route_readiness( "candidate_host": selected_host, "candidate_provider": selected_provider, "candidate_selection_reason": selected_reason, + "tag_model_route_ready": tag_model_route_ready, "model_route_ready": model_route_ready, - "generate_probe_performed": False, - "model_call_performed": False, + "generate_probe_performed": generate_probe_performed, + "generate_probe_ok_count": generate_probe_ok_count, + "generate_ready_model": generate_ready_model, + "generate_ready_host": generate_ready_host, + "generate_ready_provider": generate_ready_provider, + "generate_route_ready": generate_route_ready, + "model_call_performed": generate_probe_performed, "writes_database_count": 0, "primary_human_gate_count": 0, } @@ -188,13 +305,14 @@ def build_pixelrag_vlm_route_readiness( "status": status, "generated_at": generated_at, "timeout_seconds": timeout, + "probe_timeout_seconds": probe_timeout if probe_generate else 0, "configured_model": configured_model, "candidate_models": candidates, "summary": summary, "hosts": public_host_results, "controlled_apply": { "network_call": True, - "model_call": False, + "model_call": generate_probe_performed, "db_write": False, "writes_database": False, "writes_database_count": 0, diff --git a/tests/test_pixelrag_vlm_replay_worker_service.py b/tests/test_pixelrag_vlm_replay_worker_service.py index ea0f66c..63c6d64 100644 --- a/tests/test_pixelrag_vlm_replay_worker_service.py +++ b/tests/test_pixelrag_vlm_replay_worker_service.py @@ -257,6 +257,83 @@ def test_pixelrag_vlm_replay_worker_auto_selects_installed_candidate(tmp_path, m assert payload["summary"]["executed_ok_count"] == 1 +def test_pixelrag_vlm_replay_worker_generate_probe_blocks_tile_generate(tmp_path, monkeypatch): + from services import pixelrag_vlm_replay_worker_service as service + + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=sunscreen", + ) + + monkeypatch.setattr( + service, + "build_pixelrag_vlm_route_readiness", + lambda **kwargs: { + "policy": "read_only_pixelrag_vlm_route_readiness_v1", + "status": "critical", + "summary": { + "configured_model": "minicpm-v:latest", + "candidate_model": "gemma3:4b", + "candidate_host": "http://34.21.145.224:11434", + "tag_model_route_ready": True, + "model_route_ready": False, + "generate_probe_performed": True, + "generate_probe_ok_count": 0, + "generate_route_ready": False, + "generate_ready_model": "", + "generate_ready_host": "", + "generate_ready_provider": "", + "model_call_performed": True, + }, + "controlled_apply": {"model_call": True}, + "next_machine_action": "repair_ollama_vlm_generate_runtime_or_proxy_timeout", + }, + ) + + class FakeOllama: + def __init__(self, model): + self.model = model + + def generate(self, *args, **kwargs): + raise AssertionError("tile VLM generate should not be called") + + monkeypatch.setattr(service, "OllamaService", FakeOllama) + payload = service.run_pixelrag_ollama_vlm_replay_worker( + artifact_root=tmp_path, + output_root=tmp_path / "receipts", + platform="shopee_tw", + model="minicpm-v:latest", + execute=True, + write_receipt=True, + tile_limit=1, + auto_select_model=True, + probe_generate_before_execute=True, + ) + + assert payload["status"] == "critical" + assert payload["model"] == "gemma3:4b" + assert payload["summary"]["model_route_not_ready_count"] == 1 + assert payload["summary"]["route_model_call_performed"] is True + assert payload["summary"]["tile_model_call_performed"] is False + assert payload["summary"]["model_call_performed"] is True + assert payload["controlled_apply"]["model_call"] is True + assert payload["next_machine_action"] == "repair_ollama_vlm_generate_runtime_or_proxy_timeout" + item = payload["worker_items"][0] + assert item["worker_status"] == "model_route_not_ready" + assert item["generate_probe_performed"] is True + assert item["generate_route_ready"] is False + assert item["model_call_performed"] is False + receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-ok" / "vlm_replay_receipt.json" + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "model_route_not_ready" + assert receipt["artifact_write_performed"] is True + assert receipt["generate_route_ready"] is False + + def test_pixelrag_vlm_replay_worker_cli_outputs_machine_readable_json(tmp_path): _write_receipt( tmp_path, diff --git a/tests/test_pixelrag_vlm_route_readiness_service.py b/tests/test_pixelrag_vlm_route_readiness_service.py index c24cfd9..b0abfb0 100644 --- a/tests/test_pixelrag_vlm_route_readiness_service.py +++ b/tests/test_pixelrag_vlm_route_readiness_service.py @@ -67,6 +67,70 @@ def test_pixelrag_vlm_route_readiness_critical_when_no_candidate(monkeypatch): assert payload["next_machine_action"] == "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host" +def test_pixelrag_vlm_route_readiness_generate_probe_success(monkeypatch): + from services import pixelrag_vlm_route_readiness_service as service + + monkeypatch.setattr(service, "_approved_hosts", lambda: ["http://secondary:11434"]) + monkeypatch.setattr( + service.requests, + "get", + lambda url, timeout: FakeResponse(payload={"models": [{"name": "gemma3:4b"}]}), + ) + monkeypatch.setattr( + service.requests, + "post", + lambda url, json, timeout: FakeResponse(payload={"response": "OK", "done": True}), + ) + + payload = service.build_pixelrag_vlm_route_readiness( + model="minicpm-v:latest", + probe_generate=True, + probe_timeout_seconds=1, + ) + + assert payload["status"] == "warning" + assert payload["summary"]["candidate_model"] == "gemma3:4b" + assert payload["summary"]["tag_model_route_ready"] is True + assert payload["summary"]["model_route_ready"] is True + assert payload["summary"]["generate_probe_performed"] is True + assert payload["summary"]["generate_probe_ok_count"] == 1 + assert payload["summary"]["generate_route_ready"] is True + assert payload["summary"]["generate_ready_host"] == "http://secondary:11434" + assert payload["controlled_apply"]["model_call"] is True + + +def test_pixelrag_vlm_route_readiness_generate_probe_failure_blocks_route(monkeypatch): + from services import pixelrag_vlm_route_readiness_service as service + + monkeypatch.setattr(service, "_approved_hosts", lambda: ["http://secondary:11434"]) + monkeypatch.setattr( + service.requests, + "get", + lambda url, timeout: FakeResponse(payload={"models": [{"name": "gemma3:4b"}]}), + ) + + def fake_post(url, json, timeout): + raise TimeoutError("generate timeout") + + monkeypatch.setattr(service.requests, "post", fake_post) + + payload = service.build_pixelrag_vlm_route_readiness( + model="minicpm-v:latest", + probe_generate=True, + probe_timeout_seconds=1, + ) + + assert payload["status"] == "critical" + assert payload["success"] is False + assert payload["summary"]["candidate_model"] == "gemma3:4b" + assert payload["summary"]["tag_model_route_ready"] is True + assert payload["summary"]["model_route_ready"] is False + assert payload["summary"]["generate_probe_performed"] is True + assert payload["summary"]["generate_probe_ok_count"] == 0 + assert payload["summary"]["generate_route_ready"] is False + assert payload["next_machine_action"] == "repair_ollama_vlm_generate_runtime_or_proxy_timeout" + + def test_pixelrag_vlm_route_readiness_route_returns_readback(monkeypatch): from flask import Flask from routes import system_public_routes as routes @@ -87,6 +151,7 @@ def test_pixelrag_vlm_route_readiness_route_returns_readback(monkeypatch): app = Flask(__name__) with app.test_request_context( "/api/ai-automation/pixelrag-vlm-route-readiness?model=minicpm-v:latest" + "&probe_generate=true&probe_timeout=4" ): response = routes.ai_automation_pixelrag_vlm_route_readiness_api.__wrapped__() payload = response.get_json() From 9cf8b55b72c31e4a2e72b6d21bab7026013fbdd9 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 00:23:05 +0800 Subject: [PATCH 12/75] fix(ai): bind PixelRAG VLM worker to selected route --- TODO_NEXT_STEPS.txt | 2 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 1 + .../pixelrag_vlm_replay_worker_service.py | 123 ++++++++++++++++-- ...test_pixelrag_vlm_replay_worker_service.py | 69 +++++----- 5 files changed, 150 insertions(+), 47 deletions(-) diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 478de4e..677ff4f 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -9,7 +9,7 @@ python scripts/ops/check_production_version_truth.py 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 -本輪 source target 為 `V10.753`；部署完成前不得宣稱正式環境已是 `V10.753`。 +本輪 source target 為 `V10.754`；部署完成前不得宣稱正式環境已是 `V10.754`。 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ diff --git a/config.py b/config.py index 4a2c7ff..408f795 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.753" +SYSTEM_VERSION = "V10.754" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index bc75415..871c52f 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -883,6 +883,7 @@ POSTGRES_HOST=momo-db | 2026-07-09 | PixelRAG Ollama-first VLM replay worker 必須有 runtime monitoring | V10.751 起 `/api/ai-automation/pixelrag-vlm-replay-worker`、`scripts/ops/run_pixelrag_vlm_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled VLM replay worker / `pixelrag_vlm_replay_worker` family；readback 預設 dry-run，不呼叫模型、不寫 artifact，execute 模式只讀 saved tiles、呼叫 approved Ollama VLM route、驗證 JSON field confidence/evidence refs，並只寫 artifact receipt；model_error 也必須寫 failure artifact receipt，receipt 檔內需自證 `artifact_write_performed=true` 與 `receipt_path`。此 worker 明確標記 `writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`primary_human_gate_count=0`；blocked page 不得輸出商品欄位，ready VLM 結果仍需 identity matcher replay 與 PromotionGate。 | | 2026-07-09 | PixelRAG VLM route readiness 必須有 runtime monitoring 與 execute 前自動選模 | V10.752 起 `/api/ai-automation/pixelrag-vlm-route-readiness`、`scripts/ops/report_pixelrag_vlm_route_readiness.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 read-only approved Ollama route/model readiness / `pixelrag_vlm_route_readiness` family；readback 只打 `/api/tags`，不呼叫 `/api/generate`，輸出 reachable host、configured model available count、candidate model、candidate host、model_route_ready 與 next machine action。`run_pixelrag_vlm_replay_worker.py --execute` 預設必須使用此 readback 自動選擇已安裝候選模型；完全沒有候選時寫 `model_route_not_ready` artifact receipt，不得盲打 missing model。 | | 2026-07-10 | PixelRAG VLM execute preflight 必須驗證 generate route | V10.753 起 `/api/ai-automation/pixelrag-vlm-route-readiness?probe_generate=true` 與 `scripts/ops/report_pixelrag_vlm_route_readiness.py --probe-generate` 可對已安裝候選模型執行極小 `/api/generate` preflight；smoke 預設仍不呼叫模型。`run_pixelrag_vlm_replay_worker.py --execute` 預設先執行 generate preflight，若 GCP-A direct / 110 proxy timeout、GCP-B candidate 雖安裝但 generate 不健康，worker 必須在送入 screenshot tiles 前寫 `model_route_not_ready` artifact receipt，輸出 `tag_model_route_ready`、`generate_route_ready`、`route_model_call_performed` 與 `tile_model_call_performed=false`，下一步固定為 `repair_ollama_vlm_generate_runtime_or_proxy_timeout`；不得把 tags 可見誤當 VLM 可執行。 | +| 2026-07-10 | PixelRAG VLM tile generate 必須綁定 preflight 選出的 host / exact model | V10.754 起 `run_pixelrag_vlm_replay_worker.py --execute` 若 route readiness 已輸出 `candidate_host`，tile VLM generate 必須使用該 approved host 與 exact `candidate_model` 直呼 `/api/generate`，不得重新進入全域 Ollama resolver、不得因 111 fallback 規則把 VLM 模型降級成文字模型、不得再出現 preflight 選 111 但 tile generate tried GCP-B/GCP-A/111 的 route drift。receipt 必須輸出 `route_candidate_host`，模型錯誤時下一步為 `repair_ollama_vlm_generate_runtime_or_proxy_timeout`，仍然不寫 DB、不寫正式價格表。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/services/pixelrag_vlm_replay_worker_service.py b/services/pixelrag_vlm_replay_worker_service.py index aee70ea..d8d2306 100644 --- a/services/pixelrag_vlm_replay_worker_service.py +++ b/services/pixelrag_vlm_replay_worker_service.py @@ -16,7 +16,15 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Mapping -from services.ollama_service import OllamaService, get_host_label, get_provider_tag +import requests + +from services.ollama_service import ( + OllamaResponse, + OllamaService, + get_host_label, + get_provider_tag, + is_approved_ollama_host, +) from services.pixelrag_crawler_integration_service import ( DEFAULT_ARTIFACT_MAX_AGE_HOURS, DEFAULT_ARTIFACT_ROOT, @@ -223,6 +231,78 @@ def _validate_model_payload( } +def _generate_exact_host( + prompt: str, + *, + host: str, + model: str, + temperature: float, + timeout: int, + options: Mapping[str, Any] | None, + images: list[str], +) -> OllamaResponse: + """Call the route-readiness selected host without fallback or model downgrade.""" + clean_host = str(host or "").rstrip("/") + if not is_approved_ollama_host(clean_host): + return OllamaResponse( + success=False, + content="", + model=model, + error=f"unapproved_pixelrag_vlm_candidate_host: {clean_host}", + host=clean_host or "unknown", + ) + payload: dict[str, Any] = { + "model": model, + "prompt": prompt, + "stream": False, + "options": {"temperature": temperature}, + } + if options: + payload["options"].update(dict(options)) + if images: + payload["images"] = images + try: + response = requests.post( + f"{clean_host}/api/generate", + json=payload, + timeout=max(1, int(timeout or DEFAULT_TIMEOUT_SECONDS)), + ) + if response.status_code != 200: + return OllamaResponse( + success=False, + content="", + model=model, + error=f"HTTP {response.status_code}: {response.text[:RAW_EXCERPT_LIMIT]}", + host=clean_host, + ) + data = response.json() + return OllamaResponse( + success=True, + content=data.get("response", ""), + model=model, + total_duration=(data.get("total_duration", 0) or 0) / 1e9, + host=clean_host, + input_tokens=int(data.get("prompt_eval_count", 0) or 0), + output_tokens=int(data.get("eval_count", 0) or 0), + ) + except requests.Timeout: + return OllamaResponse( + success=False, + content="", + model=model, + error=f"timeout ({max(1, int(timeout or DEFAULT_TIMEOUT_SECONDS))}s)", + host=clean_host, + ) + except Exception as exc: + return OllamaResponse( + success=False, + content="", + model=model, + error=f"{type(exc).__name__}: {str(exc)[:RAW_EXCERPT_LIMIT]}", + host=clean_host, + ) + + def _write_replay_receipt( *, output_root: Path, @@ -324,6 +404,7 @@ def _execute_item( root: Path, output_root: Path, model: str, + route_host: str | None, timeout: int, tile_limit: int, write_receipt: bool, @@ -335,6 +416,7 @@ def _execute_item( "source_receipt_path": item.get("source_receipt_path"), "worker_status": "executing", "model": model, + "route_candidate_host": str(route_host or ""), "tile_evidence": tile_evidence, "tile_image_count": len(images), "model_call_performed": bool(images), @@ -348,14 +430,27 @@ def _execute_item( }) return base - response = OllamaService(model=model).generate( - _prompt_for_item(item), - model=model, - temperature=0.1, - timeout=max(10, int(timeout or DEFAULT_TIMEOUT_SECONDS)), - options={"num_predict": 700, "num_ctx": 4096}, - images=images, - ) + prompt = _prompt_for_item(item) + options = {"num_predict": 700, "num_ctx": 4096} + if route_host: + response = _generate_exact_host( + prompt, + host=route_host, + model=model, + temperature=0.1, + timeout=max(10, int(timeout or DEFAULT_TIMEOUT_SECONDS)), + options=options, + images=images, + ) + else: + response = OllamaService(model=model).generate( + prompt, + model=model, + temperature=0.1, + timeout=max(10, int(timeout or DEFAULT_TIMEOUT_SECONDS)), + options=options, + images=images, + ) base.update({ "host": response.host, "host_label": get_host_label(response.host or ""), @@ -369,7 +464,11 @@ def _execute_item( base.update({ "worker_status": "model_error", "model_error": str(response.error or "")[:RAW_EXCERPT_LIMIT], - "next_machine_action": "repair_ollama_vlm_runtime_or_model_route", + "next_machine_action": ( + "repair_ollama_vlm_generate_runtime_or_proxy_timeout" + if route_host + else "repair_ollama_vlm_runtime_or_model_route" + ), }) if write_receipt: base["receipt_path"] = _write_replay_receipt( @@ -454,6 +553,7 @@ def run_pixelrag_ollama_vlm_replay_worker( item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) tiles = max(1, min(int(tile_limit or DEFAULT_TILE_LIMIT), 12)) selected_model = str(model or DEFAULT_MODEL) + selected_route_host = "" selected_timeout = max(10, int(timeout or DEFAULT_TIMEOUT_SECONDS)) readiness_timeout = max(1, min(int(route_readiness_timeout or 3), 20)) generate_probe_timeout = max( @@ -489,6 +589,7 @@ def run_pixelrag_ollama_vlm_replay_worker( model_route_ready = bool(route_summary.get("model_route_ready")) if candidate_model: selected_model = candidate_model + selected_route_host = str(route_summary.get("candidate_host") or "").strip() worker_items: list[dict[str, Any]] = [] for item in replay_items: @@ -511,6 +612,7 @@ def run_pixelrag_ollama_vlm_replay_worker( root=root, output_root=output, model=selected_model, + route_host=selected_route_host, timeout=selected_timeout, tile_limit=tiles, write_receipt=write_receipt, @@ -607,6 +709,7 @@ def run_pixelrag_ollama_vlm_replay_worker( "tile_limit": tiles, "model": selected_model, "configured_model": str(model or DEFAULT_MODEL), + "route_candidate_host": selected_route_host, "timeout_seconds": selected_timeout, "execute": bool(execute), "write_receipt": bool(write_receipt), diff --git a/tests/test_pixelrag_vlm_replay_worker_service.py b/tests/test_pixelrag_vlm_replay_worker_service.py index 63c6d64..b0179bc 100644 --- a/tests/test_pixelrag_vlm_replay_worker_service.py +++ b/tests/test_pixelrag_vlm_replay_worker_service.py @@ -204,44 +204,41 @@ def test_pixelrag_vlm_replay_worker_auto_selects_installed_candidate(tmp_path, m }, ) - class FakeOllama: - def __init__(self, model): - self.model = model - - def generate(self, *args, **kwargs): - assert kwargs["model"] == "gemma3:4b" - return SimpleNamespace( - success=True, - content=json.dumps({ - "blocked_page_detected": False, - "fields": { - "title": { - "value": "防曬乳 SPF50", - "confidence": 0.92, - "evidence_refs": ["tile:1"], - }, - "price": { - "value": "399", - "confidence": 0.88, - "evidence_refs": ["tile:1"], - }, + def fake_generate_exact_host(prompt, **kwargs): + assert kwargs["host"] == "http://34.21.145.224:11434" + assert kwargs["model"] == "gemma3:4b" + return SimpleNamespace( + success=True, + content=json.dumps({ + "blocked_page_detected": False, + "fields": { + "title": { + "value": "防曬乳 SPF50", + "confidence": 0.92, + "evidence_refs": ["tile:1"], }, - "quality": { - "overall_confidence": 0.90, - "missing_required_fields": [], - "requires_identity_matcher_replay": True, - "requires_promotion_gate": True, + "price": { + "value": "399", + "confidence": 0.88, + "evidence_refs": ["tile:1"], }, - }), - model="gemma3:4b", - error=None, - total_duration=1.0, - host="http://34.21.145.224:11434", - input_tokens=10, - output_tokens=50, - ) + }, + "quality": { + "overall_confidence": 0.90, + "missing_required_fields": [], + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + }, + }), + model="gemma3:4b", + error=None, + total_duration=1.0, + host="http://34.21.145.224:11434", + input_tokens=10, + output_tokens=50, + ) - monkeypatch.setattr(service, "OllamaService", FakeOllama) + monkeypatch.setattr(service, "_generate_exact_host", fake_generate_exact_host) payload = service.run_pixelrag_ollama_vlm_replay_worker( artifact_root=tmp_path, platform="shopee_tw", @@ -253,7 +250,9 @@ def test_pixelrag_vlm_replay_worker_auto_selects_installed_candidate(tmp_path, m assert payload["model"] == "gemma3:4b" assert payload["configured_model"] == "minicpm-v:latest" + assert payload["route_candidate_host"] == "http://34.21.145.224:11434" assert payload["route_readiness"]["summary"]["candidate_model"] == "gemma3:4b" + assert payload["worker_items"][0]["route_candidate_host"] == "http://34.21.145.224:11434" assert payload["summary"]["executed_ok_count"] == 1 From 2059ab3cc231cb9f49c672a3f6eb131e51c7c378 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 00:27:39 +0800 Subject: [PATCH 13/75] fix(ai): preserve unknown VLM generate readiness --- TODO_NEXT_STEPS.txt | 2 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 1 + services/ai_automation_smoke_service.py | 10 ++++++++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 677ff4f..2cd9e3c 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -9,7 +9,7 @@ python scripts/ops/check_production_version_truth.py 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 -本輪 source target 為 `V10.754`；部署完成前不得宣稱正式環境已是 `V10.754`。 +本輪 source target 為 `V10.755`；部署完成前不得宣稱正式環境已是 `V10.755`。 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ diff --git a/config.py b/config.py index 408f795..792a9f9 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.754" +SYSTEM_VERSION = "V10.755" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 871c52f..87ff2bb 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -884,6 +884,7 @@ POSTGRES_HOST=momo-db | 2026-07-09 | PixelRAG VLM route readiness 必須有 runtime monitoring 與 execute 前自動選模 | V10.752 起 `/api/ai-automation/pixelrag-vlm-route-readiness`、`scripts/ops/report_pixelrag_vlm_route_readiness.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 read-only approved Ollama route/model readiness / `pixelrag_vlm_route_readiness` family；readback 只打 `/api/tags`，不呼叫 `/api/generate`，輸出 reachable host、configured model available count、candidate model、candidate host、model_route_ready 與 next machine action。`run_pixelrag_vlm_replay_worker.py --execute` 預設必須使用此 readback 自動選擇已安裝候選模型；完全沒有候選時寫 `model_route_not_ready` artifact receipt，不得盲打 missing model。 | | 2026-07-10 | PixelRAG VLM execute preflight 必須驗證 generate route | V10.753 起 `/api/ai-automation/pixelrag-vlm-route-readiness?probe_generate=true` 與 `scripts/ops/report_pixelrag_vlm_route_readiness.py --probe-generate` 可對已安裝候選模型執行極小 `/api/generate` preflight；smoke 預設仍不呼叫模型。`run_pixelrag_vlm_replay_worker.py --execute` 預設先執行 generate preflight，若 GCP-A direct / 110 proxy timeout、GCP-B candidate 雖安裝但 generate 不健康，worker 必須在送入 screenshot tiles 前寫 `model_route_not_ready` artifact receipt，輸出 `tag_model_route_ready`、`generate_route_ready`、`route_model_call_performed` 與 `tile_model_call_performed=false`，下一步固定為 `repair_ollama_vlm_generate_runtime_or_proxy_timeout`；不得把 tags 可見誤當 VLM 可執行。 | | 2026-07-10 | PixelRAG VLM tile generate 必須綁定 preflight 選出的 host / exact model | V10.754 起 `run_pixelrag_vlm_replay_worker.py --execute` 若 route readiness 已輸出 `candidate_host`，tile VLM generate 必須使用該 approved host 與 exact `candidate_model` 直呼 `/api/generate`，不得重新進入全域 Ollama resolver、不得因 111 fallback 規則把 VLM 模型降級成文字模型、不得再出現 preflight 選 111 但 tile generate tried GCP-B/GCP-A/111 的 route drift。receipt 必須輸出 `route_candidate_host`，模型錯誤時下一步為 `repair_ollama_vlm_generate_runtime_or_proxy_timeout`，仍然不寫 DB、不寫正式價格表。 | +| 2026-07-10 | PixelRAG VLM no-probe 監控不得把 unknown 顯示成 failed | V10.755 起 smoke / scheduled-health 的 `pixelrag_vlm_route_readiness.details.generate_route_ready` 在 `generate_probe_performed=false` 時必須保留為 `null`，表示尚未做 `/api/generate` preflight；不得把未探測狀態轉成 `false`，避免 UI 或監控把 tags-only readiness 誤讀為 generate failure。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index daa4980..a8a58d8 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -4677,8 +4677,10 @@ def build_scheduled_automation_health_summary( "generate_probe_ok_count": int( pixelrag_vlm_route_readiness_details.get("generate_probe_ok_count") or 0 ), - "generate_route_ready": bool( + "generate_route_ready": ( pixelrag_vlm_route_readiness_details.get("generate_route_ready") + if pixelrag_vlm_route_readiness_details.get("generate_probe_performed") + else None ), "generate_ready_model": pixelrag_vlm_route_readiness_details.get("generate_ready_model"), "generate_ready_host": pixelrag_vlm_route_readiness_details.get("generate_ready_host"), @@ -13543,7 +13545,11 @@ def _pixelrag_vlm_route_readiness_check() -> Dict[str, Any]: "model_route_ready": bool(summary.get("model_route_ready")), "generate_probe_performed": bool(summary.get("generate_probe_performed")), "generate_probe_ok_count": int(summary.get("generate_probe_ok_count") or 0), - "generate_route_ready": bool(summary.get("generate_route_ready")), + "generate_route_ready": ( + summary.get("generate_route_ready") + if summary.get("generate_probe_performed") + else None + ), "generate_ready_model": summary.get("generate_ready_model"), "generate_ready_host": summary.get("generate_ready_host"), "generate_ready_provider": summary.get("generate_ready_provider"), From 9c7eb6f89523f03a6182df62560476872ec3bfd7 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 00:41:39 +0800 Subject: [PATCH 14/75] fix(ai): classify empty PixelRAG VLM offers as platform barriers --- TODO_NEXT_STEPS.txt | 2 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 1 + .../pixelrag_vlm_replay_worker_service.py | 18 ++++- ...test_pixelrag_vlm_replay_worker_service.py | 67 +++++++++++++++++++ 5 files changed, 87 insertions(+), 3 deletions(-) diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 2cd9e3c..23f4408 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -9,7 +9,7 @@ python scripts/ops/check_production_version_truth.py 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 -本輪 source target 為 `V10.755`；部署完成前不得宣稱正式環境已是 `V10.755`。 +本輪 source target 為 `V10.756`；部署完成前不得宣稱正式環境已是 `V10.756`。 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ diff --git a/config.py b/config.py index 792a9f9..a7cd203 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.755" +SYSTEM_VERSION = "V10.756" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 87ff2bb..47c0afa 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -885,6 +885,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG VLM execute preflight 必須驗證 generate route | V10.753 起 `/api/ai-automation/pixelrag-vlm-route-readiness?probe_generate=true` 與 `scripts/ops/report_pixelrag_vlm_route_readiness.py --probe-generate` 可對已安裝候選模型執行極小 `/api/generate` preflight；smoke 預設仍不呼叫模型。`run_pixelrag_vlm_replay_worker.py --execute` 預設先執行 generate preflight，若 GCP-A direct / 110 proxy timeout、GCP-B candidate 雖安裝但 generate 不健康，worker 必須在送入 screenshot tiles 前寫 `model_route_not_ready` artifact receipt，輸出 `tag_model_route_ready`、`generate_route_ready`、`route_model_call_performed` 與 `tile_model_call_performed=false`，下一步固定為 `repair_ollama_vlm_generate_runtime_or_proxy_timeout`；不得把 tags 可見誤當 VLM 可執行。 | | 2026-07-10 | PixelRAG VLM tile generate 必須綁定 preflight 選出的 host / exact model | V10.754 起 `run_pixelrag_vlm_replay_worker.py --execute` 若 route readiness 已輸出 `candidate_host`，tile VLM generate 必須使用該 approved host 與 exact `candidate_model` 直呼 `/api/generate`，不得重新進入全域 Ollama resolver、不得因 111 fallback 規則把 VLM 模型降級成文字模型、不得再出現 preflight 選 111 但 tile generate tried GCP-B/GCP-A/111 的 route drift。receipt 必須輸出 `route_candidate_host`，模型錯誤時下一步為 `repair_ollama_vlm_generate_runtime_or_proxy_timeout`，仍然不寫 DB、不寫正式價格表。 | | 2026-07-10 | PixelRAG VLM no-probe 監控不得把 unknown 顯示成 failed | V10.755 起 smoke / scheduled-health 的 `pixelrag_vlm_route_readiness.details.generate_route_ready` 在 `generate_probe_performed=false` 時必須保留為 `null`，表示尚未做 `/api/generate` preflight；不得把未探測狀態轉成 `false`，避免 UI 或監控把 tags-only readiness 誤讀為 generate failure。 | +| 2026-07-10 | PixelRAG VLM 空欄位與 Shopee language interstitial 必須自動轉 platform probe | V10.756 起 VLM prompt 明確把 language / region / app-download / landing / loading / logo-only / cookie consent pages 視為非商品卡；若 VLM 回傳所有 offer fields 皆空且 required title / price 缺失，worker 會標記 `non_product_or_interstitial_detected=true`，receipt 下一步改為 `run_platform_probe_or_use_structured_api`，不得反覆要求人工審核或無限 rerun VLM。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/services/pixelrag_vlm_replay_worker_service.py b/services/pixelrag_vlm_replay_worker_service.py index d8d2306..fceb7b3 100644 --- a/services/pixelrag_vlm_replay_worker_service.py +++ b/services/pixelrag_vlm_replay_worker_service.py @@ -153,6 +153,8 @@ def _prompt_for_item(item: Mapping[str, Any]) -> str: "Use only visible tile evidence and cite evidence_refs like tile:1.\n" "If the tile is access denied, captcha, login, traffic verification, or not a product/search card, " "set blocked_page_detected=true and leave product fields empty.\n" + "Language selection, region selection, app-download, landing, loading, logo-only, or cookie consent " + "pages are not product/search cards; set blocked_page_detected=true for them.\n" "Required JSON schema:\n" "{\n" ' "blocked_page_detected": false,\n' @@ -183,6 +185,7 @@ def _validate_model_payload( missing_required: list[str] = [] field_evidence_missing: list[str] = [] low_confidence_fields: list[str] = [] + present_field_count = 0 blocked_detected = bool(parsed.get("blocked_page_detected")) for contract in list(item.get("field_contract") or []): @@ -198,6 +201,8 @@ def _validate_model_payload( confidence = 0.0 min_confidence = float(contract.get("min_confidence") or DEFAULT_CONFIDENCE_THRESHOLD) present = _field_value_present(value) + if present: + present_field_count += 1 if present and not evidence_refs: field_evidence_missing.append(field_name) if present and confidence < min_confidence: @@ -213,14 +218,22 @@ def _validate_model_payload( for field in declared_missing: if field not in missing_required: missing_required.append(field) + non_product_or_interstitial_detected = ( + not blocked_detected + and present_field_count == 0 + and bool(missing_required) + ) return { "blocked_page_detected": blocked_detected, + "non_product_or_interstitial_detected": non_product_or_interstitial_detected, + "present_field_count": present_field_count, "missing_required_fields": missing_required, "field_evidence_missing": field_evidence_missing, "low_confidence_fields": low_confidence_fields, "valid_for_identity_matcher_replay": ( not blocked_detected + and not non_product_or_interstitial_detected and not missing_required and not field_evidence_missing ), @@ -501,9 +514,12 @@ def _execute_item( missing_required = list(validation.get("missing_required_fields") or []) evidence_missing = list(validation.get("field_evidence_missing") or []) blocked_detected = bool(validation.get("blocked_page_detected")) + non_product_or_interstitial = bool( + validation.get("non_product_or_interstitial_detected") + ) status = "executed_ok" next_action = "run_identity_matcher_replay_then_promotion_gate" - if blocked_detected: + if blocked_detected or non_product_or_interstitial: status = "executed_warning" next_action = "run_platform_probe_or_use_structured_api" elif missing_required or evidence_missing: diff --git a/tests/test_pixelrag_vlm_replay_worker_service.py b/tests/test_pixelrag_vlm_replay_worker_service.py index b0179bc..64a2abc 100644 --- a/tests/test_pixelrag_vlm_replay_worker_service.py +++ b/tests/test_pixelrag_vlm_replay_worker_service.py @@ -177,6 +177,73 @@ def test_pixelrag_vlm_replay_worker_writes_model_error_receipt(tmp_path, monkeyp assert receipt["next_machine_action"] == "repair_ollama_vlm_runtime_or_model_route" +def test_pixelrag_vlm_replay_worker_empty_fields_routes_to_platform_probe(tmp_path, monkeypatch): + from services import pixelrag_vlm_replay_worker_service as service + + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-language", + title="蝦皮購物 | 花得更少買得更好", + url="https://shopee.tw/search?keyword=sunscreen", + ) + + class FakeOllama: + def __init__(self, model): + self.model = model + + def generate(self, *args, **kwargs): + return SimpleNamespace( + success=True, + content=json.dumps({ + "blocked_page_detected": False, + "fields": { + "title": {"value": None, "confidence": 0.0, "evidence_refs": []}, + "price": {"value": None, "confidence": 0.0, "evidence_refs": []}, + "currency": {"value": None, "confidence": 0.0, "evidence_refs": []}, + }, + "quality": { + "overall_confidence": 0.0, + "missing_required_fields": [], + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + }, + "notes": [], + }), + model="minicpm-v:latest", + error=None, + total_duration=2.0, + host="http://192.168.0.111:11434", + input_tokens=10, + output_tokens=40, + ) + + monkeypatch.setattr(service, "OllamaService", FakeOllama) + payload = service.run_pixelrag_ollama_vlm_replay_worker( + artifact_root=tmp_path, + output_root=tmp_path / "receipts", + platform="shopee_tw", + execute=True, + write_receipt=True, + tile_limit=4, + auto_select_model=False, + ) + + assert payload["status"] == "warning" + assert payload["summary"]["executed_warning_count"] == 1 + assert payload["summary"]["required_field_missing_count"] == 2 + assert payload["next_machine_action"] == "rerun_vlm_replay_with_more_tiles_or_platform_probe" + item = payload["worker_items"][0] + assert item["worker_status"] == "executed_warning" + assert item["next_machine_action"] == "run_platform_probe_or_use_structured_api" + assert item["validation"]["present_field_count"] == 0 + assert item["validation"]["non_product_or_interstitial_detected"] is True + assert item["validation"]["valid_for_identity_matcher_replay"] is False + receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-language" / "vlm_replay_receipt.json" + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["next_machine_action"] == "run_platform_probe_or_use_structured_api" + + def test_pixelrag_vlm_replay_worker_auto_selects_installed_candidate(tmp_path, monkeypatch): from services import pixelrag_vlm_replay_worker_service as service From c4917239613f4407c6df2fdab2b1c5e1613969f1 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 00:45:56 +0800 Subject: [PATCH 15/75] fix(ai): tune PixelRAG VLM preflight defaults --- TODO_NEXT_STEPS.txt | 2 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 1 + routes/system_public_routes.py | 4 ++-- scripts/ops/run_pixelrag_vlm_replay_worker.py | 2 +- services/pixelrag_vlm_replay_worker_service.py | 12 ++++++++++-- tests/test_pixelrag_vlm_replay_worker_service.py | 2 +- 7 files changed, 17 insertions(+), 8 deletions(-) diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 23f4408..d71274f 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -9,7 +9,7 @@ python scripts/ops/check_production_version_truth.py 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 -本輪 source target 為 `V10.756`；部署完成前不得宣稱正式環境已是 `V10.756`。 +本輪 source target 為 `V10.757`；部署完成前不得宣稱正式環境已是 `V10.757`。 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ diff --git a/config.py b/config.py index a7cd203..cf4f69c 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.756" +SYSTEM_VERSION = "V10.757" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 47c0afa..b3b3c3f 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -886,6 +886,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG VLM tile generate 必須綁定 preflight 選出的 host / exact model | V10.754 起 `run_pixelrag_vlm_replay_worker.py --execute` 若 route readiness 已輸出 `candidate_host`，tile VLM generate 必須使用該 approved host 與 exact `candidate_model` 直呼 `/api/generate`，不得重新進入全域 Ollama resolver、不得因 111 fallback 規則把 VLM 模型降級成文字模型、不得再出現 preflight 選 111 但 tile generate tried GCP-B/GCP-A/111 的 route drift。receipt 必須輸出 `route_candidate_host`，模型錯誤時下一步為 `repair_ollama_vlm_generate_runtime_or_proxy_timeout`，仍然不寫 DB、不寫正式價格表。 | | 2026-07-10 | PixelRAG VLM no-probe 監控不得把 unknown 顯示成 failed | V10.755 起 smoke / scheduled-health 的 `pixelrag_vlm_route_readiness.details.generate_route_ready` 在 `generate_probe_performed=false` 時必須保留為 `null`，表示尚未做 `/api/generate` preflight；不得把未探測狀態轉成 `false`，避免 UI 或監控把 tags-only readiness 誤讀為 generate failure。 | | 2026-07-10 | PixelRAG VLM 空欄位與 Shopee language interstitial 必須自動轉 platform probe | V10.756 起 VLM prompt 明確把 language / region / app-download / landing / loading / logo-only / cookie consent pages 視為非商品卡；若 VLM 回傳所有 offer fields 皆空且 required title / price 缺失，worker 會標記 `non_product_or_interstitial_detected=true`，receipt 下一步改為 `run_platform_probe_or_use_structured_api`，不得反覆要求人工審核或無限 rerun VLM。 | +| 2026-07-10 | PixelRAG VLM worker preflight 必須容納 111 VLM 冷啟延遲 | V10.757 起 `run_pixelrag_vlm_replay_worker.py --execute` 與對應 API 的 `route_generate_probe_timeout` 預設為 20 秒；111 `minicpm-v` 在正式環境可於約 4 到 20 秒間完成小型 `/api/generate` probe，8 秒容易造成假陰性。若 VLM receipt 已判定 `run_platform_probe_or_use_structured_api`，run-level `next_machine_action` 也必須同樣輸出 platform probe，不得停留在 rerun VLM 摘要。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 1a90750..e5d0177 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -794,7 +794,7 @@ def ai_automation_pixelrag_vlm_replay_worker_api(): } ) route_readiness_timeout = request.args.get('route_readiness_timeout', 3, type=int) - route_generate_probe_timeout = request.args.get('route_generate_probe_timeout', 8, type=int) + route_generate_probe_timeout = request.args.get('route_generate_probe_timeout', 20, type=int) return jsonify(run_pixelrag_ollama_vlm_replay_worker( platform=platforms, max_age_hours=max(1, min(max_age_hours or 168, 720)), @@ -807,7 +807,7 @@ def ai_automation_pixelrag_vlm_replay_worker_api(): auto_select_model=auto_select_model, route_readiness_timeout=max(1, min(route_readiness_timeout or 3, 20)), probe_generate_before_execute=probe_generate_before_execute, - route_generate_probe_timeout=max(1, min(route_generate_probe_timeout or 8, 30)), + route_generate_probe_timeout=max(1, min(route_generate_probe_timeout or 20, 30)), )) diff --git a/scripts/ops/run_pixelrag_vlm_replay_worker.py b/scripts/ops/run_pixelrag_vlm_replay_worker.py index a412bd5..a704896 100755 --- a/scripts/ops/run_pixelrag_vlm_replay_worker.py +++ b/scripts/ops/run_pixelrag_vlm_replay_worker.py @@ -95,7 +95,7 @@ def main() -> int: parser.add_argument( "--route-generate-probe-timeout", type=int, - default=8, + default=20, help="execute 前 /api/generate preflight timeout 秒數。", ) args = parser.parse_args() diff --git a/services/pixelrag_vlm_replay_worker_service.py b/services/pixelrag_vlm_replay_worker_service.py index fceb7b3..9bf0d67 100644 --- a/services/pixelrag_vlm_replay_worker_service.py +++ b/services/pixelrag_vlm_replay_worker_service.py @@ -42,7 +42,7 @@ POLICY = "controlled_pixelrag_ollama_vlm_replay_worker_v1" DEFAULT_LIMIT = 25 DEFAULT_TILE_LIMIT = 4 DEFAULT_TIMEOUT_SECONDS = 90 -DEFAULT_ROUTE_GENERATE_PROBE_TIMEOUT_SECONDS = 8 +DEFAULT_ROUTE_GENERATE_PROBE_TIMEOUT_SECONDS = 20 DEFAULT_OUTPUT_ROOT = os.getenv( "PIXELRAG_VLM_REPLAY_RECEIPT_ROOT", "/app/data/ai_automation/pixelrag_vlm_replay_receipts" @@ -684,7 +684,15 @@ def run_pixelrag_ollama_vlm_replay_worker( elif model_error_count or parse_error_count: next_action = "repair_ollama_vlm_runtime_or_model_route" elif executed_warning_count: - next_action = "rerun_vlm_replay_with_more_tiles_or_platform_probe" + warning_actions = { + str(item.get("next_machine_action") or "") + for item in worker_items + if item.get("worker_status") == "executed_warning" + } + if warning_actions == {"run_platform_probe_or_use_structured_api"}: + next_action = "run_platform_probe_or_use_structured_api" + else: + next_action = "rerun_vlm_replay_with_more_tiles_or_platform_probe" elif executed_ok_count: next_action = "run_identity_matcher_replay_then_promotion_gate" else: diff --git a/tests/test_pixelrag_vlm_replay_worker_service.py b/tests/test_pixelrag_vlm_replay_worker_service.py index 64a2abc..d428c65 100644 --- a/tests/test_pixelrag_vlm_replay_worker_service.py +++ b/tests/test_pixelrag_vlm_replay_worker_service.py @@ -232,7 +232,7 @@ def test_pixelrag_vlm_replay_worker_empty_fields_routes_to_platform_probe(tmp_pa assert payload["status"] == "warning" assert payload["summary"]["executed_warning_count"] == 1 assert payload["summary"]["required_field_missing_count"] == 2 - assert payload["next_machine_action"] == "rerun_vlm_replay_with_more_tiles_or_platform_probe" + assert payload["next_machine_action"] == "run_platform_probe_or_use_structured_api" item = payload["worker_items"][0] assert item["worker_status"] == "executed_warning" assert item["next_machine_action"] == "run_platform_probe_or_use_structured_api" From 83f6d2e255e807d7618830fd05b1a630e0ef3575 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 00:49:54 +0800 Subject: [PATCH 16/75] fix(ai): detect PixelRAG marketplace interstitials --- TODO_NEXT_STEPS.txt | 2 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 1 + .../pixelrag_vlm_replay_worker_service.py | 57 +++++++++++++++- ...test_pixelrag_vlm_replay_worker_service.py | 65 +++++++++++++++++++ 5 files changed, 124 insertions(+), 3 deletions(-) diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index d71274f..a9c14dd 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -9,7 +9,7 @@ python scripts/ops/check_production_version_truth.py 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 -本輪 source target 為 `V10.757`；部署完成前不得宣稱正式環境已是 `V10.757`。 +本輪 source target 為 `V10.758`；部署完成前不得宣稱正式環境已是 `V10.758`。 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ diff --git a/config.py b/config.py index cf4f69c..e474d56 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.757" +SYSTEM_VERSION = "V10.758" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index b3b3c3f..22f0c97 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -887,6 +887,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG VLM no-probe 監控不得把 unknown 顯示成 failed | V10.755 起 smoke / scheduled-health 的 `pixelrag_vlm_route_readiness.details.generate_route_ready` 在 `generate_probe_performed=false` 時必須保留為 `null`，表示尚未做 `/api/generate` preflight；不得把未探測狀態轉成 `false`，避免 UI 或監控把 tags-only readiness 誤讀為 generate failure。 | | 2026-07-10 | PixelRAG VLM 空欄位與 Shopee language interstitial 必須自動轉 platform probe | V10.756 起 VLM prompt 明確把 language / region / app-download / landing / loading / logo-only / cookie consent pages 視為非商品卡；若 VLM 回傳所有 offer fields 皆空且 required title / price 缺失，worker 會標記 `non_product_or_interstitial_detected=true`，receipt 下一步改為 `run_platform_probe_or_use_structured_api`，不得反覆要求人工審核或無限 rerun VLM。 | | 2026-07-10 | PixelRAG VLM worker preflight 必須容納 111 VLM 冷啟延遲 | V10.757 起 `run_pixelrag_vlm_replay_worker.py --execute` 與對應 API 的 `route_generate_probe_timeout` 預設為 20 秒；111 `minicpm-v` 在正式環境可於約 4 到 20 秒間完成小型 `/api/generate` probe，8 秒容易造成假陰性。若 VLM receipt 已判定 `run_platform_probe_or_use_structured_api`，run-level `next_machine_action` 也必須同樣輸出 platform probe，不得停留在 rerun VLM 摘要。 | +| 2026-07-10 | PixelRAG VLM language note / generic marketplace slogan 必須視為 interstitial | V10.758 起若 VLM notes 或 title 顯示 `Language selection page`、語言/地區選擇，或只抽到 Shopee `蝦皮購物 | 花得更少買得更好` 這類網站 slogan 且 price/shop 缺失，worker validation 會標記 `interstitial_signal_detected` / `generic_marketplace_title_detected` 與 `non_product_or_interstitial_detected=true`，run-level 與 item-level 下一步都輸出 `run_platform_probe_or_use_structured_api`。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/services/pixelrag_vlm_replay_worker_service.py b/services/pixelrag_vlm_replay_worker_service.py index 9bf0d67..32ce7da 100644 --- a/services/pixelrag_vlm_replay_worker_service.py +++ b/services/pixelrag_vlm_replay_worker_service.py @@ -55,6 +55,25 @@ DEFAULT_MODEL = ( or "minicpm-v:latest" ) RAW_EXCERPT_LIMIT = 500 +INTERSTITIAL_SIGNAL_TOKENS = ( + "language selection", + "select language", + "choose language", + "region selection", + "select region", + "app-download", + "app download", + "landing page", + "loading page", + "logo-only", + "cookie consent", + "選擇語言", + "選擇地區", + "語言", +) +GENERIC_MARKETPLACE_TITLE_TOKENS = ( + "蝦皮購物 | 花得更少買得更好", +) def _normalise_platforms( @@ -176,6 +195,23 @@ def _field_value_present(value: Any) -> bool: return True +def _stringify_signal(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, Mapping): + return " ".join(_stringify_signal(item) for item in value.values()) + if isinstance(value, list): + return " ".join(_stringify_signal(item) for item in value) + return str(value) + + +def _has_interstitial_signal(*values: Any) -> bool: + haystack = " ".join(_stringify_signal(value) for value in values).lower() + return any(token.lower() in haystack for token in INTERSTITIAL_SIGNAL_TOKENS) + + def _validate_model_payload( parsed: Mapping[str, Any], item: Mapping[str, Any], @@ -187,6 +223,7 @@ def _validate_model_payload( low_confidence_fields: list[str] = [] present_field_count = 0 blocked_detected = bool(parsed.get("blocked_page_detected")) + title_value = None for contract in list(item.get("field_contract") or []): field_name = str(contract.get("field") or "") @@ -194,6 +231,8 @@ def _validate_model_payload( if not isinstance(field_payload, Mapping): field_payload = {} value = field_payload.get("value") + if field_name == "title": + title_value = value evidence_refs = list(field_payload.get("evidence_refs") or []) try: confidence = float(field_payload.get("confidence") or 0) @@ -218,15 +257,31 @@ def _validate_model_payload( for field in declared_missing: if field not in missing_required: missing_required.append(field) + notes_payload = parsed.get("notes") + generic_marketplace_title_detected = ( + isinstance(title_value, str) + and any(token in title_value for token in GENERIC_MARKETPLACE_TITLE_TOKENS) + ) + interstitial_signal_detected = _has_interstitial_signal( + notes_payload, + title_value, + item.get("title_hint"), + ) non_product_or_interstitial_detected = ( not blocked_detected - and present_field_count == 0 + and ( + present_field_count == 0 + or interstitial_signal_detected + or generic_marketplace_title_detected + ) and bool(missing_required) ) return { "blocked_page_detected": blocked_detected, "non_product_or_interstitial_detected": non_product_or_interstitial_detected, + "interstitial_signal_detected": interstitial_signal_detected, + "generic_marketplace_title_detected": generic_marketplace_title_detected, "present_field_count": present_field_count, "missing_required_fields": missing_required, "field_evidence_missing": field_evidence_missing, diff --git a/tests/test_pixelrag_vlm_replay_worker_service.py b/tests/test_pixelrag_vlm_replay_worker_service.py index d428c65..9a37344 100644 --- a/tests/test_pixelrag_vlm_replay_worker_service.py +++ b/tests/test_pixelrag_vlm_replay_worker_service.py @@ -244,6 +244,71 @@ def test_pixelrag_vlm_replay_worker_empty_fields_routes_to_platform_probe(tmp_pa assert receipt["next_machine_action"] == "run_platform_probe_or_use_structured_api" +def test_pixelrag_vlm_replay_worker_language_note_routes_to_platform_probe(tmp_path, monkeypatch): + from services import pixelrag_vlm_replay_worker_service as service + + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-language-note", + title="蝦皮購物 | 花得更少買得更好", + url="https://shopee.tw/search?keyword=sunscreen", + ) + + class FakeOllama: + def __init__(self, model): + self.model = model + + def generate(self, *args, **kwargs): + return SimpleNamespace( + success=True, + content=json.dumps({ + "blocked_page_detected": False, + "fields": { + "title": { + "value": "蝦皮購物 | 花得更少買得更好", + "confidence": 0.95, + "evidence_refs": ["tile:1"], + }, + "price": {"value": None, "confidence": 0.0, "evidence_refs": []}, + "shop": {"value": None, "confidence": 0.0, "evidence_refs": []}, + }, + "quality": { + "overall_confidence": 0.5, + "missing_required_fields": ["title", "price", "shop"], + "requires_identity_matcher_replay": True, + "requires_promotion_gate": False, + }, + "notes": [{"note": "Language selection page"}], + }), + model="minicpm-v:latest", + error=None, + total_duration=2.0, + host="http://192.168.0.111:11434", + input_tokens=10, + output_tokens=40, + ) + + monkeypatch.setattr(service, "OllamaService", FakeOllama) + payload = service.run_pixelrag_ollama_vlm_replay_worker( + artifact_root=tmp_path, + output_root=tmp_path / "receipts", + platform="shopee_tw", + execute=True, + write_receipt=True, + tile_limit=4, + auto_select_model=False, + ) + + assert payload["next_machine_action"] == "run_platform_probe_or_use_structured_api" + item = payload["worker_items"][0] + assert item["next_machine_action"] == "run_platform_probe_or_use_structured_api" + assert item["validation"]["present_field_count"] == 1 + assert item["validation"]["interstitial_signal_detected"] is True + assert item["validation"]["generic_marketplace_title_detected"] is True + assert item["validation"]["non_product_or_interstitial_detected"] is True + + def test_pixelrag_vlm_replay_worker_auto_selects_installed_candidate(tmp_path, monkeypatch): from services import pixelrag_vlm_replay_worker_service as service From 199c857353fd859aebe4338380381a8a0c507343 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 01:21:39 +0800 Subject: [PATCH 17/75] feat(ai): add PixelRAG platform probe readiness --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 2 + routes/system_public_routes.py | 22 + .../ops/capture_pixelrag_visual_evidence.js | 39 +- .../ops/capture_pixelrag_visual_evidence.py | 46 +- scripts/ops/report_pixelrag_platform_probe.py | 65 ++ services/ai_automation_smoke_service.py | 102 ++++ services/pixelrag_platform_probe_service.py | 565 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 40 +- tests/test_pixelrag_platform_probe_service.py | 240 ++++++++ 10 files changed, 1113 insertions(+), 10 deletions(-) create mode 100644 scripts/ops/report_pixelrag_platform_probe.py create mode 100644 services/pixelrag_platform_probe_service.py create mode 100644 tests/test_pixelrag_platform_probe_service.py diff --git a/config.py b/config.py index e474d56..7e287c8 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.758" +SYSTEM_VERSION = "V10.759" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 22f0c97..d239c2c 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -118,6 +118,7 @@ - 2026-07-09 起 PixelRAG 可整合/可運用範圍必須有機器可讀 application portfolio：`/api/ai-automation/pixelrag-application-portfolio` 與 `scripts/ops/report_pixelrag_application_portfolio.py` 需輸出 commerce、RAG、UX、ops、marketing、governance lanes，每條 lane 必須包含 priority、status、integrates_with、use_cases、current_capability、next_machine_action、no-write 邊界與 forbidden guardrails。此 portfolio 不抓外站、不呼叫模型、不讀 secret、不寫 DB；其用途是把「還可以整合哪些」變成可排程、可驗證、可拒絕違規場景的主線工作項目。 - 2026-07-09 起 PixelRAG ready receipts 必須有 Ollama-first VLM replay worker：`/api/ai-automation/pixelrag-vlm-replay-worker` 與 `scripts/ops/run_pixelrag_vlm_replay_worker.py` 預設 dry-run，`execute=true&write_receipt=true` 才呼叫 approved Ollama VLM route 並寫 artifact receipt；`/api/ai-automation/smoke` 需包含 `PixelRAG VLM replay worker`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_vlm_replay_worker` family。此 worker 不讀 secret、不寫 DB、不寫 `ai_insights`、不寫正式價格表；blocked / 403 / captcha / access denied receipt 自動跳 platform probe 或 structured API，ready receipt 的 VLM 結果仍須 identity matcher replay 與 PromotionGate 才能進候選知識層。 - 2026-07-09 起 PixelRAG VLM route readiness 必須在 execute 前可讀回：`/api/ai-automation/pixelrag-vlm-route-readiness` 與 `scripts/ops/report_pixelrag_vlm_route_readiness.py` 只讀 approved Ollama `/api/tags`，輸出 configured model、candidate model、reachable host、model_route_ready 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG VLM route readiness`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_vlm_route_readiness` family。此 readback 不呼叫 `/api/generate`、不讀 secret、不寫 DB；worker execute 必須使用它自動避開缺模型路由，完全缺候選時寫 `model_route_not_ready` artifact receipt。 +- 2026-07-10 起 PixelRAG platform probe 必須把 blocked / interstitial receipt 轉成機器可執行下一步：`/api/ai-automation/pixelrag-platform-probe` 與 `scripts/ops/report_pixelrag_platform_probe.py` 需讀取 capture receipts 與 VLM replay receipts，辨識 Shopee language / region / generic marketplace slogan / traffic verification 與 Coupang 403 / access denied，輸出 public browser context probe、structured source fallback、platform backoff、blocked page not product data 與 no-write 邊界；`/api/ai-automation/smoke` 需包含 `PixelRAG platform probe readiness`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_platform_probe` family。此 readback 不抓外站、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture worker 可消化 `public_browser_context` 的 locale/timezone/Accept-Language，但必須濾掉 Cookie、Authorization、token/secret/key 類 header。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -888,6 +889,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG VLM 空欄位與 Shopee language interstitial 必須自動轉 platform probe | V10.756 起 VLM prompt 明確把 language / region / app-download / landing / loading / logo-only / cookie consent pages 視為非商品卡；若 VLM 回傳所有 offer fields 皆空且 required title / price 缺失，worker 會標記 `non_product_or_interstitial_detected=true`，receipt 下一步改為 `run_platform_probe_or_use_structured_api`，不得反覆要求人工審核或無限 rerun VLM。 | | 2026-07-10 | PixelRAG VLM worker preflight 必須容納 111 VLM 冷啟延遲 | V10.757 起 `run_pixelrag_vlm_replay_worker.py --execute` 與對應 API 的 `route_generate_probe_timeout` 預設為 20 秒；111 `minicpm-v` 在正式環境可於約 4 到 20 秒間完成小型 `/api/generate` probe，8 秒容易造成假陰性。若 VLM receipt 已判定 `run_platform_probe_or_use_structured_api`，run-level `next_machine_action` 也必須同樣輸出 platform probe，不得停留在 rerun VLM 摘要。 | | 2026-07-10 | PixelRAG VLM language note / generic marketplace slogan 必須視為 interstitial | V10.758 起若 VLM notes 或 title 顯示 `Language selection page`、語言/地區選擇，或只抽到 Shopee `蝦皮購物 | 花得更少買得更好` 這類網站 slogan 且 price/shop 缺失，worker validation 會標記 `interstitial_signal_detected` / `generic_marketplace_title_detected` 與 `non_product_or_interstitial_detected=true`，run-level 與 item-level 下一步都輸出 `run_platform_probe_or_use_structured_api`。 | +| 2026-07-10 | PixelRAG platform probe 必須自動消化 marketplace interstitial / blocked receipt | V10.759 起 `/api/ai-automation/pixelrag-platform-probe`、`scripts/ops/report_pixelrag_platform_probe.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 platform probe / `pixelrag_platform_probe` family；Shopee traffic / language / generic landing 會轉成 `run_shopee_public_context_probe_then_structured_source_fallback`，Coupang 403 / access denied 會轉成 `use_structured_source_or_platform_backoff_policy`。probe plan 只產生 public empty browser context、structured adapter fallback 與 no-write contract，不讀或注入 cookie/session/login，不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture worker 只允許 locale/timezone/非敏感 header 並濾掉 Cookie / Authorization / token / secret / key 類 header。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index e5d0177..86b3259 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -840,6 +840,28 @@ def ai_automation_pixelrag_vlm_route_readiness_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-platform-probe') +@login_required +def ai_automation_pixelrag_platform_probe_api(): + """Read-only PixelRAG platform probe readiness from barrier receipts.""" + from services.pixelrag_platform_probe_service import ( + build_pixelrag_platform_probe_readiness, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 50, type=int) + return jsonify(build_pixelrag_platform_probe_readiness( + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 50, 250)), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/capture_pixelrag_visual_evidence.js b/scripts/ops/capture_pixelrag_visual_evidence.js index beea5ae..d7953e7 100644 --- a/scripts/ops/capture_pixelrag_visual_evidence.js +++ b/scripts/ops/capture_pixelrag_visual_evidence.js @@ -17,6 +17,7 @@ const DEFAULT_OUTPUT_DIR = 'runtime_artifacts/pixelrag_visual_evidence'; const DEFAULT_TIMEOUT_MS = 30000; const DEFAULT_SETTLE_MS = 500; const DEFAULT_MAX_TILES = 80; +const FORBIDDEN_CONTEXT_HEADER_TOKENS = ['cookie', 'authorization', 'token', 'secret', 'key']; const ALLOWED_PLATFORM_HOSTS = { momo: new Set(['m.momoshop.com.tw', 'www.momoshop.com.tw']), @@ -184,6 +185,30 @@ function positiveInt(value, fallback) { return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } +function safePublicBrowserContext(manifest) { + const input = manifest.public_browser_context || {}; + const headers = {}; + const rawHeaders = input.extra_http_headers || {}; + for (const [key, value] of Object.entries(rawHeaders)) { + const cleanKey = String(key || '').trim(); + const lowerKey = cleanKey.toLowerCase(); + if (!cleanKey || FORBIDDEN_CONTEXT_HEADER_TOKENS.some((token) => lowerKey.includes(token))) { + continue; + } + headers[cleanKey] = String(value || ''); + } + return { + context_policy: 'public_empty_browser_context_no_login', + locale: String(input.locale || 'zh-TW'), + timezone_id: String(input.timezone_id || input.timezoneId || 'Asia/Taipei'), + extra_http_headers: headers, + credentialed_session_allowed: false, + storage_state_allowed: false, + raw_cookie_or_session_read_allowed: false, + login_allowed: false, + }; +} + function buildTilePlan({ width, height, tileWidth, tileHeight, maxTiles }) { const tiles = []; const tilesX = Math.max(1, Math.ceil(width / tileWidth)); @@ -273,6 +298,7 @@ function buildBaseArtifact(manifest, options, errors = []) { tile_size: { width: tileWidth, height: tileHeight }, tile_plan: tilePlan, files: [], + public_browser_context: safePublicBrowserContext(manifest), }; } @@ -288,8 +314,19 @@ async function capture(manifest, options, artifact) { launchOptions.executablePath = chromePath; } + const publicContext = safePublicBrowserContext(manifest); + const contextOptions = { + ignoreHTTPSErrors: true, + viewport, + locale: publicContext.locale, + timezoneId: publicContext.timezone_id, + }; + if (Object.keys(publicContext.extra_http_headers).length) { + contextOptions.extraHTTPHeaders = publicContext.extra_http_headers; + } + const browser = await chromium.launch(launchOptions); - const context = await browser.newContext({ ignoreHTTPSErrors: true, viewport }); + const context = await browser.newContext(contextOptions); const page = await context.newPage(); try { const response = await page.goto(manifest.capture_target.url, { diff --git a/scripts/ops/capture_pixelrag_visual_evidence.py b/scripts/ops/capture_pixelrag_visual_evidence.py index f9c2a1f..6ffba51 100644 --- a/scripts/ops/capture_pixelrag_visual_evidence.py +++ b/scripts/ops/capture_pixelrag_visual_evidence.py @@ -23,6 +23,7 @@ DEFAULT_OUTPUT_DIR = "runtime_artifacts/pixelrag_visual_evidence" DEFAULT_TIMEOUT_MS = 30000 DEFAULT_SETTLE_MS = 500 DEFAULT_MAX_TILES = 80 +FORBIDDEN_CONTEXT_HEADER_TOKENS = ("cookie", "authorization", "token", "secret", "key") ALLOWED_PLATFORM_HOSTS = { "momo": {"m.momoshop.com.tw", "www.momoshop.com.tw"}, "pchome": {"24h.pchome.com.tw", "ecshweb.pchome.com.tw", "ecapi-cdn.pchome.com.tw"}, @@ -123,6 +124,33 @@ def _build_tile_plan( } +def _safe_public_browser_context(manifest: dict[str, Any]) -> dict[str, Any]: + raw_context = manifest.get("public_browser_context") or {} + raw_headers = raw_context.get("extra_http_headers") or {} + headers: dict[str, str] = {} + if isinstance(raw_headers, dict): + for key, value in raw_headers.items(): + clean_key = str(key or "").strip() + lower_key = clean_key.lower() + if not clean_key: + continue + if any(token in lower_key for token in FORBIDDEN_CONTEXT_HEADER_TOKENS): + continue + headers[clean_key] = str(value or "") + return { + "context_policy": "public_empty_browser_context_no_login", + "locale": str(raw_context.get("locale") or "zh-TW"), + "timezone_id": str( + raw_context.get("timezone_id") or raw_context.get("timezoneId") or "Asia/Taipei" + ), + "extra_http_headers": headers, + "credentialed_session_allowed": False, + "storage_state_allowed": False, + "raw_cookie_or_session_read_allowed": False, + "login_allowed": False, + } + + def _output_paths(manifest: dict[str, Any], output_dir: str) -> dict[str, Path]: target = manifest.get("capture_target") or {} manifest_id = manifest.get("manifest_id") or _safe_name( @@ -187,6 +215,7 @@ def _base_artifact( max_tiles=_positive_int(args.max_tiles, DEFAULT_MAX_TILES), ), "files": [], + "public_browser_context": _safe_public_browser_context(manifest), } @@ -205,10 +234,19 @@ def _capture( with sync_playwright() as playwright: browser = playwright.chromium.launch(headless=True) - context = browser.new_context(ignore_https_errors=True, viewport={ - "width": viewport["width"], - "height": viewport["height"], - }) + public_context = _safe_public_browser_context(manifest) + context_kwargs = { + "ignore_https_errors": True, + "viewport": { + "width": viewport["width"], + "height": viewport["height"], + }, + "locale": public_context["locale"], + "timezone_id": public_context["timezone_id"], + } + if public_context["extra_http_headers"]: + context_kwargs["extra_http_headers"] = public_context["extra_http_headers"] + context = browser.new_context(**context_kwargs) page = context.new_page() try: response = page.goto(target["url"], wait_until=args.wait_until, timeout=timeout_ms) diff --git a/scripts/ops/report_pixelrag_platform_probe.py b/scripts/ops/report_pixelrag_platform_probe.py new file mode 100644 index 0000000..95e59d4 --- /dev/null +++ b/scripts/ops/report_pixelrag_platform_probe.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Report PixelRAG platform probe readiness from barrier receipts.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_platform_probe_service import ( # noqa: E402 + build_pixelrag_platform_probe_readiness, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="輸出 PixelRAG platform probe readiness 的機器可讀讀回。" + ) + parser.add_argument( + "--artifact-root", + help="PixelRAG visual evidence artifact root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--vlm-receipt-root", + help="PixelRAG VLM replay receipt root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=50, + help="最多輸出 probe candidate 數。", + ) + args = parser.parse_args() + + payload = build_pixelrag_platform_probe_readiness( + artifact_root=args.artifact_root, + vlm_receipt_root=args.vlm_receipt_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index a8a58d8..b0ee851 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -552,6 +552,11 @@ def build_scheduled_automation_health_summary( if not pixelrag_vlm_replay_worker or not pixelrag_vlm_replay_worker_details: pixelrag_vlm_replay_worker = _pixelrag_vlm_replay_worker_check() pixelrag_vlm_replay_worker_details = pixelrag_vlm_replay_worker.get("details") or {} + pixelrag_platform_probe = _find_check(source_result, "PixelRAG platform probe readiness") + pixelrag_platform_probe_details = pixelrag_platform_probe.get("details") or {} + if not pixelrag_platform_probe or not pixelrag_platform_probe_details: + pixelrag_platform_probe = _pixelrag_platform_probe_check() + pixelrag_platform_probe_details = pixelrag_platform_probe.get("details") or {} smoke_status = source_result.get("status") or ("warning" if latest_history else "warning") freshness_family = _history_freshness_family( latest_history, @@ -4732,6 +4737,44 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_platform_probe", + "label": "PixelRAG platform probe readiness", + "status": pixelrag_platform_probe.get("status") or "warning", + "summary": ( + pixelrag_platform_probe.get("summary") + or "PixelRAG platform probe readiness has no latest readback." + ), + "next_machine_action": pixelrag_platform_probe_details.get("next_machine_action") + or "run_pixelrag_platform_probe_readback", + "details": { + "policy": pixelrag_platform_probe_details.get("policy"), + "probe_candidate_count": int( + pixelrag_platform_probe_details.get("probe_candidate_count") or 0 + ), + "ready_for_probe_count": int( + pixelrag_platform_probe_details.get("ready_for_probe_count") or 0 + ), + "shopee_public_context_probe_count": int( + pixelrag_platform_probe_details.get("shopee_public_context_probe_count") or 0 + ), + "language_or_region_interstitial_count": int( + pixelrag_platform_probe_details.get("language_or_region_interstitial_count") or 0 + ), + "traffic_verification_count": int( + pixelrag_platform_probe_details.get("traffic_verification_count") or 0 + ), + "access_denied_count": int( + pixelrag_platform_probe_details.get("access_denied_count") or 0 + ), + "structured_source_fallback_count": int( + pixelrag_platform_probe_details.get("structured_source_fallback_count") or 0 + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, freshness_family, { "key": "daily_summary_delivery", @@ -13573,6 +13616,64 @@ def _pixelrag_vlm_route_readiness_check() -> Dict[str, Any]: ) +def _pixelrag_platform_probe_check() -> Dict[str, Any]: + """Read-only sentinel for PixelRAG platform probe automation readiness.""" + try: + from services.pixelrag_platform_probe_service import ( + build_pixelrag_platform_probe_readiness, + ) + + readback = build_pixelrag_platform_probe_readiness() + summary = readback.get("summary") or {} + candidate_count = int(summary.get("probe_candidate_count") or 0) + ready_count = int(summary.get("ready_for_probe_count") or 0) + shopee_context_count = int(summary.get("shopee_public_context_probe_count") or 0) + structured_count = int(summary.get("structured_source_fallback_count") or 0) + status = readback.get("status") or "warning" + summary_text = ( + f"PixelRAG platform probe candidates={candidate_count}, " + f"ready={ready_count}, shopee_context={shopee_context_count}, " + f"structured_fallback={structured_count}" + ) + return _check( + "PixelRAG platform probe readiness", + status, + summary_text, + { + "policy": readback.get("policy"), + "probe_candidate_count": candidate_count, + "ready_for_probe_count": ready_count, + "invalid_count": int(summary.get("invalid_count") or 0), + "capture_source_count": int(summary.get("capture_source_count") or 0), + "vlm_source_count": int(summary.get("vlm_source_count") or 0), + "shopee_public_context_probe_count": shopee_context_count, + "language_or_region_interstitial_count": int( + summary.get("language_or_region_interstitial_count") or 0 + ), + "traffic_verification_count": int( + summary.get("traffic_verification_count") or 0 + ), + "access_denied_count": int(summary.get("access_denied_count") or 0), + "structured_source_fallback_count": structured_count, + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG platform probe readiness", + "critical", + f"PixelRAG platform probe readiness 無法執行：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: int = 20) -> Dict[str, Any]: checks: List[Dict[str, Any]] = [ _event_router_check(), @@ -13611,6 +13712,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _pixelrag_ocr_vlm_replay_check(), _pixelrag_vlm_route_readiness_check(), _pixelrag_vlm_replay_worker_check(), + _pixelrag_platform_probe_check(), ] worst = max(checks, key=lambda item: STATUS_RANK.get(item["status"], 2))["status"] result = { diff --git a/services/pixelrag_platform_probe_service.py b/services/pixelrag_platform_probe_service.py new file mode 100644 index 0000000..772c2b6 --- /dev/null +++ b/services/pixelrag_platform_probe_service.py @@ -0,0 +1,565 @@ +"""Read-only PixelRAG platform probe readiness. + +This module turns PixelRAG capture/VLM barrier evidence into concrete machine +actions for public marketplace probing. It does not read sessions/cookies, +perform network calls, write DB rows, or promote price truth. +""" + +from __future__ import annotations + +import json +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping +from urllib.parse import parse_qs, unquote_plus, urlparse + +from services.market_intel.adapters.registry import get_adapter +from services.pixelrag_crawler_integration_service import ( + DEFAULT_ARTIFACT_MAX_AGE_HOURS, + DEFAULT_ARTIFACT_ROOT, + build_pixelrag_marketplace_search_manifest, + _parse_iso_datetime, +) +from services.pixelrag_rag_candidate_replay_service import ( + build_pixelrag_rag_candidate_replay_readback, +) + + +POLICY = "read_only_pixelrag_platform_probe_readiness_v1" +DEFAULT_LIMIT = 50 +DEFAULT_ACCEPT_LANGUAGE = "zh-TW,zh-Hant;q=0.95,zh;q=0.9,en;q=0.7" +DEFAULT_VLM_RECEIPT_ROOT = os.getenv( + "PIXELRAG_VLM_REPLAY_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_vlm_replay_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_vlm_replay_receipts", +) +PLATFORM_ADAPTER_ALIASES = { + "shopee_tw": "shopee", + "coupang_tw": "coupang", + "yahoo_shopping_tw": "yahoo", +} + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...]: + if isinstance(platform, str): + value = platform.strip().lower() + return (value,) if value else () + return tuple( + str(item or "").strip().lower() + for item in (platform or ()) + if str(item or "").strip() + ) + + +def _read_json(path: Path) -> tuple[dict[str, Any], str]: + try: + parsed = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return {}, str(exc)[:300] + return parsed if isinstance(parsed, dict) else {}, "" + + +def _safe_int(value: Any, default: int = 0) -> int: + try: + return int(value or default) + except (TypeError, ValueError): + return default + + +def _safe_text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, Mapping): + return " ".join(_safe_text(item) for item in value.values()) + if isinstance(value, list): + return " ".join(_safe_text(item) for item in value) + return str(value) + + +def _extract_keyword(url: str) -> str: + parsed = urlparse(str(url or "")) + query = parse_qs(parsed.query) + for key in ("keyword", "q", "query", "p", "search", "searchKeyword"): + values = query.get(key) or [] + for value in values: + text = unquote_plus(str(value or "")).strip() + if text: + return text + return "" + + +def _adapter_code(platform: str) -> str: + clean = str(platform or "").strip().lower() + return PLATFORM_ADAPTER_ALIASES.get(clean, clean) + + +def _structured_source_plan(platform: str) -> dict[str, Any]: + adapter = get_adapter(_adapter_code(platform)) + if not adapter: + return { + "available": False, + "adapter_code": _adapter_code(platform), + "network_request_allowed": False, + "database_write_allowed": False, + "sources": [], + } + plan = adapter.build_discovery_plan() + return { + "available": True, + "adapter_code": plan.get("platform_code"), + "platform_name": plan.get("platform_name"), + "base_url": plan.get("base_url"), + "safety_policy": plan.get("safety_policy"), + "network_request_allowed": bool(plan.get("network_request_allowed")), + "database_write_allowed": bool(plan.get("database_write_allowed")), + "sources": list(plan.get("sources") or []), + } + + +def _public_browser_context(platform: str) -> dict[str, Any]: + return { + "context_policy": "public_empty_browser_context_no_login", + "locale": "zh-TW", + "timezone_id": "Asia/Taipei", + "viewport": {"name": "desktop-1440", "width": 1440, "height": 950}, + "extra_http_headers": { + "Accept-Language": DEFAULT_ACCEPT_LANGUAGE, + }, + "credentialed_session_allowed": False, + "storage_state_allowed": False, + "raw_cookie_or_session_read_allowed": False, + "login_allowed": False, + "cart_or_checkout_allowed": False, + "platform": platform, + } + + +def _barrier_type_from_signals( + *, + platform: str, + http_status: int, + visual_barrier_reason: str, + url: str, + title: str, + validation: Mapping[str, Any] | None = None, + parsed_output: Mapping[str, Any] | None = None, +) -> str: + validation = validation or {} + parsed_output = parsed_output or {} + haystack = " ".join( + [ + visual_barrier_reason, + url, + title, + _safe_text(parsed_output.get("notes")), + _safe_text((parsed_output.get("fields") or {}).get("title")), + ] + ).lower() + if http_status in {401, 403} or "access denied" in haystack: + return "access_denied" + if "verify/traffic" in haystack or "traffic" in haystack: + return "traffic_verification_interstitial" + if ( + validation.get("interstitial_signal_detected") + or "language selection" in haystack + or "select language" in haystack + or "choose language" in haystack + or "region selection" in haystack + or "select region" in haystack + or "language" in haystack + or "語言" in haystack + ): + return "language_or_region_interstitial" + if validation.get("generic_marketplace_title_detected") or ( + platform == "shopee_tw" and "花得更少買得更好" in haystack + ): + return "generic_marketplace_landing" + if visual_barrier_reason: + return "platform_visual_barrier" + if validation.get("non_product_or_interstitial_detected"): + return "non_product_or_interstitial" + return "unknown_platform_barrier" + + +def _probe_status(platform: str, barrier_type: str) -> str: + if platform == "shopee_tw" and barrier_type in { + "language_or_region_interstitial", + "generic_marketplace_landing", + "traffic_verification_interstitial", + }: + return "ready_for_public_context_probe" + if barrier_type in {"access_denied", "traffic_verification_interstitial"}: + return "structured_source_or_backoff_required" + return "ready_for_platform_probe" + + +def _next_machine_action(platform: str, barrier_type: str, status: str) -> str: + if platform == "shopee_tw" and status == "ready_for_public_context_probe": + return "run_shopee_public_context_probe_then_structured_source_fallback" + if status == "structured_source_or_backoff_required": + return "use_structured_source_or_platform_backoff_policy" + return "run_public_platform_context_probe" + + +def _recommended_actions(platform: str, barrier_type: str, status: str) -> list[dict[str, Any]]: + actions: list[dict[str, Any]] = [] + if status == "ready_for_public_context_probe": + actions.append({ + "order": 1, + "action": "rerun_visual_capture_with_public_browser_context", + "machine_runnable": True, + "context_keys": ["locale", "timezone_id", "extra_http_headers", "viewport"], + }) + actions.append({ + "order": len(actions) + 1, + "action": "read_structured_market_intel_adapter_sources", + "machine_runnable": True, + "adapter_code": _adapter_code(platform), + }) + if barrier_type in {"access_denied", "traffic_verification_interstitial"}: + actions.append({ + "order": len(actions) + 1, + "action": "apply_platform_backoff_and_do_not_treat_barrier_as_product_data", + "machine_runnable": True, + }) + actions.append({ + "order": len(actions) + 1, + "action": "keep_visual_fields_out_of_formal_price_tables", + "machine_runnable": True, + }) + return actions + + +def _manifest_preview(platform: str, url: str) -> dict[str, Any] | None: + keyword = _extract_keyword(url) + if not keyword: + return None + manifest = build_pixelrag_marketplace_search_manifest( + platform=platform, + keyword=keyword, + crawler="PixelRAGPlatformProbe.public_context_visual_fallback", + trigger_reason="platform_interstitial_or_blocked_page_probe", + evidence_intent="collect_public_marketplace_offer_cards_after_platform_probe", + ) + if manifest.get("success"): + manifest["public_browser_context"] = _public_browser_context(platform) + return manifest + + +def _vlm_receipt_candidates( + root: Path, + *, + platforms: tuple[str, ...], + limit: int, +) -> list[Path]: + if not root.exists(): + return [] + candidates: list[Path] = [] + if platforms: + for platform in platforms: + candidates.extend((root / platform).glob("*/vlm_replay_receipt.json")) + else: + candidates.extend(root.glob("*/*/vlm_replay_receipt.json")) + return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] + + +def _candidate_from_capture(candidate: Mapping[str, Any]) -> dict[str, Any] | None: + next_action = str(candidate.get("next_machine_action") or "") + visual_barrier_reason = str(candidate.get("visual_barrier_reason") or "") + http_status = _safe_int(candidate.get("http_status")) + if ( + "platform_probe" not in next_action + and not visual_barrier_reason + and http_status < 400 + ): + return None + + platform = str(candidate.get("platform") or "unknown").strip().lower() + manifest_id = str(candidate.get("manifest_id") or "").strip() + url = str(candidate.get("url") or "").strip() + title = str(candidate.get("title") or "").strip() + barrier_type = _barrier_type_from_signals( + platform=platform, + http_status=http_status, + visual_barrier_reason=visual_barrier_reason, + url=url, + title=title, + ) + status = _probe_status(platform, barrier_type) + return { + "platform": platform, + "manifest_id": manifest_id, + "source_type": "capture_receipt", + "source_receipt_path": candidate.get("receipt_path"), + "generated_at": candidate.get("generated_at"), + "age_hours": candidate.get("age_hours"), + "url": url, + "title": title, + "http_status": http_status, + "barrier_type": barrier_type, + "visual_barrier_reason": visual_barrier_reason, + "probe_status": status, + "probe_ready": True, + "next_machine_action": _next_machine_action(platform, barrier_type, status), + "public_browser_context": _public_browser_context(platform), + "capture_manifest_preview": _manifest_preview(platform, url), + "structured_source_fallback": _structured_source_plan(platform), + "recommended_probe_actions": _recommended_actions(platform, barrier_type, status), + "source_next_machine_action": candidate.get("next_machine_action"), + "writes_database": False, + "primary_human_gate_count": 0, + } + + +def _candidate_from_vlm(path: Path, *, now: datetime, max_age_hours: int) -> dict[str, Any] | None: + receipt, error = _read_json(path) + if error: + return { + "platform": path.parent.parent.name, + "manifest_id": path.parent.name, + "source_type": "vlm_replay_receipt", + "source_receipt_path": str(path), + "probe_status": "invalid_vlm_receipt", + "probe_ready": False, + "barrier_type": "invalid_receipt", + "errors": [error], + "next_machine_action": "fix_invalid_pixelrag_vlm_receipt", + "writes_database": False, + "primary_human_gate_count": 0, + } + + validation = receipt.get("validation") if isinstance(receipt.get("validation"), Mapping) else {} + parsed_output = receipt.get("parsed_output") if isinstance(receipt.get("parsed_output"), Mapping) else {} + next_action = str(receipt.get("next_machine_action") or "") + if ( + "platform_probe" not in next_action + and not validation.get("non_product_or_interstitial_detected") + and not validation.get("blocked_page_detected") + ): + return None + + platform = str(receipt.get("platform") or path.parent.parent.name).strip().lower() + manifest_id = str(receipt.get("manifest_id") or path.parent.name).strip() + generated = _parse_iso_datetime(receipt.get("generated_at")) + age_hours = ((now - generated).total_seconds() / 3600) if generated else None + source_url = "" + source_title = "" + source_receipt = str(receipt.get("source_receipt_path") or "") + if source_receipt: + capture, _ = _read_json(Path(source_receipt)) + capture_target = capture.get("capture_target") or {} + page_metrics = capture.get("page_metrics") or {} + source_url = str(capture_target.get("url") or page_metrics.get("final_url") or "") + source_title = str(page_metrics.get("title") or "") + barrier_type = _barrier_type_from_signals( + platform=platform, + http_status=0, + visual_barrier_reason="", + url=source_url, + title=source_title, + validation=validation, + parsed_output=parsed_output, + ) + status = _probe_status(platform, barrier_type) + return { + "platform": platform, + "manifest_id": manifest_id, + "source_type": "vlm_replay_receipt", + "source_receipt_path": str(path), + "source_capture_receipt_path": source_receipt, + "generated_at": receipt.get("generated_at"), + "age_hours": round(age_hours, 3) if age_hours is not None else None, + "stale": age_hours is None or age_hours > max_age_hours, + "url": source_url, + "title": source_title, + "http_status": 0, + "barrier_type": barrier_type, + "probe_status": status, + "probe_ready": True, + "validation": { + "blocked_page_detected": bool(validation.get("blocked_page_detected")), + "non_product_or_interstitial_detected": bool( + validation.get("non_product_or_interstitial_detected") + ), + "interstitial_signal_detected": bool( + validation.get("interstitial_signal_detected") + ), + "generic_marketplace_title_detected": bool( + validation.get("generic_marketplace_title_detected") + ), + "present_field_count": _safe_int(validation.get("present_field_count")), + "missing_required_fields": list(validation.get("missing_required_fields") or []), + }, + "next_machine_action": _next_machine_action(platform, barrier_type, status), + "public_browser_context": _public_browser_context(platform), + "capture_manifest_preview": _manifest_preview(platform, source_url), + "structured_source_fallback": _structured_source_plan(platform), + "recommended_probe_actions": _recommended_actions(platform, barrier_type, status), + "source_next_machine_action": receipt.get("next_machine_action"), + "writes_database": False, + "primary_human_gate_count": 0, + } + + +def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + by_key: dict[tuple[str, str], dict[str, Any]] = {} + source_rank = {"vlm_replay_receipt": 2, "capture_receipt": 1} + for item in items: + key = (str(item.get("platform") or ""), str(item.get("manifest_id") or "")) + current = by_key.get(key) + if not current or source_rank.get(str(item.get("source_type")), 0) >= source_rank.get( + str(current.get("source_type")), 0 + ): + by_key[key] = item + return list(by_key.values()) + + +def build_pixelrag_platform_probe_readiness( + *, + artifact_root: str | Path | None = None, + vlm_receipt_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, +) -> dict[str, Any]: + """Build a no-write platform probe plan from PixelRAG barrier receipts.""" + capture_root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT) + vlm_root = Path(vlm_receipt_root or DEFAULT_VLM_RECEIPT_ROOT) + platforms = _normalise_platforms(platform) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + now = datetime.now(timezone.utc) + + capture_readback = build_pixelrag_rag_candidate_replay_readback( + artifact_root=capture_root, + platform=platforms, + max_age_hours=max_age, + limit=item_limit, + ) + capture_items = [ + item + for item in ( + _candidate_from_capture(candidate) + for candidate in list(capture_readback.get("candidates") or []) + ) + if item + ] + vlm_items = [ + item + for item in ( + _candidate_from_vlm(path, now=now, max_age_hours=max_age) + for path in _vlm_receipt_candidates(vlm_root, platforms=platforms, limit=item_limit) + ) + if item + ] + probe_items = _dedupe_items(capture_items + vlm_items) + invalid_count = sum(1 for item in probe_items if not item.get("probe_ready")) + ready_count = sum(1 for item in probe_items if item.get("probe_ready")) + shopee_public_context_count = sum( + 1 + for item in probe_items + if item.get("platform") == "shopee_tw" + and item.get("probe_status") == "ready_for_public_context_probe" + ) + access_denied_count = sum(1 for item in probe_items if item.get("barrier_type") == "access_denied") + traffic_count = sum( + 1 for item in probe_items if item.get("barrier_type") == "traffic_verification_interstitial" + ) + language_count = sum( + 1 for item in probe_items if item.get("barrier_type") == "language_or_region_interstitial" + ) + structured_fallback_count = sum( + 1 + for item in probe_items + if (item.get("structured_source_fallback") or {}).get("available") + ) + + if invalid_count and invalid_count == len(probe_items): + status = "critical" + elif probe_items and ready_count: + status = "ok" + else: + status = "warning" + + next_action = ( + "fix_invalid_pixelrag_platform_probe_receipts" + if invalid_count and not ready_count + else ( + "run_platform_probe_or_structured_source_fallback" + if ready_count + else "run_pixelrag_visual_capture_worker" + ) + ) + + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": now.isoformat(), + "artifact_root": str(capture_root), + "vlm_receipt_root": str(vlm_root), + "platform_filter": list(platforms), + "max_age_hours": max_age, + "limit": item_limit, + "summary": { + "probe_candidate_count": len(probe_items), + "ready_for_probe_count": ready_count, + "invalid_count": invalid_count, + "capture_source_count": len(capture_items), + "vlm_source_count": len(vlm_items), + "shopee_public_context_probe_count": shopee_public_context_count, + "language_or_region_interstitial_count": language_count, + "traffic_verification_count": traffic_count, + "access_denied_count": access_denied_count, + "structured_source_fallback_count": structured_fallback_count, + "writes_database_count": 0, + "primary_human_gate_count": 0, + "platforms": sorted({str(item.get("platform") or "unknown") for item in probe_items}), + }, + "probe_items": probe_items, + "source_capture_replay": { + "policy": capture_readback.get("policy"), + "status": capture_readback.get("status"), + "summary": capture_readback.get("summary"), + "next_machine_action": capture_readback.get("next_machine_action"), + }, + "probe_contract": { + "automation_mode": "platform_probe_plan_no_write", + "network_call": False, + "db_write": False, + "writes_database": False, + "writes_ai_insights": False, + "writes_price_tables": False, + "secret_read": False, + "raw_cookie_or_session_read": False, + "credentialed_session_allowed": False, + "login_allowed": False, + "blocked_pages_are_not_product_data": True, + "visual_fields_are_candidate_evidence_only": True, + "primary_human_gate_count": 0, + }, + "controlled_apply": { + "network_call": False, + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "secret_read": False, + "raw_cookie_or_session_read": False, + "production_price_write": False, + "artifact_write": False, + "primary_human_gate_count": 0, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "POLICY", + "build_pixelrag_platform_probe_readiness", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index 8f57ce3..a32752e 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1436,11 +1436,12 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_platform_probe_check", lambda: smoke._check("pixelrag platform probe", "ok", "ok")) result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 34, "warning": 1, "critical": 1, "total": 36} + assert result["summary"] == {"ok": 35, "warning": 1, "critical": 1, "total": 37} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3972,6 +3973,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_platform_probe_check", lambda: smoke._check("pixelrag platform probe", "ok", "ok")) first = smoke.collect_ai_automation_smoke(history_limit=5) second = smoke.collect_ai_automation_smoke(history_limit=5) @@ -4027,7 +4029,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 36, "warning": 0, "critical": 0, "total": 36}, + "summary": {"ok": 37, "warning": 0, "critical": 0, "total": 37}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4192,6 +4194,24 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "artifact_write_performed": False, "next_machine_action": "run_pixelrag_vlm_replay_worker_execute", }, + }, + { + "name": "PixelRAG platform probe readiness", + "status": "ok", + "summary": "PixelRAG platform probe candidates=1, ready=1, shopee_context=1, structured_fallback=1", + "details": { + "policy": "read_only_pixelrag_platform_probe_readiness_v1", + "probe_candidate_count": 1, + "ready_for_probe_count": 1, + "shopee_public_context_probe_count": 1, + "language_or_region_interstitial_count": 1, + "traffic_verification_count": 0, + "access_denied_count": 0, + "structured_source_fallback_count": 1, + "next_machine_action": "run_platform_probe_or_structured_source_fallback", + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, } ], }, ensure_ascii=False) + "\n", @@ -4210,7 +4230,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 33 + assert summary["summary"]["total"] == 34 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -5910,6 +5930,13 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( assert pixelrag_vlm_replay_worker_family["details"]["dry_run_count"] == 1 assert pixelrag_vlm_replay_worker_family["details"]["model_call_performed"] is False assert pixelrag_vlm_replay_worker_family["details"]["writes_database_count"] == 0 + pixelrag_platform_probe_family = next( + item for item in summary["families"] + if item["key"] == "pixelrag_platform_probe" + ) + assert pixelrag_platform_probe_family["status"] == "ok" + assert pixelrag_platform_probe_family["details"]["ready_for_probe_count"] == 1 + assert pixelrag_platform_probe_family["details"]["primary_human_gate_count"] == 0 assert summary["scheduled_outputs"]["telegram_send_in_preview"] is False assert summary["scheduled_outputs"]["writes_database"] is False assert summary["automation_policy"]["primary_flow"] == "ai_controlled" @@ -6541,6 +6568,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag vlm worker ok", )) + monkeypatch.setattr(smoke, "_pixelrag_platform_probe_check", lambda: smoke._check( + "PixelRAG platform probe readiness", + "ok", + "pixelrag platform probe ok", + )) result = smoke.collect_ai_automation_smoke(record_history=False) @@ -6556,7 +6588,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 36 + assert result["summary"]["total"] == 37 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_platform_probe_service.py b/tests/test_pixelrag_platform_probe_service.py new file mode 100644 index 0000000..8a98f08 --- /dev/null +++ b/tests/test_pixelrag_platform_probe_service.py @@ -0,0 +1,240 @@ +import json +import subprocess +import sys +from datetime import datetime, timezone + +from tests.test_pixelrag_rag_candidate_replay_service import _write_receipt + + +def _patch_receipt(path, **updates): + payload = json.loads(path.read_text(encoding="utf-8")) + for key, value in updates.items(): + if key == "page_metrics": + payload.setdefault("page_metrics", {}).update(value) + else: + payload[key] = value + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return payload + + +def test_pixelrag_platform_probe_builds_machine_actions_for_shopee_and_coupang(tmp_path): + from services.pixelrag_platform_probe_service import ( + POLICY, + build_pixelrag_platform_probe_readiness, + ) + + shopee_receipt = _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-traffic", + title="蝦皮購物 | 花得更少買得更好", + url="https://shopee.tw/search?keyword=sunscreen", + ) + _patch_receipt( + shopee_receipt, + page_metrics={ + "final_url": "https://shopee.tw/verify/traffic/error?home_url=https%3A%2F%2Fshopee.tw", + "title": "蝦皮購物 | 花得更少買得更好", + }, + ) + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + + payload = build_pixelrag_platform_probe_readiness(artifact_root=tmp_path) + by_platform = {item["platform"]: item for item in payload["probe_items"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "ok" + assert payload["summary"]["probe_candidate_count"] == 2 + assert payload["summary"]["ready_for_probe_count"] == 2 + assert payload["summary"]["traffic_verification_count"] == 1 + assert payload["summary"]["access_denied_count"] == 1 + assert payload["summary"]["writes_database_count"] == 0 + assert payload["controlled_apply"]["primary_human_gate_count"] == 0 + assert by_platform["shopee_tw"]["probe_status"] == "ready_for_public_context_probe" + assert by_platform["shopee_tw"]["next_machine_action"] == ( + "run_shopee_public_context_probe_then_structured_source_fallback" + ) + assert by_platform["shopee_tw"]["public_browser_context"]["locale"] == "zh-TW" + assert by_platform["shopee_tw"]["public_browser_context"]["storage_state_allowed"] is False + assert by_platform["shopee_tw"]["capture_manifest_preview"]["success"] is True + assert by_platform["coupang_tw"]["probe_status"] == "structured_source_or_backoff_required" + assert by_platform["coupang_tw"]["next_machine_action"] == ( + "use_structured_source_or_platform_backoff_policy" + ) + assert by_platform["coupang_tw"]["structured_source_fallback"]["available"] is True + assert payload["probe_contract"]["raw_cookie_or_session_read"] is False + assert payload["probe_contract"]["blocked_pages_are_not_product_data"] is True + + +def test_pixelrag_platform_probe_reads_vlm_interstitial_receipts(tmp_path): + from services.pixelrag_platform_probe_service import build_pixelrag_platform_probe_readiness + + capture_path = _write_receipt( + tmp_path / "capture", + platform="shopee_tw", + manifest_id="shopee-language", + title="Shopee", + url="https://shopee.tw/search?keyword=sunscreen", + ) + vlm_dir = tmp_path / "vlm" / "shopee_tw" / "shopee-language" + vlm_dir.mkdir(parents=True) + (vlm_dir / "vlm_replay_receipt.json").write_text( + json.dumps({ + "generated_at": datetime.now(timezone.utc).isoformat(), + "platform": "shopee_tw", + "manifest_id": "shopee-language", + "source_receipt_path": str(capture_path), + "worker_status": "executed_warning", + "next_machine_action": "run_platform_probe_or_use_structured_api", + "parsed_output": { + "blocked_page_detected": False, + "fields": { + "title": { + "value": "蝦皮購物 | 花得更少買得更好", + "confidence": 0.95, + "evidence_refs": ["tile:1"], + }, + "price": {"value": None, "confidence": 0.0, "evidence_refs": []}, + }, + "notes": [{"note": "Language selection page"}], + }, + "validation": { + "blocked_page_detected": False, + "non_product_or_interstitial_detected": True, + "interstitial_signal_detected": True, + "generic_marketplace_title_detected": True, + "present_field_count": 1, + "missing_required_fields": ["price"], + }, + }, ensure_ascii=False), + encoding="utf-8", + ) + + payload = build_pixelrag_platform_probe_readiness( + artifact_root=tmp_path / "capture", + vlm_receipt_root=tmp_path / "vlm", + platform="shopee_tw", + ) + + assert payload["status"] == "ok" + assert payload["summary"]["probe_candidate_count"] == 1 + assert payload["summary"]["vlm_source_count"] == 1 + assert payload["summary"]["language_or_region_interstitial_count"] == 1 + item = payload["probe_items"][0] + assert item["source_type"] == "vlm_replay_receipt" + assert item["barrier_type"] == "language_or_region_interstitial" + assert item["probe_status"] == "ready_for_public_context_probe" + assert item["primary_human_gate_count"] == 0 + + +def test_pixelrag_platform_probe_cli_outputs_machine_readable_json(tmp_path): + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/report_pixelrag_platform_probe.py", + "--artifact-root", + str(tmp_path), + "--platform", + "coupang_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["probe_candidate_count"] == 1 + assert payload["probe_items"][0]["platform"] == "coupang_tw" + assert payload["probe_contract"]["writes_database"] is False + + +def test_pixelrag_capture_worker_dry_run_uses_safe_public_context(tmp_path): + from services.pixelrag_crawler_integration_service import ( + build_pixelrag_marketplace_search_manifest, + ) + + manifest = build_pixelrag_marketplace_search_manifest( + platform="shopee_tw", + keyword="sunscreen", + crawler="PixelRAGPlatformProbe.public_context_visual_fallback", + trigger_reason="platform_interstitial_or_blocked_page_probe", + ) + manifest["public_browser_context"] = { + "locale": "zh-TW", + "timezone_id": "Asia/Taipei", + "extra_http_headers": { + "Accept-Language": "zh-TW,zh-Hant;q=0.95,zh;q=0.9,en;q=0.7", + "Cookie": "must_not_survive", + "Authorization": "must_not_survive", + }, + } + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/capture_pixelrag_visual_evidence.py", + "--manifest-json", + json.dumps(manifest, ensure_ascii=False), + "--output-dir", + str(tmp_path / "artifacts"), + "--dry-run", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + context = payload["public_browser_context"] + assert context["locale"] == "zh-TW" + assert context["timezone_id"] == "Asia/Taipei" + assert context["extra_http_headers"]["Accept-Language"].startswith("zh-TW") + assert "Cookie" not in context["extra_http_headers"] + assert "Authorization" not in context["extra_http_headers"] + assert context["raw_cookie_or_session_read_allowed"] is False + + +def test_pixelrag_platform_probe_route_returns_readback(tmp_path, monkeypatch): + from flask import Flask + from routes import system_public_routes as routes + from services import pixelrag_platform_probe_service as service + + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + monkeypatch.setattr(service, "DEFAULT_ARTIFACT_ROOT", str(tmp_path)) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-platform-probe?platform=coupang_tw" + ): + response = routes.ai_automation_pixelrag_platform_probe_api.__wrapped__() + payload = response.get_json() + + assert payload["policy"] == "read_only_pixelrag_platform_probe_readiness_v1" + assert payload["summary"]["probe_candidate_count"] == 1 + assert payload["probe_items"][0]["next_machine_action"] == ( + "use_structured_source_or_platform_backoff_policy" + ) From 49828194932306a06abd597b91c92c61a5ff1962 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 09:50:30 +0800 Subject: [PATCH 18/75] feat(ai): add PixelRAG platform probe worker --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 2 + .../ai_automation_mainline_work_items.md | 7 +- docs/guides/ai_automation_session_sop.md | 4 +- docs/guides/browse_sh_crawler_playbook.md | 6 + routes/system_public_routes.py | 40 ++ .../ops/run_pixelrag_platform_probe_worker.py | 108 ++++ services/ai_automation_smoke_service.py | 114 ++++ .../pixelrag_platform_probe_worker_service.py | 530 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 46 +- ..._pixelrag_platform_probe_worker_service.py | 248 ++++++++ 11 files changed, 1098 insertions(+), 9 deletions(-) create mode 100644 scripts/ops/run_pixelrag_platform_probe_worker.py create mode 100644 services/pixelrag_platform_probe_worker_service.py create mode 100644 tests/test_pixelrag_platform_probe_worker_service.py diff --git a/config.py b/config.py index 7e287c8..b69ad11 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.759" +SYSTEM_VERSION = "V10.760" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index d239c2c..f6f9d50 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -119,6 +119,7 @@ - 2026-07-09 起 PixelRAG ready receipts 必須有 Ollama-first VLM replay worker：`/api/ai-automation/pixelrag-vlm-replay-worker` 與 `scripts/ops/run_pixelrag_vlm_replay_worker.py` 預設 dry-run，`execute=true&write_receipt=true` 才呼叫 approved Ollama VLM route 並寫 artifact receipt；`/api/ai-automation/smoke` 需包含 `PixelRAG VLM replay worker`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_vlm_replay_worker` family。此 worker 不讀 secret、不寫 DB、不寫 `ai_insights`、不寫正式價格表；blocked / 403 / captcha / access denied receipt 自動跳 platform probe 或 structured API，ready receipt 的 VLM 結果仍須 identity matcher replay 與 PromotionGate 才能進候選知識層。 - 2026-07-09 起 PixelRAG VLM route readiness 必須在 execute 前可讀回：`/api/ai-automation/pixelrag-vlm-route-readiness` 與 `scripts/ops/report_pixelrag_vlm_route_readiness.py` 只讀 approved Ollama `/api/tags`，輸出 configured model、candidate model、reachable host、model_route_ready 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG VLM route readiness`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_vlm_route_readiness` family。此 readback 不呼叫 `/api/generate`、不讀 secret、不寫 DB；worker execute 必須使用它自動避開缺模型路由，完全缺候選時寫 `model_route_not_ready` artifact receipt。 - 2026-07-10 起 PixelRAG platform probe 必須把 blocked / interstitial receipt 轉成機器可執行下一步：`/api/ai-automation/pixelrag-platform-probe` 與 `scripts/ops/report_pixelrag_platform_probe.py` 需讀取 capture receipts 與 VLM replay receipts，辨識 Shopee language / region / generic marketplace slogan / traffic verification 與 Coupang 403 / access denied，輸出 public browser context probe、structured source fallback、platform backoff、blocked page not product data 與 no-write 邊界；`/api/ai-automation/smoke` 需包含 `PixelRAG platform probe readiness`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_platform_probe` family。此 readback 不抓外站、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture worker 可消化 `public_browser_context` 的 locale/timezone/Accept-Language，但必須濾掉 Cookie、Authorization、token/secret/key 類 header。 +- 2026-07-10 起 PixelRAG platform probe 必須有 controlled worker：`/api/ai-automation/pixelrag-platform-probe-worker` 與 `scripts/ops/run_pixelrag_platform_probe_worker.py` 預設 dry-run，`execute=true&write_receipt=true` 才會對 Shopee language / traffic / generic landing 執行 public empty-context capture，並對 Coupang 403 / access denied 自動產生 structured-source/backoff artifact receipt；`/api/ai-automation/smoke` 需包含 `PixelRAG platform probe worker`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_platform_probe_worker` family。此 worker 不登入、不讀 cookie/session/secret、不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture 成功後下一步固定為 RAG candidate replay，structured fallback 只產生機器可讀 package，不把 blocked page 當商品資料。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -890,6 +891,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG VLM worker preflight 必須容納 111 VLM 冷啟延遲 | V10.757 起 `run_pixelrag_vlm_replay_worker.py --execute` 與對應 API 的 `route_generate_probe_timeout` 預設為 20 秒；111 `minicpm-v` 在正式環境可於約 4 到 20 秒間完成小型 `/api/generate` probe，8 秒容易造成假陰性。若 VLM receipt 已判定 `run_platform_probe_or_use_structured_api`，run-level `next_machine_action` 也必須同樣輸出 platform probe，不得停留在 rerun VLM 摘要。 | | 2026-07-10 | PixelRAG VLM language note / generic marketplace slogan 必須視為 interstitial | V10.758 起若 VLM notes 或 title 顯示 `Language selection page`、語言/地區選擇，或只抽到 Shopee `蝦皮購物 | 花得更少買得更好` 這類網站 slogan 且 price/shop 缺失，worker validation 會標記 `interstitial_signal_detected` / `generic_marketplace_title_detected` 與 `non_product_or_interstitial_detected=true`，run-level 與 item-level 下一步都輸出 `run_platform_probe_or_use_structured_api`。 | | 2026-07-10 | PixelRAG platform probe 必須自動消化 marketplace interstitial / blocked receipt | V10.759 起 `/api/ai-automation/pixelrag-platform-probe`、`scripts/ops/report_pixelrag_platform_probe.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 platform probe / `pixelrag_platform_probe` family；Shopee traffic / language / generic landing 會轉成 `run_shopee_public_context_probe_then_structured_source_fallback`，Coupang 403 / access denied 會轉成 `use_structured_source_or_platform_backoff_policy`。probe plan 只產生 public empty browser context、structured adapter fallback 與 no-write contract，不讀或注入 cookie/session/login，不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture worker 只允許 locale/timezone/非敏感 header 並濾掉 Cookie / Authorization / token / secret / key 類 header。 | +| 2026-07-10 | PixelRAG platform probe worker 必須把 probe plan 變成 controlled automation | V10.760 起 `/api/ai-automation/pixelrag-platform-probe-worker`、`scripts/ops/run_pixelrag_platform_probe_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled platform probe worker / `pixelrag_platform_probe_worker` family；dry-run 只規劃 capture 或 structured fallback，execute 模式才執行 public empty-context capture 或寫 structured-source/backoff artifact receipt。此 worker 明確標記 `network_call` 只在 capture execute 時為 true、`model_call=false`、`writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`raw_cookie_or_session_read=false`、`primary_human_gate_count=0`；capture 成功後必須回到 PixelRAG RAG candidate replay，不得直接寫正式價格或把 blocked page 當商品資料。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 63e406a..e146883 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -1,6 +1,6 @@ # AI Automation Mainline Work Items -> Updated: 2026-07-09 +> Updated: 2026-07-10 > Scope: EwoooC / MOMO Pro System AI automation, PixelRAG, MCP/RAG, production UX. ## Source Of Truth @@ -20,7 +20,8 @@ | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | -| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, and PixelRAG VLM replay worker families. | +| Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture, Coupang 403/access denied goes to structured-source/backoff artifact receipts. | +| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, and PixelRAG platform probe worker families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | ## P1 @@ -31,7 +32,7 @@ | Completed | Ollama-first OCR/VLM extraction worker | `run_pixelrag_vlm_replay_worker.py --execute --write-receipt` executes the replay contract against approved Ollama multimodal routes and emits confidence/evidence artifact receipts without writing formal price truth. | | Not started | Ollama-first multimodal embedding benchmark | Verify local Qwen3-VL or equivalent visual embedding on GCP-A -> GCP-B -> 111 before any visual vector retrieval. | | Not started | pgvector-compatible visual evidence metadata | Design metadata-first retrieval without FAISS in production unless ADR approves a different store. | -| Not started | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; prefer public structured data or approved probe, never count blocked pages as product data. | +| In progress | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; `run_pixelrag_platform_probe_worker.py --execute --write-receipt` now emits structured-source/backoff receipts without network, DB writes, or product-data promotion. | | Not started | Marketplace source contracts beyond MOMO/PChome | Promote stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten receipts into source contracts only after provenance, rate-limit, public boundary, and replay evidence. | | In progress | Professional product website UI/UX text reduction | Continue compact AI surfaces, visual QA, and user-facing copy guardrails; avoid engineering logs or work-session prose in the UI. | diff --git a/docs/guides/ai_automation_session_sop.md b/docs/guides/ai_automation_session_sop.md index eeb8430..d23d821 100644 --- a/docs/guides/ai_automation_session_sop.md +++ b/docs/guides/ai_automation_session_sop.md @@ -38,7 +38,9 @@ 或 `python scripts/ops/run_pixelrag_vlm_replay_worker.py` 可讀回 dry-run/execute、model_call、artifact receipt、blocked/ready 分流、DB write=0 與 primary human gate=0；execute 結果仍只是 candidate evidence。 - PixelRAG VLM route readiness 必須確認 `/api/ai-automation/pixelrag-vlm-route-readiness` 或 `python scripts/ops/report_pixelrag_vlm_route_readiness.py` 可讀回 approved Ollama hosts、configured model 是否安裝、candidate model、自動選模下一步、model_call=false、DB write=0；缺模型不得等 execute 才暴露。 -- AI automation smoke 必須包含 external MCP/RAG integration、PixelRAG RAG candidate replay、PixelRAG OCR/VLM replay contract、PixelRAG VLM route readiness 與 PixelRAG VLM replay worker family，避免 registry 已完成但 runtime flag / receipt replay / VLM route / VLM worker 未完成時被誤報為全自動閉環。 +- PixelRAG platform probe worker 必須確認 `/api/ai-automation/pixelrag-platform-probe-worker` + 或 `python scripts/ops/run_pixelrag_platform_probe_worker.py` 可讀回 dry-run/execute、public-context capture、structured-source/backoff receipt、network_call、artifact receipt、DB write=0 與 primary human gate=0；Shopee interstitial 可 execute capture，Coupang 403 必須自動落 structured fallback，不得要求人工主流程。 +- AI automation smoke 必須包含 external MCP/RAG integration、PixelRAG RAG candidate replay、PixelRAG OCR/VLM replay contract、PixelRAG VLM route readiness、PixelRAG VLM replay worker、PixelRAG platform probe readiness 與 PixelRAG platform probe worker family，避免 registry 已完成但 runtime flag / receipt replay / VLM route / VLM worker / platform probe worker 未完成時被誤報為全自動閉環。 - AI 自動化 Prometheus 指標變更必須同步檢查 `docker/grafana/provisioning/dashboards/json/ai-automation-overview.json` 是否需要新增 panel 或查詢。 - 188 線上 active monitoring stack 以 `monitoring/prometheus.yml` 為準；110 gateway 另有 `/home/wooo/monitoring/prometheus.yml`。若 dashboard 無資料，先確認 Prometheus `momo-app` target 與 `momo-network` 連線；所有 Blackbox HTTP target 必須打 `/health`，不可打 Dashboard 首頁 `/`。 - Smoke dashboard 會保存 JSONL 趨勢；若新增檢查項目，要確保 history compact record 仍保持小而可讀。 diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index 1b50cc1..12993ae 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -192,10 +192,16 @@ python scripts/ops/report_pixelrag_vlm_route_readiness.py --model minicpm-v:late python scripts/ops/run_pixelrag_vlm_replay_worker.py python scripts/ops/run_pixelrag_vlm_replay_worker.py --platform shopee_tw --platform coupang_tw python scripts/ops/run_pixelrag_vlm_replay_worker.py --platform shopee_tw --execute --write-receipt --limit 1 +python scripts/ops/report_pixelrag_platform_probe.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_platform_probe_worker.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_platform_probe_worker.py --platform shopee_tw --execute --write-receipt --limit 1 +python scripts/ops/run_pixelrag_platform_probe_worker.py --platform coupang_tw --execute --write-receipt --limit 1 ``` API readback: `/api/ai-automation/pixelrag-vlm-route-readiness` 與 `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。worker 預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。execute 前會自動讀 approved Ollama `/api/tags` 選擇已安裝候選模型；若完全沒有候選，會寫 `model_route_not_ready` artifact receipt，不盲打 generate。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 +Platform probe worker API readback: `/api/ai-automation/pixelrag-platform-probe-worker?platform=shopee_tw`。worker 預設為 dry-run，不抓外站、不寫 artifact；`execute=true&write_receipt=true` 才對 Shopee language / traffic / generic landing 執行 public empty-context capture，或對 Coupang 403 / access denied 寫 structured-source/backoff receipt。capture 成功後仍回到 RAG candidate replay，不得直接寫 `ai_insights`、正式價格表或競品價格歷史。 + Application portfolio： ```bash diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 86b3259..5974c5a 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -862,6 +862,46 @@ def ai_automation_pixelrag_platform_probe_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-platform-probe-worker') +@login_required +def ai_automation_pixelrag_platform_probe_worker_api(): + """Controlled PixelRAG platform probe worker dry-run/execute readback.""" + from services.pixelrag_platform_probe_worker_service import ( + run_pixelrag_platform_probe_worker, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + execute = str(request.args.get('execute') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + write_receipt = str(request.args.get('write_receipt') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 25, type=int) + timeout = request.args.get('timeout', 30, type=int) + settle_ms = request.args.get('settle_ms', 800, type=int) + max_tiles = request.args.get('max_tiles', 12, type=int) + return jsonify(run_pixelrag_platform_probe_worker( + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 25, 250)), + timeout=max(1, min(timeout or 30, 120)), + settle_ms=max(0, min(settle_ms or 800, 5000)), + max_tiles=max(1, min(max_tiles or 12, 80)), + execute=execute, + write_receipt=bool(execute and write_receipt), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/run_pixelrag_platform_probe_worker.py b/scripts/ops/run_pixelrag_platform_probe_worker.py new file mode 100644 index 0000000..9c6fdea --- /dev/null +++ b/scripts/ops/run_pixelrag_platform_probe_worker.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Run or dry-run the PixelRAG platform probe worker.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_platform_probe_worker_service import ( # noqa: E402 + run_pixelrag_platform_probe_worker, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="執行或 dry-run PixelRAG platform probe worker。" + ) + parser.add_argument( + "--artifact-root", + help="PixelRAG visual evidence artifact root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--vlm-receipt-root", + help="PixelRAG VLM replay receipt root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--output-root", + help="Platform probe worker receipt output root。", + ) + parser.add_argument( + "--capture-output-root", + help="Public-context capture artifact output root；預設寫回 PixelRAG visual evidence root。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=25, + help="最多處理 probe candidate 數。", + ) + parser.add_argument( + "--timeout", + type=int, + default=30, + help="public-context capture timeout 秒數。", + ) + parser.add_argument( + "--settle-ms", + type=int, + default=800, + help="頁面載入後等待毫秒數。", + ) + parser.add_argument( + "--max-tiles", + type=int, + default=12, + help="每次 public-context capture 最多切出的 tiles。", + ) + parser.add_argument( + "--execute", + action="store_true", + help="真的執行 public-context capture 或 structured fallback package；未指定時只做 no-write dry-run。", + ) + parser.add_argument( + "--write-receipt", + action="store_true", + help="execute 後寫入 platform probe worker artifact receipt；不寫 DB。", + ) + args = parser.parse_args() + + payload = run_pixelrag_platform_probe_worker( + artifact_root=args.artifact_root, + vlm_receipt_root=args.vlm_receipt_root, + output_root=args.output_root, + capture_output_root=args.capture_output_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + timeout=args.timeout, + settle_ms=args.settle_ms, + max_tiles=args.max_tiles, + execute=args.execute, + write_receipt=bool(args.write_receipt and args.execute), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index b0ee851..ef1bc27 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -557,6 +557,11 @@ def build_scheduled_automation_health_summary( if not pixelrag_platform_probe or not pixelrag_platform_probe_details: pixelrag_platform_probe = _pixelrag_platform_probe_check() pixelrag_platform_probe_details = pixelrag_platform_probe.get("details") or {} + pixelrag_platform_probe_worker = _find_check(source_result, "PixelRAG platform probe worker") + pixelrag_platform_probe_worker_details = pixelrag_platform_probe_worker.get("details") or {} + if not pixelrag_platform_probe_worker or not pixelrag_platform_probe_worker_details: + pixelrag_platform_probe_worker = _pixelrag_platform_probe_worker_check() + pixelrag_platform_probe_worker_details = pixelrag_platform_probe_worker.get("details") or {} smoke_status = source_result.get("status") or ("warning" if latest_history else "warning") freshness_family = _history_freshness_family( latest_history, @@ -4775,6 +4780,56 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_platform_probe_worker", + "label": "PixelRAG platform probe worker", + "status": pixelrag_platform_probe_worker.get("status") or "warning", + "summary": ( + pixelrag_platform_probe_worker.get("summary") + or "PixelRAG platform probe worker has no latest readback." + ), + "next_machine_action": pixelrag_platform_probe_worker_details.get("next_machine_action") + or "run_pixelrag_platform_probe_worker_dry_run", + "details": { + "policy": pixelrag_platform_probe_worker_details.get("policy"), + "probe_candidate_count": int( + pixelrag_platform_probe_worker_details.get("probe_candidate_count") or 0 + ), + "ready_count": int( + pixelrag_platform_probe_worker_details.get("ready_count") or 0 + ), + "dry_run_count": int( + pixelrag_platform_probe_worker_details.get("dry_run_count") or 0 + ), + "capture_ready_count": int( + pixelrag_platform_probe_worker_details.get("capture_ready_count") or 0 + ), + "capture_execute_count": int( + pixelrag_platform_probe_worker_details.get("capture_execute_count") or 0 + ), + "capture_ok_count": int( + pixelrag_platform_probe_worker_details.get("capture_ok_count") or 0 + ), + "capture_error_count": int( + pixelrag_platform_probe_worker_details.get("capture_error_count") or 0 + ), + "structured_fallback_count": int( + pixelrag_platform_probe_worker_details.get("structured_fallback_count") or 0 + ), + "receipt_written_count": int( + pixelrag_platform_probe_worker_details.get("receipt_written_count") or 0 + ), + "network_call_performed": bool( + pixelrag_platform_probe_worker_details.get("network_call_performed") + ), + "artifact_write_performed": bool( + pixelrag_platform_probe_worker_details.get("artifact_write_performed") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, freshness_family, { "key": "daily_summary_delivery", @@ -13674,6 +13729,64 @@ def _pixelrag_platform_probe_check() -> Dict[str, Any]: ) +def _pixelrag_platform_probe_worker_check() -> Dict[str, Any]: + """Dry-run sentinel for PixelRAG platform probe worker automation.""" + try: + from services.pixelrag_platform_probe_worker_service import ( + run_pixelrag_platform_probe_worker, + ) + + readback = run_pixelrag_platform_probe_worker() + summary = readback.get("summary") or {} + candidate_count = int(summary.get("probe_candidate_count") or 0) + ready_count = int(summary.get("ready_count") or 0) + capture_ready_count = int(summary.get("capture_ready_count") or 0) + structured_count = int(summary.get("structured_fallback_count") or 0) + status = readback.get("status") or "warning" + summary_text = ( + f"PixelRAG platform probe worker candidates={candidate_count}, " + f"ready={ready_count}, capture_ready={capture_ready_count}, " + f"structured_fallback={structured_count}, execute=false" + ) + return _check( + "PixelRAG platform probe worker", + status, + summary_text, + { + "policy": readback.get("policy"), + "probe_candidate_count": candidate_count, + "ready_count": ready_count, + "skipped_count": int(summary.get("skipped_count") or 0), + "dry_run_count": int(summary.get("dry_run_count") or 0), + "capture_ready_count": capture_ready_count, + "capture_execute_count": int(summary.get("capture_execute_count") or 0), + "capture_ok_count": int(summary.get("capture_ok_count") or 0), + "capture_error_count": int(summary.get("capture_error_count") or 0), + "structured_fallback_count": structured_count, + "executed_structured_count": int(summary.get("executed_structured_count") or 0), + "executed_count": int(summary.get("executed_count") or 0), + "receipt_written_count": int(summary.get("receipt_written_count") or 0), + "network_call_performed": bool(summary.get("network_call_performed")), + "artifact_write_performed": bool(summary.get("artifact_write_performed")), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG platform probe worker", + "critical", + f"PixelRAG platform probe worker 無法執行：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: int = 20) -> Dict[str, Any]: checks: List[Dict[str, Any]] = [ _event_router_check(), @@ -13713,6 +13826,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _pixelrag_vlm_route_readiness_check(), _pixelrag_vlm_replay_worker_check(), _pixelrag_platform_probe_check(), + _pixelrag_platform_probe_worker_check(), ] worst = max(checks, key=lambda item: STATUS_RANK.get(item["status"], 2))["status"] result = { diff --git a/services/pixelrag_platform_probe_worker_service.py b/services/pixelrag_platform_probe_worker_service.py new file mode 100644 index 0000000..5d3b76c --- /dev/null +++ b/services/pixelrag_platform_probe_worker_service.py @@ -0,0 +1,530 @@ +"""Controlled PixelRAG platform probe worker. + +This worker executes the machine actions emitted by platform-probe readiness: +public empty-context visual capture for recoverable marketplace interstitials, +or structured-source/backoff receipts for hard blocked platforms. It never +reads sessions/cookies, logs in, writes DB rows, or promotes price truth. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.pixelrag_crawler_integration_service import ( + DEFAULT_ARTIFACT_MAX_AGE_HOURS, + DEFAULT_ARTIFACT_ROOT, +) +from services.pixelrag_platform_probe_service import ( + build_pixelrag_platform_probe_readiness, +) + + +POLICY = "controlled_pixelrag_platform_probe_worker_v1" +DEFAULT_LIMIT = 25 +DEFAULT_TIMEOUT_SECONDS = 30 +DEFAULT_SETTLE_MS = 800 +DEFAULT_MAX_TILES = 12 +DEFAULT_WORKER_RECEIPT_ROOT = os.getenv( + "PIXELRAG_PLATFORM_PROBE_WORKER_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_platform_probe_worker_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_platform_probe_worker_receipts", +) +DEFAULT_CAPTURE_OUTPUT_ROOT = os.getenv( + "PIXELRAG_PLATFORM_PROBE_CAPTURE_OUTPUT_ROOT", + str(DEFAULT_ARTIFACT_ROOT), +) +CAPTURE_READY_STATUSES = { + "ready_for_public_context_probe", + "ready_for_platform_probe", +} + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...]: + if isinstance(platform, str): + value = platform.strip().lower() + return (value,) if value else () + return tuple( + str(item or "").strip().lower() + for item in (platform or ()) + if str(item or "").strip() + ) + + +def _safe_segment(value: Any) -> str: + text = str(value or "unknown").strip().lower() + text = re.sub(r"[^a-z0-9._-]+", "-", text) + return text.strip("-") or "unknown" + + +def _capture_script_path() -> Path: + return Path(__file__).resolve().parents[1] / "scripts" / "ops" / "capture_pixelrag_visual_evidence.py" + + +def _base_item(item: Mapping[str, Any]) -> dict[str, Any]: + fallback = item.get("structured_source_fallback") + if not isinstance(fallback, Mapping): + fallback = {} + return { + "platform": item.get("platform"), + "manifest_id": item.get("manifest_id"), + "source_type": item.get("source_type"), + "source_receipt_path": item.get("source_receipt_path"), + "source_capture_receipt_path": item.get("source_capture_receipt_path"), + "probe_status": item.get("probe_status"), + "barrier_type": item.get("barrier_type"), + "url": item.get("url"), + "title": item.get("title"), + "structured_source_fallback_available": bool(fallback.get("available")), + "structured_source_count": len(list(fallback.get("sources") or [])), + "writes_database": False, + "model_call_performed": False, + "primary_human_gate_count": 0, + } + + +def _write_worker_receipt( + *, + output_root: Path, + item: Mapping[str, Any], + worker_item: Mapping[str, Any], +) -> str: + target = ( + output_root + / _safe_segment(item.get("platform")) + / _safe_segment(item.get("manifest_id")) + / "platform_probe_worker_receipt.json" + ) + target.parent.mkdir(parents=True, exist_ok=True) + receipt = dict(worker_item) + receipt["artifact_write_performed"] = True + receipt["receipt_path"] = str(target) + target.write_text( + json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + return str(target) + + +def _dry_run_item(item: Mapping[str, Any]) -> dict[str, Any]: + base = _base_item(item) + probe_status = str(item.get("probe_status") or "") + manifest = item.get("capture_manifest_preview") + capture_ready = ( + probe_status in CAPTURE_READY_STATUSES + and isinstance(manifest, Mapping) + and bool(manifest.get("success")) + ) + if capture_ready: + base.update({ + "worker_status": "dry_run_ready_for_public_context_capture", + "ready_for_execution": True, + "capture_execution_ready": True, + "capture_manifest_id": manifest.get("manifest_id"), + "public_browser_context_policy": ( + (manifest.get("public_browser_context") or {}).get("context_policy") + ), + "network_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": "run_pixelrag_platform_probe_worker_execute", + }) + return base + base.update({ + "worker_status": "dry_run_structured_source_fallback_ready", + "ready_for_execution": True, + "capture_execution_ready": False, + "network_call_performed": False, + "artifact_write_performed": False, + "structured_source_fallback": item.get("structured_source_fallback") or {}, + "next_machine_action": "run_structured_source_or_platform_backoff_policy", + }) + return base + + +def _skipped_item(item: Mapping[str, Any]) -> dict[str, Any]: + base = _base_item(item) + base.update({ + "worker_status": "skipped_not_probe_ready", + "ready_for_execution": False, + "capture_execution_ready": False, + "network_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": item.get("next_machine_action") + or "refresh_pixelrag_platform_probe_readiness", + }) + return base + + +def _structured_source_item( + item: Mapping[str, Any], + *, + output_root: Path, + execute: bool, + write_receipt: bool, +) -> dict[str, Any]: + base = _base_item(item) + fallback = item.get("structured_source_fallback") or {} + base.update({ + "worker_status": ( + "executed_structured_source_fallback_package" + if execute + else "dry_run_structured_source_fallback_ready" + ), + "ready_for_execution": True, + "capture_execution_ready": False, + "network_call_performed": False, + "artifact_write_performed": False, + "structured_source_fallback": fallback, + "structured_source_package": { + "adapter_code": fallback.get("adapter_code"), + "available": bool(fallback.get("available")), + "source_count": len(list(fallback.get("sources") or [])), + "network_request_allowed": bool(fallback.get("network_request_allowed")), + "database_write_allowed": False, + "dry_run_only": True, + "blocked_page_not_product_data": True, + }, + "next_machine_action": "run_structured_source_or_platform_backoff_policy", + }) + if execute and write_receipt: + base["receipt_path"] = _write_worker_receipt( + output_root=output_root, + item=item, + worker_item=base, + ) + base["artifact_write_performed"] = True + return base + + +def _run_capture_subprocess( + *, + manifest: Mapping[str, Any], + capture_output_root: Path, + timeout_seconds: int, + settle_ms: int, + max_tiles: int, + capture_script: Path | None = None, +) -> dict[str, Any]: + script = capture_script or _capture_script_path() + completed = subprocess.run( + [ + sys.executable, + str(script), + "--manifest-json", + json.dumps(dict(manifest), ensure_ascii=False), + "--output-dir", + str(capture_output_root), + "--timeout", + str(max(1, int(timeout_seconds or DEFAULT_TIMEOUT_SECONDS))), + "--settle-ms", + str(max(0, int(settle_ms or DEFAULT_SETTLE_MS))), + "--max-tiles", + str(max(1, int(max_tiles or DEFAULT_MAX_TILES))), + ], + capture_output=True, + check=False, + text=True, + ) + try: + payload = json.loads(completed.stdout) + except json.JSONDecodeError: + payload = {} + return { + "returncode": completed.returncode, + "stdout_json": payload, + "stdout_excerpt": completed.stdout[:500], + "stderr_excerpt": completed.stderr[:500], + } + + +def _capture_item( + item: Mapping[str, Any], + *, + output_root: Path, + capture_output_root: Path, + timeout_seconds: int, + settle_ms: int, + max_tiles: int, + write_receipt: bool, + capture_script: Path | None, +) -> dict[str, Any]: + base = _base_item(item) + manifest = item.get("capture_manifest_preview") + if not isinstance(manifest, Mapping) or not manifest.get("success"): + base.update({ + "worker_status": "skipped_missing_capture_manifest", + "ready_for_execution": False, + "capture_execution_ready": False, + "network_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": "refresh_pixelrag_platform_probe_manifest_or_use_structured_source", + }) + if write_receipt: + base["receipt_path"] = _write_worker_receipt( + output_root=output_root, + item=item, + worker_item=base, + ) + base["artifact_write_performed"] = True + return base + + result = _run_capture_subprocess( + manifest=manifest, + capture_output_root=capture_output_root, + timeout_seconds=timeout_seconds, + settle_ms=settle_ms, + max_tiles=max_tiles, + capture_script=capture_script, + ) + capture_payload = result["stdout_json"] + files = list(capture_payload.get("files") or []) if isinstance(capture_payload, Mapping) else [] + tile_file_count = sum(1 for file_item in files if file_item.get("kind") == "tile") + receipt_path = "" + planned_output = capture_payload.get("planned_output") if isinstance(capture_payload, Mapping) else {} + if isinstance(planned_output, Mapping): + receipt_path = str(planned_output.get("receipt") or "") + for file_item in files: + if file_item.get("kind") == "receipt": + receipt_path = str(file_item.get("path") or receipt_path) + + captured_ok = ( + result["returncode"] == 0 + and isinstance(capture_payload, Mapping) + and capture_payload.get("success") is True + and capture_payload.get("status") == "captured" + ) + base.update({ + "worker_status": "executed_capture_ok" if captured_ok else "capture_worker_error", + "ready_for_execution": True, + "capture_execution_ready": True, + "capture_manifest_id": manifest.get("manifest_id"), + "capture_returncode": result["returncode"], + "capture_status": capture_payload.get("status") if isinstance(capture_payload, Mapping) else "", + "capture_http_status": int(capture_payload.get("http_status") or 0) + if isinstance(capture_payload, Mapping) + else 0, + "capture_tile_file_count": tile_file_count, + "capture_receipt_path": receipt_path, + "network_call_performed": True, + "artifact_write_performed": bool(captured_ok), + "capture_stdout_excerpt": result["stdout_excerpt"] if not captured_ok else "", + "capture_stderr_excerpt": result["stderr_excerpt"] if not captured_ok else "", + "next_machine_action": ( + "run_pixelrag_rag_candidate_replay_after_probe_capture" + if captured_ok + else "repair_pixelrag_platform_probe_worker_capture_runtime_or_use_structured_source" + ), + }) + if write_receipt: + base["receipt_path"] = _write_worker_receipt( + output_root=output_root, + item=item, + worker_item=base, + ) + base["artifact_write_performed"] = True + return base + + +def run_pixelrag_platform_probe_worker( + *, + artifact_root: str | Path | None = None, + vlm_receipt_root: str | Path | None = None, + output_root: str | Path | None = None, + capture_output_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, + timeout: int | None = None, + settle_ms: int | None = None, + max_tiles: int | None = None, + execute: bool = False, + write_receipt: bool = False, + capture_script: str | Path | None = None, +) -> dict[str, Any]: + """Run or dry-run public platform probe machine actions.""" + output = Path(output_root or DEFAULT_WORKER_RECEIPT_ROOT) + capture_output = Path(capture_output_root or DEFAULT_CAPTURE_OUTPUT_ROOT) + platforms = _normalise_platforms(platform) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + timeout_seconds = max(1, min(int(timeout or DEFAULT_TIMEOUT_SECONDS), 120)) + settle = max(0, min(int(settle_ms or DEFAULT_SETTLE_MS), 5000)) + tiles = max(1, min(int(max_tiles or DEFAULT_MAX_TILES), 80)) + generated_at = datetime.now(timezone.utc).isoformat() + capture_script_path = Path(capture_script) if capture_script else None + readiness_artifact_root = artifact_root or DEFAULT_ARTIFACT_ROOT + + readiness = build_pixelrag_platform_probe_readiness( + artifact_root=readiness_artifact_root, + vlm_receipt_root=vlm_receipt_root, + platform=platforms, + max_age_hours=max_age, + limit=item_limit, + ) + probe_items = list(readiness.get("probe_items") or []) + worker_items: list[dict[str, Any]] = [] + for item in probe_items: + if not item.get("probe_ready"): + worker_items.append(_skipped_item(item)) + continue + probe_status = str(item.get("probe_status") or "") + manifest = item.get("capture_manifest_preview") + can_capture = ( + probe_status in CAPTURE_READY_STATUSES + and isinstance(manifest, Mapping) + and bool(manifest.get("success")) + ) + if not execute: + worker_items.append(_dry_run_item(item)) + continue + if can_capture: + worker_items.append(_capture_item( + item, + output_root=output, + capture_output_root=capture_output, + timeout_seconds=timeout_seconds, + settle_ms=settle, + max_tiles=tiles, + write_receipt=write_receipt, + capture_script=capture_script_path, + )) + else: + worker_items.append(_structured_source_item( + item, + output_root=output, + execute=True, + write_receipt=write_receipt, + )) + + ready_count = sum(1 for item in probe_items if item.get("probe_ready")) + dry_run_count = sum(1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_")) + structured_count = sum( + 1 for item in worker_items if "structured_source_fallback" in str(item.get("worker_status") or "") + ) + capture_ready_count = sum(1 for item in worker_items if item.get("capture_execution_ready")) + capture_execute_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_capture") + ) + capture_ok_count = sum(1 for item in worker_items if item.get("worker_status") == "executed_capture_ok") + capture_error_count = sum(1 for item in worker_items if item.get("worker_status") == "capture_worker_error") + executed_structured_count = sum( + 1 for item in worker_items if item.get("worker_status") == "executed_structured_source_fallback_package" + ) + executed_count = capture_execute_count + executed_structured_count + skipped_count = sum(1 for item in worker_items if str(item.get("worker_status") or "").startswith("skipped_")) + receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path")) + network_call_performed = any(bool(item.get("network_call_performed")) for item in worker_items) + artifact_write_performed = any(bool(item.get("artifact_write_performed")) for item in worker_items) + + if capture_error_count: + status = "critical" if execute and capture_ok_count == 0 and capture_ready_count else "warning" + elif probe_items and ready_count: + status = "ok" + elif probe_items: + status = "warning" + else: + status = "warning" + + if not probe_items: + next_action = "run_pixelrag_visual_capture_worker" + elif capture_error_count: + next_action = "repair_pixelrag_platform_probe_worker_capture_runtime_or_use_structured_source" + elif not execute and capture_ready_count: + next_action = "run_pixelrag_platform_probe_worker_execute" + elif execute and capture_ok_count: + next_action = "run_pixelrag_rag_candidate_replay_after_probe_capture" + elif structured_count: + next_action = "run_structured_source_or_platform_backoff_policy" + else: + next_action = "refresh_pixelrag_platform_probe_readiness" + + summary = { + "probe_candidate_count": len(probe_items), + "ready_count": ready_count, + "skipped_count": skipped_count, + "dry_run_count": dry_run_count, + "capture_ready_count": capture_ready_count, + "capture_execute_count": capture_execute_count, + "capture_ok_count": capture_ok_count, + "capture_error_count": capture_error_count, + "structured_fallback_count": structured_count, + "executed_structured_count": executed_structured_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "network_call_performed": network_call_performed, + "artifact_write_performed": artifact_write_performed, + "writes_database_count": 0, + "primary_human_gate_count": 0, + "platforms": sorted({str(item.get("platform") or "unknown") for item in probe_items}), + } + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": generated_at, + "artifact_root": str(readiness_artifact_root), + "vlm_receipt_root": str( + vlm_receipt_root + or readiness.get("vlm_receipt_root") + or "" + ), + "output_root": str(output), + "capture_output_root": str(capture_output), + "platform_filter": list(platforms), + "max_age_hours": max_age, + "limit": item_limit, + "timeout_seconds": timeout_seconds, + "settle_ms": settle, + "max_tiles": tiles, + "execute": bool(execute), + "write_receipt": bool(write_receipt), + "summary": summary, + "worker_items": worker_items, + "source_readiness": { + "policy": readiness.get("policy"), + "status": readiness.get("status"), + "summary": readiness.get("summary"), + "next_machine_action": readiness.get("next_machine_action"), + }, + "controlled_apply": { + "network_call": bool(execute and network_call_performed), + "model_call": False, + "artifact_write": artifact_write_performed, + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "secret_read": False, + "raw_cookie_or_session_read": False, + "credentialed_session_allowed": False, + "login_allowed": False, + "production_price_write": False, + "primary_human_gate_count": 0, + }, + "promotion_boundary": { + "writes_ai_insights": False, + "writes_price_tables": False, + "blocked_pages_are_not_product_data": True, + "visual_fields_are_candidate_evidence_only": True, + "requires_rag_candidate_replay_after_capture": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "DEFAULT_CAPTURE_OUTPUT_ROOT", + "DEFAULT_WORKER_RECEIPT_ROOT", + "POLICY", + "run_pixelrag_platform_probe_worker", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index a32752e..06ca503 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1437,11 +1437,12 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_platform_probe_check", lambda: smoke._check("pixelrag platform probe", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_platform_probe_worker_check", lambda: smoke._check("pixelrag platform probe worker", "ok", "ok")) result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 35, "warning": 1, "critical": 1, "total": 37} + assert result["summary"] == {"ok": 36, "warning": 1, "critical": 1, "total": 38} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3974,6 +3975,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_platform_probe_check", lambda: smoke._check("pixelrag platform probe", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_platform_probe_worker_check", lambda: smoke._check("pixelrag platform probe worker", "ok", "ok")) first = smoke.collect_ai_automation_smoke(history_limit=5) second = smoke.collect_ai_automation_smoke(history_limit=5) @@ -4029,7 +4031,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 37, "warning": 0, "critical": 0, "total": 37}, + "summary": {"ok": 38, "warning": 0, "critical": 0, "total": 38}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4212,6 +4214,28 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "writes_database_count": 0, "primary_human_gate_count": 0, }, + }, + { + "name": "PixelRAG platform probe worker", + "status": "ok", + "summary": "PixelRAG platform probe worker candidates=1, ready=1, capture_ready=1, structured_fallback=0, execute=false", + "details": { + "policy": "controlled_pixelrag_platform_probe_worker_v1", + "probe_candidate_count": 1, + "ready_count": 1, + "dry_run_count": 1, + "capture_ready_count": 1, + "capture_execute_count": 0, + "capture_ok_count": 0, + "capture_error_count": 0, + "structured_fallback_count": 0, + "receipt_written_count": 0, + "network_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": "run_pixelrag_platform_probe_worker_execute", + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, } ], }, ensure_ascii=False) + "\n", @@ -4230,7 +4254,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 34 + assert summary["summary"]["total"] == 35 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -5937,6 +5961,15 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( assert pixelrag_platform_probe_family["status"] == "ok" assert pixelrag_platform_probe_family["details"]["ready_for_probe_count"] == 1 assert pixelrag_platform_probe_family["details"]["primary_human_gate_count"] == 0 + pixelrag_platform_probe_worker_family = next( + item for item in summary["families"] + if item["key"] == "pixelrag_platform_probe_worker" + ) + assert pixelrag_platform_probe_worker_family["status"] == "ok" + assert pixelrag_platform_probe_worker_family["details"]["ready_count"] == 1 + assert pixelrag_platform_probe_worker_family["details"]["dry_run_count"] == 1 + assert pixelrag_platform_probe_worker_family["details"]["network_call_performed"] is False + assert pixelrag_platform_probe_worker_family["details"]["writes_database_count"] == 0 assert summary["scheduled_outputs"]["telegram_send_in_preview"] is False assert summary["scheduled_outputs"]["writes_database"] is False assert summary["automation_policy"]["primary_flow"] == "ai_controlled" @@ -6573,6 +6606,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag platform probe ok", )) + monkeypatch.setattr(smoke, "_pixelrag_platform_probe_worker_check", lambda: smoke._check( + "PixelRAG platform probe worker", + "ok", + "pixelrag platform probe worker ok", + )) result = smoke.collect_ai_automation_smoke(record_history=False) @@ -6588,7 +6626,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 37 + assert result["summary"]["total"] == 38 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_platform_probe_worker_service.py b/tests/test_pixelrag_platform_probe_worker_service.py new file mode 100644 index 0000000..8236375 --- /dev/null +++ b/tests/test_pixelrag_platform_probe_worker_service.py @@ -0,0 +1,248 @@ +import json +import subprocess +import sys +from pathlib import Path + +from tests.test_pixelrag_rag_candidate_replay_service import _write_receipt + + +def _patch_receipt(path, **updates): + payload = json.loads(path.read_text(encoding="utf-8")) + for key, value in updates.items(): + if key == "page_metrics": + payload.setdefault("page_metrics", {}).update(value) + else: + payload[key] = value + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return payload + + +def test_pixelrag_platform_probe_worker_dry_run_splits_capture_and_structured(tmp_path): + from services.pixelrag_platform_probe_worker_service import ( + POLICY, + run_pixelrag_platform_probe_worker, + ) + + shopee_receipt = _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-traffic", + title="蝦皮購物 | 花得更少買得更好", + url="https://shopee.tw/search?keyword=sunscreen", + ) + _patch_receipt( + shopee_receipt, + page_metrics={ + "final_url": "https://shopee.tw/verify/traffic/error?home_url=https%3A%2F%2Fshopee.tw", + "title": "蝦皮購物 | 花得更少買得更好", + }, + ) + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + + payload = run_pixelrag_platform_probe_worker( + artifact_root=tmp_path, + platform=("shopee_tw", "coupang_tw"), + ) + by_platform = {item["platform"]: item for item in payload["worker_items"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "ok" + assert payload["execute"] is False + assert payload["summary"]["probe_candidate_count"] == 2 + assert payload["summary"]["ready_count"] == 2 + assert payload["summary"]["dry_run_count"] == 2 + assert payload["summary"]["capture_ready_count"] == 1 + assert payload["summary"]["structured_fallback_count"] == 1 + assert payload["summary"]["network_call_performed"] is False + assert payload["summary"]["writes_database_count"] == 0 + assert payload["controlled_apply"]["primary_human_gate_count"] == 0 + assert payload["next_machine_action"] == "run_pixelrag_platform_probe_worker_execute" + assert by_platform["shopee_tw"]["worker_status"] == "dry_run_ready_for_public_context_capture" + assert by_platform["shopee_tw"]["public_browser_context_policy"] == ( + "public_empty_browser_context_no_login" + ) + assert by_platform["coupang_tw"]["worker_status"] == ( + "dry_run_structured_source_fallback_ready" + ) + assert by_platform["coupang_tw"]["structured_source_fallback_available"] is True + + +def test_pixelrag_platform_probe_worker_execute_capture_writes_receipt(tmp_path, monkeypatch): + from services import pixelrag_platform_probe_worker_service as service + + shopee_receipt = _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-traffic", + title="蝦皮購物 | 花得更少買得更好", + url="https://shopee.tw/search?keyword=sunscreen", + ) + _patch_receipt( + shopee_receipt, + page_metrics={ + "final_url": "https://shopee.tw/verify/traffic/error?home_url=https%3A%2F%2Fshopee.tw", + "title": "蝦皮購物 | 花得更少買得更好", + }, + ) + + def fake_capture_subprocess(**kwargs): + manifest = kwargs["manifest"] + capture_root = Path(kwargs["capture_output_root"]) + receipt_path = capture_root / "shopee_tw" / manifest["manifest_id"] / "capture_receipt.json" + return { + "returncode": 0, + "stdout_json": { + "success": True, + "status": "captured", + "http_status": 200, + "planned_output": {"receipt": str(receipt_path)}, + "files": [ + {"kind": "fullpage_screenshot", "path": str(receipt_path.parent / "fullpage.png")}, + {"kind": "tile", "path": str(receipt_path.parent / "tiles" / "tile_000.png")}, + {"kind": "receipt", "path": str(receipt_path)}, + ], + }, + "stdout_excerpt": "", + "stderr_excerpt": "", + } + + monkeypatch.setattr(service, "_run_capture_subprocess", fake_capture_subprocess) + payload = service.run_pixelrag_platform_probe_worker( + artifact_root=tmp_path, + output_root=tmp_path / "worker_receipts", + capture_output_root=tmp_path / "capture_output", + platform="shopee_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["capture_execute_count"] == 1 + assert payload["summary"]["capture_ok_count"] == 1 + assert payload["summary"]["receipt_written_count"] == 1 + assert payload["summary"]["network_call_performed"] is True + assert payload["controlled_apply"]["network_call"] is True + assert payload["controlled_apply"]["writes_database"] is False + item = payload["worker_items"][0] + assert item["worker_status"] == "executed_capture_ok" + assert item["next_machine_action"] == "run_pixelrag_rag_candidate_replay_after_probe_capture" + receipt_path = ( + tmp_path + / "worker_receipts" + / "shopee_tw" + / "shopee-traffic" + / "platform_probe_worker_receipt.json" + ) + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "executed_capture_ok" + assert receipt["artifact_write_performed"] is True + assert receipt["receipt_path"] == str(receipt_path) + + +def test_pixelrag_platform_probe_worker_execute_structured_fallback_writes_receipt(tmp_path): + from services.pixelrag_platform_probe_worker_service import run_pixelrag_platform_probe_worker + + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + + payload = run_pixelrag_platform_probe_worker( + artifact_root=tmp_path, + output_root=tmp_path / "worker_receipts", + platform="coupang_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["executed_structured_count"] == 1 + assert payload["summary"]["capture_execute_count"] == 0 + assert payload["summary"]["network_call_performed"] is False + assert payload["summary"]["receipt_written_count"] == 1 + item = payload["worker_items"][0] + assert item["worker_status"] == "executed_structured_source_fallback_package" + assert item["structured_source_package"]["adapter_code"] == "coupang" + assert item["structured_source_package"]["database_write_allowed"] is False + receipt_path = ( + tmp_path + / "worker_receipts" + / "coupang_tw" + / "coupang-403" + / "platform_probe_worker_receipt.json" + ) + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "executed_structured_source_fallback_package" + assert receipt["artifact_write_performed"] is True + + +def test_pixelrag_platform_probe_worker_cli_outputs_machine_readable_json(tmp_path): + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/run_pixelrag_platform_probe_worker.py", + "--artifact-root", + str(tmp_path), + "--platform", + "coupang_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["probe_candidate_count"] == 1 + assert payload["summary"]["structured_fallback_count"] == 1 + assert payload["controlled_apply"]["network_call"] is False + assert payload["controlled_apply"]["writes_database"] is False + + +def test_pixelrag_platform_probe_worker_route_returns_readback(tmp_path, monkeypatch): + from flask import Flask + from routes import system_public_routes as routes + from services import pixelrag_platform_probe_worker_service as service + + _write_receipt( + tmp_path, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + monkeypatch.setattr(service, "DEFAULT_ARTIFACT_ROOT", str(tmp_path)) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-platform-probe-worker?platform=coupang_tw" + ): + response = routes.ai_automation_pixelrag_platform_probe_worker_api.__wrapped__() + payload = response.get_json() + + assert payload["policy"] == "controlled_pixelrag_platform_probe_worker_v1" + assert payload["summary"]["probe_candidate_count"] == 1 + assert payload["summary"]["structured_fallback_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False From 39d9918f12564d509097677046798e6802f7bc4a Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 10:03:22 +0800 Subject: [PATCH 19/75] fix(ai): fallback PixelRAG platform probe runtime gaps --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 2 + .../ai_automation_mainline_work_items.md | 2 +- .../pixelrag_platform_probe_worker_service.py | 83 +++++++++++++++++-- ..._pixelrag_platform_probe_worker_service.py | 62 ++++++++++++++ 5 files changed, 143 insertions(+), 8 deletions(-) diff --git a/config.py b/config.py index b69ad11..fc4b4b1 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.760" +SYSTEM_VERSION = "V10.761" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index f6f9d50..5bd159b 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -120,6 +120,7 @@ - 2026-07-09 起 PixelRAG VLM route readiness 必須在 execute 前可讀回：`/api/ai-automation/pixelrag-vlm-route-readiness` 與 `scripts/ops/report_pixelrag_vlm_route_readiness.py` 只讀 approved Ollama `/api/tags`，輸出 configured model、candidate model、reachable host、model_route_ready 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG VLM route readiness`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_vlm_route_readiness` family。此 readback 不呼叫 `/api/generate`、不讀 secret、不寫 DB；worker execute 必須使用它自動避開缺模型路由，完全缺候選時寫 `model_route_not_ready` artifact receipt。 - 2026-07-10 起 PixelRAG platform probe 必須把 blocked / interstitial receipt 轉成機器可執行下一步：`/api/ai-automation/pixelrag-platform-probe` 與 `scripts/ops/report_pixelrag_platform_probe.py` 需讀取 capture receipts 與 VLM replay receipts，辨識 Shopee language / region / generic marketplace slogan / traffic verification 與 Coupang 403 / access denied，輸出 public browser context probe、structured source fallback、platform backoff、blocked page not product data 與 no-write 邊界；`/api/ai-automation/smoke` 需包含 `PixelRAG platform probe readiness`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_platform_probe` family。此 readback 不抓外站、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture worker 可消化 `public_browser_context` 的 locale/timezone/Accept-Language，但必須濾掉 Cookie、Authorization、token/secret/key 類 header。 - 2026-07-10 起 PixelRAG platform probe 必須有 controlled worker：`/api/ai-automation/pixelrag-platform-probe-worker` 與 `scripts/ops/run_pixelrag_platform_probe_worker.py` 預設 dry-run，`execute=true&write_receipt=true` 才會對 Shopee language / traffic / generic landing 執行 public empty-context capture，並對 Coupang 403 / access denied 自動產生 structured-source/backoff artifact receipt；`/api/ai-automation/smoke` 需包含 `PixelRAG platform probe worker`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_platform_probe_worker` family。此 worker 不登入、不讀 cookie/session/secret、不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture 成功後下一步固定為 RAG candidate replay，structured fallback 只產生機器可讀 package，不把 blocked page 當商品資料。 +- 2026-07-10 起 PixelRAG platform probe worker 必須自動處理 capture runtime gap：若 production app 容器缺 Playwright，`execute=true&write_receipt=true` 不得退回人工審核或 critical dead-end；worker 必須寫 `capture_runtime_unavailable_structured_fallback_package` artifact receipt，標記 `network_call=false`、`writes_database=false`、`primary_human_gate_count=0`，並把下一步改為 `run_structured_source_or_install_pixelrag_capture_runtime`。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -892,6 +893,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG VLM language note / generic marketplace slogan 必須視為 interstitial | V10.758 起若 VLM notes 或 title 顯示 `Language selection page`、語言/地區選擇，或只抽到 Shopee `蝦皮購物 | 花得更少買得更好` 這類網站 slogan 且 price/shop 缺失，worker validation 會標記 `interstitial_signal_detected` / `generic_marketplace_title_detected` 與 `non_product_or_interstitial_detected=true`，run-level 與 item-level 下一步都輸出 `run_platform_probe_or_use_structured_api`。 | | 2026-07-10 | PixelRAG platform probe 必須自動消化 marketplace interstitial / blocked receipt | V10.759 起 `/api/ai-automation/pixelrag-platform-probe`、`scripts/ops/report_pixelrag_platform_probe.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 platform probe / `pixelrag_platform_probe` family；Shopee traffic / language / generic landing 會轉成 `run_shopee_public_context_probe_then_structured_source_fallback`，Coupang 403 / access denied 會轉成 `use_structured_source_or_platform_backoff_policy`。probe plan 只產生 public empty browser context、structured adapter fallback 與 no-write contract，不讀或注入 cookie/session/login，不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture worker 只允許 locale/timezone/非敏感 header 並濾掉 Cookie / Authorization / token / secret / key 類 header。 | | 2026-07-10 | PixelRAG platform probe worker 必須把 probe plan 變成 controlled automation | V10.760 起 `/api/ai-automation/pixelrag-platform-probe-worker`、`scripts/ops/run_pixelrag_platform_probe_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled platform probe worker / `pixelrag_platform_probe_worker` family；dry-run 只規劃 capture 或 structured fallback，execute 模式才執行 public empty-context capture 或寫 structured-source/backoff artifact receipt。此 worker 明確標記 `network_call` 只在 capture execute 時為 true、`model_call=false`、`writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`raw_cookie_or_session_read=false`、`primary_human_gate_count=0`；capture 成功後必須回到 PixelRAG RAG candidate replay，不得直接寫正式價格或把 blocked page 當商品資料。 | +| 2026-07-10 | PixelRAG platform probe worker 缺 capture runtime 時必須自動降級 | V10.761 起若 `run_pixelrag_platform_probe_worker.py --execute --write-receipt` 在 app 容器偵測到 `ModuleNotFoundError: playwright`，worker 會自動產生 `capture_runtime_unavailable_structured_fallback_package` receipt，`network_call=false`、`artifact_write=true`、`writes_database=false`、`primary_human_gate_count=0`，不再把 Shopee/Coupang platform probe runtime gap 變成人工主流程。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index e146883..bb5f192 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -20,7 +20,7 @@ | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | -| Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture, Coupang 403/access denied goes to structured-source/backoff artifact receipts. | +| Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture when runtime is available, missing Playwright auto-falls back to structured-source/backoff receipt, and Coupang 403/access denied goes to structured-source/backoff artifact receipts. | | In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, and PixelRAG platform probe worker families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | diff --git a/services/pixelrag_platform_probe_worker_service.py b/services/pixelrag_platform_probe_worker_service.py index 5d3b76c..21f92f6 100644 --- a/services/pixelrag_platform_probe_worker_service.py +++ b/services/pixelrag_platform_probe_worker_service.py @@ -238,14 +238,40 @@ def _run_capture_subprocess( payload = json.loads(completed.stdout) except json.JSONDecodeError: payload = {} + stderr_excerpt = completed.stderr[:500] + if len(completed.stderr) > 500: + stderr_excerpt = f"{completed.stderr[:250]}...{completed.stderr[-250:]}" return { "returncode": completed.returncode, "stdout_json": payload, "stdout_excerpt": completed.stdout[:500], - "stderr_excerpt": completed.stderr[:500], + "stderr_excerpt": stderr_excerpt, } +def _capture_runtime_unavailable(result: Mapping[str, Any]) -> bool: + haystack = " ".join( + [ + str(result.get("stdout_excerpt") or ""), + str(result.get("stderr_excerpt") or ""), + ] + ).lower() + return ( + "modulenotfounderror" in haystack + and "playwright" in haystack + ) or "no module named 'playwright'" in haystack or ( + "from playwright.sync_api" in haystack + and "traceback" in haystack + ) or ( + "playwright" in haystack + and ( + "executable doesn't exist" in haystack + or "please run" in haystack + or "browsertype.launch" in haystack + ) + ) + + def _capture_item( item: Mapping[str, Any], *, @@ -302,8 +328,25 @@ def _capture_item( and capture_payload.get("success") is True and capture_payload.get("status") == "captured" ) + runtime_missing = not captured_ok and _capture_runtime_unavailable(result) + structured_fallback_available = bool( + (item.get("structured_source_fallback") or {}).get("available") + ) + capture_fallback = not captured_ok and structured_fallback_available base.update({ - "worker_status": "executed_capture_ok" if captured_ok else "capture_worker_error", + "worker_status": ( + "executed_capture_ok" + if captured_ok + else ( + ( + "capture_runtime_unavailable_structured_fallback_package" + if runtime_missing + else "capture_failed_structured_fallback_package" + ) + if capture_fallback + else "capture_worker_error" + ) + ), "ready_for_execution": True, "capture_execution_ready": True, "capture_manifest_id": manifest.get("manifest_id"), @@ -314,14 +357,39 @@ def _capture_item( else 0, "capture_tile_file_count": tile_file_count, "capture_receipt_path": receipt_path, - "network_call_performed": True, + "network_call_performed": bool(not runtime_missing), "artifact_write_performed": bool(captured_ok), + "structured_source_fallback": item.get("structured_source_fallback") or {}, + "structured_source_package": ( + { + "adapter_code": (item.get("structured_source_fallback") or {}).get("adapter_code"), + "available": True, + "source_count": len(list((item.get("structured_source_fallback") or {}).get("sources") or [])), + "network_request_allowed": bool( + (item.get("structured_source_fallback") or {}).get("network_request_allowed") + ), + "database_write_allowed": False, + "dry_run_only": True, + "blocked_page_not_product_data": True, + "capture_runtime_unavailable": runtime_missing, + } + if capture_fallback + else None + ), "capture_stdout_excerpt": result["stdout_excerpt"] if not captured_ok else "", "capture_stderr_excerpt": result["stderr_excerpt"] if not captured_ok else "", "next_machine_action": ( "run_pixelrag_rag_candidate_replay_after_probe_capture" if captured_ok - else "repair_pixelrag_platform_probe_worker_capture_runtime_or_use_structured_source" + else ( + ( + "run_structured_source_or_install_pixelrag_capture_runtime" + if runtime_missing + else "run_structured_source_or_retry_platform_capture" + ) + if capture_fallback + else "repair_pixelrag_platform_probe_worker_capture_runtime_or_use_structured_source" + ) ), }) if write_receipt: @@ -408,7 +476,7 @@ def run_pixelrag_platform_probe_worker( ready_count = sum(1 for item in probe_items if item.get("probe_ready")) dry_run_count = sum(1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_")) structured_count = sum( - 1 for item in worker_items if "structured_source_fallback" in str(item.get("worker_status") or "") + 1 for item in worker_items if "structured" in str(item.get("worker_status") or "") ) capture_ready_count = sum(1 for item in worker_items if item.get("capture_execution_ready")) capture_execute_count = sum( @@ -417,7 +485,10 @@ def run_pixelrag_platform_probe_worker( capture_ok_count = sum(1 for item in worker_items if item.get("worker_status") == "executed_capture_ok") capture_error_count = sum(1 for item in worker_items if item.get("worker_status") == "capture_worker_error") executed_structured_count = sum( - 1 for item in worker_items if item.get("worker_status") == "executed_structured_source_fallback_package" + 1 + for item in worker_items + if "structured" in str(item.get("worker_status") or "") + and not str(item.get("worker_status") or "").startswith("dry_run_") ) executed_count = capture_execute_count + executed_structured_count skipped_count = sum(1 for item in worker_items if str(item.get("worker_status") or "").startswith("skipped_")) diff --git a/tests/test_pixelrag_platform_probe_worker_service.py b/tests/test_pixelrag_platform_probe_worker_service.py index 8236375..658d6e1 100644 --- a/tests/test_pixelrag_platform_probe_worker_service.py +++ b/tests/test_pixelrag_platform_probe_worker_service.py @@ -147,6 +147,68 @@ def test_pixelrag_platform_probe_worker_execute_capture_writes_receipt(tmp_path, assert receipt["receipt_path"] == str(receipt_path) +def test_pixelrag_platform_probe_worker_missing_playwright_falls_back_to_structured_receipt( + tmp_path, + monkeypatch, +): + from services import pixelrag_platform_probe_worker_service as service + + shopee_receipt = _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-traffic", + title="蝦皮購物 | 花得更少買得更好", + url="https://shopee.tw/search?keyword=sunscreen", + ) + _patch_receipt( + shopee_receipt, + page_metrics={ + "final_url": "https://shopee.tw/verify/traffic/error?home_url=https%3A%2F%2Fshopee.tw", + "title": "蝦皮購物 | 花得更少買得更好", + }, + ) + + monkeypatch.setattr( + service, + "_run_capture_subprocess", + lambda **kwargs: { + "returncode": 1, + "stdout_json": {}, + "stdout_excerpt": "", + "stderr_excerpt": "ModuleNotFoundError: No module named 'playwright'", + }, + ) + payload = service.run_pixelrag_platform_probe_worker( + artifact_root=tmp_path, + output_root=tmp_path / "worker_receipts", + platform="shopee_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["capture_error_count"] == 0 + assert payload["summary"]["structured_fallback_count"] == 1 + assert payload["summary"]["executed_structured_count"] == 1 + assert payload["summary"]["network_call_performed"] is False + assert payload["summary"]["receipt_written_count"] == 1 + item = payload["worker_items"][0] + assert item["worker_status"] == "capture_runtime_unavailable_structured_fallback_package" + assert item["structured_source_package"]["capture_runtime_unavailable"] is True + assert item["next_machine_action"] == "run_structured_source_or_install_pixelrag_capture_runtime" + receipt_path = ( + tmp_path + / "worker_receipts" + / "shopee_tw" + / "shopee-traffic" + / "platform_probe_worker_receipt.json" + ) + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "capture_runtime_unavailable_structured_fallback_package" + assert receipt["artifact_write_performed"] is True + assert receipt["network_call_performed"] is False + + def test_pixelrag_platform_probe_worker_execute_structured_fallback_writes_receipt(tmp_path): from services.pixelrag_platform_probe_worker_service import run_pixelrag_platform_probe_worker From 905667ff4c425f7363917da7aa2e13d7fcf00712 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 10:22:20 +0800 Subject: [PATCH 20/75] feat(ai): replay platform probe worker receipts --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 5 +- .../ai_automation_mainline_work_items.md | 4 +- routes/system_public_routes.py | 3 + .../report_pixelrag_rag_candidate_replay.py | 5 + services/ai_automation_smoke_service.py | 22 +- .../pixelrag_rag_candidate_replay_service.py | 218 ++++++++++++++++-- ...t_pixelrag_rag_candidate_replay_service.py | 138 ++++++++++- 8 files changed, 376 insertions(+), 21 deletions(-) diff --git a/config.py b/config.py index fc4b4b1..2488ffa 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.761" +SYSTEM_VERSION = "V10.762" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 5bd159b..9fc5563 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,8 +1,8 @@ # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth -> **最後更新**: 2026-07-09 (台北時間) +> **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.748 +> **適用版本**: V10.762 --- @@ -894,6 +894,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG platform probe 必須自動消化 marketplace interstitial / blocked receipt | V10.759 起 `/api/ai-automation/pixelrag-platform-probe`、`scripts/ops/report_pixelrag_platform_probe.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 platform probe / `pixelrag_platform_probe` family；Shopee traffic / language / generic landing 會轉成 `run_shopee_public_context_probe_then_structured_source_fallback`，Coupang 403 / access denied 會轉成 `use_structured_source_or_platform_backoff_policy`。probe plan 只產生 public empty browser context、structured adapter fallback 與 no-write contract，不讀或注入 cookie/session/login，不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture worker 只允許 locale/timezone/非敏感 header 並濾掉 Cookie / Authorization / token / secret / key 類 header。 | | 2026-07-10 | PixelRAG platform probe worker 必須把 probe plan 變成 controlled automation | V10.760 起 `/api/ai-automation/pixelrag-platform-probe-worker`、`scripts/ops/run_pixelrag_platform_probe_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled platform probe worker / `pixelrag_platform_probe_worker` family；dry-run 只規劃 capture 或 structured fallback，execute 模式才執行 public empty-context capture 或寫 structured-source/backoff artifact receipt。此 worker 明確標記 `network_call` 只在 capture execute 時為 true、`model_call=false`、`writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`raw_cookie_or_session_read=false`、`primary_human_gate_count=0`；capture 成功後必須回到 PixelRAG RAG candidate replay，不得直接寫正式價格或把 blocked page 當商品資料。 | | 2026-07-10 | PixelRAG platform probe worker 缺 capture runtime 時必須自動降級 | V10.761 起若 `run_pixelrag_platform_probe_worker.py --execute --write-receipt` 在 app 容器偵測到 `ModuleNotFoundError: playwright`，worker 會自動產生 `capture_runtime_unavailable_structured_fallback_package` receipt，`network_call=false`、`artifact_write=true`、`writes_database=false`、`primary_human_gate_count=0`，不再把 Shopee/Coupang platform probe runtime gap 變成人工主流程。 | +| 2026-07-10 | PixelRAG platform probe worker receipts 必須回到 RAG/source-contract replay | V10.762 起 `/api/ai-automation/pixelrag-rag-candidate-replay` 與 `scripts/ops/report_pixelrag_rag_candidate_replay.py` 必須同時讀 visual capture receipts 與 `platform_probe_worker_receipt.json`；worker receipts 只能成為 `source_contract_fallback` / `eligible_for_source_contract_replay` 候選，並輸出 `platform_probe_worker_receipt_count`、`source_contract_fallback_count`、`capture_runtime_gap_count` 與下一步 `run_source_contract_replay_for_platform_probe_worker_receipts`。此路徑不寫 DB、不寫 `ai_insights`、不寫正式價格表、不把 blocked page 當商品資料，也不得退回人工審核主流程。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index bb5f192..e814f88 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -16,7 +16,7 @@ |---|---|---| | Completed | Multi-commerce PixelRAG visual evidence lane for momo, pchome, shopee_tw, coupang_tw, yahoo_shopping_tw, etmall_tw, friday_tw, rakuten_tw | API and CLI can emit expansion plans and marketplace manifests; PChome parser failures can queue manifests. | | Completed | External MCP/RAG capability inventory absorbed into internal governance readback | `/api/ai-automation/external-mcp-rag-integration` and `scripts/ops/report_external_mcp_rag_integration.py` expose 9 capabilities, absorbed/unresolved counts, and runtime flags. | -| Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read receipts, split eligible vs blocked, and require OCR/VLM replay plus PromotionGate before knowledge writes. | +| Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read visual capture receipts and platform probe worker receipts, split eligible / blocked / source-contract-fallback candidates, and require OCR/VLM replay, source contract replay, identity matching, plus PromotionGate before knowledge writes. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | @@ -33,7 +33,7 @@ | Not started | Ollama-first multimodal embedding benchmark | Verify local Qwen3-VL or equivalent visual embedding on GCP-A -> GCP-B -> 111 before any visual vector retrieval. | | Not started | pgvector-compatible visual evidence metadata | Design metadata-first retrieval without FAISS in production unless ADR approves a different store. | | In progress | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; `run_pixelrag_platform_probe_worker.py --execute --write-receipt` now emits structured-source/backoff receipts without network, DB writes, or product-data promotion. | -| Not started | Marketplace source contracts beyond MOMO/PChome | Promote stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten receipts into source contracts only after provenance, rate-limit, public boundary, and replay evidence. | +| In progress | Marketplace source contracts beyond MOMO/PChome | Platform probe worker receipts now enter RAG candidate replay as source-contract fallback candidates; next machine action is to replay stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten source contracts only after provenance, rate-limit, public boundary, and replay evidence. | | In progress | Professional product website UI/UX text reduction | Continue compact AI surfaces, visual QA, and user-facing copy guardrails; avoid engineering logs or work-session prose in the UI. | ## Always Enforced diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 5974c5a..9f31159 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -705,6 +705,9 @@ def ai_automation_pixelrag_rag_candidate_replay_api(): max_age_hours = request.args.get('max_age_hours', 168, type=int) limit = request.args.get('limit', 50, type=int) return jsonify(build_pixelrag_rag_candidate_replay_readback( + platform_probe_worker_receipt_root=str( + request.args.get('platform_probe_worker_receipt_root') or '' + ).strip() or None, platform=platforms, max_age_hours=max(1, min(max_age_hours or 168, 720)), limit=max(1, min(limit or 50, 250)), diff --git a/scripts/ops/report_pixelrag_rag_candidate_replay.py b/scripts/ops/report_pixelrag_rag_candidate_replay.py index 9520f8c..50befd6 100755 --- a/scripts/ops/report_pixelrag_rag_candidate_replay.py +++ b/scripts/ops/report_pixelrag_rag_candidate_replay.py @@ -26,6 +26,10 @@ def main() -> int: "--artifact-root", help="PixelRAG visual evidence artifact root；預設使用 production/container 設定。", ) + parser.add_argument( + "--platform-probe-worker-receipt-root", + help="PixelRAG platform probe worker receipt root；預設使用 production/container 設定。", + ) parser.add_argument( "--platform", action="append", @@ -48,6 +52,7 @@ def main() -> int: payload = build_pixelrag_rag_candidate_replay_readback( artifact_root=args.artifact_root, + platform_probe_worker_receipt_root=args.platform_probe_worker_receipt_root, platform=tuple(args.platforms or ()), max_age_hours=args.max_age_hours, limit=args.limit, diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index ef1bc27..2e98d00 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -4603,6 +4603,18 @@ def build_scheduled_automation_health_summary( "eligible_count": int(pixelrag_rag_replay_details.get("eligible_count") or 0), "blocked_count": int(pixelrag_rag_replay_details.get("blocked_count") or 0), "invalid_count": int(pixelrag_rag_replay_details.get("invalid_count") or 0), + "visual_receipt_count": int( + pixelrag_rag_replay_details.get("visual_receipt_count") or 0 + ), + "platform_probe_worker_receipt_count": int( + pixelrag_rag_replay_details.get("platform_probe_worker_receipt_count") or 0 + ), + "source_contract_fallback_count": int( + pixelrag_rag_replay_details.get("source_contract_fallback_count") or 0 + ), + "capture_runtime_gap_count": int( + pixelrag_rag_replay_details.get("capture_runtime_gap_count") or 0 + ), "visual_barrier_count": int( pixelrag_rag_replay_details.get("visual_barrier_count") or 0 ), @@ -13448,11 +13460,15 @@ def _pixelrag_rag_candidate_replay_check() -> Dict[str, Any]: blocked_count = int(summary.get("blocked_count") or 0) invalid_count = int(summary.get("invalid_count") or 0) visual_barrier_count = int(summary.get("visual_barrier_count") or 0) + worker_receipt_count = int(summary.get("platform_probe_worker_receipt_count") or 0) + source_contract_fallback_count = int(summary.get("source_contract_fallback_count") or 0) + capture_runtime_gap_count = int(summary.get("capture_runtime_gap_count") or 0) status = readback.get("status") or "warning" summary_text = ( f"PixelRAG RAG candidate replay receipts={receipt_count}, " f"eligible={eligible_count}, blocked={blocked_count}, invalid={invalid_count}, " - f"barriers={visual_barrier_count}" + f"barriers={visual_barrier_count}, worker_receipts={worker_receipt_count}, " + f"source_contract_fallback={source_contract_fallback_count}" ) return _check( "PixelRAG RAG candidate replay", @@ -13465,6 +13481,10 @@ def _pixelrag_rag_candidate_replay_check() -> Dict[str, Any]: "blocked_count": blocked_count, "invalid_count": invalid_count, "visual_barrier_count": visual_barrier_count, + "visual_receipt_count": int(summary.get("visual_receipt_count") or 0), + "platform_probe_worker_receipt_count": worker_receipt_count, + "source_contract_fallback_count": source_contract_fallback_count, + "capture_runtime_gap_count": capture_runtime_gap_count, "tile_file_count": int(summary.get("tile_file_count") or 0), "missing_file_count": int(summary.get("missing_file_count") or 0), "stale_count": int(summary.get("stale_count") or 0), diff --git a/services/pixelrag_rag_candidate_replay_service.py b/services/pixelrag_rag_candidate_replay_service.py index c390e21..9a33d6f 100644 --- a/services/pixelrag_rag_candidate_replay_service.py +++ b/services/pixelrag_rag_candidate_replay_service.py @@ -7,6 +7,7 @@ anything into the production price tables. from __future__ import annotations +import os import json from datetime import datetime, timezone from pathlib import Path @@ -27,6 +28,12 @@ from services.pixelrag_crawler_integration_service import ( POLICY = "read_only_pixelrag_rag_candidate_replay_v1" INTERNAL_RAG_TARGET = "rag_service.learning_episode_candidate_preview" DEFAULT_LIMIT = 50 +DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT = os.getenv( + "PIXELRAG_PLATFORM_PROBE_WORKER_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_platform_probe_worker_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_platform_probe_worker_receipts", +) def _status_counts(items: list[dict[str, Any]]) -> dict[str, int]: @@ -197,6 +204,150 @@ def _candidate_from_receipt_path( } +def _platform_probe_worker_receipt_candidates( + root: Path, + *, + platforms: tuple[str, ...] | list[str] | None, + limit: int, +) -> list[Path]: + if not root.exists(): + return [] + selected_platforms = [ + str(platform or "").strip().lower() + for platform in (platforms or []) + if str(platform or "").strip() + ] + candidates: list[Path] = [] + if selected_platforms: + for platform in selected_platforms: + candidates.extend((root / platform).glob("*/platform_probe_worker_receipt.json")) + else: + candidates.extend(root.glob("*/*/platform_probe_worker_receipt.json")) + return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] + + +def _candidate_from_platform_probe_worker_receipt_path( + receipt_path: Path, + *, + now: datetime, + max_age_hours: int, +) -> dict[str, Any]: + errors: list[str] = [] + try: + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + receipt = {} + errors.append(str(exc)[:300]) + + try: + file_generated_at = datetime.fromtimestamp(receipt_path.stat().st_mtime, tz=timezone.utc) + except OSError: + file_generated_at = None + generated_at = _parse_iso_datetime(receipt.get("generated_at")) or file_generated_at + age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None + stale = age_hours is None or age_hours > max_age_hours + platform = str(receipt.get("platform") or receipt_path.parent.parent.name).strip().lower() + manifest_id = str(receipt.get("manifest_id") or receipt_path.parent.name).strip() + worker_status = str(receipt.get("worker_status") or "").strip() + structured_package = ( + receipt.get("structured_source_package") + if isinstance(receipt.get("structured_source_package"), Mapping) + else {} + ) + structured_available = bool(structured_package.get("available")) + capture_runtime_gap = bool(structured_package.get("capture_runtime_unavailable")) or ( + worker_status == "capture_runtime_unavailable_structured_fallback_package" + ) + capture_receipt_path = str(receipt.get("capture_receipt_path") or "").strip() + source_receipt_path = str(receipt.get("source_receipt_path") or "").strip() + blocked_reasons: list[str] = [] + if errors: + blocked_reasons.append("platform_probe_worker_receipt_parse_error") + if stale: + blocked_reasons.append("platform_probe_worker_receipt_stale_or_missing_timestamp") + if capture_runtime_gap: + blocked_reasons.append("capture_runtime_unavailable") + barrier_type = str(receipt.get("barrier_type") or "").strip() + if barrier_type: + blocked_reasons.append(f"platform_barrier:{barrier_type}") + + if errors: + candidate_status = "invalid" + elif structured_available: + candidate_status = "source_contract_fallback" + elif worker_status == "executed_capture_ok" and capture_receipt_path: + candidate_status = "probe_capture_ready" + else: + candidate_status = "worker_handoff" + + rag_text_parts = [ + f"platform={platform}", + f"manifest_id={manifest_id}", + f"worker_status={worker_status or 'unknown'}", + f"probe_status={receipt.get('probe_status') or 'unknown'}", + f"barrier_type={barrier_type or 'none'}", + f"source_receipt_path={source_receipt_path or 'unknown'}", + f"capture_receipt_path={capture_receipt_path or 'none'}", + f"structured_adapter={structured_package.get('adapter_code') or 'none'}", + f"structured_source_count={int(structured_package.get('source_count') or 0)}", + f"capture_runtime_unavailable={capture_runtime_gap}", + ] + return { + "candidate_status": candidate_status, + "source_type": "platform_probe_worker_receipt", + "eligible_for_rag_candidate_replay": False, + "eligible_for_source_contract_replay": bool(structured_available), + "blocked_reasons": blocked_reasons, + "platform": platform, + "manifest_id": manifest_id, + "receipt_path": str(receipt_path), + "generated_at": generated_at.isoformat() if generated_at else None, + "age_hours": round(age_hours, 3) if age_hours is not None else None, + "stale": stale, + "http_status": 0, + "capture_status": worker_status, + "url": receipt.get("url"), + "title": receipt.get("title"), + "visual_barrier_detected": bool(barrier_type), + "visual_barrier_reason": barrier_type, + "tile_file_count": 0, + "missing_file_count": 0, + "platform_probe_worker": { + "worker_status": worker_status, + "probe_status": receipt.get("probe_status"), + "barrier_type": barrier_type, + "source_receipt_path": source_receipt_path, + "capture_receipt_path": capture_receipt_path, + "network_call_performed": bool(receipt.get("network_call_performed")), + "artifact_write_performed": bool(receipt.get("artifact_write_performed")), + "writes_database": bool(receipt.get("writes_database")), + "next_machine_action": receipt.get("next_machine_action"), + "structured_source_package": structured_package, + }, + "rag_candidate_text": " | ".join(rag_text_parts), + "internal_rag_target": INTERNAL_RAG_TARGET, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "requires_ocr_or_vlm_replay": False, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + "requires_source_contract_before_knowledge_write": True, + }, + "next_machine_action": ( + "run_source_contract_replay_for_platform_probe_worker_receipt" + if structured_available + else ( + "run_pixelrag_rag_candidate_replay_after_probe_capture" + if capture_receipt_path + else "refresh_pixelrag_platform_probe_worker_receipt" + ) + ), + "file_summary": {"tile_file_count": 0, "missing_file_count": 0, "files": []}, + } + + def _unique_receipt_candidates( root: Path, *, @@ -229,12 +380,14 @@ def _unique_receipt_candidates( def build_pixelrag_rag_candidate_replay_readback( *, artifact_root: str | Path | None = None, + platform_probe_worker_receipt_root: str | Path | None = None, platform: str | tuple[str, ...] | list[str] | None = None, max_age_hours: int | None = None, limit: int | None = None, ) -> dict[str, Any]: """Build a replayable RAG candidate readback from PixelRAG receipts.""" root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT) + worker_root = Path(platform_probe_worker_receipt_root or DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT) max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) if isinstance(platform, str): @@ -252,35 +405,66 @@ def build_pixelrag_rag_candidate_replay_readback( _candidate_from_receipt_path(path, now=now, max_age_hours=max_age) for path in candidates ] + worker_receipts = _platform_probe_worker_receipt_candidates( + worker_root, + platforms=platforms, + limit=item_limit, + ) + worker_candidates = [ + _candidate_from_platform_probe_worker_receipt_path( + path, + now=now, + max_age_hours=max_age, + ) + for path in worker_receipts + ] + all_candidates = replay_candidates + worker_candidates eligible_count = sum(1 for item in replay_candidates if item["candidate_status"] == "eligible") blocked_count = sum(1 for item in replay_candidates if item["candidate_status"] == "blocked") - invalid_count = sum(1 for item in replay_candidates if item["candidate_status"] == "invalid") + invalid_count = sum(1 for item in all_candidates if item["candidate_status"] == "invalid") visual_barrier_count = sum(1 for item in replay_candidates if item["visual_barrier_detected"]) missing_file_count = sum(int(item.get("missing_file_count") or 0) for item in replay_candidates) tile_file_count = sum(int(item.get("tile_file_count") or 0) for item in replay_candidates) - stale_count = sum(1 for item in replay_candidates if item.get("stale")) + stale_count = sum(1 for item in all_candidates if item.get("stale")) + platform_probe_worker_receipt_count = len(worker_candidates) + source_contract_fallback_count = sum( + 1 for item in worker_candidates if item.get("candidate_status") == "source_contract_fallback" + ) + capture_runtime_gap_count = sum( + 1 + for item in worker_candidates + if (item.get("platform_probe_worker") or {}).get("structured_source_package", {}).get( + "capture_runtime_unavailable" + ) + ) - if invalid_count and invalid_count == len(replay_candidates): + if invalid_count and invalid_count == len(all_candidates): status = "critical" elif eligible_count == 0 or blocked_count or invalid_count or visual_barrier_count: status = "warning" else: status = "ok" - if not replay_candidates: + if worker_candidates and not replay_candidates and not invalid_count: + status = "ok" + if not all_candidates: status = "warning" summary = { - "receipt_count": len(replay_candidates), + "receipt_count": len(all_candidates), + "visual_receipt_count": len(replay_candidates), + "platform_probe_worker_receipt_count": platform_probe_worker_receipt_count, "eligible_count": eligible_count, "blocked_count": blocked_count, "invalid_count": invalid_count, "visual_barrier_count": visual_barrier_count, + "source_contract_fallback_count": source_contract_fallback_count, + "capture_runtime_gap_count": capture_runtime_gap_count, "tile_file_count": tile_file_count, "missing_file_count": missing_file_count, "stale_count": stale_count, - "status_counts": _status_counts(replay_candidates), - "platforms": sorted({str(item.get("platform") or "unknown") for item in replay_candidates}), + "status_counts": _status_counts(all_candidates), + "platforms": sorted({str(item.get("platform") or "unknown") for item in all_candidates}), } return { @@ -289,10 +473,11 @@ def build_pixelrag_rag_candidate_replay_readback( "status": status, "generated_at": now.isoformat(), "artifact_root": str(root), + "platform_probe_worker_receipt_root": str(worker_root), "max_age_hours": max_age, "platform_filter": list(platforms or []), "summary": summary, - "candidates": replay_candidates, + "candidates": all_candidates, "replay_contract": { "internal_rag_target": INTERNAL_RAG_TARGET, "writes_database": False, @@ -303,20 +488,25 @@ def build_pixelrag_rag_candidate_replay_readback( "promotion_gate_required_before_knowledge_write": True, "ocr_or_vlm_required_before_field_extraction": True, "blocked_pages_are_not_product_data": True, + "platform_probe_worker_receipts_are_source_contract_candidates": True, }, "next_machine_action": ( "run_pixelrag_visual_capture_worker" - if not replay_candidates + if not all_candidates else ( "fix_invalid_pixelrag_receipts" if invalid_count and not eligible_count else ( - "run_platform_probe_or_use_structured_api" - if visual_barrier_count and not eligible_count + "run_source_contract_replay_for_platform_probe_worker_receipts" + if source_contract_fallback_count and not eligible_count else ( - "run_ocr_vlm_replay_then_promotion_gate" - if eligible_count - else "refresh_pixelrag_visual_capture_receipt" + "run_platform_probe_or_use_structured_api" + if visual_barrier_count and not eligible_count + else ( + "run_ocr_vlm_replay_then_promotion_gate" + if eligible_count + else "refresh_pixelrag_visual_capture_receipt" + ) ) ) ) diff --git a/tests/test_pixelrag_rag_candidate_replay_service.py b/tests/test_pixelrag_rag_candidate_replay_service.py index de9628a..35c89f4 100644 --- a/tests/test_pixelrag_rag_candidate_replay_service.py +++ b/tests/test_pixelrag_rag_candidate_replay_service.py @@ -40,6 +40,45 @@ def _write_receipt(root, *, platform, manifest_id, title, url, http_status=200): return path +def _write_platform_probe_worker_receipt( + root, + *, + platform, + manifest_id, + worker_status="executed_structured_source_fallback_package", + barrier_type="access_denied", + capture_runtime_unavailable=False, +): + receipt_dir = root / platform / manifest_id + receipt_dir.mkdir(parents=True) + payload = { + "platform": platform, + "manifest_id": manifest_id, + "source_type": "capture_receipt", + "source_receipt_path": f"runtime_artifacts/pixelrag_visual_evidence/{platform}/{manifest_id}/capture_receipt.json", + "worker_status": worker_status, + "probe_status": "structured_source_or_backoff_required", + "barrier_type": barrier_type, + "network_call_performed": False, + "artifact_write_performed": True, + "writes_database": False, + "next_machine_action": "run_structured_source_or_platform_backoff_policy", + "structured_source_package": { + "adapter_code": platform.replace("_tw", ""), + "available": True, + "source_count": 2, + "network_request_allowed": False, + "database_write_allowed": False, + "dry_run_only": True, + "blocked_page_not_product_data": True, + "capture_runtime_unavailable": capture_runtime_unavailable, + }, + } + path = receipt_dir / "platform_probe_worker_receipt.json" + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return path + + def test_pixelrag_rag_candidate_replay_splits_eligible_and_blocked_receipts(tmp_path): from services.pixelrag_rag_candidate_replay_service import ( POLICY, @@ -62,7 +101,10 @@ def test_pixelrag_rag_candidate_replay_splits_eligible_and_blocked_receipts(tmp_ http_status=403, ) - payload = build_pixelrag_rag_candidate_replay_readback(artifact_root=tmp_path) + payload = build_pixelrag_rag_candidate_replay_readback( + artifact_root=tmp_path, + platform_probe_worker_receipt_root=tmp_path / "worker_empty", + ) by_platform = {item["platform"]: item for item in payload["candidates"]} assert payload["policy"] == POLICY @@ -81,6 +123,93 @@ def test_pixelrag_rag_candidate_replay_splits_eligible_and_blocked_receipts(tmp_ assert payload["controlled_apply"]["db_write"] is False +def test_pixelrag_rag_candidate_replay_reads_platform_probe_worker_receipts(tmp_path): + from services.pixelrag_rag_candidate_replay_service import ( + build_pixelrag_rag_candidate_replay_readback, + ) + + worker_root = tmp_path / "worker" + _write_platform_probe_worker_receipt( + worker_root, + platform="coupang_tw", + manifest_id="coupang-403", + barrier_type="access_denied", + ) + _write_platform_probe_worker_receipt( + worker_root, + platform="shopee_tw", + manifest_id="shopee-runtime-gap", + worker_status="capture_runtime_unavailable_structured_fallback_package", + barrier_type="traffic_verification_interstitial", + capture_runtime_unavailable=True, + ) + + payload = build_pixelrag_rag_candidate_replay_readback( + artifact_root=tmp_path / "visual_empty", + platform_probe_worker_receipt_root=worker_root, + platform=("coupang_tw", "shopee_tw"), + ) + by_platform = {item["platform"]: item for item in payload["candidates"]} + + assert payload["status"] == "ok" + assert payload["summary"]["receipt_count"] == 2 + assert payload["summary"]["visual_receipt_count"] == 0 + assert payload["summary"]["platform_probe_worker_receipt_count"] == 2 + assert payload["summary"]["source_contract_fallback_count"] == 2 + assert payload["summary"]["capture_runtime_gap_count"] == 1 + assert payload["next_machine_action"] == ( + "run_source_contract_replay_for_platform_probe_worker_receipts" + ) + assert by_platform["coupang_tw"]["candidate_status"] == "source_contract_fallback" + assert by_platform["coupang_tw"]["eligible_for_source_contract_replay"] is True + assert by_platform["coupang_tw"]["eligible_for_rag_candidate_replay"] is False + assert by_platform["coupang_tw"]["promotion_boundary"][ + "requires_source_contract_before_knowledge_write" + ] is True + assert by_platform["shopee_tw"]["platform_probe_worker"][ + "structured_source_package" + ]["capture_runtime_unavailable"] is True + assert payload["replay_contract"][ + "platform_probe_worker_receipts_are_source_contract_candidates" + ] is True + assert payload["controlled_apply"]["db_write"] is False + + +def test_pixelrag_rag_candidate_replay_prefers_worker_source_contract_handoff(tmp_path): + from services.pixelrag_rag_candidate_replay_service import ( + build_pixelrag_rag_candidate_replay_readback, + ) + + visual_root = tmp_path / "visual" + worker_root = tmp_path / "worker" + _write_receipt( + visual_root, + platform="coupang_tw", + manifest_id="coupang-403", + title="Access Denied", + url="https://www.tw.coupang.com/search?q=iphone", + http_status=403, + ) + _write_platform_probe_worker_receipt( + worker_root, + platform="coupang_tw", + manifest_id="coupang-403", + barrier_type="access_denied", + ) + + payload = build_pixelrag_rag_candidate_replay_readback( + artifact_root=visual_root, + platform_probe_worker_receipt_root=worker_root, + platform="coupang_tw", + ) + + assert payload["summary"]["visual_barrier_count"] == 1 + assert payload["summary"]["source_contract_fallback_count"] == 1 + assert payload["next_machine_action"] == ( + "run_source_contract_replay_for_platform_probe_worker_receipts" + ) + + def test_pixelrag_rag_candidate_replay_cli_outputs_machine_readable_json(tmp_path): _write_receipt( tmp_path, @@ -98,6 +227,8 @@ def test_pixelrag_rag_candidate_replay_cli_outputs_machine_readable_json(tmp_pat str(tmp_path), "--platform", "shopee_tw", + "--platform-probe-worker-receipt-root", + str(tmp_path / "worker_empty"), ], capture_output=True, check=False, @@ -124,6 +255,11 @@ def test_pixelrag_rag_candidate_replay_route_returns_readback(tmp_path, monkeypa url="https://shopee.tw/search?keyword=sunscreen", ) monkeypatch.setattr(service, "DEFAULT_ARTIFACT_ROOT", str(tmp_path)) + monkeypatch.setattr( + service, + "DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT", + str(tmp_path / "worker_empty"), + ) app = Flask(__name__) with app.test_request_context( From 4775743f2acc187892901506a054aea2eae0c361 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 10:29:54 +0800 Subject: [PATCH 21/75] fix(ai): surface combined PixelRAG replay actions --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 3 +- .../ai_automation_mainline_work_items.md | 2 +- .../pixelrag_rag_candidate_replay_service.py | 37 ++++++++----------- ...t_pixelrag_rag_candidate_replay_service.py | 34 +++++++++++++++++ 5 files changed, 54 insertions(+), 24 deletions(-) diff --git a/config.py b/config.py index 2488ffa..6490c65 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.762" +SYSTEM_VERSION = "V10.763" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 9fc5563..9565795 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.762 +> **適用版本**: V10.763 --- @@ -895,6 +895,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG platform probe worker 必須把 probe plan 變成 controlled automation | V10.760 起 `/api/ai-automation/pixelrag-platform-probe-worker`、`scripts/ops/run_pixelrag_platform_probe_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled platform probe worker / `pixelrag_platform_probe_worker` family；dry-run 只規劃 capture 或 structured fallback，execute 模式才執行 public empty-context capture 或寫 structured-source/backoff artifact receipt。此 worker 明確標記 `network_call` 只在 capture execute 時為 true、`model_call=false`、`writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`raw_cookie_or_session_read=false`、`primary_human_gate_count=0`；capture 成功後必須回到 PixelRAG RAG candidate replay，不得直接寫正式價格或把 blocked page 當商品資料。 | | 2026-07-10 | PixelRAG platform probe worker 缺 capture runtime 時必須自動降級 | V10.761 起若 `run_pixelrag_platform_probe_worker.py --execute --write-receipt` 在 app 容器偵測到 `ModuleNotFoundError: playwright`，worker 會自動產生 `capture_runtime_unavailable_structured_fallback_package` receipt，`network_call=false`、`artifact_write=true`、`writes_database=false`、`primary_human_gate_count=0`，不再把 Shopee/Coupang platform probe runtime gap 變成人工主流程。 | | 2026-07-10 | PixelRAG platform probe worker receipts 必須回到 RAG/source-contract replay | V10.762 起 `/api/ai-automation/pixelrag-rag-candidate-replay` 與 `scripts/ops/report_pixelrag_rag_candidate_replay.py` 必須同時讀 visual capture receipts 與 `platform_probe_worker_receipt.json`；worker receipts 只能成為 `source_contract_fallback` / `eligible_for_source_contract_replay` 候選，並輸出 `platform_probe_worker_receipt_count`、`source_contract_fallback_count`、`capture_runtime_gap_count` 與下一步 `run_source_contract_replay_for_platform_probe_worker_receipts`。此路徑不寫 DB、不寫 `ai_insights`、不寫正式價格表、不把 blocked page 當商品資料，也不得退回人工審核主流程。 | +| 2026-07-10 | PixelRAG RAG replay 下一步不得隱藏並行主線 | V10.763 起若同一批 PixelRAG replay 同時存在 ready visual receipt 與 platform probe worker `source_contract_fallback` receipt，`next_machine_action` 必須輸出 `run_ocr_vlm_replay_and_source_contract_replay`，避免只顯示 OCR/VLM 而把 marketplace source-contract 自動化 lane 藏在 summary 裡。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index e814f88..bc9885c 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -16,7 +16,7 @@ |---|---|---| | Completed | Multi-commerce PixelRAG visual evidence lane for momo, pchome, shopee_tw, coupang_tw, yahoo_shopping_tw, etmall_tw, friday_tw, rakuten_tw | API and CLI can emit expansion plans and marketplace manifests; PChome parser failures can queue manifests. | | Completed | External MCP/RAG capability inventory absorbed into internal governance readback | `/api/ai-automation/external-mcp-rag-integration` and `scripts/ops/report_external_mcp_rag_integration.py` expose 9 capabilities, absorbed/unresolved counts, and runtime flags. | -| Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read visual capture receipts and platform probe worker receipts, split eligible / blocked / source-contract-fallback candidates, and require OCR/VLM replay, source contract replay, identity matching, plus PromotionGate before knowledge writes. | +| Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read visual capture receipts and platform probe worker receipts, split eligible / blocked / source-contract-fallback candidates, and expose combined OCR/VLM + source-contract next actions before identity matching and PromotionGate knowledge writes. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | diff --git a/services/pixelrag_rag_candidate_replay_service.py b/services/pixelrag_rag_candidate_replay_service.py index 9a33d6f..22c7ef3 100644 --- a/services/pixelrag_rag_candidate_replay_service.py +++ b/services/pixelrag_rag_candidate_replay_service.py @@ -467,6 +467,21 @@ def build_pixelrag_rag_candidate_replay_readback( "platforms": sorted({str(item.get("platform") or "unknown") for item in all_candidates}), } + if not all_candidates: + next_machine_action = "run_pixelrag_visual_capture_worker" + elif invalid_count and not eligible_count: + next_machine_action = "fix_invalid_pixelrag_receipts" + elif eligible_count and source_contract_fallback_count: + next_machine_action = "run_ocr_vlm_replay_and_source_contract_replay" + elif source_contract_fallback_count: + next_machine_action = "run_source_contract_replay_for_platform_probe_worker_receipts" + elif visual_barrier_count and not eligible_count: + next_machine_action = "run_platform_probe_or_use_structured_api" + elif eligible_count: + next_machine_action = "run_ocr_vlm_replay_then_promotion_gate" + else: + next_machine_action = "refresh_pixelrag_visual_capture_receipt" + return { "success": invalid_count == 0, "policy": POLICY, @@ -490,27 +505,7 @@ def build_pixelrag_rag_candidate_replay_readback( "blocked_pages_are_not_product_data": True, "platform_probe_worker_receipts_are_source_contract_candidates": True, }, - "next_machine_action": ( - "run_pixelrag_visual_capture_worker" - if not all_candidates - else ( - "fix_invalid_pixelrag_receipts" - if invalid_count and not eligible_count - else ( - "run_source_contract_replay_for_platform_probe_worker_receipts" - if source_contract_fallback_count and not eligible_count - else ( - "run_platform_probe_or_use_structured_api" - if visual_barrier_count and not eligible_count - else ( - "run_ocr_vlm_replay_then_promotion_gate" - if eligible_count - else "refresh_pixelrag_visual_capture_receipt" - ) - ) - ) - ) - ), + "next_machine_action": next_machine_action, "controlled_apply": { "network_call": False, "db_write": False, diff --git a/tests/test_pixelrag_rag_candidate_replay_service.py b/tests/test_pixelrag_rag_candidate_replay_service.py index 35c89f4..02e6e97 100644 --- a/tests/test_pixelrag_rag_candidate_replay_service.py +++ b/tests/test_pixelrag_rag_candidate_replay_service.py @@ -210,6 +210,40 @@ def test_pixelrag_rag_candidate_replay_prefers_worker_source_contract_handoff(tm ) +def test_pixelrag_rag_candidate_replay_combines_ready_and_source_contract_actions(tmp_path): + from services.pixelrag_rag_candidate_replay_service import ( + build_pixelrag_rag_candidate_replay_readback, + ) + + visual_root = tmp_path / "visual" + worker_root = tmp_path / "worker" + _write_receipt( + visual_root, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=sunscreen", + ) + _write_platform_probe_worker_receipt( + worker_root, + platform="coupang_tw", + manifest_id="coupang-403", + barrier_type="access_denied", + ) + + payload = build_pixelrag_rag_candidate_replay_readback( + artifact_root=visual_root, + platform_probe_worker_receipt_root=worker_root, + platform=("shopee_tw", "coupang_tw"), + ) + + assert payload["summary"]["eligible_count"] == 1 + assert payload["summary"]["source_contract_fallback_count"] == 1 + assert payload["next_machine_action"] == ( + "run_ocr_vlm_replay_and_source_contract_replay" + ) + + def test_pixelrag_rag_candidate_replay_cli_outputs_machine_readable_json(tmp_path): _write_receipt( tmp_path, From 08ed67eee40ac4fe368696b5e0c95c8564228695 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 10:49:56 +0800 Subject: [PATCH 22/75] feat(ai): add PixelRAG source contract replay worker --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 3 +- .../ai_automation_mainline_work_items.md | 3 +- docs/guides/browse_sh_crawler_playbook.md | 4 + routes/system_public_routes.py | 37 ++ ..._pixelrag_source_contract_replay_worker.py | 82 ++++ services/ai_automation_smoke_service.py | 126 ++++++ ...g_source_contract_replay_worker_service.py | 364 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 36 +- ...g_source_contract_replay_worker_service.py | 169 ++++++++ 10 files changed, 819 insertions(+), 7 deletions(-) create mode 100755 scripts/ops/run_pixelrag_source_contract_replay_worker.py create mode 100644 services/pixelrag_source_contract_replay_worker_service.py create mode 100644 tests/test_pixelrag_source_contract_replay_worker_service.py diff --git a/config.py b/config.py index 6490c65..9aef9ac 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.763" +SYSTEM_VERSION = "V10.764" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 9565795..d63f856 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.763 +> **適用版本**: V10.764 --- @@ -896,6 +896,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG platform probe worker 缺 capture runtime 時必須自動降級 | V10.761 起若 `run_pixelrag_platform_probe_worker.py --execute --write-receipt` 在 app 容器偵測到 `ModuleNotFoundError: playwright`，worker 會自動產生 `capture_runtime_unavailable_structured_fallback_package` receipt，`network_call=false`、`artifact_write=true`、`writes_database=false`、`primary_human_gate_count=0`，不再把 Shopee/Coupang platform probe runtime gap 變成人工主流程。 | | 2026-07-10 | PixelRAG platform probe worker receipts 必須回到 RAG/source-contract replay | V10.762 起 `/api/ai-automation/pixelrag-rag-candidate-replay` 與 `scripts/ops/report_pixelrag_rag_candidate_replay.py` 必須同時讀 visual capture receipts 與 `platform_probe_worker_receipt.json`；worker receipts 只能成為 `source_contract_fallback` / `eligible_for_source_contract_replay` 候選，並輸出 `platform_probe_worker_receipt_count`、`source_contract_fallback_count`、`capture_runtime_gap_count` 與下一步 `run_source_contract_replay_for_platform_probe_worker_receipts`。此路徑不寫 DB、不寫 `ai_insights`、不寫正式價格表、不把 blocked page 當商品資料，也不得退回人工審核主流程。 | | 2026-07-10 | PixelRAG RAG replay 下一步不得隱藏並行主線 | V10.763 起若同一批 PixelRAG replay 同時存在 ready visual receipt 與 platform probe worker `source_contract_fallback` receipt，`next_machine_action` 必須輸出 `run_ocr_vlm_replay_and_source_contract_replay`，避免只顯示 OCR/VLM 而把 marketplace source-contract 自動化 lane 藏在 summary 裡。 | +| 2026-07-10 | PixelRAG source-contract fallback 必須有 controlled replay worker | V10.764 起 `/api/ai-automation/pixelrag-source-contract-replay-worker`、`scripts/ops/run_pixelrag_source_contract_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG source-contract replay worker / `pixelrag_source_contract_replay_worker` family；worker 只讀 platform probe worker `source_contract_fallback` receipts，驗證 structured package、source count、no-DB、network-deferred、blocked-page-not-product-data、source receipt provenance，execute 模式只寫 `source_contract_replay_receipt.json` artifact。此路徑不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 marketplace source-contract adapter replay preflight。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index bc9885c..181bfd4 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -17,6 +17,7 @@ | Completed | Multi-commerce PixelRAG visual evidence lane for momo, pchome, shopee_tw, coupang_tw, yahoo_shopping_tw, etmall_tw, friday_tw, rakuten_tw | API and CLI can emit expansion plans and marketplace manifests; PChome parser failures can queue manifests. | | Completed | External MCP/RAG capability inventory absorbed into internal governance readback | `/api/ai-automation/external-mcp-rag-integration` and `scripts/ops/report_external_mcp_rag_integration.py` expose 9 capabilities, absorbed/unresolved counts, and runtime flags. | | Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read visual capture receipts and platform probe worker receipts, split eligible / blocked / source-contract-fallback candidates, and expose combined OCR/VLM + source-contract next actions before identity matching and PromotionGate knowledge writes. | +| Completed | PixelRAG source-contract replay worker | `/api/ai-automation/pixelrag-source-contract-replay-worker` and `scripts/ops/run_pixelrag_source_contract_replay_worker.py` turn platform probe worker `source_contract_fallback` receipts into replayable source-contract artifact receipts with provenance, public-boundary, no-DB, and blocked-page-not-product-data guards. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | @@ -33,7 +34,7 @@ | Not started | Ollama-first multimodal embedding benchmark | Verify local Qwen3-VL or equivalent visual embedding on GCP-A -> GCP-B -> 111 before any visual vector retrieval. | | Not started | pgvector-compatible visual evidence metadata | Design metadata-first retrieval without FAISS in production unless ADR approves a different store. | | In progress | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; `run_pixelrag_platform_probe_worker.py --execute --write-receipt` now emits structured-source/backoff receipts without network, DB writes, or product-data promotion. | -| In progress | Marketplace source contracts beyond MOMO/PChome | Platform probe worker receipts now enter RAG candidate replay as source-contract fallback candidates; next machine action is to replay stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten source contracts only after provenance, rate-limit, public boundary, and replay evidence. | +| In progress | Marketplace source contracts beyond MOMO/PChome | Platform probe worker receipts now enter source-contract replay as artifact receipts; next machine action is marketplace source-contract adapter replay preflight for stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten contracts after provenance, rate-limit, public boundary, and replay evidence. | | In progress | Professional product website UI/UX text reduction | Continue compact AI surfaces, visual QA, and user-facing copy guardrails; avoid engineering logs or work-session prose in the UI. | ## Always Enforced diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index 12993ae..dd9985c 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -196,12 +196,16 @@ python scripts/ops/report_pixelrag_platform_probe.py --platform shopee_tw --plat python scripts/ops/run_pixelrag_platform_probe_worker.py --platform shopee_tw --platform coupang_tw python scripts/ops/run_pixelrag_platform_probe_worker.py --platform shopee_tw --execute --write-receipt --limit 1 python scripts/ops/run_pixelrag_platform_probe_worker.py --platform coupang_tw --execute --write-receipt --limit 1 +python scripts/ops/run_pixelrag_source_contract_replay_worker.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_source_contract_replay_worker.py --platform coupang_tw --execute --write-receipt --limit 1 ``` API readback: `/api/ai-automation/pixelrag-vlm-route-readiness` 與 `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。worker 預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。execute 前會自動讀 approved Ollama `/api/tags` 選擇已安裝候選模型；若完全沒有候選，會寫 `model_route_not_ready` artifact receipt，不盲打 generate。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 Platform probe worker API readback: `/api/ai-automation/pixelrag-platform-probe-worker?platform=shopee_tw`。worker 預設為 dry-run，不抓外站、不寫 artifact；`execute=true&write_receipt=true` 才對 Shopee language / traffic / generic landing 執行 public empty-context capture，或對 Coupang 403 / access denied 寫 structured-source/backoff receipt。capture 成功後仍回到 RAG candidate replay，不得直接寫 `ai_insights`、正式價格表或競品價格歷史。 +Source-contract replay worker API readback: `/api/ai-automation/pixelrag-source-contract-replay-worker?platform=coupang_tw`。worker 預設為 dry-run，不抓外站、不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才把 platform probe worker 的 `source_contract_fallback` receipt 轉成 source-contract replay artifact。artifact 仍只是候選合約證據，不得直接寫 `ai_insights`、正式價格表或競品價格歷史。 + Application portfolio： ```bash diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 9f31159..8574ebc 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -905,6 +905,43 @@ def ai_automation_pixelrag_platform_probe_worker_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-source-contract-replay-worker') +@login_required +def ai_automation_pixelrag_source_contract_replay_worker_api(): + """Controlled PixelRAG source-contract replay worker dry-run/execute readback.""" + from services.pixelrag_source_contract_replay_worker_service import ( + run_pixelrag_source_contract_replay_worker, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + execute = str(request.args.get('execute') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + write_receipt = str(request.args.get('write_receipt') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 25, type=int) + return jsonify(run_pixelrag_source_contract_replay_worker( + platform_probe_worker_receipt_root=str( + request.args.get('platform_probe_worker_receipt_root') or '' + ).strip() or None, + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 25, 250)), + execute=execute, + write_receipt=bool(execute and write_receipt), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/run_pixelrag_source_contract_replay_worker.py b/scripts/ops/run_pixelrag_source_contract_replay_worker.py new file mode 100755 index 0000000..eb385af --- /dev/null +++ b/scripts/ops/run_pixelrag_source_contract_replay_worker.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Run or dry-run the PixelRAG source-contract replay worker.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_source_contract_replay_worker_service import ( # noqa: E402 + run_pixelrag_source_contract_replay_worker, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="執行或 dry-run PixelRAG source-contract replay worker。" + ) + parser.add_argument( + "--artifact-root", + help="PixelRAG visual evidence artifact root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--platform-probe-worker-receipt-root", + help="PixelRAG platform probe worker receipt root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--output-root", + help="Source-contract replay artifact receipt output root。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=25, + help="最多處理 source-contract fallback receipt 數。", + ) + parser.add_argument( + "--execute", + action="store_true", + help="執行 source-contract replay；未指定時只做 no-write dry-run。", + ) + parser.add_argument( + "--write-receipt", + action="store_true", + help="execute 後寫入 source-contract replay artifact receipt；不寫 DB。", + ) + args = parser.parse_args() + + payload = run_pixelrag_source_contract_replay_worker( + artifact_root=args.artifact_root, + platform_probe_worker_receipt_root=args.platform_probe_worker_receipt_root, + output_root=args.output_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + execute=args.execute, + write_receipt=bool(args.write_receipt and args.execute), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index 2e98d00..1dc7d56 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -537,6 +537,21 @@ def build_scheduled_automation_health_summary( if not pixelrag_rag_replay or not pixelrag_rag_replay_details: pixelrag_rag_replay = _pixelrag_rag_candidate_replay_check() pixelrag_rag_replay_details = pixelrag_rag_replay.get("details") or {} + pixelrag_source_contract_worker = _find_check( + source_result, + "PixelRAG source-contract replay worker", + ) + pixelrag_source_contract_worker_details = ( + pixelrag_source_contract_worker.get("details") or {} + ) + if ( + not pixelrag_source_contract_worker + or not pixelrag_source_contract_worker_details + ): + pixelrag_source_contract_worker = _pixelrag_source_contract_replay_worker_check() + pixelrag_source_contract_worker_details = ( + pixelrag_source_contract_worker.get("details") or {} + ) pixelrag_ocr_vlm_replay = _find_check(source_result, "PixelRAG OCR/VLM replay contract") pixelrag_ocr_vlm_replay_details = pixelrag_ocr_vlm_replay.get("details") or {} if not pixelrag_ocr_vlm_replay or not pixelrag_ocr_vlm_replay_details: @@ -4629,6 +4644,55 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_source_contract_replay_worker", + "label": "PixelRAG source-contract replay worker", + "status": pixelrag_source_contract_worker.get("status") or "warning", + "summary": ( + pixelrag_source_contract_worker.get("summary") + or "PixelRAG source-contract replay worker has no latest readback." + ), + "next_machine_action": pixelrag_source_contract_worker_details.get( + "next_machine_action" + ) + or "run_pixelrag_source_contract_replay_worker_dry_run", + "details": { + "policy": pixelrag_source_contract_worker_details.get("policy"), + "candidate_count": int( + pixelrag_source_contract_worker_details.get("candidate_count") or 0 + ), + "ready_count": int( + pixelrag_source_contract_worker_details.get("ready_count") or 0 + ), + "blocked_count": int( + pixelrag_source_contract_worker_details.get("blocked_count") or 0 + ), + "dry_run_count": int( + pixelrag_source_contract_worker_details.get("dry_run_count") or 0 + ), + "executed_count": int( + pixelrag_source_contract_worker_details.get("executed_count") or 0 + ), + "receipt_written_count": int( + pixelrag_source_contract_worker_details.get("receipt_written_count") or 0 + ), + "capture_runtime_gap_count": int( + pixelrag_source_contract_worker_details.get("capture_runtime_gap_count") or 0 + ), + "network_call_performed": bool( + pixelrag_source_contract_worker_details.get("network_call_performed") + ), + "model_call_performed": bool( + pixelrag_source_contract_worker_details.get("model_call_performed") + ), + "artifact_write_performed": bool( + pixelrag_source_contract_worker_details.get("artifact_write_performed") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "key": "pixelrag_ocr_vlm_replay", "label": "PixelRAG OCR/VLM replay contract", @@ -13510,6 +13574,67 @@ def _pixelrag_rag_candidate_replay_check() -> Dict[str, Any]: ) +def _pixelrag_source_contract_replay_worker_check() -> Dict[str, Any]: + """Dry-run sentinel for PixelRAG source-contract replay automation.""" + try: + from services.pixelrag_source_contract_replay_worker_service import ( + run_pixelrag_source_contract_replay_worker, + ) + + readback = run_pixelrag_source_contract_replay_worker(execute=False) + summary = readback.get("summary") or {} + candidate_count = int(summary.get("candidate_count") or 0) + ready_count = int(summary.get("ready_count") or 0) + blocked_count = int(summary.get("blocked_count") or 0) + dry_run_count = int(summary.get("dry_run_count") or 0) + executed_count = int(summary.get("executed_count") or 0) + receipt_written_count = int(summary.get("receipt_written_count") or 0) + status = readback.get("status") or "warning" + summary_text = ( + f"PixelRAG source-contract replay worker candidates={candidate_count}, " + f"ready={ready_count}, blocked={blocked_count}, dry_run={dry_run_count}, " + f"executed={executed_count}, receipts={receipt_written_count}" + ) + return _check( + "PixelRAG source-contract replay worker", + status, + summary_text, + { + "policy": readback.get("policy"), + "candidate_count": candidate_count, + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "capture_runtime_gap_count": int( + summary.get("capture_runtime_gap_count") or 0 + ), + "guard_failed_count": int(summary.get("guard_failed_count") or 0), + "network_call_performed": bool(summary.get("network_call_performed")), + "model_call_performed": bool(summary.get("model_call_performed")), + "artifact_write_performed": bool( + summary.get("artifact_write_performed") + ), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG source-contract replay worker", + "critical", + f"PixelRAG source-contract replay worker 無法執行 dry-run：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def _pixelrag_ocr_vlm_replay_check() -> Dict[str, Any]: """Read-only sentinel for PixelRAG receipts becoming OCR/VLM replay contracts.""" try: @@ -13842,6 +13967,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _sitewide_visual_qa_check(), _external_mcp_rag_integration_check(), _pixelrag_rag_candidate_replay_check(), + _pixelrag_source_contract_replay_worker_check(), _pixelrag_ocr_vlm_replay_check(), _pixelrag_vlm_route_readiness_check(), _pixelrag_vlm_replay_worker_check(), diff --git a/services/pixelrag_source_contract_replay_worker_service.py b/services/pixelrag_source_contract_replay_worker_service.py new file mode 100644 index 0000000..19fd468 --- /dev/null +++ b/services/pixelrag_source_contract_replay_worker_service.py @@ -0,0 +1,364 @@ +"""Controlled PixelRAG source-contract replay worker. + +This worker turns platform-probe structured fallback receipts into replayable +source-contract receipts. It does not fetch marketplace pages, call models, +write databases, or promote price truth. +""" + +from __future__ import annotations + +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.pixelrag_crawler_integration_service import DEFAULT_ARTIFACT_MAX_AGE_HOURS +from services.pixelrag_rag_candidate_replay_service import ( + DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT, + build_pixelrag_rag_candidate_replay_readback, +) + + +POLICY = "controlled_pixelrag_source_contract_replay_worker_v1" +SOURCE_CONTRACT_VERSION = "pixelrag_marketplace_source_contract_v1" +DEFAULT_LIMIT = 25 +DEFAULT_OUTPUT_ROOT = os.getenv( + "PIXELRAG_SOURCE_CONTRACT_REPLAY_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_source_contract_replay_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_source_contract_replay_receipts", +) + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...]: + if isinstance(platform, str): + value = platform.strip().lower() + return (value,) if value else () + return tuple( + str(item or "").strip().lower() + for item in (platform or ()) + if str(item or "").strip() + ) + + +def _safe_segment(value: Any) -> str: + text = str(value or "unknown").strip().lower() + text = re.sub(r"[^a-z0-9._-]+", "-", text) + return text.strip("-") or "unknown" + + +def _source_contract_candidates(readback: Mapping[str, Any]) -> list[dict[str, Any]]: + candidates: list[dict[str, Any]] = [] + for item in list(readback.get("candidates") or []): + if not isinstance(item, Mapping): + continue + if item.get("candidate_status") != "source_contract_fallback": + continue + if not item.get("eligible_for_source_contract_replay"): + continue + candidates.append(dict(item)) + return candidates + + +def _contract_checks(candidate: Mapping[str, Any]) -> dict[str, bool]: + worker = candidate.get("platform_probe_worker") + if not isinstance(worker, Mapping): + worker = {} + package = worker.get("structured_source_package") + if not isinstance(package, Mapping): + package = {} + return { + "structured_source_package_available": bool(package.get("available")), + "source_count_positive": int(package.get("source_count") or 0) > 0, + "database_write_disallowed": not bool(package.get("database_write_allowed")), + "network_request_deferred": not bool(package.get("network_request_allowed")), + "dry_run_only_contract": bool(package.get("dry_run_only")), + "blocked_page_not_product_data": bool(package.get("blocked_page_not_product_data")), + "source_worker_receipt_path_present": bool(candidate.get("receipt_path")), + "source_receipt_path_present": bool(worker.get("source_receipt_path")), + } + + +def _source_contract_preview(candidate: Mapping[str, Any]) -> dict[str, Any]: + worker = candidate.get("platform_probe_worker") + if not isinstance(worker, Mapping): + worker = {} + package = worker.get("structured_source_package") + if not isinstance(package, Mapping): + package = {} + platform = str(candidate.get("platform") or "unknown") + manifest_id = str(candidate.get("manifest_id") or "unknown") + adapter_code = str(package.get("adapter_code") or platform) + return { + "source_contract_version": SOURCE_CONTRACT_VERSION, + "contract_id": f"{platform}:{manifest_id}:{adapter_code}", + "adapter_code": adapter_code, + "source_count": int(package.get("source_count") or 0), + "capture_runtime_unavailable": bool(package.get("capture_runtime_unavailable")), + "barrier_type": worker.get("barrier_type"), + "expected_offer_fields": [ + "platform", + "platform_product_id", + "title", + "url", + "image_url", + "price", + "currency", + "availability", + "seller", + "promotion_badges", + "shipping_badges", + "evidence_time", + "source_method", + "confidence", + ], + "required_before_data_promotion": [ + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", + ], + } + + +def _worker_item( + candidate: Mapping[str, Any], + *, + execute: bool, +) -> dict[str, Any]: + checks = _contract_checks(candidate) + check_count = len(checks) + pass_count = sum(1 for passed in checks.values() if passed) + ready = pass_count == check_count + worker = candidate.get("platform_probe_worker") + if not isinstance(worker, Mapping): + worker = {} + package = worker.get("structured_source_package") + if not isinstance(package, Mapping): + package = {} + status = ( + "executed_source_contract_replay_ready" + if execute and ready + else ( + "dry_run_ready_for_source_contract_replay" + if ready + else "skipped_source_contract_guard_failed" + ) + ) + return { + "worker_status": status, + "platform": candidate.get("platform"), + "manifest_id": candidate.get("manifest_id"), + "source_type": "platform_probe_worker_receipt", + "source_worker_receipt_path": candidate.get("receipt_path"), + "source_receipt_path": worker.get("source_receipt_path"), + "source_contract_status": "ready" if ready else "blocked", + "ready_for_execution": ready, + "execute": bool(execute), + "adapter_code": package.get("adapter_code"), + "structured_source_count": int(package.get("source_count") or 0), + "capture_runtime_unavailable": bool(package.get("capture_runtime_unavailable")), + "barrier_type": worker.get("barrier_type"), + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": False, + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + "source_contract_checks": checks, + "source_contract_check_count": check_count, + "source_contract_check_pass_count": pass_count, + "source_contract_preview": _source_contract_preview(candidate), + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "source_contract_is_candidate_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": ( + "run_marketplace_source_contract_adapter_replay_preflight" + if execute and ready + else ( + "run_pixelrag_source_contract_replay_worker_execute" + if ready + else "repair_source_contract_fallback_package" + ) + ), + } + + +def _write_replay_receipt(*, output_root: Path, item: Mapping[str, Any]) -> str: + target = ( + output_root + / _safe_segment(item.get("platform")) + / _safe_segment(item.get("manifest_id")) + / "source_contract_replay_receipt.json" + ) + target.parent.mkdir(parents=True, exist_ok=True) + receipt = dict(item) + receipt["artifact_write_performed"] = True + receipt["receipt_path"] = str(target) + receipt["generated_at"] = datetime.now(timezone.utc).isoformat() + receipt["policy"] = POLICY + target.write_text( + json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + return str(target) + + +def run_pixelrag_source_contract_replay_worker( + *, + artifact_root: str | Path | None = None, + platform_probe_worker_receipt_root: str | Path | None = None, + output_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, + execute: bool = False, + write_receipt: bool = False, +) -> dict[str, Any]: + """Run or dry-run source-contract replay for platform probe worker receipts.""" + output = Path(output_root or DEFAULT_OUTPUT_ROOT) + worker_root = Path( + platform_probe_worker_receipt_root + or DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT + ) + platforms = _normalise_platforms(platform) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + readback = build_pixelrag_rag_candidate_replay_readback( + artifact_root=artifact_root, + platform_probe_worker_receipt_root=worker_root, + platform=platforms, + max_age_hours=max_age, + limit=item_limit, + ) + candidates = _source_contract_candidates(readback) + worker_items: list[dict[str, Any]] = [] + for candidate in candidates: + item = _worker_item(candidate, execute=execute) + if execute and write_receipt and item.get("ready_for_execution"): + item["receipt_path"] = _write_replay_receipt(output_root=output, item=item) + item["artifact_write_performed"] = True + worker_items.append(item) + + ready_count = sum(1 for item in worker_items if item.get("ready_for_execution")) + blocked_count = len(worker_items) - ready_count + dry_run_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_") + ) + executed_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_") + ) + receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path")) + capture_runtime_gap_count = sum( + 1 for item in worker_items if item.get("capture_runtime_unavailable") + ) + guard_failed_count = sum( + 1 for item in worker_items if item.get("source_contract_status") != "ready" + ) + + if not worker_items: + status = "warning" + elif guard_failed_count: + status = "warning" + else: + status = "ok" + + if not worker_items: + next_action = "run_pixelrag_platform_probe_worker_or_rag_candidate_replay" + elif not execute and ready_count: + next_action = "run_pixelrag_source_contract_replay_worker_execute" + elif execute and receipt_written_count: + next_action = "run_marketplace_source_contract_adapter_replay_preflight" + elif guard_failed_count: + next_action = "repair_source_contract_fallback_package" + else: + next_action = "refresh_pixelrag_source_contract_replay_candidates" + + summary = { + "candidate_count": len(worker_items), + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "capture_runtime_gap_count": capture_runtime_gap_count, + "guard_failed_count": guard_failed_count, + "platforms": sorted({str(item.get("platform") or "unknown") for item in worker_items}), + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": bool(receipt_written_count), + "writes_database_count": 0, + "primary_human_gate_count": 0, + } + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": datetime.now(timezone.utc).isoformat(), + "source_contract_version": SOURCE_CONTRACT_VERSION, + "artifact_root": str(artifact_root or readback.get("artifact_root") or ""), + "platform_probe_worker_receipt_root": str(worker_root), + "output_root": str(output), + "platform_filter": list(platforms), + "max_age_hours": max_age, + "limit": item_limit, + "execute": bool(execute), + "write_receipt": bool(write_receipt and execute), + "summary": summary, + "worker_items": worker_items, + "source_readback": { + "policy": readback.get("policy"), + "status": readback.get("status"), + "summary": readback.get("summary"), + "next_machine_action": readback.get("next_machine_action"), + }, + "controlled_apply": { + "network_call": False, + "model_call": False, + "artifact_write": bool(receipt_written_count), + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "secret_read": False, + "raw_cookie_or_session_read": False, + "credentialed_session_allowed": False, + "login_allowed": False, + "production_price_write": False, + "primary_human_gate_count": 0, + }, + "promotion_boundary": { + "writes_ai_insights": False, + "writes_price_tables": False, + "source_contract_is_candidate_only": True, + "blocked_pages_are_not_product_data": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "DEFAULT_OUTPUT_ROOT", + "POLICY", + "SOURCE_CONTRACT_VERSION", + "run_pixelrag_source_contract_replay_worker", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index 06ca503..47dbe77 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1433,6 +1433,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -1442,7 +1443,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 36, "warning": 1, "critical": 1, "total": 38} + assert result["summary"] == {"ok": 37, "warning": 1, "critical": 1, "total": 39} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3971,6 +3972,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -4031,7 +4033,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 38, "warning": 0, "critical": 0, "total": 38}, + "summary": {"ok": 39, "warning": 0, "critical": 0, "total": 39}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4140,6 +4142,27 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "next_machine_action": "run_ocr_vlm_replay_then_promotion_gate", }, }, + { + "name": "PixelRAG source-contract replay worker", + "status": "ok", + "summary": "PixelRAG source-contract replay worker candidates=1, ready=1, blocked=0, dry_run=1, executed=0, receipts=0", + "details": { + "policy": "controlled_pixelrag_source_contract_replay_worker_v1", + "candidate_count": 1, + "ready_count": 1, + "blocked_count": 0, + "dry_run_count": 1, + "executed_count": 0, + "receipt_written_count": 0, + "capture_runtime_gap_count": 0, + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": "run_pixelrag_source_contract_replay_worker_execute", + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "name": "PixelRAG OCR/VLM replay contract", "status": "ok", @@ -4254,7 +4277,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 35 + assert summary["summary"]["total"] == 36 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -6586,6 +6609,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag replay ok", )) + monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check( + "PixelRAG source-contract replay worker", + "ok", + "pixelrag source contract ok", + )) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check( "PixelRAG OCR/VLM replay contract", "ok", @@ -6626,7 +6654,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 38 + assert result["summary"]["total"] == 39 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_source_contract_replay_worker_service.py b/tests/test_pixelrag_source_contract_replay_worker_service.py new file mode 100644 index 0000000..be19278 --- /dev/null +++ b/tests/test_pixelrag_source_contract_replay_worker_service.py @@ -0,0 +1,169 @@ +import json +import subprocess +import sys + +from tests.test_pixelrag_rag_candidate_replay_service import ( + _write_platform_probe_worker_receipt, +) + + +def test_pixelrag_source_contract_replay_worker_dry_run_reads_worker_receipts(tmp_path): + from services.pixelrag_source_contract_replay_worker_service import ( + POLICY, + run_pixelrag_source_contract_replay_worker, + ) + + worker_root = tmp_path / "worker" + _write_platform_probe_worker_receipt( + worker_root, + platform="coupang_tw", + manifest_id="coupang-403", + barrier_type="access_denied", + ) + _write_platform_probe_worker_receipt( + worker_root, + platform="shopee_tw", + manifest_id="shopee-runtime-gap", + worker_status="capture_runtime_unavailable_structured_fallback_package", + barrier_type="traffic_verification_interstitial", + capture_runtime_unavailable=True, + ) + + payload = run_pixelrag_source_contract_replay_worker( + artifact_root=tmp_path / "visual_empty", + platform_probe_worker_receipt_root=worker_root, + platform=("coupang_tw", "shopee_tw"), + ) + by_platform = {item["platform"]: item for item in payload["worker_items"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "ok" + assert payload["execute"] is False + assert payload["summary"]["candidate_count"] == 2 + assert payload["summary"]["ready_count"] == 2 + assert payload["summary"]["dry_run_count"] == 2 + assert payload["summary"]["capture_runtime_gap_count"] == 1 + assert payload["summary"]["writes_database_count"] == 0 + assert payload["controlled_apply"]["network_call"] is False + assert payload["controlled_apply"]["writes_database"] is False + assert payload["controlled_apply"]["primary_human_gate_count"] == 0 + assert payload["next_machine_action"] == ( + "run_pixelrag_source_contract_replay_worker_execute" + ) + assert by_platform["coupang_tw"]["worker_status"] == ( + "dry_run_ready_for_source_contract_replay" + ) + assert by_platform["coupang_tw"]["source_contract_checks"][ + "blocked_page_not_product_data" + ] is True + assert by_platform["shopee_tw"]["capture_runtime_unavailable"] is True + + +def test_pixelrag_source_contract_replay_worker_execute_writes_receipts(tmp_path): + from services.pixelrag_source_contract_replay_worker_service import ( + run_pixelrag_source_contract_replay_worker, + ) + + worker_root = tmp_path / "worker" + output_root = tmp_path / "source_contract_receipts" + _write_platform_probe_worker_receipt( + worker_root, + platform="coupang_tw", + manifest_id="coupang-403", + barrier_type="access_denied", + ) + + payload = run_pixelrag_source_contract_replay_worker( + artifact_root=tmp_path / "visual_empty", + platform_probe_worker_receipt_root=worker_root, + output_root=output_root, + platform="coupang_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["executed_count"] == 1 + assert payload["summary"]["receipt_written_count"] == 1 + assert payload["controlled_apply"]["artifact_write"] is True + assert payload["controlled_apply"]["writes_database"] is False + assert payload["next_machine_action"] == ( + "run_marketplace_source_contract_adapter_replay_preflight" + ) + receipt_path = ( + output_root + / "coupang_tw" + / "coupang-403" + / "source_contract_replay_receipt.json" + ) + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "executed_source_contract_replay_ready" + assert receipt["artifact_write_performed"] is True + assert receipt["writes_database"] is False + assert receipt["source_contract_preview"]["source_contract_version"] == ( + "pixelrag_marketplace_source_contract_v1" + ) + + +def test_pixelrag_source_contract_replay_worker_cli_outputs_json(tmp_path): + worker_root = tmp_path / "worker" + _write_platform_probe_worker_receipt( + worker_root, + platform="coupang_tw", + manifest_id="coupang-403", + barrier_type="access_denied", + ) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/run_pixelrag_source_contract_replay_worker.py", + "--artifact-root", + str(tmp_path / "visual_empty"), + "--platform-probe-worker-receipt-root", + str(worker_root), + "--platform", + "coupang_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False + + +def test_pixelrag_source_contract_replay_worker_route_returns_readback(tmp_path, monkeypatch): + from flask import Flask + from routes import system_public_routes as routes + from services import pixelrag_source_contract_replay_worker_service as service + + worker_root = tmp_path / "worker" + _write_platform_probe_worker_receipt( + worker_root, + platform="coupang_tw", + manifest_id="coupang-403", + barrier_type="access_denied", + ) + monkeypatch.setattr( + service, + "DEFAULT_PLATFORM_PROBE_WORKER_RECEIPT_ROOT", + str(worker_root), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-source-contract-replay-worker?platform=coupang_tw" + ): + response = routes.ai_automation_pixelrag_source_contract_replay_worker_api.__wrapped__() + payload = response.get_json() + + assert payload["policy"] == "controlled_pixelrag_source_contract_replay_worker_v1" + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False From edd515749631d9afa260e23fb3195ee132fd8c2b Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 11:08:57 +0800 Subject: [PATCH 23/75] feat(ai): add PixelRAG marketplace adapter preflight --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 3 +- .../ai_automation_mainline_work_items.md | 3 +- docs/guides/browse_sh_crawler_playbook.md | 4 + routes/system_public_routes.py | 37 ++ ...place_source_contract_adapter_preflight.py | 77 ++++ services/ai_automation_smoke_service.py | 130 ++++++ ...urce_contract_adapter_preflight_service.py | 407 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 40 +- ...urce_contract_adapter_preflight_service.py | 232 ++++++++++ 10 files changed, 928 insertions(+), 7 deletions(-) create mode 100755 scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py create mode 100644 services/pixelrag_marketplace_source_contract_adapter_preflight_service.py create mode 100644 tests/test_pixelrag_marketplace_source_contract_adapter_preflight_service.py diff --git a/config.py b/config.py index 9aef9ac..9ee2139 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.764" +SYSTEM_VERSION = "V10.765" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index d63f856..5fd8f4d 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.764 +> **適用版本**: V10.765 --- @@ -897,6 +897,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG platform probe worker receipts 必須回到 RAG/source-contract replay | V10.762 起 `/api/ai-automation/pixelrag-rag-candidate-replay` 與 `scripts/ops/report_pixelrag_rag_candidate_replay.py` 必須同時讀 visual capture receipts 與 `platform_probe_worker_receipt.json`；worker receipts 只能成為 `source_contract_fallback` / `eligible_for_source_contract_replay` 候選，並輸出 `platform_probe_worker_receipt_count`、`source_contract_fallback_count`、`capture_runtime_gap_count` 與下一步 `run_source_contract_replay_for_platform_probe_worker_receipts`。此路徑不寫 DB、不寫 `ai_insights`、不寫正式價格表、不把 blocked page 當商品資料，也不得退回人工審核主流程。 | | 2026-07-10 | PixelRAG RAG replay 下一步不得隱藏並行主線 | V10.763 起若同一批 PixelRAG replay 同時存在 ready visual receipt 與 platform probe worker `source_contract_fallback` receipt，`next_machine_action` 必須輸出 `run_ocr_vlm_replay_and_source_contract_replay`，避免只顯示 OCR/VLM 而把 marketplace source-contract 自動化 lane 藏在 summary 裡。 | | 2026-07-10 | PixelRAG source-contract fallback 必須有 controlled replay worker | V10.764 起 `/api/ai-automation/pixelrag-source-contract-replay-worker`、`scripts/ops/run_pixelrag_source_contract_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG source-contract replay worker / `pixelrag_source_contract_replay_worker` family；worker 只讀 platform probe worker `source_contract_fallback` receipts，驗證 structured package、source count、no-DB、network-deferred、blocked-page-not-product-data、source receipt provenance，execute 模式只寫 `source_contract_replay_receipt.json` artifact。此路徑不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 marketplace source-contract adapter replay preflight。 | +| 2026-07-10 | PixelRAG marketplace source-contract adapter 必須先有 no-write preflight | V10.765 起 `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight`、`scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace adapter preflight / `pixelrag_marketplace_adapter_preflight` family；preflight 只讀 `source_contract_replay_receipt.json`，驗證 replay ready、source contract version、adapter code、source count、source checks all passed、blocked page not product data、no-DB、network-deferred、public source boundary、rate-limit contract、provenance contract、identity/promotion gates 與 canonical offer fields。execute 模式只寫 `marketplace_adapter_preflight_receipt.json` artifact，不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 marketplace source-contract adapter dry-run。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 181bfd4..7c77f95 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -18,6 +18,7 @@ | Completed | External MCP/RAG capability inventory absorbed into internal governance readback | `/api/ai-automation/external-mcp-rag-integration` and `scripts/ops/report_external_mcp_rag_integration.py` expose 9 capabilities, absorbed/unresolved counts, and runtime flags. | | Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read visual capture receipts and platform probe worker receipts, split eligible / blocked / source-contract-fallback candidates, and expose combined OCR/VLM + source-contract next actions before identity matching and PromotionGate knowledge writes. | | Completed | PixelRAG source-contract replay worker | `/api/ai-automation/pixelrag-source-contract-replay-worker` and `scripts/ops/run_pixelrag_source_contract_replay_worker.py` turn platform probe worker `source_contract_fallback` receipts into replayable source-contract artifact receipts with provenance, public-boundary, no-DB, and blocked-page-not-product-data guards. | +| Completed | PixelRAG marketplace source-contract adapter preflight | `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight` and `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py` validate source-contract replay receipts for adapter readiness, canonical offer fields, provenance/rate-limit/public-boundary requirements, and no-write promotion gates. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | @@ -34,7 +35,7 @@ | Not started | Ollama-first multimodal embedding benchmark | Verify local Qwen3-VL or equivalent visual embedding on GCP-A -> GCP-B -> 111 before any visual vector retrieval. | | Not started | pgvector-compatible visual evidence metadata | Design metadata-first retrieval without FAISS in production unless ADR approves a different store. | | In progress | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; `run_pixelrag_platform_probe_worker.py --execute --write-receipt` now emits structured-source/backoff receipts without network, DB writes, or product-data promotion. | -| In progress | Marketplace source contracts beyond MOMO/PChome | Platform probe worker receipts now enter source-contract replay as artifact receipts; next machine action is marketplace source-contract adapter replay preflight for stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten contracts after provenance, rate-limit, public boundary, and replay evidence. | +| In progress | Marketplace source contracts beyond MOMO/PChome | Source-contract replay receipts now enter marketplace adapter preflight as artifact receipts; next machine action is adapter dry-run for stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten contracts after provenance, rate-limit, public boundary, and replay evidence. | | In progress | Professional product website UI/UX text reduction | Continue compact AI surfaces, visual QA, and user-facing copy guardrails; avoid engineering logs or work-session prose in the UI. | ## Always Enforced diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index dd9985c..6a87483 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -198,6 +198,8 @@ python scripts/ops/run_pixelrag_platform_probe_worker.py --platform shopee_tw -- python scripts/ops/run_pixelrag_platform_probe_worker.py --platform coupang_tw --execute --write-receipt --limit 1 python scripts/ops/run_pixelrag_source_contract_replay_worker.py --platform shopee_tw --platform coupang_tw python scripts/ops/run_pixelrag_source_contract_replay_worker.py --platform coupang_tw --execute --write-receipt --limit 1 +python scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py --platform coupang_tw --execute --write-receipt --limit 1 ``` API readback: `/api/ai-automation/pixelrag-vlm-route-readiness` 與 `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。worker 預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。execute 前會自動讀 approved Ollama `/api/tags` 選擇已安裝候選模型；若完全沒有候選，會寫 `model_route_not_ready` artifact receipt，不盲打 generate。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 @@ -206,6 +208,8 @@ Platform probe worker API readback: `/api/ai-automation/pixelrag-platform-probe- Source-contract replay worker API readback: `/api/ai-automation/pixelrag-source-contract-replay-worker?platform=coupang_tw`。worker 預設為 dry-run，不抓外站、不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才把 platform probe worker 的 `source_contract_fallback` receipt 轉成 source-contract replay artifact。artifact 仍只是候選合約證據，不得直接寫 `ai_insights`、正式價格表或競品價格歷史。 +Marketplace adapter preflight API readback: `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight?platform=coupang_tw`。preflight 預設為 dry-run，不抓外站、不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才把 source-contract replay receipt 轉成 adapter preflight artifact。artifact 只證明 adapter dry-run 條件齊備，不得直接寫 `ai_insights`、正式價格表或競品價格歷史。 + Application portfolio： ```bash diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 8574ebc..3ea71ff 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -942,6 +942,43 @@ def ai_automation_pixelrag_source_contract_replay_worker_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight') +@login_required +def ai_automation_pixelrag_marketplace_source_contract_adapter_preflight_api(): + """Controlled PixelRAG marketplace source-contract adapter preflight readback.""" + from services.pixelrag_marketplace_source_contract_adapter_preflight_service import ( + run_pixelrag_marketplace_source_contract_adapter_preflight, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + execute = str(request.args.get('execute') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + write_receipt = str(request.args.get('write_receipt') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 25, type=int) + return jsonify(run_pixelrag_marketplace_source_contract_adapter_preflight( + source_contract_replay_receipt_root=str( + request.args.get('source_contract_replay_receipt_root') or '' + ).strip() or None, + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 25, 250)), + execute=execute, + write_receipt=bool(execute and write_receipt), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py b/scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py new file mode 100755 index 0000000..554ecce --- /dev/null +++ b/scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Run or dry-run PixelRAG marketplace source-contract adapter preflight.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_marketplace_source_contract_adapter_preflight_service import ( # noqa: E402 + run_pixelrag_marketplace_source_contract_adapter_preflight, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="執行或 dry-run PixelRAG marketplace source-contract adapter preflight。" + ) + parser.add_argument( + "--source-contract-replay-receipt-root", + help="Source-contract replay receipt root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--output-root", + help="Marketplace adapter preflight artifact receipt output root。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=25, + help="最多處理 source-contract replay receipt 數。", + ) + parser.add_argument( + "--execute", + action="store_true", + help="執行 adapter preflight；未指定時只做 no-write dry-run。", + ) + parser.add_argument( + "--write-receipt", + action="store_true", + help="execute 後寫入 adapter preflight artifact receipt；不寫 DB。", + ) + args = parser.parse_args() + + payload = run_pixelrag_marketplace_source_contract_adapter_preflight( + source_contract_replay_receipt_root=args.source_contract_replay_receipt_root, + output_root=args.output_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + execute=args.execute, + write_receipt=bool(args.write_receipt and args.execute), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index 1dc7d56..59196c2 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -552,6 +552,23 @@ def build_scheduled_automation_health_summary( pixelrag_source_contract_worker_details = ( pixelrag_source_contract_worker.get("details") or {} ) + pixelrag_marketplace_adapter_preflight = _find_check( + source_result, + "PixelRAG marketplace adapter preflight", + ) + pixelrag_marketplace_adapter_preflight_details = ( + pixelrag_marketplace_adapter_preflight.get("details") or {} + ) + if ( + not pixelrag_marketplace_adapter_preflight + or not pixelrag_marketplace_adapter_preflight_details + ): + pixelrag_marketplace_adapter_preflight = ( + _pixelrag_marketplace_adapter_preflight_check() + ) + pixelrag_marketplace_adapter_preflight_details = ( + pixelrag_marketplace_adapter_preflight.get("details") or {} + ) pixelrag_ocr_vlm_replay = _find_check(source_result, "PixelRAG OCR/VLM replay contract") pixelrag_ocr_vlm_replay_details = pixelrag_ocr_vlm_replay.get("details") or {} if not pixelrag_ocr_vlm_replay or not pixelrag_ocr_vlm_replay_details: @@ -4693,6 +4710,55 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_marketplace_adapter_preflight", + "label": "PixelRAG marketplace adapter preflight", + "status": pixelrag_marketplace_adapter_preflight.get("status") or "warning", + "summary": ( + pixelrag_marketplace_adapter_preflight.get("summary") + or "PixelRAG marketplace adapter preflight has no latest readback." + ), + "next_machine_action": pixelrag_marketplace_adapter_preflight_details.get( + "next_machine_action" + ) + or "run_pixelrag_marketplace_adapter_preflight_dry_run", + "details": { + "policy": pixelrag_marketplace_adapter_preflight_details.get("policy"), + "candidate_count": int( + pixelrag_marketplace_adapter_preflight_details.get("candidate_count") or 0 + ), + "ready_count": int( + pixelrag_marketplace_adapter_preflight_details.get("ready_count") or 0 + ), + "blocked_count": int( + pixelrag_marketplace_adapter_preflight_details.get("blocked_count") or 0 + ), + "dry_run_count": int( + pixelrag_marketplace_adapter_preflight_details.get("dry_run_count") or 0 + ), + "executed_count": int( + pixelrag_marketplace_adapter_preflight_details.get("executed_count") or 0 + ), + "receipt_written_count": int( + pixelrag_marketplace_adapter_preflight_details.get("receipt_written_count") or 0 + ), + "capture_runtime_gap_count": int( + pixelrag_marketplace_adapter_preflight_details.get("capture_runtime_gap_count") or 0 + ), + "network_call_performed": bool( + pixelrag_marketplace_adapter_preflight_details.get("network_call_performed") + ), + "model_call_performed": bool( + pixelrag_marketplace_adapter_preflight_details.get("model_call_performed") + ), + "artifact_write_performed": bool( + pixelrag_marketplace_adapter_preflight_details.get("artifact_write_performed") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "key": "pixelrag_ocr_vlm_replay", "label": "PixelRAG OCR/VLM replay contract", @@ -13635,6 +13701,69 @@ def _pixelrag_source_contract_replay_worker_check() -> Dict[str, Any]: ) +def _pixelrag_marketplace_adapter_preflight_check() -> Dict[str, Any]: + """Dry-run sentinel for PixelRAG marketplace source-contract adapter preflight.""" + try: + from services.pixelrag_marketplace_source_contract_adapter_preflight_service import ( + run_pixelrag_marketplace_source_contract_adapter_preflight, + ) + + readback = run_pixelrag_marketplace_source_contract_adapter_preflight( + execute=False, + ) + summary = readback.get("summary") or {} + candidate_count = int(summary.get("candidate_count") or 0) + ready_count = int(summary.get("ready_count") or 0) + blocked_count = int(summary.get("blocked_count") or 0) + dry_run_count = int(summary.get("dry_run_count") or 0) + executed_count = int(summary.get("executed_count") or 0) + receipt_written_count = int(summary.get("receipt_written_count") or 0) + status = readback.get("status") or "warning" + summary_text = ( + f"PixelRAG marketplace adapter preflight candidates={candidate_count}, " + f"ready={ready_count}, blocked={blocked_count}, dry_run={dry_run_count}, " + f"executed={executed_count}, receipts={receipt_written_count}" + ) + return _check( + "PixelRAG marketplace adapter preflight", + status, + summary_text, + { + "policy": readback.get("policy"), + "candidate_count": candidate_count, + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "capture_runtime_gap_count": int( + summary.get("capture_runtime_gap_count") or 0 + ), + "guard_failed_count": int(summary.get("guard_failed_count") or 0), + "network_call_performed": bool(summary.get("network_call_performed")), + "model_call_performed": bool(summary.get("model_call_performed")), + "artifact_write_performed": bool( + summary.get("artifact_write_performed") + ), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG marketplace adapter preflight", + "critical", + f"PixelRAG marketplace adapter preflight 無法執行 dry-run：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def _pixelrag_ocr_vlm_replay_check() -> Dict[str, Any]: """Read-only sentinel for PixelRAG receipts becoming OCR/VLM replay contracts.""" try: @@ -13968,6 +14097,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _external_mcp_rag_integration_check(), _pixelrag_rag_candidate_replay_check(), _pixelrag_source_contract_replay_worker_check(), + _pixelrag_marketplace_adapter_preflight_check(), _pixelrag_ocr_vlm_replay_check(), _pixelrag_vlm_route_readiness_check(), _pixelrag_vlm_replay_worker_check(), diff --git a/services/pixelrag_marketplace_source_contract_adapter_preflight_service.py b/services/pixelrag_marketplace_source_contract_adapter_preflight_service.py new file mode 100644 index 0000000..51b37c1 --- /dev/null +++ b/services/pixelrag_marketplace_source_contract_adapter_preflight_service.py @@ -0,0 +1,407 @@ +"""Controlled PixelRAG marketplace source-contract adapter preflight. + +This worker verifies source-contract replay receipts before any marketplace +adapter can run. It does not fetch external pages, call models, write databases, +or promote source-contract candidates into price truth. +""" + +from __future__ import annotations + +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.pixelrag_crawler_integration_service import DEFAULT_ARTIFACT_MAX_AGE_HOURS +from services.pixelrag_source_contract_replay_worker_service import ( + DEFAULT_OUTPUT_ROOT as DEFAULT_SOURCE_CONTRACT_REPLAY_RECEIPT_ROOT, + SOURCE_CONTRACT_VERSION, +) + + +POLICY = "controlled_pixelrag_marketplace_source_contract_adapter_preflight_v1" +ADAPTER_PREFLIGHT_VERSION = "pixelrag_marketplace_adapter_preflight_v1" +DEFAULT_LIMIT = 25 +DEFAULT_OUTPUT_ROOT = os.getenv( + "PIXELRAG_MARKETPLACE_ADAPTER_PREFLIGHT_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_marketplace_adapter_preflight_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_marketplace_adapter_preflight_receipts", +) + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...]: + if isinstance(platform, str): + value = platform.strip().lower() + return (value,) if value else () + return tuple( + str(item or "").strip().lower() + for item in (platform or ()) + if str(item or "").strip() + ) + + +def _safe_segment(value: Any) -> str: + text = str(value or "unknown").strip().lower() + text = re.sub(r"[^a-z0-9._-]+", "-", text) + return text.strip("-") or "unknown" + + +def _parse_iso_datetime(value: Any) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + + +def _receipt_candidates( + root: Path, + *, + platforms: tuple[str, ...], + limit: int, +) -> list[Path]: + if not root.exists(): + return [] + candidates: list[Path] = [] + if platforms: + for platform in platforms: + candidates.extend((root / platform).glob("*/source_contract_replay_receipt.json")) + else: + candidates.extend(root.glob("*/*/source_contract_replay_receipt.json")) + return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] + + +def _load_receipt(path: Path) -> tuple[dict[str, Any], list[str]]: + try: + return json.loads(path.read_text(encoding="utf-8")), [] + except (OSError, json.JSONDecodeError) as exc: + return {}, [str(exc)[:300]] + + +def _adapter_preflight_checks( + receipt: Mapping[str, Any], + *, + stale: bool, + errors: list[str], +) -> dict[str, bool]: + preview = receipt.get("source_contract_preview") + if not isinstance(preview, Mapping): + preview = {} + promotion = receipt.get("promotion_boundary") + if not isinstance(promotion, Mapping): + promotion = {} + source_checks = receipt.get("source_contract_checks") + if not isinstance(source_checks, Mapping): + source_checks = {} + required_before = set(str(item) for item in list(preview.get("required_before_data_promotion") or [])) + expected_fields = set(str(item) for item in list(preview.get("expected_offer_fields") or [])) + return { + "receipt_parse_ok": not errors, + "receipt_fresh": not stale, + "source_contract_replay_ready": ( + receipt.get("worker_status") == "executed_source_contract_replay_ready" + ), + "source_contract_version_supported": ( + preview.get("source_contract_version") == SOURCE_CONTRACT_VERSION + ), + "adapter_code_present": bool(preview.get("adapter_code") or receipt.get("adapter_code")), + "source_count_positive": int(preview.get("source_count") or 0) > 0, + "source_contract_checks_all_passed": ( + int(receipt.get("source_contract_check_pass_count") or 0) + == int(receipt.get("source_contract_check_count") or -1) + and int(receipt.get("source_contract_check_count") or 0) > 0 + ), + "blocked_page_not_product_data": bool( + source_checks.get("blocked_page_not_product_data") + ), + "database_write_disallowed": bool(source_checks.get("database_write_disallowed")), + "network_request_deferred": bool(source_checks.get("network_request_deferred")), + "public_source_boundary_required": "public_source_boundary" in required_before, + "rate_limit_contract_required": "rate_limit_contract" in required_before, + "provenance_contract_required": "provenance_contract" in required_before, + "identity_and_promotion_gates_required": ( + "identity_matcher_replay" in required_before + and "promotion_gate" in required_before + ), + "canonical_offer_fields_present": { + "platform", + "title", + "url", + "price", + "currency", + "evidence_time", + "source_method", + "confidence", + }.issubset(expected_fields), + "direct_price_write_disallowed": not bool( + promotion.get("direct_price_write_allowed") + ), + } + + +def _preflight_payload(receipt: Mapping[str, Any]) -> dict[str, Any]: + preview = receipt.get("source_contract_preview") + if not isinstance(preview, Mapping): + preview = {} + return { + "adapter_preflight_version": ADAPTER_PREFLIGHT_VERSION, + "source_contract_version": preview.get("source_contract_version"), + "adapter_code": preview.get("adapter_code") or receipt.get("adapter_code"), + "contract_id": preview.get("contract_id"), + "barrier_type": preview.get("barrier_type") or receipt.get("barrier_type"), + "capture_runtime_unavailable": bool( + preview.get("capture_runtime_unavailable") + or receipt.get("capture_runtime_unavailable") + ), + "source_count": int(preview.get("source_count") or 0), + "expected_offer_fields": list(preview.get("expected_offer_fields") or []), + "required_before_data_promotion": list( + preview.get("required_before_data_promotion") or [] + ), + "adapter_execution_mode": "preflight_only_no_network_no_db", + "allowed_next_step": "run_marketplace_source_contract_adapter_dry_run", + } + + +def _worker_item( + receipt_path: Path, + *, + now: datetime, + max_age_hours: int, + execute: bool, +) -> dict[str, Any]: + receipt, errors = _load_receipt(receipt_path) + generated_at = _parse_iso_datetime(receipt.get("generated_at")) + if generated_at is None: + try: + generated_at = datetime.fromtimestamp(receipt_path.stat().st_mtime, tz=timezone.utc) + except OSError: + generated_at = None + age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None + stale = age_hours is None or age_hours > max_age_hours + checks = _adapter_preflight_checks(receipt, stale=stale, errors=errors) + check_count = len(checks) + pass_count = sum(1 for passed in checks.values() if passed) + ready = pass_count == check_count + platform = str(receipt.get("platform") or receipt_path.parent.parent.name).strip().lower() + manifest_id = str(receipt.get("manifest_id") or receipt_path.parent.name).strip() + status = ( + "executed_adapter_preflight_ready" + if execute and ready + else ( + "dry_run_ready_for_adapter_preflight" + if ready + else "skipped_adapter_preflight_guard_failed" + ) + ) + return { + "worker_status": status, + "platform": platform, + "manifest_id": manifest_id, + "source_type": "source_contract_replay_receipt", + "source_contract_replay_receipt_path": str(receipt_path), + "source_worker_receipt_path": receipt.get("source_worker_receipt_path"), + "source_receipt_path": receipt.get("source_receipt_path"), + "adapter_code": (receipt.get("source_contract_preview") or {}).get("adapter_code") + if isinstance(receipt.get("source_contract_preview"), Mapping) + else receipt.get("adapter_code"), + "adapter_preflight_status": "ready" if ready else "blocked", + "ready_for_execution": ready, + "execute": bool(execute), + "generated_at": generated_at.isoformat() if generated_at else None, + "age_hours": round(age_hours, 3) if age_hours is not None else None, + "stale": stale, + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": False, + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + "adapter_preflight_checks": checks, + "adapter_preflight_check_count": check_count, + "adapter_preflight_check_pass_count": pass_count, + "adapter_preflight": _preflight_payload(receipt), + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "adapter_preflight_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": ( + "run_marketplace_source_contract_adapter_dry_run" + if execute and ready + else ( + "run_marketplace_source_contract_adapter_preflight_execute" + if ready + else "repair_marketplace_source_contract_adapter_preflight_inputs" + ) + ), + } + + +def _write_preflight_receipt(*, output_root: Path, item: Mapping[str, Any]) -> str: + target = ( + output_root + / _safe_segment(item.get("platform")) + / _safe_segment(item.get("manifest_id")) + / "marketplace_adapter_preflight_receipt.json" + ) + target.parent.mkdir(parents=True, exist_ok=True) + receipt = dict(item) + receipt["artifact_write_performed"] = True + receipt["receipt_path"] = str(target) + receipt["generated_at"] = datetime.now(timezone.utc).isoformat() + receipt["policy"] = POLICY + target.write_text( + json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + return str(target) + + +def run_pixelrag_marketplace_source_contract_adapter_preflight( + *, + source_contract_replay_receipt_root: str | Path | None = None, + output_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, + execute: bool = False, + write_receipt: bool = False, +) -> dict[str, Any]: + """Run or dry-run marketplace source-contract adapter preflight.""" + source_root = Path( + source_contract_replay_receipt_root + or DEFAULT_SOURCE_CONTRACT_REPLAY_RECEIPT_ROOT + ) + output = Path(output_root or DEFAULT_OUTPUT_ROOT) + platforms = _normalise_platforms(platform) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + now = datetime.now(timezone.utc) + receipt_paths = _receipt_candidates(source_root, platforms=platforms, limit=item_limit) + worker_items: list[dict[str, Any]] = [] + for receipt_path in receipt_paths: + item = _worker_item( + receipt_path, + now=now, + max_age_hours=max_age, + execute=execute, + ) + if execute and write_receipt and item.get("ready_for_execution"): + item["receipt_path"] = _write_preflight_receipt(output_root=output, item=item) + item["artifact_write_performed"] = True + worker_items.append(item) + + ready_count = sum(1 for item in worker_items if item.get("ready_for_execution")) + blocked_count = len(worker_items) - ready_count + dry_run_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_") + ) + executed_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_") + ) + receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path")) + guard_failed_count = sum( + 1 for item in worker_items if item.get("adapter_preflight_status") != "ready" + ) + capture_runtime_gap_count = sum( + 1 + for item in worker_items + if (item.get("adapter_preflight") or {}).get("capture_runtime_unavailable") + ) + if not worker_items: + status = "warning" + elif guard_failed_count: + status = "warning" + else: + status = "ok" + + if not worker_items: + next_action = "run_pixelrag_source_contract_replay_worker" + elif not execute and ready_count: + next_action = "run_marketplace_source_contract_adapter_preflight_execute" + elif execute and receipt_written_count: + next_action = "run_marketplace_source_contract_adapter_dry_run" + elif guard_failed_count: + next_action = "repair_marketplace_source_contract_adapter_preflight_inputs" + else: + next_action = "refresh_marketplace_source_contract_adapter_preflight_candidates" + + summary = { + "candidate_count": len(worker_items), + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "guard_failed_count": guard_failed_count, + "capture_runtime_gap_count": capture_runtime_gap_count, + "platforms": sorted({str(item.get("platform") or "unknown") for item in worker_items}), + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": bool(receipt_written_count), + "writes_database_count": 0, + "primary_human_gate_count": 0, + } + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": now.isoformat(), + "adapter_preflight_version": ADAPTER_PREFLIGHT_VERSION, + "source_contract_replay_receipt_root": str(source_root), + "output_root": str(output), + "platform_filter": list(platforms), + "max_age_hours": max_age, + "limit": item_limit, + "execute": bool(execute), + "write_receipt": bool(write_receipt and execute), + "summary": summary, + "worker_items": worker_items, + "controlled_apply": { + "network_call": False, + "model_call": False, + "artifact_write": bool(receipt_written_count), + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "secret_read": False, + "raw_cookie_or_session_read": False, + "credentialed_session_allowed": False, + "login_allowed": False, + "production_price_write": False, + "primary_human_gate_count": 0, + }, + "promotion_boundary": { + "writes_ai_insights": False, + "writes_price_tables": False, + "adapter_preflight_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "ADAPTER_PREFLIGHT_VERSION", + "DEFAULT_OUTPUT_ROOT", + "POLICY", + "run_pixelrag_marketplace_source_contract_adapter_preflight", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index 47dbe77..a60cad0 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1428,12 +1428,15 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_pchome_auto_policy_controlled_apply_final_preflight_check", lambda: smoke._check("auto-policy controlled apply final preflight", "ok", "ok")) monkeypatch.setattr(smoke, "_pchome_auto_policy_controlled_dry_run_package_check", lambda: smoke._check("auto-policy controlled dry-run package", "ok", "ok")) monkeypatch.setattr(smoke, "_pchome_auto_policy_controlled_dry_run_receipt_closeout_check", lambda: smoke._check("auto-policy controlled dry-run receipt closeout", "ok", "ok")) + monkeypatch.setattr(smoke, "_pchome_auto_policy_controlled_dry_run_runner_readiness_check", lambda: smoke._check("auto-policy controlled dry-run runner readiness", "ok", "ok")) + monkeypatch.setattr(smoke, "_pchome_auto_policy_controlled_dry_run_runner_readiness_check", lambda: smoke._check("auto-policy controlled dry-run runner readiness", "ok", "ok")) monkeypatch.setattr(smoke, "_ai_surface_html_readback_check", lambda: smoke._check("surface", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_ui_ux_agent_check", lambda: smoke._check("sitewide", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_preflight_check", lambda: smoke._check("pixelrag marketplace adapter preflight", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -1443,7 +1446,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 37, "warning": 1, "critical": 1, "total": 39} + assert result["summary"] == {"ok": 38, "warning": 1, "critical": 1, "total": 40} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3967,12 +3970,14 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_pchome_auto_policy_controlled_apply_final_preflight_check", lambda: smoke._check("auto-policy controlled apply final preflight", "ok", "ok")) monkeypatch.setattr(smoke, "_pchome_auto_policy_controlled_dry_run_package_check", lambda: smoke._check("auto-policy controlled dry-run package", "ok", "ok")) monkeypatch.setattr(smoke, "_pchome_auto_policy_controlled_dry_run_receipt_closeout_check", lambda: smoke._check("auto-policy controlled dry-run receipt closeout", "ok", "ok")) + monkeypatch.setattr(smoke, "_pchome_auto_policy_controlled_dry_run_runner_readiness_check", lambda: smoke._check("auto-policy controlled dry-run runner readiness", "ok", "ok")) monkeypatch.setattr(smoke, "_ai_surface_html_readback_check", lambda: smoke._check("surface", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_ui_ux_agent_check", lambda: smoke._check("sitewide", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_preflight_check", lambda: smoke._check("pixelrag marketplace adapter preflight", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -4033,7 +4038,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 39, "warning": 0, "critical": 0, "total": 39}, + "summary": {"ok": 40, "warning": 0, "critical": 0, "total": 40}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4067,6 +4072,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( _auto_policy_controlled_apply_final_preflight_history_check(), _auto_policy_controlled_dry_run_package_history_check(), _auto_policy_controlled_dry_run_receipt_closeout_history_check(), + _auto_policy_controlled_dry_run_runner_readiness_history_check(), { "name": "AI surface HTML readback", "status": "ok", @@ -4163,6 +4169,27 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "primary_human_gate_count": 0, }, }, + { + "name": "PixelRAG marketplace adapter preflight", + "status": "ok", + "summary": "PixelRAG marketplace adapter preflight candidates=1, ready=1, blocked=0, dry_run=1, executed=0, receipts=0", + "details": { + "policy": "controlled_pixelrag_marketplace_source_contract_adapter_preflight_v1", + "candidate_count": 1, + "ready_count": 1, + "blocked_count": 0, + "dry_run_count": 1, + "executed_count": 0, + "receipt_written_count": 0, + "capture_runtime_gap_count": 0, + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": "run_marketplace_source_contract_adapter_preflight_execute", + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "name": "PixelRAG OCR/VLM replay contract", "status": "ok", @@ -4277,7 +4304,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 36 + assert summary["summary"]["total"] == 37 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -6614,6 +6641,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag source contract ok", )) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_preflight_check", lambda: smoke._check( + "PixelRAG marketplace adapter preflight", + "ok", + "pixelrag marketplace adapter preflight ok", + )) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check( "PixelRAG OCR/VLM replay contract", "ok", @@ -6654,7 +6686,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 39 + assert result["summary"]["total"] == 40 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_marketplace_source_contract_adapter_preflight_service.py b/tests/test_pixelrag_marketplace_source_contract_adapter_preflight_service.py new file mode 100644 index 0000000..2432577 --- /dev/null +++ b/tests/test_pixelrag_marketplace_source_contract_adapter_preflight_service.py @@ -0,0 +1,232 @@ +import json +import subprocess +import sys +from datetime import datetime, timezone + + +def _write_source_contract_replay_receipt( + root, + *, + platform="coupang_tw", + manifest_id="coupang-403", + adapter_code="coupang", + barrier_type="access_denied", + capture_runtime_unavailable=False, +): + receipt_dir = root / platform / manifest_id + receipt_dir.mkdir(parents=True) + payload = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "policy": "controlled_pixelrag_source_contract_replay_worker_v1", + "worker_status": "executed_source_contract_replay_ready", + "platform": platform, + "manifest_id": manifest_id, + "adapter_code": adapter_code, + "source_worker_receipt_path": ( + f"runtime_artifacts/pixelrag_platform_probe_worker_receipts/" + f"{platform}/{manifest_id}/platform_probe_worker_receipt.json" + ), + "source_receipt_path": ( + f"runtime_artifacts/pixelrag_visual_evidence/" + f"{platform}/{manifest_id}/capture_receipt.json" + ), + "source_contract_check_count": 8, + "source_contract_check_pass_count": 8, + "source_contract_checks": { + "blocked_page_not_product_data": True, + "database_write_disallowed": True, + "dry_run_only_contract": True, + "network_request_deferred": True, + "source_count_positive": True, + "source_receipt_path_present": True, + "source_worker_receipt_path_present": True, + "structured_source_package_available": True, + }, + "source_contract_preview": { + "source_contract_version": "pixelrag_marketplace_source_contract_v1", + "contract_id": f"{platform}:{manifest_id}:{adapter_code}", + "adapter_code": adapter_code, + "barrier_type": barrier_type, + "capture_runtime_unavailable": capture_runtime_unavailable, + "source_count": 2, + "expected_offer_fields": [ + "platform", + "platform_product_id", + "title", + "url", + "image_url", + "price", + "currency", + "availability", + "seller", + "promotion_badges", + "shipping_badges", + "evidence_time", + "source_method", + "confidence", + ], + "required_before_data_promotion": [ + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", + ], + }, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + }, + "writes_database": False, + "network_call_performed": False, + "model_call_performed": False, + } + path = receipt_dir / "source_contract_replay_receipt.json" + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return path + + +def test_marketplace_source_contract_adapter_preflight_dry_run_reads_receipts(tmp_path): + from services.pixelrag_marketplace_source_contract_adapter_preflight_service import ( + POLICY, + run_pixelrag_marketplace_source_contract_adapter_preflight, + ) + + source_root = tmp_path / "source_contracts" + _write_source_contract_replay_receipt(source_root) + _write_source_contract_replay_receipt( + source_root, + platform="shopee_tw", + manifest_id="shopee-runtime-gap", + adapter_code="shopee", + barrier_type="language_or_region_interstitial", + capture_runtime_unavailable=True, + ) + + payload = run_pixelrag_marketplace_source_contract_adapter_preflight( + source_contract_replay_receipt_root=source_root, + platform=("coupang_tw", "shopee_tw"), + ) + by_platform = {item["platform"]: item for item in payload["worker_items"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "ok" + assert payload["summary"]["candidate_count"] == 2 + assert payload["summary"]["ready_count"] == 2 + assert payload["summary"]["dry_run_count"] == 2 + assert payload["summary"]["capture_runtime_gap_count"] == 1 + assert payload["summary"]["writes_database_count"] == 0 + assert payload["controlled_apply"]["network_call"] is False + assert payload["controlled_apply"]["writes_database"] is False + assert payload["next_machine_action"] == ( + "run_marketplace_source_contract_adapter_preflight_execute" + ) + assert by_platform["coupang_tw"]["worker_status"] == ( + "dry_run_ready_for_adapter_preflight" + ) + assert by_platform["coupang_tw"]["adapter_preflight_checks"][ + "rate_limit_contract_required" + ] is True + assert by_platform["shopee_tw"]["adapter_preflight"][ + "capture_runtime_unavailable" + ] is True + + +def test_marketplace_source_contract_adapter_preflight_execute_writes_receipt(tmp_path): + from services.pixelrag_marketplace_source_contract_adapter_preflight_service import ( + run_pixelrag_marketplace_source_contract_adapter_preflight, + ) + + source_root = tmp_path / "source_contracts" + output_root = tmp_path / "adapter_preflight" + _write_source_contract_replay_receipt(source_root) + + payload = run_pixelrag_marketplace_source_contract_adapter_preflight( + source_contract_replay_receipt_root=source_root, + output_root=output_root, + platform="coupang_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["executed_count"] == 1 + assert payload["summary"]["receipt_written_count"] == 1 + assert payload["controlled_apply"]["artifact_write"] is True + assert payload["controlled_apply"]["writes_database"] is False + assert payload["next_machine_action"] == "run_marketplace_source_contract_adapter_dry_run" + receipt_path = ( + output_root + / "coupang_tw" + / "coupang-403" + / "marketplace_adapter_preflight_receipt.json" + ) + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "executed_adapter_preflight_ready" + assert receipt["artifact_write_performed"] is True + assert receipt["writes_database"] is False + assert receipt["adapter_preflight"]["adapter_preflight_version"] == ( + "pixelrag_marketplace_adapter_preflight_v1" + ) + + +def test_marketplace_source_contract_adapter_preflight_cli_outputs_json(tmp_path): + source_root = tmp_path / "source_contracts" + _write_source_contract_replay_receipt(source_root) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py", + "--source-contract-replay-receipt-root", + str(source_root), + "--platform", + "coupang_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False + + +def test_marketplace_source_contract_adapter_preflight_route_returns_readback( + tmp_path, + monkeypatch, +): + from flask import Flask + from routes import system_public_routes as routes + from services import pixelrag_marketplace_source_contract_adapter_preflight_service as service + + source_root = tmp_path / "source_contracts" + _write_source_contract_replay_receipt(source_root) + monkeypatch.setattr( + service, + "DEFAULT_SOURCE_CONTRACT_REPLAY_RECEIPT_ROOT", + str(source_root), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight?platform=coupang_tw" + ): + response = ( + routes + .ai_automation_pixelrag_marketplace_source_contract_adapter_preflight_api + .__wrapped__() + ) + payload = response.get_json() + + assert payload["policy"] == ( + "controlled_pixelrag_marketplace_source_contract_adapter_preflight_v1" + ) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False From 8877261ae46d73d0579c6ba7a4e80a38ca8ed3b5 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 11:32:30 +0800 Subject: [PATCH 24/75] feat(ai): add PixelRAG marketplace adapter dry run --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 4 +- .../ai_automation_mainline_work_items.md | 3 +- docs/guides/browse_sh_crawler_playbook.md | 4 + routes/system_public_routes.py | 37 ++ ...etplace_source_contract_adapter_dry_run.py | 77 +++ services/ai_automation_smoke_service.py | 141 +++++ ...source_contract_adapter_dry_run_service.py | 539 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 38 +- ...source_contract_adapter_dry_run_service.py | 259 +++++++++ 10 files changed, 1097 insertions(+), 7 deletions(-) create mode 100755 scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py create mode 100644 services/pixelrag_marketplace_source_contract_adapter_dry_run_service.py create mode 100644 tests/test_pixelrag_marketplace_source_contract_adapter_dry_run_service.py diff --git a/config.py b/config.py index 9ee2139..9fced18 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.765" +SYSTEM_VERSION = "V10.766" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 5fd8f4d..3fe2ef2 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.765 +> **適用版本**: V10.766 --- @@ -121,6 +121,7 @@ - 2026-07-10 起 PixelRAG platform probe 必須把 blocked / interstitial receipt 轉成機器可執行下一步：`/api/ai-automation/pixelrag-platform-probe` 與 `scripts/ops/report_pixelrag_platform_probe.py` 需讀取 capture receipts 與 VLM replay receipts，辨識 Shopee language / region / generic marketplace slogan / traffic verification 與 Coupang 403 / access denied，輸出 public browser context probe、structured source fallback、platform backoff、blocked page not product data 與 no-write 邊界；`/api/ai-automation/smoke` 需包含 `PixelRAG platform probe readiness`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_platform_probe` family。此 readback 不抓外站、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture worker 可消化 `public_browser_context` 的 locale/timezone/Accept-Language，但必須濾掉 Cookie、Authorization、token/secret/key 類 header。 - 2026-07-10 起 PixelRAG platform probe 必須有 controlled worker：`/api/ai-automation/pixelrag-platform-probe-worker` 與 `scripts/ops/run_pixelrag_platform_probe_worker.py` 預設 dry-run，`execute=true&write_receipt=true` 才會對 Shopee language / traffic / generic landing 執行 public empty-context capture，並對 Coupang 403 / access denied 自動產生 structured-source/backoff artifact receipt；`/api/ai-automation/smoke` 需包含 `PixelRAG platform probe worker`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_platform_probe_worker` family。此 worker 不登入、不讀 cookie/session/secret、不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture 成功後下一步固定為 RAG candidate replay，structured fallback 只產生機器可讀 package，不把 blocked page 當商品資料。 - 2026-07-10 起 PixelRAG platform probe worker 必須自動處理 capture runtime gap：若 production app 容器缺 Playwright，`execute=true&write_receipt=true` 不得退回人工審核或 critical dead-end；worker 必須寫 `capture_runtime_unavailable_structured_fallback_package` artifact receipt，標記 `network_call=false`、`writes_database=false`、`primary_human_gate_count=0`，並把下一步改為 `run_structured_source_or_install_pixelrag_capture_runtime`。 +- 2026-07-10 起 PixelRAG marketplace source-contract adapter dry-run 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run` 與 `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py` 預設只讀 adapter preflight receipts 並做 deterministic mapping readiness；`execute=true&write_receipt=true` 只寫 `marketplace_adapter_dry_run_receipt.json` artifact，輸出 field contracts、offer candidate contracts、identity matcher / PromotionGate / embedding signature gates 與 no-write 邊界。此 worker 不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；dry-run receipt 只允許進 identity matcher replay，不得直接進價格表或候選知識層。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -898,6 +899,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG RAG replay 下一步不得隱藏並行主線 | V10.763 起若同一批 PixelRAG replay 同時存在 ready visual receipt 與 platform probe worker `source_contract_fallback` receipt，`next_machine_action` 必須輸出 `run_ocr_vlm_replay_and_source_contract_replay`，避免只顯示 OCR/VLM 而把 marketplace source-contract 自動化 lane 藏在 summary 裡。 | | 2026-07-10 | PixelRAG source-contract fallback 必須有 controlled replay worker | V10.764 起 `/api/ai-automation/pixelrag-source-contract-replay-worker`、`scripts/ops/run_pixelrag_source_contract_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG source-contract replay worker / `pixelrag_source_contract_replay_worker` family；worker 只讀 platform probe worker `source_contract_fallback` receipts，驗證 structured package、source count、no-DB、network-deferred、blocked-page-not-product-data、source receipt provenance，execute 模式只寫 `source_contract_replay_receipt.json` artifact。此路徑不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 marketplace source-contract adapter replay preflight。 | | 2026-07-10 | PixelRAG marketplace source-contract adapter 必須先有 no-write preflight | V10.765 起 `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight`、`scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace adapter preflight / `pixelrag_marketplace_adapter_preflight` family；preflight 只讀 `source_contract_replay_receipt.json`，驗證 replay ready、source contract version、adapter code、source count、source checks all passed、blocked page not product data、no-DB、network-deferred、public source boundary、rate-limit contract、provenance contract、identity/promotion gates 與 canonical offer fields。execute 模式只寫 `marketplace_adapter_preflight_receipt.json` artifact，不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 marketplace source-contract adapter dry-run。 | +| 2026-07-10 | PixelRAG marketplace adapter dry-run 必須先產生 no-write mapping receipt | V10.766 起 `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run`、`scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace adapter dry-run / `pixelrag_marketplace_adapter_dry_run` family；dry-run 只讀 `marketplace_adapter_preflight_receipt.json`，驗證 preflight ready、canonical offer fields、required gates、no-DB/no-network/no-model、blocked page not product data 與 price-write blocked。execute 模式只寫 `marketplace_adapter_dry_run_receipt.json` artifact，產生 deterministic field contracts / offer candidate contracts，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 identity matcher replay。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 7c77f95..d18ce4c 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -19,11 +19,12 @@ | Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read visual capture receipts and platform probe worker receipts, split eligible / blocked / source-contract-fallback candidates, and expose combined OCR/VLM + source-contract next actions before identity matching and PromotionGate knowledge writes. | | Completed | PixelRAG source-contract replay worker | `/api/ai-automation/pixelrag-source-contract-replay-worker` and `scripts/ops/run_pixelrag_source_contract_replay_worker.py` turn platform probe worker `source_contract_fallback` receipts into replayable source-contract artifact receipts with provenance, public-boundary, no-DB, and blocked-page-not-product-data guards. | | Completed | PixelRAG marketplace source-contract adapter preflight | `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight` and `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py` validate source-contract replay receipts for adapter readiness, canonical offer fields, provenance/rate-limit/public-boundary requirements, and no-write promotion gates. | +| Completed | PixelRAG marketplace adapter dry-run | `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run` and `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py` turn adapter preflight receipts into deterministic no-write field contracts and offer candidate contracts, with price writes blocked until identity matcher replay and PromotionGate. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | | Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture when runtime is available, missing Playwright auto-falls back to structured-source/backoff receipt, and Coupang 403/access denied goes to structured-source/backoff artifact receipts. | -| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, and PixelRAG platform probe worker families. | +| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, PixelRAG platform probe worker, source-contract replay, adapter preflight, and adapter dry-run families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | ## P1 diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index 6a87483..77b7d3b 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -200,6 +200,8 @@ python scripts/ops/run_pixelrag_source_contract_replay_worker.py --platform shop python scripts/ops/run_pixelrag_source_contract_replay_worker.py --platform coupang_tw --execute --write-receipt --limit 1 python scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py --platform shopee_tw --platform coupang_tw python scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py --platform coupang_tw --execute --write-receipt --limit 1 +python scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py --platform coupang_tw --execute --write-receipt --limit 1 ``` API readback: `/api/ai-automation/pixelrag-vlm-route-readiness` 與 `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。worker 預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。execute 前會自動讀 approved Ollama `/api/tags` 選擇已安裝候選模型；若完全沒有候選，會寫 `model_route_not_ready` artifact receipt，不盲打 generate。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 @@ -210,6 +212,8 @@ Source-contract replay worker API readback: `/api/ai-automation/pixelrag-source- Marketplace adapter preflight API readback: `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight?platform=coupang_tw`。preflight 預設為 dry-run，不抓外站、不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才把 source-contract replay receipt 轉成 adapter preflight artifact。artifact 只證明 adapter dry-run 條件齊備，不得直接寫 `ai_insights`、正式價格表或競品價格歷史。 +Marketplace adapter dry-run API readback: `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run?platform=coupang_tw`。dry-run 預設只讀 adapter preflight receipts 並建立 deterministic field contracts；`execute=true&write_receipt=true` 只寫 adapter dry-run artifact receipt，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表。dry-run artifact 下一步只能進 identity matcher replay 與 PromotionGate，不得直接變成價格資料。 + Application portfolio： ```bash diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 3ea71ff..db9b7a0 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -979,6 +979,43 @@ def ai_automation_pixelrag_marketplace_source_contract_adapter_preflight_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run') +@login_required +def ai_automation_pixelrag_marketplace_source_contract_adapter_dry_run_api(): + """Controlled PixelRAG marketplace source-contract adapter dry-run readback.""" + from services.pixelrag_marketplace_source_contract_adapter_dry_run_service import ( + run_pixelrag_marketplace_source_contract_adapter_dry_run, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + execute = str(request.args.get('execute') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + write_receipt = str(request.args.get('write_receipt') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 25, type=int) + return jsonify(run_pixelrag_marketplace_source_contract_adapter_dry_run( + adapter_preflight_receipt_root=str( + request.args.get('adapter_preflight_receipt_root') or '' + ).strip() or None, + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 25, 250)), + execute=execute, + write_receipt=bool(execute and write_receipt), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py b/scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py new file mode 100755 index 0000000..f423fc1 --- /dev/null +++ b/scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Run or dry-run PixelRAG marketplace source-contract adapter dry-run.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_marketplace_source_contract_adapter_dry_run_service import ( # noqa: E402 + run_pixelrag_marketplace_source_contract_adapter_dry_run, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="執行或 dry-run PixelRAG marketplace source-contract adapter dry-run。" + ) + parser.add_argument( + "--adapter-preflight-receipt-root", + help="Marketplace adapter preflight receipt root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--output-root", + help="Marketplace adapter dry-run artifact receipt output root。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=25, + help="最多處理 adapter preflight receipt 數。", + ) + parser.add_argument( + "--execute", + action="store_true", + help="執行 adapter dry-run；未指定時只做 no-write dry-run readiness。", + ) + parser.add_argument( + "--write-receipt", + action="store_true", + help="execute 後寫入 adapter dry-run artifact receipt；不寫 DB。", + ) + args = parser.parse_args() + + payload = run_pixelrag_marketplace_source_contract_adapter_dry_run( + adapter_preflight_receipt_root=args.adapter_preflight_receipt_root, + output_root=args.output_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + execute=args.execute, + write_receipt=bool(args.write_receipt and args.execute), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index 59196c2..e2294bc 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -569,6 +569,23 @@ def build_scheduled_automation_health_summary( pixelrag_marketplace_adapter_preflight_details = ( pixelrag_marketplace_adapter_preflight.get("details") or {} ) + pixelrag_marketplace_adapter_dry_run = _find_check( + source_result, + "PixelRAG marketplace adapter dry-run", + ) + pixelrag_marketplace_adapter_dry_run_details = ( + pixelrag_marketplace_adapter_dry_run.get("details") or {} + ) + if ( + not pixelrag_marketplace_adapter_dry_run + or not pixelrag_marketplace_adapter_dry_run_details + ): + pixelrag_marketplace_adapter_dry_run = ( + _pixelrag_marketplace_adapter_dry_run_check() + ) + pixelrag_marketplace_adapter_dry_run_details = ( + pixelrag_marketplace_adapter_dry_run.get("details") or {} + ) pixelrag_ocr_vlm_replay = _find_check(source_result, "PixelRAG OCR/VLM replay contract") pixelrag_ocr_vlm_replay_details = pixelrag_ocr_vlm_replay.get("details") or {} if not pixelrag_ocr_vlm_replay or not pixelrag_ocr_vlm_replay_details: @@ -4759,6 +4776,61 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_marketplace_adapter_dry_run", + "label": "PixelRAG marketplace adapter dry-run", + "status": pixelrag_marketplace_adapter_dry_run.get("status") or "warning", + "summary": ( + pixelrag_marketplace_adapter_dry_run.get("summary") + or "PixelRAG marketplace adapter dry-run has no latest readback." + ), + "next_machine_action": pixelrag_marketplace_adapter_dry_run_details.get( + "next_machine_action" + ) + or "run_pixelrag_marketplace_adapter_dry_run", + "details": { + "policy": pixelrag_marketplace_adapter_dry_run_details.get("policy"), + "candidate_count": int( + pixelrag_marketplace_adapter_dry_run_details.get("candidate_count") or 0 + ), + "ready_count": int( + pixelrag_marketplace_adapter_dry_run_details.get("ready_count") or 0 + ), + "blocked_count": int( + pixelrag_marketplace_adapter_dry_run_details.get("blocked_count") or 0 + ), + "dry_run_count": int( + pixelrag_marketplace_adapter_dry_run_details.get("dry_run_count") or 0 + ), + "executed_count": int( + pixelrag_marketplace_adapter_dry_run_details.get("executed_count") or 0 + ), + "receipt_written_count": int( + pixelrag_marketplace_adapter_dry_run_details.get("receipt_written_count") or 0 + ), + "capture_runtime_gap_count": int( + pixelrag_marketplace_adapter_dry_run_details.get("capture_runtime_gap_count") or 0 + ), + "field_contract_count": int( + pixelrag_marketplace_adapter_dry_run_details.get("field_contract_count") or 0 + ), + "dry_run_output_contract_count": int( + pixelrag_marketplace_adapter_dry_run_details.get("dry_run_output_contract_count") or 0 + ), + "network_call_performed": bool( + pixelrag_marketplace_adapter_dry_run_details.get("network_call_performed") + ), + "model_call_performed": bool( + pixelrag_marketplace_adapter_dry_run_details.get("model_call_performed") + ), + "artifact_write_performed": bool( + pixelrag_marketplace_adapter_dry_run_details.get("artifact_write_performed") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "key": "pixelrag_ocr_vlm_replay", "label": "PixelRAG OCR/VLM replay contract", @@ -13764,6 +13836,74 @@ def _pixelrag_marketplace_adapter_preflight_check() -> Dict[str, Any]: ) +def _pixelrag_marketplace_adapter_dry_run_check() -> Dict[str, Any]: + """Dry-run sentinel for PixelRAG marketplace adapter deterministic mapping.""" + try: + from services.pixelrag_marketplace_source_contract_adapter_dry_run_service import ( + run_pixelrag_marketplace_source_contract_adapter_dry_run, + ) + + readback = run_pixelrag_marketplace_source_contract_adapter_dry_run( + execute=False, + ) + summary = readback.get("summary") or {} + candidate_count = int(summary.get("candidate_count") or 0) + ready_count = int(summary.get("ready_count") or 0) + blocked_count = int(summary.get("blocked_count") or 0) + dry_run_count = int(summary.get("dry_run_count") or 0) + executed_count = int(summary.get("executed_count") or 0) + receipt_written_count = int(summary.get("receipt_written_count") or 0) + field_contract_count = int(summary.get("field_contract_count") or 0) + output_contract_count = int(summary.get("dry_run_output_contract_count") or 0) + status = readback.get("status") or "warning" + summary_text = ( + f"PixelRAG marketplace adapter dry-run candidates={candidate_count}, " + f"ready={ready_count}, blocked={blocked_count}, dry_run={dry_run_count}, " + f"executed={executed_count}, receipts={receipt_written_count}, " + f"fields={field_contract_count}, outputs={output_contract_count}" + ) + return _check( + "PixelRAG marketplace adapter dry-run", + status, + summary_text, + { + "policy": readback.get("policy"), + "candidate_count": candidate_count, + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "capture_runtime_gap_count": int( + summary.get("capture_runtime_gap_count") or 0 + ), + "guard_failed_count": int(summary.get("guard_failed_count") or 0), + "field_contract_count": field_contract_count, + "dry_run_output_contract_count": output_contract_count, + "network_call_performed": bool(summary.get("network_call_performed")), + "model_call_performed": bool(summary.get("model_call_performed")), + "artifact_write_performed": bool( + summary.get("artifact_write_performed") + ), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG marketplace adapter dry-run", + "critical", + f"PixelRAG marketplace adapter dry-run 無法執行 dry-run：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def _pixelrag_ocr_vlm_replay_check() -> Dict[str, Any]: """Read-only sentinel for PixelRAG receipts becoming OCR/VLM replay contracts.""" try: @@ -14098,6 +14238,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _pixelrag_rag_candidate_replay_check(), _pixelrag_source_contract_replay_worker_check(), _pixelrag_marketplace_adapter_preflight_check(), + _pixelrag_marketplace_adapter_dry_run_check(), _pixelrag_ocr_vlm_replay_check(), _pixelrag_vlm_route_readiness_check(), _pixelrag_vlm_replay_worker_check(), diff --git a/services/pixelrag_marketplace_source_contract_adapter_dry_run_service.py b/services/pixelrag_marketplace_source_contract_adapter_dry_run_service.py new file mode 100644 index 0000000..7e47c1c --- /dev/null +++ b/services/pixelrag_marketplace_source_contract_adapter_dry_run_service.py @@ -0,0 +1,539 @@ +"""Controlled PixelRAG marketplace adapter dry-run worker. + +This worker turns marketplace adapter preflight receipts into deterministic +adapter dry-run receipts. It does not fetch marketplace pages, call models, +write databases, or promote candidate prices into production truth. +""" + +from __future__ import annotations + +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.pixelrag_crawler_integration_service import DEFAULT_ARTIFACT_MAX_AGE_HOURS +from services.pixelrag_marketplace_source_contract_adapter_preflight_service import ( + ADAPTER_PREFLIGHT_VERSION, + DEFAULT_OUTPUT_ROOT as DEFAULT_ADAPTER_PREFLIGHT_RECEIPT_ROOT, +) +from services.pixelrag_source_contract_replay_worker_service import SOURCE_CONTRACT_VERSION + + +POLICY = "controlled_pixelrag_marketplace_source_contract_adapter_dry_run_v1" +ADAPTER_DRY_RUN_VERSION = "pixelrag_marketplace_adapter_dry_run_v1" +DEFAULT_LIMIT = 25 +DEFAULT_OUTPUT_ROOT = os.getenv( + "PIXELRAG_MARKETPLACE_ADAPTER_DRY_RUN_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_marketplace_adapter_dry_run_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_marketplace_adapter_dry_run_receipts", +) + +CANONICAL_OFFER_FIELDS = { + "platform", + "platform_product_id", + "title", + "url", + "image_url", + "price", + "currency", + "availability", + "seller", + "promotion_badges", + "shipping_badges", + "evidence_time", + "source_method", + "confidence", +} +REQUIRED_PROMOTION_GATES = { + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", +} + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...]: + if isinstance(platform, str): + value = platform.strip().lower() + return (value,) if value else () + return tuple( + str(item or "").strip().lower() + for item in (platform or ()) + if str(item or "").strip() + ) + + +def _safe_segment(value: Any) -> str: + text = str(value or "unknown").strip().lower() + text = re.sub(r"[^a-z0-9._-]+", "-", text) + return text.strip("-") or "unknown" + + +def _parse_iso_datetime(value: Any) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + + +def _receipt_candidates( + root: Path, + *, + platforms: tuple[str, ...], + limit: int, +) -> list[Path]: + if not root.exists(): + return [] + candidates: list[Path] = [] + if platforms: + for platform in platforms: + candidates.extend( + (root / platform).glob("*/marketplace_adapter_preflight_receipt.json") + ) + else: + candidates.extend(root.glob("*/*/marketplace_adapter_preflight_receipt.json")) + return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] + + +def _load_receipt(path: Path) -> tuple[dict[str, Any], list[str]]: + try: + return json.loads(path.read_text(encoding="utf-8")), [] + except (OSError, json.JSONDecodeError) as exc: + return {}, [str(exc)[:300]] + + +def _field_source_path(field: str) -> str: + return { + "platform": "literal.platform", + "platform_product_id": "source.product_id_or_url_identity", + "title": "source.title_text", + "url": "source.public_product_url", + "image_url": "source.public_image_url", + "price": "source.price_text_or_amount", + "currency": "source.currency_text_or_locale_default", + "availability": "source.availability_text", + "seller": "source.seller_text", + "promotion_badges": "source.promotion_badges[]", + "shipping_badges": "source.shipping_badges[]", + "evidence_time": "receipt.generated_at", + "source_method": "source_contract.source_method", + "confidence": "adapter.confidence_contract", + }.get(field, f"source.{field}") + + +def _field_contract(field: str) -> dict[str, Any]: + return { + "field": field, + "mapped": True, + "required": field in CANONICAL_OFFER_FIELDS, + "source_path": _field_source_path(field), + "value_materialized": False, + "write_target": None, + "requires_evidence": field in {"price", "title", "url", "evidence_time"}, + "requires_identity_match": field in {"platform_product_id", "title", "url"}, + "requires_promotion_gate": field in { + "price", + "promotion_badges", + "shipping_badges", + "confidence", + }, + } + + +def _preflight_payload(receipt: Mapping[str, Any]) -> Mapping[str, Any]: + payload = receipt.get("adapter_preflight") + return payload if isinstance(payload, Mapping) else {} + + +def _promotion_boundary(receipt: Mapping[str, Any]) -> Mapping[str, Any]: + payload = receipt.get("promotion_boundary") + return payload if isinstance(payload, Mapping) else {} + + +def _expected_fields(preflight: Mapping[str, Any]) -> list[str]: + fields = [str(item) for item in list(preflight.get("expected_offer_fields") or [])] + seen: set[str] = set() + ordered: list[str] = [] + for field in fields + sorted(CANONICAL_OFFER_FIELDS): + if field and field not in seen: + ordered.append(field) + seen.add(field) + return ordered + + +def _adapter_dry_run_payload( + receipt: Mapping[str, Any], + *, + receipt_path: Path, +) -> dict[str, Any]: + preflight = _preflight_payload(receipt) + fields = _expected_fields(preflight) + source_count = int(preflight.get("source_count") or 0) + output_count = max(0, min(source_count, 50)) + field_contracts = [_field_contract(field) for field in fields] + return { + "adapter_dry_run_version": ADAPTER_DRY_RUN_VERSION, + "adapter_preflight_version": preflight.get("adapter_preflight_version"), + "source_contract_version": preflight.get("source_contract_version"), + "adapter_code": preflight.get("adapter_code") or receipt.get("adapter_code"), + "contract_id": preflight.get("contract_id"), + "barrier_type": preflight.get("barrier_type") or receipt.get("barrier_type"), + "capture_runtime_unavailable": bool(preflight.get("capture_runtime_unavailable")), + "input_preflight_receipt_path": str(receipt_path), + "source_contract_replay_receipt_path": receipt.get( + "source_contract_replay_receipt_path" + ), + "source_worker_receipt_path": receipt.get("source_worker_receipt_path"), + "source_receipt_path": receipt.get("source_receipt_path"), + "dry_run_execution_mode": "deterministic_mapping_only_no_network_no_db", + "source_count": source_count, + "dry_run_output_contract_count": output_count, + "expected_offer_fields": fields, + "field_contract_count": len(field_contracts), + "field_contracts": field_contracts, + "offer_candidate_contracts": [ + { + "candidate_index": index, + "candidate_id": ( + f"{receipt.get('platform') or 'unknown'}:" + f"{receipt.get('manifest_id') or 'unknown'}:{index}" + ), + "field_count": len(field_contracts), + "field_contracts_materialized": True, + "values_materialized": False, + "ready_for_identity_matcher_replay": True, + "ready_for_price_table_write": False, + "price_write_blocked_until_promotion_gate": True, + } + for index in range(output_count) + ], + "required_before_data_promotion": list( + preflight.get("required_before_data_promotion") or [] + ), + "allowed_next_step": "run_marketplace_adapter_identity_matcher_replay", + } + + +def _adapter_dry_run_checks( + receipt: Mapping[str, Any], + *, + stale: bool, + errors: list[str], + dry_run_payload: Mapping[str, Any], +) -> dict[str, bool]: + preflight = _preflight_payload(receipt) + boundary = _promotion_boundary(receipt) + preflight_checks = receipt.get("adapter_preflight_checks") + if not isinstance(preflight_checks, Mapping): + preflight_checks = {} + required_before = set( + str(item) for item in list(preflight.get("required_before_data_promotion") or []) + ) + expected_fields = set(str(item) for item in list(preflight.get("expected_offer_fields") or [])) + return { + "receipt_parse_ok": not errors, + "receipt_fresh": not stale, + "adapter_preflight_ready": ( + receipt.get("worker_status") == "executed_adapter_preflight_ready" + ), + "adapter_preflight_version_supported": ( + preflight.get("adapter_preflight_version") == ADAPTER_PREFLIGHT_VERSION + ), + "source_contract_version_supported": ( + preflight.get("source_contract_version") == SOURCE_CONTRACT_VERSION + ), + "adapter_preflight_checks_all_passed": ( + int(receipt.get("adapter_preflight_check_pass_count") or 0) + == int(receipt.get("adapter_preflight_check_count") or -1) + and int(receipt.get("adapter_preflight_check_count") or 0) > 0 + ), + "blocked_page_not_product_data": bool( + preflight_checks.get("blocked_page_not_product_data") + ), + "adapter_code_present": bool(preflight.get("adapter_code") or receipt.get("adapter_code")), + "source_count_positive": int(preflight.get("source_count") or 0) > 0, + "canonical_offer_fields_present": CANONICAL_OFFER_FIELDS.issubset(expected_fields), + "promotion_gates_required": REQUIRED_PROMOTION_GATES.issubset(required_before), + "direct_price_write_disallowed": not bool( + boundary.get("direct_price_write_allowed") + ), + "direct_ai_insights_write_disallowed": not bool( + boundary.get("direct_ai_insights_write_allowed") + ), + "database_write_disallowed": not bool(receipt.get("writes_database")) + and int(receipt.get("writes_database_count") or 0) == 0, + "network_call_disallowed": not bool(receipt.get("network_call_performed")), + "model_call_disallowed": not bool(receipt.get("model_call_performed")), + "dry_run_schema_generated": bool(dry_run_payload.get("field_contracts")), + "dry_run_output_contract_generated": int( + dry_run_payload.get("dry_run_output_contract_count") or 0 + ) + > 0, + "price_write_blocked_until_promotion_gate": all( + bool(item.get("price_write_blocked_until_promotion_gate")) + for item in list(dry_run_payload.get("offer_candidate_contracts") or []) + ), + } + + +def _worker_item( + receipt_path: Path, + *, + now: datetime, + max_age_hours: int, + execute: bool, +) -> dict[str, Any]: + receipt, errors = _load_receipt(receipt_path) + generated_at = _parse_iso_datetime(receipt.get("generated_at")) + if generated_at is None: + try: + generated_at = datetime.fromtimestamp(receipt_path.stat().st_mtime, tz=timezone.utc) + except OSError: + generated_at = None + age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None + stale = age_hours is None or age_hours > max_age_hours + dry_run_payload = _adapter_dry_run_payload(receipt, receipt_path=receipt_path) + checks = _adapter_dry_run_checks( + receipt, + stale=stale, + errors=errors, + dry_run_payload=dry_run_payload, + ) + check_count = len(checks) + pass_count = sum(1 for passed in checks.values() if passed) + ready = pass_count == check_count + platform = str(receipt.get("platform") or receipt_path.parent.parent.name).strip().lower() + manifest_id = str(receipt.get("manifest_id") or receipt_path.parent.name).strip() + status = ( + "executed_marketplace_adapter_dry_run_ready" + if execute and ready + else ( + "dry_run_ready_for_marketplace_adapter_dry_run" + if ready + else "skipped_marketplace_adapter_dry_run_guard_failed" + ) + ) + return { + "worker_status": status, + "platform": platform, + "manifest_id": manifest_id, + "source_type": "marketplace_adapter_preflight_receipt", + "adapter_preflight_receipt_path": str(receipt_path), + "source_contract_replay_receipt_path": receipt.get( + "source_contract_replay_receipt_path" + ), + "source_worker_receipt_path": receipt.get("source_worker_receipt_path"), + "source_receipt_path": receipt.get("source_receipt_path"), + "adapter_code": dry_run_payload.get("adapter_code"), + "adapter_dry_run_status": "ready" if ready else "blocked", + "ready_for_execution": ready, + "execute": bool(execute), + "generated_at": generated_at.isoformat() if generated_at else None, + "age_hours": round(age_hours, 3) if age_hours is not None else None, + "stale": stale, + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": False, + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + "adapter_dry_run_checks": checks, + "adapter_dry_run_check_count": check_count, + "adapter_dry_run_check_pass_count": pass_count, + "adapter_dry_run": dry_run_payload, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "adapter_dry_run_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": ( + "run_marketplace_adapter_identity_matcher_replay" + if execute and ready + else ( + "run_marketplace_source_contract_adapter_dry_run_execute" + if ready + else "repair_marketplace_source_contract_adapter_dry_run_inputs" + ) + ), + } + + +def _write_dry_run_receipt(*, output_root: Path, item: Mapping[str, Any]) -> str: + target = ( + output_root + / _safe_segment(item.get("platform")) + / _safe_segment(item.get("manifest_id")) + / "marketplace_adapter_dry_run_receipt.json" + ) + target.parent.mkdir(parents=True, exist_ok=True) + receipt = dict(item) + receipt["artifact_write_performed"] = True + receipt["receipt_path"] = str(target) + receipt["generated_at"] = datetime.now(timezone.utc).isoformat() + receipt["policy"] = POLICY + target.write_text( + json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + return str(target) + + +def run_pixelrag_marketplace_source_contract_adapter_dry_run( + *, + adapter_preflight_receipt_root: str | Path | None = None, + output_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, + execute: bool = False, + write_receipt: bool = False, +) -> dict[str, Any]: + """Run or dry-run marketplace adapter deterministic mapping.""" + source_root = Path(adapter_preflight_receipt_root or DEFAULT_ADAPTER_PREFLIGHT_RECEIPT_ROOT) + output = Path(output_root or DEFAULT_OUTPUT_ROOT) + platforms = _normalise_platforms(platform) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + now = datetime.now(timezone.utc) + receipt_paths = _receipt_candidates(source_root, platforms=platforms, limit=item_limit) + worker_items: list[dict[str, Any]] = [] + for receipt_path in receipt_paths: + item = _worker_item( + receipt_path, + now=now, + max_age_hours=max_age, + execute=execute, + ) + if execute and write_receipt and item.get("ready_for_execution"): + item["receipt_path"] = _write_dry_run_receipt(output_root=output, item=item) + item["artifact_write_performed"] = True + worker_items.append(item) + + ready_count = sum(1 for item in worker_items if item.get("ready_for_execution")) + blocked_count = len(worker_items) - ready_count + dry_run_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_") + ) + executed_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_") + ) + receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path")) + guard_failed_count = sum( + 1 for item in worker_items if item.get("adapter_dry_run_status") != "ready" + ) + capture_runtime_gap_count = sum( + 1 + for item in worker_items + if (item.get("adapter_dry_run") or {}).get("capture_runtime_unavailable") + ) + field_contract_count = sum( + int((item.get("adapter_dry_run") or {}).get("field_contract_count") or 0) + for item in worker_items + ) + output_contract_count = sum( + int((item.get("adapter_dry_run") or {}).get("dry_run_output_contract_count") or 0) + for item in worker_items + ) + + if not worker_items: + status = "warning" + elif guard_failed_count: + status = "warning" + else: + status = "ok" + + if not worker_items: + next_action = "run_marketplace_source_contract_adapter_preflight" + elif not execute and ready_count: + next_action = "run_marketplace_source_contract_adapter_dry_run_execute" + elif execute and receipt_written_count: + next_action = "run_marketplace_adapter_identity_matcher_replay" + elif guard_failed_count: + next_action = "repair_marketplace_source_contract_adapter_dry_run_inputs" + else: + next_action = "refresh_marketplace_source_contract_adapter_dry_run_candidates" + + summary = { + "candidate_count": len(worker_items), + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "guard_failed_count": guard_failed_count, + "capture_runtime_gap_count": capture_runtime_gap_count, + "field_contract_count": field_contract_count, + "dry_run_output_contract_count": output_contract_count, + "platforms": sorted({str(item.get("platform") or "unknown") for item in worker_items}), + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": bool(receipt_written_count), + "writes_database_count": 0, + "primary_human_gate_count": 0, + } + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": now.isoformat(), + "adapter_dry_run_version": ADAPTER_DRY_RUN_VERSION, + "adapter_preflight_receipt_root": str(source_root), + "output_root": str(output), + "platform_filter": list(platforms), + "max_age_hours": max_age, + "limit": item_limit, + "execute": bool(execute), + "write_receipt": bool(write_receipt and execute), + "summary": summary, + "worker_items": worker_items, + "controlled_apply": { + "network_call": False, + "model_call": False, + "artifact_write": bool(receipt_written_count), + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "secret_read": False, + "raw_cookie_or_session_read": False, + "credentialed_session_allowed": False, + "login_allowed": False, + "production_price_write": False, + "primary_human_gate_count": 0, + }, + "promotion_boundary": { + "writes_ai_insights": False, + "writes_price_tables": False, + "adapter_dry_run_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "ADAPTER_DRY_RUN_VERSION", + "DEFAULT_OUTPUT_ROOT", + "POLICY", + "run_pixelrag_marketplace_source_contract_adapter_dry_run", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index a60cad0..d29dfc5 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1437,6 +1437,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_preflight_check", lambda: smoke._check("pixelrag marketplace adapter preflight", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_dry_run_check", lambda: smoke._check("pixelrag marketplace adapter dry-run", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -1446,7 +1447,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 38, "warning": 1, "critical": 1, "total": 40} + assert result["summary"] == {"ok": 39, "warning": 1, "critical": 1, "total": 41} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3978,6 +3979,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_preflight_check", lambda: smoke._check("pixelrag marketplace adapter preflight", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_dry_run_check", lambda: smoke._check("pixelrag marketplace adapter dry-run", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -4038,7 +4040,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 40, "warning": 0, "critical": 0, "total": 40}, + "summary": {"ok": 41, "warning": 0, "critical": 0, "total": 41}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4190,6 +4192,29 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "primary_human_gate_count": 0, }, }, + { + "name": "PixelRAG marketplace adapter dry-run", + "status": "ok", + "summary": "PixelRAG marketplace adapter dry-run candidates=1, ready=1, blocked=0, dry_run=1, executed=0, receipts=0, fields=14, outputs=2", + "details": { + "policy": "controlled_pixelrag_marketplace_source_contract_adapter_dry_run_v1", + "candidate_count": 1, + "ready_count": 1, + "blocked_count": 0, + "dry_run_count": 1, + "executed_count": 0, + "receipt_written_count": 0, + "capture_runtime_gap_count": 0, + "field_contract_count": 14, + "dry_run_output_contract_count": 2, + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": "run_marketplace_source_contract_adapter_dry_run_execute", + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "name": "PixelRAG OCR/VLM replay contract", "status": "ok", @@ -4304,7 +4329,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 37 + assert summary["summary"]["total"] == 38 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -6646,6 +6671,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag marketplace adapter preflight ok", )) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_dry_run_check", lambda: smoke._check( + "PixelRAG marketplace adapter dry-run", + "ok", + "pixelrag marketplace adapter dry-run ok", + )) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check( "PixelRAG OCR/VLM replay contract", "ok", @@ -6686,7 +6716,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 40 + assert result["summary"]["total"] == 41 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_marketplace_source_contract_adapter_dry_run_service.py b/tests/test_pixelrag_marketplace_source_contract_adapter_dry_run_service.py new file mode 100644 index 0000000..b9bdd62 --- /dev/null +++ b/tests/test_pixelrag_marketplace_source_contract_adapter_dry_run_service.py @@ -0,0 +1,259 @@ +import json +import subprocess +import sys +from datetime import datetime, timezone + + +def _write_adapter_preflight_receipt( + root, + *, + platform="coupang_tw", + manifest_id="coupang-403", + adapter_code="coupang", + barrier_type="access_denied", + capture_runtime_unavailable=False, +): + receipt_dir = root / platform / manifest_id + receipt_dir.mkdir(parents=True) + payload = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "policy": "controlled_pixelrag_marketplace_source_contract_adapter_preflight_v1", + "worker_status": "executed_adapter_preflight_ready", + "platform": platform, + "manifest_id": manifest_id, + "adapter_code": adapter_code, + "source_contract_replay_receipt_path": ( + f"runtime_artifacts/pixelrag_source_contract_replay_receipts/" + f"{platform}/{manifest_id}/source_contract_replay_receipt.json" + ), + "source_worker_receipt_path": ( + f"runtime_artifacts/pixelrag_platform_probe_worker_receipts/" + f"{platform}/{manifest_id}/platform_probe_worker_receipt.json" + ), + "source_receipt_path": ( + f"runtime_artifacts/pixelrag_visual_evidence/" + f"{platform}/{manifest_id}/capture_receipt.json" + ), + "adapter_preflight_check_count": 16, + "adapter_preflight_check_pass_count": 16, + "adapter_preflight_checks": { + "adapter_code_present": True, + "blocked_page_not_product_data": True, + "canonical_offer_fields_present": True, + "database_write_disallowed": True, + "direct_price_write_disallowed": True, + "identity_and_promotion_gates_required": True, + "network_request_deferred": True, + "provenance_contract_required": True, + "public_source_boundary_required": True, + "rate_limit_contract_required": True, + "receipt_fresh": True, + "receipt_parse_ok": True, + "source_contract_checks_all_passed": True, + "source_contract_replay_ready": True, + "source_contract_version_supported": True, + "source_count_positive": True, + }, + "adapter_preflight": { + "adapter_preflight_version": "pixelrag_marketplace_adapter_preflight_v1", + "source_contract_version": "pixelrag_marketplace_source_contract_v1", + "contract_id": f"{platform}:{manifest_id}:{adapter_code}", + "adapter_code": adapter_code, + "barrier_type": barrier_type, + "capture_runtime_unavailable": capture_runtime_unavailable, + "source_count": 2, + "expected_offer_fields": [ + "platform", + "platform_product_id", + "title", + "url", + "image_url", + "price", + "currency", + "availability", + "seller", + "promotion_badges", + "shipping_badges", + "evidence_time", + "source_method", + "confidence", + ], + "required_before_data_promotion": [ + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", + ], + }, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + }, + "writes_database": False, + "writes_database_count": 0, + "network_call_performed": False, + "model_call_performed": False, + } + path = receipt_dir / "marketplace_adapter_preflight_receipt.json" + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return path + + +def test_marketplace_adapter_dry_run_reads_preflight_receipts(tmp_path): + from services.pixelrag_marketplace_source_contract_adapter_dry_run_service import ( + POLICY, + run_pixelrag_marketplace_source_contract_adapter_dry_run, + ) + + preflight_root = tmp_path / "adapter_preflight" + _write_adapter_preflight_receipt(preflight_root) + _write_adapter_preflight_receipt( + preflight_root, + platform="shopee_tw", + manifest_id="shopee-runtime-gap", + adapter_code="shopee", + barrier_type="language_or_region_interstitial", + capture_runtime_unavailable=True, + ) + + payload = run_pixelrag_marketplace_source_contract_adapter_dry_run( + adapter_preflight_receipt_root=preflight_root, + platform=("coupang_tw", "shopee_tw"), + ) + by_platform = {item["platform"]: item for item in payload["worker_items"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "ok" + assert payload["summary"]["candidate_count"] == 2 + assert payload["summary"]["ready_count"] == 2 + assert payload["summary"]["dry_run_count"] == 2 + assert payload["summary"]["capture_runtime_gap_count"] == 1 + assert payload["summary"]["field_contract_count"] == 28 + assert payload["summary"]["dry_run_output_contract_count"] == 4 + assert payload["summary"]["writes_database_count"] == 0 + assert payload["controlled_apply"]["network_call"] is False + assert payload["controlled_apply"]["writes_database"] is False + assert payload["promotion_boundary"]["writes_price_tables"] is False + assert payload["next_machine_action"] == ( + "run_marketplace_source_contract_adapter_dry_run_execute" + ) + assert by_platform["coupang_tw"]["worker_status"] == ( + "dry_run_ready_for_marketplace_adapter_dry_run" + ) + assert by_platform["coupang_tw"]["adapter_dry_run_checks"][ + "price_write_blocked_until_promotion_gate" + ] is True + assert by_platform["coupang_tw"]["adapter_dry_run"]["offer_candidate_contracts"][0][ + "ready_for_price_table_write" + ] is False + assert by_platform["shopee_tw"]["adapter_dry_run"][ + "capture_runtime_unavailable" + ] is True + + +def test_marketplace_adapter_dry_run_execute_writes_receipt(tmp_path): + from services.pixelrag_marketplace_source_contract_adapter_dry_run_service import ( + run_pixelrag_marketplace_source_contract_adapter_dry_run, + ) + + preflight_root = tmp_path / "adapter_preflight" + output_root = tmp_path / "adapter_dry_run" + _write_adapter_preflight_receipt(preflight_root) + + payload = run_pixelrag_marketplace_source_contract_adapter_dry_run( + adapter_preflight_receipt_root=preflight_root, + output_root=output_root, + platform="coupang_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["executed_count"] == 1 + assert payload["summary"]["receipt_written_count"] == 1 + assert payload["controlled_apply"]["artifact_write"] is True + assert payload["controlled_apply"]["writes_database"] is False + assert payload["next_machine_action"] == "run_marketplace_adapter_identity_matcher_replay" + receipt_path = ( + output_root + / "coupang_tw" + / "coupang-403" + / "marketplace_adapter_dry_run_receipt.json" + ) + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "executed_marketplace_adapter_dry_run_ready" + assert receipt["artifact_write_performed"] is True + assert receipt["writes_database"] is False + assert receipt["adapter_dry_run_check_pass_count"] == ( + receipt["adapter_dry_run_check_count"] + ) + assert receipt["adapter_dry_run"]["adapter_dry_run_version"] == ( + "pixelrag_marketplace_adapter_dry_run_v1" + ) + assert receipt["adapter_dry_run"]["allowed_next_step"] == ( + "run_marketplace_adapter_identity_matcher_replay" + ) + + +def test_marketplace_adapter_dry_run_cli_outputs_json(tmp_path): + preflight_root = tmp_path / "adapter_preflight" + _write_adapter_preflight_receipt(preflight_root) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py", + "--adapter-preflight-receipt-root", + str(preflight_root), + "--platform", + "coupang_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["summary"]["field_contract_count"] == 14 + assert payload["controlled_apply"]["writes_database"] is False + + +def test_marketplace_adapter_dry_run_route_returns_readback( + tmp_path, + monkeypatch, +): + from flask import Flask + from routes import system_public_routes as routes + from services import pixelrag_marketplace_source_contract_adapter_dry_run_service as service + + preflight_root = tmp_path / "adapter_preflight" + _write_adapter_preflight_receipt(preflight_root) + monkeypatch.setattr( + service, + "DEFAULT_ADAPTER_PREFLIGHT_RECEIPT_ROOT", + str(preflight_root), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run?platform=coupang_tw" + ): + response = ( + routes + .ai_automation_pixelrag_marketplace_source_contract_adapter_dry_run_api + .__wrapped__() + ) + payload = response.get_json() + + assert payload["policy"] == ( + "controlled_pixelrag_marketplace_source_contract_adapter_dry_run_v1" + ) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False From e2c8baf7eaeaf885fdc52d0f0e812ac951eeb380 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 11:51:38 +0800 Subject: [PATCH 25/75] feat(ai): add PixelRAG marketplace identity matcher replay --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 4 +- .../ai_automation_mainline_work_items.md | 3 +- docs/guides/browse_sh_crawler_playbook.md | 4 + routes/system_public_routes.py | 37 ++ ...rag_marketplace_identity_matcher_replay.py | 77 +++ services/ai_automation_smoke_service.py | 136 +++++ ...etplace_identity_matcher_replay_service.py | 502 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 37 +- ...etplace_identity_matcher_replay_service.py | 289 ++++++++++ 10 files changed, 1084 insertions(+), 7 deletions(-) create mode 100755 scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py create mode 100644 services/pixelrag_marketplace_identity_matcher_replay_service.py create mode 100644 tests/test_pixelrag_marketplace_identity_matcher_replay_service.py diff --git a/config.py b/config.py index 9fced18..e370db4 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.766" +SYSTEM_VERSION = "V10.767" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 3fe2ef2..6d977c8 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.766 +> **適用版本**: V10.767 --- @@ -122,6 +122,7 @@ - 2026-07-10 起 PixelRAG platform probe 必須有 controlled worker：`/api/ai-automation/pixelrag-platform-probe-worker` 與 `scripts/ops/run_pixelrag_platform_probe_worker.py` 預設 dry-run，`execute=true&write_receipt=true` 才會對 Shopee language / traffic / generic landing 執行 public empty-context capture，並對 Coupang 403 / access denied 自動產生 structured-source/backoff artifact receipt；`/api/ai-automation/smoke` 需包含 `PixelRAG platform probe worker`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_platform_probe_worker` family。此 worker 不登入、不讀 cookie/session/secret、不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture 成功後下一步固定為 RAG candidate replay，structured fallback 只產生機器可讀 package，不把 blocked page 當商品資料。 - 2026-07-10 起 PixelRAG platform probe worker 必須自動處理 capture runtime gap：若 production app 容器缺 Playwright，`execute=true&write_receipt=true` 不得退回人工審核或 critical dead-end；worker 必須寫 `capture_runtime_unavailable_structured_fallback_package` artifact receipt，標記 `network_call=false`、`writes_database=false`、`primary_human_gate_count=0`，並把下一步改為 `run_structured_source_or_install_pixelrag_capture_runtime`。 - 2026-07-10 起 PixelRAG marketplace source-contract adapter dry-run 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run` 與 `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py` 預設只讀 adapter preflight receipts 並做 deterministic mapping readiness；`execute=true&write_receipt=true` 只寫 `marketplace_adapter_dry_run_receipt.json` artifact，輸出 field contracts、offer candidate contracts、identity matcher / PromotionGate / embedding signature gates 與 no-write 邊界。此 worker 不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；dry-run receipt 只允許進 identity matcher replay，不得直接進價格表或候選知識層。 +- 2026-07-10 起 PixelRAG marketplace identity matcher replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-identity-matcher-replay` 與 `scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py` 預設只讀 adapter dry-run receipts 並產生 deterministic identity key / match contract；`execute=true&write_receipt=true` 只寫 `marketplace_identity_matcher_replay_receipt.json` artifact，輸出 identity candidate contracts、source provenance、PromotionGate readiness 與 no-write 邊界。此 worker 不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；identity replay receipt 只允許進 PromotionGate replay，不得直接進價格表或候選知識層。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -900,6 +901,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG source-contract fallback 必須有 controlled replay worker | V10.764 起 `/api/ai-automation/pixelrag-source-contract-replay-worker`、`scripts/ops/run_pixelrag_source_contract_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG source-contract replay worker / `pixelrag_source_contract_replay_worker` family；worker 只讀 platform probe worker `source_contract_fallback` receipts，驗證 structured package、source count、no-DB、network-deferred、blocked-page-not-product-data、source receipt provenance，execute 模式只寫 `source_contract_replay_receipt.json` artifact。此路徑不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 marketplace source-contract adapter replay preflight。 | | 2026-07-10 | PixelRAG marketplace source-contract adapter 必須先有 no-write preflight | V10.765 起 `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight`、`scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace adapter preflight / `pixelrag_marketplace_adapter_preflight` family；preflight 只讀 `source_contract_replay_receipt.json`，驗證 replay ready、source contract version、adapter code、source count、source checks all passed、blocked page not product data、no-DB、network-deferred、public source boundary、rate-limit contract、provenance contract、identity/promotion gates 與 canonical offer fields。execute 模式只寫 `marketplace_adapter_preflight_receipt.json` artifact，不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 marketplace source-contract adapter dry-run。 | | 2026-07-10 | PixelRAG marketplace adapter dry-run 必須先產生 no-write mapping receipt | V10.766 起 `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run`、`scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace adapter dry-run / `pixelrag_marketplace_adapter_dry_run` family；dry-run 只讀 `marketplace_adapter_preflight_receipt.json`，驗證 preflight ready、canonical offer fields、required gates、no-DB/no-network/no-model、blocked page not product data 與 price-write blocked。execute 模式只寫 `marketplace_adapter_dry_run_receipt.json` artifact，產生 deterministic field contracts / offer candidate contracts，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 identity matcher replay。 | +| 2026-07-10 | PixelRAG marketplace identity matcher replay 必須先產生 no-write match receipt | V10.767 起 `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay`、`scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace identity matcher replay / `pixelrag_marketplace_identity_matcher_replay` family；replay 只讀 `marketplace_adapter_dry_run_receipt.json`，驗證 adapter dry-run ready、identity key fields、candidate contracts、required gates、no-DB/no-network/no-model、source provenance 與 price-write blocked。execute 模式只寫 `marketplace_identity_matcher_replay_receipt.json` artifact，產生 deterministic identity candidate contracts，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 PromotionGate replay。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index d18ce4c..f264af8 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -20,11 +20,12 @@ | Completed | PixelRAG source-contract replay worker | `/api/ai-automation/pixelrag-source-contract-replay-worker` and `scripts/ops/run_pixelrag_source_contract_replay_worker.py` turn platform probe worker `source_contract_fallback` receipts into replayable source-contract artifact receipts with provenance, public-boundary, no-DB, and blocked-page-not-product-data guards. | | Completed | PixelRAG marketplace source-contract adapter preflight | `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight` and `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py` validate source-contract replay receipts for adapter readiness, canonical offer fields, provenance/rate-limit/public-boundary requirements, and no-write promotion gates. | | Completed | PixelRAG marketplace adapter dry-run | `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run` and `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py` turn adapter preflight receipts into deterministic no-write field contracts and offer candidate contracts, with price writes blocked until identity matcher replay and PromotionGate. | +| Completed | PixelRAG marketplace identity matcher replay | `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay` and `scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py` turn adapter dry-run receipts into deterministic identity candidate contracts and source provenance checks, with price writes blocked until PromotionGate replay. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | | Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture when runtime is available, missing Playwright auto-falls back to structured-source/backoff receipt, and Coupang 403/access denied goes to structured-source/backoff artifact receipts. | -| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, PixelRAG platform probe worker, source-contract replay, adapter preflight, and adapter dry-run families. | +| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, PixelRAG platform probe worker, source-contract replay, adapter preflight, adapter dry-run, and identity matcher replay families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | ## P1 diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index 77b7d3b..1ced97c 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -202,6 +202,8 @@ python scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py python scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py --platform coupang_tw --execute --write-receipt --limit 1 python scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py --platform shopee_tw --platform coupang_tw python scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py --platform coupang_tw --execute --write-receipt --limit 1 +python scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py --platform coupang_tw --execute --write-receipt --limit 1 ``` API readback: `/api/ai-automation/pixelrag-vlm-route-readiness` 與 `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。worker 預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。execute 前會自動讀 approved Ollama `/api/tags` 選擇已安裝候選模型；若完全沒有候選，會寫 `model_route_not_ready` artifact receipt，不盲打 generate。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 @@ -214,6 +216,8 @@ Marketplace adapter preflight API readback: `/api/ai-automation/pixelrag-marketp Marketplace adapter dry-run API readback: `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run?platform=coupang_tw`。dry-run 預設只讀 adapter preflight receipts 並建立 deterministic field contracts；`execute=true&write_receipt=true` 只寫 adapter dry-run artifact receipt，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表。dry-run artifact 下一步只能進 identity matcher replay 與 PromotionGate，不得直接變成價格資料。 +Marketplace identity matcher replay API readback: `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay?platform=coupang_tw`。identity matcher replay 預設只讀 adapter dry-run receipts 並建立 deterministic identity candidate contracts；`execute=true&write_receipt=true` 只寫 identity matcher artifact receipt，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表。identity receipt 下一步只能進 PromotionGate replay，不得直接變成價格資料。 + Application portfolio： ```bash diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index db9b7a0..dd347ba 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -1016,6 +1016,43 @@ def ai_automation_pixelrag_marketplace_source_contract_adapter_dry_run_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-marketplace-identity-matcher-replay') +@login_required +def ai_automation_pixelrag_marketplace_identity_matcher_replay_api(): + """Controlled PixelRAG marketplace identity matcher replay readback.""" + from services.pixelrag_marketplace_identity_matcher_replay_service import ( + run_pixelrag_marketplace_identity_matcher_replay, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + execute = str(request.args.get('execute') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + write_receipt = str(request.args.get('write_receipt') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 25, type=int) + return jsonify(run_pixelrag_marketplace_identity_matcher_replay( + adapter_dry_run_receipt_root=str( + request.args.get('adapter_dry_run_receipt_root') or '' + ).strip() or None, + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 25, 250)), + execute=execute, + write_receipt=bool(execute and write_receipt), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py b/scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py new file mode 100755 index 0000000..d238045 --- /dev/null +++ b/scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Run or dry-run PixelRAG marketplace identity matcher replay.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_marketplace_identity_matcher_replay_service import ( # noqa: E402 + run_pixelrag_marketplace_identity_matcher_replay, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="執行或 dry-run PixelRAG marketplace identity matcher replay。" + ) + parser.add_argument( + "--adapter-dry-run-receipt-root", + help="Marketplace adapter dry-run receipt root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--output-root", + help="Marketplace identity matcher replay artifact receipt output root。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=25, + help="最多處理 adapter dry-run receipt 數。", + ) + parser.add_argument( + "--execute", + action="store_true", + help="執行 identity matcher replay；未指定時只做 no-write dry-run readiness。", + ) + parser.add_argument( + "--write-receipt", + action="store_true", + help="execute 後寫入 identity matcher replay artifact receipt；不寫 DB。", + ) + args = parser.parse_args() + + payload = run_pixelrag_marketplace_identity_matcher_replay( + adapter_dry_run_receipt_root=args.adapter_dry_run_receipt_root, + output_root=args.output_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + execute=args.execute, + write_receipt=bool(args.write_receipt and args.execute), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index e2294bc..6dfac7d 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -586,6 +586,23 @@ def build_scheduled_automation_health_summary( pixelrag_marketplace_adapter_dry_run_details = ( pixelrag_marketplace_adapter_dry_run.get("details") or {} ) + pixelrag_marketplace_identity_matcher = _find_check( + source_result, + "PixelRAG marketplace identity matcher replay", + ) + pixelrag_marketplace_identity_matcher_details = ( + pixelrag_marketplace_identity_matcher.get("details") or {} + ) + if ( + not pixelrag_marketplace_identity_matcher + or not pixelrag_marketplace_identity_matcher_details + ): + pixelrag_marketplace_identity_matcher = ( + _pixelrag_marketplace_identity_matcher_replay_check() + ) + pixelrag_marketplace_identity_matcher_details = ( + pixelrag_marketplace_identity_matcher.get("details") or {} + ) pixelrag_ocr_vlm_replay = _find_check(source_result, "PixelRAG OCR/VLM replay contract") pixelrag_ocr_vlm_replay_details = pixelrag_ocr_vlm_replay.get("details") or {} if not pixelrag_ocr_vlm_replay or not pixelrag_ocr_vlm_replay_details: @@ -4831,6 +4848,58 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_marketplace_identity_matcher_replay", + "label": "PixelRAG marketplace identity matcher replay", + "status": pixelrag_marketplace_identity_matcher.get("status") or "warning", + "summary": ( + pixelrag_marketplace_identity_matcher.get("summary") + or "PixelRAG marketplace identity matcher replay has no latest readback." + ), + "next_machine_action": pixelrag_marketplace_identity_matcher_details.get( + "next_machine_action" + ) + or "run_pixelrag_marketplace_identity_matcher_replay", + "details": { + "policy": pixelrag_marketplace_identity_matcher_details.get("policy"), + "candidate_count": int( + pixelrag_marketplace_identity_matcher_details.get("candidate_count") or 0 + ), + "ready_count": int( + pixelrag_marketplace_identity_matcher_details.get("ready_count") or 0 + ), + "blocked_count": int( + pixelrag_marketplace_identity_matcher_details.get("blocked_count") or 0 + ), + "dry_run_count": int( + pixelrag_marketplace_identity_matcher_details.get("dry_run_count") or 0 + ), + "executed_count": int( + pixelrag_marketplace_identity_matcher_details.get("executed_count") or 0 + ), + "receipt_written_count": int( + pixelrag_marketplace_identity_matcher_details.get("receipt_written_count") or 0 + ), + "capture_runtime_gap_count": int( + pixelrag_marketplace_identity_matcher_details.get("capture_runtime_gap_count") or 0 + ), + "identity_candidate_count": int( + pixelrag_marketplace_identity_matcher_details.get("identity_candidate_count") or 0 + ), + "network_call_performed": bool( + pixelrag_marketplace_identity_matcher_details.get("network_call_performed") + ), + "model_call_performed": bool( + pixelrag_marketplace_identity_matcher_details.get("model_call_performed") + ), + "artifact_write_performed": bool( + pixelrag_marketplace_identity_matcher_details.get("artifact_write_performed") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "key": "pixelrag_ocr_vlm_replay", "label": "PixelRAG OCR/VLM replay contract", @@ -13904,6 +13973,72 @@ def _pixelrag_marketplace_adapter_dry_run_check() -> Dict[str, Any]: ) +def _pixelrag_marketplace_identity_matcher_replay_check() -> Dict[str, Any]: + """Dry-run sentinel for PixelRAG marketplace identity matcher replay.""" + try: + from services.pixelrag_marketplace_identity_matcher_replay_service import ( + run_pixelrag_marketplace_identity_matcher_replay, + ) + + readback = run_pixelrag_marketplace_identity_matcher_replay( + execute=False, + ) + summary = readback.get("summary") or {} + candidate_count = int(summary.get("candidate_count") or 0) + ready_count = int(summary.get("ready_count") or 0) + blocked_count = int(summary.get("blocked_count") or 0) + dry_run_count = int(summary.get("dry_run_count") or 0) + executed_count = int(summary.get("executed_count") or 0) + receipt_written_count = int(summary.get("receipt_written_count") or 0) + identity_candidate_count = int(summary.get("identity_candidate_count") or 0) + status = readback.get("status") or "warning" + summary_text = ( + f"PixelRAG marketplace identity matcher replay candidates={candidate_count}, " + f"ready={ready_count}, blocked={blocked_count}, dry_run={dry_run_count}, " + f"executed={executed_count}, receipts={receipt_written_count}, " + f"identity_candidates={identity_candidate_count}" + ) + return _check( + "PixelRAG marketplace identity matcher replay", + status, + summary_text, + { + "policy": readback.get("policy"), + "candidate_count": candidate_count, + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "capture_runtime_gap_count": int( + summary.get("capture_runtime_gap_count") or 0 + ), + "guard_failed_count": int(summary.get("guard_failed_count") or 0), + "identity_candidate_count": identity_candidate_count, + "network_call_performed": bool(summary.get("network_call_performed")), + "model_call_performed": bool(summary.get("model_call_performed")), + "artifact_write_performed": bool( + summary.get("artifact_write_performed") + ), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG marketplace identity matcher replay", + "critical", + f"PixelRAG marketplace identity matcher replay 無法執行 dry-run：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def _pixelrag_ocr_vlm_replay_check() -> Dict[str, Any]: """Read-only sentinel for PixelRAG receipts becoming OCR/VLM replay contracts.""" try: @@ -14239,6 +14374,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _pixelrag_source_contract_replay_worker_check(), _pixelrag_marketplace_adapter_preflight_check(), _pixelrag_marketplace_adapter_dry_run_check(), + _pixelrag_marketplace_identity_matcher_replay_check(), _pixelrag_ocr_vlm_replay_check(), _pixelrag_vlm_route_readiness_check(), _pixelrag_vlm_replay_worker_check(), diff --git a/services/pixelrag_marketplace_identity_matcher_replay_service.py b/services/pixelrag_marketplace_identity_matcher_replay_service.py new file mode 100644 index 0000000..9b708cd --- /dev/null +++ b/services/pixelrag_marketplace_identity_matcher_replay_service.py @@ -0,0 +1,502 @@ +"""Controlled PixelRAG marketplace identity matcher replay worker. + +This worker turns marketplace adapter dry-run receipts into deterministic +identity matcher replay receipts. It does not fetch marketplace pages, call +models, write databases, write AI insights, or promote candidate prices. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.pixelrag_crawler_integration_service import DEFAULT_ARTIFACT_MAX_AGE_HOURS +from services.pixelrag_marketplace_source_contract_adapter_dry_run_service import ( + ADAPTER_DRY_RUN_VERSION, + DEFAULT_OUTPUT_ROOT as DEFAULT_ADAPTER_DRY_RUN_RECEIPT_ROOT, +) + + +POLICY = "controlled_pixelrag_marketplace_identity_matcher_replay_v1" +IDENTITY_MATCHER_REPLAY_VERSION = "pixelrag_marketplace_identity_matcher_replay_v1" +DEFAULT_LIMIT = 25 +DEFAULT_OUTPUT_ROOT = os.getenv( + "PIXELRAG_MARKETPLACE_IDENTITY_MATCHER_REPLAY_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_marketplace_identity_matcher_replay_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_marketplace_identity_matcher_replay_receipts", +) + +IDENTITY_KEY_FIELDS = ("platform", "platform_product_id", "title", "url") +REQUIRED_PROMOTION_GATES = { + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", +} + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...]: + if isinstance(platform, str): + value = platform.strip().lower() + return (value,) if value else () + return tuple( + str(item or "").strip().lower() + for item in (platform or ()) + if str(item or "").strip() + ) + + +def _safe_segment(value: Any) -> str: + text = str(value or "unknown").strip().lower() + text = re.sub(r"[^a-z0-9._-]+", "-", text) + return text.strip("-") or "unknown" + + +def _parse_iso_datetime(value: Any) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + + +def _receipt_candidates( + root: Path, + *, + platforms: tuple[str, ...], + limit: int, +) -> list[Path]: + if not root.exists(): + return [] + candidates: list[Path] = [] + if platforms: + for platform in platforms: + candidates.extend((root / platform).glob("*/marketplace_adapter_dry_run_receipt.json")) + else: + candidates.extend(root.glob("*/*/marketplace_adapter_dry_run_receipt.json")) + return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] + + +def _load_receipt(path: Path) -> tuple[dict[str, Any], list[str]]: + try: + return json.loads(path.read_text(encoding="utf-8")), [] + except (OSError, json.JSONDecodeError) as exc: + return {}, [str(exc)[:300]] + + +def _as_mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _as_list(value: Any) -> list[Any]: + return list(value) if isinstance(value, list) else [] + + +def _field_names(dry_run: Mapping[str, Any]) -> set[str]: + names: set[str] = set() + for item in _as_list(dry_run.get("field_contracts")): + if isinstance(item, Mapping) and item.get("field"): + names.add(str(item.get("field"))) + return names + + +def _required_before(dry_run: Mapping[str, Any]) -> set[str]: + return set(str(item) for item in _as_list(dry_run.get("required_before_data_promotion"))) + + +def _identity_fingerprint(platform: str, manifest_id: str, candidate_id: str) -> str: + digest = hashlib.sha256(f"{platform}:{manifest_id}:{candidate_id}".encode("utf-8")).hexdigest() + return digest[:24] + + +def _identity_candidate_contracts( + receipt: Mapping[str, Any], + *, + dry_run: Mapping[str, Any], +) -> list[dict[str, Any]]: + platform = str(receipt.get("platform") or "unknown") + manifest_id = str(receipt.get("manifest_id") or "unknown") + contracts: list[dict[str, Any]] = [] + for candidate in _as_list(dry_run.get("offer_candidate_contracts")): + if not isinstance(candidate, Mapping): + continue + candidate_id = str(candidate.get("candidate_id") or f"{platform}:{manifest_id}:unknown") + contracts.append( + { + "candidate_id": candidate_id, + "candidate_index": int(candidate.get("candidate_index") or 0), + "identity_fingerprint": _identity_fingerprint(platform, manifest_id, candidate_id), + "identity_key_fields": list(IDENTITY_KEY_FIELDS), + "match_strategy": "deterministic_contract_replay_no_values", + "requires_platform_scope": True, + "requires_title_url_cross_check": True, + "requires_product_id_or_url_identity": True, + "requires_evidence_time": True, + "field_contract_count": int(candidate.get("field_count") or 0), + "ready_for_promotion_gate_replay": True, + "ready_for_price_table_write": False, + "price_write_blocked_until_promotion_gate": True, + } + ) + return contracts + + +def _identity_matcher_payload( + receipt: Mapping[str, Any], + *, + receipt_path: Path, +) -> dict[str, Any]: + dry_run = _as_mapping(receipt.get("adapter_dry_run")) + identity_contracts = _identity_candidate_contracts(receipt, dry_run=dry_run) + return { + "identity_matcher_replay_version": IDENTITY_MATCHER_REPLAY_VERSION, + "adapter_dry_run_version": dry_run.get("adapter_dry_run_version"), + "adapter_preflight_version": dry_run.get("adapter_preflight_version"), + "source_contract_version": dry_run.get("source_contract_version"), + "adapter_code": dry_run.get("adapter_code") or receipt.get("adapter_code"), + "contract_id": dry_run.get("contract_id"), + "barrier_type": dry_run.get("barrier_type"), + "capture_runtime_unavailable": bool(dry_run.get("capture_runtime_unavailable")), + "input_adapter_dry_run_receipt_path": str(receipt_path), + "adapter_preflight_receipt_path": receipt.get("adapter_preflight_receipt_path"), + "source_contract_replay_receipt_path": receipt.get( + "source_contract_replay_receipt_path" + ), + "source_worker_receipt_path": receipt.get("source_worker_receipt_path"), + "source_receipt_path": receipt.get("source_receipt_path"), + "identity_match_execution_mode": "deterministic_replay_only_no_network_no_db", + "identity_key_fields": list(IDENTITY_KEY_FIELDS), + "identity_candidate_count": len(identity_contracts), + "identity_candidate_contracts": identity_contracts, + "required_before_data_promotion": list(dry_run.get("required_before_data_promotion") or []), + "allowed_next_step": "run_marketplace_promotion_gate_replay", + } + + +def _identity_matcher_checks( + receipt: Mapping[str, Any], + *, + stale: bool, + errors: list[str], + identity_payload: Mapping[str, Any], +) -> dict[str, bool]: + dry_run = _as_mapping(receipt.get("adapter_dry_run")) + boundary = _as_mapping(receipt.get("promotion_boundary")) + dry_run_checks = _as_mapping(receipt.get("adapter_dry_run_checks")) + field_names = _field_names(dry_run) + required_before = _required_before(dry_run) + identity_contracts = _as_list(identity_payload.get("identity_candidate_contracts")) + return { + "receipt_parse_ok": not errors, + "receipt_fresh": not stale, + "adapter_dry_run_ready": ( + receipt.get("worker_status") == "executed_marketplace_adapter_dry_run_ready" + ), + "adapter_dry_run_version_supported": ( + dry_run.get("adapter_dry_run_version") == ADAPTER_DRY_RUN_VERSION + ), + "adapter_dry_run_checks_all_passed": ( + int(receipt.get("adapter_dry_run_check_pass_count") or 0) + == int(receipt.get("adapter_dry_run_check_count") or -1) + and int(receipt.get("adapter_dry_run_check_count") or 0) > 0 + ), + "blocked_page_not_product_data": bool( + dry_run_checks.get("blocked_page_not_product_data") + ), + "identity_key_fields_present": set(IDENTITY_KEY_FIELDS).issubset(field_names), + "offer_candidate_contracts_present": bool( + _as_list(dry_run.get("offer_candidate_contracts")) + ), + "identity_candidate_contracts_generated": bool(identity_contracts), + "identity_candidate_count_matches_offer_count": ( + len(identity_contracts) + == len(_as_list(dry_run.get("offer_candidate_contracts"))) + ), + "promotion_gates_required": REQUIRED_PROMOTION_GATES.issubset(required_before), + "identity_gate_required": "identity_matcher_replay" in required_before, + "direct_price_write_disallowed": not bool( + boundary.get("direct_price_write_allowed") + ), + "direct_ai_insights_write_disallowed": not bool( + boundary.get("direct_ai_insights_write_allowed") + ), + "database_write_disallowed": not bool(receipt.get("writes_database")) + and int(receipt.get("writes_database_count") or 0) == 0, + "network_call_disallowed": not bool(receipt.get("network_call_performed")), + "model_call_disallowed": not bool(receipt.get("model_call_performed")), + "promotion_gate_replay_ready": all( + bool(item.get("ready_for_promotion_gate_replay")) + for item in identity_contracts + ), + "price_write_blocked_until_promotion_gate": all( + bool(item.get("price_write_blocked_until_promotion_gate")) + and not bool(item.get("ready_for_price_table_write")) + for item in identity_contracts + ), + "source_provenance_paths_present": bool( + identity_payload.get("input_adapter_dry_run_receipt_path") + and identity_payload.get("adapter_preflight_receipt_path") + and identity_payload.get("source_contract_replay_receipt_path") + ), + } + + +def _worker_item( + receipt_path: Path, + *, + now: datetime, + max_age_hours: int, + execute: bool, +) -> dict[str, Any]: + receipt, errors = _load_receipt(receipt_path) + generated_at = _parse_iso_datetime(receipt.get("generated_at")) + if generated_at is None: + try: + generated_at = datetime.fromtimestamp(receipt_path.stat().st_mtime, tz=timezone.utc) + except OSError: + generated_at = None + age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None + stale = age_hours is None or age_hours > max_age_hours + identity_payload = _identity_matcher_payload(receipt, receipt_path=receipt_path) + checks = _identity_matcher_checks( + receipt, + stale=stale, + errors=errors, + identity_payload=identity_payload, + ) + check_count = len(checks) + pass_count = sum(1 for passed in checks.values() if passed) + ready = pass_count == check_count + platform = str(receipt.get("platform") or receipt_path.parent.parent.name).strip().lower() + manifest_id = str(receipt.get("manifest_id") or receipt_path.parent.name).strip() + status = ( + "executed_marketplace_identity_matcher_replay_ready" + if execute and ready + else ( + "dry_run_ready_for_marketplace_identity_matcher_replay" + if ready + else "skipped_marketplace_identity_matcher_replay_guard_failed" + ) + ) + return { + "worker_status": status, + "platform": platform, + "manifest_id": manifest_id, + "source_type": "marketplace_adapter_dry_run_receipt", + "adapter_dry_run_receipt_path": str(receipt_path), + "adapter_preflight_receipt_path": receipt.get("adapter_preflight_receipt_path"), + "source_contract_replay_receipt_path": receipt.get( + "source_contract_replay_receipt_path" + ), + "source_worker_receipt_path": receipt.get("source_worker_receipt_path"), + "source_receipt_path": receipt.get("source_receipt_path"), + "adapter_code": identity_payload.get("adapter_code"), + "identity_matcher_replay_status": "ready" if ready else "blocked", + "ready_for_execution": ready, + "execute": bool(execute), + "generated_at": generated_at.isoformat() if generated_at else None, + "age_hours": round(age_hours, 3) if age_hours is not None else None, + "stale": stale, + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": False, + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + "identity_matcher_replay_checks": checks, + "identity_matcher_replay_check_count": check_count, + "identity_matcher_replay_check_pass_count": pass_count, + "identity_matcher_replay": identity_payload, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "identity_matcher_replay_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": ( + "run_marketplace_promotion_gate_replay" + if execute and ready + else ( + "run_marketplace_adapter_identity_matcher_replay_execute" + if ready + else "repair_marketplace_identity_matcher_replay_inputs" + ) + ), + } + + +def _write_identity_matcher_receipt(*, output_root: Path, item: Mapping[str, Any]) -> str: + target = ( + output_root + / _safe_segment(item.get("platform")) + / _safe_segment(item.get("manifest_id")) + / "marketplace_identity_matcher_replay_receipt.json" + ) + target.parent.mkdir(parents=True, exist_ok=True) + receipt = dict(item) + receipt["artifact_write_performed"] = True + receipt["receipt_path"] = str(target) + receipt["generated_at"] = datetime.now(timezone.utc).isoformat() + receipt["policy"] = POLICY + target.write_text( + json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + return str(target) + + +def run_pixelrag_marketplace_identity_matcher_replay( + *, + adapter_dry_run_receipt_root: str | Path | None = None, + output_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, + execute: bool = False, + write_receipt: bool = False, +) -> dict[str, Any]: + """Run or dry-run marketplace identity matcher replay.""" + source_root = Path(adapter_dry_run_receipt_root or DEFAULT_ADAPTER_DRY_RUN_RECEIPT_ROOT) + output = Path(output_root or DEFAULT_OUTPUT_ROOT) + platforms = _normalise_platforms(platform) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + now = datetime.now(timezone.utc) + receipt_paths = _receipt_candidates(source_root, platforms=platforms, limit=item_limit) + worker_items: list[dict[str, Any]] = [] + for receipt_path in receipt_paths: + item = _worker_item( + receipt_path, + now=now, + max_age_hours=max_age, + execute=execute, + ) + if execute and write_receipt and item.get("ready_for_execution"): + item["receipt_path"] = _write_identity_matcher_receipt( + output_root=output, + item=item, + ) + item["artifact_write_performed"] = True + worker_items.append(item) + + ready_count = sum(1 for item in worker_items if item.get("ready_for_execution")) + blocked_count = len(worker_items) - ready_count + dry_run_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_") + ) + executed_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_") + ) + receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path")) + guard_failed_count = sum( + 1 for item in worker_items if item.get("identity_matcher_replay_status") != "ready" + ) + capture_runtime_gap_count = sum( + 1 + for item in worker_items + if (item.get("identity_matcher_replay") or {}).get("capture_runtime_unavailable") + ) + identity_candidate_count = sum( + int((item.get("identity_matcher_replay") or {}).get("identity_candidate_count") or 0) + for item in worker_items + ) + + if not worker_items: + status = "warning" + elif guard_failed_count: + status = "warning" + else: + status = "ok" + + if not worker_items: + next_action = "run_marketplace_source_contract_adapter_dry_run" + elif not execute and ready_count: + next_action = "run_marketplace_adapter_identity_matcher_replay_execute" + elif execute and receipt_written_count: + next_action = "run_marketplace_promotion_gate_replay" + elif guard_failed_count: + next_action = "repair_marketplace_identity_matcher_replay_inputs" + else: + next_action = "refresh_marketplace_identity_matcher_replay_candidates" + + summary = { + "candidate_count": len(worker_items), + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "guard_failed_count": guard_failed_count, + "capture_runtime_gap_count": capture_runtime_gap_count, + "identity_candidate_count": identity_candidate_count, + "platforms": sorted({str(item.get("platform") or "unknown") for item in worker_items}), + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": bool(receipt_written_count), + "writes_database_count": 0, + "primary_human_gate_count": 0, + } + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": now.isoformat(), + "identity_matcher_replay_version": IDENTITY_MATCHER_REPLAY_VERSION, + "adapter_dry_run_receipt_root": str(source_root), + "output_root": str(output), + "platform_filter": list(platforms), + "max_age_hours": max_age, + "limit": item_limit, + "execute": bool(execute), + "write_receipt": bool(write_receipt and execute), + "summary": summary, + "worker_items": worker_items, + "controlled_apply": { + "network_call": False, + "model_call": False, + "artifact_write": bool(receipt_written_count), + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "secret_read": False, + "raw_cookie_or_session_read": False, + "credentialed_session_allowed": False, + "login_allowed": False, + "production_price_write": False, + "primary_human_gate_count": 0, + }, + "promotion_boundary": { + "writes_ai_insights": False, + "writes_price_tables": False, + "identity_matcher_replay_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "DEFAULT_OUTPUT_ROOT", + "IDENTITY_MATCHER_REPLAY_VERSION", + "POLICY", + "run_pixelrag_marketplace_identity_matcher_replay", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index d29dfc5..ec28adf 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1438,6 +1438,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_preflight_check", lambda: smoke._check("pixelrag marketplace adapter preflight", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_dry_run_check", lambda: smoke._check("pixelrag marketplace adapter dry-run", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_identity_matcher_replay_check", lambda: smoke._check("pixelrag marketplace identity matcher replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -1447,7 +1448,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 39, "warning": 1, "critical": 1, "total": 41} + assert result["summary"] == {"ok": 40, "warning": 1, "critical": 1, "total": 42} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3980,6 +3981,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_preflight_check", lambda: smoke._check("pixelrag marketplace adapter preflight", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_dry_run_check", lambda: smoke._check("pixelrag marketplace adapter dry-run", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_identity_matcher_replay_check", lambda: smoke._check("pixelrag marketplace identity matcher replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -4040,7 +4042,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 41, "warning": 0, "critical": 0, "total": 41}, + "summary": {"ok": 42, "warning": 0, "critical": 0, "total": 42}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4215,6 +4217,28 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "primary_human_gate_count": 0, }, }, + { + "name": "PixelRAG marketplace identity matcher replay", + "status": "ok", + "summary": "PixelRAG marketplace identity matcher replay candidates=1, ready=1, blocked=0, dry_run=1, executed=0, receipts=0, identity_candidates=2", + "details": { + "policy": "controlled_pixelrag_marketplace_identity_matcher_replay_v1", + "candidate_count": 1, + "ready_count": 1, + "blocked_count": 0, + "dry_run_count": 1, + "executed_count": 0, + "receipt_written_count": 0, + "capture_runtime_gap_count": 0, + "identity_candidate_count": 2, + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": "run_marketplace_adapter_identity_matcher_replay_execute", + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "name": "PixelRAG OCR/VLM replay contract", "status": "ok", @@ -4329,7 +4353,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 38 + assert summary["summary"]["total"] == 39 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -6676,6 +6700,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag marketplace adapter dry-run ok", )) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_identity_matcher_replay_check", lambda: smoke._check( + "PixelRAG marketplace identity matcher replay", + "ok", + "pixelrag marketplace identity matcher replay ok", + )) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check( "PixelRAG OCR/VLM replay contract", "ok", @@ -6716,7 +6745,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 41 + assert result["summary"]["total"] == 42 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_marketplace_identity_matcher_replay_service.py b/tests/test_pixelrag_marketplace_identity_matcher_replay_service.py new file mode 100644 index 0000000..fba617c --- /dev/null +++ b/tests/test_pixelrag_marketplace_identity_matcher_replay_service.py @@ -0,0 +1,289 @@ +import json +import subprocess +import sys +from datetime import datetime, timezone + + +def _write_adapter_dry_run_receipt( + root, + *, + platform="coupang_tw", + manifest_id="coupang-403", + adapter_code="coupang", + barrier_type="access_denied", + capture_runtime_unavailable=False, +): + receipt_dir = root / platform / manifest_id + receipt_dir.mkdir(parents=True) + field_contracts = [ + {"field": "platform", "mapped": True}, + {"field": "platform_product_id", "mapped": True, "requires_identity_match": True}, + {"field": "title", "mapped": True, "requires_identity_match": True}, + {"field": "url", "mapped": True, "requires_identity_match": True}, + {"field": "image_url", "mapped": True}, + {"field": "price", "mapped": True, "requires_promotion_gate": True}, + {"field": "currency", "mapped": True}, + {"field": "availability", "mapped": True}, + {"field": "seller", "mapped": True}, + {"field": "promotion_badges", "mapped": True, "requires_promotion_gate": True}, + {"field": "shipping_badges", "mapped": True, "requires_promotion_gate": True}, + {"field": "evidence_time", "mapped": True}, + {"field": "source_method", "mapped": True}, + {"field": "confidence", "mapped": True, "requires_promotion_gate": True}, + ] + payload = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "policy": "controlled_pixelrag_marketplace_source_contract_adapter_dry_run_v1", + "worker_status": "executed_marketplace_adapter_dry_run_ready", + "platform": platform, + "manifest_id": manifest_id, + "adapter_code": adapter_code, + "adapter_dry_run_receipt_path": ( + f"runtime_artifacts/pixelrag_marketplace_adapter_dry_run_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_dry_run_receipt.json" + ), + "adapter_preflight_receipt_path": ( + f"runtime_artifacts/pixelrag_marketplace_adapter_preflight_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_preflight_receipt.json" + ), + "source_contract_replay_receipt_path": ( + f"runtime_artifacts/pixelrag_source_contract_replay_receipts/" + f"{platform}/{manifest_id}/source_contract_replay_receipt.json" + ), + "source_worker_receipt_path": ( + f"runtime_artifacts/pixelrag_platform_probe_worker_receipts/" + f"{platform}/{manifest_id}/platform_probe_worker_receipt.json" + ), + "source_receipt_path": ( + f"runtime_artifacts/pixelrag_visual_evidence/" + f"{platform}/{manifest_id}/capture_receipt.json" + ), + "adapter_dry_run_check_count": 19, + "adapter_dry_run_check_pass_count": 19, + "adapter_dry_run_checks": { + "adapter_code_present": True, + "adapter_preflight_checks_all_passed": True, + "adapter_preflight_ready": True, + "adapter_preflight_version_supported": True, + "blocked_page_not_product_data": True, + "canonical_offer_fields_present": True, + "database_write_disallowed": True, + "direct_ai_insights_write_disallowed": True, + "direct_price_write_disallowed": True, + "dry_run_output_contract_generated": True, + "dry_run_schema_generated": True, + "model_call_disallowed": True, + "network_call_disallowed": True, + "price_write_blocked_until_promotion_gate": True, + "promotion_gates_required": True, + "receipt_fresh": True, + "receipt_parse_ok": True, + "source_contract_version_supported": True, + "source_count_positive": True, + }, + "adapter_dry_run": { + "adapter_dry_run_version": "pixelrag_marketplace_adapter_dry_run_v1", + "adapter_preflight_version": "pixelrag_marketplace_adapter_preflight_v1", + "source_contract_version": "pixelrag_marketplace_source_contract_v1", + "contract_id": f"{platform}:{manifest_id}:{adapter_code}", + "adapter_code": adapter_code, + "barrier_type": barrier_type, + "capture_runtime_unavailable": capture_runtime_unavailable, + "dry_run_execution_mode": "deterministic_mapping_only_no_network_no_db", + "field_contract_count": 14, + "field_contracts": field_contracts, + "offer_candidate_contracts": [ + { + "candidate_index": 0, + "candidate_id": f"{platform}:{manifest_id}:0", + "field_count": 14, + "ready_for_identity_matcher_replay": True, + "ready_for_price_table_write": False, + "price_write_blocked_until_promotion_gate": True, + }, + { + "candidate_index": 1, + "candidate_id": f"{platform}:{manifest_id}:1", + "field_count": 14, + "ready_for_identity_matcher_replay": True, + "ready_for_price_table_write": False, + "price_write_blocked_until_promotion_gate": True, + }, + ], + "required_before_data_promotion": [ + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", + ], + }, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + }, + "writes_database": False, + "writes_database_count": 0, + "network_call_performed": False, + "model_call_performed": False, + } + path = receipt_dir / "marketplace_adapter_dry_run_receipt.json" + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return path + + +def test_marketplace_identity_matcher_replay_reads_dry_run_receipts(tmp_path): + from services.pixelrag_marketplace_identity_matcher_replay_service import ( + POLICY, + run_pixelrag_marketplace_identity_matcher_replay, + ) + + dry_run_root = tmp_path / "adapter_dry_run" + _write_adapter_dry_run_receipt(dry_run_root) + _write_adapter_dry_run_receipt( + dry_run_root, + platform="shopee_tw", + manifest_id="shopee-runtime-gap", + adapter_code="shopee", + barrier_type="language_or_region_interstitial", + capture_runtime_unavailable=True, + ) + + payload = run_pixelrag_marketplace_identity_matcher_replay( + adapter_dry_run_receipt_root=dry_run_root, + platform=("coupang_tw", "shopee_tw"), + ) + by_platform = {item["platform"]: item for item in payload["worker_items"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "ok" + assert payload["summary"]["candidate_count"] == 2 + assert payload["summary"]["ready_count"] == 2 + assert payload["summary"]["dry_run_count"] == 2 + assert payload["summary"]["identity_candidate_count"] == 4 + assert payload["summary"]["capture_runtime_gap_count"] == 1 + assert payload["summary"]["writes_database_count"] == 0 + assert payload["controlled_apply"]["network_call"] is False + assert payload["controlled_apply"]["writes_database"] is False + assert payload["promotion_boundary"]["writes_price_tables"] is False + assert payload["next_machine_action"] == ( + "run_marketplace_adapter_identity_matcher_replay_execute" + ) + assert by_platform["coupang_tw"]["worker_status"] == ( + "dry_run_ready_for_marketplace_identity_matcher_replay" + ) + first_identity = by_platform["coupang_tw"]["identity_matcher_replay"][ + "identity_candidate_contracts" + ][0] + assert first_identity["ready_for_promotion_gate_replay"] is True + assert first_identity["ready_for_price_table_write"] is False + assert first_identity["price_write_blocked_until_promotion_gate"] is True + assert by_platform["shopee_tw"]["identity_matcher_replay"][ + "capture_runtime_unavailable" + ] is True + + +def test_marketplace_identity_matcher_replay_execute_writes_receipt(tmp_path): + from services.pixelrag_marketplace_identity_matcher_replay_service import ( + run_pixelrag_marketplace_identity_matcher_replay, + ) + + dry_run_root = tmp_path / "adapter_dry_run" + output_root = tmp_path / "identity_matcher" + _write_adapter_dry_run_receipt(dry_run_root) + + payload = run_pixelrag_marketplace_identity_matcher_replay( + adapter_dry_run_receipt_root=dry_run_root, + output_root=output_root, + platform="coupang_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["executed_count"] == 1 + assert payload["summary"]["receipt_written_count"] == 1 + assert payload["controlled_apply"]["artifact_write"] is True + assert payload["controlled_apply"]["writes_database"] is False + assert payload["next_machine_action"] == "run_marketplace_promotion_gate_replay" + receipt_path = ( + output_root + / "coupang_tw" + / "coupang-403" + / "marketplace_identity_matcher_replay_receipt.json" + ) + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == ( + "executed_marketplace_identity_matcher_replay_ready" + ) + assert receipt["artifact_write_performed"] is True + assert receipt["writes_database"] is False + assert receipt["identity_matcher_replay_check_pass_count"] == ( + receipt["identity_matcher_replay_check_count"] + ) + assert receipt["identity_matcher_replay"]["allowed_next_step"] == ( + "run_marketplace_promotion_gate_replay" + ) + + +def test_marketplace_identity_matcher_replay_cli_outputs_json(tmp_path): + dry_run_root = tmp_path / "adapter_dry_run" + _write_adapter_dry_run_receipt(dry_run_root) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py", + "--adapter-dry-run-receipt-root", + str(dry_run_root), + "--platform", + "coupang_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["summary"]["identity_candidate_count"] == 2 + assert payload["controlled_apply"]["writes_database"] is False + + +def test_marketplace_identity_matcher_replay_route_returns_readback( + tmp_path, + monkeypatch, +): + from flask import Flask + from routes import system_public_routes as routes + from services import pixelrag_marketplace_identity_matcher_replay_service as service + + dry_run_root = tmp_path / "adapter_dry_run" + _write_adapter_dry_run_receipt(dry_run_root) + monkeypatch.setattr( + service, + "DEFAULT_ADAPTER_DRY_RUN_RECEIPT_ROOT", + str(dry_run_root), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-marketplace-identity-matcher-replay?platform=coupang_tw" + ): + response = ( + routes + .ai_automation_pixelrag_marketplace_identity_matcher_replay_api + .__wrapped__() + ) + payload = response.get_json() + + assert payload["policy"] == ( + "controlled_pixelrag_marketplace_identity_matcher_replay_v1" + ) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False From d0b14577cbbc58d8ec2a079d5d855ea4ce7c7fa4 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 12:11:46 +0800 Subject: [PATCH 26/75] feat(ai): add PixelRAG marketplace PromotionGate replay --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 4 +- .../ai_automation_mainline_work_items.md | 3 +- docs/guides/browse_sh_crawler_playbook.md | 4 + routes/system_public_routes.py | 37 ++ ...elrag_marketplace_promotion_gate_replay.py | 77 +++ services/ai_automation_smoke_service.py | 144 +++++ ...rketplace_promotion_gate_replay_service.py | 555 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 38 +- ...rketplace_promotion_gate_replay_service.py | 305 ++++++++++ 10 files changed, 1162 insertions(+), 7 deletions(-) create mode 100755 scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py create mode 100644 services/pixelrag_marketplace_promotion_gate_replay_service.py create mode 100644 tests/test_pixelrag_marketplace_promotion_gate_replay_service.py diff --git a/config.py b/config.py index e370db4..1446b6a 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.767" +SYSTEM_VERSION = "V10.768" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 6d977c8..b37f60c 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.767 +> **適用版本**: V10.768 --- @@ -123,6 +123,7 @@ - 2026-07-10 起 PixelRAG platform probe worker 必須自動處理 capture runtime gap：若 production app 容器缺 Playwright，`execute=true&write_receipt=true` 不得退回人工審核或 critical dead-end；worker 必須寫 `capture_runtime_unavailable_structured_fallback_package` artifact receipt，標記 `network_call=false`、`writes_database=false`、`primary_human_gate_count=0`，並把下一步改為 `run_structured_source_or_install_pixelrag_capture_runtime`。 - 2026-07-10 起 PixelRAG marketplace source-contract adapter dry-run 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run` 與 `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py` 預設只讀 adapter preflight receipts 並做 deterministic mapping readiness；`execute=true&write_receipt=true` 只寫 `marketplace_adapter_dry_run_receipt.json` artifact，輸出 field contracts、offer candidate contracts、identity matcher / PromotionGate / embedding signature gates 與 no-write 邊界。此 worker 不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；dry-run receipt 只允許進 identity matcher replay，不得直接進價格表或候選知識層。 - 2026-07-10 起 PixelRAG marketplace identity matcher replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-identity-matcher-replay` 與 `scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py` 預設只讀 adapter dry-run receipts 並產生 deterministic identity key / match contract；`execute=true&write_receipt=true` 只寫 `marketplace_identity_matcher_replay_receipt.json` artifact，輸出 identity candidate contracts、source provenance、PromotionGate readiness 與 no-write 邊界。此 worker 不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；identity replay receipt 只允許進 PromotionGate replay，不得直接進價格表或候選知識層。 +- 2026-07-10 起 PixelRAG marketplace PromotionGate replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-promotion-gate-replay` 與 `scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py` 預設只讀 identity matcher receipts 並產生 deterministic PromotionGate contract；`execute=true&write_receipt=true` 只寫 `marketplace_promotion_gate_replay_receipt.json` artifact，輸出 promotion candidate contracts、embedding signature guard readiness、source provenance 與 no-write 邊界。此 worker 不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；PromotionGate receipt 只允許進 embedding signature guard replay，不得直接進價格表或候選知識層。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -902,6 +903,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG marketplace source-contract adapter 必須先有 no-write preflight | V10.765 起 `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight`、`scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace adapter preflight / `pixelrag_marketplace_adapter_preflight` family；preflight 只讀 `source_contract_replay_receipt.json`，驗證 replay ready、source contract version、adapter code、source count、source checks all passed、blocked page not product data、no-DB、network-deferred、public source boundary、rate-limit contract、provenance contract、identity/promotion gates 與 canonical offer fields。execute 模式只寫 `marketplace_adapter_preflight_receipt.json` artifact，不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 marketplace source-contract adapter dry-run。 | | 2026-07-10 | PixelRAG marketplace adapter dry-run 必須先產生 no-write mapping receipt | V10.766 起 `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run`、`scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace adapter dry-run / `pixelrag_marketplace_adapter_dry_run` family；dry-run 只讀 `marketplace_adapter_preflight_receipt.json`，驗證 preflight ready、canonical offer fields、required gates、no-DB/no-network/no-model、blocked page not product data 與 price-write blocked。execute 模式只寫 `marketplace_adapter_dry_run_receipt.json` artifact，產生 deterministic field contracts / offer candidate contracts，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 identity matcher replay。 | | 2026-07-10 | PixelRAG marketplace identity matcher replay 必須先產生 no-write match receipt | V10.767 起 `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay`、`scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace identity matcher replay / `pixelrag_marketplace_identity_matcher_replay` family；replay 只讀 `marketplace_adapter_dry_run_receipt.json`，驗證 adapter dry-run ready、identity key fields、candidate contracts、required gates、no-DB/no-network/no-model、source provenance 與 price-write blocked。execute 模式只寫 `marketplace_identity_matcher_replay_receipt.json` artifact，產生 deterministic identity candidate contracts，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 PromotionGate replay。 | +| 2026-07-10 | PixelRAG marketplace PromotionGate replay 必須先產生 no-write promotion receipt | V10.768 起 `/api/ai-automation/pixelrag-marketplace-promotion-gate-replay`、`scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace PromotionGate replay / `pixelrag_marketplace_promotion_gate_replay` family；replay 只讀 `marketplace_identity_matcher_replay_receipt.json`，驗證 identity matcher ready、identity checks all passed、required promotion gates、embedding signature guard readiness、no-DB/no-network/no-model、source provenance 與 price-write blocked。execute 模式只寫 `marketplace_promotion_gate_replay_receipt.json` artifact，產生 deterministic promotion candidate contracts，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 embedding signature guard replay。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index f264af8..4997b24 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -21,11 +21,12 @@ | Completed | PixelRAG marketplace source-contract adapter preflight | `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight` and `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py` validate source-contract replay receipts for adapter readiness, canonical offer fields, provenance/rate-limit/public-boundary requirements, and no-write promotion gates. | | Completed | PixelRAG marketplace adapter dry-run | `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run` and `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py` turn adapter preflight receipts into deterministic no-write field contracts and offer candidate contracts, with price writes blocked until identity matcher replay and PromotionGate. | | Completed | PixelRAG marketplace identity matcher replay | `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay` and `scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py` turn adapter dry-run receipts into deterministic identity candidate contracts and source provenance checks, with price writes blocked until PromotionGate replay. | +| Completed | PixelRAG marketplace PromotionGate replay | `/api/ai-automation/pixelrag-marketplace-promotion-gate-replay` and `scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py` turn identity matcher receipts into deterministic PromotionGate contracts and embedding-signature readiness, with price writes still blocked. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | | Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture when runtime is available, missing Playwright auto-falls back to structured-source/backoff receipt, and Coupang 403/access denied goes to structured-source/backoff artifact receipts. | -| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, PixelRAG platform probe worker, source-contract replay, adapter preflight, adapter dry-run, and identity matcher replay families. | +| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, PixelRAG platform probe worker, source-contract replay, adapter preflight, adapter dry-run, identity matcher replay, and PromotionGate replay families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | ## P1 diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index 1ced97c..cbd1434 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -204,6 +204,8 @@ python scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py - python scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py --platform coupang_tw --execute --write-receipt --limit 1 python scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py --platform shopee_tw --platform coupang_tw python scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py --platform coupang_tw --execute --write-receipt --limit 1 +python scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py --platform coupang_tw --execute --write-receipt --limit 1 ``` API readback: `/api/ai-automation/pixelrag-vlm-route-readiness` 與 `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。worker 預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。execute 前會自動讀 approved Ollama `/api/tags` 選擇已安裝候選模型；若完全沒有候選，會寫 `model_route_not_ready` artifact receipt，不盲打 generate。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 @@ -218,6 +220,8 @@ Marketplace adapter dry-run API readback: `/api/ai-automation/pixelrag-marketpla Marketplace identity matcher replay API readback: `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay?platform=coupang_tw`。identity matcher replay 預設只讀 adapter dry-run receipts 並建立 deterministic identity candidate contracts；`execute=true&write_receipt=true` 只寫 identity matcher artifact receipt，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表。identity receipt 下一步只能進 PromotionGate replay，不得直接變成價格資料。 +Marketplace PromotionGate replay API readback: `/api/ai-automation/pixelrag-marketplace-promotion-gate-replay?platform=coupang_tw`。PromotionGate replay 預設只讀 identity matcher receipts 並建立 deterministic promotion candidate contracts；`execute=true&write_receipt=true` 只寫 PromotionGate artifact receipt，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表。PromotionGate receipt 下一步只能進 embedding signature guard replay，不得直接變成價格資料。 + Application portfolio： ```bash diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index dd347ba..8af7f23 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -1053,6 +1053,43 @@ def ai_automation_pixelrag_marketplace_identity_matcher_replay_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-marketplace-promotion-gate-replay') +@login_required +def ai_automation_pixelrag_marketplace_promotion_gate_replay_api(): + """Controlled PixelRAG marketplace PromotionGate replay readback.""" + from services.pixelrag_marketplace_promotion_gate_replay_service import ( + run_pixelrag_marketplace_promotion_gate_replay, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + execute = str(request.args.get('execute') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + write_receipt = str(request.args.get('write_receipt') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 25, type=int) + return jsonify(run_pixelrag_marketplace_promotion_gate_replay( + identity_matcher_receipt_root=str( + request.args.get('identity_matcher_receipt_root') or '' + ).strip() or None, + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 25, 250)), + execute=execute, + write_receipt=bool(execute and write_receipt), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py b/scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py new file mode 100755 index 0000000..6300150 --- /dev/null +++ b/scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Run or dry-run PixelRAG marketplace PromotionGate replay.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_marketplace_promotion_gate_replay_service import ( # noqa: E402 + run_pixelrag_marketplace_promotion_gate_replay, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="執行或 dry-run PixelRAG marketplace PromotionGate replay。" + ) + parser.add_argument( + "--identity-matcher-receipt-root", + help="Marketplace identity matcher replay receipt root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--output-root", + help="Marketplace PromotionGate replay artifact receipt output root。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=25, + help="最多處理 identity matcher replay receipt 數。", + ) + parser.add_argument( + "--execute", + action="store_true", + help="執行 PromotionGate replay；未指定時只做 no-write dry-run readiness。", + ) + parser.add_argument( + "--write-receipt", + action="store_true", + help="execute 後寫入 PromotionGate replay artifact receipt；不寫 DB。", + ) + args = parser.parse_args() + + payload = run_pixelrag_marketplace_promotion_gate_replay( + identity_matcher_receipt_root=args.identity_matcher_receipt_root, + output_root=args.output_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + execute=args.execute, + write_receipt=bool(args.write_receipt and args.execute), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index 6dfac7d..e3f25cc 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -603,6 +603,23 @@ def build_scheduled_automation_health_summary( pixelrag_marketplace_identity_matcher_details = ( pixelrag_marketplace_identity_matcher.get("details") or {} ) + pixelrag_marketplace_promotion_gate = _find_check( + source_result, + "PixelRAG marketplace PromotionGate replay", + ) + pixelrag_marketplace_promotion_gate_details = ( + pixelrag_marketplace_promotion_gate.get("details") or {} + ) + if ( + not pixelrag_marketplace_promotion_gate + or not pixelrag_marketplace_promotion_gate_details + ): + pixelrag_marketplace_promotion_gate = ( + _pixelrag_marketplace_promotion_gate_replay_check() + ) + pixelrag_marketplace_promotion_gate_details = ( + pixelrag_marketplace_promotion_gate.get("details") or {} + ) pixelrag_ocr_vlm_replay = _find_check(source_result, "PixelRAG OCR/VLM replay contract") pixelrag_ocr_vlm_replay_details = pixelrag_ocr_vlm_replay.get("details") or {} if not pixelrag_ocr_vlm_replay or not pixelrag_ocr_vlm_replay_details: @@ -4900,6 +4917,61 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_marketplace_promotion_gate_replay", + "label": "PixelRAG marketplace PromotionGate replay", + "status": pixelrag_marketplace_promotion_gate.get("status") or "warning", + "summary": ( + pixelrag_marketplace_promotion_gate.get("summary") + or "PixelRAG marketplace PromotionGate replay has no latest readback." + ), + "next_machine_action": pixelrag_marketplace_promotion_gate_details.get( + "next_machine_action" + ) + or "run_marketplace_promotion_gate_replay", + "details": { + "policy": pixelrag_marketplace_promotion_gate_details.get("policy"), + "candidate_count": int( + pixelrag_marketplace_promotion_gate_details.get("candidate_count") or 0 + ), + "ready_count": int( + pixelrag_marketplace_promotion_gate_details.get("ready_count") or 0 + ), + "blocked_count": int( + pixelrag_marketplace_promotion_gate_details.get("blocked_count") or 0 + ), + "dry_run_count": int( + pixelrag_marketplace_promotion_gate_details.get("dry_run_count") or 0 + ), + "executed_count": int( + pixelrag_marketplace_promotion_gate_details.get("executed_count") or 0 + ), + "receipt_written_count": int( + pixelrag_marketplace_promotion_gate_details.get("receipt_written_count") or 0 + ), + "capture_runtime_gap_count": int( + pixelrag_marketplace_promotion_gate_details.get("capture_runtime_gap_count") or 0 + ), + "promotion_candidate_count": int( + pixelrag_marketplace_promotion_gate_details.get("promotion_candidate_count") or 0 + ), + "embedding_signature_guard_ready_count": int( + pixelrag_marketplace_promotion_gate_details.get("embedding_signature_guard_ready_count") or 0 + ), + "network_call_performed": bool( + pixelrag_marketplace_promotion_gate_details.get("network_call_performed") + ), + "model_call_performed": bool( + pixelrag_marketplace_promotion_gate_details.get("model_call_performed") + ), + "artifact_write_performed": bool( + pixelrag_marketplace_promotion_gate_details.get("artifact_write_performed") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "key": "pixelrag_ocr_vlm_replay", "label": "PixelRAG OCR/VLM replay contract", @@ -14039,6 +14111,77 @@ def _pixelrag_marketplace_identity_matcher_replay_check() -> Dict[str, Any]: ) +def _pixelrag_marketplace_promotion_gate_replay_check() -> Dict[str, Any]: + """Dry-run sentinel for PixelRAG marketplace PromotionGate replay.""" + try: + from services.pixelrag_marketplace_promotion_gate_replay_service import ( + run_pixelrag_marketplace_promotion_gate_replay, + ) + + readback = run_pixelrag_marketplace_promotion_gate_replay( + execute=False, + ) + summary = readback.get("summary") or {} + candidate_count = int(summary.get("candidate_count") or 0) + ready_count = int(summary.get("ready_count") or 0) + blocked_count = int(summary.get("blocked_count") or 0) + dry_run_count = int(summary.get("dry_run_count") or 0) + executed_count = int(summary.get("executed_count") or 0) + receipt_written_count = int(summary.get("receipt_written_count") or 0) + promotion_candidate_count = int(summary.get("promotion_candidate_count") or 0) + embedding_ready_count = int( + summary.get("embedding_signature_guard_ready_count") or 0 + ) + status = readback.get("status") or "warning" + summary_text = ( + f"PixelRAG marketplace PromotionGate replay candidates={candidate_count}, " + f"ready={ready_count}, blocked={blocked_count}, dry_run={dry_run_count}, " + f"executed={executed_count}, receipts={receipt_written_count}, " + f"promotion_candidates={promotion_candidate_count}, " + f"embedding_ready={embedding_ready_count}" + ) + return _check( + "PixelRAG marketplace PromotionGate replay", + status, + summary_text, + { + "policy": readback.get("policy"), + "candidate_count": candidate_count, + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "capture_runtime_gap_count": int( + summary.get("capture_runtime_gap_count") or 0 + ), + "guard_failed_count": int(summary.get("guard_failed_count") or 0), + "promotion_candidate_count": promotion_candidate_count, + "embedding_signature_guard_ready_count": embedding_ready_count, + "network_call_performed": bool(summary.get("network_call_performed")), + "model_call_performed": bool(summary.get("model_call_performed")), + "artifact_write_performed": bool( + summary.get("artifact_write_performed") + ), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG marketplace PromotionGate replay", + "critical", + f"PixelRAG marketplace PromotionGate replay 無法執行 dry-run：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def _pixelrag_ocr_vlm_replay_check() -> Dict[str, Any]: """Read-only sentinel for PixelRAG receipts becoming OCR/VLM replay contracts.""" try: @@ -14375,6 +14518,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _pixelrag_marketplace_adapter_preflight_check(), _pixelrag_marketplace_adapter_dry_run_check(), _pixelrag_marketplace_identity_matcher_replay_check(), + _pixelrag_marketplace_promotion_gate_replay_check(), _pixelrag_ocr_vlm_replay_check(), _pixelrag_vlm_route_readiness_check(), _pixelrag_vlm_replay_worker_check(), diff --git a/services/pixelrag_marketplace_promotion_gate_replay_service.py b/services/pixelrag_marketplace_promotion_gate_replay_service.py new file mode 100644 index 0000000..b87978b --- /dev/null +++ b/services/pixelrag_marketplace_promotion_gate_replay_service.py @@ -0,0 +1,555 @@ +"""Controlled PixelRAG marketplace PromotionGate replay worker. + +This worker turns marketplace identity matcher receipts into deterministic +PromotionGate replay receipts. It does not fetch marketplace pages, call +models, write databases, write AI insights, or promote candidate prices. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.pixelrag_crawler_integration_service import DEFAULT_ARTIFACT_MAX_AGE_HOURS +from services.pixelrag_marketplace_identity_matcher_replay_service import ( + DEFAULT_OUTPUT_ROOT as DEFAULT_IDENTITY_MATCHER_RECEIPT_ROOT, + IDENTITY_MATCHER_REPLAY_VERSION, +) + + +POLICY = "controlled_pixelrag_marketplace_promotion_gate_replay_v1" +PROMOTION_GATE_REPLAY_VERSION = "pixelrag_marketplace_promotion_gate_replay_v1" +DEFAULT_LIMIT = 25 +DEFAULT_OUTPUT_ROOT = os.getenv( + "PIXELRAG_MARKETPLACE_PROMOTION_GATE_REPLAY_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_marketplace_promotion_gate_replay_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_marketplace_promotion_gate_replay_receipts", +) + +REQUIRED_PROMOTION_GATES = { + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", +} + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...]: + if isinstance(platform, str): + value = platform.strip().lower() + return (value,) if value else () + return tuple( + str(item or "").strip().lower() + for item in (platform or ()) + if str(item or "").strip() + ) + + +def _safe_segment(value: Any) -> str: + text = str(value or "unknown").strip().lower() + text = re.sub(r"[^a-z0-9._-]+", "-", text) + return text.strip("-") or "unknown" + + +def _parse_iso_datetime(value: Any) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + + +def _receipt_candidates( + root: Path, + *, + platforms: tuple[str, ...], + limit: int, +) -> list[Path]: + if not root.exists(): + return [] + candidates: list[Path] = [] + if platforms: + for platform in platforms: + candidates.extend( + (root / platform).glob("*/marketplace_identity_matcher_replay_receipt.json") + ) + else: + candidates.extend(root.glob("*/*/marketplace_identity_matcher_replay_receipt.json")) + return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] + + +def _load_receipt(path: Path) -> tuple[dict[str, Any], list[str]]: + try: + return json.loads(path.read_text(encoding="utf-8")), [] + except (OSError, json.JSONDecodeError) as exc: + return {}, [str(exc)[:300]] + + +def _as_mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _as_list(value: Any) -> list[Any]: + return list(value) if isinstance(value, list) else [] + + +def _required_before(identity_payload: Mapping[str, Any]) -> set[str]: + return set(str(item) for item in _as_list(identity_payload.get("required_before_data_promotion"))) + + +def _promotion_fingerprint( + platform: str, + manifest_id: str, + candidate_id: str, + identity_fingerprint: str, +) -> str: + digest = hashlib.sha256( + f"{platform}:{manifest_id}:{candidate_id}:{identity_fingerprint}".encode("utf-8") + ).hexdigest() + return digest[:24] + + +def _promotion_candidate_contracts( + receipt: Mapping[str, Any], + *, + identity_payload: Mapping[str, Any], +) -> list[dict[str, Any]]: + platform = str(receipt.get("platform") or "unknown") + manifest_id = str(receipt.get("manifest_id") or "unknown") + contracts: list[dict[str, Any]] = [] + for candidate in _as_list(identity_payload.get("identity_candidate_contracts")): + if not isinstance(candidate, Mapping): + continue + candidate_id = str(candidate.get("candidate_id") or f"{platform}:{manifest_id}:unknown") + identity_fingerprint = str(candidate.get("identity_fingerprint") or "missing") + contracts.append( + { + "candidate_id": candidate_id, + "candidate_index": int(candidate.get("candidate_index") or 0), + "identity_fingerprint": identity_fingerprint, + "promotion_fingerprint": _promotion_fingerprint( + platform, + manifest_id, + candidate_id, + identity_fingerprint, + ), + "promotion_strategy": "deterministic_gate_replay_no_values", + "promotion_stage": "pre_embedding_signature_guard", + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_source_provenance": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + "requires_evidence_time": True, + "ready_for_embedding_signature_guard_replay": True, + "ready_for_candidate_knowledge_replay": True, + "ready_for_price_table_write": False, + "price_write_blocked_until_embedding_signature_guard": True, + } + ) + return contracts + + +def _promotion_gate_payload( + receipt: Mapping[str, Any], + *, + receipt_path: Path, +) -> dict[str, Any]: + identity_payload = _as_mapping(receipt.get("identity_matcher_replay")) + promotion_contracts = _promotion_candidate_contracts( + receipt, + identity_payload=identity_payload, + ) + return { + "promotion_gate_replay_version": PROMOTION_GATE_REPLAY_VERSION, + "identity_matcher_replay_version": identity_payload.get( + "identity_matcher_replay_version" + ), + "adapter_dry_run_version": identity_payload.get("adapter_dry_run_version"), + "adapter_preflight_version": identity_payload.get("adapter_preflight_version"), + "source_contract_version": identity_payload.get("source_contract_version"), + "adapter_code": identity_payload.get("adapter_code") or receipt.get("adapter_code"), + "contract_id": identity_payload.get("contract_id"), + "barrier_type": identity_payload.get("barrier_type"), + "capture_runtime_unavailable": bool( + identity_payload.get("capture_runtime_unavailable") + ), + "input_identity_matcher_receipt_path": str(receipt_path), + "input_adapter_dry_run_receipt_path": identity_payload.get( + "input_adapter_dry_run_receipt_path" + ), + "adapter_preflight_receipt_path": identity_payload.get( + "adapter_preflight_receipt_path" + ), + "source_contract_replay_receipt_path": identity_payload.get( + "source_contract_replay_receipt_path" + ), + "source_worker_receipt_path": identity_payload.get("source_worker_receipt_path"), + "source_receipt_path": identity_payload.get("source_receipt_path"), + "promotion_gate_execution_mode": "deterministic_replay_only_no_network_no_db", + "promotion_candidate_count": len(promotion_contracts), + "promotion_candidate_contracts": promotion_contracts, + "required_before_data_promotion": list( + identity_payload.get("required_before_data_promotion") or [] + ), + "allowed_next_step": "run_marketplace_embedding_signature_guard_replay", + } + + +def _promotion_gate_checks( + receipt: Mapping[str, Any], + *, + stale: bool, + errors: list[str], + promotion_payload: Mapping[str, Any], +) -> dict[str, bool]: + identity_payload = _as_mapping(receipt.get("identity_matcher_replay")) + boundary = _as_mapping(receipt.get("promotion_boundary")) + identity_checks = _as_mapping(receipt.get("identity_matcher_replay_checks")) + identity_contracts = _as_list(identity_payload.get("identity_candidate_contracts")) + promotion_contracts = _as_list(promotion_payload.get("promotion_candidate_contracts")) + required_before = _required_before(identity_payload) + return { + "receipt_parse_ok": not errors, + "receipt_fresh": not stale, + "identity_matcher_replay_ready": ( + receipt.get("worker_status") + == "executed_marketplace_identity_matcher_replay_ready" + ), + "identity_matcher_version_supported": ( + identity_payload.get("identity_matcher_replay_version") + == IDENTITY_MATCHER_REPLAY_VERSION + ), + "identity_matcher_checks_all_passed": ( + int(receipt.get("identity_matcher_replay_check_pass_count") or 0) + == int(receipt.get("identity_matcher_replay_check_count") or -1) + and int(receipt.get("identity_matcher_replay_check_count") or 0) > 0 + ), + "blocked_page_not_product_data": bool( + identity_checks.get("blocked_page_not_product_data") + ), + "identity_candidate_contracts_present": bool(identity_contracts), + "promotion_candidate_contracts_generated": bool(promotion_contracts), + "promotion_candidate_count_matches_identity_count": ( + len(promotion_contracts) == len(identity_contracts) + ), + "identity_candidates_ready_for_promotion": all( + bool(item.get("ready_for_promotion_gate_replay")) + for item in identity_contracts + ), + "required_promotion_gates_present": REQUIRED_PROMOTION_GATES.issubset( + required_before + ), + "promotion_gate_required": "promotion_gate" in required_before, + "embedding_signature_guard_required": "embedding_signature_guard" in required_before, + "embedding_signature_guard_replay_ready": all( + bool(item.get("ready_for_embedding_signature_guard_replay")) + for item in promotion_contracts + ), + "candidate_knowledge_replay_ready": all( + bool(item.get("ready_for_candidate_knowledge_replay")) + for item in promotion_contracts + ), + "price_write_blocked_until_embedding_signature_guard": all( + bool(item.get("price_write_blocked_until_embedding_signature_guard")) + and not bool(item.get("ready_for_price_table_write")) + for item in promotion_contracts + ), + "direct_price_write_disallowed": not bool( + boundary.get("direct_price_write_allowed") + ), + "direct_ai_insights_write_disallowed": not bool( + boundary.get("direct_ai_insights_write_allowed") + ), + "database_write_disallowed": not bool(receipt.get("writes_database")) + and int(receipt.get("writes_database_count") or 0) == 0, + "network_call_disallowed": not bool(receipt.get("network_call_performed")), + "model_call_disallowed": not bool(receipt.get("model_call_performed")), + "source_provenance_paths_present": bool( + promotion_payload.get("input_identity_matcher_receipt_path") + and promotion_payload.get("input_adapter_dry_run_receipt_path") + and promotion_payload.get("adapter_preflight_receipt_path") + and promotion_payload.get("source_contract_replay_receipt_path") + ), + } + + +def _worker_item( + receipt_path: Path, + *, + now: datetime, + max_age_hours: int, + execute: bool, +) -> dict[str, Any]: + receipt, errors = _load_receipt(receipt_path) + generated_at = _parse_iso_datetime(receipt.get("generated_at")) + if generated_at is None: + try: + generated_at = datetime.fromtimestamp(receipt_path.stat().st_mtime, tz=timezone.utc) + except OSError: + generated_at = None + age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None + stale = age_hours is None or age_hours > max_age_hours + promotion_payload = _promotion_gate_payload(receipt, receipt_path=receipt_path) + checks = _promotion_gate_checks( + receipt, + stale=stale, + errors=errors, + promotion_payload=promotion_payload, + ) + check_count = len(checks) + pass_count = sum(1 for passed in checks.values() if passed) + ready = pass_count == check_count + platform = str(receipt.get("platform") or receipt_path.parent.parent.name).strip().lower() + manifest_id = str(receipt.get("manifest_id") or receipt_path.parent.name).strip() + status = ( + "executed_marketplace_promotion_gate_replay_ready" + if execute and ready + else ( + "dry_run_ready_for_marketplace_promotion_gate_replay" + if ready + else "skipped_marketplace_promotion_gate_replay_guard_failed" + ) + ) + return { + "worker_status": status, + "platform": platform, + "manifest_id": manifest_id, + "source_type": "marketplace_identity_matcher_replay_receipt", + "identity_matcher_receipt_path": str(receipt_path), + "adapter_dry_run_receipt_path": promotion_payload.get( + "input_adapter_dry_run_receipt_path" + ), + "adapter_preflight_receipt_path": promotion_payload.get( + "adapter_preflight_receipt_path" + ), + "source_contract_replay_receipt_path": promotion_payload.get( + "source_contract_replay_receipt_path" + ), + "source_worker_receipt_path": promotion_payload.get("source_worker_receipt_path"), + "source_receipt_path": promotion_payload.get("source_receipt_path"), + "adapter_code": promotion_payload.get("adapter_code"), + "promotion_gate_replay_status": "ready" if ready else "blocked", + "ready_for_execution": ready, + "execute": bool(execute), + "generated_at": generated_at.isoformat() if generated_at else None, + "age_hours": round(age_hours, 3) if age_hours is not None else None, + "stale": stale, + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": False, + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + "promotion_gate_replay_checks": checks, + "promotion_gate_replay_check_count": check_count, + "promotion_gate_replay_check_pass_count": pass_count, + "promotion_gate_replay": promotion_payload, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "promotion_gate_replay_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": ( + "run_marketplace_embedding_signature_guard_replay" + if execute and ready + else ( + "run_marketplace_promotion_gate_replay_execute" + if ready + else "repair_marketplace_promotion_gate_replay_inputs" + ) + ), + } + + +def _write_promotion_gate_receipt(*, output_root: Path, item: Mapping[str, Any]) -> str: + target = ( + output_root + / _safe_segment(item.get("platform")) + / _safe_segment(item.get("manifest_id")) + / "marketplace_promotion_gate_replay_receipt.json" + ) + target.parent.mkdir(parents=True, exist_ok=True) + receipt = dict(item) + receipt["artifact_write_performed"] = True + receipt["receipt_path"] = str(target) + receipt["generated_at"] = datetime.now(timezone.utc).isoformat() + receipt["policy"] = POLICY + target.write_text( + json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + return str(target) + + +def run_pixelrag_marketplace_promotion_gate_replay( + *, + identity_matcher_receipt_root: str | Path | None = None, + output_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, + execute: bool = False, + write_receipt: bool = False, +) -> dict[str, Any]: + """Run or dry-run marketplace PromotionGate replay.""" + source_root = Path(identity_matcher_receipt_root or DEFAULT_IDENTITY_MATCHER_RECEIPT_ROOT) + output = Path(output_root or DEFAULT_OUTPUT_ROOT) + platforms = _normalise_platforms(platform) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + now = datetime.now(timezone.utc) + receipt_paths = _receipt_candidates(source_root, platforms=platforms, limit=item_limit) + worker_items: list[dict[str, Any]] = [] + for receipt_path in receipt_paths: + item = _worker_item( + receipt_path, + now=now, + max_age_hours=max_age, + execute=execute, + ) + if execute and write_receipt and item.get("ready_for_execution"): + item["receipt_path"] = _write_promotion_gate_receipt( + output_root=output, + item=item, + ) + item["artifact_write_performed"] = True + worker_items.append(item) + + ready_count = sum(1 for item in worker_items if item.get("ready_for_execution")) + blocked_count = len(worker_items) - ready_count + dry_run_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_") + ) + executed_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_") + ) + receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path")) + guard_failed_count = sum( + 1 for item in worker_items if item.get("promotion_gate_replay_status") != "ready" + ) + capture_runtime_gap_count = sum( + 1 + for item in worker_items + if (item.get("promotion_gate_replay") or {}).get("capture_runtime_unavailable") + ) + promotion_candidate_count = sum( + int((item.get("promotion_gate_replay") or {}).get("promotion_candidate_count") or 0) + for item in worker_items + ) + embedding_signature_guard_ready_count = sum( + sum( + 1 + for candidate in _as_list( + (item.get("promotion_gate_replay") or {}).get( + "promotion_candidate_contracts" + ) + ) + if isinstance(candidate, Mapping) + and candidate.get("ready_for_embedding_signature_guard_replay") + ) + for item in worker_items + ) + + if not worker_items: + status = "warning" + elif guard_failed_count: + status = "warning" + else: + status = "ok" + + if not worker_items: + next_action = "run_marketplace_adapter_identity_matcher_replay" + elif not execute and ready_count: + next_action = "run_marketplace_promotion_gate_replay_execute" + elif execute and receipt_written_count: + next_action = "run_marketplace_embedding_signature_guard_replay" + elif guard_failed_count: + next_action = "repair_marketplace_promotion_gate_replay_inputs" + else: + next_action = "refresh_marketplace_promotion_gate_replay_candidates" + + summary = { + "candidate_count": len(worker_items), + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "guard_failed_count": guard_failed_count, + "capture_runtime_gap_count": capture_runtime_gap_count, + "promotion_candidate_count": promotion_candidate_count, + "embedding_signature_guard_ready_count": embedding_signature_guard_ready_count, + "platforms": sorted({str(item.get("platform") or "unknown") for item in worker_items}), + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": bool(receipt_written_count), + "writes_database_count": 0, + "primary_human_gate_count": 0, + } + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": now.isoformat(), + "promotion_gate_replay_version": PROMOTION_GATE_REPLAY_VERSION, + "identity_matcher_receipt_root": str(source_root), + "output_root": str(output), + "platform_filter": list(platforms), + "max_age_hours": max_age, + "limit": item_limit, + "execute": bool(execute), + "write_receipt": bool(write_receipt and execute), + "summary": summary, + "worker_items": worker_items, + "controlled_apply": { + "network_call": False, + "model_call": False, + "artifact_write": bool(receipt_written_count), + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "secret_read": False, + "raw_cookie_or_session_read": False, + "credentialed_session_allowed": False, + "login_allowed": False, + "production_price_write": False, + "primary_human_gate_count": 0, + }, + "promotion_boundary": { + "writes_ai_insights": False, + "writes_price_tables": False, + "promotion_gate_replay_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_embedding_signature_guard": True, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "DEFAULT_OUTPUT_ROOT", + "POLICY", + "PROMOTION_GATE_REPLAY_VERSION", + "run_pixelrag_marketplace_promotion_gate_replay", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index ec28adf..19b0820 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1439,6 +1439,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_preflight_check", lambda: smoke._check("pixelrag marketplace adapter preflight", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_dry_run_check", lambda: smoke._check("pixelrag marketplace adapter dry-run", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_identity_matcher_replay_check", lambda: smoke._check("pixelrag marketplace identity matcher replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_promotion_gate_replay_check", lambda: smoke._check("pixelrag marketplace promotion gate replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -1448,7 +1449,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 40, "warning": 1, "critical": 1, "total": 42} + assert result["summary"] == {"ok": 41, "warning": 1, "critical": 1, "total": 43} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3982,6 +3983,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_preflight_check", lambda: smoke._check("pixelrag marketplace adapter preflight", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_dry_run_check", lambda: smoke._check("pixelrag marketplace adapter dry-run", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_identity_matcher_replay_check", lambda: smoke._check("pixelrag marketplace identity matcher replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_promotion_gate_replay_check", lambda: smoke._check("pixelrag marketplace promotion gate replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -4042,7 +4044,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 42, "warning": 0, "critical": 0, "total": 42}, + "summary": {"ok": 43, "warning": 0, "critical": 0, "total": 43}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4239,6 +4241,29 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "primary_human_gate_count": 0, }, }, + { + "name": "PixelRAG marketplace PromotionGate replay", + "status": "ok", + "summary": "PixelRAG marketplace PromotionGate replay candidates=1, ready=1, blocked=0, dry_run=1, executed=0, receipts=0, promotion_candidates=2, embedding_ready=2", + "details": { + "policy": "controlled_pixelrag_marketplace_promotion_gate_replay_v1", + "candidate_count": 1, + "ready_count": 1, + "blocked_count": 0, + "dry_run_count": 1, + "executed_count": 0, + "receipt_written_count": 0, + "capture_runtime_gap_count": 0, + "promotion_candidate_count": 2, + "embedding_signature_guard_ready_count": 2, + "network_call_performed": False, + "model_call_performed": False, + "artifact_write_performed": False, + "next_machine_action": "run_marketplace_promotion_gate_replay_execute", + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "name": "PixelRAG OCR/VLM replay contract", "status": "ok", @@ -4353,7 +4378,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 39 + assert summary["summary"]["total"] == 40 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -6705,6 +6730,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag marketplace identity matcher replay ok", )) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_promotion_gate_replay_check", lambda: smoke._check( + "PixelRAG marketplace PromotionGate replay", + "ok", + "pixelrag marketplace PromotionGate replay ok", + )) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check( "PixelRAG OCR/VLM replay contract", "ok", @@ -6745,7 +6775,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 42 + assert result["summary"]["total"] == 43 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_marketplace_promotion_gate_replay_service.py b/tests/test_pixelrag_marketplace_promotion_gate_replay_service.py new file mode 100644 index 0000000..f69e954 --- /dev/null +++ b/tests/test_pixelrag_marketplace_promotion_gate_replay_service.py @@ -0,0 +1,305 @@ +import json +import subprocess +import sys +from datetime import datetime, timezone + + +def _identity_candidate(platform, manifest_id, index): + return { + "candidate_id": f"{platform}:{manifest_id}:{index}", + "candidate_index": index, + "field_contract_count": 14, + "identity_fingerprint": f"identity-{platform}-{index}", + "identity_key_fields": ["platform", "platform_product_id", "title", "url"], + "match_strategy": "deterministic_contract_replay_no_values", + "price_write_blocked_until_promotion_gate": True, + "ready_for_price_table_write": False, + "ready_for_promotion_gate_replay": True, + "requires_evidence_time": True, + "requires_platform_scope": True, + "requires_product_id_or_url_identity": True, + "requires_title_url_cross_check": True, + } + + +def _write_identity_matcher_receipt( + root, + *, + platform="coupang_tw", + manifest_id="coupang-403", + adapter_code="coupang", + barrier_type="access_denied", + capture_runtime_unavailable=False, +): + receipt_dir = root / platform / manifest_id + receipt_dir.mkdir(parents=True) + source_root = "runtime_artifacts" + payload = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "policy": "controlled_pixelrag_marketplace_identity_matcher_replay_v1", + "worker_status": "executed_marketplace_identity_matcher_replay_ready", + "platform": platform, + "manifest_id": manifest_id, + "adapter_code": adapter_code, + "identity_matcher_receipt_path": ( + f"{source_root}/pixelrag_marketplace_identity_matcher_replay_receipts/" + f"{platform}/{manifest_id}/marketplace_identity_matcher_replay_receipt.json" + ), + "adapter_dry_run_receipt_path": ( + f"{source_root}/pixelrag_marketplace_adapter_dry_run_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_dry_run_receipt.json" + ), + "adapter_preflight_receipt_path": ( + f"{source_root}/pixelrag_marketplace_adapter_preflight_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_preflight_receipt.json" + ), + "source_contract_replay_receipt_path": ( + f"{source_root}/pixelrag_source_contract_replay_receipts/" + f"{platform}/{manifest_id}/source_contract_replay_receipt.json" + ), + "source_worker_receipt_path": ( + f"{source_root}/pixelrag_platform_probe_worker_receipts/" + f"{platform}/{manifest_id}/platform_probe_worker_receipt.json" + ), + "source_receipt_path": ( + f"{source_root}/pixelrag_visual_evidence/" + f"{platform}/{manifest_id}/capture_receipt.json" + ), + "identity_matcher_replay_check_count": 20, + "identity_matcher_replay_check_pass_count": 20, + "identity_matcher_replay_checks": { + "adapter_dry_run_checks_all_passed": True, + "adapter_dry_run_ready": True, + "adapter_dry_run_version_supported": True, + "blocked_page_not_product_data": True, + "database_write_disallowed": True, + "direct_ai_insights_write_disallowed": True, + "direct_price_write_disallowed": True, + "identity_candidate_contracts_generated": True, + "identity_candidate_count_matches_offer_count": True, + "identity_gate_required": True, + "identity_key_fields_present": True, + "model_call_disallowed": True, + "network_call_disallowed": True, + "offer_candidate_contracts_present": True, + "price_write_blocked_until_promotion_gate": True, + "promotion_gate_replay_ready": True, + "promotion_gates_required": True, + "receipt_fresh": True, + "receipt_parse_ok": True, + "source_provenance_paths_present": True, + }, + "identity_matcher_replay": { + "identity_matcher_replay_version": "pixelrag_marketplace_identity_matcher_replay_v1", + "adapter_dry_run_version": "pixelrag_marketplace_adapter_dry_run_v1", + "adapter_preflight_version": "pixelrag_marketplace_adapter_preflight_v1", + "source_contract_version": "pixelrag_marketplace_source_contract_v1", + "contract_id": f"{platform}:{manifest_id}:{adapter_code}", + "adapter_code": adapter_code, + "barrier_type": barrier_type, + "capture_runtime_unavailable": capture_runtime_unavailable, + "input_adapter_dry_run_receipt_path": ( + f"{source_root}/pixelrag_marketplace_adapter_dry_run_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_dry_run_receipt.json" + ), + "adapter_preflight_receipt_path": ( + f"{source_root}/pixelrag_marketplace_adapter_preflight_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_preflight_receipt.json" + ), + "source_contract_replay_receipt_path": ( + f"{source_root}/pixelrag_source_contract_replay_receipts/" + f"{platform}/{manifest_id}/source_contract_replay_receipt.json" + ), + "source_worker_receipt_path": ( + f"{source_root}/pixelrag_platform_probe_worker_receipts/" + f"{platform}/{manifest_id}/platform_probe_worker_receipt.json" + ), + "source_receipt_path": ( + f"{source_root}/pixelrag_visual_evidence/" + f"{platform}/{manifest_id}/capture_receipt.json" + ), + "identity_match_execution_mode": "deterministic_replay_only_no_network_no_db", + "identity_key_fields": ["platform", "platform_product_id", "title", "url"], + "identity_candidate_count": 2, + "identity_candidate_contracts": [ + _identity_candidate(platform, manifest_id, 0), + _identity_candidate(platform, manifest_id, 1), + ], + "required_before_data_promotion": [ + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", + ], + "allowed_next_step": "run_marketplace_promotion_gate_replay", + }, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + }, + "writes_database": False, + "writes_database_count": 0, + "network_call_performed": False, + "model_call_performed": False, + } + path = receipt_dir / "marketplace_identity_matcher_replay_receipt.json" + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return path + + +def test_marketplace_promotion_gate_replay_reads_identity_receipts(tmp_path): + from services.pixelrag_marketplace_promotion_gate_replay_service import ( + POLICY, + run_pixelrag_marketplace_promotion_gate_replay, + ) + + identity_root = tmp_path / "identity_matcher" + _write_identity_matcher_receipt(identity_root) + _write_identity_matcher_receipt( + identity_root, + platform="shopee_tw", + manifest_id="shopee-runtime-gap", + adapter_code="shopee", + barrier_type="language_or_region_interstitial", + capture_runtime_unavailable=True, + ) + + payload = run_pixelrag_marketplace_promotion_gate_replay( + identity_matcher_receipt_root=identity_root, + platform=("coupang_tw", "shopee_tw"), + ) + by_platform = {item["platform"]: item for item in payload["worker_items"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "ok" + assert payload["summary"]["candidate_count"] == 2 + assert payload["summary"]["ready_count"] == 2 + assert payload["summary"]["dry_run_count"] == 2 + assert payload["summary"]["promotion_candidate_count"] == 4 + assert payload["summary"]["embedding_signature_guard_ready_count"] == 4 + assert payload["summary"]["capture_runtime_gap_count"] == 1 + assert payload["summary"]["writes_database_count"] == 0 + assert payload["controlled_apply"]["network_call"] is False + assert payload["controlled_apply"]["writes_database"] is False + assert payload["promotion_boundary"]["writes_price_tables"] is False + assert payload["next_machine_action"] == "run_marketplace_promotion_gate_replay_execute" + assert by_platform["coupang_tw"]["worker_status"] == ( + "dry_run_ready_for_marketplace_promotion_gate_replay" + ) + first_promotion = by_platform["coupang_tw"]["promotion_gate_replay"][ + "promotion_candidate_contracts" + ][0] + assert first_promotion["ready_for_embedding_signature_guard_replay"] is True + assert first_promotion["ready_for_candidate_knowledge_replay"] is True + assert first_promotion["ready_for_price_table_write"] is False + assert ( + first_promotion["price_write_blocked_until_embedding_signature_guard"] is True + ) + assert by_platform["shopee_tw"]["promotion_gate_replay"][ + "capture_runtime_unavailable" + ] is True + + +def test_marketplace_promotion_gate_replay_execute_writes_receipt(tmp_path): + from services.pixelrag_marketplace_promotion_gate_replay_service import ( + run_pixelrag_marketplace_promotion_gate_replay, + ) + + identity_root = tmp_path / "identity_matcher" + output_root = tmp_path / "promotion_gate" + _write_identity_matcher_receipt(identity_root) + + payload = run_pixelrag_marketplace_promotion_gate_replay( + identity_matcher_receipt_root=identity_root, + output_root=output_root, + platform="coupang_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["executed_count"] == 1 + assert payload["summary"]["receipt_written_count"] == 1 + assert payload["controlled_apply"]["artifact_write"] is True + assert payload["controlled_apply"]["writes_database"] is False + assert payload["next_machine_action"] == "run_marketplace_embedding_signature_guard_replay" + receipt_path = ( + output_root + / "coupang_tw" + / "coupang-403" + / "marketplace_promotion_gate_replay_receipt.json" + ) + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "executed_marketplace_promotion_gate_replay_ready" + assert receipt["artifact_write_performed"] is True + assert receipt["writes_database"] is False + assert receipt["promotion_gate_replay_check_pass_count"] == ( + receipt["promotion_gate_replay_check_count"] + ) + assert receipt["promotion_gate_replay"]["allowed_next_step"] == ( + "run_marketplace_embedding_signature_guard_replay" + ) + + +def test_marketplace_promotion_gate_replay_cli_outputs_json(tmp_path): + identity_root = tmp_path / "identity_matcher" + _write_identity_matcher_receipt(identity_root) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py", + "--identity-matcher-receipt-root", + str(identity_root), + "--platform", + "coupang_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["summary"]["promotion_candidate_count"] == 2 + assert payload["controlled_apply"]["writes_database"] is False + + +def test_marketplace_promotion_gate_replay_route_returns_readback( + tmp_path, + monkeypatch, +): + from flask import Flask + from routes import system_public_routes as routes + from services import pixelrag_marketplace_promotion_gate_replay_service as service + + identity_root = tmp_path / "identity_matcher" + _write_identity_matcher_receipt(identity_root) + monkeypatch.setattr( + service, + "DEFAULT_IDENTITY_MATCHER_RECEIPT_ROOT", + str(identity_root), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-marketplace-promotion-gate-replay?platform=coupang_tw" + ): + response = ( + routes + .ai_automation_pixelrag_marketplace_promotion_gate_replay_api + .__wrapped__() + ) + payload = response.get_json() + + assert payload["policy"] == ( + "controlled_pixelrag_marketplace_promotion_gate_replay_v1" + ) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False From 9bfbcf94ce56fb00fea8e4ce5a1b15ba949afccd Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 12:26:59 +0800 Subject: [PATCH 27/75] feat(ai): add PixelRAG marketplace embedding signature guard replay --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 4 +- .../ai_automation_mainline_work_items.md | 3 +- docs/guides/browse_sh_crawler_playbook.md | 4 + routes/system_public_routes.py | 37 + ...tplace_embedding_signature_guard_replay.py | 77 +++ services/ai_automation_smoke_service.py | 166 +++++ ...mbedding_signature_guard_replay_service.py | 651 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 40 +- ...mbedding_signature_guard_replay_service.py | 332 +++++++++ 10 files changed, 1309 insertions(+), 7 deletions(-) create mode 100755 scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py create mode 100644 services/pixelrag_marketplace_embedding_signature_guard_replay_service.py create mode 100644 tests/test_pixelrag_marketplace_embedding_signature_guard_replay_service.py diff --git a/config.py b/config.py index 1446b6a..387623e 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.768" +SYSTEM_VERSION = "V10.769" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index b37f60c..ffdd70c 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.768 +> **適用版本**: V10.769 --- @@ -124,6 +124,7 @@ - 2026-07-10 起 PixelRAG marketplace source-contract adapter dry-run 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run` 與 `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py` 預設只讀 adapter preflight receipts 並做 deterministic mapping readiness；`execute=true&write_receipt=true` 只寫 `marketplace_adapter_dry_run_receipt.json` artifact，輸出 field contracts、offer candidate contracts、identity matcher / PromotionGate / embedding signature gates 與 no-write 邊界。此 worker 不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；dry-run receipt 只允許進 identity matcher replay，不得直接進價格表或候選知識層。 - 2026-07-10 起 PixelRAG marketplace identity matcher replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-identity-matcher-replay` 與 `scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py` 預設只讀 adapter dry-run receipts 並產生 deterministic identity key / match contract；`execute=true&write_receipt=true` 只寫 `marketplace_identity_matcher_replay_receipt.json` artifact，輸出 identity candidate contracts、source provenance、PromotionGate readiness 與 no-write 邊界。此 worker 不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；identity replay receipt 只允許進 PromotionGate replay，不得直接進價格表或候選知識層。 - 2026-07-10 起 PixelRAG marketplace PromotionGate replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-promotion-gate-replay` 與 `scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py` 預設只讀 identity matcher receipts 並產生 deterministic PromotionGate contract；`execute=true&write_receipt=true` 只寫 `marketplace_promotion_gate_replay_receipt.json` artifact，輸出 promotion candidate contracts、embedding signature guard readiness、source provenance 與 no-write 邊界。此 worker 不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；PromotionGate receipt 只允許進 embedding signature guard replay，不得直接進價格表或候選知識層。 +- 2026-07-10 起 PixelRAG marketplace embedding signature guard replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay` 與 `scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py` 預設只讀 PromotionGate receipts 並產生 deterministic embedding signature guard contract；`execute=true&write_receipt=true` 只寫 `marketplace_embedding_signature_guard_replay_receipt.json` artifact，輸出 expected embedding signature、signature guard contracts、candidate knowledge replay readiness 與 no-write 邊界。此 worker 不產生 embedding、不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；embedding signature guard receipt 只允許進 candidate knowledge replay，不得直接進價格表或正式知識層。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -904,6 +905,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG marketplace adapter dry-run 必須先產生 no-write mapping receipt | V10.766 起 `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run`、`scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace adapter dry-run / `pixelrag_marketplace_adapter_dry_run` family；dry-run 只讀 `marketplace_adapter_preflight_receipt.json`，驗證 preflight ready、canonical offer fields、required gates、no-DB/no-network/no-model、blocked page not product data 與 price-write blocked。execute 模式只寫 `marketplace_adapter_dry_run_receipt.json` artifact，產生 deterministic field contracts / offer candidate contracts，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 identity matcher replay。 | | 2026-07-10 | PixelRAG marketplace identity matcher replay 必須先產生 no-write match receipt | V10.767 起 `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay`、`scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace identity matcher replay / `pixelrag_marketplace_identity_matcher_replay` family；replay 只讀 `marketplace_adapter_dry_run_receipt.json`，驗證 adapter dry-run ready、identity key fields、candidate contracts、required gates、no-DB/no-network/no-model、source provenance 與 price-write blocked。execute 模式只寫 `marketplace_identity_matcher_replay_receipt.json` artifact，產生 deterministic identity candidate contracts，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 PromotionGate replay。 | | 2026-07-10 | PixelRAG marketplace PromotionGate replay 必須先產生 no-write promotion receipt | V10.768 起 `/api/ai-automation/pixelrag-marketplace-promotion-gate-replay`、`scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace PromotionGate replay / `pixelrag_marketplace_promotion_gate_replay` family；replay 只讀 `marketplace_identity_matcher_replay_receipt.json`，驗證 identity matcher ready、identity checks all passed、required promotion gates、embedding signature guard readiness、no-DB/no-network/no-model、source provenance 與 price-write blocked。execute 模式只寫 `marketplace_promotion_gate_replay_receipt.json` artifact，產生 deterministic promotion candidate contracts，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 embedding signature guard replay。 | +| 2026-07-10 | PixelRAG marketplace embedding signature guard replay 必須先產生 no-write signature receipt | V10.769 起 `/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay`、`scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace embedding signature guard replay / `pixelrag_marketplace_embedding_signature_guard_replay` family；replay 只讀 `marketplace_promotion_gate_replay_receipt.json`，驗證 PromotionGate ready、PromotionGate checks all passed、expected embedding signature、signature guard contracts、candidate knowledge readiness、no-DB/no-network/no-model、no embedding generation、source provenance 與 price-write blocked。execute 模式只寫 `marketplace_embedding_signature_guard_replay_receipt.json` artifact，產生 deterministic signature guard contracts，不產生 embedding、不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 candidate knowledge replay。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 4997b24..219e3d9 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -22,11 +22,12 @@ | Completed | PixelRAG marketplace adapter dry-run | `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run` and `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py` turn adapter preflight receipts into deterministic no-write field contracts and offer candidate contracts, with price writes blocked until identity matcher replay and PromotionGate. | | Completed | PixelRAG marketplace identity matcher replay | `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay` and `scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py` turn adapter dry-run receipts into deterministic identity candidate contracts and source provenance checks, with price writes blocked until PromotionGate replay. | | Completed | PixelRAG marketplace PromotionGate replay | `/api/ai-automation/pixelrag-marketplace-promotion-gate-replay` and `scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py` turn identity matcher receipts into deterministic PromotionGate contracts and embedding-signature readiness, with price writes still blocked. | +| Completed | PixelRAG marketplace embedding signature guard replay | `/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay` and `scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py` turn PromotionGate receipts into deterministic embedding-signature guard contracts and candidate-knowledge readiness, with DB and price writes still blocked. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | | Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture when runtime is available, missing Playwright auto-falls back to structured-source/backoff receipt, and Coupang 403/access denied goes to structured-source/backoff artifact receipts. | -| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, PixelRAG platform probe worker, source-contract replay, adapter preflight, adapter dry-run, identity matcher replay, and PromotionGate replay families. | +| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, PixelRAG platform probe worker, source-contract replay, adapter preflight, adapter dry-run, identity matcher replay, PromotionGate replay, and embedding signature guard replay families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | ## P1 diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index cbd1434..ac79f76 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -206,6 +206,8 @@ python scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py --platfor python scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py --platform coupang_tw --execute --write-receipt --limit 1 python scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py --platform shopee_tw --platform coupang_tw python scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py --platform coupang_tw --execute --write-receipt --limit 1 +python scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py --platform coupang_tw --execute --write-receipt --limit 1 ``` API readback: `/api/ai-automation/pixelrag-vlm-route-readiness` 與 `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。worker 預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。execute 前會自動讀 approved Ollama `/api/tags` 選擇已安裝候選模型；若完全沒有候選，會寫 `model_route_not_ready` artifact receipt，不盲打 generate。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 @@ -222,6 +224,8 @@ Marketplace identity matcher replay API readback: `/api/ai-automation/pixelrag-m Marketplace PromotionGate replay API readback: `/api/ai-automation/pixelrag-marketplace-promotion-gate-replay?platform=coupang_tw`。PromotionGate replay 預設只讀 identity matcher receipts 並建立 deterministic promotion candidate contracts；`execute=true&write_receipt=true` 只寫 PromotionGate artifact receipt，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表。PromotionGate receipt 下一步只能進 embedding signature guard replay，不得直接變成價格資料。 +Marketplace embedding signature guard replay API readback: `/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay?platform=coupang_tw`。embedding signature guard replay 預設只讀 PromotionGate receipts 並建立 deterministic signature guard contracts；`execute=true&write_receipt=true` 只寫 embedding signature guard artifact receipt，不產生 embedding、不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表。signature guard receipt 下一步只能進 candidate knowledge replay，不得直接變成價格資料。 + Application portfolio： ```bash diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 8af7f23..ef55f49 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -1090,6 +1090,43 @@ def ai_automation_pixelrag_marketplace_promotion_gate_replay_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay') +@login_required +def ai_automation_pixelrag_marketplace_embedding_signature_guard_replay_api(): + """Controlled PixelRAG marketplace embedding signature guard readback.""" + from services.pixelrag_marketplace_embedding_signature_guard_replay_service import ( + run_pixelrag_marketplace_embedding_signature_guard_replay, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + execute = str(request.args.get('execute') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + write_receipt = str(request.args.get('write_receipt') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 25, type=int) + return jsonify(run_pixelrag_marketplace_embedding_signature_guard_replay( + promotion_gate_receipt_root=str( + request.args.get('promotion_gate_receipt_root') or '' + ).strip() or None, + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 25, 250)), + execute=execute, + write_receipt=bool(execute and write_receipt), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py b/scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py new file mode 100755 index 0000000..8e0893d --- /dev/null +++ b/scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Run or dry-run PixelRAG marketplace embedding signature guard replay.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_marketplace_embedding_signature_guard_replay_service import ( # noqa: E402 + run_pixelrag_marketplace_embedding_signature_guard_replay, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="執行或 dry-run PixelRAG marketplace embedding signature guard replay。" + ) + parser.add_argument( + "--promotion-gate-receipt-root", + help="Marketplace PromotionGate replay receipt root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--output-root", + help="Marketplace embedding signature guard replay artifact receipt output root。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=25, + help="最多處理 PromotionGate replay receipt 數。", + ) + parser.add_argument( + "--execute", + action="store_true", + help="執行 embedding signature guard replay；未指定時只做 no-write dry-run readiness。", + ) + parser.add_argument( + "--write-receipt", + action="store_true", + help="execute 後寫入 embedding signature guard replay artifact receipt；不寫 DB。", + ) + args = parser.parse_args() + + payload = run_pixelrag_marketplace_embedding_signature_guard_replay( + promotion_gate_receipt_root=args.promotion_gate_receipt_root, + output_root=args.output_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + execute=args.execute, + write_receipt=bool(args.write_receipt and args.execute), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index e3f25cc..2e1fc48 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -620,6 +620,23 @@ def build_scheduled_automation_health_summary( pixelrag_marketplace_promotion_gate_details = ( pixelrag_marketplace_promotion_gate.get("details") or {} ) + pixelrag_marketplace_embedding_signature = _find_check( + source_result, + "PixelRAG marketplace embedding signature guard replay", + ) + pixelrag_marketplace_embedding_signature_details = ( + pixelrag_marketplace_embedding_signature.get("details") or {} + ) + if ( + not pixelrag_marketplace_embedding_signature + or not pixelrag_marketplace_embedding_signature_details + ): + pixelrag_marketplace_embedding_signature = ( + _pixelrag_marketplace_embedding_signature_guard_replay_check() + ) + pixelrag_marketplace_embedding_signature_details = ( + pixelrag_marketplace_embedding_signature.get("details") or {} + ) pixelrag_ocr_vlm_replay = _find_check(source_result, "PixelRAG OCR/VLM replay contract") pixelrag_ocr_vlm_replay_details = pixelrag_ocr_vlm_replay.get("details") or {} if not pixelrag_ocr_vlm_replay or not pixelrag_ocr_vlm_replay_details: @@ -4972,6 +4989,74 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_marketplace_embedding_signature_guard_replay", + "label": "PixelRAG marketplace embedding signature guard replay", + "status": ( + pixelrag_marketplace_embedding_signature.get("status") or "warning" + ), + "summary": ( + pixelrag_marketplace_embedding_signature.get("summary") + or "PixelRAG marketplace embedding signature guard replay has no latest readback." + ), + "next_machine_action": pixelrag_marketplace_embedding_signature_details.get( + "next_machine_action" + ) + or "run_marketplace_embedding_signature_guard_replay", + "details": { + "policy": pixelrag_marketplace_embedding_signature_details.get("policy"), + "candidate_count": int( + pixelrag_marketplace_embedding_signature_details.get("candidate_count") or 0 + ), + "ready_count": int( + pixelrag_marketplace_embedding_signature_details.get("ready_count") or 0 + ), + "blocked_count": int( + pixelrag_marketplace_embedding_signature_details.get("blocked_count") or 0 + ), + "dry_run_count": int( + pixelrag_marketplace_embedding_signature_details.get("dry_run_count") or 0 + ), + "executed_count": int( + pixelrag_marketplace_embedding_signature_details.get("executed_count") or 0 + ), + "receipt_written_count": int( + pixelrag_marketplace_embedding_signature_details.get("receipt_written_count") or 0 + ), + "capture_runtime_gap_count": int( + pixelrag_marketplace_embedding_signature_details.get("capture_runtime_gap_count") or 0 + ), + "signature_guard_candidate_count": int( + pixelrag_marketplace_embedding_signature_details.get("signature_guard_candidate_count") or 0 + ), + "candidate_knowledge_ready_count": int( + pixelrag_marketplace_embedding_signature_details.get("candidate_knowledge_ready_count") or 0 + ), + "expected_embedding_signatures": list( + pixelrag_marketplace_embedding_signature_details.get( + "expected_embedding_signatures" + ) + or [] + ), + "network_call_performed": bool( + pixelrag_marketplace_embedding_signature_details.get("network_call_performed") + ), + "model_call_performed": bool( + pixelrag_marketplace_embedding_signature_details.get("model_call_performed") + ), + "embedding_generation_performed": bool( + pixelrag_marketplace_embedding_signature_details.get( + "embedding_generation_performed" + ) + ), + "artifact_write_performed": bool( + pixelrag_marketplace_embedding_signature_details.get("artifact_write_performed") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "key": "pixelrag_ocr_vlm_replay", "label": "PixelRAG OCR/VLM replay contract", @@ -14182,6 +14267,86 @@ def _pixelrag_marketplace_promotion_gate_replay_check() -> Dict[str, Any]: ) +def _pixelrag_marketplace_embedding_signature_guard_replay_check() -> Dict[str, Any]: + """Dry-run sentinel for PixelRAG marketplace embedding signature guard.""" + try: + from services.pixelrag_marketplace_embedding_signature_guard_replay_service import ( + run_pixelrag_marketplace_embedding_signature_guard_replay, + ) + + readback = run_pixelrag_marketplace_embedding_signature_guard_replay( + execute=False, + ) + summary = readback.get("summary") or {} + candidate_count = int(summary.get("candidate_count") or 0) + ready_count = int(summary.get("ready_count") or 0) + blocked_count = int(summary.get("blocked_count") or 0) + dry_run_count = int(summary.get("dry_run_count") or 0) + executed_count = int(summary.get("executed_count") or 0) + receipt_written_count = int(summary.get("receipt_written_count") or 0) + signature_candidate_count = int( + summary.get("signature_guard_candidate_count") or 0 + ) + knowledge_ready_count = int( + summary.get("candidate_knowledge_ready_count") or 0 + ) + signatures = list(summary.get("expected_embedding_signatures") or []) + status = readback.get("status") or "warning" + signature_text = ",".join(str(item) for item in signatures) or "none" + summary_text = ( + f"PixelRAG marketplace embedding signature guard replay " + f"candidates={candidate_count}, ready={ready_count}, " + f"blocked={blocked_count}, dry_run={dry_run_count}, " + f"executed={executed_count}, receipts={receipt_written_count}, " + f"signature_candidates={signature_candidate_count}, " + f"knowledge_ready={knowledge_ready_count}, signatures={signature_text}" + ) + return _check( + "PixelRAG marketplace embedding signature guard replay", + status, + summary_text, + { + "policy": readback.get("policy"), + "candidate_count": candidate_count, + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "capture_runtime_gap_count": int( + summary.get("capture_runtime_gap_count") or 0 + ), + "guard_failed_count": int(summary.get("guard_failed_count") or 0), + "signature_guard_candidate_count": signature_candidate_count, + "candidate_knowledge_ready_count": knowledge_ready_count, + "expected_embedding_signatures": signatures, + "network_call_performed": bool(summary.get("network_call_performed")), + "model_call_performed": bool(summary.get("model_call_performed")), + "embedding_generation_performed": bool( + summary.get("embedding_generation_performed") + ), + "artifact_write_performed": bool( + summary.get("artifact_write_performed") + ), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG marketplace embedding signature guard replay", + "critical", + f"PixelRAG marketplace embedding signature guard replay 無法執行 dry-run：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def _pixelrag_ocr_vlm_replay_check() -> Dict[str, Any]: """Read-only sentinel for PixelRAG receipts becoming OCR/VLM replay contracts.""" try: @@ -14519,6 +14684,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _pixelrag_marketplace_adapter_dry_run_check(), _pixelrag_marketplace_identity_matcher_replay_check(), _pixelrag_marketplace_promotion_gate_replay_check(), + _pixelrag_marketplace_embedding_signature_guard_replay_check(), _pixelrag_ocr_vlm_replay_check(), _pixelrag_vlm_route_readiness_check(), _pixelrag_vlm_replay_worker_check(), diff --git a/services/pixelrag_marketplace_embedding_signature_guard_replay_service.py b/services/pixelrag_marketplace_embedding_signature_guard_replay_service.py new file mode 100644 index 0000000..77ca973 --- /dev/null +++ b/services/pixelrag_marketplace_embedding_signature_guard_replay_service.py @@ -0,0 +1,651 @@ +"""Controlled PixelRAG marketplace embedding signature guard replay worker. + +This worker turns marketplace PromotionGate receipts into deterministic +embedding signature guard receipts. It does not generate embeddings, fetch +marketplace pages, call models, write databases, write AI insights, or promote +candidate prices. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.pixelrag_crawler_integration_service import DEFAULT_ARTIFACT_MAX_AGE_HOURS +from services.pixelrag_marketplace_promotion_gate_replay_service import ( + DEFAULT_OUTPUT_ROOT as DEFAULT_PROMOTION_GATE_RECEIPT_ROOT, + PROMOTION_GATE_REPLAY_VERSION, +) +from services.rag_service import ( + RAG_EMBED_DIM, + RAG_EMBED_MODEL, + RAG_EMBED_NORMALIZE, + get_embedding_signature, +) + + +POLICY = "controlled_pixelrag_marketplace_embedding_signature_guard_replay_v1" +EMBEDDING_SIGNATURE_GUARD_REPLAY_VERSION = ( + "pixelrag_marketplace_embedding_signature_guard_replay_v1" +) +DEFAULT_LIMIT = 25 +DEFAULT_OUTPUT_ROOT = os.getenv( + "PIXELRAG_MARKETPLACE_EMBEDDING_SIGNATURE_GUARD_REPLAY_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_marketplace_embedding_signature_guard_replay_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_marketplace_embedding_signature_guard_replay_receipts", +) + +REQUIRED_PROMOTION_GATES = { + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", +} + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...]: + if isinstance(platform, str): + value = platform.strip().lower() + return (value,) if value else () + return tuple( + str(item or "").strip().lower() + for item in (platform or ()) + if str(item or "").strip() + ) + + +def _safe_segment(value: Any) -> str: + text = str(value or "unknown").strip().lower() + text = re.sub(r"[^a-z0-9._-]+", "-", text) + return text.strip("-") or "unknown" + + +def _parse_iso_datetime(value: Any) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + + +def _receipt_candidates( + root: Path, + *, + platforms: tuple[str, ...], + limit: int, +) -> list[Path]: + if not root.exists(): + return [] + candidates: list[Path] = [] + if platforms: + for platform in platforms: + candidates.extend( + (root / platform).glob("*/marketplace_promotion_gate_replay_receipt.json") + ) + else: + candidates.extend(root.glob("*/*/marketplace_promotion_gate_replay_receipt.json")) + return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] + + +def _load_receipt(path: Path) -> tuple[dict[str, Any], list[str]]: + try: + return json.loads(path.read_text(encoding="utf-8")), [] + except (OSError, json.JSONDecodeError) as exc: + return {}, [str(exc)[:300]] + + +def _as_mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _as_list(value: Any) -> list[Any]: + return list(value) if isinstance(value, list) else [] + + +def _required_before(promotion_payload: Mapping[str, Any]) -> set[str]: + return set(str(item) for item in _as_list(promotion_payload.get("required_before_data_promotion"))) + + +def _signature_guard_fingerprint( + platform: str, + manifest_id: str, + candidate_id: str, + promotion_fingerprint: str, + embedding_signature: str, +) -> str: + digest = hashlib.sha256( + ( + f"{platform}:{manifest_id}:{candidate_id}:" + f"{promotion_fingerprint}:{embedding_signature}" + ).encode("utf-8") + ).hexdigest() + return digest[:24] + + +def _embedding_signature_contract() -> dict[str, Any]: + signature = get_embedding_signature() + return { + "embedding_signature": signature, + "embedding_model": RAG_EMBED_MODEL, + "embedding_dim": RAG_EMBED_DIM, + "embedding_normalize": RAG_EMBED_NORMALIZE, + "signature_algorithm": "sha1(model|normalize|dim)[:12]", + "embedding_generation_performed": False, + "model_call_performed": False, + } + + +def _signature_guard_candidate_contracts( + receipt: Mapping[str, Any], + *, + promotion_payload: Mapping[str, Any], + embedding_contract: Mapping[str, Any], +) -> list[dict[str, Any]]: + platform = str(receipt.get("platform") or "unknown") + manifest_id = str(receipt.get("manifest_id") or "unknown") + embedding_signature = str(embedding_contract.get("embedding_signature") or "") + contracts: list[dict[str, Any]] = [] + for candidate in _as_list(promotion_payload.get("promotion_candidate_contracts")): + if not isinstance(candidate, Mapping): + continue + candidate_id = str(candidate.get("candidate_id") or f"{platform}:{manifest_id}:unknown") + promotion_fingerprint = str(candidate.get("promotion_fingerprint") or "missing") + contracts.append( + { + "candidate_id": candidate_id, + "candidate_index": int(candidate.get("candidate_index") or 0), + "promotion_fingerprint": promotion_fingerprint, + "identity_fingerprint": candidate.get("identity_fingerprint"), + "expected_embedding_signature": embedding_signature, + "embedding_signature": embedding_signature, + "embedding_signature_guard_fingerprint": _signature_guard_fingerprint( + platform, + manifest_id, + candidate_id, + promotion_fingerprint, + embedding_signature, + ), + "signature_guard_strategy": "deterministic_signature_contract_no_embedding_generation", + "guard_stage": "pre_candidate_knowledge_replay", + "requires_embedding_signature_guard": True, + "requires_candidate_knowledge_replay": True, + "requires_source_provenance": True, + "requires_replay_canary_before_db_write": True, + "embedding_generation_performed": False, + "ready_for_candidate_knowledge_replay": True, + "ready_for_ai_insights_write": False, + "ready_for_price_table_write": False, + "price_write_blocked_until_candidate_canary": True, + } + ) + return contracts + + +def _embedding_signature_guard_payload( + receipt: Mapping[str, Any], + *, + receipt_path: Path, +) -> dict[str, Any]: + promotion_payload = _as_mapping(receipt.get("promotion_gate_replay")) + embedding_contract = _embedding_signature_contract() + guard_contracts = _signature_guard_candidate_contracts( + receipt, + promotion_payload=promotion_payload, + embedding_contract=embedding_contract, + ) + return { + "embedding_signature_guard_replay_version": ( + EMBEDDING_SIGNATURE_GUARD_REPLAY_VERSION + ), + "promotion_gate_replay_version": promotion_payload.get( + "promotion_gate_replay_version" + ), + "identity_matcher_replay_version": promotion_payload.get( + "identity_matcher_replay_version" + ), + "adapter_dry_run_version": promotion_payload.get("adapter_dry_run_version"), + "adapter_preflight_version": promotion_payload.get("adapter_preflight_version"), + "source_contract_version": promotion_payload.get("source_contract_version"), + "adapter_code": promotion_payload.get("adapter_code") or receipt.get("adapter_code"), + "contract_id": promotion_payload.get("contract_id"), + "barrier_type": promotion_payload.get("barrier_type"), + "capture_runtime_unavailable": bool( + promotion_payload.get("capture_runtime_unavailable") + ), + "input_promotion_gate_receipt_path": str(receipt_path), + "input_identity_matcher_receipt_path": promotion_payload.get( + "input_identity_matcher_receipt_path" + ), + "input_adapter_dry_run_receipt_path": promotion_payload.get( + "input_adapter_dry_run_receipt_path" + ), + "adapter_preflight_receipt_path": promotion_payload.get( + "adapter_preflight_receipt_path" + ), + "source_contract_replay_receipt_path": promotion_payload.get( + "source_contract_replay_receipt_path" + ), + "source_worker_receipt_path": promotion_payload.get("source_worker_receipt_path"), + "source_receipt_path": promotion_payload.get("source_receipt_path"), + "embedding_signature_contract": embedding_contract, + "embedding_signature_guard_execution_mode": ( + "deterministic_signature_guard_only_no_embedding_no_network_no_db" + ), + "signature_guard_candidate_count": len(guard_contracts), + "signature_guard_candidate_contracts": guard_contracts, + "required_before_data_promotion": list( + promotion_payload.get("required_before_data_promotion") or [] + ), + "allowed_next_step": "run_marketplace_candidate_knowledge_replay", + } + + +def _embedding_signature_guard_checks( + receipt: Mapping[str, Any], + *, + stale: bool, + errors: list[str], + guard_payload: Mapping[str, Any], +) -> dict[str, bool]: + promotion_payload = _as_mapping(receipt.get("promotion_gate_replay")) + boundary = _as_mapping(receipt.get("promotion_boundary")) + promotion_checks = _as_mapping(receipt.get("promotion_gate_replay_checks")) + promotion_contracts = _as_list(promotion_payload.get("promotion_candidate_contracts")) + guard_contracts = _as_list(guard_payload.get("signature_guard_candidate_contracts")) + embedding_contract = _as_mapping(guard_payload.get("embedding_signature_contract")) + expected_signature = str(embedding_contract.get("embedding_signature") or "") + required_before = _required_before(promotion_payload) + return { + "receipt_parse_ok": not errors, + "receipt_fresh": not stale, + "promotion_gate_replay_ready": ( + receipt.get("worker_status") + == "executed_marketplace_promotion_gate_replay_ready" + ), + "promotion_gate_version_supported": ( + promotion_payload.get("promotion_gate_replay_version") + == PROMOTION_GATE_REPLAY_VERSION + ), + "promotion_gate_checks_all_passed": ( + int(receipt.get("promotion_gate_replay_check_pass_count") or 0) + == int(receipt.get("promotion_gate_replay_check_count") or -1) + and int(receipt.get("promotion_gate_replay_check_count") or 0) > 0 + ), + "blocked_page_not_product_data": bool( + promotion_checks.get("blocked_page_not_product_data") + ), + "promotion_candidate_contracts_present": bool(promotion_contracts), + "signature_guard_contracts_generated": bool(guard_contracts), + "signature_guard_count_matches_promotion_count": ( + len(guard_contracts) == len(promotion_contracts) + ), + "promotion_candidates_ready_for_embedding_signature_guard": all( + bool(item.get("ready_for_embedding_signature_guard_replay")) + for item in promotion_contracts + ), + "required_promotion_gates_present": REQUIRED_PROMOTION_GATES.issubset( + required_before + ), + "embedding_signature_guard_required": "embedding_signature_guard" in required_before, + "embedding_signature_present": ( + len(expected_signature) == 12 + and all(ch in "0123456789abcdef" for ch in expected_signature) + ), + "embedding_generation_not_performed": ( + not bool(embedding_contract.get("embedding_generation_performed")) + and not bool(embedding_contract.get("model_call_performed")) + ), + "signature_guard_all_match_expected_signature": all( + item.get("embedding_signature") == expected_signature + and item.get("expected_embedding_signature") == expected_signature + for item in guard_contracts + ), + "candidate_knowledge_replay_ready": all( + bool(item.get("ready_for_candidate_knowledge_replay")) + for item in guard_contracts + ), + "ai_insights_write_disallowed": all( + not bool(item.get("ready_for_ai_insights_write")) + for item in guard_contracts + ), + "price_write_blocked_until_candidate_canary": all( + bool(item.get("price_write_blocked_until_candidate_canary")) + and not bool(item.get("ready_for_price_table_write")) + for item in guard_contracts + ), + "direct_price_write_disallowed": not bool( + boundary.get("direct_price_write_allowed") + ), + "direct_ai_insights_write_disallowed": not bool( + boundary.get("direct_ai_insights_write_allowed") + ), + "database_write_disallowed": not bool(receipt.get("writes_database")) + and int(receipt.get("writes_database_count") or 0) == 0, + "network_call_disallowed": not bool(receipt.get("network_call_performed")), + "model_call_disallowed": not bool(receipt.get("model_call_performed")), + "source_provenance_paths_present": bool( + guard_payload.get("input_promotion_gate_receipt_path") + and guard_payload.get("input_identity_matcher_receipt_path") + and guard_payload.get("input_adapter_dry_run_receipt_path") + and guard_payload.get("adapter_preflight_receipt_path") + and guard_payload.get("source_contract_replay_receipt_path") + ), + } + + +def _worker_item( + receipt_path: Path, + *, + now: datetime, + max_age_hours: int, + execute: bool, +) -> dict[str, Any]: + receipt, errors = _load_receipt(receipt_path) + generated_at = _parse_iso_datetime(receipt.get("generated_at")) + if generated_at is None: + try: + generated_at = datetime.fromtimestamp(receipt_path.stat().st_mtime, tz=timezone.utc) + except OSError: + generated_at = None + age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None + stale = age_hours is None or age_hours > max_age_hours + guard_payload = _embedding_signature_guard_payload(receipt, receipt_path=receipt_path) + checks = _embedding_signature_guard_checks( + receipt, + stale=stale, + errors=errors, + guard_payload=guard_payload, + ) + check_count = len(checks) + pass_count = sum(1 for passed in checks.values() if passed) + ready = pass_count == check_count + platform = str(receipt.get("platform") or receipt_path.parent.parent.name).strip().lower() + manifest_id = str(receipt.get("manifest_id") or receipt_path.parent.name).strip() + status = ( + "executed_marketplace_embedding_signature_guard_replay_ready" + if execute and ready + else ( + "dry_run_ready_for_marketplace_embedding_signature_guard_replay" + if ready + else "skipped_marketplace_embedding_signature_guard_replay_guard_failed" + ) + ) + return { + "worker_status": status, + "platform": platform, + "manifest_id": manifest_id, + "source_type": "marketplace_promotion_gate_replay_receipt", + "promotion_gate_receipt_path": str(receipt_path), + "identity_matcher_receipt_path": guard_payload.get( + "input_identity_matcher_receipt_path" + ), + "adapter_dry_run_receipt_path": guard_payload.get( + "input_adapter_dry_run_receipt_path" + ), + "adapter_preflight_receipt_path": guard_payload.get( + "adapter_preflight_receipt_path" + ), + "source_contract_replay_receipt_path": guard_payload.get( + "source_contract_replay_receipt_path" + ), + "source_worker_receipt_path": guard_payload.get("source_worker_receipt_path"), + "source_receipt_path": guard_payload.get("source_receipt_path"), + "adapter_code": guard_payload.get("adapter_code"), + "embedding_signature_guard_replay_status": "ready" if ready else "blocked", + "ready_for_execution": ready, + "execute": bool(execute), + "generated_at": generated_at.isoformat() if generated_at else None, + "age_hours": round(age_hours, 3) if age_hours is not None else None, + "stale": stale, + "network_call_performed": False, + "model_call_performed": False, + "embedding_generation_performed": False, + "artifact_write_performed": False, + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + "embedding_signature_guard_replay_checks": checks, + "embedding_signature_guard_replay_check_count": check_count, + "embedding_signature_guard_replay_check_pass_count": pass_count, + "embedding_signature_guard_replay": guard_payload, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "embedding_signature_guard_replay_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_candidate_knowledge_replay": True, + }, + "next_machine_action": ( + "run_marketplace_candidate_knowledge_replay" + if execute and ready + else ( + "run_marketplace_embedding_signature_guard_replay_execute" + if ready + else "repair_marketplace_embedding_signature_guard_replay_inputs" + ) + ), + } + + +def _write_embedding_signature_guard_receipt( + *, + output_root: Path, + item: Mapping[str, Any], +) -> str: + target = ( + output_root + / _safe_segment(item.get("platform")) + / _safe_segment(item.get("manifest_id")) + / "marketplace_embedding_signature_guard_replay_receipt.json" + ) + target.parent.mkdir(parents=True, exist_ok=True) + receipt = dict(item) + receipt["artifact_write_performed"] = True + receipt["receipt_path"] = str(target) + receipt["generated_at"] = datetime.now(timezone.utc).isoformat() + receipt["policy"] = POLICY + target.write_text( + json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + return str(target) + + +def run_pixelrag_marketplace_embedding_signature_guard_replay( + *, + promotion_gate_receipt_root: str | Path | None = None, + output_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, + execute: bool = False, + write_receipt: bool = False, +) -> dict[str, Any]: + """Run or dry-run marketplace embedding signature guard replay.""" + source_root = Path(promotion_gate_receipt_root or DEFAULT_PROMOTION_GATE_RECEIPT_ROOT) + output = Path(output_root or DEFAULT_OUTPUT_ROOT) + platforms = _normalise_platforms(platform) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + now = datetime.now(timezone.utc) + receipt_paths = _receipt_candidates(source_root, platforms=platforms, limit=item_limit) + worker_items: list[dict[str, Any]] = [] + for receipt_path in receipt_paths: + item = _worker_item( + receipt_path, + now=now, + max_age_hours=max_age, + execute=execute, + ) + if execute and write_receipt and item.get("ready_for_execution"): + item["receipt_path"] = _write_embedding_signature_guard_receipt( + output_root=output, + item=item, + ) + item["artifact_write_performed"] = True + worker_items.append(item) + + ready_count = sum(1 for item in worker_items if item.get("ready_for_execution")) + blocked_count = len(worker_items) - ready_count + dry_run_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_") + ) + executed_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_") + ) + receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path")) + guard_failed_count = sum( + 1 + for item in worker_items + if item.get("embedding_signature_guard_replay_status") != "ready" + ) + capture_runtime_gap_count = sum( + 1 + for item in worker_items + if (item.get("embedding_signature_guard_replay") or {}).get( + "capture_runtime_unavailable" + ) + ) + signature_guard_candidate_count = sum( + int( + (item.get("embedding_signature_guard_replay") or {}).get( + "signature_guard_candidate_count" + ) + or 0 + ) + for item in worker_items + ) + candidate_knowledge_ready_count = sum( + sum( + 1 + for candidate in _as_list( + (item.get("embedding_signature_guard_replay") or {}).get( + "signature_guard_candidate_contracts" + ) + ) + if isinstance(candidate, Mapping) + and candidate.get("ready_for_candidate_knowledge_replay") + ) + for item in worker_items + ) + expected_signatures = sorted( + { + str( + ( + (item.get("embedding_signature_guard_replay") or {}) + .get("embedding_signature_contract") + or {} + ).get("embedding_signature") + or "" + ) + for item in worker_items + } + - {""} + ) + + if not worker_items: + status = "warning" + elif guard_failed_count: + status = "warning" + else: + status = "ok" + + if not worker_items: + next_action = "run_marketplace_promotion_gate_replay" + elif not execute and ready_count: + next_action = "run_marketplace_embedding_signature_guard_replay_execute" + elif execute and receipt_written_count: + next_action = "run_marketplace_candidate_knowledge_replay" + elif guard_failed_count: + next_action = "repair_marketplace_embedding_signature_guard_replay_inputs" + else: + next_action = "refresh_marketplace_embedding_signature_guard_replay_candidates" + + summary = { + "candidate_count": len(worker_items), + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "guard_failed_count": guard_failed_count, + "capture_runtime_gap_count": capture_runtime_gap_count, + "signature_guard_candidate_count": signature_guard_candidate_count, + "candidate_knowledge_ready_count": candidate_knowledge_ready_count, + "expected_embedding_signatures": expected_signatures, + "platforms": sorted({str(item.get("platform") or "unknown") for item in worker_items}), + "network_call_performed": False, + "model_call_performed": False, + "embedding_generation_performed": False, + "artifact_write_performed": bool(receipt_written_count), + "writes_database_count": 0, + "primary_human_gate_count": 0, + } + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": now.isoformat(), + "embedding_signature_guard_replay_version": ( + EMBEDDING_SIGNATURE_GUARD_REPLAY_VERSION + ), + "promotion_gate_receipt_root": str(source_root), + "output_root": str(output), + "platform_filter": list(platforms), + "max_age_hours": max_age, + "limit": item_limit, + "execute": bool(execute), + "write_receipt": bool(write_receipt and execute), + "summary": summary, + "worker_items": worker_items, + "controlled_apply": { + "network_call": False, + "model_call": False, + "embedding_generation": False, + "artifact_write": bool(receipt_written_count), + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "secret_read": False, + "raw_cookie_or_session_read": False, + "credentialed_session_allowed": False, + "login_allowed": False, + "production_price_write": False, + "primary_human_gate_count": 0, + }, + "promotion_boundary": { + "writes_ai_insights": False, + "writes_price_tables": False, + "embedding_signature_guard_replay_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_candidate_knowledge_replay": True, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "DEFAULT_OUTPUT_ROOT", + "EMBEDDING_SIGNATURE_GUARD_REPLAY_VERSION", + "POLICY", + "run_pixelrag_marketplace_embedding_signature_guard_replay", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index 19b0820..ae81a7c 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1440,6 +1440,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_dry_run_check", lambda: smoke._check("pixelrag marketplace adapter dry-run", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_identity_matcher_replay_check", lambda: smoke._check("pixelrag marketplace identity matcher replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_promotion_gate_replay_check", lambda: smoke._check("pixelrag marketplace promotion gate replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_embedding_signature_guard_replay_check", lambda: smoke._check("pixelrag marketplace embedding signature guard replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -1449,7 +1450,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 41, "warning": 1, "critical": 1, "total": 43} + assert result["summary"] == {"ok": 42, "warning": 1, "critical": 1, "total": 44} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3984,6 +3985,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_pixelrag_marketplace_adapter_dry_run_check", lambda: smoke._check("pixelrag marketplace adapter dry-run", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_identity_matcher_replay_check", lambda: smoke._check("pixelrag marketplace identity matcher replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_promotion_gate_replay_check", lambda: smoke._check("pixelrag marketplace promotion gate replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_embedding_signature_guard_replay_check", lambda: smoke._check("pixelrag marketplace embedding signature guard replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -4044,7 +4046,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 43, "warning": 0, "critical": 0, "total": 43}, + "summary": {"ok": 44, "warning": 0, "critical": 0, "total": 44}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4264,6 +4266,31 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "primary_human_gate_count": 0, }, }, + { + "name": "PixelRAG marketplace embedding signature guard replay", + "status": "ok", + "summary": "PixelRAG marketplace embedding signature guard replay candidates=1, ready=1, blocked=0, dry_run=1, executed=0, receipts=0, signature_candidates=2, knowledge_ready=2, signatures=8c8063c66dbe", + "details": { + "policy": "controlled_pixelrag_marketplace_embedding_signature_guard_replay_v1", + "candidate_count": 1, + "ready_count": 1, + "blocked_count": 0, + "dry_run_count": 1, + "executed_count": 0, + "receipt_written_count": 0, + "capture_runtime_gap_count": 0, + "signature_guard_candidate_count": 2, + "candidate_knowledge_ready_count": 2, + "expected_embedding_signatures": ["8c8063c66dbe"], + "network_call_performed": False, + "model_call_performed": False, + "embedding_generation_performed": False, + "artifact_write_performed": False, + "next_machine_action": "run_marketplace_embedding_signature_guard_replay_execute", + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "name": "PixelRAG OCR/VLM replay contract", "status": "ok", @@ -4378,7 +4405,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 40 + assert summary["summary"]["total"] == 41 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -6735,6 +6762,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag marketplace PromotionGate replay ok", )) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_embedding_signature_guard_replay_check", lambda: smoke._check( + "PixelRAG marketplace embedding signature guard replay", + "ok", + "pixelrag marketplace embedding signature guard replay ok", + )) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check( "PixelRAG OCR/VLM replay contract", "ok", @@ -6775,7 +6807,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 43 + assert result["summary"]["total"] == 44 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_marketplace_embedding_signature_guard_replay_service.py b/tests/test_pixelrag_marketplace_embedding_signature_guard_replay_service.py new file mode 100644 index 0000000..17215d7 --- /dev/null +++ b/tests/test_pixelrag_marketplace_embedding_signature_guard_replay_service.py @@ -0,0 +1,332 @@ +import json +import subprocess +import sys +from datetime import datetime, timezone + + +def _promotion_candidate(platform, manifest_id, index): + return { + "candidate_id": f"{platform}:{manifest_id}:{index}", + "candidate_index": index, + "identity_fingerprint": f"identity-{platform}-{index}", + "promotion_fingerprint": f"promotion-{platform}-{index}", + "promotion_stage": "pre_embedding_signature_guard", + "promotion_strategy": "deterministic_gate_replay_no_values", + "ready_for_candidate_knowledge_replay": True, + "ready_for_embedding_signature_guard_replay": True, + "ready_for_price_table_write": False, + "price_write_blocked_until_embedding_signature_guard": True, + "requires_embedding_signature_guard": True, + "requires_evidence_time": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_source_provenance": True, + } + + +def _write_promotion_gate_receipt( + root, + *, + platform="coupang_tw", + manifest_id="coupang-403", + adapter_code="coupang", + barrier_type="access_denied", + capture_runtime_unavailable=False, +): + receipt_dir = root / platform / manifest_id + receipt_dir.mkdir(parents=True) + source_root = "runtime_artifacts" + payload = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "policy": "controlled_pixelrag_marketplace_promotion_gate_replay_v1", + "worker_status": "executed_marketplace_promotion_gate_replay_ready", + "platform": platform, + "manifest_id": manifest_id, + "adapter_code": adapter_code, + "promotion_gate_receipt_path": ( + f"{source_root}/pixelrag_marketplace_promotion_gate_replay_receipts/" + f"{platform}/{manifest_id}/marketplace_promotion_gate_replay_receipt.json" + ), + "identity_matcher_receipt_path": ( + f"{source_root}/pixelrag_marketplace_identity_matcher_replay_receipts/" + f"{platform}/{manifest_id}/marketplace_identity_matcher_replay_receipt.json" + ), + "adapter_dry_run_receipt_path": ( + f"{source_root}/pixelrag_marketplace_adapter_dry_run_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_dry_run_receipt.json" + ), + "adapter_preflight_receipt_path": ( + f"{source_root}/pixelrag_marketplace_adapter_preflight_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_preflight_receipt.json" + ), + "source_contract_replay_receipt_path": ( + f"{source_root}/pixelrag_source_contract_replay_receipts/" + f"{platform}/{manifest_id}/source_contract_replay_receipt.json" + ), + "source_worker_receipt_path": ( + f"{source_root}/pixelrag_platform_probe_worker_receipts/" + f"{platform}/{manifest_id}/platform_probe_worker_receipt.json" + ), + "source_receipt_path": ( + f"{source_root}/pixelrag_visual_evidence/" + f"{platform}/{manifest_id}/capture_receipt.json" + ), + "promotion_gate_replay_check_count": 22, + "promotion_gate_replay_check_pass_count": 22, + "promotion_gate_replay_checks": { + "blocked_page_not_product_data": True, + "candidate_knowledge_replay_ready": True, + "database_write_disallowed": True, + "direct_ai_insights_write_disallowed": True, + "direct_price_write_disallowed": True, + "embedding_signature_guard_replay_ready": True, + "embedding_signature_guard_required": True, + "identity_candidate_contracts_present": True, + "identity_candidates_ready_for_promotion": True, + "identity_matcher_checks_all_passed": True, + "identity_matcher_replay_ready": True, + "identity_matcher_version_supported": True, + "model_call_disallowed": True, + "network_call_disallowed": True, + "price_write_blocked_until_embedding_signature_guard": True, + "promotion_candidate_contracts_generated": True, + "promotion_candidate_count_matches_identity_count": True, + "promotion_gate_required": True, + "receipt_fresh": True, + "receipt_parse_ok": True, + "required_promotion_gates_present": True, + "source_provenance_paths_present": True, + }, + "promotion_gate_replay": { + "promotion_gate_replay_version": "pixelrag_marketplace_promotion_gate_replay_v1", + "identity_matcher_replay_version": "pixelrag_marketplace_identity_matcher_replay_v1", + "adapter_dry_run_version": "pixelrag_marketplace_adapter_dry_run_v1", + "adapter_preflight_version": "pixelrag_marketplace_adapter_preflight_v1", + "source_contract_version": "pixelrag_marketplace_source_contract_v1", + "contract_id": f"{platform}:{manifest_id}:{adapter_code}", + "adapter_code": adapter_code, + "barrier_type": barrier_type, + "capture_runtime_unavailable": capture_runtime_unavailable, + "input_promotion_gate_receipt_path": ( + f"{source_root}/pixelrag_marketplace_promotion_gate_replay_receipts/" + f"{platform}/{manifest_id}/marketplace_promotion_gate_replay_receipt.json" + ), + "input_identity_matcher_receipt_path": ( + f"{source_root}/pixelrag_marketplace_identity_matcher_replay_receipts/" + f"{platform}/{manifest_id}/marketplace_identity_matcher_replay_receipt.json" + ), + "input_adapter_dry_run_receipt_path": ( + f"{source_root}/pixelrag_marketplace_adapter_dry_run_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_dry_run_receipt.json" + ), + "adapter_preflight_receipt_path": ( + f"{source_root}/pixelrag_marketplace_adapter_preflight_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_preflight_receipt.json" + ), + "source_contract_replay_receipt_path": ( + f"{source_root}/pixelrag_source_contract_replay_receipts/" + f"{platform}/{manifest_id}/source_contract_replay_receipt.json" + ), + "source_worker_receipt_path": ( + f"{source_root}/pixelrag_platform_probe_worker_receipts/" + f"{platform}/{manifest_id}/platform_probe_worker_receipt.json" + ), + "source_receipt_path": ( + f"{source_root}/pixelrag_visual_evidence/" + f"{platform}/{manifest_id}/capture_receipt.json" + ), + "promotion_gate_execution_mode": "deterministic_replay_only_no_network_no_db", + "promotion_candidate_count": 2, + "promotion_candidate_contracts": [ + _promotion_candidate(platform, manifest_id, 0), + _promotion_candidate(platform, manifest_id, 1), + ], + "required_before_data_promotion": [ + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", + ], + "allowed_next_step": "run_marketplace_embedding_signature_guard_replay", + }, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + }, + "writes_database": False, + "writes_database_count": 0, + "network_call_performed": False, + "model_call_performed": False, + } + path = receipt_dir / "marketplace_promotion_gate_replay_receipt.json" + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return path + + +def test_marketplace_embedding_signature_guard_reads_promotion_receipts(tmp_path): + from services.pixelrag_marketplace_embedding_signature_guard_replay_service import ( + POLICY, + run_pixelrag_marketplace_embedding_signature_guard_replay, + ) + from services.rag_service import get_embedding_signature + + promotion_root = tmp_path / "promotion_gate" + _write_promotion_gate_receipt(promotion_root) + _write_promotion_gate_receipt( + promotion_root, + platform="shopee_tw", + manifest_id="shopee-runtime-gap", + adapter_code="shopee", + barrier_type="language_or_region_interstitial", + capture_runtime_unavailable=True, + ) + + payload = run_pixelrag_marketplace_embedding_signature_guard_replay( + promotion_gate_receipt_root=promotion_root, + platform=("coupang_tw", "shopee_tw"), + ) + by_platform = {item["platform"]: item for item in payload["worker_items"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "ok" + assert payload["summary"]["candidate_count"] == 2 + assert payload["summary"]["ready_count"] == 2 + assert payload["summary"]["dry_run_count"] == 2 + assert payload["summary"]["signature_guard_candidate_count"] == 4 + assert payload["summary"]["candidate_knowledge_ready_count"] == 4 + assert payload["summary"]["capture_runtime_gap_count"] == 1 + assert payload["summary"]["expected_embedding_signatures"] == [ + get_embedding_signature() + ] + assert payload["summary"]["writes_database_count"] == 0 + assert payload["controlled_apply"]["embedding_generation"] is False + assert payload["controlled_apply"]["network_call"] is False + assert payload["promotion_boundary"]["writes_price_tables"] is False + assert payload["next_machine_action"] == ( + "run_marketplace_embedding_signature_guard_replay_execute" + ) + assert by_platform["coupang_tw"]["worker_status"] == ( + "dry_run_ready_for_marketplace_embedding_signature_guard_replay" + ) + first_guard = by_platform["coupang_tw"]["embedding_signature_guard_replay"][ + "signature_guard_candidate_contracts" + ][0] + assert first_guard["embedding_signature"] == get_embedding_signature() + assert first_guard["ready_for_candidate_knowledge_replay"] is True + assert first_guard["ready_for_ai_insights_write"] is False + assert first_guard["ready_for_price_table_write"] is False + assert first_guard["price_write_blocked_until_candidate_canary"] is True + assert by_platform["shopee_tw"]["embedding_signature_guard_replay"][ + "capture_runtime_unavailable" + ] is True + + +def test_marketplace_embedding_signature_guard_execute_writes_receipt(tmp_path): + from services.pixelrag_marketplace_embedding_signature_guard_replay_service import ( + run_pixelrag_marketplace_embedding_signature_guard_replay, + ) + + promotion_root = tmp_path / "promotion_gate" + output_root = tmp_path / "embedding_signature" + _write_promotion_gate_receipt(promotion_root) + + payload = run_pixelrag_marketplace_embedding_signature_guard_replay( + promotion_gate_receipt_root=promotion_root, + output_root=output_root, + platform="coupang_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["executed_count"] == 1 + assert payload["summary"]["receipt_written_count"] == 1 + assert payload["controlled_apply"]["artifact_write"] is True + assert payload["controlled_apply"]["writes_database"] is False + assert payload["next_machine_action"] == "run_marketplace_candidate_knowledge_replay" + receipt_path = ( + output_root + / "coupang_tw" + / "coupang-403" + / "marketplace_embedding_signature_guard_replay_receipt.json" + ) + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == ( + "executed_marketplace_embedding_signature_guard_replay_ready" + ) + assert receipt["artifact_write_performed"] is True + assert receipt["writes_database"] is False + assert receipt["embedding_signature_guard_replay_check_pass_count"] == ( + receipt["embedding_signature_guard_replay_check_count"] + ) + assert receipt["embedding_signature_guard_replay"]["allowed_next_step"] == ( + "run_marketplace_candidate_knowledge_replay" + ) + + +def test_marketplace_embedding_signature_guard_cli_outputs_json(tmp_path): + promotion_root = tmp_path / "promotion_gate" + _write_promotion_gate_receipt(promotion_root) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py", + "--promotion-gate-receipt-root", + str(promotion_root), + "--platform", + "coupang_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["summary"]["signature_guard_candidate_count"] == 2 + assert payload["controlled_apply"]["writes_database"] is False + + +def test_marketplace_embedding_signature_guard_route_returns_readback( + tmp_path, + monkeypatch, +): + from flask import Flask + from routes import system_public_routes as routes + from services import ( + pixelrag_marketplace_embedding_signature_guard_replay_service as service, + ) + + promotion_root = tmp_path / "promotion_gate" + _write_promotion_gate_receipt(promotion_root) + monkeypatch.setattr( + service, + "DEFAULT_PROMOTION_GATE_RECEIPT_ROOT", + str(promotion_root), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay?platform=coupang_tw" + ): + response = ( + routes + .ai_automation_pixelrag_marketplace_embedding_signature_guard_replay_api + .__wrapped__() + ) + payload = response.get_json() + + assert payload["policy"] == ( + "controlled_pixelrag_marketplace_embedding_signature_guard_replay_v1" + ) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False From cd12e4850573f2c64740243d11b6f4b18eba0c61 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 12:51:56 +0800 Subject: [PATCH 28/75] feat(ai): add PixelRAG marketplace candidate knowledge replay --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 4 +- .../ai_automation_mainline_work_items.md | 3 +- docs/guides/browse_sh_crawler_playbook.md | 4 + routes/system_public_routes.py | 37 + ..._marketplace_candidate_knowledge_replay.py | 79 ++ services/ai_automation_smoke_service.py | 174 +++++ ...lace_candidate_knowledge_replay_service.py | 729 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 42 +- ...lace_candidate_knowledge_replay_service.py | 325 ++++++++ 10 files changed, 1392 insertions(+), 7 deletions(-) create mode 100644 scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py create mode 100644 services/pixelrag_marketplace_candidate_knowledge_replay_service.py create mode 100644 tests/test_pixelrag_marketplace_candidate_knowledge_replay_service.py diff --git a/config.py b/config.py index 387623e..3fa492f 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.769" +SYSTEM_VERSION = "V10.770" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index ffdd70c..9d2b355 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.769 +> **適用版本**: V10.770 --- @@ -125,6 +125,7 @@ - 2026-07-10 起 PixelRAG marketplace identity matcher replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-identity-matcher-replay` 與 `scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py` 預設只讀 adapter dry-run receipts 並產生 deterministic identity key / match contract；`execute=true&write_receipt=true` 只寫 `marketplace_identity_matcher_replay_receipt.json` artifact，輸出 identity candidate contracts、source provenance、PromotionGate readiness 與 no-write 邊界。此 worker 不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；identity replay receipt 只允許進 PromotionGate replay，不得直接進價格表或候選知識層。 - 2026-07-10 起 PixelRAG marketplace PromotionGate replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-promotion-gate-replay` 與 `scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py` 預設只讀 identity matcher receipts 並產生 deterministic PromotionGate contract；`execute=true&write_receipt=true` 只寫 `marketplace_promotion_gate_replay_receipt.json` artifact，輸出 promotion candidate contracts、embedding signature guard readiness、source provenance 與 no-write 邊界。此 worker 不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；PromotionGate receipt 只允許進 embedding signature guard replay，不得直接進價格表或候選知識層。 - 2026-07-10 起 PixelRAG marketplace embedding signature guard replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay` 與 `scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py` 預設只讀 PromotionGate receipts 並產生 deterministic embedding signature guard contract；`execute=true&write_receipt=true` 只寫 `marketplace_embedding_signature_guard_replay_receipt.json` artifact，輸出 expected embedding signature、signature guard contracts、candidate knowledge replay readiness 與 no-write 邊界。此 worker 不產生 embedding、不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；embedding signature guard receipt 只允許進 candidate knowledge replay，不得直接進價格表或正式知識層。 +- 2026-07-10 起 PixelRAG marketplace candidate knowledge replay 必須是正式 smoke/scheduled family：`/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay` 與 `scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py` 預設只讀 embedding signature guard receipts 並產生 deterministic internal RAG candidate knowledge contract；`execute=true&write_receipt=true` 只寫 `marketplace_candidate_knowledge_replay_receipt.json` artifact，輸出 candidate knowledge text、internal RAG preview readiness、source provenance 與 no-write 邊界。此 worker 不產生 embedding、不抓外站、不呼叫模型、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；candidate knowledge receipt 只允許進 internal RAG candidate canary，不得直接進正式知識層或價格表。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -906,6 +907,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG marketplace identity matcher replay 必須先產生 no-write match receipt | V10.767 起 `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay`、`scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace identity matcher replay / `pixelrag_marketplace_identity_matcher_replay` family；replay 只讀 `marketplace_adapter_dry_run_receipt.json`，驗證 adapter dry-run ready、identity key fields、candidate contracts、required gates、no-DB/no-network/no-model、source provenance 與 price-write blocked。execute 模式只寫 `marketplace_identity_matcher_replay_receipt.json` artifact，產生 deterministic identity candidate contracts，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 PromotionGate replay。 | | 2026-07-10 | PixelRAG marketplace PromotionGate replay 必須先產生 no-write promotion receipt | V10.768 起 `/api/ai-automation/pixelrag-marketplace-promotion-gate-replay`、`scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace PromotionGate replay / `pixelrag_marketplace_promotion_gate_replay` family；replay 只讀 `marketplace_identity_matcher_replay_receipt.json`，驗證 identity matcher ready、identity checks all passed、required promotion gates、embedding signature guard readiness、no-DB/no-network/no-model、source provenance 與 price-write blocked。execute 模式只寫 `marketplace_promotion_gate_replay_receipt.json` artifact，產生 deterministic promotion candidate contracts，不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 embedding signature guard replay。 | | 2026-07-10 | PixelRAG marketplace embedding signature guard replay 必須先產生 no-write signature receipt | V10.769 起 `/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay`、`scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace embedding signature guard replay / `pixelrag_marketplace_embedding_signature_guard_replay` family；replay 只讀 `marketplace_promotion_gate_replay_receipt.json`，驗證 PromotionGate ready、PromotionGate checks all passed、expected embedding signature、signature guard contracts、candidate knowledge readiness、no-DB/no-network/no-model、no embedding generation、source provenance 與 price-write blocked。execute 模式只寫 `marketplace_embedding_signature_guard_replay_receipt.json` artifact，產生 deterministic signature guard contracts，不產生 embedding、不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 candidate knowledge replay。 | +| 2026-07-10 | PixelRAG marketplace candidate knowledge replay 必須先產生 no-write internal RAG candidate receipt | V10.770 起 `/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay`、`scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 PixelRAG marketplace candidate knowledge replay / `pixelrag_marketplace_candidate_knowledge_replay` family；replay 只讀 `marketplace_embedding_signature_guard_replay_receipt.json`，驗證 signature guard ready、signature checks all passed、expected embedding signature、knowledge candidate text、internal RAG preview readiness、no-DB/no-network/no-model、no embedding generation、source provenance、AI insights write blocked 與 price-write blocked。execute 模式只寫 `marketplace_candidate_knowledge_replay_receipt.json` artifact，產生 deterministic internal RAG candidate preview contracts，不產生 embedding、不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表，下一步只允許進 internal RAG candidate canary。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 219e3d9..4ed8ca6 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -23,11 +23,12 @@ | Completed | PixelRAG marketplace identity matcher replay | `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay` and `scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py` turn adapter dry-run receipts into deterministic identity candidate contracts and source provenance checks, with price writes blocked until PromotionGate replay. | | Completed | PixelRAG marketplace PromotionGate replay | `/api/ai-automation/pixelrag-marketplace-promotion-gate-replay` and `scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py` turn identity matcher receipts into deterministic PromotionGate contracts and embedding-signature readiness, with price writes still blocked. | | Completed | PixelRAG marketplace embedding signature guard replay | `/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay` and `scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py` turn PromotionGate receipts into deterministic embedding-signature guard contracts and candidate-knowledge readiness, with DB and price writes still blocked. | +| Completed | PixelRAG marketplace candidate knowledge replay | `/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay` and `scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py` turn embedding-signature guard receipts into deterministic internal RAG candidate preview contracts, with DB, `ai_insights`, and price writes still blocked until canary. | | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | | Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture when runtime is available, missing Playwright auto-falls back to structured-source/backoff receipt, and Coupang 403/access denied goes to structured-source/backoff artifact receipts. | -| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, PixelRAG platform probe worker, source-contract replay, adapter preflight, adapter dry-run, identity matcher replay, PromotionGate replay, and embedding signature guard replay families. | +| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, PixelRAG platform probe worker, source-contract replay, adapter preflight, adapter dry-run, identity matcher replay, PromotionGate replay, embedding signature guard replay, and candidate knowledge replay families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | ## P1 diff --git a/docs/guides/browse_sh_crawler_playbook.md b/docs/guides/browse_sh_crawler_playbook.md index ac79f76..123bd27 100644 --- a/docs/guides/browse_sh_crawler_playbook.md +++ b/docs/guides/browse_sh_crawler_playbook.md @@ -208,6 +208,8 @@ python scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py --platform python scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py --platform coupang_tw --execute --write-receipt --limit 1 python scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py --platform shopee_tw --platform coupang_tw python scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py --platform coupang_tw --execute --write-receipt --limit 1 +python scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py --platform shopee_tw --platform coupang_tw +python scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py --platform coupang_tw --execute --write-receipt --limit 1 ``` API readback: `/api/ai-automation/pixelrag-vlm-route-readiness` 與 `/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw`。worker 預設為 dry-run，不呼叫模型、不寫 artifact；`execute=true&write_receipt=true` 才呼叫 Ollama VLM 並寫 artifact receipt。execute 前會自動讀 approved Ollama `/api/tags` 選擇已安裝候選模型；若完全沒有候選，會寫 `model_route_not_ready` artifact receipt，不盲打 generate。即使 execute，結果仍只是 candidate evidence；不得直接寫 `ai_insights`、正式價格表或競品價格歷史，且 missing confidence/evidence 會留在 replay / probe lane。 @@ -226,6 +228,8 @@ Marketplace PromotionGate replay API readback: `/api/ai-automation/pixelrag-mark Marketplace embedding signature guard replay API readback: `/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay?platform=coupang_tw`。embedding signature guard replay 預設只讀 PromotionGate receipts 並建立 deterministic signature guard contracts；`execute=true&write_receipt=true` 只寫 embedding signature guard artifact receipt，不產生 embedding、不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表。signature guard receipt 下一步只能進 candidate knowledge replay，不得直接變成價格資料。 +Marketplace candidate knowledge replay API readback: `/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay?platform=coupang_tw`。candidate knowledge replay 預設只讀 embedding signature guard receipts 並建立 deterministic internal RAG candidate preview contracts；`execute=true&write_receipt=true` 只寫 candidate knowledge artifact receipt，不產生 embedding、不抓外站、不呼叫模型、不寫 DB、不寫 `ai_insights`、不寫正式價格表。candidate knowledge receipt 下一步只能進 internal RAG candidate canary，不得直接變成正式知識庫或價格資料。 + Application portfolio： ```bash diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index ef55f49..7f1ab0e 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -1127,6 +1127,43 @@ def ai_automation_pixelrag_marketplace_embedding_signature_guard_replay_api(): )) +@system_public_bp.route('/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay') +@login_required +def ai_automation_pixelrag_marketplace_candidate_knowledge_replay_api(): + """Controlled PixelRAG marketplace candidate knowledge readback.""" + from services.pixelrag_marketplace_candidate_knowledge_replay_service import ( + run_pixelrag_marketplace_candidate_knowledge_replay, + ) + + platforms = tuple( + str(item or '').strip() + for item in request.args.getlist('platform') + if str(item or '').strip() + ) + execute = str(request.args.get('execute') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + write_receipt = str(request.args.get('write_receipt') or '').strip().lower() in { + '1', + 'true', + 'yes', + } + max_age_hours = request.args.get('max_age_hours', 168, type=int) + limit = request.args.get('limit', 25, type=int) + return jsonify(run_pixelrag_marketplace_candidate_knowledge_replay( + embedding_signature_guard_receipt_root=str( + request.args.get('embedding_signature_guard_receipt_root') or '' + ).strip() or None, + platform=platforms, + max_age_hours=max(1, min(max_age_hours or 168, 720)), + limit=max(1, min(limit or 25, 250)), + execute=execute, + write_receipt=bool(execute and write_receipt), + )) + + @system_public_bp.route('/api/ai-automation/external-mcp-rag-integration') @login_required def ai_automation_external_mcp_rag_integration_api(): diff --git a/scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py b/scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py new file mode 100644 index 0000000..a532303 --- /dev/null +++ b/scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Run or dry-run PixelRAG marketplace candidate knowledge replay.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.pixelrag_marketplace_candidate_knowledge_replay_service import ( # noqa: E402 + run_pixelrag_marketplace_candidate_knowledge_replay, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="執行或 dry-run PixelRAG marketplace candidate knowledge replay。" + ) + parser.add_argument( + "--embedding-signature-guard-receipt-root", + help="Marketplace embedding signature guard receipt root；預設使用 production/container 設定。", + ) + parser.add_argument( + "--output-root", + help="Marketplace candidate knowledge replay artifact receipt output root。", + ) + parser.add_argument( + "--platform", + action="append", + dest="platforms", + help="限制平台，可重複指定，例如 --platform shopee_tw --platform coupang_tw。", + ) + parser.add_argument( + "--max-age-hours", + type=int, + default=168, + help="receipt 最大新鮮度小時數。", + ) + parser.add_argument( + "--limit", + type=int, + default=25, + help="最多處理 embedding signature guard receipt 數。", + ) + parser.add_argument( + "--execute", + action="store_true", + help="執行 candidate knowledge replay；未指定時只做 no-write dry-run readiness。", + ) + parser.add_argument( + "--write-receipt", + action="store_true", + help="execute 後寫入 candidate knowledge replay artifact receipt；不寫 DB。", + ) + args = parser.parse_args() + + payload = run_pixelrag_marketplace_candidate_knowledge_replay( + embedding_signature_guard_receipt_root=( + args.embedding_signature_guard_receipt_root + ), + output_root=args.output_root, + platform=tuple(args.platforms or ()), + max_age_hours=args.max_age_hours, + limit=args.limit, + execute=args.execute, + write_receipt=bool(args.write_receipt and args.execute), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if payload.get("success") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index 2e1fc48..1f9ec2f 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -637,6 +637,23 @@ def build_scheduled_automation_health_summary( pixelrag_marketplace_embedding_signature_details = ( pixelrag_marketplace_embedding_signature.get("details") or {} ) + pixelrag_marketplace_candidate_knowledge = _find_check( + source_result, + "PixelRAG marketplace candidate knowledge replay", + ) + pixelrag_marketplace_candidate_knowledge_details = ( + pixelrag_marketplace_candidate_knowledge.get("details") or {} + ) + if ( + not pixelrag_marketplace_candidate_knowledge + or not pixelrag_marketplace_candidate_knowledge_details + ): + pixelrag_marketplace_candidate_knowledge = ( + _pixelrag_marketplace_candidate_knowledge_replay_check() + ) + pixelrag_marketplace_candidate_knowledge_details = ( + pixelrag_marketplace_candidate_knowledge.get("details") or {} + ) pixelrag_ocr_vlm_replay = _find_check(source_result, "PixelRAG OCR/VLM replay contract") pixelrag_ocr_vlm_replay_details = pixelrag_ocr_vlm_replay.get("details") or {} if not pixelrag_ocr_vlm_replay or not pixelrag_ocr_vlm_replay_details: @@ -5057,6 +5074,80 @@ def build_scheduled_automation_health_summary( "primary_human_gate_count": 0, }, }, + { + "key": "pixelrag_marketplace_candidate_knowledge_replay", + "label": "PixelRAG marketplace candidate knowledge replay", + "status": ( + pixelrag_marketplace_candidate_knowledge.get("status") or "warning" + ), + "summary": ( + pixelrag_marketplace_candidate_knowledge.get("summary") + or "PixelRAG marketplace candidate knowledge replay has no latest readback." + ), + "next_machine_action": pixelrag_marketplace_candidate_knowledge_details.get( + "next_machine_action" + ) + or "run_marketplace_candidate_knowledge_replay", + "details": { + "policy": pixelrag_marketplace_candidate_knowledge_details.get("policy"), + "candidate_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("candidate_count") or 0 + ), + "ready_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("ready_count") or 0 + ), + "blocked_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("blocked_count") or 0 + ), + "dry_run_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("dry_run_count") or 0 + ), + "executed_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("executed_count") or 0 + ), + "receipt_written_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("receipt_written_count") or 0 + ), + "capture_runtime_gap_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("capture_runtime_gap_count") or 0 + ), + "knowledge_candidate_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("knowledge_candidate_count") or 0 + ), + "internal_rag_ready_count": int( + pixelrag_marketplace_candidate_knowledge_details.get("internal_rag_ready_count") or 0 + ), + "expected_embedding_signatures": list( + pixelrag_marketplace_candidate_knowledge_details.get( + "expected_embedding_signatures" + ) + or [] + ), + "network_call_performed": bool( + pixelrag_marketplace_candidate_knowledge_details.get("network_call_performed") + ), + "model_call_performed": bool( + pixelrag_marketplace_candidate_knowledge_details.get("model_call_performed") + ), + "embedding_generation_performed": bool( + pixelrag_marketplace_candidate_knowledge_details.get( + "embedding_generation_performed" + ) + ), + "artifact_write_performed": bool( + pixelrag_marketplace_candidate_knowledge_details.get("artifact_write_performed") + ), + "writes_ai_insights": bool( + pixelrag_marketplace_candidate_knowledge_details.get("writes_ai_insights") + ), + "writes_price_tables": bool( + pixelrag_marketplace_candidate_knowledge_details.get("writes_price_tables") + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "key": "pixelrag_ocr_vlm_replay", "label": "PixelRAG OCR/VLM replay contract", @@ -14347,6 +14438,88 @@ def _pixelrag_marketplace_embedding_signature_guard_replay_check() -> Dict[str, ) +def _pixelrag_marketplace_candidate_knowledge_replay_check() -> Dict[str, Any]: + """Dry-run sentinel for PixelRAG marketplace candidate knowledge replay.""" + try: + from services.pixelrag_marketplace_candidate_knowledge_replay_service import ( + run_pixelrag_marketplace_candidate_knowledge_replay, + ) + + readback = run_pixelrag_marketplace_candidate_knowledge_replay( + execute=False, + ) + summary = readback.get("summary") or {} + candidate_count = int(summary.get("candidate_count") or 0) + ready_count = int(summary.get("ready_count") or 0) + blocked_count = int(summary.get("blocked_count") or 0) + dry_run_count = int(summary.get("dry_run_count") or 0) + executed_count = int(summary.get("executed_count") or 0) + receipt_written_count = int(summary.get("receipt_written_count") or 0) + knowledge_candidate_count = int( + summary.get("knowledge_candidate_count") or 0 + ) + internal_rag_ready_count = int( + summary.get("internal_rag_ready_count") or 0 + ) + signatures = list(summary.get("expected_embedding_signatures") or []) + status = readback.get("status") or "warning" + signature_text = ",".join(str(item) for item in signatures) or "none" + summary_text = ( + f"PixelRAG marketplace candidate knowledge replay " + f"candidates={candidate_count}, ready={ready_count}, " + f"blocked={blocked_count}, dry_run={dry_run_count}, " + f"executed={executed_count}, receipts={receipt_written_count}, " + f"knowledge_candidates={knowledge_candidate_count}, " + f"internal_rag_ready={internal_rag_ready_count}, signatures={signature_text}" + ) + return _check( + "PixelRAG marketplace candidate knowledge replay", + status, + summary_text, + { + "policy": readback.get("policy"), + "candidate_count": candidate_count, + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "capture_runtime_gap_count": int( + summary.get("capture_runtime_gap_count") or 0 + ), + "guard_failed_count": int(summary.get("guard_failed_count") or 0), + "knowledge_candidate_count": knowledge_candidate_count, + "internal_rag_ready_count": internal_rag_ready_count, + "expected_embedding_signatures": signatures, + "network_call_performed": bool(summary.get("network_call_performed")), + "model_call_performed": bool(summary.get("model_call_performed")), + "embedding_generation_performed": bool( + summary.get("embedding_generation_performed") + ), + "artifact_write_performed": bool( + summary.get("artifact_write_performed") + ), + "writes_ai_insights": bool(summary.get("writes_ai_insights")), + "writes_price_tables": bool(summary.get("writes_price_tables")), + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "PixelRAG marketplace candidate knowledge replay", + "critical", + f"PixelRAG marketplace candidate knowledge replay 無法執行 dry-run：{exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def _pixelrag_ocr_vlm_replay_check() -> Dict[str, Any]: """Read-only sentinel for PixelRAG receipts becoming OCR/VLM replay contracts.""" try: @@ -14685,6 +14858,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _pixelrag_marketplace_identity_matcher_replay_check(), _pixelrag_marketplace_promotion_gate_replay_check(), _pixelrag_marketplace_embedding_signature_guard_replay_check(), + _pixelrag_marketplace_candidate_knowledge_replay_check(), _pixelrag_ocr_vlm_replay_check(), _pixelrag_vlm_route_readiness_check(), _pixelrag_vlm_replay_worker_check(), diff --git a/services/pixelrag_marketplace_candidate_knowledge_replay_service.py b/services/pixelrag_marketplace_candidate_knowledge_replay_service.py new file mode 100644 index 0000000..fc94a9e --- /dev/null +++ b/services/pixelrag_marketplace_candidate_knowledge_replay_service.py @@ -0,0 +1,729 @@ +"""Controlled PixelRAG marketplace candidate knowledge replay worker. + +This worker turns marketplace embedding signature guard receipts into +deterministic internal RAG candidate-knowledge receipts. It does not generate +embeddings, call models, call networks, write databases, write AI insights, or +promote candidate prices. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.pixelrag_crawler_integration_service import DEFAULT_ARTIFACT_MAX_AGE_HOURS +from services.pixelrag_marketplace_embedding_signature_guard_replay_service import ( + DEFAULT_OUTPUT_ROOT as DEFAULT_EMBEDDING_SIGNATURE_GUARD_RECEIPT_ROOT, + EMBEDDING_SIGNATURE_GUARD_REPLAY_VERSION, +) +from services.rag_service import get_embedding_signature + + +POLICY = "controlled_pixelrag_marketplace_candidate_knowledge_replay_v1" +CANDIDATE_KNOWLEDGE_REPLAY_VERSION = ( + "pixelrag_marketplace_candidate_knowledge_replay_v1" +) +INTERNAL_RAG_TARGET = "rag_service.learning_episode_candidate_preview" +DEFAULT_LIMIT = 25 +DEFAULT_OUTPUT_ROOT = os.getenv( + "PIXELRAG_MARKETPLACE_CANDIDATE_KNOWLEDGE_REPLAY_RECEIPT_ROOT", + "/app/data/ai_automation/pixelrag_marketplace_candidate_knowledge_replay_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pixelrag_marketplace_candidate_knowledge_replay_receipts", +) + +REQUIRED_SIGNATURE_GATES = { + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", +} + + +def _normalise_platforms( + platform: str | tuple[str, ...] | list[str] | None, +) -> tuple[str, ...]: + if isinstance(platform, str): + value = platform.strip().lower() + return (value,) if value else () + return tuple( + str(item or "").strip().lower() + for item in (platform or ()) + if str(item or "").strip() + ) + + +def _safe_segment(value: Any) -> str: + text = str(value or "unknown").strip().lower() + text = re.sub(r"[^a-z0-9._-]+", "-", text) + return text.strip("-") or "unknown" + + +def _parse_iso_datetime(value: Any) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + + +def _receipt_candidates( + root: Path, + *, + platforms: tuple[str, ...], + limit: int, +) -> list[Path]: + if not root.exists(): + return [] + candidates: list[Path] = [] + if platforms: + for platform in platforms: + candidates.extend( + (root / platform).glob( + "*/marketplace_embedding_signature_guard_replay_receipt.json" + ) + ) + else: + candidates.extend( + root.glob("*/*/marketplace_embedding_signature_guard_replay_receipt.json") + ) + return sorted(candidates, key=lambda path: path.stat().st_mtime, reverse=True)[:limit] + + +def _load_receipt(path: Path) -> tuple[dict[str, Any], list[str]]: + try: + return json.loads(path.read_text(encoding="utf-8")), [] + except (OSError, json.JSONDecodeError) as exc: + return {}, [str(exc)[:300]] + + +def _as_mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _as_list(value: Any) -> list[Any]: + return list(value) if isinstance(value, list) else [] + + +def _required_before(signature_payload: Mapping[str, Any]) -> set[str]: + return set(str(item) for item in _as_list(signature_payload.get("required_before_data_promotion"))) + + +def _knowledge_fingerprint( + platform: str, + manifest_id: str, + candidate_id: str, + signature_guard_fingerprint: str, + embedding_signature: str, +) -> str: + digest = hashlib.sha256( + ( + f"{platform}:{manifest_id}:{candidate_id}:" + f"{signature_guard_fingerprint}:{embedding_signature}" + ).encode("utf-8") + ).hexdigest() + return digest[:24] + + +def _knowledge_text( + *, + platform: str, + manifest_id: str, + adapter_code: str, + barrier_type: str, + candidate: Mapping[str, Any], + embedding_signature: str, +) -> str: + parts = [ + f"platform={platform}", + f"manifest_id={manifest_id}", + f"adapter_code={adapter_code or 'unknown'}", + f"barrier_type={barrier_type or 'none'}", + f"candidate_id={candidate.get('candidate_id') or 'unknown'}", + f"candidate_index={int(candidate.get('candidate_index') or 0)}", + f"identity_fingerprint={candidate.get('identity_fingerprint') or 'unknown'}", + f"promotion_fingerprint={candidate.get('promotion_fingerprint') or 'unknown'}", + ( + "embedding_signature_guard_fingerprint=" + f"{candidate.get('embedding_signature_guard_fingerprint') or 'unknown'}" + ), + f"embedding_signature={embedding_signature}", + "knowledge_source=pixelrag_marketplace_signature_guard_replay", + "write_target=internal_rag_candidate_preview", + ] + return " | ".join(parts) + + +def _candidate_knowledge_contracts( + receipt: Mapping[str, Any], + *, + signature_payload: Mapping[str, Any], +) -> list[dict[str, Any]]: + platform = str(receipt.get("platform") or "unknown") + manifest_id = str(receipt.get("manifest_id") or "unknown") + adapter_code = str(signature_payload.get("adapter_code") or receipt.get("adapter_code") or "") + barrier_type = str(signature_payload.get("barrier_type") or "") + embedding_contract = _as_mapping(signature_payload.get("embedding_signature_contract")) + embedding_signature = str(embedding_contract.get("embedding_signature") or "") + contracts: list[dict[str, Any]] = [] + for candidate in _as_list(signature_payload.get("signature_guard_candidate_contracts")): + if not isinstance(candidate, Mapping): + continue + candidate_id = str(candidate.get("candidate_id") or f"{platform}:{manifest_id}:unknown") + signature_guard_fingerprint = str( + candidate.get("embedding_signature_guard_fingerprint") or "missing" + ) + knowledge_fingerprint = _knowledge_fingerprint( + platform, + manifest_id, + candidate_id, + signature_guard_fingerprint, + embedding_signature, + ) + contracts.append( + { + "candidate_id": candidate_id, + "candidate_index": int(candidate.get("candidate_index") or 0), + "knowledge_candidate_id": ( + f"{platform}:{manifest_id}:{knowledge_fingerprint}" + ), + "identity_fingerprint": candidate.get("identity_fingerprint"), + "promotion_fingerprint": candidate.get("promotion_fingerprint"), + "embedding_signature_guard_fingerprint": signature_guard_fingerprint, + "candidate_knowledge_fingerprint": knowledge_fingerprint, + "embedding_signature": embedding_signature, + "expected_embedding_signature": ( + candidate.get("expected_embedding_signature") or embedding_signature + ), + "candidate_knowledge_text": _knowledge_text( + platform=platform, + manifest_id=manifest_id, + adapter_code=adapter_code, + barrier_type=barrier_type, + candidate=candidate, + embedding_signature=embedding_signature, + ), + "internal_rag_target": INTERNAL_RAG_TARGET, + "knowledge_stage": "pre_internal_rag_candidate_canary", + "knowledge_strategy": "deterministic_candidate_knowledge_replay_no_db_write", + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + "requires_candidate_knowledge_replay": True, + "requires_internal_rag_candidate_canary": True, + "embedding_generation_performed": False, + "model_call_performed": False, + "ready_for_internal_rag_candidate_replay": True, + "ready_for_rag_candidate_preview": True, + "ready_for_ai_insights_write": False, + "ready_for_price_table_write": False, + "ai_insights_write_blocked_until_canary": True, + "price_write_blocked_until_candidate_canary": True, + } + ) + return contracts + + +def _candidate_knowledge_payload( + receipt: Mapping[str, Any], + *, + receipt_path: Path, +) -> dict[str, Any]: + signature_payload = _as_mapping(receipt.get("embedding_signature_guard_replay")) + knowledge_contracts = _candidate_knowledge_contracts( + receipt, + signature_payload=signature_payload, + ) + return { + "candidate_knowledge_replay_version": CANDIDATE_KNOWLEDGE_REPLAY_VERSION, + "embedding_signature_guard_replay_version": signature_payload.get( + "embedding_signature_guard_replay_version" + ), + "promotion_gate_replay_version": signature_payload.get( + "promotion_gate_replay_version" + ), + "identity_matcher_replay_version": signature_payload.get( + "identity_matcher_replay_version" + ), + "adapter_dry_run_version": signature_payload.get("adapter_dry_run_version"), + "adapter_preflight_version": signature_payload.get("adapter_preflight_version"), + "source_contract_version": signature_payload.get("source_contract_version"), + "adapter_code": signature_payload.get("adapter_code") or receipt.get("adapter_code"), + "contract_id": signature_payload.get("contract_id"), + "barrier_type": signature_payload.get("barrier_type"), + "capture_runtime_unavailable": bool( + signature_payload.get("capture_runtime_unavailable") + ), + "input_embedding_signature_guard_receipt_path": str(receipt_path), + "input_promotion_gate_receipt_path": signature_payload.get( + "input_promotion_gate_receipt_path" + ), + "input_identity_matcher_receipt_path": signature_payload.get( + "input_identity_matcher_receipt_path" + ), + "input_adapter_dry_run_receipt_path": signature_payload.get( + "input_adapter_dry_run_receipt_path" + ), + "adapter_preflight_receipt_path": signature_payload.get( + "adapter_preflight_receipt_path" + ), + "source_contract_replay_receipt_path": signature_payload.get( + "source_contract_replay_receipt_path" + ), + "source_worker_receipt_path": signature_payload.get("source_worker_receipt_path"), + "source_receipt_path": signature_payload.get("source_receipt_path"), + "embedding_signature_contract": signature_payload.get( + "embedding_signature_contract" + ) + or {}, + "candidate_knowledge_execution_mode": ( + "deterministic_candidate_knowledge_replay_no_embedding_no_network_no_db" + ), + "candidate_knowledge_count": len(knowledge_contracts), + "candidate_knowledge_contracts": knowledge_contracts, + "required_before_data_promotion": list( + signature_payload.get("required_before_data_promotion") or [] + ), + "allowed_next_step": "run_internal_rag_candidate_canary", + } + + +def _candidate_knowledge_checks( + receipt: Mapping[str, Any], + *, + stale: bool, + errors: list[str], + knowledge_payload: Mapping[str, Any], +) -> dict[str, bool]: + signature_payload = _as_mapping(receipt.get("embedding_signature_guard_replay")) + boundary = _as_mapping(receipt.get("promotion_boundary")) + signature_checks = _as_mapping( + receipt.get("embedding_signature_guard_replay_checks") + ) + signature_contracts = _as_list( + signature_payload.get("signature_guard_candidate_contracts") + ) + knowledge_contracts = _as_list( + knowledge_payload.get("candidate_knowledge_contracts") + ) + embedding_contract = _as_mapping( + knowledge_payload.get("embedding_signature_contract") + ) + expected_signature = str(embedding_contract.get("embedding_signature") or "") + current_signature = get_embedding_signature() + required_before = _required_before(signature_payload) + return { + "receipt_parse_ok": not errors, + "receipt_fresh": not stale, + "embedding_signature_guard_replay_ready": ( + receipt.get("worker_status") + == "executed_marketplace_embedding_signature_guard_replay_ready" + ), + "embedding_signature_guard_version_supported": ( + signature_payload.get("embedding_signature_guard_replay_version") + == EMBEDDING_SIGNATURE_GUARD_REPLAY_VERSION + ), + "embedding_signature_guard_checks_all_passed": ( + int(receipt.get("embedding_signature_guard_replay_check_pass_count") or 0) + == int(receipt.get("embedding_signature_guard_replay_check_count") or -1) + and int(receipt.get("embedding_signature_guard_replay_check_count") or 0) > 0 + ), + "blocked_page_not_product_data": bool( + signature_checks.get("blocked_page_not_product_data") + ), + "signature_guard_candidate_contracts_present": bool(signature_contracts), + "candidate_knowledge_contracts_generated": bool(knowledge_contracts), + "candidate_knowledge_count_matches_signature_count": ( + len(knowledge_contracts) == len(signature_contracts) + ), + "signature_candidates_ready_for_candidate_knowledge": all( + bool(item.get("ready_for_candidate_knowledge_replay")) + for item in signature_contracts + ), + "required_signature_gates_present": REQUIRED_SIGNATURE_GATES.issubset( + required_before + ), + "candidate_knowledge_replay_required": all( + bool(item.get("requires_candidate_knowledge_replay")) + for item in signature_contracts + ), + "embedding_signature_present": ( + len(expected_signature) == 12 + and all(ch in "0123456789abcdef" for ch in expected_signature) + ), + "embedding_signature_matches_current_rag_signature": ( + expected_signature == current_signature + ), + "embedding_generation_not_performed": ( + not bool(embedding_contract.get("embedding_generation_performed")) + and not bool(embedding_contract.get("model_call_performed")) + and not bool(receipt.get("embedding_generation_performed")) + ), + "knowledge_text_generated": all( + bool(item.get("candidate_knowledge_text")) for item in knowledge_contracts + ), + "internal_rag_preview_ready": all( + bool(item.get("ready_for_internal_rag_candidate_replay")) + and item.get("internal_rag_target") == INTERNAL_RAG_TARGET + for item in knowledge_contracts + ), + "ai_insights_write_disallowed": all( + not bool(item.get("ready_for_ai_insights_write")) + and bool(item.get("ai_insights_write_blocked_until_canary")) + for item in knowledge_contracts + ), + "price_write_blocked_until_candidate_canary": all( + bool(item.get("price_write_blocked_until_candidate_canary")) + and not bool(item.get("ready_for_price_table_write")) + for item in knowledge_contracts + ), + "direct_price_write_disallowed": not bool( + boundary.get("direct_price_write_allowed") + ), + "direct_ai_insights_write_disallowed": not bool( + boundary.get("direct_ai_insights_write_allowed") + ), + "database_write_disallowed": not bool(receipt.get("writes_database")) + and int(receipt.get("writes_database_count") or 0) == 0, + "network_call_disallowed": not bool(receipt.get("network_call_performed")), + "model_call_disallowed": not bool(receipt.get("model_call_performed")), + "source_provenance_paths_present": bool( + knowledge_payload.get("input_embedding_signature_guard_receipt_path") + and knowledge_payload.get("input_promotion_gate_receipt_path") + and knowledge_payload.get("input_identity_matcher_receipt_path") + and knowledge_payload.get("input_adapter_dry_run_receipt_path") + and knowledge_payload.get("adapter_preflight_receipt_path") + and knowledge_payload.get("source_contract_replay_receipt_path") + ), + } + + +def _worker_item( + receipt_path: Path, + *, + now: datetime, + max_age_hours: int, + execute: bool, +) -> dict[str, Any]: + receipt, errors = _load_receipt(receipt_path) + generated_at = _parse_iso_datetime(receipt.get("generated_at")) + if generated_at is None: + try: + generated_at = datetime.fromtimestamp(receipt_path.stat().st_mtime, tz=timezone.utc) + except OSError: + generated_at = None + age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None + stale = age_hours is None or age_hours > max_age_hours + knowledge_payload = _candidate_knowledge_payload(receipt, receipt_path=receipt_path) + checks = _candidate_knowledge_checks( + receipt, + stale=stale, + errors=errors, + knowledge_payload=knowledge_payload, + ) + check_count = len(checks) + pass_count = sum(1 for passed in checks.values() if passed) + ready = pass_count == check_count + platform = str(receipt.get("platform") or receipt_path.parent.parent.name).strip().lower() + manifest_id = str(receipt.get("manifest_id") or receipt_path.parent.name).strip() + status = ( + "executed_marketplace_candidate_knowledge_replay_ready" + if execute and ready + else ( + "dry_run_ready_for_marketplace_candidate_knowledge_replay" + if ready + else "skipped_marketplace_candidate_knowledge_replay_guard_failed" + ) + ) + return { + "worker_status": status, + "platform": platform, + "manifest_id": manifest_id, + "source_type": "marketplace_embedding_signature_guard_replay_receipt", + "embedding_signature_guard_receipt_path": str(receipt_path), + "promotion_gate_receipt_path": knowledge_payload.get( + "input_promotion_gate_receipt_path" + ), + "identity_matcher_receipt_path": knowledge_payload.get( + "input_identity_matcher_receipt_path" + ), + "adapter_dry_run_receipt_path": knowledge_payload.get( + "input_adapter_dry_run_receipt_path" + ), + "adapter_preflight_receipt_path": knowledge_payload.get( + "adapter_preflight_receipt_path" + ), + "source_contract_replay_receipt_path": knowledge_payload.get( + "source_contract_replay_receipt_path" + ), + "source_worker_receipt_path": knowledge_payload.get("source_worker_receipt_path"), + "source_receipt_path": knowledge_payload.get("source_receipt_path"), + "adapter_code": knowledge_payload.get("adapter_code"), + "candidate_knowledge_replay_status": "ready" if ready else "blocked", + "ready_for_execution": ready, + "execute": bool(execute), + "generated_at": generated_at.isoformat() if generated_at else None, + "age_hours": round(age_hours, 3) if age_hours is not None else None, + "stale": stale, + "network_call_performed": False, + "model_call_performed": False, + "embedding_generation_performed": False, + "artifact_write_performed": False, + "writes_database": False, + "writes_database_count": 0, + "writes_ai_insights": False, + "writes_price_tables": False, + "primary_human_gate_count": 0, + "candidate_knowledge_replay_checks": checks, + "candidate_knowledge_replay_check_count": check_count, + "candidate_knowledge_replay_check_pass_count": pass_count, + "candidate_knowledge_replay": knowledge_payload, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "candidate_knowledge_replay_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + "requires_internal_rag_candidate_canary": True, + }, + "next_machine_action": ( + "run_internal_rag_candidate_canary" + if execute and ready + else ( + "run_marketplace_candidate_knowledge_replay_execute" + if ready + else "repair_marketplace_candidate_knowledge_replay_inputs" + ) + ), + } + + +def _write_candidate_knowledge_receipt( + *, + output_root: Path, + item: Mapping[str, Any], +) -> str: + target = ( + output_root + / _safe_segment(item.get("platform")) + / _safe_segment(item.get("manifest_id")) + / "marketplace_candidate_knowledge_replay_receipt.json" + ) + target.parent.mkdir(parents=True, exist_ok=True) + receipt = dict(item) + receipt["artifact_write_performed"] = True + receipt["receipt_path"] = str(target) + receipt["generated_at"] = datetime.now(timezone.utc).isoformat() + receipt["policy"] = POLICY + target.write_text( + json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + return str(target) + + +def run_pixelrag_marketplace_candidate_knowledge_replay( + *, + embedding_signature_guard_receipt_root: str | Path | None = None, + output_root: str | Path | None = None, + platform: str | tuple[str, ...] | list[str] | None = None, + max_age_hours: int | None = None, + limit: int | None = None, + execute: bool = False, + write_receipt: bool = False, +) -> dict[str, Any]: + """Run or dry-run marketplace candidate knowledge replay.""" + source_root = Path( + embedding_signature_guard_receipt_root + or DEFAULT_EMBEDDING_SIGNATURE_GUARD_RECEIPT_ROOT + ) + output = Path(output_root or DEFAULT_OUTPUT_ROOT) + platforms = _normalise_platforms(platform) + max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) + item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 250)) + now = datetime.now(timezone.utc) + receipt_paths = _receipt_candidates(source_root, platforms=platforms, limit=item_limit) + worker_items: list[dict[str, Any]] = [] + for receipt_path in receipt_paths: + item = _worker_item( + receipt_path, + now=now, + max_age_hours=max_age, + execute=execute, + ) + if execute and write_receipt and item.get("ready_for_execution"): + item["receipt_path"] = _write_candidate_knowledge_receipt( + output_root=output, + item=item, + ) + item["artifact_write_performed"] = True + worker_items.append(item) + + ready_count = sum(1 for item in worker_items if item.get("ready_for_execution")) + blocked_count = len(worker_items) - ready_count + dry_run_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_") + ) + executed_count = sum( + 1 for item in worker_items if str(item.get("worker_status") or "").startswith("executed_") + ) + receipt_written_count = sum(1 for item in worker_items if item.get("receipt_path")) + guard_failed_count = sum( + 1 + for item in worker_items + if item.get("candidate_knowledge_replay_status") != "ready" + ) + capture_runtime_gap_count = sum( + 1 + for item in worker_items + if (item.get("candidate_knowledge_replay") or {}).get( + "capture_runtime_unavailable" + ) + ) + knowledge_candidate_count = sum( + int( + (item.get("candidate_knowledge_replay") or {}).get( + "candidate_knowledge_count" + ) + or 0 + ) + for item in worker_items + ) + internal_rag_ready_count = sum( + sum( + 1 + for candidate in _as_list( + (item.get("candidate_knowledge_replay") or {}).get( + "candidate_knowledge_contracts" + ) + ) + if isinstance(candidate, Mapping) + and candidate.get("ready_for_internal_rag_candidate_replay") + ) + for item in worker_items + ) + expected_signatures = sorted( + { + str( + ( + (item.get("candidate_knowledge_replay") or {}) + .get("embedding_signature_contract") + or {} + ).get("embedding_signature") + or "" + ) + for item in worker_items + } + - {""} + ) + + if not worker_items: + status = "warning" + elif guard_failed_count: + status = "warning" + else: + status = "ok" + + if not worker_items: + next_action = "run_marketplace_embedding_signature_guard_replay" + elif not execute and ready_count: + next_action = "run_marketplace_candidate_knowledge_replay_execute" + elif execute and receipt_written_count: + next_action = "run_internal_rag_candidate_canary" + elif guard_failed_count: + next_action = "repair_marketplace_candidate_knowledge_replay_inputs" + else: + next_action = "refresh_marketplace_candidate_knowledge_replay_candidates" + + summary = { + "candidate_count": len(worker_items), + "ready_count": ready_count, + "blocked_count": blocked_count, + "dry_run_count": dry_run_count, + "executed_count": executed_count, + "receipt_written_count": receipt_written_count, + "guard_failed_count": guard_failed_count, + "capture_runtime_gap_count": capture_runtime_gap_count, + "knowledge_candidate_count": knowledge_candidate_count, + "internal_rag_ready_count": internal_rag_ready_count, + "expected_embedding_signatures": expected_signatures, + "platforms": sorted({str(item.get("platform") or "unknown") for item in worker_items}), + "network_call_performed": False, + "model_call_performed": False, + "embedding_generation_performed": False, + "artifact_write_performed": bool(receipt_written_count), + "writes_database_count": 0, + "writes_ai_insights": False, + "writes_price_tables": False, + "primary_human_gate_count": 0, + } + return { + "success": status != "critical", + "policy": POLICY, + "status": status, + "generated_at": now.isoformat(), + "candidate_knowledge_replay_version": CANDIDATE_KNOWLEDGE_REPLAY_VERSION, + "embedding_signature_guard_receipt_root": str(source_root), + "output_root": str(output), + "platform_filter": list(platforms), + "max_age_hours": max_age, + "limit": item_limit, + "execute": bool(execute), + "write_receipt": bool(write_receipt and execute), + "summary": summary, + "worker_items": worker_items, + "controlled_apply": { + "network_call": False, + "model_call": False, + "embedding_generation": False, + "artifact_write": bool(receipt_written_count), + "db_write": False, + "writes_database": False, + "writes_database_count": 0, + "writes_ai_insights": False, + "writes_price_tables": False, + "secret_read": False, + "raw_cookie_or_session_read": False, + "credentialed_session_allowed": False, + "login_allowed": False, + "production_price_write": False, + "primary_human_gate_count": 0, + }, + "promotion_boundary": { + "writes_ai_insights": False, + "writes_price_tables": False, + "candidate_knowledge_replay_only": True, + "requires_public_source_boundary": True, + "requires_rate_limit_contract": True, + "requires_provenance_contract": True, + "requires_identity_matcher_replay": True, + "requires_promotion_gate": True, + "requires_embedding_signature_guard": True, + "requires_internal_rag_candidate_canary": True, + }, + "next_machine_action": next_action, + } + + +__all__ = [ + "CANDIDATE_KNOWLEDGE_REPLAY_VERSION", + "DEFAULT_OUTPUT_ROOT", + "INTERNAL_RAG_TARGET", + "POLICY", + "run_pixelrag_marketplace_candidate_knowledge_replay", +] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index ae81a7c..69e100c 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1441,6 +1441,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_pixelrag_marketplace_identity_matcher_replay_check", lambda: smoke._check("pixelrag marketplace identity matcher replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_promotion_gate_replay_check", lambda: smoke._check("pixelrag marketplace promotion gate replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_embedding_signature_guard_replay_check", lambda: smoke._check("pixelrag marketplace embedding signature guard replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_candidate_knowledge_replay_check", lambda: smoke._check("pixelrag marketplace candidate knowledge replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -1450,7 +1451,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 42, "warning": 1, "critical": 1, "total": 44} + assert result["summary"] == {"ok": 43, "warning": 1, "critical": 1, "total": 45} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3986,6 +3987,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_pixelrag_marketplace_identity_matcher_replay_check", lambda: smoke._check("pixelrag marketplace identity matcher replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_promotion_gate_replay_check", lambda: smoke._check("pixelrag marketplace promotion gate replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_marketplace_embedding_signature_guard_replay_check", lambda: smoke._check("pixelrag marketplace embedding signature guard replay", "ok", "ok")) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_candidate_knowledge_replay_check", lambda: smoke._check("pixelrag marketplace candidate knowledge replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check("pixelrag ocr vlm", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_route_readiness_check", lambda: smoke._check("pixelrag vlm route", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_vlm_replay_worker_check", lambda: smoke._check("pixelrag vlm worker", "ok", "ok")) @@ -4046,7 +4048,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( json.dumps({ "generated_at": datetime.now().isoformat(timespec="seconds"), "status": "ok", - "summary": {"ok": 44, "warning": 0, "critical": 0, "total": 44}, + "summary": {"ok": 45, "warning": 0, "critical": 0, "total": 45}, "checks": [ { "name": "PChome 受控落地 drift monitor", @@ -4291,6 +4293,33 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( "primary_human_gate_count": 0, }, }, + { + "name": "PixelRAG marketplace candidate knowledge replay", + "status": "ok", + "summary": "PixelRAG marketplace candidate knowledge replay candidates=1, ready=1, blocked=0, dry_run=1, executed=0, receipts=0, knowledge_candidates=2, internal_rag_ready=2, signatures=8c8063c66dbe", + "details": { + "policy": "controlled_pixelrag_marketplace_candidate_knowledge_replay_v1", + "candidate_count": 1, + "ready_count": 1, + "blocked_count": 0, + "dry_run_count": 1, + "executed_count": 0, + "receipt_written_count": 0, + "capture_runtime_gap_count": 0, + "knowledge_candidate_count": 2, + "internal_rag_ready_count": 2, + "expected_embedding_signatures": ["8c8063c66dbe"], + "network_call_performed": False, + "model_call_performed": False, + "embedding_generation_performed": False, + "artifact_write_performed": False, + "writes_ai_insights": False, + "writes_price_tables": False, + "next_machine_action": "run_marketplace_candidate_knowledge_replay_execute", + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "name": "PixelRAG OCR/VLM replay contract", "status": "ok", @@ -4405,7 +4434,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 41 + assert summary["summary"]["total"] == 42 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -6767,6 +6796,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "ok", "pixelrag marketplace embedding signature guard replay ok", )) + monkeypatch.setattr(smoke, "_pixelrag_marketplace_candidate_knowledge_replay_check", lambda: smoke._check( + "PixelRAG marketplace candidate knowledge replay", + "ok", + "pixelrag marketplace candidate knowledge replay ok", + )) monkeypatch.setattr(smoke, "_pixelrag_ocr_vlm_replay_check", lambda: smoke._check( "PixelRAG OCR/VLM replay contract", "ok", @@ -6807,7 +6841,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 44 + assert result["summary"]["total"] == 45 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_pixelrag_marketplace_candidate_knowledge_replay_service.py b/tests/test_pixelrag_marketplace_candidate_knowledge_replay_service.py new file mode 100644 index 0000000..e615cf5 --- /dev/null +++ b/tests/test_pixelrag_marketplace_candidate_knowledge_replay_service.py @@ -0,0 +1,325 @@ +import json +import subprocess +import sys +from datetime import datetime, timezone + + +def _signature_candidate(platform, manifest_id, index, embedding_signature): + return { + "candidate_id": f"{platform}:{manifest_id}:{index}", + "candidate_index": index, + "identity_fingerprint": f"identity-{platform}-{index}", + "promotion_fingerprint": f"promotion-{platform}-{index}", + "embedding_signature_guard_fingerprint": f"signature-guard-{platform}-{index}", + "expected_embedding_signature": embedding_signature, + "embedding_signature": embedding_signature, + "guard_stage": "pre_candidate_knowledge_replay", + "ready_for_candidate_knowledge_replay": True, + "ready_for_ai_insights_write": False, + "ready_for_price_table_write": False, + "price_write_blocked_until_candidate_canary": True, + "requires_candidate_knowledge_replay": True, + "requires_embedding_signature_guard": True, + "requires_source_provenance": True, + "embedding_generation_performed": False, + } + + +def _write_embedding_signature_guard_receipt( + root, + *, + platform="coupang_tw", + manifest_id="coupang-403", + adapter_code="coupang", + barrier_type="access_denied", + capture_runtime_unavailable=False, +): + from services.rag_service import get_embedding_signature + + receipt_dir = root / platform / manifest_id + receipt_dir.mkdir(parents=True) + source_root = "runtime_artifacts" + embedding_signature = get_embedding_signature() + payload = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "policy": "controlled_pixelrag_marketplace_embedding_signature_guard_replay_v1", + "worker_status": "executed_marketplace_embedding_signature_guard_replay_ready", + "platform": platform, + "manifest_id": manifest_id, + "adapter_code": adapter_code, + "embedding_signature_guard_replay_check_count": 24, + "embedding_signature_guard_replay_check_pass_count": 24, + "embedding_signature_guard_replay_checks": { + "blocked_page_not_product_data": True, + "candidate_knowledge_replay_ready": True, + "database_write_disallowed": True, + "direct_ai_insights_write_disallowed": True, + "direct_price_write_disallowed": True, + "embedding_generation_not_performed": True, + "embedding_signature_guard_required": True, + "embedding_signature_present": True, + "model_call_disallowed": True, + "network_call_disallowed": True, + "price_write_blocked_until_candidate_canary": True, + "receipt_fresh": True, + "receipt_parse_ok": True, + "required_promotion_gates_present": True, + "signature_guard_contracts_generated": True, + "source_provenance_paths_present": True, + }, + "embedding_signature_guard_replay": { + "embedding_signature_guard_replay_version": "pixelrag_marketplace_embedding_signature_guard_replay_v1", + "promotion_gate_replay_version": "pixelrag_marketplace_promotion_gate_replay_v1", + "identity_matcher_replay_version": "pixelrag_marketplace_identity_matcher_replay_v1", + "adapter_dry_run_version": "pixelrag_marketplace_adapter_dry_run_v1", + "adapter_preflight_version": "pixelrag_marketplace_adapter_preflight_v1", + "source_contract_version": "pixelrag_marketplace_source_contract_v1", + "contract_id": f"{platform}:{manifest_id}:{adapter_code}", + "adapter_code": adapter_code, + "barrier_type": barrier_type, + "capture_runtime_unavailable": capture_runtime_unavailable, + "input_embedding_signature_guard_receipt_path": ( + f"{source_root}/pixelrag_marketplace_embedding_signature_guard_replay_receipts/" + f"{platform}/{manifest_id}/marketplace_embedding_signature_guard_replay_receipt.json" + ), + "input_promotion_gate_receipt_path": ( + f"{source_root}/pixelrag_marketplace_promotion_gate_replay_receipts/" + f"{platform}/{manifest_id}/marketplace_promotion_gate_replay_receipt.json" + ), + "input_identity_matcher_receipt_path": ( + f"{source_root}/pixelrag_marketplace_identity_matcher_replay_receipts/" + f"{platform}/{manifest_id}/marketplace_identity_matcher_replay_receipt.json" + ), + "input_adapter_dry_run_receipt_path": ( + f"{source_root}/pixelrag_marketplace_adapter_dry_run_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_dry_run_receipt.json" + ), + "adapter_preflight_receipt_path": ( + f"{source_root}/pixelrag_marketplace_adapter_preflight_receipts/" + f"{platform}/{manifest_id}/marketplace_adapter_preflight_receipt.json" + ), + "source_contract_replay_receipt_path": ( + f"{source_root}/pixelrag_source_contract_replay_receipts/" + f"{platform}/{manifest_id}/source_contract_replay_receipt.json" + ), + "source_worker_receipt_path": ( + f"{source_root}/pixelrag_platform_probe_worker_receipts/" + f"{platform}/{manifest_id}/platform_probe_worker_receipt.json" + ), + "source_receipt_path": ( + f"{source_root}/pixelrag_visual_evidence/" + f"{platform}/{manifest_id}/capture_receipt.json" + ), + "embedding_signature_contract": { + "embedding_signature": embedding_signature, + "embedding_model": "bge-m3:latest", + "embedding_dim": 1024, + "embedding_normalize": True, + "embedding_generation_performed": False, + "model_call_performed": False, + }, + "embedding_signature_guard_execution_mode": ( + "deterministic_signature_guard_only_no_embedding_no_network_no_db" + ), + "signature_guard_candidate_count": 2, + "signature_guard_candidate_contracts": [ + _signature_candidate(platform, manifest_id, 0, embedding_signature), + _signature_candidate(platform, manifest_id, 1, embedding_signature), + ], + "required_before_data_promotion": [ + "public_source_boundary", + "rate_limit_contract", + "provenance_contract", + "identity_matcher_replay", + "promotion_gate", + "embedding_signature_guard", + ], + "allowed_next_step": "run_marketplace_candidate_knowledge_replay", + }, + "promotion_boundary": { + "direct_ai_insights_write_allowed": False, + "direct_price_write_allowed": False, + "requires_candidate_knowledge_replay": True, + }, + "writes_database": False, + "writes_database_count": 0, + "network_call_performed": False, + "model_call_performed": False, + "embedding_generation_performed": False, + } + path = receipt_dir / "marketplace_embedding_signature_guard_replay_receipt.json" + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return path + + +def test_marketplace_candidate_knowledge_reads_signature_guard_receipts(tmp_path): + from services.pixelrag_marketplace_candidate_knowledge_replay_service import ( + INTERNAL_RAG_TARGET, + POLICY, + run_pixelrag_marketplace_candidate_knowledge_replay, + ) + from services.rag_service import get_embedding_signature + + signature_root = tmp_path / "signature" + _write_embedding_signature_guard_receipt(signature_root) + _write_embedding_signature_guard_receipt( + signature_root, + platform="shopee_tw", + manifest_id="shopee-runtime-gap", + adapter_code="shopee", + barrier_type="language_or_region_interstitial", + capture_runtime_unavailable=True, + ) + + payload = run_pixelrag_marketplace_candidate_knowledge_replay( + embedding_signature_guard_receipt_root=signature_root, + platform=("coupang_tw", "shopee_tw"), + ) + by_platform = {item["platform"]: item for item in payload["worker_items"]} + + assert payload["policy"] == POLICY + assert payload["status"] == "ok" + assert payload["summary"]["candidate_count"] == 2 + assert payload["summary"]["ready_count"] == 2 + assert payload["summary"]["dry_run_count"] == 2 + assert payload["summary"]["knowledge_candidate_count"] == 4 + assert payload["summary"]["internal_rag_ready_count"] == 4 + assert payload["summary"]["capture_runtime_gap_count"] == 1 + assert payload["summary"]["expected_embedding_signatures"] == [ + get_embedding_signature() + ] + assert payload["summary"]["writes_database_count"] == 0 + assert payload["summary"]["writes_ai_insights"] is False + assert payload["summary"]["writes_price_tables"] is False + assert payload["controlled_apply"]["embedding_generation"] is False + assert payload["controlled_apply"]["writes_ai_insights"] is False + assert payload["promotion_boundary"]["writes_price_tables"] is False + assert payload["next_machine_action"] == ( + "run_marketplace_candidate_knowledge_replay_execute" + ) + assert by_platform["coupang_tw"]["worker_status"] == ( + "dry_run_ready_for_marketplace_candidate_knowledge_replay" + ) + first_knowledge = by_platform["coupang_tw"]["candidate_knowledge_replay"][ + "candidate_knowledge_contracts" + ][0] + assert first_knowledge["embedding_signature"] == get_embedding_signature() + assert first_knowledge["internal_rag_target"] == INTERNAL_RAG_TARGET + assert first_knowledge["ready_for_internal_rag_candidate_replay"] is True + assert first_knowledge["ready_for_ai_insights_write"] is False + assert first_knowledge["ready_for_price_table_write"] is False + assert first_knowledge["price_write_blocked_until_candidate_canary"] is True + assert by_platform["shopee_tw"]["candidate_knowledge_replay"][ + "capture_runtime_unavailable" + ] is True + + +def test_marketplace_candidate_knowledge_execute_writes_receipt(tmp_path): + from services.pixelrag_marketplace_candidate_knowledge_replay_service import ( + run_pixelrag_marketplace_candidate_knowledge_replay, + ) + + signature_root = tmp_path / "signature" + output_root = tmp_path / "knowledge" + _write_embedding_signature_guard_receipt(signature_root) + + payload = run_pixelrag_marketplace_candidate_knowledge_replay( + embedding_signature_guard_receipt_root=signature_root, + output_root=output_root, + platform="coupang_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["executed_count"] == 1 + assert payload["summary"]["receipt_written_count"] == 1 + assert payload["controlled_apply"]["artifact_write"] is True + assert payload["controlled_apply"]["writes_database"] is False + assert payload["next_machine_action"] == "run_internal_rag_candidate_canary" + receipt_path = ( + output_root + / "coupang_tw" + / "coupang-403" + / "marketplace_candidate_knowledge_replay_receipt.json" + ) + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == ( + "executed_marketplace_candidate_knowledge_replay_ready" + ) + assert receipt["artifact_write_performed"] is True + assert receipt["writes_database"] is False + assert receipt["writes_ai_insights"] is False + assert receipt["writes_price_tables"] is False + assert receipt["candidate_knowledge_replay_check_pass_count"] == ( + receipt["candidate_knowledge_replay_check_count"] + ) + assert receipt["candidate_knowledge_replay"]["allowed_next_step"] == ( + "run_internal_rag_candidate_canary" + ) + + +def test_marketplace_candidate_knowledge_cli_outputs_json(tmp_path): + signature_root = tmp_path / "signature" + _write_embedding_signature_guard_receipt(signature_root) + + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py", + "--embedding-signature-guard-receipt-root", + str(signature_root), + "--platform", + "coupang_tw", + ], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0 + payload = json.loads(completed.stdout) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["summary"]["knowledge_candidate_count"] == 2 + assert payload["controlled_apply"]["writes_database"] is False + assert payload["controlled_apply"]["writes_ai_insights"] is False + + +def test_marketplace_candidate_knowledge_route_returns_readback( + tmp_path, + monkeypatch, +): + from flask import Flask + from routes import system_public_routes as routes + from services import ( + pixelrag_marketplace_candidate_knowledge_replay_service as service, + ) + + signature_root = tmp_path / "signature" + _write_embedding_signature_guard_receipt(signature_root) + monkeypatch.setattr( + service, + "DEFAULT_EMBEDDING_SIGNATURE_GUARD_RECEIPT_ROOT", + str(signature_root), + ) + + app = Flask(__name__) + with app.test_request_context( + "/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay?platform=coupang_tw" + ): + response = ( + routes + .ai_automation_pixelrag_marketplace_candidate_knowledge_replay_api + .__wrapped__() + ) + payload = response.get_json() + + assert payload["policy"] == ( + "controlled_pixelrag_marketplace_candidate_knowledge_replay_v1" + ) + assert payload["summary"]["candidate_count"] == 1 + assert payload["summary"]["ready_count"] == 1 + assert payload["controlled_apply"]["writes_database"] is False + assert payload["controlled_apply"]["writes_ai_insights"] is False From 6b8bd8bb05bffc539597387e02488893ef8f2e3e Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 18:49:32 +0800 Subject: [PATCH 29/75] feat(security): automate governance review and access controls --- .gitea/workflows/cd.yaml | 40 +- CONSTITUTION.md | 65 +- app.py | 45 +- auth.py | 62 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 16 +- docs/adr/ADR-012-agent-action-ladder.md | 1 + ...-four-agent-ai-automation-control-plane.md | 1 + ...ity-governance-automation-control-plane.md | 101 +++ docs/adr/README.md | 1 + .../ai_automation_mainline_work_items.md | 107 ++- governance/ewoooc_asset_inventory.json | 365 ++++++++++ .../security_governance_review_baseline.json | 40 ++ routes/alert_routes.py | 26 +- routes/misc_routes.py | 42 +- routes/notification_routes.py | 3 +- routes/openclaw_bot_routes.py | 22 +- routes/system_public_routes.py | 14 + .../ops/report_security_governance_review.py | 57 ++ services/ai_automation_smoke_service.py | 101 +++ services/code_review_pipeline_service.py | 3 +- services/http_access_policy_service.py | 72 ++ .../security_governance_review_service.py | 670 ++++++++++++++++++ tests/test_ai_automation_smoke_service.py | 18 + ...test_security_governance_review_service.py | 164 +++++ utils/security.py | 38 + 26 files changed, 1977 insertions(+), 99 deletions(-) create mode 100644 docs/adr/ADR-038-security-governance-automation-control-plane.md create mode 100644 governance/ewoooc_asset_inventory.json create mode 100644 governance/security_governance_review_baseline.json create mode 100644 scripts/ops/report_security_governance_review.py create mode 100644 services/http_access_policy_service.py create mode 100644 services/security_governance_review_service.py create mode 100644 tests/test_security_governance_review_service.py diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 618d7fc..c8ba063 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -17,6 +17,8 @@ name: CD Pipeline +permissions: read-all + on: push: branches: [main] @@ -32,6 +34,7 @@ on: - 'services/**' - 'routes/**' - 'database/**' + - 'governance/**' - 'templates/**' - 'static/**' - 'monitoring/prometheus.yml' @@ -58,7 +61,7 @@ on: # 新 push 立即取消舊 job，只部署最新版本 concurrency: - group: cd-deploy-${{ github.ref }} + group: cd-deploy-${{ gitea.ref }} cancel-in-progress: true jobs: @@ -67,14 +70,29 @@ jobs: runs-on: ewoooc-host steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 2 + - name: 從目前 Gitea 取回原始碼 + env: + GITEA_TOKEN: ${{ gitea.token }} + GITEA_SERVER_URL: ${{ gitea.server_url }} + GITEA_REPOSITORY: ${{ gitea.repository }} + GITEA_SHA: ${{ gitea.sha }} + run: | + set -eu + AUTH_HEADER=$(printf 'x-access-token:%s' "$GITEA_TOKEN" | base64 | tr -d '\n') + git init . + git remote remove origin 2>/dev/null || true + git remote add origin "${GITEA_SERVER_URL}/${GITEA_REPOSITORY}.git" + git -c http.extraHeader="Authorization: Basic ${AUTH_HEADER}" \ + fetch --no-tags --depth=2 origin "$GITEA_SHA" + git checkout --force --detach FETCH_HEAD + git clean -fdx - name: 取得 Commit 資訊 id: commit + env: + GITEA_SHA: ${{ gitea.sha }} run: | - echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + echo "short_sha=${GITEA_SHA::7}" >> $GITHUB_OUTPUT echo "message=$(git log -1 --pretty=%s | head -c 60)" >> $GITHUB_OUTPUT VERSION=$(sed -n "s/^SYSTEM_VERSION[[:space:]]*=[[:space:]]*[\"']\([^\"']*\)[\"'].*/\1/p" config.py | head -1) echo "version=$VERSION" >> $GITHUB_OUTPUT @@ -84,7 +102,7 @@ jobs: - name: 偵測部署類型 id: deploy_type run: | - if [[ "${{ github.event.inputs.force_rebuild }}" == "true" ]]; then + if [[ "${{ gitea.event.inputs.force_rebuild }}" == "true" ]]; then echo "type=rebuild" >> $GITHUB_OUTPUT echo "label=🔨 強制重建 Docker Image" >> $GITHUB_OUTPUT exit 0 @@ -110,6 +128,10 @@ jobs: echo "ℹ️ AI 觀測台 QA: skipped" fi + - name: 資訊安全與治理 release gate + run: | + python3 scripts/ops/report_security_governance_review.py --compact --strict + # 設定 SSH 金鑰 + 主機驗證（C2 fix: 移除 StrictHostKeyChecking no） - name: 設定 SSH 金鑰 env: @@ -139,7 +161,7 @@ jobs: env: COMMIT_MSG: ${{ steps.commit.outputs.message }} COMMIT_SHA: ${{ steps.commit.outputs.short_sha }} - COMMIT_ACTOR: ${{ github.actor }} + COMMIT_ACTOR: ${{ gitea.actor }} DEPLOY_LABEL: ${{ steps.deploy_type.outputs.label }} TG_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} TG_CHAT: ${{ secrets.TELEGRAM_CHAT_ID }} @@ -400,8 +422,8 @@ jobs: continue-on-error: true env: WEBHOOK_TOKEN: ${{ secrets.INTERNAL_WEBHOOK_TOKEN }} - COMMIT_SHA_FULL: ${{ github.sha }} - BRANCH_NAME: ${{ github.ref_name }} + COMMIT_SHA_FULL: ${{ gitea.sha }} + BRANCH_NAME: ${{ gitea.ref_name }} DEPLOY_TYPE: ${{ steps.deploy_type.outputs.type }} run: | CHANGED=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || echo "") diff --git a/CONSTITUTION.md b/CONSTITUTION.md index 55c3c72..fcb30ff 100644 --- a/CONSTITUTION.md +++ b/CONSTITUTION.md @@ -2,8 +2,9 @@ > 本文件定義專案開發的核心準則與不可違反的規範 > **建立日期**: 2026-01-12 -> **當前版本**: V10.64 (Keep only latest 50 AI product picks pending) -> **最後更新**: 2026-05-01 +> **當前版本**: V10.771 (Security governance automation review) +> **最後更新**: 2026-07-10 +> **治理基準**: `global_product_governance_v2` + ADR-038 --- @@ -155,10 +156,11 @@ - ❌ **禁止**: 使用 5888 或其他端口（已廢棄） - **配置位置**: `config.py` 的 `PUBLIC_URL` -### 第 16 條：資料庫路徑（絕對禁止違反） -- ✅ **正確**: `data/momo_database.db` -- ❌ **禁止**: 更改資料庫位置或名稱 -- **配置位置**: `config.py` 的 `DATABASE_PATH` +### 第 16 條：資料庫 runtime（絕對禁止違反） +- ✅ **正式環境**: PostgreSQL + pgvector，runtime identity 為 `momo-db`；以 production SQL receipt 與 migration readback 為真相。 +- ✅ **本機開發**: 可使用 `data/momo_database.db` 作為相容 fallback，但不得覆蓋 production truth。 +- ❌ **禁止**: 自動 restart / stop / recreate `momo-db`，或把 SQLite snapshot 誤報為正式資料庫完成證據。 +- **配置位置**: `config.py` 的 `DATABASE_PATH` / `DATABASE_TYPE` ### 第 17 條：時區設定（絕對禁止違反） - ✅ **正確**: 使用 `TAIPEI_TZ = pytz.timezone('Asia/Taipei')` @@ -195,14 +197,14 @@ - ❌ **禁止**: 修改功能但不更新版本號 ### 第 20 條：備份系統（強制要求） -- ✅ **正確**: 重大更新前必須執行 `python backup_system.py` -- ✅ **排除目錄**: `['backups', '__pycache__', '.git', '.idea', '.vscode', 'bin', 'bin 2']` -- ❌ **禁止**: 跳過備份直接上線 +- ✅ **正確**: 高風險資料變更前必須取得 fresh backup、checksum、offsite coverage 與可追溯 receipt。 +- ✅ **正確**: 備份完成不等於可恢復；必須定期在隔離環境執行 non-destructive restore drill 並量測 RPO/RTO。 +- ❌ **禁止**: 未驗證 restore receipt 就宣稱備援閉環完成。 ### 第 21 條：TODO 文件維護（強制要求） -- ✅ **正確**: 完成功能後更新 `TODO_NEXT_STEPS.txt` 標記 ✅ -- ✅ **正確**: 記錄修改的檔案與行數 -- ❌ **禁止**: 完成任務但不更新文件 +- ✅ **正確**: 工作優先序與狀態以 `docs/guides/ai_automation_mainline_work_items.md` 及機器可讀 governance review 為準。 +- ✅ **正確**: 完成狀態必須分開記錄 source/test、Gitea CD、production runtime、visible evidence。 +- ❌ **禁止**: 只更新 TODO、文件或欄位就把工作標為 completed。 --- @@ -283,9 +285,10 @@ ## 第十章：安全性規範 ### 第 31 條：敏感資訊保護（絕對禁止違反） -- ✅ **正確**: 所有 API Token、密碼存放於 `config.py` -- ❌ **禁止**: 硬編碼敏感資訊於程式碼中 -- ❌ **禁止**: 提交 `config.py` 至公開 Git 倉庫 +- ✅ **正確**: 程式碼與 `config.py` 只保存環境變數名稱、驗證與 fail-secure 邏輯；secret value 只存在受控 runtime secret store / Gitea Secret。 +- ✅ **正確**: webhook secret、API token、password、private key 必須使用 constant-time comparison、最小權限、輪替與失效機制。 +- ❌ **禁止**: 讀取、記錄、同步或提交明文 secret、raw session、authorization header、runtime secret volume。 +- ❌ **禁止**: 使用不安全預設值、query-string token 或 production login bypass。 ### 第 32 條：SQL 注入防護（絕對禁止違反） - ✅ **正確**: 使用 SQLAlchemy ORM 的參數化查詢 @@ -304,21 +307,30 @@ - ✅ **正確**: JavaScript 中使用 `textContent` 而非 `innerHTML` - ❌ **禁止**: 直接插入未驗證的使用者輸入 +### 第 33.1 條：存取控制與 Session（絕對禁止違反） +- ✅ **正確**: 除明確 public allowlist 外，業務資料、operator、admin、logs、import、CI/CD、crawler 與 notification surfaces 一律 deny-by-default。 +- ✅ **正確**: production cookie 必須 `Secure`、`HttpOnly`、適當 `SameSite`；敏感回應須 `no-store` 並具安全 response headers。 +- ✅ **正確**: internal/webhook endpoint 必須使用 signed transport 或 fail-secure token，且具 replay/duplicate guard。 +- ❌ **禁止**: 以 CSRF exemption 取代 authentication/authorization，或讓 shared password 自動等同永久 admin identity。 + +### 第 33.2 條：Software Supply Chain（絕對禁止違反） +- ✅ **正確**: Gitea CD 僅從目前 Gitea、受控內部 registry/mirror 或 repo-owned immutable artifact 取回 source/action/image。 +- ✅ **正確**: release gate 必須逐步具備 exact dependency lock、SAST、SCA、secret scan、SBOM、image digest、provenance 與部署版本 readback。 +- ❌ **禁止**: `uses: actions/*` 或任何 GitHub/raw/codeload/ghcr runtime dependency。 + --- ## 第十一章：部署與維運規範 ### 第 34 條：伺服器啟動（強制要求） -- ✅ **正確**: 使用 `nohup python3 app.py > /dev/null 2>&1 &` 背景執行 -- ✅ **正確**: 部署前檢查 Port 80 是否已被佔用 -- ❌ **禁止**: 直接執行 `python3 app.py`（關閉終端機會停止服務） +- ✅ **正確**: 正式服務由 188 Docker Compose + Gunicorn 管理，部署入口為 Gitea CD。 +- ✅ **正確**: image rebuild 必須 build 成功後才切換，並保留 internal/external `/health` readback。 +- ❌ **禁止**: 以裸 Python process、背景 shell 或未受控 SSH 指令作為正式部署方式。 ### 第 35 條：伺服器重啟（強制要求） -- ✅ **正確步驟**: - 1. `pkill -9 -f "python3 app.py"` (停止舊程序) - 2. `sleep 2` (等待端口釋放) - 3. `nohup python3 app.py > /dev/null 2>&1 &` (啟動新程序) - 4. `ps aux | grep "[p]ython3 app.py"` (確認運行) +- ✅ **正確**: sync 模式對 `momo-pro-system` 使用 Gunicorn HUP，scheduler/telegram-bot 才做 bounded restart。 +- ✅ **正確**: rebuild 只可 recreate `momo-app scheduler telegram-bot`，並以 post-verifier 確認版本與健康。 +- ❌ **禁止**: `docker compose --remove-orphans`、跨專案容器操作或任何 `momo-db` 生命週期操作。 ### 第 36 條：日誌輪替（建議執行） - ✅ **建議**: 定期清理過大的日誌檔案 (> 100MB) @@ -351,13 +363,14 @@ - **Hermes（採集層）**: Ollama 三主機級聯（GCP-A → GCP-B → 111），負責 embedding、去重、品質分數計算。成本 = $0 - **NemoTron（處理層）**: qwen3:14b Ollama-first，NVIDIA NIM Llama 3.1 8B 僅作備援，負責 tool calling 邏輯路由與 DB 寫入。NIM 限額 80 次/天 - **OpenClaw（應用層）**: Ollama-first；Gemini 僅作備援或 ADR-028 鎖定場景，負責最終 PPT 生成、洞察報告對外輸出。 -- **ElephantAlpha（編排層）**: 負責跨 Agent orchestration、HITL、AutoHeal bridge 與受控執行計畫，不可繞過安全入口 +- **ElephantAlpha（編排層）**: 負責跨 Agent orchestration、risk/policy decision、AutoHeal bridge 與受控執行計畫；low/medium/high 預設 controlled apply，critical 才走 break-glass。 - ❌ **禁止**：讓 OpenClaw 做 Hermes 層的苦力工作（高算力浪費） - ❌ **禁止**：讓 Hermes 直接生成對外報告（品質不足） - ❌ **禁止**：讓 ElephantAlpha 直接繞過 EventRouter / AutoHeal / ADR-011 執行高風險副作用 ### 第 41 條：AI 學習數據雙寫（絕對禁止違反） -- ✅ **正確**：所有 AI 產出（PPT 洞察、競品分析、對話記錄、Agent action、自癒紀錄）必須**雙寫** PostgreSQL `ai_insights` + pgvector embedding +- ✅ **正確**：通過 PromotionGate 且允許成為正式知識的 AI 產出，必須**雙寫** PostgreSQL `ai_insights` + pgvector embedding，並留下 durable acknowledgement。 +- ✅ **正確**：preview、blocked page、PixelRAG visual evidence、matcher candidate 與失敗 receipt 必須保持 evidence-only，不得為了追求雙寫而污染正式知識或價格表。 - ❌ **禁止**：只寫 DB 不寫 KM（RAG 無法語意搜尋） - ❌ **禁止**：只寫 KM 不寫 DB（精確 period/sku 查詢無法命中） - **理由**：DB 是精準命中，KM 是語意搜尋，兩者互補缺一不可（ADR-007） @@ -388,6 +401,8 @@ - ✅ **正確**：AutoHeal 是自癒副作用入口，失敗時必須安全降級為 alert / log / file queue - ✅ **正確**：L2 safe action 必須可審計、可回放、低副作用 - ✅ **正確**：AI 自動化觀測變更需同步 `/metrics`、Smoke dashboard 與 Grafana provisioning，避免告警閉環變成黑盒 +- ✅ **正確**：每個 side-effect run 必須以同一 `trace_id/run_id/work_item_id` 串接 sensor、identity、SOT diff、decision、risk、dry-run、bounded execution、independent verifier、rollback/retry 與 KM/RAG/PlayBook write acknowledgement。 +- ✅ **正確**：完成度必須同時公布 program、asset coverage、runtime closure；任一 closure 段缺失只能標示 partial/degraded。 - ❌ **禁止**：自動 restart / stop / recreate `momo-db` 或 `momo-postgres` - ❌ **禁止**：AI 分析失敗導致 Telegram 告警完全不送出 - **依據**：ADR-012、ADR-013、ADR-018 diff --git a/app.py b/app.py index 0c80c38..af48aae 100644 --- a/app.py +++ b/app.py @@ -170,9 +170,13 @@ app.config['SESSION_COOKIE_HTTPONLY'] = True # 防止 JavaScript 存取 cookie app.config['SESSION_COOKIE_SAMESITE'] = 'Lax' # 防止 CSRF 攻擊 app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=24) # Session 有效期 24 小時（延長避免長時間閒置斷線） -# 如果使用 HTTPS，啟用 SECURE cookie（本地開發時應設為 False） -# 注意：如果您的系統部署在 HTTPS 環境，請將 .env 中的 USE_HTTPS 設為 true -USE_HTTPS = os.getenv('USE_HTTPS', 'false').lower() == 'true' +# 正式入口為 HTTPS 時 secure cookie 預設開啟；本機 HTTP 開發可明確設 USE_HTTPS=false。 +_use_https_env = os.getenv('USE_HTTPS') +USE_HTTPS = ( + _use_https_env.lower() == 'true' + if _use_https_env is not None + else os.getenv('PUBLIC_URL', 'https://mo.wooo.work').lower().startswith('https://') +) if USE_HTTPS: app.config['SESSION_COOKIE_SECURE'] = True sys_log.info("[Security] ✅ HTTPS 模式已啟用，Session cookie 僅透過 HTTPS 傳輸") @@ -185,8 +189,8 @@ else: app.config['MAX_CONTENT_LENGTH'] = 100 * 1024 * 1024 sys_log.info("[Security] ✅ Flask 安全配置已載入") -sys_log.info(f"[Security] • Session 有效期: 2 小時") -sys_log.info(f"[Security] • 檔案上傳限制: 10 MB") +sys_log.info(f"[Security] • Session 有效期: 24 小時") +sys_log.info(f"[Security] • 檔案上傳限制: 100 MB") sys_log.info(f"[Security] • CSRF 防護: SameSite=Lax") sys_log.info(f"[Security] • XSS 防護: HttpOnly=True") @@ -636,6 +640,37 @@ def get_dashboard_stats(): # ================= 🛣️ 4. Flask 路由 ================= +# 敏感業務/維運端點由中央 policy 統一 fail-closed。 +from services.http_access_policy_service import enforce_http_access_policy # noqa: E402 +app.before_request(enforce_http_access_policy) + + +@app.after_request +def add_security_response_headers(response): + """全域瀏覽器安全基線；CSP 先以 report-only 收斂既有 inline 資產。""" + response.headers.setdefault('X-Content-Type-Options', 'nosniff') + response.headers.setdefault('X-Frame-Options', 'SAMEORIGIN') + response.headers.setdefault('Referrer-Policy', 'strict-origin-when-cross-origin') + response.headers.setdefault( + 'Permissions-Policy', + 'camera=(), microphone=(), geolocation=(), payment=()', + ) + response.headers.setdefault( + 'Strict-Transport-Security', + 'max-age=31536000; includeSubDomains', + ) + response.headers.setdefault( + 'Content-Security-Policy-Report-Only', + "default-src 'self'; object-src 'none'; base-uri 'self'; frame-ancestors 'self'; " + "img-src 'self' data: https:; font-src 'self' data: https:; " + "style-src 'self' 'unsafe-inline' https:; script-src 'self' 'unsafe-inline' https:; " + "connect-src 'self' https: wss:", + ) + if session.get('logged_in'): + response.headers.setdefault('Cache-Control', 'private, no-store') + return response + + # Session 自動續期機制 @app.before_request def refresh_session(): diff --git a/auth.py b/auth.py index 4fc2c5b..88d4c6f 100644 --- a/auth.py +++ b/auth.py @@ -1,6 +1,7 @@ +import hmac import os import time -from flask import render_template, redirect, url_for, request, session, flash +from flask import jsonify, render_template, redirect, url_for, request, session, flash from functools import wraps from werkzeug.security import check_password_hash from config import LOGIN_PASSWORD @@ -9,11 +10,17 @@ from datetime import datetime, timedelta # ========================================== # 🔓 登入功能開關 # ========================================== -# 設定環境變數 DISABLE_LOGIN=true 可關閉登入驗證 -DISABLE_LOGIN = os.getenv('DISABLE_LOGIN', 'false').lower() == 'true' +# 登入繞過只允許測試環境明確開啟，避免 production 因單一 env 誤設而全站失守。 +_DISABLE_LOGIN_REQUESTED = os.getenv('DISABLE_LOGIN', 'false').lower() == 'true' +_ALLOW_INSECURE_LOGIN_BYPASS = ( + os.getenv('MOMO_ALLOW_INSECURE_CONFIG_FOR_TESTS', 'false').lower() == 'true' +) +DISABLE_LOGIN = _DISABLE_LOGIN_REQUESTED and _ALLOW_INSECURE_LOGIN_BYPASS if DISABLE_LOGIN: print("⚠️ 警告：登入驗證已關閉 (DISABLE_LOGIN=true)") +elif _DISABLE_LOGIN_REQUESTED: + print("[Security] 已拒絕 DISABLE_LOGIN：僅測試環境可使用登入繞過") # ========================================== # 🔒 登入失敗追蹤與帳號鎖定機制 @@ -164,6 +171,41 @@ def login_required(f): return decorated_view +def require_authenticated_request(): + """供中央 HTTP policy 使用；API 回 401，頁面導向登入。""" + if DISABLE_LOGIN or session.get('logged_in'): + return None + + is_api_request = request.path.startswith('/api/') or '/api/' in request.path + if is_api_request: + return jsonify({ + 'success': False, + 'error': 'authentication_required', + 'message': '此端點需要登入。', + }), 401 + return redirect(url_for('login')) + + +def internal_key_required(f): + """Require the shared internal transport key for non-session service calls.""" + @wraps(f) + def decorated_view(*args, **kwargs): + expected = os.getenv('INTERNAL_API_KEY', '').strip() + provided = request.headers.get('X-Internal-Key', '').strip() + if not expected: + return jsonify({ + 'success': False, + 'error': 'internal_auth_not_configured', + }), 503 + if not provided or not hmac.compare_digest(provided, expected): + return jsonify({ + 'success': False, + 'error': 'invalid_internal_key', + }), 401 + return f(*args, **kwargs) + return decorated_view + + def get_current_user(): """ 取得目前登入的使用者資訊 @@ -176,10 +218,12 @@ def get_current_user(): return { 'logged_in': True, + 'user_id': session.get('user_id'), 'login_time': session.get('login_time'), 'client_ip': session.get('client_ip'), 'role': session.get('role', 'admin'), # 預設為 admin（向後兼容） - 'username': session.get('username', 'admin') + 'username': session.get('username', 'admin'), + 'auth_source': session.get('auth_source', 'legacy_shared_password'), } @@ -297,17 +341,21 @@ def init_auth_routes(app): is_password_valid = check_password_hash(LOGIN_PASSWORD, input_password) else: # 向後兼容：明文比對（僅用於過渡期） - is_password_valid = (input_password == LOGIN_PASSWORD) + is_password_valid = hmac.compare_digest(input_password, LOGIN_PASSWORD) if is_password_valid: print("⚠️ 警告：系統仍在使用明文密碼，請盡快執行密碼雜湊更新") # 4. 驗證結果處理 if is_password_valid: # 登入成功 + session.clear() session.permanent = True session['logged_in'] = True session['login_time'] = datetime.now().isoformat() session['client_ip'] = client_ip + session['role'] = 'admin' + session['username'] = 'legacy-admin' + session['auth_source'] = 'legacy_shared_password' # 清除失敗記錄 clear_login_attempts(client_ip) @@ -345,9 +393,7 @@ def init_auth_routes(app): 登出路由：清除 session 並導回登入頁。 """ client_ip = get_client_ip() - session.pop('logged_in', None) - session.pop('login_time', None) - session.pop('client_ip', None) + session.clear() print(f"👋 使用者已登出 | IP: {client_ip}") return redirect(url_for('login')) diff --git a/config.py b/config.py index 3fa492f..3e632f2 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.770" +SYSTEM_VERSION = "V10.771" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 9d2b355..865ed8a 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,8 +1,20 @@ # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth > **最後更新**: 2026-07-10 (台北時間) -> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.770 +> **狀態**: 🟠 Partial。四 Agent、PixelRAG、MCP/RAG registry、PChome controlled-preview families 與多項 smoke/readback 已建立；但 access control、database identity/RBAC、full asset reconciliation、software supply chain、same-run controlled apply、restore drill、internal RAG canary 與 MCP/RAG runtime closure 尚未全部完成。任何局部 receipt 不得代表整體閉環完成。 +> **適用版本**: V10.771 + +--- + +## 零之負一、資訊安全與治理自動化基準（2026-07-10） + +- ADR-038 採 NIST CSF 2.0 `GOVERN / IDENTIFY / PROTECT / DETECT / RESPOND / RECOVER` 作為 program 結構，OWASP ASVS 5.0.0 L2 作為應用基準，NIST SSDF SP 800-218 v1.1 作為供應鏈基準。 +- `governance/ewoooc_asset_inventory.json` 是 full-asset seed；欄位完整不等於 runtime reconciliation 完成。 +- `services/http_access_policy_service.py` 統一保護 vendor、import、CI/CD、crawler、category、logs 與 operator surfaces；public exception 必須有明確 transport auth 或 public allowlist。 +- `services/security_governance_review_service.py`、`scripts/ops/report_security_governance_review.py` 與 `/api/ai-automation/security-governance-review` 提供 machine-readable checks、completion、release gate 與 ordered work items。 +- Gitea CD 必須執行 `report_security_governance_review.py --strict`，且不得解析 GitHub action/source/image。 +- production 安全與治理完成度必須分開顯示 program、asset coverage、runtime closure；目前結論是 partial，不是 complete。 +- 目前 ordered P0 以 `docs/guides/ai_automation_mainline_work_items.md` 為準：先關閉 access exposure，再做 identity/RBAC、webhook trust、supply chain、asset reconciliation、controlled-apply envelope、internal RAG canary 與 MCP/RAG runtime closure。 --- diff --git a/docs/adr/ADR-012-agent-action-ladder.md b/docs/adr/ADR-012-agent-action-ladder.md index e1a0cdb..ff04b33 100644 --- a/docs/adr/ADR-012-agent-action-ladder.md +++ b/docs/adr/ADR-012-agent-action-ladder.md @@ -5,6 +5,7 @@ - **Deciders**: 統帥 - **Related**: ADR-001（三 Agent 分工）, ADR-004（NemoTron Fallback）, ADR-007（AI Dual-Write）, ADR-011（跨專案隔離）, ADR-018（四 AI Agent 自動化控制面） - **Note**: [ADR-020](ADR-020-code-review-full-autoheal.md) 局部覆寫本 ADR 對「post-deploy code review pipeline」場景的 L3 HITL 規定 — 該場景改採全自動修復 + Git/CI/CD 回滾安全網。本 ADR 對 schema migration / 流量切換 / customer-facing 廣播 / AIOps prod SSH 等其他 L3 場景仍生效。 +- **2026-07-10 Sunset**: [ADR-038](ADR-038-security-governance-automation-control-plane.md) 覆寫 generic owner/manual/HITL terminal gate。low/medium/high 改採 risk-based controlled apply；critical break-glass 類型仍保留外部授權。 ## Context diff --git a/docs/adr/ADR-018-four-agent-ai-automation-control-plane.md b/docs/adr/ADR-018-four-agent-ai-automation-control-plane.md index 18f7a7e..0128e70 100644 --- a/docs/adr/ADR-018-four-agent-ai-automation-control-plane.md +++ b/docs/adr/ADR-018-four-agent-ai-automation-control-plane.md @@ -4,6 +4,7 @@ - **Date**: 2026-04-29 - **Deciders**: 統帥 - **Related**: ADR-001, ADR-004, ADR-007, ADR-009, ADR-011, ADR-012, ADR-013 +- **2026-07-10 Update**: ADR-038 將 ElephantAlpha 的 generic HITL 改為 risk-based controlled apply；只有 critical break-glass 類型維持外部授權。 ## Context diff --git a/docs/adr/ADR-038-security-governance-automation-control-plane.md b/docs/adr/ADR-038-security-governance-automation-control-plane.md new file mode 100644 index 0000000..aef89d4 --- /dev/null +++ b/docs/adr/ADR-038-security-governance-automation-control-plane.md @@ -0,0 +1,101 @@ +# ADR-038: Security Governance Automation Control Plane + +- **Status**: Accepted +- **Date**: 2026-07-10 +- **Decider**: Product owner +- **Supersedes**: generic owner/manual/HITL terminal gates in ADR-012 and ADR-018 for low/medium/high risk work +- **Related**: ADR-011, ADR-012, ADR-013, ADR-018, ADR-020, ADR-031, ADR-032, ADR-033 + +## Context + +EwoooC has many AI automation receipts, smoke families and guarded preview stages, but the program still cannot prove a complete information-security and governance loop. Production review found four structural gaps: + +1. Sensitive routes evolved blueprint by blueprint instead of behind one deny-by-default policy. +2. Asset, identity, dependency, backup and AI runtime evidence are not reconciled by one control plane. +3. Preview/review stages do not share one `trace_id/run_id/work_item_id` through execution, verifier, rollback and learning. +4. Source/test/UI evidence has repeatedly been reported ahead of production runtime closure. + +## Decision + +### 1. Control model + +Use NIST CSF 2.0 as the program structure: + +- GOVERN: policy, ownership, risk, supply chain and measurable targets. +- IDENTIFY: canonical asset graph, dependency inventory and risk assessment. +- PROTECT: identity, authorization, secure defaults, data and platform controls. +- DETECT: fresh logs, metrics, traces, anomaly signals and control drift. +- RESPOND: correlated incident, bounded action, verifier and communication receipts. +- RECOVER: rollback/retry, backup/restore, RPO/RTO and learning writeback. + +OWASP ASVS 5.0.0 L2 is the application baseline, with selected L3 controls for operator, webhook, AI execution and admin surfaces. NIST SSDF SP 800-218 v1.1 is the software-supply-chain baseline. + +### 2. Source of truth + +Completion evidence is ordered: + +1. production runtime receipt, independent post-verifier and durable DB receipt; +2. Gitea main/deploy marker/CD run/production readback; +3. committed machine-readable inventory, policy and tests; +4. docs and snapshots; +5. conversation. + +Lower evidence cannot promote a higher incomplete layer to completed. + +### 3. Controlled apply + +Low, medium and high risk work defaults to controlled apply with blast-radius controls. Only critical break-glass classes remain externally gated: plaintext secrets, destructive database/storage operations, host/network cutovers, credentialed offensive scans, paid provider or core runtime replacement without replay/shadow/canary, destructive Git operations and raw secret-volume access. + +Every side-effect run must retain one `trace_id/run_id/work_item_id` across: + +`sensor -> normalize -> correlate -> decide -> risk/check -> bounded execute -> independent verify -> retry/rollback -> learn/writeback` + +Any missing stage produces `partial` or `degraded`, never completed. + +### 4. Machine-readable controls + +- `governance/ewoooc_asset_inventory.json` is the canonical asset seed and must be reconciled against runtime receipts. +- `services/http_access_policy_service.py` is the central policy for sensitive HTTP surfaces. +- `services/security_governance_review_service.py` produces NIST/OWASP-aligned findings, completion scores and ordered work items. +- `scripts/ops/report_security_governance_review.py --strict` is the source release gate. +- `/api/ai-automation/security-governance-review` is the protected production readback. + +### 5. Legacy policy sunset + +| Legacy policy | Owner | Expiry | Replacement | Verifier | +|---|---|---|---|---| +| Generic HITL/manual terminal for low/medium/high | product-governance | 2026-07-10 | risk-based controlled apply in ADR-038 | governance review work-item state | +| Shared password treated as complete RBAC | security | 2026-07-31 | database identity + least-privilege role migration | anonymous/role route matrix | +| `actions/checkout@v4` in Gitea CD | delivery | 2026-07-10 | Gitea job-token source checkout | Gitea-only supply-chain gate | +| Backup script treated as recovery completion | resilience | 2026-07-31 | checksum/offsite/isolated restore receipt | RPO/RTO restore verifier | +| Registry or preview readiness treated as runtime completion | AI platform | 2026-07-17 | MCP/RAG live health and internal RAG canary | production smoke/readback | + +## Alternatives Considered + +- Keep adding domain-specific receipt stages: rejected because it increases surface area without unifying assurance. +- Make every action human-approved: rejected for low/medium/high because it defeats the product's AI automation goal. +- Make every action fully autonomous without independent verification: rejected because it cannot bound production risk. +- Adopt an external governance platform as the source of truth: rejected until internal asset and receipt contracts are stable and exportable. + +## Consequences + +Positive: + +- Security and AI automation use the same measurable closure contract. +- Drift becomes a work item instead of an undocumented manual observation. +- Release gating can block critical source regressions without pretending all runtime debt is complete. +- Product, asset and runtime completion remain separate and honest. + +Costs and risks: + +- Existing routes and automation lanes require gradual migration. +- Shared identity, webhook secret activation, non-root containers and restore drills need canary rollout. +- The initial asset inventory has complete required fields but runtime reconciliation remains partial. + +## Verification + +1. `python3 scripts/ops/report_security_governance_review.py --strict` returns exit 0. +2. Anonymous requests to sensitive read/operator surfaces return 401 or login redirect. +3. Gitea workflow contains no external `uses:` action or forbidden GitHub source. +4. Production endpoint reports zero release-blocking source failures. +5. Program, asset and runtime closure percentages are reported separately. diff --git a/docs/adr/README.md b/docs/adr/README.md index e9f9337..9ae74f2 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -59,6 +59,7 @@ | [035](ADR-035-cross-platform-market-campaign-intelligence.md) | 跨平台市場活動情報系統 | Accepted | 2026-05-06 | | [036](ADR-036-fastapi-strangler-not-frontend-prerequisite.md) | FastAPI Strangler Migration，不作為前端 V3 前置條件 | Accepted | 2026-05-12 | | [037](ADR-037-webcrumbs-shared-ui-runtime.md) | Webcrumbs 共用 UI Runtime 接入 | Accepted | 2026-05-31 | +| [038](ADR-038-security-governance-automation-control-plane.md) | 資訊安全與治理自動化控制面（NIST CSF / OWASP ASVS / SSDF） | Accepted | 2026-07-10 | ## 規範 diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 4ed8ca6..31a4ef4 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -1,50 +1,85 @@ # AI Automation Mainline Work Items > Updated: 2026-07-10 -> Scope: EwoooC / MOMO Pro System AI automation, PixelRAG, MCP/RAG, production UX. +> Governance: `global_product_governance_v2` + ADR-038 +> Current P0: `SEC-P0-001 deny-by-default access control` ## Source Of Truth -- Production readback stays authoritative after deploy: `/health`, container health, AI automation API readback, and smoke. -- Gitea is the only source-control truth. GitHub remains frozen and must not be used. -- Runtime truth must stay separate from source truth: a green test or registry does not mean MCP/RAG runtime is enabled. -- PixelRAG visual receipts are evidence, not formal price truth. They cannot write `competitor_prices`, `competitor_price_history`, or `ai_insights` without replay and promotion gates. +- Production runtime receipt/post-verifier/durable DB receipt is authoritative. +- Gitea main, deploy marker, CD run and production readback are the only source/deploy truth; GitHub remains frozen. +- Source/test/UI/CD green does not mean runtime closure. +- Completion must report program, asset coverage and runtime closure separately. +- PixelRAG visual evidence cannot write formal prices or `ai_insights` before identity, PromotionGate and internal RAG canary proof. -## P0 +## Ordered P0 -| Status | Work item | Evidence / next machine action | -|---|---|---| -| Completed | Multi-commerce PixelRAG visual evidence lane for momo, pchome, shopee_tw, coupang_tw, yahoo_shopping_tw, etmall_tw, friday_tw, rakuten_tw | API and CLI can emit expansion plans and marketplace manifests; PChome parser failures can queue manifests. | -| Completed | External MCP/RAG capability inventory absorbed into internal governance readback | `/api/ai-automation/external-mcp-rag-integration` and `scripts/ops/report_external_mcp_rag_integration.py` expose 9 capabilities, absorbed/unresolved counts, and runtime flags. | -| Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read visual capture receipts and platform probe worker receipts, split eligible / blocked / source-contract-fallback candidates, and expose combined OCR/VLM + source-contract next actions before identity matching and PromotionGate knowledge writes. | -| Completed | PixelRAG source-contract replay worker | `/api/ai-automation/pixelrag-source-contract-replay-worker` and `scripts/ops/run_pixelrag_source_contract_replay_worker.py` turn platform probe worker `source_contract_fallback` receipts into replayable source-contract artifact receipts with provenance, public-boundary, no-DB, and blocked-page-not-product-data guards. | -| Completed | PixelRAG marketplace source-contract adapter preflight | `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight` and `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py` validate source-contract replay receipts for adapter readiness, canonical offer fields, provenance/rate-limit/public-boundary requirements, and no-write promotion gates. | -| Completed | PixelRAG marketplace adapter dry-run | `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run` and `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py` turn adapter preflight receipts into deterministic no-write field contracts and offer candidate contracts, with price writes blocked until identity matcher replay and PromotionGate. | -| Completed | PixelRAG marketplace identity matcher replay | `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay` and `scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py` turn adapter dry-run receipts into deterministic identity candidate contracts and source provenance checks, with price writes blocked until PromotionGate replay. | -| Completed | PixelRAG marketplace PromotionGate replay | `/api/ai-automation/pixelrag-marketplace-promotion-gate-replay` and `scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py` turn identity matcher receipts into deterministic PromotionGate contracts and embedding-signature readiness, with price writes still blocked. | -| Completed | PixelRAG marketplace embedding signature guard replay | `/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay` and `scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py` turn PromotionGate receipts into deterministic embedding-signature guard contracts and candidate-knowledge readiness, with DB and price writes still blocked. | -| Completed | PixelRAG marketplace candidate knowledge replay | `/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay` and `scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py` turn embedding-signature guard receipts into deterministic internal RAG candidate preview contracts, with DB, `ai_insights`, and price writes still blocked until canary. | -| Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | -| Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | -| Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | -| Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture when runtime is available, missing Playwright auto-falls back to structured-source/backoff receipt, and Coupang 403/access denied goes to structured-source/backoff artifact receipts. | -| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, PixelRAG platform probe worker, source-contract replay, adapter preflight, adapter dry-run, identity matcher replay, PromotionGate replay, embedding signature guard replay, and candidate knowledge replay families. | -| In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | +| Order | ID | Status | Work item | Exit evidence / next machine action | +|---:|---|---|---|---| +| 1 | `SEC-P0-001` | In progress | Deny-by-default route access control | Central policy now protects vendor/import/CI-CD/crawler/category/log/operator and sales-detail surfaces; every anonymous GET is explicitly allowlisted. Exit: anonymous production matrix passes and `/metrics` moves behind edge/internal auth. Next: `verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth`. | +| 2 | `SEC-P0-002` | Not started | Database identity + least-privilege RBAC | Replace shared admin password, default-admin fallback, process-local lockout and spoofable proxy identity. Exit: shadow auth, role matrix, durable audit/lockout, session revocation and canary cutover. | +| 3 | `SEC-P0-003` | In progress | Webhook trust and replay protection | Telegram secret-token verification code exists; production secret activation remains unproven. Exit: secret provisioned outside source, required mode enabled, invalid-secret 401 and authorized callback canary pass. | +| 4 | `SUPPLY-P0-001` | In progress | Gitea-only secure software supply chain | Gitea-native checkout and governance gate added. Exit: exact dependency lock, internal SAST/SCA/secret scan, SBOM, image digest/provenance, vulnerability SLA and production digest readback. | +| 5 | `GOV-P0-001` | In progress | Canonical full asset graph + runtime reconciliation | `governance/ewoooc_asset_inventory.json` seeds hosts, services, data, AI, routes, supply chain, observability and recovery. Exit: same-run probe receipt for every asset; drift auto-creates work items. | +| 6 | `GOV-P0-002` | Not started | Unified controlled-apply envelope | Introduce one `trace_id/run_id/work_item_id` across sensor, identity, SOT diff, decision, risk, dry-run, execution, verifier, rollback/retry and learning acknowledgement. Start with EventRouter + AutoHeal. | +| 7 | `RAG-P0-001` | Not started | Internal RAG candidate canary | V10.770 stops at candidate preview. Exit: bounded pgvector candidate canary, signature/readback/feedback receipt, no price write and no premature `ai_insights` promotion. Next: `run_internal_rag_candidate_canary`. | +| 8 | `MCP-P0-001` | In progress | MCP/RAG production runtime closure | Registry and smoke wiring exist, but runtime flags/ports must be live. Exit: MCP servers healthy, router enabled, RAG enabled, approved caller/tool boundary and production query canary pass. | +| 9 | `SEC-P0-004` | Not started | Security operations lifecycle and metrics | Add durable security incident state and publish MTTA, MTTR, recurrence, false positive, human intervention, verifier pass, rollback and freshness. Exit: detect-to-learn production receipt. | +| 10 | `REL-P0-001` | In progress | Formal deploy and visible proof discipline | Every functional change bumps version, passes Gitea CD, deploys only three app containers, leaves `momo-db` untouched, and reads back `/health`, new APIs and visible UI/security behavior. | ## P1 -| Status | Work item | Evidence / next machine action | +| Order | ID | Status | Work item | Exit evidence / next machine action | +|---:|---|---|---|---| +| 11 | `RES-P1-001` | Not started | Backup/offsite/restore automation | Fresh checksum and offsite coverage plus isolated non-destructive restore drill with measured RPO/RTO. | +| 12 | `PLAT-P1-001` | Not started | Non-root container and capability hardening | Shadow non-root image, writable-path inventory, capability drop/read-only filesystem canary, three-app rollout. | +| 13 | `APPSEC-P1-001` | In progress | CSP and DOM/XSS hardening | Security headers are present and CSP is report-only. Collect violations, remove high-risk `innerHTML`/inline sinks, then enforce CSP by canary. | +| 14 | `APPSEC-P1-002` | Not started | Unsafe shared-cache serialization removal | Replace writable pickle caches in dashboard/daily-sales/EDM/sales with constrained JSON or signed schema. | +| 15 | `ARCH-P1-001` | In progress | Split oversized policy/executor/verifier modules | Current top debts include 44k-line PChome mapping and 14k-line smoke service. Split by bounded family and independent tests. | +| 16 | `UX-P1-001` | In progress | Professional full-site UI/UX | Complete first-viewport cockpit, progressive disclosure, text-density reduction, Traditional Chinese product copy, accessibility, loading/error/empty/degraded states and desktop/mobile visual smoke. | +| 17 | `PIXELRAG-P1-001` | Not started | Ollama-first multimodal embedding benchmark | Verify approved visual embedding on GCP-A -> GCP-B -> 111 and design pgvector-compatible visual metadata; FAISS remains disallowed without ADR. | +| 18 | `MARKET-P1-001` | In progress | Marketplace source contracts | Finish stable, public-boundary, provenance and rate-limit contracts for Shopee, Coupang, Yahoo, ETMall, Friday and Rakuten; blocked pages remain non-product data. | +| 19 | `QA-P1-001` | In progress | Deterministic test and CI governance | Current baseline is 1,701 passed / 30 legacy failures / 9 skipped plus one missing report module at collection. Restore or retire the missing contract, remove wall-clock/copy drift, and split deterministic release tests from nightly integration and production canaries. | + +## P2 + +| Order | ID | Status | Work item | Exit evidence / next machine action | +|---:|---|---|---|---| +| 20 | `GOV-P2-001` | Not started | Continuous NIST/ASVS control trend | Persist governance snapshots, compare control/asset/runtime drift and create ordered work items automatically. | +| 21 | `DATA-P2-001` | Not started | Data classification and retention enforcement | Classify business, personal, operational and model data; automate retention, deletion eligibility and audit evidence without destructive default actions. | +| 22 | `CHAOS-P2-001` | Not started | Controlled resilience exercises | Run non-destructive model-host, MCP, queue, Telegram and app-container failure drills with rollback and learning receipts. | + +## Completed Foundations + +These are reusable foundations, not proof that the full program is complete. + +| Status | Capability | Current boundary | |---|---|---| -| Completed | OCR/VLM replay contract for visual fields | `/api/ai-automation/pixelrag-ocr-vlm-replay` and `scripts/ops/report_pixelrag_ocr_vlm_replay.py` turn saved tiles into no-write field contracts, output schemas, validation rules, and Ollama-first worker actions. | -| Completed | Ollama-first OCR/VLM extraction worker | `run_pixelrag_vlm_replay_worker.py --execute --write-receipt` executes the replay contract against approved Ollama multimodal routes and emits confidence/evidence artifact receipts without writing formal price truth. | -| Not started | Ollama-first multimodal embedding benchmark | Verify local Qwen3-VL or equivalent visual embedding on GCP-A -> GCP-B -> 111 before any visual vector retrieval. | -| Not started | pgvector-compatible visual evidence metadata | Design metadata-first retrieval without FAISS in production unless ADR approves a different store. | -| In progress | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; `run_pixelrag_platform_probe_worker.py --execute --write-receipt` now emits structured-source/backoff receipts without network, DB writes, or product-data promotion. | -| In progress | Marketplace source contracts beyond MOMO/PChome | Source-contract replay receipts now enter marketplace adapter preflight as artifact receipts; next machine action is adapter dry-run for stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten contracts after provenance, rate-limit, public boundary, and replay evidence. | -| In progress | Professional product website UI/UX text reduction | Continue compact AI surfaces, visual QA, and user-facing copy guardrails; avoid engineering logs or work-session prose in the UI. | +| Completed | Multi-commerce PixelRAG visual evidence for momo, pchome, shopee_tw, coupang_tw, yahoo_shopping_tw, etmall_tw, friday_tw, rakuten_tw | Evidence-only; blocked pages are not product data. | +| Completed | External MCP/RAG capability inventory | Registry/integration readback exists; runtime enablement remains P0. | +| Completed | PixelRAG receipt -> RAG candidate replay | Candidate-only; no formal knowledge or price write. | +| Completed | Source-contract replay worker | Public-boundary artifact receipts only. | +| Completed | Marketplace adapter preflight and dry-run | Deterministic no-write contracts. | +| Completed | Marketplace identity matcher replay | Candidate identity only. | +| Completed | PromotionGate replay | No production write. | +| Completed | Embedding-signature guard replay | Signature readiness only. | +| Completed | Candidate knowledge replay | Internal RAG preview only; canary remains P0. | +| Completed | PixelRAG application portfolio | Commerce/RAG/UX/ops/marketing/governance inventory. | +| Completed | Ollama-first VLM route readiness and replay worker | Evidence-bound artifact output; no direct price write. | +| Completed | Platform probe worker | Shopee/Coupang barriers become structured fallback/backoff receipts. | -## Always Enforced +## Definition Of Done -- Manual review is exception-only for low-risk, machine-verifiable controlled apply. -- Break-glass still applies to secrets, destructive DB actions, force pushes, production provider switchovers, raw runtime volumes, and unverified core model replacement. -- AI automation smoke must expose health honestly: `MCP_ROUTER_ENABLED=false` or `RAG_ENABLED=false` is a runtime gap, not a source-code failure. +A work item can be `Completed` only when the same production run contains: + +1. sensor/source receipt; +2. normalized canonical asset identity; +3. source-of-truth diff; +4. AI decision and candidate action; +5. risk/policy decision; +6. check-mode/dry-run receipt; +7. idempotent bounded execution receipt; +8. independent post-verifier and rollback/no-write terminal; +9. incident/Telegram/KM/RAG/MCP/PlayBook durable acknowledgement. + +Missing any stage means `partial`, `degraded` or `blocked_with_safe_next_action`. diff --git a/governance/ewoooc_asset_inventory.json b/governance/ewoooc_asset_inventory.json new file mode 100644 index 0000000..2b7c5cc --- /dev/null +++ b/governance/ewoooc_asset_inventory.json @@ -0,0 +1,365 @@ +{ + "schema_version": "1.0", + "product_id": "ewoooc", + "governance_profile": "global_product_governance_v2", + "updated_at": "2026-07-10T18:30:00+08:00", + "required_fields": [ + "canonical_id", + "asset_type", + "owner_lane", + "source_truth", + "runtime_identity", + "signal_freshness", + "policy", + "executor", + "verifier", + "backup_restore", + "learning_targets" + ], + "assets": [ + { + "canonical_id": "product:ewoooc", + "asset_type": "product", + "owner_lane": "product-governance", + "source_truth": "Gitea main plus production health and protected product surfaces", + "runtime_identity": "https://mo.wooo.work", + "signal_freshness": {"source": "commit", "runtime_target": "/health", "max_age_minutes": 15}, + "policy": ["global_product_governance_v2", "CONSTITUTION.md"], + "executor": "Gitea CD on ewoooc-host", + "verifier": "production version truth plus HTTP and UI smoke", + "backup_restore": "source in Gitea; runtime data handled by database and artifact lanes", + "learning_targets": ["security governance review", "AI automation smoke", "KM/PlayBook"] + }, + { + "canonical_id": "repo:gitea:wooo/ewoooc", + "asset_type": "source_repository", + "owner_lane": "source-control", + "source_truth": "ssh://git@192.168.0.110:2222/wooo/ewoooc.git", + "runtime_identity": "Gitea repository wooo/ewoooc", + "signal_freshness": {"source": "git ls-remote", "runtime_target": "main/dev heads", "max_age_minutes": 30}, + "policy": ["GitHub freeze", "protected main", "Gitea-only actions"], + "executor": "git over Gitea SSH or Gitea job token", + "verifier": "remote head readback and deploy marker", + "backup_restore": "Gitea repository backup and offsite checksum lane", + "learning_targets": ["source deploy runtime truth"] + }, + { + "canonical_id": "pipeline:gitea-actions:ewoooc-cd", + "asset_type": "deployment_pipeline", + "owner_lane": "delivery", + "source_truth": ".gitea/workflows/cd.yaml", + "runtime_identity": "runner label ewoooc-host", + "signal_freshness": {"source": "workflow commit", "runtime_target": "latest Gitea Action run", "max_age_minutes": 60}, + "policy": ["Gitea-only", "momo-db protected", "three app containers only"], + "executor": "Gitea act runner", + "verifier": "CD health, version, smoke and post-deploy review", + "backup_restore": "rollback by known-good Gitea commit and image", + "learning_targets": ["CD failure classification", "deployment playbook"] + }, + { + "canonical_id": "host:lan:110", + "asset_type": "host_gateway", + "owner_lane": "platform", + "source_truth": "host and service runtime probes", + "runtime_identity": "192.168.0.110", + "signal_freshness": {"source": "SSH probe", "runtime_target": "gateway/Gitea/Nginx", "max_age_minutes": 15}, + "policy": ["gateway and Gitea only", "cross-project isolation"], + "executor": "bounded SSH via approved operator lane", + "verifier": "independent service and public route probes", + "backup_restore": "host configuration backup and Gitea dump lane", + "learning_targets": ["host health", "routing incidents"] + }, + { + "canonical_id": "host:lan:188", + "asset_type": "production_host", + "owner_lane": "runtime", + "source_truth": "Docker runtime receipts on 192.168.0.188", + "runtime_identity": "192.168.0.188", + "signal_freshness": {"source": "SSH/Docker probe", "runtime_target": "EwoooC containers", "max_age_minutes": 10}, + "policy": ["not an Ollama node", "momo-db protected", "cross-project isolation"], + "executor": "Gitea CD bounded compose actions", + "verifier": "container health plus external /health", + "backup_restore": "database backup and application rollback lanes", + "learning_targets": ["runtime incidents", "capacity history"] + }, + { + "canonical_id": "host:gcp:ollama-a", + "asset_type": "model_host", + "owner_lane": "ai-runtime", + "source_truth": "Ollama host health receipts", + "runtime_identity": "34.87.90.216:11434", + "signal_freshness": {"source": "approved model probe", "runtime_target": "/api/tags", "max_age_minutes": 20}, + "policy": ["Ollama primary", "no secret payload logging"], + "executor": "services.ollama_service", + "verifier": "host health and model availability readback", + "backup_restore": "fallback to GCP-B then 111", + "learning_targets": ["model latency", "fallback cause"] + }, + { + "canonical_id": "host:gcp:ollama-b", + "asset_type": "model_host", + "owner_lane": "ai-runtime", + "source_truth": "Ollama host health receipts", + "runtime_identity": "34.21.145.224:11434", + "signal_freshness": {"source": "approved model probe", "runtime_target": "/api/tags", "max_age_minutes": 20}, + "policy": ["Ollama secondary", "embedding fallback"], + "executor": "services.ollama_service", + "verifier": "host health and model availability readback", + "backup_restore": "fallback to 111 for approved non-batch traffic", + "learning_targets": ["model latency", "fallback cause"] + }, + { + "canonical_id": "host:lan:ollama-111", + "asset_type": "model_host", + "owner_lane": "ai-runtime", + "source_truth": "Ollama host health receipts", + "runtime_identity": "192.168.0.111:11434", + "signal_freshness": {"source": "approved model probe", "runtime_target": "/api/tags", "max_age_minutes": 20}, + "policy": ["third fallback", "no background embedding batches"], + "executor": "services.ollama_service", + "verifier": "host health and route policy readback", + "backup_restore": "no further model host fallback", + "learning_targets": ["fallback exhaustion"] + }, + { + "canonical_id": "network:public:mo.wooo.work", + "asset_type": "dns_tls_route", + "owner_lane": "edge", + "source_truth": "public DNS/TLS/HTTP readback", + "runtime_identity": "https://mo.wooo.work", + "signal_freshness": {"source": "Blackbox", "runtime_target": "/health", "max_age_minutes": 5}, + "policy": ["TLS only for user traffic", "health endpoint is lightweight"], + "executor": "Nginx on 110", + "verifier": "external TLS and HTTP probe", + "backup_restore": "versioned Nginx configuration and rollback", + "learning_targets": ["TLS expiry", "routing drift"] + }, + { + "canonical_id": "service:ewoooc:momo-app", + "asset_type": "application_service", + "owner_lane": "runtime", + "source_truth": "Docker inspect plus /health", + "runtime_identity": "momo-pro-system", + "signal_freshness": {"source": "Docker health", "runtime_target": "container port 80", "max_age_minutes": 5}, + "policy": ["Gunicorn gthread", "HUP sync reload", "no momo-db lifecycle action"], + "executor": "Gitea CD momo-app lane", + "verifier": "internal and external /health plus version truth", + "backup_restore": "known-good image and Gitea commit rollback", + "learning_targets": ["HTTP errors", "worker latency", "security events"] + }, + { + "canonical_id": "service:ewoooc:scheduler", + "asset_type": "scheduler_service", + "owner_lane": "automation-runtime", + "source_truth": "Docker health and scheduler receipts", + "runtime_identity": "momo-scheduler", + "signal_freshness": {"source": "job receipts", "runtime_target": "scheduler process", "max_age_minutes": 15}, + "policy": ["EventRouter", "bounded retry", "no momo-db restart"], + "executor": "run_scheduler.py", + "verifier": "scheduled health summary and job receipts", + "backup_restore": "restart scheduler only; replay durable queues", + "learning_targets": ["job failure recurrence", "MTTR"] + }, + { + "canonical_id": "service:ewoooc:telegram-bot", + "asset_type": "notification_service", + "owner_lane": "automation-runtime", + "source_truth": "Telegram API receipt plus Docker health", + "runtime_identity": "momo-telegram-bot", + "signal_freshness": {"source": "delivery receipt", "runtime_target": "bot process", "max_age_minutes": 15}, + "policy": ["authorized users/groups", "webhook secret target", "queue replay"], + "executor": "run_telegram_bot.py and OpenClaw webhook", + "verifier": "delivery receipt and callback readback", + "backup_restore": "file queue replay and bot restart only", + "learning_targets": ["delivery failure", "unauthorized attempt"] + }, + { + "canonical_id": "database:ewoooc:postgresql", + "asset_type": "database", + "owner_lane": "data", + "source_truth": "PostgreSQL durable receipts and migrations", + "runtime_identity": "momo-db", + "signal_freshness": {"source": "SQL health", "runtime_target": "PostgreSQL", "max_age_minutes": 5}, + "policy": ["protected resource", "pgvector primary", "no destructive automatic action"], + "executor": "SQLAlchemy and idempotent migrations", + "verifier": "independent readback queries and schema checks", + "backup_restore": "scheduled backup, checksum, offsite copy and restore drill", + "learning_targets": ["query health", "migration drift", "restore evidence"] + }, + { + "canonical_id": "data:ewoooc:pchome-sales", + "asset_type": "business_dataset", + "owner_lane": "sales-data", + "source_truth": "PostgreSQL sales tables plus import receipts", + "runtime_identity": "daily and monthly PChome sales records", + "signal_freshness": {"source": "import job receipt", "runtime_target": "latest business date", "max_age_minutes": 180}, + "policy": ["real data only", "Taipei time", "no mock values"], + "executor": "Google Drive/import services", + "verifier": "row count, business date and duplicate checks", + "backup_restore": "database backup and source file replay", + "learning_targets": ["import errors", "freshness drift"] + }, + { + "canonical_id": "data:ewoooc:market-intelligence", + "asset_type": "business_dataset", + "owner_lane": "market-intelligence", + "source_truth": "source receipts, external_offers and guarded candidate artifacts", + "runtime_identity": "market intel tables and receipt artifacts", + "signal_freshness": {"source": "source receipt", "runtime_target": "offer freshness", "max_age_minutes": 1440}, + "policy": ["provenance required", "PromotionGate", "no visual-only price write"], + "executor": "market intel controlled apply lanes", + "verifier": "identity matcher, PromotionGate and post-write readback", + "backup_restore": "database backup and replayable evidence artifacts", + "learning_targets": ["matcher quality", "false positive", "source barrier"] + }, + { + "canonical_id": "integration:ewoooc:google-drive-import", + "asset_type": "external_integration", + "owner_lane": "sales-data", + "source_truth": "Google Drive auth and import receipts", + "runtime_identity": "auto_import service", + "signal_freshness": {"source": "auth readiness", "runtime_target": "latest import job", "max_age_minutes": 180}, + "policy": ["authenticated operator surface", "no credential exposure"], + "executor": "services.google_drive_service", + "verifier": "job status and database readback", + "backup_restore": "replay source file without duplicate write", + "learning_targets": ["auth expiry", "import parser failure"] + }, + { + "canonical_id": "integration:ewoooc:telegram-webhook", + "asset_type": "webhook", + "owner_lane": "notification-security", + "source_truth": "Telegram webhook configuration and request receipts", + "runtime_identity": "/bot/telegram/webhook", + "signal_freshness": {"source": "webhook info", "runtime_target": "secret verification", "max_age_minutes": 60}, + "policy": ["secret token required target", "authorized user and group checks"], + "executor": "OpenClaw Telegram webhook", + "verifier": "invalid-secret rejection plus authorized callback canary", + "backup_restore": "restore prior webhook URL and replay no pending updates", + "learning_targets": ["auth failures", "duplicate updates"] + }, + { + "canonical_id": "ai:ewoooc:ollama-router", + "asset_type": "ai_runtime", + "owner_lane": "ai-platform", + "source_truth": "Ollama route and call receipts", + "runtime_identity": "GCP-A -> GCP-B -> 111", + "signal_freshness": {"source": "host probe", "runtime_target": "approved model route", "max_age_minutes": 20}, + "policy": ["Ollama-first", "Gemini hard-disabled default", "188 forbidden"], + "executor": "services.ollama_service", + "verifier": "model route readback and fallback receipt", + "backup_restore": "bounded fallback chain", + "learning_targets": ["latency", "model error", "fallback rate"] + }, + { + "canonical_id": "ai:ewoooc:mcp-router", + "asset_type": "mcp_control_plane", + "owner_lane": "ai-platform", + "source_truth": "MCP registry plus live server health", + "runtime_identity": "services.mcp_router and ports 3001-3004", + "signal_freshness": {"source": "MCP smoke", "runtime_target": "tool server health", "max_age_minutes": 15}, + "policy": ["known caller/tool allowlist", "Postgres/filesystem read-only"], + "executor": "services.mcp_router", + "verifier": "live health, call receipt and boundary tests", + "backup_restore": "disable router and fall back to approved static/Ollama path", + "learning_targets": ["tool success", "cache hit", "policy rejection"] + }, + { + "canonical_id": "ai:ewoooc:rag-service", + "asset_type": "rag_control_plane", + "owner_lane": "ai-platform", + "source_truth": "pgvector records and RAG query receipts", + "runtime_identity": "services.rag_service", + "signal_freshness": {"source": "RAG smoke", "runtime_target": "query/feedback logs", "max_age_minutes": 30}, + "policy": ["pgvector only", "BGE-M3 signature", "PromotionGate"], + "executor": "services.rag_service", + "verifier": "candidate canary, signature and retrieval quality checks", + "backup_restore": "disable retrieval and preserve structured DB truth", + "learning_targets": ["hit rate", "feedback", "recurrence"] + }, + { + "canonical_id": "ai:ewoooc:pixelrag", + "asset_type": "visual_evidence_pipeline", + "owner_lane": "market-intelligence", + "source_truth": "screenshot, tile and replay receipts", + "runtime_identity": "PixelRAG artifact lanes for 8 marketplaces", + "signal_freshness": {"source": "artifact receipt", "runtime_target": "capture/replay readback", "max_age_minutes": 1440}, + "policy": ["public evidence only", "blocked page is not product data", "no direct price write"], + "executor": "PixelRAG capture and Ollama VLM workers", + "verifier": "identity, PromotionGate, embedding signature and RAG canary", + "backup_restore": "immutable artifact receipts and no-write terminal", + "learning_targets": ["capture quality", "VLM confidence", "platform barrier"] + }, + { + "canonical_id": "observability:ewoooc:security-runtime", + "asset_type": "logs_metrics_traces_alerts", + "owner_lane": "observability", + "source_truth": "Prometheus, Grafana, logs, alerts and Telegram receipts", + "runtime_identity": "/metrics plus AI automation smoke", + "signal_freshness": {"source": "scrape and alert receipt", "runtime_target": "security/governance families", "max_age_minutes": 15}, + "policy": ["no public operational logs", "lifecycle receipts", "freshness SLO"], + "executor": "Prometheus/Blackbox/EventRouter", + "verifier": "alert-chain and dashboard readback", + "backup_restore": "metrics retention and durable incident receipts", + "learning_targets": ["MTTA", "MTTR", "false positive", "rollback rate"] + }, + { + "canonical_id": "backup:ewoooc:database-artifacts", + "asset_type": "backup_restore", + "owner_lane": "resilience", + "source_truth": "backup checksum and restore drill receipts", + "runtime_identity": "database and artifact backup lanes", + "signal_freshness": {"source": "backup receipt", "runtime_target": "latest successful restore drill", "max_age_minutes": 10080}, + "policy": ["checksum required", "offsite required", "restore drill required"], + "executor": "backup services and controlled restore workflow", + "verifier": "independent checksum plus isolated restore query", + "backup_restore": "this asset owns the recovery evidence itself", + "learning_targets": ["RPO", "RTO", "restore failure"] + }, + { + "canonical_id": "supply-chain:ewoooc:python-container", + "asset_type": "packages_images_sbom", + "owner_lane": "software-supply-chain", + "source_truth": "requirements, image digest, SBOM and vulnerability receipts", + "runtime_identity": "EwoooC Python 3.11 container image", + "signal_freshness": {"source": "CD security gate", "runtime_target": "deployed image digest", "max_age_minutes": 1440}, + "policy": ["Gitea/internal sources", "exact dependency lock target", "no GitHub action dependency"], + "executor": "Gitea CD build lane", + "verifier": "SAST, SCA, secret scan, SBOM and provenance checks", + "backup_restore": "known-good immutable image digest", + "learning_targets": ["CVE age", "dependency drift", "build provenance"] + }, + { + "canonical_id": "quality:ewoooc:test-control-plane", + "asset_type": "test_and_release_evidence", + "owner_lane": "quality-governance", + "source_truth": "pytest collection, deterministic release suite and runtime canary receipts", + "runtime_identity": "Gitea release gate plus scheduled nightly and production smoke lanes", + "signal_freshness": {"source": "governance/security_governance_review_baseline.json", "runtime_target": "latest Gitea and production verifier receipts", "max_age_minutes": 1440}, + "policy": ["release tests deterministic", "nightly integration separated", "production canary read-only first"], + "executor": "pytest and Gitea Actions", + "verifier": "collection gate, focused security suites and post-deploy smoke", + "backup_restore": "Gitea test source plus immutable test result receipt", + "learning_targets": ["flake rate", "failure age", "collection drift", "runtime escape rate"] + }, + { + "canonical_id": "surface:ewoooc:primary-ui-api", + "asset_type": "product_surface", + "owner_lane": "product-experience", + "source_truth": "Flask route map, templates and production visual readback", + "runtime_identity": "primary desktop and mobile user workflows", + "signal_freshness": {"source": "sitewide visual QA", "runtime_target": "desktop/mobile screenshots", "max_age_minutes": 1440}, + "policy": ["Traditional Chinese", "progressive disclosure", "authenticated operator surfaces", "no text walls"], + "executor": "Flask/Jinja frontend", + "verifier": "route, accessibility, responsive and visual smoke", + "backup_restore": "Gitea commit and frontend asset rollback", + "learning_targets": ["task completion", "error state", "text density"] + } + ], + "reconciliation": { + "status": "partial", + "source_inventory_complete": true, + "runtime_receipts_complete": false, + "next_machine_action": "probe_all_assets_and_write_same-run_reconciliation_receipt" + } +} diff --git a/governance/security_governance_review_baseline.json b/governance/security_governance_review_baseline.json new file mode 100644 index 0000000..79940ce --- /dev/null +++ b/governance/security_governance_review_baseline.json @@ -0,0 +1,40 @@ +{ + "schema_version": "1.0", + "product_id": "ewoooc", + "captured_at": "2026-07-10T20:40:00+08:00", + "source_base_sha": "c62ec7b504c7f74ce8a6a3b5312e30c1e6947513", + "review_branch": "codex/security-governance-review", + "collection": { + "status": "failed", + "collected_count": 1740, + "error_count": 1, + "missing_contract": "scripts/ops/report_pchome_mapping_backlog.py", + "consumer": "tests/test_pchome_mapping_backlog_report.py" + }, + "focused_suites": { + "ai_automation_smoke": {"passed": 46, "failed": 0}, + "security_and_governance": {"passed": 23, "failed": 0}, + "frontend_contract": {"passed": 29, "failed": 0} + }, + "full_suite_without_missing_contract": { + "status": "partial", + "command": ".venv/bin/python -m pytest -q --ignore=tests/test_pchome_mapping_backlog_report.py", + "passed": 1701, + "failed": 30, + "skipped": 9, + "duration_seconds": 361.66, + "failure_groups": [ + "legacy manual/HITL assertions conflict with current AI-controlled product copy", + "wall-clock dependent import fixture expired", + "modularization line-count snapshot drift", + "missing or stale source/test contract assertions", + "pre-existing service behavior and fixture mismatches outside this review diff" + ] + }, + "review_change_regression_evidence": { + "focused_security_suites_pass": true, + "failed_full_suite_targets_are_outside_security_review_source_diff": true, + "claim_full_suite_green": false + }, + "next_machine_action": "restore_or_retire_missing_pchome_report_contract_then_baseline_legacy_failures" +} diff --git a/routes/alert_routes.py b/routes/alert_routes.py index 265b625..baab0ba 100644 --- a/routes/alert_routes.py +++ b/routes/alert_routes.py @@ -6,6 +6,7 @@ 並執行自動排查和修復 """ +import hmac import os import json import subprocess @@ -78,6 +79,18 @@ ALLOWED_ISSUE_TYPES = frozenset({ 'cpu', 'container_cpu', }) +ALLOWED_AUTO_FIX_CONTAINERS = frozenset({ + 'momo-pro-system', + 'momo-scheduler', + 'momo-telegram-bot', +}) +PROTECTED_CONTAINERS = frozenset({ + 'momo-db', + 'momo-postgres', + 'momo-prometheus', + 'momo-grafana', +}) + def check_alert_auth(f): """告警 Webhook 認證裝飾器（fail-secure：密碼未設定時拒絕所有請求）""" @@ -89,7 +102,10 @@ def check_alert_auth(f): auth = request.authorization if not auth: return jsonify({'error': 'Missing authentication'}), 401 - if auth.username != ALERT_WEBHOOK_USER or auth.password != ALERT_WEBHOOK_PASSWORD: + if ( + not hmac.compare_digest(auth.username or '', ALERT_WEBHOOK_USER) + or not hmac.compare_digest(auth.password or '', ALERT_WEBHOOK_PASSWORD) + ): return jsonify({'error': 'Invalid credentials'}), 401 return f(*args, **kwargs) return decorated @@ -270,10 +286,12 @@ def auto_fix_container(container_name: str, issue_type: str) -> dict: try: # 排除關鍵容器 - protected_containers = ['momo-postgres', 'momo-prometheus', 'momo-grafana'] - if container_name in protected_containers: + if container_name in PROTECTED_CONTAINERS: result['message'] = "此為受保護的關鍵容器，不自動重啟" return result + if container_name not in ALLOWED_AUTO_FIX_CONTAINERS: + result['message'] = "容器不在 EwoooC 自動修復白名單" + return result # 根據問題類型決定修復動作 if issue_type in ['container_memory', 'memory']: @@ -454,6 +472,7 @@ def manual_fix(): @alert_bp.route('/api/alert/test', methods=['POST']) +@check_alert_auth def test_alert(): """測試告警通知""" message = """ @@ -473,6 +492,7 @@ def test_alert(): @alert_bp.route('/api/alert/status') +@check_alert_auth def alert_status(): """取得告警系統狀態""" return jsonify({ diff --git a/routes/misc_routes.py b/routes/misc_routes.py index 383c14c..cdce697 100644 --- a/routes/misc_routes.py +++ b/routes/misc_routes.py @@ -6,11 +6,18 @@ """ import requests from flask import Blueprint, jsonify, render_template, request +from urllib.parse import urljoin + +from auth import login_required +from services.logger_manager import SystemLogger +from utils.security import validate_public_http_url misc_bp = Blueprint('misc', __name__) +sys_log = SystemLogger('MiscRoutes').get_logger() @misc_bp.route('/api/test_url', methods=['POST']) +@login_required def test_url(): """API: 測試網址是否有效""" try: @@ -22,16 +29,39 @@ def test_url(): headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" } - # 設定 10 秒超時，避免卡住 - response = requests.get(url, headers=headers, timeout=10) + current_url = validate_public_http_url(url) + response = None + for _ in range(4): + response = requests.get( + current_url, + headers=headers, + timeout=(3.05, 10), + allow_redirects=False, + stream=True, + ) + if response.status_code not in {301, 302, 303, 307, 308}: + break + location = response.headers.get('Location') + response.close() + if not location: + break + current_url = validate_public_http_url(urljoin(current_url, location)) - if response.status_code == 200: + if response is None: + raise ValueError('無法取得網址回應') + status_code = response.status_code + response.close() + + if status_code == 200: return jsonify({"status": "success", "message": f"✅ 連結有效 (Status: 200)"}) else: - return jsonify({"status": "warning", "message": f"⚠️ 連結回應異常 (Status: {response.status_code})"}) + return jsonify({"status": "warning", "message": f"⚠️ 連結回應異常 (Status: {status_code})"}) - except Exception as e: - return jsonify({"status": "error", "message": f"❌ 連線失敗: {str(e)}"}), 500 + except ValueError as exc: + return jsonify({"status": "error", "message": str(exc)}), 400 + except Exception as exc: + sys_log.warning(f"[Misc] [URL_CHECK] 連線失敗 | Error: {exc}") + return jsonify({"status": "error", "message": "連線失敗，請確認網址後再試。"}), 502 @misc_bp.route('/brand_assets') diff --git a/routes/notification_routes.py b/routes/notification_routes.py index c3baa9b..cf03e95 100644 --- a/routes/notification_routes.py +++ b/routes/notification_routes.py @@ -6,7 +6,7 @@ """ from flask import Blueprint, request, jsonify, render_template -from auth import login_required +from auth import internal_key_required, login_required from services.notification_service import NotificationTemplateService from services.logger_manager import SystemLogger from config import SYSTEM_VERSION @@ -87,6 +87,7 @@ def preview_template(code): @notification_bp.route('/api/notification/render', methods=['POST']) +@internal_key_required def render_for_n8n(): """ API: 為 n8n 渲染模板 diff --git a/routes/openclaw_bot_routes.py b/routes/openclaw_bot_routes.py index 2dc7dc9..372bc33 100644 --- a/routes/openclaw_bot_routes.py +++ b/routes/openclaw_bot_routes.py @@ -24,6 +24,7 @@ from __future__ import annotations import os import json +import hmac import re import threading import hashlib # Operation Ollama-First v5.0 P1: H6 PII fix — chat_id 進 meta 改 hash[:8] @@ -126,6 +127,11 @@ BOT_TOKEN = os.getenv('OPENCLAW_BOT_TOKEN') or os.getenv('TELEGRAM_BOT_TOK BOT_API_URL = f"https://api.telegram.org/bot{BOT_TOKEN}" ALLOWED_GROUP = int(os.getenv('OPENCLAW_GROUP_ID', '-1003940688311')) MOMO_BASE_URL = os.getenv('MOMO_BASE_URL', 'https://mo.wooo.work') +TELEGRAM_WEBHOOK_SECRET_TOKEN = os.getenv('TELEGRAM_WEBHOOK_SECRET_TOKEN', '').strip() +TELEGRAM_WEBHOOK_SECRET_REQUIRED = ( + os.getenv('TELEGRAM_WEBHOOK_SECRET_REQUIRED', 'false').strip().lower() + in {'1', 'true', 'yes', 'on'} +) NVIDIA_API_KEY = os.getenv('NVIDIA_API_KEY', '') NVIDIA_BASE_URL = 'https://integrate.api.nvidia.com/v1' CHAT_MODEL = 'deepseek-ai/deepseek-v3.2' @@ -8904,6 +8910,15 @@ def _identify_product_name_with_gemini_vision(img_b64: str, request_id: str) -> @openclaw_bot_bp.route('/bot/telegram/webhook', methods=['POST']) def telegram_webhook(): try: + provided_secret = request.headers.get('X-Telegram-Bot-Api-Secret-Token', '') + if TELEGRAM_WEBHOOK_SECRET_TOKEN: + if not hmac.compare_digest(provided_secret, TELEGRAM_WEBHOOK_SECRET_TOKEN): + sys_log.warning('[OpenClawBot] Telegram webhook secret 驗證失敗') + return jsonify({'ok': False, 'error': 'unauthorized'}), 401 + elif TELEGRAM_WEBHOOK_SECRET_REQUIRED: + sys_log.error('[OpenClawBot] Telegram webhook secret 必填但未設定') + return jsonify({'ok': False, 'error': 'webhook_secret_not_configured'}), 503 + # 每個 webhook request 先清空 ContextVar，避免同 worker thread 延用上一個 user。 _CURRENT_USER_ID_CTX.set(None) @@ -9398,11 +9413,14 @@ def telegram_webhook(): @openclaw_bot_bp.route('/bot/telegram/set_webhook', methods=['POST']) def set_webhook(): webhook_url = f"{MOMO_BASE_URL}/bot/telegram/webhook" - result = _tg('setWebhook', { + payload = { 'url': webhook_url, 'allowed_updates': ['message', 'callback_query'], 'drop_pending_updates': True, - }) + } + if TELEGRAM_WEBHOOK_SECRET_TOKEN: + payload['secret_token'] = TELEGRAM_WEBHOOK_SECRET_TOKEN + result = _tg('setWebhook', payload) register_commands() sys_log.info(f"[OpenClawBot] setWebhook → {result}") return jsonify({'ok': result.get('ok'), 'webhook_url': webhook_url}) diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 7f1ab0e..402b2b9 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -1177,6 +1177,20 @@ def ai_automation_external_mcp_rag_integration_api(): )) +@system_public_bp.route('/api/ai-automation/security-governance-review') +@login_required +def ai_automation_security_governance_review_api(): + """Read-only NIST/OWASP-aligned security and governance review.""" + from services.security_governance_review_service import ( + build_security_governance_review, + ) + + detail_limit = request.args.get('detail_limit', 30, type=int) or 30 + return jsonify(build_security_governance_review( + detail_limit=max(5, min(detail_limit, 100)), + )) + + @system_public_bp.route('/api/ai-automation/smoke/history/export') @login_required def ai_automation_smoke_history_export(): diff --git a/scripts/ops/report_security_governance_review.py b/scripts/ops/report_security_governance_review.py new file mode 100644 index 0000000..fbe097a --- /dev/null +++ b/scripts/ops/report_security_governance_review.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Report the machine-readable EwoooC security/governance review.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.security_governance_review_service import ( # noqa: E402 + build_security_governance_review, +) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--detail-limit", type=int, default=30) + parser.add_argument("--compact", action="store_true") + parser.add_argument( + "--strict", + action="store_true", + help="Exit non-zero only for release-blocking source governance failures.", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + report = build_security_governance_review( + root=args.root, + detail_limit=args.detail_limit, + ) + if args.compact: + output = { + "policy": report["policy"], + "status": report["status"], + "completion": report["completion"], + "release_gate": report["release_gate"], + "next_machine_action": report["next_machine_action"], + } + else: + output = report + print(json.dumps(output, ensure_ascii=False, indent=2, sort_keys=True)) + if args.strict and report["release_gate"]["status"] != "pass": + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index 1f9ec2f..992b4ea 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -679,6 +679,14 @@ def build_scheduled_automation_health_summary( if not pixelrag_platform_probe_worker or not pixelrag_platform_probe_worker_details: pixelrag_platform_probe_worker = _pixelrag_platform_probe_worker_check() pixelrag_platform_probe_worker_details = pixelrag_platform_probe_worker.get("details") or {} + security_governance = _find_check( + source_result, + "Security governance automation review", + ) + security_governance_details = security_governance.get("details") or {} + if not security_governance or not security_governance_details: + security_governance = _security_governance_review_check() + security_governance_details = security_governance.get("details") or {} smoke_status = source_result.get("status") or ("warning" if latest_history else "warning") freshness_family = _history_freshness_family( latest_history, @@ -707,6 +715,43 @@ def build_scheduled_automation_health_summary( "writes_database": False, }, }, + { + "key": "security_governance_review", + "label": "Security governance automation review", + "status": security_governance.get("status") or "warning", + "summary": ( + security_governance.get("summary") + or "Security governance review has no latest readback." + ), + "next_machine_action": security_governance_details.get( + "next_machine_action" + ) or "refresh_security_governance_review", + "details": { + "policy": security_governance_details.get("policy"), + "review_status": security_governance_details.get("review_status"), + "program_percent": security_governance_details.get("program_percent"), + "asset_field_coverage_percent": security_governance_details.get( + "asset_field_coverage_percent" + ), + "asset_runtime_reconciliation_complete": bool( + security_governance_details.get( + "asset_runtime_reconciliation_complete" + ) + ), + "runtime_closure_passed": int( + security_governance_details.get("runtime_closure_passed") or 0 + ), + "runtime_closure_total": int( + security_governance_details.get("runtime_closure_total") or 0 + ), + "blocking_failure_count": int( + security_governance_details.get("blocking_failure_count") or 0 + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "key": "pchome_controlled_apply_drift_monitor", "label": "PChome controlled apply drift monitor", @@ -5420,6 +5465,7 @@ def build_scheduled_automation_health_summary( "scheduled_outputs": { "health_summary_endpoint": "/api/ai-automation/scheduled-health-summary", "smoke_endpoint": "/api/ai-automation/smoke", + "security_governance_endpoint": "/api/ai-automation/security-governance-review", "daily_summary_send_endpoint": "/api/ai-automation/smoke/daily-summary/send", "telegram_send_in_preview": False, "writes_database": False, @@ -14817,6 +14863,60 @@ def _pixelrag_platform_probe_worker_check() -> Dict[str, Any]: ) +def _security_governance_review_check() -> Dict[str, Any]: + try: + from services.security_governance_review_service import ( + build_security_governance_review, + ) + + readback = build_security_governance_review(detail_limit=10) + completion = readback.get("completion") or {} + release_gate = readback.get("release_gate") or {} + blocking_count = int(release_gate.get("blocking_failure_count") or 0) + status = "critical" if blocking_count else ( + "ok" if readback.get("status") == "complete" else "warning" + ) + return _check( + "Security governance automation review", + status, + ( + f"Security governance program={completion.get('program_percent', 0)}%, " + f"asset_fields={completion.get('asset_field_coverage_percent', 0)}%, " + f"runtime_closure={completion.get('runtime_closure_passed', 0)}/" + f"{completion.get('runtime_closure_total', 0)}, blocking={blocking_count}" + ), + { + "policy": readback.get("policy"), + "review_status": readback.get("status"), + "program_percent": completion.get("program_percent"), + "asset_field_coverage_percent": completion.get( + "asset_field_coverage_percent" + ), + "asset_runtime_reconciliation_complete": completion.get( + "asset_runtime_reconciliation_complete" + ), + "runtime_closure_passed": completion.get("runtime_closure_passed"), + "runtime_closure_total": completion.get("runtime_closure_total"), + "blocking_failure_count": blocking_count, + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "Security governance automation review", + "critical", + f"Security governance review failed: {exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: int = 20) -> Dict[str, Any]: checks: List[Dict[str, Any]] = [ _event_router_check(), @@ -14850,6 +14950,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _ai_surface_html_readback_check(), _sitewide_ui_ux_agent_check(), _sitewide_visual_qa_check(), + _security_governance_review_check(), _external_mcp_rag_integration_check(), _pixelrag_rag_candidate_replay_check(), _pixelrag_source_contract_replay_worker_check(), diff --git a/services/code_review_pipeline_service.py b/services/code_review_pipeline_service.py index 131420e..80ebbe4 100644 --- a/services/code_review_pipeline_service.py +++ b/services/code_review_pipeline_service.py @@ -20,6 +20,7 @@ Pipeline： """ import json +import hmac import logging import os import re @@ -1268,4 +1269,4 @@ def verify_internal_token(request_token: str) -> bool: return True logger.error("[CodeReview] INTERNAL_WEBHOOK_TOKEN 未設定，拒絕 webhook") return False - return request_token == INTERNAL_TOKEN + return hmac.compare_digest(request_token or "", INTERNAL_TOKEN) diff --git a/services/http_access_policy_service.py b/services/http_access_policy_service.py new file mode 100644 index 0000000..d306ab1 --- /dev/null +++ b/services/http_access_policy_service.py @@ -0,0 +1,72 @@ +"""Central deny-by-default policy for sensitive HTTP surfaces.""" + +from __future__ import annotations + +POLICY = "ewoooc_sensitive_http_access_policy_v1" + +# Entire blueprints that expose business data or operator actions. +SESSION_REQUIRED_BLUEPRINTS = frozenset({ + "auto_import", + "category", + "cicd", + "crawler", + "crawler_management", + "vendor", +}) + +# Mixed public/private blueprints use endpoint-level protection. +SESSION_REQUIRED_ENDPOINTS = frozenset({ + "alert.alert_status", + "alert.test_alert", + "code_review.api_history", + "code_review.api_status", + "code_review.dashboard", + "misc.test_url", + "openclaw_bot.set_webhook", + "openclaw_bot.webhook_info", + "sales.abc_analysis_detail", + "system_public.get_logs_api", + "system_public.settings", + "system_public.show_logs", + "system_public.system_settings_page", +}) + +# Every anonymous GET must be intentional and small enough for public exposure. +INTENTIONALLY_PUBLIC_ENDPOINTS = frozenset({ + "bot_api.bot_api_status", + "elephant_alpha.status", + "misc.brand_assets", + "system_public.favicon", + "system_public.health_check", + "system_public.prometheus_metrics", + "system_public.webcrumbs_asset_proxy", + "user_bp.get_password_reqs", +}) + + +def is_session_required(endpoint: str | None, blueprint: str | None) -> bool: + return bool( + blueprint in SESSION_REQUIRED_BLUEPRINTS + or endpoint in SESSION_REQUIRED_ENDPOINTS + ) + + +def enforce_http_access_policy(): + """Protect sensitive routes before their route function executes.""" + from flask import request + + if not is_session_required(request.endpoint, request.blueprint): + return None + from auth import require_authenticated_request + + return require_authenticated_request() + + +__all__ = [ + "POLICY", + "SESSION_REQUIRED_BLUEPRINTS", + "SESSION_REQUIRED_ENDPOINTS", + "INTENTIONALLY_PUBLIC_ENDPOINTS", + "enforce_http_access_policy", + "is_session_required", +] diff --git a/services/security_governance_review_service.py b/services/security_governance_review_service.py new file mode 100644 index 0000000..9a29666 --- /dev/null +++ b/services/security_governance_review_service.py @@ -0,0 +1,670 @@ +"""Machine-readable security and governance review for EwoooC.""" + +from __future__ import annotations + +import ast +import json +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from services.http_access_policy_service import ( + INTENTIONALLY_PUBLIC_ENDPOINTS, + SESSION_REQUIRED_BLUEPRINTS, + SESSION_REQUIRED_ENDPOINTS, +) + + +POLICY = "global_product_governance_v2_security_review_v1" +ROOT = Path(__file__).resolve().parents[1] +ASSET_INVENTORY_PATH = Path("governance/ewoooc_asset_inventory.json") +QUALITY_BASELINE_PATH = Path("governance/security_governance_review_baseline.json") + +MUTATING_METHODS = {"POST", "PUT", "PATCH", "DELETE"} +AUTH_DECORATOR_MARKERS = ( + "login_required", + "admin_required", + "role_required", + "permission_required", + "check_alert_auth", + "internal_key_required", + "require_api_token", + "verify_webhook", +) +INLINE_AUTH_CALLS = { + "_require_internal_key", + "verify_internal_token", + "verify_webhook_signature", + "compare_digest", +} +FORBIDDEN_SOURCE_MARKERS = ( + "github.com", + "api.github.com", + "raw.githubusercontent.com", + "codeload.github.com", + "ghcr.io", +) +ASSET_REQUIRED_FIELDS = { + "canonical_id", + "asset_type", + "owner_lane", + "source_truth", + "runtime_identity", + "signal_freshness", + "policy", + "executor", + "verifier", + "backup_restore", + "learning_targets", +} + + +def _read(path: Path) -> str: + try: + return path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return "" + + +def _blueprint_names(tree: ast.Module) -> dict[str, str]: + names: dict[str, str] = {} + for node in tree.body: + if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Call): + continue + func = node.value.func + if not isinstance(func, ast.Name) or func.id != "Blueprint" or not node.value.args: + continue + first = node.value.args[0] + if not isinstance(first, ast.Constant) or not isinstance(first.value, str): + continue + for target in node.targets: + if isinstance(target, ast.Name): + names[target.id] = first.value + return names + + +def _route_metadata(decorator: ast.expr) -> tuple[str, set[str]] | None: + if not isinstance(decorator, ast.Call) or not isinstance(decorator.func, ast.Attribute): + return None + if decorator.func.attr != "route" or not decorator.args: + return None + first = decorator.args[0] + route = first.value if isinstance(first, ast.Constant) and isinstance(first.value, str) else "" + methods = {"GET"} + for keyword in decorator.keywords: + if keyword.arg != "methods" or not isinstance(keyword.value, (ast.List, ast.Tuple)): + continue + parsed = { + item.value.upper() + for item in keyword.value.elts + if isinstance(item, ast.Constant) and isinstance(item.value, str) + } + if parsed: + methods = parsed + return route, methods + + +def _called_names(node: ast.AST) -> set[str]: + names: set[str] = set() + for child in ast.walk(node): + if not isinstance(child, ast.Call): + continue + if isinstance(child.func, ast.Name): + names.add(child.func.id) + elif isinstance(child.func, ast.Attribute): + names.add(child.func.attr) + return names + + +def _scan_routes(root: Path) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] + parse_errors: list[str] = [] + for path in sorted((root / "routes").glob("*.py")): + try: + tree = ast.parse(_read(path), filename=str(path)) + except SyntaxError: + parse_errors.append(path.relative_to(root).as_posix()) + continue + blueprints = _blueprint_names(tree) + for node in tree.body: + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + decorator_text = [ast.unparse(item) for item in node.decorator_list] + explicit_guard = any( + marker in text + for marker in AUTH_DECORATOR_MARKERS + for text in decorator_text + ) + inline_guard = bool(_called_names(node) & INLINE_AUTH_CALLS) + for decorator in node.decorator_list: + metadata = _route_metadata(decorator) + if metadata is None: + continue + route, methods = metadata + blueprint_var = "" + if isinstance(decorator, ast.Call) and isinstance(decorator.func, ast.Attribute): + if isinstance(decorator.func.value, ast.Name): + blueprint_var = decorator.func.value.id + blueprint = blueprints.get(blueprint_var, blueprint_var.removesuffix("_bp")) + endpoint = f"{blueprint}.{node.name}" if blueprint else node.name + central_guard = ( + blueprint in SESSION_REQUIRED_BLUEPRINTS + or endpoint in SESSION_REQUIRED_ENDPOINTS + ) + rows.append({ + "file": path.relative_to(root).as_posix(), + "line": node.lineno, + "endpoint": endpoint, + "route": route, + "methods": sorted(methods), + "mutating": bool(methods & MUTATING_METHODS), + "guarded": explicit_guard or inline_guard or central_guard, + "guard_source": ( + "decorator" if explicit_guard else + "inline" if inline_guard else + "central_policy" if central_guard else + "none" + ), + }) + + unguarded = [item for item in rows if not item["guarded"]] + mutating_unguarded = [item for item in unguarded if item["mutating"]] + unclassified_unguarded = [ + item for item in unguarded + if item["endpoint"] not in INTENTIONALLY_PUBLIC_ENDPOINTS + ] + return { + "route_count": len(rows), + "guarded_count": sum(1 for item in rows if item["guarded"]), + "unguarded_count": len(unguarded), + "unguarded": unguarded, + "unclassified_unguarded_count": len(unclassified_unguarded), + "unclassified_unguarded": unclassified_unguarded, + "mutating_unguarded_count": len(mutating_unguarded), + "mutating_unguarded": mutating_unguarded[:30], + "parse_errors": parse_errors, + } + + +def _asset_inventory(root: Path) -> dict[str, Any]: + path = root / ASSET_INVENTORY_PATH + try: + payload = json.loads(_read(path)) + except (TypeError, json.JSONDecodeError): + payload = {} + assets = payload.get("assets") if isinstance(payload, dict) else [] + if not isinstance(assets, list): + assets = [] + missing: list[dict[str, Any]] = [] + populated_fields = 0 + expected_fields = len(assets) * len(ASSET_REQUIRED_FIELDS) + for index, asset in enumerate(assets): + if not isinstance(asset, dict): + missing.append({"index": index, "missing_fields": sorted(ASSET_REQUIRED_FIELDS)}) + continue + missing_fields = sorted(field for field in ASSET_REQUIRED_FIELDS if not asset.get(field)) + populated_fields += len(ASSET_REQUIRED_FIELDS) - len(missing_fields) + if missing_fields: + missing.append({ + "index": index, + "canonical_id": asset.get("canonical_id"), + "missing_fields": missing_fields, + }) + field_coverage = round((populated_fields / expected_fields * 100), 1) if expected_fields else 0.0 + reconciliation = payload.get("reconciliation") if isinstance(payload, dict) else {} + return { + "path": ASSET_INVENTORY_PATH.as_posix(), + "asset_count": len(assets), + "field_coverage_percent": field_coverage, + "missing_field_assets": missing, + "schema_valid": bool(assets) and not missing, + "reconciliation_status": (reconciliation or {}).get("status", "missing"), + "runtime_receipts_complete": bool((reconciliation or {}).get("runtime_receipts_complete")), + } + + +def _quality_baseline(root: Path) -> dict[str, Any]: + path = root / QUALITY_BASELINE_PATH + try: + payload = json.loads(_read(path)) + except (TypeError, json.JSONDecodeError): + payload = {} + collection = payload.get("collection") if isinstance(payload, dict) else {} + full_suite = payload.get("full_suite_without_missing_contract") if isinstance(payload, dict) else {} + focused = payload.get("focused_suites") if isinstance(payload, dict) else {} + collection = collection if isinstance(collection, dict) else {} + full_suite = full_suite if isinstance(full_suite, dict) else {} + focused = focused if isinstance(focused, dict) else {} + return { + "path": QUALITY_BASELINE_PATH.as_posix(), + "captured_at": payload.get("captured_at") if isinstance(payload, dict) else None, + "collection_status": collection.get("status", "missing"), + "collection_error_count": int(collection.get("error_count") or 0), + "missing_contract": collection.get("missing_contract"), + "full_suite_passed": int(full_suite.get("passed") or 0), + "full_suite_failed": int(full_suite.get("failed") or 0), + "full_suite_skipped": int(full_suite.get("skipped") or 0), + "focused_suites": focused, + "next_machine_action": payload.get("next_machine_action") if isinstance(payload, dict) else None, + } + + +def _workflow_supply_chain(root: Path) -> dict[str, Any]: + files = sorted((root / ".gitea" / "workflows").glob("*.y*ml")) + forbidden: list[dict[str, Any]] = [] + external_actions: list[dict[str, Any]] = [] + for path in files: + for line_number, line in enumerate(_read(path).splitlines(), start=1): + lower = line.lower() + for marker in FORBIDDEN_SOURCE_MARKERS: + if marker in lower: + forbidden.append({ + "file": path.relative_to(root).as_posix(), + "line": line_number, + "marker": marker, + }) + stripped = line.strip() + if stripped.startswith("uses:") or " uses:" in stripped: + external_actions.append({ + "file": path.relative_to(root).as_posix(), + "line": line_number, + "reference": stripped[:180], + }) + return { + "workflow_count": len(files), + "forbidden_source_references": forbidden, + "external_action_references": external_actions, + "gitea_only": bool(files) and not forbidden and not external_actions, + } + + +def _dependency_posture(root: Path) -> dict[str, Any]: + requirements = [] + for raw in _read(root / "requirements.txt").splitlines(): + line = raw.split("#", 1)[0].strip() + if line and not line.startswith(("-", "--")): + requirements.append(line) + exact = [line for line in requirements if "==" in line or " @ " in line] + sbom_names = {"sbom.json", "bom.json", "cyclonedx.json", "spdx.json"} + sbom_candidates: list[str] = [] + for name in sorted(sbom_names): + candidate = root / name + if candidate.is_file(): + sbom_candidates.append(candidate.relative_to(root).as_posix()) + for folder in ("artifacts", "build", "dist", "governance"): + base = root / folder + if not base.is_dir(): + continue + sbom_candidates.extend( + path.relative_to(root).as_posix() + for path in base.rglob("*") + if path.is_file() and path.name.lower() in sbom_names + ) + return { + "direct_requirement_count": len(requirements), + "exact_requirement_count": len(exact), + "all_direct_requirements_exact": bool(requirements) and len(exact) == len(requirements), + "lock_exists": (root / "requirements.lock").exists(), + "sbom_files": sbom_candidates, + "automated_sca_configured": any( + marker in _read(path).lower() + for path in (root / ".gitea" / "workflows").glob("*.y*ml") + for marker in ("pip-audit", "trivy", "grype", "osv-scanner") + ), + } + + +def _large_python_files(root: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for folder in ("services", "routes"): + for path in (root / folder).rglob("*.py"): + try: + line_count = sum(1 for _ in path.open(encoding="utf-8", errors="ignore")) + except OSError: + continue + if line_count > 800: + rows.append({ + "file": path.relative_to(root).as_posix(), + "lines": line_count, + }) + return sorted(rows, key=lambda item: item["lines"], reverse=True) + + +def _check( + check_id: str, + function: str, + status: str, + severity: str, + summary: str, + evidence: dict[str, Any], + next_action: str, + *, + release_blocking: bool = False, +) -> dict[str, Any]: + return { + "id": check_id, + "function": function, + "status": status, + "severity": severity, + "summary": summary, + "evidence": evidence, + "next_machine_action": next_action, + "release_blocking": release_blocking, + } + + +def _work_items(checks: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + return [ + {"order": 1, "id": "SEC-P0-001", "priority": "P0", "status": "in_progress", "lane": "deny_by_default_access", "objective": "Close all sensitive and mutating routes behind central auth or signed internal transport.", "next_machine_action": "verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth"}, + {"order": 2, "id": "SEC-P0-002", "priority": "P0", "status": "not_started", "lane": "identity_rbac", "objective": "Replace the shared admin password with database-backed identities, least-privilege RBAC, durable lockout and auditable sessions.", "next_machine_action": "inventory_active_users_then_shadow_database_auth_before_cutover"}, + {"order": 3, "id": "SEC-P0-003", "priority": "P0", "status": "in_progress", "lane": "webhook_trust", "objective": "Require Telegram and internal webhook transport secrets with replay protection.", "next_machine_action": "provision_telegram_webhook_secret_then_enable_required_mode_and_canary"}, + {"order": 4, "id": "SUPPLY-P0-001", "priority": "P0", "status": "in_progress", "lane": "software_supply_chain", "objective": "Use Gitea-only checkout, exact locks, SAST/SCA/secret scan, SBOM, image digest and provenance gates.", "next_machine_action": "add_internal_scanner_artifacts_and_exact_dependency_lock"}, + {"order": 5, "id": "GOV-P0-001", "priority": "P0", "status": "in_progress", "lane": "asset_graph", "objective": "Reconcile every host, service, data, AI, route, package and evidence asset against runtime truth.", "next_machine_action": "probe_all_assets_and_write_same_run_reconciliation_receipt"}, + {"order": 6, "id": "GOV-P0-002", "priority": "P0", "status": "not_started", "lane": "controlled_apply_contract", "objective": "Enforce one trace_id/run_id/work_item_id from sensor through verifier, rollback and learning acknowledgement.", "next_machine_action": "introduce_shared_governance_envelope_and_migrate_eventrouter_autoheal_first"}, + {"order": 7, "id": "RAG-P0-001", "priority": "P0", "status": "not_started", "lane": "internal_rag_canary", "objective": "Canary PixelRAG candidate knowledge into internal pgvector RAG without price or ai_insights promotion.", "next_machine_action": "run_internal_rag_candidate_canary"}, + {"order": 8, "id": "MCP-P0-001", "priority": "P0", "status": "in_progress", "lane": "mcp_rag_runtime", "objective": "Enable and verify MCP/RAG runtime health rather than registry-only readiness.", "next_machine_action": "close_mcp_ports_router_and_rag_runtime_readback"}, + {"order": 9, "id": "SEC-P0-004", "priority": "P0", "status": "not_started", "lane": "security_operations", "objective": "Publish security MTTA/MTTR, recurrence, false-positive, verifier and rollback metrics with incident closure receipts.", "next_machine_action": "add_security_event_metrics_and_incident_lifecycle_store"}, + {"order": 10, "id": "REL-P0-001", "priority": "P0", "status": "in_progress", "lane": "release_readback", "objective": "Separate source, test, CD, production runtime and visible proof for every release without touching momo-db.", "next_machine_action": "deploy_v10_771_then_read_back_security_matrix_and_container_identity"}, + {"order": 11, "id": "RES-P1-001", "priority": "P1", "status": "not_started", "lane": "resilience", "objective": "Automate checksum, offsite backup and isolated restore drills with measured RPO/RTO.", "next_machine_action": "build_non_destructive_restore_drill_receipt"}, + {"order": 12, "id": "PLAT-P1-001", "priority": "P1", "status": "not_started", "lane": "container_hardening", "objective": "Run application containers as non-root with minimal capabilities and read-only filesystems where compatible.", "next_machine_action": "shadow_nonroot_image_then_canary_three_app_containers"}, + {"order": 13, "id": "APPSEC-P1-001", "priority": "P1", "status": "in_progress", "lane": "browser_security", "objective": "Move CSP from report-only to enforced and remove unsafe DOM/inline sinks by measured rollout.", "next_machine_action": "collect_csp_violations_then_replace_high_risk_innerhtml_sinks"}, + {"order": 14, "id": "APPSEC-P1-002", "priority": "P1", "status": "not_started", "lane": "unsafe_cache_serialization", "objective": "Replace writable pickle caches with signed JSON or constrained serialization.", "next_machine_action": "migrate_dashboard_daily_sales_and_edm_shared_caches"}, + {"order": 15, "id": "ARCH-P1-001", "priority": "P1", "status": "in_progress", "lane": "modularization", "objective": "Split oversized services and routes so policy, executor and verifier can be independently tested.", "next_machine_action": "split_ai_automation_smoke_and_pchome_mapping_backlog_by_family"}, + {"order": 16, "id": "UX-P1-001", "priority": "P1", "status": "in_progress", "lane": "professional_product_ui", "objective": "Complete first-viewport cockpit, progressive disclosure, text-density reduction, accessibility and mobile visual smoke across primary pages.", "next_machine_action": "run_sitewide_visual_qa_then_fix_highest_traffic_text_walls"}, + {"order": 17, "id": "PIXELRAG-P1-001", "priority": "P1", "status": "not_started", "lane": "visual_embedding", "objective": "Benchmark Ollama-first multimodal embedding and pgvector-compatible visual metadata.", "next_machine_action": "benchmark_qwen3_vl_equivalent_on_gcp_a_gcp_b_111"}, + {"order": 18, "id": "MARKET-P1-001", "priority": "P1", "status": "in_progress", "lane": "marketplace_source_contracts", "objective": "Finish stable source contracts for Shopee, Coupang, Yahoo, ETMall, Friday and Rakuten with barrier classification.", "next_machine_action": "execute_provenance_rate_limit_and_public_boundary_replays"}, + {"order": 19, "id": "QA-P1-001", "priority": "P1", "status": "in_progress", "lane": "test_governance", "objective": "Restore clean collection, eliminate time/copy drift and split deterministic release tests from nightly integration and production canaries.", "next_machine_action": "restore_or_retire_missing_pchome_report_contract_then_baseline_legacy_failures"}, + {"order": 20, "id": "GOV-P2-001", "priority": "P2", "status": "not_started", "lane": "continuous_control_improvement", "objective": "Trend NIST CSF and OWASP ASVS control coverage and automatically create drift work items.", "next_machine_action": "persist_review_snapshots_and_diff_control_regressions"}, + {"order": 21, "id": "DATA-P2-001", "priority": "P2", "status": "not_started", "lane": "data_governance", "objective": "Classify business, personal, operational and model data and enforce retention with auditable non-destructive defaults.", "next_machine_action": "build_data_classification_and_retention_inventory"}, + {"order": 22, "id": "CHAOS-P2-001", "priority": "P2", "status": "not_started", "lane": "controlled_resilience", "objective": "Exercise model-host, MCP, queue, Telegram and app-container failures with bounded rollback and learning receipts.", "next_machine_action": "design_non_destructive_failure_drill_matrix"}, + ] + + +def build_security_governance_review( + *, + root: Path | str | None = None, + detail_limit: int = 30, +) -> dict[str, Any]: + """Review source and runtime flags without network, DB or secret access.""" + scan_root = Path(root).resolve() if root is not None else ROOT + detail_limit = max(5, min(int(detail_limit or 30), 100)) + + route_posture = _scan_routes(scan_root) + route_posture["unguarded"] = route_posture["unguarded"][:detail_limit] + route_posture["unclassified_unguarded"] = route_posture["unclassified_unguarded"][:detail_limit] + route_posture["mutating_unguarded"] = route_posture["mutating_unguarded"][:detail_limit] + assets = _asset_inventory(scan_root) + supply_chain = _workflow_supply_chain(scan_root) + dependencies = _dependency_posture(scan_root) + quality = _quality_baseline(scan_root) + large_files = _large_python_files(scan_root) + app_source = _read(scan_root / "app.py") + alert_source = _read(scan_root / "routes" / "alert_routes.py") + constitution = _read(scan_root / "CONSTITUTION.md") + dockerfile = _read(scan_root / "Dockerfile") + telegram_secret_configured = bool(os.getenv("TELEGRAM_WEBHOOK_SECRET_TOKEN")) + mcp_enabled = os.getenv("MCP_ROUTER_ENABLED", "false").lower() in {"1", "true", "yes", "on"} + rag_enabled = os.getenv("RAG_ENABLED", "false").lower() in {"1", "true", "yes", "on"} + + required_headers = { + "X-Content-Type-Options", + "X-Frame-Options", + "Referrer-Policy", + "Permissions-Policy", + "Strict-Transport-Security", + "Content-Security-Policy-Report-Only", + } + missing_headers = sorted(header for header in required_headers if header not in app_source) + session_secure_default = "PUBLIC_URL', 'https://mo.wooo.work'" in app_source + protected_resource_ready = all( + marker in alert_source + for marker in ("'momo-db'", "ALLOWED_AUTO_FIX_CONTAINERS", "PROTECTED_CONTAINERS") + ) + central_policy_registered = "enforce_http_access_policy" in app_source + + core_contract_source = "\n".join( + _read(scan_root / path) + for path in ( + "services/event_router.py", + "services/auto_heal_service.py", + "services/agent_actions.py", + "services/code_review_pipeline_service.py", + "services/elephant_alpha_autonomous_engine.py", + ) + ) + contract_fields = { + field: field in core_contract_source + for field in ("trace_id", "run_id", "work_item_id", "post_verifier", "rollback", "learning_recorded") + } + contract_coverage = sum(contract_fields.values()) + policy_drift_markers = [ + marker + for marker in ( + "所有 API Token、密碼存放於 `config.py`", + "nohup python3 app.py", + "pkill -9 -f \"python3 app.py\"", + ) + if marker in constitution + ] + + access_contract_ready = bool( + central_policy_registered + and route_posture["mutating_unguarded_count"] == 0 + and route_posture["unclassified_unguarded_count"] == 0 + ) + + checks = [ + _check( + "GV.AM-asset-inventory", + "GOVERN/IDENTIFY", + "partial" if assets["schema_valid"] and not assets["runtime_receipts_complete"] else "pass" if assets["schema_valid"] else "fail", + "high", + f"Asset inventory has {assets['asset_count']} assets and {assets['field_coverage_percent']}% required-field coverage; runtime reconciliation remains {assets['reconciliation_status']}.", + assets, + "probe_all_assets_and_write_same_run_reconciliation_receipt", + release_blocking=not assets["schema_valid"], + ), + _check( + "PR.AA-route-access-control", + "PROTECT", + "pass" if access_contract_ready else "fail", + "critical", + f"Route map: {route_posture['route_count']} total, {route_posture['unguarded_count']} intentionally public, {route_posture['unclassified_unguarded_count']} unclassified and {route_posture['mutating_unguarded_count']} mutating unguarded after central policy.", + route_posture, + "verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth", + release_blocking=not access_contract_ready, + ), + _check( + "PR.DS-session-and-browser-baseline", + "PROTECT", + "pass" if not missing_headers and session_secure_default else "fail", + "high", + "Security headers and secure-cookie-by-public-URL baseline are present." if not missing_headers and session_secure_default else "Browser/session baseline is incomplete.", + {"missing_headers": missing_headers, "secure_cookie_default": session_secure_default}, + "enforce_csp_after_report_only_violation_cleanup", + release_blocking=bool(missing_headers) or not session_secure_default, + ), + _check( + "PR.IR-autoheal-resource-boundary", + "PROTECT/RESPOND", + "pass" if protected_resource_ready else "fail", + "critical", + "Auto-fix is deny-by-default for containers and explicitly protects momo-db." if protected_resource_ready else "Auto-fix container boundary is incomplete.", + {"protected_resource_ready": protected_resource_ready}, + "keep_container_allowlist_and_add_independent_post_verifier", + release_blocking=not protected_resource_ready, + ), + _check( + "GV.SC-gitea-only-supply-chain", + "GOVERN/PROTECT", + "pass" if supply_chain["gitea_only"] else "fail", + "critical", + "Gitea workflows have no external action or forbidden GitHub source reference." if supply_chain["gitea_only"] else "Workflow still depends on an external action/source.", + supply_chain, + "replace_external_action_resolution_with_gitea_job_token_checkout", + release_blocking=not supply_chain["gitea_only"], + ), + _check( + "ID.RA-dependency-vulnerability-management", + "IDENTIFY/PROTECT", + "pass" if dependencies["lock_exists"] and dependencies["sbom_files"] and dependencies["automated_sca_configured"] else "fail", + "high", + "Dependency lock, SBOM and automated SCA are not yet a complete release gate.", + dependencies, + "add_exact_lock_internal_sca_secret_scan_sbom_and_image_digest_receipts", + ), + _check( + "PR.PS-container-least-privilege", + "PROTECT", + "pass" if "\nUSER " in f"\n{dockerfile}" else "fail", + "high", + "Application image declares a non-root runtime user." if "\nUSER " in f"\n{dockerfile}" else "Application image currently defaults to root.", + {"dockerfile_user_declared": "\nUSER " in f"\n{dockerfile}"}, + "shadow_nonroot_image_then_canary_three_app_containers", + ), + _check( + "RS.MA-controlled-apply-envelope", + "RESPOND/RECOVER", + "pass" if contract_coverage == len(contract_fields) else "fail", + "critical", + f"Core automation contract contains {contract_coverage}/{len(contract_fields)} required closure fields.", + {"fields": contract_fields, "coverage_count": contract_coverage}, + "introduce_shared_governance_envelope_and_migrate_eventrouter_autoheal_first", + ), + _check( + "RC.RP-restore-drill", + "RECOVER", + "partial" if (scan_root / "backup_system.py").exists() else "fail", + "high", + "Backup code exists, but no durable isolated restore-drill receipt is committed or read back.", + {"backup_code_exists": (scan_root / "backup_system.py").exists(), "restore_receipt_exists": False}, + "build_non_destructive_restore_drill_receipt", + ), + _check( + "GV.PO-policy-alignment", + "GOVERN", + "pass" if not policy_drift_markers else "fail", + "high", + "Constitution is aligned to current production/governance doctrine." if not policy_drift_markers else "Legacy security/deployment doctrine still conflicts with production truth.", + {"legacy_markers": policy_drift_markers}, + "sunset_legacy_rules_with_owner_expiry_replacement_and_verifier", + release_blocking=bool(policy_drift_markers), + ), + _check( + "GV.OV-modularization", + "GOVERN", + "pass" if not large_files else "partial", + "medium", + f"{len(large_files)} service/route files exceed 800 lines; top file has {large_files[0]['lines'] if large_files else 0} lines.", + {"oversized_file_count": len(large_files), "top_files": large_files[:15]}, + "split_ai_automation_smoke_and_pchome_mapping_backlog_by_family", + ), + _check( + "GV.QA-test-governance", + "GOVERN/IDENTIFY", + "pass" if quality["collection_status"] == "passed" and quality["full_suite_failed"] == 0 else "partial", + "high", + f"Focused security suites are green, but full test governance has {quality['collection_error_count']} collection errors and {quality['full_suite_failed']} baseline failures.", + quality, + "restore_or_retire_missing_pchome_report_contract_then_baseline_legacy_failures", + ), + _check( + "PR.AA-telegram-webhook-transport", + "PROTECT", + "pass" if telegram_secret_configured else "partial", + "critical", + "Telegram webhook secret is configured." if telegram_secret_configured else "Code supports Telegram secret verification, but production secret configuration is not proven.", + {"secret_configured": telegram_secret_configured, "secret_value_read": False}, + "provision_telegram_webhook_secret_then_enable_required_mode_and_canary", + ), + _check( + "DE.CM-mcp-rag-runtime", + "DETECT", + "pass" if mcp_enabled and rag_enabled else "partial", + "high", + f"Runtime flags: MCP_ROUTER_ENABLED={mcp_enabled}, RAG_ENABLED={rag_enabled}.", + {"mcp_router_enabled": mcp_enabled, "rag_enabled": rag_enabled}, + "close_mcp_ports_router_and_rag_runtime_readback", + ), + _check( + "AI.RAG-internal-candidate-canary", + "DETECT/RESPOND", + "pass" if (scan_root / "services/internal_rag_candidate_canary_service.py").exists() else "fail", + "high", + "Internal RAG candidate canary exists." if (scan_root / "services/internal_rag_candidate_canary_service.py").exists() else "PixelRAG candidate knowledge stops before an internal RAG canary.", + {"service_exists": (scan_root / "services/internal_rag_candidate_canary_service.py").exists()}, + "run_internal_rag_candidate_canary", + ), + ] + checks_by_id = {item["id"]: item for item in checks} + blocking = [item for item in checks if item["release_blocking"] and item["status"] == "fail"] + status_counts = { + status: sum(1 for item in checks if item["status"] == status) + for status in ("pass", "partial", "fail") + } + score_weights = {"pass": 1.0, "partial": 0.5, "fail": 0.0} + program_percent = round( + sum(score_weights[item["status"]] for item in checks) / len(checks) * 100, + 1, + ) + runtime_checks = [ + checks_by_id["DE.CM-mcp-rag-runtime"], + checks_by_id["AI.RAG-internal-candidate-canary"], + checks_by_id["RC.RP-restore-drill"], + checks_by_id["RS.MA-controlled-apply-envelope"], + ] + runtime_passed = sum(1 for item in runtime_checks if item["status"] == "pass") + + return { + "success": True, + "policy": POLICY, + "generated_at": datetime.now(timezone.utc).isoformat(), + "status": "complete" if status_counts["fail"] == 0 and status_counts["partial"] == 0 else "partial", + "answer_to_owner": "The security/governance target is not complete. The project has many local receipts, but access control, identity, asset reconciliation, supply-chain evidence, same-run controlled apply, restore proof and MCP/RAG runtime closure are not yet one production loop.", + "root_causes": [ + "Feature and receipt growth outran a single deny-by-default security policy.", + "Source/test/UI evidence was repeatedly counted ahead of production closure evidence.", + "AI lanes use many preview/review stages without one shared trace/run/work-item envelope.", + "Asset, dependency, identity, backup and runtime controls are not reconciled by one machine-readable control plane.", + "Large modules couple policy, execution and verification, making independent assurance expensive.", + ], + "benchmark_profile": { + "nist_csf_2_0": ["GOVERN", "IDENTIFY", "PROTECT", "DETECT", "RESPOND", "RECOVER"], + "owasp_asvs": "5.0.0", + "nist_ssdf": "SP 800-218 v1.1", + "application_level_target": "ASVS L2 with selected L3 controls for operator and automation surfaces", + }, + "completion": { + "program_percent": program_percent, + "asset_field_coverage_percent": assets["field_coverage_percent"], + "asset_runtime_reconciliation_complete": assets["runtime_receipts_complete"], + "runtime_closure_passed": runtime_passed, + "runtime_closure_total": len(runtime_checks), + "status_counts": status_counts, + }, + "checks": checks, + "quality_baseline": quality, + "release_gate": { + "status": "pass" if not blocking else "fail", + "blocking_failure_count": len(blocking), + "blocking_failures": [ + {"id": item["id"], "summary": item["summary"]} + for item in blocking + ], + "debt_does_not_equal_release_pass": True, + }, + "work_items": _work_items(checks_by_id), + "safety": { + "read_only": True, + "network_calls": False, + "database_reads": False, + "database_writes": False, + "secret_reads": False, + "external_side_effects": False, + }, + "next_machine_action": "verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth", + } + + +__all__ = ["POLICY", "build_security_governance_review"] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index 69e100c..d746418 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1433,6 +1433,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_ai_surface_html_readback_check", lambda: smoke._check("surface", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_ui_ux_agent_check", lambda: smoke._check("sitewide", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) + monkeypatch.setattr(smoke, "_security_governance_review_check", lambda: smoke._check("security governance", "ok", "ok")) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) @@ -3979,6 +3980,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_ai_surface_html_readback_check", lambda: smoke._check("surface", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_ui_ux_agent_check", lambda: smoke._check("sitewide", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) + monkeypatch.setattr(smoke, "_security_governance_review_check", lambda: smoke._check("security governance", "ok", "ok")) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) @@ -4043,6 +4045,17 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( from datetime import datetime from services import ai_automation_smoke_service as smoke + monkeypatch.setattr( + smoke, + "_security_governance_review_check", + lambda: smoke._check( + "Security governance automation review", + "ok", + "security governance ok", + {"next_machine_action": "keep_monitoring"}, + ), + ) + history_path = tmp_path / "smoke_history.jsonl" history_path.write_text( json.dumps({ @@ -6756,6 +6769,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "visual ok", {"route_count": 12, "viewport_count": 3, "failed_count": 0}, )) + monkeypatch.setattr(smoke, "_security_governance_review_check", lambda: smoke._check( + "Security governance automation review", + "ok", + "security governance ok", + )) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check( "External MCP/RAG integration readback", "ok", diff --git a/tests/test_security_governance_review_service.py b/tests/test_security_governance_review_service.py new file mode 100644 index 0000000..a6f9d2e --- /dev/null +++ b/tests/test_security_governance_review_service.py @@ -0,0 +1,164 @@ +import json +import subprocess +import sys +from pathlib import Path + +import pytest +from flask import Blueprint, Flask, jsonify + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_security_governance_review_is_honest_and_release_gate_passes(): + from services.security_governance_review_service import ( + build_security_governance_review, + ) + + report = build_security_governance_review(root=ROOT, detail_limit=12) + + assert report["status"] == "partial" + assert report["release_gate"]["status"] == "pass" + assert report["release_gate"]["blocking_failure_count"] == 0 + assert report["completion"]["asset_field_coverage_percent"] == 100.0 + assert report["completion"]["asset_runtime_reconciliation_complete"] is False + assert report["completion"]["runtime_closure_passed"] < report["completion"]["runtime_closure_total"] + assert report["work_items"][0]["id"] == "SEC-P0-001" + assert len(report["work_items"]) == 22 + assert report["work_items"][9]["id"] == "REL-P0-001" + assert any(item["id"] == "QA-P1-001" for item in report["work_items"]) + assert report["work_items"][-1]["id"] == "CHAOS-P2-001" + checks = {item["id"]: item for item in report["checks"]} + access = checks["PR.AA-route-access-control"]["evidence"] + assert access["mutating_unguarded_count"] == 0 + assert access["unclassified_unguarded_count"] == 0 + assert access["unguarded_count"] == 8 + dependencies = checks["ID.RA-dependency-vulnerability-management"]["evidence"] + assert dependencies["sbom_files"] == [] + quality = checks["GV.QA-test-governance"] + assert quality["status"] == "partial" + assert quality["evidence"]["collection_error_count"] == 1 + assert quality["evidence"]["full_suite_passed"] == 1701 + assert report["safety"] == { + "read_only": True, + "network_calls": False, + "database_reads": False, + "database_writes": False, + "secret_reads": False, + "external_side_effects": False, + } + + +def test_security_governance_cli_strict_mode_passes_source_gate(): + completed = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "ops" / "report_security_governance_review.py"), + "--root", + str(ROOT), + "--compact", + "--strict", + ], + cwd=ROOT, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + assert completed.returncode == 0, completed.stderr or completed.stdout + payload = json.loads(completed.stdout) + assert payload["release_gate"]["status"] == "pass" + assert payload["status"] == "partial" + + +def test_sensitive_blueprint_policy_denies_anonymous_api(monkeypatch): + import auth + from services.http_access_policy_service import enforce_http_access_policy + + monkeypatch.setattr(auth, "DISABLE_LOGIN", False) + app = Flask(__name__) + app.secret_key = "test" + vendor = Blueprint("vendor", __name__) + + @vendor.route("/vendor-stockout/api/vendor/list") + def list_vendors(): + return jsonify({"success": True}) + + app.register_blueprint(vendor) + app.before_request(enforce_http_access_policy) + client = app.test_client() + + anonymous = client.get("/vendor-stockout/api/vendor/list") + assert anonymous.status_code == 401 + assert anonymous.get_json()["error"] == "authentication_required" + + with client.session_transaction() as session: + session["logged_in"] = True + authenticated = client.get("/vendor-stockout/api/vendor/list") + assert authenticated.status_code == 200 + + +def test_public_url_validator_blocks_ssrf_targets(monkeypatch): + from utils import security + + monkeypatch.setattr( + security.socket, + "getaddrinfo", + lambda *_args, **_kwargs: [(2, 1, 6, "", ("93.184.216.34", 443))], + ) + assert security.validate_public_http_url("https://example.com/path") == "https://example.com/path" + + monkeypatch.setattr( + security.socket, + "getaddrinfo", + lambda *_args, **_kwargs: [(2, 1, 6, "", ("127.0.0.1", 80))], + ) + with pytest.raises(ValueError, match="loopback"): + security.validate_public_http_url("http://example.com/") + + +def test_alert_auto_fix_protects_database_and_denies_unknown_container(monkeypatch): + from routes import alert_routes + + def fail_if_called(*_args, **_kwargs): + raise AssertionError("subprocess must not run for denied containers") + + monkeypatch.setattr(alert_routes.subprocess, "run", fail_if_called) + + protected = alert_routes.auto_fix_container("momo-db", "memory") + assert protected["success"] is False + assert "受保護" in protected["message"] + + unknown = alert_routes.auto_fix_container("stock-platform", "memory") + assert unknown["success"] is False + assert "白名單" in unknown["message"] + + +def test_security_governance_smoke_family_is_warning_for_nonblocking_debt(monkeypatch): + from services import ai_automation_smoke_service as smoke + from services import security_governance_review_service as review_service + + monkeypatch.setattr( + review_service, + "build_security_governance_review", + lambda **_kwargs: { + "policy": "test-policy", + "status": "partial", + "completion": { + "program_percent": 45.0, + "asset_field_coverage_percent": 100.0, + "asset_runtime_reconciliation_complete": False, + "runtime_closure_passed": 0, + "runtime_closure_total": 4, + }, + "release_gate": {"blocking_failure_count": 0}, + "next_machine_action": "close_access_control", + }, + ) + + check = smoke._security_governance_review_check() + + assert check["status"] == "warning" + assert check["details"]["blocking_failure_count"] == 0 + assert check["details"]["next_machine_action"] == "close_access_control" diff --git a/utils/security.py b/utils/security.py index ae6a792..9a0dbad 100644 --- a/utils/security.py +++ b/utils/security.py @@ -3,10 +3,13 @@ 從 app.py + utils/validators.py 整併的單一權威來源。純驗證邏輯，無 Flask 依賴。 舊的 utils/validators.py 已 deprecate，僅保留 re-export 不破壞既有 import。 """ +import ipaddress import os import re +import socket import unicodedata from pathlib import Path +from urllib.parse import urlparse import pandas as pd @@ -15,6 +18,41 @@ from utils.logger_manager import SystemLogger _log = SystemLogger("Security").get_logger() +def validate_public_http_url(value): + """Validate an outbound URL and reject SSRF targets before any request.""" + candidate = str(value or '').strip() + parsed = urlparse(candidate) + if parsed.scheme not in {'http', 'https'} or not parsed.hostname: + raise ValueError('只允許完整的 HTTP/HTTPS 網址') + if parsed.username or parsed.password: + raise ValueError('網址不可包含帳號或密碼') + if parsed.port not in {None, 80, 443}: + raise ValueError('只允許標準 HTTP/HTTPS 連接埠') + + try: + addresses = { + info[4][0] + for info in socket.getaddrinfo( + parsed.hostname, + parsed.port or (443 if parsed.scheme == 'https' else 80), + type=socket.SOCK_STREAM, + ) + } + except OSError as exc: + raise ValueError('網址主機無法解析') from exc + + if not addresses: + raise ValueError('網址主機沒有可用位址') + for address in addresses: + try: + ip = ipaddress.ip_address(address.split('%', 1)[0]) + except ValueError as exc: + raise ValueError('網址主機位址無效') from exc + if not ip.is_global: + raise ValueError('禁止連線至內網、loopback、link-local 或保留位址') + return candidate + + # ──────────────────────────────────────────────────────────────────────── # SQL Injection 防護 # ──────────────────────────────────────────────────────────────────────── From d6aeb76c4b9ca0e08d64e5c4ee8a010e561f5ee2 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 18:55:10 +0800 Subject: [PATCH 30/75] test(ai): align dev smoke counts with governance check --- tests/test_ai_automation_smoke_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index d746418..ea22776 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1452,7 +1452,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 43, "warning": 1, "critical": 1, "total": 45} + assert result["summary"] == {"ok": 44, "warning": 1, "critical": 1, "total": 46} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -4447,7 +4447,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 42 + assert summary["summary"]["total"] == 43 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -6859,7 +6859,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 45 + assert result["summary"]["total"] == 46 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" From 2bbb1e8e8b8bb142c02b43f1c43ac57d5f21e7cd Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 19:13:22 +0800 Subject: [PATCH 31/75] fix(security): harden image context and runtime receipts --- .dockerignore | 48 ++++++++ .gitea/workflows/cd.yaml | 10 +- .gitignore | 4 +- CONSTITUTION.md | 2 +- config.py | 2 +- docker-compose.yml | 3 + docs/AI_INTELLIGENCE_MODULE_SOT.md | 2 +- .../ai_automation_mainline_work_items.md | 2 +- governance/ewoooc_asset_inventory.json | 4 +- .../ops/report_security_governance_review.py | 14 +++ .../security_governance_review_service.py | 116 ++++++++++++++++-- ...test_security_governance_review_service.py | 55 +++++++++ 12 files changed, 246 insertions(+), 16 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6f69cd8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,48 @@ +.git +.gitea +.github +.claude +.docker +.env +.env.* +auth.json +**/auth.json +*.pem +*.key +*.crt +*.p12 +*.pfx +id_rsa* +ssl +secrets +**/secrets +data +logs +backups +runtime_artifacts +config/google_credentials.json +config/google_token.json +config/google_token.pickle +config/*.json +config/*.pickle +*.db +*.db-* +*.sqlite +*.sqlite3 +venv +.venv +node_modules +__pycache__ +*.pyc +.pytest_cache +.coverage +htmlcov +tests +docs +memory +k8s +n8n-workflows +aiops-core +*.md +._* +.DS_Store diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index c8ba063..7298903 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -41,6 +41,7 @@ on: - 'monitoring/blackbox.yml' # 需重建 image 的檔案 - 'Dockerfile' + - '.dockerignore' - 'requirements.txt' - 'docker-compose.yml' # 腳本工具 @@ -108,7 +109,7 @@ jobs: exit 0 fi CHANGED=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || echo "") - if echo "$CHANGED" | grep -qE '^(Dockerfile|requirements\.txt|docker-compose\.yml)$'; then + if echo "$CHANGED" | grep -qE '^(Dockerfile|\.dockerignore|requirements\.txt|docker-compose\.yml)$'; then echo "type=rebuild" >> $GITHUB_OUTPUT echo "label=🔨 重建 Docker Image" >> $GITHUB_OUTPUT else @@ -129,7 +130,14 @@ jobs: fi - name: 資訊安全與治理 release gate + env: + GITEA_SHA: ${{ gitea.sha }} run: | + set -eu + mkdir -p governance + python3 scripts/ops/report_security_governance_review.py \ + --strict --source-commit "$GITEA_SHA" \ + > governance/security_governance_source_receipt.json python3 scripts/ops/report_security_governance_review.py --compact --strict # 設定 SSH 金鑰 + 主機驗證（C2 fix: 移除 StrictHostKeyChecking no） diff --git a/.gitignore b/.gitignore index 5126dbc..5c95dbe 100644 --- a/.gitignore +++ b/.gitignore @@ -137,10 +137,12 @@ ehthumbs.db Desktop.ini # Docker -.dockerignore docker-compose.override.yml .docker/ +# CD-generated governance receipts (deployed evidence, not source) +governance/security_governance_source_receipt.json + # SSL 憑證 ssl/ *.pem diff --git a/CONSTITUTION.md b/CONSTITUTION.md index fcb30ff..997199a 100644 --- a/CONSTITUTION.md +++ b/CONSTITUTION.md @@ -2,7 +2,7 @@ > 本文件定義專案開發的核心準則與不可違反的規範 > **建立日期**: 2026-01-12 -> **當前版本**: V10.771 (Security governance automation review) +> **當前版本**: V10.772 (Security governance automation review) > **最後更新**: 2026-07-10 > **治理基準**: `global_product_governance_v2` + ADR-038 diff --git a/config.py b/config.py index 3e632f2..84affc4 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.771" +SYSTEM_VERSION = "V10.772" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docker-compose.yml b/docker-compose.yml index 2bcfe3e..cfd19f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,6 +65,7 @@ services: - ./gunicorn.conf.py:/app/gunicorn.conf.py:ro - ./scheduler.py:/app/scheduler.py:ro - ./scripts:/app/scripts:ro + - ./governance:/app/governance:ro - ./migrations:/app/migrations:ro - ./services:/app/services:ro - ./routes:/app/routes:ro @@ -217,6 +218,7 @@ services: - ./scheduler.py:/app/scheduler.py:ro - ./run_scheduler.py:/app/run_scheduler.py:ro - ./scripts:/app/scripts:ro + - ./governance:/app/governance:ro - ./services:/app/services:ro - ./routes:/app/routes:ro - ./database:/app/database:ro # 資料庫模型 @@ -285,6 +287,7 @@ services: - ./config:/app/config - ./config.py:/app/config.py:ro - ./run_telegram_bot.py:/app/run_telegram_bot.py:ro + - ./governance:/app/governance:ro - ./services:/app/services:ro - ./routes:/app/routes:ro - ./database:/app/database:ro diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 865ed8a..f60c69f 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟠 Partial。四 Agent、PixelRAG、MCP/RAG registry、PChome controlled-preview families 與多項 smoke/readback 已建立；但 access control、database identity/RBAC、full asset reconciliation、software supply chain、same-run controlled apply、restore drill、internal RAG canary 與 MCP/RAG runtime closure 尚未全部完成。任何局部 receipt 不得代表整體閉環完成。 -> **適用版本**: V10.771 +> **適用版本**: V10.772 --- diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 31a4ef4..3e53a8d 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -19,7 +19,7 @@ | 1 | `SEC-P0-001` | In progress | Deny-by-default route access control | Central policy now protects vendor/import/CI-CD/crawler/category/log/operator and sales-detail surfaces; every anonymous GET is explicitly allowlisted. Exit: anonymous production matrix passes and `/metrics` moves behind edge/internal auth. Next: `verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth`. | | 2 | `SEC-P0-002` | Not started | Database identity + least-privilege RBAC | Replace shared admin password, default-admin fallback, process-local lockout and spoofable proxy identity. Exit: shadow auth, role matrix, durable audit/lockout, session revocation and canary cutover. | | 3 | `SEC-P0-003` | In progress | Webhook trust and replay protection | Telegram secret-token verification code exists; production secret activation remains unproven. Exit: secret provisioned outside source, required mode enabled, invalid-secret 401 and authorized callback canary pass. | -| 4 | `SUPPLY-P0-001` | In progress | Gitea-only secure software supply chain | Gitea-native checkout and governance gate added. Exit: exact dependency lock, internal SAST/SCA/secret scan, SBOM, image digest/provenance, vulnerability SLA and production digest readback. | +| 4 | `SUPPLY-P0-001` | In progress | Gitea-only secure software supply chain | Gitea-native checkout, secret-safe `.dockerignore`, commit-bound source receipt and governance gate are active. Exit: exact dependency lock, internal SAST/SCA/secret scan, SBOM, image digest/provenance, vulnerability SLA and production digest readback. | | 5 | `GOV-P0-001` | In progress | Canonical full asset graph + runtime reconciliation | `governance/ewoooc_asset_inventory.json` seeds hosts, services, data, AI, routes, supply chain, observability and recovery. Exit: same-run probe receipt for every asset; drift auto-creates work items. | | 6 | `GOV-P0-002` | Not started | Unified controlled-apply envelope | Introduce one `trace_id/run_id/work_item_id` across sensor, identity, SOT diff, decision, risk, dry-run, execution, verifier, rollback/retry and learning acknowledgement. Start with EventRouter + AutoHeal. | | 7 | `RAG-P0-001` | Not started | Internal RAG candidate canary | V10.770 stops at candidate preview. Exit: bounded pgvector candidate canary, signature/readback/feedback receipt, no price write and no premature `ai_insights` promotion. Next: `run_internal_rag_candidate_canary`. | diff --git a/governance/ewoooc_asset_inventory.json b/governance/ewoooc_asset_inventory.json index 2b7c5cc..6b54fcf 100644 --- a/governance/ewoooc_asset_inventory.json +++ b/governance/ewoooc_asset_inventory.json @@ -320,10 +320,10 @@ "canonical_id": "supply-chain:ewoooc:python-container", "asset_type": "packages_images_sbom", "owner_lane": "software-supply-chain", - "source_truth": "requirements, image digest, SBOM and vulnerability receipts", + "source_truth": "requirements, .dockerignore, source receipt, image digest, SBOM and vulnerability receipts", "runtime_identity": "EwoooC Python 3.11 container image", "signal_freshness": {"source": "CD security gate", "runtime_target": "deployed image digest", "max_age_minutes": 1440}, - "policy": ["Gitea/internal sources", "exact dependency lock target", "no GitHub action dependency"], + "policy": ["Gitea/internal sources", "secret-safe Docker context", "exact dependency lock target", "no GitHub action dependency"], "executor": "Gitea CD build lane", "verifier": "SAST, SCA, secret scan, SBOM and provenance checks", "backup_restore": "known-good immutable image digest", diff --git a/scripts/ops/report_security_governance_review.py b/scripts/ops/report_security_governance_review.py index fbe097a..04f98c1 100644 --- a/scripts/ops/report_security_governance_review.py +++ b/scripts/ops/report_security_governance_review.py @@ -23,6 +23,11 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--root", type=Path, default=ROOT) parser.add_argument("--detail-limit", type=int, default=30) parser.add_argument("--compact", action="store_true") + parser.add_argument( + "--source-commit", + default="", + help="Bind emitted source receipt to the caller-provided Gitea commit SHA.", + ) parser.add_argument( "--strict", action="store_true", @@ -37,6 +42,15 @@ def main(argv: list[str] | None = None) -> int: root=args.root, detail_limit=args.detail_limit, ) + source_commit = str(args.source_commit or "").strip() + if source_commit: + if len(source_commit) != 40 or any( + char not in "0123456789abcdefABCDEF" for char in source_commit + ): + print("--source-commit must be a 40-character hexadecimal Gitea SHA", file=sys.stderr) + return 2 + report["receipt_kind"] = "security_governance_source_receipt" + report["source_commit"] = source_commit if args.compact: output = { "policy": report["policy"], diff --git a/services/security_governance_review_service.py b/services/security_governance_review_service.py index 9a29666..b43cbe5 100644 --- a/services/security_governance_review_service.py +++ b/services/security_governance_review_service.py @@ -20,6 +20,7 @@ POLICY = "global_product_governance_v2_security_review_v1" ROOT = Path(__file__).resolve().parents[1] ASSET_INVENTORY_PATH = Path("governance/ewoooc_asset_inventory.json") QUALITY_BASELINE_PATH = Path("governance/security_governance_review_baseline.json") +SOURCE_RECEIPT_PATH = Path("governance/security_governance_source_receipt.json") MUTATING_METHODS = {"POST", "PUT", "PATCH", "DELETE"} AUTH_DECORATOR_MARKERS = ( @@ -58,6 +59,19 @@ ASSET_REQUIRED_FIELDS = { "backup_restore", "learning_targets", } +DOCKERIGNORE_REQUIRED_PATTERNS = { + ".git", + ".gitea", + ".env", + ".env.*", + "data", + "logs", + "backups", + "config/google_credentials.json", + "config/google_token.json", + "config/google_token.pickle", + "config/*.json", +} def _read(path: Path) -> str: @@ -250,6 +264,53 @@ def _quality_baseline(root: Path) -> dict[str, Any]: } +def _source_receipt(root: Path) -> dict[str, Any]: + path = root / SOURCE_RECEIPT_PATH + try: + payload = json.loads(_read(path)) + except (TypeError, json.JSONDecodeError): + payload = {} + checks = payload.get("checks") if isinstance(payload, dict) else [] + checks_by_id = { + item.get("id"): item + for item in checks + if isinstance(item, dict) and item.get("id") + } if isinstance(checks, list) else {} + source_commit = str(payload.get("source_commit") or "") if isinstance(payload, dict) else "" + commit_shape_valid = len(source_commit) == 40 and all( + char in "0123456789abcdefABCDEF" for char in source_commit + ) + return { + "path": SOURCE_RECEIPT_PATH.as_posix(), + "available": bool(payload) and payload.get("receipt_kind") == "security_governance_source_receipt", + "source_commit": source_commit, + "source_commit_shape_valid": commit_shape_valid, + "generated_at": payload.get("generated_at") if isinstance(payload, dict) else None, + "release_gate_status": ( + (payload.get("release_gate") or {}).get("status") + if isinstance(payload, dict) else None + ), + "_checks": checks_by_id, + } + + +def _docker_build_context_posture(root: Path) -> dict[str, Any]: + path = root / ".dockerignore" + patterns = { + line.strip().rstrip("/") + for line in _read(path).splitlines() + if line.strip() and not line.lstrip().startswith("#") + } + missing = sorted(DOCKERIGNORE_REQUIRED_PATTERNS - patterns) + return { + "path": ".dockerignore", + "exists": path.is_file(), + "required_pattern_count": len(DOCKERIGNORE_REQUIRED_PATTERNS), + "missing_patterns": missing, + "safe": path.is_file() and not missing, + } + + def _workflow_supply_chain(root: Path) -> dict[str, Any]: files = sorted((root / ".gitea" / "workflows").glob("*.y*ml")) forbidden: list[dict[str, Any]] = [] @@ -395,7 +456,25 @@ def build_security_governance_review( route_posture["unclassified_unguarded"] = route_posture["unclassified_unguarded"][:detail_limit] route_posture["mutating_unguarded"] = route_posture["mutating_unguarded"][:detail_limit] assets = _asset_inventory(scan_root) + source_receipt = _source_receipt(scan_root) supply_chain = _workflow_supply_chain(scan_root) + docker_context = _docker_build_context_posture(scan_root) + supply_chain["docker_build_context"] = docker_context + supply_chain["source_chain_ready"] = bool( + supply_chain["gitea_only"] and docker_context["safe"] + ) + receipt_supply = source_receipt["_checks"].get("GV.SC-gitea-only-supply-chain", {}) + if ( + supply_chain["workflow_count"] == 0 + and source_receipt["available"] + and source_receipt["source_commit_shape_valid"] + and receipt_supply.get("status") == "pass" + ): + receipt_evidence = receipt_supply.get("evidence") or {} + supply_chain = dict(receipt_evidence) if isinstance(receipt_evidence, dict) else {} + supply_chain["evidence_source"] = "cd_source_receipt" + supply_chain["source_commit"] = source_receipt["source_commit"] + supply_chain["source_chain_ready"] = True dependencies = _dependency_posture(scan_root) quality = _quality_baseline(scan_root) large_files = _large_python_files(scan_root) @@ -447,6 +526,24 @@ def build_security_governance_review( ) if marker in constitution ] + policy_alignment_ready = bool(constitution) and not policy_drift_markers + policy_evidence = { + "legacy_markers": policy_drift_markers, + "evidence_source": "source_file" if constitution else "missing", + } + receipt_policy = source_receipt["_checks"].get("GV.PO-policy-alignment", {}) + if ( + not constitution + and source_receipt["available"] + and source_receipt["source_commit_shape_valid"] + and receipt_policy.get("status") == "pass" + ): + policy_alignment_ready = True + policy_evidence = { + "legacy_markers": [], + "evidence_source": "cd_source_receipt", + "source_commit": source_receipt["source_commit"], + } access_contract_ready = bool( central_policy_registered @@ -498,12 +595,12 @@ def build_security_governance_review( _check( "GV.SC-gitea-only-supply-chain", "GOVERN/PROTECT", - "pass" if supply_chain["gitea_only"] else "fail", + "pass" if supply_chain["source_chain_ready"] else "fail", "critical", - "Gitea workflows have no external action or forbidden GitHub source reference." if supply_chain["gitea_only"] else "Workflow still depends on an external action/source.", + "Gitea workflow and Docker build context exclude external actions, forbidden sources, secrets and runtime data." if supply_chain["source_chain_ready"] else "Source workflow or Docker build context boundary is incomplete.", supply_chain, - "replace_external_action_resolution_with_gitea_job_token_checkout", - release_blocking=not supply_chain["gitea_only"], + "keep_gitea_checkout_and_generate_commit_bound_source_receipt", + release_blocking=not supply_chain["source_chain_ready"], ), _check( "ID.RA-dependency-vulnerability-management", @@ -544,12 +641,12 @@ def build_security_governance_review( _check( "GV.PO-policy-alignment", "GOVERN", - "pass" if not policy_drift_markers else "fail", + "pass" if policy_alignment_ready else "fail", "high", - "Constitution is aligned to current production/governance doctrine." if not policy_drift_markers else "Legacy security/deployment doctrine still conflicts with production truth.", - {"legacy_markers": policy_drift_markers}, + "Constitution is aligned to current production/governance doctrine." if policy_alignment_ready else "Policy source is missing or legacy doctrine still conflicts with production truth.", + policy_evidence, "sunset_legacy_rules_with_owner_expiry_replacement_and_verifier", - release_blocking=bool(policy_drift_markers), + release_blocking=not policy_alignment_ready, ), _check( "GV.OV-modularization", @@ -645,6 +742,9 @@ def build_security_governance_review( }, "checks": checks, "quality_baseline": quality, + "source_receipt": { + key: value for key, value in source_receipt.items() if key != "_checks" + }, "release_gate": { "status": "pass" if not blocking else "fail", "blocking_failure_count": len(blocking), diff --git a/tests/test_security_governance_review_service.py b/tests/test_security_governance_review_service.py index a6f9d2e..671b99c 100644 --- a/tests/test_security_governance_review_service.py +++ b/tests/test_security_governance_review_service.py @@ -35,6 +35,9 @@ def test_security_governance_review_is_honest_and_release_gate_passes(): assert access["unguarded_count"] == 8 dependencies = checks["ID.RA-dependency-vulnerability-management"]["evidence"] assert dependencies["sbom_files"] == [] + supply = checks["GV.SC-gitea-only-supply-chain"] + assert supply["status"] == "pass" + assert supply["evidence"]["docker_build_context"]["safe"] is True quality = checks["GV.QA-test-governance"] assert quality["status"] == "partial" assert quality["evidence"]["collection_error_count"] == 1 @@ -72,6 +75,58 @@ def test_security_governance_cli_strict_mode_passes_source_gate(): assert payload["status"] == "partial" +def test_security_governance_cli_can_emit_commit_bound_source_receipt(): + source_commit = "a" * 40 + completed = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "ops" / "report_security_governance_review.py"), + "--root", + str(ROOT), + "--strict", + "--source-commit", + source_commit, + ], + cwd=ROOT, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + assert completed.returncode == 0, completed.stderr or completed.stdout + payload = json.loads(completed.stdout) + assert payload["receipt_kind"] == "security_governance_source_receipt" + assert payload["source_commit"] == source_commit + assert payload["release_gate"]["status"] == "pass" + + +def test_runtime_subset_uses_commit_bound_source_receipt(tmp_path): + from services.security_governance_review_service import ( + build_security_governance_review, + ) + + source_report = build_security_governance_review(root=ROOT) + source_report["receipt_kind"] = "security_governance_source_receipt" + source_report["source_commit"] = "b" * 40 + governance = tmp_path / "governance" + governance.mkdir() + (governance / "security_governance_source_receipt.json").write_text( + json.dumps(source_report, ensure_ascii=False), + encoding="utf-8", + ) + + runtime_report = build_security_governance_review(root=tmp_path) + checks = {item["id"]: item for item in runtime_report["checks"]} + + supply = checks["GV.SC-gitea-only-supply-chain"] + policy = checks["GV.PO-policy-alignment"] + assert supply["status"] == "pass" + assert supply["evidence"]["evidence_source"] == "cd_source_receipt" + assert policy["status"] == "pass" + assert policy["evidence"]["evidence_source"] == "cd_source_receipt" + + def test_sensitive_blueprint_policy_denies_anonymous_api(monkeypatch): import auth from services.http_access_policy_service import enforce_http_access_policy From 97ab6ea070e586b0050d864f00e141edbc9e9341 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 19:40:04 +0800 Subject: [PATCH 32/75] feat(governance): persist production security readback --- .gitignore | 1 + CONSTITUTION.md | 2 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 2 +- .../ai_automation_mainline_work_items.md | 4 +- governance/ewoooc_asset_inventory.json | 2 +- .../security_governance_review_service.py | 102 +++++++++++++++++- ...test_security_governance_review_service.py | 51 +++++++++ 8 files changed, 155 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 5c95dbe..c09a6ea 100644 --- a/.gitignore +++ b/.gitignore @@ -142,6 +142,7 @@ docker-compose.override.yml # CD-generated governance receipts (deployed evidence, not source) governance/security_governance_source_receipt.json +governance/security_governance_runtime_receipt.json # SSL 憑證 ssl/ diff --git a/CONSTITUTION.md b/CONSTITUTION.md index 997199a..aea8944 100644 --- a/CONSTITUTION.md +++ b/CONSTITUTION.md @@ -2,7 +2,7 @@ > 本文件定義專案開發的核心準則與不可違反的規範 > **建立日期**: 2026-01-12 -> **當前版本**: V10.772 (Security governance automation review) +> **當前版本**: V10.773 (Security governance automation review) > **最後更新**: 2026-07-10 > **治理基準**: `global_product_governance_v2` + ADR-038 diff --git a/config.py b/config.py index 84affc4..a17bd0b 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.772" +SYSTEM_VERSION = "V10.773" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index f60c69f..c27dc72 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-07-10 (台北時間) > **狀態**: 🟠 Partial。四 Agent、PixelRAG、MCP/RAG registry、PChome controlled-preview families 與多項 smoke/readback 已建立；但 access control、database identity/RBAC、full asset reconciliation、software supply chain、same-run controlled apply、restore drill、internal RAG canary 與 MCP/RAG runtime closure 尚未全部完成。任何局部 receipt 不得代表整體閉環完成。 -> **適用版本**: V10.772 +> **適用版本**: V10.773 --- diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 3e53a8d..74757ac 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -16,7 +16,7 @@ | Order | ID | Status | Work item | Exit evidence / next machine action | |---:|---|---|---|---| -| 1 | `SEC-P0-001` | In progress | Deny-by-default route access control | Central policy now protects vendor/import/CI-CD/crawler/category/log/operator and sales-detail surfaces; every anonymous GET is explicitly allowlisted. Exit: anonymous production matrix passes and `/metrics` moves behind edge/internal auth. Next: `verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth`. | +| 1 | `SEC-P0-001` | In progress | Deny-by-default route access control | Production anonymous matrix now denies sensitive routes and all browser headers/Secure cookie pass. Remaining exit: move `/metrics` behind edge/internal auth without breaking Prometheus. Next: `move_metrics_behind_edge_auth_with_prometheus_canary`. | | 2 | `SEC-P0-002` | Not started | Database identity + least-privilege RBAC | Replace shared admin password, default-admin fallback, process-local lockout and spoofable proxy identity. Exit: shadow auth, role matrix, durable audit/lockout, session revocation and canary cutover. | | 3 | `SEC-P0-003` | In progress | Webhook trust and replay protection | Telegram secret-token verification code exists; production secret activation remains unproven. Exit: secret provisioned outside source, required mode enabled, invalid-secret 401 and authorized callback canary pass. | | 4 | `SUPPLY-P0-001` | In progress | Gitea-only secure software supply chain | Gitea-native checkout, secret-safe `.dockerignore`, commit-bound source receipt and governance gate are active. Exit: exact dependency lock, internal SAST/SCA/secret scan, SBOM, image digest/provenance, vulnerability SLA and production digest readback. | @@ -25,7 +25,7 @@ | 7 | `RAG-P0-001` | Not started | Internal RAG candidate canary | V10.770 stops at candidate preview. Exit: bounded pgvector candidate canary, signature/readback/feedback receipt, no price write and no premature `ai_insights` promotion. Next: `run_internal_rag_candidate_canary`. | | 8 | `MCP-P0-001` | In progress | MCP/RAG production runtime closure | Registry and smoke wiring exist, but runtime flags/ports must be live. Exit: MCP servers healthy, router enabled, RAG enabled, approved caller/tool boundary and production query canary pass. | | 9 | `SEC-P0-004` | Not started | Security operations lifecycle and metrics | Add durable security incident state and publish MTTA, MTTR, recurrence, false positive, human intervention, verifier pass, rollback and freshness. Exit: detect-to-learn production receipt. | -| 10 | `REL-P0-001` | In progress | Formal deploy and visible proof discipline | Every functional change bumps version, passes Gitea CD, deploys only three app containers, leaves `momo-db` untouched, and reads back `/health`, new APIs and visible UI/security behavior. | +| 10 | `REL-P0-001` | In progress | Formal deploy and visible proof discipline | V10.773 records the controlled production fallback with `momo-db` unchanged; Gitea CD #1095 remains waiting because no `ewoooc-host` runner is online. Exit: restore a dedicated EwoooC-only runner, replay the same commit and close CD/readback automatically. | ## P1 diff --git a/governance/ewoooc_asset_inventory.json b/governance/ewoooc_asset_inventory.json index 6b54fcf..a7976e7 100644 --- a/governance/ewoooc_asset_inventory.json +++ b/governance/ewoooc_asset_inventory.json @@ -294,7 +294,7 @@ "canonical_id": "observability:ewoooc:security-runtime", "asset_type": "logs_metrics_traces_alerts", "owner_lane": "observability", - "source_truth": "Prometheus, Grafana, logs, alerts and Telegram receipts", + "source_truth": "Prometheus, Grafana, logs, alerts, security runtime receipt and Telegram receipts", "runtime_identity": "/metrics plus AI automation smoke", "signal_freshness": {"source": "scrape and alert receipt", "runtime_target": "security/governance families", "max_age_minutes": 15}, "policy": ["no public operational logs", "lifecycle receipts", "freshness SLO"], diff --git a/services/security_governance_review_service.py b/services/security_governance_review_service.py index b43cbe5..fa0f2f2 100644 --- a/services/security_governance_review_service.py +++ b/services/security_governance_review_service.py @@ -21,6 +21,7 @@ ROOT = Path(__file__).resolve().parents[1] ASSET_INVENTORY_PATH = Path("governance/ewoooc_asset_inventory.json") QUALITY_BASELINE_PATH = Path("governance/security_governance_review_baseline.json") SOURCE_RECEIPT_PATH = Path("governance/security_governance_source_receipt.json") +RUNTIME_RECEIPT_PATH = Path("governance/security_governance_runtime_receipt.json") MUTATING_METHODS = {"POST", "PUT", "PATCH", "DELETE"} AUTH_DECORATOR_MARKERS = ( @@ -294,6 +295,60 @@ def _source_receipt(root: Path) -> dict[str, Any]: } +def _runtime_receipt(root: Path) -> dict[str, Any]: + path = root / RUNTIME_RECEIPT_PATH + try: + payload = json.loads(_read(path)) + except (TypeError, json.JSONDecodeError): + payload = {} + access = payload.get("access_control") if isinstance(payload, dict) else {} + database = payload.get("database") if isinstance(payload, dict) else {} + post_verifier = payload.get("post_verifier") if isinstance(payload, dict) else {} + cd = payload.get("cd") if isinstance(payload, dict) else {} + access = access if isinstance(access, dict) else {} + database = database if isinstance(database, dict) else {} + post_verifier = post_verifier if isinstance(post_verifier, dict) else {} + cd = cd if isinstance(cd, dict) else {} + source_commit = str(payload.get("source_commit") or "") if isinstance(payload, dict) else "" + commit_shape_valid = len(source_commit) == 40 and all( + char in "0123456789abcdefABCDEF" for char in source_commit + ) + security_readback_passed = bool( + payload.get("receipt_kind") == "security_governance_runtime_receipt" + and commit_shape_valid + and access.get("status") == "pass" + and access.get("sensitive_routes_denied") is True + and access.get("security_headers_passed") is True + and access.get("secure_cookie") is True + and database.get("unchanged") is True + and post_verifier.get("health_status") == "healthy" + and post_verifier.get("governance_gate") == "pass" + ) if isinstance(payload, dict) else False + return { + "path": RUNTIME_RECEIPT_PATH.as_posix(), + "available": bool(payload), + "trace_id": payload.get("trace_id") if isinstance(payload, dict) else None, + "run_id": payload.get("run_id") if isinstance(payload, dict) else None, + "work_item_id": payload.get("work_item_id") if isinstance(payload, dict) else None, + "source_commit": source_commit, + "source_commit_shape_valid": commit_shape_valid, + "deployed_version": payload.get("deployed_version") if isinstance(payload, dict) else None, + "deployment_mode": payload.get("deployment_mode") if isinstance(payload, dict) else None, + "security_readback_passed": security_readback_passed, + "sensitive_route_count": int(access.get("sensitive_route_count") or 0), + "denied_route_count": int(access.get("denied_route_count") or 0), + "metrics_public": bool(access.get("metrics_public")), + "database_unchanged": bool(database.get("unchanged")), + "health_status": post_verifier.get("health_status"), + "governance_gate": post_verifier.get("governance_gate"), + "cd_run_id": cd.get("run_id"), + "cd_status": cd.get("status"), + "cd_blocker": cd.get("blocker"), + "learning_recorded": bool(payload.get("learning_recorded")) if isinstance(payload, dict) else False, + "next_machine_action": payload.get("next_machine_action") if isinstance(payload, dict) else None, + } + + def _docker_build_context_posture(root: Path) -> dict[str, Any]: path = root / ".dockerignore" patterns = { @@ -415,9 +470,22 @@ def _check( } -def _work_items(checks: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: +def _work_items( + checks: dict[str, dict[str, Any]], + runtime_receipt: dict[str, Any], +) -> list[dict[str, Any]]: + access_next_action = ( + "move_metrics_behind_edge_auth_with_prometheus_canary" + if runtime_receipt["security_readback_passed"] + else "verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth" + ) + release_next_action = ( + "restore_dedicated_ewoooc_runner_then_replay_waiting_cd" + if runtime_receipt["security_readback_passed"] + else "deploy_v10_773_then_read_back_security_matrix_and_container_identity" + ) return [ - {"order": 1, "id": "SEC-P0-001", "priority": "P0", "status": "in_progress", "lane": "deny_by_default_access", "objective": "Close all sensitive and mutating routes behind central auth or signed internal transport.", "next_machine_action": "verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth"}, + {"order": 1, "id": "SEC-P0-001", "priority": "P0", "status": "in_progress", "lane": "deny_by_default_access", "objective": "Close all sensitive and mutating routes behind central auth or signed internal transport.", "next_machine_action": access_next_action}, {"order": 2, "id": "SEC-P0-002", "priority": "P0", "status": "not_started", "lane": "identity_rbac", "objective": "Replace the shared admin password with database-backed identities, least-privilege RBAC, durable lockout and auditable sessions.", "next_machine_action": "inventory_active_users_then_shadow_database_auth_before_cutover"}, {"order": 3, "id": "SEC-P0-003", "priority": "P0", "status": "in_progress", "lane": "webhook_trust", "objective": "Require Telegram and internal webhook transport secrets with replay protection.", "next_machine_action": "provision_telegram_webhook_secret_then_enable_required_mode_and_canary"}, {"order": 4, "id": "SUPPLY-P0-001", "priority": "P0", "status": "in_progress", "lane": "software_supply_chain", "objective": "Use Gitea-only checkout, exact locks, SAST/SCA/secret scan, SBOM, image digest and provenance gates.", "next_machine_action": "add_internal_scanner_artifacts_and_exact_dependency_lock"}, @@ -426,7 +494,7 @@ def _work_items(checks: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: {"order": 7, "id": "RAG-P0-001", "priority": "P0", "status": "not_started", "lane": "internal_rag_canary", "objective": "Canary PixelRAG candidate knowledge into internal pgvector RAG without price or ai_insights promotion.", "next_machine_action": "run_internal_rag_candidate_canary"}, {"order": 8, "id": "MCP-P0-001", "priority": "P0", "status": "in_progress", "lane": "mcp_rag_runtime", "objective": "Enable and verify MCP/RAG runtime health rather than registry-only readiness.", "next_machine_action": "close_mcp_ports_router_and_rag_runtime_readback"}, {"order": 9, "id": "SEC-P0-004", "priority": "P0", "status": "not_started", "lane": "security_operations", "objective": "Publish security MTTA/MTTR, recurrence, false-positive, verifier and rollback metrics with incident closure receipts.", "next_machine_action": "add_security_event_metrics_and_incident_lifecycle_store"}, - {"order": 10, "id": "REL-P0-001", "priority": "P0", "status": "in_progress", "lane": "release_readback", "objective": "Separate source, test, CD, production runtime and visible proof for every release without touching momo-db.", "next_machine_action": "deploy_v10_771_then_read_back_security_matrix_and_container_identity"}, + {"order": 10, "id": "REL-P0-001", "priority": "P0", "status": "in_progress", "lane": "release_readback", "objective": "Separate source, test, CD, production runtime and visible proof for every release without touching momo-db.", "next_machine_action": release_next_action}, {"order": 11, "id": "RES-P1-001", "priority": "P1", "status": "not_started", "lane": "resilience", "objective": "Automate checksum, offsite backup and isolated restore drills with measured RPO/RTO.", "next_machine_action": "build_non_destructive_restore_drill_receipt"}, {"order": 12, "id": "PLAT-P1-001", "priority": "P1", "status": "not_started", "lane": "container_hardening", "objective": "Run application containers as non-root with minimal capabilities and read-only filesystems where compatible.", "next_machine_action": "shadow_nonroot_image_then_canary_three_app_containers"}, {"order": 13, "id": "APPSEC-P1-001", "priority": "P1", "status": "in_progress", "lane": "browser_security", "objective": "Move CSP from report-only to enforced and remove unsafe DOM/inline sinks by measured rollout.", "next_machine_action": "collect_csp_violations_then_replace_high_risk_innerhtml_sinks"}, @@ -477,6 +545,7 @@ def build_security_governance_review( supply_chain["source_chain_ready"] = True dependencies = _dependency_posture(scan_root) quality = _quality_baseline(scan_root) + runtime_receipt = _runtime_receipt(scan_root) large_files = _large_python_files(scan_root) app_source = _read(scan_root / "app.py") alert_source = _read(scan_root / "routes" / "alert_routes.py") @@ -582,6 +651,23 @@ def build_security_governance_review( "enforce_csp_after_report_only_violation_cleanup", release_blocking=bool(missing_headers) or not session_secure_default, ), + _check( + "PR.AA-production-security-readback", + "PROTECT/DETECT", + "pass" if runtime_receipt["security_readback_passed"] else "partial", + "critical", + ( + f"Production security readback passed for {runtime_receipt['denied_route_count']}/{runtime_receipt['sensitive_route_count']} sensitive routes; database identity remained unchanged." + if runtime_receipt["security_readback_passed"] + else "Production access, header, cookie, container and database readback receipt is not complete." + ), + runtime_receipt, + ( + "move_metrics_behind_edge_auth_with_prometheus_canary" + if runtime_receipt["security_readback_passed"] + else "write_commit_bound_production_security_runtime_receipt" + ), + ), _check( "PR.IR-autoheal-resource-boundary", "PROTECT/RESPOND", @@ -706,6 +792,7 @@ def build_security_governance_review( 1, ) runtime_checks = [ + checks_by_id["PR.AA-production-security-readback"], checks_by_id["DE.CM-mcp-rag-runtime"], checks_by_id["AI.RAG-internal-candidate-canary"], checks_by_id["RC.RP-restore-drill"], @@ -742,6 +829,7 @@ def build_security_governance_review( }, "checks": checks, "quality_baseline": quality, + "runtime_receipt": runtime_receipt, "source_receipt": { key: value for key, value in source_receipt.items() if key != "_checks" }, @@ -754,7 +842,7 @@ def build_security_governance_review( ], "debt_does_not_equal_release_pass": True, }, - "work_items": _work_items(checks_by_id), + "work_items": _work_items(checks_by_id, runtime_receipt), "safety": { "read_only": True, "network_calls": False, @@ -763,7 +851,11 @@ def build_security_governance_review( "secret_reads": False, "external_side_effects": False, }, - "next_machine_action": "verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth", + "next_machine_action": ( + "move_metrics_behind_edge_auth_with_prometheus_canary" + if runtime_receipt["security_readback_passed"] + else "verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth" + ), } diff --git a/tests/test_security_governance_review_service.py b/tests/test_security_governance_review_service.py index 671b99c..a238624 100644 --- a/tests/test_security_governance_review_service.py +++ b/tests/test_security_governance_review_service.py @@ -127,6 +127,57 @@ def test_runtime_subset_uses_commit_bound_source_receipt(tmp_path): assert policy["evidence"]["evidence_source"] == "cd_source_receipt" +def test_runtime_receipt_advances_access_readback_without_hiding_remaining_debt(tmp_path): + from services.security_governance_review_service import ( + _runtime_receipt, + ) + + governance = tmp_path / "governance" + governance.mkdir() + payload = { + "receipt_kind": "security_governance_runtime_receipt", + "trace_id": "secgov-test", + "run_id": "secgov-test-run", + "work_item_id": "SEC-P0-001", + "source_commit": "c" * 40, + "deployed_version": "V10.773", + "deployment_mode": "controlled_manual_fallback", + "access_control": { + "status": "pass", + "sensitive_routes_denied": True, + "security_headers_passed": True, + "secure_cookie": True, + "sensitive_route_count": 8, + "denied_route_count": 8, + "metrics_public": True, + }, + "database": {"unchanged": True}, + "post_verifier": { + "health_status": "healthy", + "governance_gate": "pass", + }, + "cd": { + "run_id": 1095, + "status": "waiting", + "blocker": "no_ewoooc_host_runner", + }, + "learning_recorded": False, + "next_machine_action": "move_metrics_behind_edge_auth_with_prometheus_canary", + } + (governance / "security_governance_runtime_receipt.json").write_text( + json.dumps(payload, ensure_ascii=False), + encoding="utf-8", + ) + + receipt = _runtime_receipt(tmp_path) + + assert receipt["security_readback_passed"] is True + assert receipt["database_unchanged"] is True + assert receipt["metrics_public"] is True + assert receipt["cd_status"] == "waiting" + assert receipt["learning_recorded"] is False + + def test_sensitive_blueprint_policy_denies_anonymous_api(monkeypatch): import auth from services.http_access_policy_service import enforce_http_access_policy From e80e146fa261de4b4a765ca9ead153a87033ab60 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 19:57:21 +0800 Subject: [PATCH 33/75] docs(governance): bind CD identity to live receipts --- docs/guides/ai_automation_mainline_work_items.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 74757ac..75abae4 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -25,7 +25,7 @@ | 7 | `RAG-P0-001` | Not started | Internal RAG candidate canary | V10.770 stops at candidate preview. Exit: bounded pgvector candidate canary, signature/readback/feedback receipt, no price write and no premature `ai_insights` promotion. Next: `run_internal_rag_candidate_canary`. | | 8 | `MCP-P0-001` | In progress | MCP/RAG production runtime closure | Registry and smoke wiring exist, but runtime flags/ports must be live. Exit: MCP servers healthy, router enabled, RAG enabled, approved caller/tool boundary and production query canary pass. | | 9 | `SEC-P0-004` | Not started | Security operations lifecycle and metrics | Add durable security incident state and publish MTTA, MTTR, recurrence, false positive, human intervention, verifier pass, rollback and freshness. Exit: detect-to-learn production receipt. | -| 10 | `REL-P0-001` | In progress | Formal deploy and visible proof discipline | V10.773 records the controlled production fallback with `momo-db` unchanged; Gitea CD #1095 remains waiting because no `ewoooc-host` runner is online. Exit: restore a dedicated EwoooC-only runner, replay the same commit and close CD/readback automatically. | +| 10 | `REL-P0-001` | In progress | Formal deploy and visible proof discipline | V10.773 records the controlled production fallback with `momo-db` unchanged. The prior Gitea CD run was cancelled by a newer push and the latest commit-bound run remains waiting because no `ewoooc-host` runner is online; exact run identity comes from Gitea/runtime receipts, not this roadmap. Exit: restore a dedicated EwoooC-only runner, replay the same commit and close CD/readback automatically. | ## P1 From 9125afc8a79af3fc93605f06570fdd532db13713 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 19:59:57 +0800 Subject: [PATCH 34/75] fix(governance): bind baseline time to commit evidence --- governance/security_governance_review_baseline.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/governance/security_governance_review_baseline.json b/governance/security_governance_review_baseline.json index 79940ce..1989449 100644 --- a/governance/security_governance_review_baseline.json +++ b/governance/security_governance_review_baseline.json @@ -1,7 +1,8 @@ { "schema_version": "1.0", "product_id": "ewoooc", - "captured_at": "2026-07-10T20:40:00+08:00", + "captured_at": "2026-07-10T18:49:32+08:00", + "capture_time_basis": "review_commit_committer_timestamp", "source_base_sha": "c62ec7b504c7f74ce8a6a3b5312e30c1e6947513", "review_branch": "codex/security-governance-review", "collection": { From 107d8ed79716a8cf0e54b9ff0e600c0a71f43c83 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 21:22:42 +0800 Subject: [PATCH 35/75] feat(pchome): close freshness and multi-market decision gaps --- .env.example | 74 ++++ config.py | 2 +- .../pchome_ai_automation_priority_backlog.md | 30 +- .../code_modularization_inventory_20260430.md | 56 +++ governance/pchome_growth_program_review.json | 222 ++++++++++++ run_scheduler.py | 64 ++++ scheduler.py | 12 +- services/ai_automation_debt_service.py | 5 +- services/external_market_offer_service.py | 73 +++- services/import_service.py | 38 +- services/pchome_revenue_growth_service.py | 329 ++++++++++++++++-- templates/ai_intelligence.html | 112 ++++-- tests/test_auto_import_failure_boundaries.py | 39 ++- tests/test_competitor_intel_cache.py | 5 +- tests/test_external_market_offer_service.py | 52 ++- tests/test_frontend_v2_assets.py | 3 + tests/test_pchome_revenue_growth_service.py | 101 +++++- tests/test_run_scheduler_embed_consistency.py | 43 +++ web/static/css/ewoooc-tokens-v2-alias.css | 4 +- 19 files changed, 1179 insertions(+), 85 deletions(-) create mode 100644 governance/pchome_growth_program_review.json diff --git a/.env.example b/.env.example index ac91b08..f552f1e 100644 --- a/.env.example +++ b/.env.example @@ -619,3 +619,77 @@ N8N_PROTOCOL=http N8N_WEBHOOK_BASE_URL=http://192.168.0.110:5678/ N8N_USER=admin N8N_PASSWORD=change-me + +# ────────────────────────────────────────────────────────────────────────── +# 2026-07-10 runtime contract inventory +# Keep every os.getenv key discoverable without exposing production secrets. +# ────────────────────────────────────────────────────────────────────────── + +FLASK_ENV=production +KUBERNETES_SERVICE_HOST= +CICD_PROD_BASE_URL=https://mo.wooo.work +CICD_UAT_BASE_URL=https://uat.mo.wooo.work +SCHEDULER_STATS_PATH=/app/data/scheduler_stats.json + +DAILY_SALES_IMPORT_MAX_STALE_DAYS=3 +DAILY_SALES_IMPORT_STALENESS_GUARD_ENABLED=true +GOOGLE_DRIVE_CREDENTIALS_FILE=/app/config/google_credentials.json +GOOGLE_DRIVE_TOKEN_FILE=/app/config/google_token.json +GOOGLE_DRIVE_LEGACY_PICKLE_FILE=/app/config/google_token.pickle + +EXTERNAL_OFFER_SYNC_LIMIT=500 +MOMO_TARGETED_SEARCH_LIMIT_PER_TERM=3 +MOMO_TARGETED_SEARCH_MAX_PRODUCTS=8 +MOMO_TARGETED_SEARCH_MAX_TERMS=3 +MOMO_TARGETED_SEARCH_MIN_SCORE=0.5 +PCHOME_GROWTH_MOMO_BACKFILL_LIMIT=8 +PCHOME_GROWTH_BACKFILL_COOLDOWN_HOURS=4 +PCHOME_GROWTH_BACKFILL_RETRY_HOURS=1 +PCHOME_PUBLIC_API_TIMEOUT_SECONDS=4 +PCHOME_SALES_JOIN_BY_MOMO_SKU_ENABLED=false +PCHOME_STALE_RECOVERY_ENABLED=true +PCHOME_FEEDER_TTL_HOURS=24 +PCHOME_FEEDER_BACKFILL_TIMEOUT=12 +PCHOME_FEEDER_BACKFILL_MAX_SEARCH_TERMS=6 +PCHOME_FEEDER_BACKFILL_MAX_SECONDS_PER_SKU=45 +PCHOME_FEEDER_BACKFILL_SEARCH_MAX_PAGES=2 +PCHOME_BACKFILL_EXPIRED_REFRESH_LIMIT=40 +PCHOME_BACKFILL_RETRYABLE_LIMIT=40 +PCHOME_BACKFILL_REVALIDATION_LIMIT=40 +PCHOME_BACKFILL_UNMATCHED_LIMIT=40 + +MOMO_PCHOME_AUTO_POLICY_DECISION_PREFLIGHT_BATCH_SIZE=20 +MOMO_PCHOME_AUTO_POLICY_DECISION_PREFLIGHT_TIMEOUT_SECONDS=12 +MOMO_PCHOME_AUTO_POLICY_GUARD_BATCH_SIZE=20 +MOMO_PCHOME_AUTO_POLICY_GUARD_LIMIT=50 +MOMO_PCHOME_AUTO_POLICY_GUARD_TIMEOUT_SECONDS=12 +MOMO_PCHOME_DRIFT_MONITOR_DB_STATEMENT_TIMEOUT_MS=12000 + +MCP_FAST_STATIC_FALLBACK=true +MCP_OLLAMA_NUM_PREDICT=512 +MCP_OLLAMA_TIMEOUT=45 +OLLAMA_SECONDARY_MODEL_FALLBACK= +OLLAMA_SECONDARY_MODEL_FALLBACK_PATTERNS= + +PIXELRAG_VISUAL_EVIDENCE_ROOT=/app/data/pixelrag/evidence +PIXELRAG_VISUAL_EVIDENCE_MANIFEST_QUEUE_ROOT=/app/data/pixelrag/manifest_queue +PIXELRAG_VISUAL_EVIDENCE_MAX_AGE_HOURS=24 +PIXELRAG_PLATFORM_PROBE_CAPTURE_OUTPUT_ROOT=/app/data/pixelrag/platform_probe +PIXELRAG_PLATFORM_PROBE_WORKER_RECEIPT_ROOT=/app/data/pixelrag/receipts/platform_probe +PIXELRAG_SOURCE_CONTRACT_REPLAY_RECEIPT_ROOT=/app/data/pixelrag/receipts/source_contract +PIXELRAG_MARKETPLACE_ADAPTER_DRY_RUN_RECEIPT_ROOT=/app/data/pixelrag/receipts/adapter_dry_run +PIXELRAG_MARKETPLACE_ADAPTER_PREFLIGHT_RECEIPT_ROOT=/app/data/pixelrag/receipts/adapter_preflight +PIXELRAG_MARKETPLACE_CANDIDATE_KNOWLEDGE_REPLAY_RECEIPT_ROOT=/app/data/pixelrag/receipts/knowledge_replay +PIXELRAG_MARKETPLACE_EMBEDDING_SIGNATURE_GUARD_REPLAY_RECEIPT_ROOT=/app/data/pixelrag/receipts/embedding_guard +PIXELRAG_MARKETPLACE_IDENTITY_MATCHER_REPLAY_RECEIPT_ROOT=/app/data/pixelrag/receipts/identity_matcher +PIXELRAG_MARKETPLACE_PROMOTION_GATE_REPLAY_RECEIPT_ROOT=/app/data/pixelrag/receipts/promotion_gate +PIXELRAG_VLM_REPLAY_RECEIPT_ROOT=/app/data/pixelrag/receipts/vlm_replay +PIXELRAG_VLM_MODEL=qwen3-vl +PIXELRAG_VLM_MODEL_CANDIDATES=qwen3-vl,minicpm-v + +MOMO_SITEWIDE_VISUAL_QA_ARTIFACT=/app/data/runtime_artifacts/sitewide_visual_qa.json +MOMO_SITEWIDE_VISUAL_QA_MAX_AGE_HOURS=24 +PPT_SCHEDULED_FAST_FALLBACK=true + +TELEGRAM_WEBHOOK_SECRET_REQUIRED=true +TELEGRAM_WEBHOOK_SECRET_TOKEN= diff --git a/config.py b/config.py index a17bd0b..8adb219 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.773" +SYSTEM_VERSION = "V10.775" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/guides/pchome_ai_automation_priority_backlog.md b/docs/guides/pchome_ai_automation_priority_backlog.md index 5f4a14b..045720f 100644 --- a/docs/guides/pchome_ai_automation_priority_backlog.md +++ b/docs/guides/pchome_ai_automation_priority_backlog.md @@ -1,9 +1,37 @@ # PChome AI 自動化優先工作清單 -> 最後更新: 2026-07-09 Asia/Taipei +> 最後更新: 2026-07-10 Asia/Taipei > 範圍: PChome growth / MOMO mapping AI 自動化主線 > 真相來源: Gitea `dev` / `main`、正式環境 `/health`、正式 DB readback、machine-verifiable artifacts +> 狀態更新：本文件後段大量 preflight / guard / package 紀錄只保留為歷史證據， +> 不再視為業績方案完成度。現行 program truth 以 +> `governance/pchome_growth_program_review.json` 與下列 2026-07-10 重排為準， +> `superseded_by=global_product_governance_v2`。 + +## 2026-07-10 業績閉環重排 + +正式環境基線：業績最新日 `2026-06-24`，落後 16 天；13,032 個有業績商品中， +只有 35 個曾有外部報價，覆蓋約 0.27%。因此目前不能宣稱「PChome 業績提升完整方案已完成」。 + +| 順序 | 工作主線 | 狀態 | 下一個完成條件 | +|---|---|---|---| +| P0-1 | 授權業績資料自動取得與匯入 | 進行中 | 最新業績 <= 1 天，source receipt 與 post-verifier 同 run 完成 | +| P0-2 | 全商品身份圖譜與營收加權對應 | 進行中 | 覆蓋 80% 營收商品的 >= 90% | +| P0-3 | 合法多電商資料取得 | 進行中 | 每個啟用來源具 freshness SLO、normalize、matcher、verifier | +| P0-4 | 多平台營收決策引擎 | 程式已完成，待推版回讀 | 正式環境以至少 2 個已啟用平台產生可重播決策 | +| P0-5 | 調價、促銷、內容、曝光實驗與歸因 | 未開始 | 同 run 完成 apply、轉換驗證、增量營收歸因與 rollback | +| P1-1 | 專業 UI/UX 作戰 cockpit | 程式已完成，待推版與視覺 QA | desktop/mobile 首屏無文字牆、無假圖、只顯示一個主要下一步 | +| P1-2 | Ollama-first MCP / RAG / PixelRAG runtime | 進行中 | 正式 flags 啟用，PixelRAG receipt 可 replay，Qwen3-VL 驗證通過 | +| P1-3 | 資安與治理自動化 | 進行中 | low/medium/high controlled apply 自動閉環，critical 才 break-glass | + +已完成且本輪保留：Google Drive 自動匯入、MOMO 同款與單位價資料模型、 +PixelRAG blocked-page 分類、外部報價正規化、正式 DB 安全邊界。 + +本輪修復：來源 seed 不再覆寫 runtime 啟用狀態、營收服務不再只查 MOMO、 +新增 Yahoo / ETMall / friDay / Rakuten PixelRAG alias、多平台 evidence 與 actionable 分層、 +業績 freshness gate、全 catalog 覆蓋率、漏跑排程追趕，以及首屏來源摘要與過期資料阻擋。 + ## 不可遺漏的插入需求 以下是使用者在主線推進期間插入、且必須保留在完整工作項目裡的要求。後續不得再把這些需求漏掉或變成口頭提醒。 diff --git a/docs/memory/code_modularization_inventory_20260430.md b/docs/memory/code_modularization_inventory_20260430.md index a9a7900..e7d754a 100644 --- a/docs/memory/code_modularization_inventory_20260430.md +++ b/docs/memory/code_modularization_inventory_20260430.md @@ -122,6 +122,62 @@ | 861 | `routes/market_intel_review_post_routes.py` | P2 market intel review post route extension | review post route glue / canonical alias decorators / AI summary persistence preview handoff | | 813 | `services/momo_crawler.py` | P2 MOMO crawler | targeted search / browser fallback / parsing 分離 | +### 2026-07-10 Live Addendum + +下表是本次完整 review 的機械式行數回讀；同一路徑若也出現在上表，以下表較新的數字為準。 +這份更新只修正 inventory truth，不把巨型檔案誤標成已完成重構。 + +| 行數 | 檔案 | 分類 | 拆分方向 | +|---:|---|---|---| +| 44888 | `services/pchome_mapping_backlog_service.py` | P0 gate 巨型 service | 立即停止新增 closeout package；拆 policy、executor、verifier、receipt、reporter | +| 14289 | `services/ai_automation_smoke_service.py` | P0 smoke 巨型 service | 拆 family registry、collectors、health projection、metrics adapter | +| 9442 | `routes/openclaw_bot_routes.py` | P0 巨型 Blueprint | route、command、report、scheduler hook 分離 | +| 7641 | `routes/ai_routes.py` | P0 巨型 Blueprint | PChome growth、AI automation、recommendation route extension 分離 | +| 5865 | `services/marketplace_product_matcher.py` | P0 matcher 巨型 service | identity、variant、unit、scoring、explanation 分離 | +| 5513 | `services/ppt_generator.py` | P0 報表巨型 service | orchestration、slides、charts、registry 分離 | +| 4158 | `routes/admin_observability_routes.py` | P0 觀測台巨型 Blueprint | query、action、render context 與 route glue 分離 | +| 3763 | `services/competitor_price_feeder.py` | P0 feeder 巨型 service | acquisition、matching、refresh、persistence、receipt 分離 | +| 3322 | `routes/dashboard_routes.py` | P0 Dashboard Blueprint | query、decision projection、review action 分離 | +| 3290 | `routes/sales_routes.py` | P0 Sales Blueprint | chart、query、calendar、export 分離 | +| 3112 | `scheduler.py` | P0 排程總管 | task registry 與 domain jobs 分離 | +| 2961 | `services/openclaw_strategist_service.py` | P1 strategist | prompt、query、report、notification 分離 | +| 2383 | `services/competitor_intel_repository.py` | P1 repository | query、decision envelope、UI projection、cache 分離 | +| 2332 | `services/elephant_alpha_autonomous_engine.py` | P1 engine | trigger、planner、executor、notification 分離 | +| 2166 | `services/nemoton_dispatcher_service.py` | P1 dispatcher | model client、tool parser、decision、delivery 分離 | +| 2011 | `services/external_market_offer_service.py` | P1 market offers | source registry、normalizer、sync、review、readiness 分離 | +| 1917 | `services/market_intel/deployment_readiness.py` | P1 readiness | check registry、runtime probe、projection 分離 | +| 1658 | `routes/market_intel_review_report_routes.py` | P1 review route | report query、export、route glue 分離 | +| 1494 | `routes/vendor_routes.py` | P1 Vendor Blueprint | stockout mutation、email、route glue 分離 | +| 1456 | `services/ollama_service.py` | P1 AI runtime | routing、health、request、fallback 分離 | +| 1417 | `services/telegram_templates.py` | P1 templates | decision、incident、report renderer 分離 | +| 1409 | `services/import_service.py` | P1 import | acquisition、validation、writer、verifier 分離 | +| 1394 | `services/telegram_bot_service.py` | P1 bot | command、formatter、client 分離 | +| 1370 | `run_scheduler.py` | P1 scheduler entry | registration、catchup、observability wrappers 分離 | +| 1306 | `routes/system_public_routes.py` | P1 public routes | health、metrics、public projection 分離 | +| 1290 | `app.py` | P1 bootstrap | app factory 與 startup guards 分離 | +| 1272 | `services/code_review_pipeline_service.py` | P2 code review | scan、normalize、persist、report 分離 | +| 1202 | `services/pchome_revenue_growth_service.py` | P1 revenue growth | sales query、offer comparison、scoring、projection 分離 | +| 1188 | `services/ppt_auto_generation_service.py` | P2 PPT automation | resolver、queue、generator、catchup 分離 | +| 1121 | `services/pixelrag_crawler_integration_service.py` | P1 PixelRAG | capture routing、manifest、queue、readback 分離 | +| 1023 | `routes/cicd_routes.py` | P2 CI/CD Blueprint | query、deploy action、route glue 分離 | +| 1014 | `services/pchome_crawler.py` | P1 crawler | transport、parser、fallback、rate policy 分離 | +| 1014 | `routes/daily_sales_routes.py` | P2 Daily Sales Blueprint | query、format、export、route glue 分離 | +| 1013 | `routes/export_routes.py` | P2 Export Blueprint | command、path policy、download 分離 | +| 989 | `routes/market_intel_routes.py` | P2 Market Intel Blueprint | page、API、MCP registration 分離 | +| 966 | `services/trend_crawler.py` | P2 trend crawler | adapters、parser、persistence 分離 | +| 942 | `services/learning_pipeline.py` | P2 RAG learning | distill、promotion、persistence、telemetry 分離 | +| 906 | `services/competitor_match_attempt_rescore_audit.py` | P2 rescore audit | query、classifier、projection 分離 | +| 895 | `routes/market_intel_review_routes.py` | P2 review Blueprint | command、query、route glue 分離 | +| 867 | `services/token_report_service.py` | P2 token report | query、aggregate、chart、notify 分離 | +| 864 | `services/ppt_vision_service.py` | P2 vision QA | runtime、queue、probe、audit 分離 | +| 862 | `services/security_governance_review_service.py` | P1 governance | inventory、controls、runtime receipt、scorecard 分離 | +| 861 | `routes/market_intel_review_post_routes.py` | P2 review POST routes | command handlers 與 verifier 分離 | +| 846 | `services/market_intel/candidate_queue_review_ai_summary_persistence_telegram_dispatch_report_catalog_record_run_receipt.py` | P2 receipt pipeline | AI summary、persistence、delivery、catalog 分離 | +| 843 | `services/pixelrag_vlm_replay_worker_service.py` | P1 VLM worker | route selection、inference、validation、receipt 分離 | +| 826 | `routes/market_intel_mcp_run_routes.py` | P2 MCP route extension | run command 與 receipt query 分離 | +| 813 | `services/momo_crawler.py` | P2 MOMO crawler | transport、parser、normalization 分離 | +| 801 | `services/ai_surface_html_readback_service.py` | P2 UI readback | contract registry、HTML probe、result projection 分離 | + ## 市場情報開發前置禁區 - 不得把跨平台活動頁、商品池、比對審核 API 新增到 `routes/sales_routes.py`、`routes/dashboard_routes.py` 或 `app.py`。 diff --git a/governance/pchome_growth_program_review.json b/governance/pchome_growth_program_review.json new file mode 100644 index 0000000..1d9b319 --- /dev/null +++ b/governance/pchome_growth_program_review.json @@ -0,0 +1,222 @@ +{ + "version": "pchome_growth_program_review_v1", + "generated_at": "2026-07-10T20:43:51+08:00", + "supersedes_status_only_completion": true, + "goal": "以新鮮 PChome 業績、可信跨平台同款資料與可歸因動作持續提升業績", + "production_baseline": { + "runtime_source_sha": "b4284693e4a86057e24daa3aa2fb3494f046edae", + "runtime_version": "V10.773", + "latest_sales_date": "2026-06-24", + "sales_age_days": 16, + "catalog_product_count": 13032, + "catalog_products_with_external_offer": 35, + "catalog_external_offer_coverage_pct": 0.27, + "top_opportunity_count": 20, + "top_opportunity_actionable_mapped_count": 5, + "top_opportunity_mapping_rate_pct": 25.0, + "external_offer_sources_with_verified_rows": [ + "momo_reference" + ], + "verified_external_offer_count": 473, + "verified_external_product_count": 394, + "runtime_mcp_router_enabled": false, + "runtime_rag_enabled": false + }, + "codebase_findings": { + "relevant_file_count_approx": 250, + "market_intel_module_count": 140, + "market_intel_line_count": 53988, + "mapping_backlog_service_line_count": 44888, + "mapping_backlog_service_function_count": 297, + "mapping_backlog_gate_marker_count": 1317, + "ai_routes_pchome_growth_route_count": 96, + "primary_root_causes": [ + "status and gate accumulation is much larger than the production data and action loop", + "sales import polls Google Drive but does not acquire the report from the authorized upstream", + "external source seeds overwrote runtime activation on every sync", + "revenue scoring only selected momo_reference even when other normalized offers existed", + "daily mapping backfill could be missed for a full day after a late container restart", + "the first viewport did not block decisions when sales data was stale", + "the source UI expanded every paused platform into a text-heavy card" + ] + }, + "work_items": [ + { + "order": 1, + "id": "PCH-GROWTH-P0-001", + "priority": "P0", + "status": "in_progress", + "lane": "authorized_sales_acquisition", + "objective": "Automatically acquire and import the latest PChome sales report", + "completed": [ + "Google Drive polling and import", + "post-import table verification", + "staleness detection" + ], + "remaining": [ + "authorized PChome backend, scheduled export, or mailbox-to-Drive acquisition connector", + "same-run source receipt and freshness verifier" + ], + "production_acceptance": "latest sales age <= 1 day and decision_ready=true" + }, + { + "order": 2, + "id": "PCH-GROWTH-P0-002", + "priority": "P0", + "status": "in_progress", + "lane": "product_identity_graph", + "objective": "Map the PChome catalog to verified same-product offers across platforms", + "completed": [ + "MOMO exact and unit-price evidence model", + "quality and variant gates", + "bounded high-sales backfill" + ], + "remaining": [ + "raise full-catalog coverage from 0.27%", + "brand, GTIN, MPN, model, size, count, color and seller identity graph", + "automatic replay for unresolved high-sales products" + ], + "production_acceptance": "top revenue 80% weighted catalog coverage >= 90%" + }, + { + "order": 3, + "id": "PCH-GROWTH-P0-003", + "priority": "P0", + "status": "in_progress", + "lane": "lawful_multiplatform_acquisition", + "objective": "Continuously acquire normalized offer evidence from relevant commerce platforms", + "completed": [ + "15-source governed registry", + "MOMO normalized offers", + "PixelRAG evidence lane with blocked-page classification", + "Yahoo, ETMall and friDay public capture feasibility proof" + ], + "remaining": [ + "authorized or structured-data adapters for Yahoo, ETMall and friDay", + "Shopee authorized feed or seller API adapter", + "Coupang official API eligibility and 403 backoff path", + "Google Merchant price competitiveness and market insights eligibility", + "Rakuten retry and backoff adapter", + "Ruten and LINE Shopping feed contracts" + ], + "production_acceptance": "each active source has source receipt, normalized offers, verifier and freshness SLO" + }, + { + "order": 4, + "id": "PCH-GROWTH-P0-004", + "priority": "P0", + "status": "source_complete_runtime_pending", + "lane": "cross_platform_revenue_decision", + "objective": "Compare every qualified platform offer without letting blocked or observational evidence drive price actions", + "completed": [ + "multi-source latest-offer selection", + "runtime source-policy enforcement", + "observation-only evidence separation", + "sales freshness decision gate", + "catalog and platform coverage metrics" + ], + "remaining": [ + "Gitea deployment", + "production API readback", + "production multi-source offer ingestion" + ], + "production_acceptance": "fresh sales plus at least two enabled verified platforms produce replayable decisions" + }, + { + "order": 5, + "id": "PCH-GROWTH-P0-005", + "priority": "P0", + "status": "not_started", + "lane": "growth_action_and_attribution", + "objective": "Turn recommendations into bounded price, promotion, content and exposure experiments and measure incremental revenue", + "completed": [], + "remaining": [ + "action contract and authorized PChome write adapter", + "holdout or matched-control experiment design", + "margin, stock and MAP policy guard", + "post-action conversion and incremental revenue verifier", + "automatic rollback and cooldown" + ], + "production_acceptance": "same run closes decide, apply, verify, attribute and learn with rollback evidence" + }, + { + "order": 6, + "id": "PCH-GROWTH-P1-001", + "priority": "P1", + "status": "source_complete_runtime_pending", + "lane": "professional_ui_ux", + "objective": "Provide a compact decision cockpit instead of a status and text wall", + "completed": [ + "freshness-first next action", + "multi-platform terminology", + "real coverage metrics", + "collapsed pending-source summary", + "removed fake sales sparkline", + "desktop and 390px mobile visual verification without horizontal overflow", + "repaired the circular accent-token fallback that made primary buttons unreadable" + ], + "remaining": [ + "production visible readback", + "per-product cross-platform offer drawer and experiment outcome view" + ], + "production_acceptance": "first viewport shows freshness, revenue trend, coverage, risk and one executable next action without overflow" + }, + { + "order": 7, + "id": "PCH-GROWTH-P1-002", + "priority": "P1", + "status": "in_progress", + "lane": "ai_mcp_rag_runtime", + "objective": "Use Ollama-first agents, MCP and RAG to automate retrieval, matching, validation and learning", + "completed": [ + "internal MCP and RAG inventory", + "PixelRAG receipts and capture manifests", + "text RAG service implementation" + ], + "remaining": [ + "enable and verify MCP router runtime", + "enable and verify RAG runtime", + "Qwen3-VL Ollama-first visual replay", + "PixelRAG receipt to RAG candidate replay", + "agent outcome metrics and durable learning acknowledgement" + ], + "production_acceptance": "runtime flags enabled and same-run retrieval-to-decision receipts pass independent replay" + }, + { + "order": 8, + "id": "PCH-GROWTH-P1-003", + "priority": "P1", + "status": "in_progress", + "lane": "security_and_governance_automation", + "objective": "Apply risk-based controlled execution with asset, policy, verifier, rollback and learning coverage", + "completed": [ + "source and runtime review artifacts", + "read-only visual evidence boundary", + "production DB lifecycle red lines" + ], + "remaining": [ + "replace preview-only gate chains with executable work items", + "shared trace_id, run_id and work_item_id envelope", + "SBOM and dependency policy enforcement", + "automated drift work item creation", + "security control coverage and production closure metrics" + ], + "production_acceptance": "low, medium and high actions close automatically under policy; only critical actions require break-glass" + } + ], + "source_strategy": { + "priority_order": [ + "official or explicitly authorized API and feed", + "public structured product data that permits access", + "PixelRAG visual evidence with OCR or VLM replay", + "blocked-page classification, backoff and alternate source" + ], + "hard_rules": [ + "never bypass anti-bot controls", + "never treat a blocked page as product evidence", + "never write PixelRAG-only prices into the formal price table", + "never allow a paused or disabled source to drive a pricing action", + "never execute a sales action with stale PChome sales data" + ] + } +} diff --git a/run_scheduler.py b/run_scheduler.py index bf4c499..85677e3 100644 --- a/run_scheduler.py +++ b/run_scheduler.py @@ -17,6 +17,7 @@ run_scheduler.py — momo-scheduler 容器入口點 每 1 年 ：ppt_auto_generation_annual（1 月 1 日 21:20） """ import asyncio +import json import logging import os import threading @@ -60,6 +61,66 @@ _AI_CALLS_ERROR_SPIKE_LAST_PUSH_TS = 0.0 _OLLAMA_111_USAGE_LAST_PUSH_TS = 0.0 +def _scheduler_stats_path() -> str: + return os.getenv( + "SCHEDULER_STATS_PATH", + os.path.join(os.path.dirname(__file__), "data", "scheduler_stats.json"), + ) + + +def _pchome_growth_backfill_due(*, now=None) -> tuple[bool, str]: + """Return whether the bounded PChome mapping run should catch up.""" + from datetime import datetime as _dt, timedelta as _timedelta, timezone as _timezone + + current = now or _dt.now(_timezone(_timedelta(hours=8))).replace(tzinfo=None) + try: + with open(_scheduler_stats_path(), "r", encoding="utf-8") as handle: + stats = json.load(handle) + except (FileNotFoundError, json.JSONDecodeError, OSError): + return True, "missing_runtime_receipt" + + history = stats.get("pchome_growth_momo_backfill") or [] + latest = history[0] if isinstance(history, list) and history else {} + last_run_raw = str(latest.get("last_run") or "").strip() + if not last_run_raw: + return True, "missing_last_run" + try: + last_run = _dt.strptime(last_run_raw, "%Y-%m-%d %H:%M:%S") + except ValueError: + return True, "invalid_last_run" + + status = str(latest.get("status") or "").lower() + cooldown_hours = float(os.getenv("PCHOME_GROWTH_BACKFILL_COOLDOWN_HOURS", "4")) + if status == "failed": + cooldown_hours = float(os.getenv("PCHOME_GROWTH_BACKFILL_RETRY_HOURS", "1")) + elapsed_hours = max(0.0, (current - last_run).total_seconds() / 3600) + if elapsed_hours >= cooldown_hours: + return True, f"stale_receipt_{elapsed_hours:.1f}h" + return False, f"cooldown_{cooldown_hours:.1f}h" + + +def run_pchome_growth_momo_backfill_catchup_task(): + """Catch up a missed daily run without creating an unbounded crawl loop.""" + try: + due, reason = _pchome_growth_backfill_due() + if not due: + logger.debug("[PChomeGrowthBackfillCatchup] skipped reason=%s", reason) + return {"status": "skipped", "reason": reason} + logger.info("[PChomeGrowthBackfillCatchup] running reason=%s", reason) + run_pchome_growth_momo_backfill_task() + return {"status": "executed", "reason": reason} + except Exception as error: + logger.error("[PChomeGrowthBackfillCatchup] task failed: %s", error, exc_info=True) + _notify_scheduler_failure( + "run_pchome_growth_momo_backfill_catchup_task", + error, + source="Scheduler.PChomeGrowthBackfillCatchup", + event_type="pchome_growth_backfill_catchup_failure", + title="PChome 高業績商品補抓追趕失敗", + ) + return {"status": "failed", "error": str(error)} + + def _env_flag(name: str, default: bool = False) -> bool: raw = os.getenv(name) if raw is None: @@ -312,6 +373,9 @@ def _register_schedules(): schedule.every().day.at("10:45").do(run_pchome_growth_momo_backfill_task) logger.info("📅 每日 10:45：pchome_growth_momo_backfill（高業績商品補 MOMO 對應）") + schedule.every(30).minutes.do(run_pchome_growth_momo_backfill_catchup_task) + logger.info("📅 每 30 分鐘：pchome_growth_momo_backfill_catchup（錯過每日任務時自動補跑）") + # Operation Ollama-First v5.0 — Phase 1 收尾：每日 23:55 LLM Token 日報 schedule.every().day.at("23:55").do(run_daily_token_report_task) logger.info("📅 每日 23:55：daily_token_report") diff --git a/scheduler.py b/scheduler.py index 59ac206..383cdeb 100644 --- a/scheduler.py +++ b/scheduler.py @@ -94,7 +94,10 @@ def _safe_driver_title(driver, task_label, url): def _save_stats(task_name, data): """將任務統計結果寫入 JSON 檔案""" - stats_file = os.path.join(os.path.dirname(__file__), 'data', 'scheduler_stats.json') + stats_file = os.getenv( + 'SCHEDULER_STATS_PATH', + os.path.join(os.path.dirname(__file__), 'data', 'scheduler_stats.json'), + ) try: stats = {} if os.path.exists(stats_file): @@ -2043,7 +2046,12 @@ def run_auto_import_task(): stats = { "file_count": result.get('file_count', 0), "imported_count": result.get('imported_count', 0), - "status": "Success" + "status": "Degraded" if result.get('status') in {'upstream_stale', 'upstream_missing'} else "Success", + "upstream_status": result.get('status'), + "latest_sales_date": result.get('latest_sales_date'), + "data_lag_days": result.get('data_lag_days'), + "decision_ready": result.get('decision_ready'), + "requires_upstream_acquisition": result.get('requires_upstream_acquisition', False), } # V-New: 發送 Telegram 和 Line 通知（僅當有匯入新檔案時） diff --git a/services/ai_automation_debt_service.py b/services/ai_automation_debt_service.py index 14e64d7..8900cf7 100644 --- a/services/ai_automation_debt_service.py +++ b/services/ai_automation_debt_service.py @@ -139,7 +139,10 @@ def _relative(path: Path, root: Path) -> str: def _is_excluded(relative_path: str) -> bool: - if relative_path == "services/ai_automation_debt_service.py": + if relative_path in { + "services/ai_automation_debt_service.py", + "services/ai_surface_html_readback_service.py", + }: return True parts = set(relative_path.split("/")) if parts & {".git", ".pytest_cache", "__pycache__", "node_modules", "data", "tests", "migrations"}: diff --git a/services/external_market_offer_service.py b/services/external_market_offer_service.py index 62fd249..912e629 100644 --- a/services/external_market_offer_service.py +++ b/services/external_market_offer_service.py @@ -130,6 +130,28 @@ SOURCE_CONTRACTS = [ "data_quality_label": "暫不進告警", "plain_note": "保留本地平台促銷與價格監控欄位，接入前不影響現有作戰清單。", }, + { + "code": "etmall", + "display_name": "東森購物", + "platform_code": "etmall", + "status_code": "paused", + "status_label": "待接入", + "source_kind": "connector_contract", + "input_methods": ["官方 API", "公開結構化資料", "供應商 API", "手動 CSV"], + "data_quality_label": "暫不進告警", + "plain_note": "保留東森購物的商品、售價與促銷訊號；完成來源授權與商品身份驗證後才進入決策。", + }, + { + "code": "friday", + "display_name": "friDay 購物", + "platform_code": "friday", + "status_code": "paused", + "status_label": "待接入", + "source_kind": "connector_contract", + "input_methods": ["官方 API", "公開結構化資料", "供應商 API", "手動 CSV"], + "data_quality_label": "暫不進告警", + "plain_note": "保留 friDay 購物的商品、售價與促銷訊號；完成來源授權與商品身份驗證後才進入決策。", + }, { "code": "ruten", "display_name": "露天", @@ -533,6 +555,22 @@ def _normalize_source_code(value: Any) -> str: "shopee": "shopee", "酷澎": "coupang", "coupang": "coupang", + "yahoo": "yahoo_shopping", + "yahoo_shopping": "yahoo_shopping", + "yahoo_shopping_tw": "yahoo_shopping", + "yahoo購物": "yahoo_shopping", + "yahoo 購物": "yahoo_shopping", + "東森": "etmall", + "東森購物": "etmall", + "etmall": "etmall", + "etmall_tw": "etmall", + "friday": "friday", + "friday_tw": "friday", + "friday購物": "friday", + "friday 購物": "friday", + "樂天": "rakuten", + "rakuten": "rakuten", + "rakuten_tw": "rakuten", } return mapping.get(lower, mapping.get(raw, raw)) @@ -677,8 +715,6 @@ def _ensure_external_market_source_seeds(conn) -> None: display_name = EXCLUDED.display_name, platform_code = EXCLUDED.platform_code, source_kind = EXCLUDED.source_kind, - status = EXCLUDED.status, - enabled = EXCLUDED.enabled, allowed_input_methods_json = EXCLUDED.allowed_input_methods_json, quality_policy_json = EXCLUDED.quality_policy_json, plain_note = EXCLUDED.plain_note, @@ -698,8 +734,6 @@ def _ensure_external_market_source_seeds(conn) -> None: display_name = excluded.display_name, platform_code = excluded.platform_code, source_kind = excluded.source_kind, - status = excluded.status, - enabled = excluded.enabled, allowed_input_methods_json = excluded.allowed_input_methods_json, quality_policy_json = excluded.quality_policy_json, plain_note = excluded.plain_note, @@ -1709,6 +1743,17 @@ def build_external_source_readiness(engine=None) -> dict[str, Any]: for source in sources: stats = normalized_stats.get(source["code"], {}) + runtime_status = str(stats.get("status") or source["status_code"]).strip() or source["status_code"] + runtime_enabled = bool(stats.get("enabled")) if stats else source["status_code"] == "active" + source["status_code"] = runtime_status + source["enabled"] = runtime_enabled + source["status_label"] = ( + "正在使用" + if runtime_status == "active" and runtime_enabled + else "已停用" + if runtime_status in {"disabled", "retired"} + else "先暫停" + ) if source["code"] == "momo_reference": usable = max( int(stats.get("usable_offer_count") or 0), @@ -1723,8 +1768,9 @@ def build_external_source_readiness(engine=None) -> dict[str, Any]: source["usable_offer_count"] = usable source["review_offer_count"] = int(stats.get("review_offer_count") or 0) source["last_seen_at"] = last_seen_at - source["can_alert"] = source["status_code"] == "active" and usable > 0 - if source["status_code"] == "active": + source["has_runtime_data"] = usable > 0 or source["review_offer_count"] > 0 + source["can_alert"] = runtime_status == "active" and runtime_enabled and usable > 0 + if runtime_status == "active" and runtime_enabled: if usable: source["plain_state"] = "已接入，可進作戰清單" elif source["review_offer_count"]: @@ -1734,8 +1780,12 @@ def build_external_source_readiness(engine=None) -> dict[str, Any]: else: source["plain_state"] = "先保留接口，不進告警" - active_count = sum(1 for source in sources if source["status_code"] == "active") - paused_count = sum(1 for source in sources if source["status_code"] == "paused") + active_count = sum( + 1 for source in sources + if source["status_code"] == "active" and source.get("enabled") + ) + paused_count = len(sources) - active_count + sources_with_data_count = sum(1 for source in sources if source.get("has_runtime_data")) usable_count = sum(int(source["usable_offer_count"]) for source in sources) review_offer_count = sum(int(source.get("review_offer_count") or 0) for source in sources) @@ -1744,12 +1794,17 @@ def build_external_source_readiness(engine=None) -> dict[str, Any]: "schema_ready": schema_ready, "active_count": active_count, "paused_count": paused_count, + "sources_with_data_count": sources_with_data_count, + "source_coverage_rate": round(sources_with_data_count / max(len(sources), 1) * 100, 1), "usable_offer_count": usable_count, "review_offer_count": review_offer_count, "sources": sources, "connector_contract": build_connector_contracts(), "offer_evidence_contract": build_offer_evidence_contract(), - "plain_summary": "MOMO 先用；其他主流平台已列管，未接合法穩定來源前不進告警。", + "plain_summary": ( + f"已列管 {len(sources)} 個來源，{sources_with_data_count} 個已有資料；" + "只有 runtime 已啟用且通過身份與品質驗證的報價會進入決策。" + ), } diff --git a/services/import_service.py b/services/import_service.py index 176685c..23dae7c 100644 --- a/services/import_service.py +++ b/services/import_service.py @@ -1143,6 +1143,8 @@ class ImportService: if not files: logger.info("沒有找到待匯入的檔案") + data_lag_days = None + latest_sales_date = None # Staleness gate (critic-approved 2026-05-03) # 'move-then-success' 反模式：成功 import 後 move_file 把 Excel 搬到 @@ -1150,25 +1152,31 @@ class ImportService: # → 4/27~5/2 daily_sales_snapshot 停更 8 天無告警。補主動偵測： # Drive 空 + DB ≥3 天無新資料時主動發催促告警（週末跨假期不誤觸）。 try: - from database.manager import get_session from sqlalchemy import text from datetime import date from services.openclaw_strategist_service import _send_data_stale_alert - _stale_session = get_session() + _stale_session = Session() try: last_date = _stale_session.execute( - text("SELECT MAX(snapshot_date)::date FROM daily_sales_snapshot") + text("SELECT MAX(snapshot_date) FROM daily_sales_snapshot") ).scalar() finally: _stale_session.close() if last_date: - days_since = (date.today() - last_date).days + normalized_last_date = ( + last_date + if isinstance(last_date, date) + else date.fromisoformat(str(last_date)[:10]) + ) + days_since = (date.today() - normalized_last_date).days + data_lag_days = days_since + latest_sales_date = str(normalized_last_date) if days_since >= 3: _send_data_stale_alert( report_type="upstream_drive", - last_date=str(last_date), + last_date=str(normalized_last_date), period=f"已停更 {days_since} 天", ) except Exception: @@ -1179,8 +1187,24 @@ class ImportService: return { 'success': True, - 'message': '沒有找到待匯入的檔案', - 'file_count': 0 + 'status': ( + 'upstream_missing' + if data_lag_days is None + else 'upstream_stale' + if data_lag_days >= 3 + else 'no_pending_file' + ), + 'message': ( + f'上游沒有新檔，最新業績已落後 {data_lag_days} 天' + if data_lag_days is not None and data_lag_days >= 3 + else '沒有找到待匯入的檔案' + ), + 'file_count': 0, + 'imported_count': 0, + 'latest_sales_date': latest_sales_date, + 'data_lag_days': data_lag_days, + 'decision_ready': data_lag_days is not None and data_lag_days <= 1, + 'requires_upstream_acquisition': data_lag_days is None or data_lag_days >= 3, } # 處理每個檔案 diff --git a/services/pchome_revenue_growth_service.py b/services/pchome_revenue_growth_service.py index 6f227c2..0d5b438 100644 --- a/services/pchome_revenue_growth_service.py +++ b/services/pchome_revenue_growth_service.py @@ -6,7 +6,7 @@ from __future__ import annotations import json import logging -from datetime import datetime +from datetime import date, datetime from typing import Any from sqlalchemy import bindparam, inspect, text @@ -54,10 +54,53 @@ def _source_names_by_status(source_readiness: dict[str, Any], status_code: str) return [ str(source.get("display_name") or source.get("code") or "").strip() for source in source_readiness.get("sources") or [] - if source.get("status_code") == status_code and str(source.get("display_name") or source.get("code") or "").strip() + if source.get("status_code") == status_code + and (status_code != "active" or source.get("enabled", True)) + and str(source.get("display_name") or source.get("code") or "").strip() ] +def _sales_freshness(latest_sales_date: Any, *, today: date | None = None) -> dict[str, Any]: + raw = str(latest_sales_date or "").strip() + if not raw: + return { + "status": "missing", + "label": "尚無業績資料", + "age_days": None, + "decision_ready": False, + "next_action": "自動取得並匯入最新 PChome 業績檔", + } + try: + parsed = date.fromisoformat(raw[:10]) + except ValueError: + return { + "status": "invalid", + "label": "業績日期格式異常", + "age_days": None, + "decision_ready": False, + "next_action": "修正業績資料日期後重新匯入", + } + + age_days = max(0, ((today or datetime.now().date()) - parsed).days) + if age_days <= 1: + status, label, ready = "fresh", "資料新鮮", True + elif age_days == 2: + status, label, ready = "warning", "資料即將過期", False + else: + status, label, ready = "critical", "資料已過期", False + return { + "status": status, + "label": label, + "age_days": age_days, + "decision_ready": ready, + "next_action": ( + "依最新業績執行今日作戰清單" + if ready + else "自動取得並匯入最新 PChome 業績檔" + ), + } + + def _table_exists(engine, table_name: str) -> bool: try: return inspect(engine).has_table(table_name) @@ -336,7 +379,7 @@ def _external_price_basis_label(price_basis: str) -> str: return "單位價" if price_basis == "unit_price" else "商品總價" -def _fetch_normalized_external_price_map(conn, pchome_product_ids: list[str]) -> dict[str, dict[str, Any]]: +def _fetch_normalized_external_offer_map(conn, pchome_product_ids: list[str]) -> dict[str, list[dict[str, Any]]]: inspector = inspect(conn) if not inspector.has_table("external_offers"): return {} @@ -347,10 +390,13 @@ def _fetch_normalized_external_price_map(conn, pchome_product_ids: list[str]) -> if conn.dialect.name == "postgresql": sql = """ WITH latest_offer AS ( - SELECT DISTINCT ON (eo.pchome_product_id) + SELECT DISTINCT ON (eo.pchome_product_id, eo.source_code) eo.pchome_product_id, + eo.source_code, + eo.platform_code, eo.source_product_id AS momo_sku, eo.title AS momo_name, + eo.product_url, eo.price AS momo_price, eo.quality_score, eo.quality_notes_json, @@ -358,8 +404,7 @@ def _fetch_normalized_external_price_map(conn, pchome_product_ids: list[str]) -> eo.observed_at, eo.ingestion_method FROM external_offers eo - WHERE eo.source_code = 'momo_reference' - AND eo.pchome_product_id IS NOT NULL + WHERE eo.pchome_product_id IS NOT NULL AND eo.pchome_product_id IN :ids AND eo.price IS NOT NULL AND eo.price > 0 @@ -367,7 +412,7 @@ def _fetch_normalized_external_price_map(conn, pchome_product_ids: list[str]) -> AND eo.match_status IN ('verified', 'usable', 'reviewed', 'exact', 'confirmed') AND eo.data_quality_status IN ('verified', 'usable', 'reviewed') AND (eo.expires_at IS NULL OR eo.expires_at > CURRENT_TIMESTAMP) - ORDER BY eo.pchome_product_id, eo.observed_at DESC NULLS LAST, eo.id DESC + ORDER BY eo.pchome_product_id, eo.source_code, eo.observed_at DESC NULLS LAST, eo.id DESC ) SELECT * FROM latest_offer """ @@ -376,8 +421,11 @@ def _fetch_normalized_external_price_map(conn, pchome_product_ids: list[str]) -> WITH latest_offer AS ( SELECT eo.pchome_product_id, + eo.source_code, + eo.platform_code, eo.source_product_id AS momo_sku, eo.title AS momo_name, + eo.product_url, eo.price AS momo_price, eo.quality_score, eo.quality_notes_json, @@ -385,12 +433,11 @@ def _fetch_normalized_external_price_map(conn, pchome_product_ids: list[str]) -> eo.observed_at, eo.ingestion_method, ROW_NUMBER() OVER ( - PARTITION BY eo.pchome_product_id + PARTITION BY eo.pchome_product_id, eo.source_code ORDER BY eo.observed_at DESC, eo.id DESC ) AS rn FROM external_offers eo - WHERE eo.source_code = 'momo_reference' - AND eo.pchome_product_id IS NOT NULL + WHERE eo.pchome_product_id IS NOT NULL AND eo.pchome_product_id IN :ids AND eo.price IS NOT NULL AND eo.price > 0 @@ -405,7 +452,7 @@ def _fetch_normalized_external_price_map(conn, pchome_product_ids: list[str]) -> stmt = text(sql).bindparams(bindparam("ids", expanding=True)) rows = conn.execute(stmt, {"ids": ids}).mappings().all() - result: dict[str, dict[str, Any]] = {} + result: dict[str, list[dict[str, Any]]] = {} for row in rows: raw_payload = _json_dict(row.get("raw_payload_json")) price_basis = _external_price_basis(raw_payload) @@ -413,11 +460,14 @@ def _fetch_normalized_external_price_map(conn, pchome_product_ids: list[str]) -> key = str(row.get("pchome_product_id") or "").strip() if not key: continue - result[key] = { + offer = { "pchome_product_id": key, + "source_code": str(row.get("source_code") or "").strip(), + "platform_code": str(row.get("platform_code") or row.get("source_code") or "").strip(), "pchome_public_name": raw_payload.get("pchome_public_name"), "momo_sku": row.get("momo_sku"), "momo_name": row.get("momo_name"), + "product_url": row.get("product_url"), "momo_price": row.get("momo_price"), "pchome_price": raw_payload.get("pchome_public_price"), "price_basis": price_basis, @@ -435,6 +485,7 @@ def _fetch_normalized_external_price_map(conn, pchome_product_ids: list[str]) -> "data_source": "external_offers", "ingestion_method": row.get("ingestion_method"), } + result.setdefault(key, []).append(offer) return result @@ -542,15 +593,138 @@ def _fetch_legacy_external_price_map(conn, pchome_product_ids: list[str]) -> dic return result -def _fetch_external_price_map(conn, pchome_product_ids: list[str]) -> dict[str, dict[str, Any]]: - normalized_map = _fetch_normalized_external_price_map(conn, pchome_product_ids) +def _external_offer_gap(offer: dict[str, Any]) -> float: + price_basis = str(offer.get("price_basis") or "total_price") + if price_basis == "unit_price": + explicit_gap = offer.get("unit_gap_pct") + if explicit_gap is not None: + return _to_float(explicit_gap) + pchome_price = _to_float(offer.get("pchome_unit_price")) + external_price = _to_float(offer.get("momo_unit_price")) + else: + pchome_price = _to_float(offer.get("pchome_price")) + external_price = _to_float(offer.get("momo_price")) + if pchome_price > 0 and external_price > 0: + return (external_price - pchome_price) / pchome_price * 100 + return 9999.0 + + +def _fetch_external_offer_maps( + conn, + pchome_product_ids: list[str], + source_readiness: dict[str, Any], +) -> tuple[dict[str, dict[str, Any]], dict[str, list[dict[str, Any]]]]: + normalized_map = _fetch_normalized_external_offer_map(conn, pchome_product_ids) + source_policy = { + str(source.get("code") or ""): source + for source in source_readiness.get("sources") or [] + } + primary_map: dict[str, dict[str, Any]] = {} + market_map: dict[str, list[dict[str, Any]]] = {} + + for product_id, offers in normalized_map.items(): + normalized_offers = [] + for offer in offers: + source_code = str(offer.get("source_code") or "").strip() + policy = source_policy.get(source_code, {}) + enriched = { + **offer, + "source_display_name": policy.get("display_name") or source_code or "外部平台", + "source_status": policy.get("status_code") or "unregistered", + "source_enabled": bool(policy.get("enabled", False)), + "actionable": bool(policy.get("can_alert")), + } + enriched["gap_pct"] = round(_external_offer_gap(enriched), 1) + normalized_offers.append(enriched) + normalized_offers.sort(key=lambda item: (_external_offer_gap(item), item.get("source_code") or "")) + market_map[product_id] = normalized_offers + actionable = [offer for offer in normalized_offers if offer.get("actionable")] + if actionable: + primary_map[product_id] = actionable[0] + missing_ids = [ str(item).strip() for item in pchome_product_ids - if str(item or "").strip() and str(item).strip() not in normalized_map + if str(item or "").strip() and str(item).strip() not in primary_map ] legacy_map = _fetch_legacy_external_price_map(conn, missing_ids) - return {**legacy_map, **normalized_map} + momo_policy = source_policy.get("momo_reference", {}) + for product_id, legacy in legacy_map.items(): + legacy_offer = { + **legacy, + "source_code": "momo_reference", + "platform_code": "momo", + "source_display_name": momo_policy.get("display_name") or "MOMO 外部價格參考", + "source_status": momo_policy.get("status_code") or "active", + "source_enabled": bool(momo_policy.get("enabled", True)), + "actionable": bool(momo_policy.get("can_alert", True)), + } + legacy_offer["gap_pct"] = round(_external_offer_gap(legacy_offer), 1) + existing_sources = { + str(item.get("source_code") or "") for item in market_map.get(product_id, []) + } + if "momo_reference" not in existing_sources: + market_map.setdefault(product_id, []).append(legacy_offer) + if legacy_offer["actionable"]: + primary_map[product_id] = legacy_offer + + for offers in market_map.values(): + offers.sort(key=lambda item: (_external_offer_gap(item), item.get("source_code") or "")) + + return primary_map, market_map + + +def _fetch_catalog_mapping_summary(conn) -> dict[str, Any]: + cols = _daily_sales_columns(conn) + sku_column = cols.get("sku") + if not sku_column: + return { + "catalog_product_count": 0, + "mapped_product_count": 0, + "catalog_mapping_rate": 0.0, + } + + sales_rows = conn.execute(text( + f"SELECT DISTINCT TRIM({_as_text_expr(sku_column, conn.dialect.name)}) AS product_id " + "FROM daily_sales_snapshot " + f"WHERE {_quote_identifier(sku_column)} IS NOT NULL" + )).fetchall() + sales_ids = {str(row[0]).strip() for row in sales_rows if str(row[0] or "").strip()} + mapped_ids: set[str] = set() + inspector = inspect(conn) + if inspector.has_table("external_offers"): + rows = conn.execute(text(""" + SELECT DISTINCT pchome_product_id + FROM external_offers + WHERE pchome_product_id IS NOT NULL + AND price IS NOT NULL + AND price > 0 + AND COALESCE(quality_score, 0) >= 76 + AND match_status IN ('verified', 'usable', 'reviewed', 'exact', 'confirmed') + AND data_quality_status IN ('verified', 'usable', 'reviewed') + AND (expires_at IS NULL OR expires_at > CURRENT_TIMESTAMP) + """)).fetchall() + mapped_ids.update(str(row[0]).strip() for row in rows if str(row[0] or "").strip()) + if inspector.has_table("competitor_prices"): + rows = conn.execute(text(""" + SELECT DISTINCT competitor_product_id + FROM competitor_prices + WHERE source = 'pchome' + AND competitor_product_id IS NOT NULL + AND price IS NOT NULL + AND price > 0 + AND COALESCE(match_score, 0) >= 0.76 + AND (expires_at IS NULL OR expires_at > CURRENT_TIMESTAMP) + """)).fetchall() + mapped_ids.update(str(row[0]).strip() for row in rows if str(row[0] or "").strip()) + + mapped_catalog_ids = sales_ids & mapped_ids + return { + "catalog_product_count": len(sales_ids), + "mapped_product_count": len(mapped_catalog_ids), + "catalog_mapping_rate": round(len(mapped_catalog_ids) / max(len(sales_ids), 1) * 100, 2), + "unmapped_product_count": max(0, len(sales_ids) - len(mapped_catalog_ids)), + } def _fetch_review_candidate_map(conn, pchome_product_ids: list[str]) -> dict[str, dict[str, Any]]: @@ -646,10 +820,32 @@ def _fetch_review_candidate_map(conn, pchome_product_ids: list[str]) -> dict[str return result +def _serialize_market_offer(offer: dict[str, Any]) -> dict[str, Any]: + gap = _external_offer_gap(offer) + return { + "source_code": offer.get("source_code") or "momo_reference", + "source": offer.get("source_display_name") or "MOMO 外部價格參考", + "platform_code": offer.get("platform_code") or "momo", + "source_product_id": offer.get("momo_sku"), + "title": offer.get("momo_name"), + "product_url": offer.get("product_url"), + "price": round(_to_float(offer.get("momo_price")), 2) if _to_float(offer.get("momo_price")) else None, + "pchome_price": round(_to_float(offer.get("pchome_price")), 2) if _to_float(offer.get("pchome_price")) else None, + "price_basis": offer.get("price_basis") or "total_price", + "gap_pct": round(gap, 1) if gap < 9999 else None, + "match_score": round(_to_float(offer.get("match_score")), 3), + "source_status": offer.get("source_status") or "active", + "actionable": bool(offer.get("actionable")), + "evidence_state": "decision_ready" if offer.get("actionable") else "observation_only", + "updated_at": str(offer.get("crawled_at") or ""), + } + + def _score_opportunity( sales_row: dict[str, Any], external_row: dict[str, Any] | None, review_candidate: dict[str, Any] | None = None, + market_offers: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: sales_7d = _to_float(sales_row.get("sales_7d")) sales_prev_7d = _to_float(sales_row.get("sales_prev_7d")) @@ -663,13 +859,15 @@ def _score_opportunity( decline_score = min(24, abs(sales_delta_pct) / 45 * 24) if sales_delta_pct is not None and sales_delta_pct < 0 else 0 data_quality_score = 54 external_payload = None + market_offer_payloads = [_serialize_market_offer(offer) for offer in (market_offers or [])] review_candidate_payload = None action_code = "map_external_product" action_label = "先補商品對應" - action_message = "這項商品已有業績訊號，但還沒有可確認的 MOMO 對照商品。先補對應，後續才能判斷價格壓力。" + action_message = "這項商品已有業績訊號，但還沒有可確認的外部平台同款。先補對應，後續才能判斷價格與促銷壓力。" reason_lines = [] if external_row: + source_name = str(external_row.get("source_display_name") or "MOMO 外部價格參考") price_basis = str(external_row.get("price_basis") or "total_price") price_basis_label = external_row.get("price_basis_label") or _external_price_basis_label(price_basis) pchome_price = _to_float(external_row.get("pchome_price")) @@ -691,7 +889,9 @@ def _score_opportunity( tags = _load_json_tags(external_row.get("tags")) data_quality_score = 78 + min(12, _to_float(external_row.get("match_score")) * 12) external_payload = { - "source": "MOMO", + "source": source_name, + "source_code": external_row.get("source_code") or "momo_reference", + "platform_code": external_row.get("platform_code") or "momo", "data_source": external_row.get("data_source") or "competitor_prices", "data_source_label": "自動同步資料層" if external_row.get("data_source") == "external_offers" @@ -716,7 +916,7 @@ def _score_opportunity( if gap_pct is not None and gap_pct < -5: action_code = "review_price_or_promo" action_label = "檢查售價與活動" - action_message = "MOMO 外部參考價比較低，建議檢查 PChome 售價、活動組合或曝光策略。" + action_message = f"{source_name}的同款價格較低，建議檢查 PChome 售價、活動組合或曝光策略。" elif gap_pct is not None and gap_pct > 5: action_code = "amplify_price_advantage" action_label = "放大價格優勢" @@ -733,11 +933,11 @@ def _score_opportunity( if gap_pct is not None: basis_prefix = "單位價" if price_basis == "unit_price" else "價格" if gap_pct > 0: - reason_lines.append(f"PChome {basis_prefix}目前比 MOMO 低約 {abs(gap_pct):.1f}%。") + reason_lines.append(f"PChome {basis_prefix}目前比{source_name}低約 {abs(gap_pct):.1f}%。") elif gap_pct < 0: - reason_lines.append(f"MOMO {basis_prefix}目前比 PChome 低約 {abs(gap_pct):.1f}%。") + reason_lines.append(f"{source_name}{basis_prefix}目前比 PChome 低約 {abs(gap_pct):.1f}%。") else: - reason_lines.append(f"PChome 與 MOMO {basis_prefix}幾乎相同。") + reason_lines.append(f"PChome 與{source_name}{basis_prefix}幾乎相同。") else: if review_candidate: review_candidate_payload = { @@ -757,8 +957,16 @@ def _score_opportunity( action_message = "已找到 MOMO 候選，但還要確認同款、色號或組合後才能進價格判斷。" reason_lines.append("已找到 MOMO 候選，先確認同款、色號或組合。") else: - data_quality_score -= 12 - reason_lines.append("尚未找到可確認的 MOMO 對照商品。") + if market_offer_payloads: + data_quality_score = max(data_quality_score, 60) + action_code = "validate_external_source" + action_label = "驗證外部來源" + action_message = "已有外部平台同款證據，但來源尚未通過啟用政策；系統會先完成來源與商品身份驗證。" + observed_sources = sorted({str(item.get("source") or "") for item in market_offer_payloads}) + reason_lines.append(f"已有 {len(observed_sources)} 個外部平台證據，尚未進入正式價格決策。") + else: + data_quality_score -= 12 + reason_lines.append("尚未找到可確認的外部平台同款商品。") if sales_delta_pct is None: reason_lines.append("前 7 天沒有可比基準，先看近 7 天表現。") @@ -772,7 +980,7 @@ def _score_opportunity( if sales_7d > 0: reason_lines.append(f"近 7 天業績約 NT$ {sales_7d:,.0f}，銷量 {qty_7d:,.0f}。") - mapping_gap_score = 18 if not external_row and max(sales_7d, sales_prev_7d) > 0 else 0 + mapping_gap_score = 18 if not external_row and not market_offer_payloads and max(sales_7d, sales_prev_7d) > 0 else 0 priority_score = min(100, volume_score + qty_score + decline_score + mapping_gap_score + data_quality_score * 0.18) if external_payload and external_payload.get("gap_pct") is not None: gap = float(external_payload["gap_pct"]) @@ -783,7 +991,13 @@ def _score_opportunity( issues = [] if not external_row: - issues.append("需要確認 MOMO 候選" if review_candidate_payload else "需要補商品對應") + issues.append( + "外部來源尚未啟用" + if market_offer_payloads + else "需要確認 MOMO 候選" + if review_candidate_payload + else "需要補商品對應" + ) if sales_delta_pct is None: issues.append("前期業績不足") @@ -801,6 +1015,14 @@ def _score_opportunity( else None, "last_sale_date": str(sales_row.get("last_sale_date") or ""), "external_price": external_payload, + "market_offers": market_offer_payloads, + "cross_platform": { + "observed_platform_count": len({item.get("source_code") for item in market_offer_payloads}), + "decision_ready_platform_count": len({ + item.get("source_code") for item in market_offer_payloads if item.get("actionable") + }), + "lowest_external_offer": market_offer_payloads[0] if market_offer_payloads else None, + }, "review_candidate": review_candidate_payload, "priority_score": round(priority_score, 1), "recommended_action": { @@ -833,12 +1055,18 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] review_candidate_count = int(source_readiness.get("review_offer_count") or 0) active_external_sources = _source_names_by_status(source_readiness, "active") or list(ACTIVE_EXTERNAL_SOURCES) paused_external_sources = _source_names_by_status(source_readiness, "paused") or list(PAUSED_EXTERNAL_SOURCES) + observed_external_sources = [ + str(source.get("display_name") or source.get("code") or "") + for source in source_readiness.get("sources") or [] + if source.get("has_runtime_data") + ] source_scope = { "primary_goal": "提升 PChome 業績", "primary_sales_source": PRIMARY_SALES_SOURCE, "active_external_sources": active_external_sources, "paused_external_sources": paused_external_sources, - "plain_note": "MOMO 先用；其他主流平台待接入，不進作戰清單，也不發告警。", + "observed_external_sources": observed_external_sources, + "plain_note": "多平台證據會完整保留；只有 runtime 已啟用且通過同款與品質驗證的報價能驅動正式決策。", "source_readiness": source_readiness, "offer_evidence_contract": source_readiness.get("offer_evidence_contract") or {}, } @@ -856,6 +1084,7 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] "mapping_rate": 0, "needs_mapping_count": 0, "review_candidate_count": review_candidate_count, + "sales_freshness": _sales_freshness(None), }, "opportunities": [], "message": "目前還沒有 PChome 業績資料，請先完成業績匯入。", @@ -863,6 +1092,8 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] with engine.connect() as conn: sales_summary = _fetch_sales_summary(conn) + catalog_mapping_summary = _fetch_catalog_mapping_summary(conn) + summary_freshness = _sales_freshness(sales_summary.get("overall_latest_sales_date")) try: sales_rows, latest_sales_date = _fetch_sales_rows(conn, limit=limit) except RuntimeError as exc: @@ -883,28 +1114,43 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] "action_counts": {}, "action_code_counts": {}, "external_data_source_counts": {}, + "external_platform_counts": {}, + "sales_freshness": summary_freshness, + **catalog_mapping_summary, **sales_summary, }, "opportunities": [], "message": str(exc), } sales_ids = [str(row.get("pchome_product_id") or "") for row in sales_rows] - external_map = _fetch_external_price_map(conn, sales_ids) + external_map, market_offer_map = _fetch_external_offer_maps( + conn, + sales_ids, + source_readiness, + ) review_candidate_map = _fetch_review_candidate_map(conn, sales_ids) opportunities = [] for row in sales_rows: key = str(row.get("pchome_product_id") or "").strip() - opportunities.append(_score_opportunity(row, external_map.get(key), review_candidate_map.get(key))) + opportunities.append(_score_opportunity( + row, + external_map.get(key), + review_candidate_map.get(key), + market_offer_map.get(key), + )) opportunities.sort(key=lambda item: item["priority_score"], reverse=True) opportunities = opportunities[:limit] - needs_mapping_count = sum(1 for item in opportunities if not item.get("external_price")) - mapped_count = len(opportunities) - needs_mapping_count + needs_mapping_count = sum(1 for item in opportunities if not item.get("market_offers")) + observed_mapping_count = len(opportunities) - needs_mapping_count + mapped_count = sum(1 for item in opportunities if item.get("external_price")) mapping_rate = round(mapped_count / max(len(opportunities), 1) * 100, 1) + observed_mapping_rate = round(observed_mapping_count / max(len(opportunities), 1) * 100, 1) action_counts: dict[str, int] = {} action_code_counts: dict[str, int] = {} external_data_source_counts: dict[str, int] = {} + external_platform_counts: dict[str, int] = {} for item in opportunities: label = item["recommended_action"]["label"] code = item["recommended_action"]["code"] @@ -916,6 +1162,12 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] external_data_source_counts[data_source_label] = ( external_data_source_counts.get(data_source_label, 0) + 1 ) + for offer in item.get("market_offers") or []: + source_name = str(offer.get("source") or offer.get("source_code") or "外部平台") + external_platform_counts[source_name] = external_platform_counts.get(source_name, 0) + 1 + + latest_sales = latest_sales_date or sales_summary.get("overall_latest_sales_date") + sales_freshness = _sales_freshness(latest_sales) return { "success": True, @@ -923,10 +1175,13 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] "generated_at": generated_at, "source_scope": source_scope, "stats": { - "latest_sales_date": latest_sales_date or sales_summary.get("overall_latest_sales_date"), + "latest_sales_date": latest_sales, + "sales_freshness": sales_freshness, "candidate_count": len(opportunities), "mapped_count": mapped_count, "mapping_rate": mapping_rate, + "observed_mapping_count": observed_mapping_count, + "observed_mapping_rate": observed_mapping_rate, "needs_mapping_count": needs_mapping_count, "review_candidate_count": review_candidate_count, "total_sales_7d": round(sum(_to_float(item.get("sales_7d")) for item in opportunities), 2), @@ -934,8 +1189,16 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] "action_counts": action_counts, "action_code_counts": action_code_counts, "external_data_source_counts": external_data_source_counts, + "external_platform_counts": external_platform_counts, + "platforms_with_runtime_data": int(source_readiness.get("sources_with_data_count") or 0), + "platform_source_coverage_rate": _to_float(source_readiness.get("source_coverage_rate")), + **catalog_mapping_summary, **sales_summary, }, "opportunities": opportunities, - "message": "已整理今日 PChome 業績成長作戰清單。", + "message": ( + "業績資料已過期，作戰清單只供觀察；系統應先自動取得最新業績。" + if not sales_freshness["decision_ready"] + else "已依最新業績與外部平台證據整理今日作戰清單。" + ), } diff --git a/templates/ai_intelligence.html b/templates/ai_intelligence.html index a35b1f9..5d4ffad 100644 --- a/templates/ai_intelligence.html +++ b/templates/ai_intelligence.html @@ -2870,9 +2870,7 @@ — - - - + 確認資料新鮮度 等待資料
@@ -2882,7 +2880,7 @@ —%
- 等待 MOMO 對應 + 等待外部平台對應
@@ -2955,9 +2953,9 @@ -
+
-

PChome 與 MOMO 價格狀態

+

PChome 與外部平台

@@ -2972,7 +2970,7 @@
- MOMO 低價壓力 + 外部低價壓力 —
@@ -3003,7 +3001,7 @@
待補比價 — - 需要補抓 MOMO + 需要補外部平台同款
最新業績日 @@ -3913,12 +3911,12 @@ function renderOpsCommandDashboard(stats, scope = {}) { document.getElementById('opsDashboardStatus').textContent = candidateCount ? `資料就緒率 ${readyRate}%` : '等待 PChome 業績資料'; - renderNextAction(candidateCount, mappedCount, needsMapping, reviewCandidateCount); + renderNextAction(stats); setWidth('opsFunnelCandidateBar', candidateCount ? 100 : 0); setWidth('opsFunnelMappedBar', candidateCount ? (mappedCount / candidateCount) * 100 : 0); setWidth('opsFunnelNeedsBar', candidateCount ? (needsMapping / candidateCount) * 100 : 0); - renderOpsSourceBars(stats.external_data_source_counts || {}, scope); + renderOpsSourceBars(stats.external_platform_counts || {}, scope); renderGrowthExecutiveSummary(stats); renderGrowthCommandCenter(stats, latestGrowthRows); } @@ -3956,11 +3954,15 @@ function renderGrowthCommandCenter(stats = {}, rows = []) { const pendingCount = Math.max(0, needsMapping + reviewCandidateCount); const readyShare = candidateCount ? (advantageCount / candidateCount) * 100 : 0; const pressureShare = candidateCount ? (pressureCount / candidateCount) * 100 : 0; + const freshness = stats.sales_freshness || {}; setText('commandSales7d', sales7d ? formatMoney(sales7d) : '—'); setText('commandSalesDelta', prevSales7d ? `${salesDeltaPct >= 0 ? '+' : ''}${salesDeltaPct.toFixed(1)}%` : '待比'); + setText('commandSalesFreshness', freshness.age_days === null || freshness.age_days === undefined + ? (freshness.label || '尚無資料') + : `${freshness.label || '資料狀態'} · ${formatCount(freshness.age_days)} 天`); setText('commandMatchRate', `${Math.round(matchRate * 10) / 10}%`); setText('commandMappedRatio', `${formatCount(mappedCount)} / ${formatCount(candidateCount)}`); setText('commandDecliningProducts', formatCount(decliningCount)); @@ -3980,8 +3982,11 @@ function renderGrowthCommandCenter(stats = {}, rows = []) { const status = document.getElementById('growthCommandStatus'); if (status) { - const text = latestSalesDate ? `更新 ${latestSalesDate}` : '待資料'; - status.innerHTML = `${escapeHtml(text)}`; + const text = freshness.decision_ready + ? `資料可用 · ${latestSalesDate}` + : `${freshness.label || '待資料'}${latestSalesDate ? ` · ${latestSalesDate}` : ''}`; + const icon = freshness.decision_ready ? 'fa-circle-check' : 'fa-triangle-exclamation'; + status.innerHTML = `${escapeHtml(text)}`; } const rateBar = document.getElementById('commandMatchRateBar'); if (rateBar) rateBar.style.width = `${clampPercent(matchRate)}%`; @@ -3991,7 +3996,16 @@ function renderGrowthCommandCenter(stats = {}, rows = []) { donut.style.setProperty('--review', clampPercent(pressureShare)); } - if (!candidateCount) { + if (freshness.decision_ready === false) { + setCommandTask( + '先更新業績資料', + freshness.age_days === null || freshness.age_days === undefined + ? '尚未取得可用業績' + : `資料落後 ${formatCount(freshness.age_days)} 天`, + '查看匯入狀態', + () => { window.location.href = '/daily_sales'; } + ); + } else if (!candidateCount) { setCommandTask( '更新業績', '待業績', @@ -4059,11 +4073,28 @@ function renderGrowthCommandTopDecliners(rows) { }).join(''); } -function renderNextAction(candidateCount, mappedCount, needsMapping, reviewCandidateCount = 0) { +function renderNextAction(stats = {}) { const title = document.getElementById('nextActionTitle'); const reason = document.getElementById('nextActionReason'); const button = document.getElementById('nextActionButton'); if (!title || !reason || !button) return; + const candidateCount = Number(stats.candidate_count || 0); + const mappedCount = Number(stats.mapped_count || 0); + const needsMapping = Number(stats.needs_mapping_count || 0); + const reviewCandidateCount = Number(stats.review_candidate_count || 0); + const freshness = stats.sales_freshness || {}; + + if (freshness.decision_ready === false) { + const age = freshness.age_days; + title.textContent = '今天先做：更新 PChome 業績'; + reason.textContent = age === null || age === undefined + ? '尚未取得可用業績，價格與曝光決策暫不執行。' + : `最新業績已落後 ${formatCount(age)} 天，系統先恢復資料來源，再重新計算作戰清單。`; + button.textContent = '查看匯入狀態'; + delete button.dataset.action; + button.onclick = () => { window.location.href = '/daily_sales'; }; + return; + } if (!candidateCount) { title.textContent = '今天先做：更新 PChome 業績'; @@ -4076,7 +4107,7 @@ function renderNextAction(candidateCount, mappedCount, needsMapping, reviewCandi if (needsMapping > 0 && mappedCount === 0) { if (reviewCandidateCount > 0) { - title.textContent = `今天先做：確認 ${formatCount(reviewCandidateCount)} 筆 MOMO 候選`; + title.textContent = `今天先做：AI 驗證 ${formatCount(reviewCandidateCount)} 筆候選`; reason.textContent = '候選已找到，先確認同款或排除，確認後才會進入價格判斷。'; button.textContent = '確認候選'; delete button.dataset.action; @@ -4084,7 +4115,7 @@ function renderNextAction(candidateCount, mappedCount, needsMapping, reviewCandi return; } title.textContent = `今天先做：補齊 ${formatCount(needsMapping)} 件比價資料`; - reason.textContent = '這些商品有業績，但目前還看不到 MOMO 參考價，請先補齊比價資料。'; + reason.textContent = '這些商品有業績，但目前還沒有可決策的外部平台同款，系統先補齊商品對應。'; button.textContent = '補齊比價資料'; button.dataset.action = 'backfill'; button.onclick = () => backfillPchomeMatches(); @@ -4093,7 +4124,7 @@ function renderNextAction(candidateCount, mappedCount, needsMapping, reviewCandi if (needsMapping > mappedCount) { if (reviewCandidateCount > 0) { - title.textContent = `今天先做：確認 ${formatCount(reviewCandidateCount)} 筆 MOMO 候選`; + title.textContent = `今天先做：AI 驗證 ${formatCount(reviewCandidateCount)} 筆候選`; reason.textContent = '先把已找到的候選變成可用資料，再補剩下找不到同款的商品。'; button.textContent = '確認候選'; delete button.dataset.action; @@ -4121,17 +4152,27 @@ function renderGrowthExecutiveSummary(stats = {}) { const needsMapping = Number(stats.needs_mapping_count || 0); const reviewCandidateCount = Number(stats.review_candidate_count || 0); const latestSalesDate = String(stats.latest_sales_date || '').slice(0, 10); + const freshness = stats.sales_freshness || {}; document.getElementById('growthExecReady').textContent = formatCount(mappedCount); document.getElementById('growthExecGap').textContent = formatCount(needsMapping); document.getElementById('growthExecLatestDate').textContent = latestSalesDate || '—'; - document.getElementById('growthExecLatestDetail').textContent = latestSalesDate ? '已接到 PChome 業績' : '尚未確認業績日期'; + document.getElementById('growthExecLatestDetail').textContent = freshness.label || (latestSalesDate ? '已接到 PChome 業績' : '尚未確認業績日期'); const task = document.getElementById('growthExecTask'); const detail = document.getElementById('growthExecTaskDetail'); const gapCard = document.getElementById('growthExecGapCard'); if (!task || !detail) return; + if (freshness.decision_ready === false) { + task.textContent = '先恢復最新業績'; + detail.textContent = freshness.age_days === null || freshness.age_days === undefined + ? '尚未取得可用業績，營收決策暫停。' + : `業績落後 ${formatCount(freshness.age_days)} 天，系統先完成自動匯入。`; + gapCard?.classList.add('is-gap'); + return; + } + if (!candidateCount) { task.textContent = '先更新 PChome 業績'; detail.textContent = '目前還沒有足夠資料，請確認最新業績檔是否已匯入。'; @@ -4225,9 +4266,10 @@ async function loadGrowthOps(forceRefresh = false) { renderGrowthDataSourceSummary(stats); const active = (scope.active_external_sources || []).join('、') || '尚未接入'; - const paused = (scope.paused_external_sources || []).join('、') || '無'; + const observed = (scope.observed_external_sources || []).length; + const paused = (scope.paused_external_sources || []).length; document.getElementById('growthSourceNote').textContent = - `業績：${scope.primary_sales_source || 'PChome 後台業績'} · 外部：${active} · 暫停：${paused}`; + `業績：${scope.primary_sales_source || 'PChome 後台業績'} · 決策來源：${active} · 已有證據 ${observed} 個 · 待接入 ${paused} 個`; renderGrowthSourceReadiness((scope.source_readiness || {}).sources || []); latestGrowthRows = data.opportunities || []; @@ -4273,6 +4315,14 @@ function renderGrowthActionHint(stats) { const mappedCount = Number(stats.mapped_count || 0); const needsMapping = Number(stats.needs_mapping_count || 0); const reviewCandidateCount = Number(stats.review_candidate_count || 0); + const freshness = stats.sales_freshness || {}; + + if (freshness.decision_ready === false) { + hint.textContent = freshness.age_days === null || freshness.age_days === undefined + ? '尚未取得可用業績，系統先恢復上游資料。' + : `最新業績落後 ${formatCount(freshness.age_days)} 天，先完成自動匯入再執行營收動作。`; + return; + } if (!candidateCount) { hint.textContent = '還不能產生今日清單，原因是缺少最新 PChome 業績。'; @@ -4301,10 +4351,10 @@ function renderGrowthDataSourceSummary(stats) { const summary = document.getElementById('growthDataSourceSummary'); if (!summary) return; - const counts = stats.external_data_source_counts || {}; + const counts = stats.external_platform_counts || {}; const entries = Object.entries(counts) .filter(([, count]) => Number(count || 0) > 0) - .map(([label, count]) => `${formatDataSourceLabel(label)} ${Number(count).toLocaleString()} 筆`); + .map(([label, count]) => `${label} ${Number(count).toLocaleString()} 筆`); if (!entries.length) { summary.textContent = '來源：尚未接到外部參考價'; @@ -4328,7 +4378,11 @@ function renderGrowthSourceReadiness(sources) { return; } - box.innerHTML = sources.map((source) => { + const visibleSources = sources.filter((source) => + source.status_code === 'active' || source.has_runtime_data + ); + const pendingCount = Math.max(0, sources.length - visibleSources.length); + const cards = visibleSources.map((source) => { const usable = Number(source.usable_offer_count || 0); const detail = usable > 0 ? `${source.data_quality_label || '資料可用'} · ${usable.toLocaleString()} 筆` @@ -4341,7 +4395,17 @@ function renderGrowthSourceReadiness(sources) {

${escapeHtml(detail)}

`; - }).join(''); + }); + if (pendingCount > 0) { + cards.push(`
+
+ 其他平台 + ${formatCount(pendingCount)} 個待接入 +
+

完成合法來源、同款驗證與品質門檻後自動啟用。

+
`); + } + box.innerHTML = cards.join(''); } function resolveGrowthDetailKind(kind) { diff --git a/tests/test_auto_import_failure_boundaries.py b/tests/test_auto_import_failure_boundaries.py index 44eec9d..476344c 100644 --- a/tests/test_auto_import_failure_boundaries.py +++ b/tests/test_auto_import_failure_boundaries.py @@ -61,7 +61,7 @@ def test_daily_sales_import_fails_when_monthly_sync_fails(monkeypatch, tmp_path) source_df = pd.DataFrame([ { - "日期": "2026-06-24", + "日期": pd.Timestamp.today().strftime("%Y-%m-%d"), "商品ID": "A001", "商品名稱": "測試商品", "銷售金額": 1200, @@ -350,4 +350,41 @@ def test_auto_import_empty_drive_folder_remains_success(monkeypatch, tmp_path): assert result["success"] is True assert result["file_count"] == 0 + assert result["status"] == "upstream_missing" + assert result["decision_ready"] is False assert result["message"] == "沒有找到待匯入的檔案" + + +def test_auto_import_empty_drive_exposes_stale_upstream_as_degraded(monkeypatch, tmp_path): + import_service = _load_import_service(monkeypatch, f"sqlite:///{tmp_path / 'momo.db'}") + import_service.Base.metadata.create_all(import_service.engine) + with import_service.engine.begin() as conn: + conn.execute(text("CREATE TABLE IF NOT EXISTS daily_sales_snapshot (snapshot_date TEXT)")) + conn.execute(text("INSERT INTO daily_sales_snapshot (snapshot_date) VALUES ('2020-01-01')")) + + class FakeDriveService: + last_error_kind = None + last_error = None + + def list_files_in_folder(self, folder_path, file_pattern): + return [] + + import services.openclaw_strategist_service as strategist + + alerts = [] + monkeypatch.setattr(import_service, "drive_service", FakeDriveService()) + monkeypatch.setattr( + strategist, + "_send_data_stale_alert", + lambda **kwargs: alerts.append(kwargs), + ) + + result = import_service.ImportService().auto_import_from_drive() + + assert result["success"] is True + assert result["status"] == "upstream_stale" + assert result["decision_ready"] is False + assert result["requires_upstream_acquisition"] is True + assert result["latest_sales_date"] == "2020-01-01" + assert result["data_lag_days"] >= 3 + assert alerts and alerts[0]["report_type"] == "upstream_drive" diff --git a/tests/test_competitor_intel_cache.py b/tests/test_competitor_intel_cache.py index beabbfb..bcdff9e 100644 --- a/tests/test_competitor_intel_cache.py +++ b/tests/test_competitor_intel_cache.py @@ -189,7 +189,7 @@ def test_competitor_review_queue_is_canonical_unit_price_handoff(): assert "AI 單位價" in daily_template assert "competitor_intel.review_queue" in daily_template assert "chart_data.competitor_coverage" in growth_template - assert "coverage.decision_support_rate" in growth_template + assert "coverage.match_rate" in growth_template assert "coverage.decision_ready_rate" in growth_template assert "coverage.decision_support_rate" in growth_template assert "coverage.catalog_comparable_count" in growth_template @@ -367,6 +367,7 @@ def test_competitor_review_reasons_prefer_json_payload_labels(): assert envelope["recommended_action"]["requires_hitl"] is False assert envelope["recommended_action"]["owner"] == "AI Agent" assert envelope["recommended_action"]["ai_exception_mode"] == "machine_verifiable_auto_resolution" + assert envelope["recommended_action"]["requires_ai_exception"] is True assert envelope["recommended_action"]["action"] == "verify_or_reject_identity" assert any(evidence["metric"] == "reasons" for evidence in envelope["evidence"]) assert any(evidence["metric"] == "identity_evidence" for evidence in envelope["evidence"]) @@ -505,11 +506,13 @@ def test_review_decision_brief_is_shared_by_openclaw_and_ppt(): brief = summarize_review_decision_envelopes([item], limit=5) assert brief["hitl_count"] == 0 + assert brief["ai_exception_count"] == 1 assert brief["auto_execute_blocked_count"] == 1 assert brief["severity_counts"] assert brief["data_quality_counts"] == {"complete": 1} assert "AI 確認身份後採用同款" in brief["text"] assert "HITL" not in brief["text"] + assert "AI例外" in brief["text"] assert "DDAO4C-A79050612" in brief["text"] repo_source = (ROOT / "services" / "competitor_intel_repository.py").read_text(encoding="utf-8") diff --git a/tests/test_external_market_offer_service.py b/tests/test_external_market_offer_service.py index bcf3763..141eb51 100644 --- a/tests/test_external_market_offer_service.py +++ b/tests/test_external_market_offer_service.py @@ -21,6 +21,8 @@ def test_connector_contract_lists_mainstream_sources_without_auto_alerts(): "line_shopping", "rakuten", "yahoo_shopping", + "etmall", + "friday", "ruten", "shopify_brand_store", "meta_commerce", @@ -638,7 +640,55 @@ def test_external_source_readiness_uses_legacy_momo_reference_cache(): assert "image_url" in evidence_fields assert "stock_status" in evidence_fields assert payload["offer_evidence_contract"]["quality_gate"]["unverified_data_route"] == "待確認候選，不進告警" - assert payload["plain_summary"] == "MOMO 先用；其他主流平台已列管，未接合法穩定來源前不進告警。" + assert payload["sources_with_data_count"] == 1 + assert payload["source_coverage_rate"] > 0 + assert "只有 runtime 已啟用" in payload["plain_summary"] + + +def test_source_seeds_preserve_runtime_activation_and_readiness_uses_it(): + from services import external_market_offer_service as service + + engine = create_engine("sqlite:///:memory:") + _seed_external_offer_sync_tables(engine) + + with engine.begin() as conn: + service._ensure_external_market_source_seeds(conn) + conn.execute(text(""" + UPDATE external_market_sources + SET status = 'active', enabled = 1 + WHERE code = 'shopee' + """)) + service._ensure_external_market_source_seeds(conn) + source = conn.execute(text(""" + SELECT status, enabled + FROM external_market_sources + WHERE code = 'shopee' + """)).mappings().one() + + assert source["status"] == "active" + assert bool(source["enabled"]) is True + + +def test_source_aliases_normalize_pixelrag_platform_codes(): + from services.external_market_offer_service import normalize_external_offer_payload + + base = { + "source_product_id": "SKU-1", + "title": "測試商品", + "price": 100, + "observed_at": "2026-07-10T10:00:00", + "ingestion_method": "provider_api", + } + expected = { + "yahoo_shopping_tw": "yahoo_shopping", + "etmall_tw": "etmall", + "friday_tw": "friday", + "rakuten_tw": "rakuten", + } + for raw, normalized in expected.items(): + record, errors = normalize_external_offer_payload({**base, "source_code": raw}) + assert errors == [] + assert record.source_code == normalized def test_external_market_migration_creates_source_and_offer_tables(): diff --git a/tests/test_frontend_v2_assets.py b/tests/test_frontend_v2_assets.py index 4aad45d..15d27b2 100644 --- a/tests/test_frontend_v2_assets.py +++ b/tests/test_frontend_v2_assets.py @@ -245,6 +245,7 @@ def test_growth_workflow_pages_hide_raw_export_and_fallback_content(): def test_frontend_v2_syncs_latest_momo_pro_prototype_tokens_and_shell(): tokens = (ROOT / "web/static/css/ewoooc-tokens.css").read_text(encoding="utf-8") + aliases = (ROOT / "web/static/css/ewoooc-tokens-v2-alias.css").read_text(encoding="utf-8") shell = (ROOT / "web/static/css/ewoooc-shell.css").read_text(encoding="utf-8") assert "MOMO Pro × minimal grid × operational clarity" in tokens @@ -256,6 +257,8 @@ def test_frontend_v2_syncs_latest_momo_pro_prototype_tokens_and_shell(): assert "--momo-border-strong" in tokens assert "--momo-sidebar-width" in tokens assert ".momo-app button:not(.btn):not(.btn-close)" in tokens + assert "--momo-warm-caramel: var(--momo-page-accent)" not in aliases + assert "--momo-warm-caramel: #c96442" in aliases assert "background: #1f1a14;" in shell assert "rgba(250, 247, 240, 0.08)" in shell diff --git a/tests/test_pchome_revenue_growth_service.py b/tests/test_pchome_revenue_growth_service.py index 67fcc57..ea1dd90 100644 --- a/tests/test_pchome_revenue_growth_service.py +++ b/tests/test_pchome_revenue_growth_service.py @@ -129,6 +129,59 @@ def _seed_growth_unit_price_external_offer(engine): """)) +def _seed_growth_multiplatform_external_offers(engine): + with engine.begin() as conn: + conn.execute(text(""" + CREATE TABLE external_market_sources ( + code TEXT PRIMARY KEY, + status TEXT, + enabled BOOLEAN + ) + """)) + conn.execute(text(""" + CREATE TABLE external_offers ( + id INTEGER PRIMARY KEY, + source_code TEXT, + platform_code TEXT, + source_product_id TEXT, + title TEXT, + product_url TEXT, + price REAL, + observed_at TEXT, + expires_at TEXT, + ingestion_method TEXT, + pchome_product_id TEXT, + momo_sku TEXT, + match_status TEXT, + quality_score REAL, + data_quality_status TEXT, + quality_notes_json TEXT, + raw_payload_json TEXT + ) + """)) + conn.execute(text(""" + INSERT INTO external_market_sources (code, status, enabled) + VALUES ('momo_reference', 'active', 1), ('yahoo_shopping', 'active', 1) + """)) + conn.execute(text(""" + INSERT INTO external_offers ( + id, source_code, platform_code, source_product_id, title, product_url, + price, observed_at, expires_at, ingestion_method, pchome_product_id, + momo_sku, match_status, quality_score, data_quality_status, + quality_notes_json, raw_payload_json + ) + VALUES + (1, 'momo_reference', 'momo', 'MOMO-1', 'MOMO 商品', 'https://momo.test/1', + 870, '2026-07-10 10:00:00', NULL, 'official_api', 'PCH-1', 'MOMO-1', + 'verified', 92, 'verified', '[]', + '{"pchome_public_price": 1000, "pchome_public_name": "PChome 商品"}'), + (2, 'yahoo_shopping', 'yahoo_shopping', 'YAHOO-1', 'Yahoo 商品', 'https://yahoo.test/1', + 800, '2026-07-10 11:00:00', NULL, 'official_api', 'PCH-1', 'YAHOO-1', + 'verified', 90, 'verified', '[]', + '{"pchome_public_price": 1000, "pchome_public_name": "PChome 商品"}') + """)) + + def _seed_growth_review_candidate_offer(engine): with engine.begin() as conn: conn.execute(text(""" @@ -198,8 +251,10 @@ def test_pchome_growth_opportunities_use_plain_language_and_pause_shopee_coupang "品牌官網 / Shopify", "Meta Commerce", "酷澎", + "東森購物", + "friDay 購物", }.issubset(set(payload["source_scope"]["paused_external_sources"])) - assert payload["source_scope"]["plain_note"] == "MOMO 先用；其他主流平台待接入，不進作戰清單，也不發告警。" + assert "只有 runtime 已啟用" in payload["source_scope"]["plain_note"] evidence_fields = { field["name"] for field in payload["source_scope"]["offer_evidence_contract"]["professional_evidence_fields"] @@ -213,6 +268,9 @@ def test_pchome_growth_opportunities_use_plain_language_and_pause_shopee_coupang assert payload["stats"]["candidate_count"] == 2 assert payload["stats"]["mapped_count"] == 1 assert payload["stats"]["needs_mapping_count"] == 1 + assert payload["stats"]["catalog_product_count"] == 2 + assert payload["stats"]["mapped_product_count"] == 1 + assert payload["stats"]["sales_freshness"]["status"] == "critical" actions = {item["pchome_product_id"]: item["recommended_action"]["label"] for item in payload["opportunities"]} assert actions["PCH-1"] == "檢查售價與活動" @@ -291,6 +349,41 @@ def test_pchome_growth_understands_unit_price_external_offers(): assert any("單位價" in line for line in pchome_1["reason_lines"]) +def test_pchome_growth_compares_all_enabled_platforms_and_keeps_evidence(): + from services.pchome_revenue_growth_service import build_pchome_growth_opportunities + + engine = create_engine("sqlite:///:memory:") + _seed_growth_tables(engine) + _seed_growth_multiplatform_external_offers(engine) + + payload = build_pchome_growth_opportunities(engine, limit=5) + + pchome_1 = next(item for item in payload["opportunities"] if item["pchome_product_id"] == "PCH-1") + assert pchome_1["external_price"]["source_code"] == "yahoo_shopping" + assert pchome_1["external_price"]["source"] == "Yahoo 購物" + assert pchome_1["external_price"]["momo_price"] == 800 + assert {offer["source_code"] for offer in pchome_1["market_offers"]} == { + "momo_reference", + "yahoo_shopping", + } + assert pchome_1["cross_platform"]["decision_ready_platform_count"] == 2 + assert payload["stats"]["external_platform_counts"] == { + "Yahoo 購物": 1, + "MOMO 外部價格參考": 1, + } + + +def test_sales_freshness_blocks_decisions_after_two_days(): + from datetime import date + from services.pchome_revenue_growth_service import _sales_freshness + + assert _sales_freshness("2026-07-09", today=date(2026, 7, 10))["decision_ready"] is True + stale = _sales_freshness("2026-07-07", today=date(2026, 7, 10)) + assert stale["status"] == "critical" + assert stale["age_days"] == 3 + assert stale["decision_ready"] is False + + def test_pchome_growth_route_cache_respects_shared_invalidation_epoch(monkeypatch): from routes import ai_routes as routes @@ -510,7 +603,9 @@ def test_ai_intelligence_template_uses_pchome_growth_name_and_endpoint(): assert "/api/ai/pchome-growth/external-offers/csv-dry-run" in template assert "/api/ai/pchome-growth/review-candidates" in template assert "growthSourceReadiness" in template - assert "sources.map((source)" in template + assert "visibleSources.map((source)" in template + assert "source.has_runtime_data" in template + assert "其他平台" in template assert "MOMO 待確認候選" in template assert "確認同款" in template assert "不是同款" in template @@ -749,7 +844,7 @@ def test_help_empty_login_and_ppt_copy_are_action_oriented(): "templates/system_settings.html": ["會更新對應年月份的業績資料"], "templates/admin/ppt_audit_history.html": [ "先確認簡報可預覽、可審核，再把問題交給修復流程", - "先補齊視覺 QA 執行條件，再判斷簡報品質", + "補齊轉檔與視覺模型後即可自動審核", "先補排程或手動產出，再進行視覺 QA", "先補視覺 QA 條件", ], diff --git a/tests/test_run_scheduler_embed_consistency.py b/tests/test_run_scheduler_embed_consistency.py index 29a4b7b..5c35876 100644 --- a/tests/test_run_scheduler_embed_consistency.py +++ b/tests/test_run_scheduler_embed_consistency.py @@ -1,6 +1,8 @@ import importlib import inspect import ast +import json +from datetime import datetime from pathlib import Path @@ -228,10 +230,51 @@ def test_roi_ai_smoke_and_daily_report_schedules_stay_staggered(): assert 'schedule.every().day.at("09:10").do(run_ai_smoke_daily_summary_task)' in source assert 'schedule.every().day.at("10:30").do(run_pchome_match_backfill_task)' in source assert 'schedule.every().day.at("10:45").do(run_pchome_growth_momo_backfill_task)' in source + assert "schedule.every(30).minutes.do(run_pchome_growth_momo_backfill_catchup_task)" in source assert "schedule.every(6).hours.do(run_action_plan_hygiene_task)" in source assert "schedule.every(15).minutes.do(run_ollama_111_usage_guard_check)" in source +def test_pchome_growth_backfill_catchup_uses_runtime_receipt_cooldown(monkeypatch, tmp_path): + run_scheduler = _load_run_scheduler(monkeypatch) + stats_path = tmp_path / "scheduler_stats.json" + stats_path.write_text(json.dumps({ + "pchome_growth_momo_backfill": [{ + "status": "Success", + "last_run": "2026-07-10 09:00:00", + }], + }), encoding="utf-8") + monkeypatch.setenv("SCHEDULER_STATS_PATH", str(stats_path)) + + due, reason = run_scheduler._pchome_growth_backfill_due( + now=datetime(2026, 7, 10, 11, 0, 0) + ) + assert due is False + assert reason == "cooldown_4.0h" + + due, reason = run_scheduler._pchome_growth_backfill_due( + now=datetime(2026, 7, 10, 13, 30, 0) + ) + assert due is True + assert reason == "stale_receipt_4.5h" + + +def test_pchome_growth_backfill_catchup_executes_missing_receipt(monkeypatch, tmp_path): + run_scheduler = _load_run_scheduler(monkeypatch) + monkeypatch.setenv("SCHEDULER_STATS_PATH", str(tmp_path / "missing.json")) + calls = [] + monkeypatch.setattr( + run_scheduler, + "run_pchome_growth_momo_backfill_task", + lambda: calls.append("executed"), + ) + + result = run_scheduler.run_pchome_growth_momo_backfill_catchup_task() + + assert result == {"status": "executed", "reason": "missing_runtime_receipt"} + assert calls == ["executed"] + + def test_ollama_111_usage_guard_stays_observational(monkeypatch): run_scheduler = _load_run_scheduler(monkeypatch) source = inspect.getsource(run_scheduler.run_ollama_111_usage_guard_check) diff --git a/web/static/css/ewoooc-tokens-v2-alias.css b/web/static/css/ewoooc-tokens-v2-alias.css index b4c13f4..e78ca65 100644 --- a/web/static/css/ewoooc-tokens-v2-alias.css +++ b/web/static/css/ewoooc-tokens-v2-alias.css @@ -14,7 +14,9 @@ :root { /* ── 暖色主軸（最常用，~30+ 處 inline 引用）── */ - --momo-warm-caramel: var(--momo-page-accent); /* 焦糖橘主色 */ + /* Keep the V3 base token concrete. Pointing it back to page-accent creates + a cycle because the monitor page accent is defined from warm-caramel. */ + --momo-warm-caramel: #c96442; /* 焦糖橘主色 */ --momo-warm-rust: var(--momo-danger-text); /* 鐵鏽紅 */ --momo-warm-mahogany: var(--momo-tag-clay-text); /* 桃花心木 */ --momo-warm-honey: var(--momo-warning-text); /* 蜂蜜黃 */ From 2f8e54f31fafe971de77851cc987c3c2f625e959 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 21:56:00 +0800 Subject: [PATCH 36/75] fix(ops): report Ollama embed failures accurately --- governance/pchome_growth_program_review.json | 42 ++++++++++++++++---- scripts/ops/diagnose_ollama_gcp_failover.sh | 12 +++++- tests/test_ollama_failover_diagnostic.py | 15 +++++++ 3 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 tests/test_ollama_failover_diagnostic.py diff --git a/governance/pchome_growth_program_review.json b/governance/pchome_growth_program_review.json index 1d9b319..f4aa317 100644 --- a/governance/pchome_growth_program_review.json +++ b/governance/pchome_growth_program_review.json @@ -1,6 +1,6 @@ { "version": "pchome_growth_program_review_v1", - "generated_at": "2026-07-10T20:43:51+08:00", + "generated_at": "2026-07-10T21:54:36+08:00", "supersedes_status_only_completion": true, "goal": "以新鮮 PChome 業績、可信跨平台同款資料與可歸因動作持續提升業績", "production_baseline": { @@ -22,6 +22,29 @@ "runtime_mcp_router_enabled": false, "runtime_rag_enabled": false }, + "implementation_receipt": { + "feature_source_sha": "b8cc27e4c4ae8cc6a649bc04451b7e94d0611bef", + "dev_integration_sha": "107d8ed", + "runtime_version": "V10.775", + "deployment_mode": "controlled_sync_with_targeted_rollback_archive", + "gitea_cd_run": 1098, + "gitea_cd_status": "waiting_runner_unavailable", + "application_containers_healthy": 3, + "database_container_recreated": false, + "latest_sales_date": "2026-06-24", + "sales_decision_ready": false, + "valid_catalog_mapping_count": 31, + "valid_catalog_mapping_rate_pct": 0.24, + "runtime_platforms_with_data": 1, + "catchup_result": { + "scanned": 8, + "candidates": 3, + "auto_accepted": 0, + "written": 0, + "review_candidates": 3, + "next_due": "cooldown_4.0h" + } + }, "codebase_findings": { "relevant_file_count_approx": 250, "market_intel_module_count": 140, @@ -105,7 +128,7 @@ "order": 4, "id": "PCH-GROWTH-P0-004", "priority": "P0", - "status": "source_complete_runtime_pending", + "status": "runtime_deployed_dependency_blocked", "lane": "cross_platform_revenue_decision", "objective": "Compare every qualified platform offer without letting blocked or observational evidence drive price actions", "completed": [ @@ -113,11 +136,12 @@ "runtime source-policy enforcement", "observation-only evidence separation", "sales freshness decision gate", - "catalog and platform coverage metrics" + "catalog and platform coverage metrics", + "Gitea main and dev integration", + "production V10.775 service-layer readback", + "bounded mapping catch-up receipt" ], "remaining": [ - "Gitea deployment", - "production API readback", "production multi-source offer ingestion" ], "production_acceptance": "fresh sales plus at least two enabled verified platforms produce replayable decisions" @@ -143,7 +167,7 @@ "order": 6, "id": "PCH-GROWTH-P1-001", "priority": "P1", - "status": "source_complete_runtime_pending", + "status": "runtime_deployed_visible_auth_pending", "lane": "professional_ui_ux", "objective": "Provide a compact decision cockpit instead of a status and text wall", "completed": [ @@ -153,7 +177,8 @@ "collapsed pending-source summary", "removed fake sales sparkline", "desktop and 390px mobile visual verification without horizontal overflow", - "repaired the circular accent-token fallback that made primary buttons unreadable" + "repaired the circular accent-token fallback that made primary buttons unreadable", + "production V10.775 static asset readback" ], "remaining": [ "production visible readback", @@ -178,6 +203,9 @@ "enable and verify RAG runtime", "Qwen3-VL Ollama-first visual replay", "PixelRAG receipt to RAG candidate replay", + "restore GCP-A direct runtime and 110 primary proxy", + "recover GCP-B bge-m3 embedding latency below the 30 second verifier SLO", + "restore or formally retire the unreachable 111 fallback route", "agent outcome metrics and durable learning acknowledgement" ], "production_acceptance": "runtime flags enabled and same-run retrieval-to-decision receipts pass independent replay" diff --git a/scripts/ops/diagnose_ollama_gcp_failover.sh b/scripts/ops/diagnose_ollama_gcp_failover.sh index 8afa46a..30880bd 100755 --- a/scripts/ops/diagnose_ollama_gcp_failover.sh +++ b/scripts/ops/diagnose_ollama_gcp_failover.sh @@ -108,8 +108,16 @@ curl_probe "110 proxy secondary" "$PROXY_SECONDARY_URL" "/api/version" 10 && pro if [[ "$RUN_EMBED_PROBE" == "1" ]]; then say "" say "Embedding runtime probe model=${EMBED_MODEL} timeout=${EMBED_TIMEOUT}s" - [[ "$primary_version_ok" == "1" ]] && embed_probe "GCP-A embed" "$PRIMARY_URL" || status_line WARN "GCP-A embed" "skipped because /api/version is not healthy" - [[ "$secondary_version_ok" == "1" ]] && embed_probe "GCP-B embed" "$SECONDARY_URL" || status_line WARN "GCP-B embed" "skipped because /api/version is not healthy" + if [[ "$primary_version_ok" == "1" ]]; then + embed_probe "GCP-A embed" "$PRIMARY_URL" || true + else + status_line WARN "GCP-A embed" "skipped because /api/version is not healthy" + fi + if [[ "$secondary_version_ok" == "1" ]]; then + embed_probe "GCP-B embed" "$SECONDARY_URL" || true + else + status_line WARN "GCP-B embed" "skipped because /api/version is not healthy" + fi if [[ "$INCLUDE_111_EMBED" == "1" ]]; then embed_probe "111 embed" "$FALLBACK_URL" || true else diff --git a/tests/test_ollama_failover_diagnostic.py b/tests/test_ollama_failover_diagnostic.py new file mode 100644 index 0000000..be4e705 --- /dev/null +++ b/tests/test_ollama_failover_diagnostic.py @@ -0,0 +1,15 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_embedding_probe_failure_is_not_reported_as_version_skip(): + script = (ROOT / "scripts" / "ops" / "diagnose_ollama_gcp_failover.sh").read_text( + encoding="utf-8" + ) + + assert 'if [[ "$primary_version_ok" == "1" ]]; then' in script + assert 'if [[ "$secondary_version_ok" == "1" ]]; then' in script + assert 'embed_probe "GCP-B embed" "$SECONDARY_URL" || true' in script + assert '&& embed_probe "GCP-B embed"' not in script From 05d1b699840a9aef78a83ec6da285303e9f6cc41 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 22:02:19 +0800 Subject: [PATCH 37/75] chore(governance): track EwoooC release runner recovery --- governance/pchome_growth_program_review.json | 23 ++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/governance/pchome_growth_program_review.json b/governance/pchome_growth_program_review.json index f4aa317..9498f10 100644 --- a/governance/pchome_growth_program_review.json +++ b/governance/pchome_growth_program_review.json @@ -165,6 +165,25 @@ }, { "order": 6, + "id": "PCH-GROWTH-P0-006", + "priority": "P0", + "status": "blocked_with_safe_next_action", + "lane": "release_automation", + "objective": "Restore deterministic Gitea CD so every accepted main commit reaches production with independent verification", + "completed": [ + "Gitea main and dev source truth", + "controlled sync fallback with targeted rollback archive", + "production health, container and service-layer readback" + ], + "remaining": [ + "restore a registered runner with the ewoooc-host label", + "run the latest main CD to Success", + "persist source SHA, runtime version and post-deploy verifier as one durable deploy receipt" + ], + "production_acceptance": "latest main CD succeeds without manual file sync and production readback matches the deployed source SHA" + }, + { + "order": 7, "id": "PCH-GROWTH-P1-001", "priority": "P1", "status": "runtime_deployed_visible_auth_pending", @@ -187,7 +206,7 @@ "production_acceptance": "first viewport shows freshness, revenue trend, coverage, risk and one executable next action without overflow" }, { - "order": 7, + "order": 8, "id": "PCH-GROWTH-P1-002", "priority": "P1", "status": "in_progress", @@ -211,7 +230,7 @@ "production_acceptance": "runtime flags enabled and same-run retrieval-to-decision receipts pass independent replay" }, { - "order": 8, + "order": 9, "id": "PCH-GROWTH-P1-003", "priority": "P1", "status": "in_progress", From 48830d01d0216bfad1f36b5fd4ae51bac43e4942 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 01:30:42 +0800 Subject: [PATCH 38/75] feat(pchome): automate authorized sales acquisition --- .env.example | 22 + config.py | 2 +- database/import_models.py | 61 ++ docs/AI_INTELLIGENCE_MODULE_SOT.md | 1 + .../pchome_ai_automation_priority_backlog.md | 23 +- governance/pchome_growth_program_review.json | 27 +- .../046_pchome_sales_acquisition_receipts.sql | 53 ++ requirements.txt | 1 + routes/auto_import_routes.py | 32 +- scheduler.py | 60 +- services/import_service.py | 523 ++++++++------ .../pchome_sales_acquisition_providers.py | 380 +++++++++++ services/pchome_sales_acquisition_service.py | 498 ++++++++++++++ templates/auto_import_index.html | 636 ++++++++---------- tests/test_auto_import_failure_boundaries.py | 81 ++- tests/test_frontend_v2_assets.py | 11 +- tests/test_import_service_sql_params.py | 16 +- tests/test_pchome_revenue_growth_service.py | 18 +- .../test_pchome_sales_acquisition_service.py | 334 +++++++++ web/static/css/page-auto-import-bem.css | 264 +++++++- web/static/css/page-auto-import.css | 2 +- 21 files changed, 2428 insertions(+), 617 deletions(-) create mode 100644 migrations/046_pchome_sales_acquisition_receipts.sql create mode 100644 services/pchome_sales_acquisition_providers.py create mode 100644 services/pchome_sales_acquisition_service.py create mode 100644 tests/test_pchome_sales_acquisition_service.py diff --git a/.env.example b/.env.example index f552f1e..bc3d30e 100644 --- a/.env.example +++ b/.env.example @@ -637,6 +637,28 @@ GOOGLE_DRIVE_CREDENTIALS_FILE=/app/config/google_credentials.json GOOGLE_DRIVE_TOKEN_FILE=/app/config/google_token.json GOOGLE_DRIVE_LEGACY_PICKLE_FILE=/app/config/google_token.pickle +# Authorized PChome sales acquisition. Providers are opt-in and never expose +# credentials through readiness APIs or receipts. +PCHOME_SALES_HTTP_ENABLED=false +PCHOME_SALES_HTTP_URL= +PCHOME_SALES_HTTP_ALLOWED_HOSTS= +PCHOME_SALES_HTTP_ALLOWED_PORTS=443 +PCHOME_SALES_HTTP_BEARER_TOKEN= +PCHOME_SALES_HTTP_ALLOW_PRIVATE=false +PCHOME_SALES_HTTP_TIMEOUT_SECONDS=30 +PCHOME_SALES_IMAP_ENABLED=false +PCHOME_SALES_IMAP_HOST= +PCHOME_SALES_IMAP_PORT=993 +PCHOME_SALES_IMAP_USER= +PCHOME_SALES_IMAP_PASSWORD= +PCHOME_SALES_IMAP_MAILBOX=INBOX +PCHOME_SALES_IMAP_SUBJECT=即時業績 +PCHOME_SALES_IMAP_MAX_MESSAGES=20 +PCHOME_SALES_LOCAL_DROP_DIR= +PCHOME_SALES_TEMP_DIR=/app/data/temp/pchome-sales +PCHOME_SALES_MAX_FILE_BYTES=26214400 +PCHOME_SALES_MAX_FILES_PER_RUN=5 + EXTERNAL_OFFER_SYNC_LIMIT=500 MOMO_TARGETED_SEARCH_LIMIT_PER_TERM=3 MOMO_TARGETED_SEARCH_MAX_PRODUCTS=8 diff --git a/config.py b/config.py index 8adb219..083b399 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.775" +SYSTEM_VERSION = "V10.776" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/database/import_models.py b/database/import_models.py index 3f1e607..6ce3644 100644 --- a/database/import_models.py +++ b/database/import_models.py @@ -110,3 +110,64 @@ class ImportConfig(Base): 'created_at': self.created_at.isoformat() if self.created_at else None, 'updated_at': self.updated_at.isoformat() if self.updated_at else None, } + + +class SalesAcquisitionReceipt(Base): + """Durable controlled-apply receipt for PChome sales acquisition runs.""" + __tablename__ = 'pchome_sales_acquisition_receipts' + + receipt_id = Column(String(64), primary_key=True) + trace_id = Column(String(64), nullable=False, index=True) + span_id = Column(String(16), nullable=False, index=True) + run_id = Column(String(64), nullable=False, index=True) + work_item_id = Column(String(100), nullable=False, index=True) + status = Column(String(40), nullable=False, index=True) + source_type = Column(String(40), nullable=False, index=True) + source_fingerprint = Column(String(64), index=True) + source_ref_hash = Column(String(64)) + source_file_name = Column(String(500)) + risk_level = Column(String(20), nullable=False, default='medium') + policy_decision = Column(String(40), nullable=False) + decision = Column(String(80), nullable=False) + import_job_id = Column(Integer, index=True) + rows_imported = Column(Integer, default=0) + before_freshness_json = Column(Text) + stage_receipts_json = Column(Text) + verifier_json = Column(Text) + error_kind = Column(String(100)) + public_message = Column(Text) + started_at = Column(DateTime, default=taipei_now, nullable=False) + completed_at = Column(DateTime) + + def to_public_dict(self): + def parse_json(value): + if not value: + return None + try: + import json + return json.loads(value) + except Exception: + return None + + return { + 'receipt_id': self.receipt_id, + 'trace_id': self.trace_id, + 'span_id': self.span_id, + 'run_id': self.run_id, + 'work_item_id': self.work_item_id, + 'status': self.status, + 'source_type': self.source_type, + 'source_file_name': self.source_file_name, + 'risk_level': self.risk_level, + 'policy_decision': self.policy_decision, + 'decision': self.decision, + 'import_job_id': self.import_job_id, + 'rows_imported': self.rows_imported or 0, + 'before_freshness': parse_json(self.before_freshness_json), + 'stages': parse_json(self.stage_receipts_json), + 'verifier': parse_json(self.verifier_json), + 'error_kind': self.error_kind, + 'message': self.public_message, + 'started_at': self.started_at.isoformat() if self.started_at else None, + 'completed_at': self.completed_at.isoformat() if self.completed_at else None, + } diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index c27dc72..c13c86d 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -339,6 +339,7 @@ SQL漏斗(~300筆) > **重要**: 此表由 `import_service.py` 使用 `df.to_sql()` 動態建立。 > 欄位名稱**完全繼承自 PChome 後台匯出的業績 Excel 原始欄位**，加上程式碼追加的 `snapshot_date`。 > V10.605 起，匯入器會掃描所有 worksheet 與前 15 列表頭，優先選擇含「日期 / 商品名稱 / 總業績或銷售金額」的明細工作表；格式或日期真的不合格的檔案會移到 Google Drive `匯入失敗`，避免每 30 分鐘重複告警。 +> V10.776 起，`pchome_sales_acquisition_service.py` 依序探測 Google Drive、顯式授權 HTTPS、顯式授權 IMAP 與受控落地目錄；所有非 Drive 檔案先做副檔名/魔術碼/大小檢查與 SHA-256 冪等，再由 `import_service.py` 在同一 DB transaction 內 replace、寫入及驗證 `daily_sales_snapshot` 與 `realtime_sales_monthly`。任一步失敗必須整批 rollback，不得留下半同步狀態。每次 run 需把 `trace_id`、OpenTelemetry-compatible `span_id`、`run_id`、`work_item_id`、來源、政策、check、execution、post-verifier 與 terminal 寫入 `pchome_sales_acquisition_receipts`；舊版人工暫停旗標不得停止此 medium-risk controlled-apply lane。正式完成仍要求最新業績 <= 1 天，不能以 source/test/UI 綠燈取代 production freshness closure。 #### 已確認的關鍵欄位（PChome 後台報表欄位名稱） diff --git a/docs/guides/pchome_ai_automation_priority_backlog.md b/docs/guides/pchome_ai_automation_priority_backlog.md index 045720f..1e9e493 100644 --- a/docs/guides/pchome_ai_automation_priority_backlog.md +++ b/docs/guides/pchome_ai_automation_priority_backlog.md @@ -1,6 +1,6 @@ # PChome AI 自動化優先工作清單 -> 最後更新: 2026-07-10 Asia/Taipei +> 最後更新: 2026-07-11 Asia/Taipei > 範圍: PChome growth / MOMO mapping AI 自動化主線 > 真相來源: Gitea `dev` / `main`、正式環境 `/health`、正式 DB readback、machine-verifiable artifacts @@ -16,12 +16,12 @@ | 順序 | 工作主線 | 狀態 | 下一個完成條件 | |---|---|---|---| -| P0-1 | 授權業績資料自動取得與匯入 | 進行中 | 最新業績 <= 1 天，source receipt 與 post-verifier 同 run 完成 | +| P0-1 | 授權業績資料自動取得與匯入 | 程式已完成，待正式來源與新鮮資料閉環 | 最新業績 <= 1 天，source receipt 與 post-verifier 同 run 完成 | | P0-2 | 全商品身份圖譜與營收加權對應 | 進行中 | 覆蓋 80% 營收商品的 >= 90% | | P0-3 | 合法多電商資料取得 | 進行中 | 每個啟用來源具 freshness SLO、normalize、matcher、verifier | -| P0-4 | 多平台營收決策引擎 | 程式已完成，待推版回讀 | 正式環境以至少 2 個已啟用平台產生可重播決策 | +| P0-4 | 多平台營收決策引擎 | 已部署，受多來源資料缺口阻擋 | 正式環境以至少 2 個已啟用平台產生可重播決策 | | P0-5 | 調價、促銷、內容、曝光實驗與歸因 | 未開始 | 同 run 完成 apply、轉換驗證、增量營收歸因與 rollback | -| P1-1 | 專業 UI/UX 作戰 cockpit | 程式已完成，待推版與視覺 QA | desktop/mobile 首屏無文字牆、無假圖、只顯示一個主要下一步 | +| P1-1 | 專業 UI/UX 作戰 cockpit | V10.776 source-ready，待正式推版與視覺 QA | desktop/mobile 首屏無文字牆、無假圖、只顯示一個主要下一步 | | P1-2 | Ollama-first MCP / RAG / PixelRAG runtime | 進行中 | 正式 flags 啟用，PixelRAG receipt 可 replay，Qwen3-VL 驗證通過 | | P1-3 | 資安與治理自動化 | 進行中 | low/medium/high controlled apply 自動閉環，critical 才 break-glass | @@ -32,12 +32,23 @@ PixelRAG blocked-page 分類、外部報價正規化、正式 DB 安全邊界。 新增 Yahoo / ETMall / friDay / Rakuten PixelRAG alias、多平台 evidence 與 actionable 分層、 業績 freshness gate、全 catalog 覆蓋率、漏跑排程追趕，以及首屏來源摘要與過期資料阻擋。 +2026-07-11 P0-1 source receipt：新增 Drive → 授權 HTTPS → 授權 IMAP → 受控落地目錄的 +自動取得順序、SHA-256 冪等、同交易雙表 replace/verifier/rollback、 +`trace_id/span_id/run_id/work_item_id` durable receipt、舊人工暫停旗標 sunset 與首屏來源 cockpit。 +程式完成不等於 production closure；正式退出條件仍是取得日期 <= 1 天的授權報表並完成 runtime readback。 + +### 2026-07-11 外部專業基準落地 + +- OWASP SSRF Prevention Cheat Sheet：授權 HTTPS provider 採 HTTPS scheme、host/port allowlist、公開 IP 驗證、禁止 redirect，並固定連到已驗證 IP、保留原 hostname TLS 驗證，降低 DNS rebinding / TOCTOU 風險。 +- OpenTelemetry Trace Context：receipt 使用 32 位小寫 hex `trace_id` 與 16 位小寫 hex `span_id`，保留後續跨服務 log/trace correlation 契約。 +- NIST AI RMF：acquisition run 以 policy/asset mapping、風險決策、量測/verifier 與 retry/rollback/learning writeback 對應 Govern、Map、Measure、Manage；UI 或 source-ready 不得取代 production runtime closure。 + ## 不可遺漏的插入需求 以下是使用者在主線推進期間插入、且必須保留在完整工作項目裡的要求。後續不得再把這些需求漏掉或變成口頭提醒。 1. 正式環境才是最新版本真相；不得用本機檔案、舊筆記、舊分支蓋過 production truth。 -2. 版本不得搞錯；目前正式環境 `/health` 版本推進目標為 `V10.745`，完成條件必須包含明確版本 bump、部署與 readback。 +2. 版本不得搞錯；2026-07-11 開工 live truth 為正式環境 `V10.775`，本輪 source 目標為 `V10.776`；完成條件必須包含明確版本 bump、部署與 readback，後續仍以當下 `/health` 為準。 3. GitHub 全面 freeze；不得使用 GitHub、`gh`、GitHub API、GitHub Actions、PR、issue、mirror 或 read-only GitHub 流程。 4. 實作結果必須推到 Gitea `dev` 與 `main`；若改到 runtime 行為，還必須部署到正式環境並回讀。 5. 主方向是 AI 自動化，不是人工審核；人工欄位只能當 evidence / ledger / UI truth，不得阻擋低爆炸半徑、可驗證、可回滾的 controlled apply。 @@ -69,7 +80,7 @@ PixelRAG blocked-page 分類、外部報價正規化、正式 DB 安全邊界。 已完成: -- 正式環境版本真相 guard：`/health` 是最高真相，本輪推進目標為 `V10.745`。 +- 正式環境版本真相 guard：`/health` 是最高真相；2026-07-11 開工基線為 `V10.775`，本輪 source 目標為 `V10.776`。 - Gitea `dev` / `main` source truth 對齊。 - Retry exception controlled apply executor 已落地，目標表為 `pchome_product_matches`。 - 正式 DB 已 controlled apply 4 個 selectors: diff --git a/governance/pchome_growth_program_review.json b/governance/pchome_growth_program_review.json index 9498f10..cb80f5a 100644 --- a/governance/pchome_growth_program_review.json +++ b/governance/pchome_growth_program_review.json @@ -1,6 +1,6 @@ { "version": "pchome_growth_program_review_v1", - "generated_at": "2026-07-10T21:54:36+08:00", + "generated_at": "2026-07-11T12:00:00+08:00", "supersedes_status_only_completion": true, "goal": "以新鮮 PChome 業績、可信跨平台同款資料與可歸因動作持續提升業績", "production_baseline": { @@ -73,12 +73,17 @@ "objective": "Automatically acquire and import the latest PChome sales report", "completed": [ "Google Drive polling and import", + "authorized HTTPS, IMAP and controlled local-drop provider implementation", + "SHA-256 source idempotency", + "atomic two-table replace with rollback", + "same-run source, policy, execution and freshness receipt", "post-import table verification", - "staleness detection" + "staleness detection", + "compact source-readiness cockpit" ], "remaining": [ - "authorized PChome backend, scheduled export, or mailbox-to-Drive acquisition connector", - "same-run source receipt and freshness verifier" + "configure at least one production authorized upstream export provider", + "receive and import a report dated within one day in production" ], "production_acceptance": "latest sales age <= 1 day and decision_ready=true" }, @@ -197,10 +202,14 @@ "removed fake sales sparkline", "desktop and 390px mobile visual verification without horizontal overflow", "repaired the circular accent-token fallback that made primary buttons unreadable", - "production V10.775 static asset readback" + "production V10.775 static asset readback", + "sales automation first-viewport cockpit", + "four-source readiness and runtime-closure readback", + "manual upload removed from the primary workflow", + "six-column verified-run history" ], "remaining": [ - "production visible readback", + "production V10.776 visible readback", "per-product cross-platform offer drawer and experiment outcome view" ], "production_acceptance": "first viewport shows freshness, revenue trend, coverage, risk and one executable next action without overflow" @@ -239,11 +248,13 @@ "completed": [ "source and runtime review artifacts", "read-only visual evidence boundary", - "production DB lifecycle red lines" + "production DB lifecycle red lines", + "OpenTelemetry-compatible trace_id and span_id plus run_id and work_item_id envelope for sales acquisition", + "medium-risk controlled apply without legacy HITL terminal", + "durable acquisition receipt and atomic rollback verifier" ], "remaining": [ "replace preview-only gate chains with executable work items", - "shared trace_id, run_id and work_item_id envelope", "SBOM and dependency policy enforcement", "automated drift work item creation", "security control coverage and production closure metrics" diff --git a/migrations/046_pchome_sales_acquisition_receipts.sql b/migrations/046_pchome_sales_acquisition_receipts.sql new file mode 100644 index 0000000..3e8e2c0 --- /dev/null +++ b/migrations/046_pchome_sales_acquisition_receipts.sql @@ -0,0 +1,53 @@ +-- ============================================================================= +-- Migration 046: PChome authorized sales acquisition controlled-apply receipts +-- Additive only. No product, price, sales, or historical rows are rewritten. +-- ============================================================================= + +CREATE TABLE IF NOT EXISTS pchome_sales_acquisition_receipts ( + receipt_id VARCHAR(64) PRIMARY KEY, + trace_id VARCHAR(64) NOT NULL, + span_id VARCHAR(16) NOT NULL, + run_id VARCHAR(64) NOT NULL, + work_item_id VARCHAR(100) NOT NULL, + status VARCHAR(40) NOT NULL, + source_type VARCHAR(40) NOT NULL, + source_fingerprint VARCHAR(64), + source_ref_hash VARCHAR(64), + source_file_name VARCHAR(500), + risk_level VARCHAR(20) NOT NULL DEFAULT 'medium', + policy_decision VARCHAR(40) NOT NULL, + decision VARCHAR(80) NOT NULL, + import_job_id INTEGER, + rows_imported INTEGER NOT NULL DEFAULT 0, + before_freshness_json TEXT, + stage_receipts_json TEXT, + verifier_json TEXT, + error_kind VARCHAR(100), + public_message TEXT, + started_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP +); + +ALTER TABLE pchome_sales_acquisition_receipts + ADD COLUMN IF NOT EXISTS span_id VARCHAR(16); +UPDATE pchome_sales_acquisition_receipts +SET span_id = SUBSTRING(MD5(receipt_id), 1, 16) +WHERE span_id IS NULL; +ALTER TABLE pchome_sales_acquisition_receipts + ALTER COLUMN span_id SET NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_pchome_sales_acq_trace ON pchome_sales_acquisition_receipts (trace_id); +CREATE INDEX IF NOT EXISTS idx_pchome_sales_acq_span ON pchome_sales_acquisition_receipts (span_id); +CREATE INDEX IF NOT EXISTS idx_pchome_sales_acq_run ON pchome_sales_acquisition_receipts (run_id); +CREATE INDEX IF NOT EXISTS idx_pchome_sales_acq_work_item ON pchome_sales_acquisition_receipts (work_item_id); +CREATE INDEX IF NOT EXISTS idx_pchome_sales_acq_status ON pchome_sales_acquisition_receipts (status); +CREATE INDEX IF NOT EXISTS idx_pchome_sales_acq_source ON pchome_sales_acquisition_receipts (source_type); +CREATE INDEX IF NOT EXISTS idx_pchome_sales_acq_fingerprint ON pchome_sales_acquisition_receipts (source_fingerprint); +CREATE INDEX IF NOT EXISTS idx_pchome_sales_acq_job ON pchome_sales_acquisition_receipts (import_job_id); + +GRANT ALL PRIVILEGES ON pchome_sales_acquisition_receipts TO momo; + +DO $$ +BEGIN + RAISE NOTICE 'Migration 046 complete: PChome sales acquisition receipts are ready'; +END $$; diff --git a/requirements.txt b/requirements.txt index 7aced09..4370b99 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ gunicorn>=20.1 pandas>=1.5 pytz>=2026.2 openpyxl>=3.1.5 +xlrd>=2.0.1 SQLAlchemy>=1.4 psycopg2-binary>=2.9 schedule>=1.2.2 diff --git a/routes/auto_import_routes.py b/routes/auto_import_routes.py index 2b3b5de..873d2a6 100644 --- a/routes/auto_import_routes.py +++ b/routes/auto_import_routes.py @@ -8,6 +8,7 @@ from flask import Blueprint, render_template, jsonify, request import logging from services.import_service import humanize_import_error, import_service from services.google_drive_service import drive_service +from services.pchome_sales_acquisition_service import pchome_sales_acquisition_service # 建立 Blueprint auto_import_bp = Blueprint('auto_import', __name__) @@ -193,9 +194,9 @@ def list_drive_files(): @auto_import_bp.route('/api/manual_import', methods=['POST']) def manual_import(): - """手動觸發匯入""" + """Backward-compatible command endpoint for the automated acquisition run.""" try: - result = import_service.auto_import_from_drive() + result = pchome_sales_acquisition_service.run(trigger='operator_command') return jsonify(result) @@ -207,6 +208,33 @@ def manual_import(): }), 500 +@auto_import_bp.route('/api/auto_import/readiness', methods=['GET']) +def auto_import_readiness(): + """Return public-safe provider, freshness, and runtime-closure state.""" + try: + return jsonify(pchome_sales_acquisition_service.readiness()) + except Exception: + logger.exception("取得業績來源自動化狀態失敗") + return jsonify({ + 'success': False, + 'message': '暫時無法取得業績來源自動化狀態。' + }), 500 + + +@auto_import_bp.route('/api/auto_import/run', methods=['POST']) +def run_auto_import(): + """Run the governed acquisition, controlled apply, and verifier chain.""" + try: + return jsonify(pchome_sales_acquisition_service.run(trigger='operator_command')) + except Exception as e: + logger.exception("執行業績來源自動化失敗") + return jsonify({ + 'success': False, + 'status': 'failed_rolled_back', + 'message': f'自動取得與匯入失敗。{humanize_import_error(e)}' + }), 500 + + @auto_import_bp.route('/api/reset_stuck_jobs', methods=['POST']) def reset_stuck_jobs(): """重置逾時未完成的匯入任務""" diff --git a/scheduler.py b/scheduler.py index 383cdeb..3c6963a 100644 --- a/scheduler.py +++ b/scheduler.py @@ -2016,38 +2016,61 @@ def verify_import_data_sync(expected_rows: int = None, date_range: dict = None) def run_auto_import_task(): """ - V-New: 自動從 Google Drive 匯入當日業績 - 每半小時檢查一次 Google Drive 是否有新的 Excel 檔案 + 每半小時從授權來源取得當日業績，執行交易式匯入與獨立驗證。 """ - # ADR-012 Phase 4: HITL 暫停檢查 notification_sent = False try: from services.agent_actions import is_task_paused if is_task_paused("run_auto_import_task"): - logging.info("[Scheduler] [AutoImport] ⏸️ 任務被 HITL 暫停中，本次跳過") - return + logging.warning( + "[Scheduler] [AutoImport] legacy HITL pause is superseded for this medium-risk " + "controlled-apply lane; continuing with bounded execution" + ) except Exception as pause_check_error: logging.debug( - f"[Scheduler] [AutoImport] HITL 暫停檢查失敗但繼續排程 | Error: {pause_check_error}", + f"[Scheduler] [AutoImport] legacy pause readback failed; continuing | Error: {pause_check_error}", exc_info=True, ) try: - from services.import_service import import_service from services.notification_manager import NotificationManager + from services.pchome_sales_acquisition_service import pchome_sales_acquisition_service - logging.info("[Scheduler] [AutoImport] 🚀 啟動 Google Drive 自動匯入任務") + logging.info("[Scheduler] [AutoImport] 啟動授權來源自動取得與交易式匯入") - # 執行自動匯入 - result = import_service.auto_import_from_drive() + result = pchome_sales_acquisition_service.run(trigger="scheduler") + run_status = result.get('status') - if result.get('success'): + if run_status in {'blocked_with_safe_next_action', 'degraded_no_write', 'partial'}: + logging.warning( + "[Scheduler] [AutoImport] 自動取得尚未完整閉環 | status=%s | trace_id=%s | latest=%s | lag=%s", + run_status, + result.get('trace_id'), + result.get('latest_sales_date'), + result.get('data_lag_days'), + ) + stats = { + "status": "Degraded", + "upstream_status": run_status, + "trace_id": result.get('trace_id'), + "receipt_id": result.get('receipt_id'), + "receipt_persisted": result.get('receipt_persisted'), + "latest_sales_date": result.get('latest_sales_date'), + "data_lag_days": result.get('data_lag_days'), + "decision_ready": result.get('decision_ready'), + "requires_upstream_acquisition": result.get('requires_upstream_acquisition', True), + "safe_next_action": result.get('safe_next_action'), + } + elif result.get('success'): logging.info(f"[Scheduler] [AutoImport] ✅ 自動匯入完成 | Message: {result.get('message')}") stats = { "file_count": result.get('file_count', 0), "imported_count": result.get('imported_count', 0), - "status": "Degraded" if result.get('status') in {'upstream_stale', 'upstream_missing'} else "Success", - "upstream_status": result.get('status'), + "status": "Success", + "upstream_status": run_status, + "trace_id": result.get('trace_id'), + "receipt_id": result.get('receipt_id'), + "receipt_persisted": result.get('receipt_persisted'), "latest_sales_date": result.get('latest_sales_date'), "data_lag_days": result.get('data_lag_days'), "decision_ready": result.get('decision_ready'), @@ -2087,6 +2110,7 @@ def run_auto_import_task(): f"📊 當日業績自動匯入通知 ({now_str})\n" f"{'='*30}\n" f"✅ 匯入狀態：成功\n" + f"🔎 Trace：{result.get('trace_id', '')[:12]}\n" f"📁 處理檔案數：{result.get('file_count', 0)} 個\n" f"📝 共匯入記錄：{result.get('total_rows', 0)} 筆\n" f"📅 數據日期期間：{date_range_str}\n" @@ -2134,7 +2158,12 @@ def run_auto_import_task(): logging.error(f"[Scheduler] [AutoImport] ❌ 自動匯入失敗 | Message: {result.get('message')}") stats = { "status": "Failed", - "error": result.get('message', 'Unknown error') + "error": result.get('message', 'Unknown error'), + "upstream_status": run_status, + "trace_id": result.get('trace_id'), + "receipt_id": result.get('receipt_id'), + "receipt_persisted": result.get('receipt_persisted'), + "safe_next_action": result.get('safe_next_action'), } # V-New: 匯入失敗時也發送通知 @@ -2146,8 +2175,9 @@ def run_auto_import_task(): f"{'='*30}\n" f"❌ 匯入狀態：失敗\n" f"📌 錯誤訊息：{result.get('message', 'Unknown error')}\n" + f"🔎 Trace：{str(result.get('trace_id') or '')[:12]}\n" f"{'='*30}\n" - f"請檢查 Google Drive 設定或手動匯入" + f"系統將依安全重試策略再次取得授權來源" ) notifier = NotificationManager() notifier._send_line_messages([message]) diff --git a/services/import_service.py b/services/import_service.py index 23dae7c..481b328 100644 --- a/services/import_service.py +++ b/services/import_service.py @@ -10,6 +10,7 @@ import json import os import re from datetime import date, datetime, timedelta +from pathlib import Path from typing import Any, Dict, Optional import pandas as pd @@ -299,7 +300,8 @@ def _read_daily_sales_excel(file_path: str) -> tuple[pd.DataFrame, dict]: """ candidates: list[dict] = [] - with pd.ExcelFile(file_path, engine="openpyxl") as excel: + excel_engine = "xlrd" if Path(file_path).suffix.lower() == ".xls" else "openpyxl" + with pd.ExcelFile(file_path, engine=excel_engine) as excel: for sheet_name in excel.sheet_names: try: preview = pd.read_excel( @@ -399,6 +401,116 @@ def _should_quarantine_failed_import(error_message: str) -> bool: ] return any(marker in error_message for marker in permanent_error_markers) + +def _prepare_monthly_dataframe(df: pd.DataFrame, conn, job_id: int) -> tuple[pd.DataFrame, list]: + """Normalize the sales dataframe to the existing monthly-table contract.""" + monthly_table = "realtime_sales_monthly" + df_monthly = df.drop(columns=["snapshot_date"], errors="ignore").copy() + df_monthly = df_monthly.rename( + columns={ + column: column.replace("%", "_pct").replace("(", "_").replace(")", "_") + for column in df_monthly.columns + } + ) + + target_columns = _table_columns(conn, monthly_table) + date_column = _find_daily_sales_date_column(df_monthly.columns) + if date_column and date_column != "日期" and "日期" in target_columns and "日期" not in df_monthly.columns: + df_monthly = df_monthly.rename(columns={date_column: "日期"}) + + extra_columns = set(df_monthly.columns) - target_columns + if extra_columns: + logger.warning("任務 %s 移除月報目標表未定義欄位: %s", job_id, sorted(extra_columns)) + df_monthly = df_monthly.drop(columns=sorted(extra_columns), errors="ignore") + + if "日期" not in df_monthly.columns: + raise ValueError("業績分析同步缺少日期欄位，已中止整批匯入") + if df_monthly.empty or not len(df_monthly.columns): + raise ValueError("業績分析同步沒有可寫入欄位，已中止整批匯入") + + parsed_monthly_dates = pd.to_datetime(df_monthly["日期"], errors="coerce") + if parsed_monthly_dates.isna().any(): + invalid_rows = int(parsed_monthly_dates.isna().sum()) + raise ValueError(f"業績分析同步有 {invalid_rows} 筆無效日期，已中止整批匯入") + df_monthly["日期"] = parsed_monthly_dates.dt.strftime("%Y-%m-%d") + return df_monthly, sorted(df_monthly["日期"].unique().tolist()) + + +def _replace_sales_tables_atomic(df: pd.DataFrame, job_id: int) -> Dict[str, Any]: + """Replace snapshot and monthly rows in one transaction and verify both writes.""" + snapshot_dates = _normalise_date_values_for_sql(df["snapshot_date"].dropna().unique()) + if not snapshot_dates: + raise ValueError("當日業績沒有可驗證的資料日期，已中止整批匯入") + + with engine.begin() as conn: + snapshot_placeholders, snapshot_params = _build_in_clause("snapshot_date", snapshot_dates) + snapshot_expr = _date_filter_expr("snapshot_date") + deleted_snapshot = conn.execute( + text( + "DELETE FROM daily_sales_snapshot " + f"WHERE {snapshot_expr} IN ({snapshot_placeholders})" + ), + snapshot_params, + ).rowcount + + df.to_sql( + "daily_sales_snapshot", + conn, + if_exists="append", + index=False, + method="multi", + chunksize=1000, + ) + snapshot_count = conn.execute( + text( + "SELECT COUNT(*) FROM daily_sales_snapshot " + f"WHERE {snapshot_expr} IN ({snapshot_placeholders})" + ), + snapshot_params, + ).scalar() + if snapshot_count != len(df): + raise RuntimeError( + f"當日業績寫入驗證失敗: 預期 {len(df)} 筆, 實際 {snapshot_count} 筆" + ) + + df_monthly, monthly_dates = _prepare_monthly_dataframe(df, conn, job_id) + monthly_placeholders, monthly_params = _build_in_clause("monthly_date", monthly_dates) + deleted_monthly = conn.execute( + text( + 'DELETE FROM realtime_sales_monthly WHERE "日期" ' + f"IN ({monthly_placeholders})" + ), + monthly_params, + ).rowcount + df_monthly.to_sql( + "realtime_sales_monthly", + conn, + if_exists="append", + index=False, + method="multi", + chunksize=1000, + ) + monthly_count = conn.execute( + text( + 'SELECT COUNT(*) FROM realtime_sales_monthly WHERE "日期" ' + f"IN ({monthly_placeholders})" + ), + monthly_params, + ).scalar() + if monthly_count != len(df_monthly): + raise RuntimeError( + f"業績分析寫入驗證失敗: 預期 {len(df_monthly)} 筆, 實際 {monthly_count} 筆" + ) + + return { + "snapshot_rows": snapshot_count, + "monthly_rows": monthly_count, + "deleted_snapshot_rows": deleted_snapshot, + "deleted_monthly_rows": deleted_monthly, + "verified": True, + "atomic": True, + } + # 資料庫設定 - 使用 config.py 中的設定，支援 PostgreSQL 和 SQLite def _create_engine_with_pool(db_path): """建立帶有連線池配置的資料庫引擎""" @@ -512,8 +624,8 @@ class ImportService: Args: job_type: 任務類型（daily_sales 或 vendor_stockout） - drive_file_id: Google Drive 檔案 ID - drive_file_name: 檔案名稱 + drive_file_id: Google Drive ID or governed source fingerprint + drive_file_name: Source file name drive_file_size: 檔案大小 Returns: @@ -720,6 +832,92 @@ class ImportService: finally: session.close() + def import_local_daily_sales( + self, + file_path: str, + *, + source_type: str, + source_fingerprint: str, + source_file_name: Optional[str] = None, + ) -> Dict[str, Any]: + """Import a provider-fetched sales file with fingerprint idempotency.""" + if not os.path.isfile(file_path) or os.path.islink(file_path): + return { + 'success': False, + 'status': 'rejected', + 'error_kind': 'invalid_local_file', + 'message': '來源檔案不存在或不符合安全檔案規則。', + } + + safe_source = re.sub(r'[^a-z0-9_-]+', '-', str(source_type).lower()).strip('-')[:40] + if not safe_source or not re.fullmatch(r'[0-9a-f]{64}', str(source_fingerprint or '')): + return { + 'success': False, + 'status': 'rejected', + 'error_kind': 'invalid_source_identity', + 'message': '來源識別或內容指紋無效，未執行寫入。', + } + + source_identity = f'{safe_source}:sha256:{source_fingerprint}' + session = Session() + try: + existing = session.query(ImportJob).filter_by( + job_type='daily_sales', + drive_file_id=source_identity, + status='completed', + ).order_by(ImportJob.id.desc()).first() + if existing: + return { + 'success': True, + 'status': 'duplicate_no_write', + 'duplicate': True, + 'job_id': existing.id, + 'job': _public_import_job_payload(existing), + 'message': '相同內容已完成匯入，本次以冪等方式略過寫入。', + } + finally: + session.close() + + file_name = os.path.basename(source_file_name or file_path)[:500] + job_id = self.create_import_job( + 'daily_sales', + source_identity, + file_name, + os.path.getsize(file_path), + ) + if not job_id: + return { + 'success': False, + 'status': 'failed', + 'error_kind': 'job_create_failed', + 'message': '無法建立匯入任務，正式資料未變更。', + } + + session = Session() + try: + job = session.query(ImportJob).filter_by(id=job_id).first() + if job: + job.local_file_path = file_path + session.commit() + finally: + session.close() + + self.update_job_status(job_id, 'downloading', 40, '授權來源檔案已取得') + success = self.process_daily_sales_import(job_id, file_path) + public_job = self.get_job_status(job_id) + return { + 'success': success, + 'status': 'completed' if success else 'failed_rolled_back', + 'duplicate': False, + 'job_id': job_id, + 'job': public_job, + 'message': ( + '業績檔已完成交易式匯入與雙表驗證。' + if success + else '業績檔未通過交易式匯入驗證，正式資料未變更。' + ), + } + def process_daily_sales_import(self, job_id: int, file_path: str) -> bool: """ 處理當日業績匯入 @@ -769,6 +967,15 @@ class ImportService: logger.error(error_msg) self.update_job_status(job_id, 'failed', 50, '日期驗證失敗', error_msg) return False + if parsed_dates.isna().any(): + invalid_rows = int(parsed_dates.isna().sum()) + error_msg = ( + f"Excel 日期防禦失敗：日期欄位「{date_col}」有 {invalid_rows} 筆無效日期，" + "為避免部分資料寫入已中止整批匯入" + ) + logger.error(error_msg) + self.update_job_status(job_id, 'failed', 50, '日期驗證失敗', error_msg) + return False df['snapshot_date'] = parsed_dates.dt.date logger.info(f"使用日期欄位: {date_col}") @@ -789,225 +996,88 @@ class ImportService: return False else: # 使用當前日期 - df['snapshot_date'] = datetime.now(TAIPEI_TZ).date() + fallback_date = datetime.now(TAIPEI_TZ).date() + df['snapshot_date'] = fallback_date + if '日期' not in df.columns: + df['日期'] = fallback_date.isoformat() logger.info("未找到日期欄位，使用當前日期（台北時區）") - # 寫入資料庫 - 使用全域的 engine（支援 PostgreSQL 和 SQLite） - # 使用模組頂部定義的 engine，確保連接到正確的資料庫 - - # 更新進度 total_rows = len(df) self.update_job_progress(job_id, total_rows=total_rows, processed_rows=0) - - # 取得此次匯入的日期範圍 - import_dates = df['snapshot_date'].unique() - logger.info(f"本次匯入包含 {len(import_dates)} 個日期的資料") - - # 刪除資料庫中相同日期的舊資料（覆蓋邏輯） - if len(import_dates) > 0: - # 過濾掉 None 值 - valid_dates = _normalise_date_values_for_sql(import_dates) - - if valid_dates: - date_placeholders, date_params = _build_in_clause("snapshot_date", valid_dates) - snapshot_date_expr = _date_filter_expr("snapshot_date") - - with engine.connect() as conn: - # 刪除相同日期的舊資料 - delete_query = text( - f"DELETE FROM {table_name} WHERE {snapshot_date_expr} IN ({date_placeholders})" - ) - result = conn.execute(delete_query, date_params) - deleted_count = result.rowcount - conn.commit() - - if deleted_count > 0: - logger.info(f"已刪除 {deleted_count} 筆舊資料（覆蓋模式）") - - # 寫入資料庫（帶驗證和重試機制） + self.update_job_status(job_id, 'importing', 60, '交易式寫入與雙表驗證...') max_retries = 2 - retry_count = 0 - write_success = False - - while retry_count <= max_retries and not write_success: - try: - if retry_count > 0: - logger.warning(f"任務 {job_id} 第 {retry_count} 次重試寫入...") - self.update_job_status(job_id, 'importing', 60, f'重試寫入中 ({retry_count}/{max_retries})...') - - df.to_sql( - table_name, - engine, - if_exists='append', - index=False, - method='multi', - chunksize=1000 - ) - - # V-Fix: 匯入後驗證 - 確認資料已正確寫入資料庫 - self.update_job_status(job_id, 'importing', 85, '驗證資料寫入...') - - # 取得本次匯入的日期 - import_dates = df['snapshot_date'].dropna().unique() - if len(import_dates) > 0: - # 查詢資料庫中這些日期的資料筆數 - raw_valid_dates = [d for d in import_dates if d is not None] - valid_dates = _normalise_date_values_for_sql(raw_valid_dates) - date_placeholders, date_params = _build_in_clause("verify_date", valid_dates) - snapshot_date_expr = _date_filter_expr("snapshot_date") - - with engine.connect() as conn: - verify_query = text( - f"SELECT COUNT(*) FROM {table_name} WHERE {snapshot_date_expr} IN ({date_placeholders})" - ) - result = conn.execute(verify_query, date_params) - db_count = result.scalar() - - # 驗證：資料庫筆數應該 >= 本次匯入筆數（可能有其他日期的舊資料） - expected_count = len(df[df['snapshot_date'].isin(raw_valid_dates)]) - - if db_count >= expected_count: - logger.info(f"任務 {job_id} 驗證成功: 預期 {expected_count} 筆, 資料庫有 {db_count} 筆") - write_success = True - else: - logger.warning(f"任務 {job_id} 驗證失敗: 預期 {expected_count} 筆, 資料庫只有 {db_count} 筆") - retry_count += 1 - else: - # 沒有有效日期，跳過驗證 - logger.warning(f"任務 {job_id} 無法驗證: 沒有有效的 snapshot_date") - write_success = True - - except Exception as write_error: - logger.error(f"任務 {job_id} 寫入失敗 (嘗試 {retry_count + 1}): {str(write_error)}") - retry_count += 1 - if retry_count > max_retries: - raise write_error - - if not write_success: - error_msg = f"資料寫入驗證失敗，已重試 {max_retries} 次" - self.update_job_status(job_id, 'failed', 85, '驗證失敗', error_msg) - logger.error(f"任務 {job_id} {error_msg}") - return False - - # === V-New 2026-01-15: 同步寫入 realtime_sales_monthly === - # 目的：讓當日業績 raw data 同時呈現在「業績分析儀表板」 - # 2026-01-30 修復：加強欄位驗證、同步狀態追蹤、失敗告警 - self.update_job_status(job_id, 'importing', 90, '同步至業績分析儀表板...') - - sync_success = False + write_receipt = None sync_error_msg = None - monthly_table = 'realtime_sales_monthly' - - try: - # 準備資料：移除 snapshot_date 欄位（realtime_sales_monthly 不需要此欄位） - df_monthly = df.drop(columns=['snapshot_date'], errors='ignore') - - # 2026-01-30 修正：強化欄位名稱轉換 - # 將特殊字符轉換為 PostgreSQL 安全格式 - column_mapping = {} - for col in df_monthly.columns: - new_col = col.replace('%', '_pct').replace('(', '_').replace(')', '_') - column_mapping[col] = new_col - df_monthly = df_monthly.rename(columns=column_mapping) - - # 記錄轉換的欄位 - converted_cols = [f"'{k}' -> '{v}'" for k, v in column_mapping.items() if k != v] - if converted_cols: - logger.info(f"任務 {job_id} 欄位名稱轉換: {', '.join(converted_cols)}") - logger.info(f"任務 {job_id} 欄位轉換完成，共 {len(df_monthly.columns)} 個欄位") - - # 2026-01-30 新增：驗證 DataFrame 欄位和目標表欄位是否一致 - with engine.connect() as conn: - target_columns = _table_columns(conn, monthly_table) - - df_columns = set(df_monthly.columns) - missing_in_table = df_columns - target_columns - missing_in_df = target_columns - df_columns - - if missing_in_table: - logger.warning(f"任務 {job_id} 欄位警告: DataFrame 有但表中沒有: {missing_in_table}") - # 移除表中沒有的欄位，避免 INSERT 失敗 - df_monthly = df_monthly.drop(columns=list(missing_in_table), errors='ignore') - logger.info(f"任務 {job_id} 已移除多餘欄位，剩餘 {len(df_monthly.columns)} 個欄位") - - if missing_in_df: - logger.warning(f"任務 {job_id} 欄位警告: 表中有但 DataFrame 沒有: {missing_in_df}") - - # 取得本次匯入的日期列表（使用原始「日期」欄位） - unique_dates = [] - if '日期' in df.columns: - unique_dates = df['日期'].dropna().unique().tolist() - logger.info(f"任務 {job_id} 準備同步 {len(unique_dates)} 個日期的資料") - - if len(unique_dates) > 0: - # 刪除 realtime_sales_monthly 中相同日期的舊資料（去重） - date_placeholders, date_params = _build_in_clause("monthly_date", unique_dates) - - with engine.connect() as conn: - delete_monthly_query = text( - f'DELETE FROM {monthly_table} WHERE "日期" IN ({date_placeholders})' - ) - result = conn.execute(delete_monthly_query, date_params) - deleted_monthly = result.rowcount - conn.commit() - - if deleted_monthly > 0: - logger.info(f"任務 {job_id} 已從 {monthly_table} 刪除 {deleted_monthly} 筆同日期舊資料") - - # 寫入 realtime_sales_monthly - df_monthly.to_sql( - monthly_table, - engine, - if_exists='append', - index=False, - method='multi', - chunksize=1000 - ) - - logger.info(f"任務 {job_id} 已同步 {len(df_monthly)} 筆資料至 {monthly_table}") - - # 驗證同步結果 - if len(unique_dates) > 0: - with engine.connect() as conn: - date_placeholders, date_params = _build_in_clause("monthly_verify_date", unique_dates) - verify_query = text( - f'SELECT COUNT(*) FROM {monthly_table} WHERE "日期" IN ({date_placeholders})' + for attempt in range(max_retries + 1): + try: + if attempt: + self.update_job_status( + job_id, + 'importing', + 60, + f'整批交易重試中 ({attempt}/{max_retries})...' ) - verify_count = conn.execute(verify_query, date_params).scalar() + write_receipt = _replace_sales_tables_atomic(df, job_id) + break + except Exception as write_error: + sync_error_msg = str(write_error) + logger.error( + "任務 %s 交易式寫入失敗 (嘗試 %s/%s): %s", + job_id, + attempt + 1, + max_retries + 1, + sync_error_msg, + exc_info=True, + ) - if verify_count >= len(df_monthly): - logger.info(f"任務 {job_id} 同步驗證成功: {monthly_table} 現有 {verify_count} 筆資料") - sync_success = True - else: - sync_error_msg = f"同步驗證失敗: 預期 {len(df_monthly)} 筆, 實際 {verify_count} 筆" - logger.error(f"任務 {job_id} {sync_error_msg}") - else: - sync_success = True # 沒有日期資料時視為成功 + sync_success = bool(write_receipt and write_receipt.get('verified')) + monthly_table = 'realtime_sales_monthly' + if not sync_success: + summary = { + 'imported_count': 0, + 'table_name': table_name, + 'synced_to': None, + 'sync_success': False, + 'sync_error': sync_error_msg, + 'verified': False, + 'atomic': True, + 'rolled_back': True, + 'source_sheet': excel_metadata.get("sheet_name"), + 'source_header_row': excel_metadata.get("header_row"), + 'message': '業績資料驗證失敗，整批交易已回滾，正式資料未變更。', + } + session = Session() + try: + job = session.query(ImportJob).filter_by(id=job_id).first() + if job: + job.import_summary = json.dumps(summary, ensure_ascii=False) + session.commit() + finally: + session.close() - except Exception as sync_error: - # 同步失敗，記錄完整錯誤 - import traceback - sync_error_msg = str(sync_error) - logger.error(f"任務 {job_id} 同步至 {monthly_table} 失敗: {sync_error_msg}") - logger.error(f"任務 {job_id} 同步錯誤堆疊:\n{traceback.format_exc()}") - - # 2026-01-30 新增：發送同步失敗告警 + self.update_job_progress( + job_id, + processed_rows=total_rows, + success_rows=0, + error_rows=total_rows, + ) + self.update_job_status( + job_id, + 'failed', + 95, + '交易驗證失敗，已回滾', + sync_error_msg or '交易式寫入驗證失敗', + ) try: from services.notification_manager import NotificationManager - notifier = NotificationManager() - alert_msg = ( - f"⚠️ 業績資料同步失敗告警\n" - f"{'='*30}\n" - f"任務 ID: {job_id}\n" - f"目標表: {monthly_table}\n" - f"錯誤: {sync_error_msg[:200]}\n" - f"{'='*30}\n" - f"daily_sales_snapshot 已匯入成功，但業績分析儀表板需要手動同步" - ) - notifier._send_telegram_messages([alert_msg]) - logger.info(f"任務 {job_id} 已發送同步失敗告警") + NotificationManager()._send_telegram_messages([ + f"業績匯入失敗（任務 {job_id}）：雙表交易已回滾，正式資料未變更。" + ]) except Exception as notify_error: - logger.error(f"任務 {job_id} 發送告警失敗: {notify_error}") + logger.error("任務 %s 發送回滾告警失敗: %s", job_id, notify_error) + return False + + self.update_job_status(job_id, 'importing', 90, '雙表交易驗證完成') # 計算日期範圍 @@ -1170,7 +1240,7 @@ class ImportService: if isinstance(last_date, date) else date.fromisoformat(str(last_date)[:10]) ) - days_since = (date.today() - normalized_last_date).days + days_since = (datetime.now(TAIPEI_TZ).date() - normalized_last_date).days data_lag_days = days_since latest_sales_date = str(normalized_last_date) if days_since >= 3: @@ -1212,6 +1282,7 @@ class ImportService: total_rows = 0 all_dates = [] # 收集所有匯入的日期 failed_files = [] + archive_failed_files = [] for file in files: file_id = file['id'] @@ -1260,10 +1331,20 @@ class ImportService: if drive_service.move_file(file_id, archive_folder): logger.info(f"已移動 Google Drive 檔案到「{archive_folder}」: {file_name}") + self.update_job_status(job_id, 'completed', 100, '完成') else: logger.warning(f"無法移動 Google Drive 檔案: {file_name}") - - self.update_job_status(job_id, 'completed', 100, '完成') + archive_failed_files.append({ + 'file': file_name, + 'job_id': job_id, + 'error': '業績已匯入，但 Google Drive 來源封存失敗', + }) + self.update_job_status( + job_id, + 'completed', + 100, + '匯入完成，來源封存待自動重試', + ) imported_count += 1 # 讀取 job summary 取得匯入筆數和日期範圍 @@ -1347,15 +1428,29 @@ class ImportService: f'找到 {len(files)} 個檔案，成功匯入 {imported_count} 個，' f'失敗 {len(failed_files)} 個。{public_error}' ) + elif archive_failed_files: + message = ( + f'成功匯入 {imported_count} 個檔案，但有 {len(archive_failed_files)} 個來源封存待重試' + ) else: message = f'成功匯入 {imported_count} 個檔案' return { - 'success': len(failed_files) == 0, + 'success': len(failed_files) == 0 and len(archive_failed_files) == 0, + 'status': ( + 'partial_source_finalize_failed' + if archive_failed_files + else 'completed' + if not failed_files + else 'failed' + ), 'message': message, 'file_count': len(files), 'imported_count': imported_count, 'failed_count': len(failed_files), + 'source_finalize_ok': len(archive_failed_files) == 0, + 'archive_failed_count': len(archive_failed_files), + 'archive_errors': archive_failed_files, 'errors': failed_files, 'total_rows': total_rows, 'date_range': date_range diff --git a/services/pchome_sales_acquisition_providers.py b/services/pchome_sales_acquisition_providers.py new file mode 100644 index 0000000..8e8483a --- /dev/null +++ b/services/pchome_sales_acquisition_providers.py @@ -0,0 +1,380 @@ +"""Authorized PChome sales report providers and file-boundary guards.""" + +from __future__ import annotations + +import hashlib +import http.client +import imaplib +import ipaddress +import os +import re +import shutil +import socket +import ssl +import tempfile +from dataclasses import dataclass +from email import policy +from email.header import decode_header, make_header +from email.parser import BytesParser +from pathlib import Path +from typing import Callable, List, Optional +from urllib.parse import unquote, urlparse + + +VALID_EXTENSIONS = {".xlsx", ".xls"} +XLSX_MAGIC = b"PK\x03\x04" +XLS_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + + +class ProviderError(RuntimeError): + def __init__(self, kind: str, public_message: str): + super().__init__(public_message) + self.kind = kind + self.public_message = public_message + + +class _PinnedHTTPSConnection(http.client.HTTPSConnection): + """Connect to the validated IP while preserving hostname TLS verification.""" + + def __init__(self, hostname: str, port: int, pinned_ip: str, timeout: int): + super().__init__(hostname, port=port, timeout=timeout, context=ssl.create_default_context()) + self._pinned_ip = pinned_ip + + def connect(self): + self.sock = socket.create_connection( + (self._pinned_ip, self.port), + self.timeout, + self.source_address, + ) + self.sock = self._context.wrap_socket(self.sock, server_hostname=self.host) + + +@dataclass +class AcquisitionCandidate: + source_type: str + file_path: str + file_name: str + fingerprint: str + source_ref_hash: str + finalize_success: Callable[[], None] + finalize_rejected: Callable[[], None] + cleanup: Callable[[], None] + + +def env_bool(name: str, default: bool = False) -> bool: + fallback = "true" if default else "false" + return os.getenv(name, fallback).strip().lower() in {"1", "true", "yes", "on"} + + +def env_int(name: str, default: int, minimum: int = 1, maximum: int = 1000) -> int: + try: + value = int(os.getenv(name, str(default))) + except (TypeError, ValueError): + value = default + return max(minimum, min(value, maximum)) + + +def _hash_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8", errors="ignore")).hexdigest() + + +def _sha256_file(file_path: str) -> str: + digest = hashlib.sha256() + with open(file_path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _safe_file_name(value: str, default: str = "pchome-sales.xlsx") -> str: + candidate = os.path.basename(str(value or "").replace("\x00", "")).strip() + candidate = re.sub(r"[^0-9A-Za-z._()\-\u4e00-\u9fff]+", "_", candidate) + return (candidate or default)[:180] + + +def _decode_mail_header(value: Optional[str]) -> str: + if not value: + return "" + try: + return str(make_header(decode_header(value))) + except Exception: + return str(value) + + +def _make_temp_path(suffix: str) -> str: + target = Path(os.getenv("PCHOME_SALES_TEMP_DIR", "data/temp/pchome-sales")) + target.mkdir(parents=True, exist_ok=True) + fd, file_path = tempfile.mkstemp(prefix="sales-", suffix=suffix, dir=str(target)) + os.close(fd) + return file_path + + +def _validate_excel_file(file_path: str, file_name: str) -> None: + path = Path(file_path) + max_bytes = env_int("PCHOME_SALES_MAX_FILE_BYTES", 25 * 1024 * 1024, 1024, 200 * 1024 * 1024) + if not path.is_file() or path.is_symlink(): + raise ProviderError("unsafe_file", "來源檔案不存在或不符合安全檔案規則。") + if path.stat().st_size <= 0 or path.stat().st_size > max_bytes: + raise ProviderError("invalid_file_size", "來源檔案大小不符合匯入政策，未執行寫入。") + suffix = Path(file_name).suffix.lower() + if suffix not in VALID_EXTENSIONS: + raise ProviderError("unsupported_file_type", "來源不是允許的 Excel 業績檔，未執行寫入。") + with path.open("rb") as handle: + magic = handle.read(8) + signature_matches = ( + suffix == ".xlsx" and magic.startswith(XLSX_MAGIC) + ) or ( + suffix == ".xls" and magic == XLS_MAGIC + ) + if not signature_matches: + raise ProviderError("invalid_excel_signature", "來源檔案內容不是有效的 Excel 格式，未執行寫入。") + + +def _archive_file(source: Path, folder_name: str, fingerprint: str) -> None: + if not source.exists() or source.is_symlink(): + return + target_dir = source.parent / folder_name + target_dir.mkdir(parents=True, exist_ok=True) + target = target_dir / source.name + if target.exists(): + target = target_dir / f"{source.stem}-{fingerprint[:10]}{source.suffix}" + os.replace(str(source), str(target)) + + +class AuthorizedSalesProviders: + """Side-effect-bounded report providers in governed priority order.""" + + def validate_http_url(self, url: str) -> tuple[str, int, str]: + parsed = urlparse(url) + if parsed.scheme.lower() != "https" or not parsed.hostname or parsed.username or parsed.password: + raise ProviderError("http_url_rejected", "授權報表網址必須是無內嵌帳密的 HTTPS 網址。") + hostname = parsed.hostname.lower().rstrip(".") + try: + port = parsed.port or 443 + except ValueError as exc: + raise ProviderError("http_port_rejected", "授權報表網址的連接埠格式無效。") from exc + allowed_ports = { + int(item.strip()) + for item in os.getenv("PCHOME_SALES_HTTP_ALLOWED_PORTS", "443").split(",") + if item.strip().isdigit() + } + if port not in allowed_ports: + raise ProviderError("http_port_rejected", "授權報表連接埠未列入允許清單，未發出連線。") + allowed_hosts = { + item.strip().lower().rstrip(".") + for item in os.getenv("PCHOME_SALES_HTTP_ALLOWED_HOSTS", "").split(",") + if item.strip() + } + if hostname not in allowed_hosts: + raise ProviderError("http_host_not_allowed", "授權報表主機未列入允許清單，未發出連線。") + try: + addresses = { + item[4][0] + for item in socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + } + except OSError as exc: + raise ProviderError("http_dns_failed", "授權報表主機目前無法解析。") from exc + if not addresses: + raise ProviderError("http_dns_failed", "授權報表主機目前沒有可用位址。") + if not env_bool("PCHOME_SALES_HTTP_ALLOW_PRIVATE"): + for address in addresses: + ip = ipaddress.ip_address(address) + if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved: + raise ProviderError("http_private_address_rejected", "授權報表主機解析到受限制網段，未發出連線。") + return hostname, port, sorted(addresses)[0] + + def http_candidates(self) -> List[AcquisitionCandidate]: + if not env_bool("PCHOME_SALES_HTTP_ENABLED"): + return [] + url = os.getenv("PCHOME_SALES_HTTP_URL", "").strip() + if not url: + raise ProviderError("http_not_configured", "授權 HTTPS 來源已啟用但網址尚未設定。") + hostname, port, pinned_ip = self.validate_http_url(url) + parsed = urlparse(url) + headers = {"Accept": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet, application/vnd.ms-excel"} + token = os.getenv("PCHOME_SALES_HTTP_BEARER_TOKEN", "").strip() + if token: + headers["Authorization"] = f"Bearer {token}" + timeout = env_int("PCHOME_SALES_HTTP_TIMEOUT_SECONDS", 30, 3, 120) + max_bytes = env_int("PCHOME_SALES_MAX_FILE_BYTES", 25 * 1024 * 1024, 1024, 200 * 1024 * 1024) + connection = _PinnedHTTPSConnection(hostname, port, pinned_ip, timeout) + path = parsed.path or "/" + if parsed.query: + path = f"{path}?{parsed.query}" + try: + connection.request("GET", path, headers=headers) + response = connection.getresponse() + if response.status != 200: + raise ProviderError("http_status_rejected", f"授權 HTTPS 報表來源回應 {response.status}，未執行匯入。") + content_length = response.getheader("Content-Length") + if content_length and int(content_length) > max_bytes: + raise ProviderError("http_file_too_large", "授權報表超過檔案大小上限，未執行匯入。") + disposition = response.getheader("Content-Disposition", "") + match = re.search(r"filename\*?=(?:UTF-8''|\")?([^\";]+)", disposition, re.I) + raw_name = unquote(match.group(1).strip()) if match else os.path.basename(parsed.path) + file_name = _safe_file_name(raw_name) + suffix = Path(file_name).suffix.lower() + if suffix not in VALID_EXTENSIONS: + suffix = ".xlsx" + file_name = f"{Path(file_name).stem or 'pchome-sales'}{suffix}" + temp_path = _make_temp_path(suffix) + written = 0 + try: + with open(temp_path, "wb") as handle: + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + written += len(chunk) + if written > max_bytes: + raise ProviderError("http_file_too_large", "授權報表超過檔案大小上限，未執行匯入。") + handle.write(chunk) + _validate_excel_file(temp_path, file_name) + except Exception: + Path(temp_path).unlink(missing_ok=True) + raise + except ProviderError: + raise + except (OSError, ssl.SSLError, http.client.HTTPException, ValueError) as exc: + raise ProviderError("http_fetch_failed", "授權 HTTPS 報表來源目前無法連線。") from exc + finally: + connection.close() + fingerprint = _sha256_file(temp_path) + return [AcquisitionCandidate( + source_type="authorized_https", + file_path=temp_path, + file_name=file_name, + fingerprint=fingerprint, + source_ref_hash=_hash_text(f"https://{hostname}{parsed.path}"), + finalize_success=lambda: None, + finalize_rejected=lambda: None, + cleanup=lambda candidate_path=temp_path: Path(candidate_path).unlink(missing_ok=True), + )] + + def _mark_imap_seen(self, uid: bytes) -> None: + host = os.getenv("PCHOME_SALES_IMAP_HOST", "").strip() + user = os.getenv("PCHOME_SALES_IMAP_USER", "").strip() + password = os.getenv("PCHOME_SALES_IMAP_PASSWORD", "") + port = env_int("PCHOME_SALES_IMAP_PORT", 993, 1, 65535) + mailbox = os.getenv("PCHOME_SALES_IMAP_MAILBOX", "INBOX").strip() or "INBOX" + client = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context(), timeout=20) + try: + client.login(user, password) + client.select(mailbox, readonly=False) + client.uid("store", uid, "+FLAGS", "(\\Seen)") + finally: + try: + client.logout() + except Exception: + pass + + def imap_candidates(self) -> List[AcquisitionCandidate]: + if not env_bool("PCHOME_SALES_IMAP_ENABLED"): + return [] + host = os.getenv("PCHOME_SALES_IMAP_HOST", "").strip() + user = os.getenv("PCHOME_SALES_IMAP_USER", "").strip() + password = os.getenv("PCHOME_SALES_IMAP_PASSWORD", "") + if not host or not user or not password: + raise ProviderError("imap_not_configured", "授權信箱來源已啟用但連線設定不完整。") + port = env_int("PCHOME_SALES_IMAP_PORT", 993, 1, 65535) + mailbox = os.getenv("PCHOME_SALES_IMAP_MAILBOX", "INBOX").strip() or "INBOX" + subject_pattern = os.getenv("PCHOME_SALES_IMAP_SUBJECT", "即時業績").strip() + max_messages = env_int("PCHOME_SALES_IMAP_MAX_MESSAGES", 20, 1, 100) + candidates: List[AcquisitionCandidate] = [] + client = None + try: + client = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context(), timeout=20) + client.login(user, password) + status, _ = client.select(mailbox, readonly=True) + if status != "OK": + raise ProviderError("imap_mailbox_failed", "授權信箱資料夾目前無法開啟。") + status, data = client.uid("search", None, "UNSEEN") + if status != "OK": + raise ProviderError("imap_search_failed", "授權信箱目前無法搜尋新報表。") + uids = list(reversed((data[0] or b"").split()))[:max_messages] + for uid in uids: + status, payload = client.uid("fetch", uid, "(RFC822)") + if status != "OK" or not payload or not isinstance(payload[0], tuple): + continue + message = BytesParser(policy=policy.default).parsebytes(payload[0][1]) + if subject_pattern and subject_pattern not in _decode_mail_header(message.get("Subject")): + continue + for attachment in message.iter_attachments(): + file_name = _safe_file_name(_decode_mail_header(attachment.get_filename())) + suffix = Path(file_name).suffix.lower() + if suffix not in VALID_EXTENSIONS: + continue + temp_path = _make_temp_path(suffix) + try: + with open(temp_path, "wb") as handle: + handle.write(attachment.get_payload(decode=True) or b"") + _validate_excel_file(temp_path, file_name) + except Exception: + Path(temp_path).unlink(missing_ok=True) + raise + fingerprint = _sha256_file(temp_path) + candidates.append(AcquisitionCandidate( + source_type="authorized_imap", + file_path=temp_path, + file_name=file_name, + fingerprint=fingerprint, + source_ref_hash=_hash_text(f"{host}:{mailbox}:{uid.decode(errors='ignore')}"), + finalize_success=lambda message_uid=uid: self._mark_imap_seen(message_uid), + finalize_rejected=lambda message_uid=uid: self._mark_imap_seen(message_uid), + cleanup=lambda candidate_path=temp_path: Path(candidate_path).unlink(missing_ok=True), + )) + except ProviderError: + for candidate in candidates: + candidate.cleanup() + raise + except (imaplib.IMAP4.error, OSError, ssl.SSLError) as exc: + for candidate in candidates: + candidate.cleanup() + raise ProviderError("imap_connection_failed", "授權信箱來源目前無法連線或驗證。") from exc + finally: + if client is not None: + try: + client.logout() + except Exception: + pass + return candidates + + def local_candidates(self) -> List[AcquisitionCandidate]: + configured = os.getenv("PCHOME_SALES_LOCAL_DROP_DIR", "").strip() + if not configured: + return [] + root = Path(configured).expanduser().resolve() + if not root.is_dir(): + raise ProviderError("local_drop_missing", "受控落地目錄不存在，未執行檔案存取。") + max_files = env_int("PCHOME_SALES_MAX_FILES_PER_RUN", 5, 1, 25) + sources = sorted( + ( + path for path in root.iterdir() + if path.is_file() and not path.is_symlink() and path.suffix.lower() in VALID_EXTENSIONS + ), + key=lambda path: path.stat().st_mtime, + reverse=True, + )[:max_files] + candidates: List[AcquisitionCandidate] = [] + for source in sources: + temp_path = _make_temp_path(source.suffix.lower()) + try: + shutil.copyfile(str(source), temp_path) + _validate_excel_file(temp_path, source.name) + except Exception: + Path(temp_path).unlink(missing_ok=True) + for candidate in candidates: + candidate.cleanup() + raise + fingerprint = _sha256_file(temp_path) + candidates.append(AcquisitionCandidate( + source_type="controlled_local_drop", + file_path=temp_path, + file_name=_safe_file_name(source.name), + fingerprint=fingerprint, + source_ref_hash=_hash_text(str(source)), + finalize_success=lambda item=source, fp=fingerprint: _archive_file(item, "archive", fp), + finalize_rejected=lambda item=source, fp=fingerprint: _archive_file(item, "quarantine", fp), + cleanup=lambda candidate_path=temp_path: Path(candidate_path).unlink(missing_ok=True), + )) + return candidates diff --git a/services/pchome_sales_acquisition_service.py b/services/pchome_sales_acquisition_service.py new file mode 100644 index 0000000..1d3025a --- /dev/null +++ b/services/pchome_sales_acquisition_service.py @@ -0,0 +1,498 @@ +"""Authorized, replay-safe PChome sales report acquisition and import.""" + +from __future__ import annotations + +import json +import logging +import os +import uuid +from datetime import date, datetime +from pathlib import Path +from typing import Dict, List, Optional + +import pytz +from sqlalchemy import text + +from database.import_models import SalesAcquisitionReceipt +from services.google_drive_service import drive_service +from services.import_service import Session, humanize_import_error, import_service +from services.pchome_sales_acquisition_providers import ( + AcquisitionCandidate, + AuthorizedSalesProviders, + ProviderError, + env_bool, + env_int, +) + + +logger = logging.getLogger(__name__) +TAIPEI_TZ = pytz.timezone("Asia/Taipei") +WORK_ITEM_ID = "P0-PCHOME-SALES-AUTHORIZED-ACQUISITION" + + +class PChomeSalesAcquisitionService: + """Acquire authorized reports and close the import loop without manual review.""" + + def __init__(self, providers: Optional[AuthorizedSalesProviders] = None): + self.providers = providers or AuthorizedSalesProviders() + + def _freshness(self) -> Dict[str, object]: + latest = None + try: + session = Session() + try: + latest = session.execute( + text("SELECT MAX(snapshot_date) FROM daily_sales_snapshot") + ).scalar() + finally: + session.close() + except Exception: + logger.warning("Unable to read PChome sales freshness", exc_info=True) + + normalized = None + if latest: + try: + normalized = latest if isinstance(latest, date) else date.fromisoformat(str(latest)[:10]) + except (TypeError, ValueError): + normalized = None + lag_days = (datetime.now(TAIPEI_TZ).date() - normalized).days if normalized else None + return { + "latest_sales_date": normalized.isoformat() if normalized else None, + "data_lag_days": lag_days, + "decision_ready": lag_days is not None and lag_days <= 1, + "freshness_status": ( + "fresh" if lag_days is not None and lag_days <= 1 + else "degraded" if lag_days is not None + else "missing" + ), + } + + def _latest_receipt(self) -> Optional[Dict[str, object]]: + session = Session() + try: + receipt = session.query(SalesAcquisitionReceipt).order_by( + SalesAcquisitionReceipt.started_at.desc() + ).first() + return receipt.to_public_dict() if receipt else None + except Exception: + logger.warning("Unable to read latest sales acquisition receipt", exc_info=True) + return None + finally: + session.close() + + def readiness(self) -> Dict[str, object]: + try: + drive = drive_service.check_auth_readiness(refresh_expired=False) + drive_ready = bool(drive.get("ready")) + drive_status = drive.get("kind") or "unknown" + except Exception: + drive_ready = False + drive_status = "probe_failed" + + http_enabled = env_bool("PCHOME_SALES_HTTP_ENABLED") + http_ready = bool( + http_enabled + and os.getenv("PCHOME_SALES_HTTP_URL", "").strip() + and os.getenv("PCHOME_SALES_HTTP_ALLOWED_HOSTS", "").strip() + ) + imap_enabled = env_bool("PCHOME_SALES_IMAP_ENABLED") + imap_ready = bool( + imap_enabled + and os.getenv("PCHOME_SALES_IMAP_HOST", "").strip() + and os.getenv("PCHOME_SALES_IMAP_USER", "").strip() + and os.getenv("PCHOME_SALES_IMAP_PASSWORD", "").strip() + ) + local_path = os.getenv("PCHOME_SALES_LOCAL_DROP_DIR", "").strip() + local_ready = bool(local_path and Path(local_path).is_dir()) + sources = [ + {"id": "google_drive", "enabled": True, "ready": drive_ready, "status": drive_status}, + { + "id": "authorized_https", + "enabled": http_enabled, + "ready": http_ready, + "status": "ready" if http_ready else "disabled" if not http_enabled else "incomplete", + }, + { + "id": "authorized_imap", + "enabled": imap_enabled, + "ready": imap_ready, + "status": "ready" if imap_ready else "disabled" if not imap_enabled else "incomplete", + }, + { + "id": "controlled_local_drop", + "enabled": bool(local_path), + "ready": local_ready, + "status": "ready" if local_ready else "disabled" if not local_path else "path_missing", + }, + ] + ready_count = sum(1 for source in sources if source["ready"]) + freshness = self._freshness() + return { + "success": True, + "automation_mode": "controlled_apply", + "manual_review_required": False, + "risk_level": "medium", + "source_priority": [source["id"] for source in sources], + "sources": sources, + "asset_coverage": { + "ready": ready_count, + "total": len(sources), + "percent": round(ready_count / len(sources) * 100, 1), + }, + "freshness": freshness, + "runtime_closure": "closed" if freshness["decision_ready"] else "blocked_by_upstream_freshness", + "last_receipt": self._latest_receipt(), + } + + def _persist_receipt( + self, + *, + receipt_id: str, + trace_id: str, + span_id: str, + run_id: str, + status: str, + source_type: str, + candidate: Optional[AcquisitionCandidate], + decision: str, + before: Dict[str, object], + stages: Dict[str, object], + verifier: Dict[str, object], + import_job_id: Optional[int], + rows_imported: int, + error_kind: Optional[str], + message: str, + started_at: datetime, + ) -> bool: + session = Session() + try: + session.add(SalesAcquisitionReceipt( + receipt_id=receipt_id, + trace_id=trace_id, + span_id=span_id, + run_id=run_id, + work_item_id=WORK_ITEM_ID, + status=status, + source_type=source_type, + source_fingerprint=candidate.fingerprint if candidate else None, + source_ref_hash=candidate.source_ref_hash if candidate else None, + source_file_name=candidate.file_name if candidate else None, + risk_level="medium", + policy_decision="controlled_apply_allowed", + decision=decision, + import_job_id=import_job_id, + rows_imported=rows_imported, + before_freshness_json=json.dumps(before, ensure_ascii=False), + stage_receipts_json=json.dumps(stages, ensure_ascii=False), + verifier_json=json.dumps(verifier, ensure_ascii=False), + error_kind=error_kind, + public_message=message, + started_at=started_at.replace(tzinfo=None), + completed_at=datetime.now(TAIPEI_TZ).replace(tzinfo=None), + )) + session.commit() + return True + except Exception: + session.rollback() + logger.error("Unable to persist PChome sales acquisition receipt", exc_info=True) + return False + finally: + session.close() + + def run(self, *, trigger: str = "scheduler") -> Dict[str, object]: + started_at = datetime.now(TAIPEI_TZ) + receipt_id = uuid.uuid4().hex + trace_id = uuid.uuid4().hex + span_id = uuid.uuid4().hex[:16] + run_id = uuid.uuid4().hex + before = self._freshness() + stages: Dict[str, object] = { + "sensor": {"status": "started", "trigger": trigger, "trace_id": trace_id, "span_id": span_id}, + "normalize": {"asset_id": "pchome.daily_sales.authorized_report"}, + "source_diff": {"before": before}, + "ai_decision": {"decision": "probe_authorized_sources_in_policy_order"}, + "risk_policy": {"risk": "medium", "decision": "controlled_apply_allowed"}, + "check_mode": {"status": "passed", "max_files": env_int("PCHOME_SALES_MAX_FILES_PER_RUN", 5, 1, 25)}, + "providers": [], + } + candidates: List[AcquisitionCandidate] = [] + successful_results: List[Dict[str, object]] = [] + failed_results: List[Dict[str, object]] = [] + duplicate_results: List[Dict[str, object]] = [] + finalization_failures: List[Dict[str, object]] = [] + selected_candidate = None + error_kind = None + + try: + drive_result = import_service.auto_import_from_drive() + stages["providers"].append({ + "source": "google_drive", + "status": drive_result.get("status") or ("completed" if drive_result.get("success") else "failed"), + "file_count": drive_result.get("file_count", 0), + "imported_count": drive_result.get("imported_count", 0), + "error_kind": drive_result.get("error_kind"), + }) + if drive_result.get("imported_count", 0) > 0: + successful_results.append({ + "source_type": "google_drive", + "job_id": None, + "rows": int(drive_result.get("total_rows") or 0), + "date_range": drive_result.get("date_range"), + "result": drive_result, + }) + if not drive_result.get("source_finalize_ok", True): + finalization_failures.append({ + "source_type": "google_drive", + "error_kind": "source_finalize_failed", + }) + elif not drive_result.get("success"): + failed_results.append({ + "source_type": "google_drive", + "error_kind": drive_result.get("error_kind") or "drive_acquisition_failed", + "message": drive_result.get("message") or "Google Drive 授權來源目前無法取得報表。", + }) + + if not successful_results: + for source_name, provider in ( + ("authorized_https", self.providers.http_candidates), + ("authorized_imap", self.providers.imap_candidates), + ("controlled_local_drop", self.providers.local_candidates), + ): + try: + provider_candidates = provider() + candidates.extend(provider_candidates) + stages["providers"].append({ + "source": source_name, + "status": "candidate_found" if provider_candidates else "no_candidate", + "candidate_count": len(provider_candidates), + }) + except ProviderError as exc: + failed_results.append({"source_type": source_name, "error_kind": exc.kind, "message": exc.public_message}) + stages["providers"].append({"source": source_name, "status": "failed", "error_kind": exc.kind}) + + max_files = env_int("PCHOME_SALES_MAX_FILES_PER_RUN", 5, 1, 25) + for candidate in candidates[:max_files]: + selected_candidate = candidate + try: + result = import_service.import_local_daily_sales( + candidate.file_path, + source_type=candidate.source_type, + source_fingerprint=candidate.fingerprint, + source_file_name=candidate.file_name, + ) + job = result.get("job") or {} + summary = job.get("import_summary") or {} + record = { + "source_type": candidate.source_type, + "job_id": result.get("job_id"), + "rows": int(summary.get("imported_count") or 0), + "date_range": ( + {"min": summary.get("date_min"), "max": summary.get("date_max")} + if summary.get("date_min") and summary.get("date_max") + else None + ), + "status": result.get("status"), + } + if result.get("duplicate"): + duplicate_results.append(record) + try: + candidate.finalize_success() + except Exception: + logger.error("Duplicate source finalization failed", exc_info=True) + finalization_failures.append({ + "source_type": candidate.source_type, + "error_kind": "source_finalize_failed", + }) + elif result.get("success"): + successful_results.append(record) + try: + candidate.finalize_success() + except Exception: + logger.error("Imported source finalization failed", exc_info=True) + finalization_failures.append({ + "source_type": candidate.source_type, + "error_kind": "source_finalize_failed", + }) + else: + current_step = str(job.get("current_step") or "") + if "欄位" in current_step or "日期" in current_step: + try: + candidate.finalize_rejected() + except Exception: + logger.error("Rejected source finalization failed", exc_info=True) + finalization_failures.append({ + "source_type": candidate.source_type, + "error_kind": "source_finalize_failed", + }) + failed_results.append({**record, "error_kind": result.get("error_kind") or "import_failed_rolled_back"}) + finally: + candidate.cleanup() + except Exception as exc: + logger.error("PChome sales acquisition run failed", exc_info=True) + error_kind = "acquisition_runtime_error" + failed_results.append({ + "source_type": selected_candidate.source_type if selected_candidate else "orchestrator", + "error_kind": error_kind, + "message": humanize_import_error(exc), + }) + finally: + for candidate in candidates: + try: + candidate.cleanup() + except Exception: + logger.warning("Unable to clean sales acquisition temp file", exc_info=True) + + verifier = self._freshness() + rows_imported = sum(int(item.get("rows") or 0) for item in successful_results) + import_job_id = next((item.get("job_id") for item in reversed(successful_results) if item.get("job_id")), None) + source_types = sorted({ + item.get("source_type") + for item in successful_results + duplicate_results + failed_results + if item.get("source_type") + }) + source_type = source_types[0] if len(source_types) == 1 else "multiple" if source_types else "none" + date_mins = [ + item["date_range"]["min"] + for item in successful_results + if item.get("date_range") and item["date_range"].get("min") + ] + date_maxes = [ + item["date_range"]["max"] + for item in successful_results + if item.get("date_range") and item["date_range"].get("max") + ] + date_range = ( + {"min": min(date_mins), "max": max(date_maxes)} + if date_mins and date_maxes + else None + ) + + if finalization_failures and (successful_results or duplicate_results): + status = "partial" + decision = "runtime_verified_source_finalize_failed" + error_kind = "source_finalize_failed" + message = "業績資料已通過正式驗證，但來源封存回讀失敗，系統將自動重試收尾。" + elif successful_results and verifier["decision_ready"]: + status = "completed" + decision = "imported_and_verified" + message = f"已自動匯入 {rows_imported} 筆業績資料，雙表與新鮮度驗證通過。" + elif successful_results: + status = "partial" + decision = "imported_but_freshness_not_closed" + message = "業績檔已匯入並通過雙表驗證，但資料日期仍未達決策新鮮度。" + elif duplicate_results and verifier["decision_ready"]: + status = "completed_no_write" + decision = "duplicate_verified_no_write" + message = "來源內容已匯入過，冪等略過寫入且正式資料仍通過新鮮度驗證。" + elif not failed_results and verifier["decision_ready"]: + status = "completed_no_write" + decision = "no_candidate_fresh_no_write" + message = "目前沒有新業績檔，正式資料仍通過新鮮度驗證，本輪無需寫入。" + elif failed_results: + error_kind = error_kind or failed_results[0].get("error_kind") + import_failed = any( + str(item.get("error_kind") or "") in { + "import_failed_rolled_back", + "acquisition_runtime_error", + } + for item in failed_results + ) + status = "failed_rolled_back" if import_failed else "degraded_no_write" + decision = "import_failed_rolled_back" if import_failed else "provider_failed_no_write" + message = ( + "匯入驗證未通過，正式資料已回滾。" + if import_failed + else "授權來源目前無法取得報表，系統將自動重試。" + ) + else: + status = "blocked_with_safe_next_action" + decision = "no_authorized_candidate" + message = "所有已啟用授權來源目前都沒有新業績檔，正式資料未變更。" + + stages["execution"] = { + "status": status, + "successful_files": len(successful_results), + "duplicate_files": len(duplicate_results), + "failed_files": len(failed_results), + "source_finalize_failed": len(finalization_failures), + "rows_imported": rows_imported, + "idempotent": True, + "bounded": True, + } + stages["post_verifier"] = verifier + stages["terminal"] = { + "status": status, + "rollback": status == "failed_rolled_back", + "no_write": status in { + "completed_no_write", + "blocked_with_safe_next_action", + "failed_rolled_back", + "degraded_no_write", + }, + } + stages["learning_writeback"] = { + "target": "pchome_sales_acquisition_receipts", + "ack": "committed_with_receipt", + "km_rag_payload": "not_applicable_for_raw_sales_payload", + } + persisted = self._persist_receipt( + receipt_id=receipt_id, + trace_id=trace_id, + span_id=span_id, + run_id=run_id, + status=status, + source_type=source_type, + candidate=selected_candidate, + decision=decision, + before=before, + stages=stages, + verifier=verifier, + import_job_id=import_job_id, + rows_imported=rows_imported, + error_kind=error_kind, + message=message, + started_at=started_at, + ) + if not persisted and status in {"completed", "completed_no_write"}: + status = "partial" + decision = "runtime_verified_receipt_write_failed" + message = "業績資料已通過驗證，但治理 receipt 寫入失敗，閉環尚未完成。" + + return { + "success": status in {"completed", "completed_no_write"}, + "status": status, + "message": message, + "trace_id": trace_id, + "span_id": span_id, + "run_id": run_id, + "work_item_id": WORK_ITEM_ID, + "receipt_id": receipt_id, + "receipt_persisted": persisted, + "source_type": source_type, + "file_count": len(successful_results) + len(duplicate_results) + len(failed_results), + "imported_count": len(successful_results), + "failed_count": len(failed_results), + "duplicate_count": len(duplicate_results), + "source_finalize_failed_count": len(finalization_failures), + "total_rows": rows_imported, + "date_range": date_range, + "latest_sales_date": verifier.get("latest_sales_date"), + "data_lag_days": verifier.get("data_lag_days"), + "decision_ready": verifier.get("decision_ready"), + "requires_upstream_acquisition": not bool(verifier.get("decision_ready")), + "manual_review_required": False, + "safe_next_action": ( + "scheduler_retry_authorized_sources" + if status == "blocked_with_safe_next_action" + else "repair_provider_or_replay_transaction" + if status in {"failed_rolled_back", "degraded_no_write"} + else "retry_source_finalization" + if error_kind == "source_finalize_failed" + else "none" + ), + "providers": stages["providers"], + "verifier": verifier, + } + + +pchome_sales_acquisition_service = PChomeSalesAcquisitionService() diff --git a/templates/auto_import_index.html b/templates/auto_import_index.html index fd36db9..5097bd6 100644 --- a/templates/auto_import_index.html +++ b/templates/auto_import_index.html @@ -1,6 +1,6 @@ {% extends "ewoooc_base.html" %} -{% block title %}當日業績報表匯入{% endblock %} +{% block title %}業績資料自動化{% endblock %} {% block page_attrs %}data-page-group="monitor" data-page-id="auto-import"{% endblock %} {% block extra_head %} @@ -11,429 +11,357 @@ {% block content %}
- - {# ─────────── 頁首 ─────────── #}
- +
-

當日業績報表匯入

-

- - 保持 PChome 業績新鮮，讓評估、分析與建議有可靠資料。 -

+

業績資料自動化

+

正在讀取正式資料狀態

+
+
+ +
- {# ─────────── 配置區 ─────────── #} -
-
-

- Google Drive 自動匯入配置 -

-
-
-
- -
- -

每 30 分鐘檢查雲端業績檔，讓日報、成長分析與作戰清單保持新鮮。

-
- -
-
- - - 設定要監控的 Google Drive 資料夾路徑 -
-
- - - 用於過濾特定名稱的檔案 -
-
- -
- - - - -
+
+
+ 最新業績 + -- + 讀取中 +
+
+ 決策狀態 + -- + 讀取中 +
+
+ 來源覆蓋 + -- + 讀取中 +
+
+ 最近執行 + -- + 尚無 receipt
- {# ─────────── 手動上傳 ─────────── #} -
-
-

- 手動上傳匯入 -

+ + +
+
+
+ 授權來源 +

資料來源

+
+ 自動安全執行
-
-
-
- -
-

每日業績快照

-

- 上傳當日業績檔，更新日報、成長分析與今日作戰清單。 -

-

- 檔名建議：即時業績_當日_YYYYMMDD.xlsx；送出後更新日報、成長分析與今日作戰清單。 -

-
-
- -
-
- - -
-
- -
-
+
+
正在讀取來源狀態
+ +
+ Drive 來源設定 +
+
+
+
+ + +
+
+ + +
+
+
+ + + +
+
+
- {# ─────────── 任務歷史 ─────────── #} -
-
-

- 匯入任務歷史 -

-
-
- - - - - - + + +
-
{% endblock %} {% block extra_scripts %} {% endblock %} diff --git a/tests/test_auto_import_failure_boundaries.py b/tests/test_auto_import_failure_boundaries.py index 476344c..25a4ffc 100644 --- a/tests/test_auto_import_failure_boundaries.py +++ b/tests/test_auto_import_failure_boundaries.py @@ -95,12 +95,14 @@ def test_daily_sales_import_fails_when_monthly_sync_fails(monkeypatch, tmp_path) job = session.query(import_service.ImportJob).filter_by(id=job_id).one() assert job.status == "failed" assert job.progress_percent == 95 - assert job.current_step == "業績分析儀表板同步失敗" + assert job.current_step == "交易驗證失敗，已回滾" assert "monthly sync boom" in job.error_message summary = json.loads(job.import_summary) assert summary["sync_success"] is False assert summary["synced_to"] is None assert "monthly sync boom" in summary["sync_error"] + assert summary["atomic"] is True + assert summary["rolled_back"] is True finally: session.close() @@ -108,11 +110,45 @@ def test_daily_sales_import_fails_when_monthly_sync_fails(monkeypatch, tmp_path) snapshot_rows = conn.execute(text("SELECT COUNT(*) FROM daily_sales_snapshot")).scalar() monthly_rows = conn.execute(text("SELECT COUNT(*) FROM realtime_sales_monthly")).scalar() - assert snapshot_rows == 1 + assert snapshot_rows == 0 assert monthly_rows == 0 assert FakeNotificationManager.sent_messages +def test_daily_sales_import_preserves_today_fallback_when_date_column_is_missing(monkeypatch, tmp_path): + import_service = _load_import_service(monkeypatch, f"sqlite:///{tmp_path / 'momo.db'}") + _prepare_daily_sales_tables(import_service) + source_df = pd.DataFrame([{ + "商品ID": "A001", + "商品名稱": "測試商品", + "銷售金額": 1200, + }]) + monkeypatch.setattr( + import_service, + "_read_daily_sales_excel", + lambda _path: ( + source_df.copy(), + {"date_col": None, "sheet_name": "即時業績明細", "header_row": 1}, + ), + ) + service = import_service.ImportService() + job_id = service.create_import_job("daily_sales", "drive-file-1", "daily.xlsx", 1024) + + assert service.process_daily_sales_import(job_id, str(tmp_path / "daily.xlsx")) is True + + expected_date = import_service.datetime.now(import_service.TAIPEI_TZ).date().isoformat() + with import_service.engine.connect() as conn: + snapshot = conn.execute(text( + 'SELECT "日期", snapshot_date FROM daily_sales_snapshot' + )).one() + monthly = conn.execute(text( + 'SELECT "日期" FROM realtime_sales_monthly' + )).one() + assert snapshot[0] == expected_date + assert str(snapshot[1])[:10] == expected_date + assert monthly[0] == expected_date + + def test_auto_import_does_not_move_drive_file_when_import_fails(monkeypatch, tmp_path): import_service = _load_import_service(monkeypatch, f"sqlite:///{tmp_path / 'momo.db'}") import_service.Base.metadata.create_all(import_service.engine) @@ -146,6 +182,47 @@ def test_auto_import_does_not_move_drive_file_when_import_fails(monkeypatch, tmp assert fake_drive.moved_files == [] +def test_auto_import_reports_partial_when_drive_archive_fails(monkeypatch, tmp_path): + import_service = _load_import_service(monkeypatch, f"sqlite:///{tmp_path / 'momo.db'}") + import_service.Base.metadata.create_all(import_service.engine) + monkeypatch.chdir(tmp_path) + + class FakeDriveService: + last_error_kind = None + last_error = None + + def list_files_in_folder(self, folder_path, file_pattern): + return [{"id": "drive-file-1", "name": "daily.xlsx", "size": 1024}] + + def download_file(self, file_id, local_path): + os.makedirs(os.path.dirname(local_path), exist_ok=True) + with open(local_path, "wb") as handle: + handle.write(b"test") + return True + + def move_file(self, file_id, folder, create_missing=False): + return False + + monkeypatch.setattr(import_service, "drive_service", FakeDriveService()) + service = import_service.ImportService() + monkeypatch.setattr(service, "process_daily_sales_import", lambda job_id, path: True) + + result = service.auto_import_from_drive() + + assert result["success"] is False + assert result["status"] == "partial_source_finalize_failed" + assert result["imported_count"] == 1 + assert result["source_finalize_ok"] is False + assert result["archive_failed_count"] == 1 + session = import_service.Session() + try: + job = session.query(import_service.ImportJob).one() + assert job.status == "completed" + assert job.current_step == "匯入完成，來源封存待自動重試" + finally: + session.close() + + def test_auto_import_fails_closed_when_drive_auth_fails(monkeypatch, tmp_path): import_service = _load_import_service(monkeypatch, f"sqlite:///{tmp_path / 'momo.db'}") import_service.Base.metadata.create_all(import_service.engine) diff --git a/tests/test_frontend_v2_assets.py b/tests/test_frontend_v2_assets.py index 15d27b2..cb7f9d7 100644 --- a/tests/test_frontend_v2_assets.py +++ b/tests/test_frontend_v2_assets.py @@ -851,8 +851,15 @@ def test_utility_pages_keep_operator_copy_professional(): assert "簡報線上預覽" in ppt_preview assert "下載簡報檔" in ppt_history - assert "送出後更新日報、成長分析與今日作戰清單" in auto_import - assert "等待系統更新任務狀態；若重複停在異常" in auto_import + assert "業績資料自動化" in auto_import + assert "正式業績可供分析與決策" in auto_import + assert "雙表已驗證" in auto_import + assert "自動安全執行" in auto_import + assert "Controlled apply" not in auto_import + assert "AUTHORIZED SOURCES" not in auto_import + assert "VERIFIED RUNS" not in auto_import + assert "手動上傳匯入" not in auto_import + assert "uploadManualFile" not in auto_import assert "return raw ||" not in auto_import assert "供貨風險匯入" in stockout_import assert "缺少必要內容時，會先停止匯入" in stockout_import diff --git a/tests/test_import_service_sql_params.py b/tests/test_import_service_sql_params.py index 3d0dcff..f91d4e8 100644 --- a/tests/test_import_service_sql_params.py +++ b/tests/test_import_service_sql_params.py @@ -29,11 +29,23 @@ def test_daily_snapshot_delete_casts_text_date_column_on_postgres(monkeypatch): def test_daily_snapshot_delete_query_uses_snapshot_date_cast_helper(): source = Path("services/import_service.py").read_text(encoding="utf-8") - assert 'snapshot_date_expr = _date_filter_expr("snapshot_date")' in source - assert "DELETE FROM {table_name} WHERE {snapshot_date_expr} IN" in source + assert 'snapshot_expr = _date_filter_expr("snapshot_date")' in source + assert "f\"WHERE {snapshot_expr} IN ({snapshot_placeholders})\"" in source assert "DELETE FROM daily_sales_snapshot WHERE snapshot_date IN" not in source +def test_sales_table_replacement_uses_one_transaction_connection(): + source = Path("services/import_service.py").read_text(encoding="utf-8") + start = source.index("def _replace_sales_tables_atomic") + end = source.index("# 資料庫設定", start) + block = source[start:end] + + assert "with engine.begin() as conn:" in block + assert 'df.to_sql(\n "daily_sales_snapshot",\n conn,' in block + assert 'df_monthly.to_sql(\n "realtime_sales_monthly",\n conn,' in block + assert "conn.commit()" not in block + + def test_daily_snapshot_delete_uses_iso_dates_on_sqlite(monkeypatch): monkeypatch.setattr(import_service, "_db_dialect_name", lambda: "sqlite") diff --git a/tests/test_pchome_revenue_growth_service.py b/tests/test_pchome_revenue_growth_service.py index ea1dd90..af6fd61 100644 --- a/tests/test_pchome_revenue_growth_service.py +++ b/tests/test_pchome_revenue_growth_service.py @@ -749,7 +749,7 @@ def test_primary_pages_use_growth_outcome_copy_instead_of_feature_explaining(): expected = { "templates/daily_sales.html": "找出下滑與價差壓力", "templates/ai_recommend.html": "把價差、商品證據與趨勢轉成主推、調價、補比價動作", - "templates/auto_import_index.html": "保持 PChome 業績新鮮", + "templates/auto_import_index.html": "正式業績可供分析與決策", "templates/price_comparison.html": "確認同款、判斷價差、決定下一步", "templates/vendor_stockout_index_v2.html": "避免主推商品斷貨", "templates/monthly_summary_analysis.html": "判斷成長、毛利與品類結構", @@ -910,14 +910,14 @@ def test_governance_and_low_frequency_pages_avoid_engineering_status_copy(): "templates/403.html": ["權限守門", "未授權操作影響營運資料", "權限控管"], "templates/maintenance.html": ["服務維護", "確認業績、比價與匯入狀態", "台北時間"], "templates/auto_import_index.html": [ - "更新日報、成長分析與今日作戰清單", - "送出後更新日報、成長分析與今日作戰清單", - "作戰清單保持新鮮", - "共更新", - "buildImportActionHint", - "重新確認 Google Drive 授權", - "改用當日業績明細檔", - "重新匯入最新檔案", + "業績資料自動化", + "來源覆蓋", + "自動安全執行", + "執行自動取得", + "雙表已驗證", + "等待最新業績", + "loadReadiness", + "/api/auto_import/run", ], "templates/settings.html": ["比價來源同步", "補齊 MOMO 參考來源"], "templates/system_settings.html": [ diff --git a/tests/test_pchome_sales_acquisition_service.py b/tests/test_pchome_sales_acquisition_service.py new file mode 100644 index 0000000..7a94ef2 --- /dev/null +++ b/tests/test_pchome_sales_acquisition_service.py @@ -0,0 +1,334 @@ +import importlib +import json +import os +from datetime import datetime + +import pandas as pd +import pytz +from flask import Flask +from sqlalchemy import text + + +TAIPEI_TZ = pytz.timezone("Asia/Taipei") + + +def _load_services(monkeypatch, database_url): + os.environ.setdefault("MOMO_ALLOW_INSECURE_CONFIG_FOR_TESTS", "true") + import config + + monkeypatch.setattr(config, "DATABASE_PATH", database_url) + import services.import_service as import_service + + import_service = importlib.reload(import_service) + import services.pchome_sales_acquisition_service as acquisition + + acquisition = importlib.reload(acquisition) + return import_service, acquisition + + +def _prepare_tables(import_service): + import_service.Base.metadata.create_all(import_service.engine) + with import_service.engine.begin() as conn: + conn.execute(text("DROP TABLE IF EXISTS daily_sales_snapshot")) + conn.execute(text("DROP TABLE IF EXISTS realtime_sales_monthly")) + conn.execute(text(""" + CREATE TABLE daily_sales_snapshot ( + "日期" TEXT, + "商品ID" TEXT, + "商品名稱" TEXT, + "銷售金額" INTEGER, + snapshot_date TEXT + ) + """)) + conn.execute(text(""" + CREATE TABLE realtime_sales_monthly ( + "日期" TEXT, + "商品ID" TEXT, + "商品名稱" TEXT, + "銷售金額" INTEGER + ) + """)) + + +class EmptyDrive: + last_error_kind = None + last_error = None + + def list_files_in_folder(self, _folder_path, _file_pattern): + return [] + + +class ReadyDrive(EmptyDrive): + def check_auth_readiness(self, refresh_expired=False): + return {"ready": True, "kind": "ready"} + + +class FailedDrive(EmptyDrive): + last_error_kind = "authentication_failed" + last_error = "credentials missing" + + +def _write_sales_file(path): + today = datetime.now(TAIPEI_TZ).strftime("%Y-%m-%d") + pd.DataFrame([{ + "日期": today, + "商品ID": "P001", + "商品名稱": "測試商品", + "銷售金額": 1680, + }]).to_excel(path, index=False, sheet_name="即時業績明細") + + +def _disable_remote_sources(monkeypatch): + monkeypatch.setenv("PCHOME_SALES_HTTP_ENABLED", "false") + monkeypatch.setenv("PCHOME_SALES_IMAP_ENABLED", "false") + + +def test_local_drop_closes_import_and_archives_only_after_verification(monkeypatch, tmp_path): + import_service, acquisition = _load_services(monkeypatch, f"sqlite:///{tmp_path / 'momo.db'}") + _prepare_tables(import_service) + _disable_remote_sources(monkeypatch) + drop_dir = tmp_path / "drop" + drop_dir.mkdir() + source = drop_dir / "即時業績_當日.xlsx" + _write_sales_file(source) + monkeypatch.setenv("PCHOME_SALES_LOCAL_DROP_DIR", str(drop_dir)) + monkeypatch.setattr(import_service, "drive_service", EmptyDrive()) + + result = acquisition.PChomeSalesAcquisitionService().run(trigger="test") + + assert result["success"] is True + assert result["status"] == "completed" + assert result["source_type"] == "controlled_local_drop" + assert result["total_rows"] == 1 + assert result["decision_ready"] is True + assert result["receipt_persisted"] is True + assert len(result["trace_id"]) == 32 + assert len(result["span_id"]) == 16 + assert result["trace_id"] == result["trace_id"].lower() + assert result["span_id"] == result["span_id"].lower() + assert not source.exists() + assert (drop_dir / "archive" / source.name).exists() + + with import_service.engine.connect() as conn: + assert conn.execute(text("SELECT COUNT(*) FROM daily_sales_snapshot")).scalar() == 1 + assert conn.execute(text("SELECT COUNT(*) FROM realtime_sales_monthly")).scalar() == 1 + assert conn.execute(text("SELECT COUNT(*) FROM pchome_sales_acquisition_receipts")).scalar() == 1 + + +def test_same_content_is_idempotent_and_does_not_duplicate_rows(monkeypatch, tmp_path): + import_service, acquisition = _load_services(monkeypatch, f"sqlite:///{tmp_path / 'momo.db'}") + _prepare_tables(import_service) + _disable_remote_sources(monkeypatch) + drop_dir = tmp_path / "drop" + drop_dir.mkdir() + source = drop_dir / "即時業績_當日.xlsx" + _write_sales_file(source) + original_bytes = source.read_bytes() + monkeypatch.setenv("PCHOME_SALES_LOCAL_DROP_DIR", str(drop_dir)) + monkeypatch.setattr(import_service, "drive_service", EmptyDrive()) + service = acquisition.PChomeSalesAcquisitionService() + + first = service.run(trigger="test") + source.write_bytes(original_bytes) + second = service.run(trigger="test") + + assert first["status"] == "completed" + assert second["status"] == "completed_no_write" + assert second["duplicate_count"] == 1 + assert second["imported_count"] == 0 + with import_service.engine.connect() as conn: + assert conn.execute(text("SELECT COUNT(*) FROM daily_sales_snapshot")).scalar() == 1 + assert conn.execute(text("SELECT COUNT(*) FROM realtime_sales_monthly")).scalar() == 1 + + +def test_verified_import_is_partial_when_source_archive_ack_fails(monkeypatch, tmp_path): + import_service, acquisition = _load_services(monkeypatch, f"sqlite:///{tmp_path / 'momo.db'}") + import services.pchome_sales_acquisition_providers as providers + + _prepare_tables(import_service) + _disable_remote_sources(monkeypatch) + drop_dir = tmp_path / "drop" + drop_dir.mkdir() + source = drop_dir / "即時業績_當日.xlsx" + _write_sales_file(source) + monkeypatch.setenv("PCHOME_SALES_LOCAL_DROP_DIR", str(drop_dir)) + monkeypatch.setattr(import_service, "drive_service", EmptyDrive()) + monkeypatch.setattr( + providers, + "_archive_file", + lambda *_args, **_kwargs: (_ for _ in ()).throw(PermissionError("archive denied")), + ) + + result = acquisition.PChomeSalesAcquisitionService().run(trigger="test") + + assert result["success"] is False + assert result["status"] == "partial" + assert result["source_finalize_failed_count"] == 1 + assert result["safe_next_action"] == "retry_source_finalization" + assert source.exists() + with import_service.engine.connect() as conn: + assert conn.execute(text("SELECT COUNT(*) FROM daily_sales_snapshot")).scalar() == 1 + assert conn.execute(text("SELECT COUNT(*) FROM realtime_sales_monthly")).scalar() == 1 + + +def test_no_candidate_with_missing_freshness_is_blocked_and_receipted(monkeypatch, tmp_path): + import_service, acquisition = _load_services(monkeypatch, f"sqlite:///{tmp_path / 'momo.db'}") + _prepare_tables(import_service) + _disable_remote_sources(monkeypatch) + monkeypatch.delenv("PCHOME_SALES_LOCAL_DROP_DIR", raising=False) + monkeypatch.setattr(import_service, "drive_service", EmptyDrive()) + + result = acquisition.PChomeSalesAcquisitionService().run(trigger="test") + + assert result["success"] is False + assert result["status"] == "blocked_with_safe_next_action" + assert result["safe_next_action"] == "scheduler_retry_authorized_sources" + assert result["receipt_persisted"] is True + with import_service.engine.connect() as conn: + receipt = conn.execute(text( + "SELECT status, decision FROM pchome_sales_acquisition_receipts" + )).one() + assert receipt.status == "blocked_with_safe_next_action" + assert receipt.decision == "no_authorized_candidate" + + +def test_provider_failure_keeps_product_message_compact(monkeypatch, tmp_path): + import_service, acquisition = _load_services(monkeypatch, f"sqlite:///{tmp_path / 'momo.db'}") + _prepare_tables(import_service) + _disable_remote_sources(monkeypatch) + monkeypatch.delenv("PCHOME_SALES_LOCAL_DROP_DIR", raising=False) + monkeypatch.setattr(import_service, "drive_service", FailedDrive()) + + result = acquisition.PChomeSalesAcquisitionService().run(trigger="test") + + assert result["success"] is False + assert result["status"] == "degraded_no_write" + assert result["message"] == "授權來源目前無法取得報表，系統將自動重試。" + assert result["providers"][0]["error_kind"] == "authentication_failed" + + +def test_readiness_never_exposes_provider_credentials_or_urls(monkeypatch, tmp_path): + import_service, acquisition = _load_services(monkeypatch, f"sqlite:///{tmp_path / 'momo.db'}") + _prepare_tables(import_service) + monkeypatch.setattr(acquisition, "drive_service", ReadyDrive()) + monkeypatch.setenv("PCHOME_SALES_HTTP_ENABLED", "true") + monkeypatch.setenv("PCHOME_SALES_HTTP_URL", "https://reports.example.test/private/report.xlsx") + monkeypatch.setenv("PCHOME_SALES_HTTP_ALLOWED_HOSTS", "reports.example.test") + monkeypatch.setenv("PCHOME_SALES_HTTP_BEARER_TOKEN", "top-secret-http-token") + monkeypatch.setenv("PCHOME_SALES_IMAP_ENABLED", "true") + monkeypatch.setenv("PCHOME_SALES_IMAP_HOST", "mail.example.test") + monkeypatch.setenv("PCHOME_SALES_IMAP_USER", "private-user@example.test") + monkeypatch.setenv("PCHOME_SALES_IMAP_PASSWORD", "top-secret-imap-password") + + payload = acquisition.PChomeSalesAcquisitionService().readiness() + public_text = json.dumps(payload, ensure_ascii=False) + + assert payload["manual_review_required"] is False + assert payload["asset_coverage"]["ready"] == 3 + assert "top-secret" not in public_text + assert "reports.example.test" not in public_text + assert "mail.example.test" not in public_text + assert "private-user" not in public_text + + +def test_https_provider_rejects_non_https_and_non_allowlisted_hosts(monkeypatch): + import services.pchome_sales_acquisition_service as acquisition + import services.pchome_sales_acquisition_providers as provider_module + + service = acquisition.PChomeSalesAcquisitionService() + monkeypatch.setenv("PCHOME_SALES_HTTP_ALLOWED_HOSTS", "reports.example.test") + + try: + service.providers.validate_http_url("http://reports.example.test/report.xlsx") + assert False, "HTTP URL should be rejected" + except acquisition.ProviderError as exc: + assert exc.kind == "http_url_rejected" + + try: + service.providers.validate_http_url("https://untrusted.example.test/report.xlsx") + assert False, "Non-allowlisted host should be rejected" + except acquisition.ProviderError as exc: + assert exc.kind == "http_host_not_allowed" + + monkeypatch.setenv("PCHOME_SALES_HTTP_ALLOWED_HOSTS", "reports.example.test") + try: + service.providers.validate_http_url("https://reports.example.test:8443/report.xlsx") + assert False, "Non-allowlisted port should be rejected" + except acquisition.ProviderError as exc: + assert exc.kind == "http_port_rejected" + + monkeypatch.setattr( + provider_module.socket, + "getaddrinfo", + lambda *_args, **_kwargs: [(2, 1, 6, "", ("127.0.0.1", 443))], + ) + try: + service.providers.validate_http_url("https://reports.example.test/report.xlsx") + assert False, "Private resolved address should be rejected" + except acquisition.ProviderError as exc: + assert exc.kind == "http_private_address_rejected" + + +def test_auto_import_api_uses_governed_acquisition_service(monkeypatch): + import routes.auto_import_routes as routes + + class FakeAcquisitionService: + def readiness(self): + return {"success": True, "runtime_closure": "blocked_by_upstream_freshness"} + + def run(self, *, trigger): + return { + "success": False, + "status": "blocked_with_safe_next_action", + "trigger": trigger, + "receipt_persisted": True, + } + + monkeypatch.setattr(routes, "pchome_sales_acquisition_service", FakeAcquisitionService()) + app = Flask(__name__) + app.register_blueprint(routes.auto_import_bp) + client = app.test_client() + + readiness = client.get("/api/auto_import/readiness") + run = client.post("/api/auto_import/run") + legacy = client.post("/api/manual_import") + + assert readiness.status_code == 200 + assert readiness.get_json()["runtime_closure"] == "blocked_by_upstream_freshness" + assert run.get_json()["status"] == "blocked_with_safe_next_action" + assert run.get_json()["trigger"] == "operator_command" + assert legacy.get_json()["trigger"] == "operator_command" + + +def test_scheduler_continues_when_legacy_hitl_pause_is_set(monkeypatch): + import scheduler + import services.agent_actions as agent_actions + import services.pchome_sales_acquisition_service as acquisition + + calls = [] + saved = {} + monkeypatch.setattr(agent_actions, "is_task_paused", lambda _name: True) + monkeypatch.setattr( + acquisition.pchome_sales_acquisition_service, + "run", + lambda *, trigger: calls.append(trigger) or { + "success": False, + "status": "blocked_with_safe_next_action", + "message": "no candidate", + "trace_id": "trace-test", + "receipt_id": "receipt-test", + "receipt_persisted": True, + "latest_sales_date": "2026-07-10", + "data_lag_days": 1, + "decision_ready": True, + "safe_next_action": "scheduler_retry_authorized_sources", + }, + ) + monkeypatch.setattr(scheduler, "_save_stats", lambda name, stats: saved.update({"name": name, "stats": stats})) + + scheduler.run_auto_import_task() + + assert calls == ["scheduler"] + assert saved["name"] == "auto_import_task" + assert saved["stats"]["status"] == "Degraded" + assert saved["stats"]["receipt_persisted"] is True diff --git a/web/static/css/page-auto-import-bem.css b/web/static/css/page-auto-import-bem.css index 65f0b47..126695f 100644 --- a/web/static/css/page-auto-import-bem.css +++ b/web/static/css/page-auto-import-bem.css @@ -271,7 +271,7 @@ font-weight: var(--momo-font-semibold); font-size: var(--momo-text-xs); text-transform: uppercase; - letter-spacing: 0.04em; + letter-spacing: 0; padding: var(--momo-space-3); border-bottom: 1px solid var(--momo-border); white-space: nowrap; @@ -520,3 +520,265 @@ min-width: 0; } } + +/* ---------- V10.776 automation cockpit ---------- */ +.ai-head__actions { + display: flex; + align-items: center; + gap: var(--momo-space-2); + margin-left: auto; +} + +.ai-icon-button, +.ai-row-action { + display: inline-flex; + align-items: center; + justify-content: center; + width: 40px; + height: 40px; + padding: 0; + border: 1px solid var(--momo-border); + border-radius: var(--momo-radius-sm); + background: var(--momo-surface-0); + color: var(--momo-ink-secondary); +} + +.ai-icon-button:hover, +.ai-icon-button:focus-visible, +.ai-row-action:hover, +.ai-row-action:focus-visible { + border-color: #2f6f8f; + color: #205771; + outline: 2px solid rgba(47, 111, 143, 0.2); + outline-offset: 1px; +} + +.ai-cockpit { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + margin-bottom: var(--momo-space-5); + border-top: 1px solid var(--momo-border); + border-bottom: 1px solid var(--momo-border); + background: var(--momo-surface-0); +} + +.ai-metric { + display: flex; + min-width: 0; + min-height: 118px; + flex-direction: column; + justify-content: center; + padding: var(--momo-space-4) var(--momo-space-5); + border-right: 1px solid var(--momo-border-light); +} + +.ai-metric:last-child { border-right: 0; } + +.ai-metric__label, +.ai-section__eyebrow { + color: var(--momo-ink-tertiary); + font-family: var(--momo-font-mono); + font-size: var(--momo-text-xs); + font-weight: var(--momo-font-semibold); + letter-spacing: 0; + text-transform: uppercase; +} + +.ai-metric__value { + margin: var(--momo-space-2) 0 var(--momo-space-1); + color: var(--momo-ink-primary); + font-family: var(--momo-font-heading); + font-size: var(--momo-text-xl); + font-weight: var(--momo-font-bold); + line-height: 1.25; + overflow-wrap: anywhere; +} + +.ai-metric__value--compact { font-size: var(--momo-text-base); } +.ai-metric__value[data-state="ready"] { color: #287a55; } +.ai-metric__value[data-state="blocked"] { color: #a33e4a; } + +.ai-metric__meta { + color: var(--momo-ink-tertiary); + font-size: var(--momo-text-xs); + overflow-wrap: anywhere; +} + +.ai-inline-status { + display: flex; + align-items: center; + gap: var(--momo-space-2); + margin: 0 0 var(--momo-space-5); + padding: var(--momo-space-3) var(--momo-space-4); + border-left: 3px solid #2f6f8f; + background: rgba(47, 111, 143, 0.08); + color: #205771; + font-size: var(--momo-text-sm); +} + +.ai-inline-status[hidden] { display: none; } +.ai-inline-status--success { border-color: #287a55; background: rgba(40, 122, 85, 0.09); color: #1f6245; } +.ai-inline-status--warning { border-color: #a06b15; background: rgba(160, 107, 21, 0.09); color: #76500f; } +.ai-inline-status--danger { border-color: #a33e4a; background: rgba(163, 62, 74, 0.09); color: #7b2f38; } + +.ai-section { + padding: var(--momo-space-5) 0; + border-top: 1px solid var(--momo-border-light); +} + +.ai-section__head { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--momo-space-3); + margin-bottom: var(--momo-space-4); +} + +.ai-section__title { + margin: var(--momo-space-1) 0 0; + color: var(--momo-ink-primary); + font-family: var(--momo-font-heading); + font-size: var(--momo-text-lg); + font-weight: var(--momo-font-semibold); + letter-spacing: 0; +} + +.ai-mode-badge { + display: inline-flex; + align-items: center; + gap: var(--momo-space-1); + padding: var(--momo-space-1) var(--momo-space-2); + border: 1px solid rgba(40, 122, 85, 0.35); + border-radius: var(--momo-radius-sm); + background: rgba(40, 122, 85, 0.08); + color: #1f6245; + font-family: var(--momo-font-mono); + font-size: var(--momo-text-xs); + font-weight: var(--momo-font-semibold); +} + +.ai-source-list { + border-top: 1px solid var(--momo-border-light); +} + +.ai-source-row { + display: grid; + grid-template-columns: 14px minmax(0, 1fr) auto; + gap: var(--momo-space-3); + align-items: center; + min-height: 52px; + padding: var(--momo-space-3) 0; + border-bottom: 1px solid var(--momo-border-light); +} + +.ai-source-row--loading { + display: block; + color: var(--momo-ink-tertiary); + font-size: var(--momo-text-sm); +} + +.ai-source-row__indicator { + width: 9px; + height: 9px; + border-radius: 50%; + background: var(--momo-border-strong); +} + +.ai-source-row__indicator--ready { background: #287a55; } +.ai-source-row__indicator--degraded { background: #a06b15; } +.ai-source-row__indicator--disabled { background: #8a9198; } + +.ai-source-row__name { + min-width: 0; + color: var(--momo-ink-primary); + font-size: var(--momo-text-sm); + font-weight: var(--momo-font-semibold); + overflow-wrap: anywhere; +} + +.ai-source-row__state { + color: var(--momo-ink-tertiary); + font-family: var(--momo-font-mono); + font-size: var(--momo-text-xs); +} + +.ai-disclosure { + margin-top: var(--momo-space-4); + border-bottom: 1px solid var(--momo-border-light); +} + +.ai-disclosure summary { + display: flex; + align-items: center; + gap: var(--momo-space-2); + padding: var(--momo-space-3) 0; + color: var(--momo-ink-secondary); + cursor: pointer; + font-size: var(--momo-text-sm); + font-weight: var(--momo-font-semibold); + list-style: none; +} + +.ai-disclosure summary::-webkit-details-marker { display: none; } +.ai-disclosure summary::after { + margin-left: auto; + color: var(--momo-ink-tertiary); + content: "+"; +} +.ai-disclosure[open] summary::after { content: "−"; } +.ai-disclosure__body { padding: var(--momo-space-3) 0 var(--momo-space-5); } + +.ai-count-separator { + margin: 0 var(--momo-space-1); + color: var(--momo-ink-tertiary); +} + +.ai-row-action { + width: 28px; + height: 28px; + margin-left: var(--momo-space-2); + color: #a33e4a; +} + +@media (max-width: 900px) { + .ai-cockpit { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .ai-metric:nth-child(2) { border-right: 0; } + .ai-metric:nth-child(-n + 2) { border-bottom: 1px solid var(--momo-border-light); } +} + +@media (max-width: 768px) { + .ai-head { flex-wrap: wrap; } + .ai-head__main { flex-basis: calc(100% - 52px); } + .ai-head__actions { + width: 100%; + margin-left: 0; + } + .ai-head__actions .btn { flex: 1; min-height: 44px; } + .ai-cockpit { grid-template-columns: 1fr; } + .ai-metric, + .ai-metric:nth-child(2) { + min-height: 96px; + border-right: 0; + border-bottom: 1px solid var(--momo-border-light); + } + .ai-metric:last-child { border-bottom: 0; } + .ai-section__head { align-items: flex-start; } + .ai-mode-badge { max-width: 48%; text-align: right; } + .ai-source-row { grid-template-columns: 14px minmax(0, 1fr) minmax(80px, auto); } + + .ai-jobtable td:nth-child(1)::before { content: "檔案"; } + .ai-jobtable td:nth-child(2)::before { content: "狀態"; } + .ai-jobtable td:nth-child(3)::before { content: "進度"; } + .ai-jobtable td:nth-child(4)::before { content: "筆數"; } + .ai-jobtable td:nth-child(5)::before { content: "完成"; } + .ai-jobtable td:nth-child(6)::before { content: "驗證"; } +} + +@media (max-width: 380px) { + .ai-source-row { + grid-template-columns: 14px minmax(0, 1fr); + } + .ai-source-row__state { + grid-column: 2; + } +} diff --git a/web/static/css/page-auto-import.css b/web/static/css/page-auto-import.css index 7ba0070..4e6e966 100644 --- a/web/static/css/page-auto-import.css +++ b/web/static/css/page-auto-import.css @@ -234,7 +234,7 @@ font-weight: var(--momo-font-semibold); text-transform: uppercase; font-size: var(--momo-text-xs); - letter-spacing: 0.04em; + letter-spacing: 0; padding: var(--momo-space-3); white-space: nowrap; } From 603352b88ae383d86e090d320b27b86883628952 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 09:48:41 +0800 Subject: [PATCH 39/75] perf(runtime): isolate workers and compress web assets --- .env.example | 1 + app.py | 89 ++--- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 5 +- docs/guides/modularization_governance.md | 7 + .../code_modularization_inventory_20260430.md | 17 +- requirements.txt | 1 + routes/export_routes.py | 6 +- routes/openclaw_bot_routes.py | 77 +---- run_scheduler.py | 70 ++++ ...ort_codebase_modularization_performance.py | 34 ++ ...base_modularization_performance_service.py | 217 ++++++++++++ .../openclaw_bot/embedding_worker_runtime.py | 44 +++ services/openclaw_bot/scheduled_jobs.py | 96 ++++++ services/openclaw_learning_service.py | 7 +- services/web_scheduler_compat.py | 92 +++++ templates/login.html | 324 ++---------------- ...base_modularization_performance_service.py | 51 +++ tests/test_phase3f_cleanup_contracts.py | 7 +- tests/test_runtime_role_and_performance.py | 114 ++++++ web/static/css/page-login.css | 299 ++++++++++++++++ 21 files changed, 1124 insertions(+), 436 deletions(-) create mode 100644 scripts/ops/report_codebase_modularization_performance.py create mode 100644 services/codebase_modularization_performance_service.py create mode 100644 services/openclaw_bot/embedding_worker_runtime.py create mode 100644 services/openclaw_bot/scheduled_jobs.py create mode 100644 services/web_scheduler_compat.py create mode 100644 tests/test_codebase_modularization_performance_service.py create mode 100644 tests/test_runtime_role_and_performance.py create mode 100644 web/static/css/page-login.css diff --git a/.env.example b/.env.example index bc3d30e..a519b2f 100644 --- a/.env.example +++ b/.env.example @@ -406,6 +406,7 @@ MOMO_EVENT_ROUTER_REPLAY_LIMIT=3 # [選填] AI 自動化 Smoke 歷史保存 MOMO_AI_AUTOMATION_SMOKE_HISTORY=/app/data/ai_automation_smoke_history.jsonl MOMO_AI_AUTOMATION_SMOKE_HISTORY_LIMIT=200 +CODEBASE_INVENTORY_RECEIPT_PATH=/app/data/ai_automation/codebase_inventory/latest.json # [選填] OpenClaw Telegram bot OPENCLAW_BOT_TOKEN=your_openclaw_bot_token_here diff --git a/app.py b/app.py index af48aae..926e911 100644 --- a/app.py +++ b/app.py @@ -32,7 +32,6 @@ try: from flask import Flask, render_template, jsonify, request, send_file, redirect, url_for, flash, session from werkzeug.utils import secure_filename from pyngrok import ngrok, conf - import schedule from sqlalchemy import desc, and_, func, text, literal, case from sqlalchemy import inspect # V-New: 用於檢查資料表是否存在 from sqlalchemy.orm import joinedload @@ -42,7 +41,6 @@ try: # 導入自定義模組 try: - from scheduler import run_momo_task, run_edm_task, run_festival_task, run_auto_import_task, run_whitepage_check, run_competitor_price_feeder_task, run_pchome_match_backfill_task from database.manager import DatabaseManager from database.models import Base, Product, PriceRecord, MonthlySummaryAnalysis from database.edm_models import PromoProduct @@ -146,6 +144,24 @@ app = Flask(__name__, template_folder=TEMPLATE_DIR, static_folder=STATIC_DIR) app.config['SEND_FILE_MAX_AGE_DEFAULT'] = timedelta(days=7) +app.config.update( + COMPRESS_ALGORITHM=['br', 'gzip'], + COMPRESS_LEVEL=6, + COMPRESS_MIN_SIZE=1024, + COMPRESS_MIMETYPES=[ + 'text/html', + 'text/css', + 'text/plain', + 'text/javascript', + 'application/javascript', + 'application/json', + 'image/svg+xml', + ], +) + +from flask_compress import Compress # noqa: E402 + +compress = Compress(app) @app.url_defaults @@ -666,6 +682,12 @@ def add_security_response_headers(response): "style-src 'self' 'unsafe-inline' https:; script-src 'self' 'unsafe-inline' https:; " "connect-src 'self' https: wss:", ) + if request.endpoint == 'static': + if request.args.get('v') == SYSTEM_VERSION: + response.headers['Cache-Control'] = 'public, max-age=31536000, immutable' + else: + response.headers.setdefault('Cache-Control', 'public, max-age=604800') + return response if session.get('logged_in'): response.headers.setdefault('Cache-Control', 'private, no-store') return response @@ -1179,35 +1201,16 @@ def prepare_calendar_data(df, selected_month): # ================= ⚙️ 5. 服務啟動邏輯 ================= def run_schedule(): - """在背景執行緒中運行排程""" - sys_log.info("🚀 排程服務已啟動，等待任務...") - while True: - schedule.run_pending() - time.sleep(1) + """Backward-compatible local runner; production uses momo-scheduler.""" + from services.web_scheduler_compat import run_schedule as _run_schedule + + return _run_schedule(sys_log) def init_scheduler(): - """初始化排程任務（Gunicorn 模式下也會執行）""" - schedule.every(1).hours.do(run_momo_task) - schedule.every(1).hours.do(run_edm_task) - schedule.every(1).hours.do(run_festival_task) - sys_log.info(f"📅 已設定每小時執行主站、EDM與購物節爬蟲任務") + """Backward-compatible local scheduler; never imported by Gunicorn startup.""" + from services.web_scheduler_compat import init_scheduler as _init_scheduler - schedule.every(30).minutes.do(run_auto_import_task) - sys_log.info(f"📅 已設定每 30 分鐘執行 Google Drive 自動匯入任務") - - schedule.every(30).minutes.do(run_whitepage_check) - sys_log.info(f"📅 已設定每 30 分鐘執行網頁白頁監控任務") - - schedule.every(4).hours.do(run_competitor_price_feeder_task) - sys_log.info(f"📅 已設定每 4 小時執行 PChome 競品價格抓取任務") - - schedule.every().day.at("10:30").do(run_pchome_match_backfill_task) - sys_log.info(f"📅 已設定每日 10:30 執行 PChome 待比對補抓與挑品重算任務") - - # 啟動排程執行緒 - scheduler_thread = threading.Thread(target=run_schedule, daemon=True) - scheduler_thread.start() - sys_log.info("✅ 排程器已在背景執行緒中啟動") + return _init_scheduler(sys_log) # V-New: 在模組載入時自動初始化排程（Gunicorn 模式下也會執行） # 🚩 V-Fix 2026-01-14: 停用自動排程器以避免多個 gunicorn workers 重複執行任務 @@ -1224,34 +1227,10 @@ def start_flask(): app.run(host='0.0.0.0', port=80, use_reloader=False) def scheduled_job_wrapper(): - """執行 MOMO 爬蟲任務並發送通知""" - timestamp = datetime.now(TAIPEI_TZ).strftime('%H:%M:%S') - sys_log.info(f"⏰ [{timestamp}] 啟動背景抓取執行緒...") - - def job(): - # 1. 執行爬蟲 - run_momo_task() - - # 2. 發送通知 (僅發送今日異動) - try: - # 重新載入通知模組 - import importlib - import scheduler - import services.notification_manager - importlib.reload(scheduler) - importlib.reload(services.notification_manager) - from services.notification_manager import NotificationManager - - stats = get_dashboard_stats() - - # 只要有任何異動數據就發送通知 - if any(stats.values()): - screenshot_path = scheduler.capture_page_screenshot("http://127.0.0.1/", "momo_dashboard") - NotificationManager().send_momo_report(stats, screenshot_path) - except Exception as e: - sys_log.error(f"[Scheduler] ❌ 發送通知失敗: {e}") + """Backward-compatible command wrapper with lazy scheduler imports.""" + from services.web_scheduler_compat import scheduled_job_wrapper as _scheduled_job_wrapper - threading.Thread(target=job, daemon=True).start() + return _scheduled_job_wrapper(sys_log, get_dashboard_stats) if __name__ == "__main__": banner = f" MOMO 專業數據管理系統 {SYSTEM_VERSION} " diff --git a/config.py b/config.py index 083b399..0b16188 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.776" +SYSTEM_VERSION = "V10.777" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index c13c86d..9a5795d 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,8 +1,8 @@ # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth -> **最後更新**: 2026-07-10 (台北時間) +> **最後更新**: 2026-07-11 (台北時間) > **狀態**: 🟠 Partial。四 Agent、PixelRAG、MCP/RAG registry、PChome controlled-preview families 與多項 smoke/readback 已建立；但 access control、database identity/RBAC、full asset reconciliation、software supply chain、same-run controlled apply、restore drill、internal RAG canary 與 MCP/RAG runtime closure 尚未全部完成。任何局部 receipt 不得代表整體閉環完成。 -> **適用版本**: V10.773 +> **適用版本**: V10.777 --- @@ -15,6 +15,7 @@ - Gitea CD 必須執行 `report_security_governance_review.py --strict`，且不得解析 GitHub action/source/image。 - production 安全與治理完成度必須分開顯示 program、asset coverage、runtime closure；目前結論是 partial，不是 complete。 - 目前 ordered P0 以 `docs/guides/ai_automation_mainline_work_items.md` 為準：先關閉 access exposure，再做 identity/RBAC、webhook trust、supply chain、asset reconciliation、controlled-apply envelope、internal RAG canary 與 MCP/RAG runtime closure。 +- V10.777 起 web process 只能註冊 OpenClaw Blueprint，不得在 import 或 `record_once` 啟動 APScheduler、timed jobs 或 embedding thread；10 個 OpenClaw timed jobs 與唯一 embedding worker runtime owner 必須由 `momo-scheduler` 明確啟動。每日 04:15 的 codebase inventory 只掃描程式與前端資產並寫 artifact receipt，不讀 secret、不寫業務 DB。 --- diff --git a/docs/guides/modularization_governance.md b/docs/guides/modularization_governance.md index 98fca2c..44c29b2 100644 --- a/docs/guides/modularization_governance.md +++ b/docs/guides/modularization_governance.md @@ -47,3 +47,10 @@ - 是否讓既有 >800 行檔案淨增加？若是，先抽模組或更新 inventory 與拆分計畫。 - 是否新增重複 cache / DB manager / Telegram / AI client 初始化？若是，改用既有 service。 - 是否需要更新 ADR、memory、SOT 或 guide？若是，本次 commit 一起更新。 + +## 自動盤點與狀態回寫 + +- 本機或 CI 使用 `python3 scripts/ops/report_codebase_modularization_performance.py` 取得 JSON；加 `--write` 才原子寫入 receipt。 +- 正式 `momo-scheduler` 每日 04:15 自動執行，receipt 固定為 `data/ai_automation/codebase_inventory/latest.json`。 +- work item 狀態只允許 `completed`、`in_progress`、`not_started`；拆出 helper 不代表整個巨型模組 completed。 +- source/test 綠燈只代表候選變更可部署；必須另有 scheduler receipt、web process 無背景 worker、壓縮/cache header 與 production health readback 才能關閉 runtime lane。 diff --git a/docs/memory/code_modularization_inventory_20260430.md b/docs/memory/code_modularization_inventory_20260430.md index e7d754a..01e970f 100644 --- a/docs/memory/code_modularization_inventory_20260430.md +++ b/docs/memory/code_modularization_inventory_20260430.md @@ -131,7 +131,7 @@ |---:|---|---|---| | 44888 | `services/pchome_mapping_backlog_service.py` | P0 gate 巨型 service | 立即停止新增 closeout package；拆 policy、executor、verifier、receipt、reporter | | 14289 | `services/ai_automation_smoke_service.py` | P0 smoke 巨型 service | 拆 family registry、collectors、health projection、metrics adapter | -| 9442 | `routes/openclaw_bot_routes.py` | P0 巨型 Blueprint | route、command、report、scheduler hook 分離 | +| 9383 | `routes/openclaw_bot_routes.py` | P0 巨型 Blueprint、拆分進行中 | scheduler hook 已移到 `services/openclaw_bot/scheduled_jobs.py`；下一步拆 report、command | | 7641 | `routes/ai_routes.py` | P0 巨型 Blueprint | PChome growth、AI automation、recommendation route extension 分離 | | 5865 | `services/marketplace_product_matcher.py` | P0 matcher 巨型 service | identity、variant、unit、scoring、explanation 分離 | | 5513 | `services/ppt_generator.py` | P0 報表巨型 service | orchestration、slides、charts、registry 分離 | @@ -152,9 +152,9 @@ | 1417 | `services/telegram_templates.py` | P1 templates | decision、incident、report renderer 分離 | | 1409 | `services/import_service.py` | P1 import | acquisition、validation、writer、verifier 分離 | | 1394 | `services/telegram_bot_service.py` | P1 bot | command、formatter、client 分離 | -| 1370 | `run_scheduler.py` | P1 scheduler entry | registration、catchup、observability wrappers 分離 | +| 1440 | `run_scheduler.py` | P1 scheduler entry、拆分進行中 | OpenClaw timed-job registry 已外移；下一步拆 task registration 與 runtime startup | | 1306 | `routes/system_public_routes.py` | P1 public routes | health、metrics、public projection 分離 | -| 1290 | `app.py` | P1 bootstrap | app factory 與 startup guards 分離 | +| 1269 | `app.py` | P1 bootstrap、拆分進行中 | web scheduler 相容層已移到 `services/web_scheduler_compat.py`；下一步拆 extension setup | | 1272 | `services/code_review_pipeline_service.py` | P2 code review | scan、normalize、persist、report 分離 | | 1202 | `services/pchome_revenue_growth_service.py` | P1 revenue growth | sales query、offer comparison、scoring、projection 分離 | | 1188 | `services/ppt_auto_generation_service.py` | P2 PPT automation | resolver、queue、generator、catchup 分離 | @@ -162,7 +162,7 @@ | 1023 | `routes/cicd_routes.py` | P2 CI/CD Blueprint | query、deploy action、route glue 分離 | | 1014 | `services/pchome_crawler.py` | P1 crawler | transport、parser、fallback、rate policy 分離 | | 1014 | `routes/daily_sales_routes.py` | P2 Daily Sales Blueprint | query、format、export、route glue 分離 | -| 1013 | `routes/export_routes.py` | P2 Export Blueprint | command、path policy、download 分離 | +| 1017 | `routes/export_routes.py` | P2 Export Blueprint | command、path policy、download 分離；string dtype Excel sanitizer 已修正 | | 989 | `routes/market_intel_routes.py` | P2 Market Intel Blueprint | page、API、MCP registration 分離 | | 966 | `services/trend_crawler.py` | P2 trend crawler | adapters、parser、persistence 分離 | | 942 | `services/learning_pipeline.py` | P2 RAG learning | distill、promotion、persistence、telemetry 分離 | @@ -188,7 +188,7 @@ ## 工作項目 -1. P0：持續拆 `routes/openclaw_bot_routes.py`；Telegram API helper 已搬到 `services/openclaw_bot/telegram_api.py`，Inline Keyboard builders 已搬到 `services/openclaw_bot/menu_keyboards.py`，下一步拆 report formatting 或 command dispatcher。 +1. P0：持續拆 `routes/openclaw_bot_routes.py`；Telegram API helper、Inline Keyboard builders 與 10 個 timed-job scheduler hook 已搬離巨型 Blueprint，下一步拆 report formatting 或 command dispatcher。 2. P0：拆 `routes/sales_routes.py`，先把 chart/query/calendar 計算搬到 `services/sales/`。 3. P0：拆 `scheduler.py`，建立 `jobs/` 或 `services/scheduler/` task registry。 4. P0：拆 `routes/admin_observability_routes.py`，先搬純查詢函式到 `services/observability_query_service.py`，再搬 AutoHeal / Code Review / AiderHeal / throttle mutation 到 `services/observability_action_service.py`。 @@ -201,3 +201,10 @@ - `tests/test_modularization_governance.py` 會掃描所有 Python 檔案；任何新的 >800 行檔案沒有列在本 inventory 就會失敗。 - 若檔案拆小後低於 800 行，可從本清單移除並同步更新測試。 + +### 2026-07-11 自動盤點 + +- `python3 scripts/ops/report_codebase_modularization_performance.py` 會為 Python 與前端大型資產建立穩定 work item ID、P0/P1、`in_progress/not_started` 狀態與拆分方向。 +- `momo-scheduler` 每日 04:15 執行同一盤點，原子寫入 `data/ai_automation/codebase_inventory/latest.json`；失敗只記 degraded receipt，不得阻斷其他排程。 +- 本次基線：447 個 production Python modules、260,059 行、48 個大於等於 800 行模組、33 個大型前端資產、23 個 P0、6/6 runtime controls 通過；program completion 69.1%，runtime closure 仍為 partial。 +- `services/openclaw_learning_service.py` 已降至 789 行；embedding worker 的唯一 runtime owner 已移到 `momo-scheduler`，因此不再列入大檔債務，但仍需由 production worker/readback 驗證。 diff --git a/requirements.txt b/requirements.txt index 4370b99..98288f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ Flask>=2.3 +Flask-Compress>=1.24,<2 Flask-WTF>=1.2.2 gunicorn>=20.1 pandas>=1.5 diff --git a/routes/export_routes.py b/routes/export_routes.py index 530477c..868ca48 100644 --- a/routes/export_routes.py +++ b/routes/export_routes.py @@ -65,7 +65,11 @@ def _sanitize_excel_dataframe(df: pd.DataFrame) -> pd.DataFrame: return df cleaned = df.copy() for column in cleaned.columns: - if cleaned[column].dtype == object: + column_dtype = cleaned[column].dtype + if ( + pd.api.types.is_object_dtype(column_dtype) + or pd.api.types.is_string_dtype(column_dtype) + ): cleaned[column] = cleaned[column].map(_sanitize_excel_cell) return cleaned diff --git a/routes/openclaw_bot_routes.py b/routes/openclaw_bot_routes.py index 372bc33..e6d8a7a 100644 --- a/routes/openclaw_bot_routes.py +++ b/routes/openclaw_bot_routes.py @@ -346,7 +346,6 @@ TRIGGER_KEYWORDS = [] # 空 = 全部回應（小龍蝦是專用業務群組） # ── 目標管理（記憶體，跨 session 用 DB 儲存）───────────────────── _GOALS: dict = {} # {'daily','monthly','quarterly','half','yearly': float} -_scheduler = None # ── 輸入等待狀態機（chat_id → pending action）──────────────────── _input_pending: dict = {} # {chat_id: {'action': str, 'label': str}} @@ -4814,70 +4813,13 @@ def send_daily_excel(): def start_scheduler(): - """啟動排程（Flask app 啟動後呼叫）""" - global _scheduler - try: - if _scheduler is not None and _scheduler.running: - sys_log.info("[OpenClawBot] Scheduler 已在執行中，跳過重複啟動") - return - - from apscheduler.schedulers.background import BackgroundScheduler - from apscheduler.triggers.cron import CronTrigger - - _scheduler = BackgroundScheduler(timezone='Asia/Taipei') - _scheduler.add_job( - send_morning_report, - CronTrigger(hour=8, minute=30), - id="openclaw_send_morning_report", - replace_existing=True, - ) - _scheduler.add_job( - send_competitor_report, - CronTrigger(hour=8, minute=0), - id="openclaw_send_competitor_report", - replace_existing=True, - ) - _scheduler.add_job( - send_daily_excel, - CronTrigger(hour=8, minute=45), - id="openclaw_send_daily_excel", - replace_existing=True, - ) - _scheduler.add_job( - send_evening_report, - CronTrigger(hour=21, minute=0), - id="openclaw_send_evening_report", - replace_existing=True, - ) - _scheduler.add_job( - send_weekly_report, - CronTrigger(day_of_week='mon', hour=9, minute=0), - id="openclaw_send_weekly_report", - replace_existing=True, - ) - _scheduler.add_job( - check_anomalies, - CronTrigger(hour='9,12,15,18', minute=0), - id="openclaw_check_anomalies", - replace_existing=True, - ) - # ADR-019 Phase 6: 每日 09:00 主動巡檢資料新鮮度，缺資料時透過 EventRouter 發警告 - try: - from services.data_freshness_probe import run_data_freshness_probe - _scheduler.add_job( - run_data_freshness_probe, - CronTrigger(hour=9, minute=5), - id="openclaw_data_freshness_probe", - replace_existing=True, - ) - except ImportError as _e: - sys_log.warning(f"[OpenClawBot] data_freshness_probe 未安裝，跳過：{_e}") - _scheduler.start() - sys_log.info("[OpenClawBot] Scheduler started ✓ (competitor/morning/excel/evening/weekly/anomaly/freshness)") - except ImportError: - sys_log.warning("[OpenClawBot] APScheduler 未安裝 — 排程功能停用") - except Exception as e: - sys_log.error(f"[OpenClawBot] start_scheduler: {e}") + """Compatibility readback for callers of the retired web-worker scheduler.""" + sys_log.info("[OpenClawBot] Timed jobs are owned by momo-scheduler") + return { + "status": "delegated", + "runtime_owner": "momo-scheduler", + "scheduler": "schedule", + } def register_commands(): @@ -9437,6 +9379,5 @@ def webhook_info(): @openclaw_bot_bp.record_once def _on_register(_state): - """Blueprint 被 app.register_blueprint 時自動啟動排程""" - start_scheduler() - sys_log.info("[OpenClawBot] Blueprint registered — scheduler boot triggered") + """Register HTTP routes without starting background workers in Gunicorn.""" + sys_log.info("[OpenClawBot] Blueprint registered; timed jobs delegated to momo-scheduler") diff --git a/run_scheduler.py b/run_scheduler.py index 85677e3..8d103b9 100644 --- a/run_scheduler.py +++ b/run_scheduler.py @@ -358,6 +358,9 @@ def _register_schedules(): schedule.every().day.at("04:00").do(run_backup_monitor_task) logger.info("📅 每日 04:00：backup_monitor") + schedule.every().day.at("04:15").do(run_codebase_modularization_performance_task) + logger.info("📅 每日 04:15：codebase_modularization_performance inventory") + schedule.every().monday.at("06:00").do(run_weekly_strategy_task) logger.info("📅 每週一 06:00：weekly_strategy") @@ -389,6 +392,10 @@ def _register_schedules(): schedule.every().day.at("07:00").do(_monthly_report_gate) logger.info("📅 每月1日 07:00：monthly_report") + from services.openclaw_bot.scheduled_jobs import register_openclaw_schedules + + register_openclaw_schedules(schedule, logger) + def run_daily_token_report_task(): """每日 23:55 — Operation Ollama-First v5.0 Phase 1 收尾：LLM Token 日報。 @@ -1298,6 +1305,46 @@ def run_cleanup_agent_context(): session.close() +def run_codebase_modularization_performance_task(): + """Scan source/assets, generate work items, and persist a bounded receipt.""" + try: + from services.codebase_modularization_performance_service import ( + build_codebase_inventory, + write_codebase_inventory_receipt, + ) + + result = build_codebase_inventory() + receipt_path = write_codebase_inventory_receipt(result) + payload = { + "status": "completed", + "summary": result["summary"], + "receipt_path": str(receipt_path), + "top_p0_work_items": [ + item + for item in result["python_work_items"] + result["frontend_work_items"] + if item["priority"] == "P0" + ][:20], + } + _save_stats("codebase_modularization_performance", payload) + logger.info( + "[CodebaseInventory] completion=%s p0=%s receipt=%s", + payload["summary"]["program_completion_percent"], + payload["summary"]["p0_work_item_count"], + receipt_path, + ) + return payload + except Exception as error: + logger.error("[CodebaseInventory] task failed: %s", error, exc_info=True) + _notify_scheduler_failure( + "run_codebase_modularization_performance_task", + error, + source="Scheduler.CodebaseInventory", + event_type="codebase_inventory_failure", + title="Codebase 模組化與效能盤點失敗", + ) + return {"status": "failed", "error_kind": type(error).__name__} + + def _run_elephant_alpha_engine(): """Daemon thread: ElephantAlpha 自主監控引擎（獨立 asyncio loop）""" loop = None @@ -1320,6 +1367,29 @@ if __name__ == "__main__": _register_schedules() logger.info("✅ 全部排程任務已註冊") + try: + from services.openclaw_bot.embedding_worker_runtime import start_embedding_worker + + _embedding_thread = start_embedding_worker() + logger.info( + "[OpenClawRuntime] embedding worker=%s alive=%s", + _embedding_thread.name, + _embedding_thread.is_alive(), + ) + except Exception as _embedding_error: + logger.error( + "[OpenClawRuntime] embedding worker startup failed: %s", + _embedding_error, + exc_info=True, + ) + _notify_scheduler_failure( + "start_openclaw_embedding_worker", + _embedding_error, + source="Scheduler.OpenClawRuntime", + event_type="openclaw_embedding_worker_startup_failure", + title="OpenClaw embedding worker 啟動失敗", + ) + _ea_thread = threading.Thread( target=_run_elephant_alpha_engine, daemon=True, diff --git a/scripts/ops/report_codebase_modularization_performance.py b/scripts/ops/report_codebase_modularization_performance.py new file mode 100644 index 0000000..cd273de --- /dev/null +++ b/scripts/ops/report_codebase_modularization_performance.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Report or persist the current codebase modularization/performance inventory.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.codebase_modularization_performance_service import ( # noqa: E402 + build_codebase_inventory, + write_codebase_inventory_receipt, +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--write", action="store_true", help="persist the latest runtime receipt") + args = parser.parse_args() + result = build_codebase_inventory() + if args.write: + result["receipt_path"] = str(write_codebase_inventory_receipt(result)) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/codebase_modularization_performance_service.py b/services/codebase_modularization_performance_service.py new file mode 100644 index 0000000..fe2cd7c --- /dev/null +++ b/services/codebase_modularization_performance_service.py @@ -0,0 +1,217 @@ +"""Automated codebase modularization and frontend payload inventory.""" + +from __future__ import annotations + +import hashlib +import json +import os +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Iterable + + +ROOT = Path(__file__).resolve().parents[1] +TAIPEI_TZ = timezone(timedelta(hours=8)) +PYTHON_DEBT_LINES = 800 +FRONTEND_P0_BYTES = 50_000 +FRONTEND_P1_BYTES = 20_000 +IN_PROGRESS_PYTHON_PATHS = { + "app.py", + "routes/openclaw_bot_routes.py", + "run_scheduler.py", +} +IGNORED_PARTS = { + ".git", + ".pytest_cache", + ".venv", + "__pycache__", + "backups", + "tests", + "venv", +} + + +def _iter_python_files(root: Path) -> Iterable[Path]: + for path in root.rglob("*.py"): + relative = path.relative_to(root) + if any(part in IGNORED_PARTS for part in relative.parts): + continue + if relative.parts[:2] in {("docs", "design"), ("docs", "design_audit_frontend")}: + continue + yield path + + +def _iter_frontend_files(root: Path) -> Iterable[Path]: + for folder in ("templates", "static", "web/templates", "web/static"): + base = root / folder + if not base.exists(): + continue + for path in base.rglob("*"): + if path.is_file() and path.suffix.lower() in {".html", ".css", ".js"}: + yield path + + +def _line_count(path: Path) -> int: + with path.open(encoding="utf-8", errors="ignore") as handle: + return sum(1 for _ in handle) + + +def _priority_for_python(lines: int) -> str: + if lines >= 2_000: + return "P0" + if lines >= PYTHON_DEBT_LINES: + return "P1" + return "P2" + + +def _priority_for_frontend(size_bytes: int) -> str: + return "P0" if size_bytes >= FRONTEND_P0_BYTES else "P1" + + +def _split_direction(path: str) -> str: + if path.startswith("routes/"): + return "separate route glue, queries, commands, and projections" + if path == "scheduler.py" or path == "run_scheduler.py": + return "separate task registry, domain jobs, runtime workers, and receipts" + if path.startswith("services/"): + return "separate policy, adapters, execution, verification, and reporting" + if path == "app.py": + return "extract app factory, extensions, blueprint registry, and startup guards" + return "separate cohesive domain responsibilities behind stable interfaces" + + +def _work_item_id(kind: str, path: str) -> str: + digest = hashlib.sha256(path.encode("utf-8")).hexdigest()[:10] + return f"{kind}-{digest}" + + +def _runtime_controls(root: Path) -> list[dict]: + app_source = (root / "app.py").read_text(encoding="utf-8") + bot_source = (root / "routes/openclaw_bot_routes.py").read_text(encoding="utf-8") + learning_source = (root / "services/openclaw_learning_service.py").read_text(encoding="utf-8") + login_source = (root / "templates/login.html").read_text(encoding="utf-8") + registration = bot_source[bot_source.index("@openclaw_bot_bp.record_once") :] + return [ + { + "id": "web_no_eager_scheduler_import", + "passed": "from scheduler import" not in app_source, + }, + { + "id": "openclaw_web_registration_no_scheduler_boot", + "passed": "start_scheduler()" not in registration, + }, + { + "id": "embedding_worker_explicit_runtime_owner", + "passed": "Thread(target=_embedding_worker_loop" not in learning_source, + }, + { + "id": "text_compression_enabled", + "passed": "Compress(app)" in app_source and "COMPRESS_ALGORITHM" in app_source, + }, + { + "id": "versioned_static_immutable", + "passed": "max-age=31536000, immutable" in app_source, + }, + { + "id": "login_has_no_third_party_assets", + "passed": "https://" not in login_source and " + - +
@@ -295,26 +34,23 @@