diff --git a/config.py b/config.py index b69ad11..fc4b4b1 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.760" +SYSTEM_VERSION = "V10.761" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index f6f9d50..5bd159b 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -120,6 +120,7 @@ - 2026-07-09 起 PixelRAG VLM route readiness 必須在 execute 前可讀回：`/api/ai-automation/pixelrag-vlm-route-readiness` 與 `scripts/ops/report_pixelrag_vlm_route_readiness.py` 只讀 approved Ollama `/api/tags`，輸出 configured model、candidate model、reachable host、model_route_ready 與 next machine action；`/api/ai-automation/smoke` 需包含 `PixelRAG VLM route readiness`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_vlm_route_readiness` family。此 readback 不呼叫 `/api/generate`、不讀 secret、不寫 DB；worker execute 必須使用它自動避開缺模型路由，完全缺候選時寫 `model_route_not_ready` artifact receipt。 - 2026-07-10 起 PixelRAG platform probe 必須把 blocked / interstitial receipt 轉成機器可執行下一步：`/api/ai-automation/pixelrag-platform-probe` 與 `scripts/ops/report_pixelrag_platform_probe.py` 需讀取 capture receipts 與 VLM replay receipts，辨識 Shopee language / region / generic marketplace slogan / traffic verification 與 Coupang 403 / access denied，輸出 public browser context probe、structured source fallback、platform backoff、blocked page not product data 與 no-write 邊界；`/api/ai-automation/smoke` 需包含 `PixelRAG platform probe readiness`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_platform_probe` family。此 readback 不抓外站、不讀 secret/cookie/session、不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture worker 可消化 `public_browser_context` 的 locale/timezone/Accept-Language，但必須濾掉 Cookie、Authorization、token/secret/key 類 header。 - 2026-07-10 起 PixelRAG platform probe 必須有 controlled worker：`/api/ai-automation/pixelrag-platform-probe-worker` 與 `scripts/ops/run_pixelrag_platform_probe_worker.py` 預設 dry-run，`execute=true&write_receipt=true` 才會對 Shopee language / traffic / generic landing 執行 public empty-context capture，並對 Coupang 403 / access denied 自動產生 structured-source/backoff artifact receipt；`/api/ai-automation/smoke` 需包含 `PixelRAG platform probe worker`，`/api/ai-automation/scheduled-health-summary` 需輸出 `pixelrag_platform_probe_worker` family。此 worker 不登入、不讀 cookie/session/secret、不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture 成功後下一步固定為 RAG candidate replay，structured fallback 只產生機器可讀 package，不把 blocked page 當商品資料。 +- 2026-07-10 起 PixelRAG platform probe worker 必須自動處理 capture runtime gap：若 production app 容器缺 Playwright，`execute=true&write_receipt=true` 不得退回人工審核或 critical dead-end；worker 必須寫 `capture_runtime_unavailable_structured_fallback_package` artifact receipt，標記 `network_call=false`、`writes_database=false`、`primary_human_gate_count=0`，並把下一步改為 `run_structured_source_or_install_pixelrag_capture_runtime`。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 @@ -892,6 +893,7 @@ POSTGRES_HOST=momo-db | 2026-07-10 | PixelRAG VLM language note / generic marketplace slogan 必須視為 interstitial | V10.758 起若 VLM notes 或 title 顯示 `Language selection page`、語言/地區選擇，或只抽到 Shopee `蝦皮購物 | 花得更少買得更好` 這類網站 slogan 且 price/shop 缺失，worker validation 會標記 `interstitial_signal_detected` / `generic_marketplace_title_detected` 與 `non_product_or_interstitial_detected=true`，run-level 與 item-level 下一步都輸出 `run_platform_probe_or_use_structured_api`。 | | 2026-07-10 | PixelRAG platform probe 必須自動消化 marketplace interstitial / blocked receipt | V10.759 起 `/api/ai-automation/pixelrag-platform-probe`、`scripts/ops/report_pixelrag_platform_probe.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 platform probe / `pixelrag_platform_probe` family；Shopee traffic / language / generic landing 會轉成 `run_shopee_public_context_probe_then_structured_source_fallback`，Coupang 403 / access denied 會轉成 `use_structured_source_or_platform_backoff_policy`。probe plan 只產生 public empty browser context、structured adapter fallback 與 no-write contract，不讀或注入 cookie/session/login，不寫 DB、不寫 `ai_insights`、不寫正式價格表；capture worker 只允許 locale/timezone/非敏感 header 並濾掉 Cookie / Authorization / token / secret / key 類 header。 | | 2026-07-10 | PixelRAG platform probe worker 必須把 probe plan 變成 controlled automation | V10.760 起 `/api/ai-automation/pixelrag-platform-probe-worker`、`scripts/ops/run_pixelrag_platform_probe_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled platform probe worker / `pixelrag_platform_probe_worker` family；dry-run 只規劃 capture 或 structured fallback，execute 模式才執行 public empty-context capture 或寫 structured-source/backoff artifact receipt。此 worker 明確標記 `network_call` 只在 capture execute 時為 true、`model_call=false`、`writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`raw_cookie_or_session_read=false`、`primary_human_gate_count=0`；capture 成功後必須回到 PixelRAG RAG candidate replay，不得直接寫正式價格或把 blocked page 當商品資料。 | +| 2026-07-10 | PixelRAG platform probe worker 缺 capture runtime 時必須自動降級 | V10.761 起若 `run_pixelrag_platform_probe_worker.py --execute --write-receipt` 在 app 容器偵測到 `ModuleNotFoundError: playwright`，worker 會自動產生 `capture_runtime_unavailable_structured_fallback_package` receipt，`network_call=false`、`artifact_write=true`、`writes_database=false`、`primary_human_gate_count=0`，不再把 Shopee/Coupang platform probe runtime gap 變成人工主流程。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index e146883..bb5f192 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -20,7 +20,7 @@ | Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | | Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | | Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | -| Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture, Coupang 403/access denied goes to structured-source/backoff artifact receipts. | +| Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture when runtime is available, missing Playwright auto-falls back to structured-source/backoff receipt, and Coupang 403/access denied goes to structured-source/backoff artifact receipts. | | In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, and PixelRAG platform probe worker families. | | In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | diff --git a/services/pixelrag_platform_probe_worker_service.py b/services/pixelrag_platform_probe_worker_service.py index 5d3b76c..21f92f6 100644 --- a/services/pixelrag_platform_probe_worker_service.py +++ b/services/pixelrag_platform_probe_worker_service.py @@ -238,14 +238,40 @@ def _run_capture_subprocess( payload = json.loads(completed.stdout) except json.JSONDecodeError: payload = {} + stderr_excerpt = completed.stderr[:500] + if len(completed.stderr) > 500: + stderr_excerpt = f"{completed.stderr[:250]}...{completed.stderr[-250:]}" return { "returncode": completed.returncode, "stdout_json": payload, "stdout_excerpt": completed.stdout[:500], - "stderr_excerpt": completed.stderr[:500], + "stderr_excerpt": stderr_excerpt, } +def _capture_runtime_unavailable(result: Mapping[str, Any]) -> bool: + haystack = " ".join( + [ + str(result.get("stdout_excerpt") or ""), + str(result.get("stderr_excerpt") or ""), + ] + ).lower() + return ( + "modulenotfounderror" in haystack + and "playwright" in haystack + ) or "no module named 'playwright'" in haystack or ( + "from playwright.sync_api" in haystack + and "traceback" in haystack + ) or ( + "playwright" in haystack + and ( + "executable doesn't exist" in haystack + or "please run" in haystack + or "browsertype.launch" in haystack + ) + ) + + def _capture_item( item: Mapping[str, Any], *, @@ -302,8 +328,25 @@ def _capture_item( and capture_payload.get("success") is True and capture_payload.get("status") == "captured" ) + runtime_missing = not captured_ok and _capture_runtime_unavailable(result) + structured_fallback_available = bool( + (item.get("structured_source_fallback") or {}).get("available") + ) + capture_fallback = not captured_ok and structured_fallback_available base.update({ - "worker_status": "executed_capture_ok" if captured_ok else "capture_worker_error", + "worker_status": ( + "executed_capture_ok" + if captured_ok + else ( + ( + "capture_runtime_unavailable_structured_fallback_package" + if runtime_missing + else "capture_failed_structured_fallback_package" + ) + if capture_fallback + else "capture_worker_error" + ) + ), "ready_for_execution": True, "capture_execution_ready": True, "capture_manifest_id": manifest.get("manifest_id"), @@ -314,14 +357,39 @@ def _capture_item( else 0, "capture_tile_file_count": tile_file_count, "capture_receipt_path": receipt_path, - "network_call_performed": True, + "network_call_performed": bool(not runtime_missing), "artifact_write_performed": bool(captured_ok), + "structured_source_fallback": item.get("structured_source_fallback") or {}, + "structured_source_package": ( + { + "adapter_code": (item.get("structured_source_fallback") or {}).get("adapter_code"), + "available": True, + "source_count": len(list((item.get("structured_source_fallback") or {}).get("sources") or [])), + "network_request_allowed": bool( + (item.get("structured_source_fallback") or {}).get("network_request_allowed") + ), + "database_write_allowed": False, + "dry_run_only": True, + "blocked_page_not_product_data": True, + "capture_runtime_unavailable": runtime_missing, + } + if capture_fallback + else None + ), "capture_stdout_excerpt": result["stdout_excerpt"] if not captured_ok else "", "capture_stderr_excerpt": result["stderr_excerpt"] if not captured_ok else "", "next_machine_action": ( "run_pixelrag_rag_candidate_replay_after_probe_capture" if captured_ok - else "repair_pixelrag_platform_probe_worker_capture_runtime_or_use_structured_source" + else ( + ( + "run_structured_source_or_install_pixelrag_capture_runtime" + if runtime_missing + else "run_structured_source_or_retry_platform_capture" + ) + if capture_fallback + else "repair_pixelrag_platform_probe_worker_capture_runtime_or_use_structured_source" + ) ), }) if write_receipt: @@ -408,7 +476,7 @@ def run_pixelrag_platform_probe_worker( ready_count = sum(1 for item in probe_items if item.get("probe_ready")) dry_run_count = sum(1 for item in worker_items if str(item.get("worker_status") or "").startswith("dry_run_")) structured_count = sum( - 1 for item in worker_items if "structured_source_fallback" in str(item.get("worker_status") or "") + 1 for item in worker_items if "structured" in str(item.get("worker_status") or "") ) capture_ready_count = sum(1 for item in worker_items if item.get("capture_execution_ready")) capture_execute_count = sum( @@ -417,7 +485,10 @@ def run_pixelrag_platform_probe_worker( capture_ok_count = sum(1 for item in worker_items if item.get("worker_status") == "executed_capture_ok") capture_error_count = sum(1 for item in worker_items if item.get("worker_status") == "capture_worker_error") executed_structured_count = sum( - 1 for item in worker_items if item.get("worker_status") == "executed_structured_source_fallback_package" + 1 + for item in worker_items + if "structured" in str(item.get("worker_status") or "") + and not str(item.get("worker_status") or "").startswith("dry_run_") ) executed_count = capture_execute_count + executed_structured_count skipped_count = sum(1 for item in worker_items if str(item.get("worker_status") or "").startswith("skipped_")) diff --git a/tests/test_pixelrag_platform_probe_worker_service.py b/tests/test_pixelrag_platform_probe_worker_service.py index 8236375..658d6e1 100644 --- a/tests/test_pixelrag_platform_probe_worker_service.py +++ b/tests/test_pixelrag_platform_probe_worker_service.py @@ -147,6 +147,68 @@ def test_pixelrag_platform_probe_worker_execute_capture_writes_receipt(tmp_path, assert receipt["receipt_path"] == str(receipt_path) +def test_pixelrag_platform_probe_worker_missing_playwright_falls_back_to_structured_receipt( + tmp_path, + monkeypatch, +): + from services import pixelrag_platform_probe_worker_service as service + + shopee_receipt = _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-traffic", + title="蝦皮購物 | 花得更少買得更好", + url="https://shopee.tw/search?keyword=sunscreen", + ) + _patch_receipt( + shopee_receipt, + page_metrics={ + "final_url": "https://shopee.tw/verify/traffic/error?home_url=https%3A%2F%2Fshopee.tw", + "title": "蝦皮購物 | 花得更少買得更好", + }, + ) + + monkeypatch.setattr( + service, + "_run_capture_subprocess", + lambda **kwargs: { + "returncode": 1, + "stdout_json": {}, + "stdout_excerpt": "", + "stderr_excerpt": "ModuleNotFoundError: No module named 'playwright'", + }, + ) + payload = service.run_pixelrag_platform_probe_worker( + artifact_root=tmp_path, + output_root=tmp_path / "worker_receipts", + platform="shopee_tw", + execute=True, + write_receipt=True, + ) + + assert payload["status"] == "ok" + assert payload["summary"]["capture_error_count"] == 0 + assert payload["summary"]["structured_fallback_count"] == 1 + assert payload["summary"]["executed_structured_count"] == 1 + assert payload["summary"]["network_call_performed"] is False + assert payload["summary"]["receipt_written_count"] == 1 + item = payload["worker_items"][0] + assert item["worker_status"] == "capture_runtime_unavailable_structured_fallback_package" + assert item["structured_source_package"]["capture_runtime_unavailable"] is True + assert item["next_machine_action"] == "run_structured_source_or_install_pixelrag_capture_runtime" + receipt_path = ( + tmp_path + / "worker_receipts" + / "shopee_tw" + / "shopee-traffic" + / "platform_probe_worker_receipt.json" + ) + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "capture_runtime_unavailable_structured_fallback_package" + assert receipt["artifact_write_performed"] is True + assert receipt["network_call_performed"] is False + + def test_pixelrag_platform_probe_worker_execute_structured_fallback_writes_receipt(tmp_path): from services.pixelrag_platform_probe_worker_service import run_pixelrag_platform_probe_worker