From bed3a8a8ee25fea385a23fa11854852247dc39a4 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 00:17:02 +0800 Subject: [PATCH] feat(ai): add PixelRAG VLM generate preflight --- TODO_NEXT_STEPS.txt | 2 +- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 1 + routes/system_public_routes.py | 19 +++ .../report_pixelrag_vlm_route_readiness.py | 13 ++ scripts/ops/run_pixelrag_vlm_replay_worker.py | 13 ++ services/ai_automation_smoke_service.py | 22 +++ .../pixelrag_vlm_replay_worker_service.py | 48 ++++++- .../pixelrag_vlm_route_readiness_service.py | 136 ++++++++++++++++-- ...test_pixelrag_vlm_replay_worker_service.py | 77 ++++++++++ ...st_pixelrag_vlm_route_readiness_service.py | 65 +++++++++ 11 files changed, 384 insertions(+), 14 deletions(-) diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 3853591..478de4e 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -9,7 +9,7 @@ python scripts/ops/check_production_version_truth.py 目前最新版本仍以 production `https://mo.wooo.work/health` readback 為準。 -本輪 source target 為 `V10.752`；部署完成前不得宣稱正式環境已是 `V10.752`。 +本輪 source target 為 `V10.753`；部署完成前不得宣稱正式環境已是 `V10.753`。 舊 iCloud checkout 不是 Gitea dev worktree，不得拿來當最新版本真相。 ================================================================================ diff --git a/config.py b/config.py index bdada6f..4a2c7ff 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.752" +SYSTEM_VERSION = "V10.753" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 62cb6e5..bc75415 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -882,6 +882,7 @@ POSTGRES_HOST=momo-db | 2026-07-09 | PixelRAG application portfolio 必須把可整合場景轉成主線工作項目 | V10.748 起 `/api/ai-automation/pixelrag-application-portfolio` 與 `scripts/ops/report_pixelrag_application_portfolio.py` 必須輸出 PixelRAG 在 commerce、RAG、UX、ops、marketing、governance 的可整合/可運用 lanes；每條 lane 需有 priority、status、integrates_with、use_cases、current_capability、next_machine_action、no-write 邊界與 forbidden guardrails。此 readback 依據 PixelRAG visual-RAG pattern、Google Merchant product data、Google Product structured data 與 Baymard product list UX 轉成內部工作項目；它不抓外站、不呼叫模型、不讀 secret、不寫 DB、不把像素結果當正式價格。 | | 2026-07-09 | PixelRAG Ollama-first VLM replay worker 必須有 runtime monitoring | V10.751 起 `/api/ai-automation/pixelrag-vlm-replay-worker`、`scripts/ops/run_pixelrag_vlm_replay_worker.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 controlled VLM replay worker / `pixelrag_vlm_replay_worker` family；readback 預設 dry-run，不呼叫模型、不寫 artifact，execute 模式只讀 saved tiles、呼叫 approved Ollama VLM route、驗證 JSON field confidence/evidence refs，並只寫 artifact receipt；model_error 也必須寫 failure artifact receipt，receipt 檔內需自證 `artifact_write_performed=true` 與 `receipt_path`。此 worker 明確標記 `writes_database=false`、`writes_ai_insights=false`、`writes_price_tables=false`、`secret_read=false`、`primary_human_gate_count=0`；blocked page 不得輸出商品欄位，ready VLM 結果仍需 identity matcher replay 與 PromotionGate。 | | 2026-07-09 | PixelRAG VLM route readiness 必須有 runtime monitoring 與 execute 前自動選模 | V10.752 起 `/api/ai-automation/pixelrag-vlm-route-readiness`、`scripts/ops/report_pixelrag_vlm_route_readiness.py`、`/api/ai-automation/smoke` 與 `/api/ai-automation/scheduled-health-summary` 必須輸出 read-only approved Ollama route/model readiness / `pixelrag_vlm_route_readiness` family；readback 只打 `/api/tags`，不呼叫 `/api/generate`，輸出 reachable host、configured model available count、candidate model、candidate host、model_route_ready 與 next machine action。`run_pixelrag_vlm_replay_worker.py --execute` 預設必須使用此 readback 自動選擇已安裝候選模型；完全沒有候選時寫 `model_route_not_ready` artifact receipt，不得盲打 missing model。 | +| 2026-07-10 | PixelRAG VLM execute preflight 必須驗證 generate route | V10.753 起 `/api/ai-automation/pixelrag-vlm-route-readiness?probe_generate=true` 與 `scripts/ops/report_pixelrag_vlm_route_readiness.py --probe-generate` 可對已安裝候選模型執行極小 `/api/generate` preflight；smoke 預設仍不呼叫模型。`run_pixelrag_vlm_replay_worker.py --execute` 預設先執行 generate preflight，若 GCP-A direct / 110 proxy timeout、GCP-B candidate 雖安裝但 generate 不健康，worker 必須在送入 screenshot tiles 前寫 `model_route_not_ready` artifact receipt，輸出 `tag_model_route_ready`、`generate_route_ready`、`route_model_call_performed` 與 `tile_model_call_performed=false`，下一步固定為 `repair_ollama_vlm_generate_runtime_or_proxy_timeout`；不得把 tags 可見誤當 VLM 可執行。 | | 2026-06-29 | PChome DB apply 授權 lane 必須先通過 no-write guard / decision preflight / decision closeout / issuer gate / signing-decision preflight / signing-decision closeout / signing-issuer guard | V10.725 的 PChome mapping backlog auto-policy 已新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-lane-guard`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-decision-closeout`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-issuer-gate`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-preflight`、`/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-decision-closeout` 與 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-guard`；這些 endpoint 只驗證 final exact request package、same-run production truth requirement、secret rejection、rollback boundary、lane entry requirements、decision input requirements、rejection policy、post-apply verifier、future authorization decision package、final nonsecret authorization envelope、signing decision preflight inputs、unsigned signing decision package 與 signable request boundary，不讀 secret、不執行 shell/SQL、不寫 DB，也不簽發 database apply authorization。 | | 2026-06-29 | PChome DB apply 授權簽署發行者 lane 必須先產出 final signable request package | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-issuer-closeout`；此 endpoint 只把 signing-issuer guard 的 signable request boundary 收斂成 final signable request package 與 closeout contract，確認 fresh production truth、post-apply verifier、migration hash、secret boundary 與 no-side-effect checks，不讀 secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | | 2026-06-29 | PChome DB apply 授權簽署執行 lane 必須先通過 operator-held secret boundary preflight | V10.725 的 PChome mapping backlog auto-policy 新增 `/api/ai/pchome-growth/mapping-backlog/auto-policy-db-apply-authorization-signing-execution-preflight`；此 endpoint 只把 final signable request package 轉成 future signing execution preflight package、operator-held secret boundary contract、nonsecret signing inputs、command-shape preview、rollback boundary 與 abort conditions，不讀 secret、不接受 plaintext secret、不簽發 authorization、不執行 shell/SQL、不寫 DB，也不代表正式 DB apply 已授權。 | diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index a529702..1a90750 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -785,7 +785,16 @@ def ai_automation_pixelrag_vlm_replay_worker_api(): 'false', 'no', } + probe_generate_before_execute = ( + str(request.args.get('probe_generate_before_execute', 'true')).strip().lower() + not in { + '0', + 'false', + 'no', + } + ) route_readiness_timeout = request.args.get('route_readiness_timeout', 3, type=int) + route_generate_probe_timeout = request.args.get('route_generate_probe_timeout', 8, type=int) return jsonify(run_pixelrag_ollama_vlm_replay_worker( platform=platforms, max_age_hours=max(1, min(max_age_hours or 168, 720)), @@ -797,6 +806,8 @@ def ai_automation_pixelrag_vlm_replay_worker_api(): write_receipt=bool(execute and write_receipt), auto_select_model=auto_select_model, route_readiness_timeout=max(1, min(route_readiness_timeout or 3, 20)), + probe_generate_before_execute=probe_generate_before_execute, + route_generate_probe_timeout=max(1, min(route_generate_probe_timeout or 8, 30)), )) @@ -813,11 +824,19 @@ def ai_automation_pixelrag_vlm_route_readiness_api(): 'true', 'yes', } + probe_generate = str(request.args.get('probe_generate') or '').strip().lower() in { + '1', + 'true', + 'yes', + } timeout = request.args.get('timeout', 3, type=int) + probe_timeout = request.args.get('probe_timeout', 8, type=int) return jsonify(build_pixelrag_vlm_route_readiness( model=str(request.args.get('model') or '').strip() or None, timeout_seconds=max(1, min(timeout or 3, 20)), include_models=include_models, + probe_generate=probe_generate, + probe_timeout_seconds=max(1, min(probe_timeout or 8, 30)), )) diff --git a/scripts/ops/report_pixelrag_vlm_route_readiness.py b/scripts/ops/report_pixelrag_vlm_route_readiness.py index c7b9b10..fa7b6fa 100644 --- a/scripts/ops/report_pixelrag_vlm_route_readiness.py +++ b/scripts/ops/report_pixelrag_vlm_route_readiness.py @@ -34,12 +34,25 @@ def main() -> int: action="store_true", help="輸出每個 host 的 model list。", ) + parser.add_argument( + "--probe-generate", + action="store_true", + help="執行極小 /api/generate preflight；預設不呼叫模型。", + ) + parser.add_argument( + "--probe-timeout", + type=int, + default=8, + help="/api/generate preflight timeout 秒數。", + ) args = parser.parse_args() payload = build_pixelrag_vlm_route_readiness( model=args.model, timeout_seconds=args.timeout, include_models=args.include_models, + probe_generate=args.probe_generate, + probe_timeout_seconds=args.probe_timeout, ) print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) return 0 if payload.get("success") else 1 diff --git a/scripts/ops/run_pixelrag_vlm_replay_worker.py b/scripts/ops/run_pixelrag_vlm_replay_worker.py index ab78f94..a412bd5 100755 --- a/scripts/ops/run_pixelrag_vlm_replay_worker.py +++ b/scripts/ops/run_pixelrag_vlm_replay_worker.py @@ -87,6 +87,17 @@ def main() -> int: default=3, help="auto-select model 的 /api/tags readiness timeout 秒數。", ) + parser.add_argument( + "--no-probe-generate-before-execute", + action="store_true", + help="停用 execute 前的極小 /api/generate preflight。", + ) + parser.add_argument( + "--route-generate-probe-timeout", + type=int, + default=8, + help="execute 前 /api/generate preflight timeout 秒數。", + ) args = parser.parse_args() payload = run_pixelrag_ollama_vlm_replay_worker( @@ -102,6 +113,8 @@ def main() -> int: write_receipt=bool(args.write_receipt and args.execute), auto_select_model=not args.no_auto_select_model, route_readiness_timeout=args.route_readiness_timeout, + probe_generate_before_execute=not args.no_probe_generate_before_execute, + route_generate_probe_timeout=args.route_generate_probe_timeout, ) print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) return 0 if payload.get("success") else 1 diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index c8ffcb4..daa4980 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -4665,9 +4665,24 @@ def build_scheduled_automation_health_summary( "candidate_model": pixelrag_vlm_route_readiness_details.get("candidate_model"), "candidate_host": pixelrag_vlm_route_readiness_details.get("candidate_host"), "candidate_provider": pixelrag_vlm_route_readiness_details.get("candidate_provider"), + "tag_model_route_ready": bool( + pixelrag_vlm_route_readiness_details.get("tag_model_route_ready") + ), "model_route_ready": bool( pixelrag_vlm_route_readiness_details.get("model_route_ready") ), + "generate_probe_performed": bool( + pixelrag_vlm_route_readiness_details.get("generate_probe_performed") + ), + "generate_probe_ok_count": int( + pixelrag_vlm_route_readiness_details.get("generate_probe_ok_count") or 0 + ), + "generate_route_ready": bool( + pixelrag_vlm_route_readiness_details.get("generate_route_ready") + ), + "generate_ready_model": pixelrag_vlm_route_readiness_details.get("generate_ready_model"), + "generate_ready_host": pixelrag_vlm_route_readiness_details.get("generate_ready_host"), + "generate_ready_provider": pixelrag_vlm_route_readiness_details.get("generate_ready_provider"), "model_call_performed": bool( pixelrag_vlm_route_readiness_details.get("model_call_performed") ), @@ -13524,7 +13539,14 @@ def _pixelrag_vlm_route_readiness_check() -> Dict[str, Any]: "candidate_model": candidate_model, "candidate_host": summary.get("candidate_host"), "candidate_provider": summary.get("candidate_provider"), + "tag_model_route_ready": bool(summary.get("tag_model_route_ready")), "model_route_ready": bool(summary.get("model_route_ready")), + "generate_probe_performed": bool(summary.get("generate_probe_performed")), + "generate_probe_ok_count": int(summary.get("generate_probe_ok_count") or 0), + "generate_route_ready": bool(summary.get("generate_route_ready")), + "generate_ready_model": summary.get("generate_ready_model"), + "generate_ready_host": summary.get("generate_ready_host"), + "generate_ready_provider": summary.get("generate_ready_provider"), "model_call_performed": bool(summary.get("model_call_performed")), "next_machine_action": readback.get("next_machine_action"), "writes_database": False, diff --git a/services/pixelrag_vlm_replay_worker_service.py b/services/pixelrag_vlm_replay_worker_service.py index 853e765..aee70ea 100644 --- a/services/pixelrag_vlm_replay_worker_service.py +++ b/services/pixelrag_vlm_replay_worker_service.py @@ -34,6 +34,7 @@ POLICY = "controlled_pixelrag_ollama_vlm_replay_worker_v1" DEFAULT_LIMIT = 25 DEFAULT_TILE_LIMIT = 4 DEFAULT_TIMEOUT_SECONDS = 90 +DEFAULT_ROUTE_GENERATE_PROBE_TIMEOUT_SECONDS = 8 DEFAULT_OUTPUT_ROOT = os.getenv( "PIXELRAG_VLM_REPLAY_RECEIPT_ROOT", "/app/data/ai_automation/pixelrag_vlm_replay_receipts" @@ -293,6 +294,13 @@ def _model_route_not_ready_item( "model": summary.get("configured_model"), "candidate_model": summary.get("candidate_model"), "candidate_host": summary.get("candidate_host"), + "tag_model_route_ready": bool(summary.get("tag_model_route_ready")), + "generate_probe_performed": bool(summary.get("generate_probe_performed")), + "generate_probe_ok_count": int(summary.get("generate_probe_ok_count") or 0), + "generate_route_ready": bool(summary.get("generate_route_ready")), + "generate_ready_model": summary.get("generate_ready_model"), + "generate_ready_host": summary.get("generate_ready_host"), + "generate_ready_provider": summary.get("generate_ready_provider"), "model_call_performed": False, "artifact_write_performed": False, "writes_database": False, @@ -435,6 +443,8 @@ def run_pixelrag_ollama_vlm_replay_worker( write_receipt: bool = False, auto_select_model: bool = True, route_readiness_timeout: int | None = None, + probe_generate_before_execute: bool = True, + route_generate_probe_timeout: int | None = None, ) -> dict[str, Any]: """Run or dry-run the PixelRAG VLM replay worker.""" root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT) @@ -446,6 +456,16 @@ def run_pixelrag_ollama_vlm_replay_worker( selected_model = str(model or DEFAULT_MODEL) selected_timeout = max(10, int(timeout or DEFAULT_TIMEOUT_SECONDS)) readiness_timeout = max(1, min(int(route_readiness_timeout or 3), 20)) + generate_probe_timeout = max( + 1, + min( + int( + route_generate_probe_timeout + or DEFAULT_ROUTE_GENERATE_PROBE_TIMEOUT_SECONDS + ), + 30, + ), + ) generated_at = datetime.now(timezone.utc).isoformat() route_readiness: dict[str, Any] | None = None model_route_ready = True @@ -461,6 +481,8 @@ def run_pixelrag_ollama_vlm_replay_worker( route_readiness = build_pixelrag_vlm_route_readiness( model=selected_model, timeout_seconds=readiness_timeout, + probe_generate=bool(probe_generate_before_execute), + probe_timeout_seconds=generate_probe_timeout, ) route_summary = route_readiness.get("summary") or {} candidate_model = str(route_summary.get("candidate_model") or "").strip() @@ -509,7 +531,19 @@ def run_pixelrag_ollama_vlm_replay_worker( int(item.get("required_field_missing_count") or 0) for item in worker_items ) - model_call_performed = any(bool(item.get("model_call_performed")) for item in worker_items) + tile_model_call_performed = any( + bool(item.get("model_call_performed")) for item in worker_items + ) + route_model_call_performed = bool( + route_readiness + and ( + (route_readiness.get("controlled_apply") or {}).get("model_call") + or (route_readiness.get("summary") or {}).get("model_call_performed") + ) + ) + model_call_performed = bool( + tile_model_call_performed or route_model_call_performed + ) artifact_write_performed = any(bool(item.get("artifact_write_performed")) for item in worker_items) if parse_error_count or model_error_count or route_not_ready_count or no_tile_count: @@ -524,7 +558,11 @@ def run_pixelrag_ollama_vlm_replay_worker( elif not execute and ready_count: next_action = "run_pixelrag_vlm_replay_worker_execute" elif route_not_ready_count: - next_action = "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host" + next_action = ( + route_readiness.get("next_machine_action") + if route_readiness + else None + ) or "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host" elif model_error_count or parse_error_count: next_action = "repair_ollama_vlm_runtime_or_model_route" elif executed_warning_count: @@ -548,6 +586,8 @@ def run_pixelrag_ollama_vlm_replay_worker( "no_tile_count": no_tile_count, "receipt_written_count": receipt_written_count, "required_field_missing_count": required_missing_count, + "route_model_call_performed": route_model_call_performed, + "tile_model_call_performed": tile_model_call_performed, "model_call_performed": model_call_performed, "artifact_write_performed": artifact_write_performed, "writes_database_count": 0, @@ -572,6 +612,8 @@ def run_pixelrag_ollama_vlm_replay_worker( "write_receipt": bool(write_receipt), "auto_select_model": bool(auto_select_model), "route_readiness_timeout_seconds": readiness_timeout, + "probe_generate_before_execute": bool(probe_generate_before_execute), + "route_generate_probe_timeout_seconds": generate_probe_timeout, "summary": summary, "worker_items": worker_items, "route_readiness": ( @@ -591,7 +633,7 @@ def run_pixelrag_ollama_vlm_replay_worker( "next_machine_action": contract.get("next_machine_action"), }, "controlled_apply": { - "network_call": bool(execute and model_call_performed), + "network_call": bool(execute and (route_readiness or model_call_performed)), "model_call": bool(execute and model_call_performed), "artifact_write": artifact_write_performed, "db_write": False, diff --git a/services/pixelrag_vlm_route_readiness_service.py b/services/pixelrag_vlm_route_readiness_service.py index 9cc4acf..aeec0fe 100644 --- a/services/pixelrag_vlm_route_readiness_service.py +++ b/services/pixelrag_vlm_route_readiness_service.py @@ -1,14 +1,15 @@ -"""Read-only PixelRAG VLM route readiness. +"""PixelRAG VLM route readiness. This module checks approved Ollama routes for installed VLM candidate models. -It does not call /api/generate, read secrets, write DB data, or change runtime -configuration. The worker can use the result to avoid blindly executing a -missing configured model. +By default it only reads /api/tags. Execute preflight can opt into a tiny +/api/generate probe to avoid blindly sending screenshot tiles to a route that +is installed but cannot generate. """ from __future__ import annotations import os +import time from datetime import datetime, timezone from typing import Any @@ -27,6 +28,8 @@ from services.ollama_service import ( POLICY = "read_only_pixelrag_vlm_route_readiness_v1" DEFAULT_TIMEOUT_SECONDS = 3 +DEFAULT_PROBE_TIMEOUT_SECONDS = 8 +DEFAULT_PROBE_PROMPT = "Return exactly OK." DEFAULT_MODEL = ( os.getenv("PIXELRAG_VLM_MODEL") or os.getenv("PPT_VISION_MODEL") @@ -103,15 +106,83 @@ def _fetch_models(host: str, timeout: int) -> dict[str, Any]: return item +def _probe_generate( + host: str, + *, + model: str, + timeout: int, + prompt: str, +) -> dict[str, Any]: + clean_host = str(host or "").rstrip("/") + started = time.monotonic() + result: dict[str, Any] = { + "generate_probe_performed": True, + "generate_probe_model": str(model or "").strip(), + "generate_probe_ok": False, + "generate_probe_error": "", + "generate_probe_duration_ms": 0, + } + payload = { + "model": str(model or "").strip(), + "prompt": str(prompt or DEFAULT_PROBE_PROMPT), + "stream": False, + "options": { + "temperature": 0, + "num_predict": 8, + "num_ctx": 512, + }, + "keep_alive": "1m", + } + try: + response = requests.post( + f"{clean_host}/api/generate", + json=payload, + timeout=max(1, timeout), + ) + result["generate_probe_http_status"] = response.status_code + if response.status_code != 200: + result["generate_probe_error"] = ( + f"HTTP {response.status_code}: {response.text[:180]}" + ) + return result + try: + body = response.json() + except Exception as exc: + result["generate_probe_error"] = f"invalid_json: {type(exc).__name__}" + return result + result["generate_probe_ok"] = ( + isinstance(body, dict) + and ( + "response" in body + or bool(body.get("done")) + or bool(body.get("context")) + ) + ) + if not result["generate_probe_ok"]: + result["generate_probe_error"] = "generate_response_missing_output_fields" + except Exception as exc: + result["generate_probe_error"] = f"{type(exc).__name__}: {str(exc)[:180]}" + finally: + result["generate_probe_duration_ms"] = int((time.monotonic() - started) * 1000) + return result + + def build_pixelrag_vlm_route_readiness( *, model: str | None = None, timeout_seconds: int | None = None, include_models: bool = False, + probe_generate: bool = False, + probe_timeout_seconds: int | None = None, + probe_prompt: str | None = None, ) -> dict[str, Any]: """Read approved Ollama tags and recommend an installed VLM model.""" configured_model = str(model or DEFAULT_MODEL).strip() or DEFAULT_MODEL timeout = max(1, min(int(timeout_seconds or DEFAULT_TIMEOUT_SECONDS), 20)) + probe_timeout = max( + 1, + min(int(probe_timeout_seconds or DEFAULT_PROBE_TIMEOUT_SECONDS), 30), + ) generated_at = datetime.now(timezone.utc).isoformat() candidates = _candidate_models(configured_model) host_results = [_fetch_models(host, timeout) for host in _approved_hosts()] @@ -141,13 +212,48 @@ def build_pixelrag_vlm_route_readiness( if selected_model: break - model_route_ready = bool(selected_model and selected_host) + tag_model_route_ready = bool(selected_model and selected_host) + generate_probe_performed = bool(probe_generate and tag_model_route_ready) + generate_probe_ok_count = 0 + generate_ready_model = "" + generate_ready_host = "" + generate_ready_provider = "" + generate_route_ready: bool | None = None + if generate_probe_performed: + generate_route_ready = False + for host in reachable_hosts: + if selected_model not in set(host.get("models") or []): + continue + probe_result = _probe_generate( + str(host.get("host") or ""), + model=selected_model, + timeout=probe_timeout, + prompt=probe_prompt or DEFAULT_PROBE_PROMPT, + ) + host.update(probe_result) + if probe_result.get("generate_probe_ok"): + generate_probe_ok_count += 1 + if not generate_ready_host: + generate_ready_model = selected_model + generate_ready_host = str(host.get("host") or "") + generate_ready_provider = str(host.get("provider") or "") + if generate_ready_host: + selected_host = generate_ready_host + selected_provider = generate_ready_provider + generate_route_ready = True + + model_route_ready = tag_model_route_ready and ( + bool(generate_route_ready) if generate_probe_performed else True + ) if not reachable_hosts: status = "critical" next_action = "repair_approved_ollama_host_connectivity" - elif not model_route_ready: + elif not tag_model_route_ready: status = "critical" next_action = "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host" + elif generate_probe_performed and not generate_route_ready: + status = "critical" + next_action = "repair_ollama_vlm_generate_runtime_or_proxy_timeout" elif selected_model != configured_model: status = "warning" next_action = "run_pixelrag_vlm_replay_worker_execute_with_selected_model" @@ -163,6 +269,11 @@ def build_pixelrag_vlm_route_readiness( candidate for candidate in candidates if candidate in set(item.get("models") or []) ] + result.setdefault("generate_probe_performed", False) + result.setdefault("generate_probe_model", "") + result.setdefault("generate_probe_ok", False) + result.setdefault("generate_probe_error", "") + result.setdefault("generate_probe_duration_ms", 0) if not include_models: result.pop("models", None) public_host_results.append(result) @@ -176,9 +287,15 @@ def build_pixelrag_vlm_route_readiness( "candidate_host": selected_host, "candidate_provider": selected_provider, "candidate_selection_reason": selected_reason, + "tag_model_route_ready": tag_model_route_ready, "model_route_ready": model_route_ready, - "generate_probe_performed": False, - "model_call_performed": False, + "generate_probe_performed": generate_probe_performed, + "generate_probe_ok_count": generate_probe_ok_count, + "generate_ready_model": generate_ready_model, + "generate_ready_host": generate_ready_host, + "generate_ready_provider": generate_ready_provider, + "generate_route_ready": generate_route_ready, + "model_call_performed": generate_probe_performed, "writes_database_count": 0, "primary_human_gate_count": 0, } @@ -188,13 +305,14 @@ def build_pixelrag_vlm_route_readiness( "status": status, "generated_at": generated_at, "timeout_seconds": timeout, + "probe_timeout_seconds": probe_timeout if probe_generate else 0, "configured_model": configured_model, "candidate_models": candidates, "summary": summary, "hosts": public_host_results, "controlled_apply": { "network_call": True, - "model_call": False, + "model_call": generate_probe_performed, "db_write": False, "writes_database": False, "writes_database_count": 0, diff --git a/tests/test_pixelrag_vlm_replay_worker_service.py b/tests/test_pixelrag_vlm_replay_worker_service.py index ea0f66c..63c6d64 100644 --- a/tests/test_pixelrag_vlm_replay_worker_service.py +++ b/tests/test_pixelrag_vlm_replay_worker_service.py @@ -257,6 +257,83 @@ def test_pixelrag_vlm_replay_worker_auto_selects_installed_candidate(tmp_path, m assert payload["summary"]["executed_ok_count"] == 1 +def test_pixelrag_vlm_replay_worker_generate_probe_blocks_tile_generate(tmp_path, monkeypatch): + from services import pixelrag_vlm_replay_worker_service as service + + _write_receipt( + tmp_path, + platform="shopee_tw", + manifest_id="shopee-ok", + title="Shopee 防曬乳", + url="https://shopee.tw/search?keyword=sunscreen", + ) + + monkeypatch.setattr( + service, + "build_pixelrag_vlm_route_readiness", + lambda **kwargs: { + "policy": "read_only_pixelrag_vlm_route_readiness_v1", + "status": "critical", + "summary": { + "configured_model": "minicpm-v:latest", + "candidate_model": "gemma3:4b", + "candidate_host": "http://34.21.145.224:11434", + "tag_model_route_ready": True, + "model_route_ready": False, + "generate_probe_performed": True, + "generate_probe_ok_count": 0, + "generate_route_ready": False, + "generate_ready_model": "", + "generate_ready_host": "", + "generate_ready_provider": "", + "model_call_performed": True, + }, + "controlled_apply": {"model_call": True}, + "next_machine_action": "repair_ollama_vlm_generate_runtime_or_proxy_timeout", + }, + ) + + class FakeOllama: + def __init__(self, model): + self.model = model + + def generate(self, *args, **kwargs): + raise AssertionError("tile VLM generate should not be called") + + monkeypatch.setattr(service, "OllamaService", FakeOllama) + payload = service.run_pixelrag_ollama_vlm_replay_worker( + artifact_root=tmp_path, + output_root=tmp_path / "receipts", + platform="shopee_tw", + model="minicpm-v:latest", + execute=True, + write_receipt=True, + tile_limit=1, + auto_select_model=True, + probe_generate_before_execute=True, + ) + + assert payload["status"] == "critical" + assert payload["model"] == "gemma3:4b" + assert payload["summary"]["model_route_not_ready_count"] == 1 + assert payload["summary"]["route_model_call_performed"] is True + assert payload["summary"]["tile_model_call_performed"] is False + assert payload["summary"]["model_call_performed"] is True + assert payload["controlled_apply"]["model_call"] is True + assert payload["next_machine_action"] == "repair_ollama_vlm_generate_runtime_or_proxy_timeout" + item = payload["worker_items"][0] + assert item["worker_status"] == "model_route_not_ready" + assert item["generate_probe_performed"] is True + assert item["generate_route_ready"] is False + assert item["model_call_performed"] is False + receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-ok" / "vlm_replay_receipt.json" + assert receipt_path.exists() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["worker_status"] == "model_route_not_ready" + assert receipt["artifact_write_performed"] is True + assert receipt["generate_route_ready"] is False + + def test_pixelrag_vlm_replay_worker_cli_outputs_machine_readable_json(tmp_path): _write_receipt( tmp_path, diff --git a/tests/test_pixelrag_vlm_route_readiness_service.py b/tests/test_pixelrag_vlm_route_readiness_service.py index c24cfd9..b0abfb0 100644 --- a/tests/test_pixelrag_vlm_route_readiness_service.py +++ b/tests/test_pixelrag_vlm_route_readiness_service.py @@ -67,6 +67,70 @@ def test_pixelrag_vlm_route_readiness_critical_when_no_candidate(monkeypatch): assert payload["next_machine_action"] == "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host" +def test_pixelrag_vlm_route_readiness_generate_probe_success(monkeypatch): + from services import pixelrag_vlm_route_readiness_service as service + + monkeypatch.setattr(service, "_approved_hosts", lambda: ["http://secondary:11434"]) + monkeypatch.setattr( + service.requests, + "get", + lambda url, timeout: FakeResponse(payload={"models": [{"name": "gemma3:4b"}]}), + ) + monkeypatch.setattr( + service.requests, + "post", + lambda url, json, timeout: FakeResponse(payload={"response": "OK", "done": True}), + ) + + payload = service.build_pixelrag_vlm_route_readiness( + model="minicpm-v:latest", + probe_generate=True, + probe_timeout_seconds=1, + ) + + assert payload["status"] == "warning" + assert payload["summary"]["candidate_model"] == "gemma3:4b" + assert payload["summary"]["tag_model_route_ready"] is True + assert payload["summary"]["model_route_ready"] is True + assert payload["summary"]["generate_probe_performed"] is True + assert payload["summary"]["generate_probe_ok_count"] == 1 + assert payload["summary"]["generate_route_ready"] is True + assert payload["summary"]["generate_ready_host"] == "http://secondary:11434" + assert payload["controlled_apply"]["model_call"] is True + + +def test_pixelrag_vlm_route_readiness_generate_probe_failure_blocks_route(monkeypatch): + from services import pixelrag_vlm_route_readiness_service as service + + monkeypatch.setattr(service, "_approved_hosts", lambda: ["http://secondary:11434"]) + monkeypatch.setattr( + service.requests, + "get", + lambda url, timeout: FakeResponse(payload={"models": [{"name": "gemma3:4b"}]}), + ) + + def fake_post(url, json, timeout): + raise TimeoutError("generate timeout") + + monkeypatch.setattr(service.requests, "post", fake_post) + + payload = service.build_pixelrag_vlm_route_readiness( + model="minicpm-v:latest", + probe_generate=True, + probe_timeout_seconds=1, + ) + + assert payload["status"] == "critical" + assert payload["success"] is False + assert payload["summary"]["candidate_model"] == "gemma3:4b" + assert payload["summary"]["tag_model_route_ready"] is True + assert payload["summary"]["model_route_ready"] is False + assert payload["summary"]["generate_probe_performed"] is True + assert payload["summary"]["generate_probe_ok_count"] == 0 + assert payload["summary"]["generate_route_ready"] is False + assert payload["next_machine_action"] == "repair_ollama_vlm_generate_runtime_or_proxy_timeout" + + def test_pixelrag_vlm_route_readiness_route_returns_readback(monkeypatch): from flask import Flask from routes import system_public_routes as routes @@ -87,6 +151,7 @@ def test_pixelrag_vlm_route_readiness_route_returns_readback(monkeypatch): app = Flask(__name__) with app.test_request_context( "/api/ai-automation/pixelrag-vlm-route-readiness?model=minicpm-v:latest" + "&probe_generate=true&probe_timeout=4" ): response = routes.ai_automation_pixelrag_vlm_route_readiness_api.__wrapped__() payload = response.get_json()