wooo/ewoooc
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(ai): add PixelRAG VLM generate preflight

Browse Source
This commit is contained in:
ogt
2026-07-10 00:17:02 +08:00
parent d1ca067054
commit bed3a8a8ee
11 changed files with 384 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
services/ai_automation_smoke_service.py
View File

@@ -4665,9 +4665,24 @@ def build_scheduled_automation_health_summary(
"candidate_model": pixelrag_vlm_route_readiness_details.get("candidate_model"),
"candidate_host": pixelrag_vlm_route_readiness_details.get("candidate_host"),
"candidate_provider": pixelrag_vlm_route_readiness_details.get("candidate_provider"),
"tag_model_route_ready": bool(
pixelrag_vlm_route_readiness_details.get("tag_model_route_ready")
),
"model_route_ready": bool(
pixelrag_vlm_route_readiness_details.get("model_route_ready")
),
"generate_probe_performed": bool(
pixelrag_vlm_route_readiness_details.get("generate_probe_performed")
),
"generate_probe_ok_count": int(
pixelrag_vlm_route_readiness_details.get("generate_probe_ok_count") or 0
),
"generate_route_ready": bool(
pixelrag_vlm_route_readiness_details.get("generate_route_ready")
),
"generate_ready_model": pixelrag_vlm_route_readiness_details.get("generate_ready_model"),
"generate_ready_host": pixelrag_vlm_route_readiness_details.get("generate_ready_host"),
"generate_ready_provider": pixelrag_vlm_route_readiness_details.get("generate_ready_provider"),
"model_call_performed": bool(
pixelrag_vlm_route_readiness_details.get("model_call_performed")
),
@@ -13524,7 +13539,14 @@ def _pixelrag_vlm_route_readiness_check() -> Dict[str, Any]:
"candidate_model": candidate_model,
"candidate_host": summary.get("candidate_host"),
"candidate_provider": summary.get("candidate_provider"),
"tag_model_route_ready": bool(summary.get("tag_model_route_ready")),
"model_route_ready": bool(summary.get("model_route_ready")),
"generate_probe_performed": bool(summary.get("generate_probe_performed")),
"generate_probe_ok_count": int(summary.get("generate_probe_ok_count") or 0),
"generate_route_ready": bool(summary.get("generate_route_ready")),
"generate_ready_model": summary.get("generate_ready_model"),
"generate_ready_host": summary.get("generate_ready_host"),
"generate_ready_provider": summary.get("generate_ready_provider"),
"model_call_performed": bool(summary.get("model_call_performed")),
"next_machine_action": readback.get("next_machine_action"),
"writes_database": False,

48
services/pixelrag_vlm_replay_worker_service.py
View File

@@ -34,6 +34,7 @@ POLICY = "controlled_pixelrag_ollama_vlm_replay_worker_v1"
DEFAULT_LIMIT = 25
DEFAULT_TILE_LIMIT = 4
DEFAULT_TIMEOUT_SECONDS = 90
DEFAULT_ROUTE_GENERATE_PROBE_TIMEOUT_SECONDS = 8
DEFAULT_OUTPUT_ROOT = os.getenv(
"PIXELRAG_VLM_REPLAY_RECEIPT_ROOT",
"/app/data/ai_automation/pixelrag_vlm_replay_receipts"
@@ -293,6 +294,13 @@ def _model_route_not_ready_item(
"model": summary.get("configured_model"),
"candidate_model": summary.get("candidate_model"),
"candidate_host": summary.get("candidate_host"),
"tag_model_route_ready": bool(summary.get("tag_model_route_ready")),
"generate_probe_performed": bool(summary.get("generate_probe_performed")),
"generate_probe_ok_count": int(summary.get("generate_probe_ok_count") or 0),
"generate_route_ready": bool(summary.get("generate_route_ready")),
"generate_ready_model": summary.get("generate_ready_model"),
"generate_ready_host": summary.get("generate_ready_host"),
"generate_ready_provider": summary.get("generate_ready_provider"),
"model_call_performed": False,
"artifact_write_performed": False,
"writes_database": False,
@@ -435,6 +443,8 @@ def run_pixelrag_ollama_vlm_replay_worker(
write_receipt: bool = False,
auto_select_model: bool = True,
route_readiness_timeout: int | None = None,
probe_generate_before_execute: bool = True,
route_generate_probe_timeout: int | None = None,
) -> dict[str, Any]:
"""Run or dry-run the PixelRAG VLM replay worker."""
root = Path(artifact_root or DEFAULT_ARTIFACT_ROOT)
@@ -446,6 +456,16 @@ def run_pixelrag_ollama_vlm_replay_worker(
selected_model = str(model or DEFAULT_MODEL)
selected_timeout = max(10, int(timeout or DEFAULT_TIMEOUT_SECONDS))
readiness_timeout = max(1, min(int(route_readiness_timeout or 3), 20))
generate_probe_timeout = max(
1,
min(
int(
route_generate_probe_timeout
or DEFAULT_ROUTE_GENERATE_PROBE_TIMEOUT_SECONDS
),
30,
),
)
generated_at = datetime.now(timezone.utc).isoformat()
route_readiness: dict[str, Any] | None = None
model_route_ready = True
@@ -461,6 +481,8 @@ def run_pixelrag_ollama_vlm_replay_worker(
route_readiness = build_pixelrag_vlm_route_readiness(
model=selected_model,
timeout_seconds=readiness_timeout,
probe_generate=bool(probe_generate_before_execute),
probe_timeout_seconds=generate_probe_timeout,
)
route_summary = route_readiness.get("summary") or {}
candidate_model = str(route_summary.get("candidate_model") or "").strip()
@@ -509,7 +531,19 @@ def run_pixelrag_ollama_vlm_replay_worker(
int(item.get("required_field_missing_count") or 0)
for item in worker_items
)
model_call_performed = any(bool(item.get("model_call_performed")) for item in worker_items)
tile_model_call_performed = any(
bool(item.get("model_call_performed")) for item in worker_items
)
route_model_call_performed = bool(
route_readiness
and (
(route_readiness.get("controlled_apply") or {}).get("model_call")
or (route_readiness.get("summary") or {}).get("model_call_performed")
)
)
model_call_performed = bool(
tile_model_call_performed or route_model_call_performed
)
artifact_write_performed = any(bool(item.get("artifact_write_performed")) for item in worker_items)
if parse_error_count or model_error_count or route_not_ready_count or no_tile_count:
@@ -524,7 +558,11 @@ def run_pixelrag_ollama_vlm_replay_worker(
elif not execute and ready_count:
next_action = "run_pixelrag_vlm_replay_worker_execute"
elif route_not_ready_count:
next_action = "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host"
next_action = (
route_readiness.get("next_machine_action")
if route_readiness
else None
) or "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host"
elif model_error_count or parse_error_count:
next_action = "repair_ollama_vlm_runtime_or_model_route"
elif executed_warning_count:
@@ -548,6 +586,8 @@ def run_pixelrag_ollama_vlm_replay_worker(
"no_tile_count": no_tile_count,
"receipt_written_count": receipt_written_count,
"required_field_missing_count": required_missing_count,
"route_model_call_performed": route_model_call_performed,
"tile_model_call_performed": tile_model_call_performed,
"model_call_performed": model_call_performed,
"artifact_write_performed": artifact_write_performed,
"writes_database_count": 0,
@@ -572,6 +612,8 @@ def run_pixelrag_ollama_vlm_replay_worker(
"write_receipt": bool(write_receipt),
"auto_select_model": bool(auto_select_model),
"route_readiness_timeout_seconds": readiness_timeout,
"probe_generate_before_execute": bool(probe_generate_before_execute),
"route_generate_probe_timeout_seconds": generate_probe_timeout,
"summary": summary,
"worker_items": worker_items,
"route_readiness": (
@@ -591,7 +633,7 @@ def run_pixelrag_ollama_vlm_replay_worker(
"next_machine_action": contract.get("next_machine_action"),
},
"controlled_apply": {
"network_call": bool(execute and model_call_performed),
"network_call": bool(execute and (route_readiness or model_call_performed)),
"model_call": bool(execute and model_call_performed),
"artifact_write": artifact_write_performed,
"db_write": False,

136
services/pixelrag_vlm_route_readiness_service.py
View File

@@ -1,14 +1,15 @@
"""Read-only PixelRAG VLM route readiness.
"""PixelRAG VLM route readiness.
This module checks approved Ollama routes for installed VLM candidate models.
It does not call /api/generate, read secrets, write DB data, or change runtime
configuration. The worker can use the result to avoid blindly executing a
missing configured model.
By default it only reads /api/tags. Execute preflight can opt into a tiny
/api/generate probe to avoid blindly sending screenshot tiles to a route that
is installed but cannot generate.
"""
from __future__ import annotations
import os
import time
from datetime import datetime, timezone
from typing import Any
@@ -27,6 +28,8 @@ from services.ollama_service import (
POLICY = "read_only_pixelrag_vlm_route_readiness_v1"
DEFAULT_TIMEOUT_SECONDS = 3
DEFAULT_PROBE_TIMEOUT_SECONDS = 8
DEFAULT_PROBE_PROMPT = "Return exactly OK."
DEFAULT_MODEL = (
os.getenv("PIXELRAG_VLM_MODEL")
or os.getenv("PPT_VISION_MODEL")
@@ -103,15 +106,83 @@ def _fetch_models(host: str, timeout: int) -> dict[str, Any]:
return item
def _probe_generate(
host: str,
*,
model: str,
timeout: int,
prompt: str,
) -> dict[str, Any]:
clean_host = str(host or "").rstrip("/")
started = time.monotonic()
result: dict[str, Any] = {
"generate_probe_performed": True,
"generate_probe_model": str(model or "").strip(),
"generate_probe_ok": False,
"generate_probe_error": "",
"generate_probe_duration_ms": 0,
}
payload = {
"model": str(model or "").strip(),
"prompt": str(prompt or DEFAULT_PROBE_PROMPT),
"stream": False,
"options": {
"temperature": 0,
"num_predict": 8,
"num_ctx": 512,
},
"keep_alive": "1m",
}
try:
response = requests.post(
f"{clean_host}/api/generate",
json=payload,
timeout=max(1, timeout),
)
result["generate_probe_http_status"] = response.status_code
if response.status_code != 200:
result["generate_probe_error"] = (
f"HTTP {response.status_code}: {response.text[:180]}"
)
return result
try:
body = response.json()
except Exception as exc:
result["generate_probe_error"] = f"invalid_json: {type(exc).__name__}"
return result
result["generate_probe_ok"] = (
isinstance(body, dict)
and (
"response" in body
or bool(body.get("done"))
or bool(body.get("context"))
)
)
if not result["generate_probe_ok"]:
result["generate_probe_error"] = "generate_response_missing_output_fields"
except Exception as exc:
result["generate_probe_error"] = f"{type(exc).__name__}: {str(exc)[:180]}"
finally:
result["generate_probe_duration_ms"] = int((time.monotonic() - started) * 1000)
return result
def build_pixelrag_vlm_route_readiness(
*,
model: str | None = None,
timeout_seconds: int | None = None,
include_models: bool = False,
probe_generate: bool = False,
probe_timeout_seconds: int | None = None,
probe_prompt: str | None = None,
) -> dict[str, Any]:
"""Read approved Ollama tags and recommend an installed VLM model."""
configured_model = str(model or DEFAULT_MODEL).strip() or DEFAULT_MODEL
timeout = max(1, min(int(timeout_seconds or DEFAULT_TIMEOUT_SECONDS), 20))
probe_timeout = max(
1,
min(int(probe_timeout_seconds or DEFAULT_PROBE_TIMEOUT_SECONDS), 30),
)
generated_at = datetime.now(timezone.utc).isoformat()
candidates = _candidate_models(configured_model)
host_results = [_fetch_models(host, timeout) for host in _approved_hosts()]
@@ -141,13 +212,48 @@ def build_pixelrag_vlm_route_readiness(
if selected_model:
break
model_route_ready = bool(selected_model and selected_host)
tag_model_route_ready = bool(selected_model and selected_host)
generate_probe_performed = bool(probe_generate and tag_model_route_ready)
generate_probe_ok_count = 0
generate_ready_model = ""
generate_ready_host = ""
generate_ready_provider = ""
generate_route_ready: bool | None = None
if generate_probe_performed:
generate_route_ready = False
for host in reachable_hosts:
if selected_model not in set(host.get("models") or []):
continue
probe_result = _probe_generate(
str(host.get("host") or ""),
model=selected_model,
timeout=probe_timeout,
prompt=probe_prompt or DEFAULT_PROBE_PROMPT,
)
host.update(probe_result)
if probe_result.get("generate_probe_ok"):
generate_probe_ok_count += 1
if not generate_ready_host:
generate_ready_model = selected_model
generate_ready_host = str(host.get("host") or "")
generate_ready_provider = str(host.get("provider") or "")
if generate_ready_host:
selected_host = generate_ready_host
selected_provider = generate_ready_provider
generate_route_ready = True
model_route_ready = tag_model_route_ready and (
bool(generate_route_ready) if generate_probe_performed else True
)
if not reachable_hosts:
status = "critical"
next_action = "repair_approved_ollama_host_connectivity"
elif not model_route_ready:
elif not tag_model_route_ready:
status = "critical"
next_action = "install_or_configure_pixelrag_vlm_model_on_approved_ollama_host"
elif generate_probe_performed and not generate_route_ready:
status = "critical"
next_action = "repair_ollama_vlm_generate_runtime_or_proxy_timeout"
elif selected_model != configured_model:
status = "warning"
next_action = "run_pixelrag_vlm_replay_worker_execute_with_selected_model"
@@ -163,6 +269,11 @@ def build_pixelrag_vlm_route_readiness(
candidate for candidate in candidates
if candidate in set(item.get("models") or [])
]
result.setdefault("generate_probe_performed", False)
result.setdefault("generate_probe_model", "")
result.setdefault("generate_probe_ok", False)
result.setdefault("generate_probe_error", "")
result.setdefault("generate_probe_duration_ms", 0)
if not include_models:
result.pop("models", None)
public_host_results.append(result)
@@ -176,9 +287,15 @@ def build_pixelrag_vlm_route_readiness(
"candidate_host": selected_host,
"candidate_provider": selected_provider,
"candidate_selection_reason": selected_reason,
"tag_model_route_ready": tag_model_route_ready,
"model_route_ready": model_route_ready,
"generate_probe_performed": False,
"model_call_performed": False,
"generate_probe_performed": generate_probe_performed,
"generate_probe_ok_count": generate_probe_ok_count,
"generate_ready_model": generate_ready_model,
"generate_ready_host": generate_ready_host,
"generate_ready_provider": generate_ready_provider,
"generate_route_ready": generate_route_ready,
"model_call_performed": generate_probe_performed,
"writes_database_count": 0,
"primary_human_gate_count": 0,
}
@@ -188,13 +305,14 @@ def build_pixelrag_vlm_route_readiness(
"status": status,
"generated_at": generated_at,
"timeout_seconds": timeout,
"probe_timeout_seconds": probe_timeout if probe_generate else 0,
"configured_model": configured_model,
"candidate_models": candidates,
"summary": summary,
"hosts": public_host_results,
"controlled_apply": {
"network_call": True,
"model_call": False,
"model_call": generate_probe_performed,
"db_write": False,
"writes_database": False,
"writes_database_count": 0,