import json import subprocess import sys from types import SimpleNamespace from tests.test_pixelrag_ocr_vlm_replay_service import _write_receipt def test_pixelrag_vlm_replay_worker_dry_run_keeps_blocked_pages_out(tmp_path): from services.pixelrag_vlm_replay_worker_service import ( POLICY, run_pixelrag_ollama_vlm_replay_worker, ) _write_receipt( tmp_path, platform="shopee_tw", manifest_id="shopee-ok", title="Shopee 防曬乳", url="https://shopee.tw/search?keyword=sunscreen", ) _write_receipt( tmp_path, platform="coupang_tw", manifest_id="coupang-403", title="Access Denied", url="https://www.tw.coupang.com/search?q=iphone", http_status=403, ) payload = run_pixelrag_ollama_vlm_replay_worker( artifact_root=tmp_path, platform=("shopee_tw", "coupang_tw"), ) assert payload["policy"] == POLICY assert payload["status"] == "warning" assert payload["execute"] is False assert payload["summary"]["receipt_count"] == 2 assert payload["summary"]["ready_count"] == 1 assert payload["summary"]["dry_run_count"] == 1 assert payload["summary"]["skipped_count"] == 1 assert payload["summary"]["model_call_performed"] is False assert payload["summary"]["artifact_write_performed"] is False assert payload["summary"]["writes_database_count"] == 0 assert payload["controlled_apply"]["primary_human_gate_count"] == 0 assert payload["next_machine_action"] == "run_pixelrag_vlm_replay_worker_execute" by_platform = {item["platform"]: item for item in payload["worker_items"]} assert by_platform["shopee_tw"]["worker_status"] == "dry_run_ready" assert by_platform["coupang_tw"]["worker_status"] == "skipped_blocked_or_not_ready" def test_pixelrag_vlm_replay_worker_execute_writes_artifact_receipt(tmp_path, monkeypatch): from services import pixelrag_vlm_replay_worker_service as service _write_receipt( tmp_path, platform="shopee_tw", manifest_id="shopee-ok", title="Shopee 防曬乳", url="https://shopee.tw/search?keyword=sunscreen", ) class FakeOllama: def __init__(self, model): self.model = model def generate(self, *args, **kwargs): return SimpleNamespace( success=True, content=json.dumps({ "blocked_page_detected": False, "fields": { "title": { "value": "防曬乳 SPF50", "confidence": 0.92, "evidence_refs": ["tile:1"], }, "price": { "value": "399", "confidence": 0.88, "evidence_refs": ["tile:2"], }, }, "quality": { "overall_confidence": 0.89, "missing_required_fields": [], "requires_identity_matcher_replay": True, "requires_promotion_gate": True, }, "notes": [], }), model="minicpm-v:latest", error=None, total_duration=1.5, host="http://34.87.90.216:11434", input_tokens=12, output_tokens=80, ) monkeypatch.setattr(service, "OllamaService", FakeOllama) payload = service.run_pixelrag_ollama_vlm_replay_worker( artifact_root=tmp_path, output_root=tmp_path / "receipts", platform="shopee_tw", execute=True, write_receipt=True, tile_limit=1, auto_select_model=False, ) assert payload["status"] == "ok" assert payload["summary"]["executed_count"] == 1 assert payload["summary"]["executed_ok_count"] == 1 assert payload["summary"]["receipt_written_count"] == 1 assert payload["summary"]["writes_database_count"] == 0 item = payload["worker_items"][0] assert item["worker_status"] == "executed_ok" assert item["provider"] == "gcp_ollama" assert item["validation"]["valid_for_identity_matcher_replay"] is True receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-ok" / "vlm_replay_receipt.json" assert receipt_path.exists() receipt = json.loads(receipt_path.read_text(encoding="utf-8")) assert receipt["parsed_output"]["fields"]["title"]["value"] == "防曬乳 SPF50" assert receipt["artifact_write_performed"] is True assert receipt["receipt_path"] == str(receipt_path) def test_pixelrag_vlm_replay_worker_writes_model_error_receipt(tmp_path, monkeypatch): from services import pixelrag_vlm_replay_worker_service as service _write_receipt( tmp_path, platform="shopee_tw", manifest_id="shopee-ok", title="Shopee 防曬乳", url="https://shopee.tw/search?keyword=sunscreen", ) class FakeOllama: def __init__(self, model): self.model = model def generate(self, *args, **kwargs): return SimpleNamespace( success=False, content="", model="minicpm-v:latest", error="all hosts failed", total_duration=None, host="http://34.21.145.224:11434", input_tokens=0, output_tokens=0, ) monkeypatch.setattr(service, "OllamaService", FakeOllama) payload = service.run_pixelrag_ollama_vlm_replay_worker( artifact_root=tmp_path, output_root=tmp_path / "receipts", platform="shopee_tw", execute=True, write_receipt=True, tile_limit=1, auto_select_model=False, ) assert payload["status"] == "critical" assert payload["summary"]["model_error_count"] == 1 assert payload["summary"]["receipt_written_count"] == 1 assert payload["summary"]["artifact_write_performed"] is True assert payload["controlled_apply"]["writes_database"] is False receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-ok" / "vlm_replay_receipt.json" receipt = json.loads(receipt_path.read_text(encoding="utf-8")) assert receipt["worker_status"] == "model_error" assert receipt["artifact_write_performed"] is True assert receipt["receipt_path"] == str(receipt_path) assert receipt["next_machine_action"] == "repair_ollama_vlm_runtime_or_model_route" def test_pixelrag_vlm_replay_worker_empty_fields_routes_to_platform_probe(tmp_path, monkeypatch): from services import pixelrag_vlm_replay_worker_service as service _write_receipt( tmp_path, platform="shopee_tw", manifest_id="shopee-language", title="蝦皮購物 | 花得更少買得更好", url="https://shopee.tw/search?keyword=sunscreen", ) class FakeOllama: def __init__(self, model): self.model = model def generate(self, *args, **kwargs): return SimpleNamespace( success=True, content=json.dumps({ "blocked_page_detected": False, "fields": { "title": {"value": None, "confidence": 0.0, "evidence_refs": []}, "price": {"value": None, "confidence": 0.0, "evidence_refs": []}, "currency": {"value": None, "confidence": 0.0, "evidence_refs": []}, }, "quality": { "overall_confidence": 0.0, "missing_required_fields": [], "requires_identity_matcher_replay": True, "requires_promotion_gate": True, }, "notes": [], }), model="minicpm-v:latest", error=None, total_duration=2.0, host="http://192.168.0.111:11434", input_tokens=10, output_tokens=40, ) monkeypatch.setattr(service, "OllamaService", FakeOllama) payload = service.run_pixelrag_ollama_vlm_replay_worker( artifact_root=tmp_path, output_root=tmp_path / "receipts", platform="shopee_tw", execute=True, write_receipt=True, tile_limit=4, auto_select_model=False, ) assert payload["status"] == "warning" assert payload["summary"]["executed_warning_count"] == 1 assert payload["summary"]["required_field_missing_count"] == 2 assert payload["next_machine_action"] == "run_platform_probe_or_use_structured_api" item = payload["worker_items"][0] assert item["worker_status"] == "executed_warning" assert item["next_machine_action"] == "run_platform_probe_or_use_structured_api" assert item["validation"]["present_field_count"] == 0 assert item["validation"]["non_product_or_interstitial_detected"] is True assert item["validation"]["valid_for_identity_matcher_replay"] is False receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-language" / "vlm_replay_receipt.json" receipt = json.loads(receipt_path.read_text(encoding="utf-8")) assert receipt["next_machine_action"] == "run_platform_probe_or_use_structured_api" def test_pixelrag_vlm_replay_worker_language_note_routes_to_platform_probe(tmp_path, monkeypatch): from services import pixelrag_vlm_replay_worker_service as service _write_receipt( tmp_path, platform="shopee_tw", manifest_id="shopee-language-note", title="蝦皮購物 | 花得更少買得更好", url="https://shopee.tw/search?keyword=sunscreen", ) class FakeOllama: def __init__(self, model): self.model = model def generate(self, *args, **kwargs): return SimpleNamespace( success=True, content=json.dumps({ "blocked_page_detected": False, "fields": { "title": { "value": "蝦皮購物 | 花得更少買得更好", "confidence": 0.95, "evidence_refs": ["tile:1"], }, "price": {"value": None, "confidence": 0.0, "evidence_refs": []}, "shop": {"value": None, "confidence": 0.0, "evidence_refs": []}, }, "quality": { "overall_confidence": 0.5, "missing_required_fields": ["title", "price", "shop"], "requires_identity_matcher_replay": True, "requires_promotion_gate": False, }, "notes": [{"note": "Language selection page"}], }), model="minicpm-v:latest", error=None, total_duration=2.0, host="http://192.168.0.111:11434", input_tokens=10, output_tokens=40, ) monkeypatch.setattr(service, "OllamaService", FakeOllama) payload = service.run_pixelrag_ollama_vlm_replay_worker( artifact_root=tmp_path, output_root=tmp_path / "receipts", platform="shopee_tw", execute=True, write_receipt=True, tile_limit=4, auto_select_model=False, ) assert payload["next_machine_action"] == "run_platform_probe_or_use_structured_api" item = payload["worker_items"][0] assert item["next_machine_action"] == "run_platform_probe_or_use_structured_api" assert item["validation"]["present_field_count"] == 1 assert item["validation"]["interstitial_signal_detected"] is True assert item["validation"]["generic_marketplace_title_detected"] is True assert item["validation"]["non_product_or_interstitial_detected"] is True def test_pixelrag_vlm_replay_worker_auto_selects_installed_candidate(tmp_path, monkeypatch): from services import pixelrag_vlm_replay_worker_service as service _write_receipt( tmp_path, platform="shopee_tw", manifest_id="shopee-ok", title="Shopee 防曬乳", url="https://shopee.tw/search?keyword=sunscreen", ) monkeypatch.setattr( service, "build_pixelrag_vlm_route_readiness", lambda **kwargs: { "policy": "read_only_pixelrag_vlm_route_readiness_v1", "status": "warning", "summary": { "configured_model": "minicpm-v:latest", "candidate_model": "gemma3:4b", "candidate_host": "http://34.21.145.224:11434", "model_route_ready": True, }, "next_machine_action": "run_pixelrag_vlm_replay_worker_execute_with_selected_model", }, ) def fake_generate_exact_host(prompt, **kwargs): assert kwargs["host"] == "http://34.21.145.224:11434" assert kwargs["model"] == "gemma3:4b" return SimpleNamespace( success=True, content=json.dumps({ "blocked_page_detected": False, "fields": { "title": { "value": "防曬乳 SPF50", "confidence": 0.92, "evidence_refs": ["tile:1"], }, "price": { "value": "399", "confidence": 0.88, "evidence_refs": ["tile:1"], }, }, "quality": { "overall_confidence": 0.90, "missing_required_fields": [], "requires_identity_matcher_replay": True, "requires_promotion_gate": True, }, }), model="gemma3:4b", error=None, total_duration=1.0, host="http://34.21.145.224:11434", input_tokens=10, output_tokens=50, ) monkeypatch.setattr(service, "_generate_exact_host", fake_generate_exact_host) payload = service.run_pixelrag_ollama_vlm_replay_worker( artifact_root=tmp_path, platform="shopee_tw", model="minicpm-v:latest", execute=True, tile_limit=1, auto_select_model=True, ) assert payload["model"] == "gemma3:4b" assert payload["configured_model"] == "minicpm-v:latest" assert payload["route_candidate_host"] == "http://34.21.145.224:11434" assert payload["route_readiness"]["summary"]["candidate_model"] == "gemma3:4b" assert payload["worker_items"][0]["route_candidate_host"] == "http://34.21.145.224:11434" assert payload["summary"]["executed_ok_count"] == 1 def test_pixelrag_vlm_replay_worker_generate_probe_blocks_tile_generate(tmp_path, monkeypatch): from services import pixelrag_vlm_replay_worker_service as service _write_receipt( tmp_path, platform="shopee_tw", manifest_id="shopee-ok", title="Shopee 防曬乳", url="https://shopee.tw/search?keyword=sunscreen", ) monkeypatch.setattr( service, "build_pixelrag_vlm_route_readiness", lambda **kwargs: { "policy": "read_only_pixelrag_vlm_route_readiness_v1", "status": "critical", "summary": { "configured_model": "minicpm-v:latest", "candidate_model": "gemma3:4b", "candidate_host": "http://34.21.145.224:11434", "tag_model_route_ready": True, "model_route_ready": False, "generate_probe_performed": True, "generate_probe_ok_count": 0, "generate_route_ready": False, "generate_ready_model": "", "generate_ready_host": "", "generate_ready_provider": "", "model_call_performed": True, }, "controlled_apply": {"model_call": True}, "next_machine_action": "repair_ollama_vlm_generate_runtime_or_proxy_timeout", }, ) class FakeOllama: def __init__(self, model): self.model = model def generate(self, *args, **kwargs): raise AssertionError("tile VLM generate should not be called") monkeypatch.setattr(service, "OllamaService", FakeOllama) payload = service.run_pixelrag_ollama_vlm_replay_worker( artifact_root=tmp_path, output_root=tmp_path / "receipts", platform="shopee_tw", model="minicpm-v:latest", execute=True, write_receipt=True, tile_limit=1, auto_select_model=True, probe_generate_before_execute=True, ) assert payload["status"] == "critical" assert payload["model"] == "gemma3:4b" assert payload["summary"]["model_route_not_ready_count"] == 1 assert payload["summary"]["route_model_call_performed"] is True assert payload["summary"]["tile_model_call_performed"] is False assert payload["summary"]["model_call_performed"] is True assert payload["controlled_apply"]["model_call"] is True assert payload["next_machine_action"] == "repair_ollama_vlm_generate_runtime_or_proxy_timeout" item = payload["worker_items"][0] assert item["worker_status"] == "model_route_not_ready" assert item["generate_probe_performed"] is True assert item["generate_route_ready"] is False assert item["model_call_performed"] is False receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-ok" / "vlm_replay_receipt.json" assert receipt_path.exists() receipt = json.loads(receipt_path.read_text(encoding="utf-8")) assert receipt["worker_status"] == "model_route_not_ready" assert receipt["artifact_write_performed"] is True assert receipt["generate_route_ready"] is False def test_pixelrag_vlm_replay_worker_cli_outputs_machine_readable_json(tmp_path): _write_receipt( tmp_path, platform="shopee_tw", manifest_id="shopee-ok", title="Shopee 防曬乳", url="https://shopee.tw/search?keyword=sunscreen", ) completed = subprocess.run( [ sys.executable, "scripts/ops/run_pixelrag_vlm_replay_worker.py", "--artifact-root", str(tmp_path), "--platform", "shopee_tw", ], capture_output=True, check=False, text=True, ) assert completed.returncode == 0 payload = json.loads(completed.stdout) assert payload["summary"]["receipt_count"] == 1 assert payload["summary"]["dry_run_count"] == 1 assert payload["controlled_apply"]["model_call"] is False assert payload["controlled_apply"]["writes_database"] is False def test_pixelrag_vlm_replay_worker_route_returns_readback(tmp_path, monkeypatch): from flask import Flask from routes import system_public_routes as routes from services import pixelrag_vlm_replay_worker_service as service _write_receipt( tmp_path, platform="shopee_tw", manifest_id="shopee-ok", title="Shopee 防曬乳", url="https://shopee.tw/search?keyword=sunscreen", ) monkeypatch.setattr(service, "DEFAULT_ARTIFACT_ROOT", str(tmp_path)) app = Flask(__name__) with app.test_request_context( "/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw" ): response = routes.ai_automation_pixelrag_vlm_replay_worker_api.__wrapped__() payload = response.get_json() assert payload["policy"] == "controlled_pixelrag_ollama_vlm_replay_worker_v1" assert payload["summary"]["receipt_count"] == 1 assert payload["summary"]["dry_run_count"] == 1 assert payload["controlled_apply"]["model_call"] is False