wooo/ewoooc
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity
Files
0c2f2f175febcbfc4e58a95d3742eb3b3c4c928d
ewoooc/tests/test_pixelrag_vlm_replay_worker_service.py
ogt ec27e89665
Some checks failed
CD Pipeline / deploy (push) Has been cancelled
Details
fix(ai): detect PixelRAG marketplace interstitials
2026-07-10 00:49:54 +08:00

524 lines
20 KiB
Python
Raw Blame History

import json
import subprocess
import sys
from types import SimpleNamespace
from tests.test_pixelrag_ocr_vlm_replay_service import _write_receipt
def test_pixelrag_vlm_replay_worker_dry_run_keeps_blocked_pages_out(tmp_path):
from services.pixelrag_vlm_replay_worker_service import (
POLICY,
run_pixelrag_ollama_vlm_replay_worker,
)
_write_receipt(
tmp_path,
platform="shopee_tw",
manifest_id="shopee-ok",
title="Shopee 防曬乳",
url="https://shopee.tw/search?keyword=sunscreen",
)
_write_receipt(
tmp_path,
platform="coupang_tw",
manifest_id="coupang-403",
title="Access Denied",
url="https://www.tw.coupang.com/search?q=iphone",
http_status=403,
)
payload = run_pixelrag_ollama_vlm_replay_worker(
artifact_root=tmp_path,
platform=("shopee_tw", "coupang_tw"),
)
assert payload["policy"] == POLICY
assert payload["status"] == "warning"
assert payload["execute"] is False
assert payload["summary"]["receipt_count"] == 2
assert payload["summary"]["ready_count"] == 1
assert payload["summary"]["dry_run_count"] == 1
assert payload["summary"]["skipped_count"] == 1
assert payload["summary"]["model_call_performed"] is False
assert payload["summary"]["artifact_write_performed"] is False
assert payload["summary"]["writes_database_count"] == 0
assert payload["controlled_apply"]["primary_human_gate_count"] == 0
assert payload["next_machine_action"] == "run_pixelrag_vlm_replay_worker_execute"
by_platform = {item["platform"]: item for item in payload["worker_items"]}
assert by_platform["shopee_tw"]["worker_status"] == "dry_run_ready"
assert by_platform["coupang_tw"]["worker_status"] == "skipped_blocked_or_not_ready"
def test_pixelrag_vlm_replay_worker_execute_writes_artifact_receipt(tmp_path, monkeypatch):
from services import pixelrag_vlm_replay_worker_service as service
_write_receipt(
tmp_path,
platform="shopee_tw",
manifest_id="shopee-ok",
title="Shopee 防曬乳",
url="https://shopee.tw/search?keyword=sunscreen",
)
class FakeOllama:
def __init__(self, model):
self.model = model
def generate(self, *args, **kwargs):
return SimpleNamespace(
success=True,
content=json.dumps({
"blocked_page_detected": False,
"fields": {
"title": {
"value": "防曬乳 SPF50",
"confidence": 0.92,
"evidence_refs": ["tile:1"],
},
"price": {
"value": "399",
"confidence": 0.88,
"evidence_refs": ["tile:2"],
},
},
"quality": {
"overall_confidence": 0.89,
"missing_required_fields": [],
"requires_identity_matcher_replay": True,
"requires_promotion_gate": True,
},
"notes": [],
}),
model="minicpm-v:latest",
error=None,
total_duration=1.5,
host="http://34.87.90.216:11434",
input_tokens=12,
output_tokens=80,
)
monkeypatch.setattr(service, "OllamaService", FakeOllama)
payload = service.run_pixelrag_ollama_vlm_replay_worker(
artifact_root=tmp_path,
output_root=tmp_path / "receipts",
platform="shopee_tw",
execute=True,
write_receipt=True,
tile_limit=1,
auto_select_model=False,
)
assert payload["status"] == "ok"
assert payload["summary"]["executed_count"] == 1
assert payload["summary"]["executed_ok_count"] == 1
assert payload["summary"]["receipt_written_count"] == 1
assert payload["summary"]["writes_database_count"] == 0
item = payload["worker_items"][0]
assert item["worker_status"] == "executed_ok"
assert item["provider"] == "gcp_ollama"
assert item["validation"]["valid_for_identity_matcher_replay"] is True
receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-ok" / "vlm_replay_receipt.json"
assert receipt_path.exists()
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
assert receipt["parsed_output"]["fields"]["title"]["value"] == "防曬乳 SPF50"
assert receipt["artifact_write_performed"] is True
assert receipt["receipt_path"] == str(receipt_path)
def test_pixelrag_vlm_replay_worker_writes_model_error_receipt(tmp_path, monkeypatch):
from services import pixelrag_vlm_replay_worker_service as service
_write_receipt(
tmp_path,
platform="shopee_tw",
manifest_id="shopee-ok",
title="Shopee 防曬乳",
url="https://shopee.tw/search?keyword=sunscreen",
)
class FakeOllama:
def __init__(self, model):
self.model = model
def generate(self, *args, **kwargs):
return SimpleNamespace(
success=False,
content="",
model="minicpm-v:latest",
error="all hosts failed",
total_duration=None,
host="http://34.21.145.224:11434",
input_tokens=0,
output_tokens=0,
)
monkeypatch.setattr(service, "OllamaService", FakeOllama)
payload = service.run_pixelrag_ollama_vlm_replay_worker(
artifact_root=tmp_path,
output_root=tmp_path / "receipts",
platform="shopee_tw",
execute=True,
write_receipt=True,
tile_limit=1,
auto_select_model=False,
)
assert payload["status"] == "critical"
assert payload["summary"]["model_error_count"] == 1
assert payload["summary"]["receipt_written_count"] == 1
assert payload["summary"]["artifact_write_performed"] is True
assert payload["controlled_apply"]["writes_database"] is False
receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-ok" / "vlm_replay_receipt.json"
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
assert receipt["worker_status"] == "model_error"
assert receipt["artifact_write_performed"] is True
assert receipt["receipt_path"] == str(receipt_path)
assert receipt["next_machine_action"] == "repair_ollama_vlm_runtime_or_model_route"
def test_pixelrag_vlm_replay_worker_empty_fields_routes_to_platform_probe(tmp_path, monkeypatch):
from services import pixelrag_vlm_replay_worker_service as service
_write_receipt(
tmp_path,
platform="shopee_tw",
manifest_id="shopee-language",
title="蝦皮購物 | 花得更少買得更好",
url="https://shopee.tw/search?keyword=sunscreen",
)
class FakeOllama:
def __init__(self, model):
self.model = model
def generate(self, *args, **kwargs):
return SimpleNamespace(
success=True,
content=json.dumps({
"blocked_page_detected": False,
"fields": {
"title": {"value": None, "confidence": 0.0, "evidence_refs": []},
"price": {"value": None, "confidence": 0.0, "evidence_refs": []},
"currency": {"value": None, "confidence": 0.0, "evidence_refs": []},
},
"quality": {
"overall_confidence": 0.0,
"missing_required_fields": [],
"requires_identity_matcher_replay": True,
"requires_promotion_gate": True,
},
"notes": [],
}),
model="minicpm-v:latest",
error=None,
total_duration=2.0,
host="http://192.168.0.111:11434",
input_tokens=10,
output_tokens=40,
)
monkeypatch.setattr(service, "OllamaService", FakeOllama)
payload = service.run_pixelrag_ollama_vlm_replay_worker(
artifact_root=tmp_path,
output_root=tmp_path / "receipts",
platform="shopee_tw",
execute=True,
write_receipt=True,
tile_limit=4,
auto_select_model=False,
)
assert payload["status"] == "warning"
assert payload["summary"]["executed_warning_count"] == 1
assert payload["summary"]["required_field_missing_count"] == 2
assert payload["next_machine_action"] == "run_platform_probe_or_use_structured_api"
item = payload["worker_items"][0]
assert item["worker_status"] == "executed_warning"
assert item["next_machine_action"] == "run_platform_probe_or_use_structured_api"
assert item["validation"]["present_field_count"] == 0
assert item["validation"]["non_product_or_interstitial_detected"] is True
assert item["validation"]["valid_for_identity_matcher_replay"] is False
receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-language" / "vlm_replay_receipt.json"
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
assert receipt["next_machine_action"] == "run_platform_probe_or_use_structured_api"
def test_pixelrag_vlm_replay_worker_language_note_routes_to_platform_probe(tmp_path, monkeypatch):
from services import pixelrag_vlm_replay_worker_service as service
_write_receipt(
tmp_path,
platform="shopee_tw",
manifest_id="shopee-language-note",
title="蝦皮購物 | 花得更少買得更好",
url="https://shopee.tw/search?keyword=sunscreen",
)
class FakeOllama:
def __init__(self, model):
self.model = model
def generate(self, *args, **kwargs):
return SimpleNamespace(
success=True,
content=json.dumps({
"blocked_page_detected": False,
"fields": {
"title": {
"value": "蝦皮購物 | 花得更少買得更好",
"confidence": 0.95,
"evidence_refs": ["tile:1"],
},
"price": {"value": None, "confidence": 0.0, "evidence_refs": []},
"shop": {"value": None, "confidence": 0.0, "evidence_refs": []},
},
"quality": {
"overall_confidence": 0.5,
"missing_required_fields": ["title", "price", "shop"],
"requires_identity_matcher_replay": True,
"requires_promotion_gate": False,
},
"notes": [{"note": "Language selection page"}],
}),
model="minicpm-v:latest",
error=None,
total_duration=2.0,
host="http://192.168.0.111:11434",
input_tokens=10,
output_tokens=40,
)
monkeypatch.setattr(service, "OllamaService", FakeOllama)
payload = service.run_pixelrag_ollama_vlm_replay_worker(
artifact_root=tmp_path,
output_root=tmp_path / "receipts",
platform="shopee_tw",
execute=True,
write_receipt=True,
tile_limit=4,
auto_select_model=False,
)
assert payload["next_machine_action"] == "run_platform_probe_or_use_structured_api"
item = payload["worker_items"][0]
assert item["next_machine_action"] == "run_platform_probe_or_use_structured_api"
assert item["validation"]["present_field_count"] == 1
assert item["validation"]["interstitial_signal_detected"] is True
assert item["validation"]["generic_marketplace_title_detected"] is True
assert item["validation"]["non_product_or_interstitial_detected"] is True
def test_pixelrag_vlm_replay_worker_auto_selects_installed_candidate(tmp_path, monkeypatch):
from services import pixelrag_vlm_replay_worker_service as service
_write_receipt(
tmp_path,
platform="shopee_tw",
manifest_id="shopee-ok",
title="Shopee 防曬乳",
url="https://shopee.tw/search?keyword=sunscreen",
)
monkeypatch.setattr(
service,
"build_pixelrag_vlm_route_readiness",
lambda **kwargs: {
"policy": "read_only_pixelrag_vlm_route_readiness_v1",
"status": "warning",
"summary": {
"configured_model": "minicpm-v:latest",
"candidate_model": "gemma3:4b",
"candidate_host": "http://34.21.145.224:11434",
"model_route_ready": True,
},
"next_machine_action": "run_pixelrag_vlm_replay_worker_execute_with_selected_model",
},
)
def fake_generate_exact_host(prompt, **kwargs):
assert kwargs["host"] == "http://34.21.145.224:11434"
assert kwargs["model"] == "gemma3:4b"
return SimpleNamespace(
success=True,
content=json.dumps({
"blocked_page_detected": False,
"fields": {
"title": {
"value": "防曬乳 SPF50",
"confidence": 0.92,
"evidence_refs": ["tile:1"],
},
"price": {
"value": "399",
"confidence": 0.88,
"evidence_refs": ["tile:1"],
},
},
"quality": {
"overall_confidence": 0.90,
"missing_required_fields": [],
"requires_identity_matcher_replay": True,
"requires_promotion_gate": True,
},
}),
model="gemma3:4b",
error=None,
total_duration=1.0,
host="http://34.21.145.224:11434",
input_tokens=10,
output_tokens=50,
)
monkeypatch.setattr(service, "_generate_exact_host", fake_generate_exact_host)
payload = service.run_pixelrag_ollama_vlm_replay_worker(
artifact_root=tmp_path,
platform="shopee_tw",
model="minicpm-v:latest",
execute=True,
tile_limit=1,
auto_select_model=True,
)
assert payload["model"] == "gemma3:4b"
assert payload["configured_model"] == "minicpm-v:latest"
assert payload["route_candidate_host"] == "http://34.21.145.224:11434"
assert payload["route_readiness"]["summary"]["candidate_model"] == "gemma3:4b"
assert payload["worker_items"][0]["route_candidate_host"] == "http://34.21.145.224:11434"
assert payload["summary"]["executed_ok_count"] == 1
def test_pixelrag_vlm_replay_worker_generate_probe_blocks_tile_generate(tmp_path, monkeypatch):
from services import pixelrag_vlm_replay_worker_service as service
_write_receipt(
tmp_path,
platform="shopee_tw",
manifest_id="shopee-ok",
title="Shopee 防曬乳",
url="https://shopee.tw/search?keyword=sunscreen",
)
monkeypatch.setattr(
service,
"build_pixelrag_vlm_route_readiness",
lambda **kwargs: {
"policy": "read_only_pixelrag_vlm_route_readiness_v1",
"status": "critical",
"summary": {
"configured_model": "minicpm-v:latest",
"candidate_model": "gemma3:4b",
"candidate_host": "http://34.21.145.224:11434",
"tag_model_route_ready": True,
"model_route_ready": False,
"generate_probe_performed": True,
"generate_probe_ok_count": 0,
"generate_route_ready": False,
"generate_ready_model": "",
"generate_ready_host": "",
"generate_ready_provider": "",
"model_call_performed": True,
},
"controlled_apply": {"model_call": True},
"next_machine_action": "repair_ollama_vlm_generate_runtime_or_proxy_timeout",
},
)
class FakeOllama:
def __init__(self, model):
self.model = model
def generate(self, *args, **kwargs):
raise AssertionError("tile VLM generate should not be called")
monkeypatch.setattr(service, "OllamaService", FakeOllama)
payload = service.run_pixelrag_ollama_vlm_replay_worker(
artifact_root=tmp_path,
output_root=tmp_path / "receipts",
platform="shopee_tw",
model="minicpm-v:latest",
execute=True,
write_receipt=True,
tile_limit=1,
auto_select_model=True,
probe_generate_before_execute=True,
)
assert payload["status"] == "critical"
assert payload["model"] == "gemma3:4b"
assert payload["summary"]["model_route_not_ready_count"] == 1
assert payload["summary"]["route_model_call_performed"] is True
assert payload["summary"]["tile_model_call_performed"] is False
assert payload["summary"]["model_call_performed"] is True
assert payload["controlled_apply"]["model_call"] is True
assert payload["next_machine_action"] == "repair_ollama_vlm_generate_runtime_or_proxy_timeout"
item = payload["worker_items"][0]
assert item["worker_status"] == "model_route_not_ready"
assert item["generate_probe_performed"] is True
assert item["generate_route_ready"] is False
assert item["model_call_performed"] is False
receipt_path = tmp_path / "receipts" / "shopee_tw" / "shopee-ok" / "vlm_replay_receipt.json"
assert receipt_path.exists()
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
assert receipt["worker_status"] == "model_route_not_ready"
assert receipt["artifact_write_performed"] is True
assert receipt["generate_route_ready"] is False
def test_pixelrag_vlm_replay_worker_cli_outputs_machine_readable_json(tmp_path):
_write_receipt(
tmp_path,
platform="shopee_tw",
manifest_id="shopee-ok",
title="Shopee 防曬乳",
url="https://shopee.tw/search?keyword=sunscreen",
)
completed = subprocess.run(
[
sys.executable,
"scripts/ops/run_pixelrag_vlm_replay_worker.py",
"--artifact-root",
str(tmp_path),
"--platform",
"shopee_tw",
],
capture_output=True,
check=False,
text=True,
)
assert completed.returncode == 0
payload = json.loads(completed.stdout)
assert payload["summary"]["receipt_count"] == 1
assert payload["summary"]["dry_run_count"] == 1
assert payload["controlled_apply"]["model_call"] is False
assert payload["controlled_apply"]["writes_database"] is False
def test_pixelrag_vlm_replay_worker_route_returns_readback(tmp_path, monkeypatch):
from flask import Flask
from routes import system_public_routes as routes
from services import pixelrag_vlm_replay_worker_service as service
_write_receipt(
tmp_path,
platform="shopee_tw",
manifest_id="shopee-ok",
title="Shopee 防曬乳",
url="https://shopee.tw/search?keyword=sunscreen",
)
monkeypatch.setattr(service, "DEFAULT_ARTIFACT_ROOT", str(tmp_path))
app = Flask(__name__)
with app.test_request_context(
"/api/ai-automation/pixelrag-vlm-replay-worker?platform=shopee_tw"
):
response = routes.ai_automation_pixelrag_vlm_replay_worker_api.__wrapped__()
payload = response.get_json()
assert payload["policy"] == "controlled_pixelrag_ollama_vlm_replay_worker_v1"
assert payload["summary"]["receipt_count"] == 1
assert payload["summary"]["dry_run_count"] == 1
assert payload["controlled_apply"]["model_call"] is False
Reference in New Issue View Git Blame Copy Permalink