271 lines
9.7 KiB
Python
271 lines
9.7 KiB
Python
import json
|
|
import inspect
|
|
import subprocess
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
|
|
def _write_candidate_receipt(root, *, signature=None, platform="shopee_tw"):
|
|
from services.rag_service import get_embedding_signature
|
|
|
|
embedding_signature = signature or get_embedding_signature()
|
|
manifest_id = "shopee-canary-001"
|
|
target = root / platform / manifest_id
|
|
target.mkdir(parents=True)
|
|
payload = {
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"worker_status": "executed_marketplace_candidate_knowledge_replay_ready",
|
|
"platform": platform,
|
|
"manifest_id": manifest_id,
|
|
"writes_database": False,
|
|
"writes_database_count": 0,
|
|
"writes_ai_insights": False,
|
|
"writes_price_tables": False,
|
|
"candidate_knowledge_replay": {
|
|
"candidate_knowledge_replay_version": "pixelrag_marketplace_candidate_knowledge_replay_v1",
|
|
"embedding_signature_contract": {"embedding_signature": embedding_signature},
|
|
"candidate_knowledge_contracts": [{
|
|
"candidate_id": f"{platform}:{manifest_id}:0",
|
|
"candidate_knowledge_fingerprint": "candidate-fingerprint-001",
|
|
"candidate_knowledge_text": (
|
|
f"platform={platform} | manifest_id={manifest_id} | "
|
|
"candidate_id=0 | product evidence"
|
|
),
|
|
"ready_for_internal_rag_candidate_replay": True,
|
|
"ready_for_ai_insights_write": False,
|
|
"ready_for_price_table_write": False,
|
|
}],
|
|
},
|
|
}
|
|
path = target / "marketplace_candidate_knowledge_replay_receipt.json"
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def test_internal_rag_canary_dry_run_never_calls_model_or_database(tmp_path, monkeypatch):
|
|
from services import internal_rag_candidate_canary_service as service
|
|
|
|
source_root = tmp_path / "candidate"
|
|
_write_candidate_receipt(source_root)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_generate_embedding",
|
|
lambda _text: (_ for _ in ()).throw(AssertionError("model call forbidden")),
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run_pgvector_probe",
|
|
lambda *_args: (_ for _ in ()).throw(AssertionError("DB call forbidden")),
|
|
)
|
|
payload = service.run_internal_rag_candidate_canary(
|
|
candidate_knowledge_receipt_root=source_root,
|
|
)
|
|
|
|
assert payload["status"] == "ready_for_canary"
|
|
assert payload["summary"]["ready_count"] == 1
|
|
assert payload["summary"]["executed_count"] == 0
|
|
assert payload["controlled_apply"]["model_call"] is False
|
|
assert payload["controlled_apply"]["database_write"] is False
|
|
|
|
|
|
def test_internal_rag_canary_executes_read_only_pgvector_probe(tmp_path, monkeypatch):
|
|
from services import internal_rag_candidate_canary_service as service
|
|
|
|
source_root = tmp_path / "candidate"
|
|
output_root = tmp_path / "canary"
|
|
_write_candidate_receipt(source_root)
|
|
monkeypatch.setattr(
|
|
service, "_generate_embedding", lambda _text: [0.01] * service.RAG_EMBED_DIM
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_verify_embedding_consistency",
|
|
lambda: {
|
|
"ok": True,
|
|
"reachable": ["gcp_ollama", "ollama_secondary"],
|
|
"max_diff": 0.0,
|
|
"errors": [],
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run_pgvector_probe",
|
|
lambda *_args: {
|
|
"transaction_read_only": True,
|
|
"exact_similarity": 1.0,
|
|
"probe_similarity": 0.91,
|
|
"ai_insights_table_present": True,
|
|
"database_write_performed": False,
|
|
},
|
|
)
|
|
payload = service.run_internal_rag_candidate_canary(
|
|
candidate_knowledge_receipt_root=source_root,
|
|
output_root=output_root,
|
|
execute=True,
|
|
write_receipt=True,
|
|
)
|
|
|
|
assert payload["summary"]["canary_passed_count"] == 1
|
|
assert payload["status"] in {"canary_passed_activation_blocked", "complete"}
|
|
item = payload["executed_items"][0]
|
|
assert item["canary_passed"] is True
|
|
assert item["transaction_read_only"] is True
|
|
assert item["writes_database"] is False
|
|
assert item["writes_ai_insights"] is False
|
|
assert item["writes_price_tables"] is False
|
|
assert payload["latest_execution"]["canary_passed"] is True
|
|
assert payload["run_identity"]["work_item_id"] == "RAG-P0-001"
|
|
assert item["run_identity"] == payload["run_identity"]
|
|
|
|
|
|
def test_internal_rag_canary_requires_both_primary_gcp_hosts(tmp_path, monkeypatch):
|
|
from services import internal_rag_candidate_canary_service as service
|
|
|
|
source_root = tmp_path / "candidate"
|
|
_write_candidate_receipt(source_root)
|
|
monkeypatch.setattr(
|
|
service, "_generate_embedding", lambda _text: [0.01] * service.RAG_EMBED_DIM
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_verify_embedding_consistency",
|
|
lambda: {
|
|
"ok": True,
|
|
"reachable": ["gcp_ollama", "ollama_111"],
|
|
"max_diff": 0.0,
|
|
"errors": ["ollama_secondary unavailable"],
|
|
},
|
|
)
|
|
database_calls = []
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run_pgvector_probe",
|
|
lambda *_args: database_calls.append(_args) or {
|
|
"transaction_read_only": True,
|
|
"exact_similarity": 1.0,
|
|
"probe_similarity": 0.91,
|
|
"ai_insights_table_present": True,
|
|
"database_write_performed": False,
|
|
},
|
|
)
|
|
|
|
payload = service.run_internal_rag_candidate_canary(
|
|
candidate_knowledge_receipt_root=source_root,
|
|
execute=True,
|
|
)
|
|
|
|
assert payload["status"] == "canary_failed"
|
|
item = payload["executed_items"][0]
|
|
assert item["canary_checks"]["cross_host_embedding_consistent"] is False
|
|
assert item["canary_checks"]["database_call_blocked_by_preflight"] is True
|
|
assert item["required_consistency_hosts"] == ["gcp_ollama", "ollama_secondary"]
|
|
assert item["error"].startswith("embedding_host_preflight_failed")
|
|
assert item["database_call_performed"] is False
|
|
assert database_calls == []
|
|
|
|
|
|
def test_internal_rag_canary_blocks_empty_embedding_before_database(tmp_path, monkeypatch):
|
|
from services import internal_rag_candidate_canary_service as service
|
|
|
|
source_root = tmp_path / "candidate"
|
|
_write_candidate_receipt(source_root)
|
|
database_calls = []
|
|
monkeypatch.setattr(service, "_generate_embedding", lambda _text: [])
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_verify_embedding_consistency",
|
|
lambda: {
|
|
"ok": True,
|
|
"reachable": ["gcp_ollama", "ollama_secondary"],
|
|
"max_diff": 0.0,
|
|
"errors": [],
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run_pgvector_probe",
|
|
lambda *_args: database_calls.append(_args),
|
|
)
|
|
|
|
payload = service.run_internal_rag_candidate_canary(
|
|
candidate_knowledge_receipt_root=source_root,
|
|
execute=True,
|
|
)
|
|
|
|
item = payload["executed_items"][0]
|
|
assert payload["status"] == "canary_failed"
|
|
assert item["error"] == (
|
|
f"embedding_dimension_preflight_failed: candidate=0 probe=0 expected={service.RAG_EMBED_DIM}"
|
|
)
|
|
assert item["database_call_performed"] is False
|
|
assert item["rollback_terminal"] == "no_database_call_due_embedding_dimension_preflight"
|
|
assert database_calls == []
|
|
|
|
|
|
def test_internal_rag_canary_blocks_embedding_signature_drift(tmp_path):
|
|
from services.internal_rag_candidate_canary_service import run_internal_rag_candidate_canary
|
|
|
|
source_root = tmp_path / "candidate"
|
|
_write_candidate_receipt(source_root, signature="deadbeef0000")
|
|
payload = run_internal_rag_candidate_canary(
|
|
candidate_knowledge_receipt_root=source_root,
|
|
)
|
|
|
|
assert payload["status"] == "blocked"
|
|
assert payload["summary"]["ready_count"] == 0
|
|
assert payload["source_items"][0]["source_checks"][
|
|
"embedding_signature_matches_runtime"
|
|
] is False
|
|
|
|
|
|
def test_internal_rag_canary_cli_outputs_machine_readable_dry_run(tmp_path):
|
|
source_root = tmp_path / "candidate"
|
|
_write_candidate_receipt(source_root)
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"scripts/ops/run_internal_rag_candidate_canary.py",
|
|
"--candidate-knowledge-receipt-root",
|
|
str(source_root),
|
|
],
|
|
capture_output=True,
|
|
check=False,
|
|
text=True,
|
|
)
|
|
|
|
assert completed.returncode == 0
|
|
payload = json.loads(completed.stdout)
|
|
assert payload["status"] == "ready_for_canary"
|
|
assert payload["controlled_apply"]["database_write"] is False
|
|
|
|
|
|
def test_internal_rag_canary_route_returns_readback(tmp_path, monkeypatch):
|
|
from flask import Flask
|
|
from routes import system_public_routes as routes
|
|
from services import internal_rag_candidate_canary_service as service
|
|
|
|
source_root = tmp_path / "candidate"
|
|
_write_candidate_receipt(source_root, platform="coupang_tw")
|
|
monkeypatch.setattr(
|
|
service, "DEFAULT_CANDIDATE_KNOWLEDGE_RECEIPT_ROOT", str(source_root)
|
|
)
|
|
app = Flask(__name__)
|
|
with app.test_request_context(
|
|
"/api/ai-automation/internal-rag-candidate-canary?platform=coupang_tw&execute=true"
|
|
):
|
|
response = routes.ai_automation_internal_rag_candidate_canary_api.__wrapped__()
|
|
payload = response.get_json()
|
|
|
|
assert payload["policy"] == "controlled_internal_rag_candidate_canary_v1"
|
|
assert payload["summary"]["ready_count"] == 1
|
|
assert payload["execute"] is False
|
|
|
|
|
|
def test_internal_rag_canary_http_route_is_read_only():
|
|
from routes import system_public_routes as routes
|
|
|
|
source = inspect.getsource(routes.ai_automation_internal_rag_candidate_canary_api)
|
|
assert "execute=False" in source
|
|
assert "write_receipt=False" in source
|
|
assert "get_current_user" not in source
|