"""Controlled PixelRAG candidate canary for the internal pgvector RAG plane. The canary consumes candidate-knowledge receipts, generates Ollama-first text embeddings, and verifies retrieval with PostgreSQL's pgvector operator inside a read-only transaction. It never writes ai_insights or product price tables. """ from __future__ import annotations import json import os import uuid from datetime import datetime, timezone from pathlib import Path from typing import Any, Mapping from services.pixelrag_crawler_integration_service import DEFAULT_ARTIFACT_MAX_AGE_HOURS from services.pixelrag_marketplace_candidate_knowledge_replay_service import ( CANDIDATE_KNOWLEDGE_REPLAY_VERSION, DEFAULT_OUTPUT_ROOT as DEFAULT_CANDIDATE_KNOWLEDGE_RECEIPT_ROOT, ) from services.rag_service import ( RAG_EMBED_DIM, RAG_EMBED_MODEL, get_embedding_signature, is_rag_enabled, ) POLICY = "controlled_internal_rag_candidate_canary_v1" CANARY_VERSION = "internal_rag_candidate_canary_v1" DEFAULT_LIMIT = 1 DEFAULT_SIMILARITY_THRESHOLD = float( os.getenv("INTERNAL_RAG_CANDIDATE_CANARY_THRESHOLD", "0.70") ) DEFAULT_OUTPUT_ROOT = os.getenv( "INTERNAL_RAG_CANDIDATE_CANARY_RECEIPT_ROOT", "/app/data/ai_automation/internal_rag_candidate_canary_receipts" if Path("/app/data").exists() else "runtime_artifacts/internal_rag_candidate_canary_receipts", ) def _as_mapping(value: Any) -> Mapping[str, Any]: return value if isinstance(value, Mapping) else {} def _as_list(value: Any) -> list[Any]: return list(value) if isinstance(value, list) else [] def _parse_datetime(value: Any) -> datetime | None: if not value: return None try: parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) except ValueError: return None if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=timezone.utc) return parsed.astimezone(timezone.utc) def _normalise_platforms( platform: str | tuple[str, ...] | list[str] | None, ) -> tuple[str, ...]: if isinstance(platform, str): value = platform.strip().lower() return (value,) if value else () return tuple( str(item or "").strip().lower() for item in (platform or ()) if str(item or "").strip() ) def _safe_segment(value: Any) -> str: text = "".join( char if char.isalnum() or char in "._-" else "-" for char in str(value or "unknown").strip().lower() ) return text.strip("-") or "unknown" def _receipt_candidates( root: Path, *, platforms: tuple[str, ...], limit: int, ) -> list[Path]: if not root.exists(): return [] candidates: list[Path] = [] if platforms: for platform in platforms: candidates.extend( (root / platform).glob( "*/marketplace_candidate_knowledge_replay_receipt.json" ) ) else: candidates.extend( root.glob("*/*/marketplace_candidate_knowledge_replay_receipt.json") ) return sorted( candidates, key=lambda path: path.stat().st_mtime, reverse=True, )[:limit] def _latest_execution_receipt(root: Path) -> dict[str, Any]: if not root.exists(): return {} candidates = sorted( root.glob("*/*/internal_rag_candidate_canary_receipt.json"), key=lambda path: path.stat().st_mtime, reverse=True, ) if not candidates: return {} path = candidates[0] try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return {} return { "receipt_path": str(path), "generated_at": payload.get("generated_at"), "status": payload.get("status"), "canary_passed": payload.get("canary_passed") is True, "platform": payload.get("platform"), "manifest_id": payload.get("manifest_id"), "embedding_signature": payload.get("embedding_signature"), "probe_similarity": payload.get("probe_similarity"), "transaction_read_only": payload.get("transaction_read_only") is True, "writes_ai_insights": payload.get("writes_ai_insights") is True, "writes_price_tables": payload.get("writes_price_tables") is True, } def _load_receipt(path: Path) -> tuple[dict[str, Any], list[str]]: try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: return {}, [str(exc)[:300]] return payload, [] def _source_item( path: Path, *, now: datetime, max_age_hours: int, ) -> dict[str, Any]: receipt, errors = _load_receipt(path) knowledge = _as_mapping(receipt.get("candidate_knowledge_replay")) contracts = [ item for item in _as_list(knowledge.get("candidate_knowledge_contracts")) if isinstance(item, Mapping) ] generated_at = _parse_datetime(receipt.get("generated_at")) if generated_at is None: try: generated_at = datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc) except OSError: generated_at = None age_hours = ((now - generated_at).total_seconds() / 3600) if generated_at else None stale = age_hours is None or age_hours > max_age_hours expected_signature = str( _as_mapping(knowledge.get("embedding_signature_contract")).get( "embedding_signature" ) or "" ) source_checks = { "receipt_parse_ok": not errors, "receipt_fresh": not stale, "candidate_knowledge_version_supported": ( knowledge.get("candidate_knowledge_replay_version") == CANDIDATE_KNOWLEDGE_REPLAY_VERSION ), "candidate_knowledge_execute_completed": ( receipt.get("worker_status") == "executed_marketplace_candidate_knowledge_replay_ready" ), "candidate_contracts_present": bool(contracts), "candidate_contracts_ready": bool(contracts) and all( item.get("ready_for_internal_rag_candidate_replay") is True and item.get("ready_for_ai_insights_write") is False and item.get("ready_for_price_table_write") is False for item in contracts ), "embedding_signature_matches_runtime": ( bool(expected_signature) and expected_signature == get_embedding_signature() ), "source_database_write_absent": ( receipt.get("writes_database") is False and int(receipt.get("writes_database_count") or 0) == 0 ), "source_ai_insights_write_absent": receipt.get("writes_ai_insights") is False, "source_price_write_absent": receipt.get("writes_price_tables") is False, } ready = all(source_checks.values()) return { "platform": str(receipt.get("platform") or path.parent.parent.name).lower(), "manifest_id": str(receipt.get("manifest_id") or path.parent.name), "source_receipt_path": str(path), "generated_at": generated_at.isoformat() if generated_at else None, "age_hours": round(age_hours, 3) if age_hours is not None else None, "stale": stale, "expected_embedding_signature": expected_signature, "candidate_contracts": contracts, "source_checks": source_checks, "source_check_count": len(source_checks), "source_check_pass_count": sum(source_checks.values()), "ready_for_canary": ready, "errors": errors, } def _probe_text(candidate: Mapping[str, Any], *, platform: str, manifest_id: str) -> str: return " | ".join( [ f"platform={platform}", f"manifest_id={manifest_id}", f"candidate_id={candidate.get('candidate_id') or 'unknown'}", "retrieve PixelRAG marketplace evidence for internal RAG verification", ] ) def _generate_embedding(text: str) -> list[float]: from services.ollama_service import ollama_service return list( ollama_service.generate_embedding( text, model=RAG_EMBED_MODEL, allow_111_fallback=False, ) or [] ) def _verify_embedding_consistency() -> dict[str, Any]: from services.rag_service import verify_embedding_consistency return dict(verify_embedding_consistency()) def _run_pgvector_probe( candidate_embedding: list[float], probe_embedding: list[float], ) -> dict[str, Any]: from sqlalchemy import text from database.manager import get_session session = get_session() try: session.execute(text("SET TRANSACTION READ ONLY")) transaction_read_only = str( session.execute(text("SHOW transaction_read_only")).scalar() or "" ).lower() == "on" row = session.execute( text( """ SELECT 1.0 - ( CAST(:candidate_embedding AS vector) <=> CAST(:candidate_embedding AS vector) ) AS exact_similarity, 1.0 - ( CAST(:candidate_embedding AS vector) <=> CAST(:probe_embedding AS vector) ) AS probe_similarity, to_regclass('public.ai_insights') IS NOT NULL AS ai_insights_exists """ ), { "candidate_embedding": str(candidate_embedding), "probe_embedding": str(probe_embedding), }, ).mappings().one() return { "transaction_read_only": transaction_read_only, "exact_similarity": round(float(row["exact_similarity"] or 0.0), 6), "probe_similarity": round(float(row["probe_similarity"] or 0.0), 6), "ai_insights_table_present": bool(row["ai_insights_exists"]), "database_write_performed": False, } finally: session.rollback() session.close() def _execute_item( item: Mapping[str, Any], *, consistency: Mapping[str, Any], similarity_threshold: float, run_identity: Mapping[str, str], ) -> dict[str, Any]: contracts = list(item.get("candidate_contracts") or []) candidate = contracts[0] if contracts else {} candidate_text = str(candidate.get("candidate_knowledge_text") or "") probe_text = _probe_text( candidate, platform=str(item.get("platform") or "unknown"), manifest_id=str(item.get("manifest_id") or "unknown"), ) result = dict(item) result["run_identity"] = dict(run_identity) result["candidate_contracts"] = [] result["candidate_id"] = candidate.get("candidate_id") result["candidate_knowledge_fingerprint"] = candidate.get( "candidate_knowledge_fingerprint" ) result["embedding_signature"] = get_embedding_signature() result["embedding_model"] = RAG_EMBED_MODEL result["similarity_threshold"] = similarity_threshold result["network_call_performed"] = True result["model_call_performed"] = True result["writes_database"] = False result["writes_ai_insights"] = False result["writes_price_tables"] = False required_hosts = {"gcp_ollama", "ollama_secondary"} reachable = list(consistency.get("reachable") or []) consistency_ready = ( consistency.get("ok") is True and required_hosts.issubset(set(reachable)) ) if not consistency_ready: result.update( { "status": "canary_failed", "canary_passed": False, "candidate_embedding_dimension": 0, "probe_embedding_dimension": 0, "database_call_performed": False, "transaction_read_only": None, "required_consistency_hosts": sorted(required_hosts), "reachable_consistency_hosts": reachable, "canary_checks": { "cross_host_embedding_consistent": False, "database_call_blocked_by_preflight": True, "database_write_absent": True, "ai_insights_write_absent": True, "price_table_write_absent": True, }, "canary_check_count": 5, "canary_check_pass_count": 4, "rollback_terminal": "no_database_call_due_embedding_host_preflight", "error": ( "embedding_host_preflight_failed: " f"required={sorted(required_hosts)} reachable={reachable} " f"errors={list(consistency.get('errors') or [])[:2]}" )[:500], } ) return result try: candidate_embedding = _generate_embedding(candidate_text) probe_embedding = _generate_embedding(probe_text) candidate_dimension_valid = len(candidate_embedding) == RAG_EMBED_DIM probe_dimension_valid = len(probe_embedding) == RAG_EMBED_DIM if not candidate_dimension_valid or not probe_dimension_valid: checks = { "candidate_embedding_dimension_valid": candidate_dimension_valid, "probe_embedding_dimension_valid": probe_dimension_valid, "cross_host_embedding_consistent": True, "database_call_blocked_by_preflight": True, "database_write_absent": True, "ai_insights_write_absent": True, "price_table_write_absent": True, } result.update( { "status": "canary_failed", "canary_passed": False, "candidate_embedding_dimension": len(candidate_embedding), "probe_embedding_dimension": len(probe_embedding), "database_call_performed": False, "transaction_read_only": None, "required_consistency_hosts": sorted(required_hosts), "reachable_consistency_hosts": reachable, "canary_checks": checks, "canary_check_count": len(checks), "canary_check_pass_count": sum(checks.values()), "rollback_terminal": "no_database_call_due_embedding_dimension_preflight", "error": ( "embedding_dimension_preflight_failed: " f"candidate={len(candidate_embedding)} " f"probe={len(probe_embedding)} expected={RAG_EMBED_DIM}" ), } ) return result pgvector = _run_pgvector_probe(candidate_embedding, probe_embedding) checks = { "candidate_embedding_dimension_valid": ( len(candidate_embedding) == RAG_EMBED_DIM ), "probe_embedding_dimension_valid": len(probe_embedding) == RAG_EMBED_DIM, "cross_host_embedding_consistent": ( consistency.get("ok") is True and required_hosts.issubset(set(reachable)) ), "pgvector_transaction_read_only": ( pgvector.get("transaction_read_only") is True ), "pgvector_exact_similarity_valid": ( float(pgvector.get("exact_similarity") or 0.0) >= 0.999 ), "pgvector_probe_similarity_passed": ( float(pgvector.get("probe_similarity") or 0.0) >= similarity_threshold ), "database_write_absent": ( pgvector.get("database_write_performed") is False ), "ai_insights_write_absent": True, "price_table_write_absent": True, } passed = all(checks.values()) result.update( { "status": "canary_passed" if passed else "canary_failed", "canary_passed": passed, "candidate_embedding_dimension": len(candidate_embedding), "probe_embedding_dimension": len(probe_embedding), "exact_similarity": pgvector.get("exact_similarity"), "probe_similarity": pgvector.get("probe_similarity"), "transaction_read_only": pgvector.get("transaction_read_only"), "database_call_performed": True, "pgvector_probe": pgvector, "required_consistency_hosts": sorted(required_hosts), "reachable_consistency_hosts": reachable, "canary_checks": checks, "canary_check_count": len(checks), "canary_check_pass_count": sum(checks.values()), "error": None, "rollback_terminal": "transaction_rollback_after_read_only_pgvector_probe", } ) except Exception as exc: result.update( { "status": "canary_failed", "canary_passed": False, "transaction_read_only": False, "database_call_performed": True, "canary_checks": {}, "canary_check_count": 0, "canary_check_pass_count": 0, "error": f"{type(exc).__name__}: {str(exc)[:300]}", "rollback_terminal": "transaction_rollback_after_pgvector_probe_error", } ) return result def _write_receipt(root: Path, item: Mapping[str, Any]) -> str: target = ( root / _safe_segment(item.get("platform")) / _safe_segment(item.get("manifest_id")) / "internal_rag_candidate_canary_receipt.json" ) target.parent.mkdir(parents=True, exist_ok=True) payload = dict(item) payload["policy"] = POLICY payload["canary_version"] = CANARY_VERSION payload["generated_at"] = datetime.now(timezone.utc).isoformat() payload["artifact_write_performed"] = True payload["receipt_path"] = str(target) target.write_text( json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8", ) return str(target) def run_internal_rag_candidate_canary( *, candidate_knowledge_receipt_root: str | Path | None = None, output_root: str | Path | None = None, platform: str | tuple[str, ...] | list[str] | None = None, max_age_hours: int | None = None, limit: int | None = None, similarity_threshold: float | None = None, execute: bool = False, write_receipt: bool = False, trace_id: str | None = None, run_id: str | None = None, work_item_id: str = "RAG-P0-001", ) -> dict[str, Any]: """Dry-run or execute a bounded, read-only pgvector retrieval canary.""" source_root = Path( candidate_knowledge_receipt_root or DEFAULT_CANDIDATE_KNOWLEDGE_RECEIPT_ROOT ) receipt_root = Path(output_root or DEFAULT_OUTPUT_ROOT) platforms = _normalise_platforms(platform) item_limit = max(1, min(int(limit or DEFAULT_LIMIT), 5)) max_age = max(1, int(max_age_hours or DEFAULT_ARTIFACT_MAX_AGE_HOURS)) threshold = max( 0.0, min(float(similarity_threshold or DEFAULT_SIMILARITY_THRESHOLD), 1.0), ) resolved_run_id = str(run_id or f"rag-canary-{uuid.uuid4()}") run_identity = { "trace_id": str(trace_id or f"trace-{resolved_run_id}"), "run_id": resolved_run_id, "work_item_id": str(work_item_id or "RAG-P0-001"), } now = datetime.now(timezone.utc) source_items = [ _source_item(path, now=now, max_age_hours=max_age) for path in _receipt_candidates( source_root, platforms=platforms, limit=item_limit, ) ] ready_items = [item for item in source_items if item.get("ready_for_canary")] consistency: dict[str, Any] = {} executed_items: list[dict[str, Any]] = [] if execute and ready_items: consistency = _verify_embedding_consistency() for item in ready_items: executed = _execute_item( item, consistency=consistency, similarity_threshold=threshold, run_identity=run_identity, ) if write_receipt: executed["receipt_path"] = _write_receipt(receipt_root, executed) executed["artifact_write_performed"] = True executed_items.append(executed) canary_passed_count = sum( 1 for item in executed_items if item.get("canary_passed") is True ) canary_failed_count = len(executed_items) - canary_passed_count activation_blockers: list[str] = [] if not is_rag_enabled(): activation_blockers.append("rag_runtime_disabled") if RAG_EMBED_MODEL.endswith(":latest"): activation_blockers.append("rag_embedding_model_floating_tag") latest_execution = _latest_execution_receipt(receipt_root) historical_canary_passed = latest_execution.get("canary_passed") is True if not source_items: status = "warning" next_action = "run_marketplace_candidate_knowledge_replay_execute" elif not ready_items: status = "blocked" next_action = "repair_candidate_knowledge_receipt_guards" elif not execute: status = "ready_for_canary" next_action = "run_internal_rag_candidate_canary_execute" elif canary_failed_count: status = "canary_failed" next_action = "repair_embedding_or_pgvector_canary_failure" elif activation_blockers: status = "canary_passed_activation_blocked" next_action = "pin_embedding_model_then_enable_rag_controlled_canary" else: status = "complete" next_action = "continue_scheduled_internal_rag_canary" return { "success": bool(source_items) and bool(ready_items) and not canary_failed_count, "run_identity": run_identity, "policy": POLICY, "canary_version": CANARY_VERSION, "generated_at": now.isoformat(), "status": status, "execute": bool(execute), "source_receipt_root": str(source_root), "output_root": str(receipt_root), "summary": { "source_receipt_count": len(source_items), "ready_count": len(ready_items), "blocked_count": len(source_items) - len(ready_items), "executed_count": len(executed_items), "canary_passed_count": canary_passed_count, "canary_failed_count": canary_failed_count, "historical_canary_passed": historical_canary_passed, }, "embedding": { "model": RAG_EMBED_MODEL, "dimension": RAG_EMBED_DIM, "signature": get_embedding_signature(), "immutable_model_reference": not RAG_EMBED_MODEL.endswith(":latest"), "cross_host_consistency": consistency, }, "source_items": source_items, "executed_items": executed_items, "latest_execution": latest_execution, "activation_blockers": activation_blockers, "source_of_truth_diff": { "expected_embedding_signature": get_embedding_signature(), "observed_embedding_signatures": sorted({ str(item.get("expected_embedding_signature") or "") for item in source_items if item.get("expected_embedding_signature") }), "rag_runtime_expected_enabled": True, "rag_runtime_observed_enabled": is_rag_enabled(), "candidate_receipt_expected_minimum": 1, "candidate_receipt_observed": len(source_items), }, "controlled_apply": { "risk": "medium", "bounded_candidate_limit": item_limit, "network_call": bool(execute and ready_items), "model_call": bool(execute and ready_items), "database_transaction_read_only": True, "database_write": False, "ai_insights_write": False, "price_table_write": False, "artifact_write": bool(execute and write_receipt), "rollback_terminal": "transaction_rollback_after_read_only_pgvector_probe", "independent_verifier": "pgvector_read_only_similarity_probe", }, "closure_receipt": { "sensor_source_receipt": bool(source_items), "normalized_asset_identity": bool(source_items) and all( item.get("platform") and item.get("manifest_id") for item in source_items ), "source_of_truth_diff_recorded": True, "ai_candidate_decision_recorded": True, "risk_policy_decision": "medium_bounded_read_only_canary", "check_mode_passed": bool(ready_items), "bounded_execution_performed": bool(executed_items), "independent_verifier_passed": ( bool(executed_items) and canary_passed_count == len(executed_items) ), "rollback_or_no_write_terminal": ( "transaction_rollback_after_read_only_pgvector_probe" if executed_items else "no_write_terminal" ), "telegram_acknowledgement": "pending_scheduler_dispatch" if execute else "not_applicable_dry_run", "learning_write_acknowledgement": "rag_canary_receipt_written" if executed_items and write_receipt else "no_learning_write", }, "next_machine_action": next_action, } __all__ = [ "CANARY_VERSION", "POLICY", "run_internal_rag_candidate_canary", ]