"""Read-only inventory for external MCP/RAG absorption into internal control planes.""" from __future__ import annotations import os import hashlib import json from copy import deepcopy from datetime import datetime, timezone from typing import Any from zoneinfo import ZoneInfo POLICY = "read_only_external_mcp_rag_integration_readback_v1" PUBLIC_FEDERATION_SCHEMA_VERSION = "ewoooc_mcp_federation_receipt_v1" PUBLIC_FEDERATION_POLICY = "public_aggregate_read_only_mcp_federation_v1" TAIPEI_TZ = ZoneInfo("Asia/Taipei") _FEDERATED_PRODUCTS = ( { "product_id": "ewoooc", "canonical_repo": "wooo/ewoooc", "runtime_role": "mcp_rag_automation_source_plane", }, { "product_id": "momo-pro-system", "canonical_repo": "wooo/momo-pro-system", "runtime_role": "commerce_runtime_consumer_plane", }, ) def _enabled(value: str | None) -> bool: return str(value or "").strip().lower() in {"1", "true", "yes", "on"} def _controlled_apply_boundary() -> dict[str, bool]: return { "network_call": False, "db_write": False, "secret_read": False, "production_price_write": False, "external_side_effect": False, } def _public_operation_boundaries() -> dict[str, bool]: """V-New: 公開 receipt 只允許無副作用的聚合狀態讀回。""" return { "read_only_observation_allowed": True, "network_call_allowed": False, "db_write_allowed": False, "secret_read_allowed": False, "external_tool_call_allowed": False, "rag_write_allowed": False, "production_price_write_allowed": False, "request_or_response_body_storage_allowed": False, } def _safe_mcp_summary(snapshot: dict[str, Any]) -> dict[str, Any]: servers = snapshot.get("servers") servers = servers if isinstance(servers, dict) else {} registry = snapshot.get("tool_registry") registry = registry if isinstance(registry, dict) else {} tool_count = 0 for server_map in registry.values(): if not isinstance(server_map, dict): continue for tools in server_map.values(): if isinstance(tools, (list, tuple, set)): tool_count += len(tools) return { "enabled": snapshot.get("enabled") is True, "server_count": len(servers), "server_ids": sorted(str(server_id) for server_id in servers), "caller_count": int(snapshot.get("caller_count") or 0), "tool_count": tool_count, } def _safe_rag_summary(snapshot: dict[str, Any]) -> dict[str, Any]: model = str(snapshot.get("embedding_model") or "") return { "enabled": snapshot.get("enabled") is True, "vector_store": str(snapshot.get("vector_store") or "pgvector"), "embedding_model": model, "embedding_dim": int(snapshot.get("embedding_dim") or 0), "embedding_signature": str(snapshot.get("embedding_signature") or ""), "embedding_version_state": ( "floating_tag_detected" if model.endswith(":latest") else "explicit_or_unknown" ), } def _safe_pixelrag_summary(snapshot: dict[str, Any]) -> dict[str, Any]: return { "enabled": snapshot.get("enabled") is True, "platform_count": int(snapshot.get("platform_count") or 0), "platforms": sorted( str(platform) for platform in snapshot.get("platforms") or [] ), "visual_rag_stage": str( snapshot.get("visual_rag_stage") or "unavailable" ), "formal_product_write_allowed": False, } def compute_public_federation_receipt_fingerprint( receipt: dict[str, Any], ) -> str: """Return the verifier-recomputable hash for normalized public fields.""" covered = { key: value for key, value in receipt.items() if key not in {"receipt_id", "observed_at", "integrity"} } canonical = json.dumps( covered, ensure_ascii=False, sort_keys=True, separators=(",", ":"), ) return hashlib.sha256(canonical.encode("utf-8")).hexdigest() def build_public_mcp_federation_receipt( *, runtime_version: str, observed_at: str | None = None, ) -> dict[str, Any]: """Build two product-scoped, aggregate-only MCP/RAG runtime receipts. This function performs no network call, database read/write, tool call, or RAG write. Internal endpoint URLs and raw tool registries are deliberately reduced to stable IDs and counts before reaching the public response. """ timestamp = observed_at or datetime.now(TAIPEI_TZ).isoformat(timespec="seconds") mcp = _safe_mcp_summary(_mcp_runtime_snapshot()) rag = _safe_rag_summary(_rag_runtime_snapshot()) pixelrag = _safe_pixelrag_summary(_pixelrag_snapshot()) blockers: list[str] = [] if not mcp["enabled"]: blockers.append("mcp_router_runtime_not_enabled") if mcp["server_count"] == 0 or mcp["tool_count"] == 0: blockers.append("mcp_runtime_inventory_empty") if not rag["enabled"]: blockers.append("rag_runtime_not_enabled") if not rag["embedding_signature"]: blockers.append("rag_embedding_signature_missing") if rag["embedding_version_state"] == "floating_tag_detected": blockers.append("rag_embedding_model_not_immutable") if not pixelrag["enabled"]: blockers.append("pixelrag_runtime_not_enabled") blockers = sorted(blockers) normalized_runtime = { "mcp": mcp, "rag": rag, "pixelrag": pixelrag, } receipts: list[dict[str, Any]] = [] for product in _FEDERATED_PRODUCTS: receipt: dict[str, Any] = { "receipt_schema_version": PUBLIC_FEDERATION_SCHEMA_VERSION, "product_id": product["product_id"], "canonical_repo": product["canonical_repo"], "runtime_role": product["runtime_role"], "runtime_version": str(runtime_version or "unknown"), "health_status": "ready" if not blockers else "partial_degraded", "runtime": deepcopy(normalized_runtime), "data_boundaries": { "scope": "aggregate_runtime_metadata_only", "pixelrag": "public_visual_receipts_no_formal_product_write", "rag": "pgvector_internal_only_no_public_content", "mcp": "server_ids_and_counts_only_no_endpoint_or_payload", }, "operation_boundaries": _public_operation_boundaries(), "blockers": list(blockers), } fingerprint = compute_public_federation_receipt_fingerprint(receipt) receipt["observed_at"] = timestamp receipt["receipt_id"] = ( f"mcp-federation:{product['product_id']}:{fingerprint[:16]}" ) receipt["integrity"] = { "algorithm": "sha256", "canonicalization": "json_sort_keys_compact_v1", "excluded_fields": ["receipt_id", "observed_at", "integrity"], "normalized_fingerprint_sha256": fingerprint, } receipts.append(receipt) return { "success": True, "schema_version": PUBLIC_FEDERATION_SCHEMA_VERSION, "policy": PUBLIC_FEDERATION_POLICY, "generated_at": timestamp, "status": "ready" if not blockers else "partial_degraded", "receipt_count": len(receipts), "receipts": receipts, "blockers": blockers, "operation_boundaries": _public_operation_boundaries(), "next_machine_action": ( "pin_embedding_model_and_verify_runtime_receipts" if blockers else "continue_scheduled_read_only_federation" ), } def _status_counts(items: list[dict[str, Any]]) -> dict[str, int]: counts: dict[str, int] = {} for item in items: status = str(item.get("status") or "unknown") counts[status] = counts.get(status, 0) + 1 return counts def _mcp_runtime_snapshot() -> dict[str, Any]: try: from services.mcp_router import MCP_BASE_HOSTS, TOOL_REGISTRY, is_mcp_router_enabled return { "enabled": is_mcp_router_enabled(), "servers": deepcopy(MCP_BASE_HOSTS), "caller_count": len(TOOL_REGISTRY), "tool_registry": { caller: { server: list(tools) for server, tools in servers.items() } for caller, servers in TOOL_REGISTRY.items() }, } except Exception as exc: return { "enabled": False, "servers": {}, "caller_count": 0, "tool_registry": {}, "error": str(exc)[:300], } def _rag_runtime_snapshot() -> dict[str, Any]: try: from services.rag_service import ( RAG_DEFAULT_THRESHOLD, RAG_DEFAULT_TOP_K, RAG_EMBED_DIM, RAG_EMBED_MODEL, get_embedding_signature, is_rag_enabled, ) return { "enabled": is_rag_enabled(), "vector_store": "pgvector", "embedding_model": RAG_EMBED_MODEL, "embedding_dim": RAG_EMBED_DIM, "embedding_signature": get_embedding_signature(), "default_top_k": RAG_DEFAULT_TOP_K, "default_threshold": RAG_DEFAULT_THRESHOLD, } except Exception as exc: return { "enabled": False, "vector_store": "pgvector", "embedding_model": "", "embedding_dim": 0, "embedding_signature": "", "error": str(exc)[:300], } def _pixelrag_snapshot() -> dict[str, Any]: try: from services.pixelrag_crawler_integration_service import ECOMMERCE_PLATFORM_PROFILES platforms = sorted( platform for platform in ECOMMERCE_PLATFORM_PROFILES if platform not in {"market_intel", "external_market"} ) return { "enabled": True, "platform_count": len(platforms), "platforms": platforms, "visual_rag_stage": "phase1_visual_evidence_receipts", } except Exception as exc: return { "enabled": False, "platform_count": 0, "platforms": [], "visual_rag_stage": "unavailable", "error": str(exc)[:300], } def _capability_inventory() -> list[dict[str, Any]]: return [ { "id": "mcp.omnisearch.tavily_exa", "family": "external_mcp", "external_capability": "Tavily / Exa style public web search through omnisearch MCP", "internal_plane": "mcp_router", "internal_entrypoint": "services.mcp_router.TOOL_REGISTRY[mcp_collector][omnisearch]", "status": "integrated_self_hosted_gateway", "writes": ["mcp_calls_log_async"], "data_boundary": "public_search_summary_only", "next_machine_action": "keep_mcp_router_health_and_cache_readback", }, { "id": "mcp.firecrawl.scrape", "family": "external_mcp", "external_capability": "Firecrawl style public page scrape", "internal_plane": "mcp_router", "internal_entrypoint": "services.mcp_router.TOOL_REGISTRY[*][firecrawl].scrape_url", "status": "integrated_self_hosted_gateway", "writes": ["mcp_calls_log_async"], "data_boundary": "approved_public_url_only", "next_machine_action": "enforce_source_contract_before_scheduler_attach", }, { "id": "mcp.postgres.query", "family": "internal_mcp", "external_capability": "SQL read interface exposed as MCP tool", "internal_plane": "mcp_router", "internal_entrypoint": "postgres.query for approved callers", "status": "integrated_read_only_contract", "writes": ["mcp_calls_log_async"], "data_boundary": "read_only_query_allowlist", "next_machine_action": "keep_disallowing_unknown_callers_and_write_tools", }, { "id": "mcp.filesystem.readonly", "family": "internal_mcp", "external_capability": "Filesystem MCP read diagnostics", "internal_plane": "mcp_router", "internal_entrypoint": "ops_diagnostics filesystem read-only tools", "status": "integrated_read_only_contract", "writes": ["mcp_calls_log_async"], "data_boundary": "allowed_directories_read_only", "next_machine_action": "keep_write_file_and_mutation_tools_rejected", }, { "id": "rag.pixelrag.visual_evidence", "family": "external_rag", "external_capability": "PixelRAG page screenshot and tile retrieval pattern", "internal_plane": "pixelrag_visual_evidence + internal RAG candidate lane", "internal_entrypoint": "services.pixelrag_crawler_integration_service", "status": "integrated_phase1_visual_receipts", "writes": ["artifact_file_only"], "data_boundary": "public_page_screenshot_tiles_no_price_write", "next_machine_action": "add_ocr_vlm_replay_before_promoting_to_pgvector_rag", }, { "id": "rag.pgvector.bge_m3", "family": "internal_rag", "external_capability": "General text RAG retrieval pattern", "internal_plane": "rag_service", "internal_entrypoint": "services.rag_service.RAGService.query", "status": "internal_primary_ready", "writes": ["rag_query_log_async"], "data_boundary": "ai_insights_pgvector_embedding_signature_guard", "next_machine_action": "keep_embedding_signature_and_feedback_guardrails", }, { "id": "rag.qwen3_vl_embedding", "family": "external_rag", "external_capability": "Multimodal visual embedding model for tile retrieval", "internal_plane": "ollama_first_visual_embedding_benchmark", "internal_entrypoint": "not_enabled_in_production", "status": "deferred_until_ollama_first_verified", "writes": [], "data_boundary": "no_hosted_visual_embedding_api", "next_machine_action": "benchmark_local_multimodal_embedding_then_design_pgvector_metadata", }, { "id": "rag.faiss_visual_index", "family": "external_rag", "external_capability": "FAISS visual retrieval index used by some PixelRAG pipelines", "internal_plane": "pgvector_first_policy", "internal_entrypoint": "not_enabled_in_production", "status": "rejected_for_production_without_adr", "writes": [], "data_boundary": "pgvector_is_the_only_production_vector_store", "next_machine_action": "use_pgvector_compatible_metadata_or_write_adr_before_any_faiss_trial", }, { "id": "mcp.gemini_grounding", "family": "external_mcp", "external_capability": "Gemini grounding as hosted external search fallback", "internal_plane": "mcp_collector_final_fallback", "internal_entrypoint": "services.mcp_collector_service guarded fallback", "status": "disabled_by_default_fallback_only", "writes": ["mcp_cache_only_when_explicitly_enabled"], "data_boundary": "gemini_api_hard_disabled_by_default", "next_machine_action": "prefer_self_hosted_mcp_or_ollama_static_fallback", }, ] def build_external_mcp_rag_integration_readback( *, target_family: str | None = None, ) -> dict[str, Any]: """Describe how external MCP/RAG capabilities are absorbed internally.""" family_filter = str(target_family or "").strip().lower() inventory = _capability_inventory() if family_filter: inventory = [ item for item in inventory if str(item.get("family") or "").lower() == family_filter ] counts = _status_counts(inventory) unresolved = [ item for item in inventory if str(item.get("status") or "").startswith(("deferred", "rejected", "disabled")) ] absorbed = [ item for item in inventory if item not in unresolved ] all_absorbed = len(unresolved) == 0 return { "success": True, "policy": POLICY, "generated_at": datetime.now(timezone.utc).isoformat(), "status": "partially_integrated" if not all_absorbed else "fully_integrated", "answer_to_owner": ( "不是全部完成；已把主要外部 MCP/RAG 放進內部可治理 registry，" "其中 self-hosted MCP、pgvector RAG、PixelRAG phase1 已整合，" "多模態 embedding / FAISS / Gemini grounding 仍需條件或維持停用。" if not all_absorbed else "目前 registry 內的外部 MCP/RAG 都已有內部治理落點。" ), "completion": { "total_capabilities": len(inventory), "absorbed_count": len(absorbed), "unresolved_count": len(unresolved), "all_absorbed": all_absorbed, "status_counts": counts, }, "runtime": { "mcp": _mcp_runtime_snapshot(), "rag": _rag_runtime_snapshot(), "pixelrag": _pixelrag_snapshot(), "env_flags": { "MCP_ROUTER_ENABLED": _enabled(os.getenv("MCP_ROUTER_ENABLED")), "RAG_ENABLED": _enabled(os.getenv("RAG_ENABLED")), "GEMINI_API_HARD_DISABLED": _enabled(os.getenv("GEMINI_API_HARD_DISABLED", "true")), }, }, "inventory": inventory, "next_machine_actions": [ { "priority": "P0", "action": "wire_pixelrag_receipts_to_internal_rag_candidate_replay", "status": "ready_after_ocr_vlm_replay_contract", }, { "priority": "P0", "action": "add_mcp_router_health_readback_to_ai_automation_smoke", "status": "ready", }, { "priority": "P1", "action": "benchmark_ollama_first_visual_embedding", "status": "blocked_until_model_runtime_verified", }, { "priority": "P1", "action": "promote_successful_marketplace_visual_receipts_to_source_contracts", "status": "ready_for_shopee_warning_for_coupang", }, ], "controlled_apply": _controlled_apply_boundary(), } __all__ = [ "POLICY", "PUBLIC_FEDERATION_POLICY", "PUBLIC_FEDERATION_SCHEMA_VERSION", "build_external_mcp_rag_integration_readback", "build_public_mcp_federation_receipt", "compute_public_federation_receipt_fingerprint", ]