"""Read-only inventory for external MCP/RAG absorption into internal control planes.""" from __future__ import annotations import os from copy import deepcopy from datetime import datetime, timezone from typing import Any POLICY = "read_only_external_mcp_rag_integration_readback_v1" def _enabled(value: str | None) -> bool: return str(value or "").strip().lower() in {"1", "true", "yes", "on"} def _controlled_apply_boundary() -> dict[str, bool]: return { "network_call": False, "db_write": False, "secret_read": False, "production_price_write": False, "external_side_effect": False, } def _status_counts(items: list[dict[str, Any]]) -> dict[str, int]: counts: dict[str, int] = {} for item in items: status = str(item.get("status") or "unknown") counts[status] = counts.get(status, 0) + 1 return counts def _mcp_runtime_snapshot() -> dict[str, Any]: try: from services.mcp_router import MCP_BASE_HOSTS, TOOL_REGISTRY, is_mcp_router_enabled return { "enabled": is_mcp_router_enabled(), "servers": deepcopy(MCP_BASE_HOSTS), "caller_count": len(TOOL_REGISTRY), "tool_registry": { caller: { server: list(tools) for server, tools in servers.items() } for caller, servers in TOOL_REGISTRY.items() }, } except Exception as exc: return { "enabled": False, "servers": {}, "caller_count": 0, "tool_registry": {}, "error": str(exc)[:300], } def _rag_runtime_snapshot() -> dict[str, Any]: try: from services.rag_service import ( RAG_DEFAULT_THRESHOLD, RAG_DEFAULT_TOP_K, RAG_EMBED_DIM, RAG_EMBED_MODEL, get_embedding_signature, is_rag_enabled, ) return { "enabled": is_rag_enabled(), "vector_store": "pgvector", "embedding_model": RAG_EMBED_MODEL, "embedding_dim": RAG_EMBED_DIM, "embedding_signature": get_embedding_signature(), "default_top_k": RAG_DEFAULT_TOP_K, "default_threshold": RAG_DEFAULT_THRESHOLD, } except Exception as exc: return { "enabled": False, "vector_store": "pgvector", "embedding_model": "", "embedding_dim": 0, "embedding_signature": "", "error": str(exc)[:300], } def _pixelrag_snapshot() -> dict[str, Any]: try: from services.pixelrag_crawler_integration_service import ECOMMERCE_PLATFORM_PROFILES platforms = sorted( platform for platform in ECOMMERCE_PLATFORM_PROFILES if platform not in {"market_intel", "external_market"} ) return { "enabled": True, "platform_count": len(platforms), "platforms": platforms, "visual_rag_stage": "phase1_visual_evidence_receipts", } except Exception as exc: return { "enabled": False, "platform_count": 0, "platforms": [], "visual_rag_stage": "unavailable", "error": str(exc)[:300], } def _capability_inventory() -> list[dict[str, Any]]: return [ { "id": "mcp.omnisearch.tavily_exa", "family": "external_mcp", "external_capability": "Tavily / Exa style public web search through omnisearch MCP", "internal_plane": "mcp_router", "internal_entrypoint": "services.mcp_router.TOOL_REGISTRY[mcp_collector][omnisearch]", "status": "integrated_self_hosted_gateway", "writes": ["mcp_calls_log_async"], "data_boundary": "public_search_summary_only", "next_machine_action": "keep_mcp_router_health_and_cache_readback", }, { "id": "mcp.firecrawl.scrape", "family": "external_mcp", "external_capability": "Firecrawl style public page scrape", "internal_plane": "mcp_router", "internal_entrypoint": "services.mcp_router.TOOL_REGISTRY[*][firecrawl].scrape_url", "status": "integrated_self_hosted_gateway", "writes": ["mcp_calls_log_async"], "data_boundary": "approved_public_url_only", "next_machine_action": "enforce_source_contract_before_scheduler_attach", }, { "id": "mcp.postgres.query", "family": "internal_mcp", "external_capability": "SQL read interface exposed as MCP tool", "internal_plane": "mcp_router", "internal_entrypoint": "postgres.query for approved callers", "status": "integrated_read_only_contract", "writes": ["mcp_calls_log_async"], "data_boundary": "read_only_query_allowlist", "next_machine_action": "keep_disallowing_unknown_callers_and_write_tools", }, { "id": "mcp.filesystem.readonly", "family": "internal_mcp", "external_capability": "Filesystem MCP read diagnostics", "internal_plane": "mcp_router", "internal_entrypoint": "ops_diagnostics filesystem read-only tools", "status": "integrated_read_only_contract", "writes": ["mcp_calls_log_async"], "data_boundary": "allowed_directories_read_only", "next_machine_action": "keep_write_file_and_mutation_tools_rejected", }, { "id": "rag.pixelrag.visual_evidence", "family": "external_rag", "external_capability": "PixelRAG page screenshot and tile retrieval pattern", "internal_plane": "pixelrag_visual_evidence + internal RAG candidate lane", "internal_entrypoint": "services.pixelrag_crawler_integration_service", "status": "integrated_phase1_visual_receipts", "writes": ["artifact_file_only"], "data_boundary": "public_page_screenshot_tiles_no_price_write", "next_machine_action": "add_ocr_vlm_replay_before_promoting_to_pgvector_rag", }, { "id": "rag.pgvector.bge_m3", "family": "internal_rag", "external_capability": "General text RAG retrieval pattern", "internal_plane": "rag_service", "internal_entrypoint": "services.rag_service.RAGService.query", "status": "internal_primary_ready", "writes": ["rag_query_log_async"], "data_boundary": "ai_insights_pgvector_embedding_signature_guard", "next_machine_action": "keep_embedding_signature_and_feedback_guardrails", }, { "id": "rag.qwen3_vl_embedding", "family": "external_rag", "external_capability": "Multimodal visual embedding model for tile retrieval", "internal_plane": "ollama_first_visual_embedding_benchmark", "internal_entrypoint": "not_enabled_in_production", "status": "deferred_until_ollama_first_verified", "writes": [], "data_boundary": "no_hosted_visual_embedding_api", "next_machine_action": "benchmark_local_multimodal_embedding_then_design_pgvector_metadata", }, { "id": "rag.faiss_visual_index", "family": "external_rag", "external_capability": "FAISS visual retrieval index used by some PixelRAG pipelines", "internal_plane": "pgvector_first_policy", "internal_entrypoint": "not_enabled_in_production", "status": "rejected_for_production_without_adr", "writes": [], "data_boundary": "pgvector_is_the_only_production_vector_store", "next_machine_action": "use_pgvector_compatible_metadata_or_write_adr_before_any_faiss_trial", }, { "id": "mcp.gemini_grounding", "family": "external_mcp", "external_capability": "Gemini grounding as hosted external search fallback", "internal_plane": "mcp_collector_final_fallback", "internal_entrypoint": "services.mcp_collector_service guarded fallback", "status": "disabled_by_default_fallback_only", "writes": ["mcp_cache_only_when_explicitly_enabled"], "data_boundary": "gemini_api_hard_disabled_by_default", "next_machine_action": "prefer_self_hosted_mcp_or_ollama_static_fallback", }, ] def build_external_mcp_rag_integration_readback( *, target_family: str | None = None, ) -> dict[str, Any]: """Describe how external MCP/RAG capabilities are absorbed internally.""" family_filter = str(target_family or "").strip().lower() inventory = _capability_inventory() if family_filter: inventory = [ item for item in inventory if str(item.get("family") or "").lower() == family_filter ] counts = _status_counts(inventory) unresolved = [ item for item in inventory if str(item.get("status") or "").startswith(("deferred", "rejected", "disabled")) ] absorbed = [ item for item in inventory if item not in unresolved ] all_absorbed = len(unresolved) == 0 return { "success": True, "policy": POLICY, "generated_at": datetime.now(timezone.utc).isoformat(), "status": "partially_integrated" if not all_absorbed else "fully_integrated", "answer_to_owner": ( "不是全部完成；已把主要外部 MCP/RAG 放進內部可治理 registry，" "其中 self-hosted MCP、pgvector RAG、PixelRAG phase1 已整合，" "多模態 embedding / FAISS / Gemini grounding 仍需條件或維持停用。" if not all_absorbed else "目前 registry 內的外部 MCP/RAG 都已有內部治理落點。" ), "completion": { "total_capabilities": len(inventory), "absorbed_count": len(absorbed), "unresolved_count": len(unresolved), "all_absorbed": all_absorbed, "status_counts": counts, }, "runtime": { "mcp": _mcp_runtime_snapshot(), "rag": _rag_runtime_snapshot(), "pixelrag": _pixelrag_snapshot(), "env_flags": { "MCP_ROUTER_ENABLED": _enabled(os.getenv("MCP_ROUTER_ENABLED")), "RAG_ENABLED": _enabled(os.getenv("RAG_ENABLED")), "GEMINI_API_HARD_DISABLED": _enabled(os.getenv("GEMINI_API_HARD_DISABLED", "true")), }, }, "inventory": inventory, "next_machine_actions": [ { "priority": "P0", "action": "wire_pixelrag_receipts_to_internal_rag_candidate_replay", "status": "ready_after_ocr_vlm_replay_contract", }, { "priority": "P0", "action": "add_mcp_router_health_readback_to_ai_automation_smoke", "status": "ready", }, { "priority": "P1", "action": "benchmark_ollama_first_visual_embedding", "status": "blocked_until_model_runtime_verified", }, { "priority": "P1", "action": "promote_successful_marketplace_visual_receipts_to_source_contracts", "status": "ready_for_shopee_warning_for_coupang", }, ], "controlled_apply": _controlled_apply_boundary(), } __all__ = [ "POLICY", "build_external_mcp_rag_integration_readback", ]