309 lines
12 KiB
Python
309 lines
12 KiB
Python
"""Read-only inventory for external MCP/RAG absorption into internal control planes."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from copy import deepcopy
|
||
from datetime import datetime, timezone
|
||
from typing import Any
|
||
|
||
|
||
POLICY = "read_only_external_mcp_rag_integration_readback_v1"
|
||
|
||
|
||
def _enabled(value: str | None) -> bool:
|
||
return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
|
||
|
||
|
||
def _controlled_apply_boundary() -> dict[str, bool]:
|
||
return {
|
||
"network_call": False,
|
||
"db_write": False,
|
||
"secret_read": False,
|
||
"production_price_write": False,
|
||
"external_side_effect": False,
|
||
}
|
||
|
||
|
||
def _status_counts(items: list[dict[str, Any]]) -> dict[str, int]:
|
||
counts: dict[str, int] = {}
|
||
for item in items:
|
||
status = str(item.get("status") or "unknown")
|
||
counts[status] = counts.get(status, 0) + 1
|
||
return counts
|
||
|
||
|
||
def _mcp_runtime_snapshot() -> dict[str, Any]:
|
||
try:
|
||
from services.mcp_router import MCP_BASE_HOSTS, TOOL_REGISTRY, is_mcp_router_enabled
|
||
|
||
return {
|
||
"enabled": is_mcp_router_enabled(),
|
||
"servers": deepcopy(MCP_BASE_HOSTS),
|
||
"caller_count": len(TOOL_REGISTRY),
|
||
"tool_registry": {
|
||
caller: {
|
||
server: list(tools)
|
||
for server, tools in servers.items()
|
||
}
|
||
for caller, servers in TOOL_REGISTRY.items()
|
||
},
|
||
}
|
||
except Exception as exc:
|
||
return {
|
||
"enabled": False,
|
||
"servers": {},
|
||
"caller_count": 0,
|
||
"tool_registry": {},
|
||
"error": str(exc)[:300],
|
||
}
|
||
|
||
|
||
def _rag_runtime_snapshot() -> dict[str, Any]:
|
||
try:
|
||
from services.rag_service import (
|
||
RAG_DEFAULT_THRESHOLD,
|
||
RAG_DEFAULT_TOP_K,
|
||
RAG_EMBED_DIM,
|
||
RAG_EMBED_MODEL,
|
||
get_embedding_signature,
|
||
is_rag_enabled,
|
||
)
|
||
|
||
return {
|
||
"enabled": is_rag_enabled(),
|
||
"vector_store": "pgvector",
|
||
"embedding_model": RAG_EMBED_MODEL,
|
||
"embedding_dim": RAG_EMBED_DIM,
|
||
"embedding_signature": get_embedding_signature(),
|
||
"default_top_k": RAG_DEFAULT_TOP_K,
|
||
"default_threshold": RAG_DEFAULT_THRESHOLD,
|
||
}
|
||
except Exception as exc:
|
||
return {
|
||
"enabled": False,
|
||
"vector_store": "pgvector",
|
||
"embedding_model": "",
|
||
"embedding_dim": 0,
|
||
"embedding_signature": "",
|
||
"error": str(exc)[:300],
|
||
}
|
||
|
||
|
||
def _pixelrag_snapshot() -> dict[str, Any]:
|
||
try:
|
||
from services.pixelrag_crawler_integration_service import ECOMMERCE_PLATFORM_PROFILES
|
||
|
||
platforms = sorted(
|
||
platform
|
||
for platform in ECOMMERCE_PLATFORM_PROFILES
|
||
if platform not in {"market_intel", "external_market"}
|
||
)
|
||
return {
|
||
"enabled": True,
|
||
"platform_count": len(platforms),
|
||
"platforms": platforms,
|
||
"visual_rag_stage": "phase1_visual_evidence_receipts",
|
||
}
|
||
except Exception as exc:
|
||
return {
|
||
"enabled": False,
|
||
"platform_count": 0,
|
||
"platforms": [],
|
||
"visual_rag_stage": "unavailable",
|
||
"error": str(exc)[:300],
|
||
}
|
||
|
||
|
||
def _capability_inventory() -> list[dict[str, Any]]:
|
||
return [
|
||
{
|
||
"id": "mcp.omnisearch.tavily_exa",
|
||
"family": "external_mcp",
|
||
"external_capability": "Tavily / Exa style public web search through omnisearch MCP",
|
||
"internal_plane": "mcp_router",
|
||
"internal_entrypoint": "services.mcp_router.TOOL_REGISTRY[mcp_collector][omnisearch]",
|
||
"status": "integrated_self_hosted_gateway",
|
||
"writes": ["mcp_calls_log_async"],
|
||
"data_boundary": "public_search_summary_only",
|
||
"next_machine_action": "keep_mcp_router_health_and_cache_readback",
|
||
},
|
||
{
|
||
"id": "mcp.firecrawl.scrape",
|
||
"family": "external_mcp",
|
||
"external_capability": "Firecrawl style public page scrape",
|
||
"internal_plane": "mcp_router",
|
||
"internal_entrypoint": "services.mcp_router.TOOL_REGISTRY[*][firecrawl].scrape_url",
|
||
"status": "integrated_self_hosted_gateway",
|
||
"writes": ["mcp_calls_log_async"],
|
||
"data_boundary": "approved_public_url_only",
|
||
"next_machine_action": "enforce_source_contract_before_scheduler_attach",
|
||
},
|
||
{
|
||
"id": "mcp.postgres.query",
|
||
"family": "internal_mcp",
|
||
"external_capability": "SQL read interface exposed as MCP tool",
|
||
"internal_plane": "mcp_router",
|
||
"internal_entrypoint": "postgres.query for approved callers",
|
||
"status": "integrated_read_only_contract",
|
||
"writes": ["mcp_calls_log_async"],
|
||
"data_boundary": "read_only_query_allowlist",
|
||
"next_machine_action": "keep_disallowing_unknown_callers_and_write_tools",
|
||
},
|
||
{
|
||
"id": "mcp.filesystem.readonly",
|
||
"family": "internal_mcp",
|
||
"external_capability": "Filesystem MCP read diagnostics",
|
||
"internal_plane": "mcp_router",
|
||
"internal_entrypoint": "ops_diagnostics filesystem read-only tools",
|
||
"status": "integrated_read_only_contract",
|
||
"writes": ["mcp_calls_log_async"],
|
||
"data_boundary": "allowed_directories_read_only",
|
||
"next_machine_action": "keep_write_file_and_mutation_tools_rejected",
|
||
},
|
||
{
|
||
"id": "rag.pixelrag.visual_evidence",
|
||
"family": "external_rag",
|
||
"external_capability": "PixelRAG page screenshot and tile retrieval pattern",
|
||
"internal_plane": "pixelrag_visual_evidence + internal RAG candidate lane",
|
||
"internal_entrypoint": "services.pixelrag_crawler_integration_service",
|
||
"status": "integrated_phase1_visual_receipts",
|
||
"writes": ["artifact_file_only"],
|
||
"data_boundary": "public_page_screenshot_tiles_no_price_write",
|
||
"next_machine_action": "add_ocr_vlm_replay_before_promoting_to_pgvector_rag",
|
||
},
|
||
{
|
||
"id": "rag.pgvector.bge_m3",
|
||
"family": "internal_rag",
|
||
"external_capability": "General text RAG retrieval pattern",
|
||
"internal_plane": "rag_service",
|
||
"internal_entrypoint": "services.rag_service.RAGService.query",
|
||
"status": "internal_primary_ready",
|
||
"writes": ["rag_query_log_async"],
|
||
"data_boundary": "ai_insights_pgvector_embedding_signature_guard",
|
||
"next_machine_action": "keep_embedding_signature_and_feedback_guardrails",
|
||
},
|
||
{
|
||
"id": "rag.qwen3_vl_embedding",
|
||
"family": "external_rag",
|
||
"external_capability": "Multimodal visual embedding model for tile retrieval",
|
||
"internal_plane": "ollama_first_visual_embedding_benchmark",
|
||
"internal_entrypoint": "not_enabled_in_production",
|
||
"status": "deferred_until_ollama_first_verified",
|
||
"writes": [],
|
||
"data_boundary": "no_hosted_visual_embedding_api",
|
||
"next_machine_action": "benchmark_local_multimodal_embedding_then_design_pgvector_metadata",
|
||
},
|
||
{
|
||
"id": "rag.faiss_visual_index",
|
||
"family": "external_rag",
|
||
"external_capability": "FAISS visual retrieval index used by some PixelRAG pipelines",
|
||
"internal_plane": "pgvector_first_policy",
|
||
"internal_entrypoint": "not_enabled_in_production",
|
||
"status": "rejected_for_production_without_adr",
|
||
"writes": [],
|
||
"data_boundary": "pgvector_is_the_only_production_vector_store",
|
||
"next_machine_action": "use_pgvector_compatible_metadata_or_write_adr_before_any_faiss_trial",
|
||
},
|
||
{
|
||
"id": "mcp.gemini_grounding",
|
||
"family": "external_mcp",
|
||
"external_capability": "Gemini grounding as hosted external search fallback",
|
||
"internal_plane": "mcp_collector_final_fallback",
|
||
"internal_entrypoint": "services.mcp_collector_service guarded fallback",
|
||
"status": "disabled_by_default_fallback_only",
|
||
"writes": ["mcp_cache_only_when_explicitly_enabled"],
|
||
"data_boundary": "gemini_api_hard_disabled_by_default",
|
||
"next_machine_action": "prefer_self_hosted_mcp_or_ollama_static_fallback",
|
||
},
|
||
]
|
||
|
||
|
||
def build_external_mcp_rag_integration_readback(
|
||
*,
|
||
target_family: str | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Describe how external MCP/RAG capabilities are absorbed internally."""
|
||
family_filter = str(target_family or "").strip().lower()
|
||
inventory = _capability_inventory()
|
||
if family_filter:
|
||
inventory = [
|
||
item
|
||
for item in inventory
|
||
if str(item.get("family") or "").lower() == family_filter
|
||
]
|
||
|
||
counts = _status_counts(inventory)
|
||
unresolved = [
|
||
item
|
||
for item in inventory
|
||
if str(item.get("status") or "").startswith(("deferred", "rejected", "disabled"))
|
||
]
|
||
absorbed = [
|
||
item
|
||
for item in inventory
|
||
if item not in unresolved
|
||
]
|
||
all_absorbed = len(unresolved) == 0
|
||
|
||
return {
|
||
"success": True,
|
||
"policy": POLICY,
|
||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||
"status": "partially_integrated" if not all_absorbed else "fully_integrated",
|
||
"answer_to_owner": (
|
||
"不是全部完成；已把主要外部 MCP/RAG 放進內部可治理 registry，"
|
||
"其中 self-hosted MCP、pgvector RAG、PixelRAG phase1 已整合，"
|
||
"多模態 embedding / FAISS / Gemini grounding 仍需條件或維持停用。"
|
||
if not all_absorbed
|
||
else "目前 registry 內的外部 MCP/RAG 都已有內部治理落點。"
|
||
),
|
||
"completion": {
|
||
"total_capabilities": len(inventory),
|
||
"absorbed_count": len(absorbed),
|
||
"unresolved_count": len(unresolved),
|
||
"all_absorbed": all_absorbed,
|
||
"status_counts": counts,
|
||
},
|
||
"runtime": {
|
||
"mcp": _mcp_runtime_snapshot(),
|
||
"rag": _rag_runtime_snapshot(),
|
||
"pixelrag": _pixelrag_snapshot(),
|
||
"env_flags": {
|
||
"MCP_ROUTER_ENABLED": _enabled(os.getenv("MCP_ROUTER_ENABLED")),
|
||
"RAG_ENABLED": _enabled(os.getenv("RAG_ENABLED")),
|
||
"GEMINI_API_HARD_DISABLED": _enabled(os.getenv("GEMINI_API_HARD_DISABLED", "true")),
|
||
},
|
||
},
|
||
"inventory": inventory,
|
||
"next_machine_actions": [
|
||
{
|
||
"priority": "P0",
|
||
"action": "wire_pixelrag_receipts_to_internal_rag_candidate_replay",
|
||
"status": "ready_after_ocr_vlm_replay_contract",
|
||
},
|
||
{
|
||
"priority": "P0",
|
||
"action": "add_mcp_router_health_readback_to_ai_automation_smoke",
|
||
"status": "ready",
|
||
},
|
||
{
|
||
"priority": "P1",
|
||
"action": "benchmark_ollama_first_visual_embedding",
|
||
"status": "blocked_until_model_runtime_verified",
|
||
},
|
||
{
|
||
"priority": "P1",
|
||
"action": "promote_successful_marketplace_visual_receipts_to_source_contracts",
|
||
"status": "ready_for_shopee_warning_for_coupang",
|
||
},
|
||
],
|
||
"controlled_apply": _controlled_apply_boundary(),
|
||
}
|
||
|
||
|
||
__all__ = [
|
||
"POLICY",
|
||
"build_external_mcp_rag_integration_readback",
|
||
]
|