499 lines
19 KiB
Python
499 lines
19 KiB
Python
"""Read-only inventory for external MCP/RAG absorption into internal control planes."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import hashlib
|
||
import json
|
||
from copy import deepcopy
|
||
from datetime import datetime, timezone
|
||
from typing import Any
|
||
from zoneinfo import ZoneInfo
|
||
|
||
|
||
POLICY = "read_only_external_mcp_rag_integration_readback_v1"
|
||
PUBLIC_FEDERATION_SCHEMA_VERSION = "ewoooc_mcp_federation_receipt_v1"
|
||
PUBLIC_FEDERATION_POLICY = "public_aggregate_read_only_mcp_federation_v1"
|
||
TAIPEI_TZ = ZoneInfo("Asia/Taipei")
|
||
|
||
_FEDERATED_PRODUCTS = (
|
||
{
|
||
"product_id": "ewoooc",
|
||
"canonical_repo": "wooo/ewoooc",
|
||
"runtime_role": "mcp_rag_automation_source_plane",
|
||
},
|
||
{
|
||
"product_id": "momo-pro-system",
|
||
"canonical_repo": "wooo/momo-pro-system",
|
||
"runtime_role": "commerce_runtime_consumer_plane",
|
||
},
|
||
)
|
||
|
||
|
||
def _enabled(value: str | None) -> bool:
|
||
return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
|
||
|
||
|
||
def _controlled_apply_boundary() -> dict[str, bool]:
|
||
return {
|
||
"network_call": False,
|
||
"db_write": False,
|
||
"secret_read": False,
|
||
"production_price_write": False,
|
||
"external_side_effect": False,
|
||
}
|
||
|
||
|
||
def _public_operation_boundaries() -> dict[str, bool]:
|
||
"""V-New: 公開 receipt 只允許無副作用的聚合狀態讀回。"""
|
||
return {
|
||
"read_only_observation_allowed": True,
|
||
"network_call_allowed": False,
|
||
"db_write_allowed": False,
|
||
"secret_read_allowed": False,
|
||
"external_tool_call_allowed": False,
|
||
"rag_write_allowed": False,
|
||
"production_price_write_allowed": False,
|
||
"request_or_response_body_storage_allowed": False,
|
||
}
|
||
|
||
|
||
def _safe_mcp_summary(snapshot: dict[str, Any]) -> dict[str, Any]:
|
||
servers = snapshot.get("servers")
|
||
servers = servers if isinstance(servers, dict) else {}
|
||
registry = snapshot.get("tool_registry")
|
||
registry = registry if isinstance(registry, dict) else {}
|
||
tool_count = 0
|
||
for server_map in registry.values():
|
||
if not isinstance(server_map, dict):
|
||
continue
|
||
for tools in server_map.values():
|
||
if isinstance(tools, (list, tuple, set)):
|
||
tool_count += len(tools)
|
||
return {
|
||
"enabled": snapshot.get("enabled") is True,
|
||
"server_count": len(servers),
|
||
"server_ids": sorted(str(server_id) for server_id in servers),
|
||
"caller_count": int(snapshot.get("caller_count") or 0),
|
||
"tool_count": tool_count,
|
||
}
|
||
|
||
|
||
def _safe_rag_summary(snapshot: dict[str, Any]) -> dict[str, Any]:
|
||
model = str(snapshot.get("embedding_model") or "")
|
||
return {
|
||
"enabled": snapshot.get("enabled") is True,
|
||
"vector_store": str(snapshot.get("vector_store") or "pgvector"),
|
||
"embedding_model": model,
|
||
"embedding_dim": int(snapshot.get("embedding_dim") or 0),
|
||
"embedding_signature": str(snapshot.get("embedding_signature") or ""),
|
||
"embedding_version_state": (
|
||
"floating_tag_detected" if model.endswith(":latest") else "explicit_or_unknown"
|
||
),
|
||
}
|
||
|
||
|
||
def _safe_pixelrag_summary(snapshot: dict[str, Any]) -> dict[str, Any]:
|
||
return {
|
||
"enabled": snapshot.get("enabled") is True,
|
||
"platform_count": int(snapshot.get("platform_count") or 0),
|
||
"platforms": sorted(
|
||
str(platform) for platform in snapshot.get("platforms") or []
|
||
),
|
||
"visual_rag_stage": str(
|
||
snapshot.get("visual_rag_stage") or "unavailable"
|
||
),
|
||
"formal_product_write_allowed": False,
|
||
}
|
||
|
||
|
||
def compute_public_federation_receipt_fingerprint(
|
||
receipt: dict[str, Any],
|
||
) -> str:
|
||
"""Return the verifier-recomputable hash for normalized public fields."""
|
||
covered = {
|
||
key: value
|
||
for key, value in receipt.items()
|
||
if key not in {"receipt_id", "observed_at", "integrity"}
|
||
}
|
||
canonical = json.dumps(
|
||
covered,
|
||
ensure_ascii=False,
|
||
sort_keys=True,
|
||
separators=(",", ":"),
|
||
)
|
||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def build_public_mcp_federation_receipt(
|
||
*,
|
||
runtime_version: str,
|
||
observed_at: str | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Build two product-scoped, aggregate-only MCP/RAG runtime receipts.
|
||
|
||
This function performs no network call, database read/write, tool call, or
|
||
RAG write. Internal endpoint URLs and raw tool registries are deliberately
|
||
reduced to stable IDs and counts before reaching the public response.
|
||
"""
|
||
timestamp = observed_at or datetime.now(TAIPEI_TZ).isoformat(timespec="seconds")
|
||
mcp = _safe_mcp_summary(_mcp_runtime_snapshot())
|
||
rag = _safe_rag_summary(_rag_runtime_snapshot())
|
||
pixelrag = _safe_pixelrag_summary(_pixelrag_snapshot())
|
||
|
||
blockers: list[str] = []
|
||
if not mcp["enabled"]:
|
||
blockers.append("mcp_router_runtime_not_enabled")
|
||
if mcp["server_count"] == 0 or mcp["tool_count"] == 0:
|
||
blockers.append("mcp_runtime_inventory_empty")
|
||
if not rag["enabled"]:
|
||
blockers.append("rag_runtime_not_enabled")
|
||
if not rag["embedding_signature"]:
|
||
blockers.append("rag_embedding_signature_missing")
|
||
if rag["embedding_version_state"] == "floating_tag_detected":
|
||
blockers.append("rag_embedding_model_not_immutable")
|
||
if not pixelrag["enabled"]:
|
||
blockers.append("pixelrag_runtime_not_enabled")
|
||
blockers = sorted(blockers)
|
||
|
||
normalized_runtime = {
|
||
"mcp": mcp,
|
||
"rag": rag,
|
||
"pixelrag": pixelrag,
|
||
}
|
||
receipts: list[dict[str, Any]] = []
|
||
for product in _FEDERATED_PRODUCTS:
|
||
receipt: dict[str, Any] = {
|
||
"receipt_schema_version": PUBLIC_FEDERATION_SCHEMA_VERSION,
|
||
"product_id": product["product_id"],
|
||
"canonical_repo": product["canonical_repo"],
|
||
"runtime_role": product["runtime_role"],
|
||
"runtime_version": str(runtime_version or "unknown"),
|
||
"health_status": "ready" if not blockers else "partial_degraded",
|
||
"runtime": deepcopy(normalized_runtime),
|
||
"data_boundaries": {
|
||
"scope": "aggregate_runtime_metadata_only",
|
||
"pixelrag": "public_visual_receipts_no_formal_product_write",
|
||
"rag": "pgvector_internal_only_no_public_content",
|
||
"mcp": "server_ids_and_counts_only_no_endpoint_or_payload",
|
||
},
|
||
"operation_boundaries": _public_operation_boundaries(),
|
||
"blockers": list(blockers),
|
||
}
|
||
fingerprint = compute_public_federation_receipt_fingerprint(receipt)
|
||
receipt["observed_at"] = timestamp
|
||
receipt["receipt_id"] = (
|
||
f"mcp-federation:{product['product_id']}:{fingerprint[:16]}"
|
||
)
|
||
receipt["integrity"] = {
|
||
"algorithm": "sha256",
|
||
"canonicalization": "json_sort_keys_compact_v1",
|
||
"excluded_fields": ["receipt_id", "observed_at", "integrity"],
|
||
"normalized_fingerprint_sha256": fingerprint,
|
||
}
|
||
receipts.append(receipt)
|
||
|
||
return {
|
||
"success": True,
|
||
"schema_version": PUBLIC_FEDERATION_SCHEMA_VERSION,
|
||
"policy": PUBLIC_FEDERATION_POLICY,
|
||
"generated_at": timestamp,
|
||
"status": "ready" if not blockers else "partial_degraded",
|
||
"receipt_count": len(receipts),
|
||
"receipts": receipts,
|
||
"blockers": blockers,
|
||
"operation_boundaries": _public_operation_boundaries(),
|
||
"next_machine_action": (
|
||
"pin_embedding_model_and_verify_runtime_receipts"
|
||
if blockers
|
||
else "continue_scheduled_read_only_federation"
|
||
),
|
||
}
|
||
|
||
|
||
def _status_counts(items: list[dict[str, Any]]) -> dict[str, int]:
|
||
counts: dict[str, int] = {}
|
||
for item in items:
|
||
status = str(item.get("status") or "unknown")
|
||
counts[status] = counts.get(status, 0) + 1
|
||
return counts
|
||
|
||
|
||
def _mcp_runtime_snapshot() -> dict[str, Any]:
|
||
try:
|
||
from services.mcp_router import MCP_BASE_HOSTS, TOOL_REGISTRY, is_mcp_router_enabled
|
||
|
||
return {
|
||
"enabled": is_mcp_router_enabled(),
|
||
"servers": deepcopy(MCP_BASE_HOSTS),
|
||
"caller_count": len(TOOL_REGISTRY),
|
||
"tool_registry": {
|
||
caller: {
|
||
server: list(tools)
|
||
for server, tools in servers.items()
|
||
}
|
||
for caller, servers in TOOL_REGISTRY.items()
|
||
},
|
||
}
|
||
except Exception as exc:
|
||
return {
|
||
"enabled": False,
|
||
"servers": {},
|
||
"caller_count": 0,
|
||
"tool_registry": {},
|
||
"error": str(exc)[:300],
|
||
}
|
||
|
||
|
||
def _rag_runtime_snapshot() -> dict[str, Any]:
|
||
try:
|
||
from services.rag_service import (
|
||
RAG_DEFAULT_THRESHOLD,
|
||
RAG_DEFAULT_TOP_K,
|
||
RAG_EMBED_DIM,
|
||
RAG_EMBED_MODEL,
|
||
get_embedding_signature,
|
||
is_rag_enabled,
|
||
)
|
||
|
||
return {
|
||
"enabled": is_rag_enabled(),
|
||
"vector_store": "pgvector",
|
||
"embedding_model": RAG_EMBED_MODEL,
|
||
"embedding_dim": RAG_EMBED_DIM,
|
||
"embedding_signature": get_embedding_signature(),
|
||
"default_top_k": RAG_DEFAULT_TOP_K,
|
||
"default_threshold": RAG_DEFAULT_THRESHOLD,
|
||
}
|
||
except Exception as exc:
|
||
return {
|
||
"enabled": False,
|
||
"vector_store": "pgvector",
|
||
"embedding_model": "",
|
||
"embedding_dim": 0,
|
||
"embedding_signature": "",
|
||
"error": str(exc)[:300],
|
||
}
|
||
|
||
|
||
def _pixelrag_snapshot() -> dict[str, Any]:
|
||
try:
|
||
from services.pixelrag_crawler_integration_service import ECOMMERCE_PLATFORM_PROFILES
|
||
|
||
platforms = sorted(
|
||
platform
|
||
for platform in ECOMMERCE_PLATFORM_PROFILES
|
||
if platform not in {"market_intel", "external_market"}
|
||
)
|
||
return {
|
||
"enabled": True,
|
||
"platform_count": len(platforms),
|
||
"platforms": platforms,
|
||
"visual_rag_stage": "phase1_visual_evidence_receipts",
|
||
}
|
||
except Exception as exc:
|
||
return {
|
||
"enabled": False,
|
||
"platform_count": 0,
|
||
"platforms": [],
|
||
"visual_rag_stage": "unavailable",
|
||
"error": str(exc)[:300],
|
||
}
|
||
|
||
|
||
def _capability_inventory() -> list[dict[str, Any]]:
|
||
return [
|
||
{
|
||
"id": "mcp.omnisearch.tavily_exa",
|
||
"family": "external_mcp",
|
||
"external_capability": "Tavily / Exa style public web search through omnisearch MCP",
|
||
"internal_plane": "mcp_router",
|
||
"internal_entrypoint": "services.mcp_router.TOOL_REGISTRY[mcp_collector][omnisearch]",
|
||
"status": "integrated_self_hosted_gateway",
|
||
"writes": ["mcp_calls_log_async"],
|
||
"data_boundary": "public_search_summary_only",
|
||
"next_machine_action": "keep_mcp_router_health_and_cache_readback",
|
||
},
|
||
{
|
||
"id": "mcp.firecrawl.scrape",
|
||
"family": "external_mcp",
|
||
"external_capability": "Firecrawl style public page scrape",
|
||
"internal_plane": "mcp_router",
|
||
"internal_entrypoint": "services.mcp_router.TOOL_REGISTRY[*][firecrawl].scrape_url",
|
||
"status": "integrated_self_hosted_gateway",
|
||
"writes": ["mcp_calls_log_async"],
|
||
"data_boundary": "approved_public_url_only",
|
||
"next_machine_action": "enforce_source_contract_before_scheduler_attach",
|
||
},
|
||
{
|
||
"id": "mcp.postgres.query",
|
||
"family": "internal_mcp",
|
||
"external_capability": "SQL read interface exposed as MCP tool",
|
||
"internal_plane": "mcp_router",
|
||
"internal_entrypoint": "postgres.query for approved callers",
|
||
"status": "integrated_read_only_contract",
|
||
"writes": ["mcp_calls_log_async"],
|
||
"data_boundary": "read_only_query_allowlist",
|
||
"next_machine_action": "keep_disallowing_unknown_callers_and_write_tools",
|
||
},
|
||
{
|
||
"id": "mcp.filesystem.readonly",
|
||
"family": "internal_mcp",
|
||
"external_capability": "Filesystem MCP read diagnostics",
|
||
"internal_plane": "mcp_router",
|
||
"internal_entrypoint": "ops_diagnostics filesystem read-only tools",
|
||
"status": "integrated_read_only_contract",
|
||
"writes": ["mcp_calls_log_async"],
|
||
"data_boundary": "allowed_directories_read_only",
|
||
"next_machine_action": "keep_write_file_and_mutation_tools_rejected",
|
||
},
|
||
{
|
||
"id": "rag.pixelrag.visual_evidence",
|
||
"family": "external_rag",
|
||
"external_capability": "PixelRAG page screenshot and tile retrieval pattern",
|
||
"internal_plane": "pixelrag_visual_evidence + internal RAG candidate lane",
|
||
"internal_entrypoint": "services.pixelrag_crawler_integration_service",
|
||
"status": "integrated_phase1_visual_receipts",
|
||
"writes": ["artifact_file_only"],
|
||
"data_boundary": "public_page_screenshot_tiles_no_price_write",
|
||
"next_machine_action": "add_ocr_vlm_replay_before_promoting_to_pgvector_rag",
|
||
},
|
||
{
|
||
"id": "rag.pgvector.bge_m3",
|
||
"family": "internal_rag",
|
||
"external_capability": "General text RAG retrieval pattern",
|
||
"internal_plane": "rag_service",
|
||
"internal_entrypoint": "services.rag_service.RAGService.query",
|
||
"status": "internal_primary_ready",
|
||
"writes": ["rag_query_log_async"],
|
||
"data_boundary": "ai_insights_pgvector_embedding_signature_guard",
|
||
"next_machine_action": "keep_embedding_signature_and_feedback_guardrails",
|
||
},
|
||
{
|
||
"id": "rag.qwen3_vl_embedding",
|
||
"family": "external_rag",
|
||
"external_capability": "Multimodal visual embedding model for tile retrieval",
|
||
"internal_plane": "ollama_first_visual_embedding_benchmark",
|
||
"internal_entrypoint": "not_enabled_in_production",
|
||
"status": "deferred_until_ollama_first_verified",
|
||
"writes": [],
|
||
"data_boundary": "no_hosted_visual_embedding_api",
|
||
"next_machine_action": "benchmark_local_multimodal_embedding_then_design_pgvector_metadata",
|
||
},
|
||
{
|
||
"id": "rag.faiss_visual_index",
|
||
"family": "external_rag",
|
||
"external_capability": "FAISS visual retrieval index used by some PixelRAG pipelines",
|
||
"internal_plane": "pgvector_first_policy",
|
||
"internal_entrypoint": "not_enabled_in_production",
|
||
"status": "rejected_for_production_without_adr",
|
||
"writes": [],
|
||
"data_boundary": "pgvector_is_the_only_production_vector_store",
|
||
"next_machine_action": "use_pgvector_compatible_metadata_or_write_adr_before_any_faiss_trial",
|
||
},
|
||
{
|
||
"id": "mcp.gemini_grounding",
|
||
"family": "external_mcp",
|
||
"external_capability": "Gemini grounding as hosted external search fallback",
|
||
"internal_plane": "mcp_collector_final_fallback",
|
||
"internal_entrypoint": "services.mcp_collector_service guarded fallback",
|
||
"status": "disabled_by_default_fallback_only",
|
||
"writes": ["mcp_cache_only_when_explicitly_enabled"],
|
||
"data_boundary": "gemini_api_hard_disabled_by_default",
|
||
"next_machine_action": "prefer_self_hosted_mcp_or_ollama_static_fallback",
|
||
},
|
||
]
|
||
|
||
|
||
def build_external_mcp_rag_integration_readback(
|
||
*,
|
||
target_family: str | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Describe how external MCP/RAG capabilities are absorbed internally."""
|
||
family_filter = str(target_family or "").strip().lower()
|
||
inventory = _capability_inventory()
|
||
if family_filter:
|
||
inventory = [
|
||
item
|
||
for item in inventory
|
||
if str(item.get("family") or "").lower() == family_filter
|
||
]
|
||
|
||
counts = _status_counts(inventory)
|
||
unresolved = [
|
||
item
|
||
for item in inventory
|
||
if str(item.get("status") or "").startswith(("deferred", "rejected", "disabled"))
|
||
]
|
||
absorbed = [
|
||
item
|
||
for item in inventory
|
||
if item not in unresolved
|
||
]
|
||
all_absorbed = len(unresolved) == 0
|
||
|
||
return {
|
||
"success": True,
|
||
"policy": POLICY,
|
||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||
"status": "partially_integrated" if not all_absorbed else "fully_integrated",
|
||
"answer_to_owner": (
|
||
"不是全部完成；已把主要外部 MCP/RAG 放進內部可治理 registry，"
|
||
"其中 self-hosted MCP、pgvector RAG、PixelRAG phase1 已整合，"
|
||
"多模態 embedding / FAISS / Gemini grounding 仍需條件或維持停用。"
|
||
if not all_absorbed
|
||
else "目前 registry 內的外部 MCP/RAG 都已有內部治理落點。"
|
||
),
|
||
"completion": {
|
||
"total_capabilities": len(inventory),
|
||
"absorbed_count": len(absorbed),
|
||
"unresolved_count": len(unresolved),
|
||
"all_absorbed": all_absorbed,
|
||
"status_counts": counts,
|
||
},
|
||
"runtime": {
|
||
"mcp": _mcp_runtime_snapshot(),
|
||
"rag": _rag_runtime_snapshot(),
|
||
"pixelrag": _pixelrag_snapshot(),
|
||
"env_flags": {
|
||
"MCP_ROUTER_ENABLED": _enabled(os.getenv("MCP_ROUTER_ENABLED")),
|
||
"RAG_ENABLED": _enabled(os.getenv("RAG_ENABLED")),
|
||
"GEMINI_API_HARD_DISABLED": _enabled(os.getenv("GEMINI_API_HARD_DISABLED", "true")),
|
||
},
|
||
},
|
||
"inventory": inventory,
|
||
"next_machine_actions": [
|
||
{
|
||
"priority": "P0",
|
||
"action": "wire_pixelrag_receipts_to_internal_rag_candidate_replay",
|
||
"status": "ready_after_ocr_vlm_replay_contract",
|
||
},
|
||
{
|
||
"priority": "P0",
|
||
"action": "add_mcp_router_health_readback_to_ai_automation_smoke",
|
||
"status": "ready",
|
||
},
|
||
{
|
||
"priority": "P1",
|
||
"action": "benchmark_ollama_first_visual_embedding",
|
||
"status": "blocked_until_model_runtime_verified",
|
||
},
|
||
{
|
||
"priority": "P1",
|
||
"action": "promote_successful_marketplace_visual_receipts_to_source_contracts",
|
||
"status": "ready_for_shopee_warning_for_coupang",
|
||
},
|
||
],
|
||
"controlled_apply": _controlled_apply_boundary(),
|
||
}
|
||
|
||
|
||
__all__ = [
|
||
"POLICY",
|
||
"PUBLIC_FEDERATION_POLICY",
|
||
"PUBLIC_FEDERATION_SCHEMA_VERSION",
|
||
"build_external_mcp_rag_integration_readback",
|
||
"build_public_mcp_federation_receipt",
|
||
"compute_public_federation_receipt_fingerprint",
|
||
]
|