feat(ai): add external mcp rag integration readback

docs/guides/ai_automation_session_sop.md

@@ -26,6 +26,8 @@
 - Telegram 失敗必須可暫存與 replay。
 - EventRouter / AutoHeal 變更必須更新 `services/ai_automation_metrics.py` 指標或確認既有指標已覆蓋。
 - AI 自動化閉環變更必須確認 `/api/ai-automation/smoke` 與 `/ai_automation_smoke` 仍能反映新狀態。
+- 外部 MCP / RAG 能力導入內部治理時，必須確認 `/api/ai-automation/external-mcp-rag-integration`
+  或 `python scripts/ops/report_external_mcp_rag_integration.py` 可讀回每個能力的內部落點、狀態、資料邊界與下一個機器動作。
 - AI 自動化 Prometheus 指標變更必須同步檢查 `docker/grafana/provisioning/dashboards/json/ai-automation-overview.json` 是否需要新增 panel 或查詢。
 - 188 線上 active monitoring stack 以 `monitoring/prometheus.yml` 為準；110 gateway 另有 `/home/wooo/monitoring/prometheus.yml`。若 dashboard 無資料，先確認 Prometheus `momo-app` target 與 `momo-network` 連線；所有 Blackbox HTTP target 必須打 `/health`，不可打 Dashboard 首頁 `/`。
 - Smoke dashboard 會保存 JSONL 趨勢；若新增檢查項目，要確保 history compact record 仍保持小而可讀。

routes/system_public_routes.py

@@ -689,6 +689,19 @@ def ai_automation_pixelrag_visual_evidence_readback_api():
     ))
 
 
+@system_public_bp.route('/api/ai-automation/external-mcp-rag-integration')
+@login_required
+def ai_automation_external_mcp_rag_integration_api():
+    """Read-only external MCP/RAG absorption status for internal MCP/RAG planes."""
+    from services.external_mcp_rag_integration_service import (
+        build_external_mcp_rag_integration_readback,
+    )
+
+    return jsonify(build_external_mcp_rag_integration_readback(
+        target_family=str(request.args.get('family') or '').strip() or None,
+    ))
+
+
 @system_public_bp.route('/api/ai-automation/smoke/history/export')
 @login_required
 def ai_automation_smoke_history_export():

scripts/ops/report_external_mcp_rag_integration.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Report external MCP/RAG absorption into internal MCP/RAG planes."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from services.external_mcp_rag_integration_service import (  # noqa: E402
+    build_external_mcp_rag_integration_readback,
+)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="輸出外部 MCP/RAG 導入內部 MCP/RAG 治理面的機器可讀狀態。"
+    )
+    parser.add_argument(
+        "--family",
+        choices=["external_mcp", "internal_mcp", "external_rag", "internal_rag"],
+        help="只輸出指定 capability family。",
+    )
+    args = parser.parse_args()
+
+    payload = build_external_mcp_rag_integration_readback(
+        target_family=args.family,
+    )
+    print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
+    return 0 if payload.get("success") else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

services/external_mcp_rag_integration_service.py

@@ -0,0 +1,308 @@
+"""Read-only inventory for external MCP/RAG absorption into internal control planes."""
+
+from __future__ import annotations
+
+import os
+from copy import deepcopy
+from datetime import datetime, timezone
+from typing import Any
+
+
+POLICY = "read_only_external_mcp_rag_integration_readback_v1"
+
+
+def _enabled(value: str | None) -> bool:
+    return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _controlled_apply_boundary() -> dict[str, bool]:
+    return {
+        "network_call": False,
+        "db_write": False,
+        "secret_read": False,
+        "production_price_write": False,
+        "external_side_effect": False,
+    }
+
+
+def _status_counts(items: list[dict[str, Any]]) -> dict[str, int]:
+    counts: dict[str, int] = {}
+    for item in items:
+        status = str(item.get("status") or "unknown")
+        counts[status] = counts.get(status, 0) + 1
+    return counts
+
+
+def _mcp_runtime_snapshot() -> dict[str, Any]:
+    try:
+        from services.mcp_router import MCP_BASE_HOSTS, TOOL_REGISTRY, is_mcp_router_enabled
+
+        return {
+            "enabled": is_mcp_router_enabled(),
+            "servers": deepcopy(MCP_BASE_HOSTS),
+            "caller_count": len(TOOL_REGISTRY),
+            "tool_registry": {
+                caller: {
+                    server: list(tools)
+                    for server, tools in servers.items()
+                }
+                for caller, servers in TOOL_REGISTRY.items()
+            },
+        }
+    except Exception as exc:
+        return {
+            "enabled": False,
+            "servers": {},
+            "caller_count": 0,
+            "tool_registry": {},
+            "error": str(exc)[:300],
+        }
+
+
+def _rag_runtime_snapshot() -> dict[str, Any]:
+    try:
+        from services.rag_service import (
+            RAG_DEFAULT_THRESHOLD,
+            RAG_DEFAULT_TOP_K,
+            RAG_EMBED_DIM,
+            RAG_EMBED_MODEL,
+            get_embedding_signature,
+            is_rag_enabled,
+        )
+
+        return {
+            "enabled": is_rag_enabled(),
+            "vector_store": "pgvector",
+            "embedding_model": RAG_EMBED_MODEL,
+            "embedding_dim": RAG_EMBED_DIM,
+            "embedding_signature": get_embedding_signature(),
+            "default_top_k": RAG_DEFAULT_TOP_K,
+            "default_threshold": RAG_DEFAULT_THRESHOLD,
+        }
+    except Exception as exc:
+        return {
+            "enabled": False,
+            "vector_store": "pgvector",
+            "embedding_model": "",
+            "embedding_dim": 0,
+            "embedding_signature": "",
+            "error": str(exc)[:300],
+        }
+
+
+def _pixelrag_snapshot() -> dict[str, Any]:
+    try:
+        from services.pixelrag_crawler_integration_service import ECOMMERCE_PLATFORM_PROFILES
+
+        platforms = sorted(
+            platform
+            for platform in ECOMMERCE_PLATFORM_PROFILES
+            if platform not in {"market_intel", "external_market"}
+        )
+        return {
+            "enabled": True,
+            "platform_count": len(platforms),
+            "platforms": platforms,
+            "visual_rag_stage": "phase1_visual_evidence_receipts",
+        }
+    except Exception as exc:
+        return {
+            "enabled": False,
+            "platform_count": 0,
+            "platforms": [],
+            "visual_rag_stage": "unavailable",
+            "error": str(exc)[:300],
+        }
+
+
+def _capability_inventory() -> list[dict[str, Any]]:
+    return [
+        {
+            "id": "mcp.omnisearch.tavily_exa",
+            "family": "external_mcp",
+            "external_capability": "Tavily / Exa style public web search through omnisearch MCP",
+            "internal_plane": "mcp_router",
+            "internal_entrypoint": "services.mcp_router.TOOL_REGISTRY[mcp_collector][omnisearch]",
+            "status": "integrated_self_hosted_gateway",
+            "writes": ["mcp_calls_log_async"],
+            "data_boundary": "public_search_summary_only",
+            "next_machine_action": "keep_mcp_router_health_and_cache_readback",
+        },
+        {
+            "id": "mcp.firecrawl.scrape",
+            "family": "external_mcp",
+            "external_capability": "Firecrawl style public page scrape",
+            "internal_plane": "mcp_router",
+            "internal_entrypoint": "services.mcp_router.TOOL_REGISTRY[*][firecrawl].scrape_url",
+            "status": "integrated_self_hosted_gateway",
+            "writes": ["mcp_calls_log_async"],
+            "data_boundary": "approved_public_url_only",
+            "next_machine_action": "enforce_source_contract_before_scheduler_attach",
+        },
+        {
+            "id": "mcp.postgres.query",
+            "family": "internal_mcp",
+            "external_capability": "SQL read interface exposed as MCP tool",
+            "internal_plane": "mcp_router",
+            "internal_entrypoint": "postgres.query for approved callers",
+            "status": "integrated_read_only_contract",
+            "writes": ["mcp_calls_log_async"],
+            "data_boundary": "read_only_query_allowlist",
+            "next_machine_action": "keep_disallowing_unknown_callers_and_write_tools",
+        },
+        {
+            "id": "mcp.filesystem.readonly",
+            "family": "internal_mcp",
+            "external_capability": "Filesystem MCP read diagnostics",
+            "internal_plane": "mcp_router",
+            "internal_entrypoint": "ops_diagnostics filesystem read-only tools",
+            "status": "integrated_read_only_contract",
+            "writes": ["mcp_calls_log_async"],
+            "data_boundary": "allowed_directories_read_only",
+            "next_machine_action": "keep_write_file_and_mutation_tools_rejected",
+        },
+        {
+            "id": "rag.pixelrag.visual_evidence",
+            "family": "external_rag",
+            "external_capability": "PixelRAG page screenshot and tile retrieval pattern",
+            "internal_plane": "pixelrag_visual_evidence + internal RAG candidate lane",
+            "internal_entrypoint": "services.pixelrag_crawler_integration_service",
+            "status": "integrated_phase1_visual_receipts",
+            "writes": ["artifact_file_only"],
+            "data_boundary": "public_page_screenshot_tiles_no_price_write",
+            "next_machine_action": "add_ocr_vlm_replay_before_promoting_to_pgvector_rag",
+        },
+        {
+            "id": "rag.pgvector.bge_m3",
+            "family": "internal_rag",
+            "external_capability": "General text RAG retrieval pattern",
+            "internal_plane": "rag_service",
+            "internal_entrypoint": "services.rag_service.RAGService.query",
+            "status": "internal_primary_ready",
+            "writes": ["rag_query_log_async"],
+            "data_boundary": "ai_insights_pgvector_embedding_signature_guard",
+            "next_machine_action": "keep_embedding_signature_and_feedback_guardrails",
+        },
+        {
+            "id": "rag.qwen3_vl_embedding",
+            "family": "external_rag",
+            "external_capability": "Multimodal visual embedding model for tile retrieval",
+            "internal_plane": "ollama_first_visual_embedding_benchmark",
+            "internal_entrypoint": "not_enabled_in_production",
+            "status": "deferred_until_ollama_first_verified",
+            "writes": [],
+            "data_boundary": "no_hosted_visual_embedding_api",
+            "next_machine_action": "benchmark_local_multimodal_embedding_then_design_pgvector_metadata",
+        },
+        {
+            "id": "rag.faiss_visual_index",
+            "family": "external_rag",
+            "external_capability": "FAISS visual retrieval index used by some PixelRAG pipelines",
+            "internal_plane": "pgvector_first_policy",
+            "internal_entrypoint": "not_enabled_in_production",
+            "status": "rejected_for_production_without_adr",
+            "writes": [],
+            "data_boundary": "pgvector_is_the_only_production_vector_store",
+            "next_machine_action": "use_pgvector_compatible_metadata_or_write_adr_before_any_faiss_trial",
+        },
+        {
+            "id": "mcp.gemini_grounding",
+            "family": "external_mcp",
+            "external_capability": "Gemini grounding as hosted external search fallback",
+            "internal_plane": "mcp_collector_final_fallback",
+            "internal_entrypoint": "services.mcp_collector_service guarded fallback",
+            "status": "disabled_by_default_fallback_only",
+            "writes": ["mcp_cache_only_when_explicitly_enabled"],
+            "data_boundary": "gemini_api_hard_disabled_by_default",
+            "next_machine_action": "prefer_self_hosted_mcp_or_ollama_static_fallback",
+        },
+    ]
+
+
+def build_external_mcp_rag_integration_readback(
+    *,
+    target_family: str | None = None,
+) -> dict[str, Any]:
+    """Describe how external MCP/RAG capabilities are absorbed internally."""
+    family_filter = str(target_family or "").strip().lower()
+    inventory = _capability_inventory()
+    if family_filter:
+        inventory = [
+            item
+            for item in inventory
+            if str(item.get("family") or "").lower() == family_filter
+        ]
+
+    counts = _status_counts(inventory)
+    unresolved = [
+        item
+        for item in inventory
+        if str(item.get("status") or "").startswith(("deferred", "rejected", "disabled"))
+    ]
+    absorbed = [
+        item
+        for item in inventory
+        if item not in unresolved
+    ]
+    all_absorbed = len(unresolved) == 0
+
+    return {
+        "success": True,
+        "policy": POLICY,
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "status": "partially_integrated" if not all_absorbed else "fully_integrated",
+        "answer_to_owner": (
+            "不是全部完成；已把主要外部 MCP/RAG 放進內部可治理 registry，"
+            "其中 self-hosted MCP、pgvector RAG、PixelRAG phase1 已整合，"
+            "多模態 embedding / FAISS / Gemini grounding 仍需條件或維持停用。"
+            if not all_absorbed
+            else "目前 registry 內的外部 MCP/RAG 都已有內部治理落點。"
+        ),
+        "completion": {
+            "total_capabilities": len(inventory),
+            "absorbed_count": len(absorbed),
+            "unresolved_count": len(unresolved),
+            "all_absorbed": all_absorbed,
+            "status_counts": counts,
+        },
+        "runtime": {
+            "mcp": _mcp_runtime_snapshot(),
+            "rag": _rag_runtime_snapshot(),
+            "pixelrag": _pixelrag_snapshot(),
+            "env_flags": {
+                "MCP_ROUTER_ENABLED": _enabled(os.getenv("MCP_ROUTER_ENABLED")),
+                "RAG_ENABLED": _enabled(os.getenv("RAG_ENABLED")),
+                "GEMINI_API_HARD_DISABLED": _enabled(os.getenv("GEMINI_API_HARD_DISABLED", "true")),
+            },
+        },
+        "inventory": inventory,
+        "next_machine_actions": [
+            {
+                "priority": "P0",
+                "action": "wire_pixelrag_receipts_to_internal_rag_candidate_replay",
+                "status": "ready_after_ocr_vlm_replay_contract",
+            },
+            {
+                "priority": "P0",
+                "action": "add_mcp_router_health_readback_to_ai_automation_smoke",
+                "status": "ready",
+            },
+            {
+                "priority": "P1",
+                "action": "benchmark_ollama_first_visual_embedding",
+                "status": "blocked_until_model_runtime_verified",
+            },
+            {
+                "priority": "P1",
+                "action": "promote_successful_marketplace_visual_receipts_to_source_contracts",
+                "status": "ready_for_shopee_warning_for_coupang",
+            },
+        ],
+        "controlled_apply": _controlled_apply_boundary(),
+    }
+
+
+__all__ = [
+    "POLICY",
+    "build_external_mcp_rag_integration_readback",
+]

tests/test_external_mcp_rag_integration_service.py

@@ -0,0 +1,73 @@
+import json
+import subprocess
+import sys
+
+
+def test_external_mcp_rag_readback_reports_partial_truth():
+    from services.external_mcp_rag_integration_service import (
+        POLICY,
+        build_external_mcp_rag_integration_readback,
+    )
+
+    payload = build_external_mcp_rag_integration_readback()
+    inventory = {item["id"]: item for item in payload["inventory"]}
+
+    assert payload["policy"] == POLICY
+    assert payload["status"] == "partially_integrated"
+    assert payload["completion"]["all_absorbed"] is False
+    assert payload["completion"]["unresolved_count"] >= 1
+    assert "不是全部完成" in payload["answer_to_owner"]
+    assert inventory["rag.pixelrag.visual_evidence"]["status"] == "integrated_phase1_visual_receipts"
+    assert inventory["rag.qwen3_vl_embedding"]["status"] == "deferred_until_ollama_first_verified"
+    assert inventory["rag.faiss_visual_index"]["status"] == "rejected_for_production_without_adr"
+    assert inventory["mcp.gemini_grounding"]["status"] == "disabled_by_default_fallback_only"
+    assert payload["controlled_apply"]["db_write"] is False
+    assert payload["controlled_apply"]["network_call"] is False
+
+
+def test_external_mcp_rag_readback_can_filter_family():
+    from services.external_mcp_rag_integration_service import (
+        build_external_mcp_rag_integration_readback,
+    )
+
+    payload = build_external_mcp_rag_integration_readback(target_family="external_rag")
+
+    assert payload["completion"]["total_capabilities"] == 3
+    assert {item["family"] for item in payload["inventory"]} == {"external_rag"}
+
+
+def test_external_mcp_rag_cli_outputs_machine_readable_json():
+    completed = subprocess.run(
+        [
+            sys.executable,
+            "scripts/ops/report_external_mcp_rag_integration.py",
+            "--family",
+            "external_rag",
+        ],
+        capture_output=True,
+        check=False,
+        text=True,
+    )
+
+    assert completed.returncode == 0
+    payload = json.loads(completed.stdout)
+    assert payload["success"] is True
+    assert payload["completion"]["total_capabilities"] == 3
+    assert payload["inventory"][0]["family"] == "external_rag"
+
+
+def test_external_mcp_rag_ai_automation_route_returns_readback():
+    from flask import Flask
+    from routes import system_public_routes as routes
+
+    app = Flask(__name__)
+    with app.test_request_context(
+        "/api/ai-automation/external-mcp-rag-integration?family=external_mcp"
+    ):
+        response = routes.ai_automation_external_mcp_rag_integration_api.__wrapped__()
+    payload = response.get_json()
+
+    assert payload["policy"] == "read_only_external_mcp_rag_integration_readback_v1"
+    assert payload["success"] is True
+    assert {item["family"] for item in payload["inventory"]} == {"external_mcp"}
+    assert payload["runtime"]["mcp"]["caller_count"] >= 1