feat(p46): Agent 編排矩陣新頁 — OpenClaw/Hermes/NemoTron/EA × Ollama × Gemini × MCP × RAG

統帥要求：「好好把 OpenClaw/Hermes/NemoTron/ElephantAlpha + Ollama 多模型
+ 外部付費 Gemini + 內外 MCP + RAG 組合發揮出 AI 自動化新境界」

新頁面 /observability/agent_orchestration 一頁式呈現 4 Agent × 5 維度全景：

J-1: caller 自動分組
- OpenClaw: openclaw_qa/daily/meta/monthly/weekly/bot_main/bot_gemini/bot_nim
            + sales_copy + code_review_openclaw
- Hermes: hermes_analyst + hermes_intent + code_review_hermes
- NemoTron: nemotron_dispatch
- ElephantAlpha: ea_engine + code_review_elephant

J-2/3: 跨表 SQL JOIN（ai_calls × mcp_calls × rag_query_log）
每個 agent 顯示：
- 24h 呼叫 + Token + 成本
- 本地 Ollama 比例（細分 GCP-A/GCP-B/111）
- 付費 LLM 比例（細分 Gemini / 其他）
- MCP 編排率（透過 request_id 跨表 JOIN mcp_calls）
- RAG 命中率
- 錯誤率 + 平均耗時
- MCP server × caller 工作量明細

自動編排建議（5 條 rule-based）：
1. 付費比例 > 50% 且 ollama < 20% → 改 Hermes-first 短路
2. 錯誤率 > 10% → 觸發 Code Review Pipeline
3. MCP 編排率 < 5% 但 calls > 50 → 擴大 MCP omnisearch/firecrawl
4. RAG 命中率 ≥ 40% → 推 Telegram 收 feedback 強化 promotion gate
5. 111 fallback 比例 > 20% → GCP 兩台異常，查 host_health AIOps

J-4: 入口
- sidebar AI 觀測 group 加「Agent 編排矩陣」（07b）
- /observability/overview 入口卡升級為 7 項，Agent 編排矩陣放第一

整體 KPI 卡片：
- 總呼叫 / 本地 Ollama 比例 / 付費 LLM 成本 / RAG 命中率
- 「組合發揮」一目瞭然

8 表跨 JOIN：ai_calls × mcp_calls × rag_query_log × ai_insights ×
learning_episodes × incidents × heal_logs × host_health_probes

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

routes/admin_observability_routes.py

@@ -247,6 +247,262 @@ def observability_overview():
     )
 
 
+# ─────────────────────────────────────────────────────────────────────────────
+# /observability/agent_orchestration — Phase 46 編排矩陣
+# ─────────────────────────────────────────────────────────────────────────────
+
+# caller → agent 歸類規則（同 services/* 各 agent 真實 caller 值）
+_AGENT_CALLER_GROUPS = {
+    'openclaw': [
+        'openclaw_qa', 'openclaw_daily', 'openclaw_daily_insight',
+        'openclaw_meta', 'openclaw_monthly', 'openclaw_weekly',
+        'openclaw_bot_main', 'openclaw_bot_gemini', 'openclaw_bot_nim',
+        'sales_copy', 'code_review_openclaw',
+    ],
+    'hermes': [
+        'hermes_analyst', 'hermes_intent', 'code_review_hermes',
+    ],
+    'nemotron': [
+        'nemotron_dispatch',
+    ],
+    'elephant_alpha': [
+        'ea_engine', 'code_review_elephant',
+    ],
+}
+_AGENT_LABELS = {
+    'openclaw': ('🤖 OpenClaw', '主編排者 / Bot 對話 / 報告生成'),
+    'hermes': ('🔍 Hermes', '價格/程式碼分析師'),
+    'nemotron': ('🧬 NemoTron', '任務 dispatcher'),
+    'elephant_alpha': ('🐘 ElephantAlpha', '自主決策引擎'),
+}
+
+# Provider → 類別歸類
+_PROVIDER_TIER = {
+    'gcp_ollama': 'ollama_local',
+    'ollama_secondary': 'ollama_local',
+    'ollama_111': 'ollama_local',
+    'ollama_other': 'ollama_local',
+    'gemini': 'paid_external',
+    'claude': 'paid_external',
+    'nim': 'paid_external',
+    'nim_via_elephant': 'paid_external',
+    'openrouter': 'paid_external',
+}
+
+
+@admin_observability_bp.route('/agent_orchestration')
+@login_required
+def agent_orchestration_dashboard():
+    """Phase 46 — 4 Agent × Models × MCP × RAG 編排矩陣
+
+    展現「組合發揮」：每個 agent 在 24h 內如何調用 Ollama/Gemini，
+    搭配 MCP tool（外部 + 內部 mcp_collector），與 RAG 知識庫的協作。
+
+    資料來源：ai_calls × mcp_calls × rag_query_log 三表跨 JOIN + caller 分組。
+    """
+    hours = int(request.args.get('hours', '24'))
+    session = get_session()
+    try:
+        # 1. 整體統計
+        overall = session.execute(
+            sa_text("""
+                SELECT COUNT(*),
+                       COALESCE(SUM(cost_usd), 0),
+                       COUNT(*) FILTER (WHERE provider IN ('gemini','claude','nim','openrouter','nim_via_elephant')),
+                       COUNT(*) FILTER (WHERE provider IN ('gcp_ollama','ollama_secondary','ollama_111','ollama_other')),
+                       COUNT(*) FILTER (WHERE rag_hit),
+                       COALESCE(SUM(input_tokens + output_tokens), 0)
+                FROM ai_calls
+                WHERE called_at >= NOW() - (:h * INTERVAL '1 hour')
+            """),
+            {'h': hours},
+        ).fetchone()
+        total_calls = int(overall[0] or 0)
+        total_cost = float(overall[1] or 0)
+        paid_calls = int(overall[2] or 0)
+        local_calls = int(overall[3] or 0)
+        rag_hits = int(overall[4] or 0)
+        total_tokens = int(overall[5] or 0)
+
+        # 2. 每個 agent group 的細節
+        agent_matrix = []
+        for agent_key, callers in _AGENT_CALLER_GROUPS.items():
+            ag_row = session.execute(
+                sa_text("""
+                    SELECT COUNT(*) AS calls,
+                           COALESCE(SUM(input_tokens + output_tokens), 0) AS tokens,
+                           COALESCE(SUM(cost_usd), 0) AS cost,
+                           COUNT(*) FILTER (WHERE rag_hit) AS rag_hits,
+                           COUNT(*) FILTER (WHERE provider IN ('gcp_ollama','ollama_secondary','ollama_111','ollama_other')) AS ollama,
+                           COUNT(*) FILTER (WHERE provider = 'gcp_ollama') AS ollama_gcp_a,
+                           COUNT(*) FILTER (WHERE provider = 'ollama_secondary') AS ollama_gcp_b,
+                           COUNT(*) FILTER (WHERE provider = 'ollama_111') AS ollama_111,
+                           COUNT(*) FILTER (WHERE provider = 'gemini') AS gemini,
+                           COUNT(*) FILTER (WHERE provider IN ('claude','nim','openrouter','nim_via_elephant')) AS other_paid,
+                           COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')) AS errors,
+                           COALESCE(AVG(duration_ms), 0) AS avg_ms
+                    FROM ai_calls
+                    WHERE called_at >= NOW() - (:h * INTERVAL '1 hour')
+                      AND caller = ANY(:callers)
+                """),
+                {'h': hours, 'callers': callers},
+            ).fetchone()
+            calls = int(ag_row[0] or 0)
+            if calls == 0:
+                # 沒呼叫也佔位顯示
+                agent_matrix.append({
+                    'key': agent_key, 'label': _AGENT_LABELS[agent_key][0],
+                    'desc': _AGENT_LABELS[agent_key][1],
+                    'calls': 0, 'tokens': 0, 'cost': 0,
+                    'rag_hits': 0, 'rag_rate': 0,
+                    'ollama_pct': 0, 'gemini_pct': 0, 'paid_pct': 0,
+                    'ollama_gcp_a': 0, 'ollama_gcp_b': 0, 'ollama_111': 0,
+                    'gemini': 0, 'other_paid': 0,
+                    'errors': 0, 'error_rate': 0,
+                    'avg_ms': 0, 'mcp_calls': 0, 'mcp_rate': 0,
+                    'callers_in_group': callers,
+                })
+                continue
+
+            # MCP 編排率（透過 request_id 串接）
+            mcp_count = session.execute(
+                sa_text("""
+                    SELECT COUNT(DISTINCT a.request_id)
+                    FROM ai_calls a
+                    INNER JOIN mcp_calls m ON m.request_id = a.request_id
+                    WHERE a.called_at >= NOW() - (:h * INTERVAL '1 hour')
+                      AND a.caller = ANY(:callers)
+                      AND a.request_id IS NOT NULL
+                """),
+                {'h': hours, 'callers': callers},
+            ).fetchone()[0] or 0
+
+            errors = int(ag_row[10] or 0)
+            ollama = int(ag_row[4] or 0)
+            gemini = int(ag_row[8] or 0)
+            other_paid = int(ag_row[9] or 0)
+
+            agent_matrix.append({
+                'key': agent_key,
+                'label': _AGENT_LABELS[agent_key][0],
+                'desc': _AGENT_LABELS[agent_key][1],
+                'calls': calls,
+                'tokens': int(ag_row[1] or 0),
+                'cost': float(ag_row[2] or 0),
+                'rag_hits': int(ag_row[3] or 0),
+                'rag_rate': (float(ag_row[3] or 0) / calls * 100) if calls else 0,
+                'ollama': ollama, 'ollama_pct': (ollama / calls * 100) if calls else 0,
+                'ollama_gcp_a': int(ag_row[5] or 0),
+                'ollama_gcp_b': int(ag_row[6] or 0),
+                'ollama_111': int(ag_row[7] or 0),
+                'gemini': gemini, 'gemini_pct': (gemini / calls * 100) if calls else 0,
+                'other_paid': other_paid,
+                'paid_pct': ((gemini + other_paid) / calls * 100) if calls else 0,
+                'errors': errors, 'error_rate': (errors / calls * 100) if calls else 0,
+                'avg_ms': int(ag_row[11] or 0),
+                'mcp_calls': int(mcp_count),
+                'mcp_rate': (float(mcp_count) / calls * 100) if calls else 0,
+                'callers_in_group': callers,
+            })
+
+        # 3. MCP server 24h 工作量（同 host_health 邏輯）
+        mcp_servers = session.execute(
+            sa_text("""
+                SELECT server, caller, COUNT(*) AS calls,
+                       COUNT(*) FILTER (WHERE cache_hit) AS cache_hits,
+                       COALESCE(SUM(cost_usd), 0) AS cost
+                FROM mcp_calls
+                WHERE called_at >= NOW() - (:h * INTERVAL '1 hour')
+                GROUP BY server, caller
+                ORDER BY calls DESC
+                LIMIT 30
+            """),
+            {'h': hours},
+        ).fetchall()
+        mcp_matrix = [
+            {
+                'server': r[0], 'caller': r[1],
+                'calls': int(r[2] or 0),
+                'cache_hits': int(r[3] or 0),
+                'cost': float(r[4] or 0),
+                'cache_rate': (float(r[3] or 0) / float(r[2]) * 100) if r[2] else 0,
+            }
+            for r in mcp_servers
+        ]
+
+        # 4. 自動編排建議（rule-based 提案）
+        recommendations = []
+        for ag in agent_matrix:
+            if ag['calls'] == 0:
+                continue
+            # 規則 1：付費比例 > 50% 且 ollama 比例 < 20% → 建議切 Hermes-first
+            if ag['paid_pct'] > 50 and ag['ollama_pct'] < 20:
+                recommendations.append({
+                    'severity': 'high', 'agent': ag['label'],
+                    'finding': f"付費 LLM 比例 {ag['paid_pct']:.0f}%（cost ${ag['cost']:.2f}）",
+                    'suggestion': '改用 Hermes-first 短路機制：先試 Ollama 三主機 5s timeout，0 hits 才 escalate Gemini',
+                })
+            # 規則 2：錯誤率 > 10% → 建議跑 code review
+            if ag['error_rate'] > 10:
+                recommendations.append({
+                    'severity': 'high', 'agent': ag['label'],
+                    'finding': f"錯誤率 {ag['error_rate']:.1f}%（{ag['errors']}/{ag['calls']}）",
+                    'suggestion': '觸發 Code Review Pipeline 找 regression（ai_calls 觀測台一鍵）',
+                })
+            # 規則 3：MCP 編排率 < 5% 但 calls 多 → 建議擴大 MCP 使用
+            if ag['mcp_rate'] < 5 and ag['calls'] > 50:
+                recommendations.append({
+                    'severity': 'med', 'agent': ag['label'],
+                    'finding': f"MCP 編排率僅 {ag['mcp_rate']:.1f}%，未善用外部工具",
+                    'suggestion': '考慮加 MCP omnisearch / firecrawl 補強事實查證鏈',
+                })
+            # 規則 4：RAG 命中率高（≥40%）但有 saved_call=False 的多 → 提醒 feedback
+            if ag['rag_rate'] >= 40 and ag['rag_hits'] >= 20:
+                recommendations.append({
+                    'severity': 'low', 'agent': ag['label'],
+                    'finding': f"RAG 命中率 {ag['rag_rate']:.1f}%（{ag['rag_hits']} hits）— 知識庫貢獻度高",
+                    'suggestion': '推 Telegram inline button 收集 feedback_score 強化 promotion gate',
+                })
+            # 規則 5：111 fallback 比例 > 20% → 警示
+            if ag['calls'] > 0 and ag['ollama_111'] / max(ag['calls'], 1) > 0.20:
+                fb_pct = ag['ollama_111'] / ag['calls'] * 100
+                recommendations.append({
+                    'severity': 'med', 'agent': ag['label'],
+                    'finding': f"111 fallback 比例 {fb_pct:.0f}%（GCP 兩台不可達？）",
+                    'suggestion': '檢查 mo.wooo.work/observability/host_health AIOps incidents',
+                })
+
+        return render_template(
+            'admin/agent_orchestration.html',
+            active_page='obs_agent_orchestration',
+            hours=hours,
+            agent_matrix=agent_matrix,
+            mcp_matrix=mcp_matrix,
+            recommendations=recommendations,
+            overall={
+                'total_calls': total_calls,
+                'total_cost': total_cost,
+                'total_tokens': total_tokens,
+                'paid_calls': paid_calls,
+                'local_calls': local_calls,
+                'rag_hits': rag_hits,
+                'paid_pct': (paid_calls / total_calls * 100) if total_calls else 0,
+                'local_pct': (local_calls / total_calls * 100) if total_calls else 0,
+                'rag_rate': (rag_hits / total_calls * 100) if total_calls else 0,
+            },
+            error=None,
+        )
+    except Exception as e:
+        return render_template(
+            'admin/agent_orchestration.html',
+            active_page='obs_agent_orchestration', hours=hours,
+            agent_matrix=[], mcp_matrix=[], recommendations=[], overall={},
+            error=f'查詢失敗: {type(e).__name__}: {str(e)[:200]}',
+        )
+    finally:
+        session.close()
+
+
 # ─────────────────────────────────────────────────────────────────────────────
 # /observability/ai_calls — Phase 27 主入口
 # ─────────────────────────────────────────────────────────────────────────────