""" AI Router - Phase 13.3 #87 ========================== 智能 AI 路由器，根據意圖和複雜度動態選擇 AI Provider 目標: 根據請求特性自動選擇最適模型 策略: Intent Classifier + Complexity Scorer → Routing Decision 延遲目標: < 50ms (規則引擎優先) 唯一 production 路由（所有風險、複雜度與工具路徑共用）： GCP-A Ollama -> GCP-B Ollama -> host111 Ollama -> Anthropic Claude API -> Gemini API。Claude/Gemini 都必須通過 persistent enablement、原子成本閘與 per-provider circuit breaker；NVIDIA/Nemo/OpenClaw 僅保留 shadow metadata。 版本: v5.0 建立: 2026-03-26 (台北時區) 建立者: Claude Code 最後修改: 2026-07-14 (台北時區) 修改者: Codex 變更紀錄: | 版本 | 日期 | 執行者 | 變更內容 | |------|------|--------|----------| | v1.0 | 2026-03-26 | Claude Code | 初始實作 | | v2.0 | 2026-03-26 | Claude Code | 支援 IntentResult + 新意圖類型 | | v3.0 | 2026-03-26 | Claude Code | Phase 13.3 #87 完整路由決策矩陣 | | v4.0 | 2026-04-02 | ogt (首席架構師) | Phase 24 AIProvider Registry + Executor; C1 Langfuse Trace; C2 AIRouter.route(); C3 型別 typo; I4 Protocol close | | v4.1 | 2026-04-04 | ogt (首席架構師) | Phase 25 P0: DIAGNOSE Privacy-First — _local_fallback_chain; DIAGNOSE→NEMOTRON; REJECT+Telegram | | v4.2 | 2026-04-04 | Claude Code | Phase 25 P0 實測修正: _local_fallback_chain 移除 Nemotron(雲端)，僅留 Ollama(本地); timeout 依實測調整(NIM 60s/Ollama 200s) | | v4.3 | 2026-04-05 | Claude Code | Phase 25 P0 架構修正: 實測 Ollama CPU ~238s(不可用); NIM 實測 2-27s avg 10.6s; DIAGNOSE 改走 _full_fallback_chain(NIM 主力); _local_fallback_chain 廢棄 | | v4.4 | 2026-04-27 | Claude Sonnet 4.6 | A2 INC-20260425: DIAGNOSE fallback chain 移除 Ollama (CPU 238s 二次 timeout); 新增 _diagnose_fallback_chain (NEMO→GEMINI→CLAUDE); 新增 aiops_diagnose_fallback_total metric | | v5.1 | 2026-07-15 | Codex | Claude 以 paid/cost/canary gate 插入 host111 與 Gemini 之間；legacy providers 維持 shadow metadata | """ from __future__ import annotations import asyncio import hashlib import json as _json import re import time from dataclasses import dataclass, field from datetime import UTC, datetime from enum import Enum from typing import TYPE_CHECKING, Protocol import httpx as _httpx import structlog if TYPE_CHECKING: from src.services.intent_classifier import IntentResult from src.core.config import get_settings from src.services.ai_provider_policy import ( NON_EXECUTABLE_SHADOW_PROVIDERS, PAID_PROVIDER_ORDER, PRODUCTION_OLLAMA_ORDER, PRODUCTION_PROVIDER_ORDER, SANITIZED_CLOUD_PROVIDER_ORDER, normalize_production_execution_order, ) from src.services.ai_providers.interfaces import AIProvider as AIProviderProtocol from src.services.ai_providers.interfaces import AIResult from src.services.complexity_scorer import ( ComplexityScore, get_complexity_scorer, ) from src.services.intent_classifier import ( IntentResult, IntentType, RiskLevel, get_intent_classifier, normalize_intent, ) from src.services.model_registry import get_model_registry logger = structlog.get_logger(__name__) _DURABLE_OLLAMA_UNAVAILABLE_RECEIPT_SCHEMA = "ollama_unavailable_receipt_v1" _DURABLE_OLLAMA_UNAVAILABLE_RECEIPT_PREFIX = "ai:ollama:unavailable_receipt:" _DURABLE_OLLAMA_UNAVAILABLE_RECEIPT_ID = re.compile(r"^[A-Za-z0-9:_-]{1,128}$") _DURABLE_OLLAMA_UNAVAILABLE_MAX_SECONDS = 300.0 def _is_paid_provider_identity(provider: object) -> bool: """Treat decorated paid identities as paid and never cache-executable.""" normalized = str(provider or "").strip().lower() return any( normalized == paid or normalized.startswith(f"{paid}_") or normalized.startswith(f"{paid}:") or normalized.startswith(f"{paid}-") for paid in PAID_PROVIDER_ORDER ) # ============================================================================= # Provider 定義 # ============================================================================= class AIProviderEnum(str, Enum): """AI 提供者""" OLLAMA = "ollama" # 2026-05-04 ogt + Claude Sonnet 4.6: ADR-110 GCP 三層容災 # OllamaFailoverManager 回傳 provider_name="ollama_gcp_a"/"ollama_gcp_b"/"ollama_local" # 缺少 enum 值 → AIProviderEnum(primary_str) 拋 ValueError → fallback chain 清空 → 直跳 Gemini OLLAMA_GCP_A = "ollama_gcp_a" # GCP-A 34.143.170.20 Primary OLLAMA_GCP_B = "ollama_gcp_b" # GCP-B 34.21.145.224 Secondary OLLAMA_LOCAL = "ollama_local" # 192.168.0.111 Local Fallback GEMINI = "gemini" CLAUDE = "claude" # 2026-04-02 ogt: C1 修復 — 對齊 Registry 實際名稱 # OpenClawNemoProvider.name = "openclaw_nemo" (一般推理, via .188) # NemotronProvider.name = "nemotron" (Tool Calling, direct NVIDIA NIM) # 舊版 NVIDIA = "nvidia" 已移除: Registry 無此 Provider OPENCLAW_NEMO = "openclaw_nemo" NEMOTRON = "nemotron" # Provider 對應延遲預算 (ms) PROVIDER_LATENCY_BUDGET: dict[AIProviderEnum, int] = { AIProviderEnum.OLLAMA: 60000, # 本地，允許較長處理時間 # 2026-05-04 ogt: ADR-110 GCP 三層容災 — GCP NVMe SSD 推理快，60s 足夠 AIProviderEnum.OLLAMA_GCP_A: 60000, AIProviderEnum.OLLAMA_GCP_B: 60000, AIProviderEnum.OLLAMA_LOCAL: 90000, # 111 本地 HDD 稍慢 AIProviderEnum.GEMINI: 30000, # 雲端，較低延遲 AIProviderEnum.CLAUDE: 30000, # 雲端，較低延遲 # 2026-04-02 ogt: C1 修復 — 對齊 Registry 名稱 AIProviderEnum.OPENCLAW_NEMO: 60000, # via .188 → NVIDIA NIM，允許較長時間 AIProviderEnum.NEMOTRON: 60000, # Tool Calling 專用，允許較長時間 } # ============================================================================= # Interface 定義 (P1 修復 - 2026-04-01 首席架構師審查) # ============================================================================= class IAIRouter(Protocol): """ AI Router Protocol - 支援 DI 測試替換 2026-04-01 ogt: 首席架構師審查 P1 修復 - 新增 Protocol 定義支援依賴注入 - 參考: IModelRegistry, IComplexityScorer """ async def route( self, text: str, context: dict | None = None, ) -> RoutingDecision: """路由請求到最適 AI Provider""" ... def route_sync( self, text: str, context: dict | None = None, ) -> RoutingDecision: """同步版本路由""" ... def route_tool_calling( self, ) -> tuple[AIProviderEnum, str, list[tuple[AIProviderEnum, str]]]: """Tool Calling 專用路由""" ... @dataclass class RoutingDecision: """ 路由決策結果 (Phase 13.3 #87) 包含完整的路由資訊，供 OpenClaw 主流程使用 """ # 核心決策 selected_provider: AIProviderEnum # 選擇的 AI Provider selected_model: str # 選擇的模型名稱 fallback_chain: list[tuple[AIProviderEnum, str]] # 備援鏈 [(provider, model), ...] routing_reason: str # 路由決策原因 latency_budget_ms: int # 延遲預算 (毫秒) # 分類結果 intent: IntentType # 意圖分類 (正規化後) intent_result: IntentResult # 完整 Intent 分類結果 complexity: ComplexityScore # 複雜度評分 risk_level: RiskLevel = field(default=RiskLevel.MEDIUM) # 風險等級 observed_fallback_provider: str | None = None # 路由 metadata routing_latency_ms: float = 0.0 # 路由決策耗時 (ms) # 向後相容 (deprecated) model: str = "" # -> selected_model reason: str = "" # -> routing_reason fallback_models: list[str] = field(default_factory=list) # -> fallback_chain def __post_init__(self): """初始化後設定衍生欄位""" self.risk_level = self.intent_result.risk_level # 向後相容 self.model = self.selected_model self.reason = self.routing_reason self.fallback_models = [model for _, model in self.fallback_chain if model != self.selected_model] def to_dict(self) -> dict: """轉換為字典 (API 回應用)""" return { "selected_provider": self.selected_provider.value, "selected_model": self.selected_model, "fallback_chain": [ {"provider": p.value, "model": m} for p, m in self.fallback_chain ], "routing_reason": self.routing_reason, "latency_budget_ms": self.latency_budget_ms, "intent": self.intent.value, "risk_level": self.risk_level.value, "complexity_score": self.complexity.score, "routing_latency_ms": round(self.routing_latency_ms, 2), "production_route_contract": list(PRODUCTION_PROVIDER_ORDER), "selection_mode": "fixed_production_order", "observed_fallback_provider": self.observed_fallback_provider, } class AIRouter: """ AI 路由器 (Phase 13.3 #87) 整合 IntentClassifier 和 ComplexityScorer， 動態選擇最適合的 AI Provider 和模型。 Intent/risk/complexity only choose an Ollama model. They never choose a provider. Provider placement is the global five-hop production contract. """ def __init__(self): self._intent_classifier = get_intent_classifier() self._complexity_scorer = get_complexity_scorer() self._model_registry = get_model_registry() # 2026-04-25 P1.2 by Claude Engineer-A2 — failover 整合到 ai_router + lifespan # 延遲 import 避免循環依賴（ollama_failover_manager 不 import ai_router） from src.services.ollama_failover_manager import get_ollama_failover_manager self._failover_manager = get_ollama_failover_manager() # 從 ModelRegistry 取得模型配置 self._ollama_default = self._model_registry.get_model("ollama", "default") self._ollama_summary = self._model_registry.get_model("ollama", "summary") self._gemini_default = self._model_registry.get_model("gemini", "default") self._claude_default = self._model_registry.get_model("claude", "default") # 2026-04-02 ogt: C1 修復 — openclaw_nemo (一般推理) + nemotron (Tool Calling) self._openclaw_nemo_default = self._model_registry.get_model("nvidia", "default") self._nemotron_default = self._model_registry.get_model("nvidia", "default") # 向後相容別名 self._nvidia_default = self._openclaw_nemo_default # Provider 對應模型映射 self._provider_models: dict[AIProviderEnum, str] = { AIProviderEnum.OLLAMA: self._ollama_default, AIProviderEnum.GEMINI: self._gemini_default, AIProviderEnum.CLAUDE: self._claude_default, AIProviderEnum.OPENCLAW_NEMO: self._openclaw_nemo_default, AIProviderEnum.NEMOTRON: self._nemotron_default, } # 唯一 production chain。舊 provider 仍可保留在 registry 供 replay / # shadow metadata，但不得出現在任何 executable fallback chain。 self._full_fallback_chain: list[tuple[AIProviderEnum, str]] = [ (AIProviderEnum.OLLAMA_GCP_A, self._ollama_default), (AIProviderEnum.OLLAMA_GCP_B, self._ollama_default), (AIProviderEnum.OLLAMA_LOCAL, self._ollama_default), (AIProviderEnum.CLAUDE, self._claude_default), (AIProviderEnum.GEMINI, self._gemini_default), ] # 2026-04-29 ogt + Claude Code: 推翻 A2 鐵律（INC-20260425 事實基礎已過期） # 統帥鐵律 2026-04-29: 「主要優先用 111 主機的 Ollama」 # 統帥鐵律 feedback_ai_autonomous_direction.md: 以本地免費 LLM 為主 # 統帥鐵律 feedback_ollama_111_only.md: Ollama 唯一主機 = 111 # # 推翻原因： # 舊事實 (2026-04-27 A2): Ollama = CPU-only deepseek-r1:14b @ 238s（不可用） # 新事實 (2026-04-29): prod Ollama 111 = M1 Pro Apple Silicon GPU + qwen2.5:7b-instruct # VRAM 8.2GB 全載入，ctx 32k，實測 hi 0.54s # 雲端全死: OpenClaw 188 → 500，Gemini → 429 配額爆，Claude → 404 endpoint 過時 # 不推翻 → 100% incident llm_failed → AI 自動修復永遠不啟動 # # 配套修改： # - IntentType.DIAGNOSE override: OPENCLAW_NEMO → OLLAMA # - openclaw.py 注入 task_type="diagnose" 讓 Ollama 用 200s timeout # - test_p0_diagnose_routing.py / test_ai_router_diagnose_fallback.py 同步更新 self._diagnose_fallback_chain = self._full_fallback_chain.copy() # Tool Calling 專用 Fallback 鏈 (ADR-036) self._tool_calling_fallback_chain = self._full_fallback_chain.copy() # 2026-04-05 Claude Code: Phase 25 P0 v4.3 — _local_fallback_chain 廢棄 # 實測依據 (2026-04-05): # Ollama llama3.2:3b CPU-only = 238s 回 {"ok":true}（完全不可用於生產） # Nemotron NIM 實測 2.2s~27s，平均 10.6s（雲端 GPU，一直是主力） # NIM 從 Phase 22 起就接收 Incident 資料（無隱私問題，非新決策） # 結論: 不存在可用的本地 AI provider，DIAGNOSE 統一走 _full_fallback_chain（NIM 主力） self._local_fallback_chain: list[tuple[AIProviderEnum, str]] = [ # 廢棄: Ollama CPU ~238s 不可用，NIM 本非 local。保留欄位避免 attribute error。 ] # 意圖對應 Provider 強制覆寫 (None = 依複雜度決定) self._intent_provider_overrides: dict[IntentType, AIProviderEnum | None] = { # 四大核心意圖 IntentType.RESTART: None, # 依複雜度 IntentType.SCALE: None, # 依複雜度 IntentType.CONFIG: None, # 依複雜度 (但 HIGH 會升級) # 2026-04-16 ogt: 恢復 DIAGNOSE → OPENCLAW_NEMO（已過期，見 2026-04-29 推翻） # 2026-04-29 ogt + Claude Code: 推翻 A2，DIAGNOSE → OLLAMA（統帥鐵律本地優先） # 舊事實過期：M1 Pro GPU + qwen2.5:7b 0.54s（不再 CPU deepseek 238s） # OPENCLAW_NEMO 188:8088 現況 500 → 不可用 # 雲端全死 → 必須回到本地 Ollama 主推理 IntentType.DIAGNOSE: AIProviderEnum.OLLAMA, # 輔助意圖 IntentType.DELETE: AIProviderEnum.OLLAMA, IntentType.ROLLBACK: None, # 依複雜度 IntentType.UNKNOWN: None, # 舊版兼容 IntentType.CODE_REVIEW: None, IntentType.DEPLOYMENT: None, IntentType.ALERT_TRIAGE: AIProviderEnum.OLLAMA, IntentType.QUERY: AIProviderEnum.OLLAMA, IntentType.MAINTENANCE: None, } # 向後相容 self._default_model = self._ollama_default self._summary_model = self._ollama_summary self._fallback_order = [self._ollama_default, self._gemini_default] def _resolve_intent_from_context( self, context: dict | None, ) -> IntentResult | None: """ 從 context 解析集中治理的 intent hint。 僅作為 AI Router 的內部快路徑，避免呼叫端自行繞過 Router 規則。 例如 Phase 2 agent 已知屬於診斷分析，就不必再多跑一次 intent LLM。 """ if not context: return None raw_hint = str(context.get("intent_hint", "")).strip().lower() if not raw_hint: return None alias_map = { "restart": IntentType.RESTART, "scale": IntentType.SCALE, "config": IntentType.CONFIG, "diagnose": IntentType.DIAGNOSE, "delete": IntentType.DELETE, "rollback": IntentType.ROLLBACK, "unknown": IntentType.UNKNOWN, # legacy aliases "alert_triage": IntentType.ALERT_TRIAGE, "deployment": IntentType.DEPLOYMENT, "query": IntentType.QUERY, "maintenance": IntentType.MAINTENANCE, "code_review": IntentType.CODE_REVIEW, } intent = alias_map.get(raw_hint) if intent is None: logger.warning("ai_router_invalid_intent_hint", intent_hint=raw_hint) return None return IntentResult( intent=intent, confidence=1.0, method="context_hint", matched_keywords=[f"context:{raw_hint}"], detected_resources=[], reasoning=f"context intent_hint={raw_hint}", ) async def route( self, text: str, context: dict | None = None, ) -> RoutingDecision: """ 路由請求到最適 AI Provider 和模型 延遲目標: < 50ms (規則引擎優先，LLM 分類時可能稍長) Args: text: 用戶輸入或告警內容 context: 額外上下文 (服務、指標等) Returns: RoutingDecision: 完整路由決策 """ start_time = time.perf_counter() context = context or {} # Step 1: 意圖分類 (返回 IntentResult, 規則引擎 < 10ms) intent_result = self._resolve_intent_from_context(context) if intent_result is None: intent_result = await self._intent_classifier.classify(text) intent = normalize_intent(intent_result.intent) # Step 2: 複雜度評分 (< 10ms) complexity = self._complexity_scorer.score(context) # Step 3: Provider + Model 選擇 (< 1ms) provider, model, reason = self._select_provider_and_model( intent, intent_result, complexity ) # Step 3b: collect failover health evidence without allowing the probe # result to reorder or truncate the executable production chain. A # health-selected Gemini primary previously let AIRouter jump directly # to the paid provider without attempting GCP-A/GCP-B/host111. Runtime # execution is now always normalized to the concrete first hop; the # executor owns bounded failover and Gemini's atomic cost gate. observed_fallback_provider: str | None = None if provider == AIProviderEnum.OLLAMA: try: failover_result = await self._failover_manager.select_provider( task_type=( "alert_fast" if intent == IntentType.ALERT_TRIAGE else intent.value if intent else "general" ), emit_side_effects=True, ) observed_fallback = getattr( failover_result, "observed_fallback", None, ) observed_fallback_provider = ( str(observed_fallback.provider_name) if observed_fallback else None ) reason = ( f"{reason} [selected primary=ollama_gcp_a; " f"observed fallback={observed_fallback_provider or 'none'}; " "execution order remains fixed]" ) except Exception as e: # Health evidence is best-effort for free lanes. Failure to # probe cannot authorize a paid-provider jump. logger.warning("ai_router_failover_manager_error", error=str(e)) provider = AIProviderEnum.OLLAMA_GCP_A # Step 4: 建立 Fallback 鏈 # 2026-04-05 Claude Code: v4.3 — NIM 從 Phase 22 起就是主力，無隱私問題 # 2026-04-25 P1.2: 若 failover_manager 回傳了 fallback chain，優先使用 # 2026-04-27 Claude Sonnet 4.6: A2 INC-20260425 — DIAGNOSE 專用 chain（排除 Ollama） # failover_manager 只在 OLLAMA 路徑觸發（Step 3b 限制），DIAGNOSE→OPENCLAW_NEMO # 不會進入 failover 路徑，因此 fallover_fallback 此時為 None，走 _build_fallback_chain_for_intent fallback_chain = self._build_fallback_chain_for_intent(provider, intent) # Step 5: 計算延遲預算 latency_budget = PROVIDER_LATENCY_BUDGET.get(provider, 30000) # 計算路由決策耗時 routing_latency = (time.perf_counter() - start_time) * 1000 decision = RoutingDecision( selected_provider=provider, selected_model=model, fallback_chain=fallback_chain, routing_reason=reason, latency_budget_ms=latency_budget, intent=intent, intent_result=intent_result, complexity=complexity, routing_latency_ms=routing_latency, observed_fallback_provider=observed_fallback_provider, ) logger.info( "ai_routing_decision", provider=provider.value, model=model, intent=intent.value, intent_confidence=intent_result.confidence, risk_level=intent_result.risk_level.value, complexity_score=complexity.score, reason=reason, latency_budget_ms=latency_budget, routing_latency_ms=round(routing_latency, 2), fallback_count=len(fallback_chain), observed_fallback_provider=observed_fallback_provider, ) return decision def _select_provider_and_model( self, intent: IntentType, intent_result: IntentResult, complexity: ComplexityScore, ) -> tuple[AIProviderEnum, str, str]: """ 選擇 Provider 和模型 (Phase 13.3 #87 核心邏輯) 所有 intent / risk / complexity 只選擇 Ollama model；provider execution order 永遠是 GCP-A、GCP-B、host111、Claude，最後才是 Gemini。 Args: intent: 正規化後的意圖 intent_result: 完整分類結果 complexity: 複雜度評分 Returns: (provider, model, reason) """ risk = intent_result.risk_level score = complexity.score # Intent overrides may choose only the logical Ollama lane. Risk and # complexity can select a heavier Ollama model, never another provider. provider_override = self._intent_provider_overrides.get(intent) if provider_override is not None: provider = AIProviderEnum.OLLAMA if intent in ( IntentType.ALERT_TRIAGE, IntentType.QUERY ): model = self._ollama_summary else: model = self._ollama_default reason = ( f"意圖 {intent.value} 使用全域 production chain；" "provider 起點固定 GCP-A Ollama" ) return provider, model, reason provider = AIProviderEnum.OLLAMA model = self._ollama_summary if score <= 1 else self._ollama_default reason = ( f"複雜度={score}/5, 風險={risk.value}；全域 production chain " "GCP-A -> GCP-B -> host111 -> Claude -> Gemini" ) return provider, model, reason def _select_model( self, intent: IntentType, intent_result: IntentResult, complexity: ComplexityScore, ) -> tuple[str, str]: """ 選擇模型 (向後相容方法) Deprecated: 請使用 _select_provider_and_model Args: intent: 正規化後的意圖 intent_result: 完整分類結果 complexity: 複雜度評分 Returns: (model_name, reason) """ _, model, reason = self._select_provider_and_model( intent, intent_result, complexity ) return model, reason def _build_fallback_chain( self, selected_provider: AIProviderEnum ) -> list[tuple[AIProviderEnum, str]]: """ # DEPRECATED 2026-04-28 — 已由 _build_fallback_chain_for_intent 取代，無呼叫方 建立 Fallback 鏈 (排除已選 Provider) Fallback 順序: GCP-A → GCP-B → host111 → Claude → Gemini Args: selected_provider: 已選擇的 Provider Returns: Fallback 鏈 [(provider, model), ...] """ return self._build_fallback_chain_for_intent( selected_provider, IntentType.UNKNOWN, ) def _build_fallback_list(self, selected_model: str) -> list[str]: """建立 Fallback 列表 (向後相容)""" fallbacks = [m for m in self._fallback_order if m != selected_model] return fallbacks def _build_fallback_chain_for_intent( self, selected_provider: AIProviderEnum, intent: IntentType, ) -> list[tuple[AIProviderEnum, str]]: """ 建構單一 production fallback 鏈，排除已選的第一跳。 intent 參數僅為介面相容，不得改寫 provider order。 Args: selected_provider: 已選擇的 primary Provider intent: 正規化後的意圖 Returns: Fallback 鏈 [(provider, model), ...]，排除 selected_provider """ _ = intent canonical_selected = ( AIProviderEnum.OLLAMA_GCP_A if selected_provider == AIProviderEnum.OLLAMA else selected_provider ) providers = [provider for provider, _model in self._full_fallback_chain] if canonical_selected not in providers: return self._full_fallback_chain.copy() selected_index = providers.index(canonical_selected) return self._full_fallback_chain[selected_index + 1 :] def route_sync( self, text: str, context: dict | None = None, ) -> RoutingDecision: """ 同步版本路由 (僅關鍵字匹配，保證 < 50ms) 適用場景: 需要快速決策，不需要 LLM 分類的情況 Args: text: 用戶輸入或告警內容 context: 額外上下文 Returns: RoutingDecision: 路由決策 """ start_time = time.perf_counter() context = context or {} # 同步分類 (僅規則引擎, < 10ms) intent_result = self._resolve_intent_from_context(context) if intent_result is None: intent_result = self._intent_classifier.classify_sync(text) intent = normalize_intent(intent_result.intent) # 複雜度評分 (< 10ms) complexity = self._complexity_scorer.score(context) # Provider + Model 選擇 provider, model, reason = self._select_provider_and_model( intent, intent_result, complexity ) # route_sync cannot run health probes, so publish the concrete first hop # rather than the legacy logical ``ollama`` alias. if provider == AIProviderEnum.OLLAMA: provider = AIProviderEnum.OLLAMA_GCP_A reason = f"{reason} [sync concrete first hop=ollama_gcp_a]" # 建立固定 production fallback 鏈。 fallback_chain = self._build_fallback_chain_for_intent(provider, intent) # 延遲預算 latency_budget = PROVIDER_LATENCY_BUDGET.get(provider, 30000) # 計算路由決策耗時 routing_latency = (time.perf_counter() - start_time) * 1000 return RoutingDecision( selected_provider=provider, selected_model=model, fallback_chain=fallback_chain, routing_reason=reason, latency_budget_ms=latency_budget, intent=intent, intent_result=intent_result, complexity=complexity, routing_latency_ms=routing_latency, ) # ========================================================================= # Tool Calling 路由 (ADR-036) # ========================================================================= def route_tool_calling(self) -> tuple[AIProviderEnum, str, list[tuple[AIProviderEnum, str]]]: """ Tool Calling 使用同一 production 路由；Nemotron 僅保留 shadow metadata。 Returns: (provider, model, fallback_chain) """ provider = AIProviderEnum.OLLAMA_GCP_A model = self._model_registry.get_model("ollama", "trust_scoring") fallback_chain = [ (p, m) for p, m in self._tool_calling_fallback_chain if p != provider ] logger.info( "tool_calling_routing", provider=provider.value, model=model, fallback_count=len(fallback_chain), ) return provider, model, fallback_chain def get_tool_calling_fallback_chain(self) -> list[tuple[AIProviderEnum, str]]: """取得 Tool Calling Fallback 鏈""" return self._tool_calling_fallback_chain.copy() # ========================================================================= # 便捷方法 # ========================================================================= def get_provider_for_intent(self, intent: IntentType) -> AIProviderEnum: """取得意圖對應的 Provider (不考慮複雜度)""" _ = intent return AIProviderEnum.OLLAMA_GCP_A def get_model_for_provider(self, provider: AIProviderEnum) -> str: """取得 Provider 對應的模型""" return self._provider_models.get(provider, self._ollama_default) def get_routing_matrix(self) -> list[dict]: """ 取得路由決策矩陣 (用於 API 文檔或除錯) Returns: 路由規則清單 """ return [ { "rule": 1, "condition": "all intents, risks, complexities, tools and background jobs", "provider": "ollama_gcp_a", "provider_order": list(PRODUCTION_PROVIDER_ORDER), "reason": "single global production route; paid providers require cost gates and legacy providers remain shadow metadata only", } ] async def feedback_from_aider_events( self, repo: str | None = None, days: int = 7, ) -> dict[str, float]: """從 aider_events 聚合近 N 日 success rate per model。 Phase 24 ADR-052 延伸：AI 自主化 feedback loop。 目前為 read-only（未接入 route() 決策），等 USE_AIDER_FEEDBACK flag + 7 天灰度驗證後才會調整 provider 權重。 Args: repo: 若給定，只聚合該 repo 的 session；否則所有 repo。 days: 時間窗口（預設 7 天）。 Returns: {model_name: success_rate_float} e.g. {"elephant-alpha": 0.85, "gemini-pro": 0.92} 空 dict 代表無資料或查詢失敗（caller 應降級為忽略）。 """ try: from src.db.base import get_db_context from src.repositories.aider_event_repository import AiderEventRepository except ImportError: return {} try: async with get_db_context() as sess: repo_obj = AiderEventRepository(sess) stats = await repo_obj.model_stats_since(days=days) except Exception: logger.debug("ai_router_feedback_aggregation_failed") return {} return self._aggregate_feedback_stats(stats, repo=repo) @staticmethod def _aggregate_feedback_stats( stats: list[dict], repo: str | None = None ) -> dict[str, float]: """純函數：過濾 repo 並將 stats 轉換為 {model: success_rate}（可獨立單元測試）。""" out: dict[str, float] = {} for row in stats: if repo and row.get("repo") != repo: continue model = row.get("model") if not model: continue out[model] = float(row.get("success_rate") or 0) return out # ============================================================================= # Phase 24 ADR-052: AI Provider Registry + Execution Layer # ============================================================================= # 2026-04-02 ogt: 在現有 AIRouter (路由決策) 之上，加入 Provider 執行層 # 整合: ProviderRegistry + 閘門 (CB/RL/Sem) + Cache + Langfuse Trace # # 呼叫關係: # openclaw.py → AIRouterExecutor.execute() → AIRouter.route() → Provider.analyze() # ============================================================================= _settings = get_settings() class _SimpleCircuitBreaker: """ 輕量 per-provider Circuit Breaker (Phase 24 C2 修復) 不共用 OpenClawGuard — 避免 Gemini 掛掉時 Ollama 也被擋 """ def __init__(self, name: str, failure_threshold: int = 5, recovery_timeout: float = 60.0) -> None: self.name = name self._failure_threshold = failure_threshold self._recovery_timeout = recovery_timeout self._failure_count = 0 self._last_failure_time: float = 0.0 def is_open(self) -> bool: if self._failure_count < self._failure_threshold: return False # 超過 recovery timeout → half-open (允許一次嘗試) if time.time() - self._last_failure_time > self._recovery_timeout: return False return True def record_success(self) -> None: self._failure_count = 0 def record_failure(self) -> None: self._failure_count += 1 self._last_failure_time = time.time() def cooldown_remaining_seconds(self) -> int: if self._failure_count < self._failure_threshold: return 0 return max( 0, int(self._recovery_timeout - (time.time() - self._last_failure_time)), ) class AIProviderRegistry: """ AI Provider 註冊中心 — 類比 MCP ProviderRegistry (ADR-015) 動態管理 AI Provider 的生命週期與啟停狀態。 """ def __init__(self) -> None: self._providers: dict[str, AIProviderProtocol] = {} self._shadow_providers: dict[str, AIProviderProtocol] = {} def register(self, provider: AIProviderProtocol) -> None: """註冊 Provider (啟動時呼叫)""" if provider.name in NON_EXECUTABLE_SHADOW_PROVIDERS: self._shadow_providers[provider.name] = provider logger.info( "ai_provider_registered_shadow_metadata_only", name=provider.name, production_executable=False, ) return self._providers[provider.name] = provider status = "enabled" if provider.is_enabled else "disabled" logger.info("ai_provider_registered", name=provider.name, status=status, privacy=provider.privacy_level) def get(self, name: str) -> AIProviderProtocol | None: """取得已啟用的 Provider""" p = self._providers.get(name) if p and p.is_enabled: return p return None def all_enabled(self) -> list[AIProviderProtocol]: """取得所有已啟用的 Provider""" return [p for p in self._providers.values() if p.is_enabled] def names(self) -> list[str]: """所有已註冊 Provider 名稱""" return [*self._providers.keys(), *self._shadow_providers.keys()] async def health_check_all(self) -> dict[str, bool]: """所有 Provider 健康狀態""" results = {name: False for name in self._shadow_providers} for name, p in self._providers.items(): try: results[name] = await p.health_check() except Exception: results[name] = False return results async def close_all(self) -> None: """關閉所有 Provider 的 HTTP 連線 (I5 修復: shutdown hook)""" for name, p in self._providers.items(): try: if hasattr(p, "close"): await p.close() logger.info("ai_provider_closed", name=name) except Exception as e: logger.warning("ai_provider_close_failed", name=name, error=str(e)) class AIRouterExecutor: """ AI Router 執行層 (Phase 24 ADR-052) 職責: 1. Cache 檢查 (Redis, 跨 Provider 共享) — D4 2. 閘門控制 (Circuit Breaker → Rate Limiter → Semaphore) — D3 3. 呼叫 Provider.analyze() — 實際執行 4. 記錄 Langfuse Trace — D5 5. Mock Mode 攔截 — D13 設計原則: - 只依賴 AIProviderProtocol，禁止 import 具體 Provider 類別 - 閘門在 Router，Provider 保持純粹 (Stateless Compute Units) """ def __init__(self, registry: AIProviderRegistry) -> None: self._registry = registry self._semaphores: dict[str, asyncio.Semaphore] = {} # C2 修復: per-provider Circuit Breaker (不共用，避免一個掛全部擋) self._circuit_breakers: dict[str, _SimpleCircuitBreaker] = {} def _get_semaphore(self, name: str, limit: int = 3) -> asyncio.Semaphore: """取得 Provider 的並發 Semaphore (lazy init)""" if name not in self._semaphores: self._semaphores[name] = asyncio.Semaphore(limit) return self._semaphores[name] def _get_circuit_breaker(self, name: str) -> _SimpleCircuitBreaker: """取得 Provider 的 Circuit Breaker (per-provider, lazy init)""" if name not in self._circuit_breakers: # 2026-04-05 Claude Code: v4.3 — NIM 使用更寬鬆的 CB 參數 # 每次都先跑 NIM，只有真正連線錯誤（非 timeout）才累積失敗 # failure_threshold=10: 需要 10 次真實錯誤才 OPEN（timeout 不計） # recovery_timeout=30: 30s 後進入 half-open，立即重試 NIM if name == "nemotron": self._circuit_breakers[name] = _SimpleCircuitBreaker( name, failure_threshold=10, recovery_timeout=30.0 ) elif name == "claude": self._circuit_breakers[name] = _SimpleCircuitBreaker( name, failure_threshold=3, recovery_timeout=120.0 ) else: self._circuit_breakers[name] = _SimpleCircuitBreaker(name) return self._circuit_breakers[name] @staticmethod def _asset_execution_preflight(context: dict | None) -> AIResult | None: """Reject unresolved identities and open asset circuits before any model call.""" context = context or {} route = context.get("typed_target_route") or context.get( "typedTargetRoute" ) identity = context.get("canonical_asset_identity") route = route if isinstance(route, dict) else {} identity = identity if isinstance(identity, dict) else {} resolution_values = { str(value or "").strip().lower() for value in ( route.get("resolution_status"), identity.get("resolution_status"), context.get("asset_identity_resolution_status"), ) if str(value or "").strip() } canonical_asset_id = str( route.get("canonical_asset_id") or identity.get("canonical_id") or context.get("canonical_asset_id") or "" ).strip() identity_unresolved = bool( resolution_values & { "ambiguous", "asset_identity_ambiguous", "asset_identity_unresolved", "unknown", "unresolved", } ) or bool( "resolved" in resolution_values and (route or identity) and not canonical_asset_id ) if identity_unresolved: identity_seed = str( context.get("target_resource") or "asset-identity-unresolved" ) digest = hashlib.sha256(identity_seed.encode("utf-8")).hexdigest() return AIResult( raw_response="", success=False, provider="none", error="asset_identity_unresolved", audit_metadata={ "schema_version": "ai_asset_identity_fail_closed_v1", "status": "asset_identity_unresolved", "provider_network_call_performed": False, "cross_provider_fallback_allowed": False, "cross_domain_fallback_allowed": False, "drift_work_item_id": f"AIA-ASSET-DRIFT-{digest[:12].upper()}", }, ) circuit_state = str( route.get("circuit_state") or identity.get("circuit_state") or context.get("asset_circuit_state") or "" ).strip().lower() circuit_open = circuit_state == "open" or any( value is True for value in ( route.get("circuit_open"), identity.get("circuit_open"), context.get("asset_circuit_open"), ) ) if not circuit_open: return None asset_seed = canonical_asset_id or str( context.get("target_resource") or "asset-circuit" ) digest = hashlib.sha256(asset_seed.encode("utf-8")).hexdigest() domain = str( route.get("target_kind") or identity.get("asset_domain") or context.get("asset_domain") or "unknown" ).strip().lower() return AIResult( raw_response="", success=False, provider="none", error="asset_circuit_open_fail_closed", audit_metadata={ "schema_version": "ai_asset_circuit_open_terminal_v1", "status": "circuit_open", "circuit_scope": "asset", "asset_identity_sha256": digest, "canonical_domain": domain, "provider_network_call_performed": False, "cross_provider_fallback_allowed": False, "cross_domain_fallback_allowed": False, "allowed_actions": [ "same_asset_domain_bounded_cooldown_probe", "create_same_asset_domain_repair_work_item", ], "repair_work_item": { "work_item_id": f"AIA-CIRCUIT-ASSET-{digest[:16].upper()}", "domain": domain, "execution_allowed": False, "required_next_receipt": "bounded_cooldown_or_repair_verifier", }, }, ) @staticmethod def _provider_circuit_open_result( *, provider_name: str, circuit_breaker: _SimpleCircuitBreaker, context: dict | None, ) -> AIResult: work_item_seed = str((context or {}).get("work_item_id") or "") digest = hashlib.sha256( f"{provider_name}|{work_item_seed}".encode() ).hexdigest() return AIResult( raw_response="", success=False, provider="none", error=f"provider_circuit_open_fail_closed:{provider_name}", audit_metadata={ "schema_version": "ai_provider_circuit_open_terminal_v1", "status": "circuit_open", "circuit_scope": "provider", "canonical_provider_lane": provider_name, "provider_network_call_performed": False, "cross_provider_fallback_allowed": False, "cross_domain_fallback_allowed": False, "cooldown_remaining_seconds": ( circuit_breaker.cooldown_remaining_seconds() ), "allowed_actions": [ "same_provider_bounded_cooldown_probe", "create_same_provider_repair_work_item", ], "repair_work_item": { "work_item_id": f"AIA-CIRCUIT-PROVIDER-{digest[:16].upper()}", "provider": provider_name, "execution_allowed": False, "required_next_receipt": "same_provider_cooldown_verifier", }, }, ) @staticmethod async def _attach_gcp_transport_receipt( *, provider_name: str, provider_context: dict | None, ) -> tuple[dict | None, str | None]: """Issue and attach one same-run GCP transport receipt fail-closed.""" if provider_name not in {"ollama_gcp_a", "ollama_gcp_b"}: return provider_context, None if not isinstance(provider_context, dict): return None, "gcp_cloud_transport_context_missing" from src.services.cloud_transport_receipt import ( bounded_cloud_transport_receipt_hints, issue_cloud_transport_receipt, resolve_gcp_transport_endpoint, verify_cloud_transport_receipt, ) endpoint_url = resolve_gcp_transport_endpoint(provider_name) existing_reason = await verify_cloud_transport_receipt( context=provider_context, provider=provider_name, endpoint_url=endpoint_url, ) if existing_reason is None: return provider_context, None try: receipt = await issue_cloud_transport_receipt( trace_id=str(provider_context.get("trace_id") or ""), run_id=str(provider_context.get("run_id") or ""), work_item_id=str(provider_context.get("work_item_id") or ""), provider=provider_name, endpoint_url=endpoint_url, ) except Exception as error: logger.warning( "ai_router_gcp_transport_preflight_failed", provider=provider_name, existing_reason=existing_reason, error_type=type(error).__name__, ) return None, "gcp_cloud_transport_preflight_failed" receipts = bounded_cloud_transport_receipt_hints( provider_context.get("cloud_transport_receipts") ) receipts = [ item for item in receipts if str(item.get("provider") or "") != provider_name ] receipts.append(receipt) updated_context = dict(provider_context) updated_context["cloud_transport_receipts"] = receipts return updated_context, None @staticmethod def _cache_key(prompt: str, context: dict | None) -> str: """生成 Cache Key (與 openclaw.py 相容)""" ctx_hash = "" if context: cloud_context = context.get("paid_cloud_context") or {} receipt = ( cloud_context.get("cloud_sanitization_receipt") or {} if isinstance(cloud_context, dict) else {} ) ctx_hash = ( f":{context.get('alert_type', '')}:{context.get('target_resource', '')}:" f"{(cloud_context or {}).get('data_classification', context.get('data_classification', ''))}:" f"{receipt.get('receipt_id', '') if isinstance(receipt, dict) else ''}" ) content = f"{prompt}{ctx_hash}" return f"llm_cache:{hashlib.sha256(content.encode()).hexdigest()[:16]}" @staticmethod def _sre_chat_sanitization_receipt_block_reason( prompt: str, context: dict | None, ) -> str | None: """Verify the formal prompt-bound receipt for real SRE conversation.""" root = context or {} receipt = root.get("sre_chat_sanitization_receipt") if not isinstance(receipt, dict): # Preserve the established generic cloud boundary code for callers # that provide neither an alert nor a real-chat receipt. return "cloud_sanitization_receipt_missing" prompt_sha256 = hashlib.sha256(prompt.encode("utf-8")).hexdigest() if receipt.get("prompt_sha256") != prompt_sha256: return "sre_chat_sanitized_prompt_mismatch" correlation_fields = ("trace_id", "run_id", "work_item_id") if ( receipt.get("schema_version") != "sre_chat_sanitization_receipt_v1" or receipt.get("status") != "verified" or receipt.get("sanitizer") != "src.services.sanitization_service.sanitize" or receipt.get("source_label") != "awoooi_sre_group_chat" or receipt.get("data_classification") != "sanitized" or receipt.get("raw_input_forwarded") is not False or receipt.get("raw_log_payload_forwarded") is not False or receipt.get("secret_value_exposed") is not False or receipt.get("private_network_value_exposed") is not False or receipt.get("infrastructure_remediation_write_allowed") is not False or receipt.get("provider_timeout_seconds") != 35 or root.get("data_classification") != "sanitized" or root.get("infrastructure_remediation_write_allowed") is not False or root.get("provider_timeout_seconds") != 35 or root.get("executor_invocation_allowed") is not False or root.get("agent99_dispatch_allowed") is not False or root.get("cross_domain_execution_allowed") is not False or any( not str(root.get(field) or "") or receipt.get(field) != root.get(field) for field in correlation_fields ) ): return "sre_chat_sanitization_receipt_invalid" return None @staticmethod def _paid_alert_execution_inputs( prompt: str, context: dict | None, ) -> tuple[str | None, dict | None, str | None]: """Validate an allowlist sanitizer receipt and isolate cloud inputs.""" is_alert = bool(context) and any( key in context for key in ("alert_type", "alertname", "alert_name", "incident_id", "signals") ) if not is_alert: root = context or {} if root.get("synthetic_validation") is True: if ( root.get("data_classification") == "sanitized" and root.get("infrastructure_remediation_write_allowed") is False and root.get("synthetic_prompt_sha256") == hashlib.sha256(prompt.encode("utf-8")).hexdigest() ): return prompt, context, None return None, None, "cloud_synthetic_prompt_receipt_invalid" chat_block = AIRouterExecutor._sre_chat_sanitization_receipt_block_reason( prompt, context, ) if chat_block is None: return prompt, context, None return None, None, chat_block cloud_context = (context or {}).get("paid_cloud_context") paid_prompt = (context or {}).get("paid_cloud_prompt") if not isinstance(cloud_context, dict) or not isinstance(paid_prompt, str): return None, None, "alert_cloud_sanitization_receipt_missing" receipt = cloud_context.get("cloud_sanitization_receipt") payload = cloud_context.get("sanitized_payload") if not isinstance(receipt, dict) or not isinstance(payload, dict): return None, None, "alert_cloud_sanitization_receipt_missing" if ( receipt.get("schema_version") != "openclaw_paid_cloud_sanitization_v1" or receipt.get("status") != "verified" or receipt.get("sanitizer") != "src.services.sanitization_service.sanitize" or receipt.get("classifier") != "allowlisted_structured_alert_fields_v1" or receipt.get("dlp_policy") != "paid_cloud_allowlist_dlp_v1" or receipt.get("raw_payload_forwarded") is not False or receipt.get("raw_log_payload_forwarded") is not False or receipt.get("secret_value_exposed") is not False or receipt.get("private_network_value_exposed") is not False ): return None, None, "alert_cloud_sanitization_receipt_invalid" canonical = _json.dumps( payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"), ) payload_sha256 = hashlib.sha256(canonical.encode("utf-8")).hexdigest() prompt_sha256 = hashlib.sha256(paid_prompt.encode("utf-8")).hexdigest() if receipt.get("payload_sha256") != payload_sha256: return None, None, "alert_cloud_sanitization_payload_mismatch" if cloud_context.get("paid_prompt_sha256") != prompt_sha256: return None, None, "alert_cloud_sanitized_prompt_mismatch" # Receipt fields are caller-controlled until they are rebound to the # original alert through the canonical allowlist sanitizer. A matching # self-signed hash alone is not an authority boundary. from src.services.openclaw import build_paid_cloud_alert_context rebuilt_context, rebuilt_reason = build_paid_cloud_alert_context(context) if rebuilt_context is None: return None, None, rebuilt_reason rebuilt_receipt = rebuilt_context.get("cloud_sanitization_receipt") or {} if ( rebuilt_context.get("sanitized_payload") != payload or rebuilt_receipt.get("receipt_id") != receipt.get("receipt_id") ): return None, None, "alert_cloud_sanitization_receipt_untrusted" restricted_context = dict(cloud_context) for correlation_field in ("trace_id", "run_id", "work_item_id"): if (context or {}).get(correlation_field): restricted_context[correlation_field] = (context or {})[ correlation_field ] return paid_prompt, restricted_context, None @staticmethod async def _bounded_ollama_unavailable_receipts( context: dict | None, ) -> set[str]: """Verify current-run free-lane unavailability against durable truth. Caller-provided JSON is only a lookup hint. The full receipt must exist independently in Redis, carry a short expiry, and acknowledge a completed failover-manager verifier before it can replace an attempt. """ run_id = str((context or {}).get("run_id") or "") if not run_id: return set() raw_receipts = (context or {}).get("ollama_unavailable_receipts") or [] if isinstance(raw_receipts, dict): raw_receipts = list(raw_receipts.values()) valid: set[str] = set() allowed_reasons = { "endpoint_unreachable", "health_probe_failed", "provider_not_registered", "provider_disabled", "bounded_deadline_exhausted", } try: from src.core.redis_client import get_redis redis = get_redis() except Exception as redis_error: logger.warning( "ai_router_ollama_unavailable_receipt_store_unavailable", error_type=type(redis_error).__name__, ) return set() now = datetime.now(UTC) bounded_receipts = ( raw_receipts[: len(PRODUCTION_OLLAMA_ORDER)] if isinstance(raw_receipts, list) else [] ) for receipt in bounded_receipts: if not isinstance(receipt, dict): continue provider = str(receipt.get("provider") or "") receipt_id = str(receipt.get("receipt_id") or "") if not ( provider in PRODUCTION_OLLAMA_ORDER and receipt.get("schema_version") == _DURABLE_OLLAMA_UNAVAILABLE_RECEIPT_SCHEMA and receipt.get("source") == "ollama_failover_manager" and receipt.get("status") == "bounded_unavailable" and str(receipt.get("run_id") or "") == run_id and receipt.get("check_completed") is True and str(receipt.get("reason") or "") in allowed_reasons and _DURABLE_OLLAMA_UNAVAILABLE_RECEIPT_ID.fullmatch(receipt_id) ): continue try: durable_raw = await redis.get( f"{_DURABLE_OLLAMA_UNAVAILABLE_RECEIPT_PREFIX}{receipt_id}" ) if isinstance(durable_raw, bytes): durable_raw = durable_raw.decode("utf-8") durable = _json.loads(durable_raw) if durable_raw else None except Exception as receipt_error: logger.warning( "ai_router_ollama_unavailable_receipt_read_failed", provider=provider, error_type=type(receipt_error).__name__, ) continue if not isinstance(durable, dict): continue contract_fields = ( "schema_version", "receipt_id", "source", "provider", "run_id", "status", "check_completed", "reason", "observed_at", "expires_at", ) if any(durable.get(field) != receipt.get(field) for field in contract_fields): continue if ( durable.get("durable_write_ack") is not True or durable.get("verifier_status") != "verified" or durable.get("verified_by") != "ollama_failover_manager.health_probe" ): continue try: observed_at = datetime.fromisoformat( str(durable.get("observed_at") or "").replace("Z", "+00:00") ) expires_at = datetime.fromisoformat( str(durable.get("expires_at") or "").replace("Z", "+00:00") ) except ValueError: continue if observed_at.tzinfo is None or expires_at.tzinfo is None: continue lifetime = (expires_at - observed_at).total_seconds() if not ( 0 < lifetime <= _DURABLE_OLLAMA_UNAVAILABLE_MAX_SECONDS and observed_at <= now and now < expires_at ): continue valid.add(provider) return valid async def execute( self, prompt: str, provider_order: list[str], context: dict | None = None, cache_ttl: int = 3600, require_local: bool = False, ) -> AIResult: """ 核心執行方法 — 依序嘗試 Provider，含閘門 + Cache Args: prompt: LLM prompt provider_order: Provider 名稱順序 (由 AIRouter.route 決定) context: 額外上下文 cache_ttl: Cache TTL (秒) require_local: 強制 local Provider (隱私) Returns: AIResult: 標準化結果 """ asset_preflight = self._asset_execution_preflight(context) if asset_preflight is not None: logger.warning( "ai_router_asset_identity_or_circuit_blocked", reason=asset_preflight.error, audit_schema=asset_preflight.audit_metadata.get("schema_version"), ) return asset_preflight # ① Mock Mode 攔截 (D13) if _settings.MOCK_MODE: logger.info("ai_router_mock_mode") return AIResult( raw_response=_json.dumps({ "action_title": "Mock Analysis", "description": "Mock mode enabled", "risk_level": "low", "reasoning": "MOCK_MODE=true", "confidence": 0.0, }), success=True, provider="mock", ) requested_provider_order = list(provider_order) provider_order, rejected_providers = normalize_production_execution_order( provider_order, require_local=require_local, ) if rejected_providers: logger.warning( "ai_router_nonproduction_provider_insertion_rejected", rejected_providers=rejected_providers, requested_provider_order=requested_provider_order, normalized_provider_order=provider_order, ) if not provider_order: logger.error( "ai_router_production_route_missing_ollama_lane", requested_provider_order=requested_provider_order, ) return AIResult( raw_response="", success=False, provider="none", error="production_route_missing_ollama_lane", ) # Paid-provider runtime control is evaluated before cache lookup. This # prevents a cached cloud response from bypassing a newly asserted # durable disable and prevents Redis read failures from silently # re-adding either paid lane. preflight_errors: list[str] = [] from src.services.ai_control import is_provider_disabled from src.services.ai_providers.interfaces import ( cloud_context_block_reason, is_provider_enabled_by_env, ) paid_prompt, paid_context, paid_input_block = self._paid_alert_execution_inputs( prompt, context, ) cloud_privacy_block = paid_input_block or cloud_context_block_reason( paid_context, require_local=require_local, ) if cloud_privacy_block: blocked_cloud_providers = [ provider for provider in SANITIZED_CLOUD_PROVIDER_ORDER if provider in provider_order ] provider_order = [ provider for provider in provider_order if provider not in SANITIZED_CLOUD_PROVIDER_ORDER ] preflight_errors.extend( f"{provider}: {cloud_privacy_block}" for provider in blocked_cloud_providers ) logger.warning( "ai_router_cloud_context_blocked_pre_cache", reason=cloud_privacy_block, blocked_providers=blocked_cloud_providers, ) if any(provider in provider_order for provider in PAID_PROVIDER_ORDER): for paid_provider in PAID_PROVIDER_ORDER: if paid_provider not in provider_order: continue if not is_provider_enabled_by_env(paid_provider): provider_order = [ provider for provider in provider_order if provider != paid_provider ] preflight_errors.append( f"{paid_provider}: disabled_by_environment" ) logger.info( "ai_router_provider_environment_disabled_pre_cache", provider=paid_provider, ) continue try: if await is_provider_disabled( paid_provider, run_id=str((paid_context or context or {}).get("run_id") or ""), ): provider_order = [ provider for provider in provider_order if provider != paid_provider ] preflight_errors.append( f"{paid_provider}: disabled_by_runtime_control" ) logger.info( "ai_router_provider_disabled_pre_cache", provider=paid_provider, ) except Exception as disable_check_error: provider_order = [ provider for provider in provider_order if provider != paid_provider ] preflight_errors.append( f"{paid_provider}: disabled_state_unavailable" ) logger.warning( "ai_router_paid_disable_state_unavailable_pre_cache_blocked", provider=paid_provider, error_type=type(disable_check_error).__name__, ) # A circuit-open first hop is a terminal route condition, not an # unavailability receipt authorizing another host or provider. Check it # before cache lookup so stale generations cannot bypass the circuit. if provider_order: first_provider = provider_order[0] first_circuit = self._get_circuit_breaker(first_provider) if first_circuit.is_open(): logger.warning( "ai_router_provider_circuit_open_fail_closed", provider=first_provider, cross_provider_fallback_allowed=False, ) return self._provider_circuit_open_result( provider_name=first_provider, circuit_breaker=first_circuit, context=context, ) # ② Cache 檢查 (D4) cache_key = self._cache_key(prompt, context) # C3 修復: 移到 try 外避免 UnboundLocalError try: from src.core.redis_client import get_redis redis = get_redis() cached = await redis.get(cache_key) if cached: data = _json.loads(cached) cached_provider = data.get("provider", "cache") if _is_paid_provider_identity(cached_provider): logger.info( "ai_router_paid_cache_bypassed", cache_key=cache_key[:30], cached_provider=cached_provider, reason="current_policy_and_ordered_attempt_recheck_required", ) raise ValueError("paid provider cache is non-executable") provider_allowed = cached_provider in provider_order ollama_first_required = ( bool(context) and any( key in context for key in ( "alert_type", "alertname", "alert_name", "fingerprint", "incident_id", "severity", "target_resource", ) ) and bool(provider_order) and provider_order[0].startswith("ollama") ) or ( bool(context) and bool(context.get("enforce_ollama_first")) and bool(provider_order) and provider_order[0].startswith("ollama") ) if ( cached_provider == "ollama" and any(provider.startswith("ollama") for provider in provider_order) ): provider_allowed = True if ollama_first_required and not cached_provider.startswith("ollama"): provider_allowed = False if not provider_allowed: logger.info( "ai_router_cache_provider_mismatch_skip", cache_key=cache_key[:30], cached_provider=cached_provider, provider_order=provider_order, ollama_first_required=ollama_first_required, ) raise ValueError("cached provider not allowed by current provider_order") logger.info("ai_router_cache_hit", cache_key=cache_key[:30]) return AIResult( raw_response=data.get("response", ""), success=True, provider=cached_provider, from_cache=True, ) except Exception as e: logger.debug( "ai_router_cache_read_failed", error_type=type(e).__name__, ) # ③ 遍歷 Provider + 閘門 (D3) # 2026-04-02 ogt: C1 修復 — 建立 Langfuse Trace (D5) # 包住整個執行鏈，記錄每個 Provider 的 generation try: from src.services.langfuse_client import langfuse_trace alert_type_value = str((context or {}).get("alert_type", "")) _lf_trace_ctx = langfuse_trace( "ai_router_execute", metadata={ "provider_order": provider_order, "prompt_length": len(prompt), "require_local": require_local, "alert_type_length": len(alert_type_value), "alert_type_sha256": hashlib.sha256( alert_type_value.encode("utf-8") ).hexdigest(), }, ) _lf_trace_ctx.__enter__() except Exception: _lf_trace_ctx = None errors: list[str] = list(preflight_errors) attempted_providers: set[str] = set() bounded_unavailable_providers = ( await self._bounded_ollama_unavailable_receipts(context) ) provider_timeout_seconds: float | None = None try: configured_provider_timeout = float( (context or {}).get("provider_timeout_seconds") or 0 ) if configured_provider_timeout > 0: provider_timeout_seconds = configured_provider_timeout except (TypeError, ValueError): logger.warning( "ai_router_invalid_provider_timeout", value=(context or {}).get("provider_timeout_seconds"), ) alert_requires_ollama_before_cloud = bool( (context or {}).get("alert_requires_ollama_before_cloud") ) # 2026-04-27 Claude Sonnet 4.6: A2 INC-20260425 — DIAGNOSE fallback metric 追蹤 # 透過 context.get("intent_hint") 判斷是否為 DIAGNOSE，避免改動 execute() 簽名 # _last_attempted_provider 記錄上一輪嘗試的 provider，用於計算 from→to 關係 _is_diagnose_intent = str((context or {}).get("intent_hint", "")).strip().lower() == "diagnose" _last_attempted_provider: str | None = None for provider_name in provider_order: # Opening a provider circuit terminates this canonical lane. It is # checked before fallback metrics and registry/provider work so the # skipped hop cannot be reported as an attempted fallback. cb = self._get_circuit_breaker(provider_name) if cb.is_open(): logger.warning( "ai_router_provider_circuit_open_fail_closed", provider=provider_name, cross_provider_fallback_allowed=False, ) if _lf_trace_ctx: try: _lf_trace_ctx.__exit__(None, None, None) except Exception: pass return self._provider_circuit_open_result( provider_name=provider_name, circuit_breaker=cb, context=context, ) # 2026-04-27 Claude Sonnet 4.6: A2 — 若上一輪失敗且本輪開始，表示發生 fallback # 記錄 metric（DIAGNOSE intent 專屬；非 DIAGNOSE 不記，不影響其他路徑） # 2026-04-27 Claude Sonnet 4.6: F6 — fallback metric 只在真實 analyze() 失敗時觸發 # _last_attempted_provider 僅在 provider.analyze() 執行後失敗才賦值（見下方兩處）； # not_registered / privacy_skip / circuit_open / rate_limit 分支均不賦值， # 避免這些「被跳過的 provider」誤計入 from→to fallback 鏈，metric 不可信問題（F6）。 if _is_diagnose_intent and _last_attempted_provider is not None: try: from src.core.metrics import record_diagnose_fallback record_diagnose_fallback( from_provider=_last_attempted_provider, to_provider=provider_name, ) logger.info( "diagnose_fallback_recorded", from_provider=_last_attempted_provider, to_provider=provider_name, ) except Exception as _metric_e: # 2026-04-27 Claude Sonnet 4.6: F6 — 升 warning（原 debug 會 silent swallow） # + 計入 error counter 讓 metric 管道問題可被 Prometheus 偵測 logger.warning("diagnose_fallback_metric_failed", error=str(_metric_e)) try: from src.core.metrics import ( AIOPS_DIAGNOSE_FALLBACK_METRIC_ERROR_TOTAL, ) AIOPS_DIAGNOSE_FALLBACK_METRIC_ERROR_TOTAL.inc() except Exception: pass provider = self._registry.get(provider_name) if provider is None and provider_name == "ollama_gcp_a": # Compatibility for isolated unit registries. Production's # registry exposes both the concrete GCP-A identity and the # legacy logical ``ollama`` alias, while execution receipts # continue to carry the provider's concrete runtime identity. provider = self._registry.get("ollama") if provider is None and provider_name == "ollama_local" and require_local: # Older isolated/local registries expose the logical Ollama # provider only. It may satisfy the concrete host111 lane only # when the provider itself declares a local privacy boundary; # production keeps using its explicit ollama_local identity. legacy_local = self._registry.get("ollama") if ( legacy_local is not None and getattr(legacy_local, "privacy_level", None) == "local" ): provider = legacy_local if not provider: # 2026-04-14 Claude Sonnet 4.6: silent skip 改 errors 累積（觀測性） # 2026-04-27 Claude Sonnet 4.6: F6 — 不設 _last_attempted_provider（未真實執行 analyze） errors.append(f"{provider_name}: not_registered") continue if provider_name in PAID_PROVIDER_ORDER: unresolved_ollama_hops = [ hop for hop in PRODUCTION_OLLAMA_ORDER if hop not in attempted_providers and hop not in bounded_unavailable_providers ] if unresolved_ollama_hops: errors.append( f"{provider_name}: cloud_blocked_ollama_attempt_receipts_missing(" f"{','.join(unresolved_ollama_hops)})" ) logger.warning( "ai_router_cloud_blocked_missing_ollama_attempt_receipts", provider=provider_name, unresolved_ollama_hops=unresolved_ollama_hops, attempted_providers=sorted(attempted_providers), bounded_unavailable_providers=sorted( bounded_unavailable_providers ), ) continue # 隱私過濾 (D7) # 2026-04-27 Claude Sonnet 4.6: F6 — privacy_skip 不設 _last_attempted_provider（未嘗試） if require_local and provider.privacy_level != "local": errors.append(f"{provider_name}: privacy_skip(non_local)") continue # GCP-A/B are cloud-hosted Ollama and therefore non-local for data # classification, but they remain the first two free Ollama hops. # The "Ollama before cloud critic" gate applies only to paid # external critics, otherwise it would incorrectly block GCP-A/B. if ( alert_requires_ollama_before_cloud and provider_name in PAID_PROVIDER_ORDER ): if "ollama_local" not in attempted_providers: errors.append(f"{provider_name}: blocked_until_ollama_local_attempted") logger.warning( "ai_router_cloud_blocked_until_ollama_local_attempted", provider=provider_name, provider_order=provider_order, attempted_providers=sorted(attempted_providers), ) continue # 閘門 2: Rate Limiter # 2026-04-02 Claude Code: Phase 24 B3 + C1 修復 — Rate Limiter (含 openclaw_nemo) # Paid providers own atomic requests/tokens/cost reservations inside # their provider implementations; legacy free/provider lanes retain # this compatibility rate gate. if provider_name in ("openclaw_nemo", "nemotron"): try: from src.services.ai_rate_limiter import get_ai_rate_limiter rate_limiter = get_ai_rate_limiter() allowed, reason = await rate_limiter.check_and_increment(provider_name) if not allowed: errors.append(f"{provider_name}: rate_limit({reason})") logger.info("ai_router_rate_limited", provider=provider_name, reason=reason) continue except Exception as e: logger.debug("ai_router_rate_limiter_error", error=str(e)) # 閘門 3: Semaphore (並發控制) sem = self._get_semaphore(provider_name) async with sem: try: provider_prompt = ( paid_prompt if provider_name in SANITIZED_CLOUD_PROVIDER_ORDER else prompt ) provider_context = ( paid_context if provider_name in SANITIZED_CLOUD_PROVIDER_ORDER else context ) if provider_name in SANITIZED_CLOUD_PROVIDER_ORDER and ( provider_prompt is None or provider_context is None ): errors.append( f"{provider_name}: sanitized_cloud_execution_inputs_missing" ) continue if ( provider_name in {"ollama_gcp_a", "ollama_gcp_b"} and (context or {}).get("cloud_transport_auto_preflight") is True ): provider_context, transport_block = ( await self._attach_gcp_transport_receipt( provider_name=provider_name, provider_context=provider_context, ) ) if transport_block: errors.append( f"{provider_name}: {transport_block}" ) continue paid_context = provider_context attempted_providers.add(provider_name) provider_call = provider.analyze( provider_prompt, provider_context, ) result = ( await asyncio.wait_for( provider_call, timeout=provider_timeout_seconds, ) if provider_timeout_seconds is not None else await provider_call ) if result.success: # 記錄成功 (per-provider CB) cb.record_success() # 記錄費用 if ( result.cost_usd > 0 and provider_name not in PAID_PROVIDER_ORDER ): try: rate_limiter = get_ai_rate_limiter() await rate_limiter.record_cost(provider_name, result.cost_usd) except Exception: pass # Paid responses are deliberately non-cacheable: every # paid candidate must re-run current durable policy, # privacy, canary, cost, and ordered-attempt gates. if provider_name not in SANITIZED_CLOUD_PROVIDER_ORDER: try: redis = get_redis() cache_data = _json.dumps({ "response": result.raw_response, "provider": result.provider, "cached_at": time.strftime("%Y-%m-%dT%H:%M:%S+08:00"), }) await redis.set(cache_key, cache_data, ex=cache_ttl) except Exception: pass logger.info( "ai_router_execute_success", provider=provider_name, latency_ms=round(result.latency_ms, 1), tokens=result.tokens, from_cache=False, ) # D5: 記錄 Langfuse generation if _lf_trace_ctx: try: cloud_provider = ( provider_name in SANITIZED_CLOUD_PROVIDER_ORDER ) prompt_hash = hashlib.sha256( provider_prompt.encode("utf-8") ).hexdigest() response_hash = hashlib.sha256( result.raw_response.encode("utf-8") ).hexdigest() receipt = ( (provider_context or {}).get( "cloud_sanitization_receipt" ) or {} ) _lf_trace_ctx.generation( name=f"{provider_name}_call", model=provider_name, input=None if cloud_provider else provider_prompt[:500], output=( None if cloud_provider else result.raw_response[:500] ), usage={"total": result.tokens} if result.tokens else None, metadata={ "cost_usd": result.cost_usd, "latency_ms": round(result.latency_ms, 1), "content_redacted": cloud_provider, "prompt_length": len(provider_prompt), "prompt_sha256": prompt_hash, "response_length": len(result.raw_response), "response_sha256": response_hash, "sanitization_receipt_id": ( receipt.get("receipt_id") if isinstance(receipt, dict) else None ), "generation_receipt_id": result.audit_metadata.get( "generation_receipt_id" ), }, ) _lf_trace_ctx.__exit__(None, None, None) except Exception: pass return result # Provider 回傳 success=False cb.record_failure() if provider_name in PAID_PROVIDER_ORDER: error_text = str(result.error or "") error_sha256 = hashlib.sha256( error_text.encode("utf-8") ).hexdigest() errors.append( f"{provider_name}: provider_failed(error_sha256={error_sha256})" ) logger.warning( "ai_router_paid_provider_failed", provider=provider_name, error_length=len(error_text), error_sha256=error_sha256, ) else: errors.append(f"{provider_name}: {result.error}") logger.warning( "ai_router_provider_failed", provider=provider_name, error=result.error, ) # 2026-04-27 A2: 記錄失敗的 provider，供下輪迭代計算 fallback metric _last_attempted_provider = provider_name except Exception as e: if provider_name in PAID_PROVIDER_ORDER: error_text = str(e) error_sha256 = hashlib.sha256( error_text.encode("utf-8") ).hexdigest() errors.append( f"{provider_name}: provider_exception(error_sha256={error_sha256})" ) logger.warning( "ai_router_paid_provider_exception", provider=provider_name, error_type=type(e).__name__, error_length=len(error_text), error_sha256=error_sha256, ) else: errors.append(f"{provider_name}: {e}") logger.warning( "ai_router_provider_exception", provider=provider_name, error=str(e), ) # 2026-04-05 Claude Code: v4.3 — Timeout 不計 CB 失敗 # NIM 偶爾 GPU 忙碌導致 27s，timeout 不代表 NIM 故障 # 只有明確連線錯誤（非 timeout）才累積 CB 失敗次數 if not isinstance(e, _httpx.TimeoutException | TimeoutError): cb.record_failure() # 2026-04-27 A2: 記錄失敗的 provider，供下輪迭代計算 fallback metric _last_attempted_provider = provider_name # 全部失敗 logger.error("ai_router_all_providers_failed", tried=provider_order, errors=errors) if _lf_trace_ctx: try: _lf_trace_ctx.__exit__(None, None, None) except Exception: pass # 2026-04-04 ogt: Phase 25 P0 — require_local 全部失敗時 Telegram 通知（隱私邊界） # 2026-04-15 ogt: 改用 ADR-075 TYPE-1 格式，禁止純文字 raw notification if require_local: try: from src.services.telegram_gateway import get_telegram_gateway tg = get_telegram_gateway() import asyncio as _asyncio tried_str = ", ".join(provider_order) formatted = ( "⚠️ TYPE-1 | AI Provider 不可用\n" "──────────────────────\n" f"├─ 已嘗試: {tried_str}\n" "└─ 原因: require_local=True，無可用本地 Provider\n" "\n" "需要人工介入" ) _asyncio.create_task( tg.send_alert_notification( formatted, parse_mode="HTML", product_id="awoooi", signal_family="shared_infrastructure", severity="P1", ) ) except Exception as _tg_e: logger.warning("diagnose_reject_telegram_failed", error=str(_tg_e)) return AIResult( raw_response="", success=False, provider="none", error="local_providers_unavailable", ) return AIResult( raw_response="", success=False, provider="none", error=f"All providers failed: {'; '.join(errors)}", ) # ============================================================================= # 單例管理 # ============================================================================= _router: AIRouter | None = None _registry: AIProviderRegistry | None = None _executor: AIRouterExecutor | None = None def _init_registry() -> AIProviderRegistry: """初始化 Provider Registry (首次呼叫時自動註冊所有 Provider)""" from src.services.ai_providers.claude import ClaudeProvider from src.services.ai_providers.gemini import GeminiProvider from src.services.ai_providers.ollama import ( OllamaGcpBProvider, # 2026-05-04 ADR-110 GCP-B OllamaLocalProvider, OllamaProvider, ) from src.services.ai_providers.openclaw_nemo import OpenClawNemoProvider registry = AIProviderRegistry() # GCP-A Primary（name="ollama"，OLLAMA_URL） ollama_gcp_a = OllamaProvider() registry.register(ollama_gcp_a) registry.register(GeminiProvider()) registry.register(ClaudeProvider()) registry.register(OpenClawNemoProvider()) # 2026-04-02 Claude Code: Phase 24 B3 — 加入 NemotronProvider (tool_calling 優先) from src.services.ai_providers.nemotron import NemotronProvider registry.register(NemotronProvider()) # 2026-05-06 Codex: 188 不再作為 Ollama provider。 # Local fallback 統一命名為 ollama_local，端點由 OLLAMA_FALLBACK_URL 直連 host111。 ollama_local = OllamaLocalProvider() registry.register(ollama_local) # 2026-05-04 ogt + Claude Sonnet 4.6: ADR-110 GCP 三層容災修復 # 根因：OllamaFailoverManager 回傳 "ollama_gcp_a"/"ollama_gcp_b"/"ollama_local" # 但 registry 無這些名稱 → not_registered → 整條 Ollama 鏈跳過 → 直接跳 Gemini # 修復： # "ollama_gcp_a" alias → 同 OllamaProvider（OLLAMA_URL = GCP-A） # "ollama_gcp_b" → 新 OllamaGcpBProvider（OLLAMA_SECONDARY_URL = GCP-B） # "ollama_local" → OllamaLocalProvider（OLLAMA_FALLBACK_URL = host111 direct） registry._providers["ollama_gcp_a"] = ollama_gcp_a registry.register(OllamaGcpBProvider()) registry._providers["ollama_local"] = ollama_local return registry def get_ai_router() -> AIRouter: """取得 AIRouter 單例 (路由決策)""" global _router if _router is None: _router = AIRouter() return _router def get_ai_registry() -> AIProviderRegistry: """取得 AIProviderRegistry 單例""" global _registry if _registry is None: _registry = _init_registry() return _registry def get_ai_executor() -> AIRouterExecutor: """取得 AIRouterExecutor 單例 (路由決策 + 執行)""" global _executor if _executor is None: _executor = AIRouterExecutor(get_ai_registry()) return _executor def reset_ai_router() -> None: """重置所有單例 (用於測試)""" global _router, _registry, _executor _router = None _registry = None _executor = None