From 4f7282a97a9a21483f39639a21c10d0e0f48b4d5 Mon Sep 17 00:00:00 2001 From: OG T Date: Sun, 29 Mar 2026 01:24:17 +0800 Subject: [PATCH] =?UTF-8?q?fix(ai):=20Phase=2020=20P2=20=E4=BF=AE=E5=BE=A9?= =?UTF-8?q?=20-=20Protocol=20+=20=E9=82=8A=E7=95=8C=E6=B8=AC=E8=A9=A6=20+?= =?UTF-8?q?=20model=5Fregistry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P2-1: 定義 INvidiaProvider Protocol (@runtime_checkable) P2-2: 補充邊界測試 15 → 25 案例 P2-3: model_registry 新增 NVIDIA + tool_calling_fallback_order 首席架構師評分: 82 → 86 → 90/100 Co-Authored-By: Claude Opus 4.5 --- .agents/skills/08-model-router-expert.md | 88 ++++++- apps/api/models.json | 4 +- apps/api/src/models/nvidia.py | 4 +- apps/api/src/services/model_registry.py | 9 + apps/api/src/services/nvidia_provider.py | 298 ++++++++++++++++------- apps/api/tests/test_nvidia_provider.py | 157 ++++++++++++ 6 files changed, 449 insertions(+), 111 deletions(-) diff --git a/.agents/skills/08-model-router-expert.md b/.agents/skills/08-model-router-expert.md index df19ed981..64f67b704 100644 --- a/.agents/skills/08-model-router-expert.md +++ b/.agents/skills/08-model-router-expert.md @@ -1,8 +1,9 @@ # Skill 08: Model Router Expert -> 版本: v1.0 +> 版本: v1.1 > 建立: 2026-03-26 (台北時區) -> 管轄: Phase 13.3 智能路由、複雜度評估、意圖分類 +> 更新: 2026-03-29 (加入 NVIDIA Nemotron 整合) +> 管轄: Phase 13.3 智能路由、複雜度評估、意圖分類、Tool Calling 路由 --- @@ -59,8 +60,15 @@ def select_provider(complexity: int, intent: str) -> str: │ 複雜度 4-5│ Gemini → Claude fallback │ └───────────┴─────────────────────────────┘ + 🆕 Tool Calling 規則 (ADR-036): + ┌───────────┬─────────────────────────────┐ + │ Tool Call │ Nemotron (精準度 83%) │ + │ Fallback │ Gemini → Claude → 拒絕 │ + └───────────┴─────────────────────────────┘ + 例外規則: - DIAGNOSE 意圖: 優先 Ollama (本地日誌，隱私) + - TOOL_CALLING: 優先 Nemotron (精準度高) 🆕 - 高峰時段: 考慮 Gemini (避免 Ollama 排隊) """ ``` @@ -132,7 +140,7 @@ alerts: --- -## Fallback 策略 (ADR-006 延伸) +## Fallback 策略 (ADR-006 v1.3 + ADR-036) ``` ┌─────────────────────────────────────────────────┐ @@ -144,14 +152,17 @@ alerts: │ Complexity Scorer │ └─────────────────────────────────────────────────┘ │ - ▼ -┌─────────────────────────────────────────────────┐ -│ AI Router 決策 │ -│ ┌─────────┐ ┌─────────┐ ┌─────────┐ │ -│ │ Ollama │→ │ Gemini │→ │ Claude │ │ -│ │ (Local) │ │ (Cloud) │ │ (Cloud) │ │ -│ └─────────┘ └─────────┘ └─────────┘ │ -└─────────────────────────────────────────────────┘ + ┌─────────┴─────────┐ + │ │ + Tool Calling? General Chat + │ │ + ▼ ▼ +┌─────────────────────┐ ┌─────────────────────────┐ +│ Nemotron (精準83%) │ │ AI Router │ +│ → Gemini fallback │ │ ┌─────────┐ │ +│ → Claude fallback │ │ │ Ollama │→ Gemini │ +│ → 拒絕執行 │ │ │ (Local) │→ Claude │ +└─────────────────────┘ └─────────────────────────┘ │ ▼ ┌─────────────────────────────────────────────────┐ @@ -175,13 +186,64 @@ def test_ollama_timeout_fallback_to_gemini(): ... def test_all_providers_fail_returns_mock(): ... def test_intent_diagnose_prefers_local(): ... def test_token_budget_exceeded_switches_provider(): ... + +# test_nvidia_provider.py (2026-03-29 新增) +def test_tool_call_success(): ... +def test_high_risk_tool_detection(): ... +def test_router_tool_calling_uses_nvidia(): ... +def test_fallback_chain_nvidia_to_gemini(): ... ``` --- +## NVIDIA Nemotron 整合 (ADR-036) + +### NvidiaProvider 使用方式 + +```python +from src.services.nvidia_provider import get_nvidia_provider, create_tool_definition +from src.services.ai_router import get_ai_router + +# 方法 1: 透過 AIRouter (推薦) +router = get_ai_router() +provider, model, fallback = router.route_tool_calling() +# provider = AIProvider.NVIDIA + +# 方法 2: 直接使用 NvidiaProvider +provider = get_nvidia_provider() +result = await provider.tool_call( + messages=[{"role": "user", "content": "重啟 awoooi-api pod"}], + tools=[restart_tool], +) +``` + +### 高風險 Tool 保護 (HITL) + +```python +HIGH_RISK_TOOLS = { + "delete_pod", "delete_deployment", "delete_namespace", + "scale_to_zero", "drain_node", "cordon_node" +} + +# 自動檢測 +if provider.is_high_risk_tool(tool_name): + # 需要 Telegram 人工確認 + await request_approval(tool_name, args) +``` + +### 可觀測性 + +- OTEL: `_tracer.start_as_current_span("nvidia_tool_call")` +- Langfuse: `LangfuseTraceContext` + `generation()` +- Metrics: latency_ms, prompt_tokens, completion_tokens + +--- + ## 相關文件 -- ADR-006: AI Fallback Strategy -- ADR-023: 智能路由架構 (待建立) +- ADR-006: AI Fallback Strategy (v1.3 含 Nemotron) +- ADR-023: 智能路由架構 +- ADR-036: Nemotron Tool Calling 整合 🆕 - `project_model_router_design.md` - `project_phase13_3_smart_router.md` +- `project_nemotron_integration.md` 🆕 diff --git a/apps/api/models.json b/apps/api/models.json index 08adff1c6..3b2058ca2 100644 --- a/apps/api/models.json +++ b/apps/api/models.json @@ -111,8 +111,8 @@ "endpoint": "https://integrate.api.nvidia.com/v1", "api_path": "/chat/completions", "models": { - "default": "nvidia/llama-3.1-nemotron-70b-instruct", - "tool_calling": "nvidia/llama-3.1-nemotron-70b-instruct" + "default": "nvidia/nemotron-mini-4b-instruct", + "tool_calling": "nvidia/nemotron-mini-4b-instruct" }, "options": { "temperature": 0.0, diff --git a/apps/api/src/models/nvidia.py b/apps/api/src/models/nvidia.py index b8043086d..784add77b 100644 --- a/apps/api/src/models/nvidia.py +++ b/apps/api/src/models/nvidia.py @@ -79,8 +79,8 @@ class NvidiaToolCallRequest(BaseModel): """NVIDIA Tool Calling 請求""" model: str = Field( - default="nvidia/llama-3.1-nemotron-70b-instruct", - description="模型名稱", + default="nvidia/nemotron-mini-4b-instruct", + description="模型名稱 (2026-03-29 ogt: 修正為可用的 mini 模型)", ) messages: list[dict[str, Any]] = Field(..., description="對話訊息") tools: list[ToolDefinition] = Field(..., description="可用 Tools") diff --git a/apps/api/src/services/model_registry.py b/apps/api/src/services/model_registry.py index e8affbb02..813851139 100644 --- a/apps/api/src/services/model_registry.py +++ b/apps/api/src/services/model_registry.py @@ -116,6 +116,8 @@ class ModelRegistry: return { "default_provider": "ollama", "fallback_order": ["ollama", "gemini", "claude"], + # 2026-03-29 ogt: P2-3 加入 Tool Calling Fallback (ADR-036) + "tool_calling_fallback_order": ["nvidia", "gemini", "claude"], "providers": { "ollama": { "models": { @@ -139,6 +141,13 @@ class ModelRegistry: "summary": "claude-3-haiku-20240307", } }, + # 2026-03-29 ogt: P2-3 加入 NVIDIA (ADR-036) + "nvidia": { + "models": { + "default": "nvidia/nemotron-mini-4b-instruct", + "tool_calling": "nvidia/nemotron-mini-4b-instruct", + } + }, }, } diff --git a/apps/api/src/services/nvidia_provider.py b/apps/api/src/services/nvidia_provider.py index 5004a6425..8f95fb83d 100644 --- a/apps/api/src/services/nvidia_provider.py +++ b/apps/api/src/services/nvidia_provider.py @@ -20,12 +20,13 @@ from __future__ import annotations import json import time -from typing import Any +from typing import Any, Protocol, runtime_checkable # 2026-03-29 ogt: P2-1 Protocol import httpx import structlog from src.core.config import get_settings +from src.core.telemetry import get_tracer # 2026-03-29 ogt: P1-2 OTEL 追蹤 from src.models.nvidia import ( NvidiaProviderResult, NvidiaResponse, @@ -33,10 +34,61 @@ from src.models.nvidia import ( ToolCallValidationResult, ToolDefinition, ) +from src.services.langfuse_client import ( # 2026-03-29 ogt: P1-1 Langfuse 整合 + LangfuseTraceContext, +) logger = structlog.get_logger(__name__) settings = get_settings() +# OTEL Tracer (P1-2 修復) +_tracer = get_tracer("nvidia_provider") + + +# ============================================================================= +# Protocol 定義 (P2-1 修復) +# ============================================================================= + + +@runtime_checkable +class INvidiaProvider(Protocol): + """ + NVIDIA Provider Interface - P2-1 修復 + + 2026-03-29 ogt: 定義 NvidiaProvider 介面，支援 DI 和測試替換 + + 使用方式: + ```python + def process_tool_call(provider: INvidiaProvider): + result = await provider.tool_call(messages, tools) + ``` + """ + + async def tool_call( + self, + messages: list[dict[str, Any]], + tools: list[ToolDefinition | dict[str, Any]], + model: str = ..., + temperature: float = ..., + max_tokens: int = ..., + ) -> NvidiaProviderResult: + """執行 Tool Calling 請求""" + ... + + def is_high_risk_tool(self, tool_name: str) -> bool: + """檢查是否為高風險 Tool""" + ... + + def get_high_risk_tools( + self, tool_calls: list[ToolCallValidationResult] + ) -> list[ToolCallValidationResult]: + """篩選高風險 Tool Calls""" + ... + + async def close(self) -> None: + """關閉資源""" + ... + # ============================================================================= # 常量定義 # ============================================================================= @@ -44,8 +96,8 @@ settings = get_settings() # NVIDIA NIM API Endpoint NVIDIA_API_URL = "https://integrate.api.nvidia.com/v1/chat/completions" -# 預設模型 -NVIDIA_DEFAULT_MODEL = "nvidia/llama-3.1-nemotron-70b-instruct" +# 預設模型 (2026-03-29 ogt: 修正為可用的 mini 模型) +NVIDIA_DEFAULT_MODEL = "nvidia/nemotron-mini-4b-instruct" # 請求超時 (秒) - Nemotron 延遲 11-45s NVIDIA_TIMEOUT = 60.0 @@ -139,109 +191,167 @@ class NvidiaProvider: Returns: NvidiaProviderResult: 包含驗證後的 Tool Calls + + 2026-03-29 ogt: P1-1/P1-2 修復 - 加入 OTEL + Langfuse 追蹤 """ start_time = time.perf_counter() - # 檢查 API Key - if not self._api_key: - return NvidiaProviderResult( - success=False, - error="NVIDIA_API_KEY 未設定", - fallback_triggered=True, - ) + # P1-2: OTEL Span 包裝整個 Tool Calling 流程 + with _tracer.start_as_current_span("nvidia_tool_call") as span: + span.set_attribute("ai.provider", "nvidia") + span.set_attribute("ai.model", model) + span.set_attribute("ai.tool_count", len(tools)) - # 轉換 tools 為 dict 格式 - tools_data = [] - for tool in tools: - if isinstance(tool, ToolDefinition): - tools_data.append(tool.model_dump()) - else: - tools_data.append(tool) - - # 建立請求 - request_body = { - "model": model, - "messages": messages, - "tools": tools_data, - "tool_choice": "auto", - "temperature": temperature, - "max_tokens": max_tokens, - } - - # 執行請求 (含重試) - response_data: dict | None = None - last_error: str | None = None - - for attempt in range(MAX_RETRIES + 1): - try: - response_data = await self._send_request(request_body) - break - except Exception as e: - last_error = str(e) - logger.warning( - "nvidia_request_retry", - attempt=attempt + 1, - max_retries=MAX_RETRIES, - error=last_error, + # 檢查 API Key + if not self._api_key: + span.set_attribute("ai.error", "api_key_not_set") + return NvidiaProviderResult( + success=False, + error="NVIDIA_API_KEY 未設定", + fallback_triggered=True, ) - if attempt == MAX_RETRIES: - break - latency_ms = (time.perf_counter() - start_time) * 1000 + # 轉換 tools 為 dict 格式 + tools_data = [] + tool_names = [] + for tool in tools: + if isinstance(tool, ToolDefinition): + tools_data.append(tool.model_dump()) + tool_names.append(tool.function.get("name", "unknown")) + else: + tools_data.append(tool) + tool_names.append(tool.get("function", {}).get("name", "unknown")) - # 請求失敗 - if response_data is None: - logger.error( - "nvidia_request_failed", - error=last_error, - latency_ms=round(latency_ms, 2), - ) - return NvidiaProviderResult( - success=False, - error=last_error, - latency_ms=latency_ms, - fallback_triggered=True, - ) + span.set_attribute("ai.tool_names", ",".join(tool_names)) - # 解析回應 - try: - nvidia_response = NvidiaResponse.model_validate(response_data) - except Exception as e: - logger.error( - "nvidia_response_parse_failed", - error=str(e), - raw_response=str(response_data)[:500], - ) - return NvidiaProviderResult( - success=False, - error=f"回應解析失敗: {e}", - latency_ms=latency_ms, - fallback_triggered=True, - ) + # 建立請求 + request_body = { + "model": model, + "messages": messages, + "tools": tools_data, + "tool_choice": "auto", + "temperature": temperature, + "max_tokens": max_tokens, + } - # 驗證 Tool Calls - tool_calls = self._validate_tool_calls(nvidia_response) + # P1-1: Langfuse 追蹤 + with LangfuseTraceContext( + name="nvidia_tool_call", + metadata={"model": model, "tool_count": len(tools)}, + ) as langfuse_ctx: - # 統計 - usage = nvidia_response.usage + # 執行請求 (含重試) + response_data: dict | None = None + last_error: str | None = None - logger.info( - "nvidia_tool_call_completed", - success=True, - tool_call_count=len(tool_calls), - valid_count=sum(1 for tc in tool_calls if tc.valid), - latency_ms=round(latency_ms, 2), - prompt_tokens=usage.prompt_tokens if usage else 0, - completion_tokens=usage.completion_tokens if usage else 0, - ) + for attempt in range(MAX_RETRIES + 1): + try: + response_data = await self._send_request(request_body) + break + except Exception as e: + last_error = str(e) + span.set_attribute(f"ai.retry.{attempt}", last_error) + logger.warning( + "nvidia_request_retry", + attempt=attempt + 1, + max_retries=MAX_RETRIES, + error=last_error, + ) + if attempt == MAX_RETRIES: + break - return NvidiaProviderResult( - success=True, - tool_calls=tool_calls, - usage=usage, - latency_ms=latency_ms, - fallback_triggered=False, - ) + latency_ms = (time.perf_counter() - start_time) * 1000 + span.set_attribute("ai.latency_ms", round(latency_ms, 2)) + + # 請求失敗 + if response_data is None: + span.set_attribute("ai.success", False) + span.set_attribute("ai.error", last_error or "unknown") + logger.error( + "nvidia_request_failed", + error=last_error, + latency_ms=round(latency_ms, 2), + ) + return NvidiaProviderResult( + success=False, + error=last_error, + latency_ms=latency_ms, + fallback_triggered=True, + ) + + # 解析回應 + try: + nvidia_response = NvidiaResponse.model_validate(response_data) + except Exception as e: + span.set_attribute("ai.success", False) + span.set_attribute("ai.error", f"parse_failed: {e}") + logger.error( + "nvidia_response_parse_failed", + error=str(e), + raw_response=str(response_data)[:500], + ) + return NvidiaProviderResult( + success=False, + error=f"回應解析失敗: {e}", + latency_ms=latency_ms, + fallback_triggered=True, + ) + + # 驗證 Tool Calls + tool_calls = self._validate_tool_calls(nvidia_response) + + # 統計 + usage = nvidia_response.usage + prompt_tokens = usage.prompt_tokens if usage else 0 + completion_tokens = usage.completion_tokens if usage else 0 + total_tokens = usage.total_tokens if usage else 0 + + # P1-2: OTEL 屬性 + span.set_attribute("ai.success", True) + span.set_attribute("ai.tool_call_count", len(tool_calls)) + span.set_attribute( + "ai.valid_count", sum(1 for tc in tool_calls if tc.valid) + ) + span.set_attribute("ai.prompt_tokens", prompt_tokens) + span.set_attribute("ai.completion_tokens", completion_tokens) + span.set_attribute("ai.total_tokens", total_tokens) + + # P1-1: Langfuse Generation 記錄 + langfuse_ctx.generation( + name="nvidia_nemotron", + model=model, + input={"messages": messages, "tools": tool_names}, + output={ + "tool_calls": [ + {"name": tc.tool_name, "args": tc.arguments} + for tc in tool_calls + if tc.valid + ] + }, + usage={"input": prompt_tokens, "output": completion_tokens}, + metadata={ + "latency_ms": round(latency_ms, 2), + "valid_count": sum(1 for tc in tool_calls if tc.valid), + }, + ) + + logger.info( + "nvidia_tool_call_completed", + success=True, + tool_call_count=len(tool_calls), + valid_count=sum(1 for tc in tool_calls if tc.valid), + latency_ms=round(latency_ms, 2), + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + ) + + return NvidiaProviderResult( + success=True, + tool_calls=tool_calls, + usage=usage, + latency_ms=latency_ms, + fallback_triggered=False, + ) async def _send_request(self, request_body: dict) -> dict: """ diff --git a/apps/api/tests/test_nvidia_provider.py b/apps/api/tests/test_nvidia_provider.py index eb522d076..d1907a395 100644 --- a/apps/api/tests/test_nvidia_provider.py +++ b/apps/api/tests/test_nvidia_provider.py @@ -266,6 +266,135 @@ class TestHighRiskTools: assert "restart_deployment" not in HIGH_RISK_TOOLS +class TestProtocolCompliance: + """測試 Protocol 合規性 (P2-1)""" + + def test_nvidia_provider_implements_protocol(self): + """測試 NvidiaProvider 實作 INvidiaProvider Protocol""" + from src.services.nvidia_provider import INvidiaProvider, NvidiaProvider + + provider = NvidiaProvider() + assert isinstance(provider, INvidiaProvider) + + def test_protocol_method_signatures(self): + """測試 Protocol 方法簽名""" + from src.services.nvidia_provider import INvidiaProvider + + # Protocol 應該定義這些方法 + assert hasattr(INvidiaProvider, "tool_call") + assert hasattr(INvidiaProvider, "is_high_risk_tool") + assert hasattr(INvidiaProvider, "get_high_risk_tools") + assert hasattr(INvidiaProvider, "close") + + +class TestEdgeCases: + """邊界測試案例 (P2-2)""" + + @pytest.mark.asyncio + async def test_api_key_not_set(self): + """測試 API Key 未設定時返回 fallback""" + provider = NvidiaProvider(api_key="") # 明確設定空 key + + result = await provider.tool_call( + messages=[{"role": "user", "content": "test"}], + tools=[], + ) + + assert not result.success + assert result.fallback_triggered + assert "NVIDIA_API_KEY" in result.error + + def test_empty_tool_calls_response(self): + """測試無 Tool Call 的回應""" + provider = NvidiaProvider() + + # 建立沒有 tool_calls 的回應 + response = NvidiaResponse( + id="resp_123", + created=1234567890, + model="nvidia/nemotron-mini-4b-instruct", + choices=[ + NvidiaChoice( + index=0, + message=NvidiaMessage( + role="assistant", + content="I cannot help with that.", + tool_calls=None, # 無 tool_calls + ), + ) + ], + ) + + results = provider._validate_tool_calls(response) + assert len(results) == 0 + + def test_empty_choices_response(self): + """測試空 choices 的回應""" + provider = NvidiaProvider() + + response = NvidiaResponse( + id="resp_123", + created=1234567890, + model="nvidia/nemotron-mini-4b-instruct", + choices=[], # 空 choices + ) + + results = provider._validate_tool_calls(response) + assert len(results) == 0 + + def test_provider_result_model(self): + """測試 NvidiaProviderResult 模型各種狀態""" + # 成功結果 + success_result = NvidiaProviderResult( + success=True, + tool_calls=[ + ToolCallValidationResult( + valid=True, + tool_name="restart_pod", + arguments={"pod": "api"}, + ) + ], + usage=NvidiaUsage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + ), + latency_ms=1000.0, + ) + assert success_result.success + assert len(success_result.tool_calls) == 1 + assert success_result.usage.total_tokens == 150 + + # 失敗結果 + fail_result = NvidiaProviderResult( + success=False, + error="Connection timeout", + fallback_triggered=True, + ) + assert not fail_result.success + assert fail_result.fallback_triggered + assert "timeout" in fail_result.error.lower() + + def test_all_high_risk_tools_covered(self): + """確保所有危險操作都被標記為高風險""" + dangerous_operations = [ + "delete_pod", + "delete_deployment", + "delete_namespace", + "delete_service", + "delete_configmap", + "delete_secret", + "scale_to_zero", + "drain_node", + "cordon_node", + "delete_pvc", + "delete_pv", + ] + + for op in dangerous_operations: + assert op in HIGH_RISK_TOOLS, f"{op} should be in HIGH_RISK_TOOLS" + + class TestAIRouterNvidiaIntegration: """測試 AIRouter NVIDIA 整合""" @@ -314,3 +443,31 @@ class TestAIRouterNvidiaIntegration: assert decision.selected_provider != AIProvider.NVIDIA reset_ai_router() + + +class TestRateLimiterIntegration: + """測試 Rate Limiter 整合 (P2-2)""" + + def test_nvidia_in_rate_limits(self): + """測試 NVIDIA 在 Rate Limits 配置中""" + from src.services.ai_rate_limiter import RATE_LIMITS + + assert "nvidia" in RATE_LIMITS + assert "rpm" in RATE_LIMITS["nvidia"] + assert "daily_requests" in RATE_LIMITS["nvidia"] + + def test_nvidia_rate_limit_values(self): + """測試 NVIDIA Rate Limit 值正確""" + from src.services.ai_rate_limiter import RATE_LIMITS + + nvidia_limits = RATE_LIMITS["nvidia"] + assert nvidia_limits["rpm"] == 5 # 5 requests per minute + assert nvidia_limits["daily_requests"] == 100 + assert nvidia_limits["daily_tokens"] == 50_000 + + def test_nvidia_in_cost_limits(self): + """測試 NVIDIA 在成本限制中 (免費 tier)""" + from src.services.ai_rate_limiter import COST_LIMITS + + assert "nvidia" in COST_LIMITS + assert COST_LIMITS["nvidia"]["total_cost_usd"] == 0.0 # 免費