diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index c73255a0e..2ad9887e3 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -159,6 +159,9 @@ async def _push_to_telegram_background( signoz_rps: float = 0.0, signoz_rps_trend: str = "stable", signoz_error_rate: float = 0.0, + # 2026-03-29 ogt: AI Token/Cost 追蹤 + ai_tokens: int = 0, + ai_cost: float = 0.0, signoz_p99_latency: float = 0.0, signoz_latency_trend: str = "stable", signoz_trace_url: str = "", @@ -206,6 +209,9 @@ async def _push_to_telegram_background( signoz_latency_trend=signoz_latency_trend, signoz_trace_url=signoz_trace_url, auto_tuning_command=auto_tuning_command, + # 2026-03-29 ogt: AI Token/Cost 追蹤 + ai_tokens=ai_tokens, + ai_cost=ai_cost, ) logger.info( @@ -216,6 +222,8 @@ async def _push_to_telegram_background( primary_responsibility=primary_responsibility, confidence=confidence, signoz_integrated=signoz_rps > 0 or signoz_error_rate > 0, + ai_tokens=ai_tokens, + ai_cost=f"${ai_cost:.6f}", ) except TelegramGatewayError as e: @@ -887,8 +895,9 @@ async def receive_alert( } # 呼叫 OpenClaw LLM 分析 (v7.0 含 SignOz 整合) + # 2026-03-29 ogt: 加入 Token/Cost 追蹤 openclaw = get_openclaw() - analysis_result, ai_provider, raw_response, signoz_metrics, signoz_trace_url = await openclaw.analyze_alert(alert_context) + analysis_result, ai_provider, raw_response, signoz_metrics, signoz_trace_url, ai_tokens, ai_cost = await openclaw.analyze_alert(alert_context) if analysis_result: # LLM 分析成功 @@ -1017,6 +1026,9 @@ async def receive_alert( signoz_latency_trend=signoz_latency_trend, signoz_trace_url=signoz_trace_url, auto_tuning_command=auto_tuning_cmd, + # 2026-03-29 ogt: AI Token/Cost 追蹤 + ai_tokens=ai_tokens, + ai_cost=ai_cost, ) return AlertResponse( @@ -1251,8 +1263,9 @@ async def alertmanager_webhook( "labels": alert.labels, } + # 2026-03-29 ogt: 加入 Token/Cost 追蹤 openclaw = get_openclaw() - analysis_result, ai_provider, raw_response, signoz_metrics, signoz_trace_url = await openclaw.analyze_alert(alert_context) + analysis_result, ai_provider, raw_response, signoz_metrics, signoz_trace_url, ai_tokens, ai_cost = await openclaw.analyze_alert(alert_context) if analysis_result: # analysis_result 是 OpenClawDecision Pydantic 模型 @@ -1335,6 +1348,9 @@ async def alertmanager_webhook( signoz_p99_latency=signoz_metrics.p99_latency_ms if signoz_metrics else 0, signoz_latency_trend=signoz_metrics.latency_trend if signoz_metrics else "stable", signoz_trace_url=signoz_trace_url or "", + # 2026-03-29 ogt: AI Token/Cost 追蹤 + ai_tokens=ai_tokens, + ai_cost=ai_cost, ) return AlertResponse( diff --git a/apps/api/src/services/openclaw.py b/apps/api/src/services/openclaw.py index bda9b3be8..a022383a1 100644 --- a/apps/api/src/services/openclaw.py +++ b/apps/api/src/services/openclaw.py @@ -320,12 +320,21 @@ class OpenClawService: ) return str(e), False - async def _call_gemini(self, prompt: str) -> tuple[str, bool]: + async def _call_gemini(self, prompt: str) -> tuple[str, bool, int, float]: """ 呼叫 Google Gemini (支援 JSON Mode) + + Returns: + tuple: (response_text, success, total_tokens, cost_usd) + - response_text: LLM 回應文本 + - success: 是否成功 + - total_tokens: 使用的 Token 總數 + - cost_usd: 預估成本 (USD) + + 2026-03-29 ogt: 加入 Token/Cost 追蹤 """ if not settings.GEMINI_API_KEY: - return "GEMINI_API_KEY not configured", False + return "GEMINI_API_KEY not configured", False, 0, 0.0 try: client = await self._get_client() @@ -350,12 +359,29 @@ class OpenClawService: data = response.json() text = data["candidates"][0]["content"]["parts"][0]["text"] - logger.info("gemini_response_received", response_length=len(text)) - return text, True + # 2026-03-29 ogt: 擷取 Token 使用量 + usage_metadata = data.get("usageMetadata", {}) + prompt_tokens = usage_metadata.get("promptTokenCount", 0) + completion_tokens = usage_metadata.get("candidatesTokenCount", 0) + total_tokens = usage_metadata.get("totalTokenCount", prompt_tokens + completion_tokens) + + # Gemini 1.5 Flash 定價 (per 1M tokens) + # Input: $0.075 / 1M, Output: $0.30 / 1M + cost_usd = (prompt_tokens * 0.000000075) + (completion_tokens * 0.0000003) + + logger.info( + "gemini_response_received", + response_length=len(text), + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + cost_usd=f"${cost_usd:.6f}", + ) + return text, True, total_tokens, cost_usd except Exception as e: logger.warning("gemini_call_failed", error=str(e)) - return str(e), False + return str(e), False, 0, 0.0 async def _call_claude(self, prompt: str) -> tuple[str, bool]: """ @@ -672,7 +698,7 @@ class OpenClawService: alert_context: dict | None = None, signoz_metrics: GoldMetrics | None = None, cache_ttl: int = 3600, # 1 hour default - ) -> tuple[str, str, bool, bool]: + ) -> tuple[str, str, bool, bool, int, float]: """ 帶快取的 LLM 呼叫包裝器 @@ -685,7 +711,9 @@ class OpenClawService: cache_ttl: 快取存活時間 (秒) Returns: - (response, provider, success, from_cache) + (response, provider, success, from_cache, total_tokens, cost_usd) + + 2026-03-29 ogt: 加入 Token/Cost 追蹤 """ # 生成快取鍵 (基於 prompt + alert_context hash) context_hash = "" @@ -711,13 +739,15 @@ class OpenClawService: f"{cached_data['provider']}_cached", True, True, # from_cache + 0, # tokens (cache hit, no new tokens) + 0.0, # cost (cache hit, no cost) ) except Exception as e: logger.warning("llm_cache_read_failed", error=str(e)) # 2. Cache Miss - 呼叫 LLM logger.info("llm_cache_miss", cache_key=cache_key[:20]) - response, provider, success = await self._call_with_fallback( + response, provider, success, total_tokens, cost_usd = await self._call_with_fallback( prompt, alert_context, signoz_metrics ) @@ -744,7 +774,7 @@ class OpenClawService: except Exception as e: logger.warning("llm_cache_write_failed", error=str(e)) - return response, provider, success, False # from_cache=False + return response, provider, success, False, total_tokens, cost_usd # from_cache=False # ========================================================================= # Public LLM Interface (ILLMProvider Protocol) @@ -773,19 +803,23 @@ class OpenClawService: prompt: str, alert_context: dict | None = None, signoz_metrics: GoldMetrics | None = None, - ) -> tuple[str, str, bool]: + ) -> tuple[str, str, bool, int, float]: """ 依 AI_FALLBACK_ORDER 順序呼叫 AI 若 MOCK_MODE=True,直接回傳模擬結果。 若所有 Provider 失敗,fallback 到 Mock。 + Returns: + tuple: (response, provider, success, total_tokens, cost_usd) + Phase 15.1: 整合 Langfuse LLMOps 追蹤 + 2026-03-29 ogt: 加入 Token/Cost 追蹤 """ # Mock Mode: 開發測試用 if settings.MOCK_MODE: logger.info("mock_mode_enabled", using="mock_llm") - return self._generate_mock_response(alert_context or {}, signoz_metrics), "mock", True + return self._generate_mock_response(alert_context or {}, signoz_metrics), "mock", True, 0, 0.0 # Phase 15.1 + 15.3: Langfuse 追蹤整合 + SignOz Deep Linking with langfuse_trace( @@ -833,10 +867,14 @@ class OpenClawService: start_time = time.time() model_name = self._get_model_name(provider) + # 2026-03-29 ogt: Gemini 回傳 4 值 (含 token/cost),其他 Provider 補 0 + total_tokens = 0 + cost_usd = 0.0 + if provider == "ollama": response, success = await self._call_ollama(prompt) elif provider == "gemini": - response, success = await self._call_gemini(prompt) + response, success, total_tokens, cost_usd = await self._call_gemini(prompt) elif provider == "claude": response, success = await self._call_claude(prompt) else: @@ -855,21 +893,29 @@ class OpenClawService: "success": success, "latency_ms": round(latency_ms, 2), "provider": provider, + "total_tokens": total_tokens, + "cost_usd": cost_usd, }, ) if success: - logger.info("ai_provider_success", provider=provider, latency_ms=latency_ms) + logger.info( + "ai_provider_success", + provider=provider, + latency_ms=latency_ms, + total_tokens=total_tokens, + cost_usd=f"${cost_usd:.6f}", + ) # Langfuse: 記錄成功評分 trace.score(name="provider_success", value=1.0, comment=f"Success via {provider}") - return response, provider, True + return response, provider, True, total_tokens, cost_usd logger.warning("ai_provider_failed_fallback", provider=provider, latency_ms=latency_ms) # 所有 Provider 失敗時,fallback 到 Mock (優雅降級) logger.warning("all_providers_failed_using_mock", fallback="mock_llm") trace.score(name="provider_success", value=0.0, comment="All providers failed, using mock") - return self._generate_mock_response(alert_context or {}, signoz_metrics), "mock_fallback", True + return self._generate_mock_response(alert_context or {}, signoz_metrics), "mock_fallback", True, 0, 0.0 def _get_model_name(self, provider: str) -> str: """取得 provider 對應的模型名稱 (從 ModelRegistry)""" @@ -977,7 +1023,7 @@ class OpenClawService: async def analyze_alert( self, alert_context: dict, - ) -> tuple[LLMAnalysisResult | None, str, str, GoldMetrics | None, str]: + ) -> tuple[LLMAnalysisResult | None, str, str, GoldMetrics | None, str, int, float]: """ 分析告警並產生 RCA 結果 (含 SignOz 整合) @@ -985,7 +1031,9 @@ class OpenClawService: alert_context: 告警上下文 (alert_type, severity, target_resource, etc.) Returns: - (analysis_result, ai_provider, raw_response, signoz_metrics, signoz_trace_url) + (analysis_result, ai_provider, raw_response, signoz_metrics, signoz_trace_url, total_tokens, cost_usd) + + 2026-03-29 ogt: 加入 Token/Cost 追蹤 """ # Step 0: 擷取 SignOz 上下文 service_name = alert_context.get("target_resource", "unknown") @@ -1018,7 +1066,7 @@ Trace URL: {signoz_trace_url} ) # 呼叫 LLM (使用快取層保護算力) - raw_response, provider, success, from_cache = await self._call_with_cache( + raw_response, provider, success, from_cache, total_tokens, cost_usd = await self._call_with_cache( full_prompt, alert_context, signoz_metrics, @@ -1027,7 +1075,7 @@ Trace URL: {signoz_trace_url} if not success: logger.error("openclaw_all_providers_failed") - return None, provider, raw_response, signoz_metrics, signoz_trace_url + return None, provider, raw_response, signoz_metrics, signoz_trace_url, 0, 0.0 if from_cache: logger.info("openclaw_using_cached_response", provider=provider) @@ -1056,7 +1104,7 @@ Trace URL: {signoz_trace_url} raw_response=raw_response[:300], ) - return result, provider, raw_response, signoz_metrics, signoz_trace_url + return result, provider, raw_response, signoz_metrics, signoz_trace_url, total_tokens, cost_usd # ========================================================================= # Phase 6.4: LLM Proposal Generation diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 67ed0c4e2..9a961b941 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -139,6 +139,9 @@ class TelegramMessage: signoz_metrics: SignOzMetricsBlock | None = None signoz_trace_url: str = "" # 動態時間參數 URL auto_tuning_command: str = "" # kubectl 調優指令 + # 2026-03-29 ogt: AI Token/Cost 追蹤 + ai_tokens: int = 0 # LLM Token 使用量 + ai_cost: float = 0.0 # LLM 成本 (USD) def format(self) -> str: """ @@ -190,6 +193,11 @@ class TelegramMessage: safe_action = html.escape(self.suggested_action[:35]) safe_downtime = html.escape(self.estimated_downtime) + # 2026-03-29 ogt: AI Token/Cost 顯示 + ai_cost_display = "" + if self.ai_tokens > 0 or self.ai_cost > 0: + ai_cost_display = f"💰 Tokens: {self.ai_tokens:,} / ${self.ai_cost:.4f}\n" + # 組裝訊息 message = ( f"═══════════════════════════\n" @@ -201,6 +209,7 @@ class TelegramMessage: f"🤖 AI 仲裁判定\n" f"👥 責任: {resp_display}\n" f"📊 信心: {conf_emoji} {confidence_pct}%\n" + f"{ai_cost_display}" f"💡 原因: {safe_root_cause}\n" f"{signoz_block}" f"━━━━━━━━━━━━━━━━━━━\n" @@ -425,6 +434,9 @@ class TelegramGateway: signoz_latency_trend: str = "stable", signoz_trace_url: str = "", auto_tuning_command: str = "", + # 2026-03-29 ogt: AI Token/Cost 追蹤 + ai_tokens: int = 0, + ai_cost: float = 0.0, ) -> dict: """ 推送待簽核卡片到 Telegram (v7.0 含 SignOz 整合) @@ -463,7 +475,7 @@ class TelegramGateway: trace_url=signoz_trace_url, ) - # 建立訊息結構 (含 AI 仲裁 + SignOz) + # 建立訊息結構 (含 AI 仲裁 + SignOz + Token/Cost) message = TelegramMessage( status_emoji=emoji, risk_level=risk_level.upper(), @@ -478,6 +490,9 @@ class TelegramGateway: signoz_metrics=signoz_metrics, signoz_trace_url=signoz_trace_url, auto_tuning_command=auto_tuning_command, + # 2026-03-29 ogt: AI Token/Cost 追蹤 + ai_tokens=ai_tokens, + ai_cost=ai_cost, ) # 格式化訊息 diff --git a/k8s/awoooi-prod/04-configmap.yaml b/k8s/awoooi-prod/04-configmap.yaml index 39edac66a..d828d061a 100644 --- a/k8s/awoooi-prod/04-configmap.yaml +++ b/k8s/awoooi-prod/04-configmap.yaml @@ -34,9 +34,10 @@ data: CORS_ORIGINS: '["https://awoooi.wooo.work","http://localhost:3000","http://localhost:3001"]' # AI 配置 (JSON array 格式 for pydantic-settings) - # 2026-03-28: 已切回 Ollama 優先 (成本最佳化) - # 備援順序: Ollama($0) → Gemini(~$0.001) → Claude(~$0.008) - AI_FALLBACK_ORDER: '["ollama","gemini","claude"]' + # 2026-03-29 ogt: 切換 Gemini 優先 (確認 AI 仲裁運作正常) + # 備援順序: Gemini(~$0.001) → Ollama($0) → Claude(~$0.008) + # Rate Limiter: RPM=10, 每日 500 次, Token 100K + AI_FALLBACK_ORDER: '["gemini","ollama","claude"]' AI_CACHE_TTL: "3600" # 快取 TTL (秒)