diff --git a/apps/api/src/services/openclaw.py b/apps/api/src/services/openclaw.py index 7be8971f1..96687ece8 100644 --- a/apps/api/src/services/openclaw.py +++ b/apps/api/src/services/openclaw.py @@ -18,6 +18,7 @@ Features: - SignOz 失敗時優雅降級 (不阻塞主流程) """ +import hashlib import json import re import time @@ -27,6 +28,7 @@ import httpx import structlog from src.core.config import settings +from src.core.redis_client import get_redis from src.models.ai import ( AIRiskLevel, AIBlastRadius, @@ -714,6 +716,100 @@ class OpenClawService: return json.dumps(mock_response) + # ========================================================================= + # LLM Cache Layer (憲法要求: 嚴禁無快取裸奔) + # ========================================================================= + + def _generate_cache_key(self, prompt: str, context_hash: str = "") -> str: + """ + 生成 LLM 快取鍵 + + 使用 prompt 內容的 SHA256 作為快取鍵,確保相同問題不重複呼叫 LLM + """ + content = f"{prompt}:{context_hash}" + hash_digest = hashlib.sha256(content.encode()).hexdigest()[:16] + return f"llm_cache:{hash_digest}" + + async def _call_with_cache( + self, + prompt: str, + alert_context: dict | None = None, + signoz_metrics: GoldMetrics | None = None, + cache_ttl: int = 3600, # 1 hour default + ) -> tuple[str, str, bool, bool]: + """ + 帶快取的 LLM 呼叫包裝器 + + 憲法條款: 必須使用快取保護算力資源 + + Args: + prompt: LLM prompt + alert_context: 告警上下文 + signoz_metrics: SignOz 指標 + cache_ttl: 快取存活時間 (秒) + + Returns: + (response, provider, success, from_cache) + """ + # 生成快取鍵 (基於 prompt + alert_context hash) + context_hash = "" + if alert_context: + # 使用告警類型 + 目標資源作為上下文 hash + context_hash = f"{alert_context.get('alert_type', '')}:{alert_context.get('target_resource', '')}" + + cache_key = self._generate_cache_key(prompt, context_hash) + + # 1. 嘗試從快取讀取 + try: + redis_client = get_redis() + cached = await redis_client.get(cache_key) + if cached: + cached_data = json.loads(cached) + logger.info( + "llm_cache_hit", + cache_key=cache_key[:20], + provider=cached_data.get("provider", "cached"), + ) + return ( + cached_data["response"], + f"{cached_data['provider']}_cached", + True, + True, # from_cache + ) + except Exception as e: + logger.warning("llm_cache_read_failed", error=str(e)) + + # 2. Cache Miss - 呼叫 LLM + logger.info("llm_cache_miss", cache_key=cache_key[:20]) + response, provider, success = await self._call_with_fallback( + prompt, alert_context, signoz_metrics + ) + + # 3. 成功則寫入快取 + if success: + try: + redis_client = get_redis() + cache_data = { + "response": response, + "provider": provider, + "cached_at": datetime.now().isoformat(), + } + await redis_client.set( + cache_key, + json.dumps(cache_data, ensure_ascii=False), + ex=cache_ttl, + ) + logger.info( + "llm_cache_write", + cache_key=cache_key[:20], + provider=provider, + ttl=cache_ttl, + ) + except Exception as e: + logger.warning("llm_cache_write_failed", error=str(e)) + + return response, provider, success, False # from_cache=False + # ========================================================================= # Fallback Chain # ========================================================================= @@ -899,17 +995,21 @@ Trace URL: {signoz_trace_url} signoz_available=signoz_metrics is not None, ) - # 呼叫 LLM - raw_response, provider, success = await self._call_with_fallback( + # 呼叫 LLM (使用快取層保護算力) + raw_response, provider, success, from_cache = await self._call_with_cache( full_prompt, alert_context, signoz_metrics, + cache_ttl=1800, # 30 min for alert analysis ) if not success: logger.error("openclaw_all_providers_failed") return None, provider, raw_response, signoz_metrics, signoz_trace_url + if from_cache: + logger.info("openclaw_using_cached_response", provider=provider) + logger.info( "openclaw_llm_response_received", provider=provider, @@ -936,6 +1036,157 @@ Trace URL: {signoz_trace_url} return result, provider, raw_response, signoz_metrics, signoz_trace_url + # ========================================================================= + # Phase 6.4: LLM Proposal Generation + # ========================================================================= + + async def generate_incident_proposal( + self, + incident_id: str, + severity: str, + signals: list[dict], + affected_services: list[str], + ) -> tuple[dict | None, str, bool]: + """ + 為 Incident 生成 LLM-based 修復提案 + + Phase 6.4: 賦予大腦「生成解決方案」的思考能力 + + Args: + incident_id: Incident ID + severity: 嚴重度 (P0/P1/P2/P3) + signals: 關聯的告警訊號 + affected_services: 受影響服務 + + Returns: + (proposal_dict, provider, success) + proposal_dict 包含: + - action: 建議動作 + - description: 動作描述 + - kubectl_command: kubectl 指令 + - risk_level: 風險等級 + - reasoning: LLM 推理過程 + """ + # 建構 prompt + signal_summary = "\n".join([ + f"- {s.get('alert_name', 'unknown')}: {s.get('description', 'N/A')}" + for s in signals[:10] # 最多 10 筆 + ]) + + target = affected_services[0] if affected_services else "unknown-service" + + # 擷取 SignOz 指標 + signoz_metrics, signoz_trace_url = await self.get_signoz_context( + service_name=target, + namespace="awoooi-prod", + ) + + signoz_context = "" + if signoz_metrics: + signoz_context = f""" +## 📊 SignOz Real-time Metrics +{signoz_metrics.to_summary()} +""" + + proposal_prompt = f"""{OPENCLAW_SYSTEM_PROMPT} + +{signoz_context} + +## 🚨 Incident Context +- **Incident ID**: {incident_id} +- **Severity**: {severity} +- **Affected Services**: {', '.join(affected_services)} +- **Signal Count**: {len(signals)} + +## 📋 Alert Signals +{signal_summary} + +## 🎯 Your Task +Based on the above incident and signals, generate a remediation proposal. +You MUST respond with ONLY valid JSON following the schema above. +Focus on: +1. Root cause analysis based on signals and SignOz data +2. Specific kubectl command to remediate +3. Risk assessment for the proposed action +4. Preventive recommendations +""" + + logger.info( + "proposal_generation_start", + incident_id=incident_id, + severity=severity, + signal_count=len(signals), + signoz_available=signoz_metrics is not None, + ) + + # 使用快取呼叫 LLM + alert_context = { + "incident_id": incident_id, + "alert_type": signals[0].get("alert_name", "incident") if signals else "incident", + "target_resource": target, + "severity": severity, + } + + raw_response, provider, success, from_cache = await self._call_with_cache( + proposal_prompt, + alert_context, + signoz_metrics, + cache_ttl=3600, # 1 hour for proposals + ) + + if not success: + logger.error( + "proposal_generation_failed", + incident_id=incident_id, + provider=provider, + ) + return None, provider, False + + # 解析 LLM 結果 + result = self._parse_analysis_result(raw_response) + + if result: + logger.info( + "proposal_generation_complete", + incident_id=incident_id, + action_title=result.action_title, + risk_level=result.risk_level, + provider=provider, + from_cache=from_cache, + ) + + # 轉換為 proposal dict + proposal_dict = { + "action": result.action_title, + "description": result.description, + "kubectl_command": result.kubectl_command, + "target_resource": result.target_resource, + "namespace": result.namespace, + "risk_level": result.risk_level, + "reasoning": result.reasoning, + "confidence": result.confidence, + "primary_responsibility": result.primary_responsibility, + "optimization_suggestions": [ + { + "type": s.type, + "description": s.description, + "kubectl_or_config": s.kubectl_or_config, + } + for s in result.optimization_suggestions + ], + "signoz_correlation": result.signoz_correlation, + "from_cache": from_cache, + "provider": provider, + } + return proposal_dict, provider, True + + logger.warning( + "proposal_parse_failed", + incident_id=incident_id, + raw_response=raw_response[:300], + ) + return None, provider, False + # ========================================================================= # Shadow Mode Auto-Tuning # ========================================================================= diff --git a/apps/api/src/services/proposal_service.py b/apps/api/src/services/proposal_service.py index 8af5313a6..ebdafb3c2 100644 --- a/apps/api/src/services/proposal_service.py +++ b/apps/api/src/services/proposal_service.py @@ -19,7 +19,6 @@ Decision Proposal Service - Phase 6.4 決策輸出層 """ from datetime import datetime, timezone -from typing import Any from uuid import UUID import structlog @@ -43,6 +42,7 @@ from src.models.incident import ( ) from src.services.approval_db import get_approval_service from src.services.trust_engine import trust_engine, normalize_action_pattern, RiskLevel +from src.services.openclaw import get_openclaw logger = structlog.get_logger(__name__) @@ -95,14 +95,20 @@ class ProposalService: 決策提案服務 - Phase 6.4 職責: - 1. 分析 Incident 生成修復建議 + 1. 分析 Incident 生成修復建議 (LLM-based) 2. 評估風險等級 3. 建立 ApprovalRequest (向下相容前端) 4. 更新 Incident 狀態與關聯 + + Phase 6.4 升級: + - 整合 OpenClaw LLM 生成智能提案 + - 使用 _call_with_cache 保護算力資源 + - Fallback 到模板方案確保可用性 """ def __init__(self) -> None: self._approval_service = get_approval_service() + self._openclaw = get_openclaw() # ========================================================================= # 核心方法: 從 Incident 生成 Proposal @@ -147,12 +153,51 @@ class ProposalService: signal_count=len(incident.signals), ) - # 2. 分析 signals 決定修復動作 - action_type, action, description = self._determine_action(incident) - - # 3. 評估風險等級 - base_risk = SEVERITY_TO_RISK.get(incident.severity, ApprovalRiskLevel.MEDIUM) + # 2. 呼叫 OpenClaw LLM 生成提案 (Phase 6.4 核心) target = incident.affected_services[0] if incident.affected_services else "unknown" + signals_dict = [s.model_dump() for s in incident.signals] + + llm_proposal, provider, llm_success = await self._openclaw.generate_incident_proposal( + incident_id=incident_id, + severity=incident.severity.value, + signals=signals_dict, + affected_services=incident.affected_services, + ) + + # 使用 LLM 結果或 fallback 到模板 + if llm_success and llm_proposal: + action = llm_proposal["action"] + description = f"{llm_proposal['description']}\n\n**AI 推理**: {llm_proposal['reasoning']}" + action_type = llm_proposal.get("primary_responsibility", "default").lower() + + # LLM 提供的 risk_level 轉換 + llm_risk = llm_proposal.get("risk_level", "medium") + risk_map = { + "low": ApprovalRiskLevel.LOW, + "medium": ApprovalRiskLevel.MEDIUM, + "critical": ApprovalRiskLevel.CRITICAL, + } + base_risk = risk_map.get(llm_risk, ApprovalRiskLevel.MEDIUM) + + logger.info( + "llm_proposal_generated", + incident_id=incident_id, + provider=provider, + action=action[:50], + risk_level=llm_risk, + confidence=llm_proposal.get("confidence", 0), + ) + else: + # Fallback 到模板方案 + logger.warning( + "llm_proposal_fallback_to_template", + incident_id=incident_id, + provider=provider, + ) + action_type, action, description = self._determine_action(incident) + base_risk = SEVERITY_TO_RISK.get(incident.severity, ApprovalRiskLevel.MEDIUM) + + # 3. 評估風險等級 (TrustEngine 調整) action_pattern = normalize_action_pattern(action_type, {"resource": target}) risk_adjustment = trust_engine.evaluate_adjusted_risk( @@ -173,6 +218,24 @@ class ProposalService: blast_radius = self._build_blast_radius(incident) dry_run_checks = self._build_dry_run_checks(incident) + # 建立 metadata (含 LLM 資訊) + metadata = { + "incident_id": incident_id, + "severity": incident.severity.value, + "signal_count": len(incident.signals), + "affected_services": incident.affected_services, + "trust_adjustment": risk_adjustment.to_dict(), + } + + # 加入 LLM 相關資訊 (Phase 6.4) + if llm_success and llm_proposal: + metadata["llm_provider"] = llm_proposal.get("provider", "unknown") + metadata["llm_confidence"] = llm_proposal.get("confidence", 0) + metadata["llm_from_cache"] = llm_proposal.get("from_cache", False) + metadata["kubectl_command"] = llm_proposal.get("kubectl_command", "") + metadata["signoz_correlation"] = llm_proposal.get("signoz_correlation", "") + metadata["optimization_suggestions"] = llm_proposal.get("optimization_suggestions", []) + approval_create = ApprovalRequestCreate( action=action, description=description, @@ -180,13 +243,7 @@ class ProposalService: blast_radius=blast_radius, dry_run_checks=dry_run_checks, requested_by="OpenClaw AI", - metadata={ - "incident_id": incident_id, - "severity": incident.severity.value, - "signal_count": len(incident.signals), - "affected_services": incident.affected_services, - "trust_adjustment": risk_adjustment.to_dict(), - }, + metadata=metadata, ) approval = await self._approval_service.create_approval(approval_create)