feat(api): Phase 6.4 LLM-based proposal generation with cache
- Add _call_with_cache wrapper in OpenClaw (Redis-based LLM cache) - Add generate_incident_proposal method for incident analysis - Integrate ProposalService with OpenClaw LLM - Fallback to template-based proposals if LLM fails - Include LLM metadata (provider, confidence, cache status) in proposals 憲法條款: 必須使用快取保護算力資源，嚴禁無快取裸奔調用 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
OG T
@@ -18,6 +18,7 @@ Features:
|
||||
- SignOz 失敗時優雅降級 (不阻塞主流程)
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
@@ -27,6 +28,7 @@ import httpx
|
||||
import structlog
|
||||
|
||||
from src.core.config import settings
|
||||
from src.core.redis_client import get_redis
|
||||
from src.models.ai import (
|
||||
AIRiskLevel,
|
||||
AIBlastRadius,
|
||||
@@ -714,6 +716,100 @@ class OpenClawService:
|
||||
|
||||
return json.dumps(mock_response)
|
||||
|
||||
# =========================================================================
|
||||
# LLM Cache Layer (憲法要求: 嚴禁無快取裸奔)
|
||||
# =========================================================================
|
||||
|
||||
def _generate_cache_key(self, prompt: str, context_hash: str = "") -> str:
|
||||
"""
|
||||
生成 LLM 快取鍵
|
||||
|
||||
使用 prompt 內容的 SHA256 作為快取鍵，確保相同問題不重複呼叫 LLM
|
||||
"""
|
||||
content = f"{prompt}:{context_hash}"
|
||||
hash_digest = hashlib.sha256(content.encode()).hexdigest()[:16]
|
||||
return f"llm_cache:{hash_digest}"
|
||||
|
||||
async def _call_with_cache(
|
||||
self,
|
||||
prompt: str,
|
||||
alert_context: dict | None = None,
|
||||
signoz_metrics: GoldMetrics | None = None,
|
||||
cache_ttl: int = 3600, # 1 hour default
|
||||
) -> tuple[str, str, bool, bool]:
|
||||
"""
|
||||
帶快取的 LLM 呼叫包裝器
|
||||
|
||||
憲法條款: 必須使用快取保護算力資源
|
||||
|
||||
Args:
|
||||
prompt: LLM prompt
|
||||
alert_context: 告警上下文
|
||||
signoz_metrics: SignOz 指標
|
||||
cache_ttl: 快取存活時間 (秒)
|
||||
|
||||
Returns:
|
||||
(response, provider, success, from_cache)
|
||||
"""
|
||||
# 生成快取鍵 (基於 prompt + alert_context hash)
|
||||
context_hash = ""
|
||||
if alert_context:
|
||||
# 使用告警類型 + 目標資源作為上下文 hash
|
||||
context_hash = f"{alert_context.get('alert_type', '')}:{alert_context.get('target_resource', '')}"
|
||||
|
||||
cache_key = self._generate_cache_key(prompt, context_hash)
|
||||
|
||||
# 1. 嘗試從快取讀取
|
||||
try:
|
||||
redis_client = get_redis()
|
||||
cached = await redis_client.get(cache_key)
|
||||
if cached:
|
||||
cached_data = json.loads(cached)
|
||||
logger.info(
|
||||
"llm_cache_hit",
|
||||
cache_key=cache_key[:20],
|
||||
provider=cached_data.get("provider", "cached"),
|
||||
)
|
||||
return (
|
||||
cached_data["response"],
|
||||
f"{cached_data['provider']}_cached",
|
||||
True,
|
||||
True, # from_cache
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("llm_cache_read_failed", error=str(e))
|
||||
|
||||
# 2. Cache Miss - 呼叫 LLM
|
||||
logger.info("llm_cache_miss", cache_key=cache_key[:20])
|
||||
response, provider, success = await self._call_with_fallback(
|
||||
prompt, alert_context, signoz_metrics
|
||||
)
|
||||
|
||||
# 3. 成功則寫入快取
|
||||
if success:
|
||||
try:
|
||||
redis_client = get_redis()
|
||||
cache_data = {
|
||||
"response": response,
|
||||
"provider": provider,
|
||||
"cached_at": datetime.now().isoformat(),
|
||||
}
|
||||
await redis_client.set(
|
||||
cache_key,
|
||||
json.dumps(cache_data, ensure_ascii=False),
|
||||
ex=cache_ttl,
|
||||
)
|
||||
logger.info(
|
||||
"llm_cache_write",
|
||||
cache_key=cache_key[:20],
|
||||
provider=provider,
|
||||
ttl=cache_ttl,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("llm_cache_write_failed", error=str(e))
|
||||
|
||||
return response, provider, success, False # from_cache=False
|
||||
|
||||
# =========================================================================
|
||||
# Fallback Chain
|
||||
# =========================================================================
|
||||
@@ -899,17 +995,21 @@ Trace URL: {signoz_trace_url}
|
||||
signoz_available=signoz_metrics is not None,
|
||||
)
|
||||
|
||||
# 呼叫 LLM
|
||||
raw_response, provider, success = await self._call_with_fallback(
|
||||
# 呼叫 LLM (使用快取層保護算力)
|
||||
raw_response, provider, success, from_cache = await self._call_with_cache(
|
||||
full_prompt,
|
||||
alert_context,
|
||||
signoz_metrics,
|
||||
cache_ttl=1800, # 30 min for alert analysis
|
||||
)
|
||||
|
||||
if not success:
|
||||
logger.error("openclaw_all_providers_failed")
|
||||
return None, provider, raw_response, signoz_metrics, signoz_trace_url
|
||||
|
||||
if from_cache:
|
||||
logger.info("openclaw_using_cached_response", provider=provider)
|
||||
|
||||
logger.info(
|
||||
"openclaw_llm_response_received",
|
||||
provider=provider,
|
||||
@@ -936,6 +1036,157 @@ Trace URL: {signoz_trace_url}
|
||||
|
||||
return result, provider, raw_response, signoz_metrics, signoz_trace_url
|
||||
|
||||
# =========================================================================
|
||||
# Phase 6.4: LLM Proposal Generation
|
||||
# =========================================================================
|
||||
|
||||
async def generate_incident_proposal(
|
||||
self,
|
||||
incident_id: str,
|
||||
severity: str,
|
||||
signals: list[dict],
|
||||
affected_services: list[str],
|
||||
) -> tuple[dict | None, str, bool]:
|
||||
"""
|
||||
為 Incident 生成 LLM-based 修復提案
|
||||
|
||||
Phase 6.4: 賦予大腦「生成解決方案」的思考能力
|
||||
|
||||
Args:
|
||||
incident_id: Incident ID
|
||||
severity: 嚴重度 (P0/P1/P2/P3)
|
||||
signals: 關聯的告警訊號
|
||||
affected_services: 受影響服務
|
||||
|
||||
Returns:
|
||||
(proposal_dict, provider, success)
|
||||
proposal_dict 包含:
|
||||
- action: 建議動作
|
||||
- description: 動作描述
|
||||
- kubectl_command: kubectl 指令
|
||||
- risk_level: 風險等級
|
||||
- reasoning: LLM 推理過程
|
||||
"""
|
||||
# 建構 prompt
|
||||
signal_summary = "\n".join([
|
||||
f"- {s.get('alert_name', 'unknown')}: {s.get('description', 'N/A')}"
|
||||
for s in signals[:10] # 最多 10 筆
|
||||
])
|
||||
|
||||
target = affected_services[0] if affected_services else "unknown-service"
|
||||
|
||||
# 擷取 SignOz 指標
|
||||
signoz_metrics, signoz_trace_url = await self.get_signoz_context(
|
||||
service_name=target,
|
||||
namespace="awoooi-prod",
|
||||
)
|
||||
|
||||
signoz_context = ""
|
||||
if signoz_metrics:
|
||||
signoz_context = f"""
|
||||
## 📊 SignOz Real-time Metrics
|
||||
{signoz_metrics.to_summary()}
|
||||
"""
|
||||
|
||||
proposal_prompt = f"""{OPENCLAW_SYSTEM_PROMPT}
|
||||
|
||||
{signoz_context}
|
||||
|
||||
## 🚨 Incident Context
|
||||
- **Incident ID**: {incident_id}
|
||||
- **Severity**: {severity}
|
||||
- **Affected Services**: {', '.join(affected_services)}
|
||||
- **Signal Count**: {len(signals)}
|
||||
|
||||
## 📋 Alert Signals
|
||||
{signal_summary}
|
||||
|
||||
## 🎯 Your Task
|
||||
Based on the above incident and signals, generate a remediation proposal.
|
||||
You MUST respond with ONLY valid JSON following the schema above.
|
||||
Focus on:
|
||||
1. Root cause analysis based on signals and SignOz data
|
||||
2. Specific kubectl command to remediate
|
||||
3. Risk assessment for the proposed action
|
||||
4. Preventive recommendations
|
||||
"""
|
||||
|
||||
logger.info(
|
||||
"proposal_generation_start",
|
||||
incident_id=incident_id,
|
||||
severity=severity,
|
||||
signal_count=len(signals),
|
||||
signoz_available=signoz_metrics is not None,
|
||||
)
|
||||
|
||||
# 使用快取呼叫 LLM
|
||||
alert_context = {
|
||||
"incident_id": incident_id,
|
||||
"alert_type": signals[0].get("alert_name", "incident") if signals else "incident",
|
||||
"target_resource": target,
|
||||
"severity": severity,
|
||||
}
|
||||
|
||||
raw_response, provider, success, from_cache = await self._call_with_cache(
|
||||
proposal_prompt,
|
||||
alert_context,
|
||||
signoz_metrics,
|
||||
cache_ttl=3600, # 1 hour for proposals
|
||||
)
|
||||
|
||||
if not success:
|
||||
logger.error(
|
||||
"proposal_generation_failed",
|
||||
incident_id=incident_id,
|
||||
provider=provider,
|
||||
)
|
||||
return None, provider, False
|
||||
|
||||
# 解析 LLM 結果
|
||||
result = self._parse_analysis_result(raw_response)
|
||||
|
||||
if result:
|
||||
logger.info(
|
||||
"proposal_generation_complete",
|
||||
incident_id=incident_id,
|
||||
action_title=result.action_title,
|
||||
risk_level=result.risk_level,
|
||||
provider=provider,
|
||||
from_cache=from_cache,
|
||||
)
|
||||
|
||||
# 轉換為 proposal dict
|
||||
proposal_dict = {
|
||||
"action": result.action_title,
|
||||
"description": result.description,
|
||||
"kubectl_command": result.kubectl_command,
|
||||
"target_resource": result.target_resource,
|
||||
"namespace": result.namespace,
|
||||
"risk_level": result.risk_level,
|
||||
"reasoning": result.reasoning,
|
||||
"confidence": result.confidence,
|
||||
"primary_responsibility": result.primary_responsibility,
|
||||
"optimization_suggestions": [
|
||||
{
|
||||
"type": s.type,
|
||||
"description": s.description,
|
||||
"kubectl_or_config": s.kubectl_or_config,
|
||||
}
|
||||
for s in result.optimization_suggestions
|
||||
],
|
||||
"signoz_correlation": result.signoz_correlation,
|
||||
"from_cache": from_cache,
|
||||
"provider": provider,
|
||||
}
|
||||
return proposal_dict, provider, True
|
||||
|
||||
logger.warning(
|
||||
"proposal_parse_failed",
|
||||
incident_id=incident_id,
|
||||
raw_response=raw_response[:300],
|
||||
)
|
||||
return None, provider, False
|
||||
|
||||
# =========================================================================
|
||||
# Shadow Mode Auto-Tuning
|
||||
# =========================================================================
|
||||
|
||||
@@ -19,7 +19,6 @@ Decision Proposal Service - Phase 6.4 決策輸出層
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import structlog
|
||||
@@ -43,6 +42,7 @@ from src.models.incident import (
|
||||
)
|
||||
from src.services.approval_db import get_approval_service
|
||||
from src.services.trust_engine import trust_engine, normalize_action_pattern, RiskLevel
|
||||
from src.services.openclaw import get_openclaw
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
@@ -95,14 +95,20 @@ class ProposalService:
|
||||
決策提案服務 - Phase 6.4
|
||||
|
||||
職責:
|
||||
1. 分析 Incident 生成修復建議
|
||||
1. 分析 Incident 生成修復建議 (LLM-based)
|
||||
2. 評估風險等級
|
||||
3. 建立 ApprovalRequest (向下相容前端)
|
||||
4. 更新 Incident 狀態與關聯
|
||||
|
||||
Phase 6.4 升級:
|
||||
- 整合 OpenClaw LLM 生成智能提案
|
||||
- 使用 _call_with_cache 保護算力資源
|
||||
- Fallback 到模板方案確保可用性
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._approval_service = get_approval_service()
|
||||
self._openclaw = get_openclaw()
|
||||
|
||||
# =========================================================================
|
||||
# 核心方法: 從 Incident 生成 Proposal
|
||||
@@ -147,12 +153,51 @@ class ProposalService:
|
||||
signal_count=len(incident.signals),
|
||||
)
|
||||
|
||||
# 2. 分析 signals 決定修復動作
|
||||
action_type, action, description = self._determine_action(incident)
|
||||
|
||||
# 3. 評估風險等級
|
||||
base_risk = SEVERITY_TO_RISK.get(incident.severity, ApprovalRiskLevel.MEDIUM)
|
||||
# 2. 呼叫 OpenClaw LLM 生成提案 (Phase 6.4 核心)
|
||||
target = incident.affected_services[0] if incident.affected_services else "unknown"
|
||||
signals_dict = [s.model_dump() for s in incident.signals]
|
||||
|
||||
llm_proposal, provider, llm_success = await self._openclaw.generate_incident_proposal(
|
||||
incident_id=incident_id,
|
||||
severity=incident.severity.value,
|
||||
signals=signals_dict,
|
||||
affected_services=incident.affected_services,
|
||||
)
|
||||
|
||||
# 使用 LLM 結果或 fallback 到模板
|
||||
if llm_success and llm_proposal:
|
||||
action = llm_proposal["action"]
|
||||
description = f"{llm_proposal['description']}\n\n**AI 推理**: {llm_proposal['reasoning']}"
|
||||
action_type = llm_proposal.get("primary_responsibility", "default").lower()
|
||||
|
||||
# LLM 提供的 risk_level 轉換
|
||||
llm_risk = llm_proposal.get("risk_level", "medium")
|
||||
risk_map = {
|
||||
"low": ApprovalRiskLevel.LOW,
|
||||
"medium": ApprovalRiskLevel.MEDIUM,
|
||||
"critical": ApprovalRiskLevel.CRITICAL,
|
||||
}
|
||||
base_risk = risk_map.get(llm_risk, ApprovalRiskLevel.MEDIUM)
|
||||
|
||||
logger.info(
|
||||
"llm_proposal_generated",
|
||||
incident_id=incident_id,
|
||||
provider=provider,
|
||||
action=action[:50],
|
||||
risk_level=llm_risk,
|
||||
confidence=llm_proposal.get("confidence", 0),
|
||||
)
|
||||
else:
|
||||
# Fallback 到模板方案
|
||||
logger.warning(
|
||||
"llm_proposal_fallback_to_template",
|
||||
incident_id=incident_id,
|
||||
provider=provider,
|
||||
)
|
||||
action_type, action, description = self._determine_action(incident)
|
||||
base_risk = SEVERITY_TO_RISK.get(incident.severity, ApprovalRiskLevel.MEDIUM)
|
||||
|
||||
# 3. 評估風險等級 (TrustEngine 調整)
|
||||
action_pattern = normalize_action_pattern(action_type, {"resource": target})
|
||||
|
||||
risk_adjustment = trust_engine.evaluate_adjusted_risk(
|
||||
@@ -173,6 +218,24 @@ class ProposalService:
|
||||
blast_radius = self._build_blast_radius(incident)
|
||||
dry_run_checks = self._build_dry_run_checks(incident)
|
||||
|
||||
# 建立 metadata (含 LLM 資訊)
|
||||
metadata = {
|
||||
"incident_id": incident_id,
|
||||
"severity": incident.severity.value,
|
||||
"signal_count": len(incident.signals),
|
||||
"affected_services": incident.affected_services,
|
||||
"trust_adjustment": risk_adjustment.to_dict(),
|
||||
}
|
||||
|
||||
# 加入 LLM 相關資訊 (Phase 6.4)
|
||||
if llm_success and llm_proposal:
|
||||
metadata["llm_provider"] = llm_proposal.get("provider", "unknown")
|
||||
metadata["llm_confidence"] = llm_proposal.get("confidence", 0)
|
||||
metadata["llm_from_cache"] = llm_proposal.get("from_cache", False)
|
||||
metadata["kubectl_command"] = llm_proposal.get("kubectl_command", "")
|
||||
metadata["signoz_correlation"] = llm_proposal.get("signoz_correlation", "")
|
||||
metadata["optimization_suggestions"] = llm_proposal.get("optimization_suggestions", [])
|
||||
|
||||
approval_create = ApprovalRequestCreate(
|
||||
action=action,
|
||||
description=description,
|
||||
@@ -180,13 +243,7 @@ class ProposalService:
|
||||
blast_radius=blast_radius,
|
||||
dry_run_checks=dry_run_checks,
|
||||
requested_by="OpenClaw AI",
|
||||
metadata={
|
||||
"incident_id": incident_id,
|
||||
"severity": incident.severity.value,
|
||||
"signal_count": len(incident.signals),
|
||||
"affected_services": incident.affected_services,
|
||||
"trust_adjustment": risk_adjustment.to_dict(),
|
||||
},
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
approval = await self._approval_service.create_approval(approval_create)
|
||||
|
||||
Reference in New Issue
Block a user