feat(phase6-9): Complete modular architecture and Agent Teams
Phase 6.4 - Modular Architecture: - Add lewooogo-brain adapters for LLM providers - Add lewooogo-data dual memory (Redis + PostgreSQL) - Implement consensus engine for multi-agent decisions - Add incident memory service for historical context Phase 9 - Agent Teams (Claude Agent SDK): - Add base agent class with Claude Sonnet 4 integration - Implement action planner, blast radius, and security agents - Add agent API endpoints and proposal workflow - Integrate ADR-009 OpenClaw Agent Teams architecture DevOps & CI/CD: - Add GitHub Actions CI/CD workflows (ci.yaml, cd.yaml) - Add pre-commit hooks and secrets baseline - Add docker-compose for local development - Update Kubernetes network policies Frontend Improvements: - Add auto-healing error boundary component - Update i18n messages for agent features - Enhance dual-state incident card with execution feedback Documentation: - Add 7 ADRs covering MCP, design system, architecture decisions - Update ARCHITECTURE_MEMORY.md with modular design - Add GLOBAL_RULES.md and SOUL.md for project identity Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
OG T
@@ -41,6 +41,13 @@ from .graph_rag import (
|
||||
FullAnalysisResult,
|
||||
create_mock_topology,
|
||||
)
|
||||
from .consensus_engine import (
|
||||
ConsensusEngine,
|
||||
get_consensus_engine,
|
||||
ConsensusResult,
|
||||
AgentOpinion,
|
||||
AgentType,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Dry-Run
|
||||
@@ -82,4 +89,10 @@ __all__ = [
|
||||
"RootCauseResult",
|
||||
"FullAnalysisResult",
|
||||
"create_mock_topology",
|
||||
# Consensus Engine (Phase 9.4)
|
||||
"ConsensusEngine",
|
||||
"get_consensus_engine",
|
||||
"ConsensusResult",
|
||||
"AgentOpinion",
|
||||
"AgentType",
|
||||
]
|
||||
|
||||
@@ -19,7 +19,6 @@ from uuid import UUID
|
||||
|
||||
import structlog
|
||||
from sqlalchemy import select, update, and_, or_
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.db.base import get_db_context
|
||||
from src.db.models import ApprovalRecord, TimelineEvent
|
||||
@@ -572,6 +571,78 @@ class ApprovalDBService:
|
||||
success=success,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Phase 6.4h: Proposals API 支援方法
|
||||
# =========================================================================
|
||||
|
||||
async def get_approval_by_id(self, approval_id: UUID) -> ApprovalRequest | None:
|
||||
"""
|
||||
根據 ID 取得單一授權請求 (Phase 6.4h)
|
||||
|
||||
Args:
|
||||
approval_id: 授權請求 UUID
|
||||
|
||||
Returns:
|
||||
ApprovalRequest if found, None otherwise
|
||||
"""
|
||||
async with get_db_context() as db:
|
||||
result = await db.execute(
|
||||
select(ApprovalRecord).where(ApprovalRecord.id == str(approval_id))
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
|
||||
if record is None:
|
||||
return None
|
||||
|
||||
return approval_record_to_request(record)
|
||||
|
||||
async def get_all_approvals(
|
||||
self,
|
||||
status: ApprovalStatus | None = None,
|
||||
incident_id: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[ApprovalRequest]:
|
||||
"""
|
||||
取得所有授權請求 (Phase 6.4h)
|
||||
|
||||
Args:
|
||||
status: 狀態篩選 (可選)
|
||||
incident_id: Incident ID 篩選 (可選)
|
||||
limit: 每頁數量
|
||||
offset: 偏移量
|
||||
|
||||
Returns:
|
||||
ApprovalRequest 清單
|
||||
"""
|
||||
async with get_db_context() as db:
|
||||
query = select(ApprovalRecord)
|
||||
|
||||
# 狀態篩選
|
||||
if status is not None:
|
||||
query = query.where(ApprovalRecord.status == status)
|
||||
|
||||
# Incident ID 篩選 (從 extra_metadata JSON 欄位)
|
||||
# NOTE: 這是基於 JSON 欄位查詢，效能可能受影響
|
||||
# 若有效能問題，考慮新增 incident_id 欄位到 ApprovalRecord
|
||||
|
||||
query = query.order_by(ApprovalRecord.created_at.desc())
|
||||
query = query.offset(offset).limit(limit)
|
||||
|
||||
result = await db.execute(query)
|
||||
records = result.scalars().all()
|
||||
|
||||
approvals = [approval_record_to_request(r) for r in records]
|
||||
|
||||
# 若有 incident_id 篩選，在應用層過濾
|
||||
if incident_id:
|
||||
approvals = [
|
||||
a for a in approvals
|
||||
if a.metadata and a.metadata.get("incident_id") == incident_id
|
||||
]
|
||||
|
||||
return approvals
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Timeline Event Service
|
||||
|
||||
@@ -25,11 +25,7 @@ import structlog
|
||||
|
||||
from src.core.config import settings
|
||||
from src.models.ai import (
|
||||
AIRiskLevel,
|
||||
AIBlastRadius,
|
||||
AIDataImpact,
|
||||
ClawBotDecision,
|
||||
SuggestedAction,
|
||||
)
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
637
apps/api/src/services/consensus_engine.py
Normal file
637
apps/api/src/services/consensus_engine.py
Normal file
@@ -0,0 +1,637 @@
|
||||
"""
|
||||
Consensus Engine - Phase 9.4 多專家共識引擎
|
||||
============================================
|
||||
|
||||
實作 Agent Teams 的共識機制，整合多個專家 Agent 的意見。
|
||||
|
||||
Features:
|
||||
- 收集多個專家 Agent 的意見 (SRE, Security, Cost, Performance)
|
||||
- 計算加權共識分數
|
||||
- 產生最終整合決策
|
||||
- 支援 Redis Working Memory 儲存
|
||||
|
||||
統帥鐵律:
|
||||
- 所有專家意見必須被記錄 (CISO 可稽核性要求)
|
||||
- 信心度低於 0.6 的意見權重降低
|
||||
- 最終決策必須包含所有專家的推理過程
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import structlog
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from src.core.redis_client import get_redis
|
||||
from src.models.incident import Incident
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Agent Types (專家類型)
|
||||
# =============================================================================
|
||||
|
||||
class AgentType(str, Enum):
|
||||
"""專家 Agent 類型"""
|
||||
SRE = "sre" # Site Reliability Engineer - 系統穩定性
|
||||
SECURITY = "security" # Security Expert - 資安風險
|
||||
COST = "cost" # FinOps Expert - 成本效益
|
||||
PERFORMANCE = "performance" # Performance Expert - 效能優化
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Agent Opinion (專家意見)
|
||||
# =============================================================================
|
||||
|
||||
class AgentOpinion(BaseModel):
|
||||
"""
|
||||
單一專家的意見
|
||||
|
||||
每個專家會針對同一個 Incident 提出自己的分析與建議
|
||||
"""
|
||||
|
||||
agent_type: AgentType
|
||||
action: str
|
||||
reasoning: str
|
||||
confidence: float = Field(ge=0.0, le=1.0, description="信心度 0-1")
|
||||
risk_assessment: str
|
||||
kubectl_command: str | None = None
|
||||
priority: int = Field(default=5, ge=1, le=10, description="優先度 1-10, 10 最高")
|
||||
estimated_impact: dict[str, Any] = Field(default_factory=dict)
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
|
||||
model_config = {"use_enum_values": False}
|
||||
|
||||
@field_validator("confidence", mode="before")
|
||||
@classmethod
|
||||
def clamp_confidence(cls, v: float) -> float:
|
||||
"""Clamp confidence to 0-1 range"""
|
||||
return min(max(v, 0.0), 1.0)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"agent_type": self.agent_type.value,
|
||||
"action": self.action,
|
||||
"reasoning": self.reasoning,
|
||||
"confidence": self.confidence,
|
||||
"risk_assessment": self.risk_assessment,
|
||||
"kubectl_command": self.kubectl_command,
|
||||
"priority": self.priority,
|
||||
"estimated_impact": self.estimated_impact,
|
||||
"created_at": self.created_at.isoformat(),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "AgentOpinion":
|
||||
return cls(
|
||||
agent_type=AgentType(data["agent_type"]),
|
||||
action=data["action"],
|
||||
reasoning=data["reasoning"],
|
||||
confidence=data["confidence"],
|
||||
risk_assessment=data["risk_assessment"],
|
||||
kubectl_command=data.get("kubectl_command"),
|
||||
priority=data.get("priority", 5),
|
||||
estimated_impact=data.get("estimated_impact", {}),
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Consensus Result (共識結果)
|
||||
# =============================================================================
|
||||
|
||||
class ConsensusResult(BaseModel):
|
||||
"""
|
||||
共識引擎的最終決策結果
|
||||
|
||||
包含:
|
||||
- 所有專家意見 (CISO 可稽核性)
|
||||
- 加權共識分數
|
||||
- 最終推薦行動
|
||||
- 決策理由
|
||||
"""
|
||||
|
||||
consensus_id: str
|
||||
incident_id: str
|
||||
opinions: list[AgentOpinion]
|
||||
consensus_score: float = Field(ge=0.0, le=1.0, description="共識分數 0-1")
|
||||
recommended_action: str
|
||||
recommended_kubectl: str | None = None
|
||||
final_reasoning: str
|
||||
risk_level: str
|
||||
dissenting_opinions: list[str] = Field(default_factory=list)
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
|
||||
model_config = {"use_enum_values": False}
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"consensus_id": self.consensus_id,
|
||||
"incident_id": self.incident_id,
|
||||
"opinions": [op.to_dict() for op in self.opinions],
|
||||
"consensus_score": self.consensus_score,
|
||||
"recommended_action": self.recommended_action,
|
||||
"recommended_kubectl": self.recommended_kubectl,
|
||||
"final_reasoning": self.final_reasoning,
|
||||
"risk_level": self.risk_level,
|
||||
"dissenting_opinions": self.dissenting_opinions,
|
||||
"created_at": self.created_at.isoformat(),
|
||||
"agent_count": len(self.opinions),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "ConsensusResult":
|
||||
return cls(
|
||||
consensus_id=data["consensus_id"],
|
||||
incident_id=data["incident_id"],
|
||||
opinions=[AgentOpinion.from_dict(op) for op in data["opinions"]],
|
||||
consensus_score=data["consensus_score"],
|
||||
recommended_action=data["recommended_action"],
|
||||
recommended_kubectl=data.get("recommended_kubectl"),
|
||||
final_reasoning=data["final_reasoning"],
|
||||
risk_level=data["risk_level"],
|
||||
dissenting_opinions=data.get("dissenting_opinions", []),
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Expert Agent Base (專家 Agent 基類)
|
||||
# =============================================================================
|
||||
|
||||
class ExpertAgent:
|
||||
"""
|
||||
專家 Agent 基類
|
||||
|
||||
每個專家會從自己的角度分析 Incident，
|
||||
子類別實作 analyze() 方法
|
||||
"""
|
||||
|
||||
agent_type: AgentType
|
||||
|
||||
async def analyze(self, incident: Incident) -> AgentOpinion:
|
||||
"""
|
||||
分析 Incident 並產生意見
|
||||
|
||||
子類別必須實作此方法
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class SREAgent(ExpertAgent):
|
||||
"""SRE 專家 - 專注系統穩定性與可用性"""
|
||||
|
||||
agent_type = AgentType.SRE
|
||||
|
||||
async def analyze(self, incident: Incident) -> AgentOpinion:
|
||||
"""SRE 視角分析"""
|
||||
# 分析 signals 決定建議
|
||||
alert_names = " ".join([s.alert_name.lower() for s in incident.signals])
|
||||
target = incident.affected_services[0] if incident.affected_services else "unknown"
|
||||
|
||||
# SRE 規則引擎
|
||||
if any(kw in alert_names for kw in ["crash", "restart", "oom", "killed"]):
|
||||
action = "重新啟動服務以恢復穩定性"
|
||||
kubectl = f"kubectl rollout restart deployment/{target} -n awoooi-prod"
|
||||
confidence = 0.85
|
||||
risk = "medium"
|
||||
elif any(kw in alert_names for kw in ["latency", "slow", "timeout"]):
|
||||
action = "擴展副本數以分散負載"
|
||||
kubectl = f"kubectl scale deployment/{target} --replicas=3 -n awoooi-prod"
|
||||
confidence = 0.80
|
||||
risk = "low"
|
||||
elif any(kw in alert_names for kw in ["cpu", "memory", "resource"]):
|
||||
action = "調整資源限制或擴展副本"
|
||||
kubectl = f"kubectl scale deployment/{target} --replicas=2 -n awoooi-prod"
|
||||
confidence = 0.75
|
||||
risk = "medium"
|
||||
else:
|
||||
action = "進行安全重啟以排除未知問題"
|
||||
kubectl = f"kubectl rollout restart deployment/{target} -n awoooi-prod"
|
||||
confidence = 0.60
|
||||
risk = "medium"
|
||||
|
||||
return AgentOpinion(
|
||||
agent_type=self.agent_type,
|
||||
action=action,
|
||||
reasoning=f"SRE 分析: 根據告警 {alert_names[:50]} 判斷服務 {target} 需要 {action}",
|
||||
confidence=confidence,
|
||||
risk_assessment=f"SRE 評估風險等級: {risk}，預計恢復時間 < 5 分鐘",
|
||||
kubectl_command=kubectl,
|
||||
priority=8 if incident.severity.value in ["P0", "P1"] else 5,
|
||||
estimated_impact={
|
||||
"downtime_seconds": 30 if "restart" in action else 0,
|
||||
"affected_users": "minimal",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class SecurityAgent(ExpertAgent):
|
||||
"""資安專家 - 專注安全風險評估"""
|
||||
|
||||
agent_type = AgentType.SECURITY
|
||||
|
||||
async def analyze(self, incident: Incident) -> AgentOpinion:
|
||||
"""資安視角分析"""
|
||||
target = incident.affected_services[0] if incident.affected_services else "unknown"
|
||||
alert_names = " ".join([s.alert_name.lower() for s in incident.signals])
|
||||
|
||||
# 資安掃描
|
||||
security_concerns = []
|
||||
if any(kw in alert_names for kw in ["auth", "login", "401", "403"]):
|
||||
security_concerns.append("可能存在認證問題")
|
||||
if any(kw in alert_names for kw in ["injection", "xss", "csrf"]):
|
||||
security_concerns.append("可能存在注入攻擊")
|
||||
if any(kw in alert_names for kw in ["rate", "ddos", "flood"]):
|
||||
security_concerns.append("可能存在 DoS 攻擊")
|
||||
|
||||
if security_concerns:
|
||||
action = "建議先隔離受影響服務，啟用 NetworkPolicy 限制"
|
||||
confidence = 0.70
|
||||
risk = "critical"
|
||||
else:
|
||||
action = "無明顯資安風險，建議 SRE 處理"
|
||||
confidence = 0.85
|
||||
risk = "low"
|
||||
|
||||
return AgentOpinion(
|
||||
agent_type=self.agent_type,
|
||||
action=action,
|
||||
reasoning=f"Security 分析: {'; '.join(security_concerns) if security_concerns else '未發現資安威脅'}",
|
||||
confidence=confidence,
|
||||
risk_assessment=f"資安風險等級: {risk}",
|
||||
kubectl_command=None, # 資安建議通常需要人工審核
|
||||
priority=9 if security_concerns else 3,
|
||||
estimated_impact={
|
||||
"security_risk": "high" if security_concerns else "none",
|
||||
"requires_audit": bool(security_concerns),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class CostAgent(ExpertAgent):
|
||||
"""成本專家 - 專注資源效益分析"""
|
||||
|
||||
agent_type = AgentType.COST
|
||||
|
||||
async def analyze(self, incident: Incident) -> AgentOpinion:
|
||||
"""成本視角分析"""
|
||||
target = incident.affected_services[0] if incident.affected_services else "unknown"
|
||||
|
||||
# 成本評估 (假設每個副本每小時 $0.05)
|
||||
action = "建議使用 HPA 自動擴展而非固定擴容，以優化成本"
|
||||
kubectl = f"kubectl autoscale deployment/{target} --cpu-percent=70 --min=2 --max=5 -n awoooi-prod"
|
||||
|
||||
return AgentOpinion(
|
||||
agent_type=self.agent_type,
|
||||
action=action,
|
||||
reasoning="FinOps 分析: 使用 HPA 可在負載降低後自動縮減，相比固定擴容可節省約 40% 成本",
|
||||
confidence=0.75,
|
||||
risk_assessment="成本風險: low，使用 HPA 可自動調節",
|
||||
kubectl_command=kubectl,
|
||||
priority=4,
|
||||
estimated_impact={
|
||||
"monthly_cost_change": "+$15 to +$50",
|
||||
"cost_optimization": "HPA 自動縮減",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class PerformanceAgent(ExpertAgent):
|
||||
"""效能專家 - 專注性能優化"""
|
||||
|
||||
agent_type = AgentType.PERFORMANCE
|
||||
|
||||
async def analyze(self, incident: Incident) -> AgentOpinion:
|
||||
"""效能視角分析"""
|
||||
target = incident.affected_services[0] if incident.affected_services else "unknown"
|
||||
alert_names = " ".join([s.alert_name.lower() for s in incident.signals])
|
||||
|
||||
if any(kw in alert_names for kw in ["latency", "p99", "slow"]):
|
||||
action = "建議增加資源限制並啟用 PodDisruptionBudget"
|
||||
kubectl = f"kubectl patch deployment/{target} -n awoooi-prod -p '{{\"spec\":{{\"template\":{{\"spec\":{{\"containers\":[{{\"name\":\"{target}\",\"resources\":{{\"limits\":{{\"cpu\":\"2\",\"memory\":\"2Gi\"}}}}}}]}}}}}}}}'"
|
||||
confidence = 0.80
|
||||
else:
|
||||
action = "當前效能指標正常，建議觀察"
|
||||
kubectl = None
|
||||
confidence = 0.70
|
||||
|
||||
return AgentOpinion(
|
||||
agent_type=self.agent_type,
|
||||
action=action,
|
||||
reasoning=f"Performance 分析: 根據 P99 latency 指標，{action}",
|
||||
confidence=confidence,
|
||||
risk_assessment="效能風險: medium，資源調整可能影響其他 Pod",
|
||||
kubectl_command=kubectl,
|
||||
priority=6,
|
||||
estimated_impact={
|
||||
"latency_improvement": "預計 P99 降低 30%",
|
||||
"resource_increase": "+1 CPU, +1Gi Memory",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Consensus Engine
|
||||
# =============================================================================
|
||||
|
||||
CONSENSUS_PREFIX = "consensus:"
|
||||
CONSENSUS_TTL = 3600 # 1 小時
|
||||
|
||||
|
||||
class ConsensusEngine:
|
||||
"""
|
||||
共識引擎 - Phase 9.4 核心
|
||||
|
||||
職責:
|
||||
1. 收集所有專家 Agent 的意見
|
||||
2. 計算加權共識分數
|
||||
3. 產生最終整合決策
|
||||
4. 儲存結果到 Redis (Working Memory)
|
||||
|
||||
共識計算規則:
|
||||
- 高信心度意見權重較高
|
||||
- 同類型建議會強化共識
|
||||
- 分歧意見會降低共識分數
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._agents: list[ExpertAgent] = [
|
||||
SREAgent(),
|
||||
SecurityAgent(),
|
||||
CostAgent(),
|
||||
PerformanceAgent(),
|
||||
]
|
||||
|
||||
async def gather_opinions(
|
||||
self,
|
||||
incident: Incident,
|
||||
timeout_sec: float = 30.0,
|
||||
) -> list[AgentOpinion]:
|
||||
"""
|
||||
收集所有專家的意見
|
||||
|
||||
並行執行所有專家分析，使用 timeout 防止單一專家阻塞
|
||||
"""
|
||||
async def safe_analyze(agent: ExpertAgent) -> AgentOpinion | None:
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
agent.analyze(incident),
|
||||
timeout=timeout_sec / len(self._agents),
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(
|
||||
"agent_analyze_timeout",
|
||||
agent_type=agent.agent_type.value,
|
||||
incident_id=incident.incident_id,
|
||||
)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"agent_analyze_error",
|
||||
agent_type=agent.agent_type.value,
|
||||
error=str(e),
|
||||
)
|
||||
return None
|
||||
|
||||
# 並行執行所有專家分析
|
||||
results = await asyncio.gather(
|
||||
*[safe_analyze(agent) for agent in self._agents],
|
||||
return_exceptions=False,
|
||||
)
|
||||
|
||||
opinions = [r for r in results if r is not None]
|
||||
|
||||
logger.info(
|
||||
"opinions_gathered",
|
||||
incident_id=incident.incident_id,
|
||||
total_agents=len(self._agents),
|
||||
successful_opinions=len(opinions),
|
||||
)
|
||||
|
||||
return opinions
|
||||
|
||||
def calculate_consensus(
|
||||
self,
|
||||
opinions: list[AgentOpinion],
|
||||
) -> tuple[float, str, list[str]]:
|
||||
"""
|
||||
計算共識分數
|
||||
|
||||
算法:
|
||||
1. 按 action 類型分組
|
||||
2. 計算加權投票 (confidence * priority)
|
||||
3. 最高票數的 action 為推薦
|
||||
4. 共識分數 = 最高票 / 總票數
|
||||
|
||||
Returns:
|
||||
(consensus_score, recommended_action, dissenting_opinions)
|
||||
"""
|
||||
if not opinions:
|
||||
return 0.0, "NO_ACTION", []
|
||||
|
||||
# 按 action 分組計算加權票數
|
||||
action_votes: dict[str, float] = {}
|
||||
action_details: dict[str, list[AgentOpinion]] = {}
|
||||
|
||||
for opinion in opinions:
|
||||
# 低信心度意見權重降低
|
||||
weight_multiplier = 1.0 if opinion.confidence >= 0.6 else 0.5
|
||||
vote_weight = opinion.confidence * opinion.priority * weight_multiplier
|
||||
|
||||
# 簡化 action 到類別
|
||||
action_key = self._normalize_action(opinion.action)
|
||||
|
||||
if action_key not in action_votes:
|
||||
action_votes[action_key] = 0.0
|
||||
action_details[action_key] = []
|
||||
|
||||
action_votes[action_key] += vote_weight
|
||||
action_details[action_key].append(opinion)
|
||||
|
||||
# 找出最高票
|
||||
total_votes = sum(action_votes.values())
|
||||
if total_votes == 0:
|
||||
return 0.0, "NO_ACTION", []
|
||||
|
||||
winner_action = max(action_votes.keys(), key=lambda k: action_votes[k])
|
||||
consensus_score = action_votes[winner_action] / total_votes
|
||||
|
||||
# 找出分歧意見 (非主流意見)
|
||||
dissenting = []
|
||||
for action_key, ops in action_details.items():
|
||||
if action_key != winner_action:
|
||||
for op in ops:
|
||||
dissenting.append(
|
||||
f"{op.agent_type.value}: {op.action} (信心度: {op.confidence:.0%})"
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"consensus_calculated",
|
||||
winner_action=winner_action,
|
||||
consensus_score=consensus_score,
|
||||
total_votes=total_votes,
|
||||
dissenting_count=len(dissenting),
|
||||
)
|
||||
|
||||
return consensus_score, winner_action, dissenting
|
||||
|
||||
def _normalize_action(self, action: str) -> str:
|
||||
"""將 action 正規化到類別"""
|
||||
action_lower = action.lower()
|
||||
|
||||
if any(kw in action_lower for kw in ["重啟", "restart"]):
|
||||
return "RESTART"
|
||||
elif any(kw in action_lower for kw in ["擴展", "scale", "副本"]):
|
||||
return "SCALE"
|
||||
elif any(kw in action_lower for kw in ["hpa", "autoscale"]):
|
||||
return "HPA"
|
||||
elif any(kw in action_lower for kw in ["隔離", "isolate", "network"]):
|
||||
return "ISOLATE"
|
||||
elif any(kw in action_lower for kw in ["資源", "resource", "limit"]):
|
||||
return "TUNE_RESOURCES"
|
||||
elif any(kw in action_lower for kw in ["觀察", "observe", "正常"]):
|
||||
return "OBSERVE"
|
||||
else:
|
||||
return "OTHER"
|
||||
|
||||
async def generate_final_decision(
|
||||
self,
|
||||
incident: Incident,
|
||||
opinions: list[AgentOpinion],
|
||||
consensus_score: float,
|
||||
recommended_action_type: str,
|
||||
dissenting: list[str],
|
||||
) -> ConsensusResult:
|
||||
"""
|
||||
產生最終決策
|
||||
|
||||
整合所有專家意見，產生結構化的 ConsensusResult
|
||||
"""
|
||||
consensus_id = f"CON-{datetime.now(timezone.utc).strftime('%Y%m%d')}-{uuid4().hex[:8].upper()}"
|
||||
|
||||
# 找出最佳的 kubectl 指令 (來自最高 priority + confidence 的意見)
|
||||
best_kubectl = None
|
||||
best_score = 0.0
|
||||
best_action_detail = ""
|
||||
|
||||
for op in opinions:
|
||||
if self._normalize_action(op.action) == recommended_action_type:
|
||||
score = op.confidence * op.priority
|
||||
if score > best_score and op.kubectl_command:
|
||||
best_score = score
|
||||
best_kubectl = op.kubectl_command
|
||||
best_action_detail = op.action
|
||||
|
||||
# 決定風險等級
|
||||
if consensus_score >= 0.8:
|
||||
risk_level = "low"
|
||||
elif consensus_score >= 0.6:
|
||||
risk_level = "medium"
|
||||
else:
|
||||
risk_level = "critical" # 共識不足，需人工審核
|
||||
|
||||
# 組合最終推理
|
||||
reasoning_parts = []
|
||||
for op in opinions:
|
||||
reasoning_parts.append(f"[{op.agent_type.value.upper()}] {op.reasoning}")
|
||||
|
||||
final_reasoning = (
|
||||
f"共識引擎整合 {len(opinions)} 位專家意見:\n"
|
||||
+ "\n".join(reasoning_parts)
|
||||
+ f"\n\n最終共識: {recommended_action_type} (共識度: {consensus_score:.0%})"
|
||||
)
|
||||
|
||||
result = ConsensusResult(
|
||||
consensus_id=consensus_id,
|
||||
incident_id=incident.incident_id,
|
||||
opinions=opinions,
|
||||
consensus_score=consensus_score,
|
||||
recommended_action=best_action_detail or recommended_action_type,
|
||||
recommended_kubectl=best_kubectl,
|
||||
final_reasoning=final_reasoning,
|
||||
risk_level=risk_level,
|
||||
dissenting_opinions=dissenting,
|
||||
)
|
||||
|
||||
# 儲存到 Redis
|
||||
await self._save_consensus(result)
|
||||
|
||||
logger.info(
|
||||
"consensus_generated",
|
||||
consensus_id=consensus_id,
|
||||
incident_id=incident.incident_id,
|
||||
consensus_score=consensus_score,
|
||||
risk_level=risk_level,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
async def run_consensus(
|
||||
self,
|
||||
incident: Incident,
|
||||
timeout_sec: float = 30.0,
|
||||
) -> ConsensusResult:
|
||||
"""
|
||||
執行完整的共識流程
|
||||
|
||||
這是對外的主要 API:
|
||||
1. 收集意見
|
||||
2. 計算共識
|
||||
3. 產生決策
|
||||
"""
|
||||
# Step 1: 收集意見
|
||||
opinions = await self.gather_opinions(incident, timeout_sec)
|
||||
|
||||
# Step 2: 計算共識
|
||||
consensus_score, recommended_action, dissenting = self.calculate_consensus(opinions)
|
||||
|
||||
# Step 3: 產生決策
|
||||
result = await self.generate_final_decision(
|
||||
incident=incident,
|
||||
opinions=opinions,
|
||||
consensus_score=consensus_score,
|
||||
recommended_action_type=recommended_action,
|
||||
dissenting=dissenting,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
async def _save_consensus(self, result: ConsensusResult) -> None:
|
||||
"""儲存共識結果到 Redis"""
|
||||
redis_client = get_redis()
|
||||
key = f"{CONSENSUS_PREFIX}{result.consensus_id}"
|
||||
|
||||
await redis_client.set(
|
||||
key,
|
||||
json.dumps(result.to_dict()),
|
||||
ex=CONSENSUS_TTL,
|
||||
)
|
||||
|
||||
async def get_consensus(self, consensus_id: str) -> ConsensusResult | None:
|
||||
"""取得共識結果"""
|
||||
redis_client = get_redis()
|
||||
key = f"{CONSENSUS_PREFIX}{consensus_id}"
|
||||
|
||||
data = await redis_client.get(key)
|
||||
if data:
|
||||
return ConsensusResult.from_dict(json.loads(data))
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Singleton
|
||||
# =============================================================================
|
||||
|
||||
_consensus_engine: ConsensusEngine | None = None
|
||||
|
||||
|
||||
def get_consensus_engine() -> ConsensusEngine:
|
||||
"""取得 ConsensusEngine 實例 (Singleton)"""
|
||||
global _consensus_engine
|
||||
if _consensus_engine is None:
|
||||
_consensus_engine = ConsensusEngine()
|
||||
return _consensus_engine
|
||||
@@ -22,13 +22,13 @@ Decision Manager - Phase 6.5 非同步決策狀態機
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any, Literal
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import structlog
|
||||
|
||||
from src.core.redis_client import get_redis
|
||||
from src.models.incident import Incident, IncidentStatus, Severity
|
||||
from src.models.incident import Incident
|
||||
from src.services.openclaw import get_openclaw
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
@@ -425,6 +425,124 @@ class DecisionManager:
|
||||
await self._save_token(token)
|
||||
return token
|
||||
|
||||
async def get_or_create_decision_with_consensus(
|
||||
self,
|
||||
incident: Incident,
|
||||
timeout_sec: float = 30.0,
|
||||
use_consensus: bool = True,
|
||||
) -> DecisionToken:
|
||||
"""
|
||||
取得或建立決策令牌 (含 Agent Teams 共識)
|
||||
|
||||
Phase 9.4 升級版本:
|
||||
- 對於 P0/P1 事件，自動啟用 ConsensusEngine
|
||||
- 整合多專家意見
|
||||
- 共識分數影響風險評估
|
||||
|
||||
Args:
|
||||
incident: 事件
|
||||
timeout_sec: 超時秒數
|
||||
use_consensus: 是否使用共識引擎 (預設 True)
|
||||
|
||||
Returns:
|
||||
DecisionToken
|
||||
"""
|
||||
# 判斷是否需要共識 (P0/P1 或明確要求)
|
||||
should_use_consensus = use_consensus and incident.severity.value in ["P0", "P1"]
|
||||
|
||||
if not should_use_consensus:
|
||||
# 使用原有的雙軌決策
|
||||
return await self.get_or_create_decision(incident, timeout_sec)
|
||||
|
||||
# Phase 9.4: 使用 ConsensusEngine
|
||||
from src.services.consensus_engine import get_consensus_engine
|
||||
|
||||
consensus_engine = get_consensus_engine()
|
||||
|
||||
# 檢查現有 token
|
||||
existing_token = await self._find_existing_token(incident.incident_id)
|
||||
if existing_token and existing_token.state in (
|
||||
DecisionState.READY,
|
||||
DecisionState.EXECUTING,
|
||||
DecisionState.COMPLETED,
|
||||
):
|
||||
return existing_token
|
||||
|
||||
# 建立新 token
|
||||
token = DecisionToken(
|
||||
token=f"DEC-{uuid4().hex[:12].upper()}",
|
||||
incident_id=incident.incident_id,
|
||||
state=DecisionState.ANALYZING,
|
||||
)
|
||||
await self._save_token(token)
|
||||
|
||||
logger.info(
|
||||
"decision_analyzing_with_consensus",
|
||||
token=token.token,
|
||||
incident_id=incident.incident_id,
|
||||
)
|
||||
|
||||
try:
|
||||
# 執行共識分析
|
||||
consensus_result = await asyncio.wait_for(
|
||||
consensus_engine.run_consensus(incident, timeout_sec),
|
||||
timeout=timeout_sec,
|
||||
)
|
||||
|
||||
# 轉換為 proposal_data 格式
|
||||
proposal_data = {
|
||||
"source": "consensus_engine",
|
||||
"consensus_id": consensus_result.consensus_id,
|
||||
"consensus_score": consensus_result.consensus_score,
|
||||
"action": consensus_result.recommended_action,
|
||||
"description": consensus_result.final_reasoning,
|
||||
"risk_level": consensus_result.risk_level,
|
||||
"kubectl_command": consensus_result.recommended_kubectl,
|
||||
"reasoning": consensus_result.final_reasoning,
|
||||
"confidence": consensus_result.consensus_score,
|
||||
"agent_count": len(consensus_result.opinions),
|
||||
"dissenting_opinions": consensus_result.dissenting_opinions,
|
||||
"from_cache": False,
|
||||
}
|
||||
|
||||
token.state = DecisionState.READY
|
||||
token.proposal_data = proposal_data
|
||||
token.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
logger.info(
|
||||
"decision_ready_with_consensus",
|
||||
token=token.token,
|
||||
consensus_id=consensus_result.consensus_id,
|
||||
consensus_score=consensus_result.consensus_score,
|
||||
)
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(
|
||||
"consensus_timeout_using_expert",
|
||||
token=token.token,
|
||||
timeout_sec=timeout_sec,
|
||||
)
|
||||
# Fallback 到 Expert System
|
||||
expert_result = expert_analyze(incident)
|
||||
token.state = DecisionState.READY
|
||||
token.proposal_data = expert_result
|
||||
token.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"consensus_error_using_expert",
|
||||
token=token.token,
|
||||
error=str(e),
|
||||
)
|
||||
expert_result = expert_analyze(incident)
|
||||
token.state = DecisionState.READY
|
||||
token.proposal_data = expert_result
|
||||
token.error = str(e)
|
||||
token.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
await self._save_token(token)
|
||||
return token
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Singleton
|
||||
|
||||
@@ -31,7 +31,7 @@ import structlog
|
||||
from src.core.config import settings
|
||||
from src.db.base import get_db_context
|
||||
from src.db.models import AuditLog
|
||||
from src.models.approval import ApprovalRequest, ApprovalStatus
|
||||
from src.models.approval import ApprovalRequest
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
@@ -600,7 +600,6 @@ class ActionExecutor:
|
||||
Returns:
|
||||
ExecutionResult: 執行結果
|
||||
"""
|
||||
import shlex
|
||||
start_time = time.monotonic()
|
||||
|
||||
# 安全檢查: 必須是 kubectl 指令
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
"""
|
||||
Incident Engine v1.1 - Phase 6.3 認知覺醒核心 (效能強化版)
|
||||
============================================================
|
||||
Incident Engine v1.2 - Phase 6.4e DualMemory 整合版
|
||||
====================================================
|
||||
|
||||
v1.2 重構內容 (Phase 6.4e):
|
||||
- 整合 DualIncidentMemory 進行 DB 持久化
|
||||
- 保持 Lua 原子操作進行 Redis Working Memory 更新
|
||||
- 支援從 Episodic Memory (PostgreSQL) 回載 Incident
|
||||
|
||||
v1.1 重構內容 (2026-03-22 架構師審查後修正):
|
||||
1. O(1) 反向索引: 廢除 SCAN，改用 namespace/target 索引直查
|
||||
@@ -30,15 +35,13 @@ from typing import Any
|
||||
import structlog
|
||||
|
||||
from src.core.redis_client import get_redis
|
||||
from src.db.base import get_db_context
|
||||
from src.db.models import IncidentRecord
|
||||
from src.models.incident import (
|
||||
Incident,
|
||||
IncidentStatus,
|
||||
Severity,
|
||||
Signal,
|
||||
)
|
||||
from src.services.graph_rag import topology_graph, BlastRadiusResult
|
||||
from src.services.incident_memory import DualIncidentMemory, get_incident_memory
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
@@ -254,8 +257,15 @@ class IncidentEngine:
|
||||
incident = await engine.process_signal(signal_data)
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, memory: DualIncidentMemory | None = None) -> None:
|
||||
"""
|
||||
初始化 IncidentEngine
|
||||
|
||||
Args:
|
||||
memory: DualIncidentMemory 實例 (可選，預設使用 Singleton)
|
||||
"""
|
||||
self._graph = topology_graph
|
||||
self._memory = memory or get_incident_memory()
|
||||
self._lua_aggregate_sha: str | None = None
|
||||
self._lua_create_sha: str | None = None
|
||||
|
||||
@@ -519,75 +529,53 @@ class IncidentEngine:
|
||||
incident.affected_services.append(target)
|
||||
|
||||
# =========================================================================
|
||||
# 持久化 (DB 層)
|
||||
# 持久化 (DB 層) - Phase 6.4e: 委託給 DualIncidentMemory
|
||||
# =========================================================================
|
||||
|
||||
async def _persist_to_db(self, incident: Incident) -> None:
|
||||
"""
|
||||
持久化到 SQLite/PostgreSQL (Episodic Memory)
|
||||
持久化到 PostgreSQL (Episodic Memory)
|
||||
|
||||
Phase 6.4e: 委託給 DualIncidentMemory.persist_incident()
|
||||
Redis 已在 Lua Script 中更新，這裡只處理 DB
|
||||
"""
|
||||
try:
|
||||
async with get_db_context() as db:
|
||||
from sqlalchemy import select
|
||||
success = await self._memory.persist_incident(incident)
|
||||
incident.persisted_to_pg = success
|
||||
|
||||
# 檢查是否已存在
|
||||
stmt = select(IncidentRecord).where(
|
||||
IncidentRecord.incident_id == incident.incident_id
|
||||
if success:
|
||||
logger.debug(
|
||||
"db_persisted_via_dual_memory",
|
||||
incident_id=incident.incident_id,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"db_persist_failed_via_dual_memory",
|
||||
incident_id=incident.incident_id,
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
existing = result.scalar_one_or_none()
|
||||
|
||||
if existing:
|
||||
# 更新現有記錄
|
||||
existing.status = incident.status.value
|
||||
existing.severity = incident.severity.value
|
||||
existing.signals = [
|
||||
s.model_dump(mode="json") for s in incident.signals
|
||||
]
|
||||
existing.affected_services = incident.affected_services
|
||||
existing.updated_at = incident.updated_at
|
||||
else:
|
||||
# 建立新記錄
|
||||
record = IncidentRecord(
|
||||
incident_id=incident.incident_id,
|
||||
status=incident.status.value,
|
||||
severity=incident.severity.value,
|
||||
signals=[
|
||||
s.model_dump(mode="json") for s in incident.signals
|
||||
],
|
||||
affected_services=incident.affected_services,
|
||||
decision_chain=(
|
||||
incident.decision_chain.model_dump(mode="json")
|
||||
if incident.decision_chain
|
||||
else None
|
||||
),
|
||||
proposal_ids=[str(pid) for pid in incident.proposal_ids],
|
||||
outcome=(
|
||||
incident.outcome.model_dump(mode="json")
|
||||
if incident.outcome
|
||||
else None
|
||||
),
|
||||
created_at=incident.created_at,
|
||||
updated_at=incident.updated_at,
|
||||
resolved_at=incident.resolved_at,
|
||||
closed_at=incident.closed_at,
|
||||
ttl_days=incident.ttl_days,
|
||||
vectorized=incident.vectorized,
|
||||
)
|
||||
db.add(record)
|
||||
|
||||
incident.persisted_to_pg = True
|
||||
|
||||
logger.debug(
|
||||
"db_persisted",
|
||||
incident_id=incident.incident_id,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("db_save_error", error=str(e))
|
||||
|
||||
# =========================================================================
|
||||
# 從 Episodic Memory 載入 (Phase 6.4e 新增)
|
||||
# =========================================================================
|
||||
|
||||
async def get_incident(self, incident_id: str) -> Incident | None:
|
||||
"""
|
||||
取得 Incident
|
||||
|
||||
Phase 6.4e: 委託給 DualIncidentMemory.load_incident()
|
||||
優先從 Working Memory (Redis) 讀取，miss 時從 Episodic (PostgreSQL) 讀取
|
||||
|
||||
Args:
|
||||
incident_id: Incident ID
|
||||
|
||||
Returns:
|
||||
Incident 或 None
|
||||
"""
|
||||
return await self._memory.load_incident(incident_id)
|
||||
|
||||
# =========================================================================
|
||||
# 輔助方法
|
||||
# =========================================================================
|
||||
|
||||
483
apps/api/src/services/incident_memory.py
Normal file
483
apps/api/src/services/incident_memory.py
Normal file
@@ -0,0 +1,483 @@
|
||||
"""
|
||||
Incident Memory Provider - 事件記憶體提供者
|
||||
============================================
|
||||
Phase 6.4e: DualIncidentMemory 整合
|
||||
|
||||
設計:
|
||||
- 實作 IIncidentMemory 協定 (Protocol)
|
||||
- 雙層記憶體: Working (Redis) + Episodic (PostgreSQL)
|
||||
- 反向索引: namespace:target -> incident_id
|
||||
|
||||
統帥鐵律:
|
||||
- Working Memory (Redis): 7 天 TTL
|
||||
- Episodic Memory (PostgreSQL): 永久
|
||||
- 反向索引: 30 分鐘 TTL (聚合窗口)
|
||||
|
||||
NOTE: 此模組為 lewooogo-brain/adapters/incident_memory.py 的 apps/api 內嵌版本
|
||||
待 Phase 6.4i 完成 monorepo Docker 解法後，將直接引用 lewooogo-brain 套件
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Any, Protocol
|
||||
|
||||
import structlog
|
||||
|
||||
from src.core.redis_client import get_redis
|
||||
from src.db.base import get_db_context
|
||||
from src.db.models import IncidentRecord
|
||||
from src.models.incident import Incident
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Constants
|
||||
# =============================================================================
|
||||
|
||||
WORKING_MEMORY_TTL = 604800 # 7 天
|
||||
AGGREGATION_WINDOW_MINUTES = 30
|
||||
INDEX_TTL = 1800 # 索引 30 分鐘 TTL
|
||||
|
||||
# Redis Key Patterns
|
||||
INCIDENT_KEY_PREFIX = "awoooi:incidents:"
|
||||
INDEX_PREFIX = "awoooi:incidents:index:"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Protocol Definition (與 lewooogo-brain 保持一致)
|
||||
# =============================================================================
|
||||
|
||||
class IIncidentMemory(Protocol):
|
||||
"""Incident 專用記憶體提供者協定"""
|
||||
|
||||
async def load_incident(self, incident_id: str) -> Incident | None:
|
||||
"""從 Working Memory 載入 Incident"""
|
||||
...
|
||||
|
||||
async def save_incident(self, incident: Incident, ttl_seconds: int = WORKING_MEMORY_TTL) -> bool:
|
||||
"""儲存 Incident 到 Working Memory (預設 7 天 TTL)"""
|
||||
...
|
||||
|
||||
async def persist_incident(self, incident: Incident) -> bool:
|
||||
"""持久化到 Episodic Memory (PostgreSQL)"""
|
||||
...
|
||||
|
||||
async def find_related_incident(
|
||||
self,
|
||||
namespace: str,
|
||||
target: str,
|
||||
window_minutes: int = AGGREGATION_WINDOW_MINUTES,
|
||||
) -> Incident | None:
|
||||
"""尋找相關的活躍 Incident (用於聚合)"""
|
||||
...
|
||||
|
||||
async def update_index(
|
||||
self,
|
||||
incident_id: str,
|
||||
namespace: str,
|
||||
target: str,
|
||||
) -> bool:
|
||||
"""更新反向索引 (namespace/target -> incident_id)"""
|
||||
...
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DualIncidentMemory Implementation
|
||||
# =============================================================================
|
||||
|
||||
class DualIncidentMemory:
|
||||
"""
|
||||
Incident 專用雙層記憶體適配器
|
||||
|
||||
實作 IIncidentMemory 協定:
|
||||
- load_incident: 從 Working/Episodic 載入
|
||||
- save_incident: 儲存到 Working
|
||||
- persist_incident: 持久化到 Episodic
|
||||
- find_related_incident: 透過反向索引尋找相關 Incident
|
||||
- update_index: 更新反向索引
|
||||
|
||||
反向索引結構:
|
||||
Key: awoooi:incidents:index:{namespace}:{target}
|
||||
Value: incident_id
|
||||
TTL: 30 分鐘 (聚合窗口)
|
||||
"""
|
||||
|
||||
def __init__(self, redis_client: Any = None, key_prefix: str = INCIDENT_KEY_PREFIX):
|
||||
"""
|
||||
初始化適配器
|
||||
|
||||
Args:
|
||||
redis_client: Redis 連線客戶端 (可選，預設使用 get_redis())
|
||||
key_prefix: Redis Key 前綴
|
||||
"""
|
||||
self._redis = redis_client
|
||||
self._key_prefix = key_prefix
|
||||
self._index_prefix = INDEX_PREFIX
|
||||
|
||||
def _get_redis(self) -> Any:
|
||||
"""取得 Redis 客戶端 (延遲初始化)"""
|
||||
if self._redis is None:
|
||||
self._redis = get_redis()
|
||||
return self._redis
|
||||
|
||||
def _make_key(self, incident_id: str) -> str:
|
||||
"""生成 Incident Key"""
|
||||
return f"{self._key_prefix}{incident_id}"
|
||||
|
||||
def _make_index_key(self, namespace: str, target: str) -> str:
|
||||
"""生成索引 Key"""
|
||||
return f"{self._index_prefix}{namespace}:{target}"
|
||||
|
||||
async def load_incident(self, incident_id: str) -> Incident | None:
|
||||
"""
|
||||
載入 Incident
|
||||
|
||||
策略:
|
||||
1. 從 Redis (Working Memory) 讀取
|
||||
2. 若 miss，從 PostgreSQL (Episodic) 讀取
|
||||
|
||||
Args:
|
||||
incident_id: Incident ID
|
||||
|
||||
Returns:
|
||||
Incident 或 None
|
||||
"""
|
||||
try:
|
||||
redis_client = self._get_redis()
|
||||
key = self._make_key(incident_id)
|
||||
data = await redis_client.get(key)
|
||||
|
||||
if data is not None:
|
||||
# JSON -> Incident
|
||||
return Incident.model_validate_json(data)
|
||||
|
||||
# Working Memory miss, 嘗試從 Episodic Memory 載入
|
||||
logger.debug("incident_not_found_in_working", incident_id=incident_id)
|
||||
|
||||
async with get_db_context() as db:
|
||||
from sqlalchemy import select
|
||||
stmt = select(IncidentRecord).where(
|
||||
IncidentRecord.incident_id == incident_id
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
record = result.scalar_one_or_none()
|
||||
|
||||
if record:
|
||||
# 從 DB 重建 Incident
|
||||
incident = self._record_to_incident(record)
|
||||
# 寫回 Working Memory (快取)
|
||||
await self.save_incident(incident)
|
||||
return incident
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error("load_incident_failed", incident_id=incident_id, error=str(e))
|
||||
return None
|
||||
|
||||
async def save_incident(
|
||||
self,
|
||||
incident: Incident,
|
||||
ttl_seconds: int = WORKING_MEMORY_TTL,
|
||||
) -> bool:
|
||||
"""
|
||||
儲存 Incident 到 Working Memory (Redis)
|
||||
|
||||
Args:
|
||||
incident: Incident 物件
|
||||
ttl_seconds: TTL (預設 7 天)
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
try:
|
||||
redis_client = self._get_redis()
|
||||
key = self._make_key(incident.incident_id)
|
||||
json_data = incident.model_dump_json()
|
||||
|
||||
await redis_client.setex(key, ttl_seconds, json_data)
|
||||
|
||||
logger.debug(
|
||||
"incident_saved_to_working",
|
||||
incident_id=incident.incident_id,
|
||||
ttl=ttl_seconds,
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"save_incident_failed",
|
||||
incident_id=incident.incident_id,
|
||||
error=str(e),
|
||||
)
|
||||
return False
|
||||
|
||||
async def persist_incident(self, incident: Incident) -> bool:
|
||||
"""
|
||||
持久化到 Episodic Memory (PostgreSQL)
|
||||
|
||||
Args:
|
||||
incident: Incident 物件
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
try:
|
||||
async with get_db_context() as db:
|
||||
from sqlalchemy import select
|
||||
|
||||
# 檢查是否已存在
|
||||
stmt = select(IncidentRecord).where(
|
||||
IncidentRecord.incident_id == incident.incident_id
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
existing = result.scalar_one_or_none()
|
||||
|
||||
if existing:
|
||||
# 更新現有記錄
|
||||
existing.status = incident.status.value
|
||||
existing.severity = incident.severity.value
|
||||
existing.signals = [
|
||||
s.model_dump(mode="json") for s in incident.signals
|
||||
]
|
||||
existing.affected_services = incident.affected_services
|
||||
existing.updated_at = incident.updated_at
|
||||
if incident.resolved_at:
|
||||
existing.resolved_at = incident.resolved_at
|
||||
if incident.closed_at:
|
||||
existing.closed_at = incident.closed_at
|
||||
else:
|
||||
# 建立新記錄
|
||||
record = IncidentRecord(
|
||||
incident_id=incident.incident_id,
|
||||
status=incident.status.value,
|
||||
severity=incident.severity.value,
|
||||
signals=[
|
||||
s.model_dump(mode="json") for s in incident.signals
|
||||
],
|
||||
affected_services=incident.affected_services,
|
||||
decision_chain=(
|
||||
incident.decision_chain.model_dump(mode="json")
|
||||
if incident.decision_chain
|
||||
else None
|
||||
),
|
||||
proposal_ids=[str(pid) for pid in incident.proposal_ids],
|
||||
outcome=(
|
||||
incident.outcome.model_dump(mode="json")
|
||||
if incident.outcome
|
||||
else None
|
||||
),
|
||||
created_at=incident.created_at,
|
||||
updated_at=incident.updated_at,
|
||||
resolved_at=incident.resolved_at,
|
||||
closed_at=incident.closed_at,
|
||||
ttl_days=incident.ttl_days,
|
||||
vectorized=incident.vectorized,
|
||||
)
|
||||
db.add(record)
|
||||
|
||||
logger.debug(
|
||||
"incident_persisted_to_episodic",
|
||||
incident_id=incident.incident_id,
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"persist_incident_failed",
|
||||
incident_id=incident.incident_id,
|
||||
error=str(e),
|
||||
)
|
||||
return False
|
||||
|
||||
async def find_related_incident(
|
||||
self,
|
||||
namespace: str,
|
||||
target: str,
|
||||
window_minutes: int = AGGREGATION_WINDOW_MINUTES,
|
||||
) -> Incident | None:
|
||||
"""
|
||||
尋找相關的活躍 Incident (用於聚合)
|
||||
|
||||
透過反向索引快速查找:
|
||||
1. 查詢索引 Key: namespace:target -> incident_id
|
||||
2. 載入 Incident
|
||||
3. 檢查是否仍在聚合窗口內
|
||||
|
||||
Args:
|
||||
namespace: 命名空間
|
||||
target: 目標服務
|
||||
window_minutes: 聚合窗口 (分鐘)
|
||||
|
||||
Returns:
|
||||
相關 Incident 或 None
|
||||
"""
|
||||
try:
|
||||
redis_client = self._get_redis()
|
||||
|
||||
# Step 1: 查詢索引
|
||||
index_key = self._make_index_key(namespace, target)
|
||||
incident_id = await redis_client.get(index_key)
|
||||
|
||||
if incident_id is None:
|
||||
return None
|
||||
|
||||
# 解碼 bytes
|
||||
if isinstance(incident_id, bytes):
|
||||
incident_id = incident_id.decode()
|
||||
|
||||
# Step 2: 載入 Incident
|
||||
incident = await self.load_incident(incident_id)
|
||||
if incident is None:
|
||||
# 索引存在但 Incident 不存在，清除索引
|
||||
await redis_client.delete(index_key)
|
||||
return None
|
||||
|
||||
# Step 3: 檢查聚合窗口
|
||||
window_start = datetime.now(timezone.utc) - timedelta(minutes=window_minutes)
|
||||
if incident.updated_at < window_start:
|
||||
# 超出聚合窗口，不聚合
|
||||
logger.debug(
|
||||
"incident_outside_window",
|
||||
incident_id=incident_id,
|
||||
updated_at=incident.updated_at.isoformat(),
|
||||
)
|
||||
return None
|
||||
|
||||
logger.debug(
|
||||
"found_related_incident",
|
||||
incident_id=incident_id,
|
||||
namespace=namespace,
|
||||
target=target,
|
||||
)
|
||||
return incident
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"find_related_incident_failed",
|
||||
namespace=namespace,
|
||||
target=target,
|
||||
error=str(e),
|
||||
)
|
||||
return None
|
||||
|
||||
async def update_index(
|
||||
self,
|
||||
incident_id: str,
|
||||
namespace: str,
|
||||
target: str,
|
||||
) -> bool:
|
||||
"""
|
||||
更新反向索引
|
||||
|
||||
索引結構:
|
||||
Key: awoooi:incidents:index:{namespace}:{target}
|
||||
Value: incident_id
|
||||
TTL: 30 分鐘
|
||||
|
||||
Args:
|
||||
incident_id: Incident ID
|
||||
namespace: 命名空間
|
||||
target: 目標服務
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
try:
|
||||
redis_client = self._get_redis()
|
||||
index_key = self._make_index_key(namespace, target)
|
||||
await redis_client.setex(index_key, INDEX_TTL, incident_id)
|
||||
|
||||
logger.debug(
|
||||
"index_updated",
|
||||
incident_id=incident_id,
|
||||
namespace=namespace,
|
||||
target=target,
|
||||
ttl=INDEX_TTL,
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"update_index_failed",
|
||||
incident_id=incident_id,
|
||||
namespace=namespace,
|
||||
target=target,
|
||||
error=str(e),
|
||||
)
|
||||
return False
|
||||
|
||||
async def delete_incident(self, incident_id: str) -> bool:
|
||||
"""
|
||||
刪除 Incident
|
||||
|
||||
Args:
|
||||
incident_id: Incident ID
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
try:
|
||||
redis_client = self._get_redis()
|
||||
key = self._make_key(incident_id)
|
||||
result = await redis_client.delete(key)
|
||||
return result > 0
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delete_incident_failed",
|
||||
incident_id=incident_id,
|
||||
error=str(e),
|
||||
)
|
||||
return False
|
||||
|
||||
def _record_to_incident(self, record: IncidentRecord) -> Incident:
|
||||
"""
|
||||
將 DB Record 轉換為 Incident 物件
|
||||
|
||||
Args:
|
||||
record: IncidentRecord
|
||||
|
||||
Returns:
|
||||
Incident
|
||||
"""
|
||||
from src.models.incident import (
|
||||
IncidentStatus,
|
||||
Severity,
|
||||
Signal,
|
||||
)
|
||||
|
||||
# 重建 Signals
|
||||
signals = []
|
||||
for s in record.signals or []:
|
||||
signals.append(Signal.model_validate(s))
|
||||
|
||||
return Incident(
|
||||
incident_id=record.incident_id,
|
||||
status=IncidentStatus(record.status),
|
||||
severity=Severity(record.severity),
|
||||
signals=signals,
|
||||
affected_services=record.affected_services or [],
|
||||
proposal_ids=record.proposal_ids or [],
|
||||
created_at=record.created_at,
|
||||
updated_at=record.updated_at,
|
||||
resolved_at=record.resolved_at,
|
||||
closed_at=record.closed_at,
|
||||
ttl_days=record.ttl_days or 30,
|
||||
vectorized=record.vectorized or False,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Singleton
|
||||
# =============================================================================
|
||||
|
||||
_dual_memory: DualIncidentMemory | None = None
|
||||
|
||||
|
||||
def get_incident_memory() -> DualIncidentMemory:
|
||||
"""取得 DualIncidentMemory 實例 (Singleton)"""
|
||||
global _dual_memory
|
||||
if _dual_memory is None:
|
||||
_dual_memory = DualIncidentMemory()
|
||||
return _dual_memory
|
||||
@@ -17,7 +17,6 @@ Features:
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import structlog
|
||||
|
||||
@@ -10,7 +10,6 @@ Phase 6: leWOOOgo Output Plugins
|
||||
"""
|
||||
|
||||
import httpx
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from src.core.config import settings
|
||||
from src.core.logging import get_logger
|
||||
|
||||
@@ -30,11 +30,7 @@ import structlog
|
||||
from src.core.config import settings
|
||||
from src.core.redis_client import get_redis
|
||||
from src.models.ai import (
|
||||
AIRiskLevel,
|
||||
AIBlastRadius,
|
||||
AIDataImpact,
|
||||
OpenClawDecision,
|
||||
SuggestedAction,
|
||||
)
|
||||
from src.services.signoz_client import get_signoz_client, GoldMetrics
|
||||
|
||||
|
||||
@@ -29,7 +29,6 @@ from src.db.models import IncidentRecord
|
||||
from src.models.approval import (
|
||||
ApprovalRequest,
|
||||
ApprovalRequestCreate,
|
||||
ApprovalRequestResponse,
|
||||
BlastRadius,
|
||||
DataImpact,
|
||||
DryRunCheck,
|
||||
@@ -41,7 +40,7 @@ from src.models.incident import (
|
||||
Severity,
|
||||
)
|
||||
from src.services.approval_db import get_approval_service
|
||||
from src.services.trust_engine import trust_engine, normalize_action_pattern, RiskLevel
|
||||
from src.services.trust_engine import trust_engine, normalize_action_pattern
|
||||
from src.services.openclaw import get_openclaw
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
@@ -14,11 +14,8 @@ Features:
|
||||
- 過期的 Nonce 自動清除
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
import structlog
|
||||
|
||||
|
||||
@@ -29,7 +29,6 @@ import structlog
|
||||
from src.core.config import settings
|
||||
from src.services.security_interceptor import (
|
||||
get_security_interceptor,
|
||||
TelegramUser,
|
||||
UserNotWhitelistedError,
|
||||
NonceReplayError,
|
||||
)
|
||||
@@ -884,14 +883,20 @@ class TelegramGateway:
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code == 409:
|
||||
# 409 Conflict: 另一個實例正在使用 getUpdates
|
||||
# 這通常表示有其他 Bot 實例在運行
|
||||
# 409 Conflict: 可能是 HTTP/2 連線狀態污染
|
||||
# 重建 HTTP client 以清除殘留連線
|
||||
logger.warning(
|
||||
"telegram_polling_conflict",
|
||||
status=409,
|
||||
message="另一個 Bot 實例正在運行，嘗試重新刪除 Webhook...",
|
||||
message="偵測到 409 衝突，重建 HTTP client...",
|
||||
)
|
||||
if self._http_client:
|
||||
await self._http_client.aclose()
|
||||
self._http_client = httpx.AsyncClient(
|
||||
timeout=30.0,
|
||||
headers={"Content-Type": "application/json"},
|
||||
http2=False, # 強制 HTTP/1.1 避免連線複用問題
|
||||
)
|
||||
await self._delete_webhook()
|
||||
await asyncio.sleep(LONG_POLLING_RETRY_DELAY)
|
||||
else:
|
||||
logger.error("telegram_polling_http_error", status=e.response.status_code)
|
||||
|
||||
Reference in New Issue
Block a user