feat(aiops): ADR-070 全自動化方向 — 三大修復
Some checks failed
CD Pipeline / build-and-deploy (push) Has been cancelled
Some checks failed
CD Pipeline / build-and-deploy (push) Has been cancelled
1. auto_approve.py: 允許 high risk 自動執行 (low/medium/high 全開放)
- min_confidence 0.65→0.50 (信心門檻降低)
- 新增 DESTRUCTIVE_PATTERNS 攔截真正危險指令
(scale=0, delete deployment/pvc/namespace, drop table)
- 核心: critical + 破壞性操作 → 人工; 其他 → 全自動
2. decision_manager.py: 新增 _collect_mcp_context()
- LLM 分析前先收集真實環境狀態 (SSH/K8s MCP)
- Host/Docker 告警 → ssh_get_container_status + ssh_get_top_processes
- K8s 告警 → k8s_get_events
- 注入 diagnosis_context "當前環境狀態 (MCP 實時查詢)" 區段
3. webhooks.py: 修復 target_resource 提取
- 新增 name/container/job label 提取
- DockerContainerUnhealthy 不再 target=alertname
- IP 位址自動排除 (192.x 開頭不作為 target)
🔴 Tier 3 紅區 — 需首席架構師批准
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1384,20 +1384,91 @@ class DecisionManager:
|
||||
logger.error("kb_rag_unexpected_error", incident_id=incident.incident_id, error=str(e))
|
||||
return ""
|
||||
|
||||
async def _collect_mcp_context(self, incident: Incident) -> str:
|
||||
"""
|
||||
ADR-070 全自動 AIOps: 分析前用 MCP 收集真實環境狀態
|
||||
讓 LLM 拿到真實資訊做決策,而非只憑 alert labels
|
||||
|
||||
策略:
|
||||
- K8s 告警 → K8s MCP 查 Pod 狀態/事件
|
||||
- 主機/Docker 告警 → SSH MCP 查容器狀態/資源
|
||||
|
||||
2026-04-11 Claude Sonnet 4.6 Asia/Taipei
|
||||
"""
|
||||
if not incident.signals:
|
||||
return ""
|
||||
|
||||
labels = incident.signals[0].labels
|
||||
alertname = labels.get("alertname", "")
|
||||
host = labels.get("instance", "").split(":")[0] or labels.get("host", "")
|
||||
container = labels.get("name") or labels.get("container") or incident.affected_services[0] if incident.affected_services else ""
|
||||
ns = labels.get("namespace", "awoooi-prod")
|
||||
|
||||
ctx_parts: list[str] = []
|
||||
|
||||
# 主機/Docker 告警 → SSH MCP 診斷
|
||||
_HOST_ALERT_PREFIXES = ("Host", "Docker", "Sentry", "Harbor", "Ollama", "Backup")
|
||||
if alertname.startswith(_HOST_ALERT_PREFIXES) and host:
|
||||
try:
|
||||
from src.plugins.mcp.providers.ssh_provider import SSHProvider
|
||||
ssh = SSHProvider()
|
||||
if ssh.enabled and host in ("192.168.0.188", "192.168.0.110"):
|
||||
# 查容器狀態
|
||||
if container and container != alertname:
|
||||
status_result = await ssh.execute(
|
||||
tool_name="ssh_get_container_status",
|
||||
params={"host": host, "container_name": container},
|
||||
)
|
||||
if status_result.get("success"):
|
||||
ctx_parts.append(f"[SSH] 容器 {container} 狀態: {status_result.get('output', '')[:300]}")
|
||||
# 查主機資源
|
||||
if "CpuLoad" in alertname or "Memory" in alertname:
|
||||
top_result = await ssh.execute(
|
||||
tool_name="ssh_get_top_processes",
|
||||
params={"host": host, "top_n": 5},
|
||||
)
|
||||
if top_result.get("success"):
|
||||
ctx_parts.append(f"[SSH] 主機 {host} Top processes: {top_result.get('output', '')[:300]}")
|
||||
except Exception as e:
|
||||
logger.debug("mcp_context_ssh_failed", alertname=alertname, error=str(e))
|
||||
|
||||
# K8s 告警 → K8s MCP 查 Pod 狀態
|
||||
if alertname.startswith(("Kube", "K3s")) or labels.get("pod"):
|
||||
try:
|
||||
from src.plugins.mcp.providers.k8s_provider import K8sProvider
|
||||
k8s = K8sProvider()
|
||||
if k8s.enabled:
|
||||
pod = labels.get("pod", "")
|
||||
if pod:
|
||||
events_result = await k8s.execute(
|
||||
tool_name="k8s_get_events",
|
||||
params={"namespace": ns, "field_selector": f"involvedObject.name={pod}"},
|
||||
)
|
||||
if events_result.get("success"):
|
||||
ctx_parts.append(f"[K8s] Pod {pod} 事件: {events_result.get('output', '')[:300]}")
|
||||
except Exception as e:
|
||||
logger.debug("mcp_context_k8s_failed", alertname=alertname, error=str(e))
|
||||
|
||||
return "\n".join(ctx_parts)
|
||||
|
||||
async def _dual_engine_analyze(
|
||||
self,
|
||||
incident: Incident,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
三軌決策分析 (Phase 7.5 升級 + KB Phase 2 RAG 整合)
|
||||
三軌決策分析 (Phase 7.5 升級 + KB Phase 2 RAG 整合 + ADR-070 MCP 前置收集)
|
||||
|
||||
策略:
|
||||
1. 先檢查 Playbook 是否有高度匹配 (similarity >= 85%)
|
||||
2. Playbook 命中則直接使用 (最快、經驗驗證)
|
||||
3. 否則 LLM + Expert System 雙軌 + KB RAG context 注入
|
||||
1. MCP 前置收集真實環境狀態(ADR-070)
|
||||
2. 先檢查 Playbook 是否有高度匹配 (similarity >= 85%)
|
||||
3. Playbook 命中則直接使用 (最快、經驗驗證)
|
||||
4. 否則 LLM + Expert System 雙軌 + KB RAG context + MCP context 注入
|
||||
|
||||
優先順序: Playbook > LLM > Expert System
|
||||
"""
|
||||
# ADR-070: 分析前用 MCP 收集真實環境狀態
|
||||
mcp_context = await self._collect_mcp_context(incident)
|
||||
|
||||
# Phase 7.5: 先嘗試 Playbook 匹配
|
||||
playbook_result = await self._try_playbook_match(incident)
|
||||
if playbook_result:
|
||||
@@ -1418,13 +1489,19 @@ class DecisionManager:
|
||||
try:
|
||||
signals_dict = [s.model_dump() for s in incident.signals]
|
||||
|
||||
# 將 KB context 注入 expert_context 傳給 LLM
|
||||
# 將 KB context + MCP 實時狀態 注入 expert_context 傳給 LLM
|
||||
# ADR-070: MCP context 優先放最前面,讓 LLM 看到真實環境狀態再做決策
|
||||
llm_expert_context: dict[str, Any] = {**expert_result} if expert_result else {}
|
||||
existing = str(llm_expert_context.get("diagnosis_context", ""))
|
||||
context_parts = []
|
||||
if mcp_context:
|
||||
context_parts.append(f"## 當前環境狀態 (MCP 實時查詢)\n{mcp_context}")
|
||||
if kb_context:
|
||||
existing = str(llm_expert_context.get("diagnosis_context", ""))
|
||||
llm_expert_context["diagnosis_context"] = (
|
||||
f"{kb_context}\n\n{existing}" if existing else kb_context
|
||||
)
|
||||
context_parts.append(f"## 相關歷史知識\n{kb_context}")
|
||||
if existing:
|
||||
context_parts.append(existing)
|
||||
if context_parts:
|
||||
llm_expert_context["diagnosis_context"] = "\n\n".join(context_parts)
|
||||
|
||||
llm_result, provider, success = await self._openclaw.generate_incident_proposal_with_tools(
|
||||
incident_id=incident.incident_id,
|
||||
|
||||
Reference in New Issue
Block a user