From f0d14ab6c415462d9b93a11dce96f38610306d39 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 30 Apr 2026 23:49:17 +0800 Subject: [PATCH] fix(aiops): escalate blocked auto repair --- apps/api/alert_rules.yaml | 15 +-- apps/api/src/api/v1/webhooks.py | 65 +++++++++- apps/api/src/services/decision_manager.py | 119 +++++++++++++++++- .../services/emergency_escalation_service.py | 21 ++++ .../tests/test_alertmanager_rule_bypass.py | 32 ++++- docs/LOGBOOK.md | 17 +++ ops/monitoring/alerts-unified.yml | 12 +- 7 files changed, 264 insertions(+), 17 deletions(-) diff --git a/apps/api/alert_rules.yaml b/apps/api/alert_rules.yaml index 7d32d0cd8..53b97b201 100644 --- a/apps/api/alert_rules.yaml +++ b/apps/api/alert_rules.yaml @@ -53,6 +53,7 @@ rules: alertname: - TargetDown - InstanceDown + - NodeExporterDown response: action_title: "重啟 {job} exporter on {host}" description: "⚙️ 規則匹配: Prometheus 無法抓取 {instance} ({job}) 指標。自動重啟主機上的 exporter container。" @@ -664,17 +665,17 @@ rules: - VeleroBackupNotRun - BackupJobFailed response: - action_title: "備份失敗，需人工確認" - description: "⚠️ 備份任務失敗，無自動修復動作。請人工確認備份腳本及磁碟空間。" - suggested_action: NO_ACTION - kubectl_command: "" + action_title: "🔍 備份失敗自動診斷 — SSH 收集備份與磁碟狀態" + description: "⚠️ 備份任務失敗。先自動 SSH 收集 backup log、last_success 與磁碟空間；若無法確認安全修復，立即升級緊急介入。" + suggested_action: SSH_DIAGNOSE + kubectl_command: "ssh {host} 'echo \"=== BACKUP STATUS ===\"; ls -lah /home/ollama/backup/110 2>/dev/null || true; echo \"=== LAST SUCCESS ===\"; cat /home/ollama/backup/110/last_success 2>/dev/null || true; echo \"=== BACKUP LOG ===\"; tail -80 /home/ollama/backup/110/backup.log 2>/dev/null || true; echo \"=== DISK ===\"; df -h /home/ollama /backup / 2>/dev/null || df -h'" estimated_downtime: "N/A" - risk: medium + risk: low responsibility: INFRA - responsibility_reasoning: "備份失敗屬基礎設施維運問題，需人工介入確認根因" + responsibility_reasoning: "備份失敗屬基礎設施維運問題，先自動收集只讀證據，再交由緊急介入或後續 Playbook 修復" secondary_teams: [] optimization: [] - reasoning: "[規則匹配] 備份失敗無法自動修復，需人工排查備份腳本、磁碟空間及網路連通性。" + reasoning: "[規則匹配] 備份失敗先自動 SSH 只讀診斷，避免 LLM 誤判為 K8s deployment 重啟。" # ── DevOps 工具層 ───────────────────────────────────────── # 2026-04-14 Claude Sonnet 4.6: Task 2.2 ADR-076 — 新增 devops_tool / ssl_cert / external_site 三類規則 diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index d1077d90e..e963ee364 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -111,6 +111,26 @@ def _should_bypass_alertmanager_llm( ) +def _should_use_alertmanager_rule_first( + rule_response: dict | None, + alert_category: str, +) -> bool: + """Host 類告警命中權威 YAML 規則時，避免再被 LLM 污染成 K8s 動作。""" + + if not rule_response or alert_category != "host_resource": + return False + if rule_response.get("rule_id", "") in ("generic_fallback", ""): + return False + + suggested_action = str(rule_response.get("suggested_action", "")).upper() + command = str(rule_response.get("kubectl_command", "")).strip().lower() + return ( + suggested_action == "NO_ACTION" + or suggested_action.startswith("SSH_") + or command.startswith("ssh ") + ) + + async def _escalate_auto_repair_unavailable( *, incident_id: str, @@ -1334,7 +1354,7 @@ async def _process_new_alert_background( openclaw = get_openclaw() rule_response = match_rule(alert_context) - should_bypass_llm = _should_bypass_alertmanager_llm(rule_response, alert_category) + should_bypass_llm = _should_use_alertmanager_rule_first(rule_response, alert_category) if should_bypass_llm: logger.info( @@ -1343,7 +1363,7 @@ async def _process_new_alert_background( alertname=alertname, rule_id=rule_response.get("rule_id", ""), alert_category=alert_category, - reason="host_resource YAML NO_ACTION 規則命中，跳過 LLM 產生人工排查卡片", + reason="host_resource YAML 權威規則命中，跳過 LLM 避免產生錯誤 K8s 動作", ) risk_mapping = { "low": RiskLevel.LOW, @@ -1517,6 +1537,37 @@ async def _process_new_alert_background( error=str(_meta_err), ) + _is_heartbeat = is_heartbeat_alertname(alertname) + if can_auto_repair and not _is_heartbeat: + await _try_auto_repair_background( + incident_id=incident_id, + approval_id=str(approval.id), + alert_type=alert_type, + target_resource=target_resource, + namespace=namespace, + ) + elif not can_auto_repair and not _is_heartbeat: + from src.repositories.alert_operation_log_repository import get_alert_operation_log_repository + _op_log_rule = get_alert_operation_log_repository() + await _op_log_rule.append( + "GUARDRAIL_BLOCKED", + incident_id=incident_id, + approval_id=str(approval.id), + actor="prometheus-rule", + action_detail=f"Prometheus rule 設定 auto_repair=false，轉入緊急介入: {alertname}", + success=False, + context={"alertname": alertname, "auto_repair_flag": False}, + ) + await _escalate_auto_repair_unavailable( + incident_id=incident_id, + approval_id=str(approval.id), + alert_type=alert_type, + target_resource=target_resource, + namespace=namespace, + failure_reason="Prometheus rule auto_repair=false，未進入自動修復評估", + attempted_actions="rule_engine -> rule_first -> emergency_intervention", + ) + await _push_to_telegram_background( approval_id=str(approval.id), risk_level=rule_risk.value, @@ -1710,6 +1761,16 @@ async def _process_new_alert_background( success=False, context={"alertname": alertname, "auto_repair_flag": False}, ) + if not _is_heartbeat: + await _escalate_auto_repair_unavailable( + incident_id=incident_id, + approval_id=str(approval.id), + alert_type=alert_type, + target_resource=target_resource, + namespace=namespace, + failure_reason="Prometheus rule auto_repair=false，未進入自動修復評估", + attempted_actions="llm_analysis -> guardrail:auto_repair_false -> emergency_intervention", + ) await _push_to_telegram_background( approval_id=str(approval.id), diff --git a/apps/api/src/services/decision_manager.py b/apps/api/src/services/decision_manager.py index 20cecff6c..ed1f931ee 100644 --- a/apps/api/src/services/decision_manager.py +++ b/apps/api/src/services/decision_manager.py @@ -74,6 +74,61 @@ def _phase2_fallback_reason(package: Any) -> str | None: return None +def _should_escalate_auto_approve_rejection(reason: Any) -> bool: + """Return True for manual gates that mean the automation path went blind.""" + + value = getattr(reason, "value", str(reason or "")).lower() + return value in { + "no_playbook", + "no_executable_action", + "low_trust", + } + + +async def _escalate_decision_auto_repair_unavailable( + *, + incident: Incident, + token: Any, + failure_reason: str, + attempted_actions: str, +) -> None: + """Send the same emergency channel used by auto-repair when decision gates block.""" + + try: + from src.services.emergency_escalation_service import ( + escalate_auto_repair_unavailable, + ) + + labels = incident.signals[0].labels if incident.signals else {} + alertname = labels.get("alertname") or getattr(incident, "alertname", "") or "DecisionAutoRepairBlocked" + target = ( + (incident.affected_services or [None])[0] + or labels.get("deployment") + or labels.get("pod") + or labels.get("service") + or labels.get("component") + or labels.get("instance") + or "unknown" + ) + namespace = labels.get("namespace") or "awoooi-prod" + + await escalate_auto_repair_unavailable( + incident_id=incident.incident_id, + approval_id=getattr(token, "proposal_id", None) or getattr(token, "token", ""), + alert_type=str(alertname), + target_resource=str(target), + namespace=str(namespace), + failure_reason=failure_reason, + attempted_actions=attempted_actions, + ) + except Exception as exc: + logger.warning( + "decision_auto_repair_escalation_failed", + incident_id=getattr(incident, "incident_id", ""), + error=str(exc), + ) + + # ============================================================================= # Phase 31 (ADR-067 2026-04-10): Log 異常摘要 — NemoTron deepseek-r1:14b # ============================================================================= @@ -1634,6 +1689,20 @@ class DecisionManager: # ADR-071: 注入 decision_state + auto_executed 供 classify_notification 使用 token.proposal_data["decision_state"] = token.state.value if token.state else "" token.proposal_data["auto_executed"] = False + token.proposal_data["auto_approve_reason"] = auto_decision.reason.value + token.proposal_data["blocked_reason"] = auto_decision.reason_detail + if _should_escalate_auto_approve_rejection(auto_decision.reason): + _fire_and_forget( + _escalate_decision_auto_repair_unavailable( + incident=incident, + token=token, + failure_reason=auto_decision.reason_detail, + attempted_actions=( + "decision_manager -> auto_approve_policy:" + f"{auto_decision.reason.value} -> emergency_intervention" + ), + ) + ) _fire_and_forget( _push_decision_to_telegram(incident, token.proposal_data) ) @@ -1925,7 +1994,7 @@ class DecisionManager: # alert_category = "infrastructure" 表示 Docker 告警，非 kubectl action → SSH # P1-1 fix 2026-04-12: 必須在 kubectl safety guard 之前 routing，否則 docker 指令被 _action_safe=False 攔截 _alert_category = getattr(incident, "alert_category", None) or "" - if _alert_category == "infrastructure" and action and not action.startswith("kubectl"): + if _alert_category in {"infrastructure", "host_resource"} and action and not action.startswith("kubectl"): await self._ssh_execute(incident, token, action, _target) return @@ -1945,6 +2014,14 @@ class DecisionManager: token.proposal_data["mcp_all_failed"] = True token.proposal_data["blocked_reason"] = "host_resource 告警禁止 K8s kubectl，請人工排查主機" await self._save_token(token) + _fire_and_forget( + _escalate_decision_auto_repair_unavailable( + incident=incident, + token=token, + failure_reason=token.proposal_data["blocked_reason"], + attempted_actions="auto_execute -> host_resource_k8s_block -> emergency_intervention", + ) + ) _fire_and_forget(_push_decision_to_telegram(incident, token.proposal_data)) return @@ -1973,6 +2050,17 @@ class DecisionManager: token.proposal_data["auto_executed"] = False token.proposal_data["mcp_all_failed"] = True # 標記讓 classify_notification → TYPE-4 await self._save_token(token) + _fire_and_forget( + _escalate_decision_auto_repair_unavailable( + incident=incident, + token=token, + failure_reason=( + "Auto-execute safety guard blocked action: " + f"{_action_parse.reason}" + ), + attempted_actions="auto_execute -> action_parser_guard -> emergency_intervention", + ) + ) _fire_and_forget( _push_decision_to_telegram(incident, token.proposal_data) ) @@ -1999,6 +2087,16 @@ class DecisionManager: token.proposal_data["auto_executed"] = False token.proposal_data["mcp_all_failed"] = True await self._save_token(token) + _fire_and_forget( + _escalate_decision_auto_repair_unavailable( + incident=incident, + token=token, + failure_reason=( + f"K8s target '{_target}' not found in namespace '{_ns}'" + ), + attempted_actions="auto_execute -> k8s_existence_check -> emergency_intervention", + ) + ) _fire_and_forget( _push_decision_to_telegram(incident, token.proposal_data) ) @@ -3159,6 +3257,17 @@ class DecisionManager: # 2026-04-27 Claude Sonnet 4.6: 加入主機診斷路徑 # 根因：只支援 docker/systemctl restart，主機告警 ssh {host} '...' 格式全降級人工 # 修復：識別 ssh_diagnose 模式，路由到 ssh_get_top_processes / ssh_get_disk_usage + _labels = incident.signals[0].labels if incident.signals else {} + if not target or target == "unknown": + target = ( + _labels.get("container") + or _labels.get("name") + or _labels.get("component") + or _labels.get("service") + or _labels.get("job") + or "unknown" + ) + _action_lower = action.lower().strip() if _action_lower.startswith("docker restart"): _tool = "docker_restart" @@ -3166,6 +3275,14 @@ class DecisionManager: elif _action_lower.startswith("systemctl restart"): _tool = "service_restart" _service = target + elif _action_lower.startswith("ssh ") and "systemctl restart" in _action_lower: + import re as _ssh_re + _tool = "service_restart" + _svc_match = _ssh_re.search(r"systemctl\s+restart\s+([a-z0-9_.-]+)", _action_lower) + _service = _svc_match.group(1) if _svc_match else target + elif _action_lower.startswith("ssh ") and "docker restart" in _action_lower and target != "unknown": + _tool = "docker_restart" + _container = target elif _action_lower.startswith("ssh ") and ("ps aux" in _action_lower or "top" in _action_lower or "free" in _action_lower or "df -h" in _action_lower or "uptime" in _action_lower): # 主機診斷指令：自動收集 CPU/記憶體/磁碟，不修改任何狀態 _tool = "ssh_diagnose" diff --git a/apps/api/src/services/emergency_escalation_service.py b/apps/api/src/services/emergency_escalation_service.py index 13a136bdf..21b17a225 100644 --- a/apps/api/src/services/emergency_escalation_service.py +++ b/apps/api/src/services/emergency_escalation_service.py @@ -41,6 +41,7 @@ async def escalate_auto_repair_unavailable( from src.repositories.alert_operation_log_repository import ( get_alert_operation_log_repository, ) + from src.services.approval_db import get_timeline_service from src.services.telegram_gateway import get_telegram_gateway await get_telegram_gateway().send_escalation_card( @@ -69,6 +70,26 @@ async def escalate_auto_repair_unavailable( "attempted_actions": attempted_actions, }, ) + try: + await get_timeline_service().add_event( + event_type="agent", + status="warning", + title="AI emergency intervention requested", + description=( + f"{failure_reason} | attempted={attempted_actions}" + )[:500], + actor="auto_repair", + actor_role="emergency_intervention", + approval_id=approval_id, + incident_id=incident_id, + ) + except Exception as timeline_exc: + logger.warning( + "auto_repair_emergency_timeline_failed", + incident_id=incident_id, + approval_id=approval_id, + error=str(timeline_exc), + ) logger.warning( "auto_repair_emergency_escalated", incident_id=incident_id, diff --git a/apps/api/tests/test_alertmanager_rule_bypass.py b/apps/api/tests/test_alertmanager_rule_bypass.py index 4f227ee18..0f58f6555 100644 --- a/apps/api/tests/test_alertmanager_rule_bypass.py +++ b/apps/api/tests/test_alertmanager_rule_bypass.py @@ -1,6 +1,10 @@ from datetime import datetime -from src.api.v1.webhooks import _should_bypass_alertmanager_llm +from src.api.v1.webhooks import ( + _should_bypass_alertmanager_llm, + _should_use_alertmanager_rule_first, +) +from src.services.decision_manager import _should_escalate_auto_approve_rejection from src.services.telegram_gateway import _format_resolved_guard_stamp @@ -34,6 +38,32 @@ def test_non_host_category_does_not_bypass_llm(): assert _should_bypass_alertmanager_llm(rule_response, "kubernetes") is False +def test_host_resource_ssh_rule_uses_rule_first(): + rule_response = { + "rule_id": "host_resource_alert", + "suggested_action": "SSH_DIAGNOSE", + "kubectl_command": "ssh {host} 'df -h'", + } + + assert _should_use_alertmanager_rule_first(rule_response, "host_resource") is True + + +def test_generic_fallback_does_not_use_rule_first(): + rule_response = { + "rule_id": "generic_fallback", + "suggested_action": "SSH_DIAGNOSE", + "kubectl_command": "ssh {host} 'df -h'", + } + + assert _should_use_alertmanager_rule_first(rule_response, "host_resource") is False + + +def test_manual_gate_reasons_escalate_to_emergency_intervention(): + assert _should_escalate_auto_approve_rejection("no_executable_action") is True + assert _should_escalate_auto_approve_rejection("no_playbook") is True + assert _should_escalate_auto_approve_rejection("critical_operation") is False + + def test_resolved_guard_stamp_without_timestamp_is_clean(): assert _format_resolved_guard_stamp(None) == "✅ 此事件已解決" diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 16f91af47..0fe34581e 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -6,6 +6,23 @@ --- +## 2026-04-30 | Auto Repair 緊急介入補洞 — rule-first + host SSH + +統帥批准繼續推進「所有異常要自動修復；無法自修復要有緊急通道」。Live 盤查確認 AI 診斷不是全死，而是主機/備份/磁碟告警常在 `auto_repair=false`、Phase2 空動作、或 LLM 產生 K8s 垃圾 target 後，只回到人工卡。 + +### 完成 +- Alertmanager host_resource 命中 YAML 權威規則時改為 rule-first，`SSH_DIAGNOSE` / SSH 指令不再被 LLM 覆蓋成 `kubectl get pods`、`unknown-service`。 +- `auto_repair=false` 不再靜默等待人工；會寫 `GUARDRAIL_BLOCKED`，並送 TYPE-7E emergency escalation 到 SRE 戰情室。 +- DecisionManager 的 auto-approve manual gate（no_playbook / no_executable_action / low_trust）、host_resource K8s block、action parser block、K8s target missing 全部補 emergency escalation。 +- Emergency escalation 追加 `timeline_events` agent warning，讓 WarRoom timeline 看得到「AI emergency intervention requested」。 +- HostDiskUsageHigh/Critical、HostOutOfMemory、HostOutOfDiskSpace、HostBackupFailed、NodeExporterDown 改進自動修復評估；備份失敗改為 SSH 只讀診斷，NodeExporterDown 補入 YAML rule catalog。 +- DecisionManager SSH route 支援 host_resource 非 kubectl 動作，並可從 `ssh ... systemctl restart` / `ssh ... docker restart` 包裝指令轉進 SSH MCP tool。 + +### 驗證 +- `python3 -m py_compile apps/api/src/api/v1/webhooks.py apps/api/src/services/decision_manager.py apps/api/src/services/emergency_escalation_service.py` 通過。 +- `python3` YAML parse `apps/api/alert_rules.yaml`、`ops/monitoring/alerts-unified.yml` 通過。 +- `cd apps/api && pytest tests/test_alertmanager_rule_bypass.py tests/test_phase2_fallback.py tests/test_telegram_ai_automation_block.py tests/test_ai_router_diagnose_fallback.py tests/test_p0_diagnose_routing.py -q` → 29 passed。 + ## 2026-04-30 | ADR-104 LLM Playbook Generator 第一段落地 承接統帥 AI 自動化目標中「自動建立 Playbook」最低分缺口，先把成功修復後的 learn 階段從 deterministic extraction 擴成 local LLM Playbook generation。 diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 7a601db4e..cba08d8f4 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -55,7 +55,7 @@ groups: severity: warning layer: systemd-188 team: ops - auto_repair: "false" + auto_repair: "true" mcp_provider: "ssh_host" host_type: "bare_metal" alert_category: "host_resource" @@ -70,7 +70,7 @@ groups: severity: warning layer: systemd-188 team: ops - auto_repair: "false" + auto_repair: "true" mcp_provider: "ssh_host" host_type: "bare_metal" alert_category: "host_resource" @@ -169,7 +169,7 @@ groups: team: ops component: backup host: "188" - auto_repair: "false" + auto_repair: "true" alert_category: host_resource annotations: summary: "188 Host 備份超過 25 小時未成功" @@ -927,7 +927,7 @@ groups: layer: systemd-188 alert_category: infrastructure notification_type: TYPE-3 - auto_repair: "false" + auto_repair: "true" supersedes: PostgreSQLDiskGrowthRate annotations: summary: "🔴 主機 {{ $labels.instance }} 根目錄磁碟使用率 {{ $value | humanize }}% (>90%, critical)" @@ -948,7 +948,7 @@ groups: layer: systemd-188 alert_category: infrastructure notification_type: TYPE-3 - auto_repair: "false" + auto_repair: "true" annotations: summary: "主機 {{ $labels.instance }} 無法連通" description: "TCP probe 到 {{ $labels.instance }} 失敗超過 5 分鐘，可能發生網路分區。" @@ -976,7 +976,7 @@ groups: team: ops alert_category: infrastructure notification_type: TYPE-3 - auto_repair: "false" + auto_repair: "true" annotations: summary: "cAdvisor ({{ $labels.instance }}) 停擺" description: "主機 {{ $labels.instance }} 的 cadvisor 已停擺 5 分鐘，容器監控中斷。"