From 2ef7daccdecf20a1441583d1b23ba466b53a24fe Mon Sep 17 00:00:00 2001 From: OG T Date: Thu, 26 Mar 2026 21:35:20 +0800 Subject: [PATCH] =?UTF-8?q?feat(api):=20Expert=20System=20=E6=99=BA?= =?UTF-8?q?=E8=83=BD=E8=A8=BA=E6=96=B7=E9=87=8D=E6=A7=8B=20-=20=E6=A0=B9?= =?UTF-8?q?=E5=9B=A0=E5=84=AA=E5=85=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 問題: 原本的 Expert System 只會建議「重啟」，不診斷根因 重構: 1. 分層診斷: - 層 1: 測試資源過濾 (test/demo/tmp 自動忽略) - 層 2: 規則匹配 (更精確的 pattern) - 層 3: 診斷指令 (提供 kubectl 診斷命令) 2. 根因優先: - OOM → 檢查記憶體用量，非重啟 - CrashLoop → 查看崩潰日誌，非重啟 - ImagePull → 檢查映像配置，非重啟 - Default → 人工診斷，非盲目重啟 3. 人工標記: - 未知問題標記 human_review_required - 降低 confidence (0.5) 這才是正確的自動化修復：先診斷根因，再決定行動 Co-Authored-By: Claude Opus 4.5 --- apps/api/src/services/decision_manager.py | 210 ++++++++++++++++++---- 1 file changed, 172 insertions(+), 38 deletions(-) diff --git a/apps/api/src/services/decision_manager.py b/apps/api/src/services/decision_manager.py index 970c36b63..86bbb13fe 100644 --- a/apps/api/src/services/decision_manager.py +++ b/apps/api/src/services/decision_manager.py @@ -142,47 +142,128 @@ class DecisionState(str, Enum): # ============================================================================= # Expert System - 規則引擎 (Local Fallback) # ============================================================================= +# 2026-03-27 重構: 分層診斷 + 根因優先 + 避免盲目重啟 +# +# 設計原則: +# 1. 診斷優先於修復 - 先了解問題再行動 +# 2. 測試資源忽略 - 避免處理臨時測試告警 +# 3. 根因導向 - 提供診斷指令而非直接重啟 +# 4. 人工判斷 - 未知問題建議人工介入 +# ============================================================================= + +# 測試資源黑名單 (自動忽略) +TEST_RESOURCE_PATTERNS = [ + "test", "demo", "tmp", "temp", "debug", "dev-", + "sandbox", "experiment", "trial", "mock", +] EXPERT_RULES: dict[str, dict[str, Any]] = { - # Pod 崩潰 → 重啟 - "pod_crash": { - "patterns": ["crash", "restart", "oom", "killed", "failed"], - "action": "kubectl rollout restart deployment/{target}", - "description": "Expert System: 偵測到 Pod 異常，建議重啟部署", + # ========== 第一類: 明確根因的自動修復 ========== + + # OOM Kill → 建議增加記憶體限制 (非重啟) + "oom_killed": { + "patterns": ["oomkill", "oom", "out of memory", "memory limit"], + "action": "kubectl describe pod {target} -n awoooi-prod | grep -A5 'Last State'", + "description": "偵測到 OOM Kill，建議檢查記憶體用量後調整 limits", "risk_level": "medium", - "reasoning": "根據歷史數據，重啟可解決 85% 的 Pod 崩潰問題", + "reasoning": "OOM 通常是記憶體 limits 不足或記憶體洩漏，重啟無法解決根因", + "diagnosis_commands": [ + "kubectl top pod {target} -n awoooi-prod", + "kubectl logs {target} -n awoooi-prod --tail=100 | grep -i memory", + ], }, - # 高延遲 → 擴容 - "high_latency": { - "patterns": ["latency", "slow", "timeout", "p99"], - "action": "kubectl scale deployment/{target} --replicas=3", - "description": "Expert System: 偵測到高延遲，建議擴容至 3 副本", + + # CrashLoopBackOff → 查日誌找根因 (非重啟) + "crash_loop": { + "patterns": ["crashloop", "backoff", "crash loop"], + "action": "kubectl logs {target} -n awoooi-prod --previous --tail=50", + "description": "偵測到 CrashLoopBackOff，需查看崩潰日誌找根因", + "risk_level": "high", + "reasoning": "CrashLoop 表示容器持續崩潰，重啟無效，需從日誌找根因", + "diagnosis_commands": [ + "kubectl describe pod {target} -n awoooi-prod | grep -A10 'Events'", + "kubectl logs {target} -n awoooi-prod --previous", + ], + }, + + # ImagePullBackOff → 檢查映像名稱 (非重啟) + "image_pull_error": { + "patterns": ["imagepull", "pull error", "image not found", "errimagepull"], + "action": "kubectl describe pod {target} -n awoooi-prod | grep -A5 'Events'", + "description": "偵測到映像拉取失敗，需檢查映像名稱或 Registry 連線", + "risk_level": "high", + "reasoning": "映像問題需修正配置或檢查 Harbor 連線，重啟無法解決", + "diagnosis_commands": [ + "kubectl get pod {target} -n awoooi-prod -o jsonpath='{.spec.containers[*].image}'", + ], + }, + + # ========== 第二類: 可能需要擴容的情況 ========== + + # 高 CPU 使用率 → 先診斷是否正常負載 + "high_cpu": { + "patterns": ["cpu", "high cpu", "cpu throttl"], + "action": "kubectl top pod -n awoooi-prod -l app={target_app}", + "description": "偵測到高 CPU，建議先確認是否為正常負載高峰", "risk_level": "low", - "reasoning": "擴容可分散負載，降低單一 Pod 壓力", + "reasoning": "CPU 高可能是正常負載，需先診斷再決定是否擴容", + "diagnosis_commands": [ + "kubectl top pod -n awoooi-prod", + "kubectl get hpa -n awoooi-prod", + ], }, - # 高錯誤率 → 回滾 - "high_error_rate": { - "patterns": ["error", "5xx", "fail", "exception"], - "action": "kubectl rollout undo deployment/{target}", - "description": "Expert System: 偵測到高錯誤率，建議回滾至上一版", - "risk_level": "critical", - "reasoning": "錯誤率突增通常源自最近部署，回滾是最快修復方式", - }, - # 資源耗盡 → 擴容 - "resource_exhaustion": { - "patterns": ["cpu", "memory", "resource", "quota"], - "action": "kubectl scale deployment/{target} --replicas=2", - "description": "Expert System: 偵測到資源耗盡，建議擴容", + + # 高延遲 → 先診斷瓶頸在哪 + "high_latency": { + "patterns": ["latency", "slow", "p99", "p95"], + "action": "kubectl logs -n awoooi-prod -l app={target_app} --tail=50 | grep -E 'latency|slow|timeout'", + "description": "偵測到高延遲，建議先診斷瓶頸位置", "risk_level": "medium", - "reasoning": "增加副本可分散資源壓力", + "reasoning": "延遲可能來自 DB、外部 API 或代碼，需診斷後對症下藥", + "diagnosis_commands": [ + "查看 SignOz Trace: http://192.168.0.188:3301/traces", + ], }, - # 預設 → 重啟 (最保守) + + # ========== 第三類: 需要謹慎的高風險操作 ========== + + # 高錯誤率 → 建議查日誌，回滾需人工確認 + "high_error_rate": { + "patterns": ["error rate", "5xx", "500 error", "exception rate"], + "action": "kubectl logs -n awoooi-prod -l app={target_app} --tail=100 | grep -i error", + "description": "偵測到高錯誤率，建議先查日誌確認錯誤類型", + "risk_level": "high", + "reasoning": "錯誤原因多樣，需先診斷是代碼問題還是依賴服務問題", + "diagnosis_commands": [ + "查看 Sentry: http://192.168.0.110:9000", + "kubectl logs -n awoooi-prod -l app={target_app} | grep -i exception", + ], + "human_review_required": True, + }, + + # ========== 第四類: 已確認可安全重啟的情況 ========== + + # 明確的 Pod 異常 (非 CrashLoop) + "pod_unhealthy": { + "patterns": ["unhealthy", "not ready", "readiness", "liveness"], + "action": "kubectl rollout restart deployment/{target_app} -n awoooi-prod", + "description": "Pod 健康檢查失敗，重啟可能解決", + "risk_level": "medium", + "reasoning": "健康檢查失敗且非 CrashLoop，重啟通常有效", + }, + + # ========== 預設: 不要盲目重啟，建議人工診斷 ========== "default": { "patterns": [], - "action": "kubectl rollout restart deployment/{target}", - "description": "Expert System: 無法確定具體問題，建議安全重啟", - "risk_level": "medium", - "reasoning": "重啟是最安全的通用修復動作", + "action": "kubectl describe pod {target} -n awoooi-prod", + "description": "無法自動判斷問題類型，建議人工查看詳情後決定", + "risk_level": "low", + "reasoning": "未知問題不應盲目重啟，需人工判斷根因", + "diagnosis_commands": [ + "kubectl get events -n awoooi-prod --sort-by='.lastTimestamp' | tail -20", + "kubectl logs -n awoooi-prod {target} --tail=50", + ], + "human_review_required": True, }, } @@ -191,34 +272,87 @@ def expert_analyze(incident: Incident) -> dict[str, Any]: """ Expert System 規則引擎分析 + 2026-03-27 重構: + - 分層診斷 (測試資源過濾 → 規則匹配 → 診斷指令) + - 根因優先 (提供診斷指令而非盲目重啟) + - 人工判斷標記 (未知問題標記需人工介入) + 這是 100% 本地執行，永不失敗的保底方案 """ target = incident.affected_services[0] if incident.affected_services else "unknown-service" - alert_names = " ".join([s.alert_name.lower() for s in incident.signals]) + target_lower = target.lower() - # 匹配規則 + # 從 target 提取 app 名稱 (去除 pod hash) + # e.g., "awoooi-api-649986569-2sgch" → "awoooi-api" + target_app = "-".join(target.split("-")[:2]) if "-" in target else target + + alert_names = " ".join([s.alert_name.lower() for s in incident.signals]) + all_text = f"{alert_names} {target_lower}" + + # ========== 第一層: 測試資源過濾 ========== + is_test_resource = any(pattern in target_lower for pattern in TEST_RESOURCE_PATTERNS) + if is_test_resource: + return { + "source": "expert_system", + "action": "# 測試資源，建議忽略或手動清理", + "description": f"偵測到測試資源 ({target})，建議確認是否需要清理", + "risk_level": "low", + "reasoning": "測試資源告警通常是臨時性的，不需要自動修復", + "confidence": 0.9, + "kubectl_command": f"kubectl delete pod {target} -n awoooi-prod --grace-period=0", + "matched_rule": "test_resource_filter", + "from_cache": False, + "human_review_required": True, + "is_test_resource": True, + } + + # ========== 第二層: 規則匹配 ========== matched_rule = "default" for rule_name, rule in EXPERT_RULES.items(): if rule_name == "default": continue - if any(pattern in alert_names for pattern in rule["patterns"]): + if any(pattern in all_text for pattern in rule["patterns"]): matched_rule = rule_name break rule = EXPERT_RULES[matched_rule] - return { + # 格式化指令 (支援 {target} 和 {target_app}) + format_vars = {"target": target, "target_app": target_app} + action = rule["action"].format(**format_vars) + + # 格式化診斷指令 + diagnosis_commands = [] + if "diagnosis_commands" in rule: + diagnosis_commands = [ + cmd.format(**format_vars) if "{" in cmd else cmd + for cmd in rule["diagnosis_commands"] + ] + + # ========== 第三層: 建構回應 ========== + result = { "source": "expert_system", - "action": rule["action"].format(target=target), + "action": action, "description": rule["description"], "risk_level": rule["risk_level"], "reasoning": rule["reasoning"], - "confidence": 0.75, # Expert System 固定信心分數 - "kubectl_command": rule["action"].format(target=target), + "confidence": 0.75 if matched_rule != "default" else 0.5, + "kubectl_command": action, "matched_rule": matched_rule, "from_cache": False, } + # 新增診斷指令 (如果有) + if diagnosis_commands: + result["diagnosis_commands"] = diagnosis_commands + + # 標記是否需要人工審查 + if rule.get("human_review_required"): + result["human_review_required"] = True + result["description"] += " (建議人工確認)" + + return result + # ============================================================================= # Decision Token (Redis)