feat(api): Expert System 智能診斷重構 - 根因優先
問題: 原本的 Expert System 只會建議「重啟」，不診斷根因 重構: 1. 分層診斷: - 層 1: 測試資源過濾 (test/demo/tmp 自動忽略) - 層 2: 規則匹配 (更精確的 pattern) - 層 3: 診斷指令 (提供 kubectl 診斷命令) 2. 根因優先: - OOM → 檢查記憶體用量，非重啟 - CrashLoop → 查看崩潰日誌，非重啟 - ImagePull → 檢查映像配置，非重啟 - Default → 人工診斷，非盲目重啟 3. 人工標記: - 未知問題標記 human_review_required - 降低 confidence (0.5) 這才是正確的自動化修復：先診斷根因，再決定行動 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
OG T
@@ -142,47 +142,128 @@ class DecisionState(str, Enum):
|
||||
# =============================================================================
|
||||
# Expert System - 規則引擎 (Local Fallback)
|
||||
# =============================================================================
|
||||
# 2026-03-27 重構: 分層診斷 + 根因優先 + 避免盲目重啟
|
||||
#
|
||||
# 設計原則:
|
||||
# 1. 診斷優先於修復 - 先了解問題再行動
|
||||
# 2. 測試資源忽略 - 避免處理臨時測試告警
|
||||
# 3. 根因導向 - 提供診斷指令而非直接重啟
|
||||
# 4. 人工判斷 - 未知問題建議人工介入
|
||||
# =============================================================================
|
||||
|
||||
# 測試資源黑名單 (自動忽略)
|
||||
TEST_RESOURCE_PATTERNS = [
|
||||
"test", "demo", "tmp", "temp", "debug", "dev-",
|
||||
"sandbox", "experiment", "trial", "mock",
|
||||
]
|
||||
|
||||
EXPERT_RULES: dict[str, dict[str, Any]] = {
|
||||
# Pod 崩潰 → 重啟
|
||||
"pod_crash": {
|
||||
"patterns": ["crash", "restart", "oom", "killed", "failed"],
|
||||
"action": "kubectl rollout restart deployment/{target}",
|
||||
"description": "Expert System: 偵測到 Pod 異常，建議重啟部署",
|
||||
# ========== 第一類: 明確根因的自動修復 ==========
|
||||
|
||||
# OOM Kill → 建議增加記憶體限制 (非重啟)
|
||||
"oom_killed": {
|
||||
"patterns": ["oomkill", "oom", "out of memory", "memory limit"],
|
||||
"action": "kubectl describe pod {target} -n awoooi-prod | grep -A5 'Last State'",
|
||||
"description": "偵測到 OOM Kill，建議檢查記憶體用量後調整 limits",
|
||||
"risk_level": "medium",
|
||||
"reasoning": "根據歷史數據，重啟可解決 85% 的 Pod 崩潰問題",
|
||||
"reasoning": "OOM 通常是記憶體 limits 不足或記憶體洩漏，重啟無法解決根因",
|
||||
"diagnosis_commands": [
|
||||
"kubectl top pod {target} -n awoooi-prod",
|
||||
"kubectl logs {target} -n awoooi-prod --tail=100 | grep -i memory",
|
||||
],
|
||||
},
|
||||
# 高延遲 → 擴容
|
||||
"high_latency": {
|
||||
"patterns": ["latency", "slow", "timeout", "p99"],
|
||||
"action": "kubectl scale deployment/{target} --replicas=3",
|
||||
"description": "Expert System: 偵測到高延遲，建議擴容至 3 副本",
|
||||
|
||||
# CrashLoopBackOff → 查日誌找根因 (非重啟)
|
||||
"crash_loop": {
|
||||
"patterns": ["crashloop", "backoff", "crash loop"],
|
||||
"action": "kubectl logs {target} -n awoooi-prod --previous --tail=50",
|
||||
"description": "偵測到 CrashLoopBackOff，需查看崩潰日誌找根因",
|
||||
"risk_level": "high",
|
||||
"reasoning": "CrashLoop 表示容器持續崩潰，重啟無效，需從日誌找根因",
|
||||
"diagnosis_commands": [
|
||||
"kubectl describe pod {target} -n awoooi-prod | grep -A10 'Events'",
|
||||
"kubectl logs {target} -n awoooi-prod --previous",
|
||||
],
|
||||
},
|
||||
|
||||
# ImagePullBackOff → 檢查映像名稱 (非重啟)
|
||||
"image_pull_error": {
|
||||
"patterns": ["imagepull", "pull error", "image not found", "errimagepull"],
|
||||
"action": "kubectl describe pod {target} -n awoooi-prod | grep -A5 'Events'",
|
||||
"description": "偵測到映像拉取失敗，需檢查映像名稱或 Registry 連線",
|
||||
"risk_level": "high",
|
||||
"reasoning": "映像問題需修正配置或檢查 Harbor 連線，重啟無法解決",
|
||||
"diagnosis_commands": [
|
||||
"kubectl get pod {target} -n awoooi-prod -o jsonpath='{.spec.containers[*].image}'",
|
||||
],
|
||||
},
|
||||
|
||||
# ========== 第二類: 可能需要擴容的情況 ==========
|
||||
|
||||
# 高 CPU 使用率 → 先診斷是否正常負載
|
||||
"high_cpu": {
|
||||
"patterns": ["cpu", "high cpu", "cpu throttl"],
|
||||
"action": "kubectl top pod -n awoooi-prod -l app={target_app}",
|
||||
"description": "偵測到高 CPU，建議先確認是否為正常負載高峰",
|
||||
"risk_level": "low",
|
||||
"reasoning": "擴容可分散負載，降低單一 Pod 壓力",
|
||||
"reasoning": "CPU 高可能是正常負載，需先診斷再決定是否擴容",
|
||||
"diagnosis_commands": [
|
||||
"kubectl top pod -n awoooi-prod",
|
||||
"kubectl get hpa -n awoooi-prod",
|
||||
],
|
||||
},
|
||||
# 高錯誤率 → 回滾
|
||||
"high_error_rate": {
|
||||
"patterns": ["error", "5xx", "fail", "exception"],
|
||||
"action": "kubectl rollout undo deployment/{target}",
|
||||
"description": "Expert System: 偵測到高錯誤率，建議回滾至上一版",
|
||||
"risk_level": "critical",
|
||||
"reasoning": "錯誤率突增通常源自最近部署，回滾是最快修復方式",
|
||||
},
|
||||
# 資源耗盡 → 擴容
|
||||
"resource_exhaustion": {
|
||||
"patterns": ["cpu", "memory", "resource", "quota"],
|
||||
"action": "kubectl scale deployment/{target} --replicas=2",
|
||||
"description": "Expert System: 偵測到資源耗盡，建議擴容",
|
||||
|
||||
# 高延遲 → 先診斷瓶頸在哪
|
||||
"high_latency": {
|
||||
"patterns": ["latency", "slow", "p99", "p95"],
|
||||
"action": "kubectl logs -n awoooi-prod -l app={target_app} --tail=50 | grep -E 'latency|slow|timeout'",
|
||||
"description": "偵測到高延遲，建議先診斷瓶頸位置",
|
||||
"risk_level": "medium",
|
||||
"reasoning": "增加副本可分散資源壓力",
|
||||
"reasoning": "延遲可能來自 DB、外部 API 或代碼，需診斷後對症下藥",
|
||||
"diagnosis_commands": [
|
||||
"查看 SignOz Trace: http://192.168.0.188:3301/traces",
|
||||
],
|
||||
},
|
||||
# 預設 → 重啟 (最保守)
|
||||
|
||||
# ========== 第三類: 需要謹慎的高風險操作 ==========
|
||||
|
||||
# 高錯誤率 → 建議查日誌，回滾需人工確認
|
||||
"high_error_rate": {
|
||||
"patterns": ["error rate", "5xx", "500 error", "exception rate"],
|
||||
"action": "kubectl logs -n awoooi-prod -l app={target_app} --tail=100 | grep -i error",
|
||||
"description": "偵測到高錯誤率，建議先查日誌確認錯誤類型",
|
||||
"risk_level": "high",
|
||||
"reasoning": "錯誤原因多樣，需先診斷是代碼問題還是依賴服務問題",
|
||||
"diagnosis_commands": [
|
||||
"查看 Sentry: http://192.168.0.110:9000",
|
||||
"kubectl logs -n awoooi-prod -l app={target_app} | grep -i exception",
|
||||
],
|
||||
"human_review_required": True,
|
||||
},
|
||||
|
||||
# ========== 第四類: 已確認可安全重啟的情況 ==========
|
||||
|
||||
# 明確的 Pod 異常 (非 CrashLoop)
|
||||
"pod_unhealthy": {
|
||||
"patterns": ["unhealthy", "not ready", "readiness", "liveness"],
|
||||
"action": "kubectl rollout restart deployment/{target_app} -n awoooi-prod",
|
||||
"description": "Pod 健康檢查失敗，重啟可能解決",
|
||||
"risk_level": "medium",
|
||||
"reasoning": "健康檢查失敗且非 CrashLoop，重啟通常有效",
|
||||
},
|
||||
|
||||
# ========== 預設: 不要盲目重啟，建議人工診斷 ==========
|
||||
"default": {
|
||||
"patterns": [],
|
||||
"action": "kubectl rollout restart deployment/{target}",
|
||||
"description": "Expert System: 無法確定具體問題，建議安全重啟",
|
||||
"risk_level": "medium",
|
||||
"reasoning": "重啟是最安全的通用修復動作",
|
||||
"action": "kubectl describe pod {target} -n awoooi-prod",
|
||||
"description": "無法自動判斷問題類型，建議人工查看詳情後決定",
|
||||
"risk_level": "low",
|
||||
"reasoning": "未知問題不應盲目重啟，需人工判斷根因",
|
||||
"diagnosis_commands": [
|
||||
"kubectl get events -n awoooi-prod --sort-by='.lastTimestamp' | tail -20",
|
||||
"kubectl logs -n awoooi-prod {target} --tail=50",
|
||||
],
|
||||
"human_review_required": True,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -191,34 +272,87 @@ def expert_analyze(incident: Incident) -> dict[str, Any]:
|
||||
"""
|
||||
Expert System 規則引擎分析
|
||||
|
||||
2026-03-27 重構:
|
||||
- 分層診斷 (測試資源過濾 → 規則匹配 → 診斷指令)
|
||||
- 根因優先 (提供診斷指令而非盲目重啟)
|
||||
- 人工判斷標記 (未知問題標記需人工介入)
|
||||
|
||||
這是 100% 本地執行，永不失敗的保底方案
|
||||
"""
|
||||
target = incident.affected_services[0] if incident.affected_services else "unknown-service"
|
||||
alert_names = " ".join([s.alert_name.lower() for s in incident.signals])
|
||||
target_lower = target.lower()
|
||||
|
||||
# 匹配規則
|
||||
# 從 target 提取 app 名稱 (去除 pod hash)
|
||||
# e.g., "awoooi-api-649986569-2sgch" → "awoooi-api"
|
||||
target_app = "-".join(target.split("-")[:2]) if "-" in target else target
|
||||
|
||||
alert_names = " ".join([s.alert_name.lower() for s in incident.signals])
|
||||
all_text = f"{alert_names} {target_lower}"
|
||||
|
||||
# ========== 第一層: 測試資源過濾 ==========
|
||||
is_test_resource = any(pattern in target_lower for pattern in TEST_RESOURCE_PATTERNS)
|
||||
if is_test_resource:
|
||||
return {
|
||||
"source": "expert_system",
|
||||
"action": "# 測試資源，建議忽略或手動清理",
|
||||
"description": f"偵測到測試資源 ({target})，建議確認是否需要清理",
|
||||
"risk_level": "low",
|
||||
"reasoning": "測試資源告警通常是臨時性的，不需要自動修復",
|
||||
"confidence": 0.9,
|
||||
"kubectl_command": f"kubectl delete pod {target} -n awoooi-prod --grace-period=0",
|
||||
"matched_rule": "test_resource_filter",
|
||||
"from_cache": False,
|
||||
"human_review_required": True,
|
||||
"is_test_resource": True,
|
||||
}
|
||||
|
||||
# ========== 第二層: 規則匹配 ==========
|
||||
matched_rule = "default"
|
||||
for rule_name, rule in EXPERT_RULES.items():
|
||||
if rule_name == "default":
|
||||
continue
|
||||
if any(pattern in alert_names for pattern in rule["patterns"]):
|
||||
if any(pattern in all_text for pattern in rule["patterns"]):
|
||||
matched_rule = rule_name
|
||||
break
|
||||
|
||||
rule = EXPERT_RULES[matched_rule]
|
||||
|
||||
return {
|
||||
# 格式化指令 (支援 {target} 和 {target_app})
|
||||
format_vars = {"target": target, "target_app": target_app}
|
||||
action = rule["action"].format(**format_vars)
|
||||
|
||||
# 格式化診斷指令
|
||||
diagnosis_commands = []
|
||||
if "diagnosis_commands" in rule:
|
||||
diagnosis_commands = [
|
||||
cmd.format(**format_vars) if "{" in cmd else cmd
|
||||
for cmd in rule["diagnosis_commands"]
|
||||
]
|
||||
|
||||
# ========== 第三層: 建構回應 ==========
|
||||
result = {
|
||||
"source": "expert_system",
|
||||
"action": rule["action"].format(target=target),
|
||||
"action": action,
|
||||
"description": rule["description"],
|
||||
"risk_level": rule["risk_level"],
|
||||
"reasoning": rule["reasoning"],
|
||||
"confidence": 0.75, # Expert System 固定信心分數
|
||||
"kubectl_command": rule["action"].format(target=target),
|
||||
"confidence": 0.75 if matched_rule != "default" else 0.5,
|
||||
"kubectl_command": action,
|
||||
"matched_rule": matched_rule,
|
||||
"from_cache": False,
|
||||
}
|
||||
|
||||
# 新增診斷指令 (如果有)
|
||||
if diagnosis_commands:
|
||||
result["diagnosis_commands"] = diagnosis_commands
|
||||
|
||||
# 標記是否需要人工審查
|
||||
if rule.get("human_review_required"):
|
||||
result["human_review_required"] = True
|
||||
result["description"] += " (建議人工確認)"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Decision Token (Redis)
|
||||
|
||||
Reference in New Issue
Block a user