From 54a4e59af97278668485f562fa84176710a78ce8 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 4 May 2026 14:15:05 +0800 Subject: [PATCH] =?UTF-8?q?fix(auto-approve):=20=E4=B8=BB=E6=A9=9F?= =?UTF-8?q?=E5=91=8A=E8=AD=A6=20SSH=20=E8=A8=BA=E6=96=B7=E6=8C=87=E4=BB=A4?= =?UTF-8?q?=E8=B1=81=E5=85=8D=20bad=5Ftarget=20=E9=A9=97=E8=AD=89=20?= =?UTF-8?q?=E2=80=94=20=E4=BF=AE=E5=BE=A9=20no=5Fexecutable=5Faction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 根因：host_resource_alert 規則使用 {host}（由 instance label 派生）， 與 {target} 無關；但 host 告警缺少 K8s deployment label 導致 target=unknown， _is_bad_target=True → kubectl_command 被清空 → auto_approve 以 no_executable_action 拒絕 → 每日 3 次人工攔截。 修復： - alert_rule_engine.py: SSH 指令（startswith "ssh "）跳過 bad_target 驗證 - prompts.py: 主 + Nemo prompt 補 Host* 告警 SSH 診斷規則，防 LLM fallback 路徑輸出 kubectl - ssh_command_whitelist.py: 新建唯讀 SSH 指令白名單模組（供 _ssh_execute() 執行前驗證） Co-Authored-By: Claude Sonnet 4.6 --- apps/api/src/core/prompts.py | 6 +- apps/api/src/services/alert_rule_engine.py | 9 +- .../api/src/services/ssh_command_whitelist.py | 121 ++++++++++++++++++ 3 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 apps/api/src/services/ssh_command_whitelist.py diff --git a/apps/api/src/core/prompts.py b/apps/api/src/core/prompts.py index 6dec04734..139ed6616 100644 --- a/apps/api/src/core/prompts.py +++ b/apps/api/src/core/prompts.py @@ -108,10 +108,11 @@ The `alertname` field is your PRIMARY signal. Use it to determine the problem ty | Alert category / alertname pattern | suggested_action | kubectl_command guidance | |-------------------------------------|-----------------|--------------------------| +| starts with "Host" (HostHighCpuLoad, HostHighMemoryUsage, HostHighLoad, HostOutOfMemory, HostDisk*, etc.) | INVESTIGATE | `ssh 'ps aux --sort=-%cpu \| head -15; free -h; uptime'` — use labels.instance for host IP; do NOT use kubectl | | contains "Disk", "Storage", "PVC", "Volume" | NO_ACTION | `kubectl exec -- df -h` or `kubectl get pvc -n ` | | contains "Postgres", "MySQL", "Redis", "DB", "Database" | NO_ACTION | `kubectl exec -- psql` or `kubectl logs ` | | contains "CrashLoop", "OOMKilled", "Pod" | DELETE_POD or RESTART_DEPLOYMENT | `kubectl delete pod -n ` | -| contains "CPU", "Memory", "Resource" | TUNE_RESOURCES or SCALE_DEPLOYMENT | `kubectl top pod -n ` or HPA command | +| contains "CPU", "Memory", "Resource" (K8s Pod alerts only — NOT Host* alerts) | TUNE_RESOURCES or SCALE_DEPLOYMENT | `kubectl top pod -n ` or HPA command | | contains "Node", "NodeNotReady" | NO_ACTION | `kubectl describe node ` | | contains "SSL", "Certificate", "Cert" | NO_ACTION | `kubectl get certificate -n ` | | alert_category = "database" | NO_ACTION | DB investigation commands only | @@ -184,10 +185,11 @@ You are an SRE AI. Analyze the alert and respond with ONLY valid JSON. ## CRITICAL: Read alertname first The `alertname` field tells you what kind of problem this is. Use it: +- starts with "Host" (HostHighCpuLoad, HostHighMemoryUsage, HostHighLoad, HostOutOfMemory, HostDisk*, etc.) → suggested_action=INVESTIGATE, kubectl_command="ssh 'ps aux --sort=-%cpu | head -15; free -h; uptime'" — NO kubectl commands for host alerts - "Disk/Storage/PVC/Volume" → suggested_action=NO_ACTION, kubectl_command="kubectl get pvc" or "kubectl exec -- df -h" - "Postgres/MySQL/Redis/DB/Database" → suggested_action=NO_ACTION, DB investigation commands - "CrashLoop/OOM/Pod" → suggested_action=DELETE_POD or RESTART_DEPLOYMENT -- "CPU/Memory/Resource" → suggested_action=TUNE_RESOURCES or SCALE_DEPLOYMENT +- "CPU/Memory/Resource" (K8s Pod alerts only) → suggested_action=TUNE_RESOURCES or SCALE_DEPLOYMENT - "SSL/Cert" → suggested_action=NO_ACTION NEVER use "kubectl rollout restart deployment/awoooi-prod" (that is the NAMESPACE, not a deployment). Make action_title describe the ACTUAL problem (not generic "自動修復 AWOOOI 服務"). diff --git a/apps/api/src/services/alert_rule_engine.py b/apps/api/src/services/alert_rule_engine.py index 97c3ffb04..25033e20f 100644 --- a/apps/api/src/services/alert_rule_engine.py +++ b/apps/api/src/services/alert_rule_engine.py @@ -397,8 +397,15 @@ def match_rule(alert_context: dict) -> dict[str, Any] | None: # GAP-A4 (2026-04-14 Claude Sonnet 4.6): 後置驗證 — 垃圾 target 丟棄 command # 避免 `kubectl rollout restart deployment unknown/HostHighCpuLoad/...` 這類無效指令 # 清空 kubectl_command 讓 decision_manager 降級給 LLM 處理 + # 2026-05-04 ogt + Claude Sonnet 4.6: SSH 指令豁免 bad_target 驗證 + # 根因：host_resource_alert 規則的 kubectl_command 以 "ssh {host} '...'" 組成， + # {host} 由 instance label 派生（_extract_vars 第 184-185 行），與 {target} 無關。 + # 但 host 告警缺少 K8s deployment label → target="unknown" → _is_bad_target=True + # → kubectl_command 被清空 → auto_approve 以 no_executable_action 拒絕 → 人工攔截。 + # 修復：SSH 指令不依賴 target，跳過 bad_target 驗證，保留指令讓自動診斷路徑通行。 _invalid_target = False - if kubectl_command and _is_bad_target(vars["target"], alertname): + _is_ssh_command = kubectl_command.startswith("ssh ") + if kubectl_command and not _is_ssh_command and _is_bad_target(vars["target"], alertname): logger.warning( "rule_kubectl_command_discarded_bad_target", rule_id=matched_rule["id"], diff --git a/apps/api/src/services/ssh_command_whitelist.py b/apps/api/src/services/ssh_command_whitelist.py new file mode 100644 index 000000000..00107ab0f --- /dev/null +++ b/apps/api/src/services/ssh_command_whitelist.py @@ -0,0 +1,121 @@ +""" +SSH 唯讀診斷指令白名單 — 防止 RCE + +2026-05-04 ogt + Claude Sonnet 4.6: +設計原則： + - 只允許純唯讀指令（top/uptime/free/df/ps/cat /proc/*） + - 禁止任何 shell metachar（compound/pipeline/substitution/redirect） + - 由 validate_kubectl_command() 的 "ssh " 豁免確保進規則路徑； + 本模組為補充安全層，供未來 _ssh_execute() 執行前再次驗證。 +""" + +# 允許的指令前綴（唯讀診斷用） +_SAFE_PREFIXES: tuple[str, ...] = ( + "top -bn1", + "uptime", + "free -m", + "free -h", + "df -h", + "df -hT", + "iostat", + "vmstat", + "ps aux", + "ps -ef", + "cat /proc/loadavg", + "cat /proc/meminfo", + "cat /proc/cpuinfo", + "echo ", +) + +# 禁止的 shell metachar — 防止指令注入 +_DANGEROUS_PATTERNS: tuple[str, ...] = ( + ";", + "&&", + "||", + "|", + ">", + "<", + "`", + "$(", + "${", + "\\n", + "\n", + "rm ", + "kill ", + "pkill ", + "reboot", + "shutdown", + "poweroff", + "dd ", + "mkfs", + "fdisk", + "wget ", + "curl ", + "nc ", + "ncat ", + "bash ", + "sh ", + "python", + "perl", + "ruby", + "exec ", +) + + +def is_safe_ssh_command(cmd: str) -> bool: + """ + 驗證 SSH 指令是否為唯讀安全指令。 + + 規則： + 1. 必須以 _SAFE_PREFIXES 其中一個前綴開頭 + 2. 不得含有任何 _DANGEROUS_PATTERNS + + Args: + cmd: 要驗證的 SSH 指令字串（不含前導 "ssh '" 包裝） + + Returns: + True — 安全，可執行 + False — 含危險模式，應拒絕 + """ + if not cmd or not cmd.strip(): + return False + + cmd_stripped = cmd.strip() + + # 先阻擋危險模式（優先於前綴白名單） + for pattern in _DANGEROUS_PATTERNS: + if pattern in cmd_stripped: + return False + + # 必須以白名單前綴開頭 + return any(cmd_stripped.startswith(prefix) for prefix in _SAFE_PREFIXES) + + +def extract_inner_command(ssh_cmd: str) -> str: + """ + 從完整 SSH 指令中提取內層指令供 is_safe_ssh_command() 驗證。 + + 範例： + "ssh 192.168.1.1 'uptime'" → "uptime" + "ssh -o StrictHostKeyChecking=no host 'free -m'" → "free -m" + + Args: + ssh_cmd: 完整 SSH 指令字串 + + Returns: + 內層指令字串；若解析失敗返回原始字串 + """ + if not ssh_cmd.startswith("ssh "): + return ssh_cmd + + # 找最後一對單引號或雙引號包裝的指令 + for quote in ("'", '"'): + last_open = ssh_cmd.rfind(quote) + if last_open == -1: + continue + first_open = ssh_cmd.find(quote) + if first_open == last_open: + continue + return ssh_cmd[first_open + 1 : last_open] + + return ssh_cmd