fix(GAP-A4): 規則 Action 模板 placeholder 解析修復 — 解開 8.3h 飛輪沉默

🚨 真因診斷（統帥逮到）: API log 顯示最近 1 小時爆發大量 auto_execute_blocked_unresolved_placeholder: - action: "kubectl rollout restart deployment HostHighCpuLoad" ← target=alertname - action: "kubectl rollout restart deployment unknown" - action: "kubectl scale deployment unknown --replicas=3" 根因：alert_rule_engine._extract_vars() target 解析邏輯不夠強健，當 Prometheus 告警無 deployment label 時，退回 alertname 或 "unknown"，產生垃圾指令。GAP-A1 防注入閘正確攔下，但自動修復路徑因此卡死， KM 不寫入 → 飛輪沈默。修復（三層防護）: 1. 新增 _strip_pod_suffix() — K8s Pod 名稱還原 Deployment base - Deployment 格式: awoooi-api-7d6b776f78-4sgjl → awoooi-api - StatefulSet: postgresql-0 → postgresql - Legacy: my-job-x2m4k → my-job 2. 新增 _is_bad_target() — 垃圾 target 識別 - 空串 / "unknown" / "none" / "null" - target == alertname 本身 - IP:port 格式、純 IP、含空白/括號/引號 - 未解析 {placeholder} 3. 重寫 _extract_vars() — 多層 label 查找（權威優先）: deployment > app > statefulset > pod(去後綴) > container > service > target_resource 每層都過 _is_bad_target 驗證，全失敗 → target="unknown" 4. match_rule() 後置雙驗證: - bad target → 清空 kubectl_command (降級 LLM) - 殘留 { or } → 清空 kubectl_command (模板未填完) 測試覆蓋： - 33 個新單元測試（GAP-A4 四大場景全覆蓋） - 214/214 回歸測試全過影響： - 原本產出「kubectl rollout restart deployment HostHighCpuLoad」的路徑 → 現在會 `rule_kubectl_command_discarded_bad_target` 並降級 LLM - LLM 若能從錯誤 log 推理真實 deployment，飛輪恢復正常運轉 - 若 LLM 也無解，進 TYPE-4 人工扶梯 2026-04-14 Claude Sonnet 4.6（MASTER 藍圖之外的隱性 Bug 殲滅） Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
2026-04-14 18:43:29 +08:00
parent 88a33eb4d7
commit 10b74affcf
2 changed files with 336 additions and 10 deletions
--- a/apps/api/src/services/alert_rule_engine.py
+++ b/apps/api/src/services/alert_rule_engine.py
@@ -91,26 +91,122 @@ def validate_kubectl_command(command: str) -> bool:
 # ── 變數提取 ────────────────────────────────────────────────


+_POD_SUFFIX_DEPLOYMENT_RE = __import__("re").compile(
+    r"-[a-z0-9]{5,10}-[a-z0-9]{5}$"
+)
+_POD_SUFFIX_LEGACY_RE = __import__("re").compile(
+    r"-[a-z0-9]{5}$"
+)
+_POD_SUFFIX_STATEFULSET_RE = __import__("re").compile(
+    r"-\d+$"
+)
+
+
+def _strip_pod_suffix(pod_name: str) -> str:
+    """
+    由 Pod 名稱推斷 Deployment/StatefulSet base name。
+
+    優先順序（由嚴格到寬鬆）:
+    1. Deployment: {name}-{rs_hash 5-10 chars}-{pod_hash 5 chars}
+       範例: awoooi-api-7d6b776f78-4sgjl → awoooi-api
+    2. StatefulSet: {name}-{ordinal}
+       範例: postgresql-0 → postgresql
+    3. Legacy single-hash Pod: {name}-{hash 5 chars}
+       範例: my-job-x2m4k → my-job
+
+    GAP-A4 (2026-04-14 Claude Sonnet 4.6): Placeholder 解析缺漏修復。
+    """
+    # 先試 Deployment 格式（最常見）
+    stripped = _POD_SUFFIX_DEPLOYMENT_RE.sub("", pod_name)
+    if stripped != pod_name and stripped:
+        return stripped
+    # 再試 StatefulSet
+    stripped = _POD_SUFFIX_STATEFULSET_RE.sub("", pod_name)
+    if stripped != pod_name and stripped:
+        return stripped
+    # 最後試 legacy single-hash
+    stripped = _POD_SUFFIX_LEGACY_RE.sub("", pod_name)
+    if stripped != pod_name and stripped and "-" in stripped:
+        return stripped
+    return pod_name
+
+
+def _is_bad_target(target: str, alertname: str) -> bool:
+    """
+    判斷 target 是否為「垃圾值」，不得組合成 kubectl 指令。
+
+    垃圾值:
+    - 空字串 / "unknown"
+    - 包含空白、冒號（IP:port）、括號、引號
+    - 等於 alertname 本身（LLM/規則填錯）
+    - 純數字或 IP 格式
+    """
+    if not target or target in ("unknown", "none", "null", ""):
+        return True
+    if target == alertname:
+        return True
+    if any(c in target for c in (" ", ":", "(", ")", '"', "'", "<", ">", "{", "}")):
+        return True
+    # 純 IP 格式
+    if target.replace(".", "").isdigit() and target.count(".") == 3:
+        return True
+    return False
+
+
 def _extract_vars(alert_context: dict) -> dict[str, str]:
-    """從 alert_context 提取模板變數"""
+    """
+    從 alert_context 提取模板變數。
+
+    GAP-A4 (2026-04-14 Claude Sonnet 4.6): 強化 target 解析，新增多層 label 查找順序:
+    1. labels.deployment (最權威)
+    2. labels.app / labels.app.kubernetes.io/name
+    3. labels.statefulset
+    4. labels.pod → 去除 replicaset/pod hash 後綴
+    5. labels.container / labels.name
+    6. labels.service
+    7. target_resource（但排除 IP:port 和 alertname）
+
+    若全部提取失敗 → target="unknown"，由 match_rule() 的後置驗證丟棄此規則。
+    """
    labels = alert_context.get("labels", {})
+    alertname = labels.get("alertname", alert_context.get("alert_type", ""))
    raw_target = alert_context.get("target_resource", "unknown")
    instance = labels.get("instance", raw_target)
    host = instance.split(":")[0] if ":" in instance else instance
-    container = labels.get("name", labels.get("container", raw_target))
    job = labels.get("job", "exporter")
    namespace = alert_context.get("namespace", "awoooi-prod")

-    # target: 優先用 pod label，否則用 raw_target（排除純 IP:port 和 alertname）
-    pod = labels.get("pod", "")
-    if pod:
-        target = pod
-    elif ":" in raw_target or raw_target == alert_context.get("labels", {}).get("alertname", ""):
-        # raw_target 是 IP:port 或 alertname — 用 job 或 container 代替
-        target = container if container != raw_target else job
-    else:
+    # GAP-A4: 多層 label 查找，由最權威到最弱
+    target = ""
+    for key in ("deployment", "app", "app.kubernetes.io/name", "statefulset"):
+        val = labels.get(key, "")
+        if val and not _is_bad_target(val, alertname):
+            target = val
+            break
+
+    # Pod label 需去除 hash 後綴還原 Deployment 名稱
+    if not target:
+        pod = labels.get("pod", "")
+        if pod and not _is_bad_target(pod, alertname):
+            target = _strip_pod_suffix(pod)
+
+    # container / name 次優
+    if not target:
+        for key in ("container", "name", "service"):
+            val = labels.get(key, "")
+            if val and not _is_bad_target(val, alertname):
+                target = val
+                break
+
+    # raw_target 末位（且必須通過 bad_target 驗證）
+    if not target and not _is_bad_target(raw_target, alertname):
        target = raw_target

+    # 若全部失敗 → 保留 "unknown" 讓後置驗證層 reject
+    if not target:
+        target = "unknown"
+
+    container = labels.get("name", labels.get("container", "")) or target
    return {
        "target": target,
        "host": host,
@@ -267,6 +363,29 @@ def match_rule(alert_context: dict) -> dict[str, Any] | None:
        )
        kubectl_command = ""

+    # GAP-A4 (2026-04-14 Claude Sonnet 4.6): 後置驗證 — 垃圾 target 丟棄 command
+    # 避免 `kubectl rollout restart deployment unknown/HostHighCpuLoad/...` 這類無效指令
+    # 清空 kubectl_command 讓 decision_manager 降級給 LLM 處理
+    if kubectl_command and _is_bad_target(vars["target"], alertname):
+        logger.warning(
+            "rule_kubectl_command_discarded_bad_target",
+            rule_id=matched_rule["id"],
+            target=vars["target"],
+            alertname=alertname,
+            reason="target 未解析為真實 deployment/app，拒絕組裝指令 → fallback LLM",
+            original_command=kubectl_command[:120],
+        )
+        kubectl_command = ""
+
+    # 還有 {var} 殘留 → 模板變數未被 _fill 填滿（可能 vars 缺少對應 key）
+    if kubectl_command and ("{" in kubectl_command or "}" in kubectl_command):
+        logger.warning(
+            "rule_kubectl_command_discarded_unfilled_placeholder",
+            rule_id=matched_rule["id"],
+            command=kubectl_command[:120],
+        )
+        kubectl_command = ""
+
    return {
        "rule_id": matched_rule["id"],
        "action_title": _fill(resp["action_title"], vars),