fix(aiops-p1): Playbook 學習閉環 5斷點全修 + DB Migration（ADR-092 B4）

【P0.4 補丁】pre_decision_investigator Prometheus query 欄位缺失 - _build_tool_params() 補 "query" 欄位（prometheus_query tool 必要參數） - 新增 _build_prometheus_query() — 依告警類型生成 PromQL（CPU/Memory/Crash/Disk/HTTP/Pod/fallback） - 修復後 D3_METRICS 感官維度實際取得資料（原本 100% 回 missing_query_parameter）【P1 Playbook 學習閉環 B1-B5 全修】 - B2 db/models.py: ApprovalRecord 新增 matched_playbook_id 欄位 + ix_approval_matched_playbook index - B2 db/models.py: TimelineEvent 新增 incident_id 欄位（MCP 稽核用）+ index - B3 approval_db.py: record→ApprovalRequest 補回 incident_id + matched_playbook_id - B4 approval_repository.py: 同 B3（兩個轉換函式必須同步） - B5 approval_db.py: approval_request_to_record_data 補 matched_playbook_id → DB 才能存值【P1.5 KM 寫入】approval_execution.py: fire-and-forget → await wait_for(30s) - 根因：asyncio.create_task 在 Pod recycle 時被殺，KM 寫入靜默遺失 - 修復：await asyncio.wait_for(..., timeout=30.0) + TimeoutError log 【Migration 文件】adr092_p1_learning_chain_fix.sql - ALTER TABLE approval_records ADD COLUMN matched_playbook_id VARCHAR(36) - ALTER TABLE timeline_events ADD COLUMN incident_id VARCHAR(64) - 執行：psql $DATABASE_URL -f apps/api/migrations/adr092_p1_learning_chain_fix.sql 【附帶 Agent 改動】 - decision_manager: Phase 2 YAML NO_ACTION 優先門（主機層/外部服務跳過 Agent Debate） - alert_rules.yaml: Sentry/ClickHouse + HostDiskUsageHigh/Critical 新規則 - solver_agent: action_title 語意合成兜底（取代靜默丟棄） Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-24 15:41:35 +08:00
parent 7f4088bcd0
commit 04ff22563e
11 changed files with 321 additions and 20 deletions
--- a/apps/api/src/services/approval_db.py
+++ b/apps/api/src/services/approval_db.py
@@ -109,6 +109,11 @@ def approval_record_to_request(record: ApprovalRecord) -> ApprovalRequest:
        fingerprint=record.fingerprint,
        hit_count=record.hit_count,
        last_seen_at=record.last_seen_at,
+        # B3 fix 2026-04-24 ogt + Claude Sonnet 4.6: 補回 DB 欄位（人工審核路徑讀回必要）
+        # incident_id / matched_playbook_id 在 ApprovalRequest 基礎模型中有定義
+        # telegram_message_id / telegram_chat_id 只在 DB model，不在 Pydantic ApprovalRequest
+        incident_id=getattr(record, "incident_id", None),
+        matched_playbook_id=getattr(record, "matched_playbook_id", None),
    )


@@ -164,6 +169,9 @@ def approval_request_to_record_data(
        # 不在 dict 裡導致 DB 欄位永遠 NULL，Telegram 卡片顯示 INC 號是空白
        # 用戶在 Telegram 根本認不出對應的告警，審核閉環名存實亡
        "incident_id": request.incident_id,
+        # B5 fix 2026-04-24 ogt + Claude Sonnet 4.6: 補 matched_playbook_id
+        # 原本缺失 → Playbook 人工審核後 trust score 永遠不更新（學習閉環斷鏈）
+        "matched_playbook_id": getattr(request, "matched_playbook_id", None),
    }


--- a/apps/api/src/services/approval_execution.py
+++ b/apps/api/src/services/approval_execution.py
@@ -718,9 +718,19 @@ class ApprovalExecutionService:

        # 2026-04-04 ogt: 執行結果沉澱到 KM — 移出 try/except 確保 learning 失敗也寫入
        # 統帥鐵律: 所有異常與自動修復紀錄必須回寫 KM
-        asyncio.create_task(
-            self._write_execution_result_to_km(approval, success, error_message)
-        )
+        # P1.5 fix 2026-04-24 ogt + Claude Sonnet 4.6: fire-and-forget → await（30s 熔斷）
+        # 根因：asyncio.create_task 在 Pod recycle 時被殺，KM 寫入遺失（audit D 每天+5）
+        try:
+            await asyncio.wait_for(
+                self._write_execution_result_to_km(approval, success, error_message),
+                timeout=30.0,
+            )
+        except asyncio.TimeoutError:
+            logger.warning(
+                "km_write_timeout",
+                approval_id=str(approval.id),
+                timeout_sec=30.0,
+            )

    async def _run_post_execution_verify(
        self,
--- a/apps/api/src/services/decision_manager.py
+++ b/apps/api/src/services/decision_manager.py
@@ -1140,6 +1140,7 @@ def _package_to_proposal_data(package: Any) -> dict[str, Any]:
        "confidence": confidence,
        "risk_level": risk_level,
        "source": "phase2_agent_debate",
+        "provider": "phase2_agents",  # 2026-04-24 ogt: 讓 Telegram 顯示 "Phase2 Agents" 而非通用 "AI 仲裁"
        "requires_human_review": package.requires_human_approval,
        # Phase 2 診斷摘要（供 Audit Trail / 學習閉環，不直接顯示給用戶）
        "debate_summary": package.debate_summary or "",
@@ -2283,6 +2284,45 @@ class DecisionManager:
        # 需要 EvidenceSnapshot；若 P1 未開啟則自行收集
        # 2026-04-15 ogt + Claude Sonnet 4.6（亞太）
        if aiops_flags.is_phase_enabled(2):  # Gate 2: 用 is_phase_enabled 統一父 Phase 守衛
+            # 2026-04-24 ogt + Claude Sonnet 4.6: YAML NO_ACTION 優先門
+            # 根因：Phase 2 五 agent 對主機層/外部服務告警（HostDiskUsageHigh /
+            #       SentryClickHouseMemoryPressure）做 kubectl 分析 → Solver 永遠降級
+            #       (無 K8s target) → confidence=20% + kubectl get pods（無意義）
+            # 修復：YAML 匹配到 NO_ACTION 且 kubectl_command="" → 跳過 Agent Debate，
+            #       直接返回 YAML 規則響應，讓 Telegram 推送人工排查通知
+            try:
+                from src.services.alert_rule_engine import match_rule as _p2_match_rule
+                _p2_labels = incident.signals[0].labels if incident.signals else {}
+                _p2_alertname = _p2_labels.get("alertname", "")
+                _p2_yaml = _p2_match_rule({
+                    "labels": _p2_labels,
+                    "alert_type": _p2_alertname,
+                    "message": (
+                        incident.signals[0].annotations.get("summary", "")
+                        if incident.signals else ""
+                    ),
+                    "target_resource": incident.affected_services[0] if incident.affected_services else "unknown",
+                    "namespace": _p2_labels.get("namespace", "awoooi-prod"),
+                    "severity": incident.severity.value if hasattr(incident.severity, "value") else "medium",
+                })
+                _is_no_action_yaml = (
+                    _p2_yaml is not None
+                    and _p2_yaml.get("suggested_action") == "NO_ACTION"
+                    and not _p2_yaml.get("kubectl_command", "").strip()
+                    and _p2_yaml.get("rule_id", "") not in ("generic_fallback", "")
+                )
+                if _is_no_action_yaml:
+                    logger.info(
+                        "p2_yaml_no_action_bypass",
+                        incident_id=incident.incident_id,
+                        alertname=_p2_alertname,
+                        rule_id=_p2_yaml.get("rule_id", ""),
+                        reason="YAML NO_ACTION 規則命中，跳過 Agent Debate",
+                    )
+                    return _p2_yaml
+            except Exception as _p2_yaml_err:
+                logger.debug("p2_yaml_precheck_error", error=str(_p2_yaml_err))
+
            p2_snapshot = evidence_snapshot
            if p2_snapshot is None:
                try:
--- a/apps/api/src/services/pre_decision_investigator.py
+++ b/apps/api/src/services/pre_decision_investigator.py
@@ -466,18 +466,53 @@ SSH_MCP_ALLOWED_HOSTS 使用完整 IP（如 "192.168.0.110"）。
 """


+def _build_prometheus_query(alertname: str, namespace: str, pod_name: str) -> str:
+    """依告警類型生成 Prometheus PromQL 查詢（供 prometheus_query tool 使用）。
+    2026-04-24 ogt + Claude Sonnet 4.6: P0.4 fix — _build_tool_params 補 query 欄位"""
+    an = alertname.lower()
+    # CPU / 負載
+    if any(k in an for k in ("cpu", "load", "throttl")):
+        filter_pod = f',pod=~"{pod_name}.*"' if pod_name else ""
+        return f'avg(rate(container_cpu_usage_seconds_total{{namespace="{namespace}"{filter_pod}}}[5m]))'
+    # 記憶體
+    elif any(k in an for k in ("memory", "mem", "oom")):
+        filter_pod = f',pod=~"{pod_name}.*"' if pod_name else ""
+        return f'avg(container_memory_working_set_bytes{{namespace="{namespace}"{filter_pod}}}) / 1048576'
+    # CrashLoop / 重啟
+    elif any(k in an for k in ("crash", "restart", "oom", "backoff")):
+        return f'sum(increase(kube_pod_container_status_restarts_total{{namespace="{namespace}"}}[15m]))'
+    # 磁碟 / 儲存
+    elif any(k in an for k in ("disk", "storage", "pvc", "volume", "capacity")):
+        return 'sum(kubelet_volume_stats_used_bytes) by (persistentvolumeclaim)'
+    # HTTP / 可用性
+    elif any(k in an for k in ("http", "error", "5xx", "probe", "down", "unhealthy")):
+        return '1 - avg(probe_success)'
+    # Pod / Container 狀態
+    elif any(k in an for k in ("pod", "container", "deploy", "replicaset")):
+        return f'kube_pod_status_phase{{namespace="{namespace}"}}'
+    # 通用 fallback
+    else:
+        return f'up{{namespace="{namespace}"}}'
+
+
 def _build_tool_params(incident: "Incident") -> dict[str, Any]:
    """從 Incident 提取 MCP 工具呼叫所需的公共參數。"""
    labels = _get_labels(incident)
    raw_host = labels.get("instance", "").split(":")[0] or labels.get("host", "")
    host = _SHORT_HOST_MAP.get(raw_host, raw_host)  # 短名 → 完整 IP
+    namespace = labels.get("namespace", "awoooi-prod")
+    pod_name = labels.get("pod", labels.get("name", ""))
+    alertname = labels.get("alertname", "")
    return {
-        "namespace": labels.get("namespace", "awoooi-prod"),
-        "pod_name": labels.get("pod", labels.get("name", "")),
+        "namespace": namespace,
+        "pod_name": pod_name,
        "deployment": labels.get("deployment", ""),
        "host": host,
        "container": labels.get("container", labels.get("name", "")),
-        "alertname": labels.get("alertname", ""),
+        "alertname": alertname,
+        # P0.4 fix 2026-04-24 ogt + Claude Sonnet 4.6: Prometheus tool 需要 query 欄位
+        # 原本缺少此欄位 → prometheus_query/range tool 傳入空 query → 回傳 error dict
+        "query": _build_prometheus_query(alertname, namespace, pod_name),
    }