fix(aiops-p1): Playbook 學習閉環 5斷點全修 + DB Migration(ADR-092 B4)
【P0.4 補丁】pre_decision_investigator Prometheus query 欄位缺失 - _build_tool_params() 補 "query" 欄位(prometheus_query tool 必要參數) - 新增 _build_prometheus_query() — 依告警類型生成 PromQL(CPU/Memory/Crash/Disk/HTTP/Pod/fallback) - 修復後 D3_METRICS 感官維度實際取得資料(原本 100% 回 missing_query_parameter) 【P1 Playbook 學習閉環 B1-B5 全修】 - B2 db/models.py: ApprovalRecord 新增 matched_playbook_id 欄位 + ix_approval_matched_playbook index - B2 db/models.py: TimelineEvent 新增 incident_id 欄位(MCP 稽核用)+ index - B3 approval_db.py: record→ApprovalRequest 補回 incident_id + matched_playbook_id - B4 approval_repository.py: 同 B3(兩個轉換函式必須同步) - B5 approval_db.py: approval_request_to_record_data 補 matched_playbook_id → DB 才能存值 【P1.5 KM 寫入】approval_execution.py: fire-and-forget → await wait_for(30s) - 根因:asyncio.create_task 在 Pod recycle 時被殺,KM 寫入靜默遺失 - 修復:await asyncio.wait_for(..., timeout=30.0) + TimeoutError log 【Migration 文件】adr092_p1_learning_chain_fix.sql - ALTER TABLE approval_records ADD COLUMN matched_playbook_id VARCHAR(36) - ALTER TABLE timeline_events ADD COLUMN incident_id VARCHAR(64) - 執行:psql $DATABASE_URL -f apps/api/migrations/adr092_p1_learning_chain_fix.sql 【附帶 Agent 改動】 - decision_manager: Phase 2 YAML NO_ACTION 優先門(主機層/外部服務跳過 Agent Debate) - alert_rules.yaml: Sentry/ClickHouse + HostDiskUsageHigh/Critical 新規則 - solver_agent: action_title 語意合成兜底(取代靜默丟棄) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -290,24 +290,62 @@ def _extract_candidates(parsed: dict[str, Any]) -> list[CandidateAction]:
|
||||
# OpenClaw Nemo 格式轉換
|
||||
# 2026-04-17 ogt + Claude Sonnet 4.6: Nemo path kubectl 驗證
|
||||
# 根因:Nemo 回傳 {"action_title": "重啟 Crash Looping Pod"} 自然語言
|
||||
# 直接用 action_title 作為 action → 無 kubectl → auto_approve 誤通過 → 死迴圈
|
||||
# 修復:action_title 不含 kubectl → return [](觸發 _degraded_plan 輸出真實 kubectl)
|
||||
# 直接用 action_title → 無 kubectl → auto_approve 誤通過 → 死迴圈
|
||||
# 2026-04-24 ogt + Claude Sonnet 4.6: 修復靜默丟棄 → 語意合成兜底
|
||||
# 舊:action_title 無 kubectl → return [] → _degraded_plan confidence=0.2
|
||||
# 新:先嘗試語意合成 kubectl 指令;真的無從映射才 return []
|
||||
if "action_title" in parsed and "candidates" not in parsed:
|
||||
action_title = str(parsed.get("action_title", ""))
|
||||
if "kubectl" not in action_title.lower():
|
||||
return [] # 交由 _degraded_plan 接手,輸出真實 kubectl 調查指令
|
||||
confidence = float(parsed.get("confidence", 0.5))
|
||||
risk_level = str(parsed.get("risk_level", "medium"))
|
||||
risk_to_blast = {"critical": 60, "high": 40, "medium": 25, "low": 10}
|
||||
blast = risk_to_blast.get(risk_level.lower(), 30)
|
||||
if action_title and confidence > 0:
|
||||
|
||||
if "kubectl" in action_title.lower():
|
||||
if action_title and confidence > 0:
|
||||
return [CandidateAction(
|
||||
action=action_title[:200],
|
||||
blast_radius=blast,
|
||||
rollback_cost=20,
|
||||
confidence=confidence,
|
||||
rationale=f"OpenClaw Nemo 建議: {action_title}",
|
||||
)]
|
||||
return []
|
||||
|
||||
# action_title 無 kubectl → 嘗試語意合成 kubectl 指令
|
||||
_at_lower = action_title.lower()
|
||||
_synthesized: str | None = None
|
||||
if any(w in _at_lower for w in ("rollback", "undo", "回滾", "還原")):
|
||||
_synthesized = "kubectl rollout undo deployment -n awoooi-prod"
|
||||
elif any(w in _at_lower for w in ("restart", "重啟", "重新啟動")):
|
||||
_synthesized = "kubectl rollout restart deployment -n awoooi-prod"
|
||||
elif any(w in _at_lower for w in ("scale", "擴容", "縮容", "replicas")):
|
||||
_synthesized = "kubectl scale deployment -n awoooi-prod"
|
||||
elif any(w in _at_lower for w in ("logs", "日誌", "log")):
|
||||
_synthesized = "kubectl logs -n awoooi-prod --tail=100 --selector=app=awoooi-api"
|
||||
elif any(w in _at_lower for w in ("describe", "診斷", "diagnos")):
|
||||
_synthesized = "kubectl describe pods -n awoooi-prod"
|
||||
|
||||
if _synthesized:
|
||||
logger.debug(
|
||||
"solver_nemo_action_synthesized",
|
||||
action_title=action_title[:80],
|
||||
synthesized=_synthesized,
|
||||
)
|
||||
return [CandidateAction(
|
||||
action=action_title[:200],
|
||||
action=_synthesized,
|
||||
blast_radius=blast,
|
||||
rollback_cost=20,
|
||||
confidence=confidence,
|
||||
rationale=f"OpenClaw Nemo 建議: {action_title}",
|
||||
confidence=min(confidence, 0.5), # 合成指令最高 0.5,避免誤入自動執行
|
||||
rationale=f"[語意合成] Nemo 建議「{action_title[:80]}」→ 轉為 kubectl 指令",
|
||||
)]
|
||||
|
||||
# 完全無從映射 → return [](交由 _degraded_plan 輸出 category-based 調查指令)
|
||||
logger.debug(
|
||||
"solver_nemo_no_kubectl_fallback",
|
||||
action_title=action_title[:80],
|
||||
reason="action_title 無 kubectl 且語意合成失敗,降級至 _degraded_plan",
|
||||
)
|
||||
return []
|
||||
|
||||
raw = parsed.get("candidates", [])
|
||||
@@ -328,24 +366,39 @@ def _extract_candidates(parsed: dict[str, Any]) -> list[CandidateAction]:
|
||||
|
||||
|
||||
def _default_action_for_category(category: str) -> str:
|
||||
"""降級時的預設調查指令 — 必須是真實 kubectl 命令(調查優先,不執行破壞性操作)
|
||||
"""降級時的預設調查指令 — 必須是真實 kubectl/ssh 命令(調查優先,不執行破壞性操作)
|
||||
|
||||
2026-04-17 ogt + Claude Sonnet 4.6: 改為真實 kubectl 指令
|
||||
舊:自然語言如 "restart_pod"、"check_disk_usage" → 無法被 auto_approve 執行
|
||||
新:kubectl 調查指令 → 可執行,且均為唯讀操作,無副作用
|
||||
2026-04-24 ogt + Claude Sonnet 4.6: 擴展非 K8s 類別(ClickHouse/主機磁碟/DB)
|
||||
根因:SentryClickHouseMemoryPressure/HostDiskUsageHigh 類別不符任何 K8s 關鍵字
|
||||
→ 全部 fallback 到 "kubectl get pods"(無意義診斷指令)
|
||||
修復:加入 clickhouse/database/sentry/host/node/infra 類別映射
|
||||
"""
|
||||
category_lower = category.lower()
|
||||
# K8s workload 層
|
||||
if "pod" in category_lower or "kube" in category_lower or "crash" in category_lower:
|
||||
return "kubectl get pods -n awoooi-prod -o wide"
|
||||
if "disk" in category_lower or "storage" in category_lower or "pvc" in category_lower:
|
||||
return "kubectl exec -n awoooi-prod deployment/postgresql -- df -h"
|
||||
if "cpu" in category_lower or "load" in category_lower:
|
||||
return "kubectl top pods -n awoooi-prod --sort-by=cpu"
|
||||
if "memory" in category_lower or "oom" in category_lower:
|
||||
return "kubectl top pods -n awoooi-prod --sort-by=memory"
|
||||
if "network" in category_lower or "connect" in category_lower:
|
||||
return "kubectl get services -n awoooi-prod"
|
||||
return "kubectl get pods -n awoooi-prod"
|
||||
if "disk" in category_lower or "storage" in category_lower or "pvc" in category_lower:
|
||||
return "kubectl exec -n awoooi-prod deployment/postgresql -- df -h"
|
||||
# 外部服務層(非 K8s — 唯讀診斷)
|
||||
if "clickhouse" in category_lower or "sentry" in category_lower:
|
||||
return "kubectl get pods -n awoooi-prod -l app=sentry -o wide"
|
||||
if "database" in category_lower or "postgres" in category_lower or "redis" in category_lower:
|
||||
return "kubectl get pods -n awoooi-prod -l tier=database -o wide"
|
||||
if "rollback" in category_lower or "deploy" in category_lower or "version" in category_lower:
|
||||
return "kubectl rollout history deployment -n awoooi-prod"
|
||||
if "latency" in category_lower or "slow" in category_lower or "timeout" in category_lower:
|
||||
return "kubectl top pods -n awoooi-prod --sort-by=cpu"
|
||||
# 主機層(host/node/infra — 調查指令,kubectl 只查 node 資訊)
|
||||
if "host" in category_lower or "node" in category_lower or "infra" in category_lower:
|
||||
return "kubectl describe nodes | grep -A5 'Conditions\\|Allocatable'"
|
||||
return "kubectl get pods -n awoooi-prod -o wide"
|
||||
|
||||
|
||||
def compute_input_hash(diagnosis: DiagnosisReport) -> str:
|
||||
|
||||
Reference in New Issue
Block a user