fix(aiops-p0): 六大病根 P0 全面修復(ADR-092 B4)
【P0.1】knowledge_extractor_service.py:210 — AttributeError 修復 - Signal.description 欄位不存在(100% 失敗,KM 每天+5 根因) - 改用 alert_name + annotations.summary 拼接文字 【P0.2+P0.3】Gate 9+11 唯讀指令鬆綁 - blast_radius_calculator: kubectl get/top/describe/logs/version → score=1(非 50) - operation_parser: 增加 INVESTIGATE 類型識別(唯讀 kubectl 不回 None) - executor.py: OperationType 新增 INVESTIGATE enum - approval_execution.py: INVESTIGATE 路徑直接呼叫 execute_kubectl_command 【P0.4】MCP SSH/K8s Provider 修復 - decision_manager: params= → parameters=(符合 MCPToolProvider.execute 簽名) - decision_manager: MCPToolResult .get() → .success/.output(dataclass 用法) - decision_manager + ssh_provider: 補入 hosts 120/121(原 default 缺失) - auto_approve: phase2_agent_debate source bypass confidence 閾值 【P0.5】告警規則語義矛盾修復 - alert_rules.yaml: 8 條 kubectl 查詢規則 RESTART_DEPLOYMENT → NO_ACTION (CrashLoopBackOff/PostgreSQL 連線/慢查詢/MinIO 磁碟/K3s 節點/告警鏈路/SSL/CoreDNS 等) - incident_service.py: cAdvisor/CoreDNS 從 general 拆出獨立分類 【P0.6】proactive_inspector 動態基線 PromQL 全修 - 5 個 MONITORED_METRICS PromQL 全部修正(cadvisor label/datname/blackbox) - db_connection_pool: datname="awoooi" → "awoooi_prod" - http_error_rate: 無效 http_requests_total → blackbox probe_success - cpu/memory: namespace label → name=~"k8s_api_awoooi-api.*" Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -37,36 +37,46 @@ DEDUP_KEY_PREFIX = "proactive:dedup:"
|
||||
K8S_NAMESPACE = "awoooi-prod"
|
||||
|
||||
# 需要監控的 metrics(Prometheus PromQL + 警戒閾值)
|
||||
# 2026-04-24 ogt + Claude Sonnet 4.6: P0.6 修復 — 修正 PromQL labels 使其對應實際 Prometheus 資料
|
||||
# - CPU/Memory: cadvisor 無 namespace label,改用 kube_pod_container_status_restarts_total 確認存在的 namespace 篩法
|
||||
# - pod_restart_rate: 改用 sum() 聚合,避免回傳多 vector 使 _fetch_current_value 只取第一筆
|
||||
# - db_connection_pool: datname 實際為 awoooi_prod(非 awoooi)
|
||||
# - http_error_rate: cadvisor 無 http_requests_total,改用 probe_success 替代
|
||||
MONITORED_METRICS: list[dict[str, Any]] = [
|
||||
{
|
||||
"name": "http_error_rate",
|
||||
"promql": 'sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m]))',
|
||||
"threshold": 0.05, # > 5% error rate = 警戒
|
||||
"description": "HTTP 5xx 錯誤率",
|
||||
# blackbox probe 失敗率:1 - 平均探測成功率(全部 target 聚合)
|
||||
"promql": '1 - avg(blackbox_probe_success)',
|
||||
"threshold": 0.05, # > 5% probe 失敗 = 警戒
|
||||
"description": "HTTP Probe 失敗率(Blackbox Exporter)",
|
||||
},
|
||||
{
|
||||
"name": "cpu_usage_awoooi_api",
|
||||
"promql": 'avg(rate(container_cpu_usage_seconds_total{namespace="awoooi-prod",container="awoooi-api"}[5m]))',
|
||||
"threshold": 0.85, # > 85% CPU
|
||||
# cadvisor: awoooi-prod namespace 的 api container(name label 格式為 k8s_api_awoooi-api-*_awoooi-prod_*_*)
|
||||
"promql": 'avg(rate(container_cpu_usage_seconds_total{name=~"k8s_api_awoooi-api.*"}[5m]))',
|
||||
"threshold": 0.85, # > 85% CPU(單核心比例)
|
||||
"description": "API 容器 CPU 使用率",
|
||||
},
|
||||
{
|
||||
"name": "memory_usage_awoooi_api",
|
||||
"promql": 'avg(container_memory_usage_bytes{namespace="awoooi-prod",container="awoooi-api"}) / avg(container_spec_memory_limit_bytes{namespace="awoooi-prod",container="awoooi-api"})',
|
||||
"threshold": 0.90, # > 90% memory
|
||||
"description": "API 容器記憶體使用率",
|
||||
# cadvisor memory working set(不含 cache)
|
||||
"promql": 'avg(container_memory_working_set_bytes{name=~"k8s_api_awoooi-api.*"})',
|
||||
"threshold": 1073741824.0, # > 1 GiB = 警戒
|
||||
"description": "API 容器記憶體使用(working set bytes)",
|
||||
},
|
||||
{
|
||||
"name": "pod_restart_rate",
|
||||
"promql": 'increase(kube_pod_container_status_restarts_total{namespace="awoooi-prod"}[15m])',
|
||||
# kube-state-metrics: namespace=awoooi-prod,sum 聚合避免 multi-vector
|
||||
"promql": 'sum(increase(kube_pod_container_status_restarts_total{namespace="awoooi-prod"}[15m]))',
|
||||
"threshold": 2.0, # 15 分鐘內 > 2 次重啟
|
||||
"description": "Pod 重啟次數(15分鐘窗口)",
|
||||
},
|
||||
{
|
||||
"name": "db_connection_pool",
|
||||
"promql": 'pg_stat_activity_count{datname="awoooi"}',
|
||||
# datname 實際值為 awoooi_prod;sum 聚合所有 state
|
||||
"promql": 'sum(pg_stat_activity_count{datname="awoooi_prod"})',
|
||||
"threshold": 80.0, # > 80 個 DB 連線
|
||||
"description": "PostgreSQL 連線數",
|
||||
"description": "PostgreSQL 連線數(awoooi_prod)",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user