wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(ops): add docker host resource baseline guardrails
Some checks failed
CD Pipeline / tests (push) Failing after 1m50s
Details
CD Pipeline / build-and-deploy (push) Has been skipped
Details
CD Pipeline / post-deploy-checks (push) Has been skipped
Details
Code Review / ai-code-review (push) Successful in 25s
Details
Deploy Alert Rules / Deploy Prometheus Alert Rules (push) Successful in 38s
Details

Browse Source
This commit is contained in:
Your Name
2026-05-05 13:45:09 +08:00
parent a57e3d3d75
commit e8e6748f70
9 changed files with 586 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
apps/api/src/services/proactive_inspector.py
View File

@@ -68,20 +68,66 @@ MONITORED_METRICS: list[dict[str, Any]] = [
# cadvisor up=0（prod-docker-188 離線），改用 node-exporter node-level CPU
# 實測確認：avg(rate(node_cpu_seconds_total{mode!="idle"}[5m])) → 有資料
# threshold 0.85 = 85% CPU 使用率（node level，0-1 比例）
"promql": 'avg(rate(node_cpu_seconds_total{mode!="idle"}[5m]))',
"promql": 'avg(rate(node_cpu_seconds_total{host="188",mode!="idle"}[5m]))',
"threshold": 0.85, # > 85% node CPU（所有 core 平均）
"description": "Node 188 CPU 使用率（node-exporter，cadvisor 停止時替代）",
},
{
# 2026-05-05 ogt + Codex: 110/188 CPU 過載事故後補齊 110 動態基線。
# Why: 事故主熱點在 110 Sentry ClickHouse/Snuba/Kafka，舊 inspector 只看未過濾的全域 CPU。
"name": "cpu_usage_node_110",
"promql": 'avg(rate(node_cpu_seconds_total{host="110",mode!="idle"}[5m]))',
"threshold": 0.85,
"description": "Node 110 CPU 使用率（node-exporter）",
},
{
# 2026-04-26 H2 hotfix: 改名 _node_188
"name": "memory_usage_node_188",
# cadvisor 停止，改用 node-exporter 節點記憶體使用率比例（0-1）
# 實測確認：188 機器 62.76 GiB，當前 ~30% 使用率
# threshold 0.85 = 85% node memory usage
"promql": '(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes',
"promql": '(node_memory_MemTotal_bytes{host="188"} - node_memory_MemAvailable_bytes{host="188"}) / node_memory_MemTotal_bytes{host="188"}',
"threshold": 0.85, # > 85% node memory（0-1 比例）
"description": "Node 188 記憶體使用率（node-exporter，cadvisor 停止時替代）",
},
{
# 2026-05-05 ogt + Codex: 110/188 CPU 過載事故後補齊 110 記憶體基線。
"name": "memory_usage_node_110",
"promql": '(node_memory_MemTotal_bytes{host="110"} - node_memory_MemAvailable_bytes{host="110"}) / node_memory_MemTotal_bytes{host="110"}',
"threshold": 0.85,
"description": "Node 110 記憶體使用率（node-exporter）",
},
{
# 2026-05-05 ogt + Codex: 用 load/core 比例抓長時間過載，避免 CPU% 被 runnable queue 稀釋。
"name": "load5_per_core_110",
"promql": 'node_load5{host="110"} / count(count by(cpu) (node_cpu_seconds_total{host="110",mode="idle"}))',
"threshold": 1.50,
"description": "Node 110 load5 / CPU core 比例",
},
{
"name": "load5_per_core_188",
"promql": 'node_load5{host="188"} / count(count by(cpu) (node_cpu_seconds_total{host="188",mode="idle"}))',
"threshold": 1.50,
"description": "Node 188 load5 / CPU core 比例",
},
{
# 2026-05-05 ogt + Codex: 補 Docker Compose 服務層基線，抓單容器長期吃超過 2 core。
"name": "docker_max_container_cpu_cores",
"promql": 'max(docker_container_cpu_cores) or max(rate(container_cpu_usage_seconds_total{name!="",id!="/"}[5m])) or on() vector(0)',
"threshold": 2.0,
"description": "Docker 單容器 CPU core 使用量上限",
},
{
# 2026-05-05 ogt + Codex: cAdvisor v0.47 無 container restart metric，改吃 node-exporter textfile。
"name": "docker_restart_spike_15m",
"promql": (
'sum(increase(docker_container_restart_count[15m])) '
'or sum(increase(docker_container_inspect_restart_count[15m])) '
'or on() vector(0)'
),
"threshold": 5.0,
"description": "Docker 容器 15 分鐘重啟增量",
},
{
"name": "pod_restart_rate",
# kube-state-metrics: namespace=awoooi-prod，sum 聚合避免 multi-vector
@@ -222,8 +268,8 @@ class ProactiveInspector:
async def _inspect_dynamic_baseline(self, report: InspectionReport) -> None:
"""從 Prometheus 抓取當前值，與 Holt-Winters 基線比對。"""
from src.services.dynamic_baseline_service import get_dynamic_baseline_service
from src.core.feature_flags import aiops_flags
from src.services.dynamic_baseline_service import get_dynamic_baseline_service
if not aiops_flags.AIOPS_P4_DYNAMIC_BASELINE:
return
@@ -273,8 +319,8 @@ class ProactiveInspector:
async def _inspect_log_patterns(self, report: InspectionReport) -> None:
"""掃描 K8s Pod 日誌，偵測新 log pattern。"""
from src.services.log_anomaly_detector import get_log_anomaly_detector
from src.core.feature_flags import aiops_flags
from src.services.log_anomaly_detector import get_log_anomaly_detector
if not aiops_flags.AIOPS_P4_LOG_ANOMALY:
return
@@ -313,8 +359,8 @@ class ProactiveInspector:
async def _inspect_trends(self, report: InspectionReport) -> None:
"""對各 metric 做 4h 趨勢外推。"""
from src.services.trend_predictor import get_trend_predictor
from src.core.feature_flags import aiops_flags
from src.services.trend_predictor import get_trend_predictor
if not aiops_flags.AIOPS_P4_TREND_PREDICTOR:
return
@@ -365,8 +411,8 @@ class ProactiveInspector:
async def _retrain_baselines_background(self) -> None:
"""背景重訓所有 Holt-Winters 基線（不阻塞巡檢）。"""
from src.services.dynamic_baseline_service import get_dynamic_baseline_service
from src.core.feature_flags import aiops_flags
from src.services.dynamic_baseline_service import get_dynamic_baseline_service
if not aiops_flags.AIOPS_P4_DYNAMIC_BASELINE:
return
@@ -392,6 +438,7 @@ class ProactiveInspector:
async def _fetch_current_value(self, promql: str) -> float | None:
"""從 Prometheus 抓取當前值（instant query）。"""
import httpx
from src.core.config import settings
try: