fix(ops): add docker host resource baseline guardrails

apps/api/alert_rules.yaml

@@ -163,6 +163,37 @@ rules:
       responsibility: INFRA
       reasoning: "[規則匹配] 主機層資源告警，自動 SSH 執行診斷指令（只讀，不修改），收集根因資訊後推送 Telegram 讓 SRE 決策。"
 
+  # 2026-05-05 ogt + Codex: 110/188 長時間過載事故後補 Docker Compose 過載與 restart spike 路由。
+  # 原則：過載與重啟暴增只能先診斷，禁止通用 docker restart；由 LLM + Playbook trust 決定 service-specific 修復。
+  - id: docker_baseline_overload_alert
+    priority: 44
+    description: Docker Compose 服務過載 / restart spike 基線告警（cadvisor + textfile exporter）
+    match:
+      alertname:
+        - HostLoadAverageSustainedHigh
+        - DockerContainerCpuSustainedHigh
+        - DockerContainerCpuRunawayCritical
+        - DockerContainerMemoryLimitPressure
+        - DockerContainerRestartSpike
+    response:
+      action_title: "🔍 Docker/Host 過載自動診斷 — 禁止通用重啟"
+      description: "110/188 Docker Compose 或主機 load 長時間偏離 baseline。AI 需先收集容器 CPU、restart、logs、ClickHouse/Kafka/爬蟲狀態，再選擇限流、降併發或服務專屬 playbook。"
+      suggested_action: SSH_DIAGNOSE
+      kubectl_command: "ssh {host} 'echo \"=== LOAD ===\"; uptime; echo \"=== TOP ===\"; ps aux --sort=-%cpu | head -20; echo \"=== DOCKER ===\"; docker stats --no-stream | head -40'"
+      estimated_downtime: "N/A"
+      risk: low
+      responsibility: INFRA
+      responsibility_reasoning: "Docker Compose / bare-metal 過載屬主機與平台資源治理，不能交給 K8s restart 處理"
+      secondary_teams: [BE, SRE]
+      optimization:
+        - type: BASELINE_CHECK
+          description: "比較 load5/core、單容器 CPU core、restart spike 與 24h 動態基線"
+          command: "Prometheus query: node_load5/core + rate(container_cpu_usage_seconds_total[5m]) + increase(docker_container_restart_count[15m])"
+        - type: SERVICE_SPECIFIC_REPAIR
+          description: "依服務選擇專屬修復：ClickHouse 降 merge / scheduler 限 concurrency / litellm 修 health 或路由 / exporter 降 collector"
+          command: "由 AI 根據 evidence snapshot 選擇已驗證 playbook"
+      reasoning: "[規則匹配] 長期過載先 read-only 診斷與分流，禁止通用 docker restart；修復必須服務專屬且可回寫 Playbook trust。"
+
   - id: high_cpu
     priority: 40
     description: K8s Pod/Deployment CPU 使用率過高

apps/api/src/services/proactive_inspector.py

@@ -68,20 +68,66 @@ MONITORED_METRICS: list[dict[str, Any]] = [
         # cadvisor up=0（prod-docker-188 離線），改用 node-exporter node-level CPU
         # 實測確認：avg(rate(node_cpu_seconds_total{mode!="idle"}[5m])) → 有資料
         # threshold 0.85 = 85% CPU 使用率（node level，0-1 比例）
-        "promql": 'avg(rate(node_cpu_seconds_total{mode!="idle"}[5m]))',
+        "promql": 'avg(rate(node_cpu_seconds_total{host="188",mode!="idle"}[5m]))',
         "threshold": 0.85,   # > 85% node CPU（所有 core 平均）
         "description": "Node 188 CPU 使用率（node-exporter，cadvisor 停止時替代）",
     },
+    {
+        # 2026-05-05 ogt + Codex: 110/188 CPU 過載事故後補齊 110 動態基線。
+        # Why: 事故主熱點在 110 Sentry ClickHouse/Snuba/Kafka，舊 inspector 只看未過濾的全域 CPU。
+        "name": "cpu_usage_node_110",
+        "promql": 'avg(rate(node_cpu_seconds_total{host="110",mode!="idle"}[5m]))',
+        "threshold": 0.85,
+        "description": "Node 110 CPU 使用率（node-exporter）",
+    },
     {
         # 2026-04-26 H2 hotfix: 改名 _node_188
         "name": "memory_usage_node_188",
         # cadvisor 停止，改用 node-exporter 節點記憶體使用率比例（0-1）
         # 實測確認：188 機器 62.76 GiB，當前 ~30% 使用率
         # threshold 0.85 = 85% node memory usage
-        "promql": '(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes',
+        "promql": '(node_memory_MemTotal_bytes{host="188"} - node_memory_MemAvailable_bytes{host="188"}) / node_memory_MemTotal_bytes{host="188"}',
         "threshold": 0.85,   # > 85% node memory（0-1 比例）
         "description": "Node 188 記憶體使用率（node-exporter，cadvisor 停止時替代）",
     },
+    {
+        # 2026-05-05 ogt + Codex: 110/188 CPU 過載事故後補齊 110 記憶體基線。
+        "name": "memory_usage_node_110",
+        "promql": '(node_memory_MemTotal_bytes{host="110"} - node_memory_MemAvailable_bytes{host="110"}) / node_memory_MemTotal_bytes{host="110"}',
+        "threshold": 0.85,
+        "description": "Node 110 記憶體使用率（node-exporter）",
+    },
+    {
+        # 2026-05-05 ogt + Codex: 用 load/core 比例抓長時間過載，避免 CPU% 被 runnable queue 稀釋。
+        "name": "load5_per_core_110",
+        "promql": 'node_load5{host="110"} / count(count by(cpu) (node_cpu_seconds_total{host="110",mode="idle"}))',
+        "threshold": 1.50,
+        "description": "Node 110 load5 / CPU core 比例",
+    },
+    {
+        "name": "load5_per_core_188",
+        "promql": 'node_load5{host="188"} / count(count by(cpu) (node_cpu_seconds_total{host="188",mode="idle"}))',
+        "threshold": 1.50,
+        "description": "Node 188 load5 / CPU core 比例",
+    },
+    {
+        # 2026-05-05 ogt + Codex: 補 Docker Compose 服務層基線，抓單容器長期吃超過 2 core。
+        "name": "docker_max_container_cpu_cores",
+        "promql": 'max(docker_container_cpu_cores) or max(rate(container_cpu_usage_seconds_total{name!="",id!="/"}[5m])) or on() vector(0)',
+        "threshold": 2.0,
+        "description": "Docker 單容器 CPU core 使用量上限",
+    },
+    {
+        # 2026-05-05 ogt + Codex: cAdvisor v0.47 無 container restart metric，改吃 node-exporter textfile。
+        "name": "docker_restart_spike_15m",
+        "promql": (
+            'sum(increase(docker_container_restart_count[15m])) '
+            'or sum(increase(docker_container_inspect_restart_count[15m])) '
+            'or on() vector(0)'
+        ),
+        "threshold": 5.0,
+        "description": "Docker 容器 15 分鐘重啟增量",
+    },
     {
         "name": "pod_restart_rate",
         # kube-state-metrics: namespace=awoooi-prod，sum 聚合避免 multi-vector
@@ -222,8 +268,8 @@ class ProactiveInspector:
 
     async def _inspect_dynamic_baseline(self, report: InspectionReport) -> None:
         """從 Prometheus 抓取當前值，與 Holt-Winters 基線比對。"""
-        from src.services.dynamic_baseline_service import get_dynamic_baseline_service
         from src.core.feature_flags import aiops_flags
+        from src.services.dynamic_baseline_service import get_dynamic_baseline_service
 
         if not aiops_flags.AIOPS_P4_DYNAMIC_BASELINE:
             return
@@ -273,8 +319,8 @@ class ProactiveInspector:
 
     async def _inspect_log_patterns(self, report: InspectionReport) -> None:
         """掃描 K8s Pod 日誌，偵測新 log pattern。"""
-        from src.services.log_anomaly_detector import get_log_anomaly_detector
         from src.core.feature_flags import aiops_flags
+        from src.services.log_anomaly_detector import get_log_anomaly_detector
 
         if not aiops_flags.AIOPS_P4_LOG_ANOMALY:
             return
@@ -313,8 +359,8 @@ class ProactiveInspector:
 
     async def _inspect_trends(self, report: InspectionReport) -> None:
         """對各 metric 做 4h 趨勢外推。"""
-        from src.services.trend_predictor import get_trend_predictor
         from src.core.feature_flags import aiops_flags
+        from src.services.trend_predictor import get_trend_predictor
 
         if not aiops_flags.AIOPS_P4_TREND_PREDICTOR:
             return
@@ -365,8 +411,8 @@ class ProactiveInspector:
 
     async def _retrain_baselines_background(self) -> None:
         """背景重訓所有 Holt-Winters 基線（不阻塞巡檢）。"""
-        from src.services.dynamic_baseline_service import get_dynamic_baseline_service
         from src.core.feature_flags import aiops_flags
+        from src.services.dynamic_baseline_service import get_dynamic_baseline_service
 
         if not aiops_flags.AIOPS_P4_DYNAMIC_BASELINE:
             return
@@ -392,6 +438,7 @@ class ProactiveInspector:
     async def _fetch_current_value(self, promql: str) -> float | None:
         """從 Prometheus 抓取當前值（instant query）。"""
         import httpx
+
         from src.core.config import settings
 
         try: