fix(ops): add docker host resource baseline guardrails

ops/monitoring/alerts-unified.yml

@@ -48,6 +48,25 @@ groups:
           summary: "主機 {{ $labels.host }} CPU 高負載"
           description: "CPU 使用率超過 80%"
 
+      - alert: HostLoadAverageSustainedHigh
+        # 2026-05-05 ogt + Codex: 110/188 長時間過載基線。
+        # Why: CPU% 只看 busy time，無法充分表達 ClickHouse merge、Kafka 回補、Chrome/Ollama 造成的 runnable queue。
+        expr: node_load5{host=~"110|188"} / on(host) count by(host) (count by(host,cpu) (node_cpu_seconds_total{mode="idle",host=~"110|188"})) > 1.5
+        for: 15m
+        labels:
+          severity: critical
+          layer: systemd-188
+          team: ops
+          auto_repair: "true"
+          mcp_provider: "ssh_host"
+          host_type: "bare_metal"
+          alert_category: "host_resource"
+        annotations:
+          summary: "主機 {{ $labels.host }} load5/core 長時間過高"
+          description: "load5 / CPU core > 1.5 持續 15 分鐘；這通常代表 runnable queue 已長期塞車，不是短暫尖峰。"
+          auto_repair_action: "ssh {{ $labels.instance }} 'echo \"=== LOAD ===\"; uptime; echo \"=== CPU TOP ===\"; ps aux --sort=-%cpu | head -25; echo \"=== DOCKER STATS ===\"; docker stats --no-stream | head -40'"
+          runbook: "先判斷高 load 來源：ClickHouse merge / Kafka backlog / Chrome scraper / Ollama runner / exporter；只允許 read-only 診斷，自動修復需走服務專屬 playbook。"
+
       - alert: HostOutOfMemory
         expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
         for: 5m
@@ -576,6 +595,74 @@ groups:
           summary: "容器 {{ $labels.container }} 已停止"
           description: "主機 {{ $labels.host }} 容器 {{ $labels.container }} 已 exited/dead，持續 1 分鐘"
 
+      - alert: DockerContainerCpuSustainedHigh
+        # 2026-05-05 ogt + Codex: Docker Compose 長期過載基線。
+        # Baseline: 單容器 >2 core 10m 為 warning；用於提早抓 cadvisor、ClickHouse、momo-scheduler、Ollama runner 類問題。
+        expr: docker_container_cpu_cores > 2
+        for: 10m
+        labels:
+          severity: warning
+          layer: docker
+          team: ops
+          alert_category: infrastructure
+          notification_type: TYPE-1
+          auto_repair: "true"
+        annotations:
+          summary: "容器 {{ $labels.container_name }} CPU 持續超過 2 core"
+          description: "{{ $labels.container_name }} 5m CPU 使用 >2 core 且持續 10 分鐘，需檢查是否缺 cpus limit、merge backlog、爬蟲尖峰或模型推理。"
+          auto_repair_action: "ssh {{ $labels.instance }} 'docker stats --no-stream {{ $labels.container_name }}; docker inspect {{ $labels.container_name }} | head -80'"
+          runbook: "自動階段只做診斷；若是已知服務，交給 service-specific playbook：ClickHouse 降 merge/查 backlog、scheduler 限 concurrency、litellm/blackbox 查 liveliness。"
+
+      - alert: DockerContainerCpuRunawayCritical
+        expr: docker_container_cpu_cores > 4
+        for: 15m
+        labels:
+          severity: critical
+          layer: docker
+          team: ops
+          alert_category: infrastructure
+          notification_type: TYPE-3
+          auto_repair: "true"
+        annotations:
+          summary: "容器 {{ $labels.container_name }} CPU 持續超過 4 core"
+          description: "{{ $labels.container_name }} 已持續吃超過 4 core，會拖垮 110/188 主機；需要 AI 判斷是否限流、暫停 worker 或套用既有 playbook。"
+          auto_repair_action: "ssh {{ $labels.instance }} 'echo \"=== CONTAINER ===\"; docker stats --no-stream {{ $labels.container_name }}; echo \"=== TOP ===\"; docker top {{ $labels.container_name }} -eo pid,ppid,stat,pcpu,pmem,comm,args | head -30'"
+          runbook: "禁止通用 docker restart；先抓根因，只有 health down 或 crash loop 才可走重啟。"
+
+      - alert: DockerContainerMemoryLimitPressure
+        # 2026-05-05 ogt + Codex: 防止亂設 memory limit 把 Kafka/Gitea/Taskworker 類服務卡死。
+        expr: docker_container_memory_limit_bytes > 0 and docker_container_memory_usage_bytes / docker_container_memory_limit_bytes > 0.85
+        for: 10m
+        labels:
+          severity: warning
+          layer: docker
+          team: ops
+          alert_category: infrastructure
+          notification_type: TYPE-1
+          auto_repair: "true"
+        annotations:
+          summary: "容器 {{ $labels.container_name }} 記憶體超過 limit 85%"
+          description: "{{ $labels.container_name }} memory / limit > 85% 持續 10 分鐘。若是 Kafka/ClickHouse/Gitea/Taskworker，需先判斷 workload，不可直接降 limit。"
+          auto_repair_action: "ssh {{ $labels.instance }} 'docker stats --no-stream {{ $labels.container_name }}; docker inspect {{ $labels.container_name }} | head -80'"
+          runbook: "若服務已接近 limit：優先調整 retention/concurrency/cache，再評估提高 memory；禁止用更低 memory limit 當止血。"
+
+      - alert: DockerContainerRestartSpike
+        # 2026-05-05 ogt + Codex: cAdvisor v0.47 無 restart metric，吃 node-exporter textfile docker_container_restart_count。
+        expr: increase(docker_container_restart_count[15m]) > 5 or increase(docker_container_inspect_restart_count[15m]) > 5
+        for: 3m
+        labels:
+          severity: critical
+          layer: docker
+          team: ops
+          alert_category: infrastructure
+          notification_type: TYPE-3
+          auto_repair: "true"
+        annotations:
+          summary: "容器 {{ $labels.container_name }} 15 分鐘重啟超過 5 次"
+          description: "Docker restart textfile exporter 顯示 {{ $labels.container_name }} 重啟暴增，避免再次出現 litellm 24,464 次靜默崩潰。"
+          auto_repair_action: "ssh {{ $labels.instance }} 'docker ps -a --filter name={{ $labels.container_name }}; docker logs --tail=120 {{ $labels.container_name }}'"
+          runbook: "先抓 crash signature；若是 config/DB/網路問題，修設定，不用無限 restart。"
+
   # =========================================================================
   # MinIO / Kali 告警
   # =========================================================================

ops/monitoring/alerts.yml

@@ -51,6 +51,25 @@ groups:
           auto_repair_action: "ssh {{ $labels.instance }} 'ps aux --sort=-%cpu | head -20' (host CPU 診斷；禁 kubectl restart awoooi-* — 主因常為第三方服務 Sentry/ClickHouse/Snuba)"
           runbook: "host CPU 高負載排查：先 SSH ps aux 看 top 進程；若為第三方服務（Sentry/ClickHouse 等）寫 ADR 升級資源或調 limit，禁止 kubectl restart 跨 domain"
 
+      - alert: HostLoadAverageSustainedHigh
+        # 2026-05-05 ogt + Codex: 110/188 長時間過載基線。
+        # Why: CPU% 只看 busy time，無法充分表達 ClickHouse merge、Kafka 回補、Chrome/Ollama 造成的 runnable queue。
+        expr: node_load5{host=~"110|188"} / on(host) count by(host) (count by(host,cpu) (node_cpu_seconds_total{mode="idle",host=~"110|188"})) > 1.5
+        for: 15m
+        labels:
+          severity: critical
+          layer: systemd-188
+          team: ops
+          auto_repair: "true"
+          mcp_provider: "ssh_host"
+          host_type: "bare_metal"
+          alert_category: "host_resource"
+        annotations:
+          summary: "主機 {{ $labels.host }} load5/core 長時間過高"
+          description: "load5 / CPU core > 1.5 持續 15 分鐘；這通常代表 runnable queue 已長期塞車，不是短暫尖峰。"
+          auto_repair_action: "ssh {{ $labels.instance }} 'echo \"=== LOAD ===\"; uptime; echo \"=== CPU TOP ===\"; ps aux --sort=-%cpu | head -25; echo \"=== DOCKER STATS ===\"; docker stats --no-stream | head -40'"
+          runbook: "先判斷高 load 來源：ClickHouse merge / Kafka backlog / Chrome scraper / Ollama runner / exporter；只允許 read-only 診斷，自動修復需走服務專屬 playbook。"
+
       - alert: HostOutOfMemory
         expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
         for: 5m
@@ -582,6 +601,74 @@ groups:
           summary: "容器 {{ $labels.container }} 已停止"
           description: "主機 {{ $labels.host }} 容器 {{ $labels.container }} 已 exited/dead，持續 1 分鐘"
 
+      - alert: DockerContainerCpuSustainedHigh
+        # 2026-05-05 ogt + Codex: Docker Compose 長期過載基線。
+        # Baseline: 單容器 >2 core 10m 為 warning；用於提早抓 cadvisor、ClickHouse、momo-scheduler、Ollama runner 類問題。
+        expr: docker_container_cpu_cores > 2
+        for: 10m
+        labels:
+          severity: warning
+          layer: docker
+          team: ops
+          alert_category: infrastructure
+          notification_type: TYPE-1
+          auto_repair: "true"
+        annotations:
+          summary: "容器 {{ $labels.container_name }} CPU 持續超過 2 core"
+          description: "{{ $labels.container_name }} 5m CPU 使用 >2 core 且持續 10 分鐘，需檢查是否缺 cpus limit、merge backlog、爬蟲尖峰或模型推理。"
+          auto_repair_action: "ssh {{ $labels.instance }} 'docker stats --no-stream {{ $labels.container_name }}; docker inspect {{ $labels.container_name }} | head -80'"
+          runbook: "自動階段只做診斷；若是已知服務，交給 service-specific playbook：ClickHouse 降 merge/查 backlog、scheduler 限 concurrency、litellm/blackbox 查 liveliness。"
+
+      - alert: DockerContainerCpuRunawayCritical
+        expr: docker_container_cpu_cores > 4
+        for: 15m
+        labels:
+          severity: critical
+          layer: docker
+          team: ops
+          alert_category: infrastructure
+          notification_type: TYPE-3
+          auto_repair: "true"
+        annotations:
+          summary: "容器 {{ $labels.container_name }} CPU 持續超過 4 core"
+          description: "{{ $labels.container_name }} 已持續吃超過 4 core，會拖垮 110/188 主機；需要 AI 判斷是否限流、暫停 worker 或套用既有 playbook。"
+          auto_repair_action: "ssh {{ $labels.instance }} 'echo \"=== CONTAINER ===\"; docker stats --no-stream {{ $labels.container_name }}; echo \"=== TOP ===\"; docker top {{ $labels.container_name }} -eo pid,ppid,stat,pcpu,pmem,comm,args | head -30'"
+          runbook: "禁止通用 docker restart；先抓根因，只有 health down 或 crash loop 才可走重啟。"
+
+      - alert: DockerContainerMemoryLimitPressure
+        # 2026-05-05 ogt + Codex: 防止亂設 memory limit 把 Kafka/Gitea/Taskworker 類服務卡死。
+        expr: docker_container_memory_limit_bytes > 0 and docker_container_memory_usage_bytes / docker_container_memory_limit_bytes > 0.85
+        for: 10m
+        labels:
+          severity: warning
+          layer: docker
+          team: ops
+          alert_category: infrastructure
+          notification_type: TYPE-1
+          auto_repair: "true"
+        annotations:
+          summary: "容器 {{ $labels.container_name }} 記憶體超過 limit 85%"
+          description: "{{ $labels.container_name }} memory / limit > 85% 持續 10 分鐘。若是 Kafka/ClickHouse/Gitea/Taskworker，需先判斷 workload，不可直接降 limit。"
+          auto_repair_action: "ssh {{ $labels.instance }} 'docker stats --no-stream {{ $labels.container_name }}; docker inspect {{ $labels.container_name }} | head -80'"
+          runbook: "若服務已接近 limit：優先調整 retention/concurrency/cache，再評估提高 memory；禁止用更低 memory limit 當止血。"
+
+      - alert: DockerContainerRestartSpike
+        # 2026-05-05 ogt + Codex: cAdvisor v0.47 無 restart metric，吃 node-exporter textfile docker_container_restart_count。
+        expr: increase(docker_container_restart_count[15m]) > 5 or increase(docker_container_inspect_restart_count[15m]) > 5
+        for: 3m
+        labels:
+          severity: critical
+          layer: docker
+          team: ops
+          alert_category: infrastructure
+          notification_type: TYPE-3
+          auto_repair: "true"
+        annotations:
+          summary: "容器 {{ $labels.container_name }} 15 分鐘重啟超過 5 次"
+          description: "Docker restart textfile exporter 顯示 {{ $labels.container_name }} 重啟暴增，避免再次出現 litellm 24,464 次靜默崩潰。"
+          auto_repair_action: "ssh {{ $labels.instance }} 'docker ps -a --filter name={{ $labels.container_name }}; docker logs --tail=120 {{ $labels.container_name }}'"
+          runbook: "先抓 crash signature；若是 config/DB/網路問題，修設定，不用無限 restart。"
+
   # =========================================================================
   # MinIO / Kali 告警
   # =========================================================================