fix(ops): route systemd runner baseline alerts
This commit is contained in:
@@ -194,6 +194,35 @@ rules:
|
||||
command: "由 AI 根據 evidence snapshot 選擇已驗證 playbook"
|
||||
reasoning: "[規則匹配] 長期過載先 read-only 診斷與分流,禁止通用 docker restart;修復必須服務專屬且可回寫 Playbook trust。"
|
||||
|
||||
# 2026-05-05 ogt + Codex: 110 self-hosted runner 是 systemd service,不在 Docker/cAdvisor 覆蓋內。
|
||||
# 原則:AI 可自動診斷 watchdog/quota/restart storm;套用 systemd drop-in 需要 sudo,必須走人工批准或 sudo playbook。
|
||||
- id: systemd_runner_baseline_alert
|
||||
priority: 43
|
||||
description: 110 self-hosted runner systemd watchdog / restart / quota 基線告警
|
||||
match:
|
||||
alertname:
|
||||
- SystemdRunnerRestartSpike
|
||||
- SystemdRunnerWatchdogEnabled
|
||||
- SystemdRunnerMissingResourceQuota
|
||||
response:
|
||||
action_title: "🔍 Systemd Runner 基線診斷 — 需要 sudo 才可修復"
|
||||
description: "110 self-hosted runner 發生 watchdog/restart storm 或缺 CPU/Memory quota。這會讓 CI 與 Sentry/ClickHouse/Gitea 搶主機資源,且 Docker/cAdvisor 看不到。"
|
||||
suggested_action: SSH_DIAGNOSE
|
||||
kubectl_command: "ssh {host} 'systemctl show {unit} -p WatchdogUSec -p NRestarts -p DropInPaths -p CPUQuotaPerSecUSec -p MemoryMax -p ActiveState -p SubState; journalctl -u {unit} --since \"20 minutes ago\" --no-pager | tail -120'"
|
||||
estimated_downtime: "N/A"
|
||||
risk: low
|
||||
responsibility: INFRA
|
||||
responsibility_reasoning: "self-hosted runner 是 bare-metal systemd 資源治理,非 K8s 或 Docker workload"
|
||||
secondary_teams: [SRE]
|
||||
optimization:
|
||||
- type: SYSTEMD_GUARDRAIL
|
||||
description: "人工批准後停用錯誤 watchdog drop-in,並為 runner 加 CPUQuota=200%、MemoryMax=2G"
|
||||
command: "bash scripts/ops/apply-runner-systemd-guardrails.sh --apply"
|
||||
- type: CI_CAPACITY
|
||||
description: "若 110 同時承載 Sentry/ClickHouse/Gitea,不應讓多個 runner 無限制並行"
|
||||
command: "檢查 active jobs、runner 數量與 Gitea Actions concurrency,必要時分流 runner"
|
||||
reasoning: "[規則匹配] systemd runner 過載先 read-only 診斷;改 systemd drop-in 需 sudo 與人工批准,避免 AI 擅自改 host unit。"
|
||||
|
||||
- id: high_cpu
|
||||
priority: 40
|
||||
description: K8s Pod/Deployment CPU 使用率過高
|
||||
|
||||
Reference in New Issue
Block a user