fix(ops): monitor systemd runners in host baseline
Some checks failed
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
Code Review / ai-code-review (push) Has been cancelled
Deploy Alert Rules / Deploy Prometheus Alert Rules (push) Successful in 39s

This commit is contained in:
Your Name
2026-05-05 14:08:43 +08:00
parent 8e22110030
commit fe618960a8
5 changed files with 297 additions and 10 deletions

View File

@@ -128,6 +128,20 @@ MONITORED_METRICS: list[dict[str, Any]] = [
"threshold": 5.0,
"description": "Docker 容器 15 分鐘重啟增量",
},
{
# 2026-05-05 ogt + Codex: 110 runner watchdog loop was invisible to Docker metrics.
"name": "systemd_runner_restart_spike_15m",
"promql": 'sum(increase(systemd_unit_restarts_total{unit=~"actions\\\\.runner\\\\..*"}[15m])) or on() vector(0)',
"threshold": 2.0,
"description": "Systemd GitHub/Gitea runner 15 分鐘重啟增量",
},
{
# 2026-05-05 ogt + Codex: GitHub runner service should not use systemd WatchdogSec.
"name": "systemd_runner_watchdog_enabled",
"promql": 'max(systemd_unit_watchdog_seconds{unit=~"actions\\\\.runner\\\\..*"}) or on() vector(0)',
"threshold": 0.0,
"description": "Systemd runner WatchdogSec 是否啟用",
},
{
"name": "pod_restart_rate",
# kube-state-metrics: namespace=awoooi-prodsum 聚合避免 multi-vector