From fe618960a8e8726d15bdc1ba859a37f299c41344 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 5 May 2026 14:08:43 +0800 Subject: [PATCH] fix(ops): monitor systemd runners in host baseline --- apps/api/src/services/proactive_inspector.py | 14 ++ .../HOST-RESOURCE-BASELINE-110-188.md | 28 ++++ ops/monitoring/alerts-unified.yml | 59 ++++++- ops/monitoring/alerts.yml | 59 ++++++- .../ops/systemd-units-textfile-exporter.py | 147 ++++++++++++++++++ 5 files changed, 297 insertions(+), 10 deletions(-) create mode 100644 scripts/ops/systemd-units-textfile-exporter.py diff --git a/apps/api/src/services/proactive_inspector.py b/apps/api/src/services/proactive_inspector.py index ba49df7b0..73fefc94e 100644 --- a/apps/api/src/services/proactive_inspector.py +++ b/apps/api/src/services/proactive_inspector.py @@ -128,6 +128,20 @@ MONITORED_METRICS: list[dict[str, Any]] = [ "threshold": 5.0, "description": "Docker 容器 15 分鐘重啟增量", }, + { + # 2026-05-05 ogt + Codex: 110 runner watchdog loop was invisible to Docker metrics. + "name": "systemd_runner_restart_spike_15m", + "promql": 'sum(increase(systemd_unit_restarts_total{unit=~"actions\\\\.runner\\\\..*"}[15m])) or on() vector(0)', + "threshold": 2.0, + "description": "Systemd GitHub/Gitea runner 15 分鐘重啟增量", + }, + { + # 2026-05-05 ogt + Codex: GitHub runner service should not use systemd WatchdogSec. + "name": "systemd_runner_watchdog_enabled", + "promql": 'max(systemd_unit_watchdog_seconds{unit=~"actions\\\\.runner\\\\..*"}) or on() vector(0)', + "threshold": 0.0, + "description": "Systemd runner WatchdogSec 是否啟用", + }, { "name": "pod_restart_rate", # kube-state-metrics: namespace=awoooi-prod，sum 聚合避免 multi-vector diff --git a/docs/runbooks/HOST-RESOURCE-BASELINE-110-188.md b/docs/runbooks/HOST-RESOURCE-BASELINE-110-188.md index 31a23eb5c..657844887 100644 --- a/docs/runbooks/HOST-RESOURCE-BASELINE-110-188.md +++ b/docs/runbooks/HOST-RESOURCE-BASELINE-110-188.md @@ -15,6 +15,7 @@ | Sentry taskbroker | 1 CPU / 512 MiB | ~70-98% CPU / 160 MiB | CPU is tight; increasing may improve backlog but can raise host load. | | Sentry taskscheduler | 0.5 CPU / 512 MiB | ~13% CPU / 387 MiB (76%) | Memory is tight; alert at 85% before it stalls. | | Gitea | 3 CPU / 3 GiB | ~4% CPU / 2.18 GiB (73%) | Good cap; memory headroom is not huge. | +| GitHub/Gitea runners | unlimited systemd services | one runner had WatchdogSec=5min and 8,490 restarts; `act` CI containers caused load spikes | Must be monitored outside Docker. Remove bad watchdog drop-in and apply per-runner CPU/Memory quotas. | | node-exporter | 1 CPU / 256 MiB | ~0-5% CPU / 8 MiB | Good after disabling expensive `arp`, `netclass`, and `netdev` collectors. | | cadvisor | 1 CPU / 512 MiB | ~0% CPU / 27 MiB | Good safety cap. | | Harbor / Langfuse / Prometheus / Grafana / Nginx | mostly unlimited | currently low | Needs staged limits, but not during Sentry backlog drain. | @@ -42,6 +43,9 @@ Use these thresholds for alerting and AI triage: | Docker container CPU | > 4 cores for 15m | Critical diagnosis; never generic restart. | | Docker memory / limit | > 85% for 10m | Warning; raise memory or reduce workload, never lower the limit. | | Docker restarts | > 5 in 15m | Critical; pull logs and fix crash signature. | +| Systemd runner restarts | > 2 in 15m | Critical; inspect watchdog/drop-ins and active CI jobs. | +| Systemd runner WatchdogSec | > 0 for 10m | Warning; GitHub Actions runner should not be killed by systemd watchdog. | +| Systemd runner quota | CPU or memory unlimited for 30m | Warning; apply CPUQuota/MemoryMax or move CI away from Sentry host. | ## Rules @@ -53,6 +57,8 @@ Use these thresholds for alerting and AI triage: 6. For monitoring tools, caps are required, but every cap must be paired with self-monitoring. 7. Every Docker Compose host must emit `docker_container_cpu_cores`, `docker_container_memory_*`, and a restart counter via node-exporter textfile. 8. Disable node-exporter collectors that are slow or failing on each host; exporter scrape time is part of the resource baseline. +9. Runner services are part of the resource baseline even when jobs run outside Docker. Export `systemd_unit_*` metrics for all `actions.runner.*` services. +10. CI runner limits must be set at the systemd unit level. Docker container limits do not protect host load from runner listener, worker, and `act` helper processes. ## Next Safe Rollout Order @@ -62,6 +68,27 @@ Use these thresholds for alerting and AI triage: 4. Tune `momo-scheduler` crawler concurrency on 188; keep 2 CPU / 2 GiB until success rate and latency prove it is too low. 5. Fix 188 Elephant Alpha/OpenClaw allowed-action drift before enabling resource auto-repair beyond diagnosis. 6. Add modest caps to currently unlimited low-risk services in small batches. +7. Fix 110 runner services with sudo-capable host maintenance: + +```bash +unit=actions.runner.owenhytsai-awoooi.awoooi-110.service +sudo mv /etc/systemd/system/$unit.d/watchdog.conf /etc/systemd/system/$unit.d/watchdog.conf.disabled-20260505 + +for u in \ + actions.runner.owenhytsai-awoooi.awoooi-110.service \ + actions.runner.owenhytsai-awoooi.awoooi-110-3.service \ + actions.runner.owenhytsai-wooo-aiops.wooo-110-runner-2.service \ + actions.runner.owenhytsai-wooo-aiops.wooo-110-runner-3.service \ + actions.runner.owenhytsai-wooo-aiops.wooo-runner-110.service +do + sudo mkdir -p /etc/systemd/system/$u.d + printf "[Service]\nCPUAccounting=yes\nCPUQuota=200%%\nMemoryAccounting=yes\nMemoryMax=2G\n" \ + | sudo tee /etc/systemd/system/$u.d/resource-guard.conf +done + +sudo systemctl daemon-reload +sudo systemctl restart actions.runner.owenhytsai-awoooi.awoooi-110.service +``` ## Known Anti-Patterns @@ -70,3 +97,4 @@ Use these thresholds for alerting and AI triage: - Capping Kafka below current working set during backlog recovery. - Treating "no alert" as healthy when cAdvisor or textfile exporters are missing. - Letting monitoring collectors spend seconds per scrape; this turns observability into load. +- Leaving self-hosted runners unlimited on the same host as Sentry/ClickHouse/Gitea. diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index cc4decc84..e39f990a6 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -64,7 +64,7 @@ groups: annotations: summary: "主機 {{ $labels.host }} load5/core 長時間過高" description: "load5 / CPU core > 1.5 持續 15 分鐘；這通常代表 runnable queue 已長期塞車，不是短暫尖峰。" - auto_repair_action: "ssh {{ $labels.instance }} 'echo \"=== LOAD ===\"; uptime; echo \"=== CPU TOP ===\"; ps aux --sort=-%cpu | head -25; echo \"=== DOCKER STATS ===\"; docker stats --no-stream | head -40'" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'echo \"=== LOAD ===\"; uptime; echo \"=== CPU TOP ===\"; ps aux --sort=-%cpu | head -25; echo \"=== DOCKER STATS ===\"; docker stats --no-stream | head -40'" runbook: "先判斷高 load 來源：ClickHouse merge / Kafka backlog / Chrome scraper / Ollama runner / exporter；只允許 read-only 診斷，自動修復需走服務專屬 playbook。" - alert: HostOutOfMemory @@ -610,7 +610,7 @@ groups: annotations: summary: "容器 {{ $labels.container_name }} CPU 持續超過 2 core" description: "{{ $labels.container_name }} 5m CPU 使用 >2 core 且持續 10 分鐘，需檢查是否缺 cpus limit、merge backlog、爬蟲尖峰或模型推理。" - auto_repair_action: "ssh {{ $labels.instance }} 'docker stats --no-stream {{ $labels.container_name }}; docker inspect {{ $labels.container_name }} | head -80'" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'docker stats --no-stream {{ $labels.container_name }}; docker inspect {{ $labels.container_name }} | head -80'" runbook: "自動階段只做診斷；若是已知服務，交給 service-specific playbook：ClickHouse 降 merge/查 backlog、scheduler 限 concurrency、litellm/blackbox 查 liveliness。" - alert: DockerContainerCpuRunawayCritical @@ -626,7 +626,7 @@ groups: annotations: summary: "容器 {{ $labels.container_name }} CPU 持續超過 4 core" description: "{{ $labels.container_name }} 已持續吃超過 4 core，會拖垮 110/188 主機；需要 AI 判斷是否限流、暫停 worker 或套用既有 playbook。" - auto_repair_action: "ssh {{ $labels.instance }} 'echo \"=== CONTAINER ===\"; docker stats --no-stream {{ $labels.container_name }}; echo \"=== TOP ===\"; docker top {{ $labels.container_name }} -eo pid,ppid,stat,pcpu,pmem,comm,args | head -30'" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'echo \"=== CONTAINER ===\"; docker stats --no-stream {{ $labels.container_name }}; echo \"=== TOP ===\"; docker top {{ $labels.container_name }} -eo pid,ppid,stat,pcpu,pmem,comm,args | head -30'" runbook: "禁止通用 docker restart；先抓根因，只有 health down 或 crash loop 才可走重啟。" - alert: DockerContainerMemoryLimitPressure @@ -643,7 +643,7 @@ groups: annotations: summary: "容器 {{ $labels.container_name }} 記憶體超過 limit 85%" description: "{{ $labels.container_name }} memory / limit > 85% 持續 10 分鐘。若是 Kafka/ClickHouse/Gitea/Taskworker，需先判斷 workload，不可直接降 limit。" - auto_repair_action: "ssh {{ $labels.instance }} 'docker stats --no-stream {{ $labels.container_name }}; docker inspect {{ $labels.container_name }} | head -80'" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'docker stats --no-stream {{ $labels.container_name }}; docker inspect {{ $labels.container_name }} | head -80'" runbook: "若服務已接近 limit：優先調整 retention/concurrency/cache，再評估提高 memory；禁止用更低 memory limit 當止血。" - alert: DockerContainerRestartSpike @@ -660,9 +660,58 @@ groups: annotations: summary: "容器 {{ $labels.container_name }} 15 分鐘重啟超過 5 次" description: "Docker restart textfile exporter 顯示 {{ $labels.container_name }} 重啟暴增，避免再次出現 litellm 24,464 次靜默崩潰。" - auto_repair_action: "ssh {{ $labels.instance }} 'docker ps -a --filter name={{ $labels.container_name }}; docker logs --tail=120 {{ $labels.container_name }}'" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'docker ps -a --filter name={{ $labels.container_name }}; docker logs --tail=120 {{ $labels.container_name }}'" runbook: "先抓 crash signature；若是 config/DB/網路問題，修設定，不用無限 restart。" + - alert: SystemdRunnerRestartSpike + # 2026-05-05 ogt + Codex: 110 GitHub Actions runner watchdog loop was outside Docker/cAdvisor coverage. + expr: increase(systemd_unit_restarts_total{unit=~"actions\\.runner\\..*"}[15m]) > 2 + for: 3m + labels: + severity: critical + layer: systemd-110 + team: ops + alert_category: infrastructure + notification_type: TYPE-3 + auto_repair: "true" + annotations: + summary: "Systemd runner {{ $labels.unit }} 15 分鐘重啟超過 2 次" + description: "{{ $labels.unit }} 在 15 分鐘內重啟暴增；110 曾發生 WatchdogSec=5min 造成 runner 每 5 分鐘自殺重啟。" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'systemctl show {{ $labels.unit }} -p WatchdogUSec -p NRestarts -p DropInPaths -p CPUQuotaPerSecUSec -p MemoryMax; journalctl -u {{ $labels.unit }} --since \"20 minutes ago\" --no-pager | tail -120'" + runbook: "自動階段先診斷。若確認 WatchdogSec 對 GitHub runner 誤設，需人工或 sudo playbook 停用 watchdog drop-in，並加 CPUQuota/MemoryMax。" + + - alert: SystemdRunnerWatchdogEnabled + expr: systemd_unit_watchdog_seconds{unit=~"actions\\.runner\\..*"} > 0 + for: 10m + labels: + severity: warning + layer: systemd-110 + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "true" + annotations: + summary: "Systemd runner {{ $labels.unit }} 啟用了 WatchdogSec" + description: "{{ $labels.unit }} WatchdogSec={{ $value }} 秒。GitHub Actions runner service 不應被 systemd watchdog 週期性殺掉。" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'systemctl show {{ $labels.unit }} -p WatchdogUSec -p DropInPaths -p NRestarts'" + runbook: "確認 drop-in 來源；需要 sudo 時由人工套用：移除 watchdog.conf、daemon-reload、restart service。" + + - alert: SystemdRunnerMissingResourceQuota + expr: systemd_unit_cpu_quota_cores{unit=~"actions\\.runner\\..*"} == 0 or systemd_unit_memory_max_bytes{unit=~"actions\\.runner\\..*"} == 0 + for: 30m + labels: + severity: warning + layer: systemd-110 + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "true" + annotations: + summary: "Systemd runner {{ $labels.unit }} 缺 CPU 或 memory quota" + description: "{{ $labels.unit }} 仍為 unlimited；CI runner 會與 Sentry/ClickHouse/Gitea 搶主機 CPU。" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'systemctl show {{ $labels.unit }} -p CPUQuotaPerSecUSec -p MemoryMax -p ActiveState -p SubState'" + runbook: "建議 baseline：每個 runner CPUQuota=200%、MemoryMax=2G；若同時有多個 runner，需限制並行度或分流到非 Sentry 主機。" + # ========================================================================= # MinIO / Kali 告警 # ========================================================================= diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml index 0325fac3f..9e73a858f 100644 --- a/ops/monitoring/alerts.yml +++ b/ops/monitoring/alerts.yml @@ -67,7 +67,7 @@ groups: annotations: summary: "主機 {{ $labels.host }} load5/core 長時間過高" description: "load5 / CPU core > 1.5 持續 15 分鐘；這通常代表 runnable queue 已長期塞車，不是短暫尖峰。" - auto_repair_action: "ssh {{ $labels.instance }} 'echo \"=== LOAD ===\"; uptime; echo \"=== CPU TOP ===\"; ps aux --sort=-%cpu | head -25; echo \"=== DOCKER STATS ===\"; docker stats --no-stream | head -40'" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'echo \"=== LOAD ===\"; uptime; echo \"=== CPU TOP ===\"; ps aux --sort=-%cpu | head -25; echo \"=== DOCKER STATS ===\"; docker stats --no-stream | head -40'" runbook: "先判斷高 load 來源：ClickHouse merge / Kafka backlog / Chrome scraper / Ollama runner / exporter；只允許 read-only 診斷，自動修復需走服務專屬 playbook。" - alert: HostOutOfMemory @@ -616,7 +616,7 @@ groups: annotations: summary: "容器 {{ $labels.container_name }} CPU 持續超過 2 core" description: "{{ $labels.container_name }} 5m CPU 使用 >2 core 且持續 10 分鐘，需檢查是否缺 cpus limit、merge backlog、爬蟲尖峰或模型推理。" - auto_repair_action: "ssh {{ $labels.instance }} 'docker stats --no-stream {{ $labels.container_name }}; docker inspect {{ $labels.container_name }} | head -80'" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'docker stats --no-stream {{ $labels.container_name }}; docker inspect {{ $labels.container_name }} | head -80'" runbook: "自動階段只做診斷；若是已知服務，交給 service-specific playbook：ClickHouse 降 merge/查 backlog、scheduler 限 concurrency、litellm/blackbox 查 liveliness。" - alert: DockerContainerCpuRunawayCritical @@ -632,7 +632,7 @@ groups: annotations: summary: "容器 {{ $labels.container_name }} CPU 持續超過 4 core" description: "{{ $labels.container_name }} 已持續吃超過 4 core，會拖垮 110/188 主機；需要 AI 判斷是否限流、暫停 worker 或套用既有 playbook。" - auto_repair_action: "ssh {{ $labels.instance }} 'echo \"=== CONTAINER ===\"; docker stats --no-stream {{ $labels.container_name }}; echo \"=== TOP ===\"; docker top {{ $labels.container_name }} -eo pid,ppid,stat,pcpu,pmem,comm,args | head -30'" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'echo \"=== CONTAINER ===\"; docker stats --no-stream {{ $labels.container_name }}; echo \"=== TOP ===\"; docker top {{ $labels.container_name }} -eo pid,ppid,stat,pcpu,pmem,comm,args | head -30'" runbook: "禁止通用 docker restart；先抓根因，只有 health down 或 crash loop 才可走重啟。" - alert: DockerContainerMemoryLimitPressure @@ -649,7 +649,7 @@ groups: annotations: summary: "容器 {{ $labels.container_name }} 記憶體超過 limit 85%" description: "{{ $labels.container_name }} memory / limit > 85% 持續 10 分鐘。若是 Kafka/ClickHouse/Gitea/Taskworker，需先判斷 workload，不可直接降 limit。" - auto_repair_action: "ssh {{ $labels.instance }} 'docker stats --no-stream {{ $labels.container_name }}; docker inspect {{ $labels.container_name }} | head -80'" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'docker stats --no-stream {{ $labels.container_name }}; docker inspect {{ $labels.container_name }} | head -80'" runbook: "若服務已接近 limit：優先調整 retention/concurrency/cache，再評估提高 memory；禁止用更低 memory limit 當止血。" - alert: DockerContainerRestartSpike @@ -666,9 +666,58 @@ groups: annotations: summary: "容器 {{ $labels.container_name }} 15 分鐘重啟超過 5 次" description: "Docker restart textfile exporter 顯示 {{ $labels.container_name }} 重啟暴增，避免再次出現 litellm 24,464 次靜默崩潰。" - auto_repair_action: "ssh {{ $labels.instance }} 'docker ps -a --filter name={{ $labels.container_name }}; docker logs --tail=120 {{ $labels.container_name }}'" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'docker ps -a --filter name={{ $labels.container_name }}; docker logs --tail=120 {{ $labels.container_name }}'" runbook: "先抓 crash signature；若是 config/DB/網路問題，修設定，不用無限 restart。" + - alert: SystemdRunnerRestartSpike + # 2026-05-05 ogt + Codex: 110 GitHub Actions runner watchdog loop was outside Docker/cAdvisor coverage. + expr: increase(systemd_unit_restarts_total{unit=~"actions\\.runner\\..*"}[15m]) > 2 + for: 3m + labels: + severity: critical + layer: systemd-110 + team: ops + alert_category: infrastructure + notification_type: TYPE-3 + auto_repair: "true" + annotations: + summary: "Systemd runner {{ $labels.unit }} 15 分鐘重啟超過 2 次" + description: "{{ $labels.unit }} 在 15 分鐘內重啟暴增；110 曾發生 WatchdogSec=5min 造成 runner 每 5 分鐘自殺重啟。" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'systemctl show {{ $labels.unit }} -p WatchdogUSec -p NRestarts -p DropInPaths -p CPUQuotaPerSecUSec -p MemoryMax; journalctl -u {{ $labels.unit }} --since \"20 minutes ago\" --no-pager | tail -120'" + runbook: "自動階段先診斷。若確認 WatchdogSec 對 GitHub runner 誤設，需人工或 sudo playbook 停用 watchdog drop-in，並加 CPUQuota/MemoryMax。" + + - alert: SystemdRunnerWatchdogEnabled + expr: systemd_unit_watchdog_seconds{unit=~"actions\\.runner\\..*"} > 0 + for: 10m + labels: + severity: warning + layer: systemd-110 + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "true" + annotations: + summary: "Systemd runner {{ $labels.unit }} 啟用了 WatchdogSec" + description: "{{ $labels.unit }} WatchdogSec={{ $value }} 秒。GitHub Actions runner service 不應被 systemd watchdog 週期性殺掉。" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'systemctl show {{ $labels.unit }} -p WatchdogUSec -p DropInPaths -p NRestarts'" + runbook: "確認 drop-in 來源；需要 sudo 時由人工套用：移除 watchdog.conf、daemon-reload、restart service。" + + - alert: SystemdRunnerMissingResourceQuota + expr: systemd_unit_cpu_quota_cores{unit=~"actions\\.runner\\..*"} == 0 or systemd_unit_memory_max_bytes{unit=~"actions\\.runner\\..*"} == 0 + for: 30m + labels: + severity: warning + layer: systemd-110 + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "true" + annotations: + summary: "Systemd runner {{ $labels.unit }} 缺 CPU 或 memory quota" + description: "{{ $labels.unit }} 仍為 unlimited；CI runner 會與 Sentry/ClickHouse/Gitea 搶主機 CPU。" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'systemctl show {{ $labels.unit }} -p CPUQuotaPerSecUSec -p MemoryMax -p ActiveState -p SubState'" + runbook: "建議 baseline：每個 runner CPUQuota=200%、MemoryMax=2G；若同時有多個 runner，需限制並行度或分流到非 Sentry 主機。" + # ========================================================================= # MinIO / Kali 告警 # ========================================================================= diff --git a/scripts/ops/systemd-units-textfile-exporter.py b/scripts/ops/systemd-units-textfile-exporter.py new file mode 100644 index 000000000..66c53dfea --- /dev/null +++ b/scripts/ops/systemd-units-textfile-exporter.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +""" +Systemd unit textfile exporter for host-level AIOps baselines. + +2026-05-05 ogt + Codex: 110 runner overload follow-up. +Why: GitHub/Gitea runner services are outside Docker metrics. A bad +WatchdogSec drop-in caused repeated runner restarts on 110, so the AI monitor +needs direct visibility into systemd restarts, watchdog settings, and limits. +""" + +from __future__ import annotations + +import os +import re +import subprocess +import tempfile +from pathlib import Path + + +TEXTFILE_DIR = Path(os.environ.get("NODE_EXPORTER_TEXTFILE_DIR", "/var/lib/node_exporter/textfile_collector")) +OUTPUT_NAME = "systemd_units.prom" +HOST_LABEL = os.environ.get("AIOPS_HOST_LABEL", os.uname().nodename) +UNIT_NAMES = [ + unit.strip() + for unit in os.environ.get("AIOPS_SYSTEMD_UNITS", "").split(",") + if unit.strip() +] +LABEL_RE = re.compile(r'["\\\n]') + + +def _escape_label(value: str) -> str: + return LABEL_RE.sub(lambda m: {"\n": r"\n", "\\": r"\\", '"': r"\""}[m.group(0)], value) + + +def _parse_usec(value: str) -> float: + value = value.strip() + if value in {"", "0", "infinity"}: + return 0.0 + multipliers = { + "us": 0.000001, + "ms": 0.001, + "s": 1.0, + "min": 60.0, + "h": 3600.0, + } + match = re.fullmatch(r"([0-9.]+)\s*([A-Za-z]+)?", value) + if not match: + return 0.0 + number, unit = match.groups() + if unit is None: + return float(number) / 1_000_000 + return float(number) * multipliers.get(unit, 0.0) + + +def _parse_bytes(value: str) -> float: + if value in {"", "infinity"}: + return 0.0 + try: + return float(value) + except ValueError: + return 0.0 + + +def _show_unit(unit: str) -> dict[str, str]: + result = subprocess.run( + [ + "systemctl", + "show", + unit, + "-p", + "ActiveState", + "-p", + "SubState", + "-p", + "NRestarts", + "-p", + "WatchdogUSec", + "-p", + "CPUQuotaPerSecUSec", + "-p", + "MemoryMax", + ], + check=True, + capture_output=True, + text=True, + timeout=10, + ) + values = {} + for line in result.stdout.splitlines(): + key, _, raw = line.partition("=") + values[key] = raw + return values + + +def collect() -> str: + lines = [ + "# HELP systemd_unit_info Systemd unit inventory exposed by the textfile exporter.", + "# TYPE systemd_unit_info gauge", + "# HELP systemd_unit_restarts_total Systemd unit restart counter from systemctl show NRestarts.", + "# TYPE systemd_unit_restarts_total counter", + "# HELP systemd_unit_watchdog_seconds Systemd unit WatchdogSec setting in seconds, 0 when disabled.", + "# TYPE systemd_unit_watchdog_seconds gauge", + "# HELP systemd_unit_cpu_quota_cores Systemd unit CPUQuota converted to CPU cores, 0 when unlimited.", + "# TYPE systemd_unit_cpu_quota_cores gauge", + "# HELP systemd_unit_memory_max_bytes Systemd unit MemoryMax in bytes, 0 when unlimited.", + "# TYPE systemd_unit_memory_max_bytes gauge", + ] + for unit in UNIT_NAMES: + try: + values = _show_unit(unit) + except Exception as exc: + labels = f'host="{_escape_label(HOST_LABEL)}",unit="{_escape_label(unit)}"' + lines.append(f'systemd_unit_info{{{labels},active_state="scrape_error",sub_state="{_escape_label(str(exc))}"}} 0') + continue + + labels = f'host="{_escape_label(HOST_LABEL)}",unit="{_escape_label(unit)}"' + active_state = values.get("ActiveState", "") + sub_state = values.get("SubState", "") + restarts = int(values.get("NRestarts") or 0) + watchdog_seconds = _parse_usec(values.get("WatchdogUSec", "0")) + cpu_quota = _parse_usec(values.get("CPUQuotaPerSecUSec", "0")) + memory_max = _parse_bytes(values.get("MemoryMax", "0")) + lines.append( + f'systemd_unit_info{{{labels},active_state="{_escape_label(active_state)}",sub_state="{_escape_label(sub_state)}"}} 1' + ) + lines.append(f"systemd_unit_restarts_total{{{labels}}} {restarts}") + lines.append(f"systemd_unit_watchdog_seconds{{{labels}}} {watchdog_seconds:.6f}") + lines.append(f"systemd_unit_cpu_quota_cores{{{labels}}} {cpu_quota:.6f}") + lines.append(f"systemd_unit_memory_max_bytes{{{labels}}} {memory_max:.0f}") + return "\n".join(lines) + "\n" + + +def main() -> None: + if not UNIT_NAMES: + raise SystemExit("AIOPS_SYSTEMD_UNITS is required") + TEXTFILE_DIR.mkdir(parents=True, exist_ok=True) + payload = collect() + with tempfile.NamedTemporaryFile("w", dir=TEXTFILE_DIR, delete=False, encoding="utf-8") as tmp: + tmp.write(payload) + tmp_path = Path(tmp.name) + output_path = TEXTFILE_DIR / OUTPUT_NAME + tmp_path.replace(output_path) + output_path.chmod(0o644) + + +if __name__ == "__main__": + main()