diff --git a/apps/api/alert_rules.yaml b/apps/api/alert_rules.yaml index 3d494b1be..1b20f44b6 100644 --- a/apps/api/alert_rules.yaml +++ b/apps/api/alert_rules.yaml @@ -175,6 +175,7 @@ rules: - DockerContainerCpuRunawayCritical - DockerContainerMemoryLimitPressure - DockerContainerRestartSpike + - DockerGiteaActionsJobStale response: action_title: "🔍 Docker/Host 過載自動診斷 — 禁止通用重啟" description: "110/188 Docker Compose 或主機 load 長時間偏離 baseline。AI 需先收集容器 CPU、restart、logs、ClickHouse/Kafka/爬蟲狀態，再選擇限流、降併發或服務專屬 playbook。" diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 09fe8126b..8c16d6834 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1605,6 +1605,8 @@ psql $DATABASE_URL -f apps/api/migrations/cleanup_duplicate_deprecated_playbooks | `0e14935` | `SystemdRunner*` early classification + `{unit}` template variable | | `ab0f0a8` | deploy API image `runner-classify-20260505-0e14935` | | `2e128f9` | Gitea Code Review stale-run guard，避免快速連推堆疊多個 runner job | +| `3b73cc7` | CD paths 收斂，workflow-only commits 不再觸發完整 image build/deploy | +| `pending` | Docker textfile 補 `docker_container_started_seconds` + `DockerGiteaActionsJobStale` | ### 下一步 diff --git a/docs/runbooks/HOST-RESOURCE-BASELINE-110-188.md b/docs/runbooks/HOST-RESOURCE-BASELINE-110-188.md index 657844887..e48b99c33 100644 --- a/docs/runbooks/HOST-RESOURCE-BASELINE-110-188.md +++ b/docs/runbooks/HOST-RESOURCE-BASELINE-110-188.md @@ -46,6 +46,7 @@ Use these thresholds for alerting and AI triage: | Systemd runner restarts | > 2 in 15m | Critical; inspect watchdog/drop-ins and active CI jobs. | | Systemd runner WatchdogSec | > 0 for 10m | Warning; GitHub Actions runner should not be killed by systemd watchdog. | | Systemd runner quota | CPU or memory unlimited for 30m | Warning; apply CPUQuota/MemoryMax or move CI away from Sentry host. | +| Gitea Actions job runtime | > 20m for 5m | Warning; inspect logs and stop stale job containers if they outlive workflow timeout. | ## Rules @@ -59,6 +60,7 @@ Use these thresholds for alerting and AI triage: 8. Disable node-exporter collectors that are slow or failing on each host; exporter scrape time is part of the resource baseline. 9. Runner services are part of the resource baseline even when jobs run outside Docker. Export `systemd_unit_*` metrics for all `actions.runner.*` services. 10. CI runner limits must be set at the systemd unit level. Docker container limits do not protect host load from runner listener, worker, and `act` helper processes. +11. Workflow-only commits must not trigger full CD image builds; CD should run only when runtime inputs change. ## Next Safe Rollout Order diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index e39f990a6..e0ac963fa 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -663,6 +663,23 @@ groups: auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'docker ps -a --filter name={{ $labels.container_name }}; docker logs --tail=120 {{ $labels.container_name }}'" runbook: "先抓 crash signature；若是 config/DB/網路問題，修設定，不用無限 restart。" + - alert: DockerGiteaActionsJobStale + # 2026-05-05 ogt + Codex: Gitea/act job containers can outlive workflow timeout and keep 110 hot. + expr: time() - docker_container_started_seconds{container_name=~"GITEA-ACTIONS-.*"} > 1200 + for: 5m + labels: + severity: warning + layer: docker-110 + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "true" + annotations: + summary: "Gitea Actions job {{ $labels.container_name }} 執行超過 20 分鐘" + description: "{{ $labels.container_name }} 已超過 20 分鐘，110 曾出現 code-review/CD job timeout 失效而持續佔用 runner。" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'docker ps --filter name={{ $labels.container_name }}; docker logs --tail=120 {{ $labels.container_name }}'" + runbook: "先確認是否仍有輸出與是否為 CD 關鍵步驟；若 logs 空白且超過 workflow timeout，可人工停止 stale job container。" + - alert: SystemdRunnerRestartSpike # 2026-05-05 ogt + Codex: 110 GitHub Actions runner watchdog loop was outside Docker/cAdvisor coverage. expr: increase(systemd_unit_restarts_total{unit=~"actions\\.runner\\..*"}[15m]) > 2 diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml index 9e73a858f..6abb823d3 100644 --- a/ops/monitoring/alerts.yml +++ b/ops/monitoring/alerts.yml @@ -669,6 +669,23 @@ groups: auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'docker ps -a --filter name={{ $labels.container_name }}; docker logs --tail=120 {{ $labels.container_name }}'" runbook: "先抓 crash signature；若是 config/DB/網路問題，修設定，不用無限 restart。" + - alert: DockerGiteaActionsJobStale + # 2026-05-05 ogt + Codex: Gitea/act job containers can outlive workflow timeout and keep 110 hot. + expr: time() - docker_container_started_seconds{container_name=~"GITEA-ACTIONS-.*"} > 1200 + for: 5m + labels: + severity: warning + layer: docker-110 + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "true" + annotations: + summary: "Gitea Actions job {{ $labels.container_name }} 執行超過 20 分鐘" + description: "{{ $labels.container_name }} 已超過 20 分鐘，110 曾出現 code-review/CD job timeout 失效而持續佔用 runner。" + auto_repair_action: "ssh 192.168.0.{{ $labels.host }} 'docker ps --filter name={{ $labels.container_name }}; docker logs --tail=120 {{ $labels.container_name }}'" + runbook: "先確認是否仍有輸出與是否為 CD 關鍵步驟；若 logs 空白且超過 workflow timeout，可人工停止 stale job container。" + - alert: SystemdRunnerRestartSpike # 2026-05-05 ogt + Codex: 110 GitHub Actions runner watchdog loop was outside Docker/cAdvisor coverage. expr: increase(systemd_unit_restarts_total{unit=~"actions\\.runner\\..*"}[15m]) > 2 diff --git a/scripts/ops/docker-stats-textfile-exporter.py b/scripts/ops/docker-stats-textfile-exporter.py index 12841672b..81d1952a0 100755 --- a/scripts/ops/docker-stats-textfile-exporter.py +++ b/scripts/ops/docker-stats-textfile-exporter.py @@ -15,6 +15,7 @@ import os import re import subprocess import tempfile +from datetime import datetime, timezone from pathlib import Path @@ -67,6 +68,17 @@ def _memory_bytes(value: str) -> float: return float(number) * scale +def _started_at_seconds(value: str) -> float: + if not value: + return 0.0 + try: + normalized = re.sub(r"\.(\d{6})\d+(Z|[+-]\d\d:\d\d)$", r".\1\2", value) + normalized = normalized.replace("Z", "+00:00") + return datetime.fromisoformat(normalized).astimezone(timezone.utc).timestamp() + except ValueError: + return 0.0 + + def collect() -> str: stats = _run_json_lines([ "docker", @@ -102,6 +114,8 @@ def collect() -> str: "# TYPE docker_container_inspect_restart_count gauge", "# HELP docker_container_info Docker container inventory exposed by the textfile exporter.", "# TYPE docker_container_info gauge", + "# HELP docker_container_started_seconds Docker container start timestamp from Docker inspect.", + "# TYPE docker_container_started_seconds gauge", ] for row in stats: @@ -114,6 +128,7 @@ def collect() -> str: nano_cpus = float(host_config.get("NanoCpus") or 0) memory_limit = float(host_config.get("Memory") or 0) restart_count = int(inspected.get("RestartCount") or 0) + started_seconds = _started_at_seconds(state.get("StartedAt", "")) labels = f'host="{_escape_label(HOST_LABEL)}",container_name="{_escape_label(name)}"' mem_current = (row.get("MemUsage") or "0 B / 0 B").split("/", 1)[0].strip() pids = row.get("PIDs") or "0" @@ -123,6 +138,7 @@ def collect() -> str: lines.append(f"docker_container_memory_limit_bytes{{{labels}}} {memory_limit:.0f}") lines.append(f"docker_container_pids{{{labels}}} {int(float(pids))}") lines.append(f"docker_container_inspect_restart_count{{{labels}}} {restart_count}") + lines.append(f"docker_container_started_seconds{{{labels}}} {started_seconds:.6f}") lines.append( f'docker_container_info{{{labels},status="{_escape_label(state.get("Status", ""))}"}} 1' )