diff --git a/apps/api/src/services/alert_rule_engine.py b/apps/api/src/services/alert_rule_engine.py index 25033e20f..7b96173ab 100644 --- a/apps/api/src/services/alert_rule_engine.py +++ b/apps/api/src/services/alert_rule_engine.py @@ -7,7 +7,7 @@ OpenClaw 告警規則匹配引擎 - 規則在 YAML 定義,不需要改 Python 代碼 - 匹配邏輯: alertname 完全匹配 > alert_type 部分匹配 > message 關鍵字 - priority 越小越優先,999 = 通用兜底 -- 變數替換: {target} {host} {container} {instance} {job} {namespace} +- 變數替換: {target} {host} {container} {instance} {job} {namespace} {unit} 自動規則生成: - 只有 generic_fallback 觸發時才生成(具體規則不觸發) @@ -185,6 +185,7 @@ def _extract_vars(alert_context: dict) -> dict[str, str]: host = instance.split(":")[0] if ":" in instance else instance job = labels.get("job", "exporter") namespace = alert_context.get("namespace", "awoooi-prod") + unit = labels.get("unit", "") # GAP-A4: 多層 label 查找,由最權威到最弱 target = "" @@ -245,6 +246,7 @@ def _extract_vars(alert_context: dict) -> dict[str, str]: "instance": instance, "job": job, "namespace": namespace, + "unit": unit, } diff --git a/apps/api/src/services/incident_service.py b/apps/api/src/services/incident_service.py index ab97b2cad..bcf463a70 100644 --- a/apps/api/src/services/incident_service.py +++ b/apps/api/src/services/incident_service.py @@ -116,7 +116,7 @@ def classify_alert_early( 1. ConfigurationDrift / KubeConfigDrift → TYPE-4D (Config Drift 卡片) 2. severity=info/none → TYPE-1 (純資訊,無按鈕) 3. backup/heartbeat 關鍵字 → TYPE-1(但 backup failure age > 24h → TYPE-3,見下) - 4. Docker/Host 前綴 → infrastructure TYPE-3 + 4. Docker/Host/Systemd runner 前綴 → infrastructure/host_resource TYPE-3 5. Kube/Pod/Deploy/Node/Velero/ArgoCD 前綴 → kubernetes TYPE-3 6. Postgres/Redis 前綴 → database TYPE-3 7. 預設 → general TYPE-3 @@ -183,6 +183,12 @@ def classify_alert_early( if severity in ("info", "none"): return "info", "TYPE-1" + # 2026-05-05 ogt + Codex: self-hosted runners are host-level systemd services. + # Must run before the generic "watchdog" heartbeat rule because + # SystemdRunnerWatchdogEnabled contains "Watchdog" but is not a heartbeat. + if alertname.startswith("SystemdRunner"): + return "host_resource", "TYPE-3" + # 5. Backup / Heartbeat — 純資訊,不進 LLM # HostBackupFailed 必須在 Host prefix 前攔截,否則被歸 host_resource/TYPE-3 # 2026-04-12 ogt: 只針對已知主機備份監控 alertname,不用寬泛關鍵字 diff --git a/apps/api/tests/test_classify_alert_early.py b/apps/api/tests/test_classify_alert_early.py index e1008ebe3..c4f133d2d 100644 --- a/apps/api/tests/test_classify_alert_early.py +++ b/apps/api/tests/test_classify_alert_early.py @@ -12,8 +12,8 @@ classify_alert_early() 分類函數單元測試 — ADR-073 Phase 2-2 + ADR-075 """ import pytest -from src.services.incident_service import classify_alert_early +from src.services.incident_service import classify_alert_early # --------------------------------------------------------------------------- # # TYPE-4D: Config Drift @@ -110,6 +110,16 @@ class TestInfrastructure: ac, nt = classify_alert_early("HostOutOfDiskSpace", "warning", {}) assert ac == "host_resource" + @pytest.mark.parametrize("alertname", [ + "SystemdRunnerRestartSpike", + "SystemdRunnerWatchdogEnabled", + "SystemdRunnerMissingResourceQuota", + ]) + def test_systemd_runner_is_host_resource_not_heartbeat(self, alertname): + ac, nt = classify_alert_early(alertname, "warning", {}) + assert nt == "TYPE-3" + assert ac == "host_resource" + # --------------------------------------------------------------------------- # # ADR-075: alertchain_health (TYPE-8M) diff --git a/apps/api/tests/test_gap_a4_placeholder_resolution.py b/apps/api/tests/test_gap_a4_placeholder_resolution.py index e70f0093a..a468e35c0 100644 --- a/apps/api/tests/test_gap_a4_placeholder_resolution.py +++ b/apps/api/tests/test_gap_a4_placeholder_resolution.py @@ -25,7 +25,6 @@ from src.services.alert_rule_engine import ( match_rule, ) - # ============================================================================= # _strip_pod_suffix # ============================================================================= @@ -158,6 +157,20 @@ class TestExtractVarsGapA4: } assert _extract_vars(ctx)["target"] == "awoooi-web" + def test_systemd_unit_label_extracted_for_host_rule_templates(self): + """Systemd runner alert labels must fill {unit} in SSH diagnostics.""" + ctx = { + "target_resource": "SystemdRunnerWatchdogEnabled", + "labels": { + "alertname": "SystemdRunnerWatchdogEnabled", + "instance": "192.168.0.110:9100", + "unit": "actions.runner.owenhytsai-awoooi.awoooi-110.service", + }, + "namespace": "awoooi-prod", + } + vars = _extract_vars(ctx) + assert vars["unit"] == "actions.runner.owenhytsai-awoooi.awoooi-110.service" + # ============================================================================= # match_rule 後置驗證 — 最後一道防線 @@ -205,3 +218,25 @@ class TestMatchRuleRejection: assert "awoooi-api" in result["kubectl_command"] assert "unknown" not in result["kubectl_command"] assert "KubePodCrashLooping" not in result["kubectl_command"] + + def test_systemd_runner_rule_preserves_unit_ssh_command(self): + """SystemdRunner* must keep a filled read-only SSH diagnostic command.""" + ctx = { + "alert_type": "infrastructure", + "severity": "warning", + "source": "prometheus", + "target_resource": "SystemdRunnerWatchdogEnabled", + "namespace": "awoooi-prod", + "message": "runner watchdog enabled", + "labels": { + "alertname": "SystemdRunnerWatchdogEnabled", + "instance": "192.168.0.110:9100", + "unit": "actions.runner.owenhytsai-awoooi.awoooi-110.service", + }, + } + result = match_rule(ctx) + assert result is not None + assert result["rule_id"] == "systemd_runner_baseline_alert" + assert result["kubectl_command"].startswith("ssh 192.168.0.110 ") + assert "actions.runner.owenhytsai-awoooi.awoooi-110.service" in result["kubectl_command"] + assert "{unit}" not in result["kubectl_command"] diff --git a/apps/api/tests/test_golden_regression.py b/apps/api/tests/test_golden_regression.py index 64253397a..acccb8488 100644 --- a/apps/api/tests/test_golden_regression.py +++ b/apps/api/tests/test_golden_regression.py @@ -10,7 +10,6 @@ """ from __future__ import annotations -import pytest from src.services.incident_service import classify_alert_early @@ -49,6 +48,10 @@ class TestHostResource: cat, ntype = classify_alert_early("HostDiskSpaceFull", "critical") assert cat == "host_resource" and ntype == "TYPE-3" + def test_systemd_runner_watchdog_is_not_heartbeat(self): + cat, ntype = classify_alert_early("SystemdRunnerWatchdogEnabled", "warning") + assert cat == "host_resource" and ntype == "TYPE-3" + class TestHighCpuVariants: """HighCPU* prefix 規則覆蓋"""