fix(backup): preserve alert receipt identity

scripts/ops/backup-alert-label-contract-check.py

@@ -147,15 +147,22 @@ def static_check(path: Path, baseline_path: Path) -> list[str]:
     rule = _require_alert(alerts, "BackupAlertReceiptStageMissing")
     expr = str(rule.get("expr", ""))
     for required_fragment in [
+        "max by (required_alert, host) (",
+        "label_replace(",
         "ALERTS{",
         'alertstate="firing"',
         'alertname=~"Backup.*|GiteaPrivateBundle.*|GiteaRestoreDrill.*"',
         'alertname!="BackupAlertReceiptStageMissing"',
-        "unless on (alertname, host)",
-        "label_replace(",
-        "awoooi_backup_alert_receipt_fresh == 1",
+        '"required_alert", "$1", "alertname", "(.*)"',
+        "unless on (required_alert, host)",
+        "max by (required_alert, host) (awoooi_backup_alert_receipt_fresh == 1)",
     ]:
         _require_contains(expr, required_fragment, "BackupAlertReceiptStageMissing expr")
+    _require_not_contains(
+        expr,
+        "unless on (alertname, host)",
+        "BackupAlertReceiptStageMissing expr",
+    )
     _require_not_contains(
         expr,
         "sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == bool 0) > 0",
@@ -164,10 +171,14 @@ def static_check(path: Path, baseline_path: Path) -> list[str]:
     text = _annotation_text(rule)
     for required_label in [
         "$labels.host",
-        "$labels.alertname",
-        "required_alert",
+        "$labels.required_alert",
     ]:
         _require_contains(text, required_label, "BackupAlertReceiptStageMissing annotations")
+    _require_not_contains(
+        text,
+        "$labels.alertname",
+        "BackupAlertReceiptStageMissing annotations",
+    )
     lines.append("OK BackupAlertReceiptStageMissing requires receipts only for actively firing backup/Gitea alerts")
 
     return lines

scripts/ops/backup-alert-live-visibility-check.py

@@ -188,35 +188,37 @@ def _escrow_required_alerts(prometheus_url: str, host: str, timeout: int) -> lis
 
 
 def _receipt_stage_required_alerts(prometheus_url: str, host: str, timeout: int) -> list[RequiredAlert]:
+    # 直接從已 firing 的 meta alert 讀取 required_alert，再對 Prometheus
+    # 與 Alertmanager 做同一 identity 核對。不得從缺 marker 推導假 receipt。
     expr = (
-        "sum by (host, receipt_channel) "
-        f'(awoooi_backup_alert_receipt_stage_fresh{{host="{host}"}} == bool 0) > 0'
+        'ALERTS{alertname="BackupAlertReceiptStageMissing",alertstate="firing",'
+        f'host="{host}"}}'
     )
     rows = _prom_query(prometheus_url, expr, timeout)
     required: list[RequiredAlert] = []
     for row in rows:
         labels = _metric_labels(row)
-        receipt_channel = labels.get("receipt_channel")
+        required_alert = labels.get("required_alert")
         missing = [
             label
             for label, value in {
-                "receipt_channel": receipt_channel,
+                "required_alert": required_alert,
             }.items()
             if not value
         ]
         if missing:
-            raise VisibilityError(f"Backup alert receipt gap metric missing labels {missing}: {row}")
+            raise VisibilityError(f"Backup alert receipt meta alert missing labels {missing}: {row}")
         required.append(
             RequiredAlert(
                 "BackupAlertReceiptStageMissing",
                 {
                     **COMMON_LABELS,
                     "host": host,
-                    "receipt_channel": str(receipt_channel),
+                    "required_alert": str(required_alert),
                 },
             )
         )
-    return sorted(required, key=lambda alert: alert.labels["receipt_channel"])
+    return sorted(required, key=lambda alert: alert.labels["required_alert"])
 
 
 def live_check(prometheus_url: str, alertmanager_url: str, host: str, timeout: int) -> list[str]:
@@ -241,12 +243,12 @@ def live_check(prometheus_url: str, alertmanager_url: str, host: str, timeout: i
     required_alerts.extend(receipt_stage_alerts)
     if receipt_stage_alerts:
         lines.append(
-            "OK backup alert receipt gap metrics require "
-            f"{len(receipt_stage_alerts)} aggregated alert(s): "
-            + ", ".join(alert.labels["receipt_channel"] for alert in receipt_stage_alerts)
+            "OK firing backup alert receipt gaps preserve "
+            f"{len(receipt_stage_alerts)} required alert identity label(s): "
+            + ", ".join(alert.labels["required_alert"] for alert in receipt_stage_alerts)
         )
     else:
-        lines.append("OK backup alert receipt markers are fresh; no receipt stage alert required")
+        lines.append("OK no BackupAlertReceiptStageMissing alert is currently firing")
 
     prom_alerts = _prom_alerts(prometheus_url, timeout)
     for required in required_alerts:

scripts/ops/backup-health-textfile-exporter.py

@@ -67,6 +67,13 @@ BACKUP_ALERT_RECEIPT_REQUIREMENTS = [
     {"host": "188", "alertname": "BackupHealthMonitorMissing188", "scope": "monitor"},
     {"host": "110", "alertname": "BackupHealthMonitorStale", "scope": "monitor"},
     {"host": "110", "alertname": "BackupExpectedJobMissing", "scope": "schedule"},
+    # 只宣告 requirement；沒有真實收據 marker 時 fresh 必須維持 0。
+    {
+        "host": "110",
+        "alertname": "BackupScheduleSingletonMismatch",
+        "scope": "schedule",
+        "notification_type": "TYPE-1",
+    },
     {"host": "110", "alertname": "BackupScriptMissing", "scope": "script"},
     {"host": "110", "alertname": "BackupJobStale", "scope": "freshness"},
     {"host": "110", "alertname": "BackupCoverageDomainStale", "scope": "coverage"},
@@ -160,11 +167,12 @@ def _backup_alert_receipt_metric_lines(host: str) -> list[str]:
             continue
         alertname = requirement["alertname"]
         scope = requirement["scope"]
+        notification_type = requirement.get("notification_type", "TYPE-3")
         base_labels = (
             f'host="{_escape_label(host)}",'
             f'required_alert="{_escape_label(alertname)}",'
             f'scope="{_escape_label(scope)}",'
-            'required_notification_type="TYPE-3",'
+            f'required_notification_type="{_escape_label(notification_type)}",'
             'receipt_channel="telegram_awooop",'
             f'max_age_hours="{BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS:g}"'
         )

scripts/ops/tests/test_backup_alert_live_visibility.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+import pytest
+
+
+SCRIPT_ROOT = Path(__file__).resolve().parents[1]
+CHECK_PATH = SCRIPT_ROOT / "backup-alert-live-visibility-check.py"
+
+
+def load_check():
+    spec = importlib.util.spec_from_file_location(
+        "backup_alert_live_visibility_check",
+        CHECK_PATH,
+    )
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_receipt_verifier_preserves_each_required_alert_identity(monkeypatch) -> None:
+    check = load_check()
+    observed: dict[str, object] = {}
+
+    def fake_query(base_url: str, expr: str, timeout: int):
+        observed.update(base_url=base_url, expr=expr, timeout=timeout)
+        return [
+            {
+                "metric": {
+                    "alertname": "BackupAlertReceiptStageMissing",
+                    "alertstate": "firing",
+                    "host": "110",
+                    "required_alert": "BackupScheduleSingletonMismatch",
+                },
+                "value": [1_783_000_000, "1"],
+            },
+            {
+                "metric": {
+                    "alertname": "BackupAlertReceiptStageMissing",
+                    "alertstate": "firing",
+                    "host": "110",
+                    "required_alert": "BackupRetentionPolicyNotLatestOnly",
+                },
+                "value": [1_783_000_000, "1"],
+            },
+        ]
+
+    monkeypatch.setattr(check, "_prom_query", fake_query)
+
+    required = check._receipt_stage_required_alerts("http://prometheus", "110", 8)
+
+    assert observed == {
+        "base_url": "http://prometheus",
+        "expr": (
+            'ALERTS{alertname="BackupAlertReceiptStageMissing",alertstate="firing",'
+            'host="110"}'
+        ),
+        "timeout": 8,
+    }
+    assert [alert.labels["required_alert"] for alert in required] == [
+        "BackupRetentionPolicyNotLatestOnly",
+        "BackupScheduleSingletonMismatch",
+    ]
+    assert all("receipt_channel" not in alert.labels for alert in required)
+
+
+def test_receipt_verifier_rejects_meta_alert_without_required_alert(monkeypatch) -> None:
+    check = load_check()
+    monkeypatch.setattr(
+        check,
+        "_prom_query",
+        lambda *_args: [
+            {
+                "metric": {
+                    "alertname": "BackupAlertReceiptStageMissing",
+                    "alertstate": "firing",
+                    "host": "110",
+                },
+                "value": [1_783_000_000, "1"],
+            }
+        ],
+    )
+
+    with pytest.raises(check.VisibilityError, match="required_alert"):
+        check._receipt_stage_required_alerts("http://prometheus", "110", 8)

scripts/ops/tests/test_backup_health_textfile_exporter.py

@@ -248,6 +248,45 @@ def test_backup_alert_receipt_metrics_require_all_stages(tmp_path: Path, monkeyp
     )
 
 
+def test_backup_schedule_singleton_receipt_is_required_without_false_receipt(
+    tmp_path: Path, monkeypatch
+) -> None:
+    exporter = load_exporter()
+    receipt_dir = tmp_path / "alert-receipts"
+    receipt_dir.mkdir()
+    now = 1_782_900_000
+
+    monkeypatch.setattr(exporter, "BACKUP_ALERT_RECEIPT_DIR", receipt_dir)
+    monkeypatch.setattr(exporter, "BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS", 168)
+    monkeypatch.setattr(exporter.time, "time", lambda: now)
+
+    requirement = next(
+        row
+        for row in exporter.BACKUP_ALERT_RECEIPT_REQUIREMENTS
+        if row["alertname"] == "BackupScheduleSingletonMismatch"
+    )
+    assert requirement == {
+        "host": "110",
+        "alertname": "BackupScheduleSingletonMismatch",
+        "scope": "schedule",
+        "notification_type": "TYPE-1",
+    }
+
+    rendered = "\n".join(exporter._backup_alert_receipt_metric_lines("110"))
+    label_prefix = (
+        'host="110",required_alert="BackupScheduleSingletonMismatch",scope="schedule",'
+        'required_notification_type="TYPE-1",receipt_channel="telegram_awooop",max_age_hours="168"'
+    )
+    assert f"awoooi_backup_alert_receipt_expected_info{{{label_prefix}}} 1" in rendered
+    assert f"awoooi_backup_alert_receipt_fresh{{{label_prefix}}} 0" in rendered
+    for stage in exporter.BACKUP_ALERT_RECEIPT_STAGES:
+        assert (
+            f"awoooi_backup_alert_receipt_stage_fresh{{{label_prefix},stage=\"{stage}\"}} 0"
+            in rendered
+        )
+    assert list(receipt_dir.iterdir()) == []
+
+
 def test_backup_alert_receipt_shared_marker_can_satisfy_all_stages(tmp_path: Path, monkeypatch) -> None:
     exporter = load_exporter()
     receipt_dir = tmp_path / "alert-receipts"

scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py

@@ -654,7 +654,7 @@ def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None:
     assert 'alertname!="BackupAlertReceiptStageMissing"' in alerts
     assert "label_replace(" in alerts
     assert 'awoooi_backup_alert_receipt_fresh == 1' in alerts
-    assert "$labels.alertname" in alerts
+    assert "$labels.required_alert" in alerts
 
 
 def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None: