From a535fa652fe9c5041a8dde8f31f415b9c80f5bb0 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 23:34:55 +0800 Subject: [PATCH] fix(backup): preserve alert receipt identity --- .../src/services/platform_operator_service.py | 10 ++- .../test_awooop_operator_timeline_labels.py | 78 ++++++++++++++++ ops/monitoring/alerts-unified.yml | 30 ++++--- .../tests/test_backup_alert_receipt_rules.yml | 45 ++++++++++ .../ops/backup-alert-label-contract-check.py | 21 +++-- .../ops/backup-alert-live-visibility-check.py | 24 ++--- .../ops/backup-health-textfile-exporter.py | 10 ++- .../test_backup_alert_live_visibility.py | 90 +++++++++++++++++++ .../test_backup_health_textfile_exporter.py | 39 ++++++++ .../test_reboot_p0_operational_contract.py | 2 +- 10 files changed, 315 insertions(+), 34 deletions(-) create mode 100644 ops/monitoring/tests/test_backup_alert_receipt_rules.yml create mode 100644 scripts/ops/tests/test_backup_alert_live_visibility.py diff --git a/apps/api/src/services/platform_operator_service.py b/apps/api/src/services/platform_operator_service.py index 3005bd2ee..f80c0e5e0 100644 --- a/apps/api/src/services/platform_operator_service.py +++ b/apps/api/src/services/platform_operator_service.py @@ -176,7 +176,10 @@ _AI_ALERT_CARD_LEARNING_REGISTRY_TARGETS: tuple[dict[str, str], ...] = ( "source_field": "ai_agent_context_ref", }, ) -_INCIDENT_ID_RE = re.compile(r"\bINC-\d{8}-[A-Z0-9]{4,}\b") +# Backup/restore ingress 使用穩定 digest ID；只加入精確 16 位大寫 hex 格式。 +_INCIDENT_ID_RE = re.compile( + r"\b(?:INC-\d{8}-[A-Z0-9]{4,}|INC-BRR-[A-F0-9]{16})\b" +) _REMEDIATION_STATUS_FILTERS = { "mcp_observed", "no_evidence", @@ -7671,7 +7674,10 @@ def _validate_incident_id_filter(value: str | None) -> None: if not _INCIDENT_ID_RE.fullmatch(value): raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail="incident_id 格式錯誤，必須是 INC-YYYYMMDD-XXXX", + detail=( + "incident_id 格式錯誤，必須是 INC-YYYYMMDD-XXXX " + "或 INC-BRR-<16 位大寫十六進位字元>" + ), ) diff --git a/apps/api/tests/test_awooop_operator_timeline_labels.py b/apps/api/tests/test_awooop_operator_timeline_labels.py index bc77f85a3..cd2d185d6 100644 --- a/apps/api/tests/test_awooop_operator_timeline_labels.py +++ b/apps/api/tests/test_awooop_operator_timeline_labels.py @@ -63,6 +63,7 @@ from src.services.platform_operator_service import ( _validate_callback_reply_status_filter, _validate_cicd_stage_filter, _validate_cicd_status_filter, + _validate_incident_id_filter, ) @@ -3859,6 +3860,83 @@ def test_remediation_summary_matches_incident_id_filter() -> None: assert _remediation_summary_matches_incident_id(None, None) +@pytest.mark.parametrize( + "incident_id", + [ + "INC-20260714-A1B2", + "INC-20260714-A1B2C3D4", + "INC-BRR-544862858131AD18", + "INC-BRR-0123456789ABCDEF", + ], +) +def test_validate_incident_id_filter_accepts_legacy_and_backup_restore_ids( + incident_id: str, +) -> None: + _validate_incident_id_filter(incident_id) + + +@pytest.mark.parametrize( + "incident_id", + [ + "INC-BRR-544862858131AD1", + "INC-BRR-544862858131AD188", + "INC-BRR-544862858131ad18", + "INC-BRR-544862858131ZZ18", + "INC-BRR-544862858131AD18-extra", + "BRR-544862858131AD18", + "arbitrary-incident", + ], +) +def test_validate_incident_id_filter_rejects_noncanonical_backup_restore_ids( + incident_id: str, +) -> None: + with pytest.raises(HTTPException) as exc_info: + _validate_incident_id_filter(incident_id) + + assert exc_info.value.status_code == 422 + + +@pytest.mark.asyncio +async def test_status_chain_queries_existing_backup_restore_incident( + monkeypatch, +) -> None: + incident_id = "INC-BRR-544862858131AD18" + observed: dict[str, object] = {} + + async def fake_history(incident_ids, *, limit): + observed["history"] = (incident_ids, limit) + return [] + + async def fake_status_chain(*, incident_ids, project_id, remediation_history): + observed["fetch"] = (incident_ids, project_id, remediation_history) + return { + "schema_version": "awooop_status_chain_v1", + "source_id": incident_ids[0], + } + + monkeypatch.setattr( + platform_operator_service, + "_fetch_run_remediation_history", + fake_history, + ) + monkeypatch.setattr( + platform_operator_service, + "_fetch_awooop_status_chain", + fake_status_chain, + ) + + result = await platform_operator_service.get_awooop_status_chain( + project_id="awoooi", + incident_ids=[incident_id], + ) + + assert result["source_id"] == incident_id + assert observed == { + "history": ([incident_id], 5), + "fetch": ([incident_id], "awoooi", []), + } + + def test_list_filter_context_limit_scales_with_candidate_rows() -> None: assert _list_filter_context_limit(2) == 500 assert _list_filter_context_limit(4176) == 16704 diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 20b4b368c..253873d5f 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -1721,19 +1721,21 @@ groups: runbook: "部署新版 scripts/ops/backup-health-textfile-exporter.py 到 188，刷新 /home/ollama/node_exporter_textfiles/backup_health.prom；不得送測試 Telegram、不得讀 token、不得 restore 到 production。" - alert: BackupAlertReceiptStageMissing + # Prometheus 會把來源 alertname 覆寫成本規則名稱；先投影為 + # required_alert 再聚合，避免同一 host 多個 firing alert 產生重複 labelset。 expr: | - max by (alertname, host) ( - ALERTS{ - alertstate="firing", - alertname=~"Backup.*|GiteaPrivateBundle.*|GiteaRestoreDrill.*", - alertname!="BackupAlertReceiptStageMissing" - } - ) - unless on (alertname, host) - label_replace( - max by (required_alert, host) (awoooi_backup_alert_receipt_fresh == 1), - "alertname", "$1", "required_alert", "(.*)" + max by (required_alert, host) ( + label_replace( + ALERTS{ + alertstate="firing", + alertname=~"Backup.*|GiteaPrivateBundle.*|GiteaRestoreDrill.*", + alertname!="BackupAlertReceiptStageMissing" + }, + "required_alert", "$1", "alertname", "(.*)" + ) ) + unless on (required_alert, host) + max by (required_alert, host) (awoooi_backup_alert_receipt_fresh == 1) for: 30m labels: severity: warning @@ -1744,9 +1746,9 @@ groups: notification_type: TYPE-1 auto_repair: "false" annotations: - summary: "{{ $labels.host }} 正在發生的備份告警尚未完成 receipt 閉環" - description: "{{ $labels.alertname }} 正在 firing，但尚無新鮮的 Telegram/AwoooP sent/received/dedup/escalation 完整 receipt；未 firing 的規則不再產生假告警。" - runbook: "查 awoooi_backup_alert_receipt_stage_fresh{host=\"{{ $labels.host }}\",required_alert=\"{{ $labels.alertname }}\"} 取得缺少階段；只補 timestamp/evidence id，不得寫入 token、chat raw payload 或 secret。" + summary: "{{ $labels.host }} 正在發生的備份告警 {{ $labels.required_alert }} 尚未完成 receipt 閉環" + description: "{{ $labels.required_alert }} 正在 firing，但尚無新鮮的 Telegram/AwoooP sent/received/dedup/escalation 完整 receipt；未 firing 的規則不再產生假告警。" + runbook: "查 awoooi_backup_alert_receipt_stage_fresh{host=\"{{ $labels.host }}\",required_alert=\"{{ $labels.required_alert }}\"} 取得缺少階段；只補 timestamp/evidence id，不得寫入 token、chat raw payload 或 secret。" - alert: BackupAggregateRunFailed expr: sum by (host) (awoooi_backup_last_run_failed_count{host="110",exported_job!="backup_all"}) > 0 diff --git a/ops/monitoring/tests/test_backup_alert_receipt_rules.yml b/ops/monitoring/tests/test_backup_alert_receipt_rules.yml new file mode 100644 index 000000000..1363c29e7 --- /dev/null +++ b/ops/monitoring/tests/test_backup_alert_receipt_rules.yml @@ -0,0 +1,45 @@ +rule_files: + - ../alerts-unified.yml + +evaluation_interval: 1m + +tests: + - name: backup receipt alert preserves each required alert identity + interval: 1m + input_series: + - series: 'ALERTS{alertname="BackupScheduleSingletonMismatch",alertstate="firing",host="110"}' + values: '1x40' + - series: 'ALERTS{alertname="BackupRetentionPolicyNotLatestOnly",alertstate="firing",host="110"}' + values: '1x40' + alert_rule_test: + - eval_time: 31m + alertname: BackupAlertReceiptStageMissing + exp_alerts: + - exp_labels: + alert_category: infrastructure + auto_repair: "false" + component: backup-alert-receipt + host: "110" + layer: host-backup + notification_type: TYPE-1 + required_alert: BackupScheduleSingletonMismatch + severity: warning + team: ops + exp_annotations: + summary: "110 正在發生的備份告警 BackupScheduleSingletonMismatch 尚未完成 receipt 閉環" + description: "BackupScheduleSingletonMismatch 正在 firing，但尚無新鮮的 Telegram/AwoooP sent/received/dedup/escalation 完整 receipt；未 firing 的規則不再產生假告警。" + runbook: '查 awoooi_backup_alert_receipt_stage_fresh{host="110",required_alert="BackupScheduleSingletonMismatch"} 取得缺少階段；只補 timestamp/evidence id，不得寫入 token、chat raw payload 或 secret。' + - exp_labels: + alert_category: infrastructure + auto_repair: "false" + component: backup-alert-receipt + host: "110" + layer: host-backup + notification_type: TYPE-1 + required_alert: BackupRetentionPolicyNotLatestOnly + severity: warning + team: ops + exp_annotations: + summary: "110 正在發生的備份告警 BackupRetentionPolicyNotLatestOnly 尚未完成 receipt 閉環" + description: "BackupRetentionPolicyNotLatestOnly 正在 firing，但尚無新鮮的 Telegram/AwoooP sent/received/dedup/escalation 完整 receipt；未 firing 的規則不再產生假告警。" + runbook: '查 awoooi_backup_alert_receipt_stage_fresh{host="110",required_alert="BackupRetentionPolicyNotLatestOnly"} 取得缺少階段；只補 timestamp/evidence id，不得寫入 token、chat raw payload 或 secret。' diff --git a/scripts/ops/backup-alert-label-contract-check.py b/scripts/ops/backup-alert-label-contract-check.py index 3fcfb658b..56143a1da 100755 --- a/scripts/ops/backup-alert-label-contract-check.py +++ b/scripts/ops/backup-alert-label-contract-check.py @@ -147,15 +147,22 @@ def static_check(path: Path, baseline_path: Path) -> list[str]: rule = _require_alert(alerts, "BackupAlertReceiptStageMissing") expr = str(rule.get("expr", "")) for required_fragment in [ + "max by (required_alert, host) (", + "label_replace(", "ALERTS{", 'alertstate="firing"', 'alertname=~"Backup.*|GiteaPrivateBundle.*|GiteaRestoreDrill.*"', 'alertname!="BackupAlertReceiptStageMissing"', - "unless on (alertname, host)", - "label_replace(", - "awoooi_backup_alert_receipt_fresh == 1", + '"required_alert", "$1", "alertname", "(.*)"', + "unless on (required_alert, host)", + "max by (required_alert, host) (awoooi_backup_alert_receipt_fresh == 1)", ]: _require_contains(expr, required_fragment, "BackupAlertReceiptStageMissing expr") + _require_not_contains( + expr, + "unless on (alertname, host)", + "BackupAlertReceiptStageMissing expr", + ) _require_not_contains( expr, "sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == bool 0) > 0", @@ -164,10 +171,14 @@ def static_check(path: Path, baseline_path: Path) -> list[str]: text = _annotation_text(rule) for required_label in [ "$labels.host", - "$labels.alertname", - "required_alert", + "$labels.required_alert", ]: _require_contains(text, required_label, "BackupAlertReceiptStageMissing annotations") + _require_not_contains( + text, + "$labels.alertname", + "BackupAlertReceiptStageMissing annotations", + ) lines.append("OK BackupAlertReceiptStageMissing requires receipts only for actively firing backup/Gitea alerts") return lines diff --git a/scripts/ops/backup-alert-live-visibility-check.py b/scripts/ops/backup-alert-live-visibility-check.py index a35f94f25..1b5f1fd1e 100755 --- a/scripts/ops/backup-alert-live-visibility-check.py +++ b/scripts/ops/backup-alert-live-visibility-check.py @@ -188,35 +188,37 @@ def _escrow_required_alerts(prometheus_url: str, host: str, timeout: int) -> lis def _receipt_stage_required_alerts(prometheus_url: str, host: str, timeout: int) -> list[RequiredAlert]: + # 直接從已 firing 的 meta alert 讀取 required_alert，再對 Prometheus + # 與 Alertmanager 做同一 identity 核對。不得從缺 marker 推導假 receipt。 expr = ( - "sum by (host, receipt_channel) " - f'(awoooi_backup_alert_receipt_stage_fresh{{host="{host}"}} == bool 0) > 0' + 'ALERTS{alertname="BackupAlertReceiptStageMissing",alertstate="firing",' + f'host="{host}"}}' ) rows = _prom_query(prometheus_url, expr, timeout) required: list[RequiredAlert] = [] for row in rows: labels = _metric_labels(row) - receipt_channel = labels.get("receipt_channel") + required_alert = labels.get("required_alert") missing = [ label for label, value in { - "receipt_channel": receipt_channel, + "required_alert": required_alert, }.items() if not value ] if missing: - raise VisibilityError(f"Backup alert receipt gap metric missing labels {missing}: {row}") + raise VisibilityError(f"Backup alert receipt meta alert missing labels {missing}: {row}") required.append( RequiredAlert( "BackupAlertReceiptStageMissing", { **COMMON_LABELS, "host": host, - "receipt_channel": str(receipt_channel), + "required_alert": str(required_alert), }, ) ) - return sorted(required, key=lambda alert: alert.labels["receipt_channel"]) + return sorted(required, key=lambda alert: alert.labels["required_alert"]) def live_check(prometheus_url: str, alertmanager_url: str, host: str, timeout: int) -> list[str]: @@ -241,12 +243,12 @@ def live_check(prometheus_url: str, alertmanager_url: str, host: str, timeout: i required_alerts.extend(receipt_stage_alerts) if receipt_stage_alerts: lines.append( - "OK backup alert receipt gap metrics require " - f"{len(receipt_stage_alerts)} aggregated alert(s): " - + ", ".join(alert.labels["receipt_channel"] for alert in receipt_stage_alerts) + "OK firing backup alert receipt gaps preserve " + f"{len(receipt_stage_alerts)} required alert identity label(s): " + + ", ".join(alert.labels["required_alert"] for alert in receipt_stage_alerts) ) else: - lines.append("OK backup alert receipt markers are fresh; no receipt stage alert required") + lines.append("OK no BackupAlertReceiptStageMissing alert is currently firing") prom_alerts = _prom_alerts(prometheus_url, timeout) for required in required_alerts: diff --git a/scripts/ops/backup-health-textfile-exporter.py b/scripts/ops/backup-health-textfile-exporter.py index 77572b77c..05379b4bd 100755 --- a/scripts/ops/backup-health-textfile-exporter.py +++ b/scripts/ops/backup-health-textfile-exporter.py @@ -67,6 +67,13 @@ BACKUP_ALERT_RECEIPT_REQUIREMENTS = [ {"host": "188", "alertname": "BackupHealthMonitorMissing188", "scope": "monitor"}, {"host": "110", "alertname": "BackupHealthMonitorStale", "scope": "monitor"}, {"host": "110", "alertname": "BackupExpectedJobMissing", "scope": "schedule"}, + # 只宣告 requirement；沒有真實收據 marker 時 fresh 必須維持 0。 + { + "host": "110", + "alertname": "BackupScheduleSingletonMismatch", + "scope": "schedule", + "notification_type": "TYPE-1", + }, {"host": "110", "alertname": "BackupScriptMissing", "scope": "script"}, {"host": "110", "alertname": "BackupJobStale", "scope": "freshness"}, {"host": "110", "alertname": "BackupCoverageDomainStale", "scope": "coverage"}, @@ -160,11 +167,12 @@ def _backup_alert_receipt_metric_lines(host: str) -> list[str]: continue alertname = requirement["alertname"] scope = requirement["scope"] + notification_type = requirement.get("notification_type", "TYPE-3") base_labels = ( f'host="{_escape_label(host)}",' f'required_alert="{_escape_label(alertname)}",' f'scope="{_escape_label(scope)}",' - 'required_notification_type="TYPE-3",' + f'required_notification_type="{_escape_label(notification_type)}",' 'receipt_channel="telegram_awooop",' f'max_age_hours="{BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS:g}"' ) diff --git a/scripts/ops/tests/test_backup_alert_live_visibility.py b/scripts/ops/tests/test_backup_alert_live_visibility.py new file mode 100644 index 000000000..9e7f1e745 --- /dev/null +++ b/scripts/ops/tests/test_backup_alert_live_visibility.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + + +SCRIPT_ROOT = Path(__file__).resolve().parents[1] +CHECK_PATH = SCRIPT_ROOT / "backup-alert-live-visibility-check.py" + + +def load_check(): + spec = importlib.util.spec_from_file_location( + "backup_alert_live_visibility_check", + CHECK_PATH, + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_receipt_verifier_preserves_each_required_alert_identity(monkeypatch) -> None: + check = load_check() + observed: dict[str, object] = {} + + def fake_query(base_url: str, expr: str, timeout: int): + observed.update(base_url=base_url, expr=expr, timeout=timeout) + return [ + { + "metric": { + "alertname": "BackupAlertReceiptStageMissing", + "alertstate": "firing", + "host": "110", + "required_alert": "BackupScheduleSingletonMismatch", + }, + "value": [1_783_000_000, "1"], + }, + { + "metric": { + "alertname": "BackupAlertReceiptStageMissing", + "alertstate": "firing", + "host": "110", + "required_alert": "BackupRetentionPolicyNotLatestOnly", + }, + "value": [1_783_000_000, "1"], + }, + ] + + monkeypatch.setattr(check, "_prom_query", fake_query) + + required = check._receipt_stage_required_alerts("http://prometheus", "110", 8) + + assert observed == { + "base_url": "http://prometheus", + "expr": ( + 'ALERTS{alertname="BackupAlertReceiptStageMissing",alertstate="firing",' + 'host="110"}' + ), + "timeout": 8, + } + assert [alert.labels["required_alert"] for alert in required] == [ + "BackupRetentionPolicyNotLatestOnly", + "BackupScheduleSingletonMismatch", + ] + assert all("receipt_channel" not in alert.labels for alert in required) + + +def test_receipt_verifier_rejects_meta_alert_without_required_alert(monkeypatch) -> None: + check = load_check() + monkeypatch.setattr( + check, + "_prom_query", + lambda *_args: [ + { + "metric": { + "alertname": "BackupAlertReceiptStageMissing", + "alertstate": "firing", + "host": "110", + }, + "value": [1_783_000_000, "1"], + } + ], + ) + + with pytest.raises(check.VisibilityError, match="required_alert"): + check._receipt_stage_required_alerts("http://prometheus", "110", 8) diff --git a/scripts/ops/tests/test_backup_health_textfile_exporter.py b/scripts/ops/tests/test_backup_health_textfile_exporter.py index 06b163a1a..20834a1b8 100644 --- a/scripts/ops/tests/test_backup_health_textfile_exporter.py +++ b/scripts/ops/tests/test_backup_health_textfile_exporter.py @@ -248,6 +248,45 @@ def test_backup_alert_receipt_metrics_require_all_stages(tmp_path: Path, monkeyp ) +def test_backup_schedule_singleton_receipt_is_required_without_false_receipt( + tmp_path: Path, monkeypatch +) -> None: + exporter = load_exporter() + receipt_dir = tmp_path / "alert-receipts" + receipt_dir.mkdir() + now = 1_782_900_000 + + monkeypatch.setattr(exporter, "BACKUP_ALERT_RECEIPT_DIR", receipt_dir) + monkeypatch.setattr(exporter, "BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS", 168) + monkeypatch.setattr(exporter.time, "time", lambda: now) + + requirement = next( + row + for row in exporter.BACKUP_ALERT_RECEIPT_REQUIREMENTS + if row["alertname"] == "BackupScheduleSingletonMismatch" + ) + assert requirement == { + "host": "110", + "alertname": "BackupScheduleSingletonMismatch", + "scope": "schedule", + "notification_type": "TYPE-1", + } + + rendered = "\n".join(exporter._backup_alert_receipt_metric_lines("110")) + label_prefix = ( + 'host="110",required_alert="BackupScheduleSingletonMismatch",scope="schedule",' + 'required_notification_type="TYPE-1",receipt_channel="telegram_awooop",max_age_hours="168"' + ) + assert f"awoooi_backup_alert_receipt_expected_info{{{label_prefix}}} 1" in rendered + assert f"awoooi_backup_alert_receipt_fresh{{{label_prefix}}} 0" in rendered + for stage in exporter.BACKUP_ALERT_RECEIPT_STAGES: + assert ( + f"awoooi_backup_alert_receipt_stage_fresh{{{label_prefix},stage=\"{stage}\"}} 0" + in rendered + ) + assert list(receipt_dir.iterdir()) == [] + + def test_backup_alert_receipt_shared_marker_can_satisfy_all_stages(tmp_path: Path, monkeypatch) -> None: exporter = load_exporter() receipt_dir = tmp_path / "alert-receipts" diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index dcfec31b8..f4612c7e5 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -654,7 +654,7 @@ def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None: assert 'alertname!="BackupAlertReceiptStageMissing"' in alerts assert "label_replace(" in alerts assert 'awoooi_backup_alert_receipt_fresh == 1' in alerts - assert "$labels.alertname" in alerts + assert "$labels.required_alert" in alerts def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None: