fix(ops): close Gitea backup restore verification
This commit is contained in:
56
scripts/ops/tests/test_backup_alert_label_contract.py
Normal file
56
scripts/ops/tests/test_backup_alert_label_contract.py
Normal file
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
CHECKER = ROOT / "scripts" / "ops" / "backup-alert-label-contract-check.py"
|
||||
RULES = ROOT / "ops" / "monitoring" / "alerts-unified.yml"
|
||||
BASELINE = ROOT / "ops" / "reboot-recovery" / "full-stack-backup-baseline.yml"
|
||||
|
||||
|
||||
def _run(rules: Path) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(CHECKER),
|
||||
"--rules",
|
||||
str(rules),
|
||||
"--baseline",
|
||||
str(BASELINE),
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def test_checker_accepts_firing_only_receipt_contract() -> None:
|
||||
result = _run(RULES)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "requires receipts only for actively firing backup/Gitea alerts" in result.stdout
|
||||
assert "BACKUP_ALERT_LABEL_CONTRACT_OK" in result.stdout
|
||||
|
||||
|
||||
def test_checker_rejects_non_firing_receipt_selector(tmp_path: Path) -> None:
|
||||
payload = yaml.safe_load(RULES.read_text(encoding="utf-8"))
|
||||
for group in payload["groups"]:
|
||||
for rule in group.get("rules", []):
|
||||
if rule.get("alert") == "BackupAlertReceiptStageMissing":
|
||||
rule["expr"] = str(rule["expr"]).replace(
|
||||
'alertstate="firing"',
|
||||
'alertstate="pending"',
|
||||
)
|
||||
candidate = tmp_path / "alerts.yml"
|
||||
candidate.write_text(yaml.safe_dump(payload, allow_unicode=True, sort_keys=False), encoding="utf-8")
|
||||
|
||||
result = _run(candidate)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert 'alertstate="firing"' in result.stderr
|
||||
@@ -35,6 +35,50 @@ def test_default_gitea_bundle_expected_repos_cover_current_project_set() -> None
|
||||
"wooo/bitan-pharmacy",
|
||||
"wooo/vtuber",
|
||||
]
|
||||
assert any(
|
||||
row["alertname"] == "GiteaRestoreDrillMissingOrFailed"
|
||||
for row in exporter.BACKUP_ALERT_RECEIPT_REQUIREMENTS
|
||||
)
|
||||
|
||||
|
||||
def test_gitea_restore_drill_metrics_require_inventory_and_artifacts(tmp_path: Path, monkeypatch) -> None:
|
||||
exporter = load_exporter()
|
||||
status_file = tmp_path / "gitea-restore-drill.status"
|
||||
status_file.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"timestamp=2000000000",
|
||||
"success=1",
|
||||
"reason=verified",
|
||||
"dump_size_bytes=6928031514",
|
||||
"backup_regular_repo_count=15",
|
||||
"backup_wiki_repo_count=0",
|
||||
"live_regular_repo_count=15",
|
||||
"live_wiki_repo_count=0",
|
||||
"database_dump_present=1",
|
||||
"config_present=2",
|
||||
"archive_crc_ok=1",
|
||||
"inventory_digest_match=1",
|
||||
]
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(exporter, "GITEA_RESTORE_DRILL_STATUS_FILE", status_file)
|
||||
monkeypatch.setattr(exporter.time, "time", lambda: 2000000060)
|
||||
|
||||
rendered = "\n".join(exporter._gitea_restore_drill_metric_lines("110"))
|
||||
assert 'awoooi_gitea_restore_drill_fresh{host="110",max_age_hours="744"} 1' in rendered
|
||||
assert 'awoooi_gitea_restore_drill_verified{host="110",max_age_hours="744"} 1' in rendered
|
||||
assert 'awoooi_gitea_restore_drill_inventory_match{host="110",max_age_hours="744"} 1' in rendered
|
||||
|
||||
status_file.write_text(
|
||||
status_file.read_text(encoding="utf-8").replace("inventory_digest_match=1", "inventory_digest_match=0"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
rendered = "\n".join(exporter._gitea_restore_drill_metric_lines("110"))
|
||||
assert 'awoooi_gitea_restore_drill_fresh{host="110",max_age_hours="744"} 0' in rendered
|
||||
assert 'awoooi_gitea_restore_drill_verified{host="110",max_age_hours="744"} 0' in rendered
|
||||
|
||||
|
||||
def test_gitea_bundle_metrics_require_all_expected_repos(tmp_path: Path, monkeypatch) -> None:
|
||||
|
||||
Reference in New Issue
Block a user