fix(ops): close Gitea backup restore verification

This commit is contained in:
ogt
2026-07-10 20:26:00 +08:00
parent 866811bbb3
commit 872a3ed5eb
22 changed files with 1460 additions and 71 deletions

View File

@@ -0,0 +1,56 @@
from __future__ import annotations
import subprocess
import sys
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[3]
CHECKER = ROOT / "scripts" / "ops" / "backup-alert-label-contract-check.py"
RULES = ROOT / "ops" / "monitoring" / "alerts-unified.yml"
BASELINE = ROOT / "ops" / "reboot-recovery" / "full-stack-backup-baseline.yml"
def _run(rules: Path) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[
sys.executable,
str(CHECKER),
"--rules",
str(rules),
"--baseline",
str(BASELINE),
],
cwd=ROOT,
capture_output=True,
text=True,
check=False,
)
def test_checker_accepts_firing_only_receipt_contract() -> None:
result = _run(RULES)
assert result.returncode == 0, result.stderr
assert "requires receipts only for actively firing backup/Gitea alerts" in result.stdout
assert "BACKUP_ALERT_LABEL_CONTRACT_OK" in result.stdout
def test_checker_rejects_non_firing_receipt_selector(tmp_path: Path) -> None:
payload = yaml.safe_load(RULES.read_text(encoding="utf-8"))
for group in payload["groups"]:
for rule in group.get("rules", []):
if rule.get("alert") == "BackupAlertReceiptStageMissing":
rule["expr"] = str(rule["expr"]).replace(
'alertstate="firing"',
'alertstate="pending"',
)
candidate = tmp_path / "alerts.yml"
candidate.write_text(yaml.safe_dump(payload, allow_unicode=True, sort_keys=False), encoding="utf-8")
result = _run(candidate)
assert result.returncode != 0
assert 'alertstate="firing"' in result.stderr

View File

@@ -35,6 +35,50 @@ def test_default_gitea_bundle_expected_repos_cover_current_project_set() -> None
"wooo/bitan-pharmacy",
"wooo/vtuber",
]
assert any(
row["alertname"] == "GiteaRestoreDrillMissingOrFailed"
for row in exporter.BACKUP_ALERT_RECEIPT_REQUIREMENTS
)
def test_gitea_restore_drill_metrics_require_inventory_and_artifacts(tmp_path: Path, monkeypatch) -> None:
exporter = load_exporter()
status_file = tmp_path / "gitea-restore-drill.status"
status_file.write_text(
"\n".join(
[
"timestamp=2000000000",
"success=1",
"reason=verified",
"dump_size_bytes=6928031514",
"backup_regular_repo_count=15",
"backup_wiki_repo_count=0",
"live_regular_repo_count=15",
"live_wiki_repo_count=0",
"database_dump_present=1",
"config_present=2",
"archive_crc_ok=1",
"inventory_digest_match=1",
]
)
+ "\n",
encoding="utf-8",
)
monkeypatch.setattr(exporter, "GITEA_RESTORE_DRILL_STATUS_FILE", status_file)
monkeypatch.setattr(exporter.time, "time", lambda: 2000000060)
rendered = "\n".join(exporter._gitea_restore_drill_metric_lines("110"))
assert 'awoooi_gitea_restore_drill_fresh{host="110",max_age_hours="744"} 1' in rendered
assert 'awoooi_gitea_restore_drill_verified{host="110",max_age_hours="744"} 1' in rendered
assert 'awoooi_gitea_restore_drill_inventory_match{host="110",max_age_hours="744"} 1' in rendered
status_file.write_text(
status_file.read_text(encoding="utf-8").replace("inventory_digest_match=1", "inventory_digest_match=0"),
encoding="utf-8",
)
rendered = "\n".join(exporter._gitea_restore_drill_metric_lines("110"))
assert 'awoooi_gitea_restore_drill_fresh{host="110",max_age_hours="744"} 0' in rendered
assert 'awoooi_gitea_restore_drill_verified{host="110",max_age_hours="744"} 0' in rendered
def test_gitea_bundle_metrics_require_all_expected_repos(tmp_path: Path, monkeypatch) -> None: