wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(ops): close Gitea backup restore verification

Browse Source
This commit is contained in:
ogt
2026-07-10 20:26:00 +08:00
parent 866811bbb3
commit 872a3ed5eb
22 changed files with 1460 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
scripts/ops/backup-alert-label-contract-check.py
View File

@@ -6,6 +6,9 @@ Node exporter textfile metrics use labels such as job="backup_all" locally, but
Prometheus rewrites that metric label to exported_job because the scrape target
already has job="node-exporter-110". Backup alerts must therefore use
$labels.exported_job in user-facing text and exported_job="..." in expressions.
Receipt alerts must also be demand-driven: only an actively firing backup or
Gitea alert may require a fresh receipt. Missing markers for inactive rules are
inventory evidence, not user-facing incidents.
"""
from __future__ import annotations
@@ -142,20 +145,30 @@ def static_check(path: Path, baseline_path: Path) -> list[str]:
lines.append("OK BackupAlertReceiptMetricMissing188 checks 188 receipt requirement metric")
rule = _require_alert(alerts, "BackupAlertReceiptStageMissing")
_require_contains(
str(rule.get("expr", "")),
expr = str(rule.get("expr", ""))
for required_fragment in [
"ALERTS{",
'alertstate="firing"',
'alertname=~"Backup.*|GiteaPrivateBundle.*|GiteaRestoreDrill.*"',
'alertname!="BackupAlertReceiptStageMissing"',
"unless on (alertname, host)",
"label_replace(",
"awoooi_backup_alert_receipt_fresh == 1",
]:
_require_contains(expr, required_fragment, "BackupAlertReceiptStageMissing expr")
_require_not_contains(
expr,
"sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == bool 0) > 0",
"BackupAlertReceiptStageMissing expr",
)
text = _annotation_text(rule)
for required_label in [
"$labels.host",
"$labels.receipt_channel",
"$value",
"required_alert/scope/stage/required_notification_type",
"$labels.alertname",
"required_alert",
]:
_require_contains(text, required_label, "BackupAlertReceiptStageMissing annotations")
lines.append("OK BackupAlertReceiptStageMissing aggregates by host/receipt channel and points to detail labels")
lines.append("OK BackupAlertReceiptStageMissing requires receipts only for actively firing backup/Gitea alerts")
return lines

95
scripts/ops/backup-health-textfile-exporter.py
View File

@@ -34,6 +34,10 @@ BACKUP_OFFSITE_ENV = Path(os.environ.get("AIOPS_BACKUP_OFFSITE_ENV", "/backup/sc
OFFSITE_STATUS_DIR = Path(os.environ.get("AIOPS_OFFSITE_STATUS_DIR", "/backup/offsite"))
ESCROW_EVIDENCE_DIR = Path(os.environ.get("AIOPS_ESCROW_EVIDENCE_DIR", "/backup/escrow-evidence"))
CONFIG_CAPTURE_STATUS_FILE = Path(os.environ.get("AIOPS_CONFIG_CAPTURE_STATUS_FILE", "/backup/status/backup-configs-last-status.json"))
GITEA_RESTORE_DRILL_STATUS_FILE = Path(
os.environ.get("AIOPS_GITEA_RESTORE_DRILL_STATUS_FILE", "/backup/integrity/gitea-restore-drill.status")
)
GITEA_RESTORE_DRILL_MAX_AGE_HOURS = float(os.environ.get("AIOPS_GITEA_RESTORE_DRILL_MAX_AGE_HOURS", "744"))
BACKUP_ALERT_RECEIPT_DIR = Path(os.environ.get("AIOPS_BACKUP_ALERT_RECEIPT_DIR", "/backup/alert-receipts"))
BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS = float(os.environ.get("AIOPS_BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS", "168"))
DEFAULT_GITEA_BUNDLE_EXPECTED_REPOS = [
@@ -73,6 +77,7 @@ BACKUP_ALERT_RECEIPT_REQUIREMENTS = [
{"host": "110", "alertname": "BackupConfigCaptureStatusStale", "scope": "config_capture"},
{"host": "110", "alertname": "BackupIntegrityCheckMissingOrFailed", "scope": "integrity"},
{"host": "110", "alertname": "BackupRestoreDrillMissingOrFailed", "scope": "restore_drill"},
{"host": "110", "alertname": "GiteaRestoreDrillMissingOrFailed", "scope": "gitea_restore_drill"},
{"host": "110", "alertname": "BackupRestoreTestMissing", "scope": "velero_restore_test"},
{"host": "110", "alertname": "BackupRestoreTestCronMissing", "scope": "velero_restore_test"},
{"host": "110", "alertname": "BackupRestoreTestFailed", "scope": "velero_restore_test"},
@@ -573,6 +578,46 @@ def _integrity_metric_lines(host: str) -> list[str]:
return lines
def _gitea_restore_drill_metric_lines(host: str) -> list[str]:
values = _read_key_value_status(str(GITEA_RESTORE_DRILL_STATUS_FILE))
now = int(time.time())
timestamp = int(values.get("timestamp", 0))
success = int(values.get("success", 0))
dump_size = int(values.get("dump_size_bytes", 0))
backup_regular = int(values.get("backup_regular_repo_count", 0))
backup_wiki = int(values.get("backup_wiki_repo_count", 0))
live_regular = int(values.get("live_regular_repo_count", 0))
live_wiki = int(values.get("live_wiki_repo_count", 0))
database_present = int(values.get("database_dump_present", 0))
config_present = int(values.get("config_present", 0))
crc_ok = int(values.get("archive_crc_ok", 0))
inventory_match = int(values.get("inventory_digest_match", 0))
age = max(0, now - timestamp) if timestamp else 0
count_match = int(backup_regular == live_regular and backup_wiki == live_wiki and backup_regular > 0)
required_artifacts_ok = int(database_present > 0 and config_present > 0)
verified = int(success == 1 and crc_ok == 1 and required_artifacts_ok == 1 and count_match == 1 and inventory_match == 1)
fresh = int(timestamp > 0 and age <= int(GITEA_RESTORE_DRILL_MAX_AGE_HOURS * 3600) and verified == 1)
reason = str(values.get("reason", "status_missing"))
labels = f'host="{_escape_label(host)}",max_age_hours="{GITEA_RESTORE_DRILL_MAX_AGE_HOURS:g}"'
return [
f'awoooi_gitea_restore_drill_status_info{{{labels},reason="{_escape_label(reason)}"}} 1',
f"awoooi_gitea_restore_drill_last_success_timestamp{{{labels}}} {timestamp if verified else 0}",
f"awoooi_gitea_restore_drill_age_seconds{{{labels}}} {age}",
f"awoooi_gitea_restore_drill_fresh{{{labels}}} {fresh}",
f"awoooi_gitea_restore_drill_verified{{{labels}}} {verified}",
f"awoooi_gitea_restore_drill_archive_crc_ok{{{labels}}} {crc_ok}",
f"awoooi_gitea_restore_drill_required_artifacts_ok{{{labels}}} {required_artifacts_ok}",
f"awoooi_gitea_restore_drill_repo_count_match{{{labels}}} {count_match}",
f"awoooi_gitea_restore_drill_inventory_match{{{labels}}} {inventory_match}",
f"awoooi_gitea_restore_drill_dump_size_bytes{{{labels}}} {dump_size}",
f'awoooi_gitea_restore_drill_repo_count{{{labels},source="backup",kind="regular"}} {backup_regular}',
f'awoooi_gitea_restore_drill_repo_count{{{labels},source="backup",kind="wiki"}} {backup_wiki}',
f'awoooi_gitea_restore_drill_repo_count{{{labels},source="live",kind="regular"}} {live_regular}',
f'awoooi_gitea_restore_drill_repo_count{{{labels},source="live",kind="wiki"}} {live_wiki}',
]
def _config_capture_metric_lines(host: str) -> list[str]:
now = int(time.time())
labels = f'host="{_escape_label(host)}"'
@@ -975,6 +1020,28 @@ def _base_lines(host: str) -> list[str]:
"# TYPE awoooi_backup_integrity_failed_repo_count gauge",
"# HELP awoooi_backup_integrity_checked_repo_count Checked repository count from backup integrity or restore drill run.",
"# TYPE awoooi_backup_integrity_checked_repo_count gauge",
"# HELP awoooi_gitea_restore_drill_status_info Bounded result reason from the isolated Gitea restore drill.",
"# TYPE awoooi_gitea_restore_drill_status_info gauge",
"# HELP awoooi_gitea_restore_drill_last_success_timestamp Unix timestamp of the latest fully verified isolated Gitea restore drill.",
"# TYPE awoooi_gitea_restore_drill_last_success_timestamp gauge",
"# HELP awoooi_gitea_restore_drill_age_seconds Age of the latest isolated Gitea restore drill evidence.",
"# TYPE awoooi_gitea_restore_drill_age_seconds gauge",
"# HELP awoooi_gitea_restore_drill_fresh Whether the isolated Gitea restore drill is verified and within max age.",
"# TYPE awoooi_gitea_restore_drill_fresh gauge",
"# HELP awoooi_gitea_restore_drill_verified Whether CRC, required assets, counts, and inventory all passed.",
"# TYPE awoooi_gitea_restore_drill_verified gauge",
"# HELP awoooi_gitea_restore_drill_archive_crc_ok Whether every Gitea dump ZIP member passed CRC verification.",
"# TYPE awoooi_gitea_restore_drill_archive_crc_ok gauge",
"# HELP awoooi_gitea_restore_drill_required_artifacts_ok Whether database dump and configuration are present.",
"# TYPE awoooi_gitea_restore_drill_required_artifacts_ok gauge",
"# HELP awoooi_gitea_restore_drill_repo_count_match Whether backup and live regular/wiki repo counts match.",
"# TYPE awoooi_gitea_restore_drill_repo_count_match gauge",
"# HELP awoooi_gitea_restore_drill_inventory_match Whether privacy-preserving backup and live repo inventories match.",
"# TYPE awoooi_gitea_restore_drill_inventory_match gauge",
"# HELP awoooi_gitea_restore_drill_dump_size_bytes Size of the verified Gitea dump archive.",
"# TYPE awoooi_gitea_restore_drill_dump_size_bytes gauge",
"# HELP awoooi_gitea_restore_drill_repo_count Regular or wiki repo count in backup or live inventory.",
"# TYPE awoooi_gitea_restore_drill_repo_count gauge",
"# HELP awoooi_backup_config_capture_status_timestamp Unix timestamp of the latest config-capture coverage status.",
"# TYPE awoooi_backup_config_capture_status_timestamp gauge",
"# HELP awoooi_backup_config_capture_status_age_seconds Age of the latest config-capture coverage status.",
@@ -1104,6 +1171,7 @@ def _collect_110(host: str) -> list[str]:
"offsite_full_sync_verify": "/backup/scripts/verify-offsite-full-sync.sh --write-textfile",
"backup_integrity_check": "/backup/scripts/check-backup-integrity.sh --mode check",
"backup_restore_drill": "/backup/scripts/check-backup-integrity.sh --mode restore-drill",
"gitea_bundle_sync_188": "/backup/scripts/gitea-bundle-backup-sync-188.sh",
}
for job, pattern in expected_crons.items():
labels = f'host="{_escape_label(host)}",job="{_escape_label(job)}"'
@@ -1124,6 +1192,9 @@ def _collect_110(host: str) -> list[str]:
"verify-offsite-full-sync.sh",
"mark-credential-escrow-verified.sh",
"check-backup-integrity.sh",
"verify-gitea-backup.sh",
"gitea-repo-bundle-backup.sh",
"gitea-bundle-backup-sync-188.sh",
"backup-gitea.sh",
"backup-harbor.sh",
"backup-momo.sh",
@@ -1193,6 +1264,7 @@ def _collect_110(host: str) -> list[str]:
lines.append(f"awoooi_backup_last_run_failed_count{{{labels}}} {failed_count}")
lines.append(f"awoooi_backup_job_last_success_timestamp{{{labels},type=\"aggregate\",source=\"110-cron-log\",target=\"/backup/logs/cron.log\",max_age_hours=\"48\"}} {backup_all_ts if failed_count == 0 else 0}")
lines.extend(_integrity_metric_lines(host))
lines.extend(_gitea_restore_drill_metric_lines(host))
lines.extend(_config_capture_metric_lines(host))
lines.extend(_offsite_and_escrow_metric_lines(host))
lines.extend(_retention_metric_lines(host))
@@ -1215,11 +1287,13 @@ def _collect_188(host: str) -> list[str]:
for script in [
"/home/ollama/bin/backup-from-110.sh",
"/home/ollama/bin/momo-pg-backup.sh",
"/home/ollama/scripts/gitea-bundle-sample-restore-dry-run.sh",
"/home/ollama/awoooi-ops/pg-backup.sh",
]:
labels = f'host="{_escape_label(host)}",script="{_escape_label(Path(script).name)}"'
lines.append(f"awoooi_backup_script_present{{{labels}}} {int(Path(script).exists() and os.access(script, os.X_OK))}")
backup_from_110_ts = _read_backup_110_timestamp()
lines.extend(
_metric_lines_for_job(
host=host,
@@ -1227,12 +1301,13 @@ def _collect_188(host: str) -> list[str]:
source="188-rsync",
target="/home/ollama/backup/110",
backup_type="rsync",
last_success=_read_backup_110_timestamp(),
last_success=backup_from_110_ts,
max_age_hours=25,
sample_count=1,
)
)
gitea_mirror_ts, gitea_mirror_count = _newest_tree_timestamp(Path("/home/ollama/backup/110/gitea"))
_, gitea_mirror_count = _newest_tree_timestamp(Path("/home/ollama/backup/110/gitea"))
gitea_mirror_ts = backup_from_110_ts
gitea_mirror_fresh = 1 if gitea_mirror_ts and int(time.time()) - gitea_mirror_ts <= 25 * 3600 else 0
lines.extend(
_metric_lines_for_job(
@@ -1248,6 +1323,20 @@ def _collect_188(host: str) -> list[str]:
)
gitea_bundle_lines, gitea_bundle_ok = _gitea_bundle_metric_lines(host)
lines.extend(gitea_bundle_lines)
gitea_bundle_ts, gitea_bundle_count = _newest_tree_timestamp(GITEA_BUNDLE_ROOT, max_entries=20000)
gitea_bundle_fresh = 1 if gitea_bundle_ts and int(time.time()) - gitea_bundle_ts <= 25 * 3600 else 0
lines.extend(
_metric_lines_for_job(
host=host,
job="gitea_private_bundle",
source="188-git-bundle",
target=str(GITEA_BUNDLE_ROOT),
backup_type="git_bundle",
last_success=gitea_bundle_ts,
max_age_hours=25,
sample_count=gitea_bundle_count,
)
)
coverage_labels = (
f'host="{_escape_label(host)}",'
'domain="service",'
@@ -1256,7 +1345,7 @@ def _collect_188(host: str) -> list[str]:
lines.append(f"awoooi_backup_coverage_domain_expected_info{{{coverage_labels}}} 1")
lines.append(
"awoooi_backup_coverage_domain_fresh"
f"{{{coverage_labels}}} {1 if gitea_mirror_fresh and gitea_bundle_ok else 0}"
f"{{{coverage_labels}}} {1 if gitea_mirror_fresh and gitea_bundle_fresh and gitea_bundle_ok else 0}"
)
momo_ts = _newest_file_timestamp([
"/home/ollama/momo_backups/*.sql.gz",

138
scripts/ops/prometheus-rule-drift-guard.sh
View File

@@ -13,10 +13,16 @@ CURRENT_RULES="${CURRENT_RULES:-/home/wooo/monitoring/alerts.yml}"
CANONICAL_RULES="${CANONICAL_RULES:-/home/wooo/monitoring/alerts-unified.canonical.yml}"
TEXTFILE="${TEXTFILE:-/home/wooo/node_exporter_textfiles/prometheus_rule_drift_guard.prom}"
LOG_FILE="${LOG_FILE:-/home/wooo/logs/prometheus-rule-drift-guard.log}"
PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
PROMETHEUS_RUNTIME_RULES="${PROMETHEUS_RUNTIME_RULES:-/etc/prometheus/alerts.yml}"
PROMETHEUS_COMPOSE_FILE="${PROMETHEUS_COMPOSE_FILE:-/home/wooo/monitoring/docker-compose.yml}"
PROMETHEUS_COMPOSE_SERVICE="${PROMETHEUS_COMPOSE_SERVICE:-prometheus}"
ALLOW_RUNTIME_RECREATE="${ALLOW_RUNTIME_RECREATE:-1}"
REQUIRED_RULES=(
"BackupCredentialEscrowEvidenceMissing"
"BackupExpectedJobMissing"
"GiteaRestoreDrillMissingOrFailed"
"awoooi_recovery_core_ready"
"awoooi_recovery_dr_offsite_ready"
"ColdStartRecoveryBlocked"
@@ -32,6 +38,7 @@ write_textfile() {
local repaired="$2"
local missing_count="$3"
local matches_canonical="$4"
local runtime_matches_current="$5"
local tmp
mkdir -p "$(dirname "$TEXTFILE")" 2>/dev/null || true
tmp="$(mktemp "${TEXTFILE}.tmp.XXXXXX")" || return 0
@@ -48,6 +55,9 @@ awoooi_prometheus_rule_drift_guard_missing_required_count{host="${HOST_LABEL}"}
# HELP awoooi_prometheus_rule_drift_guard_current_matches_canonical Whether active alerts.yml matches canonical copy.
# TYPE awoooi_prometheus_rule_drift_guard_current_matches_canonical gauge
awoooi_prometheus_rule_drift_guard_current_matches_canonical{host="${HOST_LABEL}"} ${matches_canonical}
# HELP awoooi_prometheus_rule_drift_guard_runtime_matches_current Whether the Prometheus container mount matches the active host rules file.
# TYPE awoooi_prometheus_rule_drift_guard_runtime_matches_current gauge
awoooi_prometheus_rule_drift_guard_runtime_matches_current{host="${HOST_LABEL}"} ${runtime_matches_current}
EOF
chmod 0644 "$tmp" 2>/dev/null || true
mv "$tmp" "$TEXTFILE" 2>/dev/null || rm -f "$tmp"
@@ -67,16 +77,32 @@ try:
if payload.get("status") != "success":
raise RuntimeError(payload)
loaded = {
str(rule.get("name") or rule.get("alert") or rule.get("record"))
str(rule.get("name") or rule.get("alert") or rule.get("record")): rule
for group in payload.get("data", {}).get("groups") or []
for rule in group.get("rules") or []
}
print(len(required - loaded))
missing = required - set(loaded)
unhealthy = {name for name in required & set(loaded) if loaded[name].get("health") != "ok"}
print(len(missing | unhealthy))
except Exception as exc:
print(f"QUERY_FAILED:{exc}")
PY
}
wait_for_required_rules() {
local attempt missing
for attempt in $(seq 1 30); do
missing="$(rules_missing_count)"
if [ "$missing" = "0" ]; then
log "runtime rules converged after attempt=${attempt}"
return 0
fi
sleep 2
done
log "runtime rules did not converge within 60s: ${missing}"
return 1
}
matches_canonical() {
if [ ! -f "$CURRENT_RULES" ] || [ ! -f "$CANONICAL_RULES" ]; then
echo 0
@@ -89,60 +115,136 @@ matches_canonical() {
fi
}
file_sha256() {
sha256sum "$1" 2>/dev/null | awk '{print $1}'
}
runtime_rules_sha256() {
docker exec "${PROMETHEUS_CONTAINER}" sha256sum "${PROMETHEUS_RUNTIME_RULES}" 2>/dev/null | awk '{print $1}'
}
runtime_matches_current() {
local current_hash runtime_hash
current_hash="$(file_sha256 "${CURRENT_RULES}")"
runtime_hash="$(runtime_rules_sha256)"
if [ -n "${current_hash}" ] && [ "${current_hash}" = "${runtime_hash}" ]; then
echo 1
else
echo 0
fi
}
validate_canonical_rules() {
local candidate_path rc
candidate_path="/tmp/awoooi-prometheus-rule-guard-candidate.$$.yml"
docker cp "${CANONICAL_RULES}" "${PROMETHEUS_CONTAINER}:${candidate_path}" >/dev/null 2>&1 || return 1
docker exec "${PROMETHEUS_CONTAINER}" promtool check rules "${candidate_path}" >>"${LOG_FILE}" 2>&1
rc=$?
docker exec -u 0 "${PROMETHEUS_CONTAINER}" rm -f "${candidate_path}" >/dev/null 2>&1 || true
return "${rc}"
}
recreate_prometheus_runtime() {
[ "${ALLOW_RUNTIME_RECREATE}" = "1" ] || return 1
[ -f "${PROMETHEUS_COMPOSE_FILE}" ] || return 1
docker compose -f "${PROMETHEUS_COMPOSE_FILE}" up -d --no-deps --force-recreate \
"${PROMETHEUS_COMPOSE_SERVICE}" >>"${LOG_FILE}" 2>&1
}
restore_rules() {
local backup_path
local backup_path needs_recreate
backup_path="${CURRENT_RULES}.guard.bak.$(date +%Y%m%d%H%M%S)"
cp "$CURRENT_RULES" "$backup_path" 2>/dev/null || true
cp "$CANONICAL_RULES" "$CURRENT_RULES"
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
if ! cp "$CURRENT_RULES" "$backup_path" 2>/dev/null; then
log "failed to create active rules rollback copy"
return 1
fi
if ! validate_canonical_rules; then
log "canonical promtool validation failed"
return 1
fi
if ! cat "$CANONICAL_RULES" >"$CURRENT_RULES"; then
log "failed to update active rules file"
return 1
fi
needs_recreate=0
[ "$(runtime_matches_current)" -eq 1 ] || needs_recreate=1
if [ "${needs_recreate}" -eq 1 ]; then
log "runtime bind mount differs from active file; recreating Prometheus only"
if ! recreate_prometheus_runtime; then
cat "$backup_path" >"$CURRENT_RULES" 2>/dev/null || true
log "Prometheus recreate failed; active file rolled back"
return 1
fi
elif ! curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null; then
cat "$backup_path" >"$CURRENT_RULES" 2>/dev/null || true
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null 2>&1 || true
log "Prometheus reload failed; active file rolled back"
return 1
fi
if wait_for_required_rules && [ "$(runtime_matches_current)" -eq 1 ]; then
return 0
fi
cat "$backup_path" >"$CURRENT_RULES" 2>/dev/null || true
if [ "${needs_recreate}" -eq 1 ]; then
recreate_prometheus_runtime >/dev/null 2>&1 || true
else
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null 2>&1 || true
fi
log "post-apply verifier failed; active rules rolled back"
return 1
}
main() {
if [ ! -f "$CANONICAL_RULES" ]; then
log "canonical rules missing: ${CANONICAL_RULES}"
write_textfile "canonical_missing" 0 999 0
write_textfile "canonical_missing" 0 999 0 0
return 1
fi
local missing before_matches repaired after_missing after_matches
local missing before_matches before_runtime_matches repaired after_missing after_matches after_runtime_matches
missing="$(rules_missing_count)"
before_matches="$(matches_canonical)"
before_runtime_matches="$(runtime_matches_current)"
repaired=0
if [[ "$missing" == QUERY_FAILED:* ]]; then
log "Prometheus query failed: ${missing}"
write_textfile "query_failed" 0 999 "$before_matches"
write_textfile "query_failed" 0 999 "$before_matches" "$before_runtime_matches"
return 1
fi
if [ "$missing" -gt 0 ] || [ "$before_matches" -eq 0 ]; then
log "rule drift detected: missing=${missing} current_matches_canonical=${before_matches}; restoring"
if [ "$missing" -gt 0 ] || [ "$before_matches" -eq 0 ] || [ "$before_runtime_matches" -eq 0 ]; then
log "rule drift detected: missing=${missing} current_matches_canonical=${before_matches} runtime_matches_current=${before_runtime_matches}; restoring"
if restore_rules; then
repaired=1
sleep 3
else
log "restore failed"
write_textfile "restore_failed" 0 "$missing" "$before_matches"
write_textfile "restore_failed" 0 "$missing" "$before_matches" "$before_runtime_matches"
return 1
fi
fi
after_missing="$(rules_missing_count)"
after_matches="$(matches_canonical)"
after_runtime_matches="$(runtime_matches_current)"
if [[ "$after_missing" == QUERY_FAILED:* ]]; then
log "post-restore Prometheus query failed: ${after_missing}"
write_textfile "post_query_failed" "$repaired" 999 "$after_matches"
write_textfile "post_query_failed" "$repaired" 999 "$after_matches" "$after_runtime_matches"
return 1
fi
if [ "$after_missing" -eq 0 ] && [ "$after_matches" -eq 1 ]; then
write_textfile "ok" "$repaired" "$after_missing" "$after_matches"
if [ "$after_missing" -eq 0 ] && [ "$after_matches" -eq 1 ] && [ "$after_runtime_matches" -eq 1 ]; then
write_textfile "ok" "$repaired" "$after_missing" "$after_matches" "$after_runtime_matches"
log "ok repaired=${repaired}"
return 0
fi
log "still drifted after check: missing=${after_missing} current_matches_canonical=${after_matches}"
write_textfile "drifted" "$repaired" "$after_missing" "$after_matches"
log "still drifted after check: missing=${after_missing} current_matches_canonical=${after_matches} runtime_matches_current=${after_runtime_matches}"
write_textfile "drifted" "$repaired" "$after_missing" "$after_matches" "$after_runtime_matches"
return 1
}

56
scripts/ops/tests/test_backup_alert_label_contract.py Normal file
View File

@@ -0,0 +1,56 @@
from __future__ import annotations
import subprocess
import sys
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[3]
CHECKER = ROOT / "scripts" / "ops" / "backup-alert-label-contract-check.py"
RULES = ROOT / "ops" / "monitoring" / "alerts-unified.yml"
BASELINE = ROOT / "ops" / "reboot-recovery" / "full-stack-backup-baseline.yml"
def _run(rules: Path) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[
sys.executable,
str(CHECKER),
"--rules",
str(rules),
"--baseline",
str(BASELINE),
],
cwd=ROOT,
capture_output=True,
text=True,
check=False,
)
def test_checker_accepts_firing_only_receipt_contract() -> None:
result = _run(RULES)
assert result.returncode == 0, result.stderr
assert "requires receipts only for actively firing backup/Gitea alerts" in result.stdout
assert "BACKUP_ALERT_LABEL_CONTRACT_OK" in result.stdout
def test_checker_rejects_non_firing_receipt_selector(tmp_path: Path) -> None:
payload = yaml.safe_load(RULES.read_text(encoding="utf-8"))
for group in payload["groups"]:
for rule in group.get("rules", []):
if rule.get("alert") == "BackupAlertReceiptStageMissing":
rule["expr"] = str(rule["expr"]).replace(
'alertstate="firing"',
'alertstate="pending"',
)
candidate = tmp_path / "alerts.yml"
candidate.write_text(yaml.safe_dump(payload, allow_unicode=True, sort_keys=False), encoding="utf-8")
result = _run(candidate)
assert result.returncode != 0
assert 'alertstate="firing"' in result.stderr

44
scripts/ops/tests/test_backup_health_textfile_exporter.py
View File

@@ -35,6 +35,50 @@ def test_default_gitea_bundle_expected_repos_cover_current_project_set() -> None
"wooo/bitan-pharmacy",
"wooo/vtuber",
]
assert any(
row["alertname"] == "GiteaRestoreDrillMissingOrFailed"
for row in exporter.BACKUP_ALERT_RECEIPT_REQUIREMENTS
)
def test_gitea_restore_drill_metrics_require_inventory_and_artifacts(tmp_path: Path, monkeypatch) -> None:
exporter = load_exporter()
status_file = tmp_path / "gitea-restore-drill.status"
status_file.write_text(
"\n".join(
[
"timestamp=2000000000",
"success=1",
"reason=verified",
"dump_size_bytes=6928031514",
"backup_regular_repo_count=15",
"backup_wiki_repo_count=0",
"live_regular_repo_count=15",
"live_wiki_repo_count=0",
"database_dump_present=1",
"config_present=2",
"archive_crc_ok=1",
"inventory_digest_match=1",
]
)
+ "\n",
encoding="utf-8",
)
monkeypatch.setattr(exporter, "GITEA_RESTORE_DRILL_STATUS_FILE", status_file)
monkeypatch.setattr(exporter.time, "time", lambda: 2000000060)
rendered = "\n".join(exporter._gitea_restore_drill_metric_lines("110"))
assert 'awoooi_gitea_restore_drill_fresh{host="110",max_age_hours="744"} 1' in rendered
assert 'awoooi_gitea_restore_drill_verified{host="110",max_age_hours="744"} 1' in rendered
assert 'awoooi_gitea_restore_drill_inventory_match{host="110",max_age_hours="744"} 1' in rendered
status_file.write_text(
status_file.read_text(encoding="utf-8").replace("inventory_digest_match=1", "inventory_digest_match=0"),
encoding="utf-8",
)
rendered = "\n".join(exporter._gitea_restore_drill_metric_lines("110"))
assert 'awoooi_gitea_restore_drill_fresh{host="110",max_age_hours="744"} 0' in rendered
assert 'awoooi_gitea_restore_drill_verified{host="110",max_age_hours="744"} 0' in rendered
def test_gitea_bundle_metrics_require_all_expected_repos(tmp_path: Path, monkeypatch) -> None: