From 7fd76b3bcfc5f4771fcf409213f2c323509d30ea Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 09:45:12 +0800 Subject: [PATCH] fix(gitea): add full backup restore drill receipt path --- .../gitea_full_server_backup_dr_receipts.py | 42 +++- ...itea_full_server_backup_dr_receipts_api.py | 64 +++++- scripts/backup/backup-gitea.sh | 2 +- .../backup/gitea-full-backup-restore-drill.sh | 182 ++++++++++++++++++ ...test_backup_gitea_full_receipt_contract.py | 6 + .../test_gitea_full_backup_restore_drill.py | 99 ++++++++++ 6 files changed, 387 insertions(+), 8 deletions(-) create mode 100755 scripts/backup/gitea-full-backup-restore-drill.sh create mode 100644 scripts/backup/tests/test_gitea_full_backup_restore_drill.py diff --git a/apps/api/src/services/gitea_full_server_backup_dr_receipts.py b/apps/api/src/services/gitea_full_server_backup_dr_receipts.py index 1d4826941..6c148aafd 100644 --- a/apps/api/src/services/gitea_full_server_backup_dr_receipts.py +++ b/apps/api/src/services/gitea_full_server_backup_dr_receipts.py @@ -73,9 +73,11 @@ def load_latest_gitea_full_server_backup_dr_receipts( def build_gitea_full_server_backup_source_contract(repo_root: Path) -> dict[str, Any]: """Inspect committed Gitea backup/offsite source without reading secrets.""" backup_script = repo_root / "scripts" / "backup" / "backup-gitea.sh" + restore_drill_script = repo_root / "scripts" / "backup" / "gitea-full-backup-restore-drill.sh" sync_script = repo_root / "scripts" / "backup" / "sync-offsite-backups.sh" verify_script = repo_root / "scripts" / "backup" / "verify-offsite-full-sync.sh" backup_text = _read_text(backup_script) + restore_drill_text = _read_text(restore_drill_script) sync_text = _read_text(sync_script) verify_text = _read_text(verify_script) metric_writer_present = ( @@ -83,6 +85,13 @@ def build_gitea_full_server_backup_source_contract(repo_root: Path) -> dict[str, and "secret_value_collected" in backup_text and "production_restore_performed" in backup_text ) + restore_drill_source_ready = ( + "awoooi_gitea_full_backup_restore_drill_performed" in restore_drill_text + and "structural_metadata_only" in restore_drill_text + and "never restores to production" in restore_drill_text.lower() + and "SECRET_VALUES_COLLECTED=0" in restore_drill_text + and "PRODUCTION_RESTORE_PERFORMED=0" in restore_drill_text + ) gitea_dump_present = "gitea dump" in backup_text and "/tmp/gitea-dump.zip" in backup_text restic_target_present = "restic -r" in backup_text and ( "/backup" in backup_text or "BACKUP_BASE" in backup_text @@ -108,12 +117,16 @@ def build_gitea_full_server_backup_source_contract(repo_root: Path) -> dict[str, "schema_version": "gitea_full_server_backup_source_contract_v1", "source_refs": [ "scripts/backup/backup-gitea.sh", + "scripts/backup/gitea-full-backup-restore-drill.sh", "scripts/backup/sync-offsite-backups.sh", "scripts/backup/verify-offsite-full-sync.sh", ], "gitea_dump_source_ready": gitea_dump_present, "restic_target_source_ready": restic_target_present, "component_metric_writer_source_ready": metric_writer_present, + "restore_drill_source_ready": restore_drill_source_ready, + "restore_drill_receipt_metric_writer_source_ready": restore_drill_source_ready, + "restore_drill_scope": "structural_metadata_only", "google_drive_offsite_source_ready": offsite_source_ready, "component_rows": component_rows, "source_ready_component_count": sum( @@ -151,6 +164,7 @@ def build_gitea_full_server_backup_dr_receipts( "secret_values_collected_by_source_inspection": False, } receipt_metrics = _receipt_metric_rollup(metric_samples or [], host) + restore_drill_source_ready = contract.get("restore_drill_source_ready") is True component_rows = [ { @@ -192,20 +206,35 @@ def build_gitea_full_server_backup_dr_receipts( "status": "verified_ready" if receipt_metrics["restore_drill_performed"] else "approval_required", - "evidence_refs": ["GET /api/v1/agents/backup-dr-readiness-matrix"], + "evidence_refs": [ + "scripts/backup/gitea-full-backup-restore-drill.sh", + "GET /api/v1/agents/backup-dr-readiness-matrix", + "GET /api/v1/agents/backup-restore-drill-approval-package-template", + ], "receipt_state": "restore_drill_receipt_present" if receipt_metrics["restore_drill_performed"] + else "restore_drill_source_ready_live_receipt_pending" + if restore_drill_source_ready else "restore_drill_not_executed_by_readback", - "restore_scope": "restore requires controlled drill approval", + "restore_scope": str( + contract.get("restore_drill_scope") or "controlled_restore_drill" + ), "production_restore_performed": False, "metrics": { "gitea_restore_drill_status": str( gitea_readiness.get("restore_drill_status") or "" ), "restore_execution_allowed": False, + "restore_drill_source_ready": restore_drill_source_ready, + "restore_drill_receipt_metric_writer_source_ready": contract.get( + "restore_drill_receipt_metric_writer_source_ready" + ) + is True, "restore_drill_performed_metric": receipt_metrics[ "restore_drill_performed" ], + "secret_value_collection_allowed": False, + "production_restore_allowed": False, }, }, *_component_receipt_rows(contract, receipt_metrics), @@ -277,14 +306,19 @@ def build_gitea_full_server_backup_dr_receipts( "component_receipt_count" ], "gitea_full_backup_metric_error": metrics_error, + "restore_drill_source_ready": restore_drill_source_ready, + "restore_drill_receipt_pending": not receipt_metrics[ + "restore_drill_performed" + ], "production_restore_performed": False, }, "source_contract": contract, "receipt_metric_rollup": receipt_metrics, "component_rows": component_rows, "safe_next_actions": [ - "deploy backup-gitea.sh receipt metric writer and rerun the normal Gitea backup cadence", - "keep production restore disabled; use controlled restore-drill approval only", + "deploy gitea-full-backup-restore-drill.sh and run it against the latest Gitea dump in an isolated non-production drill", + "write the restore drill textfile receipt only after the structural drill passes", + "keep production restore disabled; production restore metrics must remain zero", "verify Google Drive/offsite freshness with redacted non-secret receipts", ], } diff --git a/apps/api/tests/test_gitea_full_server_backup_dr_receipts_api.py b/apps/api/tests/test_gitea_full_server_backup_dr_receipts_api.py index 663c66d9a..a6519bbcf 100644 --- a/apps/api/tests/test_gitea_full_server_backup_dr_receipts_api.py +++ b/apps/api/tests/test_gitea_full_server_backup_dr_receipts_api.py @@ -65,7 +65,11 @@ def _payload() -> dict: ) -def _full_backup_metric_samples() -> list[dict]: +def _full_backup_metric_samples( + *, + restore_drill_performed: bool = False, + production_restore_performed: bool = False, +) -> list[dict]: def sample(name: str, value: float, **labels: str) -> dict: return {"name": name, "labels": {"host": "110", **labels}, "value": value} @@ -73,8 +77,10 @@ def _full_backup_metric_samples() -> list[dict]: sample("awoooi_gitea_full_backup_last_run_failed", 0, service="gitea"), sample( "awoooi_gitea_full_backup_restore_drill_performed", - 0, + 1 if restore_drill_performed else 0, service="gitea", + drill_scope="structural_metadata_only", + production_restore="false", ), sample( "awoooi_gitea_full_backup_secret_value_collected", @@ -83,7 +89,7 @@ def _full_backup_metric_samples() -> list[dict]: ), sample( "awoooi_gitea_full_backup_production_restore_performed", - 0, + 1 if production_restore_performed else 0, service="gitea", ), ] @@ -130,10 +136,17 @@ def test_gitea_full_server_backup_dr_receipts_separates_bundle_from_full_dr() -> assert by_id["gitea_server_backup_target"]["status"] == "verified_ready" assert by_id["google_drive_offsite_mirror"]["status"] == "verified_ready" assert by_id["restore_drill_approval_gate"]["status"] == "approval_required" + assert by_id["restore_drill_approval_gate"]["receipt_state"] == ( + "restore_drill_source_ready_live_receipt_pending" + ) + assert by_id["restore_drill_approval_gate"]["metrics"][ + "restore_drill_source_ready" + ] is True assert by_id["gitea_database_receipt"]["status"] == "source_ready" assert by_id["gitea_packages_receipt"]["status"] == "source_ready" assert by_id["gitea_attachments_receipt"]["production_restore_performed"] is False assert payload["source_contract"]["component_metric_writer_source_ready"] is True + assert payload["source_contract"]["restore_drill_source_ready"] is True assert payload["source_contract"]["secret_values_collected_by_source_inspection"] is False @@ -155,6 +168,51 @@ def test_gitea_full_server_backup_dr_receipts_promotes_live_component_metrics() assert payload["receipt_metric_rollup"]["production_restore_performed"] is False +def test_gitea_full_server_backup_dr_receipts_promotes_restore_drill_metric() -> None: + repo_root = Path(__file__).resolve().parents[3] + payload = build_gitea_full_server_backup_dr_receipts( + repo_bundle_backup_readback=_repo_bundle_payload(), + backup_dr_readiness_matrix=load_latest_backup_dr_readiness_matrix(), + backup_dr_target_inventory=load_latest_backup_dr_target_inventory(), + source_contract=build_gitea_full_server_backup_source_contract(repo_root), + metric_samples=_full_backup_metric_samples(restore_drill_performed=True), + generated_at="2026-07-10T00:03:00+08:00", + ) + + assert payload["status"] == "gitea_full_server_backup_dr_receipts_ready" + assert payload["ready"] is True + assert payload["active_gap_ids"] == [] + assert payload["summary"]["verified_ready_component_count"] == 10 + assert payload["summary"]["restore_drill_source_ready"] is True + assert payload["summary"]["restore_drill_receipt_pending"] is False + by_id = {row["component_id"]: row for row in payload["component_rows"]} + assert by_id["restore_drill_approval_gate"]["status"] == "verified_ready" + assert by_id["restore_drill_approval_gate"]["metrics"][ + "restore_drill_performed_metric" + ] is True + assert payload["receipt_metric_rollup"]["production_restore_performed"] is False + + +def test_gitea_full_server_backup_dr_receipts_rejects_production_restore_metric() -> None: + repo_root = Path(__file__).resolve().parents[3] + try: + build_gitea_full_server_backup_dr_receipts( + repo_bundle_backup_readback=_repo_bundle_payload(), + backup_dr_readiness_matrix=load_latest_backup_dr_readiness_matrix(), + backup_dr_target_inventory=load_latest_backup_dr_target_inventory(), + source_contract=build_gitea_full_server_backup_source_contract(repo_root), + metric_samples=_full_backup_metric_samples( + restore_drill_performed=True, + production_restore_performed=True, + ), + generated_at="2026-07-10T00:04:00+08:00", + ) + except ValueError as exc: + assert "must not restore production" in str(exc) + else: + raise AssertionError("expected production restore metric to fail closed") + + def test_gitea_full_server_backup_dr_receipts_endpoint_returns_payload(monkeypatch) -> None: monkeypatch.setattr( agents, diff --git a/scripts/backup/backup-gitea.sh b/scripts/backup/backup-gitea.sh index 31c0de30f..94ee6a958 100755 --- a/scripts/backup/backup-gitea.sh +++ b/scripts/backup/backup-gitea.sh @@ -60,7 +60,7 @@ write_gitea_full_backup_receipt() { echo "# TYPE awoooi_gitea_full_backup_restic_snapshot_present gauge" echo "# HELP awoooi_gitea_full_backup_component_receipt Whether the Gitea full-server dump includes a redacted component receipt." echo "# TYPE awoooi_gitea_full_backup_component_receipt gauge" - echo "# HELP awoooi_gitea_full_backup_restore_drill_performed Whether this receipt performed a production restore drill." + echo "# HELP awoooi_gitea_full_backup_restore_drill_performed Whether a non-production Gitea full-backup restore drill receipt is present." echo "# TYPE awoooi_gitea_full_backup_restore_drill_performed gauge" echo "# HELP awoooi_gitea_full_backup_secret_value_collected Whether this receipt collected secret values." echo "# TYPE awoooi_gitea_full_backup_secret_value_collected gauge" diff --git a/scripts/backup/gitea-full-backup-restore-drill.sh b/scripts/backup/gitea-full-backup-restore-drill.sh new file mode 100755 index 000000000..e84e1222d --- /dev/null +++ b/scripts/backup/gitea-full-backup-restore-drill.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Verify a Gitea full-server backup dump with a non-production, metadata-only +# structural drill. This never restores into Gitea, never contacts production, +# never reads tokens, and never prints archive member names or file contents. + +set -euo pipefail + +DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}" +HOST_LABEL="${AIOPS_HOST_LABEL:-110}" +TEXTFILE_PATH="${AIOPS_GITEA_FULL_BACKUP_RESTORE_DRILL_TEXTFILE:-/home/wooo/node_exporter_textfiles/gitea_full_backup_restore_drill.prom}" +WRITE_TEXTFILE=0 +TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-60}" +DRILL_SCOPE="structural_metadata_only" +REQUIRED_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments hooks_custom_assets" + +usage() { + cat <<'USAGE' +Usage: scripts/backup/gitea-full-backup-restore-drill.sh [options] + +Options: + --dump-zip PATH Gitea dump zip to verify structurally. + --write-textfile Write node-exporter textfile metric after verification. + --textfile PATH Textfile metric path. + --host LABEL Host label for metrics. Default: 110. + -h, --help Show this help. + +This verifier inspects only the Gitea dump zip central directory and component +coverage. It never restores to production, never contacts Gitea, never reads +credentials, never prints archive member names, and never extracts contents. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --dump-zip) + shift + DUMP_ZIP="${1:?--dump-zip requires PATH}" + ;; + --write-textfile) + WRITE_TEXTFILE=1 + ;; + --textfile) + shift + TEXTFILE_PATH="${1:?--textfile requires PATH}" + ;; + --host) + shift + HOST_LABEL="${1:?--host requires LABEL}" + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 64 + ;; + esac + shift +done + +if [ -z "$DUMP_ZIP" ]; then + echo "ERROR=dump_zip_required" >&2 + usage >&2 + exit 64 +fi + +ok=0 +entry_count=0 +missing_components="" +error_reason="" + +if [ ! -f "$DUMP_ZIP" ]; then + error_reason="dump_zip_not_found" +else + set +e + drill_result="$( + timeout "$TIMEOUT_SECONDS" python3 - "$DUMP_ZIP" $REQUIRED_COMPONENTS <<'PY' +from __future__ import annotations + +import sys +import zipfile +from pathlib import Path + +dump_zip = Path(sys.argv[1]) +required = sys.argv[2:] + +try: + with zipfile.ZipFile(dump_zip) as archive: + names = archive.namelist() +except zipfile.BadZipFile: + print("ERROR=bad_zip") + sys.exit(1) +except OSError: + print("ERROR=zip_open_failed") + sys.exit(1) + +lowered = [name.lower() for name in names] + +patterns = { + "database": ("gitea-db", "database", ".sql"), + "repositories": ("repo", "repos", "repositories", ".git"), + "app_settings": ("app.ini", "app.example.ini", "conf/"), + "lfs_objects": ("lfs",), + "package_registry": ("package", "packages"), + "attachments": ("attachment", "attachments"), + "hooks_custom_assets": ("hook", "custom", "public/"), +} + +missing = [] +for component in required: + needles = patterns.get(component, (component,)) + if not any(any(needle in name for needle in needles) for name in lowered): + missing.append(component) + +print(f"ENTRY_COUNT={len(names)}") +print(f"MISSING_COMPONENTS={','.join(missing)}") +print("OK=1" if not missing and names else "OK=0") +PY + )" + drill_status=$? + set -e + entry_count="$(printf '%s\n' "$drill_result" | awk -F= '$1 == "ENTRY_COUNT" {print $2; exit}')" + missing_components="$(printf '%s\n' "$drill_result" | awk -F= '$1 == "MISSING_COMPONENTS" {print $2; exit}')" + if [ "$drill_status" -ne 0 ]; then + error_reason="$(printf '%s\n' "$drill_result" | awk -F= '$1 == "ERROR" {print $2; exit}')" + [ -n "$error_reason" ] || error_reason="drill_inspection_failed" + elif printf '%s\n' "$drill_result" | grep -q '^OK=1$'; then + ok=1 + else + error_reason="component_coverage_gap" + fi +fi + +timestamp="$(date +%s)" +dump_label="$(basename "${DUMP_ZIP:-missing}")" +host_label="${HOST_LABEL//\\/\\\\}" +host_label="${host_label//\"/\\\"}" +dump_label="${dump_label//\\/\\\\}" +dump_label="${dump_label//\"/\\\"}" +error_label="${error_reason//\\/\\\\}" +error_label="${error_label//\"/\\\"}" + +component_present() { + local component="$1" + if [ "$ok" -eq 1 ]; then + printf '1' + elif printf ',%s,' "$missing_components" | grep -q ",${component},"; then + printf '0' + else + printf '0' + fi +} + +if [ "$WRITE_TEXTFILE" -eq 1 ]; then + mkdir -p "$(dirname "$TEXTFILE_PATH")" + tmp_textfile="${TEXTFILE_PATH}.$$" + { + echo '# HELP awoooi_gitea_full_backup_restore_drill_performed Whether a non-production Gitea full-backup structural restore drill succeeded.' + echo '# TYPE awoooi_gitea_full_backup_restore_drill_performed gauge' + printf 'awoooi_gitea_full_backup_restore_drill_performed{host="%s",service="gitea",drill_scope="%s",production_restore="false",dump="%s",error="%s"} %s\n' "$host_label" "$DRILL_SCOPE" "$dump_label" "$error_label" "$ok" + printf 'awoooi_gitea_full_backup_restore_drill_timestamp{host="%s",service="gitea",drill_scope="%s"} %s\n' "$host_label" "$DRILL_SCOPE" "$timestamp" + printf 'awoooi_gitea_full_backup_restore_drill_entry_count{host="%s",service="gitea",drill_scope="%s"} %s\n' "$host_label" "$DRILL_SCOPE" "${entry_count:-0}" + for component in $REQUIRED_COMPONENTS; do + printf 'awoooi_gitea_full_backup_restore_drill_component_receipt{host="%s",service="gitea",component="%s",drill_scope="%s"} %s\n' "$host_label" "$component" "$DRILL_SCOPE" "$(component_present "$component")" + done + printf 'awoooi_gitea_full_backup_secret_value_collected{host="%s",service="gitea",drill_scope="%s"} 0\n' "$host_label" "$DRILL_SCOPE" + printf 'awoooi_gitea_full_backup_production_restore_performed{host="%s",service="gitea",drill_scope="%s"} 0\n' "$host_label" "$DRILL_SCOPE" + } > "$tmp_textfile" + mv "$tmp_textfile" "$TEXTFILE_PATH" +fi + +printf 'GITEA_FULL_BACKUP_RESTORE_DRILL_OK=%s\n' "$ok" +printf 'ENTRY_COUNT=%s\n' "${entry_count:-0}" +printf 'TEXTFILE_WRITTEN=%s\n' "$WRITE_TEXTFILE" +printf 'SECRET_VALUES_COLLECTED=0\n' +printf 'PRODUCTION_RESTORE_PERFORMED=0\n' +if [ -n "$error_reason" ]; then + printf 'ERROR=%s\n' "$error_reason" + exit 1 +fi diff --git a/scripts/backup/tests/test_backup_gitea_full_receipt_contract.py b/scripts/backup/tests/test_backup_gitea_full_receipt_contract.py index 29a80e343..d2ddbdd52 100644 --- a/scripts/backup/tests/test_backup_gitea_full_receipt_contract.py +++ b/scripts/backup/tests/test_backup_gitea_full_receipt_contract.py @@ -12,6 +12,7 @@ def test_backup_gitea_writes_full_server_component_receipts() -> None: assert "write_gitea_full_backup_receipt" in text assert "awoooi_gitea_full_backup_component_receipt" in text + assert "awoooi_gitea_full_backup_restore_drill_performed" in text assert "awoooi_gitea_full_backup_secret_value_collected" in text assert "awoooi_gitea_full_backup_production_restore_performed" in text assert "gitea dump -c /data/gitea/conf/app.ini" in text @@ -34,6 +35,11 @@ def test_backup_gitea_receipt_contract_stays_no_secret_no_restore() -> None: assert "awoooi_gitea_full_backup_secret_value_collected" in text assert "awoooi_gitea_full_backup_production_restore_performed" in text + assert "awoooi_gitea_full_backup_restore_drill_performed" in text + assert ( + 'restore_drill_performed{host=\\"${host}\\",service=\\"${service_label}\\"} 0' + in text + ) assert ( 'secret_value_collected{host=\\"${host}\\",service=\\"${service_label}\\"} 0' in text diff --git a/scripts/backup/tests/test_gitea_full_backup_restore_drill.py b/scripts/backup/tests/test_gitea_full_backup_restore_drill.py new file mode 100644 index 000000000..3d4f48eef --- /dev/null +++ b/scripts/backup/tests/test_gitea_full_backup_restore_drill.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import subprocess +import zipfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "gitea-full-backup-restore-drill.sh" + + +def _write_dump(path: Path, *, include_lfs: bool = True) -> None: + entries = { + "gitea-db.sql": "-- redacted fixture\n", + "repos/wooo/awoooi.git/HEAD": "ref: refs/heads/main\n", + "custom/conf/app.ini": "[server]\nAPP_NAME=fixture\n", + "data/packages/package.bin": "package fixture\n", + "data/attachments/attachment.bin": "attachment fixture\n", + "custom/hooks/pre-receive": "#!/bin/sh\n", + } + if include_lfs: + entries["data/lfs/objects/aa/bb/object"] = "lfs fixture\n" + with zipfile.ZipFile(path, "w") as archive: + for name, content in entries.items(): + archive.writestr(name, content) + + +def test_gitea_full_backup_restore_drill_writes_non_production_receipt( + tmp_path: Path, +) -> None: + dump = tmp_path / "gitea-dump.zip" + textfile = tmp_path / "gitea_full_backup_restore_drill.prom" + _write_dump(dump) + + result = subprocess.run( + [ + "bash", + str(SCRIPT), + "--dump-zip", + str(dump), + "--write-textfile", + "--textfile", + str(textfile), + "--host", + "110", + ], + text=True, + capture_output=True, + check=True, + ) + + assert "GITEA_FULL_BACKUP_RESTORE_DRILL_OK=1" in result.stdout + assert "SECRET_VALUES_COLLECTED=0" in result.stdout + assert "PRODUCTION_RESTORE_PERFORMED=0" in result.stdout + rendered = textfile.read_text(encoding="utf-8") + assert ( + 'awoooi_gitea_full_backup_restore_drill_performed{host="110",service="gitea",' + in rendered + ) + assert 'drill_scope="structural_metadata_only"' in rendered + assert 'production_restore="false"' in rendered + assert '} 1' in rendered + assert ( + 'awoooi_gitea_full_backup_secret_value_collected{host="110",service="gitea",' + in rendered + ) + assert ( + 'awoooi_gitea_full_backup_production_restore_performed{host="110",service="gitea",' + in rendered + ) + + +def test_gitea_full_backup_restore_drill_fails_closed_when_component_missing( + tmp_path: Path, +) -> None: + dump = tmp_path / "gitea-dump.zip" + textfile = tmp_path / "gitea_full_backup_restore_drill.prom" + _write_dump(dump, include_lfs=False) + + result = subprocess.run( + [ + "bash", + str(SCRIPT), + "--dump-zip", + str(dump), + "--write-textfile", + "--textfile", + str(textfile), + ], + text=True, + capture_output=True, + ) + + assert result.returncode == 1 + assert "GITEA_FULL_BACKUP_RESTORE_DRILL_OK=0" in result.stdout + assert "ERROR=component_coverage_gap" in result.stdout + rendered = textfile.read_text(encoding="utf-8") + assert 'component="lfs_objects",drill_scope="structural_metadata_only"} 0' in rendered + assert 'production_restore_performed{host="110",service="gitea",' in rendered