wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(gitea): add full backup restore drill receipt path

Browse Source
This commit is contained in:
ogt
2026-07-10 09:45:12 +08:00
parent 97312fa37a
commit 7fd76b3bcf
6 changed files with 387 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
apps/api/src/services/gitea_full_server_backup_dr_receipts.py
View File

@@ -73,9 +73,11 @@ def load_latest_gitea_full_server_backup_dr_receipts(
def build_gitea_full_server_backup_source_contract(repo_root: Path) -> dict[str, Any]:
"""Inspect committed Gitea backup/offsite source without reading secrets."""
backup_script = repo_root / "scripts" / "backup" / "backup-gitea.sh"
restore_drill_script = repo_root / "scripts" / "backup" / "gitea-full-backup-restore-drill.sh"
sync_script = repo_root / "scripts" / "backup" / "sync-offsite-backups.sh"
verify_script = repo_root / "scripts" / "backup" / "verify-offsite-full-sync.sh"
backup_text = _read_text(backup_script)
restore_drill_text = _read_text(restore_drill_script)
sync_text = _read_text(sync_script)
verify_text = _read_text(verify_script)
metric_writer_present = (
@@ -83,6 +85,13 @@ def build_gitea_full_server_backup_source_contract(repo_root: Path) -> dict[str,
and "secret_value_collected" in backup_text
and "production_restore_performed" in backup_text
)
restore_drill_source_ready = (
"awoooi_gitea_full_backup_restore_drill_performed" in restore_drill_text
and "structural_metadata_only" in restore_drill_text
and "never restores to production" in restore_drill_text.lower()
and "SECRET_VALUES_COLLECTED=0" in restore_drill_text
and "PRODUCTION_RESTORE_PERFORMED=0" in restore_drill_text
)
gitea_dump_present = "gitea dump" in backup_text and "/tmp/gitea-dump.zip" in backup_text
restic_target_present = "restic -r" in backup_text and (
"/backup" in backup_text or "BACKUP_BASE" in backup_text
@@ -108,12 +117,16 @@ def build_gitea_full_server_backup_source_contract(repo_root: Path) -> dict[str,
"schema_version": "gitea_full_server_backup_source_contract_v1",
"source_refs": [
"scripts/backup/backup-gitea.sh",
"scripts/backup/gitea-full-backup-restore-drill.sh",
"scripts/backup/sync-offsite-backups.sh",
"scripts/backup/verify-offsite-full-sync.sh",
],
"gitea_dump_source_ready": gitea_dump_present,
"restic_target_source_ready": restic_target_present,
"component_metric_writer_source_ready": metric_writer_present,
"restore_drill_source_ready": restore_drill_source_ready,
"restore_drill_receipt_metric_writer_source_ready": restore_drill_source_ready,
"restore_drill_scope": "structural_metadata_only",
"google_drive_offsite_source_ready": offsite_source_ready,
"component_rows": component_rows,
"source_ready_component_count": sum(
@@ -151,6 +164,7 @@ def build_gitea_full_server_backup_dr_receipts(
"secret_values_collected_by_source_inspection": False,
}
receipt_metrics = _receipt_metric_rollup(metric_samples or [], host)
restore_drill_source_ready = contract.get("restore_drill_source_ready") is True
component_rows = [
{
@@ -192,20 +206,35 @@ def build_gitea_full_server_backup_dr_receipts(
"status": "verified_ready"
if receipt_metrics["restore_drill_performed"]
else "approval_required",
"evidence_refs": ["GET /api/v1/agents/backup-dr-readiness-matrix"],
"evidence_refs": [
"scripts/backup/gitea-full-backup-restore-drill.sh",
"GET /api/v1/agents/backup-dr-readiness-matrix",
"GET /api/v1/agents/backup-restore-drill-approval-package-template",
],
"receipt_state": "restore_drill_receipt_present"
if receipt_metrics["restore_drill_performed"]
else "restore_drill_source_ready_live_receipt_pending"
if restore_drill_source_ready
else "restore_drill_not_executed_by_readback",
"restore_scope": "restore requires controlled drill approval",
"restore_scope": str(
contract.get("restore_drill_scope") or "controlled_restore_drill"
),
"production_restore_performed": False,
"metrics": {
"gitea_restore_drill_status": str(
gitea_readiness.get("restore_drill_status") or ""
),
"restore_execution_allowed": False,
"restore_drill_source_ready": restore_drill_source_ready,
"restore_drill_receipt_metric_writer_source_ready": contract.get(
"restore_drill_receipt_metric_writer_source_ready"
)
is True,
"restore_drill_performed_metric": receipt_metrics[
"restore_drill_performed"
],
"secret_value_collection_allowed": False,
"production_restore_allowed": False,
},
},
*_component_receipt_rows(contract, receipt_metrics),
@@ -277,14 +306,19 @@ def build_gitea_full_server_backup_dr_receipts(
"component_receipt_count"
],
"gitea_full_backup_metric_error": metrics_error,
"restore_drill_source_ready": restore_drill_source_ready,
"restore_drill_receipt_pending": not receipt_metrics[
"restore_drill_performed"
],
"production_restore_performed": False,
},
"source_contract": contract,
"receipt_metric_rollup": receipt_metrics,
"component_rows": component_rows,
"safe_next_actions": [
"deploy backup-gitea.sh receipt metric writer and rerun the normal Gitea backup cadence",
"keep production restore disabled; use controlled restore-drill approval only",
"deploy gitea-full-backup-restore-drill.sh and run it against the latest Gitea dump in an isolated non-production drill",
"write the restore drill textfile receipt only after the structural drill passes",
"keep production restore disabled; production restore metrics must remain zero",
"verify Google Drive/offsite freshness with redacted non-secret receipts",
],
}

64
apps/api/tests/test_gitea_full_server_backup_dr_receipts_api.py
View File

@@ -65,7 +65,11 @@ def _payload() -> dict:
)
def _full_backup_metric_samples() -> list[dict]:
def _full_backup_metric_samples(
*,
restore_drill_performed: bool = False,
production_restore_performed: bool = False,
) -> list[dict]:
def sample(name: str, value: float, **labels: str) -> dict:
return {"name": name, "labels": {"host": "110", **labels}, "value": value}
@@ -73,8 +77,10 @@ def _full_backup_metric_samples() -> list[dict]:
sample("awoooi_gitea_full_backup_last_run_failed", 0, service="gitea"),
sample(
"awoooi_gitea_full_backup_restore_drill_performed",
0,
1 if restore_drill_performed else 0,
service="gitea",
drill_scope="structural_metadata_only",
production_restore="false",
),
sample(
"awoooi_gitea_full_backup_secret_value_collected",
@@ -83,7 +89,7 @@ def _full_backup_metric_samples() -> list[dict]:
),
sample(
"awoooi_gitea_full_backup_production_restore_performed",
0,
1 if production_restore_performed else 0,
service="gitea",
),
]
@@ -130,10 +136,17 @@ def test_gitea_full_server_backup_dr_receipts_separates_bundle_from_full_dr() ->
assert by_id["gitea_server_backup_target"]["status"] == "verified_ready"
assert by_id["google_drive_offsite_mirror"]["status"] == "verified_ready"
assert by_id["restore_drill_approval_gate"]["status"] == "approval_required"
assert by_id["restore_drill_approval_gate"]["receipt_state"] == (
"restore_drill_source_ready_live_receipt_pending"
)
assert by_id["restore_drill_approval_gate"]["metrics"][
"restore_drill_source_ready"
] is True
assert by_id["gitea_database_receipt"]["status"] == "source_ready"
assert by_id["gitea_packages_receipt"]["status"] == "source_ready"
assert by_id["gitea_attachments_receipt"]["production_restore_performed"] is False
assert payload["source_contract"]["component_metric_writer_source_ready"] is True
assert payload["source_contract"]["restore_drill_source_ready"] is True
assert payload["source_contract"]["secret_values_collected_by_source_inspection"] is False
@@ -155,6 +168,51 @@ def test_gitea_full_server_backup_dr_receipts_promotes_live_component_metrics()
assert payload["receipt_metric_rollup"]["production_restore_performed"] is False
def test_gitea_full_server_backup_dr_receipts_promotes_restore_drill_metric() -> None:
repo_root = Path(__file__).resolve().parents[3]
payload = build_gitea_full_server_backup_dr_receipts(
repo_bundle_backup_readback=_repo_bundle_payload(),
backup_dr_readiness_matrix=load_latest_backup_dr_readiness_matrix(),
backup_dr_target_inventory=load_latest_backup_dr_target_inventory(),
source_contract=build_gitea_full_server_backup_source_contract(repo_root),
metric_samples=_full_backup_metric_samples(restore_drill_performed=True),
generated_at="2026-07-10T00:03:00+08:00",
)
assert payload["status"] == "gitea_full_server_backup_dr_receipts_ready"
assert payload["ready"] is True
assert payload["active_gap_ids"] == []
assert payload["summary"]["verified_ready_component_count"] == 10
assert payload["summary"]["restore_drill_source_ready"] is True
assert payload["summary"]["restore_drill_receipt_pending"] is False
by_id = {row["component_id"]: row for row in payload["component_rows"]}
assert by_id["restore_drill_approval_gate"]["status"] == "verified_ready"
assert by_id["restore_drill_approval_gate"]["metrics"][
"restore_drill_performed_metric"
] is True
assert payload["receipt_metric_rollup"]["production_restore_performed"] is False
def test_gitea_full_server_backup_dr_receipts_rejects_production_restore_metric() -> None:
repo_root = Path(__file__).resolve().parents[3]
try:
build_gitea_full_server_backup_dr_receipts(
repo_bundle_backup_readback=_repo_bundle_payload(),
backup_dr_readiness_matrix=load_latest_backup_dr_readiness_matrix(),
backup_dr_target_inventory=load_latest_backup_dr_target_inventory(),
source_contract=build_gitea_full_server_backup_source_contract(repo_root),
metric_samples=_full_backup_metric_samples(
restore_drill_performed=True,
production_restore_performed=True,
),
generated_at="2026-07-10T00:04:00+08:00",
)
except ValueError as exc:
assert "must not restore production" in str(exc)
else:
raise AssertionError("expected production restore metric to fail closed")
def test_gitea_full_server_backup_dr_receipts_endpoint_returns_payload(monkeypatch) -> None:
monkeypatch.setattr(
agents,

2
scripts/backup/backup-gitea.sh
View File

@@ -60,7 +60,7 @@ write_gitea_full_backup_receipt() {
echo "# TYPE awoooi_gitea_full_backup_restic_snapshot_present gauge"
echo "# HELP awoooi_gitea_full_backup_component_receipt Whether the Gitea full-server dump includes a redacted component receipt."
echo "# TYPE awoooi_gitea_full_backup_component_receipt gauge"
echo "# HELP awoooi_gitea_full_backup_restore_drill_performed Whether this receipt performed a production restore drill."
echo "# HELP awoooi_gitea_full_backup_restore_drill_performed Whether a non-production Gitea full-backup restore drill receipt is present."
echo "# TYPE awoooi_gitea_full_backup_restore_drill_performed gauge"
echo "# HELP awoooi_gitea_full_backup_secret_value_collected Whether this receipt collected secret values."
echo "# TYPE awoooi_gitea_full_backup_secret_value_collected gauge"

182
scripts/backup/gitea-full-backup-restore-drill.sh Executable file
View File

@@ -0,0 +1,182 @@
#!/usr/bin/env bash
# Verify a Gitea full-server backup dump with a non-production, metadata-only
# structural drill. This never restores into Gitea, never contacts production,
# never reads tokens, and never prints archive member names or file contents.
set -euo pipefail
DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}"
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
TEXTFILE_PATH="${AIOPS_GITEA_FULL_BACKUP_RESTORE_DRILL_TEXTFILE:-/home/wooo/node_exporter_textfiles/gitea_full_backup_restore_drill.prom}"
WRITE_TEXTFILE=0
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-60}"
DRILL_SCOPE="structural_metadata_only"
REQUIRED_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments hooks_custom_assets"
usage() {
cat <<'USAGE'
Usage: scripts/backup/gitea-full-backup-restore-drill.sh [options]
Options:
--dump-zip PATH Gitea dump zip to verify structurally.
--write-textfile Write node-exporter textfile metric after verification.
--textfile PATH Textfile metric path.
--host LABEL Host label for metrics. Default: 110.
-h, --help Show this help.
This verifier inspects only the Gitea dump zip central directory and component
coverage. It never restores to production, never contacts Gitea, never reads
credentials, never prints archive member names, and never extracts contents.
USAGE
}
while [ "$#" -gt 0 ]; do
case "$1" in
--dump-zip)
shift
DUMP_ZIP="${1:?--dump-zip requires PATH}"
;;
--write-textfile)
WRITE_TEXTFILE=1
;;
--textfile)
shift
TEXTFILE_PATH="${1:?--textfile requires PATH}"
;;
--host)
shift
HOST_LABEL="${1:?--host requires LABEL}"
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown argument: $1" >&2
usage >&2
exit 64
;;
esac
shift
done
if [ -z "$DUMP_ZIP" ]; then
echo "ERROR=dump_zip_required" >&2
usage >&2
exit 64
fi
ok=0
entry_count=0
missing_components=""
error_reason=""
if [ ! -f "$DUMP_ZIP" ]; then
error_reason="dump_zip_not_found"
else
set +e
drill_result="$(
timeout "$TIMEOUT_SECONDS" python3 - "$DUMP_ZIP" $REQUIRED_COMPONENTS <<'PY'
from __future__ import annotations
import sys
import zipfile
from pathlib import Path
dump_zip = Path(sys.argv[1])
required = sys.argv[2:]
try:
with zipfile.ZipFile(dump_zip) as archive:
names = archive.namelist()
except zipfile.BadZipFile:
print("ERROR=bad_zip")
sys.exit(1)
except OSError:
print("ERROR=zip_open_failed")
sys.exit(1)
lowered = [name.lower() for name in names]
patterns = {
"database": ("gitea-db", "database", ".sql"),
"repositories": ("repo", "repos", "repositories", ".git"),
"app_settings": ("app.ini", "app.example.ini", "conf/"),
"lfs_objects": ("lfs",),
"package_registry": ("package", "packages"),
"attachments": ("attachment", "attachments"),
"hooks_custom_assets": ("hook", "custom", "public/"),
}
missing = []
for component in required:
needles = patterns.get(component, (component,))
if not any(any(needle in name for needle in needles) for name in lowered):
missing.append(component)
print(f"ENTRY_COUNT={len(names)}")
print(f"MISSING_COMPONENTS={','.join(missing)}")
print("OK=1" if not missing and names else "OK=0")
PY
)"
drill_status=$?
set -e
entry_count="$(printf '%s\n' "$drill_result" | awk -F= '$1 == "ENTRY_COUNT" {print $2; exit}')"
missing_components="$(printf '%s\n' "$drill_result" | awk -F= '$1 == "MISSING_COMPONENTS" {print $2; exit}')"
if [ "$drill_status" -ne 0 ]; then
error_reason="$(printf '%s\n' "$drill_result" | awk -F= '$1 == "ERROR" {print $2; exit}')"
[ -n "$error_reason" ] || error_reason="drill_inspection_failed"
elif printf '%s\n' "$drill_result" | grep -q '^OK=1$'; then
ok=1
else
error_reason="component_coverage_gap"
fi
fi
timestamp="$(date +%s)"
dump_label="$(basename "${DUMP_ZIP:-missing}")"
host_label="${HOST_LABEL//\\/\\\\}"
host_label="${host_label//\"/\\\"}"
dump_label="${dump_label//\\/\\\\}"
dump_label="${dump_label//\"/\\\"}"
error_label="${error_reason//\\/\\\\}"
error_label="${error_label//\"/\\\"}"
component_present() {
local component="$1"
if [ "$ok" -eq 1 ]; then
printf '1'
elif printf ',%s,' "$missing_components" | grep -q ",${component},"; then
printf '0'
else
printf '0'
fi
}
if [ "$WRITE_TEXTFILE" -eq 1 ]; then
mkdir -p "$(dirname "$TEXTFILE_PATH")"
tmp_textfile="${TEXTFILE_PATH}.$$"
{
echo '# HELP awoooi_gitea_full_backup_restore_drill_performed Whether a non-production Gitea full-backup structural restore drill succeeded.'
echo '# TYPE awoooi_gitea_full_backup_restore_drill_performed gauge'
printf 'awoooi_gitea_full_backup_restore_drill_performed{host="%s",service="gitea",drill_scope="%s",production_restore="false",dump="%s",error="%s"} %s\n' "$host_label" "$DRILL_SCOPE" "$dump_label" "$error_label" "$ok"
printf 'awoooi_gitea_full_backup_restore_drill_timestamp{host="%s",service="gitea",drill_scope="%s"} %s\n' "$host_label" "$DRILL_SCOPE" "$timestamp"
printf 'awoooi_gitea_full_backup_restore_drill_entry_count{host="%s",service="gitea",drill_scope="%s"} %s\n' "$host_label" "$DRILL_SCOPE" "${entry_count:-0}"
for component in $REQUIRED_COMPONENTS; do
printf 'awoooi_gitea_full_backup_restore_drill_component_receipt{host="%s",service="gitea",component="%s",drill_scope="%s"} %s\n' "$host_label" "$component" "$DRILL_SCOPE" "$(component_present "$component")"
done
printf 'awoooi_gitea_full_backup_secret_value_collected{host="%s",service="gitea",drill_scope="%s"} 0\n' "$host_label" "$DRILL_SCOPE"
printf 'awoooi_gitea_full_backup_production_restore_performed{host="%s",service="gitea",drill_scope="%s"} 0\n' "$host_label" "$DRILL_SCOPE"
} > "$tmp_textfile"
mv "$tmp_textfile" "$TEXTFILE_PATH"
fi
printf 'GITEA_FULL_BACKUP_RESTORE_DRILL_OK=%s\n' "$ok"
printf 'ENTRY_COUNT=%s\n' "${entry_count:-0}"
printf 'TEXTFILE_WRITTEN=%s\n' "$WRITE_TEXTFILE"
printf 'SECRET_VALUES_COLLECTED=0\n'
printf 'PRODUCTION_RESTORE_PERFORMED=0\n'
if [ -n "$error_reason" ]; then
printf 'ERROR=%s\n' "$error_reason"
exit 1
fi

6
scripts/backup/tests/test_backup_gitea_full_receipt_contract.py
View File

@@ -12,6 +12,7 @@ def test_backup_gitea_writes_full_server_component_receipts() -> None:
assert "write_gitea_full_backup_receipt" in text
assert "awoooi_gitea_full_backup_component_receipt" in text
assert "awoooi_gitea_full_backup_restore_drill_performed" in text
assert "awoooi_gitea_full_backup_secret_value_collected" in text
assert "awoooi_gitea_full_backup_production_restore_performed" in text
assert "gitea dump -c /data/gitea/conf/app.ini" in text
@@ -34,6 +35,11 @@ def test_backup_gitea_receipt_contract_stays_no_secret_no_restore() -> None:
assert "awoooi_gitea_full_backup_secret_value_collected" in text
assert "awoooi_gitea_full_backup_production_restore_performed" in text
assert "awoooi_gitea_full_backup_restore_drill_performed" in text
assert (
'restore_drill_performed{host=\\"${host}\\",service=\\"${service_label}\\"} 0'
in text
)
assert (
'secret_value_collected{host=\\"${host}\\",service=\\"${service_label}\\"} 0'
in text

99
scripts/backup/tests/test_gitea_full_backup_restore_drill.py Normal file
View File

@@ -0,0 +1,99 @@
from __future__ import annotations
import subprocess
import zipfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SCRIPT = ROOT / "gitea-full-backup-restore-drill.sh"
def _write_dump(path: Path, *, include_lfs: bool = True) -> None:
entries = {
"gitea-db.sql": "-- redacted fixture\n",
"repos/wooo/awoooi.git/HEAD": "ref: refs/heads/main\n",
"custom/conf/app.ini": "[server]\nAPP_NAME=fixture\n",
"data/packages/package.bin": "package fixture\n",
"data/attachments/attachment.bin": "attachment fixture\n",
"custom/hooks/pre-receive": "#!/bin/sh\n",
}
if include_lfs:
entries["data/lfs/objects/aa/bb/object"] = "lfs fixture\n"
with zipfile.ZipFile(path, "w") as archive:
for name, content in entries.items():
archive.writestr(name, content)
def test_gitea_full_backup_restore_drill_writes_non_production_receipt(
tmp_path: Path,
) -> None:
dump = tmp_path / "gitea-dump.zip"
textfile = tmp_path / "gitea_full_backup_restore_drill.prom"
_write_dump(dump)
result = subprocess.run(
[
"bash",
str(SCRIPT),
"--dump-zip",
str(dump),
"--write-textfile",
"--textfile",
str(textfile),
"--host",
"110",
],
text=True,
capture_output=True,
check=True,
)
assert "GITEA_FULL_BACKUP_RESTORE_DRILL_OK=1" in result.stdout
assert "SECRET_VALUES_COLLECTED=0" in result.stdout
assert "PRODUCTION_RESTORE_PERFORMED=0" in result.stdout
rendered = textfile.read_text(encoding="utf-8")
assert (
'awoooi_gitea_full_backup_restore_drill_performed{host="110",service="gitea",'
in rendered
)
assert 'drill_scope="structural_metadata_only"' in rendered
assert 'production_restore="false"' in rendered
assert '} 1' in rendered
assert (
'awoooi_gitea_full_backup_secret_value_collected{host="110",service="gitea",'
in rendered
)
assert (
'awoooi_gitea_full_backup_production_restore_performed{host="110",service="gitea",'
in rendered
)
def test_gitea_full_backup_restore_drill_fails_closed_when_component_missing(
tmp_path: Path,
) -> None:
dump = tmp_path / "gitea-dump.zip"
textfile = tmp_path / "gitea_full_backup_restore_drill.prom"
_write_dump(dump, include_lfs=False)
result = subprocess.run(
[
"bash",
str(SCRIPT),
"--dump-zip",
str(dump),
"--write-textfile",
"--textfile",
str(textfile),
],
text=True,
capture_output=True,
)
assert result.returncode == 1
assert "GITEA_FULL_BACKUP_RESTORE_DRILL_OK=0" in result.stdout
assert "ERROR=component_coverage_gap" in result.stdout
rendered = textfile.read_text(encoding="utf-8")
assert 'component="lfs_objects",drill_scope="structural_metadata_only"} 0' in rendered
assert 'production_restore_performed{host="110",service="gitea",' in rendered