fix(gitea): add full backup restore drill receipt path
This commit is contained in:
ogt
@@ -60,7 +60,7 @@ write_gitea_full_backup_receipt() {
|
||||
echo "# TYPE awoooi_gitea_full_backup_restic_snapshot_present gauge"
|
||||
echo "# HELP awoooi_gitea_full_backup_component_receipt Whether the Gitea full-server dump includes a redacted component receipt."
|
||||
echo "# TYPE awoooi_gitea_full_backup_component_receipt gauge"
|
||||
echo "# HELP awoooi_gitea_full_backup_restore_drill_performed Whether this receipt performed a production restore drill."
|
||||
echo "# HELP awoooi_gitea_full_backup_restore_drill_performed Whether a non-production Gitea full-backup restore drill receipt is present."
|
||||
echo "# TYPE awoooi_gitea_full_backup_restore_drill_performed gauge"
|
||||
echo "# HELP awoooi_gitea_full_backup_secret_value_collected Whether this receipt collected secret values."
|
||||
echo "# TYPE awoooi_gitea_full_backup_secret_value_collected gauge"
|
||||
|
||||
182
scripts/backup/gitea-full-backup-restore-drill.sh
Executable file
182
scripts/backup/gitea-full-backup-restore-drill.sh
Executable file
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env bash
|
||||
# Verify a Gitea full-server backup dump with a non-production, metadata-only
|
||||
# structural drill. This never restores into Gitea, never contacts production,
|
||||
# never reads tokens, and never prints archive member names or file contents.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}"
|
||||
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
|
||||
TEXTFILE_PATH="${AIOPS_GITEA_FULL_BACKUP_RESTORE_DRILL_TEXTFILE:-/home/wooo/node_exporter_textfiles/gitea_full_backup_restore_drill.prom}"
|
||||
WRITE_TEXTFILE=0
|
||||
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-60}"
|
||||
DRILL_SCOPE="structural_metadata_only"
|
||||
REQUIRED_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments hooks_custom_assets"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: scripts/backup/gitea-full-backup-restore-drill.sh [options]
|
||||
|
||||
Options:
|
||||
--dump-zip PATH Gitea dump zip to verify structurally.
|
||||
--write-textfile Write node-exporter textfile metric after verification.
|
||||
--textfile PATH Textfile metric path.
|
||||
--host LABEL Host label for metrics. Default: 110.
|
||||
-h, --help Show this help.
|
||||
|
||||
This verifier inspects only the Gitea dump zip central directory and component
|
||||
coverage. It never restores to production, never contacts Gitea, never reads
|
||||
credentials, never prints archive member names, and never extracts contents.
|
||||
USAGE
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--dump-zip)
|
||||
shift
|
||||
DUMP_ZIP="${1:?--dump-zip requires PATH}"
|
||||
;;
|
||||
--write-textfile)
|
||||
WRITE_TEXTFILE=1
|
||||
;;
|
||||
--textfile)
|
||||
shift
|
||||
TEXTFILE_PATH="${1:?--textfile requires PATH}"
|
||||
;;
|
||||
--host)
|
||||
shift
|
||||
HOST_LABEL="${1:?--host requires LABEL}"
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [ -z "$DUMP_ZIP" ]; then
|
||||
echo "ERROR=dump_zip_required" >&2
|
||||
usage >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
ok=0
|
||||
entry_count=0
|
||||
missing_components=""
|
||||
error_reason=""
|
||||
|
||||
if [ ! -f "$DUMP_ZIP" ]; then
|
||||
error_reason="dump_zip_not_found"
|
||||
else
|
||||
set +e
|
||||
drill_result="$(
|
||||
timeout "$TIMEOUT_SECONDS" python3 - "$DUMP_ZIP" $REQUIRED_COMPONENTS <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
dump_zip = Path(sys.argv[1])
|
||||
required = sys.argv[2:]
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(dump_zip) as archive:
|
||||
names = archive.namelist()
|
||||
except zipfile.BadZipFile:
|
||||
print("ERROR=bad_zip")
|
||||
sys.exit(1)
|
||||
except OSError:
|
||||
print("ERROR=zip_open_failed")
|
||||
sys.exit(1)
|
||||
|
||||
lowered = [name.lower() for name in names]
|
||||
|
||||
patterns = {
|
||||
"database": ("gitea-db", "database", ".sql"),
|
||||
"repositories": ("repo", "repos", "repositories", ".git"),
|
||||
"app_settings": ("app.ini", "app.example.ini", "conf/"),
|
||||
"lfs_objects": ("lfs",),
|
||||
"package_registry": ("package", "packages"),
|
||||
"attachments": ("attachment", "attachments"),
|
||||
"hooks_custom_assets": ("hook", "custom", "public/"),
|
||||
}
|
||||
|
||||
missing = []
|
||||
for component in required:
|
||||
needles = patterns.get(component, (component,))
|
||||
if not any(any(needle in name for needle in needles) for name in lowered):
|
||||
missing.append(component)
|
||||
|
||||
print(f"ENTRY_COUNT={len(names)}")
|
||||
print(f"MISSING_COMPONENTS={','.join(missing)}")
|
||||
print("OK=1" if not missing and names else "OK=0")
|
||||
PY
|
||||
)"
|
||||
drill_status=$?
|
||||
set -e
|
||||
entry_count="$(printf '%s\n' "$drill_result" | awk -F= '$1 == "ENTRY_COUNT" {print $2; exit}')"
|
||||
missing_components="$(printf '%s\n' "$drill_result" | awk -F= '$1 == "MISSING_COMPONENTS" {print $2; exit}')"
|
||||
if [ "$drill_status" -ne 0 ]; then
|
||||
error_reason="$(printf '%s\n' "$drill_result" | awk -F= '$1 == "ERROR" {print $2; exit}')"
|
||||
[ -n "$error_reason" ] || error_reason="drill_inspection_failed"
|
||||
elif printf '%s\n' "$drill_result" | grep -q '^OK=1$'; then
|
||||
ok=1
|
||||
else
|
||||
error_reason="component_coverage_gap"
|
||||
fi
|
||||
fi
|
||||
|
||||
timestamp="$(date +%s)"
|
||||
dump_label="$(basename "${DUMP_ZIP:-missing}")"
|
||||
host_label="${HOST_LABEL//\\/\\\\}"
|
||||
host_label="${host_label//\"/\\\"}"
|
||||
dump_label="${dump_label//\\/\\\\}"
|
||||
dump_label="${dump_label//\"/\\\"}"
|
||||
error_label="${error_reason//\\/\\\\}"
|
||||
error_label="${error_label//\"/\\\"}"
|
||||
|
||||
component_present() {
|
||||
local component="$1"
|
||||
if [ "$ok" -eq 1 ]; then
|
||||
printf '1'
|
||||
elif printf ',%s,' "$missing_components" | grep -q ",${component},"; then
|
||||
printf '0'
|
||||
else
|
||||
printf '0'
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "$WRITE_TEXTFILE" -eq 1 ]; then
|
||||
mkdir -p "$(dirname "$TEXTFILE_PATH")"
|
||||
tmp_textfile="${TEXTFILE_PATH}.$$"
|
||||
{
|
||||
echo '# HELP awoooi_gitea_full_backup_restore_drill_performed Whether a non-production Gitea full-backup structural restore drill succeeded.'
|
||||
echo '# TYPE awoooi_gitea_full_backup_restore_drill_performed gauge'
|
||||
printf 'awoooi_gitea_full_backup_restore_drill_performed{host="%s",service="gitea",drill_scope="%s",production_restore="false",dump="%s",error="%s"} %s\n' "$host_label" "$DRILL_SCOPE" "$dump_label" "$error_label" "$ok"
|
||||
printf 'awoooi_gitea_full_backup_restore_drill_timestamp{host="%s",service="gitea",drill_scope="%s"} %s\n' "$host_label" "$DRILL_SCOPE" "$timestamp"
|
||||
printf 'awoooi_gitea_full_backup_restore_drill_entry_count{host="%s",service="gitea",drill_scope="%s"} %s\n' "$host_label" "$DRILL_SCOPE" "${entry_count:-0}"
|
||||
for component in $REQUIRED_COMPONENTS; do
|
||||
printf 'awoooi_gitea_full_backup_restore_drill_component_receipt{host="%s",service="gitea",component="%s",drill_scope="%s"} %s\n' "$host_label" "$component" "$DRILL_SCOPE" "$(component_present "$component")"
|
||||
done
|
||||
printf 'awoooi_gitea_full_backup_secret_value_collected{host="%s",service="gitea",drill_scope="%s"} 0\n' "$host_label" "$DRILL_SCOPE"
|
||||
printf 'awoooi_gitea_full_backup_production_restore_performed{host="%s",service="gitea",drill_scope="%s"} 0\n' "$host_label" "$DRILL_SCOPE"
|
||||
} > "$tmp_textfile"
|
||||
mv "$tmp_textfile" "$TEXTFILE_PATH"
|
||||
fi
|
||||
|
||||
printf 'GITEA_FULL_BACKUP_RESTORE_DRILL_OK=%s\n' "$ok"
|
||||
printf 'ENTRY_COUNT=%s\n' "${entry_count:-0}"
|
||||
printf 'TEXTFILE_WRITTEN=%s\n' "$WRITE_TEXTFILE"
|
||||
printf 'SECRET_VALUES_COLLECTED=0\n'
|
||||
printf 'PRODUCTION_RESTORE_PERFORMED=0\n'
|
||||
if [ -n "$error_reason" ]; then
|
||||
printf 'ERROR=%s\n' "$error_reason"
|
||||
exit 1
|
||||
fi
|
||||
@@ -12,6 +12,7 @@ def test_backup_gitea_writes_full_server_component_receipts() -> None:
|
||||
|
||||
assert "write_gitea_full_backup_receipt" in text
|
||||
assert "awoooi_gitea_full_backup_component_receipt" in text
|
||||
assert "awoooi_gitea_full_backup_restore_drill_performed" in text
|
||||
assert "awoooi_gitea_full_backup_secret_value_collected" in text
|
||||
assert "awoooi_gitea_full_backup_production_restore_performed" in text
|
||||
assert "gitea dump -c /data/gitea/conf/app.ini" in text
|
||||
@@ -34,6 +35,11 @@ def test_backup_gitea_receipt_contract_stays_no_secret_no_restore() -> None:
|
||||
|
||||
assert "awoooi_gitea_full_backup_secret_value_collected" in text
|
||||
assert "awoooi_gitea_full_backup_production_restore_performed" in text
|
||||
assert "awoooi_gitea_full_backup_restore_drill_performed" in text
|
||||
assert (
|
||||
'restore_drill_performed{host=\\"${host}\\",service=\\"${service_label}\\"} 0'
|
||||
in text
|
||||
)
|
||||
assert (
|
||||
'secret_value_collected{host=\\"${host}\\",service=\\"${service_label}\\"} 0'
|
||||
in text
|
||||
|
||||
99
scripts/backup/tests/test_gitea_full_backup_restore_drill.py
Normal file
99
scripts/backup/tests/test_gitea_full_backup_restore_drill.py
Normal file
@@ -0,0 +1,99 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPT = ROOT / "gitea-full-backup-restore-drill.sh"
|
||||
|
||||
|
||||
def _write_dump(path: Path, *, include_lfs: bool = True) -> None:
|
||||
entries = {
|
||||
"gitea-db.sql": "-- redacted fixture\n",
|
||||
"repos/wooo/awoooi.git/HEAD": "ref: refs/heads/main\n",
|
||||
"custom/conf/app.ini": "[server]\nAPP_NAME=fixture\n",
|
||||
"data/packages/package.bin": "package fixture\n",
|
||||
"data/attachments/attachment.bin": "attachment fixture\n",
|
||||
"custom/hooks/pre-receive": "#!/bin/sh\n",
|
||||
}
|
||||
if include_lfs:
|
||||
entries["data/lfs/objects/aa/bb/object"] = "lfs fixture\n"
|
||||
with zipfile.ZipFile(path, "w") as archive:
|
||||
for name, content in entries.items():
|
||||
archive.writestr(name, content)
|
||||
|
||||
|
||||
def test_gitea_full_backup_restore_drill_writes_non_production_receipt(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
dump = tmp_path / "gitea-dump.zip"
|
||||
textfile = tmp_path / "gitea_full_backup_restore_drill.prom"
|
||||
_write_dump(dump)
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(SCRIPT),
|
||||
"--dump-zip",
|
||||
str(dump),
|
||||
"--write-textfile",
|
||||
"--textfile",
|
||||
str(textfile),
|
||||
"--host",
|
||||
"110",
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
assert "GITEA_FULL_BACKUP_RESTORE_DRILL_OK=1" in result.stdout
|
||||
assert "SECRET_VALUES_COLLECTED=0" in result.stdout
|
||||
assert "PRODUCTION_RESTORE_PERFORMED=0" in result.stdout
|
||||
rendered = textfile.read_text(encoding="utf-8")
|
||||
assert (
|
||||
'awoooi_gitea_full_backup_restore_drill_performed{host="110",service="gitea",'
|
||||
in rendered
|
||||
)
|
||||
assert 'drill_scope="structural_metadata_only"' in rendered
|
||||
assert 'production_restore="false"' in rendered
|
||||
assert '} 1' in rendered
|
||||
assert (
|
||||
'awoooi_gitea_full_backup_secret_value_collected{host="110",service="gitea",'
|
||||
in rendered
|
||||
)
|
||||
assert (
|
||||
'awoooi_gitea_full_backup_production_restore_performed{host="110",service="gitea",'
|
||||
in rendered
|
||||
)
|
||||
|
||||
|
||||
def test_gitea_full_backup_restore_drill_fails_closed_when_component_missing(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
dump = tmp_path / "gitea-dump.zip"
|
||||
textfile = tmp_path / "gitea_full_backup_restore_drill.prom"
|
||||
_write_dump(dump, include_lfs=False)
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(SCRIPT),
|
||||
"--dump-zip",
|
||||
str(dump),
|
||||
"--write-textfile",
|
||||
"--textfile",
|
||||
str(textfile),
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "GITEA_FULL_BACKUP_RESTORE_DRILL_OK=0" in result.stdout
|
||||
assert "ERROR=component_coverage_gap" in result.stdout
|
||||
rendered = textfile.read_text(encoding="utf-8")
|
||||
assert 'component="lfs_objects",drill_scope="structural_metadata_only"} 0' in rendered
|
||||
assert 'production_restore_performed{host="110",service="gitea",' in rendered
|
||||
Reference in New Issue
Block a user