253 lines
9.1 KiB
Bash
Executable File
253 lines
9.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Verify a Gitea full-server backup dump with a non-production, metadata-only
|
|
# structural drill. This never restores into Gitea, never contacts production,
|
|
# never reads tokens, and never prints archive member names or file contents.
|
|
|
|
set -euo pipefail
|
|
|
|
resolve_backup_runtime_script_path() {
|
|
local raw_path="$1" candidate="" resolved="" resolved_directory=""
|
|
[ -n "${raw_path}" ] || return 1
|
|
case "${raw_path}" in /*) candidate="${raw_path}" ;; *) candidate="${PWD}/${raw_path}" ;; esac
|
|
if command -v realpath >/dev/null 2>&1; then
|
|
resolved="$(realpath -e -- "${candidate}" 2>/dev/null || realpath -- "${candidate}" 2>/dev/null)" || resolved=""
|
|
if [ -n "${resolved}" ]; then printf '%s\n' "${resolved}"; return 0; fi
|
|
fi
|
|
if command -v readlink >/dev/null 2>&1; then
|
|
resolved="$(readlink -f -- "${candidate}" 2>/dev/null)" || resolved=""
|
|
if [ -n "${resolved}" ]; then printf '%s\n' "${resolved}"; return 0; fi
|
|
fi
|
|
[ ! -L "${candidate}" ] || return 1
|
|
resolved_directory="$(cd -P -- "$(dirname -- "${candidate}")" 2>/dev/null && pwd -P)" || return 1
|
|
[ -e "${resolved_directory}/$(basename -- "${candidate}")" ] || return 1
|
|
printf '%s/%s\n' "${resolved_directory}" "$(basename -- "${candidate}")"
|
|
}
|
|
|
|
backup_runtime_inherited_fd_status() {
|
|
local expected_path="$1" fd_path="" fd_target=""
|
|
if [ -d "/proc/$$/fd" ]; then
|
|
fd_path="/proc/$$/fd/197"
|
|
[ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1
|
|
fd_target="$(resolve_backup_runtime_script_path "${fd_path}")" || return 2
|
|
[ "${fd_target}" = "${expected_path}" ] || return 2
|
|
return 0
|
|
fi
|
|
fd_path="/dev/fd/197"
|
|
[ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1
|
|
command -v python3 >/dev/null 2>&1 || return 2
|
|
fd_target="$(python3 - <<'PY'
|
|
import fcntl
|
|
import os
|
|
|
|
try:
|
|
target = os.readlink("/dev/fd/197")
|
|
except OSError:
|
|
value = fcntl.fcntl(197, getattr(fcntl, "F_GETPATH", 50), b"\0" * 1024)
|
|
target = os.fsdecode(value.split(b"\0", 1)[0])
|
|
print(os.path.realpath(target))
|
|
PY
|
|
)" || return 2
|
|
[ "${fd_target}" = "${expected_path}" ] || return 2
|
|
}
|
|
|
|
runtime_script_path="$(resolve_backup_runtime_script_path "${BASH_SOURCE[0]:-}")" || {
|
|
echo "backup runtime gate unavailable: script identity unresolved" >&2
|
|
exit 69
|
|
}
|
|
if [[ "${runtime_script_path}" == /backup/scripts/* ]]; then
|
|
command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; }
|
|
[ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; }
|
|
(umask 077; : >> /tmp/agent99-host110-backup-runtime.lock)
|
|
[ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; }
|
|
runtime_lock_path="$(resolve_backup_runtime_script_path /tmp/agent99-host110-backup-runtime.lock)" \
|
|
|| { echo "backup runtime gate unavailable: lock identity unresolved" >&2; exit 69; }
|
|
if backup_runtime_inherited_fd_status "${runtime_lock_path}"; then
|
|
:
|
|
else
|
|
inherited_fd_status=$?
|
|
if [ "${inherited_fd_status}" -eq 1 ]; then
|
|
exec 197>>/tmp/agent99-host110-backup-runtime.lock
|
|
else
|
|
echo "backup runtime gate unsafe: inherited fd 197 identity mismatch" >&2
|
|
exit 69
|
|
fi
|
|
fi
|
|
flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; }
|
|
fi
|
|
|
|
DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}"
|
|
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
|
|
TEXTFILE_PATH="${AIOPS_GITEA_FULL_BACKUP_RESTORE_DRILL_TEXTFILE:-/home/wooo/node_exporter_textfiles/gitea_full_backup_restore_drill.prom}"
|
|
WRITE_TEXTFILE=0
|
|
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-60}"
|
|
DRILL_SCOPE="structural_metadata_only"
|
|
REQUIRED_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments hooks_custom_assets"
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage: scripts/backup/gitea-full-backup-restore-drill.sh [options]
|
|
|
|
Options:
|
|
--dump-zip PATH Gitea dump zip to verify structurally.
|
|
--write-textfile Write node-exporter textfile metric after verification.
|
|
--textfile PATH Textfile metric path.
|
|
--host LABEL Host label for metrics. Default: 110.
|
|
-h, --help Show this help.
|
|
|
|
This verifier inspects only the Gitea dump zip central directory and component
|
|
coverage. It never restores to production, never contacts Gitea, never reads
|
|
credentials, never prints archive member names, and never extracts contents.
|
|
USAGE
|
|
}
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--dump-zip)
|
|
shift
|
|
DUMP_ZIP="${1:?--dump-zip requires PATH}"
|
|
;;
|
|
--write-textfile)
|
|
WRITE_TEXTFILE=1
|
|
;;
|
|
--textfile)
|
|
shift
|
|
TEXTFILE_PATH="${1:?--textfile requires PATH}"
|
|
;;
|
|
--host)
|
|
shift
|
|
HOST_LABEL="${1:?--host requires LABEL}"
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "Unknown argument: $1" >&2
|
|
usage >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
if [ -z "$DUMP_ZIP" ]; then
|
|
echo "ERROR=dump_zip_required" >&2
|
|
usage >&2
|
|
exit 64
|
|
fi
|
|
|
|
ok=0
|
|
entry_count=0
|
|
missing_components=""
|
|
error_reason=""
|
|
|
|
if [ ! -f "$DUMP_ZIP" ]; then
|
|
error_reason="dump_zip_not_found"
|
|
else
|
|
set +e
|
|
drill_result="$(
|
|
timeout "$TIMEOUT_SECONDS" python3 - "$DUMP_ZIP" $REQUIRED_COMPONENTS <<'PY'
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
dump_zip = Path(sys.argv[1])
|
|
required = sys.argv[2:]
|
|
|
|
try:
|
|
with zipfile.ZipFile(dump_zip) as archive:
|
|
names = archive.namelist()
|
|
except zipfile.BadZipFile:
|
|
print("ERROR=bad_zip")
|
|
sys.exit(1)
|
|
except OSError:
|
|
print("ERROR=zip_open_failed")
|
|
sys.exit(1)
|
|
|
|
lowered = [name.lower() for name in names]
|
|
|
|
patterns = {
|
|
"database": ("gitea-db", "database", ".sql"),
|
|
"repositories": ("repo", "repos", "repositories", ".git"),
|
|
"app_settings": ("app.ini", "app.example.ini", "conf/"),
|
|
"lfs_objects": ("lfs",),
|
|
"package_registry": ("package", "packages"),
|
|
"attachments": ("attachment", "attachments"),
|
|
"hooks_custom_assets": ("hook", "custom", "public/"),
|
|
}
|
|
|
|
missing = []
|
|
for component in required:
|
|
needles = patterns.get(component, (component,))
|
|
if not any(any(needle in name for needle in needles) for name in lowered):
|
|
missing.append(component)
|
|
|
|
print(f"ENTRY_COUNT={len(names)}")
|
|
print(f"MISSING_COMPONENTS={','.join(missing)}")
|
|
print("OK=1" if not missing and names else "OK=0")
|
|
PY
|
|
)"
|
|
drill_status=$?
|
|
set -e
|
|
entry_count="$(printf '%s\n' "$drill_result" | awk -F= '$1 == "ENTRY_COUNT" {print $2; exit}')"
|
|
missing_components="$(printf '%s\n' "$drill_result" | awk -F= '$1 == "MISSING_COMPONENTS" {print $2; exit}')"
|
|
if [ "$drill_status" -ne 0 ]; then
|
|
error_reason="$(printf '%s\n' "$drill_result" | awk -F= '$1 == "ERROR" {print $2; exit}')"
|
|
[ -n "$error_reason" ] || error_reason="drill_inspection_failed"
|
|
elif printf '%s\n' "$drill_result" | grep -q '^OK=1$'; then
|
|
ok=1
|
|
else
|
|
error_reason="component_coverage_gap"
|
|
fi
|
|
fi
|
|
|
|
timestamp="$(date +%s)"
|
|
dump_label="$(basename "${DUMP_ZIP:-missing}")"
|
|
host_label="${HOST_LABEL//\\/\\\\}"
|
|
host_label="${host_label//\"/\\\"}"
|
|
dump_label="${dump_label//\\/\\\\}"
|
|
dump_label="${dump_label//\"/\\\"}"
|
|
error_label="${error_reason//\\/\\\\}"
|
|
error_label="${error_label//\"/\\\"}"
|
|
|
|
component_present() {
|
|
local component="$1"
|
|
if [ "$ok" -eq 1 ]; then
|
|
printf '1'
|
|
elif printf ',%s,' "$missing_components" | grep -q ",${component},"; then
|
|
printf '0'
|
|
else
|
|
printf '0'
|
|
fi
|
|
}
|
|
|
|
if [ "$WRITE_TEXTFILE" -eq 1 ]; then
|
|
mkdir -p "$(dirname "$TEXTFILE_PATH")"
|
|
tmp_textfile="${TEXTFILE_PATH}.$$"
|
|
{
|
|
echo '# HELP awoooi_gitea_full_backup_restore_drill_performed Whether a non-production Gitea full-backup structural restore drill succeeded.'
|
|
echo '# TYPE awoooi_gitea_full_backup_restore_drill_performed gauge'
|
|
printf 'awoooi_gitea_full_backup_restore_drill_performed{host="%s",service="gitea",drill_scope="%s",production_restore="false",dump="%s",error="%s"} %s\n' "$host_label" "$DRILL_SCOPE" "$dump_label" "$error_label" "$ok"
|
|
printf 'awoooi_gitea_full_backup_restore_drill_timestamp{host="%s",service="gitea",drill_scope="%s"} %s\n' "$host_label" "$DRILL_SCOPE" "$timestamp"
|
|
printf 'awoooi_gitea_full_backup_restore_drill_entry_count{host="%s",service="gitea",drill_scope="%s"} %s\n' "$host_label" "$DRILL_SCOPE" "${entry_count:-0}"
|
|
for component in $REQUIRED_COMPONENTS; do
|
|
printf 'awoooi_gitea_full_backup_restore_drill_component_receipt{host="%s",service="gitea",component="%s",drill_scope="%s"} %s\n' "$host_label" "$component" "$DRILL_SCOPE" "$(component_present "$component")"
|
|
done
|
|
printf 'awoooi_gitea_full_backup_secret_value_collected{host="%s",service="gitea",drill_scope="%s"} 0\n' "$host_label" "$DRILL_SCOPE"
|
|
printf 'awoooi_gitea_full_backup_production_restore_performed{host="%s",service="gitea",drill_scope="%s"} 0\n' "$host_label" "$DRILL_SCOPE"
|
|
} > "$tmp_textfile"
|
|
mv "$tmp_textfile" "$TEXTFILE_PATH"
|
|
fi
|
|
|
|
printf 'GITEA_FULL_BACKUP_RESTORE_DRILL_OK=%s\n' "$ok"
|
|
printf 'ENTRY_COUNT=%s\n' "${entry_count:-0}"
|
|
printf 'TEXTFILE_WRITTEN=%s\n' "$WRITE_TEXTFILE"
|
|
printf 'SECRET_VALUES_COLLECTED=0\n'
|
|
printf 'PRODUCTION_RESTORE_PERFORMED=0\n'
|
|
if [ -n "$error_reason" ]; then
|
|
printf 'ERROR=%s\n' "$error_reason"
|
|
exit 1
|
|
fi
|