289 lines
12 KiB
Bash
Executable File
289 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
||
|
||
# Isolated Gitea restore drill. It validates the latest dump without touching
|
||
# the running container or exposing repository names, database rows, or config.
|
||
set -euo pipefail
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
TEST_MODE="${GITEA_RESTORE_DRILL_TEST_MODE:-0}"
|
||
CALLER_BACKUP_BASE="${BACKUP_BASE:-}"
|
||
CALLER_BACKUP_LOG_DIR="${BACKUP_LOG_DIR:-}"
|
||
CALLER_RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-}"
|
||
if [ "${TEST_MODE}" != "1" ] && [ -f "${SCRIPT_DIR}/common.sh" ]; then
|
||
# shellcheck disable=SC1091
|
||
source "${SCRIPT_DIR}/common.sh"
|
||
else
|
||
BACKUP_BASE="${BACKUP_BASE:-/backup}"
|
||
RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-/nonexistent}"
|
||
log_info() { printf '[INFO] %s\n' "$*"; }
|
||
log_success() { printf '[SUCCESS] %s\n' "$*"; }
|
||
log_error() { printf '[ERROR] %s\n' "$*" >&2; }
|
||
notify_clawbot() { return 0; }
|
||
fi
|
||
if [ -n "${CALLER_BACKUP_BASE}" ]; then
|
||
BACKUP_BASE="${CALLER_BACKUP_BASE}"
|
||
BACKUP_LOG_DIR="${CALLER_BACKUP_LOG_DIR:-${BACKUP_BASE}/logs}"
|
||
RESTIC_PASSWORD_FILE="${CALLER_RESTIC_PASSWORD_FILE:-${BACKUP_BASE}/scripts/.restic-password}"
|
||
export BACKUP_BASE BACKUP_LOG_DIR RESTIC_PASSWORD_FILE
|
||
fi
|
||
|
||
LOCAL_REPO="${GITEA_BACKUP_REPO:-${BACKUP_BASE}/gitea}"
|
||
STATE_DIR="${GITEA_RESTORE_DRILL_STATE_DIR:-${BACKUP_BASE}/integrity}"
|
||
WORK_ROOT="${GITEA_RESTORE_DRILL_WORK_ROOT:-${BACKUP_BASE}/tmp}"
|
||
STATUS_FILE="${STATE_DIR}/gitea-restore-drill.status"
|
||
MIN_FREE_BUFFER_BYTES="${GITEA_RESTORE_DRILL_MIN_FREE_BUFFER_BYTES:-2147483648}"
|
||
NOTIFY=0
|
||
SOURCE_ZIP="${GITEA_RESTORE_DRILL_SOURCE_ZIP:-}"
|
||
LIVE_COUNTS="${GITEA_RESTORE_DRILL_LIVE_COUNTS:-}"
|
||
LIVE_INVENTORY_FILE="${GITEA_RESTORE_DRILL_LIVE_INVENTORY_FILE:-}"
|
||
START_TIME="$(date +%s)"
|
||
WORK_DIR=""
|
||
|
||
while [ "$#" -gt 0 ]; do
|
||
case "$1" in
|
||
--notify) NOTIFY=1 ;;
|
||
--no-notify) NOTIFY=0 ;;
|
||
-h|--help)
|
||
printf 'Usage: verify-gitea-backup.sh [--notify|--no-notify]\n'
|
||
exit 0
|
||
;;
|
||
*)
|
||
printf 'Unknown argument: %s\n' "$1" >&2
|
||
exit 2
|
||
;;
|
||
esac
|
||
shift
|
||
done
|
||
|
||
cleanup() {
|
||
if [ -n "${WORK_DIR}" ] && [ -d "${WORK_DIR}" ]; then
|
||
rm -rf "${WORK_DIR}"
|
||
fi
|
||
}
|
||
trap cleanup EXIT
|
||
|
||
low_priority() {
|
||
if command -v ionice >/dev/null 2>&1; then
|
||
ionice -c2 -n7 nice -n 10 "$@"
|
||
else
|
||
nice -n 10 "$@"
|
||
fi
|
||
}
|
||
|
||
file_size() {
|
||
stat -c '%s' "$1" 2>/dev/null || stat -f '%z' "$1"
|
||
}
|
||
|
||
write_status() {
|
||
local success="$1"
|
||
local reason="$2"
|
||
local snapshot_id="$3"
|
||
local snapshot_timestamp="$4"
|
||
local dump_size="$5"
|
||
local backup_regular="$6"
|
||
local backup_wiki="$7"
|
||
local live_regular="$8"
|
||
local live_wiki="$9"
|
||
local database_present="${10}"
|
||
local config_present="${11}"
|
||
local crc_ok="${12}"
|
||
local inventory_match="${13}"
|
||
local duration
|
||
local temp_status
|
||
|
||
duration=$(($(date +%s) - START_TIME))
|
||
install -d -m 700 "${STATE_DIR}"
|
||
temp_status="${STATUS_FILE}.tmp.$$"
|
||
{
|
||
printf 'timestamp=%s\n' "$(date +%s)"
|
||
printf 'success=%s\n' "${success}"
|
||
printf 'reason=%s\n' "${reason}"
|
||
printf 'snapshot_id=%s\n' "${snapshot_id}"
|
||
printf 'snapshot_timestamp=%s\n' "${snapshot_timestamp}"
|
||
printf 'dump_size_bytes=%s\n' "${dump_size}"
|
||
printf 'backup_regular_repo_count=%s\n' "${backup_regular}"
|
||
printf 'backup_wiki_repo_count=%s\n' "${backup_wiki}"
|
||
printf 'live_regular_repo_count=%s\n' "${live_regular}"
|
||
printf 'live_wiki_repo_count=%s\n' "${live_wiki}"
|
||
printf 'database_dump_present=%s\n' "${database_present}"
|
||
printf 'config_present=%s\n' "${config_present}"
|
||
printf 'archive_crc_ok=%s\n' "${crc_ok}"
|
||
printf 'inventory_digest_match=%s\n' "${inventory_match}"
|
||
printf 'duration_seconds=%s\n' "${duration}"
|
||
} > "${temp_status}"
|
||
chmod 600 "${temp_status}"
|
||
mv "${temp_status}" "${STATUS_FILE}"
|
||
}
|
||
|
||
fail_drill() {
|
||
local reason="$1"
|
||
local snapshot_id="${2:-unknown}"
|
||
local snapshot_timestamp="${3:-unknown}"
|
||
local dump_size="${4:-0}"
|
||
local backup_regular="${5:-0}"
|
||
local backup_wiki="${6:-0}"
|
||
local live_regular="${7:-0}"
|
||
local live_wiki="${8:-0}"
|
||
local database_present="${9:-0}"
|
||
local config_present="${10:-0}"
|
||
local crc_ok="${11:-0}"
|
||
local inventory_match="${12:-0}"
|
||
|
||
write_status 0 "${reason}" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" \
|
||
"${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" \
|
||
"${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}"
|
||
log_error "Gitea restore drill failed: ${reason}; evidence=${STATUS_FILE}"
|
||
if [ "${NOTIFY}" -eq 1 ]; then
|
||
notify_clawbot "failed" "gitea-restore-drill" "Gitea 備份還原演練失敗：${reason}" "$(($(date +%s) - START_TIME))"
|
||
fi
|
||
exit 1
|
||
}
|
||
|
||
if [ "${TEST_MODE}" != "1" ] && { [ -n "${SOURCE_ZIP}" ] || [ -n "${LIVE_COUNTS}" ] || [ -n "${LIVE_INVENTORY_FILE}" ]; }; then
|
||
fail_drill "test_override_rejected"
|
||
fi
|
||
|
||
install -d -m 700 "${WORK_ROOT}" "${STATE_DIR}"
|
||
WORK_DIR="$(mktemp -d "${WORK_ROOT%/}/gitea-restore-drill.XXXXXX")"
|
||
chmod 700 "${WORK_DIR}"
|
||
RESTORED_ZIP="${WORK_DIR}/gitea-dump.zip"
|
||
|
||
snapshot_id="unknown"
|
||
snapshot_timestamp="unknown"
|
||
dump_size=0
|
||
|
||
if [ "${TEST_MODE}" = "1" ]; then
|
||
[ -f "${SOURCE_ZIP}" ] || fail_drill "test_source_zip_missing"
|
||
cp "${SOURCE_ZIP}" "${RESTORED_ZIP}"
|
||
dump_size="$(file_size "${RESTORED_ZIP}")"
|
||
snapshot_id="synthetic"
|
||
snapshot_timestamp="synthetic"
|
||
else
|
||
[ -d "${LOCAL_REPO}/data" ] || fail_drill "restic_repo_missing"
|
||
[ -r "${RESTIC_PASSWORD_FILE}" ] || fail_drill "restic_password_file_unavailable"
|
||
|
||
snapshot_meta=""
|
||
if ! snapshot_meta="$(
|
||
restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \
|
||
python3 -c 'import json,sys
|
||
rows=json.load(sys.stdin)
|
||
if rows:
|
||
row=rows[-1]
|
||
print("%s\t%s" % (row.get("short_id", "unknown"), row.get("time", "unknown")))'
|
||
)"; then
|
||
fail_drill "snapshot_metadata_read_failed"
|
||
fi
|
||
[ -n "${snapshot_meta}" ] || fail_drill "snapshot_metadata_missing"
|
||
IFS=$'\t' read -r snapshot_id snapshot_timestamp <<<"${snapshot_meta}"
|
||
|
||
dump_meta=""
|
||
if ! dump_meta="$(
|
||
restic -r "${LOCAL_REPO}" ls latest --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \
|
||
python3 -c 'import json,os,sys
|
||
for line in sys.stdin:
|
||
try: item=json.loads(line)
|
||
except json.JSONDecodeError: continue
|
||
if item.get("struct_type")=="node" and os.path.basename(item.get("path", ""))=="gitea-dump.zip":
|
||
print("%s\t%s" % (item.get("path", ""), item.get("size", 0)))
|
||
raise SystemExit'
|
||
)"; then
|
||
fail_drill "dump_metadata_read_failed" "${snapshot_id}" "${snapshot_timestamp}"
|
||
fi
|
||
[ -n "${dump_meta}" ] || fail_drill "gitea_dump_member_missing" "${snapshot_id}" "${snapshot_timestamp}"
|
||
IFS=$'\t' read -r dump_path dump_size <<<"${dump_meta}"
|
||
|
||
available_bytes="$(df -Pk "${WORK_ROOT}" | awk 'NR==2 {printf "%.0f\n", $4 * 1024}')"
|
||
required_bytes=$((dump_size + MIN_FREE_BUFFER_BYTES))
|
||
if [ "${available_bytes}" -lt "${required_bytes}" ]; then
|
||
fail_drill "insufficient_free_space" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}"
|
||
fi
|
||
|
||
log_info "Restoring isolated Gitea dump snapshot=${snapshot_id} size_bytes=${dump_size}"
|
||
if ! low_priority restic -r "${LOCAL_REPO}" dump latest "${dump_path}" \
|
||
--password-file "${RESTIC_PASSWORD_FILE}" > "${RESTORED_ZIP}"; then
|
||
fail_drill "restic_dump_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}"
|
||
fi
|
||
fi
|
||
|
||
actual_size="$(file_size "${RESTORED_ZIP}")"
|
||
if [ "${actual_size}" -ne "${dump_size}" ]; then
|
||
fail_drill "dump_size_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}"
|
||
fi
|
||
|
||
if ! low_priority unzip -tqq "${RESTORED_ZIP}"; then
|
||
fail_drill "archive_crc_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}"
|
||
fi
|
||
crc_ok=1
|
||
|
||
inventory=""
|
||
if ! inventory="$(
|
||
unzip -Z1 "${RESTORED_ZIP}" | awk '
|
||
/\.wiki\.git\/HEAD$/ { wiki += 1 }
|
||
/\.git\/HEAD$/ && $0 !~ /\.wiki\.git\/HEAD$/ { regular += 1 }
|
||
/(^|\/)gitea-db\.sql$/ { database += 1 }
|
||
/(^|\/)app\.ini$/ { config += 1 }
|
||
END { printf "%d %d %d %d", regular, wiki, database, config }
|
||
'
|
||
)"; then
|
||
fail_drill "archive_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" 0 0 0 0 0 0 "${crc_ok}"
|
||
fi
|
||
read -r backup_regular backup_wiki database_present config_present <<<"${inventory}"
|
||
|
||
backup_inventory_file="${WORK_DIR}/backup-repo-inventory"
|
||
live_inventory_file="${WORK_DIR}/live-repo-inventory"
|
||
if ! unzip -Z1 "${RESTORED_ZIP}" | awk -F/ '
|
||
/\.git\/HEAD$/ { print $(NF - 2) "/" $(NF - 1) }
|
||
' | LC_ALL=C sort -u > "${backup_inventory_file}"; then
|
||
fail_drill "backup_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}"
|
||
fi
|
||
|
||
if [ -n "${LIVE_INVENTORY_FILE}" ]; then
|
||
[ -f "${LIVE_INVENTORY_FILE}" ] || fail_drill "test_live_inventory_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}"
|
||
LC_ALL=C sort -u "${LIVE_INVENTORY_FILE}" > "${live_inventory_file}"
|
||
else
|
||
if ! docker exec gitea sh -c 'find /data/git/repositories -type f -path "*.git/HEAD"' 2>/dev/null | awk -F/ '
|
||
/\.git\/HEAD$/ { print $(NF - 2) "/" $(NF - 1) }
|
||
' | LC_ALL=C sort -u > "${live_inventory_file}"; then
|
||
fail_drill "live_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}"
|
||
fi
|
||
fi
|
||
|
||
live_inventory="$(awk '
|
||
/\.wiki\.git$/ { wiki += 1; next }
|
||
/\.git$/ { regular += 1 }
|
||
END { printf "%d %d", regular, wiki }
|
||
' "${live_inventory_file}")"
|
||
read -r live_regular live_wiki <<<"${live_inventory}"
|
||
|
||
hash_file() {
|
||
if command -v sha256sum >/dev/null 2>&1; then
|
||
sha256sum "$1" | awk '{print $1}'
|
||
else
|
||
shasum -a 256 "$1" | awk '{print $1}'
|
||
fi
|
||
}
|
||
|
||
backup_inventory_digest="$(hash_file "${backup_inventory_file}")"
|
||
live_inventory_digest="$(hash_file "${live_inventory_file}")"
|
||
inventory_match=0
|
||
[ "${backup_inventory_digest}" = "${live_inventory_digest}" ] && inventory_match=1
|
||
|
||
[ "${database_present}" -gt 0 ] || fail_drill "database_dump_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" 0 "${config_present}" "${crc_ok}"
|
||
[ "${config_present}" -gt 0 ] || fail_drill "config_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" 0 "${crc_ok}"
|
||
[ "${backup_regular}" -gt 0 ] || fail_drill "repository_inventory_empty" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" 0 "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}"
|
||
|
||
if [ "${backup_regular}" -ne "${live_regular}" ] || [ "${backup_wiki}" -ne "${live_wiki}" ]; then
|
||
fail_drill "repository_count_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}"
|
||
fi
|
||
if [ "${inventory_match}" -ne 1 ]; then
|
||
fail_drill "repository_inventory_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}" 0
|
||
fi
|
||
|
||
write_status 1 "verified" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" \
|
||
"${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" \
|
||
"${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}"
|
||
log_success "Gitea restore drill verified: regular=${backup_regular}/${live_regular} wiki=${backup_wiki}/${live_wiki}; evidence=${STATUS_FILE}"
|
||
if [ "${NOTIFY}" -eq 1 ]; then
|
||
notify_clawbot "success" "gitea-restore-drill" "Gitea 備份還原演練完成，repo=${backup_regular}/${live_regular}" "$(($(date +%s) - START_TIME))"
|
||
fi
|