#!/bin/bash # Isolated Gitea restore drill. It validates the latest dump without touching # the running container or exposing repository names, database rows, or config. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_MODE="${GITEA_RESTORE_DRILL_TEST_MODE:-0}" CALLER_BACKUP_BASE="${BACKUP_BASE:-}" CALLER_BACKUP_LOG_DIR="${BACKUP_LOG_DIR:-}" CALLER_RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-}" if [ "${TEST_MODE}" != "1" ] && [ -f "${SCRIPT_DIR}/common.sh" ]; then # shellcheck disable=SC1091 source "${SCRIPT_DIR}/common.sh" else BACKUP_BASE="${BACKUP_BASE:-/backup}" RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-/nonexistent}" log_info() { printf '[INFO] %s\n' "$*"; } log_success() { printf '[SUCCESS] %s\n' "$*"; } log_error() { printf '[ERROR] %s\n' "$*" >&2; } notify_clawbot() { return 0; } fi if [ -n "${CALLER_BACKUP_BASE}" ]; then BACKUP_BASE="${CALLER_BACKUP_BASE}" BACKUP_LOG_DIR="${CALLER_BACKUP_LOG_DIR:-${BACKUP_BASE}/logs}" RESTIC_PASSWORD_FILE="${CALLER_RESTIC_PASSWORD_FILE:-${BACKUP_BASE}/scripts/.restic-password}" export BACKUP_BASE BACKUP_LOG_DIR RESTIC_PASSWORD_FILE fi LOCAL_REPO="${GITEA_BACKUP_REPO:-${BACKUP_BASE}/gitea}" STATE_DIR="${GITEA_RESTORE_DRILL_STATE_DIR:-${BACKUP_BASE}/integrity}" WORK_ROOT="${GITEA_RESTORE_DRILL_WORK_ROOT:-${BACKUP_BASE}/tmp}" STATUS_FILE="${STATE_DIR}/gitea-restore-drill.status" MIN_FREE_BUFFER_BYTES="${GITEA_RESTORE_DRILL_MIN_FREE_BUFFER_BYTES:-2147483648}" NOTIFY=0 SOURCE_ZIP="${GITEA_RESTORE_DRILL_SOURCE_ZIP:-}" LIVE_COUNTS="${GITEA_RESTORE_DRILL_LIVE_COUNTS:-}" LIVE_INVENTORY_FILE="${GITEA_RESTORE_DRILL_LIVE_INVENTORY_FILE:-}" START_TIME="$(date +%s)" WORK_DIR="" while [ "$#" -gt 0 ]; do case "$1" in --notify) NOTIFY=1 ;; --no-notify) NOTIFY=0 ;; -h|--help) printf 'Usage: verify-gitea-backup.sh [--notify|--no-notify]\n' exit 0 ;; *) printf 'Unknown argument: %s\n' "$1" >&2 exit 2 ;; esac shift done cleanup() { if [ -n "${WORK_DIR}" ] && [ -d "${WORK_DIR}" ]; then rm -rf "${WORK_DIR}" fi } trap cleanup EXIT low_priority() { if command -v ionice >/dev/null 2>&1; then ionice -c2 -n7 nice -n 10 "$@" else nice -n 10 "$@" fi } file_size() { stat -c '%s' "$1" 2>/dev/null || stat -f '%z' "$1" } write_status() { local success="$1" local reason="$2" local snapshot_id="$3" local snapshot_timestamp="$4" local dump_size="$5" local backup_regular="$6" local backup_wiki="$7" local live_regular="$8" local live_wiki="$9" local database_present="${10}" local config_present="${11}" local crc_ok="${12}" local inventory_match="${13}" local duration local temp_status duration=$(($(date +%s) - START_TIME)) install -d -m 700 "${STATE_DIR}" temp_status="${STATUS_FILE}.tmp.$$" { printf 'timestamp=%s\n' "$(date +%s)" printf 'success=%s\n' "${success}" printf 'reason=%s\n' "${reason}" printf 'snapshot_id=%s\n' "${snapshot_id}" printf 'snapshot_timestamp=%s\n' "${snapshot_timestamp}" printf 'dump_size_bytes=%s\n' "${dump_size}" printf 'backup_regular_repo_count=%s\n' "${backup_regular}" printf 'backup_wiki_repo_count=%s\n' "${backup_wiki}" printf 'live_regular_repo_count=%s\n' "${live_regular}" printf 'live_wiki_repo_count=%s\n' "${live_wiki}" printf 'database_dump_present=%s\n' "${database_present}" printf 'config_present=%s\n' "${config_present}" printf 'archive_crc_ok=%s\n' "${crc_ok}" printf 'inventory_digest_match=%s\n' "${inventory_match}" printf 'duration_seconds=%s\n' "${duration}" } > "${temp_status}" chmod 600 "${temp_status}" mv "${temp_status}" "${STATUS_FILE}" } fail_drill() { local reason="$1" local snapshot_id="${2:-unknown}" local snapshot_timestamp="${3:-unknown}" local dump_size="${4:-0}" local backup_regular="${5:-0}" local backup_wiki="${6:-0}" local live_regular="${7:-0}" local live_wiki="${8:-0}" local database_present="${9:-0}" local config_present="${10:-0}" local crc_ok="${11:-0}" local inventory_match="${12:-0}" write_status 0 "${reason}" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" \ "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" \ "${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}" log_error "Gitea restore drill failed: ${reason}; evidence=${STATUS_FILE}" if [ "${NOTIFY}" -eq 1 ]; then notify_clawbot "failed" "gitea-restore-drill" "Gitea 備份還原演練失敗:${reason}" "$(($(date +%s) - START_TIME))" fi exit 1 } if [ "${TEST_MODE}" != "1" ] && { [ -n "${SOURCE_ZIP}" ] || [ -n "${LIVE_COUNTS}" ] || [ -n "${LIVE_INVENTORY_FILE}" ]; }; then fail_drill "test_override_rejected" fi install -d -m 700 "${WORK_ROOT}" "${STATE_DIR}" WORK_DIR="$(mktemp -d "${WORK_ROOT%/}/gitea-restore-drill.XXXXXX")" chmod 700 "${WORK_DIR}" RESTORED_ZIP="${WORK_DIR}/gitea-dump.zip" snapshot_id="unknown" snapshot_timestamp="unknown" dump_size=0 if [ "${TEST_MODE}" = "1" ]; then [ -f "${SOURCE_ZIP}" ] || fail_drill "test_source_zip_missing" cp "${SOURCE_ZIP}" "${RESTORED_ZIP}" dump_size="$(file_size "${RESTORED_ZIP}")" snapshot_id="synthetic" snapshot_timestamp="synthetic" else [ -d "${LOCAL_REPO}/data" ] || fail_drill "restic_repo_missing" [ -r "${RESTIC_PASSWORD_FILE}" ] || fail_drill "restic_password_file_unavailable" snapshot_meta="" if ! snapshot_meta="$( restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \ python3 -c 'import json,sys rows=json.load(sys.stdin) if rows: row=rows[-1] print("%s\t%s" % (row.get("short_id", "unknown"), row.get("time", "unknown")))' )"; then fail_drill "snapshot_metadata_read_failed" fi [ -n "${snapshot_meta}" ] || fail_drill "snapshot_metadata_missing" IFS=$'\t' read -r snapshot_id snapshot_timestamp <<<"${snapshot_meta}" dump_meta="" if ! dump_meta="$( restic -r "${LOCAL_REPO}" ls latest --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \ python3 -c 'import json,os,sys for line in sys.stdin: try: item=json.loads(line) except json.JSONDecodeError: continue if item.get("struct_type")=="node" and os.path.basename(item.get("path", ""))=="gitea-dump.zip": print("%s\t%s" % (item.get("path", ""), item.get("size", 0))) raise SystemExit' )"; then fail_drill "dump_metadata_read_failed" "${snapshot_id}" "${snapshot_timestamp}" fi [ -n "${dump_meta}" ] || fail_drill "gitea_dump_member_missing" "${snapshot_id}" "${snapshot_timestamp}" IFS=$'\t' read -r dump_path dump_size <<<"${dump_meta}" available_bytes="$(df -Pk "${WORK_ROOT}" | awk 'NR==2 {printf "%.0f\n", $4 * 1024}')" required_bytes=$((dump_size + MIN_FREE_BUFFER_BYTES)) if [ "${available_bytes}" -lt "${required_bytes}" ]; then fail_drill "insufficient_free_space" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" fi log_info "Restoring isolated Gitea dump snapshot=${snapshot_id} size_bytes=${dump_size}" if ! low_priority restic -r "${LOCAL_REPO}" dump latest "${dump_path}" \ --password-file "${RESTIC_PASSWORD_FILE}" > "${RESTORED_ZIP}"; then fail_drill "restic_dump_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" fi fi actual_size="$(file_size "${RESTORED_ZIP}")" if [ "${actual_size}" -ne "${dump_size}" ]; then fail_drill "dump_size_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" fi if ! low_priority unzip -tqq "${RESTORED_ZIP}"; then fail_drill "archive_crc_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" fi crc_ok=1 inventory="" if ! inventory="$( unzip -Z1 "${RESTORED_ZIP}" | awk ' /\.wiki\.git\/HEAD$/ { wiki += 1 } /\.git\/HEAD$/ && $0 !~ /\.wiki\.git\/HEAD$/ { regular += 1 } /(^|\/)gitea-db\.sql$/ { database += 1 } /(^|\/)app\.ini$/ { config += 1 } END { printf "%d %d %d %d", regular, wiki, database, config } ' )"; then fail_drill "archive_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" 0 0 0 0 0 0 "${crc_ok}" fi read -r backup_regular backup_wiki database_present config_present <<<"${inventory}" backup_inventory_file="${WORK_DIR}/backup-repo-inventory" live_inventory_file="${WORK_DIR}/live-repo-inventory" if ! unzip -Z1 "${RESTORED_ZIP}" | awk -F/ ' /\.git\/HEAD$/ { print $(NF - 2) "/" $(NF - 1) } ' | LC_ALL=C sort -u > "${backup_inventory_file}"; then fail_drill "backup_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}" fi if [ -n "${LIVE_INVENTORY_FILE}" ]; then [ -f "${LIVE_INVENTORY_FILE}" ] || fail_drill "test_live_inventory_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}" LC_ALL=C sort -u "${LIVE_INVENTORY_FILE}" > "${live_inventory_file}" else if ! docker exec gitea sh -c 'find /data/git/repositories -type f -path "*.git/HEAD"' 2>/dev/null | awk -F/ ' /\.git\/HEAD$/ { print $(NF - 2) "/" $(NF - 1) } ' | LC_ALL=C sort -u > "${live_inventory_file}"; then fail_drill "live_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}" fi fi live_inventory="$(awk ' /\.wiki\.git$/ { wiki += 1; next } /\.git$/ { regular += 1 } END { printf "%d %d", regular, wiki } ' "${live_inventory_file}")" read -r live_regular live_wiki <<<"${live_inventory}" hash_file() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}' else shasum -a 256 "$1" | awk '{print $1}' fi } backup_inventory_digest="$(hash_file "${backup_inventory_file}")" live_inventory_digest="$(hash_file "${live_inventory_file}")" inventory_match=0 [ "${backup_inventory_digest}" = "${live_inventory_digest}" ] && inventory_match=1 [ "${database_present}" -gt 0 ] || fail_drill "database_dump_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" 0 "${config_present}" "${crc_ok}" [ "${config_present}" -gt 0 ] || fail_drill "config_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" 0 "${crc_ok}" [ "${backup_regular}" -gt 0 ] || fail_drill "repository_inventory_empty" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" 0 "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}" if [ "${backup_regular}" -ne "${live_regular}" ] || [ "${backup_wiki}" -ne "${live_wiki}" ]; then fail_drill "repository_count_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}" fi if [ "${inventory_match}" -ne 1 ]; then fail_drill "repository_inventory_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}" 0 fi write_status 1 "verified" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" \ "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" \ "${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}" log_success "Gitea restore drill verified: regular=${backup_regular}/${live_regular} wiki=${backup_wiki}/${live_wiki}; evidence=${STATUS_FILE}" if [ "${NOTIFY}" -eq 1 ]; then notify_clawbot "success" "gitea-restore-drill" "Gitea 備份還原演練完成,repo=${backup_regular}/${live_regular}" "$(($(date +%s) - START_TIME))" fi