wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(ops): close Gitea backup restore verification

Browse Source
This commit is contained in:
ogt
2026-07-10 20:26:00 +08:00
parent 866811bbb3
commit 872a3ed5eb
22 changed files with 1460 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
scripts/backup/backup-status.sh
View File

@@ -221,8 +221,10 @@ configured_missing_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_conf
configured_missing_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_configured" 0)"
script_missing_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_script_present" 0)"
script_missing_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_script_present" 0)"
fresh_total_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh")"
fresh_total_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh")"
monitored_total_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh")"
monitored_total_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh")"
fresh_ok_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh" 1)"
fresh_ok_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh" 1)"
stale_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh" 0)"
stale_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh" 0)"
failed_total_110="$(metric_sum "${TEXTFILE_110}" "awoooi_backup_last_run_failed_count")"
@@ -236,7 +238,12 @@ offsite_rclone_configured="$(awk '/^awoooi_backup_offsite_configured\{.*provider
offsite_rclone_fresh="$(awk '/^awoooi_backup_offsite_fresh\{.*provider="rclone"/ { print $2; found=1; exit } END { if (!found) print 0 }' "${TEXTFILE_110}" 2>/dev/null || echo 0)"
escrow_missing="$(metric_first "${TEXTFILE_110}" "awoooi_backup_dr_credential_escrow_missing_count")"
core_blockers=$((host_110_missing + host_188_missing + configured_missing_110 + configured_missing_188 + script_missing_110 + script_missing_188 + stale_110 + stale_188 + component_failed_110 + component_failed_188 + integrity_stale_110))
freshness_metrics_missing_110=0
freshness_metrics_missing_188=0
[ "${monitored_total_110}" -gt 0 ] || freshness_metrics_missing_110=1
[ "${monitored_total_188}" -gt 0 ] || freshness_metrics_missing_188=1
core_blockers=$((host_110_missing + host_188_missing + configured_missing_110 + configured_missing_188 + script_missing_110 + script_missing_188 + freshness_metrics_missing_110 + freshness_metrics_missing_188 + stale_110 + stale_188 + component_failed_110 + component_failed_188 + integrity_stale_110))
dr_warnings=0
if [ "${offsite_configured%.*}" -lt 1 ] 2>/dev/null; then
dr_warnings=$((dr_warnings + 1))
@@ -265,7 +272,7 @@ missing_scripts_188="$(label_list_for_zero "${TEXTFILE_188_TMP}" "awoooi_backup_
backup_all_ts="$(metric_value_for_label "${TEXTFILE_110}" "awoooi_backup_job_last_success_timestamp" "job" "backup_all")"
last_backup_all="$(human_timestamp "${backup_all_ts}")"
message="${headline}; 110備份=${fresh_total_110}/13 fresh component_failed=${component_failed_110} aggregate_failed=${failed_total_110}; 188備份=${fresh_total_188}/2 fresh component_failed=${component_failed_188} aggregate_failed=${failed_total_188}; integrity_stale=${integrity_stale_110}; offsite_configured=${offsite_configured}; offsite_fresh=${offsite_fresh}; rclone_gdrive_configured=${offsite_rclone_configured}; rclone_gdrive_fresh=${offsite_rclone_fresh}; escrow_missing=${escrow_missing}; last_backup_all=${last_backup_all}"
message="${headline}; 110備份=${fresh_ok_110}/${monitored_total_110} fresh component_failed=${component_failed_110} aggregate_failed=${failed_total_110}; 188備份=${fresh_ok_188}/${monitored_total_188} fresh component_failed=${component_failed_188} aggregate_failed=${failed_total_188}; freshness_metrics_missing=${freshness_metrics_missing_110}/${freshness_metrics_missing_188}; integrity_stale=${integrity_stale_110}; offsite_configured=${offsite_configured}; offsite_fresh=${offsite_fresh}; rclone_gdrive_configured=${offsite_rclone_configured}; rclone_gdrive_fresh=${offsite_rclone_fresh}; escrow_missing=${escrow_missing}; last_backup_all=${last_backup_all}"
if [ "${core_blockers}" -gt 0 ]; then
message="${message}; stale110=${stale_jobs_110:-none}; stale188=${stale_jobs_188:-none}; missing_script110=${missing_scripts_110:-none}; missing_script188=${missing_scripts_188:-none}"
@@ -273,7 +280,7 @@ fi
{
log_line "${message}"
log_line "DETAIL core_blockers=${core_blockers} dr_warnings=${dr_warnings} configured_missing_110=${configured_missing_110} configured_missing_188=${configured_missing_188} script_missing_110=${script_missing_110} script_missing_188=${script_missing_188}"
log_line "DETAIL core_blockers=${core_blockers} dr_warnings=${dr_warnings} configured_missing_110=${configured_missing_110} configured_missing_188=${configured_missing_188} script_missing_110=${script_missing_110} script_missing_188=${script_missing_188} freshness_metrics_missing_110=${freshness_metrics_missing_110} freshness_metrics_missing_188=${freshness_metrics_missing_188}"
} | tee -a "${LOG_DIR}/backup-status.log"
if [ "${NOTIFY}" -eq 1 ]; then
@@ -285,7 +292,7 @@ if [ "${NOTIFY}" -eq 1 ]; then
failed_interval_hours="${BACKUP_STATUS_FAILED_INTERVAL_HOURS:-6}"
now_ts="$(date +%s)"
notify_fingerprint="$(
printf '%s' "status=${status};core=${core_blockers};dr=${dr_warnings};cm110=${configured_missing_110};cm188=${configured_missing_188};sm110=${script_missing_110};sm188=${script_missing_188};stale110=${stale_jobs_110:-none};stale188=${stale_jobs_188:-none};offsite=${offsite_configured}:${offsite_fresh};escrow=${escrow_missing}" \
printf '%s' "status=${status};core=${core_blockers};dr=${dr_warnings};cm110=${configured_missing_110};cm188=${configured_missing_188};sm110=${script_missing_110};sm188=${script_missing_188};fm110=${freshness_metrics_missing_110};fm188=${freshness_metrics_missing_188};fresh110=${fresh_ok_110}/${monitored_total_110};fresh188=${fresh_ok_188}/${monitored_total_188};stale110=${stale_jobs_110:-none};stale188=${stale_jobs_188:-none};offsite=${offsite_configured}:${offsite_fresh};escrow=${escrow_missing}" \
| cksum \
| awk '{print $1}'
)"

12
scripts/backup/check-backup-integrity.sh
View File

@@ -190,6 +190,18 @@ main() {
failed=$((failed + 1))
fi
else
if [ "${name}" = "gitea" ] && [ -x "${BACKUP_BASE}/scripts/verify-gitea-backup.sh" ]; then
log_info "Gitea specialized restore drill: archive CRC, required assets, and live inventory match"
if "${BACKUP_BASE}/scripts/verify-gitea-backup.sh" --no-notify >> "${LOG_FILE}" 2>&1; then
log_success "repo ${name} specialized restore drill OK"
echo "repo=${name} status=restore_drill_specialized_ok snapshots=${count} latest=${latest_ts}" >> "${LOG_FILE}"
else
log_error "repo ${name} specialized restore drill failed"
echo "repo=${name} status=restore_drill_specialized_failed snapshots=${count} latest=${latest_ts}" >> "${LOG_FILE}"
failed=$((failed + 1))
fi
continue
fi
local sample
local sample_out
sample=$(MAX_SAMPLE_BYTES="${MAX_SAMPLE_BYTES}" sample_path_for_repo "${repo}")

172
scripts/backup/gitea-bundle-backup-sync-188.sh Executable file
View File

@@ -0,0 +1,172 @@
#!/usr/bin/env bash
# Generate Gitea Git bundles on host 110 from the local Gitea container, sync
# the complete set to 188 without deletion, then run an isolated sample restore.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
LOCAL_OUTPUT_ROOT="${GITEA_BUNDLE_LOCAL_ROOT:-/backup/gitea-bundles}"
LOCAL_CONTAINER="${GITEA_BUNDLE_LOCAL_CONTAINER:-gitea}"
REMOTE_HOST="${GITEA_BUNDLE_REMOTE_HOST:-ollama@192.168.0.188}"
REMOTE_OUTPUT_ROOT="${GITEA_BUNDLE_REMOTE_ROOT:-/home/ollama/backup/110/gitea/git-bundles}"
REMOTE_VERIFY_SCRIPT="${GITEA_BUNDLE_REMOTE_VERIFY_SCRIPT:-/home/ollama/scripts/gitea-bundle-sample-restore-dry-run.sh}"
REMOTE_EXPORTER_SCRIPT="${GITEA_BUNDLE_REMOTE_EXPORTER_SCRIPT:-/home/ollama/scripts/backup-health-textfile-exporter.py}"
BUNDLE_SCRIPT="${GITEA_BUNDLE_SCRIPT:-${SCRIPT_DIR}/gitea-repo-bundle-backup.sh}"
STATUS_FILE="${GITEA_BUNDLE_SYNC_STATUS_FILE:-/backup/status/gitea-bundle-sync-188.status}"
LOCK_FILE="${GITEA_BUNDLE_SYNC_LOCK_FILE:-${LOCAL_OUTPUT_ROOT}/.sync-188.lock}"
START_TIME="$(date +%s)"
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
phase="preflight"
promoted=0
previous_complete=""
previous_latest=""
expected_count=0
stamp="unknown"
mkdir -p "${LOCAL_OUTPUT_ROOT}" "$(dirname "${STATUS_FILE}")"
exec 9>"${LOCK_FILE}"
if ! flock -n 9; then
printf 'SKIP Gitea bundle sync already running\n'
exit 0
fi
write_status() {
local success="$1"
local reason="$2"
local temp_status="${STATUS_FILE}.tmp.$$"
{
printf 'timestamp=%s\n' "$(date +%s)"
printf 'success=%s\n' "${success}"
printf 'reason=%s\n' "${reason}"
printf 'phase=%s\n' "${phase}"
printf 'snapshot_stamp=%s\n' "${stamp}"
printf 'expected_repo_count=%s\n' "${expected_count}"
printf 'duration_seconds=%s\n' "$(($(date +%s) - START_TIME))"
} >"${temp_status}"
chmod 600 "${temp_status}"
mv "${temp_status}" "${STATUS_FILE}"
}
refresh_remote_exporter() {
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
"AIOPS_HOST_LABEL=188 NODE_EXPORTER_TEXTFILE_DIR=/home/ollama/node_exporter_textfiles '${REMOTE_EXPORTER_SCRIPT}'" \
>/dev/null 2>&1 || true
}
rollback_remote_links() {
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
python3 - "${REMOTE_OUTPUT_ROOT}" "${previous_complete}" "${previous_latest}" <<'PY'
import os
import sys
from pathlib import Path
root = Path(sys.argv[1])
previous = {
root / "latest-private-complete": sys.argv[2],
root / "latest": sys.argv[3],
}
for link, target in previous.items():
if target:
temporary = link.with_name(f".{link.name}.rollback.{os.getpid()}")
temporary.unlink(missing_ok=True)
temporary.symlink_to(target)
os.replace(temporary, link)
elif link.is_symlink():
link.unlink()
PY
}
finish() {
local rc=$?
trap - EXIT
if [ "${rc}" -eq 0 ] && [ "${phase}" = "complete" ]; then
write_status 1 verified
else
if [ "${promoted}" -eq 1 ]; then
rollback_remote_links >/dev/null 2>&1 || true
fi
refresh_remote_exporter
write_status 0 "${phase}_failed"
fi
exit "${rc}"
}
trap finish EXIT
[ -x "${BUNDLE_SCRIPT}" ] || { printf 'missing bundle script: %s\n' "${BUNDLE_SCRIPT}" >&2; exit 1; }
command -v docker >/dev/null
command -v rsync >/dev/null
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
"test -x '${REMOTE_VERIFY_SCRIPT}' && test -x '${REMOTE_EXPORTER_SCRIPT}'"
phase="bundle_local_container"
"${BUNDLE_SCRIPT}" \
--local-container "${LOCAL_CONTAINER}" \
--output-root "${LOCAL_OUTPUT_ROOT}"
local_snapshot="$(readlink -f "${LOCAL_OUTPUT_ROOT}/latest-private-complete")"
stamp="$(basename "${local_snapshot}")"
manifest="${local_snapshot}/manifest.tsv"
[ -f "${manifest}" ]
expected_count="$(awk -F '\t' 'NR > 1 {total += 1; if ($2 != "ok") failed += 1} END {if (failed > 0 || total == 0) exit 1; print total}' "${manifest}")"
phase="sync_complete_set_to_188"
previous_complete="$(ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" "readlink -f '${REMOTE_OUTPUT_ROOT}/latest-private-complete' 2>/dev/null || true")"
previous_latest="$(ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" "readlink -f '${REMOTE_OUTPUT_ROOT}/latest' 2>/dev/null || true")"
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" "mkdir -p '${REMOTE_OUTPUT_ROOT}/${stamp}'"
rsync -a --checksum \
-e "ssh ${SSH_OPTS[*]}" \
"${local_snapshot}/" \
"${REMOTE_HOST}:${REMOTE_OUTPUT_ROOT}/${stamp}/"
phase="remote_manifest_verification"
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
python3 - "${REMOTE_OUTPUT_ROOT}/${stamp}" "${expected_count}" <<'PY'
import csv
import hashlib
import sys
from pathlib import Path
snapshot = Path(sys.argv[1])
expected = int(sys.argv[2])
with (snapshot / "manifest.tsv").open(encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle, delimiter="\t"))
ok = len(rows) == expected and expected > 0
verified_checksums = 0
for row in rows:
bundle = snapshot / Path(row.get("bundle", "")).name
checksum = snapshot / Path(row.get("checksum", "")).name
row_ok = row.get("status") == "ok" and bundle.is_file() and checksum.is_file()
if row_ok:
expected_digest = checksum.read_text(encoding="utf-8", errors="replace").split()[0].lower()
digest = hashlib.sha256()
with bundle.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
row_ok = len(expected_digest) == 64 and digest.hexdigest() == expected_digest
if row_ok:
verified_checksums += 1
ok = ok and row_ok
print(
f"remote_manifest_expected={expected} remote_manifest_rows={len(rows)} "
f"remote_manifest_checksum_ok={verified_checksums} remote_manifest_ok={int(ok)}"
)
raise SystemExit(0 if ok else 1)
PY
phase="remote_promote_complete"
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
"for link in '${REMOTE_OUTPUT_ROOT}/latest-private-complete' '${REMOTE_OUTPUT_ROOT}/latest'; do test ! -e \"\$link\" || test -L \"\$link\" || exit 1; done"
promoted=1
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
"ln -sfn '${REMOTE_OUTPUT_ROOT}/${stamp}' '${REMOTE_OUTPUT_ROOT}/latest-private-complete' && ln -sfn '${REMOTE_OUTPUT_ROOT}/${stamp}' '${REMOTE_OUTPUT_ROOT}/latest'"
phase="remote_sample_restore_fsck"
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
"AIOPS_GITEA_BUNDLE_ROOT='${REMOTE_OUTPUT_ROOT}/latest-private-complete' AIOPS_HOST_LABEL=188 '${REMOTE_VERIFY_SCRIPT}' --write-textfile"
phase="refresh_remote_backup_health"
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
"AIOPS_HOST_LABEL=188 NODE_EXPORTER_TEXTFILE_DIR=/home/ollama/node_exporter_textfiles '${REMOTE_EXPORTER_SCRIPT}'"
phase="complete"
printf 'GITEA_BUNDLE_SYNC_COMPLETE=1 expected=%s evidence=%s\n' "${expected_count}" "${STATUS_FILE}"

118
scripts/backup/gitea-repo-bundle-backup.sh
View File

@@ -7,6 +7,8 @@ set -euo pipefail
GITEA_BASE_URL="${GITEA_BASE_URL:-http://192.168.0.110:3001}"
GIT_REMOTE_BASE="${GIT_REMOTE_BASE:-}"
LOCAL_GITEA_CONTAINER="${LOCAL_GITEA_CONTAINER:-}"
LOCAL_REPO_ROOT="${LOCAL_REPO_ROOT:-/data/git/repositories}"
OUTPUT_ROOT="${OUTPUT_ROOT:-/home/ollama/backup/110/gitea/git-bundles}"
TIMEOUT_LS_REMOTE_SECONDS="${TIMEOUT_LS_REMOTE_SECONDS:-60}"
TIMEOUT_CLONE_SECONDS="${TIMEOUT_CLONE_SECONDS:-180}"
@@ -27,6 +29,11 @@ Options:
--git-remote-base URL
Git remote base, such as ssh://git@192.168.0.110:2222.
When set, it is used instead of --gitea-base.
--local-container NAME
Read bare repos directly from a local Gitea container.
This avoids tokens and SSH keys on the Gitea host.
--local-repo-root PATH
Repo root inside the local container.
--stamp STAMP Output directory stamp. Default: current local timestamp.
--dry-run Print target URLs without cloning or writing bundles.
-h, --help Show this help.
@@ -56,6 +63,14 @@ while [ "$#" -gt 0 ]; do
shift
GIT_REMOTE_BASE="${1:?--git-remote-base requires URL}"
;;
--local-container)
shift
LOCAL_GITEA_CONTAINER="${1:?--local-container requires NAME}"
;;
--local-repo-root)
shift
LOCAL_REPO_ROOT="${1:?--local-repo-root requires PATH}"
;;
--stamp)
shift
STAMP="${1:?--stamp requires value}"
@@ -104,7 +119,11 @@ repo_remote_url() {
if [ "$DRY_RUN" -eq 1 ]; then
for repo in "${REPOS[@]}"; do
printf 'DRY_RUN repo=%s url=%s\n' "$repo" "$(repo_remote_url "$repo")"
if [ -n "$LOCAL_GITEA_CONTAINER" ]; then
printf 'DRY_RUN repo=%s transport=container container=%s\n' "$repo" "$LOCAL_GITEA_CONTAINER"
else
printf 'DRY_RUN repo=%s url=%s\n' "$repo" "$(repo_remote_url "$repo")"
fi
done
exit 0
fi
@@ -116,19 +135,80 @@ mkdir -p "$backup_root"
trap 'rm -rf "$tmp_root"' EXIT
printf 'repo\tstatus\thead_count\tbundle\tchecksum\n' > "$manifest"
failed_count=0
success_count=0
for repo in "${REPOS[@]}"; do
slug="${repo//\//__}"
url="$(repo_remote_url "$repo")"
refs_file="${backup_root}/${slug}.refs"
err_file="${backup_root}/${slug}.err"
bundle_file="${backup_root}/${slug}.bundle"
checksum_file="${bundle_file}.sha256"
repo_tmp="${tmp_root}/${slug}.git"
if [ -n "$LOCAL_GITEA_CONTAINER" ]; then
container_repo="${LOCAL_REPO_ROOT%/}/${repo}.git"
container_bundle="/tmp/awoooi-${slug}-${STAMP}.bundle"
if ! docker exec -u git "$LOCAL_GITEA_CONTAINER" test -d "$container_repo" >/dev/null 2>&1; then
owner="${repo%%/*}"
name="${repo#*/}"
resolved_paths="$(docker exec -u git "$LOCAL_GITEA_CONTAINER" sh -c \
'find "$1/$2" -mindepth 1 -maxdepth 1 -type d -iname "$3.git" -print' \
sh "${LOCAL_REPO_ROOT%/}" "$owner" "$name" 2>>"$err_file" || true)"
resolved_count="$(printf '%s\n' "$resolved_paths" | awk 'NF {count += 1} END {print count + 0}')"
if [ "$resolved_count" -ne 1 ]; then
printf '%s\tlocal_repo_read_failed\t0\t\t\n' "$repo" >> "$manifest"
failed_count=$((failed_count + 1))
continue
fi
container_repo="$resolved_paths"
fi
if ! timeout "$TIMEOUT_LS_REMOTE_SECONDS" docker exec -u git "$LOCAL_GITEA_CONTAINER" \
git -C "$container_repo" for-each-ref --format='%(objectname)%09%(refname)' \
>"$refs_file" 2>"$err_file"; then
printf '%s\tlocal_repo_read_failed\t0\t\t\n' "$repo" >> "$manifest"
failed_count=$((failed_count + 1))
continue
fi
head_count="$(wc -l < "$refs_file" | tr -d ' ')"
if ! timeout "$TIMEOUT_BUNDLE_SECONDS" docker exec -u git "$LOCAL_GITEA_CONTAINER" \
git -C "$container_repo" bundle create "$container_bundle" --all \
>/dev/null 2>>"$err_file"; then
docker exec -u git "$LOCAL_GITEA_CONTAINER" rm -f "$container_bundle" >/dev/null 2>&1 || true
printf '%s\tbundle_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest"
failed_count=$((failed_count + 1))
continue
fi
if ! timeout "$TIMEOUT_VERIFY_SECONDS" docker exec -u git "$LOCAL_GITEA_CONTAINER" \
git -C "$container_repo" bundle verify "$container_bundle" \
>/dev/null 2>>"$err_file"; then
docker exec -u git "$LOCAL_GITEA_CONTAINER" rm -f "$container_bundle" >/dev/null 2>&1 || true
printf '%s\tverify_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest"
failed_count=$((failed_count + 1))
continue
fi
if ! docker cp "${LOCAL_GITEA_CONTAINER}:${container_bundle}" "$bundle_file" 2>>"$err_file"; then
docker exec -u git "$LOCAL_GITEA_CONTAINER" rm -f "$container_bundle" >/dev/null 2>&1 || true
printf '%s\tcopy_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest"
failed_count=$((failed_count + 1))
continue
fi
docker exec -u git "$LOCAL_GITEA_CONTAINER" rm -f "$container_bundle" >/dev/null 2>&1 || true
(sha256sum "$bundle_file" 2>/dev/null || shasum -a 256 "$bundle_file") > "$checksum_file"
printf '%s\tok\t%s\t%s\t%s\n' "$repo" "$head_count" "$bundle_file" "$checksum_file" >> "$manifest"
success_count=$((success_count + 1))
continue
fi
url="$(repo_remote_url "$repo")"
if ! GIT_TERMINAL_PROMPT=0 timeout "$TIMEOUT_LS_REMOTE_SECONDS" \
git ls-remote --heads "$url" >"$refs_file" 2>"$err_file"; then
printf '%s\tls_remote_failed\t0\t\t\n' "$repo" >> "$manifest"
failed_count=$((failed_count + 1))
continue
fi
@@ -137,30 +217,52 @@ for repo in "${REPOS[@]}"; do
if ! GIT_TERMINAL_PROMPT=0 timeout "$TIMEOUT_CLONE_SECONDS" \
git clone --mirror --quiet "$url" "$repo_tmp" 2>>"$err_file"; then
printf '%s\tclone_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest"
failed_count=$((failed_count + 1))
continue
fi
if ! timeout "$TIMEOUT_BUNDLE_SECONDS" \
git -C "$repo_tmp" bundle create "$bundle_file" --all >/dev/null 2>>"$err_file"; then
printf '%s\tbundle_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest"
failed_count=$((failed_count + 1))
continue
fi
if ! timeout "$TIMEOUT_VERIFY_SECONDS" \
git -C "$repo_tmp" bundle verify "$bundle_file" >/dev/null 2>>"$err_file"; then
printf '%s\tverify_failed\t%s\t%s\t\n' "$repo" "$head_count" "$bundle_file" >> "$manifest"
failed_count=$((failed_count + 1))
continue
fi
(sha256sum "$bundle_file" 2>/dev/null || shasum -a 256 "$bundle_file") > "$checksum_file"
rm -rf "$repo_tmp"
printf '%s\tok\t%s\t%s\t%s\n' "$repo" "$head_count" "$bundle_file" "$checksum_file" >> "$manifest"
success_count=$((success_count + 1))
done
latest_link="${OUTPUT_ROOT%/}/latest"
if [ ! -e "$latest_link" ] || [ -L "$latest_link" ]; then
ln -sfn "$backup_root" "$latest_link"
else
echo "WARN latest path exists and is not a symlink: $latest_link" >&2
fi
expected_count="${#REPOS[@]}"
cat "$manifest"
if [ "$failed_count" -ne 0 ] || [ "$success_count" -ne "$expected_count" ]; then
printf 'ERROR bundle backup incomplete success=%s expected=%s failed=%s; complete links unchanged\n' \
"$success_count" "$expected_count" "$failed_count" >&2
exit 1
fi
promotion_path_ready() {
link="$1"
if [ ! -e "$link" ] || [ -L "$link" ]; then
return 0
fi
printf 'ERROR promotion path exists and is not a symlink: %s\n' "$link" >&2
return 1
}
latest_link="${OUTPUT_ROOT%/}/latest"
complete_link="${OUTPUT_ROOT%/}/latest-private-complete"
promotion_path_ready "$latest_link"
promotion_path_ready "$complete_link"
ln -sfn "$backup_root" "$latest_link"
ln -sfn "$backup_root" "$complete_link"
printf 'BUNDLE_BACKUP_COMPLETE=1 success=%s expected=%s root=%s\n' "$success_count" "$expected_count" "$backup_root"

14
scripts/backup/tests/test_backup_status_contract.py
View File

@@ -19,3 +19,17 @@ def test_backup_status_keeps_aggregate_failure_out_of_core_blockers() -> None:
assert "component_failed_188" in core_line
assert "failed_total_110" not in core_line
assert "failed_total_188" not in core_line
def test_backup_status_uses_dynamic_healthy_over_monitored_ratio() -> None:
text = BACKUP_STATUS.read_text(encoding="utf-8")
core_line = next(line for line in text.splitlines() if line.startswith("core_blockers="))
assert 'fresh_ok_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh" 1)"' in text
assert 'fresh_ok_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh" 1)"' in text
assert "110備份=${fresh_ok_110}/${monitored_total_110}" in text
assert "188備份=${fresh_ok_188}/${monitored_total_188}" in text
assert "freshness_metrics_missing_110" in core_line
assert "freshness_metrics_missing_188" in core_line
assert "/13 fresh" not in text
assert "/2 fresh" not in text

309
scripts/backup/tests/test_gitea_backup_automation.py Normal file
View File

@@ -0,0 +1,309 @@
from __future__ import annotations
import os
import shutil
import subprocess
import textwrap
import zipfile
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[3]
BUNDLE_SCRIPT = ROOT / "scripts" / "backup" / "gitea-repo-bundle-backup.sh"
RESTORE_DRILL_SCRIPT = ROOT / "scripts" / "backup" / "verify-gitea-backup.sh"
def _fake_git(tmp_path: Path) -> Path:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
git = fake_bin / "git"
git.write_text(
textwrap.dedent(
"""\
#!/usr/bin/env bash
set -euo pipefail
if [ -n "${FAKE_GIT_FAIL_PATTERN:-}" ] && [[ "$*" == *"${FAKE_GIT_FAIL_PATTERN}"* ]]; then
exit 42
fi
if [ "$1" = "ls-remote" ]; then
printf '0000000000000000000000000000000000000001\trefs/heads/main\n'
exit 0
fi
if [ "$1" = "clone" ]; then
mkdir -p "${!#}"
exit 0
fi
if [ "$1" = "-C" ] && [ "$3" = "bundle" ] && [ "$4" = "create" ]; then
printf 'synthetic bundle\n' > "$5"
exit 0
fi
if [ "$1" = "-C" ] && [ "$3" = "bundle" ] && [ "$4" = "verify" ]; then
exit 0
fi
printf 'unexpected fake git invocation: %s\n' "$*" >&2
exit 64
"""
),
encoding="utf-8",
)
git.chmod(0o755)
return fake_bin
def _fake_docker(fake_bin: Path, tmp_path: Path) -> None:
docker = fake_bin / "docker"
docker.write_text(
textwrap.dedent(
"""\
#!/usr/bin/env bash
set -euo pipefail
root="${FAKE_DOCKER_ROOT:?}"
if [ "$1" = "exec" ]; then
shift 4
if [ "$1" = "test" ]; then
exit 0
fi
if [ "$1" = "git" ] && [ "$4" = "for-each-ref" ]; then
printf '0000000000000000000000000000000000000001\trefs/heads/main\n'
exit 0
fi
if [ "$1" = "git" ] && [ "$4" = "bundle" ] && [ "$5" = "create" ]; then
mkdir -p "${root}$(dirname "$6")"
printf 'synthetic container bundle\n' > "${root}$6"
exit 0
fi
if [ "$1" = "git" ] && [ "$4" = "bundle" ] && [ "$5" = "verify" ]; then
exit 0
fi
if [ "$1" = "rm" ]; then
rm -f "${root}${3}"
exit 0
fi
fi
if [ "$1" = "cp" ]; then
source_path="${2#*:}"
cp "${root}${source_path}" "$3"
exit 0
fi
printf 'unexpected fake docker invocation: %s\n' "$*" >&2
exit 64
"""
),
encoding="utf-8",
)
docker.chmod(0o755)
def test_bundle_backup_promotes_only_a_complete_repo_set(tmp_path: Path) -> None:
fake_bin = _fake_git(tmp_path)
output_root = tmp_path / "bundles"
env = os.environ | {"PATH": f"{fake_bin}:{os.environ['PATH']}"}
complete = subprocess.run(
[
"bash",
str(BUNDLE_SCRIPT),
"--output-root",
str(output_root),
"--git-remote-base",
"ssh://git@example.invalid:2222",
"--stamp",
"complete",
"--repo",
"owner/one",
"--repo",
"owner/two",
],
env=env,
capture_output=True,
text=True,
check=False,
)
assert complete.returncode == 0, complete.stderr
assert (output_root / "latest-private-complete").resolve() == output_root / "complete"
assert (output_root / "latest").resolve() == output_root / "complete"
assert "BUNDLE_BACKUP_COMPLETE=1 success=2 expected=2" in complete.stdout
failed = subprocess.run(
[
"bash",
str(BUNDLE_SCRIPT),
"--output-root",
str(output_root),
"--git-remote-base",
"ssh://git@example.invalid:2222",
"--stamp",
"partial",
"--repo",
"owner/one",
"--repo",
"owner/fail",
],
env=env | {"FAKE_GIT_FAIL_PATTERN": "owner/fail"},
capture_output=True,
text=True,
check=False,
)
assert failed.returncode != 0
assert "complete links unchanged" in failed.stderr
assert (output_root / "latest-private-complete").resolve() == output_root / "complete"
assert "owner/fail\tls_remote_failed" in (output_root / "partial" / "manifest.tsv").read_text(
encoding="utf-8"
)
def test_bundle_backup_supports_local_gitea_container_without_credentials(tmp_path: Path) -> None:
fake_bin = _fake_git(tmp_path)
_fake_docker(fake_bin, tmp_path)
output_root = tmp_path / "local-bundles"
docker_root = tmp_path / "fake-docker"
docker_root.mkdir()
env = os.environ | {
"PATH": f"{fake_bin}:{os.environ['PATH']}",
"FAKE_DOCKER_ROOT": str(docker_root),
}
result = subprocess.run(
[
"bash",
str(BUNDLE_SCRIPT),
"--output-root",
str(output_root),
"--local-container",
"gitea",
"--stamp",
"local-complete",
"--repo",
"owner/one",
"--repo",
"owner/two",
],
env=env,
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, result.stderr
assert (output_root / "latest-private-complete").resolve() == output_root / "local-complete"
assert "BUNDLE_BACKUP_COMPLETE=1 success=2 expected=2" in result.stdout
def test_bundle_backup_preflights_both_promotion_links(tmp_path: Path) -> None:
fake_bin = _fake_git(tmp_path)
output_root = tmp_path / "blocked-promotion"
output_root.mkdir()
(output_root / "latest-private-complete").mkdir()
env = os.environ | {"PATH": f"{fake_bin}:{os.environ['PATH']}"}
result = subprocess.run(
[
"bash",
str(BUNDLE_SCRIPT),
"--output-root",
str(output_root),
"--git-remote-base",
"ssh://git@example.invalid:2222",
"--stamp",
"candidate",
"--repo",
"owner/one",
],
env=env,
capture_output=True,
text=True,
check=False,
)
assert result.returncode != 0
assert "promotion path exists and is not a symlink" in result.stderr
assert not (output_root / "latest").exists()
assert (output_root / "latest-private-complete").is_dir()
@pytest.mark.skipif(shutil.which("unzip") is None, reason="unzip is required by the production verifier")
def test_gitea_restore_drill_blocks_same_count_inventory_mismatch(tmp_path: Path) -> None:
archive = tmp_path / "gitea-dump.zip"
with zipfile.ZipFile(archive, "w") as payload:
payload.writestr("gitea-repo/owner/one.git/HEAD", "ref: refs/heads/main\n")
payload.writestr("gitea-repo/owner/two.git/HEAD", "ref: refs/heads/main\n")
payload.writestr("gitea-db.sql", "synthetic database placeholder\n")
payload.writestr("config/app.ini", "synthetic config placeholder\n")
state_dir = tmp_path / "state"
work_dir = tmp_path / "work"
state_dir.mkdir()
work_dir.mkdir()
matching_inventory = tmp_path / "matching.txt"
matching_inventory.write_text("owner/one.git\nowner/two.git\n", encoding="utf-8")
mismatch_inventory = tmp_path / "mismatch.txt"
mismatch_inventory.write_text("owner/one.git\nowner/different.git\n", encoding="utf-8")
base_env = os.environ | {
"GITEA_RESTORE_DRILL_TEST_MODE": "1",
"GITEA_RESTORE_DRILL_SOURCE_ZIP": str(archive),
"GITEA_RESTORE_DRILL_STATE_DIR": str(state_dir),
"GITEA_RESTORE_DRILL_WORK_ROOT": str(work_dir),
"BACKUP_BASE": str(tmp_path),
}
success = subprocess.run(
["bash", str(RESTORE_DRILL_SCRIPT), "--no-notify"],
env=base_env | {"GITEA_RESTORE_DRILL_LIVE_INVENTORY_FILE": str(matching_inventory)},
capture_output=True,
text=True,
check=False,
)
assert success.returncode == 0, success.stderr
status = (state_dir / "gitea-restore-drill.status").read_text(encoding="utf-8")
assert "success=1" in status
assert "inventory_digest_match=1" in status
mismatch = subprocess.run(
["bash", str(RESTORE_DRILL_SCRIPT), "--no-notify"],
env=base_env | {"GITEA_RESTORE_DRILL_LIVE_INVENTORY_FILE": str(mismatch_inventory)},
capture_output=True,
text=True,
check=False,
)
assert mismatch.returncode != 0
status = (state_dir / "gitea-restore-drill.status").read_text(encoding="utf-8")
assert "success=0" in status
assert "reason=repository_inventory_mismatch" in status
@pytest.mark.skipif(shutil.which("unzip") is None, reason="unzip is required by the production verifier")
def test_gitea_restore_drill_records_live_inventory_read_failure(tmp_path: Path) -> None:
archive = tmp_path / "gitea-dump.zip"
with zipfile.ZipFile(archive, "w") as payload:
payload.writestr("gitea-repo/owner/one.git/HEAD", "ref: refs/heads/main\n")
payload.writestr("gitea-db.sql", "synthetic database placeholder\n")
payload.writestr("config/app.ini", "synthetic config placeholder\n")
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
docker = fake_bin / "docker"
docker.write_text("#!/usr/bin/env bash\nexit 42\n", encoding="utf-8")
docker.chmod(0o755)
state_dir = tmp_path / "state"
work_dir = tmp_path / "work"
state_dir.mkdir()
work_dir.mkdir()
result = subprocess.run(
["bash", str(RESTORE_DRILL_SCRIPT), "--no-notify"],
env=os.environ
| {
"PATH": f"{fake_bin}:{os.environ['PATH']}",
"GITEA_RESTORE_DRILL_TEST_MODE": "1",
"GITEA_RESTORE_DRILL_SOURCE_ZIP": str(archive),
"GITEA_RESTORE_DRILL_STATE_DIR": str(state_dir),
"GITEA_RESTORE_DRILL_WORK_ROOT": str(work_dir),
"BACKUP_BASE": str(tmp_path),
},
capture_output=True,
text=True,
check=False,
)
assert result.returncode != 0
status = (state_dir / "gitea-restore-drill.status").read_text(encoding="utf-8")
assert "success=0" in status
assert "reason=live_inventory_read_failed" in status

288
scripts/backup/verify-gitea-backup.sh Executable file
View File

@@ -0,0 +1,288 @@
#!/bin/bash
# Isolated Gitea restore drill. It validates the latest dump without touching
# the running container or exposing repository names, database rows, or config.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TEST_MODE="${GITEA_RESTORE_DRILL_TEST_MODE:-0}"
CALLER_BACKUP_BASE="${BACKUP_BASE:-}"
CALLER_BACKUP_LOG_DIR="${BACKUP_LOG_DIR:-}"
CALLER_RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-}"
if [ "${TEST_MODE}" != "1" ] && [ -f "${SCRIPT_DIR}/common.sh" ]; then
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/common.sh"
else
BACKUP_BASE="${BACKUP_BASE:-/backup}"
RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-/nonexistent}"
log_info() { printf '[INFO] %s\n' "$*"; }
log_success() { printf '[SUCCESS] %s\n' "$*"; }
log_error() { printf '[ERROR] %s\n' "$*" >&2; }
notify_clawbot() { return 0; }
fi
if [ -n "${CALLER_BACKUP_BASE}" ]; then
BACKUP_BASE="${CALLER_BACKUP_BASE}"
BACKUP_LOG_DIR="${CALLER_BACKUP_LOG_DIR:-${BACKUP_BASE}/logs}"
RESTIC_PASSWORD_FILE="${CALLER_RESTIC_PASSWORD_FILE:-${BACKUP_BASE}/scripts/.restic-password}"
export BACKUP_BASE BACKUP_LOG_DIR RESTIC_PASSWORD_FILE
fi
LOCAL_REPO="${GITEA_BACKUP_REPO:-${BACKUP_BASE}/gitea}"
STATE_DIR="${GITEA_RESTORE_DRILL_STATE_DIR:-${BACKUP_BASE}/integrity}"
WORK_ROOT="${GITEA_RESTORE_DRILL_WORK_ROOT:-${BACKUP_BASE}/tmp}"
STATUS_FILE="${STATE_DIR}/gitea-restore-drill.status"
MIN_FREE_BUFFER_BYTES="${GITEA_RESTORE_DRILL_MIN_FREE_BUFFER_BYTES:-2147483648}"
NOTIFY=0
SOURCE_ZIP="${GITEA_RESTORE_DRILL_SOURCE_ZIP:-}"
LIVE_COUNTS="${GITEA_RESTORE_DRILL_LIVE_COUNTS:-}"
LIVE_INVENTORY_FILE="${GITEA_RESTORE_DRILL_LIVE_INVENTORY_FILE:-}"
START_TIME="$(date +%s)"
WORK_DIR=""
while [ "$#" -gt 0 ]; do
case "$1" in
--notify) NOTIFY=1 ;;
--no-notify) NOTIFY=0 ;;
-h|--help)
printf 'Usage: verify-gitea-backup.sh [--notify|--no-notify]\n'
exit 0
;;
*)
printf 'Unknown argument: %s\n' "$1" >&2
exit 2
;;
esac
shift
done
cleanup() {
if [ -n "${WORK_DIR}" ] && [ -d "${WORK_DIR}" ]; then
rm -rf "${WORK_DIR}"
fi
}
trap cleanup EXIT
low_priority() {
if command -v ionice >/dev/null 2>&1; then
ionice -c2 -n7 nice -n 10 "$@"
else
nice -n 10 "$@"
fi
}
file_size() {
stat -c '%s' "$1" 2>/dev/null || stat -f '%z' "$1"
}
write_status() {
local success="$1"
local reason="$2"
local snapshot_id="$3"
local snapshot_timestamp="$4"
local dump_size="$5"
local backup_regular="$6"
local backup_wiki="$7"
local live_regular="$8"
local live_wiki="$9"
local database_present="${10}"
local config_present="${11}"
local crc_ok="${12}"
local inventory_match="${13}"
local duration
local temp_status
duration=$(($(date +%s) - START_TIME))
install -d -m 700 "${STATE_DIR}"
temp_status="${STATUS_FILE}.tmp.$$"
{
printf 'timestamp=%s\n' "$(date +%s)"
printf 'success=%s\n' "${success}"
printf 'reason=%s\n' "${reason}"
printf 'snapshot_id=%s\n' "${snapshot_id}"
printf 'snapshot_timestamp=%s\n' "${snapshot_timestamp}"
printf 'dump_size_bytes=%s\n' "${dump_size}"
printf 'backup_regular_repo_count=%s\n' "${backup_regular}"
printf 'backup_wiki_repo_count=%s\n' "${backup_wiki}"
printf 'live_regular_repo_count=%s\n' "${live_regular}"
printf 'live_wiki_repo_count=%s\n' "${live_wiki}"
printf 'database_dump_present=%s\n' "${database_present}"
printf 'config_present=%s\n' "${config_present}"
printf 'archive_crc_ok=%s\n' "${crc_ok}"
printf 'inventory_digest_match=%s\n' "${inventory_match}"
printf 'duration_seconds=%s\n' "${duration}"
} > "${temp_status}"
chmod 600 "${temp_status}"
mv "${temp_status}" "${STATUS_FILE}"
}
fail_drill() {
local reason="$1"
local snapshot_id="${2:-unknown}"
local snapshot_timestamp="${3:-unknown}"
local dump_size="${4:-0}"
local backup_regular="${5:-0}"
local backup_wiki="${6:-0}"
local live_regular="${7:-0}"
local live_wiki="${8:-0}"
local database_present="${9:-0}"
local config_present="${10:-0}"
local crc_ok="${11:-0}"
local inventory_match="${12:-0}"
write_status 0 "${reason}" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" \
"${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" \
"${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}"
log_error "Gitea restore drill failed: ${reason}; evidence=${STATUS_FILE}"
if [ "${NOTIFY}" -eq 1 ]; then
notify_clawbot "failed" "gitea-restore-drill" "Gitea 備份還原演練失敗：${reason}" "$(($(date +%s) - START_TIME))"
fi
exit 1
}
if [ "${TEST_MODE}" != "1" ] && { [ -n "${SOURCE_ZIP}" ] || [ -n "${LIVE_COUNTS}" ] || [ -n "${LIVE_INVENTORY_FILE}" ]; }; then
fail_drill "test_override_rejected"
fi
install -d -m 700 "${WORK_ROOT}" "${STATE_DIR}"
WORK_DIR="$(mktemp -d "${WORK_ROOT%/}/gitea-restore-drill.XXXXXX")"
chmod 700 "${WORK_DIR}"
RESTORED_ZIP="${WORK_DIR}/gitea-dump.zip"
snapshot_id="unknown"
snapshot_timestamp="unknown"
dump_size=0
if [ "${TEST_MODE}" = "1" ]; then
[ -f "${SOURCE_ZIP}" ] || fail_drill "test_source_zip_missing"
cp "${SOURCE_ZIP}" "${RESTORED_ZIP}"
dump_size="$(file_size "${RESTORED_ZIP}")"
snapshot_id="synthetic"
snapshot_timestamp="synthetic"
else
[ -d "${LOCAL_REPO}/data" ] || fail_drill "restic_repo_missing"
[ -r "${RESTIC_PASSWORD_FILE}" ] || fail_drill "restic_password_file_unavailable"
snapshot_meta=""
if ! snapshot_meta="$(
restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \
python3 -c 'import json,sys
rows=json.load(sys.stdin)
if rows:
row=rows[-1]
print("%s\t%s" % (row.get("short_id", "unknown"), row.get("time", "unknown")))'
)"; then
fail_drill "snapshot_metadata_read_failed"
fi
[ -n "${snapshot_meta}" ] || fail_drill "snapshot_metadata_missing"
IFS=$'\t' read -r snapshot_id snapshot_timestamp <<<"${snapshot_meta}"
dump_meta=""
if ! dump_meta="$(
restic -r "${LOCAL_REPO}" ls latest --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \
python3 -c 'import json,os,sys
for line in sys.stdin:
try: item=json.loads(line)
except json.JSONDecodeError: continue
if item.get("struct_type")=="node" and os.path.basename(item.get("path", ""))=="gitea-dump.zip":
print("%s\t%s" % (item.get("path", ""), item.get("size", 0)))
raise SystemExit'
)"; then
fail_drill "dump_metadata_read_failed" "${snapshot_id}" "${snapshot_timestamp}"
fi
[ -n "${dump_meta}" ] || fail_drill "gitea_dump_member_missing" "${snapshot_id}" "${snapshot_timestamp}"
IFS=$'\t' read -r dump_path dump_size <<<"${dump_meta}"
available_bytes="$(df -Pk "${WORK_ROOT}" | awk 'NR==2 {printf "%.0f\n", $4 * 1024}')"
required_bytes=$((dump_size + MIN_FREE_BUFFER_BYTES))
if [ "${available_bytes}" -lt "${required_bytes}" ]; then
fail_drill "insufficient_free_space" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}"
fi
log_info "Restoring isolated Gitea dump snapshot=${snapshot_id} size_bytes=${dump_size}"
if ! low_priority restic -r "${LOCAL_REPO}" dump latest "${dump_path}" \
--password-file "${RESTIC_PASSWORD_FILE}" > "${RESTORED_ZIP}"; then
fail_drill "restic_dump_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}"
fi
fi
actual_size="$(file_size "${RESTORED_ZIP}")"
if [ "${actual_size}" -ne "${dump_size}" ]; then
fail_drill "dump_size_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}"
fi
if ! low_priority unzip -tqq "${RESTORED_ZIP}"; then
fail_drill "archive_crc_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}"
fi
crc_ok=1
inventory=""
if ! inventory="$(
unzip -Z1 "${RESTORED_ZIP}" | awk '
/\.wiki\.git\/HEAD$/ { wiki += 1 }
/\.git\/HEAD$/ && $0 !~ /\.wiki\.git\/HEAD$/ { regular += 1 }
/(^|\/)gitea-db\.sql$/ { database += 1 }
/(^|\/)app\.ini$/ { config += 1 }
END { printf "%d %d %d %d", regular, wiki, database, config }
'
)"; then
fail_drill "archive_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" 0 0 0 0 0 0 "${crc_ok}"
fi
read -r backup_regular backup_wiki database_present config_present <<<"${inventory}"
backup_inventory_file="${WORK_DIR}/backup-repo-inventory"
live_inventory_file="${WORK_DIR}/live-repo-inventory"
if ! unzip -Z1 "${RESTORED_ZIP}" | awk -F/ '
/\.git\/HEAD$/ { print $(NF - 2) "/" $(NF - 1) }
' | LC_ALL=C sort -u > "${backup_inventory_file}"; then
fail_drill "backup_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}"
fi
if [ -n "${LIVE_INVENTORY_FILE}" ]; then
[ -f "${LIVE_INVENTORY_FILE}" ] || fail_drill "test_live_inventory_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}"
LC_ALL=C sort -u "${LIVE_INVENTORY_FILE}" > "${live_inventory_file}"
else
if ! docker exec gitea sh -c 'find /data/git/repositories -type f -path "*.git/HEAD"' 2>/dev/null | awk -F/ '
/\.git\/HEAD$/ { print $(NF - 2) "/" $(NF - 1) }
' | LC_ALL=C sort -u > "${live_inventory_file}"; then
fail_drill "live_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}"
fi
fi
live_inventory="$(awk '
/\.wiki\.git$/ { wiki += 1; next }
/\.git$/ { regular += 1 }
END { printf "%d %d", regular, wiki }
' "${live_inventory_file}")"
read -r live_regular live_wiki <<<"${live_inventory}"
hash_file() {
if command -v sha256sum >/dev/null 2>&1; then
sha256sum "$1" | awk '{print $1}'
else
shasum -a 256 "$1" | awk '{print $1}'
fi
}
backup_inventory_digest="$(hash_file "${backup_inventory_file}")"
live_inventory_digest="$(hash_file "${live_inventory_file}")"
inventory_match=0
[ "${backup_inventory_digest}" = "${live_inventory_digest}" ] && inventory_match=1
[ "${database_present}" -gt 0 ] || fail_drill "database_dump_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" 0 "${config_present}" "${crc_ok}"
[ "${config_present}" -gt 0 ] || fail_drill "config_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" 0 "${crc_ok}"
[ "${backup_regular}" -gt 0 ] || fail_drill "repository_inventory_empty" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" 0 "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}"
if [ "${backup_regular}" -ne "${live_regular}" ] || [ "${backup_wiki}" -ne "${live_wiki}" ]; then
fail_drill "repository_count_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}"
fi
if [ "${inventory_match}" -ne 1 ]; then
fail_drill "repository_inventory_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}" 0
fi
write_status 1 "verified" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" \
"${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" \
"${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}"
log_success "Gitea restore drill verified: regular=${backup_regular}/${live_regular} wiki=${backup_wiki}/${live_wiki}; evidence=${STATUS_FILE}"
if [ "${NOTIFY}" -eq 1 ]; then
notify_clawbot "success" "gitea-restore-drill" "Gitea 備份還原演練完成，repo=${backup_regular}/${live_regular}" "$(($(date +%s) - START_TIME))"
fi