wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(backup): make Gitea dump fail-safe

Browse Source
This commit is contained in:
Your Name
2026-07-18 20:21:03 +08:00
parent e7e3bcf8a5
commit 56d1d396b8
9 changed files with 740 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
scripts/backup/backup-configs.sh
View File

@@ -201,6 +201,13 @@ main() {
write_cmd_output "110-systemd-unit-files" systemctl list-unit-files || failed=$((failed + 1))
write_cmd_output "110-docker-containers" docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' || true
if [ -r /home/wooo/vibework-act-runner/config.yaml ]; then
record_config_status "110-gitea-runner-config" true true "110"
else
record_config_status "110-gitea-runner-config" true false "110"
failed=$((failed + 1))
fi
if tar_local "110-host-configs" \
/etc/nginx \
/etc/systemd/system \
@@ -224,6 +231,7 @@ main() {
/home/wooo/scripts \
/home/wooo/awoooi \
/home/wooo/awoooi-ops \
/home/wooo/vibework-act-runner/config.yaml \
/backup/scripts; then
record_config_status "110-host-configs" true true "110"
else

290
scripts/backup/backup-gitea.sh
View File

@@ -7,6 +7,7 @@
# =============================================================================
set -euo pipefail
umask 077
source "$(dirname "$0")/common.sh"
@@ -17,7 +18,22 @@ DUMP_DIR="/tmp/gitea-backup-$$"
TEXTFILE_DIR="${NODE_EXPORTER_TEXTFILE_DIR:-/home/wooo/node_exporter_textfiles}"
TEXTFILE_PATH="${GITEA_FULL_BACKUP_RECEIPT_TEXTFILE:-${TEXTFILE_DIR}/gitea_full_backup_receipt.prom}"
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
GITEA_FULL_BACKUP_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments hooks_custom_assets"
GITEA_FULL_BACKUP_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments actions_logs actions_artifacts avatars hooks_custom_assets"
GITEA_RESTORE_DRILL="$(dirname "$0")/gitea-full-backup-restore-drill.sh"
GITEA_BACKUP_LOCK="${GITEA_BACKUP_LOCK:-/tmp/gitea-backup.lock}"
GITEA_STOP_TIMEOUT_SECONDS="${GITEA_STOP_TIMEOUT_SECONDS:-15}"
GITEA_DUMP_TIMEOUT_SECONDS="${GITEA_DUMP_TIMEOUT_SECONDS:-360}"
GITEA_DUMP_KILL_AFTER_SECONDS="${GITEA_DUMP_KILL_AFTER_SECONDS:-10}"
GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS="${GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS:-10}"
GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS="${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS:-3}"
GITEA_START_TIMEOUT_SECONDS="${GITEA_START_TIMEOUT_SECONDS:-60}"
GITEA_MAX_OUTAGE_SECONDS="${GITEA_MAX_OUTAGE_SECONDS:-600}"
GITEA_LOCAL_HEALTH_URL="${GITEA_LOCAL_HEALTH_URL:-http://127.0.0.1:3001/api/healthz}"
TRANSIENT_CONTAINER=""
PRIMARY_STOPPED=0
PRIMARY_RESTART_VERIFIED=0
OUTAGE_STARTED_AT=0
OUTAGE_SECONDS=0
label_escape() {
printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'
@@ -28,6 +44,10 @@ write_gitea_full_backup_receipt() {
local timestamp="$2"
local duration="$3"
local snapshot_id="${4:-unknown}"
local consistent_offline_window="${5:-0}"
local service_restart_verified="${6:-0}"
local outage_seconds="${7:-0}"
local structural_drill_performed="${8:-0}"
local tmp
local host
local service_label
@@ -66,6 +86,12 @@ write_gitea_full_backup_receipt() {
echo "# TYPE awoooi_gitea_full_backup_secret_value_collected gauge"
echo "# HELP awoooi_gitea_full_backup_production_restore_performed Whether this receipt restored anything into production."
echo "# TYPE awoooi_gitea_full_backup_production_restore_performed gauge"
echo "# HELP awoooi_gitea_full_backup_consistent_offline_window Whether the dump was created while the primary Gitea service was stopped."
echo "# TYPE awoooi_gitea_full_backup_consistent_offline_window gauge"
echo "# HELP awoooi_gitea_full_backup_service_restart_verified Whether Gitea was independently reachable after the bounded offline window."
echo "# TYPE awoooi_gitea_full_backup_service_restart_verified gauge"
echo "# HELP awoooi_gitea_full_backup_service_outage_seconds Duration of the bounded Gitea offline backup window."
echo "# TYPE awoooi_gitea_full_backup_service_outage_seconds gauge"
echo "awoooi_gitea_full_backup_last_success_timestamp{host=\"${host}\",service=\"${service_label}\"} $([ "${ok}" = "1" ] && echo "${timestamp}" || echo 0)"
echo "awoooi_gitea_full_backup_last_run_failed{host=\"${host}\",service=\"${service_label}\",status=\"${status_label}\"} ${failed}"
echo "awoooi_gitea_full_backup_duration_seconds{host=\"${host}\",service=\"${service_label}\"} ${duration}"
@@ -74,16 +100,231 @@ write_gitea_full_backup_receipt() {
for component in ${GITEA_FULL_BACKUP_COMPONENTS}; do
echo "awoooi_gitea_full_backup_component_receipt{host=\"${host}\",service=\"${service_label}\",component=\"$(label_escape "${component}")\",receipt=\"gitea_dump_restic\"} ${ok}"
done
echo "awoooi_gitea_full_backup_restore_drill_performed{host=\"${host}\",service=\"${service_label}\"} 0"
echo "awoooi_gitea_full_backup_restore_drill_performed{host=\"${host}\",service=\"${service_label}\"} ${structural_drill_performed}"
echo "awoooi_gitea_full_backup_secret_value_collected{host=\"${host}\",service=\"${service_label}\"} 0"
echo "awoooi_gitea_full_backup_production_restore_performed{host=\"${host}\",service=\"${service_label}\"} 0"
echo "awoooi_gitea_full_backup_consistent_offline_window{host=\"${host}\",service=\"${service_label}\"} ${consistent_offline_window}"
echo "awoooi_gitea_full_backup_service_restart_verified{host=\"${host}\",service=\"${service_label}\"} ${service_restart_verified}"
echo "awoooi_gitea_full_backup_service_outage_seconds{host=\"${host}\",service=\"${service_label}\"} ${outage_seconds}"
} >"${tmp}"
mv "${tmp}" "${TEXTFILE_PATH}" || return 0
chmod 0644 "${TEXTFILE_PATH}" || true
}
cleanup() {
local status=$?
trap - EXIT HUP INT TERM
if [ -n "${TRANSIENT_CONTAINER}" ]; then
if ! remove_transient_container; then
status=1
fi
fi
if [ "${PRIMARY_STOPPED}" -eq 1 ]; then
if ! start_and_verify_primary; then
status=1
fi
fi
rm -rf "${DUMP_DIR}"
exit "${status}"
}
validate_positive_integer() {
case "$2" in
''|*[!0-9]*|0) log_error "$1 must be a positive integer"; return 64 ;;
esac
}
require_commands() {
local command_name configured_outage_bound
for command_name in curl docker flock python3 restic timeout; do
if ! command -v "${command_name}" >/dev/null 2>&1; then
log_error "Required command missing: ${command_name}"
return 69
fi
done
[ -x "${GITEA_RESTORE_DRILL}" ] || {
log_error "Gitea structural restore verifier is unavailable"
return 69
}
[ -r "${RESTIC_PASSWORD_FILE}" ] || {
log_error "Restic password reference is unavailable"
return 69
}
validate_positive_integer GITEA_STOP_TIMEOUT_SECONDS "${GITEA_STOP_TIMEOUT_SECONDS}"
validate_positive_integer GITEA_DUMP_TIMEOUT_SECONDS "${GITEA_DUMP_TIMEOUT_SECONDS}"
validate_positive_integer GITEA_DUMP_KILL_AFTER_SECONDS "${GITEA_DUMP_KILL_AFTER_SECONDS}"
validate_positive_integer GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS "${GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS}"
validate_positive_integer GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS "${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS}"
validate_positive_integer GITEA_START_TIMEOUT_SECONDS "${GITEA_START_TIMEOUT_SECONDS}"
validate_positive_integer GITEA_MAX_OUTAGE_SECONDS "${GITEA_MAX_OUTAGE_SECONDS}"
configured_outage_bound=$((
GITEA_STOP_TIMEOUT_SECONDS
+ GITEA_DUMP_TIMEOUT_SECONDS
+ GITEA_DUMP_KILL_AFTER_SECONDS
+ GITEA_START_TIMEOUT_SECONDS
+ (4 * GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS)
+ (4 * GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS)
))
if [ "${configured_outage_bound}" -gt "${GITEA_MAX_OUTAGE_SECONDS}" ]; then
log_error "Configured Gitea backup outage bound exceeds ${GITEA_MAX_OUTAGE_SECONDS}s"
return 64
fi
}
bounded_docker_control() {
timeout --signal=TERM \
--kill-after="${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS}s" \
"${GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS}s" \
docker "$@"
}
remove_transient_container() {
local transient="${TRANSIENT_CONTAINER}"
local remove_status=0
[ -n "${transient}" ] || return 0
if ! bounded_docker_control rm -f "${transient}" >/dev/null 2>&1; then
log_error "Gitea one-shot backup container removal timed out or failed"
remove_status=1
fi
TRANSIENT_CONTAINER=""
return "${remove_status}"
}
wait_for_primary_health() {
local deadline=$((SECONDS + GITEA_START_TIMEOUT_SECONDS))
local health_body
while [ "${SECONDS}" -lt "${deadline}" ]; do
if [ "$(bounded_docker_control inspect -f '{{.State.Running}}' "${GITEA_CONTAINER}" 2>/dev/null || true)" = "true" ]; then
health_body="$(curl -fsS --max-time 5 "${GITEA_LOCAL_HEALTH_URL}" 2>/dev/null || true)"
if printf '%s' "${health_body}" | grep -Eq '"status"[[:space:]]*:[[:space:]]*"pass"'; then
return 0
fi
fi
sleep 2
done
return 1
}
start_and_verify_primary() {
local running
running="$(bounded_docker_control inspect -f '{{.State.Running}}' "${GITEA_CONTAINER}" 2>/dev/null || true)"
if [ "${running}" != "true" ]; then
if ! bounded_docker_control start "${GITEA_CONTAINER}" >/dev/null; then
log_error "Gitea failed to restart after the backup window"
return 1
fi
fi
if ! wait_for_primary_health; then
log_error "Gitea post-backup health verification timed out"
return 1
fi
PRIMARY_STOPPED=0
PRIMARY_RESTART_VERIFIED=1
if [ "${OUTAGE_STARTED_AT}" -gt 0 ]; then
OUTAGE_SECONDS=$(($(date +%s) - OUTAGE_STARTED_AT))
fi
if [ "${OUTAGE_SECONDS}" -gt "${GITEA_MAX_OUTAGE_SECONDS}" ]; then
log_error "Gitea backup outage exceeded ${GITEA_MAX_OUTAGE_SECONDS}s"
return 1
fi
log_success "Gitea primary restarted and independently reachable (outage=${OUTAGE_SECONDS}s)"
}
offline_gitea_dump() {
local image_id network_rows network_name network_count git_uid host_uid timestamp dump_status stop_status transient_cleanup_status
[ "$(docker inspect -f '{{.State.Running}}' "${GITEA_CONTAINER}" 2>/dev/null || true)" = "true" ] || {
log_error "Gitea primary is not running before backup"
return 1
}
wait_for_primary_health || {
log_error "Gitea primary is not reachable before backup"
return 1
}
image_id="$(docker inspect -f '{{.Image}}' "${GITEA_CONTAINER}")"
[[ "${image_id}" =~ ^sha256:[0-9a-f]{64}$ ]] || {
log_error "Gitea image identity is invalid"
return 1
}
network_rows="$(docker inspect -f '{{range $name, $value := .NetworkSettings.Networks}}{{$name}}{{"\n"}}{{end}}' "${GITEA_CONTAINER}" | sed '/^$/d')"
network_count="$(printf '%s\n' "${network_rows}" | sed '/^$/d' | wc -l | tr -d ' ')"
[ "${network_count}" -eq 1 ] || {
log_error "Gitea backup requires exactly one fixed Docker network"
return 1
}
network_name="$(printf '%s\n' "${network_rows}" | head -n 1)"
[[ "${network_name}" =~ ^[A-Za-z0-9_.-]+$ ]] || {
log_error "Gitea Docker network identity is invalid"
return 1
}
git_uid="$(docker exec -u git "${GITEA_CONTAINER}" id -u)"
host_uid="$(id -u)"
[ "${git_uid}" = "${host_uid}" ] || {
log_error "Gitea dump staging UID does not match the controller UID"
return 1
}
timestamp="$(date +%Y%m%d%H%M%S)"
TRANSIENT_CONTAINER="agent99-gitea-backup-${timestamp}-$$"
OUTAGE_STARTED_AT="$(date +%s)"
PRIMARY_STOPPED=1
set +e
timeout --signal=TERM \
--kill-after="${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS}s" \
"$((GITEA_STOP_TIMEOUT_SECONDS + GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS))s" \
docker stop --time "${GITEA_STOP_TIMEOUT_SECONDS}" "${GITEA_CONTAINER}" >/dev/null
stop_status=$?
set -e
if [ "${stop_status}" -ne 0 ]; then
log_error "Gitea primary could not enter the bounded offline backup window"
start_and_verify_primary || true
return 1
fi
set +e
timeout --signal=TERM \
--kill-after="${GITEA_DUMP_KILL_AFTER_SECONDS}s" \
"${GITEA_DUMP_TIMEOUT_SECONDS}s" docker run --pull=never \
--name "${TRANSIENT_CONTAINER}" \
--network "${network_name}" \
--volumes-from "${GITEA_CONTAINER}" \
--mount "type=bind,src=${DUMP_DIR},dst=/backup-out" \
--user "${git_uid}" \
--workdir /backup-out \
--cpus "${BACKUP_DOCKER_CPUS}" \
--memory "${BACKUP_DOCKER_MEMORY}" \
--memory-swap "${BACKUP_DOCKER_MEMORY_SWAP}" \
--security-opt no-new-privileges:true \
--entrypoint /usr/local/bin/gitea \
"${image_id}" dump \
-c /data/gitea/conf/app.ini \
-f /backup-out/gitea-dump.zip \
-t /tmp
dump_status=$?
set -e
transient_cleanup_status=0
remove_transient_container || transient_cleanup_status=$?
if ! start_and_verify_primary; then
return 1
fi
if [ "${transient_cleanup_status}" -ne 0 ]; then
return "${transient_cleanup_status}"
fi
if [ "${dump_status}" -ne 0 ]; then
log_error "Offline Gitea dump failed (exit=${dump_status})"
return "${dump_status}"
fi
[ -s "${DUMP_DIR}/gitea-dump.zip" ] || {
log_error "Offline Gitea dump did not produce a non-empty archive"
return 1
}
log_success "Gitea consistent offline dump completed"
}
main() {
@@ -91,31 +332,44 @@ main() {
local tags
local snapshot_id
local duration
local structural_drill_performed=0
start_time=$(date +%s)
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM HUP
log_info "========== 開始 Gitea 備份 =========="
mkdir -p "${DUMP_DIR}"
log_info "執行 Gitea dump..."
if docker exec -u git "${GITEA_CONTAINER}" gitea dump -c /data/gitea/conf/app.ini -f /tmp/gitea-dump.zip 2>&1; then
docker cp "${GITEA_CONTAINER}:/tmp/gitea-dump.zip" "${DUMP_DIR}/gitea-dump.zip"
docker exec -u git "${GITEA_CONTAINER}" rm -f /tmp/gitea-dump.zip
log_success "Gitea dump 完成"
else
duration=$(($(date +%s) - start_time))
write_gitea_full_backup_receipt "dump_failed" "0" "${duration}" "none" || true
log_error "Gitea dump 失敗"
notify_clawbot "failed" "${SERVICE}" "Gitea dump 失敗"
exit 1
require_commands
exec 9>"${GITEA_BACKUP_LOCK}"
if ! flock -n 9; then
log_error "Gitea backup is already running"
return 75
fi
install -d -m 700 "${DUMP_DIR}"
log_info "執行一致性 Gitea offline dump..."
if ! offline_gitea_dump; then
duration=$(($(date +%s) - start_time))
write_gitea_full_backup_receipt "offline_dump_failed" "0" "${duration}" "none" 0 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" 0 || true
notify_clawbot "failed" "${SERVICE}" "Gitea 一致性備份失敗；服務重啟驗證=${PRIMARY_RESTART_VERIFIED}" "${duration}"
return 1
fi
if ! "${GITEA_RESTORE_DRILL}" --dump-zip "${DUMP_DIR}/gitea-dump.zip"; then
duration=$(($(date +%s) - start_time))
write_gitea_full_backup_receipt "structural_drill_failed" "0" "${duration}" "none" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" 0 || true
log_error "Gitea dump structural verifier failed"
notify_clawbot "failed" "${SERVICE}" "Gitea 備份結構驗證失敗" "${duration}"
return 1
fi
structural_drill_performed=1
if [ ! -d "${LOCAL_REPO}/data" ]; then
log_info "初始化本地 Restic 倉庫..."
if ! restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}"; then
duration=$(($(date +%s) - start_time))
write_gitea_full_backup_receipt "restic_init_failed" "0" "${duration}" "none" || true
write_gitea_full_backup_receipt "restic_init_failed" "0" "${duration}" "none" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" "${structural_drill_performed}" || true
log_error "Restic 初始化失敗"
notify_clawbot "failed" "${SERVICE}" "Gitea Restic 初始化失敗"
exit 1
@@ -127,7 +381,7 @@ main() {
--password-file "${RESTIC_PASSWORD_FILE}" \
${tags}; then
duration=$(($(date +%s) - start_time))
write_gitea_full_backup_receipt "restic_backup_failed" "0" "${duration}" "none" || true
write_gitea_full_backup_receipt "restic_backup_failed" "0" "${duration}" "none" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" "${structural_drill_performed}" || true
log_error "Restic 備份失敗"
notify_clawbot "failed" "${SERVICE}" "Gitea Restic 備份失敗"
exit 1
@@ -142,7 +396,7 @@ main() {
log_info "Offsite copy is handled by sync-offsite-backups.sh; no direct rclone sync here."
duration=$(($(date +%s) - start_time))
write_gitea_full_backup_receipt "success" "$(date +%s)" "${duration}" "${snapshot_id:-unknown}" || true
write_gitea_full_backup_receipt "success" "$(date +%s)" "${duration}" "${snapshot_id:-unknown}" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" "${structural_drill_performed}" || true
log_success "========== Gitea 備份完成 (${duration}s) =========="
notify_clawbot "success" "${SERVICE}" "Gitea 備份完成" "${duration}"
}

55
scripts/backup/gitea-full-backup-restore-drill.sh
View File

@@ -9,9 +9,9 @@ DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}"
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
TEXTFILE_PATH="${AIOPS_GITEA_FULL_BACKUP_RESTORE_DRILL_TEXTFILE:-/home/wooo/node_exporter_textfiles/gitea_full_backup_restore_drill.prom}"
WRITE_TEXTFILE=0
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-60}"
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-300}"
DRILL_SCOPE="structural_metadata_only"
REQUIRED_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments hooks_custom_assets"
REQUIRED_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments actions_logs actions_artifacts avatars hooks_custom_assets"
usage() {
cat <<'USAGE'
@@ -88,35 +88,50 @@ required = sys.argv[2:]
try:
with zipfile.ZipFile(dump_zip) as archive:
names = archive.namelist()
bad_member = archive.testzip()
infos = archive.infolist()
if bad_member is not None:
print("ERROR=crc_failed")
sys.exit(1)
except zipfile.BadZipFile:
print("ERROR=bad_zip")
sys.exit(1)
except OSError:
except (OSError, RuntimeError):
print("ERROR=zip_open_failed")
sys.exit(1)
lowered = [name.lower() for name in names]
members = [
(info.filename.lower().lstrip("./"), info.file_size)
for info in infos
if not info.is_dir()
]
patterns = {
"database": ("gitea-db", "database", ".sql"),
"repositories": ("repo", "repos", "repositories", ".git"),
"app_settings": ("app.ini", "app.example.ini", "conf/"),
"lfs_objects": ("lfs",),
"package_registry": ("package", "packages"),
"attachments": ("attachment", "attachments"),
"hooks_custom_assets": ("hook", "custom", "public/"),
def has_nonempty(*needles: str) -> bool:
return any(size > 0 and any(needle in name for needle in needles) for name, size in members)
data_bundle = has_nonempty("gitea-data.zip")
custom_bundle = has_nonempty("gitea-custom.zip")
coverage = {
"database": has_nonempty("gitea-db", "database", ".sql"),
"repositories": has_nonempty("gitea-repo.zip", "gitea-repos.zip", "repos/", "repositories/", ".git/"),
"app_settings": has_nonempty("app.ini", "app.example.ini"),
"lfs_objects": data_bundle or has_nonempty("data/lfs/", "/lfs/"),
"package_registry": data_bundle or has_nonempty("data/packages/", "/packages/"),
"attachments": data_bundle or has_nonempty("data/attachments/", "/attachments/"),
"actions_logs": data_bundle or has_nonempty("data/actions_log/", "/actions_log/"),
"actions_artifacts": data_bundle or has_nonempty("data/actions_artifacts/", "/actions_artifacts/"),
"avatars": data_bundle or has_nonempty("data/avatars/", "data/repo-avatars/", "/avatars/", "/repo-avatars/"),
"hooks_custom_assets": custom_bundle or has_nonempty("custom/", "/hooks/", "public/"),
}
missing = []
for component in required:
needles = patterns.get(component, (component,))
if not any(any(needle in name for needle in needles) for name in lowered):
missing.append(component)
missing = [component for component in required if not coverage.get(component, False)]
print(f"ENTRY_COUNT={len(names)}")
print(f"ENTRY_COUNT={len(infos)}")
print(f"MISSING_COMPONENTS={','.join(missing)}")
print("OK=1" if not missing and names else "OK=0")
print("OK=1" if not missing and infos else "OK=0")
PY
)"
drill_status=$?

5
scripts/backup/tests/test_110_backup_runtime_deploy_playbook.py
View File

@@ -18,6 +18,9 @@ EXPECTED_FILES = {
"backup-host188-products.sh",
"verify-host188-products-backup.sh",
"verify-host188-products-archive.py",
"backup-gitea.sh",
"gitea-full-backup-restore-drill.sh",
"backup-configs.sh",
"check-backup-integrity.sh",
"sync-offsite-backups.sh",
"backup-offsite-readiness-gate.sh",
@@ -26,7 +29,7 @@ EXPECTED_FILES = {
}
def test_backup_runtime_deploy_is_fixed_to_host_110_and_fourteen_files() -> None:
def test_backup_runtime_deploy_is_fixed_to_host_110_and_seventeen_files() -> None:
plays = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8"))
assert len(plays) == 1
play = plays[0]

44
scripts/backup/tests/test_backup_gitea_full_receipt_contract.py
View File

@@ -5,6 +5,7 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
BACKUP_GITEA = ROOT / "scripts" / "backup" / "backup-gitea.sh"
BACKUP_CONFIGS = ROOT / "scripts" / "backup" / "backup-configs.sh"
def test_backup_gitea_writes_full_server_component_receipts() -> None:
@@ -15,7 +16,12 @@ def test_backup_gitea_writes_full_server_component_receipts() -> None:
assert "awoooi_gitea_full_backup_restore_drill_performed" in text
assert "awoooi_gitea_full_backup_secret_value_collected" in text
assert "awoooi_gitea_full_backup_production_restore_performed" in text
assert "gitea dump -c /data/gitea/conf/app.ini" in text
assert "offline_gitea_dump" in text
assert "--volumes-from" in text
assert "--pull=never" in text
assert "docker stop --time" in text
assert "start_and_verify_primary" in text
assert "gitea-full-backup-restore-drill.sh" in text
assert "GITEA_FULL_BACKUP_COMPONENTS" in text
for component in [
@@ -25,6 +31,9 @@ def test_backup_gitea_writes_full_server_component_receipts() -> None:
"lfs_objects",
"package_registry",
"attachments",
"actions_logs",
"actions_artifacts",
"avatars",
"hooks_custom_assets",
]:
assert component in text
@@ -37,7 +46,7 @@ def test_backup_gitea_receipt_contract_stays_no_secret_no_restore() -> None:
assert "awoooi_gitea_full_backup_production_restore_performed" in text
assert "awoooi_gitea_full_backup_restore_drill_performed" in text
assert (
'restore_drill_performed{host=\\"${host}\\",service=\\"${service_label}\\"} 0'
'restore_drill_performed{host=\\"${host}\\",service=\\"${service_label}\\"} ${structural_drill_performed}'
in text
)
assert (
@@ -48,3 +57,34 @@ def test_backup_gitea_receipt_contract_stays_no_secret_no_restore() -> None:
'production_restore_performed{host=\\"${host}\\",service=\\"${service_label}\\"} 0'
in text
)
def test_backup_gitea_is_bounded_and_fail_safe_around_the_offline_window() -> None:
text = BACKUP_GITEA.read_text(encoding="utf-8")
assert 'GITEA_BACKUP_LOCK="${GITEA_BACKUP_LOCK:-/tmp/gitea-backup.lock}"' in text
assert "flock -n 9" in text
assert "trap cleanup EXIT" in text
assert 'if [ "${PRIMARY_STOPPED}" -eq 1 ]' in text
assert "wait_for_primary_health" in text
assert "http://127.0.0.1:3001/api/healthz" in text
assert '--kill-after="${GITEA_DUMP_KILL_AFTER_SECONDS}s"' in text
assert "bounded_docker_control rm -f" in text
assert 'GITEA_MAX_OUTAGE_SECONDS="${GITEA_MAX_OUTAGE_SECONDS:-600}"' in text
assert "configured_outage_bound" in text
assert "--security-opt no-new-privileges:true" in text
assert '--cpus "${BACKUP_DOCKER_CPUS}"' in text
assert '--memory "${BACKUP_DOCKER_MEMORY}"' in text
assert '--memory-swap "${BACKUP_DOCKER_MEMORY_SWAP}"' in text
assert "awoooi_gitea_full_backup_consistent_offline_window" in text
assert "awoooi_gitea_full_backup_service_restart_verified" in text
assert "awoooi_gitea_full_backup_service_outage_seconds" in text
def test_gitea_runner_configuration_is_in_the_encrypted_config_backup() -> None:
text = BACKUP_CONFIGS.read_text(encoding="utf-8")
assert "/home/wooo/vibework-act-runner/config.yaml" in text
assert 'record_config_status "110-gitea-runner-config" true true "110"' in text
assert 'record_config_status "110-gitea-runner-config" true false "110"' in text
assert "/home/wooo/vibework-act-runner/data" not in text

300
scripts/backup/tests/test_backup_gitea_offline_window_behavior.py Normal file
View File

@@ -0,0 +1,300 @@
from __future__ import annotations
import os
import subprocess
import textwrap
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts" / "backup" / "backup-gitea.sh"
def _write_executable(path: Path, source: str) -> None:
path.write_text(textwrap.dedent(source), encoding="utf-8")
path.chmod(0o755)
def _fake_runtime(tmp_path: Path) -> tuple[dict[str, str], Path, Path]:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
state_dir = tmp_path / "state"
state_dir.mkdir()
(state_dir / "running").write_text("true\n", encoding="utf-8")
command_log = state_dir / "commands.log"
_write_executable(
fake_bin / "docker",
r"""
#!/usr/bin/env bash
set -euo pipefail
state="${FAKE_DOCKER_STATE:?}"
printf '%s\n' "$*" >> "$state/commands.log"
command_name="${1:-}"
shift || true
case "$command_name" in
inspect)
args="$*"
case "$args" in
*State.Running*) cat "$state/running" ;;
*NetworkSettings.Networks*) printf 'gitea_gitea\n' ;;
*'{{.Image}}'*) printf 'sha256:%064d\n' 0 ;;
*) exit 64 ;;
esac
;;
exec)
args="$*"
if [[ "$args" == *' id -u'* ]]; then
id -u
elif [[ "$args" == *'gitea --version'* ]]; then
printf 'gitea version 1.25.5 built with test\n'
else
exit 64
fi
;;
stop)
printf 'false\n' > "$state/running"
if [ "${FAKE_DOCKER_STOP_FAIL:-0}" = "1" ]; then
exit 42
fi
;;
start)
printf 'true\n' > "$state/running"
;;
run)
output=""
for arg in "$@"; do
case "$arg" in
type=bind,src=*,dst=/backup-out)
output="${arg#type=bind,src=}"
output="${output%,dst=/backup-out}"
;;
esac
done
[ -n "$output" ] || exit 65
if [ "${FAKE_DOCKER_RUN_FAIL:-0}" = "1" ]; then
exit 42
fi
python3 - "$output/gitea-dump.zip" <<'PY'
import sys
import zipfile
entries = {
"gitea-db.sql": "fixture\n",
"repos/wooo/awoooi.git/HEAD": "ref: refs/heads/main\n",
"custom/conf/app.ini": "[server]\n",
"data/lfs/objects/aa/bb/object": "lfs\n",
"data/packages/package.bin": "package\n",
"data/attachments/attachment.bin": "attachment\n",
"data/actions_artifacts/aa/bb/artifact.zip": "artifact\n",
"data/actions_log/aa/bb/job.log": "log\n",
"data/avatars/aa/bb/avatar.png": "avatar\n",
"custom/hooks/pre-receive": "#!/bin/sh\n",
}
with zipfile.ZipFile(sys.argv[1], "w") as archive:
for name, value in entries.items():
archive.writestr(name, value)
PY
;;
rm) ;;
*) exit 64 ;;
esac
""",
)
_write_executable(
fake_bin / "curl",
r"""
#!/usr/bin/env bash
url="${!#}"
if [ "$url" = "http://127.0.0.1:3001/api/healthz" ]; then
printf '{"status":"pass"}'
else
printf '<html>another service</html>'
fi
""",
)
_write_executable(
fake_bin / "timeout",
"""
#!/usr/bin/env bash
set -euo pipefail
while [[ "${1:-}" == --* ]]; do shift; done
shift
if [ "${FAKE_DOCKER_RUN_TIMEOUT:-0}" = "1" ] && [[ " $* " == *' docker run '* ]]; then
exit 124
fi
exec "$@"
""",
)
_write_executable(
fake_bin / "flock",
"""
#!/usr/bin/env bash
exit 0
""",
)
_write_executable(
fake_bin / "restic",
r"""
#!/usr/bin/env bash
set -euo pipefail
repo=""
args=("$@")
for ((i=0; i<${#args[@]}; i++)); do
if [ "${args[$i]}" = "-r" ]; then repo="${args[$((i+1))]}"; fi
done
if [[ " $* " == *' init '* ]]; then
mkdir -p "$repo/data" "$repo/snapshots"
exit 0
fi
if [[ " $* " == *' backup '* ]]; then
mkdir -p "$repo/data" "$repo/snapshots"
exit 0
fi
if [[ " $* " == *' snapshots '* ]]; then
printf '[{"short_id":"abcd1234","time":"2026-07-18T00:00:00Z"}]\n'
exit 0
fi
if [[ " $* " == *' forget '* ]]; then exit 0; fi
exit 64
""",
)
backup_root = tmp_path / "backup"
(backup_root / "scripts").mkdir(parents=True)
(backup_root / "scripts" / ".restic-password").write_text("fixture\n", encoding="utf-8")
textfile = tmp_path / "gitea.prom"
environment = os.environ | {
"PATH": f"{fake_bin}:{os.environ['PATH']}",
"BACKUP_BASE": str(backup_root),
"BACKUP_LOG_DIR": str(backup_root / "logs"),
"RESTIC_PASSWORD_FILE": str(backup_root / "scripts" / ".restic-password"),
"GITEA_FULL_BACKUP_RECEIPT_TEXTFILE": str(textfile),
"NODE_EXPORTER_TEXTFILE_DIR": str(tmp_path),
"GITEA_BACKUP_LOCK": str(tmp_path / "gitea.lock"),
"GITEA_START_TIMEOUT_SECONDS": "4",
"GITEA_DUMP_TIMEOUT_SECONDS": "30",
"GITEA_DUMP_KILL_AFTER_SECONDS": "2",
"GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS": "2",
"GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS": "1",
"GITEA_STOP_TIMEOUT_SECONDS": "2",
"FAKE_DOCKER_STATE": str(state_dir),
"BACKUP_RETENTION_MODE": "latest",
}
return environment, command_log, textfile
def test_offline_backup_restarts_and_verifies_primary_on_success(tmp_path: Path) -> None:
environment, command_log, textfile = _fake_runtime(tmp_path)
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode == 0, result.stderr
commands = command_log.read_text(encoding="utf-8")
assert commands.index("stop --time 2 gitea") < commands.index("run --pull=never")
assert commands.index("run --pull=never") < commands.index("rm -f agent99-gitea-backup-")
assert commands.index("rm -f agent99-gitea-backup-") < commands.index("start gitea")
rendered = textfile.read_text(encoding="utf-8")
assert "awoooi_gitea_full_backup_consistent_offline_window" in rendered
assert "awoooi_gitea_full_backup_service_restart_verified" in rendered
assert 'service="gitea"} 1' in rendered
def test_offline_backup_restarts_primary_when_dump_fails(tmp_path: Path) -> None:
environment, command_log, textfile = _fake_runtime(tmp_path)
environment["FAKE_DOCKER_RUN_FAIL"] = "1"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
assert (Path(environment["FAKE_DOCKER_STATE"]) / "running").read_text(encoding="utf-8").strip() == "true"
commands = command_log.read_text(encoding="utf-8")
assert "stop --time 2 gitea" in commands
assert commands.index("rm -f agent99-gitea-backup-") < commands.index("start gitea")
rendered = textfile.read_text(encoding="utf-8")
assert 'awoooi_gitea_full_backup_service_restart_verified{host="110",service="gitea"} 1' in rendered
assert 'status="offline_dump_failed"} 1' in rendered
def test_offline_backup_restarts_primary_when_dump_hits_hard_timeout(tmp_path: Path) -> None:
environment, command_log, textfile = _fake_runtime(tmp_path)
environment["FAKE_DOCKER_RUN_TIMEOUT"] = "1"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
assert (Path(environment["FAKE_DOCKER_STATE"]) / "running").read_text(encoding="utf-8").strip() == "true"
commands = command_log.read_text(encoding="utf-8")
assert commands.index("rm -f agent99-gitea-backup-") < commands.index("start gitea")
assert 'status="offline_dump_failed"} 1' in textfile.read_text(encoding="utf-8")
def test_offline_backup_recovers_when_stop_has_side_effect_then_fails(tmp_path: Path) -> None:
environment, command_log, textfile = _fake_runtime(tmp_path)
environment["FAKE_DOCKER_STOP_FAIL"] = "1"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
assert (Path(environment["FAKE_DOCKER_STATE"]) / "running").read_text(encoding="utf-8").strip() == "true"
commands = command_log.read_text(encoding="utf-8")
assert commands.index("stop --time 2 gitea") < commands.index("start gitea")
assert 'service_restart_verified{host="110",service="gitea"} 1' in textfile.read_text(encoding="utf-8")
def test_wrong_service_health_response_cannot_open_offline_window(tmp_path: Path) -> None:
environment, command_log, _ = _fake_runtime(tmp_path)
environment["GITEA_LOCAL_HEALTH_URL"] = "http://127.0.0.1:3000/"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
commands = command_log.read_text(encoding="utf-8")
assert "stop --time" not in commands
def test_oversized_outage_budget_fails_before_primary_stop(tmp_path: Path) -> None:
environment, command_log, _ = _fake_runtime(tmp_path)
environment["GITEA_MAX_OUTAGE_SECONDS"] = "10"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
assert "Configured Gitea backup outage bound exceeds 10s" in result.stdout
assert not command_log.exists() or "stop --time" not in command_log.read_text(encoding="utf-8")

77
scripts/backup/tests/test_gitea_full_backup_restore_drill.py
View File

@@ -9,15 +9,25 @@ ROOT = Path(__file__).resolve().parents[1]
SCRIPT = ROOT / "gitea-full-backup-restore-drill.sh"
def _write_dump(path: Path, *, include_lfs: bool = True) -> None:
def _write_dump(
path: Path,
*,
include_lfs: bool = True,
include_actions_artifact: bool = True,
empty_critical_payloads: bool = False,
) -> None:
entries = {
"gitea-db.sql": "-- redacted fixture\n",
"repos/wooo/awoooi.git/HEAD": "ref: refs/heads/main\n",
"gitea-db.sql": "" if empty_critical_payloads else "-- redacted fixture\n",
"repos/wooo/awoooi.git/HEAD": "" if empty_critical_payloads else "ref: refs/heads/main\n",
"custom/conf/app.ini": "[server]\nAPP_NAME=fixture\n",
"data/packages/package.bin": "package fixture\n",
"data/attachments/attachment.bin": "attachment fixture\n",
"data/actions_log/aa/bb/job.log": "actions log fixture\n",
"data/avatars/aa/bb/avatar.png": "avatar fixture\n",
"custom/hooks/pre-receive": "#!/bin/sh\n",
}
if include_actions_artifact:
entries["data/actions_artifacts/aa/bb/artifact.zip"] = "actions artifact fixture\n"
if include_lfs:
entries["data/lfs/objects/aa/bb/object"] = "lfs fixture\n"
with zipfile.ZipFile(path, "w") as archive:
@@ -97,3 +107,64 @@ def test_gitea_full_backup_restore_drill_fails_closed_when_component_missing(
rendered = textfile.read_text(encoding="utf-8")
assert 'component="lfs_objects",drill_scope="structural_metadata_only"} 0' in rendered
assert 'production_restore_performed{host="110",service="gitea",' in rendered
def test_gitea_full_backup_restore_drill_rejects_empty_database_and_repository(
tmp_path: Path,
) -> None:
dump = tmp_path / "gitea-dump.zip"
_write_dump(dump, empty_critical_payloads=True)
result = subprocess.run(
["bash", str(SCRIPT), "--dump-zip", str(dump)],
text=True,
capture_output=True,
)
assert result.returncode == 1
assert "ERROR=component_coverage_gap" in result.stdout
def test_gitea_full_backup_restore_drill_requires_actions_log_and_artifact(
tmp_path: Path,
) -> None:
dump = tmp_path / "gitea-dump.zip"
textfile = tmp_path / "gitea_full_backup_restore_drill.prom"
_write_dump(dump, include_actions_artifact=False)
result = subprocess.run(
[
"bash",
str(SCRIPT),
"--dump-zip",
str(dump),
"--write-textfile",
"--textfile",
str(textfile),
],
text=True,
capture_output=True,
)
assert result.returncode == 1
rendered = textfile.read_text(encoding="utf-8")
assert 'component="actions_logs",drill_scope="structural_metadata_only"} 1' not in rendered
assert 'component="actions_artifacts",drill_scope="structural_metadata_only"} 0' in rendered
def test_gitea_full_backup_restore_drill_rejects_crc_corruption(tmp_path: Path) -> None:
dump = tmp_path / "gitea-dump.zip"
_write_dump(dump)
payload = dump.read_bytes()
fixture = b"-- redacted fixture\n"
assert fixture in payload
dump.write_bytes(payload.replace(fixture, b"X" + fixture[1:], 1))
result = subprocess.run(
["bash", str(SCRIPT), "--dump-zip", str(dump)],
text=True,
capture_output=True,
)
assert result.returncode == 1
assert "ERROR=crc_failed" in result.stdout