diff --git a/infra/ansible/playbooks/110-backup-runtime-deploy.yml b/infra/ansible/playbooks/110-backup-runtime-deploy.yml index 679f1c3c9..48e533413 100644 --- a/infra/ansible/playbooks/110-backup-runtime-deploy.yml +++ b/infra/ansible/playbooks/110-backup-runtime-deploy.yml @@ -17,6 +17,9 @@ - backup-host188-products.sh - verify-host188-products-backup.sh - verify-host188-products-archive.py + - backup-gitea.sh + - gitea-full-backup-restore-drill.sh + - backup-configs.sh backup_runtime_stage2_files: - backup-awoooi.sh - backup-awoooi-frequent.sh @@ -35,7 +38,7 @@ ansible.builtin.assert: that: - inventory_hostname == "host_110" - - backup_runtime_files | length == 14 + - backup_runtime_files | length == 17 - backup_runtime_files | unique | length == backup_runtime_files | length - backup_runtime_fault_injection_after_stage1 | type_debug == "bool" fail_msg: backup_runtime_deploy_boundary_failed diff --git a/infra/ansible/playbooks/110-devops.yml b/infra/ansible/playbooks/110-devops.yml index 29205ef4f..608b68d6a 100644 --- a/infra/ansible/playbooks/110-devops.yml +++ b/infra/ansible/playbooks/110-devops.yml @@ -224,6 +224,7 @@ - backup-all.sh - backup-status.sh - backup-gitea.sh + - gitea-full-backup-restore-drill.sh - gitea-repo-bundle-backup.sh - gitea-bundle-backup-sync-188.sh - backup-harbor.sh diff --git a/scripts/backup/backup-configs.sh b/scripts/backup/backup-configs.sh index 50fcfe28d..e6290eac3 100755 --- a/scripts/backup/backup-configs.sh +++ b/scripts/backup/backup-configs.sh @@ -201,6 +201,13 @@ main() { write_cmd_output "110-systemd-unit-files" systemctl list-unit-files || failed=$((failed + 1)) write_cmd_output "110-docker-containers" docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' || true + if [ -r /home/wooo/vibework-act-runner/config.yaml ]; then + record_config_status "110-gitea-runner-config" true true "110" + else + record_config_status "110-gitea-runner-config" true false "110" + failed=$((failed + 1)) + fi + if tar_local "110-host-configs" \ /etc/nginx \ /etc/systemd/system \ @@ -224,6 +231,7 @@ main() { /home/wooo/scripts \ /home/wooo/awoooi \ /home/wooo/awoooi-ops \ + /home/wooo/vibework-act-runner/config.yaml \ /backup/scripts; then record_config_status "110-host-configs" true true "110" else diff --git a/scripts/backup/backup-gitea.sh b/scripts/backup/backup-gitea.sh index 94ee6a958..1807b8da5 100755 --- a/scripts/backup/backup-gitea.sh +++ b/scripts/backup/backup-gitea.sh @@ -7,6 +7,7 @@ # ============================================================================= set -euo pipefail +umask 077 source "$(dirname "$0")/common.sh" @@ -17,7 +18,22 @@ DUMP_DIR="/tmp/gitea-backup-$$" TEXTFILE_DIR="${NODE_EXPORTER_TEXTFILE_DIR:-/home/wooo/node_exporter_textfiles}" TEXTFILE_PATH="${GITEA_FULL_BACKUP_RECEIPT_TEXTFILE:-${TEXTFILE_DIR}/gitea_full_backup_receipt.prom}" HOST_LABEL="${AIOPS_HOST_LABEL:-110}" -GITEA_FULL_BACKUP_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments hooks_custom_assets" +GITEA_FULL_BACKUP_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments actions_logs actions_artifacts avatars hooks_custom_assets" +GITEA_RESTORE_DRILL="$(dirname "$0")/gitea-full-backup-restore-drill.sh" +GITEA_BACKUP_LOCK="${GITEA_BACKUP_LOCK:-/tmp/gitea-backup.lock}" +GITEA_STOP_TIMEOUT_SECONDS="${GITEA_STOP_TIMEOUT_SECONDS:-15}" +GITEA_DUMP_TIMEOUT_SECONDS="${GITEA_DUMP_TIMEOUT_SECONDS:-360}" +GITEA_DUMP_KILL_AFTER_SECONDS="${GITEA_DUMP_KILL_AFTER_SECONDS:-10}" +GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS="${GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS:-10}" +GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS="${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS:-3}" +GITEA_START_TIMEOUT_SECONDS="${GITEA_START_TIMEOUT_SECONDS:-60}" +GITEA_MAX_OUTAGE_SECONDS="${GITEA_MAX_OUTAGE_SECONDS:-600}" +GITEA_LOCAL_HEALTH_URL="${GITEA_LOCAL_HEALTH_URL:-http://127.0.0.1:3001/api/healthz}" +TRANSIENT_CONTAINER="" +PRIMARY_STOPPED=0 +PRIMARY_RESTART_VERIFIED=0 +OUTAGE_STARTED_AT=0 +OUTAGE_SECONDS=0 label_escape() { printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' @@ -28,6 +44,10 @@ write_gitea_full_backup_receipt() { local timestamp="$2" local duration="$3" local snapshot_id="${4:-unknown}" + local consistent_offline_window="${5:-0}" + local service_restart_verified="${6:-0}" + local outage_seconds="${7:-0}" + local structural_drill_performed="${8:-0}" local tmp local host local service_label @@ -66,6 +86,12 @@ write_gitea_full_backup_receipt() { echo "# TYPE awoooi_gitea_full_backup_secret_value_collected gauge" echo "# HELP awoooi_gitea_full_backup_production_restore_performed Whether this receipt restored anything into production." echo "# TYPE awoooi_gitea_full_backup_production_restore_performed gauge" + echo "# HELP awoooi_gitea_full_backup_consistent_offline_window Whether the dump was created while the primary Gitea service was stopped." + echo "# TYPE awoooi_gitea_full_backup_consistent_offline_window gauge" + echo "# HELP awoooi_gitea_full_backup_service_restart_verified Whether Gitea was independently reachable after the bounded offline window." + echo "# TYPE awoooi_gitea_full_backup_service_restart_verified gauge" + echo "# HELP awoooi_gitea_full_backup_service_outage_seconds Duration of the bounded Gitea offline backup window." + echo "# TYPE awoooi_gitea_full_backup_service_outage_seconds gauge" echo "awoooi_gitea_full_backup_last_success_timestamp{host=\"${host}\",service=\"${service_label}\"} $([ "${ok}" = "1" ] && echo "${timestamp}" || echo 0)" echo "awoooi_gitea_full_backup_last_run_failed{host=\"${host}\",service=\"${service_label}\",status=\"${status_label}\"} ${failed}" echo "awoooi_gitea_full_backup_duration_seconds{host=\"${host}\",service=\"${service_label}\"} ${duration}" @@ -74,16 +100,231 @@ write_gitea_full_backup_receipt() { for component in ${GITEA_FULL_BACKUP_COMPONENTS}; do echo "awoooi_gitea_full_backup_component_receipt{host=\"${host}\",service=\"${service_label}\",component=\"$(label_escape "${component}")\",receipt=\"gitea_dump_restic\"} ${ok}" done - echo "awoooi_gitea_full_backup_restore_drill_performed{host=\"${host}\",service=\"${service_label}\"} 0" + echo "awoooi_gitea_full_backup_restore_drill_performed{host=\"${host}\",service=\"${service_label}\"} ${structural_drill_performed}" echo "awoooi_gitea_full_backup_secret_value_collected{host=\"${host}\",service=\"${service_label}\"} 0" echo "awoooi_gitea_full_backup_production_restore_performed{host=\"${host}\",service=\"${service_label}\"} 0" + echo "awoooi_gitea_full_backup_consistent_offline_window{host=\"${host}\",service=\"${service_label}\"} ${consistent_offline_window}" + echo "awoooi_gitea_full_backup_service_restart_verified{host=\"${host}\",service=\"${service_label}\"} ${service_restart_verified}" + echo "awoooi_gitea_full_backup_service_outage_seconds{host=\"${host}\",service=\"${service_label}\"} ${outage_seconds}" } >"${tmp}" mv "${tmp}" "${TEXTFILE_PATH}" || return 0 chmod 0644 "${TEXTFILE_PATH}" || true } cleanup() { + local status=$? + trap - EXIT HUP INT TERM + if [ -n "${TRANSIENT_CONTAINER}" ]; then + if ! remove_transient_container; then + status=1 + fi + fi + if [ "${PRIMARY_STOPPED}" -eq 1 ]; then + if ! start_and_verify_primary; then + status=1 + fi + fi rm -rf "${DUMP_DIR}" + exit "${status}" +} + +validate_positive_integer() { + case "$2" in + ''|*[!0-9]*|0) log_error "$1 must be a positive integer"; return 64 ;; + esac +} + +require_commands() { + local command_name configured_outage_bound + for command_name in curl docker flock python3 restic timeout; do + if ! command -v "${command_name}" >/dev/null 2>&1; then + log_error "Required command missing: ${command_name}" + return 69 + fi + done + [ -x "${GITEA_RESTORE_DRILL}" ] || { + log_error "Gitea structural restore verifier is unavailable" + return 69 + } + [ -r "${RESTIC_PASSWORD_FILE}" ] || { + log_error "Restic password reference is unavailable" + return 69 + } + validate_positive_integer GITEA_STOP_TIMEOUT_SECONDS "${GITEA_STOP_TIMEOUT_SECONDS}" + validate_positive_integer GITEA_DUMP_TIMEOUT_SECONDS "${GITEA_DUMP_TIMEOUT_SECONDS}" + validate_positive_integer GITEA_DUMP_KILL_AFTER_SECONDS "${GITEA_DUMP_KILL_AFTER_SECONDS}" + validate_positive_integer GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS "${GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS}" + validate_positive_integer GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS "${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS}" + validate_positive_integer GITEA_START_TIMEOUT_SECONDS "${GITEA_START_TIMEOUT_SECONDS}" + validate_positive_integer GITEA_MAX_OUTAGE_SECONDS "${GITEA_MAX_OUTAGE_SECONDS}" + + configured_outage_bound=$(( + GITEA_STOP_TIMEOUT_SECONDS + + GITEA_DUMP_TIMEOUT_SECONDS + + GITEA_DUMP_KILL_AFTER_SECONDS + + GITEA_START_TIMEOUT_SECONDS + + (4 * GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS) + + (4 * GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS) + )) + if [ "${configured_outage_bound}" -gt "${GITEA_MAX_OUTAGE_SECONDS}" ]; then + log_error "Configured Gitea backup outage bound exceeds ${GITEA_MAX_OUTAGE_SECONDS}s" + return 64 + fi +} + +bounded_docker_control() { + timeout --signal=TERM \ + --kill-after="${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS}s" \ + "${GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS}s" \ + docker "$@" +} + +remove_transient_container() { + local transient="${TRANSIENT_CONTAINER}" + local remove_status=0 + + [ -n "${transient}" ] || return 0 + if ! bounded_docker_control rm -f "${transient}" >/dev/null 2>&1; then + log_error "Gitea one-shot backup container removal timed out or failed" + remove_status=1 + fi + TRANSIENT_CONTAINER="" + return "${remove_status}" +} + +wait_for_primary_health() { + local deadline=$((SECONDS + GITEA_START_TIMEOUT_SECONDS)) + local health_body + while [ "${SECONDS}" -lt "${deadline}" ]; do + if [ "$(bounded_docker_control inspect -f '{{.State.Running}}' "${GITEA_CONTAINER}" 2>/dev/null || true)" = "true" ]; then + health_body="$(curl -fsS --max-time 5 "${GITEA_LOCAL_HEALTH_URL}" 2>/dev/null || true)" + if printf '%s' "${health_body}" | grep -Eq '"status"[[:space:]]*:[[:space:]]*"pass"'; then + return 0 + fi + fi + sleep 2 + done + return 1 +} + +start_and_verify_primary() { + local running + + running="$(bounded_docker_control inspect -f '{{.State.Running}}' "${GITEA_CONTAINER}" 2>/dev/null || true)" + if [ "${running}" != "true" ]; then + if ! bounded_docker_control start "${GITEA_CONTAINER}" >/dev/null; then + log_error "Gitea failed to restart after the backup window" + return 1 + fi + fi + if ! wait_for_primary_health; then + log_error "Gitea post-backup health verification timed out" + return 1 + fi + PRIMARY_STOPPED=0 + PRIMARY_RESTART_VERIFIED=1 + if [ "${OUTAGE_STARTED_AT}" -gt 0 ]; then + OUTAGE_SECONDS=$(($(date +%s) - OUTAGE_STARTED_AT)) + fi + if [ "${OUTAGE_SECONDS}" -gt "${GITEA_MAX_OUTAGE_SECONDS}" ]; then + log_error "Gitea backup outage exceeded ${GITEA_MAX_OUTAGE_SECONDS}s" + return 1 + fi + log_success "Gitea primary restarted and independently reachable (outage=${OUTAGE_SECONDS}s)" +} + +offline_gitea_dump() { + local image_id network_rows network_name network_count git_uid host_uid timestamp dump_status stop_status transient_cleanup_status + + [ "$(docker inspect -f '{{.State.Running}}' "${GITEA_CONTAINER}" 2>/dev/null || true)" = "true" ] || { + log_error "Gitea primary is not running before backup" + return 1 + } + wait_for_primary_health || { + log_error "Gitea primary is not reachable before backup" + return 1 + } + + image_id="$(docker inspect -f '{{.Image}}' "${GITEA_CONTAINER}")" + [[ "${image_id}" =~ ^sha256:[0-9a-f]{64}$ ]] || { + log_error "Gitea image identity is invalid" + return 1 + } + network_rows="$(docker inspect -f '{{range $name, $value := .NetworkSettings.Networks}}{{$name}}{{"\n"}}{{end}}' "${GITEA_CONTAINER}" | sed '/^$/d')" + network_count="$(printf '%s\n' "${network_rows}" | sed '/^$/d' | wc -l | tr -d ' ')" + [ "${network_count}" -eq 1 ] || { + log_error "Gitea backup requires exactly one fixed Docker network" + return 1 + } + network_name="$(printf '%s\n' "${network_rows}" | head -n 1)" + [[ "${network_name}" =~ ^[A-Za-z0-9_.-]+$ ]] || { + log_error "Gitea Docker network identity is invalid" + return 1 + } + git_uid="$(docker exec -u git "${GITEA_CONTAINER}" id -u)" + host_uid="$(id -u)" + [ "${git_uid}" = "${host_uid}" ] || { + log_error "Gitea dump staging UID does not match the controller UID" + return 1 + } + + timestamp="$(date +%Y%m%d%H%M%S)" + TRANSIENT_CONTAINER="agent99-gitea-backup-${timestamp}-$$" + OUTAGE_STARTED_AT="$(date +%s)" + PRIMARY_STOPPED=1 + set +e + timeout --signal=TERM \ + --kill-after="${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS}s" \ + "$((GITEA_STOP_TIMEOUT_SECONDS + GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS))s" \ + docker stop --time "${GITEA_STOP_TIMEOUT_SECONDS}" "${GITEA_CONTAINER}" >/dev/null + stop_status=$? + set -e + if [ "${stop_status}" -ne 0 ]; then + log_error "Gitea primary could not enter the bounded offline backup window" + start_and_verify_primary || true + return 1 + fi + + set +e + timeout --signal=TERM \ + --kill-after="${GITEA_DUMP_KILL_AFTER_SECONDS}s" \ + "${GITEA_DUMP_TIMEOUT_SECONDS}s" docker run --pull=never \ + --name "${TRANSIENT_CONTAINER}" \ + --network "${network_name}" \ + --volumes-from "${GITEA_CONTAINER}" \ + --mount "type=bind,src=${DUMP_DIR},dst=/backup-out" \ + --user "${git_uid}" \ + --workdir /backup-out \ + --cpus "${BACKUP_DOCKER_CPUS}" \ + --memory "${BACKUP_DOCKER_MEMORY}" \ + --memory-swap "${BACKUP_DOCKER_MEMORY_SWAP}" \ + --security-opt no-new-privileges:true \ + --entrypoint /usr/local/bin/gitea \ + "${image_id}" dump \ + -c /data/gitea/conf/app.ini \ + -f /backup-out/gitea-dump.zip \ + -t /tmp + dump_status=$? + set -e + + transient_cleanup_status=0 + remove_transient_container || transient_cleanup_status=$? + + if ! start_and_verify_primary; then + return 1 + fi + if [ "${transient_cleanup_status}" -ne 0 ]; then + return "${transient_cleanup_status}" + fi + if [ "${dump_status}" -ne 0 ]; then + log_error "Offline Gitea dump failed (exit=${dump_status})" + return "${dump_status}" + fi + [ -s "${DUMP_DIR}/gitea-dump.zip" ] || { + log_error "Offline Gitea dump did not produce a non-empty archive" + return 1 + } + log_success "Gitea consistent offline dump completed" } main() { @@ -91,31 +332,44 @@ main() { local tags local snapshot_id local duration + local structural_drill_performed=0 start_time=$(date +%s) trap cleanup EXIT + trap 'exit 130' INT + trap 'exit 143' TERM HUP log_info "========== 開始 Gitea 備份 ==========" - mkdir -p "${DUMP_DIR}" - - log_info "執行 Gitea dump..." - if docker exec -u git "${GITEA_CONTAINER}" gitea dump -c /data/gitea/conf/app.ini -f /tmp/gitea-dump.zip 2>&1; then - docker cp "${GITEA_CONTAINER}:/tmp/gitea-dump.zip" "${DUMP_DIR}/gitea-dump.zip" - docker exec -u git "${GITEA_CONTAINER}" rm -f /tmp/gitea-dump.zip - log_success "Gitea dump 完成" - else - duration=$(($(date +%s) - start_time)) - write_gitea_full_backup_receipt "dump_failed" "0" "${duration}" "none" || true - log_error "Gitea dump 失敗" - notify_clawbot "failed" "${SERVICE}" "Gitea dump 失敗" - exit 1 + require_commands + exec 9>"${GITEA_BACKUP_LOCK}" + if ! flock -n 9; then + log_error "Gitea backup is already running" + return 75 fi + install -d -m 700 "${DUMP_DIR}" + + log_info "執行一致性 Gitea offline dump..." + if ! offline_gitea_dump; then + duration=$(($(date +%s) - start_time)) + write_gitea_full_backup_receipt "offline_dump_failed" "0" "${duration}" "none" 0 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" 0 || true + notify_clawbot "failed" "${SERVICE}" "Gitea 一致性備份失敗；服務重啟驗證=${PRIMARY_RESTART_VERIFIED}" "${duration}" + return 1 + fi + + if ! "${GITEA_RESTORE_DRILL}" --dump-zip "${DUMP_DIR}/gitea-dump.zip"; then + duration=$(($(date +%s) - start_time)) + write_gitea_full_backup_receipt "structural_drill_failed" "0" "${duration}" "none" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" 0 || true + log_error "Gitea dump structural verifier failed" + notify_clawbot "failed" "${SERVICE}" "Gitea 備份結構驗證失敗" "${duration}" + return 1 + fi + structural_drill_performed=1 if [ ! -d "${LOCAL_REPO}/data" ]; then log_info "初始化本地 Restic 倉庫..." if ! restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}"; then duration=$(($(date +%s) - start_time)) - write_gitea_full_backup_receipt "restic_init_failed" "0" "${duration}" "none" || true + write_gitea_full_backup_receipt "restic_init_failed" "0" "${duration}" "none" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" "${structural_drill_performed}" || true log_error "Restic 初始化失敗" notify_clawbot "failed" "${SERVICE}" "Gitea Restic 初始化失敗" exit 1 @@ -127,7 +381,7 @@ main() { --password-file "${RESTIC_PASSWORD_FILE}" \ ${tags}; then duration=$(($(date +%s) - start_time)) - write_gitea_full_backup_receipt "restic_backup_failed" "0" "${duration}" "none" || true + write_gitea_full_backup_receipt "restic_backup_failed" "0" "${duration}" "none" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" "${structural_drill_performed}" || true log_error "Restic 備份失敗" notify_clawbot "failed" "${SERVICE}" "Gitea Restic 備份失敗" exit 1 @@ -142,7 +396,7 @@ main() { log_info "Offsite copy is handled by sync-offsite-backups.sh; no direct rclone sync here." duration=$(($(date +%s) - start_time)) - write_gitea_full_backup_receipt "success" "$(date +%s)" "${duration}" "${snapshot_id:-unknown}" || true + write_gitea_full_backup_receipt "success" "$(date +%s)" "${duration}" "${snapshot_id:-unknown}" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" "${structural_drill_performed}" || true log_success "========== Gitea 備份完成 (${duration}s) ==========" notify_clawbot "success" "${SERVICE}" "Gitea 備份完成" "${duration}" } diff --git a/scripts/backup/gitea-full-backup-restore-drill.sh b/scripts/backup/gitea-full-backup-restore-drill.sh index e84e1222d..39b3683df 100755 --- a/scripts/backup/gitea-full-backup-restore-drill.sh +++ b/scripts/backup/gitea-full-backup-restore-drill.sh @@ -9,9 +9,9 @@ DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}" HOST_LABEL="${AIOPS_HOST_LABEL:-110}" TEXTFILE_PATH="${AIOPS_GITEA_FULL_BACKUP_RESTORE_DRILL_TEXTFILE:-/home/wooo/node_exporter_textfiles/gitea_full_backup_restore_drill.prom}" WRITE_TEXTFILE=0 -TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-60}" +TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-300}" DRILL_SCOPE="structural_metadata_only" -REQUIRED_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments hooks_custom_assets" +REQUIRED_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments actions_logs actions_artifacts avatars hooks_custom_assets" usage() { cat <<'USAGE' @@ -88,35 +88,50 @@ required = sys.argv[2:] try: with zipfile.ZipFile(dump_zip) as archive: - names = archive.namelist() + bad_member = archive.testzip() + infos = archive.infolist() + if bad_member is not None: + print("ERROR=crc_failed") + sys.exit(1) except zipfile.BadZipFile: print("ERROR=bad_zip") sys.exit(1) -except OSError: +except (OSError, RuntimeError): print("ERROR=zip_open_failed") sys.exit(1) -lowered = [name.lower() for name in names] +members = [ + (info.filename.lower().lstrip("./"), info.file_size) + for info in infos + if not info.is_dir() +] -patterns = { - "database": ("gitea-db", "database", ".sql"), - "repositories": ("repo", "repos", "repositories", ".git"), - "app_settings": ("app.ini", "app.example.ini", "conf/"), - "lfs_objects": ("lfs",), - "package_registry": ("package", "packages"), - "attachments": ("attachment", "attachments"), - "hooks_custom_assets": ("hook", "custom", "public/"), + +def has_nonempty(*needles: str) -> bool: + return any(size > 0 and any(needle in name for needle in needles) for name, size in members) + + +data_bundle = has_nonempty("gitea-data.zip") +custom_bundle = has_nonempty("gitea-custom.zip") + +coverage = { + "database": has_nonempty("gitea-db", "database", ".sql"), + "repositories": has_nonempty("gitea-repo.zip", "gitea-repos.zip", "repos/", "repositories/", ".git/"), + "app_settings": has_nonempty("app.ini", "app.example.ini"), + "lfs_objects": data_bundle or has_nonempty("data/lfs/", "/lfs/"), + "package_registry": data_bundle or has_nonempty("data/packages/", "/packages/"), + "attachments": data_bundle or has_nonempty("data/attachments/", "/attachments/"), + "actions_logs": data_bundle or has_nonempty("data/actions_log/", "/actions_log/"), + "actions_artifacts": data_bundle or has_nonempty("data/actions_artifacts/", "/actions_artifacts/"), + "avatars": data_bundle or has_nonempty("data/avatars/", "data/repo-avatars/", "/avatars/", "/repo-avatars/"), + "hooks_custom_assets": custom_bundle or has_nonempty("custom/", "/hooks/", "public/"), } -missing = [] -for component in required: - needles = patterns.get(component, (component,)) - if not any(any(needle in name for needle in needles) for name in lowered): - missing.append(component) +missing = [component for component in required if not coverage.get(component, False)] -print(f"ENTRY_COUNT={len(names)}") +print(f"ENTRY_COUNT={len(infos)}") print(f"MISSING_COMPONENTS={','.join(missing)}") -print("OK=1" if not missing and names else "OK=0") +print("OK=1" if not missing and infos else "OK=0") PY )" drill_status=$? diff --git a/scripts/backup/tests/test_110_backup_runtime_deploy_playbook.py b/scripts/backup/tests/test_110_backup_runtime_deploy_playbook.py index 7cbfd38c7..4fbccf207 100644 --- a/scripts/backup/tests/test_110_backup_runtime_deploy_playbook.py +++ b/scripts/backup/tests/test_110_backup_runtime_deploy_playbook.py @@ -18,6 +18,9 @@ EXPECTED_FILES = { "backup-host188-products.sh", "verify-host188-products-backup.sh", "verify-host188-products-archive.py", + "backup-gitea.sh", + "gitea-full-backup-restore-drill.sh", + "backup-configs.sh", "check-backup-integrity.sh", "sync-offsite-backups.sh", "backup-offsite-readiness-gate.sh", @@ -26,7 +29,7 @@ EXPECTED_FILES = { } -def test_backup_runtime_deploy_is_fixed_to_host_110_and_fourteen_files() -> None: +def test_backup_runtime_deploy_is_fixed_to_host_110_and_seventeen_files() -> None: plays = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8")) assert len(plays) == 1 play = plays[0] diff --git a/scripts/backup/tests/test_backup_gitea_full_receipt_contract.py b/scripts/backup/tests/test_backup_gitea_full_receipt_contract.py index d2ddbdd52..ce1e4121b 100644 --- a/scripts/backup/tests/test_backup_gitea_full_receipt_contract.py +++ b/scripts/backup/tests/test_backup_gitea_full_receipt_contract.py @@ -5,6 +5,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[3] BACKUP_GITEA = ROOT / "scripts" / "backup" / "backup-gitea.sh" +BACKUP_CONFIGS = ROOT / "scripts" / "backup" / "backup-configs.sh" def test_backup_gitea_writes_full_server_component_receipts() -> None: @@ -15,7 +16,12 @@ def test_backup_gitea_writes_full_server_component_receipts() -> None: assert "awoooi_gitea_full_backup_restore_drill_performed" in text assert "awoooi_gitea_full_backup_secret_value_collected" in text assert "awoooi_gitea_full_backup_production_restore_performed" in text - assert "gitea dump -c /data/gitea/conf/app.ini" in text + assert "offline_gitea_dump" in text + assert "--volumes-from" in text + assert "--pull=never" in text + assert "docker stop --time" in text + assert "start_and_verify_primary" in text + assert "gitea-full-backup-restore-drill.sh" in text assert "GITEA_FULL_BACKUP_COMPONENTS" in text for component in [ @@ -25,6 +31,9 @@ def test_backup_gitea_writes_full_server_component_receipts() -> None: "lfs_objects", "package_registry", "attachments", + "actions_logs", + "actions_artifacts", + "avatars", "hooks_custom_assets", ]: assert component in text @@ -37,7 +46,7 @@ def test_backup_gitea_receipt_contract_stays_no_secret_no_restore() -> None: assert "awoooi_gitea_full_backup_production_restore_performed" in text assert "awoooi_gitea_full_backup_restore_drill_performed" in text assert ( - 'restore_drill_performed{host=\\"${host}\\",service=\\"${service_label}\\"} 0' + 'restore_drill_performed{host=\\"${host}\\",service=\\"${service_label}\\"} ${structural_drill_performed}' in text ) assert ( @@ -48,3 +57,34 @@ def test_backup_gitea_receipt_contract_stays_no_secret_no_restore() -> None: 'production_restore_performed{host=\\"${host}\\",service=\\"${service_label}\\"} 0' in text ) + + +def test_backup_gitea_is_bounded_and_fail_safe_around_the_offline_window() -> None: + text = BACKUP_GITEA.read_text(encoding="utf-8") + + assert 'GITEA_BACKUP_LOCK="${GITEA_BACKUP_LOCK:-/tmp/gitea-backup.lock}"' in text + assert "flock -n 9" in text + assert "trap cleanup EXIT" in text + assert 'if [ "${PRIMARY_STOPPED}" -eq 1 ]' in text + assert "wait_for_primary_health" in text + assert "http://127.0.0.1:3001/api/healthz" in text + assert '--kill-after="${GITEA_DUMP_KILL_AFTER_SECONDS}s"' in text + assert "bounded_docker_control rm -f" in text + assert 'GITEA_MAX_OUTAGE_SECONDS="${GITEA_MAX_OUTAGE_SECONDS:-600}"' in text + assert "configured_outage_bound" in text + assert "--security-opt no-new-privileges:true" in text + assert '--cpus "${BACKUP_DOCKER_CPUS}"' in text + assert '--memory "${BACKUP_DOCKER_MEMORY}"' in text + assert '--memory-swap "${BACKUP_DOCKER_MEMORY_SWAP}"' in text + assert "awoooi_gitea_full_backup_consistent_offline_window" in text + assert "awoooi_gitea_full_backup_service_restart_verified" in text + assert "awoooi_gitea_full_backup_service_outage_seconds" in text + + +def test_gitea_runner_configuration_is_in_the_encrypted_config_backup() -> None: + text = BACKUP_CONFIGS.read_text(encoding="utf-8") + + assert "/home/wooo/vibework-act-runner/config.yaml" in text + assert 'record_config_status "110-gitea-runner-config" true true "110"' in text + assert 'record_config_status "110-gitea-runner-config" true false "110"' in text + assert "/home/wooo/vibework-act-runner/data" not in text diff --git a/scripts/backup/tests/test_backup_gitea_offline_window_behavior.py b/scripts/backup/tests/test_backup_gitea_offline_window_behavior.py new file mode 100644 index 000000000..08382ef96 --- /dev/null +++ b/scripts/backup/tests/test_backup_gitea_offline_window_behavior.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +import os +import subprocess +import textwrap +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "backup" / "backup-gitea.sh" + + +def _write_executable(path: Path, source: str) -> None: + path.write_text(textwrap.dedent(source), encoding="utf-8") + path.chmod(0o755) + + +def _fake_runtime(tmp_path: Path) -> tuple[dict[str, str], Path, Path]: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + state_dir = tmp_path / "state" + state_dir.mkdir() + (state_dir / "running").write_text("true\n", encoding="utf-8") + command_log = state_dir / "commands.log" + + _write_executable( + fake_bin / "docker", + r""" + #!/usr/bin/env bash + set -euo pipefail + state="${FAKE_DOCKER_STATE:?}" + printf '%s\n' "$*" >> "$state/commands.log" + command_name="${1:-}" + shift || true + case "$command_name" in + inspect) + args="$*" + case "$args" in + *State.Running*) cat "$state/running" ;; + *NetworkSettings.Networks*) printf 'gitea_gitea\n' ;; + *'{{.Image}}'*) printf 'sha256:%064d\n' 0 ;; + *) exit 64 ;; + esac + ;; + exec) + args="$*" + if [[ "$args" == *' id -u'* ]]; then + id -u + elif [[ "$args" == *'gitea --version'* ]]; then + printf 'gitea version 1.25.5 built with test\n' + else + exit 64 + fi + ;; + stop) + printf 'false\n' > "$state/running" + if [ "${FAKE_DOCKER_STOP_FAIL:-0}" = "1" ]; then + exit 42 + fi + ;; + start) + printf 'true\n' > "$state/running" + ;; + run) + output="" + for arg in "$@"; do + case "$arg" in + type=bind,src=*,dst=/backup-out) + output="${arg#type=bind,src=}" + output="${output%,dst=/backup-out}" + ;; + esac + done + [ -n "$output" ] || exit 65 + if [ "${FAKE_DOCKER_RUN_FAIL:-0}" = "1" ]; then + exit 42 + fi + python3 - "$output/gitea-dump.zip" <<'PY' +import sys +import zipfile + +entries = { + "gitea-db.sql": "fixture\n", + "repos/wooo/awoooi.git/HEAD": "ref: refs/heads/main\n", + "custom/conf/app.ini": "[server]\n", + "data/lfs/objects/aa/bb/object": "lfs\n", + "data/packages/package.bin": "package\n", + "data/attachments/attachment.bin": "attachment\n", + "data/actions_artifacts/aa/bb/artifact.zip": "artifact\n", + "data/actions_log/aa/bb/job.log": "log\n", + "data/avatars/aa/bb/avatar.png": "avatar\n", + "custom/hooks/pre-receive": "#!/bin/sh\n", +} +with zipfile.ZipFile(sys.argv[1], "w") as archive: + for name, value in entries.items(): + archive.writestr(name, value) +PY + ;; + rm) ;; + *) exit 64 ;; + esac + """, + ) + _write_executable( + fake_bin / "curl", + r""" + #!/usr/bin/env bash + url="${!#}" + if [ "$url" = "http://127.0.0.1:3001/api/healthz" ]; then + printf '{"status":"pass"}' + else + printf 'another service' + fi + """, + ) + _write_executable( + fake_bin / "timeout", + """ + #!/usr/bin/env bash + set -euo pipefail + while [[ "${1:-}" == --* ]]; do shift; done + shift + if [ "${FAKE_DOCKER_RUN_TIMEOUT:-0}" = "1" ] && [[ " $* " == *' docker run '* ]]; then + exit 124 + fi + exec "$@" + """, + ) + _write_executable( + fake_bin / "flock", + """ + #!/usr/bin/env bash + exit 0 + """, + ) + _write_executable( + fake_bin / "restic", + r""" + #!/usr/bin/env bash + set -euo pipefail + repo="" + args=("$@") + for ((i=0; i<${#args[@]}; i++)); do + if [ "${args[$i]}" = "-r" ]; then repo="${args[$((i+1))]}"; fi + done + if [[ " $* " == *' init '* ]]; then + mkdir -p "$repo/data" "$repo/snapshots" + exit 0 + fi + if [[ " $* " == *' backup '* ]]; then + mkdir -p "$repo/data" "$repo/snapshots" + exit 0 + fi + if [[ " $* " == *' snapshots '* ]]; then + printf '[{"short_id":"abcd1234","time":"2026-07-18T00:00:00Z"}]\n' + exit 0 + fi + if [[ " $* " == *' forget '* ]]; then exit 0; fi + exit 64 + """, + ) + + backup_root = tmp_path / "backup" + (backup_root / "scripts").mkdir(parents=True) + (backup_root / "scripts" / ".restic-password").write_text("fixture\n", encoding="utf-8") + textfile = tmp_path / "gitea.prom" + environment = os.environ | { + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "BACKUP_BASE": str(backup_root), + "BACKUP_LOG_DIR": str(backup_root / "logs"), + "RESTIC_PASSWORD_FILE": str(backup_root / "scripts" / ".restic-password"), + "GITEA_FULL_BACKUP_RECEIPT_TEXTFILE": str(textfile), + "NODE_EXPORTER_TEXTFILE_DIR": str(tmp_path), + "GITEA_BACKUP_LOCK": str(tmp_path / "gitea.lock"), + "GITEA_START_TIMEOUT_SECONDS": "4", + "GITEA_DUMP_TIMEOUT_SECONDS": "30", + "GITEA_DUMP_KILL_AFTER_SECONDS": "2", + "GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS": "2", + "GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS": "1", + "GITEA_STOP_TIMEOUT_SECONDS": "2", + "FAKE_DOCKER_STATE": str(state_dir), + "BACKUP_RETENTION_MODE": "latest", + } + return environment, command_log, textfile + + +def test_offline_backup_restarts_and_verifies_primary_on_success(tmp_path: Path) -> None: + environment, command_log, textfile = _fake_runtime(tmp_path) + + result = subprocess.run( + ["bash", str(SCRIPT)], + env=environment, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + commands = command_log.read_text(encoding="utf-8") + assert commands.index("stop --time 2 gitea") < commands.index("run --pull=never") + assert commands.index("run --pull=never") < commands.index("rm -f agent99-gitea-backup-") + assert commands.index("rm -f agent99-gitea-backup-") < commands.index("start gitea") + rendered = textfile.read_text(encoding="utf-8") + assert "awoooi_gitea_full_backup_consistent_offline_window" in rendered + assert "awoooi_gitea_full_backup_service_restart_verified" in rendered + assert 'service="gitea"} 1' in rendered + + +def test_offline_backup_restarts_primary_when_dump_fails(tmp_path: Path) -> None: + environment, command_log, textfile = _fake_runtime(tmp_path) + environment["FAKE_DOCKER_RUN_FAIL"] = "1" + + result = subprocess.run( + ["bash", str(SCRIPT)], + env=environment, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode != 0 + assert (Path(environment["FAKE_DOCKER_STATE"]) / "running").read_text(encoding="utf-8").strip() == "true" + commands = command_log.read_text(encoding="utf-8") + assert "stop --time 2 gitea" in commands + assert commands.index("rm -f agent99-gitea-backup-") < commands.index("start gitea") + rendered = textfile.read_text(encoding="utf-8") + assert 'awoooi_gitea_full_backup_service_restart_verified{host="110",service="gitea"} 1' in rendered + assert 'status="offline_dump_failed"} 1' in rendered + + +def test_offline_backup_restarts_primary_when_dump_hits_hard_timeout(tmp_path: Path) -> None: + environment, command_log, textfile = _fake_runtime(tmp_path) + environment["FAKE_DOCKER_RUN_TIMEOUT"] = "1" + + result = subprocess.run( + ["bash", str(SCRIPT)], + env=environment, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode != 0 + assert (Path(environment["FAKE_DOCKER_STATE"]) / "running").read_text(encoding="utf-8").strip() == "true" + commands = command_log.read_text(encoding="utf-8") + assert commands.index("rm -f agent99-gitea-backup-") < commands.index("start gitea") + assert 'status="offline_dump_failed"} 1' in textfile.read_text(encoding="utf-8") + + +def test_offline_backup_recovers_when_stop_has_side_effect_then_fails(tmp_path: Path) -> None: + environment, command_log, textfile = _fake_runtime(tmp_path) + environment["FAKE_DOCKER_STOP_FAIL"] = "1" + + result = subprocess.run( + ["bash", str(SCRIPT)], + env=environment, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode != 0 + assert (Path(environment["FAKE_DOCKER_STATE"]) / "running").read_text(encoding="utf-8").strip() == "true" + commands = command_log.read_text(encoding="utf-8") + assert commands.index("stop --time 2 gitea") < commands.index("start gitea") + assert 'service_restart_verified{host="110",service="gitea"} 1' in textfile.read_text(encoding="utf-8") + + +def test_wrong_service_health_response_cannot_open_offline_window(tmp_path: Path) -> None: + environment, command_log, _ = _fake_runtime(tmp_path) + environment["GITEA_LOCAL_HEALTH_URL"] = "http://127.0.0.1:3000/" + + result = subprocess.run( + ["bash", str(SCRIPT)], + env=environment, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode != 0 + commands = command_log.read_text(encoding="utf-8") + assert "stop --time" not in commands + + +def test_oversized_outage_budget_fails_before_primary_stop(tmp_path: Path) -> None: + environment, command_log, _ = _fake_runtime(tmp_path) + environment["GITEA_MAX_OUTAGE_SECONDS"] = "10" + + result = subprocess.run( + ["bash", str(SCRIPT)], + env=environment, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode != 0 + assert "Configured Gitea backup outage bound exceeds 10s" in result.stdout + assert not command_log.exists() or "stop --time" not in command_log.read_text(encoding="utf-8") diff --git a/scripts/backup/tests/test_gitea_full_backup_restore_drill.py b/scripts/backup/tests/test_gitea_full_backup_restore_drill.py index 3d4f48eef..543163806 100644 --- a/scripts/backup/tests/test_gitea_full_backup_restore_drill.py +++ b/scripts/backup/tests/test_gitea_full_backup_restore_drill.py @@ -9,15 +9,25 @@ ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "gitea-full-backup-restore-drill.sh" -def _write_dump(path: Path, *, include_lfs: bool = True) -> None: +def _write_dump( + path: Path, + *, + include_lfs: bool = True, + include_actions_artifact: bool = True, + empty_critical_payloads: bool = False, +) -> None: entries = { - "gitea-db.sql": "-- redacted fixture\n", - "repos/wooo/awoooi.git/HEAD": "ref: refs/heads/main\n", + "gitea-db.sql": "" if empty_critical_payloads else "-- redacted fixture\n", + "repos/wooo/awoooi.git/HEAD": "" if empty_critical_payloads else "ref: refs/heads/main\n", "custom/conf/app.ini": "[server]\nAPP_NAME=fixture\n", "data/packages/package.bin": "package fixture\n", "data/attachments/attachment.bin": "attachment fixture\n", + "data/actions_log/aa/bb/job.log": "actions log fixture\n", + "data/avatars/aa/bb/avatar.png": "avatar fixture\n", "custom/hooks/pre-receive": "#!/bin/sh\n", } + if include_actions_artifact: + entries["data/actions_artifacts/aa/bb/artifact.zip"] = "actions artifact fixture\n" if include_lfs: entries["data/lfs/objects/aa/bb/object"] = "lfs fixture\n" with zipfile.ZipFile(path, "w") as archive: @@ -97,3 +107,64 @@ def test_gitea_full_backup_restore_drill_fails_closed_when_component_missing( rendered = textfile.read_text(encoding="utf-8") assert 'component="lfs_objects",drill_scope="structural_metadata_only"} 0' in rendered assert 'production_restore_performed{host="110",service="gitea",' in rendered + + +def test_gitea_full_backup_restore_drill_rejects_empty_database_and_repository( + tmp_path: Path, +) -> None: + dump = tmp_path / "gitea-dump.zip" + _write_dump(dump, empty_critical_payloads=True) + + result = subprocess.run( + ["bash", str(SCRIPT), "--dump-zip", str(dump)], + text=True, + capture_output=True, + ) + + assert result.returncode == 1 + assert "ERROR=component_coverage_gap" in result.stdout + + +def test_gitea_full_backup_restore_drill_requires_actions_log_and_artifact( + tmp_path: Path, +) -> None: + dump = tmp_path / "gitea-dump.zip" + textfile = tmp_path / "gitea_full_backup_restore_drill.prom" + _write_dump(dump, include_actions_artifact=False) + + result = subprocess.run( + [ + "bash", + str(SCRIPT), + "--dump-zip", + str(dump), + "--write-textfile", + "--textfile", + str(textfile), + ], + text=True, + capture_output=True, + ) + + assert result.returncode == 1 + rendered = textfile.read_text(encoding="utf-8") + assert 'component="actions_logs",drill_scope="structural_metadata_only"} 1' not in rendered + assert 'component="actions_artifacts",drill_scope="structural_metadata_only"} 0' in rendered + + +def test_gitea_full_backup_restore_drill_rejects_crc_corruption(tmp_path: Path) -> None: + dump = tmp_path / "gitea-dump.zip" + _write_dump(dump) + payload = dump.read_bytes() + fixture = b"-- redacted fixture\n" + assert fixture in payload + dump.write_bytes(payload.replace(fixture, b"X" + fixture[1:], 1)) + + result = subprocess.run( + ["bash", str(SCRIPT), "--dump-zip", str(dump)], + text=True, + capture_output=True, + ) + + assert result.returncode == 1 + assert "ERROR=crc_failed" in result.stdout