From d3d731be3ee6daa11256a99cee2cff4c429e13c6 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 21:55:07 +0800 Subject: [PATCH] fix(backup): harden Google Drive freshness readback --- .../full-stack-backup-baseline.yml | 2 +- .../backup/backup-offsite-readiness-gate.sh | 1 + scripts/backup/common.sh | 16 ++++ scripts/backup/sync-offsite-backups.sh | 2 + .../test_verify_offsite_full_sync_runtime.py | 95 +++++++++++++++++++ scripts/backup/verify-offsite-full-sync.sh | 24 ++++- 6 files changed, 135 insertions(+), 5 deletions(-) create mode 100644 scripts/backup/tests/test_verify_offsite_full_sync_runtime.py diff --git a/ops/reboot-recovery/full-stack-backup-baseline.yml b/ops/reboot-recovery/full-stack-backup-baseline.yml index 19984a7fd..5125f6661 100644 --- a/ops/reboot-recovery/full-stack-backup-baseline.yml +++ b/ops/reboot-recovery/full-stack-backup-baseline.yml @@ -304,7 +304,7 @@ release_gate: strict_dr_exit_conditions: - "Google Drive/rclone provider configured on 110 host-local rclone.conf; /backup/scripts/offsite.env keeps only non-secret remote/path with mode 0600" - "credential escrow markers fresh for restic_repository_password, offsite_provider_credentials, break_glass_admin_credentials, dns_registrar_recovery, oauth_ai_provider_recovery" - - "full offsite marker /backup/offsite/rclone-last-success fresh after full 13 repo sync" + - "full offsite marker /backup/offsite/rclone-last-success fresh after full 14 repo sync" - "full-stack-recovery-scorecard.sh --require-dr exits 0" - "recovery-scorecard-contract-check.py --expect-dr-ready exits 0 against 110 Prometheus" - "dr-offsite-operator-checklist.sh --require-dr exits 0 after scorecard, Prometheus recording rule, and backup alert visibility contract agree" diff --git a/scripts/backup/backup-offsite-readiness-gate.sh b/scripts/backup/backup-offsite-readiness-gate.sh index f793d9850..bec98e8ed 100755 --- a/scripts/backup/backup-offsite-readiness-gate.sh +++ b/scripts/backup/backup-offsite-readiness-gate.sh @@ -199,6 +199,7 @@ check_offsite_env() { check_configured() { load_offsite_env + bind_rclone_config_to_runtime_user if command -v rclone >/dev/null 2>&1; then ok "rclone command is available" else diff --git a/scripts/backup/common.sh b/scripts/backup/common.sh index fbeaadaf4..955d03498 100644 --- a/scripts/backup/common.sh +++ b/scripts/backup/common.sh @@ -63,6 +63,22 @@ export BACKUP_DOCKER_CPUS="${BACKUP_DOCKER_CPUS:-1.0}" export BACKUP_DOCKER_MEMORY="${BACKUP_DOCKER_MEMORY:-1g}" export BACKUP_DOCKER_MEMORY_SWAP="${BACKUP_DOCKER_MEMORY_SWAP:-1g}" +# Cron and service managers do not always preserve HOME. Bind rclone to the +# runtime account's config path when the operator did not provide an explicit +# RCLONE_CONFIG, without reading or printing the credential-bearing file. +bind_rclone_config_to_runtime_user() { + [ -z "${RCLONE_CONFIG:-}" ] || return 0 + command -v getent >/dev/null 2>&1 || return 0 + + local runtime_home + local candidate + runtime_home="$(getent passwd "$(id -u)" 2>/dev/null | awk -F: 'NR == 1 {print $6}')" + candidate="${runtime_home}/.config/rclone/rclone.conf" + if [ -n "${runtime_home}" ] && [ -r "${candidate}" ]; then + export RCLONE_CONFIG="${candidate}" + fi +} + # ----------------------------------------------------------------------------- # 日誌函式 # ----------------------------------------------------------------------------- diff --git a/scripts/backup/sync-offsite-backups.sh b/scripts/backup/sync-offsite-backups.sh index 03a582061..19c22727b 100755 --- a/scripts/backup/sync-offsite-backups.sh +++ b/scripts/backup/sync-offsite-backups.sh @@ -121,6 +121,8 @@ prepare_rclone() { return 1 fi + bind_rclone_config_to_runtime_user + if [ "${PROVIDER}" = "b2" ]; then if ! check_b2_config; then return 1 diff --git a/scripts/backup/tests/test_verify_offsite_full_sync_runtime.py b/scripts/backup/tests/test_verify_offsite_full_sync_runtime.py new file mode 100644 index 000000000..d342e8c30 --- /dev/null +++ b/scripts/backup/tests/test_verify_offsite_full_sync_runtime.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "backup" / "verify-offsite-full-sync.sh" + + +def run_verifier(tmp_path: Path, mode: str) -> tuple[subprocess.CompletedProcess[str], str, Path]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + runtime_home = tmp_path / "runtime-home" + config = runtime_home / ".config" / "rclone" / "rclone.conf" + config.parent.mkdir(parents=True) + config.write_text("test fixture only\n", encoding="utf-8") + event_log = tmp_path / "rclone.events" + + getent = bin_dir / "getent" + getent.write_text( + "#!/bin/sh\nprintf 'wooo:x:1000:1000::%s:/bin/bash\\n' \"$TEST_RUNTIME_HOME\"\n", + encoding="utf-8", + ) + getent.chmod(0o755) + + rclone = bin_dir / "rclone" + rclone.write_text( + """#!/bin/sh +printf 'CMD=%s CONFIG=%s\n' "$*" "${RCLONE_CONFIG:-}" >> "$TEST_EVENT_LOG" +case "$1" in + listremotes) + [ "$TEST_RCLONE_MODE" = config_missing ] && printf '%s\n' 'other:' || printf '%s\n' 'gdrive:' + ;; + lsf) + if [ "$TEST_RCLONE_MODE" = remote_down ]; then + exit 9 + fi + case "$2" in + */snapshots) printf '%s\n' snapshot-file ;; + *) printf '%s\n' awoooi/ ;; + esac + ;; +esac +""", + encoding="utf-8", + ) + rclone.chmod(0o755) + + env = os.environ.copy() + env.pop("RCLONE_CONFIG", None) + env.update( + { + "BACKUP_BASE": str(tmp_path / "backup"), + "HOME": "/", + "OFFSITE_REPOS": "awoooi", + "PATH": f"{bin_dir}:{env['PATH']}", + "TEST_EVENT_LOG": str(event_log), + "TEST_RCLONE_MODE": mode, + "TEST_RUNTIME_HOME": str(runtime_home), + } + ) + result = subprocess.run( + ["bash", str(SCRIPT), "--no-color"], + env=env, + text=True, + capture_output=True, + check=False, + ) + return result, event_log.read_text(encoding="utf-8"), config + + +def test_verifier_binds_rclone_config_to_runtime_account(tmp_path: Path) -> None: + result, events, config = run_verifier(tmp_path, "ok") + + assert result.returncode == 1 # The full-sync marker is intentionally absent. + assert "rclone provider readback succeeded" in result.stdout + assert f"CONFIG={config}" in events + + +def test_verifier_distinguishes_missing_config_without_remote_calls(tmp_path: Path) -> None: + result, events, _ = run_verifier(tmp_path, "config_missing") + + assert result.returncode == 1 + assert "rclone remote configuration unavailable" in result.stdout + assert " lsf " not in f" {events} " + + +def test_verifier_distinguishes_provider_failure_and_stops_fanout(tmp_path: Path) -> None: + result, events, _ = run_verifier(tmp_path, "remote_down") + + assert result.returncode == 1 + assert "rclone provider readback failed" in result.stdout + assert events.count("CMD=lsf ") == 1 diff --git a/scripts/backup/verify-offsite-full-sync.sh b/scripts/backup/verify-offsite-full-sync.sh index 4a0f60e07..9d130a4ba 100644 --- a/scripts/backup/verify-offsite-full-sync.sh +++ b/scripts/backup/verify-offsite-full-sync.sh @@ -113,11 +113,16 @@ low_priority() { fi } -rclone_ready() { +rclone_configured() { command -v rclone >/dev/null 2>&1 || return 1 rclone listremotes 2>/dev/null | grep -Fxq "${RCLONE_REMOTE}:" } +rclone_remote_readable() { + low_priority timeout 60s rclone lsf "$(remote_root)" \ + --dirs-only --max-depth 1 >/dev/null 2>&1 +} + count_remote_snapshots() { local repo="$1" local remote_snapshots @@ -201,6 +206,7 @@ main() { local ok local latest_only_ok=1 local verify_ok=0 + local remote_readable=0 local success_marker="${OFFSITE_DIR}/${PROVIDER}-full-verify-last-success" local success_ts local success_age @@ -225,13 +231,19 @@ main() { echo "WRITE_TEXTFILE=${WRITE_TEXTFILE}" echo + bind_rclone_config_to_runtime_user if [ "${PROVIDER}" != "rclone" ]; then printf "%sBLOCKED%s unsupported provider for remote snapshot verification: %s\n" "${red}" "${reset}" "${PROVIDER}" failed=1 - elif rclone_ready; then + elif ! rclone_configured; then + printf "%sBLOCKED%s rclone remote configuration unavailable: %s:\n" "${red}" "${reset}" "${RCLONE_REMOTE}" + failed=1 + elif rclone_remote_readable; then printf "%sOK%s rclone remote configured: %s:\n" "${green}" "${reset}" "${RCLONE_REMOTE}" + printf "%sOK%s rclone provider readback succeeded: %s\n" "${green}" "${reset}" "$(remote_root)" + remote_readable=1 else - printf "%sBLOCKED%s rclone remote unavailable: %s:\n" "${red}" "${reset}" "${RCLONE_REMOTE}" + printf "%sBLOCKED%s rclone provider readback failed: %s\n" "${red}" "${reset}" "$(remote_root)" failed=1 fi @@ -245,7 +257,11 @@ main() { echo echo "== remote snapshot counts ==" for repo in ${EXPECTED_REPOS}; do - count="$(count_remote_snapshots "${repo}" || true)" + if [ "${remote_readable}" = "1" ]; then + count="$(count_remote_snapshots "${repo}" || true)" + else + count=-1 + fi ok=0 if [ "${count}" = "1" ]; then ok=1