fix(backup): harden Google Drive freshness readback

ops/reboot-recovery/full-stack-backup-baseline.yml

@@ -304,7 +304,7 @@ release_gate:
   strict_dr_exit_conditions:
     - "Google Drive/rclone provider configured on 110 host-local rclone.conf; /backup/scripts/offsite.env keeps only non-secret remote/path with mode 0600"
     - "credential escrow markers fresh for restic_repository_password, offsite_provider_credentials, break_glass_admin_credentials, dns_registrar_recovery, oauth_ai_provider_recovery"
-    - "full offsite marker /backup/offsite/rclone-last-success fresh after full 13 repo sync"
+    - "full offsite marker /backup/offsite/rclone-last-success fresh after full 14 repo sync"
     - "full-stack-recovery-scorecard.sh --require-dr exits 0"
     - "recovery-scorecard-contract-check.py --expect-dr-ready exits 0 against 110 Prometheus"
     - "dr-offsite-operator-checklist.sh --require-dr exits 0 after scorecard, Prometheus recording rule, and backup alert visibility contract agree"

scripts/backup/backup-offsite-readiness-gate.sh

@@ -199,6 +199,7 @@ check_offsite_env() {
 
 check_configured() {
     load_offsite_env
+    bind_rclone_config_to_runtime_user
     if command -v rclone >/dev/null 2>&1; then
         ok "rclone command is available"
     else

scripts/backup/common.sh

@@ -63,6 +63,22 @@ export BACKUP_DOCKER_CPUS="${BACKUP_DOCKER_CPUS:-1.0}"
 export BACKUP_DOCKER_MEMORY="${BACKUP_DOCKER_MEMORY:-1g}"
 export BACKUP_DOCKER_MEMORY_SWAP="${BACKUP_DOCKER_MEMORY_SWAP:-1g}"
 
+# Cron and service managers do not always preserve HOME. Bind rclone to the
+# runtime account's config path when the operator did not provide an explicit
+# RCLONE_CONFIG, without reading or printing the credential-bearing file.
+bind_rclone_config_to_runtime_user() {
+    [ -z "${RCLONE_CONFIG:-}" ] || return 0
+    command -v getent >/dev/null 2>&1 || return 0
+
+    local runtime_home
+    local candidate
+    runtime_home="$(getent passwd "$(id -u)" 2>/dev/null | awk -F: 'NR == 1 {print $6}')"
+    candidate="${runtime_home}/.config/rclone/rclone.conf"
+    if [ -n "${runtime_home}" ] && [ -r "${candidate}" ]; then
+        export RCLONE_CONFIG="${candidate}"
+    fi
+}
+
 # -----------------------------------------------------------------------------
 # 日誌函式
 # -----------------------------------------------------------------------------

scripts/backup/sync-offsite-backups.sh

@@ -121,6 +121,8 @@ prepare_rclone() {
         return 1
     fi
 
+    bind_rclone_config_to_runtime_user
+
     if [ "${PROVIDER}" = "b2" ]; then
         if ! check_b2_config; then
             return 1

scripts/backup/tests/test_verify_offsite_full_sync_runtime.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import os
+import subprocess
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "backup" / "verify-offsite-full-sync.sh"
+
+
+def run_verifier(tmp_path: Path, mode: str) -> tuple[subprocess.CompletedProcess[str], str, Path]:
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    runtime_home = tmp_path / "runtime-home"
+    config = runtime_home / ".config" / "rclone" / "rclone.conf"
+    config.parent.mkdir(parents=True)
+    config.write_text("test fixture only\n", encoding="utf-8")
+    event_log = tmp_path / "rclone.events"
+
+    getent = bin_dir / "getent"
+    getent.write_text(
+        "#!/bin/sh\nprintf 'wooo:x:1000:1000::%s:/bin/bash\\n' \"$TEST_RUNTIME_HOME\"\n",
+        encoding="utf-8",
+    )
+    getent.chmod(0o755)
+
+    rclone = bin_dir / "rclone"
+    rclone.write_text(
+        """#!/bin/sh
+printf 'CMD=%s CONFIG=%s\n' "$*" "${RCLONE_CONFIG:-}" >> "$TEST_EVENT_LOG"
+case "$1" in
+  listremotes)
+    [ "$TEST_RCLONE_MODE" = config_missing ] && printf '%s\n' 'other:' || printf '%s\n' 'gdrive:'
+    ;;
+  lsf)
+    if [ "$TEST_RCLONE_MODE" = remote_down ]; then
+      exit 9
+    fi
+    case "$2" in
+      */snapshots) printf '%s\n' snapshot-file ;;
+      *) printf '%s\n' awoooi/ ;;
+    esac
+    ;;
+esac
+""",
+        encoding="utf-8",
+    )
+    rclone.chmod(0o755)
+
+    env = os.environ.copy()
+    env.pop("RCLONE_CONFIG", None)
+    env.update(
+        {
+            "BACKUP_BASE": str(tmp_path / "backup"),
+            "HOME": "/",
+            "OFFSITE_REPOS": "awoooi",
+            "PATH": f"{bin_dir}:{env['PATH']}",
+            "TEST_EVENT_LOG": str(event_log),
+            "TEST_RCLONE_MODE": mode,
+            "TEST_RUNTIME_HOME": str(runtime_home),
+        }
+    )
+    result = subprocess.run(
+        ["bash", str(SCRIPT), "--no-color"],
+        env=env,
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    return result, event_log.read_text(encoding="utf-8"), config
+
+
+def test_verifier_binds_rclone_config_to_runtime_account(tmp_path: Path) -> None:
+    result, events, config = run_verifier(tmp_path, "ok")
+
+    assert result.returncode == 1  # The full-sync marker is intentionally absent.
+    assert "rclone provider readback succeeded" in result.stdout
+    assert f"CONFIG={config}" in events
+
+
+def test_verifier_distinguishes_missing_config_without_remote_calls(tmp_path: Path) -> None:
+    result, events, _ = run_verifier(tmp_path, "config_missing")
+
+    assert result.returncode == 1
+    assert "rclone remote configuration unavailable" in result.stdout
+    assert " lsf " not in f" {events} "
+
+
+def test_verifier_distinguishes_provider_failure_and_stops_fanout(tmp_path: Path) -> None:
+    result, events, _ = run_verifier(tmp_path, "remote_down")
+
+    assert result.returncode == 1
+    assert "rclone provider readback failed" in result.stdout
+    assert events.count("CMD=lsf ") == 1

scripts/backup/verify-offsite-full-sync.sh

@@ -113,11 +113,16 @@ low_priority() {
     fi
 }
 
-rclone_ready() {
+rclone_configured() {
     command -v rclone >/dev/null 2>&1 || return 1
     rclone listremotes 2>/dev/null | grep -Fxq "${RCLONE_REMOTE}:"
 }
 
+rclone_remote_readable() {
+    low_priority timeout 60s rclone lsf "$(remote_root)" \
+        --dirs-only --max-depth 1 >/dev/null 2>&1
+}
+
 count_remote_snapshots() {
     local repo="$1"
     local remote_snapshots
@@ -201,6 +206,7 @@ main() {
     local ok
     local latest_only_ok=1
     local verify_ok=0
+    local remote_readable=0
     local success_marker="${OFFSITE_DIR}/${PROVIDER}-full-verify-last-success"
     local success_ts
     local success_age
@@ -225,13 +231,19 @@ main() {
     echo "WRITE_TEXTFILE=${WRITE_TEXTFILE}"
     echo
 
+    bind_rclone_config_to_runtime_user
     if [ "${PROVIDER}" != "rclone" ]; then
         printf "%sBLOCKED%s unsupported provider for remote snapshot verification: %s\n" "${red}" "${reset}" "${PROVIDER}"
         failed=1
-    elif rclone_ready; then
+    elif ! rclone_configured; then
+        printf "%sBLOCKED%s rclone remote configuration unavailable: %s:\n" "${red}" "${reset}" "${RCLONE_REMOTE}"
+        failed=1
+    elif rclone_remote_readable; then
         printf "%sOK%s rclone remote configured: %s:\n" "${green}" "${reset}" "${RCLONE_REMOTE}"
+        printf "%sOK%s rclone provider readback succeeded: %s\n" "${green}" "${reset}" "$(remote_root)"
+        remote_readable=1
     else
-        printf "%sBLOCKED%s rclone remote unavailable: %s:\n" "${red}" "${reset}" "${RCLONE_REMOTE}"
+        printf "%sBLOCKED%s rclone provider readback failed: %s\n" "${red}" "${reset}" "$(remote_root)"
         failed=1
     fi
 
@@ -245,7 +257,11 @@ main() {
     echo
     echo "== remote snapshot counts =="
     for repo in ${EXPECTED_REPOS}; do
-        count="$(count_remote_snapshots "${repo}" || true)"
+        if [ "${remote_readable}" = "1" ]; then
+            count="$(count_remote_snapshots "${repo}" || true)"
+        else
+            count=-1
+        fi
         ok=0
         if [ "${count}" = "1" ]; then
             ok=1