fix(backup): verify host188 recovery end to end

2026-07-18 19:11:59 +08:00
parent 420738514e
commit e614f061f7
18 changed files with 1211 additions and 54 deletions
--- a/infra/ansible/playbooks/110-backup-runtime-deploy.yml
+++ b/infra/ansible/playbooks/110-backup-runtime-deploy.yml
@@ -8,64 +8,269 @@

  vars:
    backup_runtime_root: /backup/scripts
-    backup_runtime_files:
+    backup_runtime_stage_root: /backup/.agent99-backup-runtime-stage
+    backup_health_exporter_root: /home/wooo/scripts
+    backup_health_exporter_name: backup-health-textfile-exporter.py
+    backup_runtime_stage1_files:
      - common.sh
      - backup-all.sh
+      - backup-host188-products.sh
+      - verify-host188-products-backup.sh
+      - verify-host188-products-archive.py
+    backup_runtime_stage2_files:
      - backup-awoooi.sh
      - backup-awoooi-frequent.sh
      - backup-clawbot.sh
      - backup-sentry.sh
-      - backup-host188-products.sh
+      - check-backup-integrity.sh
+      - sync-offsite-backups.sh
+      - backup-offsite-readiness-gate.sh
+      - verify-offsite-full-sync.sh
+      - enforce-latest-only-retention.sh
+    backup_runtime_files: "{{ backup_runtime_stage1_files + backup_runtime_stage2_files }}"
+    backup_runtime_fault_injection_after_stage1: false

  pre_tasks:
-    - name: Verify the fixed host boundary
+    - name: Verify the fixed host and transaction boundary
      ansible.builtin.assert:
        that:
          - inventory_hostname == "host_110"
-          - backup_runtime_files | length == 7
+          - backup_runtime_files | length == 14
          - backup_runtime_files | unique | length == backup_runtime_files | length
+          - backup_runtime_fault_injection_after_stage1 | type_debug == "bool"
        fail_msg: backup_runtime_deploy_boundary_failed

  tasks:
-    - name: Ensure the backup runtime directory exists
+    - name: Ensure fixed runtime directories exist
      ansible.builtin.file:
-        path: "{{ backup_runtime_root }}"
+        path: "{{ item.path }}"
        state: directory
-        owner: wooo
-        group: wooo
-        mode: "0755"
+        owner: "{{ item.owner }}"
+        group: "{{ item.group }}"
+        mode: "{{ item.mode }}"
+      loop:
+        - { path: "{{ backup_runtime_root }}", owner: wooo, group: wooo, mode: "0755" }
+        - { path: "{{ backup_health_exporter_root }}", owner: wooo, group: wooo, mode: "0755" }

-    - name: Validate and deploy the fixed backup runtime files
+    - name: Reset the transaction staging directory
+      ansible.builtin.file:
+        path: "{{ backup_runtime_stage_root }}"
+        state: absent
+
+    - name: Create the transaction staging directory
+      ansible.builtin.file:
+        path: "{{ backup_runtime_stage_root }}"
+        state: directory
+        owner: root
+        group: root
+        mode: "0700"
+
+    - name: Stage and shell-validate backup runtime files
      ansible.builtin.copy:
        src: "{{ playbook_dir }}/../../../scripts/backup/{{ item }}"
-        dest: "{{ backup_runtime_root }}/{{ item }}"
-        owner: wooo
-        group: wooo
-        mode: "0755"
-        backup: true
+        dest: "{{ backup_runtime_stage_root }}/{{ item }}"
+        owner: root
+        group: root
+        mode: "0700"
        force: true
        validate: "/bin/bash -n %s"
-      loop: "{{ backup_runtime_files }}"
-      register: backup_runtime_copy
+      loop: "{{ backup_runtime_files | reject('match', '.*[.]py$') | list }}"

-    - name: Read back deployed backup runtime metadata
+    - name: Stage and Python-validate backup runtime files
+      ansible.builtin.copy:
+        src: "{{ playbook_dir }}/../../../scripts/backup/{{ item }}"
+        dest: "{{ backup_runtime_stage_root }}/{{ item }}"
+        owner: root
+        group: root
+        mode: "0700"
+        force: true
+        validate: "/usr/bin/python3 -m py_compile %s"
+      loop: "{{ backup_runtime_files | select('match', '.*[.]py$') | list }}"
+
+    - name: Stage and Python-validate backup health exporter
+      ansible.builtin.copy:
+        src: "{{ playbook_dir }}/../../../scripts/ops/{{ backup_health_exporter_name }}"
+        dest: "{{ backup_runtime_stage_root }}/{{ backup_health_exporter_name }}"
+        owner: root
+        group: root
+        mode: "0700"
+        force: true
+        validate: "/usr/bin/python3 -m py_compile %s"
+
+    - name: Read back staged backup runtime metadata
      ansible.builtin.stat:
-        path: "{{ backup_runtime_root }}/{{ item }}"
+        path: "{{ backup_runtime_stage_root }}/{{ item }}"
        checksum_algorithm: sha256
      loop: "{{ backup_runtime_files }}"
-      register: backup_runtime_stat
+      register: backup_runtime_stage_stat
      changed_when: false

-    - name: Verify deployed backup runtime identity
+    - name: Verify staged backup runtime identity
      ansible.builtin.assert:
        that:
          - item.stat.exists
          - item.stat.isreg
-          - item.stat.pw_name == "wooo"
-          - item.stat.gr_name == "wooo"
-          - item.stat.mode == "0755"
          - item.stat.checksum == (lookup('ansible.builtin.file', playbook_dir + '/../../../scripts/backup/' + item.item, rstrip=false) | hash('sha256'))
-        fail_msg: "backup_runtime_identity_failed:{{ item.item }}"
-      loop: "{{ backup_runtime_stat.results }}"
+        fail_msg: "backup_runtime_stage_identity_failed:{{ item.item }}"
+      loop: "{{ backup_runtime_stage_stat.results }}"
      loop_control:
        label: "{{ item.item }}"
+
+    - name: Read current backup runtime metadata for rollback
+      ansible.builtin.stat:
+        path: "{{ backup_runtime_root }}/{{ item }}"
+      loop: "{{ backup_runtime_files }}"
+      register: backup_runtime_before_stat
+      changed_when: false
+
+    - name: Capture current backup runtime content for rollback
+      ansible.builtin.slurp:
+        path: "{{ backup_runtime_root }}/{{ item.item }}"
+      loop: "{{ backup_runtime_before_stat.results }}"
+      when: item.stat.exists
+      register: backup_runtime_before_content
+      no_log: true
+
+    - name: Read current backup exporter metadata for rollback
+      ansible.builtin.stat:
+        path: "{{ backup_health_exporter_root }}/{{ backup_health_exporter_name }}"
+      register: backup_exporter_before_stat
+      changed_when: false
+
+    - name: Capture current backup exporter content for rollback
+      ansible.builtin.slurp:
+        path: "{{ backup_health_exporter_root }}/{{ backup_health_exporter_name }}"
+      when: backup_exporter_before_stat.stat.exists
+      register: backup_exporter_before_content
+      no_log: true
+
+    - name: Promote the staged backup runtime transaction
+      block:
+        - name: Promote transaction stage one
+          ansible.builtin.copy:
+            src: "{{ backup_runtime_stage_root }}/{{ item }}"
+            dest: "{{ backup_runtime_root }}/{{ item }}"
+            remote_src: true
+            owner: wooo
+            group: wooo
+            mode: "0755"
+            backup: true
+            force: true
+          loop: "{{ backup_runtime_stage1_files }}"
+
+        - name: Exercise bounded rollback fault injection when explicitly requested
+          ansible.builtin.fail:
+            msg: backup_runtime_fault_injection_after_stage1
+          when: backup_runtime_fault_injection_after_stage1
+
+        - name: Promote transaction stage two
+          ansible.builtin.copy:
+            src: "{{ backup_runtime_stage_root }}/{{ item }}"
+            dest: "{{ backup_runtime_root }}/{{ item }}"
+            remote_src: true
+            owner: wooo
+            group: wooo
+            mode: "0755"
+            backup: true
+            force: true
+          loop: "{{ backup_runtime_stage2_files }}"
+
+        - name: Promote the staged backup health exporter
+          ansible.builtin.copy:
+            src: "{{ backup_runtime_stage_root }}/{{ backup_health_exporter_name }}"
+            dest: "{{ backup_health_exporter_root }}/{{ backup_health_exporter_name }}"
+            remote_src: true
+            owner: wooo
+            group: wooo
+            mode: "0755"
+            backup: true
+            force: true
+
+        - name: Read back promoted backup runtime metadata
+          ansible.builtin.stat:
+            path: "{{ backup_runtime_root }}/{{ item }}"
+            checksum_algorithm: sha256
+          loop: "{{ backup_runtime_files }}"
+          register: backup_runtime_after_stat
+          changed_when: false
+
+        - name: Verify promoted backup runtime identity
+          ansible.builtin.assert:
+            that:
+              - item.stat.exists
+              - item.stat.isreg
+              - item.stat.pw_name == "wooo"
+              - item.stat.gr_name == "wooo"
+              - item.stat.mode == "0755"
+              - item.stat.checksum == (lookup('ansible.builtin.file', playbook_dir + '/../../../scripts/backup/' + item.item, rstrip=false) | hash('sha256'))
+            fail_msg: "backup_runtime_identity_failed:{{ item.item }}"
+          loop: "{{ backup_runtime_after_stat.results }}"
+          loop_control:
+            label: "{{ item.item }}"
+
+        - name: Read back promoted backup exporter metadata
+          ansible.builtin.stat:
+            path: "{{ backup_health_exporter_root }}/{{ backup_health_exporter_name }}"
+            checksum_algorithm: sha256
+          register: backup_exporter_after_stat
+          changed_when: false
+
+        - name: Verify promoted backup exporter identity
+          ansible.builtin.assert:
+            that:
+              - backup_exporter_after_stat.stat.exists
+              - backup_exporter_after_stat.stat.pw_name == "wooo"
+              - backup_exporter_after_stat.stat.gr_name == "wooo"
+              - backup_exporter_after_stat.stat.mode == "0755"
+              - backup_exporter_after_stat.stat.checksum == (lookup('ansible.builtin.file', playbook_dir + '/../../../scripts/ops/' + backup_health_exporter_name, rstrip=false) | hash('sha256'))
+            fail_msg: backup_exporter_identity_failed
+      rescue:
+        - name: Restore runtime files that existed before promotion
+          ansible.builtin.copy:
+            content: "{{ item.content | b64decode }}"
+            dest: "{{ backup_runtime_root }}/{{ item.item.item }}"
+            owner: wooo
+            group: wooo
+            mode: "0755"
+            force: true
+          loop: "{{ backup_runtime_before_content.results | default([]) }}"
+          when: not (item.skipped | default(false))
+          no_log: true
+
+        - name: Remove runtime files that did not exist before promotion
+          ansible.builtin.file:
+            path: "{{ backup_runtime_root }}/{{ item.item }}"
+            state: absent
+          loop: "{{ backup_runtime_before_stat.results }}"
+          when: not item.stat.exists
+
+        - name: Restore the exporter that existed before promotion
+          ansible.builtin.copy:
+            content: "{{ backup_exporter_before_content.content | b64decode }}"
+            dest: "{{ backup_health_exporter_root }}/{{ backup_health_exporter_name }}"
+            owner: wooo
+            group: wooo
+            mode: "0755"
+            force: true
+          when: backup_exporter_before_stat.stat.exists
+          no_log: true
+
+        - name: Remove exporter created by the failed promotion
+          ansible.builtin.file:
+            path: "{{ backup_health_exporter_root }}/{{ backup_health_exporter_name }}"
+            state: absent
+          when: not backup_exporter_before_stat.stat.exists
+
+        - name: Remove failed transaction staging data
+          ansible.builtin.file:
+            path: "{{ backup_runtime_stage_root }}"
+            state: absent
+
+        - name: Fail closed after transactional rollback
+          ansible.builtin.fail:
+            msg: backup_runtime_promotion_failed_and_rolled_back
+
+    - name: Remove successful transaction staging data
+      ansible.builtin.file:
+        path: "{{ backup_runtime_stage_root }}"
+        state: absent
--- a/infra/ansible/playbooks/110-devops.yml
+++ b/infra/ansible/playbooks/110-devops.yml
@@ -239,6 +239,8 @@
        - backup-ai-artifacts.sh
        - backup-public-routes.sh
        - backup-host188-products.sh
+        - verify-host188-products-backup.sh
+        - verify-host188-products-archive.py
        - clickhouse-native-backup.sh
        - clickhouse-native-restore-drill.sh
        - clickhouse-restore-inventory.py
@@ -250,6 +252,7 @@
        - backup-offsite-readiness-gate.sh
        - offsite-escrow-evidence-report.sh
        - verify-offsite-full-sync.sh
+        - enforce-latest-only-retention.sh
      tags: backup_jobs

    - name: "Backup | 建立 SigNoz native backup/restore config 目錄"
--- a/scripts/backup/backup-host188-products.sh
+++ b/scripts/backup/backup-host188-products.sh
@@ -9,8 +9,12 @@ source "$(dirname "$0")/common.sh"
 SERVICE="host188-products"
 REMOTE_HOST="ollama@192.168.0.188"
 LOCAL_REPO="${BACKUP_BASE}/host188-products"
-DUMP_DIR="/tmp/host188-products-backup-$$"
+RESTORE_VERIFIER="$(dirname "$0")/verify-host188-products-backup.sh"
+DUMP_DIR="${BACKUP_BASE}/staging/host188-products-backup-$$"
+LOCK_FILE="/tmp/host188-products-backup.lock"
 MIN_POSTGRES_DUMP_BYTES="${MIN_POSTGRES_DUMP_BYTES:-4096}"
+MIN_FREE_BYTES="${HOST188_BACKUP_MIN_FREE_BYTES:-8589934592}"
+SSH_TIMEOUT_SECONDS="${HOST188_BACKUP_SSH_TIMEOUT_SECONDS:-1800}"
 SSH_OPTIONS=(
    -o BatchMode=yes
    -o StrictHostKeyChecking=yes
@@ -35,7 +39,7 @@ on_error() {

 require_commands() {
    local command_name
-    for command_name in restic ssh python3; do
+    for command_name in restic ssh python3 flock timeout; do
        if ! command -v "${command_name}" >/dev/null 2>&1; then
            log_error "Required command missing: ${command_name}"
            return 69
@@ -45,6 +49,27 @@ require_commands() {
        log_error "Restic password reference is unavailable"
        return 69
    fi
+    case "${MIN_FREE_BYTES}:${SSH_TIMEOUT_SECONDS}" in
+        *[!0-9:]*|:*|*:) log_error "Host188 backup numeric limits are invalid"; return 64 ;;
+    esac
+    if [ "${MIN_FREE_BYTES}" -le 0 ] || [ "${SSH_TIMEOUT_SECONDS}" -le 0 ]; then
+        log_error "Host188 backup numeric limits must be positive"
+        return 64
+    fi
+}
+
+run_remote() {
+    timeout --signal=TERM "${SSH_TIMEOUT_SECONDS}s" \
+        ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "$@"
+}
+
+require_disk_headroom() {
+    local available_bytes
+    available_bytes="$(df -Pk "${BACKUP_BASE}" 2>/dev/null | awk 'NR==2 {printf "%.0f", $4 * 1024}')"
+    if [ -z "${available_bytes}" ] || [ "${available_bytes}" -lt "${MIN_FREE_BYTES}" ]; then
+        log_error "Host188 backup staging headroom is insufficient"
+        return 75
+    fi
 }

 assert_fixed_container() {
@@ -69,7 +94,7 @@ dump_redis_container() {
        'test "$(docker inspect -f '\''{{.State.Running}}'\'' %q 2>/dev/null)" = true && docker exec %q sh -eu -c %q' \
        "${container}" "${container}" "${inner_command}"

-    if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${remote_command}" >"${temporary_file}"; then
+    if ! run_remote "${remote_command}" >"${temporary_file}"; then
        log_error "Redis RDB dump failed: ${container}"
        return 1
    fi
@@ -95,7 +120,7 @@ dump_postgres_container() {
        'test "$(docker inspect -f '\''{{.State.Running}}'\'' %q 2>/dev/null)" = true && docker exec %q sh -eu -c %q' \
        "${container}" "${container}" "${dump_inner}"

-    if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${dump_command}" >"${temporary_file}"; then
+    if ! run_remote "${dump_command}" >"${temporary_file}"; then
        log_error "PostgreSQL dump failed: ${container}"
        return 1
    fi
@@ -108,7 +133,7 @@ dump_postgres_container() {
    fi

    printf -v verify_command 'docker exec -i %q pg_restore --list >/dev/null' "${container}"
-    if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${verify_command}" <"${temporary_file}"; then
+    if ! run_remote "${verify_command}" <"${temporary_file}"; then
        log_error "PostgreSQL archive verifier failed: ${container}"
        return 1
    fi
@@ -131,7 +156,7 @@ backup_n8n_online() {
        'test "$(docker inspect -f '\''{{.State.Running}}'\'' n8n 2>/dev/null)" = true && docker exec n8n sh -eu -c %q' \
        "${inner_command}"

-    if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${remote_command}" >"${temporary_file}"; then
+    if ! run_remote "${remote_command}" >"${temporary_file}"; then
        log_error "n8n online backup failed"
        return 1
    fi
@@ -246,6 +271,9 @@ for name in (
            "credentialsDecrypted": False,
            "serviceStopped": False,
            "offsiteWritePerformed": False,
+            "recoveryReadinessClaimed": False,
+            "n8nEncryptionKeyEscrowRequired": True,
+            "n8nEncryptionKeyEscrowVerified": False,
        },
        separators=(",", ":"),
    )
@@ -269,6 +297,12 @@ main() {
    trap on_error ERR

    require_commands
+    exec 9>"${LOCK_FILE}"
+    if ! flock -n 9; then
+        log_error "Host188 product backup is already running"
+        return 75
+    fi
+    require_disk_headroom
    install -d -m 700 "${DUMP_DIR}"
    log_info "Host188 product backup started"

@@ -291,10 +325,14 @@ main() {
        --password-file "${RESTIC_PASSWORD_FILE}" \
        --tag "service:${SERVICE}" --tag "source:host188" --tag "timestamp:${timestamp}"
    snapshot_id="$(latest_snapshot_id)"
-    verify_backup "${LOCAL_REPO}" "${snapshot_id}"
+    [ -r "${RESTORE_VERIFIER}" ] || {
+        log_error "Host188 snapshot restore verifier missing"
+        return 69
+    }
+    bash "${RESTORE_VERIFIER}" --snapshot "${snapshot_id}" --data-only

    duration="$(($(date +%s) - start_time))"
-    log_success "Host188 product backup completed snapshot=${snapshot_id} duration=${duration}s"
+    log_success "Host188 product data snapshot verified snapshot=${snapshot_id} duration=${duration}s; DR escrow remains a separate gate"
    if [ "${BACKUP_NOTIFY_SUCCESS:-0}" = "1" ]; then
        notify_clawbot "success" "${SERVICE}" "Host188 product backup completed and verified" "${duration}"
    fi
--- a/scripts/backup/check-backup-integrity.sh
+++ b/scripts/backup/check-backup-integrity.sh
@@ -202,6 +202,18 @@ main() {
                fi
                continue
            fi
+            if [ "${name}" = "host188-products" ] && [ -r "${BACKUP_BASE}/scripts/verify-host188-products-backup.sh" ]; then
+                log_info "Host188 specialized isolated restore drill: PostgreSQL, Redis, n8n, manifest, and escrow"
+                if bash "${BACKUP_BASE}/scripts/verify-host188-products-backup.sh" >> "${LOG_FILE}" 2>&1; then
+                    log_success "repo ${name} specialized restore drill OK"
+                    echo "repo=${name} status=restore_drill_specialized_ok snapshots=${count} latest=${latest_ts}" >> "${LOG_FILE}"
+                else
+                    log_error "repo ${name} specialized restore drill blocked or failed"
+                    echo "repo=${name} status=restore_drill_specialized_failed snapshots=${count} latest=${latest_ts}" >> "${LOG_FILE}"
+                    failed=$((failed + 1))
+                fi
+                continue
+            fi
            local sample
            local sample_out
            sample=$(MAX_SAMPLE_BYTES="${MAX_SAMPLE_BYTES}" sample_path_for_repo "${repo}")
--- a/scripts/backup/common.sh
+++ b/scripts/backup/common.sh
@@ -8,9 +8,9 @@
 # -----------------------------------------------------------------------------
 # 配置區 (待 CEO 提供 B2 帳號後更新)
 # -----------------------------------------------------------------------------
-export BACKUP_BASE="/backup"
-export BACKUP_LOG_DIR="${BACKUP_BASE}/logs"
-export RESTIC_PASSWORD_FILE="${BACKUP_BASE}/scripts/.restic-password"
+export BACKUP_BASE="${BACKUP_BASE:-/backup}"
+export BACKUP_LOG_DIR="${BACKUP_LOG_DIR:-${BACKUP_BASE}/logs}"
+export RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-${BACKUP_BASE}/scripts/.restic-password}"

 # Backblaze B2 配置 (待填入)
 export B2_ACCOUNT_ID=""          # 待 CEO 提供
@@ -115,10 +115,35 @@ build_tags() {
 verify_backup() {
    local repo="$1"
    local snapshot_id="$2"
-    
+    local snapshot_rows snapshot_files
+
+    if ! [[ "${snapshot_id}" =~ ^[0-9a-fA-F]{8,64}$ ]]; then
+        log_error "無效 snapshot id"
+        return 64
+    fi
    log_info "驗證備份快照: ${snapshot_id}"
-    restic -r "${repo}" check --read-data-subset=1% 2>&1
-    return $?
+    snapshot_rows="$(restic -r "${repo}" snapshots "${snapshot_id}" --json \
+        --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null)" || return 1
+    if ! printf '%s' "${snapshot_rows}" | python3 -c \
+        'import json,sys; rows=json.load(sys.stdin); raise SystemExit(0 if len(rows) == 1 else 1)'; then
+        log_error "指定 snapshot 無法唯一讀回: ${snapshot_id}"
+        return 1
+    fi
+    snapshot_files="$(restic -r "${repo}" ls "${snapshot_id}" --json \
+        --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \
+        python3 -c 'import json,sys
+count=0
+for line in sys.stdin:
+    try: row=json.loads(line)
+    except json.JSONDecodeError: continue
+    if row.get("type") == "file": count += 1
+print(count)')" || return 1
+    if [ "${snapshot_files}" -le 0 ]; then
+        log_error "指定 snapshot 不含可還原檔案: ${snapshot_id}"
+        return 1
+    fi
+    restic -r "${repo}" check --read-data-subset=1% \
+        --password-file "${RESTIC_PASSWORD_FILE}" 2>&1
 }

 # -----------------------------------------------------------------------------
@@ -126,13 +151,31 @@ verify_backup() {
 # -----------------------------------------------------------------------------
 cleanup_old_backups() {
    local repo="$1"
-    
+    local mode="${BACKUP_RETENTION_MODE:-gfs}"
+    local keep_last="${BACKUP_KEEP_LAST:-1}"
+
+    if [ "${mode}" = "latest" ]; then
+        case "${keep_last}" in
+            ''|*[!0-9]*|0) log_error "BACKUP_KEEP_LAST 必須是正整數"; return 64 ;;
+        esac
+        log_info "執行 latest-only 清理策略 keep-last=${keep_last}"
+        restic -r "${repo}" forget \
+            --keep-last "${keep_last}" \
+            --prune \
+            --password-file "${RESTIC_PASSWORD_FILE}" 2>&1
+        return $?
+    fi
+    if [ "${mode}" != "gfs" ]; then
+        log_error "未知 BACKUP_RETENTION_MODE: ${mode}"
+        return 64
+    fi
    log_info "執行 GFS 清理策略"
    restic -r "${repo}" forget \
-        --keep-daily ${KEEP_DAILY} \
-        --keep-weekly ${KEEP_WEEKLY} \
-        --keep-monthly ${KEEP_MONTHLY} \
-        --prune 2>&1
+        --keep-daily "${KEEP_DAILY}" \
+        --keep-weekly "${KEEP_WEEKLY}" \
+        --keep-monthly "${KEEP_MONTHLY}" \
+        --prune \
+        --password-file "${RESTIC_PASSWORD_FILE}" 2>&1
 }

 # -----------------------------------------------------------------------------
--- a/scripts/backup/enforce-latest-only-retention.sh
+++ b/scripts/backup/enforce-latest-only-retention.sh
@@ -13,6 +13,7 @@ source "$(dirname "$0")/common.sh"

 EXPECTED_REPOS_DEFAULT="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes host188-products"
 REPOS="${BACKUP_RETENTION_REPOS:-${EXPECTED_REPOS_DEFAULT}}"
+KEEP_LAST="${BACKUP_KEEP_LAST:-1}"

 main() {
    local failed=0
@@ -26,7 +27,8 @@ main() {
        fi

        log_info "Enforce latest-only retention: ${name}"
-        if ! BACKUP_RETENTION_MODE=latest cleanup_old_backups "${repo}"; then
+        if ! BACKUP_RETENTION_MODE=latest BACKUP_KEEP_LAST="${KEEP_LAST}" \
+            cleanup_old_backups "${repo}"; then
            failed=$((failed + 1))
        fi
    done
--- a/scripts/backup/sync-offsite-backups.sh
+++ b/scripts/backup/sync-offsite-backups.sh
@@ -19,7 +19,7 @@

 set -euo pipefail

-source "$(dirname "$0")/common.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"

 SERVICE="offsite-backup"
 MODE="status"
@@ -183,7 +183,7 @@ backup_disk_used_pct() {
 active_backup_processes() {
    ps -eo pid=,args= | awk -v self="$$" '
        $1 == self { next }
-        /\/backup\/scripts\/backup-(all|awoooi|awoooi-frequent|gitea|harbor|momo|langfuse|monitoring|signoz|open-webui|clawbot|sentry|ai-artifacts|public-routes|configs)\.sh/ {
+        /\/backup\/scripts\/backup-(all|awoooi|awoooi-frequent|gitea|harbor|momo|langfuse|monitoring|signoz|open-webui|clawbot|sentry|ai-artifacts|public-routes|configs|host188-products)\.sh/ {
            print
        }
    '
@@ -411,4 +411,6 @@ main() {
    return "${failed}"
 }

-main "$@"
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+    main "$@"
+fi
--- a/scripts/backup/tests/test_110_backup_runtime_deploy_playbook.py
+++ b/scripts/backup/tests/test_110_backup_runtime_deploy_playbook.py
@@ -16,10 +16,17 @@ EXPECTED_FILES = {
    "backup-clawbot.sh",
    "backup-sentry.sh",
    "backup-host188-products.sh",
+    "verify-host188-products-backup.sh",
+    "verify-host188-products-archive.py",
+    "check-backup-integrity.sh",
+    "sync-offsite-backups.sh",
+    "backup-offsite-readiness-gate.sh",
+    "verify-offsite-full-sync.sh",
+    "enforce-latest-only-retention.sh",
 }


-def test_backup_runtime_deploy_is_fixed_to_host_110_and_seven_files() -> None:
+def test_backup_runtime_deploy_is_fixed_to_host_110_and_fourteen_files() -> None:
    plays = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8"))
    assert len(plays) == 1
    play = plays[0]
@@ -30,7 +37,11 @@ def test_backup_runtime_deploy_is_fixed_to_host_110_and_seven_files() -> None:
    assert play["serial"] == 1
    assert play["any_errors_fatal"] is True
    assert play["vars"]["backup_runtime_root"] == "/backup/scripts"
-    assert set(play["vars"]["backup_runtime_files"]) == EXPECTED_FILES
+    stage1 = set(play["vars"]["backup_runtime_stage1_files"])
+    stage2 = set(play["vars"]["backup_runtime_stage2_files"])
+    assert stage1 | stage2 == EXPECTED_FILES
+    assert not stage1 & stage2
+    assert play["vars"]["backup_runtime_fault_injection_after_stage1"] is False


 def test_backup_runtime_deploy_has_validation_rollback_and_identity_readback() -> None:
@@ -42,6 +53,13 @@ def test_backup_runtime_deploy_has_validation_rollback_and_identity_readback() -
    assert "checksum_algorithm: sha256" in source
    assert "rstrip=false" in source
    assert "backup_runtime_identity_failed" in source
+    assert "backup_runtime_stage_identity_failed" in source
+    assert "backup_runtime_fault_injection_after_stage1" in source
+    assert "backup_runtime_promotion_failed_and_rolled_back" in source
+    assert "rescue:" in source
+    assert "ansible.builtin.slurp:" in source
+    assert "remote_src: true" in source
+    assert "backup-health-textfile-exporter.py" in source
    assert "changed_when: false" in source

    for forbidden in (
@@ -55,3 +73,25 @@ def test_backup_runtime_deploy_has_validation_rollback_and_identity_readback() -
        "kubectl",
    ):
        assert forbidden not in source
+
+
+def test_fault_injection_occurs_mid_promotion_and_rescue_restores_prior_state() -> None:
+    play = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8"))[0]
+    transaction = next(
+        task
+        for task in play["tasks"]
+        if task["name"] == "Promote the staged backup runtime transaction"
+    )
+    block_names = [task["name"] for task in transaction["block"]]
+    rescue_names = [task["name"] for task in transaction["rescue"]]
+
+    assert block_names.index("Promote transaction stage one") < block_names.index(
+        "Exercise bounded rollback fault injection when explicitly requested"
+    )
+    assert block_names.index(
+        "Exercise bounded rollback fault injection when explicitly requested"
+    ) < block_names.index("Promote transaction stage two")
+    assert "Restore runtime files that existed before promotion" in rescue_names
+    assert "Remove runtime files that did not exist before promotion" in rescue_names
+    assert "Restore the exporter that existed before promotion" in rescue_names
+    assert rescue_names[-1] == "Fail closed after transactional rollback"
--- a/scripts/backup/tests/test_backup_common_retention_contract.py
+++ b/scripts/backup/tests/test_backup_common_retention_contract.py
@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import os
+import subprocess
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+COMMON = ROOT / "scripts" / "backup" / "common.sh"
+
+
+def test_latest_only_retention_uses_keep_last_one_and_explicit_password(tmp_path: Path) -> None:
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    event_log = tmp_path / "restic.args"
+    restic = bin_dir / "restic"
+    restic.write_text(
+        '#!/bin/sh\nprintf "%s\\n" "$*" >> "$TEST_EVENT_LOG"\n',
+        encoding="utf-8",
+    )
+    restic.chmod(0o755)
+    backup_base = tmp_path / "backup"
+    password_file = backup_base / "scripts" / ".restic-password"
+    password_file.parent.mkdir(parents=True)
+    password_file.write_text("test-only\n", encoding="utf-8")
+
+    env = os.environ.copy()
+    env.update(
+        {
+            "BACKUP_BASE": str(backup_base),
+            "RESTIC_PASSWORD_FILE": str(password_file),
+            "PATH": f"{bin_dir}:{env['PATH']}",
+            "TEST_EVENT_LOG": str(event_log),
+        }
+    )
+    subprocess.run(
+        [
+            "bash",
+            "-c",
+            f'source "{COMMON}"; BACKUP_RETENTION_MODE=latest '
+            'BACKUP_KEEP_LAST=1 cleanup_old_backups "$BACKUP_BASE/demo"',
+        ],
+        env=env,
+        check=True,
+        text=True,
+        capture_output=True,
+    )
+
+    args = event_log.read_text(encoding="utf-8")
+    assert "forget --keep-last 1 --prune --password-file" in args
+
+
+def test_retention_rejects_invalid_keep_last_before_restic(tmp_path: Path) -> None:
+    backup_base = tmp_path / "backup"
+    env = os.environ.copy()
+    env["BACKUP_BASE"] = str(backup_base)
+    result = subprocess.run(
+        [
+            "bash",
+            "-c",
+            f'source "{COMMON}"; BACKUP_RETENTION_MODE=latest '
+            'BACKUP_KEEP_LAST=0 cleanup_old_backups "$BACKUP_BASE/demo"',
+        ],
+        env=env,
+        text=True,
+        capture_output=True,
+    )
+
+    assert result.returncode == 64
+    assert "BACKUP_KEEP_LAST" in result.stdout
+
+
+def test_verify_backup_reads_the_requested_snapshot_before_repository_check(
+    tmp_path: Path,
+) -> None:
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    event_log = tmp_path / "restic.args"
+    restic = bin_dir / "restic"
+    restic.write_text(
+        """#!/bin/sh
+printf '%s\n' "$*" >> "$TEST_EVENT_LOG"
+case " $* " in
+  *" snapshots "*) printf '%s\n' '[{"short_id":"abcdef12"}]' ;;
+  *" ls "*) printf '%s\n' '{"type":"file","path":"/backup/data.dump"}' ;;
+  *" check "*) ;;
+esac
+""",
+        encoding="utf-8",
+    )
+    restic.chmod(0o755)
+    backup_base = tmp_path / "backup"
+    password_file = backup_base / "scripts" / ".restic-password"
+    password_file.parent.mkdir(parents=True)
+    password_file.write_text("test-only\n", encoding="utf-8")
+    env = os.environ.copy()
+    env.update(
+        {
+            "BACKUP_BASE": str(backup_base),
+            "RESTIC_PASSWORD_FILE": str(password_file),
+            "PATH": f"{bin_dir}:{env['PATH']}",
+            "TEST_EVENT_LOG": str(event_log),
+        }
+    )
+
+    subprocess.run(
+        ["bash", "-c", f'source "{COMMON}"; verify_backup /repo abcdef12'],
+        env=env,
+        check=True,
+        text=True,
+        capture_output=True,
+    )
+
+    calls = event_log.read_text(encoding="utf-8")
+    assert "snapshots abcdef12 --json" in calls
+    assert "ls abcdef12 --json" in calls
+    assert "check --read-data-subset=1%" in calls
--- a/scripts/backup/tests/test_backup_host188_products_contract.py
+++ b/scripts/backup/tests/test_backup_host188_products_contract.py
@@ -47,7 +47,8 @@ def test_host188_product_backup_has_fixed_sources_and_online_verifiers() -> None
        "n8n export:workflow --backup",
        "n8n export:credentials --backup",
        'LOCAL_REPO="${BACKUP_BASE}/host188-products"',
-        "verify_backup",
+        "verify-host188-products-backup.sh",
+        '--snapshot "${snapshot_id}" --data-only',
    ):
        assert expected in source

@@ -63,6 +64,13 @@ def test_host188_product_backup_preserves_secret_and_runtime_boundaries() -> Non
    assert "REDISCLI_AUTH" in source
    assert 'if [ "${BACKUP_NOTIFY_SUCCESS:-0}" = "1" ]' in source
    assert "handle.read(1024 * 1024)" in source
+    assert '"recoveryReadinessClaimed": False' in source
+    assert '"n8nEncryptionKeyEscrowRequired": True' in source
+    assert 'DUMP_DIR="${BACKUP_BASE}/staging/' in source
+    assert "flock -n 9" in source
+    assert "HOST188_BACKUP_MIN_FREE_BYTES" in source
+    assert "HOST188_BACKUP_SSH_TIMEOUT_SECONDS" in source
+    assert "timeout --signal=TERM" in source

    for forbidden in (
        "--decrypted",
@@ -89,7 +97,9 @@ def test_host188_product_backup_is_wired_into_health_integrity_and_offsite() ->

    exporter = EXPORTER.read_text(encoding="utf-8")
    assert '"backup-host188-products.sh"' in exporter
-    assert '("host188_products", "/backup/host188-products", 48)' in exporter
+    assert '"verify-host188-products-backup.sh"' in exporter
+    assert '"verify-host188-products-archive.py"' in exporter
+    assert '("host188_products", "/backup/host188-products", 30)' in exporter

    agent99 = AGENT99.read_text(encoding="utf-8")
    assert 'name = "host188-products"' in agent99
@@ -99,3 +109,37 @@ def test_host188_product_backup_is_wired_into_health_integrity_and_offsite() ->
    for path in (FULL_PLAYBOOK, BOUNDED_PLAYBOOK):
        assert "backup-host188-products.sh" in path.read_text(encoding="utf-8")
    assert "backup-host188-products.sh" in REBOOT_READINESS.read_text(encoding="utf-8")
+
+    offsite_sync = OFFSITE_SCRIPTS[0].read_text(encoding="utf-8")
+    assert "configs|host188-products" in offsite_sync
+
+
+def test_host188_specialized_restore_drill_is_snapshot_specific_and_fail_closed() -> None:
+    verifier = (
+        ROOT / "scripts" / "backup" / "verify-host188-products-backup.sh"
+    ).read_text(encoding="utf-8")
+
+    for expected in (
+        'verify_backup "${REPO}" "${SNAPSHOT_ID}"',
+        'restic -r "${REPO}" dump "${SNAPSHOT_ID}"',
+        "verify-host188-products-archive.py",
+        "pg_restore --list",
+        "redis-check-rdb",
+        "n8n_encryption_key.last_verified",
+        "n8n_encryption_key_escrow_missing_or_stale",
+        "HOST188_RESTORE_DRILL_DATA_VERIFIED=1",
+        "HOST188_RESTORE_DRILL_RECOVERY_READY=0",
+    ):
+        assert expected in verifier
+    for forbidden in ("psql ", "redis-cli FLUSH", "docker restart", "docker stop"):
+        assert forbidden not in verifier
+
+    archive_validator = (
+        ROOT / "scripts" / "backup" / "verify-host188-products-archive.py"
+    ).read_text(encoding="utf-8")
+    assert "handle.read(1024 * 1024)" in archive_validator
+    assert ".read_bytes()[:5]" not in archive_validator
+
+    integrity = INTEGRITY.read_text(encoding="utf-8")
+    assert 'name}" = "host188-products"' in integrity
+    assert "verify-host188-products-backup.sh" in integrity
--- a/scripts/backup/tests/test_sync_offsite_active_backup_guard.py
+++ b/scripts/backup/tests/test_sync_offsite_active_backup_guard.py
@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import os
+import subprocess
+import time
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "backup" / "sync-offsite-backups.sh"
+
+
+@pytest.mark.parametrize("name", ["backup-host188-products.sh", "backup-all.sh"])
+def test_active_backup_guard_detects_direct_and_aggregate_runs(
+    tmp_path: Path, name: str
+) -> None:
+    process = subprocess.Popen(
+        ["bash", "-c", f"exec -a /backup/scripts/{name} sleep 30"],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    try:
+        time.sleep(0.1)
+        env = os.environ.copy()
+        env["BACKUP_BASE"] = str(tmp_path / "backup")
+        result = subprocess.run(
+            ["bash", "-c", f'source "{SCRIPT}"; active_backup_processes'],
+            env=env,
+            text=True,
+            capture_output=True,
+            check=True,
+        )
+        assert f"/backup/scripts/{name}" in result.stdout
+    finally:
+        process.terminate()
+        process.wait(timeout=5)
--- a/scripts/backup/tests/test_verify_host188_products_archive.py
+++ b/scripts/backup/tests/test_verify_host188_products_archive.py
@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import json
+import sqlite3
+import tarfile
+from pathlib import Path
+from types import ModuleType
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "backup" / "verify-host188-products-archive.py"
+
+
+def load_module() -> ModuleType:
+    spec = importlib.util.spec_from_file_location("host188_archive_verifier", SCRIPT)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def build_restore(root: Path) -> None:
+    timestamp = "20260718_120000"
+    vibe = root / f"vibework-postgres-{timestamp}.dump"
+    fifa = root / f"fifa-postgres-{timestamp}.dump"
+    redis = root / f"fifa-redis-{timestamp}.rdb"
+    n8n = root / f"n8n-online-{timestamp}.tar.gz"
+    validation = root / "n8n-validation.json"
+    vibe.write_bytes(b"PGDMP" + b"v" * 32)
+    fifa.write_bytes(b"PGDMP" + b"f" * 32)
+    redis.write_bytes(b"REDIS0011" + b"r" * 32)
+
+    n8n_root = root / "n8n-source"
+    (n8n_root / "workflows").mkdir(parents=True)
+    (n8n_root / "credentials").mkdir()
+    (n8n_root / "workflows" / "one.json").write_text('{"id":"one"}\n')
+    (n8n_root / "credentials" / "one.json").write_text('{"id":"encrypted"}\n')
+    with sqlite3.connect(n8n_root / "database.sqlite") as connection:
+        connection.execute("CREATE TABLE workflow_entity (id TEXT PRIMARY KEY)")
+        connection.execute("INSERT INTO workflow_entity VALUES ('one')")
+    with tarfile.open(n8n, "w:gz") as archive:
+        for path in sorted(n8n_root.rglob("*")):
+            archive.add(path, arcname=path.relative_to(n8n_root))
+    validation.write_text(
+        json.dumps(
+            {
+                "sqliteIntegrity": "ok",
+                "workflowFiles": 1,
+                "credentialFiles": 1,
+                "credentialsDecrypted": False,
+            }
+        )
+    )
+    for path in sorted(n8n_root.rglob("*"), reverse=True):
+        if path.is_file():
+            path.unlink()
+        else:
+            path.rmdir()
+    n8n_root.rmdir()
+
+    assets = []
+    for path in (vibe, fifa, redis, n8n, validation):
+        assets.append(
+            {
+                "name": path.name,
+                "sizeBytes": path.stat().st_size,
+                "sha256": sha256(path),
+            }
+        )
+    (root / "backup-manifest.json").write_text(
+        json.dumps(
+            {
+                "schemaVersion": "host188_product_backup_manifest_v1",
+                "sourceHost": "192.168.0.188",
+                "controllerHost": "192.168.0.110",
+                "assets": assets,
+                "credentialsDecrypted": False,
+                "serviceStopped": False,
+                "offsiteWritePerformed": False,
+                "recoveryReadinessClaimed": False,
+                "n8nEncryptionKeyEscrowRequired": True,
+            }
+        )
+    )
+
+
+def test_validates_all_restored_host188_assets(tmp_path: Path) -> None:
+    module = load_module()
+    build_restore(tmp_path)
+
+    receipt = module.validate_restore(tmp_path)
+
+    assert receipt["dataVerified"] is True
+    assert receipt["assetCount"] == 5
+    assert receipt["postgresArchiveCount"] == 2
+    assert receipt["redisArchiveCount"] == 1
+    assert receipt["n8nSqliteIntegrity"] == "ok"
+    assert receipt["recoveryReadinessClaimed"] is False
+
+
+def test_rejects_manifest_hash_mismatch(tmp_path: Path) -> None:
+    module = load_module()
+    build_restore(tmp_path)
+    next(tmp_path.glob("fifa-redis-*.rdb")).write_bytes(b"REDIS0011tampered")
+
+    with pytest.raises(ValueError, match="asset_(size|hash)_mismatch"):
+        module.validate_restore(tmp_path)
--- a/scripts/backup/tests/test_verify_host188_products_backup_shell.py
+++ b/scripts/backup/tests/test_verify_host188_products_backup_shell.py
@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import importlib.util
+import os
+import subprocess
+import time
+from pathlib import Path
+from types import ModuleType
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "backup" / "verify-host188-products-backup.sh"
+FIXTURE_MODULE = ROOT / "scripts" / "backup" / "tests" / "test_verify_host188_products_archive.py"
+
+
+def load_fixture_module() -> ModuleType:
+    spec = importlib.util.spec_from_file_location("host188_restore_fixture", FIXTURE_MODULE)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def prepare_runtime(tmp_path: Path) -> tuple[dict[str, str], Path]:
+    restored = tmp_path / "restored"
+    restored.mkdir()
+    load_fixture_module().build_restore(restored)
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    restic = bin_dir / "restic"
+    restic.write_text(
+        """#!/usr/bin/env python3
+import json
+import os
+import sys
+from pathlib import Path
+
+args = sys.argv[1:]
+root = Path(os.environ["FAKE_RESTORE_ROOT"])
+prefix = "/tmp/host188-products-backup-test"
+if "snapshots" in args:
+    print(json.dumps([{"id": "abcdef1234567890", "short_id": "abcdef12"}]))
+elif "ls" in args:
+    print(json.dumps({"struct_type": "snapshot", "id": "abcdef1234567890"}))
+    for path in sorted(root.iterdir()):
+        if path.is_file():
+            print(json.dumps({"type": "file", "path": f"{prefix}/{path.name}"}))
+elif "dump" in args:
+    selected = next((root / Path(value).name for value in args if (root / Path(value).name).is_file()), None)
+    if selected is None:
+        raise SystemExit(2)
+    sys.stdout.buffer.write(selected.read_bytes())
+elif "check" in args:
+    pass
+else:
+    raise SystemExit(2)
+""",
+        encoding="utf-8",
+    )
+    restic.chmod(0o755)
+    ssh = bin_dir / "ssh"
+    ssh.write_text("#!/bin/sh\ncat >/dev/null\n", encoding="utf-8")
+    ssh.chmod(0o755)
+    timeout = bin_dir / "timeout"
+    timeout.write_text(
+        '#!/bin/sh\ncase "$1" in --*) shift ;; esac\nshift\nexec "$@"\n',
+        encoding="utf-8",
+    )
+    timeout.chmod(0o755)
+    backup_base = tmp_path / "backup"
+    password = backup_base / "scripts" / ".restic-password"
+    password.parent.mkdir(parents=True)
+    password.write_text("test-only\n", encoding="utf-8")
+    env = os.environ.copy()
+    env.update(
+        {
+            "BACKUP_BASE": str(backup_base),
+            "BACKUP_LOG_DIR": str(backup_base / "logs"),
+            "RESTIC_PASSWORD_FILE": str(password),
+            "FAKE_RESTORE_ROOT": str(restored),
+            "HOST188_RESTORE_MIN_FREE_BYTES": "1",
+            "PATH": f"{bin_dir}:{env['PATH']}",
+        }
+    )
+    return env, backup_base
+
+
+def run_verifier(env: dict[str, str], *args: str) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["bash", str(SCRIPT), "--snapshot", "abcdef12", *args],
+        env=env,
+        text=True,
+        capture_output=True,
+    )
+
+
+def test_data_only_snapshot_restore_replay_passes_without_recovery_claim(tmp_path: Path) -> None:
+    env, _ = prepare_runtime(tmp_path)
+
+    result = run_verifier(env, "--data-only")
+
+    assert result.returncode == 0, result.stderr
+    assert "HOST188_RESTORE_DRILL_DATA_VERIFIED=1" in result.stdout
+    assert "HOST188_RESTORE_DRILL_RECOVERY_READY=0" in result.stdout
+    assert "HOST188_RESTORE_DRILL_SCOPE=data_only" in result.stdout
+
+
+def test_full_restore_replay_blocks_until_matching_escrow_marker(tmp_path: Path) -> None:
+    env, backup_base = prepare_runtime(tmp_path)
+
+    blocked = run_verifier(env)
+    assert blocked.returncode == 78
+    assert "n8n_encryption_key_escrow_missing_or_stale" in blocked.stdout
+
+    marker = backup_base / "escrow-evidence" / "n8n_encryption_key.last_verified"
+    marker.parent.mkdir(parents=True)
+    marker.write_text(
+        f"timestamp={int(time.time())}\n"
+        "item=n8n_encryption_key\n"
+        "evidence_id=TEST-NON-SECRET-EVIDENCE\n",
+        encoding="utf-8",
+    )
+    ready = run_verifier(env)
+    assert ready.returncode == 0, ready.stderr
+    assert "HOST188_RESTORE_DRILL_RECOVERY_READY=1" in ready.stdout
--- a/scripts/backup/verify-host188-products-archive.py
+++ b/scripts/backup/verify-host188-products-archive.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""Validate an isolated Host188 product backup restore without data writes."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sqlite3
+import tarfile
+import tempfile
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+
+EXPECTED_PREFIXES = {
+    "vibework-postgres-": ".dump",
+    "fifa-postgres-": ".dump",
+    "fifa-redis-": ".rdb",
+    "n8n-online-": ".tar.gz",
+}
+FIXED_ASSETS = {"n8n-validation.json"}
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def read_magic(path: Path) -> bytes:
+    with path.open("rb") as handle:
+        return handle.read(5)
+
+
+def _single_prefixed_asset(names: set[str], prefix: str, suffix: str) -> str:
+    matches = sorted(name for name in names if name.startswith(prefix) and name.endswith(suffix))
+    if len(matches) != 1:
+        raise ValueError(f"asset_cardinality_invalid:{prefix}")
+    return matches[0]
+
+
+def _extract_n8n_archive(archive_path: Path, target: Path) -> None:
+    with tarfile.open(archive_path, "r:gz") as archive:
+        members = archive.getmembers()
+        for member in members:
+            path = PurePosixPath(member.name)
+            if path.is_absolute() or ".." in path.parts:
+                raise ValueError("n8n_archive_unsafe_path")
+            if not (member.isfile() or member.isdir()):
+                raise ValueError("n8n_archive_unsafe_member")
+        archive.extractall(target, members=members)
+
+
+def validate_restore(root: Path) -> dict[str, Any]:
+    manifest_path = root / "backup-manifest.json"
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    if manifest.get("schemaVersion") != "host188_product_backup_manifest_v1":
+        raise ValueError("manifest_schema_invalid")
+    if manifest.get("sourceHost") != "192.168.0.188":
+        raise ValueError("manifest_source_invalid")
+    if manifest.get("controllerHost") != "192.168.0.110":
+        raise ValueError("manifest_controller_invalid")
+    if manifest.get("credentialsDecrypted") is not False:
+        raise ValueError("credential_boundary_invalid")
+    if manifest.get("serviceStopped") is not False:
+        raise ValueError("service_stop_boundary_invalid")
+    if manifest.get("offsiteWritePerformed") is not False:
+        raise ValueError("offsite_boundary_invalid")
+    if manifest.get("recoveryReadinessClaimed") is not False:
+        raise ValueError("recovery_readiness_false_green")
+    if manifest.get("n8nEncryptionKeyEscrowRequired") is not True:
+        raise ValueError("n8n_escrow_requirement_missing")
+
+    assets = manifest.get("assets")
+    if not isinstance(assets, list):
+        raise ValueError("manifest_assets_invalid")
+    by_name: dict[str, dict[str, Any]] = {}
+    for item in assets:
+        if not isinstance(item, dict):
+            raise ValueError("manifest_asset_invalid")
+        name = str(item.get("name") or "")
+        if not name or name in by_name or Path(name).name != name:
+            raise ValueError("manifest_asset_name_invalid")
+        by_name[name] = item
+
+    names = set(by_name)
+    if not FIXED_ASSETS.issubset(names):
+        raise ValueError("fixed_asset_missing")
+    selected = {
+        prefix: _single_prefixed_asset(names, prefix, suffix)
+        for prefix, suffix in EXPECTED_PREFIXES.items()
+    }
+    expected_names = set(selected.values()) | FIXED_ASSETS
+    if names != expected_names:
+        raise ValueError("unexpected_manifest_asset")
+
+    for name, item in by_name.items():
+        path = root / name
+        if not path.is_file():
+            raise ValueError(f"asset_missing:{name}")
+        if path.stat().st_size != int(item.get("sizeBytes") or -1):
+            raise ValueError(f"asset_size_mismatch:{name}")
+        if sha256_file(path) != str(item.get("sha256") or ""):
+            raise ValueError(f"asset_hash_mismatch:{name}")
+
+    for prefix in ("vibework-postgres-", "fifa-postgres-"):
+        if read_magic(root / selected[prefix]) != b"PGDMP":
+            raise ValueError(f"postgres_magic_invalid:{prefix}")
+    if read_magic(root / selected["fifa-redis-"]) != b"REDIS":
+        raise ValueError("redis_magic_invalid")
+
+    validation = json.loads((root / "n8n-validation.json").read_text(encoding="utf-8"))
+    if validation.get("sqliteIntegrity") != "ok":
+        raise ValueError("n8n_prior_validation_invalid")
+    if validation.get("credentialsDecrypted") is not False:
+        raise ValueError("n8n_prior_credential_boundary_invalid")
+
+    with tempfile.TemporaryDirectory(
+        prefix="host188-n8n-restore-", dir=root.parent
+    ) as temp_dir:
+        extracted = Path(temp_dir)
+        _extract_n8n_archive(root / selected["n8n-online-"], extracted)
+        database = extracted / "database.sqlite"
+        with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection:
+            integrity = connection.execute("PRAGMA integrity_check").fetchone()[0]
+        if integrity != "ok":
+            raise ValueError("n8n_sqlite_integrity_invalid")
+        counts: dict[str, int] = {}
+        for category in ("workflows", "credentials"):
+            directory = extracted / category
+            rows = sorted(directory.rglob("*.json")) if directory.is_dir() else []
+            for path in rows:
+                json.loads(path.read_text(encoding="utf-8"))
+            counts[category] = len(rows)
+        if counts["workflows"] != int(validation.get("workflowFiles") or 0):
+            raise ValueError("n8n_workflow_count_mismatch")
+        if counts["credentials"] != int(validation.get("credentialFiles") or 0):
+            raise ValueError("n8n_credential_count_mismatch")
+
+    return {
+        "schemaVersion": "host188_product_restore_validation_v1",
+        "dataVerified": True,
+        "assetCount": len(by_name),
+        "postgresArchiveCount": 2,
+        "redisArchiveCount": 1,
+        "n8nSqliteIntegrity": "ok",
+        "credentialsDecrypted": False,
+        "recoveryReadinessClaimed": False,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", type=Path, required=True)
+    args = parser.parse_args()
+    print(json.dumps(validate_restore(args.root), separators=(",", ":")))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
--- a/scripts/backup/verify-host188-products-backup.sh
+++ b/scripts/backup/verify-host188-products-backup.sh
@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+# Snapshot-specific isolated restore drill for Host188 product data.
+
+set -Eeuo pipefail
+umask 077
+
+source "$(dirname "$0")/common.sh"
+
+SERVICE="host188-products-restore-drill"
+REPO="${BACKUP_BASE}/host188-products"
+REMOTE_HOST="ollama@192.168.0.188"
+VALIDATOR="$(dirname "$0")/verify-host188-products-archive.py"
+ESCROW_MARKER="${N8N_ESCROW_EVIDENCE_FILE:-${BACKUP_BASE}/escrow-evidence/n8n_encryption_key.last_verified}"
+ESCROW_MAX_AGE_SECONDS="${N8N_ESCROW_MAX_AGE_SECONDS:-2678400}"
+SSH_TIMEOUT_SECONDS="${HOST188_RESTORE_SSH_TIMEOUT_SECONDS:-1800}"
+MIN_FREE_BYTES="${HOST188_RESTORE_MIN_FREE_BYTES:-8589934592}"
+SNAPSHOT_ID="latest"
+DATA_ONLY=0
+RESTORE_DIR="${BACKUP_BASE}/restore-drill/host188-products-$$"
+SSH_OPTIONS=(
+    -o BatchMode=yes
+    -o StrictHostKeyChecking=yes
+    -o ConnectTimeout=8
+    -o ServerAliveInterval=10
+    -o ServerAliveCountMax=2
+)
+
+cleanup() {
+    rm -rf "${RESTORE_DIR}"
+}
+
+usage() {
+    echo "Usage: verify-host188-products-backup.sh [--snapshot <id|latest>] [--data-only]"
+}
+
+while [ "$#" -gt 0 ]; do
+    case "$1" in
+        --snapshot) SNAPSHOT_ID="${2:-}"; shift 2 ;;
+        --data-only) DATA_ONLY=1; shift ;;
+        -h|--help) usage; exit 0 ;;
+        *) usage >&2; exit 2 ;;
+    esac
+done
+
+latest_snapshot_id() {
+    restic -r "${REPO}" snapshots --latest 1 --json \
+        --password-file "${RESTIC_PASSWORD_FILE}" | \
+        python3 -c 'import json,sys; rows=json.load(sys.stdin); print(rows[-1]["short_id"] if rows else "")'
+}
+
+snapshot_paths() {
+    restic -r "${REPO}" ls "${SNAPSHOT_ID}" --json \
+        --password-file "${RESTIC_PASSWORD_FILE}" | python3 -c '
+import json
+import re
+import sys
+
+patterns = (
+    re.compile(r"vibework-postgres-[0-9]{8}_[0-9]{6}[.]dump$"),
+    re.compile(r"fifa-postgres-[0-9]{8}_[0-9]{6}[.]dump$"),
+    re.compile(r"fifa-redis-[0-9]{8}_[0-9]{6}[.]rdb$"),
+    re.compile(r"n8n-online-[0-9]{8}_[0-9]{6}[.]tar[.]gz$"),
+    re.compile(r"n8n-validation[.]json$"),
+    re.compile(r"backup-manifest[.]json$"),
+)
+found = {pattern.pattern: [] for pattern in patterns}
+for line in sys.stdin:
+    try:
+        row = json.loads(line)
+    except json.JSONDecodeError:
+        continue
+    if row.get("type") != "file":
+        continue
+    path = str(row.get("path") or "")
+    for pattern in patterns:
+        if pattern.search(path):
+            found[pattern.pattern].append(path)
+for pattern in patterns:
+    rows = found[pattern.pattern]
+    if len(rows) != 1:
+        raise SystemExit("snapshot_asset_cardinality_invalid")
+    print(rows[0])
+'
+}
+
+verify_postgres_archive() {
+    local container="$1"
+    local archive="$2"
+    local command
+    printf -v command 'docker exec -i %q pg_restore --list >/dev/null' "${container}"
+    timeout --signal=TERM "${SSH_TIMEOUT_SECONDS}s" \
+        ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${command}" <"${archive}"
+}
+
+verify_redis_archive() {
+    local archive="$1"
+    local inner command
+    inner='set -eu; umask 077; tmp=/tmp/agent99-fifa-redis-restore-drill.rdb; rm -f "$tmp"; trap '\''rm -f "$tmp"'\'' EXIT HUP INT TERM; cat >"$tmp"; redis-check-rdb "$tmp" >/dev/null'
+    printf -v command \
+        'test "$(docker inspect -f '\''{{.State.Running}}'\'' current-fifa2026-redis-1 2>/dev/null)" = true && docker exec -i current-fifa2026-redis-1 sh -eu -c %q' \
+        "${inner}"
+    timeout --signal=TERM "${SSH_TIMEOUT_SECONDS}s" \
+        ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${command}" <"${archive}"
+}
+
+escrow_marker_fresh() {
+    python3 - "${ESCROW_MARKER}" "${ESCROW_MAX_AGE_SECONDS}" <<'PY'
+import sys
+import time
+from pathlib import Path
+
+path = Path(sys.argv[1])
+max_age = int(sys.argv[2])
+if not path.is_file():
+    raise SystemExit(1)
+values = {}
+for line in path.read_text(encoding="utf-8").splitlines():
+    key, separator, value = line.partition("=")
+    if separator:
+        values[key] = value
+timestamp = int(values.get("timestamp") or 0)
+valid = (
+    values.get("item") == "n8n_encryption_key"
+    and bool(values.get("evidence_id"))
+    and 0 <= int(time.time()) - timestamp <= max_age
+)
+raise SystemExit(0 if valid else 1)
+PY
+}
+
+main() {
+    local path name available_bytes
+    local paths=()
+    trap cleanup EXIT HUP INT TERM
+    install -d -m 700 "${RESTORE_DIR}"
+    for command_name in restic ssh python3 timeout; do
+        command -v "${command_name}" >/dev/null 2>&1 || {
+            log_error "Host188 restore command missing: ${command_name}"
+            return 69
+        }
+    done
+    case "${SSH_TIMEOUT_SECONDS}:${ESCROW_MAX_AGE_SECONDS}:${MIN_FREE_BYTES}" in
+        *[!0-9:]*|:*|*:) log_error "Host188 restore numeric limits are invalid"; return 64 ;;
+    esac
+    if [ "${SSH_TIMEOUT_SECONDS}" -le 0 ] || [ "${ESCROW_MAX_AGE_SECONDS}" -le 0 ] || [ "${MIN_FREE_BYTES}" -le 0 ]; then
+        log_error "Host188 restore numeric limits must be positive"
+        return 64
+    fi
+    available_bytes="$(df -Pk "${BACKUP_BASE}" 2>/dev/null | awk 'NR==2 {printf "%.0f", $4 * 1024}')"
+    if [ -z "${available_bytes}" ] || [ "${available_bytes}" -lt "${MIN_FREE_BYTES}" ]; then
+        log_error "Host188 isolated restore headroom is insufficient"
+        return 75
+    fi
+    [ -r "${VALIDATOR}" ] || { log_error "Host188 restore validator missing"; return 69; }
+    [ -r "${RESTIC_PASSWORD_FILE}" ] || { log_error "Restic password reference unavailable"; return 69; }
+
+    if [ "${SNAPSHOT_ID}" = "latest" ]; then
+        SNAPSHOT_ID="$(latest_snapshot_id)"
+    fi
+    [[ "${SNAPSHOT_ID}" =~ ^[0-9a-fA-F]{8,64}$ ]] || {
+        log_error "Host188 snapshot id invalid"
+        return 64
+    }
+    verify_backup "${REPO}" "${SNAPSHOT_ID}"
+    while IFS= read -r path; do
+        paths+=("${path}")
+    done < <(snapshot_paths)
+    [ "${#paths[@]}" -eq 6 ] || { log_error "Host188 snapshot asset list incomplete"; return 1; }
+    for path in "${paths[@]}"; do
+        name="${path##*/}"
+        restic -r "${REPO}" dump "${SNAPSHOT_ID}" "${path}" \
+            --password-file "${RESTIC_PASSWORD_FILE}" >"${RESTORE_DIR}/${name}"
+    done
+
+    python3 "${VALIDATOR}" --root "${RESTORE_DIR}" >"${RESTORE_DIR}/validation-receipt.json"
+    verify_postgres_archive \
+        "vibework-production-postgres-1" \
+        "$(find "${RESTORE_DIR}" -maxdepth 1 -name 'vibework-postgres-*.dump' -print -quit)"
+    verify_postgres_archive \
+        "current-fifa2026-postgres-1" \
+        "$(find "${RESTORE_DIR}" -maxdepth 1 -name 'fifa-postgres-*.dump' -print -quit)"
+    verify_redis_archive \
+        "$(find "${RESTORE_DIR}" -maxdepth 1 -name 'fifa-redis-*.rdb' -print -quit)"
+
+    if [ "${DATA_ONLY}" = "1" ]; then
+        echo "HOST188_RESTORE_DRILL_DATA_VERIFIED=1"
+        echo "HOST188_RESTORE_DRILL_RECOVERY_READY=0"
+        echo "HOST188_RESTORE_DRILL_SCOPE=data_only"
+        return 0
+    fi
+    if ! escrow_marker_fresh; then
+        echo "HOST188_RESTORE_DRILL_DATA_VERIFIED=1"
+        echo "HOST188_RESTORE_DRILL_RECOVERY_READY=0"
+        echo "HOST188_RESTORE_DRILL_BLOCKER=n8n_encryption_key_escrow_missing_or_stale"
+        return 78
+    fi
+    echo "HOST188_RESTORE_DRILL_DATA_VERIFIED=1"
+    echo "HOST188_RESTORE_DRILL_RECOVERY_READY=1"
+}
+
+main "$@"
--- a/scripts/ops/ansible-validate.sh
+++ b/scripts/ops/ansible-validate.sh
@@ -65,6 +65,7 @@ bash -n \
  scripts/backup/backup-ai-artifacts.sh \
  scripts/backup/backup-public-routes.sh \
  scripts/backup/backup-host188-products.sh \
+  scripts/backup/verify-host188-products-backup.sh \
  scripts/backup/configure-offsite-rclone.sh \
  scripts/backup/configure-offsite-b2.sh \
  scripts/backup/sync-offsite-backups.sh \
@@ -86,6 +87,7 @@ python3 -m py_compile \
  scripts/ops/backup-alert-live-visibility-check.py \
  scripts/ops/recovery-scorecard-contract-check.py \
  scripts/backup/clickhouse-restore-inventory.py \
+  scripts/backup/verify-host188-products-archive.py \
  scripts/ops/doc-secrets-sanity-check.py
 echo "Python 語法 OK"

@@ -107,6 +109,7 @@ echo "== Ansible syntax-check =="
 for playbook in \
  infra/ansible/playbooks/site.yml \
  infra/ansible/playbooks/110-devops.yml \
+  infra/ansible/playbooks/110-backup-runtime-deploy.yml \
  infra/ansible/playbooks/188-ai-web.yml \
  infra/ansible/playbooks/nginx-sync.yml; do
  ansible-playbook -i infra/ansible/inventory/hosts.yml "$playbook" --syntax-check
--- a/scripts/ops/backup-health-textfile-exporter.py
+++ b/scripts/ops/backup-health-textfile-exporter.py
@@ -1193,6 +1193,9 @@ def _collect_110(host: str) -> list[str]:
        "backup-ai-artifacts.sh",
        "backup-public-routes.sh",
        "backup-host188-products.sh",
+        "verify-host188-products-backup.sh",
+        "verify-host188-products-archive.py",
+        "enforce-latest-only-retention.sh",
        "configure-offsite-rclone.sh",
        "configure-offsite-b2.sh",
        "sync-offsite-backups.sh",
@@ -1231,7 +1234,7 @@ def _collect_110(host: str) -> list[str]:
        ("clawbot", "/backup/clawbot", 48),
        ("ai_artifacts", "/backup/ai-artifacts", 48),
        ("public_routes", "/backup/public-routes", 168),
-        ("host188_products", "/backup/host188-products", 48),
+        ("host188_products", "/backup/host188-products", 30),
    ]:
        timestamp, count = _latest_restic_snapshot(repo)
        age = int(time.time()) - timestamp if timestamp else 0
--- a/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh
+++ b/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh
@@ -279,6 +279,8 @@ require_file scripts/backup/backup-sentry.sh "Sentry dedicated data backup"
 require_file scripts/backup/backup-ai-artifacts.sh "AI artifacts and Ollama manifest backup"
 require_file scripts/backup/backup-public-routes.sh "Public routes DNS/TLS evidence backup"
 require_file scripts/backup/backup-host188-products.sh "Host188 product data-plane backup"
+require_file scripts/backup/verify-host188-products-backup.sh "Host188 snapshot-specific restore drill"
+require_file scripts/backup/verify-host188-products-archive.py "Host188 isolated restore archive validator"
 require_file scripts/backup/configure-offsite-rclone.sh "Offsite Google Drive/rclone host-local config helper"
 require_pattern "create-root-remote" scripts/backup/configure-offsite-rclone.sh "Offsite Google Drive root-scoped remote helper"
 require_pattern "gdrive_awoooi_restic" docs/runbooks/OFFSITE-BACKUP-ESCROW-RUNBOOK.md "Offsite Google Drive root-scoped remote runbook"