#!/usr/bin/env bash # Snapshot-specific isolated restore drill for Host188 product data. set -Eeuo pipefail umask 077 source "$(dirname "$0")/common.sh" SERVICE="host188-products-restore-drill" REPO="${BACKUP_BASE}/host188-products" REMOTE_HOST="ollama@192.168.0.188" VALIDATOR="$(dirname "$0")/verify-host188-products-archive.py" ESCROW_MARKER="${N8N_ESCROW_EVIDENCE_FILE:-${BACKUP_BASE}/escrow-evidence/n8n_encryption_key.last_verified}" ESCROW_MAX_AGE_SECONDS="${N8N_ESCROW_MAX_AGE_SECONDS:-2678400}" SSH_TIMEOUT_SECONDS="${HOST188_RESTORE_SSH_TIMEOUT_SECONDS:-1800}" MIN_FREE_BYTES="${HOST188_RESTORE_MIN_FREE_BYTES:-8589934592}" SNAPSHOT_ID="latest" DATA_ONLY=0 RESTORE_DIR="${BACKUP_BASE}/restore-drill/host188-products-$$" SSH_OPTIONS=( -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o ServerAliveCountMax=2 ) cleanup() { rm -rf "${RESTORE_DIR}" } usage() { echo "Usage: verify-host188-products-backup.sh [--snapshot ] [--data-only]" } while [ "$#" -gt 0 ]; do case "$1" in --snapshot) SNAPSHOT_ID="${2:-}"; shift 2 ;; --data-only) DATA_ONLY=1; shift ;; -h|--help) usage; exit 0 ;; *) usage >&2; exit 2 ;; esac done latest_snapshot_id() { restic -r "${REPO}" snapshots --latest 1 --json \ --password-file "${RESTIC_PASSWORD_FILE}" | \ python3 -c 'import json,sys; rows=json.load(sys.stdin); print(rows[-1]["short_id"] if rows else "")' } snapshot_paths() { restic -r "${REPO}" ls "${SNAPSHOT_ID}" --json \ --password-file "${RESTIC_PASSWORD_FILE}" | python3 -c ' import json import re import sys patterns = ( re.compile(r"vibework-postgres-[0-9]{8}_[0-9]{6}[.]dump$"), re.compile(r"fifa-postgres-[0-9]{8}_[0-9]{6}[.]dump$"), re.compile(r"fifa-redis-[0-9]{8}_[0-9]{6}[.]rdb$"), re.compile(r"n8n-online-[0-9]{8}_[0-9]{6}[.]tar[.]gz$"), re.compile(r"n8n-validation[.]json$"), re.compile(r"backup-manifest[.]json$"), ) found = {pattern.pattern: [] for pattern in patterns} for line in sys.stdin: try: row = json.loads(line) except json.JSONDecodeError: continue if row.get("type") != "file": continue path = str(row.get("path") or "") for pattern in patterns: if pattern.search(path): found[pattern.pattern].append(path) for pattern in patterns: rows = found[pattern.pattern] if len(rows) != 1: raise SystemExit("snapshot_asset_cardinality_invalid") print(rows[0]) ' } verify_postgres_archive() { local container="$1" local archive="$2" local command printf -v command 'docker exec -i %q pg_restore --list >/dev/null' "${container}" timeout --signal=TERM "${SSH_TIMEOUT_SECONDS}s" \ ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${command}" <"${archive}" } verify_redis_archive() { local archive="$1" local inner command inner='set -eu; umask 077; tmp=/tmp/agent99-fifa-redis-restore-drill.rdb; rm -f "$tmp"; trap '\''rm -f "$tmp"'\'' EXIT HUP INT TERM; cat >"$tmp"; redis-check-rdb "$tmp" >/dev/null' printf -v command \ 'test "$(docker inspect -f '\''{{.State.Running}}'\'' current-fifa2026-redis-1 2>/dev/null)" = true && docker exec -i current-fifa2026-redis-1 sh -eu -c %q' \ "${inner}" timeout --signal=TERM "${SSH_TIMEOUT_SECONDS}s" \ ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${command}" <"${archive}" } escrow_marker_fresh() { python3 - "${ESCROW_MARKER}" "${ESCROW_MAX_AGE_SECONDS}" <<'PY' import sys import time from pathlib import Path path = Path(sys.argv[1]) max_age = int(sys.argv[2]) if not path.is_file(): raise SystemExit(1) values = {} for line in path.read_text(encoding="utf-8").splitlines(): key, separator, value = line.partition("=") if separator: values[key] = value timestamp = int(values.get("timestamp") or 0) valid = ( values.get("item") == "n8n_encryption_key" and bool(values.get("evidence_id")) and 0 <= int(time.time()) - timestamp <= max_age ) raise SystemExit(0 if valid else 1) PY } main() { local path name available_bytes local paths=() trap cleanup EXIT HUP INT TERM install -d -m 700 "${RESTORE_DIR}" for command_name in restic ssh python3 timeout; do command -v "${command_name}" >/dev/null 2>&1 || { log_error "Host188 restore command missing: ${command_name}" return 69 } done case "${SSH_TIMEOUT_SECONDS}:${ESCROW_MAX_AGE_SECONDS}:${MIN_FREE_BYTES}" in *[!0-9:]*|:*|*:) log_error "Host188 restore numeric limits are invalid"; return 64 ;; esac if [ "${SSH_TIMEOUT_SECONDS}" -le 0 ] || [ "${ESCROW_MAX_AGE_SECONDS}" -le 0 ] || [ "${MIN_FREE_BYTES}" -le 0 ]; then log_error "Host188 restore numeric limits must be positive" return 64 fi available_bytes="$(df -Pk "${BACKUP_BASE}" 2>/dev/null | awk 'NR==2 {printf "%.0f", $4 * 1024}')" if [ -z "${available_bytes}" ] || [ "${available_bytes}" -lt "${MIN_FREE_BYTES}" ]; then log_error "Host188 isolated restore headroom is insufficient" return 75 fi [ -r "${VALIDATOR}" ] || { log_error "Host188 restore validator missing"; return 69; } [ -r "${RESTIC_PASSWORD_FILE}" ] || { log_error "Restic password reference unavailable"; return 69; } if [ "${SNAPSHOT_ID}" = "latest" ]; then SNAPSHOT_ID="$(latest_snapshot_id)" fi [[ "${SNAPSHOT_ID}" =~ ^[0-9a-fA-F]{8,64}$ ]] || { log_error "Host188 snapshot id invalid" return 64 } verify_backup "${REPO}" "${SNAPSHOT_ID}" while IFS= read -r path; do paths+=("${path}") done < <(snapshot_paths) [ "${#paths[@]}" -eq 6 ] || { log_error "Host188 snapshot asset list incomplete"; return 1; } for path in "${paths[@]}"; do name="${path##*/}" restic -r "${REPO}" dump "${SNAPSHOT_ID}" "${path}" \ --password-file "${RESTIC_PASSWORD_FILE}" >"${RESTORE_DIR}/${name}" done python3 "${VALIDATOR}" --root "${RESTORE_DIR}" >"${RESTORE_DIR}/validation-receipt.json" verify_postgres_archive \ "vibework-production-postgres-1" \ "$(find "${RESTORE_DIR}" -maxdepth 1 -name 'vibework-postgres-*.dump' -print -quit)" verify_postgres_archive \ "current-fifa2026-postgres-1" \ "$(find "${RESTORE_DIR}" -maxdepth 1 -name 'fifa-postgres-*.dump' -print -quit)" verify_redis_archive \ "$(find "${RESTORE_DIR}" -maxdepth 1 -name 'fifa-redis-*.rdb' -print -quit)" if [ "${DATA_ONLY}" = "1" ]; then echo "HOST188_RESTORE_DRILL_DATA_VERIFIED=1" echo "HOST188_RESTORE_DRILL_RECOVERY_READY=0" echo "HOST188_RESTORE_DRILL_SCOPE=data_only" return 0 fi if ! escrow_marker_fresh; then echo "HOST188_RESTORE_DRILL_DATA_VERIFIED=1" echo "HOST188_RESTORE_DRILL_RECOVERY_READY=0" echo "HOST188_RESTORE_DRILL_BLOCKER=n8n_encryption_key_escrow_missing_or_stale" return 78 fi echo "HOST188_RESTORE_DRILL_DATA_VERIFIED=1" echo "HOST188_RESTORE_DRILL_RECOVERY_READY=1" } main "$@"