awoooi/scripts/backup/verify-host188-products-backup.sh

#!/usr/bin/env bash
# Snapshot-specific isolated restore drill for Host188 product data.

set -Eeuo pipefail
umask 077

source "$(dirname "$0")/common.sh"

SERVICE="host188-products-restore-drill"
REPO="${BACKUP_BASE}/host188-products"
REMOTE_HOST="ollama@192.168.0.188"
VALIDATOR="$(dirname "$0")/verify-host188-products-archive.py"
ESCROW_MARKER="${N8N_ESCROW_EVIDENCE_FILE:-${BACKUP_BASE}/escrow-evidence/n8n_encryption_key.last_verified}"
ESCROW_MAX_AGE_SECONDS="${N8N_ESCROW_MAX_AGE_SECONDS:-2678400}"
SSH_TIMEOUT_SECONDS="${HOST188_RESTORE_SSH_TIMEOUT_SECONDS:-1800}"
MIN_FREE_BYTES="${HOST188_RESTORE_MIN_FREE_BYTES:-8589934592}"
SNAPSHOT_ID="latest"
DATA_ONLY=0
RESTORE_DIR="${BACKUP_BASE}/restore-drill/host188-products-$$"
SSH_OPTIONS=(
    -o BatchMode=yes
    -o StrictHostKeyChecking=yes
    -o ConnectTimeout=8
    -o ServerAliveInterval=10
    -o ServerAliveCountMax=2
)

cleanup() {
    rm -rf "${RESTORE_DIR}"
}

usage() {
    echo "Usage: verify-host188-products-backup.sh [--snapshot <id|latest>] [--data-only]"
}

while [ "$#" -gt 0 ]; do
    case "$1" in
        --snapshot) SNAPSHOT_ID="${2:-}"; shift 2 ;;
        --data-only) DATA_ONLY=1; shift ;;
        -h|--help) usage; exit 0 ;;
        *) usage >&2; exit 2 ;;
    esac
done

latest_snapshot_id() {
    restic -r "${REPO}" snapshots --latest 1 --json \
        --password-file "${RESTIC_PASSWORD_FILE}" | \
        python3 -c 'import json,sys; rows=json.load(sys.stdin); print(rows[-1]["short_id"] if rows else "")'
}

snapshot_paths() {
    restic -r "${REPO}" ls "${SNAPSHOT_ID}" --json \
        --password-file "${RESTIC_PASSWORD_FILE}" | python3 -c '
import json
import re
import sys

patterns = (
    re.compile(r"vibework-postgres-[0-9]{8}_[0-9]{6}[.]dump$"),
    re.compile(r"fifa-postgres-[0-9]{8}_[0-9]{6}[.]dump$"),
    re.compile(r"fifa-redis-[0-9]{8}_[0-9]{6}[.]rdb$"),
    re.compile(r"n8n-online-[0-9]{8}_[0-9]{6}[.]tar[.]gz$"),
    re.compile(r"n8n-validation[.]json$"),
    re.compile(r"backup-manifest[.]json$"),
)
found = {pattern.pattern: [] for pattern in patterns}
for line in sys.stdin:
    try:
        row = json.loads(line)
    except json.JSONDecodeError:
        continue
    if row.get("type") != "file":
        continue
    path = str(row.get("path") or "")
    for pattern in patterns:
        if pattern.search(path):
            found[pattern.pattern].append(path)
for pattern in patterns:
    rows = found[pattern.pattern]
    if len(rows) != 1:
        raise SystemExit("snapshot_asset_cardinality_invalid")
    print(rows[0])
'
}

verify_postgres_archive() {
    local container="$1"
    local archive="$2"
    local command
    printf -v command 'docker exec -i %q pg_restore --list >/dev/null' "${container}"
    timeout --signal=TERM "${SSH_TIMEOUT_SECONDS}s" \
        ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${command}" <"${archive}"
}

verify_redis_archive() {
    local archive="$1"
    local inner command
    inner='set -eu; umask 077; tmp=/tmp/agent99-fifa-redis-restore-drill.rdb; rm -f "$tmp"; trap '\''rm -f "$tmp"'\'' EXIT HUP INT TERM; cat >"$tmp"; redis-check-rdb "$tmp" >/dev/null'
    printf -v command \
        'test "$(docker inspect -f '\''{{.State.Running}}'\'' current-fifa2026-redis-1 2>/dev/null)" = true && docker exec -i current-fifa2026-redis-1 sh -eu -c %q' \
        "${inner}"
    timeout --signal=TERM "${SSH_TIMEOUT_SECONDS}s" \
        ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${command}" <"${archive}"
}

escrow_marker_fresh() {
    python3 - "${ESCROW_MARKER}" "${ESCROW_MAX_AGE_SECONDS}" <<'PY'
import sys
import time
from pathlib import Path

path = Path(sys.argv[1])
max_age = int(sys.argv[2])
if not path.is_file():
    raise SystemExit(1)
values = {}
for line in path.read_text(encoding="utf-8").splitlines():
    key, separator, value = line.partition("=")
    if separator:
        values[key] = value
timestamp = int(values.get("timestamp") or 0)
valid = (
    values.get("item") == "n8n_encryption_key"
    and bool(values.get("evidence_id"))
    and 0 <= int(time.time()) - timestamp <= max_age
)
raise SystemExit(0 if valid else 1)
PY
}

main() {
    local path name available_bytes
    local paths=()
    trap cleanup EXIT HUP INT TERM
    install -d -m 700 "${RESTORE_DIR}"
    for command_name in restic ssh python3 timeout; do
        command -v "${command_name}" >/dev/null 2>&1 || {
            log_error "Host188 restore command missing: ${command_name}"
            return 69
        }
    done
    case "${SSH_TIMEOUT_SECONDS}:${ESCROW_MAX_AGE_SECONDS}:${MIN_FREE_BYTES}" in
        *[!0-9:]*|:*|*:) log_error "Host188 restore numeric limits are invalid"; return 64 ;;
    esac
    if [ "${SSH_TIMEOUT_SECONDS}" -le 0 ] || [ "${ESCROW_MAX_AGE_SECONDS}" -le 0 ] || [ "${MIN_FREE_BYTES}" -le 0 ]; then
        log_error "Host188 restore numeric limits must be positive"
        return 64
    fi
    available_bytes="$(df -Pk "${BACKUP_BASE}" 2>/dev/null | awk 'NR==2 {printf "%.0f", $4 * 1024}')"
    if [ -z "${available_bytes}" ] || [ "${available_bytes}" -lt "${MIN_FREE_BYTES}" ]; then
        log_error "Host188 isolated restore headroom is insufficient"
        return 75
    fi
    [ -r "${VALIDATOR}" ] || { log_error "Host188 restore validator missing"; return 69; }
    [ -r "${RESTIC_PASSWORD_FILE}" ] || { log_error "Restic password reference unavailable"; return 69; }

    if [ "${SNAPSHOT_ID}" = "latest" ]; then
        SNAPSHOT_ID="$(latest_snapshot_id)"
    fi
    [[ "${SNAPSHOT_ID}" =~ ^[0-9a-fA-F]{8,64}$ ]] || {
        log_error "Host188 snapshot id invalid"
        return 64
    }
    verify_backup "${REPO}" "${SNAPSHOT_ID}"
    while IFS= read -r path; do
        paths+=("${path}")
    done < <(snapshot_paths)
    [ "${#paths[@]}" -eq 6 ] || { log_error "Host188 snapshot asset list incomplete"; return 1; }
    for path in "${paths[@]}"; do
        name="${path##*/}"
        restic -r "${REPO}" dump "${SNAPSHOT_ID}" "${path}" \
            --password-file "${RESTIC_PASSWORD_FILE}" >"${RESTORE_DIR}/${name}"
    done

    python3 "${VALIDATOR}" --root "${RESTORE_DIR}" >"${RESTORE_DIR}/validation-receipt.json"
    verify_postgres_archive \
        "vibework-production-postgres-1" \
        "$(find "${RESTORE_DIR}" -maxdepth 1 -name 'vibework-postgres-*.dump' -print -quit)"
    verify_postgres_archive \
        "current-fifa2026-postgres-1" \
        "$(find "${RESTORE_DIR}" -maxdepth 1 -name 'fifa-postgres-*.dump' -print -quit)"
    verify_redis_archive \
        "$(find "${RESTORE_DIR}" -maxdepth 1 -name 'fifa-redis-*.rdb' -print -quit)"

    if [ "${DATA_ONLY}" = "1" ]; then
        echo "HOST188_RESTORE_DRILL_DATA_VERIFIED=1"
        echo "HOST188_RESTORE_DRILL_RECOVERY_READY=0"
        echo "HOST188_RESTORE_DRILL_SCOPE=data_only"
        return 0
    fi
    if ! escrow_marker_fresh; then
        echo "HOST188_RESTORE_DRILL_DATA_VERIFIED=1"
        echo "HOST188_RESTORE_DRILL_RECOVERY_READY=0"
        echo "HOST188_RESTORE_DRILL_BLOCKER=n8n_encryption_key_escrow_missing_or_stale"
        return 78
    fi
    echo "HOST188_RESTORE_DRILL_DATA_VERIFIED=1"
    echo "HOST188_RESTORE_DRILL_RECOVERY_READY=1"
}

main "$@"