#!/usr/bin/env bash # Fixed n8n, VibeWork, and FIFA data-plane backup from host 188 to host 110. set -Eeuo pipefail umask 077 source "$(dirname "$0")/common.sh" SERVICE="host188-products" REMOTE_HOST="ollama@192.168.0.188" LOCAL_REPO="${BACKUP_BASE}/host188-products" RESTORE_VERIFIER="$(dirname "$0")/verify-host188-products-backup.sh" DUMP_DIR="${BACKUP_BASE}/staging/host188-products-backup-$$" LOCK_FILE="/tmp/host188-products-backup.lock" MIN_POSTGRES_DUMP_BYTES="${MIN_POSTGRES_DUMP_BYTES:-4096}" MIN_FREE_BYTES="${HOST188_BACKUP_MIN_FREE_BYTES:-8589934592}" SSH_TIMEOUT_SECONDS="${HOST188_BACKUP_SSH_TIMEOUT_SECONDS:-1800}" SSH_OPTIONS=( -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o ServerAliveCountMax=2 ) cleanup() { rm -rf "${DUMP_DIR}" } on_error() { local status=$? trap - ERR set +e log_error "Host188 product backup failed (exit=${status})" notify_clawbot "failed" "${SERVICE}" "Host188 product backup failed; inspect the controller receipt" 0 cleanup exit "${status}" } require_commands() { local command_name for command_name in restic ssh python3 flock timeout; do if ! command -v "${command_name}" >/dev/null 2>&1; then log_error "Required command missing: ${command_name}" return 69 fi done if [ ! -r "${RESTIC_PASSWORD_FILE}" ]; then log_error "Restic password reference is unavailable" return 69 fi case "${MIN_FREE_BYTES}:${SSH_TIMEOUT_SECONDS}" in *[!0-9:]*|:*|*:) log_error "Host188 backup numeric limits are invalid"; return 64 ;; esac if [ "${MIN_FREE_BYTES}" -le 0 ] || [ "${SSH_TIMEOUT_SECONDS}" -le 0 ]; then log_error "Host188 backup numeric limits must be positive" return 64 fi } run_remote() { timeout --signal=TERM "${SSH_TIMEOUT_SECONDS}s" \ ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "$@" } require_disk_headroom() { local available_bytes available_bytes="$(df -Pk "${BACKUP_BASE}" 2>/dev/null | awk 'NR==2 {printf "%.0f", $4 * 1024}')" if [ -z "${available_bytes}" ] || [ "${available_bytes}" -lt "${MIN_FREE_BYTES}" ]; then log_error "Host188 backup staging headroom is insufficient" return 75 fi } assert_fixed_container() { case "$1" in vibework-production-postgres-1|current-fifa2026-postgres-1|current-fifa2026-redis-1|n8n) return 0 ;; *) return 64 ;; esac } dump_redis_container() { local container="$1" local output_file="$2" local temporary_file="${output_file}.tmp" local remote_tmp="/tmp/agent99-fifa-redis-backup-$$.rdb" local inner_command remote_command size_bytes magic assert_fixed_container "${container}" printf -v inner_command \ 'set -eu; umask 077; tmp=%q; rm -f "$tmp"; trap '\''rm -f "$tmp"'\'' EXIT HUP INT TERM; if [ -n "${REDIS_PASSWORD:-}" ]; then export REDISCLI_AUTH="$REDIS_PASSWORD"; fi; redis-cli --rdb "$tmp" >/dev/null; test -s "$tmp"; cat "$tmp"' \ "${remote_tmp}" printf -v remote_command \ 'test "$(docker inspect -f '\''{{.State.Running}}'\'' %q 2>/dev/null)" = true && docker exec %q sh -eu -c %q' \ "${container}" "${container}" "${inner_command}" if ! run_remote "${remote_command}" >"${temporary_file}"; then log_error "Redis RDB dump failed: ${container}" return 1 fi size_bytes="$(stat -c%s "${temporary_file}" 2>/dev/null || echo 0)" magic="$(head -c 5 "${temporary_file}" 2>/dev/null || true)" if [ "${size_bytes}" -lt 9 ] || [ "${magic}" != "REDIS" ]; then log_error "Redis RDB validation failed: ${container} size=${size_bytes}" return 1 fi mv "${temporary_file}" "${output_file}" } dump_postgres_container() { local container="$1" local output_file="$2" local temporary_file="${output_file}.tmp" local dump_inner dump_command verify_command size_bytes magic assert_fixed_container "${container}" dump_inner='exec pg_dump -Fc -U "$POSTGRES_USER" -d "$POSTGRES_DB" --no-password --no-owner --no-acl' printf -v dump_command \ 'test "$(docker inspect -f '\''{{.State.Running}}'\'' %q 2>/dev/null)" = true && docker exec %q sh -eu -c %q' \ "${container}" "${container}" "${dump_inner}" if ! run_remote "${dump_command}" >"${temporary_file}"; then log_error "PostgreSQL dump failed: ${container}" return 1 fi size_bytes="$(stat -c%s "${temporary_file}" 2>/dev/null || echo 0)" magic="$(head -c 5 "${temporary_file}" 2>/dev/null || true)" if [ "${size_bytes}" -lt "${MIN_POSTGRES_DUMP_BYTES}" ] || [ "${magic}" != "PGDMP" ]; then log_error "PostgreSQL dump validation failed: ${container} size=${size_bytes}" return 1 fi printf -v verify_command 'docker exec -i %q pg_restore --list >/dev/null' "${container}" if ! run_remote "${verify_command}" <"${temporary_file}"; then log_error "PostgreSQL archive verifier failed: ${container}" return 1 fi mv "${temporary_file}" "${output_file}" } backup_n8n_online() { local timestamp="$1" local output_file="$2" local temporary_file="${output_file}.tmp" local remote_tmp="/tmp/agent99-n8n-backup-${timestamp}" local node_script inner_command remote_command validation_dir assert_fixed_container "n8n" node_script='const p=require.resolve("sqlite3",{paths:["/usr/local/lib/node_modules/n8n"]});const sqlite3=require(p);const src=process.env.N8N_SQLITE_SOURCE;const dst=process.env.N8N_SQLITE_DEST;const db=new sqlite3.Database(src,sqlite3.OPEN_READONLY,(openError)=>{if(openError){console.error("sqlite_open_failed");process.exit(2);}db.backup(dst,(backupError)=>{db.close((closeError)=>{if(backupError||closeError){console.error("sqlite_backup_failed");process.exit(3);}});});});' printf -v inner_command \ 'set -eu; umask 077; tmp=%q; rm -rf "$tmp"; trap '\''rm -rf "$tmp"'\'' EXIT HUP INT TERM; mkdir -p "$tmp/workflows" "$tmp/credentials"; N8N_SQLITE_SOURCE=/home/node/.n8n/database.sqlite N8N_SQLITE_DEST="$tmp/database.sqlite" node -e %q >/dev/null 2>&1; n8n export:workflow --backup --output="$tmp/workflows" >/dev/null 2>&1; n8n export:credentials --backup --output="$tmp/credentials" >/dev/null 2>&1; tar czf - -C "$tmp" .' \ "${remote_tmp}" "${node_script}" printf -v remote_command \ 'test "$(docker inspect -f '\''{{.State.Running}}'\'' n8n 2>/dev/null)" = true && docker exec n8n sh -eu -c %q' \ "${inner_command}" if ! run_remote "${remote_command}" >"${temporary_file}"; then log_error "n8n online backup failed" return 1 fi validation_dir="${DUMP_DIR}/n8n-validation" install -d -m 700 "${validation_dir}" python3 - "${temporary_file}" "${validation_dir}" "${DUMP_DIR}/n8n-validation.json" <<'PY' import json import sqlite3 import sys import tarfile from pathlib import Path, PurePosixPath archive_path = Path(sys.argv[1]) root = Path(sys.argv[2]) output = Path(sys.argv[3]) try: with tarfile.open(archive_path, "r:gz") as archive: members = archive.getmembers() for member in members: path = PurePosixPath(member.name) if path.is_absolute() or ".." in path.parts: raise SystemExit("n8n_archive_unsafe_path") if not (member.isfile() or member.isdir()): raise SystemExit("n8n_archive_unsafe_member") archive.extractall(root, members=members) except (tarfile.TarError, OSError): raise SystemExit("n8n_archive_invalid") database = root / "database.sqlite" if not database.is_file() or database.stat().st_size <= 0: raise SystemExit("n8n_sqlite_backup_missing") with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection: integrity = connection.execute("PRAGMA integrity_check").fetchone()[0] if integrity != "ok": raise SystemExit("n8n_sqlite_integrity_failed") counts = {} for category in ("workflows", "credentials"): directory = root / category if not directory.is_dir(): raise SystemExit(f"n8n_{category}_directory_missing") rows = list(directory.glob("*.json")) for path in rows: json.loads(path.read_text(encoding="utf-8")) counts[category] = len(rows) output.write_text( json.dumps( { "schemaVersion": "n8n_backup_validation_v1", "sqliteIntegrity": "ok", "workflowFiles": counts["workflows"], "credentialFiles": counts["credentials"], "credentialsDecrypted": False, }, separators=(",", ":"), ) + "\n", encoding="utf-8", ) PY rm -rf "${validation_dir}" mv "${temporary_file}" "${output_file}" } write_manifest() { local timestamp="$1" python3 - "${DUMP_DIR}" "${timestamp}" <<'PY' import hashlib import json import sys from pathlib import Path root = Path(sys.argv[1]) timestamp = sys.argv[2] def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() assets = [] for name in ( f"vibework-postgres-{timestamp}.dump", f"fifa-postgres-{timestamp}.dump", f"fifa-redis-{timestamp}.rdb", f"n8n-online-{timestamp}.tar.gz", "n8n-validation.json", ): path = root / name assets.append( { "name": name, "sizeBytes": path.stat().st_size, "sha256": sha256_file(path), } ) (root / "backup-manifest.json").write_text( json.dumps( { "schemaVersion": "host188_product_backup_manifest_v1", "sourceHost": "192.168.0.188", "controllerHost": "192.168.0.110", "timestamp": timestamp, "assets": assets, "secretValueExported": False, "credentialsDecrypted": False, "serviceStopped": False, "offsiteWritePerformed": False, "recoveryReadinessClaimed": False, "n8nEncryptionKeyEscrowRequired": True, "n8nEncryptionKeyEscrowVerified": False, }, separators=(",", ":"), ) + "\n", encoding="utf-8", ) PY } latest_snapshot_id() { restic -r "${LOCAL_REPO}" snapshots --latest 1 --json \ --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \ python3 -c 'import json,sys; rows=json.load(sys.stdin); print(rows[-1].get("short_id","unknown") if rows else "unknown")' } main() { local start_time timestamp snapshot_id duration start_time="$(date +%s)" timestamp="$(date +%Y%m%d_%H%M%S)" trap cleanup EXIT HUP INT TERM trap on_error ERR require_commands exec 9>"${LOCK_FILE}" if ! flock -n 9; then log_error "Host188 product backup is already running" return 75 fi require_disk_headroom install -d -m 700 "${DUMP_DIR}" log_info "Host188 product backup started" dump_postgres_container \ "vibework-production-postgres-1" \ "${DUMP_DIR}/vibework-postgres-${timestamp}.dump" dump_postgres_container \ "current-fifa2026-postgres-1" \ "${DUMP_DIR}/fifa-postgres-${timestamp}.dump" dump_redis_container \ "current-fifa2026-redis-1" \ "${DUMP_DIR}/fifa-redis-${timestamp}.rdb" backup_n8n_online "${timestamp}" "${DUMP_DIR}/n8n-online-${timestamp}.tar.gz" write_manifest "${timestamp}" if ! restic -r "${LOCAL_REPO}" snapshots --json --password-file "${RESTIC_PASSWORD_FILE}" >/dev/null 2>&1; then restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" fi restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" \ --password-file "${RESTIC_PASSWORD_FILE}" \ --tag "service:${SERVICE}" --tag "source:host188" --tag "timestamp:${timestamp}" snapshot_id="$(latest_snapshot_id)" [ -r "${RESTORE_VERIFIER}" ] || { log_error "Host188 snapshot restore verifier missing" return 69 } bash "${RESTORE_VERIFIER}" --snapshot "${snapshot_id}" --data-only duration="$(($(date +%s) - start_time))" log_success "Host188 product data snapshot verified snapshot=${snapshot_id} duration=${duration}s; DR escrow remains a separate gate" if [ "${BACKUP_NOTIFY_SUCCESS:-0}" = "1" ]; then notify_clawbot "success" "${SERVICE}" "Host188 product backup completed and verified" "${duration}" fi } main "$@"