304 lines
11 KiB
Bash
304 lines
11 KiB
Bash
#!/usr/bin/env bash
|
|
# Fixed n8n, VibeWork, and FIFA data-plane backup from host 188 to host 110.
|
|
|
|
set -Eeuo pipefail
|
|
umask 077
|
|
|
|
source "$(dirname "$0")/common.sh"
|
|
|
|
SERVICE="host188-products"
|
|
REMOTE_HOST="ollama@192.168.0.188"
|
|
LOCAL_REPO="${BACKUP_BASE}/host188-products"
|
|
DUMP_DIR="/tmp/host188-products-backup-$$"
|
|
MIN_POSTGRES_DUMP_BYTES="${MIN_POSTGRES_DUMP_BYTES:-4096}"
|
|
SSH_OPTIONS=(
|
|
-o BatchMode=yes
|
|
-o StrictHostKeyChecking=yes
|
|
-o ConnectTimeout=8
|
|
-o ServerAliveInterval=10
|
|
-o ServerAliveCountMax=2
|
|
)
|
|
|
|
cleanup() {
|
|
rm -rf "${DUMP_DIR}"
|
|
}
|
|
|
|
on_error() {
|
|
local status=$?
|
|
trap - ERR
|
|
set +e
|
|
log_error "Host188 product backup failed (exit=${status})"
|
|
notify_clawbot "failed" "${SERVICE}" "Host188 product backup failed; inspect the controller receipt" 0
|
|
cleanup
|
|
exit "${status}"
|
|
}
|
|
|
|
require_commands() {
|
|
local command_name
|
|
for command_name in restic ssh python3; do
|
|
if ! command -v "${command_name}" >/dev/null 2>&1; then
|
|
log_error "Required command missing: ${command_name}"
|
|
return 69
|
|
fi
|
|
done
|
|
if [ ! -r "${RESTIC_PASSWORD_FILE}" ]; then
|
|
log_error "Restic password reference is unavailable"
|
|
return 69
|
|
fi
|
|
}
|
|
|
|
assert_fixed_container() {
|
|
case "$1" in
|
|
vibework-production-postgres-1|current-fifa2026-postgres-1|current-fifa2026-redis-1|n8n) return 0 ;;
|
|
*) return 64 ;;
|
|
esac
|
|
}
|
|
|
|
dump_redis_container() {
|
|
local container="$1"
|
|
local output_file="$2"
|
|
local temporary_file="${output_file}.tmp"
|
|
local remote_tmp="/tmp/agent99-fifa-redis-backup-$$.rdb"
|
|
local inner_command remote_command size_bytes magic
|
|
|
|
assert_fixed_container "${container}"
|
|
printf -v inner_command \
|
|
'set -eu; umask 077; tmp=%q; rm -f "$tmp"; trap '\''rm -f "$tmp"'\'' EXIT HUP INT TERM; if [ -n "${REDIS_PASSWORD:-}" ]; then export REDISCLI_AUTH="$REDIS_PASSWORD"; fi; redis-cli --rdb "$tmp" >/dev/null; test -s "$tmp"; cat "$tmp"' \
|
|
"${remote_tmp}"
|
|
printf -v remote_command \
|
|
'test "$(docker inspect -f '\''{{.State.Running}}'\'' %q 2>/dev/null)" = true && docker exec %q sh -eu -c %q' \
|
|
"${container}" "${container}" "${inner_command}"
|
|
|
|
if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${remote_command}" >"${temporary_file}"; then
|
|
log_error "Redis RDB dump failed: ${container}"
|
|
return 1
|
|
fi
|
|
|
|
size_bytes="$(stat -c%s "${temporary_file}" 2>/dev/null || echo 0)"
|
|
magic="$(head -c 5 "${temporary_file}" 2>/dev/null || true)"
|
|
if [ "${size_bytes}" -lt 9 ] || [ "${magic}" != "REDIS" ]; then
|
|
log_error "Redis RDB validation failed: ${container} size=${size_bytes}"
|
|
return 1
|
|
fi
|
|
mv "${temporary_file}" "${output_file}"
|
|
}
|
|
|
|
dump_postgres_container() {
|
|
local container="$1"
|
|
local output_file="$2"
|
|
local temporary_file="${output_file}.tmp"
|
|
local dump_inner dump_command verify_command size_bytes magic
|
|
|
|
assert_fixed_container "${container}"
|
|
dump_inner='exec pg_dump -Fc -U "$POSTGRES_USER" -d "$POSTGRES_DB" --no-password --no-owner --no-acl'
|
|
printf -v dump_command \
|
|
'test "$(docker inspect -f '\''{{.State.Running}}'\'' %q 2>/dev/null)" = true && docker exec %q sh -eu -c %q' \
|
|
"${container}" "${container}" "${dump_inner}"
|
|
|
|
if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${dump_command}" >"${temporary_file}"; then
|
|
log_error "PostgreSQL dump failed: ${container}"
|
|
return 1
|
|
fi
|
|
|
|
size_bytes="$(stat -c%s "${temporary_file}" 2>/dev/null || echo 0)"
|
|
magic="$(head -c 5 "${temporary_file}" 2>/dev/null || true)"
|
|
if [ "${size_bytes}" -lt "${MIN_POSTGRES_DUMP_BYTES}" ] || [ "${magic}" != "PGDMP" ]; then
|
|
log_error "PostgreSQL dump validation failed: ${container} size=${size_bytes}"
|
|
return 1
|
|
fi
|
|
|
|
printf -v verify_command 'docker exec -i %q pg_restore --list >/dev/null' "${container}"
|
|
if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${verify_command}" <"${temporary_file}"; then
|
|
log_error "PostgreSQL archive verifier failed: ${container}"
|
|
return 1
|
|
fi
|
|
mv "${temporary_file}" "${output_file}"
|
|
}
|
|
|
|
backup_n8n_online() {
|
|
local timestamp="$1"
|
|
local output_file="$2"
|
|
local temporary_file="${output_file}.tmp"
|
|
local remote_tmp="/tmp/agent99-n8n-backup-${timestamp}"
|
|
local node_script inner_command remote_command validation_dir
|
|
|
|
assert_fixed_container "n8n"
|
|
node_script='const p=require.resolve("sqlite3",{paths:["/usr/local/lib/node_modules/n8n"]});const sqlite3=require(p);const src=process.env.N8N_SQLITE_SOURCE;const dst=process.env.N8N_SQLITE_DEST;const db=new sqlite3.Database(src,sqlite3.OPEN_READONLY,(openError)=>{if(openError){console.error("sqlite_open_failed");process.exit(2);}db.backup(dst,(backupError)=>{db.close((closeError)=>{if(backupError||closeError){console.error("sqlite_backup_failed");process.exit(3);}});});});'
|
|
printf -v inner_command \
|
|
'set -eu; umask 077; tmp=%q; rm -rf "$tmp"; trap '\''rm -rf "$tmp"'\'' EXIT HUP INT TERM; mkdir -p "$tmp/workflows" "$tmp/credentials"; N8N_SQLITE_SOURCE=/home/node/.n8n/database.sqlite N8N_SQLITE_DEST="$tmp/database.sqlite" node -e %q >/dev/null 2>&1; n8n export:workflow --backup --output="$tmp/workflows" >/dev/null 2>&1; n8n export:credentials --backup --output="$tmp/credentials" >/dev/null 2>&1; tar czf - -C "$tmp" .' \
|
|
"${remote_tmp}" "${node_script}"
|
|
printf -v remote_command \
|
|
'test "$(docker inspect -f '\''{{.State.Running}}'\'' n8n 2>/dev/null)" = true && docker exec n8n sh -eu -c %q' \
|
|
"${inner_command}"
|
|
|
|
if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${remote_command}" >"${temporary_file}"; then
|
|
log_error "n8n online backup failed"
|
|
return 1
|
|
fi
|
|
validation_dir="${DUMP_DIR}/n8n-validation"
|
|
install -d -m 700 "${validation_dir}"
|
|
python3 - "${temporary_file}" "${validation_dir}" "${DUMP_DIR}/n8n-validation.json" <<'PY'
|
|
import json
|
|
import sqlite3
|
|
import sys
|
|
import tarfile
|
|
from pathlib import Path, PurePosixPath
|
|
|
|
archive_path = Path(sys.argv[1])
|
|
root = Path(sys.argv[2])
|
|
output = Path(sys.argv[3])
|
|
try:
|
|
with tarfile.open(archive_path, "r:gz") as archive:
|
|
members = archive.getmembers()
|
|
for member in members:
|
|
path = PurePosixPath(member.name)
|
|
if path.is_absolute() or ".." in path.parts:
|
|
raise SystemExit("n8n_archive_unsafe_path")
|
|
if not (member.isfile() or member.isdir()):
|
|
raise SystemExit("n8n_archive_unsafe_member")
|
|
archive.extractall(root, members=members)
|
|
except (tarfile.TarError, OSError):
|
|
raise SystemExit("n8n_archive_invalid")
|
|
|
|
database = root / "database.sqlite"
|
|
if not database.is_file() or database.stat().st_size <= 0:
|
|
raise SystemExit("n8n_sqlite_backup_missing")
|
|
with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection:
|
|
integrity = connection.execute("PRAGMA integrity_check").fetchone()[0]
|
|
if integrity != "ok":
|
|
raise SystemExit("n8n_sqlite_integrity_failed")
|
|
|
|
counts = {}
|
|
for category in ("workflows", "credentials"):
|
|
directory = root / category
|
|
if not directory.is_dir():
|
|
raise SystemExit(f"n8n_{category}_directory_missing")
|
|
rows = list(directory.glob("*.json"))
|
|
for path in rows:
|
|
json.loads(path.read_text(encoding="utf-8"))
|
|
counts[category] = len(rows)
|
|
|
|
output.write_text(
|
|
json.dumps(
|
|
{
|
|
"schemaVersion": "n8n_backup_validation_v1",
|
|
"sqliteIntegrity": "ok",
|
|
"workflowFiles": counts["workflows"],
|
|
"credentialFiles": counts["credentials"],
|
|
"credentialsDecrypted": False,
|
|
},
|
|
separators=(",", ":"),
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
PY
|
|
rm -rf "${validation_dir}"
|
|
mv "${temporary_file}" "${output_file}"
|
|
}
|
|
|
|
write_manifest() {
|
|
local timestamp="$1"
|
|
python3 - "${DUMP_DIR}" "${timestamp}" <<'PY'
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
root = Path(sys.argv[1])
|
|
timestamp = sys.argv[2]
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
assets = []
|
|
for name in (
|
|
f"vibework-postgres-{timestamp}.dump",
|
|
f"fifa-postgres-{timestamp}.dump",
|
|
f"fifa-redis-{timestamp}.rdb",
|
|
f"n8n-online-{timestamp}.tar.gz",
|
|
"n8n-validation.json",
|
|
):
|
|
path = root / name
|
|
assets.append(
|
|
{
|
|
"name": name,
|
|
"sizeBytes": path.stat().st_size,
|
|
"sha256": sha256_file(path),
|
|
}
|
|
)
|
|
|
|
(root / "backup-manifest.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schemaVersion": "host188_product_backup_manifest_v1",
|
|
"sourceHost": "192.168.0.188",
|
|
"controllerHost": "192.168.0.110",
|
|
"timestamp": timestamp,
|
|
"assets": assets,
|
|
"secretValueExported": False,
|
|
"credentialsDecrypted": False,
|
|
"serviceStopped": False,
|
|
"offsiteWritePerformed": False,
|
|
},
|
|
separators=(",", ":"),
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
PY
|
|
}
|
|
|
|
latest_snapshot_id() {
|
|
restic -r "${LOCAL_REPO}" snapshots --latest 1 --json \
|
|
--password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \
|
|
python3 -c 'import json,sys; rows=json.load(sys.stdin); print(rows[-1].get("short_id","unknown") if rows else "unknown")'
|
|
}
|
|
|
|
main() {
|
|
local start_time timestamp snapshot_id duration
|
|
start_time="$(date +%s)"
|
|
timestamp="$(date +%Y%m%d_%H%M%S)"
|
|
trap cleanup EXIT HUP INT TERM
|
|
trap on_error ERR
|
|
|
|
require_commands
|
|
install -d -m 700 "${DUMP_DIR}"
|
|
log_info "Host188 product backup started"
|
|
|
|
dump_postgres_container \
|
|
"vibework-production-postgres-1" \
|
|
"${DUMP_DIR}/vibework-postgres-${timestamp}.dump"
|
|
dump_postgres_container \
|
|
"current-fifa2026-postgres-1" \
|
|
"${DUMP_DIR}/fifa-postgres-${timestamp}.dump"
|
|
dump_redis_container \
|
|
"current-fifa2026-redis-1" \
|
|
"${DUMP_DIR}/fifa-redis-${timestamp}.rdb"
|
|
backup_n8n_online "${timestamp}" "${DUMP_DIR}/n8n-online-${timestamp}.tar.gz"
|
|
write_manifest "${timestamp}"
|
|
|
|
if ! restic -r "${LOCAL_REPO}" snapshots --json --password-file "${RESTIC_PASSWORD_FILE}" >/dev/null 2>&1; then
|
|
restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}"
|
|
fi
|
|
restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" \
|
|
--password-file "${RESTIC_PASSWORD_FILE}" \
|
|
--tag "service:${SERVICE}" --tag "source:host188" --tag "timestamp:${timestamp}"
|
|
snapshot_id="$(latest_snapshot_id)"
|
|
verify_backup "${LOCAL_REPO}" "${snapshot_id}"
|
|
|
|
duration="$(($(date +%s) - start_time))"
|
|
log_success "Host188 product backup completed snapshot=${snapshot_id} duration=${duration}s"
|
|
if [ "${BACKUP_NOTIFY_SUCCESS:-0}" = "1" ]; then
|
|
notify_clawbot "success" "${SERVICE}" "Host188 product backup completed and verified" "${duration}"
|
|
fi
|
|
}
|
|
|
|
main "$@"
|