diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index d7b77125e..656ceabac 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -4959,7 +4959,8 @@ function Get-AgentBackupHealthConfig { [pscustomobject]@{ name = "sentry"; path = "/backup/sentry/snapshots"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "ai-artifacts"; path = "/backup/ai-artifacts/snapshots"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "configs"; path = "/backup/configs/snapshots"; maxAgeMinutes = 1800; required = $true }, - [pscustomobject]@{ name = "public-routes"; path = "/backup/public-routes/snapshots"; maxAgeMinutes = 10080; required = $false } + [pscustomobject]@{ name = "public-routes"; path = "/backup/public-routes/snapshots"; maxAgeMinutes = 10080; required = $false }, + [pscustomobject]@{ name = "host188-products"; path = "/backup/host188-products/snapshots"; maxAgeMinutes = 1800; required = $true } ) $fileChecks = @( [pscustomobject]@{ name = "backup_log"; path = "/backup/logs/backup.log"; maxAgeMinutes = 1800; required = $true }, diff --git a/infra/ansible/playbooks/110-backup-runtime-deploy.yml b/infra/ansible/playbooks/110-backup-runtime-deploy.yml index 96edcc0ab..b7cade204 100644 --- a/infra/ansible/playbooks/110-backup-runtime-deploy.yml +++ b/infra/ansible/playbooks/110-backup-runtime-deploy.yml @@ -15,13 +15,14 @@ - backup-awoooi-frequent.sh - backup-clawbot.sh - backup-sentry.sh + - backup-host188-products.sh pre_tasks: - name: Verify the fixed host boundary ansible.builtin.assert: that: - inventory_hostname == "host_110" - - backup_runtime_files | length == 6 + - backup_runtime_files | length == 7 - backup_runtime_files | unique | length == backup_runtime_files | length fail_msg: backup_runtime_deploy_boundary_failed diff --git a/infra/ansible/playbooks/110-devops.yml b/infra/ansible/playbooks/110-devops.yml index 894e51d90..7c3c6745c 100644 --- a/infra/ansible/playbooks/110-devops.yml +++ b/infra/ansible/playbooks/110-devops.yml @@ -238,6 +238,7 @@ - backup-sentry.sh - backup-ai-artifacts.sh - backup-public-routes.sh + - backup-host188-products.sh - clickhouse-native-backup.sh - clickhouse-native-restore-drill.sh - clickhouse-restore-inventory.py diff --git a/scripts/backup/backup-all.sh b/scripts/backup/backup-all.sh index 2e137dafa..9ff7bf183 100755 --- a/scripts/backup/backup-all.sh +++ b/scripts/backup/backup-all.sh @@ -14,7 +14,7 @@ source "$(dirname "$0")/common.sh" main() { local start_time=$(date +%s) local failed=0 - local total=13 + local total=14 log_info "╔══════════════════════════════════════════════════════════════╗" log_info "║ WOOO AIOps - 全服務備份開始 (v3.0) ║" @@ -143,6 +143,15 @@ main() { failed=$((failed+1)) fi + # Host188 product data planes: VibeWork/FIFA PostgreSQL, FIFA Redis, and n8n. + log_info ">>> [14/${total}] 備份 Host188 產品資料..." + if /backup/scripts/backup-host188-products.sh; then + log_success " Host188 產品資料備份成功" + else + log_error " Host188 產品資料備份失敗" + failed=$((failed+1)) + fi + local end_time=$(date +%s) local duration=$((end_time - start_time)) diff --git a/scripts/backup/backup-host188-products.sh b/scripts/backup/backup-host188-products.sh new file mode 100644 index 000000000..c639f10f2 --- /dev/null +++ b/scripts/backup/backup-host188-products.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash +# Fixed n8n, VibeWork, and FIFA data-plane backup from host 188 to host 110. + +set -Eeuo pipefail +umask 077 + +source "$(dirname "$0")/common.sh" + +SERVICE="host188-products" +REMOTE_HOST="ollama@192.168.0.188" +LOCAL_REPO="${BACKUP_BASE}/host188-products" +DUMP_DIR="/tmp/host188-products-backup-$$" +MIN_POSTGRES_DUMP_BYTES="${MIN_POSTGRES_DUMP_BYTES:-4096}" +SSH_OPTIONS=( + -o BatchMode=yes + -o StrictHostKeyChecking=yes + -o ConnectTimeout=8 + -o ServerAliveInterval=10 + -o ServerAliveCountMax=2 +) + +cleanup() { + rm -rf "${DUMP_DIR}" +} + +on_error() { + local status=$? + trap - ERR + set +e + log_error "Host188 product backup failed (exit=${status})" + notify_clawbot "failed" "${SERVICE}" "Host188 product backup failed; inspect the controller receipt" 0 + cleanup + exit "${status}" +} + +require_commands() { + local command_name + for command_name in restic ssh python3; do + if ! command -v "${command_name}" >/dev/null 2>&1; then + log_error "Required command missing: ${command_name}" + return 69 + fi + done + if [ ! -r "${RESTIC_PASSWORD_FILE}" ]; then + log_error "Restic password reference is unavailable" + return 69 + fi +} + +assert_fixed_container() { + case "$1" in + vibework-production-postgres-1|current-fifa2026-postgres-1|current-fifa2026-redis-1|n8n) return 0 ;; + *) return 64 ;; + esac +} + +dump_redis_container() { + local container="$1" + local output_file="$2" + local temporary_file="${output_file}.tmp" + local remote_tmp="/tmp/agent99-fifa-redis-backup-$$.rdb" + local inner_command remote_command size_bytes magic + + assert_fixed_container "${container}" + printf -v inner_command \ + 'set -eu; umask 077; tmp=%q; rm -f "$tmp"; trap '\''rm -f "$tmp"'\'' EXIT HUP INT TERM; if [ -n "${REDIS_PASSWORD:-}" ]; then export REDISCLI_AUTH="$REDIS_PASSWORD"; fi; redis-cli --rdb "$tmp" >/dev/null; test -s "$tmp"; cat "$tmp"' \ + "${remote_tmp}" + printf -v remote_command \ + 'test "$(docker inspect -f '\''{{.State.Running}}'\'' %q 2>/dev/null)" = true && docker exec %q sh -eu -c %q' \ + "${container}" "${container}" "${inner_command}" + + if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${remote_command}" >"${temporary_file}"; then + log_error "Redis RDB dump failed: ${container}" + return 1 + fi + + size_bytes="$(stat -c%s "${temporary_file}" 2>/dev/null || echo 0)" + magic="$(head -c 5 "${temporary_file}" 2>/dev/null || true)" + if [ "${size_bytes}" -lt 9 ] || [ "${magic}" != "REDIS" ]; then + log_error "Redis RDB validation failed: ${container} size=${size_bytes}" + return 1 + fi + mv "${temporary_file}" "${output_file}" +} + +dump_postgres_container() { + local container="$1" + local output_file="$2" + local temporary_file="${output_file}.tmp" + local dump_inner dump_command verify_command size_bytes magic + + assert_fixed_container "${container}" + dump_inner='exec pg_dump -Fc -U "$POSTGRES_USER" -d "$POSTGRES_DB" --no-password --no-owner --no-acl' + printf -v dump_command \ + 'test "$(docker inspect -f '\''{{.State.Running}}'\'' %q 2>/dev/null)" = true && docker exec %q sh -eu -c %q' \ + "${container}" "${container}" "${dump_inner}" + + if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${dump_command}" >"${temporary_file}"; then + log_error "PostgreSQL dump failed: ${container}" + return 1 + fi + + size_bytes="$(stat -c%s "${temporary_file}" 2>/dev/null || echo 0)" + magic="$(head -c 5 "${temporary_file}" 2>/dev/null || true)" + if [ "${size_bytes}" -lt "${MIN_POSTGRES_DUMP_BYTES}" ] || [ "${magic}" != "PGDMP" ]; then + log_error "PostgreSQL dump validation failed: ${container} size=${size_bytes}" + return 1 + fi + + printf -v verify_command 'docker exec -i %q pg_restore --list >/dev/null' "${container}" + if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${verify_command}" <"${temporary_file}"; then + log_error "PostgreSQL archive verifier failed: ${container}" + return 1 + fi + mv "${temporary_file}" "${output_file}" +} + +backup_n8n_online() { + local timestamp="$1" + local output_file="$2" + local temporary_file="${output_file}.tmp" + local remote_tmp="/tmp/agent99-n8n-backup-${timestamp}" + local node_script inner_command remote_command validation_dir + + assert_fixed_container "n8n" + node_script='const p=require.resolve("sqlite3",{paths:["/usr/local/lib/node_modules/n8n"]});const sqlite3=require(p);const src=process.env.N8N_SQLITE_SOURCE;const dst=process.env.N8N_SQLITE_DEST;const db=new sqlite3.Database(src,sqlite3.OPEN_READONLY,(openError)=>{if(openError){console.error("sqlite_open_failed");process.exit(2);}db.backup(dst,(backupError)=>{db.close((closeError)=>{if(backupError||closeError){console.error("sqlite_backup_failed");process.exit(3);}});});});' + printf -v inner_command \ + 'set -eu; umask 077; tmp=%q; rm -rf "$tmp"; trap '\''rm -rf "$tmp"'\'' EXIT HUP INT TERM; mkdir -p "$tmp/workflows" "$tmp/credentials"; N8N_SQLITE_SOURCE=/home/node/.n8n/database.sqlite N8N_SQLITE_DEST="$tmp/database.sqlite" node -e %q >/dev/null 2>&1; n8n export:workflow --backup --output="$tmp/workflows" >/dev/null 2>&1; n8n export:credentials --backup --output="$tmp/credentials" >/dev/null 2>&1; tar czf - -C "$tmp" .' \ + "${remote_tmp}" "${node_script}" + printf -v remote_command \ + 'test "$(docker inspect -f '\''{{.State.Running}}'\'' n8n 2>/dev/null)" = true && docker exec n8n sh -eu -c %q' \ + "${inner_command}" + + if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${remote_command}" >"${temporary_file}"; then + log_error "n8n online backup failed" + return 1 + fi + validation_dir="${DUMP_DIR}/n8n-validation" + install -d -m 700 "${validation_dir}" + python3 - "${temporary_file}" "${validation_dir}" "${DUMP_DIR}/n8n-validation.json" <<'PY' +import json +import sqlite3 +import sys +import tarfile +from pathlib import Path, PurePosixPath + +archive_path = Path(sys.argv[1]) +root = Path(sys.argv[2]) +output = Path(sys.argv[3]) +try: + with tarfile.open(archive_path, "r:gz") as archive: + members = archive.getmembers() + for member in members: + path = PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts: + raise SystemExit("n8n_archive_unsafe_path") + if not (member.isfile() or member.isdir()): + raise SystemExit("n8n_archive_unsafe_member") + archive.extractall(root, members=members) +except (tarfile.TarError, OSError): + raise SystemExit("n8n_archive_invalid") + +database = root / "database.sqlite" +if not database.is_file() or database.stat().st_size <= 0: + raise SystemExit("n8n_sqlite_backup_missing") +with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection: + integrity = connection.execute("PRAGMA integrity_check").fetchone()[0] +if integrity != "ok": + raise SystemExit("n8n_sqlite_integrity_failed") + +counts = {} +for category in ("workflows", "credentials"): + directory = root / category + if not directory.is_dir(): + raise SystemExit(f"n8n_{category}_directory_missing") + rows = list(directory.glob("*.json")) + for path in rows: + json.loads(path.read_text(encoding="utf-8")) + counts[category] = len(rows) + +output.write_text( + json.dumps( + { + "schemaVersion": "n8n_backup_validation_v1", + "sqliteIntegrity": "ok", + "workflowFiles": counts["workflows"], + "credentialFiles": counts["credentials"], + "credentialsDecrypted": False, + }, + separators=(",", ":"), + ) + + "\n", + encoding="utf-8", +) +PY + rm -rf "${validation_dir}" + mv "${temporary_file}" "${output_file}" +} + +write_manifest() { + local timestamp="$1" + python3 - "${DUMP_DIR}" "${timestamp}" <<'PY' +import hashlib +import json +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +timestamp = sys.argv[2] + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +assets = [] +for name in ( + f"vibework-postgres-{timestamp}.dump", + f"fifa-postgres-{timestamp}.dump", + f"fifa-redis-{timestamp}.rdb", + f"n8n-online-{timestamp}.tar.gz", + "n8n-validation.json", +): + path = root / name + assets.append( + { + "name": name, + "sizeBytes": path.stat().st_size, + "sha256": sha256_file(path), + } + ) + +(root / "backup-manifest.json").write_text( + json.dumps( + { + "schemaVersion": "host188_product_backup_manifest_v1", + "sourceHost": "192.168.0.188", + "controllerHost": "192.168.0.110", + "timestamp": timestamp, + "assets": assets, + "secretValueExported": False, + "credentialsDecrypted": False, + "serviceStopped": False, + "offsiteWritePerformed": False, + }, + separators=(",", ":"), + ) + + "\n", + encoding="utf-8", +) +PY +} + +latest_snapshot_id() { + restic -r "${LOCAL_REPO}" snapshots --latest 1 --json \ + --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \ + python3 -c 'import json,sys; rows=json.load(sys.stdin); print(rows[-1].get("short_id","unknown") if rows else "unknown")' +} + +main() { + local start_time timestamp snapshot_id duration + start_time="$(date +%s)" + timestamp="$(date +%Y%m%d_%H%M%S)" + trap cleanup EXIT HUP INT TERM + trap on_error ERR + + require_commands + install -d -m 700 "${DUMP_DIR}" + log_info "Host188 product backup started" + + dump_postgres_container \ + "vibework-production-postgres-1" \ + "${DUMP_DIR}/vibework-postgres-${timestamp}.dump" + dump_postgres_container \ + "current-fifa2026-postgres-1" \ + "${DUMP_DIR}/fifa-postgres-${timestamp}.dump" + dump_redis_container \ + "current-fifa2026-redis-1" \ + "${DUMP_DIR}/fifa-redis-${timestamp}.rdb" + backup_n8n_online "${timestamp}" "${DUMP_DIR}/n8n-online-${timestamp}.tar.gz" + write_manifest "${timestamp}" + + if ! restic -r "${LOCAL_REPO}" snapshots --json --password-file "${RESTIC_PASSWORD_FILE}" >/dev/null 2>&1; then + restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" + fi + restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" \ + --password-file "${RESTIC_PASSWORD_FILE}" \ + --tag "service:${SERVICE}" --tag "source:host188" --tag "timestamp:${timestamp}" + snapshot_id="$(latest_snapshot_id)" + verify_backup "${LOCAL_REPO}" "${snapshot_id}" + + duration="$(($(date +%s) - start_time))" + log_success "Host188 product backup completed snapshot=${snapshot_id} duration=${duration}s" + if [ "${BACKUP_NOTIFY_SUCCESS:-0}" = "1" ]; then + notify_clawbot "success" "${SERVICE}" "Host188 product backup completed and verified" "${duration}" + fi +} + +main "$@" diff --git a/scripts/backup/backup-offsite-readiness-gate.sh b/scripts/backup/backup-offsite-readiness-gate.sh index 237bdb685..a260286e2 100755 --- a/scripts/backup/backup-offsite-readiness-gate.sh +++ b/scripts/backup/backup-offsite-readiness-gate.sh @@ -23,7 +23,7 @@ REQUIRE_CONFIGURED=0 REQUIRE_ESCROW=0 NO_COLOR=0 SMALL_REPOS="ai-artifacts public-routes" -EXPECTED_REPOS="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes" +EXPECTED_REPOS="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes host188-products" pass=0 warn=0 diff --git a/scripts/backup/check-backup-integrity.sh b/scripts/backup/check-backup-integrity.sh index 8f5a0a267..76176f4cc 100755 --- a/scripts/backup/check-backup-integrity.sh +++ b/scripts/backup/check-backup-integrity.sh @@ -22,7 +22,7 @@ MAX_SAMPLE_BYTES="${RESTIC_RESTORE_DRILL_MAX_SAMPLE_BYTES:-20971520}" STATE_DIR="${BACKUP_BASE}/integrity" LOG_FILE="${BACKUP_LOG_DIR}/backup-integrity.log" RESTORE_DIR="/tmp/backup-restore-drill-$$" -REPOS_DEFAULT="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes" +REPOS_DEFAULT="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes host188-products" REPOS="${BACKUP_INTEGRITY_REPOS:-${REPOS_DEFAULT}}" while [ "$#" -gt 0 ]; do diff --git a/scripts/backup/enforce-latest-only-retention.sh b/scripts/backup/enforce-latest-only-retention.sh index 6eb3730e5..772323ff1 100755 --- a/scripts/backup/enforce-latest-only-retention.sh +++ b/scripts/backup/enforce-latest-only-retention.sh @@ -11,7 +11,7 @@ set -euo pipefail source "$(dirname "$0")/common.sh" -EXPECTED_REPOS_DEFAULT="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes" +EXPECTED_REPOS_DEFAULT="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes host188-products" REPOS="${BACKUP_RETENTION_REPOS:-${EXPECTED_REPOS_DEFAULT}}" main() { diff --git a/scripts/backup/sync-offsite-backups.sh b/scripts/backup/sync-offsite-backups.sh index 19e9e6614..5cf31d5a7 100755 --- a/scripts/backup/sync-offsite-backups.sh +++ b/scripts/backup/sync-offsite-backups.sh @@ -43,7 +43,7 @@ OFFSITE_SYNC_FULL_MIN_RUNWAY_MINUTES="${OFFSITE_SYNC_FULL_MIN_RUNWAY_MINUTES:-27 OFFSITE_SYNC_BACKUP_SCHEDULE_MINUTES="${OFFSITE_SYNC_BACKUP_SCHEDULE_MINUTES:-120 480 840 1200}" OFFSITE_SYNC_NOTIFY_SKIPPED="${OFFSITE_SYNC_NOTIFY_SKIPPED:-0}" OFFSITE_SYNC_NOTIFY_SUCCESS="${OFFSITE_SYNC_NOTIFY_SUCCESS:-0}" -EXPECTED_REPOS_DEFAULT="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes" +EXPECTED_REPOS_DEFAULT="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes host188-products" REPOS="${OFFSITE_REPOS:-${EXPECTED_REPOS_DEFAULT}}" DRY_RUN_ARGS=() diff --git a/scripts/backup/tests/test_110_backup_runtime_deploy_playbook.py b/scripts/backup/tests/test_110_backup_runtime_deploy_playbook.py index 27c1b6d05..ec3f91ed0 100644 --- a/scripts/backup/tests/test_110_backup_runtime_deploy_playbook.py +++ b/scripts/backup/tests/test_110_backup_runtime_deploy_playbook.py @@ -15,10 +15,11 @@ EXPECTED_FILES = { "backup-awoooi-frequent.sh", "backup-clawbot.sh", "backup-sentry.sh", + "backup-host188-products.sh", } -def test_backup_runtime_deploy_is_fixed_to_host_110_and_six_files() -> None: +def test_backup_runtime_deploy_is_fixed_to_host_110_and_seven_files() -> None: plays = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8")) assert len(plays) == 1 play = plays[0] diff --git a/scripts/backup/tests/test_backup_all_required_domains.py b/scripts/backup/tests/test_backup_all_required_domains.py index 41d853d01..1cb1bf93f 100644 --- a/scripts/backup/tests/test_backup_all_required_domains.py +++ b/scripts/backup/tests/test_backup_all_required_domains.py @@ -22,15 +22,16 @@ REQUIRED_BACKUP_SCRIPTS = ( "backup-ai-artifacts.sh", "backup-configs.sh", "backup-public-routes.sh", + "backup-host188-products.sh", ) def test_backup_all_runs_every_required_domain_once() -> None: source = BACKUP_ALL.read_text(encoding="utf-8") - assert "local total=13" in source + assert "local total=14" in source assert [int(value) for value in re.findall(r">>> \[(\d+)/\$\{total\}\]", source)] == list( - range(1, 14) + range(1, 15) ) for script in REQUIRED_BACKUP_SCRIPTS: assert source.count(f"/backup/scripts/{script}") == 1 diff --git a/scripts/backup/tests/test_backup_host188_products_contract.py b/scripts/backup/tests/test_backup_host188_products_contract.py new file mode 100644 index 000000000..4b4b2278e --- /dev/null +++ b/scripts/backup/tests/test_backup_host188_products_contract.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "backup" / "backup-host188-products.sh" +BACKUP_ALL = ROOT / "scripts" / "backup" / "backup-all.sh" +INTEGRITY = ROOT / "scripts" / "backup" / "check-backup-integrity.sh" +OFFSITE_SCRIPTS = ( + ROOT / "scripts" / "backup" / "sync-offsite-backups.sh", + ROOT / "scripts" / "backup" / "backup-offsite-readiness-gate.sh", + ROOT / "scripts" / "backup" / "verify-offsite-full-sync.sh", + ROOT / "scripts" / "backup" / "enforce-latest-only-retention.sh", +) +EXPORTER = ROOT / "scripts" / "ops" / "backup-health-textfile-exporter.py" +AGENT99 = ROOT / "agent99-control-plane.ps1" +FULL_PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml" +BOUNDED_PLAYBOOK = ( + ROOT / "infra" / "ansible" / "playbooks" / "110-backup-runtime-deploy.yml" +) +REBOOT_READINESS = ( + ROOT + / "scripts" + / "reboot-recovery" + / "reboot-recovery-readiness-audit.sh" +) + + +def test_host188_product_backup_has_fixed_sources_and_online_verifiers() -> None: + source = SCRIPT.read_text(encoding="utf-8") + + for expected in ( + 'REMOTE_HOST="ollama@192.168.0.188"', + "vibework-production-postgres-1", + "current-fifa2026-postgres-1", + "current-fifa2026-redis-1", + "n8n", + "pg_dump -Fc", + "PGDMP", + "pg_restore --list", + "redis-cli --rdb", + 'magic}" != "REDIS"', + "sqlite3.OPEN_READONLY", + "db.backup", + "PRAGMA integrity_check", + "n8n export:workflow --backup", + "n8n export:credentials --backup", + 'LOCAL_REPO="${BACKUP_BASE}/host188-products"', + "verify_backup", + ): + assert expected in source + + +def test_host188_product_backup_preserves_secret_and_runtime_boundaries() -> None: + source = SCRIPT.read_text(encoding="utf-8") + + assert "umask 077" in source + assert '"credentialsDecrypted": False' in source + assert '"offsiteWritePerformed": False' in source + assert "tarfile.open" in source + assert "n8n_archive_unsafe_path" in source + assert "REDISCLI_AUTH" in source + assert 'if [ "${BACKUP_NOTIFY_SUCCESS:-0}" = "1" ]' in source + assert "handle.read(1024 * 1024)" in source + + for forbidden in ( + "--decrypted", + "docker stop", + "docker restart", + "docker pause", + "docker compose down", + "docker inspect -f '{{range .Config.Env}}", + "rclone ", + "sync-offsite-backups.sh", + "rm -rf /home/node/.n8n", + "path.read_bytes()", + ): + assert forbidden not in source + + +def test_host188_product_backup_is_wired_into_health_integrity_and_offsite() -> None: + assert "/backup/scripts/backup-host188-products.sh" in BACKUP_ALL.read_text( + encoding="utf-8" + ) + assert "host188-products" in INTEGRITY.read_text(encoding="utf-8") + for path in OFFSITE_SCRIPTS: + assert "host188-products" in path.read_text(encoding="utf-8") + + exporter = EXPORTER.read_text(encoding="utf-8") + assert '"backup-host188-products.sh"' in exporter + assert '("host188_products", "/backup/host188-products", 48)' in exporter + + agent99 = AGENT99.read_text(encoding="utf-8") + assert 'name = "host188-products"' in agent99 + assert 'path = "/backup/host188-products/snapshots"' in agent99 + assert "maxAgeMinutes = 1800; required = $true" in agent99 + + for path in (FULL_PLAYBOOK, BOUNDED_PLAYBOOK): + assert "backup-host188-products.sh" in path.read_text(encoding="utf-8") + assert "backup-host188-products.sh" in REBOOT_READINESS.read_text(encoding="utf-8") diff --git a/scripts/backup/verify-offsite-full-sync.sh b/scripts/backup/verify-offsite-full-sync.sh index 5e09d8845..4a0f60e07 100644 --- a/scripts/backup/verify-offsite-full-sync.sh +++ b/scripts/backup/verify-offsite-full-sync.sh @@ -2,7 +2,7 @@ # ============================================================================= # WOOO AIOps - Offsite full sync verifier # 2026-05-19 ogt + Codex: full sync 後驗證 Google Drive/rclone 遠端仍符合 -# latest-only:13 個 repo 都可列出,且 snapshots/ 只保留 1 份。 +# latest-only:14 個 repo 都可列出,且 snapshots/ 只保留 1 份。 # # 規則: # - 只讀 Google Drive/rclone remote,不讀、不輸出 token 或 rclone.conf。 @@ -22,7 +22,7 @@ OFFSITE_DIR="${BACKUP_BASE}/offsite" TEXTFILE_DIR="${NODE_EXPORTER_TEXTFILE_DIR:-/home/wooo/node_exporter_textfiles}" TEXTFILE_PATH="${TEXTFILE_DIR}/offsite_full_sync_verify.prom" HOST_LABEL="${AIOPS_HOST_LABEL:-110}" -EXPECTED_REPOS="${OFFSITE_REPOS:-awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes}" +EXPECTED_REPOS="${OFFSITE_REPOS:-awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes host188-products}" MAX_AGE_HOURS="${OFFSITE_FULL_VERIFY_MAX_AGE_HOURS:-48}" WRITE_TEXTFILE=0 NO_COLOR=0 diff --git a/scripts/ops/ansible-validate.sh b/scripts/ops/ansible-validate.sh index 36a1ef359..4c03d54be 100755 --- a/scripts/ops/ansible-validate.sh +++ b/scripts/ops/ansible-validate.sh @@ -64,6 +64,7 @@ bash -n \ scripts/backup/backup-sentry.sh \ scripts/backup/backup-ai-artifacts.sh \ scripts/backup/backup-public-routes.sh \ + scripts/backup/backup-host188-products.sh \ scripts/backup/configure-offsite-rclone.sh \ scripts/backup/configure-offsite-b2.sh \ scripts/backup/sync-offsite-backups.sh \ diff --git a/scripts/ops/backup-health-textfile-exporter.py b/scripts/ops/backup-health-textfile-exporter.py index 05379b4bd..ad2063111 100755 --- a/scripts/ops/backup-health-textfile-exporter.py +++ b/scripts/ops/backup-health-textfile-exporter.py @@ -1192,6 +1192,7 @@ def _collect_110(host: str) -> list[str]: "backup-sentry.sh", "backup-ai-artifacts.sh", "backup-public-routes.sh", + "backup-host188-products.sh", "configure-offsite-rclone.sh", "configure-offsite-b2.sh", "sync-offsite-backups.sh", @@ -1230,6 +1231,7 @@ def _collect_110(host: str) -> list[str]: ("clawbot", "/backup/clawbot", 48), ("ai_artifacts", "/backup/ai-artifacts", 48), ("public_routes", "/backup/public-routes", 168), + ("host188_products", "/backup/host188-products", 48), ]: timestamp, count = _latest_restic_snapshot(repo) age = int(time.time()) - timestamp if timestamp else 0 @@ -1249,11 +1251,11 @@ def _collect_110(host: str) -> list[str]: coverage_domains = { "host": ["configs"], - "database": ["awoooi_db"], - "website": ["public_routes", "momo"], - "service": ["gitea", "harbor", "sentry", "monitoring", "signoz"], + "database": ["awoooi_db", "host188_products"], + "website": ["public_routes", "momo", "host188_products"], + "service": ["gitea", "harbor", "sentry", "monitoring", "signoz", "host188_products"], "package": ["configs", "ai_artifacts"], - "tool": ["ai_artifacts", "open_webui", "clawbot", "langfuse"], + "tool": ["ai_artifacts", "open_webui", "clawbot", "langfuse", "host188_products"], "log": ["monitoring", "signoz", "sentry"], } for domain, jobs in coverage_domains.items(): diff --git a/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh b/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh index 9b0527662..43752b81b 100755 --- a/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh +++ b/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh @@ -278,6 +278,7 @@ require_file scripts/backup/backup-momo-188-pg.sh "188 momo PostgreSQL backup sc require_file scripts/backup/backup-sentry.sh "Sentry dedicated data backup" require_file scripts/backup/backup-ai-artifacts.sh "AI artifacts and Ollama manifest backup" require_file scripts/backup/backup-public-routes.sh "Public routes DNS/TLS evidence backup" +require_file scripts/backup/backup-host188-products.sh "Host188 product data-plane backup" require_file scripts/backup/configure-offsite-rclone.sh "Offsite Google Drive/rclone host-local config helper" require_pattern "create-root-remote" scripts/backup/configure-offsite-rclone.sh "Offsite Google Drive root-scoped remote helper" require_pattern "gdrive_awoooi_restic" docs/runbooks/OFFSITE-BACKUP-ESCROW-RUNBOOK.md "Offsite Google Drive root-scoped remote runbook"