wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

feat(backup): protect host188 product data

Browse Source
This commit is contained in:
Your Name
2026-07-18 18:39:35 +08:00
parent 592a195676
commit 420738514e
16 changed files with 438 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
scripts/backup/backup-all.sh
View File

@@ -14,7 +14,7 @@ source "$(dirname "$0")/common.sh"
main() {
local start_time=$(date +%s)
local failed=0
local total=13
local total=14
log_info "╔══════════════════════════════════════════════════════════════╗"
log_info "║ WOOO AIOps - 全服務備份開始 (v3.0) ║"
@@ -143,6 +143,15 @@ main() {
failed=$((failed+1))
fi
# Host188 product data planes: VibeWork/FIFA PostgreSQL, FIFA Redis, and n8n.
log_info ">>> [14/${total}] 備份 Host188 產品資料..."
if /backup/scripts/backup-host188-products.sh; then
log_success " Host188 產品資料備份成功"
else
log_error " Host188 產品資料備份失敗"
failed=$((failed+1))
fi
local end_time=$(date +%s)
local duration=$((end_time - start_time))

303
scripts/backup/backup-host188-products.sh Normal file
View File

@@ -0,0 +1,303 @@
#!/usr/bin/env bash
# Fixed n8n, VibeWork, and FIFA data-plane backup from host 188 to host 110.
set -Eeuo pipefail
umask 077
source "$(dirname "$0")/common.sh"
SERVICE="host188-products"
REMOTE_HOST="ollama@192.168.0.188"
LOCAL_REPO="${BACKUP_BASE}/host188-products"
DUMP_DIR="/tmp/host188-products-backup-$$"
MIN_POSTGRES_DUMP_BYTES="${MIN_POSTGRES_DUMP_BYTES:-4096}"
SSH_OPTIONS=(
-o BatchMode=yes
-o StrictHostKeyChecking=yes
-o ConnectTimeout=8
-o ServerAliveInterval=10
-o ServerAliveCountMax=2
)
cleanup() {
rm -rf "${DUMP_DIR}"
}
on_error() {
local status=$?
trap - ERR
set +e
log_error "Host188 product backup failed (exit=${status})"
notify_clawbot "failed" "${SERVICE}" "Host188 product backup failed; inspect the controller receipt" 0
cleanup
exit "${status}"
}
require_commands() {
local command_name
for command_name in restic ssh python3; do
if ! command -v "${command_name}" >/dev/null 2>&1; then
log_error "Required command missing: ${command_name}"
return 69
fi
done
if [ ! -r "${RESTIC_PASSWORD_FILE}" ]; then
log_error "Restic password reference is unavailable"
return 69
fi
}
assert_fixed_container() {
case "$1" in
vibework-production-postgres-1|current-fifa2026-postgres-1|current-fifa2026-redis-1|n8n) return 0 ;;
*) return 64 ;;
esac
}
dump_redis_container() {
local container="$1"
local output_file="$2"
local temporary_file="${output_file}.tmp"
local remote_tmp="/tmp/agent99-fifa-redis-backup-$$.rdb"
local inner_command remote_command size_bytes magic
assert_fixed_container "${container}"
printf -v inner_command \
'set -eu; umask 077; tmp=%q; rm -f "$tmp"; trap '\''rm -f "$tmp"'\'' EXIT HUP INT TERM; if [ -n "${REDIS_PASSWORD:-}" ]; then export REDISCLI_AUTH="$REDIS_PASSWORD"; fi; redis-cli --rdb "$tmp" >/dev/null; test -s "$tmp"; cat "$tmp"' \
"${remote_tmp}"
printf -v remote_command \
'test "$(docker inspect -f '\''{{.State.Running}}'\'' %q 2>/dev/null)" = true && docker exec %q sh -eu -c %q' \
"${container}" "${container}" "${inner_command}"
if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${remote_command}" >"${temporary_file}"; then
log_error "Redis RDB dump failed: ${container}"
return 1
fi
size_bytes="$(stat -c%s "${temporary_file}" 2>/dev/null || echo 0)"
magic="$(head -c 5 "${temporary_file}" 2>/dev/null || true)"
if [ "${size_bytes}" -lt 9 ] || [ "${magic}" != "REDIS" ]; then
log_error "Redis RDB validation failed: ${container} size=${size_bytes}"
return 1
fi
mv "${temporary_file}" "${output_file}"
}
dump_postgres_container() {
local container="$1"
local output_file="$2"
local temporary_file="${output_file}.tmp"
local dump_inner dump_command verify_command size_bytes magic
assert_fixed_container "${container}"
dump_inner='exec pg_dump -Fc -U "$POSTGRES_USER" -d "$POSTGRES_DB" --no-password --no-owner --no-acl'
printf -v dump_command \
'test "$(docker inspect -f '\''{{.State.Running}}'\'' %q 2>/dev/null)" = true && docker exec %q sh -eu -c %q' \
"${container}" "${container}" "${dump_inner}"
if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${dump_command}" >"${temporary_file}"; then
log_error "PostgreSQL dump failed: ${container}"
return 1
fi
size_bytes="$(stat -c%s "${temporary_file}" 2>/dev/null || echo 0)"
magic="$(head -c 5 "${temporary_file}" 2>/dev/null || true)"
if [ "${size_bytes}" -lt "${MIN_POSTGRES_DUMP_BYTES}" ] || [ "${magic}" != "PGDMP" ]; then
log_error "PostgreSQL dump validation failed: ${container} size=${size_bytes}"
return 1
fi
printf -v verify_command 'docker exec -i %q pg_restore --list >/dev/null' "${container}"
if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${verify_command}" <"${temporary_file}"; then
log_error "PostgreSQL archive verifier failed: ${container}"
return 1
fi
mv "${temporary_file}" "${output_file}"
}
backup_n8n_online() {
local timestamp="$1"
local output_file="$2"
local temporary_file="${output_file}.tmp"
local remote_tmp="/tmp/agent99-n8n-backup-${timestamp}"
local node_script inner_command remote_command validation_dir
assert_fixed_container "n8n"
node_script='const p=require.resolve("sqlite3",{paths:["/usr/local/lib/node_modules/n8n"]});const sqlite3=require(p);const src=process.env.N8N_SQLITE_SOURCE;const dst=process.env.N8N_SQLITE_DEST;const db=new sqlite3.Database(src,sqlite3.OPEN_READONLY,(openError)=>{if(openError){console.error("sqlite_open_failed");process.exit(2);}db.backup(dst,(backupError)=>{db.close((closeError)=>{if(backupError||closeError){console.error("sqlite_backup_failed");process.exit(3);}});});});'
printf -v inner_command \
'set -eu; umask 077; tmp=%q; rm -rf "$tmp"; trap '\''rm -rf "$tmp"'\'' EXIT HUP INT TERM; mkdir -p "$tmp/workflows" "$tmp/credentials"; N8N_SQLITE_SOURCE=/home/node/.n8n/database.sqlite N8N_SQLITE_DEST="$tmp/database.sqlite" node -e %q >/dev/null 2>&1; n8n export:workflow --backup --output="$tmp/workflows" >/dev/null 2>&1; n8n export:credentials --backup --output="$tmp/credentials" >/dev/null 2>&1; tar czf - -C "$tmp" .' \
"${remote_tmp}" "${node_script}"
printf -v remote_command \
'test "$(docker inspect -f '\''{{.State.Running}}'\'' n8n 2>/dev/null)" = true && docker exec n8n sh -eu -c %q' \
"${inner_command}"
if ! ssh "${SSH_OPTIONS[@]}" "${REMOTE_HOST}" "${remote_command}" >"${temporary_file}"; then
log_error "n8n online backup failed"
return 1
fi
validation_dir="${DUMP_DIR}/n8n-validation"
install -d -m 700 "${validation_dir}"
python3 - "${temporary_file}" "${validation_dir}" "${DUMP_DIR}/n8n-validation.json" <<'PY'
import json
import sqlite3
import sys
import tarfile
from pathlib import Path, PurePosixPath
archive_path = Path(sys.argv[1])
root = Path(sys.argv[2])
output = Path(sys.argv[3])
try:
with tarfile.open(archive_path, "r:gz") as archive:
members = archive.getmembers()
for member in members:
path = PurePosixPath(member.name)
if path.is_absolute() or ".." in path.parts:
raise SystemExit("n8n_archive_unsafe_path")
if not (member.isfile() or member.isdir()):
raise SystemExit("n8n_archive_unsafe_member")
archive.extractall(root, members=members)
except (tarfile.TarError, OSError):
raise SystemExit("n8n_archive_invalid")
database = root / "database.sqlite"
if not database.is_file() or database.stat().st_size <= 0:
raise SystemExit("n8n_sqlite_backup_missing")
with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection:
integrity = connection.execute("PRAGMA integrity_check").fetchone()[0]
if integrity != "ok":
raise SystemExit("n8n_sqlite_integrity_failed")
counts = {}
for category in ("workflows", "credentials"):
directory = root / category
if not directory.is_dir():
raise SystemExit(f"n8n_{category}_directory_missing")
rows = list(directory.glob("*.json"))
for path in rows:
json.loads(path.read_text(encoding="utf-8"))
counts[category] = len(rows)
output.write_text(
json.dumps(
{
"schemaVersion": "n8n_backup_validation_v1",
"sqliteIntegrity": "ok",
"workflowFiles": counts["workflows"],
"credentialFiles": counts["credentials"],
"credentialsDecrypted": False,
},
separators=(",", ":"),
)
+ "\n",
encoding="utf-8",
)
PY
rm -rf "${validation_dir}"
mv "${temporary_file}" "${output_file}"
}
write_manifest() {
local timestamp="$1"
python3 - "${DUMP_DIR}" "${timestamp}" <<'PY'
import hashlib
import json
import sys
from pathlib import Path
root = Path(sys.argv[1])
timestamp = sys.argv[2]
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
assets = []
for name in (
f"vibework-postgres-{timestamp}.dump",
f"fifa-postgres-{timestamp}.dump",
f"fifa-redis-{timestamp}.rdb",
f"n8n-online-{timestamp}.tar.gz",
"n8n-validation.json",
):
path = root / name
assets.append(
{
"name": name,
"sizeBytes": path.stat().st_size,
"sha256": sha256_file(path),
}
)
(root / "backup-manifest.json").write_text(
json.dumps(
{
"schemaVersion": "host188_product_backup_manifest_v1",
"sourceHost": "192.168.0.188",
"controllerHost": "192.168.0.110",
"timestamp": timestamp,
"assets": assets,
"secretValueExported": False,
"credentialsDecrypted": False,
"serviceStopped": False,
"offsiteWritePerformed": False,
},
separators=(",", ":"),
)
+ "\n",
encoding="utf-8",
)
PY
}
latest_snapshot_id() {
restic -r "${LOCAL_REPO}" snapshots --latest 1 --json \
--password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \
python3 -c 'import json,sys; rows=json.load(sys.stdin); print(rows[-1].get("short_id","unknown") if rows else "unknown")'
}
main() {
local start_time timestamp snapshot_id duration
start_time="$(date +%s)"
timestamp="$(date +%Y%m%d_%H%M%S)"
trap cleanup EXIT HUP INT TERM
trap on_error ERR
require_commands
install -d -m 700 "${DUMP_DIR}"
log_info "Host188 product backup started"
dump_postgres_container \
"vibework-production-postgres-1" \
"${DUMP_DIR}/vibework-postgres-${timestamp}.dump"
dump_postgres_container \
"current-fifa2026-postgres-1" \
"${DUMP_DIR}/fifa-postgres-${timestamp}.dump"
dump_redis_container \
"current-fifa2026-redis-1" \
"${DUMP_DIR}/fifa-redis-${timestamp}.rdb"
backup_n8n_online "${timestamp}" "${DUMP_DIR}/n8n-online-${timestamp}.tar.gz"
write_manifest "${timestamp}"
if ! restic -r "${LOCAL_REPO}" snapshots --json --password-file "${RESTIC_PASSWORD_FILE}" >/dev/null 2>&1; then
restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}"
fi
restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" \
--password-file "${RESTIC_PASSWORD_FILE}" \
--tag "service:${SERVICE}" --tag "source:host188" --tag "timestamp:${timestamp}"
snapshot_id="$(latest_snapshot_id)"
verify_backup "${LOCAL_REPO}" "${snapshot_id}"
duration="$(($(date +%s) - start_time))"
log_success "Host188 product backup completed snapshot=${snapshot_id} duration=${duration}s"
if [ "${BACKUP_NOTIFY_SUCCESS:-0}" = "1" ]; then
notify_clawbot "success" "${SERVICE}" "Host188 product backup completed and verified" "${duration}"
fi
}
main "$@"

2
scripts/backup/backup-offsite-readiness-gate.sh
View File

@@ -23,7 +23,7 @@ REQUIRE_CONFIGURED=0
REQUIRE_ESCROW=0
NO_COLOR=0
SMALL_REPOS="ai-artifacts public-routes"
EXPECTED_REPOS="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes"
EXPECTED_REPOS="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes host188-products"
pass=0
warn=0

2
scripts/backup/check-backup-integrity.sh
View File

@@ -22,7 +22,7 @@ MAX_SAMPLE_BYTES="${RESTIC_RESTORE_DRILL_MAX_SAMPLE_BYTES:-20971520}"
STATE_DIR="${BACKUP_BASE}/integrity"
LOG_FILE="${BACKUP_LOG_DIR}/backup-integrity.log"
RESTORE_DIR="/tmp/backup-restore-drill-$$"
REPOS_DEFAULT="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes"
REPOS_DEFAULT="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes host188-products"
REPOS="${BACKUP_INTEGRITY_REPOS:-${REPOS_DEFAULT}}"
while [ "$#" -gt 0 ]; do

2
scripts/backup/enforce-latest-only-retention.sh
View File

@@ -11,7 +11,7 @@ set -euo pipefail
source "$(dirname "$0")/common.sh"
EXPECTED_REPOS_DEFAULT="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes"
EXPECTED_REPOS_DEFAULT="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes host188-products"
REPOS="${BACKUP_RETENTION_REPOS:-${EXPECTED_REPOS_DEFAULT}}"
main() {

2
scripts/backup/sync-offsite-backups.sh
View File

@@ -43,7 +43,7 @@ OFFSITE_SYNC_FULL_MIN_RUNWAY_MINUTES="${OFFSITE_SYNC_FULL_MIN_RUNWAY_MINUTES:-27
OFFSITE_SYNC_BACKUP_SCHEDULE_MINUTES="${OFFSITE_SYNC_BACKUP_SCHEDULE_MINUTES:-120 480 840 1200}"
OFFSITE_SYNC_NOTIFY_SKIPPED="${OFFSITE_SYNC_NOTIFY_SKIPPED:-0}"
OFFSITE_SYNC_NOTIFY_SUCCESS="${OFFSITE_SYNC_NOTIFY_SUCCESS:-0}"
EXPECTED_REPOS_DEFAULT="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes"
EXPECTED_REPOS_DEFAULT="awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes host188-products"
REPOS="${OFFSITE_REPOS:-${EXPECTED_REPOS_DEFAULT}}"
DRY_RUN_ARGS=()

3
scripts/backup/tests/test_110_backup_runtime_deploy_playbook.py
View File

@@ -15,10 +15,11 @@ EXPECTED_FILES = {
"backup-awoooi-frequent.sh",
"backup-clawbot.sh",
"backup-sentry.sh",
"backup-host188-products.sh",
}
def test_backup_runtime_deploy_is_fixed_to_host_110_and_six_files() -> None:
def test_backup_runtime_deploy_is_fixed_to_host_110_and_seven_files() -> None:
plays = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8"))
assert len(plays) == 1
play = plays[0]

5
scripts/backup/tests/test_backup_all_required_domains.py
View File

@@ -22,15 +22,16 @@ REQUIRED_BACKUP_SCRIPTS = (
"backup-ai-artifacts.sh",
"backup-configs.sh",
"backup-public-routes.sh",
"backup-host188-products.sh",
)
def test_backup_all_runs_every_required_domain_once() -> None:
source = BACKUP_ALL.read_text(encoding="utf-8")
assert "local total=13" in source
assert "local total=14" in source
assert [int(value) for value in re.findall(r">>> \[(\d+)/\$\{total\}\]", source)] == list(
range(1, 14)
range(1, 15)
)
for script in REQUIRED_BACKUP_SCRIPTS:
assert source.count(f"/backup/scripts/{script}") == 1

101
scripts/backup/tests/test_backup_host188_products_contract.py Normal file
View File

@@ -0,0 +1,101 @@
from __future__ import annotations
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts" / "backup" / "backup-host188-products.sh"
BACKUP_ALL = ROOT / "scripts" / "backup" / "backup-all.sh"
INTEGRITY = ROOT / "scripts" / "backup" / "check-backup-integrity.sh"
OFFSITE_SCRIPTS = (
ROOT / "scripts" / "backup" / "sync-offsite-backups.sh",
ROOT / "scripts" / "backup" / "backup-offsite-readiness-gate.sh",
ROOT / "scripts" / "backup" / "verify-offsite-full-sync.sh",
ROOT / "scripts" / "backup" / "enforce-latest-only-retention.sh",
)
EXPORTER = ROOT / "scripts" / "ops" / "backup-health-textfile-exporter.py"
AGENT99 = ROOT / "agent99-control-plane.ps1"
FULL_PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
BOUNDED_PLAYBOOK = (
ROOT / "infra" / "ansible" / "playbooks" / "110-backup-runtime-deploy.yml"
)
REBOOT_READINESS = (
ROOT
/ "scripts"
/ "reboot-recovery"
/ "reboot-recovery-readiness-audit.sh"
)
def test_host188_product_backup_has_fixed_sources_and_online_verifiers() -> None:
source = SCRIPT.read_text(encoding="utf-8")
for expected in (
'REMOTE_HOST="ollama@192.168.0.188"',
"vibework-production-postgres-1",
"current-fifa2026-postgres-1",
"current-fifa2026-redis-1",
"n8n",
"pg_dump -Fc",
"PGDMP",
"pg_restore --list",
"redis-cli --rdb",
'magic}" != "REDIS"',
"sqlite3.OPEN_READONLY",
"db.backup",
"PRAGMA integrity_check",
"n8n export:workflow --backup",
"n8n export:credentials --backup",
'LOCAL_REPO="${BACKUP_BASE}/host188-products"',
"verify_backup",
):
assert expected in source
def test_host188_product_backup_preserves_secret_and_runtime_boundaries() -> None:
source = SCRIPT.read_text(encoding="utf-8")
assert "umask 077" in source
assert '"credentialsDecrypted": False' in source
assert '"offsiteWritePerformed": False' in source
assert "tarfile.open" in source
assert "n8n_archive_unsafe_path" in source
assert "REDISCLI_AUTH" in source
assert 'if [ "${BACKUP_NOTIFY_SUCCESS:-0}" = "1" ]' in source
assert "handle.read(1024 * 1024)" in source
for forbidden in (
"--decrypted",
"docker stop",
"docker restart",
"docker pause",
"docker compose down",
"docker inspect -f '{{range .Config.Env}}",
"rclone ",
"sync-offsite-backups.sh",
"rm -rf /home/node/.n8n",
"path.read_bytes()",
):
assert forbidden not in source
def test_host188_product_backup_is_wired_into_health_integrity_and_offsite() -> None:
assert "/backup/scripts/backup-host188-products.sh" in BACKUP_ALL.read_text(
encoding="utf-8"
)
assert "host188-products" in INTEGRITY.read_text(encoding="utf-8")
for path in OFFSITE_SCRIPTS:
assert "host188-products" in path.read_text(encoding="utf-8")
exporter = EXPORTER.read_text(encoding="utf-8")
assert '"backup-host188-products.sh"' in exporter
assert '("host188_products", "/backup/host188-products", 48)' in exporter
agent99 = AGENT99.read_text(encoding="utf-8")
assert 'name = "host188-products"' in agent99
assert 'path = "/backup/host188-products/snapshots"' in agent99
assert "maxAgeMinutes = 1800; required = $true" in agent99
for path in (FULL_PLAYBOOK, BOUNDED_PLAYBOOK):
assert "backup-host188-products.sh" in path.read_text(encoding="utf-8")
assert "backup-host188-products.sh" in REBOOT_READINESS.read_text(encoding="utf-8")

4
scripts/backup/verify-offsite-full-sync.sh
View File

@@ -2,7 +2,7 @@
# =============================================================================
# WOOO AIOps - Offsite full sync verifier
# 2026-05-19 ogt + Codex: full sync 後驗證 Google Drive/rclone 遠端仍符合
# latest-only：13 個 repo 都可列出，且 snapshots/ 只保留 1 份。
# latest-only：14 個 repo 都可列出，且 snapshots/ 只保留 1 份。
#
# 規則：
# - 只讀 Google Drive/rclone remote，不讀、不輸出 token 或 rclone.conf。
@@ -22,7 +22,7 @@ OFFSITE_DIR="${BACKUP_BASE}/offsite"
TEXTFILE_DIR="${NODE_EXPORTER_TEXTFILE_DIR:-/home/wooo/node_exporter_textfiles}"
TEXTFILE_PATH="${TEXTFILE_DIR}/offsite_full_sync_verify.prom"
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
EXPECTED_REPOS="${OFFSITE_REPOS:-awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes}"
EXPECTED_REPOS="${OFFSITE_REPOS:-awoooi configs gitea harbor momo langfuse monitoring signoz open-webui clawbot sentry ai-artifacts public-routes host188-products}"
MAX_AGE_HOURS="${OFFSITE_FULL_VERIFY_MAX_AGE_HOURS:-48}"
WRITE_TEXTFILE=0
NO_COLOR=0

1
scripts/ops/ansible-validate.sh
View File

@@ -64,6 +64,7 @@ bash -n \
scripts/backup/backup-sentry.sh \
scripts/backup/backup-ai-artifacts.sh \
scripts/backup/backup-public-routes.sh \
scripts/backup/backup-host188-products.sh \
scripts/backup/configure-offsite-rclone.sh \
scripts/backup/configure-offsite-b2.sh \
scripts/backup/sync-offsite-backups.sh \

10
scripts/ops/backup-health-textfile-exporter.py
View File

@@ -1192,6 +1192,7 @@ def _collect_110(host: str) -> list[str]:
"backup-sentry.sh",
"backup-ai-artifacts.sh",
"backup-public-routes.sh",
"backup-host188-products.sh",
"configure-offsite-rclone.sh",
"configure-offsite-b2.sh",
"sync-offsite-backups.sh",
@@ -1230,6 +1231,7 @@ def _collect_110(host: str) -> list[str]:
("clawbot", "/backup/clawbot", 48),
("ai_artifacts", "/backup/ai-artifacts", 48),
("public_routes", "/backup/public-routes", 168),
("host188_products", "/backup/host188-products", 48),
]:
timestamp, count = _latest_restic_snapshot(repo)
age = int(time.time()) - timestamp if timestamp else 0
@@ -1249,11 +1251,11 @@ def _collect_110(host: str) -> list[str]:
coverage_domains = {
"host": ["configs"],
"database": ["awoooi_db"],
"website": ["public_routes", "momo"],
"service": ["gitea", "harbor", "sentry", "monitoring", "signoz"],
"database": ["awoooi_db", "host188_products"],
"website": ["public_routes", "momo", "host188_products"],
"service": ["gitea", "harbor", "sentry", "monitoring", "signoz", "host188_products"],
"package": ["configs", "ai_artifacts"],
"tool": ["ai_artifacts", "open_webui", "clawbot", "langfuse"],
"tool": ["ai_artifacts", "open_webui", "clawbot", "langfuse", "host188_products"],
"log": ["monitoring", "signoz", "sentry"],
}
for domain, jobs in coverage_domains.items():

1
scripts/reboot-recovery/reboot-recovery-readiness-audit.sh
View File

@@ -278,6 +278,7 @@ require_file scripts/backup/backup-momo-188-pg.sh "188 momo PostgreSQL backup sc
require_file scripts/backup/backup-sentry.sh "Sentry dedicated data backup"
require_file scripts/backup/backup-ai-artifacts.sh "AI artifacts and Ollama manifest backup"
require_file scripts/backup/backup-public-routes.sh "Public routes DNS/TLS evidence backup"
require_file scripts/backup/backup-host188-products.sh "Host188 product data-plane backup"
require_file scripts/backup/configure-offsite-rclone.sh "Offsite Google Drive/rclone host-local config helper"
require_pattern "create-root-remote" scripts/backup/configure-offsite-rclone.sh "Offsite Google Drive root-scoped remote helper"
require_pattern "gdrive_awoooi_restic" docs/runbooks/OFFSITE-BACKUP-ESCROW-RUNBOOK.md "Offsite Google Drive root-scoped remote runbook"