173 lines
6.6 KiB
Bash
Executable File
173 lines
6.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Generate Gitea Git bundles on host 110 from the local Gitea container, sync
|
|
# the complete set to 188 without deletion, then run an isolated sample restore.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
LOCAL_OUTPUT_ROOT="${GITEA_BUNDLE_LOCAL_ROOT:-/backup/gitea-bundles}"
|
|
LOCAL_CONTAINER="${GITEA_BUNDLE_LOCAL_CONTAINER:-gitea}"
|
|
REMOTE_HOST="${GITEA_BUNDLE_REMOTE_HOST:-ollama@192.168.0.188}"
|
|
REMOTE_OUTPUT_ROOT="${GITEA_BUNDLE_REMOTE_ROOT:-/home/ollama/backup/110/gitea/git-bundles}"
|
|
REMOTE_VERIFY_SCRIPT="${GITEA_BUNDLE_REMOTE_VERIFY_SCRIPT:-/home/ollama/scripts/gitea-bundle-sample-restore-dry-run.sh}"
|
|
REMOTE_EXPORTER_SCRIPT="${GITEA_BUNDLE_REMOTE_EXPORTER_SCRIPT:-/home/ollama/scripts/backup-health-textfile-exporter.py}"
|
|
BUNDLE_SCRIPT="${GITEA_BUNDLE_SCRIPT:-${SCRIPT_DIR}/gitea-repo-bundle-backup.sh}"
|
|
STATUS_FILE="${GITEA_BUNDLE_SYNC_STATUS_FILE:-/backup/status/gitea-bundle-sync-188.status}"
|
|
LOCK_FILE="${GITEA_BUNDLE_SYNC_LOCK_FILE:-${LOCAL_OUTPUT_ROOT}/.sync-188.lock}"
|
|
START_TIME="$(date +%s)"
|
|
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
|
phase="preflight"
|
|
promoted=0
|
|
previous_complete=""
|
|
previous_latest=""
|
|
expected_count=0
|
|
stamp="unknown"
|
|
|
|
mkdir -p "${LOCAL_OUTPUT_ROOT}" "$(dirname "${STATUS_FILE}")"
|
|
exec 9>"${LOCK_FILE}"
|
|
if ! flock -n 9; then
|
|
printf 'SKIP Gitea bundle sync already running\n'
|
|
exit 0
|
|
fi
|
|
|
|
write_status() {
|
|
local success="$1"
|
|
local reason="$2"
|
|
local temp_status="${STATUS_FILE}.tmp.$$"
|
|
{
|
|
printf 'timestamp=%s\n' "$(date +%s)"
|
|
printf 'success=%s\n' "${success}"
|
|
printf 'reason=%s\n' "${reason}"
|
|
printf 'phase=%s\n' "${phase}"
|
|
printf 'snapshot_stamp=%s\n' "${stamp}"
|
|
printf 'expected_repo_count=%s\n' "${expected_count}"
|
|
printf 'duration_seconds=%s\n' "$(($(date +%s) - START_TIME))"
|
|
} >"${temp_status}"
|
|
chmod 600 "${temp_status}"
|
|
mv "${temp_status}" "${STATUS_FILE}"
|
|
}
|
|
|
|
refresh_remote_exporter() {
|
|
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
|
"AIOPS_HOST_LABEL=188 NODE_EXPORTER_TEXTFILE_DIR=/home/ollama/node_exporter_textfiles '${REMOTE_EXPORTER_SCRIPT}'" \
|
|
>/dev/null 2>&1 || true
|
|
}
|
|
|
|
rollback_remote_links() {
|
|
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
|
python3 - "${REMOTE_OUTPUT_ROOT}" "${previous_complete}" "${previous_latest}" <<'PY'
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
root = Path(sys.argv[1])
|
|
previous = {
|
|
root / "latest-private-complete": sys.argv[2],
|
|
root / "latest": sys.argv[3],
|
|
}
|
|
for link, target in previous.items():
|
|
if target:
|
|
temporary = link.with_name(f".{link.name}.rollback.{os.getpid()}")
|
|
temporary.unlink(missing_ok=True)
|
|
temporary.symlink_to(target)
|
|
os.replace(temporary, link)
|
|
elif link.is_symlink():
|
|
link.unlink()
|
|
PY
|
|
}
|
|
|
|
finish() {
|
|
local rc=$?
|
|
trap - EXIT
|
|
if [ "${rc}" -eq 0 ] && [ "${phase}" = "complete" ]; then
|
|
write_status 1 verified
|
|
else
|
|
if [ "${promoted}" -eq 1 ]; then
|
|
rollback_remote_links >/dev/null 2>&1 || true
|
|
fi
|
|
refresh_remote_exporter
|
|
write_status 0 "${phase}_failed"
|
|
fi
|
|
exit "${rc}"
|
|
}
|
|
trap finish EXIT
|
|
|
|
[ -x "${BUNDLE_SCRIPT}" ] || { printf 'missing bundle script: %s\n' "${BUNDLE_SCRIPT}" >&2; exit 1; }
|
|
command -v docker >/dev/null
|
|
command -v rsync >/dev/null
|
|
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
|
"test -x '${REMOTE_VERIFY_SCRIPT}' && test -x '${REMOTE_EXPORTER_SCRIPT}'"
|
|
|
|
phase="bundle_local_container"
|
|
"${BUNDLE_SCRIPT}" \
|
|
--local-container "${LOCAL_CONTAINER}" \
|
|
--output-root "${LOCAL_OUTPUT_ROOT}"
|
|
|
|
local_snapshot="$(readlink -f "${LOCAL_OUTPUT_ROOT}/latest-private-complete")"
|
|
stamp="$(basename "${local_snapshot}")"
|
|
manifest="${local_snapshot}/manifest.tsv"
|
|
[ -f "${manifest}" ]
|
|
expected_count="$(awk -F '\t' 'NR > 1 {total += 1; if ($2 != "ok") failed += 1} END {if (failed > 0 || total == 0) exit 1; print total}' "${manifest}")"
|
|
|
|
phase="sync_complete_set_to_188"
|
|
previous_complete="$(ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" "readlink -f '${REMOTE_OUTPUT_ROOT}/latest-private-complete' 2>/dev/null || true")"
|
|
previous_latest="$(ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" "readlink -f '${REMOTE_OUTPUT_ROOT}/latest' 2>/dev/null || true")"
|
|
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" "mkdir -p '${REMOTE_OUTPUT_ROOT}/${stamp}'"
|
|
rsync -a --checksum \
|
|
-e "ssh ${SSH_OPTS[*]}" \
|
|
"${local_snapshot}/" \
|
|
"${REMOTE_HOST}:${REMOTE_OUTPUT_ROOT}/${stamp}/"
|
|
|
|
phase="remote_manifest_verification"
|
|
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
|
python3 - "${REMOTE_OUTPUT_ROOT}/${stamp}" "${expected_count}" <<'PY'
|
|
import csv
|
|
import hashlib
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
snapshot = Path(sys.argv[1])
|
|
expected = int(sys.argv[2])
|
|
with (snapshot / "manifest.tsv").open(encoding="utf-8", newline="") as handle:
|
|
rows = list(csv.DictReader(handle, delimiter="\t"))
|
|
ok = len(rows) == expected and expected > 0
|
|
verified_checksums = 0
|
|
for row in rows:
|
|
bundle = snapshot / Path(row.get("bundle", "")).name
|
|
checksum = snapshot / Path(row.get("checksum", "")).name
|
|
row_ok = row.get("status") == "ok" and bundle.is_file() and checksum.is_file()
|
|
if row_ok:
|
|
expected_digest = checksum.read_text(encoding="utf-8", errors="replace").split()[0].lower()
|
|
digest = hashlib.sha256()
|
|
with bundle.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
row_ok = len(expected_digest) == 64 and digest.hexdigest() == expected_digest
|
|
if row_ok:
|
|
verified_checksums += 1
|
|
ok = ok and row_ok
|
|
print(
|
|
f"remote_manifest_expected={expected} remote_manifest_rows={len(rows)} "
|
|
f"remote_manifest_checksum_ok={verified_checksums} remote_manifest_ok={int(ok)}"
|
|
)
|
|
raise SystemExit(0 if ok else 1)
|
|
PY
|
|
|
|
phase="remote_promote_complete"
|
|
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
|
"for link in '${REMOTE_OUTPUT_ROOT}/latest-private-complete' '${REMOTE_OUTPUT_ROOT}/latest'; do test ! -e \"\$link\" || test -L \"\$link\" || exit 1; done"
|
|
promoted=1
|
|
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
|
"ln -sfn '${REMOTE_OUTPUT_ROOT}/${stamp}' '${REMOTE_OUTPUT_ROOT}/latest-private-complete' && ln -sfn '${REMOTE_OUTPUT_ROOT}/${stamp}' '${REMOTE_OUTPUT_ROOT}/latest'"
|
|
|
|
phase="remote_sample_restore_fsck"
|
|
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
|
"AIOPS_GITEA_BUNDLE_ROOT='${REMOTE_OUTPUT_ROOT}/latest-private-complete' AIOPS_HOST_LABEL=188 '${REMOTE_VERIFY_SCRIPT}' --write-textfile"
|
|
|
|
phase="refresh_remote_backup_health"
|
|
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
|
"AIOPS_HOST_LABEL=188 NODE_EXPORTER_TEXTFILE_DIR=/home/ollama/node_exporter_textfiles '${REMOTE_EXPORTER_SCRIPT}'"
|
|
|
|
phase="complete"
|
|
printf 'GITEA_BUNDLE_SYNC_COMPLETE=1 expected=%s evidence=%s\n' "${expected_count}" "${STATUS_FILE}"
|