feat(agent99): bind host110 backup runtime deploy
This commit is contained in:
@@ -9,6 +9,16 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then
|
||||
command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; }
|
||||
[ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; }
|
||||
(umask 077; : >> /tmp/agent99-host110-backup-runtime.lock)
|
||||
[ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; }
|
||||
exec 197>>/tmp/agent99-host110-backup-runtime.lock
|
||||
flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; }
|
||||
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
|
||||
fi
|
||||
|
||||
BACKUP_BASE="${BACKUP_BASE:-/backup}"
|
||||
OFFSITE_ENV_FILE="${BACKUP_OFFSITE_ENV_FILE:-${BACKUP_BASE}/scripts/offsite.env}"
|
||||
OFFSITE_DIR="${BACKUP_OFFSITE_STATUS_DIR:-${BACKUP_BASE}/offsite}"
|
||||
|
||||
@@ -1,5 +1,38 @@
|
||||
#\!/bin/bash
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
|
||||
# Production backup entrypoints share this stable gate. Agent99 takes the
|
||||
# exclusive side while promoting a complete, digest-bound runtime bundle.
|
||||
acquire_backup_runtime_shared_lock() {
|
||||
local caller_path="${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}"
|
||||
local lock_path="/tmp/agent99-host110-backup-runtime.lock"
|
||||
case "${caller_path}" in
|
||||
/backup/scripts/*) ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
[ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ] || return 0
|
||||
command -v flock >/dev/null 2>&1 || {
|
||||
echo "backup runtime gate unavailable: flock missing" >&2
|
||||
return 69
|
||||
}
|
||||
[ ! -L "${lock_path}" ] || {
|
||||
echo "backup runtime gate unsafe: lock is a symlink" >&2
|
||||
return 69
|
||||
}
|
||||
(umask 077; : >> "${lock_path}") || return 69
|
||||
[ -f "${lock_path}" ] && [ -O "${lock_path}" ] || return 69
|
||||
exec 197>>"${lock_path}"
|
||||
flock -s -w 60 197 || {
|
||||
echo "backup runtime deployment is active; retry later" >&2
|
||||
return 75
|
||||
}
|
||||
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
|
||||
}
|
||||
|
||||
acquire_backup_runtime_shared_lock || {
|
||||
runtime_lock_status=$?
|
||||
return "${runtime_lock_status}" 2>/dev/null || exit "${runtime_lock_status}"
|
||||
}
|
||||
# WOOO AIOps - 備份共用函式庫
|
||||
# 版本: 1.0.0
|
||||
# 建立日期: 2026-03-12
|
||||
|
||||
@@ -5,6 +5,16 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then
|
||||
command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; }
|
||||
[ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; }
|
||||
(umask 077; : >> /tmp/agent99-host110-backup-runtime.lock)
|
||||
[ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; }
|
||||
exec 197>>/tmp/agent99-host110-backup-runtime.lock
|
||||
flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; }
|
||||
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
|
||||
fi
|
||||
|
||||
DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}"
|
||||
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
|
||||
TEXTFILE_PATH="${AIOPS_GITEA_FULL_BACKUP_RESTORE_DRILL_TEXTFILE:-/home/wooo/node_exporter_textfiles/gitea_full_backup_restore_drill.prom}"
|
||||
|
||||
431
scripts/backup/host110-backup-runtime-executor.sh
Executable file
431
scripts/backup/host110-backup-runtime-executor.sh
Executable file
@@ -0,0 +1,431 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fixed Host110 executor for Windows99 Agent99 backup-runtime deployment.
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
readonly EXPECTED_HOST_IP="192.168.0.110"
|
||||
readonly EXPECTED_USER="wooo"
|
||||
readonly DEST_ROOT="/backup/scripts"
|
||||
readonly STATUS_ROOT="/backup/status"
|
||||
readonly STAGE_ROOT="/backup/.agent99-backup-runtime-stage"
|
||||
readonly ROLLBACK_ROOT="/backup/.agent99-backup-runtime-rollback"
|
||||
readonly LOCK_PATH="/tmp/agent99-host110-backup-runtime.lock"
|
||||
|
||||
readonly -a RUNTIME_FILES=(
|
||||
common.sh
|
||||
backup-all.sh
|
||||
backup-host188-products.sh
|
||||
verify-host188-products-backup.sh
|
||||
verify-host188-products-archive.py
|
||||
backup-gitea.sh
|
||||
gitea-full-backup-restore-drill.sh
|
||||
backup-configs.sh
|
||||
backup-awoooi.sh
|
||||
backup-awoooi-frequent.sh
|
||||
backup-clawbot.sh
|
||||
backup-sentry.sh
|
||||
check-backup-integrity.sh
|
||||
sync-offsite-backups.sh
|
||||
backup-offsite-readiness-gate.sh
|
||||
verify-offsite-full-sync.sh
|
||||
enforce-latest-only-retention.sh
|
||||
)
|
||||
|
||||
MODE=""
|
||||
SOURCE_REVISION=""
|
||||
RUN_ID=""
|
||||
SOURCE_STAGE=""
|
||||
SOURCE_HEAD=""
|
||||
EXECUTOR_DIGEST=""
|
||||
VERIFIER_DIGEST=""
|
||||
STAGE_DIR=""
|
||||
ROLLBACK_DIR=""
|
||||
APPLY_STARTED=0
|
||||
APPLY_COMPLETE=0
|
||||
ROLLBACK_VERIFIED=0
|
||||
declare -A FILE_EXISTED=()
|
||||
declare -A PREVIOUS_DIGEST=()
|
||||
declare -A EXPECTED_DIGEST=()
|
||||
|
||||
usage() {
|
||||
printf '%s\n' \
|
||||
"Usage: $0 --mode check|apply|verify --source-revision SHA --run-id ID --source-stage PATH" \
|
||||
"The executor is fixed to host 192.168.0.110, an Agent99 run-bound source stage," \
|
||||
"and the 17-file /backup/scripts transaction."
|
||||
}
|
||||
|
||||
fail() {
|
||||
printf 'HOST110_BACKUP_RUNTIME_OK=0\nERROR=%s\n' "$1" >&2
|
||||
exit 64
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--mode)
|
||||
shift
|
||||
MODE="${1:-}"
|
||||
;;
|
||||
--source-revision)
|
||||
shift
|
||||
SOURCE_REVISION="${1:-}"
|
||||
;;
|
||||
--run-id)
|
||||
shift
|
||||
RUN_ID="${1:-}"
|
||||
;;
|
||||
--source-stage)
|
||||
shift
|
||||
SOURCE_STAGE="${1:-}"
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*) fail "unknown_argument" ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
case "$MODE" in check|apply|verify) ;; *) fail "invalid_mode" ;; esac
|
||||
[[ "$SOURCE_REVISION" =~ ^[0-9a-f]{40}$ ]] || fail "invalid_source_revision"
|
||||
[[ "$RUN_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$ ]] || fail "invalid_run_id"
|
||||
expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"
|
||||
[ "$SOURCE_STAGE" = "$expected_source_stage" ] || fail "invalid_source_stage"
|
||||
|
||||
for command_name in awk bash cp flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do
|
||||
command -v "$command_name" >/dev/null 2>&1 || fail "required_command_missing_${command_name}"
|
||||
done
|
||||
|
||||
current_user="$(id -un)"
|
||||
[ "$current_user" = "$EXPECTED_USER" ] || fail "unexpected_executor_user"
|
||||
hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx "$EXPECTED_HOST_IP" || fail "unexpected_executor_host"
|
||||
[ -d "$SOURCE_STAGE" ] && [ ! -L "$SOURCE_STAGE" ] || fail "canonical_source_stage_unavailable"
|
||||
[ -d "$DEST_ROOT" ] && [ ! -L "$DEST_ROOT" ] || fail "backup_runtime_destination_unavailable"
|
||||
|
||||
manifest_rows="$(python3 - "$SOURCE_STAGE/manifest.json" "$SOURCE_REVISION" "$RUN_ID" "${RUNTIME_FILES[@]}" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
source_revision = sys.argv[2]
|
||||
run_id = sys.argv[3]
|
||||
expected_names = sys.argv[4:]
|
||||
try:
|
||||
document = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
raise SystemExit(65)
|
||||
rows = document.get("files")
|
||||
if (
|
||||
document.get("schemaVersion") != "agent99_host110_backup_runtime_source_manifest_v1"
|
||||
or document.get("sourceRevision") != source_revision
|
||||
or document.get("runId") != run_id
|
||||
or not re.fullmatch(r"[0-9a-f]{40}", str(document.get("sourceHead", "")))
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("executorSha256", "")))
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("verifierSha256", "")))
|
||||
or not isinstance(rows, list)
|
||||
or len(rows) != len(expected_names)
|
||||
):
|
||||
raise SystemExit(66)
|
||||
by_name = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
raise SystemExit(66)
|
||||
name = str(row.get("name", ""))
|
||||
digest = str(row.get("sha256", ""))
|
||||
if name in by_name or not re.fullmatch(r"[A-Za-z0-9_.-]+", name) or not re.fullmatch(r"[0-9a-f]{64}", digest):
|
||||
raise SystemExit(66)
|
||||
by_name[name] = digest
|
||||
if set(by_name) != set(expected_names):
|
||||
raise SystemExit(66)
|
||||
print(f"SOURCE_HEAD {document['sourceHead']}")
|
||||
print(f"EXECUTOR_SHA256 {document['executorSha256']}")
|
||||
print(f"VERIFIER_SHA256 {document['verifierSha256']}")
|
||||
for name in expected_names:
|
||||
print(f"{name} {by_name[name]}")
|
||||
PY
|
||||
)" || fail "source_manifest_invalid"
|
||||
|
||||
while read -r manifest_name manifest_digest; do
|
||||
case "$manifest_name" in
|
||||
SOURCE_HEAD) SOURCE_HEAD="$manifest_digest" ;;
|
||||
EXECUTOR_SHA256) EXECUTOR_DIGEST="$manifest_digest" ;;
|
||||
VERIFIER_SHA256) VERIFIER_DIGEST="$manifest_digest" ;;
|
||||
*) EXPECTED_DIGEST[$manifest_name]="$manifest_digest" ;;
|
||||
esac
|
||||
done <<< "$manifest_rows"
|
||||
[[ "$SOURCE_HEAD" =~ ^[0-9a-f]{40}$ ]] || fail "source_head_invalid"
|
||||
|
||||
working_digest() {
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
}
|
||||
|
||||
[ "$(working_digest "$0")" = "$EXECUTOR_DIGEST" ] || fail "executor_identity_failed"
|
||||
[ -f "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \
|
||||
&& [ ! -L "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \
|
||||
&& [ "$(working_digest "$SOURCE_STAGE/verify-host110-backup-runtime.py")" = "$VERIFIER_DIGEST" ] \
|
||||
|| fail "verifier_identity_failed"
|
||||
|
||||
validate_file() {
|
||||
local path="$1"
|
||||
case "$path" in
|
||||
*.sh) bash -n "$path" ;;
|
||||
*.py) python3 -c 'import ast, pathlib, sys; ast.parse(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))' "$path" ;;
|
||||
*) return 64 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
source_digest() {
|
||||
local name="$1"
|
||||
printf '%s\n' "${EXPECTED_DIGEST[$name]:-}"
|
||||
}
|
||||
|
||||
validate_source() {
|
||||
local name source_path expected_digest actual_digest resolved
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
source_path="$SOURCE_STAGE/$name"
|
||||
[ -f "$source_path" ] && [ ! -L "$source_path" ] || return 65
|
||||
resolved="$(readlink -f "$source_path")"
|
||||
[[ "$resolved" == "$SOURCE_STAGE/"* ]] || return 65
|
||||
expected_digest="$(source_digest "$name")"
|
||||
actual_digest="$(working_digest "$source_path")"
|
||||
[ "$actual_digest" = "$expected_digest" ] || return 66
|
||||
validate_file "$source_path" || return 67
|
||||
done
|
||||
}
|
||||
|
||||
verify_destination() {
|
||||
local name dest_path expected_digest actual_digest
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
dest_path="$DEST_ROOT/$name"
|
||||
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] || return 68
|
||||
expected_digest="$(source_digest "$name")"
|
||||
actual_digest="$(working_digest "$dest_path")"
|
||||
[ "$actual_digest" = "$expected_digest" ] || return 69
|
||||
[ "$(stat -c '%U:%G:%a' "$dest_path")" = "wooo:wooo:755" ] || return 70
|
||||
done
|
||||
}
|
||||
|
||||
write_receipt() {
|
||||
local status="$1"
|
||||
local rollback_verified="$2"
|
||||
local receipt_path="$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json"
|
||||
install -d -m 700 "$STATUS_ROOT"
|
||||
python3 - "$receipt_path" "$status" "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$rollback_verified" "$VERIFIER_DIGEST" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
document = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v1",
|
||||
"status": sys.argv[2],
|
||||
"sourceRevision": sys.argv[3],
|
||||
"sourceHead": sys.argv[4],
|
||||
"runId": sys.argv[5],
|
||||
"fileCount": 17,
|
||||
"rollbackPerformed": sys.argv[6] == "1",
|
||||
"rollbackVerified": sys.argv[6] == "1",
|
||||
"verifierSha256": sys.argv[7],
|
||||
"executorHost": "192.168.0.110",
|
||||
"productionServiceRestarted": False,
|
||||
"secretValuesRead": False,
|
||||
"writtenAt": int(time.time()),
|
||||
}
|
||||
temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
|
||||
temporary.write_text(json.dumps(document, ensure_ascii=True, sort_keys=True) + "\n", encoding="utf-8")
|
||||
os.chmod(temporary, 0o600)
|
||||
try:
|
||||
os.link(temporary, path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
PY
|
||||
printf '%s' "$receipt_path"
|
||||
}
|
||||
|
||||
record_prestate() {
|
||||
local name dest_path digest
|
||||
: > "$ROLLBACK_DIR/prestate.tsv"
|
||||
chmod 600 "$ROLLBACK_DIR/prestate.tsv"
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
dest_path="$DEST_ROOT/$name"
|
||||
if [ -f "$dest_path" ] && [ ! -L "$dest_path" ]; then
|
||||
[ "$(stat -c '%U:%G' "$dest_path")" = "wooo:wooo" ] || return 71
|
||||
FILE_EXISTED[$name]=1
|
||||
digest="$(working_digest "$dest_path")"
|
||||
PREVIOUS_DIGEST[$name]="$digest"
|
||||
printf '%s\t1\t%s\n' "$name" "$digest" >> "$ROLLBACK_DIR/prestate.tsv"
|
||||
cp -p -- "$dest_path" "$ROLLBACK_DIR/$name"
|
||||
[ "$(working_digest "$ROLLBACK_DIR/$name")" = "$digest" ] || return 72
|
||||
elif [ ! -e "$dest_path" ]; then
|
||||
FILE_EXISTED[$name]=0
|
||||
PREVIOUS_DIGEST[$name]="-"
|
||||
printf '%s\t0\t-\n' "$name" >> "$ROLLBACK_DIR/prestate.tsv"
|
||||
else
|
||||
return 73
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
rollback_transaction() {
|
||||
local name dest_path backup_path temporary failed=0
|
||||
set +e
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
[ -n "${FILE_EXISTED[$name]+x}" ] || { failed=1; continue; }
|
||||
dest_path="$DEST_ROOT/$name"
|
||||
if [ "${FILE_EXISTED[$name]}" = "1" ]; then
|
||||
backup_path="$ROLLBACK_DIR/$name"
|
||||
temporary="$DEST_ROOT/.${name}.agent99-rollback-${RUN_ID}"
|
||||
cp -p -- "$backup_path" "$temporary" \
|
||||
&& mv -f -- "$temporary" "$dest_path" \
|
||||
|| failed=1
|
||||
else
|
||||
rm -f -- "$dest_path" || failed=1
|
||||
fi
|
||||
done
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
dest_path="$DEST_ROOT/$name"
|
||||
if [ "${FILE_EXISTED[$name]:-}" = "1" ]; then
|
||||
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] \
|
||||
&& [ "$(working_digest "$dest_path" 2>/dev/null)" = "${PREVIOUS_DIGEST[$name]:-invalid}" ] \
|
||||
|| failed=1
|
||||
elif [ "${FILE_EXISTED[$name]:-}" = "0" ]; then
|
||||
[ ! -e "$dest_path" ] || failed=1
|
||||
else
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
set -e
|
||||
if [ "$failed" -eq 0 ]; then
|
||||
ROLLBACK_VERIFIED=1
|
||||
return 0
|
||||
fi
|
||||
ROLLBACK_VERIFIED=0
|
||||
return 1
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
local rollback_status="rollback_unverified"
|
||||
trap - EXIT HUP INT TERM
|
||||
if [ "$APPLY_STARTED" -eq 1 ] && [ "$APPLY_COMPLETE" -ne 1 ]; then
|
||||
if rollback_transaction; then
|
||||
rollback_status="failed_rolled_back"
|
||||
fi
|
||||
write_receipt "$rollback_status" "$ROLLBACK_VERIFIED" >/dev/null || true
|
||||
fi
|
||||
[ -z "$STAGE_DIR" ] || rm -rf -- "$STAGE_DIR"
|
||||
if [ "$ROLLBACK_VERIFIED" -eq 1 ]; then
|
||||
[ -z "$ROLLBACK_DIR" ] || rm -rf -- "$ROLLBACK_DIR"
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
validate_source || fail "source_validation_failed"
|
||||
|
||||
if [ "$MODE" = "check" ]; then
|
||||
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":17,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$MODE" = "verify" ]; then
|
||||
verify_destination || fail "destination_verification_failed"
|
||||
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":17,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[ ! -e "$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json" ] || fail "run_identity_receipt_exists"
|
||||
STAGE_DIR="$STAGE_ROOT/$RUN_ID"
|
||||
ROLLBACK_DIR="$ROLLBACK_ROOT/$RUN_ID"
|
||||
[ ! -e "$STAGE_DIR" ] || fail "run_identity_stage_exists"
|
||||
[ ! -e "$ROLLBACK_DIR" ] || fail "run_identity_rollback_exists"
|
||||
|
||||
[ ! -L "$LOCK_PATH" ] || fail "deployment_lock_symlink_unsafe"
|
||||
(umask 077; : >> "$LOCK_PATH") || fail "deployment_lock_unavailable"
|
||||
[ -f "$LOCK_PATH" ] && [ -O "$LOCK_PATH" ] || fail "deployment_lock_owner_invalid"
|
||||
exec 9>>"$LOCK_PATH"
|
||||
flock -x -w 30 9 || fail "deployment_lock_timeout"
|
||||
|
||||
# Old runtime revisions predate the shared gate. Refuse the first promotion if
|
||||
# one of them is already active; after common.sh moves first, new starts block.
|
||||
if pgrep -afu "$EXPECTED_USER" '/backup/scripts/(backup-|check-backup|sync-offsite|verify-offsite|enforce-latest)' \
|
||||
| grep -vF "host110-backup-runtime-executor.sh" >/dev/null 2>&1; then
|
||||
fail "legacy_backup_runtime_active"
|
||||
fi
|
||||
|
||||
install -d -m 700 "$STAGE_DIR" "$ROLLBACK_DIR"
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM HUP
|
||||
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
install -m 700 "$SOURCE_STAGE/$name" "$STAGE_DIR/$name"
|
||||
[ "$(working_digest "$STAGE_DIR/$name")" = "$(source_digest "$name")" ] || fail "stage_identity_failed"
|
||||
validate_file "$STAGE_DIR/$name" || fail "stage_validation_failed"
|
||||
done
|
||||
|
||||
record_prestate || fail "rollback_prestate_capture_failed"
|
||||
APPLY_STARTED=1
|
||||
|
||||
# common.sh is promoted first. Every managed mutating backup entrypoint then
|
||||
# takes the shared side of LOCK_PATH before doing work, while this process holds
|
||||
# the exclusive side until the independent verifier has completed.
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
temporary="$DEST_ROOT/.${name}.agent99-${RUN_ID}"
|
||||
install -m 755 "$STAGE_DIR/$name" "$temporary"
|
||||
mv -f -- "$temporary" "$DEST_ROOT/$name"
|
||||
done
|
||||
|
||||
verify_destination || fail "post_apply_verification_failed"
|
||||
verifier_result="$(BACKUP_RUNTIME_DEPLOY_CONTEXT=1 python3 "$SOURCE_STAGE/verify-host110-backup-runtime.py" \
|
||||
--manifest "$SOURCE_STAGE/manifest.json" \
|
||||
--destination "$DEST_ROOT" \
|
||||
--source-revision "$SOURCE_REVISION" \
|
||||
--run-id "$RUN_ID" \
|
||||
--expected-verifier-sha256 "$VERIFIER_DIGEST")" || fail "independent_verifier_failed"
|
||||
python3 - "$verifier_result" "$SOURCE_REVISION" "$RUN_ID" <<'PY' || fail "independent_verifier_contract_failed"
|
||||
import json
|
||||
import sys
|
||||
|
||||
try:
|
||||
row = json.loads(sys.argv[1])
|
||||
except json.JSONDecodeError:
|
||||
raise SystemExit(1)
|
||||
if not (
|
||||
row.get("schemaVersion") == "agent99_host110_backup_runtime_verifier_v1"
|
||||
and row.get("ok") is True
|
||||
and row.get("sourceRevision") == sys.argv[2]
|
||||
and row.get("runId") == sys.argv[3]
|
||||
and row.get("fileCount") == 17
|
||||
and row.get("remoteWritePerformed") is False
|
||||
and row.get("selfIdentityVerified") is True
|
||||
):
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
|
||||
APPLY_COMPLETE=1
|
||||
receipt_path="$(write_receipt "verified" 0)"
|
||||
rm -rf -- "$STAGE_DIR" "$ROLLBACK_DIR"
|
||||
STAGE_DIR=""
|
||||
ROLLBACK_DIR=""
|
||||
trap - EXIT HUP INT TERM
|
||||
python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$receipt_path" "$verifier_result" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
print(json.dumps({
|
||||
"schemaVersion": "agent99_host110_backup_runtime_executor_v1",
|
||||
"mode": "apply",
|
||||
"ok": True,
|
||||
"sourceRevision": sys.argv[1],
|
||||
"sourceHead": sys.argv[2],
|
||||
"fileCount": 17,
|
||||
"rollbackPerformed": False,
|
||||
"receipt": sys.argv[3],
|
||||
"verifier": json.loads(sys.argv[4]),
|
||||
}, ensure_ascii=True, sort_keys=True))
|
||||
PY
|
||||
164
scripts/backup/verify-host110-backup-runtime.py
Executable file
164
scripts/backup/verify-host110-backup-runtime.py
Executable file
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independent no-write verifier for the Host110 backup runtime transaction."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
EXPECTED_DESTINATION = Path("/backup/scripts")
|
||||
RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")
|
||||
EXPECTED_FILES = (
|
||||
"common.sh",
|
||||
"backup-all.sh",
|
||||
"backup-host188-products.sh",
|
||||
"verify-host188-products-backup.sh",
|
||||
"verify-host188-products-archive.py",
|
||||
"backup-gitea.sh",
|
||||
"gitea-full-backup-restore-drill.sh",
|
||||
"backup-configs.sh",
|
||||
"backup-awoooi.sh",
|
||||
"backup-awoooi-frequent.sh",
|
||||
"backup-clawbot.sh",
|
||||
"backup-sentry.sh",
|
||||
"check-backup-integrity.sh",
|
||||
"sync-offsite-backups.sh",
|
||||
"backup-offsite-readiness-gate.sh",
|
||||
"verify-offsite-full-sync.sh",
|
||||
"enforce-latest-only-retention.sh",
|
||||
)
|
||||
|
||||
|
||||
def fail(reason: str) -> None:
|
||||
print(f"HOST110_BACKUP_RUNTIME_VERIFY_OK=0\nERROR={reason}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def acquire_runtime_read_lock(destination: Path):
|
||||
if destination != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_DEPLOY_CONTEXT") == "1":
|
||||
return None
|
||||
if RUNTIME_LOCK.is_symlink() or not RUNTIME_LOCK.is_file():
|
||||
fail("runtime_lock_unavailable")
|
||||
handle = RUNTIME_LOCK.open("rb")
|
||||
if RUNTIME_LOCK.stat().st_uid != os.getuid():
|
||||
handle.close()
|
||||
fail("runtime_lock_owner_invalid")
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
|
||||
return handle
|
||||
except BlockingIOError:
|
||||
if time.monotonic() >= deadline:
|
||||
handle.close()
|
||||
fail("runtime_deployment_active")
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
manifest_source = parser.add_mutually_exclusive_group(required=True)
|
||||
manifest_source.add_argument("--manifest")
|
||||
manifest_source.add_argument("--manifest-base64")
|
||||
parser.add_argument("--destination", required=True)
|
||||
parser.add_argument("--source-revision", required=True)
|
||||
parser.add_argument("--run-id", required=True)
|
||||
parser.add_argument("--expected-verifier-sha256", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", args.source_revision):
|
||||
fail("invalid_source_revision")
|
||||
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}", args.run_id):
|
||||
fail("invalid_run_id")
|
||||
destination = Path(args.destination)
|
||||
if destination != EXPECTED_DESTINATION or destination.is_symlink() or not destination.is_dir():
|
||||
fail("invalid_destination")
|
||||
runtime_lock = acquire_runtime_read_lock(destination)
|
||||
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", args.expected_verifier_sha256):
|
||||
fail("invalid_verifier_identity")
|
||||
try:
|
||||
if args.manifest:
|
||||
manifest_text = Path(args.manifest).read_text(encoding="utf-8")
|
||||
else:
|
||||
manifest_text = base64.b64decode(args.manifest_base64, validate=True).decode("utf-8")
|
||||
document = json.loads(manifest_text)
|
||||
except (OSError, UnicodeDecodeError, ValueError, json.JSONDecodeError):
|
||||
fail("manifest_unavailable")
|
||||
rows = document.get("files")
|
||||
if (
|
||||
document.get("schemaVersion") != "agent99_host110_backup_runtime_source_manifest_v1"
|
||||
or document.get("sourceRevision") != args.source_revision
|
||||
or document.get("runId") != args.run_id
|
||||
or not re.fullmatch(r"[0-9a-f]{40}", str(document.get("sourceHead", "")))
|
||||
or document.get("verifierSha256") != args.expected_verifier_sha256
|
||||
or not isinstance(rows, list)
|
||||
or len(rows) != len(EXPECTED_FILES)
|
||||
):
|
||||
fail("manifest_contract_failed")
|
||||
by_name: dict[str, str] = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
fail("manifest_file_set_failed")
|
||||
name = str(row.get("name", ""))
|
||||
if name in by_name:
|
||||
fail("manifest_file_set_failed")
|
||||
by_name[name] = str(row.get("sha256", ""))
|
||||
if set(by_name) != set(EXPECTED_FILES) or not all(re.fullmatch(r"[0-9a-f]{64}", value) for value in by_name.values()):
|
||||
fail("manifest_file_set_failed")
|
||||
|
||||
verifier_path = Path(__file__)
|
||||
self_identity_verified = verifier_path.is_file() and sha256(verifier_path) == args.expected_verifier_sha256
|
||||
if args.manifest and not self_identity_verified:
|
||||
fail("verifier_identity_failed")
|
||||
|
||||
root = destination.resolve(strict=True)
|
||||
for name in EXPECTED_FILES:
|
||||
path = destination / name
|
||||
if path.is_symlink() or not path.is_file():
|
||||
fail("runtime_file_type_failed")
|
||||
resolved = path.resolve(strict=True)
|
||||
if resolved.parent != root:
|
||||
fail("runtime_file_boundary_failed")
|
||||
stat = resolved.stat()
|
||||
if stat.st_uid != os.getuid() or stat.st_gid != os.getgid() or stat.st_mode & 0o777 != 0o755:
|
||||
fail("runtime_file_metadata_failed")
|
||||
if sha256(resolved) != by_name[name]:
|
||||
fail("runtime_file_identity_failed")
|
||||
|
||||
result = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_verifier_v1",
|
||||
"ok": True,
|
||||
"sourceRevision": args.source_revision,
|
||||
"runId": args.run_id,
|
||||
"fileCount": len(EXPECTED_FILES),
|
||||
"destination": str(EXPECTED_DESTINATION),
|
||||
"remoteWritePerformed": False,
|
||||
"secretValuesRead": False,
|
||||
"selfIdentityVerified": self_identity_verified,
|
||||
"verifierSha256": args.expected_verifier_sha256,
|
||||
}
|
||||
print(json.dumps(result, ensure_ascii=True, sort_keys=True))
|
||||
if runtime_lock is not None:
|
||||
runtime_lock.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -4,11 +4,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
@@ -20,6 +23,28 @@ EXPECTED_PREFIXES = {
|
||||
"n8n-online-": ".tar.gz",
|
||||
}
|
||||
FIXED_ASSETS = {"n8n-validation.json"}
|
||||
RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")
|
||||
|
||||
|
||||
def acquire_runtime_lock():
|
||||
if Path(__file__).resolve().parent != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_SHARED_LOCK_HELD") == "1":
|
||||
return None
|
||||
if RUNTIME_LOCK.is_symlink():
|
||||
raise RuntimeError("backup_runtime_gate_unsafe")
|
||||
handle = RUNTIME_LOCK.open("a+")
|
||||
if RUNTIME_LOCK.stat().st_uid != os.getuid():
|
||||
handle.close()
|
||||
raise RuntimeError("backup_runtime_gate_owner_invalid")
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
|
||||
return handle
|
||||
except BlockingIOError:
|
||||
if time.monotonic() >= deadline:
|
||||
handle.close()
|
||||
raise RuntimeError("backup_runtime_deployment_active")
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
@@ -153,10 +178,13 @@ def validate_restore(root: Path) -> dict[str, Any]:
|
||||
|
||||
|
||||
def main() -> int:
|
||||
runtime_lock = acquire_runtime_lock()
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(validate_restore(args.root), separators=(",", ":")))
|
||||
if runtime_lock is not None:
|
||||
runtime_lock.close()
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user