feat(agent99): bind host110 backup runtime deploy
This commit is contained in:
Your Name
431
scripts/backup/host110-backup-runtime-executor.sh
Executable file
431
scripts/backup/host110-backup-runtime-executor.sh
Executable file
@@ -0,0 +1,431 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fixed Host110 executor for Windows99 Agent99 backup-runtime deployment.
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
readonly EXPECTED_HOST_IP="192.168.0.110"
|
||||
readonly EXPECTED_USER="wooo"
|
||||
readonly DEST_ROOT="/backup/scripts"
|
||||
readonly STATUS_ROOT="/backup/status"
|
||||
readonly STAGE_ROOT="/backup/.agent99-backup-runtime-stage"
|
||||
readonly ROLLBACK_ROOT="/backup/.agent99-backup-runtime-rollback"
|
||||
readonly LOCK_PATH="/tmp/agent99-host110-backup-runtime.lock"
|
||||
|
||||
readonly -a RUNTIME_FILES=(
|
||||
common.sh
|
||||
backup-all.sh
|
||||
backup-host188-products.sh
|
||||
verify-host188-products-backup.sh
|
||||
verify-host188-products-archive.py
|
||||
backup-gitea.sh
|
||||
gitea-full-backup-restore-drill.sh
|
||||
backup-configs.sh
|
||||
backup-awoooi.sh
|
||||
backup-awoooi-frequent.sh
|
||||
backup-clawbot.sh
|
||||
backup-sentry.sh
|
||||
check-backup-integrity.sh
|
||||
sync-offsite-backups.sh
|
||||
backup-offsite-readiness-gate.sh
|
||||
verify-offsite-full-sync.sh
|
||||
enforce-latest-only-retention.sh
|
||||
)
|
||||
|
||||
MODE=""
|
||||
SOURCE_REVISION=""
|
||||
RUN_ID=""
|
||||
SOURCE_STAGE=""
|
||||
SOURCE_HEAD=""
|
||||
EXECUTOR_DIGEST=""
|
||||
VERIFIER_DIGEST=""
|
||||
STAGE_DIR=""
|
||||
ROLLBACK_DIR=""
|
||||
APPLY_STARTED=0
|
||||
APPLY_COMPLETE=0
|
||||
ROLLBACK_VERIFIED=0
|
||||
declare -A FILE_EXISTED=()
|
||||
declare -A PREVIOUS_DIGEST=()
|
||||
declare -A EXPECTED_DIGEST=()
|
||||
|
||||
usage() {
|
||||
printf '%s\n' \
|
||||
"Usage: $0 --mode check|apply|verify --source-revision SHA --run-id ID --source-stage PATH" \
|
||||
"The executor is fixed to host 192.168.0.110, an Agent99 run-bound source stage," \
|
||||
"and the 17-file /backup/scripts transaction."
|
||||
}
|
||||
|
||||
fail() {
|
||||
printf 'HOST110_BACKUP_RUNTIME_OK=0\nERROR=%s\n' "$1" >&2
|
||||
exit 64
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--mode)
|
||||
shift
|
||||
MODE="${1:-}"
|
||||
;;
|
||||
--source-revision)
|
||||
shift
|
||||
SOURCE_REVISION="${1:-}"
|
||||
;;
|
||||
--run-id)
|
||||
shift
|
||||
RUN_ID="${1:-}"
|
||||
;;
|
||||
--source-stage)
|
||||
shift
|
||||
SOURCE_STAGE="${1:-}"
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*) fail "unknown_argument" ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
case "$MODE" in check|apply|verify) ;; *) fail "invalid_mode" ;; esac
|
||||
[[ "$SOURCE_REVISION" =~ ^[0-9a-f]{40}$ ]] || fail "invalid_source_revision"
|
||||
[[ "$RUN_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$ ]] || fail "invalid_run_id"
|
||||
expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"
|
||||
[ "$SOURCE_STAGE" = "$expected_source_stage" ] || fail "invalid_source_stage"
|
||||
|
||||
for command_name in awk bash cp flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do
|
||||
command -v "$command_name" >/dev/null 2>&1 || fail "required_command_missing_${command_name}"
|
||||
done
|
||||
|
||||
current_user="$(id -un)"
|
||||
[ "$current_user" = "$EXPECTED_USER" ] || fail "unexpected_executor_user"
|
||||
hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx "$EXPECTED_HOST_IP" || fail "unexpected_executor_host"
|
||||
[ -d "$SOURCE_STAGE" ] && [ ! -L "$SOURCE_STAGE" ] || fail "canonical_source_stage_unavailable"
|
||||
[ -d "$DEST_ROOT" ] && [ ! -L "$DEST_ROOT" ] || fail "backup_runtime_destination_unavailable"
|
||||
|
||||
manifest_rows="$(python3 - "$SOURCE_STAGE/manifest.json" "$SOURCE_REVISION" "$RUN_ID" "${RUNTIME_FILES[@]}" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
source_revision = sys.argv[2]
|
||||
run_id = sys.argv[3]
|
||||
expected_names = sys.argv[4:]
|
||||
try:
|
||||
document = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
raise SystemExit(65)
|
||||
rows = document.get("files")
|
||||
if (
|
||||
document.get("schemaVersion") != "agent99_host110_backup_runtime_source_manifest_v1"
|
||||
or document.get("sourceRevision") != source_revision
|
||||
or document.get("runId") != run_id
|
||||
or not re.fullmatch(r"[0-9a-f]{40}", str(document.get("sourceHead", "")))
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("executorSha256", "")))
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("verifierSha256", "")))
|
||||
or not isinstance(rows, list)
|
||||
or len(rows) != len(expected_names)
|
||||
):
|
||||
raise SystemExit(66)
|
||||
by_name = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
raise SystemExit(66)
|
||||
name = str(row.get("name", ""))
|
||||
digest = str(row.get("sha256", ""))
|
||||
if name in by_name or not re.fullmatch(r"[A-Za-z0-9_.-]+", name) or not re.fullmatch(r"[0-9a-f]{64}", digest):
|
||||
raise SystemExit(66)
|
||||
by_name[name] = digest
|
||||
if set(by_name) != set(expected_names):
|
||||
raise SystemExit(66)
|
||||
print(f"SOURCE_HEAD {document['sourceHead']}")
|
||||
print(f"EXECUTOR_SHA256 {document['executorSha256']}")
|
||||
print(f"VERIFIER_SHA256 {document['verifierSha256']}")
|
||||
for name in expected_names:
|
||||
print(f"{name} {by_name[name]}")
|
||||
PY
|
||||
)" || fail "source_manifest_invalid"
|
||||
|
||||
while read -r manifest_name manifest_digest; do
|
||||
case "$manifest_name" in
|
||||
SOURCE_HEAD) SOURCE_HEAD="$manifest_digest" ;;
|
||||
EXECUTOR_SHA256) EXECUTOR_DIGEST="$manifest_digest" ;;
|
||||
VERIFIER_SHA256) VERIFIER_DIGEST="$manifest_digest" ;;
|
||||
*) EXPECTED_DIGEST[$manifest_name]="$manifest_digest" ;;
|
||||
esac
|
||||
done <<< "$manifest_rows"
|
||||
[[ "$SOURCE_HEAD" =~ ^[0-9a-f]{40}$ ]] || fail "source_head_invalid"
|
||||
|
||||
working_digest() {
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
}
|
||||
|
||||
[ "$(working_digest "$0")" = "$EXECUTOR_DIGEST" ] || fail "executor_identity_failed"
|
||||
[ -f "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \
|
||||
&& [ ! -L "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \
|
||||
&& [ "$(working_digest "$SOURCE_STAGE/verify-host110-backup-runtime.py")" = "$VERIFIER_DIGEST" ] \
|
||||
|| fail "verifier_identity_failed"
|
||||
|
||||
validate_file() {
|
||||
local path="$1"
|
||||
case "$path" in
|
||||
*.sh) bash -n "$path" ;;
|
||||
*.py) python3 -c 'import ast, pathlib, sys; ast.parse(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))' "$path" ;;
|
||||
*) return 64 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
source_digest() {
|
||||
local name="$1"
|
||||
printf '%s\n' "${EXPECTED_DIGEST[$name]:-}"
|
||||
}
|
||||
|
||||
validate_source() {
|
||||
local name source_path expected_digest actual_digest resolved
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
source_path="$SOURCE_STAGE/$name"
|
||||
[ -f "$source_path" ] && [ ! -L "$source_path" ] || return 65
|
||||
resolved="$(readlink -f "$source_path")"
|
||||
[[ "$resolved" == "$SOURCE_STAGE/"* ]] || return 65
|
||||
expected_digest="$(source_digest "$name")"
|
||||
actual_digest="$(working_digest "$source_path")"
|
||||
[ "$actual_digest" = "$expected_digest" ] || return 66
|
||||
validate_file "$source_path" || return 67
|
||||
done
|
||||
}
|
||||
|
||||
verify_destination() {
|
||||
local name dest_path expected_digest actual_digest
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
dest_path="$DEST_ROOT/$name"
|
||||
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] || return 68
|
||||
expected_digest="$(source_digest "$name")"
|
||||
actual_digest="$(working_digest "$dest_path")"
|
||||
[ "$actual_digest" = "$expected_digest" ] || return 69
|
||||
[ "$(stat -c '%U:%G:%a' "$dest_path")" = "wooo:wooo:755" ] || return 70
|
||||
done
|
||||
}
|
||||
|
||||
write_receipt() {
|
||||
local status="$1"
|
||||
local rollback_verified="$2"
|
||||
local receipt_path="$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json"
|
||||
install -d -m 700 "$STATUS_ROOT"
|
||||
python3 - "$receipt_path" "$status" "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$rollback_verified" "$VERIFIER_DIGEST" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
document = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v1",
|
||||
"status": sys.argv[2],
|
||||
"sourceRevision": sys.argv[3],
|
||||
"sourceHead": sys.argv[4],
|
||||
"runId": sys.argv[5],
|
||||
"fileCount": 17,
|
||||
"rollbackPerformed": sys.argv[6] == "1",
|
||||
"rollbackVerified": sys.argv[6] == "1",
|
||||
"verifierSha256": sys.argv[7],
|
||||
"executorHost": "192.168.0.110",
|
||||
"productionServiceRestarted": False,
|
||||
"secretValuesRead": False,
|
||||
"writtenAt": int(time.time()),
|
||||
}
|
||||
temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
|
||||
temporary.write_text(json.dumps(document, ensure_ascii=True, sort_keys=True) + "\n", encoding="utf-8")
|
||||
os.chmod(temporary, 0o600)
|
||||
try:
|
||||
os.link(temporary, path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
PY
|
||||
printf '%s' "$receipt_path"
|
||||
}
|
||||
|
||||
record_prestate() {
|
||||
local name dest_path digest
|
||||
: > "$ROLLBACK_DIR/prestate.tsv"
|
||||
chmod 600 "$ROLLBACK_DIR/prestate.tsv"
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
dest_path="$DEST_ROOT/$name"
|
||||
if [ -f "$dest_path" ] && [ ! -L "$dest_path" ]; then
|
||||
[ "$(stat -c '%U:%G' "$dest_path")" = "wooo:wooo" ] || return 71
|
||||
FILE_EXISTED[$name]=1
|
||||
digest="$(working_digest "$dest_path")"
|
||||
PREVIOUS_DIGEST[$name]="$digest"
|
||||
printf '%s\t1\t%s\n' "$name" "$digest" >> "$ROLLBACK_DIR/prestate.tsv"
|
||||
cp -p -- "$dest_path" "$ROLLBACK_DIR/$name"
|
||||
[ "$(working_digest "$ROLLBACK_DIR/$name")" = "$digest" ] || return 72
|
||||
elif [ ! -e "$dest_path" ]; then
|
||||
FILE_EXISTED[$name]=0
|
||||
PREVIOUS_DIGEST[$name]="-"
|
||||
printf '%s\t0\t-\n' "$name" >> "$ROLLBACK_DIR/prestate.tsv"
|
||||
else
|
||||
return 73
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
rollback_transaction() {
|
||||
local name dest_path backup_path temporary failed=0
|
||||
set +e
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
[ -n "${FILE_EXISTED[$name]+x}" ] || { failed=1; continue; }
|
||||
dest_path="$DEST_ROOT/$name"
|
||||
if [ "${FILE_EXISTED[$name]}" = "1" ]; then
|
||||
backup_path="$ROLLBACK_DIR/$name"
|
||||
temporary="$DEST_ROOT/.${name}.agent99-rollback-${RUN_ID}"
|
||||
cp -p -- "$backup_path" "$temporary" \
|
||||
&& mv -f -- "$temporary" "$dest_path" \
|
||||
|| failed=1
|
||||
else
|
||||
rm -f -- "$dest_path" || failed=1
|
||||
fi
|
||||
done
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
dest_path="$DEST_ROOT/$name"
|
||||
if [ "${FILE_EXISTED[$name]:-}" = "1" ]; then
|
||||
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] \
|
||||
&& [ "$(working_digest "$dest_path" 2>/dev/null)" = "${PREVIOUS_DIGEST[$name]:-invalid}" ] \
|
||||
|| failed=1
|
||||
elif [ "${FILE_EXISTED[$name]:-}" = "0" ]; then
|
||||
[ ! -e "$dest_path" ] || failed=1
|
||||
else
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
set -e
|
||||
if [ "$failed" -eq 0 ]; then
|
||||
ROLLBACK_VERIFIED=1
|
||||
return 0
|
||||
fi
|
||||
ROLLBACK_VERIFIED=0
|
||||
return 1
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
local rollback_status="rollback_unverified"
|
||||
trap - EXIT HUP INT TERM
|
||||
if [ "$APPLY_STARTED" -eq 1 ] && [ "$APPLY_COMPLETE" -ne 1 ]; then
|
||||
if rollback_transaction; then
|
||||
rollback_status="failed_rolled_back"
|
||||
fi
|
||||
write_receipt "$rollback_status" "$ROLLBACK_VERIFIED" >/dev/null || true
|
||||
fi
|
||||
[ -z "$STAGE_DIR" ] || rm -rf -- "$STAGE_DIR"
|
||||
if [ "$ROLLBACK_VERIFIED" -eq 1 ]; then
|
||||
[ -z "$ROLLBACK_DIR" ] || rm -rf -- "$ROLLBACK_DIR"
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
validate_source || fail "source_validation_failed"
|
||||
|
||||
if [ "$MODE" = "check" ]; then
|
||||
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":17,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$MODE" = "verify" ]; then
|
||||
verify_destination || fail "destination_verification_failed"
|
||||
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":17,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[ ! -e "$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json" ] || fail "run_identity_receipt_exists"
|
||||
STAGE_DIR="$STAGE_ROOT/$RUN_ID"
|
||||
ROLLBACK_DIR="$ROLLBACK_ROOT/$RUN_ID"
|
||||
[ ! -e "$STAGE_DIR" ] || fail "run_identity_stage_exists"
|
||||
[ ! -e "$ROLLBACK_DIR" ] || fail "run_identity_rollback_exists"
|
||||
|
||||
[ ! -L "$LOCK_PATH" ] || fail "deployment_lock_symlink_unsafe"
|
||||
(umask 077; : >> "$LOCK_PATH") || fail "deployment_lock_unavailable"
|
||||
[ -f "$LOCK_PATH" ] && [ -O "$LOCK_PATH" ] || fail "deployment_lock_owner_invalid"
|
||||
exec 9>>"$LOCK_PATH"
|
||||
flock -x -w 30 9 || fail "deployment_lock_timeout"
|
||||
|
||||
# Old runtime revisions predate the shared gate. Refuse the first promotion if
|
||||
# one of them is already active; after common.sh moves first, new starts block.
|
||||
if pgrep -afu "$EXPECTED_USER" '/backup/scripts/(backup-|check-backup|sync-offsite|verify-offsite|enforce-latest)' \
|
||||
| grep -vF "host110-backup-runtime-executor.sh" >/dev/null 2>&1; then
|
||||
fail "legacy_backup_runtime_active"
|
||||
fi
|
||||
|
||||
install -d -m 700 "$STAGE_DIR" "$ROLLBACK_DIR"
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM HUP
|
||||
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
install -m 700 "$SOURCE_STAGE/$name" "$STAGE_DIR/$name"
|
||||
[ "$(working_digest "$STAGE_DIR/$name")" = "$(source_digest "$name")" ] || fail "stage_identity_failed"
|
||||
validate_file "$STAGE_DIR/$name" || fail "stage_validation_failed"
|
||||
done
|
||||
|
||||
record_prestate || fail "rollback_prestate_capture_failed"
|
||||
APPLY_STARTED=1
|
||||
|
||||
# common.sh is promoted first. Every managed mutating backup entrypoint then
|
||||
# takes the shared side of LOCK_PATH before doing work, while this process holds
|
||||
# the exclusive side until the independent verifier has completed.
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
temporary="$DEST_ROOT/.${name}.agent99-${RUN_ID}"
|
||||
install -m 755 "$STAGE_DIR/$name" "$temporary"
|
||||
mv -f -- "$temporary" "$DEST_ROOT/$name"
|
||||
done
|
||||
|
||||
verify_destination || fail "post_apply_verification_failed"
|
||||
verifier_result="$(BACKUP_RUNTIME_DEPLOY_CONTEXT=1 python3 "$SOURCE_STAGE/verify-host110-backup-runtime.py" \
|
||||
--manifest "$SOURCE_STAGE/manifest.json" \
|
||||
--destination "$DEST_ROOT" \
|
||||
--source-revision "$SOURCE_REVISION" \
|
||||
--run-id "$RUN_ID" \
|
||||
--expected-verifier-sha256 "$VERIFIER_DIGEST")" || fail "independent_verifier_failed"
|
||||
python3 - "$verifier_result" "$SOURCE_REVISION" "$RUN_ID" <<'PY' || fail "independent_verifier_contract_failed"
|
||||
import json
|
||||
import sys
|
||||
|
||||
try:
|
||||
row = json.loads(sys.argv[1])
|
||||
except json.JSONDecodeError:
|
||||
raise SystemExit(1)
|
||||
if not (
|
||||
row.get("schemaVersion") == "agent99_host110_backup_runtime_verifier_v1"
|
||||
and row.get("ok") is True
|
||||
and row.get("sourceRevision") == sys.argv[2]
|
||||
and row.get("runId") == sys.argv[3]
|
||||
and row.get("fileCount") == 17
|
||||
and row.get("remoteWritePerformed") is False
|
||||
and row.get("selfIdentityVerified") is True
|
||||
):
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
|
||||
APPLY_COMPLETE=1
|
||||
receipt_path="$(write_receipt "verified" 0)"
|
||||
rm -rf -- "$STAGE_DIR" "$ROLLBACK_DIR"
|
||||
STAGE_DIR=""
|
||||
ROLLBACK_DIR=""
|
||||
trap - EXIT HUP INT TERM
|
||||
python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$receipt_path" "$verifier_result" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
print(json.dumps({
|
||||
"schemaVersion": "agent99_host110_backup_runtime_executor_v1",
|
||||
"mode": "apply",
|
||||
"ok": True,
|
||||
"sourceRevision": sys.argv[1],
|
||||
"sourceHead": sys.argv[2],
|
||||
"fileCount": 17,
|
||||
"rollbackPerformed": False,
|
||||
"receipt": sys.argv[3],
|
||||
"verifier": json.loads(sys.argv[4]),
|
||||
}, ensure_ascii=True, sort_keys=True))
|
||||
PY
|
||||
Reference in New Issue
Block a user