wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(agent99): deploy backup freshness exporter atomically

Browse Source
This commit is contained in:
Your Name
2026-07-18 21:33:50 +08:00
parent 68dcc6bbd4
commit 5c4f22f7da
5 changed files with 122 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
scripts/backup/host110-backup-runtime-executor.sh
View File

@@ -7,6 +7,8 @@ umask 077
readonly EXPECTED_HOST_IP="192.168.0.110"
readonly EXPECTED_USER="wooo"
readonly DEST_ROOT="/backup/scripts"
readonly EXPORTER_ROOT="/home/wooo/scripts"
readonly EXPORTER_FILE="backup-health-textfile-exporter.py"
readonly STATUS_ROOT="/backup/status"
readonly STAGE_ROOT="/backup/.agent99-backup-runtime-stage"
readonly ROLLBACK_ROOT="/backup/.agent99-backup-runtime-rollback"
@@ -31,6 +33,7 @@ readonly -a RUNTIME_FILES=(
verify-offsite-full-sync.sh
enforce-latest-only-retention.sh
)
readonly -a PAYLOAD_FILES=("${RUNTIME_FILES[@]}" "$EXPORTER_FILE")
MODE=""
SOURCE_REVISION=""
@@ -46,13 +49,14 @@ APPLY_COMPLETE=0
ROLLBACK_VERIFIED=0
declare -A FILE_EXISTED=()
declare -A PREVIOUS_DIGEST=()
declare -A PREVIOUS_METADATA=()
declare -A EXPECTED_DIGEST=()
usage() {
printf '%s\n' \
"Usage: $0 --mode check|apply|verify --source-revision SHA --run-id ID --source-stage PATH" \
"The executor is fixed to host 192.168.0.110, an Agent99 run-bound source stage," \
"and the 17-file /backup/scripts transaction."
"and the 18-file transaction: 17 /backup/scripts files plus the backup health exporter."
}
fail() {
@@ -93,7 +97,7 @@ case "$MODE" in check|apply|verify) ;; *) fail "invalid_mode" ;; esac
expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"
[ "$SOURCE_STAGE" = "$expected_source_stage" ] || fail "invalid_source_stage"
for command_name in awk bash cp flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do
for command_name in awk bash cp dirname flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do
command -v "$command_name" >/dev/null 2>&1 || fail "required_command_missing_${command_name}"
done
@@ -102,8 +106,9 @@ current_user="$(id -un)"
hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx "$EXPECTED_HOST_IP" || fail "unexpected_executor_host"
[ -d "$SOURCE_STAGE" ] && [ ! -L "$SOURCE_STAGE" ] || fail "canonical_source_stage_unavailable"
[ -d "$DEST_ROOT" ] && [ ! -L "$DEST_ROOT" ] || fail "backup_runtime_destination_unavailable"
[ -d "$EXPORTER_ROOT" ] && [ ! -L "$EXPORTER_ROOT" ] || fail "backup_exporter_destination_unavailable"
manifest_rows="$(python3 - "$SOURCE_STAGE/manifest.json" "$SOURCE_REVISION" "$RUN_ID" "${RUNTIME_FILES[@]}" <<'PY'
manifest_rows="$(python3 - "$SOURCE_STAGE/manifest.json" "$SOURCE_REVISION" "$RUN_ID" "${PAYLOAD_FILES[@]}" <<'PY'
import json
import re
import sys
@@ -184,7 +189,7 @@ source_digest() {
validate_source() {
local name source_path expected_digest actual_digest resolved
for name in "${RUNTIME_FILES[@]}"; do
for name in "${PAYLOAD_FILES[@]}"; do
source_path="$SOURCE_STAGE/$name"
[ -f "$source_path" ] && [ ! -L "$source_path" ] || return 65
resolved="$(readlink -f "$source_path")"
@@ -196,10 +201,19 @@ validate_source() {
done
}
destination_path() {
local name="$1"
if [ "$name" = "$EXPORTER_FILE" ]; then
printf '%s/%s\n' "$EXPORTER_ROOT" "$name"
else
printf '%s/%s\n' "$DEST_ROOT" "$name"
fi
}
verify_destination() {
local name dest_path expected_digest actual_digest
for name in "${RUNTIME_FILES[@]}"; do
dest_path="$DEST_ROOT/$name"
for name in "${PAYLOAD_FILES[@]}"; do
dest_path="$(destination_path "$name")"
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] || return 68
expected_digest="$(source_digest "$name")"
actual_digest="$(working_digest "$dest_path")"
@@ -227,7 +241,9 @@ document = {
"sourceRevision": sys.argv[3],
"sourceHead": sys.argv[4],
"runId": sys.argv[5],
"fileCount": 17,
"fileCount": 18,
"backupScriptFileCount": 17,
"backupHealthExporterIncluded": True,
"rollbackPerformed": sys.argv[6] == "1",
"rollbackVerified": sys.argv[6] == "1",
"verifierSha256": sys.argv[7],
@@ -248,23 +264,26 @@ PY
}
record_prestate() {
local name dest_path digest
local name dest_path digest metadata
: > "$ROLLBACK_DIR/prestate.tsv"
chmod 600 "$ROLLBACK_DIR/prestate.tsv"
for name in "${RUNTIME_FILES[@]}"; do
dest_path="$DEST_ROOT/$name"
for name in "${PAYLOAD_FILES[@]}"; do
dest_path="$(destination_path "$name")"
if [ -f "$dest_path" ] && [ ! -L "$dest_path" ]; then
[ "$(stat -c '%U:%G' "$dest_path")" = "wooo:wooo" ] || return 71
FILE_EXISTED[$name]=1
digest="$(working_digest "$dest_path")"
metadata="$(stat -c '%u:%g:%a' "$dest_path")"
PREVIOUS_DIGEST[$name]="$digest"
printf '%s\t1\t%s\n' "$name" "$digest" >> "$ROLLBACK_DIR/prestate.tsv"
PREVIOUS_METADATA[$name]="$metadata"
printf '%s\t1\t%s\t%s\n' "$name" "$digest" "$metadata" >> "$ROLLBACK_DIR/prestate.tsv"
cp -p -- "$dest_path" "$ROLLBACK_DIR/$name"
[ "$(working_digest "$ROLLBACK_DIR/$name")" = "$digest" ] || return 72
elif [ ! -e "$dest_path" ]; then
FILE_EXISTED[$name]=0
PREVIOUS_DIGEST[$name]="-"
printf '%s\t0\t-\n' "$name" >> "$ROLLBACK_DIR/prestate.tsv"
PREVIOUS_METADATA[$name]="-"
printf '%s\t0\t-\t-\n' "$name" >> "$ROLLBACK_DIR/prestate.tsv"
else
return 73
fi
@@ -272,14 +291,15 @@ record_prestate() {
}
rollback_transaction() {
local name dest_path backup_path temporary failed=0
local name dest_path destination_parent backup_path temporary failed=0
set +e
for name in "${RUNTIME_FILES[@]}"; do
for name in "${PAYLOAD_FILES[@]}"; do
[ -n "${FILE_EXISTED[$name]+x}" ] || { failed=1; continue; }
dest_path="$DEST_ROOT/$name"
dest_path="$(destination_path "$name")"
if [ "${FILE_EXISTED[$name]}" = "1" ]; then
backup_path="$ROLLBACK_DIR/$name"
temporary="$DEST_ROOT/.${name}.agent99-rollback-${RUN_ID}"
destination_parent="$(dirname "$dest_path")"
temporary="$destination_parent/.${name}.agent99-rollback-${RUN_ID}"
cp -p -- "$backup_path" "$temporary" \
&& mv -f -- "$temporary" "$dest_path" \
|| failed=1
@@ -287,11 +307,12 @@ rollback_transaction() {
rm -f -- "$dest_path" || failed=1
fi
done
for name in "${RUNTIME_FILES[@]}"; do
dest_path="$DEST_ROOT/$name"
for name in "${PAYLOAD_FILES[@]}"; do
dest_path="$(destination_path "$name")"
if [ "${FILE_EXISTED[$name]:-}" = "1" ]; then
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] \
&& [ "$(working_digest "$dest_path" 2>/dev/null)" = "${PREVIOUS_DIGEST[$name]:-invalid}" ] \
&& [ "$(stat -c '%u:%g:%a' "$dest_path" 2>/dev/null)" = "${PREVIOUS_METADATA[$name]:-invalid}" ] \
|| failed=1
elif [ "${FILE_EXISTED[$name]:-}" = "0" ]; then
[ ! -e "$dest_path" ] || failed=1
@@ -328,13 +349,13 @@ cleanup() {
validate_source || fail "source_validation_failed"
if [ "$MODE" = "check" ]; then
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":17,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
exit 0
fi
if [ "$MODE" = "verify" ]; then
verify_destination || fail "destination_verification_failed"
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":17,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
exit 0
fi
@@ -362,7 +383,7 @@ trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM HUP
for name in "${RUNTIME_FILES[@]}"; do
for name in "${PAYLOAD_FILES[@]}"; do
install -m 700 "$SOURCE_STAGE/$name" "$STAGE_DIR/$name"
[ "$(working_digest "$STAGE_DIR/$name")" = "$(source_digest "$name")" ] || fail "stage_identity_failed"
validate_file "$STAGE_DIR/$name" || fail "stage_validation_failed"
@@ -374,16 +395,19 @@ APPLY_STARTED=1
# common.sh is promoted first. Every managed mutating backup entrypoint then
# takes the shared side of LOCK_PATH before doing work, while this process holds
# the exclusive side until the independent verifier has completed.
for name in "${RUNTIME_FILES[@]}"; do
temporary="$DEST_ROOT/.${name}.agent99-${RUN_ID}"
for name in "${PAYLOAD_FILES[@]}"; do
dest_path="$(destination_path "$name")"
destination_parent="$(dirname "$dest_path")"
temporary="$destination_parent/.${name}.agent99-${RUN_ID}"
install -m 755 "$STAGE_DIR/$name" "$temporary"
mv -f -- "$temporary" "$DEST_ROOT/$name"
mv -f -- "$temporary" "$dest_path"
done
verify_destination || fail "post_apply_verification_failed"
verifier_result="$(BACKUP_RUNTIME_DEPLOY_CONTEXT=1 python3 "$SOURCE_STAGE/verify-host110-backup-runtime.py" \
--manifest "$SOURCE_STAGE/manifest.json" \
--destination "$DEST_ROOT" \
--exporter-destination "$EXPORTER_ROOT/$EXPORTER_FILE" \
--source-revision "$SOURCE_REVISION" \
--run-id "$RUN_ID" \
--expected-verifier-sha256 "$VERIFIER_DIGEST")" || fail "independent_verifier_failed"
@@ -400,7 +424,9 @@ if not (
and row.get("ok") is True
and row.get("sourceRevision") == sys.argv[2]
and row.get("runId") == sys.argv[3]
and row.get("fileCount") == 17
and row.get("fileCount") == 18
and row.get("backupScriptFileCount") == 17
and row.get("backupHealthExporterIncluded") is True
and row.get("remoteWritePerformed") is False
and row.get("selfIdentityVerified") is True
):
@@ -423,7 +449,9 @@ print(json.dumps({
"ok": True,
"sourceRevision": sys.argv[1],
"sourceHead": sys.argv[2],
"fileCount": 17,
"fileCount": 18,
"backupScriptFileCount": 17,
"backupHealthExporterIncluded": True,
"rollbackPerformed": False,
"receipt": sys.argv[3],
"verifier": json.loads(sys.argv[4]),

19
scripts/backup/verify-host110-backup-runtime.py
View File

@@ -16,8 +16,9 @@ from pathlib import Path
EXPECTED_DESTINATION = Path("/backup/scripts")
EXPECTED_EXPORTER_DESTINATION = Path("/home/wooo/scripts/backup-health-textfile-exporter.py")
RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")
EXPECTED_FILES = (
EXPECTED_BACKUP_FILES = (
"common.sh",
"backup-all.sh",
"backup-host188-products.sh",
@@ -36,6 +37,8 @@ EXPECTED_FILES = (
"verify-offsite-full-sync.sh",
"enforce-latest-only-retention.sh",
)
EXPORTER_FILE = "backup-health-textfile-exporter.py"
EXPECTED_FILES = EXPECTED_BACKUP_FILES + (EXPORTER_FILE,)
def fail(reason: str) -> None:
@@ -78,6 +81,7 @@ def main() -> None:
manifest_source.add_argument("--manifest")
manifest_source.add_argument("--manifest-base64")
parser.add_argument("--destination", required=True)
parser.add_argument("--exporter-destination", required=True)
parser.add_argument("--source-revision", required=True)
parser.add_argument("--run-id", required=True)
parser.add_argument("--expected-verifier-sha256", required=True)
@@ -90,6 +94,9 @@ def main() -> None:
destination = Path(args.destination)
if destination != EXPECTED_DESTINATION or destination.is_symlink() or not destination.is_dir():
fail("invalid_destination")
exporter_destination = Path(args.exporter_destination)
if exporter_destination != EXPECTED_EXPORTER_DESTINATION:
fail("invalid_exporter_destination")
runtime_lock = acquire_runtime_read_lock(destination)
if not re.fullmatch(r"[0-9a-f]{64}", args.expected_verifier_sha256):
@@ -131,11 +138,14 @@ def main() -> None:
root = destination.resolve(strict=True)
for name in EXPECTED_FILES:
path = destination / name
path = exporter_destination if name == EXPORTER_FILE else destination / name
if path.is_symlink() or not path.is_file():
fail("runtime_file_type_failed")
resolved = path.resolve(strict=True)
if resolved.parent != root:
if name == EXPORTER_FILE:
if resolved != exporter_destination:
fail("runtime_file_boundary_failed")
elif resolved.parent != root:
fail("runtime_file_boundary_failed")
stat = resolved.stat()
if stat.st_uid != os.getuid() or stat.st_gid != os.getgid() or stat.st_mode & 0o777 != 0o755:
@@ -149,7 +159,10 @@ def main() -> None:
"sourceRevision": args.source_revision,
"runId": args.run_id,
"fileCount": len(EXPECTED_FILES),
"backupScriptFileCount": len(EXPECTED_BACKUP_FILES),
"backupHealthExporterIncluded": True,
"destination": str(EXPECTED_DESTINATION),
"exporterDestination": str(EXPECTED_EXPORTER_DESTINATION),
"remoteWritePerformed": False,
"secretValuesRead": False,
"selfIdentityVerified": self_identity_verified,