fix(agent99): close host110 backup commit races

2026-07-19 01:08:45 +08:00
parent 8f11ef3362
commit d1797a1c87
5 changed files with 1282 additions and 42 deletions
--- a/scripts/backup/host110-backup-runtime-executor.sh
+++ b/scripts/backup/host110-backup-runtime-executor.sh
@@ -45,8 +45,12 @@ VERIFIER_DIGEST=""
 STAGE_DIR=""
 ROLLBACK_DIR=""
 RECEIPT_PATH=""
+TERMINAL_RECEIPT_PATH=""
 APPLY_STARTED=0
 APPLY_COMPLETE=0
+COMMIT_CRITICAL=0
+DEFERRED_SIGNAL=""
+DEFERRED_SIGNAL_EXIT=0
 ROLLBACK_ATTEMPTED=0
 ROLLBACK_PERFORMED=0
 ROLLBACK_VERIFIED=0
@@ -103,10 +107,17 @@ case "$MODE" in check|apply|verify) ;; *) fail "invalid_mode" ;; esac
 expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"
 [ "$SOURCE_STAGE" = "$expected_source_stage" ] || fail "invalid_source_stage"
 RECEIPT_PATH="$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json"
+TERMINAL_RECEIPT_PATH="$STATUS_ROOT/agent99-backup-runtime-terminal-${RUN_ID}.json"
 if [ "${HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS:-}" = "1" ] \
  && [ "$DEST_ROOT" != "/backup/scripts" ]; then
  FAULT_INJECTION_ENABLED=1
 fi
+if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then
+  case "${HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL:-}" in ""|INT|HUP|TERM) ;; *) fail "invalid_test_commit_signal" ;; esac
+  case "${HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL:-}" in ""|INT|HUP|TERM) ;; *) fail "invalid_test_terminal_signal" ;; esac
+  case "${HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT:-}" in ""|write|link|fsync|readback) ;; *) fail "invalid_test_terminal_receipt_fault" ;; esac
+  case "${HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT:-}" in ""|stage|rollback_prestate) ;; *) fail "invalid_test_cleanup_fault" ;; esac
+fi

 for command_name in awk bash cp dirname flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do
  command -v "$command_name" >/dev/null 2>&1 || fail "required_command_missing_${command_name}"
@@ -314,9 +325,9 @@ rollback_attempted = sys.argv[6] == "1"
 rollback_performed = sys.argv[7] == "1"
 rollback_verified = sys.argv[8] == "1"
 zero_residue_verified = sys.argv[9] == "1"
-if status not in {"verified", "failed_rolled_back", "rollback_unverified"}:
+if status not in {"payload_verified", "failed_rolled_back", "rollback_unverified"}:
    raise SystemExit(78)
-if status == "verified" and (rollback_attempted or rollback_performed or rollback_verified or not zero_residue_verified):
+if status == "payload_verified" and (rollback_attempted or rollback_performed or rollback_verified or not zero_residue_verified):
    raise SystemExit(78)
 if status == "failed_rolled_back" and not (
    rollback_attempted and rollback_performed and rollback_verified and zero_residue_verified
@@ -325,7 +336,7 @@ if status == "failed_rolled_back" and not (
 if status == "rollback_unverified" and (not rollback_attempted or rollback_verified):
    raise SystemExit(78)
 document = {
-    "schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v2",
+    "schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v3",
    "status": status,
    "sourceRevision": sys.argv[3],
    "sourceHead": sys.argv[4],
@@ -337,6 +348,7 @@ document = {
    "rollbackPerformed": rollback_performed,
    "rollbackVerified": rollback_verified,
    "zeroResidueVerified": zero_residue_verified,
+    "zeroResidueScope": "run_owned_destination_temps",
    "verifierSha256": sys.argv[10],
    "executorHost": "192.168.0.110",
    "productionServiceRestarted": False,
@@ -349,7 +361,7 @@ temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
 fault_enabled = sys.argv[11] == "1"
 fault = (
    os.environ.get("HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT", "")
-    if status == "verified" and fault_enabled
+    if status == "payload_verified" and fault_enabled
    else ""
 )
 if fault not in {"", "write", "link", "fsync", "readback"}:
@@ -412,6 +424,172 @@ PY
  return 0
 }

+write_terminal_receipt() {
+  local terminal_status="$1"
+  local stage_cleanup_verified="$2"
+  local rollback_prestate_cleanup_verified="$3"
+  local deferred_signal="$4"
+  local terminal_exit_code="$5"
+  install -d -m 700 "$STATUS_ROOT" || return 1
+  [ -d "$STATUS_ROOT" ] && [ ! -L "$STATUS_ROOT" ] || return 1
+  [ "$(stat -c '%U:%G:%a' "$STATUS_ROOT")" = "wooo:wooo:700" ] || return 1
+  if ! python3 - "$TERMINAL_RECEIPT_PATH" "$RECEIPT_PATH" "$terminal_status" \
+    "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$stage_cleanup_verified" \
+    "$rollback_prestate_cleanup_verified" "$deferred_signal" "$terminal_exit_code" \
+    "$VERIFIER_DIGEST" "$FAULT_INJECTION_ENABLED" <<'PY'
+import hashlib
+import json
+import os
+import signal
+import sys
+import time
+from pathlib import Path
+
+path = Path(sys.argv[1])
+payload_receipt_path = Path(sys.argv[2])
+status = sys.argv[3]
+source_revision = sys.argv[4]
+source_head = sys.argv[5]
+run_id = sys.argv[6]
+stage_cleanup_verified = sys.argv[7] == "1"
+rollback_prestate_cleanup_verified = sys.argv[8] == "1"
+deferred_signal = sys.argv[9]
+terminal_exit_code = int(sys.argv[10])
+verifier_digest = sys.argv[11]
+fault_enabled = sys.argv[12] == "1"
+signal_exit_codes = {"INT": 130, "HUP": 129, "TERM": 143}
+
+terminal_signal = os.environ.get("HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL", "") if fault_enabled else ""
+terminal_fault = os.environ.get("HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT", "") if fault_enabled else ""
+if terminal_signal:
+    os.kill(os.getppid(), getattr(signal, f"SIG{terminal_signal}"))
+    time.sleep(0.05)
+
+try:
+    payload_receipt = json.loads(payload_receipt_path.read_text(encoding="utf-8"))
+except (OSError, json.JSONDecodeError):
+    raise SystemExit(79)
+if not (
+    payload_receipt.get("schemaVersion") == "agent99_host110_backup_runtime_deploy_receipt_v3"
+    and payload_receipt.get("status") == "payload_verified"
+    and payload_receipt.get("sourceRevision") == source_revision
+    and payload_receipt.get("sourceHead") == source_head
+    and payload_receipt.get("runId") == run_id
+    and payload_receipt.get("zeroResidueVerified") is True
+):
+    raise SystemExit(79)
+
+if status == "verified":
+    valid = (
+        stage_cleanup_verified
+        and rollback_prestate_cleanup_verified
+        and not deferred_signal
+        and terminal_exit_code == 0
+    )
+elif status == "cleanup_pending":
+    valid = (
+        not (stage_cleanup_verified and rollback_prestate_cleanup_verified)
+        and deferred_signal in {"", *signal_exit_codes}
+        and terminal_exit_code != 0
+    )
+elif status == "committed_signal_deferred":
+    valid = (
+        stage_cleanup_verified
+        and rollback_prestate_cleanup_verified
+        and deferred_signal in signal_exit_codes
+        and terminal_exit_code == signal_exit_codes[deferred_signal]
+    )
+else:
+    valid = False
+if not valid:
+    raise SystemExit(79)
+
+document = {
+    "schemaVersion": "agent99_host110_backup_runtime_terminal_receipt_v1",
+    "status": status,
+    "ok": status == "verified",
+    "payloadCommitted": True,
+    "payloadReceipt": str(payload_receipt_path),
+    "sourceRevision": source_revision,
+    "sourceHead": source_head,
+    "runId": run_id,
+    "fileCount": 18,
+    "backupScriptFileCount": 17,
+    "backupHealthExporterIncluded": True,
+    "stageCleanupVerified": stage_cleanup_verified,
+    "rollbackPrestateCleanupVerified": rollback_prestate_cleanup_verified,
+    "cleanupVerified": stage_cleanup_verified and rollback_prestate_cleanup_verified,
+    "independentVerifierVerified": True,
+    "zeroResidueScope": "run_owned_destination_temps_and_internal_stage_prestate",
+    "deferredSignal": deferred_signal or None,
+    "terminalExitCode": terminal_exit_code,
+    "verifierSha256": verifier_digest,
+    "executorHost": "192.168.0.110",
+    "productionServiceRestarted": False,
+    "secretValuesRead": False,
+    "writtenAt": int(time.time()),
+}
+payload = (json.dumps(document, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")
+expected_digest = hashlib.sha256(payload).hexdigest()
+temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
+created_final = False
+directory_fd = None
+try:
+    if os.path.lexists(path) or os.path.lexists(temporary):
+        raise FileExistsError(path)
+    if terminal_fault == "write":
+        raise OSError("fault_injected_terminal_receipt_write")
+    descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    with os.fdopen(descriptor, "wb") as handle:
+        if handle.write(payload) != len(payload):
+            raise OSError("terminal_receipt_short_write")
+        handle.flush()
+        if terminal_fault == "fsync":
+            raise OSError("fault_injected_terminal_receipt_fsync")
+        os.fsync(handle.fileno())
+    os.chmod(temporary, 0o600)
+    temporary_readback = temporary.read_bytes()
+    if temporary_readback != payload or hashlib.sha256(temporary_readback).hexdigest() != expected_digest:
+        raise OSError("terminal_receipt_temporary_readback_failed")
+    if terminal_fault == "link":
+        raise OSError("fault_injected_terminal_receipt_link")
+    os.link(temporary, path)
+    created_final = True
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_fd = os.open(path.parent, directory_flags)
+    os.fsync(directory_fd)
+    if terminal_fault == "readback":
+        raise OSError("fault_injected_terminal_receipt_readback")
+    readback = path.read_bytes()
+    if readback != payload or hashlib.sha256(readback).hexdigest() != expected_digest:
+        raise OSError("terminal_receipt_final_readback_failed")
+    if json.loads(readback.decode("utf-8")) != document:
+        raise OSError("terminal_receipt_document_readback_failed")
+    temporary.unlink()
+    os.fsync(directory_fd)
+except BaseException:
+    try:
+        if created_final:
+            path.unlink(missing_ok=True)
+        temporary.unlink(missing_ok=True)
+        if directory_fd is None and path.parent.is_dir():
+            directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+            directory_fd = os.open(path.parent, directory_flags)
+        if directory_fd is not None:
+            os.fsync(directory_fd)
+    finally:
+        raise
+finally:
+    if directory_fd is not None:
+        os.close(directory_fd)
+PY
+  then
+    return 1
+  fi
+  [ -f "$TERMINAL_RECEIPT_PATH" ] && [ ! -L "$TERMINAL_RECEIPT_PATH" ] || return 1
+  return 0
+}
+
 record_prestate() {
  local name dest_path digest metadata
  : > "$ROLLBACK_DIR/prestate.tsv" || return 74
@@ -530,6 +708,35 @@ rollback_transaction() {
  return 1
 }

+handle_agent99_signal() {
+  local signal_name="$1"
+  local signal_exit_code="$2"
+  if [ "$COMMIT_CRITICAL" -eq 1 ] || [ "$APPLY_COMPLETE" -eq 1 ]; then
+    if [ -z "$DEFERRED_SIGNAL" ]; then
+      DEFERRED_SIGNAL="$signal_name"
+      DEFERRED_SIGNAL_EXIT="$signal_exit_code"
+    fi
+    return 0
+  fi
+  exit "$signal_exit_code"
+}
+
+cleanup_postcommit_directory() {
+  local cleanup_kind="$1"
+  local cleanup_path="$2"
+  local fault=""
+  if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then
+    fault="${HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT:-}"
+  fi
+  if [ "$fault" = "$cleanup_kind" ]; then
+    return 1
+  fi
+  rm -rf -- "$cleanup_path" || return 1
+  [ ! -e "$cleanup_path" ] && [ ! -L "$cleanup_path" ] || return 1
+  fsync_paths_and_parents "$cleanup_path" || return 1
+  [ ! -e "$cleanup_path" ] && [ ! -L "$cleanup_path" ]
+}
+
 cleanup() {
  local exit_status=$?
  local rollback_status="rollback_unverified"
@@ -544,10 +751,12 @@ cleanup() {
      failure_receipt_written=1
    fi
  fi
-  if [ -n "$STAGE_DIR" ]; then
+  if [ "$APPLY_COMPLETE" -ne 1 ] && [ -n "$STAGE_DIR" ]; then
    rm -rf -- "$STAGE_DIR" || true
  fi
-  if [ "$ROLLBACK_VERIFIED" -eq 1 ] && [ "$failure_receipt_written" -eq 1 ]; then
+  if [ "$APPLY_COMPLETE" -ne 1 ] \
+    && [ "$ROLLBACK_VERIFIED" -eq 1 ] \
+    && [ "$failure_receipt_written" -eq 1 ]; then
    if [ -n "$ROLLBACK_DIR" ]; then
      rm -rf -- "$ROLLBACK_DIR" || true
    fi
@@ -558,17 +767,19 @@ cleanup() {
 validate_source || fail "source_validation_failed"

 if [ "$MODE" = "check" ]; then
-  printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
+  printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","runId":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID"
  exit 0
 fi

 if [ "$MODE" = "verify" ]; then
  verify_destination || fail "destination_verification_failed"
-  printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
+  printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","runId":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID"
  exit 0
 fi

 [ ! -e "$RECEIPT_PATH" ] && [ ! -L "$RECEIPT_PATH" ] || fail "run_identity_receipt_exists"
+[ ! -e "$TERMINAL_RECEIPT_PATH" ] && [ ! -L "$TERMINAL_RECEIPT_PATH" ] \
+  || fail "run_identity_terminal_receipt_exists"
 STAGE_DIR="$STAGE_ROOT/$RUN_ID"
 ROLLBACK_DIR="$ROLLBACK_ROOT/$RUN_ID"
 [ ! -e "$STAGE_DIR" ] || fail "run_identity_stage_exists"
@@ -589,8 +800,9 @@ fi

 install -d -m 700 "$STAGE_DIR" "$ROLLBACK_DIR"
 trap cleanup EXIT
-trap 'exit 130' INT
-trap 'exit 143' TERM HUP
+trap 'handle_agent99_signal INT 130' INT
+trap 'handle_agent99_signal HUP 129' HUP
+trap 'handle_agent99_signal TERM 143' TERM

 for name in "${PAYLOAD_FILES[@]}"; do
  install -m 700 "$SOURCE_STAGE/$name" "$STAGE_DIR/$name"
@@ -651,43 +863,116 @@ PY

 verify_run_owned_temp_absence || fail "post_apply_temp_residue_detected"
 RUN_TEMP_RESIDUE_VERIFIED=1
-if ! write_receipt "verified" 0 0 0 "$RUN_TEMP_RESIDUE_VERIFIED"; then
+COMMIT_CRITICAL=1
+if ! write_receipt "payload_verified" 0 0 0 "$RUN_TEMP_RESIDUE_VERIFIED"; then
+  COMMIT_CRITICAL=0
  fail "durable_receipt_failed"
 fi
-receipt_path="$RECEIPT_PATH"
+if [ "$FAULT_INJECTION_ENABLED" -eq 1 ] \
+  && [ -n "${HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL:-}" ]; then
+  case "$HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL" in
+    INT|HUP|TERM) kill -s "$HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL" "$$" ;;
+  esac
+fi
 APPLY_COMPLETE=1
-trap - EXIT HUP INT TERM
+COMMIT_CRITICAL=0
 stage_cleanup_verified=0
 rollback_prestate_cleanup_verified=0
-if rm -rf -- "$STAGE_DIR" && [ ! -e "$STAGE_DIR" ] && [ ! -L "$STAGE_DIR" ]; then
+if cleanup_postcommit_directory "stage" "$STAGE_DIR"; then
  stage_cleanup_verified=1
 fi
-if rm -rf -- "$ROLLBACK_DIR" && [ ! -e "$ROLLBACK_DIR" ] && [ ! -L "$ROLLBACK_DIR" ]; then
+if cleanup_postcommit_directory "rollback_prestate" "$ROLLBACK_DIR"; then
  rollback_prestate_cleanup_verified=1
 fi
-STAGE_DIR=""
-ROLLBACK_DIR=""
-python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$receipt_path" "$verifier_result" \
-  "$stage_cleanup_verified" "$rollback_prestate_cleanup_verified" <<'PY'
+terminal_status="verified"
+terminal_exit_code=0
+# Cleanup has completed. Freeze terminal signal state before the immutable
+# terminal writer so its receipt, stdout, and exit status cannot diverge.
+trap '' HUP INT TERM
+if [ "$stage_cleanup_verified" -ne 1 ] || [ "$rollback_prestate_cleanup_verified" -ne 1 ]; then
+  terminal_status="cleanup_pending"
+  terminal_exit_code=75
+elif [ -n "$DEFERRED_SIGNAL" ]; then
+  terminal_status="committed_signal_deferred"
+  terminal_exit_code="$DEFERRED_SIGNAL_EXIT"
+fi
+if ! write_terminal_receipt "$terminal_status" "$stage_cleanup_verified" \
+  "$rollback_prestate_cleanup_verified" "$DEFERRED_SIGNAL" "$terminal_exit_code"; then
+  trap - EXIT
+  python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$RECEIPT_PATH" \
+    "$TERMINAL_RECEIPT_PATH" "$stage_cleanup_verified" \
+    "$rollback_prestate_cleanup_verified" "$DEFERRED_SIGNAL" <<'PY'
 import json
 import sys

 print(json.dumps({
    "schemaVersion": "agent99_host110_backup_runtime_executor_v1",
    "mode": "apply",
-    "ok": True,
+    "status": "terminal_receipt_failed",
+    "ok": False,
    "sourceRevision": sys.argv[1],
    "sourceHead": sys.argv[2],
+    "runId": sys.argv[3],
    "fileCount": 18,
    "backupScriptFileCount": 17,
    "backupHealthExporterIncluded": True,
+    "payloadCommitted": True,
    "rollbackAttempted": False,
    "rollbackPerformed": False,
    "rollbackVerified": False,
    "zeroResidueVerified": True,
-    "stageCleanupVerified": sys.argv[5] == "1",
-    "rollbackPrestateCleanupVerified": sys.argv[6] == "1",
-    "receipt": sys.argv[3],
-    "verifier": json.loads(sys.argv[4]),
+    "zeroResidueScope": "run_owned_destination_temps",
+    "stageCleanupVerified": sys.argv[6] == "1",
+    "rollbackPrestateCleanupVerified": sys.argv[7] == "1",
+    "receipt": sys.argv[4],
+    "terminalReceipt": sys.argv[5],
+    "terminalReceiptPublished": False,
+    "terminalExitCode": 76,
+    "deferredSignal": sys.argv[8] or None,
+    "independentVerifierVerified": True,
+    "verifier": None,
 }, ensure_ascii=True, sort_keys=True))
 PY
+  printf 'HOST110_BACKUP_RUNTIME_OK=0\nERROR=terminal_receipt_failed\n' >&2
+  exit 76
+fi
+if [ "$stage_cleanup_verified" -eq 1 ]; then STAGE_DIR=""; fi
+if [ "$rollback_prestate_cleanup_verified" -eq 1 ]; then ROLLBACK_DIR=""; fi
+trap - EXIT
+python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$RECEIPT_PATH" "$TERMINAL_RECEIPT_PATH" \
+  "$verifier_result" "$stage_cleanup_verified" "$rollback_prestate_cleanup_verified" \
+  "$terminal_status" "$terminal_exit_code" "$DEFERRED_SIGNAL" "$RUN_ID" <<'PY'
+import json
+import sys
+
+status = sys.argv[8]
+verifier = json.loads(sys.argv[5])
+print(json.dumps({
+    "schemaVersion": "agent99_host110_backup_runtime_executor_v1",
+    "mode": "apply",
+    "status": status,
+    "ok": status == "verified",
+    "sourceRevision": sys.argv[1],
+    "sourceHead": sys.argv[2],
+    "runId": sys.argv[11],
+    "fileCount": 18,
+    "backupScriptFileCount": 17,
+    "backupHealthExporterIncluded": True,
+    "payloadCommitted": True,
+    "rollbackAttempted": False,
+    "rollbackPerformed": False,
+    "rollbackVerified": False,
+    "zeroResidueVerified": True,
+    "zeroResidueScope": "run_owned_destination_temps",
+    "stageCleanupVerified": sys.argv[6] == "1",
+    "rollbackPrestateCleanupVerified": sys.argv[7] == "1",
+    "receipt": sys.argv[3],
+    "terminalReceipt": sys.argv[4],
+    "terminalReceiptPublished": True,
+    "terminalExitCode": int(sys.argv[9]),
+    "deferredSignal": sys.argv[10] or None,
+    "independentVerifierVerified": True,
+    "verifier": verifier if status == "verified" else None,
+}, ensure_ascii=True, sort_keys=True))
+PY
+exit "$terminal_exit_code"