fix(agent99): close host110 backup commit races

2026-07-19 01:08:45 +08:00
parent 8f11ef3362
commit d1797a1c87
5 changed files with 1282 additions and 42 deletions
--- a/scripts/backup/host110-backup-runtime-executor.sh
+++ b/scripts/backup/host110-backup-runtime-executor.sh
@@ -45,8 +45,12 @@ VERIFIER_DIGEST=""
 STAGE_DIR=""
 ROLLBACK_DIR=""
 RECEIPT_PATH=""
+TERMINAL_RECEIPT_PATH=""
 APPLY_STARTED=0
 APPLY_COMPLETE=0
+COMMIT_CRITICAL=0
+DEFERRED_SIGNAL=""
+DEFERRED_SIGNAL_EXIT=0
 ROLLBACK_ATTEMPTED=0
 ROLLBACK_PERFORMED=0
 ROLLBACK_VERIFIED=0
@@ -103,10 +107,17 @@ case "$MODE" in check|apply|verify) ;; *) fail "invalid_mode" ;; esac
 expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"
 [ "$SOURCE_STAGE" = "$expected_source_stage" ] || fail "invalid_source_stage"
 RECEIPT_PATH="$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json"
+TERMINAL_RECEIPT_PATH="$STATUS_ROOT/agent99-backup-runtime-terminal-${RUN_ID}.json"
 if [ "${HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS:-}" = "1" ] \
  && [ "$DEST_ROOT" != "/backup/scripts" ]; then
  FAULT_INJECTION_ENABLED=1
 fi
+if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then
+  case "${HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL:-}" in ""|INT|HUP|TERM) ;; *) fail "invalid_test_commit_signal" ;; esac
+  case "${HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL:-}" in ""|INT|HUP|TERM) ;; *) fail "invalid_test_terminal_signal" ;; esac
+  case "${HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT:-}" in ""|write|link|fsync|readback) ;; *) fail "invalid_test_terminal_receipt_fault" ;; esac
+  case "${HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT:-}" in ""|stage|rollback_prestate) ;; *) fail "invalid_test_cleanup_fault" ;; esac
+fi

 for command_name in awk bash cp dirname flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do
  command -v "$command_name" >/dev/null 2>&1 || fail "required_command_missing_${command_name}"
@@ -314,9 +325,9 @@ rollback_attempted = sys.argv[6] == "1"
 rollback_performed = sys.argv[7] == "1"
 rollback_verified = sys.argv[8] == "1"
 zero_residue_verified = sys.argv[9] == "1"
-if status not in {"verified", "failed_rolled_back", "rollback_unverified"}:
+if status not in {"payload_verified", "failed_rolled_back", "rollback_unverified"}:
    raise SystemExit(78)
-if status == "verified" and (rollback_attempted or rollback_performed or rollback_verified or not zero_residue_verified):
+if status == "payload_verified" and (rollback_attempted or rollback_performed or rollback_verified or not zero_residue_verified):
    raise SystemExit(78)
 if status == "failed_rolled_back" and not (
    rollback_attempted and rollback_performed and rollback_verified and zero_residue_verified
@@ -325,7 +336,7 @@ if status == "failed_rolled_back" and not (
 if status == "rollback_unverified" and (not rollback_attempted or rollback_verified):
    raise SystemExit(78)
 document = {
-    "schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v2",
+    "schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v3",
    "status": status,
    "sourceRevision": sys.argv[3],
    "sourceHead": sys.argv[4],
@@ -337,6 +348,7 @@ document = {
    "rollbackPerformed": rollback_performed,
    "rollbackVerified": rollback_verified,
    "zeroResidueVerified": zero_residue_verified,
+    "zeroResidueScope": "run_owned_destination_temps",
    "verifierSha256": sys.argv[10],
    "executorHost": "192.168.0.110",
    "productionServiceRestarted": False,
@@ -349,7 +361,7 @@ temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
 fault_enabled = sys.argv[11] == "1"
 fault = (
    os.environ.get("HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT", "")
-    if status == "verified" and fault_enabled
+    if status == "payload_verified" and fault_enabled
    else ""
 )
 if fault not in {"", "write", "link", "fsync", "readback"}:
@@ -412,6 +424,172 @@ PY
  return 0
 }

+write_terminal_receipt() {
+  local terminal_status="$1"
+  local stage_cleanup_verified="$2"
+  local rollback_prestate_cleanup_verified="$3"
+  local deferred_signal="$4"
+  local terminal_exit_code="$5"
+  install -d -m 700 "$STATUS_ROOT" || return 1
+  [ -d "$STATUS_ROOT" ] && [ ! -L "$STATUS_ROOT" ] || return 1
+  [ "$(stat -c '%U:%G:%a' "$STATUS_ROOT")" = "wooo:wooo:700" ] || return 1
+  if ! python3 - "$TERMINAL_RECEIPT_PATH" "$RECEIPT_PATH" "$terminal_status" \
+    "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$stage_cleanup_verified" \
+    "$rollback_prestate_cleanup_verified" "$deferred_signal" "$terminal_exit_code" \
+    "$VERIFIER_DIGEST" "$FAULT_INJECTION_ENABLED" <<'PY'
+import hashlib
+import json
+import os
+import signal
+import sys
+import time
+from pathlib import Path
+
+path = Path(sys.argv[1])
+payload_receipt_path = Path(sys.argv[2])
+status = sys.argv[3]
+source_revision = sys.argv[4]
+source_head = sys.argv[5]
+run_id = sys.argv[6]
+stage_cleanup_verified = sys.argv[7] == "1"
+rollback_prestate_cleanup_verified = sys.argv[8] == "1"
+deferred_signal = sys.argv[9]
+terminal_exit_code = int(sys.argv[10])
+verifier_digest = sys.argv[11]
+fault_enabled = sys.argv[12] == "1"
+signal_exit_codes = {"INT": 130, "HUP": 129, "TERM": 143}
+
+terminal_signal = os.environ.get("HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL", "") if fault_enabled else ""
+terminal_fault = os.environ.get("HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT", "") if fault_enabled else ""
+if terminal_signal:
+    os.kill(os.getppid(), getattr(signal, f"SIG{terminal_signal}"))
+    time.sleep(0.05)
+
+try:
+    payload_receipt = json.loads(payload_receipt_path.read_text(encoding="utf-8"))
+except (OSError, json.JSONDecodeError):
+    raise SystemExit(79)
+if not (
+    payload_receipt.get("schemaVersion") == "agent99_host110_backup_runtime_deploy_receipt_v3"
+    and payload_receipt.get("status") == "payload_verified"
+    and payload_receipt.get("sourceRevision") == source_revision
+    and payload_receipt.get("sourceHead") == source_head
+    and payload_receipt.get("runId") == run_id
+    and payload_receipt.get("zeroResidueVerified") is True
+):
+    raise SystemExit(79)
+
+if status == "verified":
+    valid = (
+        stage_cleanup_verified
+        and rollback_prestate_cleanup_verified
+        and not deferred_signal
+        and terminal_exit_code == 0
+    )
+elif status == "cleanup_pending":
+    valid = (
+        not (stage_cleanup_verified and rollback_prestate_cleanup_verified)
+        and deferred_signal in {"", *signal_exit_codes}
+        and terminal_exit_code != 0
+    )
+elif status == "committed_signal_deferred":
+    valid = (
+        stage_cleanup_verified
+        and rollback_prestate_cleanup_verified
+        and deferred_signal in signal_exit_codes
+        and terminal_exit_code == signal_exit_codes[deferred_signal]
+    )
+else:
+    valid = False
+if not valid:
+    raise SystemExit(79)
+
+document = {
+    "schemaVersion": "agent99_host110_backup_runtime_terminal_receipt_v1",
+    "status": status,
+    "ok": status == "verified",
+    "payloadCommitted": True,
+    "payloadReceipt": str(payload_receipt_path),
+    "sourceRevision": source_revision,
+    "sourceHead": source_head,
+    "runId": run_id,
+    "fileCount": 18,
+    "backupScriptFileCount": 17,
+    "backupHealthExporterIncluded": True,
+    "stageCleanupVerified": stage_cleanup_verified,
+    "rollbackPrestateCleanupVerified": rollback_prestate_cleanup_verified,
+    "cleanupVerified": stage_cleanup_verified and rollback_prestate_cleanup_verified,
+    "independentVerifierVerified": True,
+    "zeroResidueScope": "run_owned_destination_temps_and_internal_stage_prestate",
+    "deferredSignal": deferred_signal or None,
+    "terminalExitCode": terminal_exit_code,
+    "verifierSha256": verifier_digest,
+    "executorHost": "192.168.0.110",
+    "productionServiceRestarted": False,
+    "secretValuesRead": False,
+    "writtenAt": int(time.time()),
+}
+payload = (json.dumps(document, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")
+expected_digest = hashlib.sha256(payload).hexdigest()
+temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
+created_final = False
+directory_fd = None
+try:
+    if os.path.lexists(path) or os.path.lexists(temporary):
+        raise FileExistsError(path)
+    if terminal_fault == "write":
+        raise OSError("fault_injected_terminal_receipt_write")
+    descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    with os.fdopen(descriptor, "wb") as handle:
+        if handle.write(payload) != len(payload):
+            raise OSError("terminal_receipt_short_write")
+        handle.flush()
+        if terminal_fault == "fsync":
+            raise OSError("fault_injected_terminal_receipt_fsync")
+        os.fsync(handle.fileno())
+    os.chmod(temporary, 0o600)
+    temporary_readback = temporary.read_bytes()
+    if temporary_readback != payload or hashlib.sha256(temporary_readback).hexdigest() != expected_digest:
+        raise OSError("terminal_receipt_temporary_readback_failed")
+    if terminal_fault == "link":
+        raise OSError("fault_injected_terminal_receipt_link")
+    os.link(temporary, path)
+    created_final = True
+    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+    directory_fd = os.open(path.parent, directory_flags)
+    os.fsync(directory_fd)
+    if terminal_fault == "readback":
+        raise OSError("fault_injected_terminal_receipt_readback")
+    readback = path.read_bytes()
+    if readback != payload or hashlib.sha256(readback).hexdigest() != expected_digest:
+        raise OSError("terminal_receipt_final_readback_failed")
+    if json.loads(readback.decode("utf-8")) != document:
+        raise OSError("terminal_receipt_document_readback_failed")
+    temporary.unlink()
+    os.fsync(directory_fd)
+except BaseException:
+    try:
+        if created_final:
+            path.unlink(missing_ok=True)
+        temporary.unlink(missing_ok=True)
+        if directory_fd is None and path.parent.is_dir():
+            directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
+            directory_fd = os.open(path.parent, directory_flags)
+        if directory_fd is not None:
+            os.fsync(directory_fd)
+    finally:
+        raise
+finally:
+    if directory_fd is not None:
+        os.close(directory_fd)
+PY
+  then
+    return 1
+  fi
+  [ -f "$TERMINAL_RECEIPT_PATH" ] && [ ! -L "$TERMINAL_RECEIPT_PATH" ] || return 1
+  return 0
+}
+
 record_prestate() {
  local name dest_path digest metadata
  : > "$ROLLBACK_DIR/prestate.tsv" || return 74
@@ -530,6 +708,35 @@ rollback_transaction() {
  return 1
 }

+handle_agent99_signal() {
+  local signal_name="$1"
+  local signal_exit_code="$2"
+  if [ "$COMMIT_CRITICAL" -eq 1 ] || [ "$APPLY_COMPLETE" -eq 1 ]; then
+    if [ -z "$DEFERRED_SIGNAL" ]; then
+      DEFERRED_SIGNAL="$signal_name"
+      DEFERRED_SIGNAL_EXIT="$signal_exit_code"
+    fi
+    return 0
+  fi
+  exit "$signal_exit_code"
+}
+
+cleanup_postcommit_directory() {
+  local cleanup_kind="$1"
+  local cleanup_path="$2"
+  local fault=""
+  if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then
+    fault="${HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT:-}"
+  fi
+  if [ "$fault" = "$cleanup_kind" ]; then
+    return 1
+  fi
+  rm -rf -- "$cleanup_path" || return 1
+  [ ! -e "$cleanup_path" ] && [ ! -L "$cleanup_path" ] || return 1
+  fsync_paths_and_parents "$cleanup_path" || return 1
+  [ ! -e "$cleanup_path" ] && [ ! -L "$cleanup_path" ]
+}
+
 cleanup() {
  local exit_status=$?
  local rollback_status="rollback_unverified"
@@ -544,10 +751,12 @@ cleanup() {
      failure_receipt_written=1
    fi
  fi
-  if [ -n "$STAGE_DIR" ]; then
+  if [ "$APPLY_COMPLETE" -ne 1 ] && [ -n "$STAGE_DIR" ]; then
    rm -rf -- "$STAGE_DIR" || true
  fi
-  if [ "$ROLLBACK_VERIFIED" -eq 1 ] && [ "$failure_receipt_written" -eq 1 ]; then
+  if [ "$APPLY_COMPLETE" -ne 1 ] \
+    && [ "$ROLLBACK_VERIFIED" -eq 1 ] \
+    && [ "$failure_receipt_written" -eq 1 ]; then
    if [ -n "$ROLLBACK_DIR" ]; then
      rm -rf -- "$ROLLBACK_DIR" || true
    fi
@@ -558,17 +767,19 @@ cleanup() {
 validate_source || fail "source_validation_failed"

 if [ "$MODE" = "check" ]; then
-  printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
+  printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","runId":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID"
  exit 0
 fi

 if [ "$MODE" = "verify" ]; then
  verify_destination || fail "destination_verification_failed"
-  printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
+  printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","runId":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID"
  exit 0
 fi

 [ ! -e "$RECEIPT_PATH" ] && [ ! -L "$RECEIPT_PATH" ] || fail "run_identity_receipt_exists"
+[ ! -e "$TERMINAL_RECEIPT_PATH" ] && [ ! -L "$TERMINAL_RECEIPT_PATH" ] \
+  || fail "run_identity_terminal_receipt_exists"
 STAGE_DIR="$STAGE_ROOT/$RUN_ID"
 ROLLBACK_DIR="$ROLLBACK_ROOT/$RUN_ID"
 [ ! -e "$STAGE_DIR" ] || fail "run_identity_stage_exists"
@@ -589,8 +800,9 @@ fi

 install -d -m 700 "$STAGE_DIR" "$ROLLBACK_DIR"
 trap cleanup EXIT
-trap 'exit 130' INT
-trap 'exit 143' TERM HUP
+trap 'handle_agent99_signal INT 130' INT
+trap 'handle_agent99_signal HUP 129' HUP
+trap 'handle_agent99_signal TERM 143' TERM

 for name in "${PAYLOAD_FILES[@]}"; do
  install -m 700 "$SOURCE_STAGE/$name" "$STAGE_DIR/$name"
@@ -651,43 +863,116 @@ PY

 verify_run_owned_temp_absence || fail "post_apply_temp_residue_detected"
 RUN_TEMP_RESIDUE_VERIFIED=1
-if ! write_receipt "verified" 0 0 0 "$RUN_TEMP_RESIDUE_VERIFIED"; then
+COMMIT_CRITICAL=1
+if ! write_receipt "payload_verified" 0 0 0 "$RUN_TEMP_RESIDUE_VERIFIED"; then
+  COMMIT_CRITICAL=0
  fail "durable_receipt_failed"
 fi
-receipt_path="$RECEIPT_PATH"
+if [ "$FAULT_INJECTION_ENABLED" -eq 1 ] \
+  && [ -n "${HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL:-}" ]; then
+  case "$HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL" in
+    INT|HUP|TERM) kill -s "$HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL" "$$" ;;
+  esac
+fi
 APPLY_COMPLETE=1
-trap - EXIT HUP INT TERM
+COMMIT_CRITICAL=0
 stage_cleanup_verified=0
 rollback_prestate_cleanup_verified=0
-if rm -rf -- "$STAGE_DIR" && [ ! -e "$STAGE_DIR" ] && [ ! -L "$STAGE_DIR" ]; then
+if cleanup_postcommit_directory "stage" "$STAGE_DIR"; then
  stage_cleanup_verified=1
 fi
-if rm -rf -- "$ROLLBACK_DIR" && [ ! -e "$ROLLBACK_DIR" ] && [ ! -L "$ROLLBACK_DIR" ]; then
+if cleanup_postcommit_directory "rollback_prestate" "$ROLLBACK_DIR"; then
  rollback_prestate_cleanup_verified=1
 fi
-STAGE_DIR=""
-ROLLBACK_DIR=""
-python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$receipt_path" "$verifier_result" \
-  "$stage_cleanup_verified" "$rollback_prestate_cleanup_verified" <<'PY'
+terminal_status="verified"
+terminal_exit_code=0
+# Cleanup has completed. Freeze terminal signal state before the immutable
+# terminal writer so its receipt, stdout, and exit status cannot diverge.
+trap '' HUP INT TERM
+if [ "$stage_cleanup_verified" -ne 1 ] || [ "$rollback_prestate_cleanup_verified" -ne 1 ]; then
+  terminal_status="cleanup_pending"
+  terminal_exit_code=75
+elif [ -n "$DEFERRED_SIGNAL" ]; then
+  terminal_status="committed_signal_deferred"
+  terminal_exit_code="$DEFERRED_SIGNAL_EXIT"
+fi
+if ! write_terminal_receipt "$terminal_status" "$stage_cleanup_verified" \
+  "$rollback_prestate_cleanup_verified" "$DEFERRED_SIGNAL" "$terminal_exit_code"; then
+  trap - EXIT
+  python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$RECEIPT_PATH" \
+    "$TERMINAL_RECEIPT_PATH" "$stage_cleanup_verified" \
+    "$rollback_prestate_cleanup_verified" "$DEFERRED_SIGNAL" <<'PY'
 import json
 import sys

 print(json.dumps({
    "schemaVersion": "agent99_host110_backup_runtime_executor_v1",
    "mode": "apply",
-    "ok": True,
+    "status": "terminal_receipt_failed",
+    "ok": False,
    "sourceRevision": sys.argv[1],
    "sourceHead": sys.argv[2],
+    "runId": sys.argv[3],
    "fileCount": 18,
    "backupScriptFileCount": 17,
    "backupHealthExporterIncluded": True,
+    "payloadCommitted": True,
    "rollbackAttempted": False,
    "rollbackPerformed": False,
    "rollbackVerified": False,
    "zeroResidueVerified": True,
-    "stageCleanupVerified": sys.argv[5] == "1",
-    "rollbackPrestateCleanupVerified": sys.argv[6] == "1",
-    "receipt": sys.argv[3],
-    "verifier": json.loads(sys.argv[4]),
+    "zeroResidueScope": "run_owned_destination_temps",
+    "stageCleanupVerified": sys.argv[6] == "1",
+    "rollbackPrestateCleanupVerified": sys.argv[7] == "1",
+    "receipt": sys.argv[4],
+    "terminalReceipt": sys.argv[5],
+    "terminalReceiptPublished": False,
+    "terminalExitCode": 76,
+    "deferredSignal": sys.argv[8] or None,
+    "independentVerifierVerified": True,
+    "verifier": None,
 }, ensure_ascii=True, sort_keys=True))
 PY
+  printf 'HOST110_BACKUP_RUNTIME_OK=0\nERROR=terminal_receipt_failed\n' >&2
+  exit 76
+fi
+if [ "$stage_cleanup_verified" -eq 1 ]; then STAGE_DIR=""; fi
+if [ "$rollback_prestate_cleanup_verified" -eq 1 ]; then ROLLBACK_DIR=""; fi
+trap - EXIT
+python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$RECEIPT_PATH" "$TERMINAL_RECEIPT_PATH" \
+  "$verifier_result" "$stage_cleanup_verified" "$rollback_prestate_cleanup_verified" \
+  "$terminal_status" "$terminal_exit_code" "$DEFERRED_SIGNAL" "$RUN_ID" <<'PY'
+import json
+import sys
+
+status = sys.argv[8]
+verifier = json.loads(sys.argv[5])
+print(json.dumps({
+    "schemaVersion": "agent99_host110_backup_runtime_executor_v1",
+    "mode": "apply",
+    "status": status,
+    "ok": status == "verified",
+    "sourceRevision": sys.argv[1],
+    "sourceHead": sys.argv[2],
+    "runId": sys.argv[11],
+    "fileCount": 18,
+    "backupScriptFileCount": 17,
+    "backupHealthExporterIncluded": True,
+    "payloadCommitted": True,
+    "rollbackAttempted": False,
+    "rollbackPerformed": False,
+    "rollbackVerified": False,
+    "zeroResidueVerified": True,
+    "zeroResidueScope": "run_owned_destination_temps",
+    "stageCleanupVerified": sys.argv[6] == "1",
+    "rollbackPrestateCleanupVerified": sys.argv[7] == "1",
+    "receipt": sys.argv[3],
+    "terminalReceipt": sys.argv[4],
+    "terminalReceiptPublished": True,
+    "terminalExitCode": int(sys.argv[9]),
+    "deferredSignal": sys.argv[10] or None,
+    "independentVerifierVerified": True,
+    "verifier": verifier if status == "verified" else None,
+}, ensure_ascii=True, sort_keys=True))
+PY
+exit "$terminal_exit_code"
--- a/scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1
+++ b/scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1
@@ -0,0 +1,220 @@
+$ErrorActionPreference = "Stop"
+
+function Write-ReplayTrace {
+  param([string]$Step)
+  if ($ReplayTrace) { Write-Output "REPLAY_STEP=$Step" }
+}
+
+if (-not $BrokerGzipBase64) { throw "broker_payload_missing" }
+Write-ReplayTrace "decode_source"
+$compressed = [Convert]::FromBase64String([string]$BrokerGzipBase64)
+$inputStream = New-Object IO.MemoryStream(,$compressed)
+$gzip = New-Object IO.Compression.GzipStream(
+  $inputStream,
+  [IO.Compression.CompressionMode]::Decompress
+)
+$reader = New-Object IO.StreamReader($gzip, [Text.Encoding]::UTF8)
+try {
+  $brokerSource = $reader.ReadToEnd()
+} finally {
+  $reader.Dispose()
+  $gzip.Dispose()
+  $inputStream.Dispose()
+}
+
+$tokens = $null
+$parseErrors = $null
+$ast = [Management.Automation.Language.Parser]::ParseInput(
+  $brokerSource,
+  [ref]$tokens,
+  [ref]$parseErrors
+)
+if ($parseErrors.Count -ne 0) { throw "broker_parse_failed" }
+Write-ReplayTrace "define_functions"
+
+$functionNames = @(
+  "Test-Agent99JsonField",
+  "Assert-Agent99JsonFields",
+  "Convert-Agent99ExecutorDocument",
+  "Assert-Agent99ApplyResultFields",
+  "Assert-Agent99VerifierFields",
+  "Get-Agent99ExecutorResult",
+  "Get-Agent99CommittedFailureResult"
+)
+foreach ($functionName in $functionNames) {
+  $node = $ast.Find(
+    {
+      param($candidate)
+      $candidate -is [Management.Automation.Language.FunctionDefinitionAst] -and
+        $candidate.Name -eq $functionName
+    },
+    $true
+  )
+  if ($null -eq $node) { throw "broker_function_missing_$functionName" }
+  Invoke-Expression $node.Extent.Text
+}
+
+$SourceRevision = "a" * 40
+$RunId = "windows99-contract-replay"
+$ExpectedFileCount = 18
+$sourceHead = "b" * 40
+$payloadReceipt = "/backup/status/agent99-backup-runtime-deploy-$RunId.json"
+$terminalReceipt = "/backup/status/agent99-backup-runtime-terminal-$RunId.json"
+
+function Copy-ReplayDocument {
+  param([object]$Document)
+  return ($Document | ConvertTo-Json -Compress -Depth 12 | ConvertFrom-Json)
+}
+
+function New-ReplayTransport {
+  param([object]$Document, [int]$ExitCode)
+  return [pscustomobject]@{
+    ok = [bool]($ExitCode -eq 0)
+    exitCode = $ExitCode
+    reason = if ($ExitCode -eq 0) { "completed" } else { "process_failed" }
+    stdout = $Document | ConvertTo-Json -Compress -Depth 12
+    stderrPresent = $false
+  }
+}
+
+function Assert-Rejected {
+  param([scriptblock]$Operation, [string]$Name)
+  $accepted = $false
+  try {
+    & $Operation | Out-Null
+    $accepted = $true
+  } catch {}
+  if ($accepted) { throw "contract_mutation_accepted_$Name" }
+}
+
+$verifier = [pscustomobject]@{
+  schemaVersion = "agent99_host110_backup_runtime_verifier_v1"
+  ok = $true
+  sourceRevision = $SourceRevision
+  runId = $RunId
+  fileCount = 18
+  backupScriptFileCount = 17
+  backupHealthExporterIncluded = $true
+  remoteWritePerformed = $false
+  selfIdentityVerified = $true
+}
+$success = [pscustomobject]@{
+  schemaVersion = "agent99_host110_backup_runtime_executor_v1"
+  mode = "apply"
+  status = "verified"
+  ok = $true
+  sourceRevision = $SourceRevision
+  sourceHead = $sourceHead
+  runId = $RunId
+  fileCount = 18
+  backupScriptFileCount = 17
+  backupHealthExporterIncluded = $true
+  payloadCommitted = $true
+  rollbackAttempted = $false
+  rollbackPerformed = $false
+  rollbackVerified = $false
+  zeroResidueVerified = $true
+  zeroResidueScope = "run_owned_destination_temps"
+  stageCleanupVerified = $true
+  rollbackPrestateCleanupVerified = $true
+  receipt = $payloadReceipt
+  terminalReceipt = $terminalReceipt
+  terminalReceiptPublished = $true
+  terminalExitCode = 0
+  deferredSignal = $null
+  independentVerifierVerified = $true
+  verifier = $verifier
+}
+
+Write-ReplayTrace "accept_success"
+Get-Agent99ExecutorResult (New-ReplayTransport $success 0) "Apply" | Out-Null
+$topLevelRequired = @(
+  "schemaVersion", "mode", "status", "ok", "sourceRevision", "sourceHead",
+  "runId", "fileCount", "backupScriptFileCount", "backupHealthExporterIncluded",
+  "payloadCommitted", "rollbackAttempted", "rollbackPerformed", "rollbackVerified",
+  "zeroResidueVerified", "zeroResidueScope", "stageCleanupVerified",
+  "rollbackPrestateCleanupVerified", "receipt", "terminalReceipt", "terminalExitCode",
+  "terminalReceiptPublished", "deferredSignal", "independentVerifierVerified", "verifier"
+)
+$verifierRequired = @(
+  "schemaVersion", "ok", "sourceRevision", "runId", "fileCount",
+  "backupScriptFileCount", "backupHealthExporterIncluded", "remoteWritePerformed",
+  "selfIdentityVerified"
+)
+$missingFieldRejections = 0
+Write-ReplayTrace "reject_missing_top"
+foreach ($field in $topLevelRequired) {
+  $mutated = Copy-ReplayDocument $success
+  $mutated.PSObject.Properties.Remove($field)
+  Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_$field"
+  $missingFieldRejections++
+}
+Write-ReplayTrace "reject_missing_verifier"
+foreach ($field in $verifierRequired) {
+  $mutated = Copy-ReplayDocument $success
+  $mutated.verifier.PSObject.Properties.Remove($field)
+  Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_verifier_$field"
+  $missingFieldRejections++
+}
+
+$typeMutations = @(
+  @{ field = "rollbackAttempted"; value = "false" },
+  @{ field = "rollbackPerformed"; value = "false" },
+  @{ field = "rollbackVerified"; value = "false" },
+  @{ field = "terminalExitCode"; value = "0" },
+  @{ field = "deferredSignal"; value = 0 }
+)
+$typeRejections = 0
+Write-ReplayTrace "reject_types"
+foreach ($mutation in $typeMutations) {
+  $mutated = Copy-ReplayDocument $success
+  $mutated.($mutation.field) = $mutation.value
+  Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_$($mutation.field)"
+  $typeRejections++
+}
+$mutated = Copy-ReplayDocument $success
+$mutated.verifier.remoteWritePerformed = "false"
+Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_verifier_remoteWritePerformed"
+$typeRejections++
+
+$cleanupPending = Copy-ReplayDocument $success
+$cleanupPending.ok = $false
+$cleanupPending.status = "cleanup_pending"
+$cleanupPending.stageCleanupVerified = $false
+$cleanupPending.terminalExitCode = 75
+$cleanupPending.verifier = $null
+Write-ReplayTrace "accept_cleanup_pending"
+Get-Agent99CommittedFailureResult (New-ReplayTransport $cleanupPending 75) | Out-Null
+
+$signalDeferred = Copy-ReplayDocument $success
+$signalDeferred.ok = $false
+$signalDeferred.status = "committed_signal_deferred"
+$signalDeferred.terminalExitCode = 143
+$signalDeferred.deferredSignal = "TERM"
+$signalDeferred.verifier = $null
+Write-ReplayTrace "accept_signal_deferred"
+Get-Agent99CommittedFailureResult (New-ReplayTransport $signalDeferred 143) | Out-Null
+
+$terminalReceiptFailed = Copy-ReplayDocument $success
+$terminalReceiptFailed.ok = $false
+$terminalReceiptFailed.status = "terminal_receipt_failed"
+$terminalReceiptFailed.terminalReceiptPublished = $false
+$terminalReceiptFailed.terminalExitCode = 76
+$terminalReceiptFailed.verifier = $null
+Write-ReplayTrace "accept_terminal_receipt_failed"
+Get-Agent99CommittedFailureResult (New-ReplayTransport $terminalReceiptFailed 76) | Out-Null
+
+Write-ReplayTrace "complete"
+[pscustomobject]@{
+  schemaVersion = "agent99_host110_broker_contract_replay_v1"
+  ok = $true
+  parser = "WindowsPowerShell"
+  successAccepted = $true
+  cleanupPendingPreserved = $true
+  committedSignalPreserved = $true
+  terminalReceiptFailurePreserved = $true
+  missingFieldRejections = $missingFieldRejections
+  typeRejections = $typeRejections
+  remoteWritePerformed = $false
+  secretValuesRead = $false
+} | ConvertTo-Json -Compress
--- a/scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py
+++ b/scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py
@@ -20,6 +20,9 @@ ROOT = Path(__file__).resolve().parents[3]
 BROKER = ROOT / "agent99-host110-backup-runtime-broker.ps1"
 EXECUTOR = ROOT / "scripts" / "backup" / "host110-backup-runtime-executor.sh"
 VERIFIER = ROOT / "scripts" / "backup" / "verify-host110-backup-runtime.py"
+BROKER_CONTRACT_REPLAY = (
+    ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-host110-broker-contract-replay.ps1"
+)


 EXPECTED_RUNTIME_FILES = {
@@ -71,6 +74,12 @@ def _powershell_array(source: str, variable: str) -> tuple[str, ...]:
    return tuple(re.findall(r'"([^"]+)"', match.group(1)))


+def _powershell_function(source: str, name: str) -> str:
+    start = source.index(f"function {name} {{")
+    next_function = source.find("\nfunction ", start + 1)
+    return source[start:] if next_function < 0 else source[start:next_function]
+
+
 def _executor_payload_order(source: str) -> tuple[str, ...]:
    match = re.search(r"readonly -a RUNTIME_FILES=\((.*?)\n\)", source, re.DOTALL)
    assert match is not None
@@ -240,6 +249,10 @@ def _run_executor_harness(harness: dict[str, object], **extra_env: str) -> subpr
        "HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT",
        "HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT",
        "HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT",
+        "HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL",
+        "HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL",
+        "HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT",
+        "HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT",
    ):
        environment.pop(name, None)
    environment["HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS"] = "1"
@@ -281,6 +294,16 @@ def _assert_prestate_restored(harness: dict[str, object]) -> None:
        assert (target.read_bytes(), target.stat().st_mode & 0o777) == before_state[name]


+def _assert_payload_committed(harness: dict[str, object]) -> None:
+    destination = Path(harness["destination"])
+    exporter_root = Path(harness["exporter_root"])
+    source_stage = Path(harness["source_stage"])
+    for name in harness["payload_order"]:
+        target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name
+        assert target.read_bytes() == (source_stage / name).read_bytes()
+        assert target.stat().st_mode & 0o777 == 0o755
+
+
 def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
    source = BROKER.read_text(encoding="utf-8")

@@ -304,6 +327,100 @@ def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
    assert "taskkill.exe /PID $process.Id /T /F" in source
    assert "$process.Kill()" in source
    assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths' in source
+    assert '[string]$parsed.status -ne "verified"' in source
+    assert '-not [bool]$parsed.payloadCommitted' in source
+    assert '-not [bool]$parsed.stageCleanupVerified' in source
+    assert '-not [bool]$parsed.rollbackPrestateCleanupVerified' in source
+    assert '[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps"' in source
+    assert '[int64]$parsed.terminalExitCode -ne 0' in source
+    assert 'agent99-backup-runtime-terminal-$RunId.json' in source
+    assert '$property = $Object.PSObject.Properties[$Name]' in source
+    assert 'if ($null -eq $property) { return $false }' in source
+    assert 'Get-Agent99CommittedFailureResult $applyTransport' in source
+    assert 'throw "host110_apply_$([string]$apply.status)"' in source
+
+
+def test_broker_required_field_contract_rejects_missing_fields_before_cast() -> None:
+    source = BROKER.read_text(encoding="utf-8")
+    field_guard = _powershell_function(source, "Test-Agent99JsonField")
+    common_contract = _powershell_function(source, "Convert-Agent99ExecutorDocument")
+    apply_contract = _powershell_function(source, "Assert-Agent99ApplyResultFields")
+    verifier_contract = _powershell_function(source, "Assert-Agent99VerifierFields")
+    committed_failure = _powershell_function(source, "Get-Agent99CommittedFailureResult")
+
+    assert '$property = $Object.PSObject.Properties[$Name]' in field_guard
+    assert 'if ($null -eq $property) { return $false }' in field_guard
+    for field in (
+        "schemaVersion",
+        "mode",
+        "ok",
+        "sourceRevision",
+        "sourceHead",
+        "runId",
+        "fileCount",
+        "backupScriptFileCount",
+        "backupHealthExporterIncluded",
+    ):
+        assert re.search(rf"^\s+{field} = \"", common_contract, re.MULTILINE), field
+    for field in (
+        "status",
+        "payloadCommitted",
+        "rollbackAttempted",
+        "rollbackPerformed",
+        "rollbackVerified",
+        "zeroResidueVerified",
+        "zeroResidueScope",
+        "stageCleanupVerified",
+        "rollbackPrestateCleanupVerified",
+        "receipt",
+        "terminalReceipt",
+        "terminalReceiptPublished",
+        "terminalExitCode",
+        "deferredSignal",
+        "independentVerifierVerified",
+        "verifier",
+    ):
+        assert re.search(rf"^\s+{field} = ", apply_contract, re.MULTILINE), field
+    for field in (
+        "schemaVersion",
+        "ok",
+        "sourceRevision",
+        "runId",
+        "fileCount",
+        "backupScriptFileCount",
+        "backupHealthExporterIncluded",
+        "remoteWritePerformed",
+        "selfIdentityVerified",
+    ):
+        assert re.search(rf"^\s+{field} = \"", verifier_contract, re.MULTILINE), field
+    assert 'status -eq "cleanup_pending"' in committed_failure
+    assert 'status -eq "committed_signal_deferred"' in committed_failure
+    assert 'verifier = $VerifierKind' in apply_contract
+
+
+def test_windows99_contract_replay_uses_actual_broker_functions_without_remote_write() -> None:
+    source = BROKER_CONTRACT_REPLAY.read_text(encoding="utf-8")
+
+    assert "FunctionDefinitionAst" in source
+    for function_name in (
+        "Test-Agent99JsonField",
+        "Assert-Agent99JsonFields",
+        "Convert-Agent99ExecutorDocument",
+        "Assert-Agent99ApplyResultFields",
+        "Assert-Agent99VerifierFields",
+        "Get-Agent99ExecutorResult",
+        "Get-Agent99CommittedFailureResult",
+    ):
+        assert f'"{function_name}"' in source
+    assert "$mutated.PSObject.Properties.Remove($field)" in source
+    assert "$mutated.verifier.PSObject.Properties.Remove($field)" in source
+    assert 'status = "cleanup_pending"' in source
+    assert 'status = "committed_signal_deferred"' in source
+    assert 'status = "terminal_receipt_failed"' in source
+    assert "remoteWritePerformed = $false" in source
+    assert "secretValuesRead = $false" in source
+    for forbidden in ("New-Item", "Set-Content", "WriteAllText", "ssh.exe", "scp.exe"):
+        assert forbidden not in source


 def test_broker_scp_basenames_match_executor_source_stage_contract() -> None:
@@ -385,11 +502,16 @@ def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
    assert '"rollbackPerformed": rollback_performed' in source
    assert '"rollbackVerified": rollback_verified' in source
    assert '"zeroResidueVerified": zero_residue_verified' in source
+    assert '"zeroResidueScope": "run_owned_destination_temps"' in source
    assert 'cleanup_run_owned_temps' in source
    assert 'verify_run_owned_temp_absence' in source
    assert 'HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT' in source
    assert 'HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT' in source
    assert 'HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT' in source
+    assert 'HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL' in source
+    assert 'HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL' in source
+    assert 'HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT' in source
+    assert 'HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT' in source
    assert 'HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS' in source
    assert '[ "$DEST_ROOT" != "/backup/scripts" ]' in source
    assert 'fsync_rollback_prestate || return 75' in source
@@ -401,8 +523,18 @@ def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
    assert 'os.fsync(handle.fileno())' in source
    assert 'os.fsync(directory_fd)' in source
    assert 'hashlib.sha256(readback).hexdigest()' in source
-    assert source.index('write_receipt "verified"') < source.index("\nAPPLY_COMPLETE=1")
-    assert source.index("\nAPPLY_COMPLETE=1") < source.rindex('rm -rf -- "$ROLLBACK_DIR"')
+    assert source.index("COMMIT_CRITICAL=1") < source.index('write_receipt "payload_verified"')
+    assert source.index('write_receipt "payload_verified"') < source.index("\nAPPLY_COMPLETE=1")
+    assert source.index("\nAPPLY_COMPLETE=1") < source.rindex("\nCOMMIT_CRITICAL=0")
+    assert source.index("\nAPPLY_COMPLETE=1") < source.rindex(
+        'cleanup_postcommit_directory "rollback_prestate"'
+    )
+    assert 'handle_agent99_signal HUP 129' in source
+    assert 'handle_agent99_signal TERM 143' in source
+    assert source.index("trap '' HUP INT TERM") < source.index('write_terminal_receipt "$terminal_status"')
+    assert 'write_terminal_receipt "$terminal_status"' in source
+    assert 'terminal_status="cleanup_pending"' in source
+    assert 'terminal_status="committed_signal_deferred"' in source
    assert 'run_identity_receipt_exists' in source
    assert 'run_identity_stage_exists' in source
    assert 'run_identity_rollback_exists' in source
@@ -432,22 +564,205 @@ def test_apply_success_publishes_durable_receipt_before_prestate_cleanup(tmp_pat
    assert output["zeroResidueVerified"] is True
    assert output["stageCleanupVerified"] is True
    assert output["rollbackPrestateCleanupVerified"] is True
+    assert output["status"] == "verified"
+    assert output["payloadCommitted"] is True
+    assert output["runId"] == run_id
+    assert output["terminalExitCode"] == 0
+    assert output["terminalReceiptPublished"] is True
+    assert output["deferredSignal"] is None
    receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
    receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
-    assert receipt["status"] == "verified"
+    assert receipt["status"] == "payload_verified"
    assert receipt["rollbackAttempted"] is False
    assert receipt["rollbackPerformed"] is False
    assert receipt["rollbackVerified"] is False
    assert receipt["zeroResidueVerified"] is True
+    assert receipt["zeroResidueScope"] == "run_owned_destination_temps"
+    terminal_path = Path(harness["status_root"]) / f"agent99-backup-runtime-terminal-{run_id}.json"
+    terminal = json.loads(terminal_path.read_text(encoding="utf-8"))
+    assert terminal["status"] == "verified"
+    assert terminal["ok"] is True
+    assert terminal["payloadCommitted"] is True
+    assert terminal["stageCleanupVerified"] is True
+    assert terminal["rollbackPrestateCleanupVerified"] is True
+    assert terminal["terminalExitCode"] == 0
+    assert terminal["zeroResidueScope"] == "run_owned_destination_temps_and_internal_stage_prestate"
+    assert output["terminalReceipt"] == str(terminal_path)
    assert not (Path(harness["rollback_root"]) / run_id).exists()
    assert not (Path(harness["stage_root"]) / run_id).exists()
-    for name in harness["payload_order"]:
-        target = (
-            Path(harness["exporter_root"]) / name
-            if name == "backup-health-textfile-exporter.py"
-            else Path(harness["destination"]) / name
-        )
-        assert target.read_bytes() == (Path(harness["source_stage"]) / name).read_bytes()
+    _assert_payload_committed(harness)
+
+
+@pytest.mark.parametrize(("signal_name", "exit_code"), [("TERM", 143), ("HUP", 129), ("INT", 130)])
+def test_commit_window_signal_is_deferred_until_payload_and_terminal_are_consistent(
+    tmp_path: Path,
+    signal_name: str,
+    exit_code: int,
+) -> None:
+    run_id = f"commit-signal-{signal_name.lower()}"
+    harness = _build_executor_harness(tmp_path, run_id)
+
+    result = _run_executor_harness(
+        harness,
+        HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL=signal_name,
+    )
+
+    assert result.returncode == exit_code, result.stderr
+    output = json.loads(result.stdout)
+    assert output["ok"] is False
+    assert output["status"] == "committed_signal_deferred"
+    assert output["payloadCommitted"] is True
+    assert output["runId"] == run_id
+    assert output["deferredSignal"] == signal_name
+    assert output["terminalExitCode"] == exit_code
+    assert output["terminalReceiptPublished"] is True
+    assert output["stageCleanupVerified"] is True
+    assert output["rollbackPrestateCleanupVerified"] is True
+    _assert_payload_committed(harness)
+    payload_receipt = json.loads(
+        (
+            Path(harness["status_root"])
+            / f"agent99-backup-runtime-deploy-{run_id}.json"
+        ).read_text(encoding="utf-8")
+    )
+    assert payload_receipt["status"] == "payload_verified"
+    terminal = json.loads(
+        (
+            Path(harness["status_root"])
+            / f"agent99-backup-runtime-terminal-{run_id}.json"
+        ).read_text(encoding="utf-8")
+    )
+    assert terminal["status"] == "committed_signal_deferred"
+    assert terminal["ok"] is False
+    assert terminal["deferredSignal"] == signal_name
+    assert terminal["terminalExitCode"] == exit_code
+    assert not (Path(harness["rollback_root"]) / run_id).exists()
+    assert not (Path(harness["stage_root"]) / run_id).exists()
+
+
+@pytest.mark.parametrize("signal_name", ["TERM", "HUP", "INT"])
+def test_terminal_publication_masks_late_signal_without_receipt_stdout_exit_divergence(
+    tmp_path: Path,
+    signal_name: str,
+) -> None:
+    run_id = f"terminal-signal-{signal_name.lower()}"
+    harness = _build_executor_harness(tmp_path, run_id)
+
+    result = _run_executor_harness(
+        harness,
+        HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL=signal_name,
+    )
+
+    assert result.returncode == 0, result.stderr
+    output = json.loads(result.stdout)
+    assert output["ok"] is True
+    assert output["status"] == "verified"
+    assert output["payloadCommitted"] is True
+    assert output["runId"] == run_id
+    assert output["deferredSignal"] is None
+    assert output["terminalExitCode"] == 0
+    assert output["terminalReceiptPublished"] is True
+    terminal = json.loads(
+        (
+            Path(harness["status_root"])
+            / f"agent99-backup-runtime-terminal-{run_id}.json"
+        ).read_text(encoding="utf-8")
+    )
+    assert terminal["ok"] is True
+    assert terminal["status"] == output["status"]
+    assert terminal["deferredSignal"] == output["deferredSignal"]
+    assert terminal["terminalExitCode"] == output["terminalExitCode"]
+    _assert_payload_committed(harness)
+
+
+@pytest.mark.parametrize("fault", ["write", "link", "fsync", "readback"])
+def test_terminal_receipt_fault_is_nonzero_with_payload_receipt_only(
+    tmp_path: Path,
+    fault: str,
+) -> None:
+    run_id = f"terminal-receipt-{fault}"
+    harness = _build_executor_harness(tmp_path, run_id)
+
+    result = _run_executor_harness(
+        harness,
+        HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT=fault,
+    )
+
+    assert result.returncode == 76
+    assert '"ok": true' not in result.stdout.lower()
+    assert "terminal_receipt_failed" in result.stderr
+    output = json.loads(result.stdout)
+    assert output["ok"] is False
+    assert output["status"] == "terminal_receipt_failed"
+    assert output["payloadCommitted"] is True
+    assert output["runId"] == run_id
+    assert output["terminalReceiptPublished"] is False
+    assert output["terminalExitCode"] == 76
+    _assert_payload_committed(harness)
+    payload_receipt = json.loads(
+        (
+            Path(harness["status_root"])
+            / f"agent99-backup-runtime-deploy-{run_id}.json"
+        ).read_text(encoding="utf-8")
+    )
+    assert payload_receipt["status"] == "payload_verified"
+    assert not (
+        Path(harness["status_root"])
+        / f"agent99-backup-runtime-terminal-{run_id}.json"
+    ).exists()
+    assert not (Path(harness["rollback_root"]) / run_id).exists()
+    assert not (Path(harness["stage_root"]) / run_id).exists()
+
+
+@pytest.mark.parametrize(
+    ("fault", "stage_clean", "rollback_clean"),
+    [("stage", False, True), ("rollback_prestate", True, False)],
+)
+def test_postcommit_cleanup_failure_is_nonzero_and_never_false_green(
+    tmp_path: Path,
+    fault: str,
+    stage_clean: bool,
+    rollback_clean: bool,
+) -> None:
+    run_id = f"cleanup-{fault}"
+    harness = _build_executor_harness(tmp_path, run_id)
+
+    result = _run_executor_harness(
+        harness,
+        HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT=fault,
+    )
+
+    assert result.returncode == 75, result.stderr
+    assert '"ok": true' not in result.stdout.lower()
+    output = json.loads(result.stdout)
+    assert output["ok"] is False
+    assert output["status"] == "cleanup_pending"
+    assert output["payloadCommitted"] is True
+    assert output["runId"] == run_id
+    assert output["stageCleanupVerified"] is stage_clean
+    assert output["rollbackPrestateCleanupVerified"] is rollback_clean
+    assert output["terminalReceiptPublished"] is True
+    _assert_payload_committed(harness)
+    payload_receipt = json.loads(
+        (
+            Path(harness["status_root"])
+            / f"agent99-backup-runtime-deploy-{run_id}.json"
+        ).read_text(encoding="utf-8")
+    )
+    assert payload_receipt["status"] == "payload_verified"
+    terminal = json.loads(
+        (
+            Path(harness["status_root"])
+            / f"agent99-backup-runtime-terminal-{run_id}.json"
+        ).read_text(encoding="utf-8")
+    )
+    assert terminal["status"] == "cleanup_pending"
+    assert terminal["ok"] is False
+    assert terminal["payloadCommitted"] is True
+    assert terminal["stageCleanupVerified"] is stage_clean
+    assert terminal["rollbackPrestateCleanupVerified"] is rollback_clean
+    assert (Path(harness["stage_root"]) / run_id).exists() is (not stage_clean)
+    assert (Path(harness["rollback_root"]) / run_id).exists() is (not rollback_clean)


@pytest.mark.parametrize("fault", ["write", "link", "fsync", "readback"])