From d1797a1c8766722ae00ac14e20fed00f4e3f3a5b Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 01:08:45 +0800 Subject: [PATCH] fix(agent99): close host110 backup commit races --- agent99-host110-backup-runtime-broker.ps1 | 219 +++++++++++- .../AWOOOI-AG99-H110-BACKUP-RUNTIME-V10.md | 217 ++++++++++++ .../backup/host110-backup-runtime-executor.sh | 333 +++++++++++++++-- ...agent99-host110-broker-contract-replay.ps1 | 220 ++++++++++++ ...t_agent99_host110_backup_runtime_broker.py | 335 +++++++++++++++++- 5 files changed, 1282 insertions(+), 42 deletions(-) create mode 100644 docs/proposals/AWOOOI-AG99-H110-BACKUP-RUNTIME-V10.md create mode 100644 scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1 diff --git a/agent99-host110-backup-runtime-broker.ps1 b/agent99-host110-backup-runtime-broker.ps1 index ad842e7c3..83147c373 100644 --- a/agent99-host110-backup-runtime-broker.ps1 +++ b/agent99-host110-backup-runtime-broker.ps1 @@ -268,30 +268,157 @@ function New-Agent99SourcePackage { } } -function Get-Agent99ExecutorResult { - param([object]$Transport, [string]$ExpectedMode) +function Test-Agent99JsonField { + param( + [object]$Object, + [string]$Name, + [ValidateSet("String", "Boolean", "Integer", "Object", "Null", "NullOrString")] + [string]$Kind + ) - if (-not $Transport.ok) { throw "host110_${ExpectedMode}_transport_failed" } + if ($null -eq $Object) { return $false } + $property = $Object.PSObject.Properties[$Name] + if ($null -eq $property) { return $false } + $value = $property.Value + switch ($Kind) { + "String" { return [bool]($value -is [string]) } + "Boolean" { return [bool]($value -is [bool]) } + "Integer" { return [bool]($value -is [int] -or $value -is [long]) } + "Object" { return [bool]($value -is [System.Management.Automation.PSCustomObject]) } + "Null" { return [bool]($null -eq $value) } + "NullOrString" { return [bool]($null -eq $value -or $value -is [string]) } + } + return $false +} + +function Assert-Agent99JsonFields { + param([object]$Object, [hashtable]$Contract, [string]$ErrorCode) + + foreach ($name in $Contract.Keys) { + if (-not (Test-Agent99JsonField $Object $name ([string]$Contract[$name]))) { + throw $ErrorCode + } + } +} + +function Convert-Agent99ExecutorDocument { + param( + [object]$Transport, + [string]$ExpectedMode, + [switch]$AllowProcessFailure + ) + + if (-not $Transport.ok) { + if ( + -not $AllowProcessFailure -or + [string]$Transport.reason -ne "process_failed" -or + [int]$Transport.exitCode -le 0 -or + [string]::IsNullOrWhiteSpace([string]$Transport.stdout) + ) { + throw "host110_${ExpectedMode}_transport_failed" + } + } try { $parsed = [string]$Transport.stdout | ConvertFrom-Json } catch { throw "host110_${ExpectedMode}_receipt_invalid" } + Assert-Agent99JsonFields $parsed @{ + schemaVersion = "String" + mode = "String" + ok = "Boolean" + sourceRevision = "String" + sourceHead = "String" + runId = "String" + fileCount = "Integer" + backupScriptFileCount = "Integer" + backupHealthExporterIncluded = "Boolean" + } "host110_${ExpectedMode}_receipt_required_field_failed" if ( [string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_executor_v1" -or [string]$parsed.mode -ne $ExpectedMode.ToLowerInvariant() -or - -not [bool]$parsed.ok -or [string]$parsed.sourceRevision -ne $SourceRevision -or - [int]$parsed.fileCount -ne $ExpectedFileCount -or - [int]$parsed.backupScriptFileCount -ne 17 -or + [string]$parsed.sourceHead -notmatch "^[0-9a-f]{40}$" -or + [string]$parsed.runId -ne $RunId -or + [int64]$parsed.fileCount -ne $ExpectedFileCount -or + [int64]$parsed.backupScriptFileCount -ne 17 -or -not [bool]$parsed.backupHealthExporterIncluded ) { throw "host110_${ExpectedMode}_receipt_contract_failed" } + return $parsed +} + +function Assert-Agent99ApplyResultFields { + param([object]$Parsed, [string]$VerifierKind) + + Assert-Agent99JsonFields $Parsed @{ + status = "String" + payloadCommitted = "Boolean" + rollbackAttempted = "Boolean" + rollbackPerformed = "Boolean" + rollbackVerified = "Boolean" + zeroResidueVerified = "Boolean" + zeroResidueScope = "String" + stageCleanupVerified = "Boolean" + rollbackPrestateCleanupVerified = "Boolean" + receipt = "String" + terminalReceipt = "String" + terminalReceiptPublished = "Boolean" + terminalExitCode = "Integer" + deferredSignal = "NullOrString" + independentVerifierVerified = "Boolean" + verifier = $VerifierKind + } "host110_apply_receipt_required_field_failed" +} + +function Assert-Agent99VerifierFields { + param([object]$Verifier) + + Assert-Agent99JsonFields $Verifier @{ + schemaVersion = "String" + ok = "Boolean" + sourceRevision = "String" + runId = "String" + fileCount = "Integer" + backupScriptFileCount = "Integer" + backupHealthExporterIncluded = "Boolean" + remoteWritePerformed = "Boolean" + selfIdentityVerified = "Boolean" + } "host110_apply_verifier_required_field_failed" +} + +function Get-Agent99ExecutorResult { + param([object]$Transport, [string]$ExpectedMode) + + $parsed = Convert-Agent99ExecutorDocument $Transport $ExpectedMode + if (-not [bool]$parsed.ok) { throw "host110_${ExpectedMode}_receipt_contract_failed" } if ($ExpectedMode -eq "Apply") { + Assert-Agent99ApplyResultFields $parsed "Object" + Assert-Agent99VerifierFields $parsed.verifier if ( + [string]$parsed.status -ne "verified" -or + -not [bool]$parsed.payloadCommitted -or + [bool]$parsed.rollbackAttempted -or + [bool]$parsed.rollbackPerformed -or + [bool]$parsed.rollbackVerified -or + -not [bool]$parsed.zeroResidueVerified -or + [string]$parsed.zeroResidueScope -ne "run_owned_destination_temps" -or + -not [bool]$parsed.stageCleanupVerified -or + -not [bool]$parsed.rollbackPrestateCleanupVerified -or + [int64]$parsed.terminalExitCode -ne 0 -or + $null -ne $parsed.deferredSignal -or + -not [bool]$parsed.independentVerifierVerified -or + [string]$parsed.receipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or + [string]$parsed.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json" -or + -not [bool]$parsed.terminalReceiptPublished -or [string]$parsed.verifier.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or -not [bool]$parsed.verifier.ok -or + [string]$parsed.verifier.sourceRevision -ne $SourceRevision -or + [string]$parsed.verifier.runId -ne $RunId -or + [int64]$parsed.verifier.fileCount -ne $ExpectedFileCount -or + [int64]$parsed.verifier.backupScriptFileCount -ne 17 -or + -not [bool]$parsed.verifier.backupHealthExporterIncluded -or [bool]$parsed.verifier.remoteWritePerformed -or -not [bool]$parsed.verifier.selfIdentityVerified ) { @@ -301,6 +428,66 @@ function Get-Agent99ExecutorResult { return $parsed } +function Get-Agent99CommittedFailureResult { + param([object]$Transport) + + $parsed = Convert-Agent99ExecutorDocument $Transport "Apply" -AllowProcessFailure + Assert-Agent99ApplyResultFields $parsed "Null" + if ( + [bool]$parsed.ok -or + -not [bool]$parsed.payloadCommitted -or + [bool]$parsed.rollbackAttempted -or + [bool]$parsed.rollbackPerformed -or + [bool]$parsed.rollbackVerified -or + -not [bool]$parsed.zeroResidueVerified -or + [string]$parsed.zeroResidueScope -ne "run_owned_destination_temps" -or + -not [bool]$parsed.independentVerifierVerified -or + [string]$parsed.receipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or + [string]$parsed.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json" + ) { + throw "host110_apply_committed_failure_contract_failed" + } + $signalExitCodes = @{ INT = 130; HUP = 129; TERM = 143 } + if ([string]$parsed.status -eq "cleanup_pending") { + if ( + [int]$Transport.exitCode -ne 75 -or + [int64]$parsed.terminalExitCode -ne 75 -or + ([bool]$parsed.stageCleanupVerified -and [bool]$parsed.rollbackPrestateCleanupVerified) -or + ($null -ne $parsed.deferredSignal -and -not $signalExitCodes.ContainsKey([string]$parsed.deferredSignal)) + ) { + throw "host110_apply_cleanup_pending_contract_failed" + } + if (-not [bool]$parsed.terminalReceiptPublished) { + throw "host110_apply_cleanup_pending_terminal_receipt_missing" + } + } elseif ([string]$parsed.status -eq "committed_signal_deferred") { + $signalName = [string]$parsed.deferredSignal + if ( + -not $signalExitCodes.ContainsKey($signalName) -or + -not [bool]$parsed.stageCleanupVerified -or + -not [bool]$parsed.rollbackPrestateCleanupVerified -or + [int]$Transport.exitCode -ne [int]$signalExitCodes[$signalName] -or + [int64]$parsed.terminalExitCode -ne [int]$signalExitCodes[$signalName] + ) { + throw "host110_apply_deferred_signal_contract_failed" + } + if (-not [bool]$parsed.terminalReceiptPublished) { + throw "host110_apply_deferred_signal_terminal_receipt_missing" + } + } elseif ([string]$parsed.status -eq "terminal_receipt_failed") { + if ( + [bool]$parsed.terminalReceiptPublished -or + [int]$Transport.exitCode -ne 76 -or + [int64]$parsed.terminalExitCode -ne 76 + ) { + throw "host110_apply_terminal_receipt_failure_contract_failed" + } + } else { + throw "host110_apply_committed_failure_status_invalid" + } + return $parsed +} + function Get-Agent99VerifierResult { param([object]$Transport, [object]$SourcePackage) @@ -310,6 +497,17 @@ function Get-Agent99VerifierResult { } catch { throw "host110_verify_receipt_invalid" } + Assert-Agent99JsonFields $parsed @{ + schemaVersion = "String" + ok = "Boolean" + sourceRevision = "String" + runId = "String" + fileCount = "Integer" + backupScriptFileCount = "Integer" + backupHealthExporterIncluded = "Boolean" + remoteWritePerformed = "Boolean" + verifierSha256 = "String" + } "host110_verify_receipt_required_field_failed" if ( [string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or -not [bool]$parsed.ok -or @@ -421,7 +619,12 @@ try { $executorPath = "$remoteStage/host110-backup-runtime-executor.sh" $command = "timeout --signal=TERM --kill-after=10s 300s $executorPath --mode apply --source-revision $SourceRevision --run-id $RunId --source-stage $remoteStage" $applyTransport = Invoke-Agent99BoundedSsh $command 330 - $apply = Get-Agent99ExecutorResult $applyTransport "Apply" + if ($applyTransport.ok) { + $apply = Get-Agent99ExecutorResult $applyTransport "Apply" + } else { + $apply = Get-Agent99CommittedFailureResult $applyTransport + throw "host110_apply_$([string]$apply.status)" + } $verify = $apply.verifier $cleanupVerified = Remove-Agent99RemoteStage $remoteStage $true $remoteStageCreated = $false @@ -445,7 +648,7 @@ try { } $result = [pscustomobject]@{ - schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v2" + schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v3" ok = [bool]($terminalStatus -ne "failed") status = $terminalStatus mode = $Mode diff --git a/docs/proposals/AWOOOI-AG99-H110-BACKUP-RUNTIME-V10.md b/docs/proposals/AWOOOI-AG99-H110-BACKUP-RUNTIME-V10.md new file mode 100644 index 000000000..9a650f291 --- /dev/null +++ b/docs/proposals/AWOOOI-AG99-H110-BACKUP-RUNTIME-V10.md @@ -0,0 +1,217 @@ +# AWOOOI Agent99 Host110 Backup Runtime V10 + +Status: proposed; central review and explicit owner approval are required before +any production apply. + +This artifact supersedes V2 through V9. Every V9 identity below is void for +approval and remains audit evidence only: + +- branch/ref: `codex/host110-backup-runtime-v9-20260718` +- candidate/revision: `8f11ef3362a8d7f66a46308b58fd142b0c91a8bc` +- artifact SHA-256: `929978eb3da15b04791a8614e6d5ee504ae2a4c2f205b694307a7bad7713bcad` +- exact diff SHA-256: `8085d49450b74e7c43577ba118cbd74d3a2e3dd2d22eeba90ce749d56c72c0f2` +- every other V9 proposal, content, revision, and derived hash + +No V9 hash, ref, review, receipt, or test grants production execution +authority. + +## Exact Scope + +- Agent99 runtime bundle: 19 files, guarded by one producer/consumer parity + test. +- Host110 backup payload: 18 ordered files, comprising 17 backup scripts and + `backup-health-textfile-exporter.py`. +- Host110 remote stage: 21 unique basenames, comprising the ordered 18-file + payload, fixed executor, independent verifier, and `manifest.json`. +- Fixed target: `wooo@192.168.0.110`, dispatched by Windows99 Agent99 from an + exact Gitea revision and digest manifest. +- The candidate includes a 10-line runtime-lock addition to + `scripts/backup/gitea-full-backup-restore-drill.sh`. A future drill invocation + acquires the shared Host110 backup-runtime lock before its pre-existing drill + behavior. Applying this candidate replaces the script but does not invoke the + drill or any container lifecycle. +- `scripts/backup/backup-gitea.sh` has zero diff from the production base. No + Gitea primary stop/start or backup-container creation is part of this + candidate apply. +- V7, V8, and V9 proposal documents remain tracked audit artifacts and are not + runtime authority. +- `agent99-host110-broker-contract-replay.ps1` is a no-write Windows99 test + asset; it is not part of the 19-file Agent99 runtime or 18-file Host110 + payload. + +## V9 Defects Closed + +### Signal-Safe Commit Boundary + +The executor enters a signal-deferring critical section before publishing the +payload receipt. `INT`, `HUP`, and `TERM` are recorded without exiting until the +durable payload-receipt readback and `APPLY_COMPLETE=1` commit state are both +established. A deferred signal can never enter the pre-commit EXIT rollback +path after the payload receipt exists. + +After commit, exact stage and rollback-prestate cleanup still run. If cleanup +is complete, a deferred signal produces the non-success terminal +`committed_signal_deferred`, a durable terminal receipt, the corresponding +129/130/143 exit, and an applied payload. TERM, HUP, and INT are replayed in the +exact receipt-to-commit window. + +After cleanup, terminal publication uses a bounded signal mask. Signals captured +before that boundary remain part of the non-success terminal; signals delivered +during the immutable terminal writer cannot mutate its already-final state. +Terminal-writer TERM, HUP, and INT replays require receipt, stdout, payload, and +exit to remain consistent. + +### Payload And Terminal Receipts + +The immutable payload receipt has status `payload_verified`; it proves exact +payload, independent verifier, durable receipt readback, and absence of +run-owned destination temporaries. It does not claim internal stage or rollback +prestate cleanup. + +A separate immutable terminal receipt is the authority for overall success. +Only status `verified` with `stageCleanupVerified=true`, +`rollbackPrestateCleanupVerified=true`, no deferred signal, and exit code zero +may yield executor `ok:true`. Each successful cleanup is absence-read back and +its parent directory is fsynced before terminal publication. + +If either cleanup fails, the payload remains committed, exact residue is +preserved, terminal status is `cleanup_pending`, executor output is +`ok:false`, and exit is nonzero. A payload receipt cannot be used as a +zero-residue or broker-success shortcut. The Windows99 broker validates every +status, payload, rollback, residue-scope, cleanup, signal, exit, receipt-path, +and independent-verifier field fail-closed. +Committed non-success outputs are parsed and retained in broker evidence before +the broker fails overall; `cleanup_pending`, `committed_signal_deferred`, and +`terminal_receipt_failed` can never be collapsed into an ambiguous generic +transport failure. + +### Existing Transaction Guarantees Retained + +- Producer and executor payload basenames are exact-equal in count, set, and + order: 18 payload files and 21 unique staged files. +- Receipt write, hard-link, fsync, and readback failures roll back exact content + and metadata without publishing success. +- Terminal-receipt write, hard-link, fsync, and readback failures keep the + verified payload, publish no terminal success, emit structured committed + failure evidence with `terminalReceiptPublished=false`, and exit nonzero. +- Apply and rollback destination temporaries are run-owned, tracked, removed, + and absence-read back. +- Rollback attempted, performed, verified, and residue truth remain separate; + partial or unknown rollback terminates `rollback_unverified`. +- Fault hooks require both the explicit test flag and a transformed + non-production destination; canonical `/backup/scripts` cannot enable them. + +## Apply Effects + +- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded + manifest and receipt state. +- Host110 Apply replaces the ordered 18 payload paths through one exclusive, + digest-bound filesystem transaction, and writes payload/terminal receipts + below `/backup/status`. +- The deployed restore-drill script gains shared-lock behavior. The apply does + not execute that script. +- Apply does not run a backup, invoke a Gitea container lifecycle, sync Google + Drive, prune snapshots, restart a VM or service, or change a database, + network, firewall, credential, or secret. + +## Reproducible Validation + +Focused replay and contract suite: + +```sh +pytest -q \ + scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \ + scripts/backup/tests/test_verify_offsite_full_sync_runtime.py \ + scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py \ + scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py \ + scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py \ + scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py \ + scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py +``` + +Exact signal and post-commit cleanup replays: + +```sh +pytest -vv \ + scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \ + -k 'commit_window_signal or terminal_publication or terminal_receipt_fault or postcommit_cleanup' +``` + +Static, syntax, and repository checks: + +```sh +ruff check scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py +bash -n scripts/backup/host110-backup-runtime-executor.sh +git diff --check +PATH=/Users/ogt/.pyenv/shims:$PATH bash scripts/ops/ansible-validate.sh +``` + +`ansible-validate.sh` parses +`ops/reboot-recovery/full-stack-backup-baseline.yml` as YAML metadata. It runs +`ansible-playbook --syntax-check` only for the actual playbooks listed by the +script; the metadata mapping is not represented as a playbook. + +Windows99 no-file-write PowerShell parser check: + +```sh +PARSER_COMMAND='$source=[Console]::In.ReadToEnd(); $tokens=$null; $errors=$null; [System.Management.Automation.Language.Parser]::ParseInput($source,[ref]$tokens,[ref]$errors) | Out-Null; [pscustomobject]@{ok=($errors.Count -eq 0); errorCount=$errors.Count; parser="WindowsPowerShell"} | ConvertTo-Json -Compress; if ($errors.Count -ne 0) { exit 1 }' +ENCODED=$(printf '%s' "$PARSER_COMMAND" | iconv -f UTF-8 -t UTF-16LE | base64 | tr -d '\n') +ssh -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=5 \ + -o ConnectionAttempts=1 Administrator@192.168.0.99 \ + "powershell.exe -NoProfile -NonInteractive -EncodedCommand $ENCODED" \ + < agent99-host110-backup-runtime-broker.ps1 +``` + +Windows99 actual-function missing-field/type contract replay: + +```sh +WRAPPER='$script=[Console]::In.ReadToEnd(); & ([ScriptBlock]::Create($script))' +WRAPPER_ENCODED=$(printf '%s' "$WRAPPER" | iconv -f UTF-8 -t UTF-16LE | base64 | tr -d '\n') +BROKER_GZIP_BASE64=$(gzip -c agent99-host110-backup-runtime-broker.ps1 | base64 | tr -d '\n') +awk -v payload="$BROKER_GZIP_BASE64" \ + 'BEGIN { print "$BrokerGzipBase64 = \"" payload "\"" } { print }' \ + scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1 | + ssh -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=5 \ + -o ConnectionAttempts=1 Administrator@192.168.0.99 \ + "powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand $WRAPPER_ENCODED" +``` + +The replay loads the candidate broker's actual function AST, accepts the valid +success and three committed non-success terminals, then removes every required +field and injects wrong JSON types. Expected readback is 34 missing-field and 6 +type rejections. It performs no remote file write. + +## Exact Diff Serialization + +Substitute the V10 Gitea live-ref SHA for `` and hash the +exact stdout bytes: + +```sh +git diff --binary --full-index \ + e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3.. | + shasum -a 256 +``` + +Central review must independently bind the live feature ref and candidate Git +SHA, this tracked artifact path and bytes hash, the serialized diff hash/path +count/line counts, the 19/18/21 count contracts, and the exact restore-drill and +`backup-gitea.sh` diffs. + +## Rollback + +- Source rollback reverts the exact V10 diff to + `e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing. +- Agent99 rollback restores the prior runtime files and manifest and verifies + prior hashes independently. +- Before payload commit, Host110 rollback restores all prior payload content, + SHA-256, uid, gid, and mode under the exclusive lock and proves no run-owned + destination temporary remains. +- After payload commit, the executor never rolls the payload back merely due to + a signal or cleanup error. It preserves exact cleanup evidence and reports a + non-success terminal. Any later cleanup-only remediation is a separate + controlled action scoped to the run-specific stage/prestate paths and may not + rewrite the verified payload. +- Source rollback restores the restore-drill script's prior unlocked behavior. + It does not invoke the drill or any container lifecycle. +- Any canonical, metadata, receipt, signal, cleanup, or residue mismatch fails + closed and cannot produce broker `verified`. diff --git a/scripts/backup/host110-backup-runtime-executor.sh b/scripts/backup/host110-backup-runtime-executor.sh index fc976169b..f97ab1dc8 100755 --- a/scripts/backup/host110-backup-runtime-executor.sh +++ b/scripts/backup/host110-backup-runtime-executor.sh @@ -45,8 +45,12 @@ VERIFIER_DIGEST="" STAGE_DIR="" ROLLBACK_DIR="" RECEIPT_PATH="" +TERMINAL_RECEIPT_PATH="" APPLY_STARTED=0 APPLY_COMPLETE=0 +COMMIT_CRITICAL=0 +DEFERRED_SIGNAL="" +DEFERRED_SIGNAL_EXIT=0 ROLLBACK_ATTEMPTED=0 ROLLBACK_PERFORMED=0 ROLLBACK_VERIFIED=0 @@ -103,10 +107,17 @@ case "$MODE" in check|apply|verify) ;; *) fail "invalid_mode" ;; esac expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}" [ "$SOURCE_STAGE" = "$expected_source_stage" ] || fail "invalid_source_stage" RECEIPT_PATH="$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json" +TERMINAL_RECEIPT_PATH="$STATUS_ROOT/agent99-backup-runtime-terminal-${RUN_ID}.json" if [ "${HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS:-}" = "1" ] \ && [ "$DEST_ROOT" != "/backup/scripts" ]; then FAULT_INJECTION_ENABLED=1 fi +if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then + case "${HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL:-}" in ""|INT|HUP|TERM) ;; *) fail "invalid_test_commit_signal" ;; esac + case "${HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL:-}" in ""|INT|HUP|TERM) ;; *) fail "invalid_test_terminal_signal" ;; esac + case "${HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT:-}" in ""|write|link|fsync|readback) ;; *) fail "invalid_test_terminal_receipt_fault" ;; esac + case "${HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT:-}" in ""|stage|rollback_prestate) ;; *) fail "invalid_test_cleanup_fault" ;; esac +fi for command_name in awk bash cp dirname flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do command -v "$command_name" >/dev/null 2>&1 || fail "required_command_missing_${command_name}" @@ -314,9 +325,9 @@ rollback_attempted = sys.argv[6] == "1" rollback_performed = sys.argv[7] == "1" rollback_verified = sys.argv[8] == "1" zero_residue_verified = sys.argv[9] == "1" -if status not in {"verified", "failed_rolled_back", "rollback_unverified"}: +if status not in {"payload_verified", "failed_rolled_back", "rollback_unverified"}: raise SystemExit(78) -if status == "verified" and (rollback_attempted or rollback_performed or rollback_verified or not zero_residue_verified): +if status == "payload_verified" and (rollback_attempted or rollback_performed or rollback_verified or not zero_residue_verified): raise SystemExit(78) if status == "failed_rolled_back" and not ( rollback_attempted and rollback_performed and rollback_verified and zero_residue_verified @@ -325,7 +336,7 @@ if status == "failed_rolled_back" and not ( if status == "rollback_unverified" and (not rollback_attempted or rollback_verified): raise SystemExit(78) document = { - "schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v2", + "schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v3", "status": status, "sourceRevision": sys.argv[3], "sourceHead": sys.argv[4], @@ -337,6 +348,7 @@ document = { "rollbackPerformed": rollback_performed, "rollbackVerified": rollback_verified, "zeroResidueVerified": zero_residue_verified, + "zeroResidueScope": "run_owned_destination_temps", "verifierSha256": sys.argv[10], "executorHost": "192.168.0.110", "productionServiceRestarted": False, @@ -349,7 +361,7 @@ temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}") fault_enabled = sys.argv[11] == "1" fault = ( os.environ.get("HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT", "") - if status == "verified" and fault_enabled + if status == "payload_verified" and fault_enabled else "" ) if fault not in {"", "write", "link", "fsync", "readback"}: @@ -412,6 +424,172 @@ PY return 0 } +write_terminal_receipt() { + local terminal_status="$1" + local stage_cleanup_verified="$2" + local rollback_prestate_cleanup_verified="$3" + local deferred_signal="$4" + local terminal_exit_code="$5" + install -d -m 700 "$STATUS_ROOT" || return 1 + [ -d "$STATUS_ROOT" ] && [ ! -L "$STATUS_ROOT" ] || return 1 + [ "$(stat -c '%U:%G:%a' "$STATUS_ROOT")" = "wooo:wooo:700" ] || return 1 + if ! python3 - "$TERMINAL_RECEIPT_PATH" "$RECEIPT_PATH" "$terminal_status" \ + "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$stage_cleanup_verified" \ + "$rollback_prestate_cleanup_verified" "$deferred_signal" "$terminal_exit_code" \ + "$VERIFIER_DIGEST" "$FAULT_INJECTION_ENABLED" <<'PY' +import hashlib +import json +import os +import signal +import sys +import time +from pathlib import Path + +path = Path(sys.argv[1]) +payload_receipt_path = Path(sys.argv[2]) +status = sys.argv[3] +source_revision = sys.argv[4] +source_head = sys.argv[5] +run_id = sys.argv[6] +stage_cleanup_verified = sys.argv[7] == "1" +rollback_prestate_cleanup_verified = sys.argv[8] == "1" +deferred_signal = sys.argv[9] +terminal_exit_code = int(sys.argv[10]) +verifier_digest = sys.argv[11] +fault_enabled = sys.argv[12] == "1" +signal_exit_codes = {"INT": 130, "HUP": 129, "TERM": 143} + +terminal_signal = os.environ.get("HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL", "") if fault_enabled else "" +terminal_fault = os.environ.get("HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT", "") if fault_enabled else "" +if terminal_signal: + os.kill(os.getppid(), getattr(signal, f"SIG{terminal_signal}")) + time.sleep(0.05) + +try: + payload_receipt = json.loads(payload_receipt_path.read_text(encoding="utf-8")) +except (OSError, json.JSONDecodeError): + raise SystemExit(79) +if not ( + payload_receipt.get("schemaVersion") == "agent99_host110_backup_runtime_deploy_receipt_v3" + and payload_receipt.get("status") == "payload_verified" + and payload_receipt.get("sourceRevision") == source_revision + and payload_receipt.get("sourceHead") == source_head + and payload_receipt.get("runId") == run_id + and payload_receipt.get("zeroResidueVerified") is True +): + raise SystemExit(79) + +if status == "verified": + valid = ( + stage_cleanup_verified + and rollback_prestate_cleanup_verified + and not deferred_signal + and terminal_exit_code == 0 + ) +elif status == "cleanup_pending": + valid = ( + not (stage_cleanup_verified and rollback_prestate_cleanup_verified) + and deferred_signal in {"", *signal_exit_codes} + and terminal_exit_code != 0 + ) +elif status == "committed_signal_deferred": + valid = ( + stage_cleanup_verified + and rollback_prestate_cleanup_verified + and deferred_signal in signal_exit_codes + and terminal_exit_code == signal_exit_codes[deferred_signal] + ) +else: + valid = False +if not valid: + raise SystemExit(79) + +document = { + "schemaVersion": "agent99_host110_backup_runtime_terminal_receipt_v1", + "status": status, + "ok": status == "verified", + "payloadCommitted": True, + "payloadReceipt": str(payload_receipt_path), + "sourceRevision": source_revision, + "sourceHead": source_head, + "runId": run_id, + "fileCount": 18, + "backupScriptFileCount": 17, + "backupHealthExporterIncluded": True, + "stageCleanupVerified": stage_cleanup_verified, + "rollbackPrestateCleanupVerified": rollback_prestate_cleanup_verified, + "cleanupVerified": stage_cleanup_verified and rollback_prestate_cleanup_verified, + "independentVerifierVerified": True, + "zeroResidueScope": "run_owned_destination_temps_and_internal_stage_prestate", + "deferredSignal": deferred_signal or None, + "terminalExitCode": terminal_exit_code, + "verifierSha256": verifier_digest, + "executorHost": "192.168.0.110", + "productionServiceRestarted": False, + "secretValuesRead": False, + "writtenAt": int(time.time()), +} +payload = (json.dumps(document, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") +expected_digest = hashlib.sha256(payload).hexdigest() +temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}") +created_final = False +directory_fd = None +try: + if os.path.lexists(path) or os.path.lexists(temporary): + raise FileExistsError(path) + if terminal_fault == "write": + raise OSError("fault_injected_terminal_receipt_write") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as handle: + if handle.write(payload) != len(payload): + raise OSError("terminal_receipt_short_write") + handle.flush() + if terminal_fault == "fsync": + raise OSError("fault_injected_terminal_receipt_fsync") + os.fsync(handle.fileno()) + os.chmod(temporary, 0o600) + temporary_readback = temporary.read_bytes() + if temporary_readback != payload or hashlib.sha256(temporary_readback).hexdigest() != expected_digest: + raise OSError("terminal_receipt_temporary_readback_failed") + if terminal_fault == "link": + raise OSError("fault_injected_terminal_receipt_link") + os.link(temporary, path) + created_final = True + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + directory_fd = os.open(path.parent, directory_flags) + os.fsync(directory_fd) + if terminal_fault == "readback": + raise OSError("fault_injected_terminal_receipt_readback") + readback = path.read_bytes() + if readback != payload or hashlib.sha256(readback).hexdigest() != expected_digest: + raise OSError("terminal_receipt_final_readback_failed") + if json.loads(readback.decode("utf-8")) != document: + raise OSError("terminal_receipt_document_readback_failed") + temporary.unlink() + os.fsync(directory_fd) +except BaseException: + try: + if created_final: + path.unlink(missing_ok=True) + temporary.unlink(missing_ok=True) + if directory_fd is None and path.parent.is_dir(): + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + directory_fd = os.open(path.parent, directory_flags) + if directory_fd is not None: + os.fsync(directory_fd) + finally: + raise +finally: + if directory_fd is not None: + os.close(directory_fd) +PY + then + return 1 + fi + [ -f "$TERMINAL_RECEIPT_PATH" ] && [ ! -L "$TERMINAL_RECEIPT_PATH" ] || return 1 + return 0 +} + record_prestate() { local name dest_path digest metadata : > "$ROLLBACK_DIR/prestate.tsv" || return 74 @@ -530,6 +708,35 @@ rollback_transaction() { return 1 } +handle_agent99_signal() { + local signal_name="$1" + local signal_exit_code="$2" + if [ "$COMMIT_CRITICAL" -eq 1 ] || [ "$APPLY_COMPLETE" -eq 1 ]; then + if [ -z "$DEFERRED_SIGNAL" ]; then + DEFERRED_SIGNAL="$signal_name" + DEFERRED_SIGNAL_EXIT="$signal_exit_code" + fi + return 0 + fi + exit "$signal_exit_code" +} + +cleanup_postcommit_directory() { + local cleanup_kind="$1" + local cleanup_path="$2" + local fault="" + if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then + fault="${HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT:-}" + fi + if [ "$fault" = "$cleanup_kind" ]; then + return 1 + fi + rm -rf -- "$cleanup_path" || return 1 + [ ! -e "$cleanup_path" ] && [ ! -L "$cleanup_path" ] || return 1 + fsync_paths_and_parents "$cleanup_path" || return 1 + [ ! -e "$cleanup_path" ] && [ ! -L "$cleanup_path" ] +} + cleanup() { local exit_status=$? local rollback_status="rollback_unverified" @@ -544,10 +751,12 @@ cleanup() { failure_receipt_written=1 fi fi - if [ -n "$STAGE_DIR" ]; then + if [ "$APPLY_COMPLETE" -ne 1 ] && [ -n "$STAGE_DIR" ]; then rm -rf -- "$STAGE_DIR" || true fi - if [ "$ROLLBACK_VERIFIED" -eq 1 ] && [ "$failure_receipt_written" -eq 1 ]; then + if [ "$APPLY_COMPLETE" -ne 1 ] \ + && [ "$ROLLBACK_VERIFIED" -eq 1 ] \ + && [ "$failure_receipt_written" -eq 1 ]; then if [ -n "$ROLLBACK_DIR" ]; then rm -rf -- "$ROLLBACK_DIR" || true fi @@ -558,17 +767,19 @@ cleanup() { validate_source || fail "source_validation_failed" if [ "$MODE" = "check" ]; then - printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" + printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","runId":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" exit 0 fi if [ "$MODE" = "verify" ]; then verify_destination || fail "destination_verification_failed" - printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" + printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","runId":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" exit 0 fi [ ! -e "$RECEIPT_PATH" ] && [ ! -L "$RECEIPT_PATH" ] || fail "run_identity_receipt_exists" +[ ! -e "$TERMINAL_RECEIPT_PATH" ] && [ ! -L "$TERMINAL_RECEIPT_PATH" ] \ + || fail "run_identity_terminal_receipt_exists" STAGE_DIR="$STAGE_ROOT/$RUN_ID" ROLLBACK_DIR="$ROLLBACK_ROOT/$RUN_ID" [ ! -e "$STAGE_DIR" ] || fail "run_identity_stage_exists" @@ -589,8 +800,9 @@ fi install -d -m 700 "$STAGE_DIR" "$ROLLBACK_DIR" trap cleanup EXIT -trap 'exit 130' INT -trap 'exit 143' TERM HUP +trap 'handle_agent99_signal INT 130' INT +trap 'handle_agent99_signal HUP 129' HUP +trap 'handle_agent99_signal TERM 143' TERM for name in "${PAYLOAD_FILES[@]}"; do install -m 700 "$SOURCE_STAGE/$name" "$STAGE_DIR/$name" @@ -651,43 +863,116 @@ PY verify_run_owned_temp_absence || fail "post_apply_temp_residue_detected" RUN_TEMP_RESIDUE_VERIFIED=1 -if ! write_receipt "verified" 0 0 0 "$RUN_TEMP_RESIDUE_VERIFIED"; then +COMMIT_CRITICAL=1 +if ! write_receipt "payload_verified" 0 0 0 "$RUN_TEMP_RESIDUE_VERIFIED"; then + COMMIT_CRITICAL=0 fail "durable_receipt_failed" fi -receipt_path="$RECEIPT_PATH" +if [ "$FAULT_INJECTION_ENABLED" -eq 1 ] \ + && [ -n "${HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL:-}" ]; then + case "$HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL" in + INT|HUP|TERM) kill -s "$HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL" "$$" ;; + esac +fi APPLY_COMPLETE=1 -trap - EXIT HUP INT TERM +COMMIT_CRITICAL=0 stage_cleanup_verified=0 rollback_prestate_cleanup_verified=0 -if rm -rf -- "$STAGE_DIR" && [ ! -e "$STAGE_DIR" ] && [ ! -L "$STAGE_DIR" ]; then +if cleanup_postcommit_directory "stage" "$STAGE_DIR"; then stage_cleanup_verified=1 fi -if rm -rf -- "$ROLLBACK_DIR" && [ ! -e "$ROLLBACK_DIR" ] && [ ! -L "$ROLLBACK_DIR" ]; then +if cleanup_postcommit_directory "rollback_prestate" "$ROLLBACK_DIR"; then rollback_prestate_cleanup_verified=1 fi -STAGE_DIR="" -ROLLBACK_DIR="" -python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$receipt_path" "$verifier_result" \ - "$stage_cleanup_verified" "$rollback_prestate_cleanup_verified" <<'PY' +terminal_status="verified" +terminal_exit_code=0 +# Cleanup has completed. Freeze terminal signal state before the immutable +# terminal writer so its receipt, stdout, and exit status cannot diverge. +trap '' HUP INT TERM +if [ "$stage_cleanup_verified" -ne 1 ] || [ "$rollback_prestate_cleanup_verified" -ne 1 ]; then + terminal_status="cleanup_pending" + terminal_exit_code=75 +elif [ -n "$DEFERRED_SIGNAL" ]; then + terminal_status="committed_signal_deferred" + terminal_exit_code="$DEFERRED_SIGNAL_EXIT" +fi +if ! write_terminal_receipt "$terminal_status" "$stage_cleanup_verified" \ + "$rollback_prestate_cleanup_verified" "$DEFERRED_SIGNAL" "$terminal_exit_code"; then + trap - EXIT + python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$RECEIPT_PATH" \ + "$TERMINAL_RECEIPT_PATH" "$stage_cleanup_verified" \ + "$rollback_prestate_cleanup_verified" "$DEFERRED_SIGNAL" <<'PY' import json import sys print(json.dumps({ "schemaVersion": "agent99_host110_backup_runtime_executor_v1", "mode": "apply", - "ok": True, + "status": "terminal_receipt_failed", + "ok": False, "sourceRevision": sys.argv[1], "sourceHead": sys.argv[2], + "runId": sys.argv[3], "fileCount": 18, "backupScriptFileCount": 17, "backupHealthExporterIncluded": True, + "payloadCommitted": True, "rollbackAttempted": False, "rollbackPerformed": False, "rollbackVerified": False, "zeroResidueVerified": True, - "stageCleanupVerified": sys.argv[5] == "1", - "rollbackPrestateCleanupVerified": sys.argv[6] == "1", - "receipt": sys.argv[3], - "verifier": json.loads(sys.argv[4]), + "zeroResidueScope": "run_owned_destination_temps", + "stageCleanupVerified": sys.argv[6] == "1", + "rollbackPrestateCleanupVerified": sys.argv[7] == "1", + "receipt": sys.argv[4], + "terminalReceipt": sys.argv[5], + "terminalReceiptPublished": False, + "terminalExitCode": 76, + "deferredSignal": sys.argv[8] or None, + "independentVerifierVerified": True, + "verifier": None, }, ensure_ascii=True, sort_keys=True)) PY + printf 'HOST110_BACKUP_RUNTIME_OK=0\nERROR=terminal_receipt_failed\n' >&2 + exit 76 +fi +if [ "$stage_cleanup_verified" -eq 1 ]; then STAGE_DIR=""; fi +if [ "$rollback_prestate_cleanup_verified" -eq 1 ]; then ROLLBACK_DIR=""; fi +trap - EXIT +python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$RECEIPT_PATH" "$TERMINAL_RECEIPT_PATH" \ + "$verifier_result" "$stage_cleanup_verified" "$rollback_prestate_cleanup_verified" \ + "$terminal_status" "$terminal_exit_code" "$DEFERRED_SIGNAL" "$RUN_ID" <<'PY' +import json +import sys + +status = sys.argv[8] +verifier = json.loads(sys.argv[5]) +print(json.dumps({ + "schemaVersion": "agent99_host110_backup_runtime_executor_v1", + "mode": "apply", + "status": status, + "ok": status == "verified", + "sourceRevision": sys.argv[1], + "sourceHead": sys.argv[2], + "runId": sys.argv[11], + "fileCount": 18, + "backupScriptFileCount": 17, + "backupHealthExporterIncluded": True, + "payloadCommitted": True, + "rollbackAttempted": False, + "rollbackPerformed": False, + "rollbackVerified": False, + "zeroResidueVerified": True, + "zeroResidueScope": "run_owned_destination_temps", + "stageCleanupVerified": sys.argv[6] == "1", + "rollbackPrestateCleanupVerified": sys.argv[7] == "1", + "receipt": sys.argv[3], + "terminalReceipt": sys.argv[4], + "terminalReceiptPublished": True, + "terminalExitCode": int(sys.argv[9]), + "deferredSignal": sys.argv[10] or None, + "independentVerifierVerified": True, + "verifier": verifier if status == "verified" else None, +}, ensure_ascii=True, sort_keys=True)) +PY +exit "$terminal_exit_code" diff --git a/scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1 b/scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1 new file mode 100644 index 000000000..96d5d85ab --- /dev/null +++ b/scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1 @@ -0,0 +1,220 @@ +$ErrorActionPreference = "Stop" + +function Write-ReplayTrace { + param([string]$Step) + if ($ReplayTrace) { Write-Output "REPLAY_STEP=$Step" } +} + +if (-not $BrokerGzipBase64) { throw "broker_payload_missing" } +Write-ReplayTrace "decode_source" +$compressed = [Convert]::FromBase64String([string]$BrokerGzipBase64) +$inputStream = New-Object IO.MemoryStream(,$compressed) +$gzip = New-Object IO.Compression.GzipStream( + $inputStream, + [IO.Compression.CompressionMode]::Decompress +) +$reader = New-Object IO.StreamReader($gzip, [Text.Encoding]::UTF8) +try { + $brokerSource = $reader.ReadToEnd() +} finally { + $reader.Dispose() + $gzip.Dispose() + $inputStream.Dispose() +} + +$tokens = $null +$parseErrors = $null +$ast = [Management.Automation.Language.Parser]::ParseInput( + $brokerSource, + [ref]$tokens, + [ref]$parseErrors +) +if ($parseErrors.Count -ne 0) { throw "broker_parse_failed" } +Write-ReplayTrace "define_functions" + +$functionNames = @( + "Test-Agent99JsonField", + "Assert-Agent99JsonFields", + "Convert-Agent99ExecutorDocument", + "Assert-Agent99ApplyResultFields", + "Assert-Agent99VerifierFields", + "Get-Agent99ExecutorResult", + "Get-Agent99CommittedFailureResult" +) +foreach ($functionName in $functionNames) { + $node = $ast.Find( + { + param($candidate) + $candidate -is [Management.Automation.Language.FunctionDefinitionAst] -and + $candidate.Name -eq $functionName + }, + $true + ) + if ($null -eq $node) { throw "broker_function_missing_$functionName" } + Invoke-Expression $node.Extent.Text +} + +$SourceRevision = "a" * 40 +$RunId = "windows99-contract-replay" +$ExpectedFileCount = 18 +$sourceHead = "b" * 40 +$payloadReceipt = "/backup/status/agent99-backup-runtime-deploy-$RunId.json" +$terminalReceipt = "/backup/status/agent99-backup-runtime-terminal-$RunId.json" + +function Copy-ReplayDocument { + param([object]$Document) + return ($Document | ConvertTo-Json -Compress -Depth 12 | ConvertFrom-Json) +} + +function New-ReplayTransport { + param([object]$Document, [int]$ExitCode) + return [pscustomobject]@{ + ok = [bool]($ExitCode -eq 0) + exitCode = $ExitCode + reason = if ($ExitCode -eq 0) { "completed" } else { "process_failed" } + stdout = $Document | ConvertTo-Json -Compress -Depth 12 + stderrPresent = $false + } +} + +function Assert-Rejected { + param([scriptblock]$Operation, [string]$Name) + $accepted = $false + try { + & $Operation | Out-Null + $accepted = $true + } catch {} + if ($accepted) { throw "contract_mutation_accepted_$Name" } +} + +$verifier = [pscustomobject]@{ + schemaVersion = "agent99_host110_backup_runtime_verifier_v1" + ok = $true + sourceRevision = $SourceRevision + runId = $RunId + fileCount = 18 + backupScriptFileCount = 17 + backupHealthExporterIncluded = $true + remoteWritePerformed = $false + selfIdentityVerified = $true +} +$success = [pscustomobject]@{ + schemaVersion = "agent99_host110_backup_runtime_executor_v1" + mode = "apply" + status = "verified" + ok = $true + sourceRevision = $SourceRevision + sourceHead = $sourceHead + runId = $RunId + fileCount = 18 + backupScriptFileCount = 17 + backupHealthExporterIncluded = $true + payloadCommitted = $true + rollbackAttempted = $false + rollbackPerformed = $false + rollbackVerified = $false + zeroResidueVerified = $true + zeroResidueScope = "run_owned_destination_temps" + stageCleanupVerified = $true + rollbackPrestateCleanupVerified = $true + receipt = $payloadReceipt + terminalReceipt = $terminalReceipt + terminalReceiptPublished = $true + terminalExitCode = 0 + deferredSignal = $null + independentVerifierVerified = $true + verifier = $verifier +} + +Write-ReplayTrace "accept_success" +Get-Agent99ExecutorResult (New-ReplayTransport $success 0) "Apply" | Out-Null +$topLevelRequired = @( + "schemaVersion", "mode", "status", "ok", "sourceRevision", "sourceHead", + "runId", "fileCount", "backupScriptFileCount", "backupHealthExporterIncluded", + "payloadCommitted", "rollbackAttempted", "rollbackPerformed", "rollbackVerified", + "zeroResidueVerified", "zeroResidueScope", "stageCleanupVerified", + "rollbackPrestateCleanupVerified", "receipt", "terminalReceipt", "terminalExitCode", + "terminalReceiptPublished", "deferredSignal", "independentVerifierVerified", "verifier" +) +$verifierRequired = @( + "schemaVersion", "ok", "sourceRevision", "runId", "fileCount", + "backupScriptFileCount", "backupHealthExporterIncluded", "remoteWritePerformed", + "selfIdentityVerified" +) +$missingFieldRejections = 0 +Write-ReplayTrace "reject_missing_top" +foreach ($field in $topLevelRequired) { + $mutated = Copy-ReplayDocument $success + $mutated.PSObject.Properties.Remove($field) + Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_$field" + $missingFieldRejections++ +} +Write-ReplayTrace "reject_missing_verifier" +foreach ($field in $verifierRequired) { + $mutated = Copy-ReplayDocument $success + $mutated.verifier.PSObject.Properties.Remove($field) + Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_verifier_$field" + $missingFieldRejections++ +} + +$typeMutations = @( + @{ field = "rollbackAttempted"; value = "false" }, + @{ field = "rollbackPerformed"; value = "false" }, + @{ field = "rollbackVerified"; value = "false" }, + @{ field = "terminalExitCode"; value = "0" }, + @{ field = "deferredSignal"; value = 0 } +) +$typeRejections = 0 +Write-ReplayTrace "reject_types" +foreach ($mutation in $typeMutations) { + $mutated = Copy-ReplayDocument $success + $mutated.($mutation.field) = $mutation.value + Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_$($mutation.field)" + $typeRejections++ +} +$mutated = Copy-ReplayDocument $success +$mutated.verifier.remoteWritePerformed = "false" +Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_verifier_remoteWritePerformed" +$typeRejections++ + +$cleanupPending = Copy-ReplayDocument $success +$cleanupPending.ok = $false +$cleanupPending.status = "cleanup_pending" +$cleanupPending.stageCleanupVerified = $false +$cleanupPending.terminalExitCode = 75 +$cleanupPending.verifier = $null +Write-ReplayTrace "accept_cleanup_pending" +Get-Agent99CommittedFailureResult (New-ReplayTransport $cleanupPending 75) | Out-Null + +$signalDeferred = Copy-ReplayDocument $success +$signalDeferred.ok = $false +$signalDeferred.status = "committed_signal_deferred" +$signalDeferred.terminalExitCode = 143 +$signalDeferred.deferredSignal = "TERM" +$signalDeferred.verifier = $null +Write-ReplayTrace "accept_signal_deferred" +Get-Agent99CommittedFailureResult (New-ReplayTransport $signalDeferred 143) | Out-Null + +$terminalReceiptFailed = Copy-ReplayDocument $success +$terminalReceiptFailed.ok = $false +$terminalReceiptFailed.status = "terminal_receipt_failed" +$terminalReceiptFailed.terminalReceiptPublished = $false +$terminalReceiptFailed.terminalExitCode = 76 +$terminalReceiptFailed.verifier = $null +Write-ReplayTrace "accept_terminal_receipt_failed" +Get-Agent99CommittedFailureResult (New-ReplayTransport $terminalReceiptFailed 76) | Out-Null + +Write-ReplayTrace "complete" +[pscustomobject]@{ + schemaVersion = "agent99_host110_broker_contract_replay_v1" + ok = $true + parser = "WindowsPowerShell" + successAccepted = $true + cleanupPendingPreserved = $true + committedSignalPreserved = $true + terminalReceiptFailurePreserved = $true + missingFieldRejections = $missingFieldRejections + typeRejections = $typeRejections + remoteWritePerformed = $false + secretValuesRead = $false +} | ConvertTo-Json -Compress diff --git a/scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py b/scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py index dacd13100..b025a3556 100644 --- a/scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py +++ b/scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py @@ -20,6 +20,9 @@ ROOT = Path(__file__).resolve().parents[3] BROKER = ROOT / "agent99-host110-backup-runtime-broker.ps1" EXECUTOR = ROOT / "scripts" / "backup" / "host110-backup-runtime-executor.sh" VERIFIER = ROOT / "scripts" / "backup" / "verify-host110-backup-runtime.py" +BROKER_CONTRACT_REPLAY = ( + ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-host110-broker-contract-replay.ps1" +) EXPECTED_RUNTIME_FILES = { @@ -71,6 +74,12 @@ def _powershell_array(source: str, variable: str) -> tuple[str, ...]: return tuple(re.findall(r'"([^"]+)"', match.group(1))) +def _powershell_function(source: str, name: str) -> str: + start = source.index(f"function {name} {{") + next_function = source.find("\nfunction ", start + 1) + return source[start:] if next_function < 0 else source[start:next_function] + + def _executor_payload_order(source: str) -> tuple[str, ...]: match = re.search(r"readonly -a RUNTIME_FILES=\((.*?)\n\)", source, re.DOTALL) assert match is not None @@ -240,6 +249,10 @@ def _run_executor_harness(harness: dict[str, object], **extra_env: str) -> subpr "HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT", "HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT", "HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT", + "HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL", + "HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL", + "HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT", + "HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT", ): environment.pop(name, None) environment["HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS"] = "1" @@ -281,6 +294,16 @@ def _assert_prestate_restored(harness: dict[str, object]) -> None: assert (target.read_bytes(), target.stat().st_mode & 0o777) == before_state[name] +def _assert_payload_committed(harness: dict[str, object]) -> None: + destination = Path(harness["destination"]) + exporter_root = Path(harness["exporter_root"]) + source_stage = Path(harness["source_stage"]) + for name in harness["payload_order"]: + target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name + assert target.read_bytes() == (source_stage / name).read_bytes() + assert target.stat().st_mode & 0o777 == 0o755 + + def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None: source = BROKER.read_text(encoding="utf-8") @@ -304,6 +327,100 @@ def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None: assert "taskkill.exe /PID $process.Id /T /F" in source assert "$process.Kill()" in source assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths' in source + assert '[string]$parsed.status -ne "verified"' in source + assert '-not [bool]$parsed.payloadCommitted' in source + assert '-not [bool]$parsed.stageCleanupVerified' in source + assert '-not [bool]$parsed.rollbackPrestateCleanupVerified' in source + assert '[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps"' in source + assert '[int64]$parsed.terminalExitCode -ne 0' in source + assert 'agent99-backup-runtime-terminal-$RunId.json' in source + assert '$property = $Object.PSObject.Properties[$Name]' in source + assert 'if ($null -eq $property) { return $false }' in source + assert 'Get-Agent99CommittedFailureResult $applyTransport' in source + assert 'throw "host110_apply_$([string]$apply.status)"' in source + + +def test_broker_required_field_contract_rejects_missing_fields_before_cast() -> None: + source = BROKER.read_text(encoding="utf-8") + field_guard = _powershell_function(source, "Test-Agent99JsonField") + common_contract = _powershell_function(source, "Convert-Agent99ExecutorDocument") + apply_contract = _powershell_function(source, "Assert-Agent99ApplyResultFields") + verifier_contract = _powershell_function(source, "Assert-Agent99VerifierFields") + committed_failure = _powershell_function(source, "Get-Agent99CommittedFailureResult") + + assert '$property = $Object.PSObject.Properties[$Name]' in field_guard + assert 'if ($null -eq $property) { return $false }' in field_guard + for field in ( + "schemaVersion", + "mode", + "ok", + "sourceRevision", + "sourceHead", + "runId", + "fileCount", + "backupScriptFileCount", + "backupHealthExporterIncluded", + ): + assert re.search(rf"^\s+{field} = \"", common_contract, re.MULTILINE), field + for field in ( + "status", + "payloadCommitted", + "rollbackAttempted", + "rollbackPerformed", + "rollbackVerified", + "zeroResidueVerified", + "zeroResidueScope", + "stageCleanupVerified", + "rollbackPrestateCleanupVerified", + "receipt", + "terminalReceipt", + "terminalReceiptPublished", + "terminalExitCode", + "deferredSignal", + "independentVerifierVerified", + "verifier", + ): + assert re.search(rf"^\s+{field} = ", apply_contract, re.MULTILINE), field + for field in ( + "schemaVersion", + "ok", + "sourceRevision", + "runId", + "fileCount", + "backupScriptFileCount", + "backupHealthExporterIncluded", + "remoteWritePerformed", + "selfIdentityVerified", + ): + assert re.search(rf"^\s+{field} = \"", verifier_contract, re.MULTILINE), field + assert 'status -eq "cleanup_pending"' in committed_failure + assert 'status -eq "committed_signal_deferred"' in committed_failure + assert 'verifier = $VerifierKind' in apply_contract + + +def test_windows99_contract_replay_uses_actual_broker_functions_without_remote_write() -> None: + source = BROKER_CONTRACT_REPLAY.read_text(encoding="utf-8") + + assert "FunctionDefinitionAst" in source + for function_name in ( + "Test-Agent99JsonField", + "Assert-Agent99JsonFields", + "Convert-Agent99ExecutorDocument", + "Assert-Agent99ApplyResultFields", + "Assert-Agent99VerifierFields", + "Get-Agent99ExecutorResult", + "Get-Agent99CommittedFailureResult", + ): + assert f'"{function_name}"' in source + assert "$mutated.PSObject.Properties.Remove($field)" in source + assert "$mutated.verifier.PSObject.Properties.Remove($field)" in source + assert 'status = "cleanup_pending"' in source + assert 'status = "committed_signal_deferred"' in source + assert 'status = "terminal_receipt_failed"' in source + assert "remoteWritePerformed = $false" in source + assert "secretValuesRead = $false" in source + for forbidden in ("New-Item", "Set-Content", "WriteAllText", "ssh.exe", "scp.exe"): + assert forbidden not in source def test_broker_scp_basenames_match_executor_source_stage_contract() -> None: @@ -385,11 +502,16 @@ def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None: assert '"rollbackPerformed": rollback_performed' in source assert '"rollbackVerified": rollback_verified' in source assert '"zeroResidueVerified": zero_residue_verified' in source + assert '"zeroResidueScope": "run_owned_destination_temps"' in source assert 'cleanup_run_owned_temps' in source assert 'verify_run_owned_temp_absence' in source assert 'HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT' in source assert 'HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT' in source assert 'HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT' in source + assert 'HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL' in source + assert 'HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL' in source + assert 'HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT' in source + assert 'HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT' in source assert 'HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS' in source assert '[ "$DEST_ROOT" != "/backup/scripts" ]' in source assert 'fsync_rollback_prestate || return 75' in source @@ -401,8 +523,18 @@ def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None: assert 'os.fsync(handle.fileno())' in source assert 'os.fsync(directory_fd)' in source assert 'hashlib.sha256(readback).hexdigest()' in source - assert source.index('write_receipt "verified"') < source.index("\nAPPLY_COMPLETE=1") - assert source.index("\nAPPLY_COMPLETE=1") < source.rindex('rm -rf -- "$ROLLBACK_DIR"') + assert source.index("COMMIT_CRITICAL=1") < source.index('write_receipt "payload_verified"') + assert source.index('write_receipt "payload_verified"') < source.index("\nAPPLY_COMPLETE=1") + assert source.index("\nAPPLY_COMPLETE=1") < source.rindex("\nCOMMIT_CRITICAL=0") + assert source.index("\nAPPLY_COMPLETE=1") < source.rindex( + 'cleanup_postcommit_directory "rollback_prestate"' + ) + assert 'handle_agent99_signal HUP 129' in source + assert 'handle_agent99_signal TERM 143' in source + assert source.index("trap '' HUP INT TERM") < source.index('write_terminal_receipt "$terminal_status"') + assert 'write_terminal_receipt "$terminal_status"' in source + assert 'terminal_status="cleanup_pending"' in source + assert 'terminal_status="committed_signal_deferred"' in source assert 'run_identity_receipt_exists' in source assert 'run_identity_stage_exists' in source assert 'run_identity_rollback_exists' in source @@ -432,22 +564,205 @@ def test_apply_success_publishes_durable_receipt_before_prestate_cleanup(tmp_pat assert output["zeroResidueVerified"] is True assert output["stageCleanupVerified"] is True assert output["rollbackPrestateCleanupVerified"] is True + assert output["status"] == "verified" + assert output["payloadCommitted"] is True + assert output["runId"] == run_id + assert output["terminalExitCode"] == 0 + assert output["terminalReceiptPublished"] is True + assert output["deferredSignal"] is None receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json" receipt = json.loads(receipt_path.read_text(encoding="utf-8")) - assert receipt["status"] == "verified" + assert receipt["status"] == "payload_verified" assert receipt["rollbackAttempted"] is False assert receipt["rollbackPerformed"] is False assert receipt["rollbackVerified"] is False assert receipt["zeroResidueVerified"] is True + assert receipt["zeroResidueScope"] == "run_owned_destination_temps" + terminal_path = Path(harness["status_root"]) / f"agent99-backup-runtime-terminal-{run_id}.json" + terminal = json.loads(terminal_path.read_text(encoding="utf-8")) + assert terminal["status"] == "verified" + assert terminal["ok"] is True + assert terminal["payloadCommitted"] is True + assert terminal["stageCleanupVerified"] is True + assert terminal["rollbackPrestateCleanupVerified"] is True + assert terminal["terminalExitCode"] == 0 + assert terminal["zeroResidueScope"] == "run_owned_destination_temps_and_internal_stage_prestate" + assert output["terminalReceipt"] == str(terminal_path) assert not (Path(harness["rollback_root"]) / run_id).exists() assert not (Path(harness["stage_root"]) / run_id).exists() - for name in harness["payload_order"]: - target = ( - Path(harness["exporter_root"]) / name - if name == "backup-health-textfile-exporter.py" - else Path(harness["destination"]) / name - ) - assert target.read_bytes() == (Path(harness["source_stage"]) / name).read_bytes() + _assert_payload_committed(harness) + + +@pytest.mark.parametrize(("signal_name", "exit_code"), [("TERM", 143), ("HUP", 129), ("INT", 130)]) +def test_commit_window_signal_is_deferred_until_payload_and_terminal_are_consistent( + tmp_path: Path, + signal_name: str, + exit_code: int, +) -> None: + run_id = f"commit-signal-{signal_name.lower()}" + harness = _build_executor_harness(tmp_path, run_id) + + result = _run_executor_harness( + harness, + HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL=signal_name, + ) + + assert result.returncode == exit_code, result.stderr + output = json.loads(result.stdout) + assert output["ok"] is False + assert output["status"] == "committed_signal_deferred" + assert output["payloadCommitted"] is True + assert output["runId"] == run_id + assert output["deferredSignal"] == signal_name + assert output["terminalExitCode"] == exit_code + assert output["terminalReceiptPublished"] is True + assert output["stageCleanupVerified"] is True + assert output["rollbackPrestateCleanupVerified"] is True + _assert_payload_committed(harness) + payload_receipt = json.loads( + ( + Path(harness["status_root"]) + / f"agent99-backup-runtime-deploy-{run_id}.json" + ).read_text(encoding="utf-8") + ) + assert payload_receipt["status"] == "payload_verified" + terminal = json.loads( + ( + Path(harness["status_root"]) + / f"agent99-backup-runtime-terminal-{run_id}.json" + ).read_text(encoding="utf-8") + ) + assert terminal["status"] == "committed_signal_deferred" + assert terminal["ok"] is False + assert terminal["deferredSignal"] == signal_name + assert terminal["terminalExitCode"] == exit_code + assert not (Path(harness["rollback_root"]) / run_id).exists() + assert not (Path(harness["stage_root"]) / run_id).exists() + + +@pytest.mark.parametrize("signal_name", ["TERM", "HUP", "INT"]) +def test_terminal_publication_masks_late_signal_without_receipt_stdout_exit_divergence( + tmp_path: Path, + signal_name: str, +) -> None: + run_id = f"terminal-signal-{signal_name.lower()}" + harness = _build_executor_harness(tmp_path, run_id) + + result = _run_executor_harness( + harness, + HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL=signal_name, + ) + + assert result.returncode == 0, result.stderr + output = json.loads(result.stdout) + assert output["ok"] is True + assert output["status"] == "verified" + assert output["payloadCommitted"] is True + assert output["runId"] == run_id + assert output["deferredSignal"] is None + assert output["terminalExitCode"] == 0 + assert output["terminalReceiptPublished"] is True + terminal = json.loads( + ( + Path(harness["status_root"]) + / f"agent99-backup-runtime-terminal-{run_id}.json" + ).read_text(encoding="utf-8") + ) + assert terminal["ok"] is True + assert terminal["status"] == output["status"] + assert terminal["deferredSignal"] == output["deferredSignal"] + assert terminal["terminalExitCode"] == output["terminalExitCode"] + _assert_payload_committed(harness) + + +@pytest.mark.parametrize("fault", ["write", "link", "fsync", "readback"]) +def test_terminal_receipt_fault_is_nonzero_with_payload_receipt_only( + tmp_path: Path, + fault: str, +) -> None: + run_id = f"terminal-receipt-{fault}" + harness = _build_executor_harness(tmp_path, run_id) + + result = _run_executor_harness( + harness, + HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT=fault, + ) + + assert result.returncode == 76 + assert '"ok": true' not in result.stdout.lower() + assert "terminal_receipt_failed" in result.stderr + output = json.loads(result.stdout) + assert output["ok"] is False + assert output["status"] == "terminal_receipt_failed" + assert output["payloadCommitted"] is True + assert output["runId"] == run_id + assert output["terminalReceiptPublished"] is False + assert output["terminalExitCode"] == 76 + _assert_payload_committed(harness) + payload_receipt = json.loads( + ( + Path(harness["status_root"]) + / f"agent99-backup-runtime-deploy-{run_id}.json" + ).read_text(encoding="utf-8") + ) + assert payload_receipt["status"] == "payload_verified" + assert not ( + Path(harness["status_root"]) + / f"agent99-backup-runtime-terminal-{run_id}.json" + ).exists() + assert not (Path(harness["rollback_root"]) / run_id).exists() + assert not (Path(harness["stage_root"]) / run_id).exists() + + +@pytest.mark.parametrize( + ("fault", "stage_clean", "rollback_clean"), + [("stage", False, True), ("rollback_prestate", True, False)], +) +def test_postcommit_cleanup_failure_is_nonzero_and_never_false_green( + tmp_path: Path, + fault: str, + stage_clean: bool, + rollback_clean: bool, +) -> None: + run_id = f"cleanup-{fault}" + harness = _build_executor_harness(tmp_path, run_id) + + result = _run_executor_harness( + harness, + HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT=fault, + ) + + assert result.returncode == 75, result.stderr + assert '"ok": true' not in result.stdout.lower() + output = json.loads(result.stdout) + assert output["ok"] is False + assert output["status"] == "cleanup_pending" + assert output["payloadCommitted"] is True + assert output["runId"] == run_id + assert output["stageCleanupVerified"] is stage_clean + assert output["rollbackPrestateCleanupVerified"] is rollback_clean + assert output["terminalReceiptPublished"] is True + _assert_payload_committed(harness) + payload_receipt = json.loads( + ( + Path(harness["status_root"]) + / f"agent99-backup-runtime-deploy-{run_id}.json" + ).read_text(encoding="utf-8") + ) + assert payload_receipt["status"] == "payload_verified" + terminal = json.loads( + ( + Path(harness["status_root"]) + / f"agent99-backup-runtime-terminal-{run_id}.json" + ).read_text(encoding="utf-8") + ) + assert terminal["status"] == "cleanup_pending" + assert terminal["ok"] is False + assert terminal["payloadCommitted"] is True + assert terminal["stageCleanupVerified"] is stage_clean + assert terminal["rollbackPrestateCleanupVerified"] is rollback_clean + assert (Path(harness["stage_root"]) / run_id).exists() is (not stage_clean) + assert (Path(harness["rollback_root"]) / run_id).exists() is (not rollback_clean) @pytest.mark.parametrize("fault", ["write", "link", "fsync", "readback"])