Compare commits

...

1 Commits

Author SHA1 Message Date
Your Name
d1797a1c87 fix(agent99): close host110 backup commit races 2026-07-19 01:08:45 +08:00
5 changed files with 1282 additions and 42 deletions

View File

@@ -268,30 +268,157 @@ function New-Agent99SourcePackage {
}
}
function Get-Agent99ExecutorResult {
param([object]$Transport, [string]$ExpectedMode)
function Test-Agent99JsonField {
param(
[object]$Object,
[string]$Name,
[ValidateSet("String", "Boolean", "Integer", "Object", "Null", "NullOrString")]
[string]$Kind
)
if (-not $Transport.ok) { throw "host110_${ExpectedMode}_transport_failed" }
if ($null -eq $Object) { return $false }
$property = $Object.PSObject.Properties[$Name]
if ($null -eq $property) { return $false }
$value = $property.Value
switch ($Kind) {
"String" { return [bool]($value -is [string]) }
"Boolean" { return [bool]($value -is [bool]) }
"Integer" { return [bool]($value -is [int] -or $value -is [long]) }
"Object" { return [bool]($value -is [System.Management.Automation.PSCustomObject]) }
"Null" { return [bool]($null -eq $value) }
"NullOrString" { return [bool]($null -eq $value -or $value -is [string]) }
}
return $false
}
function Assert-Agent99JsonFields {
param([object]$Object, [hashtable]$Contract, [string]$ErrorCode)
foreach ($name in $Contract.Keys) {
if (-not (Test-Agent99JsonField $Object $name ([string]$Contract[$name]))) {
throw $ErrorCode
}
}
}
function Convert-Agent99ExecutorDocument {
param(
[object]$Transport,
[string]$ExpectedMode,
[switch]$AllowProcessFailure
)
if (-not $Transport.ok) {
if (
-not $AllowProcessFailure -or
[string]$Transport.reason -ne "process_failed" -or
[int]$Transport.exitCode -le 0 -or
[string]::IsNullOrWhiteSpace([string]$Transport.stdout)
) {
throw "host110_${ExpectedMode}_transport_failed"
}
}
try {
$parsed = [string]$Transport.stdout | ConvertFrom-Json
} catch {
throw "host110_${ExpectedMode}_receipt_invalid"
}
Assert-Agent99JsonFields $parsed @{
schemaVersion = "String"
mode = "String"
ok = "Boolean"
sourceRevision = "String"
sourceHead = "String"
runId = "String"
fileCount = "Integer"
backupScriptFileCount = "Integer"
backupHealthExporterIncluded = "Boolean"
} "host110_${ExpectedMode}_receipt_required_field_failed"
if (
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_executor_v1" -or
[string]$parsed.mode -ne $ExpectedMode.ToLowerInvariant() -or
-not [bool]$parsed.ok -or
[string]$parsed.sourceRevision -ne $SourceRevision -or
[int]$parsed.fileCount -ne $ExpectedFileCount -or
[int]$parsed.backupScriptFileCount -ne 17 -or
[string]$parsed.sourceHead -notmatch "^[0-9a-f]{40}$" -or
[string]$parsed.runId -ne $RunId -or
[int64]$parsed.fileCount -ne $ExpectedFileCount -or
[int64]$parsed.backupScriptFileCount -ne 17 -or
-not [bool]$parsed.backupHealthExporterIncluded
) {
throw "host110_${ExpectedMode}_receipt_contract_failed"
}
return $parsed
}
function Assert-Agent99ApplyResultFields {
param([object]$Parsed, [string]$VerifierKind)
Assert-Agent99JsonFields $Parsed @{
status = "String"
payloadCommitted = "Boolean"
rollbackAttempted = "Boolean"
rollbackPerformed = "Boolean"
rollbackVerified = "Boolean"
zeroResidueVerified = "Boolean"
zeroResidueScope = "String"
stageCleanupVerified = "Boolean"
rollbackPrestateCleanupVerified = "Boolean"
receipt = "String"
terminalReceipt = "String"
terminalReceiptPublished = "Boolean"
terminalExitCode = "Integer"
deferredSignal = "NullOrString"
independentVerifierVerified = "Boolean"
verifier = $VerifierKind
} "host110_apply_receipt_required_field_failed"
}
function Assert-Agent99VerifierFields {
param([object]$Verifier)
Assert-Agent99JsonFields $Verifier @{
schemaVersion = "String"
ok = "Boolean"
sourceRevision = "String"
runId = "String"
fileCount = "Integer"
backupScriptFileCount = "Integer"
backupHealthExporterIncluded = "Boolean"
remoteWritePerformed = "Boolean"
selfIdentityVerified = "Boolean"
} "host110_apply_verifier_required_field_failed"
}
function Get-Agent99ExecutorResult {
param([object]$Transport, [string]$ExpectedMode)
$parsed = Convert-Agent99ExecutorDocument $Transport $ExpectedMode
if (-not [bool]$parsed.ok) { throw "host110_${ExpectedMode}_receipt_contract_failed" }
if ($ExpectedMode -eq "Apply") {
Assert-Agent99ApplyResultFields $parsed "Object"
Assert-Agent99VerifierFields $parsed.verifier
if (
[string]$parsed.status -ne "verified" -or
-not [bool]$parsed.payloadCommitted -or
[bool]$parsed.rollbackAttempted -or
[bool]$parsed.rollbackPerformed -or
[bool]$parsed.rollbackVerified -or
-not [bool]$parsed.zeroResidueVerified -or
[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps" -or
-not [bool]$parsed.stageCleanupVerified -or
-not [bool]$parsed.rollbackPrestateCleanupVerified -or
[int64]$parsed.terminalExitCode -ne 0 -or
$null -ne $parsed.deferredSignal -or
-not [bool]$parsed.independentVerifierVerified -or
[string]$parsed.receipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or
[string]$parsed.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json" -or
-not [bool]$parsed.terminalReceiptPublished -or
[string]$parsed.verifier.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
-not [bool]$parsed.verifier.ok -or
[string]$parsed.verifier.sourceRevision -ne $SourceRevision -or
[string]$parsed.verifier.runId -ne $RunId -or
[int64]$parsed.verifier.fileCount -ne $ExpectedFileCount -or
[int64]$parsed.verifier.backupScriptFileCount -ne 17 -or
-not [bool]$parsed.verifier.backupHealthExporterIncluded -or
[bool]$parsed.verifier.remoteWritePerformed -or
-not [bool]$parsed.verifier.selfIdentityVerified
) {
@@ -301,6 +428,66 @@ function Get-Agent99ExecutorResult {
return $parsed
}
function Get-Agent99CommittedFailureResult {
param([object]$Transport)
$parsed = Convert-Agent99ExecutorDocument $Transport "Apply" -AllowProcessFailure
Assert-Agent99ApplyResultFields $parsed "Null"
if (
[bool]$parsed.ok -or
-not [bool]$parsed.payloadCommitted -or
[bool]$parsed.rollbackAttempted -or
[bool]$parsed.rollbackPerformed -or
[bool]$parsed.rollbackVerified -or
-not [bool]$parsed.zeroResidueVerified -or
[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps" -or
-not [bool]$parsed.independentVerifierVerified -or
[string]$parsed.receipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or
[string]$parsed.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json"
) {
throw "host110_apply_committed_failure_contract_failed"
}
$signalExitCodes = @{ INT = 130; HUP = 129; TERM = 143 }
if ([string]$parsed.status -eq "cleanup_pending") {
if (
[int]$Transport.exitCode -ne 75 -or
[int64]$parsed.terminalExitCode -ne 75 -or
([bool]$parsed.stageCleanupVerified -and [bool]$parsed.rollbackPrestateCleanupVerified) -or
($null -ne $parsed.deferredSignal -and -not $signalExitCodes.ContainsKey([string]$parsed.deferredSignal))
) {
throw "host110_apply_cleanup_pending_contract_failed"
}
if (-not [bool]$parsed.terminalReceiptPublished) {
throw "host110_apply_cleanup_pending_terminal_receipt_missing"
}
} elseif ([string]$parsed.status -eq "committed_signal_deferred") {
$signalName = [string]$parsed.deferredSignal
if (
-not $signalExitCodes.ContainsKey($signalName) -or
-not [bool]$parsed.stageCleanupVerified -or
-not [bool]$parsed.rollbackPrestateCleanupVerified -or
[int]$Transport.exitCode -ne [int]$signalExitCodes[$signalName] -or
[int64]$parsed.terminalExitCode -ne [int]$signalExitCodes[$signalName]
) {
throw "host110_apply_deferred_signal_contract_failed"
}
if (-not [bool]$parsed.terminalReceiptPublished) {
throw "host110_apply_deferred_signal_terminal_receipt_missing"
}
} elseif ([string]$parsed.status -eq "terminal_receipt_failed") {
if (
[bool]$parsed.terminalReceiptPublished -or
[int]$Transport.exitCode -ne 76 -or
[int64]$parsed.terminalExitCode -ne 76
) {
throw "host110_apply_terminal_receipt_failure_contract_failed"
}
} else {
throw "host110_apply_committed_failure_status_invalid"
}
return $parsed
}
function Get-Agent99VerifierResult {
param([object]$Transport, [object]$SourcePackage)
@@ -310,6 +497,17 @@ function Get-Agent99VerifierResult {
} catch {
throw "host110_verify_receipt_invalid"
}
Assert-Agent99JsonFields $parsed @{
schemaVersion = "String"
ok = "Boolean"
sourceRevision = "String"
runId = "String"
fileCount = "Integer"
backupScriptFileCount = "Integer"
backupHealthExporterIncluded = "Boolean"
remoteWritePerformed = "Boolean"
verifierSha256 = "String"
} "host110_verify_receipt_required_field_failed"
if (
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
-not [bool]$parsed.ok -or
@@ -421,7 +619,12 @@ try {
$executorPath = "$remoteStage/host110-backup-runtime-executor.sh"
$command = "timeout --signal=TERM --kill-after=10s 300s $executorPath --mode apply --source-revision $SourceRevision --run-id $RunId --source-stage $remoteStage"
$applyTransport = Invoke-Agent99BoundedSsh $command 330
$apply = Get-Agent99ExecutorResult $applyTransport "Apply"
if ($applyTransport.ok) {
$apply = Get-Agent99ExecutorResult $applyTransport "Apply"
} else {
$apply = Get-Agent99CommittedFailureResult $applyTransport
throw "host110_apply_$([string]$apply.status)"
}
$verify = $apply.verifier
$cleanupVerified = Remove-Agent99RemoteStage $remoteStage $true
$remoteStageCreated = $false
@@ -445,7 +648,7 @@ try {
}
$result = [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v2"
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v3"
ok = [bool]($terminalStatus -ne "failed")
status = $terminalStatus
mode = $Mode

View File

@@ -0,0 +1,217 @@
# AWOOOI Agent99 Host110 Backup Runtime V10
Status: proposed; central review and explicit owner approval are required before
any production apply.
This artifact supersedes V2 through V9. Every V9 identity below is void for
approval and remains audit evidence only:
- branch/ref: `codex/host110-backup-runtime-v9-20260718`
- candidate/revision: `8f11ef3362a8d7f66a46308b58fd142b0c91a8bc`
- artifact SHA-256: `929978eb3da15b04791a8614e6d5ee504ae2a4c2f205b694307a7bad7713bcad`
- exact diff SHA-256: `8085d49450b74e7c43577ba118cbd74d3a2e3dd2d22eeba90ce749d56c72c0f2`
- every other V9 proposal, content, revision, and derived hash
No V9 hash, ref, review, receipt, or test grants production execution
authority.
## Exact Scope
- Agent99 runtime bundle: 19 files, guarded by one producer/consumer parity
test.
- Host110 backup payload: 18 ordered files, comprising 17 backup scripts and
`backup-health-textfile-exporter.py`.
- Host110 remote stage: 21 unique basenames, comprising the ordered 18-file
payload, fixed executor, independent verifier, and `manifest.json`.
- Fixed target: `wooo@192.168.0.110`, dispatched by Windows99 Agent99 from an
exact Gitea revision and digest manifest.
- The candidate includes a 10-line runtime-lock addition to
`scripts/backup/gitea-full-backup-restore-drill.sh`. A future drill invocation
acquires the shared Host110 backup-runtime lock before its pre-existing drill
behavior. Applying this candidate replaces the script but does not invoke the
drill or any container lifecycle.
- `scripts/backup/backup-gitea.sh` has zero diff from the production base. No
Gitea primary stop/start or backup-container creation is part of this
candidate apply.
- V7, V8, and V9 proposal documents remain tracked audit artifacts and are not
runtime authority.
- `agent99-host110-broker-contract-replay.ps1` is a no-write Windows99 test
asset; it is not part of the 19-file Agent99 runtime or 18-file Host110
payload.
## V9 Defects Closed
### Signal-Safe Commit Boundary
The executor enters a signal-deferring critical section before publishing the
payload receipt. `INT`, `HUP`, and `TERM` are recorded without exiting until the
durable payload-receipt readback and `APPLY_COMPLETE=1` commit state are both
established. A deferred signal can never enter the pre-commit EXIT rollback
path after the payload receipt exists.
After commit, exact stage and rollback-prestate cleanup still run. If cleanup
is complete, a deferred signal produces the non-success terminal
`committed_signal_deferred`, a durable terminal receipt, the corresponding
129/130/143 exit, and an applied payload. TERM, HUP, and INT are replayed in the
exact receipt-to-commit window.
After cleanup, terminal publication uses a bounded signal mask. Signals captured
before that boundary remain part of the non-success terminal; signals delivered
during the immutable terminal writer cannot mutate its already-final state.
Terminal-writer TERM, HUP, and INT replays require receipt, stdout, payload, and
exit to remain consistent.
### Payload And Terminal Receipts
The immutable payload receipt has status `payload_verified`; it proves exact
payload, independent verifier, durable receipt readback, and absence of
run-owned destination temporaries. It does not claim internal stage or rollback
prestate cleanup.
A separate immutable terminal receipt is the authority for overall success.
Only status `verified` with `stageCleanupVerified=true`,
`rollbackPrestateCleanupVerified=true`, no deferred signal, and exit code zero
may yield executor `ok:true`. Each successful cleanup is absence-read back and
its parent directory is fsynced before terminal publication.
If either cleanup fails, the payload remains committed, exact residue is
preserved, terminal status is `cleanup_pending`, executor output is
`ok:false`, and exit is nonzero. A payload receipt cannot be used as a
zero-residue or broker-success shortcut. The Windows99 broker validates every
status, payload, rollback, residue-scope, cleanup, signal, exit, receipt-path,
and independent-verifier field fail-closed.
Committed non-success outputs are parsed and retained in broker evidence before
the broker fails overall; `cleanup_pending`, `committed_signal_deferred`, and
`terminal_receipt_failed` can never be collapsed into an ambiguous generic
transport failure.
### Existing Transaction Guarantees Retained
- Producer and executor payload basenames are exact-equal in count, set, and
order: 18 payload files and 21 unique staged files.
- Receipt write, hard-link, fsync, and readback failures roll back exact content
and metadata without publishing success.
- Terminal-receipt write, hard-link, fsync, and readback failures keep the
verified payload, publish no terminal success, emit structured committed
failure evidence with `terminalReceiptPublished=false`, and exit nonzero.
- Apply and rollback destination temporaries are run-owned, tracked, removed,
and absence-read back.
- Rollback attempted, performed, verified, and residue truth remain separate;
partial or unknown rollback terminates `rollback_unverified`.
- Fault hooks require both the explicit test flag and a transformed
non-production destination; canonical `/backup/scripts` cannot enable them.
## Apply Effects
- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded
manifest and receipt state.
- Host110 Apply replaces the ordered 18 payload paths through one exclusive,
digest-bound filesystem transaction, and writes payload/terminal receipts
below `/backup/status`.
- The deployed restore-drill script gains shared-lock behavior. The apply does
not execute that script.
- Apply does not run a backup, invoke a Gitea container lifecycle, sync Google
Drive, prune snapshots, restart a VM or service, or change a database,
network, firewall, credential, or secret.
## Reproducible Validation
Focused replay and contract suite:
```sh
pytest -q \
scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \
scripts/backup/tests/test_verify_offsite_full_sync_runtime.py \
scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py \
scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py \
scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py \
scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py \
scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py
```
Exact signal and post-commit cleanup replays:
```sh
pytest -vv \
scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \
-k 'commit_window_signal or terminal_publication or terminal_receipt_fault or postcommit_cleanup'
```
Static, syntax, and repository checks:
```sh
ruff check scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py
bash -n scripts/backup/host110-backup-runtime-executor.sh
git diff --check
PATH=/Users/ogt/.pyenv/shims:$PATH bash scripts/ops/ansible-validate.sh
```
`ansible-validate.sh` parses
`ops/reboot-recovery/full-stack-backup-baseline.yml` as YAML metadata. It runs
`ansible-playbook --syntax-check` only for the actual playbooks listed by the
script; the metadata mapping is not represented as a playbook.
Windows99 no-file-write PowerShell parser check:
```sh
PARSER_COMMAND='$source=[Console]::In.ReadToEnd(); $tokens=$null; $errors=$null; [System.Management.Automation.Language.Parser]::ParseInput($source,[ref]$tokens,[ref]$errors) | Out-Null; [pscustomobject]@{ok=($errors.Count -eq 0); errorCount=$errors.Count; parser="WindowsPowerShell"} | ConvertTo-Json -Compress; if ($errors.Count -ne 0) { exit 1 }'
ENCODED=$(printf '%s' "$PARSER_COMMAND" | iconv -f UTF-8 -t UTF-16LE | base64 | tr -d '\n')
ssh -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=5 \
-o ConnectionAttempts=1 Administrator@192.168.0.99 \
"powershell.exe -NoProfile -NonInteractive -EncodedCommand $ENCODED" \
< agent99-host110-backup-runtime-broker.ps1
```
Windows99 actual-function missing-field/type contract replay:
```sh
WRAPPER='$script=[Console]::In.ReadToEnd(); & ([ScriptBlock]::Create($script))'
WRAPPER_ENCODED=$(printf '%s' "$WRAPPER" | iconv -f UTF-8 -t UTF-16LE | base64 | tr -d '\n')
BROKER_GZIP_BASE64=$(gzip -c agent99-host110-backup-runtime-broker.ps1 | base64 | tr -d '\n')
awk -v payload="$BROKER_GZIP_BASE64" \
'BEGIN { print "$BrokerGzipBase64 = \"" payload "\"" } { print }' \
scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1 |
ssh -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=5 \
-o ConnectionAttempts=1 Administrator@192.168.0.99 \
"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand $WRAPPER_ENCODED"
```
The replay loads the candidate broker's actual function AST, accepts the valid
success and three committed non-success terminals, then removes every required
field and injects wrong JSON types. Expected readback is 34 missing-field and 6
type rejections. It performs no remote file write.
## Exact Diff Serialization
Substitute the V10 Gitea live-ref SHA for `<candidate_git_sha>` and hash the
exact stdout bytes:
```sh
git diff --binary --full-index \
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
shasum -a 256
```
Central review must independently bind the live feature ref and candidate Git
SHA, this tracked artifact path and bytes hash, the serialized diff hash/path
count/line counts, the 19/18/21 count contracts, and the exact restore-drill and
`backup-gitea.sh` diffs.
## Rollback
- Source rollback reverts the exact V10 diff to
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
- Agent99 rollback restores the prior runtime files and manifest and verifies
prior hashes independently.
- Before payload commit, Host110 rollback restores all prior payload content,
SHA-256, uid, gid, and mode under the exclusive lock and proves no run-owned
destination temporary remains.
- After payload commit, the executor never rolls the payload back merely due to
a signal or cleanup error. It preserves exact cleanup evidence and reports a
non-success terminal. Any later cleanup-only remediation is a separate
controlled action scoped to the run-specific stage/prestate paths and may not
rewrite the verified payload.
- Source rollback restores the restore-drill script's prior unlocked behavior.
It does not invoke the drill or any container lifecycle.
- Any canonical, metadata, receipt, signal, cleanup, or residue mismatch fails
closed and cannot produce broker `verified`.

View File

@@ -45,8 +45,12 @@ VERIFIER_DIGEST=""
STAGE_DIR=""
ROLLBACK_DIR=""
RECEIPT_PATH=""
TERMINAL_RECEIPT_PATH=""
APPLY_STARTED=0
APPLY_COMPLETE=0
COMMIT_CRITICAL=0
DEFERRED_SIGNAL=""
DEFERRED_SIGNAL_EXIT=0
ROLLBACK_ATTEMPTED=0
ROLLBACK_PERFORMED=0
ROLLBACK_VERIFIED=0
@@ -103,10 +107,17 @@ case "$MODE" in check|apply|verify) ;; *) fail "invalid_mode" ;; esac
expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"
[ "$SOURCE_STAGE" = "$expected_source_stage" ] || fail "invalid_source_stage"
RECEIPT_PATH="$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json"
TERMINAL_RECEIPT_PATH="$STATUS_ROOT/agent99-backup-runtime-terminal-${RUN_ID}.json"
if [ "${HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS:-}" = "1" ] \
&& [ "$DEST_ROOT" != "/backup/scripts" ]; then
FAULT_INJECTION_ENABLED=1
fi
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then
case "${HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL:-}" in ""|INT|HUP|TERM) ;; *) fail "invalid_test_commit_signal" ;; esac
case "${HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL:-}" in ""|INT|HUP|TERM) ;; *) fail "invalid_test_terminal_signal" ;; esac
case "${HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT:-}" in ""|write|link|fsync|readback) ;; *) fail "invalid_test_terminal_receipt_fault" ;; esac
case "${HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT:-}" in ""|stage|rollback_prestate) ;; *) fail "invalid_test_cleanup_fault" ;; esac
fi
for command_name in awk bash cp dirname flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do
command -v "$command_name" >/dev/null 2>&1 || fail "required_command_missing_${command_name}"
@@ -314,9 +325,9 @@ rollback_attempted = sys.argv[6] == "1"
rollback_performed = sys.argv[7] == "1"
rollback_verified = sys.argv[8] == "1"
zero_residue_verified = sys.argv[9] == "1"
if status not in {"verified", "failed_rolled_back", "rollback_unverified"}:
if status not in {"payload_verified", "failed_rolled_back", "rollback_unverified"}:
raise SystemExit(78)
if status == "verified" and (rollback_attempted or rollback_performed or rollback_verified or not zero_residue_verified):
if status == "payload_verified" and (rollback_attempted or rollback_performed or rollback_verified or not zero_residue_verified):
raise SystemExit(78)
if status == "failed_rolled_back" and not (
rollback_attempted and rollback_performed and rollback_verified and zero_residue_verified
@@ -325,7 +336,7 @@ if status == "failed_rolled_back" and not (
if status == "rollback_unverified" and (not rollback_attempted or rollback_verified):
raise SystemExit(78)
document = {
"schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v2",
"schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v3",
"status": status,
"sourceRevision": sys.argv[3],
"sourceHead": sys.argv[4],
@@ -337,6 +348,7 @@ document = {
"rollbackPerformed": rollback_performed,
"rollbackVerified": rollback_verified,
"zeroResidueVerified": zero_residue_verified,
"zeroResidueScope": "run_owned_destination_temps",
"verifierSha256": sys.argv[10],
"executorHost": "192.168.0.110",
"productionServiceRestarted": False,
@@ -349,7 +361,7 @@ temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
fault_enabled = sys.argv[11] == "1"
fault = (
os.environ.get("HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT", "")
if status == "verified" and fault_enabled
if status == "payload_verified" and fault_enabled
else ""
)
if fault not in {"", "write", "link", "fsync", "readback"}:
@@ -412,6 +424,172 @@ PY
return 0
}
write_terminal_receipt() {
local terminal_status="$1"
local stage_cleanup_verified="$2"
local rollback_prestate_cleanup_verified="$3"
local deferred_signal="$4"
local terminal_exit_code="$5"
install -d -m 700 "$STATUS_ROOT" || return 1
[ -d "$STATUS_ROOT" ] && [ ! -L "$STATUS_ROOT" ] || return 1
[ "$(stat -c '%U:%G:%a' "$STATUS_ROOT")" = "wooo:wooo:700" ] || return 1
if ! python3 - "$TERMINAL_RECEIPT_PATH" "$RECEIPT_PATH" "$terminal_status" \
"$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$stage_cleanup_verified" \
"$rollback_prestate_cleanup_verified" "$deferred_signal" "$terminal_exit_code" \
"$VERIFIER_DIGEST" "$FAULT_INJECTION_ENABLED" <<'PY'
import hashlib
import json
import os
import signal
import sys
import time
from pathlib import Path
path = Path(sys.argv[1])
payload_receipt_path = Path(sys.argv[2])
status = sys.argv[3]
source_revision = sys.argv[4]
source_head = sys.argv[5]
run_id = sys.argv[6]
stage_cleanup_verified = sys.argv[7] == "1"
rollback_prestate_cleanup_verified = sys.argv[8] == "1"
deferred_signal = sys.argv[9]
terminal_exit_code = int(sys.argv[10])
verifier_digest = sys.argv[11]
fault_enabled = sys.argv[12] == "1"
signal_exit_codes = {"INT": 130, "HUP": 129, "TERM": 143}
terminal_signal = os.environ.get("HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL", "") if fault_enabled else ""
terminal_fault = os.environ.get("HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT", "") if fault_enabled else ""
if terminal_signal:
os.kill(os.getppid(), getattr(signal, f"SIG{terminal_signal}"))
time.sleep(0.05)
try:
payload_receipt = json.loads(payload_receipt_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
raise SystemExit(79)
if not (
payload_receipt.get("schemaVersion") == "agent99_host110_backup_runtime_deploy_receipt_v3"
and payload_receipt.get("status") == "payload_verified"
and payload_receipt.get("sourceRevision") == source_revision
and payload_receipt.get("sourceHead") == source_head
and payload_receipt.get("runId") == run_id
and payload_receipt.get("zeroResidueVerified") is True
):
raise SystemExit(79)
if status == "verified":
valid = (
stage_cleanup_verified
and rollback_prestate_cleanup_verified
and not deferred_signal
and terminal_exit_code == 0
)
elif status == "cleanup_pending":
valid = (
not (stage_cleanup_verified and rollback_prestate_cleanup_verified)
and deferred_signal in {"", *signal_exit_codes}
and terminal_exit_code != 0
)
elif status == "committed_signal_deferred":
valid = (
stage_cleanup_verified
and rollback_prestate_cleanup_verified
and deferred_signal in signal_exit_codes
and terminal_exit_code == signal_exit_codes[deferred_signal]
)
else:
valid = False
if not valid:
raise SystemExit(79)
document = {
"schemaVersion": "agent99_host110_backup_runtime_terminal_receipt_v1",
"status": status,
"ok": status == "verified",
"payloadCommitted": True,
"payloadReceipt": str(payload_receipt_path),
"sourceRevision": source_revision,
"sourceHead": source_head,
"runId": run_id,
"fileCount": 18,
"backupScriptFileCount": 17,
"backupHealthExporterIncluded": True,
"stageCleanupVerified": stage_cleanup_verified,
"rollbackPrestateCleanupVerified": rollback_prestate_cleanup_verified,
"cleanupVerified": stage_cleanup_verified and rollback_prestate_cleanup_verified,
"independentVerifierVerified": True,
"zeroResidueScope": "run_owned_destination_temps_and_internal_stage_prestate",
"deferredSignal": deferred_signal or None,
"terminalExitCode": terminal_exit_code,
"verifierSha256": verifier_digest,
"executorHost": "192.168.0.110",
"productionServiceRestarted": False,
"secretValuesRead": False,
"writtenAt": int(time.time()),
}
payload = (json.dumps(document, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")
expected_digest = hashlib.sha256(payload).hexdigest()
temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
created_final = False
directory_fd = None
try:
if os.path.lexists(path) or os.path.lexists(temporary):
raise FileExistsError(path)
if terminal_fault == "write":
raise OSError("fault_injected_terminal_receipt_write")
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "wb") as handle:
if handle.write(payload) != len(payload):
raise OSError("terminal_receipt_short_write")
handle.flush()
if terminal_fault == "fsync":
raise OSError("fault_injected_terminal_receipt_fsync")
os.fsync(handle.fileno())
os.chmod(temporary, 0o600)
temporary_readback = temporary.read_bytes()
if temporary_readback != payload or hashlib.sha256(temporary_readback).hexdigest() != expected_digest:
raise OSError("terminal_receipt_temporary_readback_failed")
if terminal_fault == "link":
raise OSError("fault_injected_terminal_receipt_link")
os.link(temporary, path)
created_final = True
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
directory_fd = os.open(path.parent, directory_flags)
os.fsync(directory_fd)
if terminal_fault == "readback":
raise OSError("fault_injected_terminal_receipt_readback")
readback = path.read_bytes()
if readback != payload or hashlib.sha256(readback).hexdigest() != expected_digest:
raise OSError("terminal_receipt_final_readback_failed")
if json.loads(readback.decode("utf-8")) != document:
raise OSError("terminal_receipt_document_readback_failed")
temporary.unlink()
os.fsync(directory_fd)
except BaseException:
try:
if created_final:
path.unlink(missing_ok=True)
temporary.unlink(missing_ok=True)
if directory_fd is None and path.parent.is_dir():
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
directory_fd = os.open(path.parent, directory_flags)
if directory_fd is not None:
os.fsync(directory_fd)
finally:
raise
finally:
if directory_fd is not None:
os.close(directory_fd)
PY
then
return 1
fi
[ -f "$TERMINAL_RECEIPT_PATH" ] && [ ! -L "$TERMINAL_RECEIPT_PATH" ] || return 1
return 0
}
record_prestate() {
local name dest_path digest metadata
: > "$ROLLBACK_DIR/prestate.tsv" || return 74
@@ -530,6 +708,35 @@ rollback_transaction() {
return 1
}
handle_agent99_signal() {
local signal_name="$1"
local signal_exit_code="$2"
if [ "$COMMIT_CRITICAL" -eq 1 ] || [ "$APPLY_COMPLETE" -eq 1 ]; then
if [ -z "$DEFERRED_SIGNAL" ]; then
DEFERRED_SIGNAL="$signal_name"
DEFERRED_SIGNAL_EXIT="$signal_exit_code"
fi
return 0
fi
exit "$signal_exit_code"
}
cleanup_postcommit_directory() {
local cleanup_kind="$1"
local cleanup_path="$2"
local fault=""
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then
fault="${HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT:-}"
fi
if [ "$fault" = "$cleanup_kind" ]; then
return 1
fi
rm -rf -- "$cleanup_path" || return 1
[ ! -e "$cleanup_path" ] && [ ! -L "$cleanup_path" ] || return 1
fsync_paths_and_parents "$cleanup_path" || return 1
[ ! -e "$cleanup_path" ] && [ ! -L "$cleanup_path" ]
}
cleanup() {
local exit_status=$?
local rollback_status="rollback_unverified"
@@ -544,10 +751,12 @@ cleanup() {
failure_receipt_written=1
fi
fi
if [ -n "$STAGE_DIR" ]; then
if [ "$APPLY_COMPLETE" -ne 1 ] && [ -n "$STAGE_DIR" ]; then
rm -rf -- "$STAGE_DIR" || true
fi
if [ "$ROLLBACK_VERIFIED" -eq 1 ] && [ "$failure_receipt_written" -eq 1 ]; then
if [ "$APPLY_COMPLETE" -ne 1 ] \
&& [ "$ROLLBACK_VERIFIED" -eq 1 ] \
&& [ "$failure_receipt_written" -eq 1 ]; then
if [ -n "$ROLLBACK_DIR" ]; then
rm -rf -- "$ROLLBACK_DIR" || true
fi
@@ -558,17 +767,19 @@ cleanup() {
validate_source || fail "source_validation_failed"
if [ "$MODE" = "check" ]; then
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","runId":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID"
exit 0
fi
if [ "$MODE" = "verify" ]; then
verify_destination || fail "destination_verification_failed"
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","runId":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID"
exit 0
fi
[ ! -e "$RECEIPT_PATH" ] && [ ! -L "$RECEIPT_PATH" ] || fail "run_identity_receipt_exists"
[ ! -e "$TERMINAL_RECEIPT_PATH" ] && [ ! -L "$TERMINAL_RECEIPT_PATH" ] \
|| fail "run_identity_terminal_receipt_exists"
STAGE_DIR="$STAGE_ROOT/$RUN_ID"
ROLLBACK_DIR="$ROLLBACK_ROOT/$RUN_ID"
[ ! -e "$STAGE_DIR" ] || fail "run_identity_stage_exists"
@@ -589,8 +800,9 @@ fi
install -d -m 700 "$STAGE_DIR" "$ROLLBACK_DIR"
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM HUP
trap 'handle_agent99_signal INT 130' INT
trap 'handle_agent99_signal HUP 129' HUP
trap 'handle_agent99_signal TERM 143' TERM
for name in "${PAYLOAD_FILES[@]}"; do
install -m 700 "$SOURCE_STAGE/$name" "$STAGE_DIR/$name"
@@ -651,43 +863,116 @@ PY
verify_run_owned_temp_absence || fail "post_apply_temp_residue_detected"
RUN_TEMP_RESIDUE_VERIFIED=1
if ! write_receipt "verified" 0 0 0 "$RUN_TEMP_RESIDUE_VERIFIED"; then
COMMIT_CRITICAL=1
if ! write_receipt "payload_verified" 0 0 0 "$RUN_TEMP_RESIDUE_VERIFIED"; then
COMMIT_CRITICAL=0
fail "durable_receipt_failed"
fi
receipt_path="$RECEIPT_PATH"
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ] \
&& [ -n "${HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL:-}" ]; then
case "$HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL" in
INT|HUP|TERM) kill -s "$HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL" "$$" ;;
esac
fi
APPLY_COMPLETE=1
trap - EXIT HUP INT TERM
COMMIT_CRITICAL=0
stage_cleanup_verified=0
rollback_prestate_cleanup_verified=0
if rm -rf -- "$STAGE_DIR" && [ ! -e "$STAGE_DIR" ] && [ ! -L "$STAGE_DIR" ]; then
if cleanup_postcommit_directory "stage" "$STAGE_DIR"; then
stage_cleanup_verified=1
fi
if rm -rf -- "$ROLLBACK_DIR" && [ ! -e "$ROLLBACK_DIR" ] && [ ! -L "$ROLLBACK_DIR" ]; then
if cleanup_postcommit_directory "rollback_prestate" "$ROLLBACK_DIR"; then
rollback_prestate_cleanup_verified=1
fi
STAGE_DIR=""
ROLLBACK_DIR=""
python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$receipt_path" "$verifier_result" \
"$stage_cleanup_verified" "$rollback_prestate_cleanup_verified" <<'PY'
terminal_status="verified"
terminal_exit_code=0
# Cleanup has completed. Freeze terminal signal state before the immutable
# terminal writer so its receipt, stdout, and exit status cannot diverge.
trap '' HUP INT TERM
if [ "$stage_cleanup_verified" -ne 1 ] || [ "$rollback_prestate_cleanup_verified" -ne 1 ]; then
terminal_status="cleanup_pending"
terminal_exit_code=75
elif [ -n "$DEFERRED_SIGNAL" ]; then
terminal_status="committed_signal_deferred"
terminal_exit_code="$DEFERRED_SIGNAL_EXIT"
fi
if ! write_terminal_receipt "$terminal_status" "$stage_cleanup_verified" \
"$rollback_prestate_cleanup_verified" "$DEFERRED_SIGNAL" "$terminal_exit_code"; then
trap - EXIT
python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$RECEIPT_PATH" \
"$TERMINAL_RECEIPT_PATH" "$stage_cleanup_verified" \
"$rollback_prestate_cleanup_verified" "$DEFERRED_SIGNAL" <<'PY'
import json
import sys
print(json.dumps({
"schemaVersion": "agent99_host110_backup_runtime_executor_v1",
"mode": "apply",
"ok": True,
"status": "terminal_receipt_failed",
"ok": False,
"sourceRevision": sys.argv[1],
"sourceHead": sys.argv[2],
"runId": sys.argv[3],
"fileCount": 18,
"backupScriptFileCount": 17,
"backupHealthExporterIncluded": True,
"payloadCommitted": True,
"rollbackAttempted": False,
"rollbackPerformed": False,
"rollbackVerified": False,
"zeroResidueVerified": True,
"stageCleanupVerified": sys.argv[5] == "1",
"rollbackPrestateCleanupVerified": sys.argv[6] == "1",
"receipt": sys.argv[3],
"verifier": json.loads(sys.argv[4]),
"zeroResidueScope": "run_owned_destination_temps",
"stageCleanupVerified": sys.argv[6] == "1",
"rollbackPrestateCleanupVerified": sys.argv[7] == "1",
"receipt": sys.argv[4],
"terminalReceipt": sys.argv[5],
"terminalReceiptPublished": False,
"terminalExitCode": 76,
"deferredSignal": sys.argv[8] or None,
"independentVerifierVerified": True,
"verifier": None,
}, ensure_ascii=True, sort_keys=True))
PY
printf 'HOST110_BACKUP_RUNTIME_OK=0\nERROR=terminal_receipt_failed\n' >&2
exit 76
fi
if [ "$stage_cleanup_verified" -eq 1 ]; then STAGE_DIR=""; fi
if [ "$rollback_prestate_cleanup_verified" -eq 1 ]; then ROLLBACK_DIR=""; fi
trap - EXIT
python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$RECEIPT_PATH" "$TERMINAL_RECEIPT_PATH" \
"$verifier_result" "$stage_cleanup_verified" "$rollback_prestate_cleanup_verified" \
"$terminal_status" "$terminal_exit_code" "$DEFERRED_SIGNAL" "$RUN_ID" <<'PY'
import json
import sys
status = sys.argv[8]
verifier = json.loads(sys.argv[5])
print(json.dumps({
"schemaVersion": "agent99_host110_backup_runtime_executor_v1",
"mode": "apply",
"status": status,
"ok": status == "verified",
"sourceRevision": sys.argv[1],
"sourceHead": sys.argv[2],
"runId": sys.argv[11],
"fileCount": 18,
"backupScriptFileCount": 17,
"backupHealthExporterIncluded": True,
"payloadCommitted": True,
"rollbackAttempted": False,
"rollbackPerformed": False,
"rollbackVerified": False,
"zeroResidueVerified": True,
"zeroResidueScope": "run_owned_destination_temps",
"stageCleanupVerified": sys.argv[6] == "1",
"rollbackPrestateCleanupVerified": sys.argv[7] == "1",
"receipt": sys.argv[3],
"terminalReceipt": sys.argv[4],
"terminalReceiptPublished": True,
"terminalExitCode": int(sys.argv[9]),
"deferredSignal": sys.argv[10] or None,
"independentVerifierVerified": True,
"verifier": verifier if status == "verified" else None,
}, ensure_ascii=True, sort_keys=True))
PY
exit "$terminal_exit_code"

View File

@@ -0,0 +1,220 @@
$ErrorActionPreference = "Stop"
function Write-ReplayTrace {
param([string]$Step)
if ($ReplayTrace) { Write-Output "REPLAY_STEP=$Step" }
}
if (-not $BrokerGzipBase64) { throw "broker_payload_missing" }
Write-ReplayTrace "decode_source"
$compressed = [Convert]::FromBase64String([string]$BrokerGzipBase64)
$inputStream = New-Object IO.MemoryStream(,$compressed)
$gzip = New-Object IO.Compression.GzipStream(
$inputStream,
[IO.Compression.CompressionMode]::Decompress
)
$reader = New-Object IO.StreamReader($gzip, [Text.Encoding]::UTF8)
try {
$brokerSource = $reader.ReadToEnd()
} finally {
$reader.Dispose()
$gzip.Dispose()
$inputStream.Dispose()
}
$tokens = $null
$parseErrors = $null
$ast = [Management.Automation.Language.Parser]::ParseInput(
$brokerSource,
[ref]$tokens,
[ref]$parseErrors
)
if ($parseErrors.Count -ne 0) { throw "broker_parse_failed" }
Write-ReplayTrace "define_functions"
$functionNames = @(
"Test-Agent99JsonField",
"Assert-Agent99JsonFields",
"Convert-Agent99ExecutorDocument",
"Assert-Agent99ApplyResultFields",
"Assert-Agent99VerifierFields",
"Get-Agent99ExecutorResult",
"Get-Agent99CommittedFailureResult"
)
foreach ($functionName in $functionNames) {
$node = $ast.Find(
{
param($candidate)
$candidate -is [Management.Automation.Language.FunctionDefinitionAst] -and
$candidate.Name -eq $functionName
},
$true
)
if ($null -eq $node) { throw "broker_function_missing_$functionName" }
Invoke-Expression $node.Extent.Text
}
$SourceRevision = "a" * 40
$RunId = "windows99-contract-replay"
$ExpectedFileCount = 18
$sourceHead = "b" * 40
$payloadReceipt = "/backup/status/agent99-backup-runtime-deploy-$RunId.json"
$terminalReceipt = "/backup/status/agent99-backup-runtime-terminal-$RunId.json"
function Copy-ReplayDocument {
param([object]$Document)
return ($Document | ConvertTo-Json -Compress -Depth 12 | ConvertFrom-Json)
}
function New-ReplayTransport {
param([object]$Document, [int]$ExitCode)
return [pscustomobject]@{
ok = [bool]($ExitCode -eq 0)
exitCode = $ExitCode
reason = if ($ExitCode -eq 0) { "completed" } else { "process_failed" }
stdout = $Document | ConvertTo-Json -Compress -Depth 12
stderrPresent = $false
}
}
function Assert-Rejected {
param([scriptblock]$Operation, [string]$Name)
$accepted = $false
try {
& $Operation | Out-Null
$accepted = $true
} catch {}
if ($accepted) { throw "contract_mutation_accepted_$Name" }
}
$verifier = [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_verifier_v1"
ok = $true
sourceRevision = $SourceRevision
runId = $RunId
fileCount = 18
backupScriptFileCount = 17
backupHealthExporterIncluded = $true
remoteWritePerformed = $false
selfIdentityVerified = $true
}
$success = [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_executor_v1"
mode = "apply"
status = "verified"
ok = $true
sourceRevision = $SourceRevision
sourceHead = $sourceHead
runId = $RunId
fileCount = 18
backupScriptFileCount = 17
backupHealthExporterIncluded = $true
payloadCommitted = $true
rollbackAttempted = $false
rollbackPerformed = $false
rollbackVerified = $false
zeroResidueVerified = $true
zeroResidueScope = "run_owned_destination_temps"
stageCleanupVerified = $true
rollbackPrestateCleanupVerified = $true
receipt = $payloadReceipt
terminalReceipt = $terminalReceipt
terminalReceiptPublished = $true
terminalExitCode = 0
deferredSignal = $null
independentVerifierVerified = $true
verifier = $verifier
}
Write-ReplayTrace "accept_success"
Get-Agent99ExecutorResult (New-ReplayTransport $success 0) "Apply" | Out-Null
$topLevelRequired = @(
"schemaVersion", "mode", "status", "ok", "sourceRevision", "sourceHead",
"runId", "fileCount", "backupScriptFileCount", "backupHealthExporterIncluded",
"payloadCommitted", "rollbackAttempted", "rollbackPerformed", "rollbackVerified",
"zeroResidueVerified", "zeroResidueScope", "stageCleanupVerified",
"rollbackPrestateCleanupVerified", "receipt", "terminalReceipt", "terminalExitCode",
"terminalReceiptPublished", "deferredSignal", "independentVerifierVerified", "verifier"
)
$verifierRequired = @(
"schemaVersion", "ok", "sourceRevision", "runId", "fileCount",
"backupScriptFileCount", "backupHealthExporterIncluded", "remoteWritePerformed",
"selfIdentityVerified"
)
$missingFieldRejections = 0
Write-ReplayTrace "reject_missing_top"
foreach ($field in $topLevelRequired) {
$mutated = Copy-ReplayDocument $success
$mutated.PSObject.Properties.Remove($field)
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_$field"
$missingFieldRejections++
}
Write-ReplayTrace "reject_missing_verifier"
foreach ($field in $verifierRequired) {
$mutated = Copy-ReplayDocument $success
$mutated.verifier.PSObject.Properties.Remove($field)
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_verifier_$field"
$missingFieldRejections++
}
$typeMutations = @(
@{ field = "rollbackAttempted"; value = "false" },
@{ field = "rollbackPerformed"; value = "false" },
@{ field = "rollbackVerified"; value = "false" },
@{ field = "terminalExitCode"; value = "0" },
@{ field = "deferredSignal"; value = 0 }
)
$typeRejections = 0
Write-ReplayTrace "reject_types"
foreach ($mutation in $typeMutations) {
$mutated = Copy-ReplayDocument $success
$mutated.($mutation.field) = $mutation.value
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_$($mutation.field)"
$typeRejections++
}
$mutated = Copy-ReplayDocument $success
$mutated.verifier.remoteWritePerformed = "false"
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_verifier_remoteWritePerformed"
$typeRejections++
$cleanupPending = Copy-ReplayDocument $success
$cleanupPending.ok = $false
$cleanupPending.status = "cleanup_pending"
$cleanupPending.stageCleanupVerified = $false
$cleanupPending.terminalExitCode = 75
$cleanupPending.verifier = $null
Write-ReplayTrace "accept_cleanup_pending"
Get-Agent99CommittedFailureResult (New-ReplayTransport $cleanupPending 75) | Out-Null
$signalDeferred = Copy-ReplayDocument $success
$signalDeferred.ok = $false
$signalDeferred.status = "committed_signal_deferred"
$signalDeferred.terminalExitCode = 143
$signalDeferred.deferredSignal = "TERM"
$signalDeferred.verifier = $null
Write-ReplayTrace "accept_signal_deferred"
Get-Agent99CommittedFailureResult (New-ReplayTransport $signalDeferred 143) | Out-Null
$terminalReceiptFailed = Copy-ReplayDocument $success
$terminalReceiptFailed.ok = $false
$terminalReceiptFailed.status = "terminal_receipt_failed"
$terminalReceiptFailed.terminalReceiptPublished = $false
$terminalReceiptFailed.terminalExitCode = 76
$terminalReceiptFailed.verifier = $null
Write-ReplayTrace "accept_terminal_receipt_failed"
Get-Agent99CommittedFailureResult (New-ReplayTransport $terminalReceiptFailed 76) | Out-Null
Write-ReplayTrace "complete"
[pscustomobject]@{
schemaVersion = "agent99_host110_broker_contract_replay_v1"
ok = $true
parser = "WindowsPowerShell"
successAccepted = $true
cleanupPendingPreserved = $true
committedSignalPreserved = $true
terminalReceiptFailurePreserved = $true
missingFieldRejections = $missingFieldRejections
typeRejections = $typeRejections
remoteWritePerformed = $false
secretValuesRead = $false
} | ConvertTo-Json -Compress

View File

@@ -20,6 +20,9 @@ ROOT = Path(__file__).resolve().parents[3]
BROKER = ROOT / "agent99-host110-backup-runtime-broker.ps1"
EXECUTOR = ROOT / "scripts" / "backup" / "host110-backup-runtime-executor.sh"
VERIFIER = ROOT / "scripts" / "backup" / "verify-host110-backup-runtime.py"
BROKER_CONTRACT_REPLAY = (
ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-host110-broker-contract-replay.ps1"
)
EXPECTED_RUNTIME_FILES = {
@@ -71,6 +74,12 @@ def _powershell_array(source: str, variable: str) -> tuple[str, ...]:
return tuple(re.findall(r'"([^"]+)"', match.group(1)))
def _powershell_function(source: str, name: str) -> str:
start = source.index(f"function {name} {{")
next_function = source.find("\nfunction ", start + 1)
return source[start:] if next_function < 0 else source[start:next_function]
def _executor_payload_order(source: str) -> tuple[str, ...]:
match = re.search(r"readonly -a RUNTIME_FILES=\((.*?)\n\)", source, re.DOTALL)
assert match is not None
@@ -240,6 +249,10 @@ def _run_executor_harness(harness: dict[str, object], **extra_env: str) -> subpr
"HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT",
"HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT",
"HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT",
"HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL",
"HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL",
"HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT",
"HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT",
):
environment.pop(name, None)
environment["HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS"] = "1"
@@ -281,6 +294,16 @@ def _assert_prestate_restored(harness: dict[str, object]) -> None:
assert (target.read_bytes(), target.stat().st_mode & 0o777) == before_state[name]
def _assert_payload_committed(harness: dict[str, object]) -> None:
destination = Path(harness["destination"])
exporter_root = Path(harness["exporter_root"])
source_stage = Path(harness["source_stage"])
for name in harness["payload_order"]:
target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name
assert target.read_bytes() == (source_stage / name).read_bytes()
assert target.stat().st_mode & 0o777 == 0o755
def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
source = BROKER.read_text(encoding="utf-8")
@@ -304,6 +327,100 @@ def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
assert "taskkill.exe /PID $process.Id /T /F" in source
assert "$process.Kill()" in source
assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths' in source
assert '[string]$parsed.status -ne "verified"' in source
assert '-not [bool]$parsed.payloadCommitted' in source
assert '-not [bool]$parsed.stageCleanupVerified' in source
assert '-not [bool]$parsed.rollbackPrestateCleanupVerified' in source
assert '[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps"' in source
assert '[int64]$parsed.terminalExitCode -ne 0' in source
assert 'agent99-backup-runtime-terminal-$RunId.json' in source
assert '$property = $Object.PSObject.Properties[$Name]' in source
assert 'if ($null -eq $property) { return $false }' in source
assert 'Get-Agent99CommittedFailureResult $applyTransport' in source
assert 'throw "host110_apply_$([string]$apply.status)"' in source
def test_broker_required_field_contract_rejects_missing_fields_before_cast() -> None:
source = BROKER.read_text(encoding="utf-8")
field_guard = _powershell_function(source, "Test-Agent99JsonField")
common_contract = _powershell_function(source, "Convert-Agent99ExecutorDocument")
apply_contract = _powershell_function(source, "Assert-Agent99ApplyResultFields")
verifier_contract = _powershell_function(source, "Assert-Agent99VerifierFields")
committed_failure = _powershell_function(source, "Get-Agent99CommittedFailureResult")
assert '$property = $Object.PSObject.Properties[$Name]' in field_guard
assert 'if ($null -eq $property) { return $false }' in field_guard
for field in (
"schemaVersion",
"mode",
"ok",
"sourceRevision",
"sourceHead",
"runId",
"fileCount",
"backupScriptFileCount",
"backupHealthExporterIncluded",
):
assert re.search(rf"^\s+{field} = \"", common_contract, re.MULTILINE), field
for field in (
"status",
"payloadCommitted",
"rollbackAttempted",
"rollbackPerformed",
"rollbackVerified",
"zeroResidueVerified",
"zeroResidueScope",
"stageCleanupVerified",
"rollbackPrestateCleanupVerified",
"receipt",
"terminalReceipt",
"terminalReceiptPublished",
"terminalExitCode",
"deferredSignal",
"independentVerifierVerified",
"verifier",
):
assert re.search(rf"^\s+{field} = ", apply_contract, re.MULTILINE), field
for field in (
"schemaVersion",
"ok",
"sourceRevision",
"runId",
"fileCount",
"backupScriptFileCount",
"backupHealthExporterIncluded",
"remoteWritePerformed",
"selfIdentityVerified",
):
assert re.search(rf"^\s+{field} = \"", verifier_contract, re.MULTILINE), field
assert 'status -eq "cleanup_pending"' in committed_failure
assert 'status -eq "committed_signal_deferred"' in committed_failure
assert 'verifier = $VerifierKind' in apply_contract
def test_windows99_contract_replay_uses_actual_broker_functions_without_remote_write() -> None:
source = BROKER_CONTRACT_REPLAY.read_text(encoding="utf-8")
assert "FunctionDefinitionAst" in source
for function_name in (
"Test-Agent99JsonField",
"Assert-Agent99JsonFields",
"Convert-Agent99ExecutorDocument",
"Assert-Agent99ApplyResultFields",
"Assert-Agent99VerifierFields",
"Get-Agent99ExecutorResult",
"Get-Agent99CommittedFailureResult",
):
assert f'"{function_name}"' in source
assert "$mutated.PSObject.Properties.Remove($field)" in source
assert "$mutated.verifier.PSObject.Properties.Remove($field)" in source
assert 'status = "cleanup_pending"' in source
assert 'status = "committed_signal_deferred"' in source
assert 'status = "terminal_receipt_failed"' in source
assert "remoteWritePerformed = $false" in source
assert "secretValuesRead = $false" in source
for forbidden in ("New-Item", "Set-Content", "WriteAllText", "ssh.exe", "scp.exe"):
assert forbidden not in source
def test_broker_scp_basenames_match_executor_source_stage_contract() -> None:
@@ -385,11 +502,16 @@ def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
assert '"rollbackPerformed": rollback_performed' in source
assert '"rollbackVerified": rollback_verified' in source
assert '"zeroResidueVerified": zero_residue_verified' in source
assert '"zeroResidueScope": "run_owned_destination_temps"' in source
assert 'cleanup_run_owned_temps' in source
assert 'verify_run_owned_temp_absence' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT' in source
assert 'HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS' in source
assert '[ "$DEST_ROOT" != "/backup/scripts" ]' in source
assert 'fsync_rollback_prestate || return 75' in source
@@ -401,8 +523,18 @@ def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
assert 'os.fsync(handle.fileno())' in source
assert 'os.fsync(directory_fd)' in source
assert 'hashlib.sha256(readback).hexdigest()' in source
assert source.index('write_receipt "verified"') < source.index("\nAPPLY_COMPLETE=1")
assert source.index("\nAPPLY_COMPLETE=1") < source.rindex('rm -rf -- "$ROLLBACK_DIR"')
assert source.index("COMMIT_CRITICAL=1") < source.index('write_receipt "payload_verified"')
assert source.index('write_receipt "payload_verified"') < source.index("\nAPPLY_COMPLETE=1")
assert source.index("\nAPPLY_COMPLETE=1") < source.rindex("\nCOMMIT_CRITICAL=0")
assert source.index("\nAPPLY_COMPLETE=1") < source.rindex(
'cleanup_postcommit_directory "rollback_prestate"'
)
assert 'handle_agent99_signal HUP 129' in source
assert 'handle_agent99_signal TERM 143' in source
assert source.index("trap '' HUP INT TERM") < source.index('write_terminal_receipt "$terminal_status"')
assert 'write_terminal_receipt "$terminal_status"' in source
assert 'terminal_status="cleanup_pending"' in source
assert 'terminal_status="committed_signal_deferred"' in source
assert 'run_identity_receipt_exists' in source
assert 'run_identity_stage_exists' in source
assert 'run_identity_rollback_exists' in source
@@ -432,22 +564,205 @@ def test_apply_success_publishes_durable_receipt_before_prestate_cleanup(tmp_pat
assert output["zeroResidueVerified"] is True
assert output["stageCleanupVerified"] is True
assert output["rollbackPrestateCleanupVerified"] is True
assert output["status"] == "verified"
assert output["payloadCommitted"] is True
assert output["runId"] == run_id
assert output["terminalExitCode"] == 0
assert output["terminalReceiptPublished"] is True
assert output["deferredSignal"] is None
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
assert receipt["status"] == "verified"
assert receipt["status"] == "payload_verified"
assert receipt["rollbackAttempted"] is False
assert receipt["rollbackPerformed"] is False
assert receipt["rollbackVerified"] is False
assert receipt["zeroResidueVerified"] is True
assert receipt["zeroResidueScope"] == "run_owned_destination_temps"
terminal_path = Path(harness["status_root"]) / f"agent99-backup-runtime-terminal-{run_id}.json"
terminal = json.loads(terminal_path.read_text(encoding="utf-8"))
assert terminal["status"] == "verified"
assert terminal["ok"] is True
assert terminal["payloadCommitted"] is True
assert terminal["stageCleanupVerified"] is True
assert terminal["rollbackPrestateCleanupVerified"] is True
assert terminal["terminalExitCode"] == 0
assert terminal["zeroResidueScope"] == "run_owned_destination_temps_and_internal_stage_prestate"
assert output["terminalReceipt"] == str(terminal_path)
assert not (Path(harness["rollback_root"]) / run_id).exists()
assert not (Path(harness["stage_root"]) / run_id).exists()
for name in harness["payload_order"]:
target = (
Path(harness["exporter_root"]) / name
if name == "backup-health-textfile-exporter.py"
else Path(harness["destination"]) / name
)
assert target.read_bytes() == (Path(harness["source_stage"]) / name).read_bytes()
_assert_payload_committed(harness)
@pytest.mark.parametrize(("signal_name", "exit_code"), [("TERM", 143), ("HUP", 129), ("INT", 130)])
def test_commit_window_signal_is_deferred_until_payload_and_terminal_are_consistent(
tmp_path: Path,
signal_name: str,
exit_code: int,
) -> None:
run_id = f"commit-signal-{signal_name.lower()}"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL=signal_name,
)
assert result.returncode == exit_code, result.stderr
output = json.loads(result.stdout)
assert output["ok"] is False
assert output["status"] == "committed_signal_deferred"
assert output["payloadCommitted"] is True
assert output["runId"] == run_id
assert output["deferredSignal"] == signal_name
assert output["terminalExitCode"] == exit_code
assert output["terminalReceiptPublished"] is True
assert output["stageCleanupVerified"] is True
assert output["rollbackPrestateCleanupVerified"] is True
_assert_payload_committed(harness)
payload_receipt = json.loads(
(
Path(harness["status_root"])
/ f"agent99-backup-runtime-deploy-{run_id}.json"
).read_text(encoding="utf-8")
)
assert payload_receipt["status"] == "payload_verified"
terminal = json.loads(
(
Path(harness["status_root"])
/ f"agent99-backup-runtime-terminal-{run_id}.json"
).read_text(encoding="utf-8")
)
assert terminal["status"] == "committed_signal_deferred"
assert terminal["ok"] is False
assert terminal["deferredSignal"] == signal_name
assert terminal["terminalExitCode"] == exit_code
assert not (Path(harness["rollback_root"]) / run_id).exists()
assert not (Path(harness["stage_root"]) / run_id).exists()
@pytest.mark.parametrize("signal_name", ["TERM", "HUP", "INT"])
def test_terminal_publication_masks_late_signal_without_receipt_stdout_exit_divergence(
tmp_path: Path,
signal_name: str,
) -> None:
run_id = f"terminal-signal-{signal_name.lower()}"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL=signal_name,
)
assert result.returncode == 0, result.stderr
output = json.loads(result.stdout)
assert output["ok"] is True
assert output["status"] == "verified"
assert output["payloadCommitted"] is True
assert output["runId"] == run_id
assert output["deferredSignal"] is None
assert output["terminalExitCode"] == 0
assert output["terminalReceiptPublished"] is True
terminal = json.loads(
(
Path(harness["status_root"])
/ f"agent99-backup-runtime-terminal-{run_id}.json"
).read_text(encoding="utf-8")
)
assert terminal["ok"] is True
assert terminal["status"] == output["status"]
assert terminal["deferredSignal"] == output["deferredSignal"]
assert terminal["terminalExitCode"] == output["terminalExitCode"]
_assert_payload_committed(harness)
@pytest.mark.parametrize("fault", ["write", "link", "fsync", "readback"])
def test_terminal_receipt_fault_is_nonzero_with_payload_receipt_only(
tmp_path: Path,
fault: str,
) -> None:
run_id = f"terminal-receipt-{fault}"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT=fault,
)
assert result.returncode == 76
assert '"ok": true' not in result.stdout.lower()
assert "terminal_receipt_failed" in result.stderr
output = json.loads(result.stdout)
assert output["ok"] is False
assert output["status"] == "terminal_receipt_failed"
assert output["payloadCommitted"] is True
assert output["runId"] == run_id
assert output["terminalReceiptPublished"] is False
assert output["terminalExitCode"] == 76
_assert_payload_committed(harness)
payload_receipt = json.loads(
(
Path(harness["status_root"])
/ f"agent99-backup-runtime-deploy-{run_id}.json"
).read_text(encoding="utf-8")
)
assert payload_receipt["status"] == "payload_verified"
assert not (
Path(harness["status_root"])
/ f"agent99-backup-runtime-terminal-{run_id}.json"
).exists()
assert not (Path(harness["rollback_root"]) / run_id).exists()
assert not (Path(harness["stage_root"]) / run_id).exists()
@pytest.mark.parametrize(
("fault", "stage_clean", "rollback_clean"),
[("stage", False, True), ("rollback_prestate", True, False)],
)
def test_postcommit_cleanup_failure_is_nonzero_and_never_false_green(
tmp_path: Path,
fault: str,
stage_clean: bool,
rollback_clean: bool,
) -> None:
run_id = f"cleanup-{fault}"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT=fault,
)
assert result.returncode == 75, result.stderr
assert '"ok": true' not in result.stdout.lower()
output = json.loads(result.stdout)
assert output["ok"] is False
assert output["status"] == "cleanup_pending"
assert output["payloadCommitted"] is True
assert output["runId"] == run_id
assert output["stageCleanupVerified"] is stage_clean
assert output["rollbackPrestateCleanupVerified"] is rollback_clean
assert output["terminalReceiptPublished"] is True
_assert_payload_committed(harness)
payload_receipt = json.loads(
(
Path(harness["status_root"])
/ f"agent99-backup-runtime-deploy-{run_id}.json"
).read_text(encoding="utf-8")
)
assert payload_receipt["status"] == "payload_verified"
terminal = json.loads(
(
Path(harness["status_root"])
/ f"agent99-backup-runtime-terminal-{run_id}.json"
).read_text(encoding="utf-8")
)
assert terminal["status"] == "cleanup_pending"
assert terminal["ok"] is False
assert terminal["payloadCommitted"] is True
assert terminal["stageCleanupVerified"] is stage_clean
assert terminal["rollbackPrestateCleanupVerified"] is rollback_clean
assert (Path(harness["stage_root"]) / run_id).exists() is (not stage_clean)
assert (Path(harness["rollback_root"]) / run_id).exists() is (not rollback_clean)
@pytest.mark.parametrize("fault", ["write", "link", "fsync", "readback"])