Compare commits

..

6 Commits

Author SHA1 Message Date
Your Name
1df830d8cc docs(governance): bind Host110 runtime V6 evidence 2026-07-18 22:16:53 +08:00
Your Name
c96e00d6e9 fix(agent99): enforce runtime manifest count parity 2026-07-18 22:04:04 +08:00
Your Name
8391d8d5a6 fix(backup): harden Google Drive freshness readback 2026-07-18 22:04:04 +08:00
Your Name
5c4f22f7da fix(agent99): deploy backup freshness exporter atomically 2026-07-18 22:04:04 +08:00
Your Name
68dcc6bbd4 feat(agent99): bind host110 backup runtime deploy 2026-07-18 22:04:04 +08:00
Your Name
56d1d396b8 fix(backup): make Gitea dump fail-safe 2026-07-18 22:04:04 +08:00
18 changed files with 865 additions and 2253 deletions

View File

@@ -268,157 +268,30 @@ function New-Agent99SourcePackage {
}
}
function Test-Agent99JsonField {
param(
[object]$Object,
[string]$Name,
[ValidateSet("String", "Boolean", "Integer", "Object", "Null", "NullOrString")]
[string]$Kind
)
function Get-Agent99ExecutorResult {
param([object]$Transport, [string]$ExpectedMode)
if ($null -eq $Object) { return $false }
$property = $Object.PSObject.Properties[$Name]
if ($null -eq $property) { return $false }
$value = $property.Value
switch ($Kind) {
"String" { return [bool]($value -is [string]) }
"Boolean" { return [bool]($value -is [bool]) }
"Integer" { return [bool]($value -is [int] -or $value -is [long]) }
"Object" { return [bool]($value -is [System.Management.Automation.PSCustomObject]) }
"Null" { return [bool]($null -eq $value) }
"NullOrString" { return [bool]($null -eq $value -or $value -is [string]) }
}
return $false
}
function Assert-Agent99JsonFields {
param([object]$Object, [hashtable]$Contract, [string]$ErrorCode)
foreach ($name in $Contract.Keys) {
if (-not (Test-Agent99JsonField $Object $name ([string]$Contract[$name]))) {
throw $ErrorCode
}
}
}
function Convert-Agent99ExecutorDocument {
param(
[object]$Transport,
[string]$ExpectedMode,
[switch]$AllowProcessFailure
)
if (-not $Transport.ok) {
if (
-not $AllowProcessFailure -or
[string]$Transport.reason -ne "process_failed" -or
[int]$Transport.exitCode -le 0 -or
[string]::IsNullOrWhiteSpace([string]$Transport.stdout)
) {
throw "host110_${ExpectedMode}_transport_failed"
}
}
if (-not $Transport.ok) { throw "host110_${ExpectedMode}_transport_failed" }
try {
$parsed = [string]$Transport.stdout | ConvertFrom-Json
} catch {
throw "host110_${ExpectedMode}_receipt_invalid"
}
Assert-Agent99JsonFields $parsed @{
schemaVersion = "String"
mode = "String"
ok = "Boolean"
sourceRevision = "String"
sourceHead = "String"
runId = "String"
fileCount = "Integer"
backupScriptFileCount = "Integer"
backupHealthExporterIncluded = "Boolean"
} "host110_${ExpectedMode}_receipt_required_field_failed"
if (
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_executor_v1" -or
[string]$parsed.mode -ne $ExpectedMode.ToLowerInvariant() -or
-not [bool]$parsed.ok -or
[string]$parsed.sourceRevision -ne $SourceRevision -or
[string]$parsed.sourceHead -notmatch "^[0-9a-f]{40}$" -or
[string]$parsed.runId -ne $RunId -or
[int64]$parsed.fileCount -ne $ExpectedFileCount -or
[int64]$parsed.backupScriptFileCount -ne 17 -or
[int]$parsed.fileCount -ne $ExpectedFileCount -or
[int]$parsed.backupScriptFileCount -ne 17 -or
-not [bool]$parsed.backupHealthExporterIncluded
) {
throw "host110_${ExpectedMode}_receipt_contract_failed"
}
return $parsed
}
function Assert-Agent99ApplyResultFields {
param([object]$Parsed, [string]$VerifierKind)
Assert-Agent99JsonFields $Parsed @{
status = "String"
payloadCommitted = "Boolean"
rollbackAttempted = "Boolean"
rollbackPerformed = "Boolean"
rollbackVerified = "Boolean"
zeroResidueVerified = "Boolean"
zeroResidueScope = "String"
stageCleanupVerified = "Boolean"
rollbackPrestateCleanupVerified = "Boolean"
receipt = "String"
terminalReceipt = "String"
terminalReceiptPublished = "Boolean"
terminalExitCode = "Integer"
deferredSignal = "NullOrString"
independentVerifierVerified = "Boolean"
verifier = $VerifierKind
} "host110_apply_receipt_required_field_failed"
}
function Assert-Agent99VerifierFields {
param([object]$Verifier)
Assert-Agent99JsonFields $Verifier @{
schemaVersion = "String"
ok = "Boolean"
sourceRevision = "String"
runId = "String"
fileCount = "Integer"
backupScriptFileCount = "Integer"
backupHealthExporterIncluded = "Boolean"
remoteWritePerformed = "Boolean"
selfIdentityVerified = "Boolean"
} "host110_apply_verifier_required_field_failed"
}
function Get-Agent99ExecutorResult {
param([object]$Transport, [string]$ExpectedMode)
$parsed = Convert-Agent99ExecutorDocument $Transport $ExpectedMode
if (-not [bool]$parsed.ok) { throw "host110_${ExpectedMode}_receipt_contract_failed" }
if ($ExpectedMode -eq "Apply") {
Assert-Agent99ApplyResultFields $parsed "Object"
Assert-Agent99VerifierFields $parsed.verifier
if (
[string]$parsed.status -ne "verified" -or
-not [bool]$parsed.payloadCommitted -or
[bool]$parsed.rollbackAttempted -or
[bool]$parsed.rollbackPerformed -or
[bool]$parsed.rollbackVerified -or
-not [bool]$parsed.zeroResidueVerified -or
[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps" -or
-not [bool]$parsed.stageCleanupVerified -or
-not [bool]$parsed.rollbackPrestateCleanupVerified -or
[int64]$parsed.terminalExitCode -ne 0 -or
$null -ne $parsed.deferredSignal -or
-not [bool]$parsed.independentVerifierVerified -or
[string]$parsed.receipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or
[string]$parsed.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json" -or
-not [bool]$parsed.terminalReceiptPublished -or
[string]$parsed.verifier.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
-not [bool]$parsed.verifier.ok -or
[string]$parsed.verifier.sourceRevision -ne $SourceRevision -or
[string]$parsed.verifier.runId -ne $RunId -or
[int64]$parsed.verifier.fileCount -ne $ExpectedFileCount -or
[int64]$parsed.verifier.backupScriptFileCount -ne 17 -or
-not [bool]$parsed.verifier.backupHealthExporterIncluded -or
[bool]$parsed.verifier.remoteWritePerformed -or
-not [bool]$parsed.verifier.selfIdentityVerified
) {
@@ -428,66 +301,6 @@ function Get-Agent99ExecutorResult {
return $parsed
}
function Get-Agent99CommittedFailureResult {
param([object]$Transport)
$parsed = Convert-Agent99ExecutorDocument $Transport "Apply" -AllowProcessFailure
Assert-Agent99ApplyResultFields $parsed "Null"
if (
[bool]$parsed.ok -or
-not [bool]$parsed.payloadCommitted -or
[bool]$parsed.rollbackAttempted -or
[bool]$parsed.rollbackPerformed -or
[bool]$parsed.rollbackVerified -or
-not [bool]$parsed.zeroResidueVerified -or
[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps" -or
-not [bool]$parsed.independentVerifierVerified -or
[string]$parsed.receipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or
[string]$parsed.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json"
) {
throw "host110_apply_committed_failure_contract_failed"
}
$signalExitCodes = @{ INT = 130; HUP = 129; TERM = 143 }
if ([string]$parsed.status -eq "cleanup_pending") {
if (
[int]$Transport.exitCode -ne 75 -or
[int64]$parsed.terminalExitCode -ne 75 -or
([bool]$parsed.stageCleanupVerified -and [bool]$parsed.rollbackPrestateCleanupVerified) -or
($null -ne $parsed.deferredSignal -and -not $signalExitCodes.ContainsKey([string]$parsed.deferredSignal))
) {
throw "host110_apply_cleanup_pending_contract_failed"
}
if (-not [bool]$parsed.terminalReceiptPublished) {
throw "host110_apply_cleanup_pending_terminal_receipt_missing"
}
} elseif ([string]$parsed.status -eq "committed_signal_deferred") {
$signalName = [string]$parsed.deferredSignal
if (
-not $signalExitCodes.ContainsKey($signalName) -or
-not [bool]$parsed.stageCleanupVerified -or
-not [bool]$parsed.rollbackPrestateCleanupVerified -or
[int]$Transport.exitCode -ne [int]$signalExitCodes[$signalName] -or
[int64]$parsed.terminalExitCode -ne [int]$signalExitCodes[$signalName]
) {
throw "host110_apply_deferred_signal_contract_failed"
}
if (-not [bool]$parsed.terminalReceiptPublished) {
throw "host110_apply_deferred_signal_terminal_receipt_missing"
}
} elseif ([string]$parsed.status -eq "terminal_receipt_failed") {
if (
[bool]$parsed.terminalReceiptPublished -or
[int]$Transport.exitCode -ne 76 -or
[int64]$parsed.terminalExitCode -ne 76
) {
throw "host110_apply_terminal_receipt_failure_contract_failed"
}
} else {
throw "host110_apply_committed_failure_status_invalid"
}
return $parsed
}
function Get-Agent99VerifierResult {
param([object]$Transport, [object]$SourcePackage)
@@ -497,17 +310,6 @@ function Get-Agent99VerifierResult {
} catch {
throw "host110_verify_receipt_invalid"
}
Assert-Agent99JsonFields $parsed @{
schemaVersion = "String"
ok = "Boolean"
sourceRevision = "String"
runId = "String"
fileCount = "Integer"
backupScriptFileCount = "Integer"
backupHealthExporterIncluded = "Boolean"
remoteWritePerformed = "Boolean"
verifierSha256 = "String"
} "host110_verify_receipt_required_field_failed"
if (
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
-not [bool]$parsed.ok -or
@@ -582,7 +384,7 @@ if (Test-Path -LiteralPath $evidencePath -PathType Leaf) {
}
$sourceRoot = Join-Path $SourceRootBase "host110-backup-source-$RunId"
$manifestPath = Join-Path $sourceRoot "manifest.json"
$manifestPath = Join-Path $sourceRoot "agent99-host110-backup-runtime-manifest.json"
$remoteStage = "/tmp/agent99-host110-backup-runtime-$RunId"
$startedAt = [DateTime]::UtcNow
$mutex = [Threading.Mutex]::new($false, $TransportMutexName)
@@ -619,12 +421,7 @@ try {
$executorPath = "$remoteStage/host110-backup-runtime-executor.sh"
$command = "timeout --signal=TERM --kill-after=10s 300s $executorPath --mode apply --source-revision $SourceRevision --run-id $RunId --source-stage $remoteStage"
$applyTransport = Invoke-Agent99BoundedSsh $command 330
if ($applyTransport.ok) {
$apply = Get-Agent99ExecutorResult $applyTransport "Apply"
} else {
$apply = Get-Agent99CommittedFailureResult $applyTransport
throw "host110_apply_$([string]$apply.status)"
}
$apply = Get-Agent99ExecutorResult $applyTransport "Apply"
$verify = $apply.verifier
$cleanupVerified = Remove-Agent99RemoteStage $remoteStage $true
$remoteStageCreated = $false
@@ -648,7 +445,7 @@ try {
}
$result = [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v3"
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v2"
ok = [bool]($terminalStatus -ne "failed")
status = $terminalStatus
mode = $Mode

View File

@@ -1,217 +0,0 @@
# AWOOOI Agent99 Host110 Backup Runtime V10
Status: proposed; central review and explicit owner approval are required before
any production apply.
This artifact supersedes V2 through V9. Every V9 identity below is void for
approval and remains audit evidence only:
- branch/ref: `codex/host110-backup-runtime-v9-20260718`
- candidate/revision: `8f11ef3362a8d7f66a46308b58fd142b0c91a8bc`
- artifact SHA-256: `929978eb3da15b04791a8614e6d5ee504ae2a4c2f205b694307a7bad7713bcad`
- exact diff SHA-256: `8085d49450b74e7c43577ba118cbd74d3a2e3dd2d22eeba90ce749d56c72c0f2`
- every other V9 proposal, content, revision, and derived hash
No V9 hash, ref, review, receipt, or test grants production execution
authority.
## Exact Scope
- Agent99 runtime bundle: 19 files, guarded by one producer/consumer parity
test.
- Host110 backup payload: 18 ordered files, comprising 17 backup scripts and
`backup-health-textfile-exporter.py`.
- Host110 remote stage: 21 unique basenames, comprising the ordered 18-file
payload, fixed executor, independent verifier, and `manifest.json`.
- Fixed target: `wooo@192.168.0.110`, dispatched by Windows99 Agent99 from an
exact Gitea revision and digest manifest.
- The candidate includes a 10-line runtime-lock addition to
`scripts/backup/gitea-full-backup-restore-drill.sh`. A future drill invocation
acquires the shared Host110 backup-runtime lock before its pre-existing drill
behavior. Applying this candidate replaces the script but does not invoke the
drill or any container lifecycle.
- `scripts/backup/backup-gitea.sh` has zero diff from the production base. No
Gitea primary stop/start or backup-container creation is part of this
candidate apply.
- V7, V8, and V9 proposal documents remain tracked audit artifacts and are not
runtime authority.
- `agent99-host110-broker-contract-replay.ps1` is a no-write Windows99 test
asset; it is not part of the 19-file Agent99 runtime or 18-file Host110
payload.
## V9 Defects Closed
### Signal-Safe Commit Boundary
The executor enters a signal-deferring critical section before publishing the
payload receipt. `INT`, `HUP`, and `TERM` are recorded without exiting until the
durable payload-receipt readback and `APPLY_COMPLETE=1` commit state are both
established. A deferred signal can never enter the pre-commit EXIT rollback
path after the payload receipt exists.
After commit, exact stage and rollback-prestate cleanup still run. If cleanup
is complete, a deferred signal produces the non-success terminal
`committed_signal_deferred`, a durable terminal receipt, the corresponding
129/130/143 exit, and an applied payload. TERM, HUP, and INT are replayed in the
exact receipt-to-commit window.
After cleanup, terminal publication uses a bounded signal mask. Signals captured
before that boundary remain part of the non-success terminal; signals delivered
during the immutable terminal writer cannot mutate its already-final state.
Terminal-writer TERM, HUP, and INT replays require receipt, stdout, payload, and
exit to remain consistent.
### Payload And Terminal Receipts
The immutable payload receipt has status `payload_verified`; it proves exact
payload, independent verifier, durable receipt readback, and absence of
run-owned destination temporaries. It does not claim internal stage or rollback
prestate cleanup.
A separate immutable terminal receipt is the authority for overall success.
Only status `verified` with `stageCleanupVerified=true`,
`rollbackPrestateCleanupVerified=true`, no deferred signal, and exit code zero
may yield executor `ok:true`. Each successful cleanup is absence-read back and
its parent directory is fsynced before terminal publication.
If either cleanup fails, the payload remains committed, exact residue is
preserved, terminal status is `cleanup_pending`, executor output is
`ok:false`, and exit is nonzero. A payload receipt cannot be used as a
zero-residue or broker-success shortcut. The Windows99 broker validates every
status, payload, rollback, residue-scope, cleanup, signal, exit, receipt-path,
and independent-verifier field fail-closed.
Committed non-success outputs are parsed and retained in broker evidence before
the broker fails overall; `cleanup_pending`, `committed_signal_deferred`, and
`terminal_receipt_failed` can never be collapsed into an ambiguous generic
transport failure.
### Existing Transaction Guarantees Retained
- Producer and executor payload basenames are exact-equal in count, set, and
order: 18 payload files and 21 unique staged files.
- Receipt write, hard-link, fsync, and readback failures roll back exact content
and metadata without publishing success.
- Terminal-receipt write, hard-link, fsync, and readback failures keep the
verified payload, publish no terminal success, emit structured committed
failure evidence with `terminalReceiptPublished=false`, and exit nonzero.
- Apply and rollback destination temporaries are run-owned, tracked, removed,
and absence-read back.
- Rollback attempted, performed, verified, and residue truth remain separate;
partial or unknown rollback terminates `rollback_unverified`.
- Fault hooks require both the explicit test flag and a transformed
non-production destination; canonical `/backup/scripts` cannot enable them.
## Apply Effects
- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded
manifest and receipt state.
- Host110 Apply replaces the ordered 18 payload paths through one exclusive,
digest-bound filesystem transaction, and writes payload/terminal receipts
below `/backup/status`.
- The deployed restore-drill script gains shared-lock behavior. The apply does
not execute that script.
- Apply does not run a backup, invoke a Gitea container lifecycle, sync Google
Drive, prune snapshots, restart a VM or service, or change a database,
network, firewall, credential, or secret.
## Reproducible Validation
Focused replay and contract suite:
```sh
pytest -q \
scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \
scripts/backup/tests/test_verify_offsite_full_sync_runtime.py \
scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py \
scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py \
scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py \
scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py \
scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py
```
Exact signal and post-commit cleanup replays:
```sh
pytest -vv \
scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \
-k 'commit_window_signal or terminal_publication or terminal_receipt_fault or postcommit_cleanup'
```
Static, syntax, and repository checks:
```sh
ruff check scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py
bash -n scripts/backup/host110-backup-runtime-executor.sh
git diff --check
PATH=/Users/ogt/.pyenv/shims:$PATH bash scripts/ops/ansible-validate.sh
```
`ansible-validate.sh` parses
`ops/reboot-recovery/full-stack-backup-baseline.yml` as YAML metadata. It runs
`ansible-playbook --syntax-check` only for the actual playbooks listed by the
script; the metadata mapping is not represented as a playbook.
Windows99 no-file-write PowerShell parser check:
```sh
PARSER_COMMAND='$source=[Console]::In.ReadToEnd(); $tokens=$null; $errors=$null; [System.Management.Automation.Language.Parser]::ParseInput($source,[ref]$tokens,[ref]$errors) | Out-Null; [pscustomobject]@{ok=($errors.Count -eq 0); errorCount=$errors.Count; parser="WindowsPowerShell"} | ConvertTo-Json -Compress; if ($errors.Count -ne 0) { exit 1 }'
ENCODED=$(printf '%s' "$PARSER_COMMAND" | iconv -f UTF-8 -t UTF-16LE | base64 | tr -d '\n')
ssh -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=5 \
-o ConnectionAttempts=1 Administrator@192.168.0.99 \
"powershell.exe -NoProfile -NonInteractive -EncodedCommand $ENCODED" \
< agent99-host110-backup-runtime-broker.ps1
```
Windows99 actual-function missing-field/type contract replay:
```sh
WRAPPER='$script=[Console]::In.ReadToEnd(); & ([ScriptBlock]::Create($script))'
WRAPPER_ENCODED=$(printf '%s' "$WRAPPER" | iconv -f UTF-8 -t UTF-16LE | base64 | tr -d '\n')
BROKER_GZIP_BASE64=$(gzip -c agent99-host110-backup-runtime-broker.ps1 | base64 | tr -d '\n')
awk -v payload="$BROKER_GZIP_BASE64" \
'BEGIN { print "$BrokerGzipBase64 = \"" payload "\"" } { print }' \
scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1 |
ssh -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=5 \
-o ConnectionAttempts=1 Administrator@192.168.0.99 \
"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand $WRAPPER_ENCODED"
```
The replay loads the candidate broker's actual function AST, accepts the valid
success and three committed non-success terminals, then removes every required
field and injects wrong JSON types. Expected readback is 34 missing-field and 6
type rejections. It performs no remote file write.
## Exact Diff Serialization
Substitute the V10 Gitea live-ref SHA for `<candidate_git_sha>` and hash the
exact stdout bytes:
```sh
git diff --binary --full-index \
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
shasum -a 256
```
Central review must independently bind the live feature ref and candidate Git
SHA, this tracked artifact path and bytes hash, the serialized diff hash/path
count/line counts, the 19/18/21 count contracts, and the exact restore-drill and
`backup-gitea.sh` diffs.
## Rollback
- Source rollback reverts the exact V10 diff to
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
- Agent99 rollback restores the prior runtime files and manifest and verifies
prior hashes independently.
- Before payload commit, Host110 rollback restores all prior payload content,
SHA-256, uid, gid, and mode under the exclusive lock and proves no run-owned
destination temporary remains.
- After payload commit, the executor never rolls the payload back merely due to
a signal or cleanup error. It preserves exact cleanup evidence and reports a
non-success terminal. Any later cleanup-only remediation is a separate
controlled action scoped to the run-specific stage/prestate paths and may not
rewrite the verified payload.
- Source rollback restores the restore-drill script's prior unlocked behavior.
It does not invoke the drill or any container lifecycle.
- Any canonical, metadata, receipt, signal, cleanup, or residue mismatch fails
closed and cannot produce broker `verified`.

View File

@@ -0,0 +1,67 @@
# AWOOOI Agent99 Host110 Backup Runtime V6
Status: proposed; owner approval required before any production apply.
This artifact supersedes V2 through V5. It is evidence for central review, not
an active policy and not production execution authority.
## Scope
- Agent99 runtime bundle: 19 files. Deploy, bootstrap, sender, receiver,
preflight, Signoz consumer, and contract checks must agree on this count.
- Host110 backup payload: 18 files, comprising 17 backup scripts and
`backup-health-textfile-exporter.py`.
- Host110 remote stage: 21 files, comprising the 18 payload files, the fixed
executor, the independent verifier, and the source manifest.
- Fixed production target: `wooo@192.168.0.110`, dispatched from Windows99
Agent99 with an exact Gitea revision and digest manifest.
## Exact Diff Serialization
The reviewer must substitute the V6 Gitea live-ref SHA for
`<candidate_git_sha>` and hash these exact stdout bytes:
```sh
git diff --binary --full-index \
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
shasum -a 256
```
The proposal message must bind all of the following independently:
- Gitea live ref and exact 40-character candidate Git SHA.
- This tracked artifact path and its SHA-256 bytes hash.
- The exact diff serialization command above and resulting SHA-256.
## Apply Effects
- Agent99 runtime promotion writes the 19-file runtime bundle on Windows99 and
writes its bounded deployment manifest and receipts.
- Host110 Apply writes/replaces the 18 payload destinations through one
exclusive, digest-bound filesystem transaction.
- The transaction does not itself run a backup, sync Google Drive, prune or
delete snapshots, restart a VM, or change a network, firewall, DB, or service.
- The broader source diff also changes the Gitea backup route. A later scheduled
or explicit Gitea offline-consistency backup may stop and restart the Gitea
container within its bounded backup window. That deferred runtime effect is
not part of the Host110 filesystem deployment transaction and requires its
own verifier receipt.
## Validation
- Post-rebase focused suite: 51 passed, 4 skipped.
- Central read-only review: 36 passed, 4 skipped; Host110 broker: 7 passed.
- Ansible validation: green.
- Changed PowerShell parser readback on Windows99: zero errors.
- Prior broad regression baseline: 552 passed, 4 skipped.
## Rollback
- Source rollback reverts the exact V6 candidate diff to
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
- Agent99 rollback restores the complete prior runtime bundle and prior runtime
manifest, then independently verifies prior hashes.
- Host110 rollback restores every prior payload's content, SHA-256, uid, gid,
and mode under the exclusive lock.
- Any mismatch terminates as `rollback_unverified`; it cannot be reported as a
successful rollback.

View File

@@ -1,65 +0,0 @@
# AWOOOI Agent99 Host110 Backup Runtime V7
Status: proposed; owner approval required before any production apply.
This artifact supersedes V2 through V6. All V5 proposal, artifact, and diff
hashes are void. This file is review evidence only; it is not active policy or
production execution authority.
## Scope
- Agent99 runtime bundle: 19 files. Every producer and consumer is guarded by
one exact parity test.
- Host110 backup payload: 18 files, comprising 17 backup scripts and
`backup-health-textfile-exporter.py`.
- Host110 remote stage: 21 files, comprising the 18 payload files, the fixed
executor, the independent verifier, and the source manifest.
- Fixed target: `wooo@192.168.0.110`, dispatched by Windows99 Agent99 from an
exact Gitea live revision and digest manifest.
- Gitea offline backup and container lifecycle changes are excluded from this
proposal. `backup-gitea.sh` has no candidate diff. The only Gitea-related
diff is a shared deployment lock in the read-only restore-drill entrypoint;
it adds no Docker command and is not executed by the deployment transaction.
## Exact Diff Serialization
Substitute the V7 Gitea live-ref SHA for `<candidate_git_sha>` and hash the
exact stdout bytes from this command:
```sh
git diff --binary --full-index \
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
shasum -a 256
```
The review message independently binds the Gitea live ref and Git SHA, this
tracked artifact path and bytes hash, and the serialized diff hash.
## Apply Effects
- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded
manifest and receipt state.
- Host110 Apply writes/replaces the 18 payload paths through one exclusive,
digest-bound filesystem transaction.
- Apply does not run a backup, invoke a Gitea container lifecycle, sync Google
Drive, prune or delete snapshots, restart a VM or service, or change a DB,
network, or firewall.
## Validation
- Post-split focused runtime, broker, backup, and parity suites must pass.
- Ansible and shell/Python static validation must pass.
- Changed PowerShell must parse on Windows99 with zero errors.
- Central review must independently reproduce the live ref, artifact hash,
diff hash, and 19/18/21 counts.
## Rollback
- Source rollback reverts the exact V7 diff to
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
- Agent99 rollback restores the prior runtime files and manifest and verifies
the prior hashes independently.
- Host110 rollback restores every prior payload's content, SHA-256, uid, gid,
and mode under the exclusive lock.
- Any mismatch terminates as `rollback_unverified` and cannot be reported as a
successful rollback.

View File

@@ -1,78 +0,0 @@
# AWOOOI Agent99 Host110 Backup Runtime V8
Status: proposed; owner approval required before any production apply.
This artifact supersedes V2 through V7. The V7 candidate, revision, artifact,
and diff hashes are void for approval. All V5 hashes remain void. Prior refs
remain immutable audit evidence only; none is production execution authority.
## Scope
- Agent99 runtime bundle: 19 files, guarded by one producer/consumer parity
test.
- Host110 backup payload: 18 files, comprising 17 backup scripts and
`backup-health-textfile-exporter.py`.
- Host110 remote stage: 21 unique basenames, comprising the 18 payload files,
the fixed executor, the independent verifier, and `manifest.json`.
- Fixed target: `wooo@192.168.0.110`, dispatched by Windows99 Agent99 from an
exact Gitea main revision and digest manifest.
- Gitea offline backup and container lifecycle changes remain excluded.
`backup-gitea.sh` has no candidate diff.
## V7 Defect Closed
V7 created and uploaded
`agent99-host110-backup-runtime-manifest.json`, while the fixed Host110
executor consumed `$SOURCE_STAGE/manifest.json`. V8 uses `manifest.json` at
the broker source stage, so SCP preserves the exact basename consumed by the
executor.
The integration test derives all broker SCP basenames from the actual
PowerShell file list and manifest path, requires 21 unique stage files, and
matches the manifest and verifier basenames to the executor's exact
`$SOURCE_STAGE` paths. A hand-built temporary manifest cannot satisfy this
test by itself.
## Exact Diff Serialization
Substitute the V8 Gitea live-ref SHA for `<candidate_git_sha>` and hash the
exact stdout bytes from this command:
```sh
git diff --binary --full-index \
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
shasum -a 256
```
The review message independently binds the Gitea live ref and Git SHA, this
tracked artifact path and bytes hash, and the serialized diff hash.
## Apply Effects
- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded
manifest and receipt state.
- Host110 Apply writes/replaces the 18 payload paths through one exclusive,
digest-bound filesystem transaction.
- Apply does not run a backup, invoke a Gitea container lifecycle, sync Google
Drive, prune or delete snapshots, restart a VM or service, or change a DB,
network, or firewall.
## Validation
- The exact broker/SCP basename to executor path integration test must pass.
- Focused runtime, broker, backup, parity, and Ansible suites must pass.
- Ansible and shell/Python static validation must pass.
- Changed PowerShell must parse on Windows99 with zero errors.
- Central review must independently reproduce the live ref, artifact hash,
diff hash, 19/18/21 counts, and zero `backup-gitea.sh` diff.
## Rollback
- Source rollback reverts the exact V8 diff to
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
- Agent99 rollback restores the prior runtime files and manifest and verifies
the prior hashes independently.
- Host110 rollback restores every prior payload's content, SHA-256, uid, gid,
and mode under the exclusive lock.
- Any mismatch terminates as `rollback_unverified` and cannot be reported as a
successful rollback.

View File

@@ -1,133 +0,0 @@
# AWOOOI Agent99 Host110 Backup Runtime V9
Status: proposed; owner approval required before any production apply.
This artifact supersedes V2 through V8. The following V8 identities are void
for approval and remain audit evidence only:
- candidate/revision: `0b9c0284006c8d892c4617c2bfc16869bb6637f0`
- artifact SHA-256: `ea9537613c462a07f3155263da5ad0f444bda0ba6773b49208de31b0acdef228`
- exact diff SHA-256: `0867d9221ecfa7bcbe6b073a6a02da401bbed5087b9f2bd7fa494f2953b2ef2e`
- every other proposal/content hash derived from V8 candidate bytes
No V8 hash, ref, review, or receipt grants production execution authority.
## Scope
- Agent99 runtime bundle: 19 files, guarded by one producer/consumer parity
test.
- Host110 backup payload: 18 ordered files, comprising 17 backup scripts and
`backup-health-textfile-exporter.py`.
- Host110 remote stage: 21 unique basenames, comprising the ordered 18-file
payload, fixed executor, independent verifier, and `manifest.json`.
- Fixed target: `wooo@192.168.0.110`, dispatched by Windows99 Agent99 from an
exact Gitea revision and digest manifest.
- Gitea offline backup and container lifecycle changes remain excluded;
`backup-gitea.sh` has no candidate diff from the production base.
## V8 Defects Closed
### Durable Receipt Transaction
The executor no longer captures `write_receipt` through command substitution
and cannot mask an internal failure with a later `printf`. Receipt publication
uses an exclusive temporary file, exact full-write check, file `fsync`,
temporary-byte SHA-256 readback, no-replace hard link, directory `fsync`, exact
final-byte/document/SHA-256 readback, temporary unlink, and a second directory
`fsync`. Any write, link, fsync, cleanup, or readback failure is nonzero and
removes a final path created by that failed attempt when possible.
`APPLY_COMPLETE=1` is set only after the durable verified receipt passes this
transaction. Stage and rollback prestate are never cleared before that point.
If verified receipt publication fails, the EXIT path performs rollback first,
writes a distinct failure receipt, and only removes rollback prestate after
both rollback verification and durable failure-receipt publication succeed.
Payload files, destination directories, rollback prestate files, and the
rollback directory are explicitly fsynced before their corresponding durable
claim can be published.
### Run-Owned Temporary Files
Every apply and rollback temporary destination is tracked for the exact run.
Rollback removes and reads back all tracked paths, including a failure between
`install` and `mv`. Canonical content/metadata verification plus zero hidden
residue are both required before rollback can be called verified.
### Rollback Truth
Receipt schema `agent99_host110_backup_runtime_deploy_receipt_v2` records
`rollbackAttempted`, `rollbackPerformed`, `rollbackVerified`, and
`zeroResidueVerified` independently. A partial, unknown, or residue-bearing
rollback terminates as `rollback_unverified`; only a fully restored and
zero-residue transaction may terminate as `failed_rolled_back`.
### Producer/Consumer Parity
The integration test parses both actual sources and requires the broker's 18
payload basenames to equal the executor payload in count, exact set, and exact
order. It separately requires executor, verifier, and `manifest.json`, yielding
21 unique remote-stage basenames.
## Fault-Injection Contract
The bounded local harness executes the real executor control flow against
isolated temporary destinations and covers:
- successful apply, durable receipt, verifier, and post-receipt prestate cleanup;
- receipt write, link, fsync, and final-readback failures;
- apply failure after hidden-temp installation but before canonical `mv`;
- forced temp-cleanup failure and `rollback_unverified` evidence preservation.
Every receipt fault must restore exact prestate, emit no success JSON, and
produce a durable `failed_rolled_back` receipt. A cleanup fault must preserve
`prestate.tsv`, expose rollback attempted/performed separately, and must not
claim rollback verification.
Fault hooks require an explicit test enable flag and a transformed
non-production destination. The canonical `/backup/scripts` executor can never
enable them, even if a fault environment variable is present.
## Exact Diff Serialization
Substitute the V9 Gitea live-ref SHA for `<candidate_git_sha>` and hash the
exact stdout bytes from this command:
```sh
git diff --binary --full-index \
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
shasum -a 256
```
The review message independently binds the Gitea live ref and Git SHA, this
tracked artifact path and bytes hash, and the serialized diff hash.
## Apply Effects
- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded
manifest and receipt state.
- Host110 Apply writes/replaces the 18 payload paths through one exclusive,
digest-bound filesystem transaction.
- Apply does not run a backup, invoke a Gitea container lifecycle, sync Google
Drive, prune snapshots, restart a VM or service, or change a database,
network, or firewall.
## Validation
- The exact broker payload/order and 21-file stage parity test must pass.
- Success and all six fault-injection paths must execute and pass.
- Focused runtime, broker, backup, parity, Ansible, shell, and Python checks
must pass.
- Changed PowerShell must parse on Windows99 with zero errors when applicable.
- Central review must independently reproduce the live ref, artifact hash,
exact diff hash, 19/18/21 counts, and zero `backup-gitea.sh` diff.
## Rollback
- Source rollback reverts the exact V9 diff to
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
- Agent99 rollback restores the prior runtime files and manifest and verifies
prior hashes independently.
- Host110 rollback restores all prior payload content, SHA-256, uid, gid, and
mode under the exclusive lock and proves zero run-owned temporary residue.
- Any canonical, metadata, receipt, or residue mismatch terminates as
`rollback_unverified`; rollback prestate remains available for diagnosis.

View File

@@ -17,6 +17,9 @@
- backup-host188-products.sh
- verify-host188-products-backup.sh
- verify-host188-products-archive.py
- backup-gitea.sh
- gitea-full-backup-restore-drill.sh
- backup-configs.sh
backup_runtime_stage2_files:
- backup-awoooi.sh
- backup-awoooi-frequent.sh
@@ -35,7 +38,7 @@
ansible.builtin.assert:
that:
- inventory_hostname == "host_110"
- backup_runtime_files | length == 14
- backup_runtime_files | length == 17
- backup_runtime_files | unique | length == backup_runtime_files | length
- backup_runtime_fault_injection_after_stage1 | type_debug == "bool"
fail_msg: backup_runtime_deploy_boundary_failed

View File

@@ -224,6 +224,7 @@
- backup-all.sh
- backup-status.sh
- backup-gitea.sh
- gitea-full-backup-restore-drill.sh
- gitea-repo-bundle-backup.sh
- gitea-bundle-backup-sync-188.sh
- backup-harbor.sh

View File

@@ -201,6 +201,13 @@ main() {
write_cmd_output "110-systemd-unit-files" systemctl list-unit-files || failed=$((failed + 1))
write_cmd_output "110-docker-containers" docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' || true
if [ -r /home/wooo/vibework-act-runner/config.yaml ]; then
record_config_status "110-gitea-runner-config" true true "110"
else
record_config_status "110-gitea-runner-config" true false "110"
failed=$((failed + 1))
fi
if tar_local "110-host-configs" \
/etc/nginx \
/etc/systemd/system \
@@ -224,6 +231,7 @@ main() {
/home/wooo/scripts \
/home/wooo/awoooi \
/home/wooo/awoooi-ops \
/home/wooo/vibework-act-runner/config.yaml \
/backup/scripts; then
record_config_status "110-host-configs" true true "110"
else

View File

@@ -7,6 +7,7 @@
# =============================================================================
set -euo pipefail
umask 077
source "$(dirname "$0")/common.sh"
@@ -17,7 +18,22 @@ DUMP_DIR="/tmp/gitea-backup-$$"
TEXTFILE_DIR="${NODE_EXPORTER_TEXTFILE_DIR:-/home/wooo/node_exporter_textfiles}"
TEXTFILE_PATH="${GITEA_FULL_BACKUP_RECEIPT_TEXTFILE:-${TEXTFILE_DIR}/gitea_full_backup_receipt.prom}"
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
GITEA_FULL_BACKUP_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments hooks_custom_assets"
GITEA_FULL_BACKUP_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments actions_logs actions_artifacts avatars hooks_custom_assets"
GITEA_RESTORE_DRILL="$(dirname "$0")/gitea-full-backup-restore-drill.sh"
GITEA_BACKUP_LOCK="${GITEA_BACKUP_LOCK:-/tmp/gitea-backup.lock}"
GITEA_STOP_TIMEOUT_SECONDS="${GITEA_STOP_TIMEOUT_SECONDS:-15}"
GITEA_DUMP_TIMEOUT_SECONDS="${GITEA_DUMP_TIMEOUT_SECONDS:-360}"
GITEA_DUMP_KILL_AFTER_SECONDS="${GITEA_DUMP_KILL_AFTER_SECONDS:-10}"
GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS="${GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS:-10}"
GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS="${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS:-3}"
GITEA_START_TIMEOUT_SECONDS="${GITEA_START_TIMEOUT_SECONDS:-60}"
GITEA_MAX_OUTAGE_SECONDS="${GITEA_MAX_OUTAGE_SECONDS:-600}"
GITEA_LOCAL_HEALTH_URL="${GITEA_LOCAL_HEALTH_URL:-http://127.0.0.1:3001/api/healthz}"
TRANSIENT_CONTAINER=""
PRIMARY_STOPPED=0
PRIMARY_RESTART_VERIFIED=0
OUTAGE_STARTED_AT=0
OUTAGE_SECONDS=0
label_escape() {
printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'
@@ -28,6 +44,10 @@ write_gitea_full_backup_receipt() {
local timestamp="$2"
local duration="$3"
local snapshot_id="${4:-unknown}"
local consistent_offline_window="${5:-0}"
local service_restart_verified="${6:-0}"
local outage_seconds="${7:-0}"
local structural_drill_performed="${8:-0}"
local tmp
local host
local service_label
@@ -66,6 +86,12 @@ write_gitea_full_backup_receipt() {
echo "# TYPE awoooi_gitea_full_backup_secret_value_collected gauge"
echo "# HELP awoooi_gitea_full_backup_production_restore_performed Whether this receipt restored anything into production."
echo "# TYPE awoooi_gitea_full_backup_production_restore_performed gauge"
echo "# HELP awoooi_gitea_full_backup_consistent_offline_window Whether the dump was created while the primary Gitea service was stopped."
echo "# TYPE awoooi_gitea_full_backup_consistent_offline_window gauge"
echo "# HELP awoooi_gitea_full_backup_service_restart_verified Whether Gitea was independently reachable after the bounded offline window."
echo "# TYPE awoooi_gitea_full_backup_service_restart_verified gauge"
echo "# HELP awoooi_gitea_full_backup_service_outage_seconds Duration of the bounded Gitea offline backup window."
echo "# TYPE awoooi_gitea_full_backup_service_outage_seconds gauge"
echo "awoooi_gitea_full_backup_last_success_timestamp{host=\"${host}\",service=\"${service_label}\"} $([ "${ok}" = "1" ] && echo "${timestamp}" || echo 0)"
echo "awoooi_gitea_full_backup_last_run_failed{host=\"${host}\",service=\"${service_label}\",status=\"${status_label}\"} ${failed}"
echo "awoooi_gitea_full_backup_duration_seconds{host=\"${host}\",service=\"${service_label}\"} ${duration}"
@@ -74,16 +100,231 @@ write_gitea_full_backup_receipt() {
for component in ${GITEA_FULL_BACKUP_COMPONENTS}; do
echo "awoooi_gitea_full_backup_component_receipt{host=\"${host}\",service=\"${service_label}\",component=\"$(label_escape "${component}")\",receipt=\"gitea_dump_restic\"} ${ok}"
done
echo "awoooi_gitea_full_backup_restore_drill_performed{host=\"${host}\",service=\"${service_label}\"} 0"
echo "awoooi_gitea_full_backup_restore_drill_performed{host=\"${host}\",service=\"${service_label}\"} ${structural_drill_performed}"
echo "awoooi_gitea_full_backup_secret_value_collected{host=\"${host}\",service=\"${service_label}\"} 0"
echo "awoooi_gitea_full_backup_production_restore_performed{host=\"${host}\",service=\"${service_label}\"} 0"
echo "awoooi_gitea_full_backup_consistent_offline_window{host=\"${host}\",service=\"${service_label}\"} ${consistent_offline_window}"
echo "awoooi_gitea_full_backup_service_restart_verified{host=\"${host}\",service=\"${service_label}\"} ${service_restart_verified}"
echo "awoooi_gitea_full_backup_service_outage_seconds{host=\"${host}\",service=\"${service_label}\"} ${outage_seconds}"
} >"${tmp}"
mv "${tmp}" "${TEXTFILE_PATH}" || return 0
chmod 0644 "${TEXTFILE_PATH}" || true
}
cleanup() {
local status=$?
trap - EXIT HUP INT TERM
if [ -n "${TRANSIENT_CONTAINER}" ]; then
if ! remove_transient_container; then
status=1
fi
fi
if [ "${PRIMARY_STOPPED}" -eq 1 ]; then
if ! start_and_verify_primary; then
status=1
fi
fi
rm -rf "${DUMP_DIR}"
exit "${status}"
}
validate_positive_integer() {
case "$2" in
''|*[!0-9]*|0) log_error "$1 must be a positive integer"; return 64 ;;
esac
}
require_commands() {
local command_name configured_outage_bound
for command_name in curl docker flock python3 restic timeout; do
if ! command -v "${command_name}" >/dev/null 2>&1; then
log_error "Required command missing: ${command_name}"
return 69
fi
done
[ -x "${GITEA_RESTORE_DRILL}" ] || {
log_error "Gitea structural restore verifier is unavailable"
return 69
}
[ -r "${RESTIC_PASSWORD_FILE}" ] || {
log_error "Restic password reference is unavailable"
return 69
}
validate_positive_integer GITEA_STOP_TIMEOUT_SECONDS "${GITEA_STOP_TIMEOUT_SECONDS}"
validate_positive_integer GITEA_DUMP_TIMEOUT_SECONDS "${GITEA_DUMP_TIMEOUT_SECONDS}"
validate_positive_integer GITEA_DUMP_KILL_AFTER_SECONDS "${GITEA_DUMP_KILL_AFTER_SECONDS}"
validate_positive_integer GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS "${GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS}"
validate_positive_integer GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS "${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS}"
validate_positive_integer GITEA_START_TIMEOUT_SECONDS "${GITEA_START_TIMEOUT_SECONDS}"
validate_positive_integer GITEA_MAX_OUTAGE_SECONDS "${GITEA_MAX_OUTAGE_SECONDS}"
configured_outage_bound=$((
GITEA_STOP_TIMEOUT_SECONDS
+ GITEA_DUMP_TIMEOUT_SECONDS
+ GITEA_DUMP_KILL_AFTER_SECONDS
+ GITEA_START_TIMEOUT_SECONDS
+ (4 * GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS)
+ (4 * GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS)
))
if [ "${configured_outage_bound}" -gt "${GITEA_MAX_OUTAGE_SECONDS}" ]; then
log_error "Configured Gitea backup outage bound exceeds ${GITEA_MAX_OUTAGE_SECONDS}s"
return 64
fi
}
bounded_docker_control() {
timeout --signal=TERM \
--kill-after="${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS}s" \
"${GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS}s" \
docker "$@"
}
remove_transient_container() {
local transient="${TRANSIENT_CONTAINER}"
local remove_status=0
[ -n "${transient}" ] || return 0
if ! bounded_docker_control rm -f "${transient}" >/dev/null 2>&1; then
log_error "Gitea one-shot backup container removal timed out or failed"
remove_status=1
fi
TRANSIENT_CONTAINER=""
return "${remove_status}"
}
wait_for_primary_health() {
local deadline=$((SECONDS + GITEA_START_TIMEOUT_SECONDS))
local health_body
while [ "${SECONDS}" -lt "${deadline}" ]; do
if [ "$(bounded_docker_control inspect -f '{{.State.Running}}' "${GITEA_CONTAINER}" 2>/dev/null || true)" = "true" ]; then
health_body="$(curl -fsS --max-time 5 "${GITEA_LOCAL_HEALTH_URL}" 2>/dev/null || true)"
if printf '%s' "${health_body}" | grep -Eq '"status"[[:space:]]*:[[:space:]]*"pass"'; then
return 0
fi
fi
sleep 2
done
return 1
}
start_and_verify_primary() {
local running
running="$(bounded_docker_control inspect -f '{{.State.Running}}' "${GITEA_CONTAINER}" 2>/dev/null || true)"
if [ "${running}" != "true" ]; then
if ! bounded_docker_control start "${GITEA_CONTAINER}" >/dev/null; then
log_error "Gitea failed to restart after the backup window"
return 1
fi
fi
if ! wait_for_primary_health; then
log_error "Gitea post-backup health verification timed out"
return 1
fi
PRIMARY_STOPPED=0
PRIMARY_RESTART_VERIFIED=1
if [ "${OUTAGE_STARTED_AT}" -gt 0 ]; then
OUTAGE_SECONDS=$(($(date +%s) - OUTAGE_STARTED_AT))
fi
if [ "${OUTAGE_SECONDS}" -gt "${GITEA_MAX_OUTAGE_SECONDS}" ]; then
log_error "Gitea backup outage exceeded ${GITEA_MAX_OUTAGE_SECONDS}s"
return 1
fi
log_success "Gitea primary restarted and independently reachable (outage=${OUTAGE_SECONDS}s)"
}
offline_gitea_dump() {
local image_id network_rows network_name network_count git_uid host_uid timestamp dump_status stop_status transient_cleanup_status
[ "$(docker inspect -f '{{.State.Running}}' "${GITEA_CONTAINER}" 2>/dev/null || true)" = "true" ] || {
log_error "Gitea primary is not running before backup"
return 1
}
wait_for_primary_health || {
log_error "Gitea primary is not reachable before backup"
return 1
}
image_id="$(docker inspect -f '{{.Image}}' "${GITEA_CONTAINER}")"
[[ "${image_id}" =~ ^sha256:[0-9a-f]{64}$ ]] || {
log_error "Gitea image identity is invalid"
return 1
}
network_rows="$(docker inspect -f '{{range $name, $value := .NetworkSettings.Networks}}{{$name}}{{"\n"}}{{end}}' "${GITEA_CONTAINER}" | sed '/^$/d')"
network_count="$(printf '%s\n' "${network_rows}" | sed '/^$/d' | wc -l | tr -d ' ')"
[ "${network_count}" -eq 1 ] || {
log_error "Gitea backup requires exactly one fixed Docker network"
return 1
}
network_name="$(printf '%s\n' "${network_rows}" | head -n 1)"
[[ "${network_name}" =~ ^[A-Za-z0-9_.-]+$ ]] || {
log_error "Gitea Docker network identity is invalid"
return 1
}
git_uid="$(docker exec -u git "${GITEA_CONTAINER}" id -u)"
host_uid="$(id -u)"
[ "${git_uid}" = "${host_uid}" ] || {
log_error "Gitea dump staging UID does not match the controller UID"
return 1
}
timestamp="$(date +%Y%m%d%H%M%S)"
TRANSIENT_CONTAINER="agent99-gitea-backup-${timestamp}-$$"
OUTAGE_STARTED_AT="$(date +%s)"
PRIMARY_STOPPED=1
set +e
timeout --signal=TERM \
--kill-after="${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS}s" \
"$((GITEA_STOP_TIMEOUT_SECONDS + GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS))s" \
docker stop --time "${GITEA_STOP_TIMEOUT_SECONDS}" "${GITEA_CONTAINER}" >/dev/null
stop_status=$?
set -e
if [ "${stop_status}" -ne 0 ]; then
log_error "Gitea primary could not enter the bounded offline backup window"
start_and_verify_primary || true
return 1
fi
set +e
timeout --signal=TERM \
--kill-after="${GITEA_DUMP_KILL_AFTER_SECONDS}s" \
"${GITEA_DUMP_TIMEOUT_SECONDS}s" docker run --pull=never \
--name "${TRANSIENT_CONTAINER}" \
--network "${network_name}" \
--volumes-from "${GITEA_CONTAINER}" \
--mount "type=bind,src=${DUMP_DIR},dst=/backup-out" \
--user "${git_uid}" \
--workdir /backup-out \
--cpus "${BACKUP_DOCKER_CPUS}" \
--memory "${BACKUP_DOCKER_MEMORY}" \
--memory-swap "${BACKUP_DOCKER_MEMORY_SWAP}" \
--security-opt no-new-privileges:true \
--entrypoint /usr/local/bin/gitea \
"${image_id}" dump \
-c /data/gitea/conf/app.ini \
-f /backup-out/gitea-dump.zip \
-t /tmp
dump_status=$?
set -e
transient_cleanup_status=0
remove_transient_container || transient_cleanup_status=$?
if ! start_and_verify_primary; then
return 1
fi
if [ "${transient_cleanup_status}" -ne 0 ]; then
return "${transient_cleanup_status}"
fi
if [ "${dump_status}" -ne 0 ]; then
log_error "Offline Gitea dump failed (exit=${dump_status})"
return "${dump_status}"
fi
[ -s "${DUMP_DIR}/gitea-dump.zip" ] || {
log_error "Offline Gitea dump did not produce a non-empty archive"
return 1
}
log_success "Gitea consistent offline dump completed"
}
main() {
@@ -91,31 +332,44 @@ main() {
local tags
local snapshot_id
local duration
local structural_drill_performed=0
start_time=$(date +%s)
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM HUP
log_info "========== 開始 Gitea 備份 =========="
mkdir -p "${DUMP_DIR}"
log_info "執行 Gitea dump..."
if docker exec -u git "${GITEA_CONTAINER}" gitea dump -c /data/gitea/conf/app.ini -f /tmp/gitea-dump.zip 2>&1; then
docker cp "${GITEA_CONTAINER}:/tmp/gitea-dump.zip" "${DUMP_DIR}/gitea-dump.zip"
docker exec -u git "${GITEA_CONTAINER}" rm -f /tmp/gitea-dump.zip
log_success "Gitea dump 完成"
else
duration=$(($(date +%s) - start_time))
write_gitea_full_backup_receipt "dump_failed" "0" "${duration}" "none" || true
log_error "Gitea dump 失敗"
notify_clawbot "failed" "${SERVICE}" "Gitea dump 失敗"
exit 1
require_commands
exec 9>"${GITEA_BACKUP_LOCK}"
if ! flock -n 9; then
log_error "Gitea backup is already running"
return 75
fi
install -d -m 700 "${DUMP_DIR}"
log_info "執行一致性 Gitea offline dump..."
if ! offline_gitea_dump; then
duration=$(($(date +%s) - start_time))
write_gitea_full_backup_receipt "offline_dump_failed" "0" "${duration}" "none" 0 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" 0 || true
notify_clawbot "failed" "${SERVICE}" "Gitea 一致性備份失敗;服務重啟驗證=${PRIMARY_RESTART_VERIFIED}" "${duration}"
return 1
fi
if ! "${GITEA_RESTORE_DRILL}" --dump-zip "${DUMP_DIR}/gitea-dump.zip"; then
duration=$(($(date +%s) - start_time))
write_gitea_full_backup_receipt "structural_drill_failed" "0" "${duration}" "none" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" 0 || true
log_error "Gitea dump structural verifier failed"
notify_clawbot "failed" "${SERVICE}" "Gitea 備份結構驗證失敗" "${duration}"
return 1
fi
structural_drill_performed=1
if [ ! -d "${LOCAL_REPO}/data" ]; then
log_info "初始化本地 Restic 倉庫..."
if ! restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}"; then
duration=$(($(date +%s) - start_time))
write_gitea_full_backup_receipt "restic_init_failed" "0" "${duration}" "none" || true
write_gitea_full_backup_receipt "restic_init_failed" "0" "${duration}" "none" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" "${structural_drill_performed}" || true
log_error "Restic 初始化失敗"
notify_clawbot "failed" "${SERVICE}" "Gitea Restic 初始化失敗"
exit 1
@@ -127,7 +381,7 @@ main() {
--password-file "${RESTIC_PASSWORD_FILE}" \
${tags}; then
duration=$(($(date +%s) - start_time))
write_gitea_full_backup_receipt "restic_backup_failed" "0" "${duration}" "none" || true
write_gitea_full_backup_receipt "restic_backup_failed" "0" "${duration}" "none" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" "${structural_drill_performed}" || true
log_error "Restic 備份失敗"
notify_clawbot "failed" "${SERVICE}" "Gitea Restic 備份失敗"
exit 1
@@ -142,7 +396,7 @@ main() {
log_info "Offsite copy is handled by sync-offsite-backups.sh; no direct rclone sync here."
duration=$(($(date +%s) - start_time))
write_gitea_full_backup_receipt "success" "$(date +%s)" "${duration}" "${snapshot_id:-unknown}" || true
write_gitea_full_backup_receipt "success" "$(date +%s)" "${duration}" "${snapshot_id:-unknown}" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" "${structural_drill_performed}" || true
log_success "========== Gitea 備份完成 (${duration}s) =========="
notify_clawbot "success" "${SERVICE}" "Gitea 備份完成" "${duration}"
}

View File

@@ -19,9 +19,9 @@ DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}"
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
TEXTFILE_PATH="${AIOPS_GITEA_FULL_BACKUP_RESTORE_DRILL_TEXTFILE:-/home/wooo/node_exporter_textfiles/gitea_full_backup_restore_drill.prom}"
WRITE_TEXTFILE=0
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-60}"
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-300}"
DRILL_SCOPE="structural_metadata_only"
REQUIRED_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments hooks_custom_assets"
REQUIRED_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments actions_logs actions_artifacts avatars hooks_custom_assets"
usage() {
cat <<'USAGE'
@@ -98,35 +98,50 @@ required = sys.argv[2:]
try:
with zipfile.ZipFile(dump_zip) as archive:
names = archive.namelist()
bad_member = archive.testzip()
infos = archive.infolist()
if bad_member is not None:
print("ERROR=crc_failed")
sys.exit(1)
except zipfile.BadZipFile:
print("ERROR=bad_zip")
sys.exit(1)
except OSError:
except (OSError, RuntimeError):
print("ERROR=zip_open_failed")
sys.exit(1)
lowered = [name.lower() for name in names]
members = [
(info.filename.lower().lstrip("./"), info.file_size)
for info in infos
if not info.is_dir()
]
patterns = {
"database": ("gitea-db", "database", ".sql"),
"repositories": ("repo", "repos", "repositories", ".git"),
"app_settings": ("app.ini", "app.example.ini", "conf/"),
"lfs_objects": ("lfs",),
"package_registry": ("package", "packages"),
"attachments": ("attachment", "attachments"),
"hooks_custom_assets": ("hook", "custom", "public/"),
def has_nonempty(*needles: str) -> bool:
return any(size > 0 and any(needle in name for needle in needles) for name, size in members)
data_bundle = has_nonempty("gitea-data.zip")
custom_bundle = has_nonempty("gitea-custom.zip")
coverage = {
"database": has_nonempty("gitea-db", "database", ".sql"),
"repositories": has_nonempty("gitea-repo.zip", "gitea-repos.zip", "repos/", "repositories/", ".git/"),
"app_settings": has_nonempty("app.ini", "app.example.ini"),
"lfs_objects": data_bundle or has_nonempty("data/lfs/", "/lfs/"),
"package_registry": data_bundle or has_nonempty("data/packages/", "/packages/"),
"attachments": data_bundle or has_nonempty("data/attachments/", "/attachments/"),
"actions_logs": data_bundle or has_nonempty("data/actions_log/", "/actions_log/"),
"actions_artifacts": data_bundle or has_nonempty("data/actions_artifacts/", "/actions_artifacts/"),
"avatars": data_bundle or has_nonempty("data/avatars/", "data/repo-avatars/", "/avatars/", "/repo-avatars/"),
"hooks_custom_assets": custom_bundle or has_nonempty("custom/", "/hooks/", "public/"),
}
missing = []
for component in required:
needles = patterns.get(component, (component,))
if not any(any(needle in name for needle in needles) for name in lowered):
missing.append(component)
missing = [component for component in required if not coverage.get(component, False)]
print(f"ENTRY_COUNT={len(names)}")
print(f"ENTRY_COUNT={len(infos)}")
print(f"MISSING_COMPONENTS={','.join(missing)}")
print("OK=1" if not missing and names else "OK=0")
print("OK=1" if not missing and infos else "OK=0")
PY
)"
drill_status=$?

View File

@@ -44,23 +44,13 @@ EXECUTOR_DIGEST=""
VERIFIER_DIGEST=""
STAGE_DIR=""
ROLLBACK_DIR=""
RECEIPT_PATH=""
TERMINAL_RECEIPT_PATH=""
APPLY_STARTED=0
APPLY_COMPLETE=0
COMMIT_CRITICAL=0
DEFERRED_SIGNAL=""
DEFERRED_SIGNAL_EXIT=0
ROLLBACK_ATTEMPTED=0
ROLLBACK_PERFORMED=0
ROLLBACK_VERIFIED=0
RUN_TEMP_RESIDUE_VERIFIED=0
FAULT_INJECTION_ENABLED=0
declare -A FILE_EXISTED=()
declare -A PREVIOUS_DIGEST=()
declare -A PREVIOUS_METADATA=()
declare -A EXPECTED_DIGEST=()
declare -A RUN_OWNED_TEMP_PATHS=()
usage() {
printf '%s\n' \
@@ -106,18 +96,6 @@ case "$MODE" in check|apply|verify) ;; *) fail "invalid_mode" ;; esac
[[ "$RUN_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$ ]] || fail "invalid_run_id"
expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"
[ "$SOURCE_STAGE" = "$expected_source_stage" ] || fail "invalid_source_stage"
RECEIPT_PATH="$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json"
TERMINAL_RECEIPT_PATH="$STATUS_ROOT/agent99-backup-runtime-terminal-${RUN_ID}.json"
if [ "${HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS:-}" = "1" ] \
&& [ "$DEST_ROOT" != "/backup/scripts" ]; then
FAULT_INJECTION_ENABLED=1
fi
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then
case "${HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL:-}" in ""|INT|HUP|TERM) ;; *) fail "invalid_test_commit_signal" ;; esac
case "${HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL:-}" in ""|INT|HUP|TERM) ;; *) fail "invalid_test_terminal_signal" ;; esac
case "${HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT:-}" in ""|write|link|fsync|readback) ;; *) fail "invalid_test_terminal_receipt_fault" ;; esac
case "${HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT:-}" in ""|stage|rollback_prestate) ;; *) fail "invalid_test_cleanup_fault" ;; esac
fi
for command_name in awk bash cp dirname flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do
command -v "$command_name" >/dev/null 2>&1 || fail "required_command_missing_${command_name}"
@@ -196,10 +174,10 @@ working_digest() {
|| fail "verifier_identity_failed"
validate_file() {
local validation_path="$1"
case "$validation_path" in
*.sh) bash -n "$validation_path" ;;
*.py) python3 -c 'import ast, pathlib, sys; ast.parse(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))' "$validation_path" ;;
local path="$1"
case "$path" in
*.sh) bash -n "$path" ;;
*.py) python3 -c 'import ast, pathlib, sys; ast.parse(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))' "$path" ;;
*) return 64 ;;
esac
}
@@ -244,75 +222,12 @@ verify_destination() {
done
}
fsync_paths_and_parents() {
python3 - "$@" <<'PY'
import os
import stat
import sys
from pathlib import Path
parents = set()
for raw_path in sys.argv[1:]:
path = Path(raw_path)
parents.add(path.parent)
if not os.path.lexists(path):
continue
if path.is_symlink() or not path.is_file():
raise SystemExit(76)
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(path, flags)
try:
if not stat.S_ISREG(os.fstat(descriptor).st_mode):
raise SystemExit(76)
os.fsync(descriptor)
finally:
os.close(descriptor)
for parent in sorted(parents, key=str):
if parent.is_symlink() or not parent.is_dir():
raise SystemExit(76)
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(parent, flags)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
PY
}
fsync_destination_state() {
local name dest_path
local -a fsync_targets=()
for name in "${PAYLOAD_FILES[@]}"; do
dest_path="$(destination_path "$name")"
fsync_targets+=("$dest_path")
done
fsync_paths_and_parents "${fsync_targets[@]}"
}
fsync_rollback_prestate() {
local name
local -a fsync_targets=("$ROLLBACK_DIR/prestate.tsv")
for name in "${PAYLOAD_FILES[@]}"; do
if [ "${FILE_EXISTED[$name]:-}" = "1" ]; then
fsync_targets+=("$ROLLBACK_DIR/$name")
fi
done
fsync_paths_and_parents "${fsync_targets[@]}"
}
write_receipt() {
local receipt_status="$1"
local rollback_attempted="$2"
local rollback_performed="$3"
local rollback_verified="$4"
local zero_residue_verified="$5"
install -d -m 700 "$STATUS_ROOT" || return 1
[ -d "$STATUS_ROOT" ] && [ ! -L "$STATUS_ROOT" ] || return 1
[ "$(stat -c '%U:%G:%a' "$STATUS_ROOT")" = "wooo:wooo:700" ] || return 1
if ! python3 - "$RECEIPT_PATH" "$receipt_status" "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" \
"$rollback_attempted" "$rollback_performed" "$rollback_verified" \
"$zero_residue_verified" "$VERIFIER_DIGEST" "$FAULT_INJECTION_ENABLED" <<'PY'
import hashlib
local status="$1"
local rollback_verified="$2"
local receipt_path="$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json"
install -d -m 700 "$STATUS_ROOT"
python3 - "$receipt_path" "$status" "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$rollback_verified" "$VERIFIER_DIGEST" <<'PY'
import json
import os
import sys
@@ -320,343 +235,64 @@ import time
from pathlib import Path
path = Path(sys.argv[1])
status = sys.argv[2]
rollback_attempted = sys.argv[6] == "1"
rollback_performed = sys.argv[7] == "1"
rollback_verified = sys.argv[8] == "1"
zero_residue_verified = sys.argv[9] == "1"
if status not in {"payload_verified", "failed_rolled_back", "rollback_unverified"}:
raise SystemExit(78)
if status == "payload_verified" and (rollback_attempted or rollback_performed or rollback_verified or not zero_residue_verified):
raise SystemExit(78)
if status == "failed_rolled_back" and not (
rollback_attempted and rollback_performed and rollback_verified and zero_residue_verified
):
raise SystemExit(78)
if status == "rollback_unverified" and (not rollback_attempted or rollback_verified):
raise SystemExit(78)
document = {
"schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v3",
"status": status,
"schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v1",
"status": sys.argv[2],
"sourceRevision": sys.argv[3],
"sourceHead": sys.argv[4],
"runId": sys.argv[5],
"fileCount": 18,
"backupScriptFileCount": 17,
"backupHealthExporterIncluded": True,
"rollbackAttempted": rollback_attempted,
"rollbackPerformed": rollback_performed,
"rollbackVerified": rollback_verified,
"zeroResidueVerified": zero_residue_verified,
"zeroResidueScope": "run_owned_destination_temps",
"verifierSha256": sys.argv[10],
"rollbackPerformed": sys.argv[6] == "1",
"rollbackVerified": sys.argv[6] == "1",
"verifierSha256": sys.argv[7],
"executorHost": "192.168.0.110",
"productionServiceRestarted": False,
"secretValuesRead": False,
"writtenAt": int(time.time()),
}
payload = (json.dumps(document, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")
expected_digest = hashlib.sha256(payload).hexdigest()
temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
fault_enabled = sys.argv[11] == "1"
fault = (
os.environ.get("HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT", "")
if status == "payload_verified" and fault_enabled
else ""
)
if fault not in {"", "write", "link", "fsync", "readback"}:
raise SystemExit(78)
created_final = False
directory_fd = None
temporary.write_text(json.dumps(document, ensure_ascii=True, sort_keys=True) + "\n", encoding="utf-8")
os.chmod(temporary, 0o600)
try:
if os.path.lexists(path) or os.path.lexists(temporary):
raise FileExistsError(path)
if fault == "write":
raise OSError("fault_injected_receipt_write")
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "wb") as handle:
if handle.write(payload) != len(payload):
raise OSError("receipt_short_write")
handle.flush()
if fault == "fsync":
raise OSError("fault_injected_receipt_fsync")
os.fsync(handle.fileno())
os.chmod(temporary, 0o600)
temporary_readback = temporary.read_bytes()
if temporary_readback != payload or hashlib.sha256(temporary_readback).hexdigest() != expected_digest:
raise OSError("receipt_temporary_readback_failed")
if fault == "link":
raise OSError("fault_injected_receipt_link")
os.link(temporary, path)
created_final = True
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
directory_fd = os.open(path.parent, directory_flags)
os.fsync(directory_fd)
if fault == "readback":
raise OSError("fault_injected_receipt_readback")
readback = path.read_bytes()
if readback != payload or hashlib.sha256(readback).hexdigest() != expected_digest:
raise OSError("receipt_final_readback_failed")
if json.loads(readback.decode("utf-8")) != document:
raise OSError("receipt_document_readback_failed")
temporary.unlink()
os.fsync(directory_fd)
except BaseException:
try:
if created_final:
path.unlink(missing_ok=True)
temporary.unlink(missing_ok=True)
if directory_fd is None and path.parent.is_dir():
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
directory_fd = os.open(path.parent, directory_flags)
if directory_fd is not None:
os.fsync(directory_fd)
finally:
raise
finally:
if directory_fd is not None:
os.close(directory_fd)
temporary.unlink(missing_ok=True)
PY
then
return 1
fi
[ -f "$RECEIPT_PATH" ] && [ ! -L "$RECEIPT_PATH" ] || return 1
return 0
}
write_terminal_receipt() {
local terminal_status="$1"
local stage_cleanup_verified="$2"
local rollback_prestate_cleanup_verified="$3"
local deferred_signal="$4"
local terminal_exit_code="$5"
install -d -m 700 "$STATUS_ROOT" || return 1
[ -d "$STATUS_ROOT" ] && [ ! -L "$STATUS_ROOT" ] || return 1
[ "$(stat -c '%U:%G:%a' "$STATUS_ROOT")" = "wooo:wooo:700" ] || return 1
if ! python3 - "$TERMINAL_RECEIPT_PATH" "$RECEIPT_PATH" "$terminal_status" \
"$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$stage_cleanup_verified" \
"$rollback_prestate_cleanup_verified" "$deferred_signal" "$terminal_exit_code" \
"$VERIFIER_DIGEST" "$FAULT_INJECTION_ENABLED" <<'PY'
import hashlib
import json
import os
import signal
import sys
import time
from pathlib import Path
path = Path(sys.argv[1])
payload_receipt_path = Path(sys.argv[2])
status = sys.argv[3]
source_revision = sys.argv[4]
source_head = sys.argv[5]
run_id = sys.argv[6]
stage_cleanup_verified = sys.argv[7] == "1"
rollback_prestate_cleanup_verified = sys.argv[8] == "1"
deferred_signal = sys.argv[9]
terminal_exit_code = int(sys.argv[10])
verifier_digest = sys.argv[11]
fault_enabled = sys.argv[12] == "1"
signal_exit_codes = {"INT": 130, "HUP": 129, "TERM": 143}
terminal_signal = os.environ.get("HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL", "") if fault_enabled else ""
terminal_fault = os.environ.get("HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT", "") if fault_enabled else ""
if terminal_signal:
os.kill(os.getppid(), getattr(signal, f"SIG{terminal_signal}"))
time.sleep(0.05)
try:
payload_receipt = json.loads(payload_receipt_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
raise SystemExit(79)
if not (
payload_receipt.get("schemaVersion") == "agent99_host110_backup_runtime_deploy_receipt_v3"
and payload_receipt.get("status") == "payload_verified"
and payload_receipt.get("sourceRevision") == source_revision
and payload_receipt.get("sourceHead") == source_head
and payload_receipt.get("runId") == run_id
and payload_receipt.get("zeroResidueVerified") is True
):
raise SystemExit(79)
if status == "verified":
valid = (
stage_cleanup_verified
and rollback_prestate_cleanup_verified
and not deferred_signal
and terminal_exit_code == 0
)
elif status == "cleanup_pending":
valid = (
not (stage_cleanup_verified and rollback_prestate_cleanup_verified)
and deferred_signal in {"", *signal_exit_codes}
and terminal_exit_code != 0
)
elif status == "committed_signal_deferred":
valid = (
stage_cleanup_verified
and rollback_prestate_cleanup_verified
and deferred_signal in signal_exit_codes
and terminal_exit_code == signal_exit_codes[deferred_signal]
)
else:
valid = False
if not valid:
raise SystemExit(79)
document = {
"schemaVersion": "agent99_host110_backup_runtime_terminal_receipt_v1",
"status": status,
"ok": status == "verified",
"payloadCommitted": True,
"payloadReceipt": str(payload_receipt_path),
"sourceRevision": source_revision,
"sourceHead": source_head,
"runId": run_id,
"fileCount": 18,
"backupScriptFileCount": 17,
"backupHealthExporterIncluded": True,
"stageCleanupVerified": stage_cleanup_verified,
"rollbackPrestateCleanupVerified": rollback_prestate_cleanup_verified,
"cleanupVerified": stage_cleanup_verified and rollback_prestate_cleanup_verified,
"independentVerifierVerified": True,
"zeroResidueScope": "run_owned_destination_temps_and_internal_stage_prestate",
"deferredSignal": deferred_signal or None,
"terminalExitCode": terminal_exit_code,
"verifierSha256": verifier_digest,
"executorHost": "192.168.0.110",
"productionServiceRestarted": False,
"secretValuesRead": False,
"writtenAt": int(time.time()),
}
payload = (json.dumps(document, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")
expected_digest = hashlib.sha256(payload).hexdigest()
temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
created_final = False
directory_fd = None
try:
if os.path.lexists(path) or os.path.lexists(temporary):
raise FileExistsError(path)
if terminal_fault == "write":
raise OSError("fault_injected_terminal_receipt_write")
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "wb") as handle:
if handle.write(payload) != len(payload):
raise OSError("terminal_receipt_short_write")
handle.flush()
if terminal_fault == "fsync":
raise OSError("fault_injected_terminal_receipt_fsync")
os.fsync(handle.fileno())
os.chmod(temporary, 0o600)
temporary_readback = temporary.read_bytes()
if temporary_readback != payload or hashlib.sha256(temporary_readback).hexdigest() != expected_digest:
raise OSError("terminal_receipt_temporary_readback_failed")
if terminal_fault == "link":
raise OSError("fault_injected_terminal_receipt_link")
os.link(temporary, path)
created_final = True
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
directory_fd = os.open(path.parent, directory_flags)
os.fsync(directory_fd)
if terminal_fault == "readback":
raise OSError("fault_injected_terminal_receipt_readback")
readback = path.read_bytes()
if readback != payload or hashlib.sha256(readback).hexdigest() != expected_digest:
raise OSError("terminal_receipt_final_readback_failed")
if json.loads(readback.decode("utf-8")) != document:
raise OSError("terminal_receipt_document_readback_failed")
temporary.unlink()
os.fsync(directory_fd)
except BaseException:
try:
if created_final:
path.unlink(missing_ok=True)
temporary.unlink(missing_ok=True)
if directory_fd is None and path.parent.is_dir():
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
directory_fd = os.open(path.parent, directory_flags)
if directory_fd is not None:
os.fsync(directory_fd)
finally:
raise
finally:
if directory_fd is not None:
os.close(directory_fd)
PY
then
return 1
fi
[ -f "$TERMINAL_RECEIPT_PATH" ] && [ ! -L "$TERMINAL_RECEIPT_PATH" ] || return 1
return 0
printf '%s' "$receipt_path"
}
record_prestate() {
local name dest_path digest metadata
: > "$ROLLBACK_DIR/prestate.tsv" || return 74
chmod 600 "$ROLLBACK_DIR/prestate.tsv" || return 74
: > "$ROLLBACK_DIR/prestate.tsv"
chmod 600 "$ROLLBACK_DIR/prestate.tsv"
for name in "${PAYLOAD_FILES[@]}"; do
dest_path="$(destination_path "$name")"
if [ -f "$dest_path" ] && [ ! -L "$dest_path" ]; then
[ "$(stat -c '%U:%G' "$dest_path")" = "wooo:wooo" ] || return 71
FILE_EXISTED[$name]=1
digest="$(working_digest "$dest_path")" || return 72
metadata="$(stat -c '%u:%g:%a' "$dest_path")" || return 72
digest="$(working_digest "$dest_path")"
metadata="$(stat -c '%u:%g:%a' "$dest_path")"
PREVIOUS_DIGEST[$name]="$digest"
PREVIOUS_METADATA[$name]="$metadata"
printf '%s\t1\t%s\t%s\n' "$name" "$digest" "$metadata" >> "$ROLLBACK_DIR/prestate.tsv" || return 72
cp -p -- "$dest_path" "$ROLLBACK_DIR/$name" || return 72
printf '%s\t1\t%s\t%s\n' "$name" "$digest" "$metadata" >> "$ROLLBACK_DIR/prestate.tsv"
cp -p -- "$dest_path" "$ROLLBACK_DIR/$name"
[ "$(working_digest "$ROLLBACK_DIR/$name")" = "$digest" ] || return 72
elif [ ! -e "$dest_path" ]; then
FILE_EXISTED[$name]=0
PREVIOUS_DIGEST[$name]="-"
PREVIOUS_METADATA[$name]="-"
printf '%s\t0\t-\t-\n' "$name" >> "$ROLLBACK_DIR/prestate.tsv" || return 72
printf '%s\t0\t-\t-\n' "$name" >> "$ROLLBACK_DIR/prestate.tsv"
else
return 73
fi
done
fsync_rollback_prestate || return 75
}
register_run_owned_temp() {
RUN_OWNED_TEMP_PATHS["$1"]=1
}
verify_run_owned_temp_absence() {
local temporary
for temporary in "${!RUN_OWNED_TEMP_PATHS[@]}"; do
[ ! -e "$temporary" ] && [ ! -L "$temporary" ] || return 1
done
}
cleanup_run_owned_temps() {
local temporary failed=0 preserved=0
local fault=""
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then
fault="${HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT:-}"
fi
for temporary in "${!RUN_OWNED_TEMP_PATHS[@]}"; do
if [ "$fault" = "preserve_first_temp" ] \
&& [ "$preserved" -eq 0 ] \
&& { [ -e "$temporary" ] || [ -L "$temporary" ]; }; then
preserved=1
failed=1
continue
fi
rm -f -- "$temporary" || failed=1
done
verify_run_owned_temp_absence || failed=1
[ "$failed" -eq 0 ]
}
rollback_transaction() {
local name dest_path destination_parent backup_path temporary failed=0
ROLLBACK_ATTEMPTED=1
ROLLBACK_PERFORMED=0
ROLLBACK_VERIFIED=0
RUN_TEMP_RESIDUE_VERIFIED=0
set +e
cleanup_run_owned_temps || failed=1
for name in "${PAYLOAD_FILES[@]}"; do
[ -n "${FILE_EXISTED[$name]+x}" ] || { failed=1; continue; }
dest_path="$(destination_path "$name")"
@@ -664,22 +300,13 @@ rollback_transaction() {
backup_path="$ROLLBACK_DIR/$name"
destination_parent="$(dirname "$dest_path")"
temporary="$destination_parent/.${name}.agent99-rollback-${RUN_ID}"
register_run_owned_temp "$temporary"
if cp -p -- "$backup_path" "$temporary" && mv -f -- "$temporary" "$dest_path"; then
ROLLBACK_PERFORMED=1
else
failed=1
fi
cp -p -- "$backup_path" "$temporary" \
&& mv -f -- "$temporary" "$dest_path" \
|| failed=1
else
if rm -f -- "$dest_path"; then
ROLLBACK_PERFORMED=1
else
failed=1
fi
rm -f -- "$dest_path" || failed=1
fi
done
cleanup_run_owned_temps || failed=1
fsync_destination_state || failed=1
for name in "${PAYLOAD_FILES[@]}"; do
dest_path="$(destination_path "$name")"
if [ "${FILE_EXISTED[$name]:-}" = "1" ]; then
@@ -693,14 +320,8 @@ rollback_transaction() {
failed=1
fi
done
if verify_run_owned_temp_absence; then
RUN_TEMP_RESIDUE_VERIFIED=1
else
failed=1
RUN_TEMP_RESIDUE_VERIFIED=0
fi
set -e
if [ "$failed" -eq 0 ] && [ "$ROLLBACK_PERFORMED" -eq 1 ] && [ "$RUN_TEMP_RESIDUE_VERIFIED" -eq 1 ]; then
if [ "$failed" -eq 0 ]; then
ROLLBACK_VERIFIED=1
return 0
fi
@@ -708,78 +329,37 @@ rollback_transaction() {
return 1
}
handle_agent99_signal() {
local signal_name="$1"
local signal_exit_code="$2"
if [ "$COMMIT_CRITICAL" -eq 1 ] || [ "$APPLY_COMPLETE" -eq 1 ]; then
if [ -z "$DEFERRED_SIGNAL" ]; then
DEFERRED_SIGNAL="$signal_name"
DEFERRED_SIGNAL_EXIT="$signal_exit_code"
fi
return 0
fi
exit "$signal_exit_code"
}
cleanup_postcommit_directory() {
local cleanup_kind="$1"
local cleanup_path="$2"
local fault=""
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then
fault="${HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT:-}"
fi
if [ "$fault" = "$cleanup_kind" ]; then
return 1
fi
rm -rf -- "$cleanup_path" || return 1
[ ! -e "$cleanup_path" ] && [ ! -L "$cleanup_path" ] || return 1
fsync_paths_and_parents "$cleanup_path" || return 1
[ ! -e "$cleanup_path" ] && [ ! -L "$cleanup_path" ]
}
cleanup() {
local exit_status=$?
local status=$?
local rollback_status="rollback_unverified"
local failure_receipt_written=0
trap - EXIT HUP INT TERM
if [ "$APPLY_STARTED" -eq 1 ] && [ "$APPLY_COMPLETE" -ne 1 ]; then
if rollback_transaction; then
rollback_status="failed_rolled_back"
fi
if write_receipt "$rollback_status" "$ROLLBACK_ATTEMPTED" "$ROLLBACK_PERFORMED" \
"$ROLLBACK_VERIFIED" "$RUN_TEMP_RESIDUE_VERIFIED"; then
failure_receipt_written=1
fi
write_receipt "$rollback_status" "$ROLLBACK_VERIFIED" >/dev/null || true
fi
if [ "$APPLY_COMPLETE" -ne 1 ] && [ -n "$STAGE_DIR" ]; then
rm -rf -- "$STAGE_DIR" || true
[ -z "$STAGE_DIR" ] || rm -rf -- "$STAGE_DIR"
if [ "$ROLLBACK_VERIFIED" -eq 1 ]; then
[ -z "$ROLLBACK_DIR" ] || rm -rf -- "$ROLLBACK_DIR"
fi
if [ "$APPLY_COMPLETE" -ne 1 ] \
&& [ "$ROLLBACK_VERIFIED" -eq 1 ] \
&& [ "$failure_receipt_written" -eq 1 ]; then
if [ -n "$ROLLBACK_DIR" ]; then
rm -rf -- "$ROLLBACK_DIR" || true
fi
fi
exit "$exit_status"
exit "$status"
}
validate_source || fail "source_validation_failed"
if [ "$MODE" = "check" ]; then
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","runId":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID"
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
exit 0
fi
if [ "$MODE" = "verify" ]; then
verify_destination || fail "destination_verification_failed"
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","runId":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID"
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
exit 0
fi
[ ! -e "$RECEIPT_PATH" ] && [ ! -L "$RECEIPT_PATH" ] || fail "run_identity_receipt_exists"
[ ! -e "$TERMINAL_RECEIPT_PATH" ] && [ ! -L "$TERMINAL_RECEIPT_PATH" ] \
|| fail "run_identity_terminal_receipt_exists"
[ ! -e "$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json" ] || fail "run_identity_receipt_exists"
STAGE_DIR="$STAGE_ROOT/$RUN_ID"
ROLLBACK_DIR="$ROLLBACK_ROOT/$RUN_ID"
[ ! -e "$STAGE_DIR" ] || fail "run_identity_stage_exists"
@@ -800,9 +380,8 @@ fi
install -d -m 700 "$STAGE_DIR" "$ROLLBACK_DIR"
trap cleanup EXIT
trap 'handle_agent99_signal INT 130' INT
trap 'handle_agent99_signal HUP 129' HUP
trap 'handle_agent99_signal TERM 143' TERM
trap 'exit 130' INT
trap 'exit 143' TERM HUP
for name in "${PAYLOAD_FILES[@]}"; do
install -m 700 "$SOURCE_STAGE/$name" "$STAGE_DIR/$name"
@@ -820,17 +399,10 @@ for name in "${PAYLOAD_FILES[@]}"; do
dest_path="$(destination_path "$name")"
destination_parent="$(dirname "$dest_path")"
temporary="$destination_parent/.${name}.agent99-${RUN_ID}"
register_run_owned_temp "$temporary"
install -m 755 "$STAGE_DIR/$name" "$temporary"
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ] \
&& [ "${HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT:-}" = "after_temp_install" ] \
&& [ "$name" = "${PAYLOAD_FILES[0]}" ]; then
fail "fault_injected_after_temp_install"
fi
mv -f -- "$temporary" "$dest_path"
done
fsync_destination_state || fail "post_apply_durability_failed"
verify_destination || fail "post_apply_verification_failed"
verifier_result="$(BACKUP_RUNTIME_DEPLOY_CONTEXT=1 python3 "$SOURCE_STAGE/verify-host110-backup-runtime.py" \
--manifest "$SOURCE_STAGE/manifest.json" \
@@ -861,118 +433,27 @@ if not (
raise SystemExit(1)
PY
verify_run_owned_temp_absence || fail "post_apply_temp_residue_detected"
RUN_TEMP_RESIDUE_VERIFIED=1
COMMIT_CRITICAL=1
if ! write_receipt "payload_verified" 0 0 0 "$RUN_TEMP_RESIDUE_VERIFIED"; then
COMMIT_CRITICAL=0
fail "durable_receipt_failed"
fi
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ] \
&& [ -n "${HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL:-}" ]; then
case "$HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL" in
INT|HUP|TERM) kill -s "$HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL" "$$" ;;
esac
fi
APPLY_COMPLETE=1
COMMIT_CRITICAL=0
stage_cleanup_verified=0
rollback_prestate_cleanup_verified=0
if cleanup_postcommit_directory "stage" "$STAGE_DIR"; then
stage_cleanup_verified=1
fi
if cleanup_postcommit_directory "rollback_prestate" "$ROLLBACK_DIR"; then
rollback_prestate_cleanup_verified=1
fi
terminal_status="verified"
terminal_exit_code=0
# Cleanup has completed. Freeze terminal signal state before the immutable
# terminal writer so its receipt, stdout, and exit status cannot diverge.
trap '' HUP INT TERM
if [ "$stage_cleanup_verified" -ne 1 ] || [ "$rollback_prestate_cleanup_verified" -ne 1 ]; then
terminal_status="cleanup_pending"
terminal_exit_code=75
elif [ -n "$DEFERRED_SIGNAL" ]; then
terminal_status="committed_signal_deferred"
terminal_exit_code="$DEFERRED_SIGNAL_EXIT"
fi
if ! write_terminal_receipt "$terminal_status" "$stage_cleanup_verified" \
"$rollback_prestate_cleanup_verified" "$DEFERRED_SIGNAL" "$terminal_exit_code"; then
trap - EXIT
python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$RECEIPT_PATH" \
"$TERMINAL_RECEIPT_PATH" "$stage_cleanup_verified" \
"$rollback_prestate_cleanup_verified" "$DEFERRED_SIGNAL" <<'PY'
receipt_path="$(write_receipt "verified" 0)"
rm -rf -- "$STAGE_DIR" "$ROLLBACK_DIR"
STAGE_DIR=""
ROLLBACK_DIR=""
trap - EXIT HUP INT TERM
python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$receipt_path" "$verifier_result" <<'PY'
import json
import sys
print(json.dumps({
"schemaVersion": "agent99_host110_backup_runtime_executor_v1",
"mode": "apply",
"status": "terminal_receipt_failed",
"ok": False,
"ok": True,
"sourceRevision": sys.argv[1],
"sourceHead": sys.argv[2],
"runId": sys.argv[3],
"fileCount": 18,
"backupScriptFileCount": 17,
"backupHealthExporterIncluded": True,
"payloadCommitted": True,
"rollbackAttempted": False,
"rollbackPerformed": False,
"rollbackVerified": False,
"zeroResidueVerified": True,
"zeroResidueScope": "run_owned_destination_temps",
"stageCleanupVerified": sys.argv[6] == "1",
"rollbackPrestateCleanupVerified": sys.argv[7] == "1",
"receipt": sys.argv[4],
"terminalReceipt": sys.argv[5],
"terminalReceiptPublished": False,
"terminalExitCode": 76,
"deferredSignal": sys.argv[8] or None,
"independentVerifierVerified": True,
"verifier": None,
}, ensure_ascii=True, sort_keys=True))
PY
printf 'HOST110_BACKUP_RUNTIME_OK=0\nERROR=terminal_receipt_failed\n' >&2
exit 76
fi
if [ "$stage_cleanup_verified" -eq 1 ]; then STAGE_DIR=""; fi
if [ "$rollback_prestate_cleanup_verified" -eq 1 ]; then ROLLBACK_DIR=""; fi
trap - EXIT
python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$RECEIPT_PATH" "$TERMINAL_RECEIPT_PATH" \
"$verifier_result" "$stage_cleanup_verified" "$rollback_prestate_cleanup_verified" \
"$terminal_status" "$terminal_exit_code" "$DEFERRED_SIGNAL" "$RUN_ID" <<'PY'
import json
import sys
status = sys.argv[8]
verifier = json.loads(sys.argv[5])
print(json.dumps({
"schemaVersion": "agent99_host110_backup_runtime_executor_v1",
"mode": "apply",
"status": status,
"ok": status == "verified",
"sourceRevision": sys.argv[1],
"sourceHead": sys.argv[2],
"runId": sys.argv[11],
"fileCount": 18,
"backupScriptFileCount": 17,
"backupHealthExporterIncluded": True,
"payloadCommitted": True,
"rollbackAttempted": False,
"rollbackPerformed": False,
"rollbackVerified": False,
"zeroResidueVerified": True,
"zeroResidueScope": "run_owned_destination_temps",
"stageCleanupVerified": sys.argv[6] == "1",
"rollbackPrestateCleanupVerified": sys.argv[7] == "1",
"receipt": sys.argv[3],
"terminalReceipt": sys.argv[4],
"terminalReceiptPublished": True,
"terminalExitCode": int(sys.argv[9]),
"deferredSignal": sys.argv[10] or None,
"independentVerifierVerified": True,
"verifier": verifier if status == "verified" else None,
"verifier": json.loads(sys.argv[4]),
}, ensure_ascii=True, sort_keys=True))
PY
exit "$terminal_exit_code"

View File

@@ -18,6 +18,9 @@ EXPECTED_FILES = {
"backup-host188-products.sh",
"verify-host188-products-backup.sh",
"verify-host188-products-archive.py",
"backup-gitea.sh",
"gitea-full-backup-restore-drill.sh",
"backup-configs.sh",
"check-backup-integrity.sh",
"sync-offsite-backups.sh",
"backup-offsite-readiness-gate.sh",
@@ -26,7 +29,7 @@ EXPECTED_FILES = {
}
def test_backup_runtime_deploy_is_fixed_to_host_110_and_fourteen_files() -> None:
def test_backup_runtime_deploy_is_fixed_to_host_110_and_seventeen_files() -> None:
plays = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8"))
assert len(plays) == 1
play = plays[0]

View File

@@ -5,6 +5,7 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
BACKUP_GITEA = ROOT / "scripts" / "backup" / "backup-gitea.sh"
BACKUP_CONFIGS = ROOT / "scripts" / "backup" / "backup-configs.sh"
def test_backup_gitea_writes_full_server_component_receipts() -> None:
@@ -15,7 +16,12 @@ def test_backup_gitea_writes_full_server_component_receipts() -> None:
assert "awoooi_gitea_full_backup_restore_drill_performed" in text
assert "awoooi_gitea_full_backup_secret_value_collected" in text
assert "awoooi_gitea_full_backup_production_restore_performed" in text
assert "gitea dump -c /data/gitea/conf/app.ini" in text
assert "offline_gitea_dump" in text
assert "--volumes-from" in text
assert "--pull=never" in text
assert "docker stop --time" in text
assert "start_and_verify_primary" in text
assert "gitea-full-backup-restore-drill.sh" in text
assert "GITEA_FULL_BACKUP_COMPONENTS" in text
for component in [
@@ -25,6 +31,9 @@ def test_backup_gitea_writes_full_server_component_receipts() -> None:
"lfs_objects",
"package_registry",
"attachments",
"actions_logs",
"actions_artifacts",
"avatars",
"hooks_custom_assets",
]:
assert component in text
@@ -37,7 +46,7 @@ def test_backup_gitea_receipt_contract_stays_no_secret_no_restore() -> None:
assert "awoooi_gitea_full_backup_production_restore_performed" in text
assert "awoooi_gitea_full_backup_restore_drill_performed" in text
assert (
'restore_drill_performed{host=\\"${host}\\",service=\\"${service_label}\\"} 0'
'restore_drill_performed{host=\\"${host}\\",service=\\"${service_label}\\"} ${structural_drill_performed}'
in text
)
assert (
@@ -48,3 +57,34 @@ def test_backup_gitea_receipt_contract_stays_no_secret_no_restore() -> None:
'production_restore_performed{host=\\"${host}\\",service=\\"${service_label}\\"} 0'
in text
)
def test_backup_gitea_is_bounded_and_fail_safe_around_the_offline_window() -> None:
text = BACKUP_GITEA.read_text(encoding="utf-8")
assert 'GITEA_BACKUP_LOCK="${GITEA_BACKUP_LOCK:-/tmp/gitea-backup.lock}"' in text
assert "flock -n 9" in text
assert "trap cleanup EXIT" in text
assert 'if [ "${PRIMARY_STOPPED}" -eq 1 ]' in text
assert "wait_for_primary_health" in text
assert "http://127.0.0.1:3001/api/healthz" in text
assert '--kill-after="${GITEA_DUMP_KILL_AFTER_SECONDS}s"' in text
assert "bounded_docker_control rm -f" in text
assert 'GITEA_MAX_OUTAGE_SECONDS="${GITEA_MAX_OUTAGE_SECONDS:-600}"' in text
assert "configured_outage_bound" in text
assert "--security-opt no-new-privileges:true" in text
assert '--cpus "${BACKUP_DOCKER_CPUS}"' in text
assert '--memory "${BACKUP_DOCKER_MEMORY}"' in text
assert '--memory-swap "${BACKUP_DOCKER_MEMORY_SWAP}"' in text
assert "awoooi_gitea_full_backup_consistent_offline_window" in text
assert "awoooi_gitea_full_backup_service_restart_verified" in text
assert "awoooi_gitea_full_backup_service_outage_seconds" in text
def test_gitea_runner_configuration_is_in_the_encrypted_config_backup() -> None:
text = BACKUP_CONFIGS.read_text(encoding="utf-8")
assert "/home/wooo/vibework-act-runner/config.yaml" in text
assert 'record_config_status "110-gitea-runner-config" true true "110"' in text
assert 'record_config_status "110-gitea-runner-config" true false "110"' in text
assert "/home/wooo/vibework-act-runner/data" not in text

View File

@@ -0,0 +1,300 @@
from __future__ import annotations
import os
import subprocess
import textwrap
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts" / "backup" / "backup-gitea.sh"
def _write_executable(path: Path, source: str) -> None:
path.write_text(textwrap.dedent(source), encoding="utf-8")
path.chmod(0o755)
def _fake_runtime(tmp_path: Path) -> tuple[dict[str, str], Path, Path]:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
state_dir = tmp_path / "state"
state_dir.mkdir()
(state_dir / "running").write_text("true\n", encoding="utf-8")
command_log = state_dir / "commands.log"
_write_executable(
fake_bin / "docker",
r"""
#!/usr/bin/env bash
set -euo pipefail
state="${FAKE_DOCKER_STATE:?}"
printf '%s\n' "$*" >> "$state/commands.log"
command_name="${1:-}"
shift || true
case "$command_name" in
inspect)
args="$*"
case "$args" in
*State.Running*) cat "$state/running" ;;
*NetworkSettings.Networks*) printf 'gitea_gitea\n' ;;
*'{{.Image}}'*) printf 'sha256:%064d\n' 0 ;;
*) exit 64 ;;
esac
;;
exec)
args="$*"
if [[ "$args" == *' id -u'* ]]; then
id -u
elif [[ "$args" == *'gitea --version'* ]]; then
printf 'gitea version 1.25.5 built with test\n'
else
exit 64
fi
;;
stop)
printf 'false\n' > "$state/running"
if [ "${FAKE_DOCKER_STOP_FAIL:-0}" = "1" ]; then
exit 42
fi
;;
start)
printf 'true\n' > "$state/running"
;;
run)
output=""
for arg in "$@"; do
case "$arg" in
type=bind,src=*,dst=/backup-out)
output="${arg#type=bind,src=}"
output="${output%,dst=/backup-out}"
;;
esac
done
[ -n "$output" ] || exit 65
if [ "${FAKE_DOCKER_RUN_FAIL:-0}" = "1" ]; then
exit 42
fi
python3 - "$output/gitea-dump.zip" <<'PY'
import sys
import zipfile
entries = {
"gitea-db.sql": "fixture\n",
"repos/wooo/awoooi.git/HEAD": "ref: refs/heads/main\n",
"custom/conf/app.ini": "[server]\n",
"data/lfs/objects/aa/bb/object": "lfs\n",
"data/packages/package.bin": "package\n",
"data/attachments/attachment.bin": "attachment\n",
"data/actions_artifacts/aa/bb/artifact.zip": "artifact\n",
"data/actions_log/aa/bb/job.log": "log\n",
"data/avatars/aa/bb/avatar.png": "avatar\n",
"custom/hooks/pre-receive": "#!/bin/sh\n",
}
with zipfile.ZipFile(sys.argv[1], "w") as archive:
for name, value in entries.items():
archive.writestr(name, value)
PY
;;
rm) ;;
*) exit 64 ;;
esac
""",
)
_write_executable(
fake_bin / "curl",
r"""
#!/usr/bin/env bash
url="${!#}"
if [ "$url" = "http://127.0.0.1:3001/api/healthz" ]; then
printf '{"status":"pass"}'
else
printf '<html>another service</html>'
fi
""",
)
_write_executable(
fake_bin / "timeout",
"""
#!/usr/bin/env bash
set -euo pipefail
while [[ "${1:-}" == --* ]]; do shift; done
shift
if [ "${FAKE_DOCKER_RUN_TIMEOUT:-0}" = "1" ] && [[ " $* " == *' docker run '* ]]; then
exit 124
fi
exec "$@"
""",
)
_write_executable(
fake_bin / "flock",
"""
#!/usr/bin/env bash
exit 0
""",
)
_write_executable(
fake_bin / "restic",
r"""
#!/usr/bin/env bash
set -euo pipefail
repo=""
args=("$@")
for ((i=0; i<${#args[@]}; i++)); do
if [ "${args[$i]}" = "-r" ]; then repo="${args[$((i+1))]}"; fi
done
if [[ " $* " == *' init '* ]]; then
mkdir -p "$repo/data" "$repo/snapshots"
exit 0
fi
if [[ " $* " == *' backup '* ]]; then
mkdir -p "$repo/data" "$repo/snapshots"
exit 0
fi
if [[ " $* " == *' snapshots '* ]]; then
printf '[{"short_id":"abcd1234","time":"2026-07-18T00:00:00Z"}]\n'
exit 0
fi
if [[ " $* " == *' forget '* ]]; then exit 0; fi
exit 64
""",
)
backup_root = tmp_path / "backup"
(backup_root / "scripts").mkdir(parents=True)
(backup_root / "scripts" / ".restic-password").write_text("fixture\n", encoding="utf-8")
textfile = tmp_path / "gitea.prom"
environment = os.environ | {
"PATH": f"{fake_bin}:{os.environ['PATH']}",
"BACKUP_BASE": str(backup_root),
"BACKUP_LOG_DIR": str(backup_root / "logs"),
"RESTIC_PASSWORD_FILE": str(backup_root / "scripts" / ".restic-password"),
"GITEA_FULL_BACKUP_RECEIPT_TEXTFILE": str(textfile),
"NODE_EXPORTER_TEXTFILE_DIR": str(tmp_path),
"GITEA_BACKUP_LOCK": str(tmp_path / "gitea.lock"),
"GITEA_START_TIMEOUT_SECONDS": "4",
"GITEA_DUMP_TIMEOUT_SECONDS": "30",
"GITEA_DUMP_KILL_AFTER_SECONDS": "2",
"GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS": "2",
"GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS": "1",
"GITEA_STOP_TIMEOUT_SECONDS": "2",
"FAKE_DOCKER_STATE": str(state_dir),
"BACKUP_RETENTION_MODE": "latest",
}
return environment, command_log, textfile
def test_offline_backup_restarts_and_verifies_primary_on_success(tmp_path: Path) -> None:
environment, command_log, textfile = _fake_runtime(tmp_path)
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode == 0, result.stderr
commands = command_log.read_text(encoding="utf-8")
assert commands.index("stop --time 2 gitea") < commands.index("run --pull=never")
assert commands.index("run --pull=never") < commands.index("rm -f agent99-gitea-backup-")
assert commands.index("rm -f agent99-gitea-backup-") < commands.index("start gitea")
rendered = textfile.read_text(encoding="utf-8")
assert "awoooi_gitea_full_backup_consistent_offline_window" in rendered
assert "awoooi_gitea_full_backup_service_restart_verified" in rendered
assert 'service="gitea"} 1' in rendered
def test_offline_backup_restarts_primary_when_dump_fails(tmp_path: Path) -> None:
environment, command_log, textfile = _fake_runtime(tmp_path)
environment["FAKE_DOCKER_RUN_FAIL"] = "1"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
assert (Path(environment["FAKE_DOCKER_STATE"]) / "running").read_text(encoding="utf-8").strip() == "true"
commands = command_log.read_text(encoding="utf-8")
assert "stop --time 2 gitea" in commands
assert commands.index("rm -f agent99-gitea-backup-") < commands.index("start gitea")
rendered = textfile.read_text(encoding="utf-8")
assert 'awoooi_gitea_full_backup_service_restart_verified{host="110",service="gitea"} 1' in rendered
assert 'status="offline_dump_failed"} 1' in rendered
def test_offline_backup_restarts_primary_when_dump_hits_hard_timeout(tmp_path: Path) -> None:
environment, command_log, textfile = _fake_runtime(tmp_path)
environment["FAKE_DOCKER_RUN_TIMEOUT"] = "1"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
assert (Path(environment["FAKE_DOCKER_STATE"]) / "running").read_text(encoding="utf-8").strip() == "true"
commands = command_log.read_text(encoding="utf-8")
assert commands.index("rm -f agent99-gitea-backup-") < commands.index("start gitea")
assert 'status="offline_dump_failed"} 1' in textfile.read_text(encoding="utf-8")
def test_offline_backup_recovers_when_stop_has_side_effect_then_fails(tmp_path: Path) -> None:
environment, command_log, textfile = _fake_runtime(tmp_path)
environment["FAKE_DOCKER_STOP_FAIL"] = "1"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
assert (Path(environment["FAKE_DOCKER_STATE"]) / "running").read_text(encoding="utf-8").strip() == "true"
commands = command_log.read_text(encoding="utf-8")
assert commands.index("stop --time 2 gitea") < commands.index("start gitea")
assert 'service_restart_verified{host="110",service="gitea"} 1' in textfile.read_text(encoding="utf-8")
def test_wrong_service_health_response_cannot_open_offline_window(tmp_path: Path) -> None:
environment, command_log, _ = _fake_runtime(tmp_path)
environment["GITEA_LOCAL_HEALTH_URL"] = "http://127.0.0.1:3000/"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
commands = command_log.read_text(encoding="utf-8")
assert "stop --time" not in commands
def test_oversized_outage_budget_fails_before_primary_stop(tmp_path: Path) -> None:
environment, command_log, _ = _fake_runtime(tmp_path)
environment["GITEA_MAX_OUTAGE_SECONDS"] = "10"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
assert "Configured Gitea backup outage bound exceeds 10s" in result.stdout
assert not command_log.exists() or "stop --time" not in command_log.read_text(encoding="utf-8")

View File

@@ -9,15 +9,25 @@ ROOT = Path(__file__).resolve().parents[1]
SCRIPT = ROOT / "gitea-full-backup-restore-drill.sh"
def _write_dump(path: Path, *, include_lfs: bool = True) -> None:
def _write_dump(
path: Path,
*,
include_lfs: bool = True,
include_actions_artifact: bool = True,
empty_critical_payloads: bool = False,
) -> None:
entries = {
"gitea-db.sql": "-- redacted fixture\n",
"repos/wooo/awoooi.git/HEAD": "ref: refs/heads/main\n",
"gitea-db.sql": "" if empty_critical_payloads else "-- redacted fixture\n",
"repos/wooo/awoooi.git/HEAD": "" if empty_critical_payloads else "ref: refs/heads/main\n",
"custom/conf/app.ini": "[server]\nAPP_NAME=fixture\n",
"data/packages/package.bin": "package fixture\n",
"data/attachments/attachment.bin": "attachment fixture\n",
"data/actions_log/aa/bb/job.log": "actions log fixture\n",
"data/avatars/aa/bb/avatar.png": "avatar fixture\n",
"custom/hooks/pre-receive": "#!/bin/sh\n",
}
if include_actions_artifact:
entries["data/actions_artifacts/aa/bb/artifact.zip"] = "actions artifact fixture\n"
if include_lfs:
entries["data/lfs/objects/aa/bb/object"] = "lfs fixture\n"
with zipfile.ZipFile(path, "w") as archive:
@@ -97,3 +107,64 @@ def test_gitea_full_backup_restore_drill_fails_closed_when_component_missing(
rendered = textfile.read_text(encoding="utf-8")
assert 'component="lfs_objects",drill_scope="structural_metadata_only"} 0' in rendered
assert 'production_restore_performed{host="110",service="gitea",' in rendered
def test_gitea_full_backup_restore_drill_rejects_empty_database_and_repository(
tmp_path: Path,
) -> None:
dump = tmp_path / "gitea-dump.zip"
_write_dump(dump, empty_critical_payloads=True)
result = subprocess.run(
["bash", str(SCRIPT), "--dump-zip", str(dump)],
text=True,
capture_output=True,
)
assert result.returncode == 1
assert "ERROR=component_coverage_gap" in result.stdout
def test_gitea_full_backup_restore_drill_requires_actions_log_and_artifact(
tmp_path: Path,
) -> None:
dump = tmp_path / "gitea-dump.zip"
textfile = tmp_path / "gitea_full_backup_restore_drill.prom"
_write_dump(dump, include_actions_artifact=False)
result = subprocess.run(
[
"bash",
str(SCRIPT),
"--dump-zip",
str(dump),
"--write-textfile",
"--textfile",
str(textfile),
],
text=True,
capture_output=True,
)
assert result.returncode == 1
rendered = textfile.read_text(encoding="utf-8")
assert 'component="actions_logs",drill_scope="structural_metadata_only"} 1' not in rendered
assert 'component="actions_artifacts",drill_scope="structural_metadata_only"} 0' in rendered
def test_gitea_full_backup_restore_drill_rejects_crc_corruption(tmp_path: Path) -> None:
dump = tmp_path / "gitea-dump.zip"
_write_dump(dump)
payload = dump.read_bytes()
fixture = b"-- redacted fixture\n"
assert fixture in payload
dump.write_bytes(payload.replace(fixture, b"X" + fixture[1:], 1))
result = subprocess.run(
["bash", str(SCRIPT), "--dump-zip", str(dump)],
text=True,
capture_output=True,
)
assert result.returncode == 1
assert "ERROR=crc_failed" in result.stdout

View File

@@ -1,220 +0,0 @@
$ErrorActionPreference = "Stop"
function Write-ReplayTrace {
param([string]$Step)
if ($ReplayTrace) { Write-Output "REPLAY_STEP=$Step" }
}
if (-not $BrokerGzipBase64) { throw "broker_payload_missing" }
Write-ReplayTrace "decode_source"
$compressed = [Convert]::FromBase64String([string]$BrokerGzipBase64)
$inputStream = New-Object IO.MemoryStream(,$compressed)
$gzip = New-Object IO.Compression.GzipStream(
$inputStream,
[IO.Compression.CompressionMode]::Decompress
)
$reader = New-Object IO.StreamReader($gzip, [Text.Encoding]::UTF8)
try {
$brokerSource = $reader.ReadToEnd()
} finally {
$reader.Dispose()
$gzip.Dispose()
$inputStream.Dispose()
}
$tokens = $null
$parseErrors = $null
$ast = [Management.Automation.Language.Parser]::ParseInput(
$brokerSource,
[ref]$tokens,
[ref]$parseErrors
)
if ($parseErrors.Count -ne 0) { throw "broker_parse_failed" }
Write-ReplayTrace "define_functions"
$functionNames = @(
"Test-Agent99JsonField",
"Assert-Agent99JsonFields",
"Convert-Agent99ExecutorDocument",
"Assert-Agent99ApplyResultFields",
"Assert-Agent99VerifierFields",
"Get-Agent99ExecutorResult",
"Get-Agent99CommittedFailureResult"
)
foreach ($functionName in $functionNames) {
$node = $ast.Find(
{
param($candidate)
$candidate -is [Management.Automation.Language.FunctionDefinitionAst] -and
$candidate.Name -eq $functionName
},
$true
)
if ($null -eq $node) { throw "broker_function_missing_$functionName" }
Invoke-Expression $node.Extent.Text
}
$SourceRevision = "a" * 40
$RunId = "windows99-contract-replay"
$ExpectedFileCount = 18
$sourceHead = "b" * 40
$payloadReceipt = "/backup/status/agent99-backup-runtime-deploy-$RunId.json"
$terminalReceipt = "/backup/status/agent99-backup-runtime-terminal-$RunId.json"
function Copy-ReplayDocument {
param([object]$Document)
return ($Document | ConvertTo-Json -Compress -Depth 12 | ConvertFrom-Json)
}
function New-ReplayTransport {
param([object]$Document, [int]$ExitCode)
return [pscustomobject]@{
ok = [bool]($ExitCode -eq 0)
exitCode = $ExitCode
reason = if ($ExitCode -eq 0) { "completed" } else { "process_failed" }
stdout = $Document | ConvertTo-Json -Compress -Depth 12
stderrPresent = $false
}
}
function Assert-Rejected {
param([scriptblock]$Operation, [string]$Name)
$accepted = $false
try {
& $Operation | Out-Null
$accepted = $true
} catch {}
if ($accepted) { throw "contract_mutation_accepted_$Name" }
}
$verifier = [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_verifier_v1"
ok = $true
sourceRevision = $SourceRevision
runId = $RunId
fileCount = 18
backupScriptFileCount = 17
backupHealthExporterIncluded = $true
remoteWritePerformed = $false
selfIdentityVerified = $true
}
$success = [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_executor_v1"
mode = "apply"
status = "verified"
ok = $true
sourceRevision = $SourceRevision
sourceHead = $sourceHead
runId = $RunId
fileCount = 18
backupScriptFileCount = 17
backupHealthExporterIncluded = $true
payloadCommitted = $true
rollbackAttempted = $false
rollbackPerformed = $false
rollbackVerified = $false
zeroResidueVerified = $true
zeroResidueScope = "run_owned_destination_temps"
stageCleanupVerified = $true
rollbackPrestateCleanupVerified = $true
receipt = $payloadReceipt
terminalReceipt = $terminalReceipt
terminalReceiptPublished = $true
terminalExitCode = 0
deferredSignal = $null
independentVerifierVerified = $true
verifier = $verifier
}
Write-ReplayTrace "accept_success"
Get-Agent99ExecutorResult (New-ReplayTransport $success 0) "Apply" | Out-Null
$topLevelRequired = @(
"schemaVersion", "mode", "status", "ok", "sourceRevision", "sourceHead",
"runId", "fileCount", "backupScriptFileCount", "backupHealthExporterIncluded",
"payloadCommitted", "rollbackAttempted", "rollbackPerformed", "rollbackVerified",
"zeroResidueVerified", "zeroResidueScope", "stageCleanupVerified",
"rollbackPrestateCleanupVerified", "receipt", "terminalReceipt", "terminalExitCode",
"terminalReceiptPublished", "deferredSignal", "independentVerifierVerified", "verifier"
)
$verifierRequired = @(
"schemaVersion", "ok", "sourceRevision", "runId", "fileCount",
"backupScriptFileCount", "backupHealthExporterIncluded", "remoteWritePerformed",
"selfIdentityVerified"
)
$missingFieldRejections = 0
Write-ReplayTrace "reject_missing_top"
foreach ($field in $topLevelRequired) {
$mutated = Copy-ReplayDocument $success
$mutated.PSObject.Properties.Remove($field)
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_$field"
$missingFieldRejections++
}
Write-ReplayTrace "reject_missing_verifier"
foreach ($field in $verifierRequired) {
$mutated = Copy-ReplayDocument $success
$mutated.verifier.PSObject.Properties.Remove($field)
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_verifier_$field"
$missingFieldRejections++
}
$typeMutations = @(
@{ field = "rollbackAttempted"; value = "false" },
@{ field = "rollbackPerformed"; value = "false" },
@{ field = "rollbackVerified"; value = "false" },
@{ field = "terminalExitCode"; value = "0" },
@{ field = "deferredSignal"; value = 0 }
)
$typeRejections = 0
Write-ReplayTrace "reject_types"
foreach ($mutation in $typeMutations) {
$mutated = Copy-ReplayDocument $success
$mutated.($mutation.field) = $mutation.value
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_$($mutation.field)"
$typeRejections++
}
$mutated = Copy-ReplayDocument $success
$mutated.verifier.remoteWritePerformed = "false"
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_verifier_remoteWritePerformed"
$typeRejections++
$cleanupPending = Copy-ReplayDocument $success
$cleanupPending.ok = $false
$cleanupPending.status = "cleanup_pending"
$cleanupPending.stageCleanupVerified = $false
$cleanupPending.terminalExitCode = 75
$cleanupPending.verifier = $null
Write-ReplayTrace "accept_cleanup_pending"
Get-Agent99CommittedFailureResult (New-ReplayTransport $cleanupPending 75) | Out-Null
$signalDeferred = Copy-ReplayDocument $success
$signalDeferred.ok = $false
$signalDeferred.status = "committed_signal_deferred"
$signalDeferred.terminalExitCode = 143
$signalDeferred.deferredSignal = "TERM"
$signalDeferred.verifier = $null
Write-ReplayTrace "accept_signal_deferred"
Get-Agent99CommittedFailureResult (New-ReplayTransport $signalDeferred 143) | Out-Null
$terminalReceiptFailed = Copy-ReplayDocument $success
$terminalReceiptFailed.ok = $false
$terminalReceiptFailed.status = "terminal_receipt_failed"
$terminalReceiptFailed.terminalReceiptPublished = $false
$terminalReceiptFailed.terminalExitCode = 76
$terminalReceiptFailed.verifier = $null
Write-ReplayTrace "accept_terminal_receipt_failed"
Get-Agent99CommittedFailureResult (New-ReplayTransport $terminalReceiptFailed 76) | Out-Null
Write-ReplayTrace "complete"
[pscustomobject]@{
schemaVersion = "agent99_host110_broker_contract_replay_v1"
ok = $true
parser = "WindowsPowerShell"
successAccepted = $true
cleanupPendingPreserved = $true
committedSignalPreserved = $true
terminalReceiptFailurePreserved = $true
missingFieldRejections = $missingFieldRejections
typeRejections = $typeRejections
remoteWritePerformed = $false
secretValuesRead = $false
} | ConvertTo-Json -Compress

View File

@@ -1,15 +1,10 @@
from __future__ import annotations
import base64
import grp
import hashlib
import importlib.util
import json
import os
import pwd
import re
import shutil
import subprocess
import sys
from pathlib import Path
@@ -20,9 +15,6 @@ ROOT = Path(__file__).resolve().parents[3]
BROKER = ROOT / "agent99-host110-backup-runtime-broker.ps1"
EXECUTOR = ROOT / "scripts" / "backup" / "host110-backup-runtime-executor.sh"
VERIFIER = ROOT / "scripts" / "backup" / "verify-host110-backup-runtime.py"
BROKER_CONTRACT_REPLAY = (
ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-host110-broker-contract-replay.ps1"
)
EXPECTED_RUNTIME_FILES = {
@@ -58,252 +50,6 @@ def _sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _powershell_string(source: str, variable: str) -> str:
match = re.search(rf'\${re.escape(variable)}\s*=\s*"([^"]+)"', source)
assert match is not None, variable
return match.group(1)
def _powershell_array(source: str, variable: str) -> tuple[str, ...]:
match = re.search(
rf'\${re.escape(variable)}\s*=\s*@\((.*?)\n\)',
source,
re.DOTALL,
)
assert match is not None, variable
return tuple(re.findall(r'"([^"]+)"', match.group(1)))
def _powershell_function(source: str, name: str) -> str:
start = source.index(f"function {name} {{")
next_function = source.find("\nfunction ", start + 1)
return source[start:] if next_function < 0 else source[start:next_function]
def _executor_payload_order(source: str) -> tuple[str, ...]:
match = re.search(r"readonly -a RUNTIME_FILES=\((.*?)\n\)", source, re.DOTALL)
assert match is not None
runtime_files = tuple(re.findall(r"^\s+([A-Za-z0-9_.-]+)\s*$", match.group(1), re.MULTILINE))
exporter = re.search(r'^readonly EXPORTER_FILE="([^"]+)"$', source, re.MULTILINE)
assert exporter is not None
return (*runtime_files, exporter.group(1))
def _write_executable(path: Path, source: str) -> None:
path.write_text(source, encoding="utf-8")
path.chmod(0o755)
def _build_executor_harness(tmp_path: Path, run_id: str) -> dict[str, object]:
destination = tmp_path / "backup-scripts"
exporter_root = tmp_path / "exporter"
status_root = tmp_path / "status"
stage_root = tmp_path / "stage"
rollback_root = tmp_path / "rollback"
source_stage = tmp_path / f"source-{run_id}"
lock_path = tmp_path / "runtime.lock"
fake_bin = tmp_path / "fake-bin"
for path in (destination, exporter_root, source_stage, fake_bin):
path.mkdir(parents=True)
user_name = pwd.getpwuid(os.getuid()).pw_name
group_name = grp.getgrgid(os.getgid()).gr_name
shell_path = shutil.which("bash") or "/bin/bash"
shell_version = subprocess.run(
[shell_path, "-c", 'printf "%s" "${BASH_VERSINFO[0]:-0}"'],
check=False,
capture_output=True,
text=True,
).stdout
use_zsh_adapter = not shell_version.isdigit() or int(shell_version) < 4
if use_zsh_adapter:
shell_path = shutil.which("zsh") or "/bin/zsh"
executor_source = EXECUTOR.read_text(encoding="utf-8")
replacements = {
'readonly EXPECTED_USER="wooo"': f'readonly EXPECTED_USER="{user_name}"',
'readonly DEST_ROOT="/backup/scripts"': f'readonly DEST_ROOT="{destination}"',
'readonly EXPORTER_ROOT="/home/wooo/scripts"': f'readonly EXPORTER_ROOT="{exporter_root}"',
'readonly STATUS_ROOT="/backup/status"': f'readonly STATUS_ROOT="{status_root}"',
'readonly STAGE_ROOT="/backup/.agent99-backup-runtime-stage"': f'readonly STAGE_ROOT="{stage_root}"',
'readonly ROLLBACK_ROOT="/backup/.agent99-backup-runtime-rollback"': f'readonly ROLLBACK_ROOT="{rollback_root}"',
'readonly LOCK_PATH="/tmp/agent99-host110-backup-runtime.lock"': f'readonly LOCK_PATH="{lock_path}"',
'expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"': f'expected_source_stage="{source_stage}"',
'"wooo:wooo:755"': f'"{user_name}:{group_name}:755"',
'"wooo:wooo:700"': f'"{user_name}:{group_name}:700"',
'"wooo:wooo"': f'"{user_name}:{group_name}"',
}
for before, after in replacements.items():
assert before in executor_source
executor_source = executor_source.replace(before, after)
if use_zsh_adapter:
executor_source = executor_source.replace(
'"${!RUN_OWNED_TEMP_PATHS[@]}"',
'"${(@k)RUN_OWNED_TEMP_PATHS}"',
).replace(
'RUN_OWNED_TEMP_PATHS["$1"]=1',
'RUN_OWNED_TEMP_PATHS[$1]=1',
).replace('${PAYLOAD_FILES[0]}', '${PAYLOAD_FILES[1]}')
executor_path = source_stage / EXECUTOR.name
_write_executable(executor_path, executor_source)
verifier_source = VERIFIER.read_text(encoding="utf-8")
verifier_replacements = {
'EXPECTED_DESTINATION = Path("/backup/scripts")': f"EXPECTED_DESTINATION = Path({str(destination)!r})",
'EXPECTED_EXPORTER_DESTINATION = Path("/home/wooo/scripts/backup-health-textfile-exporter.py")': (
f"EXPECTED_EXPORTER_DESTINATION = Path({str(exporter_root / 'backup-health-textfile-exporter.py')!r})"
),
'RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")': f"RUNTIME_LOCK = Path({str(lock_path)!r})",
}
for before, after in verifier_replacements.items():
assert before in verifier_source
verifier_source = verifier_source.replace(before, after)
verifier_path = source_stage / VERIFIER.name
_write_executable(verifier_path, verifier_source)
payload_order = _executor_payload_order(executor_source)
rows = []
before_state: dict[str, tuple[bytes, int]] = {}
for name in payload_order:
source = (
ROOT / "scripts" / "ops" / name
if name == "backup-health-textfile-exporter.py"
else ROOT / "scripts" / "backup" / name
)
staged = source_stage / name
shutil.copy2(source, staged)
staged.chmod(0o755)
rows.append({"name": name, "sha256": _sha256(staged)})
target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name
target.write_text(f"prestate:{name}\n", encoding="utf-8")
target.chmod(0o755)
before_state[name] = (target.read_bytes(), target.stat().st_mode & 0o777)
source_revision = "a" * 40
manifest = {
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
"sourceRevision": source_revision,
"sourceHead": "b" * 40,
"runId": run_id,
"executorSha256": _sha256(executor_path),
"verifierSha256": _sha256(verifier_path),
"files": rows,
}
(source_stage / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
_write_executable(fake_bin / "hostname", "#!/bin/sh\nprintf '%s\\n' '192.168.0.110'\n")
_write_executable(fake_bin / "pgrep", "#!/bin/sh\nexit 1\n")
_write_executable(fake_bin / "flock", "#!/bin/sh\nexit 0\n")
_write_executable(fake_bin / "timeout", "#!/bin/sh\nexit 0\n")
_write_executable(
fake_bin / "readlink",
"""#!/usr/bin/env python3
import pathlib
import sys
if len(sys.argv) != 3 or sys.argv[1] != "-f":
raise SystemExit(2)
print(pathlib.Path(sys.argv[2]).resolve(strict=True))
""",
)
_write_executable(
fake_bin / "stat",
"""#!/usr/bin/env python3
import grp
import os
import pwd
import sys
if len(sys.argv) != 4 or sys.argv[1] != "-c":
raise SystemExit(2)
row = os.stat(sys.argv[3])
mode = format(row.st_mode & 0o777, "o")
values = {
"%U:%G:%a": f"{pwd.getpwuid(row.st_uid).pw_name}:{grp.getgrgid(row.st_gid).gr_name}:{mode}",
"%U:%G": f"{pwd.getpwuid(row.st_uid).pw_name}:{grp.getgrgid(row.st_gid).gr_name}",
"%u:%g:%a": f"{row.st_uid}:{row.st_gid}:{mode}",
}
if sys.argv[2] not in values:
raise SystemExit(2)
print(values[sys.argv[2]])
""",
)
return {
"executor": executor_path,
"source_stage": source_stage,
"source_revision": source_revision,
"run_id": run_id,
"destination": destination,
"exporter_root": exporter_root,
"status_root": status_root,
"stage_root": stage_root,
"rollback_root": rollback_root,
"fake_bin": fake_bin,
"shell": shell_path,
"use_zsh_adapter": use_zsh_adapter,
"payload_order": payload_order,
"before_state": before_state,
}
def _run_executor_harness(harness: dict[str, object], **extra_env: str) -> subprocess.CompletedProcess[str]:
environment = os.environ.copy()
for name in (
"HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT",
"HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT",
"HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT",
"HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL",
"HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL",
"HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT",
"HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT",
):
environment.pop(name, None)
environment["HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS"] = "1"
environment.update(extra_env)
environment["PATH"] = f"{harness['fake_bin']}:{environment['PATH']}"
command = [str(harness["shell"])]
if harness["use_zsh_adapter"]:
environment["ZDOTDIR"] = str(harness["fake_bin"])
command.append("-f")
command.extend(
[
str(harness["executor"]),
"--mode",
"apply",
"--source-revision",
str(harness["source_revision"]),
"--run-id",
str(harness["run_id"]),
"--source-stage",
str(harness["source_stage"]),
]
)
return subprocess.run(
command,
check=False,
capture_output=True,
text=True,
env=environment,
)
def _assert_prestate_restored(harness: dict[str, object]) -> None:
destination = Path(harness["destination"])
exporter_root = Path(harness["exporter_root"])
before_state = harness["before_state"]
assert isinstance(before_state, dict)
for name in harness["payload_order"]:
target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name
assert (target.read_bytes(), target.stat().st_mode & 0o777) == before_state[name]
def _assert_payload_committed(harness: dict[str, object]) -> None:
destination = Path(harness["destination"])
exporter_root = Path(harness["exporter_root"])
source_stage = Path(harness["source_stage"])
for name in harness["payload_order"]:
target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name
assert target.read_bytes() == (source_stage / name).read_bytes()
assert target.stat().st_mode & 0o777 == 0o755
def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
source = BROKER.read_text(encoding="utf-8")
@@ -327,136 +73,6 @@ def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
assert "taskkill.exe /PID $process.Id /T /F" in source
assert "$process.Kill()" in source
assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths' in source
assert '[string]$parsed.status -ne "verified"' in source
assert '-not [bool]$parsed.payloadCommitted' in source
assert '-not [bool]$parsed.stageCleanupVerified' in source
assert '-not [bool]$parsed.rollbackPrestateCleanupVerified' in source
assert '[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps"' in source
assert '[int64]$parsed.terminalExitCode -ne 0' in source
assert 'agent99-backup-runtime-terminal-$RunId.json' in source
assert '$property = $Object.PSObject.Properties[$Name]' in source
assert 'if ($null -eq $property) { return $false }' in source
assert 'Get-Agent99CommittedFailureResult $applyTransport' in source
assert 'throw "host110_apply_$([string]$apply.status)"' in source
def test_broker_required_field_contract_rejects_missing_fields_before_cast() -> None:
source = BROKER.read_text(encoding="utf-8")
field_guard = _powershell_function(source, "Test-Agent99JsonField")
common_contract = _powershell_function(source, "Convert-Agent99ExecutorDocument")
apply_contract = _powershell_function(source, "Assert-Agent99ApplyResultFields")
verifier_contract = _powershell_function(source, "Assert-Agent99VerifierFields")
committed_failure = _powershell_function(source, "Get-Agent99CommittedFailureResult")
assert '$property = $Object.PSObject.Properties[$Name]' in field_guard
assert 'if ($null -eq $property) { return $false }' in field_guard
for field in (
"schemaVersion",
"mode",
"ok",
"sourceRevision",
"sourceHead",
"runId",
"fileCount",
"backupScriptFileCount",
"backupHealthExporterIncluded",
):
assert re.search(rf"^\s+{field} = \"", common_contract, re.MULTILINE), field
for field in (
"status",
"payloadCommitted",
"rollbackAttempted",
"rollbackPerformed",
"rollbackVerified",
"zeroResidueVerified",
"zeroResidueScope",
"stageCleanupVerified",
"rollbackPrestateCleanupVerified",
"receipt",
"terminalReceipt",
"terminalReceiptPublished",
"terminalExitCode",
"deferredSignal",
"independentVerifierVerified",
"verifier",
):
assert re.search(rf"^\s+{field} = ", apply_contract, re.MULTILINE), field
for field in (
"schemaVersion",
"ok",
"sourceRevision",
"runId",
"fileCount",
"backupScriptFileCount",
"backupHealthExporterIncluded",
"remoteWritePerformed",
"selfIdentityVerified",
):
assert re.search(rf"^\s+{field} = \"", verifier_contract, re.MULTILINE), field
assert 'status -eq "cleanup_pending"' in committed_failure
assert 'status -eq "committed_signal_deferred"' in committed_failure
assert 'verifier = $VerifierKind' in apply_contract
def test_windows99_contract_replay_uses_actual_broker_functions_without_remote_write() -> None:
source = BROKER_CONTRACT_REPLAY.read_text(encoding="utf-8")
assert "FunctionDefinitionAst" in source
for function_name in (
"Test-Agent99JsonField",
"Assert-Agent99JsonFields",
"Convert-Agent99ExecutorDocument",
"Assert-Agent99ApplyResultFields",
"Assert-Agent99VerifierFields",
"Get-Agent99ExecutorResult",
"Get-Agent99CommittedFailureResult",
):
assert f'"{function_name}"' in source
assert "$mutated.PSObject.Properties.Remove($field)" in source
assert "$mutated.verifier.PSObject.Properties.Remove($field)" in source
assert 'status = "cleanup_pending"' in source
assert 'status = "committed_signal_deferred"' in source
assert 'status = "terminal_receipt_failed"' in source
assert "remoteWritePerformed = $false" in source
assert "secretValuesRead = $false" in source
for forbidden in ("New-Item", "Set-Content", "WriteAllText", "ssh.exe", "scp.exe"):
assert forbidden not in source
def test_broker_scp_basenames_match_executor_source_stage_contract() -> None:
broker = BROKER.read_text(encoding="utf-8")
executor = EXECUTOR.read_text(encoding="utf-8")
runtime_paths = _powershell_array(broker, "RuntimeRelativePaths")
broker_payload_basenames = tuple(Path(path).name for path in runtime_paths)
executor_payload_basenames = _executor_payload_order(executor)
executor_path = _powershell_string(broker, "ExecutorRelativePath")
verifier_path = _powershell_string(broker, "VerifierRelativePath")
manifest_match = re.search(
r'\$manifestPath\s*=\s*Join-Path\s+\$sourceRoot\s+"([^"]+)"',
broker,
)
assert manifest_match is not None
manifest_name = manifest_match.group(1)
assert len(broker_payload_basenames) == 18
assert len(executor_payload_basenames) == 18
assert set(broker_payload_basenames) == set(executor_payload_basenames)
assert broker_payload_basenames == executor_payload_basenames
staged_basenames = (
*broker_payload_basenames,
Path(executor_path).name,
Path(verifier_path).name,
manifest_name,
)
assert len(staged_basenames) == 21
assert len(set(staged_basenames)) == 21
assert manifest_name == "manifest.json"
assert '$localPaths += $executorPath, $verifierPath, $ManifestPath' in broker
assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths $remoteStage' in broker
assert f'$SOURCE_STAGE/{manifest_name}' in executor
assert f'$SOURCE_STAGE/{Path(verifier_path).name}' in executor
assert Path(executor_path).name == "host110-backup-runtime-executor.sh"
def test_windows99_broker_verify_is_independent_no_write_and_evidence_is_immutable() -> None:
@@ -496,45 +112,10 @@ def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
assert source.index("for name in \"${PAYLOAD_FILES[@]}\"") < source.index("mv -f -- \"$temporary\"")
assert 'rollback_unverified' in source
assert 'failed_rolled_back' in source
assert 'ROLLBACK_ATTEMPTED=1' in source
assert 'ROLLBACK_PERFORMED=1' in source
assert '"rollbackAttempted": rollback_attempted' in source
assert '"rollbackPerformed": rollback_performed' in source
assert '"rollbackVerified": rollback_verified' in source
assert '"zeroResidueVerified": zero_residue_verified' in source
assert '"zeroResidueScope": "run_owned_destination_temps"' in source
assert 'cleanup_run_owned_temps' in source
assert 'verify_run_owned_temp_absence' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT' in source
assert 'HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS' in source
assert '[ "$DEST_ROOT" != "/backup/scripts" ]' in source
assert 'fsync_rollback_prestate || return 75' in source
assert 'fsync_destination_state || fail "post_apply_durability_failed"' in source
assert 'working_digest "$dest_path"' in source
assert 'PREVIOUS_METADATA' in source
assert "stat -c '%u:%g:%a'" in source
assert 'os.link(temporary, path)' in source
assert 'os.fsync(handle.fileno())' in source
assert 'os.fsync(directory_fd)' in source
assert 'hashlib.sha256(readback).hexdigest()' in source
assert source.index("COMMIT_CRITICAL=1") < source.index('write_receipt "payload_verified"')
assert source.index('write_receipt "payload_verified"') < source.index("\nAPPLY_COMPLETE=1")
assert source.index("\nAPPLY_COMPLETE=1") < source.rindex("\nCOMMIT_CRITICAL=0")
assert source.index("\nAPPLY_COMPLETE=1") < source.rindex(
'cleanup_postcommit_directory "rollback_prestate"'
)
assert 'handle_agent99_signal HUP 129' in source
assert 'handle_agent99_signal TERM 143' in source
assert source.index("trap '' HUP INT TERM") < source.index('write_terminal_receipt "$terminal_status"')
assert 'write_terminal_receipt "$terminal_status"' in source
assert 'terminal_status="cleanup_pending"' in source
assert 'terminal_status="committed_signal_deferred"' in source
assert 'run_identity_receipt_exists' in source
assert 'run_identity_stage_exists' in source
assert 'run_identity_rollback_exists' in source
@@ -549,302 +130,6 @@ def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
assert "systemctl " not in source
def test_apply_success_publishes_durable_receipt_before_prestate_cleanup(tmp_path: Path) -> None:
run_id = "apply-success"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(harness)
assert result.returncode == 0, result.stderr
output = json.loads(result.stdout)
assert output["ok"] is True
assert output["rollbackAttempted"] is False
assert output["rollbackPerformed"] is False
assert output["rollbackVerified"] is False
assert output["zeroResidueVerified"] is True
assert output["stageCleanupVerified"] is True
assert output["rollbackPrestateCleanupVerified"] is True
assert output["status"] == "verified"
assert output["payloadCommitted"] is True
assert output["runId"] == run_id
assert output["terminalExitCode"] == 0
assert output["terminalReceiptPublished"] is True
assert output["deferredSignal"] is None
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
assert receipt["status"] == "payload_verified"
assert receipt["rollbackAttempted"] is False
assert receipt["rollbackPerformed"] is False
assert receipt["rollbackVerified"] is False
assert receipt["zeroResidueVerified"] is True
assert receipt["zeroResidueScope"] == "run_owned_destination_temps"
terminal_path = Path(harness["status_root"]) / f"agent99-backup-runtime-terminal-{run_id}.json"
terminal = json.loads(terminal_path.read_text(encoding="utf-8"))
assert terminal["status"] == "verified"
assert terminal["ok"] is True
assert terminal["payloadCommitted"] is True
assert terminal["stageCleanupVerified"] is True
assert terminal["rollbackPrestateCleanupVerified"] is True
assert terminal["terminalExitCode"] == 0
assert terminal["zeroResidueScope"] == "run_owned_destination_temps_and_internal_stage_prestate"
assert output["terminalReceipt"] == str(terminal_path)
assert not (Path(harness["rollback_root"]) / run_id).exists()
assert not (Path(harness["stage_root"]) / run_id).exists()
_assert_payload_committed(harness)
@pytest.mark.parametrize(("signal_name", "exit_code"), [("TERM", 143), ("HUP", 129), ("INT", 130)])
def test_commit_window_signal_is_deferred_until_payload_and_terminal_are_consistent(
tmp_path: Path,
signal_name: str,
exit_code: int,
) -> None:
run_id = f"commit-signal-{signal_name.lower()}"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL=signal_name,
)
assert result.returncode == exit_code, result.stderr
output = json.loads(result.stdout)
assert output["ok"] is False
assert output["status"] == "committed_signal_deferred"
assert output["payloadCommitted"] is True
assert output["runId"] == run_id
assert output["deferredSignal"] == signal_name
assert output["terminalExitCode"] == exit_code
assert output["terminalReceiptPublished"] is True
assert output["stageCleanupVerified"] is True
assert output["rollbackPrestateCleanupVerified"] is True
_assert_payload_committed(harness)
payload_receipt = json.loads(
(
Path(harness["status_root"])
/ f"agent99-backup-runtime-deploy-{run_id}.json"
).read_text(encoding="utf-8")
)
assert payload_receipt["status"] == "payload_verified"
terminal = json.loads(
(
Path(harness["status_root"])
/ f"agent99-backup-runtime-terminal-{run_id}.json"
).read_text(encoding="utf-8")
)
assert terminal["status"] == "committed_signal_deferred"
assert terminal["ok"] is False
assert terminal["deferredSignal"] == signal_name
assert terminal["terminalExitCode"] == exit_code
assert not (Path(harness["rollback_root"]) / run_id).exists()
assert not (Path(harness["stage_root"]) / run_id).exists()
@pytest.mark.parametrize("signal_name", ["TERM", "HUP", "INT"])
def test_terminal_publication_masks_late_signal_without_receipt_stdout_exit_divergence(
tmp_path: Path,
signal_name: str,
) -> None:
run_id = f"terminal-signal-{signal_name.lower()}"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL=signal_name,
)
assert result.returncode == 0, result.stderr
output = json.loads(result.stdout)
assert output["ok"] is True
assert output["status"] == "verified"
assert output["payloadCommitted"] is True
assert output["runId"] == run_id
assert output["deferredSignal"] is None
assert output["terminalExitCode"] == 0
assert output["terminalReceiptPublished"] is True
terminal = json.loads(
(
Path(harness["status_root"])
/ f"agent99-backup-runtime-terminal-{run_id}.json"
).read_text(encoding="utf-8")
)
assert terminal["ok"] is True
assert terminal["status"] == output["status"]
assert terminal["deferredSignal"] == output["deferredSignal"]
assert terminal["terminalExitCode"] == output["terminalExitCode"]
_assert_payload_committed(harness)
@pytest.mark.parametrize("fault", ["write", "link", "fsync", "readback"])
def test_terminal_receipt_fault_is_nonzero_with_payload_receipt_only(
tmp_path: Path,
fault: str,
) -> None:
run_id = f"terminal-receipt-{fault}"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT=fault,
)
assert result.returncode == 76
assert '"ok": true' not in result.stdout.lower()
assert "terminal_receipt_failed" in result.stderr
output = json.loads(result.stdout)
assert output["ok"] is False
assert output["status"] == "terminal_receipt_failed"
assert output["payloadCommitted"] is True
assert output["runId"] == run_id
assert output["terminalReceiptPublished"] is False
assert output["terminalExitCode"] == 76
_assert_payload_committed(harness)
payload_receipt = json.loads(
(
Path(harness["status_root"])
/ f"agent99-backup-runtime-deploy-{run_id}.json"
).read_text(encoding="utf-8")
)
assert payload_receipt["status"] == "payload_verified"
assert not (
Path(harness["status_root"])
/ f"agent99-backup-runtime-terminal-{run_id}.json"
).exists()
assert not (Path(harness["rollback_root"]) / run_id).exists()
assert not (Path(harness["stage_root"]) / run_id).exists()
@pytest.mark.parametrize(
("fault", "stage_clean", "rollback_clean"),
[("stage", False, True), ("rollback_prestate", True, False)],
)
def test_postcommit_cleanup_failure_is_nonzero_and_never_false_green(
tmp_path: Path,
fault: str,
stage_clean: bool,
rollback_clean: bool,
) -> None:
run_id = f"cleanup-{fault}"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT=fault,
)
assert result.returncode == 75, result.stderr
assert '"ok": true' not in result.stdout.lower()
output = json.loads(result.stdout)
assert output["ok"] is False
assert output["status"] == "cleanup_pending"
assert output["payloadCommitted"] is True
assert output["runId"] == run_id
assert output["stageCleanupVerified"] is stage_clean
assert output["rollbackPrestateCleanupVerified"] is rollback_clean
assert output["terminalReceiptPublished"] is True
_assert_payload_committed(harness)
payload_receipt = json.loads(
(
Path(harness["status_root"])
/ f"agent99-backup-runtime-deploy-{run_id}.json"
).read_text(encoding="utf-8")
)
assert payload_receipt["status"] == "payload_verified"
terminal = json.loads(
(
Path(harness["status_root"])
/ f"agent99-backup-runtime-terminal-{run_id}.json"
).read_text(encoding="utf-8")
)
assert terminal["status"] == "cleanup_pending"
assert terminal["ok"] is False
assert terminal["payloadCommitted"] is True
assert terminal["stageCleanupVerified"] is stage_clean
assert terminal["rollbackPrestateCleanupVerified"] is rollback_clean
assert (Path(harness["stage_root"]) / run_id).exists() is (not stage_clean)
assert (Path(harness["rollback_root"]) / run_id).exists() is (not rollback_clean)
@pytest.mark.parametrize("fault", ["write", "link", "fsync", "readback"])
def test_receipt_fault_rolls_back_without_false_success_or_premature_prestate_cleanup(
tmp_path: Path,
fault: str,
) -> None:
run_id = f"receipt-{fault}"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT=fault,
)
assert result.returncode != 0
assert '"ok": true' not in result.stdout.lower()
assert "durable_receipt_failed" in result.stderr
_assert_prestate_restored(harness)
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
assert receipt["status"] == "failed_rolled_back"
assert receipt["rollbackAttempted"] is True
assert receipt["rollbackPerformed"] is True
assert receipt["rollbackVerified"] is True
assert receipt["zeroResidueVerified"] is True
assert not (Path(harness["rollback_root"]) / run_id).exists()
assert not (Path(harness["stage_root"]) / run_id).exists()
for parent in (Path(harness["destination"]), Path(harness["exporter_root"])):
assert not list(parent.glob(f".*.agent99-*{run_id}"))
def test_apply_temp_fault_is_removed_and_zero_residue_is_verified(tmp_path: Path) -> None:
run_id = "apply-temp-cleanup"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT="after_temp_install",
)
assert result.returncode != 0
assert '"ok": true' not in result.stdout.lower()
assert "fault_injected_after_temp_install" in result.stderr
_assert_prestate_restored(harness)
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
assert receipt["status"] == "failed_rolled_back"
assert receipt["rollbackAttempted"] is True
assert receipt["rollbackPerformed"] is True
assert receipt["rollbackVerified"] is True
assert receipt["zeroResidueVerified"] is True
for parent in (Path(harness["destination"]), Path(harness["exporter_root"])):
assert not list(parent.glob(f".*.agent99-*{run_id}"))
def test_temp_cleanup_fault_is_rollback_unverified_and_preserves_prestate(tmp_path: Path) -> None:
run_id = "temp-residue-unverified"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT="after_temp_install",
HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT="preserve_first_temp",
)
assert result.returncode != 0
assert '"ok": true' not in result.stdout.lower()
_assert_prestate_restored(harness)
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
assert receipt["status"] == "rollback_unverified"
assert receipt["rollbackAttempted"] is True
assert receipt["rollbackPerformed"] is True
assert receipt["rollbackVerified"] is False
assert receipt["zeroResidueVerified"] is False
assert (Path(harness["rollback_root"]) / run_id / "prestate.tsv").is_file()
residue = []
for parent in (Path(harness["destination"]), Path(harness["exporter_root"])):
residue.extend(parent.glob(f".*.agent99-*{run_id}"))
assert residue
def test_backup_runtime_entrypoints_share_the_deployment_gate() -> None:
common = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8")
restore = (ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh").read_text(encoding="utf-8")