Compare commits
1 Commits
codex/host
...
codex/cicd
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5c43b3da2e |
@@ -133,10 +133,7 @@ jobs:
|
||||
# 2026-06-28 Codex: awoooi-non110-host maps to the dedicated
|
||||
# non-110 runner lane. Bootstrap tools defensively because host
|
||||
# runners can start without the CI toolchain preinstalled.
|
||||
run: |
|
||||
if command -v apk >/dev/null 2>&1; then
|
||||
apk add --no-cache nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
|
||||
fi
|
||||
run: bash scripts/ci/bootstrap-host-runner-tools.sh
|
||||
|
||||
- name: Checkout source from Gitea
|
||||
env:
|
||||
@@ -1454,10 +1451,7 @@ jobs:
|
||||
- name: Bootstrap Host Runner Tools
|
||||
# 2026-05-05 Codex: keep the host-mode runner self-healing before
|
||||
# actions/checkout@v4 and Telegram failure notifications run.
|
||||
run: |
|
||||
if command -v apk >/dev/null 2>&1; then
|
||||
apk add --no-cache nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
|
||||
fi
|
||||
run: bash scripts/ci/bootstrap-host-runner-tools.sh
|
||||
|
||||
- name: Checkout source from Gitea
|
||||
env:
|
||||
@@ -5226,10 +5220,7 @@ jobs:
|
||||
- name: Bootstrap Host Runner Tools
|
||||
# 2026-05-05 Codex: post-deploy also uses checkout and curl-based
|
||||
# notifications, so it needs the same runner bootstrap as earlier jobs.
|
||||
run: |
|
||||
if command -v apk >/dev/null 2>&1; then
|
||||
apk add --no-cache nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
|
||||
fi
|
||||
run: bash scripts/ci/bootstrap-host-runner-tools.sh
|
||||
|
||||
- name: Checkout source from Gitea
|
||||
env:
|
||||
|
||||
@@ -224,7 +224,6 @@ $agentFiles = @(
|
||||
"agent99-control-plane.ps1",
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
|
||||
@@ -20,7 +20,6 @@ $requiredFiles = @(
|
||||
"agent99-control-plane.ps1",
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
@@ -132,7 +131,6 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js
|
||||
$control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane.ps1"), [Text.Encoding]::UTF8)
|
||||
$dbBoundedExecutor = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-db-bounded-executor.ps1"), [Text.Encoding]::UTF8)
|
||||
$signozMetadataExecutor = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-signoz-metadata-executor.ps1"), [Text.Encoding]::UTF8)
|
||||
$host110BackupRuntimeBroker = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-host110-backup-runtime-broker.ps1"), [Text.Encoding]::UTF8)
|
||||
$inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8)
|
||||
$telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8)
|
||||
$bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8)
|
||||
@@ -192,8 +190,7 @@ $signozMetadataExecutorContract = [bool](
|
||||
$signozMetadataExecutor.Contains('agent99_active:') -and
|
||||
$signozMetadataExecutor.Contains('$RemoteResult.completed -ne $true') -and
|
||||
$signozMetadataExecutor.Contains('Get-Agent99RuntimeSourceBinding') -and
|
||||
$signozMetadataExecutor.Contains('$ExpectedRuntimeFileCount = 19') -and
|
||||
$signozMetadataExecutor.Contains('$result.fileCount -eq $ExpectedRuntimeFileCount') -and
|
||||
$signozMetadataExecutor.Contains('$result.fileCount -eq 18') -and
|
||||
$signozMetadataExecutor.Contains('pass_export_verified_restore_pending') -and
|
||||
$signozMetadataExecutor.Contains('productionRuntimeMutationPerformed = $false') -and
|
||||
$signozMetadataExecutor.Contains('rawSqliteRead = $false') -and
|
||||
@@ -206,29 +203,6 @@ $signozMetadataExecutorContract = [bool](
|
||||
-not $signozMetadataExecutor.Contains('Get-Content -LiteralPath $CredentialFile')
|
||||
)
|
||||
Add-Check "backup:signoz_metadata_fixed_executor" $signozMetadataExecutorContract "Windows99 dispatches one fixed host110 metadata export contract, consumes only a root-owned credential reference on target, and requires the independent bundle verifier without raw SQLite or secret output"
|
||||
$host110BackupRuntimeContract = [bool](
|
||||
$host110BackupRuntimeBroker.Contains('$TargetHost = "192.168.0.110"') -and
|
||||
$host110BackupRuntimeBroker.Contains('$TargetUser = "wooo"') -and
|
||||
$host110BackupRuntimeBroker.Contains('$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"') -and
|
||||
$host110BackupRuntimeBroker.Contains('"credential.helper="') -and
|
||||
$host110BackupRuntimeBroker.Contains('$env:GIT_TERMINAL_PROMPT = "0"') -and
|
||||
$host110BackupRuntimeBroker.Contains('"merge-base", "--is-ancestor", $SourceRevision, $sourceHead') -and
|
||||
$host110BackupRuntimeBroker.Contains('executorSha256 = $executorDigest') -and
|
||||
$host110BackupRuntimeBroker.Contains('verifierSha256 = $verifierDigest') -and
|
||||
$host110BackupRuntimeBroker.Contains('"scripts/ops/backup-health-textfile-exporter.py"') -and
|
||||
$host110BackupRuntimeBroker.Contains('$ExpectedFileCount = 18') -and
|
||||
$host110BackupRuntimeBroker.Contains('StrictHostKeyChecking=yes') -and
|
||||
$host110BackupRuntimeBroker.Contains('$process.WaitForExit($TimeoutSeconds * 1000)') -and
|
||||
$host110BackupRuntimeBroker.Contains('taskkill.exe /PID $process.Id /T /F') -and
|
||||
$host110BackupRuntimeBroker.Contains('Invoke-Agent99BoundedScp $sourcePackage.localPaths') -and
|
||||
$host110BackupRuntimeBroker.Contains('python3 - --manifest-base64') -and
|
||||
$host110BackupRuntimeBroker.Contains('Write-Agent99ImmutableJson') -and
|
||||
$host110BackupRuntimeBroker.Contains('decisionProvider = "deterministic_only"') -and
|
||||
$host110BackupRuntimeBroker.Contains('criticProvider = "deterministic_only"') -and
|
||||
-not $host110BackupRuntimeBroker.Contains('/home/wooo/awoooi') -and
|
||||
-not $host110BackupRuntimeBroker.Contains('Invoke-Expression')
|
||||
)
|
||||
Add-Check "backup:host110_runtime_fixed_broker" $host110BackupRuntimeContract "Windows99 fetches one exact Gitea revision, binds executor and verifier digests, performs one fixed process-bounded Host110 transaction, and preserves an independent no-write verifier readback"
|
||||
Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("Enter-AgentRecoveryTriggerReservation") -and $control.Contains('"recovery-trigger-reservation.lock"') -and $control.Contains('[System.IO.FileShare]::None') -and $control.Contains('reason = "recovery_trigger_reservation_busy"') -and $control.Contains("recovery_already_queued")) "boot and host-down recovery reserve one atomic single-flight queue decision"
|
||||
Add-Check "recovery:auto_trigger_controlled_identity" ($control.Contains("New-AgentObservedRecoveryDispatchIdentity") -and $control.Contains('-RouteId "agent99_local_observer"') -and $control.Contains('schema_version = "agent99_controlled_dispatch_identity_v1"') -and $control.Contains('correlationKey = $dispatchIdentity.idempotencyKey') -and $control.Contains('canonicalDigest = $dispatchIdentity.canonicalDigest') -and $control.Contains("Invoke-AgentControlledDispatchIdentitySelfTest")) "observer-created Recover requests carry a stable canonical controlled dispatch identity before queue processing"
|
||||
Add-Check "recovery:queue_identity_fail_closed" ($control.Contains('dispatchIdentitySchemaVersion = $dispatchIdentity.schemaVersion') -and $control.Contains('Test-AgentControlledDispatchIdentity $dispatchIdentityEnvelope') -and $control.Contains('"controlled_dispatch_identity_invalid"') -and $control.IndexOf('"controlled_dispatch_identity_invalid"') -lt $control.IndexOf('$process = Start-Process -FilePath "powershell.exe" -ArgumentList $args')) "queue consumer rebuilds and validates canonical identity before any controlled executor starts"
|
||||
|
||||
@@ -35,7 +35,6 @@ $runtimeFiles = @(
|
||||
"agent99-control-plane.ps1",
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
|
||||
@@ -1,684 +0,0 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateSet("Check", "Apply", "Verify")]
|
||||
[string]$Mode,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$SourceRevision,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RunId,
|
||||
[string]$AgentRoot = "C:\Wooo\Agent99"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ExpectedAgentRoot = "C:\Wooo\Agent99"
|
||||
$TargetHost = "192.168.0.110"
|
||||
$TargetUser = "wooo"
|
||||
$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"
|
||||
$GitPath = "C:\Program Files\Git\cmd\git.exe"
|
||||
$IdentityFile = Join-Path $ExpectedAgentRoot "keys\agent99_ed25519"
|
||||
$EvidenceRoot = Join-Path $ExpectedAgentRoot "evidence\host110-backup-runtime"
|
||||
$SourceRootBase = Join-Path $ExpectedAgentRoot "deploy"
|
||||
$TransportMutexName = "Global\WoooAgent99Host110BackupRuntimeV2"
|
||||
$ExpectedFileCount = 18
|
||||
$ExpectedRemoteStageFileCount = 21
|
||||
$RuntimeRelativePaths = @(
|
||||
"scripts/backup/common.sh",
|
||||
"scripts/backup/backup-all.sh",
|
||||
"scripts/backup/backup-host188-products.sh",
|
||||
"scripts/backup/verify-host188-products-backup.sh",
|
||||
"scripts/backup/verify-host188-products-archive.py",
|
||||
"scripts/backup/backup-gitea.sh",
|
||||
"scripts/backup/gitea-full-backup-restore-drill.sh",
|
||||
"scripts/backup/backup-configs.sh",
|
||||
"scripts/backup/backup-awoooi.sh",
|
||||
"scripts/backup/backup-awoooi-frequent.sh",
|
||||
"scripts/backup/backup-clawbot.sh",
|
||||
"scripts/backup/backup-sentry.sh",
|
||||
"scripts/backup/check-backup-integrity.sh",
|
||||
"scripts/backup/sync-offsite-backups.sh",
|
||||
"scripts/backup/backup-offsite-readiness-gate.sh",
|
||||
"scripts/backup/verify-offsite-full-sync.sh",
|
||||
"scripts/backup/enforce-latest-only-retention.sh",
|
||||
"scripts/ops/backup-health-textfile-exporter.py"
|
||||
)
|
||||
$ExecutorRelativePath = "scripts/backup/host110-backup-runtime-executor.sh"
|
||||
$VerifierRelativePath = "scripts/backup/verify-host110-backup-runtime.py"
|
||||
|
||||
function Test-Agent99SafeIdentity {
|
||||
param([string]$Value)
|
||||
return [bool]($Value -and $Value -match "^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$")
|
||||
}
|
||||
|
||||
function Write-Agent99ImmutableJson {
|
||||
param([string]$Path, [object]$Value)
|
||||
|
||||
if (Test-Path -LiteralPath $Path) { throw "run_identity_evidence_exists" }
|
||||
$temporary = "$Path.tmp-$PID-$([Guid]::NewGuid().ToString('N'))"
|
||||
$encoding = New-Object System.Text.UTF8Encoding($false)
|
||||
try {
|
||||
$json = $Value | ConvertTo-Json -Depth 12
|
||||
[IO.File]::WriteAllText($temporary, $json + [Environment]::NewLine, $encoding)
|
||||
[IO.File]::Move($temporary, $Path)
|
||||
} finally {
|
||||
Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Agent99BoundedProcess {
|
||||
param(
|
||||
[string]$FilePath,
|
||||
[string[]]$Arguments,
|
||||
[int]$TimeoutSeconds,
|
||||
[string]$WorkingDirectory = "",
|
||||
[string]$StandardInputPath = ""
|
||||
)
|
||||
|
||||
$token = [Guid]::NewGuid().ToString("N")
|
||||
$stdoutPath = Join-Path $env:TEMP "agent99-host110-backup-$token.out"
|
||||
$stderrPath = Join-Path $env:TEMP "agent99-host110-backup-$token.err"
|
||||
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
$process = $null
|
||||
try {
|
||||
$start = @{
|
||||
FilePath = $FilePath
|
||||
ArgumentList = $Arguments
|
||||
NoNewWindow = $true
|
||||
RedirectStandardOutput = $stdoutPath
|
||||
RedirectStandardError = $stderrPath
|
||||
PassThru = $true
|
||||
}
|
||||
if ($WorkingDirectory) { $start.WorkingDirectory = $WorkingDirectory }
|
||||
if ($StandardInputPath) { $start.RedirectStandardInput = $StandardInputPath }
|
||||
$process = Start-Process @start
|
||||
$null = $process.Handle
|
||||
if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
|
||||
try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {}
|
||||
if (-not $process.WaitForExit(2000) -and -not $process.HasExited) {
|
||||
$process.Kill()
|
||||
$process.WaitForExit(2000) | Out-Null
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
ok = $false
|
||||
exitCode = -2
|
||||
reason = "transport_timeout"
|
||||
elapsedMs = $stopwatch.ElapsedMilliseconds
|
||||
stdout = ""
|
||||
stderrPresent = $false
|
||||
}
|
||||
}
|
||||
$process.WaitForExit() | Out-Null
|
||||
$process.Refresh()
|
||||
$stdout = if (Test-Path -LiteralPath $stdoutPath) { Get-Content -LiteralPath $stdoutPath -Raw } else { "" }
|
||||
$stderr = if (Test-Path -LiteralPath $stderrPath) { Get-Content -LiteralPath $stderrPath -Raw } else { "" }
|
||||
if ($stdout.Length -gt 65536 -or $stderr.Length -gt 65536) {
|
||||
return [pscustomobject]@{
|
||||
ok = $false
|
||||
exitCode = -3
|
||||
reason = "transport_output_oversized"
|
||||
elapsedMs = $stopwatch.ElapsedMilliseconds
|
||||
stdout = ""
|
||||
stderrPresent = [bool]$stderr
|
||||
}
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
ok = [bool]($process.ExitCode -eq 0)
|
||||
exitCode = [int]$process.ExitCode
|
||||
reason = if ($process.ExitCode -eq 0) { "completed" } else { "process_failed" }
|
||||
elapsedMs = $stopwatch.ElapsedMilliseconds
|
||||
stdout = [string]$stdout
|
||||
stderrPresent = [bool]$stderr
|
||||
}
|
||||
} catch {
|
||||
return [pscustomobject]@{
|
||||
ok = $false
|
||||
exitCode = -1
|
||||
reason = "transport_exception"
|
||||
elapsedMs = $stopwatch.ElapsedMilliseconds
|
||||
stdout = ""
|
||||
stderrPresent = $false
|
||||
errorType = $_.Exception.GetType().Name
|
||||
}
|
||||
} finally {
|
||||
$stopwatch.Stop()
|
||||
Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null
|
||||
if ($process) { $process.Dispose() }
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Agent99BoundedSsh {
|
||||
param([string]$RemoteCommand, [int]$TimeoutSeconds, [string]$StandardInputPath = "")
|
||||
|
||||
$arguments = @(
|
||||
"-i", $IdentityFile,
|
||||
"-o", "IdentitiesOnly=yes",
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "NumberOfPasswordPrompts=0",
|
||||
"-o", "StrictHostKeyChecking=yes",
|
||||
"-o", "ConnectTimeout=8",
|
||||
"-o", "ConnectionAttempts=1",
|
||||
"-o", "ServerAliveInterval=10",
|
||||
"-o", "ServerAliveCountMax=2",
|
||||
"-l", $TargetUser,
|
||||
$TargetHost,
|
||||
$RemoteCommand
|
||||
)
|
||||
return Invoke-Agent99BoundedProcess "ssh.exe" $arguments $TimeoutSeconds "" $StandardInputPath
|
||||
}
|
||||
|
||||
function Invoke-Agent99BoundedScp {
|
||||
param([string[]]$LocalPaths, [string]$RemoteDirectory, [int]$TimeoutSeconds)
|
||||
|
||||
$arguments = @(
|
||||
"-i", $IdentityFile,
|
||||
"-o", "IdentitiesOnly=yes",
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "NumberOfPasswordPrompts=0",
|
||||
"-o", "StrictHostKeyChecking=yes",
|
||||
"-o", "ConnectTimeout=8",
|
||||
"-o", "ConnectionAttempts=1"
|
||||
)
|
||||
$arguments += $LocalPaths
|
||||
$arguments += "${TargetUser}@${TargetHost}:${RemoteDirectory}/"
|
||||
return Invoke-Agent99BoundedProcess "scp.exe" $arguments $TimeoutSeconds
|
||||
}
|
||||
|
||||
function Invoke-Agent99Git {
|
||||
param([string[]]$Arguments, [int]$TimeoutSeconds, [string]$WorkingDirectory = "")
|
||||
|
||||
$previousPrompt = $env:GIT_TERMINAL_PROMPT
|
||||
try {
|
||||
$env:GIT_TERMINAL_PROMPT = "0"
|
||||
return Invoke-Agent99BoundedProcess $GitPath $Arguments $TimeoutSeconds $WorkingDirectory
|
||||
} finally {
|
||||
$env:GIT_TERMINAL_PROMPT = $previousPrompt
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Agent99ProcessOk {
|
||||
param([object]$Result, [string]$Reason)
|
||||
if (-not $Result.ok) { throw $Reason }
|
||||
}
|
||||
|
||||
function New-Agent99SourcePackage {
|
||||
param([string]$SourceRoot, [string]$ManifestPath)
|
||||
|
||||
if (Test-Path -LiteralPath $SourceRoot) { throw "run_identity_source_stage_exists" }
|
||||
$clone = Invoke-Agent99Git @(
|
||||
"-c", "credential.helper=",
|
||||
"-c", "core.autocrlf=false",
|
||||
"clone", "--quiet", "--no-tags", "--depth", "200", "--branch", "main",
|
||||
$GiteaSourceUrl, $SourceRoot
|
||||
) 180 $SourceRootBase
|
||||
Assert-Agent99ProcessOk $clone "gitea_source_fetch_failed"
|
||||
|
||||
$headResult = Invoke-Agent99Git @("rev-parse", "HEAD") 30 $SourceRoot
|
||||
Assert-Agent99ProcessOk $headResult "gitea_source_head_failed"
|
||||
$sourceHead = ([string]$headResult.stdout).Trim().ToLowerInvariant()
|
||||
if ($sourceHead -notmatch "^[0-9a-f]{40}$") { throw "gitea_source_head_invalid" }
|
||||
|
||||
$commitResult = Invoke-Agent99Git @("cat-file", "-e", "$SourceRevision`^{commit}") 30 $SourceRoot
|
||||
Assert-Agent99ProcessOk $commitResult "source_revision_not_fetched"
|
||||
$ancestorResult = Invoke-Agent99Git @("merge-base", "--is-ancestor", $SourceRevision, $sourceHead) 30 $SourceRoot
|
||||
Assert-Agent99ProcessOk $ancestorResult "source_revision_not_on_gitea_main"
|
||||
|
||||
$allPaths = @($RuntimeRelativePaths) + @($ExecutorRelativePath, $VerifierRelativePath)
|
||||
$checkoutResult = Invoke-Agent99Git (@("-c", "core.autocrlf=false", "checkout", "--quiet", $SourceRevision, "--") + $allPaths) 60 $SourceRoot
|
||||
Assert-Agent99ProcessOk $checkoutResult "source_revision_checkout_failed"
|
||||
|
||||
$rows = @()
|
||||
$localPaths = @()
|
||||
foreach ($relativePath in $RuntimeRelativePaths) {
|
||||
$path = Join-Path $SourceRoot ($relativePath -replace "/", "\")
|
||||
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { throw "runtime_source_file_missing" }
|
||||
$rows += [ordered]@{
|
||||
name = [IO.Path]::GetFileName($path)
|
||||
sha256 = (Get-FileHash -LiteralPath $path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
$localPaths += $path
|
||||
}
|
||||
if (($rows.name | Sort-Object -Unique).Count -ne $ExpectedFileCount) { throw "runtime_source_file_set_invalid" }
|
||||
|
||||
$executorPath = Join-Path $SourceRoot ($ExecutorRelativePath -replace "/", "\")
|
||||
$verifierPath = Join-Path $SourceRoot ($VerifierRelativePath -replace "/", "\")
|
||||
if (-not (Test-Path -LiteralPath $executorPath -PathType Leaf)) { throw "executor_source_file_missing" }
|
||||
if (-not (Test-Path -LiteralPath $verifierPath -PathType Leaf)) { throw "verifier_source_file_missing" }
|
||||
$executorDigest = (Get-FileHash -LiteralPath $executorPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$verifierDigest = (Get-FileHash -LiteralPath $verifierPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$manifest = [ordered]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_source_manifest_v1"
|
||||
sourceRevision = $SourceRevision
|
||||
sourceHead = $sourceHead
|
||||
runId = $RunId
|
||||
executorSha256 = $executorDigest
|
||||
verifierSha256 = $verifierDigest
|
||||
files = $rows
|
||||
}
|
||||
$encoding = New-Object System.Text.UTF8Encoding($false)
|
||||
[IO.File]::WriteAllText($ManifestPath, ($manifest | ConvertTo-Json -Depth 8 -Compress), $encoding)
|
||||
$localPaths += $executorPath, $verifierPath, $ManifestPath
|
||||
return [pscustomobject]@{
|
||||
sourceHead = $sourceHead
|
||||
executorSha256 = $executorDigest
|
||||
verifierSha256 = $verifierDigest
|
||||
verifierPath = $verifierPath
|
||||
manifestPath = $ManifestPath
|
||||
manifestBase64 = [Convert]::ToBase64String([IO.File]::ReadAllBytes($ManifestPath))
|
||||
localPaths = $localPaths
|
||||
}
|
||||
}
|
||||
|
||||
function Test-Agent99JsonField {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$Name,
|
||||
[ValidateSet("String", "Boolean", "Integer", "Object", "Null", "NullOrString")]
|
||||
[string]$Kind
|
||||
)
|
||||
|
||||
if ($null -eq $Object) { return $false }
|
||||
$property = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $false }
|
||||
$value = $property.Value
|
||||
switch ($Kind) {
|
||||
"String" { return [bool]($value -is [string]) }
|
||||
"Boolean" { return [bool]($value -is [bool]) }
|
||||
"Integer" { return [bool]($value -is [int] -or $value -is [long]) }
|
||||
"Object" { return [bool]($value -is [System.Management.Automation.PSCustomObject]) }
|
||||
"Null" { return [bool]($null -eq $value) }
|
||||
"NullOrString" { return [bool]($null -eq $value -or $value -is [string]) }
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
function Assert-Agent99JsonFields {
|
||||
param([object]$Object, [hashtable]$Contract, [string]$ErrorCode)
|
||||
|
||||
foreach ($name in $Contract.Keys) {
|
||||
if (-not (Test-Agent99JsonField $Object $name ([string]$Contract[$name]))) {
|
||||
throw $ErrorCode
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Convert-Agent99ExecutorDocument {
|
||||
param(
|
||||
[object]$Transport,
|
||||
[string]$ExpectedMode,
|
||||
[switch]$AllowProcessFailure
|
||||
)
|
||||
|
||||
if (-not $Transport.ok) {
|
||||
if (
|
||||
-not $AllowProcessFailure -or
|
||||
[string]$Transport.reason -ne "process_failed" -or
|
||||
[int]$Transport.exitCode -le 0 -or
|
||||
[string]::IsNullOrWhiteSpace([string]$Transport.stdout)
|
||||
) {
|
||||
throw "host110_${ExpectedMode}_transport_failed"
|
||||
}
|
||||
}
|
||||
try {
|
||||
$parsed = [string]$Transport.stdout | ConvertFrom-Json
|
||||
} catch {
|
||||
throw "host110_${ExpectedMode}_receipt_invalid"
|
||||
}
|
||||
Assert-Agent99JsonFields $parsed @{
|
||||
schemaVersion = "String"
|
||||
mode = "String"
|
||||
ok = "Boolean"
|
||||
sourceRevision = "String"
|
||||
sourceHead = "String"
|
||||
runId = "String"
|
||||
fileCount = "Integer"
|
||||
backupScriptFileCount = "Integer"
|
||||
backupHealthExporterIncluded = "Boolean"
|
||||
} "host110_${ExpectedMode}_receipt_required_field_failed"
|
||||
if (
|
||||
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_executor_v1" -or
|
||||
[string]$parsed.mode -ne $ExpectedMode.ToLowerInvariant() -or
|
||||
[string]$parsed.sourceRevision -ne $SourceRevision -or
|
||||
[string]$parsed.sourceHead -notmatch "^[0-9a-f]{40}$" -or
|
||||
[string]$parsed.runId -ne $RunId -or
|
||||
[int64]$parsed.fileCount -ne $ExpectedFileCount -or
|
||||
[int64]$parsed.backupScriptFileCount -ne 17 -or
|
||||
-not [bool]$parsed.backupHealthExporterIncluded
|
||||
) {
|
||||
throw "host110_${ExpectedMode}_receipt_contract_failed"
|
||||
}
|
||||
return $parsed
|
||||
}
|
||||
|
||||
function Assert-Agent99ApplyResultFields {
|
||||
param([object]$Parsed, [string]$VerifierKind)
|
||||
|
||||
Assert-Agent99JsonFields $Parsed @{
|
||||
status = "String"
|
||||
payloadCommitted = "Boolean"
|
||||
rollbackAttempted = "Boolean"
|
||||
rollbackPerformed = "Boolean"
|
||||
rollbackVerified = "Boolean"
|
||||
zeroResidueVerified = "Boolean"
|
||||
zeroResidueScope = "String"
|
||||
stageCleanupVerified = "Boolean"
|
||||
rollbackPrestateCleanupVerified = "Boolean"
|
||||
receipt = "String"
|
||||
terminalReceipt = "String"
|
||||
terminalReceiptPublished = "Boolean"
|
||||
terminalExitCode = "Integer"
|
||||
deferredSignal = "NullOrString"
|
||||
independentVerifierVerified = "Boolean"
|
||||
verifier = $VerifierKind
|
||||
} "host110_apply_receipt_required_field_failed"
|
||||
}
|
||||
|
||||
function Assert-Agent99VerifierFields {
|
||||
param([object]$Verifier)
|
||||
|
||||
Assert-Agent99JsonFields $Verifier @{
|
||||
schemaVersion = "String"
|
||||
ok = "Boolean"
|
||||
sourceRevision = "String"
|
||||
runId = "String"
|
||||
fileCount = "Integer"
|
||||
backupScriptFileCount = "Integer"
|
||||
backupHealthExporterIncluded = "Boolean"
|
||||
remoteWritePerformed = "Boolean"
|
||||
selfIdentityVerified = "Boolean"
|
||||
} "host110_apply_verifier_required_field_failed"
|
||||
}
|
||||
|
||||
function Get-Agent99ExecutorResult {
|
||||
param([object]$Transport, [string]$ExpectedMode)
|
||||
|
||||
$parsed = Convert-Agent99ExecutorDocument $Transport $ExpectedMode
|
||||
if (-not [bool]$parsed.ok) { throw "host110_${ExpectedMode}_receipt_contract_failed" }
|
||||
if ($ExpectedMode -eq "Apply") {
|
||||
Assert-Agent99ApplyResultFields $parsed "Object"
|
||||
Assert-Agent99VerifierFields $parsed.verifier
|
||||
if (
|
||||
[string]$parsed.status -ne "verified" -or
|
||||
-not [bool]$parsed.payloadCommitted -or
|
||||
[bool]$parsed.rollbackAttempted -or
|
||||
[bool]$parsed.rollbackPerformed -or
|
||||
[bool]$parsed.rollbackVerified -or
|
||||
-not [bool]$parsed.zeroResidueVerified -or
|
||||
[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps" -or
|
||||
-not [bool]$parsed.stageCleanupVerified -or
|
||||
-not [bool]$parsed.rollbackPrestateCleanupVerified -or
|
||||
[int64]$parsed.terminalExitCode -ne 0 -or
|
||||
$null -ne $parsed.deferredSignal -or
|
||||
-not [bool]$parsed.independentVerifierVerified -or
|
||||
[string]$parsed.receipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or
|
||||
[string]$parsed.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json" -or
|
||||
-not [bool]$parsed.terminalReceiptPublished -or
|
||||
[string]$parsed.verifier.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
|
||||
-not [bool]$parsed.verifier.ok -or
|
||||
[string]$parsed.verifier.sourceRevision -ne $SourceRevision -or
|
||||
[string]$parsed.verifier.runId -ne $RunId -or
|
||||
[int64]$parsed.verifier.fileCount -ne $ExpectedFileCount -or
|
||||
[int64]$parsed.verifier.backupScriptFileCount -ne 17 -or
|
||||
-not [bool]$parsed.verifier.backupHealthExporterIncluded -or
|
||||
[bool]$parsed.verifier.remoteWritePerformed -or
|
||||
-not [bool]$parsed.verifier.selfIdentityVerified
|
||||
) {
|
||||
throw "host110_apply_independent_verifier_contract_failed"
|
||||
}
|
||||
}
|
||||
return $parsed
|
||||
}
|
||||
|
||||
function Get-Agent99CommittedFailureResult {
|
||||
param([object]$Transport)
|
||||
|
||||
$parsed = Convert-Agent99ExecutorDocument $Transport "Apply" -AllowProcessFailure
|
||||
Assert-Agent99ApplyResultFields $parsed "Null"
|
||||
if (
|
||||
[bool]$parsed.ok -or
|
||||
-not [bool]$parsed.payloadCommitted -or
|
||||
[bool]$parsed.rollbackAttempted -or
|
||||
[bool]$parsed.rollbackPerformed -or
|
||||
[bool]$parsed.rollbackVerified -or
|
||||
-not [bool]$parsed.zeroResidueVerified -or
|
||||
[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps" -or
|
||||
-not [bool]$parsed.independentVerifierVerified -or
|
||||
[string]$parsed.receipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or
|
||||
[string]$parsed.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json"
|
||||
) {
|
||||
throw "host110_apply_committed_failure_contract_failed"
|
||||
}
|
||||
$signalExitCodes = @{ INT = 130; HUP = 129; TERM = 143 }
|
||||
if ([string]$parsed.status -eq "cleanup_pending") {
|
||||
if (
|
||||
[int]$Transport.exitCode -ne 75 -or
|
||||
[int64]$parsed.terminalExitCode -ne 75 -or
|
||||
([bool]$parsed.stageCleanupVerified -and [bool]$parsed.rollbackPrestateCleanupVerified) -or
|
||||
($null -ne $parsed.deferredSignal -and -not $signalExitCodes.ContainsKey([string]$parsed.deferredSignal))
|
||||
) {
|
||||
throw "host110_apply_cleanup_pending_contract_failed"
|
||||
}
|
||||
if (-not [bool]$parsed.terminalReceiptPublished) {
|
||||
throw "host110_apply_cleanup_pending_terminal_receipt_missing"
|
||||
}
|
||||
} elseif ([string]$parsed.status -eq "committed_signal_deferred") {
|
||||
$signalName = [string]$parsed.deferredSignal
|
||||
if (
|
||||
-not $signalExitCodes.ContainsKey($signalName) -or
|
||||
-not [bool]$parsed.stageCleanupVerified -or
|
||||
-not [bool]$parsed.rollbackPrestateCleanupVerified -or
|
||||
[int]$Transport.exitCode -ne [int]$signalExitCodes[$signalName] -or
|
||||
[int64]$parsed.terminalExitCode -ne [int]$signalExitCodes[$signalName]
|
||||
) {
|
||||
throw "host110_apply_deferred_signal_contract_failed"
|
||||
}
|
||||
if (-not [bool]$parsed.terminalReceiptPublished) {
|
||||
throw "host110_apply_deferred_signal_terminal_receipt_missing"
|
||||
}
|
||||
} elseif ([string]$parsed.status -eq "terminal_receipt_failed") {
|
||||
if (
|
||||
[bool]$parsed.terminalReceiptPublished -or
|
||||
[int]$Transport.exitCode -ne 76 -or
|
||||
[int64]$parsed.terminalExitCode -ne 76
|
||||
) {
|
||||
throw "host110_apply_terminal_receipt_failure_contract_failed"
|
||||
}
|
||||
} else {
|
||||
throw "host110_apply_committed_failure_status_invalid"
|
||||
}
|
||||
return $parsed
|
||||
}
|
||||
|
||||
function Get-Agent99VerifierResult {
|
||||
param([object]$Transport, [object]$SourcePackage)
|
||||
|
||||
if (-not $Transport.ok) { throw "host110_verify_transport_failed" }
|
||||
try {
|
||||
$parsed = [string]$Transport.stdout | ConvertFrom-Json
|
||||
} catch {
|
||||
throw "host110_verify_receipt_invalid"
|
||||
}
|
||||
Assert-Agent99JsonFields $parsed @{
|
||||
schemaVersion = "String"
|
||||
ok = "Boolean"
|
||||
sourceRevision = "String"
|
||||
runId = "String"
|
||||
fileCount = "Integer"
|
||||
backupScriptFileCount = "Integer"
|
||||
backupHealthExporterIncluded = "Boolean"
|
||||
remoteWritePerformed = "Boolean"
|
||||
verifierSha256 = "String"
|
||||
} "host110_verify_receipt_required_field_failed"
|
||||
if (
|
||||
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
|
||||
-not [bool]$parsed.ok -or
|
||||
[string]$parsed.sourceRevision -ne $SourceRevision -or
|
||||
[string]$parsed.runId -ne $RunId -or
|
||||
[int]$parsed.fileCount -ne $ExpectedFileCount -or
|
||||
[int]$parsed.backupScriptFileCount -ne 17 -or
|
||||
-not [bool]$parsed.backupHealthExporterIncluded -or
|
||||
[bool]$parsed.remoteWritePerformed -or
|
||||
[string]$parsed.verifierSha256 -ne [string]$SourcePackage.verifierSha256
|
||||
) {
|
||||
throw "host110_verify_receipt_contract_failed"
|
||||
}
|
||||
return $parsed
|
||||
}
|
||||
|
||||
function Invoke-Agent99ReadOnlyPreflight {
|
||||
param([object]$SourcePackage)
|
||||
|
||||
$command = 'test "$(id -un)" = wooo && hostname -I | tr " " "\n" | grep -qx 192.168.0.110 && test -d /backup/scripts && test ! -L /backup/scripts && test -d /home/wooo/scripts && test ! -L /home/wooo/scripts && command -v timeout >/dev/null && command -v flock >/dev/null && command -v python3 >/dev/null && printf HOST110_BACKUP_PREFLIGHT_OK=1'
|
||||
$transport = Invoke-Agent99BoundedSsh $command 45
|
||||
if (-not $transport.ok -or ([string]$transport.stdout).Trim() -ne "HOST110_BACKUP_PREFLIGHT_OK=1") {
|
||||
throw "host110_read_only_preflight_failed"
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_preflight_v1"
|
||||
ok = $true
|
||||
sourceRevision = $SourceRevision
|
||||
sourceHead = [string]$SourcePackage.sourceHead
|
||||
fileCount = $ExpectedFileCount
|
||||
backupScriptFileCount = 17
|
||||
backupHealthExporterIncluded = $true
|
||||
remoteWritePerformed = $false
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Agent99NoWriteVerifier {
|
||||
param([object]$SourcePackage)
|
||||
|
||||
$command = "timeout --signal=TERM --kill-after=5s 60s python3 - --manifest-base64 $($SourcePackage.manifestBase64) --destination /backup/scripts --exporter-destination /home/wooo/scripts/backup-health-textfile-exporter.py --source-revision $SourceRevision --run-id $RunId --expected-verifier-sha256 $($SourcePackage.verifierSha256)"
|
||||
$transport = Invoke-Agent99BoundedSsh $command 90 ([string]$SourcePackage.verifierPath)
|
||||
return Get-Agent99VerifierResult $transport $SourcePackage
|
||||
}
|
||||
|
||||
function Remove-Agent99RemoteStage {
|
||||
param([string]$RemoteStage, [bool]$Required)
|
||||
|
||||
$cleanup = Invoke-Agent99BoundedSsh "rm -rf -- $RemoteStage && test ! -e $RemoteStage" 45
|
||||
if ($Required -and -not $cleanup.ok) { throw "remote_source_stage_cleanup_failed" }
|
||||
return [bool]$cleanup.ok
|
||||
}
|
||||
|
||||
if ($AgentRoot.TrimEnd("\") -ne $ExpectedAgentRoot) { throw "agent_root_override_forbidden" }
|
||||
$SourceRevision = $SourceRevision.ToLowerInvariant()
|
||||
if ($SourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
|
||||
if (-not (Test-Agent99SafeIdentity $RunId)) { throw "invalid_run_id" }
|
||||
if (-not (Test-Path -LiteralPath $IdentityFile -PathType Leaf)) { throw "agent99_ssh_identity_unavailable" }
|
||||
if (-not (Test-Path -LiteralPath $GitPath -PathType Leaf)) { throw "windows_git_unavailable" }
|
||||
if (-not (Get-Command ssh.exe -ErrorAction SilentlyContinue)) { throw "windows_openssh_unavailable" }
|
||||
if (-not (Get-Command scp.exe -ErrorAction SilentlyContinue)) { throw "windows_openscp_unavailable" }
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $EvidenceRoot, $SourceRootBase | Out-Null
|
||||
$evidencePath = Join-Path $EvidenceRoot "agent99-host110-backup-runtime-$RunId.json"
|
||||
if (Test-Path -LiteralPath $evidencePath -PathType Leaf) {
|
||||
try { $existing = Get-Content -LiteralPath $evidencePath -Raw | ConvertFrom-Json } catch { throw "run_identity_evidence_invalid" }
|
||||
if ([string]$existing.sourceRevision -ne $SourceRevision -or [string]$existing.mode -ne $Mode) {
|
||||
throw "run_identity_conflict"
|
||||
}
|
||||
$existing | ConvertTo-Json -Compress -Depth 12
|
||||
if (-not [bool]$existing.ok) { exit 1 }
|
||||
exit 0
|
||||
}
|
||||
|
||||
$sourceRoot = Join-Path $SourceRootBase "host110-backup-source-$RunId"
|
||||
$manifestPath = Join-Path $sourceRoot "manifest.json"
|
||||
$remoteStage = "/tmp/agent99-host110-backup-runtime-$RunId"
|
||||
$startedAt = [DateTime]::UtcNow
|
||||
$mutex = [Threading.Mutex]::new($false, $TransportMutexName)
|
||||
$acquired = $false
|
||||
$remoteStageCreated = $false
|
||||
$sourcePackage = $null
|
||||
$check = $null
|
||||
$apply = $null
|
||||
$verify = $null
|
||||
$terminalStatus = "failed"
|
||||
$errorCode = ""
|
||||
$cleanupVerified = $false
|
||||
|
||||
try {
|
||||
try {
|
||||
$acquired = [bool]$mutex.WaitOne(30000)
|
||||
} catch [Threading.AbandonedMutexException] {
|
||||
$acquired = $true
|
||||
}
|
||||
if (-not $acquired) { throw "host110_backup_runtime_transport_busy" }
|
||||
|
||||
$sourcePackage = New-Agent99SourcePackage $sourceRoot $manifestPath
|
||||
$check = Invoke-Agent99ReadOnlyPreflight $sourcePackage
|
||||
|
||||
if ($Mode -eq "Apply") {
|
||||
$createStage = Invoke-Agent99BoundedSsh "test ! -e $remoteStage && install -d -m 700 $remoteStage" 45
|
||||
Assert-Agent99ProcessOk $createStage "remote_source_stage_identity_conflict"
|
||||
$remoteStageCreated = $true
|
||||
$copyStage = Invoke-Agent99BoundedScp $sourcePackage.localPaths $remoteStage 120
|
||||
Assert-Agent99ProcessOk $copyStage "remote_source_stage_copy_failed"
|
||||
$stageCheck = Invoke-Agent99BoundedSsh "chmod 700 $remoteStage/host110-backup-runtime-executor.sh $remoteStage/verify-host110-backup-runtime.py && test `$(find $remoteStage -mindepth 1 -maxdepth 1 -type f | wc -l) -eq $ExpectedRemoteStageFileCount" 45
|
||||
Assert-Agent99ProcessOk $stageCheck "remote_source_stage_contract_failed"
|
||||
|
||||
$executorPath = "$remoteStage/host110-backup-runtime-executor.sh"
|
||||
$command = "timeout --signal=TERM --kill-after=10s 300s $executorPath --mode apply --source-revision $SourceRevision --run-id $RunId --source-stage $remoteStage"
|
||||
$applyTransport = Invoke-Agent99BoundedSsh $command 330
|
||||
if ($applyTransport.ok) {
|
||||
$apply = Get-Agent99ExecutorResult $applyTransport "Apply"
|
||||
} else {
|
||||
$apply = Get-Agent99CommittedFailureResult $applyTransport
|
||||
throw "host110_apply_$([string]$apply.status)"
|
||||
}
|
||||
$verify = $apply.verifier
|
||||
$cleanupVerified = Remove-Agent99RemoteStage $remoteStage $true
|
||||
$remoteStageCreated = $false
|
||||
$terminalStatus = "verified"
|
||||
} elseif ($Mode -eq "Verify") {
|
||||
$verify = Invoke-Agent99NoWriteVerifier $sourcePackage
|
||||
$terminalStatus = "readback_ok"
|
||||
$cleanupVerified = $true
|
||||
} else {
|
||||
$terminalStatus = "readback_ok"
|
||||
$cleanupVerified = $true
|
||||
}
|
||||
} catch {
|
||||
$errorCode = [string]$_.Exception.Message
|
||||
} finally {
|
||||
if ($remoteStageCreated) {
|
||||
try { $cleanupVerified = Remove-Agent99RemoteStage $remoteStage $false } catch {}
|
||||
}
|
||||
if ($acquired) { try { $mutex.ReleaseMutex() } catch {} }
|
||||
$mutex.Dispose()
|
||||
}
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v3"
|
||||
ok = [bool]($terminalStatus -ne "failed")
|
||||
status = $terminalStatus
|
||||
mode = $Mode
|
||||
sourceRevision = $SourceRevision
|
||||
sourceHead = if ($sourcePackage) { [string]$sourcePackage.sourceHead } else { "" }
|
||||
runId = $RunId
|
||||
targetHost = $TargetHost
|
||||
sourceTransport = "windows99_gitea_exact_revision_manifest"
|
||||
executor = "host110_backup_runtime_executor"
|
||||
verifier = "independent_host110_backup_runtime_python_readback"
|
||||
decisionProvider = "deterministic_only"
|
||||
criticProvider = "deterministic_only"
|
||||
agentAction = if ($Mode -eq "Apply") { "controlled_apply" } else { "readback" }
|
||||
check = $check
|
||||
apply = $apply
|
||||
verify = $verify
|
||||
cleanupVerified = $cleanupVerified
|
||||
errorCode = $errorCode
|
||||
elapsedSeconds = [math]::Round(([DateTime]::UtcNow - $startedAt).TotalSeconds, 3)
|
||||
evidence = $evidencePath
|
||||
secretValuesRead = $false
|
||||
rawSessionStored = $false
|
||||
}
|
||||
|
||||
try {
|
||||
Write-Agent99ImmutableJson $evidencePath $result
|
||||
} finally {
|
||||
if (Test-Path -LiteralPath $sourceRoot) {
|
||||
Remove-Item -LiteralPath $sourceRoot -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
$result | ConvertTo-Json -Compress -Depth 12
|
||||
if (-not $result.ok) { exit 1 }
|
||||
@@ -21,7 +21,6 @@ $IdentityFile = Join-Path $AgentRoot "keys\agent99_ed25519"
|
||||
$EvidenceDir = Join-Path $AgentRoot "evidence"
|
||||
$RuntimeManifestPath = Join-Path $AgentRoot "state\runtime-manifest.json"
|
||||
$EntrypointPath = [string]$MyInvocation.MyCommand.Path
|
||||
$ExpectedRuntimeFileCount = 19
|
||||
$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"
|
||||
$BundleId = "bb6d78b67f506072b2e45e92bc1d415364e25852282a9b068900665c578df011"
|
||||
$ToolchainRoot = "/backup/toolchains/signoz-metadata/$BundleId"
|
||||
@@ -221,7 +220,7 @@ function Get-Agent99RuntimeSourceBinding {
|
||||
}
|
||||
$result.ok = [bool](
|
||||
$result.runtimeMatched -and
|
||||
$result.fileCount -eq $ExpectedRuntimeFileCount -and
|
||||
$result.fileCount -eq 18 -and
|
||||
$result.mismatchCount -eq 0 -and
|
||||
$result.sourceRevision -eq $SourceRevision -and
|
||||
$result.entrypointTracked -and
|
||||
|
||||
@@ -1,217 +0,0 @@
|
||||
# AWOOOI Agent99 Host110 Backup Runtime V10
|
||||
|
||||
Status: proposed; central review and explicit owner approval are required before
|
||||
any production apply.
|
||||
|
||||
This artifact supersedes V2 through V9. Every V9 identity below is void for
|
||||
approval and remains audit evidence only:
|
||||
|
||||
- branch/ref: `codex/host110-backup-runtime-v9-20260718`
|
||||
- candidate/revision: `8f11ef3362a8d7f66a46308b58fd142b0c91a8bc`
|
||||
- artifact SHA-256: `929978eb3da15b04791a8614e6d5ee504ae2a4c2f205b694307a7bad7713bcad`
|
||||
- exact diff SHA-256: `8085d49450b74e7c43577ba118cbd74d3a2e3dd2d22eeba90ce749d56c72c0f2`
|
||||
- every other V9 proposal, content, revision, and derived hash
|
||||
|
||||
No V9 hash, ref, review, receipt, or test grants production execution
|
||||
authority.
|
||||
|
||||
## Exact Scope
|
||||
|
||||
- Agent99 runtime bundle: 19 files, guarded by one producer/consumer parity
|
||||
test.
|
||||
- Host110 backup payload: 18 ordered files, comprising 17 backup scripts and
|
||||
`backup-health-textfile-exporter.py`.
|
||||
- Host110 remote stage: 21 unique basenames, comprising the ordered 18-file
|
||||
payload, fixed executor, independent verifier, and `manifest.json`.
|
||||
- Fixed target: `wooo@192.168.0.110`, dispatched by Windows99 Agent99 from an
|
||||
exact Gitea revision and digest manifest.
|
||||
- The candidate includes a 10-line runtime-lock addition to
|
||||
`scripts/backup/gitea-full-backup-restore-drill.sh`. A future drill invocation
|
||||
acquires the shared Host110 backup-runtime lock before its pre-existing drill
|
||||
behavior. Applying this candidate replaces the script but does not invoke the
|
||||
drill or any container lifecycle.
|
||||
- `scripts/backup/backup-gitea.sh` has zero diff from the production base. No
|
||||
Gitea primary stop/start or backup-container creation is part of this
|
||||
candidate apply.
|
||||
- V7, V8, and V9 proposal documents remain tracked audit artifacts and are not
|
||||
runtime authority.
|
||||
- `agent99-host110-broker-contract-replay.ps1` is a no-write Windows99 test
|
||||
asset; it is not part of the 19-file Agent99 runtime or 18-file Host110
|
||||
payload.
|
||||
|
||||
## V9 Defects Closed
|
||||
|
||||
### Signal-Safe Commit Boundary
|
||||
|
||||
The executor enters a signal-deferring critical section before publishing the
|
||||
payload receipt. `INT`, `HUP`, and `TERM` are recorded without exiting until the
|
||||
durable payload-receipt readback and `APPLY_COMPLETE=1` commit state are both
|
||||
established. A deferred signal can never enter the pre-commit EXIT rollback
|
||||
path after the payload receipt exists.
|
||||
|
||||
After commit, exact stage and rollback-prestate cleanup still run. If cleanup
|
||||
is complete, a deferred signal produces the non-success terminal
|
||||
`committed_signal_deferred`, a durable terminal receipt, the corresponding
|
||||
129/130/143 exit, and an applied payload. TERM, HUP, and INT are replayed in the
|
||||
exact receipt-to-commit window.
|
||||
|
||||
After cleanup, terminal publication uses a bounded signal mask. Signals captured
|
||||
before that boundary remain part of the non-success terminal; signals delivered
|
||||
during the immutable terminal writer cannot mutate its already-final state.
|
||||
Terminal-writer TERM, HUP, and INT replays require receipt, stdout, payload, and
|
||||
exit to remain consistent.
|
||||
|
||||
### Payload And Terminal Receipts
|
||||
|
||||
The immutable payload receipt has status `payload_verified`; it proves exact
|
||||
payload, independent verifier, durable receipt readback, and absence of
|
||||
run-owned destination temporaries. It does not claim internal stage or rollback
|
||||
prestate cleanup.
|
||||
|
||||
A separate immutable terminal receipt is the authority for overall success.
|
||||
Only status `verified` with `stageCleanupVerified=true`,
|
||||
`rollbackPrestateCleanupVerified=true`, no deferred signal, and exit code zero
|
||||
may yield executor `ok:true`. Each successful cleanup is absence-read back and
|
||||
its parent directory is fsynced before terminal publication.
|
||||
|
||||
If either cleanup fails, the payload remains committed, exact residue is
|
||||
preserved, terminal status is `cleanup_pending`, executor output is
|
||||
`ok:false`, and exit is nonzero. A payload receipt cannot be used as a
|
||||
zero-residue or broker-success shortcut. The Windows99 broker validates every
|
||||
status, payload, rollback, residue-scope, cleanup, signal, exit, receipt-path,
|
||||
and independent-verifier field fail-closed.
|
||||
Committed non-success outputs are parsed and retained in broker evidence before
|
||||
the broker fails overall; `cleanup_pending`, `committed_signal_deferred`, and
|
||||
`terminal_receipt_failed` can never be collapsed into an ambiguous generic
|
||||
transport failure.
|
||||
|
||||
### Existing Transaction Guarantees Retained
|
||||
|
||||
- Producer and executor payload basenames are exact-equal in count, set, and
|
||||
order: 18 payload files and 21 unique staged files.
|
||||
- Receipt write, hard-link, fsync, and readback failures roll back exact content
|
||||
and metadata without publishing success.
|
||||
- Terminal-receipt write, hard-link, fsync, and readback failures keep the
|
||||
verified payload, publish no terminal success, emit structured committed
|
||||
failure evidence with `terminalReceiptPublished=false`, and exit nonzero.
|
||||
- Apply and rollback destination temporaries are run-owned, tracked, removed,
|
||||
and absence-read back.
|
||||
- Rollback attempted, performed, verified, and residue truth remain separate;
|
||||
partial or unknown rollback terminates `rollback_unverified`.
|
||||
- Fault hooks require both the explicit test flag and a transformed
|
||||
non-production destination; canonical `/backup/scripts` cannot enable them.
|
||||
|
||||
## Apply Effects
|
||||
|
||||
- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded
|
||||
manifest and receipt state.
|
||||
- Host110 Apply replaces the ordered 18 payload paths through one exclusive,
|
||||
digest-bound filesystem transaction, and writes payload/terminal receipts
|
||||
below `/backup/status`.
|
||||
- The deployed restore-drill script gains shared-lock behavior. The apply does
|
||||
not execute that script.
|
||||
- Apply does not run a backup, invoke a Gitea container lifecycle, sync Google
|
||||
Drive, prune snapshots, restart a VM or service, or change a database,
|
||||
network, firewall, credential, or secret.
|
||||
|
||||
## Reproducible Validation
|
||||
|
||||
Focused replay and contract suite:
|
||||
|
||||
```sh
|
||||
pytest -q \
|
||||
scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \
|
||||
scripts/backup/tests/test_verify_offsite_full_sync_runtime.py \
|
||||
scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py \
|
||||
scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py \
|
||||
scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py \
|
||||
scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py \
|
||||
scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py
|
||||
```
|
||||
|
||||
Exact signal and post-commit cleanup replays:
|
||||
|
||||
```sh
|
||||
pytest -vv \
|
||||
scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \
|
||||
-k 'commit_window_signal or terminal_publication or terminal_receipt_fault or postcommit_cleanup'
|
||||
```
|
||||
|
||||
Static, syntax, and repository checks:
|
||||
|
||||
```sh
|
||||
ruff check scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py
|
||||
bash -n scripts/backup/host110-backup-runtime-executor.sh
|
||||
git diff --check
|
||||
PATH=/Users/ogt/.pyenv/shims:$PATH bash scripts/ops/ansible-validate.sh
|
||||
```
|
||||
|
||||
`ansible-validate.sh` parses
|
||||
`ops/reboot-recovery/full-stack-backup-baseline.yml` as YAML metadata. It runs
|
||||
`ansible-playbook --syntax-check` only for the actual playbooks listed by the
|
||||
script; the metadata mapping is not represented as a playbook.
|
||||
|
||||
Windows99 no-file-write PowerShell parser check:
|
||||
|
||||
```sh
|
||||
PARSER_COMMAND='$source=[Console]::In.ReadToEnd(); $tokens=$null; $errors=$null; [System.Management.Automation.Language.Parser]::ParseInput($source,[ref]$tokens,[ref]$errors) | Out-Null; [pscustomobject]@{ok=($errors.Count -eq 0); errorCount=$errors.Count; parser="WindowsPowerShell"} | ConvertTo-Json -Compress; if ($errors.Count -ne 0) { exit 1 }'
|
||||
ENCODED=$(printf '%s' "$PARSER_COMMAND" | iconv -f UTF-8 -t UTF-16LE | base64 | tr -d '\n')
|
||||
ssh -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=5 \
|
||||
-o ConnectionAttempts=1 Administrator@192.168.0.99 \
|
||||
"powershell.exe -NoProfile -NonInteractive -EncodedCommand $ENCODED" \
|
||||
< agent99-host110-backup-runtime-broker.ps1
|
||||
```
|
||||
|
||||
Windows99 actual-function missing-field/type contract replay:
|
||||
|
||||
```sh
|
||||
WRAPPER='$script=[Console]::In.ReadToEnd(); & ([ScriptBlock]::Create($script))'
|
||||
WRAPPER_ENCODED=$(printf '%s' "$WRAPPER" | iconv -f UTF-8 -t UTF-16LE | base64 | tr -d '\n')
|
||||
BROKER_GZIP_BASE64=$(gzip -c agent99-host110-backup-runtime-broker.ps1 | base64 | tr -d '\n')
|
||||
awk -v payload="$BROKER_GZIP_BASE64" \
|
||||
'BEGIN { print "$BrokerGzipBase64 = \"" payload "\"" } { print }' \
|
||||
scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1 |
|
||||
ssh -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=5 \
|
||||
-o ConnectionAttempts=1 Administrator@192.168.0.99 \
|
||||
"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand $WRAPPER_ENCODED"
|
||||
```
|
||||
|
||||
The replay loads the candidate broker's actual function AST, accepts the valid
|
||||
success and three committed non-success terminals, then removes every required
|
||||
field and injects wrong JSON types. Expected readback is 34 missing-field and 6
|
||||
type rejections. It performs no remote file write.
|
||||
|
||||
## Exact Diff Serialization
|
||||
|
||||
Substitute the V10 Gitea live-ref SHA for `<candidate_git_sha>` and hash the
|
||||
exact stdout bytes:
|
||||
|
||||
```sh
|
||||
git diff --binary --full-index \
|
||||
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
|
||||
shasum -a 256
|
||||
```
|
||||
|
||||
Central review must independently bind the live feature ref and candidate Git
|
||||
SHA, this tracked artifact path and bytes hash, the serialized diff hash/path
|
||||
count/line counts, the 19/18/21 count contracts, and the exact restore-drill and
|
||||
`backup-gitea.sh` diffs.
|
||||
|
||||
## Rollback
|
||||
|
||||
- Source rollback reverts the exact V10 diff to
|
||||
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
|
||||
- Agent99 rollback restores the prior runtime files and manifest and verifies
|
||||
prior hashes independently.
|
||||
- Before payload commit, Host110 rollback restores all prior payload content,
|
||||
SHA-256, uid, gid, and mode under the exclusive lock and proves no run-owned
|
||||
destination temporary remains.
|
||||
- After payload commit, the executor never rolls the payload back merely due to
|
||||
a signal or cleanup error. It preserves exact cleanup evidence and reports a
|
||||
non-success terminal. Any later cleanup-only remediation is a separate
|
||||
controlled action scoped to the run-specific stage/prestate paths and may not
|
||||
rewrite the verified payload.
|
||||
- Source rollback restores the restore-drill script's prior unlocked behavior.
|
||||
It does not invoke the drill or any container lifecycle.
|
||||
- Any canonical, metadata, receipt, signal, cleanup, or residue mismatch fails
|
||||
closed and cannot produce broker `verified`.
|
||||
@@ -1,65 +0,0 @@
|
||||
# AWOOOI Agent99 Host110 Backup Runtime V7
|
||||
|
||||
Status: proposed; owner approval required before any production apply.
|
||||
|
||||
This artifact supersedes V2 through V6. All V5 proposal, artifact, and diff
|
||||
hashes are void. This file is review evidence only; it is not active policy or
|
||||
production execution authority.
|
||||
|
||||
## Scope
|
||||
|
||||
- Agent99 runtime bundle: 19 files. Every producer and consumer is guarded by
|
||||
one exact parity test.
|
||||
- Host110 backup payload: 18 files, comprising 17 backup scripts and
|
||||
`backup-health-textfile-exporter.py`.
|
||||
- Host110 remote stage: 21 files, comprising the 18 payload files, the fixed
|
||||
executor, the independent verifier, and the source manifest.
|
||||
- Fixed target: `wooo@192.168.0.110`, dispatched by Windows99 Agent99 from an
|
||||
exact Gitea live revision and digest manifest.
|
||||
- Gitea offline backup and container lifecycle changes are excluded from this
|
||||
proposal. `backup-gitea.sh` has no candidate diff. The only Gitea-related
|
||||
diff is a shared deployment lock in the read-only restore-drill entrypoint;
|
||||
it adds no Docker command and is not executed by the deployment transaction.
|
||||
|
||||
## Exact Diff Serialization
|
||||
|
||||
Substitute the V7 Gitea live-ref SHA for `<candidate_git_sha>` and hash the
|
||||
exact stdout bytes from this command:
|
||||
|
||||
```sh
|
||||
git diff --binary --full-index \
|
||||
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
|
||||
shasum -a 256
|
||||
```
|
||||
|
||||
The review message independently binds the Gitea live ref and Git SHA, this
|
||||
tracked artifact path and bytes hash, and the serialized diff hash.
|
||||
|
||||
## Apply Effects
|
||||
|
||||
- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded
|
||||
manifest and receipt state.
|
||||
- Host110 Apply writes/replaces the 18 payload paths through one exclusive,
|
||||
digest-bound filesystem transaction.
|
||||
- Apply does not run a backup, invoke a Gitea container lifecycle, sync Google
|
||||
Drive, prune or delete snapshots, restart a VM or service, or change a DB,
|
||||
network, or firewall.
|
||||
|
||||
## Validation
|
||||
|
||||
- Post-split focused runtime, broker, backup, and parity suites must pass.
|
||||
- Ansible and shell/Python static validation must pass.
|
||||
- Changed PowerShell must parse on Windows99 with zero errors.
|
||||
- Central review must independently reproduce the live ref, artifact hash,
|
||||
diff hash, and 19/18/21 counts.
|
||||
|
||||
## Rollback
|
||||
|
||||
- Source rollback reverts the exact V7 diff to
|
||||
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
|
||||
- Agent99 rollback restores the prior runtime files and manifest and verifies
|
||||
the prior hashes independently.
|
||||
- Host110 rollback restores every prior payload's content, SHA-256, uid, gid,
|
||||
and mode under the exclusive lock.
|
||||
- Any mismatch terminates as `rollback_unverified` and cannot be reported as a
|
||||
successful rollback.
|
||||
@@ -1,78 +0,0 @@
|
||||
# AWOOOI Agent99 Host110 Backup Runtime V8
|
||||
|
||||
Status: proposed; owner approval required before any production apply.
|
||||
|
||||
This artifact supersedes V2 through V7. The V7 candidate, revision, artifact,
|
||||
and diff hashes are void for approval. All V5 hashes remain void. Prior refs
|
||||
remain immutable audit evidence only; none is production execution authority.
|
||||
|
||||
## Scope
|
||||
|
||||
- Agent99 runtime bundle: 19 files, guarded by one producer/consumer parity
|
||||
test.
|
||||
- Host110 backup payload: 18 files, comprising 17 backup scripts and
|
||||
`backup-health-textfile-exporter.py`.
|
||||
- Host110 remote stage: 21 unique basenames, comprising the 18 payload files,
|
||||
the fixed executor, the independent verifier, and `manifest.json`.
|
||||
- Fixed target: `wooo@192.168.0.110`, dispatched by Windows99 Agent99 from an
|
||||
exact Gitea main revision and digest manifest.
|
||||
- Gitea offline backup and container lifecycle changes remain excluded.
|
||||
`backup-gitea.sh` has no candidate diff.
|
||||
|
||||
## V7 Defect Closed
|
||||
|
||||
V7 created and uploaded
|
||||
`agent99-host110-backup-runtime-manifest.json`, while the fixed Host110
|
||||
executor consumed `$SOURCE_STAGE/manifest.json`. V8 uses `manifest.json` at
|
||||
the broker source stage, so SCP preserves the exact basename consumed by the
|
||||
executor.
|
||||
|
||||
The integration test derives all broker SCP basenames from the actual
|
||||
PowerShell file list and manifest path, requires 21 unique stage files, and
|
||||
matches the manifest and verifier basenames to the executor's exact
|
||||
`$SOURCE_STAGE` paths. A hand-built temporary manifest cannot satisfy this
|
||||
test by itself.
|
||||
|
||||
## Exact Diff Serialization
|
||||
|
||||
Substitute the V8 Gitea live-ref SHA for `<candidate_git_sha>` and hash the
|
||||
exact stdout bytes from this command:
|
||||
|
||||
```sh
|
||||
git diff --binary --full-index \
|
||||
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
|
||||
shasum -a 256
|
||||
```
|
||||
|
||||
The review message independently binds the Gitea live ref and Git SHA, this
|
||||
tracked artifact path and bytes hash, and the serialized diff hash.
|
||||
|
||||
## Apply Effects
|
||||
|
||||
- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded
|
||||
manifest and receipt state.
|
||||
- Host110 Apply writes/replaces the 18 payload paths through one exclusive,
|
||||
digest-bound filesystem transaction.
|
||||
- Apply does not run a backup, invoke a Gitea container lifecycle, sync Google
|
||||
Drive, prune or delete snapshots, restart a VM or service, or change a DB,
|
||||
network, or firewall.
|
||||
|
||||
## Validation
|
||||
|
||||
- The exact broker/SCP basename to executor path integration test must pass.
|
||||
- Focused runtime, broker, backup, parity, and Ansible suites must pass.
|
||||
- Ansible and shell/Python static validation must pass.
|
||||
- Changed PowerShell must parse on Windows99 with zero errors.
|
||||
- Central review must independently reproduce the live ref, artifact hash,
|
||||
diff hash, 19/18/21 counts, and zero `backup-gitea.sh` diff.
|
||||
|
||||
## Rollback
|
||||
|
||||
- Source rollback reverts the exact V8 diff to
|
||||
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
|
||||
- Agent99 rollback restores the prior runtime files and manifest and verifies
|
||||
the prior hashes independently.
|
||||
- Host110 rollback restores every prior payload's content, SHA-256, uid, gid,
|
||||
and mode under the exclusive lock.
|
||||
- Any mismatch terminates as `rollback_unverified` and cannot be reported as a
|
||||
successful rollback.
|
||||
@@ -1,133 +0,0 @@
|
||||
# AWOOOI Agent99 Host110 Backup Runtime V9
|
||||
|
||||
Status: proposed; owner approval required before any production apply.
|
||||
|
||||
This artifact supersedes V2 through V8. The following V8 identities are void
|
||||
for approval and remain audit evidence only:
|
||||
|
||||
- candidate/revision: `0b9c0284006c8d892c4617c2bfc16869bb6637f0`
|
||||
- artifact SHA-256: `ea9537613c462a07f3155263da5ad0f444bda0ba6773b49208de31b0acdef228`
|
||||
- exact diff SHA-256: `0867d9221ecfa7bcbe6b073a6a02da401bbed5087b9f2bd7fa494f2953b2ef2e`
|
||||
- every other proposal/content hash derived from V8 candidate bytes
|
||||
|
||||
No V8 hash, ref, review, or receipt grants production execution authority.
|
||||
|
||||
## Scope
|
||||
|
||||
- Agent99 runtime bundle: 19 files, guarded by one producer/consumer parity
|
||||
test.
|
||||
- Host110 backup payload: 18 ordered files, comprising 17 backup scripts and
|
||||
`backup-health-textfile-exporter.py`.
|
||||
- Host110 remote stage: 21 unique basenames, comprising the ordered 18-file
|
||||
payload, fixed executor, independent verifier, and `manifest.json`.
|
||||
- Fixed target: `wooo@192.168.0.110`, dispatched by Windows99 Agent99 from an
|
||||
exact Gitea revision and digest manifest.
|
||||
- Gitea offline backup and container lifecycle changes remain excluded;
|
||||
`backup-gitea.sh` has no candidate diff from the production base.
|
||||
|
||||
## V8 Defects Closed
|
||||
|
||||
### Durable Receipt Transaction
|
||||
|
||||
The executor no longer captures `write_receipt` through command substitution
|
||||
and cannot mask an internal failure with a later `printf`. Receipt publication
|
||||
uses an exclusive temporary file, exact full-write check, file `fsync`,
|
||||
temporary-byte SHA-256 readback, no-replace hard link, directory `fsync`, exact
|
||||
final-byte/document/SHA-256 readback, temporary unlink, and a second directory
|
||||
`fsync`. Any write, link, fsync, cleanup, or readback failure is nonzero and
|
||||
removes a final path created by that failed attempt when possible.
|
||||
|
||||
`APPLY_COMPLETE=1` is set only after the durable verified receipt passes this
|
||||
transaction. Stage and rollback prestate are never cleared before that point.
|
||||
If verified receipt publication fails, the EXIT path performs rollback first,
|
||||
writes a distinct failure receipt, and only removes rollback prestate after
|
||||
both rollback verification and durable failure-receipt publication succeed.
|
||||
Payload files, destination directories, rollback prestate files, and the
|
||||
rollback directory are explicitly fsynced before their corresponding durable
|
||||
claim can be published.
|
||||
|
||||
### Run-Owned Temporary Files
|
||||
|
||||
Every apply and rollback temporary destination is tracked for the exact run.
|
||||
Rollback removes and reads back all tracked paths, including a failure between
|
||||
`install` and `mv`. Canonical content/metadata verification plus zero hidden
|
||||
residue are both required before rollback can be called verified.
|
||||
|
||||
### Rollback Truth
|
||||
|
||||
Receipt schema `agent99_host110_backup_runtime_deploy_receipt_v2` records
|
||||
`rollbackAttempted`, `rollbackPerformed`, `rollbackVerified`, and
|
||||
`zeroResidueVerified` independently. A partial, unknown, or residue-bearing
|
||||
rollback terminates as `rollback_unverified`; only a fully restored and
|
||||
zero-residue transaction may terminate as `failed_rolled_back`.
|
||||
|
||||
### Producer/Consumer Parity
|
||||
|
||||
The integration test parses both actual sources and requires the broker's 18
|
||||
payload basenames to equal the executor payload in count, exact set, and exact
|
||||
order. It separately requires executor, verifier, and `manifest.json`, yielding
|
||||
21 unique remote-stage basenames.
|
||||
|
||||
## Fault-Injection Contract
|
||||
|
||||
The bounded local harness executes the real executor control flow against
|
||||
isolated temporary destinations and covers:
|
||||
|
||||
- successful apply, durable receipt, verifier, and post-receipt prestate cleanup;
|
||||
- receipt write, link, fsync, and final-readback failures;
|
||||
- apply failure after hidden-temp installation but before canonical `mv`;
|
||||
- forced temp-cleanup failure and `rollback_unverified` evidence preservation.
|
||||
|
||||
Every receipt fault must restore exact prestate, emit no success JSON, and
|
||||
produce a durable `failed_rolled_back` receipt. A cleanup fault must preserve
|
||||
`prestate.tsv`, expose rollback attempted/performed separately, and must not
|
||||
claim rollback verification.
|
||||
|
||||
Fault hooks require an explicit test enable flag and a transformed
|
||||
non-production destination. The canonical `/backup/scripts` executor can never
|
||||
enable them, even if a fault environment variable is present.
|
||||
|
||||
## Exact Diff Serialization
|
||||
|
||||
Substitute the V9 Gitea live-ref SHA for `<candidate_git_sha>` and hash the
|
||||
exact stdout bytes from this command:
|
||||
|
||||
```sh
|
||||
git diff --binary --full-index \
|
||||
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
|
||||
shasum -a 256
|
||||
```
|
||||
|
||||
The review message independently binds the Gitea live ref and Git SHA, this
|
||||
tracked artifact path and bytes hash, and the serialized diff hash.
|
||||
|
||||
## Apply Effects
|
||||
|
||||
- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded
|
||||
manifest and receipt state.
|
||||
- Host110 Apply writes/replaces the 18 payload paths through one exclusive,
|
||||
digest-bound filesystem transaction.
|
||||
- Apply does not run a backup, invoke a Gitea container lifecycle, sync Google
|
||||
Drive, prune snapshots, restart a VM or service, or change a database,
|
||||
network, or firewall.
|
||||
|
||||
## Validation
|
||||
|
||||
- The exact broker payload/order and 21-file stage parity test must pass.
|
||||
- Success and all six fault-injection paths must execute and pass.
|
||||
- Focused runtime, broker, backup, parity, Ansible, shell, and Python checks
|
||||
must pass.
|
||||
- Changed PowerShell must parse on Windows99 with zero errors when applicable.
|
||||
- Central review must independently reproduce the live ref, artifact hash,
|
||||
exact diff hash, 19/18/21 counts, and zero `backup-gitea.sh` diff.
|
||||
|
||||
## Rollback
|
||||
|
||||
- Source rollback reverts the exact V9 diff to
|
||||
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
|
||||
- Agent99 rollback restores the prior runtime files and manifest and verifies
|
||||
prior hashes independently.
|
||||
- Host110 rollback restores all prior payload content, SHA-256, uid, gid, and
|
||||
mode under the exclusive lock and proves zero run-owned temporary residue.
|
||||
- Any canonical, metadata, receipt, or residue mismatch terminates as
|
||||
`rollback_unverified`; rollback prestate remains available for diagnosis.
|
||||
@@ -304,7 +304,7 @@ release_gate:
|
||||
strict_dr_exit_conditions:
|
||||
- "Google Drive/rclone provider configured on 110 host-local rclone.conf; /backup/scripts/offsite.env keeps only non-secret remote/path with mode 0600"
|
||||
- "credential escrow markers fresh for restic_repository_password, offsite_provider_credentials, break_glass_admin_credentials, dns_registrar_recovery, oauth_ai_provider_recovery"
|
||||
- "full offsite marker /backup/offsite/rclone-last-success fresh after full 14 repo sync"
|
||||
- "full offsite marker /backup/offsite/rclone-last-success fresh after full 13 repo sync"
|
||||
- "full-stack-recovery-scorecard.sh --require-dr exits 0"
|
||||
- "recovery-scorecard-contract-check.py --expect-dr-ready exits 0 against 110 Prometheus"
|
||||
- "dr-offsite-operator-checklist.sh --require-dr exits 0 after scorecard, Prometheus recording rule, and backup alert visibility contract agree"
|
||||
|
||||
@@ -9,16 +9,6 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then
|
||||
command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; }
|
||||
[ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; }
|
||||
(umask 077; : >> /tmp/agent99-host110-backup-runtime.lock)
|
||||
[ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; }
|
||||
exec 197>>/tmp/agent99-host110-backup-runtime.lock
|
||||
flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; }
|
||||
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
|
||||
fi
|
||||
|
||||
BACKUP_BASE="${BACKUP_BASE:-/backup}"
|
||||
OFFSITE_ENV_FILE="${BACKUP_OFFSITE_ENV_FILE:-${BACKUP_BASE}/scripts/offsite.env}"
|
||||
OFFSITE_DIR="${BACKUP_OFFSITE_STATUS_DIR:-${BACKUP_BASE}/offsite}"
|
||||
@@ -199,7 +189,6 @@ check_offsite_env() {
|
||||
|
||||
check_configured() {
|
||||
load_offsite_env
|
||||
bind_rclone_config_to_runtime_user
|
||||
if command -v rclone >/dev/null 2>&1; then
|
||||
ok "rclone command is available"
|
||||
else
|
||||
|
||||
@@ -1,38 +1,5 @@
|
||||
#!/bin/bash
|
||||
#\!/bin/bash
|
||||
# =============================================================================
|
||||
|
||||
# Production backup entrypoints share this stable gate. Agent99 takes the
|
||||
# exclusive side while promoting a complete, digest-bound runtime bundle.
|
||||
acquire_backup_runtime_shared_lock() {
|
||||
local caller_path="${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}"
|
||||
local lock_path="/tmp/agent99-host110-backup-runtime.lock"
|
||||
case "${caller_path}" in
|
||||
/backup/scripts/*) ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
[ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ] || return 0
|
||||
command -v flock >/dev/null 2>&1 || {
|
||||
echo "backup runtime gate unavailable: flock missing" >&2
|
||||
return 69
|
||||
}
|
||||
[ ! -L "${lock_path}" ] || {
|
||||
echo "backup runtime gate unsafe: lock is a symlink" >&2
|
||||
return 69
|
||||
}
|
||||
(umask 077; : >> "${lock_path}") || return 69
|
||||
[ -f "${lock_path}" ] && [ -O "${lock_path}" ] || return 69
|
||||
exec 197>>"${lock_path}"
|
||||
flock -s -w 60 197 || {
|
||||
echo "backup runtime deployment is active; retry later" >&2
|
||||
return 75
|
||||
}
|
||||
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
|
||||
}
|
||||
|
||||
acquire_backup_runtime_shared_lock || {
|
||||
runtime_lock_status=$?
|
||||
return "${runtime_lock_status}" 2>/dev/null || exit "${runtime_lock_status}"
|
||||
}
|
||||
# WOOO AIOps - 備份共用函式庫
|
||||
# 版本: 1.0.0
|
||||
# 建立日期: 2026-03-12
|
||||
@@ -63,22 +30,6 @@ export BACKUP_DOCKER_CPUS="${BACKUP_DOCKER_CPUS:-1.0}"
|
||||
export BACKUP_DOCKER_MEMORY="${BACKUP_DOCKER_MEMORY:-1g}"
|
||||
export BACKUP_DOCKER_MEMORY_SWAP="${BACKUP_DOCKER_MEMORY_SWAP:-1g}"
|
||||
|
||||
# Cron and service managers do not always preserve HOME. Bind rclone to the
|
||||
# runtime account's config path when the operator did not provide an explicit
|
||||
# RCLONE_CONFIG, without reading or printing the credential-bearing file.
|
||||
bind_rclone_config_to_runtime_user() {
|
||||
[ -z "${RCLONE_CONFIG:-}" ] || return 0
|
||||
command -v getent >/dev/null 2>&1 || return 0
|
||||
|
||||
local runtime_home
|
||||
local candidate
|
||||
runtime_home="$(getent passwd "$(id -u)" 2>/dev/null | awk -F: 'NR == 1 {print $6}')"
|
||||
candidate="${runtime_home}/.config/rclone/rclone.conf"
|
||||
if [ -n "${runtime_home}" ] && [ -r "${candidate}" ]; then
|
||||
export RCLONE_CONFIG="${candidate}"
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 日誌函式
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
@@ -5,16 +5,6 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then
|
||||
command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; }
|
||||
[ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; }
|
||||
(umask 077; : >> /tmp/agent99-host110-backup-runtime.lock)
|
||||
[ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; }
|
||||
exec 197>>/tmp/agent99-host110-backup-runtime.lock
|
||||
flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; }
|
||||
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
|
||||
fi
|
||||
|
||||
DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}"
|
||||
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
|
||||
TEXTFILE_PATH="${AIOPS_GITEA_FULL_BACKUP_RESTORE_DRILL_TEXTFILE:-/home/wooo/node_exporter_textfiles/gitea_full_backup_restore_drill.prom}"
|
||||
|
||||
@@ -1,978 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fixed Host110 executor for Windows99 Agent99 backup-runtime deployment.
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
readonly EXPECTED_HOST_IP="192.168.0.110"
|
||||
readonly EXPECTED_USER="wooo"
|
||||
readonly DEST_ROOT="/backup/scripts"
|
||||
readonly EXPORTER_ROOT="/home/wooo/scripts"
|
||||
readonly EXPORTER_FILE="backup-health-textfile-exporter.py"
|
||||
readonly STATUS_ROOT="/backup/status"
|
||||
readonly STAGE_ROOT="/backup/.agent99-backup-runtime-stage"
|
||||
readonly ROLLBACK_ROOT="/backup/.agent99-backup-runtime-rollback"
|
||||
readonly LOCK_PATH="/tmp/agent99-host110-backup-runtime.lock"
|
||||
|
||||
readonly -a RUNTIME_FILES=(
|
||||
common.sh
|
||||
backup-all.sh
|
||||
backup-host188-products.sh
|
||||
verify-host188-products-backup.sh
|
||||
verify-host188-products-archive.py
|
||||
backup-gitea.sh
|
||||
gitea-full-backup-restore-drill.sh
|
||||
backup-configs.sh
|
||||
backup-awoooi.sh
|
||||
backup-awoooi-frequent.sh
|
||||
backup-clawbot.sh
|
||||
backup-sentry.sh
|
||||
check-backup-integrity.sh
|
||||
sync-offsite-backups.sh
|
||||
backup-offsite-readiness-gate.sh
|
||||
verify-offsite-full-sync.sh
|
||||
enforce-latest-only-retention.sh
|
||||
)
|
||||
readonly -a PAYLOAD_FILES=("${RUNTIME_FILES[@]}" "$EXPORTER_FILE")
|
||||
|
||||
MODE=""
|
||||
SOURCE_REVISION=""
|
||||
RUN_ID=""
|
||||
SOURCE_STAGE=""
|
||||
SOURCE_HEAD=""
|
||||
EXECUTOR_DIGEST=""
|
||||
VERIFIER_DIGEST=""
|
||||
STAGE_DIR=""
|
||||
ROLLBACK_DIR=""
|
||||
RECEIPT_PATH=""
|
||||
TERMINAL_RECEIPT_PATH=""
|
||||
APPLY_STARTED=0
|
||||
APPLY_COMPLETE=0
|
||||
COMMIT_CRITICAL=0
|
||||
DEFERRED_SIGNAL=""
|
||||
DEFERRED_SIGNAL_EXIT=0
|
||||
ROLLBACK_ATTEMPTED=0
|
||||
ROLLBACK_PERFORMED=0
|
||||
ROLLBACK_VERIFIED=0
|
||||
RUN_TEMP_RESIDUE_VERIFIED=0
|
||||
FAULT_INJECTION_ENABLED=0
|
||||
declare -A FILE_EXISTED=()
|
||||
declare -A PREVIOUS_DIGEST=()
|
||||
declare -A PREVIOUS_METADATA=()
|
||||
declare -A EXPECTED_DIGEST=()
|
||||
declare -A RUN_OWNED_TEMP_PATHS=()
|
||||
|
||||
usage() {
|
||||
printf '%s\n' \
|
||||
"Usage: $0 --mode check|apply|verify --source-revision SHA --run-id ID --source-stage PATH" \
|
||||
"The executor is fixed to host 192.168.0.110, an Agent99 run-bound source stage," \
|
||||
"and the 18-file transaction: 17 /backup/scripts files plus the backup health exporter."
|
||||
}
|
||||
|
||||
fail() {
|
||||
printf 'HOST110_BACKUP_RUNTIME_OK=0\nERROR=%s\n' "$1" >&2
|
||||
exit 64
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--mode)
|
||||
shift
|
||||
MODE="${1:-}"
|
||||
;;
|
||||
--source-revision)
|
||||
shift
|
||||
SOURCE_REVISION="${1:-}"
|
||||
;;
|
||||
--run-id)
|
||||
shift
|
||||
RUN_ID="${1:-}"
|
||||
;;
|
||||
--source-stage)
|
||||
shift
|
||||
SOURCE_STAGE="${1:-}"
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*) fail "unknown_argument" ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
case "$MODE" in check|apply|verify) ;; *) fail "invalid_mode" ;; esac
|
||||
[[ "$SOURCE_REVISION" =~ ^[0-9a-f]{40}$ ]] || fail "invalid_source_revision"
|
||||
[[ "$RUN_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$ ]] || fail "invalid_run_id"
|
||||
expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"
|
||||
[ "$SOURCE_STAGE" = "$expected_source_stage" ] || fail "invalid_source_stage"
|
||||
RECEIPT_PATH="$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json"
|
||||
TERMINAL_RECEIPT_PATH="$STATUS_ROOT/agent99-backup-runtime-terminal-${RUN_ID}.json"
|
||||
if [ "${HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS:-}" = "1" ] \
|
||||
&& [ "$DEST_ROOT" != "/backup/scripts" ]; then
|
||||
FAULT_INJECTION_ENABLED=1
|
||||
fi
|
||||
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then
|
||||
case "${HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL:-}" in ""|INT|HUP|TERM) ;; *) fail "invalid_test_commit_signal" ;; esac
|
||||
case "${HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL:-}" in ""|INT|HUP|TERM) ;; *) fail "invalid_test_terminal_signal" ;; esac
|
||||
case "${HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT:-}" in ""|write|link|fsync|readback) ;; *) fail "invalid_test_terminal_receipt_fault" ;; esac
|
||||
case "${HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT:-}" in ""|stage|rollback_prestate) ;; *) fail "invalid_test_cleanup_fault" ;; esac
|
||||
fi
|
||||
|
||||
for command_name in awk bash cp dirname flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do
|
||||
command -v "$command_name" >/dev/null 2>&1 || fail "required_command_missing_${command_name}"
|
||||
done
|
||||
|
||||
current_user="$(id -un)"
|
||||
[ "$current_user" = "$EXPECTED_USER" ] || fail "unexpected_executor_user"
|
||||
hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx "$EXPECTED_HOST_IP" || fail "unexpected_executor_host"
|
||||
[ -d "$SOURCE_STAGE" ] && [ ! -L "$SOURCE_STAGE" ] || fail "canonical_source_stage_unavailable"
|
||||
[ -d "$DEST_ROOT" ] && [ ! -L "$DEST_ROOT" ] || fail "backup_runtime_destination_unavailable"
|
||||
[ -d "$EXPORTER_ROOT" ] && [ ! -L "$EXPORTER_ROOT" ] || fail "backup_exporter_destination_unavailable"
|
||||
|
||||
manifest_rows="$(python3 - "$SOURCE_STAGE/manifest.json" "$SOURCE_REVISION" "$RUN_ID" "${PAYLOAD_FILES[@]}" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
source_revision = sys.argv[2]
|
||||
run_id = sys.argv[3]
|
||||
expected_names = sys.argv[4:]
|
||||
try:
|
||||
document = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
raise SystemExit(65)
|
||||
rows = document.get("files")
|
||||
if (
|
||||
document.get("schemaVersion") != "agent99_host110_backup_runtime_source_manifest_v1"
|
||||
or document.get("sourceRevision") != source_revision
|
||||
or document.get("runId") != run_id
|
||||
or not re.fullmatch(r"[0-9a-f]{40}", str(document.get("sourceHead", "")))
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("executorSha256", "")))
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("verifierSha256", "")))
|
||||
or not isinstance(rows, list)
|
||||
or len(rows) != len(expected_names)
|
||||
):
|
||||
raise SystemExit(66)
|
||||
by_name = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
raise SystemExit(66)
|
||||
name = str(row.get("name", ""))
|
||||
digest = str(row.get("sha256", ""))
|
||||
if name in by_name or not re.fullmatch(r"[A-Za-z0-9_.-]+", name) or not re.fullmatch(r"[0-9a-f]{64}", digest):
|
||||
raise SystemExit(66)
|
||||
by_name[name] = digest
|
||||
if set(by_name) != set(expected_names):
|
||||
raise SystemExit(66)
|
||||
print(f"SOURCE_HEAD {document['sourceHead']}")
|
||||
print(f"EXECUTOR_SHA256 {document['executorSha256']}")
|
||||
print(f"VERIFIER_SHA256 {document['verifierSha256']}")
|
||||
for name in expected_names:
|
||||
print(f"{name} {by_name[name]}")
|
||||
PY
|
||||
)" || fail "source_manifest_invalid"
|
||||
|
||||
while read -r manifest_name manifest_digest; do
|
||||
case "$manifest_name" in
|
||||
SOURCE_HEAD) SOURCE_HEAD="$manifest_digest" ;;
|
||||
EXECUTOR_SHA256) EXECUTOR_DIGEST="$manifest_digest" ;;
|
||||
VERIFIER_SHA256) VERIFIER_DIGEST="$manifest_digest" ;;
|
||||
*) EXPECTED_DIGEST[$manifest_name]="$manifest_digest" ;;
|
||||
esac
|
||||
done <<< "$manifest_rows"
|
||||
[[ "$SOURCE_HEAD" =~ ^[0-9a-f]{40}$ ]] || fail "source_head_invalid"
|
||||
|
||||
working_digest() {
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
}
|
||||
|
||||
[ "$(working_digest "$0")" = "$EXECUTOR_DIGEST" ] || fail "executor_identity_failed"
|
||||
[ -f "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \
|
||||
&& [ ! -L "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \
|
||||
&& [ "$(working_digest "$SOURCE_STAGE/verify-host110-backup-runtime.py")" = "$VERIFIER_DIGEST" ] \
|
||||
|| fail "verifier_identity_failed"
|
||||
|
||||
validate_file() {
|
||||
local validation_path="$1"
|
||||
case "$validation_path" in
|
||||
*.sh) bash -n "$validation_path" ;;
|
||||
*.py) python3 -c 'import ast, pathlib, sys; ast.parse(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))' "$validation_path" ;;
|
||||
*) return 64 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
source_digest() {
|
||||
local name="$1"
|
||||
printf '%s\n' "${EXPECTED_DIGEST[$name]:-}"
|
||||
}
|
||||
|
||||
validate_source() {
|
||||
local name source_path expected_digest actual_digest resolved
|
||||
for name in "${PAYLOAD_FILES[@]}"; do
|
||||
source_path="$SOURCE_STAGE/$name"
|
||||
[ -f "$source_path" ] && [ ! -L "$source_path" ] || return 65
|
||||
resolved="$(readlink -f "$source_path")"
|
||||
[[ "$resolved" == "$SOURCE_STAGE/"* ]] || return 65
|
||||
expected_digest="$(source_digest "$name")"
|
||||
actual_digest="$(working_digest "$source_path")"
|
||||
[ "$actual_digest" = "$expected_digest" ] || return 66
|
||||
validate_file "$source_path" || return 67
|
||||
done
|
||||
}
|
||||
|
||||
destination_path() {
|
||||
local name="$1"
|
||||
if [ "$name" = "$EXPORTER_FILE" ]; then
|
||||
printf '%s/%s\n' "$EXPORTER_ROOT" "$name"
|
||||
else
|
||||
printf '%s/%s\n' "$DEST_ROOT" "$name"
|
||||
fi
|
||||
}
|
||||
|
||||
verify_destination() {
|
||||
local name dest_path expected_digest actual_digest
|
||||
for name in "${PAYLOAD_FILES[@]}"; do
|
||||
dest_path="$(destination_path "$name")"
|
||||
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] || return 68
|
||||
expected_digest="$(source_digest "$name")"
|
||||
actual_digest="$(working_digest "$dest_path")"
|
||||
[ "$actual_digest" = "$expected_digest" ] || return 69
|
||||
[ "$(stat -c '%U:%G:%a' "$dest_path")" = "wooo:wooo:755" ] || return 70
|
||||
done
|
||||
}
|
||||
|
||||
fsync_paths_and_parents() {
|
||||
python3 - "$@" <<'PY'
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
parents = set()
|
||||
for raw_path in sys.argv[1:]:
|
||||
path = Path(raw_path)
|
||||
parents.add(path.parent)
|
||||
if not os.path.lexists(path):
|
||||
continue
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise SystemExit(76)
|
||||
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
|
||||
descriptor = os.open(path, flags)
|
||||
try:
|
||||
if not stat.S_ISREG(os.fstat(descriptor).st_mode):
|
||||
raise SystemExit(76)
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
for parent in sorted(parents, key=str):
|
||||
if parent.is_symlink() or not parent.is_dir():
|
||||
raise SystemExit(76)
|
||||
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
||||
descriptor = os.open(parent, flags)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
PY
|
||||
}
|
||||
|
||||
fsync_destination_state() {
|
||||
local name dest_path
|
||||
local -a fsync_targets=()
|
||||
for name in "${PAYLOAD_FILES[@]}"; do
|
||||
dest_path="$(destination_path "$name")"
|
||||
fsync_targets+=("$dest_path")
|
||||
done
|
||||
fsync_paths_and_parents "${fsync_targets[@]}"
|
||||
}
|
||||
|
||||
fsync_rollback_prestate() {
|
||||
local name
|
||||
local -a fsync_targets=("$ROLLBACK_DIR/prestate.tsv")
|
||||
for name in "${PAYLOAD_FILES[@]}"; do
|
||||
if [ "${FILE_EXISTED[$name]:-}" = "1" ]; then
|
||||
fsync_targets+=("$ROLLBACK_DIR/$name")
|
||||
fi
|
||||
done
|
||||
fsync_paths_and_parents "${fsync_targets[@]}"
|
||||
}
|
||||
|
||||
write_receipt() {
|
||||
local receipt_status="$1"
|
||||
local rollback_attempted="$2"
|
||||
local rollback_performed="$3"
|
||||
local rollback_verified="$4"
|
||||
local zero_residue_verified="$5"
|
||||
install -d -m 700 "$STATUS_ROOT" || return 1
|
||||
[ -d "$STATUS_ROOT" ] && [ ! -L "$STATUS_ROOT" ] || return 1
|
||||
[ "$(stat -c '%U:%G:%a' "$STATUS_ROOT")" = "wooo:wooo:700" ] || return 1
|
||||
if ! python3 - "$RECEIPT_PATH" "$receipt_status" "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" \
|
||||
"$rollback_attempted" "$rollback_performed" "$rollback_verified" \
|
||||
"$zero_residue_verified" "$VERIFIER_DIGEST" "$FAULT_INJECTION_ENABLED" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
status = sys.argv[2]
|
||||
rollback_attempted = sys.argv[6] == "1"
|
||||
rollback_performed = sys.argv[7] == "1"
|
||||
rollback_verified = sys.argv[8] == "1"
|
||||
zero_residue_verified = sys.argv[9] == "1"
|
||||
if status not in {"payload_verified", "failed_rolled_back", "rollback_unverified"}:
|
||||
raise SystemExit(78)
|
||||
if status == "payload_verified" and (rollback_attempted or rollback_performed or rollback_verified or not zero_residue_verified):
|
||||
raise SystemExit(78)
|
||||
if status == "failed_rolled_back" and not (
|
||||
rollback_attempted and rollback_performed and rollback_verified and zero_residue_verified
|
||||
):
|
||||
raise SystemExit(78)
|
||||
if status == "rollback_unverified" and (not rollback_attempted or rollback_verified):
|
||||
raise SystemExit(78)
|
||||
document = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v3",
|
||||
"status": status,
|
||||
"sourceRevision": sys.argv[3],
|
||||
"sourceHead": sys.argv[4],
|
||||
"runId": sys.argv[5],
|
||||
"fileCount": 18,
|
||||
"backupScriptFileCount": 17,
|
||||
"backupHealthExporterIncluded": True,
|
||||
"rollbackAttempted": rollback_attempted,
|
||||
"rollbackPerformed": rollback_performed,
|
||||
"rollbackVerified": rollback_verified,
|
||||
"zeroResidueVerified": zero_residue_verified,
|
||||
"zeroResidueScope": "run_owned_destination_temps",
|
||||
"verifierSha256": sys.argv[10],
|
||||
"executorHost": "192.168.0.110",
|
||||
"productionServiceRestarted": False,
|
||||
"secretValuesRead": False,
|
||||
"writtenAt": int(time.time()),
|
||||
}
|
||||
payload = (json.dumps(document, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")
|
||||
expected_digest = hashlib.sha256(payload).hexdigest()
|
||||
temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
|
||||
fault_enabled = sys.argv[11] == "1"
|
||||
fault = (
|
||||
os.environ.get("HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT", "")
|
||||
if status == "payload_verified" and fault_enabled
|
||||
else ""
|
||||
)
|
||||
if fault not in {"", "write", "link", "fsync", "readback"}:
|
||||
raise SystemExit(78)
|
||||
created_final = False
|
||||
directory_fd = None
|
||||
try:
|
||||
if os.path.lexists(path) or os.path.lexists(temporary):
|
||||
raise FileExistsError(path)
|
||||
if fault == "write":
|
||||
raise OSError("fault_injected_receipt_write")
|
||||
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
if handle.write(payload) != len(payload):
|
||||
raise OSError("receipt_short_write")
|
||||
handle.flush()
|
||||
if fault == "fsync":
|
||||
raise OSError("fault_injected_receipt_fsync")
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, 0o600)
|
||||
temporary_readback = temporary.read_bytes()
|
||||
if temporary_readback != payload or hashlib.sha256(temporary_readback).hexdigest() != expected_digest:
|
||||
raise OSError("receipt_temporary_readback_failed")
|
||||
if fault == "link":
|
||||
raise OSError("fault_injected_receipt_link")
|
||||
os.link(temporary, path)
|
||||
created_final = True
|
||||
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
|
||||
directory_fd = os.open(path.parent, directory_flags)
|
||||
os.fsync(directory_fd)
|
||||
if fault == "readback":
|
||||
raise OSError("fault_injected_receipt_readback")
|
||||
readback = path.read_bytes()
|
||||
if readback != payload or hashlib.sha256(readback).hexdigest() != expected_digest:
|
||||
raise OSError("receipt_final_readback_failed")
|
||||
if json.loads(readback.decode("utf-8")) != document:
|
||||
raise OSError("receipt_document_readback_failed")
|
||||
temporary.unlink()
|
||||
os.fsync(directory_fd)
|
||||
except BaseException:
|
||||
try:
|
||||
if created_final:
|
||||
path.unlink(missing_ok=True)
|
||||
temporary.unlink(missing_ok=True)
|
||||
if directory_fd is None and path.parent.is_dir():
|
||||
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
|
||||
directory_fd = os.open(path.parent, directory_flags)
|
||||
if directory_fd is not None:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
raise
|
||||
finally:
|
||||
if directory_fd is not None:
|
||||
os.close(directory_fd)
|
||||
PY
|
||||
then
|
||||
return 1
|
||||
fi
|
||||
[ -f "$RECEIPT_PATH" ] && [ ! -L "$RECEIPT_PATH" ] || return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
write_terminal_receipt() {
|
||||
local terminal_status="$1"
|
||||
local stage_cleanup_verified="$2"
|
||||
local rollback_prestate_cleanup_verified="$3"
|
||||
local deferred_signal="$4"
|
||||
local terminal_exit_code="$5"
|
||||
install -d -m 700 "$STATUS_ROOT" || return 1
|
||||
[ -d "$STATUS_ROOT" ] && [ ! -L "$STATUS_ROOT" ] || return 1
|
||||
[ "$(stat -c '%U:%G:%a' "$STATUS_ROOT")" = "wooo:wooo:700" ] || return 1
|
||||
if ! python3 - "$TERMINAL_RECEIPT_PATH" "$RECEIPT_PATH" "$terminal_status" \
|
||||
"$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$stage_cleanup_verified" \
|
||||
"$rollback_prestate_cleanup_verified" "$deferred_signal" "$terminal_exit_code" \
|
||||
"$VERIFIER_DIGEST" "$FAULT_INJECTION_ENABLED" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
payload_receipt_path = Path(sys.argv[2])
|
||||
status = sys.argv[3]
|
||||
source_revision = sys.argv[4]
|
||||
source_head = sys.argv[5]
|
||||
run_id = sys.argv[6]
|
||||
stage_cleanup_verified = sys.argv[7] == "1"
|
||||
rollback_prestate_cleanup_verified = sys.argv[8] == "1"
|
||||
deferred_signal = sys.argv[9]
|
||||
terminal_exit_code = int(sys.argv[10])
|
||||
verifier_digest = sys.argv[11]
|
||||
fault_enabled = sys.argv[12] == "1"
|
||||
signal_exit_codes = {"INT": 130, "HUP": 129, "TERM": 143}
|
||||
|
||||
terminal_signal = os.environ.get("HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL", "") if fault_enabled else ""
|
||||
terminal_fault = os.environ.get("HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT", "") if fault_enabled else ""
|
||||
if terminal_signal:
|
||||
os.kill(os.getppid(), getattr(signal, f"SIG{terminal_signal}"))
|
||||
time.sleep(0.05)
|
||||
|
||||
try:
|
||||
payload_receipt = json.loads(payload_receipt_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
raise SystemExit(79)
|
||||
if not (
|
||||
payload_receipt.get("schemaVersion") == "agent99_host110_backup_runtime_deploy_receipt_v3"
|
||||
and payload_receipt.get("status") == "payload_verified"
|
||||
and payload_receipt.get("sourceRevision") == source_revision
|
||||
and payload_receipt.get("sourceHead") == source_head
|
||||
and payload_receipt.get("runId") == run_id
|
||||
and payload_receipt.get("zeroResidueVerified") is True
|
||||
):
|
||||
raise SystemExit(79)
|
||||
|
||||
if status == "verified":
|
||||
valid = (
|
||||
stage_cleanup_verified
|
||||
and rollback_prestate_cleanup_verified
|
||||
and not deferred_signal
|
||||
and terminal_exit_code == 0
|
||||
)
|
||||
elif status == "cleanup_pending":
|
||||
valid = (
|
||||
not (stage_cleanup_verified and rollback_prestate_cleanup_verified)
|
||||
and deferred_signal in {"", *signal_exit_codes}
|
||||
and terminal_exit_code != 0
|
||||
)
|
||||
elif status == "committed_signal_deferred":
|
||||
valid = (
|
||||
stage_cleanup_verified
|
||||
and rollback_prestate_cleanup_verified
|
||||
and deferred_signal in signal_exit_codes
|
||||
and terminal_exit_code == signal_exit_codes[deferred_signal]
|
||||
)
|
||||
else:
|
||||
valid = False
|
||||
if not valid:
|
||||
raise SystemExit(79)
|
||||
|
||||
document = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_terminal_receipt_v1",
|
||||
"status": status,
|
||||
"ok": status == "verified",
|
||||
"payloadCommitted": True,
|
||||
"payloadReceipt": str(payload_receipt_path),
|
||||
"sourceRevision": source_revision,
|
||||
"sourceHead": source_head,
|
||||
"runId": run_id,
|
||||
"fileCount": 18,
|
||||
"backupScriptFileCount": 17,
|
||||
"backupHealthExporterIncluded": True,
|
||||
"stageCleanupVerified": stage_cleanup_verified,
|
||||
"rollbackPrestateCleanupVerified": rollback_prestate_cleanup_verified,
|
||||
"cleanupVerified": stage_cleanup_verified and rollback_prestate_cleanup_verified,
|
||||
"independentVerifierVerified": True,
|
||||
"zeroResidueScope": "run_owned_destination_temps_and_internal_stage_prestate",
|
||||
"deferredSignal": deferred_signal or None,
|
||||
"terminalExitCode": terminal_exit_code,
|
||||
"verifierSha256": verifier_digest,
|
||||
"executorHost": "192.168.0.110",
|
||||
"productionServiceRestarted": False,
|
||||
"secretValuesRead": False,
|
||||
"writtenAt": int(time.time()),
|
||||
}
|
||||
payload = (json.dumps(document, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")
|
||||
expected_digest = hashlib.sha256(payload).hexdigest()
|
||||
temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
|
||||
created_final = False
|
||||
directory_fd = None
|
||||
try:
|
||||
if os.path.lexists(path) or os.path.lexists(temporary):
|
||||
raise FileExistsError(path)
|
||||
if terminal_fault == "write":
|
||||
raise OSError("fault_injected_terminal_receipt_write")
|
||||
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
if handle.write(payload) != len(payload):
|
||||
raise OSError("terminal_receipt_short_write")
|
||||
handle.flush()
|
||||
if terminal_fault == "fsync":
|
||||
raise OSError("fault_injected_terminal_receipt_fsync")
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, 0o600)
|
||||
temporary_readback = temporary.read_bytes()
|
||||
if temporary_readback != payload or hashlib.sha256(temporary_readback).hexdigest() != expected_digest:
|
||||
raise OSError("terminal_receipt_temporary_readback_failed")
|
||||
if terminal_fault == "link":
|
||||
raise OSError("fault_injected_terminal_receipt_link")
|
||||
os.link(temporary, path)
|
||||
created_final = True
|
||||
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
|
||||
directory_fd = os.open(path.parent, directory_flags)
|
||||
os.fsync(directory_fd)
|
||||
if terminal_fault == "readback":
|
||||
raise OSError("fault_injected_terminal_receipt_readback")
|
||||
readback = path.read_bytes()
|
||||
if readback != payload or hashlib.sha256(readback).hexdigest() != expected_digest:
|
||||
raise OSError("terminal_receipt_final_readback_failed")
|
||||
if json.loads(readback.decode("utf-8")) != document:
|
||||
raise OSError("terminal_receipt_document_readback_failed")
|
||||
temporary.unlink()
|
||||
os.fsync(directory_fd)
|
||||
except BaseException:
|
||||
try:
|
||||
if created_final:
|
||||
path.unlink(missing_ok=True)
|
||||
temporary.unlink(missing_ok=True)
|
||||
if directory_fd is None and path.parent.is_dir():
|
||||
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
|
||||
directory_fd = os.open(path.parent, directory_flags)
|
||||
if directory_fd is not None:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
raise
|
||||
finally:
|
||||
if directory_fd is not None:
|
||||
os.close(directory_fd)
|
||||
PY
|
||||
then
|
||||
return 1
|
||||
fi
|
||||
[ -f "$TERMINAL_RECEIPT_PATH" ] && [ ! -L "$TERMINAL_RECEIPT_PATH" ] || return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
record_prestate() {
|
||||
local name dest_path digest metadata
|
||||
: > "$ROLLBACK_DIR/prestate.tsv" || return 74
|
||||
chmod 600 "$ROLLBACK_DIR/prestate.tsv" || return 74
|
||||
for name in "${PAYLOAD_FILES[@]}"; do
|
||||
dest_path="$(destination_path "$name")"
|
||||
if [ -f "$dest_path" ] && [ ! -L "$dest_path" ]; then
|
||||
[ "$(stat -c '%U:%G' "$dest_path")" = "wooo:wooo" ] || return 71
|
||||
FILE_EXISTED[$name]=1
|
||||
digest="$(working_digest "$dest_path")" || return 72
|
||||
metadata="$(stat -c '%u:%g:%a' "$dest_path")" || return 72
|
||||
PREVIOUS_DIGEST[$name]="$digest"
|
||||
PREVIOUS_METADATA[$name]="$metadata"
|
||||
printf '%s\t1\t%s\t%s\n' "$name" "$digest" "$metadata" >> "$ROLLBACK_DIR/prestate.tsv" || return 72
|
||||
cp -p -- "$dest_path" "$ROLLBACK_DIR/$name" || return 72
|
||||
[ "$(working_digest "$ROLLBACK_DIR/$name")" = "$digest" ] || return 72
|
||||
elif [ ! -e "$dest_path" ]; then
|
||||
FILE_EXISTED[$name]=0
|
||||
PREVIOUS_DIGEST[$name]="-"
|
||||
PREVIOUS_METADATA[$name]="-"
|
||||
printf '%s\t0\t-\t-\n' "$name" >> "$ROLLBACK_DIR/prestate.tsv" || return 72
|
||||
else
|
||||
return 73
|
||||
fi
|
||||
done
|
||||
fsync_rollback_prestate || return 75
|
||||
}
|
||||
|
||||
register_run_owned_temp() {
|
||||
RUN_OWNED_TEMP_PATHS["$1"]=1
|
||||
}
|
||||
|
||||
verify_run_owned_temp_absence() {
|
||||
local temporary
|
||||
for temporary in "${!RUN_OWNED_TEMP_PATHS[@]}"; do
|
||||
[ ! -e "$temporary" ] && [ ! -L "$temporary" ] || return 1
|
||||
done
|
||||
}
|
||||
|
||||
cleanup_run_owned_temps() {
|
||||
local temporary failed=0 preserved=0
|
||||
local fault=""
|
||||
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then
|
||||
fault="${HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT:-}"
|
||||
fi
|
||||
for temporary in "${!RUN_OWNED_TEMP_PATHS[@]}"; do
|
||||
if [ "$fault" = "preserve_first_temp" ] \
|
||||
&& [ "$preserved" -eq 0 ] \
|
||||
&& { [ -e "$temporary" ] || [ -L "$temporary" ]; }; then
|
||||
preserved=1
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
rm -f -- "$temporary" || failed=1
|
||||
done
|
||||
verify_run_owned_temp_absence || failed=1
|
||||
[ "$failed" -eq 0 ]
|
||||
}
|
||||
|
||||
rollback_transaction() {
|
||||
local name dest_path destination_parent backup_path temporary failed=0
|
||||
ROLLBACK_ATTEMPTED=1
|
||||
ROLLBACK_PERFORMED=0
|
||||
ROLLBACK_VERIFIED=0
|
||||
RUN_TEMP_RESIDUE_VERIFIED=0
|
||||
set +e
|
||||
cleanup_run_owned_temps || failed=1
|
||||
for name in "${PAYLOAD_FILES[@]}"; do
|
||||
[ -n "${FILE_EXISTED[$name]+x}" ] || { failed=1; continue; }
|
||||
dest_path="$(destination_path "$name")"
|
||||
if [ "${FILE_EXISTED[$name]}" = "1" ]; then
|
||||
backup_path="$ROLLBACK_DIR/$name"
|
||||
destination_parent="$(dirname "$dest_path")"
|
||||
temporary="$destination_parent/.${name}.agent99-rollback-${RUN_ID}"
|
||||
register_run_owned_temp "$temporary"
|
||||
if cp -p -- "$backup_path" "$temporary" && mv -f -- "$temporary" "$dest_path"; then
|
||||
ROLLBACK_PERFORMED=1
|
||||
else
|
||||
failed=1
|
||||
fi
|
||||
else
|
||||
if rm -f -- "$dest_path"; then
|
||||
ROLLBACK_PERFORMED=1
|
||||
else
|
||||
failed=1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
cleanup_run_owned_temps || failed=1
|
||||
fsync_destination_state || failed=1
|
||||
for name in "${PAYLOAD_FILES[@]}"; do
|
||||
dest_path="$(destination_path "$name")"
|
||||
if [ "${FILE_EXISTED[$name]:-}" = "1" ]; then
|
||||
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] \
|
||||
&& [ "$(working_digest "$dest_path" 2>/dev/null)" = "${PREVIOUS_DIGEST[$name]:-invalid}" ] \
|
||||
&& [ "$(stat -c '%u:%g:%a' "$dest_path" 2>/dev/null)" = "${PREVIOUS_METADATA[$name]:-invalid}" ] \
|
||||
|| failed=1
|
||||
elif [ "${FILE_EXISTED[$name]:-}" = "0" ]; then
|
||||
[ ! -e "$dest_path" ] || failed=1
|
||||
else
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
if verify_run_owned_temp_absence; then
|
||||
RUN_TEMP_RESIDUE_VERIFIED=1
|
||||
else
|
||||
failed=1
|
||||
RUN_TEMP_RESIDUE_VERIFIED=0
|
||||
fi
|
||||
set -e
|
||||
if [ "$failed" -eq 0 ] && [ "$ROLLBACK_PERFORMED" -eq 1 ] && [ "$RUN_TEMP_RESIDUE_VERIFIED" -eq 1 ]; then
|
||||
ROLLBACK_VERIFIED=1
|
||||
return 0
|
||||
fi
|
||||
ROLLBACK_VERIFIED=0
|
||||
return 1
|
||||
}
|
||||
|
||||
handle_agent99_signal() {
|
||||
local signal_name="$1"
|
||||
local signal_exit_code="$2"
|
||||
if [ "$COMMIT_CRITICAL" -eq 1 ] || [ "$APPLY_COMPLETE" -eq 1 ]; then
|
||||
if [ -z "$DEFERRED_SIGNAL" ]; then
|
||||
DEFERRED_SIGNAL="$signal_name"
|
||||
DEFERRED_SIGNAL_EXIT="$signal_exit_code"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
exit "$signal_exit_code"
|
||||
}
|
||||
|
||||
cleanup_postcommit_directory() {
|
||||
local cleanup_kind="$1"
|
||||
local cleanup_path="$2"
|
||||
local fault=""
|
||||
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then
|
||||
fault="${HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT:-}"
|
||||
fi
|
||||
if [ "$fault" = "$cleanup_kind" ]; then
|
||||
return 1
|
||||
fi
|
||||
rm -rf -- "$cleanup_path" || return 1
|
||||
[ ! -e "$cleanup_path" ] && [ ! -L "$cleanup_path" ] || return 1
|
||||
fsync_paths_and_parents "$cleanup_path" || return 1
|
||||
[ ! -e "$cleanup_path" ] && [ ! -L "$cleanup_path" ]
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local exit_status=$?
|
||||
local rollback_status="rollback_unverified"
|
||||
local failure_receipt_written=0
|
||||
trap - EXIT HUP INT TERM
|
||||
if [ "$APPLY_STARTED" -eq 1 ] && [ "$APPLY_COMPLETE" -ne 1 ]; then
|
||||
if rollback_transaction; then
|
||||
rollback_status="failed_rolled_back"
|
||||
fi
|
||||
if write_receipt "$rollback_status" "$ROLLBACK_ATTEMPTED" "$ROLLBACK_PERFORMED" \
|
||||
"$ROLLBACK_VERIFIED" "$RUN_TEMP_RESIDUE_VERIFIED"; then
|
||||
failure_receipt_written=1
|
||||
fi
|
||||
fi
|
||||
if [ "$APPLY_COMPLETE" -ne 1 ] && [ -n "$STAGE_DIR" ]; then
|
||||
rm -rf -- "$STAGE_DIR" || true
|
||||
fi
|
||||
if [ "$APPLY_COMPLETE" -ne 1 ] \
|
||||
&& [ "$ROLLBACK_VERIFIED" -eq 1 ] \
|
||||
&& [ "$failure_receipt_written" -eq 1 ]; then
|
||||
if [ -n "$ROLLBACK_DIR" ]; then
|
||||
rm -rf -- "$ROLLBACK_DIR" || true
|
||||
fi
|
||||
fi
|
||||
exit "$exit_status"
|
||||
}
|
||||
|
||||
validate_source || fail "source_validation_failed"
|
||||
|
||||
if [ "$MODE" = "check" ]; then
|
||||
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","runId":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$MODE" = "verify" ]; then
|
||||
verify_destination || fail "destination_verification_failed"
|
||||
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","runId":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[ ! -e "$RECEIPT_PATH" ] && [ ! -L "$RECEIPT_PATH" ] || fail "run_identity_receipt_exists"
|
||||
[ ! -e "$TERMINAL_RECEIPT_PATH" ] && [ ! -L "$TERMINAL_RECEIPT_PATH" ] \
|
||||
|| fail "run_identity_terminal_receipt_exists"
|
||||
STAGE_DIR="$STAGE_ROOT/$RUN_ID"
|
||||
ROLLBACK_DIR="$ROLLBACK_ROOT/$RUN_ID"
|
||||
[ ! -e "$STAGE_DIR" ] || fail "run_identity_stage_exists"
|
||||
[ ! -e "$ROLLBACK_DIR" ] || fail "run_identity_rollback_exists"
|
||||
|
||||
[ ! -L "$LOCK_PATH" ] || fail "deployment_lock_symlink_unsafe"
|
||||
(umask 077; : >> "$LOCK_PATH") || fail "deployment_lock_unavailable"
|
||||
[ -f "$LOCK_PATH" ] && [ -O "$LOCK_PATH" ] || fail "deployment_lock_owner_invalid"
|
||||
exec 9>>"$LOCK_PATH"
|
||||
flock -x -w 30 9 || fail "deployment_lock_timeout"
|
||||
|
||||
# Old runtime revisions predate the shared gate. Refuse the first promotion if
|
||||
# one of them is already active; after common.sh moves first, new starts block.
|
||||
if pgrep -afu "$EXPECTED_USER" '/backup/scripts/(backup-|check-backup|sync-offsite|verify-offsite|enforce-latest)' \
|
||||
| grep -vF "host110-backup-runtime-executor.sh" >/dev/null 2>&1; then
|
||||
fail "legacy_backup_runtime_active"
|
||||
fi
|
||||
|
||||
install -d -m 700 "$STAGE_DIR" "$ROLLBACK_DIR"
|
||||
trap cleanup EXIT
|
||||
trap 'handle_agent99_signal INT 130' INT
|
||||
trap 'handle_agent99_signal HUP 129' HUP
|
||||
trap 'handle_agent99_signal TERM 143' TERM
|
||||
|
||||
for name in "${PAYLOAD_FILES[@]}"; do
|
||||
install -m 700 "$SOURCE_STAGE/$name" "$STAGE_DIR/$name"
|
||||
[ "$(working_digest "$STAGE_DIR/$name")" = "$(source_digest "$name")" ] || fail "stage_identity_failed"
|
||||
validate_file "$STAGE_DIR/$name" || fail "stage_validation_failed"
|
||||
done
|
||||
|
||||
record_prestate || fail "rollback_prestate_capture_failed"
|
||||
APPLY_STARTED=1
|
||||
|
||||
# common.sh is promoted first. Every managed mutating backup entrypoint then
|
||||
# takes the shared side of LOCK_PATH before doing work, while this process holds
|
||||
# the exclusive side until the independent verifier has completed.
|
||||
for name in "${PAYLOAD_FILES[@]}"; do
|
||||
dest_path="$(destination_path "$name")"
|
||||
destination_parent="$(dirname "$dest_path")"
|
||||
temporary="$destination_parent/.${name}.agent99-${RUN_ID}"
|
||||
register_run_owned_temp "$temporary"
|
||||
install -m 755 "$STAGE_DIR/$name" "$temporary"
|
||||
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ] \
|
||||
&& [ "${HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT:-}" = "after_temp_install" ] \
|
||||
&& [ "$name" = "${PAYLOAD_FILES[0]}" ]; then
|
||||
fail "fault_injected_after_temp_install"
|
||||
fi
|
||||
mv -f -- "$temporary" "$dest_path"
|
||||
done
|
||||
|
||||
fsync_destination_state || fail "post_apply_durability_failed"
|
||||
verify_destination || fail "post_apply_verification_failed"
|
||||
verifier_result="$(BACKUP_RUNTIME_DEPLOY_CONTEXT=1 python3 "$SOURCE_STAGE/verify-host110-backup-runtime.py" \
|
||||
--manifest "$SOURCE_STAGE/manifest.json" \
|
||||
--destination "$DEST_ROOT" \
|
||||
--exporter-destination "$EXPORTER_ROOT/$EXPORTER_FILE" \
|
||||
--source-revision "$SOURCE_REVISION" \
|
||||
--run-id "$RUN_ID" \
|
||||
--expected-verifier-sha256 "$VERIFIER_DIGEST")" || fail "independent_verifier_failed"
|
||||
python3 - "$verifier_result" "$SOURCE_REVISION" "$RUN_ID" <<'PY' || fail "independent_verifier_contract_failed"
|
||||
import json
|
||||
import sys
|
||||
|
||||
try:
|
||||
row = json.loads(sys.argv[1])
|
||||
except json.JSONDecodeError:
|
||||
raise SystemExit(1)
|
||||
if not (
|
||||
row.get("schemaVersion") == "agent99_host110_backup_runtime_verifier_v1"
|
||||
and row.get("ok") is True
|
||||
and row.get("sourceRevision") == sys.argv[2]
|
||||
and row.get("runId") == sys.argv[3]
|
||||
and row.get("fileCount") == 18
|
||||
and row.get("backupScriptFileCount") == 17
|
||||
and row.get("backupHealthExporterIncluded") is True
|
||||
and row.get("remoteWritePerformed") is False
|
||||
and row.get("selfIdentityVerified") is True
|
||||
):
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
|
||||
verify_run_owned_temp_absence || fail "post_apply_temp_residue_detected"
|
||||
RUN_TEMP_RESIDUE_VERIFIED=1
|
||||
COMMIT_CRITICAL=1
|
||||
if ! write_receipt "payload_verified" 0 0 0 "$RUN_TEMP_RESIDUE_VERIFIED"; then
|
||||
COMMIT_CRITICAL=0
|
||||
fail "durable_receipt_failed"
|
||||
fi
|
||||
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ] \
|
||||
&& [ -n "${HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL:-}" ]; then
|
||||
case "$HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL" in
|
||||
INT|HUP|TERM) kill -s "$HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL" "$$" ;;
|
||||
esac
|
||||
fi
|
||||
APPLY_COMPLETE=1
|
||||
COMMIT_CRITICAL=0
|
||||
stage_cleanup_verified=0
|
||||
rollback_prestate_cleanup_verified=0
|
||||
if cleanup_postcommit_directory "stage" "$STAGE_DIR"; then
|
||||
stage_cleanup_verified=1
|
||||
fi
|
||||
if cleanup_postcommit_directory "rollback_prestate" "$ROLLBACK_DIR"; then
|
||||
rollback_prestate_cleanup_verified=1
|
||||
fi
|
||||
terminal_status="verified"
|
||||
terminal_exit_code=0
|
||||
# Cleanup has completed. Freeze terminal signal state before the immutable
|
||||
# terminal writer so its receipt, stdout, and exit status cannot diverge.
|
||||
trap '' HUP INT TERM
|
||||
if [ "$stage_cleanup_verified" -ne 1 ] || [ "$rollback_prestate_cleanup_verified" -ne 1 ]; then
|
||||
terminal_status="cleanup_pending"
|
||||
terminal_exit_code=75
|
||||
elif [ -n "$DEFERRED_SIGNAL" ]; then
|
||||
terminal_status="committed_signal_deferred"
|
||||
terminal_exit_code="$DEFERRED_SIGNAL_EXIT"
|
||||
fi
|
||||
if ! write_terminal_receipt "$terminal_status" "$stage_cleanup_verified" \
|
||||
"$rollback_prestate_cleanup_verified" "$DEFERRED_SIGNAL" "$terminal_exit_code"; then
|
||||
trap - EXIT
|
||||
python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$RECEIPT_PATH" \
|
||||
"$TERMINAL_RECEIPT_PATH" "$stage_cleanup_verified" \
|
||||
"$rollback_prestate_cleanup_verified" "$DEFERRED_SIGNAL" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
print(json.dumps({
|
||||
"schemaVersion": "agent99_host110_backup_runtime_executor_v1",
|
||||
"mode": "apply",
|
||||
"status": "terminal_receipt_failed",
|
||||
"ok": False,
|
||||
"sourceRevision": sys.argv[1],
|
||||
"sourceHead": sys.argv[2],
|
||||
"runId": sys.argv[3],
|
||||
"fileCount": 18,
|
||||
"backupScriptFileCount": 17,
|
||||
"backupHealthExporterIncluded": True,
|
||||
"payloadCommitted": True,
|
||||
"rollbackAttempted": False,
|
||||
"rollbackPerformed": False,
|
||||
"rollbackVerified": False,
|
||||
"zeroResidueVerified": True,
|
||||
"zeroResidueScope": "run_owned_destination_temps",
|
||||
"stageCleanupVerified": sys.argv[6] == "1",
|
||||
"rollbackPrestateCleanupVerified": sys.argv[7] == "1",
|
||||
"receipt": sys.argv[4],
|
||||
"terminalReceipt": sys.argv[5],
|
||||
"terminalReceiptPublished": False,
|
||||
"terminalExitCode": 76,
|
||||
"deferredSignal": sys.argv[8] or None,
|
||||
"independentVerifierVerified": True,
|
||||
"verifier": None,
|
||||
}, ensure_ascii=True, sort_keys=True))
|
||||
PY
|
||||
printf 'HOST110_BACKUP_RUNTIME_OK=0\nERROR=terminal_receipt_failed\n' >&2
|
||||
exit 76
|
||||
fi
|
||||
if [ "$stage_cleanup_verified" -eq 1 ]; then STAGE_DIR=""; fi
|
||||
if [ "$rollback_prestate_cleanup_verified" -eq 1 ]; then ROLLBACK_DIR=""; fi
|
||||
trap - EXIT
|
||||
python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$RECEIPT_PATH" "$TERMINAL_RECEIPT_PATH" \
|
||||
"$verifier_result" "$stage_cleanup_verified" "$rollback_prestate_cleanup_verified" \
|
||||
"$terminal_status" "$terminal_exit_code" "$DEFERRED_SIGNAL" "$RUN_ID" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
status = sys.argv[8]
|
||||
verifier = json.loads(sys.argv[5])
|
||||
print(json.dumps({
|
||||
"schemaVersion": "agent99_host110_backup_runtime_executor_v1",
|
||||
"mode": "apply",
|
||||
"status": status,
|
||||
"ok": status == "verified",
|
||||
"sourceRevision": sys.argv[1],
|
||||
"sourceHead": sys.argv[2],
|
||||
"runId": sys.argv[11],
|
||||
"fileCount": 18,
|
||||
"backupScriptFileCount": 17,
|
||||
"backupHealthExporterIncluded": True,
|
||||
"payloadCommitted": True,
|
||||
"rollbackAttempted": False,
|
||||
"rollbackPerformed": False,
|
||||
"rollbackVerified": False,
|
||||
"zeroResidueVerified": True,
|
||||
"zeroResidueScope": "run_owned_destination_temps",
|
||||
"stageCleanupVerified": sys.argv[6] == "1",
|
||||
"rollbackPrestateCleanupVerified": sys.argv[7] == "1",
|
||||
"receipt": sys.argv[3],
|
||||
"terminalReceipt": sys.argv[4],
|
||||
"terminalReceiptPublished": True,
|
||||
"terminalExitCode": int(sys.argv[9]),
|
||||
"deferredSignal": sys.argv[10] or None,
|
||||
"independentVerifierVerified": True,
|
||||
"verifier": verifier if status == "verified" else None,
|
||||
}, ensure_ascii=True, sort_keys=True))
|
||||
PY
|
||||
exit "$terminal_exit_code"
|
||||
@@ -121,8 +121,6 @@ prepare_rclone() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
bind_rclone_config_to_runtime_user
|
||||
|
||||
if [ "${PROVIDER}" = "b2" ]; then
|
||||
if ! check_b2_config; then
|
||||
return 1
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts" / "backup" / "verify-offsite-full-sync.sh"
|
||||
|
||||
|
||||
def run_verifier(tmp_path: Path, mode: str) -> tuple[subprocess.CompletedProcess[str], str, Path]:
|
||||
bin_dir = tmp_path / "bin"
|
||||
bin_dir.mkdir()
|
||||
runtime_home = tmp_path / "runtime-home"
|
||||
config = runtime_home / ".config" / "rclone" / "rclone.conf"
|
||||
config.parent.mkdir(parents=True)
|
||||
config.write_text("test fixture only\n", encoding="utf-8")
|
||||
event_log = tmp_path / "rclone.events"
|
||||
|
||||
getent = bin_dir / "getent"
|
||||
getent.write_text(
|
||||
"#!/bin/sh\nprintf 'wooo:x:1000:1000::%s:/bin/bash\\n' \"$TEST_RUNTIME_HOME\"\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
getent.chmod(0o755)
|
||||
|
||||
rclone = bin_dir / "rclone"
|
||||
rclone.write_text(
|
||||
"""#!/bin/sh
|
||||
printf 'CMD=%s CONFIG=%s\n' "$*" "${RCLONE_CONFIG:-}" >> "$TEST_EVENT_LOG"
|
||||
case "$1" in
|
||||
listremotes)
|
||||
[ "$TEST_RCLONE_MODE" = config_missing ] && printf '%s\n' 'other:' || printf '%s\n' 'gdrive:'
|
||||
;;
|
||||
lsf)
|
||||
if [ "$TEST_RCLONE_MODE" = remote_down ]; then
|
||||
exit 9
|
||||
fi
|
||||
case "$2" in
|
||||
*/snapshots) printf '%s\n' snapshot-file ;;
|
||||
*) printf '%s\n' awoooi/ ;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
rclone.chmod(0o755)
|
||||
|
||||
env = os.environ.copy()
|
||||
env.pop("RCLONE_CONFIG", None)
|
||||
env.update(
|
||||
{
|
||||
"BACKUP_BASE": str(tmp_path / "backup"),
|
||||
"HOME": "/",
|
||||
"OFFSITE_REPOS": "awoooi",
|
||||
"PATH": f"{bin_dir}:{env['PATH']}",
|
||||
"TEST_EVENT_LOG": str(event_log),
|
||||
"TEST_RCLONE_MODE": mode,
|
||||
"TEST_RUNTIME_HOME": str(runtime_home),
|
||||
}
|
||||
)
|
||||
result = subprocess.run(
|
||||
["bash", str(SCRIPT), "--no-color"],
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
return result, event_log.read_text(encoding="utf-8"), config
|
||||
|
||||
|
||||
def test_verifier_binds_rclone_config_to_runtime_account(tmp_path: Path) -> None:
|
||||
result, events, config = run_verifier(tmp_path, "ok")
|
||||
|
||||
assert result.returncode == 1 # The full-sync marker is intentionally absent.
|
||||
assert "rclone provider readback succeeded" in result.stdout
|
||||
assert f"CONFIG={config}" in events
|
||||
|
||||
|
||||
def test_verifier_distinguishes_missing_config_without_remote_calls(tmp_path: Path) -> None:
|
||||
result, events, _ = run_verifier(tmp_path, "config_missing")
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "rclone remote configuration unavailable" in result.stdout
|
||||
assert " lsf " not in f" {events} "
|
||||
|
||||
|
||||
def test_verifier_distinguishes_provider_failure_and_stops_fanout(tmp_path: Path) -> None:
|
||||
result, events, _ = run_verifier(tmp_path, "remote_down")
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "rclone provider readback failed" in result.stdout
|
||||
assert events.count("CMD=lsf ") == 1
|
||||
@@ -1,177 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independent no-write verifier for the Host110 backup runtime transaction."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
EXPECTED_DESTINATION = Path("/backup/scripts")
|
||||
EXPECTED_EXPORTER_DESTINATION = Path("/home/wooo/scripts/backup-health-textfile-exporter.py")
|
||||
RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")
|
||||
EXPECTED_BACKUP_FILES = (
|
||||
"common.sh",
|
||||
"backup-all.sh",
|
||||
"backup-host188-products.sh",
|
||||
"verify-host188-products-backup.sh",
|
||||
"verify-host188-products-archive.py",
|
||||
"backup-gitea.sh",
|
||||
"gitea-full-backup-restore-drill.sh",
|
||||
"backup-configs.sh",
|
||||
"backup-awoooi.sh",
|
||||
"backup-awoooi-frequent.sh",
|
||||
"backup-clawbot.sh",
|
||||
"backup-sentry.sh",
|
||||
"check-backup-integrity.sh",
|
||||
"sync-offsite-backups.sh",
|
||||
"backup-offsite-readiness-gate.sh",
|
||||
"verify-offsite-full-sync.sh",
|
||||
"enforce-latest-only-retention.sh",
|
||||
)
|
||||
EXPORTER_FILE = "backup-health-textfile-exporter.py"
|
||||
EXPECTED_FILES = EXPECTED_BACKUP_FILES + (EXPORTER_FILE,)
|
||||
|
||||
|
||||
def fail(reason: str) -> None:
|
||||
print(f"HOST110_BACKUP_RUNTIME_VERIFY_OK=0\nERROR={reason}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def acquire_runtime_read_lock(destination: Path):
|
||||
if destination != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_DEPLOY_CONTEXT") == "1":
|
||||
return None
|
||||
if RUNTIME_LOCK.is_symlink() or not RUNTIME_LOCK.is_file():
|
||||
fail("runtime_lock_unavailable")
|
||||
handle = RUNTIME_LOCK.open("rb")
|
||||
if RUNTIME_LOCK.stat().st_uid != os.getuid():
|
||||
handle.close()
|
||||
fail("runtime_lock_owner_invalid")
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
|
||||
return handle
|
||||
except BlockingIOError:
|
||||
if time.monotonic() >= deadline:
|
||||
handle.close()
|
||||
fail("runtime_deployment_active")
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
manifest_source = parser.add_mutually_exclusive_group(required=True)
|
||||
manifest_source.add_argument("--manifest")
|
||||
manifest_source.add_argument("--manifest-base64")
|
||||
parser.add_argument("--destination", required=True)
|
||||
parser.add_argument("--exporter-destination", required=True)
|
||||
parser.add_argument("--source-revision", required=True)
|
||||
parser.add_argument("--run-id", required=True)
|
||||
parser.add_argument("--expected-verifier-sha256", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", args.source_revision):
|
||||
fail("invalid_source_revision")
|
||||
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}", args.run_id):
|
||||
fail("invalid_run_id")
|
||||
destination = Path(args.destination)
|
||||
if destination != EXPECTED_DESTINATION or destination.is_symlink() or not destination.is_dir():
|
||||
fail("invalid_destination")
|
||||
exporter_destination = Path(args.exporter_destination)
|
||||
if exporter_destination != EXPECTED_EXPORTER_DESTINATION:
|
||||
fail("invalid_exporter_destination")
|
||||
runtime_lock = acquire_runtime_read_lock(destination)
|
||||
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", args.expected_verifier_sha256):
|
||||
fail("invalid_verifier_identity")
|
||||
try:
|
||||
if args.manifest:
|
||||
manifest_text = Path(args.manifest).read_text(encoding="utf-8")
|
||||
else:
|
||||
manifest_text = base64.b64decode(args.manifest_base64, validate=True).decode("utf-8")
|
||||
document = json.loads(manifest_text)
|
||||
except (OSError, UnicodeDecodeError, ValueError, json.JSONDecodeError):
|
||||
fail("manifest_unavailable")
|
||||
rows = document.get("files")
|
||||
if (
|
||||
document.get("schemaVersion") != "agent99_host110_backup_runtime_source_manifest_v1"
|
||||
or document.get("sourceRevision") != args.source_revision
|
||||
or document.get("runId") != args.run_id
|
||||
or not re.fullmatch(r"[0-9a-f]{40}", str(document.get("sourceHead", "")))
|
||||
or document.get("verifierSha256") != args.expected_verifier_sha256
|
||||
or not isinstance(rows, list)
|
||||
or len(rows) != len(EXPECTED_FILES)
|
||||
):
|
||||
fail("manifest_contract_failed")
|
||||
by_name: dict[str, str] = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
fail("manifest_file_set_failed")
|
||||
name = str(row.get("name", ""))
|
||||
if name in by_name:
|
||||
fail("manifest_file_set_failed")
|
||||
by_name[name] = str(row.get("sha256", ""))
|
||||
if set(by_name) != set(EXPECTED_FILES) or not all(re.fullmatch(r"[0-9a-f]{64}", value) for value in by_name.values()):
|
||||
fail("manifest_file_set_failed")
|
||||
|
||||
verifier_path = Path(__file__)
|
||||
self_identity_verified = verifier_path.is_file() and sha256(verifier_path) == args.expected_verifier_sha256
|
||||
if args.manifest and not self_identity_verified:
|
||||
fail("verifier_identity_failed")
|
||||
|
||||
root = destination.resolve(strict=True)
|
||||
for name in EXPECTED_FILES:
|
||||
path = exporter_destination if name == EXPORTER_FILE else destination / name
|
||||
if path.is_symlink() or not path.is_file():
|
||||
fail("runtime_file_type_failed")
|
||||
resolved = path.resolve(strict=True)
|
||||
if name == EXPORTER_FILE:
|
||||
if resolved != exporter_destination:
|
||||
fail("runtime_file_boundary_failed")
|
||||
elif resolved.parent != root:
|
||||
fail("runtime_file_boundary_failed")
|
||||
stat = resolved.stat()
|
||||
if stat.st_uid != os.getuid() or stat.st_gid != os.getgid() or stat.st_mode & 0o777 != 0o755:
|
||||
fail("runtime_file_metadata_failed")
|
||||
if sha256(resolved) != by_name[name]:
|
||||
fail("runtime_file_identity_failed")
|
||||
|
||||
result = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_verifier_v1",
|
||||
"ok": True,
|
||||
"sourceRevision": args.source_revision,
|
||||
"runId": args.run_id,
|
||||
"fileCount": len(EXPECTED_FILES),
|
||||
"backupScriptFileCount": len(EXPECTED_BACKUP_FILES),
|
||||
"backupHealthExporterIncluded": True,
|
||||
"destination": str(EXPECTED_DESTINATION),
|
||||
"exporterDestination": str(EXPECTED_EXPORTER_DESTINATION),
|
||||
"remoteWritePerformed": False,
|
||||
"secretValuesRead": False,
|
||||
"selfIdentityVerified": self_identity_verified,
|
||||
"verifierSha256": args.expected_verifier_sha256,
|
||||
}
|
||||
print(json.dumps(result, ensure_ascii=True, sort_keys=True))
|
||||
if runtime_lock is not None:
|
||||
runtime_lock.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -4,14 +4,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
@@ -23,28 +20,6 @@ EXPECTED_PREFIXES = {
|
||||
"n8n-online-": ".tar.gz",
|
||||
}
|
||||
FIXED_ASSETS = {"n8n-validation.json"}
|
||||
RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")
|
||||
|
||||
|
||||
def acquire_runtime_lock():
|
||||
if Path(__file__).resolve().parent != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_SHARED_LOCK_HELD") == "1":
|
||||
return None
|
||||
if RUNTIME_LOCK.is_symlink():
|
||||
raise RuntimeError("backup_runtime_gate_unsafe")
|
||||
handle = RUNTIME_LOCK.open("a+")
|
||||
if RUNTIME_LOCK.stat().st_uid != os.getuid():
|
||||
handle.close()
|
||||
raise RuntimeError("backup_runtime_gate_owner_invalid")
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
|
||||
return handle
|
||||
except BlockingIOError:
|
||||
if time.monotonic() >= deadline:
|
||||
handle.close()
|
||||
raise RuntimeError("backup_runtime_deployment_active")
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
@@ -178,13 +153,10 @@ def validate_restore(root: Path) -> dict[str, Any]:
|
||||
|
||||
|
||||
def main() -> int:
|
||||
runtime_lock = acquire_runtime_lock()
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(validate_restore(args.root), separators=(",", ":")))
|
||||
if runtime_lock is not None:
|
||||
runtime_lock.close()
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -113,16 +113,11 @@ low_priority() {
|
||||
fi
|
||||
}
|
||||
|
||||
rclone_configured() {
|
||||
rclone_ready() {
|
||||
command -v rclone >/dev/null 2>&1 || return 1
|
||||
rclone listremotes 2>/dev/null | grep -Fxq "${RCLONE_REMOTE}:"
|
||||
}
|
||||
|
||||
rclone_remote_readable() {
|
||||
low_priority timeout 60s rclone lsf "$(remote_root)" \
|
||||
--dirs-only --max-depth 1 >/dev/null 2>&1
|
||||
}
|
||||
|
||||
count_remote_snapshots() {
|
||||
local repo="$1"
|
||||
local remote_snapshots
|
||||
@@ -206,7 +201,6 @@ main() {
|
||||
local ok
|
||||
local latest_only_ok=1
|
||||
local verify_ok=0
|
||||
local remote_readable=0
|
||||
local success_marker="${OFFSITE_DIR}/${PROVIDER}-full-verify-last-success"
|
||||
local success_ts
|
||||
local success_age
|
||||
@@ -231,19 +225,13 @@ main() {
|
||||
echo "WRITE_TEXTFILE=${WRITE_TEXTFILE}"
|
||||
echo
|
||||
|
||||
bind_rclone_config_to_runtime_user
|
||||
if [ "${PROVIDER}" != "rclone" ]; then
|
||||
printf "%sBLOCKED%s unsupported provider for remote snapshot verification: %s\n" "${red}" "${reset}" "${PROVIDER}"
|
||||
failed=1
|
||||
elif ! rclone_configured; then
|
||||
printf "%sBLOCKED%s rclone remote configuration unavailable: %s:\n" "${red}" "${reset}" "${RCLONE_REMOTE}"
|
||||
failed=1
|
||||
elif rclone_remote_readable; then
|
||||
elif rclone_ready; then
|
||||
printf "%sOK%s rclone remote configured: %s:\n" "${green}" "${reset}" "${RCLONE_REMOTE}"
|
||||
printf "%sOK%s rclone provider readback succeeded: %s\n" "${green}" "${reset}" "$(remote_root)"
|
||||
remote_readable=1
|
||||
else
|
||||
printf "%sBLOCKED%s rclone provider readback failed: %s\n" "${red}" "${reset}" "$(remote_root)"
|
||||
printf "%sBLOCKED%s rclone remote unavailable: %s:\n" "${red}" "${reset}" "${RCLONE_REMOTE}"
|
||||
failed=1
|
||||
fi
|
||||
|
||||
@@ -257,11 +245,7 @@ main() {
|
||||
echo
|
||||
echo "== remote snapshot counts =="
|
||||
for repo in ${EXPECTED_REPOS}; do
|
||||
if [ "${remote_readable}" = "1" ]; then
|
||||
count="$(count_remote_snapshots "${repo}" || true)"
|
||||
else
|
||||
count=-1
|
||||
fi
|
||||
count="$(count_remote_snapshots "${repo}" || true)"
|
||||
ok=0
|
||||
if [ "${count}" = "1" ]; then
|
||||
ok=1
|
||||
|
||||
43
scripts/ci/bootstrap-host-runner-tools.sh
Executable file
43
scripts/ci/bootstrap-host-runner-tools.sh
Executable file
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
required_commands=(node npm git curl bash timeout python3 ssh docker)
|
||||
missing_commands=()
|
||||
|
||||
for command_name in "${required_commands[@]}"; do
|
||||
if ! command -v "${command_name}" >/dev/null 2>&1; then
|
||||
missing_commands+=("${command_name}")
|
||||
fi
|
||||
done
|
||||
|
||||
if command -v docker >/dev/null 2>&1 && \
|
||||
! docker buildx version >/dev/null 2>&1; then
|
||||
missing_commands+=(docker-buildx)
|
||||
fi
|
||||
|
||||
if [ "${#missing_commands[@]}" -eq 0 ]; then
|
||||
echo "host_runner_tools_ready=1 bootstrap_install_performed=0"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
printf 'host_runner_tools_missing=%s\n' "$(IFS=,; echo "${missing_commands[*]}")"
|
||||
if ! command -v apk >/dev/null 2>&1; then
|
||||
echo "BLOCKER host_runner_tools_missing_and_apk_unavailable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
apk add --no-cache \
|
||||
nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
|
||||
|
||||
for command_name in "${required_commands[@]}"; do
|
||||
command -v "${command_name}" >/dev/null 2>&1 || {
|
||||
echo "BLOCKER host_runner_tool_install_failed command=${command_name}"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
docker buildx version >/dev/null 2>&1 || {
|
||||
echo "BLOCKER host_runner_tool_install_failed command=docker-buildx"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "host_runner_tools_ready=1 bootstrap_install_performed=1"
|
||||
96
scripts/ci/tests/test_bootstrap_host_runner_tools.py
Normal file
96
scripts/ci/tests/test_bootstrap_host_runner_tools.py
Normal file
@@ -0,0 +1,96 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts/ci/bootstrap-host-runner-tools.sh"
|
||||
REQUIRED = ("node", "npm", "git", "curl", "bash", "timeout", "python3", "ssh")
|
||||
|
||||
|
||||
def _write_command(path: Path, body: str = "exit 0\n") -> None:
|
||||
path.write_text("#!/bin/sh\n" + body, encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def _fake_tool_path(directory: Path, *, include_npm: bool) -> None:
|
||||
for command in REQUIRED:
|
||||
if command != "npm" or include_npm:
|
||||
_write_command(directory / command)
|
||||
_write_command(directory / "docker")
|
||||
|
||||
|
||||
def _run(directory: Path) -> subprocess.CompletedProcess[str]:
|
||||
env = os.environ.copy()
|
||||
env["PATH"] = str(directory)
|
||||
return subprocess.run(
|
||||
["/bin/bash", str(SCRIPT)],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
env=env,
|
||||
)
|
||||
|
||||
|
||||
def test_ready_runner_skips_package_install() -> None:
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
directory = Path(temp)
|
||||
_fake_tool_path(directory, include_npm=True)
|
||||
|
||||
completed = _run(directory)
|
||||
|
||||
assert completed.returncode == 0
|
||||
assert "host_runner_tools_ready=1 bootstrap_install_performed=0" in completed.stdout
|
||||
|
||||
|
||||
def test_missing_tool_without_apk_fails_closed() -> None:
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
directory = Path(temp)
|
||||
_fake_tool_path(directory, include_npm=False)
|
||||
|
||||
completed = _run(directory)
|
||||
|
||||
assert completed.returncode == 1
|
||||
assert "host_runner_tools_missing=npm" in completed.stdout
|
||||
assert "BLOCKER host_runner_tools_missing_and_apk_unavailable" in completed.stdout
|
||||
|
||||
|
||||
def test_missing_tool_uses_apk_once_and_revalidates() -> None:
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
directory = Path(temp)
|
||||
_fake_tool_path(directory, include_npm=False)
|
||||
apk_log = directory / "apk.log"
|
||||
_write_command(
|
||||
directory / "apk",
|
||||
'printf "%s\\n" "$*" > "$APK_LOG"\n'
|
||||
'printf "#!/bin/sh\\nexit 0\\n" > "$FAKE_BIN/npm"\n'
|
||||
'/bin/chmod +x "$FAKE_BIN/npm"\n',
|
||||
)
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"PATH": str(directory),
|
||||
"APK_LOG": str(apk_log),
|
||||
"FAKE_BIN": str(directory),
|
||||
}
|
||||
)
|
||||
|
||||
completed = subprocess.run(
|
||||
["/bin/bash", str(SCRIPT)],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
env=env,
|
||||
)
|
||||
|
||||
install_args = apk_log.read_text(encoding="utf-8")
|
||||
|
||||
assert completed.returncode == 0
|
||||
assert "bootstrap_install_performed=1" in completed.stdout
|
||||
assert install_args.startswith("add --no-cache ")
|
||||
assert "docker-cli-buildx" in install_args
|
||||
@@ -26,8 +26,7 @@ def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None:
|
||||
assert "._+" not in source
|
||||
assert '$EvidenceRunId = if ($RunId -match $SafeIdPattern)' in source
|
||||
assert "Get-Agent99RuntimeSourceBinding" in source
|
||||
assert "$ExpectedRuntimeFileCount = 19" in source
|
||||
assert "$result.fileCount -eq $ExpectedRuntimeFileCount" in source
|
||||
assert "$result.fileCount -eq 18" in source
|
||||
assert "$result.sourceRevision -eq $SourceRevision" in source
|
||||
assert "$result.entrypointHashMatched" in source
|
||||
assert '$CredentialFile = "/etc/awoooi/secrets/signoz-metadata-api-key"' in source
|
||||
@@ -294,8 +293,8 @@ def test_entrypoint_is_in_exact_agent99_runtime_bundle() -> None:
|
||||
assert expected in _quoted_array(contract, "$requiredFiles = @(", ")")
|
||||
assert expected in _quoted_array(sender, "RUNTIME_FILES=(", ")")
|
||||
assert expected in _quoted_array(receiver, "$FixedRuntimeFiles = @(", ")")
|
||||
assert 'expectedRuntimeFileCount": 19' in sender
|
||||
assert '$runtimeManifest.fileCount -eq 19' in receiver
|
||||
assert 'expectedRuntimeFileCount": 18' in sender
|
||||
assert '$runtimeManifest.fileCount -eq 18' in receiver
|
||||
|
||||
|
||||
def test_runtime_contract_check_requires_typed_signoz_transport_readback() -> None:
|
||||
|
||||
@@ -122,7 +122,7 @@ def test_agent99_atomic_bundle_owns_the_dispatch_entrypoint() -> None:
|
||||
assert "$dbExecutorRecoveryContract" in contract
|
||||
assert "awoooi_db_bounded_executor_receipt_v2" in contract
|
||||
assert "Receipt.database_sqlstate" in contract
|
||||
assert 'expectedRuntimeFileCount": 19' in transport
|
||||
assert "expectedRuntimeFileCount=19" in transport
|
||||
assert "expectedRuntimeFileCount -ne 19" in receiver
|
||||
assert "fileCount = 19" in receiver
|
||||
assert 'expectedRuntimeFileCount": 18' in transport
|
||||
assert "expectedRuntimeFileCount=18" in transport
|
||||
assert "expectedRuntimeFileCount -ne 18" in receiver
|
||||
assert "fileCount = 18" in receiver
|
||||
|
||||
@@ -486,7 +486,7 @@ $decision = Get-AgentLivePreflightDecision `
|
||||
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
|
||||
-Manifest $manifest `
|
||||
-ManifestCheckRequired ([bool]($ReadinessScope -eq "FullRuntime")) `
|
||||
-ExpectedRuntimeFileCount 19 `
|
||||
-ExpectedRuntimeFileCount 18 `
|
||||
-TaskFailureCount $taskFailures.Count `
|
||||
-RelayListenerCount $relayListeners.Count `
|
||||
-RequiredEvidenceStaleCount $requiredEvidenceStale.Count `
|
||||
|
||||
@@ -34,7 +34,6 @@ $FixedRuntimeFiles = @(
|
||||
"agent99-control-plane.ps1",
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
@@ -952,7 +951,7 @@ try {
|
||||
}
|
||||
$sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant()
|
||||
if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
|
||||
if ([int]$envelope.expectedRuntimeFileCount -ne 19) { throw "invalid_expected_runtime_file_count" }
|
||||
if ([int]$envelope.expectedRuntimeFileCount -ne 18) { throw "invalid_expected_runtime_file_count" }
|
||||
|
||||
$traceId = [string]$envelope.traceId
|
||||
$runId = [string]$envelope.runId
|
||||
@@ -987,7 +986,7 @@ try {
|
||||
$sourceManifest = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_source_manifest_v1"
|
||||
sourceRevision = $sourceRevision
|
||||
fileCount = 19
|
||||
fileCount = 18
|
||||
manifestSha256 = $manifestDigest
|
||||
files = @($FixedRuntimeFiles | ForEach-Object {
|
||||
[pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() }
|
||||
@@ -1024,7 +1023,7 @@ try {
|
||||
status = "check_ready"
|
||||
mode = "check"
|
||||
sourceRevision = $sourceRevision
|
||||
expectedRuntimeFileCount = 19
|
||||
expectedRuntimeFileCount = 18
|
||||
manifestSha256 = $manifestDigest
|
||||
plannedStagingPath = $stagePath
|
||||
runtimeManifest = $currentManifest
|
||||
@@ -1111,7 +1110,7 @@ try {
|
||||
if (
|
||||
[string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or
|
||||
[string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or
|
||||
[int]$existingSourceManifest.fileCount -ne 19 -or
|
||||
[int]$existingSourceManifest.fileCount -ne 18 -or
|
||||
[string]$existingSourceManifest.manifestSha256 -ne $manifestDigest
|
||||
) { throw "existing_staging_manifest_mismatch" }
|
||||
} catch {
|
||||
@@ -1135,7 +1134,7 @@ try {
|
||||
[string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and
|
||||
$live.runtimeMatched -and
|
||||
$live.sourceRevision -eq $sourceRevision -and
|
||||
$live.fileCount -eq 19 -and
|
||||
$live.fileCount -eq 18 -and
|
||||
$live.mismatchCount -eq 0 -and
|
||||
$liveLauncherContract.ok -and
|
||||
[string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256
|
||||
@@ -1448,7 +1447,7 @@ try {
|
||||
$runtimeManifest.exists -and
|
||||
$runtimeManifest.runtimeMatched -and
|
||||
$runtimeManifest.sourceRevision -eq $sourceRevision -and
|
||||
$runtimeManifest.fileCount -eq 19 -and
|
||||
$runtimeManifest.fileCount -eq 18 -and
|
||||
$runtimeManifest.mismatchCount -eq 0 -and
|
||||
$launcherContract.ok -and
|
||||
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and
|
||||
|
||||
@@ -18,7 +18,6 @@ RUNTIME_FILES=(
|
||||
"agent99-control-plane.ps1"
|
||||
"agent99-db-bounded-executor.ps1"
|
||||
"agent99-signoz-metadata-executor.ps1"
|
||||
"agent99-host110-backup-runtime-broker.ps1"
|
||||
"agent99-deploy.ps1"
|
||||
"agent99-register-tasks.ps1"
|
||||
"agent99-alertmanager-alertchain-poll.ps1"
|
||||
@@ -228,7 +227,7 @@ trace_id, run_id, work_item_id = sys.argv[4:7]
|
||||
output_path = Path(sys.argv[7])
|
||||
runtime_names = sys.argv[8:]
|
||||
|
||||
if len(runtime_names) != 19 or len(set(runtime_names)) != 19:
|
||||
if len(runtime_names) != 18 or len(set(runtime_names)) != 18:
|
||||
raise SystemExit("fixed_runtime_file_contract_failed")
|
||||
|
||||
files: list[dict[str, str]] = []
|
||||
@@ -268,7 +267,7 @@ envelope = {
|
||||
"runId": run_id,
|
||||
"workItemId": work_item_id,
|
||||
"sourceRevision": source_revision,
|
||||
"expectedRuntimeFileCount": 19,
|
||||
"expectedRuntimeFileCount": 18,
|
||||
"manifestSha256": manifest_sha256,
|
||||
"files": files,
|
||||
"livePreflight": {
|
||||
@@ -420,14 +419,14 @@ stage_token = hashlib.sha256(stage_identity.encode("utf-8")).hexdigest()[:20]
|
||||
script = r'''$ErrorActionPreference='Stop';$ProgressPreference='SilentlyContinue'
|
||||
$a='C:\Wooo\Agent99';$b=Join-Path $a 'bin';$p=Join-Path $a 'state\runtime-manifest.json'
|
||||
$r=[ordered]@{exists=$false;sourceRevision='';runtimeMatched=$false;recordedRuntimeMatched=$false;fileCount=0;mismatchCount=-1;recordedMismatchCount=-1;parseError=''}
|
||||
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 19-and$m.mismatchCount-eq 0-and$rows.Count-eq 19-and$seen.Count-eq 19-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
|
||||
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 18-and$m.mismatchCount-eq 0-and$rows.Count-eq 18-and$seen.Count-eq 18-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
|
||||
$c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''}
|
||||
try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name}
|
||||
$z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false}
|
||||
$ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0)
|
||||
$status=if($ready){'check_ready'}else{'check_failed_no_apply'}
|
||||
$next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'}
|
||||
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=19;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
|
||||
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=18;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
|
||||
if(-not$ready){exit 65}'''
|
||||
script = (
|
||||
script.replace("__SOURCE_REVISION__", source_revision)
|
||||
|
||||
@@ -1,220 +0,0 @@
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Write-ReplayTrace {
|
||||
param([string]$Step)
|
||||
if ($ReplayTrace) { Write-Output "REPLAY_STEP=$Step" }
|
||||
}
|
||||
|
||||
if (-not $BrokerGzipBase64) { throw "broker_payload_missing" }
|
||||
Write-ReplayTrace "decode_source"
|
||||
$compressed = [Convert]::FromBase64String([string]$BrokerGzipBase64)
|
||||
$inputStream = New-Object IO.MemoryStream(,$compressed)
|
||||
$gzip = New-Object IO.Compression.GzipStream(
|
||||
$inputStream,
|
||||
[IO.Compression.CompressionMode]::Decompress
|
||||
)
|
||||
$reader = New-Object IO.StreamReader($gzip, [Text.Encoding]::UTF8)
|
||||
try {
|
||||
$brokerSource = $reader.ReadToEnd()
|
||||
} finally {
|
||||
$reader.Dispose()
|
||||
$gzip.Dispose()
|
||||
$inputStream.Dispose()
|
||||
}
|
||||
|
||||
$tokens = $null
|
||||
$parseErrors = $null
|
||||
$ast = [Management.Automation.Language.Parser]::ParseInput(
|
||||
$brokerSource,
|
||||
[ref]$tokens,
|
||||
[ref]$parseErrors
|
||||
)
|
||||
if ($parseErrors.Count -ne 0) { throw "broker_parse_failed" }
|
||||
Write-ReplayTrace "define_functions"
|
||||
|
||||
$functionNames = @(
|
||||
"Test-Agent99JsonField",
|
||||
"Assert-Agent99JsonFields",
|
||||
"Convert-Agent99ExecutorDocument",
|
||||
"Assert-Agent99ApplyResultFields",
|
||||
"Assert-Agent99VerifierFields",
|
||||
"Get-Agent99ExecutorResult",
|
||||
"Get-Agent99CommittedFailureResult"
|
||||
)
|
||||
foreach ($functionName in $functionNames) {
|
||||
$node = $ast.Find(
|
||||
{
|
||||
param($candidate)
|
||||
$candidate -is [Management.Automation.Language.FunctionDefinitionAst] -and
|
||||
$candidate.Name -eq $functionName
|
||||
},
|
||||
$true
|
||||
)
|
||||
if ($null -eq $node) { throw "broker_function_missing_$functionName" }
|
||||
Invoke-Expression $node.Extent.Text
|
||||
}
|
||||
|
||||
$SourceRevision = "a" * 40
|
||||
$RunId = "windows99-contract-replay"
|
||||
$ExpectedFileCount = 18
|
||||
$sourceHead = "b" * 40
|
||||
$payloadReceipt = "/backup/status/agent99-backup-runtime-deploy-$RunId.json"
|
||||
$terminalReceipt = "/backup/status/agent99-backup-runtime-terminal-$RunId.json"
|
||||
|
||||
function Copy-ReplayDocument {
|
||||
param([object]$Document)
|
||||
return ($Document | ConvertTo-Json -Compress -Depth 12 | ConvertFrom-Json)
|
||||
}
|
||||
|
||||
function New-ReplayTransport {
|
||||
param([object]$Document, [int]$ExitCode)
|
||||
return [pscustomobject]@{
|
||||
ok = [bool]($ExitCode -eq 0)
|
||||
exitCode = $ExitCode
|
||||
reason = if ($ExitCode -eq 0) { "completed" } else { "process_failed" }
|
||||
stdout = $Document | ConvertTo-Json -Compress -Depth 12
|
||||
stderrPresent = $false
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Rejected {
|
||||
param([scriptblock]$Operation, [string]$Name)
|
||||
$accepted = $false
|
||||
try {
|
||||
& $Operation | Out-Null
|
||||
$accepted = $true
|
||||
} catch {}
|
||||
if ($accepted) { throw "contract_mutation_accepted_$Name" }
|
||||
}
|
||||
|
||||
$verifier = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_verifier_v1"
|
||||
ok = $true
|
||||
sourceRevision = $SourceRevision
|
||||
runId = $RunId
|
||||
fileCount = 18
|
||||
backupScriptFileCount = 17
|
||||
backupHealthExporterIncluded = $true
|
||||
remoteWritePerformed = $false
|
||||
selfIdentityVerified = $true
|
||||
}
|
||||
$success = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_executor_v1"
|
||||
mode = "apply"
|
||||
status = "verified"
|
||||
ok = $true
|
||||
sourceRevision = $SourceRevision
|
||||
sourceHead = $sourceHead
|
||||
runId = $RunId
|
||||
fileCount = 18
|
||||
backupScriptFileCount = 17
|
||||
backupHealthExporterIncluded = $true
|
||||
payloadCommitted = $true
|
||||
rollbackAttempted = $false
|
||||
rollbackPerformed = $false
|
||||
rollbackVerified = $false
|
||||
zeroResidueVerified = $true
|
||||
zeroResidueScope = "run_owned_destination_temps"
|
||||
stageCleanupVerified = $true
|
||||
rollbackPrestateCleanupVerified = $true
|
||||
receipt = $payloadReceipt
|
||||
terminalReceipt = $terminalReceipt
|
||||
terminalReceiptPublished = $true
|
||||
terminalExitCode = 0
|
||||
deferredSignal = $null
|
||||
independentVerifierVerified = $true
|
||||
verifier = $verifier
|
||||
}
|
||||
|
||||
Write-ReplayTrace "accept_success"
|
||||
Get-Agent99ExecutorResult (New-ReplayTransport $success 0) "Apply" | Out-Null
|
||||
$topLevelRequired = @(
|
||||
"schemaVersion", "mode", "status", "ok", "sourceRevision", "sourceHead",
|
||||
"runId", "fileCount", "backupScriptFileCount", "backupHealthExporterIncluded",
|
||||
"payloadCommitted", "rollbackAttempted", "rollbackPerformed", "rollbackVerified",
|
||||
"zeroResidueVerified", "zeroResidueScope", "stageCleanupVerified",
|
||||
"rollbackPrestateCleanupVerified", "receipt", "terminalReceipt", "terminalExitCode",
|
||||
"terminalReceiptPublished", "deferredSignal", "independentVerifierVerified", "verifier"
|
||||
)
|
||||
$verifierRequired = @(
|
||||
"schemaVersion", "ok", "sourceRevision", "runId", "fileCount",
|
||||
"backupScriptFileCount", "backupHealthExporterIncluded", "remoteWritePerformed",
|
||||
"selfIdentityVerified"
|
||||
)
|
||||
$missingFieldRejections = 0
|
||||
Write-ReplayTrace "reject_missing_top"
|
||||
foreach ($field in $topLevelRequired) {
|
||||
$mutated = Copy-ReplayDocument $success
|
||||
$mutated.PSObject.Properties.Remove($field)
|
||||
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_$field"
|
||||
$missingFieldRejections++
|
||||
}
|
||||
Write-ReplayTrace "reject_missing_verifier"
|
||||
foreach ($field in $verifierRequired) {
|
||||
$mutated = Copy-ReplayDocument $success
|
||||
$mutated.verifier.PSObject.Properties.Remove($field)
|
||||
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_verifier_$field"
|
||||
$missingFieldRejections++
|
||||
}
|
||||
|
||||
$typeMutations = @(
|
||||
@{ field = "rollbackAttempted"; value = "false" },
|
||||
@{ field = "rollbackPerformed"; value = "false" },
|
||||
@{ field = "rollbackVerified"; value = "false" },
|
||||
@{ field = "terminalExitCode"; value = "0" },
|
||||
@{ field = "deferredSignal"; value = 0 }
|
||||
)
|
||||
$typeRejections = 0
|
||||
Write-ReplayTrace "reject_types"
|
||||
foreach ($mutation in $typeMutations) {
|
||||
$mutated = Copy-ReplayDocument $success
|
||||
$mutated.($mutation.field) = $mutation.value
|
||||
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_$($mutation.field)"
|
||||
$typeRejections++
|
||||
}
|
||||
$mutated = Copy-ReplayDocument $success
|
||||
$mutated.verifier.remoteWritePerformed = "false"
|
||||
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_verifier_remoteWritePerformed"
|
||||
$typeRejections++
|
||||
|
||||
$cleanupPending = Copy-ReplayDocument $success
|
||||
$cleanupPending.ok = $false
|
||||
$cleanupPending.status = "cleanup_pending"
|
||||
$cleanupPending.stageCleanupVerified = $false
|
||||
$cleanupPending.terminalExitCode = 75
|
||||
$cleanupPending.verifier = $null
|
||||
Write-ReplayTrace "accept_cleanup_pending"
|
||||
Get-Agent99CommittedFailureResult (New-ReplayTransport $cleanupPending 75) | Out-Null
|
||||
|
||||
$signalDeferred = Copy-ReplayDocument $success
|
||||
$signalDeferred.ok = $false
|
||||
$signalDeferred.status = "committed_signal_deferred"
|
||||
$signalDeferred.terminalExitCode = 143
|
||||
$signalDeferred.deferredSignal = "TERM"
|
||||
$signalDeferred.verifier = $null
|
||||
Write-ReplayTrace "accept_signal_deferred"
|
||||
Get-Agent99CommittedFailureResult (New-ReplayTransport $signalDeferred 143) | Out-Null
|
||||
|
||||
$terminalReceiptFailed = Copy-ReplayDocument $success
|
||||
$terminalReceiptFailed.ok = $false
|
||||
$terminalReceiptFailed.status = "terminal_receipt_failed"
|
||||
$terminalReceiptFailed.terminalReceiptPublished = $false
|
||||
$terminalReceiptFailed.terminalExitCode = 76
|
||||
$terminalReceiptFailed.verifier = $null
|
||||
Write-ReplayTrace "accept_terminal_receipt_failed"
|
||||
Get-Agent99CommittedFailureResult (New-ReplayTransport $terminalReceiptFailed 76) | Out-Null
|
||||
|
||||
Write-ReplayTrace "complete"
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_broker_contract_replay_v1"
|
||||
ok = $true
|
||||
parser = "WindowsPowerShell"
|
||||
successAccepted = $true
|
||||
cleanupPendingPreserved = $true
|
||||
committedSignalPreserved = $true
|
||||
terminalReceiptFailurePreserved = $true
|
||||
missingFieldRejections = $missingFieldRejections
|
||||
typeRejections = $typeRejections
|
||||
remoteWritePerformed = $false
|
||||
secretValuesRead = $false
|
||||
} | ConvertTo-Json -Compress
|
||||
@@ -1,997 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import grp
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import pwd
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
BROKER = ROOT / "agent99-host110-backup-runtime-broker.ps1"
|
||||
EXECUTOR = ROOT / "scripts" / "backup" / "host110-backup-runtime-executor.sh"
|
||||
VERIFIER = ROOT / "scripts" / "backup" / "verify-host110-backup-runtime.py"
|
||||
BROKER_CONTRACT_REPLAY = (
|
||||
ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-host110-broker-contract-replay.ps1"
|
||||
)
|
||||
|
||||
|
||||
EXPECTED_RUNTIME_FILES = {
|
||||
"common.sh",
|
||||
"backup-all.sh",
|
||||
"backup-host188-products.sh",
|
||||
"verify-host188-products-backup.sh",
|
||||
"verify-host188-products-archive.py",
|
||||
"backup-gitea.sh",
|
||||
"gitea-full-backup-restore-drill.sh",
|
||||
"backup-configs.sh",
|
||||
"backup-awoooi.sh",
|
||||
"backup-awoooi-frequent.sh",
|
||||
"backup-clawbot.sh",
|
||||
"backup-sentry.sh",
|
||||
"check-backup-integrity.sh",
|
||||
"sync-offsite-backups.sh",
|
||||
"backup-offsite-readiness-gate.sh",
|
||||
"verify-offsite-full-sync.sh",
|
||||
"enforce-latest-only-retention.sh",
|
||||
}
|
||||
|
||||
|
||||
def _load_verifier():
|
||||
spec = importlib.util.spec_from_file_location("host110_runtime_verifier", VERIFIER)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _powershell_string(source: str, variable: str) -> str:
|
||||
match = re.search(rf'\${re.escape(variable)}\s*=\s*"([^"]+)"', source)
|
||||
assert match is not None, variable
|
||||
return match.group(1)
|
||||
|
||||
|
||||
def _powershell_array(source: str, variable: str) -> tuple[str, ...]:
|
||||
match = re.search(
|
||||
rf'\${re.escape(variable)}\s*=\s*@\((.*?)\n\)',
|
||||
source,
|
||||
re.DOTALL,
|
||||
)
|
||||
assert match is not None, variable
|
||||
return tuple(re.findall(r'"([^"]+)"', match.group(1)))
|
||||
|
||||
|
||||
def _powershell_function(source: str, name: str) -> str:
|
||||
start = source.index(f"function {name} {{")
|
||||
next_function = source.find("\nfunction ", start + 1)
|
||||
return source[start:] if next_function < 0 else source[start:next_function]
|
||||
|
||||
|
||||
def _executor_payload_order(source: str) -> tuple[str, ...]:
|
||||
match = re.search(r"readonly -a RUNTIME_FILES=\((.*?)\n\)", source, re.DOTALL)
|
||||
assert match is not None
|
||||
runtime_files = tuple(re.findall(r"^\s+([A-Za-z0-9_.-]+)\s*$", match.group(1), re.MULTILINE))
|
||||
exporter = re.search(r'^readonly EXPORTER_FILE="([^"]+)"$', source, re.MULTILINE)
|
||||
assert exporter is not None
|
||||
return (*runtime_files, exporter.group(1))
|
||||
|
||||
|
||||
def _write_executable(path: Path, source: str) -> None:
|
||||
path.write_text(source, encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def _build_executor_harness(tmp_path: Path, run_id: str) -> dict[str, object]:
|
||||
destination = tmp_path / "backup-scripts"
|
||||
exporter_root = tmp_path / "exporter"
|
||||
status_root = tmp_path / "status"
|
||||
stage_root = tmp_path / "stage"
|
||||
rollback_root = tmp_path / "rollback"
|
||||
source_stage = tmp_path / f"source-{run_id}"
|
||||
lock_path = tmp_path / "runtime.lock"
|
||||
fake_bin = tmp_path / "fake-bin"
|
||||
for path in (destination, exporter_root, source_stage, fake_bin):
|
||||
path.mkdir(parents=True)
|
||||
|
||||
user_name = pwd.getpwuid(os.getuid()).pw_name
|
||||
group_name = grp.getgrgid(os.getgid()).gr_name
|
||||
shell_path = shutil.which("bash") or "/bin/bash"
|
||||
shell_version = subprocess.run(
|
||||
[shell_path, "-c", 'printf "%s" "${BASH_VERSINFO[0]:-0}"'],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout
|
||||
use_zsh_adapter = not shell_version.isdigit() or int(shell_version) < 4
|
||||
if use_zsh_adapter:
|
||||
shell_path = shutil.which("zsh") or "/bin/zsh"
|
||||
executor_source = EXECUTOR.read_text(encoding="utf-8")
|
||||
replacements = {
|
||||
'readonly EXPECTED_USER="wooo"': f'readonly EXPECTED_USER="{user_name}"',
|
||||
'readonly DEST_ROOT="/backup/scripts"': f'readonly DEST_ROOT="{destination}"',
|
||||
'readonly EXPORTER_ROOT="/home/wooo/scripts"': f'readonly EXPORTER_ROOT="{exporter_root}"',
|
||||
'readonly STATUS_ROOT="/backup/status"': f'readonly STATUS_ROOT="{status_root}"',
|
||||
'readonly STAGE_ROOT="/backup/.agent99-backup-runtime-stage"': f'readonly STAGE_ROOT="{stage_root}"',
|
||||
'readonly ROLLBACK_ROOT="/backup/.agent99-backup-runtime-rollback"': f'readonly ROLLBACK_ROOT="{rollback_root}"',
|
||||
'readonly LOCK_PATH="/tmp/agent99-host110-backup-runtime.lock"': f'readonly LOCK_PATH="{lock_path}"',
|
||||
'expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"': f'expected_source_stage="{source_stage}"',
|
||||
'"wooo:wooo:755"': f'"{user_name}:{group_name}:755"',
|
||||
'"wooo:wooo:700"': f'"{user_name}:{group_name}:700"',
|
||||
'"wooo:wooo"': f'"{user_name}:{group_name}"',
|
||||
}
|
||||
for before, after in replacements.items():
|
||||
assert before in executor_source
|
||||
executor_source = executor_source.replace(before, after)
|
||||
if use_zsh_adapter:
|
||||
executor_source = executor_source.replace(
|
||||
'"${!RUN_OWNED_TEMP_PATHS[@]}"',
|
||||
'"${(@k)RUN_OWNED_TEMP_PATHS}"',
|
||||
).replace(
|
||||
'RUN_OWNED_TEMP_PATHS["$1"]=1',
|
||||
'RUN_OWNED_TEMP_PATHS[$1]=1',
|
||||
).replace('${PAYLOAD_FILES[0]}', '${PAYLOAD_FILES[1]}')
|
||||
executor_path = source_stage / EXECUTOR.name
|
||||
_write_executable(executor_path, executor_source)
|
||||
|
||||
verifier_source = VERIFIER.read_text(encoding="utf-8")
|
||||
verifier_replacements = {
|
||||
'EXPECTED_DESTINATION = Path("/backup/scripts")': f"EXPECTED_DESTINATION = Path({str(destination)!r})",
|
||||
'EXPECTED_EXPORTER_DESTINATION = Path("/home/wooo/scripts/backup-health-textfile-exporter.py")': (
|
||||
f"EXPECTED_EXPORTER_DESTINATION = Path({str(exporter_root / 'backup-health-textfile-exporter.py')!r})"
|
||||
),
|
||||
'RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")': f"RUNTIME_LOCK = Path({str(lock_path)!r})",
|
||||
}
|
||||
for before, after in verifier_replacements.items():
|
||||
assert before in verifier_source
|
||||
verifier_source = verifier_source.replace(before, after)
|
||||
verifier_path = source_stage / VERIFIER.name
|
||||
_write_executable(verifier_path, verifier_source)
|
||||
|
||||
payload_order = _executor_payload_order(executor_source)
|
||||
rows = []
|
||||
before_state: dict[str, tuple[bytes, int]] = {}
|
||||
for name in payload_order:
|
||||
source = (
|
||||
ROOT / "scripts" / "ops" / name
|
||||
if name == "backup-health-textfile-exporter.py"
|
||||
else ROOT / "scripts" / "backup" / name
|
||||
)
|
||||
staged = source_stage / name
|
||||
shutil.copy2(source, staged)
|
||||
staged.chmod(0o755)
|
||||
rows.append({"name": name, "sha256": _sha256(staged)})
|
||||
target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name
|
||||
target.write_text(f"prestate:{name}\n", encoding="utf-8")
|
||||
target.chmod(0o755)
|
||||
before_state[name] = (target.read_bytes(), target.stat().st_mode & 0o777)
|
||||
|
||||
source_revision = "a" * 40
|
||||
manifest = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
|
||||
"sourceRevision": source_revision,
|
||||
"sourceHead": "b" * 40,
|
||||
"runId": run_id,
|
||||
"executorSha256": _sha256(executor_path),
|
||||
"verifierSha256": _sha256(verifier_path),
|
||||
"files": rows,
|
||||
}
|
||||
(source_stage / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
|
||||
_write_executable(fake_bin / "hostname", "#!/bin/sh\nprintf '%s\\n' '192.168.0.110'\n")
|
||||
_write_executable(fake_bin / "pgrep", "#!/bin/sh\nexit 1\n")
|
||||
_write_executable(fake_bin / "flock", "#!/bin/sh\nexit 0\n")
|
||||
_write_executable(fake_bin / "timeout", "#!/bin/sh\nexit 0\n")
|
||||
_write_executable(
|
||||
fake_bin / "readlink",
|
||||
"""#!/usr/bin/env python3
|
||||
import pathlib
|
||||
import sys
|
||||
if len(sys.argv) != 3 or sys.argv[1] != "-f":
|
||||
raise SystemExit(2)
|
||||
print(pathlib.Path(sys.argv[2]).resolve(strict=True))
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "stat",
|
||||
"""#!/usr/bin/env python3
|
||||
import grp
|
||||
import os
|
||||
import pwd
|
||||
import sys
|
||||
if len(sys.argv) != 4 or sys.argv[1] != "-c":
|
||||
raise SystemExit(2)
|
||||
row = os.stat(sys.argv[3])
|
||||
mode = format(row.st_mode & 0o777, "o")
|
||||
values = {
|
||||
"%U:%G:%a": f"{pwd.getpwuid(row.st_uid).pw_name}:{grp.getgrgid(row.st_gid).gr_name}:{mode}",
|
||||
"%U:%G": f"{pwd.getpwuid(row.st_uid).pw_name}:{grp.getgrgid(row.st_gid).gr_name}",
|
||||
"%u:%g:%a": f"{row.st_uid}:{row.st_gid}:{mode}",
|
||||
}
|
||||
if sys.argv[2] not in values:
|
||||
raise SystemExit(2)
|
||||
print(values[sys.argv[2]])
|
||||
""",
|
||||
)
|
||||
return {
|
||||
"executor": executor_path,
|
||||
"source_stage": source_stage,
|
||||
"source_revision": source_revision,
|
||||
"run_id": run_id,
|
||||
"destination": destination,
|
||||
"exporter_root": exporter_root,
|
||||
"status_root": status_root,
|
||||
"stage_root": stage_root,
|
||||
"rollback_root": rollback_root,
|
||||
"fake_bin": fake_bin,
|
||||
"shell": shell_path,
|
||||
"use_zsh_adapter": use_zsh_adapter,
|
||||
"payload_order": payload_order,
|
||||
"before_state": before_state,
|
||||
}
|
||||
|
||||
|
||||
def _run_executor_harness(harness: dict[str, object], **extra_env: str) -> subprocess.CompletedProcess[str]:
|
||||
environment = os.environ.copy()
|
||||
for name in (
|
||||
"HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT",
|
||||
"HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT",
|
||||
"HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT",
|
||||
"HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL",
|
||||
"HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL",
|
||||
"HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT",
|
||||
"HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT",
|
||||
):
|
||||
environment.pop(name, None)
|
||||
environment["HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS"] = "1"
|
||||
environment.update(extra_env)
|
||||
environment["PATH"] = f"{harness['fake_bin']}:{environment['PATH']}"
|
||||
command = [str(harness["shell"])]
|
||||
if harness["use_zsh_adapter"]:
|
||||
environment["ZDOTDIR"] = str(harness["fake_bin"])
|
||||
command.append("-f")
|
||||
command.extend(
|
||||
[
|
||||
str(harness["executor"]),
|
||||
"--mode",
|
||||
"apply",
|
||||
"--source-revision",
|
||||
str(harness["source_revision"]),
|
||||
"--run-id",
|
||||
str(harness["run_id"]),
|
||||
"--source-stage",
|
||||
str(harness["source_stage"]),
|
||||
]
|
||||
)
|
||||
return subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
)
|
||||
|
||||
|
||||
def _assert_prestate_restored(harness: dict[str, object]) -> None:
|
||||
destination = Path(harness["destination"])
|
||||
exporter_root = Path(harness["exporter_root"])
|
||||
before_state = harness["before_state"]
|
||||
assert isinstance(before_state, dict)
|
||||
for name in harness["payload_order"]:
|
||||
target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name
|
||||
assert (target.read_bytes(), target.stat().st_mode & 0o777) == before_state[name]
|
||||
|
||||
|
||||
def _assert_payload_committed(harness: dict[str, object]) -> None:
|
||||
destination = Path(harness["destination"])
|
||||
exporter_root = Path(harness["exporter_root"])
|
||||
source_stage = Path(harness["source_stage"])
|
||||
for name in harness["payload_order"]:
|
||||
target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name
|
||||
assert target.read_bytes() == (source_stage / name).read_bytes()
|
||||
assert target.stat().st_mode & 0o777 == 0o755
|
||||
|
||||
|
||||
def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
|
||||
assert '[ValidateSet("Check", "Apply", "Verify")]' in source
|
||||
assert '$TargetHost = "192.168.0.110"' in source
|
||||
assert '$TargetUser = "wooo"' in source
|
||||
assert '$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"' in source
|
||||
assert 'keys\\agent99_ed25519' in source
|
||||
assert '"credential.helper="' in source
|
||||
assert '$env:GIT_TERMINAL_PROMPT = "0"' in source
|
||||
assert '"merge-base", "--is-ancestor", $SourceRevision, $sourceHead' in source
|
||||
assert '"checkout", "--quiet", $SourceRevision, "--"' in source
|
||||
assert 'executorSha256 = $executorDigest' in source
|
||||
assert 'verifierSha256 = $verifierDigest' in source
|
||||
assert '"scripts/ops/backup-health-textfile-exporter.py"' in source
|
||||
assert "/home/wooo/awoooi" not in source
|
||||
assert '"StrictHostKeyChecking=yes"' in source
|
||||
assert '"NumberOfPasswordPrompts=0"' in source
|
||||
assert '"ConnectionAttempts=1"' in source
|
||||
assert "$process.WaitForExit($TimeoutSeconds * 1000)" in source
|
||||
assert "taskkill.exe /PID $process.Id /T /F" in source
|
||||
assert "$process.Kill()" in source
|
||||
assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths' in source
|
||||
assert '[string]$parsed.status -ne "verified"' in source
|
||||
assert '-not [bool]$parsed.payloadCommitted' in source
|
||||
assert '-not [bool]$parsed.stageCleanupVerified' in source
|
||||
assert '-not [bool]$parsed.rollbackPrestateCleanupVerified' in source
|
||||
assert '[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps"' in source
|
||||
assert '[int64]$parsed.terminalExitCode -ne 0' in source
|
||||
assert 'agent99-backup-runtime-terminal-$RunId.json' in source
|
||||
assert '$property = $Object.PSObject.Properties[$Name]' in source
|
||||
assert 'if ($null -eq $property) { return $false }' in source
|
||||
assert 'Get-Agent99CommittedFailureResult $applyTransport' in source
|
||||
assert 'throw "host110_apply_$([string]$apply.status)"' in source
|
||||
|
||||
|
||||
def test_broker_required_field_contract_rejects_missing_fields_before_cast() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
field_guard = _powershell_function(source, "Test-Agent99JsonField")
|
||||
common_contract = _powershell_function(source, "Convert-Agent99ExecutorDocument")
|
||||
apply_contract = _powershell_function(source, "Assert-Agent99ApplyResultFields")
|
||||
verifier_contract = _powershell_function(source, "Assert-Agent99VerifierFields")
|
||||
committed_failure = _powershell_function(source, "Get-Agent99CommittedFailureResult")
|
||||
|
||||
assert '$property = $Object.PSObject.Properties[$Name]' in field_guard
|
||||
assert 'if ($null -eq $property) { return $false }' in field_guard
|
||||
for field in (
|
||||
"schemaVersion",
|
||||
"mode",
|
||||
"ok",
|
||||
"sourceRevision",
|
||||
"sourceHead",
|
||||
"runId",
|
||||
"fileCount",
|
||||
"backupScriptFileCount",
|
||||
"backupHealthExporterIncluded",
|
||||
):
|
||||
assert re.search(rf"^\s+{field} = \"", common_contract, re.MULTILINE), field
|
||||
for field in (
|
||||
"status",
|
||||
"payloadCommitted",
|
||||
"rollbackAttempted",
|
||||
"rollbackPerformed",
|
||||
"rollbackVerified",
|
||||
"zeroResidueVerified",
|
||||
"zeroResidueScope",
|
||||
"stageCleanupVerified",
|
||||
"rollbackPrestateCleanupVerified",
|
||||
"receipt",
|
||||
"terminalReceipt",
|
||||
"terminalReceiptPublished",
|
||||
"terminalExitCode",
|
||||
"deferredSignal",
|
||||
"independentVerifierVerified",
|
||||
"verifier",
|
||||
):
|
||||
assert re.search(rf"^\s+{field} = ", apply_contract, re.MULTILINE), field
|
||||
for field in (
|
||||
"schemaVersion",
|
||||
"ok",
|
||||
"sourceRevision",
|
||||
"runId",
|
||||
"fileCount",
|
||||
"backupScriptFileCount",
|
||||
"backupHealthExporterIncluded",
|
||||
"remoteWritePerformed",
|
||||
"selfIdentityVerified",
|
||||
):
|
||||
assert re.search(rf"^\s+{field} = \"", verifier_contract, re.MULTILINE), field
|
||||
assert 'status -eq "cleanup_pending"' in committed_failure
|
||||
assert 'status -eq "committed_signal_deferred"' in committed_failure
|
||||
assert 'verifier = $VerifierKind' in apply_contract
|
||||
|
||||
|
||||
def test_windows99_contract_replay_uses_actual_broker_functions_without_remote_write() -> None:
|
||||
source = BROKER_CONTRACT_REPLAY.read_text(encoding="utf-8")
|
||||
|
||||
assert "FunctionDefinitionAst" in source
|
||||
for function_name in (
|
||||
"Test-Agent99JsonField",
|
||||
"Assert-Agent99JsonFields",
|
||||
"Convert-Agent99ExecutorDocument",
|
||||
"Assert-Agent99ApplyResultFields",
|
||||
"Assert-Agent99VerifierFields",
|
||||
"Get-Agent99ExecutorResult",
|
||||
"Get-Agent99CommittedFailureResult",
|
||||
):
|
||||
assert f'"{function_name}"' in source
|
||||
assert "$mutated.PSObject.Properties.Remove($field)" in source
|
||||
assert "$mutated.verifier.PSObject.Properties.Remove($field)" in source
|
||||
assert 'status = "cleanup_pending"' in source
|
||||
assert 'status = "committed_signal_deferred"' in source
|
||||
assert 'status = "terminal_receipt_failed"' in source
|
||||
assert "remoteWritePerformed = $false" in source
|
||||
assert "secretValuesRead = $false" in source
|
||||
for forbidden in ("New-Item", "Set-Content", "WriteAllText", "ssh.exe", "scp.exe"):
|
||||
assert forbidden not in source
|
||||
|
||||
|
||||
def test_broker_scp_basenames_match_executor_source_stage_contract() -> None:
|
||||
broker = BROKER.read_text(encoding="utf-8")
|
||||
executor = EXECUTOR.read_text(encoding="utf-8")
|
||||
|
||||
runtime_paths = _powershell_array(broker, "RuntimeRelativePaths")
|
||||
broker_payload_basenames = tuple(Path(path).name for path in runtime_paths)
|
||||
executor_payload_basenames = _executor_payload_order(executor)
|
||||
executor_path = _powershell_string(broker, "ExecutorRelativePath")
|
||||
verifier_path = _powershell_string(broker, "VerifierRelativePath")
|
||||
manifest_match = re.search(
|
||||
r'\$manifestPath\s*=\s*Join-Path\s+\$sourceRoot\s+"([^"]+)"',
|
||||
broker,
|
||||
)
|
||||
assert manifest_match is not None
|
||||
manifest_name = manifest_match.group(1)
|
||||
|
||||
assert len(broker_payload_basenames) == 18
|
||||
assert len(executor_payload_basenames) == 18
|
||||
assert set(broker_payload_basenames) == set(executor_payload_basenames)
|
||||
assert broker_payload_basenames == executor_payload_basenames
|
||||
staged_basenames = (
|
||||
*broker_payload_basenames,
|
||||
Path(executor_path).name,
|
||||
Path(verifier_path).name,
|
||||
manifest_name,
|
||||
)
|
||||
assert len(staged_basenames) == 21
|
||||
assert len(set(staged_basenames)) == 21
|
||||
assert manifest_name == "manifest.json"
|
||||
assert '$localPaths += $executorPath, $verifierPath, $ManifestPath' in broker
|
||||
assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths $remoteStage' in broker
|
||||
assert f'$SOURCE_STAGE/{manifest_name}' in executor
|
||||
assert f'$SOURCE_STAGE/{Path(verifier_path).name}' in executor
|
||||
assert Path(executor_path).name == "host110-backup-runtime-executor.sh"
|
||||
|
||||
|
||||
def test_windows99_broker_verify_is_independent_no_write_and_evidence_is_immutable() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
|
||||
assert 'RedirectStandardInput = $StandardInputPath' in source
|
||||
assert "python3 - --manifest-base64" in source
|
||||
assert 'remoteWritePerformed' in source
|
||||
assert '[IO.File]::Move($temporary, $Path)' in source
|
||||
assert 'Move-Item -LiteralPath $temporary -Destination $Path -Force' not in source
|
||||
assert 'run_identity_conflict' in source
|
||||
assert 'run_identity_evidence_exists' in source
|
||||
assert 'decisionProvider = "deterministic_only"' in source
|
||||
assert 'criticProvider = "deterministic_only"' in source
|
||||
assert 'verifier = "independent_host110_backup_runtime_python_readback"' in source
|
||||
|
||||
|
||||
def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
|
||||
source = EXECUTOR.read_text(encoding="utf-8")
|
||||
array_body = source.split("readonly -a RUNTIME_FILES=(", 1)[1].split(")", 1)[0]
|
||||
runtime_files = set(re.findall(r"^\s+([A-Za-z0-9_.-]+)\s*$", array_body, re.MULTILINE))
|
||||
|
||||
assert runtime_files == EXPECTED_RUNTIME_FILES
|
||||
assert 'readonly EXPECTED_HOST_IP="192.168.0.110"' in source
|
||||
assert 'readonly EXPECTED_USER="wooo"' in source
|
||||
assert 'readonly DEST_ROOT="/backup/scripts"' in source
|
||||
assert 'readonly EXPORTER_ROOT="/home/wooo/scripts"' in source
|
||||
assert 'readonly EXPORTER_FILE="backup-health-textfile-exporter.py"' in source
|
||||
assert 'readonly -a PAYLOAD_FILES=("${RUNTIME_FILES[@]}" "$EXPORTER_FILE")' in source
|
||||
assert '--source-stage' in source
|
||||
assert 'document.get("executorSha256"' in source
|
||||
assert 'document.get("verifierSha256"' in source
|
||||
assert 'executor_identity_failed' in source
|
||||
assert 'verifier_identity_failed' in source
|
||||
assert 'flock -x -w 30 9' in source
|
||||
assert 'legacy_backup_runtime_active' in source
|
||||
assert source.index("for name in \"${PAYLOAD_FILES[@]}\"") < source.index("mv -f -- \"$temporary\"")
|
||||
assert 'rollback_unverified' in source
|
||||
assert 'failed_rolled_back' in source
|
||||
assert 'ROLLBACK_ATTEMPTED=1' in source
|
||||
assert 'ROLLBACK_PERFORMED=1' in source
|
||||
assert '"rollbackAttempted": rollback_attempted' in source
|
||||
assert '"rollbackPerformed": rollback_performed' in source
|
||||
assert '"rollbackVerified": rollback_verified' in source
|
||||
assert '"zeroResidueVerified": zero_residue_verified' in source
|
||||
assert '"zeroResidueScope": "run_owned_destination_temps"' in source
|
||||
assert 'cleanup_run_owned_temps' in source
|
||||
assert 'verify_run_owned_temp_absence' in source
|
||||
assert 'HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT' in source
|
||||
assert 'HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT' in source
|
||||
assert 'HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT' in source
|
||||
assert 'HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL' in source
|
||||
assert 'HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL' in source
|
||||
assert 'HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT' in source
|
||||
assert 'HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT' in source
|
||||
assert 'HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS' in source
|
||||
assert '[ "$DEST_ROOT" != "/backup/scripts" ]' in source
|
||||
assert 'fsync_rollback_prestate || return 75' in source
|
||||
assert 'fsync_destination_state || fail "post_apply_durability_failed"' in source
|
||||
assert 'working_digest "$dest_path"' in source
|
||||
assert 'PREVIOUS_METADATA' in source
|
||||
assert "stat -c '%u:%g:%a'" in source
|
||||
assert 'os.link(temporary, path)' in source
|
||||
assert 'os.fsync(handle.fileno())' in source
|
||||
assert 'os.fsync(directory_fd)' in source
|
||||
assert 'hashlib.sha256(readback).hexdigest()' in source
|
||||
assert source.index("COMMIT_CRITICAL=1") < source.index('write_receipt "payload_verified"')
|
||||
assert source.index('write_receipt "payload_verified"') < source.index("\nAPPLY_COMPLETE=1")
|
||||
assert source.index("\nAPPLY_COMPLETE=1") < source.rindex("\nCOMMIT_CRITICAL=0")
|
||||
assert source.index("\nAPPLY_COMPLETE=1") < source.rindex(
|
||||
'cleanup_postcommit_directory "rollback_prestate"'
|
||||
)
|
||||
assert 'handle_agent99_signal HUP 129' in source
|
||||
assert 'handle_agent99_signal TERM 143' in source
|
||||
assert source.index("trap '' HUP INT TERM") < source.index('write_terminal_receipt "$terminal_status"')
|
||||
assert 'write_terminal_receipt "$terminal_status"' in source
|
||||
assert 'terminal_status="cleanup_pending"' in source
|
||||
assert 'terminal_status="committed_signal_deferred"' in source
|
||||
assert 'run_identity_receipt_exists' in source
|
||||
assert 'run_identity_stage_exists' in source
|
||||
assert 'run_identity_rollback_exists' in source
|
||||
assert 'verify-host110-backup-runtime.py' in source
|
||||
assert 'selfIdentityVerified' in source
|
||||
assert '"backupScriptFileCount": 17' in source
|
||||
assert '"backupHealthExporterIncluded": True' in source
|
||||
assert '"productionServiceRestarted": False' in source
|
||||
assert '"secretValuesRead": False' in source
|
||||
assert "sudo " not in source
|
||||
assert "docker " not in source
|
||||
assert "systemctl " not in source
|
||||
|
||||
|
||||
def test_apply_success_publishes_durable_receipt_before_prestate_cleanup(tmp_path: Path) -> None:
|
||||
run_id = "apply-success"
|
||||
harness = _build_executor_harness(tmp_path, run_id)
|
||||
|
||||
result = _run_executor_harness(harness)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
output = json.loads(result.stdout)
|
||||
assert output["ok"] is True
|
||||
assert output["rollbackAttempted"] is False
|
||||
assert output["rollbackPerformed"] is False
|
||||
assert output["rollbackVerified"] is False
|
||||
assert output["zeroResidueVerified"] is True
|
||||
assert output["stageCleanupVerified"] is True
|
||||
assert output["rollbackPrestateCleanupVerified"] is True
|
||||
assert output["status"] == "verified"
|
||||
assert output["payloadCommitted"] is True
|
||||
assert output["runId"] == run_id
|
||||
assert output["terminalExitCode"] == 0
|
||||
assert output["terminalReceiptPublished"] is True
|
||||
assert output["deferredSignal"] is None
|
||||
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
|
||||
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
|
||||
assert receipt["status"] == "payload_verified"
|
||||
assert receipt["rollbackAttempted"] is False
|
||||
assert receipt["rollbackPerformed"] is False
|
||||
assert receipt["rollbackVerified"] is False
|
||||
assert receipt["zeroResidueVerified"] is True
|
||||
assert receipt["zeroResidueScope"] == "run_owned_destination_temps"
|
||||
terminal_path = Path(harness["status_root"]) / f"agent99-backup-runtime-terminal-{run_id}.json"
|
||||
terminal = json.loads(terminal_path.read_text(encoding="utf-8"))
|
||||
assert terminal["status"] == "verified"
|
||||
assert terminal["ok"] is True
|
||||
assert terminal["payloadCommitted"] is True
|
||||
assert terminal["stageCleanupVerified"] is True
|
||||
assert terminal["rollbackPrestateCleanupVerified"] is True
|
||||
assert terminal["terminalExitCode"] == 0
|
||||
assert terminal["zeroResidueScope"] == "run_owned_destination_temps_and_internal_stage_prestate"
|
||||
assert output["terminalReceipt"] == str(terminal_path)
|
||||
assert not (Path(harness["rollback_root"]) / run_id).exists()
|
||||
assert not (Path(harness["stage_root"]) / run_id).exists()
|
||||
_assert_payload_committed(harness)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("signal_name", "exit_code"), [("TERM", 143), ("HUP", 129), ("INT", 130)])
|
||||
def test_commit_window_signal_is_deferred_until_payload_and_terminal_are_consistent(
|
||||
tmp_path: Path,
|
||||
signal_name: str,
|
||||
exit_code: int,
|
||||
) -> None:
|
||||
run_id = f"commit-signal-{signal_name.lower()}"
|
||||
harness = _build_executor_harness(tmp_path, run_id)
|
||||
|
||||
result = _run_executor_harness(
|
||||
harness,
|
||||
HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL=signal_name,
|
||||
)
|
||||
|
||||
assert result.returncode == exit_code, result.stderr
|
||||
output = json.loads(result.stdout)
|
||||
assert output["ok"] is False
|
||||
assert output["status"] == "committed_signal_deferred"
|
||||
assert output["payloadCommitted"] is True
|
||||
assert output["runId"] == run_id
|
||||
assert output["deferredSignal"] == signal_name
|
||||
assert output["terminalExitCode"] == exit_code
|
||||
assert output["terminalReceiptPublished"] is True
|
||||
assert output["stageCleanupVerified"] is True
|
||||
assert output["rollbackPrestateCleanupVerified"] is True
|
||||
_assert_payload_committed(harness)
|
||||
payload_receipt = json.loads(
|
||||
(
|
||||
Path(harness["status_root"])
|
||||
/ f"agent99-backup-runtime-deploy-{run_id}.json"
|
||||
).read_text(encoding="utf-8")
|
||||
)
|
||||
assert payload_receipt["status"] == "payload_verified"
|
||||
terminal = json.loads(
|
||||
(
|
||||
Path(harness["status_root"])
|
||||
/ f"agent99-backup-runtime-terminal-{run_id}.json"
|
||||
).read_text(encoding="utf-8")
|
||||
)
|
||||
assert terminal["status"] == "committed_signal_deferred"
|
||||
assert terminal["ok"] is False
|
||||
assert terminal["deferredSignal"] == signal_name
|
||||
assert terminal["terminalExitCode"] == exit_code
|
||||
assert not (Path(harness["rollback_root"]) / run_id).exists()
|
||||
assert not (Path(harness["stage_root"]) / run_id).exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("signal_name", ["TERM", "HUP", "INT"])
|
||||
def test_terminal_publication_masks_late_signal_without_receipt_stdout_exit_divergence(
|
||||
tmp_path: Path,
|
||||
signal_name: str,
|
||||
) -> None:
|
||||
run_id = f"terminal-signal-{signal_name.lower()}"
|
||||
harness = _build_executor_harness(tmp_path, run_id)
|
||||
|
||||
result = _run_executor_harness(
|
||||
harness,
|
||||
HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL=signal_name,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
output = json.loads(result.stdout)
|
||||
assert output["ok"] is True
|
||||
assert output["status"] == "verified"
|
||||
assert output["payloadCommitted"] is True
|
||||
assert output["runId"] == run_id
|
||||
assert output["deferredSignal"] is None
|
||||
assert output["terminalExitCode"] == 0
|
||||
assert output["terminalReceiptPublished"] is True
|
||||
terminal = json.loads(
|
||||
(
|
||||
Path(harness["status_root"])
|
||||
/ f"agent99-backup-runtime-terminal-{run_id}.json"
|
||||
).read_text(encoding="utf-8")
|
||||
)
|
||||
assert terminal["ok"] is True
|
||||
assert terminal["status"] == output["status"]
|
||||
assert terminal["deferredSignal"] == output["deferredSignal"]
|
||||
assert terminal["terminalExitCode"] == output["terminalExitCode"]
|
||||
_assert_payload_committed(harness)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fault", ["write", "link", "fsync", "readback"])
|
||||
def test_terminal_receipt_fault_is_nonzero_with_payload_receipt_only(
|
||||
tmp_path: Path,
|
||||
fault: str,
|
||||
) -> None:
|
||||
run_id = f"terminal-receipt-{fault}"
|
||||
harness = _build_executor_harness(tmp_path, run_id)
|
||||
|
||||
result = _run_executor_harness(
|
||||
harness,
|
||||
HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT=fault,
|
||||
)
|
||||
|
||||
assert result.returncode == 76
|
||||
assert '"ok": true' not in result.stdout.lower()
|
||||
assert "terminal_receipt_failed" in result.stderr
|
||||
output = json.loads(result.stdout)
|
||||
assert output["ok"] is False
|
||||
assert output["status"] == "terminal_receipt_failed"
|
||||
assert output["payloadCommitted"] is True
|
||||
assert output["runId"] == run_id
|
||||
assert output["terminalReceiptPublished"] is False
|
||||
assert output["terminalExitCode"] == 76
|
||||
_assert_payload_committed(harness)
|
||||
payload_receipt = json.loads(
|
||||
(
|
||||
Path(harness["status_root"])
|
||||
/ f"agent99-backup-runtime-deploy-{run_id}.json"
|
||||
).read_text(encoding="utf-8")
|
||||
)
|
||||
assert payload_receipt["status"] == "payload_verified"
|
||||
assert not (
|
||||
Path(harness["status_root"])
|
||||
/ f"agent99-backup-runtime-terminal-{run_id}.json"
|
||||
).exists()
|
||||
assert not (Path(harness["rollback_root"]) / run_id).exists()
|
||||
assert not (Path(harness["stage_root"]) / run_id).exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("fault", "stage_clean", "rollback_clean"),
|
||||
[("stage", False, True), ("rollback_prestate", True, False)],
|
||||
)
|
||||
def test_postcommit_cleanup_failure_is_nonzero_and_never_false_green(
|
||||
tmp_path: Path,
|
||||
fault: str,
|
||||
stage_clean: bool,
|
||||
rollback_clean: bool,
|
||||
) -> None:
|
||||
run_id = f"cleanup-{fault}"
|
||||
harness = _build_executor_harness(tmp_path, run_id)
|
||||
|
||||
result = _run_executor_harness(
|
||||
harness,
|
||||
HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT=fault,
|
||||
)
|
||||
|
||||
assert result.returncode == 75, result.stderr
|
||||
assert '"ok": true' not in result.stdout.lower()
|
||||
output = json.loads(result.stdout)
|
||||
assert output["ok"] is False
|
||||
assert output["status"] == "cleanup_pending"
|
||||
assert output["payloadCommitted"] is True
|
||||
assert output["runId"] == run_id
|
||||
assert output["stageCleanupVerified"] is stage_clean
|
||||
assert output["rollbackPrestateCleanupVerified"] is rollback_clean
|
||||
assert output["terminalReceiptPublished"] is True
|
||||
_assert_payload_committed(harness)
|
||||
payload_receipt = json.loads(
|
||||
(
|
||||
Path(harness["status_root"])
|
||||
/ f"agent99-backup-runtime-deploy-{run_id}.json"
|
||||
).read_text(encoding="utf-8")
|
||||
)
|
||||
assert payload_receipt["status"] == "payload_verified"
|
||||
terminal = json.loads(
|
||||
(
|
||||
Path(harness["status_root"])
|
||||
/ f"agent99-backup-runtime-terminal-{run_id}.json"
|
||||
).read_text(encoding="utf-8")
|
||||
)
|
||||
assert terminal["status"] == "cleanup_pending"
|
||||
assert terminal["ok"] is False
|
||||
assert terminal["payloadCommitted"] is True
|
||||
assert terminal["stageCleanupVerified"] is stage_clean
|
||||
assert terminal["rollbackPrestateCleanupVerified"] is rollback_clean
|
||||
assert (Path(harness["stage_root"]) / run_id).exists() is (not stage_clean)
|
||||
assert (Path(harness["rollback_root"]) / run_id).exists() is (not rollback_clean)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fault", ["write", "link", "fsync", "readback"])
|
||||
def test_receipt_fault_rolls_back_without_false_success_or_premature_prestate_cleanup(
|
||||
tmp_path: Path,
|
||||
fault: str,
|
||||
) -> None:
|
||||
run_id = f"receipt-{fault}"
|
||||
harness = _build_executor_harness(tmp_path, run_id)
|
||||
|
||||
result = _run_executor_harness(
|
||||
harness,
|
||||
HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT=fault,
|
||||
)
|
||||
assert result.returncode != 0
|
||||
assert '"ok": true' not in result.stdout.lower()
|
||||
assert "durable_receipt_failed" in result.stderr
|
||||
_assert_prestate_restored(harness)
|
||||
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
|
||||
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
|
||||
assert receipt["status"] == "failed_rolled_back"
|
||||
assert receipt["rollbackAttempted"] is True
|
||||
assert receipt["rollbackPerformed"] is True
|
||||
assert receipt["rollbackVerified"] is True
|
||||
assert receipt["zeroResidueVerified"] is True
|
||||
assert not (Path(harness["rollback_root"]) / run_id).exists()
|
||||
assert not (Path(harness["stage_root"]) / run_id).exists()
|
||||
for parent in (Path(harness["destination"]), Path(harness["exporter_root"])):
|
||||
assert not list(parent.glob(f".*.agent99-*{run_id}"))
|
||||
|
||||
|
||||
def test_apply_temp_fault_is_removed_and_zero_residue_is_verified(tmp_path: Path) -> None:
|
||||
run_id = "apply-temp-cleanup"
|
||||
harness = _build_executor_harness(tmp_path, run_id)
|
||||
|
||||
result = _run_executor_harness(
|
||||
harness,
|
||||
HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT="after_temp_install",
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert '"ok": true' not in result.stdout.lower()
|
||||
assert "fault_injected_after_temp_install" in result.stderr
|
||||
_assert_prestate_restored(harness)
|
||||
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
|
||||
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
|
||||
assert receipt["status"] == "failed_rolled_back"
|
||||
assert receipt["rollbackAttempted"] is True
|
||||
assert receipt["rollbackPerformed"] is True
|
||||
assert receipt["rollbackVerified"] is True
|
||||
assert receipt["zeroResidueVerified"] is True
|
||||
for parent in (Path(harness["destination"]), Path(harness["exporter_root"])):
|
||||
assert not list(parent.glob(f".*.agent99-*{run_id}"))
|
||||
|
||||
|
||||
def test_temp_cleanup_fault_is_rollback_unverified_and_preserves_prestate(tmp_path: Path) -> None:
|
||||
run_id = "temp-residue-unverified"
|
||||
harness = _build_executor_harness(tmp_path, run_id)
|
||||
|
||||
result = _run_executor_harness(
|
||||
harness,
|
||||
HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT="after_temp_install",
|
||||
HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT="preserve_first_temp",
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert '"ok": true' not in result.stdout.lower()
|
||||
_assert_prestate_restored(harness)
|
||||
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
|
||||
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
|
||||
assert receipt["status"] == "rollback_unverified"
|
||||
assert receipt["rollbackAttempted"] is True
|
||||
assert receipt["rollbackPerformed"] is True
|
||||
assert receipt["rollbackVerified"] is False
|
||||
assert receipt["zeroResidueVerified"] is False
|
||||
assert (Path(harness["rollback_root"]) / run_id / "prestate.tsv").is_file()
|
||||
residue = []
|
||||
for parent in (Path(harness["destination"]), Path(harness["exporter_root"])):
|
||||
residue.extend(parent.glob(f".*.agent99-*{run_id}"))
|
||||
assert residue
|
||||
|
||||
|
||||
def test_backup_runtime_entrypoints_share_the_deployment_gate() -> None:
|
||||
common = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8")
|
||||
restore = (ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh").read_text(encoding="utf-8")
|
||||
offsite = (ROOT / "scripts/backup/backup-offsite-readiness-gate.sh").read_text(encoding="utf-8")
|
||||
archive = (ROOT / "scripts/backup/verify-host188-products-archive.py").read_text(encoding="utf-8")
|
||||
|
||||
for source in (common, restore, offsite):
|
||||
assert "/tmp/agent99-host110-backup-runtime.lock" in source
|
||||
assert "flock -s -w 60" in source
|
||||
assert "fcntl.LOCK_SH | fcntl.LOCK_NB" in archive
|
||||
assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" in archive
|
||||
|
||||
|
||||
def test_independent_verifier_accepts_exact_bundle_without_writing(tmp_path, monkeypatch, capsys) -> None:
|
||||
verifier = _load_verifier()
|
||||
destination = tmp_path / "scripts"
|
||||
destination.mkdir()
|
||||
exporter_destination = tmp_path / verifier.EXPORTER_FILE
|
||||
rows = []
|
||||
for name in verifier.EXPECTED_BACKUP_FILES:
|
||||
path = destination / name
|
||||
path.write_text(f"runtime:{name}\n", encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
rows.append({"name": name, "sha256": _sha256(path)})
|
||||
exporter_destination.write_text("exporter\n", encoding="utf-8")
|
||||
exporter_destination.chmod(0o755)
|
||||
rows.append({"name": verifier.EXPORTER_FILE, "sha256": _sha256(exporter_destination)})
|
||||
verifier_digest = _sha256(VERIFIER)
|
||||
source_revision = "a" * 40
|
||||
run_id = "unit-exact"
|
||||
manifest = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
|
||||
"sourceRevision": source_revision,
|
||||
"sourceHead": "b" * 40,
|
||||
"runId": run_id,
|
||||
"executorSha256": "c" * 64,
|
||||
"verifierSha256": verifier_digest,
|
||||
"files": rows,
|
||||
}
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
before = {
|
||||
path.name: (path.stat().st_mtime_ns, _sha256(path))
|
||||
for path in [*destination.iterdir(), exporter_destination]
|
||||
}
|
||||
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
|
||||
monkeypatch.setattr(verifier, "EXPECTED_EXPORTER_DESTINATION", exporter_destination)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
str(VERIFIER),
|
||||
"--manifest",
|
||||
str(manifest_path),
|
||||
"--destination",
|
||||
str(destination),
|
||||
"--exporter-destination",
|
||||
str(exporter_destination),
|
||||
"--source-revision",
|
||||
source_revision,
|
||||
"--run-id",
|
||||
run_id,
|
||||
"--expected-verifier-sha256",
|
||||
verifier_digest,
|
||||
],
|
||||
)
|
||||
|
||||
verifier.main()
|
||||
|
||||
result = json.loads(capsys.readouterr().out)
|
||||
after = {
|
||||
path.name: (path.stat().st_mtime_ns, _sha256(path))
|
||||
for path in [*destination.iterdir(), exporter_destination]
|
||||
}
|
||||
assert result["ok"] is True
|
||||
assert result["fileCount"] == 18
|
||||
assert result["backupScriptFileCount"] == 17
|
||||
assert result["backupHealthExporterIncluded"] is True
|
||||
assert result["remoteWritePerformed"] is False
|
||||
assert result["selfIdentityVerified"] is True
|
||||
assert before == after
|
||||
|
||||
|
||||
def test_independent_verifier_rejects_drifted_bundle(tmp_path, monkeypatch, capsys) -> None:
|
||||
verifier = _load_verifier()
|
||||
destination = tmp_path / "scripts"
|
||||
destination.mkdir()
|
||||
exporter_destination = tmp_path / verifier.EXPORTER_FILE
|
||||
rows = []
|
||||
for name in verifier.EXPECTED_BACKUP_FILES:
|
||||
path = destination / name
|
||||
path.write_text(f"runtime:{name}\n", encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
rows.append({"name": name, "sha256": _sha256(path)})
|
||||
exporter_destination.write_text("exporter\n", encoding="utf-8")
|
||||
exporter_destination.chmod(0o755)
|
||||
rows.append({"name": verifier.EXPORTER_FILE, "sha256": _sha256(exporter_destination)})
|
||||
(destination / verifier.EXPECTED_BACKUP_FILES[0]).write_text("drifted\n", encoding="utf-8")
|
||||
verifier_digest = _sha256(VERIFIER)
|
||||
source_revision = "d" * 40
|
||||
run_id = "unit-drift"
|
||||
manifest = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
|
||||
"sourceRevision": source_revision,
|
||||
"sourceHead": "e" * 40,
|
||||
"runId": run_id,
|
||||
"executorSha256": "f" * 64,
|
||||
"verifierSha256": verifier_digest,
|
||||
"files": rows,
|
||||
}
|
||||
encoded = base64.b64encode(json.dumps(manifest).encode()).decode()
|
||||
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
|
||||
monkeypatch.setattr(verifier, "EXPECTED_EXPORTER_DESTINATION", exporter_destination)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
str(VERIFIER),
|
||||
"--manifest-base64",
|
||||
encoded,
|
||||
"--destination",
|
||||
str(destination),
|
||||
"--exporter-destination",
|
||||
str(exporter_destination),
|
||||
"--source-revision",
|
||||
source_revision,
|
||||
"--run-id",
|
||||
run_id,
|
||||
"--expected-verifier-sha256",
|
||||
verifier_digest,
|
||||
],
|
||||
)
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
verifier.main()
|
||||
assert "runtime_file_identity_failed" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_agent99_runtime_lists_include_the_host110_broker_once() -> None:
|
||||
sources = [
|
||||
ROOT / "agent99-bootstrap.ps1",
|
||||
ROOT / "agent99-contract-check.ps1",
|
||||
ROOT / "agent99-deploy.ps1",
|
||||
ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh",
|
||||
ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1",
|
||||
]
|
||||
marker = "agent99-host110-backup-runtime-broker.ps1"
|
||||
|
||||
for path in sources:
|
||||
assert marker in path.read_text(encoding="utf-8"), path
|
||||
@@ -36,7 +36,7 @@ def _base_case() -> dict[str, Any]:
|
||||
"exists": True,
|
||||
"sourceRevision": "a" * 40,
|
||||
"runtimeMatched": True,
|
||||
"fileCount": 19,
|
||||
"fileCount": 18,
|
||||
"mismatchCount": 0,
|
||||
"parseError": "",
|
||||
},
|
||||
@@ -84,7 +84,7 @@ $parameters = @{{
|
||||
AgentRootPresent = [bool]$case.agentRootPresent
|
||||
Manifest = $case.manifest
|
||||
ManifestCheckRequired = [bool]$case.manifestCheckRequired
|
||||
ExpectedRuntimeFileCount = 19
|
||||
ExpectedRuntimeFileCount = 18
|
||||
TaskFailureCount = [int]$case.taskFailureCount
|
||||
RelayListenerCount = [int]$case.relayListenerCount
|
||||
RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount
|
||||
@@ -133,7 +133,7 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
|
||||
)
|
||||
assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision
|
||||
assert "preflightGreen = [bool]$decision.deploymentEligible" in source
|
||||
assert "-ExpectedRuntimeFileCount 19" in source
|
||||
assert "-ExpectedRuntimeFileCount 18" in source
|
||||
assert "warningReasons = $warningReasons" in source
|
||||
assert "selfHealthPerformanceIssues = @(" in source
|
||||
assert 'if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$")' in source
|
||||
|
||||
@@ -18,9 +18,6 @@ RECEIVER = (
|
||||
/ "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
|
||||
)
|
||||
DEPLOYER = ROOT / "agent99-deploy.ps1"
|
||||
BOOTSTRAP = ROOT / "agent99-bootstrap.ps1"
|
||||
SIGNOZ_EXECUTOR = ROOT / "agent99-signoz-metadata-executor.ps1"
|
||||
CONTRACT_CHECK = ROOT / "agent99-contract-check.ps1"
|
||||
LIVE_PREFLIGHT = ROOT / "scripts/reboot-recovery/agent99-live-preflight.ps1"
|
||||
|
||||
EXPECTED_RUNTIME_FILES = (
|
||||
@@ -29,7 +26,6 @@ EXPECTED_RUNTIME_FILES = (
|
||||
"agent99-control-plane.ps1",
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
@@ -74,41 +70,25 @@ def test_sender_is_valid_shell_with_check_as_the_default() -> None:
|
||||
assert "timeout --signal=TERM --kill-after=10" in source
|
||||
|
||||
|
||||
def test_runtime_manifest_producers_and_consumers_share_exact_bundle_count() -> None:
|
||||
def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None:
|
||||
sender = SENDER.read_text(encoding="utf-8")
|
||||
receiver = RECEIVER.read_text(encoding="utf-8")
|
||||
deployer = DEPLOYER.read_text(encoding="utf-8")
|
||||
bootstrap = BOOTSTRAP.read_text(encoding="utf-8")
|
||||
signoz_executor = SIGNOZ_EXECUTOR.read_text(encoding="utf-8")
|
||||
contract_check = CONTRACT_CHECK.read_text(encoding="utf-8")
|
||||
live_preflight = LIVE_PREFLIGHT.read_text(encoding="utf-8")
|
||||
|
||||
sender_files = _quoted_array(sender, "RUNTIME_FILES=(", ")")
|
||||
receiver_files = _quoted_array(receiver, "$FixedRuntimeFiles = @(", ")")
|
||||
deployer_files = _quoted_array(deployer, "$runtimeFiles = @(", ")")
|
||||
bootstrap_files = _quoted_array(bootstrap, "$agentFiles = @(", ")")
|
||||
|
||||
assert sender_files == EXPECTED_RUNTIME_FILES
|
||||
assert receiver_files == EXPECTED_RUNTIME_FILES
|
||||
assert deployer_files == EXPECTED_RUNTIME_FILES
|
||||
assert bootstrap_files == EXPECTED_RUNTIME_FILES
|
||||
expected_count = len(EXPECTED_RUNTIME_FILES)
|
||||
preflight_count = re.search(
|
||||
r"-ExpectedRuntimeFileCount\s+(\d+)", live_preflight
|
||||
)
|
||||
assert preflight_count is not None
|
||||
signoz_count = re.search(
|
||||
r"\$ExpectedRuntimeFileCount\s*=\s*(\d+)", signoz_executor
|
||||
)
|
||||
assert signoz_count is not None
|
||||
assert int(preflight_count.group(1)) == expected_count
|
||||
assert int(signoz_count.group(1)) == expected_count
|
||||
assert f'expectedRuntimeFileCount\": {expected_count}' in sender
|
||||
assert f"[int]$envelope.expectedRuntimeFileCount -ne {expected_count}" in receiver
|
||||
assert f"fileCount = {expected_count}" in receiver
|
||||
assert "$result.fileCount -eq $ExpectedRuntimeFileCount" in signoz_executor
|
||||
assert "$ExpectedRuntimeFileCount = 19" in contract_check
|
||||
assert "$result.fileCount -eq $ExpectedRuntimeFileCount" in contract_check
|
||||
assert int(preflight_count.group(1)) == len(EXPECTED_RUNTIME_FILES)
|
||||
assert "expectedRuntimeFileCount\": 18" in sender
|
||||
assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver
|
||||
assert "required_runtime_file_missing" in receiver
|
||||
assert "duplicate_runtime_file" in receiver
|
||||
@@ -283,7 +263,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
|
||||
)
|
||||
assert "$livePreflight.sourceRevision -eq $sourceRevision" in source
|
||||
assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source
|
||||
assert '$runtimeManifest.fileCount -eq 19' in source
|
||||
assert '$runtimeManifest.fileCount -eq 18' in source
|
||||
assert "function Invoke-AgentWindowsControlBaseline" in source
|
||||
assert "function Get-AgentRunEvidenceToken" in source
|
||||
assert "$evidenceRunToken = Get-AgentRunEvidenceToken -Value $RunId" in source
|
||||
|
||||
@@ -516,8 +516,7 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
|
||||
for parameter in ("AutomationRunId", "TraceId", "WorkItemId"):
|
||||
assert f"[string]${parameter} = \"\"" in control
|
||||
assert f'[string]`${parameter} = ""' in bootstrap
|
||||
assert '"controlled_dispatch_identity_unsafe"' in control
|
||||
assert "reason = $identityRejectionReason" in control
|
||||
assert 'reason = "controlled_dispatch_identity_unsafe"' in control
|
||||
assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
|
||||
assert '$dryRun.managerPreflightStatus -eq "ready"' in function
|
||||
assert "-not $dryRun.artifactTransactionBlocked" in function
|
||||
|
||||
@@ -19,9 +19,9 @@ from typing import Any, Sequence
|
||||
|
||||
|
||||
SCHEMA_VERSION = "awoooi_runtime_image_mirror_policy_v1"
|
||||
RECEIPT_SCHEMA_VERSION = "awoooi_runtime_image_mirror_receipt_v1"
|
||||
RECEIPT_SCHEMA_VERSION = "awoooi_runtime_image_mirror_receipt_v2"
|
||||
STAGING_SCHEMA_VERSION = "awoooi_runtime_image_mirror_staging_policy_v1"
|
||||
STAGING_RECEIPT_SCHEMA_VERSION = "awoooi_runtime_image_mirror_staging_receipt_v1"
|
||||
STAGING_RECEIPT_SCHEMA_VERSION = "awoooi_runtime_image_mirror_staging_receipt_v2"
|
||||
ALLOWED_KINDS = {"deployment", "daemonset", "statefulset"}
|
||||
ALLOWED_CONTAINER_KINDS = {"container", "init_container"}
|
||||
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
@@ -665,16 +665,25 @@ def mirror_images(
|
||||
known_hosts: Path,
|
||||
apply: bool,
|
||||
) -> dict[str, Any]:
|
||||
cache = RuntimeCache(policy, ssh_key, known_hosts)
|
||||
cache: RuntimeCache | None = None
|
||||
image_receipts: list[dict[str, Any]] = []
|
||||
missing_count = 0
|
||||
for image in policy.images:
|
||||
source_ref = cache.verify_source(image)
|
||||
source_fingerprint = "sha256:" + hashlib.sha256(source_ref.encode()).hexdigest()
|
||||
target_present = _target_digest_present(image)
|
||||
source_fingerprint = None
|
||||
source_cache_verified = False
|
||||
source_cache_verification_skipped_reason = "target_digest_already_verified"
|
||||
execution = "already_present"
|
||||
if not target_present:
|
||||
missing_count += 1
|
||||
if cache is None:
|
||||
cache = RuntimeCache(policy, ssh_key, known_hosts)
|
||||
source_ref = cache.verify_source(image)
|
||||
source_fingerprint = (
|
||||
"sha256:" + hashlib.sha256(source_ref.encode()).hexdigest()
|
||||
)
|
||||
source_cache_verified = True
|
||||
source_cache_verification_skipped_reason = None
|
||||
execution = "check_only_missing"
|
||||
if apply:
|
||||
source_preexisting = _local_image_present(source_ref)
|
||||
@@ -727,7 +736,10 @@ def mirror_images(
|
||||
"platform_digest": image.platform_digest,
|
||||
"target_registry_digest": image.target_registry_digest,
|
||||
"target_digest_ref": _target_digest_ref(image),
|
||||
"source_cache_verified": True,
|
||||
"source_cache_verified": source_cache_verified,
|
||||
"source_cache_verification_skipped_reason": (
|
||||
source_cache_verification_skipped_reason
|
||||
),
|
||||
"target_digest_verified": target_present,
|
||||
"execution": execution,
|
||||
}
|
||||
@@ -743,6 +755,7 @@ def mirror_images(
|
||||
"mode": "apply" if apply else "check",
|
||||
"external_pull_allowed": False,
|
||||
"source_cache_only": True,
|
||||
"source_cache_accessed": cache is not None,
|
||||
"candidate_count": len(image_receipts),
|
||||
"missing_before_count": missing_count,
|
||||
"verified_count": sum(
|
||||
@@ -766,20 +779,30 @@ def stage_images(
|
||||
known_hosts: Path,
|
||||
apply: bool,
|
||||
) -> dict[str, Any]:
|
||||
cache = RuntimeCache(policy, ssh_key, known_hosts)
|
||||
cache: RuntimeCache | None = None
|
||||
image_receipts: list[dict[str, Any]] = []
|
||||
missing_count = 0
|
||||
for image in policy.images:
|
||||
source_ref = cache.verify_source(image)
|
||||
source_fingerprint = "sha256:" + hashlib.sha256(source_ref.encode()).hexdigest()
|
||||
source_config_digest = cache.source_config_digest(image.platform_digest)
|
||||
if source_config_digest != image.source_config_digest:
|
||||
raise ControllerError("runtime_cache_config_digest_mismatch")
|
||||
|
||||
descriptor = _registry_descriptor(image.push_ref)
|
||||
source_fingerprint = None
|
||||
source_cache_verified = False
|
||||
source_cache_verification_skipped_reason = (
|
||||
"internal_registry_provenance_already_verified"
|
||||
)
|
||||
execution = "already_present"
|
||||
if descriptor is None:
|
||||
missing_count += 1
|
||||
if cache is None:
|
||||
cache = RuntimeCache(policy, ssh_key, known_hosts)
|
||||
source_ref = cache.verify_source(image)
|
||||
source_fingerprint = (
|
||||
"sha256:" + hashlib.sha256(source_ref.encode()).hexdigest()
|
||||
)
|
||||
source_config_digest = cache.source_config_digest(image.platform_digest)
|
||||
if source_config_digest != image.source_config_digest:
|
||||
raise ControllerError("runtime_cache_config_digest_mismatch")
|
||||
source_cache_verified = True
|
||||
source_cache_verification_skipped_reason = None
|
||||
execution = "check_only_missing"
|
||||
if apply:
|
||||
source_preexisting = _local_image_present(source_ref)
|
||||
@@ -836,7 +859,10 @@ def stage_images(
|
||||
raise ControllerError("internal_registry_descriptor_missing")
|
||||
execution = "staged_from_runtime_cache"
|
||||
|
||||
if descriptor is not None and descriptor.config_digest != source_config_digest:
|
||||
if (
|
||||
descriptor is not None
|
||||
and descriptor.config_digest != image.source_config_digest
|
||||
):
|
||||
raise ControllerError("internal_registry_provenance_mismatch")
|
||||
target_digest_ref = None
|
||||
runtime_ref = None
|
||||
@@ -858,13 +884,16 @@ def stage_images(
|
||||
"source_ref_fingerprint": source_fingerprint,
|
||||
"source_index_digest": image.source_index_digest,
|
||||
"platform_digest": image.platform_digest,
|
||||
"source_config_digest": source_config_digest,
|
||||
"source_config_digest": image.source_config_digest,
|
||||
"target_registry_digest": (
|
||||
descriptor.digest if descriptor is not None else None
|
||||
),
|
||||
"target_digest_ref": target_digest_ref,
|
||||
"runtime_ref": runtime_ref,
|
||||
"source_cache_verified": True,
|
||||
"source_cache_verified": source_cache_verified,
|
||||
"source_cache_verification_skipped_reason": (
|
||||
source_cache_verification_skipped_reason
|
||||
),
|
||||
"target_digest_verified": target_verified,
|
||||
"provenance_config_digest_match": descriptor is not None,
|
||||
"execution": execution,
|
||||
@@ -889,6 +918,7 @@ def stage_images(
|
||||
"risk_level": policy.risk_level,
|
||||
"external_pull_allowed": False,
|
||||
"source_cache_only": True,
|
||||
"source_cache_accessed": cache is not None,
|
||||
"candidate_count": len(image_receipts),
|
||||
"missing_before_count": missing_count,
|
||||
"verified_count": sum(
|
||||
|
||||
@@ -46,6 +46,10 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase):
|
||||
policy = controller.load_staging_policy(STAGING_POLICY_PATH)
|
||||
return replace(policy, images=(policy.images[0],))
|
||||
|
||||
def _single_image_policy(self):
|
||||
policy = controller.load_policy(POLICY_PATH)
|
||||
return replace(policy, images=(policy.images[0],))
|
||||
|
||||
def _image_with_availability_guard(self):
|
||||
image = controller.load_policy(POLICY_PATH).images[0]
|
||||
return replace(
|
||||
@@ -904,12 +908,12 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase):
|
||||
controller.RuntimeCache,
|
||||
"verify_source",
|
||||
return_value="quay.io/argoproj/argocd:v3.3.6",
|
||||
),
|
||||
) as verify_source,
|
||||
patch.object(
|
||||
controller.RuntimeCache,
|
||||
"source_config_digest",
|
||||
return_value=image.source_config_digest,
|
||||
),
|
||||
) as source_config_digest,
|
||||
patch.object(controller.RuntimeCache, "export") as export,
|
||||
patch.object(
|
||||
controller, "_registry_descriptor", return_value=descriptor
|
||||
@@ -928,11 +932,85 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase):
|
||||
|
||||
self.assertEqual(receipt["state"], "verified")
|
||||
self.assertEqual(receipt["missing_before_count"], 0)
|
||||
self.assertFalse(receipt["source_cache_accessed"])
|
||||
self.assertEqual(receipt["images"][0]["execution"], "already_present")
|
||||
self.assertTrue(receipt["images"][0]["provenance_config_digest_match"])
|
||||
self.assertFalse(receipt["images"][0]["source_cache_verified"])
|
||||
self.assertEqual(
|
||||
receipt["images"][0]["source_cache_verification_skipped_reason"],
|
||||
"internal_registry_provenance_already_verified",
|
||||
)
|
||||
verify_source.assert_not_called()
|
||||
source_config_digest.assert_not_called()
|
||||
export.assert_not_called()
|
||||
run.assert_not_called()
|
||||
|
||||
def test_mirror_reuses_verified_target_digest_without_runtime_cache(self) -> None:
|
||||
policy = self._single_image_policy()
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
receipt_path = Path(temp) / "receipt.json"
|
||||
with (
|
||||
patch.object(
|
||||
controller.RuntimeCache,
|
||||
"verify_source",
|
||||
return_value="quay.io/example/image:v1",
|
||||
) as verify_source,
|
||||
patch.object(controller.RuntimeCache, "export") as export,
|
||||
patch.object(controller, "_target_digest_present", return_value=True),
|
||||
patch.object(controller, "_run") as run,
|
||||
):
|
||||
receipt = controller.mirror_images(
|
||||
policy,
|
||||
trace_id="aia-p0-006-02a-existing",
|
||||
receipt_path=receipt_path,
|
||||
ssh_key=Path("/unused/key"),
|
||||
known_hosts=Path("/unused/known-hosts"),
|
||||
apply=True,
|
||||
)
|
||||
|
||||
self.assertEqual(receipt["state"], "verified")
|
||||
self.assertEqual(receipt["missing_before_count"], 0)
|
||||
self.assertFalse(receipt["source_cache_accessed"])
|
||||
self.assertEqual(receipt["images"][0]["execution"], "already_present")
|
||||
self.assertFalse(receipt["images"][0]["source_cache_verified"])
|
||||
self.assertEqual(
|
||||
receipt["images"][0]["source_cache_verification_skipped_reason"],
|
||||
"target_digest_already_verified",
|
||||
)
|
||||
verify_source.assert_not_called()
|
||||
export.assert_not_called()
|
||||
run.assert_not_called()
|
||||
|
||||
def test_mirror_missing_target_still_verifies_runtime_cache(self) -> None:
|
||||
policy = self._single_image_policy()
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
receipt_path = Path(temp) / "receipt.json"
|
||||
with (
|
||||
patch.object(
|
||||
controller.RuntimeCache,
|
||||
"verify_source",
|
||||
return_value="quay.io/example/image:v1",
|
||||
) as verify_source,
|
||||
patch.object(controller.RuntimeCache, "export") as export,
|
||||
patch.object(controller, "_target_digest_present", return_value=False),
|
||||
):
|
||||
receipt = controller.mirror_images(
|
||||
policy,
|
||||
trace_id="aia-p0-006-02a-missing-check",
|
||||
receipt_path=receipt_path,
|
||||
ssh_key=Path("/unused/key"),
|
||||
known_hosts=Path("/unused/known-hosts"),
|
||||
apply=False,
|
||||
)
|
||||
|
||||
self.assertEqual(receipt["state"], "blocked_with_safe_next_action")
|
||||
self.assertEqual(receipt["missing_before_count"], 1)
|
||||
self.assertTrue(receipt["source_cache_accessed"])
|
||||
self.assertEqual(receipt["images"][0]["execution"], "check_only_missing")
|
||||
self.assertTrue(receipt["images"][0]["source_cache_verified"])
|
||||
verify_source.assert_called_once_with(policy.images[0])
|
||||
export.assert_not_called()
|
||||
|
||||
def test_staging_exports_cached_source_and_verifies_remote_provenance(
|
||||
self,
|
||||
) -> None:
|
||||
@@ -985,6 +1063,7 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase):
|
||||
|
||||
self.assertEqual(receipt["state"], "verified")
|
||||
self.assertEqual(receipt["missing_before_count"], 1)
|
||||
self.assertTrue(receipt["source_cache_accessed"])
|
||||
self.assertEqual(receipt["images"][0]["execution"], "staged_from_runtime_cache")
|
||||
self.assertEqual(
|
||||
receipt["images"][0]["target_registry_digest"], descriptor.digest
|
||||
|
||||
Reference in New Issue
Block a user