Compare commits

..

7 Commits

Author SHA1 Message Date
Your Name
8f11ef3362 fix(backup): make Host110 runtime receipt durable 2026-07-18 23:37:52 +08:00
Your Name
0b9c028400 fix(backup): align Host110 runtime manifest staging 2026-07-18 22:38:53 +08:00
Your Name
c8f8bc431d docs(governance): bind Host110 runtime V7 evidence 2026-07-18 22:25:14 +08:00
Your Name
18ba20acb8 fix(agent99): enforce runtime manifest count parity 2026-07-18 22:22:04 +08:00
Your Name
d3d731be3e fix(backup): harden Google Drive freshness readback 2026-07-18 22:22:04 +08:00
Your Name
98b0682a6e fix(agent99): deploy backup freshness exporter atomically 2026-07-18 22:22:04 +08:00
Your Name
4b2b73f1a7 feat(agent99): bind host110 backup runtime deploy 2026-07-18 22:21:38 +08:00
32 changed files with 2637 additions and 303 deletions

View File

@@ -133,7 +133,10 @@ jobs:
# 2026-06-28 Codex: awoooi-non110-host maps to the dedicated
# non-110 runner lane. Bootstrap tools defensively because host
# runners can start without the CI toolchain preinstalled.
run: bash scripts/ci/bootstrap-host-runner-tools.sh
run: |
if command -v apk >/dev/null 2>&1; then
apk add --no-cache nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
fi
- name: Checkout source from Gitea
env:
@@ -1451,7 +1454,10 @@ jobs:
- name: Bootstrap Host Runner Tools
# 2026-05-05 Codex: keep the host-mode runner self-healing before
# actions/checkout@v4 and Telegram failure notifications run.
run: bash scripts/ci/bootstrap-host-runner-tools.sh
run: |
if command -v apk >/dev/null 2>&1; then
apk add --no-cache nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
fi
- name: Checkout source from Gitea
env:
@@ -5220,7 +5226,10 @@ jobs:
- name: Bootstrap Host Runner Tools
# 2026-05-05 Codex: post-deploy also uses checkout and curl-based
# notifications, so it needs the same runner bootstrap as earlier jobs.
run: bash scripts/ci/bootstrap-host-runner-tools.sh
run: |
if command -v apk >/dev/null 2>&1; then
apk add --no-cache nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
fi
- name: Checkout source from Gitea
env:

View File

@@ -224,6 +224,7 @@ $agentFiles = @(
"agent99-control-plane.ps1",
"agent99-db-bounded-executor.ps1",
"agent99-signoz-metadata-executor.ps1",
"agent99-host110-backup-runtime-broker.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",

View File

@@ -20,6 +20,7 @@ $requiredFiles = @(
"agent99-control-plane.ps1",
"agent99-db-bounded-executor.ps1",
"agent99-signoz-metadata-executor.ps1",
"agent99-host110-backup-runtime-broker.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",
@@ -131,6 +132,7 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js
$control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane.ps1"), [Text.Encoding]::UTF8)
$dbBoundedExecutor = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-db-bounded-executor.ps1"), [Text.Encoding]::UTF8)
$signozMetadataExecutor = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-signoz-metadata-executor.ps1"), [Text.Encoding]::UTF8)
$host110BackupRuntimeBroker = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-host110-backup-runtime-broker.ps1"), [Text.Encoding]::UTF8)
$inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8)
$telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8)
$bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8)
@@ -190,7 +192,8 @@ $signozMetadataExecutorContract = [bool](
$signozMetadataExecutor.Contains('agent99_active:') -and
$signozMetadataExecutor.Contains('$RemoteResult.completed -ne $true') -and
$signozMetadataExecutor.Contains('Get-Agent99RuntimeSourceBinding') -and
$signozMetadataExecutor.Contains('$result.fileCount -eq 18') -and
$signozMetadataExecutor.Contains('$ExpectedRuntimeFileCount = 19') -and
$signozMetadataExecutor.Contains('$result.fileCount -eq $ExpectedRuntimeFileCount') -and
$signozMetadataExecutor.Contains('pass_export_verified_restore_pending') -and
$signozMetadataExecutor.Contains('productionRuntimeMutationPerformed = $false') -and
$signozMetadataExecutor.Contains('rawSqliteRead = $false') -and
@@ -203,6 +206,29 @@ $signozMetadataExecutorContract = [bool](
-not $signozMetadataExecutor.Contains('Get-Content -LiteralPath $CredentialFile')
)
Add-Check "backup:signoz_metadata_fixed_executor" $signozMetadataExecutorContract "Windows99 dispatches one fixed host110 metadata export contract, consumes only a root-owned credential reference on target, and requires the independent bundle verifier without raw SQLite or secret output"
$host110BackupRuntimeContract = [bool](
$host110BackupRuntimeBroker.Contains('$TargetHost = "192.168.0.110"') -and
$host110BackupRuntimeBroker.Contains('$TargetUser = "wooo"') -and
$host110BackupRuntimeBroker.Contains('$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"') -and
$host110BackupRuntimeBroker.Contains('"credential.helper="') -and
$host110BackupRuntimeBroker.Contains('$env:GIT_TERMINAL_PROMPT = "0"') -and
$host110BackupRuntimeBroker.Contains('"merge-base", "--is-ancestor", $SourceRevision, $sourceHead') -and
$host110BackupRuntimeBroker.Contains('executorSha256 = $executorDigest') -and
$host110BackupRuntimeBroker.Contains('verifierSha256 = $verifierDigest') -and
$host110BackupRuntimeBroker.Contains('"scripts/ops/backup-health-textfile-exporter.py"') -and
$host110BackupRuntimeBroker.Contains('$ExpectedFileCount = 18') -and
$host110BackupRuntimeBroker.Contains('StrictHostKeyChecking=yes') -and
$host110BackupRuntimeBroker.Contains('$process.WaitForExit($TimeoutSeconds * 1000)') -and
$host110BackupRuntimeBroker.Contains('taskkill.exe /PID $process.Id /T /F') -and
$host110BackupRuntimeBroker.Contains('Invoke-Agent99BoundedScp $sourcePackage.localPaths') -and
$host110BackupRuntimeBroker.Contains('python3 - --manifest-base64') -and
$host110BackupRuntimeBroker.Contains('Write-Agent99ImmutableJson') -and
$host110BackupRuntimeBroker.Contains('decisionProvider = "deterministic_only"') -and
$host110BackupRuntimeBroker.Contains('criticProvider = "deterministic_only"') -and
-not $host110BackupRuntimeBroker.Contains('/home/wooo/awoooi') -and
-not $host110BackupRuntimeBroker.Contains('Invoke-Expression')
)
Add-Check "backup:host110_runtime_fixed_broker" $host110BackupRuntimeContract "Windows99 fetches one exact Gitea revision, binds executor and verifier digests, performs one fixed process-bounded Host110 transaction, and preserves an independent no-write verifier readback"
Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("Enter-AgentRecoveryTriggerReservation") -and $control.Contains('"recovery-trigger-reservation.lock"') -and $control.Contains('[System.IO.FileShare]::None') -and $control.Contains('reason = "recovery_trigger_reservation_busy"') -and $control.Contains("recovery_already_queued")) "boot and host-down recovery reserve one atomic single-flight queue decision"
Add-Check "recovery:auto_trigger_controlled_identity" ($control.Contains("New-AgentObservedRecoveryDispatchIdentity") -and $control.Contains('-RouteId "agent99_local_observer"') -and $control.Contains('schema_version = "agent99_controlled_dispatch_identity_v1"') -and $control.Contains('correlationKey = $dispatchIdentity.idempotencyKey') -and $control.Contains('canonicalDigest = $dispatchIdentity.canonicalDigest') -and $control.Contains("Invoke-AgentControlledDispatchIdentitySelfTest")) "observer-created Recover requests carry a stable canonical controlled dispatch identity before queue processing"
Add-Check "recovery:queue_identity_fail_closed" ($control.Contains('dispatchIdentitySchemaVersion = $dispatchIdentity.schemaVersion') -and $control.Contains('Test-AgentControlledDispatchIdentity $dispatchIdentityEnvelope') -and $control.Contains('"controlled_dispatch_identity_invalid"') -and $control.IndexOf('"controlled_dispatch_identity_invalid"') -lt $control.IndexOf('$process = Start-Process -FilePath "powershell.exe" -ArgumentList $args')) "queue consumer rebuilds and validates canonical identity before any controlled executor starts"

View File

@@ -35,6 +35,7 @@ $runtimeFiles = @(
"agent99-control-plane.ps1",
"agent99-db-bounded-executor.ps1",
"agent99-signoz-metadata-executor.ps1",
"agent99-host110-backup-runtime-broker.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",

View File

@@ -0,0 +1,481 @@
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[ValidateSet("Check", "Apply", "Verify")]
[string]$Mode,
[Parameter(Mandatory = $true)]
[string]$SourceRevision,
[Parameter(Mandatory = $true)]
[string]$RunId,
[string]$AgentRoot = "C:\Wooo\Agent99"
)
$ErrorActionPreference = "Stop"
$ExpectedAgentRoot = "C:\Wooo\Agent99"
$TargetHost = "192.168.0.110"
$TargetUser = "wooo"
$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"
$GitPath = "C:\Program Files\Git\cmd\git.exe"
$IdentityFile = Join-Path $ExpectedAgentRoot "keys\agent99_ed25519"
$EvidenceRoot = Join-Path $ExpectedAgentRoot "evidence\host110-backup-runtime"
$SourceRootBase = Join-Path $ExpectedAgentRoot "deploy"
$TransportMutexName = "Global\WoooAgent99Host110BackupRuntimeV2"
$ExpectedFileCount = 18
$ExpectedRemoteStageFileCount = 21
$RuntimeRelativePaths = @(
"scripts/backup/common.sh",
"scripts/backup/backup-all.sh",
"scripts/backup/backup-host188-products.sh",
"scripts/backup/verify-host188-products-backup.sh",
"scripts/backup/verify-host188-products-archive.py",
"scripts/backup/backup-gitea.sh",
"scripts/backup/gitea-full-backup-restore-drill.sh",
"scripts/backup/backup-configs.sh",
"scripts/backup/backup-awoooi.sh",
"scripts/backup/backup-awoooi-frequent.sh",
"scripts/backup/backup-clawbot.sh",
"scripts/backup/backup-sentry.sh",
"scripts/backup/check-backup-integrity.sh",
"scripts/backup/sync-offsite-backups.sh",
"scripts/backup/backup-offsite-readiness-gate.sh",
"scripts/backup/verify-offsite-full-sync.sh",
"scripts/backup/enforce-latest-only-retention.sh",
"scripts/ops/backup-health-textfile-exporter.py"
)
$ExecutorRelativePath = "scripts/backup/host110-backup-runtime-executor.sh"
$VerifierRelativePath = "scripts/backup/verify-host110-backup-runtime.py"
function Test-Agent99SafeIdentity {
param([string]$Value)
return [bool]($Value -and $Value -match "^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$")
}
function Write-Agent99ImmutableJson {
param([string]$Path, [object]$Value)
if (Test-Path -LiteralPath $Path) { throw "run_identity_evidence_exists" }
$temporary = "$Path.tmp-$PID-$([Guid]::NewGuid().ToString('N'))"
$encoding = New-Object System.Text.UTF8Encoding($false)
try {
$json = $Value | ConvertTo-Json -Depth 12
[IO.File]::WriteAllText($temporary, $json + [Environment]::NewLine, $encoding)
[IO.File]::Move($temporary, $Path)
} finally {
Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue | Out-Null
}
}
function Invoke-Agent99BoundedProcess {
param(
[string]$FilePath,
[string[]]$Arguments,
[int]$TimeoutSeconds,
[string]$WorkingDirectory = "",
[string]$StandardInputPath = ""
)
$token = [Guid]::NewGuid().ToString("N")
$stdoutPath = Join-Path $env:TEMP "agent99-host110-backup-$token.out"
$stderrPath = Join-Path $env:TEMP "agent99-host110-backup-$token.err"
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
$process = $null
try {
$start = @{
FilePath = $FilePath
ArgumentList = $Arguments
NoNewWindow = $true
RedirectStandardOutput = $stdoutPath
RedirectStandardError = $stderrPath
PassThru = $true
}
if ($WorkingDirectory) { $start.WorkingDirectory = $WorkingDirectory }
if ($StandardInputPath) { $start.RedirectStandardInput = $StandardInputPath }
$process = Start-Process @start
$null = $process.Handle
if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {}
if (-not $process.WaitForExit(2000) -and -not $process.HasExited) {
$process.Kill()
$process.WaitForExit(2000) | Out-Null
}
return [pscustomobject]@{
ok = $false
exitCode = -2
reason = "transport_timeout"
elapsedMs = $stopwatch.ElapsedMilliseconds
stdout = ""
stderrPresent = $false
}
}
$process.WaitForExit() | Out-Null
$process.Refresh()
$stdout = if (Test-Path -LiteralPath $stdoutPath) { Get-Content -LiteralPath $stdoutPath -Raw } else { "" }
$stderr = if (Test-Path -LiteralPath $stderrPath) { Get-Content -LiteralPath $stderrPath -Raw } else { "" }
if ($stdout.Length -gt 65536 -or $stderr.Length -gt 65536) {
return [pscustomobject]@{
ok = $false
exitCode = -3
reason = "transport_output_oversized"
elapsedMs = $stopwatch.ElapsedMilliseconds
stdout = ""
stderrPresent = [bool]$stderr
}
}
return [pscustomobject]@{
ok = [bool]($process.ExitCode -eq 0)
exitCode = [int]$process.ExitCode
reason = if ($process.ExitCode -eq 0) { "completed" } else { "process_failed" }
elapsedMs = $stopwatch.ElapsedMilliseconds
stdout = [string]$stdout
stderrPresent = [bool]$stderr
}
} catch {
return [pscustomobject]@{
ok = $false
exitCode = -1
reason = "transport_exception"
elapsedMs = $stopwatch.ElapsedMilliseconds
stdout = ""
stderrPresent = $false
errorType = $_.Exception.GetType().Name
}
} finally {
$stopwatch.Stop()
Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null
if ($process) { $process.Dispose() }
}
}
function Invoke-Agent99BoundedSsh {
param([string]$RemoteCommand, [int]$TimeoutSeconds, [string]$StandardInputPath = "")
$arguments = @(
"-i", $IdentityFile,
"-o", "IdentitiesOnly=yes",
"-o", "BatchMode=yes",
"-o", "NumberOfPasswordPrompts=0",
"-o", "StrictHostKeyChecking=yes",
"-o", "ConnectTimeout=8",
"-o", "ConnectionAttempts=1",
"-o", "ServerAliveInterval=10",
"-o", "ServerAliveCountMax=2",
"-l", $TargetUser,
$TargetHost,
$RemoteCommand
)
return Invoke-Agent99BoundedProcess "ssh.exe" $arguments $TimeoutSeconds "" $StandardInputPath
}
function Invoke-Agent99BoundedScp {
param([string[]]$LocalPaths, [string]$RemoteDirectory, [int]$TimeoutSeconds)
$arguments = @(
"-i", $IdentityFile,
"-o", "IdentitiesOnly=yes",
"-o", "BatchMode=yes",
"-o", "NumberOfPasswordPrompts=0",
"-o", "StrictHostKeyChecking=yes",
"-o", "ConnectTimeout=8",
"-o", "ConnectionAttempts=1"
)
$arguments += $LocalPaths
$arguments += "${TargetUser}@${TargetHost}:${RemoteDirectory}/"
return Invoke-Agent99BoundedProcess "scp.exe" $arguments $TimeoutSeconds
}
function Invoke-Agent99Git {
param([string[]]$Arguments, [int]$TimeoutSeconds, [string]$WorkingDirectory = "")
$previousPrompt = $env:GIT_TERMINAL_PROMPT
try {
$env:GIT_TERMINAL_PROMPT = "0"
return Invoke-Agent99BoundedProcess $GitPath $Arguments $TimeoutSeconds $WorkingDirectory
} finally {
$env:GIT_TERMINAL_PROMPT = $previousPrompt
}
}
function Assert-Agent99ProcessOk {
param([object]$Result, [string]$Reason)
if (-not $Result.ok) { throw $Reason }
}
function New-Agent99SourcePackage {
param([string]$SourceRoot, [string]$ManifestPath)
if (Test-Path -LiteralPath $SourceRoot) { throw "run_identity_source_stage_exists" }
$clone = Invoke-Agent99Git @(
"-c", "credential.helper=",
"-c", "core.autocrlf=false",
"clone", "--quiet", "--no-tags", "--depth", "200", "--branch", "main",
$GiteaSourceUrl, $SourceRoot
) 180 $SourceRootBase
Assert-Agent99ProcessOk $clone "gitea_source_fetch_failed"
$headResult = Invoke-Agent99Git @("rev-parse", "HEAD") 30 $SourceRoot
Assert-Agent99ProcessOk $headResult "gitea_source_head_failed"
$sourceHead = ([string]$headResult.stdout).Trim().ToLowerInvariant()
if ($sourceHead -notmatch "^[0-9a-f]{40}$") { throw "gitea_source_head_invalid" }
$commitResult = Invoke-Agent99Git @("cat-file", "-e", "$SourceRevision`^{commit}") 30 $SourceRoot
Assert-Agent99ProcessOk $commitResult "source_revision_not_fetched"
$ancestorResult = Invoke-Agent99Git @("merge-base", "--is-ancestor", $SourceRevision, $sourceHead) 30 $SourceRoot
Assert-Agent99ProcessOk $ancestorResult "source_revision_not_on_gitea_main"
$allPaths = @($RuntimeRelativePaths) + @($ExecutorRelativePath, $VerifierRelativePath)
$checkoutResult = Invoke-Agent99Git (@("-c", "core.autocrlf=false", "checkout", "--quiet", $SourceRevision, "--") + $allPaths) 60 $SourceRoot
Assert-Agent99ProcessOk $checkoutResult "source_revision_checkout_failed"
$rows = @()
$localPaths = @()
foreach ($relativePath in $RuntimeRelativePaths) {
$path = Join-Path $SourceRoot ($relativePath -replace "/", "\")
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { throw "runtime_source_file_missing" }
$rows += [ordered]@{
name = [IO.Path]::GetFileName($path)
sha256 = (Get-FileHash -LiteralPath $path -Algorithm SHA256).Hash.ToLowerInvariant()
}
$localPaths += $path
}
if (($rows.name | Sort-Object -Unique).Count -ne $ExpectedFileCount) { throw "runtime_source_file_set_invalid" }
$executorPath = Join-Path $SourceRoot ($ExecutorRelativePath -replace "/", "\")
$verifierPath = Join-Path $SourceRoot ($VerifierRelativePath -replace "/", "\")
if (-not (Test-Path -LiteralPath $executorPath -PathType Leaf)) { throw "executor_source_file_missing" }
if (-not (Test-Path -LiteralPath $verifierPath -PathType Leaf)) { throw "verifier_source_file_missing" }
$executorDigest = (Get-FileHash -LiteralPath $executorPath -Algorithm SHA256).Hash.ToLowerInvariant()
$verifierDigest = (Get-FileHash -LiteralPath $verifierPath -Algorithm SHA256).Hash.ToLowerInvariant()
$manifest = [ordered]@{
schemaVersion = "agent99_host110_backup_runtime_source_manifest_v1"
sourceRevision = $SourceRevision
sourceHead = $sourceHead
runId = $RunId
executorSha256 = $executorDigest
verifierSha256 = $verifierDigest
files = $rows
}
$encoding = New-Object System.Text.UTF8Encoding($false)
[IO.File]::WriteAllText($ManifestPath, ($manifest | ConvertTo-Json -Depth 8 -Compress), $encoding)
$localPaths += $executorPath, $verifierPath, $ManifestPath
return [pscustomobject]@{
sourceHead = $sourceHead
executorSha256 = $executorDigest
verifierSha256 = $verifierDigest
verifierPath = $verifierPath
manifestPath = $ManifestPath
manifestBase64 = [Convert]::ToBase64String([IO.File]::ReadAllBytes($ManifestPath))
localPaths = $localPaths
}
}
function Get-Agent99ExecutorResult {
param([object]$Transport, [string]$ExpectedMode)
if (-not $Transport.ok) { throw "host110_${ExpectedMode}_transport_failed" }
try {
$parsed = [string]$Transport.stdout | ConvertFrom-Json
} catch {
throw "host110_${ExpectedMode}_receipt_invalid"
}
if (
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_executor_v1" -or
[string]$parsed.mode -ne $ExpectedMode.ToLowerInvariant() -or
-not [bool]$parsed.ok -or
[string]$parsed.sourceRevision -ne $SourceRevision -or
[int]$parsed.fileCount -ne $ExpectedFileCount -or
[int]$parsed.backupScriptFileCount -ne 17 -or
-not [bool]$parsed.backupHealthExporterIncluded
) {
throw "host110_${ExpectedMode}_receipt_contract_failed"
}
if ($ExpectedMode -eq "Apply") {
if (
[string]$parsed.verifier.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
-not [bool]$parsed.verifier.ok -or
[bool]$parsed.verifier.remoteWritePerformed -or
-not [bool]$parsed.verifier.selfIdentityVerified
) {
throw "host110_apply_independent_verifier_contract_failed"
}
}
return $parsed
}
function Get-Agent99VerifierResult {
param([object]$Transport, [object]$SourcePackage)
if (-not $Transport.ok) { throw "host110_verify_transport_failed" }
try {
$parsed = [string]$Transport.stdout | ConvertFrom-Json
} catch {
throw "host110_verify_receipt_invalid"
}
if (
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
-not [bool]$parsed.ok -or
[string]$parsed.sourceRevision -ne $SourceRevision -or
[string]$parsed.runId -ne $RunId -or
[int]$parsed.fileCount -ne $ExpectedFileCount -or
[int]$parsed.backupScriptFileCount -ne 17 -or
-not [bool]$parsed.backupHealthExporterIncluded -or
[bool]$parsed.remoteWritePerformed -or
[string]$parsed.verifierSha256 -ne [string]$SourcePackage.verifierSha256
) {
throw "host110_verify_receipt_contract_failed"
}
return $parsed
}
function Invoke-Agent99ReadOnlyPreflight {
param([object]$SourcePackage)
$command = 'test "$(id -un)" = wooo && hostname -I | tr " " "\n" | grep -qx 192.168.0.110 && test -d /backup/scripts && test ! -L /backup/scripts && test -d /home/wooo/scripts && test ! -L /home/wooo/scripts && command -v timeout >/dev/null && command -v flock >/dev/null && command -v python3 >/dev/null && printf HOST110_BACKUP_PREFLIGHT_OK=1'
$transport = Invoke-Agent99BoundedSsh $command 45
if (-not $transport.ok -or ([string]$transport.stdout).Trim() -ne "HOST110_BACKUP_PREFLIGHT_OK=1") {
throw "host110_read_only_preflight_failed"
}
return [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_preflight_v1"
ok = $true
sourceRevision = $SourceRevision
sourceHead = [string]$SourcePackage.sourceHead
fileCount = $ExpectedFileCount
backupScriptFileCount = 17
backupHealthExporterIncluded = $true
remoteWritePerformed = $false
}
}
function Invoke-Agent99NoWriteVerifier {
param([object]$SourcePackage)
$command = "timeout --signal=TERM --kill-after=5s 60s python3 - --manifest-base64 $($SourcePackage.manifestBase64) --destination /backup/scripts --exporter-destination /home/wooo/scripts/backup-health-textfile-exporter.py --source-revision $SourceRevision --run-id $RunId --expected-verifier-sha256 $($SourcePackage.verifierSha256)"
$transport = Invoke-Agent99BoundedSsh $command 90 ([string]$SourcePackage.verifierPath)
return Get-Agent99VerifierResult $transport $SourcePackage
}
function Remove-Agent99RemoteStage {
param([string]$RemoteStage, [bool]$Required)
$cleanup = Invoke-Agent99BoundedSsh "rm -rf -- $RemoteStage && test ! -e $RemoteStage" 45
if ($Required -and -not $cleanup.ok) { throw "remote_source_stage_cleanup_failed" }
return [bool]$cleanup.ok
}
if ($AgentRoot.TrimEnd("\") -ne $ExpectedAgentRoot) { throw "agent_root_override_forbidden" }
$SourceRevision = $SourceRevision.ToLowerInvariant()
if ($SourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
if (-not (Test-Agent99SafeIdentity $RunId)) { throw "invalid_run_id" }
if (-not (Test-Path -LiteralPath $IdentityFile -PathType Leaf)) { throw "agent99_ssh_identity_unavailable" }
if (-not (Test-Path -LiteralPath $GitPath -PathType Leaf)) { throw "windows_git_unavailable" }
if (-not (Get-Command ssh.exe -ErrorAction SilentlyContinue)) { throw "windows_openssh_unavailable" }
if (-not (Get-Command scp.exe -ErrorAction SilentlyContinue)) { throw "windows_openscp_unavailable" }
New-Item -ItemType Directory -Force -Path $EvidenceRoot, $SourceRootBase | Out-Null
$evidencePath = Join-Path $EvidenceRoot "agent99-host110-backup-runtime-$RunId.json"
if (Test-Path -LiteralPath $evidencePath -PathType Leaf) {
try { $existing = Get-Content -LiteralPath $evidencePath -Raw | ConvertFrom-Json } catch { throw "run_identity_evidence_invalid" }
if ([string]$existing.sourceRevision -ne $SourceRevision -or [string]$existing.mode -ne $Mode) {
throw "run_identity_conflict"
}
$existing | ConvertTo-Json -Compress -Depth 12
if (-not [bool]$existing.ok) { exit 1 }
exit 0
}
$sourceRoot = Join-Path $SourceRootBase "host110-backup-source-$RunId"
$manifestPath = Join-Path $sourceRoot "manifest.json"
$remoteStage = "/tmp/agent99-host110-backup-runtime-$RunId"
$startedAt = [DateTime]::UtcNow
$mutex = [Threading.Mutex]::new($false, $TransportMutexName)
$acquired = $false
$remoteStageCreated = $false
$sourcePackage = $null
$check = $null
$apply = $null
$verify = $null
$terminalStatus = "failed"
$errorCode = ""
$cleanupVerified = $false
try {
try {
$acquired = [bool]$mutex.WaitOne(30000)
} catch [Threading.AbandonedMutexException] {
$acquired = $true
}
if (-not $acquired) { throw "host110_backup_runtime_transport_busy" }
$sourcePackage = New-Agent99SourcePackage $sourceRoot $manifestPath
$check = Invoke-Agent99ReadOnlyPreflight $sourcePackage
if ($Mode -eq "Apply") {
$createStage = Invoke-Agent99BoundedSsh "test ! -e $remoteStage && install -d -m 700 $remoteStage" 45
Assert-Agent99ProcessOk $createStage "remote_source_stage_identity_conflict"
$remoteStageCreated = $true
$copyStage = Invoke-Agent99BoundedScp $sourcePackage.localPaths $remoteStage 120
Assert-Agent99ProcessOk $copyStage "remote_source_stage_copy_failed"
$stageCheck = Invoke-Agent99BoundedSsh "chmod 700 $remoteStage/host110-backup-runtime-executor.sh $remoteStage/verify-host110-backup-runtime.py && test `$(find $remoteStage -mindepth 1 -maxdepth 1 -type f | wc -l) -eq $ExpectedRemoteStageFileCount" 45
Assert-Agent99ProcessOk $stageCheck "remote_source_stage_contract_failed"
$executorPath = "$remoteStage/host110-backup-runtime-executor.sh"
$command = "timeout --signal=TERM --kill-after=10s 300s $executorPath --mode apply --source-revision $SourceRevision --run-id $RunId --source-stage $remoteStage"
$applyTransport = Invoke-Agent99BoundedSsh $command 330
$apply = Get-Agent99ExecutorResult $applyTransport "Apply"
$verify = $apply.verifier
$cleanupVerified = Remove-Agent99RemoteStage $remoteStage $true
$remoteStageCreated = $false
$terminalStatus = "verified"
} elseif ($Mode -eq "Verify") {
$verify = Invoke-Agent99NoWriteVerifier $sourcePackage
$terminalStatus = "readback_ok"
$cleanupVerified = $true
} else {
$terminalStatus = "readback_ok"
$cleanupVerified = $true
}
} catch {
$errorCode = [string]$_.Exception.Message
} finally {
if ($remoteStageCreated) {
try { $cleanupVerified = Remove-Agent99RemoteStage $remoteStage $false } catch {}
}
if ($acquired) { try { $mutex.ReleaseMutex() } catch {} }
$mutex.Dispose()
}
$result = [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v2"
ok = [bool]($terminalStatus -ne "failed")
status = $terminalStatus
mode = $Mode
sourceRevision = $SourceRevision
sourceHead = if ($sourcePackage) { [string]$sourcePackage.sourceHead } else { "" }
runId = $RunId
targetHost = $TargetHost
sourceTransport = "windows99_gitea_exact_revision_manifest"
executor = "host110_backup_runtime_executor"
verifier = "independent_host110_backup_runtime_python_readback"
decisionProvider = "deterministic_only"
criticProvider = "deterministic_only"
agentAction = if ($Mode -eq "Apply") { "controlled_apply" } else { "readback" }
check = $check
apply = $apply
verify = $verify
cleanupVerified = $cleanupVerified
errorCode = $errorCode
elapsedSeconds = [math]::Round(([DateTime]::UtcNow - $startedAt).TotalSeconds, 3)
evidence = $evidencePath
secretValuesRead = $false
rawSessionStored = $false
}
try {
Write-Agent99ImmutableJson $evidencePath $result
} finally {
if (Test-Path -LiteralPath $sourceRoot) {
Remove-Item -LiteralPath $sourceRoot -Recurse -Force -ErrorAction SilentlyContinue
}
}
$result | ConvertTo-Json -Compress -Depth 12
if (-not $result.ok) { exit 1 }

View File

@@ -21,6 +21,7 @@ $IdentityFile = Join-Path $AgentRoot "keys\agent99_ed25519"
$EvidenceDir = Join-Path $AgentRoot "evidence"
$RuntimeManifestPath = Join-Path $AgentRoot "state\runtime-manifest.json"
$EntrypointPath = [string]$MyInvocation.MyCommand.Path
$ExpectedRuntimeFileCount = 19
$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"
$BundleId = "bb6d78b67f506072b2e45e92bc1d415364e25852282a9b068900665c578df011"
$ToolchainRoot = "/backup/toolchains/signoz-metadata/$BundleId"
@@ -220,7 +221,7 @@ function Get-Agent99RuntimeSourceBinding {
}
$result.ok = [bool](
$result.runtimeMatched -and
$result.fileCount -eq 18 -and
$result.fileCount -eq $ExpectedRuntimeFileCount -and
$result.mismatchCount -eq 0 -and
$result.sourceRevision -eq $SourceRevision -and
$result.entrypointTracked -and

View File

@@ -0,0 +1,65 @@
# AWOOOI Agent99 Host110 Backup Runtime V7
Status: proposed; owner approval required before any production apply.
This artifact supersedes V2 through V6. All V5 proposal, artifact, and diff
hashes are void. This file is review evidence only; it is not active policy or
production execution authority.
## Scope
- Agent99 runtime bundle: 19 files. Every producer and consumer is guarded by
one exact parity test.
- Host110 backup payload: 18 files, comprising 17 backup scripts and
`backup-health-textfile-exporter.py`.
- Host110 remote stage: 21 files, comprising the 18 payload files, the fixed
executor, the independent verifier, and the source manifest.
- Fixed target: `wooo@192.168.0.110`, dispatched by Windows99 Agent99 from an
exact Gitea live revision and digest manifest.
- Gitea offline backup and container lifecycle changes are excluded from this
proposal. `backup-gitea.sh` has no candidate diff. The only Gitea-related
diff is a shared deployment lock in the read-only restore-drill entrypoint;
it adds no Docker command and is not executed by the deployment transaction.
## Exact Diff Serialization
Substitute the V7 Gitea live-ref SHA for `<candidate_git_sha>` and hash the
exact stdout bytes from this command:
```sh
git diff --binary --full-index \
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
shasum -a 256
```
The review message independently binds the Gitea live ref and Git SHA, this
tracked artifact path and bytes hash, and the serialized diff hash.
## Apply Effects
- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded
manifest and receipt state.
- Host110 Apply writes/replaces the 18 payload paths through one exclusive,
digest-bound filesystem transaction.
- Apply does not run a backup, invoke a Gitea container lifecycle, sync Google
Drive, prune or delete snapshots, restart a VM or service, or change a DB,
network, or firewall.
## Validation
- Post-split focused runtime, broker, backup, and parity suites must pass.
- Ansible and shell/Python static validation must pass.
- Changed PowerShell must parse on Windows99 with zero errors.
- Central review must independently reproduce the live ref, artifact hash,
diff hash, and 19/18/21 counts.
## Rollback
- Source rollback reverts the exact V7 diff to
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
- Agent99 rollback restores the prior runtime files and manifest and verifies
the prior hashes independently.
- Host110 rollback restores every prior payload's content, SHA-256, uid, gid,
and mode under the exclusive lock.
- Any mismatch terminates as `rollback_unverified` and cannot be reported as a
successful rollback.

View File

@@ -0,0 +1,78 @@
# AWOOOI Agent99 Host110 Backup Runtime V8
Status: proposed; owner approval required before any production apply.
This artifact supersedes V2 through V7. The V7 candidate, revision, artifact,
and diff hashes are void for approval. All V5 hashes remain void. Prior refs
remain immutable audit evidence only; none is production execution authority.
## Scope
- Agent99 runtime bundle: 19 files, guarded by one producer/consumer parity
test.
- Host110 backup payload: 18 files, comprising 17 backup scripts and
`backup-health-textfile-exporter.py`.
- Host110 remote stage: 21 unique basenames, comprising the 18 payload files,
the fixed executor, the independent verifier, and `manifest.json`.
- Fixed target: `wooo@192.168.0.110`, dispatched by Windows99 Agent99 from an
exact Gitea main revision and digest manifest.
- Gitea offline backup and container lifecycle changes remain excluded.
`backup-gitea.sh` has no candidate diff.
## V7 Defect Closed
V7 created and uploaded
`agent99-host110-backup-runtime-manifest.json`, while the fixed Host110
executor consumed `$SOURCE_STAGE/manifest.json`. V8 uses `manifest.json` at
the broker source stage, so SCP preserves the exact basename consumed by the
executor.
The integration test derives all broker SCP basenames from the actual
PowerShell file list and manifest path, requires 21 unique stage files, and
matches the manifest and verifier basenames to the executor's exact
`$SOURCE_STAGE` paths. A hand-built temporary manifest cannot satisfy this
test by itself.
## Exact Diff Serialization
Substitute the V8 Gitea live-ref SHA for `<candidate_git_sha>` and hash the
exact stdout bytes from this command:
```sh
git diff --binary --full-index \
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
shasum -a 256
```
The review message independently binds the Gitea live ref and Git SHA, this
tracked artifact path and bytes hash, and the serialized diff hash.
## Apply Effects
- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded
manifest and receipt state.
- Host110 Apply writes/replaces the 18 payload paths through one exclusive,
digest-bound filesystem transaction.
- Apply does not run a backup, invoke a Gitea container lifecycle, sync Google
Drive, prune or delete snapshots, restart a VM or service, or change a DB,
network, or firewall.
## Validation
- The exact broker/SCP basename to executor path integration test must pass.
- Focused runtime, broker, backup, parity, and Ansible suites must pass.
- Ansible and shell/Python static validation must pass.
- Changed PowerShell must parse on Windows99 with zero errors.
- Central review must independently reproduce the live ref, artifact hash,
diff hash, 19/18/21 counts, and zero `backup-gitea.sh` diff.
## Rollback
- Source rollback reverts the exact V8 diff to
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
- Agent99 rollback restores the prior runtime files and manifest and verifies
the prior hashes independently.
- Host110 rollback restores every prior payload's content, SHA-256, uid, gid,
and mode under the exclusive lock.
- Any mismatch terminates as `rollback_unverified` and cannot be reported as a
successful rollback.

View File

@@ -0,0 +1,133 @@
# AWOOOI Agent99 Host110 Backup Runtime V9
Status: proposed; owner approval required before any production apply.
This artifact supersedes V2 through V8. The following V8 identities are void
for approval and remain audit evidence only:
- candidate/revision: `0b9c0284006c8d892c4617c2bfc16869bb6637f0`
- artifact SHA-256: `ea9537613c462a07f3155263da5ad0f444bda0ba6773b49208de31b0acdef228`
- exact diff SHA-256: `0867d9221ecfa7bcbe6b073a6a02da401bbed5087b9f2bd7fa494f2953b2ef2e`
- every other proposal/content hash derived from V8 candidate bytes
No V8 hash, ref, review, or receipt grants production execution authority.
## Scope
- Agent99 runtime bundle: 19 files, guarded by one producer/consumer parity
test.
- Host110 backup payload: 18 ordered files, comprising 17 backup scripts and
`backup-health-textfile-exporter.py`.
- Host110 remote stage: 21 unique basenames, comprising the ordered 18-file
payload, fixed executor, independent verifier, and `manifest.json`.
- Fixed target: `wooo@192.168.0.110`, dispatched by Windows99 Agent99 from an
exact Gitea revision and digest manifest.
- Gitea offline backup and container lifecycle changes remain excluded;
`backup-gitea.sh` has no candidate diff from the production base.
## V8 Defects Closed
### Durable Receipt Transaction
The executor no longer captures `write_receipt` through command substitution
and cannot mask an internal failure with a later `printf`. Receipt publication
uses an exclusive temporary file, exact full-write check, file `fsync`,
temporary-byte SHA-256 readback, no-replace hard link, directory `fsync`, exact
final-byte/document/SHA-256 readback, temporary unlink, and a second directory
`fsync`. Any write, link, fsync, cleanup, or readback failure is nonzero and
removes a final path created by that failed attempt when possible.
`APPLY_COMPLETE=1` is set only after the durable verified receipt passes this
transaction. Stage and rollback prestate are never cleared before that point.
If verified receipt publication fails, the EXIT path performs rollback first,
writes a distinct failure receipt, and only removes rollback prestate after
both rollback verification and durable failure-receipt publication succeed.
Payload files, destination directories, rollback prestate files, and the
rollback directory are explicitly fsynced before their corresponding durable
claim can be published.
### Run-Owned Temporary Files
Every apply and rollback temporary destination is tracked for the exact run.
Rollback removes and reads back all tracked paths, including a failure between
`install` and `mv`. Canonical content/metadata verification plus zero hidden
residue are both required before rollback can be called verified.
### Rollback Truth
Receipt schema `agent99_host110_backup_runtime_deploy_receipt_v2` records
`rollbackAttempted`, `rollbackPerformed`, `rollbackVerified`, and
`zeroResidueVerified` independently. A partial, unknown, or residue-bearing
rollback terminates as `rollback_unverified`; only a fully restored and
zero-residue transaction may terminate as `failed_rolled_back`.
### Producer/Consumer Parity
The integration test parses both actual sources and requires the broker's 18
payload basenames to equal the executor payload in count, exact set, and exact
order. It separately requires executor, verifier, and `manifest.json`, yielding
21 unique remote-stage basenames.
## Fault-Injection Contract
The bounded local harness executes the real executor control flow against
isolated temporary destinations and covers:
- successful apply, durable receipt, verifier, and post-receipt prestate cleanup;
- receipt write, link, fsync, and final-readback failures;
- apply failure after hidden-temp installation but before canonical `mv`;
- forced temp-cleanup failure and `rollback_unverified` evidence preservation.
Every receipt fault must restore exact prestate, emit no success JSON, and
produce a durable `failed_rolled_back` receipt. A cleanup fault must preserve
`prestate.tsv`, expose rollback attempted/performed separately, and must not
claim rollback verification.
Fault hooks require an explicit test enable flag and a transformed
non-production destination. The canonical `/backup/scripts` executor can never
enable them, even if a fault environment variable is present.
## Exact Diff Serialization
Substitute the V9 Gitea live-ref SHA for `<candidate_git_sha>` and hash the
exact stdout bytes from this command:
```sh
git diff --binary --full-index \
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
shasum -a 256
```
The review message independently binds the Gitea live ref and Git SHA, this
tracked artifact path and bytes hash, and the serialized diff hash.
## Apply Effects
- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded
manifest and receipt state.
- Host110 Apply writes/replaces the 18 payload paths through one exclusive,
digest-bound filesystem transaction.
- Apply does not run a backup, invoke a Gitea container lifecycle, sync Google
Drive, prune snapshots, restart a VM or service, or change a database,
network, or firewall.
## Validation
- The exact broker payload/order and 21-file stage parity test must pass.
- Success and all six fault-injection paths must execute and pass.
- Focused runtime, broker, backup, parity, Ansible, shell, and Python checks
must pass.
- Changed PowerShell must parse on Windows99 with zero errors when applicable.
- Central review must independently reproduce the live ref, artifact hash,
exact diff hash, 19/18/21 counts, and zero `backup-gitea.sh` diff.
## Rollback
- Source rollback reverts the exact V9 diff to
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
- Agent99 rollback restores the prior runtime files and manifest and verifies
prior hashes independently.
- Host110 rollback restores all prior payload content, SHA-256, uid, gid, and
mode under the exclusive lock and proves zero run-owned temporary residue.
- Any canonical, metadata, receipt, or residue mismatch terminates as
`rollback_unverified`; rollback prestate remains available for diagnosis.

View File

@@ -304,7 +304,7 @@ release_gate:
strict_dr_exit_conditions:
- "Google Drive/rclone provider configured on 110 host-local rclone.conf; /backup/scripts/offsite.env keeps only non-secret remote/path with mode 0600"
- "credential escrow markers fresh for restic_repository_password, offsite_provider_credentials, break_glass_admin_credentials, dns_registrar_recovery, oauth_ai_provider_recovery"
- "full offsite marker /backup/offsite/rclone-last-success fresh after full 13 repo sync"
- "full offsite marker /backup/offsite/rclone-last-success fresh after full 14 repo sync"
- "full-stack-recovery-scorecard.sh --require-dr exits 0"
- "recovery-scorecard-contract-check.py --expect-dr-ready exits 0 against 110 Prometheus"
- "dr-offsite-operator-checklist.sh --require-dr exits 0 after scorecard, Prometheus recording rule, and backup alert visibility contract agree"

View File

@@ -9,6 +9,16 @@
set -euo pipefail
if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then
command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; }
[ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; }
(umask 077; : >> /tmp/agent99-host110-backup-runtime.lock)
[ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; }
exec 197>>/tmp/agent99-host110-backup-runtime.lock
flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; }
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
fi
BACKUP_BASE="${BACKUP_BASE:-/backup}"
OFFSITE_ENV_FILE="${BACKUP_OFFSITE_ENV_FILE:-${BACKUP_BASE}/scripts/offsite.env}"
OFFSITE_DIR="${BACKUP_OFFSITE_STATUS_DIR:-${BACKUP_BASE}/offsite}"
@@ -189,6 +199,7 @@ check_offsite_env() {
check_configured() {
load_offsite_env
bind_rclone_config_to_runtime_user
if command -v rclone >/dev/null 2>&1; then
ok "rclone command is available"
else

View File

@@ -1,5 +1,38 @@
#\!/bin/bash
#!/bin/bash
# =============================================================================
# Production backup entrypoints share this stable gate. Agent99 takes the
# exclusive side while promoting a complete, digest-bound runtime bundle.
acquire_backup_runtime_shared_lock() {
local caller_path="${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}"
local lock_path="/tmp/agent99-host110-backup-runtime.lock"
case "${caller_path}" in
/backup/scripts/*) ;;
*) return 0 ;;
esac
[ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ] || return 0
command -v flock >/dev/null 2>&1 || {
echo "backup runtime gate unavailable: flock missing" >&2
return 69
}
[ ! -L "${lock_path}" ] || {
echo "backup runtime gate unsafe: lock is a symlink" >&2
return 69
}
(umask 077; : >> "${lock_path}") || return 69
[ -f "${lock_path}" ] && [ -O "${lock_path}" ] || return 69
exec 197>>"${lock_path}"
flock -s -w 60 197 || {
echo "backup runtime deployment is active; retry later" >&2
return 75
}
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
}
acquire_backup_runtime_shared_lock || {
runtime_lock_status=$?
return "${runtime_lock_status}" 2>/dev/null || exit "${runtime_lock_status}"
}
# WOOO AIOps - 備份共用函式庫
# 版本: 1.0.0
# 建立日期: 2026-03-12
@@ -30,6 +63,22 @@ export BACKUP_DOCKER_CPUS="${BACKUP_DOCKER_CPUS:-1.0}"
export BACKUP_DOCKER_MEMORY="${BACKUP_DOCKER_MEMORY:-1g}"
export BACKUP_DOCKER_MEMORY_SWAP="${BACKUP_DOCKER_MEMORY_SWAP:-1g}"
# Cron and service managers do not always preserve HOME. Bind rclone to the
# runtime account's config path when the operator did not provide an explicit
# RCLONE_CONFIG, without reading or printing the credential-bearing file.
bind_rclone_config_to_runtime_user() {
[ -z "${RCLONE_CONFIG:-}" ] || return 0
command -v getent >/dev/null 2>&1 || return 0
local runtime_home
local candidate
runtime_home="$(getent passwd "$(id -u)" 2>/dev/null | awk -F: 'NR == 1 {print $6}')"
candidate="${runtime_home}/.config/rclone/rclone.conf"
if [ -n "${runtime_home}" ] && [ -r "${candidate}" ]; then
export RCLONE_CONFIG="${candidate}"
fi
}
# -----------------------------------------------------------------------------
# 日誌函式
# -----------------------------------------------------------------------------

View File

@@ -5,6 +5,16 @@
set -euo pipefail
if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then
command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; }
[ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; }
(umask 077; : >> /tmp/agent99-host110-backup-runtime.lock)
[ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; }
exec 197>>/tmp/agent99-host110-backup-runtime.lock
flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; }
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
fi
DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}"
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
TEXTFILE_PATH="${AIOPS_GITEA_FULL_BACKUP_RESTORE_DRILL_TEXTFILE:-/home/wooo/node_exporter_textfiles/gitea_full_backup_restore_drill.prom}"

View File

@@ -0,0 +1,693 @@
#!/usr/bin/env bash
# Fixed Host110 executor for Windows99 Agent99 backup-runtime deployment.
set -euo pipefail
umask 077
readonly EXPECTED_HOST_IP="192.168.0.110"
readonly EXPECTED_USER="wooo"
readonly DEST_ROOT="/backup/scripts"
readonly EXPORTER_ROOT="/home/wooo/scripts"
readonly EXPORTER_FILE="backup-health-textfile-exporter.py"
readonly STATUS_ROOT="/backup/status"
readonly STAGE_ROOT="/backup/.agent99-backup-runtime-stage"
readonly ROLLBACK_ROOT="/backup/.agent99-backup-runtime-rollback"
readonly LOCK_PATH="/tmp/agent99-host110-backup-runtime.lock"
readonly -a RUNTIME_FILES=(
common.sh
backup-all.sh
backup-host188-products.sh
verify-host188-products-backup.sh
verify-host188-products-archive.py
backup-gitea.sh
gitea-full-backup-restore-drill.sh
backup-configs.sh
backup-awoooi.sh
backup-awoooi-frequent.sh
backup-clawbot.sh
backup-sentry.sh
check-backup-integrity.sh
sync-offsite-backups.sh
backup-offsite-readiness-gate.sh
verify-offsite-full-sync.sh
enforce-latest-only-retention.sh
)
readonly -a PAYLOAD_FILES=("${RUNTIME_FILES[@]}" "$EXPORTER_FILE")
MODE=""
SOURCE_REVISION=""
RUN_ID=""
SOURCE_STAGE=""
SOURCE_HEAD=""
EXECUTOR_DIGEST=""
VERIFIER_DIGEST=""
STAGE_DIR=""
ROLLBACK_DIR=""
RECEIPT_PATH=""
APPLY_STARTED=0
APPLY_COMPLETE=0
ROLLBACK_ATTEMPTED=0
ROLLBACK_PERFORMED=0
ROLLBACK_VERIFIED=0
RUN_TEMP_RESIDUE_VERIFIED=0
FAULT_INJECTION_ENABLED=0
declare -A FILE_EXISTED=()
declare -A PREVIOUS_DIGEST=()
declare -A PREVIOUS_METADATA=()
declare -A EXPECTED_DIGEST=()
declare -A RUN_OWNED_TEMP_PATHS=()
usage() {
printf '%s\n' \
"Usage: $0 --mode check|apply|verify --source-revision SHA --run-id ID --source-stage PATH" \
"The executor is fixed to host 192.168.0.110, an Agent99 run-bound source stage," \
"and the 18-file transaction: 17 /backup/scripts files plus the backup health exporter."
}
fail() {
printf 'HOST110_BACKUP_RUNTIME_OK=0\nERROR=%s\n' "$1" >&2
exit 64
}
while [ "$#" -gt 0 ]; do
case "$1" in
--mode)
shift
MODE="${1:-}"
;;
--source-revision)
shift
SOURCE_REVISION="${1:-}"
;;
--run-id)
shift
RUN_ID="${1:-}"
;;
--source-stage)
shift
SOURCE_STAGE="${1:-}"
;;
-h|--help)
usage
exit 0
;;
*) fail "unknown_argument" ;;
esac
shift
done
case "$MODE" in check|apply|verify) ;; *) fail "invalid_mode" ;; esac
[[ "$SOURCE_REVISION" =~ ^[0-9a-f]{40}$ ]] || fail "invalid_source_revision"
[[ "$RUN_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$ ]] || fail "invalid_run_id"
expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"
[ "$SOURCE_STAGE" = "$expected_source_stage" ] || fail "invalid_source_stage"
RECEIPT_PATH="$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json"
if [ "${HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS:-}" = "1" ] \
&& [ "$DEST_ROOT" != "/backup/scripts" ]; then
FAULT_INJECTION_ENABLED=1
fi
for command_name in awk bash cp dirname flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do
command -v "$command_name" >/dev/null 2>&1 || fail "required_command_missing_${command_name}"
done
current_user="$(id -un)"
[ "$current_user" = "$EXPECTED_USER" ] || fail "unexpected_executor_user"
hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx "$EXPECTED_HOST_IP" || fail "unexpected_executor_host"
[ -d "$SOURCE_STAGE" ] && [ ! -L "$SOURCE_STAGE" ] || fail "canonical_source_stage_unavailable"
[ -d "$DEST_ROOT" ] && [ ! -L "$DEST_ROOT" ] || fail "backup_runtime_destination_unavailable"
[ -d "$EXPORTER_ROOT" ] && [ ! -L "$EXPORTER_ROOT" ] || fail "backup_exporter_destination_unavailable"
manifest_rows="$(python3 - "$SOURCE_STAGE/manifest.json" "$SOURCE_REVISION" "$RUN_ID" "${PAYLOAD_FILES[@]}" <<'PY'
import json
import re
import sys
from pathlib import Path
path = Path(sys.argv[1])
source_revision = sys.argv[2]
run_id = sys.argv[3]
expected_names = sys.argv[4:]
try:
document = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
raise SystemExit(65)
rows = document.get("files")
if (
document.get("schemaVersion") != "agent99_host110_backup_runtime_source_manifest_v1"
or document.get("sourceRevision") != source_revision
or document.get("runId") != run_id
or not re.fullmatch(r"[0-9a-f]{40}", str(document.get("sourceHead", "")))
or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("executorSha256", "")))
or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("verifierSha256", "")))
or not isinstance(rows, list)
or len(rows) != len(expected_names)
):
raise SystemExit(66)
by_name = {}
for row in rows:
if not isinstance(row, dict):
raise SystemExit(66)
name = str(row.get("name", ""))
digest = str(row.get("sha256", ""))
if name in by_name or not re.fullmatch(r"[A-Za-z0-9_.-]+", name) or not re.fullmatch(r"[0-9a-f]{64}", digest):
raise SystemExit(66)
by_name[name] = digest
if set(by_name) != set(expected_names):
raise SystemExit(66)
print(f"SOURCE_HEAD {document['sourceHead']}")
print(f"EXECUTOR_SHA256 {document['executorSha256']}")
print(f"VERIFIER_SHA256 {document['verifierSha256']}")
for name in expected_names:
print(f"{name} {by_name[name]}")
PY
)" || fail "source_manifest_invalid"
while read -r manifest_name manifest_digest; do
case "$manifest_name" in
SOURCE_HEAD) SOURCE_HEAD="$manifest_digest" ;;
EXECUTOR_SHA256) EXECUTOR_DIGEST="$manifest_digest" ;;
VERIFIER_SHA256) VERIFIER_DIGEST="$manifest_digest" ;;
*) EXPECTED_DIGEST[$manifest_name]="$manifest_digest" ;;
esac
done <<< "$manifest_rows"
[[ "$SOURCE_HEAD" =~ ^[0-9a-f]{40}$ ]] || fail "source_head_invalid"
working_digest() {
sha256sum "$1" | awk '{print $1}'
}
[ "$(working_digest "$0")" = "$EXECUTOR_DIGEST" ] || fail "executor_identity_failed"
[ -f "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \
&& [ ! -L "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \
&& [ "$(working_digest "$SOURCE_STAGE/verify-host110-backup-runtime.py")" = "$VERIFIER_DIGEST" ] \
|| fail "verifier_identity_failed"
validate_file() {
local validation_path="$1"
case "$validation_path" in
*.sh) bash -n "$validation_path" ;;
*.py) python3 -c 'import ast, pathlib, sys; ast.parse(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))' "$validation_path" ;;
*) return 64 ;;
esac
}
source_digest() {
local name="$1"
printf '%s\n' "${EXPECTED_DIGEST[$name]:-}"
}
validate_source() {
local name source_path expected_digest actual_digest resolved
for name in "${PAYLOAD_FILES[@]}"; do
source_path="$SOURCE_STAGE/$name"
[ -f "$source_path" ] && [ ! -L "$source_path" ] || return 65
resolved="$(readlink -f "$source_path")"
[[ "$resolved" == "$SOURCE_STAGE/"* ]] || return 65
expected_digest="$(source_digest "$name")"
actual_digest="$(working_digest "$source_path")"
[ "$actual_digest" = "$expected_digest" ] || return 66
validate_file "$source_path" || return 67
done
}
destination_path() {
local name="$1"
if [ "$name" = "$EXPORTER_FILE" ]; then
printf '%s/%s\n' "$EXPORTER_ROOT" "$name"
else
printf '%s/%s\n' "$DEST_ROOT" "$name"
fi
}
verify_destination() {
local name dest_path expected_digest actual_digest
for name in "${PAYLOAD_FILES[@]}"; do
dest_path="$(destination_path "$name")"
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] || return 68
expected_digest="$(source_digest "$name")"
actual_digest="$(working_digest "$dest_path")"
[ "$actual_digest" = "$expected_digest" ] || return 69
[ "$(stat -c '%U:%G:%a' "$dest_path")" = "wooo:wooo:755" ] || return 70
done
}
fsync_paths_and_parents() {
python3 - "$@" <<'PY'
import os
import stat
import sys
from pathlib import Path
parents = set()
for raw_path in sys.argv[1:]:
path = Path(raw_path)
parents.add(path.parent)
if not os.path.lexists(path):
continue
if path.is_symlink() or not path.is_file():
raise SystemExit(76)
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(path, flags)
try:
if not stat.S_ISREG(os.fstat(descriptor).st_mode):
raise SystemExit(76)
os.fsync(descriptor)
finally:
os.close(descriptor)
for parent in sorted(parents, key=str):
if parent.is_symlink() or not parent.is_dir():
raise SystemExit(76)
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(parent, flags)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
PY
}
fsync_destination_state() {
local name dest_path
local -a fsync_targets=()
for name in "${PAYLOAD_FILES[@]}"; do
dest_path="$(destination_path "$name")"
fsync_targets+=("$dest_path")
done
fsync_paths_and_parents "${fsync_targets[@]}"
}
fsync_rollback_prestate() {
local name
local -a fsync_targets=("$ROLLBACK_DIR/prestate.tsv")
for name in "${PAYLOAD_FILES[@]}"; do
if [ "${FILE_EXISTED[$name]:-}" = "1" ]; then
fsync_targets+=("$ROLLBACK_DIR/$name")
fi
done
fsync_paths_and_parents "${fsync_targets[@]}"
}
write_receipt() {
local receipt_status="$1"
local rollback_attempted="$2"
local rollback_performed="$3"
local rollback_verified="$4"
local zero_residue_verified="$5"
install -d -m 700 "$STATUS_ROOT" || return 1
[ -d "$STATUS_ROOT" ] && [ ! -L "$STATUS_ROOT" ] || return 1
[ "$(stat -c '%U:%G:%a' "$STATUS_ROOT")" = "wooo:wooo:700" ] || return 1
if ! python3 - "$RECEIPT_PATH" "$receipt_status" "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" \
"$rollback_attempted" "$rollback_performed" "$rollback_verified" \
"$zero_residue_verified" "$VERIFIER_DIGEST" "$FAULT_INJECTION_ENABLED" <<'PY'
import hashlib
import json
import os
import sys
import time
from pathlib import Path
path = Path(sys.argv[1])
status = sys.argv[2]
rollback_attempted = sys.argv[6] == "1"
rollback_performed = sys.argv[7] == "1"
rollback_verified = sys.argv[8] == "1"
zero_residue_verified = sys.argv[9] == "1"
if status not in {"verified", "failed_rolled_back", "rollback_unverified"}:
raise SystemExit(78)
if status == "verified" and (rollback_attempted or rollback_performed or rollback_verified or not zero_residue_verified):
raise SystemExit(78)
if status == "failed_rolled_back" and not (
rollback_attempted and rollback_performed and rollback_verified and zero_residue_verified
):
raise SystemExit(78)
if status == "rollback_unverified" and (not rollback_attempted or rollback_verified):
raise SystemExit(78)
document = {
"schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v2",
"status": status,
"sourceRevision": sys.argv[3],
"sourceHead": sys.argv[4],
"runId": sys.argv[5],
"fileCount": 18,
"backupScriptFileCount": 17,
"backupHealthExporterIncluded": True,
"rollbackAttempted": rollback_attempted,
"rollbackPerformed": rollback_performed,
"rollbackVerified": rollback_verified,
"zeroResidueVerified": zero_residue_verified,
"verifierSha256": sys.argv[10],
"executorHost": "192.168.0.110",
"productionServiceRestarted": False,
"secretValuesRead": False,
"writtenAt": int(time.time()),
}
payload = (json.dumps(document, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")
expected_digest = hashlib.sha256(payload).hexdigest()
temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
fault_enabled = sys.argv[11] == "1"
fault = (
os.environ.get("HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT", "")
if status == "verified" and fault_enabled
else ""
)
if fault not in {"", "write", "link", "fsync", "readback"}:
raise SystemExit(78)
created_final = False
directory_fd = None
try:
if os.path.lexists(path) or os.path.lexists(temporary):
raise FileExistsError(path)
if fault == "write":
raise OSError("fault_injected_receipt_write")
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "wb") as handle:
if handle.write(payload) != len(payload):
raise OSError("receipt_short_write")
handle.flush()
if fault == "fsync":
raise OSError("fault_injected_receipt_fsync")
os.fsync(handle.fileno())
os.chmod(temporary, 0o600)
temporary_readback = temporary.read_bytes()
if temporary_readback != payload or hashlib.sha256(temporary_readback).hexdigest() != expected_digest:
raise OSError("receipt_temporary_readback_failed")
if fault == "link":
raise OSError("fault_injected_receipt_link")
os.link(temporary, path)
created_final = True
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
directory_fd = os.open(path.parent, directory_flags)
os.fsync(directory_fd)
if fault == "readback":
raise OSError("fault_injected_receipt_readback")
readback = path.read_bytes()
if readback != payload or hashlib.sha256(readback).hexdigest() != expected_digest:
raise OSError("receipt_final_readback_failed")
if json.loads(readback.decode("utf-8")) != document:
raise OSError("receipt_document_readback_failed")
temporary.unlink()
os.fsync(directory_fd)
except BaseException:
try:
if created_final:
path.unlink(missing_ok=True)
temporary.unlink(missing_ok=True)
if directory_fd is None and path.parent.is_dir():
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
directory_fd = os.open(path.parent, directory_flags)
if directory_fd is not None:
os.fsync(directory_fd)
finally:
raise
finally:
if directory_fd is not None:
os.close(directory_fd)
PY
then
return 1
fi
[ -f "$RECEIPT_PATH" ] && [ ! -L "$RECEIPT_PATH" ] || return 1
return 0
}
record_prestate() {
local name dest_path digest metadata
: > "$ROLLBACK_DIR/prestate.tsv" || return 74
chmod 600 "$ROLLBACK_DIR/prestate.tsv" || return 74
for name in "${PAYLOAD_FILES[@]}"; do
dest_path="$(destination_path "$name")"
if [ -f "$dest_path" ] && [ ! -L "$dest_path" ]; then
[ "$(stat -c '%U:%G' "$dest_path")" = "wooo:wooo" ] || return 71
FILE_EXISTED[$name]=1
digest="$(working_digest "$dest_path")" || return 72
metadata="$(stat -c '%u:%g:%a' "$dest_path")" || return 72
PREVIOUS_DIGEST[$name]="$digest"
PREVIOUS_METADATA[$name]="$metadata"
printf '%s\t1\t%s\t%s\n' "$name" "$digest" "$metadata" >> "$ROLLBACK_DIR/prestate.tsv" || return 72
cp -p -- "$dest_path" "$ROLLBACK_DIR/$name" || return 72
[ "$(working_digest "$ROLLBACK_DIR/$name")" = "$digest" ] || return 72
elif [ ! -e "$dest_path" ]; then
FILE_EXISTED[$name]=0
PREVIOUS_DIGEST[$name]="-"
PREVIOUS_METADATA[$name]="-"
printf '%s\t0\t-\t-\n' "$name" >> "$ROLLBACK_DIR/prestate.tsv" || return 72
else
return 73
fi
done
fsync_rollback_prestate || return 75
}
register_run_owned_temp() {
RUN_OWNED_TEMP_PATHS["$1"]=1
}
verify_run_owned_temp_absence() {
local temporary
for temporary in "${!RUN_OWNED_TEMP_PATHS[@]}"; do
[ ! -e "$temporary" ] && [ ! -L "$temporary" ] || return 1
done
}
cleanup_run_owned_temps() {
local temporary failed=0 preserved=0
local fault=""
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ]; then
fault="${HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT:-}"
fi
for temporary in "${!RUN_OWNED_TEMP_PATHS[@]}"; do
if [ "$fault" = "preserve_first_temp" ] \
&& [ "$preserved" -eq 0 ] \
&& { [ -e "$temporary" ] || [ -L "$temporary" ]; }; then
preserved=1
failed=1
continue
fi
rm -f -- "$temporary" || failed=1
done
verify_run_owned_temp_absence || failed=1
[ "$failed" -eq 0 ]
}
rollback_transaction() {
local name dest_path destination_parent backup_path temporary failed=0
ROLLBACK_ATTEMPTED=1
ROLLBACK_PERFORMED=0
ROLLBACK_VERIFIED=0
RUN_TEMP_RESIDUE_VERIFIED=0
set +e
cleanup_run_owned_temps || failed=1
for name in "${PAYLOAD_FILES[@]}"; do
[ -n "${FILE_EXISTED[$name]+x}" ] || { failed=1; continue; }
dest_path="$(destination_path "$name")"
if [ "${FILE_EXISTED[$name]}" = "1" ]; then
backup_path="$ROLLBACK_DIR/$name"
destination_parent="$(dirname "$dest_path")"
temporary="$destination_parent/.${name}.agent99-rollback-${RUN_ID}"
register_run_owned_temp "$temporary"
if cp -p -- "$backup_path" "$temporary" && mv -f -- "$temporary" "$dest_path"; then
ROLLBACK_PERFORMED=1
else
failed=1
fi
else
if rm -f -- "$dest_path"; then
ROLLBACK_PERFORMED=1
else
failed=1
fi
fi
done
cleanup_run_owned_temps || failed=1
fsync_destination_state || failed=1
for name in "${PAYLOAD_FILES[@]}"; do
dest_path="$(destination_path "$name")"
if [ "${FILE_EXISTED[$name]:-}" = "1" ]; then
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] \
&& [ "$(working_digest "$dest_path" 2>/dev/null)" = "${PREVIOUS_DIGEST[$name]:-invalid}" ] \
&& [ "$(stat -c '%u:%g:%a' "$dest_path" 2>/dev/null)" = "${PREVIOUS_METADATA[$name]:-invalid}" ] \
|| failed=1
elif [ "${FILE_EXISTED[$name]:-}" = "0" ]; then
[ ! -e "$dest_path" ] || failed=1
else
failed=1
fi
done
if verify_run_owned_temp_absence; then
RUN_TEMP_RESIDUE_VERIFIED=1
else
failed=1
RUN_TEMP_RESIDUE_VERIFIED=0
fi
set -e
if [ "$failed" -eq 0 ] && [ "$ROLLBACK_PERFORMED" -eq 1 ] && [ "$RUN_TEMP_RESIDUE_VERIFIED" -eq 1 ]; then
ROLLBACK_VERIFIED=1
return 0
fi
ROLLBACK_VERIFIED=0
return 1
}
cleanup() {
local exit_status=$?
local rollback_status="rollback_unverified"
local failure_receipt_written=0
trap - EXIT HUP INT TERM
if [ "$APPLY_STARTED" -eq 1 ] && [ "$APPLY_COMPLETE" -ne 1 ]; then
if rollback_transaction; then
rollback_status="failed_rolled_back"
fi
if write_receipt "$rollback_status" "$ROLLBACK_ATTEMPTED" "$ROLLBACK_PERFORMED" \
"$ROLLBACK_VERIFIED" "$RUN_TEMP_RESIDUE_VERIFIED"; then
failure_receipt_written=1
fi
fi
if [ -n "$STAGE_DIR" ]; then
rm -rf -- "$STAGE_DIR" || true
fi
if [ "$ROLLBACK_VERIFIED" -eq 1 ] && [ "$failure_receipt_written" -eq 1 ]; then
if [ -n "$ROLLBACK_DIR" ]; then
rm -rf -- "$ROLLBACK_DIR" || true
fi
fi
exit "$exit_status"
}
validate_source || fail "source_validation_failed"
if [ "$MODE" = "check" ]; then
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
exit 0
fi
if [ "$MODE" = "verify" ]; then
verify_destination || fail "destination_verification_failed"
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":18,"backupScriptFileCount":17,"backupHealthExporterIncluded":true,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
exit 0
fi
[ ! -e "$RECEIPT_PATH" ] && [ ! -L "$RECEIPT_PATH" ] || fail "run_identity_receipt_exists"
STAGE_DIR="$STAGE_ROOT/$RUN_ID"
ROLLBACK_DIR="$ROLLBACK_ROOT/$RUN_ID"
[ ! -e "$STAGE_DIR" ] || fail "run_identity_stage_exists"
[ ! -e "$ROLLBACK_DIR" ] || fail "run_identity_rollback_exists"
[ ! -L "$LOCK_PATH" ] || fail "deployment_lock_symlink_unsafe"
(umask 077; : >> "$LOCK_PATH") || fail "deployment_lock_unavailable"
[ -f "$LOCK_PATH" ] && [ -O "$LOCK_PATH" ] || fail "deployment_lock_owner_invalid"
exec 9>>"$LOCK_PATH"
flock -x -w 30 9 || fail "deployment_lock_timeout"
# Old runtime revisions predate the shared gate. Refuse the first promotion if
# one of them is already active; after common.sh moves first, new starts block.
if pgrep -afu "$EXPECTED_USER" '/backup/scripts/(backup-|check-backup|sync-offsite|verify-offsite|enforce-latest)' \
| grep -vF "host110-backup-runtime-executor.sh" >/dev/null 2>&1; then
fail "legacy_backup_runtime_active"
fi
install -d -m 700 "$STAGE_DIR" "$ROLLBACK_DIR"
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM HUP
for name in "${PAYLOAD_FILES[@]}"; do
install -m 700 "$SOURCE_STAGE/$name" "$STAGE_DIR/$name"
[ "$(working_digest "$STAGE_DIR/$name")" = "$(source_digest "$name")" ] || fail "stage_identity_failed"
validate_file "$STAGE_DIR/$name" || fail "stage_validation_failed"
done
record_prestate || fail "rollback_prestate_capture_failed"
APPLY_STARTED=1
# common.sh is promoted first. Every managed mutating backup entrypoint then
# takes the shared side of LOCK_PATH before doing work, while this process holds
# the exclusive side until the independent verifier has completed.
for name in "${PAYLOAD_FILES[@]}"; do
dest_path="$(destination_path "$name")"
destination_parent="$(dirname "$dest_path")"
temporary="$destination_parent/.${name}.agent99-${RUN_ID}"
register_run_owned_temp "$temporary"
install -m 755 "$STAGE_DIR/$name" "$temporary"
if [ "$FAULT_INJECTION_ENABLED" -eq 1 ] \
&& [ "${HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT:-}" = "after_temp_install" ] \
&& [ "$name" = "${PAYLOAD_FILES[0]}" ]; then
fail "fault_injected_after_temp_install"
fi
mv -f -- "$temporary" "$dest_path"
done
fsync_destination_state || fail "post_apply_durability_failed"
verify_destination || fail "post_apply_verification_failed"
verifier_result="$(BACKUP_RUNTIME_DEPLOY_CONTEXT=1 python3 "$SOURCE_STAGE/verify-host110-backup-runtime.py" \
--manifest "$SOURCE_STAGE/manifest.json" \
--destination "$DEST_ROOT" \
--exporter-destination "$EXPORTER_ROOT/$EXPORTER_FILE" \
--source-revision "$SOURCE_REVISION" \
--run-id "$RUN_ID" \
--expected-verifier-sha256 "$VERIFIER_DIGEST")" || fail "independent_verifier_failed"
python3 - "$verifier_result" "$SOURCE_REVISION" "$RUN_ID" <<'PY' || fail "independent_verifier_contract_failed"
import json
import sys
try:
row = json.loads(sys.argv[1])
except json.JSONDecodeError:
raise SystemExit(1)
if not (
row.get("schemaVersion") == "agent99_host110_backup_runtime_verifier_v1"
and row.get("ok") is True
and row.get("sourceRevision") == sys.argv[2]
and row.get("runId") == sys.argv[3]
and row.get("fileCount") == 18
and row.get("backupScriptFileCount") == 17
and row.get("backupHealthExporterIncluded") is True
and row.get("remoteWritePerformed") is False
and row.get("selfIdentityVerified") is True
):
raise SystemExit(1)
PY
verify_run_owned_temp_absence || fail "post_apply_temp_residue_detected"
RUN_TEMP_RESIDUE_VERIFIED=1
if ! write_receipt "verified" 0 0 0 "$RUN_TEMP_RESIDUE_VERIFIED"; then
fail "durable_receipt_failed"
fi
receipt_path="$RECEIPT_PATH"
APPLY_COMPLETE=1
trap - EXIT HUP INT TERM
stage_cleanup_verified=0
rollback_prestate_cleanup_verified=0
if rm -rf -- "$STAGE_DIR" && [ ! -e "$STAGE_DIR" ] && [ ! -L "$STAGE_DIR" ]; then
stage_cleanup_verified=1
fi
if rm -rf -- "$ROLLBACK_DIR" && [ ! -e "$ROLLBACK_DIR" ] && [ ! -L "$ROLLBACK_DIR" ]; then
rollback_prestate_cleanup_verified=1
fi
STAGE_DIR=""
ROLLBACK_DIR=""
python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$receipt_path" "$verifier_result" \
"$stage_cleanup_verified" "$rollback_prestate_cleanup_verified" <<'PY'
import json
import sys
print(json.dumps({
"schemaVersion": "agent99_host110_backup_runtime_executor_v1",
"mode": "apply",
"ok": True,
"sourceRevision": sys.argv[1],
"sourceHead": sys.argv[2],
"fileCount": 18,
"backupScriptFileCount": 17,
"backupHealthExporterIncluded": True,
"rollbackAttempted": False,
"rollbackPerformed": False,
"rollbackVerified": False,
"zeroResidueVerified": True,
"stageCleanupVerified": sys.argv[5] == "1",
"rollbackPrestateCleanupVerified": sys.argv[6] == "1",
"receipt": sys.argv[3],
"verifier": json.loads(sys.argv[4]),
}, ensure_ascii=True, sort_keys=True))
PY

View File

@@ -121,6 +121,8 @@ prepare_rclone() {
return 1
fi
bind_rclone_config_to_runtime_user
if [ "${PROVIDER}" = "b2" ]; then
if ! check_b2_config; then
return 1

View File

@@ -0,0 +1,95 @@
from __future__ import annotations
import os
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts" / "backup" / "verify-offsite-full-sync.sh"
def run_verifier(tmp_path: Path, mode: str) -> tuple[subprocess.CompletedProcess[str], str, Path]:
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
runtime_home = tmp_path / "runtime-home"
config = runtime_home / ".config" / "rclone" / "rclone.conf"
config.parent.mkdir(parents=True)
config.write_text("test fixture only\n", encoding="utf-8")
event_log = tmp_path / "rclone.events"
getent = bin_dir / "getent"
getent.write_text(
"#!/bin/sh\nprintf 'wooo:x:1000:1000::%s:/bin/bash\\n' \"$TEST_RUNTIME_HOME\"\n",
encoding="utf-8",
)
getent.chmod(0o755)
rclone = bin_dir / "rclone"
rclone.write_text(
"""#!/bin/sh
printf 'CMD=%s CONFIG=%s\n' "$*" "${RCLONE_CONFIG:-}" >> "$TEST_EVENT_LOG"
case "$1" in
listremotes)
[ "$TEST_RCLONE_MODE" = config_missing ] && printf '%s\n' 'other:' || printf '%s\n' 'gdrive:'
;;
lsf)
if [ "$TEST_RCLONE_MODE" = remote_down ]; then
exit 9
fi
case "$2" in
*/snapshots) printf '%s\n' snapshot-file ;;
*) printf '%s\n' awoooi/ ;;
esac
;;
esac
""",
encoding="utf-8",
)
rclone.chmod(0o755)
env = os.environ.copy()
env.pop("RCLONE_CONFIG", None)
env.update(
{
"BACKUP_BASE": str(tmp_path / "backup"),
"HOME": "/",
"OFFSITE_REPOS": "awoooi",
"PATH": f"{bin_dir}:{env['PATH']}",
"TEST_EVENT_LOG": str(event_log),
"TEST_RCLONE_MODE": mode,
"TEST_RUNTIME_HOME": str(runtime_home),
}
)
result = subprocess.run(
["bash", str(SCRIPT), "--no-color"],
env=env,
text=True,
capture_output=True,
check=False,
)
return result, event_log.read_text(encoding="utf-8"), config
def test_verifier_binds_rclone_config_to_runtime_account(tmp_path: Path) -> None:
result, events, config = run_verifier(tmp_path, "ok")
assert result.returncode == 1 # The full-sync marker is intentionally absent.
assert "rclone provider readback succeeded" in result.stdout
assert f"CONFIG={config}" in events
def test_verifier_distinguishes_missing_config_without_remote_calls(tmp_path: Path) -> None:
result, events, _ = run_verifier(tmp_path, "config_missing")
assert result.returncode == 1
assert "rclone remote configuration unavailable" in result.stdout
assert " lsf " not in f" {events} "
def test_verifier_distinguishes_provider_failure_and_stops_fanout(tmp_path: Path) -> None:
result, events, _ = run_verifier(tmp_path, "remote_down")
assert result.returncode == 1
assert "rclone provider readback failed" in result.stdout
assert events.count("CMD=lsf ") == 1

View File

@@ -0,0 +1,177 @@
#!/usr/bin/env python3
"""Independent no-write verifier for the Host110 backup runtime transaction."""
from __future__ import annotations
import argparse
import base64
import fcntl
import hashlib
import json
import os
import re
import sys
import time
from pathlib import Path
EXPECTED_DESTINATION = Path("/backup/scripts")
EXPECTED_EXPORTER_DESTINATION = Path("/home/wooo/scripts/backup-health-textfile-exporter.py")
RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")
EXPECTED_BACKUP_FILES = (
"common.sh",
"backup-all.sh",
"backup-host188-products.sh",
"verify-host188-products-backup.sh",
"verify-host188-products-archive.py",
"backup-gitea.sh",
"gitea-full-backup-restore-drill.sh",
"backup-configs.sh",
"backup-awoooi.sh",
"backup-awoooi-frequent.sh",
"backup-clawbot.sh",
"backup-sentry.sh",
"check-backup-integrity.sh",
"sync-offsite-backups.sh",
"backup-offsite-readiness-gate.sh",
"verify-offsite-full-sync.sh",
"enforce-latest-only-retention.sh",
)
EXPORTER_FILE = "backup-health-textfile-exporter.py"
EXPECTED_FILES = EXPECTED_BACKUP_FILES + (EXPORTER_FILE,)
def fail(reason: str) -> None:
print(f"HOST110_BACKUP_RUNTIME_VERIFY_OK=0\nERROR={reason}", file=sys.stderr)
raise SystemExit(1)
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def acquire_runtime_read_lock(destination: Path):
if destination != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_DEPLOY_CONTEXT") == "1":
return None
if RUNTIME_LOCK.is_symlink() or not RUNTIME_LOCK.is_file():
fail("runtime_lock_unavailable")
handle = RUNTIME_LOCK.open("rb")
if RUNTIME_LOCK.stat().st_uid != os.getuid():
handle.close()
fail("runtime_lock_owner_invalid")
deadline = time.monotonic() + 60
while True:
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
return handle
except BlockingIOError:
if time.monotonic() >= deadline:
handle.close()
fail("runtime_deployment_active")
time.sleep(0.2)
def main() -> None:
parser = argparse.ArgumentParser()
manifest_source = parser.add_mutually_exclusive_group(required=True)
manifest_source.add_argument("--manifest")
manifest_source.add_argument("--manifest-base64")
parser.add_argument("--destination", required=True)
parser.add_argument("--exporter-destination", required=True)
parser.add_argument("--source-revision", required=True)
parser.add_argument("--run-id", required=True)
parser.add_argument("--expected-verifier-sha256", required=True)
args = parser.parse_args()
if not re.fullmatch(r"[0-9a-f]{40}", args.source_revision):
fail("invalid_source_revision")
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}", args.run_id):
fail("invalid_run_id")
destination = Path(args.destination)
if destination != EXPECTED_DESTINATION or destination.is_symlink() or not destination.is_dir():
fail("invalid_destination")
exporter_destination = Path(args.exporter_destination)
if exporter_destination != EXPECTED_EXPORTER_DESTINATION:
fail("invalid_exporter_destination")
runtime_lock = acquire_runtime_read_lock(destination)
if not re.fullmatch(r"[0-9a-f]{64}", args.expected_verifier_sha256):
fail("invalid_verifier_identity")
try:
if args.manifest:
manifest_text = Path(args.manifest).read_text(encoding="utf-8")
else:
manifest_text = base64.b64decode(args.manifest_base64, validate=True).decode("utf-8")
document = json.loads(manifest_text)
except (OSError, UnicodeDecodeError, ValueError, json.JSONDecodeError):
fail("manifest_unavailable")
rows = document.get("files")
if (
document.get("schemaVersion") != "agent99_host110_backup_runtime_source_manifest_v1"
or document.get("sourceRevision") != args.source_revision
or document.get("runId") != args.run_id
or not re.fullmatch(r"[0-9a-f]{40}", str(document.get("sourceHead", "")))
or document.get("verifierSha256") != args.expected_verifier_sha256
or not isinstance(rows, list)
or len(rows) != len(EXPECTED_FILES)
):
fail("manifest_contract_failed")
by_name: dict[str, str] = {}
for row in rows:
if not isinstance(row, dict):
fail("manifest_file_set_failed")
name = str(row.get("name", ""))
if name in by_name:
fail("manifest_file_set_failed")
by_name[name] = str(row.get("sha256", ""))
if set(by_name) != set(EXPECTED_FILES) or not all(re.fullmatch(r"[0-9a-f]{64}", value) for value in by_name.values()):
fail("manifest_file_set_failed")
verifier_path = Path(__file__)
self_identity_verified = verifier_path.is_file() and sha256(verifier_path) == args.expected_verifier_sha256
if args.manifest and not self_identity_verified:
fail("verifier_identity_failed")
root = destination.resolve(strict=True)
for name in EXPECTED_FILES:
path = exporter_destination if name == EXPORTER_FILE else destination / name
if path.is_symlink() or not path.is_file():
fail("runtime_file_type_failed")
resolved = path.resolve(strict=True)
if name == EXPORTER_FILE:
if resolved != exporter_destination:
fail("runtime_file_boundary_failed")
elif resolved.parent != root:
fail("runtime_file_boundary_failed")
stat = resolved.stat()
if stat.st_uid != os.getuid() or stat.st_gid != os.getgid() or stat.st_mode & 0o777 != 0o755:
fail("runtime_file_metadata_failed")
if sha256(resolved) != by_name[name]:
fail("runtime_file_identity_failed")
result = {
"schemaVersion": "agent99_host110_backup_runtime_verifier_v1",
"ok": True,
"sourceRevision": args.source_revision,
"runId": args.run_id,
"fileCount": len(EXPECTED_FILES),
"backupScriptFileCount": len(EXPECTED_BACKUP_FILES),
"backupHealthExporterIncluded": True,
"destination": str(EXPECTED_DESTINATION),
"exporterDestination": str(EXPECTED_EXPORTER_DESTINATION),
"remoteWritePerformed": False,
"secretValuesRead": False,
"selfIdentityVerified": self_identity_verified,
"verifierSha256": args.expected_verifier_sha256,
}
print(json.dumps(result, ensure_ascii=True, sort_keys=True))
if runtime_lock is not None:
runtime_lock.close()
if __name__ == "__main__":
main()

View File

@@ -4,11 +4,14 @@
from __future__ import annotations
import argparse
import fcntl
import hashlib
import json
import os
import sqlite3
import tarfile
import tempfile
import time
from pathlib import Path, PurePosixPath
from typing import Any
@@ -20,6 +23,28 @@ EXPECTED_PREFIXES = {
"n8n-online-": ".tar.gz",
}
FIXED_ASSETS = {"n8n-validation.json"}
RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")
def acquire_runtime_lock():
if Path(__file__).resolve().parent != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_SHARED_LOCK_HELD") == "1":
return None
if RUNTIME_LOCK.is_symlink():
raise RuntimeError("backup_runtime_gate_unsafe")
handle = RUNTIME_LOCK.open("a+")
if RUNTIME_LOCK.stat().st_uid != os.getuid():
handle.close()
raise RuntimeError("backup_runtime_gate_owner_invalid")
deadline = time.monotonic() + 60
while True:
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
return handle
except BlockingIOError:
if time.monotonic() >= deadline:
handle.close()
raise RuntimeError("backup_runtime_deployment_active")
time.sleep(0.2)
def sha256_file(path: Path) -> str:
@@ -153,10 +178,13 @@ def validate_restore(root: Path) -> dict[str, Any]:
def main() -> int:
runtime_lock = acquire_runtime_lock()
parser = argparse.ArgumentParser()
parser.add_argument("--root", type=Path, required=True)
args = parser.parse_args()
print(json.dumps(validate_restore(args.root), separators=(",", ":")))
if runtime_lock is not None:
runtime_lock.close()
return 0

View File

@@ -113,11 +113,16 @@ low_priority() {
fi
}
rclone_ready() {
rclone_configured() {
command -v rclone >/dev/null 2>&1 || return 1
rclone listremotes 2>/dev/null | grep -Fxq "${RCLONE_REMOTE}:"
}
rclone_remote_readable() {
low_priority timeout 60s rclone lsf "$(remote_root)" \
--dirs-only --max-depth 1 >/dev/null 2>&1
}
count_remote_snapshots() {
local repo="$1"
local remote_snapshots
@@ -201,6 +206,7 @@ main() {
local ok
local latest_only_ok=1
local verify_ok=0
local remote_readable=0
local success_marker="${OFFSITE_DIR}/${PROVIDER}-full-verify-last-success"
local success_ts
local success_age
@@ -225,13 +231,19 @@ main() {
echo "WRITE_TEXTFILE=${WRITE_TEXTFILE}"
echo
bind_rclone_config_to_runtime_user
if [ "${PROVIDER}" != "rclone" ]; then
printf "%sBLOCKED%s unsupported provider for remote snapshot verification: %s\n" "${red}" "${reset}" "${PROVIDER}"
failed=1
elif rclone_ready; then
elif ! rclone_configured; then
printf "%sBLOCKED%s rclone remote configuration unavailable: %s:\n" "${red}" "${reset}" "${RCLONE_REMOTE}"
failed=1
elif rclone_remote_readable; then
printf "%sOK%s rclone remote configured: %s:\n" "${green}" "${reset}" "${RCLONE_REMOTE}"
printf "%sOK%s rclone provider readback succeeded: %s\n" "${green}" "${reset}" "$(remote_root)"
remote_readable=1
else
printf "%sBLOCKED%s rclone remote unavailable: %s:\n" "${red}" "${reset}" "${RCLONE_REMOTE}"
printf "%sBLOCKED%s rclone provider readback failed: %s\n" "${red}" "${reset}" "$(remote_root)"
failed=1
fi
@@ -245,7 +257,11 @@ main() {
echo
echo "== remote snapshot counts =="
for repo in ${EXPECTED_REPOS}; do
count="$(count_remote_snapshots "${repo}" || true)"
if [ "${remote_readable}" = "1" ]; then
count="$(count_remote_snapshots "${repo}" || true)"
else
count=-1
fi
ok=0
if [ "${count}" = "1" ]; then
ok=1

View File

@@ -1,43 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
required_commands=(node npm git curl bash timeout python3 ssh docker)
missing_commands=()
for command_name in "${required_commands[@]}"; do
if ! command -v "${command_name}" >/dev/null 2>&1; then
missing_commands+=("${command_name}")
fi
done
if command -v docker >/dev/null 2>&1 && \
! docker buildx version >/dev/null 2>&1; then
missing_commands+=(docker-buildx)
fi
if [ "${#missing_commands[@]}" -eq 0 ]; then
echo "host_runner_tools_ready=1 bootstrap_install_performed=0"
exit 0
fi
printf 'host_runner_tools_missing=%s\n' "$(IFS=,; echo "${missing_commands[*]}")"
if ! command -v apk >/dev/null 2>&1; then
echo "BLOCKER host_runner_tools_missing_and_apk_unavailable"
exit 1
fi
apk add --no-cache \
nodejs npm git curl bash coreutils python3 openssh-client docker-cli docker-cli-buildx
for command_name in "${required_commands[@]}"; do
command -v "${command_name}" >/dev/null 2>&1 || {
echo "BLOCKER host_runner_tool_install_failed command=${command_name}"
exit 1
}
done
docker buildx version >/dev/null 2>&1 || {
echo "BLOCKER host_runner_tool_install_failed command=docker-buildx"
exit 1
}
echo "host_runner_tools_ready=1 bootstrap_install_performed=1"

View File

@@ -1,96 +0,0 @@
from __future__ import annotations
import os
import subprocess
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts/ci/bootstrap-host-runner-tools.sh"
REQUIRED = ("node", "npm", "git", "curl", "bash", "timeout", "python3", "ssh")
def _write_command(path: Path, body: str = "exit 0\n") -> None:
path.write_text("#!/bin/sh\n" + body, encoding="utf-8")
path.chmod(0o755)
def _fake_tool_path(directory: Path, *, include_npm: bool) -> None:
for command in REQUIRED:
if command != "npm" or include_npm:
_write_command(directory / command)
_write_command(directory / "docker")
def _run(directory: Path) -> subprocess.CompletedProcess[str]:
env = os.environ.copy()
env["PATH"] = str(directory)
return subprocess.run(
["/bin/bash", str(SCRIPT)],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env,
)
def test_ready_runner_skips_package_install() -> None:
with tempfile.TemporaryDirectory() as temp:
directory = Path(temp)
_fake_tool_path(directory, include_npm=True)
completed = _run(directory)
assert completed.returncode == 0
assert "host_runner_tools_ready=1 bootstrap_install_performed=0" in completed.stdout
def test_missing_tool_without_apk_fails_closed() -> None:
with tempfile.TemporaryDirectory() as temp:
directory = Path(temp)
_fake_tool_path(directory, include_npm=False)
completed = _run(directory)
assert completed.returncode == 1
assert "host_runner_tools_missing=npm" in completed.stdout
assert "BLOCKER host_runner_tools_missing_and_apk_unavailable" in completed.stdout
def test_missing_tool_uses_apk_once_and_revalidates() -> None:
with tempfile.TemporaryDirectory() as temp:
directory = Path(temp)
_fake_tool_path(directory, include_npm=False)
apk_log = directory / "apk.log"
_write_command(
directory / "apk",
'printf "%s\\n" "$*" > "$APK_LOG"\n'
'printf "#!/bin/sh\\nexit 0\\n" > "$FAKE_BIN/npm"\n'
'/bin/chmod +x "$FAKE_BIN/npm"\n',
)
env = os.environ.copy()
env.update(
{
"PATH": str(directory),
"APK_LOG": str(apk_log),
"FAKE_BIN": str(directory),
}
)
completed = subprocess.run(
["/bin/bash", str(SCRIPT)],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env,
)
install_args = apk_log.read_text(encoding="utf-8")
assert completed.returncode == 0
assert "bootstrap_install_performed=1" in completed.stdout
assert install_args.startswith("add --no-cache ")
assert "docker-cli-buildx" in install_args

View File

@@ -26,7 +26,8 @@ def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None:
assert "._+" not in source
assert '$EvidenceRunId = if ($RunId -match $SafeIdPattern)' in source
assert "Get-Agent99RuntimeSourceBinding" in source
assert "$result.fileCount -eq 18" in source
assert "$ExpectedRuntimeFileCount = 19" in source
assert "$result.fileCount -eq $ExpectedRuntimeFileCount" in source
assert "$result.sourceRevision -eq $SourceRevision" in source
assert "$result.entrypointHashMatched" in source
assert '$CredentialFile = "/etc/awoooi/secrets/signoz-metadata-api-key"' in source
@@ -293,8 +294,8 @@ def test_entrypoint_is_in_exact_agent99_runtime_bundle() -> None:
assert expected in _quoted_array(contract, "$requiredFiles = @(", ")")
assert expected in _quoted_array(sender, "RUNTIME_FILES=(", ")")
assert expected in _quoted_array(receiver, "$FixedRuntimeFiles = @(", ")")
assert 'expectedRuntimeFileCount": 18' in sender
assert '$runtimeManifest.fileCount -eq 18' in receiver
assert 'expectedRuntimeFileCount": 19' in sender
assert '$runtimeManifest.fileCount -eq 19' in receiver
def test_runtime_contract_check_requires_typed_signoz_transport_readback() -> None:

View File

@@ -122,7 +122,7 @@ def test_agent99_atomic_bundle_owns_the_dispatch_entrypoint() -> None:
assert "$dbExecutorRecoveryContract" in contract
assert "awoooi_db_bounded_executor_receipt_v2" in contract
assert "Receipt.database_sqlstate" in contract
assert 'expectedRuntimeFileCount": 18' in transport
assert "expectedRuntimeFileCount=18" in transport
assert "expectedRuntimeFileCount -ne 18" in receiver
assert "fileCount = 18" in receiver
assert 'expectedRuntimeFileCount": 19' in transport
assert "expectedRuntimeFileCount=19" in transport
assert "expectedRuntimeFileCount -ne 19" in receiver
assert "fileCount = 19" in receiver

View File

@@ -486,7 +486,7 @@ $decision = Get-AgentLivePreflightDecision `
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
-Manifest $manifest `
-ManifestCheckRequired ([bool]($ReadinessScope -eq "FullRuntime")) `
-ExpectedRuntimeFileCount 18 `
-ExpectedRuntimeFileCount 19 `
-TaskFailureCount $taskFailures.Count `
-RelayListenerCount $relayListeners.Count `
-RequiredEvidenceStaleCount $requiredEvidenceStale.Count `

View File

@@ -34,6 +34,7 @@ $FixedRuntimeFiles = @(
"agent99-control-plane.ps1",
"agent99-db-bounded-executor.ps1",
"agent99-signoz-metadata-executor.ps1",
"agent99-host110-backup-runtime-broker.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",
@@ -951,7 +952,7 @@ try {
}
$sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant()
if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
if ([int]$envelope.expectedRuntimeFileCount -ne 18) { throw "invalid_expected_runtime_file_count" }
if ([int]$envelope.expectedRuntimeFileCount -ne 19) { throw "invalid_expected_runtime_file_count" }
$traceId = [string]$envelope.traceId
$runId = [string]$envelope.runId
@@ -986,7 +987,7 @@ try {
$sourceManifest = [pscustomobject]@{
schemaVersion = "agent99_remote_source_manifest_v1"
sourceRevision = $sourceRevision
fileCount = 18
fileCount = 19
manifestSha256 = $manifestDigest
files = @($FixedRuntimeFiles | ForEach-Object {
[pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() }
@@ -1023,7 +1024,7 @@ try {
status = "check_ready"
mode = "check"
sourceRevision = $sourceRevision
expectedRuntimeFileCount = 18
expectedRuntimeFileCount = 19
manifestSha256 = $manifestDigest
plannedStagingPath = $stagePath
runtimeManifest = $currentManifest
@@ -1110,7 +1111,7 @@ try {
if (
[string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or
[string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or
[int]$existingSourceManifest.fileCount -ne 18 -or
[int]$existingSourceManifest.fileCount -ne 19 -or
[string]$existingSourceManifest.manifestSha256 -ne $manifestDigest
) { throw "existing_staging_manifest_mismatch" }
} catch {
@@ -1134,7 +1135,7 @@ try {
[string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and
$live.runtimeMatched -and
$live.sourceRevision -eq $sourceRevision -and
$live.fileCount -eq 18 -and
$live.fileCount -eq 19 -and
$live.mismatchCount -eq 0 -and
$liveLauncherContract.ok -and
[string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256
@@ -1447,7 +1448,7 @@ try {
$runtimeManifest.exists -and
$runtimeManifest.runtimeMatched -and
$runtimeManifest.sourceRevision -eq $sourceRevision -and
$runtimeManifest.fileCount -eq 18 -and
$runtimeManifest.fileCount -eq 19 -and
$runtimeManifest.mismatchCount -eq 0 -and
$launcherContract.ok -and
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and

View File

@@ -18,6 +18,7 @@ RUNTIME_FILES=(
"agent99-control-plane.ps1"
"agent99-db-bounded-executor.ps1"
"agent99-signoz-metadata-executor.ps1"
"agent99-host110-backup-runtime-broker.ps1"
"agent99-deploy.ps1"
"agent99-register-tasks.ps1"
"agent99-alertmanager-alertchain-poll.ps1"
@@ -227,7 +228,7 @@ trace_id, run_id, work_item_id = sys.argv[4:7]
output_path = Path(sys.argv[7])
runtime_names = sys.argv[8:]
if len(runtime_names) != 18 or len(set(runtime_names)) != 18:
if len(runtime_names) != 19 or len(set(runtime_names)) != 19:
raise SystemExit("fixed_runtime_file_contract_failed")
files: list[dict[str, str]] = []
@@ -267,7 +268,7 @@ envelope = {
"runId": run_id,
"workItemId": work_item_id,
"sourceRevision": source_revision,
"expectedRuntimeFileCount": 18,
"expectedRuntimeFileCount": 19,
"manifestSha256": manifest_sha256,
"files": files,
"livePreflight": {
@@ -419,14 +420,14 @@ stage_token = hashlib.sha256(stage_identity.encode("utf-8")).hexdigest()[:20]
script = r'''$ErrorActionPreference='Stop';$ProgressPreference='SilentlyContinue'
$a='C:\Wooo\Agent99';$b=Join-Path $a 'bin';$p=Join-Path $a 'state\runtime-manifest.json'
$r=[ordered]@{exists=$false;sourceRevision='';runtimeMatched=$false;recordedRuntimeMatched=$false;fileCount=0;mismatchCount=-1;recordedMismatchCount=-1;parseError=''}
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 18-and$m.mismatchCount-eq 0-and$rows.Count-eq 18-and$seen.Count-eq 18-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 19-and$m.mismatchCount-eq 0-and$rows.Count-eq 19-and$seen.Count-eq 19-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
$c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''}
try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name}
$z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false}
$ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0)
$status=if($ready){'check_ready'}else{'check_failed_no_apply'}
$next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'}
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=18;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=19;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
if(-not$ready){exit 65}'''
script = (
script.replace("__SOURCE_REVISION__", source_revision)

View File

@@ -0,0 +1,682 @@
from __future__ import annotations
import base64
import grp
import hashlib
import importlib.util
import json
import os
import pwd
import re
import shutil
import subprocess
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[3]
BROKER = ROOT / "agent99-host110-backup-runtime-broker.ps1"
EXECUTOR = ROOT / "scripts" / "backup" / "host110-backup-runtime-executor.sh"
VERIFIER = ROOT / "scripts" / "backup" / "verify-host110-backup-runtime.py"
EXPECTED_RUNTIME_FILES = {
"common.sh",
"backup-all.sh",
"backup-host188-products.sh",
"verify-host188-products-backup.sh",
"verify-host188-products-archive.py",
"backup-gitea.sh",
"gitea-full-backup-restore-drill.sh",
"backup-configs.sh",
"backup-awoooi.sh",
"backup-awoooi-frequent.sh",
"backup-clawbot.sh",
"backup-sentry.sh",
"check-backup-integrity.sh",
"sync-offsite-backups.sh",
"backup-offsite-readiness-gate.sh",
"verify-offsite-full-sync.sh",
"enforce-latest-only-retention.sh",
}
def _load_verifier():
spec = importlib.util.spec_from_file_location("host110_runtime_verifier", VERIFIER)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _powershell_string(source: str, variable: str) -> str:
match = re.search(rf'\${re.escape(variable)}\s*=\s*"([^"]+)"', source)
assert match is not None, variable
return match.group(1)
def _powershell_array(source: str, variable: str) -> tuple[str, ...]:
match = re.search(
rf'\${re.escape(variable)}\s*=\s*@\((.*?)\n\)',
source,
re.DOTALL,
)
assert match is not None, variable
return tuple(re.findall(r'"([^"]+)"', match.group(1)))
def _executor_payload_order(source: str) -> tuple[str, ...]:
match = re.search(r"readonly -a RUNTIME_FILES=\((.*?)\n\)", source, re.DOTALL)
assert match is not None
runtime_files = tuple(re.findall(r"^\s+([A-Za-z0-9_.-]+)\s*$", match.group(1), re.MULTILINE))
exporter = re.search(r'^readonly EXPORTER_FILE="([^"]+)"$', source, re.MULTILINE)
assert exporter is not None
return (*runtime_files, exporter.group(1))
def _write_executable(path: Path, source: str) -> None:
path.write_text(source, encoding="utf-8")
path.chmod(0o755)
def _build_executor_harness(tmp_path: Path, run_id: str) -> dict[str, object]:
destination = tmp_path / "backup-scripts"
exporter_root = tmp_path / "exporter"
status_root = tmp_path / "status"
stage_root = tmp_path / "stage"
rollback_root = tmp_path / "rollback"
source_stage = tmp_path / f"source-{run_id}"
lock_path = tmp_path / "runtime.lock"
fake_bin = tmp_path / "fake-bin"
for path in (destination, exporter_root, source_stage, fake_bin):
path.mkdir(parents=True)
user_name = pwd.getpwuid(os.getuid()).pw_name
group_name = grp.getgrgid(os.getgid()).gr_name
shell_path = shutil.which("bash") or "/bin/bash"
shell_version = subprocess.run(
[shell_path, "-c", 'printf "%s" "${BASH_VERSINFO[0]:-0}"'],
check=False,
capture_output=True,
text=True,
).stdout
use_zsh_adapter = not shell_version.isdigit() or int(shell_version) < 4
if use_zsh_adapter:
shell_path = shutil.which("zsh") or "/bin/zsh"
executor_source = EXECUTOR.read_text(encoding="utf-8")
replacements = {
'readonly EXPECTED_USER="wooo"': f'readonly EXPECTED_USER="{user_name}"',
'readonly DEST_ROOT="/backup/scripts"': f'readonly DEST_ROOT="{destination}"',
'readonly EXPORTER_ROOT="/home/wooo/scripts"': f'readonly EXPORTER_ROOT="{exporter_root}"',
'readonly STATUS_ROOT="/backup/status"': f'readonly STATUS_ROOT="{status_root}"',
'readonly STAGE_ROOT="/backup/.agent99-backup-runtime-stage"': f'readonly STAGE_ROOT="{stage_root}"',
'readonly ROLLBACK_ROOT="/backup/.agent99-backup-runtime-rollback"': f'readonly ROLLBACK_ROOT="{rollback_root}"',
'readonly LOCK_PATH="/tmp/agent99-host110-backup-runtime.lock"': f'readonly LOCK_PATH="{lock_path}"',
'expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"': f'expected_source_stage="{source_stage}"',
'"wooo:wooo:755"': f'"{user_name}:{group_name}:755"',
'"wooo:wooo:700"': f'"{user_name}:{group_name}:700"',
'"wooo:wooo"': f'"{user_name}:{group_name}"',
}
for before, after in replacements.items():
assert before in executor_source
executor_source = executor_source.replace(before, after)
if use_zsh_adapter:
executor_source = executor_source.replace(
'"${!RUN_OWNED_TEMP_PATHS[@]}"',
'"${(@k)RUN_OWNED_TEMP_PATHS}"',
).replace(
'RUN_OWNED_TEMP_PATHS["$1"]=1',
'RUN_OWNED_TEMP_PATHS[$1]=1',
).replace('${PAYLOAD_FILES[0]}', '${PAYLOAD_FILES[1]}')
executor_path = source_stage / EXECUTOR.name
_write_executable(executor_path, executor_source)
verifier_source = VERIFIER.read_text(encoding="utf-8")
verifier_replacements = {
'EXPECTED_DESTINATION = Path("/backup/scripts")': f"EXPECTED_DESTINATION = Path({str(destination)!r})",
'EXPECTED_EXPORTER_DESTINATION = Path("/home/wooo/scripts/backup-health-textfile-exporter.py")': (
f"EXPECTED_EXPORTER_DESTINATION = Path({str(exporter_root / 'backup-health-textfile-exporter.py')!r})"
),
'RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")': f"RUNTIME_LOCK = Path({str(lock_path)!r})",
}
for before, after in verifier_replacements.items():
assert before in verifier_source
verifier_source = verifier_source.replace(before, after)
verifier_path = source_stage / VERIFIER.name
_write_executable(verifier_path, verifier_source)
payload_order = _executor_payload_order(executor_source)
rows = []
before_state: dict[str, tuple[bytes, int]] = {}
for name in payload_order:
source = (
ROOT / "scripts" / "ops" / name
if name == "backup-health-textfile-exporter.py"
else ROOT / "scripts" / "backup" / name
)
staged = source_stage / name
shutil.copy2(source, staged)
staged.chmod(0o755)
rows.append({"name": name, "sha256": _sha256(staged)})
target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name
target.write_text(f"prestate:{name}\n", encoding="utf-8")
target.chmod(0o755)
before_state[name] = (target.read_bytes(), target.stat().st_mode & 0o777)
source_revision = "a" * 40
manifest = {
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
"sourceRevision": source_revision,
"sourceHead": "b" * 40,
"runId": run_id,
"executorSha256": _sha256(executor_path),
"verifierSha256": _sha256(verifier_path),
"files": rows,
}
(source_stage / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
_write_executable(fake_bin / "hostname", "#!/bin/sh\nprintf '%s\\n' '192.168.0.110'\n")
_write_executable(fake_bin / "pgrep", "#!/bin/sh\nexit 1\n")
_write_executable(fake_bin / "flock", "#!/bin/sh\nexit 0\n")
_write_executable(fake_bin / "timeout", "#!/bin/sh\nexit 0\n")
_write_executable(
fake_bin / "readlink",
"""#!/usr/bin/env python3
import pathlib
import sys
if len(sys.argv) != 3 or sys.argv[1] != "-f":
raise SystemExit(2)
print(pathlib.Path(sys.argv[2]).resolve(strict=True))
""",
)
_write_executable(
fake_bin / "stat",
"""#!/usr/bin/env python3
import grp
import os
import pwd
import sys
if len(sys.argv) != 4 or sys.argv[1] != "-c":
raise SystemExit(2)
row = os.stat(sys.argv[3])
mode = format(row.st_mode & 0o777, "o")
values = {
"%U:%G:%a": f"{pwd.getpwuid(row.st_uid).pw_name}:{grp.getgrgid(row.st_gid).gr_name}:{mode}",
"%U:%G": f"{pwd.getpwuid(row.st_uid).pw_name}:{grp.getgrgid(row.st_gid).gr_name}",
"%u:%g:%a": f"{row.st_uid}:{row.st_gid}:{mode}",
}
if sys.argv[2] not in values:
raise SystemExit(2)
print(values[sys.argv[2]])
""",
)
return {
"executor": executor_path,
"source_stage": source_stage,
"source_revision": source_revision,
"run_id": run_id,
"destination": destination,
"exporter_root": exporter_root,
"status_root": status_root,
"stage_root": stage_root,
"rollback_root": rollback_root,
"fake_bin": fake_bin,
"shell": shell_path,
"use_zsh_adapter": use_zsh_adapter,
"payload_order": payload_order,
"before_state": before_state,
}
def _run_executor_harness(harness: dict[str, object], **extra_env: str) -> subprocess.CompletedProcess[str]:
environment = os.environ.copy()
for name in (
"HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT",
"HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT",
"HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT",
):
environment.pop(name, None)
environment["HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS"] = "1"
environment.update(extra_env)
environment["PATH"] = f"{harness['fake_bin']}:{environment['PATH']}"
command = [str(harness["shell"])]
if harness["use_zsh_adapter"]:
environment["ZDOTDIR"] = str(harness["fake_bin"])
command.append("-f")
command.extend(
[
str(harness["executor"]),
"--mode",
"apply",
"--source-revision",
str(harness["source_revision"]),
"--run-id",
str(harness["run_id"]),
"--source-stage",
str(harness["source_stage"]),
]
)
return subprocess.run(
command,
check=False,
capture_output=True,
text=True,
env=environment,
)
def _assert_prestate_restored(harness: dict[str, object]) -> None:
destination = Path(harness["destination"])
exporter_root = Path(harness["exporter_root"])
before_state = harness["before_state"]
assert isinstance(before_state, dict)
for name in harness["payload_order"]:
target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name
assert (target.read_bytes(), target.stat().st_mode & 0o777) == before_state[name]
def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
source = BROKER.read_text(encoding="utf-8")
assert '[ValidateSet("Check", "Apply", "Verify")]' in source
assert '$TargetHost = "192.168.0.110"' in source
assert '$TargetUser = "wooo"' in source
assert '$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"' in source
assert 'keys\\agent99_ed25519' in source
assert '"credential.helper="' in source
assert '$env:GIT_TERMINAL_PROMPT = "0"' in source
assert '"merge-base", "--is-ancestor", $SourceRevision, $sourceHead' in source
assert '"checkout", "--quiet", $SourceRevision, "--"' in source
assert 'executorSha256 = $executorDigest' in source
assert 'verifierSha256 = $verifierDigest' in source
assert '"scripts/ops/backup-health-textfile-exporter.py"' in source
assert "/home/wooo/awoooi" not in source
assert '"StrictHostKeyChecking=yes"' in source
assert '"NumberOfPasswordPrompts=0"' in source
assert '"ConnectionAttempts=1"' in source
assert "$process.WaitForExit($TimeoutSeconds * 1000)" in source
assert "taskkill.exe /PID $process.Id /T /F" in source
assert "$process.Kill()" in source
assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths' in source
def test_broker_scp_basenames_match_executor_source_stage_contract() -> None:
broker = BROKER.read_text(encoding="utf-8")
executor = EXECUTOR.read_text(encoding="utf-8")
runtime_paths = _powershell_array(broker, "RuntimeRelativePaths")
broker_payload_basenames = tuple(Path(path).name for path in runtime_paths)
executor_payload_basenames = _executor_payload_order(executor)
executor_path = _powershell_string(broker, "ExecutorRelativePath")
verifier_path = _powershell_string(broker, "VerifierRelativePath")
manifest_match = re.search(
r'\$manifestPath\s*=\s*Join-Path\s+\$sourceRoot\s+"([^"]+)"',
broker,
)
assert manifest_match is not None
manifest_name = manifest_match.group(1)
assert len(broker_payload_basenames) == 18
assert len(executor_payload_basenames) == 18
assert set(broker_payload_basenames) == set(executor_payload_basenames)
assert broker_payload_basenames == executor_payload_basenames
staged_basenames = (
*broker_payload_basenames,
Path(executor_path).name,
Path(verifier_path).name,
manifest_name,
)
assert len(staged_basenames) == 21
assert len(set(staged_basenames)) == 21
assert manifest_name == "manifest.json"
assert '$localPaths += $executorPath, $verifierPath, $ManifestPath' in broker
assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths $remoteStage' in broker
assert f'$SOURCE_STAGE/{manifest_name}' in executor
assert f'$SOURCE_STAGE/{Path(verifier_path).name}' in executor
assert Path(executor_path).name == "host110-backup-runtime-executor.sh"
def test_windows99_broker_verify_is_independent_no_write_and_evidence_is_immutable() -> None:
source = BROKER.read_text(encoding="utf-8")
assert 'RedirectStandardInput = $StandardInputPath' in source
assert "python3 - --manifest-base64" in source
assert 'remoteWritePerformed' in source
assert '[IO.File]::Move($temporary, $Path)' in source
assert 'Move-Item -LiteralPath $temporary -Destination $Path -Force' not in source
assert 'run_identity_conflict' in source
assert 'run_identity_evidence_exists' in source
assert 'decisionProvider = "deterministic_only"' in source
assert 'criticProvider = "deterministic_only"' in source
assert 'verifier = "independent_host110_backup_runtime_python_readback"' in source
def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
source = EXECUTOR.read_text(encoding="utf-8")
array_body = source.split("readonly -a RUNTIME_FILES=(", 1)[1].split(")", 1)[0]
runtime_files = set(re.findall(r"^\s+([A-Za-z0-9_.-]+)\s*$", array_body, re.MULTILINE))
assert runtime_files == EXPECTED_RUNTIME_FILES
assert 'readonly EXPECTED_HOST_IP="192.168.0.110"' in source
assert 'readonly EXPECTED_USER="wooo"' in source
assert 'readonly DEST_ROOT="/backup/scripts"' in source
assert 'readonly EXPORTER_ROOT="/home/wooo/scripts"' in source
assert 'readonly EXPORTER_FILE="backup-health-textfile-exporter.py"' in source
assert 'readonly -a PAYLOAD_FILES=("${RUNTIME_FILES[@]}" "$EXPORTER_FILE")' in source
assert '--source-stage' in source
assert 'document.get("executorSha256"' in source
assert 'document.get("verifierSha256"' in source
assert 'executor_identity_failed' in source
assert 'verifier_identity_failed' in source
assert 'flock -x -w 30 9' in source
assert 'legacy_backup_runtime_active' in source
assert source.index("for name in \"${PAYLOAD_FILES[@]}\"") < source.index("mv -f -- \"$temporary\"")
assert 'rollback_unverified' in source
assert 'failed_rolled_back' in source
assert 'ROLLBACK_ATTEMPTED=1' in source
assert 'ROLLBACK_PERFORMED=1' in source
assert '"rollbackAttempted": rollback_attempted' in source
assert '"rollbackPerformed": rollback_performed' in source
assert '"rollbackVerified": rollback_verified' in source
assert '"zeroResidueVerified": zero_residue_verified' in source
assert 'cleanup_run_owned_temps' in source
assert 'verify_run_owned_temp_absence' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT' in source
assert 'HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT' in source
assert 'HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS' in source
assert '[ "$DEST_ROOT" != "/backup/scripts" ]' in source
assert 'fsync_rollback_prestate || return 75' in source
assert 'fsync_destination_state || fail "post_apply_durability_failed"' in source
assert 'working_digest "$dest_path"' in source
assert 'PREVIOUS_METADATA' in source
assert "stat -c '%u:%g:%a'" in source
assert 'os.link(temporary, path)' in source
assert 'os.fsync(handle.fileno())' in source
assert 'os.fsync(directory_fd)' in source
assert 'hashlib.sha256(readback).hexdigest()' in source
assert source.index('write_receipt "verified"') < source.index("\nAPPLY_COMPLETE=1")
assert source.index("\nAPPLY_COMPLETE=1") < source.rindex('rm -rf -- "$ROLLBACK_DIR"')
assert 'run_identity_receipt_exists' in source
assert 'run_identity_stage_exists' in source
assert 'run_identity_rollback_exists' in source
assert 'verify-host110-backup-runtime.py' in source
assert 'selfIdentityVerified' in source
assert '"backupScriptFileCount": 17' in source
assert '"backupHealthExporterIncluded": True' in source
assert '"productionServiceRestarted": False' in source
assert '"secretValuesRead": False' in source
assert "sudo " not in source
assert "docker " not in source
assert "systemctl " not in source
def test_apply_success_publishes_durable_receipt_before_prestate_cleanup(tmp_path: Path) -> None:
run_id = "apply-success"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(harness)
assert result.returncode == 0, result.stderr
output = json.loads(result.stdout)
assert output["ok"] is True
assert output["rollbackAttempted"] is False
assert output["rollbackPerformed"] is False
assert output["rollbackVerified"] is False
assert output["zeroResidueVerified"] is True
assert output["stageCleanupVerified"] is True
assert output["rollbackPrestateCleanupVerified"] is True
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
assert receipt["status"] == "verified"
assert receipt["rollbackAttempted"] is False
assert receipt["rollbackPerformed"] is False
assert receipt["rollbackVerified"] is False
assert receipt["zeroResidueVerified"] is True
assert not (Path(harness["rollback_root"]) / run_id).exists()
assert not (Path(harness["stage_root"]) / run_id).exists()
for name in harness["payload_order"]:
target = (
Path(harness["exporter_root"]) / name
if name == "backup-health-textfile-exporter.py"
else Path(harness["destination"]) / name
)
assert target.read_bytes() == (Path(harness["source_stage"]) / name).read_bytes()
@pytest.mark.parametrize("fault", ["write", "link", "fsync", "readback"])
def test_receipt_fault_rolls_back_without_false_success_or_premature_prestate_cleanup(
tmp_path: Path,
fault: str,
) -> None:
run_id = f"receipt-{fault}"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT=fault,
)
assert result.returncode != 0
assert '"ok": true' not in result.stdout.lower()
assert "durable_receipt_failed" in result.stderr
_assert_prestate_restored(harness)
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
assert receipt["status"] == "failed_rolled_back"
assert receipt["rollbackAttempted"] is True
assert receipt["rollbackPerformed"] is True
assert receipt["rollbackVerified"] is True
assert receipt["zeroResidueVerified"] is True
assert not (Path(harness["rollback_root"]) / run_id).exists()
assert not (Path(harness["stage_root"]) / run_id).exists()
for parent in (Path(harness["destination"]), Path(harness["exporter_root"])):
assert not list(parent.glob(f".*.agent99-*{run_id}"))
def test_apply_temp_fault_is_removed_and_zero_residue_is_verified(tmp_path: Path) -> None:
run_id = "apply-temp-cleanup"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT="after_temp_install",
)
assert result.returncode != 0
assert '"ok": true' not in result.stdout.lower()
assert "fault_injected_after_temp_install" in result.stderr
_assert_prestate_restored(harness)
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
assert receipt["status"] == "failed_rolled_back"
assert receipt["rollbackAttempted"] is True
assert receipt["rollbackPerformed"] is True
assert receipt["rollbackVerified"] is True
assert receipt["zeroResidueVerified"] is True
for parent in (Path(harness["destination"]), Path(harness["exporter_root"])):
assert not list(parent.glob(f".*.agent99-*{run_id}"))
def test_temp_cleanup_fault_is_rollback_unverified_and_preserves_prestate(tmp_path: Path) -> None:
run_id = "temp-residue-unverified"
harness = _build_executor_harness(tmp_path, run_id)
result = _run_executor_harness(
harness,
HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT="after_temp_install",
HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT="preserve_first_temp",
)
assert result.returncode != 0
assert '"ok": true' not in result.stdout.lower()
_assert_prestate_restored(harness)
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
assert receipt["status"] == "rollback_unverified"
assert receipt["rollbackAttempted"] is True
assert receipt["rollbackPerformed"] is True
assert receipt["rollbackVerified"] is False
assert receipt["zeroResidueVerified"] is False
assert (Path(harness["rollback_root"]) / run_id / "prestate.tsv").is_file()
residue = []
for parent in (Path(harness["destination"]), Path(harness["exporter_root"])):
residue.extend(parent.glob(f".*.agent99-*{run_id}"))
assert residue
def test_backup_runtime_entrypoints_share_the_deployment_gate() -> None:
common = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8")
restore = (ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh").read_text(encoding="utf-8")
offsite = (ROOT / "scripts/backup/backup-offsite-readiness-gate.sh").read_text(encoding="utf-8")
archive = (ROOT / "scripts/backup/verify-host188-products-archive.py").read_text(encoding="utf-8")
for source in (common, restore, offsite):
assert "/tmp/agent99-host110-backup-runtime.lock" in source
assert "flock -s -w 60" in source
assert "fcntl.LOCK_SH | fcntl.LOCK_NB" in archive
assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" in archive
def test_independent_verifier_accepts_exact_bundle_without_writing(tmp_path, monkeypatch, capsys) -> None:
verifier = _load_verifier()
destination = tmp_path / "scripts"
destination.mkdir()
exporter_destination = tmp_path / verifier.EXPORTER_FILE
rows = []
for name in verifier.EXPECTED_BACKUP_FILES:
path = destination / name
path.write_text(f"runtime:{name}\n", encoding="utf-8")
path.chmod(0o755)
rows.append({"name": name, "sha256": _sha256(path)})
exporter_destination.write_text("exporter\n", encoding="utf-8")
exporter_destination.chmod(0o755)
rows.append({"name": verifier.EXPORTER_FILE, "sha256": _sha256(exporter_destination)})
verifier_digest = _sha256(VERIFIER)
source_revision = "a" * 40
run_id = "unit-exact"
manifest = {
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
"sourceRevision": source_revision,
"sourceHead": "b" * 40,
"runId": run_id,
"executorSha256": "c" * 64,
"verifierSha256": verifier_digest,
"files": rows,
}
manifest_path = tmp_path / "manifest.json"
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
before = {
path.name: (path.stat().st_mtime_ns, _sha256(path))
for path in [*destination.iterdir(), exporter_destination]
}
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
monkeypatch.setattr(verifier, "EXPECTED_EXPORTER_DESTINATION", exporter_destination)
monkeypatch.setattr(
sys,
"argv",
[
str(VERIFIER),
"--manifest",
str(manifest_path),
"--destination",
str(destination),
"--exporter-destination",
str(exporter_destination),
"--source-revision",
source_revision,
"--run-id",
run_id,
"--expected-verifier-sha256",
verifier_digest,
],
)
verifier.main()
result = json.loads(capsys.readouterr().out)
after = {
path.name: (path.stat().st_mtime_ns, _sha256(path))
for path in [*destination.iterdir(), exporter_destination]
}
assert result["ok"] is True
assert result["fileCount"] == 18
assert result["backupScriptFileCount"] == 17
assert result["backupHealthExporterIncluded"] is True
assert result["remoteWritePerformed"] is False
assert result["selfIdentityVerified"] is True
assert before == after
def test_independent_verifier_rejects_drifted_bundle(tmp_path, monkeypatch, capsys) -> None:
verifier = _load_verifier()
destination = tmp_path / "scripts"
destination.mkdir()
exporter_destination = tmp_path / verifier.EXPORTER_FILE
rows = []
for name in verifier.EXPECTED_BACKUP_FILES:
path = destination / name
path.write_text(f"runtime:{name}\n", encoding="utf-8")
path.chmod(0o755)
rows.append({"name": name, "sha256": _sha256(path)})
exporter_destination.write_text("exporter\n", encoding="utf-8")
exporter_destination.chmod(0o755)
rows.append({"name": verifier.EXPORTER_FILE, "sha256": _sha256(exporter_destination)})
(destination / verifier.EXPECTED_BACKUP_FILES[0]).write_text("drifted\n", encoding="utf-8")
verifier_digest = _sha256(VERIFIER)
source_revision = "d" * 40
run_id = "unit-drift"
manifest = {
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
"sourceRevision": source_revision,
"sourceHead": "e" * 40,
"runId": run_id,
"executorSha256": "f" * 64,
"verifierSha256": verifier_digest,
"files": rows,
}
encoded = base64.b64encode(json.dumps(manifest).encode()).decode()
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
monkeypatch.setattr(verifier, "EXPECTED_EXPORTER_DESTINATION", exporter_destination)
monkeypatch.setattr(
sys,
"argv",
[
str(VERIFIER),
"--manifest-base64",
encoded,
"--destination",
str(destination),
"--exporter-destination",
str(exporter_destination),
"--source-revision",
source_revision,
"--run-id",
run_id,
"--expected-verifier-sha256",
verifier_digest,
],
)
with pytest.raises(SystemExit):
verifier.main()
assert "runtime_file_identity_failed" in capsys.readouterr().err
def test_agent99_runtime_lists_include_the_host110_broker_once() -> None:
sources = [
ROOT / "agent99-bootstrap.ps1",
ROOT / "agent99-contract-check.ps1",
ROOT / "agent99-deploy.ps1",
ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh",
ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1",
]
marker = "agent99-host110-backup-runtime-broker.ps1"
for path in sources:
assert marker in path.read_text(encoding="utf-8"), path

View File

@@ -36,7 +36,7 @@ def _base_case() -> dict[str, Any]:
"exists": True,
"sourceRevision": "a" * 40,
"runtimeMatched": True,
"fileCount": 18,
"fileCount": 19,
"mismatchCount": 0,
"parseError": "",
},
@@ -84,7 +84,7 @@ $parameters = @{{
AgentRootPresent = [bool]$case.agentRootPresent
Manifest = $case.manifest
ManifestCheckRequired = [bool]$case.manifestCheckRequired
ExpectedRuntimeFileCount = 18
ExpectedRuntimeFileCount = 19
TaskFailureCount = [int]$case.taskFailureCount
RelayListenerCount = [int]$case.relayListenerCount
RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount
@@ -133,7 +133,7 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
)
assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision
assert "preflightGreen = [bool]$decision.deploymentEligible" in source
assert "-ExpectedRuntimeFileCount 18" in source
assert "-ExpectedRuntimeFileCount 19" in source
assert "warningReasons = $warningReasons" in source
assert "selfHealthPerformanceIssues = @(" in source
assert 'if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$")' in source

View File

@@ -18,6 +18,9 @@ RECEIVER = (
/ "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
)
DEPLOYER = ROOT / "agent99-deploy.ps1"
BOOTSTRAP = ROOT / "agent99-bootstrap.ps1"
SIGNOZ_EXECUTOR = ROOT / "agent99-signoz-metadata-executor.ps1"
CONTRACT_CHECK = ROOT / "agent99-contract-check.ps1"
LIVE_PREFLIGHT = ROOT / "scripts/reboot-recovery/agent99-live-preflight.ps1"
EXPECTED_RUNTIME_FILES = (
@@ -26,6 +29,7 @@ EXPECTED_RUNTIME_FILES = (
"agent99-control-plane.ps1",
"agent99-db-bounded-executor.ps1",
"agent99-signoz-metadata-executor.ps1",
"agent99-host110-backup-runtime-broker.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",
@@ -70,25 +74,41 @@ def test_sender_is_valid_shell_with_check_as_the_default() -> None:
assert "timeout --signal=TERM --kill-after=10" in source
def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None:
def test_runtime_manifest_producers_and_consumers_share_exact_bundle_count() -> None:
sender = SENDER.read_text(encoding="utf-8")
receiver = RECEIVER.read_text(encoding="utf-8")
deployer = DEPLOYER.read_text(encoding="utf-8")
bootstrap = BOOTSTRAP.read_text(encoding="utf-8")
signoz_executor = SIGNOZ_EXECUTOR.read_text(encoding="utf-8")
contract_check = CONTRACT_CHECK.read_text(encoding="utf-8")
live_preflight = LIVE_PREFLIGHT.read_text(encoding="utf-8")
sender_files = _quoted_array(sender, "RUNTIME_FILES=(", ")")
receiver_files = _quoted_array(receiver, "$FixedRuntimeFiles = @(", ")")
deployer_files = _quoted_array(deployer, "$runtimeFiles = @(", ")")
bootstrap_files = _quoted_array(bootstrap, "$agentFiles = @(", ")")
assert sender_files == EXPECTED_RUNTIME_FILES
assert receiver_files == EXPECTED_RUNTIME_FILES
assert deployer_files == EXPECTED_RUNTIME_FILES
assert bootstrap_files == EXPECTED_RUNTIME_FILES
expected_count = len(EXPECTED_RUNTIME_FILES)
preflight_count = re.search(
r"-ExpectedRuntimeFileCount\s+(\d+)", live_preflight
)
assert preflight_count is not None
assert int(preflight_count.group(1)) == len(EXPECTED_RUNTIME_FILES)
assert "expectedRuntimeFileCount\": 18" in sender
signoz_count = re.search(
r"\$ExpectedRuntimeFileCount\s*=\s*(\d+)", signoz_executor
)
assert signoz_count is not None
assert int(preflight_count.group(1)) == expected_count
assert int(signoz_count.group(1)) == expected_count
assert f'expectedRuntimeFileCount\": {expected_count}' in sender
assert f"[int]$envelope.expectedRuntimeFileCount -ne {expected_count}" in receiver
assert f"fileCount = {expected_count}" in receiver
assert "$result.fileCount -eq $ExpectedRuntimeFileCount" in signoz_executor
assert "$ExpectedRuntimeFileCount = 19" in contract_check
assert "$result.fileCount -eq $ExpectedRuntimeFileCount" in contract_check
assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver
assert "required_runtime_file_missing" in receiver
assert "duplicate_runtime_file" in receiver
@@ -263,7 +283,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
)
assert "$livePreflight.sourceRevision -eq $sourceRevision" in source
assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source
assert '$runtimeManifest.fileCount -eq 18' in source
assert '$runtimeManifest.fileCount -eq 19' in source
assert "function Invoke-AgentWindowsControlBaseline" in source
assert "function Get-AgentRunEvidenceToken" in source
assert "$evidenceRunToken = Get-AgentRunEvidenceToken -Value $RunId" in source

View File

@@ -516,7 +516,8 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
for parameter in ("AutomationRunId", "TraceId", "WorkItemId"):
assert f"[string]${parameter} = \"\"" in control
assert f'[string]`${parameter} = ""' in bootstrap
assert 'reason = "controlled_dispatch_identity_unsafe"' in control
assert '"controlled_dispatch_identity_unsafe"' in control
assert "reason = $identityRejectionReason" in control
assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
assert '$dryRun.managerPreflightStatus -eq "ready"' in function
assert "-not $dryRun.artifactTransactionBlocked" in function

View File

@@ -19,9 +19,9 @@ from typing import Any, Sequence
SCHEMA_VERSION = "awoooi_runtime_image_mirror_policy_v1"
RECEIPT_SCHEMA_VERSION = "awoooi_runtime_image_mirror_receipt_v2"
RECEIPT_SCHEMA_VERSION = "awoooi_runtime_image_mirror_receipt_v1"
STAGING_SCHEMA_VERSION = "awoooi_runtime_image_mirror_staging_policy_v1"
STAGING_RECEIPT_SCHEMA_VERSION = "awoooi_runtime_image_mirror_staging_receipt_v2"
STAGING_RECEIPT_SCHEMA_VERSION = "awoooi_runtime_image_mirror_staging_receipt_v1"
ALLOWED_KINDS = {"deployment", "daemonset", "statefulset"}
ALLOWED_CONTAINER_KINDS = {"container", "init_container"}
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
@@ -665,25 +665,16 @@ def mirror_images(
known_hosts: Path,
apply: bool,
) -> dict[str, Any]:
cache: RuntimeCache | None = None
cache = RuntimeCache(policy, ssh_key, known_hosts)
image_receipts: list[dict[str, Any]] = []
missing_count = 0
for image in policy.images:
source_ref = cache.verify_source(image)
source_fingerprint = "sha256:" + hashlib.sha256(source_ref.encode()).hexdigest()
target_present = _target_digest_present(image)
source_fingerprint = None
source_cache_verified = False
source_cache_verification_skipped_reason = "target_digest_already_verified"
execution = "already_present"
if not target_present:
missing_count += 1
if cache is None:
cache = RuntimeCache(policy, ssh_key, known_hosts)
source_ref = cache.verify_source(image)
source_fingerprint = (
"sha256:" + hashlib.sha256(source_ref.encode()).hexdigest()
)
source_cache_verified = True
source_cache_verification_skipped_reason = None
execution = "check_only_missing"
if apply:
source_preexisting = _local_image_present(source_ref)
@@ -736,10 +727,7 @@ def mirror_images(
"platform_digest": image.platform_digest,
"target_registry_digest": image.target_registry_digest,
"target_digest_ref": _target_digest_ref(image),
"source_cache_verified": source_cache_verified,
"source_cache_verification_skipped_reason": (
source_cache_verification_skipped_reason
),
"source_cache_verified": True,
"target_digest_verified": target_present,
"execution": execution,
}
@@ -755,7 +743,6 @@ def mirror_images(
"mode": "apply" if apply else "check",
"external_pull_allowed": False,
"source_cache_only": True,
"source_cache_accessed": cache is not None,
"candidate_count": len(image_receipts),
"missing_before_count": missing_count,
"verified_count": sum(
@@ -779,30 +766,20 @@ def stage_images(
known_hosts: Path,
apply: bool,
) -> dict[str, Any]:
cache: RuntimeCache | None = None
cache = RuntimeCache(policy, ssh_key, known_hosts)
image_receipts: list[dict[str, Any]] = []
missing_count = 0
for image in policy.images:
source_ref = cache.verify_source(image)
source_fingerprint = "sha256:" + hashlib.sha256(source_ref.encode()).hexdigest()
source_config_digest = cache.source_config_digest(image.platform_digest)
if source_config_digest != image.source_config_digest:
raise ControllerError("runtime_cache_config_digest_mismatch")
descriptor = _registry_descriptor(image.push_ref)
source_fingerprint = None
source_cache_verified = False
source_cache_verification_skipped_reason = (
"internal_registry_provenance_already_verified"
)
execution = "already_present"
if descriptor is None:
missing_count += 1
if cache is None:
cache = RuntimeCache(policy, ssh_key, known_hosts)
source_ref = cache.verify_source(image)
source_fingerprint = (
"sha256:" + hashlib.sha256(source_ref.encode()).hexdigest()
)
source_config_digest = cache.source_config_digest(image.platform_digest)
if source_config_digest != image.source_config_digest:
raise ControllerError("runtime_cache_config_digest_mismatch")
source_cache_verified = True
source_cache_verification_skipped_reason = None
execution = "check_only_missing"
if apply:
source_preexisting = _local_image_present(source_ref)
@@ -859,10 +836,7 @@ def stage_images(
raise ControllerError("internal_registry_descriptor_missing")
execution = "staged_from_runtime_cache"
if (
descriptor is not None
and descriptor.config_digest != image.source_config_digest
):
if descriptor is not None and descriptor.config_digest != source_config_digest:
raise ControllerError("internal_registry_provenance_mismatch")
target_digest_ref = None
runtime_ref = None
@@ -884,16 +858,13 @@ def stage_images(
"source_ref_fingerprint": source_fingerprint,
"source_index_digest": image.source_index_digest,
"platform_digest": image.platform_digest,
"source_config_digest": image.source_config_digest,
"source_config_digest": source_config_digest,
"target_registry_digest": (
descriptor.digest if descriptor is not None else None
),
"target_digest_ref": target_digest_ref,
"runtime_ref": runtime_ref,
"source_cache_verified": source_cache_verified,
"source_cache_verification_skipped_reason": (
source_cache_verification_skipped_reason
),
"source_cache_verified": True,
"target_digest_verified": target_verified,
"provenance_config_digest_match": descriptor is not None,
"execution": execution,
@@ -918,7 +889,6 @@ def stage_images(
"risk_level": policy.risk_level,
"external_pull_allowed": False,
"source_cache_only": True,
"source_cache_accessed": cache is not None,
"candidate_count": len(image_receipts),
"missing_before_count": missing_count,
"verified_count": sum(

View File

@@ -46,10 +46,6 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase):
policy = controller.load_staging_policy(STAGING_POLICY_PATH)
return replace(policy, images=(policy.images[0],))
def _single_image_policy(self):
policy = controller.load_policy(POLICY_PATH)
return replace(policy, images=(policy.images[0],))
def _image_with_availability_guard(self):
image = controller.load_policy(POLICY_PATH).images[0]
return replace(
@@ -908,12 +904,12 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase):
controller.RuntimeCache,
"verify_source",
return_value="quay.io/argoproj/argocd:v3.3.6",
) as verify_source,
),
patch.object(
controller.RuntimeCache,
"source_config_digest",
return_value=image.source_config_digest,
) as source_config_digest,
),
patch.object(controller.RuntimeCache, "export") as export,
patch.object(
controller, "_registry_descriptor", return_value=descriptor
@@ -932,85 +928,11 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase):
self.assertEqual(receipt["state"], "verified")
self.assertEqual(receipt["missing_before_count"], 0)
self.assertFalse(receipt["source_cache_accessed"])
self.assertEqual(receipt["images"][0]["execution"], "already_present")
self.assertTrue(receipt["images"][0]["provenance_config_digest_match"])
self.assertFalse(receipt["images"][0]["source_cache_verified"])
self.assertEqual(
receipt["images"][0]["source_cache_verification_skipped_reason"],
"internal_registry_provenance_already_verified",
)
verify_source.assert_not_called()
source_config_digest.assert_not_called()
export.assert_not_called()
run.assert_not_called()
def test_mirror_reuses_verified_target_digest_without_runtime_cache(self) -> None:
policy = self._single_image_policy()
with tempfile.TemporaryDirectory() as temp:
receipt_path = Path(temp) / "receipt.json"
with (
patch.object(
controller.RuntimeCache,
"verify_source",
return_value="quay.io/example/image:v1",
) as verify_source,
patch.object(controller.RuntimeCache, "export") as export,
patch.object(controller, "_target_digest_present", return_value=True),
patch.object(controller, "_run") as run,
):
receipt = controller.mirror_images(
policy,
trace_id="aia-p0-006-02a-existing",
receipt_path=receipt_path,
ssh_key=Path("/unused/key"),
known_hosts=Path("/unused/known-hosts"),
apply=True,
)
self.assertEqual(receipt["state"], "verified")
self.assertEqual(receipt["missing_before_count"], 0)
self.assertFalse(receipt["source_cache_accessed"])
self.assertEqual(receipt["images"][0]["execution"], "already_present")
self.assertFalse(receipt["images"][0]["source_cache_verified"])
self.assertEqual(
receipt["images"][0]["source_cache_verification_skipped_reason"],
"target_digest_already_verified",
)
verify_source.assert_not_called()
export.assert_not_called()
run.assert_not_called()
def test_mirror_missing_target_still_verifies_runtime_cache(self) -> None:
policy = self._single_image_policy()
with tempfile.TemporaryDirectory() as temp:
receipt_path = Path(temp) / "receipt.json"
with (
patch.object(
controller.RuntimeCache,
"verify_source",
return_value="quay.io/example/image:v1",
) as verify_source,
patch.object(controller.RuntimeCache, "export") as export,
patch.object(controller, "_target_digest_present", return_value=False),
):
receipt = controller.mirror_images(
policy,
trace_id="aia-p0-006-02a-missing-check",
receipt_path=receipt_path,
ssh_key=Path("/unused/key"),
known_hosts=Path("/unused/known-hosts"),
apply=False,
)
self.assertEqual(receipt["state"], "blocked_with_safe_next_action")
self.assertEqual(receipt["missing_before_count"], 1)
self.assertTrue(receipt["source_cache_accessed"])
self.assertEqual(receipt["images"][0]["execution"], "check_only_missing")
self.assertTrue(receipt["images"][0]["source_cache_verified"])
verify_source.assert_called_once_with(policy.images[0])
export.assert_not_called()
def test_staging_exports_cached_source_and_verifies_remote_provenance(
self,
) -> None:
@@ -1063,7 +985,6 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase):
self.assertEqual(receipt["state"], "verified")
self.assertEqual(receipt["missing_before_count"], 1)
self.assertTrue(receipt["source_cache_accessed"])
self.assertEqual(receipt["images"][0]["execution"], "staged_from_runtime_cache")
self.assertEqual(
receipt["images"][0]["target_registry_digest"], descriptor.digest