feat(agent99): bind host110 backup runtime deploy

This commit is contained in:
Your Name
2026-07-18 21:21:05 +08:00
parent e7e3bcf8a5
commit 4b2b73f1a7
19 changed files with 1453 additions and 23 deletions

View File

@@ -224,6 +224,7 @@ $agentFiles = @(
"agent99-control-plane.ps1",
"agent99-db-bounded-executor.ps1",
"agent99-signoz-metadata-executor.ps1",
"agent99-host110-backup-runtime-broker.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",

View File

@@ -20,6 +20,7 @@ $requiredFiles = @(
"agent99-control-plane.ps1",
"agent99-db-bounded-executor.ps1",
"agent99-signoz-metadata-executor.ps1",
"agent99-host110-backup-runtime-broker.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",
@@ -131,6 +132,7 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js
$control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane.ps1"), [Text.Encoding]::UTF8)
$dbBoundedExecutor = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-db-bounded-executor.ps1"), [Text.Encoding]::UTF8)
$signozMetadataExecutor = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-signoz-metadata-executor.ps1"), [Text.Encoding]::UTF8)
$host110BackupRuntimeBroker = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-host110-backup-runtime-broker.ps1"), [Text.Encoding]::UTF8)
$inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8)
$telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8)
$bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8)
@@ -203,6 +205,27 @@ $signozMetadataExecutorContract = [bool](
-not $signozMetadataExecutor.Contains('Get-Content -LiteralPath $CredentialFile')
)
Add-Check "backup:signoz_metadata_fixed_executor" $signozMetadataExecutorContract "Windows99 dispatches one fixed host110 metadata export contract, consumes only a root-owned credential reference on target, and requires the independent bundle verifier without raw SQLite or secret output"
$host110BackupRuntimeContract = [bool](
$host110BackupRuntimeBroker.Contains('$TargetHost = "192.168.0.110"') -and
$host110BackupRuntimeBroker.Contains('$TargetUser = "wooo"') -and
$host110BackupRuntimeBroker.Contains('$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"') -and
$host110BackupRuntimeBroker.Contains('"credential.helper="') -and
$host110BackupRuntimeBroker.Contains('$env:GIT_TERMINAL_PROMPT = "0"') -and
$host110BackupRuntimeBroker.Contains('"merge-base", "--is-ancestor", $SourceRevision, $sourceHead') -and
$host110BackupRuntimeBroker.Contains('executorSha256 = $executorDigest') -and
$host110BackupRuntimeBroker.Contains('verifierSha256 = $verifierDigest') -and
$host110BackupRuntimeBroker.Contains('StrictHostKeyChecking=yes') -and
$host110BackupRuntimeBroker.Contains('$process.WaitForExit($TimeoutSeconds * 1000)') -and
$host110BackupRuntimeBroker.Contains('taskkill.exe /PID $process.Id /T /F') -and
$host110BackupRuntimeBroker.Contains('Invoke-Agent99BoundedScp $sourcePackage.localPaths') -and
$host110BackupRuntimeBroker.Contains('python3 - --manifest-base64') -and
$host110BackupRuntimeBroker.Contains('Write-Agent99ImmutableJson') -and
$host110BackupRuntimeBroker.Contains('decisionProvider = "deterministic_only"') -and
$host110BackupRuntimeBroker.Contains('criticProvider = "deterministic_only"') -and
-not $host110BackupRuntimeBroker.Contains('/home/wooo/awoooi') -and
-not $host110BackupRuntimeBroker.Contains('Invoke-Expression')
)
Add-Check "backup:host110_runtime_fixed_broker" $host110BackupRuntimeContract "Windows99 fetches one exact Gitea revision, binds executor and verifier digests, performs one fixed process-bounded Host110 transaction, and preserves an independent no-write verifier readback"
Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("Enter-AgentRecoveryTriggerReservation") -and $control.Contains('"recovery-trigger-reservation.lock"') -and $control.Contains('[System.IO.FileShare]::None') -and $control.Contains('reason = "recovery_trigger_reservation_busy"') -and $control.Contains("recovery_already_queued")) "boot and host-down recovery reserve one atomic single-flight queue decision"
Add-Check "recovery:auto_trigger_controlled_identity" ($control.Contains("New-AgentObservedRecoveryDispatchIdentity") -and $control.Contains('-RouteId "agent99_local_observer"') -and $control.Contains('schema_version = "agent99_controlled_dispatch_identity_v1"') -and $control.Contains('correlationKey = $dispatchIdentity.idempotencyKey') -and $control.Contains('canonicalDigest = $dispatchIdentity.canonicalDigest') -and $control.Contains("Invoke-AgentControlledDispatchIdentitySelfTest")) "observer-created Recover requests carry a stable canonical controlled dispatch identity before queue processing"
Add-Check "recovery:queue_identity_fail_closed" ($control.Contains('dispatchIdentitySchemaVersion = $dispatchIdentity.schemaVersion') -and $control.Contains('Test-AgentControlledDispatchIdentity $dispatchIdentityEnvelope') -and $control.Contains('"controlled_dispatch_identity_invalid"') -and $control.IndexOf('"controlled_dispatch_identity_invalid"') -lt $control.IndexOf('$process = Start-Process -FilePath "powershell.exe" -ArgumentList $args')) "queue consumer rebuilds and validates canonical identity before any controlled executor starts"

View File

@@ -35,6 +35,7 @@ $runtimeFiles = @(
"agent99-control-plane.ps1",
"agent99-db-bounded-executor.ps1",
"agent99-signoz-metadata-executor.ps1",
"agent99-host110-backup-runtime-broker.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",

View File

@@ -0,0 +1,474 @@
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[ValidateSet("Check", "Apply", "Verify")]
[string]$Mode,
[Parameter(Mandatory = $true)]
[string]$SourceRevision,
[Parameter(Mandatory = $true)]
[string]$RunId,
[string]$AgentRoot = "C:\Wooo\Agent99"
)
$ErrorActionPreference = "Stop"
$ExpectedAgentRoot = "C:\Wooo\Agent99"
$TargetHost = "192.168.0.110"
$TargetUser = "wooo"
$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"
$GitPath = "C:\Program Files\Git\cmd\git.exe"
$IdentityFile = Join-Path $ExpectedAgentRoot "keys\agent99_ed25519"
$EvidenceRoot = Join-Path $ExpectedAgentRoot "evidence\host110-backup-runtime"
$SourceRootBase = Join-Path $ExpectedAgentRoot "deploy"
$TransportMutexName = "Global\WoooAgent99Host110BackupRuntimeV2"
$ExpectedFileCount = 17
$ExpectedRemoteStageFileCount = 20
$RuntimeRelativePaths = @(
"scripts/backup/common.sh",
"scripts/backup/backup-all.sh",
"scripts/backup/backup-host188-products.sh",
"scripts/backup/verify-host188-products-backup.sh",
"scripts/backup/verify-host188-products-archive.py",
"scripts/backup/backup-gitea.sh",
"scripts/backup/gitea-full-backup-restore-drill.sh",
"scripts/backup/backup-configs.sh",
"scripts/backup/backup-awoooi.sh",
"scripts/backup/backup-awoooi-frequent.sh",
"scripts/backup/backup-clawbot.sh",
"scripts/backup/backup-sentry.sh",
"scripts/backup/check-backup-integrity.sh",
"scripts/backup/sync-offsite-backups.sh",
"scripts/backup/backup-offsite-readiness-gate.sh",
"scripts/backup/verify-offsite-full-sync.sh",
"scripts/backup/enforce-latest-only-retention.sh"
)
$ExecutorRelativePath = "scripts/backup/host110-backup-runtime-executor.sh"
$VerifierRelativePath = "scripts/backup/verify-host110-backup-runtime.py"
function Test-Agent99SafeIdentity {
param([string]$Value)
return [bool]($Value -and $Value -match "^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$")
}
function Write-Agent99ImmutableJson {
param([string]$Path, [object]$Value)
if (Test-Path -LiteralPath $Path) { throw "run_identity_evidence_exists" }
$temporary = "$Path.tmp-$PID-$([Guid]::NewGuid().ToString('N'))"
$encoding = New-Object System.Text.UTF8Encoding($false)
try {
$json = $Value | ConvertTo-Json -Depth 12
[IO.File]::WriteAllText($temporary, $json + [Environment]::NewLine, $encoding)
[IO.File]::Move($temporary, $Path)
} finally {
Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue | Out-Null
}
}
function Invoke-Agent99BoundedProcess {
param(
[string]$FilePath,
[string[]]$Arguments,
[int]$TimeoutSeconds,
[string]$WorkingDirectory = "",
[string]$StandardInputPath = ""
)
$token = [Guid]::NewGuid().ToString("N")
$stdoutPath = Join-Path $env:TEMP "agent99-host110-backup-$token.out"
$stderrPath = Join-Path $env:TEMP "agent99-host110-backup-$token.err"
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
$process = $null
try {
$start = @{
FilePath = $FilePath
ArgumentList = $Arguments
NoNewWindow = $true
RedirectStandardOutput = $stdoutPath
RedirectStandardError = $stderrPath
PassThru = $true
}
if ($WorkingDirectory) { $start.WorkingDirectory = $WorkingDirectory }
if ($StandardInputPath) { $start.RedirectStandardInput = $StandardInputPath }
$process = Start-Process @start
$null = $process.Handle
if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {}
if (-not $process.WaitForExit(2000) -and -not $process.HasExited) {
$process.Kill()
$process.WaitForExit(2000) | Out-Null
}
return [pscustomobject]@{
ok = $false
exitCode = -2
reason = "transport_timeout"
elapsedMs = $stopwatch.ElapsedMilliseconds
stdout = ""
stderrPresent = $false
}
}
$process.WaitForExit() | Out-Null
$process.Refresh()
$stdout = if (Test-Path -LiteralPath $stdoutPath) { Get-Content -LiteralPath $stdoutPath -Raw } else { "" }
$stderr = if (Test-Path -LiteralPath $stderrPath) { Get-Content -LiteralPath $stderrPath -Raw } else { "" }
if ($stdout.Length -gt 65536 -or $stderr.Length -gt 65536) {
return [pscustomobject]@{
ok = $false
exitCode = -3
reason = "transport_output_oversized"
elapsedMs = $stopwatch.ElapsedMilliseconds
stdout = ""
stderrPresent = [bool]$stderr
}
}
return [pscustomobject]@{
ok = [bool]($process.ExitCode -eq 0)
exitCode = [int]$process.ExitCode
reason = if ($process.ExitCode -eq 0) { "completed" } else { "process_failed" }
elapsedMs = $stopwatch.ElapsedMilliseconds
stdout = [string]$stdout
stderrPresent = [bool]$stderr
}
} catch {
return [pscustomobject]@{
ok = $false
exitCode = -1
reason = "transport_exception"
elapsedMs = $stopwatch.ElapsedMilliseconds
stdout = ""
stderrPresent = $false
errorType = $_.Exception.GetType().Name
}
} finally {
$stopwatch.Stop()
Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null
if ($process) { $process.Dispose() }
}
}
function Invoke-Agent99BoundedSsh {
param([string]$RemoteCommand, [int]$TimeoutSeconds, [string]$StandardInputPath = "")
$arguments = @(
"-i", $IdentityFile,
"-o", "IdentitiesOnly=yes",
"-o", "BatchMode=yes",
"-o", "NumberOfPasswordPrompts=0",
"-o", "StrictHostKeyChecking=yes",
"-o", "ConnectTimeout=8",
"-o", "ConnectionAttempts=1",
"-o", "ServerAliveInterval=10",
"-o", "ServerAliveCountMax=2",
"-l", $TargetUser,
$TargetHost,
$RemoteCommand
)
return Invoke-Agent99BoundedProcess "ssh.exe" $arguments $TimeoutSeconds "" $StandardInputPath
}
function Invoke-Agent99BoundedScp {
param([string[]]$LocalPaths, [string]$RemoteDirectory, [int]$TimeoutSeconds)
$arguments = @(
"-i", $IdentityFile,
"-o", "IdentitiesOnly=yes",
"-o", "BatchMode=yes",
"-o", "NumberOfPasswordPrompts=0",
"-o", "StrictHostKeyChecking=yes",
"-o", "ConnectTimeout=8",
"-o", "ConnectionAttempts=1"
)
$arguments += $LocalPaths
$arguments += "${TargetUser}@${TargetHost}:${RemoteDirectory}/"
return Invoke-Agent99BoundedProcess "scp.exe" $arguments $TimeoutSeconds
}
function Invoke-Agent99Git {
param([string[]]$Arguments, [int]$TimeoutSeconds, [string]$WorkingDirectory = "")
$previousPrompt = $env:GIT_TERMINAL_PROMPT
try {
$env:GIT_TERMINAL_PROMPT = "0"
return Invoke-Agent99BoundedProcess $GitPath $Arguments $TimeoutSeconds $WorkingDirectory
} finally {
$env:GIT_TERMINAL_PROMPT = $previousPrompt
}
}
function Assert-Agent99ProcessOk {
param([object]$Result, [string]$Reason)
if (-not $Result.ok) { throw $Reason }
}
function New-Agent99SourcePackage {
param([string]$SourceRoot, [string]$ManifestPath)
if (Test-Path -LiteralPath $SourceRoot) { throw "run_identity_source_stage_exists" }
$clone = Invoke-Agent99Git @(
"-c", "credential.helper=",
"-c", "core.autocrlf=false",
"clone", "--quiet", "--no-tags", "--depth", "200", "--branch", "main",
$GiteaSourceUrl, $SourceRoot
) 180 $SourceRootBase
Assert-Agent99ProcessOk $clone "gitea_source_fetch_failed"
$headResult = Invoke-Agent99Git @("rev-parse", "HEAD") 30 $SourceRoot
Assert-Agent99ProcessOk $headResult "gitea_source_head_failed"
$sourceHead = ([string]$headResult.stdout).Trim().ToLowerInvariant()
if ($sourceHead -notmatch "^[0-9a-f]{40}$") { throw "gitea_source_head_invalid" }
$commitResult = Invoke-Agent99Git @("cat-file", "-e", "$SourceRevision`^{commit}") 30 $SourceRoot
Assert-Agent99ProcessOk $commitResult "source_revision_not_fetched"
$ancestorResult = Invoke-Agent99Git @("merge-base", "--is-ancestor", $SourceRevision, $sourceHead) 30 $SourceRoot
Assert-Agent99ProcessOk $ancestorResult "source_revision_not_on_gitea_main"
$allPaths = @($RuntimeRelativePaths) + @($ExecutorRelativePath, $VerifierRelativePath)
$checkoutResult = Invoke-Agent99Git (@("-c", "core.autocrlf=false", "checkout", "--quiet", $SourceRevision, "--") + $allPaths) 60 $SourceRoot
Assert-Agent99ProcessOk $checkoutResult "source_revision_checkout_failed"
$rows = @()
$localPaths = @()
foreach ($relativePath in $RuntimeRelativePaths) {
$path = Join-Path $SourceRoot ($relativePath -replace "/", "\")
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { throw "runtime_source_file_missing" }
$rows += [ordered]@{
name = [IO.Path]::GetFileName($path)
sha256 = (Get-FileHash -LiteralPath $path -Algorithm SHA256).Hash.ToLowerInvariant()
}
$localPaths += $path
}
if (($rows.name | Sort-Object -Unique).Count -ne $ExpectedFileCount) { throw "runtime_source_file_set_invalid" }
$executorPath = Join-Path $SourceRoot ($ExecutorRelativePath -replace "/", "\")
$verifierPath = Join-Path $SourceRoot ($VerifierRelativePath -replace "/", "\")
if (-not (Test-Path -LiteralPath $executorPath -PathType Leaf)) { throw "executor_source_file_missing" }
if (-not (Test-Path -LiteralPath $verifierPath -PathType Leaf)) { throw "verifier_source_file_missing" }
$executorDigest = (Get-FileHash -LiteralPath $executorPath -Algorithm SHA256).Hash.ToLowerInvariant()
$verifierDigest = (Get-FileHash -LiteralPath $verifierPath -Algorithm SHA256).Hash.ToLowerInvariant()
$manifest = [ordered]@{
schemaVersion = "agent99_host110_backup_runtime_source_manifest_v1"
sourceRevision = $SourceRevision
sourceHead = $sourceHead
runId = $RunId
executorSha256 = $executorDigest
verifierSha256 = $verifierDigest
files = $rows
}
$encoding = New-Object System.Text.UTF8Encoding($false)
[IO.File]::WriteAllText($ManifestPath, ($manifest | ConvertTo-Json -Depth 8 -Compress), $encoding)
$localPaths += $executorPath, $verifierPath, $ManifestPath
return [pscustomobject]@{
sourceHead = $sourceHead
executorSha256 = $executorDigest
verifierSha256 = $verifierDigest
verifierPath = $verifierPath
manifestPath = $ManifestPath
manifestBase64 = [Convert]::ToBase64String([IO.File]::ReadAllBytes($ManifestPath))
localPaths = $localPaths
}
}
function Get-Agent99ExecutorResult {
param([object]$Transport, [string]$ExpectedMode)
if (-not $Transport.ok) { throw "host110_${ExpectedMode}_transport_failed" }
try {
$parsed = [string]$Transport.stdout | ConvertFrom-Json
} catch {
throw "host110_${ExpectedMode}_receipt_invalid"
}
if (
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_executor_v1" -or
[string]$parsed.mode -ne $ExpectedMode.ToLowerInvariant() -or
-not [bool]$parsed.ok -or
[string]$parsed.sourceRevision -ne $SourceRevision -or
[int]$parsed.fileCount -ne $ExpectedFileCount
) {
throw "host110_${ExpectedMode}_receipt_contract_failed"
}
if ($ExpectedMode -eq "Apply") {
if (
[string]$parsed.verifier.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
-not [bool]$parsed.verifier.ok -or
[bool]$parsed.verifier.remoteWritePerformed -or
-not [bool]$parsed.verifier.selfIdentityVerified
) {
throw "host110_apply_independent_verifier_contract_failed"
}
}
return $parsed
}
function Get-Agent99VerifierResult {
param([object]$Transport, [object]$SourcePackage)
if (-not $Transport.ok) { throw "host110_verify_transport_failed" }
try {
$parsed = [string]$Transport.stdout | ConvertFrom-Json
} catch {
throw "host110_verify_receipt_invalid"
}
if (
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
-not [bool]$parsed.ok -or
[string]$parsed.sourceRevision -ne $SourceRevision -or
[string]$parsed.runId -ne $RunId -or
[int]$parsed.fileCount -ne $ExpectedFileCount -or
[bool]$parsed.remoteWritePerformed -or
[string]$parsed.verifierSha256 -ne [string]$SourcePackage.verifierSha256
) {
throw "host110_verify_receipt_contract_failed"
}
return $parsed
}
function Invoke-Agent99ReadOnlyPreflight {
param([object]$SourcePackage)
$command = 'test "$(id -un)" = wooo && hostname -I | tr " " "\n" | grep -qx 192.168.0.110 && test -d /backup/scripts && test ! -L /backup/scripts && command -v timeout >/dev/null && command -v flock >/dev/null && command -v python3 >/dev/null && printf HOST110_BACKUP_PREFLIGHT_OK=1'
$transport = Invoke-Agent99BoundedSsh $command 45
if (-not $transport.ok -or ([string]$transport.stdout).Trim() -ne "HOST110_BACKUP_PREFLIGHT_OK=1") {
throw "host110_read_only_preflight_failed"
}
return [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_preflight_v1"
ok = $true
sourceRevision = $SourceRevision
sourceHead = [string]$SourcePackage.sourceHead
fileCount = $ExpectedFileCount
remoteWritePerformed = $false
}
}
function Invoke-Agent99NoWriteVerifier {
param([object]$SourcePackage)
$command = "timeout --signal=TERM --kill-after=5s 60s python3 - --manifest-base64 $($SourcePackage.manifestBase64) --destination /backup/scripts --source-revision $SourceRevision --run-id $RunId --expected-verifier-sha256 $($SourcePackage.verifierSha256)"
$transport = Invoke-Agent99BoundedSsh $command 90 ([string]$SourcePackage.verifierPath)
return Get-Agent99VerifierResult $transport $SourcePackage
}
function Remove-Agent99RemoteStage {
param([string]$RemoteStage, [bool]$Required)
$cleanup = Invoke-Agent99BoundedSsh "rm -rf -- $RemoteStage && test ! -e $RemoteStage" 45
if ($Required -and -not $cleanup.ok) { throw "remote_source_stage_cleanup_failed" }
return [bool]$cleanup.ok
}
if ($AgentRoot.TrimEnd("\") -ne $ExpectedAgentRoot) { throw "agent_root_override_forbidden" }
$SourceRevision = $SourceRevision.ToLowerInvariant()
if ($SourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
if (-not (Test-Agent99SafeIdentity $RunId)) { throw "invalid_run_id" }
if (-not (Test-Path -LiteralPath $IdentityFile -PathType Leaf)) { throw "agent99_ssh_identity_unavailable" }
if (-not (Test-Path -LiteralPath $GitPath -PathType Leaf)) { throw "windows_git_unavailable" }
if (-not (Get-Command ssh.exe -ErrorAction SilentlyContinue)) { throw "windows_openssh_unavailable" }
if (-not (Get-Command scp.exe -ErrorAction SilentlyContinue)) { throw "windows_openscp_unavailable" }
New-Item -ItemType Directory -Force -Path $EvidenceRoot, $SourceRootBase | Out-Null
$evidencePath = Join-Path $EvidenceRoot "agent99-host110-backup-runtime-$RunId.json"
if (Test-Path -LiteralPath $evidencePath -PathType Leaf) {
try { $existing = Get-Content -LiteralPath $evidencePath -Raw | ConvertFrom-Json } catch { throw "run_identity_evidence_invalid" }
if ([string]$existing.sourceRevision -ne $SourceRevision -or [string]$existing.mode -ne $Mode) {
throw "run_identity_conflict"
}
$existing | ConvertTo-Json -Compress -Depth 12
if (-not [bool]$existing.ok) { exit 1 }
exit 0
}
$sourceRoot = Join-Path $SourceRootBase "host110-backup-source-$RunId"
$manifestPath = Join-Path $sourceRoot "agent99-host110-backup-runtime-manifest.json"
$remoteStage = "/tmp/agent99-host110-backup-runtime-$RunId"
$startedAt = [DateTime]::UtcNow
$mutex = [Threading.Mutex]::new($false, $TransportMutexName)
$acquired = $false
$remoteStageCreated = $false
$sourcePackage = $null
$check = $null
$apply = $null
$verify = $null
$terminalStatus = "failed"
$errorCode = ""
$cleanupVerified = $false
try {
try {
$acquired = [bool]$mutex.WaitOne(30000)
} catch [Threading.AbandonedMutexException] {
$acquired = $true
}
if (-not $acquired) { throw "host110_backup_runtime_transport_busy" }
$sourcePackage = New-Agent99SourcePackage $sourceRoot $manifestPath
$check = Invoke-Agent99ReadOnlyPreflight $sourcePackage
if ($Mode -eq "Apply") {
$createStage = Invoke-Agent99BoundedSsh "test ! -e $remoteStage && install -d -m 700 $remoteStage" 45
Assert-Agent99ProcessOk $createStage "remote_source_stage_identity_conflict"
$remoteStageCreated = $true
$copyStage = Invoke-Agent99BoundedScp $sourcePackage.localPaths $remoteStage 120
Assert-Agent99ProcessOk $copyStage "remote_source_stage_copy_failed"
$stageCheck = Invoke-Agent99BoundedSsh "chmod 700 $remoteStage/host110-backup-runtime-executor.sh $remoteStage/verify-host110-backup-runtime.py && test `$(find $remoteStage -mindepth 1 -maxdepth 1 -type f | wc -l) -eq $ExpectedRemoteStageFileCount" 45
Assert-Agent99ProcessOk $stageCheck "remote_source_stage_contract_failed"
$executorPath = "$remoteStage/host110-backup-runtime-executor.sh"
$command = "timeout --signal=TERM --kill-after=10s 300s $executorPath --mode apply --source-revision $SourceRevision --run-id $RunId --source-stage $remoteStage"
$applyTransport = Invoke-Agent99BoundedSsh $command 330
$apply = Get-Agent99ExecutorResult $applyTransport "Apply"
$verify = $apply.verifier
$cleanupVerified = Remove-Agent99RemoteStage $remoteStage $true
$remoteStageCreated = $false
$terminalStatus = "verified"
} elseif ($Mode -eq "Verify") {
$verify = Invoke-Agent99NoWriteVerifier $sourcePackage
$terminalStatus = "readback_ok"
$cleanupVerified = $true
} else {
$terminalStatus = "readback_ok"
$cleanupVerified = $true
}
} catch {
$errorCode = [string]$_.Exception.Message
} finally {
if ($remoteStageCreated) {
try { $cleanupVerified = Remove-Agent99RemoteStage $remoteStage $false } catch {}
}
if ($acquired) { try { $mutex.ReleaseMutex() } catch {} }
$mutex.Dispose()
}
$result = [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v2"
ok = [bool]($terminalStatus -ne "failed")
status = $terminalStatus
mode = $Mode
sourceRevision = $SourceRevision
sourceHead = if ($sourcePackage) { [string]$sourcePackage.sourceHead } else { "" }
runId = $RunId
targetHost = $TargetHost
sourceTransport = "windows99_gitea_exact_revision_manifest"
executor = "host110_backup_runtime_executor"
verifier = "independent_host110_backup_runtime_python_readback"
decisionProvider = "deterministic_only"
criticProvider = "deterministic_only"
agentAction = if ($Mode -eq "Apply") { "controlled_apply" } else { "readback" }
check = $check
apply = $apply
verify = $verify
cleanupVerified = $cleanupVerified
errorCode = $errorCode
elapsedSeconds = [math]::Round(([DateTime]::UtcNow - $startedAt).TotalSeconds, 3)
evidence = $evidencePath
secretValuesRead = $false
rawSessionStored = $false
}
try {
Write-Agent99ImmutableJson $evidencePath $result
} finally {
if (Test-Path -LiteralPath $sourceRoot) {
Remove-Item -LiteralPath $sourceRoot -Recurse -Force -ErrorAction SilentlyContinue
}
}
$result | ConvertTo-Json -Compress -Depth 12
if (-not $result.ok) { exit 1 }

View File

@@ -9,6 +9,16 @@
set -euo pipefail
if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then
command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; }
[ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; }
(umask 077; : >> /tmp/agent99-host110-backup-runtime.lock)
[ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; }
exec 197>>/tmp/agent99-host110-backup-runtime.lock
flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; }
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
fi
BACKUP_BASE="${BACKUP_BASE:-/backup}"
OFFSITE_ENV_FILE="${BACKUP_OFFSITE_ENV_FILE:-${BACKUP_BASE}/scripts/offsite.env}"
OFFSITE_DIR="${BACKUP_OFFSITE_STATUS_DIR:-${BACKUP_BASE}/offsite}"

View File

@@ -1,5 +1,38 @@
#\!/bin/bash
#!/bin/bash
# =============================================================================
# Production backup entrypoints share this stable gate. Agent99 takes the
# exclusive side while promoting a complete, digest-bound runtime bundle.
acquire_backup_runtime_shared_lock() {
local caller_path="${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}"
local lock_path="/tmp/agent99-host110-backup-runtime.lock"
case "${caller_path}" in
/backup/scripts/*) ;;
*) return 0 ;;
esac
[ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ] || return 0
command -v flock >/dev/null 2>&1 || {
echo "backup runtime gate unavailable: flock missing" >&2
return 69
}
[ ! -L "${lock_path}" ] || {
echo "backup runtime gate unsafe: lock is a symlink" >&2
return 69
}
(umask 077; : >> "${lock_path}") || return 69
[ -f "${lock_path}" ] && [ -O "${lock_path}" ] || return 69
exec 197>>"${lock_path}"
flock -s -w 60 197 || {
echo "backup runtime deployment is active; retry later" >&2
return 75
}
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
}
acquire_backup_runtime_shared_lock || {
runtime_lock_status=$?
return "${runtime_lock_status}" 2>/dev/null || exit "${runtime_lock_status}"
}
# WOOO AIOps - 備份共用函式庫
# 版本: 1.0.0
# 建立日期: 2026-03-12

View File

@@ -5,6 +5,16 @@
set -euo pipefail
if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then
command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; }
[ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; }
(umask 077; : >> /tmp/agent99-host110-backup-runtime.lock)
[ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; }
exec 197>>/tmp/agent99-host110-backup-runtime.lock
flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; }
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
fi
DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}"
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
TEXTFILE_PATH="${AIOPS_GITEA_FULL_BACKUP_RESTORE_DRILL_TEXTFILE:-/home/wooo/node_exporter_textfiles/gitea_full_backup_restore_drill.prom}"

View File

@@ -0,0 +1,431 @@
#!/usr/bin/env bash
# Fixed Host110 executor for Windows99 Agent99 backup-runtime deployment.
set -euo pipefail
umask 077
readonly EXPECTED_HOST_IP="192.168.0.110"
readonly EXPECTED_USER="wooo"
readonly DEST_ROOT="/backup/scripts"
readonly STATUS_ROOT="/backup/status"
readonly STAGE_ROOT="/backup/.agent99-backup-runtime-stage"
readonly ROLLBACK_ROOT="/backup/.agent99-backup-runtime-rollback"
readonly LOCK_PATH="/tmp/agent99-host110-backup-runtime.lock"
readonly -a RUNTIME_FILES=(
common.sh
backup-all.sh
backup-host188-products.sh
verify-host188-products-backup.sh
verify-host188-products-archive.py
backup-gitea.sh
gitea-full-backup-restore-drill.sh
backup-configs.sh
backup-awoooi.sh
backup-awoooi-frequent.sh
backup-clawbot.sh
backup-sentry.sh
check-backup-integrity.sh
sync-offsite-backups.sh
backup-offsite-readiness-gate.sh
verify-offsite-full-sync.sh
enforce-latest-only-retention.sh
)
MODE=""
SOURCE_REVISION=""
RUN_ID=""
SOURCE_STAGE=""
SOURCE_HEAD=""
EXECUTOR_DIGEST=""
VERIFIER_DIGEST=""
STAGE_DIR=""
ROLLBACK_DIR=""
APPLY_STARTED=0
APPLY_COMPLETE=0
ROLLBACK_VERIFIED=0
declare -A FILE_EXISTED=()
declare -A PREVIOUS_DIGEST=()
declare -A EXPECTED_DIGEST=()
usage() {
printf '%s\n' \
"Usage: $0 --mode check|apply|verify --source-revision SHA --run-id ID --source-stage PATH" \
"The executor is fixed to host 192.168.0.110, an Agent99 run-bound source stage," \
"and the 17-file /backup/scripts transaction."
}
fail() {
printf 'HOST110_BACKUP_RUNTIME_OK=0\nERROR=%s\n' "$1" >&2
exit 64
}
while [ "$#" -gt 0 ]; do
case "$1" in
--mode)
shift
MODE="${1:-}"
;;
--source-revision)
shift
SOURCE_REVISION="${1:-}"
;;
--run-id)
shift
RUN_ID="${1:-}"
;;
--source-stage)
shift
SOURCE_STAGE="${1:-}"
;;
-h|--help)
usage
exit 0
;;
*) fail "unknown_argument" ;;
esac
shift
done
case "$MODE" in check|apply|verify) ;; *) fail "invalid_mode" ;; esac
[[ "$SOURCE_REVISION" =~ ^[0-9a-f]{40}$ ]] || fail "invalid_source_revision"
[[ "$RUN_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$ ]] || fail "invalid_run_id"
expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"
[ "$SOURCE_STAGE" = "$expected_source_stage" ] || fail "invalid_source_stage"
for command_name in awk bash cp flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do
command -v "$command_name" >/dev/null 2>&1 || fail "required_command_missing_${command_name}"
done
current_user="$(id -un)"
[ "$current_user" = "$EXPECTED_USER" ] || fail "unexpected_executor_user"
hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx "$EXPECTED_HOST_IP" || fail "unexpected_executor_host"
[ -d "$SOURCE_STAGE" ] && [ ! -L "$SOURCE_STAGE" ] || fail "canonical_source_stage_unavailable"
[ -d "$DEST_ROOT" ] && [ ! -L "$DEST_ROOT" ] || fail "backup_runtime_destination_unavailable"
manifest_rows="$(python3 - "$SOURCE_STAGE/manifest.json" "$SOURCE_REVISION" "$RUN_ID" "${RUNTIME_FILES[@]}" <<'PY'
import json
import re
import sys
from pathlib import Path
path = Path(sys.argv[1])
source_revision = sys.argv[2]
run_id = sys.argv[3]
expected_names = sys.argv[4:]
try:
document = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
raise SystemExit(65)
rows = document.get("files")
if (
document.get("schemaVersion") != "agent99_host110_backup_runtime_source_manifest_v1"
or document.get("sourceRevision") != source_revision
or document.get("runId") != run_id
or not re.fullmatch(r"[0-9a-f]{40}", str(document.get("sourceHead", "")))
or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("executorSha256", "")))
or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("verifierSha256", "")))
or not isinstance(rows, list)
or len(rows) != len(expected_names)
):
raise SystemExit(66)
by_name = {}
for row in rows:
if not isinstance(row, dict):
raise SystemExit(66)
name = str(row.get("name", ""))
digest = str(row.get("sha256", ""))
if name in by_name or not re.fullmatch(r"[A-Za-z0-9_.-]+", name) or not re.fullmatch(r"[0-9a-f]{64}", digest):
raise SystemExit(66)
by_name[name] = digest
if set(by_name) != set(expected_names):
raise SystemExit(66)
print(f"SOURCE_HEAD {document['sourceHead']}")
print(f"EXECUTOR_SHA256 {document['executorSha256']}")
print(f"VERIFIER_SHA256 {document['verifierSha256']}")
for name in expected_names:
print(f"{name} {by_name[name]}")
PY
)" || fail "source_manifest_invalid"
while read -r manifest_name manifest_digest; do
case "$manifest_name" in
SOURCE_HEAD) SOURCE_HEAD="$manifest_digest" ;;
EXECUTOR_SHA256) EXECUTOR_DIGEST="$manifest_digest" ;;
VERIFIER_SHA256) VERIFIER_DIGEST="$manifest_digest" ;;
*) EXPECTED_DIGEST[$manifest_name]="$manifest_digest" ;;
esac
done <<< "$manifest_rows"
[[ "$SOURCE_HEAD" =~ ^[0-9a-f]{40}$ ]] || fail "source_head_invalid"
working_digest() {
sha256sum "$1" | awk '{print $1}'
}
[ "$(working_digest "$0")" = "$EXECUTOR_DIGEST" ] || fail "executor_identity_failed"
[ -f "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \
&& [ ! -L "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \
&& [ "$(working_digest "$SOURCE_STAGE/verify-host110-backup-runtime.py")" = "$VERIFIER_DIGEST" ] \
|| fail "verifier_identity_failed"
validate_file() {
local path="$1"
case "$path" in
*.sh) bash -n "$path" ;;
*.py) python3 -c 'import ast, pathlib, sys; ast.parse(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))' "$path" ;;
*) return 64 ;;
esac
}
source_digest() {
local name="$1"
printf '%s\n' "${EXPECTED_DIGEST[$name]:-}"
}
validate_source() {
local name source_path expected_digest actual_digest resolved
for name in "${RUNTIME_FILES[@]}"; do
source_path="$SOURCE_STAGE/$name"
[ -f "$source_path" ] && [ ! -L "$source_path" ] || return 65
resolved="$(readlink -f "$source_path")"
[[ "$resolved" == "$SOURCE_STAGE/"* ]] || return 65
expected_digest="$(source_digest "$name")"
actual_digest="$(working_digest "$source_path")"
[ "$actual_digest" = "$expected_digest" ] || return 66
validate_file "$source_path" || return 67
done
}
verify_destination() {
local name dest_path expected_digest actual_digest
for name in "${RUNTIME_FILES[@]}"; do
dest_path="$DEST_ROOT/$name"
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] || return 68
expected_digest="$(source_digest "$name")"
actual_digest="$(working_digest "$dest_path")"
[ "$actual_digest" = "$expected_digest" ] || return 69
[ "$(stat -c '%U:%G:%a' "$dest_path")" = "wooo:wooo:755" ] || return 70
done
}
write_receipt() {
local status="$1"
local rollback_verified="$2"
local receipt_path="$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json"
install -d -m 700 "$STATUS_ROOT"
python3 - "$receipt_path" "$status" "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$rollback_verified" "$VERIFIER_DIGEST" <<'PY'
import json
import os
import sys
import time
from pathlib import Path
path = Path(sys.argv[1])
document = {
"schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v1",
"status": sys.argv[2],
"sourceRevision": sys.argv[3],
"sourceHead": sys.argv[4],
"runId": sys.argv[5],
"fileCount": 17,
"rollbackPerformed": sys.argv[6] == "1",
"rollbackVerified": sys.argv[6] == "1",
"verifierSha256": sys.argv[7],
"executorHost": "192.168.0.110",
"productionServiceRestarted": False,
"secretValuesRead": False,
"writtenAt": int(time.time()),
}
temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
temporary.write_text(json.dumps(document, ensure_ascii=True, sort_keys=True) + "\n", encoding="utf-8")
os.chmod(temporary, 0o600)
try:
os.link(temporary, path)
finally:
temporary.unlink(missing_ok=True)
PY
printf '%s' "$receipt_path"
}
record_prestate() {
local name dest_path digest
: > "$ROLLBACK_DIR/prestate.tsv"
chmod 600 "$ROLLBACK_DIR/prestate.tsv"
for name in "${RUNTIME_FILES[@]}"; do
dest_path="$DEST_ROOT/$name"
if [ -f "$dest_path" ] && [ ! -L "$dest_path" ]; then
[ "$(stat -c '%U:%G' "$dest_path")" = "wooo:wooo" ] || return 71
FILE_EXISTED[$name]=1
digest="$(working_digest "$dest_path")"
PREVIOUS_DIGEST[$name]="$digest"
printf '%s\t1\t%s\n' "$name" "$digest" >> "$ROLLBACK_DIR/prestate.tsv"
cp -p -- "$dest_path" "$ROLLBACK_DIR/$name"
[ "$(working_digest "$ROLLBACK_DIR/$name")" = "$digest" ] || return 72
elif [ ! -e "$dest_path" ]; then
FILE_EXISTED[$name]=0
PREVIOUS_DIGEST[$name]="-"
printf '%s\t0\t-\n' "$name" >> "$ROLLBACK_DIR/prestate.tsv"
else
return 73
fi
done
}
rollback_transaction() {
local name dest_path backup_path temporary failed=0
set +e
for name in "${RUNTIME_FILES[@]}"; do
[ -n "${FILE_EXISTED[$name]+x}" ] || { failed=1; continue; }
dest_path="$DEST_ROOT/$name"
if [ "${FILE_EXISTED[$name]}" = "1" ]; then
backup_path="$ROLLBACK_DIR/$name"
temporary="$DEST_ROOT/.${name}.agent99-rollback-${RUN_ID}"
cp -p -- "$backup_path" "$temporary" \
&& mv -f -- "$temporary" "$dest_path" \
|| failed=1
else
rm -f -- "$dest_path" || failed=1
fi
done
for name in "${RUNTIME_FILES[@]}"; do
dest_path="$DEST_ROOT/$name"
if [ "${FILE_EXISTED[$name]:-}" = "1" ]; then
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] \
&& [ "$(working_digest "$dest_path" 2>/dev/null)" = "${PREVIOUS_DIGEST[$name]:-invalid}" ] \
|| failed=1
elif [ "${FILE_EXISTED[$name]:-}" = "0" ]; then
[ ! -e "$dest_path" ] || failed=1
else
failed=1
fi
done
set -e
if [ "$failed" -eq 0 ]; then
ROLLBACK_VERIFIED=1
return 0
fi
ROLLBACK_VERIFIED=0
return 1
}
cleanup() {
local status=$?
local rollback_status="rollback_unverified"
trap - EXIT HUP INT TERM
if [ "$APPLY_STARTED" -eq 1 ] && [ "$APPLY_COMPLETE" -ne 1 ]; then
if rollback_transaction; then
rollback_status="failed_rolled_back"
fi
write_receipt "$rollback_status" "$ROLLBACK_VERIFIED" >/dev/null || true
fi
[ -z "$STAGE_DIR" ] || rm -rf -- "$STAGE_DIR"
if [ "$ROLLBACK_VERIFIED" -eq 1 ]; then
[ -z "$ROLLBACK_DIR" ] || rm -rf -- "$ROLLBACK_DIR"
fi
exit "$status"
}
validate_source || fail "source_validation_failed"
if [ "$MODE" = "check" ]; then
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":17,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
exit 0
fi
if [ "$MODE" = "verify" ]; then
verify_destination || fail "destination_verification_failed"
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":17,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
exit 0
fi
[ ! -e "$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json" ] || fail "run_identity_receipt_exists"
STAGE_DIR="$STAGE_ROOT/$RUN_ID"
ROLLBACK_DIR="$ROLLBACK_ROOT/$RUN_ID"
[ ! -e "$STAGE_DIR" ] || fail "run_identity_stage_exists"
[ ! -e "$ROLLBACK_DIR" ] || fail "run_identity_rollback_exists"
[ ! -L "$LOCK_PATH" ] || fail "deployment_lock_symlink_unsafe"
(umask 077; : >> "$LOCK_PATH") || fail "deployment_lock_unavailable"
[ -f "$LOCK_PATH" ] && [ -O "$LOCK_PATH" ] || fail "deployment_lock_owner_invalid"
exec 9>>"$LOCK_PATH"
flock -x -w 30 9 || fail "deployment_lock_timeout"
# Old runtime revisions predate the shared gate. Refuse the first promotion if
# one of them is already active; after common.sh moves first, new starts block.
if pgrep -afu "$EXPECTED_USER" '/backup/scripts/(backup-|check-backup|sync-offsite|verify-offsite|enforce-latest)' \
| grep -vF "host110-backup-runtime-executor.sh" >/dev/null 2>&1; then
fail "legacy_backup_runtime_active"
fi
install -d -m 700 "$STAGE_DIR" "$ROLLBACK_DIR"
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM HUP
for name in "${RUNTIME_FILES[@]}"; do
install -m 700 "$SOURCE_STAGE/$name" "$STAGE_DIR/$name"
[ "$(working_digest "$STAGE_DIR/$name")" = "$(source_digest "$name")" ] || fail "stage_identity_failed"
validate_file "$STAGE_DIR/$name" || fail "stage_validation_failed"
done
record_prestate || fail "rollback_prestate_capture_failed"
APPLY_STARTED=1
# common.sh is promoted first. Every managed mutating backup entrypoint then
# takes the shared side of LOCK_PATH before doing work, while this process holds
# the exclusive side until the independent verifier has completed.
for name in "${RUNTIME_FILES[@]}"; do
temporary="$DEST_ROOT/.${name}.agent99-${RUN_ID}"
install -m 755 "$STAGE_DIR/$name" "$temporary"
mv -f -- "$temporary" "$DEST_ROOT/$name"
done
verify_destination || fail "post_apply_verification_failed"
verifier_result="$(BACKUP_RUNTIME_DEPLOY_CONTEXT=1 python3 "$SOURCE_STAGE/verify-host110-backup-runtime.py" \
--manifest "$SOURCE_STAGE/manifest.json" \
--destination "$DEST_ROOT" \
--source-revision "$SOURCE_REVISION" \
--run-id "$RUN_ID" \
--expected-verifier-sha256 "$VERIFIER_DIGEST")" || fail "independent_verifier_failed"
python3 - "$verifier_result" "$SOURCE_REVISION" "$RUN_ID" <<'PY' || fail "independent_verifier_contract_failed"
import json
import sys
try:
row = json.loads(sys.argv[1])
except json.JSONDecodeError:
raise SystemExit(1)
if not (
row.get("schemaVersion") == "agent99_host110_backup_runtime_verifier_v1"
and row.get("ok") is True
and row.get("sourceRevision") == sys.argv[2]
and row.get("runId") == sys.argv[3]
and row.get("fileCount") == 17
and row.get("remoteWritePerformed") is False
and row.get("selfIdentityVerified") is True
):
raise SystemExit(1)
PY
APPLY_COMPLETE=1
receipt_path="$(write_receipt "verified" 0)"
rm -rf -- "$STAGE_DIR" "$ROLLBACK_DIR"
STAGE_DIR=""
ROLLBACK_DIR=""
trap - EXIT HUP INT TERM
python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$receipt_path" "$verifier_result" <<'PY'
import json
import sys
print(json.dumps({
"schemaVersion": "agent99_host110_backup_runtime_executor_v1",
"mode": "apply",
"ok": True,
"sourceRevision": sys.argv[1],
"sourceHead": sys.argv[2],
"fileCount": 17,
"rollbackPerformed": False,
"receipt": sys.argv[3],
"verifier": json.loads(sys.argv[4]),
}, ensure_ascii=True, sort_keys=True))
PY

View File

@@ -0,0 +1,164 @@
#!/usr/bin/env python3
"""Independent no-write verifier for the Host110 backup runtime transaction."""
from __future__ import annotations
import argparse
import base64
import fcntl
import hashlib
import json
import os
import re
import sys
import time
from pathlib import Path
EXPECTED_DESTINATION = Path("/backup/scripts")
RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")
EXPECTED_FILES = (
"common.sh",
"backup-all.sh",
"backup-host188-products.sh",
"verify-host188-products-backup.sh",
"verify-host188-products-archive.py",
"backup-gitea.sh",
"gitea-full-backup-restore-drill.sh",
"backup-configs.sh",
"backup-awoooi.sh",
"backup-awoooi-frequent.sh",
"backup-clawbot.sh",
"backup-sentry.sh",
"check-backup-integrity.sh",
"sync-offsite-backups.sh",
"backup-offsite-readiness-gate.sh",
"verify-offsite-full-sync.sh",
"enforce-latest-only-retention.sh",
)
def fail(reason: str) -> None:
print(f"HOST110_BACKUP_RUNTIME_VERIFY_OK=0\nERROR={reason}", file=sys.stderr)
raise SystemExit(1)
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def acquire_runtime_read_lock(destination: Path):
if destination != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_DEPLOY_CONTEXT") == "1":
return None
if RUNTIME_LOCK.is_symlink() or not RUNTIME_LOCK.is_file():
fail("runtime_lock_unavailable")
handle = RUNTIME_LOCK.open("rb")
if RUNTIME_LOCK.stat().st_uid != os.getuid():
handle.close()
fail("runtime_lock_owner_invalid")
deadline = time.monotonic() + 60
while True:
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
return handle
except BlockingIOError:
if time.monotonic() >= deadline:
handle.close()
fail("runtime_deployment_active")
time.sleep(0.2)
def main() -> None:
parser = argparse.ArgumentParser()
manifest_source = parser.add_mutually_exclusive_group(required=True)
manifest_source.add_argument("--manifest")
manifest_source.add_argument("--manifest-base64")
parser.add_argument("--destination", required=True)
parser.add_argument("--source-revision", required=True)
parser.add_argument("--run-id", required=True)
parser.add_argument("--expected-verifier-sha256", required=True)
args = parser.parse_args()
if not re.fullmatch(r"[0-9a-f]{40}", args.source_revision):
fail("invalid_source_revision")
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}", args.run_id):
fail("invalid_run_id")
destination = Path(args.destination)
if destination != EXPECTED_DESTINATION or destination.is_symlink() or not destination.is_dir():
fail("invalid_destination")
runtime_lock = acquire_runtime_read_lock(destination)
if not re.fullmatch(r"[0-9a-f]{64}", args.expected_verifier_sha256):
fail("invalid_verifier_identity")
try:
if args.manifest:
manifest_text = Path(args.manifest).read_text(encoding="utf-8")
else:
manifest_text = base64.b64decode(args.manifest_base64, validate=True).decode("utf-8")
document = json.loads(manifest_text)
except (OSError, UnicodeDecodeError, ValueError, json.JSONDecodeError):
fail("manifest_unavailable")
rows = document.get("files")
if (
document.get("schemaVersion") != "agent99_host110_backup_runtime_source_manifest_v1"
or document.get("sourceRevision") != args.source_revision
or document.get("runId") != args.run_id
or not re.fullmatch(r"[0-9a-f]{40}", str(document.get("sourceHead", "")))
or document.get("verifierSha256") != args.expected_verifier_sha256
or not isinstance(rows, list)
or len(rows) != len(EXPECTED_FILES)
):
fail("manifest_contract_failed")
by_name: dict[str, str] = {}
for row in rows:
if not isinstance(row, dict):
fail("manifest_file_set_failed")
name = str(row.get("name", ""))
if name in by_name:
fail("manifest_file_set_failed")
by_name[name] = str(row.get("sha256", ""))
if set(by_name) != set(EXPECTED_FILES) or not all(re.fullmatch(r"[0-9a-f]{64}", value) for value in by_name.values()):
fail("manifest_file_set_failed")
verifier_path = Path(__file__)
self_identity_verified = verifier_path.is_file() and sha256(verifier_path) == args.expected_verifier_sha256
if args.manifest and not self_identity_verified:
fail("verifier_identity_failed")
root = destination.resolve(strict=True)
for name in EXPECTED_FILES:
path = destination / name
if path.is_symlink() or not path.is_file():
fail("runtime_file_type_failed")
resolved = path.resolve(strict=True)
if resolved.parent != root:
fail("runtime_file_boundary_failed")
stat = resolved.stat()
if stat.st_uid != os.getuid() or stat.st_gid != os.getgid() or stat.st_mode & 0o777 != 0o755:
fail("runtime_file_metadata_failed")
if sha256(resolved) != by_name[name]:
fail("runtime_file_identity_failed")
result = {
"schemaVersion": "agent99_host110_backup_runtime_verifier_v1",
"ok": True,
"sourceRevision": args.source_revision,
"runId": args.run_id,
"fileCount": len(EXPECTED_FILES),
"destination": str(EXPECTED_DESTINATION),
"remoteWritePerformed": False,
"secretValuesRead": False,
"selfIdentityVerified": self_identity_verified,
"verifierSha256": args.expected_verifier_sha256,
}
print(json.dumps(result, ensure_ascii=True, sort_keys=True))
if runtime_lock is not None:
runtime_lock.close()
if __name__ == "__main__":
main()

View File

@@ -4,11 +4,14 @@
from __future__ import annotations
import argparse
import fcntl
import hashlib
import json
import os
import sqlite3
import tarfile
import tempfile
import time
from pathlib import Path, PurePosixPath
from typing import Any
@@ -20,6 +23,28 @@ EXPECTED_PREFIXES = {
"n8n-online-": ".tar.gz",
}
FIXED_ASSETS = {"n8n-validation.json"}
RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")
def acquire_runtime_lock():
if Path(__file__).resolve().parent != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_SHARED_LOCK_HELD") == "1":
return None
if RUNTIME_LOCK.is_symlink():
raise RuntimeError("backup_runtime_gate_unsafe")
handle = RUNTIME_LOCK.open("a+")
if RUNTIME_LOCK.stat().st_uid != os.getuid():
handle.close()
raise RuntimeError("backup_runtime_gate_owner_invalid")
deadline = time.monotonic() + 60
while True:
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
return handle
except BlockingIOError:
if time.monotonic() >= deadline:
handle.close()
raise RuntimeError("backup_runtime_deployment_active")
time.sleep(0.2)
def sha256_file(path: Path) -> str:
@@ -153,10 +178,13 @@ def validate_restore(root: Path) -> dict[str, Any]:
def main() -> int:
runtime_lock = acquire_runtime_lock()
parser = argparse.ArgumentParser()
parser.add_argument("--root", type=Path, required=True)
args = parser.parse_args()
print(json.dumps(validate_restore(args.root), separators=(",", ":")))
if runtime_lock is not None:
runtime_lock.close()
return 0

View File

@@ -293,8 +293,8 @@ def test_entrypoint_is_in_exact_agent99_runtime_bundle() -> None:
assert expected in _quoted_array(contract, "$requiredFiles = @(", ")")
assert expected in _quoted_array(sender, "RUNTIME_FILES=(", ")")
assert expected in _quoted_array(receiver, "$FixedRuntimeFiles = @(", ")")
assert 'expectedRuntimeFileCount": 18' in sender
assert '$runtimeManifest.fileCount -eq 18' in receiver
assert 'expectedRuntimeFileCount": 19' in sender
assert '$runtimeManifest.fileCount -eq 19' in receiver
def test_runtime_contract_check_requires_typed_signoz_transport_readback() -> None:

View File

@@ -122,7 +122,7 @@ def test_agent99_atomic_bundle_owns_the_dispatch_entrypoint() -> None:
assert "$dbExecutorRecoveryContract" in contract
assert "awoooi_db_bounded_executor_receipt_v2" in contract
assert "Receipt.database_sqlstate" in contract
assert 'expectedRuntimeFileCount": 18' in transport
assert "expectedRuntimeFileCount=18" in transport
assert "expectedRuntimeFileCount -ne 18" in receiver
assert "fileCount = 18" in receiver
assert 'expectedRuntimeFileCount": 19' in transport
assert "expectedRuntimeFileCount=19" in transport
assert "expectedRuntimeFileCount -ne 19" in receiver
assert "fileCount = 19" in receiver

View File

@@ -486,7 +486,7 @@ $decision = Get-AgentLivePreflightDecision `
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
-Manifest $manifest `
-ManifestCheckRequired ([bool]($ReadinessScope -eq "FullRuntime")) `
-ExpectedRuntimeFileCount 18 `
-ExpectedRuntimeFileCount 19 `
-TaskFailureCount $taskFailures.Count `
-RelayListenerCount $relayListeners.Count `
-RequiredEvidenceStaleCount $requiredEvidenceStale.Count `

View File

@@ -34,6 +34,7 @@ $FixedRuntimeFiles = @(
"agent99-control-plane.ps1",
"agent99-db-bounded-executor.ps1",
"agent99-signoz-metadata-executor.ps1",
"agent99-host110-backup-runtime-broker.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",
@@ -951,7 +952,7 @@ try {
}
$sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant()
if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
if ([int]$envelope.expectedRuntimeFileCount -ne 18) { throw "invalid_expected_runtime_file_count" }
if ([int]$envelope.expectedRuntimeFileCount -ne 19) { throw "invalid_expected_runtime_file_count" }
$traceId = [string]$envelope.traceId
$runId = [string]$envelope.runId
@@ -986,7 +987,7 @@ try {
$sourceManifest = [pscustomobject]@{
schemaVersion = "agent99_remote_source_manifest_v1"
sourceRevision = $sourceRevision
fileCount = 18
fileCount = 19
manifestSha256 = $manifestDigest
files = @($FixedRuntimeFiles | ForEach-Object {
[pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() }
@@ -1023,7 +1024,7 @@ try {
status = "check_ready"
mode = "check"
sourceRevision = $sourceRevision
expectedRuntimeFileCount = 18
expectedRuntimeFileCount = 19
manifestSha256 = $manifestDigest
plannedStagingPath = $stagePath
runtimeManifest = $currentManifest
@@ -1110,7 +1111,7 @@ try {
if (
[string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or
[string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or
[int]$existingSourceManifest.fileCount -ne 18 -or
[int]$existingSourceManifest.fileCount -ne 19 -or
[string]$existingSourceManifest.manifestSha256 -ne $manifestDigest
) { throw "existing_staging_manifest_mismatch" }
} catch {
@@ -1134,7 +1135,7 @@ try {
[string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and
$live.runtimeMatched -and
$live.sourceRevision -eq $sourceRevision -and
$live.fileCount -eq 18 -and
$live.fileCount -eq 19 -and
$live.mismatchCount -eq 0 -and
$liveLauncherContract.ok -and
[string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256
@@ -1447,7 +1448,7 @@ try {
$runtimeManifest.exists -and
$runtimeManifest.runtimeMatched -and
$runtimeManifest.sourceRevision -eq $sourceRevision -and
$runtimeManifest.fileCount -eq 18 -and
$runtimeManifest.fileCount -eq 19 -and
$runtimeManifest.mismatchCount -eq 0 -and
$launcherContract.ok -and
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and

View File

@@ -18,6 +18,7 @@ RUNTIME_FILES=(
"agent99-control-plane.ps1"
"agent99-db-bounded-executor.ps1"
"agent99-signoz-metadata-executor.ps1"
"agent99-host110-backup-runtime-broker.ps1"
"agent99-deploy.ps1"
"agent99-register-tasks.ps1"
"agent99-alertmanager-alertchain-poll.ps1"
@@ -227,7 +228,7 @@ trace_id, run_id, work_item_id = sys.argv[4:7]
output_path = Path(sys.argv[7])
runtime_names = sys.argv[8:]
if len(runtime_names) != 18 or len(set(runtime_names)) != 18:
if len(runtime_names) != 19 or len(set(runtime_names)) != 19:
raise SystemExit("fixed_runtime_file_contract_failed")
files: list[dict[str, str]] = []
@@ -267,7 +268,7 @@ envelope = {
"runId": run_id,
"workItemId": work_item_id,
"sourceRevision": source_revision,
"expectedRuntimeFileCount": 18,
"expectedRuntimeFileCount": 19,
"manifestSha256": manifest_sha256,
"files": files,
"livePreflight": {
@@ -419,14 +420,14 @@ stage_token = hashlib.sha256(stage_identity.encode("utf-8")).hexdigest()[:20]
script = r'''$ErrorActionPreference='Stop';$ProgressPreference='SilentlyContinue'
$a='C:\Wooo\Agent99';$b=Join-Path $a 'bin';$p=Join-Path $a 'state\runtime-manifest.json'
$r=[ordered]@{exists=$false;sourceRevision='';runtimeMatched=$false;recordedRuntimeMatched=$false;fileCount=0;mismatchCount=-1;recordedMismatchCount=-1;parseError=''}
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 18-and$m.mismatchCount-eq 0-and$rows.Count-eq 18-and$seen.Count-eq 18-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 19-and$m.mismatchCount-eq 0-and$rows.Count-eq 19-and$seen.Count-eq 19-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
$c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''}
try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name}
$z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false}
$ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0)
$status=if($ready){'check_ready'}else{'check_failed_no_apply'}
$next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'}
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=18;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=19;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
if(-not$ready){exit 65}'''
script = (
script.replace("__SOURCE_REVISION__", source_revision)

View File

@@ -0,0 +1,251 @@
from __future__ import annotations
import base64
import hashlib
import importlib.util
import json
import re
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[3]
BROKER = ROOT / "agent99-host110-backup-runtime-broker.ps1"
EXECUTOR = ROOT / "scripts" / "backup" / "host110-backup-runtime-executor.sh"
VERIFIER = ROOT / "scripts" / "backup" / "verify-host110-backup-runtime.py"
EXPECTED_RUNTIME_FILES = {
"common.sh",
"backup-all.sh",
"backup-host188-products.sh",
"verify-host188-products-backup.sh",
"verify-host188-products-archive.py",
"backup-gitea.sh",
"gitea-full-backup-restore-drill.sh",
"backup-configs.sh",
"backup-awoooi.sh",
"backup-awoooi-frequent.sh",
"backup-clawbot.sh",
"backup-sentry.sh",
"check-backup-integrity.sh",
"sync-offsite-backups.sh",
"backup-offsite-readiness-gate.sh",
"verify-offsite-full-sync.sh",
"enforce-latest-only-retention.sh",
}
def _load_verifier():
spec = importlib.util.spec_from_file_location("host110_runtime_verifier", VERIFIER)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
source = BROKER.read_text(encoding="utf-8")
assert '[ValidateSet("Check", "Apply", "Verify")]' in source
assert '$TargetHost = "192.168.0.110"' in source
assert '$TargetUser = "wooo"' in source
assert '$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"' in source
assert 'keys\\agent99_ed25519' in source
assert '"credential.helper="' in source
assert '$env:GIT_TERMINAL_PROMPT = "0"' in source
assert '"merge-base", "--is-ancestor", $SourceRevision, $sourceHead' in source
assert '"checkout", "--quiet", $SourceRevision, "--"' in source
assert 'executorSha256 = $executorDigest' in source
assert 'verifierSha256 = $verifierDigest' in source
assert "/home/wooo/awoooi" not in source
assert '"StrictHostKeyChecking=yes"' in source
assert '"NumberOfPasswordPrompts=0"' in source
assert '"ConnectionAttempts=1"' in source
assert "$process.WaitForExit($TimeoutSeconds * 1000)" in source
assert "taskkill.exe /PID $process.Id /T /F" in source
assert "$process.Kill()" in source
assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths' in source
def test_windows99_broker_verify_is_independent_no_write_and_evidence_is_immutable() -> None:
source = BROKER.read_text(encoding="utf-8")
assert 'RedirectStandardInput = $StandardInputPath' in source
assert "python3 - --manifest-base64" in source
assert 'remoteWritePerformed' in source
assert '[IO.File]::Move($temporary, $Path)' in source
assert 'Move-Item -LiteralPath $temporary -Destination $Path -Force' not in source
assert 'run_identity_conflict' in source
assert 'run_identity_evidence_exists' in source
assert 'decisionProvider = "deterministic_only"' in source
assert 'criticProvider = "deterministic_only"' in source
assert 'verifier = "independent_host110_backup_runtime_python_readback"' in source
def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
source = EXECUTOR.read_text(encoding="utf-8")
array_body = source.split("readonly -a RUNTIME_FILES=(", 1)[1].split(")", 1)[0]
runtime_files = set(re.findall(r"^\s+([A-Za-z0-9_.-]+)\s*$", array_body, re.MULTILINE))
assert runtime_files == EXPECTED_RUNTIME_FILES
assert 'readonly EXPECTED_HOST_IP="192.168.0.110"' in source
assert 'readonly EXPECTED_USER="wooo"' in source
assert 'readonly DEST_ROOT="/backup/scripts"' in source
assert '--source-stage' in source
assert 'document.get("executorSha256"' in source
assert 'document.get("verifierSha256"' in source
assert 'executor_identity_failed' in source
assert 'verifier_identity_failed' in source
assert 'flock -x -w 30 9' in source
assert 'legacy_backup_runtime_active' in source
assert source.index("for name in \"${RUNTIME_FILES[@]}\"") < source.index("mv -f -- \"$temporary\"")
assert 'rollback_unverified' in source
assert 'failed_rolled_back' in source
assert 'working_digest "$dest_path"' in source
assert 'os.link(temporary, path)' in source
assert 'run_identity_receipt_exists' in source
assert 'run_identity_stage_exists' in source
assert 'run_identity_rollback_exists' in source
assert 'verify-host110-backup-runtime.py' in source
assert 'selfIdentityVerified' in source
assert '"productionServiceRestarted": False' in source
assert '"secretValuesRead": False' in source
assert "sudo " not in source
assert "docker " not in source
assert "systemctl " not in source
def test_backup_runtime_entrypoints_share_the_deployment_gate() -> None:
common = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8")
restore = (ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh").read_text(encoding="utf-8")
offsite = (ROOT / "scripts/backup/backup-offsite-readiness-gate.sh").read_text(encoding="utf-8")
archive = (ROOT / "scripts/backup/verify-host188-products-archive.py").read_text(encoding="utf-8")
for source in (common, restore, offsite):
assert "/tmp/agent99-host110-backup-runtime.lock" in source
assert "flock -s -w 60" in source
assert "fcntl.LOCK_SH | fcntl.LOCK_NB" in archive
assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" in archive
def test_independent_verifier_accepts_exact_bundle_without_writing(tmp_path, monkeypatch, capsys) -> None:
verifier = _load_verifier()
destination = tmp_path / "scripts"
destination.mkdir()
rows = []
for name in verifier.EXPECTED_FILES:
path = destination / name
path.write_text(f"runtime:{name}\n", encoding="utf-8")
path.chmod(0o755)
rows.append({"name": name, "sha256": _sha256(path)})
verifier_digest = _sha256(VERIFIER)
source_revision = "a" * 40
run_id = "unit-exact"
manifest = {
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
"sourceRevision": source_revision,
"sourceHead": "b" * 40,
"runId": run_id,
"executorSha256": "c" * 64,
"verifierSha256": verifier_digest,
"files": rows,
}
manifest_path = tmp_path / "manifest.json"
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
before = {path.name: (path.stat().st_mtime_ns, _sha256(path)) for path in destination.iterdir()}
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
monkeypatch.setattr(
sys,
"argv",
[
str(VERIFIER),
"--manifest",
str(manifest_path),
"--destination",
str(destination),
"--source-revision",
source_revision,
"--run-id",
run_id,
"--expected-verifier-sha256",
verifier_digest,
],
)
verifier.main()
result = json.loads(capsys.readouterr().out)
after = {path.name: (path.stat().st_mtime_ns, _sha256(path)) for path in destination.iterdir()}
assert result["ok"] is True
assert result["remoteWritePerformed"] is False
assert result["selfIdentityVerified"] is True
assert before == after
def test_independent_verifier_rejects_drifted_bundle(tmp_path, monkeypatch, capsys) -> None:
verifier = _load_verifier()
destination = tmp_path / "scripts"
destination.mkdir()
rows = []
for name in verifier.EXPECTED_FILES:
path = destination / name
path.write_text(f"runtime:{name}\n", encoding="utf-8")
path.chmod(0o755)
rows.append({"name": name, "sha256": _sha256(path)})
(destination / verifier.EXPECTED_FILES[0]).write_text("drifted\n", encoding="utf-8")
verifier_digest = _sha256(VERIFIER)
source_revision = "d" * 40
run_id = "unit-drift"
manifest = {
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
"sourceRevision": source_revision,
"sourceHead": "e" * 40,
"runId": run_id,
"executorSha256": "f" * 64,
"verifierSha256": verifier_digest,
"files": rows,
}
encoded = base64.b64encode(json.dumps(manifest).encode()).decode()
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
monkeypatch.setattr(
sys,
"argv",
[
str(VERIFIER),
"--manifest-base64",
encoded,
"--destination",
str(destination),
"--source-revision",
source_revision,
"--run-id",
run_id,
"--expected-verifier-sha256",
verifier_digest,
],
)
with pytest.raises(SystemExit):
verifier.main()
assert "runtime_file_identity_failed" in capsys.readouterr().err
def test_agent99_runtime_lists_include_the_host110_broker_once() -> None:
sources = [
ROOT / "agent99-bootstrap.ps1",
ROOT / "agent99-contract-check.ps1",
ROOT / "agent99-deploy.ps1",
ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh",
ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1",
]
marker = "agent99-host110-backup-runtime-broker.ps1"
for path in sources:
assert marker in path.read_text(encoding="utf-8"), path

View File

@@ -36,7 +36,7 @@ def _base_case() -> dict[str, Any]:
"exists": True,
"sourceRevision": "a" * 40,
"runtimeMatched": True,
"fileCount": 18,
"fileCount": 19,
"mismatchCount": 0,
"parseError": "",
},
@@ -133,7 +133,7 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
)
assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision
assert "preflightGreen = [bool]$decision.deploymentEligible" in source
assert "-ExpectedRuntimeFileCount 18" in source
assert "-ExpectedRuntimeFileCount 19" in source
assert "warningReasons = $warningReasons" in source
assert "selfHealthPerformanceIssues = @(" in source
assert 'if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$")' in source

View File

@@ -26,6 +26,7 @@ EXPECTED_RUNTIME_FILES = (
"agent99-control-plane.ps1",
"agent99-db-bounded-executor.ps1",
"agent99-signoz-metadata-executor.ps1",
"agent99-host110-backup-runtime-broker.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",
@@ -88,7 +89,7 @@ def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None
)
assert preflight_count is not None
assert int(preflight_count.group(1)) == len(EXPECTED_RUNTIME_FILES)
assert "expectedRuntimeFileCount\": 18" in sender
assert "expectedRuntimeFileCount\": 19" in sender
assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver
assert "required_runtime_file_missing" in receiver
assert "duplicate_runtime_file" in receiver
@@ -263,7 +264,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
)
assert "$livePreflight.sourceRevision -eq $sourceRevision" in source
assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source
assert '$runtimeManifest.fileCount -eq 18' in source
assert '$runtimeManifest.fileCount -eq 19' in source
assert "function Invoke-AgentWindowsControlBaseline" in source
assert "function Get-AgentRunEvidenceToken" in source
assert "$evidenceRunToken = Get-AgentRunEvidenceToken -Value $RunId" in source

View File

@@ -516,7 +516,8 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
for parameter in ("AutomationRunId", "TraceId", "WorkItemId"):
assert f"[string]${parameter} = \"\"" in control
assert f'[string]`${parameter} = ""' in bootstrap
assert 'reason = "controlled_dispatch_identity_unsafe"' in control
assert '"controlled_dispatch_identity_unsafe"' in control
assert "reason = $identityRejectionReason" in control
assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
assert '$dryRun.managerPreflightStatus -eq "ready"' in function
assert "-not $dryRun.artifactTransactionBlocked" in function