feat(agent99): bind host110 backup runtime deploy
This commit is contained in:
@@ -224,6 +224,7 @@ $agentFiles = @(
|
||||
"agent99-control-plane.ps1",
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
|
||||
@@ -20,6 +20,7 @@ $requiredFiles = @(
|
||||
"agent99-control-plane.ps1",
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
@@ -131,6 +132,7 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js
|
||||
$control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane.ps1"), [Text.Encoding]::UTF8)
|
||||
$dbBoundedExecutor = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-db-bounded-executor.ps1"), [Text.Encoding]::UTF8)
|
||||
$signozMetadataExecutor = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-signoz-metadata-executor.ps1"), [Text.Encoding]::UTF8)
|
||||
$host110BackupRuntimeBroker = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-host110-backup-runtime-broker.ps1"), [Text.Encoding]::UTF8)
|
||||
$inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8)
|
||||
$telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8)
|
||||
$bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8)
|
||||
@@ -203,6 +205,27 @@ $signozMetadataExecutorContract = [bool](
|
||||
-not $signozMetadataExecutor.Contains('Get-Content -LiteralPath $CredentialFile')
|
||||
)
|
||||
Add-Check "backup:signoz_metadata_fixed_executor" $signozMetadataExecutorContract "Windows99 dispatches one fixed host110 metadata export contract, consumes only a root-owned credential reference on target, and requires the independent bundle verifier without raw SQLite or secret output"
|
||||
$host110BackupRuntimeContract = [bool](
|
||||
$host110BackupRuntimeBroker.Contains('$TargetHost = "192.168.0.110"') -and
|
||||
$host110BackupRuntimeBroker.Contains('$TargetUser = "wooo"') -and
|
||||
$host110BackupRuntimeBroker.Contains('$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"') -and
|
||||
$host110BackupRuntimeBroker.Contains('"credential.helper="') -and
|
||||
$host110BackupRuntimeBroker.Contains('$env:GIT_TERMINAL_PROMPT = "0"') -and
|
||||
$host110BackupRuntimeBroker.Contains('"merge-base", "--is-ancestor", $SourceRevision, $sourceHead') -and
|
||||
$host110BackupRuntimeBroker.Contains('executorSha256 = $executorDigest') -and
|
||||
$host110BackupRuntimeBroker.Contains('verifierSha256 = $verifierDigest') -and
|
||||
$host110BackupRuntimeBroker.Contains('StrictHostKeyChecking=yes') -and
|
||||
$host110BackupRuntimeBroker.Contains('$process.WaitForExit($TimeoutSeconds * 1000)') -and
|
||||
$host110BackupRuntimeBroker.Contains('taskkill.exe /PID $process.Id /T /F') -and
|
||||
$host110BackupRuntimeBroker.Contains('Invoke-Agent99BoundedScp $sourcePackage.localPaths') -and
|
||||
$host110BackupRuntimeBroker.Contains('python3 - --manifest-base64') -and
|
||||
$host110BackupRuntimeBroker.Contains('Write-Agent99ImmutableJson') -and
|
||||
$host110BackupRuntimeBroker.Contains('decisionProvider = "deterministic_only"') -and
|
||||
$host110BackupRuntimeBroker.Contains('criticProvider = "deterministic_only"') -and
|
||||
-not $host110BackupRuntimeBroker.Contains('/home/wooo/awoooi') -and
|
||||
-not $host110BackupRuntimeBroker.Contains('Invoke-Expression')
|
||||
)
|
||||
Add-Check "backup:host110_runtime_fixed_broker" $host110BackupRuntimeContract "Windows99 fetches one exact Gitea revision, binds executor and verifier digests, performs one fixed process-bounded Host110 transaction, and preserves an independent no-write verifier readback"
|
||||
Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("Enter-AgentRecoveryTriggerReservation") -and $control.Contains('"recovery-trigger-reservation.lock"') -and $control.Contains('[System.IO.FileShare]::None') -and $control.Contains('reason = "recovery_trigger_reservation_busy"') -and $control.Contains("recovery_already_queued")) "boot and host-down recovery reserve one atomic single-flight queue decision"
|
||||
Add-Check "recovery:auto_trigger_controlled_identity" ($control.Contains("New-AgentObservedRecoveryDispatchIdentity") -and $control.Contains('-RouteId "agent99_local_observer"') -and $control.Contains('schema_version = "agent99_controlled_dispatch_identity_v1"') -and $control.Contains('correlationKey = $dispatchIdentity.idempotencyKey') -and $control.Contains('canonicalDigest = $dispatchIdentity.canonicalDigest') -and $control.Contains("Invoke-AgentControlledDispatchIdentitySelfTest")) "observer-created Recover requests carry a stable canonical controlled dispatch identity before queue processing"
|
||||
Add-Check "recovery:queue_identity_fail_closed" ($control.Contains('dispatchIdentitySchemaVersion = $dispatchIdentity.schemaVersion') -and $control.Contains('Test-AgentControlledDispatchIdentity $dispatchIdentityEnvelope') -and $control.Contains('"controlled_dispatch_identity_invalid"') -and $control.IndexOf('"controlled_dispatch_identity_invalid"') -lt $control.IndexOf('$process = Start-Process -FilePath "powershell.exe" -ArgumentList $args')) "queue consumer rebuilds and validates canonical identity before any controlled executor starts"
|
||||
|
||||
@@ -35,6 +35,7 @@ $runtimeFiles = @(
|
||||
"agent99-control-plane.ps1",
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
|
||||
474
agent99-host110-backup-runtime-broker.ps1
Normal file
474
agent99-host110-backup-runtime-broker.ps1
Normal file
@@ -0,0 +1,474 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateSet("Check", "Apply", "Verify")]
|
||||
[string]$Mode,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$SourceRevision,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RunId,
|
||||
[string]$AgentRoot = "C:\Wooo\Agent99"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ExpectedAgentRoot = "C:\Wooo\Agent99"
|
||||
$TargetHost = "192.168.0.110"
|
||||
$TargetUser = "wooo"
|
||||
$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"
|
||||
$GitPath = "C:\Program Files\Git\cmd\git.exe"
|
||||
$IdentityFile = Join-Path $ExpectedAgentRoot "keys\agent99_ed25519"
|
||||
$EvidenceRoot = Join-Path $ExpectedAgentRoot "evidence\host110-backup-runtime"
|
||||
$SourceRootBase = Join-Path $ExpectedAgentRoot "deploy"
|
||||
$TransportMutexName = "Global\WoooAgent99Host110BackupRuntimeV2"
|
||||
$ExpectedFileCount = 17
|
||||
$ExpectedRemoteStageFileCount = 20
|
||||
$RuntimeRelativePaths = @(
|
||||
"scripts/backup/common.sh",
|
||||
"scripts/backup/backup-all.sh",
|
||||
"scripts/backup/backup-host188-products.sh",
|
||||
"scripts/backup/verify-host188-products-backup.sh",
|
||||
"scripts/backup/verify-host188-products-archive.py",
|
||||
"scripts/backup/backup-gitea.sh",
|
||||
"scripts/backup/gitea-full-backup-restore-drill.sh",
|
||||
"scripts/backup/backup-configs.sh",
|
||||
"scripts/backup/backup-awoooi.sh",
|
||||
"scripts/backup/backup-awoooi-frequent.sh",
|
||||
"scripts/backup/backup-clawbot.sh",
|
||||
"scripts/backup/backup-sentry.sh",
|
||||
"scripts/backup/check-backup-integrity.sh",
|
||||
"scripts/backup/sync-offsite-backups.sh",
|
||||
"scripts/backup/backup-offsite-readiness-gate.sh",
|
||||
"scripts/backup/verify-offsite-full-sync.sh",
|
||||
"scripts/backup/enforce-latest-only-retention.sh"
|
||||
)
|
||||
$ExecutorRelativePath = "scripts/backup/host110-backup-runtime-executor.sh"
|
||||
$VerifierRelativePath = "scripts/backup/verify-host110-backup-runtime.py"
|
||||
|
||||
function Test-Agent99SafeIdentity {
|
||||
param([string]$Value)
|
||||
return [bool]($Value -and $Value -match "^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$")
|
||||
}
|
||||
|
||||
function Write-Agent99ImmutableJson {
|
||||
param([string]$Path, [object]$Value)
|
||||
|
||||
if (Test-Path -LiteralPath $Path) { throw "run_identity_evidence_exists" }
|
||||
$temporary = "$Path.tmp-$PID-$([Guid]::NewGuid().ToString('N'))"
|
||||
$encoding = New-Object System.Text.UTF8Encoding($false)
|
||||
try {
|
||||
$json = $Value | ConvertTo-Json -Depth 12
|
||||
[IO.File]::WriteAllText($temporary, $json + [Environment]::NewLine, $encoding)
|
||||
[IO.File]::Move($temporary, $Path)
|
||||
} finally {
|
||||
Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Agent99BoundedProcess {
|
||||
param(
|
||||
[string]$FilePath,
|
||||
[string[]]$Arguments,
|
||||
[int]$TimeoutSeconds,
|
||||
[string]$WorkingDirectory = "",
|
||||
[string]$StandardInputPath = ""
|
||||
)
|
||||
|
||||
$token = [Guid]::NewGuid().ToString("N")
|
||||
$stdoutPath = Join-Path $env:TEMP "agent99-host110-backup-$token.out"
|
||||
$stderrPath = Join-Path $env:TEMP "agent99-host110-backup-$token.err"
|
||||
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
$process = $null
|
||||
try {
|
||||
$start = @{
|
||||
FilePath = $FilePath
|
||||
ArgumentList = $Arguments
|
||||
NoNewWindow = $true
|
||||
RedirectStandardOutput = $stdoutPath
|
||||
RedirectStandardError = $stderrPath
|
||||
PassThru = $true
|
||||
}
|
||||
if ($WorkingDirectory) { $start.WorkingDirectory = $WorkingDirectory }
|
||||
if ($StandardInputPath) { $start.RedirectStandardInput = $StandardInputPath }
|
||||
$process = Start-Process @start
|
||||
$null = $process.Handle
|
||||
if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
|
||||
try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {}
|
||||
if (-not $process.WaitForExit(2000) -and -not $process.HasExited) {
|
||||
$process.Kill()
|
||||
$process.WaitForExit(2000) | Out-Null
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
ok = $false
|
||||
exitCode = -2
|
||||
reason = "transport_timeout"
|
||||
elapsedMs = $stopwatch.ElapsedMilliseconds
|
||||
stdout = ""
|
||||
stderrPresent = $false
|
||||
}
|
||||
}
|
||||
$process.WaitForExit() | Out-Null
|
||||
$process.Refresh()
|
||||
$stdout = if (Test-Path -LiteralPath $stdoutPath) { Get-Content -LiteralPath $stdoutPath -Raw } else { "" }
|
||||
$stderr = if (Test-Path -LiteralPath $stderrPath) { Get-Content -LiteralPath $stderrPath -Raw } else { "" }
|
||||
if ($stdout.Length -gt 65536 -or $stderr.Length -gt 65536) {
|
||||
return [pscustomobject]@{
|
||||
ok = $false
|
||||
exitCode = -3
|
||||
reason = "transport_output_oversized"
|
||||
elapsedMs = $stopwatch.ElapsedMilliseconds
|
||||
stdout = ""
|
||||
stderrPresent = [bool]$stderr
|
||||
}
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
ok = [bool]($process.ExitCode -eq 0)
|
||||
exitCode = [int]$process.ExitCode
|
||||
reason = if ($process.ExitCode -eq 0) { "completed" } else { "process_failed" }
|
||||
elapsedMs = $stopwatch.ElapsedMilliseconds
|
||||
stdout = [string]$stdout
|
||||
stderrPresent = [bool]$stderr
|
||||
}
|
||||
} catch {
|
||||
return [pscustomobject]@{
|
||||
ok = $false
|
||||
exitCode = -1
|
||||
reason = "transport_exception"
|
||||
elapsedMs = $stopwatch.ElapsedMilliseconds
|
||||
stdout = ""
|
||||
stderrPresent = $false
|
||||
errorType = $_.Exception.GetType().Name
|
||||
}
|
||||
} finally {
|
||||
$stopwatch.Stop()
|
||||
Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null
|
||||
if ($process) { $process.Dispose() }
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Agent99BoundedSsh {
|
||||
param([string]$RemoteCommand, [int]$TimeoutSeconds, [string]$StandardInputPath = "")
|
||||
|
||||
$arguments = @(
|
||||
"-i", $IdentityFile,
|
||||
"-o", "IdentitiesOnly=yes",
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "NumberOfPasswordPrompts=0",
|
||||
"-o", "StrictHostKeyChecking=yes",
|
||||
"-o", "ConnectTimeout=8",
|
||||
"-o", "ConnectionAttempts=1",
|
||||
"-o", "ServerAliveInterval=10",
|
||||
"-o", "ServerAliveCountMax=2",
|
||||
"-l", $TargetUser,
|
||||
$TargetHost,
|
||||
$RemoteCommand
|
||||
)
|
||||
return Invoke-Agent99BoundedProcess "ssh.exe" $arguments $TimeoutSeconds "" $StandardInputPath
|
||||
}
|
||||
|
||||
function Invoke-Agent99BoundedScp {
|
||||
param([string[]]$LocalPaths, [string]$RemoteDirectory, [int]$TimeoutSeconds)
|
||||
|
||||
$arguments = @(
|
||||
"-i", $IdentityFile,
|
||||
"-o", "IdentitiesOnly=yes",
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "NumberOfPasswordPrompts=0",
|
||||
"-o", "StrictHostKeyChecking=yes",
|
||||
"-o", "ConnectTimeout=8",
|
||||
"-o", "ConnectionAttempts=1"
|
||||
)
|
||||
$arguments += $LocalPaths
|
||||
$arguments += "${TargetUser}@${TargetHost}:${RemoteDirectory}/"
|
||||
return Invoke-Agent99BoundedProcess "scp.exe" $arguments $TimeoutSeconds
|
||||
}
|
||||
|
||||
function Invoke-Agent99Git {
|
||||
param([string[]]$Arguments, [int]$TimeoutSeconds, [string]$WorkingDirectory = "")
|
||||
|
||||
$previousPrompt = $env:GIT_TERMINAL_PROMPT
|
||||
try {
|
||||
$env:GIT_TERMINAL_PROMPT = "0"
|
||||
return Invoke-Agent99BoundedProcess $GitPath $Arguments $TimeoutSeconds $WorkingDirectory
|
||||
} finally {
|
||||
$env:GIT_TERMINAL_PROMPT = $previousPrompt
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Agent99ProcessOk {
|
||||
param([object]$Result, [string]$Reason)
|
||||
if (-not $Result.ok) { throw $Reason }
|
||||
}
|
||||
|
||||
function New-Agent99SourcePackage {
|
||||
param([string]$SourceRoot, [string]$ManifestPath)
|
||||
|
||||
if (Test-Path -LiteralPath $SourceRoot) { throw "run_identity_source_stage_exists" }
|
||||
$clone = Invoke-Agent99Git @(
|
||||
"-c", "credential.helper=",
|
||||
"-c", "core.autocrlf=false",
|
||||
"clone", "--quiet", "--no-tags", "--depth", "200", "--branch", "main",
|
||||
$GiteaSourceUrl, $SourceRoot
|
||||
) 180 $SourceRootBase
|
||||
Assert-Agent99ProcessOk $clone "gitea_source_fetch_failed"
|
||||
|
||||
$headResult = Invoke-Agent99Git @("rev-parse", "HEAD") 30 $SourceRoot
|
||||
Assert-Agent99ProcessOk $headResult "gitea_source_head_failed"
|
||||
$sourceHead = ([string]$headResult.stdout).Trim().ToLowerInvariant()
|
||||
if ($sourceHead -notmatch "^[0-9a-f]{40}$") { throw "gitea_source_head_invalid" }
|
||||
|
||||
$commitResult = Invoke-Agent99Git @("cat-file", "-e", "$SourceRevision`^{commit}") 30 $SourceRoot
|
||||
Assert-Agent99ProcessOk $commitResult "source_revision_not_fetched"
|
||||
$ancestorResult = Invoke-Agent99Git @("merge-base", "--is-ancestor", $SourceRevision, $sourceHead) 30 $SourceRoot
|
||||
Assert-Agent99ProcessOk $ancestorResult "source_revision_not_on_gitea_main"
|
||||
|
||||
$allPaths = @($RuntimeRelativePaths) + @($ExecutorRelativePath, $VerifierRelativePath)
|
||||
$checkoutResult = Invoke-Agent99Git (@("-c", "core.autocrlf=false", "checkout", "--quiet", $SourceRevision, "--") + $allPaths) 60 $SourceRoot
|
||||
Assert-Agent99ProcessOk $checkoutResult "source_revision_checkout_failed"
|
||||
|
||||
$rows = @()
|
||||
$localPaths = @()
|
||||
foreach ($relativePath in $RuntimeRelativePaths) {
|
||||
$path = Join-Path $SourceRoot ($relativePath -replace "/", "\")
|
||||
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { throw "runtime_source_file_missing" }
|
||||
$rows += [ordered]@{
|
||||
name = [IO.Path]::GetFileName($path)
|
||||
sha256 = (Get-FileHash -LiteralPath $path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
$localPaths += $path
|
||||
}
|
||||
if (($rows.name | Sort-Object -Unique).Count -ne $ExpectedFileCount) { throw "runtime_source_file_set_invalid" }
|
||||
|
||||
$executorPath = Join-Path $SourceRoot ($ExecutorRelativePath -replace "/", "\")
|
||||
$verifierPath = Join-Path $SourceRoot ($VerifierRelativePath -replace "/", "\")
|
||||
if (-not (Test-Path -LiteralPath $executorPath -PathType Leaf)) { throw "executor_source_file_missing" }
|
||||
if (-not (Test-Path -LiteralPath $verifierPath -PathType Leaf)) { throw "verifier_source_file_missing" }
|
||||
$executorDigest = (Get-FileHash -LiteralPath $executorPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$verifierDigest = (Get-FileHash -LiteralPath $verifierPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$manifest = [ordered]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_source_manifest_v1"
|
||||
sourceRevision = $SourceRevision
|
||||
sourceHead = $sourceHead
|
||||
runId = $RunId
|
||||
executorSha256 = $executorDigest
|
||||
verifierSha256 = $verifierDigest
|
||||
files = $rows
|
||||
}
|
||||
$encoding = New-Object System.Text.UTF8Encoding($false)
|
||||
[IO.File]::WriteAllText($ManifestPath, ($manifest | ConvertTo-Json -Depth 8 -Compress), $encoding)
|
||||
$localPaths += $executorPath, $verifierPath, $ManifestPath
|
||||
return [pscustomobject]@{
|
||||
sourceHead = $sourceHead
|
||||
executorSha256 = $executorDigest
|
||||
verifierSha256 = $verifierDigest
|
||||
verifierPath = $verifierPath
|
||||
manifestPath = $ManifestPath
|
||||
manifestBase64 = [Convert]::ToBase64String([IO.File]::ReadAllBytes($ManifestPath))
|
||||
localPaths = $localPaths
|
||||
}
|
||||
}
|
||||
|
||||
function Get-Agent99ExecutorResult {
|
||||
param([object]$Transport, [string]$ExpectedMode)
|
||||
|
||||
if (-not $Transport.ok) { throw "host110_${ExpectedMode}_transport_failed" }
|
||||
try {
|
||||
$parsed = [string]$Transport.stdout | ConvertFrom-Json
|
||||
} catch {
|
||||
throw "host110_${ExpectedMode}_receipt_invalid"
|
||||
}
|
||||
if (
|
||||
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_executor_v1" -or
|
||||
[string]$parsed.mode -ne $ExpectedMode.ToLowerInvariant() -or
|
||||
-not [bool]$parsed.ok -or
|
||||
[string]$parsed.sourceRevision -ne $SourceRevision -or
|
||||
[int]$parsed.fileCount -ne $ExpectedFileCount
|
||||
) {
|
||||
throw "host110_${ExpectedMode}_receipt_contract_failed"
|
||||
}
|
||||
if ($ExpectedMode -eq "Apply") {
|
||||
if (
|
||||
[string]$parsed.verifier.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
|
||||
-not [bool]$parsed.verifier.ok -or
|
||||
[bool]$parsed.verifier.remoteWritePerformed -or
|
||||
-not [bool]$parsed.verifier.selfIdentityVerified
|
||||
) {
|
||||
throw "host110_apply_independent_verifier_contract_failed"
|
||||
}
|
||||
}
|
||||
return $parsed
|
||||
}
|
||||
|
||||
function Get-Agent99VerifierResult {
|
||||
param([object]$Transport, [object]$SourcePackage)
|
||||
|
||||
if (-not $Transport.ok) { throw "host110_verify_transport_failed" }
|
||||
try {
|
||||
$parsed = [string]$Transport.stdout | ConvertFrom-Json
|
||||
} catch {
|
||||
throw "host110_verify_receipt_invalid"
|
||||
}
|
||||
if (
|
||||
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
|
||||
-not [bool]$parsed.ok -or
|
||||
[string]$parsed.sourceRevision -ne $SourceRevision -or
|
||||
[string]$parsed.runId -ne $RunId -or
|
||||
[int]$parsed.fileCount -ne $ExpectedFileCount -or
|
||||
[bool]$parsed.remoteWritePerformed -or
|
||||
[string]$parsed.verifierSha256 -ne [string]$SourcePackage.verifierSha256
|
||||
) {
|
||||
throw "host110_verify_receipt_contract_failed"
|
||||
}
|
||||
return $parsed
|
||||
}
|
||||
|
||||
function Invoke-Agent99ReadOnlyPreflight {
|
||||
param([object]$SourcePackage)
|
||||
|
||||
$command = 'test "$(id -un)" = wooo && hostname -I | tr " " "\n" | grep -qx 192.168.0.110 && test -d /backup/scripts && test ! -L /backup/scripts && command -v timeout >/dev/null && command -v flock >/dev/null && command -v python3 >/dev/null && printf HOST110_BACKUP_PREFLIGHT_OK=1'
|
||||
$transport = Invoke-Agent99BoundedSsh $command 45
|
||||
if (-not $transport.ok -or ([string]$transport.stdout).Trim() -ne "HOST110_BACKUP_PREFLIGHT_OK=1") {
|
||||
throw "host110_read_only_preflight_failed"
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_preflight_v1"
|
||||
ok = $true
|
||||
sourceRevision = $SourceRevision
|
||||
sourceHead = [string]$SourcePackage.sourceHead
|
||||
fileCount = $ExpectedFileCount
|
||||
remoteWritePerformed = $false
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Agent99NoWriteVerifier {
|
||||
param([object]$SourcePackage)
|
||||
|
||||
$command = "timeout --signal=TERM --kill-after=5s 60s python3 - --manifest-base64 $($SourcePackage.manifestBase64) --destination /backup/scripts --source-revision $SourceRevision --run-id $RunId --expected-verifier-sha256 $($SourcePackage.verifierSha256)"
|
||||
$transport = Invoke-Agent99BoundedSsh $command 90 ([string]$SourcePackage.verifierPath)
|
||||
return Get-Agent99VerifierResult $transport $SourcePackage
|
||||
}
|
||||
|
||||
function Remove-Agent99RemoteStage {
|
||||
param([string]$RemoteStage, [bool]$Required)
|
||||
|
||||
$cleanup = Invoke-Agent99BoundedSsh "rm -rf -- $RemoteStage && test ! -e $RemoteStage" 45
|
||||
if ($Required -and -not $cleanup.ok) { throw "remote_source_stage_cleanup_failed" }
|
||||
return [bool]$cleanup.ok
|
||||
}
|
||||
|
||||
if ($AgentRoot.TrimEnd("\") -ne $ExpectedAgentRoot) { throw "agent_root_override_forbidden" }
|
||||
$SourceRevision = $SourceRevision.ToLowerInvariant()
|
||||
if ($SourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
|
||||
if (-not (Test-Agent99SafeIdentity $RunId)) { throw "invalid_run_id" }
|
||||
if (-not (Test-Path -LiteralPath $IdentityFile -PathType Leaf)) { throw "agent99_ssh_identity_unavailable" }
|
||||
if (-not (Test-Path -LiteralPath $GitPath -PathType Leaf)) { throw "windows_git_unavailable" }
|
||||
if (-not (Get-Command ssh.exe -ErrorAction SilentlyContinue)) { throw "windows_openssh_unavailable" }
|
||||
if (-not (Get-Command scp.exe -ErrorAction SilentlyContinue)) { throw "windows_openscp_unavailable" }
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $EvidenceRoot, $SourceRootBase | Out-Null
|
||||
$evidencePath = Join-Path $EvidenceRoot "agent99-host110-backup-runtime-$RunId.json"
|
||||
if (Test-Path -LiteralPath $evidencePath -PathType Leaf) {
|
||||
try { $existing = Get-Content -LiteralPath $evidencePath -Raw | ConvertFrom-Json } catch { throw "run_identity_evidence_invalid" }
|
||||
if ([string]$existing.sourceRevision -ne $SourceRevision -or [string]$existing.mode -ne $Mode) {
|
||||
throw "run_identity_conflict"
|
||||
}
|
||||
$existing | ConvertTo-Json -Compress -Depth 12
|
||||
if (-not [bool]$existing.ok) { exit 1 }
|
||||
exit 0
|
||||
}
|
||||
|
||||
$sourceRoot = Join-Path $SourceRootBase "host110-backup-source-$RunId"
|
||||
$manifestPath = Join-Path $sourceRoot "agent99-host110-backup-runtime-manifest.json"
|
||||
$remoteStage = "/tmp/agent99-host110-backup-runtime-$RunId"
|
||||
$startedAt = [DateTime]::UtcNow
|
||||
$mutex = [Threading.Mutex]::new($false, $TransportMutexName)
|
||||
$acquired = $false
|
||||
$remoteStageCreated = $false
|
||||
$sourcePackage = $null
|
||||
$check = $null
|
||||
$apply = $null
|
||||
$verify = $null
|
||||
$terminalStatus = "failed"
|
||||
$errorCode = ""
|
||||
$cleanupVerified = $false
|
||||
|
||||
try {
|
||||
try {
|
||||
$acquired = [bool]$mutex.WaitOne(30000)
|
||||
} catch [Threading.AbandonedMutexException] {
|
||||
$acquired = $true
|
||||
}
|
||||
if (-not $acquired) { throw "host110_backup_runtime_transport_busy" }
|
||||
|
||||
$sourcePackage = New-Agent99SourcePackage $sourceRoot $manifestPath
|
||||
$check = Invoke-Agent99ReadOnlyPreflight $sourcePackage
|
||||
|
||||
if ($Mode -eq "Apply") {
|
||||
$createStage = Invoke-Agent99BoundedSsh "test ! -e $remoteStage && install -d -m 700 $remoteStage" 45
|
||||
Assert-Agent99ProcessOk $createStage "remote_source_stage_identity_conflict"
|
||||
$remoteStageCreated = $true
|
||||
$copyStage = Invoke-Agent99BoundedScp $sourcePackage.localPaths $remoteStage 120
|
||||
Assert-Agent99ProcessOk $copyStage "remote_source_stage_copy_failed"
|
||||
$stageCheck = Invoke-Agent99BoundedSsh "chmod 700 $remoteStage/host110-backup-runtime-executor.sh $remoteStage/verify-host110-backup-runtime.py && test `$(find $remoteStage -mindepth 1 -maxdepth 1 -type f | wc -l) -eq $ExpectedRemoteStageFileCount" 45
|
||||
Assert-Agent99ProcessOk $stageCheck "remote_source_stage_contract_failed"
|
||||
|
||||
$executorPath = "$remoteStage/host110-backup-runtime-executor.sh"
|
||||
$command = "timeout --signal=TERM --kill-after=10s 300s $executorPath --mode apply --source-revision $SourceRevision --run-id $RunId --source-stage $remoteStage"
|
||||
$applyTransport = Invoke-Agent99BoundedSsh $command 330
|
||||
$apply = Get-Agent99ExecutorResult $applyTransport "Apply"
|
||||
$verify = $apply.verifier
|
||||
$cleanupVerified = Remove-Agent99RemoteStage $remoteStage $true
|
||||
$remoteStageCreated = $false
|
||||
$terminalStatus = "verified"
|
||||
} elseif ($Mode -eq "Verify") {
|
||||
$verify = Invoke-Agent99NoWriteVerifier $sourcePackage
|
||||
$terminalStatus = "readback_ok"
|
||||
$cleanupVerified = $true
|
||||
} else {
|
||||
$terminalStatus = "readback_ok"
|
||||
$cleanupVerified = $true
|
||||
}
|
||||
} catch {
|
||||
$errorCode = [string]$_.Exception.Message
|
||||
} finally {
|
||||
if ($remoteStageCreated) {
|
||||
try { $cleanupVerified = Remove-Agent99RemoteStage $remoteStage $false } catch {}
|
||||
}
|
||||
if ($acquired) { try { $mutex.ReleaseMutex() } catch {} }
|
||||
$mutex.Dispose()
|
||||
}
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v2"
|
||||
ok = [bool]($terminalStatus -ne "failed")
|
||||
status = $terminalStatus
|
||||
mode = $Mode
|
||||
sourceRevision = $SourceRevision
|
||||
sourceHead = if ($sourcePackage) { [string]$sourcePackage.sourceHead } else { "" }
|
||||
runId = $RunId
|
||||
targetHost = $TargetHost
|
||||
sourceTransport = "windows99_gitea_exact_revision_manifest"
|
||||
executor = "host110_backup_runtime_executor"
|
||||
verifier = "independent_host110_backup_runtime_python_readback"
|
||||
decisionProvider = "deterministic_only"
|
||||
criticProvider = "deterministic_only"
|
||||
agentAction = if ($Mode -eq "Apply") { "controlled_apply" } else { "readback" }
|
||||
check = $check
|
||||
apply = $apply
|
||||
verify = $verify
|
||||
cleanupVerified = $cleanupVerified
|
||||
errorCode = $errorCode
|
||||
elapsedSeconds = [math]::Round(([DateTime]::UtcNow - $startedAt).TotalSeconds, 3)
|
||||
evidence = $evidencePath
|
||||
secretValuesRead = $false
|
||||
rawSessionStored = $false
|
||||
}
|
||||
|
||||
try {
|
||||
Write-Agent99ImmutableJson $evidencePath $result
|
||||
} finally {
|
||||
if (Test-Path -LiteralPath $sourceRoot) {
|
||||
Remove-Item -LiteralPath $sourceRoot -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
$result | ConvertTo-Json -Compress -Depth 12
|
||||
if (-not $result.ok) { exit 1 }
|
||||
@@ -9,6 +9,16 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then
|
||||
command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; }
|
||||
[ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; }
|
||||
(umask 077; : >> /tmp/agent99-host110-backup-runtime.lock)
|
||||
[ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; }
|
||||
exec 197>>/tmp/agent99-host110-backup-runtime.lock
|
||||
flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; }
|
||||
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
|
||||
fi
|
||||
|
||||
BACKUP_BASE="${BACKUP_BASE:-/backup}"
|
||||
OFFSITE_ENV_FILE="${BACKUP_OFFSITE_ENV_FILE:-${BACKUP_BASE}/scripts/offsite.env}"
|
||||
OFFSITE_DIR="${BACKUP_OFFSITE_STATUS_DIR:-${BACKUP_BASE}/offsite}"
|
||||
|
||||
@@ -1,5 +1,38 @@
|
||||
#\!/bin/bash
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
|
||||
# Production backup entrypoints share this stable gate. Agent99 takes the
|
||||
# exclusive side while promoting a complete, digest-bound runtime bundle.
|
||||
acquire_backup_runtime_shared_lock() {
|
||||
local caller_path="${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}"
|
||||
local lock_path="/tmp/agent99-host110-backup-runtime.lock"
|
||||
case "${caller_path}" in
|
||||
/backup/scripts/*) ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
[ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ] || return 0
|
||||
command -v flock >/dev/null 2>&1 || {
|
||||
echo "backup runtime gate unavailable: flock missing" >&2
|
||||
return 69
|
||||
}
|
||||
[ ! -L "${lock_path}" ] || {
|
||||
echo "backup runtime gate unsafe: lock is a symlink" >&2
|
||||
return 69
|
||||
}
|
||||
(umask 077; : >> "${lock_path}") || return 69
|
||||
[ -f "${lock_path}" ] && [ -O "${lock_path}" ] || return 69
|
||||
exec 197>>"${lock_path}"
|
||||
flock -s -w 60 197 || {
|
||||
echo "backup runtime deployment is active; retry later" >&2
|
||||
return 75
|
||||
}
|
||||
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
|
||||
}
|
||||
|
||||
acquire_backup_runtime_shared_lock || {
|
||||
runtime_lock_status=$?
|
||||
return "${runtime_lock_status}" 2>/dev/null || exit "${runtime_lock_status}"
|
||||
}
|
||||
# WOOO AIOps - 備份共用函式庫
|
||||
# 版本: 1.0.0
|
||||
# 建立日期: 2026-03-12
|
||||
|
||||
@@ -5,6 +5,16 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then
|
||||
command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; }
|
||||
[ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; }
|
||||
(umask 077; : >> /tmp/agent99-host110-backup-runtime.lock)
|
||||
[ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; }
|
||||
exec 197>>/tmp/agent99-host110-backup-runtime.lock
|
||||
flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; }
|
||||
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
|
||||
fi
|
||||
|
||||
DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}"
|
||||
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
|
||||
TEXTFILE_PATH="${AIOPS_GITEA_FULL_BACKUP_RESTORE_DRILL_TEXTFILE:-/home/wooo/node_exporter_textfiles/gitea_full_backup_restore_drill.prom}"
|
||||
|
||||
431
scripts/backup/host110-backup-runtime-executor.sh
Executable file
431
scripts/backup/host110-backup-runtime-executor.sh
Executable file
@@ -0,0 +1,431 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fixed Host110 executor for Windows99 Agent99 backup-runtime deployment.
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
readonly EXPECTED_HOST_IP="192.168.0.110"
|
||||
readonly EXPECTED_USER="wooo"
|
||||
readonly DEST_ROOT="/backup/scripts"
|
||||
readonly STATUS_ROOT="/backup/status"
|
||||
readonly STAGE_ROOT="/backup/.agent99-backup-runtime-stage"
|
||||
readonly ROLLBACK_ROOT="/backup/.agent99-backup-runtime-rollback"
|
||||
readonly LOCK_PATH="/tmp/agent99-host110-backup-runtime.lock"
|
||||
|
||||
readonly -a RUNTIME_FILES=(
|
||||
common.sh
|
||||
backup-all.sh
|
||||
backup-host188-products.sh
|
||||
verify-host188-products-backup.sh
|
||||
verify-host188-products-archive.py
|
||||
backup-gitea.sh
|
||||
gitea-full-backup-restore-drill.sh
|
||||
backup-configs.sh
|
||||
backup-awoooi.sh
|
||||
backup-awoooi-frequent.sh
|
||||
backup-clawbot.sh
|
||||
backup-sentry.sh
|
||||
check-backup-integrity.sh
|
||||
sync-offsite-backups.sh
|
||||
backup-offsite-readiness-gate.sh
|
||||
verify-offsite-full-sync.sh
|
||||
enforce-latest-only-retention.sh
|
||||
)
|
||||
|
||||
MODE=""
|
||||
SOURCE_REVISION=""
|
||||
RUN_ID=""
|
||||
SOURCE_STAGE=""
|
||||
SOURCE_HEAD=""
|
||||
EXECUTOR_DIGEST=""
|
||||
VERIFIER_DIGEST=""
|
||||
STAGE_DIR=""
|
||||
ROLLBACK_DIR=""
|
||||
APPLY_STARTED=0
|
||||
APPLY_COMPLETE=0
|
||||
ROLLBACK_VERIFIED=0
|
||||
declare -A FILE_EXISTED=()
|
||||
declare -A PREVIOUS_DIGEST=()
|
||||
declare -A EXPECTED_DIGEST=()
|
||||
|
||||
usage() {
|
||||
printf '%s\n' \
|
||||
"Usage: $0 --mode check|apply|verify --source-revision SHA --run-id ID --source-stage PATH" \
|
||||
"The executor is fixed to host 192.168.0.110, an Agent99 run-bound source stage," \
|
||||
"and the 17-file /backup/scripts transaction."
|
||||
}
|
||||
|
||||
fail() {
|
||||
printf 'HOST110_BACKUP_RUNTIME_OK=0\nERROR=%s\n' "$1" >&2
|
||||
exit 64
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--mode)
|
||||
shift
|
||||
MODE="${1:-}"
|
||||
;;
|
||||
--source-revision)
|
||||
shift
|
||||
SOURCE_REVISION="${1:-}"
|
||||
;;
|
||||
--run-id)
|
||||
shift
|
||||
RUN_ID="${1:-}"
|
||||
;;
|
||||
--source-stage)
|
||||
shift
|
||||
SOURCE_STAGE="${1:-}"
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*) fail "unknown_argument" ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
case "$MODE" in check|apply|verify) ;; *) fail "invalid_mode" ;; esac
|
||||
[[ "$SOURCE_REVISION" =~ ^[0-9a-f]{40}$ ]] || fail "invalid_source_revision"
|
||||
[[ "$RUN_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$ ]] || fail "invalid_run_id"
|
||||
expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"
|
||||
[ "$SOURCE_STAGE" = "$expected_source_stage" ] || fail "invalid_source_stage"
|
||||
|
||||
for command_name in awk bash cp flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do
|
||||
command -v "$command_name" >/dev/null 2>&1 || fail "required_command_missing_${command_name}"
|
||||
done
|
||||
|
||||
current_user="$(id -un)"
|
||||
[ "$current_user" = "$EXPECTED_USER" ] || fail "unexpected_executor_user"
|
||||
hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx "$EXPECTED_HOST_IP" || fail "unexpected_executor_host"
|
||||
[ -d "$SOURCE_STAGE" ] && [ ! -L "$SOURCE_STAGE" ] || fail "canonical_source_stage_unavailable"
|
||||
[ -d "$DEST_ROOT" ] && [ ! -L "$DEST_ROOT" ] || fail "backup_runtime_destination_unavailable"
|
||||
|
||||
manifest_rows="$(python3 - "$SOURCE_STAGE/manifest.json" "$SOURCE_REVISION" "$RUN_ID" "${RUNTIME_FILES[@]}" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
source_revision = sys.argv[2]
|
||||
run_id = sys.argv[3]
|
||||
expected_names = sys.argv[4:]
|
||||
try:
|
||||
document = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
raise SystemExit(65)
|
||||
rows = document.get("files")
|
||||
if (
|
||||
document.get("schemaVersion") != "agent99_host110_backup_runtime_source_manifest_v1"
|
||||
or document.get("sourceRevision") != source_revision
|
||||
or document.get("runId") != run_id
|
||||
or not re.fullmatch(r"[0-9a-f]{40}", str(document.get("sourceHead", "")))
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("executorSha256", "")))
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("verifierSha256", "")))
|
||||
or not isinstance(rows, list)
|
||||
or len(rows) != len(expected_names)
|
||||
):
|
||||
raise SystemExit(66)
|
||||
by_name = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
raise SystemExit(66)
|
||||
name = str(row.get("name", ""))
|
||||
digest = str(row.get("sha256", ""))
|
||||
if name in by_name or not re.fullmatch(r"[A-Za-z0-9_.-]+", name) or not re.fullmatch(r"[0-9a-f]{64}", digest):
|
||||
raise SystemExit(66)
|
||||
by_name[name] = digest
|
||||
if set(by_name) != set(expected_names):
|
||||
raise SystemExit(66)
|
||||
print(f"SOURCE_HEAD {document['sourceHead']}")
|
||||
print(f"EXECUTOR_SHA256 {document['executorSha256']}")
|
||||
print(f"VERIFIER_SHA256 {document['verifierSha256']}")
|
||||
for name in expected_names:
|
||||
print(f"{name} {by_name[name]}")
|
||||
PY
|
||||
)" || fail "source_manifest_invalid"
|
||||
|
||||
while read -r manifest_name manifest_digest; do
|
||||
case "$manifest_name" in
|
||||
SOURCE_HEAD) SOURCE_HEAD="$manifest_digest" ;;
|
||||
EXECUTOR_SHA256) EXECUTOR_DIGEST="$manifest_digest" ;;
|
||||
VERIFIER_SHA256) VERIFIER_DIGEST="$manifest_digest" ;;
|
||||
*) EXPECTED_DIGEST[$manifest_name]="$manifest_digest" ;;
|
||||
esac
|
||||
done <<< "$manifest_rows"
|
||||
[[ "$SOURCE_HEAD" =~ ^[0-9a-f]{40}$ ]] || fail "source_head_invalid"
|
||||
|
||||
working_digest() {
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
}
|
||||
|
||||
[ "$(working_digest "$0")" = "$EXECUTOR_DIGEST" ] || fail "executor_identity_failed"
|
||||
[ -f "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \
|
||||
&& [ ! -L "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \
|
||||
&& [ "$(working_digest "$SOURCE_STAGE/verify-host110-backup-runtime.py")" = "$VERIFIER_DIGEST" ] \
|
||||
|| fail "verifier_identity_failed"
|
||||
|
||||
validate_file() {
|
||||
local path="$1"
|
||||
case "$path" in
|
||||
*.sh) bash -n "$path" ;;
|
||||
*.py) python3 -c 'import ast, pathlib, sys; ast.parse(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))' "$path" ;;
|
||||
*) return 64 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
source_digest() {
|
||||
local name="$1"
|
||||
printf '%s\n' "${EXPECTED_DIGEST[$name]:-}"
|
||||
}
|
||||
|
||||
validate_source() {
|
||||
local name source_path expected_digest actual_digest resolved
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
source_path="$SOURCE_STAGE/$name"
|
||||
[ -f "$source_path" ] && [ ! -L "$source_path" ] || return 65
|
||||
resolved="$(readlink -f "$source_path")"
|
||||
[[ "$resolved" == "$SOURCE_STAGE/"* ]] || return 65
|
||||
expected_digest="$(source_digest "$name")"
|
||||
actual_digest="$(working_digest "$source_path")"
|
||||
[ "$actual_digest" = "$expected_digest" ] || return 66
|
||||
validate_file "$source_path" || return 67
|
||||
done
|
||||
}
|
||||
|
||||
verify_destination() {
|
||||
local name dest_path expected_digest actual_digest
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
dest_path="$DEST_ROOT/$name"
|
||||
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] || return 68
|
||||
expected_digest="$(source_digest "$name")"
|
||||
actual_digest="$(working_digest "$dest_path")"
|
||||
[ "$actual_digest" = "$expected_digest" ] || return 69
|
||||
[ "$(stat -c '%U:%G:%a' "$dest_path")" = "wooo:wooo:755" ] || return 70
|
||||
done
|
||||
}
|
||||
|
||||
write_receipt() {
|
||||
local status="$1"
|
||||
local rollback_verified="$2"
|
||||
local receipt_path="$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json"
|
||||
install -d -m 700 "$STATUS_ROOT"
|
||||
python3 - "$receipt_path" "$status" "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$rollback_verified" "$VERIFIER_DIGEST" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
document = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v1",
|
||||
"status": sys.argv[2],
|
||||
"sourceRevision": sys.argv[3],
|
||||
"sourceHead": sys.argv[4],
|
||||
"runId": sys.argv[5],
|
||||
"fileCount": 17,
|
||||
"rollbackPerformed": sys.argv[6] == "1",
|
||||
"rollbackVerified": sys.argv[6] == "1",
|
||||
"verifierSha256": sys.argv[7],
|
||||
"executorHost": "192.168.0.110",
|
||||
"productionServiceRestarted": False,
|
||||
"secretValuesRead": False,
|
||||
"writtenAt": int(time.time()),
|
||||
}
|
||||
temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
|
||||
temporary.write_text(json.dumps(document, ensure_ascii=True, sort_keys=True) + "\n", encoding="utf-8")
|
||||
os.chmod(temporary, 0o600)
|
||||
try:
|
||||
os.link(temporary, path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
PY
|
||||
printf '%s' "$receipt_path"
|
||||
}
|
||||
|
||||
record_prestate() {
|
||||
local name dest_path digest
|
||||
: > "$ROLLBACK_DIR/prestate.tsv"
|
||||
chmod 600 "$ROLLBACK_DIR/prestate.tsv"
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
dest_path="$DEST_ROOT/$name"
|
||||
if [ -f "$dest_path" ] && [ ! -L "$dest_path" ]; then
|
||||
[ "$(stat -c '%U:%G' "$dest_path")" = "wooo:wooo" ] || return 71
|
||||
FILE_EXISTED[$name]=1
|
||||
digest="$(working_digest "$dest_path")"
|
||||
PREVIOUS_DIGEST[$name]="$digest"
|
||||
printf '%s\t1\t%s\n' "$name" "$digest" >> "$ROLLBACK_DIR/prestate.tsv"
|
||||
cp -p -- "$dest_path" "$ROLLBACK_DIR/$name"
|
||||
[ "$(working_digest "$ROLLBACK_DIR/$name")" = "$digest" ] || return 72
|
||||
elif [ ! -e "$dest_path" ]; then
|
||||
FILE_EXISTED[$name]=0
|
||||
PREVIOUS_DIGEST[$name]="-"
|
||||
printf '%s\t0\t-\n' "$name" >> "$ROLLBACK_DIR/prestate.tsv"
|
||||
else
|
||||
return 73
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
rollback_transaction() {
|
||||
local name dest_path backup_path temporary failed=0
|
||||
set +e
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
[ -n "${FILE_EXISTED[$name]+x}" ] || { failed=1; continue; }
|
||||
dest_path="$DEST_ROOT/$name"
|
||||
if [ "${FILE_EXISTED[$name]}" = "1" ]; then
|
||||
backup_path="$ROLLBACK_DIR/$name"
|
||||
temporary="$DEST_ROOT/.${name}.agent99-rollback-${RUN_ID}"
|
||||
cp -p -- "$backup_path" "$temporary" \
|
||||
&& mv -f -- "$temporary" "$dest_path" \
|
||||
|| failed=1
|
||||
else
|
||||
rm -f -- "$dest_path" || failed=1
|
||||
fi
|
||||
done
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
dest_path="$DEST_ROOT/$name"
|
||||
if [ "${FILE_EXISTED[$name]:-}" = "1" ]; then
|
||||
[ -f "$dest_path" ] && [ ! -L "$dest_path" ] \
|
||||
&& [ "$(working_digest "$dest_path" 2>/dev/null)" = "${PREVIOUS_DIGEST[$name]:-invalid}" ] \
|
||||
|| failed=1
|
||||
elif [ "${FILE_EXISTED[$name]:-}" = "0" ]; then
|
||||
[ ! -e "$dest_path" ] || failed=1
|
||||
else
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
set -e
|
||||
if [ "$failed" -eq 0 ]; then
|
||||
ROLLBACK_VERIFIED=1
|
||||
return 0
|
||||
fi
|
||||
ROLLBACK_VERIFIED=0
|
||||
return 1
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
local rollback_status="rollback_unverified"
|
||||
trap - EXIT HUP INT TERM
|
||||
if [ "$APPLY_STARTED" -eq 1 ] && [ "$APPLY_COMPLETE" -ne 1 ]; then
|
||||
if rollback_transaction; then
|
||||
rollback_status="failed_rolled_back"
|
||||
fi
|
||||
write_receipt "$rollback_status" "$ROLLBACK_VERIFIED" >/dev/null || true
|
||||
fi
|
||||
[ -z "$STAGE_DIR" ] || rm -rf -- "$STAGE_DIR"
|
||||
if [ "$ROLLBACK_VERIFIED" -eq 1 ]; then
|
||||
[ -z "$ROLLBACK_DIR" ] || rm -rf -- "$ROLLBACK_DIR"
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
validate_source || fail "source_validation_failed"
|
||||
|
||||
if [ "$MODE" = "check" ]; then
|
||||
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":17,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$MODE" = "verify" ]; then
|
||||
verify_destination || fail "destination_verification_failed"
|
||||
printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":17,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[ ! -e "$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json" ] || fail "run_identity_receipt_exists"
|
||||
STAGE_DIR="$STAGE_ROOT/$RUN_ID"
|
||||
ROLLBACK_DIR="$ROLLBACK_ROOT/$RUN_ID"
|
||||
[ ! -e "$STAGE_DIR" ] || fail "run_identity_stage_exists"
|
||||
[ ! -e "$ROLLBACK_DIR" ] || fail "run_identity_rollback_exists"
|
||||
|
||||
[ ! -L "$LOCK_PATH" ] || fail "deployment_lock_symlink_unsafe"
|
||||
(umask 077; : >> "$LOCK_PATH") || fail "deployment_lock_unavailable"
|
||||
[ -f "$LOCK_PATH" ] && [ -O "$LOCK_PATH" ] || fail "deployment_lock_owner_invalid"
|
||||
exec 9>>"$LOCK_PATH"
|
||||
flock -x -w 30 9 || fail "deployment_lock_timeout"
|
||||
|
||||
# Old runtime revisions predate the shared gate. Refuse the first promotion if
|
||||
# one of them is already active; after common.sh moves first, new starts block.
|
||||
if pgrep -afu "$EXPECTED_USER" '/backup/scripts/(backup-|check-backup|sync-offsite|verify-offsite|enforce-latest)' \
|
||||
| grep -vF "host110-backup-runtime-executor.sh" >/dev/null 2>&1; then
|
||||
fail "legacy_backup_runtime_active"
|
||||
fi
|
||||
|
||||
install -d -m 700 "$STAGE_DIR" "$ROLLBACK_DIR"
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM HUP
|
||||
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
install -m 700 "$SOURCE_STAGE/$name" "$STAGE_DIR/$name"
|
||||
[ "$(working_digest "$STAGE_DIR/$name")" = "$(source_digest "$name")" ] || fail "stage_identity_failed"
|
||||
validate_file "$STAGE_DIR/$name" || fail "stage_validation_failed"
|
||||
done
|
||||
|
||||
record_prestate || fail "rollback_prestate_capture_failed"
|
||||
APPLY_STARTED=1
|
||||
|
||||
# common.sh is promoted first. Every managed mutating backup entrypoint then
|
||||
# takes the shared side of LOCK_PATH before doing work, while this process holds
|
||||
# the exclusive side until the independent verifier has completed.
|
||||
for name in "${RUNTIME_FILES[@]}"; do
|
||||
temporary="$DEST_ROOT/.${name}.agent99-${RUN_ID}"
|
||||
install -m 755 "$STAGE_DIR/$name" "$temporary"
|
||||
mv -f -- "$temporary" "$DEST_ROOT/$name"
|
||||
done
|
||||
|
||||
verify_destination || fail "post_apply_verification_failed"
|
||||
verifier_result="$(BACKUP_RUNTIME_DEPLOY_CONTEXT=1 python3 "$SOURCE_STAGE/verify-host110-backup-runtime.py" \
|
||||
--manifest "$SOURCE_STAGE/manifest.json" \
|
||||
--destination "$DEST_ROOT" \
|
||||
--source-revision "$SOURCE_REVISION" \
|
||||
--run-id "$RUN_ID" \
|
||||
--expected-verifier-sha256 "$VERIFIER_DIGEST")" || fail "independent_verifier_failed"
|
||||
python3 - "$verifier_result" "$SOURCE_REVISION" "$RUN_ID" <<'PY' || fail "independent_verifier_contract_failed"
|
||||
import json
|
||||
import sys
|
||||
|
||||
try:
|
||||
row = json.loads(sys.argv[1])
|
||||
except json.JSONDecodeError:
|
||||
raise SystemExit(1)
|
||||
if not (
|
||||
row.get("schemaVersion") == "agent99_host110_backup_runtime_verifier_v1"
|
||||
and row.get("ok") is True
|
||||
and row.get("sourceRevision") == sys.argv[2]
|
||||
and row.get("runId") == sys.argv[3]
|
||||
and row.get("fileCount") == 17
|
||||
and row.get("remoteWritePerformed") is False
|
||||
and row.get("selfIdentityVerified") is True
|
||||
):
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
|
||||
APPLY_COMPLETE=1
|
||||
receipt_path="$(write_receipt "verified" 0)"
|
||||
rm -rf -- "$STAGE_DIR" "$ROLLBACK_DIR"
|
||||
STAGE_DIR=""
|
||||
ROLLBACK_DIR=""
|
||||
trap - EXIT HUP INT TERM
|
||||
python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$receipt_path" "$verifier_result" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
print(json.dumps({
|
||||
"schemaVersion": "agent99_host110_backup_runtime_executor_v1",
|
||||
"mode": "apply",
|
||||
"ok": True,
|
||||
"sourceRevision": sys.argv[1],
|
||||
"sourceHead": sys.argv[2],
|
||||
"fileCount": 17,
|
||||
"rollbackPerformed": False,
|
||||
"receipt": sys.argv[3],
|
||||
"verifier": json.loads(sys.argv[4]),
|
||||
}, ensure_ascii=True, sort_keys=True))
|
||||
PY
|
||||
164
scripts/backup/verify-host110-backup-runtime.py
Executable file
164
scripts/backup/verify-host110-backup-runtime.py
Executable file
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independent no-write verifier for the Host110 backup runtime transaction."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
EXPECTED_DESTINATION = Path("/backup/scripts")
|
||||
RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")
|
||||
EXPECTED_FILES = (
|
||||
"common.sh",
|
||||
"backup-all.sh",
|
||||
"backup-host188-products.sh",
|
||||
"verify-host188-products-backup.sh",
|
||||
"verify-host188-products-archive.py",
|
||||
"backup-gitea.sh",
|
||||
"gitea-full-backup-restore-drill.sh",
|
||||
"backup-configs.sh",
|
||||
"backup-awoooi.sh",
|
||||
"backup-awoooi-frequent.sh",
|
||||
"backup-clawbot.sh",
|
||||
"backup-sentry.sh",
|
||||
"check-backup-integrity.sh",
|
||||
"sync-offsite-backups.sh",
|
||||
"backup-offsite-readiness-gate.sh",
|
||||
"verify-offsite-full-sync.sh",
|
||||
"enforce-latest-only-retention.sh",
|
||||
)
|
||||
|
||||
|
||||
def fail(reason: str) -> None:
|
||||
print(f"HOST110_BACKUP_RUNTIME_VERIFY_OK=0\nERROR={reason}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def acquire_runtime_read_lock(destination: Path):
|
||||
if destination != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_DEPLOY_CONTEXT") == "1":
|
||||
return None
|
||||
if RUNTIME_LOCK.is_symlink() or not RUNTIME_LOCK.is_file():
|
||||
fail("runtime_lock_unavailable")
|
||||
handle = RUNTIME_LOCK.open("rb")
|
||||
if RUNTIME_LOCK.stat().st_uid != os.getuid():
|
||||
handle.close()
|
||||
fail("runtime_lock_owner_invalid")
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
|
||||
return handle
|
||||
except BlockingIOError:
|
||||
if time.monotonic() >= deadline:
|
||||
handle.close()
|
||||
fail("runtime_deployment_active")
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
manifest_source = parser.add_mutually_exclusive_group(required=True)
|
||||
manifest_source.add_argument("--manifest")
|
||||
manifest_source.add_argument("--manifest-base64")
|
||||
parser.add_argument("--destination", required=True)
|
||||
parser.add_argument("--source-revision", required=True)
|
||||
parser.add_argument("--run-id", required=True)
|
||||
parser.add_argument("--expected-verifier-sha256", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", args.source_revision):
|
||||
fail("invalid_source_revision")
|
||||
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}", args.run_id):
|
||||
fail("invalid_run_id")
|
||||
destination = Path(args.destination)
|
||||
if destination != EXPECTED_DESTINATION or destination.is_symlink() or not destination.is_dir():
|
||||
fail("invalid_destination")
|
||||
runtime_lock = acquire_runtime_read_lock(destination)
|
||||
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", args.expected_verifier_sha256):
|
||||
fail("invalid_verifier_identity")
|
||||
try:
|
||||
if args.manifest:
|
||||
manifest_text = Path(args.manifest).read_text(encoding="utf-8")
|
||||
else:
|
||||
manifest_text = base64.b64decode(args.manifest_base64, validate=True).decode("utf-8")
|
||||
document = json.loads(manifest_text)
|
||||
except (OSError, UnicodeDecodeError, ValueError, json.JSONDecodeError):
|
||||
fail("manifest_unavailable")
|
||||
rows = document.get("files")
|
||||
if (
|
||||
document.get("schemaVersion") != "agent99_host110_backup_runtime_source_manifest_v1"
|
||||
or document.get("sourceRevision") != args.source_revision
|
||||
or document.get("runId") != args.run_id
|
||||
or not re.fullmatch(r"[0-9a-f]{40}", str(document.get("sourceHead", "")))
|
||||
or document.get("verifierSha256") != args.expected_verifier_sha256
|
||||
or not isinstance(rows, list)
|
||||
or len(rows) != len(EXPECTED_FILES)
|
||||
):
|
||||
fail("manifest_contract_failed")
|
||||
by_name: dict[str, str] = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
fail("manifest_file_set_failed")
|
||||
name = str(row.get("name", ""))
|
||||
if name in by_name:
|
||||
fail("manifest_file_set_failed")
|
||||
by_name[name] = str(row.get("sha256", ""))
|
||||
if set(by_name) != set(EXPECTED_FILES) or not all(re.fullmatch(r"[0-9a-f]{64}", value) for value in by_name.values()):
|
||||
fail("manifest_file_set_failed")
|
||||
|
||||
verifier_path = Path(__file__)
|
||||
self_identity_verified = verifier_path.is_file() and sha256(verifier_path) == args.expected_verifier_sha256
|
||||
if args.manifest and not self_identity_verified:
|
||||
fail("verifier_identity_failed")
|
||||
|
||||
root = destination.resolve(strict=True)
|
||||
for name in EXPECTED_FILES:
|
||||
path = destination / name
|
||||
if path.is_symlink() or not path.is_file():
|
||||
fail("runtime_file_type_failed")
|
||||
resolved = path.resolve(strict=True)
|
||||
if resolved.parent != root:
|
||||
fail("runtime_file_boundary_failed")
|
||||
stat = resolved.stat()
|
||||
if stat.st_uid != os.getuid() or stat.st_gid != os.getgid() or stat.st_mode & 0o777 != 0o755:
|
||||
fail("runtime_file_metadata_failed")
|
||||
if sha256(resolved) != by_name[name]:
|
||||
fail("runtime_file_identity_failed")
|
||||
|
||||
result = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_verifier_v1",
|
||||
"ok": True,
|
||||
"sourceRevision": args.source_revision,
|
||||
"runId": args.run_id,
|
||||
"fileCount": len(EXPECTED_FILES),
|
||||
"destination": str(EXPECTED_DESTINATION),
|
||||
"remoteWritePerformed": False,
|
||||
"secretValuesRead": False,
|
||||
"selfIdentityVerified": self_identity_verified,
|
||||
"verifierSha256": args.expected_verifier_sha256,
|
||||
}
|
||||
print(json.dumps(result, ensure_ascii=True, sort_keys=True))
|
||||
if runtime_lock is not None:
|
||||
runtime_lock.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -4,11 +4,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
@@ -20,6 +23,28 @@ EXPECTED_PREFIXES = {
|
||||
"n8n-online-": ".tar.gz",
|
||||
}
|
||||
FIXED_ASSETS = {"n8n-validation.json"}
|
||||
RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")
|
||||
|
||||
|
||||
def acquire_runtime_lock():
|
||||
if Path(__file__).resolve().parent != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_SHARED_LOCK_HELD") == "1":
|
||||
return None
|
||||
if RUNTIME_LOCK.is_symlink():
|
||||
raise RuntimeError("backup_runtime_gate_unsafe")
|
||||
handle = RUNTIME_LOCK.open("a+")
|
||||
if RUNTIME_LOCK.stat().st_uid != os.getuid():
|
||||
handle.close()
|
||||
raise RuntimeError("backup_runtime_gate_owner_invalid")
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
|
||||
return handle
|
||||
except BlockingIOError:
|
||||
if time.monotonic() >= deadline:
|
||||
handle.close()
|
||||
raise RuntimeError("backup_runtime_deployment_active")
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
@@ -153,10 +178,13 @@ def validate_restore(root: Path) -> dict[str, Any]:
|
||||
|
||||
|
||||
def main() -> int:
|
||||
runtime_lock = acquire_runtime_lock()
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(validate_restore(args.root), separators=(",", ":")))
|
||||
if runtime_lock is not None:
|
||||
runtime_lock.close()
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -293,8 +293,8 @@ def test_entrypoint_is_in_exact_agent99_runtime_bundle() -> None:
|
||||
assert expected in _quoted_array(contract, "$requiredFiles = @(", ")")
|
||||
assert expected in _quoted_array(sender, "RUNTIME_FILES=(", ")")
|
||||
assert expected in _quoted_array(receiver, "$FixedRuntimeFiles = @(", ")")
|
||||
assert 'expectedRuntimeFileCount": 18' in sender
|
||||
assert '$runtimeManifest.fileCount -eq 18' in receiver
|
||||
assert 'expectedRuntimeFileCount": 19' in sender
|
||||
assert '$runtimeManifest.fileCount -eq 19' in receiver
|
||||
|
||||
|
||||
def test_runtime_contract_check_requires_typed_signoz_transport_readback() -> None:
|
||||
|
||||
@@ -122,7 +122,7 @@ def test_agent99_atomic_bundle_owns_the_dispatch_entrypoint() -> None:
|
||||
assert "$dbExecutorRecoveryContract" in contract
|
||||
assert "awoooi_db_bounded_executor_receipt_v2" in contract
|
||||
assert "Receipt.database_sqlstate" in contract
|
||||
assert 'expectedRuntimeFileCount": 18' in transport
|
||||
assert "expectedRuntimeFileCount=18" in transport
|
||||
assert "expectedRuntimeFileCount -ne 18" in receiver
|
||||
assert "fileCount = 18" in receiver
|
||||
assert 'expectedRuntimeFileCount": 19' in transport
|
||||
assert "expectedRuntimeFileCount=19" in transport
|
||||
assert "expectedRuntimeFileCount -ne 19" in receiver
|
||||
assert "fileCount = 19" in receiver
|
||||
|
||||
@@ -486,7 +486,7 @@ $decision = Get-AgentLivePreflightDecision `
|
||||
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
|
||||
-Manifest $manifest `
|
||||
-ManifestCheckRequired ([bool]($ReadinessScope -eq "FullRuntime")) `
|
||||
-ExpectedRuntimeFileCount 18 `
|
||||
-ExpectedRuntimeFileCount 19 `
|
||||
-TaskFailureCount $taskFailures.Count `
|
||||
-RelayListenerCount $relayListeners.Count `
|
||||
-RequiredEvidenceStaleCount $requiredEvidenceStale.Count `
|
||||
|
||||
@@ -34,6 +34,7 @@ $FixedRuntimeFiles = @(
|
||||
"agent99-control-plane.ps1",
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
@@ -951,7 +952,7 @@ try {
|
||||
}
|
||||
$sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant()
|
||||
if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
|
||||
if ([int]$envelope.expectedRuntimeFileCount -ne 18) { throw "invalid_expected_runtime_file_count" }
|
||||
if ([int]$envelope.expectedRuntimeFileCount -ne 19) { throw "invalid_expected_runtime_file_count" }
|
||||
|
||||
$traceId = [string]$envelope.traceId
|
||||
$runId = [string]$envelope.runId
|
||||
@@ -986,7 +987,7 @@ try {
|
||||
$sourceManifest = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_source_manifest_v1"
|
||||
sourceRevision = $sourceRevision
|
||||
fileCount = 18
|
||||
fileCount = 19
|
||||
manifestSha256 = $manifestDigest
|
||||
files = @($FixedRuntimeFiles | ForEach-Object {
|
||||
[pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() }
|
||||
@@ -1023,7 +1024,7 @@ try {
|
||||
status = "check_ready"
|
||||
mode = "check"
|
||||
sourceRevision = $sourceRevision
|
||||
expectedRuntimeFileCount = 18
|
||||
expectedRuntimeFileCount = 19
|
||||
manifestSha256 = $manifestDigest
|
||||
plannedStagingPath = $stagePath
|
||||
runtimeManifest = $currentManifest
|
||||
@@ -1110,7 +1111,7 @@ try {
|
||||
if (
|
||||
[string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or
|
||||
[string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or
|
||||
[int]$existingSourceManifest.fileCount -ne 18 -or
|
||||
[int]$existingSourceManifest.fileCount -ne 19 -or
|
||||
[string]$existingSourceManifest.manifestSha256 -ne $manifestDigest
|
||||
) { throw "existing_staging_manifest_mismatch" }
|
||||
} catch {
|
||||
@@ -1134,7 +1135,7 @@ try {
|
||||
[string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and
|
||||
$live.runtimeMatched -and
|
||||
$live.sourceRevision -eq $sourceRevision -and
|
||||
$live.fileCount -eq 18 -and
|
||||
$live.fileCount -eq 19 -and
|
||||
$live.mismatchCount -eq 0 -and
|
||||
$liveLauncherContract.ok -and
|
||||
[string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256
|
||||
@@ -1447,7 +1448,7 @@ try {
|
||||
$runtimeManifest.exists -and
|
||||
$runtimeManifest.runtimeMatched -and
|
||||
$runtimeManifest.sourceRevision -eq $sourceRevision -and
|
||||
$runtimeManifest.fileCount -eq 18 -and
|
||||
$runtimeManifest.fileCount -eq 19 -and
|
||||
$runtimeManifest.mismatchCount -eq 0 -and
|
||||
$launcherContract.ok -and
|
||||
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and
|
||||
|
||||
@@ -18,6 +18,7 @@ RUNTIME_FILES=(
|
||||
"agent99-control-plane.ps1"
|
||||
"agent99-db-bounded-executor.ps1"
|
||||
"agent99-signoz-metadata-executor.ps1"
|
||||
"agent99-host110-backup-runtime-broker.ps1"
|
||||
"agent99-deploy.ps1"
|
||||
"agent99-register-tasks.ps1"
|
||||
"agent99-alertmanager-alertchain-poll.ps1"
|
||||
@@ -227,7 +228,7 @@ trace_id, run_id, work_item_id = sys.argv[4:7]
|
||||
output_path = Path(sys.argv[7])
|
||||
runtime_names = sys.argv[8:]
|
||||
|
||||
if len(runtime_names) != 18 or len(set(runtime_names)) != 18:
|
||||
if len(runtime_names) != 19 or len(set(runtime_names)) != 19:
|
||||
raise SystemExit("fixed_runtime_file_contract_failed")
|
||||
|
||||
files: list[dict[str, str]] = []
|
||||
@@ -267,7 +268,7 @@ envelope = {
|
||||
"runId": run_id,
|
||||
"workItemId": work_item_id,
|
||||
"sourceRevision": source_revision,
|
||||
"expectedRuntimeFileCount": 18,
|
||||
"expectedRuntimeFileCount": 19,
|
||||
"manifestSha256": manifest_sha256,
|
||||
"files": files,
|
||||
"livePreflight": {
|
||||
@@ -419,14 +420,14 @@ stage_token = hashlib.sha256(stage_identity.encode("utf-8")).hexdigest()[:20]
|
||||
script = r'''$ErrorActionPreference='Stop';$ProgressPreference='SilentlyContinue'
|
||||
$a='C:\Wooo\Agent99';$b=Join-Path $a 'bin';$p=Join-Path $a 'state\runtime-manifest.json'
|
||||
$r=[ordered]@{exists=$false;sourceRevision='';runtimeMatched=$false;recordedRuntimeMatched=$false;fileCount=0;mismatchCount=-1;recordedMismatchCount=-1;parseError=''}
|
||||
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 18-and$m.mismatchCount-eq 0-and$rows.Count-eq 18-and$seen.Count-eq 18-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
|
||||
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 19-and$m.mismatchCount-eq 0-and$rows.Count-eq 19-and$seen.Count-eq 19-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
|
||||
$c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''}
|
||||
try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name}
|
||||
$z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false}
|
||||
$ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0)
|
||||
$status=if($ready){'check_ready'}else{'check_failed_no_apply'}
|
||||
$next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'}
|
||||
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=18;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
|
||||
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=19;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
|
||||
if(-not$ready){exit 65}'''
|
||||
script = (
|
||||
script.replace("__SOURCE_REVISION__", source_revision)
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
BROKER = ROOT / "agent99-host110-backup-runtime-broker.ps1"
|
||||
EXECUTOR = ROOT / "scripts" / "backup" / "host110-backup-runtime-executor.sh"
|
||||
VERIFIER = ROOT / "scripts" / "backup" / "verify-host110-backup-runtime.py"
|
||||
|
||||
|
||||
EXPECTED_RUNTIME_FILES = {
|
||||
"common.sh",
|
||||
"backup-all.sh",
|
||||
"backup-host188-products.sh",
|
||||
"verify-host188-products-backup.sh",
|
||||
"verify-host188-products-archive.py",
|
||||
"backup-gitea.sh",
|
||||
"gitea-full-backup-restore-drill.sh",
|
||||
"backup-configs.sh",
|
||||
"backup-awoooi.sh",
|
||||
"backup-awoooi-frequent.sh",
|
||||
"backup-clawbot.sh",
|
||||
"backup-sentry.sh",
|
||||
"check-backup-integrity.sh",
|
||||
"sync-offsite-backups.sh",
|
||||
"backup-offsite-readiness-gate.sh",
|
||||
"verify-offsite-full-sync.sh",
|
||||
"enforce-latest-only-retention.sh",
|
||||
}
|
||||
|
||||
|
||||
def _load_verifier():
|
||||
spec = importlib.util.spec_from_file_location("host110_runtime_verifier", VERIFIER)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
|
||||
assert '[ValidateSet("Check", "Apply", "Verify")]' in source
|
||||
assert '$TargetHost = "192.168.0.110"' in source
|
||||
assert '$TargetUser = "wooo"' in source
|
||||
assert '$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"' in source
|
||||
assert 'keys\\agent99_ed25519' in source
|
||||
assert '"credential.helper="' in source
|
||||
assert '$env:GIT_TERMINAL_PROMPT = "0"' in source
|
||||
assert '"merge-base", "--is-ancestor", $SourceRevision, $sourceHead' in source
|
||||
assert '"checkout", "--quiet", $SourceRevision, "--"' in source
|
||||
assert 'executorSha256 = $executorDigest' in source
|
||||
assert 'verifierSha256 = $verifierDigest' in source
|
||||
assert "/home/wooo/awoooi" not in source
|
||||
assert '"StrictHostKeyChecking=yes"' in source
|
||||
assert '"NumberOfPasswordPrompts=0"' in source
|
||||
assert '"ConnectionAttempts=1"' in source
|
||||
assert "$process.WaitForExit($TimeoutSeconds * 1000)" in source
|
||||
assert "taskkill.exe /PID $process.Id /T /F" in source
|
||||
assert "$process.Kill()" in source
|
||||
assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths' in source
|
||||
|
||||
|
||||
def test_windows99_broker_verify_is_independent_no_write_and_evidence_is_immutable() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
|
||||
assert 'RedirectStandardInput = $StandardInputPath' in source
|
||||
assert "python3 - --manifest-base64" in source
|
||||
assert 'remoteWritePerformed' in source
|
||||
assert '[IO.File]::Move($temporary, $Path)' in source
|
||||
assert 'Move-Item -LiteralPath $temporary -Destination $Path -Force' not in source
|
||||
assert 'run_identity_conflict' in source
|
||||
assert 'run_identity_evidence_exists' in source
|
||||
assert 'decisionProvider = "deterministic_only"' in source
|
||||
assert 'criticProvider = "deterministic_only"' in source
|
||||
assert 'verifier = "independent_host110_backup_runtime_python_readback"' in source
|
||||
|
||||
|
||||
def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
|
||||
source = EXECUTOR.read_text(encoding="utf-8")
|
||||
array_body = source.split("readonly -a RUNTIME_FILES=(", 1)[1].split(")", 1)[0]
|
||||
runtime_files = set(re.findall(r"^\s+([A-Za-z0-9_.-]+)\s*$", array_body, re.MULTILINE))
|
||||
|
||||
assert runtime_files == EXPECTED_RUNTIME_FILES
|
||||
assert 'readonly EXPECTED_HOST_IP="192.168.0.110"' in source
|
||||
assert 'readonly EXPECTED_USER="wooo"' in source
|
||||
assert 'readonly DEST_ROOT="/backup/scripts"' in source
|
||||
assert '--source-stage' in source
|
||||
assert 'document.get("executorSha256"' in source
|
||||
assert 'document.get("verifierSha256"' in source
|
||||
assert 'executor_identity_failed' in source
|
||||
assert 'verifier_identity_failed' in source
|
||||
assert 'flock -x -w 30 9' in source
|
||||
assert 'legacy_backup_runtime_active' in source
|
||||
assert source.index("for name in \"${RUNTIME_FILES[@]}\"") < source.index("mv -f -- \"$temporary\"")
|
||||
assert 'rollback_unverified' in source
|
||||
assert 'failed_rolled_back' in source
|
||||
assert 'working_digest "$dest_path"' in source
|
||||
assert 'os.link(temporary, path)' in source
|
||||
assert 'run_identity_receipt_exists' in source
|
||||
assert 'run_identity_stage_exists' in source
|
||||
assert 'run_identity_rollback_exists' in source
|
||||
assert 'verify-host110-backup-runtime.py' in source
|
||||
assert 'selfIdentityVerified' in source
|
||||
assert '"productionServiceRestarted": False' in source
|
||||
assert '"secretValuesRead": False' in source
|
||||
assert "sudo " not in source
|
||||
assert "docker " not in source
|
||||
assert "systemctl " not in source
|
||||
|
||||
|
||||
def test_backup_runtime_entrypoints_share_the_deployment_gate() -> None:
|
||||
common = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8")
|
||||
restore = (ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh").read_text(encoding="utf-8")
|
||||
offsite = (ROOT / "scripts/backup/backup-offsite-readiness-gate.sh").read_text(encoding="utf-8")
|
||||
archive = (ROOT / "scripts/backup/verify-host188-products-archive.py").read_text(encoding="utf-8")
|
||||
|
||||
for source in (common, restore, offsite):
|
||||
assert "/tmp/agent99-host110-backup-runtime.lock" in source
|
||||
assert "flock -s -w 60" in source
|
||||
assert "fcntl.LOCK_SH | fcntl.LOCK_NB" in archive
|
||||
assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" in archive
|
||||
|
||||
|
||||
def test_independent_verifier_accepts_exact_bundle_without_writing(tmp_path, monkeypatch, capsys) -> None:
|
||||
verifier = _load_verifier()
|
||||
destination = tmp_path / "scripts"
|
||||
destination.mkdir()
|
||||
rows = []
|
||||
for name in verifier.EXPECTED_FILES:
|
||||
path = destination / name
|
||||
path.write_text(f"runtime:{name}\n", encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
rows.append({"name": name, "sha256": _sha256(path)})
|
||||
verifier_digest = _sha256(VERIFIER)
|
||||
source_revision = "a" * 40
|
||||
run_id = "unit-exact"
|
||||
manifest = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
|
||||
"sourceRevision": source_revision,
|
||||
"sourceHead": "b" * 40,
|
||||
"runId": run_id,
|
||||
"executorSha256": "c" * 64,
|
||||
"verifierSha256": verifier_digest,
|
||||
"files": rows,
|
||||
}
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
before = {path.name: (path.stat().st_mtime_ns, _sha256(path)) for path in destination.iterdir()}
|
||||
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
str(VERIFIER),
|
||||
"--manifest",
|
||||
str(manifest_path),
|
||||
"--destination",
|
||||
str(destination),
|
||||
"--source-revision",
|
||||
source_revision,
|
||||
"--run-id",
|
||||
run_id,
|
||||
"--expected-verifier-sha256",
|
||||
verifier_digest,
|
||||
],
|
||||
)
|
||||
|
||||
verifier.main()
|
||||
|
||||
result = json.loads(capsys.readouterr().out)
|
||||
after = {path.name: (path.stat().st_mtime_ns, _sha256(path)) for path in destination.iterdir()}
|
||||
assert result["ok"] is True
|
||||
assert result["remoteWritePerformed"] is False
|
||||
assert result["selfIdentityVerified"] is True
|
||||
assert before == after
|
||||
|
||||
|
||||
def test_independent_verifier_rejects_drifted_bundle(tmp_path, monkeypatch, capsys) -> None:
|
||||
verifier = _load_verifier()
|
||||
destination = tmp_path / "scripts"
|
||||
destination.mkdir()
|
||||
rows = []
|
||||
for name in verifier.EXPECTED_FILES:
|
||||
path = destination / name
|
||||
path.write_text(f"runtime:{name}\n", encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
rows.append({"name": name, "sha256": _sha256(path)})
|
||||
(destination / verifier.EXPECTED_FILES[0]).write_text("drifted\n", encoding="utf-8")
|
||||
verifier_digest = _sha256(VERIFIER)
|
||||
source_revision = "d" * 40
|
||||
run_id = "unit-drift"
|
||||
manifest = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
|
||||
"sourceRevision": source_revision,
|
||||
"sourceHead": "e" * 40,
|
||||
"runId": run_id,
|
||||
"executorSha256": "f" * 64,
|
||||
"verifierSha256": verifier_digest,
|
||||
"files": rows,
|
||||
}
|
||||
encoded = base64.b64encode(json.dumps(manifest).encode()).decode()
|
||||
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
str(VERIFIER),
|
||||
"--manifest-base64",
|
||||
encoded,
|
||||
"--destination",
|
||||
str(destination),
|
||||
"--source-revision",
|
||||
source_revision,
|
||||
"--run-id",
|
||||
run_id,
|
||||
"--expected-verifier-sha256",
|
||||
verifier_digest,
|
||||
],
|
||||
)
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
verifier.main()
|
||||
assert "runtime_file_identity_failed" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_agent99_runtime_lists_include_the_host110_broker_once() -> None:
|
||||
sources = [
|
||||
ROOT / "agent99-bootstrap.ps1",
|
||||
ROOT / "agent99-contract-check.ps1",
|
||||
ROOT / "agent99-deploy.ps1",
|
||||
ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh",
|
||||
ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1",
|
||||
]
|
||||
marker = "agent99-host110-backup-runtime-broker.ps1"
|
||||
|
||||
for path in sources:
|
||||
assert marker in path.read_text(encoding="utf-8"), path
|
||||
@@ -36,7 +36,7 @@ def _base_case() -> dict[str, Any]:
|
||||
"exists": True,
|
||||
"sourceRevision": "a" * 40,
|
||||
"runtimeMatched": True,
|
||||
"fileCount": 18,
|
||||
"fileCount": 19,
|
||||
"mismatchCount": 0,
|
||||
"parseError": "",
|
||||
},
|
||||
@@ -133,7 +133,7 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
|
||||
)
|
||||
assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision
|
||||
assert "preflightGreen = [bool]$decision.deploymentEligible" in source
|
||||
assert "-ExpectedRuntimeFileCount 18" in source
|
||||
assert "-ExpectedRuntimeFileCount 19" in source
|
||||
assert "warningReasons = $warningReasons" in source
|
||||
assert "selfHealthPerformanceIssues = @(" in source
|
||||
assert 'if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$")' in source
|
||||
|
||||
@@ -26,6 +26,7 @@ EXPECTED_RUNTIME_FILES = (
|
||||
"agent99-control-plane.ps1",
|
||||
"agent99-db-bounded-executor.ps1",
|
||||
"agent99-signoz-metadata-executor.ps1",
|
||||
"agent99-host110-backup-runtime-broker.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
@@ -88,7 +89,7 @@ def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None
|
||||
)
|
||||
assert preflight_count is not None
|
||||
assert int(preflight_count.group(1)) == len(EXPECTED_RUNTIME_FILES)
|
||||
assert "expectedRuntimeFileCount\": 18" in sender
|
||||
assert "expectedRuntimeFileCount\": 19" in sender
|
||||
assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver
|
||||
assert "required_runtime_file_missing" in receiver
|
||||
assert "duplicate_runtime_file" in receiver
|
||||
@@ -263,7 +264,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
|
||||
)
|
||||
assert "$livePreflight.sourceRevision -eq $sourceRevision" in source
|
||||
assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source
|
||||
assert '$runtimeManifest.fileCount -eq 18' in source
|
||||
assert '$runtimeManifest.fileCount -eq 19' in source
|
||||
assert "function Invoke-AgentWindowsControlBaseline" in source
|
||||
assert "function Get-AgentRunEvidenceToken" in source
|
||||
assert "$evidenceRunToken = Get-AgentRunEvidenceToken -Value $RunId" in source
|
||||
|
||||
@@ -516,7 +516,8 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
|
||||
for parameter in ("AutomationRunId", "TraceId", "WorkItemId"):
|
||||
assert f"[string]${parameter} = \"\"" in control
|
||||
assert f'[string]`${parameter} = ""' in bootstrap
|
||||
assert 'reason = "controlled_dispatch_identity_unsafe"' in control
|
||||
assert '"controlled_dispatch_identity_unsafe"' in control
|
||||
assert "reason = $identityRejectionReason" in control
|
||||
assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
|
||||
assert '$dryRun.managerPreflightStatus -eq "ready"' in function
|
||||
assert "-not $dryRun.artifactTransactionBlocked" in function
|
||||
|
||||
Reference in New Issue
Block a user