Files
awoooi/agent99-host110-backup-runtime-broker.ps1

482 lines
19 KiB
PowerShell

[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[ValidateSet("Check", "Apply", "Verify")]
[string]$Mode,
[Parameter(Mandatory = $true)]
[string]$SourceRevision,
[Parameter(Mandatory = $true)]
[string]$RunId,
[string]$AgentRoot = "C:\Wooo\Agent99"
)
$ErrorActionPreference = "Stop"
$ExpectedAgentRoot = "C:\Wooo\Agent99"
$TargetHost = "192.168.0.110"
$TargetUser = "wooo"
$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"
$GitPath = "C:\Program Files\Git\cmd\git.exe"
$IdentityFile = Join-Path $ExpectedAgentRoot "keys\agent99_ed25519"
$EvidenceRoot = Join-Path $ExpectedAgentRoot "evidence\host110-backup-runtime"
$SourceRootBase = Join-Path $ExpectedAgentRoot "deploy"
$TransportMutexName = "Global\WoooAgent99Host110BackupRuntimeV2"
$ExpectedFileCount = 18
$ExpectedRemoteStageFileCount = 21
$RuntimeRelativePaths = @(
"scripts/backup/common.sh",
"scripts/backup/backup-all.sh",
"scripts/backup/backup-host188-products.sh",
"scripts/backup/verify-host188-products-backup.sh",
"scripts/backup/verify-host188-products-archive.py",
"scripts/backup/backup-gitea.sh",
"scripts/backup/gitea-full-backup-restore-drill.sh",
"scripts/backup/backup-configs.sh",
"scripts/backup/backup-awoooi.sh",
"scripts/backup/backup-awoooi-frequent.sh",
"scripts/backup/backup-clawbot.sh",
"scripts/backup/backup-sentry.sh",
"scripts/backup/check-backup-integrity.sh",
"scripts/backup/sync-offsite-backups.sh",
"scripts/backup/backup-offsite-readiness-gate.sh",
"scripts/backup/verify-offsite-full-sync.sh",
"scripts/backup/enforce-latest-only-retention.sh",
"scripts/ops/backup-health-textfile-exporter.py"
)
$ExecutorRelativePath = "scripts/backup/host110-backup-runtime-executor.sh"
$VerifierRelativePath = "scripts/backup/verify-host110-backup-runtime.py"
function Test-Agent99SafeIdentity {
param([string]$Value)
return [bool]($Value -and $Value -match "^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$")
}
function Write-Agent99ImmutableJson {
param([string]$Path, [object]$Value)
if (Test-Path -LiteralPath $Path) { throw "run_identity_evidence_exists" }
$temporary = "$Path.tmp-$PID-$([Guid]::NewGuid().ToString('N'))"
$encoding = New-Object System.Text.UTF8Encoding($false)
try {
$json = $Value | ConvertTo-Json -Depth 12
[IO.File]::WriteAllText($temporary, $json + [Environment]::NewLine, $encoding)
[IO.File]::Move($temporary, $Path)
} finally {
Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue | Out-Null
}
}
function Invoke-Agent99BoundedProcess {
param(
[string]$FilePath,
[string[]]$Arguments,
[int]$TimeoutSeconds,
[string]$WorkingDirectory = "",
[string]$StandardInputPath = ""
)
$token = [Guid]::NewGuid().ToString("N")
$stdoutPath = Join-Path $env:TEMP "agent99-host110-backup-$token.out"
$stderrPath = Join-Path $env:TEMP "agent99-host110-backup-$token.err"
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
$process = $null
try {
$start = @{
FilePath = $FilePath
ArgumentList = $Arguments
NoNewWindow = $true
RedirectStandardOutput = $stdoutPath
RedirectStandardError = $stderrPath
PassThru = $true
}
if ($WorkingDirectory) { $start.WorkingDirectory = $WorkingDirectory }
if ($StandardInputPath) { $start.RedirectStandardInput = $StandardInputPath }
$process = Start-Process @start
$null = $process.Handle
if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {}
if (-not $process.WaitForExit(2000) -and -not $process.HasExited) {
$process.Kill()
$process.WaitForExit(2000) | Out-Null
}
return [pscustomobject]@{
ok = $false
exitCode = -2
reason = "transport_timeout"
elapsedMs = $stopwatch.ElapsedMilliseconds
stdout = ""
stderrPresent = $false
}
}
$process.WaitForExit() | Out-Null
$process.Refresh()
$stdout = if (Test-Path -LiteralPath $stdoutPath) { Get-Content -LiteralPath $stdoutPath -Raw } else { "" }
$stderr = if (Test-Path -LiteralPath $stderrPath) { Get-Content -LiteralPath $stderrPath -Raw } else { "" }
if ($stdout.Length -gt 65536 -or $stderr.Length -gt 65536) {
return [pscustomobject]@{
ok = $false
exitCode = -3
reason = "transport_output_oversized"
elapsedMs = $stopwatch.ElapsedMilliseconds
stdout = ""
stderrPresent = [bool]$stderr
}
}
return [pscustomobject]@{
ok = [bool]($process.ExitCode -eq 0)
exitCode = [int]$process.ExitCode
reason = if ($process.ExitCode -eq 0) { "completed" } else { "process_failed" }
elapsedMs = $stopwatch.ElapsedMilliseconds
stdout = [string]$stdout
stderrPresent = [bool]$stderr
}
} catch {
return [pscustomobject]@{
ok = $false
exitCode = -1
reason = "transport_exception"
elapsedMs = $stopwatch.ElapsedMilliseconds
stdout = ""
stderrPresent = $false
errorType = $_.Exception.GetType().Name
}
} finally {
$stopwatch.Stop()
Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null
if ($process) { $process.Dispose() }
}
}
function Invoke-Agent99BoundedSsh {
param([string]$RemoteCommand, [int]$TimeoutSeconds, [string]$StandardInputPath = "")
$arguments = @(
"-i", $IdentityFile,
"-o", "IdentitiesOnly=yes",
"-o", "BatchMode=yes",
"-o", "NumberOfPasswordPrompts=0",
"-o", "StrictHostKeyChecking=yes",
"-o", "ConnectTimeout=8",
"-o", "ConnectionAttempts=1",
"-o", "ServerAliveInterval=10",
"-o", "ServerAliveCountMax=2",
"-l", $TargetUser,
$TargetHost,
$RemoteCommand
)
return Invoke-Agent99BoundedProcess "ssh.exe" $arguments $TimeoutSeconds "" $StandardInputPath
}
function Invoke-Agent99BoundedScp {
param([string[]]$LocalPaths, [string]$RemoteDirectory, [int]$TimeoutSeconds)
$arguments = @(
"-i", $IdentityFile,
"-o", "IdentitiesOnly=yes",
"-o", "BatchMode=yes",
"-o", "NumberOfPasswordPrompts=0",
"-o", "StrictHostKeyChecking=yes",
"-o", "ConnectTimeout=8",
"-o", "ConnectionAttempts=1"
)
$arguments += $LocalPaths
$arguments += "${TargetUser}@${TargetHost}:${RemoteDirectory}/"
return Invoke-Agent99BoundedProcess "scp.exe" $arguments $TimeoutSeconds
}
function Invoke-Agent99Git {
param([string[]]$Arguments, [int]$TimeoutSeconds, [string]$WorkingDirectory = "")
$previousPrompt = $env:GIT_TERMINAL_PROMPT
try {
$env:GIT_TERMINAL_PROMPT = "0"
return Invoke-Agent99BoundedProcess $GitPath $Arguments $TimeoutSeconds $WorkingDirectory
} finally {
$env:GIT_TERMINAL_PROMPT = $previousPrompt
}
}
function Assert-Agent99ProcessOk {
param([object]$Result, [string]$Reason)
if (-not $Result.ok) { throw $Reason }
}
function New-Agent99SourcePackage {
param([string]$SourceRoot, [string]$ManifestPath)
if (Test-Path -LiteralPath $SourceRoot) { throw "run_identity_source_stage_exists" }
$clone = Invoke-Agent99Git @(
"-c", "credential.helper=",
"-c", "core.autocrlf=false",
"clone", "--quiet", "--no-tags", "--depth", "200", "--branch", "main",
$GiteaSourceUrl, $SourceRoot
) 180 $SourceRootBase
Assert-Agent99ProcessOk $clone "gitea_source_fetch_failed"
$headResult = Invoke-Agent99Git @("rev-parse", "HEAD") 30 $SourceRoot
Assert-Agent99ProcessOk $headResult "gitea_source_head_failed"
$sourceHead = ([string]$headResult.stdout).Trim().ToLowerInvariant()
if ($sourceHead -notmatch "^[0-9a-f]{40}$") { throw "gitea_source_head_invalid" }
$commitResult = Invoke-Agent99Git @("cat-file", "-e", "$SourceRevision`^{commit}") 30 $SourceRoot
Assert-Agent99ProcessOk $commitResult "source_revision_not_fetched"
$ancestorResult = Invoke-Agent99Git @("merge-base", "--is-ancestor", $SourceRevision, $sourceHead) 30 $SourceRoot
Assert-Agent99ProcessOk $ancestorResult "source_revision_not_on_gitea_main"
$allPaths = @($RuntimeRelativePaths) + @($ExecutorRelativePath, $VerifierRelativePath)
$checkoutResult = Invoke-Agent99Git (@("-c", "core.autocrlf=false", "checkout", "--quiet", $SourceRevision, "--") + $allPaths) 60 $SourceRoot
Assert-Agent99ProcessOk $checkoutResult "source_revision_checkout_failed"
$rows = @()
$localPaths = @()
foreach ($relativePath in $RuntimeRelativePaths) {
$path = Join-Path $SourceRoot ($relativePath -replace "/", "\")
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { throw "runtime_source_file_missing" }
$rows += [ordered]@{
name = [IO.Path]::GetFileName($path)
sha256 = (Get-FileHash -LiteralPath $path -Algorithm SHA256).Hash.ToLowerInvariant()
}
$localPaths += $path
}
if (($rows.name | Sort-Object -Unique).Count -ne $ExpectedFileCount) { throw "runtime_source_file_set_invalid" }
$executorPath = Join-Path $SourceRoot ($ExecutorRelativePath -replace "/", "\")
$verifierPath = Join-Path $SourceRoot ($VerifierRelativePath -replace "/", "\")
if (-not (Test-Path -LiteralPath $executorPath -PathType Leaf)) { throw "executor_source_file_missing" }
if (-not (Test-Path -LiteralPath $verifierPath -PathType Leaf)) { throw "verifier_source_file_missing" }
$executorDigest = (Get-FileHash -LiteralPath $executorPath -Algorithm SHA256).Hash.ToLowerInvariant()
$verifierDigest = (Get-FileHash -LiteralPath $verifierPath -Algorithm SHA256).Hash.ToLowerInvariant()
$manifest = [ordered]@{
schemaVersion = "agent99_host110_backup_runtime_source_manifest_v1"
sourceRevision = $SourceRevision
sourceHead = $sourceHead
runId = $RunId
executorSha256 = $executorDigest
verifierSha256 = $verifierDigest
files = $rows
}
$encoding = New-Object System.Text.UTF8Encoding($false)
[IO.File]::WriteAllText($ManifestPath, ($manifest | ConvertTo-Json -Depth 8 -Compress), $encoding)
$localPaths += $executorPath, $verifierPath, $ManifestPath
return [pscustomobject]@{
sourceHead = $sourceHead
executorSha256 = $executorDigest
verifierSha256 = $verifierDigest
verifierPath = $verifierPath
manifestPath = $ManifestPath
manifestBase64 = [Convert]::ToBase64String([IO.File]::ReadAllBytes($ManifestPath))
localPaths = $localPaths
}
}
function Get-Agent99ExecutorResult {
param([object]$Transport, [string]$ExpectedMode)
if (-not $Transport.ok) { throw "host110_${ExpectedMode}_transport_failed" }
try {
$parsed = [string]$Transport.stdout | ConvertFrom-Json
} catch {
throw "host110_${ExpectedMode}_receipt_invalid"
}
if (
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_executor_v1" -or
[string]$parsed.mode -ne $ExpectedMode.ToLowerInvariant() -or
-not [bool]$parsed.ok -or
[string]$parsed.sourceRevision -ne $SourceRevision -or
[int]$parsed.fileCount -ne $ExpectedFileCount -or
[int]$parsed.backupScriptFileCount -ne 17 -or
-not [bool]$parsed.backupHealthExporterIncluded
) {
throw "host110_${ExpectedMode}_receipt_contract_failed"
}
if ($ExpectedMode -eq "Apply") {
if (
[string]$parsed.verifier.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
-not [bool]$parsed.verifier.ok -or
[bool]$parsed.verifier.remoteWritePerformed -or
-not [bool]$parsed.verifier.selfIdentityVerified
) {
throw "host110_apply_independent_verifier_contract_failed"
}
}
return $parsed
}
function Get-Agent99VerifierResult {
param([object]$Transport, [object]$SourcePackage)
if (-not $Transport.ok) { throw "host110_verify_transport_failed" }
try {
$parsed = [string]$Transport.stdout | ConvertFrom-Json
} catch {
throw "host110_verify_receipt_invalid"
}
if (
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
-not [bool]$parsed.ok -or
[string]$parsed.sourceRevision -ne $SourceRevision -or
[string]$parsed.runId -ne $RunId -or
[int]$parsed.fileCount -ne $ExpectedFileCount -or
[int]$parsed.backupScriptFileCount -ne 17 -or
-not [bool]$parsed.backupHealthExporterIncluded -or
[bool]$parsed.remoteWritePerformed -or
[string]$parsed.verifierSha256 -ne [string]$SourcePackage.verifierSha256
) {
throw "host110_verify_receipt_contract_failed"
}
return $parsed
}
function Invoke-Agent99ReadOnlyPreflight {
param([object]$SourcePackage)
$command = 'test "$(id -un)" = wooo && hostname -I | tr " " "\n" | grep -qx 192.168.0.110 && test -d /backup/scripts && test ! -L /backup/scripts && test -d /home/wooo/scripts && test ! -L /home/wooo/scripts && command -v timeout >/dev/null && command -v flock >/dev/null && command -v python3 >/dev/null && printf HOST110_BACKUP_PREFLIGHT_OK=1'
$transport = Invoke-Agent99BoundedSsh $command 45
if (-not $transport.ok -or ([string]$transport.stdout).Trim() -ne "HOST110_BACKUP_PREFLIGHT_OK=1") {
throw "host110_read_only_preflight_failed"
}
return [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_preflight_v1"
ok = $true
sourceRevision = $SourceRevision
sourceHead = [string]$SourcePackage.sourceHead
fileCount = $ExpectedFileCount
backupScriptFileCount = 17
backupHealthExporterIncluded = $true
remoteWritePerformed = $false
}
}
function Invoke-Agent99NoWriteVerifier {
param([object]$SourcePackage)
$command = "timeout --signal=TERM --kill-after=5s 60s python3 - --manifest-base64 $($SourcePackage.manifestBase64) --destination /backup/scripts --exporter-destination /home/wooo/scripts/backup-health-textfile-exporter.py --source-revision $SourceRevision --run-id $RunId --expected-verifier-sha256 $($SourcePackage.verifierSha256)"
$transport = Invoke-Agent99BoundedSsh $command 90 ([string]$SourcePackage.verifierPath)
return Get-Agent99VerifierResult $transport $SourcePackage
}
function Remove-Agent99RemoteStage {
param([string]$RemoteStage, [bool]$Required)
$cleanup = Invoke-Agent99BoundedSsh "rm -rf -- $RemoteStage && test ! -e $RemoteStage" 45
if ($Required -and -not $cleanup.ok) { throw "remote_source_stage_cleanup_failed" }
return [bool]$cleanup.ok
}
if ($AgentRoot.TrimEnd("\") -ne $ExpectedAgentRoot) { throw "agent_root_override_forbidden" }
$SourceRevision = $SourceRevision.ToLowerInvariant()
if ($SourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
if (-not (Test-Agent99SafeIdentity $RunId)) { throw "invalid_run_id" }
if (-not (Test-Path -LiteralPath $IdentityFile -PathType Leaf)) { throw "agent99_ssh_identity_unavailable" }
if (-not (Test-Path -LiteralPath $GitPath -PathType Leaf)) { throw "windows_git_unavailable" }
if (-not (Get-Command ssh.exe -ErrorAction SilentlyContinue)) { throw "windows_openssh_unavailable" }
if (-not (Get-Command scp.exe -ErrorAction SilentlyContinue)) { throw "windows_openscp_unavailable" }
New-Item -ItemType Directory -Force -Path $EvidenceRoot, $SourceRootBase | Out-Null
$evidencePath = Join-Path $EvidenceRoot "agent99-host110-backup-runtime-$RunId.json"
if (Test-Path -LiteralPath $evidencePath -PathType Leaf) {
try { $existing = Get-Content -LiteralPath $evidencePath -Raw | ConvertFrom-Json } catch { throw "run_identity_evidence_invalid" }
if ([string]$existing.sourceRevision -ne $SourceRevision -or [string]$existing.mode -ne $Mode) {
throw "run_identity_conflict"
}
$existing | ConvertTo-Json -Compress -Depth 12
if (-not [bool]$existing.ok) { exit 1 }
exit 0
}
$sourceRoot = Join-Path $SourceRootBase "host110-backup-source-$RunId"
$manifestPath = Join-Path $sourceRoot "agent99-host110-backup-runtime-manifest.json"
$remoteStage = "/tmp/agent99-host110-backup-runtime-$RunId"
$startedAt = [DateTime]::UtcNow
$mutex = [Threading.Mutex]::new($false, $TransportMutexName)
$acquired = $false
$remoteStageCreated = $false
$sourcePackage = $null
$check = $null
$apply = $null
$verify = $null
$terminalStatus = "failed"
$errorCode = ""
$cleanupVerified = $false
try {
try {
$acquired = [bool]$mutex.WaitOne(30000)
} catch [Threading.AbandonedMutexException] {
$acquired = $true
}
if (-not $acquired) { throw "host110_backup_runtime_transport_busy" }
$sourcePackage = New-Agent99SourcePackage $sourceRoot $manifestPath
$check = Invoke-Agent99ReadOnlyPreflight $sourcePackage
if ($Mode -eq "Apply") {
$createStage = Invoke-Agent99BoundedSsh "test ! -e $remoteStage && install -d -m 700 $remoteStage" 45
Assert-Agent99ProcessOk $createStage "remote_source_stage_identity_conflict"
$remoteStageCreated = $true
$copyStage = Invoke-Agent99BoundedScp $sourcePackage.localPaths $remoteStage 120
Assert-Agent99ProcessOk $copyStage "remote_source_stage_copy_failed"
$stageCheck = Invoke-Agent99BoundedSsh "chmod 700 $remoteStage/host110-backup-runtime-executor.sh $remoteStage/verify-host110-backup-runtime.py && test `$(find $remoteStage -mindepth 1 -maxdepth 1 -type f | wc -l) -eq $ExpectedRemoteStageFileCount" 45
Assert-Agent99ProcessOk $stageCheck "remote_source_stage_contract_failed"
$executorPath = "$remoteStage/host110-backup-runtime-executor.sh"
$command = "timeout --signal=TERM --kill-after=10s 300s $executorPath --mode apply --source-revision $SourceRevision --run-id $RunId --source-stage $remoteStage"
$applyTransport = Invoke-Agent99BoundedSsh $command 330
$apply = Get-Agent99ExecutorResult $applyTransport "Apply"
$verify = $apply.verifier
$cleanupVerified = Remove-Agent99RemoteStage $remoteStage $true
$remoteStageCreated = $false
$terminalStatus = "verified"
} elseif ($Mode -eq "Verify") {
$verify = Invoke-Agent99NoWriteVerifier $sourcePackage
$terminalStatus = "readback_ok"
$cleanupVerified = $true
} else {
$terminalStatus = "readback_ok"
$cleanupVerified = $true
}
} catch {
$errorCode = [string]$_.Exception.Message
} finally {
if ($remoteStageCreated) {
try { $cleanupVerified = Remove-Agent99RemoteStage $remoteStage $false } catch {}
}
if ($acquired) { try { $mutex.ReleaseMutex() } catch {} }
$mutex.Dispose()
}
$result = [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v2"
ok = [bool]($terminalStatus -ne "failed")
status = $terminalStatus
mode = $Mode
sourceRevision = $SourceRevision
sourceHead = if ($sourcePackage) { [string]$sourcePackage.sourceHead } else { "" }
runId = $RunId
targetHost = $TargetHost
sourceTransport = "windows99_gitea_exact_revision_manifest"
executor = "host110_backup_runtime_executor"
verifier = "independent_host110_backup_runtime_python_readback"
decisionProvider = "deterministic_only"
criticProvider = "deterministic_only"
agentAction = if ($Mode -eq "Apply") { "controlled_apply" } else { "readback" }
check = $check
apply = $apply
verify = $verify
cleanupVerified = $cleanupVerified
errorCode = $errorCode
elapsedSeconds = [math]::Round(([DateTime]::UtcNow - $startedAt).TotalSeconds, 3)
evidence = $evidencePath
secretValuesRead = $false
rawSessionStored = $false
}
try {
Write-Agent99ImmutableJson $evidencePath $result
} finally {
if (Test-Path -LiteralPath $sourceRoot) {
Remove-Item -LiteralPath $sourceRoot -Recurse -Force -ErrorAction SilentlyContinue
}
}
$result | ConvertTo-Json -Compress -Depth 12
if (-not $result.ok) { exit 1 }