From 4b2b73f1a7f168877adc4cd998ca8deaa392ca3a Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 21:21:05 +0800 Subject: [PATCH] feat(agent99): bind host110 backup runtime deploy --- agent99-bootstrap.ps1 | 1 + agent99-contract-check.ps1 | 23 + agent99-deploy.ps1 | 1 + agent99-host110-backup-runtime-broker.ps1 | 474 ++++++++++++++++++ .../backup/backup-offsite-readiness-gate.sh | 10 + scripts/backup/common.sh | 35 +- .../backup/gitea-full-backup-restore-drill.sh | 10 + .../backup/host110-backup-runtime-executor.sh | 431 ++++++++++++++++ .../backup/verify-host110-backup-runtime.py | 164 ++++++ .../backup/verify-host188-products-archive.py | 28 ++ ...oz_metadata_executor_runtime_entrypoint.py | 4 +- ..._db_bounded_executor_runtime_entrypoint.py | 8 +- .../agent99-live-preflight.ps1 | 2 +- .../agent99-remote-atomic-deploy-receiver.ps1 | 13 +- .../deploy-agent99-via-windows99-ssh.sh | 9 +- ...t_agent99_host110_backup_runtime_broker.py | 251 ++++++++++ .../test_agent99_live_preflight_decision.py | 4 +- ...remote_atomic_deploy_transport_contract.py | 5 +- .../test_host112_manager_recovery_contract.py | 3 +- 19 files changed, 1453 insertions(+), 23 deletions(-) create mode 100644 agent99-host110-backup-runtime-broker.ps1 create mode 100755 scripts/backup/host110-backup-runtime-executor.sh create mode 100755 scripts/backup/verify-host110-backup-runtime.py create mode 100644 scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 44d7482d8..a9d2b0359 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -224,6 +224,7 @@ $agentFiles = @( "agent99-control-plane.ps1", "agent99-db-bounded-executor.ps1", "agent99-signoz-metadata-executor.ps1", + "agent99-host110-backup-runtime-broker.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index 2a4c701e0..7748c73f4 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -20,6 +20,7 @@ $requiredFiles = @( "agent99-control-plane.ps1", "agent99-db-bounded-executor.ps1", "agent99-signoz-metadata-executor.ps1", + "agent99-host110-backup-runtime-broker.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", @@ -131,6 +132,7 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js $control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane.ps1"), [Text.Encoding]::UTF8) $dbBoundedExecutor = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-db-bounded-executor.ps1"), [Text.Encoding]::UTF8) $signozMetadataExecutor = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-signoz-metadata-executor.ps1"), [Text.Encoding]::UTF8) +$host110BackupRuntimeBroker = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-host110-backup-runtime-broker.ps1"), [Text.Encoding]::UTF8) $inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8) $telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8) $bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8) @@ -203,6 +205,27 @@ $signozMetadataExecutorContract = [bool]( -not $signozMetadataExecutor.Contains('Get-Content -LiteralPath $CredentialFile') ) Add-Check "backup:signoz_metadata_fixed_executor" $signozMetadataExecutorContract "Windows99 dispatches one fixed host110 metadata export contract, consumes only a root-owned credential reference on target, and requires the independent bundle verifier without raw SQLite or secret output" +$host110BackupRuntimeContract = [bool]( + $host110BackupRuntimeBroker.Contains('$TargetHost = "192.168.0.110"') -and + $host110BackupRuntimeBroker.Contains('$TargetUser = "wooo"') -and + $host110BackupRuntimeBroker.Contains('$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"') -and + $host110BackupRuntimeBroker.Contains('"credential.helper="') -and + $host110BackupRuntimeBroker.Contains('$env:GIT_TERMINAL_PROMPT = "0"') -and + $host110BackupRuntimeBroker.Contains('"merge-base", "--is-ancestor", $SourceRevision, $sourceHead') -and + $host110BackupRuntimeBroker.Contains('executorSha256 = $executorDigest') -and + $host110BackupRuntimeBroker.Contains('verifierSha256 = $verifierDigest') -and + $host110BackupRuntimeBroker.Contains('StrictHostKeyChecking=yes') -and + $host110BackupRuntimeBroker.Contains('$process.WaitForExit($TimeoutSeconds * 1000)') -and + $host110BackupRuntimeBroker.Contains('taskkill.exe /PID $process.Id /T /F') -and + $host110BackupRuntimeBroker.Contains('Invoke-Agent99BoundedScp $sourcePackage.localPaths') -and + $host110BackupRuntimeBroker.Contains('python3 - --manifest-base64') -and + $host110BackupRuntimeBroker.Contains('Write-Agent99ImmutableJson') -and + $host110BackupRuntimeBroker.Contains('decisionProvider = "deterministic_only"') -and + $host110BackupRuntimeBroker.Contains('criticProvider = "deterministic_only"') -and + -not $host110BackupRuntimeBroker.Contains('/home/wooo/awoooi') -and + -not $host110BackupRuntimeBroker.Contains('Invoke-Expression') +) +Add-Check "backup:host110_runtime_fixed_broker" $host110BackupRuntimeContract "Windows99 fetches one exact Gitea revision, binds executor and verifier digests, performs one fixed process-bounded Host110 transaction, and preserves an independent no-write verifier readback" Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("Enter-AgentRecoveryTriggerReservation") -and $control.Contains('"recovery-trigger-reservation.lock"') -and $control.Contains('[System.IO.FileShare]::None') -and $control.Contains('reason = "recovery_trigger_reservation_busy"') -and $control.Contains("recovery_already_queued")) "boot and host-down recovery reserve one atomic single-flight queue decision" Add-Check "recovery:auto_trigger_controlled_identity" ($control.Contains("New-AgentObservedRecoveryDispatchIdentity") -and $control.Contains('-RouteId "agent99_local_observer"') -and $control.Contains('schema_version = "agent99_controlled_dispatch_identity_v1"') -and $control.Contains('correlationKey = $dispatchIdentity.idempotencyKey') -and $control.Contains('canonicalDigest = $dispatchIdentity.canonicalDigest') -and $control.Contains("Invoke-AgentControlledDispatchIdentitySelfTest")) "observer-created Recover requests carry a stable canonical controlled dispatch identity before queue processing" Add-Check "recovery:queue_identity_fail_closed" ($control.Contains('dispatchIdentitySchemaVersion = $dispatchIdentity.schemaVersion') -and $control.Contains('Test-AgentControlledDispatchIdentity $dispatchIdentityEnvelope') -and $control.Contains('"controlled_dispatch_identity_invalid"') -and $control.IndexOf('"controlled_dispatch_identity_invalid"') -lt $control.IndexOf('$process = Start-Process -FilePath "powershell.exe" -ArgumentList $args')) "queue consumer rebuilds and validates canonical identity before any controlled executor starts" diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index 8096d7cf9..564801b90 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -35,6 +35,7 @@ $runtimeFiles = @( "agent99-control-plane.ps1", "agent99-db-bounded-executor.ps1", "agent99-signoz-metadata-executor.ps1", + "agent99-host110-backup-runtime-broker.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", diff --git a/agent99-host110-backup-runtime-broker.ps1 b/agent99-host110-backup-runtime-broker.ps1 new file mode 100644 index 000000000..8bccacbde --- /dev/null +++ b/agent99-host110-backup-runtime-broker.ps1 @@ -0,0 +1,474 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidateSet("Check", "Apply", "Verify")] + [string]$Mode, + [Parameter(Mandatory = $true)] + [string]$SourceRevision, + [Parameter(Mandatory = $true)] + [string]$RunId, + [string]$AgentRoot = "C:\Wooo\Agent99" +) + +$ErrorActionPreference = "Stop" +$ExpectedAgentRoot = "C:\Wooo\Agent99" +$TargetHost = "192.168.0.110" +$TargetUser = "wooo" +$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git" +$GitPath = "C:\Program Files\Git\cmd\git.exe" +$IdentityFile = Join-Path $ExpectedAgentRoot "keys\agent99_ed25519" +$EvidenceRoot = Join-Path $ExpectedAgentRoot "evidence\host110-backup-runtime" +$SourceRootBase = Join-Path $ExpectedAgentRoot "deploy" +$TransportMutexName = "Global\WoooAgent99Host110BackupRuntimeV2" +$ExpectedFileCount = 17 +$ExpectedRemoteStageFileCount = 20 +$RuntimeRelativePaths = @( + "scripts/backup/common.sh", + "scripts/backup/backup-all.sh", + "scripts/backup/backup-host188-products.sh", + "scripts/backup/verify-host188-products-backup.sh", + "scripts/backup/verify-host188-products-archive.py", + "scripts/backup/backup-gitea.sh", + "scripts/backup/gitea-full-backup-restore-drill.sh", + "scripts/backup/backup-configs.sh", + "scripts/backup/backup-awoooi.sh", + "scripts/backup/backup-awoooi-frequent.sh", + "scripts/backup/backup-clawbot.sh", + "scripts/backup/backup-sentry.sh", + "scripts/backup/check-backup-integrity.sh", + "scripts/backup/sync-offsite-backups.sh", + "scripts/backup/backup-offsite-readiness-gate.sh", + "scripts/backup/verify-offsite-full-sync.sh", + "scripts/backup/enforce-latest-only-retention.sh" +) +$ExecutorRelativePath = "scripts/backup/host110-backup-runtime-executor.sh" +$VerifierRelativePath = "scripts/backup/verify-host110-backup-runtime.py" + +function Test-Agent99SafeIdentity { + param([string]$Value) + return [bool]($Value -and $Value -match "^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$") +} + +function Write-Agent99ImmutableJson { + param([string]$Path, [object]$Value) + + if (Test-Path -LiteralPath $Path) { throw "run_identity_evidence_exists" } + $temporary = "$Path.tmp-$PID-$([Guid]::NewGuid().ToString('N'))" + $encoding = New-Object System.Text.UTF8Encoding($false) + try { + $json = $Value | ConvertTo-Json -Depth 12 + [IO.File]::WriteAllText($temporary, $json + [Environment]::NewLine, $encoding) + [IO.File]::Move($temporary, $Path) + } finally { + Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue | Out-Null + } +} + +function Invoke-Agent99BoundedProcess { + param( + [string]$FilePath, + [string[]]$Arguments, + [int]$TimeoutSeconds, + [string]$WorkingDirectory = "", + [string]$StandardInputPath = "" + ) + + $token = [Guid]::NewGuid().ToString("N") + $stdoutPath = Join-Path $env:TEMP "agent99-host110-backup-$token.out" + $stderrPath = Join-Path $env:TEMP "agent99-host110-backup-$token.err" + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + $process = $null + try { + $start = @{ + FilePath = $FilePath + ArgumentList = $Arguments + NoNewWindow = $true + RedirectStandardOutput = $stdoutPath + RedirectStandardError = $stderrPath + PassThru = $true + } + if ($WorkingDirectory) { $start.WorkingDirectory = $WorkingDirectory } + if ($StandardInputPath) { $start.RedirectStandardInput = $StandardInputPath } + $process = Start-Process @start + $null = $process.Handle + if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { + try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {} + if (-not $process.WaitForExit(2000) -and -not $process.HasExited) { + $process.Kill() + $process.WaitForExit(2000) | Out-Null + } + return [pscustomobject]@{ + ok = $false + exitCode = -2 + reason = "transport_timeout" + elapsedMs = $stopwatch.ElapsedMilliseconds + stdout = "" + stderrPresent = $false + } + } + $process.WaitForExit() | Out-Null + $process.Refresh() + $stdout = if (Test-Path -LiteralPath $stdoutPath) { Get-Content -LiteralPath $stdoutPath -Raw } else { "" } + $stderr = if (Test-Path -LiteralPath $stderrPath) { Get-Content -LiteralPath $stderrPath -Raw } else { "" } + if ($stdout.Length -gt 65536 -or $stderr.Length -gt 65536) { + return [pscustomobject]@{ + ok = $false + exitCode = -3 + reason = "transport_output_oversized" + elapsedMs = $stopwatch.ElapsedMilliseconds + stdout = "" + stderrPresent = [bool]$stderr + } + } + return [pscustomobject]@{ + ok = [bool]($process.ExitCode -eq 0) + exitCode = [int]$process.ExitCode + reason = if ($process.ExitCode -eq 0) { "completed" } else { "process_failed" } + elapsedMs = $stopwatch.ElapsedMilliseconds + stdout = [string]$stdout + stderrPresent = [bool]$stderr + } + } catch { + return [pscustomobject]@{ + ok = $false + exitCode = -1 + reason = "transport_exception" + elapsedMs = $stopwatch.ElapsedMilliseconds + stdout = "" + stderrPresent = $false + errorType = $_.Exception.GetType().Name + } + } finally { + $stopwatch.Stop() + Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null + if ($process) { $process.Dispose() } + } +} + +function Invoke-Agent99BoundedSsh { + param([string]$RemoteCommand, [int]$TimeoutSeconds, [string]$StandardInputPath = "") + + $arguments = @( + "-i", $IdentityFile, + "-o", "IdentitiesOnly=yes", + "-o", "BatchMode=yes", + "-o", "NumberOfPasswordPrompts=0", + "-o", "StrictHostKeyChecking=yes", + "-o", "ConnectTimeout=8", + "-o", "ConnectionAttempts=1", + "-o", "ServerAliveInterval=10", + "-o", "ServerAliveCountMax=2", + "-l", $TargetUser, + $TargetHost, + $RemoteCommand + ) + return Invoke-Agent99BoundedProcess "ssh.exe" $arguments $TimeoutSeconds "" $StandardInputPath +} + +function Invoke-Agent99BoundedScp { + param([string[]]$LocalPaths, [string]$RemoteDirectory, [int]$TimeoutSeconds) + + $arguments = @( + "-i", $IdentityFile, + "-o", "IdentitiesOnly=yes", + "-o", "BatchMode=yes", + "-o", "NumberOfPasswordPrompts=0", + "-o", "StrictHostKeyChecking=yes", + "-o", "ConnectTimeout=8", + "-o", "ConnectionAttempts=1" + ) + $arguments += $LocalPaths + $arguments += "${TargetUser}@${TargetHost}:${RemoteDirectory}/" + return Invoke-Agent99BoundedProcess "scp.exe" $arguments $TimeoutSeconds +} + +function Invoke-Agent99Git { + param([string[]]$Arguments, [int]$TimeoutSeconds, [string]$WorkingDirectory = "") + + $previousPrompt = $env:GIT_TERMINAL_PROMPT + try { + $env:GIT_TERMINAL_PROMPT = "0" + return Invoke-Agent99BoundedProcess $GitPath $Arguments $TimeoutSeconds $WorkingDirectory + } finally { + $env:GIT_TERMINAL_PROMPT = $previousPrompt + } +} + +function Assert-Agent99ProcessOk { + param([object]$Result, [string]$Reason) + if (-not $Result.ok) { throw $Reason } +} + +function New-Agent99SourcePackage { + param([string]$SourceRoot, [string]$ManifestPath) + + if (Test-Path -LiteralPath $SourceRoot) { throw "run_identity_source_stage_exists" } + $clone = Invoke-Agent99Git @( + "-c", "credential.helper=", + "-c", "core.autocrlf=false", + "clone", "--quiet", "--no-tags", "--depth", "200", "--branch", "main", + $GiteaSourceUrl, $SourceRoot + ) 180 $SourceRootBase + Assert-Agent99ProcessOk $clone "gitea_source_fetch_failed" + + $headResult = Invoke-Agent99Git @("rev-parse", "HEAD") 30 $SourceRoot + Assert-Agent99ProcessOk $headResult "gitea_source_head_failed" + $sourceHead = ([string]$headResult.stdout).Trim().ToLowerInvariant() + if ($sourceHead -notmatch "^[0-9a-f]{40}$") { throw "gitea_source_head_invalid" } + + $commitResult = Invoke-Agent99Git @("cat-file", "-e", "$SourceRevision`^{commit}") 30 $SourceRoot + Assert-Agent99ProcessOk $commitResult "source_revision_not_fetched" + $ancestorResult = Invoke-Agent99Git @("merge-base", "--is-ancestor", $SourceRevision, $sourceHead) 30 $SourceRoot + Assert-Agent99ProcessOk $ancestorResult "source_revision_not_on_gitea_main" + + $allPaths = @($RuntimeRelativePaths) + @($ExecutorRelativePath, $VerifierRelativePath) + $checkoutResult = Invoke-Agent99Git (@("-c", "core.autocrlf=false", "checkout", "--quiet", $SourceRevision, "--") + $allPaths) 60 $SourceRoot + Assert-Agent99ProcessOk $checkoutResult "source_revision_checkout_failed" + + $rows = @() + $localPaths = @() + foreach ($relativePath in $RuntimeRelativePaths) { + $path = Join-Path $SourceRoot ($relativePath -replace "/", "\") + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { throw "runtime_source_file_missing" } + $rows += [ordered]@{ + name = [IO.Path]::GetFileName($path) + sha256 = (Get-FileHash -LiteralPath $path -Algorithm SHA256).Hash.ToLowerInvariant() + } + $localPaths += $path + } + if (($rows.name | Sort-Object -Unique).Count -ne $ExpectedFileCount) { throw "runtime_source_file_set_invalid" } + + $executorPath = Join-Path $SourceRoot ($ExecutorRelativePath -replace "/", "\") + $verifierPath = Join-Path $SourceRoot ($VerifierRelativePath -replace "/", "\") + if (-not (Test-Path -LiteralPath $executorPath -PathType Leaf)) { throw "executor_source_file_missing" } + if (-not (Test-Path -LiteralPath $verifierPath -PathType Leaf)) { throw "verifier_source_file_missing" } + $executorDigest = (Get-FileHash -LiteralPath $executorPath -Algorithm SHA256).Hash.ToLowerInvariant() + $verifierDigest = (Get-FileHash -LiteralPath $verifierPath -Algorithm SHA256).Hash.ToLowerInvariant() + $manifest = [ordered]@{ + schemaVersion = "agent99_host110_backup_runtime_source_manifest_v1" + sourceRevision = $SourceRevision + sourceHead = $sourceHead + runId = $RunId + executorSha256 = $executorDigest + verifierSha256 = $verifierDigest + files = $rows + } + $encoding = New-Object System.Text.UTF8Encoding($false) + [IO.File]::WriteAllText($ManifestPath, ($manifest | ConvertTo-Json -Depth 8 -Compress), $encoding) + $localPaths += $executorPath, $verifierPath, $ManifestPath + return [pscustomobject]@{ + sourceHead = $sourceHead + executorSha256 = $executorDigest + verifierSha256 = $verifierDigest + verifierPath = $verifierPath + manifestPath = $ManifestPath + manifestBase64 = [Convert]::ToBase64String([IO.File]::ReadAllBytes($ManifestPath)) + localPaths = $localPaths + } +} + +function Get-Agent99ExecutorResult { + param([object]$Transport, [string]$ExpectedMode) + + if (-not $Transport.ok) { throw "host110_${ExpectedMode}_transport_failed" } + try { + $parsed = [string]$Transport.stdout | ConvertFrom-Json + } catch { + throw "host110_${ExpectedMode}_receipt_invalid" + } + if ( + [string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_executor_v1" -or + [string]$parsed.mode -ne $ExpectedMode.ToLowerInvariant() -or + -not [bool]$parsed.ok -or + [string]$parsed.sourceRevision -ne $SourceRevision -or + [int]$parsed.fileCount -ne $ExpectedFileCount + ) { + throw "host110_${ExpectedMode}_receipt_contract_failed" + } + if ($ExpectedMode -eq "Apply") { + if ( + [string]$parsed.verifier.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or + -not [bool]$parsed.verifier.ok -or + [bool]$parsed.verifier.remoteWritePerformed -or + -not [bool]$parsed.verifier.selfIdentityVerified + ) { + throw "host110_apply_independent_verifier_contract_failed" + } + } + return $parsed +} + +function Get-Agent99VerifierResult { + param([object]$Transport, [object]$SourcePackage) + + if (-not $Transport.ok) { throw "host110_verify_transport_failed" } + try { + $parsed = [string]$Transport.stdout | ConvertFrom-Json + } catch { + throw "host110_verify_receipt_invalid" + } + if ( + [string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or + -not [bool]$parsed.ok -or + [string]$parsed.sourceRevision -ne $SourceRevision -or + [string]$parsed.runId -ne $RunId -or + [int]$parsed.fileCount -ne $ExpectedFileCount -or + [bool]$parsed.remoteWritePerformed -or + [string]$parsed.verifierSha256 -ne [string]$SourcePackage.verifierSha256 + ) { + throw "host110_verify_receipt_contract_failed" + } + return $parsed +} + +function Invoke-Agent99ReadOnlyPreflight { + param([object]$SourcePackage) + + $command = 'test "$(id -un)" = wooo && hostname -I | tr " " "\n" | grep -qx 192.168.0.110 && test -d /backup/scripts && test ! -L /backup/scripts && command -v timeout >/dev/null && command -v flock >/dev/null && command -v python3 >/dev/null && printf HOST110_BACKUP_PREFLIGHT_OK=1' + $transport = Invoke-Agent99BoundedSsh $command 45 + if (-not $transport.ok -or ([string]$transport.stdout).Trim() -ne "HOST110_BACKUP_PREFLIGHT_OK=1") { + throw "host110_read_only_preflight_failed" + } + return [pscustomobject]@{ + schemaVersion = "agent99_host110_backup_runtime_preflight_v1" + ok = $true + sourceRevision = $SourceRevision + sourceHead = [string]$SourcePackage.sourceHead + fileCount = $ExpectedFileCount + remoteWritePerformed = $false + } +} + +function Invoke-Agent99NoWriteVerifier { + param([object]$SourcePackage) + + $command = "timeout --signal=TERM --kill-after=5s 60s python3 - --manifest-base64 $($SourcePackage.manifestBase64) --destination /backup/scripts --source-revision $SourceRevision --run-id $RunId --expected-verifier-sha256 $($SourcePackage.verifierSha256)" + $transport = Invoke-Agent99BoundedSsh $command 90 ([string]$SourcePackage.verifierPath) + return Get-Agent99VerifierResult $transport $SourcePackage +} + +function Remove-Agent99RemoteStage { + param([string]$RemoteStage, [bool]$Required) + + $cleanup = Invoke-Agent99BoundedSsh "rm -rf -- $RemoteStage && test ! -e $RemoteStage" 45 + if ($Required -and -not $cleanup.ok) { throw "remote_source_stage_cleanup_failed" } + return [bool]$cleanup.ok +} + +if ($AgentRoot.TrimEnd("\") -ne $ExpectedAgentRoot) { throw "agent_root_override_forbidden" } +$SourceRevision = $SourceRevision.ToLowerInvariant() +if ($SourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" } +if (-not (Test-Agent99SafeIdentity $RunId)) { throw "invalid_run_id" } +if (-not (Test-Path -LiteralPath $IdentityFile -PathType Leaf)) { throw "agent99_ssh_identity_unavailable" } +if (-not (Test-Path -LiteralPath $GitPath -PathType Leaf)) { throw "windows_git_unavailable" } +if (-not (Get-Command ssh.exe -ErrorAction SilentlyContinue)) { throw "windows_openssh_unavailable" } +if (-not (Get-Command scp.exe -ErrorAction SilentlyContinue)) { throw "windows_openscp_unavailable" } + +New-Item -ItemType Directory -Force -Path $EvidenceRoot, $SourceRootBase | Out-Null +$evidencePath = Join-Path $EvidenceRoot "agent99-host110-backup-runtime-$RunId.json" +if (Test-Path -LiteralPath $evidencePath -PathType Leaf) { + try { $existing = Get-Content -LiteralPath $evidencePath -Raw | ConvertFrom-Json } catch { throw "run_identity_evidence_invalid" } + if ([string]$existing.sourceRevision -ne $SourceRevision -or [string]$existing.mode -ne $Mode) { + throw "run_identity_conflict" + } + $existing | ConvertTo-Json -Compress -Depth 12 + if (-not [bool]$existing.ok) { exit 1 } + exit 0 +} + +$sourceRoot = Join-Path $SourceRootBase "host110-backup-source-$RunId" +$manifestPath = Join-Path $sourceRoot "agent99-host110-backup-runtime-manifest.json" +$remoteStage = "/tmp/agent99-host110-backup-runtime-$RunId" +$startedAt = [DateTime]::UtcNow +$mutex = [Threading.Mutex]::new($false, $TransportMutexName) +$acquired = $false +$remoteStageCreated = $false +$sourcePackage = $null +$check = $null +$apply = $null +$verify = $null +$terminalStatus = "failed" +$errorCode = "" +$cleanupVerified = $false + +try { + try { + $acquired = [bool]$mutex.WaitOne(30000) + } catch [Threading.AbandonedMutexException] { + $acquired = $true + } + if (-not $acquired) { throw "host110_backup_runtime_transport_busy" } + + $sourcePackage = New-Agent99SourcePackage $sourceRoot $manifestPath + $check = Invoke-Agent99ReadOnlyPreflight $sourcePackage + + if ($Mode -eq "Apply") { + $createStage = Invoke-Agent99BoundedSsh "test ! -e $remoteStage && install -d -m 700 $remoteStage" 45 + Assert-Agent99ProcessOk $createStage "remote_source_stage_identity_conflict" + $remoteStageCreated = $true + $copyStage = Invoke-Agent99BoundedScp $sourcePackage.localPaths $remoteStage 120 + Assert-Agent99ProcessOk $copyStage "remote_source_stage_copy_failed" + $stageCheck = Invoke-Agent99BoundedSsh "chmod 700 $remoteStage/host110-backup-runtime-executor.sh $remoteStage/verify-host110-backup-runtime.py && test `$(find $remoteStage -mindepth 1 -maxdepth 1 -type f | wc -l) -eq $ExpectedRemoteStageFileCount" 45 + Assert-Agent99ProcessOk $stageCheck "remote_source_stage_contract_failed" + + $executorPath = "$remoteStage/host110-backup-runtime-executor.sh" + $command = "timeout --signal=TERM --kill-after=10s 300s $executorPath --mode apply --source-revision $SourceRevision --run-id $RunId --source-stage $remoteStage" + $applyTransport = Invoke-Agent99BoundedSsh $command 330 + $apply = Get-Agent99ExecutorResult $applyTransport "Apply" + $verify = $apply.verifier + $cleanupVerified = Remove-Agent99RemoteStage $remoteStage $true + $remoteStageCreated = $false + $terminalStatus = "verified" + } elseif ($Mode -eq "Verify") { + $verify = Invoke-Agent99NoWriteVerifier $sourcePackage + $terminalStatus = "readback_ok" + $cleanupVerified = $true + } else { + $terminalStatus = "readback_ok" + $cleanupVerified = $true + } +} catch { + $errorCode = [string]$_.Exception.Message +} finally { + if ($remoteStageCreated) { + try { $cleanupVerified = Remove-Agent99RemoteStage $remoteStage $false } catch {} + } + if ($acquired) { try { $mutex.ReleaseMutex() } catch {} } + $mutex.Dispose() +} + +$result = [pscustomobject]@{ + schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v2" + ok = [bool]($terminalStatus -ne "failed") + status = $terminalStatus + mode = $Mode + sourceRevision = $SourceRevision + sourceHead = if ($sourcePackage) { [string]$sourcePackage.sourceHead } else { "" } + runId = $RunId + targetHost = $TargetHost + sourceTransport = "windows99_gitea_exact_revision_manifest" + executor = "host110_backup_runtime_executor" + verifier = "independent_host110_backup_runtime_python_readback" + decisionProvider = "deterministic_only" + criticProvider = "deterministic_only" + agentAction = if ($Mode -eq "Apply") { "controlled_apply" } else { "readback" } + check = $check + apply = $apply + verify = $verify + cleanupVerified = $cleanupVerified + errorCode = $errorCode + elapsedSeconds = [math]::Round(([DateTime]::UtcNow - $startedAt).TotalSeconds, 3) + evidence = $evidencePath + secretValuesRead = $false + rawSessionStored = $false +} + +try { + Write-Agent99ImmutableJson $evidencePath $result +} finally { + if (Test-Path -LiteralPath $sourceRoot) { + Remove-Item -LiteralPath $sourceRoot -Recurse -Force -ErrorAction SilentlyContinue + } +} +$result | ConvertTo-Json -Compress -Depth 12 +if (-not $result.ok) { exit 1 } diff --git a/scripts/backup/backup-offsite-readiness-gate.sh b/scripts/backup/backup-offsite-readiness-gate.sh index a260286e2..f793d9850 100755 --- a/scripts/backup/backup-offsite-readiness-gate.sh +++ b/scripts/backup/backup-offsite-readiness-gate.sh @@ -9,6 +9,16 @@ set -euo pipefail +if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then + command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; } + [ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; } + (umask 077; : >> /tmp/agent99-host110-backup-runtime.lock) + [ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; } + exec 197>>/tmp/agent99-host110-backup-runtime.lock + flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; } + export BACKUP_RUNTIME_SHARED_LOCK_HELD=1 +fi + BACKUP_BASE="${BACKUP_BASE:-/backup}" OFFSITE_ENV_FILE="${BACKUP_OFFSITE_ENV_FILE:-${BACKUP_BASE}/scripts/offsite.env}" OFFSITE_DIR="${BACKUP_OFFSITE_STATUS_DIR:-${BACKUP_BASE}/offsite}" diff --git a/scripts/backup/common.sh b/scripts/backup/common.sh index 79c56acce..fbeaadaf4 100644 --- a/scripts/backup/common.sh +++ b/scripts/backup/common.sh @@ -1,5 +1,38 @@ -#\!/bin/bash +#!/bin/bash # ============================================================================= + +# Production backup entrypoints share this stable gate. Agent99 takes the +# exclusive side while promoting a complete, digest-bound runtime bundle. +acquire_backup_runtime_shared_lock() { + local caller_path="${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}" + local lock_path="/tmp/agent99-host110-backup-runtime.lock" + case "${caller_path}" in + /backup/scripts/*) ;; + *) return 0 ;; + esac + [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ] || return 0 + command -v flock >/dev/null 2>&1 || { + echo "backup runtime gate unavailable: flock missing" >&2 + return 69 + } + [ ! -L "${lock_path}" ] || { + echo "backup runtime gate unsafe: lock is a symlink" >&2 + return 69 + } + (umask 077; : >> "${lock_path}") || return 69 + [ -f "${lock_path}" ] && [ -O "${lock_path}" ] || return 69 + exec 197>>"${lock_path}" + flock -s -w 60 197 || { + echo "backup runtime deployment is active; retry later" >&2 + return 75 + } + export BACKUP_RUNTIME_SHARED_LOCK_HELD=1 +} + +acquire_backup_runtime_shared_lock || { + runtime_lock_status=$? + return "${runtime_lock_status}" 2>/dev/null || exit "${runtime_lock_status}" +} # WOOO AIOps - 備份共用函式庫 # 版本: 1.0.0 # 建立日期: 2026-03-12 diff --git a/scripts/backup/gitea-full-backup-restore-drill.sh b/scripts/backup/gitea-full-backup-restore-drill.sh index e84e1222d..13415e1c6 100755 --- a/scripts/backup/gitea-full-backup-restore-drill.sh +++ b/scripts/backup/gitea-full-backup-restore-drill.sh @@ -5,6 +5,16 @@ set -euo pipefail +if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then + command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; } + [ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; } + (umask 077; : >> /tmp/agent99-host110-backup-runtime.lock) + [ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; } + exec 197>>/tmp/agent99-host110-backup-runtime.lock + flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; } + export BACKUP_RUNTIME_SHARED_LOCK_HELD=1 +fi + DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}" HOST_LABEL="${AIOPS_HOST_LABEL:-110}" TEXTFILE_PATH="${AIOPS_GITEA_FULL_BACKUP_RESTORE_DRILL_TEXTFILE:-/home/wooo/node_exporter_textfiles/gitea_full_backup_restore_drill.prom}" diff --git a/scripts/backup/host110-backup-runtime-executor.sh b/scripts/backup/host110-backup-runtime-executor.sh new file mode 100755 index 000000000..e1ef53614 --- /dev/null +++ b/scripts/backup/host110-backup-runtime-executor.sh @@ -0,0 +1,431 @@ +#!/usr/bin/env bash +# Fixed Host110 executor for Windows99 Agent99 backup-runtime deployment. + +set -euo pipefail +umask 077 + +readonly EXPECTED_HOST_IP="192.168.0.110" +readonly EXPECTED_USER="wooo" +readonly DEST_ROOT="/backup/scripts" +readonly STATUS_ROOT="/backup/status" +readonly STAGE_ROOT="/backup/.agent99-backup-runtime-stage" +readonly ROLLBACK_ROOT="/backup/.agent99-backup-runtime-rollback" +readonly LOCK_PATH="/tmp/agent99-host110-backup-runtime.lock" + +readonly -a RUNTIME_FILES=( + common.sh + backup-all.sh + backup-host188-products.sh + verify-host188-products-backup.sh + verify-host188-products-archive.py + backup-gitea.sh + gitea-full-backup-restore-drill.sh + backup-configs.sh + backup-awoooi.sh + backup-awoooi-frequent.sh + backup-clawbot.sh + backup-sentry.sh + check-backup-integrity.sh + sync-offsite-backups.sh + backup-offsite-readiness-gate.sh + verify-offsite-full-sync.sh + enforce-latest-only-retention.sh +) + +MODE="" +SOURCE_REVISION="" +RUN_ID="" +SOURCE_STAGE="" +SOURCE_HEAD="" +EXECUTOR_DIGEST="" +VERIFIER_DIGEST="" +STAGE_DIR="" +ROLLBACK_DIR="" +APPLY_STARTED=0 +APPLY_COMPLETE=0 +ROLLBACK_VERIFIED=0 +declare -A FILE_EXISTED=() +declare -A PREVIOUS_DIGEST=() +declare -A EXPECTED_DIGEST=() + +usage() { + printf '%s\n' \ + "Usage: $0 --mode check|apply|verify --source-revision SHA --run-id ID --source-stage PATH" \ + "The executor is fixed to host 192.168.0.110, an Agent99 run-bound source stage," \ + "and the 17-file /backup/scripts transaction." +} + +fail() { + printf 'HOST110_BACKUP_RUNTIME_OK=0\nERROR=%s\n' "$1" >&2 + exit 64 +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --mode) + shift + MODE="${1:-}" + ;; + --source-revision) + shift + SOURCE_REVISION="${1:-}" + ;; + --run-id) + shift + RUN_ID="${1:-}" + ;; + --source-stage) + shift + SOURCE_STAGE="${1:-}" + ;; + -h|--help) + usage + exit 0 + ;; + *) fail "unknown_argument" ;; + esac + shift +done + +case "$MODE" in check|apply|verify) ;; *) fail "invalid_mode" ;; esac +[[ "$SOURCE_REVISION" =~ ^[0-9a-f]{40}$ ]] || fail "invalid_source_revision" +[[ "$RUN_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$ ]] || fail "invalid_run_id" +expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}" +[ "$SOURCE_STAGE" = "$expected_source_stage" ] || fail "invalid_source_stage" + +for command_name in awk bash cp flock grep hostname id install mv pgrep python3 readlink rm sha256sum stat timeout tr; do + command -v "$command_name" >/dev/null 2>&1 || fail "required_command_missing_${command_name}" +done + +current_user="$(id -un)" +[ "$current_user" = "$EXPECTED_USER" ] || fail "unexpected_executor_user" +hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx "$EXPECTED_HOST_IP" || fail "unexpected_executor_host" +[ -d "$SOURCE_STAGE" ] && [ ! -L "$SOURCE_STAGE" ] || fail "canonical_source_stage_unavailable" +[ -d "$DEST_ROOT" ] && [ ! -L "$DEST_ROOT" ] || fail "backup_runtime_destination_unavailable" + +manifest_rows="$(python3 - "$SOURCE_STAGE/manifest.json" "$SOURCE_REVISION" "$RUN_ID" "${RUNTIME_FILES[@]}" <<'PY' +import json +import re +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +source_revision = sys.argv[2] +run_id = sys.argv[3] +expected_names = sys.argv[4:] +try: + document = json.loads(path.read_text(encoding="utf-8")) +except (OSError, json.JSONDecodeError): + raise SystemExit(65) +rows = document.get("files") +if ( + document.get("schemaVersion") != "agent99_host110_backup_runtime_source_manifest_v1" + or document.get("sourceRevision") != source_revision + or document.get("runId") != run_id + or not re.fullmatch(r"[0-9a-f]{40}", str(document.get("sourceHead", ""))) + or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("executorSha256", ""))) + or not re.fullmatch(r"[0-9a-f]{64}", str(document.get("verifierSha256", ""))) + or not isinstance(rows, list) + or len(rows) != len(expected_names) +): + raise SystemExit(66) +by_name = {} +for row in rows: + if not isinstance(row, dict): + raise SystemExit(66) + name = str(row.get("name", "")) + digest = str(row.get("sha256", "")) + if name in by_name or not re.fullmatch(r"[A-Za-z0-9_.-]+", name) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise SystemExit(66) + by_name[name] = digest +if set(by_name) != set(expected_names): + raise SystemExit(66) +print(f"SOURCE_HEAD {document['sourceHead']}") +print(f"EXECUTOR_SHA256 {document['executorSha256']}") +print(f"VERIFIER_SHA256 {document['verifierSha256']}") +for name in expected_names: + print(f"{name} {by_name[name]}") +PY +)" || fail "source_manifest_invalid" + +while read -r manifest_name manifest_digest; do + case "$manifest_name" in + SOURCE_HEAD) SOURCE_HEAD="$manifest_digest" ;; + EXECUTOR_SHA256) EXECUTOR_DIGEST="$manifest_digest" ;; + VERIFIER_SHA256) VERIFIER_DIGEST="$manifest_digest" ;; + *) EXPECTED_DIGEST[$manifest_name]="$manifest_digest" ;; + esac +done <<< "$manifest_rows" +[[ "$SOURCE_HEAD" =~ ^[0-9a-f]{40}$ ]] || fail "source_head_invalid" + +working_digest() { + sha256sum "$1" | awk '{print $1}' +} + +[ "$(working_digest "$0")" = "$EXECUTOR_DIGEST" ] || fail "executor_identity_failed" +[ -f "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \ + && [ ! -L "$SOURCE_STAGE/verify-host110-backup-runtime.py" ] \ + && [ "$(working_digest "$SOURCE_STAGE/verify-host110-backup-runtime.py")" = "$VERIFIER_DIGEST" ] \ + || fail "verifier_identity_failed" + +validate_file() { + local path="$1" + case "$path" in + *.sh) bash -n "$path" ;; + *.py) python3 -c 'import ast, pathlib, sys; ast.parse(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))' "$path" ;; + *) return 64 ;; + esac +} + +source_digest() { + local name="$1" + printf '%s\n' "${EXPECTED_DIGEST[$name]:-}" +} + +validate_source() { + local name source_path expected_digest actual_digest resolved + for name in "${RUNTIME_FILES[@]}"; do + source_path="$SOURCE_STAGE/$name" + [ -f "$source_path" ] && [ ! -L "$source_path" ] || return 65 + resolved="$(readlink -f "$source_path")" + [[ "$resolved" == "$SOURCE_STAGE/"* ]] || return 65 + expected_digest="$(source_digest "$name")" + actual_digest="$(working_digest "$source_path")" + [ "$actual_digest" = "$expected_digest" ] || return 66 + validate_file "$source_path" || return 67 + done +} + +verify_destination() { + local name dest_path expected_digest actual_digest + for name in "${RUNTIME_FILES[@]}"; do + dest_path="$DEST_ROOT/$name" + [ -f "$dest_path" ] && [ ! -L "$dest_path" ] || return 68 + expected_digest="$(source_digest "$name")" + actual_digest="$(working_digest "$dest_path")" + [ "$actual_digest" = "$expected_digest" ] || return 69 + [ "$(stat -c '%U:%G:%a' "$dest_path")" = "wooo:wooo:755" ] || return 70 + done +} + +write_receipt() { + local status="$1" + local rollback_verified="$2" + local receipt_path="$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json" + install -d -m 700 "$STATUS_ROOT" + python3 - "$receipt_path" "$status" "$SOURCE_REVISION" "$SOURCE_HEAD" "$RUN_ID" "$rollback_verified" "$VERIFIER_DIGEST" <<'PY' +import json +import os +import sys +import time +from pathlib import Path + +path = Path(sys.argv[1]) +document = { + "schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v1", + "status": sys.argv[2], + "sourceRevision": sys.argv[3], + "sourceHead": sys.argv[4], + "runId": sys.argv[5], + "fileCount": 17, + "rollbackPerformed": sys.argv[6] == "1", + "rollbackVerified": sys.argv[6] == "1", + "verifierSha256": sys.argv[7], + "executorHost": "192.168.0.110", + "productionServiceRestarted": False, + "secretValuesRead": False, + "writtenAt": int(time.time()), +} +temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}") +temporary.write_text(json.dumps(document, ensure_ascii=True, sort_keys=True) + "\n", encoding="utf-8") +os.chmod(temporary, 0o600) +try: + os.link(temporary, path) +finally: + temporary.unlink(missing_ok=True) +PY + printf '%s' "$receipt_path" +} + +record_prestate() { + local name dest_path digest + : > "$ROLLBACK_DIR/prestate.tsv" + chmod 600 "$ROLLBACK_DIR/prestate.tsv" + for name in "${RUNTIME_FILES[@]}"; do + dest_path="$DEST_ROOT/$name" + if [ -f "$dest_path" ] && [ ! -L "$dest_path" ]; then + [ "$(stat -c '%U:%G' "$dest_path")" = "wooo:wooo" ] || return 71 + FILE_EXISTED[$name]=1 + digest="$(working_digest "$dest_path")" + PREVIOUS_DIGEST[$name]="$digest" + printf '%s\t1\t%s\n' "$name" "$digest" >> "$ROLLBACK_DIR/prestate.tsv" + cp -p -- "$dest_path" "$ROLLBACK_DIR/$name" + [ "$(working_digest "$ROLLBACK_DIR/$name")" = "$digest" ] || return 72 + elif [ ! -e "$dest_path" ]; then + FILE_EXISTED[$name]=0 + PREVIOUS_DIGEST[$name]="-" + printf '%s\t0\t-\n' "$name" >> "$ROLLBACK_DIR/prestate.tsv" + else + return 73 + fi + done +} + +rollback_transaction() { + local name dest_path backup_path temporary failed=0 + set +e + for name in "${RUNTIME_FILES[@]}"; do + [ -n "${FILE_EXISTED[$name]+x}" ] || { failed=1; continue; } + dest_path="$DEST_ROOT/$name" + if [ "${FILE_EXISTED[$name]}" = "1" ]; then + backup_path="$ROLLBACK_DIR/$name" + temporary="$DEST_ROOT/.${name}.agent99-rollback-${RUN_ID}" + cp -p -- "$backup_path" "$temporary" \ + && mv -f -- "$temporary" "$dest_path" \ + || failed=1 + else + rm -f -- "$dest_path" || failed=1 + fi + done + for name in "${RUNTIME_FILES[@]}"; do + dest_path="$DEST_ROOT/$name" + if [ "${FILE_EXISTED[$name]:-}" = "1" ]; then + [ -f "$dest_path" ] && [ ! -L "$dest_path" ] \ + && [ "$(working_digest "$dest_path" 2>/dev/null)" = "${PREVIOUS_DIGEST[$name]:-invalid}" ] \ + || failed=1 + elif [ "${FILE_EXISTED[$name]:-}" = "0" ]; then + [ ! -e "$dest_path" ] || failed=1 + else + failed=1 + fi + done + set -e + if [ "$failed" -eq 0 ]; then + ROLLBACK_VERIFIED=1 + return 0 + fi + ROLLBACK_VERIFIED=0 + return 1 +} + +cleanup() { + local status=$? + local rollback_status="rollback_unverified" + trap - EXIT HUP INT TERM + if [ "$APPLY_STARTED" -eq 1 ] && [ "$APPLY_COMPLETE" -ne 1 ]; then + if rollback_transaction; then + rollback_status="failed_rolled_back" + fi + write_receipt "$rollback_status" "$ROLLBACK_VERIFIED" >/dev/null || true + fi + [ -z "$STAGE_DIR" ] || rm -rf -- "$STAGE_DIR" + if [ "$ROLLBACK_VERIFIED" -eq 1 ]; then + [ -z "$ROLLBACK_DIR" ] || rm -rf -- "$ROLLBACK_DIR" + fi + exit "$status" +} + +validate_source || fail "source_validation_failed" + +if [ "$MODE" = "check" ]; then + printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"check","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":17,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" + exit 0 +fi + +if [ "$MODE" = "verify" ]; then + verify_destination || fail "destination_verification_failed" + printf '{"schemaVersion":"agent99_host110_backup_runtime_executor_v1","mode":"verify","ok":true,"sourceRevision":"%s","sourceHead":"%s","fileCount":17,"remoteWritePerformed":false}\n' "$SOURCE_REVISION" "$SOURCE_HEAD" + exit 0 +fi + +[ ! -e "$STATUS_ROOT/agent99-backup-runtime-deploy-${RUN_ID}.json" ] || fail "run_identity_receipt_exists" +STAGE_DIR="$STAGE_ROOT/$RUN_ID" +ROLLBACK_DIR="$ROLLBACK_ROOT/$RUN_ID" +[ ! -e "$STAGE_DIR" ] || fail "run_identity_stage_exists" +[ ! -e "$ROLLBACK_DIR" ] || fail "run_identity_rollback_exists" + +[ ! -L "$LOCK_PATH" ] || fail "deployment_lock_symlink_unsafe" +(umask 077; : >> "$LOCK_PATH") || fail "deployment_lock_unavailable" +[ -f "$LOCK_PATH" ] && [ -O "$LOCK_PATH" ] || fail "deployment_lock_owner_invalid" +exec 9>>"$LOCK_PATH" +flock -x -w 30 9 || fail "deployment_lock_timeout" + +# Old runtime revisions predate the shared gate. Refuse the first promotion if +# one of them is already active; after common.sh moves first, new starts block. +if pgrep -afu "$EXPECTED_USER" '/backup/scripts/(backup-|check-backup|sync-offsite|verify-offsite|enforce-latest)' \ + | grep -vF "host110-backup-runtime-executor.sh" >/dev/null 2>&1; then + fail "legacy_backup_runtime_active" +fi + +install -d -m 700 "$STAGE_DIR" "$ROLLBACK_DIR" +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM HUP + +for name in "${RUNTIME_FILES[@]}"; do + install -m 700 "$SOURCE_STAGE/$name" "$STAGE_DIR/$name" + [ "$(working_digest "$STAGE_DIR/$name")" = "$(source_digest "$name")" ] || fail "stage_identity_failed" + validate_file "$STAGE_DIR/$name" || fail "stage_validation_failed" +done + +record_prestate || fail "rollback_prestate_capture_failed" +APPLY_STARTED=1 + +# common.sh is promoted first. Every managed mutating backup entrypoint then +# takes the shared side of LOCK_PATH before doing work, while this process holds +# the exclusive side until the independent verifier has completed. +for name in "${RUNTIME_FILES[@]}"; do + temporary="$DEST_ROOT/.${name}.agent99-${RUN_ID}" + install -m 755 "$STAGE_DIR/$name" "$temporary" + mv -f -- "$temporary" "$DEST_ROOT/$name" +done + +verify_destination || fail "post_apply_verification_failed" +verifier_result="$(BACKUP_RUNTIME_DEPLOY_CONTEXT=1 python3 "$SOURCE_STAGE/verify-host110-backup-runtime.py" \ + --manifest "$SOURCE_STAGE/manifest.json" \ + --destination "$DEST_ROOT" \ + --source-revision "$SOURCE_REVISION" \ + --run-id "$RUN_ID" \ + --expected-verifier-sha256 "$VERIFIER_DIGEST")" || fail "independent_verifier_failed" +python3 - "$verifier_result" "$SOURCE_REVISION" "$RUN_ID" <<'PY' || fail "independent_verifier_contract_failed" +import json +import sys + +try: + row = json.loads(sys.argv[1]) +except json.JSONDecodeError: + raise SystemExit(1) +if not ( + row.get("schemaVersion") == "agent99_host110_backup_runtime_verifier_v1" + and row.get("ok") is True + and row.get("sourceRevision") == sys.argv[2] + and row.get("runId") == sys.argv[3] + and row.get("fileCount") == 17 + and row.get("remoteWritePerformed") is False + and row.get("selfIdentityVerified") is True +): + raise SystemExit(1) +PY + +APPLY_COMPLETE=1 +receipt_path="$(write_receipt "verified" 0)" +rm -rf -- "$STAGE_DIR" "$ROLLBACK_DIR" +STAGE_DIR="" +ROLLBACK_DIR="" +trap - EXIT HUP INT TERM +python3 - "$SOURCE_REVISION" "$SOURCE_HEAD" "$receipt_path" "$verifier_result" <<'PY' +import json +import sys + +print(json.dumps({ + "schemaVersion": "agent99_host110_backup_runtime_executor_v1", + "mode": "apply", + "ok": True, + "sourceRevision": sys.argv[1], + "sourceHead": sys.argv[2], + "fileCount": 17, + "rollbackPerformed": False, + "receipt": sys.argv[3], + "verifier": json.loads(sys.argv[4]), +}, ensure_ascii=True, sort_keys=True)) +PY diff --git a/scripts/backup/verify-host110-backup-runtime.py b/scripts/backup/verify-host110-backup-runtime.py new file mode 100755 index 000000000..826750e77 --- /dev/null +++ b/scripts/backup/verify-host110-backup-runtime.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Independent no-write verifier for the Host110 backup runtime transaction.""" + +from __future__ import annotations + +import argparse +import base64 +import fcntl +import hashlib +import json +import os +import re +import sys +import time +from pathlib import Path + + +EXPECTED_DESTINATION = Path("/backup/scripts") +RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock") +EXPECTED_FILES = ( + "common.sh", + "backup-all.sh", + "backup-host188-products.sh", + "verify-host188-products-backup.sh", + "verify-host188-products-archive.py", + "backup-gitea.sh", + "gitea-full-backup-restore-drill.sh", + "backup-configs.sh", + "backup-awoooi.sh", + "backup-awoooi-frequent.sh", + "backup-clawbot.sh", + "backup-sentry.sh", + "check-backup-integrity.sh", + "sync-offsite-backups.sh", + "backup-offsite-readiness-gate.sh", + "verify-offsite-full-sync.sh", + "enforce-latest-only-retention.sh", +) + + +def fail(reason: str) -> None: + print(f"HOST110_BACKUP_RUNTIME_VERIFY_OK=0\nERROR={reason}", file=sys.stderr) + raise SystemExit(1) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def acquire_runtime_read_lock(destination: Path): + if destination != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_DEPLOY_CONTEXT") == "1": + return None + if RUNTIME_LOCK.is_symlink() or not RUNTIME_LOCK.is_file(): + fail("runtime_lock_unavailable") + handle = RUNTIME_LOCK.open("rb") + if RUNTIME_LOCK.stat().st_uid != os.getuid(): + handle.close() + fail("runtime_lock_owner_invalid") + deadline = time.monotonic() + 60 + while True: + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + return handle + except BlockingIOError: + if time.monotonic() >= deadline: + handle.close() + fail("runtime_deployment_active") + time.sleep(0.2) + + +def main() -> None: + parser = argparse.ArgumentParser() + manifest_source = parser.add_mutually_exclusive_group(required=True) + manifest_source.add_argument("--manifest") + manifest_source.add_argument("--manifest-base64") + parser.add_argument("--destination", required=True) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--expected-verifier-sha256", required=True) + args = parser.parse_args() + + if not re.fullmatch(r"[0-9a-f]{40}", args.source_revision): + fail("invalid_source_revision") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}", args.run_id): + fail("invalid_run_id") + destination = Path(args.destination) + if destination != EXPECTED_DESTINATION or destination.is_symlink() or not destination.is_dir(): + fail("invalid_destination") + runtime_lock = acquire_runtime_read_lock(destination) + + if not re.fullmatch(r"[0-9a-f]{64}", args.expected_verifier_sha256): + fail("invalid_verifier_identity") + try: + if args.manifest: + manifest_text = Path(args.manifest).read_text(encoding="utf-8") + else: + manifest_text = base64.b64decode(args.manifest_base64, validate=True).decode("utf-8") + document = json.loads(manifest_text) + except (OSError, UnicodeDecodeError, ValueError, json.JSONDecodeError): + fail("manifest_unavailable") + rows = document.get("files") + if ( + document.get("schemaVersion") != "agent99_host110_backup_runtime_source_manifest_v1" + or document.get("sourceRevision") != args.source_revision + or document.get("runId") != args.run_id + or not re.fullmatch(r"[0-9a-f]{40}", str(document.get("sourceHead", ""))) + or document.get("verifierSha256") != args.expected_verifier_sha256 + or not isinstance(rows, list) + or len(rows) != len(EXPECTED_FILES) + ): + fail("manifest_contract_failed") + by_name: dict[str, str] = {} + for row in rows: + if not isinstance(row, dict): + fail("manifest_file_set_failed") + name = str(row.get("name", "")) + if name in by_name: + fail("manifest_file_set_failed") + by_name[name] = str(row.get("sha256", "")) + if set(by_name) != set(EXPECTED_FILES) or not all(re.fullmatch(r"[0-9a-f]{64}", value) for value in by_name.values()): + fail("manifest_file_set_failed") + + verifier_path = Path(__file__) + self_identity_verified = verifier_path.is_file() and sha256(verifier_path) == args.expected_verifier_sha256 + if args.manifest and not self_identity_verified: + fail("verifier_identity_failed") + + root = destination.resolve(strict=True) + for name in EXPECTED_FILES: + path = destination / name + if path.is_symlink() or not path.is_file(): + fail("runtime_file_type_failed") + resolved = path.resolve(strict=True) + if resolved.parent != root: + fail("runtime_file_boundary_failed") + stat = resolved.stat() + if stat.st_uid != os.getuid() or stat.st_gid != os.getgid() or stat.st_mode & 0o777 != 0o755: + fail("runtime_file_metadata_failed") + if sha256(resolved) != by_name[name]: + fail("runtime_file_identity_failed") + + result = { + "schemaVersion": "agent99_host110_backup_runtime_verifier_v1", + "ok": True, + "sourceRevision": args.source_revision, + "runId": args.run_id, + "fileCount": len(EXPECTED_FILES), + "destination": str(EXPECTED_DESTINATION), + "remoteWritePerformed": False, + "secretValuesRead": False, + "selfIdentityVerified": self_identity_verified, + "verifierSha256": args.expected_verifier_sha256, + } + print(json.dumps(result, ensure_ascii=True, sort_keys=True)) + if runtime_lock is not None: + runtime_lock.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/backup/verify-host188-products-archive.py b/scripts/backup/verify-host188-products-archive.py index 44bdcd799..79c4ed3b6 100644 --- a/scripts/backup/verify-host188-products-archive.py +++ b/scripts/backup/verify-host188-products-archive.py @@ -4,11 +4,14 @@ from __future__ import annotations import argparse +import fcntl import hashlib import json +import os import sqlite3 import tarfile import tempfile +import time from pathlib import Path, PurePosixPath from typing import Any @@ -20,6 +23,28 @@ EXPECTED_PREFIXES = { "n8n-online-": ".tar.gz", } FIXED_ASSETS = {"n8n-validation.json"} +RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock") + + +def acquire_runtime_lock(): + if Path(__file__).resolve().parent != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_SHARED_LOCK_HELD") == "1": + return None + if RUNTIME_LOCK.is_symlink(): + raise RuntimeError("backup_runtime_gate_unsafe") + handle = RUNTIME_LOCK.open("a+") + if RUNTIME_LOCK.stat().st_uid != os.getuid(): + handle.close() + raise RuntimeError("backup_runtime_gate_owner_invalid") + deadline = time.monotonic() + 60 + while True: + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + return handle + except BlockingIOError: + if time.monotonic() >= deadline: + handle.close() + raise RuntimeError("backup_runtime_deployment_active") + time.sleep(0.2) def sha256_file(path: Path) -> str: @@ -153,10 +178,13 @@ def validate_restore(root: Path) -> dict[str, Any]: def main() -> int: + runtime_lock = acquire_runtime_lock() parser = argparse.ArgumentParser() parser.add_argument("--root", type=Path, required=True) args = parser.parse_args() print(json.dumps(validate_restore(args.root), separators=(",", ":"))) + if runtime_lock is not None: + runtime_lock.close() return 0 diff --git a/scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py b/scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py index 1109d1197..54e787232 100644 --- a/scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py +++ b/scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py @@ -293,8 +293,8 @@ def test_entrypoint_is_in_exact_agent99_runtime_bundle() -> None: assert expected in _quoted_array(contract, "$requiredFiles = @(", ")") assert expected in _quoted_array(sender, "RUNTIME_FILES=(", ")") assert expected in _quoted_array(receiver, "$FixedRuntimeFiles = @(", ")") - assert 'expectedRuntimeFileCount": 18' in sender - assert '$runtimeManifest.fileCount -eq 18' in receiver + assert 'expectedRuntimeFileCount": 19' in sender + assert '$runtimeManifest.fileCount -eq 19' in receiver def test_runtime_contract_check_requires_typed_signoz_transport_readback() -> None: diff --git a/scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py b/scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py index f453ca371..e93d016b9 100644 --- a/scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py +++ b/scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py @@ -122,7 +122,7 @@ def test_agent99_atomic_bundle_owns_the_dispatch_entrypoint() -> None: assert "$dbExecutorRecoveryContract" in contract assert "awoooi_db_bounded_executor_receipt_v2" in contract assert "Receipt.database_sqlstate" in contract - assert 'expectedRuntimeFileCount": 18' in transport - assert "expectedRuntimeFileCount=18" in transport - assert "expectedRuntimeFileCount -ne 18" in receiver - assert "fileCount = 18" in receiver + assert 'expectedRuntimeFileCount": 19' in transport + assert "expectedRuntimeFileCount=19" in transport + assert "expectedRuntimeFileCount -ne 19" in receiver + assert "fileCount = 19" in receiver diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index 92189091a..b4b180226 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -486,7 +486,7 @@ $decision = Get-AgentLivePreflightDecision ` -AgentRootPresent ([bool](Test-Path $AgentRoot)) ` -Manifest $manifest ` -ManifestCheckRequired ([bool]($ReadinessScope -eq "FullRuntime")) ` - -ExpectedRuntimeFileCount 18 ` + -ExpectedRuntimeFileCount 19 ` -TaskFailureCount $taskFailures.Count ` -RelayListenerCount $relayListeners.Count ` -RequiredEvidenceStaleCount $requiredEvidenceStale.Count ` diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index af83a6372..49c31c4e3 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -34,6 +34,7 @@ $FixedRuntimeFiles = @( "agent99-control-plane.ps1", "agent99-db-bounded-executor.ps1", "agent99-signoz-metadata-executor.ps1", + "agent99-host110-backup-runtime-broker.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", @@ -951,7 +952,7 @@ try { } $sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant() if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" } - if ([int]$envelope.expectedRuntimeFileCount -ne 18) { throw "invalid_expected_runtime_file_count" } + if ([int]$envelope.expectedRuntimeFileCount -ne 19) { throw "invalid_expected_runtime_file_count" } $traceId = [string]$envelope.traceId $runId = [string]$envelope.runId @@ -986,7 +987,7 @@ try { $sourceManifest = [pscustomobject]@{ schemaVersion = "agent99_remote_source_manifest_v1" sourceRevision = $sourceRevision - fileCount = 18 + fileCount = 19 manifestSha256 = $manifestDigest files = @($FixedRuntimeFiles | ForEach-Object { [pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() } @@ -1023,7 +1024,7 @@ try { status = "check_ready" mode = "check" sourceRevision = $sourceRevision - expectedRuntimeFileCount = 18 + expectedRuntimeFileCount = 19 manifestSha256 = $manifestDigest plannedStagingPath = $stagePath runtimeManifest = $currentManifest @@ -1110,7 +1111,7 @@ try { if ( [string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or [string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or - [int]$existingSourceManifest.fileCount -ne 18 -or + [int]$existingSourceManifest.fileCount -ne 19 -or [string]$existingSourceManifest.manifestSha256 -ne $manifestDigest ) { throw "existing_staging_manifest_mismatch" } } catch { @@ -1134,7 +1135,7 @@ try { [string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and $live.runtimeMatched -and $live.sourceRevision -eq $sourceRevision -and - $live.fileCount -eq 18 -and + $live.fileCount -eq 19 -and $live.mismatchCount -eq 0 -and $liveLauncherContract.ok -and [string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256 @@ -1447,7 +1448,7 @@ try { $runtimeManifest.exists -and $runtimeManifest.runtimeMatched -and $runtimeManifest.sourceRevision -eq $sourceRevision -and - $runtimeManifest.fileCount -eq 18 -and + $runtimeManifest.fileCount -eq 19 -and $runtimeManifest.mismatchCount -eq 0 -and $launcherContract.ok -and [string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and diff --git a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh index 658e5f375..189098f14 100755 --- a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh +++ b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh @@ -18,6 +18,7 @@ RUNTIME_FILES=( "agent99-control-plane.ps1" "agent99-db-bounded-executor.ps1" "agent99-signoz-metadata-executor.ps1" + "agent99-host110-backup-runtime-broker.ps1" "agent99-deploy.ps1" "agent99-register-tasks.ps1" "agent99-alertmanager-alertchain-poll.ps1" @@ -227,7 +228,7 @@ trace_id, run_id, work_item_id = sys.argv[4:7] output_path = Path(sys.argv[7]) runtime_names = sys.argv[8:] -if len(runtime_names) != 18 or len(set(runtime_names)) != 18: +if len(runtime_names) != 19 or len(set(runtime_names)) != 19: raise SystemExit("fixed_runtime_file_contract_failed") files: list[dict[str, str]] = [] @@ -267,7 +268,7 @@ envelope = { "runId": run_id, "workItemId": work_item_id, "sourceRevision": source_revision, - "expectedRuntimeFileCount": 18, + "expectedRuntimeFileCount": 19, "manifestSha256": manifest_sha256, "files": files, "livePreflight": { @@ -419,14 +420,14 @@ stage_token = hashlib.sha256(stage_identity.encode("utf-8")).hexdigest()[:20] script = r'''$ErrorActionPreference='Stop';$ProgressPreference='SilentlyContinue' $a='C:\Wooo\Agent99';$b=Join-Path $a 'bin';$p=Join-Path $a 'state\runtime-manifest.json' $r=[ordered]@{exists=$false;sourceRevision='';runtimeMatched=$false;recordedRuntimeMatched=$false;fileCount=0;mismatchCount=-1;recordedMismatchCount=-1;parseError=''} -if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 18-and$m.mismatchCount-eq 0-and$rows.Count-eq 18-and$seen.Count-eq 18-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}} +if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 19-and$m.mismatchCount-eq 0-and$rows.Count-eq 19-and$seen.Count-eq 19-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}} $c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''} try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name} $z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false} $ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0) $status=if($ready){'check_ready'}else{'check_failed_no_apply'} $next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'} -[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=18;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8 +[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=19;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8 if(-not$ready){exit 65}''' script = ( script.replace("__SOURCE_REVISION__", source_revision) diff --git a/scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py b/scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py new file mode 100644 index 000000000..00ac91317 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import base64 +import hashlib +import importlib.util +import json +import re +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[3] +BROKER = ROOT / "agent99-host110-backup-runtime-broker.ps1" +EXECUTOR = ROOT / "scripts" / "backup" / "host110-backup-runtime-executor.sh" +VERIFIER = ROOT / "scripts" / "backup" / "verify-host110-backup-runtime.py" + + +EXPECTED_RUNTIME_FILES = { + "common.sh", + "backup-all.sh", + "backup-host188-products.sh", + "verify-host188-products-backup.sh", + "verify-host188-products-archive.py", + "backup-gitea.sh", + "gitea-full-backup-restore-drill.sh", + "backup-configs.sh", + "backup-awoooi.sh", + "backup-awoooi-frequent.sh", + "backup-clawbot.sh", + "backup-sentry.sh", + "check-backup-integrity.sh", + "sync-offsite-backups.sh", + "backup-offsite-readiness-gate.sh", + "verify-offsite-full-sync.sh", + "enforce-latest-only-retention.sh", +} + + +def _load_verifier(): + spec = importlib.util.spec_from_file_location("host110_runtime_verifier", VERIFIER) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None: + source = BROKER.read_text(encoding="utf-8") + + assert '[ValidateSet("Check", "Apply", "Verify")]' in source + assert '$TargetHost = "192.168.0.110"' in source + assert '$TargetUser = "wooo"' in source + assert '$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"' in source + assert 'keys\\agent99_ed25519' in source + assert '"credential.helper="' in source + assert '$env:GIT_TERMINAL_PROMPT = "0"' in source + assert '"merge-base", "--is-ancestor", $SourceRevision, $sourceHead' in source + assert '"checkout", "--quiet", $SourceRevision, "--"' in source + assert 'executorSha256 = $executorDigest' in source + assert 'verifierSha256 = $verifierDigest' in source + assert "/home/wooo/awoooi" not in source + assert '"StrictHostKeyChecking=yes"' in source + assert '"NumberOfPasswordPrompts=0"' in source + assert '"ConnectionAttempts=1"' in source + assert "$process.WaitForExit($TimeoutSeconds * 1000)" in source + assert "taskkill.exe /PID $process.Id /T /F" in source + assert "$process.Kill()" in source + assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths' in source + + +def test_windows99_broker_verify_is_independent_no_write_and_evidence_is_immutable() -> None: + source = BROKER.read_text(encoding="utf-8") + + assert 'RedirectStandardInput = $StandardInputPath' in source + assert "python3 - --manifest-base64" in source + assert 'remoteWritePerformed' in source + assert '[IO.File]::Move($temporary, $Path)' in source + assert 'Move-Item -LiteralPath $temporary -Destination $Path -Force' not in source + assert 'run_identity_conflict' in source + assert 'run_identity_evidence_exists' in source + assert 'decisionProvider = "deterministic_only"' in source + assert 'criticProvider = "deterministic_only"' in source + assert 'verifier = "independent_host110_backup_runtime_python_readback"' in source + + +def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None: + source = EXECUTOR.read_text(encoding="utf-8") + array_body = source.split("readonly -a RUNTIME_FILES=(", 1)[1].split(")", 1)[0] + runtime_files = set(re.findall(r"^\s+([A-Za-z0-9_.-]+)\s*$", array_body, re.MULTILINE)) + + assert runtime_files == EXPECTED_RUNTIME_FILES + assert 'readonly EXPECTED_HOST_IP="192.168.0.110"' in source + assert 'readonly EXPECTED_USER="wooo"' in source + assert 'readonly DEST_ROOT="/backup/scripts"' in source + assert '--source-stage' in source + assert 'document.get("executorSha256"' in source + assert 'document.get("verifierSha256"' in source + assert 'executor_identity_failed' in source + assert 'verifier_identity_failed' in source + assert 'flock -x -w 30 9' in source + assert 'legacy_backup_runtime_active' in source + assert source.index("for name in \"${RUNTIME_FILES[@]}\"") < source.index("mv -f -- \"$temporary\"") + assert 'rollback_unverified' in source + assert 'failed_rolled_back' in source + assert 'working_digest "$dest_path"' in source + assert 'os.link(temporary, path)' in source + assert 'run_identity_receipt_exists' in source + assert 'run_identity_stage_exists' in source + assert 'run_identity_rollback_exists' in source + assert 'verify-host110-backup-runtime.py' in source + assert 'selfIdentityVerified' in source + assert '"productionServiceRestarted": False' in source + assert '"secretValuesRead": False' in source + assert "sudo " not in source + assert "docker " not in source + assert "systemctl " not in source + + +def test_backup_runtime_entrypoints_share_the_deployment_gate() -> None: + common = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8") + restore = (ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh").read_text(encoding="utf-8") + offsite = (ROOT / "scripts/backup/backup-offsite-readiness-gate.sh").read_text(encoding="utf-8") + archive = (ROOT / "scripts/backup/verify-host188-products-archive.py").read_text(encoding="utf-8") + + for source in (common, restore, offsite): + assert "/tmp/agent99-host110-backup-runtime.lock" in source + assert "flock -s -w 60" in source + assert "fcntl.LOCK_SH | fcntl.LOCK_NB" in archive + assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" in archive + + +def test_independent_verifier_accepts_exact_bundle_without_writing(tmp_path, monkeypatch, capsys) -> None: + verifier = _load_verifier() + destination = tmp_path / "scripts" + destination.mkdir() + rows = [] + for name in verifier.EXPECTED_FILES: + path = destination / name + path.write_text(f"runtime:{name}\n", encoding="utf-8") + path.chmod(0o755) + rows.append({"name": name, "sha256": _sha256(path)}) + verifier_digest = _sha256(VERIFIER) + source_revision = "a" * 40 + run_id = "unit-exact" + manifest = { + "schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1", + "sourceRevision": source_revision, + "sourceHead": "b" * 40, + "runId": run_id, + "executorSha256": "c" * 64, + "verifierSha256": verifier_digest, + "files": rows, + } + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + before = {path.name: (path.stat().st_mtime_ns, _sha256(path)) for path in destination.iterdir()} + monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination) + monkeypatch.setattr( + sys, + "argv", + [ + str(VERIFIER), + "--manifest", + str(manifest_path), + "--destination", + str(destination), + "--source-revision", + source_revision, + "--run-id", + run_id, + "--expected-verifier-sha256", + verifier_digest, + ], + ) + + verifier.main() + + result = json.loads(capsys.readouterr().out) + after = {path.name: (path.stat().st_mtime_ns, _sha256(path)) for path in destination.iterdir()} + assert result["ok"] is True + assert result["remoteWritePerformed"] is False + assert result["selfIdentityVerified"] is True + assert before == after + + +def test_independent_verifier_rejects_drifted_bundle(tmp_path, monkeypatch, capsys) -> None: + verifier = _load_verifier() + destination = tmp_path / "scripts" + destination.mkdir() + rows = [] + for name in verifier.EXPECTED_FILES: + path = destination / name + path.write_text(f"runtime:{name}\n", encoding="utf-8") + path.chmod(0o755) + rows.append({"name": name, "sha256": _sha256(path)}) + (destination / verifier.EXPECTED_FILES[0]).write_text("drifted\n", encoding="utf-8") + verifier_digest = _sha256(VERIFIER) + source_revision = "d" * 40 + run_id = "unit-drift" + manifest = { + "schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1", + "sourceRevision": source_revision, + "sourceHead": "e" * 40, + "runId": run_id, + "executorSha256": "f" * 64, + "verifierSha256": verifier_digest, + "files": rows, + } + encoded = base64.b64encode(json.dumps(manifest).encode()).decode() + monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination) + monkeypatch.setattr( + sys, + "argv", + [ + str(VERIFIER), + "--manifest-base64", + encoded, + "--destination", + str(destination), + "--source-revision", + source_revision, + "--run-id", + run_id, + "--expected-verifier-sha256", + verifier_digest, + ], + ) + + with pytest.raises(SystemExit): + verifier.main() + assert "runtime_file_identity_failed" in capsys.readouterr().err + + +def test_agent99_runtime_lists_include_the_host110_broker_once() -> None: + sources = [ + ROOT / "agent99-bootstrap.ps1", + ROOT / "agent99-contract-check.ps1", + ROOT / "agent99-deploy.ps1", + ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh", + ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1", + ] + marker = "agent99-host110-backup-runtime-broker.ps1" + + for path in sources: + assert marker in path.read_text(encoding="utf-8"), path diff --git a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py index cdf725aef..5c6be47c7 100644 --- a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py +++ b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py @@ -36,7 +36,7 @@ def _base_case() -> dict[str, Any]: "exists": True, "sourceRevision": "a" * 40, "runtimeMatched": True, - "fileCount": 18, + "fileCount": 19, "mismatchCount": 0, "parseError": "", }, @@ -133,7 +133,7 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No ) assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision assert "preflightGreen = [bool]$decision.deploymentEligible" in source - assert "-ExpectedRuntimeFileCount 18" in source + assert "-ExpectedRuntimeFileCount 19" in source assert "warningReasons = $warningReasons" in source assert "selfHealthPerformanceIssues = @(" in source assert 'if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$")' in source diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index c00631130..d09be65f3 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -26,6 +26,7 @@ EXPECTED_RUNTIME_FILES = ( "agent99-control-plane.ps1", "agent99-db-bounded-executor.ps1", "agent99-signoz-metadata-executor.ps1", + "agent99-host110-backup-runtime-broker.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", @@ -88,7 +89,7 @@ def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None ) assert preflight_count is not None assert int(preflight_count.group(1)) == len(EXPECTED_RUNTIME_FILES) - assert "expectedRuntimeFileCount\": 18" in sender + assert "expectedRuntimeFileCount\": 19" in sender assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver assert "required_runtime_file_missing" in receiver assert "duplicate_runtime_file" in receiver @@ -263,7 +264,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier() ) assert "$livePreflight.sourceRevision -eq $sourceRevision" in source assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source - assert '$runtimeManifest.fileCount -eq 18' in source + assert '$runtimeManifest.fileCount -eq 19' in source assert "function Invoke-AgentWindowsControlBaseline" in source assert "function Get-AgentRunEvidenceToken" in source assert "$evidenceRunToken = Get-AgentRunEvidenceToken -Value $RunId" in source diff --git a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py index a17ba8670..65139e9b7 100644 --- a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py +++ b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py @@ -516,7 +516,8 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None: for parameter in ("AutomationRunId", "TraceId", "WorkItemId"): assert f"[string]${parameter} = \"\"" in control assert f'[string]`${parameter} = ""' in bootstrap - assert 'reason = "controlled_dispatch_identity_unsafe"' in control + assert '"controlled_dispatch_identity_unsafe"' in control + assert "reason = $identityRejectionReason" in control assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function assert '$dryRun.managerPreflightStatus -eq "ready"' in function assert "-not $dryRun.artifactTransactionBlocked" in function