diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 new file mode 100644 index 000000000..8c7e86617 --- /dev/null +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -0,0 +1,729 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$EnvelopeJson +) + +$ErrorActionPreference = "Stop" +$AgentRoot = "C:\Wooo\Agent99" +$BinDir = Join-Path $AgentRoot "bin" +$ConfigPath = Join-Path $AgentRoot "config\agent99.config.json" +$StateDir = Join-Path $AgentRoot "state" +$EvidenceDir = Join-Path $AgentRoot "evidence" +$DeployRoot = Join-Path $AgentRoot "deploy" +$ManifestPath = Join-Path $StateDir "runtime-manifest.json" +$WindowsPowerShell = "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" +$FixedRuntimeFiles = @( + "agent99-bootstrap.ps1", + "agent99-contract-check.ps1", + "agent99-control-plane.ps1", + "agent99-deploy.ps1", + "agent99-register-tasks.ps1", + "agent99-sre-alert-inbox.ps1", + "agent99-sre-alert-relay.ps1", + "agent99-submit-request.ps1", + "agent99-synthetic-tests.ps1", + "agent99-telegram-inbox.ps1", + "agent99-windows-update-policy.ps1", + "host-reboot-scorecard.ps1", + "agent99.config.example.json", + "agent99.config.99.example.json" +) +$script:DeployLock = $null + +function Get-AgentSha256Bytes { + param([byte[]]$Bytes) + + $sha = [Security.Cryptography.SHA256]::Create() + try { + return (($sha.ComputeHash($Bytes) | ForEach-Object { $_.ToString("x2") }) -join "") + } finally { + $sha.Dispose() + } +} + +function Get-AgentSha256File { + param([string]$Path) + + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { return "missing" } + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} + +function Get-AgentSafeRuntimeManifest { + if (-not (Test-Path -LiteralPath $ManifestPath -PathType Leaf)) { + return [pscustomobject]@{ + exists = $false + sourceRevision = "" + runtimeMatched = $false + recordedRuntimeMatched = $false + fileCount = 0 + mismatchCount = -1 + recordedMismatchCount = -1 + parseError = "" + } + } + try { + $manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json + $manifestRows = @($manifest.files) + $rowsByName = @{} + $shapeValid = [bool]($manifestRows.Count -eq $FixedRuntimeFiles.Count) + foreach ($row in $manifestRows) { + $rowName = [string]$row.name + if (-not $rowName -or $rowsByName.ContainsKey($rowName)) { + $shapeValid = $false + continue + } + $rowsByName[$rowName] = $row + } + $actualMismatchCount = 0 + foreach ($name in $FixedRuntimeFiles) { + if (-not $rowsByName.ContainsKey($name)) { + $shapeValid = $false + $actualMismatchCount += 1 + continue + } + $recordedRuntimeSha = ([string]($rowsByName[$name].runtimeSha256)).ToLowerInvariant() + $actualRuntimeSha = Get-AgentSha256File (Join-Path $BinDir $name) + if (-not $recordedRuntimeSha -or $recordedRuntimeSha -ne $actualRuntimeSha) { + $actualMismatchCount += 1 + } + } + $currentMatched = [bool]( + $shapeValid -and + [bool]$manifest.runtimeMatched -and + [int]$manifest.fileCount -eq $FixedRuntimeFiles.Count -and + [int]$manifest.mismatchCount -eq 0 -and + $actualMismatchCount -eq 0 + ) + return [pscustomobject]@{ + exists = $true + sourceRevision = [string]$manifest.sourceRevision + runtimeMatched = $currentMatched + recordedRuntimeMatched = [bool]$manifest.runtimeMatched + fileCount = [int]$manifest.fileCount + mismatchCount = [int]$actualMismatchCount + recordedMismatchCount = [int]$manifest.mismatchCount + parseError = "" + } + } catch { + return [pscustomobject]@{ + exists = $true + sourceRevision = "" + runtimeMatched = $false + recordedRuntimeMatched = $false + fileCount = 0 + mismatchCount = -1 + recordedMismatchCount = -1 + parseError = $_.Exception.GetType().Name + } + } +} + +function Get-AgentLiveBundleDigest { + $rows = @() + foreach ($name in $FixedRuntimeFiles) { + $rows += "$name`t$(Get-AgentSha256File (Join-Path $BinDir $name))`n" + } + return Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes(($rows -join ""))) +} + +function Test-AgentSafeIdentity { + param([string]$Value) + return [bool]($Value -and $Value -match "^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$") +} + +function Test-AgentManifestEqual { + param([object]$Left, [object]$Right) + + return [bool]( + $Left.exists -eq $Right.exists -and + $Left.sourceRevision -eq $Right.sourceRevision -and + $Left.runtimeMatched -eq $Right.runtimeMatched -and + $Left.fileCount -eq $Right.fileCount -and + $Left.mismatchCount -eq $Right.mismatchCount + ) +} + +function Convert-AgentChildJson { + param([object[]]$Lines) + + $text = (@($Lines | ForEach-Object { [string]$_ }) -join "`n").Trim() + $start = $text.IndexOf("{") + $end = $text.LastIndexOf("}") + if ($start -lt 0 -or $end -le $start) { return $null } + try { + return $text.Substring($start, $end - $start + 1) | ConvertFrom-Json + } catch { + return $null + } +} + +function Invoke-AgentBoundedPowerShell { + param( + [string[]]$Arguments, + [int]$TimeoutSeconds + ) + + if ($TimeoutSeconds -lt 1 -or @($Arguments | Where-Object { $_ -match '[\s"]' }).Count -gt 0) { + throw "unsafe_or_invalid_child_process_contract" + } + $startInfo = New-Object System.Diagnostics.ProcessStartInfo + $startInfo.FileName = $WindowsPowerShell + $startInfo.Arguments = $Arguments -join " " + $startInfo.UseShellExecute = $false + $startInfo.CreateNoWindow = $true + $startInfo.RedirectStandardOutput = $true + $startInfo.RedirectStandardError = $true + $process = New-Object System.Diagnostics.Process + $process.StartInfo = $startInfo + try { + if (-not $process.Start()) { throw "child_process_start_failed" } + $stdoutTask = $process.StandardOutput.ReadToEndAsync() + $stderrTask = $process.StandardError.ReadToEndAsync() + $finished = $process.WaitForExit($TimeoutSeconds * 1000) + $terminated = $finished + if (-not $finished) { + try { $process.Kill() } catch {} + try { $terminated = $process.WaitForExit(10000) } catch { $terminated = $false } + } + $stdout = "" + if ($terminated) { + try { $stdout = [string]$stdoutTask.Result } catch { $stdout = "" } + try { $null = [string]$stderrTask.Result } catch {} + } + return [pscustomobject]@{ + exitCode = if ($finished) { [int]$process.ExitCode } else { 124 } + timedOut = [bool](-not $finished) + stdout = $stdout + } + } finally { + $process.Dispose() + } +} + +function Invoke-AgentDeployChild { + param( + [string]$SourceRoot, + [string]$SourceRevision, + [switch]$ValidateOnly + ) + + $arguments = @( + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", "Bypass", + "-File", (Join-Path $SourceRoot "agent99-deploy.ps1"), + "-SourceRoot", $SourceRoot, + "-AgentRoot", $AgentRoot, + "-SourceRevision", $SourceRevision + ) + if ($ValidateOnly) { $arguments += "-ValidateOnly" } + $timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 } + $processResult = Invoke-AgentBoundedPowerShell -Arguments $arguments -TimeoutSeconds $timeoutSeconds + return [pscustomobject]@{ + ok = [bool]($processResult.exitCode -eq 0 -and -not $processResult.timedOut) + exitCode = [int]$processResult.exitCode + timedOut = [bool]$processResult.timedOut + payload = Convert-AgentChildJson @($processResult.stdout) + validateOnly = [bool]$ValidateOnly + } +} + +function Invoke-AgentLivePreflight { + param([string]$PreflightPath) + + $arguments = @("-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", $PreflightPath, "-Mode", "Verify", "-AgentRoot", $AgentRoot) + $processResult = Invoke-AgentBoundedPowerShell -Arguments $arguments -TimeoutSeconds 120 + $text = [string]$processResult.stdout + $match = [regex]::Match($text, "(?s)JSON_BEGIN\r?\n(?\{.*?\})\r?\nJSON_END") + $payload = $null + if ($match.Success) { + try { $payload = $match.Groups["json"].Value | ConvertFrom-Json } catch { $payload = $null } + } + $summary = if ($payload -and $payload.PSObject.Properties["summary"]) { $payload.summary } else { $null } + return [pscustomobject]@{ + ok = [bool]($processResult.exitCode -eq 0 -and -not $processResult.timedOut -and $summary -and $summary.preflightGreen) + exitCode = [int]$processResult.exitCode + timedOut = [bool]$processResult.timedOut + parsed = [bool]($null -ne $payload) + sourceRevision = if ($payload -and $payload.manifest) { [string]$payload.manifest.sourceRevision } else { "" } + runtimeMatched = [bool]($payload -and $payload.manifest -and $payload.manifest.runtimeMatched) + healthyTaskCount = if ($summary) { [int]$summary.healthyTaskCount } else { 0 } + requiredTaskCount = if ($summary) { [int]$summary.requiredTaskCount } else { 0 } + relayListenerCount = if ($summary) { [int]$summary.relayListenerCount } else { 0 } + pendingCommandCount = if ($summary) { [int]$summary.pendingCommandCount } else { -1 } + preflightGreen = [bool]($summary -and $summary.preflightGreen) + blockingReasons = if ($processResult.timedOut) { @("preflight_process_timeout") } elseif ($summary) { @($summary.blockingReasons | ForEach-Object { [string]$_ }) } else { @("preflight_output_unparseable") } + } +} + +function Copy-AgentFileWithRetry { + param([string]$Source, [string]$Destination) + + for ($attempt = 1; $attempt -le 20; $attempt += 1) { + try { + Copy-Item -LiteralPath $Source -Destination $Destination -Force + return + } catch [System.IO.IOException] { + if ($attempt -ge 20) { throw } + Start-Sleep -Milliseconds 500 + } + } +} + +function Test-AgentBackupPath { + param([string]$BackupPath) + + if (-not $BackupPath) { return $false } + $expectedPrefix = [IO.Path]::GetFullPath((Join-Path $DeployRoot "backup-")) + $candidate = [IO.Path]::GetFullPath($BackupPath) + return [bool]($candidate.StartsWith($expectedPrefix, [StringComparison]::OrdinalIgnoreCase)) +} + +function Restore-AgentBundle { + param( + [string]$BackupPath, + [hashtable]$BeforeFilePresence, + [bool]$BeforeConfigPresent, + [bool]$BeforeManifestPresent + ) + + if (-not (Test-AgentBackupPath $BackupPath) -or -not (Test-Path -LiteralPath $BackupPath -PathType Container)) { + return [pscustomobject]@{ attempted = $false; restored = $false; reason = "bounded_backup_path_unavailable" } + } + try { + foreach ($name in $FixedRuntimeFiles) { + $livePath = Join-Path $BinDir $name + $backupFile = Join-Path $BackupPath $name + if ($BeforeFilePresence[$name]) { + if (-not (Test-Path -LiteralPath $backupFile -PathType Leaf)) { throw "required_runtime_backup_missing" } + Copy-AgentFileWithRetry $backupFile $livePath + } elseif (Test-Path -LiteralPath $livePath -PathType Leaf) { + Remove-Item -LiteralPath $livePath -Force + } + } + $backupConfig = Join-Path $BackupPath "agent99.config.json" + if ($BeforeConfigPresent) { + if (-not (Test-Path -LiteralPath $backupConfig -PathType Leaf)) { throw "required_config_backup_missing" } + Copy-AgentFileWithRetry $backupConfig $ConfigPath + } elseif (Test-Path -LiteralPath $ConfigPath -PathType Leaf) { + Remove-Item -LiteralPath $ConfigPath -Force + } + $backupManifest = Join-Path $BackupPath "runtime-manifest.json" + if ($BeforeManifestPresent) { + if (-not (Test-Path -LiteralPath $backupManifest -PathType Leaf)) { throw "required_manifest_backup_missing" } + Copy-AgentFileWithRetry $backupManifest $ManifestPath + } elseif (Test-Path -LiteralPath $ManifestPath -PathType Leaf) { + Remove-Item -LiteralPath $ManifestPath -Force + } + return [pscustomobject]@{ attempted = $true; restored = $true; reason = "bounded_bundle_restored" } + } catch { + return [pscustomobject]@{ attempted = $true; restored = $false; reason = $_.Exception.GetType().Name } + } +} + +function Write-AgentRemoteDeployReceipt { + param([string]$Path, [object]$Receipt) + + $temporary = "$Path.tmp.$PID" + $Receipt | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $temporary -Encoding UTF8 + Move-Item -LiteralPath $temporary -Destination $Path -Force +} + +function Write-AgentResultAndExit { + param([object]$Result, [int]$ExitCode) + + $Result | ConvertTo-Json -Compress -Depth 12 + exit $ExitCode +} + +try { + $envelope = $EnvelopeJson | ConvertFrom-Json + if ([string]$envelope.schemaVersion -ne "agent99_remote_atomic_deploy_envelope_v1") { throw "invalid_envelope_schema" } + if ([string]$envelope.sourceHostAlias -ne "110") { throw "invalid_source_host_alias" } + if ([string]$envelope.targetHostAlias -ne "99") { throw "invalid_target_host_alias" } + if ([string]$envelope.transport -ne "110_to_99_ssh_publickey") { throw "invalid_transport" } + $mode = [string]$envelope.mode + if ($mode -notin @("check", "apply")) { throw "invalid_mode" } + $sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant() + if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" } + if ([int]$envelope.expectedRuntimeFileCount -ne 14) { throw "invalid_expected_runtime_file_count" } + + $traceId = [string]$envelope.traceId + $runId = [string]$envelope.runId + $workItemId = [string]$envelope.workItemId + if ($mode -eq "apply") { + foreach ($identity in @($traceId, $runId, $workItemId)) { + if (-not (Test-AgentSafeIdentity $identity)) { throw "apply_identity_missing_or_invalid" } + } + } + + $filesByName = @{} + foreach ($file in @($envelope.files)) { + $name = [string]$file.name + if ($filesByName.ContainsKey($name)) { throw "duplicate_runtime_file" } + $filesByName[$name] = $file + } + if ($filesByName.Count -ne $FixedRuntimeFiles.Count) { throw "runtime_file_count_mismatch" } + + $manifestText = "" + $decodedFiles = @{} + foreach ($name in $FixedRuntimeFiles) { + if (-not $filesByName.ContainsKey($name)) { throw "required_runtime_file_missing" } + $row = $filesByName[$name] + $bytes = [Convert]::FromBase64String([string]$row.contentBase64) + $actualSha = Get-AgentSha256Bytes $bytes + if ($actualSha -ne ([string]$row.sha256).ToLowerInvariant()) { throw "runtime_file_sha256_mismatch" } + $decodedFiles[$name] = $bytes + $manifestText += "$name`t$actualSha`n" + } + $manifestDigest = Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes($manifestText)) + if ($manifestDigest -ne ([string]$envelope.manifestSha256).ToLowerInvariant()) { throw "manifest_sha256_mismatch" } + $sourceManifest = [pscustomobject]@{ + schemaVersion = "agent99_remote_source_manifest_v1" + sourceRevision = $sourceRevision + fileCount = 14 + manifestSha256 = $manifestDigest + files = @($FixedRuntimeFiles | ForEach-Object { + [pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() } + }) + } + + $preflight = $envelope.livePreflight + if (-not $preflight -or [string]$preflight.name -ne "agent99-live-preflight.ps1") { throw "live_preflight_missing" } + $preflightBytes = [Convert]::FromBase64String([string]$preflight.contentBase64) + if ((Get-AgentSha256Bytes $preflightBytes) -ne ([string]$preflight.sha256).ToLowerInvariant()) { throw "live_preflight_sha256_mismatch" } + + $stageIdentity = if ($mode -eq "apply") { "$runId|$sourceRevision" } else { "check|$sourceRevision" } + $stageToken = (Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes($stageIdentity))).Substring(0, 20) + $stagePath = Join-Path $DeployRoot "remote-source-$stageToken" + $receiptPath = Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken.json" + $currentManifest = Get-AgentSafeRuntimeManifest + + if ($mode -eq "check") { + Write-AgentResultAndExit ([pscustomobject]@{ + schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" + status = "check_ready" + mode = "check" + sourceRevision = $sourceRevision + expectedRuntimeFileCount = 14 + manifestSha256 = $manifestDigest + plannedStagingPath = $stagePath + runtimeManifest = $currentManifest + livePreflight = [pscustomobject]@{ executed = $false; reason = "apply_only_after_validate_only" } + operationBoundaries = [pscustomobject]@{ + remoteWritePerformed = $false + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + environmentSecretRead = $false + uiInteraction = $false + vmPowerChange = $false + hostReboot = $false + serviceRestart = $false + scheduledTaskModification = $false + } + nextSafeAction = "rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check" + }) 0 + } + + New-Item -ItemType Directory -Force -Path $StateDir, $EvidenceDir, $DeployRoot | Out-Null + $lockPath = Join-Path $StateDir "remote-atomic-deploy.lock" + try { + $script:DeployLock = [IO.File]::Open($lockPath, [IO.FileMode]::OpenOrCreate, [IO.FileAccess]::ReadWrite, [IO.FileShare]::None) + } catch { + Write-AgentResultAndExit ([pscustomobject]@{ + schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" + status = "blocked_single_flight_lock_unavailable" + mode = "apply" + traceId = $traceId + runId = $runId + workItemId = $workItemId + sourceRevision = $sourceRevision + remoteWritePerformed = $false + liveRuntimeWritePerformed = $false + livePromotionPerformed = $false + }) 75 + } + + $prior = $null + if (Test-Path -LiteralPath $receiptPath -PathType Leaf) { + try { $prior = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json } catch { $prior = $null } + } + + $stageWrittenThisRun = $false + if (-not (Test-Path -LiteralPath $stagePath -PathType Container)) { + $stageBuildPath = "$stagePath.partial.$PID.$([Guid]::NewGuid().ToString('N'))" + New-Item -ItemType Directory -Path $stageBuildPath | Out-Null + foreach ($name in $FixedRuntimeFiles) { + [IO.File]::WriteAllBytes((Join-Path $stageBuildPath $name), [byte[]]$decodedFiles[$name]) + } + [IO.File]::WriteAllBytes((Join-Path $stageBuildPath "agent99-live-preflight.ps1"), $preflightBytes) + $sourceManifest | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath (Join-Path $stageBuildPath "source-manifest.json") -Encoding UTF8 + Move-Item -LiteralPath $stageBuildPath -Destination $stagePath + $stageWrittenThisRun = $true + } else { + foreach ($name in $FixedRuntimeFiles) { + if ((Get-AgentSha256File (Join-Path $stagePath $name)) -ne ([string]($filesByName[$name].sha256)).ToLowerInvariant()) { + throw "existing_staging_file_mismatch" + } + } + if ((Get-AgentSha256File (Join-Path $stagePath "agent99-live-preflight.ps1")) -ne ([string]$preflight.sha256).ToLowerInvariant()) { + throw "existing_staging_preflight_mismatch" + } + try { + $existingSourceManifest = Get-Content -LiteralPath (Join-Path $stagePath "source-manifest.json") -Raw | ConvertFrom-Json + if ( + [string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or + [string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or + [int]$existingSourceManifest.fileCount -ne 14 -or + [string]$existingSourceManifest.manifestSha256 -ne $manifestDigest + ) { throw "existing_staging_manifest_mismatch" } + } catch { + throw "existing_staging_manifest_invalid" + } + } + + $live = Get-AgentSafeRuntimeManifest + if ( + $prior -and + [string]$prior.status -eq "deployed_verified" -and + [string]$prior.traceId -eq $traceId -and + [string]$prior.runId -eq $runId -and + [string]$prior.workItemId -eq $workItemId -and + [string]$prior.sourceRevision -eq $sourceRevision -and + $live.runtimeMatched -and + $live.sourceRevision -eq $sourceRevision -and + $live.fileCount -eq 14 -and + $live.mismatchCount -eq 0 + ) { + $replayPreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") + if ($replayPreflight.ok -and $replayPreflight.sourceRevision -eq $sourceRevision -and $replayPreflight.runtimeMatched) { + $originalOperationBoundaries = $prior.operationBoundaries + $prior | Add-Member -NotePropertyName idempotentReplay -NotePropertyValue $true -Force + $prior | Add-Member -NotePropertyName runtimeManifest -NotePropertyValue $live -Force + $prior | Add-Member -NotePropertyName livePreflight -NotePropertyValue $replayPreflight -Force + $prior | Add-Member -NotePropertyName originalOperationBoundaries -NotePropertyValue $originalOperationBoundaries -Force + $prior | Add-Member -NotePropertyName operationBoundaries -NotePropertyValue ([pscustomobject]@{ + remoteWritePerformed = $stageWrittenThisRun + liveRuntimeWritePerformed = $false + livePromotionPerformed = $false + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + environmentSecretRead = $false + uiInteraction = $false + vmPowerChange = $false + hostReboot = $false + serviceRestart = $false + scheduledTaskModification = $false + }) -Force + Write-AgentResultAndExit $prior 0 + } + Write-AgentResultAndExit ([pscustomobject]@{ + schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" + status = "idempotent_replay_post_verifier_failed_no_promotion" + mode = "apply" + traceId = $traceId + runId = $runId + workItemId = $workItemId + sourceRevision = $sourceRevision + manifestSha256 = $manifestDigest + stagingPath = $stagePath + runtimeManifest = $live + livePreflight = $replayPreflight + remoteWritePerformed = $stageWrittenThisRun + liveRuntimeWritePerformed = $false + livePromotionPerformed = $false + nextSafeAction = "repair_reported_live_preflight_blocker_then_retry_same_identity" + }) 2 + } + + $validate = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision -ValidateOnly + if (-not $validate.ok -or -not $validate.payload -or [string]$validate.payload.status -ne "validated") { + $receipt = [pscustomobject]@{ + schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" + status = "validate_only_failed_no_promotion" + mode = "apply" + traceId = $traceId + runId = $runId + workItemId = $workItemId + sourceRevision = $sourceRevision + manifestSha256 = $manifestDigest + stagingPath = $stagePath + validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut } + remoteWritePerformed = $true + liveRuntimeWritePerformed = $false + livePromotionPerformed = $false + } + Write-AgentRemoteDeployReceipt $receiptPath $receipt + Write-AgentResultAndExit $receipt 2 + } + + $beforeManifest = Get-AgentSafeRuntimeManifest + $beforeBundleDigest = Get-AgentLiveBundleDigest + $beforeConfigDigest = Get-AgentSha256File $ConfigPath + $beforeManifestDigest = Get-AgentSha256File $ManifestPath + $beforeFilePresence = @{} + foreach ($name in $FixedRuntimeFiles) { $beforeFilePresence[$name] = Test-Path -LiteralPath (Join-Path $BinDir $name) -PathType Leaf } + $beforeConfigPresent = Test-Path -LiteralPath $ConfigPath -PathType Leaf + $beforeManifestPresent = Test-Path -LiteralPath $ManifestPath -PathType Leaf + $backupDirsBefore = @{} + foreach ($directory in @(Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue)) { + $backupDirsBefore[$directory.FullName] = $true + } + + $deploy = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision + $backupPath = if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) { [string]$deploy.payload.backupDir } else { "" } + if (-not $backupPath) { + $newBackup = Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue | + Where-Object { -not $backupDirsBefore.ContainsKey($_.FullName) } | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 + if ($newBackup) { $backupPath = $newBackup.FullName } + } + + if (-not $deploy.ok -or -not $deploy.payload -or [string]$deploy.payload.status -ne "deployed" -or $deploy.payload.taskRegistration) { + $rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent + $afterManifest = Get-AgentSafeRuntimeManifest + $rollbackVerified = [bool]( + (Test-AgentManifestEqual $beforeManifest $afterManifest) -and + (Get-AgentLiveBundleDigest) -eq $beforeBundleDigest -and + (Get-AgentSha256File $ConfigPath) -eq $beforeConfigDigest -and + (Get-AgentSha256File $ManifestPath) -eq $beforeManifestDigest + ) + $receipt = [pscustomobject]@{ + schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" + status = if ($rollbackVerified) { "deploy_failed_rollback_verified" } else { "deploy_failed_rollback_unverified" } + mode = "apply" + traceId = $traceId + runId = $runId + workItemId = $workItemId + sourceRevision = $sourceRevision + manifestSha256 = $manifestDigest + stagingPath = $stagePath + backupPath = $backupPath + validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut } + deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut } + rollback = $rollback + rollbackVerified = $rollbackVerified + runtimeManifest = $afterManifest + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + uiInteraction = $false + vmPowerChange = $false + hostReboot = $false + serviceRestart = $false + scheduledTaskModification = $false + } + Write-AgentRemoteDeployReceipt $receiptPath $receipt + Write-AgentResultAndExit $receipt $(if ($rollbackVerified) { 2 } else { 3 }) + } + + $runtimeManifest = Get-AgentSafeRuntimeManifest + $livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") + $postVerified = [bool]( + $runtimeManifest.exists -and + $runtimeManifest.runtimeMatched -and + $runtimeManifest.sourceRevision -eq $sourceRevision -and + $runtimeManifest.fileCount -eq 14 -and + $runtimeManifest.mismatchCount -eq 0 -and + $livePreflight.ok -and + $livePreflight.sourceRevision -eq $sourceRevision -and + $livePreflight.runtimeMatched + ) + + if (-not $postVerified) { + $rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent + $afterManifest = Get-AgentSafeRuntimeManifest + $rollbackVerified = [bool]( + $rollback.restored -and + (Test-AgentManifestEqual $beforeManifest $afterManifest) -and + (Get-AgentLiveBundleDigest) -eq $beforeBundleDigest -and + (Get-AgentSha256File $ConfigPath) -eq $beforeConfigDigest -and + (Get-AgentSha256File $ManifestPath) -eq $beforeManifestDigest + ) + $receipt = [pscustomobject]@{ + schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" + status = if ($rollbackVerified) { "rolled_back_post_verifier_failed" } else { "rollback_failed_after_post_verifier" } + mode = "apply" + traceId = $traceId + runId = $runId + workItemId = $workItemId + sourceRevision = $sourceRevision + manifestSha256 = $manifestDigest + stagingPath = $stagePath + backupPath = $backupPath + validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut } + deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut } + failedRuntimeManifest = $runtimeManifest + failedLivePreflight = $livePreflight + rollback = $rollback + rollbackVerified = $rollbackVerified + runtimeManifest = $afterManifest + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + uiInteraction = $false + vmPowerChange = $false + hostReboot = $false + serviceRestart = $false + scheduledTaskModification = $false + } + Write-AgentRemoteDeployReceipt $receiptPath $receipt + Write-AgentResultAndExit $receipt $(if ($rollbackVerified) { 2 } else { 3 }) + } + + $receipt = [pscustomobject]@{ + schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" + status = "deployed_verified" + mode = "apply" + traceId = $traceId + runId = $runId + workItemId = $workItemId + sourceRevision = $sourceRevision + manifestSha256 = $manifestDigest + stagingPath = $stagePath + backupPath = $backupPath + validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut; status = [string]$validate.payload.status } + deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status } + runtimeManifest = $runtimeManifest + livePreflight = $livePreflight + idempotentReplay = $false + operationBoundaries = [pscustomobject]@{ + remoteWritePerformed = $true + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + environmentSecretRead = $false + uiInteraction = $false + vmPowerChange = $false + hostReboot = $false + serviceRestart = $false + scheduledTaskModification = $false + } + nextSafeAction = "run_existing_production_principal_verifier_and_same_trace_completion_readback" + } + Write-AgentRemoteDeployReceipt $receiptPath $receipt + Write-AgentResultAndExit $receipt 0 +} catch { + Write-AgentResultAndExit ([pscustomobject]@{ + schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" + status = "transport_receiver_failed" + errorType = $_.Exception.GetType().Name + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + rawEnvelopeStored = $false + environmentSecretRead = $false + uiInteraction = $false + vmPowerChange = $false + hostReboot = $false + serviceRestart = $false + }) 70 +} finally { + if ($script:DeployLock) { + try { $script:DeployLock.Dispose() } catch {} + } +} diff --git a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh new file mode 100755 index 000000000..fe5881509 --- /dev/null +++ b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh @@ -0,0 +1,316 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly DEFAULT_SOURCE_ROOT="$(CDPATH= cd -- "${SCRIPT_DIR}/../.." && pwd)" +readonly TARGET="Administrator@192.168.0.99" +readonly SOURCE_HOST_IPV4="192.168.0.110" +readonly RECEIVER_RELATIVE="scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1" +readonly PREFLIGHT_RELATIVE="scripts/reboot-recovery/agent99-live-preflight.ps1" +readonly WRAPPER_RELATIVE="scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh" + +RUNTIME_FILES=( + "agent99-bootstrap.ps1" + "agent99-contract-check.ps1" + "agent99-control-plane.ps1" + "agent99-deploy.ps1" + "agent99-register-tasks.ps1" + "agent99-sre-alert-inbox.ps1" + "agent99-sre-alert-relay.ps1" + "agent99-submit-request.ps1" + "agent99-synthetic-tests.ps1" + "agent99-telegram-inbox.ps1" + "agent99-windows-update-policy.ps1" + "host-reboot-scorecard.ps1" + "agent99.config.example.json" + "agent99.config.99.example.json" +) + +MODE="check" +MODE_SELECTION="default" +SOURCE_ROOT="${DEFAULT_SOURCE_ROOT}" +SOURCE_REVISION="" +TRACE_ID="" +RUN_ID="" +WORK_ITEM_ID="" +IDENTITY_FILE="" +KNOWN_HOSTS_FILE="${HOME:-}/.ssh/known_hosts" +CONNECT_TIMEOUT_SECONDS="8" +REMOTE_EXECUTION_TIMEOUT_SECONDS="45" + +usage() { + printf '%s\n' \ + "Usage: $0 [--check|--apply] [options]" \ + "" \ + "Default is a remote no-write check/plan against fixed target ${TARGET}." \ + "Apply requires --apply plus --trace-id, --run-id, and --work-item-id." \ + "" \ + "Options:" \ + " --check No-write transport and envelope check (default)." \ + " --apply Stage, ValidateOnly, bounded promote, and verify." \ + " --trace-id ID Controlled-apply trace identity." \ + " --run-id ID Controlled-apply run identity." \ + " --work-item-id ID Controlled-apply work-item identity." \ + " --source-root PATH Checkout root (default: repository root)." \ + " --source-revision SHA Exact 40-character Git commit (default: HEAD)." \ + " --identity-file PATH Existing SSH identity; wrapper never reads or prints its contents." \ + " --known-hosts-file PATH Pinned known_hosts path (default: ~/.ssh/known_hosts)." \ + " --connect-timeout SECONDS SSH connect timeout from 1 through 30 (default: 8)." \ + " -h, --help Show this help." +} + +fail() { + printf 'agent99_transport_error=%s\n' "$1" >&2 + exit 64 +} + +require_value() { + [[ $# -ge 2 && -n "$2" ]] || fail "missing_option_value" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --check) + [[ "${MODE_SELECTION}" != "apply" ]] || fail "conflicting_mode_flags" + MODE="check" + MODE_SELECTION="check" + shift + ;; + --apply) + [[ "${MODE_SELECTION}" != "check" ]] || fail "conflicting_mode_flags" + MODE="apply" + MODE_SELECTION="apply" + shift + ;; + --trace-id) + require_value "$@" + TRACE_ID="$2" + shift 2 + ;; + --run-id) + require_value "$@" + RUN_ID="$2" + shift 2 + ;; + --work-item-id) + require_value "$@" + WORK_ITEM_ID="$2" + shift 2 + ;; + --source-root) + require_value "$@" + SOURCE_ROOT="$2" + shift 2 + ;; + --source-revision) + require_value "$@" + SOURCE_REVISION="$2" + shift 2 + ;; + --identity-file) + require_value "$@" + IDENTITY_FILE="$2" + shift 2 + ;; + --known-hosts-file) + require_value "$@" + KNOWN_HOSTS_FILE="$2" + shift 2 + ;; + --connect-timeout) + require_value "$@" + CONNECT_TIMEOUT_SECONDS="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unsupported_option" + ;; + esac +done + +[[ "${CONNECT_TIMEOUT_SECONDS}" =~ ^[0-9]+$ ]] || fail "invalid_connect_timeout" +(( CONNECT_TIMEOUT_SECONDS >= 1 && CONNECT_TIMEOUT_SECONDS <= 30 )) || fail "invalid_connect_timeout" + +if [[ "${MODE}" == "apply" ]]; then + REMOTE_EXECUTION_TIMEOUT_SECONDS="1500" + readonly SAFE_ID_PATTERN='^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$' + [[ "${TRACE_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "apply_trace_id_missing_or_invalid" + [[ "${RUN_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "apply_run_id_missing_or_invalid" + [[ "${WORK_ITEM_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "apply_work_item_id_missing_or_invalid" +fi + +for dependency in git python3 ssh mktemp hostname timeout; do + command -v "${dependency}" >/dev/null 2>&1 || fail "required_command_missing_${dependency}" +done + +SOURCE_HOST_VERIFIED=0 +read -r -a LOCAL_ADDRESSES <<<"$(hostname -I 2>/dev/null || true)" +for local_address in "${LOCAL_ADDRESSES[@]}"; do + if [[ "${local_address%%/*}" == "${SOURCE_HOST_IPV4}" ]]; then + SOURCE_HOST_VERIFIED=1 + break + fi +done +[[ "${SOURCE_HOST_VERIFIED}" == "1" ]] || fail "source_host_is_not_110" + +[[ -d "${SOURCE_ROOT}" ]] || fail "source_root_missing" +SOURCE_ROOT="$(CDPATH= cd -- "${SOURCE_ROOT}" && pwd)" +[[ -f "${KNOWN_HOSTS_FILE}" ]] || fail "known_hosts_file_missing" +if [[ -n "${IDENTITY_FILE}" ]]; then + [[ -f "${IDENTITY_FILE}" ]] || fail "identity_file_missing" +fi + +if [[ -z "${SOURCE_REVISION}" ]]; then + SOURCE_REVISION="$(git -C "${SOURCE_ROOT}" rev-parse --verify 'HEAD^{commit}')" || fail "source_revision_unavailable" +fi +SOURCE_REVISION="$(printf '%s' "${SOURCE_REVISION}" | tr '[:upper:]' '[:lower:]')" +[[ "${SOURCE_REVISION}" =~ ^[0-9a-f]{40}$ ]] || fail "source_revision_must_be_full_commit_sha" +git -C "${SOURCE_ROOT}" cat-file -e "${SOURCE_REVISION}^{commit}" 2>/dev/null || fail "source_revision_not_found" + +SOURCE_BOUND_FILES=( + "${RUNTIME_FILES[@]}" + "${PREFLIGHT_RELATIVE}" + "${RECEIVER_RELATIVE}" + "${WRAPPER_RELATIVE}" +) +for relative_path in "${SOURCE_BOUND_FILES[@]}"; do + [[ -f "${SOURCE_ROOT}/${relative_path}" ]] || fail "source_bound_file_missing" + git -C "${SOURCE_ROOT}" ls-files --error-unmatch -- "${relative_path}" >/dev/null 2>&1 || fail "source_bound_file_untracked" +done +git -C "${SOURCE_ROOT}" diff --quiet "${SOURCE_REVISION}" -- "${SOURCE_BOUND_FILES[@]}" || fail "source_bundle_differs_from_revision" + +umask 077 +WORK_DIR="$(mktemp -d /tmp/awoooi-agent99-transport.XXXXXXXX)" +cleanup() { + rm -rf -- "${WORK_DIR}" +} +trap cleanup EXIT HUP INT TERM + +ENVELOPE_PATH="${WORK_DIR}/envelope.json" +BOOTSTRAP_PATH="${WORK_DIR}/remote-bootstrap.ps1" + +python3 - \ + "${SOURCE_ROOT}" \ + "${SOURCE_REVISION}" \ + "${MODE}" \ + "${TRACE_ID}" \ + "${RUN_ID}" \ + "${WORK_ITEM_ID}" \ + "${ENVELOPE_PATH}" \ + "${RUNTIME_FILES[@]}" <<'PY' +from __future__ import annotations + +import base64 +import hashlib +import json +import sys +from pathlib import Path + +source_root = Path(sys.argv[1]) +source_revision, mode = sys.argv[2:4] +trace_id, run_id, work_item_id = sys.argv[4:7] +output_path = Path(sys.argv[7]) +runtime_names = sys.argv[8:] + +if len(runtime_names) != 14 or len(set(runtime_names)) != 14: + raise SystemExit("fixed_runtime_file_contract_failed") + +files: list[dict[str, str]] = [] +manifest_rows: list[str] = [] +for name in runtime_names: + if Path(name).name != name: + raise SystemExit("runtime_filename_not_flat") + content = (source_root / name).read_bytes() + digest = hashlib.sha256(content).hexdigest() + files.append( + { + "name": name, + "sha256": digest, + "contentBase64": base64.b64encode(content).decode("ascii"), + } + ) + manifest_rows.append(f"{name}\t{digest}\n") + +manifest_sha256 = hashlib.sha256("".join(manifest_rows).encode("utf-8")).hexdigest() +preflight_path = source_root / "scripts/reboot-recovery/agent99-live-preflight.ps1" +preflight_content = preflight_path.read_bytes() + +envelope = { + "schemaVersion": "agent99_remote_atomic_deploy_envelope_v1", + "sourceHostAlias": "110", + "targetHostAlias": "99", + "transport": "110_to_99_ssh_publickey", + "mode": mode, + "traceId": trace_id, + "runId": run_id, + "workItemId": work_item_id, + "sourceRevision": source_revision, + "expectedRuntimeFileCount": 14, + "manifestSha256": manifest_sha256, + "files": files, + "livePreflight": { + "name": preflight_path.name, + "sha256": hashlib.sha256(preflight_content).hexdigest(), + "contentBase64": base64.b64encode(preflight_content).decode("ascii"), + }, +} +output_path.write_text( + json.dumps(envelope, separators=(",", ":"), ensure_ascii=True), + encoding="utf-8", +) +PY + +python3 - \ + "${SOURCE_ROOT}/${RECEIVER_RELATIVE}" \ + "${ENVELOPE_PATH}" \ + "${BOOTSTRAP_PATH}" <<'PY' +from __future__ import annotations + +import base64 +import sys +from pathlib import Path + +receiver_path, envelope_path, output_path = map(Path, sys.argv[1:4]) +receiver_base64 = base64.b64encode(receiver_path.read_bytes()).decode("ascii") +envelope_base64 = base64.b64encode(envelope_path.read_bytes()).decode("ascii") +bootstrap = ( + "$ErrorActionPreference = 'Stop'\r\n" + f"$receiverText = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('{receiver_base64}'))\r\n" + f"$envelopeText = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('{envelope_base64}'))\r\n" + "& ([ScriptBlock]::Create($receiverText)) -EnvelopeJson $envelopeText\r\n" +) +output_path.write_text(bootstrap, encoding="utf-8") +PY + +SSH_OPTIONS=( + -T + -o BatchMode=yes + -o PreferredAuthentications=publickey + -o PubkeyAuthentication=yes + -o PasswordAuthentication=no + -o KbdInteractiveAuthentication=no + -o NumberOfPasswordPrompts=0 + -o StrictHostKeyChecking=yes + -o "UserKnownHostsFile=${KNOWN_HOSTS_FILE}" + -o "ConnectTimeout=${CONNECT_TIMEOUT_SECONDS}" + -o ConnectionAttempts=1 + -o ServerAliveInterval=5 + -o ServerAliveCountMax=2 + -o LogLevel=ERROR + -o IdentitiesOnly=yes +) +if [[ -n "${IDENTITY_FILE}" ]]; then + SSH_OPTIONS+=( -i "${IDENTITY_FILE}" ) +fi + +readonly REMOTE_COMMAND='powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command "$inputText=[Console]::In.ReadToEnd(); & ([ScriptBlock]::Create($inputText))"' +set +e +timeout --signal=TERM --kill-after=10 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}s" \ + ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_COMMAND}" <"${BOOTSTRAP_PATH}" +status=$? +set -e +exit "${status}" diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py new file mode 100644 index 000000000..0a92ac623 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -0,0 +1,318 @@ +from __future__ import annotations + +import os +import re +import shlex +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +SENDER = ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh" +RECEIVER = ( + ROOT + / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1" +) +DEPLOYER = ROOT / "agent99-deploy.ps1" + +EXPECTED_RUNTIME_FILES = ( + "agent99-bootstrap.ps1", + "agent99-contract-check.ps1", + "agent99-control-plane.ps1", + "agent99-deploy.ps1", + "agent99-register-tasks.ps1", + "agent99-sre-alert-inbox.ps1", + "agent99-sre-alert-relay.ps1", + "agent99-submit-request.ps1", + "agent99-synthetic-tests.ps1", + "agent99-telegram-inbox.ps1", + "agent99-windows-update-policy.ps1", + "host-reboot-scorecard.ps1", + "agent99.config.example.json", + "agent99.config.99.example.json", +) + + +def _quoted_array(source: str, start: str, end: str) -> tuple[str, ...]: + body = source[source.index(start) + len(start) :] + body = body[: body.index(end)] + return tuple(re.findall(r'"([^"\n]+)"', body)) + + +def test_sender_is_valid_shell_with_check_as_the_default() -> None: + result = subprocess.run( + ["bash", "-n", str(SENDER)], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + source = SENDER.read_text(encoding="utf-8") + assert 'MODE="check"' in source + assert "--apply" in source + assert 'MODE="apply"' in source + assert "apply_trace_id_missing_or_invalid" in source + assert "apply_run_id_missing_or_invalid" in source + assert "apply_work_item_id_missing_or_invalid" in source + assert 'MODE_SELECTION="default"' in source + assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="45"' in source + assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="1500"' in source + assert "timeout --signal=TERM --kill-after=10" in source + + +def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None: + sender = SENDER.read_text(encoding="utf-8") + receiver = RECEIVER.read_text(encoding="utf-8") + deployer = DEPLOYER.read_text(encoding="utf-8") + + sender_files = _quoted_array(sender, "RUNTIME_FILES=(", ")") + receiver_files = _quoted_array(receiver, "$FixedRuntimeFiles = @(", ")") + deployer_files = _quoted_array(deployer, "$runtimeFiles = @(", ")") + + assert sender_files == EXPECTED_RUNTIME_FILES + assert receiver_files == EXPECTED_RUNTIME_FILES + assert deployer_files == EXPECTED_RUNTIME_FILES + assert "expectedRuntimeFileCount\": 14" in sender + assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver + assert "required_runtime_file_missing" in receiver + assert "duplicate_runtime_file" in receiver + + +def test_sender_is_fixed_to_110_to_windows99_public_key_transport() -> None: + source = SENDER.read_text(encoding="utf-8") + + assert 'readonly TARGET="Administrator@192.168.0.99"' in source + assert 'readonly SOURCE_HOST_IPV4="192.168.0.110"' in source + assert "source_host_is_not_110" in source + assert '"sourceHostAlias": "110"' in source + assert '"transport": "110_to_99_ssh_publickey"' in source + assert "-o BatchMode=yes" in source + assert "-o PreferredAuthentications=publickey" in source + assert "-o PubkeyAuthentication=yes" in source + assert "-o PasswordAuthentication=no" in source + assert "-o KbdInteractiveAuthentication=no" in source + assert "-o NumberOfPasswordPrompts=0" in source + assert "-o StrictHostKeyChecking=yes" in source + assert 'UserKnownHostsFile=${KNOWN_HOSTS_FILE}' in source + assert "-o IdentitiesOnly=yes" in source + assert 'SSH_OPTIONS+=( -i "${IDENTITY_FILE}" )' in source + assert "StrictHostKeyChecking=no" not in source + assert "StrictHostKeyChecking=accept-new" not in source + assert "192.168.0.112" not in source + assert "--host" not in source + assert "--user" not in source + + +def test_sender_binds_every_transported_source_to_a_full_git_revision() -> None: + source = SENDER.read_text(encoding="utf-8") + + assert "source_revision_must_be_full_commit_sha" in source + assert "source_revision_not_found" in source + assert "source_bound_file_untracked" in source + assert "source_bundle_differs_from_revision" in source + assert 'git -C "${SOURCE_ROOT}" diff --quiet "${SOURCE_REVISION}"' in source + assert '"${PREFLIGHT_RELATIVE}"' in source + assert '"${RECEIVER_RELATIVE}"' in source + assert '"${WRAPPER_RELATIVE}"' in source + + +def test_sender_builds_sha256_envelope_and_streams_it_without_remote_cli_data() -> None: + source = SENDER.read_text(encoding="utf-8") + + assert '"schemaVersion": "agent99_remote_atomic_deploy_envelope_v1"' in source + assert 'manifest_rows.append(f"{name}\\t{digest}\\n")' in source + assert "hashlib.sha256(content).hexdigest()" in source + assert '"manifestSha256": manifest_sha256' in source + assert '"contentBase64": base64.b64encode(content).decode("ascii")' in source + assert "[Console]::In.ReadToEnd()" in source + assert '<"${BOOTSTRAP_PATH}"' in source + assert "rawEnvelope" not in source + assert "sshpass" not in source + + +def test_receiver_validates_manifest_before_staging_and_validate_only_before_promote() -> None: + source = RECEIVER.read_text(encoding="utf-8") + + for marker in ( + "runtime_file_sha256_mismatch", + "manifest_sha256_mismatch", + "live_preflight_sha256_mismatch", + "agent99_remote_source_manifest_v1", + "source-manifest.json", + 'Join-Path $DeployRoot "remote-source-$stageToken"', + '"$stagePath.partial.$PID.', + ): + assert marker in source + + manifest_check = source.index("manifest_sha256_mismatch") + stage_write = source.index("[IO.File]::WriteAllBytes") + validate = source.index( + "$validate = Invoke-AgentDeployChild -SourceRoot $stagePath " + "-SourceRevision $sourceRevision -ValidateOnly" + ) + promote = source.index( + "$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath " + "-SourceRevision $sourceRevision" + ) + assert manifest_check < stage_write < validate < promote + assert 'status = "validate_only_failed_no_promotion"' in source + assert "liveRuntimeWritePerformed = $false" in source + assert "livePromotionPerformed = $false" in source + + +def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier() -> None: + source = RECEIVER.read_text(encoding="utf-8") + + assert "function Restore-AgentBundle" in source + assert "Test-AgentBackupPath" in source + assert 'Join-Path $DeployRoot "backup-"' in source + assert source.count("Restore-AgentBundle $backupPath") == 2 + assert 'status = if ($rollbackVerified) { "deploy_failed_rollback_verified"' in source + assert 'status = if ($rollbackVerified) { "rolled_back_post_verifier_failed"' in source + assert "Get-AgentLiveBundleDigest" in source + assert "Test-AgentManifestEqual $beforeManifest $afterManifest" in source + assert "Get-AgentSha256File $ConfigPath" in source + assert "Get-AgentSha256File $ManifestPath" in source + assert "$actualRuntimeSha = Get-AgentSha256File" in source + assert "$recordedRuntimeSha -ne $actualRuntimeSha" in source + assert "Invoke-AgentLivePreflight" in source + assert "function Invoke-AgentBoundedPowerShell" in source + assert "$process.WaitForExit($TimeoutSeconds * 1000)" in source + assert "$process.Kill()" in source + assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source + assert "$livePreflight.sourceRevision -eq $sourceRevision" in source + assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source + assert '$runtimeManifest.fileCount -eq 14' in source + assert '$runtimeManifest.mismatchCount -eq 0' in source + assert 'status = "deployed_verified"' in source + + +def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary() -> None: + source = RECEIVER.read_text(encoding="utf-8") + + assert "remote-atomic-deploy.lock" in source + assert "FileShare]::None" in source + assert 'status = "blocked_single_flight_lock_unavailable"' in source + assert "idempotentReplay" in source + assert "idempotent_replay_post_verifier_failed_no_promotion" in source + assert "originalOperationBoundaries" in source + assert source.count("Invoke-AgentLivePreflight") >= 3 + assert 'status = "check_ready"' in source + assert "remoteWritePerformed = $false" in source + assert "secretValueRead = $false" in source + assert "privateKeyValueRead = $false" in source + assert "tokenValueRead = $false" in source + assert "environmentSecretRead = $false" in source + assert "rawEnvelopeStored = $false" in source + assert "$env:" not in source + assert "GetEnvironmentVariable" not in source + + +def test_transport_does_not_expose_ui_vm_power_reboot_or_service_actions() -> None: + source = "\n".join( + ( + SENDER.read_text(encoding="utf-8"), + RECEIVER.read_text(encoding="utf-8"), + ) + ) + forbidden_actions = ( + "Restart-Computer", + "Restart-Service", + "Start-Service", + "Stop-Service", + "vmrun", + "Start-VM", + "Stop-VM", + "shutdown.exe", + "Register-ScheduledTask", + "Unregister-ScheduledTask", + "-RegisterTasks", + ) + for action in forbidden_actions: + assert action not in source + + assert "uiInteraction = $false" in source + assert "vmPowerChange = $false" in source + assert "hostReboot = $false" in source + assert "serviceRestart = $false" in source + assert "scheduledTaskModification = $false" in source + assert "host112" not in source.lower() + assert "192.168.0.112" not in source + + +def test_apply_without_all_three_identities_fails_before_ssh() -> None: + result = subprocess.run( + ["bash", str(SENDER), "--apply"], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 64 + assert "apply_trace_id_missing_or_invalid" in result.stderr + assert "known_hosts_file_missing" not in result.stderr + + +def test_check_mode_builds_and_streams_bundle_over_bounded_fake_transport( + tmp_path: Path, +) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + args_path = tmp_path / "ssh-args.txt" + bytes_path = tmp_path / "ssh-stdin-bytes.txt" + known_hosts = tmp_path / "known_hosts" + identity = tmp_path / "existing_identity" + known_hosts.write_text("fixed-host-key-placeholder\n", encoding="utf-8") + identity.write_text("identity-content-is-not-read-by-wrapper\n", encoding="utf-8") + + fake_hostname = fake_bin / "hostname" + fake_hostname.write_text( + "#!/usr/bin/env bash\n" + "[[ \"${1:-}\" == '-I' ]] || exit 65\n" + "printf '%s\\n' '192.168.0.110 127.0.0.1'\n", + encoding="utf-8", + ) + fake_ssh = fake_bin / "ssh" + fake_ssh.write_text( + "#!/usr/bin/env bash\n" + f"printf '%s\\n' \"$@\" >{shlex.quote(str(args_path))}\n" + f"wc -c | tr -d ' ' >{shlex.quote(str(bytes_path))}\n" + "printf '%s\\n' '{\"status\":\"check_ready\"}'\n", + encoding="utf-8", + ) + fake_timeout = fake_bin / "timeout" + fake_timeout.write_text( + "#!/usr/bin/env bash\n" + "shift 3\n" + "exec \"$@\"\n", + encoding="utf-8", + ) + for executable in (fake_hostname, fake_ssh, fake_timeout): + executable.chmod(0o700) + + environment = os.environ.copy() + environment["PATH"] = f"{fake_bin}:{environment['PATH']}" + result = subprocess.run( + [ + "bash", + str(SENDER), + "--check", + "--known-hosts-file", + str(known_hosts), + "--identity-file", + str(identity), + ], + check=False, + capture_output=True, + text=True, + env=environment, + ) + + assert result.returncode == 0, result.stderr + assert '"status":"check_ready"' in result.stdout + args = args_path.read_text(encoding="utf-8") + assert "BatchMode=yes" in args + assert "StrictHostKeyChecking=yes" in args + assert "PasswordAuthentication=no" in args + assert "Administrator@192.168.0.99" in args + assert str(identity) in args + assert int(bytes_path.read_text(encoding="utf-8")) > 500_000