diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index 8c7e86617..1508e9fdc 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -13,6 +13,9 @@ $EvidenceDir = Join-Path $AgentRoot "evidence" $DeployRoot = Join-Path $AgentRoot "deploy" $ManifestPath = Join-Path $StateDir "runtime-manifest.json" $WindowsPowerShell = "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" +$RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay" +$RelayTaskPath = "\" +$RelayPort = 8787 $FixedRuntimeFiles = @( "agent99-bootstrap.ps1", "agent99-contract-check.ps1", @@ -251,12 +254,183 @@ function Invoke-AgentLivePreflight { healthyTaskCount = if ($summary) { [int]$summary.healthyTaskCount } else { 0 } requiredTaskCount = if ($summary) { [int]$summary.requiredTaskCount } else { 0 } relayListenerCount = if ($summary) { [int]$summary.relayListenerCount } else { 0 } + relayListenerGenerations = if ($payload) { + @($payload.relayListeners | ForEach-Object { + $processId = [int]$_.processId + $process = Get-Process -Id $processId -ErrorAction SilentlyContinue + if ($process) { + try { + $startedAt = $process.StartTime.ToUniversalTime().ToString("o") + Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes("$processId|$startedAt")) + } catch {} + } + } | Where-Object { $_ } | Sort-Object -Unique) + } else { @() } pendingCommandCount = if ($summary) { [int]$summary.pendingCommandCount } else { -1 } preflightGreen = [bool]($summary -and $summary.preflightGreen) blockingReasons = if ($processResult.timedOut) { @("preflight_process_timeout") } elseif ($summary) { @($summary.blockingReasons | ForEach-Object { [string]$_ }) } else { @("preflight_output_unparseable") } } } +function Get-AgentRelayRuntime { + $task = $null + $taskError = "" + try { + $tasks = @(Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop) + if ($tasks.Count -ne 1) { + $taskError = "exact_relay_task_count_$($tasks.Count)" + } else { + $task = $tasks[0] + } + } catch { + $taskError = $_.Exception.GetType().Name + } + + $listeners = @() + $listenerError = "" + try { + $connections = @(Get-NetTCPConnection -LocalPort $RelayPort -State Listen -ErrorAction Stop) + foreach ($connection in $connections) { + $processId = [int]$connection.OwningProcess + $process = Get-Process -Id $processId -ErrorAction SilentlyContinue + $startedAt = "" + $generation = "" + if ($process) { + try { + $startedAt = $process.StartTime.ToUniversalTime().ToString("o") + $generation = Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes("$processId|$startedAt")) + } catch {} + } + $listeners += [pscustomobject]@{ + address = [string]$connection.LocalAddress + port = [int]$connection.LocalPort + processId = $processId + processName = if ($process) { [string]$process.ProcessName } else { "" } + startedAt = $startedAt + generation = $generation + } + } + } catch { + $listenerError = $_.Exception.GetType().Name + } + + $processIds = @($listeners | ForEach-Object { [int]$_.processId } | Sort-Object -Unique) + $listenerGenerations = @($listeners | ForEach-Object { [string]$_.generation } | Where-Object { $_ } | Sort-Object -Unique) + $taskEnabled = $false + if ($task) { + try { $taskEnabled = [bool]$task.Settings.Enabled } catch { $taskEnabled = $false } + } + return [pscustomobject]@{ + taskName = $RelayTaskName + taskPath = $RelayTaskPath + taskExists = [bool]($null -ne $task) + taskState = if ($task) { [string]$task.State } else { "" } + taskRunning = [bool]($task -and [string]$task.State -eq "Running") + taskEnabled = $taskEnabled + taskError = $taskError + listenerPort = $RelayPort + listenerCount = $listeners.Count + listenerProcessCount = $processIds.Count + listenerGenerations = $listenerGenerations + generationReady = [bool]($listeners.Count -gt 0 -and $processIds.Count -gt 0 -and $listenerGenerations.Count -eq $processIds.Count) + listeners = $listeners + listenerError = $listenerError + } +} + +function Test-AgentGenerationIntersection { + param([string[]]$Left, [string[]]$Right) + + foreach ($value in @($Left)) { + if ($value -and $value -in @($Right)) { return $true } + } + return $false +} + +function Invoke-AgentRelayTaskRestart { + param([string[]]$ForbiddenGenerations = @()) + + $before = Get-AgentRelayRuntime + $generationCandidates = @($ForbiddenGenerations) + @($before.listenerGenerations) + $forbidden = @($generationCandidates | Where-Object { $_ } | Sort-Object -Unique) + if (-not $before.taskExists -or -not $before.taskEnabled) { + return [pscustomobject]@{ + attempted = $false + ok = $false + stage = "precondition" + reason = if ($before.taskExists) { "exact_existing_relay_task_disabled" } else { "exact_existing_relay_task_missing" } + before = $before + after = $before + forbiddenGenerations = $forbidden + oldGenerationsGone = $false + newGenerationVerified = $false + } + } + + $stage = "stop" + $afterStop = $before + $after = $before + try { + Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop + $stopDeadline = (Get-Date).AddSeconds(60) + do { + Start-Sleep -Seconds 1 + $afterStop = Get-AgentRelayRuntime + $oldGone = -not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden) + $stopVerified = [bool]($oldGone -and -not $afterStop.taskRunning) + if ($stopVerified) { break } + } while ((Get-Date) -lt $stopDeadline) + if (-not $stopVerified) { throw "relay_old_listener_generation_stop_timeout" } + + $stage = "start" + Start-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop + $startDeadline = (Get-Date).AddSeconds(120) + do { + Start-Sleep -Seconds 1 + $after = Get-AgentRelayRuntime + $oldGone = -not (Test-AgentGenerationIntersection @($after.listenerGenerations) $forbidden) + $newGenerations = @($after.listenerGenerations | Where-Object { $_ -and $_ -notin $forbidden }) + $newVerified = [bool]( + $after.taskRunning -and + $after.taskEnabled -and + $after.generationReady -and + $after.listenerCount -gt 0 -and + $oldGone -and + $newGenerations.Count -gt 0 + ) + if ($newVerified) { break } + } while ((Get-Date) -lt $startDeadline) + if (-not $newVerified) { throw "relay_new_listener_generation_start_timeout" } + + return [pscustomobject]@{ + attempted = $true + ok = $true + stage = "verified" + reason = "exact_existing_relay_task_restarted" + before = $before + afterStop = $afterStop + after = $after + forbiddenGenerations = $forbidden + oldGenerationsGone = $oldGone + newGenerationVerified = $newVerified + } + } catch { + return [pscustomobject]@{ + attempted = $true + ok = $false + stage = $stage + reason = "relay_task_restart_$($stage)_failed" + errorType = $_.Exception.GetType().Name + before = $before + afterStop = $afterStop + after = (Get-AgentRelayRuntime) + forbiddenGenerations = $forbidden + oldGenerationsGone = [bool](-not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden)) + newGenerationVerified = $false + } + } +} + function Copy-AgentFileWithRetry { param([string]$Source, [string]$Destination) @@ -275,9 +449,13 @@ function Test-AgentBackupPath { param([string]$BackupPath) if (-not $BackupPath) { return $false } - $expectedPrefix = [IO.Path]::GetFullPath((Join-Path $DeployRoot "backup-")) - $candidate = [IO.Path]::GetFullPath($BackupPath) - return [bool]($candidate.StartsWith($expectedPrefix, [StringComparison]::OrdinalIgnoreCase)) + try { + $expectedPrefix = [IO.Path]::GetFullPath((Join-Path $DeployRoot "backup-")) + $candidate = [IO.Path]::GetFullPath($BackupPath) + return [bool]($candidate.StartsWith($expectedPrefix, [StringComparison]::OrdinalIgnoreCase)) + } catch { + return $false + } } function Restore-AgentBundle { @@ -394,7 +572,7 @@ try { $preflightBytes = [Convert]::FromBase64String([string]$preflight.contentBase64) if ((Get-AgentSha256Bytes $preflightBytes) -ne ([string]$preflight.sha256).ToLowerInvariant()) { throw "live_preflight_sha256_mismatch" } - $stageIdentity = if ($mode -eq "apply") { "$runId|$sourceRevision" } else { "check|$sourceRevision" } + $stageIdentity = "$mode|$traceId|$runId|$workItemId|$sourceRevision" $stageToken = (Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes($stageIdentity))).Substring(0, 20) $stagePath = Join-Path $DeployRoot "remote-source-$stageToken" $receiptPath = Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken.json" @@ -421,6 +599,7 @@ try { vmPowerChange = $false hostReboot = $false serviceRestart = $false + scheduledTaskRestart = $false scheduledTaskModification = $false } nextSafeAction = "rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check" @@ -492,6 +671,8 @@ try { [string]$prior.runId -eq $runId -and [string]$prior.workItemId -eq $workItemId -and [string]$prior.sourceRevision -eq $sourceRevision -and + $prior.PSObject.Properties["relayReload"] -and + [bool]$prior.relayReload.newGenerationVerified -and $live.runtimeMatched -and $live.sourceRevision -eq $sourceRevision -and $live.fileCount -eq 14 -and @@ -516,6 +697,7 @@ try { vmPowerChange = $false hostReboot = $false serviceRestart = $false + scheduledTaskRestart = $false scheduledTaskModification = $false }) -Force Write-AgentResultAndExit $prior 0 @@ -568,33 +750,22 @@ try { foreach ($name in $FixedRuntimeFiles) { $beforeFilePresence[$name] = Test-Path -LiteralPath (Join-Path $BinDir $name) -PathType Leaf } $beforeConfigPresent = Test-Path -LiteralPath $ConfigPath -PathType Leaf $beforeManifestPresent = Test-Path -LiteralPath $ManifestPath -PathType Leaf + $beforeRelayRuntime = Get-AgentRelayRuntime $backupDirsBefore = @{} foreach ($directory in @(Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue)) { $backupDirsBefore[$directory.FullName] = $true } - $deploy = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision - $backupPath = if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) { [string]$deploy.payload.backupDir } else { "" } - if (-not $backupPath) { - $newBackup = Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue | - Where-Object { -not $backupDirsBefore.ContainsKey($_.FullName) } | - Sort-Object LastWriteTime -Descending | - Select-Object -First 1 - if ($newBackup) { $backupPath = $newBackup.FullName } - } - - if (-not $deploy.ok -or -not $deploy.payload -or [string]$deploy.payload.status -ne "deployed" -or $deploy.payload.taskRegistration) { - $rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent - $afterManifest = Get-AgentSafeRuntimeManifest - $rollbackVerified = [bool]( - (Test-AgentManifestEqual $beforeManifest $afterManifest) -and - (Get-AgentLiveBundleDigest) -eq $beforeBundleDigest -and - (Get-AgentSha256File $ConfigPath) -eq $beforeConfigDigest -and - (Get-AgentSha256File $ManifestPath) -eq $beforeManifestDigest - ) + if ( + -not $beforeRelayRuntime.taskExists -or + -not $beforeRelayRuntime.taskEnabled -or + -not $beforeRelayRuntime.taskRunning -or + -not $beforeRelayRuntime.generationReady -or + $beforeRelayRuntime.listenerCount -lt 1 + ) { $receipt = [pscustomobject]@{ schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" - status = if ($rollbackVerified) { "deploy_failed_rollback_verified" } else { "deploy_failed_rollback_unverified" } + status = "blocked_relay_runtime_precondition_no_promotion" mode = "apply" traceId = $traceId runId = $runId @@ -602,76 +773,208 @@ try { sourceRevision = $sourceRevision manifestSha256 = $manifestDigest stagingPath = $stagePath - backupPath = $backupPath validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut } - deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut } - rollback = $rollback - rollbackVerified = $rollbackVerified - runtimeManifest = $afterManifest + relayRuntime = $beforeRelayRuntime + remoteWritePerformed = $true + liveRuntimeWritePerformed = $false + livePromotionPerformed = $false secretValueRead = $false privateKeyValueRead = $false tokenValueRead = $false + environmentSecretRead = $false uiInteraction = $false vmPowerChange = $false hostReboot = $false serviceRestart = $false + scheduledTaskRestart = $false scheduledTaskModification = $false + nextSafeAction = "repair_exact_existing_relay_task_runtime_then_retry_same_identity" } Write-AgentRemoteDeployReceipt $receiptPath $receipt - Write-AgentResultAndExit $receipt $(if ($rollbackVerified) { 2 } else { 3 }) + Write-AgentResultAndExit $receipt 2 } - $runtimeManifest = Get-AgentSafeRuntimeManifest - $livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") - $postVerified = [bool]( - $runtimeManifest.exists -and - $runtimeManifest.runtimeMatched -and - $runtimeManifest.sourceRevision -eq $sourceRevision -and - $runtimeManifest.fileCount -eq 14 -and - $runtimeManifest.mismatchCount -eq 0 -and - $livePreflight.ok -and - $livePreflight.sourceRevision -eq $sourceRevision -and - $livePreflight.runtimeMatched - ) - - if (-not $postVerified) { - $rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent - $afterManifest = Get-AgentSafeRuntimeManifest - $rollbackVerified = [bool]( - $rollback.restored -and - (Test-AgentManifestEqual $beforeManifest $afterManifest) -and - (Get-AgentLiveBundleDigest) -eq $beforeBundleDigest -and - (Get-AgentSha256File $ConfigPath) -eq $beforeConfigDigest -and - (Get-AgentSha256File $ManifestPath) -eq $beforeManifestDigest - ) - $receipt = [pscustomobject]@{ - schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" - status = if ($rollbackVerified) { "rolled_back_post_verifier_failed" } else { "rollback_failed_after_post_verifier" } - mode = "apply" - traceId = $traceId - runId = $runId - workItemId = $workItemId - sourceRevision = $sourceRevision - manifestSha256 = $manifestDigest - stagingPath = $stagePath - backupPath = $backupPath - validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut } - deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut } - failedRuntimeManifest = $runtimeManifest - failedLivePreflight = $livePreflight - rollback = $rollback - rollbackVerified = $rollbackVerified - runtimeManifest = $afterManifest - secretValueRead = $false - privateKeyValueRead = $false - tokenValueRead = $false - uiInteraction = $false - vmPowerChange = $false - hostReboot = $false - serviceRestart = $false - scheduledTaskModification = $false + $deploy = $null + $relayReload = $null + $runtimeManifest = $null + $livePreflight = $null + $postVerified = $false + $failurePhase = "" + $failureType = "" + $backupPath = "" + try { + $deploy = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision + if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) { + $backupPath = [string]$deploy.payload.backupDir + } + if (-not $deploy.ok -or -not $deploy.payload -or [string]$deploy.payload.status -ne "deployed" -or $deploy.payload.taskRegistration) { + $failurePhase = "deploy" + $failureType = "deploy_child_failed_or_invalid" + } else { + $relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations @($beforeRelayRuntime.listenerGenerations) + if (-not $relayReload.ok) { + $failurePhase = "relay_reload" + $failureType = "relay_reload_failed" + } else { + $runtimeManifest = Get-AgentSafeRuntimeManifest + $livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") + $postVerified = [bool]( + $runtimeManifest.exists -and + $runtimeManifest.runtimeMatched -and + $runtimeManifest.sourceRevision -eq $sourceRevision -and + $runtimeManifest.fileCount -eq 14 -and + $runtimeManifest.mismatchCount -eq 0 -and + $relayReload.oldGenerationsGone -and + $relayReload.newGenerationVerified -and + $relayReload.after.taskEnabled -and + $relayReload.after.taskRunning -and + $livePreflight.ok -and + $livePreflight.sourceRevision -eq $sourceRevision -and + $livePreflight.runtimeMatched -and + $livePreflight.relayListenerCount -gt 0 -and + (Test-AgentGenerationIntersection @($livePreflight.relayListenerGenerations) @($relayReload.after.listenerGenerations)) + ) + if (-not $postVerified) { + $failurePhase = "post_verifier" + $failureType = "independent_post_verifier_failed" + } + } + } + } catch { + $failurePhase = if ($deploy -and $deploy.ok) { "reload_or_post_verifier_exception" } else { "deploy_exception" } + $failureType = $_.Exception.GetType().Name + } + + if ($failurePhase) { + if (-not $backupPath) { + $newBackup = Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue | + Where-Object { -not $backupDirsBefore.ContainsKey($_.FullName) } | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 + if ($newBackup) { $backupPath = $newBackup.FullName } + } + $rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent + $rollbackForbiddenGenerations = @($beforeRelayRuntime.listenerGenerations) + if ($relayReload) { + $rollbackForbiddenGenerations += @($relayReload.before.listenerGenerations) + $rollbackForbiddenGenerations += @($relayReload.after.listenerGenerations) + } + $relayBeforeRollback = Get-AgentRelayRuntime + $rollbackForbiddenGenerations += @($relayBeforeRollback.listenerGenerations) + $rollbackForbiddenGenerations = @($rollbackForbiddenGenerations | Where-Object { $_ } | Sort-Object -Unique) + $rollbackRelayRestart = if ($rollback.restored) { + Invoke-AgentRelayTaskRestart -ForbiddenGenerations $rollbackForbiddenGenerations + } else { + [pscustomobject]@{ + attempted = $false + ok = $false + stage = "restore" + reason = "bundle_restore_not_verified_relay_restart_withheld" + before = $relayBeforeRollback + after = $relayBeforeRollback + forbiddenGenerations = $rollbackForbiddenGenerations + oldGenerationsGone = $false + newGenerationVerified = $false + } + } + $afterManifest = [pscustomobject]@{ + exists = $false + sourceRevision = "" + runtimeMatched = $false + fileCount = 0 + mismatchCount = -1 + parseError = "rollback_verifier_not_completed" + } + $rollbackVerified = $false + $rollbackVerifierErrorType = "" + try { + $afterManifest = Get-AgentSafeRuntimeManifest + $rollbackVerified = [bool]( + $rollback.restored -and + (Test-AgentManifestEqual $beforeManifest $afterManifest) -and + (Get-AgentLiveBundleDigest) -eq $beforeBundleDigest -and + (Get-AgentSha256File $ConfigPath) -eq $beforeConfigDigest -and + (Get-AgentSha256File $ManifestPath) -eq $beforeManifestDigest -and + $rollbackRelayRestart.ok -and + $rollbackRelayRestart.oldGenerationsGone -and + $rollbackRelayRestart.newGenerationVerified -and + $rollbackRelayRestart.after.taskEnabled -and + $rollbackRelayRestart.after.taskRunning -and + $rollbackRelayRestart.after.listenerCount -gt 0 + ) + } catch { + $rollbackVerified = $false + $rollbackVerifierErrorType = $_.Exception.GetType().Name + } + try { + $receipt = [pscustomobject]@{ + schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" + status = if ($failurePhase -eq "deploy") { + if ($rollbackVerified) { "deploy_failed_rollback_verified" } else { "deploy_failed_rollback_unverified" } + } else { + if ($rollbackVerified) { "rolled_back_post_verifier_failed" } else { "rollback_failed_after_post_verifier" } + } + mode = "apply" + traceId = $traceId + runId = $runId + workItemId = $workItemId + sourceRevision = $sourceRevision + manifestSha256 = $manifestDigest + stagingPath = $stagePath + backupPath = $backupPath + validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut } + deploy = if ($deploy) { [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut } } else { [pscustomobject]@{ ok = $false; exitCode = -1; timedOut = $false } } + failurePhase = $failurePhase + failureType = $failureType + relayReload = $relayReload + failedRuntimeManifest = $runtimeManifest + failedLivePreflight = $livePreflight + rollback = $rollback + rollbackRelayRestart = $rollbackRelayRestart + rollbackVerified = $rollbackVerified + rollbackVerifierErrorType = $rollbackVerifierErrorType + runtimeManifest = $afterManifest + durableReceiptWritten = $true + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + environmentSecretRead = $false + uiInteraction = $false + vmPowerChange = $false + hostReboot = $false + serviceRestart = $false + scheduledTaskRestart = [bool](($relayReload -and $relayReload.attempted) -or $rollbackRelayRestart.attempted) + scheduledTaskModification = $false + } + } catch { + $receipt = [pscustomobject]@{ + schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" + status = "rollback_receipt_construction_failed" + mode = "apply" + traceId = $traceId + runId = $runId + workItemId = $workItemId + sourceRevision = $sourceRevision + failurePhase = $failurePhase + failureType = $failureType + rollbackVerified = $false + runtimeRollbackWasVerifiedBeforeReceiptFailure = $rollbackVerified + receiptConstructionErrorType = $_.Exception.GetType().Name + durableReceiptWritten = $true + secretValueRead = $false + environmentSecretRead = $false + scheduledTaskModification = $false + } + $rollbackVerified = $false + } + try { + Write-AgentRemoteDeployReceipt $receiptPath $receipt + } catch { + $receipt.status = "rollback_receipt_write_failed" + $receipt.durableReceiptWritten = $false + $receipt | Add-Member -NotePropertyName receiptWriteErrorType -NotePropertyValue $_.Exception.GetType().Name -Force + Write-AgentResultAndExit $receipt 3 } - Write-AgentRemoteDeployReceipt $receiptPath $receipt Write-AgentResultAndExit $receipt $(if ($rollbackVerified) { 2 } else { 3 }) } @@ -688,6 +991,7 @@ try { backupPath = $backupPath validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut; status = [string]$validate.payload.status } deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status } + relayReload = $relayReload runtimeManifest = $runtimeManifest livePreflight = $livePreflight idempotentReplay = $false @@ -701,6 +1005,7 @@ try { vmPowerChange = $false hostReboot = $false serviceRestart = $false + scheduledTaskRestart = $true scheduledTaskModification = $false } nextSafeAction = "run_existing_production_principal_verifier_and_same_trace_completion_readback" @@ -721,6 +1026,8 @@ try { vmPowerChange = $false hostReboot = $false serviceRestart = $false + scheduledTaskRestart = $false + scheduledTaskModification = $false }) 70 } finally { if ($script:DeployLock) { diff --git a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh index fe5881509..d04145f74 100755 --- a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh +++ b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh @@ -5,6 +5,9 @@ readonly SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) readonly DEFAULT_SOURCE_ROOT="$(CDPATH= cd -- "${SCRIPT_DIR}/../.." && pwd)" readonly TARGET="Administrator@192.168.0.99" readonly SOURCE_HOST_IPV4="192.168.0.110" +readonly GITEA_ORIGIN_HTTPS="https://gitea.wooo.work/wooo/awoooi.git" +readonly GITEA_ORIGIN_HTTP_INTERNAL="http://192.168.0.110:3000/wooo/awoooi.git" +readonly GITEA_ORIGIN_SSH_INTERNAL="ssh://git@192.168.0.110:2222/wooo/awoooi.git" readonly RECEIVER_RELATIVE="scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1" readonly PREFLIGHT_RELATIVE="scripts/reboot-recovery/agent99-live-preflight.ps1" readonly WRAPPER_RELATIVE="scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh" @@ -136,7 +139,7 @@ done (( CONNECT_TIMEOUT_SECONDS >= 1 && CONNECT_TIMEOUT_SECONDS <= 30 )) || fail "invalid_connect_timeout" if [[ "${MODE}" == "apply" ]]; then - REMOTE_EXECUTION_TIMEOUT_SECONDS="1500" + REMOTE_EXECUTION_TIMEOUT_SECONDS="2400" readonly SAFE_ID_PATTERN='^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$' [[ "${TRACE_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "apply_trace_id_missing_or_invalid" [[ "${RUN_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "apply_run_id_missing_or_invalid" @@ -159,17 +162,28 @@ done [[ -d "${SOURCE_ROOT}" ]] || fail "source_root_missing" SOURCE_ROOT="$(CDPATH= cd -- "${SOURCE_ROOT}" && pwd)" +ORIGIN_URL="$(git -C "${SOURCE_ROOT}" remote get-url origin 2>/dev/null)" || fail "origin_remote_missing" +case "${ORIGIN_URL}" in + "${GITEA_ORIGIN_HTTPS}"|"${GITEA_ORIGIN_HTTP_INTERNAL}"|"${GITEA_ORIGIN_SSH_INTERNAL}") ;; + *) fail "origin_is_not_fixed_internal_gitea" ;; +esac +git -C "${SOURCE_ROOT}" fetch --quiet --no-tags origin \ + '+refs/heads/main:refs/remotes/origin/main' || fail "fresh_origin_main_fetch_failed" +ORIGIN_MAIN_REVISION="$(git -C "${SOURCE_ROOT}" rev-parse --verify 'refs/remotes/origin/main^{commit}' 2>/dev/null)" \ + || fail "fresh_origin_main_revision_unavailable" +ORIGIN_MAIN_REVISION="$(printf '%s' "${ORIGIN_MAIN_REVISION}" | tr '[:upper:]' '[:lower:]')" [[ -f "${KNOWN_HOSTS_FILE}" ]] || fail "known_hosts_file_missing" if [[ -n "${IDENTITY_FILE}" ]]; then [[ -f "${IDENTITY_FILE}" ]] || fail "identity_file_missing" fi if [[ -z "${SOURCE_REVISION}" ]]; then - SOURCE_REVISION="$(git -C "${SOURCE_ROOT}" rev-parse --verify 'HEAD^{commit}')" || fail "source_revision_unavailable" + SOURCE_REVISION="${ORIGIN_MAIN_REVISION}" fi SOURCE_REVISION="$(printf '%s' "${SOURCE_REVISION}" | tr '[:upper:]' '[:lower:]')" [[ "${SOURCE_REVISION}" =~ ^[0-9a-f]{40}$ ]] || fail "source_revision_must_be_full_commit_sha" git -C "${SOURCE_ROOT}" cat-file -e "${SOURCE_REVISION}^{commit}" 2>/dev/null || fail "source_revision_not_found" +[[ "${SOURCE_REVISION}" == "${ORIGIN_MAIN_REVISION}" ]] || fail "source_revision_is_not_fresh_origin_main" SOURCE_BOUND_FILES=( "${RUNTIME_FILES[@]}" diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index 0a92ac623..6f9e48923 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -3,6 +3,7 @@ from __future__ import annotations import os import re import shlex +import shutil import subprocess from pathlib import Path @@ -56,7 +57,7 @@ def test_sender_is_valid_shell_with_check_as_the_default() -> None: assert "apply_work_item_id_missing_or_invalid" in source assert 'MODE_SELECTION="default"' in source assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="45"' in source - assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="1500"' in source + assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="2400"' in source assert "timeout --signal=TERM --kill-after=10" in source @@ -106,6 +107,23 @@ def test_sender_is_fixed_to_110_to_windows99_public_key_transport() -> None: def test_sender_binds_every_transported_source_to_a_full_git_revision() -> None: source = SENDER.read_text(encoding="utf-8") + assert 'GITEA_ORIGIN_HTTPS="https://gitea.wooo.work/wooo/awoooi.git"' in source + assert ( + 'GITEA_ORIGIN_HTTP_INTERNAL="http://192.168.0.110:3000/wooo/awoooi.git"' + in source + ) + assert ( + 'GITEA_ORIGIN_SSH_INTERNAL="ssh://git@192.168.0.110:2222/wooo/awoooi.git"' + in source + ) + assert 'remote get-url origin' in source + assert "origin_is_not_fixed_internal_gitea" in source + assert "fetch --quiet --no-tags origin" in source + assert "+refs/heads/main:refs/remotes/origin/main" in source + assert "fresh_origin_main_fetch_failed" in source + assert "refs/remotes/origin/main^{commit}" in source + assert '[[ "${SOURCE_REVISION}" == "${ORIGIN_MAIN_REVISION}" ]]' in source + assert "source_revision_is_not_fresh_origin_main" in source assert "source_revision_must_be_full_commit_sha" in source assert "source_revision_not_found" in source assert "source_bound_file_untracked" in source @@ -166,9 +184,11 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier() assert "function Restore-AgentBundle" in source assert "Test-AgentBackupPath" in source assert 'Join-Path $DeployRoot "backup-"' in source - assert source.count("Restore-AgentBundle $backupPath") == 2 - assert 'status = if ($rollbackVerified) { "deploy_failed_rollback_verified"' in source - assert 'status = if ($rollbackVerified) { "rolled_back_post_verifier_failed"' in source + assert source.count("Restore-AgentBundle $backupPath") == 1 + assert '"deploy_failed_rollback_verified"' in source + assert '"deploy_failed_rollback_unverified"' in source + assert '"rolled_back_post_verifier_failed"' in source + assert '"rollback_failed_after_post_verifier"' in source assert "Get-AgentLiveBundleDigest" in source assert "Test-AgentManifestEqual $beforeManifest $afterManifest" in source assert "Get-AgentSha256File $ConfigPath" in source @@ -185,6 +205,29 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier() assert '$runtimeManifest.fileCount -eq 14' in source assert '$runtimeManifest.mismatchCount -eq 0' in source assert 'status = "deployed_verified"' in source + assert '$failurePhase = "deploy"' in source + assert '$failurePhase = "relay_reload"' in source + assert '$failurePhase = "post_verifier"' in source + assert ( + "Invoke-AgentRelayTaskRestart -ForbiddenGenerations " + "$rollbackForbiddenGenerations" + in source + ) + assert "$rollbackRelayRestart.newGenerationVerified" in source + assert "$rollbackVerifierErrorType = $_.Exception.GetType().Name" in source + assert 'status = "rollback_receipt_construction_failed"' in source + assert '$receipt.status = "rollback_receipt_write_failed"' in source + assert "durableReceiptWritten = $true" in source + assert "$receipt.durableReceiptWritten = $false" in source + rollback = source.index( + "$rollback = Restore-AgentBundle $backupPath $beforeFilePresence" + ) + rollback_restart = source.index( + "Invoke-AgentRelayTaskRestart -ForbiddenGenerations " + "$rollbackForbiddenGenerations" + ) + rollback_verify = source.index("$rollbackVerified = [bool](") + assert rollback < rollback_restart < rollback_verify def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary() -> None: @@ -206,9 +249,53 @@ def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary() assert "rawEnvelopeStored = $false" in source assert "$env:" not in source assert "GetEnvironmentVariable" not in source + assert ( + '$stageIdentity = "$mode|$traceId|$runId|$workItemId|$sourceRevision"' + in source + ) + assert 'Join-Path $DeployRoot "remote-source-$stageToken"' in source + assert 'Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken.json"' in source + assert '$prior.PSObject.Properties["relayReload"]' in source + assert "[bool]$prior.relayReload.newGenerationVerified" in source -def test_transport_does_not_expose_ui_vm_power_reboot_or_service_actions() -> None: +def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generation() -> None: + source = RECEIVER.read_text(encoding="utf-8") + + assert '$RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay"' in source + assert '$RelayTaskPath = "\\"' in source + assert "$RelayPort = 8787" in source + assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source + assert "Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source + assert "Start-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source + assert "Get-NetTCPConnection -LocalPort $RelayPort -State Listen" in source + assert 'GetBytes("$processId|$startedAt")' in source + assert "$afterStop.taskRunning" in source + assert "$beforeRelayRuntime.taskEnabled" in source + assert "$after.taskEnabled" in source + assert "$relayReload.oldGenerationsGone" in source + assert "$relayReload.newGenerationVerified" in source + assert "$relayReload.after.taskRunning" in source + assert "$livePreflight.relayListenerGenerations" in source + assert "scheduledTaskRestart = $true" in source + + deploy = source.index( + "$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath " + "-SourceRevision $sourceRevision" + ) + reload_task = source.index( + "$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations " + "@($beforeRelayRuntime.listenerGenerations)" + ) + postverify = source.index( + '$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath ' + '"agent99-live-preflight.ps1")', + reload_task, + ) + assert deploy < reload_task < postverify + + +def test_transport_only_restarts_existing_task_without_mutating_its_definition() -> None: source = "\n".join( ( SENDER.read_text(encoding="utf-8"), @@ -226,6 +313,8 @@ def test_transport_does_not_expose_ui_vm_power_reboot_or_service_actions() -> No "shutdown.exe", "Register-ScheduledTask", "Unregister-ScheduledTask", + "Set-ScheduledTask", + "New-ScheduledTask", "-RegisterTasks", ) for action in forbidden_actions: @@ -235,6 +324,7 @@ def test_transport_does_not_expose_ui_vm_power_reboot_or_service_actions() -> No assert "vmPowerChange = $false" in source assert "hostReboot = $false" in source assert "serviceRestart = $false" in source + assert "scheduledTaskRestart = $true" in source assert "scheduledTaskModification = $false" in source assert "host112" not in source.lower() assert "192.168.0.112" not in source @@ -286,7 +376,24 @@ def test_check_mode_builds_and_streams_bundle_over_bounded_fake_transport( "exec \"$@\"\n", encoding="utf-8", ) - for executable in (fake_hostname, fake_ssh, fake_timeout): + real_git = shutil.which("git") + assert real_git is not None + fake_git = fake_bin / "git" + fake_git.write_text( + "#!/usr/bin/env bash\n" + "case \" $* \" in\n" + " *\" remote get-url origin \"*) " + "printf '%s\\n' 'https://gitea.wooo.work/wooo/awoooi.git'; exit 0 ;;\n" + " *\" fetch --quiet --no-tags origin \"*) exit 0 ;;\n" + " *\" rev-parse --verify refs/remotes/origin/main^{commit} \"*) " + "printf '%040d\\n' 0; exit 0 ;;\n" + " *\" cat-file -e \"*) exit 0 ;;\n" + " *\" diff --quiet \"*) exit 0 ;;\n" + "esac\n" + f"exec {shlex.quote(real_git)} \"$@\"\n", + encoding="utf-8", + ) + for executable in (fake_hostname, fake_ssh, fake_timeout, fake_git): executable.chmod(0o700) environment = os.environ.copy()