From b86229e9fb1a1cdab7dc4b891450f343ff2ffc68 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 21:21:27 +0800 Subject: [PATCH] fix(agent99): restore autonomous recovery transport --- agent99-bootstrap.ps1 | 13 +- agent99-contract-check.ps1 | 95 +++++++ agent99-control-plane.ps1 | 236 ++++++++++++++++- agent99-deploy.ps1 | 243 ++++++++++++++++++ agent99-synthetic-tests.ps1 | 86 +++++++ agent99.config.99.example.json | 18 ++ agent99.config.example.json | 23 ++ ...nt99_transport_recovery_deploy_contract.py | 70 +++++ 8 files changed, 772 insertions(+), 12 deletions(-) create mode 100644 agent99-contract-check.ps1 create mode 100644 agent99-deploy.ps1 create mode 100644 agent99-synthetic-tests.ps1 create mode 100644 apps/api/tests/test_agent99_transport_recovery_deploy_contract.py diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 430d2ba91..f94933f4d 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -42,13 +42,20 @@ function Copy-AgentFile { } $agentFiles = @( + "agent99-bootstrap.ps1", + "agent99-contract-check.ps1", "agent99-control-plane.ps1", + "agent99-deploy.ps1", "agent99-register-tasks.ps1", - "agent99-submit-request.ps1", - "agent99-telegram-inbox.ps1", "agent99-sre-alert-inbox.ps1", "agent99-sre-alert-relay.ps1", - "host-reboot-scorecard.ps1" + "agent99-submit-request.ps1", + "agent99-synthetic-tests.ps1", + "agent99-telegram-inbox.ps1", + "agent99-windows-update-policy.ps1", + "host-reboot-scorecard.ps1", + "agent99.config.example.json", + "agent99.config.99.example.json" ) foreach ($agentFile in $agentFiles) { Copy-AgentFile $agentFile diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 new file mode 100644 index 000000000..0b197c145 --- /dev/null +++ b/agent99-contract-check.ps1 @@ -0,0 +1,95 @@ +param( + [string]$SourceRoot = (Split-Path -Parent $MyInvocation.MyCommand.Path), + [string]$OutputPath = "" +) + +$ErrorActionPreference = "Stop" +$checks = @() +$failures = @() + +function Add-Check { + param([string]$Name, [bool]$Ok, [string]$Detail) + $item = [pscustomobject]@{ name = $Name; ok = $Ok; detail = $Detail } + $script:checks += $item + if (-not $Ok) { $script:failures += $item } +} + +$requiredFiles = @( + "agent99-bootstrap.ps1", + "agent99-contract-check.ps1", + "agent99-control-plane.ps1", + "agent99-deploy.ps1", + "agent99-register-tasks.ps1", + "agent99-sre-alert-inbox.ps1", + "agent99-sre-alert-relay.ps1", + "agent99-submit-request.ps1", + "agent99-synthetic-tests.ps1", + "agent99-telegram-inbox.ps1", + "agent99-windows-update-policy.ps1", + "host-reboot-scorecard.ps1" +) + +foreach ($name in $requiredFiles) { + $path = Join-Path $SourceRoot $name + Add-Check "file:$name" (Test-Path $path) $path +} + +foreach ($file in Get-ChildItem -Path $SourceRoot -Filter "*.ps1" -File) { + $tokens = $null + $errors = $null + $sourceText = [IO.File]::ReadAllText($file.FullName, [Text.Encoding]::UTF8) + [System.Management.Automation.Language.Parser]::ParseInput($sourceText, [ref]$tokens, [ref]$errors) | Out-Null + Add-Check "powershell_parse:$($file.Name)" ($errors.Count -eq 0) $(if ($errors.Count -eq 0) { "parse_ok" } else { "parse_errors=$($errors.Count)" }) +} + +foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.json")) { + $path = Join-Path $SourceRoot $name + try { + $config = Get-Content $path -Raw | ConvertFrom-Json + $hasHosts = @($config.hosts).Count -eq 5 + $hasRoutes = @($config.publicUrls).Count -ge 5 + $hasSlo = [bool]($config.recoverySlo -and [int]$config.recoverySlo.targetMinutes -eq 10) + $hasIdentity = [bool]($config.sshIdentityFile -and $config.sshIdentityFile -match "agent99_ed25519$") + $hasJump = [bool]($config.k3s -and $config.k3s.jumpHost -and $config.k3s.preferJumpHost -eq $true) + $hasAutoRecovery = [bool]($config.autoRecovery -and $config.autoRecovery.enabled -eq $true) + $ok = $hasHosts -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasJump -and $hasAutoRecovery + Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity jump=$hasJump autoRecovery=$hasAutoRecovery" + } catch { + Add-Check "config:$name" $false "json_parse_failed: $($_.Exception.Message)" + } +} + +$control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane.ps1"), [Text.Encoding]::UTF8) +$inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8) +$telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8) +$bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8) + +Add-Check "transport:dedicated_identity" ($control.Contains("sshIdentityFile") -and $control.Contains("IdentitiesOnly=yes")) "dedicated SSH identity is enforced" +Add-Check "transport:jump_first" ($control.Contains("preferJumpHost") -and $control.Contains("ssh_jump_fallback")) "preferred jump with bounded fallback is present" +Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("recovery_already_queued")) "boot and host-down recovery use a single-flight queue" +Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present" +Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present" +Add-Check "problem_counts:v2" ($control.Contains('schemaVersion = "agent99_problem_counts_v2"') -and $control.Contains("totalVerifiedResolved")) "verified problem counts are present" +Add-Check "runtime:manifest" ($control.Contains("Test-AgentRuntimeManifest") -and $bootstrap.Contains('schemaVersion = "agent99_runtime_manifest_v1"')) "runtime source drift is self-checked" +Add-Check "routing:structured" ($inbox.Contains("Resolve-AgentAlertRoute") -and $inbox.Contains('schemaVersion = "agent99_sre_alert_single_flight_v1"')) "structured route and local single-flight are present" +Add-Check "routing:telegram" ($telegram.Contains('schemaVersion = "agent99_alert_route_v1"') -and $telegram.Contains("autoIngestMonitoringAlerts")) "Telegram structured ingress is present" + +$result = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + sourceRoot = $SourceRoot + ok = [bool](@($failures).Count -eq 0) + checkCount = @($checks).Count + failureCount = @($failures).Count + checks = $checks + failures = $failures +} + +if ($OutputPath) { + $parent = Split-Path -Parent $OutputPath + if ($parent) { New-Item -ItemType Directory -Force -Path $parent | Out-Null } + $result | ConvertTo-Json -Depth 8 | Set-Content -Path $OutputPath -Encoding UTF8 +} + +$result | ConvertTo-Json -Depth 8 +if (-not $result.ok) { exit 1 } +exit 0 diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 2534c181f..14d4b929b 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -34,6 +34,11 @@ function Load-Config { $Config = Load-Config $SshUser = if ($Config.sshUser) { $Config.sshUser } else { "wooo" } +$SshIdentityFile = if ($Config.PSObject.Properties["sshIdentityFile"] -and $Config.sshIdentityFile) { + [Environment]::ExpandEnvironmentVariables([string]$Config.sshIdentityFile) +} else { + "" +} $script:Events = @() $script:TelegramAttempts = @() $script:SuppressAgentAlerts = [bool]$SuppressAlerts @@ -56,6 +61,11 @@ function Get-HostSshUser { $SshUser } +function Get-AgentSshIdentityArguments { + if (-not $SshIdentityFile) { return @() } + @("-i", $SshIdentityFile, "-o", "IdentitiesOnly=yes") +} + function Send-AgentTelegramRelay { param( [string]$Message, @@ -85,7 +95,7 @@ function Send-AgentTelegramRelay { $remotePhotoPath = "/tmp/" + $safePhotoName $containerPhotoPath = "/tmp/" + $safePhotoName $relayUser = Get-HostSshUser $relayHost - $scpArgs = @( + $scpArgs = @(Get-AgentSshIdentityArguments) + @( "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", "-o", "NumberOfPasswordPrompts=0", @@ -1044,6 +1054,143 @@ function Record-AgentEvent { } } +function Get-AgentBootState { + $stateDir = Join-Path $Config.agentRoot "state" + $statePath = Join-Path $stateDir "boot-state.json" + New-Item -ItemType Directory -Force -Path $stateDir | Out-Null + try { + $os = Get-CimInstance -ClassName Win32_OperatingSystem -ErrorAction Stop + $bootTime = ([datetime]$os.LastBootUpTime).ToString("o") + } catch { + Write-AgentLog "boot_state_read_failed error=$($_.Exception.Message)" + return [pscustomobject]@{ ok = $false; detected = $false; reason = "boot_time_unavailable"; statePath = $statePath } + } + + $previous = $null + if (Test-Path $statePath) { + try { $previous = Get-Content $statePath -Raw | ConvertFrom-Json } catch { $previous = $null } + } + $detected = [bool]($previous -and $previous.bootTime -and ([string]$previous.bootTime -ne $bootTime)) + $result = [pscustomobject]@{ + ok = $true + detected = $detected + bootTime = $bootTime + previousBootTime = if ($previous) { [string]$previous.bootTime } else { $null } + statePath = $statePath + } + $result | ConvertTo-Json -Depth 5 | Set-Content -Path $statePath -Encoding UTF8 + if ($detected) { + Record-AgentEvent "host_reboot_detected" "warning" "偵測到 99 主機重新啟動；Agent99 已進入受控恢復主線。" $result -Alert + } + $result +} + +function Get-AgentAutoRecoveryConfig { + $configured = if ($Config.PSObject.Properties["autoRecovery"]) { $Config.autoRecovery } else { $null } + [pscustomobject]@{ + enabled = [bool](-not $configured -or -not $configured.PSObject.Properties["enabled"] -or (Convert-AgentBool $configured.enabled)) + triggerOnHostUnreachable = [bool](-not $configured -or -not $configured.PSObject.Properties["triggerOnHostUnreachable"] -or (Convert-AgentBool $configured.triggerOnHostUnreachable)) + minimumUnreachableHosts = if ($configured -and $configured.PSObject.Properties["minimumUnreachableHosts"]) { [math]::Max(1, [int]$configured.minimumUnreachableHosts) } else { 1 } + cooldownMinutes = if ($configured -and $configured.PSObject.Properties["cooldownMinutes"]) { [math]::Max(1, [int]$configured.cooldownMinutes) } else { 10 } + } +} + +function Start-AgentRecoveryFromObservation { + param( + [string]$Reason, + [object]$Observation = $null + ) + + $autoRecovery = Get-AgentAutoRecoveryConfig + if (-not $autoRecovery.enabled) { + return [pscustomobject]@{ ok = $false; queued = $false; reason = "auto_recovery_disabled"; trigger = $Reason } + } + + $queueDir = Join-Path $Config.agentRoot "queue" + $stateDir = Join-Path $Config.agentRoot "state" + $statePath = Join-Path $stateDir "recovery-trigger-state.json" + New-Item -ItemType Directory -Force -Path $queueDir, $stateDir | Out-Null + + foreach ($file in @(Get-ChildItem -Path $queueDir -Filter "*.json" -File -ErrorAction SilentlyContinue)) { + try { + $pending = Get-Content $file.FullName -Raw | ConvertFrom-Json + if ([string]$pending.mode -eq "Recover") { + return [pscustomobject]@{ ok = $true; queued = $false; suppressed = $true; reason = "recovery_already_queued"; queuePath = $file.FullName; trigger = $Reason } + } + } catch { + Write-AgentLog "recovery_queue_probe_failed path=$($file.FullName) error=$($_.Exception.Message)" + } + } + + if (Test-Path $statePath) { + try { + $state = Get-Content $statePath -Raw | ConvertFrom-Json + $lastRequestedAt = [datetime]$state.lastRequestedAt + $ageMinutes = ((Get-Date) - $lastRequestedAt).TotalMinutes + if ($ageMinutes -lt $autoRecovery.cooldownMinutes) { + return [pscustomobject]@{ ok = $true; queued = $false; suppressed = $true; reason = "recovery_trigger_cooldown"; ageMinutes = [math]::Round($ageMinutes, 2); trigger = $Reason } + } + } catch { + Write-AgentLog "recovery_trigger_state_read_failed error=$($_.Exception.Message)" + } + } + + $id = "auto-recover-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) + $queuePath = Join-Path $queueDir "$id.json" + $temporaryPath = "$queuePath.tmp" + $createdAt = Get-Date -Format o + $request = [pscustomobject]@{ + id = $id + mode = "Recover" + controlledApply = $true + requestedBy = "Agent99" + source = "agent99-recovery-observer" + externalId = $Reason + reason = $Reason + instruction = "Agent99 detected $Reason; run the allowlisted VM, host, Harbor, K3s, public-route, and freshness recovery chain." + observation = $Observation + createdAt = $createdAt + } + $request | ConvertTo-Json -Depth 8 | Set-Content -Path $temporaryPath -Encoding UTF8 + Move-Item -Force $temporaryPath $queuePath + [pscustomobject]@{ + schemaVersion = "agent99_recovery_trigger_state_v1" + lastRequestedAt = $createdAt + reason = $Reason + queueId = $id + queuePath = $queuePath + } | ConvertTo-Json -Depth 6 | Set-Content -Path $statePath -Encoding UTF8 + + $result = [pscustomobject]@{ ok = $true; queued = $true; suppressed = $false; reason = $Reason; id = $id; queuePath = $queuePath } + Record-AgentEvent "recovery_auto_triggered" "warning" "偵測到 $Reason；已排入單一受控 Recover 佇列。" $result -Alert + $result +} + +function Get-AgentRecoverySloConfig { + $configured = if ($Config.PSObject.Properties["recoverySlo"]) { $Config.recoverySlo } else { $null } + [pscustomobject]@{ + targetMinutes = if ($configured -and $configured.PSObject.Properties["targetMinutes"]) { [int]$configured.targetMinutes } else { 10 } + hostTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["hostTimeoutSeconds"]) { [int]$configured.hostTimeoutSeconds } else { 90 } + harborTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["harborTimeoutSeconds"]) { [int]$configured.harborTimeoutSeconds } else { 120 } + workloadTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["workloadTimeoutSeconds"]) { [int]$configured.workloadTimeoutSeconds } else { 180 } + } +} + +function New-AgentRecoveryPhase { + param( + [string]$Name, + [string]$Status, + [double]$ElapsedSeconds, + [string]$Evidence = "" + ) + [pscustomobject]@{ + name = $Name + status = $Status + elapsedSeconds = [math]::Round($ElapsedSeconds, 1) + evidence = if ($Evidence) { $Evidence } else { $null } + } +} + function Invoke-SshText { param( [string]$TargetHost, @@ -1056,7 +1203,7 @@ function Invoke-SshText { $last = $null $userToUse = if ($User) { $User } else { Get-HostSshUser $TargetHost } for ($attempt = 1; $attempt -le $Retries; $attempt++) { - $args = @( + $args = @(Get-AgentSshIdentityArguments) + @( "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", "-o", "NumberOfPasswordPrompts=0", @@ -1143,17 +1290,30 @@ function Invoke-HostSshText { [int]$Retries = 1 ) - $direct = Invoke-SshText $TargetHost $Command $TimeoutSeconds $Retries - if ($direct.ok -or -not $Config.k3s.jumpHost -or $TargetHost -eq $Config.k3s.jumpHost) { - $direct | Add-Member -NotePropertyName route -NotePropertyValue "direct" -Force - return $direct - } + $jumpHost = if ($Config.k3s -and $Config.k3s.PSObject.Properties["jumpHost"]) { [string]$Config.k3s.jumpHost } else { "" } + $preferJumpHost = [bool]($Config.k3s -and $Config.k3s.PSObject.Properties["preferJumpHost"] -and (Convert-AgentBool $Config.k3s.preferJumpHost)) + $directHosts = if ($Config.k3s -and $Config.k3s.PSObject.Properties["directHosts"]) { @($Config.k3s.directHosts) } else { @() } + $canUseJump = [bool]($jumpHost -and $TargetHost -ne $jumpHost -and $TargetHost -notin $directHosts) $quotedCommand = Quote-ShSingle $Command $targetUser = Get-HostSshUser $TargetHost $jumpCommand = "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o NumberOfPasswordPrompts=0 -o ConnectTimeout=10 -l $targetUser $TargetHost $quotedCommand" - $viaJump = Invoke-SshText $Config.k3s.jumpHost $jumpCommand $TimeoutSeconds $Retries - $viaJump | Add-Member -NotePropertyName route -NotePropertyValue "via_$($Config.k3s.jumpHost)" -Force + + if ($preferJumpHost -and $canUseJump) { + $viaJump = Invoke-SshText $jumpHost $jumpCommand $TimeoutSeconds $Retries + $viaJump | Add-Member -NotePropertyName route -NotePropertyValue "via_$jumpHost" -Force + if ($viaJump.ok) { return $viaJump } + Write-AgentLog "ssh_jump_fallback host=$TargetHost jumpHost=$jumpHost exit=$($viaJump.exitCode)" + } + + $direct = Invoke-SshText $TargetHost $Command $TimeoutSeconds $Retries + if ($direct.ok -or -not $canUseJump -or $preferJumpHost) { + $direct | Add-Member -NotePropertyName route -NotePropertyValue "direct" -Force + return $direct + } + + $viaJump = Invoke-SshText $jumpHost $jumpCommand $TimeoutSeconds $Retries + $viaJump | Add-Member -NotePropertyName route -NotePropertyValue "via_$jumpHost" -Force return $viaJump } @@ -3684,13 +3844,38 @@ $backupHealth = $null $providerFreshness = $null $securityTriage = $null $controlTick = $null +$bootState = $null +$recoveryTrigger = $null +$recoverySlo = $null +$recoveryPhases = @() +$recoveryStartedAt = if ($Mode -eq "Recover") { Get-Date } else { $null } +$recoveryConfig = Get-AgentRecoverySloConfig + +if ($Mode -in @("Status", "Recover")) { + $bootState = Get-AgentBootState +} +if ($Mode -eq "Status" -and $bootState -and $bootState.detected) { + $recoveryTrigger = Start-AgentRecoveryFromObservation "host_reboot_detected" $bootState +} if ($Mode -in @("Recover", "StartVMs")) { $vmResults = Start-ConfiguredVMs + if ($Mode -eq "Recover") { + $vmFailures = @($vmResults | Where-Object { $_.ok -eq $false -and -not $_.skipped }).Count + $recoveryPhases += New-AgentRecoveryPhase "vm_start" $(if ($vmFailures -eq 0) { "ok" } else { "degraded" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds) + } } if ($Mode -in @("Status", "Recover")) { $hostResults = Test-HostReachability + $unreachableHosts = @($hostResults | Where-Object { -not $_.ping }) + if ($Mode -eq "Recover") { + $recoveryPhases += New-AgentRecoveryPhase "host_reachability" $(if ($unreachableHosts.Count -eq 0) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds) + } + $autoRecovery = Get-AgentAutoRecoveryConfig + if ($Mode -eq "Status" -and -not $recoveryTrigger -and $autoRecovery.triggerOnHostUnreachable -and $unreachableHosts.Count -ge $autoRecovery.minimumUnreachableHosts) { + $recoveryTrigger = Start-AgentRecoveryFromObservation "host_unreachable_detected" ([pscustomobject]@{ hosts = @($unreachableHosts.host); count = $unreachableHosts.Count }) + } } if ($Mode -in @("Status", "Recover", "HarborRepair")) { @@ -3703,6 +3888,10 @@ if ($Mode -in @("Recover", "HarborRepair") -and $harbor) { Start-Sleep -Seconds 3 $harbor = Get-HarborState } + if ($Mode -eq "Recover") { + $harborReady = [bool]($harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy) + $recoveryPhases += New-AgentRecoveryPhase "harbor_registry" $(if ($harborReady) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds) + } } if ($Mode -in @("Status", "Recover", "AwoooRepair")) { @@ -3716,10 +3905,17 @@ if ($Mode -in @("Recover", "AwoooRepair") -and $awooo) { Start-Sleep -Seconds 3 $awooo = Get-AwoooState } + if ($Mode -eq "Recover") { + $workloadsReady = [bool]($awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure) + $recoveryPhases += New-AgentRecoveryPhase "k3s_workloads" $(if ($workloadsReady) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds) + } } if ($Mode -in @("Status", "Recover", "PublicSmoke")) { $public = Test-PublicRoutes + if ($Mode -eq "Recover") { + $recoveryPhases += New-AgentRecoveryPhase "public_routes" $(if (@($public | Where-Object { -not $_.ok }).Count -eq 0) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds) + } } if ($Mode -in @("Status", "Recover", "Perf")) { @@ -3769,12 +3965,34 @@ $backupWarning = if ($backupHealth -and $backupHealth.severity -eq "warning") { $providerFreshnessWarning = if ($providerFreshness) { 1 } else { 0 } $securityTriageWarning = if ($securityTriage -and $securityTriage.hitCount -gt 0) { 1 } else { 0 } $summarySeverity = if ($publicFailures -gt 0 -or $aiFailures -gt 0 -or $perfCritical -gt 0 -or $selfHealthCritical -gt 0 -or $backupCritical -gt 0) { "critical" } elseif ($perfWarning -gt 0 -or $selfHealthWarning -gt 0 -or $backupWarning -gt 0 -or $providerFreshnessWarning -gt 0 -or $securityTriageWarning -gt 0) { "warning" } else { "info" } +if ($Mode -eq "Recover") { + $elapsedSeconds = [math]::Round(((Get-Date) - $recoveryStartedAt).TotalSeconds, 1) + $unreachableCount = @($hostResults | Where-Object { -not $_.ping }).Count + $recoveryCompleted = [bool]($unreachableCount -eq 0 -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0) + $withinSlo = [bool]($recoveryCompleted -and $elapsedSeconds -le ($recoveryConfig.targetMinutes * 60)) + $recoverySlo = [pscustomobject]@{ + targetMinutes = $recoveryConfig.targetMinutes + startedAt = $recoveryStartedAt.ToString("o") + completedAt = (Get-Date).ToString("o") + elapsedSeconds = $elapsedSeconds + deadline = $recoveryStartedAt.AddMinutes($recoveryConfig.targetMinutes).ToString("o") + completed = $recoveryCompleted + withinSlo = $withinSlo + phases = @($recoveryPhases) + } + $sloSeverity = if ($withinSlo) { "info" } elseif ($recoveryCompleted) { "warning" } else { "critical" } + $sloMessage = if ($withinSlo) { "恢復完成，耗時 $elapsedSeconds 秒，符合 $($recoveryConfig.targetMinutes) 分鐘 SLA。" } elseif ($recoveryCompleted) { "恢復完成但超過 $($recoveryConfig.targetMinutes) 分鐘 SLA，耗時 $elapsedSeconds 秒。" } else { "恢復尚未完成，耗時 $elapsedSeconds 秒；請依階段 evidence 繼續處置。" } + Record-AgentEvent "recovery_slo_result" $sloSeverity $sloMessage $recoverySlo -Alert +} Record-AgentEvent "agent_complete" $summarySeverity "mode=$Mode publicFailures=$publicFailures aiFailures=$aiFailures perfCritical=$perfCritical perfWarning=$perfWarning selfHealth=$($selfHealth.severity) backupHealth=$($backupHealth.severity) providerFreshness=$($providerFreshness.status) securityTriage=$($securityTriage.status) evidence=$jsonPath" $null -NoAlert $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) mode = $Mode controlledApply = [bool]$ControlledApply + bootState = $bootState + recoveryTrigger = $recoveryTrigger + recoverySlo = $recoverySlo vmResults = $vmResults hosts = $hostResults harbor = $harbor diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 new file mode 100644 index 000000000..4296d3734 --- /dev/null +++ b/agent99-deploy.ps1 @@ -0,0 +1,243 @@ +param( + [string]$SourceRoot = (Split-Path -Parent $MyInvocation.MyCommand.Path), + [string]$AgentRoot = "C:\Wooo\Agent99", + [string]$SourceRevision = "", + [switch]$RegisterTasks, + [switch]$ValidateOnly +) + +$ErrorActionPreference = "Stop" +$stamp = Get-Date -Format "yyyyMMdd-HHmmss" +$BinDir = Join-Path $AgentRoot "bin" +$ConfigPath = Join-Path $AgentRoot "config\agent99.config.json" +$EvidenceDir = Join-Path $AgentRoot "evidence" +$StateDir = Join-Path $AgentRoot "state" +$DeployRoot = Join-Path $AgentRoot "deploy" +$StageDir = Join-Path $DeployRoot "staging-$stamp" +$BackupDir = Join-Path $DeployRoot "backup-$stamp" +$ManifestPath = Join-Path $StateDir "runtime-manifest.json" +$EvidencePath = Join-Path $EvidenceDir "agent99-deploy-$stamp.json" + +$runtimeFiles = @( + "agent99-bootstrap.ps1", + "agent99-contract-check.ps1", + "agent99-control-plane.ps1", + "agent99-deploy.ps1", + "agent99-register-tasks.ps1", + "agent99-sre-alert-inbox.ps1", + "agent99-sre-alert-relay.ps1", + "agent99-submit-request.ps1", + "agent99-synthetic-tests.ps1", + "agent99-telegram-inbox.ps1", + "agent99-windows-update-policy.ps1", + "host-reboot-scorecard.ps1", + "agent99.config.example.json", + "agent99.config.99.example.json" +) + +function Ensure-Utf8Bom { + param([string]$Path) + $bytes = [IO.File]::ReadAllBytes($Path) + $text = [Text.Encoding]::UTF8.GetString($bytes) + if ($text.Length -gt 0 -and $text[0] -eq [char]0xFEFF) { $text = $text.Substring(1) } + [IO.File]::WriteAllText($Path, $text, (New-Object System.Text.UTF8Encoding($true))) +} + +function Add-DefaultProperty { + param([object]$Object, [string]$Name, [object]$Value) + if (-not $Object.PSObject.Properties[$Name] -or $null -eq $Object.PSObject.Properties[$Name].Value) { + if ($Object.PSObject.Properties[$Name]) { + $Object.PSObject.Properties[$Name].Value = $Value + } else { + $Object | Add-Member -MemberType NoteProperty -Name $Name -Value $Value + } + } +} + +function Write-DeployEvidence { + param([bool]$Ok, [string]$Status, [string]$ErrorText, [object[]]$Files, [object]$TaskResult) + New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null + [pscustomobject]@{ + schemaVersion = "agent99_deploy_receipt_v2" + timestamp = (Get-Date -Format o) + ok = $Ok + status = $Status + sourceRoot = $SourceRoot + sourceRevision = $SourceRevision + agentRoot = $AgentRoot + stageDir = $StageDir + backupDir = $BackupDir + runtimeManifest = $ManifestPath + validateOnly = [bool]$ValidateOnly + registerTasks = [bool]$RegisterTasks + error = $ErrorText + files = $Files + taskRegistration = $TaskResult + } | ConvertTo-Json -Depth 10 | Set-Content -Path $EvidencePath -Encoding UTF8 +} + +function Restore-Agent99Deployment { + param([object[]]$Rows, [bool]$ConfigBackedUp, [bool]$ManifestBackedUp) + foreach ($item in $Rows) { + $backupPath = Join-Path $BackupDir $item.name + if ($item.existed -and (Test-Path $backupPath)) { + Copy-Item -Force $backupPath $item.livePath + } elseif (-not $item.existed -and (Test-Path $item.livePath)) { + Remove-Item -Force $item.livePath + } + } + if ($ConfigBackedUp) { Copy-Item -Force (Join-Path $BackupDir "agent99.config.json") $ConfigPath } + if ($ManifestBackedUp) { + Copy-Item -Force (Join-Path $BackupDir "runtime-manifest.json") $ManifestPath + } elseif (Test-Path $ManifestPath) { + Remove-Item -Force $ManifestPath + } +} + +if (-not $SourceRevision) { + try { + $revision = & git -C $SourceRoot rev-parse HEAD 2>$null + if ($LASTEXITCODE -eq 0 -and $revision) { $SourceRevision = ([string]$revision).Trim() } + } catch { + $SourceRevision = "" + } +} +if (-not $SourceRevision) { $SourceRevision = "unknown" } + +New-Item -ItemType Directory -Force -Path $StageDir, $EvidenceDir, $StateDir | Out-Null +foreach ($name in $runtimeFiles) { + $source = Join-Path $SourceRoot $name + if (-not (Test-Path $source)) { throw "Required deployment file missing: $source" } + $target = Join-Path $StageDir $name + Copy-Item -Force $source $target + if ($name.EndsWith(".ps1")) { Ensure-Utf8Bom $target } +} + +& (Join-Path $StageDir "agent99-contract-check.ps1") -SourceRoot $StageDir | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-DeployEvidence $false "preflight_failed" "contract_check_failed" @() $null + throw "Agent99 staged contract check failed." +} +& (Join-Path $StageDir "agent99-synthetic-tests.ps1") -SourceRoot $StageDir -WorkRoot (Join-Path $StageDir "synthetic-work") | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-DeployEvidence $false "preflight_failed" "synthetic_tests_failed" @() $null + throw "Agent99 staged synthetic tests failed." +} + +$stagedFiles = @($runtimeFiles | ForEach-Object { + $path = Join-Path $StageDir $_ + [pscustomobject]@{ name = $_; sha256 = (Get-FileHash -Algorithm SHA256 -Path $path).Hash.ToLowerInvariant() } +}) + +if ($ValidateOnly) { + Write-DeployEvidence $true "validated" $null $stagedFiles $null + [pscustomobject]@{ ok = $true; status = "validated"; sourceRevision = $SourceRevision; evidence = $EvidencePath; files = $stagedFiles } | ConvertTo-Json -Depth 8 + exit 0 +} + +New-Item -ItemType Directory -Force -Path $BinDir, $BackupDir | Out-Null +$promotionRows = @() +$configBackedUp = $false +$manifestBackedUp = $false +try { + foreach ($name in $runtimeFiles) { + $livePath = Join-Path $BinDir $name + $existed = Test-Path $livePath + if ($existed) { Copy-Item -Force $livePath (Join-Path $BackupDir $name) } + $promotionRows += [pscustomobject]@{ name = $name; livePath = $livePath; existed = [bool]$existed } + Copy-Item -Force (Join-Path $StageDir $name) $livePath + } + + if (-not (Test-Path $ConfigPath)) { throw "Live Agent99 config not found: $ConfigPath" } + Copy-Item -Force $ConfigPath (Join-Path $BackupDir "agent99.config.json") + $configBackedUp = $true + if (Test-Path $ManifestPath) { + Copy-Item -Force $ManifestPath (Join-Path $BackupDir "runtime-manifest.json") + $manifestBackedUp = $true + } + + $config = Get-Content $ConfigPath -Raw | ConvertFrom-Json + Add-DefaultProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519") + Add-DefaultProperty $config "k3s" ([pscustomobject]@{}) + Add-DefaultProperty $config.k3s "jumpHost" "192.168.0.110" + Add-DefaultProperty $config.k3s "preferJumpHost" $true + Add-DefaultProperty $config.k3s "directHosts" @("192.168.0.110", "192.168.0.112") + Add-DefaultProperty $config "autoRecovery" ([pscustomobject]@{}) + Add-DefaultProperty $config.autoRecovery "enabled" $true + Add-DefaultProperty $config.autoRecovery "triggerOnHostUnreachable" $true + Add-DefaultProperty $config.autoRecovery "minimumUnreachableHosts" 1 + Add-DefaultProperty $config.autoRecovery "cooldownMinutes" 10 + Add-DefaultProperty $config "recoverySlo" ([pscustomobject]@{}) + Add-DefaultProperty $config.recoverySlo "targetMinutes" 10 + Add-DefaultProperty $config.recoverySlo "hostTimeoutSeconds" 90 + Add-DefaultProperty $config.recoverySlo "harborTimeoutSeconds" 120 + Add-DefaultProperty $config.recoverySlo "workloadTimeoutSeconds" 180 + Add-DefaultProperty $config "telegram" ([pscustomobject]@{}) + Add-DefaultProperty $config.telegram "repeatAfterMinutes" 120 + Add-DefaultProperty $config "performance" ([pscustomobject]@{}) + Add-DefaultProperty $config.performance "processCpuHostPercentWarning" 25 + Add-DefaultProperty $config.performance "processCpuHostPercentCritical" 50 + $config | ConvertTo-Json -Depth 20 | Set-Content -Path $ConfigPath -Encoding UTF8 + + $manifestRows = @() + foreach ($staged in $stagedFiles) { + $livePath = Join-Path $BinDir $staged.name + $runtimeSha = if (Test-Path $livePath) { (Get-FileHash -Algorithm SHA256 -Path $livePath).Hash.ToLowerInvariant() } else { "" } + $matched = [bool]($runtimeSha -and $runtimeSha -eq $staged.sha256) + $manifestRows += [pscustomobject]@{ + name = $staged.name + sourceExists = $true + runtimeExists = [bool](Test-Path $livePath) + sourceSha256 = $staged.sha256 + runtimeSha256 = $runtimeSha + contentMatched = $matched + matched = $matched + } + } + $runtimeMatched = [bool](@($manifestRows | Where-Object { -not $_.matched }).Count -eq 0) + [pscustomobject]@{ + schemaVersion = "agent99_runtime_manifest_v1" + generatedAt = (Get-Date -Format o) + sourceRevision = $SourceRevision + runtimeMatched = $runtimeMatched + fileCount = $manifestRows.Count + mismatchCount = @($manifestRows | Where-Object { -not $_.matched }).Count + files = $manifestRows + } | ConvertTo-Json -Depth 10 | Set-Content -Path $ManifestPath -Encoding UTF8 + if (-not $runtimeMatched) { throw "Promoted Agent99 runtime does not match staged source." } + + & (Join-Path $BinDir "agent99-contract-check.ps1") -SourceRoot $BinDir | Out-Null + if ($LASTEXITCODE -ne 0) { throw "Post-promotion Agent99 contract check failed." } + & (Join-Path $BinDir "agent99-synthetic-tests.ps1") -SourceRoot $BinDir -WorkRoot (Join-Path $StageDir "live-synthetic-work") | Out-Null + if ($LASTEXITCODE -ne 0) { throw "Post-promotion Agent99 synthetic tests failed." } +} catch { + Restore-Agent99Deployment $promotionRows $configBackedUp $manifestBackedUp + Write-DeployEvidence $false "rolled_back" $_.Exception.Message $stagedFiles $null + throw +} + +$taskResult = $null +if ($RegisterTasks) { + try { + & (Join-Path $BinDir "agent99-register-tasks.ps1") -AgentRoot $AgentRoot | Out-Null + $taskResult = [pscustomobject]@{ ok = $true; status = "registered" } + } catch { + $taskResult = [pscustomobject]@{ ok = $false; status = "registration_failed"; error = $_.Exception.Message } + } +} + +$deployOk = [bool](-not $taskResult -or $taskResult.ok) +$deployStatus = if ($deployOk) { "deployed" } else { "deployed_task_registration_failed" } +Write-DeployEvidence $deployOk $deployStatus $(if ($taskResult -and -not $taskResult.ok) { $taskResult.error } else { $null }) $stagedFiles $taskResult +[pscustomobject]@{ + ok = $deployOk + status = $deployStatus + sourceRevision = $SourceRevision + evidence = $EvidencePath + backupDir = $BackupDir + runtimeManifest = $ManifestPath + taskRegistration = $taskResult + files = $stagedFiles +} | ConvertTo-Json -Depth 10 +if (-not $deployOk) { exit 2 } +exit 0 diff --git a/agent99-synthetic-tests.ps1 b/agent99-synthetic-tests.ps1 new file mode 100644 index 000000000..88820152e --- /dev/null +++ b/agent99-synthetic-tests.ps1 @@ -0,0 +1,86 @@ +param( + [string]$SourceRoot = (Split-Path -Parent $MyInvocation.MyCommand.Path), + [string]$WorkRoot = (Join-Path $env:TEMP ("agent99-synthetic-" + (Get-Date -Format "yyyyMMdd-HHmmss"))) +) + +$ErrorActionPreference = "Stop" +$checks = @() + +function Add-SyntheticCheck { + param([string]$Name, [bool]$Ok, [string]$Detail) + $script:checks += [pscustomobject]@{ name = $Name; ok = $Ok; detail = $Detail } +} + +Remove-Item -Recurse -Force $WorkRoot -ErrorAction SilentlyContinue +New-Item -ItemType Directory -Force -Path $WorkRoot | Out-Null + +$inbox = Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1" +$routingOutput = @(& $inbox -AgentRoot $WorkRoot -SelfTest 2>&1) -join "`n" +$routingExitCode = $LASTEXITCODE +try { + $routing = $routingOutput | ConvertFrom-Json + Add-SyntheticCheck "routing:self_test" ($routingExitCode -eq 0 -and $routing.ok -eq $true) "passed=$($routing.passedCount)/$($routing.caseCount) exit=$routingExitCode" + Add-SyntheticCheck "routing:provider_freshness" (@($routing.results | Where-Object { $_.alertId -eq "selftest-provider-freshness" -and $_.mode -eq "ProviderFreshness" }).Count -eq 1) "provider freshness structured route" + Add-SyntheticCheck "routing:reboot_slo" (@($routing.results | Where-Object { $_.alertId -eq "selftest-reboot-slo" -and $_.mode -eq "Recover" -and $_.routeSource -eq "suggestedMode" }).Count -eq 1) "reboot SLO has priority over contaminated text" +} catch { + Add-SyntheticCheck "routing:self_test" $false "invalid self-test JSON: $($_.Exception.Message)" +} + +$tokens = $null +$errors = $null +$controlPath = Join-Path $SourceRoot "agent99-control-plane.ps1" +$controlSource = [IO.File]::ReadAllText($controlPath, [Text.Encoding]::UTF8) +$ast = [System.Management.Automation.Language.Parser]::ParseInput($controlSource, [ref]$tokens, [ref]$errors) +$formatFunctions = @{} +foreach ($functionName in @("Get-AgentObjectValue", "Limit-AgentTextLine", "Format-AgentMetricForCard", "Format-AgentTelegramText")) { + $definition = $ast.Find({ + param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $functionName + }, $true) + if ($definition) { + Invoke-Expression $definition.Extent.Text + $formatFunctions[$functionName] = $true + } +} +if ($formatFunctions.ContainsKey("Format-AgentTelegramText")) { + $perfCard = Format-AgentTelegramText "warning" "performance_warning" "test" ([pscustomobject]@{ + host = "192.168.0.120" + reasons = @("perf_readback_failed") + loadPerCore = $null + memAvailablePercent = $null + diskUsedPercent = $null + }) + $providerCard = Format-AgentTelegramText "warning" "provider_freshness_triage" "test" ([pscustomobject]@{ + candidateId = "pf-test" + target = "provider_freshness_signal" + hitCount = 2 + missingFields = @("lastSeen") + status = "handled_blocked_by_trust_gate" + candidatePath = "C:\Wooo\Agent99\state\pf-test.json" + }) + $cards = "$perfCard`n$providerCard" + Add-SyntheticCheck "telegram:localized" ([regex]::IsMatch($cards, "[^\x00-\x7F]")) "localized card is present" + Add-SyntheticCheck "telegram:no_blank_metrics" (-not $perfCard.Contains("load/core=, ")) "failed readback renders unavailable values" + Add-SyntheticCheck "telegram:provider_handled" ($providerCard.Contains("handled_blocked_by_trust_gate") -and $providerCard.Contains("Agent99")) "handled state and agent are visible" + Add-SyntheticCheck "telegram:no_raw_python" (-not $cards.Contains("urllib.request")) "relay source is absent from user-facing card" +} else { + Add-SyntheticCheck "telegram:formatter" $false "Format-AgentTelegramText not found" +} + +$control = $controlSource +Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_already_queued") -and $control.Contains("recovery_trigger_cooldown") -and $control.Contains('mode = "Recover"')) "automatic recovery is queued and deduplicated" +Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present" + +$failures = @($checks | Where-Object { -not $_.ok }) +$result = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + ok = [bool]($failures.Count -eq 0) + workRoot = $WorkRoot + checkCount = $checks.Count + failureCount = $failures.Count + checks = $checks + failures = $failures +} +$result | ConvertTo-Json -Depth 8 +if (-not $result.ok) { exit 1 } +exit 0 diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 99fb6b33f..317d02866 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -2,6 +2,7 @@ "agentName": "Agent99", "agentRoot": "C:\\Wooo\\Agent99", "sshUser": "wooo", + "sshIdentityFile": "C:\\Wooo\\Agent99\\keys\\agent99_ed25519", "sshUsers": { "192.168.0.112": "kali", "192.168.0.188": "ollama" @@ -48,9 +49,26 @@ ], "k3s": { "jumpHost": "192.168.0.110", + "preferJumpHost": true, + "directHosts": [ + "192.168.0.110", + "192.168.0.112" + ], "primary": "192.168.0.120", "namespace": "awoooi-prod" }, + "autoRecovery": { + "enabled": true, + "triggerOnHostUnreachable": true, + "minimumUnreachableHosts": 1, + "cooldownMinutes": 10 + }, + "recoverySlo": { + "targetMinutes": 10, + "hostTimeoutSeconds": 90, + "harborTimeoutSeconds": 120, + "workloadTimeoutSeconds": 180 + }, "publicUrls": [ "awoooi.wooo.work", "stock.wooo.work", diff --git a/agent99.config.example.json b/agent99.config.example.json index 8508d5693..3855fb95d 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -2,6 +2,7 @@ "agentName": "Agent99", "agentRoot": "C:\\Wooo\\Agent99", "sshUser": "wooo", + "sshIdentityFile": "C:\\Wooo\\Agent99\\keys\\agent99_ed25519", "sshUsers": {}, "vmrunPath": "", "hosts": [ @@ -38,6 +39,28 @@ "vmx": "" } ], + "k3s": { + "jumpHost": "192.168.0.110", + "preferJumpHost": true, + "directHosts": [ + "192.168.0.110", + "192.168.0.112" + ], + "primary": "192.168.0.120", + "namespace": "awoooi-prod" + }, + "autoRecovery": { + "enabled": true, + "triggerOnHostUnreachable": true, + "minimumUnreachableHosts": 1, + "cooldownMinutes": 10 + }, + "recoverySlo": { + "targetMinutes": 10, + "hostTimeoutSeconds": 90, + "harborTimeoutSeconds": 120, + "workloadTimeoutSeconds": 180 + }, "publicUrls": [ "awoooi.wooo.work", "stock.wooo.work", diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py new file mode 100644 index 000000000..d01fc3bba --- /dev/null +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +CONTROL = ROOT / "agent99-control-plane.ps1" +DEPLOY = ROOT / "agent99-deploy.ps1" +BOOTSTRAP = ROOT / "agent99-bootstrap.ps1" + + +def test_agent99_uses_dedicated_identity_and_preferred_jump_route() -> None: + source = CONTROL.read_text(encoding="utf-8") + function = source[source.index("function Invoke-HostSshText") :] + function = function[: function.index("function Convert-AgentDouble")] + + assert '"sshIdentityFile"' in source + assert '"IdentitiesOnly=yes"' in source + assert 'PSObject.Properties["preferJumpHost"]' in function + assert 'PSObject.Properties["directHosts"]' in function + assert function.index("if ($preferJumpHost -and $canUseJump)") < function.index( + "$direct = Invoke-SshText" + ) + assert 'Write-AgentLog "ssh_jump_fallback' in function + + +def test_agent99_auto_recovery_is_single_flight_and_slo_measured() -> None: + source = CONTROL.read_text(encoding="utf-8") + trigger = source[source.index("function Start-AgentRecoveryFromObservation") :] + trigger = trigger[: trigger.index("function Get-AgentRecoverySloConfig")] + + assert 'reason = "recovery_already_queued"' in trigger + assert 'reason = "recovery_trigger_cooldown"' in trigger + assert 'source = "agent99-recovery-observer"' in trigger + assert 'mode = "Recover"' in trigger + assert "Start-Process" not in trigger + assert 'Record-AgentEvent "recovery_slo_result"' in source + assert "withinSlo = $withinSlo" in source + + +def test_agent99_99_config_has_bounded_transport_and_recovery_defaults() -> None: + config = json.loads((ROOT / "agent99.config.99.example.json").read_text()) + + assert config["sshIdentityFile"].endswith(r"keys\agent99_ed25519") + assert config["k3s"]["jumpHost"] == "192.168.0.110" + assert config["k3s"]["preferJumpHost"] is True + assert config["k3s"]["directHosts"] == ["192.168.0.110", "192.168.0.112"] + assert config["autoRecovery"]["enabled"] is True + assert config["autoRecovery"]["cooldownMinutes"] == 10 + assert config["recoverySlo"]["targetMinutes"] == 10 + + +def test_agent99_deployer_verifies_promoted_runtime_and_writes_revision() -> None: + deploy = DEPLOY.read_text(encoding="utf-8") + bootstrap = BOOTSTRAP.read_text(encoding="utf-8") + + assert 'schemaVersion = "agent99_deploy_receipt_v2"' in deploy + assert 'schemaVersion = "agent99_runtime_manifest_v1"' in deploy + assert "sourceRevision = $SourceRevision" in deploy + assert 'Join-Path $BinDir "agent99-contract-check.ps1"' in deploy + assert 'Join-Path $BinDir "agent99-synthetic-tests.ps1"' in deploy + assert "if (-not $runtimeMatched)" in deploy + for name in ( + "agent99-contract-check.ps1", + "agent99-deploy.ps1", + "agent99-synthetic-tests.ps1", + ): + assert f'"{name}"' in deploy + assert f'"{name}"' in bootstrap