From 3b0d6a2ed30d11a3ed0df9d260024977d456ad2f Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 13:47:55 +0800 Subject: [PATCH] fix(agent99): recover external host through full closure chain --- agent99-contract-check.ps1 | 17 +- agent99-control-plane.ps1 | 219 +++++++++++++++++- agent99-deploy.ps1 | 86 ++++++- agent99-register-tasks.ps1 | 9 +- agent99.config.99.example.json | 29 +++ agent99.config.example.json | 29 +++ ...nt99_transport_recovery_deploy_contract.py | 67 +++++- ...t_agent99_recovery_coordinator_contract.py | 17 ++ 8 files changed, 462 insertions(+), 11 deletions(-) diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index 181b0c58c..bc8333ff7 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -46,7 +46,16 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js $path = Join-Path $SourceRoot $name try { $config = Get-Content $path -Raw | ConvertFrom-Json - $hasHosts = @($config.hosts).Count -eq 5 + $hasHosts = @($config.hosts).Count -eq 6 -and "192.168.0.111" -in @($config.hosts) + $host111External = @($config.externalHosts | Where-Object { $_.host -eq "192.168.0.111" } | Select-Object -First 1) + $hasExternalHostRecovery = [bool]( + $host111External.Count -eq 1 -and + $host111External[0].recoveryAction -eq "wake_on_lan" -and + $host111External[0].enabled -eq $true -and + $host111External[0].required -eq $true -and + @($host111External[0].macAddresses).Count -ge 1 -and + 22 -in @($host111External[0].verifyTcpPorts) + ) $hasRoutes = @($config.publicUrls).Count -ge 5 $hasSlo = [bool]($config.recoverySlo -and [int]$config.recoverySlo.targetMinutes -eq 10) $hasIdentity = [bool]($config.sshIdentityFile -and $config.sshIdentityFile -match "agent99_ed25519$") @@ -71,8 +80,8 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js [string]$_.vmx -eq "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" }).Count -eq 1) } else { $true } - $ok = $hasHosts -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasCompletionCallback -and $hasCanonicalHost112Vmx - Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator completionCallback=$hasCompletionCallback canonicalHost112Vmx=$hasCanonicalHost112Vmx" + $ok = $hasHosts -and $hasExternalHostRecovery -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasCompletionCallback -and $hasCanonicalHost112Vmx + Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) externalHostRecovery=$hasExternalHostRecovery routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator completionCallback=$hasCompletionCallback canonicalHost112Vmx=$hasCanonicalHost112Vmx" } catch { Add-Check "config:$name" $false "json_parse_failed: $($_.Exception.Message)" } @@ -94,6 +103,8 @@ Add-Check "recovery:durable_boot_event" ($control.Contains("pendingRecovery") -a Add-Check "recovery:terminal_evidence_budget" ($taskRegistration.Contains('$recoverySettings') -and $taskRegistration.Contains('New-TimeSpan -Minutes 20')) "startup recovery outlives the ten-minute SLO long enough to write terminal evidence" Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present" Add-Check "recovery:full_sop_coordinator" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_scorecard") -and $control.Contains("SCORECARD_FRESH_REBOOT_WINDOW") -and $control.Contains('scope = $recoveryScope')) "Agent99 cannot close reboot recovery without the 110 full-SOP scorecard" +Add-Check "recovery:host111_wol_executor" ($control.Contains("Invoke-AgentExternalHostRecovery") -and $control.Contains("Send-AgentWakeOnLan") -and $control.Contains('Get-AgentObjectValue $externalHost "recoveryAction" "wake_on_lan"') -and $control.Contains('New-AgentRecoveryPhase "external_host_recovery"') -and $control.Contains('New-AgentOutcomeCheck "external_hosts_recovered"')) "Host111 has an allowlisted WOL executor with independent ping/TCP verification" +Add-Check "recovery:startup_queue_closure" ($taskRegistration.Contains('$submitRequest') -and $taskRegistration.Contains('-Mode Recover -ControlledApply -RunNow') -and $taskRegistration.Contains('-Source agent99-startup-recovery')) "startup recovery runs through queue, outcome, callback, Telegram lifecycle, and KM/RAG" Add-Check "recovery:host112_immutable_apply_closure" ($control.Contains("managerApplyVerified") -and $control.Contains("durableReceiptVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('artifactTransactionStatus -eq "committed_verified_pair"') -and $control.Contains('artifactTransactionPriorPairVerified') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply closure binds transport, fixed WAL, attributed start, immutable receipt/readback and independent verifier" Add-Check "recovery:host112_direct_transport" ($control.Contains("Invoke-AgentHost112SshText") -and $control.Contains('expectedUser = "kali"') -and $control.Contains('route = "direct_host112_kali"') -and $deployer.Contains('name = "host112_canonical_direct_route"') -and $bootstrap.Contains("Ensure-AgentHost112CanonicalSshConfig")) "Host112 recovery uses the canonical 99 direct key route" Add-Check "recovery:host112_canonical_vmx" ($deployer.Contains('name = "host112_canonical_vmx_path"') -and $deployer.Contains('Get-AgentHost112CanonicalVm $policyConfigPath') -and $deployer.Contains('Test-Path -LiteralPath $canonicalHost112Vmx -PathType Leaf') -and $bootstrap.Contains('Get-AgentHost112CanonicalVm $PolicyPath') -and $bootstrap.Contains('Set-AgentBootstrapProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))')) "Host112 VM identity is normalized from one staged policy row and fails closed when absent" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index b7f7edb99..b1015e916 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -843,7 +843,7 @@ function Get-AgentIncidentCardModel { "^performance_|^load_shed_" { "host_performance"; break } "^backup_" { "backup"; break } "provider_freshness" { "provider_freshness"; break } - "^recovery_|^host_reboot_|^host112_guest_" { "recovery"; break } + "^recovery_|^host_reboot_|^host112_guest_|^external_host_" { "recovery"; break } "^operator_command_" { "automation_result"; break } "^security_" { "security"; break } "^ai_service_" { "ai_service"; break } @@ -1004,7 +1004,7 @@ function Get-AgentIncidentCardModel { "BackupCheck" { "讀回 freshness、cron、integrity 與 restore drill receipt；不讀備份內容。" } "Perf" { "讀回 CPU、記憶體、磁碟；必要時只做 allowlisted 降載。" } "LoadShed" { "執行 allowlisted 降載並立即做 post-apply verifier。" } - "Recover" { "依序驗證 VM、SSH、112 readiness、Harbor/K3s、網站與 full-SOP scorecard。" } + "Recover" { "依序驗證 VM、Host111 WOL、SSH、112 readiness、Harbor/K3s、網站與 full-SOP scorecard。" } "StartVMs" { "檢查 VMX/lock/guest identity 後，只啟動 allowlisted VM。" } "HarborRepair" { "檢查 core、Redis、Registry 後執行可回滾修復。" } "AwoooRepair" { "檢查 K3s workload/image pull 後執行受控修復。" } @@ -1035,6 +1035,13 @@ function Get-AgentIncidentCardModel { $action = "執行 allowlisted 降載，保留 rollback 與 cooldown receipt。" $verification = if ($executionOk -and $postVerifyOk) { "執行與 post-verifier 均通過。" } else { "post-verifier 未通過，事件保持開啟且不建立成功 cooldown。" } $visualKind = "automation" + } elseif ($EventType -match "^external_host_") { + $target = if ($targetName) { $targetName } elseif ($hostName) { $hostName } else { "Host111 workstation" } + $title = if ($lifecycle -eq "recovered") { "$target 已自動喚醒" } else { "$target 自動喚醒未通過驗證" } + $impact = "必要工作站離線會阻斷全主機 cold-start 與 10 分鐘 SLA 證明。" + $action = "只送 allowlisted WOL magic packet；不重啟、不切電、不改防火牆。" + $verification = if ($lifecycle -eq "recovered") { "ping 與必要 TCP port 均已通過。" } else { "WOL 已執行，但 ping/TCP verifier 仍失敗；事件保持開啟。" } + $visualKind = "recovery" } elseif ($EventType -match "^host112_guest_") { $target = "192.168.0.112 Kali" $title = if ($lifecycle -eq "recovered") { "Kali 112 已完整恢復" } else { "Kali 112 尚未完整開機" } @@ -1374,6 +1381,7 @@ function New-AgentTelegramCardImage { $phaseBrush = if ($passed) { $goodBrush } else { $badBrush } $phaseLabel = switch ([string]$phase.name) { "vm_start" { "VMware" } + "external_host_recovery" { "Host111 WOL" } "host_reachability" { "主機連線" } "host112_guest_readiness" { "Kali 112" } "harbor_registry" { "Harbor" } @@ -4286,6 +4294,9 @@ function Get-AgentOutcomeContract { $expectedVms = @($Config.vms).Count $vmRows = @($data.vmResults) $vmsOk = [bool]($expectedVms -gt 0 -and $vmRows.Count -ge $expectedVms -and @($vmRows | Where-Object { -not $_.ok }).Count -eq 0) + $expectedExternalHosts = @($Config.externalHosts | Where-Object { -not $_.PSObject.Properties["enabled"] -or (Convert-AgentBool $_.enabled) } | Where-Object { -not $_.PSObject.Properties["required"] -or (Convert-AgentBool $_.required) }).Count + $externalHostRows = @($data.externalHostRecovery | Where-Object { $_.required }) + $externalHostsOk = [bool]($expectedExternalHosts -eq 0 -or ($externalHostRows.Count -ge $expectedExternalHosts -and @($externalHostRows | Where-Object { -not $_.verified }).Count -eq 0)) $expectedHosts = @($Config.hosts).Count $hostRows = @($data.hosts) $hostsOk = [bool]($expectedHosts -gt 0 -and $hostRows.Count -ge $expectedHosts -and @($hostRows | Where-Object { -not $_.ping }).Count -eq 0) @@ -4296,13 +4307,14 @@ function Get-AgentOutcomeContract { $coordinatorOk = [bool]($data.recoveryCoordinator -and $data.recoveryCoordinator.verified) $host112Ok = [bool]($data.host112Recovery -and $data.host112Recovery.verified) $checks += New-AgentOutcomeCheck "configured_vms_ready" $vmsOk $true "expected=$expectedVms actual=$($vmRows.Count)" + $checks += New-AgentOutcomeCheck "external_hosts_recovered" $externalHostsOk $true "expected=$expectedExternalHosts actual=$($externalHostRows.Count)" $checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)" $checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk $checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk $checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk $checks += New-AgentOutcomeCheck "host112_guest_ready" $host112Ok $true $([string](@($data.host112Recovery.after.failedChecks) -join ',')) $checks += New-AgentOutcomeCheck "full_sop_coordinator_verified" $coordinatorOk $true $([string](@($data.recoveryCoordinator.failedChecks) -join ',')) - $modeVerifierPassed = [bool]($vmsOk -and $hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk) + $modeVerifierPassed = [bool]($vmsOk -and $externalHostsOk -and $hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk) } "StartVMs" { $expectedVms = @($Config.vms).Count @@ -6508,6 +6520,195 @@ function Invoke-AgentHost112GuestRecovery { $result } +function Test-AgentTcpPort { + param( + [string]$TargetHost, + [int]$Port, + [int]$TimeoutMilliseconds = 1200 + ) + + $client = New-Object System.Net.Sockets.TcpClient + try { + $async = $client.BeginConnect($TargetHost, $Port, $null, $null) + if (-not $async.AsyncWaitHandle.WaitOne($TimeoutMilliseconds, $false)) { + return $false + } + $client.EndConnect($async) + return [bool]$client.Connected + } catch { + return $false + } finally { + $client.Close() + } +} + +function Test-AgentExternalHostReadiness { + param([object]$ExternalHost) + + $targetHost = [string](Get-AgentObjectValue $ExternalHost "host" "") + $requiredPorts = @( + @(Get-AgentObjectValue $ExternalHost "verifyTcpPorts" @(22)) | + ForEach-Object { [int]$_ } | + Where-Object { $_ -gt 0 -and $_ -le 65535 } | + Select-Object -Unique + ) + $ping = [bool]($targetHost -and (Test-Connection -ComputerName $targetHost -Count 1 -Quiet -ErrorAction SilentlyContinue)) + $portChecks = @($requiredPorts | ForEach-Object { + [pscustomobject]@{ + port = [int]$_ + ready = [bool](Test-AgentTcpPort -TargetHost $targetHost -Port ([int]$_)) + } + }) + [pscustomobject]@{ + targetHost = $targetHost + ping = $ping + requiredPorts = $requiredPorts + portChecks = $portChecks + ready = [bool]($targetHost -and $ping -and @($portChecks | Where-Object { -not $_.ready }).Count -eq 0) + observedAt = (Get-Date -Format o) + } +} + +function Send-AgentWakeOnLan { + param([object]$ExternalHost) + + $macAddresses = @( + @(Get-AgentObjectValue $ExternalHost "macAddresses" @()) | + ForEach-Object { [string]$_ } | + Where-Object { $_ } + ) + $broadcastAddresses = @( + @(Get-AgentObjectValue $ExternalHost "broadcastAddresses" @("255.255.255.255")) | + ForEach-Object { [string]$_ } | + Where-Object { $_ } + ) + $ports = @( + @(Get-AgentObjectValue $ExternalHost "ports" @(7, 9)) | + ForEach-Object { [int]$_ } | + Where-Object { $_ -gt 0 -and $_ -le 65535 } | + Select-Object -Unique + ) + $packetBursts = [math]::Max(1, [math]::Min(10, [int](Get-AgentObjectValue $ExternalHost "packetBursts" 5))) + $sentPackets = 0 + $validMacCount = 0 + $udp = New-Object System.Net.Sockets.UdpClient + $udp.EnableBroadcast = $true + try { + for ($burst = 0; $burst -lt $packetBursts; $burst += 1) { + foreach ($macAddress in $macAddresses) { + $hex = $macAddress -replace "[^0-9A-Fa-f]", "" + if ($hex.Length -ne 12) { continue } + $targetBytes = New-Object byte[] 6 + for ($index = 0; $index -lt 6; $index += 1) { + $targetBytes[$index] = [Convert]::ToByte($hex.Substring($index * 2, 2), 16) + } + $validMacCount += 1 + $packet = New-Object byte[] 102 + for ($index = 0; $index -lt 6; $index += 1) { $packet[$index] = 0xFF } + for ($repeat = 0; $repeat -lt 16; $repeat += 1) { + [Array]::Copy($targetBytes, 0, $packet, 6 + ($repeat * 6), 6) + } + foreach ($broadcastAddress in $broadcastAddresses) { + foreach ($port in $ports) { + $endpoint = New-Object System.Net.IPEndPoint ([System.Net.IPAddress]::Parse($broadcastAddress)), ([int]$port) + [void]$udp.Send($packet, $packet.Length, $endpoint) + $sentPackets += 1 + } + } + } + Start-Sleep -Milliseconds 250 + } + [pscustomobject]@{ + ok = [bool]($validMacCount -gt 0 -and $sentPackets -gt 0) + transport = "windows99_udp_broadcast" + packetBursts = $packetBursts + validMacCount = $validMacCount + destinationCount = $broadcastAddresses.Count + ports = $ports + sentPackets = $sentPackets + errorType = $null + } + } catch { + [pscustomobject]@{ + ok = $false + transport = "windows99_udp_broadcast" + packetBursts = $packetBursts + validMacCount = $validMacCount + destinationCount = $broadcastAddresses.Count + ports = $ports + sentPackets = $sentPackets + errorType = $_.Exception.GetType().Name + } + } finally { + $udp.Close() + } +} + +function Invoke-AgentExternalHostRecovery { + $configuredHosts = if ($Config.PSObject.Properties["externalHosts"]) { @($Config.externalHosts) } else { @() } + $results = @() + foreach ($externalHost in $configuredHosts) { + $enabled = [bool](-not $externalHost.PSObject.Properties["enabled"] -or (Convert-AgentBool $externalHost.enabled)) + if (-not $enabled) { continue } + + $name = [string](Get-AgentObjectValue $externalHost "name" (Get-AgentObjectValue $externalHost "host" "external-host")) + $targetHost = [string](Get-AgentObjectValue $externalHost "host" "") + $required = [bool](-not $externalHost.PSObject.Properties["required"] -or (Convert-AgentBool $externalHost.required)) + $recoveryAction = [string](Get-AgentObjectValue $externalHost "recoveryAction" "wake_on_lan") + $before = Test-AgentExternalHostReadiness $externalHost + $after = $before + $actionAttempted = $false + $actionReceipt = $null + $terminal = if ($before.ready) { "idempotent_already_verified_healthy" } else { "observed_unreachable" } + + if (-not $before.ready -and -not $ControlledApply) { + $terminal = "controlled_apply_required" + } elseif (-not $before.ready -and $recoveryAction -ne "wake_on_lan") { + $terminal = "unsupported_recovery_action" + } elseif (-not $before.ready) { + $actionAttempted = $true + Write-AgentLog "CONTROLLED_APPLY external_host_wol name=$name host=$targetHost" + $actionReceipt = Send-AgentWakeOnLan $externalHost + $verifyTimeoutSeconds = [math]::Max(5, [math]::Min(300, [int](Get-AgentObjectValue $externalHost "verifyTimeoutSeconds" 180))) + $verifyIntervalSeconds = [math]::Max(1, [math]::Min(10, [int](Get-AgentObjectValue $externalHost "verifyIntervalSeconds" 5))) + $deadline = (Get-Date).AddSeconds($verifyTimeoutSeconds) + $after = Test-AgentExternalHostReadiness $externalHost + while (-not $after.ready -and (Get-Date) -lt $deadline) { + Start-Sleep -Seconds $verifyIntervalSeconds + $after = Test-AgentExternalHostReadiness $externalHost + } + $terminal = if ($after.ready) { "verified_ready" } elseif (-not $actionReceipt.ok) { "wol_transport_failed" } else { "wake_sent_not_verified" } + } + + $result = [pscustomobject]@{ + schemaVersion = "agent99_external_host_recovery_v1" + name = $name + targetHost = $targetHost + required = $required + recoveryAction = $recoveryAction + controlledApply = [bool]$ControlledApply + actionAttempted = $actionAttempted + runtimeWritePerformed = [bool]($actionAttempted -and $actionReceipt -and $actionReceipt.sentPackets -gt 0) + changed = [bool]($actionAttempted -and -not $before.ready -and $after.ready) + verified = [bool]$after.ready + before = $before + actionReceipt = $actionReceipt + after = $after + terminal = $terminal + rollback = "not_applicable_magic_packet_is_stateless" + verifier = "icmp_and_required_tcp_ports" + prohibitedActions = @("host_reboot", "power_cut", "service_restart", "secret_read", "firewall_change", "database_write") + } + if ($result.changed) { + Record-AgentEvent "external_host_recovered" "info" "$name 已由 Agent99 WOL 喚醒並通過 ping/TCP verifier。" $result -Alert + } elseif ($actionAttempted -and -not $result.verified) { + Record-AgentEvent "external_host_recovery_failed" "critical" "$name 已執行 WOL，但仍未通過 ping/TCP verifier。" $result -Alert + } + $results += $result + } + $results +} + function Test-HostReachability { $results = @() foreach ($hostIp in $Config.hosts) { @@ -6798,6 +6999,8 @@ if ($ReconcileOnly -or $ReconcileQueueOnly) { $vmResults = @() $hostResults = @() +$externalHostRecovery = @() +$requiredExternalHostFailures = @() $host112Recovery = $null $harbor = $null $awooo = $null @@ -6845,6 +7048,13 @@ if ($Mode -in @("Recover", "StartVMs")) { } } +if ($Mode -eq "Recover") { + $externalHostRecovery = Invoke-AgentExternalHostRecovery + $requiredExternalHostFailures = @($externalHostRecovery | Where-Object { $_.required -and -not $_.verified }) + $externalHostStatus = if ($requiredExternalHostFailures.Count -eq 0) { "ok" } else { "failed" } + $recoveryPhases += New-AgentRecoveryPhase "external_host_recovery" $externalHostStatus (((Get-Date) - $recoveryStartedAt).TotalSeconds) +} + if ($Mode -in @("Status", "Recover")) { $hostResults = Test-HostReachability $unreachableHosts = @($hostResults | Where-Object { -not $_.ping }) @@ -6988,7 +7198,7 @@ $summarySeverity = if ($publicFailures -gt 0 -or $aiFailures -gt 0 -or $perfCrit if ($Mode -eq "Recover") { $elapsedSeconds = [math]::Round(((Get-Date) - $recoveryStartedAt).TotalSeconds, 1) $unreachableCount = @($hostResults | Where-Object { -not $_.ping }).Count - $builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $host112Recovery -and $host112Recovery.verified -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0) + $builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $requiredExternalHostFailures.Count -eq 0 -and $host112Recovery -and $host112Recovery.verified -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0) $coordinatorVerified = [bool]($recoveryCoordinator -and $recoveryCoordinator.verified) $requiresFreshWindow = [bool]($recoveryCoordinator -and $recoveryCoordinator.requireFreshRebootWindow) $rebootSloClaimed = [bool]($requiresFreshWindow -and $recoveryCoordinator.rebootSloClaimed) @@ -7045,6 +7255,7 @@ $result = [pscustomobject]@{ recoverySlo = $recoverySlo recoveryCoordinator = $recoveryCoordinator vmResults = $vmResults + externalHostRecovery = $externalHostRecovery hosts = $hostResults host112Recovery = $host112Recovery harbor = $harbor diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index 6e1b08a63..a25ab60f9 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -245,6 +245,65 @@ try { Add-DefaultProperty $config.sshProcessGuard "enabled" $true Add-DefaultProperty $config.sshProcessGuard "staleMinutes" 15 Add-DefaultProperty $config.sshProcessGuard "hardStaleMinutes" 60 + Add-DefaultProperty $config "hosts" @("192.168.0.110", "192.168.0.111", "192.168.0.112", "192.168.0.120", "192.168.0.121", "192.168.0.188") + $configuredHosts = @($config.hosts | ForEach-Object { [string]$_ } | Where-Object { $_ }) + if ("192.168.0.111" -notin $configuredHosts) { + Set-AgentProperty $config "hosts" @($configuredHosts + "192.168.0.111" | Select-Object -Unique) + $script:ConfigMigrations += [pscustomobject]@{ + name = "host111_canonical_recovery_target" + from = ($configuredHosts -join ",") + to = ((@($configuredHosts + "192.168.0.111") | Select-Object -Unique) -join ",") + } + } + Add-DefaultProperty $config "externalHosts" @() + $externalHosts = @($config.externalHosts) + $host111External = $externalHosts | Where-Object { [string]$_.host -eq "192.168.0.111" } | Select-Object -First 1 + if (-not $host111External) { + $host111External = [pscustomobject]@{ + name = "host111-workstation" + host = "192.168.0.111" + recoveryAction = "wake_on_lan" + macAddresses = @("00:E0:4C:AF:10:0A", "DA:64:76:98:F1:0E") + broadcastAddresses = @("255.255.255.255", "192.168.0.255") + ports = @(7, 9) + packetBursts = 5 + verifyTimeoutSeconds = 180 + verifyIntervalSeconds = 5 + verifyTcpPorts = @(22) + enabled = $true + required = $true + } + Set-AgentProperty $config "externalHosts" @($externalHosts + $host111External) + $script:ConfigMigrations += [pscustomobject]@{ + name = "host111_wol_executor" + from = "missing" + to = "wake_on_lan" + } + } else { + Add-DefaultProperty $host111External "name" "host111-workstation" + Add-DefaultProperty $host111External "recoveryAction" "wake_on_lan" + Add-DefaultProperty $host111External "macAddresses" @("00:E0:4C:AF:10:0A", "DA:64:76:98:F1:0E") + Add-DefaultProperty $host111External "broadcastAddresses" @("255.255.255.255", "192.168.0.255") + Add-DefaultProperty $host111External "ports" @(7, 9) + Add-DefaultProperty $host111External "packetBursts" 5 + Add-DefaultProperty $host111External "verifyTimeoutSeconds" 180 + Add-DefaultProperty $host111External "verifyIntervalSeconds" 5 + Add-DefaultProperty $host111External "verifyTcpPorts" @(22) + Add-DefaultProperty $host111External "enabled" $true + Add-DefaultProperty $host111External "required" $true + } + Add-DefaultProperty $config "vms" @() + $host112Vm = @($config.vms | Where-Object { [string]$_.host -eq "192.168.0.112" -or [string]$_.name -eq "host112-kali" } | Select-Object -First 1) + $host112CanonicalVmx = "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" + if ($host112Vm.Count -gt 0 -and [string]$host112Vm[0].vmx -ne $host112CanonicalVmx) { + $previousHost112Vmx = [string]$host112Vm[0].vmx + Set-AgentProperty $host112Vm[0] "vmx" $host112CanonicalVmx + $script:ConfigMigrations += [pscustomobject]@{ + name = "host112_ssd_vmx_source" + from = if ($previousHost112Vmx) { $previousHost112Vmx } else { "missing" } + to = $host112CanonicalVmx + } + } Add-DefaultProperty $config "k3s" ([pscustomobject]@{}) Add-DefaultProperty $config.k3s "jumpHost" "192.168.0.110" Add-DefaultProperty $config.k3s "preferJumpHost" $true @@ -307,6 +366,17 @@ try { Add-DefaultProperty $config.recoveryCoordinator "maxArtifactAgeSeconds" 600 Add-DefaultProperty $config.recoveryCoordinator "artifactClockSkewSeconds" 90 Add-DefaultProperty $config.recoveryCoordinator "requiredHostAliases" @("99", "110", "111", "112", "120", "121", "188") + Add-DefaultProperty $config "selfHealth" ([pscustomobject]@{}) + Add-DefaultProperty $config.selfHealth "requiredHosts" @("192.168.0.110", "192.168.0.111", "192.168.0.112", "192.168.0.120", "192.168.0.121", "192.168.0.188") + $selfHealthHosts = @($config.selfHealth.requiredHosts | ForEach-Object { [string]$_ } | Where-Object { $_ }) + if ("192.168.0.111" -notin $selfHealthHosts) { + Set-AgentProperty $config.selfHealth "requiredHosts" @($selfHealthHosts + "192.168.0.111" | Select-Object -Unique) + $script:ConfigMigrations += [pscustomobject]@{ + name = "host111_self_health_coverage" + from = ($selfHealthHosts -join ",") + to = ((@($selfHealthHosts + "192.168.0.111") | Select-Object -Unique) -join ",") + } + } Add-DefaultProperty $config "completionCallback" ([pscustomobject]@{}) Add-DefaultProperty $config.completionCallback "enabled" $true Add-DefaultProperty $config.completionCallback "url" "https://awoooi.wooo.work/api/v1/agents/agent99/completion-callback" @@ -346,16 +416,30 @@ try { [string]$_.host -eq "192.168.0.112" -or [string]$_.name -eq "host112-kali" }) $persistedHost112Vmx = if ($persistedHost112Vm.Count -eq 1) { [string]$persistedHost112Vm[0].vmx } else { "" } + $persistedHosts = @($persistedConfig.hosts | ForEach-Object { [string]$_ }) + $persistedSelfHealthHosts = @($persistedConfig.selfHealth.requiredHosts | ForEach-Object { [string]$_ }) + $persistedHost111External = @($persistedConfig.externalHosts | Where-Object { [string]$_.host -eq "192.168.0.111" }) + $host111ExternalReady = [bool]( + $persistedHost111External.Count -eq 1 -and + [string]$persistedHost111External[0].recoveryAction -eq "wake_on_lan" -and + [bool]$persistedHost111External[0].enabled -and + [bool]$persistedHost111External[0].required -and + @($persistedHost111External[0].macAddresses).Count -ge 1 -and + 22 -in @($persistedHost111External[0].verifyTcpPorts | ForEach-Object { [int]$_ }) + ) if ( $persistedHost112User -ne "kali" -or "192.168.0.112" -notin $persistedDirectHosts -or + "192.168.0.111" -notin $persistedHosts -or + "192.168.0.111" -notin $persistedSelfHealthHosts -or + -not $host111ExternalReady -or $persistedHost112Vm.Count -ne 1 -or [string]$persistedHost112Vm[0].name -ne [string]$canonicalHost112Vm.name -or [string]$persistedHost112Vm[0].host -ne [string]$canonicalHost112Vm.host -or $persistedHost112Vmx -ne $canonicalHost112Vmx -or -not (Test-Path -LiteralPath $persistedHost112Vmx -PathType Leaf) ) { - throw "Host112 canonical SSH and VMX config post-verifier failed." + throw "Canonical Host111 recovery and Host112 direct/SSD config post-verifier failed." } $manifestRows = @() diff --git a/agent99-register-tasks.ps1 b/agent99-register-tasks.ps1 index f78f16fee..de6c6a91b 100644 --- a/agent99-register-tasks.ps1 +++ b/agent99-register-tasks.ps1 @@ -8,6 +8,10 @@ $launcher = Join-Path $AgentRoot "agent99-run.ps1" if (-not (Test-Path $launcher)) { throw "Agent99 launcher not found: $launcher" } +$submitRequest = Join-Path $AgentRoot "bin\agent99-submit-request.ps1" +if (-not (Test-Path $submitRequest)) { + throw "Agent99 submit request entrypoint not found: $submitRequest" +} $ps = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" $principalUser = if ($env:COMPUTERNAME) { "$env:COMPUTERNAME\$env:USERNAME" } else { $env:USERNAME } @@ -31,7 +35,8 @@ $serviceSettings = New-ScheduledTaskSettingsSet ` -MultipleInstances IgnoreNew ` -ExecutionTimeLimit ([TimeSpan]::Zero) -$recoverArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$launcher`" -Mode Recover -ControlledApply" +$recoverInstruction = "Windows99 startup detected; run the allowlisted all-host, VM, service, website, freshness, verifier, callback, and learning recovery chain." +$recoverArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$submitRequest`" -Instruction `"$recoverInstruction`" -Mode Recover -ControlledApply -RunNow -Source agent99-startup-recovery -ExternalId windows99-startup" $recoverAction = New-ScheduledTaskAction -Execute $ps -Argument $recoverArgs $startupTrigger = New-ScheduledTaskTrigger -AtStartup $logonTrigger = New-ScheduledTaskTrigger -AtLogOn -User $principalUser @@ -41,7 +46,7 @@ Register-ScheduledTask ` -Trigger @($startupTrigger, $logonTrigger) ` -Principal $principal ` -Settings $recoverySettings ` - -Description "Agent99 startup recovery for VMs, Harbor, K3s, and public routes." ` + -Description "Agent99 startup recovery through the controlled queue, verifier, callback, Telegram lifecycle, and KM/RAG chain." ` -Force | Out-Null $heartbeatArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$launcher`" -Mode Status" diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 4e6d52afc..77e567ba3 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -20,6 +20,7 @@ "vmrunPath": "C:\\Program Files (x86)\\VMware\\VMware Workstation\\vmrun.exe", "hosts": [ "192.168.0.110", + "192.168.0.111", "192.168.0.112", "192.168.0.120", "192.168.0.121", @@ -57,6 +58,33 @@ "priority": 50 } ], + "externalHosts": [ + { + "name": "host111-workstation", + "host": "192.168.0.111", + "recoveryAction": "wake_on_lan", + "macAddresses": [ + "00:E0:4C:AF:10:0A", + "DA:64:76:98:F1:0E" + ], + "broadcastAddresses": [ + "255.255.255.255", + "192.168.0.255" + ], + "ports": [ + 7, + 9 + ], + "packetBursts": 5, + "verifyTimeoutSeconds": 180, + "verifyIntervalSeconds": 5, + "verifyTcpPorts": [ + 22 + ], + "enabled": true, + "required": true + } + ], "k3s": { "jumpHost": "192.168.0.110", "preferJumpHost": true, @@ -268,6 +296,7 @@ ], "requiredHosts": [ "192.168.0.110", + "192.168.0.111", "192.168.0.112", "192.168.0.120", "192.168.0.121", diff --git a/agent99.config.example.json b/agent99.config.example.json index aa7c6469c..617f4f31b 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -19,6 +19,7 @@ "vmrunPath": "", "hosts": [ "192.168.0.110", + "192.168.0.111", "192.168.0.112", "192.168.0.120", "192.168.0.121", @@ -51,6 +52,33 @@ "vmx": "" } ], + "externalHosts": [ + { + "name": "host111-workstation", + "host": "192.168.0.111", + "recoveryAction": "wake_on_lan", + "macAddresses": [ + "00:E0:4C:AF:10:0A", + "DA:64:76:98:F1:0E" + ], + "broadcastAddresses": [ + "255.255.255.255", + "192.168.0.255" + ], + "ports": [ + 7, + 9 + ], + "packetBursts": 5, + "verifyTimeoutSeconds": 180, + "verifyIntervalSeconds": 5, + "verifyTcpPorts": [ + 22 + ], + "enabled": true, + "required": true + } + ], "k3s": { "jumpHost": "192.168.0.110", "preferJumpHost": true, @@ -206,6 +234,7 @@ ], "requiredHosts": [ "192.168.0.110", + "192.168.0.111", "192.168.0.112", "192.168.0.120", "192.168.0.121", diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index f850d9d70..7ea4532d8 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -73,7 +73,7 @@ def test_agent99_telegram_v5_hides_raw_evidence_and_has_operator_fields() -> Non assert "kmPath" not in formatter assert "verifierName=" not in formatter assert 'messageFormat = "incident_card_zh_tw_v5"' in source - for phase_label in ("VMware", "主機連線", "Kali 112", "公開網站", "Full SOP"): + for phase_label in ("VMware", "Host111 WOL", "主機連線", "Kali 112", "公開網站", "Full SOP"): assert f'{{ "{phase_label}" }}' in source @@ -169,6 +169,71 @@ def test_agent99_99_config_has_bounded_transport_and_recovery_defaults() -> None assert config["autoRecovery"]["enabled"] is True assert config["autoRecovery"]["cooldownMinutes"] == 10 assert config["recoverySlo"]["targetMinutes"] == 10 + assert "192.168.0.111" in config["hosts"] + assert "192.168.0.111" in config["selfHealth"]["requiredHosts"] + host111 = next(item for item in config["externalHosts"] if item["host"] == "192.168.0.111") + assert host111["recoveryAction"] == "wake_on_lan" + assert host111["required"] is True + assert host111["enabled"] is True + assert host111["verifyTcpPorts"] == [22] + assert host111["verifyTimeoutSeconds"] <= 300 + host112 = next(item for item in config["vms"] if item["host"] == "192.168.0.112") + assert host112["vmx"] == r"S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" + + +def test_agent99_host111_recovery_is_allowlisted_and_independently_verified() -> None: + source = CONTROL.read_text(encoding="utf-8") + function = source[source.index("function Test-AgentTcpPort") :] + function = function[: function.index("function Test-HostReachability")] + + assert "function Send-AgentWakeOnLan" in function + assert "System.Net.Sockets.UdpClient" in function + assert 'transport = "windows99_udp_broadcast"' in function + assert "Invoke-AgentExternalHostRecovery" in function + assert "Test-AgentExternalHostReadiness" in function + assert 'terminal = if ($after.ready) { "verified_ready" }' in function + assert 'verifier = "icmp_and_required_tcp_ports"' in function + for prohibited in ( + "host_reboot", + "power_cut", + "service_restart", + "secret_read", + "firewall_change", + "database_write", + ): + assert f'"{prohibited}"' in function + + recover = source[source.index('if ($Mode -eq "Recover") {\n $externalHostRecovery') :] + assert 'New-AgentRecoveryPhase "external_host_recovery"' in recover + assert "$requiredExternalHostFailures.Count -eq 0" in recover + assert 'New-AgentOutcomeCheck "external_hosts_recovered"' in source + + +def test_agent99_startup_recovery_uses_full_outcome_callback_and_learning_chain() -> None: + tasks = (ROOT / "agent99-register-tasks.ps1").read_text(encoding="utf-8") + + assert '$submitRequest = Join-Path $AgentRoot "bin\\agent99-submit-request.ps1"' in tasks + assert '-Mode Recover -ControlledApply -RunNow' in tasks + assert '-Source agent99-startup-recovery' in tasks + assert '-ExternalId windows99-startup' in tasks + recover_action = tasks[tasks.index("$recoverInstruction") : tasks.index("$heartbeatArgs")] + assert '$launcher`" -Mode Recover' not in recover_action + assert "callback, Telegram lifecycle, and KM/RAG" in tasks + + +def test_agent99_deployer_migrates_and_verifies_host111_and_host112_sources() -> None: + deploy = DEPLOY.read_text(encoding="utf-8") + + for migration in ( + "host111_canonical_recovery_target", + "host111_wol_executor", + "host111_self_health_coverage", + "host112_ssd_vmx_source", + ): + assert migration in deploy + assert '"S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx"' in deploy + assert "$host111ExternalReady" in deploy + assert "$host112VmxReady" in deploy def test_agent99_deployer_verifies_promoted_runtime_and_writes_revision() -> None: diff --git a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py index 1c409f433..5232c5668 100644 --- a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py @@ -18,6 +18,23 @@ def test_recover_requires_full_sop_coordinator_before_verified_resolution() -> N assert 'schemaVersion = "agent99_recovery_slo_v2"' in control assert 'scope = $recoveryScope' in control assert 'rebootSloClaimed = $rebootSloClaimed' in control + assert 'New-AgentRecoveryPhase "external_host_recovery"' in control + assert 'New-AgentOutcomeCheck "external_hosts_recovered"' in control + + +def test_host111_is_in_recovery_executor_and_all_host_readiness_scope() -> None: + control = read("agent99-control-plane.ps1") + config = read("agent99.config.99.example.json") + deploy = read("agent99-deploy.ps1") + tasks = read("agent99-register-tasks.ps1") + + assert '"host": "192.168.0.111"' in config + assert '"recoveryAction": "wake_on_lan"' in config + assert "Invoke-AgentExternalHostRecovery" in control + assert 'verifier = "icmp_and_required_tcp_ports"' in control + assert "host111_wol_executor" in deploy + assert '-Mode Recover -ControlledApply -RunNow' in tasks + assert '-Source agent99-startup-recovery' in tasks def test_coordinator_covers_all_hosts_and_full_recovery_evidence() -> None: