param( [string]$SourceRoot = (Split-Path -Parent $MyInvocation.MyCommand.Path), [string]$OutputPath = "" ) $ErrorActionPreference = "Stop" $checks = @() $failures = @() function Add-Check { param([string]$Name, [bool]$Ok, [string]$Detail) $item = [pscustomobject]@{ name = $Name; ok = $Ok; detail = $Detail } $script:checks += $item if (-not $Ok) { $script:failures += $item } } $requiredFiles = @( "agent99-bootstrap.ps1", "agent99-contract-check.ps1", "agent99-control-plane.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-sre-alert-inbox.ps1", "agent99-sre-alert-relay.ps1", "agent99-submit-request.ps1", "agent99-synthetic-tests.ps1", "agent99-telegram-inbox.ps1", "agent99-windows-update-policy.ps1", "host-reboot-scorecard.ps1" ) foreach ($name in $requiredFiles) { $path = Join-Path $SourceRoot $name Add-Check "file:$name" (Test-Path $path) $path } foreach ($file in Get-ChildItem -Path $SourceRoot -Filter "*.ps1" -File) { $tokens = $null $errors = $null $sourceText = [IO.File]::ReadAllText($file.FullName, [Text.Encoding]::UTF8) [System.Management.Automation.Language.Parser]::ParseInput($sourceText, [ref]$tokens, [ref]$errors) | Out-Null Add-Check "powershell_parse:$($file.Name)" ($errors.Count -eq 0) $(if ($errors.Count -eq 0) { "parse_ok" } else { "parse_errors=$($errors.Count)" }) } foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.json")) { $path = Join-Path $SourceRoot $name try { $config = Get-Content $path -Raw | ConvertFrom-Json $hasHosts = @($config.hosts).Count -eq 6 -and "192.168.0.111" -in @($config.hosts) $host111External = @($config.externalHosts | Where-Object { $_.host -eq "192.168.0.111" } | Select-Object -First 1) $hasExternalHostRecovery = [bool]( $host111External.Count -eq 1 -and $host111External[0].recoveryAction -eq "wake_on_lan" -and $host111External[0].enabled -eq $true -and $host111External[0].required -eq $true -and @($host111External[0].macAddresses).Count -ge 1 -and 22 -in @($host111External[0].verifyTcpPorts) ) $hasRoutes = [bool]( @($config.publicUrls).Count -ge 7 -and "https://n8n.wooo.work" -in @($config.publicUrls) -and "https://ollama.wooo.work" -in @($config.publicUrls) ) $hasSlo = [bool]($config.recoverySlo -and [int]$config.recoverySlo.targetMinutes -eq 10) $hasIdentity = [bool]($config.sshIdentityFile -and $config.sshIdentityFile -match "agent99_ed25519$") $hasCanonicalHost111User = [bool]( $config.sshUsers -and $config.sshUsers.PSObject.Properties["192.168.0.111"] -and [string]$config.sshUsers.PSObject.Properties["192.168.0.111"].Value -eq "ooo" ) $hasJump = [bool]($config.k3s -and $config.k3s.jumpHost -and $config.k3s.preferJumpHost -eq $true) $hasAutoRecovery = [bool]($config.autoRecovery -and $config.autoRecovery.enabled -eq $true) $hasRecoveryCoordinator = [bool]( $config.recoveryCoordinator -and $config.recoveryCoordinator.enabled -eq $true -and $config.recoveryCoordinator.controllerHost -eq "192.168.0.110" -and @($config.recoveryCoordinator.requiredHostAliases).Count -eq 7 ) $hasHost188EdgeRecovery = [bool]( $config.edgeServiceRecovery -and $config.edgeServiceRecovery.enabled -eq $true -and $config.edgeServiceRecovery.host -eq "192.168.0.188" -and $config.edgeServiceRecovery.scriptPath -eq "/home/ollama/bin/agent99-host188-edge-services-recover.sh" -and "n8n" -in @($config.edgeServiceRecovery.requiredServices) -and "open-webui" -in @($config.edgeServiceRecovery.requiredServices) ) $hasCompletionCallback = [bool]( $config.completionCallback -and $config.completionCallback.enabled -eq $true -and $config.completionCallback.url -match "^https://" -and $config.completionCallback.tokenEnv -eq "AGENT99_SRE_RELAY_TOKEN" ) $hasCanonicalHost112Vmx = if ($name -eq "agent99.config.99.example.json") { [bool](@($config.vms | Where-Object { [string]$_.name -eq "host112-kali" -and [string]$_.host -eq "192.168.0.112" -and [string]$_.vmx -eq "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" }).Count -eq 1) } else { $true } $ok = $hasHosts -and $hasExternalHostRecovery -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasCanonicalHost111User -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasHost188EdgeRecovery -and $hasCompletionCallback -and $hasCanonicalHost112Vmx Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) externalHostRecovery=$hasExternalHostRecovery routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity canonicalHost111User=$hasCanonicalHost111User jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator host188EdgeRecovery=$hasHost188EdgeRecovery completionCallback=$hasCompletionCallback canonicalHost112Vmx=$hasCanonicalHost112Vmx" } catch { Add-Check "config:$name" $false "json_parse_failed: $($_.Exception.Message)" } } $control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane.ps1"), [Text.Encoding]::UTF8) $inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8) $telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8) $bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8) $deployer = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-deploy.ps1"), [Text.Encoding]::UTF8) $taskRegistration = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-register-tasks.ps1"), [Text.Encoding]::UTF8) $submitRequest = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-submit-request.ps1"), [Text.Encoding]::UTF8) Add-Check "transport:dedicated_identity" ($control.Contains("sshIdentityFile") -and $control.Contains("IdentitiesOnly=yes")) "dedicated SSH identity is enforced" Add-Check "transport:jump_first" ($control.Contains("preferJumpHost") -and $control.Contains("ssh_jump_fallback")) "preferred jump with bounded fallback is present" Add-Check "transport:stale_process_guard" ($control.Contains("Invoke-AgentStaleSshProcessGuard") -and $control.Contains("taskkill.exe /PID") -and $control.Contains("ssh_stale_process_guard")) "timed-out and stale SSH clients are bounded" Add-Check "transport:cross_process_serialization" ($control.Contains("Enter-AgentSshTransportLock") -and $control.Contains("[IO.FileShare]::None") -and $control.Contains("ssh_transport_lock_failed")) "cross-process SSH sessions are serialized" Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("recovery_already_queued")) "boot and host-down recovery use a single-flight queue" Add-Check "recovery:durable_boot_event" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("Update-AgentBootRecoveryState") -and $control.Contains("recovered_late")) "boot event remains pending until verified recovery terminal" Add-Check "recovery:terminal_evidence_budget" ($taskRegistration.Contains('$recoverySettings') -and $taskRegistration.Contains('New-TimeSpan -Minutes 20')) "startup recovery outlives the ten-minute SLO long enough to write terminal evidence" Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present" Add-Check "recovery:full_sop_coordinator" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_scorecard") -and $control.Contains("SCORECARD_FRESH_REBOOT_WINDOW") -and $control.Contains('scope = $recoveryScope')) "Agent99 cannot close reboot recovery without the 110 full-SOP scorecard" Add-Check "recovery:host111_wol_executor" ($control.Contains("Invoke-AgentExternalHostRecovery") -and $control.Contains("Send-AgentWakeOnLan") -and $control.Contains('Get-AgentObjectValue $externalHost "recoveryAction" "wake_on_lan"') -and $control.Contains('New-AgentRecoveryPhase "external_host_recovery"') -and $control.Contains('New-AgentOutcomeCheck "external_hosts_recovered"')) "Host111 has an allowlisted WOL executor with independent ping/TCP verification" Add-Check "recovery:startup_queue_closure" ($taskRegistration.Contains('$submitRequest') -and $taskRegistration.Contains('-Mode Recover -ControlledApply -RunNow') -and $taskRegistration.Contains('-Source agent99-startup-recovery')) "startup recovery runs through queue, outcome, callback, Telegram lifecycle, and KM/RAG" Add-Check "recovery:startup_controlled_identity" ($submitRequest.Contains("New-AgentControlledDispatchIdentity") -and $submitRequest.Contains("New-AgentUuidV5") -and $submitRequest.Contains("agent99_controlled_dispatch_identity_v1") -and $submitRequest.Contains("canonicalDigest") -and $submitRequest.Contains("singleFlightKey") -and $submitRequest.Contains("approvalId")) "local and startup controlled requests carry the canonical AWOOOI dispatch identity" Add-Check "recovery:host112_immutable_apply_closure" ($control.Contains("managerApplyVerified") -and $control.Contains("durableReceiptVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('artifactTransactionStatus -eq "committed_verified_pair"') -and $control.Contains('artifactTransactionPriorPairVerified') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply closure binds transport, fixed WAL, attributed start, immutable receipt/readback and independent verifier" Add-Check "recovery:host112_direct_transport" ($control.Contains("Invoke-AgentHost112SshText") -and $control.Contains('expectedUser = "kali"') -and $control.Contains('route = "direct_host112_kali"') -and $deployer.Contains('name = "host112_canonical_direct_route"') -and $bootstrap.Contains("Ensure-AgentHost112CanonicalSshConfig")) "Host112 recovery uses the canonical 99 direct key route" Add-Check "recovery:host112_canonical_vmx" ($deployer.Contains('name = "host112_canonical_vmx_path"') -and $deployer.Contains('Get-AgentHost112CanonicalVm $policyConfigPath') -and $deployer.Contains('Test-Path -LiteralPath $canonicalHost112Vmx -PathType Leaf') -and $bootstrap.Contains('Get-AgentHost112CanonicalVm $PolicyPath') -and $bootstrap.Contains('Set-AgentBootstrapProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))')) "Host112 VM identity is normalized from one staged policy row and fails closed when absent" Add-Check "recovery:host112_timeout_reserve" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120")) "Host112 timeout chain has executor, queue and client reserves" Add-Check "recovery:host188_edge_services" ($control.Contains("Invoke-AgentHost188EdgeRecovery") -and $control.Contains('New-AgentRecoveryPhase "host188_edge_services"') -and $control.Contains('New-AgentOutcomeCheck "host188_edge_services_ready"') -and $control.Contains("container_local_upstream_and_public_https") -and $deployer.Contains('name = "canonical_public_route_coverage"')) "Host188 n8n and Open WebUI use a pinned controlled apply and independent public-route verifier" Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present" Add-Check "problem_counts:v2" ($control.Contains('schemaVersion = "agent99_problem_counts_v2"') -and $control.Contains("totalVerifiedResolved")) "verified problem counts are present" Add-Check "callback:durable_completion" ($control.Contains('schema_version = "agent99_completion_callback_v1"') -and $control.Contains("durable_readback") -and $control.Contains("completion-callbacks\pending")) "Agent99 completion callbacks are queued until durable AWOOOI/AwoooP readback" Add-Check "runtime:manifest" ($control.Contains("Test-AgentRuntimeManifest") -and $bootstrap.Contains('schemaVersion = "agent99_runtime_manifest_v1"')) "runtime source drift is self-checked" Add-Check "routing:structured" ($inbox.Contains("Resolve-AgentAlertRoute") -and $inbox.Contains('schemaVersion = "agent99_sre_alert_single_flight_v1"')) "structured route and local single-flight are present" Add-Check "routing:telegram" ($telegram.Contains('schemaVersion = "agent99_alert_route_v1"') -and $telegram.Contains("autoIngestMonitoringAlerts")) "Telegram structured ingress is present" $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) sourceRoot = $SourceRoot ok = [bool](@($failures).Count -eq 0) checkCount = @($checks).Count failureCount = @($failures).Count checks = $checks failures = $failures } if ($OutputPath) { $parent = Split-Path -Parent $OutputPath if ($parent) { New-Item -ItemType Directory -Force -Path $parent | Out-Null } $result | ConvertTo-Json -Depth 8 | Set-Content -Path $OutputPath -Encoding UTF8 } $result | ConvertTo-Json -Depth 8 if (-not $result.ok) { exit 1 } exit 0