All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m41s
CD Pipeline / build-and-deploy (push) Successful in 9m24s
CD Pipeline / post-deploy-checks (push) Successful in 1m56s
145 lines
12 KiB
PowerShell
145 lines
12 KiB
PowerShell
param(
|
|
[string]$SourceRoot = (Split-Path -Parent $MyInvocation.MyCommand.Path),
|
|
[string]$OutputPath = ""
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$checks = @()
|
|
$failures = @()
|
|
|
|
function Add-Check {
|
|
param([string]$Name, [bool]$Ok, [string]$Detail)
|
|
$item = [pscustomobject]@{ name = $Name; ok = $Ok; detail = $Detail }
|
|
$script:checks += $item
|
|
if (-not $Ok) { $script:failures += $item }
|
|
}
|
|
|
|
$requiredFiles = @(
|
|
"agent99-bootstrap.ps1",
|
|
"agent99-contract-check.ps1",
|
|
"agent99-control-plane.ps1",
|
|
"agent99-deploy.ps1",
|
|
"agent99-register-tasks.ps1",
|
|
"agent99-sre-alert-inbox.ps1",
|
|
"agent99-sre-alert-relay.ps1",
|
|
"agent99-submit-request.ps1",
|
|
"agent99-synthetic-tests.ps1",
|
|
"agent99-telegram-inbox.ps1",
|
|
"agent99-windows-update-policy.ps1",
|
|
"host-reboot-scorecard.ps1"
|
|
)
|
|
|
|
foreach ($name in $requiredFiles) {
|
|
$path = Join-Path $SourceRoot $name
|
|
Add-Check "file:$name" (Test-Path $path) $path
|
|
}
|
|
|
|
foreach ($file in Get-ChildItem -Path $SourceRoot -Filter "*.ps1" -File) {
|
|
$tokens = $null
|
|
$errors = $null
|
|
$sourceText = [IO.File]::ReadAllText($file.FullName, [Text.Encoding]::UTF8)
|
|
[System.Management.Automation.Language.Parser]::ParseInput($sourceText, [ref]$tokens, [ref]$errors) | Out-Null
|
|
Add-Check "powershell_parse:$($file.Name)" ($errors.Count -eq 0) $(if ($errors.Count -eq 0) { "parse_ok" } else { "parse_errors=$($errors.Count)" })
|
|
}
|
|
|
|
foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.json")) {
|
|
$path = Join-Path $SourceRoot $name
|
|
try {
|
|
$config = Get-Content $path -Raw | ConvertFrom-Json
|
|
$hasHosts = @($config.hosts).Count -eq 6 -and "192.168.0.111" -in @($config.hosts)
|
|
$host111External = @($config.externalHosts | Where-Object { $_.host -eq "192.168.0.111" } | Select-Object -First 1)
|
|
$hasExternalHostRecovery = [bool](
|
|
$host111External.Count -eq 1 -and
|
|
$host111External[0].recoveryAction -eq "wake_on_lan" -and
|
|
$host111External[0].enabled -eq $true -and
|
|
$host111External[0].required -eq $true -and
|
|
@($host111External[0].macAddresses).Count -ge 1 -and
|
|
22 -in @($host111External[0].verifyTcpPorts)
|
|
)
|
|
$hasRoutes = @($config.publicUrls).Count -ge 5
|
|
$hasSlo = [bool]($config.recoverySlo -and [int]$config.recoverySlo.targetMinutes -eq 10)
|
|
$hasIdentity = [bool]($config.sshIdentityFile -and $config.sshIdentityFile -match "agent99_ed25519$")
|
|
$hasCanonicalHost111User = [bool](
|
|
$config.sshUsers -and
|
|
$config.sshUsers.PSObject.Properties["192.168.0.111"] -and
|
|
[string]$config.sshUsers.PSObject.Properties["192.168.0.111"].Value -eq "ooo"
|
|
)
|
|
$hasJump = [bool]($config.k3s -and $config.k3s.jumpHost -and $config.k3s.preferJumpHost -eq $true)
|
|
$hasAutoRecovery = [bool]($config.autoRecovery -and $config.autoRecovery.enabled -eq $true)
|
|
$hasRecoveryCoordinator = [bool](
|
|
$config.recoveryCoordinator -and
|
|
$config.recoveryCoordinator.enabled -eq $true -and
|
|
$config.recoveryCoordinator.controllerHost -eq "192.168.0.110" -and
|
|
@($config.recoveryCoordinator.requiredHostAliases).Count -eq 7
|
|
)
|
|
$hasCompletionCallback = [bool](
|
|
$config.completionCallback -and
|
|
$config.completionCallback.enabled -eq $true -and
|
|
$config.completionCallback.url -match "^https://" -and
|
|
$config.completionCallback.tokenEnv -eq "AGENT99_SRE_RELAY_TOKEN"
|
|
)
|
|
$hasCanonicalHost112Vmx = if ($name -eq "agent99.config.99.example.json") {
|
|
[bool](@($config.vms | Where-Object {
|
|
[string]$_.name -eq "host112-kali" -and
|
|
[string]$_.host -eq "192.168.0.112" -and
|
|
[string]$_.vmx -eq "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx"
|
|
}).Count -eq 1)
|
|
} else { $true }
|
|
$ok = $hasHosts -and $hasExternalHostRecovery -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasCanonicalHost111User -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasCompletionCallback -and $hasCanonicalHost112Vmx
|
|
Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) externalHostRecovery=$hasExternalHostRecovery routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity canonicalHost111User=$hasCanonicalHost111User jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator completionCallback=$hasCompletionCallback canonicalHost112Vmx=$hasCanonicalHost112Vmx"
|
|
} catch {
|
|
Add-Check "config:$name" $false "json_parse_failed: $($_.Exception.Message)"
|
|
}
|
|
}
|
|
|
|
$control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane.ps1"), [Text.Encoding]::UTF8)
|
|
$inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8)
|
|
$telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8)
|
|
$bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8)
|
|
$deployer = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-deploy.ps1"), [Text.Encoding]::UTF8)
|
|
$taskRegistration = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-register-tasks.ps1"), [Text.Encoding]::UTF8)
|
|
$submitRequest = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-submit-request.ps1"), [Text.Encoding]::UTF8)
|
|
|
|
Add-Check "transport:dedicated_identity" ($control.Contains("sshIdentityFile") -and $control.Contains("IdentitiesOnly=yes")) "dedicated SSH identity is enforced"
|
|
Add-Check "transport:jump_first" ($control.Contains("preferJumpHost") -and $control.Contains("ssh_jump_fallback")) "preferred jump with bounded fallback is present"
|
|
Add-Check "transport:stale_process_guard" ($control.Contains("Invoke-AgentStaleSshProcessGuard") -and $control.Contains("taskkill.exe /PID") -and $control.Contains("ssh_stale_process_guard")) "timed-out and stale SSH clients are bounded"
|
|
Add-Check "transport:cross_process_serialization" ($control.Contains("Enter-AgentSshTransportLock") -and $control.Contains("[IO.FileShare]::None") -and $control.Contains("ssh_transport_lock_failed")) "cross-process SSH sessions are serialized"
|
|
Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("recovery_already_queued")) "boot and host-down recovery use a single-flight queue"
|
|
Add-Check "recovery:durable_boot_event" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("Update-AgentBootRecoveryState") -and $control.Contains("recovered_late")) "boot event remains pending until verified recovery terminal"
|
|
Add-Check "recovery:terminal_evidence_budget" ($taskRegistration.Contains('$recoverySettings') -and $taskRegistration.Contains('New-TimeSpan -Minutes 20')) "startup recovery outlives the ten-minute SLO long enough to write terminal evidence"
|
|
Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present"
|
|
Add-Check "recovery:full_sop_coordinator" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_scorecard") -and $control.Contains("SCORECARD_FRESH_REBOOT_WINDOW") -and $control.Contains('scope = $recoveryScope')) "Agent99 cannot close reboot recovery without the 110 full-SOP scorecard"
|
|
Add-Check "recovery:host111_wol_executor" ($control.Contains("Invoke-AgentExternalHostRecovery") -and $control.Contains("Send-AgentWakeOnLan") -and $control.Contains('Get-AgentObjectValue $externalHost "recoveryAction" "wake_on_lan"') -and $control.Contains('New-AgentRecoveryPhase "external_host_recovery"') -and $control.Contains('New-AgentOutcomeCheck "external_hosts_recovered"')) "Host111 has an allowlisted WOL executor with independent ping/TCP verification"
|
|
Add-Check "recovery:startup_queue_closure" ($taskRegistration.Contains('$submitRequest') -and $taskRegistration.Contains('-Mode Recover -ControlledApply -RunNow') -and $taskRegistration.Contains('-Source agent99-startup-recovery')) "startup recovery runs through queue, outcome, callback, Telegram lifecycle, and KM/RAG"
|
|
Add-Check "recovery:startup_controlled_identity" ($submitRequest.Contains("New-AgentControlledDispatchIdentity") -and $submitRequest.Contains("New-AgentUuidV5") -and $submitRequest.Contains("agent99_controlled_dispatch_identity_v1") -and $submitRequest.Contains("canonicalDigest") -and $submitRequest.Contains("singleFlightKey") -and $submitRequest.Contains("approvalId")) "local and startup controlled requests carry the canonical AWOOOI dispatch identity"
|
|
Add-Check "recovery:host112_immutable_apply_closure" ($control.Contains("managerApplyVerified") -and $control.Contains("durableReceiptVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('artifactTransactionStatus -eq "committed_verified_pair"') -and $control.Contains('artifactTransactionPriorPairVerified') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply closure binds transport, fixed WAL, attributed start, immutable receipt/readback and independent verifier"
|
|
Add-Check "recovery:host112_direct_transport" ($control.Contains("Invoke-AgentHost112SshText") -and $control.Contains('expectedUser = "kali"') -and $control.Contains('route = "direct_host112_kali"') -and $deployer.Contains('name = "host112_canonical_direct_route"') -and $bootstrap.Contains("Ensure-AgentHost112CanonicalSshConfig")) "Host112 recovery uses the canonical 99 direct key route"
|
|
Add-Check "recovery:host112_canonical_vmx" ($deployer.Contains('name = "host112_canonical_vmx_path"') -and $deployer.Contains('Get-AgentHost112CanonicalVm $policyConfigPath') -and $deployer.Contains('Test-Path -LiteralPath $canonicalHost112Vmx -PathType Leaf') -and $bootstrap.Contains('Get-AgentHost112CanonicalVm $PolicyPath') -and $bootstrap.Contains('Set-AgentBootstrapProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))')) "Host112 VM identity is normalized from one staged policy row and fails closed when absent"
|
|
Add-Check "recovery:host112_timeout_reserve" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120")) "Host112 timeout chain has executor, queue and client reserves"
|
|
Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present"
|
|
Add-Check "problem_counts:v2" ($control.Contains('schemaVersion = "agent99_problem_counts_v2"') -and $control.Contains("totalVerifiedResolved")) "verified problem counts are present"
|
|
Add-Check "callback:durable_completion" ($control.Contains('schema_version = "agent99_completion_callback_v1"') -and $control.Contains("durable_readback") -and $control.Contains("completion-callbacks\pending")) "Agent99 completion callbacks are queued until durable AWOOOI/AwoooP readback"
|
|
Add-Check "runtime:manifest" ($control.Contains("Test-AgentRuntimeManifest") -and $bootstrap.Contains('schemaVersion = "agent99_runtime_manifest_v1"')) "runtime source drift is self-checked"
|
|
Add-Check "routing:structured" ($inbox.Contains("Resolve-AgentAlertRoute") -and $inbox.Contains('schemaVersion = "agent99_sre_alert_single_flight_v1"')) "structured route and local single-flight are present"
|
|
Add-Check "routing:telegram" ($telegram.Contains('schemaVersion = "agent99_alert_route_v1"') -and $telegram.Contains("autoIngestMonitoringAlerts")) "Telegram structured ingress is present"
|
|
|
|
$result = [pscustomobject]@{
|
|
timestamp = (Get-Date -Format o)
|
|
sourceRoot = $SourceRoot
|
|
ok = [bool](@($failures).Count -eq 0)
|
|
checkCount = @($checks).Count
|
|
failureCount = @($failures).Count
|
|
checks = $checks
|
|
failures = $failures
|
|
}
|
|
|
|
if ($OutputPath) {
|
|
$parent = Split-Path -Parent $OutputPath
|
|
if ($parent) { New-Item -ItemType Directory -Force -Path $parent | Out-Null }
|
|
$result | ConvertTo-Json -Depth 8 | Set-Content -Path $OutputPath -Encoding UTF8
|
|
}
|
|
|
|
$result | ConvertTo-Json -Depth 8
|
|
if (-not $result.ok) { exit 1 }
|
|
exit 0
|