142 lines
9.1 KiB
PowerShell
142 lines
9.1 KiB
PowerShell
param(
|
|
[string]$SourceRoot = (Split-Path -Parent $MyInvocation.MyCommand.Path),
|
|
[string]$WorkRoot = (Join-Path $env:TEMP ("agent99-synthetic-" + (Get-Date -Format "yyyyMMdd-HHmmss")))
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$checks = @()
|
|
|
|
function Add-SyntheticCheck {
|
|
param([string]$Name, [bool]$Ok, [string]$Detail)
|
|
$script:checks += [pscustomobject]@{ name = $Name; ok = $Ok; detail = $Detail }
|
|
}
|
|
|
|
Remove-Item -Recurse -Force $WorkRoot -ErrorAction SilentlyContinue
|
|
New-Item -ItemType Directory -Force -Path $WorkRoot | Out-Null
|
|
|
|
$inbox = Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"
|
|
$routingOutput = @(& $inbox -AgentRoot $WorkRoot -SelfTest 2>&1) -join "`n"
|
|
$routingExitCode = $LASTEXITCODE
|
|
try {
|
|
$routing = $routingOutput | ConvertFrom-Json
|
|
Add-SyntheticCheck "routing:self_test" ($routingExitCode -eq 0 -and $routing.ok -eq $true) "passed=$($routing.passedCount)/$($routing.caseCount) exit=$routingExitCode"
|
|
Add-SyntheticCheck "routing:provider_freshness" (@($routing.results | Where-Object { $_.alertId -eq "selftest-provider-freshness" -and $_.mode -eq "ProviderFreshness" }).Count -eq 1) "provider freshness structured route"
|
|
Add-SyntheticCheck "routing:reboot_slo" (@($routing.results | Where-Object { $_.alertId -eq "selftest-reboot-slo" -and $_.mode -eq "Recover" -and $_.routeSource -eq "suggestedMode" }).Count -eq 1) "reboot SLO has priority over contaminated text"
|
|
} catch {
|
|
Add-SyntheticCheck "routing:self_test" $false "invalid self-test JSON: $($_.Exception.Message)"
|
|
}
|
|
|
|
$tokens = $null
|
|
$errors = $null
|
|
$controlPath = Join-Path $SourceRoot "agent99-control-plane.ps1"
|
|
$controlSource = [IO.File]::ReadAllText($controlPath, [Text.Encoding]::UTF8)
|
|
$ast = [System.Management.Automation.Language.Parser]::ParseInput($controlSource, [ref]$tokens, [ref]$errors)
|
|
$formatFunctions = @{}
|
|
foreach ($functionName in @("Get-AgentObjectValue", "Limit-AgentTextLine", "Format-AgentMetricForCard", "Format-AgentTelegramText", "Convert-AgentRecoveryReadback")) {
|
|
$definition = $ast.Find({
|
|
param($node)
|
|
$node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $functionName
|
|
}, $true)
|
|
if ($definition) {
|
|
Invoke-Expression $definition.Extent.Text
|
|
$formatFunctions[$functionName] = $true
|
|
}
|
|
}
|
|
if ($formatFunctions.ContainsKey("Format-AgentTelegramText")) {
|
|
$perfCard = Format-AgentTelegramText "warning" "performance_warning" "test" ([pscustomobject]@{
|
|
host = "192.168.0.120"
|
|
reasons = @("perf_readback_failed")
|
|
loadPerCore = $null
|
|
memAvailablePercent = $null
|
|
diskUsedPercent = $null
|
|
})
|
|
$providerCard = Format-AgentTelegramText "warning" "provider_freshness_triage" "test" ([pscustomobject]@{
|
|
candidateId = "pf-test"
|
|
target = "provider_freshness_signal"
|
|
hitCount = 2
|
|
missingFields = @("lastSeen")
|
|
status = "handled_blocked_by_trust_gate"
|
|
candidatePath = "C:\Wooo\Agent99\state\pf-test.json"
|
|
})
|
|
$cards = "$perfCard`n$providerCard"
|
|
Add-SyntheticCheck "telegram:localized" ([regex]::IsMatch($cards, "[^\x00-\x7F]")) "localized card is present"
|
|
Add-SyntheticCheck "telegram:no_blank_metrics" (-not $perfCard.Contains("load/core=, ")) "failed readback renders unavailable values"
|
|
Add-SyntheticCheck "telegram:provider_handled" ($providerCard.Contains("handled_blocked_by_trust_gate") -and $providerCard.Contains("Agent99")) "handled state and agent are visible"
|
|
Add-SyntheticCheck "telegram:no_raw_python" (-not $cards.Contains("urllib.request")) "relay source is absent from user-facing card"
|
|
} else {
|
|
Add-SyntheticCheck "telegram:formatter" $false "Format-AgentTelegramText not found"
|
|
}
|
|
|
|
if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) {
|
|
$fixtureEpoch = [DateTimeOffset]::Now.ToUnixTimeSeconds()
|
|
$fixtureConfig = [pscustomobject]@{
|
|
maxArtifactAgeSeconds = 600
|
|
artifactClockSkewSeconds = 90
|
|
requiredHostAliases = @("99", "110", "111", "112", "120", "121", "188")
|
|
}
|
|
$fixtureLines = @(
|
|
"RECOVERY_READBACK=1",
|
|
"RECOVERY_ARTIFACT_DIR=/home/wooo/reboot-recovery/reboot-auto-recovery-slo-selftest",
|
|
"RECOVERY_ARTIFACT_EPOCH=$fixtureEpoch",
|
|
"POST_START_RESULT=GREEN_WITH_EVIDENCE_WARNINGS",
|
|
"POST_START_BLOCKED=0",
|
|
"SERVICE_GREEN=1",
|
|
"PRODUCT_DATA_GREEN=1",
|
|
"STOCK_FRESHNESS_STATUS=ok",
|
|
"BACKUP_CORE_GREEN=1",
|
|
"HOST_188_SERVICE_GREEN=1",
|
|
"HOST_188_HYGIENE_BLOCKED=0",
|
|
"EDGE_FALLBACK_READY=1",
|
|
"VMWARE_AUTOSTART_VERIFY_READY=1",
|
|
"WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1",
|
|
"SCORECARD_STATUS=ready_reboot_auto_recovery_slo",
|
|
"SCORECARD_BLOCKER_COUNT=0",
|
|
"SCORECARD_READINESS_PERCENT=100",
|
|
"SCORECARD_PRIMARY_BLOCKER=",
|
|
"SCORECARD_CAN_CLAIM=1",
|
|
"SCORECARD_FRESH_REBOOT_WINDOW=1",
|
|
"SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0",
|
|
"SCORECARD_SOURCE_CONTROLS_READY=1",
|
|
"SCORECARD_PUBLIC_FALLBACK_READY=1",
|
|
"HOST_BOOT alias=99 target=192.168.0.99 startup_unit=vmware-host-autostart reachable=1 boot_id=windows uptime_seconds=30 systemd_state=windows_ssh startup_enabled=unknown startup_active=unknown",
|
|
"HOST_BOOT alias=110 target=wooo@192.168.0.110 startup_unit=awoooi-startup-110.service reachable=1 boot_id=linux110 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active",
|
|
"HOST_BOOT alias=111 target=ooo@192.168.0.111 startup_unit=com.momo.ollama111-allow-proxy reachable=1 boot_id=darwin uptime_seconds=40 systemd_state=darwin_ssh startup_enabled=enabled startup_active=active",
|
|
"HOST_BOOT alias=112 target=192.168.0.112 startup_unit=vm-host-boot reachable=1 boot_id=linux112 uptime_seconds=35 systemd_state=node_exporter startup_enabled=unknown startup_active=unknown",
|
|
"HOST_BOOT alias=120 target=wooo@192.168.0.120 startup_unit=k3s.service reachable=1 boot_id=linux120 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active",
|
|
"HOST_BOOT alias=121 target=wooo@192.168.0.121 startup_unit=k3s.service reachable=1 boot_id=linux121 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active",
|
|
"HOST_BOOT alias=188 target=ollama@192.168.0.188 startup_unit=awoooi-startup.service reachable=1 boot_id=linux188 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=inactive"
|
|
) -join "`n"
|
|
$freshReadback = Convert-AgentRecoveryReadback $fixtureLines (Get-Date).AddSeconds(-30) $true $fixtureConfig
|
|
Add-SyntheticCheck "recovery:coordinator_fresh_fixture" ($freshReadback.verified -and $freshReadback.rebootSloClaimed -and @($freshReadback.hosts).Count -eq 7) "fresh seven-host scorecard is verified"
|
|
|
|
$blockedFixture = $fixtureLines.Replace("SCORECARD_CAN_CLAIM=1", "SCORECARD_CAN_CLAIM=0").Replace("SCORECARD_BLOCKER_COUNT=0", "SCORECARD_BLOCKER_COUNT=3")
|
|
$blockedReadback = Convert-AgentRecoveryReadback $blockedFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig
|
|
Add-SyntheticCheck "recovery:coordinator_false_green_guard" (-not $blockedReadback.verified -and -not $blockedReadback.rebootSloClaimed -and @($blockedReadback.failedChecks).Count -ge 1) "non-SLO blockers fail closed even when service evidence is otherwise green"
|
|
|
|
$lateFixture = $blockedFixture.Replace("SCORECARD_FRESH_REBOOT_WINDOW=1", "SCORECARD_FRESH_REBOOT_WINDOW=0").Replace("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0", "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=1")
|
|
$lateReadback = Convert-AgentRecoveryReadback $lateFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig
|
|
Add-SyntheticCheck "recovery:coordinator_late_recovery_terminal" ($lateReadback.verified -and -not $lateReadback.rebootSloClaimed -and @($lateReadback.failedChecks).Count -eq 0 -and @($lateReadback.sloFailedChecks).Count -ge 2) "service recovery can close late without rewriting the reboot SLO breach"
|
|
} else {
|
|
Add-SyntheticCheck "recovery:coordinator_parser" $false "Convert-AgentRecoveryReadback not found"
|
|
}
|
|
|
|
$control = $controlSource
|
|
Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_already_queued") -and $control.Contains("recovery_trigger_cooldown") -and $control.Contains('mode = "Recover"')) "automatic recovery is queued and deduplicated"
|
|
Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence"
|
|
Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("recovered_late")) "boot recovery remains durable through first-run termination and late closure"
|
|
Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present"
|
|
|
|
$failures = @($checks | Where-Object { -not $_.ok })
|
|
$result = [pscustomobject]@{
|
|
timestamp = (Get-Date -Format o)
|
|
ok = [bool]($failures.Count -eq 0)
|
|
workRoot = $WorkRoot
|
|
checkCount = $checks.Count
|
|
failureCount = $failures.Count
|
|
checks = $checks
|
|
failures = $failures
|
|
}
|
|
$result | ConvertTo-Json -Depth 8
|
|
if (-not $result.ok) { exit 1 }
|
|
exit 0
|