param( [string]$SourceRoot = (Split-Path -Parent $MyInvocation.MyCommand.Path), [string]$WorkRoot = (Join-Path $env:TEMP ("agent99-synthetic-" + (Get-Date -Format "yyyyMMdd-HHmmss"))) ) $ErrorActionPreference = "Stop" $checks = @() function Add-SyntheticCheck { param([string]$Name, [bool]$Ok, [string]$Detail) $script:checks += [pscustomobject]@{ name = $Name; ok = $Ok; detail = $Detail } } Remove-Item -Recurse -Force $WorkRoot -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path $WorkRoot | Out-Null $inbox = Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1" $routingOutput = @(& $inbox -AgentRoot $WorkRoot -SelfTest 2>&1) -join "`n" $routingExitCode = $LASTEXITCODE try { $routing = $routingOutput | ConvertFrom-Json Add-SyntheticCheck "routing:self_test" ($routingExitCode -eq 0 -and $routing.ok -eq $true) "passed=$($routing.passedCount)/$($routing.caseCount) exit=$routingExitCode" Add-SyntheticCheck "routing:provider_freshness" (@($routing.results | Where-Object { $_.alertId -eq "selftest-provider-freshness" -and $_.mode -eq "ProviderFreshness" }).Count -eq 1) "provider freshness structured route" Add-SyntheticCheck "routing:reboot_slo" (@($routing.results | Where-Object { $_.alertId -eq "selftest-reboot-slo" -and $_.mode -eq "Recover" -and $_.routeSource -eq "suggestedMode" }).Count -eq 1) "reboot SLO has priority over contaminated text" } catch { Add-SyntheticCheck "routing:self_test" $false "invalid self-test JSON: $($_.Exception.Message)" } $tokens = $null $errors = $null $controlPath = Join-Path $SourceRoot "agent99-control-plane.ps1" $controlSource = [IO.File]::ReadAllText($controlPath, [Text.Encoding]::UTF8) $ast = [System.Management.Automation.Language.Parser]::ParseInput($controlSource, [ref]$tokens, [ref]$errors) $formatFunctions = @{} foreach ($functionName in @("Get-AgentObjectValue", "Limit-AgentTextLine", "Format-AgentMetricForCard", "Format-AgentTelegramText", "Convert-AgentRecoveryReadback")) { $definition = $ast.Find({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $functionName }, $true) if ($definition) { Invoke-Expression $definition.Extent.Text $formatFunctions[$functionName] = $true } } if ($formatFunctions.ContainsKey("Format-AgentTelegramText")) { $perfCard = Format-AgentTelegramText "warning" "performance_warning" "test" ([pscustomobject]@{ host = "192.168.0.120" reasons = @("perf_readback_failed") loadPerCore = $null memAvailablePercent = $null diskUsedPercent = $null }) $providerCard = Format-AgentTelegramText "warning" "provider_freshness_triage" "test" ([pscustomobject]@{ candidateId = "pf-test" target = "provider_freshness_signal" hitCount = 2 missingFields = @("lastSeen") status = "handled_blocked_by_trust_gate" candidatePath = "C:\Wooo\Agent99\state\pf-test.json" }) $cards = "$perfCard`n$providerCard" Add-SyntheticCheck "telegram:localized" ([regex]::IsMatch($cards, "[^\x00-\x7F]")) "localized card is present" Add-SyntheticCheck "telegram:no_blank_metrics" (-not $perfCard.Contains("load/core=, ")) "failed readback renders unavailable values" Add-SyntheticCheck "telegram:provider_handled" ($providerCard.Contains("handled_blocked_by_trust_gate") -and $providerCard.Contains("Agent99")) "handled state and agent are visible" Add-SyntheticCheck "telegram:no_raw_python" (-not $cards.Contains("urllib.request")) "relay source is absent from user-facing card" } else { Add-SyntheticCheck "telegram:formatter" $false "Format-AgentTelegramText not found" } if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) { $fixtureEpoch = [DateTimeOffset]::Now.ToUnixTimeSeconds() $fixtureConfig = [pscustomobject]@{ maxArtifactAgeSeconds = 600 artifactClockSkewSeconds = 90 requiredHostAliases = @("99", "110", "111", "112", "120", "121", "188") } $fixtureLines = @( "RECOVERY_READBACK=1", "RECOVERY_ARTIFACT_DIR=/home/wooo/reboot-recovery/reboot-auto-recovery-slo-selftest", "RECOVERY_ARTIFACT_EPOCH=$fixtureEpoch", "POST_START_RESULT=GREEN_WITH_EVIDENCE_WARNINGS", "POST_START_BLOCKED=0", "SERVICE_GREEN=1", "PRODUCT_DATA_GREEN=1", "STOCK_FRESHNESS_STATUS=ok", "BACKUP_CORE_GREEN=1", "HOST_188_SERVICE_GREEN=1", "HOST_188_HYGIENE_BLOCKED=0", "EDGE_FALLBACK_READY=1", "VMWARE_AUTOSTART_VERIFY_READY=1", "WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1", "SCORECARD_STATUS=ready_reboot_auto_recovery_slo", "SCORECARD_BLOCKER_COUNT=0", "SCORECARD_READINESS_PERCENT=100", "SCORECARD_PRIMARY_BLOCKER=", "SCORECARD_CAN_CLAIM=1", "SCORECARD_FRESH_REBOOT_WINDOW=1", "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0", "SCORECARD_SOURCE_CONTROLS_READY=1", "SCORECARD_PUBLIC_FALLBACK_READY=1", "HOST_BOOT alias=99 target=192.168.0.99 startup_unit=vmware-host-autostart reachable=1 boot_id=windows uptime_seconds=30 systemd_state=windows_ssh startup_enabled=unknown startup_active=unknown", "HOST_BOOT alias=110 target=wooo@192.168.0.110 startup_unit=awoooi-startup-110.service reachable=1 boot_id=linux110 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active", "HOST_BOOT alias=111 target=ooo@192.168.0.111 startup_unit=com.momo.ollama111-allow-proxy reachable=1 boot_id=darwin uptime_seconds=40 systemd_state=darwin_ssh startup_enabled=enabled startup_active=active", "HOST_BOOT alias=112 target=192.168.0.112 startup_unit=vm-host-boot reachable=1 boot_id=linux112 uptime_seconds=35 systemd_state=node_exporter startup_enabled=unknown startup_active=unknown", "HOST_BOOT alias=120 target=wooo@192.168.0.120 startup_unit=k3s.service reachable=1 boot_id=linux120 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active", "HOST_BOOT alias=121 target=wooo@192.168.0.121 startup_unit=k3s.service reachable=1 boot_id=linux121 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active", "HOST_BOOT alias=188 target=ollama@192.168.0.188 startup_unit=awoooi-startup.service reachable=1 boot_id=linux188 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=inactive" ) -join "`n" $freshReadback = Convert-AgentRecoveryReadback $fixtureLines (Get-Date).AddSeconds(-30) $true $fixtureConfig Add-SyntheticCheck "recovery:coordinator_fresh_fixture" ($freshReadback.verified -and $freshReadback.rebootSloClaimed -and @($freshReadback.hosts).Count -eq 7) "fresh seven-host scorecard is verified" $blockedFixture = $fixtureLines.Replace("SCORECARD_CAN_CLAIM=1", "SCORECARD_CAN_CLAIM=0").Replace("SCORECARD_BLOCKER_COUNT=0", "SCORECARD_BLOCKER_COUNT=3") $blockedReadback = Convert-AgentRecoveryReadback $blockedFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig Add-SyntheticCheck "recovery:coordinator_false_green_guard" (-not $blockedReadback.verified -and -not $blockedReadback.rebootSloClaimed -and @($blockedReadback.failedChecks).Count -ge 1) "non-SLO blockers fail closed even when service evidence is otherwise green" $lateFixture = $blockedFixture.Replace("SCORECARD_FRESH_REBOOT_WINDOW=1", "SCORECARD_FRESH_REBOOT_WINDOW=0").Replace("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0", "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=1") $lateReadback = Convert-AgentRecoveryReadback $lateFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig Add-SyntheticCheck "recovery:coordinator_late_recovery_terminal" ($lateReadback.verified -and -not $lateReadback.rebootSloClaimed -and @($lateReadback.failedChecks).Count -eq 0 -and @($lateReadback.sloFailedChecks).Count -ge 2) "service recovery can close late without rewriting the reboot SLO breach" } else { Add-SyntheticCheck "recovery:coordinator_parser" $false "Convert-AgentRecoveryReadback not found" } $control = $controlSource Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_already_queued") -and $control.Contains("recovery_trigger_cooldown") -and $control.Contains('mode = "Recover"')) "automatic recovery is queued and deduplicated" Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence" Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("recovered_late")) "boot recovery remains durable through first-run termination and late closure" Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present" $failures = @($checks | Where-Object { -not $_.ok }) $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) ok = [bool]($failures.Count -eq 0) workRoot = $WorkRoot checkCount = $checks.Count failureCount = $failures.Count checks = $checks failures = $failures } $result | ConvertTo-Json -Depth 8 if (-not $result.ok) { exit 1 } exit 0