161 lines
10 KiB
PowerShell
161 lines
10 KiB
PowerShell
param(
|
||
[string]$SourceRoot = (Split-Path -Parent $MyInvocation.MyCommand.Path),
|
||
[string]$WorkRoot = (Join-Path $env:TEMP ("agent99-synthetic-" + (Get-Date -Format "yyyyMMdd-HHmmss")))
|
||
)
|
||
|
||
$ErrorActionPreference = "Stop"
|
||
$checks = @()
|
||
|
||
function Add-SyntheticCheck {
|
||
param([string]$Name, [bool]$Ok, [string]$Detail)
|
||
$script:checks += [pscustomobject]@{ name = $Name; ok = $Ok; detail = $Detail }
|
||
}
|
||
|
||
Remove-Item -Recurse -Force $WorkRoot -ErrorAction SilentlyContinue
|
||
New-Item -ItemType Directory -Force -Path $WorkRoot | Out-Null
|
||
|
||
$inbox = Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"
|
||
$routingOutput = @(& $inbox -AgentRoot $WorkRoot -SelfTest 2>&1) -join "`n"
|
||
$routingExitCode = $LASTEXITCODE
|
||
try {
|
||
$routing = $routingOutput | ConvertFrom-Json
|
||
Add-SyntheticCheck "routing:self_test" ($routingExitCode -eq 0 -and $routing.ok -eq $true) "passed=$($routing.passedCount)/$($routing.caseCount) exit=$routingExitCode"
|
||
Add-SyntheticCheck "routing:provider_freshness" (@($routing.results | Where-Object { $_.alertId -eq "selftest-provider-freshness" -and $_.mode -eq "ProviderFreshness" }).Count -eq 1) "provider freshness structured route"
|
||
Add-SyntheticCheck "routing:reboot_slo" (@($routing.results | Where-Object { $_.alertId -eq "selftest-reboot-slo" -and $_.mode -eq "Recover" -and $_.routeSource -eq "suggestedMode" }).Count -eq 1) "reboot SLO has priority over contaminated text"
|
||
} catch {
|
||
Add-SyntheticCheck "routing:self_test" $false "invalid self-test JSON: $($_.Exception.Message)"
|
||
}
|
||
|
||
$tokens = $null
|
||
$errors = $null
|
||
$controlPath = Join-Path $SourceRoot "agent99-control-plane.ps1"
|
||
$controlSource = [IO.File]::ReadAllText($controlPath, [Text.Encoding]::UTF8)
|
||
$ast = [System.Management.Automation.Language.Parser]::ParseInput($controlSource, [ref]$tokens, [ref]$errors)
|
||
$formatFunctions = @{}
|
||
foreach ($functionName in @(
|
||
"Convert-AgentBool",
|
||
"Get-AgentObjectValue",
|
||
"Limit-AgentTextLine",
|
||
"Format-AgentMetricForCard",
|
||
"Format-AgentTelegramLegacyText",
|
||
"Get-AgentSha256Hex",
|
||
"Get-AgentIncidentCardModel",
|
||
"Format-AgentTelegramText",
|
||
"Format-AgentTelegramCaption",
|
||
"Convert-AgentRecoveryReadback"
|
||
)) {
|
||
$definition = $ast.Find({
|
||
param($node)
|
||
$node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $functionName
|
||
}, $true)
|
||
if ($definition) {
|
||
Invoke-Expression $definition.Extent.Text
|
||
$formatFunctions[$functionName] = $true
|
||
}
|
||
}
|
||
if ($formatFunctions.ContainsKey("Format-AgentTelegramText")) {
|
||
$perfCard = Format-AgentTelegramText "warning" "performance_warning" "test" ([pscustomobject]@{
|
||
host = "192.168.0.120"
|
||
reasons = @("perf_readback_failed")
|
||
loadPerCore = $null
|
||
memAvailablePercent = $null
|
||
diskUsedPercent = $null
|
||
})
|
||
$providerCard = Format-AgentTelegramText "warning" "provider_freshness_triage" "test" ([pscustomobject]@{
|
||
candidateId = "pf-test"
|
||
target = "provider_freshness_signal"
|
||
hitCount = 2
|
||
missingFields = @("lastSeen")
|
||
status = "handled_blocked_by_trust_gate"
|
||
candidatePath = "C:\Wooo\Agent99\state\pf-test.json"
|
||
})
|
||
$providerCaption = Format-AgentTelegramCaption "warning" "provider_freshness_triage" "test" ([pscustomobject]@{
|
||
target = "provider_freshness_signal"
|
||
missingFields = @("lastSeen")
|
||
status = "handled_blocked_by_trust_gate"
|
||
candidatePath = "C:\Wooo\Agent99\state\pf-test.json"
|
||
})
|
||
$cards = "$perfCard`n$providerCard"
|
||
Add-SyntheticCheck "telegram:localized" ([regex]::IsMatch($cards, "[^\x00-\x7F]")) "localized card is present"
|
||
Add-SyntheticCheck "telegram:no_blank_metrics" ($perfCard.Contains("數值目前不可信") -and -not $perfCard.Contains("load/core=, ")) "failed readback is explained without blank metrics"
|
||
Add-SyntheticCheck "telegram:provider_handled" ($providerCard.Contains("Provider freshness") -and $providerCard.Contains("事件：INC-") -and $providerCard.Contains("Agent99：")) "provider incident has human lifecycle fields"
|
||
Add-SyntheticCheck "telegram:no_internal_paths" (-not $cards.Contains("C:\Wooo") -and -not $cards.Contains("KM:") -and -not $cards.Contains("verifierName=")) "raw evidence and verifier fields are absent"
|
||
Add-SyntheticCheck "telegram:caption_bounded" ($providerCaption.Length -lt 900 -and $providerCaption.Contains("下一次更新：")) "visual caption is concise and actionable"
|
||
Add-SyntheticCheck "telegram:no_raw_python" (-not $cards.Contains("urllib.request")) "relay source is absent from user-facing card"
|
||
} else {
|
||
Add-SyntheticCheck "telegram:formatter" $false "Format-AgentTelegramText not found"
|
||
}
|
||
|
||
if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) {
|
||
$fixtureEpoch = [DateTimeOffset]::Now.ToUnixTimeSeconds()
|
||
$fixtureConfig = [pscustomobject]@{
|
||
maxArtifactAgeSeconds = 600
|
||
artifactClockSkewSeconds = 90
|
||
requiredHostAliases = @("99", "110", "111", "112", "120", "121", "188")
|
||
}
|
||
$fixtureLines = @(
|
||
"RECOVERY_READBACK=1",
|
||
"RECOVERY_ARTIFACT_DIR=/home/wooo/reboot-recovery/reboot-auto-recovery-slo-selftest",
|
||
"RECOVERY_ARTIFACT_EPOCH=$fixtureEpoch",
|
||
"POST_START_RESULT=GREEN_WITH_EVIDENCE_WARNINGS",
|
||
"POST_START_BLOCKED=0",
|
||
"SERVICE_GREEN=1",
|
||
"PRODUCT_DATA_GREEN=1",
|
||
"STOCK_FRESHNESS_STATUS=ok",
|
||
"BACKUP_CORE_GREEN=1",
|
||
"HOST_188_SERVICE_GREEN=1",
|
||
"HOST_188_HYGIENE_BLOCKED=0",
|
||
"EDGE_FALLBACK_READY=1",
|
||
"VMWARE_AUTOSTART_VERIFY_READY=1",
|
||
"WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1",
|
||
"SCORECARD_STATUS=ready_reboot_auto_recovery_slo",
|
||
"SCORECARD_BLOCKER_COUNT=0",
|
||
"SCORECARD_READINESS_PERCENT=100",
|
||
"SCORECARD_PRIMARY_BLOCKER=",
|
||
"SCORECARD_CAN_CLAIM=1",
|
||
"SCORECARD_FRESH_REBOOT_WINDOW=1",
|
||
"SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0",
|
||
"SCORECARD_SOURCE_CONTROLS_READY=1",
|
||
"SCORECARD_PUBLIC_FALLBACK_READY=1",
|
||
"HOST_BOOT alias=99 target=192.168.0.99 startup_unit=vmware-host-autostart reachable=1 boot_id=windows uptime_seconds=30 systemd_state=windows_ssh startup_enabled=unknown startup_active=unknown",
|
||
"HOST_BOOT alias=110 target=wooo@192.168.0.110 startup_unit=awoooi-startup-110.service reachable=1 boot_id=linux110 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active",
|
||
"HOST_BOOT alias=111 target=ooo@192.168.0.111 startup_unit=com.momo.ollama111-allow-proxy reachable=1 boot_id=darwin uptime_seconds=40 systemd_state=darwin_ssh startup_enabled=enabled startup_active=active",
|
||
"HOST_BOOT alias=112 target=192.168.0.112 startup_unit=vm-host-boot reachable=1 boot_id=linux112 uptime_seconds=35 systemd_state=node_exporter startup_enabled=unknown startup_active=unknown",
|
||
"HOST_BOOT alias=120 target=wooo@192.168.0.120 startup_unit=k3s.service reachable=1 boot_id=linux120 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active",
|
||
"HOST_BOOT alias=121 target=wooo@192.168.0.121 startup_unit=k3s.service reachable=1 boot_id=linux121 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active",
|
||
"HOST_BOOT alias=188 target=ollama@192.168.0.188 startup_unit=awoooi-startup.service reachable=1 boot_id=linux188 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=inactive"
|
||
) -join "`n"
|
||
$freshReadback = Convert-AgentRecoveryReadback $fixtureLines (Get-Date).AddSeconds(-30) $true $fixtureConfig
|
||
Add-SyntheticCheck "recovery:coordinator_fresh_fixture" ($freshReadback.verified -and $freshReadback.rebootSloClaimed -and @($freshReadback.hosts).Count -eq 7) "fresh seven-host scorecard is verified"
|
||
|
||
$blockedFixture = $fixtureLines.Replace("SCORECARD_CAN_CLAIM=1", "SCORECARD_CAN_CLAIM=0").Replace("SCORECARD_BLOCKER_COUNT=0", "SCORECARD_BLOCKER_COUNT=3")
|
||
$blockedReadback = Convert-AgentRecoveryReadback $blockedFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig
|
||
Add-SyntheticCheck "recovery:coordinator_false_green_guard" (-not $blockedReadback.verified -and -not $blockedReadback.rebootSloClaimed -and @($blockedReadback.failedChecks).Count -ge 1) "non-SLO blockers fail closed even when service evidence is otherwise green"
|
||
|
||
$lateFixture = $blockedFixture.Replace("SCORECARD_FRESH_REBOOT_WINDOW=1", "SCORECARD_FRESH_REBOOT_WINDOW=0").Replace("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0", "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=1")
|
||
$lateReadback = Convert-AgentRecoveryReadback $lateFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig
|
||
Add-SyntheticCheck "recovery:coordinator_late_recovery_terminal" ($lateReadback.verified -and -not $lateReadback.rebootSloClaimed -and @($lateReadback.failedChecks).Count -eq 0 -and @($lateReadback.sloFailedChecks).Count -ge 2) "service recovery can close late without rewriting the reboot SLO breach"
|
||
} else {
|
||
Add-SyntheticCheck "recovery:coordinator_parser" $false "Convert-AgentRecoveryReadback not found"
|
||
}
|
||
|
||
$control = $controlSource
|
||
Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_already_queued") -and $control.Contains("recovery_trigger_cooldown") -and $control.Contains('mode = "Recover"')) "automatic recovery is queued and deduplicated"
|
||
Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence"
|
||
Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("recovered_late")) "boot recovery remains durable through first-run termination and late closure"
|
||
Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present"
|
||
|
||
$failures = @($checks | Where-Object { -not $_.ok })
|
||
$result = [pscustomobject]@{
|
||
timestamp = (Get-Date -Format o)
|
||
ok = [bool]($failures.Count -eq 0)
|
||
workRoot = $WorkRoot
|
||
checkCount = $checks.Count
|
||
failureCount = $failures.Count
|
||
checks = $checks
|
||
failures = $failures
|
||
}
|
||
$result | ConvertTo-Json -Depth 8
|
||
if (-not $result.ok) { exit 1 }
|
||
exit 0
|