Files
awoooi/agent99-synthetic-tests.ps1
ogt a6307e785d
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 3m41s
CD Pipeline / build-and-deploy (push) Successful in 7m17s
CD Pipeline / post-deploy-checks (push) Successful in 1m59s
feat(automation): harden D037 runtime closure receipts
2026-07-14 14:32:43 +08:00

284 lines
19 KiB
PowerShell
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
param(
[string]$SourceRoot = (Split-Path -Parent $MyInvocation.MyCommand.Path),
[string]$WorkRoot = (Join-Path $env:TEMP ("agent99-synthetic-" + (Get-Date -Format "yyyyMMdd-HHmmss")))
)
$ErrorActionPreference = "Stop"
$checks = @()
function Add-SyntheticCheck {
param([string]$Name, [bool]$Ok, [string]$Detail)
$script:checks += [pscustomobject]@{ name = $Name; ok = $Ok; detail = $Detail }
}
Remove-Item -Recurse -Force $WorkRoot -ErrorAction SilentlyContinue
New-Item -ItemType Directory -Force -Path $WorkRoot | Out-Null
$inbox = Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"
$routingOutput = @(& $inbox -AgentRoot $WorkRoot -SelfTest 2>&1) -join "`n"
$routingExitCode = $LASTEXITCODE
try {
$routing = $routingOutput | ConvertFrom-Json
Add-SyntheticCheck "routing:self_test" ($routingExitCode -eq 0 -and $routing.ok -eq $true) "passed=$($routing.passedCount)/$($routing.caseCount) exit=$routingExitCode"
Add-SyntheticCheck "routing:provider_freshness" (@($routing.results | Where-Object { $_.alertId -eq "selftest-provider-freshness" -and $_.mode -eq "ProviderFreshness" }).Count -eq 1) "provider freshness structured route"
Add-SyntheticCheck "routing:reboot_slo" (@($routing.results | Where-Object { $_.alertId -eq "selftest-reboot-slo" -and $_.mode -eq "Recover" -and $_.routeSource -eq "suggestedMode" }).Count -eq 1) "reboot SLO has priority over contaminated text"
} catch {
Add-SyntheticCheck "routing:self_test" $false "invalid self-test JSON: $($_.Exception.Message)"
}
$tokens = $null
$errors = $null
$controlPath = Join-Path $SourceRoot "agent99-control-plane.ps1"
$controlSource = [IO.File]::ReadAllText($controlPath, [Text.Encoding]::UTF8)
$ast = [System.Management.Automation.Language.Parser]::ParseInput($controlSource, [ref]$tokens, [ref]$errors)
$formatFunctions = @{}
foreach ($functionName in @(
"Convert-AgentBool",
"Get-AgentObjectValue",
"Limit-AgentTextLine",
"Format-AgentMetricForCard",
"Format-AgentTelegramLegacyText",
"Get-AgentSha256Hex",
"Get-AgentIncidentCardModel",
"Format-AgentTelegramText",
"Format-AgentTelegramCaption",
"Convert-AgentRecoveryReadback",
"Get-HostSshUser",
"Get-AgentSshTransportConfig",
"Get-AgentHost112TransportContract",
"Get-AgentHost112TimeoutContract",
"Convert-AgentHost112GuestReadback",
"Invoke-AgentHost112GuestRecovery"
)) {
$definition = $ast.Find({
param($node)
$node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $functionName
}, $true)
if ($definition) {
Invoke-Expression $definition.Extent.Text
$formatFunctions[$functionName] = $true
}
}
if ($formatFunctions.ContainsKey("Format-AgentTelegramText")) {
$perfCard = Format-AgentTelegramText "warning" "performance_warning" "test" ([pscustomobject]@{
host = "192.168.0.120"
reasons = @("perf_readback_failed")
loadPerCore = $null
memAvailablePercent = $null
diskUsedPercent = $null
})
$providerCard = Format-AgentTelegramText "warning" "provider_freshness_triage" "test" ([pscustomobject]@{
candidateId = "pf-test"
target = "provider_freshness_signal"
hitCount = 2
missingFields = @("lastSeen")
status = "handled_blocked_by_trust_gate"
candidatePath = "C:\Wooo\Agent99\state\pf-test.json"
})
$providerCaption = Format-AgentTelegramCaption "warning" "provider_freshness_triage" "test" ([pscustomobject]@{
target = "provider_freshness_signal"
missingFields = @("lastSeen")
status = "handled_blocked_by_trust_gate"
candidatePath = "C:\Wooo\Agent99\state\pf-test.json"
})
$cards = "$perfCard`n$providerCard"
Add-SyntheticCheck "telegram:localized" ([regex]::IsMatch($cards, "[^\x00-\x7F]")) "localized card is present"
Add-SyntheticCheck "telegram:no_blank_metrics" ($perfCard.Contains("數值目前不可信") -and -not $perfCard.Contains("load/core=, ")) "failed readback is explained without blank metrics"
Add-SyntheticCheck "telegram:provider_handled" ($providerCard.Contains("Provider freshness") -and $providerCard.Contains("事件INC-") -and $providerCard.Contains("Agent99")) "provider incident has human lifecycle fields"
Add-SyntheticCheck "telegram:no_internal_paths" (-not $cards.Contains("C:\Wooo") -and -not $cards.Contains("KM:") -and -not $cards.Contains("verifierName=")) "raw evidence and verifier fields are absent"
Add-SyntheticCheck "telegram:caption_bounded" ($providerCaption.Length -lt 900 -and $providerCaption.Contains("下一次更新:")) "visual caption is concise and actionable"
Add-SyntheticCheck "telegram:no_raw_python" (-not $cards.Contains("urllib.request")) "relay source is absent from user-facing card"
} else {
Add-SyntheticCheck "telegram:formatter" $false "Format-AgentTelegramText not found"
}
if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) {
$fixtureEpoch = [DateTimeOffset]::Now.ToUnixTimeSeconds()
$fixtureConfig = [pscustomobject]@{
maxArtifactAgeSeconds = 600
artifactClockSkewSeconds = 90
requiredHostAliases = @("99", "110", "111", "112", "120", "121", "188")
}
$fixtureLines = @(
"RECOVERY_READBACK=1",
"RECOVERY_ARTIFACT_DIR=/home/wooo/reboot-recovery/reboot-auto-recovery-slo-selftest",
"RECOVERY_ARTIFACT_EPOCH=$fixtureEpoch",
"POST_START_RESULT=GREEN_WITH_EVIDENCE_WARNINGS",
"POST_START_BLOCKED=0",
"SERVICE_GREEN=1",
"PRODUCT_DATA_GREEN=1",
"STOCK_FRESHNESS_STATUS=ok",
"BACKUP_CORE_GREEN=1",
"HOST_188_SERVICE_GREEN=1",
"HOST_188_HYGIENE_BLOCKED=0",
"EDGE_FALLBACK_READY=1",
"VMWARE_AUTOSTART_VERIFY_READY=1",
"WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1",
"SCORECARD_STATUS=ready_reboot_auto_recovery_slo",
"SCORECARD_BLOCKER_COUNT=0",
"SCORECARD_READINESS_PERCENT=100",
"SCORECARD_PRIMARY_BLOCKER=",
"SCORECARD_CAN_CLAIM=1",
"SCORECARD_FRESH_REBOOT_WINDOW=1",
"SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0",
"SCORECARD_SOURCE_CONTROLS_READY=1",
"SCORECARD_PUBLIC_FALLBACK_READY=1",
"HOST_BOOT alias=99 target=192.168.0.99 startup_unit=vmware-host-autostart reachable=1 boot_id=windows uptime_seconds=30 systemd_state=windows_ssh startup_enabled=unknown startup_active=unknown",
"HOST_BOOT alias=110 target=wooo@192.168.0.110 startup_unit=awoooi-startup-110.service reachable=1 boot_id=linux110 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active",
"HOST_BOOT alias=111 target=ooo@192.168.0.111 startup_unit=com.momo.ollama111-allow-proxy reachable=1 boot_id=darwin uptime_seconds=40 systemd_state=darwin_ssh startup_enabled=enabled startup_active=active",
"HOST_BOOT alias=112 target=192.168.0.112 startup_unit=vm-host-boot reachable=1 boot_id=linux112 uptime_seconds=35 systemd_state=node_exporter startup_enabled=unknown startup_active=unknown",
"HOST_BOOT alias=120 target=wooo@192.168.0.120 startup_unit=k3s.service reachable=1 boot_id=linux120 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active",
"HOST_BOOT alias=121 target=wooo@192.168.0.121 startup_unit=k3s.service reachable=1 boot_id=linux121 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active",
"HOST_BOOT alias=188 target=ollama@192.168.0.188 startup_unit=awoooi-startup.service reachable=1 boot_id=linux188 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=inactive"
) -join "`n"
$freshReadback = Convert-AgentRecoveryReadback $fixtureLines (Get-Date).AddSeconds(-30) $true $fixtureConfig
Add-SyntheticCheck "recovery:coordinator_fresh_fixture" ($freshReadback.verified -and $freshReadback.rebootSloClaimed -and @($freshReadback.hosts).Count -eq 7) "fresh seven-host scorecard is verified"
$blockedFixture = $fixtureLines.Replace("SCORECARD_CAN_CLAIM=1", "SCORECARD_CAN_CLAIM=0").Replace("SCORECARD_BLOCKER_COUNT=0", "SCORECARD_BLOCKER_COUNT=3")
$blockedReadback = Convert-AgentRecoveryReadback $blockedFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig
Add-SyntheticCheck "recovery:coordinator_false_green_guard" (-not $blockedReadback.verified -and -not $blockedReadback.rebootSloClaimed -and @($blockedReadback.failedChecks).Count -ge 1) "non-SLO blockers fail closed even when service evidence is otherwise green"
$lateFixture = $blockedFixture.Replace("SCORECARD_FRESH_REBOOT_WINDOW=1", "SCORECARD_FRESH_REBOOT_WINDOW=0").Replace("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0", "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=1")
$lateReadback = Convert-AgentRecoveryReadback $lateFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig
Add-SyntheticCheck "recovery:coordinator_late_recovery_terminal" ($lateReadback.verified -and -not $lateReadback.rebootSloClaimed -and @($lateReadback.failedChecks).Count -eq 0 -and @($lateReadback.sloFailedChecks).Count -ge 2) "service recovery can close late without rewriting the reboot SLO breach"
} else {
Add-SyntheticCheck "recovery:coordinator_parser" $false "Convert-AgentRecoveryReadback not found"
}
if (
$formatFunctions.ContainsKey("Get-HostSshUser") -and
$formatFunctions.ContainsKey("Get-AgentSshTransportConfig") -and
$formatFunctions.ContainsKey("Get-AgentHost112TransportContract") -and
$formatFunctions.ContainsKey("Get-AgentHost112TimeoutContract") -and
$formatFunctions.ContainsKey("Convert-AgentHost112GuestReadback") -and
$formatFunctions.ContainsKey("Invoke-AgentHost112GuestRecovery")
) {
$Config = [pscustomobject]@{
agentRoot = $WorkRoot
sshUser = "wooo"
sshUsers = [pscustomobject]@{ "192.168.0.112" = "kali" }
sshTransport = [pscustomobject]@{ serialized = $true; lockTimeoutSeconds = 90; lockPollMilliseconds = 200 }
k3s = [pscustomobject]@{ jumpHost = "192.168.0.110"; preferJumpHost = $true; directHosts = @("192.168.0.110", "192.168.0.112") }
}
$SshUser = "wooo"
$SshIdentityFile = Join-Path $WorkRoot "keys\agent99_ed25519"
$ControlledApply = $true
$script:Agent99DispatchClientDeadlineSeconds = 1800
$script:Agent99QueueOuterDeadlineSeconds = 1680
$script:Agent99Host112CheckTimeoutSeconds = 30
$script:Agent99Host112DryRunTimeoutSeconds = 30
$script:Agent99Host112ExecutorBudgetSeconds = 420
$script:Agent99Host112ApplyTransportTimeoutSeconds = 510
$script:Agent99Host112ReceiptTimeoutSeconds = 30
$script:Agent99Host112AfterTimeoutSeconds = 30
$script:Agent99Host112SshCallCount = 5
$script:Host112SyntheticCall = 0
$script:Host112SyntheticScenario = "general_only"
function Write-AgentLog { param([string]$Message) }
function Record-AgentEvent {
param([string]$Type, [string]$Severity, [string]$Message, [object]$Data, [switch]$Alert)
}
function New-Host112SyntheticRow {
param(
[string]$Mode,
[int]$ConsoleReady,
[int]$RuntimeWrite,
[string]$Actions,
[int]$ReceiptWrite,
[string]$ReceiptStatus,
[int]$DurablePresent,
[string]$DurableTerminal,
[int]$ExecutorExit
)
$guestReady = $ConsoleReady
$receiptRef = if ($DurablePresent -eq 1) { "host112-manager-recovery:fixture" } else { "none" }
$receiptPath = if ($DurablePresent -eq 1) { "/var/lib/awoooi-host112-recovery/receipts/fixture.txt" } else { "none" }
$receiptSha = if ($DurablePresent -eq 1) { "a" * 64 } else { "none" }
$durableReadbackStatus = if ($DurablePresent -ne 1) { "not_requested" } elseif ($Mode -eq "apply") { "written_immutable" } else { "loaded_immutable_verified" }
$preflight = "already_verified_healthy"
"schema_version=host112_guest_recovery_v2 mode=$Mode trace_id=trace:fixture run_id=run:fixture work_item_id=HOST112-FIXTURE boot_id=boot-fixture uptime_seconds=999 systemd_state=running default_target=graphical.target graphical_target=active display_manager=active lightdm=active vmware_tools=active xorg=active wazuh_indexer=active wazuh_manager=active wazuh_manager_result=success wazuh_analysisd_process_count=1 wazuh_dashboard=active filebeat=active ssh_service=active ssh_enabled=enabled ssh_port_listening=1 ssh_ready=1 console_ready=$ConsoleReady services_ready=1 recovery_timer_ready=1 guest_ready=$guestReady manager_preflight_status=$preflight manager_resource_ready=1 wazuh_agent_event_port_1514_ready=1 wazuh_agent_enrollment_port_1515_tcp=1 wazuh_manager_api_port_55000_tcp=1 manager_independent_verifier_ready=1 manager_attempt_count=0 manager_start_exit=not_attempted executor_deadline_seconds=420 runtime_write_performed=$RuntimeWrite state_write_performed=0 manager_marker_write_performed=0 manager_runtime_write_performed=0 receipt_write_performed=$ReceiptWrite receipt_status=$ReceiptStatus receipt_ref=$receiptRef receipt_path=$receiptPath receipt_sha256=$receiptSha receipt_reused=0 durable_receipt_present=$DurablePresent durable_receipt_identity_match=$DurablePresent durable_receipt_boot_match=$DurablePresent durable_receipt_terminal=$DurableTerminal durable_readback_present=$DurablePresent durable_readback_identity_match=$DurablePresent durable_readback_boot_match=$DurablePresent durable_readback_receipt_link_match=$DurablePresent durable_readback_terminal_match=$DurablePresent durable_readback_status=$durableReadbackStatus durable_readback_path=$receiptPath.readback durable_readback_sha256=$receiptSha rollback_attempted=0 rollback_verified=0 rollback_terminal=not_required action_count=1 actions=$Actions executor_exit_code=$ExecutorExit"
}
function Invoke-AgentHost112SshText {
param([string]$Command, [int]$TimeoutSeconds, [int]$Retries = 1)
$script:Host112SyntheticCall += 1
$row = if ($script:Host112SyntheticScenario -eq "failed_receipt_external_recovery") {
New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "rolled_back_verifier_failed" 0
} else {
switch ($script:Host112SyntheticCall) {
1 { New-Host112SyntheticRow "check" 0 0 "none" 0 "not_requested" 0 "unknown" 1 }
2 { New-Host112SyntheticRow "dry_run" 0 0 "none" 0 "not_requested" 0 "unknown" 0 }
3 { New-Host112SyntheticRow "apply" 1 1 "start_lightdm" 1 "written_immutable" 1 "verified_healthy" 0 }
4 { New-Host112SyntheticRow "dry_run" 1 0 "none" 0 "not_requested" 1 "verified_healthy" 0 }
default { New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "verified_healthy" 0 }
}
}
[pscustomobject]@{
ok = $true
exitCode = 0
output = $row
elapsedMs = 1
transportSerialized = $true
transportLockWaitMs = 0
route = "direct_host112_kali"
}
}
$generalOnly = Invoke-AgentHost112GuestRecovery `
-RequestedRunId "run:fixture" `
-RequestedTraceId "trace:fixture" `
-RequestedWorkItemId "HOST112-FIXTURE"
Add-SyntheticCheck "recovery:host112_general_candidate_not_blocked_by_healthy_manager" (
$generalOnly.applyAttempted -and
$generalOnly.generalCandidateRequired -and
$generalOnly.generalApplyVerified -and
-not $generalOnly.managerCandidateRequired -and
-not $generalOnly.applyReceipt.managerRuntimeWritePerformed -and
[string]$generalOnly.applyReceipt.managerAttemptCount -eq "0" -and
$generalOnly.durableReceiptVerified -and
$generalOnly.verified
) "manager healthy plus console degraded executes only general repair and closes from immutable receipt plus second verifier"
$script:Host112SyntheticCall = 0
$script:Host112SyntheticScenario = "failed_receipt_external_recovery"
$failedReplay = Invoke-AgentHost112GuestRecovery `
-RequestedRunId "run:fixture" `
-RequestedTraceId "trace:fixture" `
-RequestedWorkItemId "HOST112-FIXTURE"
Add-SyntheticCheck "recovery:host112_failed_apply_cannot_be_washed_green" (
-not $failedReplay.applyAttempted -and
-not $failedReplay.verified -and
$failedReplay.managerVerified -and
$failedReplay.before.durableReceiptTerminal -eq "rolled_back_verifier_failed" -and
$failedReplay.terminal -eq "no_write_immutable_receipt_replay_not_successful"
) "a later healthy manager cannot replace a failed immutable receipt for the same run"
} else {
Add-SyntheticCheck "recovery:host112_general_candidate_fixture" $false "Host112 control functions not found"
}
$control = $controlSource
Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_already_queued") -and $control.Contains("recovery_trigger_cooldown") -and $control.Contains('mode = "Recover"')) "automatic recovery is queued and deduplicated"
Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence"
Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("recovered_late")) "boot recovery remains durable through first-run termination and late closure"
Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present"
Add-SyntheticCheck "recovery:host112_apply_receipt_closure" ($control.Contains("managerApplyVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('artifactTransactionStatus -eq "committed_verified_pair"') -and $control.Contains('artifactTransactionPriorPairVerified') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains("durableReceiptVerified") -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply cannot be closed without its fixed-WAL commit, attributed start and immutable receipt/readback"
Add-SyntheticCheck "recovery:host112_candidate_split" ($control.Contains("generalCandidateRequired") -and $control.Contains("managerCandidateRequired") -and $control.Contains("generalApplyEligible") -and $control.Contains("managerApplyEligible") -and $control.Contains("managerNoAttemptPreserved")) "Host112 general guest repair is independent from manager eligibility"
Add-SyntheticCheck "recovery:host112_timeout_contract" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120") -and $control.Contains('$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)')) "Host112 executor, SSH lock, queue and client deadlines have explicit reserves"
$deployer = Get-Content (Join-Path $SourceRoot "agent99-deploy.ps1") -Raw
$hostConfig = Get-Content (Join-Path $SourceRoot "agent99.config.99.example.json") -Raw
Add-SyntheticCheck "recovery:host112_canonical_vmx" ($deployer.Contains('name = "host112_canonical_vmx_path"') -and $deployer.Contains('Get-AgentHost112CanonicalVm $policyConfigPath') -and $deployer.Contains('Set-AgentProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))') -and $hostConfig.Contains('S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx')) "Agent99 normalizes Host112 from one staged VMX identity with a bounded post-verifier"
$failures = @($checks | Where-Object { -not $_.ok })
$result = [pscustomobject]@{
timestamp = (Get-Date -Format o)
ok = [bool]($failures.Count -eq 0)
workRoot = $WorkRoot
checkCount = $checks.Count
failureCount = $failures.Count
checks = $checks
failures = $failures
}
$result | ConvertTo-Json -Depth 8
if (-not $result.ok) { exit 1 }
exit 0