param( [string]$SourceRoot = (Split-Path -Parent $MyInvocation.MyCommand.Path), [string]$WorkRoot = (Join-Path $env:TEMP ("agent99-synthetic-" + (Get-Date -Format "yyyyMMdd-HHmmss"))) ) $ErrorActionPreference = "Stop" $checks = @() function Add-SyntheticCheck { param([string]$Name, [bool]$Ok, [string]$Detail) $script:checks += [pscustomobject]@{ name = $Name; ok = $Ok; detail = $Detail } } Remove-Item -Recurse -Force $WorkRoot -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path $WorkRoot | Out-Null $inbox = Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1" $routingOutput = @(& $inbox -AgentRoot $WorkRoot -SelfTest 2>&1) -join "`n" $routingExitCode = $LASTEXITCODE try { $routing = $routingOutput | ConvertFrom-Json Add-SyntheticCheck "routing:self_test" ($routingExitCode -eq 0 -and $routing.ok -eq $true) "passed=$($routing.passedCount)/$($routing.caseCount) exit=$routingExitCode" Add-SyntheticCheck "routing:provider_freshness" (@($routing.results | Where-Object { $_.alertId -eq "selftest-provider-freshness" -and $_.mode -eq "ProviderFreshness" }).Count -eq 1) "provider freshness structured route" Add-SyntheticCheck "routing:reboot_slo" (@($routing.results | Where-Object { $_.alertId -eq "selftest-reboot-slo" -and $_.mode -eq "Recover" -and $_.routeSource -eq "suggestedMode" }).Count -eq 1) "reboot SLO has priority over contaminated text" } catch { Add-SyntheticCheck "routing:self_test" $false "invalid self-test JSON: $($_.Exception.Message)" } $tokens = $null $errors = $null $controlPath = Join-Path $SourceRoot "agent99-control-plane.ps1" $controlSource = [IO.File]::ReadAllText($controlPath, [Text.Encoding]::UTF8) $ast = [System.Management.Automation.Language.Parser]::ParseInput($controlSource, [ref]$tokens, [ref]$errors) $formatFunctions = @{} foreach ($functionName in @( "Convert-AgentBool", "Get-AgentObjectValue", "Limit-AgentTextLine", "Format-AgentMetricForCard", "Format-AgentTelegramLegacyText", "Get-AgentSha256Hex", "Get-AgentIncidentCardModel", "Format-AgentTelegramText", "Format-AgentTelegramCaption", "Convert-AgentRecoveryReadback", "Get-HostSshUser", "Get-AgentSshTransportConfig", "Get-AgentHost112TransportContract", "Get-AgentHost112TimeoutContract", "Convert-AgentHost112GuestReadback", "Invoke-AgentHost112GuestRecovery" )) { $definition = $ast.Find({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $functionName }, $true) if ($definition) { Invoke-Expression $definition.Extent.Text $formatFunctions[$functionName] = $true } } if ($formatFunctions.ContainsKey("Format-AgentTelegramText")) { $perfCard = Format-AgentTelegramText "warning" "performance_warning" "test" ([pscustomobject]@{ host = "192.168.0.120" reasons = @("perf_readback_failed") loadPerCore = $null memAvailablePercent = $null diskUsedPercent = $null }) $providerCard = Format-AgentTelegramText "warning" "provider_freshness_triage" "test" ([pscustomobject]@{ candidateId = "pf-test" target = "provider_freshness_signal" hitCount = 2 missingFields = @("lastSeen") status = "handled_blocked_by_trust_gate" candidatePath = "C:\Wooo\Agent99\state\pf-test.json" }) $providerCaption = Format-AgentTelegramCaption "warning" "provider_freshness_triage" "test" ([pscustomobject]@{ target = "provider_freshness_signal" missingFields = @("lastSeen") status = "handled_blocked_by_trust_gate" candidatePath = "C:\Wooo\Agent99\state\pf-test.json" }) $cards = "$perfCard`n$providerCard" Add-SyntheticCheck "telegram:localized" ([regex]::IsMatch($cards, "[^\x00-\x7F]")) "localized card is present" Add-SyntheticCheck "telegram:no_blank_metrics" ($perfCard.Contains("數值目前不可信") -and -not $perfCard.Contains("load/core=, ")) "failed readback is explained without blank metrics" Add-SyntheticCheck "telegram:provider_handled" ($providerCard.Contains("Provider freshness") -and $providerCard.Contains("事件：INC-") -and $providerCard.Contains("Agent99：")) "provider incident has human lifecycle fields" Add-SyntheticCheck "telegram:no_internal_paths" (-not $cards.Contains("C:\Wooo") -and -not $cards.Contains("KM:") -and -not $cards.Contains("verifierName=")) "raw evidence and verifier fields are absent" Add-SyntheticCheck "telegram:caption_bounded" ($providerCaption.Length -lt 900 -and $providerCaption.Contains("下一次更新：")) "visual caption is concise and actionable" Add-SyntheticCheck "telegram:no_raw_python" (-not $cards.Contains("urllib.request")) "relay source is absent from user-facing card" } else { Add-SyntheticCheck "telegram:formatter" $false "Format-AgentTelegramText not found" } if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) { $fixtureEpoch = [DateTimeOffset]::Now.ToUnixTimeSeconds() $fixtureConfig = [pscustomobject]@{ maxArtifactAgeSeconds = 600 artifactClockSkewSeconds = 90 requiredHostAliases = @("99", "110", "111", "112", "120", "121", "188") } $fixtureLines = @( "RECOVERY_READBACK=1", "RECOVERY_ARTIFACT_DIR=/home/wooo/reboot-recovery/reboot-auto-recovery-slo-selftest", "RECOVERY_ARTIFACT_EPOCH=$fixtureEpoch", "POST_START_RESULT=GREEN_WITH_EVIDENCE_WARNINGS", "POST_START_BLOCKED=0", "SERVICE_GREEN=1", "PRODUCT_DATA_GREEN=1", "STOCK_FRESHNESS_STATUS=ok", "BACKUP_CORE_GREEN=1", "HOST_188_SERVICE_GREEN=1", "HOST_188_HYGIENE_BLOCKED=0", "EDGE_FALLBACK_READY=1", "VMWARE_AUTOSTART_VERIFY_READY=1", "WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1", "SCORECARD_STATUS=ready_reboot_auto_recovery_slo", "SCORECARD_BLOCKER_COUNT=0", "SCORECARD_READINESS_PERCENT=100", "SCORECARD_PRIMARY_BLOCKER=", "SCORECARD_CAN_CLAIM=1", "SCORECARD_FRESH_REBOOT_WINDOW=1", "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0", "SCORECARD_SOURCE_CONTROLS_READY=1", "SCORECARD_PUBLIC_FALLBACK_READY=1", "HOST_BOOT alias=99 target=192.168.0.99 startup_unit=vmware-host-autostart reachable=1 boot_id=windows uptime_seconds=30 systemd_state=windows_ssh startup_enabled=unknown startup_active=unknown", "HOST_BOOT alias=110 target=wooo@192.168.0.110 startup_unit=awoooi-startup-110.service reachable=1 boot_id=linux110 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active", "HOST_BOOT alias=111 target=ooo@192.168.0.111 startup_unit=com.momo.ollama111-allow-proxy reachable=1 boot_id=darwin uptime_seconds=40 systemd_state=darwin_ssh startup_enabled=enabled startup_active=active", "HOST_BOOT alias=112 target=192.168.0.112 startup_unit=vm-host-boot reachable=1 boot_id=linux112 uptime_seconds=35 systemd_state=node_exporter startup_enabled=unknown startup_active=unknown", "HOST_BOOT alias=120 target=wooo@192.168.0.120 startup_unit=k3s.service reachable=1 boot_id=linux120 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active", "HOST_BOOT alias=121 target=wooo@192.168.0.121 startup_unit=k3s.service reachable=1 boot_id=linux121 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active", "HOST_BOOT alias=188 target=ollama@192.168.0.188 startup_unit=awoooi-startup.service reachable=1 boot_id=linux188 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=inactive" ) -join "`n" $freshReadback = Convert-AgentRecoveryReadback $fixtureLines (Get-Date).AddSeconds(-30) $true $fixtureConfig Add-SyntheticCheck "recovery:coordinator_fresh_fixture" ($freshReadback.verified -and $freshReadback.rebootSloClaimed -and @($freshReadback.hosts).Count -eq 7) "fresh seven-host scorecard is verified" $blockedFixture = $fixtureLines.Replace("SCORECARD_CAN_CLAIM=1", "SCORECARD_CAN_CLAIM=0").Replace("SCORECARD_BLOCKER_COUNT=0", "SCORECARD_BLOCKER_COUNT=3") $blockedReadback = Convert-AgentRecoveryReadback $blockedFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig Add-SyntheticCheck "recovery:coordinator_false_green_guard" (-not $blockedReadback.verified -and -not $blockedReadback.rebootSloClaimed -and @($blockedReadback.failedChecks).Count -ge 1) "non-SLO blockers fail closed even when service evidence is otherwise green" $lateFixture = $blockedFixture.Replace("SCORECARD_FRESH_REBOOT_WINDOW=1", "SCORECARD_FRESH_REBOOT_WINDOW=0").Replace("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0", "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=1") $lateReadback = Convert-AgentRecoveryReadback $lateFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig Add-SyntheticCheck "recovery:coordinator_late_recovery_terminal" ($lateReadback.verified -and -not $lateReadback.rebootSloClaimed -and @($lateReadback.failedChecks).Count -eq 0 -and @($lateReadback.sloFailedChecks).Count -ge 2) "service recovery can close late without rewriting the reboot SLO breach" } else { Add-SyntheticCheck "recovery:coordinator_parser" $false "Convert-AgentRecoveryReadback not found" } if ( $formatFunctions.ContainsKey("Get-HostSshUser") -and $formatFunctions.ContainsKey("Get-AgentSshTransportConfig") -and $formatFunctions.ContainsKey("Get-AgentHost112TransportContract") -and $formatFunctions.ContainsKey("Get-AgentHost112TimeoutContract") -and $formatFunctions.ContainsKey("Convert-AgentHost112GuestReadback") -and $formatFunctions.ContainsKey("Invoke-AgentHost112GuestRecovery") ) { $Config = [pscustomobject]@{ agentRoot = $WorkRoot sshUser = "wooo" sshUsers = [pscustomobject]@{ "192.168.0.112" = "kali" } sshTransport = [pscustomobject]@{ serialized = $true; lockTimeoutSeconds = 90; lockPollMilliseconds = 200 } k3s = [pscustomobject]@{ jumpHost = "192.168.0.110"; preferJumpHost = $true; directHosts = @("192.168.0.110", "192.168.0.112") } } $SshUser = "wooo" $SshIdentityFile = Join-Path $WorkRoot "keys\agent99_ed25519" $ControlledApply = $true $script:Agent99DispatchClientDeadlineSeconds = 1800 $script:Agent99QueueOuterDeadlineSeconds = 1680 $script:Agent99Host112CheckTimeoutSeconds = 30 $script:Agent99Host112DryRunTimeoutSeconds = 30 $script:Agent99Host112ExecutorBudgetSeconds = 420 $script:Agent99Host112ApplyTransportTimeoutSeconds = 510 $script:Agent99Host112ReceiptTimeoutSeconds = 30 $script:Agent99Host112AfterTimeoutSeconds = 30 $script:Agent99Host112SshCallCount = 5 $script:Host112SyntheticCall = 0 $script:Host112SyntheticScenario = "general_only" function Write-AgentLog { param([string]$Message) } function Record-AgentEvent { param([string]$Type, [string]$Severity, [string]$Message, [object]$Data, [switch]$Alert) } function New-Host112SyntheticRow { param( [string]$Mode, [int]$ConsoleReady, [int]$RuntimeWrite, [string]$Actions, [int]$ReceiptWrite, [string]$ReceiptStatus, [int]$DurablePresent, [string]$DurableTerminal, [int]$ExecutorExit ) $guestReady = $ConsoleReady $receiptRef = if ($DurablePresent -eq 1) { "host112-manager-recovery:fixture" } else { "none" } $receiptPath = if ($DurablePresent -eq 1) { "/var/lib/awoooi-host112-recovery/receipts/fixture.txt" } else { "none" } $receiptSha = if ($DurablePresent -eq 1) { "a" * 64 } else { "none" } $durableReadbackStatus = if ($DurablePresent -ne 1) { "not_requested" } elseif ($Mode -eq "apply") { "written_immutable" } else { "loaded_immutable_verified" } $preflight = "already_verified_healthy" "schema_version=host112_guest_recovery_v2 mode=$Mode trace_id=trace:fixture run_id=run:fixture work_item_id=HOST112-FIXTURE boot_id=boot-fixture uptime_seconds=999 systemd_state=running default_target=graphical.target graphical_target=active display_manager=active lightdm=active vmware_tools=active xorg=active wazuh_indexer=active wazuh_manager=active wazuh_manager_result=success wazuh_analysisd_process_count=1 wazuh_dashboard=active filebeat=active ssh_service=active ssh_enabled=enabled ssh_port_listening=1 ssh_ready=1 console_ready=$ConsoleReady services_ready=1 recovery_timer_ready=1 guest_ready=$guestReady manager_preflight_status=$preflight manager_resource_ready=1 wazuh_agent_event_port_1514_ready=1 wazuh_agent_enrollment_port_1515_tcp=1 wazuh_manager_api_port_55000_tcp=1 manager_independent_verifier_ready=1 manager_attempt_count=0 manager_start_exit=not_attempted executor_deadline_seconds=420 runtime_write_performed=$RuntimeWrite state_write_performed=0 manager_marker_write_performed=0 manager_runtime_write_performed=0 receipt_write_performed=$ReceiptWrite receipt_status=$ReceiptStatus receipt_ref=$receiptRef receipt_path=$receiptPath receipt_sha256=$receiptSha receipt_reused=0 durable_receipt_present=$DurablePresent durable_receipt_identity_match=$DurablePresent durable_receipt_boot_match=$DurablePresent durable_receipt_terminal=$DurableTerminal durable_readback_present=$DurablePresent durable_readback_identity_match=$DurablePresent durable_readback_boot_match=$DurablePresent durable_readback_receipt_link_match=$DurablePresent durable_readback_terminal_match=$DurablePresent durable_readback_status=$durableReadbackStatus durable_readback_path=$receiptPath.readback durable_readback_sha256=$receiptSha rollback_attempted=0 rollback_verified=0 rollback_terminal=not_required action_count=1 actions=$Actions executor_exit_code=$ExecutorExit" } function Invoke-AgentHost112SshText { param([string]$Command, [int]$TimeoutSeconds, [int]$Retries = 1) $script:Host112SyntheticCall += 1 $row = if ($script:Host112SyntheticScenario -eq "failed_receipt_external_recovery") { New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "rolled_back_verifier_failed" 0 } else { switch ($script:Host112SyntheticCall) { 1 { New-Host112SyntheticRow "check" 0 0 "none" 0 "not_requested" 0 "unknown" 1 } 2 { New-Host112SyntheticRow "dry_run" 0 0 "none" 0 "not_requested" 0 "unknown" 0 } 3 { New-Host112SyntheticRow "apply" 1 1 "start_lightdm" 1 "written_immutable" 1 "verified_healthy" 0 } 4 { New-Host112SyntheticRow "dry_run" 1 0 "none" 0 "not_requested" 1 "verified_healthy" 0 } default { New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "verified_healthy" 0 } } } [pscustomobject]@{ ok = $true exitCode = 0 output = $row elapsedMs = 1 transportSerialized = $true transportLockWaitMs = 0 route = "direct_host112_kali" } } $generalOnly = Invoke-AgentHost112GuestRecovery ` -RequestedRunId "run:fixture" ` -RequestedTraceId "trace:fixture" ` -RequestedWorkItemId "HOST112-FIXTURE" Add-SyntheticCheck "recovery:host112_general_candidate_not_blocked_by_healthy_manager" ( $generalOnly.applyAttempted -and $generalOnly.generalCandidateRequired -and $generalOnly.generalApplyVerified -and -not $generalOnly.managerCandidateRequired -and -not $generalOnly.applyReceipt.managerRuntimeWritePerformed -and [string]$generalOnly.applyReceipt.managerAttemptCount -eq "0" -and $generalOnly.durableReceiptVerified -and $generalOnly.verified ) "manager healthy plus console degraded executes only general repair and closes from immutable receipt plus second verifier" $script:Host112SyntheticCall = 0 $script:Host112SyntheticScenario = "failed_receipt_external_recovery" $failedReplay = Invoke-AgentHost112GuestRecovery ` -RequestedRunId "run:fixture" ` -RequestedTraceId "trace:fixture" ` -RequestedWorkItemId "HOST112-FIXTURE" Add-SyntheticCheck "recovery:host112_failed_apply_cannot_be_washed_green" ( -not $failedReplay.applyAttempted -and -not $failedReplay.verified -and $failedReplay.managerVerified -and $failedReplay.before.durableReceiptTerminal -eq "rolled_back_verifier_failed" -and $failedReplay.terminal -eq "no_write_immutable_receipt_replay_not_successful" ) "a later healthy manager cannot replace a failed immutable receipt for the same run" } else { Add-SyntheticCheck "recovery:host112_general_candidate_fixture" $false "Host112 control functions not found" } $control = $controlSource Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_already_queued") -and $control.Contains("recovery_trigger_cooldown") -and $control.Contains('mode = "Recover"')) "automatic recovery is queued and deduplicated" Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence" Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("recovered_late")) "boot recovery remains durable through first-run termination and late closure" Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present" Add-SyntheticCheck "recovery:host112_apply_receipt_closure" ($control.Contains("managerApplyVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains("durableReceiptVerified") -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply cannot be closed by a later healthy observation without its attributed start and immutable receipt/readback" Add-SyntheticCheck "recovery:host112_candidate_split" ($control.Contains("generalCandidateRequired") -and $control.Contains("managerCandidateRequired") -and $control.Contains("generalApplyEligible") -and $control.Contains("managerApplyEligible") -and $control.Contains("managerNoAttemptPreserved")) "Host112 general guest repair is independent from manager eligibility" Add-SyntheticCheck "recovery:host112_timeout_contract" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120") -and $control.Contains('$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)')) "Host112 executor, SSH lock, queue and client deadlines have explicit reserves" $failures = @($checks | Where-Object { -not $_.ok }) $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) ok = [bool]($failures.Count -eq 0) workRoot = $WorkRoot checkCount = $checks.Count failureCount = $failures.Count checks = $checks failures = $failures } $result | ConvertTo-Json -Depth 8 if (-not $result.ok) { exit 1 } exit 0