320 lines
22 KiB
PowerShell
320 lines
22 KiB
PowerShell
param(
|
||
[string]$SourceRoot = (Split-Path -Parent $MyInvocation.MyCommand.Path),
|
||
[string]$WorkRoot = (Join-Path $env:TEMP ("agent99-synthetic-" + (Get-Date -Format "yyyyMMdd-HHmmss")))
|
||
)
|
||
|
||
$ErrorActionPreference = "Stop"
|
||
$checks = @()
|
||
|
||
function Add-SyntheticCheck {
|
||
param([string]$Name, [bool]$Ok, [string]$Detail)
|
||
$script:checks += [pscustomobject]@{ name = $Name; ok = $Ok; detail = $Detail }
|
||
}
|
||
|
||
Remove-Item -Recurse -Force $WorkRoot -ErrorAction SilentlyContinue
|
||
New-Item -ItemType Directory -Force -Path $WorkRoot | Out-Null
|
||
|
||
$inbox = Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"
|
||
$routingOutput = @(& $inbox -AgentRoot $WorkRoot -SelfTest 2>&1) -join "`n"
|
||
$routingExitCode = $LASTEXITCODE
|
||
try {
|
||
$routing = $routingOutput | ConvertFrom-Json
|
||
Add-SyntheticCheck "routing:self_test" ($routingExitCode -eq 0 -and $routing.ok -eq $true) "passed=$($routing.passedCount)/$($routing.caseCount) exit=$routingExitCode"
|
||
Add-SyntheticCheck "routing:provider_freshness" (@($routing.results | Where-Object { $_.alertId -eq "selftest-provider-freshness" -and $_.mode -eq "ProviderFreshness" }).Count -eq 1) "provider freshness structured route"
|
||
Add-SyntheticCheck "routing:reboot_slo" (@($routing.results | Where-Object { $_.alertId -eq "selftest-reboot-slo" -and $_.mode -eq "Recover" -and $_.routeSource -eq "suggestedMode" }).Count -eq 1) "reboot SLO has priority over contaminated text"
|
||
} catch {
|
||
Add-SyntheticCheck "routing:self_test" $false "invalid self-test JSON: $($_.Exception.Message)"
|
||
}
|
||
|
||
$tokens = $null
|
||
$errors = $null
|
||
$controlPath = Join-Path $SourceRoot "agent99-control-plane.ps1"
|
||
$controlSource = [IO.File]::ReadAllText($controlPath, [Text.Encoding]::UTF8)
|
||
$ast = [System.Management.Automation.Language.Parser]::ParseInput($controlSource, [ref]$tokens, [ref]$errors)
|
||
$formatFunctions = @{}
|
||
foreach ($functionName in @(
|
||
"Convert-AgentBool",
|
||
"Get-AgentObjectValue",
|
||
"Limit-AgentTextLine",
|
||
"Format-AgentMetricForCard",
|
||
"Format-AgentTelegramLegacyText",
|
||
"Get-AgentSha256Hex",
|
||
"Get-AgentIncidentCardModel",
|
||
"Format-AgentTelegramText",
|
||
"Format-AgentTelegramCaption",
|
||
"Convert-AgentRecoveryReadback",
|
||
"Get-HostSshUser",
|
||
"Get-AgentSshTransportConfig",
|
||
"Get-AgentHost112TransportContract",
|
||
"Get-AgentHost112TimeoutContract",
|
||
"Convert-AgentHost112GuestReadback",
|
||
"Invoke-AgentHost112GuestRecovery",
|
||
"Convert-AgentAwoooDeploymentReadback"
|
||
)) {
|
||
$definition = $ast.Find({
|
||
param($node)
|
||
$node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $functionName
|
||
}, $true)
|
||
if ($definition) {
|
||
Invoke-Expression $definition.Extent.Text
|
||
$formatFunctions[$functionName] = $true
|
||
}
|
||
}
|
||
if ($formatFunctions.ContainsKey("Format-AgentTelegramText")) {
|
||
$perfCard = Format-AgentTelegramText "warning" "performance_warning" "test" ([pscustomobject]@{
|
||
host = "192.168.0.120"
|
||
reasons = @("perf_readback_failed")
|
||
loadPerCore = $null
|
||
memAvailablePercent = $null
|
||
diskUsedPercent = $null
|
||
})
|
||
$providerCard = Format-AgentTelegramText "warning" "provider_freshness_triage" "test" ([pscustomobject]@{
|
||
candidateId = "pf-test"
|
||
target = "provider_freshness_signal"
|
||
hitCount = 2
|
||
missingFields = @("lastSeen")
|
||
status = "handled_blocked_by_trust_gate"
|
||
candidatePath = "C:\Wooo\Agent99\state\pf-test.json"
|
||
})
|
||
$providerCaption = Format-AgentTelegramCaption "warning" "provider_freshness_triage" "test" ([pscustomobject]@{
|
||
target = "provider_freshness_signal"
|
||
missingFields = @("lastSeen")
|
||
status = "handled_blocked_by_trust_gate"
|
||
candidatePath = "C:\Wooo\Agent99\state\pf-test.json"
|
||
})
|
||
$cards = "$perfCard`n$providerCard"
|
||
Add-SyntheticCheck "telegram:localized" ([regex]::IsMatch($cards, "[^\x00-\x7F]")) "localized card is present"
|
||
Add-SyntheticCheck "telegram:no_blank_metrics" ($perfCard.Contains("數值目前不可信") -and -not $perfCard.Contains("load/core=, ")) "failed readback is explained without blank metrics"
|
||
Add-SyntheticCheck "telegram:provider_handled" ($providerCard.Contains("Provider freshness") -and $providerCard.Contains("事件：INC-") -and $providerCard.Contains("Agent99：")) "provider incident has human lifecycle fields"
|
||
Add-SyntheticCheck "telegram:no_internal_paths" (-not $cards.Contains("C:\Wooo") -and -not $cards.Contains("KM:") -and -not $cards.Contains("verifierName=")) "raw evidence and verifier fields are absent"
|
||
Add-SyntheticCheck "telegram:caption_bounded" ($providerCaption.Length -lt 900 -and $providerCaption.Contains("下一次更新：")) "visual caption is concise and actionable"
|
||
Add-SyntheticCheck "telegram:no_raw_python" (-not $cards.Contains("urllib.request")) "relay source is absent from user-facing card"
|
||
} else {
|
||
Add-SyntheticCheck "telegram:formatter" $false "Format-AgentTelegramText not found"
|
||
}
|
||
|
||
if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) {
|
||
$fixtureEpoch = [DateTimeOffset]::Now.ToUnixTimeSeconds()
|
||
$fixtureConfig = [pscustomobject]@{
|
||
maxArtifactAgeSeconds = 600
|
||
artifactClockSkewSeconds = 90
|
||
requiredHostAliases = @("99", "110", "111", "112", "120", "121", "188")
|
||
}
|
||
$fixtureLines = @(
|
||
"RECOVERY_READBACK=1",
|
||
"RECOVERY_ARTIFACT_DIR=/home/wooo/reboot-recovery/reboot-auto-recovery-slo-selftest",
|
||
"RECOVERY_ARTIFACT_EPOCH=$fixtureEpoch",
|
||
"POST_START_RESULT=GREEN_WITH_EVIDENCE_WARNINGS",
|
||
"POST_START_BLOCKED=0",
|
||
"SERVICE_GREEN=1",
|
||
"PRODUCT_DATA_GREEN=1",
|
||
"STOCK_FRESHNESS_STATUS=ok",
|
||
"BACKUP_CORE_GREEN=1",
|
||
"HOST_188_SERVICE_GREEN=1",
|
||
"HOST_188_HYGIENE_BLOCKED=0",
|
||
"EDGE_FALLBACK_READY=1",
|
||
"VMWARE_AUTOSTART_VERIFY_READY=1",
|
||
"WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1",
|
||
"SCORECARD_STATUS=ready_reboot_auto_recovery_slo",
|
||
"SCORECARD_BLOCKER_COUNT=0",
|
||
"SCORECARD_READINESS_PERCENT=100",
|
||
"SCORECARD_PRIMARY_BLOCKER=",
|
||
"SCORECARD_CAN_CLAIM=1",
|
||
"SCORECARD_FRESH_REBOOT_WINDOW=1",
|
||
"SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0",
|
||
"SCORECARD_SOURCE_CONTROLS_READY=1",
|
||
"SCORECARD_PUBLIC_FALLBACK_READY=1",
|
||
"HOST_BOOT alias=99 target=192.168.0.99 startup_unit=vmware-host-autostart reachable=1 boot_id=windows uptime_seconds=30 systemd_state=windows_ssh startup_enabled=unknown startup_active=unknown",
|
||
"HOST_BOOT alias=110 target=wooo@192.168.0.110 startup_unit=awoooi-startup-110.service reachable=1 boot_id=linux110 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active",
|
||
"HOST_BOOT alias=111 target=ooo@192.168.0.111 startup_unit=com.momo.ollama111-allow-proxy reachable=1 boot_id=darwin uptime_seconds=40 systemd_state=darwin_ssh startup_enabled=enabled startup_active=active",
|
||
"HOST_BOOT alias=112 target=192.168.0.112 startup_unit=vm-host-boot reachable=1 boot_id=linux112 uptime_seconds=35 systemd_state=node_exporter startup_enabled=unknown startup_active=unknown",
|
||
"HOST_BOOT alias=120 target=wooo@192.168.0.120 startup_unit=k3s.service reachable=1 boot_id=linux120 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active",
|
||
"HOST_BOOT alias=121 target=wooo@192.168.0.121 startup_unit=k3s.service reachable=1 boot_id=linux121 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active",
|
||
"HOST_BOOT alias=188 target=ollama@192.168.0.188 startup_unit=awoooi-startup.service reachable=1 boot_id=linux188 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=inactive"
|
||
) -join "`n"
|
||
$freshReadback = Convert-AgentRecoveryReadback $fixtureLines (Get-Date).AddSeconds(-30) $true $fixtureConfig
|
||
Add-SyntheticCheck "recovery:coordinator_fresh_fixture" ($freshReadback.verified -and $freshReadback.rebootSloClaimed -and @($freshReadback.hosts).Count -eq 7) "fresh seven-host scorecard is verified"
|
||
|
||
$blockedFixture = $fixtureLines.Replace("SCORECARD_CAN_CLAIM=1", "SCORECARD_CAN_CLAIM=0").Replace("SCORECARD_BLOCKER_COUNT=0", "SCORECARD_BLOCKER_COUNT=3")
|
||
$blockedReadback = Convert-AgentRecoveryReadback $blockedFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig
|
||
Add-SyntheticCheck "recovery:coordinator_false_green_guard" (-not $blockedReadback.verified -and -not $blockedReadback.rebootSloClaimed -and @($blockedReadback.failedChecks).Count -ge 1) "non-SLO blockers fail closed even when service evidence is otherwise green"
|
||
|
||
$lateFixture = $blockedFixture.Replace("SCORECARD_FRESH_REBOOT_WINDOW=1", "SCORECARD_FRESH_REBOOT_WINDOW=0").Replace("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0", "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=1")
|
||
$lateReadback = Convert-AgentRecoveryReadback $lateFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig
|
||
Add-SyntheticCheck "recovery:coordinator_late_recovery_terminal" ($lateReadback.verified -and -not $lateReadback.rebootSloClaimed -and @($lateReadback.failedChecks).Count -eq 0 -and @($lateReadback.sloFailedChecks).Count -ge 2) "service recovery can close late without rewriting the reboot SLO breach"
|
||
} else {
|
||
Add-SyntheticCheck "recovery:coordinator_parser" $false "Convert-AgentRecoveryReadback not found"
|
||
}
|
||
|
||
if ($formatFunctions.ContainsKey("Convert-AgentAwoooDeploymentReadback")) {
|
||
$greenDeployments = @(
|
||
"awoooi-api 2 2 2 2",
|
||
"awoooi-web 2 2 2 2",
|
||
"awoooi-worker 1 1 1 1"
|
||
) -join "`n"
|
||
$greenPods = @(
|
||
"awoooi-api-a Running <none>",
|
||
"awoooi-api-b Running <none>",
|
||
"awoooi-web-a Running <none>",
|
||
"awoooi-worker-a Running <none>"
|
||
) -join "`n"
|
||
$greenWorkloads = Convert-AgentAwoooDeploymentReadback $greenDeployments $greenPods 0 0
|
||
Add-SyntheticCheck "recovery:awoooi_dynamic_replica_readback" (
|
||
$greenWorkloads.deployReady -and
|
||
@($greenWorkloads.deployments | Where-Object { $_.name -eq "awoooi-api" -and $_.desired -eq 2 -and $_.ready -eq 2 }).Count -eq 1 -and
|
||
-not $greenWorkloads.rawDeploymentSpecRead -and
|
||
-not $greenWorkloads.secretValueRead
|
||
) "dynamic desired/updated/ready/available counts close without raw deployment specs"
|
||
|
||
$partialWorkloads = Convert-AgentAwoooDeploymentReadback ($greenDeployments.Replace("awoooi-api 2 2 2 2", "awoooi-api 2 2 1 1")) $greenPods 0 0
|
||
Add-SyntheticCheck "recovery:awoooi_partial_rollout_fails_closed" (
|
||
-not $partialWorkloads.deployReady -and
|
||
@($partialWorkloads.notReadyDeployments) -contains "awoooi-api"
|
||
) "one unavailable replica remains a named blocker"
|
||
} else {
|
||
Add-SyntheticCheck "recovery:awoooi_deployment_parser" $false "Convert-AgentAwoooDeploymentReadback not found"
|
||
}
|
||
|
||
if (
|
||
$formatFunctions.ContainsKey("Get-HostSshUser") -and
|
||
$formatFunctions.ContainsKey("Get-AgentSshTransportConfig") -and
|
||
$formatFunctions.ContainsKey("Get-AgentHost112TransportContract") -and
|
||
$formatFunctions.ContainsKey("Get-AgentHost112TimeoutContract") -and
|
||
$formatFunctions.ContainsKey("Convert-AgentHost112GuestReadback") -and
|
||
$formatFunctions.ContainsKey("Invoke-AgentHost112GuestRecovery")
|
||
) {
|
||
$Config = [pscustomobject]@{
|
||
agentRoot = $WorkRoot
|
||
sshUser = "wooo"
|
||
sshUsers = [pscustomobject]@{ "192.168.0.112" = "kali" }
|
||
sshTransport = [pscustomobject]@{ serialized = $true; lockTimeoutSeconds = 90; lockPollMilliseconds = 200 }
|
||
k3s = [pscustomobject]@{ jumpHost = "192.168.0.110"; preferJumpHost = $true; directHosts = @("192.168.0.110", "192.168.0.112") }
|
||
}
|
||
$SshUser = "wooo"
|
||
$SshIdentityFile = Join-Path $WorkRoot "keys\agent99_ed25519"
|
||
$ControlledApply = $true
|
||
$script:Agent99DispatchClientDeadlineSeconds = 1800
|
||
$script:Agent99QueueOuterDeadlineSeconds = 1680
|
||
$script:Agent99Host112CheckTimeoutSeconds = 30
|
||
$script:Agent99Host112DryRunTimeoutSeconds = 30
|
||
$script:Agent99Host112ExecutorBudgetSeconds = 420
|
||
$script:Agent99Host112ApplyTransportTimeoutSeconds = 510
|
||
$script:Agent99Host112ReceiptTimeoutSeconds = 30
|
||
$script:Agent99Host112AfterTimeoutSeconds = 30
|
||
$script:Agent99Host112SshCallCount = 5
|
||
$script:Host112SyntheticCall = 0
|
||
$script:Host112SyntheticScenario = "general_only"
|
||
|
||
function Write-AgentLog { param([string]$Message) }
|
||
function Record-AgentEvent {
|
||
param([string]$Type, [string]$Severity, [string]$Message, [object]$Data, [switch]$Alert)
|
||
}
|
||
function New-Host112SyntheticRow {
|
||
param(
|
||
[string]$Mode,
|
||
[int]$ConsoleReady,
|
||
[int]$RuntimeWrite,
|
||
[string]$Actions,
|
||
[int]$ReceiptWrite,
|
||
[string]$ReceiptStatus,
|
||
[int]$DurablePresent,
|
||
[string]$DurableTerminal,
|
||
[int]$ExecutorExit
|
||
)
|
||
$guestReady = $ConsoleReady
|
||
$receiptRef = if ($DurablePresent -eq 1) { "host112-manager-recovery:fixture" } else { "none" }
|
||
$receiptPath = if ($DurablePresent -eq 1) { "/var/lib/awoooi-host112-recovery/receipts/fixture.txt" } else { "none" }
|
||
$receiptSha = if ($DurablePresent -eq 1) { "a" * 64 } else { "none" }
|
||
$durableReadbackStatus = if ($DurablePresent -ne 1) { "not_requested" } elseif ($Mode -eq "apply") { "written_immutable" } else { "loaded_immutable_verified" }
|
||
$artifactTransactionStatus = if ($Mode -eq "apply" -and $RuntimeWrite -eq 1) { "committed_verified_pair" } elseif ($DurablePresent -eq 1) { "existing_pair_verified" } else { "none" }
|
||
$artifactTransactionPhase = if ($DurablePresent -eq 1) { "independent_pair_verifier_passed" } else { "no_prior_transaction" }
|
||
$artifactTransactionStarted = if ($Mode -eq "apply" -and $RuntimeWrite -eq 1) { 1 } else { 0 }
|
||
$artifactTransactionWritePerformed = $artifactTransactionStarted
|
||
$artifactTransactionPairVerified = $DurablePresent
|
||
$artifactTransactionPriorPairVerified = if ($DurablePresent -eq 1 -and $Mode -ne "apply") { 1 } else { 0 }
|
||
$preflight = "already_verified_healthy"
|
||
"schema_version=host112_guest_recovery_v2 mode=$Mode trace_id=trace:fixture run_id=run:fixture work_item_id=HOST112-FIXTURE boot_id=boot-fixture uptime_seconds=999 systemd_state=running default_target=graphical.target graphical_target=active display_manager=active lightdm=active vmware_tools=active xorg=active wazuh_indexer=active wazuh_manager=active wazuh_manager_result=success wazuh_analysisd_process_count=1 wazuh_dashboard=active filebeat=active ssh_service=active ssh_enabled=enabled ssh_port_listening=1 ssh_ready=1 console_ready=$ConsoleReady services_ready=1 recovery_timer_ready=1 guest_ready=$guestReady manager_preflight_status=$preflight manager_resource_ready=1 wazuh_agent_event_port_1514_ready=1 wazuh_agent_enrollment_port_1515_tcp=1 wazuh_manager_api_port_55000_tcp=1 manager_independent_verifier_ready=1 manager_attempt_count=0 manager_start_exit=not_attempted executor_deadline_seconds=420 runtime_write_performed=$RuntimeWrite state_write_performed=0 manager_marker_write_performed=0 manager_runtime_write_performed=0 artifact_transaction_status=$artifactTransactionStatus artifact_transaction_phase=$artifactTransactionPhase artifact_transaction_started=$artifactTransactionStarted artifact_transaction_blocked=0 artifact_transaction_write_performed=$artifactTransactionWritePerformed artifact_transaction_pair_verified=$artifactTransactionPairVerified artifact_transaction_prior_pair_verified=$artifactTransactionPriorPairVerified receipt_write_performed=$ReceiptWrite receipt_status=$ReceiptStatus receipt_ref=$receiptRef receipt_path=$receiptPath receipt_sha256=$receiptSha receipt_reused=0 durable_receipt_present=$DurablePresent durable_receipt_identity_match=$DurablePresent durable_receipt_boot_match=$DurablePresent durable_receipt_terminal=$DurableTerminal durable_readback_present=$DurablePresent durable_readback_identity_match=$DurablePresent durable_readback_boot_match=$DurablePresent durable_readback_receipt_link_match=$DurablePresent durable_readback_terminal_match=$DurablePresent durable_readback_status=$durableReadbackStatus durable_readback_path=$receiptPath.readback durable_readback_sha256=$receiptSha rollback_attempted=0 rollback_verified=0 rollback_terminal=not_required action_count=1 actions=$Actions executor_exit_code=$ExecutorExit"
|
||
}
|
||
function Invoke-AgentHost112SshText {
|
||
param([string]$Command, [int]$TimeoutSeconds, [int]$Retries = 1)
|
||
$script:Host112SyntheticCall += 1
|
||
$row = if ($script:Host112SyntheticScenario -eq "failed_receipt_external_recovery") {
|
||
New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "rolled_back_verifier_failed" 0
|
||
} else {
|
||
switch ($script:Host112SyntheticCall) {
|
||
1 { New-Host112SyntheticRow "check" 0 0 "none" 0 "not_requested" 0 "unknown" 1 }
|
||
2 { New-Host112SyntheticRow "dry_run" 0 0 "none" 0 "not_requested" 0 "unknown" 0 }
|
||
3 { New-Host112SyntheticRow "apply" 1 1 "start_lightdm" 1 "written_immutable" 1 "verified_healthy" 0 }
|
||
4 { New-Host112SyntheticRow "dry_run" 1 0 "none" 0 "not_requested" 1 "verified_healthy" 0 }
|
||
default { New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "verified_healthy" 0 }
|
||
}
|
||
}
|
||
[pscustomobject]@{
|
||
ok = $true
|
||
exitCode = 0
|
||
output = $row
|
||
elapsedMs = 1
|
||
transportSerialized = $true
|
||
transportLockWaitMs = 0
|
||
route = "direct_host112_kali"
|
||
}
|
||
}
|
||
|
||
$generalOnly = Invoke-AgentHost112GuestRecovery `
|
||
-RequestedRunId "run:fixture" `
|
||
-RequestedTraceId "trace:fixture" `
|
||
-RequestedWorkItemId "HOST112-FIXTURE"
|
||
Add-SyntheticCheck "recovery:host112_general_candidate_not_blocked_by_healthy_manager" (
|
||
$generalOnly.applyAttempted -and
|
||
$generalOnly.generalCandidateRequired -and
|
||
$generalOnly.generalApplyVerified -and
|
||
-not $generalOnly.managerCandidateRequired -and
|
||
-not $generalOnly.applyReceipt.managerRuntimeWritePerformed -and
|
||
[string]$generalOnly.applyReceipt.managerAttemptCount -eq "0" -and
|
||
$generalOnly.durableReceiptVerified -and
|
||
$generalOnly.verified
|
||
) "manager healthy plus console degraded executes only general repair and closes from immutable receipt plus second verifier"
|
||
|
||
$script:Host112SyntheticCall = 0
|
||
$script:Host112SyntheticScenario = "failed_receipt_external_recovery"
|
||
$failedReplay = Invoke-AgentHost112GuestRecovery `
|
||
-RequestedRunId "run:fixture" `
|
||
-RequestedTraceId "trace:fixture" `
|
||
-RequestedWorkItemId "HOST112-FIXTURE"
|
||
Add-SyntheticCheck "recovery:host112_failed_apply_cannot_be_washed_green" (
|
||
-not $failedReplay.applyAttempted -and
|
||
-not $failedReplay.verified -and
|
||
$failedReplay.managerVerified -and
|
||
$failedReplay.before.durableReceiptTerminal -eq "rolled_back_verifier_failed" -and
|
||
$failedReplay.terminal -eq "no_write_immutable_receipt_replay_not_successful"
|
||
) "a later healthy manager cannot replace a failed immutable receipt for the same run"
|
||
} else {
|
||
Add-SyntheticCheck "recovery:host112_general_candidate_fixture" $false "Host112 control functions not found"
|
||
}
|
||
|
||
$control = $controlSource
|
||
Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_already_queued") -and $control.Contains("recovery_trigger_cooldown") -and $control.Contains('mode = "Recover"')) "automatic recovery is queued and deduplicated"
|
||
Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence"
|
||
Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("recovered_late")) "boot recovery remains durable through first-run termination and late closure"
|
||
Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present"
|
||
Add-SyntheticCheck "recovery:host112_apply_receipt_closure" ($control.Contains("managerApplyVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('artifactTransactionStatus -eq "committed_verified_pair"') -and $control.Contains('artifactTransactionPriorPairVerified') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains("durableReceiptVerified") -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply cannot be closed without its fixed-WAL commit, attributed start and immutable receipt/readback"
|
||
Add-SyntheticCheck "recovery:host112_candidate_split" ($control.Contains("generalCandidateRequired") -and $control.Contains("managerCandidateRequired") -and $control.Contains("generalApplyEligible") -and $control.Contains("managerApplyEligible") -and $control.Contains("managerNoAttemptPreserved")) "Host112 general guest repair is independent from manager eligibility"
|
||
Add-SyntheticCheck "recovery:host112_timeout_contract" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120") -and $control.Contains('$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)')) "Host112 executor, SSH lock, queue and client deadlines have explicit reserves"
|
||
$deployer = Get-Content (Join-Path $SourceRoot "agent99-deploy.ps1") -Raw
|
||
$hostConfig = Get-Content (Join-Path $SourceRoot "agent99.config.99.example.json") -Raw
|
||
Add-SyntheticCheck "recovery:host112_canonical_vmx" ($deployer.Contains('name = "host112_canonical_vmx_path"') -and $deployer.Contains('Get-AgentHost112CanonicalVm $policyConfigPath') -and $deployer.Contains('Set-AgentProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))') -and $hostConfig.Contains('S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx')) "Agent99 normalizes Host112 from one staged VMX identity with a bounded post-verifier"
|
||
|
||
$failures = @($checks | Where-Object { -not $_.ok })
|
||
$result = [pscustomobject]@{
|
||
timestamp = (Get-Date -Format o)
|
||
ok = [bool]($failures.Count -eq 0)
|
||
workRoot = $WorkRoot
|
||
checkCount = $checks.Count
|
||
failureCount = $failures.Count
|
||
checks = $checks
|
||
failures = $failures
|
||
}
|
||
$result | ConvertTo-Json -Depth 8
|
||
if (-not $result.ok) { exit 1 }
|
||
exit 0
|