fix(agent99): bind host112 apply to immutable receipt

This commit is contained in:
ogt
2026-07-14 11:54:05 +08:00
parent 316605df87
commit d7a17e58b2
8 changed files with 766 additions and 61 deletions

View File

@@ -41,6 +41,46 @@ function Copy-AgentFile {
}
}
function Set-AgentBootstrapProperty {
param([object]$Object, [string]$Name, [object]$Value)
if ($Object.PSObject.Properties[$Name]) {
$Object.PSObject.Properties[$Name].Value = $Value
} else {
$Object | Add-Member -MemberType NoteProperty -Name $Name -Value $Value
}
}
function Ensure-AgentHost112CanonicalSshConfig {
param([string]$Path)
$config = Get-Content $Path -Raw | ConvertFrom-Json
if (-not $config.PSObject.Properties["sshIdentityFile"] -or -not $config.sshIdentityFile) {
Set-AgentBootstrapProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519")
}
if (-not $config.PSObject.Properties["sshUsers"] -or $null -eq $config.sshUsers) {
Set-AgentBootstrapProperty $config "sshUsers" ([pscustomobject]@{})
}
Set-AgentBootstrapProperty $config.sshUsers "192.168.0.112" "kali"
if (-not $config.PSObject.Properties["k3s"] -or $null -eq $config.k3s) {
Set-AgentBootstrapProperty $config "k3s" ([pscustomobject]@{})
}
if (-not $config.k3s.PSObject.Properties["jumpHost"] -or -not $config.k3s.jumpHost) {
Set-AgentBootstrapProperty $config.k3s "jumpHost" "192.168.0.110"
}
if (-not $config.k3s.PSObject.Properties["preferJumpHost"]) {
Set-AgentBootstrapProperty $config.k3s "preferJumpHost" $true
}
$directHosts = if ($config.k3s.PSObject.Properties["directHosts"]) { @($config.k3s.directHosts | ForEach-Object { [string]$_ } | Where-Object { $_ }) } else { @() }
Set-AgentBootstrapProperty $config.k3s "directHosts" @($directHosts + @("192.168.0.110", "192.168.0.112") | Select-Object -Unique)
$config | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $Path
$persisted = Get-Content $Path -Raw | ConvertFrom-Json
$persistedUser = if ($persisted.sshUsers -and $persisted.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$persisted.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" }
$persistedDirectHosts = if ($persisted.k3s -and $persisted.k3s.PSObject.Properties["directHosts"]) { @($persisted.k3s.directHosts | ForEach-Object { [string]$_ }) } else { @() }
if ($persistedUser -ne "kali" -or "192.168.0.112" -notin $persistedDirectHosts) {
throw "Host112 canonical direct SSH config bootstrap verifier failed."
}
}
$agentFiles = @(
"agent99-bootstrap.ps1",
"agent99-contract-check.ps1",
@@ -122,10 +162,17 @@ if (-not (Test-Path $configPath)) {
"agentName": "Agent99",
"agentRoot": "$($AgentRoot.Replace('\', '\\'))",
"sshUser": "wooo",
"sshUsers": {},
"sshIdentityFile": "$($AgentRoot.Replace('\', '\\'))\\keys\\agent99_ed25519",
"sshUsers": { "192.168.0.112": "kali" },
"sshTransport": { "serialized": true, "lockTimeoutSeconds": 90, "lockPollMilliseconds": 200 },
"vmrunPath": "",
"hosts": ["192.168.0.110", "192.168.0.112", "192.168.0.120", "192.168.0.121", "192.168.0.188"],
"vms": [],
"k3s": {
"jumpHost": "192.168.0.110",
"preferJumpHost": true,
"directHosts": ["192.168.0.110", "192.168.0.112"]
},
"publicUrls": ["awoooi.wooo.work", "stock.wooo.work", "2026fifa.wooo.work", "gitea.wooo.work", "harbor.wooo.work"],
"sreAlertRelay": {
"enabled": true,
@@ -182,6 +229,8 @@ if (-not (Test-Path $configPath)) {
}
}
Ensure-AgentHost112CanonicalSshConfig $configPath
$launcher = Join-Path $AgentRoot "agent99-run.ps1"
@"
param(

View File

@@ -75,6 +75,7 @@ $control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane.
$inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8)
$telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8)
$bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8)
$deployer = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-deploy.ps1"), [Text.Encoding]::UTF8)
$taskRegistration = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-register-tasks.ps1"), [Text.Encoding]::UTF8)
Add-Check "transport:dedicated_identity" ($control.Contains("sshIdentityFile") -and $control.Contains("IdentitiesOnly=yes")) "dedicated SSH identity is enforced"
@@ -86,6 +87,9 @@ Add-Check "recovery:durable_boot_event" ($control.Contains("pendingRecovery") -a
Add-Check "recovery:terminal_evidence_budget" ($taskRegistration.Contains('$recoverySettings') -and $taskRegistration.Contains('New-TimeSpan -Minutes 20')) "startup recovery outlives the ten-minute SLO long enough to write terminal evidence"
Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present"
Add-Check "recovery:full_sop_coordinator" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_scorecard") -and $control.Contains("SCORECARD_FRESH_REBOOT_WINDOW") -and $control.Contains('scope = $recoveryScope')) "Agent99 cannot close reboot recovery without the 110 full-SOP scorecard"
Add-Check "recovery:host112_immutable_apply_closure" ($control.Contains("managerApplyVerified") -and $control.Contains("durableReceiptVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply closure binds transport, immutable receipt and independent verifier"
Add-Check "recovery:host112_direct_transport" ($control.Contains("Invoke-AgentHost112SshText") -and $control.Contains('expectedUser = "kali"') -and $control.Contains('route = "direct_host112_kali"') -and $deployer.Contains('name = "host112_canonical_direct_route"') -and $bootstrap.Contains("Ensure-AgentHost112CanonicalSshConfig")) "Host112 recovery uses the canonical 99 direct key route"
Add-Check "recovery:host112_timeout_reserve" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120")) "Host112 timeout chain has executor, queue and client reserves"
Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present"
Add-Check "problem_counts:v2" ($control.Contains('schemaVersion = "agent99_problem_counts_v2"') -and $control.Contains("totalVerifiedResolved")) "verified problem counts are present"
Add-Check "callback:durable_completion" ($control.Contains('schema_version = "agent99_completion_callback_v1"') -and $control.Contains("durable_readback") -and $control.Contains("completion-callbacks\pending")) "Agent99 completion callbacks are queued until durable AWOOOI/AwoooP readback"

View File

@@ -47,6 +47,15 @@ $script:Events = @()
$script:TelegramAttempts = @()
$script:SuppressAgentAlerts = [bool]$SuppressAlerts
$script:SuppressAgentAlertReason = if ($SuppressReason) { $SuppressReason } elseif ($SuppressAlerts) { "suppress_alerts" } else { "" }
$script:Agent99DispatchClientDeadlineSeconds = 1800
$script:Agent99QueueOuterDeadlineSeconds = 1680
$script:Agent99Host112CheckTimeoutSeconds = 30
$script:Agent99Host112DryRunTimeoutSeconds = 30
$script:Agent99Host112ExecutorBudgetSeconds = 420
$script:Agent99Host112ApplyTransportTimeoutSeconds = 510
$script:Agent99Host112ReceiptTimeoutSeconds = 30
$script:Agent99Host112AfterTimeoutSeconds = 30
$script:Agent99Host112SshCallCount = 5
function Convert-AgentBool {
param([object]$Value)
@@ -139,7 +148,7 @@ function Get-AgentSshTransportConfig {
$configured = if ($Config.PSObject.Properties["sshTransport"]) { $Config.sshTransport } else { $null }
[pscustomobject]@{
serialized = [bool](-not $configured -or -not $configured.PSObject.Properties["serialized"] -or (Convert-AgentBool $configured.serialized))
lockTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["lockTimeoutSeconds"]) { [math]::Max(10, [int]$configured.lockTimeoutSeconds) } else { 90 }
lockTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["lockTimeoutSeconds"]) { [math]::Min(90, [math]::Max(10, [int]$configured.lockTimeoutSeconds)) } else { 90 }
lockPollMilliseconds = if ($configured -and $configured.PSObject.Properties["lockPollMilliseconds"]) { [math]::Min(2000, [math]::Max(50, [int]$configured.lockPollMilliseconds)) } else { 200 }
lockPath = Join-Path (Join-Path $Config.agentRoot "state") "ssh-transport.lock"
}
@@ -2326,6 +2335,84 @@ function Invoke-HostSshText {
return $viaJump
}
function Get-AgentHost112TransportContract {
$targetHost = "192.168.0.112"
$configuredUser = Get-HostSshUser $targetHost
$directHosts = if ($Config.k3s -and $Config.k3s.PSObject.Properties["directHosts"]) { @($Config.k3s.directHosts | ForEach-Object { [string]$_ }) } else { @() }
$identityConfigured = [bool]($SshIdentityFile -and (Split-Path -Leaf $SshIdentityFile) -eq "agent99_ed25519")
[pscustomobject]@{
targetHost = $targetHost
configuredUser = $configuredUser
expectedUser = "kali"
directHostConfigured = [bool]($targetHost -in $directHosts)
identityConfigured = $identityConfigured
ready = [bool]($configuredUser -eq "kali" -and $targetHost -in $directHosts -and $identityConfigured)
route = "direct_host112_kali"
}
}
function Get-AgentHost112TimeoutContract {
$sshTransport = Get-AgentSshTransportConfig
$executorReserveSeconds = $script:Agent99Host112ApplyTransportTimeoutSeconds - $script:Agent99Host112ExecutorBudgetSeconds
$host112ChainBoundSeconds = (
$script:Agent99Host112CheckTimeoutSeconds +
$script:Agent99Host112DryRunTimeoutSeconds +
$script:Agent99Host112ApplyTransportTimeoutSeconds +
$script:Agent99Host112ReceiptTimeoutSeconds +
$script:Agent99Host112AfterTimeoutSeconds +
($script:Agent99Host112SshCallCount * $sshTransport.lockTimeoutSeconds)
)
$queueReserveSeconds = $script:Agent99QueueOuterDeadlineSeconds - $host112ChainBoundSeconds
$clientReserveSeconds = $script:Agent99DispatchClientDeadlineSeconds - $script:Agent99QueueOuterDeadlineSeconds
[pscustomobject]@{
schemaVersion = "agent99_host112_timeout_contract_v1"
lockWaitPerCallSeconds = [int]$sshTransport.lockTimeoutSeconds
sshCallCount = [int]$script:Agent99Host112SshCallCount
checkTimeoutSeconds = [int]$script:Agent99Host112CheckTimeoutSeconds
dryRunTimeoutSeconds = [int]$script:Agent99Host112DryRunTimeoutSeconds
executorBudgetSeconds = [int]$script:Agent99Host112ExecutorBudgetSeconds
applyTransportTimeoutSeconds = [int]$script:Agent99Host112ApplyTransportTimeoutSeconds
executorReserveSeconds = [int]$executorReserveSeconds
receiptTimeoutSeconds = [int]$script:Agent99Host112ReceiptTimeoutSeconds
afterTimeoutSeconds = [int]$script:Agent99Host112AfterTimeoutSeconds
host112ChainBoundSeconds = [int]$host112ChainBoundSeconds
queueOuterDeadlineSeconds = [int]$script:Agent99QueueOuterDeadlineSeconds
queueReserveSeconds = [int]$queueReserveSeconds
dispatchClientDeadlineSeconds = [int]$script:Agent99DispatchClientDeadlineSeconds
clientReserveSeconds = [int]$clientReserveSeconds
valid = [bool](
$executorReserveSeconds -ge 60 -and
$queueReserveSeconds -ge 300 -and
$clientReserveSeconds -ge 120
)
}
}
function Invoke-AgentHost112SshText {
param(
[string]$Command,
[int]$TimeoutSeconds,
[int]$Retries = 1
)
$transportContract = Get-AgentHost112TransportContract
if (-not $transportContract.ready) {
return [pscustomobject]@{
ok = $false
exitCode = -5
output = "host112_canonical_direct_transport_not_ready"
elapsedMs = 0
transportSerialized = $false
transportLockWaitMs = 0
route = $transportContract.route
transportContract = $transportContract
}
}
$result = Invoke-SshText -TargetHost $transportContract.targetHost -Command $Command -TimeoutSeconds $TimeoutSeconds -Retries $Retries -User $transportContract.expectedUser
$result | Add-Member -NotePropertyName route -NotePropertyValue $transportContract.route -Force
$result | Add-Member -NotePropertyName transportContract -NotePropertyValue $transportContract -Force
$result
}
function Get-AgentRecoveryCoordinatorConfig {
$configured = if ($Config.PSObject.Properties["recoveryCoordinator"]) { $Config.recoveryCoordinator } else { $null }
[pscustomobject]@{
@@ -4995,7 +5082,7 @@ function Invoke-AgentQueuedCommands {
if ($workItemId) { $args += @("-WorkItemId", $workItemId) }
if ($suppressTelegram) { $args += @("-SuppressAlerts", "-SuppressReason", "do_not_page") }
$process = Start-Process -FilePath "powershell.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru
$completed = $process.WaitForExit(600000)
$completed = $process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)
if (-not $completed) {
$process.Kill()
$process.WaitForExit(2000) | Out-Null
@@ -5400,11 +5487,14 @@ function Convert-AgentHost112GuestReadback {
param([string]$Name)
[bool]((& $get $Name "0") -eq "1")
}
$reportedSchemaVersion = & $get "schema_version"
$schemaMatches = [bool]($reportedSchemaVersion -eq "host112_guest_recovery_v2")
$readbackPresent = [bool](
$values.ContainsKey("schema_version") -and
$schemaMatches -and
$values.ContainsKey("boot_id") -and
$values.ContainsKey("guest_ready") -and
$values.ContainsKey("manager_preflight_status")
$values.ContainsKey("manager_preflight_status") -and
$values.ContainsKey("manager_independent_verifier_ready")
)
$identityPresent = [bool](
$values.ContainsKey("trace_id") -and
@@ -5422,15 +5512,34 @@ function Convert-AgentHost112GuestReadback {
$agentEventPortReady = (& $asBool "wazuh_agent_event_port_1514_ready")
$agentEnrollmentPortReady = (& $asBool "wazuh_agent_enrollment_port_1515_tcp")
$managerApiPortReady = (& $asBool "wazuh_manager_api_port_55000_tcp")
$managerIndependentVerifierReady = (& $asBool "manager_independent_verifier_ready")
$managerVerified = [bool](
$managerServiceActive -and
$analysisdPresent -and
$agentEventPortReady -and
$agentEnrollmentPortReady -and
$managerApiPortReady
$managerApiPortReady -and
$managerIndependentVerifierReady
)
$consoleReady = (& $asBool "console_ready")
$sshReady = (& $asBool "ssh_ready")
$recoveryTimerReady = (& $asBool "recovery_timer_ready")
$generalChecks = @(
[pscustomobject]@{ name = "systemd_running"; passed = [bool]((& $get "systemd_state") -eq "running") },
[pscustomobject]@{ name = "console_ready"; passed = $consoleReady },
[pscustomobject]@{ name = "wazuh_indexer_active"; passed = [bool]((& $get "wazuh_indexer") -eq "active") },
[pscustomobject]@{ name = "wazuh_dashboard_active"; passed = [bool]((& $get "wazuh_dashboard") -eq "active") },
[pscustomobject]@{ name = "filebeat_active"; passed = [bool]((& $get "filebeat") -eq "active") },
[pscustomobject]@{ name = "ssh_ready"; passed = $sshReady },
[pscustomobject]@{ name = "recovery_timer_ready"; passed = $recoveryTimerReady }
)
$generalFailedChecks = @($generalChecks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
$generalReady = [bool]($generalFailedChecks.Count -eq 0)
$actions = & $get "actions" "none"
$generalRuntimeWritePerformed = [bool]($actions -match "(?:^|,)(?:set_default_graphical_target|start_graphical_target|start_vmware_tools|start_lightdm|enable_ssh|start_ssh|retry_wazuh_indexer|enable_recovery_timer|start_recovery_timer)(?:,|$)")
$checks = @(
[pscustomobject]@{ name = "transport_ok"; passed = [bool]($Transport -and $Transport.ok) },
[pscustomobject]@{ name = "schema_matches"; passed = $schemaMatches },
[pscustomobject]@{ name = "readback_present"; passed = $readbackPresent },
[pscustomobject]@{ name = "control_identity_present"; passed = $identityPresent },
[pscustomobject]@{ name = "control_identity_matches"; passed = $identityMatches },
@@ -5441,15 +5550,20 @@ function Convert-AgentHost112GuestReadback {
[pscustomobject]@{ name = "wazuh_agent_event_1514_ready"; passed = $agentEventPortReady },
[pscustomobject]@{ name = "wazuh_agent_enrollment_1515_tcp"; passed = $agentEnrollmentPortReady },
[pscustomobject]@{ name = "wazuh_manager_api_55000_tcp"; passed = $managerApiPortReady },
[pscustomobject]@{ name = "wazuh_manager_independent_verifier_ready"; passed = $managerIndependentVerifierReady },
[pscustomobject]@{ name = "services_ready"; passed = (& $asBool "services_ready") },
[pscustomobject]@{ name = "recovery_timer_ready"; passed = (& $asBool "recovery_timer_ready") },
[pscustomobject]@{ name = "guest_ready"; passed = (& $asBool "guest_ready") }
)
$failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
[pscustomobject]@{
schemaVersion = "agent99_host112_guest_readback_v2"
schemaVersion = "agent99_host112_guest_readback_v3"
reportedSchemaVersion = $reportedSchemaVersion
schemaMatches = $schemaMatches
readbackPresent = $readbackPresent
ready = [bool]($failedChecks.Count -eq 0)
generalReady = $generalReady
generalFailedChecks = $generalFailedChecks
identityPresent = $identityPresent
identityMatches = $identityMatches
traceId = & $get "trace_id"
@@ -5475,7 +5589,9 @@ function Convert-AgentHost112GuestReadback {
managerDiskUsedPercent = & $get "manager_disk_used_percent" "100"
managerResourceReady = (& $asBool "manager_resource_ready")
managerPreflightStatus = & $get "manager_preflight_status"
managerPreflightPresent = [bool]($values.ContainsKey("manager_preflight_status") -and ((& $get "manager_preflight_status") -ne "unknown"))
managerVerified = $managerVerified
managerIndependentVerifierReady = $managerIndependentVerifierReady
wazuhAgentEventPort1514Tcp = (& $asBool "wazuh_agent_event_port_1514_tcp")
wazuhAgentEventPort1514Udp = (& $asBool "wazuh_agent_event_port_1514_udp")
wazuhAgentEventPort1514Ready = $agentEventPortReady
@@ -5483,17 +5599,37 @@ function Convert-AgentHost112GuestReadback {
wazuhManagerApiPort55000Tcp = $managerApiPortReady
wazuhDashboard = & $get "wazuh_dashboard"
filebeat = & $get "filebeat"
consoleReady = (& $asBool "console_ready")
sshService = & $get "ssh_service"
sshEnabled = & $get "ssh_enabled"
sshPortListening = (& $asBool "ssh_port_listening")
sshReady = $sshReady
consoleReady = $consoleReady
servicesReady = (& $asBool "services_ready")
recoveryTimerReady = (& $asBool "recovery_timer_ready")
recoveryTimerReady = $recoveryTimerReady
guestReady = (& $asBool "guest_ready")
actionCount = & $get "action_count" "0"
actions = & $get "actions" "none"
actions = $actions
generalRuntimeWritePerformed = $generalRuntimeWritePerformed
executorExitCode = & $get "executor_exit_code" "unknown"
executorDeadlineSeconds = & $get "executor_deadline_seconds" "unknown"
runtimeWritePerformed = (& $asBool "runtime_write_performed")
stateWritePerformed = (& $asBool "state_write_performed")
managerMarkerWritePerformed = (& $asBool "manager_marker_write_performed")
managerRuntimeWritePerformed = (& $asBool "manager_runtime_write_performed")
receiptWritePerformed = (& $asBool "receipt_write_performed")
receiptStatus = & $get "receipt_status"
receiptRef = & $get "receipt_ref" "none"
receiptPath = & $get "receipt_path" "none"
receiptSha256 = & $get "receipt_sha256" "none"
receiptReused = (& $asBool "receipt_reused")
durableReceiptPresent = (& $asBool "durable_receipt_present")
durableReceiptIdentityMatch = (& $asBool "durable_receipt_identity_match")
durableReceiptBootMatch = (& $asBool "durable_receipt_boot_match")
durableReceiptTerminal = & $get "durable_receipt_terminal" "unknown"
durableReadbackStatus = & $get "durable_readback_status" "unknown"
durableReadbackPath = & $get "durable_readback_path" "none"
durableReadbackSha256 = & $get "durable_readback_sha256" "none"
latestReadbackPointerWritten = (& $asBool "latest_readback_pointer_written")
managerAttemptCount = & $get "manager_attempt_count" "0"
managerStartExit = & $get "manager_start_exit" "not_attempted"
rollbackAttempted = (& $asBool "rollback_attempted")
@@ -5539,7 +5675,7 @@ function Invoke-AgentHost112GuestRecovery {
if (-not $identitySafe) {
$before = Convert-AgentHost112GuestReadback $null $effectiveTraceId $effectiveRunId $effectiveWorkItemId
return [pscustomobject]@{
schemaVersion = "agent99_host112_guest_recovery_v2"
schemaVersion = "agent99_host112_guest_recovery_v3"
targetHost = $targetHost
traceId = $effectiveTraceId
runId = $effectiveRunId
@@ -5556,67 +5692,271 @@ function Invoke-AgentHost112GuestRecovery {
terminal = "no_write_control_identity_incomplete_or_unsafe"
before = $before
after = $before
dryRun = $null
applyReceipt = $null
durableReceiptReadback = $null
rollback = [pscustomobject]@{ attempted = $false; verified = $false; terminal = "not_required" }
verifier = "wazuh_manager_service_process_agent_event_1514_enrollment_1515_and_api_55000"
prohibitedActions = @("host_reboot", "vm_power_change", "vm_reset", "secret_read", "firewall_change", "database_write")
}
}
$checkCommand = "/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId"
$applyCommand = "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId"
$beforeTransport = Invoke-HostSshText $targetHost $checkCommand 30 1
$before = Convert-AgentHost112GuestReadback $beforeTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
$applyTransport = $null
$applyReceipt = $null
$after = $before
$applyAttempted = $false
$receiptVerified = $false
$applyEligible = [bool](
$before.transportOk -and
$before.identityMatches -and
-not $before.managerVerified -and
$before.managerPreflightStatus -eq "ready"
)
$applyBlockedReason = if ($before.managerVerified) {
"idempotent_already_verified_healthy"
} elseif (-not $before.transportOk) {
"host112_readback_transport_unavailable"
} elseif (-not $before.identityMatches) {
"control_identity_readback_mismatch"
} elseif ($before.managerPreflightStatus -ne "ready") {
[string]$before.managerPreflightStatus
} else {
"none"
$timeoutContract = Get-AgentHost112TimeoutContract
$transportContract = Get-AgentHost112TransportContract
if (-not $timeoutContract.valid -or -not $transportContract.ready) {
$before = Convert-AgentHost112GuestReadback $null $effectiveTraceId $effectiveRunId $effectiveWorkItemId
$contractFailure = if (-not $transportContract.ready) { "canonical_direct_transport_not_ready" } else { "timeout_contract_invalid" }
return [pscustomobject]@{
schemaVersion = "agent99_host112_guest_recovery_v3"
targetHost = $targetHost
traceId = $effectiveTraceId
runId = $effectiveRunId
workItemId = $effectiveWorkItemId
controlledApply = [bool]$ControlledApply
applyEligible = $false
applyAttempted = $false
applyBlockedReason = $contractFailure
runtimeWritePerformed = $false
changed = $false
managerVerified = $false
verified = $false
receiptVerified = $false
terminal = "no_write_$contractFailure"
before = $before
after = $before
dryRun = $null
applyReceipt = $null
durableReceiptReadback = $null
timeoutContract = $timeoutContract
transportContract = $transportContract
rollback = [pscustomobject]@{ attempted = $false; verified = $false; terminal = "not_required" }
verifier = "wazuh_manager_service_process_agent_event_1514_enrollment_1515_and_api_55000"
prohibitedActions = @("host_reboot", "vm_power_change", "vm_reset", "secret_read", "firewall_change", "database_write")
}
}
# The guest executor uses exit 1 for a completed-but-degraded readback. Keep
# that semantic result in executor_exit_code while reserving transport.ok for
# SSH/sudo/executor delivery failures.
$checkCommand = ('/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf "executor_exit_code=%s\n" "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId)
$dryRunCommand = ('sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --dry-run --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf "executor_exit_code=%s\n" "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId)
$applyCommand = ('sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf "executor_exit_code=%s\n" "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId)
$beforeTransport = Invoke-AgentHost112SshText $checkCommand $script:Agent99Host112CheckTimeoutSeconds 1
$before = Convert-AgentHost112GuestReadback $beforeTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
$beforeBootBound = [bool]($before.bootId -and $before.bootId -ne "unknown")
$beforeVerified = [bool](
$before.transportOk -and
$before.executorExitCode -in @("0", "1") -and
$before.readbackPresent -and
$before.identityMatches -and
$beforeBootBound
)
$existingReceiptTerminalAccepted = [bool](
-not $before.durableReceiptPresent -or
$before.durableReceiptTerminal -in @("verified_healthy", "idempotent_already_verified_healthy")
)
$existingReceiptBound = [bool](
-not $before.durableReceiptPresent -or
(
$before.durableReceiptIdentityMatch -and
$before.durableReceiptBootMatch -and
$before.receiptSha256 -match "^[a-fA-F0-9]{64}$"
)
)
$generalCandidateRequired = [bool]($beforeVerified -and -not $before.generalReady)
$managerCandidateRequired = [bool]($beforeVerified -and -not $before.managerVerified)
$dryRunTransport = $null
$dryRun = $null
$generalDryRunVerified = $false
$managerDryRunVerified = $false
if ($ControlledApply -and $beforeVerified -and ($generalCandidateRequired -or $managerCandidateRequired)) {
$dryRunTransport = Invoke-AgentHost112SshText $dryRunCommand $script:Agent99Host112DryRunTimeoutSeconds 1
$dryRun = Convert-AgentHost112GuestReadback $dryRunTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
$generalDryRunVerified = [bool](
$dryRun.transportOk -and
$dryRun.executorExitCode -in @("0", "1") -and
$dryRun.readbackPresent -and
$dryRun.identityMatches -and
$dryRun.bootId -eq $before.bootId -and
$dryRun.managerPreflightPresent -and
-not $dryRun.runtimeWritePerformed -and
-not $dryRun.managerRuntimeWritePerformed -and
[string]$dryRun.managerAttemptCount -eq "0" -and
-not $dryRun.receiptWritePerformed -and
-not $dryRun.durableReceiptPresent
)
$managerDryRunVerified = [bool](
$generalDryRunVerified -and
$dryRun.executorExitCode -eq "0" -and
$dryRun.managerPreflightStatus -eq "ready"
)
}
$generalApplyEligible = [bool]($generalCandidateRequired -and $generalDryRunVerified)
$managerApplyEligible = [bool]($managerCandidateRequired -and $managerDryRunVerified)
$applyEligible = [bool]($generalApplyEligible -or $managerApplyEligible)
$managerApplyBlockedReason = if (-not $managerCandidateRequired) {
"not_required"
} elseif ($managerDryRunVerified) {
"none"
} elseif ($dryRun -and $dryRun.managerPreflightPresent) {
[string]$dryRun.managerPreflightStatus
} else {
"sudo_dry_run_not_verified"
}
$applyBlockedReason = if (-not $beforeVerified) {
if (-not $before.transportOk) { "host112_readback_transport_unavailable" } else { "host112_readback_identity_or_boot_unverified" }
} elseif ($before.durableReceiptPresent -and (-not $existingReceiptBound -or -not $existingReceiptTerminalAccepted)) {
"immutable_receipt_replay_not_successful"
} elseif (-not $generalCandidateRequired -and -not $managerCandidateRequired) {
"idempotent_already_verified_healthy"
} elseif (-not $ControlledApply) {
"controlled_apply_not_requested"
} elseif ($applyEligible) {
"none"
} elseif ($managerCandidateRequired) {
$managerApplyBlockedReason
} else {
"sudo_dry_run_not_verified"
}
$applyTransport = $null
$applyReceipt = $null
$durableReceiptTransport = $null
$durableReceiptReadback = $null
$afterTransport = $null
$after = $before
$applyAttempted = $false
if ($ControlledApply -and $applyEligible) {
$applyAttempted = $true
Write-AgentLog "CONTROLLED_APPLY host112_guest_recovery traceId=$effectiveTraceId runId=$effectiveRunId workItemId=$effectiveWorkItemId preflight=$($before.managerPreflightStatus)"
$applyTransport = Invoke-HostSshText $targetHost $applyCommand 590 1
Write-AgentLog "CONTROLLED_APPLY host112_guest_recovery traceId=$effectiveTraceId runId=$effectiveRunId workItemId=$effectiveWorkItemId generalCandidate=$generalCandidateRequired managerCandidate=$managerCandidateRequired managerPreflight=$($dryRun.managerPreflightStatus)"
$applyTransport = Invoke-AgentHost112SshText $applyCommand $script:Agent99Host112ApplyTransportTimeoutSeconds 1
$applyReceipt = Convert-AgentHost112GuestReadback $applyTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
$receiptVerified = [bool](
$applyReceipt.identityMatches -and
$applyReceipt.receiptWritePerformed -and
$applyReceipt.receiptStatus -eq "written" -and
$applyReceipt.receiptRef -ne "none"
)
$afterTransport = Invoke-HostSshText $targetHost $checkCommand 30 1
# This is the independent durable-receipt channel. The correlated sudo
# dry-run reloads the immutable receipt and proves identity, boot and hash;
# it does not repeat the apply.
$durableReceiptTransport = Invoke-AgentHost112SshText $dryRunCommand $script:Agent99Host112ReceiptTimeoutSeconds 1
$durableReceiptReadback = Convert-AgentHost112GuestReadback $durableReceiptTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
# The ordinary check is intentionally last and is only a second runtime
# verifier. It can never repair or replace a failed apply receipt.
$afterTransport = Invoke-AgentHost112SshText $checkCommand $script:Agent99Host112AfterTimeoutSeconds 1
$after = Convert-AgentHost112GuestReadback $afterTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
}
$managerVerified = [bool]$after.managerVerified
$verified = [bool]($after.ready -and (-not $applyAttempted -or $receiptVerified))
$changed = [bool](
$applyReceiptCommonVerified = [bool](
$applyAttempted -and
$applyReceipt -and
$applyReceipt.managerRuntimeWritePerformed -and
$managerVerified -and
$receiptVerified
$applyTransport -and $applyTransport.ok -and
$applyReceipt -and $applyReceipt.transportOk -and
$applyReceipt.executorExitCode -in @("0", "1") -and
$applyReceipt.identityMatches -and
$applyReceipt.bootId -eq $before.bootId -and
$applyReceipt.runtimeWritePerformed -and
$applyReceipt.receiptWritePerformed -and
$applyReceipt.receiptStatus -eq "written_immutable" -and
-not $applyReceipt.receiptReused -and
$applyReceipt.receiptRef -ne "none" -and
$applyReceipt.receiptPath -ne "none" -and
$applyReceipt.receiptSha256 -match "^[a-fA-F0-9]{64}$" -and
$applyReceipt.durableReceiptPresent -and
$applyReceipt.durableReceiptIdentityMatch -and
$applyReceipt.durableReceiptBootMatch -and
$applyReceipt.durableReceiptTerminal -ne "unknown"
)
$terminal = if ($applyAttempted -and $applyReceipt) {
[string]$applyReceipt.terminal
} elseif ($before.managerVerified) {
$durableReceiptVerified = [bool](
$applyReceiptCommonVerified -and
$durableReceiptTransport -and $durableReceiptTransport.ok -and
$durableReceiptReadback -and $durableReceiptReadback.transportOk -and
$durableReceiptReadback.executorExitCode -in @("0", "1") -and
$durableReceiptReadback.identityMatches -and
$durableReceiptReadback.bootId -eq $before.bootId -and
$durableReceiptReadback.durableReceiptPresent -and
$durableReceiptReadback.durableReceiptIdentityMatch -and
$durableReceiptReadback.durableReceiptBootMatch -and
$durableReceiptReadback.receiptRef -eq $applyReceipt.receiptRef -and
$durableReceiptReadback.receiptPath -eq $applyReceipt.receiptPath -and
$durableReceiptReadback.receiptSha256 -eq $applyReceipt.receiptSha256 -and
$durableReceiptReadback.durableReceiptTerminal -eq $applyReceipt.durableReceiptTerminal -and
-not $durableReceiptReadback.runtimeWritePerformed -and
-not $durableReceiptReadback.managerRuntimeWritePerformed -and
[string]$durableReceiptReadback.managerAttemptCount -eq "0"
)
$afterGeneralVerifierVerified = [bool](
$afterTransport -and $afterTransport.ok -and
$after.transportOk -and
$after.executorExitCode -in @("0", "1") -and
$after.identityMatches -and
$after.bootId -eq $before.bootId -and
$after.generalReady
)
$afterManagerVerifierVerified = [bool](
$afterTransport -and $afterTransport.ok -and
$after.transportOk -and
$after.identityMatches -and
$after.bootId -eq $before.bootId -and
$after.managerVerified -and
$after.managerIndependentVerifierReady
)
$managerApplyVerified = [bool](
$managerApplyEligible -and
$durableReceiptVerified -and
$applyReceipt.executorExitCode -eq "0" -and
$applyReceipt.durableReceiptTerminal -eq "verified_healthy" -and
$applyReceipt.managerVerified -and
$applyReceipt.managerIndependentVerifierReady -and
$applyReceipt.managerRuntimeWritePerformed -and
[string]$applyReceipt.managerAttemptCount -eq "1" -and
-not $applyReceipt.rollbackAttempted -and
$durableReceiptReadback.executorExitCode -eq "0" -and
$durableReceiptReadback.durableReceiptTerminal -eq "verified_healthy" -and
$afterManagerVerifierVerified
)
$managerNoAttemptPreserved = [bool](
$managerCandidateRequired -or
(
$applyReceipt -and
-not $applyReceipt.managerRuntimeWritePerformed -and
[string]$applyReceipt.managerAttemptCount -eq "0"
)
)
$generalApplyVerified = [bool](
$generalApplyEligible -and
$durableReceiptVerified -and
$applyReceipt.generalRuntimeWritePerformed -and
$managerNoAttemptPreserved -and
$afterGeneralVerifierVerified
)
$receiptVerified = [bool](
$durableReceiptVerified -and
(-not $managerApplyEligible -or $managerApplyVerified)
)
$managerVerified = [bool]$after.managerVerified
$verified = if ($applyAttempted) {
[bool](
$receiptVerified -and
$after.ready -and
(-not $generalCandidateRequired -or $generalApplyVerified) -and
(-not $managerCandidateRequired -or $managerApplyVerified)
)
} else {
[bool]($beforeVerified -and $before.ready -and $existingReceiptBound -and $existingReceiptTerminalAccepted)
}
$changed = [bool]($applyAttempted -and $verified -and $applyReceipt.runtimeWritePerformed)
$reportedApplyTerminal = if ($applyReceipt) { [string]$applyReceipt.durableReceiptTerminal } else { "unknown" }
$terminal = if ($applyAttempted -and -not $applyReceiptCommonVerified) {
"apply_receipt_unverified:$reportedApplyTerminal"
} elseif ($applyAttempted -and $managerCandidateRequired -and -not $managerApplyVerified) {
"manager_apply_unverified:$reportedApplyTerminal"
} elseif ($applyAttempted -and $generalCandidateRequired -and -not $generalApplyVerified) {
"general_apply_unverified:$reportedApplyTerminal"
} elseif ($applyAttempted) {
$reportedApplyTerminal
} elseif ($before.durableReceiptPresent -and (-not $existingReceiptBound -or -not $existingReceiptTerminalAccepted)) {
"no_write_immutable_receipt_replay_not_successful"
} elseif ($before.ready) {
"idempotent_already_verified_healthy"
} elseif ($ControlledApply) {
"no_write_$applyBlockedReason"
@@ -5624,7 +5964,7 @@ function Invoke-AgentHost112GuestRecovery {
"observed_no_write"
}
$result = [pscustomobject]@{
schemaVersion = "agent99_host112_guest_recovery_v2"
schemaVersion = "agent99_host112_guest_recovery_v3"
targetHost = $targetHost
traceId = $effectiveTraceId
runId = $effectiveRunId
@@ -5633,6 +5973,13 @@ function Invoke-AgentHost112GuestRecovery {
applyEligible = $applyEligible
applyAttempted = $applyAttempted
applyBlockedReason = $applyBlockedReason
generalCandidateRequired = $generalCandidateRequired
generalApplyEligible = $generalApplyEligible
generalApplyVerified = $generalApplyVerified
managerCandidateRequired = $managerCandidateRequired
managerApplyEligible = $managerApplyEligible
managerApplyBlockedReason = $managerApplyBlockedReason
managerApplyVerified = $managerApplyVerified
applyExitCode = if ($applyTransport) { $applyTransport.exitCode } else { $null }
runtimeWritePerformed = [bool]($applyReceipt -and $applyReceipt.runtimeWritePerformed)
changed = $changed
@@ -5642,7 +5989,16 @@ function Invoke-AgentHost112GuestRecovery {
terminal = $terminal
before = $before
after = $after
dryRun = $dryRun
applyReceipt = $applyReceipt
durableReceiptReadback = $durableReceiptReadback
applyReceiptCommonVerified = $applyReceiptCommonVerified
durableReceiptVerified = $durableReceiptVerified
afterGeneralVerifierVerified = $afterGeneralVerifierVerified
afterManagerVerifierVerified = $afterManagerVerifierVerified
afterVerifierRole = "independent_second_verifier_only"
timeoutContract = $timeoutContract
transportContract = $transportContract
rollback = [pscustomobject]@{
attempted = [bool]($applyReceipt -and $applyReceipt.rollbackAttempted)
verified = [bool]($applyReceipt -and $applyReceipt.rollbackVerified)

View File

@@ -195,6 +195,16 @@ try {
$config = Get-Content $ConfigPath -Raw | ConvertFrom-Json
Add-DefaultProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519")
Add-DefaultProperty $config "sshUsers" ([pscustomobject]@{})
$previousHost112User = if ($config.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$config.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" }
if ($previousHost112User -ne "kali") {
Set-AgentProperty $config.sshUsers "192.168.0.112" "kali"
$script:ConfigMigrations += [pscustomobject]@{
name = "host112_canonical_ssh_user"
from = if ($previousHost112User) { $previousHost112User } else { "missing" }
to = "kali"
}
}
Add-DefaultProperty $config "sshTransport" ([pscustomobject]@{})
Add-DefaultProperty $config.sshTransport "serialized" $true
Add-DefaultProperty $config.sshTransport "lockTimeoutSeconds" 90
@@ -207,6 +217,15 @@ try {
Add-DefaultProperty $config.k3s "jumpHost" "192.168.0.110"
Add-DefaultProperty $config.k3s "preferJumpHost" $true
Add-DefaultProperty $config.k3s "directHosts" @("192.168.0.110", "192.168.0.112")
$directHosts = @($config.k3s.directHosts | ForEach-Object { [string]$_ } | Where-Object { $_ })
if ("192.168.0.112" -notin $directHosts) {
Set-AgentProperty $config.k3s "directHosts" @($directHosts + "192.168.0.112" | Select-Object -Unique)
$script:ConfigMigrations += [pscustomobject]@{
name = "host112_canonical_direct_route"
from = ($directHosts -join ",")
to = ((@($directHosts + "192.168.0.112") | Select-Object -Unique) -join ",")
}
}
Add-DefaultProperty $config "autoRecovery" ([pscustomobject]@{})
Add-DefaultProperty $config.autoRecovery "enabled" $true
Add-DefaultProperty $config.autoRecovery "triggerOnHostUnreachable" $true
@@ -259,6 +278,12 @@ try {
}
}
$config | ConvertTo-Json -Depth 20 | Set-Content -Path $ConfigPath -Encoding UTF8
$persistedConfig = Get-Content $ConfigPath -Raw | ConvertFrom-Json
$persistedHost112User = if ($persistedConfig.sshUsers -and $persistedConfig.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$persistedConfig.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" }
$persistedDirectHosts = if ($persistedConfig.k3s -and $persistedConfig.k3s.PSObject.Properties["directHosts"]) { @($persistedConfig.k3s.directHosts | ForEach-Object { [string]$_ }) } else { @() }
if ($persistedHost112User -ne "kali" -or "192.168.0.112" -notin $persistedDirectHosts) {
throw "Host112 canonical direct SSH config post-verifier failed."
}
$manifestRows = @()
foreach ($staged in $stagedFiles) {

View File

@@ -42,7 +42,13 @@ foreach ($functionName in @(
"Get-AgentIncidentCardModel",
"Format-AgentTelegramText",
"Format-AgentTelegramCaption",
"Convert-AgentRecoveryReadback"
"Convert-AgentRecoveryReadback",
"Get-HostSshUser",
"Get-AgentSshTransportConfig",
"Get-AgentHost112TransportContract",
"Get-AgentHost112TimeoutContract",
"Convert-AgentHost112GuestReadback",
"Invoke-AgentHost112GuestRecovery"
)) {
$definition = $ast.Find({
param($node)
@@ -139,11 +145,124 @@ if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) {
Add-SyntheticCheck "recovery:coordinator_parser" $false "Convert-AgentRecoveryReadback not found"
}
if (
$formatFunctions.ContainsKey("Get-HostSshUser") -and
$formatFunctions.ContainsKey("Get-AgentSshTransportConfig") -and
$formatFunctions.ContainsKey("Get-AgentHost112TransportContract") -and
$formatFunctions.ContainsKey("Get-AgentHost112TimeoutContract") -and
$formatFunctions.ContainsKey("Convert-AgentHost112GuestReadback") -and
$formatFunctions.ContainsKey("Invoke-AgentHost112GuestRecovery")
) {
$Config = [pscustomobject]@{
agentRoot = $WorkRoot
sshUser = "wooo"
sshUsers = [pscustomobject]@{ "192.168.0.112" = "kali" }
sshTransport = [pscustomobject]@{ serialized = $true; lockTimeoutSeconds = 90; lockPollMilliseconds = 200 }
k3s = [pscustomobject]@{ jumpHost = "192.168.0.110"; preferJumpHost = $true; directHosts = @("192.168.0.110", "192.168.0.112") }
}
$SshUser = "wooo"
$SshIdentityFile = Join-Path $WorkRoot "keys\agent99_ed25519"
$ControlledApply = $true
$script:Agent99DispatchClientDeadlineSeconds = 1800
$script:Agent99QueueOuterDeadlineSeconds = 1680
$script:Agent99Host112CheckTimeoutSeconds = 30
$script:Agent99Host112DryRunTimeoutSeconds = 30
$script:Agent99Host112ExecutorBudgetSeconds = 420
$script:Agent99Host112ApplyTransportTimeoutSeconds = 510
$script:Agent99Host112ReceiptTimeoutSeconds = 30
$script:Agent99Host112AfterTimeoutSeconds = 30
$script:Agent99Host112SshCallCount = 5
$script:Host112SyntheticCall = 0
$script:Host112SyntheticScenario = "general_only"
function Write-AgentLog { param([string]$Message) }
function Record-AgentEvent {
param([string]$Type, [string]$Severity, [string]$Message, [object]$Data, [switch]$Alert)
}
function New-Host112SyntheticRow {
param(
[string]$Mode,
[int]$ConsoleReady,
[int]$RuntimeWrite,
[string]$Actions,
[int]$ReceiptWrite,
[string]$ReceiptStatus,
[int]$DurablePresent,
[string]$DurableTerminal,
[int]$ExecutorExit
)
$guestReady = $ConsoleReady
$receiptRef = if ($DurablePresent -eq 1) { "host112-manager-recovery:fixture" } else { "none" }
$receiptPath = if ($DurablePresent -eq 1) { "/var/lib/awoooi-host112-recovery/receipts/fixture.txt" } else { "none" }
$receiptSha = if ($DurablePresent -eq 1) { "a" * 64 } else { "none" }
$preflight = "already_verified_healthy"
"schema_version=host112_guest_recovery_v2 mode=$Mode trace_id=trace:fixture run_id=run:fixture work_item_id=HOST112-FIXTURE boot_id=boot-fixture uptime_seconds=999 systemd_state=running default_target=graphical.target graphical_target=active display_manager=active lightdm=active vmware_tools=active xorg=active wazuh_indexer=active wazuh_manager=active wazuh_manager_result=success wazuh_analysisd_process_count=1 wazuh_dashboard=active filebeat=active ssh_service=active ssh_enabled=enabled ssh_port_listening=1 ssh_ready=1 console_ready=$ConsoleReady services_ready=1 recovery_timer_ready=1 guest_ready=$guestReady manager_preflight_status=$preflight manager_resource_ready=1 wazuh_agent_event_port_1514_ready=1 wazuh_agent_enrollment_port_1515_tcp=1 wazuh_manager_api_port_55000_tcp=1 manager_independent_verifier_ready=1 manager_attempt_count=0 manager_start_exit=not_attempted executor_deadline_seconds=420 runtime_write_performed=$RuntimeWrite state_write_performed=0 manager_marker_write_performed=0 manager_runtime_write_performed=0 receipt_write_performed=$ReceiptWrite receipt_status=$ReceiptStatus receipt_ref=$receiptRef receipt_path=$receiptPath receipt_sha256=$receiptSha receipt_reused=0 durable_receipt_present=$DurablePresent durable_receipt_identity_match=$DurablePresent durable_receipt_boot_match=$DurablePresent durable_receipt_terminal=$DurableTerminal rollback_attempted=0 rollback_verified=0 rollback_terminal=not_required action_count=1 actions=$Actions executor_exit_code=$ExecutorExit"
}
function Invoke-AgentHost112SshText {
param([string]$Command, [int]$TimeoutSeconds, [int]$Retries = 1)
$script:Host112SyntheticCall += 1
$row = if ($script:Host112SyntheticScenario -eq "failed_receipt_external_recovery") {
New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "rolled_back_verifier_failed" 0
} else {
switch ($script:Host112SyntheticCall) {
1 { New-Host112SyntheticRow "check" 0 0 "none" 0 "not_requested" 0 "unknown" 1 }
2 { New-Host112SyntheticRow "dry_run" 0 0 "none" 0 "not_requested" 0 "unknown" 0 }
3 { New-Host112SyntheticRow "apply" 1 1 "start_lightdm" 1 "written_immutable" 1 "idempotent_already_verified_healthy" 0 }
4 { New-Host112SyntheticRow "dry_run" 1 0 "none" 0 "not_requested" 1 "idempotent_already_verified_healthy" 0 }
default { New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "idempotent_already_verified_healthy" 0 }
}
}
[pscustomobject]@{
ok = $true
exitCode = 0
output = $row
elapsedMs = 1
transportSerialized = $true
transportLockWaitMs = 0
route = "direct_host112_kali"
}
}
$generalOnly = Invoke-AgentHost112GuestRecovery `
-RequestedRunId "run:fixture" `
-RequestedTraceId "trace:fixture" `
-RequestedWorkItemId "HOST112-FIXTURE"
Add-SyntheticCheck "recovery:host112_general_candidate_not_blocked_by_healthy_manager" (
$generalOnly.applyAttempted -and
$generalOnly.generalCandidateRequired -and
$generalOnly.generalApplyVerified -and
-not $generalOnly.managerCandidateRequired -and
-not $generalOnly.applyReceipt.managerRuntimeWritePerformed -and
[string]$generalOnly.applyReceipt.managerAttemptCount -eq "0" -and
$generalOnly.durableReceiptVerified -and
$generalOnly.verified
) "manager healthy plus console degraded executes only general repair and closes from immutable receipt plus second verifier"
$script:Host112SyntheticCall = 0
$script:Host112SyntheticScenario = "failed_receipt_external_recovery"
$failedReplay = Invoke-AgentHost112GuestRecovery `
-RequestedRunId "run:fixture" `
-RequestedTraceId "trace:fixture" `
-RequestedWorkItemId "HOST112-FIXTURE"
Add-SyntheticCheck "recovery:host112_failed_apply_cannot_be_washed_green" (
-not $failedReplay.applyAttempted -and
-not $failedReplay.verified -and
$failedReplay.managerVerified -and
$failedReplay.before.durableReceiptTerminal -eq "rolled_back_verifier_failed" -and
$failedReplay.terminal -eq "no_write_immutable_receipt_replay_not_successful"
) "a later healthy manager cannot replace a failed immutable receipt for the same run"
} else {
Add-SyntheticCheck "recovery:host112_general_candidate_fixture" $false "Host112 control functions not found"
}
$control = $controlSource
Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_already_queued") -and $control.Contains("recovery_trigger_cooldown") -and $control.Contains('mode = "Recover"')) "automatic recovery is queued and deduplicated"
Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence"
Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("recovered_late")) "boot recovery remains durable through first-run termination and late closure"
Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present"
Add-SyntheticCheck "recovery:host112_apply_receipt_closure" ($control.Contains("managerApplyVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains("durableReceiptVerified") -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply cannot be closed by a later healthy observation without its immutable receipt"
Add-SyntheticCheck "recovery:host112_candidate_split" ($control.Contains("generalCandidateRequired") -and $control.Contains("managerCandidateRequired") -and $control.Contains("generalApplyEligible") -and $control.Contains("managerApplyEligible") -and $control.Contains("managerNoAttemptPreserved")) "Host112 general guest repair is independent from manager eligibility"
Add-SyntheticCheck "recovery:host112_timeout_contract" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120") -and $control.Contains('$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)')) "Host112 executor, SSH lock, queue and client deadlines have explicit reserves"
$failures = @($checks | Where-Object { -not $_.ok })
$result = [pscustomobject]@{

View File

@@ -13,7 +13,9 @@
"staleMinutes": 15,
"hardStaleMinutes": 60
},
"sshUsers": {},
"sshUsers": {
"192.168.0.112": "kali"
},
"vmrunPath": "",
"hosts": [
"192.168.0.110",

View File

@@ -0,0 +1,53 @@
# Agent99 Host112 controlled recovery contract
Host112 recovery is a bounded, correlated controlled-apply lane. Agent99 uses
the Windows99 `agent99_ed25519` identity directly to `kali@192.168.0.112`; it
does not fall back through the 110 jump route for this lane.
## Candidate separation
- The general guest candidate covers systemd, console/LightDM, VMware Tools,
SSH, Wazuh Indexer, and the recovery timer.
- The Wazuh manager candidate is evaluated separately. A healthy manager, a
manager cooldown, or a manager resource block does not suppress a valid
general guest repair candidate.
- When the manager was already verified healthy, a general-only apply must
retain `manager_attempt_count=0` and
`manager_runtime_write_performed=0`.
## Apply closure
Before any apply, Agent99 runs the correlated `sudo --dry-run` with the same
`trace_id`, `run_id`, and `work_item_id`. The dry-run must have a working direct
transport, matching identity and boot, a readable preflight result, and no
runtime or receipt write.
A manager recovery is verified only when all of the following are true for the
same apply:
1. The direct apply transport completed and the executor returned healthy.
2. The apply emitted a new immutable receipt with the matching identity and
boot, `terminal=verified_healthy`, independent manager verifier ready,
`manager_runtime_write_performed=1`, and `manager_attempt_count=1`.
3. A second correlated `sudo --dry-run` reloads the durable receipt and returns
the exact same receipt path, SHA-256, identity, boot, and terminal.
4. A final ordinary `--check` independently verifies current guest and manager
runtime health on the same boot.
The last check is evidence only. A timer or external process that makes the
manager healthy after a failed apply cannot replace the apply transport or its
immutable receipt and therefore cannot turn that run green.
## Timeout chain
- Host executor absolute deadline: 420 seconds.
- Agent99 apply transport: 510 seconds, leaving 90 seconds for transport
teardown and receipt delivery.
- Worst-case Host112 chain, including five 90-second SSH lock waits: 1080
seconds.
- Queue child outer deadline: 1680 seconds, leaving 600 seconds after the
Host112 chain.
- Dispatch client deadline: 1800 seconds, leaving 120 seconds after the queue
child deadline.
The control plane validates these reserves before attempting Host112 recovery.

View File

@@ -235,7 +235,13 @@ def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None:
assert 'if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt "$EXECUTOR_DEADLINE_SECONDS" ]; then' in source
assert 'if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -gt "$MANAGER_VERIFY_TIMEOUT_SECONDS" ]; then' in source
assert "TimeoutStartSec=480" in service
assert "Invoke-HostSshText $targetHost $applyCommand 590 1" in control
assert "$script:Agent99Host112ExecutorBudgetSeconds = 420" in control
assert "$script:Agent99Host112ApplyTransportTimeoutSeconds = 510" in control
assert "Invoke-AgentHost112SshText $applyCommand $script:Agent99Host112ApplyTransportTimeoutSeconds 1" in control
assert "executorReserveSeconds -ge 60" in control
assert "queueReserveSeconds -ge 300" in control
assert "clientReserveSeconds -ge 120" in control
assert '$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)' in control
def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> None:
@@ -396,8 +402,8 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
assert f'[string]`${parameter} = ""' in bootstrap
assert 'reason = "controlled_dispatch_identity_unsafe"' in control
assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
assert '$before.managerPreflightStatus -eq "ready"' in function
assert 'receiptStatus -eq "written"' in function
assert '$dryRun.managerPreflightStatus -eq "ready"' in function
assert 'receiptStatus -eq "written_immutable"' in function
assert '$after.managerVerified' in function
assert 'name = "wazuh_manager_service_active"' in function
assert 'name = "wazuh_analysisd_process_present"' in function
@@ -406,3 +412,94 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
assert 'name = "wazuh_manager_api_55000_tcp"' in function
assert "host112_readback_transport_unavailable" in function
assert 'terminal = "no_write_control_identity_incomplete_or_unsafe"' in function
def test_agent99_manager_success_requires_apply_and_immutable_receipt_chain() -> None:
control = read(CONTROL)
function = control[control.index("function Invoke-AgentHost112GuestRecovery") :]
function = function[: function.index("function Test-HostReachability")]
assert "$applyTransport -and $applyTransport.ok" in function
assert '$applyReceipt.durableReceiptTerminal -eq "verified_healthy"' in function
assert "$applyReceipt.managerIndependentVerifierReady" in function
assert "$applyReceipt.managerRuntimeWritePerformed" in function
assert '[string]$applyReceipt.managerAttemptCount -eq "1"' in function
assert "$durableReceiptReadback.receiptPath -eq $applyReceipt.receiptPath" in function
assert "$durableReceiptReadback.receiptSha256 -eq $applyReceipt.receiptSha256" in function
assert "$durableReceiptReadback.durableReceiptTerminal -eq $applyReceipt.durableReceiptTerminal" in function
assert '$durableReceiptReadback.durableReceiptTerminal -eq "verified_healthy"' in function
assert "$afterManagerVerifierVerified" in function
assert 'afterVerifierRole = "independent_second_verifier_only"' in function
assert "$after.ready -and" in function
assert "$managerApplyVerified" in function
assert "existingReceiptTerminalAccepted" in function
assert "immutable_receipt_replay_not_successful" in function
assert "host112_failed_apply_cannot_be_washed_green" in read(
ROOT / "agent99-synthetic-tests.ps1"
)
def test_agent99_uses_correlated_sudo_dry_run_over_canonical_direct_key() -> None:
control = read(CONTROL)
assert "function Invoke-AgentHost112SshText" in control
assert 'expectedUser = "kali"' in control
assert 'route = "direct_host112_kali"' in control
assert "Invoke-SshText -TargetHost $transportContract.targetHost" in control
assert "--dry-run --trace-id {0} --run-id {1} --work-item-id {2}" in control
assert "Invoke-AgentHost112SshText $dryRunCommand" in control
assert "$dryRun.identityMatches" in control
assert "$dryRun.managerPreflightPresent" in control
assert "$dryRun.transportOk" in control
assert "-not $dryRun.durableReceiptPresent" in control
def test_unhealthy_executor_exit_one_is_delivered_readback_not_transport_failure() -> None:
control = read(CONTROL)
function = control[control.index("function Invoke-AgentHost112GuestRecovery") :]
function = function[: function.index("function Test-HostReachability")]
synthetic = read(ROOT / "agent99-synthetic-tests.ps1")
assert function.count('case "$executor_exit" in 0|1) exit 0') == 3
assert "$before.executorExitCode -in @(\"0\", \"1\")" in function
assert "$before.readbackPresent -and" in function
assert "$before.identityMatches -and" in function
assert '$beforeTransport = Invoke-AgentHost112SshText $checkCommand' in function
assert 'New-Host112SyntheticRow "check" 0 0 "none" 0 "not_requested" 0 "unknown" 1' in synthetic
assert "host112_general_candidate_not_blocked_by_healthy_manager" in synthetic
def test_general_guest_candidate_is_not_blocked_by_manager_candidate() -> None:
control = read(CONTROL)
function = control[control.index("function Invoke-AgentHost112GuestRecovery") :]
function = function[: function.index("function Test-HostReachability")]
synthetic = read(ROOT / "agent99-synthetic-tests.ps1")
assert "$generalCandidateRequired = [bool]($beforeVerified -and -not $before.generalReady)" in function
assert "$managerCandidateRequired = [bool]($beforeVerified -and -not $before.managerVerified)" in function
assert "$generalApplyEligible = [bool]($generalCandidateRequired -and $generalDryRunVerified)" in function
assert "$managerApplyEligible = [bool]($managerCandidateRequired -and $managerDryRunVerified)" in function
assert "$applyEligible = [bool]($generalApplyEligible -or $managerApplyEligible)" in function
assert "managerNoAttemptPreserved" in function
assert "host112_general_candidate_not_blocked_by_healthy_manager" in synthetic
assert '-not $generalOnly.managerCandidateRequired' in synthetic
assert '[string]$generalOnly.applyReceipt.managerAttemptCount -eq "0"' in synthetic
assert '-not $generalOnly.applyReceipt.managerRuntimeWritePerformed' in synthetic
def test_deploy_and_bootstrap_canonicalize_host112_direct_transport() -> None:
deploy = read(ROOT / "agent99-deploy.ps1")
bootstrap = read(BOOTSTRAP)
generic_config = read(ROOT / "agent99.config.example.json")
host_config = read(ROOT / "agent99.config.99.example.json")
assert 'Set-AgentProperty $config.sshUsers "192.168.0.112" "kali"' in deploy
assert 'name = "host112_canonical_direct_route"' in deploy
assert 'Host112 canonical direct SSH config post-verifier failed.' in deploy
assert "Ensure-AgentHost112CanonicalSshConfig" in bootstrap
assert 'Set-AgentBootstrapProperty $config.sshUsers "192.168.0.112" "kali"' in bootstrap
assert 'Host112 canonical direct SSH config bootstrap verifier failed.' in bootstrap
for config in (generic_config, host_config):
assert '"192.168.0.112": "kali"' in config
assert '"directHosts"' in config
assert '"192.168.0.112"' in config