From d7a17e58b2cfc65d9ce800d0c8d08ab680796144 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 11:54:05 +0800 Subject: [PATCH] fix(agent99): bind host112 apply to immutable receipt --- agent99-bootstrap.ps1 | 51 +- agent99-contract-check.ps1 | 4 + agent99-control-plane.ps1 | 466 +++++++++++++++--- agent99-deploy.ps1 | 25 + agent99-synthetic-tests.ps1 | 121 ++++- agent99.config.example.json | 4 +- ...99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md | 53 ++ .../test_host112_manager_recovery_contract.py | 103 +++- 8 files changed, 766 insertions(+), 61 deletions(-) create mode 100644 docs/operations/AGENT99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 33be09e6f..bc452ec22 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -41,6 +41,46 @@ function Copy-AgentFile { } } +function Set-AgentBootstrapProperty { + param([object]$Object, [string]$Name, [object]$Value) + if ($Object.PSObject.Properties[$Name]) { + $Object.PSObject.Properties[$Name].Value = $Value + } else { + $Object | Add-Member -MemberType NoteProperty -Name $Name -Value $Value + } +} + +function Ensure-AgentHost112CanonicalSshConfig { + param([string]$Path) + $config = Get-Content $Path -Raw | ConvertFrom-Json + if (-not $config.PSObject.Properties["sshIdentityFile"] -or -not $config.sshIdentityFile) { + Set-AgentBootstrapProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519") + } + if (-not $config.PSObject.Properties["sshUsers"] -or $null -eq $config.sshUsers) { + Set-AgentBootstrapProperty $config "sshUsers" ([pscustomobject]@{}) + } + Set-AgentBootstrapProperty $config.sshUsers "192.168.0.112" "kali" + if (-not $config.PSObject.Properties["k3s"] -or $null -eq $config.k3s) { + Set-AgentBootstrapProperty $config "k3s" ([pscustomobject]@{}) + } + if (-not $config.k3s.PSObject.Properties["jumpHost"] -or -not $config.k3s.jumpHost) { + Set-AgentBootstrapProperty $config.k3s "jumpHost" "192.168.0.110" + } + if (-not $config.k3s.PSObject.Properties["preferJumpHost"]) { + Set-AgentBootstrapProperty $config.k3s "preferJumpHost" $true + } + $directHosts = if ($config.k3s.PSObject.Properties["directHosts"]) { @($config.k3s.directHosts | ForEach-Object { [string]$_ } | Where-Object { $_ }) } else { @() } + Set-AgentBootstrapProperty $config.k3s "directHosts" @($directHosts + @("192.168.0.110", "192.168.0.112") | Select-Object -Unique) + $config | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $Path + + $persisted = Get-Content $Path -Raw | ConvertFrom-Json + $persistedUser = if ($persisted.sshUsers -and $persisted.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$persisted.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" } + $persistedDirectHosts = if ($persisted.k3s -and $persisted.k3s.PSObject.Properties["directHosts"]) { @($persisted.k3s.directHosts | ForEach-Object { [string]$_ }) } else { @() } + if ($persistedUser -ne "kali" -or "192.168.0.112" -notin $persistedDirectHosts) { + throw "Host112 canonical direct SSH config bootstrap verifier failed." + } +} + $agentFiles = @( "agent99-bootstrap.ps1", "agent99-contract-check.ps1", @@ -122,10 +162,17 @@ if (-not (Test-Path $configPath)) { "agentName": "Agent99", "agentRoot": "$($AgentRoot.Replace('\', '\\'))", "sshUser": "wooo", - "sshUsers": {}, + "sshIdentityFile": "$($AgentRoot.Replace('\', '\\'))\\keys\\agent99_ed25519", + "sshUsers": { "192.168.0.112": "kali" }, + "sshTransport": { "serialized": true, "lockTimeoutSeconds": 90, "lockPollMilliseconds": 200 }, "vmrunPath": "", "hosts": ["192.168.0.110", "192.168.0.112", "192.168.0.120", "192.168.0.121", "192.168.0.188"], "vms": [], + "k3s": { + "jumpHost": "192.168.0.110", + "preferJumpHost": true, + "directHosts": ["192.168.0.110", "192.168.0.112"] + }, "publicUrls": ["awoooi.wooo.work", "stock.wooo.work", "2026fifa.wooo.work", "gitea.wooo.work", "harbor.wooo.work"], "sreAlertRelay": { "enabled": true, @@ -182,6 +229,8 @@ if (-not (Test-Path $configPath)) { } } +Ensure-AgentHost112CanonicalSshConfig $configPath + $launcher = Join-Path $AgentRoot "agent99-run.ps1" @" param( diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index f63fea863..80e03acf9 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -75,6 +75,7 @@ $control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane. $inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8) $telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8) $bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8) +$deployer = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-deploy.ps1"), [Text.Encoding]::UTF8) $taskRegistration = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-register-tasks.ps1"), [Text.Encoding]::UTF8) Add-Check "transport:dedicated_identity" ($control.Contains("sshIdentityFile") -and $control.Contains("IdentitiesOnly=yes")) "dedicated SSH identity is enforced" @@ -86,6 +87,9 @@ Add-Check "recovery:durable_boot_event" ($control.Contains("pendingRecovery") -a Add-Check "recovery:terminal_evidence_budget" ($taskRegistration.Contains('$recoverySettings') -and $taskRegistration.Contains('New-TimeSpan -Minutes 20')) "startup recovery outlives the ten-minute SLO long enough to write terminal evidence" Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present" Add-Check "recovery:full_sop_coordinator" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_scorecard") -and $control.Contains("SCORECARD_FRESH_REBOOT_WINDOW") -and $control.Contains('scope = $recoveryScope')) "Agent99 cannot close reboot recovery without the 110 full-SOP scorecard" +Add-Check "recovery:host112_immutable_apply_closure" ($control.Contains("managerApplyVerified") -and $control.Contains("durableReceiptVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply closure binds transport, immutable receipt and independent verifier" +Add-Check "recovery:host112_direct_transport" ($control.Contains("Invoke-AgentHost112SshText") -and $control.Contains('expectedUser = "kali"') -and $control.Contains('route = "direct_host112_kali"') -and $deployer.Contains('name = "host112_canonical_direct_route"') -and $bootstrap.Contains("Ensure-AgentHost112CanonicalSshConfig")) "Host112 recovery uses the canonical 99 direct key route" +Add-Check "recovery:host112_timeout_reserve" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120")) "Host112 timeout chain has executor, queue and client reserves" Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present" Add-Check "problem_counts:v2" ($control.Contains('schemaVersion = "agent99_problem_counts_v2"') -and $control.Contains("totalVerifiedResolved")) "verified problem counts are present" Add-Check "callback:durable_completion" ($control.Contains('schema_version = "agent99_completion_callback_v1"') -and $control.Contains("durable_readback") -and $control.Contains("completion-callbacks\pending")) "Agent99 completion callbacks are queued until durable AWOOOI/AwoooP readback" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 9ad3fd96d..f480c3dbb 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -47,6 +47,15 @@ $script:Events = @() $script:TelegramAttempts = @() $script:SuppressAgentAlerts = [bool]$SuppressAlerts $script:SuppressAgentAlertReason = if ($SuppressReason) { $SuppressReason } elseif ($SuppressAlerts) { "suppress_alerts" } else { "" } +$script:Agent99DispatchClientDeadlineSeconds = 1800 +$script:Agent99QueueOuterDeadlineSeconds = 1680 +$script:Agent99Host112CheckTimeoutSeconds = 30 +$script:Agent99Host112DryRunTimeoutSeconds = 30 +$script:Agent99Host112ExecutorBudgetSeconds = 420 +$script:Agent99Host112ApplyTransportTimeoutSeconds = 510 +$script:Agent99Host112ReceiptTimeoutSeconds = 30 +$script:Agent99Host112AfterTimeoutSeconds = 30 +$script:Agent99Host112SshCallCount = 5 function Convert-AgentBool { param([object]$Value) @@ -139,7 +148,7 @@ function Get-AgentSshTransportConfig { $configured = if ($Config.PSObject.Properties["sshTransport"]) { $Config.sshTransport } else { $null } [pscustomobject]@{ serialized = [bool](-not $configured -or -not $configured.PSObject.Properties["serialized"] -or (Convert-AgentBool $configured.serialized)) - lockTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["lockTimeoutSeconds"]) { [math]::Max(10, [int]$configured.lockTimeoutSeconds) } else { 90 } + lockTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["lockTimeoutSeconds"]) { [math]::Min(90, [math]::Max(10, [int]$configured.lockTimeoutSeconds)) } else { 90 } lockPollMilliseconds = if ($configured -and $configured.PSObject.Properties["lockPollMilliseconds"]) { [math]::Min(2000, [math]::Max(50, [int]$configured.lockPollMilliseconds)) } else { 200 } lockPath = Join-Path (Join-Path $Config.agentRoot "state") "ssh-transport.lock" } @@ -2326,6 +2335,84 @@ function Invoke-HostSshText { return $viaJump } +function Get-AgentHost112TransportContract { + $targetHost = "192.168.0.112" + $configuredUser = Get-HostSshUser $targetHost + $directHosts = if ($Config.k3s -and $Config.k3s.PSObject.Properties["directHosts"]) { @($Config.k3s.directHosts | ForEach-Object { [string]$_ }) } else { @() } + $identityConfigured = [bool]($SshIdentityFile -and (Split-Path -Leaf $SshIdentityFile) -eq "agent99_ed25519") + [pscustomobject]@{ + targetHost = $targetHost + configuredUser = $configuredUser + expectedUser = "kali" + directHostConfigured = [bool]($targetHost -in $directHosts) + identityConfigured = $identityConfigured + ready = [bool]($configuredUser -eq "kali" -and $targetHost -in $directHosts -and $identityConfigured) + route = "direct_host112_kali" + } +} + +function Get-AgentHost112TimeoutContract { + $sshTransport = Get-AgentSshTransportConfig + $executorReserveSeconds = $script:Agent99Host112ApplyTransportTimeoutSeconds - $script:Agent99Host112ExecutorBudgetSeconds + $host112ChainBoundSeconds = ( + $script:Agent99Host112CheckTimeoutSeconds + + $script:Agent99Host112DryRunTimeoutSeconds + + $script:Agent99Host112ApplyTransportTimeoutSeconds + + $script:Agent99Host112ReceiptTimeoutSeconds + + $script:Agent99Host112AfterTimeoutSeconds + + ($script:Agent99Host112SshCallCount * $sshTransport.lockTimeoutSeconds) + ) + $queueReserveSeconds = $script:Agent99QueueOuterDeadlineSeconds - $host112ChainBoundSeconds + $clientReserveSeconds = $script:Agent99DispatchClientDeadlineSeconds - $script:Agent99QueueOuterDeadlineSeconds + [pscustomobject]@{ + schemaVersion = "agent99_host112_timeout_contract_v1" + lockWaitPerCallSeconds = [int]$sshTransport.lockTimeoutSeconds + sshCallCount = [int]$script:Agent99Host112SshCallCount + checkTimeoutSeconds = [int]$script:Agent99Host112CheckTimeoutSeconds + dryRunTimeoutSeconds = [int]$script:Agent99Host112DryRunTimeoutSeconds + executorBudgetSeconds = [int]$script:Agent99Host112ExecutorBudgetSeconds + applyTransportTimeoutSeconds = [int]$script:Agent99Host112ApplyTransportTimeoutSeconds + executorReserveSeconds = [int]$executorReserveSeconds + receiptTimeoutSeconds = [int]$script:Agent99Host112ReceiptTimeoutSeconds + afterTimeoutSeconds = [int]$script:Agent99Host112AfterTimeoutSeconds + host112ChainBoundSeconds = [int]$host112ChainBoundSeconds + queueOuterDeadlineSeconds = [int]$script:Agent99QueueOuterDeadlineSeconds + queueReserveSeconds = [int]$queueReserveSeconds + dispatchClientDeadlineSeconds = [int]$script:Agent99DispatchClientDeadlineSeconds + clientReserveSeconds = [int]$clientReserveSeconds + valid = [bool]( + $executorReserveSeconds -ge 60 -and + $queueReserveSeconds -ge 300 -and + $clientReserveSeconds -ge 120 + ) + } +} + +function Invoke-AgentHost112SshText { + param( + [string]$Command, + [int]$TimeoutSeconds, + [int]$Retries = 1 + ) + $transportContract = Get-AgentHost112TransportContract + if (-not $transportContract.ready) { + return [pscustomobject]@{ + ok = $false + exitCode = -5 + output = "host112_canonical_direct_transport_not_ready" + elapsedMs = 0 + transportSerialized = $false + transportLockWaitMs = 0 + route = $transportContract.route + transportContract = $transportContract + } + } + $result = Invoke-SshText -TargetHost $transportContract.targetHost -Command $Command -TimeoutSeconds $TimeoutSeconds -Retries $Retries -User $transportContract.expectedUser + $result | Add-Member -NotePropertyName route -NotePropertyValue $transportContract.route -Force + $result | Add-Member -NotePropertyName transportContract -NotePropertyValue $transportContract -Force + $result +} + function Get-AgentRecoveryCoordinatorConfig { $configured = if ($Config.PSObject.Properties["recoveryCoordinator"]) { $Config.recoveryCoordinator } else { $null } [pscustomobject]@{ @@ -4995,7 +5082,7 @@ function Invoke-AgentQueuedCommands { if ($workItemId) { $args += @("-WorkItemId", $workItemId) } if ($suppressTelegram) { $args += @("-SuppressAlerts", "-SuppressReason", "do_not_page") } $process = Start-Process -FilePath "powershell.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru - $completed = $process.WaitForExit(600000) + $completed = $process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000) if (-not $completed) { $process.Kill() $process.WaitForExit(2000) | Out-Null @@ -5400,11 +5487,14 @@ function Convert-AgentHost112GuestReadback { param([string]$Name) [bool]((& $get $Name "0") -eq "1") } + $reportedSchemaVersion = & $get "schema_version" + $schemaMatches = [bool]($reportedSchemaVersion -eq "host112_guest_recovery_v2") $readbackPresent = [bool]( - $values.ContainsKey("schema_version") -and + $schemaMatches -and $values.ContainsKey("boot_id") -and $values.ContainsKey("guest_ready") -and - $values.ContainsKey("manager_preflight_status") + $values.ContainsKey("manager_preflight_status") -and + $values.ContainsKey("manager_independent_verifier_ready") ) $identityPresent = [bool]( $values.ContainsKey("trace_id") -and @@ -5422,15 +5512,34 @@ function Convert-AgentHost112GuestReadback { $agentEventPortReady = (& $asBool "wazuh_agent_event_port_1514_ready") $agentEnrollmentPortReady = (& $asBool "wazuh_agent_enrollment_port_1515_tcp") $managerApiPortReady = (& $asBool "wazuh_manager_api_port_55000_tcp") + $managerIndependentVerifierReady = (& $asBool "manager_independent_verifier_ready") $managerVerified = [bool]( $managerServiceActive -and $analysisdPresent -and $agentEventPortReady -and $agentEnrollmentPortReady -and - $managerApiPortReady + $managerApiPortReady -and + $managerIndependentVerifierReady ) + $consoleReady = (& $asBool "console_ready") + $sshReady = (& $asBool "ssh_ready") + $recoveryTimerReady = (& $asBool "recovery_timer_ready") + $generalChecks = @( + [pscustomobject]@{ name = "systemd_running"; passed = [bool]((& $get "systemd_state") -eq "running") }, + [pscustomobject]@{ name = "console_ready"; passed = $consoleReady }, + [pscustomobject]@{ name = "wazuh_indexer_active"; passed = [bool]((& $get "wazuh_indexer") -eq "active") }, + [pscustomobject]@{ name = "wazuh_dashboard_active"; passed = [bool]((& $get "wazuh_dashboard") -eq "active") }, + [pscustomobject]@{ name = "filebeat_active"; passed = [bool]((& $get "filebeat") -eq "active") }, + [pscustomobject]@{ name = "ssh_ready"; passed = $sshReady }, + [pscustomobject]@{ name = "recovery_timer_ready"; passed = $recoveryTimerReady } + ) + $generalFailedChecks = @($generalChecks | Where-Object { -not $_.passed } | ForEach-Object { $_.name }) + $generalReady = [bool]($generalFailedChecks.Count -eq 0) + $actions = & $get "actions" "none" + $generalRuntimeWritePerformed = [bool]($actions -match "(?:^|,)(?:set_default_graphical_target|start_graphical_target|start_vmware_tools|start_lightdm|enable_ssh|start_ssh|retry_wazuh_indexer|enable_recovery_timer|start_recovery_timer)(?:,|$)") $checks = @( [pscustomobject]@{ name = "transport_ok"; passed = [bool]($Transport -and $Transport.ok) }, + [pscustomobject]@{ name = "schema_matches"; passed = $schemaMatches }, [pscustomobject]@{ name = "readback_present"; passed = $readbackPresent }, [pscustomobject]@{ name = "control_identity_present"; passed = $identityPresent }, [pscustomobject]@{ name = "control_identity_matches"; passed = $identityMatches }, @@ -5441,15 +5550,20 @@ function Convert-AgentHost112GuestReadback { [pscustomobject]@{ name = "wazuh_agent_event_1514_ready"; passed = $agentEventPortReady }, [pscustomobject]@{ name = "wazuh_agent_enrollment_1515_tcp"; passed = $agentEnrollmentPortReady }, [pscustomobject]@{ name = "wazuh_manager_api_55000_tcp"; passed = $managerApiPortReady }, + [pscustomobject]@{ name = "wazuh_manager_independent_verifier_ready"; passed = $managerIndependentVerifierReady }, [pscustomobject]@{ name = "services_ready"; passed = (& $asBool "services_ready") }, [pscustomobject]@{ name = "recovery_timer_ready"; passed = (& $asBool "recovery_timer_ready") }, [pscustomobject]@{ name = "guest_ready"; passed = (& $asBool "guest_ready") } ) $failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name }) [pscustomobject]@{ - schemaVersion = "agent99_host112_guest_readback_v2" + schemaVersion = "agent99_host112_guest_readback_v3" + reportedSchemaVersion = $reportedSchemaVersion + schemaMatches = $schemaMatches readbackPresent = $readbackPresent ready = [bool]($failedChecks.Count -eq 0) + generalReady = $generalReady + generalFailedChecks = $generalFailedChecks identityPresent = $identityPresent identityMatches = $identityMatches traceId = & $get "trace_id" @@ -5475,7 +5589,9 @@ function Convert-AgentHost112GuestReadback { managerDiskUsedPercent = & $get "manager_disk_used_percent" "100" managerResourceReady = (& $asBool "manager_resource_ready") managerPreflightStatus = & $get "manager_preflight_status" + managerPreflightPresent = [bool]($values.ContainsKey("manager_preflight_status") -and ((& $get "manager_preflight_status") -ne "unknown")) managerVerified = $managerVerified + managerIndependentVerifierReady = $managerIndependentVerifierReady wazuhAgentEventPort1514Tcp = (& $asBool "wazuh_agent_event_port_1514_tcp") wazuhAgentEventPort1514Udp = (& $asBool "wazuh_agent_event_port_1514_udp") wazuhAgentEventPort1514Ready = $agentEventPortReady @@ -5483,17 +5599,37 @@ function Convert-AgentHost112GuestReadback { wazuhManagerApiPort55000Tcp = $managerApiPortReady wazuhDashboard = & $get "wazuh_dashboard" filebeat = & $get "filebeat" - consoleReady = (& $asBool "console_ready") + sshService = & $get "ssh_service" + sshEnabled = & $get "ssh_enabled" + sshPortListening = (& $asBool "ssh_port_listening") + sshReady = $sshReady + consoleReady = $consoleReady servicesReady = (& $asBool "services_ready") - recoveryTimerReady = (& $asBool "recovery_timer_ready") + recoveryTimerReady = $recoveryTimerReady guestReady = (& $asBool "guest_ready") actionCount = & $get "action_count" "0" - actions = & $get "actions" "none" + actions = $actions + generalRuntimeWritePerformed = $generalRuntimeWritePerformed + executorExitCode = & $get "executor_exit_code" "unknown" + executorDeadlineSeconds = & $get "executor_deadline_seconds" "unknown" runtimeWritePerformed = (& $asBool "runtime_write_performed") + stateWritePerformed = (& $asBool "state_write_performed") + managerMarkerWritePerformed = (& $asBool "manager_marker_write_performed") managerRuntimeWritePerformed = (& $asBool "manager_runtime_write_performed") receiptWritePerformed = (& $asBool "receipt_write_performed") receiptStatus = & $get "receipt_status" receiptRef = & $get "receipt_ref" "none" + receiptPath = & $get "receipt_path" "none" + receiptSha256 = & $get "receipt_sha256" "none" + receiptReused = (& $asBool "receipt_reused") + durableReceiptPresent = (& $asBool "durable_receipt_present") + durableReceiptIdentityMatch = (& $asBool "durable_receipt_identity_match") + durableReceiptBootMatch = (& $asBool "durable_receipt_boot_match") + durableReceiptTerminal = & $get "durable_receipt_terminal" "unknown" + durableReadbackStatus = & $get "durable_readback_status" "unknown" + durableReadbackPath = & $get "durable_readback_path" "none" + durableReadbackSha256 = & $get "durable_readback_sha256" "none" + latestReadbackPointerWritten = (& $asBool "latest_readback_pointer_written") managerAttemptCount = & $get "manager_attempt_count" "0" managerStartExit = & $get "manager_start_exit" "not_attempted" rollbackAttempted = (& $asBool "rollback_attempted") @@ -5539,7 +5675,7 @@ function Invoke-AgentHost112GuestRecovery { if (-not $identitySafe) { $before = Convert-AgentHost112GuestReadback $null $effectiveTraceId $effectiveRunId $effectiveWorkItemId return [pscustomobject]@{ - schemaVersion = "agent99_host112_guest_recovery_v2" + schemaVersion = "agent99_host112_guest_recovery_v3" targetHost = $targetHost traceId = $effectiveTraceId runId = $effectiveRunId @@ -5556,67 +5692,271 @@ function Invoke-AgentHost112GuestRecovery { terminal = "no_write_control_identity_incomplete_or_unsafe" before = $before after = $before + dryRun = $null applyReceipt = $null + durableReceiptReadback = $null rollback = [pscustomobject]@{ attempted = $false; verified = $false; terminal = "not_required" } verifier = "wazuh_manager_service_process_agent_event_1514_enrollment_1515_and_api_55000" prohibitedActions = @("host_reboot", "vm_power_change", "vm_reset", "secret_read", "firewall_change", "database_write") } } - $checkCommand = "/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId" - $applyCommand = "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId" - $beforeTransport = Invoke-HostSshText $targetHost $checkCommand 30 1 - $before = Convert-AgentHost112GuestReadback $beforeTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId - $applyTransport = $null - $applyReceipt = $null - $after = $before - $applyAttempted = $false - $receiptVerified = $false - $applyEligible = [bool]( - $before.transportOk -and - $before.identityMatches -and - -not $before.managerVerified -and - $before.managerPreflightStatus -eq "ready" - ) - $applyBlockedReason = if ($before.managerVerified) { - "idempotent_already_verified_healthy" - } elseif (-not $before.transportOk) { - "host112_readback_transport_unavailable" - } elseif (-not $before.identityMatches) { - "control_identity_readback_mismatch" - } elseif ($before.managerPreflightStatus -ne "ready") { - [string]$before.managerPreflightStatus - } else { - "none" + $timeoutContract = Get-AgentHost112TimeoutContract + $transportContract = Get-AgentHost112TransportContract + if (-not $timeoutContract.valid -or -not $transportContract.ready) { + $before = Convert-AgentHost112GuestReadback $null $effectiveTraceId $effectiveRunId $effectiveWorkItemId + $contractFailure = if (-not $transportContract.ready) { "canonical_direct_transport_not_ready" } else { "timeout_contract_invalid" } + return [pscustomobject]@{ + schemaVersion = "agent99_host112_guest_recovery_v3" + targetHost = $targetHost + traceId = $effectiveTraceId + runId = $effectiveRunId + workItemId = $effectiveWorkItemId + controlledApply = [bool]$ControlledApply + applyEligible = $false + applyAttempted = $false + applyBlockedReason = $contractFailure + runtimeWritePerformed = $false + changed = $false + managerVerified = $false + verified = $false + receiptVerified = $false + terminal = "no_write_$contractFailure" + before = $before + after = $before + dryRun = $null + applyReceipt = $null + durableReceiptReadback = $null + timeoutContract = $timeoutContract + transportContract = $transportContract + rollback = [pscustomobject]@{ attempted = $false; verified = $false; terminal = "not_required" } + verifier = "wazuh_manager_service_process_agent_event_1514_enrollment_1515_and_api_55000" + prohibitedActions = @("host_reboot", "vm_power_change", "vm_reset", "secret_read", "firewall_change", "database_write") + } } + # The guest executor uses exit 1 for a completed-but-degraded readback. Keep + # that semantic result in executor_exit_code while reserving transport.ok for + # SSH/sudo/executor delivery failures. + $checkCommand = ('/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf "executor_exit_code=%s\n" "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId) + $dryRunCommand = ('sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --dry-run --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf "executor_exit_code=%s\n" "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId) + $applyCommand = ('sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf "executor_exit_code=%s\n" "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId) + + $beforeTransport = Invoke-AgentHost112SshText $checkCommand $script:Agent99Host112CheckTimeoutSeconds 1 + $before = Convert-AgentHost112GuestReadback $beforeTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId + $beforeBootBound = [bool]($before.bootId -and $before.bootId -ne "unknown") + $beforeVerified = [bool]( + $before.transportOk -and + $before.executorExitCode -in @("0", "1") -and + $before.readbackPresent -and + $before.identityMatches -and + $beforeBootBound + ) + $existingReceiptTerminalAccepted = [bool]( + -not $before.durableReceiptPresent -or + $before.durableReceiptTerminal -in @("verified_healthy", "idempotent_already_verified_healthy") + ) + $existingReceiptBound = [bool]( + -not $before.durableReceiptPresent -or + ( + $before.durableReceiptIdentityMatch -and + $before.durableReceiptBootMatch -and + $before.receiptSha256 -match "^[a-fA-F0-9]{64}$" + ) + ) + $generalCandidateRequired = [bool]($beforeVerified -and -not $before.generalReady) + $managerCandidateRequired = [bool]($beforeVerified -and -not $before.managerVerified) + + $dryRunTransport = $null + $dryRun = $null + $generalDryRunVerified = $false + $managerDryRunVerified = $false + if ($ControlledApply -and $beforeVerified -and ($generalCandidateRequired -or $managerCandidateRequired)) { + $dryRunTransport = Invoke-AgentHost112SshText $dryRunCommand $script:Agent99Host112DryRunTimeoutSeconds 1 + $dryRun = Convert-AgentHost112GuestReadback $dryRunTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId + $generalDryRunVerified = [bool]( + $dryRun.transportOk -and + $dryRun.executorExitCode -in @("0", "1") -and + $dryRun.readbackPresent -and + $dryRun.identityMatches -and + $dryRun.bootId -eq $before.bootId -and + $dryRun.managerPreflightPresent -and + -not $dryRun.runtimeWritePerformed -and + -not $dryRun.managerRuntimeWritePerformed -and + [string]$dryRun.managerAttemptCount -eq "0" -and + -not $dryRun.receiptWritePerformed -and + -not $dryRun.durableReceiptPresent + ) + $managerDryRunVerified = [bool]( + $generalDryRunVerified -and + $dryRun.executorExitCode -eq "0" -and + $dryRun.managerPreflightStatus -eq "ready" + ) + } + + $generalApplyEligible = [bool]($generalCandidateRequired -and $generalDryRunVerified) + $managerApplyEligible = [bool]($managerCandidateRequired -and $managerDryRunVerified) + $applyEligible = [bool]($generalApplyEligible -or $managerApplyEligible) + $managerApplyBlockedReason = if (-not $managerCandidateRequired) { + "not_required" + } elseif ($managerDryRunVerified) { + "none" + } elseif ($dryRun -and $dryRun.managerPreflightPresent) { + [string]$dryRun.managerPreflightStatus + } else { + "sudo_dry_run_not_verified" + } + $applyBlockedReason = if (-not $beforeVerified) { + if (-not $before.transportOk) { "host112_readback_transport_unavailable" } else { "host112_readback_identity_or_boot_unverified" } + } elseif ($before.durableReceiptPresent -and (-not $existingReceiptBound -or -not $existingReceiptTerminalAccepted)) { + "immutable_receipt_replay_not_successful" + } elseif (-not $generalCandidateRequired -and -not $managerCandidateRequired) { + "idempotent_already_verified_healthy" + } elseif (-not $ControlledApply) { + "controlled_apply_not_requested" + } elseif ($applyEligible) { + "none" + } elseif ($managerCandidateRequired) { + $managerApplyBlockedReason + } else { + "sudo_dry_run_not_verified" + } + + $applyTransport = $null + $applyReceipt = $null + $durableReceiptTransport = $null + $durableReceiptReadback = $null + $afterTransport = $null + $after = $before + $applyAttempted = $false if ($ControlledApply -and $applyEligible) { $applyAttempted = $true - Write-AgentLog "CONTROLLED_APPLY host112_guest_recovery traceId=$effectiveTraceId runId=$effectiveRunId workItemId=$effectiveWorkItemId preflight=$($before.managerPreflightStatus)" - $applyTransport = Invoke-HostSshText $targetHost $applyCommand 590 1 + Write-AgentLog "CONTROLLED_APPLY host112_guest_recovery traceId=$effectiveTraceId runId=$effectiveRunId workItemId=$effectiveWorkItemId generalCandidate=$generalCandidateRequired managerCandidate=$managerCandidateRequired managerPreflight=$($dryRun.managerPreflightStatus)" + $applyTransport = Invoke-AgentHost112SshText $applyCommand $script:Agent99Host112ApplyTransportTimeoutSeconds 1 $applyReceipt = Convert-AgentHost112GuestReadback $applyTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId - $receiptVerified = [bool]( - $applyReceipt.identityMatches -and - $applyReceipt.receiptWritePerformed -and - $applyReceipt.receiptStatus -eq "written" -and - $applyReceipt.receiptRef -ne "none" - ) - $afterTransport = Invoke-HostSshText $targetHost $checkCommand 30 1 + + # This is the independent durable-receipt channel. The correlated sudo + # dry-run reloads the immutable receipt and proves identity, boot and hash; + # it does not repeat the apply. + $durableReceiptTransport = Invoke-AgentHost112SshText $dryRunCommand $script:Agent99Host112ReceiptTimeoutSeconds 1 + $durableReceiptReadback = Convert-AgentHost112GuestReadback $durableReceiptTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId + + # The ordinary check is intentionally last and is only a second runtime + # verifier. It can never repair or replace a failed apply receipt. + $afterTransport = Invoke-AgentHost112SshText $checkCommand $script:Agent99Host112AfterTimeoutSeconds 1 $after = Convert-AgentHost112GuestReadback $afterTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId } - $managerVerified = [bool]$after.managerVerified - $verified = [bool]($after.ready -and (-not $applyAttempted -or $receiptVerified)) - $changed = [bool]( + $applyReceiptCommonVerified = [bool]( $applyAttempted -and - $applyReceipt -and - $applyReceipt.managerRuntimeWritePerformed -and - $managerVerified -and - $receiptVerified + $applyTransport -and $applyTransport.ok -and + $applyReceipt -and $applyReceipt.transportOk -and + $applyReceipt.executorExitCode -in @("0", "1") -and + $applyReceipt.identityMatches -and + $applyReceipt.bootId -eq $before.bootId -and + $applyReceipt.runtimeWritePerformed -and + $applyReceipt.receiptWritePerformed -and + $applyReceipt.receiptStatus -eq "written_immutable" -and + -not $applyReceipt.receiptReused -and + $applyReceipt.receiptRef -ne "none" -and + $applyReceipt.receiptPath -ne "none" -and + $applyReceipt.receiptSha256 -match "^[a-fA-F0-9]{64}$" -and + $applyReceipt.durableReceiptPresent -and + $applyReceipt.durableReceiptIdentityMatch -and + $applyReceipt.durableReceiptBootMatch -and + $applyReceipt.durableReceiptTerminal -ne "unknown" ) - $terminal = if ($applyAttempted -and $applyReceipt) { - [string]$applyReceipt.terminal - } elseif ($before.managerVerified) { + $durableReceiptVerified = [bool]( + $applyReceiptCommonVerified -and + $durableReceiptTransport -and $durableReceiptTransport.ok -and + $durableReceiptReadback -and $durableReceiptReadback.transportOk -and + $durableReceiptReadback.executorExitCode -in @("0", "1") -and + $durableReceiptReadback.identityMatches -and + $durableReceiptReadback.bootId -eq $before.bootId -and + $durableReceiptReadback.durableReceiptPresent -and + $durableReceiptReadback.durableReceiptIdentityMatch -and + $durableReceiptReadback.durableReceiptBootMatch -and + $durableReceiptReadback.receiptRef -eq $applyReceipt.receiptRef -and + $durableReceiptReadback.receiptPath -eq $applyReceipt.receiptPath -and + $durableReceiptReadback.receiptSha256 -eq $applyReceipt.receiptSha256 -and + $durableReceiptReadback.durableReceiptTerminal -eq $applyReceipt.durableReceiptTerminal -and + -not $durableReceiptReadback.runtimeWritePerformed -and + -not $durableReceiptReadback.managerRuntimeWritePerformed -and + [string]$durableReceiptReadback.managerAttemptCount -eq "0" + ) + $afterGeneralVerifierVerified = [bool]( + $afterTransport -and $afterTransport.ok -and + $after.transportOk -and + $after.executorExitCode -in @("0", "1") -and + $after.identityMatches -and + $after.bootId -eq $before.bootId -and + $after.generalReady + ) + $afterManagerVerifierVerified = [bool]( + $afterTransport -and $afterTransport.ok -and + $after.transportOk -and + $after.identityMatches -and + $after.bootId -eq $before.bootId -and + $after.managerVerified -and + $after.managerIndependentVerifierReady + ) + $managerApplyVerified = [bool]( + $managerApplyEligible -and + $durableReceiptVerified -and + $applyReceipt.executorExitCode -eq "0" -and + $applyReceipt.durableReceiptTerminal -eq "verified_healthy" -and + $applyReceipt.managerVerified -and + $applyReceipt.managerIndependentVerifierReady -and + $applyReceipt.managerRuntimeWritePerformed -and + [string]$applyReceipt.managerAttemptCount -eq "1" -and + -not $applyReceipt.rollbackAttempted -and + $durableReceiptReadback.executorExitCode -eq "0" -and + $durableReceiptReadback.durableReceiptTerminal -eq "verified_healthy" -and + $afterManagerVerifierVerified + ) + $managerNoAttemptPreserved = [bool]( + $managerCandidateRequired -or + ( + $applyReceipt -and + -not $applyReceipt.managerRuntimeWritePerformed -and + [string]$applyReceipt.managerAttemptCount -eq "0" + ) + ) + $generalApplyVerified = [bool]( + $generalApplyEligible -and + $durableReceiptVerified -and + $applyReceipt.generalRuntimeWritePerformed -and + $managerNoAttemptPreserved -and + $afterGeneralVerifierVerified + ) + $receiptVerified = [bool]( + $durableReceiptVerified -and + (-not $managerApplyEligible -or $managerApplyVerified) + ) + $managerVerified = [bool]$after.managerVerified + $verified = if ($applyAttempted) { + [bool]( + $receiptVerified -and + $after.ready -and + (-not $generalCandidateRequired -or $generalApplyVerified) -and + (-not $managerCandidateRequired -or $managerApplyVerified) + ) + } else { + [bool]($beforeVerified -and $before.ready -and $existingReceiptBound -and $existingReceiptTerminalAccepted) + } + $changed = [bool]($applyAttempted -and $verified -and $applyReceipt.runtimeWritePerformed) + $reportedApplyTerminal = if ($applyReceipt) { [string]$applyReceipt.durableReceiptTerminal } else { "unknown" } + $terminal = if ($applyAttempted -and -not $applyReceiptCommonVerified) { + "apply_receipt_unverified:$reportedApplyTerminal" + } elseif ($applyAttempted -and $managerCandidateRequired -and -not $managerApplyVerified) { + "manager_apply_unverified:$reportedApplyTerminal" + } elseif ($applyAttempted -and $generalCandidateRequired -and -not $generalApplyVerified) { + "general_apply_unverified:$reportedApplyTerminal" + } elseif ($applyAttempted) { + $reportedApplyTerminal + } elseif ($before.durableReceiptPresent -and (-not $existingReceiptBound -or -not $existingReceiptTerminalAccepted)) { + "no_write_immutable_receipt_replay_not_successful" + } elseif ($before.ready) { "idempotent_already_verified_healthy" } elseif ($ControlledApply) { "no_write_$applyBlockedReason" @@ -5624,7 +5964,7 @@ function Invoke-AgentHost112GuestRecovery { "observed_no_write" } $result = [pscustomobject]@{ - schemaVersion = "agent99_host112_guest_recovery_v2" + schemaVersion = "agent99_host112_guest_recovery_v3" targetHost = $targetHost traceId = $effectiveTraceId runId = $effectiveRunId @@ -5633,6 +5973,13 @@ function Invoke-AgentHost112GuestRecovery { applyEligible = $applyEligible applyAttempted = $applyAttempted applyBlockedReason = $applyBlockedReason + generalCandidateRequired = $generalCandidateRequired + generalApplyEligible = $generalApplyEligible + generalApplyVerified = $generalApplyVerified + managerCandidateRequired = $managerCandidateRequired + managerApplyEligible = $managerApplyEligible + managerApplyBlockedReason = $managerApplyBlockedReason + managerApplyVerified = $managerApplyVerified applyExitCode = if ($applyTransport) { $applyTransport.exitCode } else { $null } runtimeWritePerformed = [bool]($applyReceipt -and $applyReceipt.runtimeWritePerformed) changed = $changed @@ -5642,7 +5989,16 @@ function Invoke-AgentHost112GuestRecovery { terminal = $terminal before = $before after = $after + dryRun = $dryRun applyReceipt = $applyReceipt + durableReceiptReadback = $durableReceiptReadback + applyReceiptCommonVerified = $applyReceiptCommonVerified + durableReceiptVerified = $durableReceiptVerified + afterGeneralVerifierVerified = $afterGeneralVerifierVerified + afterManagerVerifierVerified = $afterManagerVerifierVerified + afterVerifierRole = "independent_second_verifier_only" + timeoutContract = $timeoutContract + transportContract = $transportContract rollback = [pscustomobject]@{ attempted = [bool]($applyReceipt -and $applyReceipt.rollbackAttempted) verified = [bool]($applyReceipt -and $applyReceipt.rollbackVerified) diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index f3b3f0eee..25f96b168 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -195,6 +195,16 @@ try { $config = Get-Content $ConfigPath -Raw | ConvertFrom-Json Add-DefaultProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519") + Add-DefaultProperty $config "sshUsers" ([pscustomobject]@{}) + $previousHost112User = if ($config.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$config.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" } + if ($previousHost112User -ne "kali") { + Set-AgentProperty $config.sshUsers "192.168.0.112" "kali" + $script:ConfigMigrations += [pscustomobject]@{ + name = "host112_canonical_ssh_user" + from = if ($previousHost112User) { $previousHost112User } else { "missing" } + to = "kali" + } + } Add-DefaultProperty $config "sshTransport" ([pscustomobject]@{}) Add-DefaultProperty $config.sshTransport "serialized" $true Add-DefaultProperty $config.sshTransport "lockTimeoutSeconds" 90 @@ -207,6 +217,15 @@ try { Add-DefaultProperty $config.k3s "jumpHost" "192.168.0.110" Add-DefaultProperty $config.k3s "preferJumpHost" $true Add-DefaultProperty $config.k3s "directHosts" @("192.168.0.110", "192.168.0.112") + $directHosts = @($config.k3s.directHosts | ForEach-Object { [string]$_ } | Where-Object { $_ }) + if ("192.168.0.112" -notin $directHosts) { + Set-AgentProperty $config.k3s "directHosts" @($directHosts + "192.168.0.112" | Select-Object -Unique) + $script:ConfigMigrations += [pscustomobject]@{ + name = "host112_canonical_direct_route" + from = ($directHosts -join ",") + to = ((@($directHosts + "192.168.0.112") | Select-Object -Unique) -join ",") + } + } Add-DefaultProperty $config "autoRecovery" ([pscustomobject]@{}) Add-DefaultProperty $config.autoRecovery "enabled" $true Add-DefaultProperty $config.autoRecovery "triggerOnHostUnreachable" $true @@ -259,6 +278,12 @@ try { } } $config | ConvertTo-Json -Depth 20 | Set-Content -Path $ConfigPath -Encoding UTF8 + $persistedConfig = Get-Content $ConfigPath -Raw | ConvertFrom-Json + $persistedHost112User = if ($persistedConfig.sshUsers -and $persistedConfig.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$persistedConfig.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" } + $persistedDirectHosts = if ($persistedConfig.k3s -and $persistedConfig.k3s.PSObject.Properties["directHosts"]) { @($persistedConfig.k3s.directHosts | ForEach-Object { [string]$_ }) } else { @() } + if ($persistedHost112User -ne "kali" -or "192.168.0.112" -notin $persistedDirectHosts) { + throw "Host112 canonical direct SSH config post-verifier failed." + } $manifestRows = @() foreach ($staged in $stagedFiles) { diff --git a/agent99-synthetic-tests.ps1 b/agent99-synthetic-tests.ps1 index 29b0403c2..b2d746d0a 100644 --- a/agent99-synthetic-tests.ps1 +++ b/agent99-synthetic-tests.ps1 @@ -42,7 +42,13 @@ foreach ($functionName in @( "Get-AgentIncidentCardModel", "Format-AgentTelegramText", "Format-AgentTelegramCaption", - "Convert-AgentRecoveryReadback" + "Convert-AgentRecoveryReadback", + "Get-HostSshUser", + "Get-AgentSshTransportConfig", + "Get-AgentHost112TransportContract", + "Get-AgentHost112TimeoutContract", + "Convert-AgentHost112GuestReadback", + "Invoke-AgentHost112GuestRecovery" )) { $definition = $ast.Find({ param($node) @@ -139,11 +145,124 @@ if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) { Add-SyntheticCheck "recovery:coordinator_parser" $false "Convert-AgentRecoveryReadback not found" } +if ( + $formatFunctions.ContainsKey("Get-HostSshUser") -and + $formatFunctions.ContainsKey("Get-AgentSshTransportConfig") -and + $formatFunctions.ContainsKey("Get-AgentHost112TransportContract") -and + $formatFunctions.ContainsKey("Get-AgentHost112TimeoutContract") -and + $formatFunctions.ContainsKey("Convert-AgentHost112GuestReadback") -and + $formatFunctions.ContainsKey("Invoke-AgentHost112GuestRecovery") +) { + $Config = [pscustomobject]@{ + agentRoot = $WorkRoot + sshUser = "wooo" + sshUsers = [pscustomobject]@{ "192.168.0.112" = "kali" } + sshTransport = [pscustomobject]@{ serialized = $true; lockTimeoutSeconds = 90; lockPollMilliseconds = 200 } + k3s = [pscustomobject]@{ jumpHost = "192.168.0.110"; preferJumpHost = $true; directHosts = @("192.168.0.110", "192.168.0.112") } + } + $SshUser = "wooo" + $SshIdentityFile = Join-Path $WorkRoot "keys\agent99_ed25519" + $ControlledApply = $true + $script:Agent99DispatchClientDeadlineSeconds = 1800 + $script:Agent99QueueOuterDeadlineSeconds = 1680 + $script:Agent99Host112CheckTimeoutSeconds = 30 + $script:Agent99Host112DryRunTimeoutSeconds = 30 + $script:Agent99Host112ExecutorBudgetSeconds = 420 + $script:Agent99Host112ApplyTransportTimeoutSeconds = 510 + $script:Agent99Host112ReceiptTimeoutSeconds = 30 + $script:Agent99Host112AfterTimeoutSeconds = 30 + $script:Agent99Host112SshCallCount = 5 + $script:Host112SyntheticCall = 0 + $script:Host112SyntheticScenario = "general_only" + + function Write-AgentLog { param([string]$Message) } + function Record-AgentEvent { + param([string]$Type, [string]$Severity, [string]$Message, [object]$Data, [switch]$Alert) + } + function New-Host112SyntheticRow { + param( + [string]$Mode, + [int]$ConsoleReady, + [int]$RuntimeWrite, + [string]$Actions, + [int]$ReceiptWrite, + [string]$ReceiptStatus, + [int]$DurablePresent, + [string]$DurableTerminal, + [int]$ExecutorExit + ) + $guestReady = $ConsoleReady + $receiptRef = if ($DurablePresent -eq 1) { "host112-manager-recovery:fixture" } else { "none" } + $receiptPath = if ($DurablePresent -eq 1) { "/var/lib/awoooi-host112-recovery/receipts/fixture.txt" } else { "none" } + $receiptSha = if ($DurablePresent -eq 1) { "a" * 64 } else { "none" } + $preflight = "already_verified_healthy" + "schema_version=host112_guest_recovery_v2 mode=$Mode trace_id=trace:fixture run_id=run:fixture work_item_id=HOST112-FIXTURE boot_id=boot-fixture uptime_seconds=999 systemd_state=running default_target=graphical.target graphical_target=active display_manager=active lightdm=active vmware_tools=active xorg=active wazuh_indexer=active wazuh_manager=active wazuh_manager_result=success wazuh_analysisd_process_count=1 wazuh_dashboard=active filebeat=active ssh_service=active ssh_enabled=enabled ssh_port_listening=1 ssh_ready=1 console_ready=$ConsoleReady services_ready=1 recovery_timer_ready=1 guest_ready=$guestReady manager_preflight_status=$preflight manager_resource_ready=1 wazuh_agent_event_port_1514_ready=1 wazuh_agent_enrollment_port_1515_tcp=1 wazuh_manager_api_port_55000_tcp=1 manager_independent_verifier_ready=1 manager_attempt_count=0 manager_start_exit=not_attempted executor_deadline_seconds=420 runtime_write_performed=$RuntimeWrite state_write_performed=0 manager_marker_write_performed=0 manager_runtime_write_performed=0 receipt_write_performed=$ReceiptWrite receipt_status=$ReceiptStatus receipt_ref=$receiptRef receipt_path=$receiptPath receipt_sha256=$receiptSha receipt_reused=0 durable_receipt_present=$DurablePresent durable_receipt_identity_match=$DurablePresent durable_receipt_boot_match=$DurablePresent durable_receipt_terminal=$DurableTerminal rollback_attempted=0 rollback_verified=0 rollback_terminal=not_required action_count=1 actions=$Actions executor_exit_code=$ExecutorExit" + } + function Invoke-AgentHost112SshText { + param([string]$Command, [int]$TimeoutSeconds, [int]$Retries = 1) + $script:Host112SyntheticCall += 1 + $row = if ($script:Host112SyntheticScenario -eq "failed_receipt_external_recovery") { + New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "rolled_back_verifier_failed" 0 + } else { + switch ($script:Host112SyntheticCall) { + 1 { New-Host112SyntheticRow "check" 0 0 "none" 0 "not_requested" 0 "unknown" 1 } + 2 { New-Host112SyntheticRow "dry_run" 0 0 "none" 0 "not_requested" 0 "unknown" 0 } + 3 { New-Host112SyntheticRow "apply" 1 1 "start_lightdm" 1 "written_immutable" 1 "idempotent_already_verified_healthy" 0 } + 4 { New-Host112SyntheticRow "dry_run" 1 0 "none" 0 "not_requested" 1 "idempotent_already_verified_healthy" 0 } + default { New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "idempotent_already_verified_healthy" 0 } + } + } + [pscustomobject]@{ + ok = $true + exitCode = 0 + output = $row + elapsedMs = 1 + transportSerialized = $true + transportLockWaitMs = 0 + route = "direct_host112_kali" + } + } + + $generalOnly = Invoke-AgentHost112GuestRecovery ` + -RequestedRunId "run:fixture" ` + -RequestedTraceId "trace:fixture" ` + -RequestedWorkItemId "HOST112-FIXTURE" + Add-SyntheticCheck "recovery:host112_general_candidate_not_blocked_by_healthy_manager" ( + $generalOnly.applyAttempted -and + $generalOnly.generalCandidateRequired -and + $generalOnly.generalApplyVerified -and + -not $generalOnly.managerCandidateRequired -and + -not $generalOnly.applyReceipt.managerRuntimeWritePerformed -and + [string]$generalOnly.applyReceipt.managerAttemptCount -eq "0" -and + $generalOnly.durableReceiptVerified -and + $generalOnly.verified + ) "manager healthy plus console degraded executes only general repair and closes from immutable receipt plus second verifier" + + $script:Host112SyntheticCall = 0 + $script:Host112SyntheticScenario = "failed_receipt_external_recovery" + $failedReplay = Invoke-AgentHost112GuestRecovery ` + -RequestedRunId "run:fixture" ` + -RequestedTraceId "trace:fixture" ` + -RequestedWorkItemId "HOST112-FIXTURE" + Add-SyntheticCheck "recovery:host112_failed_apply_cannot_be_washed_green" ( + -not $failedReplay.applyAttempted -and + -not $failedReplay.verified -and + $failedReplay.managerVerified -and + $failedReplay.before.durableReceiptTerminal -eq "rolled_back_verifier_failed" -and + $failedReplay.terminal -eq "no_write_immutable_receipt_replay_not_successful" + ) "a later healthy manager cannot replace a failed immutable receipt for the same run" +} else { + Add-SyntheticCheck "recovery:host112_general_candidate_fixture" $false "Host112 control functions not found" +} + $control = $controlSource Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_already_queued") -and $control.Contains("recovery_trigger_cooldown") -and $control.Contains('mode = "Recover"')) "automatic recovery is queued and deduplicated" Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence" Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("recovered_late")) "boot recovery remains durable through first-run termination and late closure" Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present" +Add-SyntheticCheck "recovery:host112_apply_receipt_closure" ($control.Contains("managerApplyVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains("durableReceiptVerified") -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply cannot be closed by a later healthy observation without its immutable receipt" +Add-SyntheticCheck "recovery:host112_candidate_split" ($control.Contains("generalCandidateRequired") -and $control.Contains("managerCandidateRequired") -and $control.Contains("generalApplyEligible") -and $control.Contains("managerApplyEligible") -and $control.Contains("managerNoAttemptPreserved")) "Host112 general guest repair is independent from manager eligibility" +Add-SyntheticCheck "recovery:host112_timeout_contract" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120") -and $control.Contains('$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)')) "Host112 executor, SSH lock, queue and client deadlines have explicit reserves" $failures = @($checks | Where-Object { -not $_.ok }) $result = [pscustomobject]@{ diff --git a/agent99.config.example.json b/agent99.config.example.json index 083f9eed2..aa7c6469c 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -13,7 +13,9 @@ "staleMinutes": 15, "hardStaleMinutes": 60 }, - "sshUsers": {}, + "sshUsers": { + "192.168.0.112": "kali" + }, "vmrunPath": "", "hosts": [ "192.168.0.110", diff --git a/docs/operations/AGENT99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md b/docs/operations/AGENT99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md new file mode 100644 index 000000000..e4b2a9002 --- /dev/null +++ b/docs/operations/AGENT99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md @@ -0,0 +1,53 @@ +# Agent99 Host112 controlled recovery contract + +Host112 recovery is a bounded, correlated controlled-apply lane. Agent99 uses +the Windows99 `agent99_ed25519` identity directly to `kali@192.168.0.112`; it +does not fall back through the 110 jump route for this lane. + +## Candidate separation + +- The general guest candidate covers systemd, console/LightDM, VMware Tools, + SSH, Wazuh Indexer, and the recovery timer. +- The Wazuh manager candidate is evaluated separately. A healthy manager, a + manager cooldown, or a manager resource block does not suppress a valid + general guest repair candidate. +- When the manager was already verified healthy, a general-only apply must + retain `manager_attempt_count=0` and + `manager_runtime_write_performed=0`. + +## Apply closure + +Before any apply, Agent99 runs the correlated `sudo --dry-run` with the same +`trace_id`, `run_id`, and `work_item_id`. The dry-run must have a working direct +transport, matching identity and boot, a readable preflight result, and no +runtime or receipt write. + +A manager recovery is verified only when all of the following are true for the +same apply: + +1. The direct apply transport completed and the executor returned healthy. +2. The apply emitted a new immutable receipt with the matching identity and + boot, `terminal=verified_healthy`, independent manager verifier ready, + `manager_runtime_write_performed=1`, and `manager_attempt_count=1`. +3. A second correlated `sudo --dry-run` reloads the durable receipt and returns + the exact same receipt path, SHA-256, identity, boot, and terminal. +4. A final ordinary `--check` independently verifies current guest and manager + runtime health on the same boot. + +The last check is evidence only. A timer or external process that makes the +manager healthy after a failed apply cannot replace the apply transport or its +immutable receipt and therefore cannot turn that run green. + +## Timeout chain + +- Host executor absolute deadline: 420 seconds. +- Agent99 apply transport: 510 seconds, leaving 90 seconds for transport + teardown and receipt delivery. +- Worst-case Host112 chain, including five 90-second SSH lock waits: 1080 + seconds. +- Queue child outer deadline: 1680 seconds, leaving 600 seconds after the + Host112 chain. +- Dispatch client deadline: 1800 seconds, leaving 120 seconds after the queue + child deadline. + +The control plane validates these reserves before attempting Host112 recovery. diff --git a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py index b44a43b4a..ba3223349 100644 --- a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py +++ b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py @@ -235,7 +235,13 @@ def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None: assert 'if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt "$EXECUTOR_DEADLINE_SECONDS" ]; then' in source assert 'if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -gt "$MANAGER_VERIFY_TIMEOUT_SECONDS" ]; then' in source assert "TimeoutStartSec=480" in service - assert "Invoke-HostSshText $targetHost $applyCommand 590 1" in control + assert "$script:Agent99Host112ExecutorBudgetSeconds = 420" in control + assert "$script:Agent99Host112ApplyTransportTimeoutSeconds = 510" in control + assert "Invoke-AgentHost112SshText $applyCommand $script:Agent99Host112ApplyTransportTimeoutSeconds 1" in control + assert "executorReserveSeconds -ge 60" in control + assert "queueReserveSeconds -ge 300" in control + assert "clientReserveSeconds -ge 120" in control + assert '$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)' in control def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> None: @@ -396,8 +402,8 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None: assert f'[string]`${parameter} = ""' in bootstrap assert 'reason = "controlled_dispatch_identity_unsafe"' in control assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function - assert '$before.managerPreflightStatus -eq "ready"' in function - assert 'receiptStatus -eq "written"' in function + assert '$dryRun.managerPreflightStatus -eq "ready"' in function + assert 'receiptStatus -eq "written_immutable"' in function assert '$after.managerVerified' in function assert 'name = "wazuh_manager_service_active"' in function assert 'name = "wazuh_analysisd_process_present"' in function @@ -406,3 +412,94 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None: assert 'name = "wazuh_manager_api_55000_tcp"' in function assert "host112_readback_transport_unavailable" in function assert 'terminal = "no_write_control_identity_incomplete_or_unsafe"' in function + + +def test_agent99_manager_success_requires_apply_and_immutable_receipt_chain() -> None: + control = read(CONTROL) + function = control[control.index("function Invoke-AgentHost112GuestRecovery") :] + function = function[: function.index("function Test-HostReachability")] + + assert "$applyTransport -and $applyTransport.ok" in function + assert '$applyReceipt.durableReceiptTerminal -eq "verified_healthy"' in function + assert "$applyReceipt.managerIndependentVerifierReady" in function + assert "$applyReceipt.managerRuntimeWritePerformed" in function + assert '[string]$applyReceipt.managerAttemptCount -eq "1"' in function + assert "$durableReceiptReadback.receiptPath -eq $applyReceipt.receiptPath" in function + assert "$durableReceiptReadback.receiptSha256 -eq $applyReceipt.receiptSha256" in function + assert "$durableReceiptReadback.durableReceiptTerminal -eq $applyReceipt.durableReceiptTerminal" in function + assert '$durableReceiptReadback.durableReceiptTerminal -eq "verified_healthy"' in function + assert "$afterManagerVerifierVerified" in function + assert 'afterVerifierRole = "independent_second_verifier_only"' in function + assert "$after.ready -and" in function + assert "$managerApplyVerified" in function + assert "existingReceiptTerminalAccepted" in function + assert "immutable_receipt_replay_not_successful" in function + assert "host112_failed_apply_cannot_be_washed_green" in read( + ROOT / "agent99-synthetic-tests.ps1" + ) + + +def test_agent99_uses_correlated_sudo_dry_run_over_canonical_direct_key() -> None: + control = read(CONTROL) + + assert "function Invoke-AgentHost112SshText" in control + assert 'expectedUser = "kali"' in control + assert 'route = "direct_host112_kali"' in control + assert "Invoke-SshText -TargetHost $transportContract.targetHost" in control + assert "--dry-run --trace-id {0} --run-id {1} --work-item-id {2}" in control + assert "Invoke-AgentHost112SshText $dryRunCommand" in control + assert "$dryRun.identityMatches" in control + assert "$dryRun.managerPreflightPresent" in control + assert "$dryRun.transportOk" in control + assert "-not $dryRun.durableReceiptPresent" in control + + +def test_unhealthy_executor_exit_one_is_delivered_readback_not_transport_failure() -> None: + control = read(CONTROL) + function = control[control.index("function Invoke-AgentHost112GuestRecovery") :] + function = function[: function.index("function Test-HostReachability")] + synthetic = read(ROOT / "agent99-synthetic-tests.ps1") + + assert function.count('case "$executor_exit" in 0|1) exit 0') == 3 + assert "$before.executorExitCode -in @(\"0\", \"1\")" in function + assert "$before.readbackPresent -and" in function + assert "$before.identityMatches -and" in function + assert '$beforeTransport = Invoke-AgentHost112SshText $checkCommand' in function + assert 'New-Host112SyntheticRow "check" 0 0 "none" 0 "not_requested" 0 "unknown" 1' in synthetic + assert "host112_general_candidate_not_blocked_by_healthy_manager" in synthetic + + +def test_general_guest_candidate_is_not_blocked_by_manager_candidate() -> None: + control = read(CONTROL) + function = control[control.index("function Invoke-AgentHost112GuestRecovery") :] + function = function[: function.index("function Test-HostReachability")] + synthetic = read(ROOT / "agent99-synthetic-tests.ps1") + + assert "$generalCandidateRequired = [bool]($beforeVerified -and -not $before.generalReady)" in function + assert "$managerCandidateRequired = [bool]($beforeVerified -and -not $before.managerVerified)" in function + assert "$generalApplyEligible = [bool]($generalCandidateRequired -and $generalDryRunVerified)" in function + assert "$managerApplyEligible = [bool]($managerCandidateRequired -and $managerDryRunVerified)" in function + assert "$applyEligible = [bool]($generalApplyEligible -or $managerApplyEligible)" in function + assert "managerNoAttemptPreserved" in function + assert "host112_general_candidate_not_blocked_by_healthy_manager" in synthetic + assert '-not $generalOnly.managerCandidateRequired' in synthetic + assert '[string]$generalOnly.applyReceipt.managerAttemptCount -eq "0"' in synthetic + assert '-not $generalOnly.applyReceipt.managerRuntimeWritePerformed' in synthetic + + +def test_deploy_and_bootstrap_canonicalize_host112_direct_transport() -> None: + deploy = read(ROOT / "agent99-deploy.ps1") + bootstrap = read(BOOTSTRAP) + generic_config = read(ROOT / "agent99.config.example.json") + host_config = read(ROOT / "agent99.config.99.example.json") + + assert 'Set-AgentProperty $config.sshUsers "192.168.0.112" "kali"' in deploy + assert 'name = "host112_canonical_direct_route"' in deploy + assert 'Host112 canonical direct SSH config post-verifier failed.' in deploy + assert "Ensure-AgentHost112CanonicalSshConfig" in bootstrap + assert 'Set-AgentBootstrapProperty $config.sshUsers "192.168.0.112" "kali"' in bootstrap + assert 'Host112 canonical direct SSH config bootstrap verifier failed.' in bootstrap + for config in (generic_config, host_config): + assert '"192.168.0.112": "kali"' in config + assert '"directHosts"' in config + assert '"192.168.0.112"' in config