fix(agent99): bind host112 apply to immutable receipt
This commit is contained in:
@@ -41,6 +41,46 @@ function Copy-AgentFile {
|
||||
}
|
||||
}
|
||||
|
||||
function Set-AgentBootstrapProperty {
|
||||
param([object]$Object, [string]$Name, [object]$Value)
|
||||
if ($Object.PSObject.Properties[$Name]) {
|
||||
$Object.PSObject.Properties[$Name].Value = $Value
|
||||
} else {
|
||||
$Object | Add-Member -MemberType NoteProperty -Name $Name -Value $Value
|
||||
}
|
||||
}
|
||||
|
||||
function Ensure-AgentHost112CanonicalSshConfig {
|
||||
param([string]$Path)
|
||||
$config = Get-Content $Path -Raw | ConvertFrom-Json
|
||||
if (-not $config.PSObject.Properties["sshIdentityFile"] -or -not $config.sshIdentityFile) {
|
||||
Set-AgentBootstrapProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519")
|
||||
}
|
||||
if (-not $config.PSObject.Properties["sshUsers"] -or $null -eq $config.sshUsers) {
|
||||
Set-AgentBootstrapProperty $config "sshUsers" ([pscustomobject]@{})
|
||||
}
|
||||
Set-AgentBootstrapProperty $config.sshUsers "192.168.0.112" "kali"
|
||||
if (-not $config.PSObject.Properties["k3s"] -or $null -eq $config.k3s) {
|
||||
Set-AgentBootstrapProperty $config "k3s" ([pscustomobject]@{})
|
||||
}
|
||||
if (-not $config.k3s.PSObject.Properties["jumpHost"] -or -not $config.k3s.jumpHost) {
|
||||
Set-AgentBootstrapProperty $config.k3s "jumpHost" "192.168.0.110"
|
||||
}
|
||||
if (-not $config.k3s.PSObject.Properties["preferJumpHost"]) {
|
||||
Set-AgentBootstrapProperty $config.k3s "preferJumpHost" $true
|
||||
}
|
||||
$directHosts = if ($config.k3s.PSObject.Properties["directHosts"]) { @($config.k3s.directHosts | ForEach-Object { [string]$_ } | Where-Object { $_ }) } else { @() }
|
||||
Set-AgentBootstrapProperty $config.k3s "directHosts" @($directHosts + @("192.168.0.110", "192.168.0.112") | Select-Object -Unique)
|
||||
$config | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $Path
|
||||
|
||||
$persisted = Get-Content $Path -Raw | ConvertFrom-Json
|
||||
$persistedUser = if ($persisted.sshUsers -and $persisted.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$persisted.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" }
|
||||
$persistedDirectHosts = if ($persisted.k3s -and $persisted.k3s.PSObject.Properties["directHosts"]) { @($persisted.k3s.directHosts | ForEach-Object { [string]$_ }) } else { @() }
|
||||
if ($persistedUser -ne "kali" -or "192.168.0.112" -notin $persistedDirectHosts) {
|
||||
throw "Host112 canonical direct SSH config bootstrap verifier failed."
|
||||
}
|
||||
}
|
||||
|
||||
$agentFiles = @(
|
||||
"agent99-bootstrap.ps1",
|
||||
"agent99-contract-check.ps1",
|
||||
@@ -122,10 +162,17 @@ if (-not (Test-Path $configPath)) {
|
||||
"agentName": "Agent99",
|
||||
"agentRoot": "$($AgentRoot.Replace('\', '\\'))",
|
||||
"sshUser": "wooo",
|
||||
"sshUsers": {},
|
||||
"sshIdentityFile": "$($AgentRoot.Replace('\', '\\'))\\keys\\agent99_ed25519",
|
||||
"sshUsers": { "192.168.0.112": "kali" },
|
||||
"sshTransport": { "serialized": true, "lockTimeoutSeconds": 90, "lockPollMilliseconds": 200 },
|
||||
"vmrunPath": "",
|
||||
"hosts": ["192.168.0.110", "192.168.0.112", "192.168.0.120", "192.168.0.121", "192.168.0.188"],
|
||||
"vms": [],
|
||||
"k3s": {
|
||||
"jumpHost": "192.168.0.110",
|
||||
"preferJumpHost": true,
|
||||
"directHosts": ["192.168.0.110", "192.168.0.112"]
|
||||
},
|
||||
"publicUrls": ["awoooi.wooo.work", "stock.wooo.work", "2026fifa.wooo.work", "gitea.wooo.work", "harbor.wooo.work"],
|
||||
"sreAlertRelay": {
|
||||
"enabled": true,
|
||||
@@ -182,6 +229,8 @@ if (-not (Test-Path $configPath)) {
|
||||
}
|
||||
}
|
||||
|
||||
Ensure-AgentHost112CanonicalSshConfig $configPath
|
||||
|
||||
$launcher = Join-Path $AgentRoot "agent99-run.ps1"
|
||||
@"
|
||||
param(
|
||||
|
||||
@@ -75,6 +75,7 @@ $control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane.
|
||||
$inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8)
|
||||
$telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8)
|
||||
$bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8)
|
||||
$deployer = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-deploy.ps1"), [Text.Encoding]::UTF8)
|
||||
$taskRegistration = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-register-tasks.ps1"), [Text.Encoding]::UTF8)
|
||||
|
||||
Add-Check "transport:dedicated_identity" ($control.Contains("sshIdentityFile") -and $control.Contains("IdentitiesOnly=yes")) "dedicated SSH identity is enforced"
|
||||
@@ -86,6 +87,9 @@ Add-Check "recovery:durable_boot_event" ($control.Contains("pendingRecovery") -a
|
||||
Add-Check "recovery:terminal_evidence_budget" ($taskRegistration.Contains('$recoverySettings') -and $taskRegistration.Contains('New-TimeSpan -Minutes 20')) "startup recovery outlives the ten-minute SLO long enough to write terminal evidence"
|
||||
Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present"
|
||||
Add-Check "recovery:full_sop_coordinator" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_scorecard") -and $control.Contains("SCORECARD_FRESH_REBOOT_WINDOW") -and $control.Contains('scope = $recoveryScope')) "Agent99 cannot close reboot recovery without the 110 full-SOP scorecard"
|
||||
Add-Check "recovery:host112_immutable_apply_closure" ($control.Contains("managerApplyVerified") -and $control.Contains("durableReceiptVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply closure binds transport, immutable receipt and independent verifier"
|
||||
Add-Check "recovery:host112_direct_transport" ($control.Contains("Invoke-AgentHost112SshText") -and $control.Contains('expectedUser = "kali"') -and $control.Contains('route = "direct_host112_kali"') -and $deployer.Contains('name = "host112_canonical_direct_route"') -and $bootstrap.Contains("Ensure-AgentHost112CanonicalSshConfig")) "Host112 recovery uses the canonical 99 direct key route"
|
||||
Add-Check "recovery:host112_timeout_reserve" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120")) "Host112 timeout chain has executor, queue and client reserves"
|
||||
Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present"
|
||||
Add-Check "problem_counts:v2" ($control.Contains('schemaVersion = "agent99_problem_counts_v2"') -and $control.Contains("totalVerifiedResolved")) "verified problem counts are present"
|
||||
Add-Check "callback:durable_completion" ($control.Contains('schema_version = "agent99_completion_callback_v1"') -and $control.Contains("durable_readback") -and $control.Contains("completion-callbacks\pending")) "Agent99 completion callbacks are queued until durable AWOOOI/AwoooP readback"
|
||||
|
||||
@@ -47,6 +47,15 @@ $script:Events = @()
|
||||
$script:TelegramAttempts = @()
|
||||
$script:SuppressAgentAlerts = [bool]$SuppressAlerts
|
||||
$script:SuppressAgentAlertReason = if ($SuppressReason) { $SuppressReason } elseif ($SuppressAlerts) { "suppress_alerts" } else { "" }
|
||||
$script:Agent99DispatchClientDeadlineSeconds = 1800
|
||||
$script:Agent99QueueOuterDeadlineSeconds = 1680
|
||||
$script:Agent99Host112CheckTimeoutSeconds = 30
|
||||
$script:Agent99Host112DryRunTimeoutSeconds = 30
|
||||
$script:Agent99Host112ExecutorBudgetSeconds = 420
|
||||
$script:Agent99Host112ApplyTransportTimeoutSeconds = 510
|
||||
$script:Agent99Host112ReceiptTimeoutSeconds = 30
|
||||
$script:Agent99Host112AfterTimeoutSeconds = 30
|
||||
$script:Agent99Host112SshCallCount = 5
|
||||
|
||||
function Convert-AgentBool {
|
||||
param([object]$Value)
|
||||
@@ -139,7 +148,7 @@ function Get-AgentSshTransportConfig {
|
||||
$configured = if ($Config.PSObject.Properties["sshTransport"]) { $Config.sshTransport } else { $null }
|
||||
[pscustomobject]@{
|
||||
serialized = [bool](-not $configured -or -not $configured.PSObject.Properties["serialized"] -or (Convert-AgentBool $configured.serialized))
|
||||
lockTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["lockTimeoutSeconds"]) { [math]::Max(10, [int]$configured.lockTimeoutSeconds) } else { 90 }
|
||||
lockTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["lockTimeoutSeconds"]) { [math]::Min(90, [math]::Max(10, [int]$configured.lockTimeoutSeconds)) } else { 90 }
|
||||
lockPollMilliseconds = if ($configured -and $configured.PSObject.Properties["lockPollMilliseconds"]) { [math]::Min(2000, [math]::Max(50, [int]$configured.lockPollMilliseconds)) } else { 200 }
|
||||
lockPath = Join-Path (Join-Path $Config.agentRoot "state") "ssh-transport.lock"
|
||||
}
|
||||
@@ -2326,6 +2335,84 @@ function Invoke-HostSshText {
|
||||
return $viaJump
|
||||
}
|
||||
|
||||
function Get-AgentHost112TransportContract {
|
||||
$targetHost = "192.168.0.112"
|
||||
$configuredUser = Get-HostSshUser $targetHost
|
||||
$directHosts = if ($Config.k3s -and $Config.k3s.PSObject.Properties["directHosts"]) { @($Config.k3s.directHosts | ForEach-Object { [string]$_ }) } else { @() }
|
||||
$identityConfigured = [bool]($SshIdentityFile -and (Split-Path -Leaf $SshIdentityFile) -eq "agent99_ed25519")
|
||||
[pscustomobject]@{
|
||||
targetHost = $targetHost
|
||||
configuredUser = $configuredUser
|
||||
expectedUser = "kali"
|
||||
directHostConfigured = [bool]($targetHost -in $directHosts)
|
||||
identityConfigured = $identityConfigured
|
||||
ready = [bool]($configuredUser -eq "kali" -and $targetHost -in $directHosts -and $identityConfigured)
|
||||
route = "direct_host112_kali"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-AgentHost112TimeoutContract {
|
||||
$sshTransport = Get-AgentSshTransportConfig
|
||||
$executorReserveSeconds = $script:Agent99Host112ApplyTransportTimeoutSeconds - $script:Agent99Host112ExecutorBudgetSeconds
|
||||
$host112ChainBoundSeconds = (
|
||||
$script:Agent99Host112CheckTimeoutSeconds +
|
||||
$script:Agent99Host112DryRunTimeoutSeconds +
|
||||
$script:Agent99Host112ApplyTransportTimeoutSeconds +
|
||||
$script:Agent99Host112ReceiptTimeoutSeconds +
|
||||
$script:Agent99Host112AfterTimeoutSeconds +
|
||||
($script:Agent99Host112SshCallCount * $sshTransport.lockTimeoutSeconds)
|
||||
)
|
||||
$queueReserveSeconds = $script:Agent99QueueOuterDeadlineSeconds - $host112ChainBoundSeconds
|
||||
$clientReserveSeconds = $script:Agent99DispatchClientDeadlineSeconds - $script:Agent99QueueOuterDeadlineSeconds
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_host112_timeout_contract_v1"
|
||||
lockWaitPerCallSeconds = [int]$sshTransport.lockTimeoutSeconds
|
||||
sshCallCount = [int]$script:Agent99Host112SshCallCount
|
||||
checkTimeoutSeconds = [int]$script:Agent99Host112CheckTimeoutSeconds
|
||||
dryRunTimeoutSeconds = [int]$script:Agent99Host112DryRunTimeoutSeconds
|
||||
executorBudgetSeconds = [int]$script:Agent99Host112ExecutorBudgetSeconds
|
||||
applyTransportTimeoutSeconds = [int]$script:Agent99Host112ApplyTransportTimeoutSeconds
|
||||
executorReserveSeconds = [int]$executorReserveSeconds
|
||||
receiptTimeoutSeconds = [int]$script:Agent99Host112ReceiptTimeoutSeconds
|
||||
afterTimeoutSeconds = [int]$script:Agent99Host112AfterTimeoutSeconds
|
||||
host112ChainBoundSeconds = [int]$host112ChainBoundSeconds
|
||||
queueOuterDeadlineSeconds = [int]$script:Agent99QueueOuterDeadlineSeconds
|
||||
queueReserveSeconds = [int]$queueReserveSeconds
|
||||
dispatchClientDeadlineSeconds = [int]$script:Agent99DispatchClientDeadlineSeconds
|
||||
clientReserveSeconds = [int]$clientReserveSeconds
|
||||
valid = [bool](
|
||||
$executorReserveSeconds -ge 60 -and
|
||||
$queueReserveSeconds -ge 300 -and
|
||||
$clientReserveSeconds -ge 120
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-AgentHost112SshText {
|
||||
param(
|
||||
[string]$Command,
|
||||
[int]$TimeoutSeconds,
|
||||
[int]$Retries = 1
|
||||
)
|
||||
$transportContract = Get-AgentHost112TransportContract
|
||||
if (-not $transportContract.ready) {
|
||||
return [pscustomobject]@{
|
||||
ok = $false
|
||||
exitCode = -5
|
||||
output = "host112_canonical_direct_transport_not_ready"
|
||||
elapsedMs = 0
|
||||
transportSerialized = $false
|
||||
transportLockWaitMs = 0
|
||||
route = $transportContract.route
|
||||
transportContract = $transportContract
|
||||
}
|
||||
}
|
||||
$result = Invoke-SshText -TargetHost $transportContract.targetHost -Command $Command -TimeoutSeconds $TimeoutSeconds -Retries $Retries -User $transportContract.expectedUser
|
||||
$result | Add-Member -NotePropertyName route -NotePropertyValue $transportContract.route -Force
|
||||
$result | Add-Member -NotePropertyName transportContract -NotePropertyValue $transportContract -Force
|
||||
$result
|
||||
}
|
||||
|
||||
function Get-AgentRecoveryCoordinatorConfig {
|
||||
$configured = if ($Config.PSObject.Properties["recoveryCoordinator"]) { $Config.recoveryCoordinator } else { $null }
|
||||
[pscustomobject]@{
|
||||
@@ -4995,7 +5082,7 @@ function Invoke-AgentQueuedCommands {
|
||||
if ($workItemId) { $args += @("-WorkItemId", $workItemId) }
|
||||
if ($suppressTelegram) { $args += @("-SuppressAlerts", "-SuppressReason", "do_not_page") }
|
||||
$process = Start-Process -FilePath "powershell.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru
|
||||
$completed = $process.WaitForExit(600000)
|
||||
$completed = $process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)
|
||||
if (-not $completed) {
|
||||
$process.Kill()
|
||||
$process.WaitForExit(2000) | Out-Null
|
||||
@@ -5400,11 +5487,14 @@ function Convert-AgentHost112GuestReadback {
|
||||
param([string]$Name)
|
||||
[bool]((& $get $Name "0") -eq "1")
|
||||
}
|
||||
$reportedSchemaVersion = & $get "schema_version"
|
||||
$schemaMatches = [bool]($reportedSchemaVersion -eq "host112_guest_recovery_v2")
|
||||
$readbackPresent = [bool](
|
||||
$values.ContainsKey("schema_version") -and
|
||||
$schemaMatches -and
|
||||
$values.ContainsKey("boot_id") -and
|
||||
$values.ContainsKey("guest_ready") -and
|
||||
$values.ContainsKey("manager_preflight_status")
|
||||
$values.ContainsKey("manager_preflight_status") -and
|
||||
$values.ContainsKey("manager_independent_verifier_ready")
|
||||
)
|
||||
$identityPresent = [bool](
|
||||
$values.ContainsKey("trace_id") -and
|
||||
@@ -5422,15 +5512,34 @@ function Convert-AgentHost112GuestReadback {
|
||||
$agentEventPortReady = (& $asBool "wazuh_agent_event_port_1514_ready")
|
||||
$agentEnrollmentPortReady = (& $asBool "wazuh_agent_enrollment_port_1515_tcp")
|
||||
$managerApiPortReady = (& $asBool "wazuh_manager_api_port_55000_tcp")
|
||||
$managerIndependentVerifierReady = (& $asBool "manager_independent_verifier_ready")
|
||||
$managerVerified = [bool](
|
||||
$managerServiceActive -and
|
||||
$analysisdPresent -and
|
||||
$agentEventPortReady -and
|
||||
$agentEnrollmentPortReady -and
|
||||
$managerApiPortReady
|
||||
$managerApiPortReady -and
|
||||
$managerIndependentVerifierReady
|
||||
)
|
||||
$consoleReady = (& $asBool "console_ready")
|
||||
$sshReady = (& $asBool "ssh_ready")
|
||||
$recoveryTimerReady = (& $asBool "recovery_timer_ready")
|
||||
$generalChecks = @(
|
||||
[pscustomobject]@{ name = "systemd_running"; passed = [bool]((& $get "systemd_state") -eq "running") },
|
||||
[pscustomobject]@{ name = "console_ready"; passed = $consoleReady },
|
||||
[pscustomobject]@{ name = "wazuh_indexer_active"; passed = [bool]((& $get "wazuh_indexer") -eq "active") },
|
||||
[pscustomobject]@{ name = "wazuh_dashboard_active"; passed = [bool]((& $get "wazuh_dashboard") -eq "active") },
|
||||
[pscustomobject]@{ name = "filebeat_active"; passed = [bool]((& $get "filebeat") -eq "active") },
|
||||
[pscustomobject]@{ name = "ssh_ready"; passed = $sshReady },
|
||||
[pscustomobject]@{ name = "recovery_timer_ready"; passed = $recoveryTimerReady }
|
||||
)
|
||||
$generalFailedChecks = @($generalChecks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
|
||||
$generalReady = [bool]($generalFailedChecks.Count -eq 0)
|
||||
$actions = & $get "actions" "none"
|
||||
$generalRuntimeWritePerformed = [bool]($actions -match "(?:^|,)(?:set_default_graphical_target|start_graphical_target|start_vmware_tools|start_lightdm|enable_ssh|start_ssh|retry_wazuh_indexer|enable_recovery_timer|start_recovery_timer)(?:,|$)")
|
||||
$checks = @(
|
||||
[pscustomobject]@{ name = "transport_ok"; passed = [bool]($Transport -and $Transport.ok) },
|
||||
[pscustomobject]@{ name = "schema_matches"; passed = $schemaMatches },
|
||||
[pscustomobject]@{ name = "readback_present"; passed = $readbackPresent },
|
||||
[pscustomobject]@{ name = "control_identity_present"; passed = $identityPresent },
|
||||
[pscustomobject]@{ name = "control_identity_matches"; passed = $identityMatches },
|
||||
@@ -5441,15 +5550,20 @@ function Convert-AgentHost112GuestReadback {
|
||||
[pscustomobject]@{ name = "wazuh_agent_event_1514_ready"; passed = $agentEventPortReady },
|
||||
[pscustomobject]@{ name = "wazuh_agent_enrollment_1515_tcp"; passed = $agentEnrollmentPortReady },
|
||||
[pscustomobject]@{ name = "wazuh_manager_api_55000_tcp"; passed = $managerApiPortReady },
|
||||
[pscustomobject]@{ name = "wazuh_manager_independent_verifier_ready"; passed = $managerIndependentVerifierReady },
|
||||
[pscustomobject]@{ name = "services_ready"; passed = (& $asBool "services_ready") },
|
||||
[pscustomobject]@{ name = "recovery_timer_ready"; passed = (& $asBool "recovery_timer_ready") },
|
||||
[pscustomobject]@{ name = "guest_ready"; passed = (& $asBool "guest_ready") }
|
||||
)
|
||||
$failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_host112_guest_readback_v2"
|
||||
schemaVersion = "agent99_host112_guest_readback_v3"
|
||||
reportedSchemaVersion = $reportedSchemaVersion
|
||||
schemaMatches = $schemaMatches
|
||||
readbackPresent = $readbackPresent
|
||||
ready = [bool]($failedChecks.Count -eq 0)
|
||||
generalReady = $generalReady
|
||||
generalFailedChecks = $generalFailedChecks
|
||||
identityPresent = $identityPresent
|
||||
identityMatches = $identityMatches
|
||||
traceId = & $get "trace_id"
|
||||
@@ -5475,7 +5589,9 @@ function Convert-AgentHost112GuestReadback {
|
||||
managerDiskUsedPercent = & $get "manager_disk_used_percent" "100"
|
||||
managerResourceReady = (& $asBool "manager_resource_ready")
|
||||
managerPreflightStatus = & $get "manager_preflight_status"
|
||||
managerPreflightPresent = [bool]($values.ContainsKey("manager_preflight_status") -and ((& $get "manager_preflight_status") -ne "unknown"))
|
||||
managerVerified = $managerVerified
|
||||
managerIndependentVerifierReady = $managerIndependentVerifierReady
|
||||
wazuhAgentEventPort1514Tcp = (& $asBool "wazuh_agent_event_port_1514_tcp")
|
||||
wazuhAgentEventPort1514Udp = (& $asBool "wazuh_agent_event_port_1514_udp")
|
||||
wazuhAgentEventPort1514Ready = $agentEventPortReady
|
||||
@@ -5483,17 +5599,37 @@ function Convert-AgentHost112GuestReadback {
|
||||
wazuhManagerApiPort55000Tcp = $managerApiPortReady
|
||||
wazuhDashboard = & $get "wazuh_dashboard"
|
||||
filebeat = & $get "filebeat"
|
||||
consoleReady = (& $asBool "console_ready")
|
||||
sshService = & $get "ssh_service"
|
||||
sshEnabled = & $get "ssh_enabled"
|
||||
sshPortListening = (& $asBool "ssh_port_listening")
|
||||
sshReady = $sshReady
|
||||
consoleReady = $consoleReady
|
||||
servicesReady = (& $asBool "services_ready")
|
||||
recoveryTimerReady = (& $asBool "recovery_timer_ready")
|
||||
recoveryTimerReady = $recoveryTimerReady
|
||||
guestReady = (& $asBool "guest_ready")
|
||||
actionCount = & $get "action_count" "0"
|
||||
actions = & $get "actions" "none"
|
||||
actions = $actions
|
||||
generalRuntimeWritePerformed = $generalRuntimeWritePerformed
|
||||
executorExitCode = & $get "executor_exit_code" "unknown"
|
||||
executorDeadlineSeconds = & $get "executor_deadline_seconds" "unknown"
|
||||
runtimeWritePerformed = (& $asBool "runtime_write_performed")
|
||||
stateWritePerformed = (& $asBool "state_write_performed")
|
||||
managerMarkerWritePerformed = (& $asBool "manager_marker_write_performed")
|
||||
managerRuntimeWritePerformed = (& $asBool "manager_runtime_write_performed")
|
||||
receiptWritePerformed = (& $asBool "receipt_write_performed")
|
||||
receiptStatus = & $get "receipt_status"
|
||||
receiptRef = & $get "receipt_ref" "none"
|
||||
receiptPath = & $get "receipt_path" "none"
|
||||
receiptSha256 = & $get "receipt_sha256" "none"
|
||||
receiptReused = (& $asBool "receipt_reused")
|
||||
durableReceiptPresent = (& $asBool "durable_receipt_present")
|
||||
durableReceiptIdentityMatch = (& $asBool "durable_receipt_identity_match")
|
||||
durableReceiptBootMatch = (& $asBool "durable_receipt_boot_match")
|
||||
durableReceiptTerminal = & $get "durable_receipt_terminal" "unknown"
|
||||
durableReadbackStatus = & $get "durable_readback_status" "unknown"
|
||||
durableReadbackPath = & $get "durable_readback_path" "none"
|
||||
durableReadbackSha256 = & $get "durable_readback_sha256" "none"
|
||||
latestReadbackPointerWritten = (& $asBool "latest_readback_pointer_written")
|
||||
managerAttemptCount = & $get "manager_attempt_count" "0"
|
||||
managerStartExit = & $get "manager_start_exit" "not_attempted"
|
||||
rollbackAttempted = (& $asBool "rollback_attempted")
|
||||
@@ -5539,7 +5675,7 @@ function Invoke-AgentHost112GuestRecovery {
|
||||
if (-not $identitySafe) {
|
||||
$before = Convert-AgentHost112GuestReadback $null $effectiveTraceId $effectiveRunId $effectiveWorkItemId
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_host112_guest_recovery_v2"
|
||||
schemaVersion = "agent99_host112_guest_recovery_v3"
|
||||
targetHost = $targetHost
|
||||
traceId = $effectiveTraceId
|
||||
runId = $effectiveRunId
|
||||
@@ -5556,67 +5692,271 @@ function Invoke-AgentHost112GuestRecovery {
|
||||
terminal = "no_write_control_identity_incomplete_or_unsafe"
|
||||
before = $before
|
||||
after = $before
|
||||
dryRun = $null
|
||||
applyReceipt = $null
|
||||
durableReceiptReadback = $null
|
||||
rollback = [pscustomobject]@{ attempted = $false; verified = $false; terminal = "not_required" }
|
||||
verifier = "wazuh_manager_service_process_agent_event_1514_enrollment_1515_and_api_55000"
|
||||
prohibitedActions = @("host_reboot", "vm_power_change", "vm_reset", "secret_read", "firewall_change", "database_write")
|
||||
}
|
||||
}
|
||||
|
||||
$checkCommand = "/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId"
|
||||
$applyCommand = "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId"
|
||||
$beforeTransport = Invoke-HostSshText $targetHost $checkCommand 30 1
|
||||
$before = Convert-AgentHost112GuestReadback $beforeTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
|
||||
$applyTransport = $null
|
||||
$applyReceipt = $null
|
||||
$after = $before
|
||||
$applyAttempted = $false
|
||||
$receiptVerified = $false
|
||||
$applyEligible = [bool](
|
||||
$before.transportOk -and
|
||||
$before.identityMatches -and
|
||||
-not $before.managerVerified -and
|
||||
$before.managerPreflightStatus -eq "ready"
|
||||
)
|
||||
$applyBlockedReason = if ($before.managerVerified) {
|
||||
"idempotent_already_verified_healthy"
|
||||
} elseif (-not $before.transportOk) {
|
||||
"host112_readback_transport_unavailable"
|
||||
} elseif (-not $before.identityMatches) {
|
||||
"control_identity_readback_mismatch"
|
||||
} elseif ($before.managerPreflightStatus -ne "ready") {
|
||||
[string]$before.managerPreflightStatus
|
||||
} else {
|
||||
"none"
|
||||
$timeoutContract = Get-AgentHost112TimeoutContract
|
||||
$transportContract = Get-AgentHost112TransportContract
|
||||
if (-not $timeoutContract.valid -or -not $transportContract.ready) {
|
||||
$before = Convert-AgentHost112GuestReadback $null $effectiveTraceId $effectiveRunId $effectiveWorkItemId
|
||||
$contractFailure = if (-not $transportContract.ready) { "canonical_direct_transport_not_ready" } else { "timeout_contract_invalid" }
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_host112_guest_recovery_v3"
|
||||
targetHost = $targetHost
|
||||
traceId = $effectiveTraceId
|
||||
runId = $effectiveRunId
|
||||
workItemId = $effectiveWorkItemId
|
||||
controlledApply = [bool]$ControlledApply
|
||||
applyEligible = $false
|
||||
applyAttempted = $false
|
||||
applyBlockedReason = $contractFailure
|
||||
runtimeWritePerformed = $false
|
||||
changed = $false
|
||||
managerVerified = $false
|
||||
verified = $false
|
||||
receiptVerified = $false
|
||||
terminal = "no_write_$contractFailure"
|
||||
before = $before
|
||||
after = $before
|
||||
dryRun = $null
|
||||
applyReceipt = $null
|
||||
durableReceiptReadback = $null
|
||||
timeoutContract = $timeoutContract
|
||||
transportContract = $transportContract
|
||||
rollback = [pscustomobject]@{ attempted = $false; verified = $false; terminal = "not_required" }
|
||||
verifier = "wazuh_manager_service_process_agent_event_1514_enrollment_1515_and_api_55000"
|
||||
prohibitedActions = @("host_reboot", "vm_power_change", "vm_reset", "secret_read", "firewall_change", "database_write")
|
||||
}
|
||||
}
|
||||
|
||||
# The guest executor uses exit 1 for a completed-but-degraded readback. Keep
|
||||
# that semantic result in executor_exit_code while reserving transport.ok for
|
||||
# SSH/sudo/executor delivery failures.
|
||||
$checkCommand = ('/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf "executor_exit_code=%s\n" "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId)
|
||||
$dryRunCommand = ('sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --dry-run --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf "executor_exit_code=%s\n" "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId)
|
||||
$applyCommand = ('sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf "executor_exit_code=%s\n" "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId)
|
||||
|
||||
$beforeTransport = Invoke-AgentHost112SshText $checkCommand $script:Agent99Host112CheckTimeoutSeconds 1
|
||||
$before = Convert-AgentHost112GuestReadback $beforeTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
|
||||
$beforeBootBound = [bool]($before.bootId -and $before.bootId -ne "unknown")
|
||||
$beforeVerified = [bool](
|
||||
$before.transportOk -and
|
||||
$before.executorExitCode -in @("0", "1") -and
|
||||
$before.readbackPresent -and
|
||||
$before.identityMatches -and
|
||||
$beforeBootBound
|
||||
)
|
||||
$existingReceiptTerminalAccepted = [bool](
|
||||
-not $before.durableReceiptPresent -or
|
||||
$before.durableReceiptTerminal -in @("verified_healthy", "idempotent_already_verified_healthy")
|
||||
)
|
||||
$existingReceiptBound = [bool](
|
||||
-not $before.durableReceiptPresent -or
|
||||
(
|
||||
$before.durableReceiptIdentityMatch -and
|
||||
$before.durableReceiptBootMatch -and
|
||||
$before.receiptSha256 -match "^[a-fA-F0-9]{64}$"
|
||||
)
|
||||
)
|
||||
$generalCandidateRequired = [bool]($beforeVerified -and -not $before.generalReady)
|
||||
$managerCandidateRequired = [bool]($beforeVerified -and -not $before.managerVerified)
|
||||
|
||||
$dryRunTransport = $null
|
||||
$dryRun = $null
|
||||
$generalDryRunVerified = $false
|
||||
$managerDryRunVerified = $false
|
||||
if ($ControlledApply -and $beforeVerified -and ($generalCandidateRequired -or $managerCandidateRequired)) {
|
||||
$dryRunTransport = Invoke-AgentHost112SshText $dryRunCommand $script:Agent99Host112DryRunTimeoutSeconds 1
|
||||
$dryRun = Convert-AgentHost112GuestReadback $dryRunTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
|
||||
$generalDryRunVerified = [bool](
|
||||
$dryRun.transportOk -and
|
||||
$dryRun.executorExitCode -in @("0", "1") -and
|
||||
$dryRun.readbackPresent -and
|
||||
$dryRun.identityMatches -and
|
||||
$dryRun.bootId -eq $before.bootId -and
|
||||
$dryRun.managerPreflightPresent -and
|
||||
-not $dryRun.runtimeWritePerformed -and
|
||||
-not $dryRun.managerRuntimeWritePerformed -and
|
||||
[string]$dryRun.managerAttemptCount -eq "0" -and
|
||||
-not $dryRun.receiptWritePerformed -and
|
||||
-not $dryRun.durableReceiptPresent
|
||||
)
|
||||
$managerDryRunVerified = [bool](
|
||||
$generalDryRunVerified -and
|
||||
$dryRun.executorExitCode -eq "0" -and
|
||||
$dryRun.managerPreflightStatus -eq "ready"
|
||||
)
|
||||
}
|
||||
|
||||
$generalApplyEligible = [bool]($generalCandidateRequired -and $generalDryRunVerified)
|
||||
$managerApplyEligible = [bool]($managerCandidateRequired -and $managerDryRunVerified)
|
||||
$applyEligible = [bool]($generalApplyEligible -or $managerApplyEligible)
|
||||
$managerApplyBlockedReason = if (-not $managerCandidateRequired) {
|
||||
"not_required"
|
||||
} elseif ($managerDryRunVerified) {
|
||||
"none"
|
||||
} elseif ($dryRun -and $dryRun.managerPreflightPresent) {
|
||||
[string]$dryRun.managerPreflightStatus
|
||||
} else {
|
||||
"sudo_dry_run_not_verified"
|
||||
}
|
||||
$applyBlockedReason = if (-not $beforeVerified) {
|
||||
if (-not $before.transportOk) { "host112_readback_transport_unavailable" } else { "host112_readback_identity_or_boot_unverified" }
|
||||
} elseif ($before.durableReceiptPresent -and (-not $existingReceiptBound -or -not $existingReceiptTerminalAccepted)) {
|
||||
"immutable_receipt_replay_not_successful"
|
||||
} elseif (-not $generalCandidateRequired -and -not $managerCandidateRequired) {
|
||||
"idempotent_already_verified_healthy"
|
||||
} elseif (-not $ControlledApply) {
|
||||
"controlled_apply_not_requested"
|
||||
} elseif ($applyEligible) {
|
||||
"none"
|
||||
} elseif ($managerCandidateRequired) {
|
||||
$managerApplyBlockedReason
|
||||
} else {
|
||||
"sudo_dry_run_not_verified"
|
||||
}
|
||||
|
||||
$applyTransport = $null
|
||||
$applyReceipt = $null
|
||||
$durableReceiptTransport = $null
|
||||
$durableReceiptReadback = $null
|
||||
$afterTransport = $null
|
||||
$after = $before
|
||||
$applyAttempted = $false
|
||||
if ($ControlledApply -and $applyEligible) {
|
||||
$applyAttempted = $true
|
||||
Write-AgentLog "CONTROLLED_APPLY host112_guest_recovery traceId=$effectiveTraceId runId=$effectiveRunId workItemId=$effectiveWorkItemId preflight=$($before.managerPreflightStatus)"
|
||||
$applyTransport = Invoke-HostSshText $targetHost $applyCommand 590 1
|
||||
Write-AgentLog "CONTROLLED_APPLY host112_guest_recovery traceId=$effectiveTraceId runId=$effectiveRunId workItemId=$effectiveWorkItemId generalCandidate=$generalCandidateRequired managerCandidate=$managerCandidateRequired managerPreflight=$($dryRun.managerPreflightStatus)"
|
||||
$applyTransport = Invoke-AgentHost112SshText $applyCommand $script:Agent99Host112ApplyTransportTimeoutSeconds 1
|
||||
$applyReceipt = Convert-AgentHost112GuestReadback $applyTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
|
||||
$receiptVerified = [bool](
|
||||
$applyReceipt.identityMatches -and
|
||||
$applyReceipt.receiptWritePerformed -and
|
||||
$applyReceipt.receiptStatus -eq "written" -and
|
||||
$applyReceipt.receiptRef -ne "none"
|
||||
)
|
||||
$afterTransport = Invoke-HostSshText $targetHost $checkCommand 30 1
|
||||
|
||||
# This is the independent durable-receipt channel. The correlated sudo
|
||||
# dry-run reloads the immutable receipt and proves identity, boot and hash;
|
||||
# it does not repeat the apply.
|
||||
$durableReceiptTransport = Invoke-AgentHost112SshText $dryRunCommand $script:Agent99Host112ReceiptTimeoutSeconds 1
|
||||
$durableReceiptReadback = Convert-AgentHost112GuestReadback $durableReceiptTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
|
||||
|
||||
# The ordinary check is intentionally last and is only a second runtime
|
||||
# verifier. It can never repair or replace a failed apply receipt.
|
||||
$afterTransport = Invoke-AgentHost112SshText $checkCommand $script:Agent99Host112AfterTimeoutSeconds 1
|
||||
$after = Convert-AgentHost112GuestReadback $afterTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
|
||||
}
|
||||
|
||||
$managerVerified = [bool]$after.managerVerified
|
||||
$verified = [bool]($after.ready -and (-not $applyAttempted -or $receiptVerified))
|
||||
$changed = [bool](
|
||||
$applyReceiptCommonVerified = [bool](
|
||||
$applyAttempted -and
|
||||
$applyReceipt -and
|
||||
$applyReceipt.managerRuntimeWritePerformed -and
|
||||
$managerVerified -and
|
||||
$receiptVerified
|
||||
$applyTransport -and $applyTransport.ok -and
|
||||
$applyReceipt -and $applyReceipt.transportOk -and
|
||||
$applyReceipt.executorExitCode -in @("0", "1") -and
|
||||
$applyReceipt.identityMatches -and
|
||||
$applyReceipt.bootId -eq $before.bootId -and
|
||||
$applyReceipt.runtimeWritePerformed -and
|
||||
$applyReceipt.receiptWritePerformed -and
|
||||
$applyReceipt.receiptStatus -eq "written_immutable" -and
|
||||
-not $applyReceipt.receiptReused -and
|
||||
$applyReceipt.receiptRef -ne "none" -and
|
||||
$applyReceipt.receiptPath -ne "none" -and
|
||||
$applyReceipt.receiptSha256 -match "^[a-fA-F0-9]{64}$" -and
|
||||
$applyReceipt.durableReceiptPresent -and
|
||||
$applyReceipt.durableReceiptIdentityMatch -and
|
||||
$applyReceipt.durableReceiptBootMatch -and
|
||||
$applyReceipt.durableReceiptTerminal -ne "unknown"
|
||||
)
|
||||
$terminal = if ($applyAttempted -and $applyReceipt) {
|
||||
[string]$applyReceipt.terminal
|
||||
} elseif ($before.managerVerified) {
|
||||
$durableReceiptVerified = [bool](
|
||||
$applyReceiptCommonVerified -and
|
||||
$durableReceiptTransport -and $durableReceiptTransport.ok -and
|
||||
$durableReceiptReadback -and $durableReceiptReadback.transportOk -and
|
||||
$durableReceiptReadback.executorExitCode -in @("0", "1") -and
|
||||
$durableReceiptReadback.identityMatches -and
|
||||
$durableReceiptReadback.bootId -eq $before.bootId -and
|
||||
$durableReceiptReadback.durableReceiptPresent -and
|
||||
$durableReceiptReadback.durableReceiptIdentityMatch -and
|
||||
$durableReceiptReadback.durableReceiptBootMatch -and
|
||||
$durableReceiptReadback.receiptRef -eq $applyReceipt.receiptRef -and
|
||||
$durableReceiptReadback.receiptPath -eq $applyReceipt.receiptPath -and
|
||||
$durableReceiptReadback.receiptSha256 -eq $applyReceipt.receiptSha256 -and
|
||||
$durableReceiptReadback.durableReceiptTerminal -eq $applyReceipt.durableReceiptTerminal -and
|
||||
-not $durableReceiptReadback.runtimeWritePerformed -and
|
||||
-not $durableReceiptReadback.managerRuntimeWritePerformed -and
|
||||
[string]$durableReceiptReadback.managerAttemptCount -eq "0"
|
||||
)
|
||||
$afterGeneralVerifierVerified = [bool](
|
||||
$afterTransport -and $afterTransport.ok -and
|
||||
$after.transportOk -and
|
||||
$after.executorExitCode -in @("0", "1") -and
|
||||
$after.identityMatches -and
|
||||
$after.bootId -eq $before.bootId -and
|
||||
$after.generalReady
|
||||
)
|
||||
$afterManagerVerifierVerified = [bool](
|
||||
$afterTransport -and $afterTransport.ok -and
|
||||
$after.transportOk -and
|
||||
$after.identityMatches -and
|
||||
$after.bootId -eq $before.bootId -and
|
||||
$after.managerVerified -and
|
||||
$after.managerIndependentVerifierReady
|
||||
)
|
||||
$managerApplyVerified = [bool](
|
||||
$managerApplyEligible -and
|
||||
$durableReceiptVerified -and
|
||||
$applyReceipt.executorExitCode -eq "0" -and
|
||||
$applyReceipt.durableReceiptTerminal -eq "verified_healthy" -and
|
||||
$applyReceipt.managerVerified -and
|
||||
$applyReceipt.managerIndependentVerifierReady -and
|
||||
$applyReceipt.managerRuntimeWritePerformed -and
|
||||
[string]$applyReceipt.managerAttemptCount -eq "1" -and
|
||||
-not $applyReceipt.rollbackAttempted -and
|
||||
$durableReceiptReadback.executorExitCode -eq "0" -and
|
||||
$durableReceiptReadback.durableReceiptTerminal -eq "verified_healthy" -and
|
||||
$afterManagerVerifierVerified
|
||||
)
|
||||
$managerNoAttemptPreserved = [bool](
|
||||
$managerCandidateRequired -or
|
||||
(
|
||||
$applyReceipt -and
|
||||
-not $applyReceipt.managerRuntimeWritePerformed -and
|
||||
[string]$applyReceipt.managerAttemptCount -eq "0"
|
||||
)
|
||||
)
|
||||
$generalApplyVerified = [bool](
|
||||
$generalApplyEligible -and
|
||||
$durableReceiptVerified -and
|
||||
$applyReceipt.generalRuntimeWritePerformed -and
|
||||
$managerNoAttemptPreserved -and
|
||||
$afterGeneralVerifierVerified
|
||||
)
|
||||
$receiptVerified = [bool](
|
||||
$durableReceiptVerified -and
|
||||
(-not $managerApplyEligible -or $managerApplyVerified)
|
||||
)
|
||||
$managerVerified = [bool]$after.managerVerified
|
||||
$verified = if ($applyAttempted) {
|
||||
[bool](
|
||||
$receiptVerified -and
|
||||
$after.ready -and
|
||||
(-not $generalCandidateRequired -or $generalApplyVerified) -and
|
||||
(-not $managerCandidateRequired -or $managerApplyVerified)
|
||||
)
|
||||
} else {
|
||||
[bool]($beforeVerified -and $before.ready -and $existingReceiptBound -and $existingReceiptTerminalAccepted)
|
||||
}
|
||||
$changed = [bool]($applyAttempted -and $verified -and $applyReceipt.runtimeWritePerformed)
|
||||
$reportedApplyTerminal = if ($applyReceipt) { [string]$applyReceipt.durableReceiptTerminal } else { "unknown" }
|
||||
$terminal = if ($applyAttempted -and -not $applyReceiptCommonVerified) {
|
||||
"apply_receipt_unverified:$reportedApplyTerminal"
|
||||
} elseif ($applyAttempted -and $managerCandidateRequired -and -not $managerApplyVerified) {
|
||||
"manager_apply_unverified:$reportedApplyTerminal"
|
||||
} elseif ($applyAttempted -and $generalCandidateRequired -and -not $generalApplyVerified) {
|
||||
"general_apply_unverified:$reportedApplyTerminal"
|
||||
} elseif ($applyAttempted) {
|
||||
$reportedApplyTerminal
|
||||
} elseif ($before.durableReceiptPresent -and (-not $existingReceiptBound -or -not $existingReceiptTerminalAccepted)) {
|
||||
"no_write_immutable_receipt_replay_not_successful"
|
||||
} elseif ($before.ready) {
|
||||
"idempotent_already_verified_healthy"
|
||||
} elseif ($ControlledApply) {
|
||||
"no_write_$applyBlockedReason"
|
||||
@@ -5624,7 +5964,7 @@ function Invoke-AgentHost112GuestRecovery {
|
||||
"observed_no_write"
|
||||
}
|
||||
$result = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host112_guest_recovery_v2"
|
||||
schemaVersion = "agent99_host112_guest_recovery_v3"
|
||||
targetHost = $targetHost
|
||||
traceId = $effectiveTraceId
|
||||
runId = $effectiveRunId
|
||||
@@ -5633,6 +5973,13 @@ function Invoke-AgentHost112GuestRecovery {
|
||||
applyEligible = $applyEligible
|
||||
applyAttempted = $applyAttempted
|
||||
applyBlockedReason = $applyBlockedReason
|
||||
generalCandidateRequired = $generalCandidateRequired
|
||||
generalApplyEligible = $generalApplyEligible
|
||||
generalApplyVerified = $generalApplyVerified
|
||||
managerCandidateRequired = $managerCandidateRequired
|
||||
managerApplyEligible = $managerApplyEligible
|
||||
managerApplyBlockedReason = $managerApplyBlockedReason
|
||||
managerApplyVerified = $managerApplyVerified
|
||||
applyExitCode = if ($applyTransport) { $applyTransport.exitCode } else { $null }
|
||||
runtimeWritePerformed = [bool]($applyReceipt -and $applyReceipt.runtimeWritePerformed)
|
||||
changed = $changed
|
||||
@@ -5642,7 +5989,16 @@ function Invoke-AgentHost112GuestRecovery {
|
||||
terminal = $terminal
|
||||
before = $before
|
||||
after = $after
|
||||
dryRun = $dryRun
|
||||
applyReceipt = $applyReceipt
|
||||
durableReceiptReadback = $durableReceiptReadback
|
||||
applyReceiptCommonVerified = $applyReceiptCommonVerified
|
||||
durableReceiptVerified = $durableReceiptVerified
|
||||
afterGeneralVerifierVerified = $afterGeneralVerifierVerified
|
||||
afterManagerVerifierVerified = $afterManagerVerifierVerified
|
||||
afterVerifierRole = "independent_second_verifier_only"
|
||||
timeoutContract = $timeoutContract
|
||||
transportContract = $transportContract
|
||||
rollback = [pscustomobject]@{
|
||||
attempted = [bool]($applyReceipt -and $applyReceipt.rollbackAttempted)
|
||||
verified = [bool]($applyReceipt -and $applyReceipt.rollbackVerified)
|
||||
|
||||
@@ -195,6 +195,16 @@ try {
|
||||
|
||||
$config = Get-Content $ConfigPath -Raw | ConvertFrom-Json
|
||||
Add-DefaultProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519")
|
||||
Add-DefaultProperty $config "sshUsers" ([pscustomobject]@{})
|
||||
$previousHost112User = if ($config.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$config.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" }
|
||||
if ($previousHost112User -ne "kali") {
|
||||
Set-AgentProperty $config.sshUsers "192.168.0.112" "kali"
|
||||
$script:ConfigMigrations += [pscustomobject]@{
|
||||
name = "host112_canonical_ssh_user"
|
||||
from = if ($previousHost112User) { $previousHost112User } else { "missing" }
|
||||
to = "kali"
|
||||
}
|
||||
}
|
||||
Add-DefaultProperty $config "sshTransport" ([pscustomobject]@{})
|
||||
Add-DefaultProperty $config.sshTransport "serialized" $true
|
||||
Add-DefaultProperty $config.sshTransport "lockTimeoutSeconds" 90
|
||||
@@ -207,6 +217,15 @@ try {
|
||||
Add-DefaultProperty $config.k3s "jumpHost" "192.168.0.110"
|
||||
Add-DefaultProperty $config.k3s "preferJumpHost" $true
|
||||
Add-DefaultProperty $config.k3s "directHosts" @("192.168.0.110", "192.168.0.112")
|
||||
$directHosts = @($config.k3s.directHosts | ForEach-Object { [string]$_ } | Where-Object { $_ })
|
||||
if ("192.168.0.112" -notin $directHosts) {
|
||||
Set-AgentProperty $config.k3s "directHosts" @($directHosts + "192.168.0.112" | Select-Object -Unique)
|
||||
$script:ConfigMigrations += [pscustomobject]@{
|
||||
name = "host112_canonical_direct_route"
|
||||
from = ($directHosts -join ",")
|
||||
to = ((@($directHosts + "192.168.0.112") | Select-Object -Unique) -join ",")
|
||||
}
|
||||
}
|
||||
Add-DefaultProperty $config "autoRecovery" ([pscustomobject]@{})
|
||||
Add-DefaultProperty $config.autoRecovery "enabled" $true
|
||||
Add-DefaultProperty $config.autoRecovery "triggerOnHostUnreachable" $true
|
||||
@@ -259,6 +278,12 @@ try {
|
||||
}
|
||||
}
|
||||
$config | ConvertTo-Json -Depth 20 | Set-Content -Path $ConfigPath -Encoding UTF8
|
||||
$persistedConfig = Get-Content $ConfigPath -Raw | ConvertFrom-Json
|
||||
$persistedHost112User = if ($persistedConfig.sshUsers -and $persistedConfig.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$persistedConfig.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" }
|
||||
$persistedDirectHosts = if ($persistedConfig.k3s -and $persistedConfig.k3s.PSObject.Properties["directHosts"]) { @($persistedConfig.k3s.directHosts | ForEach-Object { [string]$_ }) } else { @() }
|
||||
if ($persistedHost112User -ne "kali" -or "192.168.0.112" -notin $persistedDirectHosts) {
|
||||
throw "Host112 canonical direct SSH config post-verifier failed."
|
||||
}
|
||||
|
||||
$manifestRows = @()
|
||||
foreach ($staged in $stagedFiles) {
|
||||
|
||||
@@ -42,7 +42,13 @@ foreach ($functionName in @(
|
||||
"Get-AgentIncidentCardModel",
|
||||
"Format-AgentTelegramText",
|
||||
"Format-AgentTelegramCaption",
|
||||
"Convert-AgentRecoveryReadback"
|
||||
"Convert-AgentRecoveryReadback",
|
||||
"Get-HostSshUser",
|
||||
"Get-AgentSshTransportConfig",
|
||||
"Get-AgentHost112TransportContract",
|
||||
"Get-AgentHost112TimeoutContract",
|
||||
"Convert-AgentHost112GuestReadback",
|
||||
"Invoke-AgentHost112GuestRecovery"
|
||||
)) {
|
||||
$definition = $ast.Find({
|
||||
param($node)
|
||||
@@ -139,11 +145,124 @@ if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) {
|
||||
Add-SyntheticCheck "recovery:coordinator_parser" $false "Convert-AgentRecoveryReadback not found"
|
||||
}
|
||||
|
||||
if (
|
||||
$formatFunctions.ContainsKey("Get-HostSshUser") -and
|
||||
$formatFunctions.ContainsKey("Get-AgentSshTransportConfig") -and
|
||||
$formatFunctions.ContainsKey("Get-AgentHost112TransportContract") -and
|
||||
$formatFunctions.ContainsKey("Get-AgentHost112TimeoutContract") -and
|
||||
$formatFunctions.ContainsKey("Convert-AgentHost112GuestReadback") -and
|
||||
$formatFunctions.ContainsKey("Invoke-AgentHost112GuestRecovery")
|
||||
) {
|
||||
$Config = [pscustomobject]@{
|
||||
agentRoot = $WorkRoot
|
||||
sshUser = "wooo"
|
||||
sshUsers = [pscustomobject]@{ "192.168.0.112" = "kali" }
|
||||
sshTransport = [pscustomobject]@{ serialized = $true; lockTimeoutSeconds = 90; lockPollMilliseconds = 200 }
|
||||
k3s = [pscustomobject]@{ jumpHost = "192.168.0.110"; preferJumpHost = $true; directHosts = @("192.168.0.110", "192.168.0.112") }
|
||||
}
|
||||
$SshUser = "wooo"
|
||||
$SshIdentityFile = Join-Path $WorkRoot "keys\agent99_ed25519"
|
||||
$ControlledApply = $true
|
||||
$script:Agent99DispatchClientDeadlineSeconds = 1800
|
||||
$script:Agent99QueueOuterDeadlineSeconds = 1680
|
||||
$script:Agent99Host112CheckTimeoutSeconds = 30
|
||||
$script:Agent99Host112DryRunTimeoutSeconds = 30
|
||||
$script:Agent99Host112ExecutorBudgetSeconds = 420
|
||||
$script:Agent99Host112ApplyTransportTimeoutSeconds = 510
|
||||
$script:Agent99Host112ReceiptTimeoutSeconds = 30
|
||||
$script:Agent99Host112AfterTimeoutSeconds = 30
|
||||
$script:Agent99Host112SshCallCount = 5
|
||||
$script:Host112SyntheticCall = 0
|
||||
$script:Host112SyntheticScenario = "general_only"
|
||||
|
||||
function Write-AgentLog { param([string]$Message) }
|
||||
function Record-AgentEvent {
|
||||
param([string]$Type, [string]$Severity, [string]$Message, [object]$Data, [switch]$Alert)
|
||||
}
|
||||
function New-Host112SyntheticRow {
|
||||
param(
|
||||
[string]$Mode,
|
||||
[int]$ConsoleReady,
|
||||
[int]$RuntimeWrite,
|
||||
[string]$Actions,
|
||||
[int]$ReceiptWrite,
|
||||
[string]$ReceiptStatus,
|
||||
[int]$DurablePresent,
|
||||
[string]$DurableTerminal,
|
||||
[int]$ExecutorExit
|
||||
)
|
||||
$guestReady = $ConsoleReady
|
||||
$receiptRef = if ($DurablePresent -eq 1) { "host112-manager-recovery:fixture" } else { "none" }
|
||||
$receiptPath = if ($DurablePresent -eq 1) { "/var/lib/awoooi-host112-recovery/receipts/fixture.txt" } else { "none" }
|
||||
$receiptSha = if ($DurablePresent -eq 1) { "a" * 64 } else { "none" }
|
||||
$preflight = "already_verified_healthy"
|
||||
"schema_version=host112_guest_recovery_v2 mode=$Mode trace_id=trace:fixture run_id=run:fixture work_item_id=HOST112-FIXTURE boot_id=boot-fixture uptime_seconds=999 systemd_state=running default_target=graphical.target graphical_target=active display_manager=active lightdm=active vmware_tools=active xorg=active wazuh_indexer=active wazuh_manager=active wazuh_manager_result=success wazuh_analysisd_process_count=1 wazuh_dashboard=active filebeat=active ssh_service=active ssh_enabled=enabled ssh_port_listening=1 ssh_ready=1 console_ready=$ConsoleReady services_ready=1 recovery_timer_ready=1 guest_ready=$guestReady manager_preflight_status=$preflight manager_resource_ready=1 wazuh_agent_event_port_1514_ready=1 wazuh_agent_enrollment_port_1515_tcp=1 wazuh_manager_api_port_55000_tcp=1 manager_independent_verifier_ready=1 manager_attempt_count=0 manager_start_exit=not_attempted executor_deadline_seconds=420 runtime_write_performed=$RuntimeWrite state_write_performed=0 manager_marker_write_performed=0 manager_runtime_write_performed=0 receipt_write_performed=$ReceiptWrite receipt_status=$ReceiptStatus receipt_ref=$receiptRef receipt_path=$receiptPath receipt_sha256=$receiptSha receipt_reused=0 durable_receipt_present=$DurablePresent durable_receipt_identity_match=$DurablePresent durable_receipt_boot_match=$DurablePresent durable_receipt_terminal=$DurableTerminal rollback_attempted=0 rollback_verified=0 rollback_terminal=not_required action_count=1 actions=$Actions executor_exit_code=$ExecutorExit"
|
||||
}
|
||||
function Invoke-AgentHost112SshText {
|
||||
param([string]$Command, [int]$TimeoutSeconds, [int]$Retries = 1)
|
||||
$script:Host112SyntheticCall += 1
|
||||
$row = if ($script:Host112SyntheticScenario -eq "failed_receipt_external_recovery") {
|
||||
New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "rolled_back_verifier_failed" 0
|
||||
} else {
|
||||
switch ($script:Host112SyntheticCall) {
|
||||
1 { New-Host112SyntheticRow "check" 0 0 "none" 0 "not_requested" 0 "unknown" 1 }
|
||||
2 { New-Host112SyntheticRow "dry_run" 0 0 "none" 0 "not_requested" 0 "unknown" 0 }
|
||||
3 { New-Host112SyntheticRow "apply" 1 1 "start_lightdm" 1 "written_immutable" 1 "idempotent_already_verified_healthy" 0 }
|
||||
4 { New-Host112SyntheticRow "dry_run" 1 0 "none" 0 "not_requested" 1 "idempotent_already_verified_healthy" 0 }
|
||||
default { New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "idempotent_already_verified_healthy" 0 }
|
||||
}
|
||||
}
|
||||
[pscustomobject]@{
|
||||
ok = $true
|
||||
exitCode = 0
|
||||
output = $row
|
||||
elapsedMs = 1
|
||||
transportSerialized = $true
|
||||
transportLockWaitMs = 0
|
||||
route = "direct_host112_kali"
|
||||
}
|
||||
}
|
||||
|
||||
$generalOnly = Invoke-AgentHost112GuestRecovery `
|
||||
-RequestedRunId "run:fixture" `
|
||||
-RequestedTraceId "trace:fixture" `
|
||||
-RequestedWorkItemId "HOST112-FIXTURE"
|
||||
Add-SyntheticCheck "recovery:host112_general_candidate_not_blocked_by_healthy_manager" (
|
||||
$generalOnly.applyAttempted -and
|
||||
$generalOnly.generalCandidateRequired -and
|
||||
$generalOnly.generalApplyVerified -and
|
||||
-not $generalOnly.managerCandidateRequired -and
|
||||
-not $generalOnly.applyReceipt.managerRuntimeWritePerformed -and
|
||||
[string]$generalOnly.applyReceipt.managerAttemptCount -eq "0" -and
|
||||
$generalOnly.durableReceiptVerified -and
|
||||
$generalOnly.verified
|
||||
) "manager healthy plus console degraded executes only general repair and closes from immutable receipt plus second verifier"
|
||||
|
||||
$script:Host112SyntheticCall = 0
|
||||
$script:Host112SyntheticScenario = "failed_receipt_external_recovery"
|
||||
$failedReplay = Invoke-AgentHost112GuestRecovery `
|
||||
-RequestedRunId "run:fixture" `
|
||||
-RequestedTraceId "trace:fixture" `
|
||||
-RequestedWorkItemId "HOST112-FIXTURE"
|
||||
Add-SyntheticCheck "recovery:host112_failed_apply_cannot_be_washed_green" (
|
||||
-not $failedReplay.applyAttempted -and
|
||||
-not $failedReplay.verified -and
|
||||
$failedReplay.managerVerified -and
|
||||
$failedReplay.before.durableReceiptTerminal -eq "rolled_back_verifier_failed" -and
|
||||
$failedReplay.terminal -eq "no_write_immutable_receipt_replay_not_successful"
|
||||
) "a later healthy manager cannot replace a failed immutable receipt for the same run"
|
||||
} else {
|
||||
Add-SyntheticCheck "recovery:host112_general_candidate_fixture" $false "Host112 control functions not found"
|
||||
}
|
||||
|
||||
$control = $controlSource
|
||||
Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_already_queued") -and $control.Contains("recovery_trigger_cooldown") -and $control.Contains('mode = "Recover"')) "automatic recovery is queued and deduplicated"
|
||||
Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence"
|
||||
Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("recovered_late")) "boot recovery remains durable through first-run termination and late closure"
|
||||
Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present"
|
||||
Add-SyntheticCheck "recovery:host112_apply_receipt_closure" ($control.Contains("managerApplyVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains("durableReceiptVerified") -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply cannot be closed by a later healthy observation without its immutable receipt"
|
||||
Add-SyntheticCheck "recovery:host112_candidate_split" ($control.Contains("generalCandidateRequired") -and $control.Contains("managerCandidateRequired") -and $control.Contains("generalApplyEligible") -and $control.Contains("managerApplyEligible") -and $control.Contains("managerNoAttemptPreserved")) "Host112 general guest repair is independent from manager eligibility"
|
||||
Add-SyntheticCheck "recovery:host112_timeout_contract" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120") -and $control.Contains('$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)')) "Host112 executor, SSH lock, queue and client deadlines have explicit reserves"
|
||||
|
||||
$failures = @($checks | Where-Object { -not $_.ok })
|
||||
$result = [pscustomobject]@{
|
||||
|
||||
@@ -13,7 +13,9 @@
|
||||
"staleMinutes": 15,
|
||||
"hardStaleMinutes": 60
|
||||
},
|
||||
"sshUsers": {},
|
||||
"sshUsers": {
|
||||
"192.168.0.112": "kali"
|
||||
},
|
||||
"vmrunPath": "",
|
||||
"hosts": [
|
||||
"192.168.0.110",
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# Agent99 Host112 controlled recovery contract
|
||||
|
||||
Host112 recovery is a bounded, correlated controlled-apply lane. Agent99 uses
|
||||
the Windows99 `agent99_ed25519` identity directly to `kali@192.168.0.112`; it
|
||||
does not fall back through the 110 jump route for this lane.
|
||||
|
||||
## Candidate separation
|
||||
|
||||
- The general guest candidate covers systemd, console/LightDM, VMware Tools,
|
||||
SSH, Wazuh Indexer, and the recovery timer.
|
||||
- The Wazuh manager candidate is evaluated separately. A healthy manager, a
|
||||
manager cooldown, or a manager resource block does not suppress a valid
|
||||
general guest repair candidate.
|
||||
- When the manager was already verified healthy, a general-only apply must
|
||||
retain `manager_attempt_count=0` and
|
||||
`manager_runtime_write_performed=0`.
|
||||
|
||||
## Apply closure
|
||||
|
||||
Before any apply, Agent99 runs the correlated `sudo --dry-run` with the same
|
||||
`trace_id`, `run_id`, and `work_item_id`. The dry-run must have a working direct
|
||||
transport, matching identity and boot, a readable preflight result, and no
|
||||
runtime or receipt write.
|
||||
|
||||
A manager recovery is verified only when all of the following are true for the
|
||||
same apply:
|
||||
|
||||
1. The direct apply transport completed and the executor returned healthy.
|
||||
2. The apply emitted a new immutable receipt with the matching identity and
|
||||
boot, `terminal=verified_healthy`, independent manager verifier ready,
|
||||
`manager_runtime_write_performed=1`, and `manager_attempt_count=1`.
|
||||
3. A second correlated `sudo --dry-run` reloads the durable receipt and returns
|
||||
the exact same receipt path, SHA-256, identity, boot, and terminal.
|
||||
4. A final ordinary `--check` independently verifies current guest and manager
|
||||
runtime health on the same boot.
|
||||
|
||||
The last check is evidence only. A timer or external process that makes the
|
||||
manager healthy after a failed apply cannot replace the apply transport or its
|
||||
immutable receipt and therefore cannot turn that run green.
|
||||
|
||||
## Timeout chain
|
||||
|
||||
- Host executor absolute deadline: 420 seconds.
|
||||
- Agent99 apply transport: 510 seconds, leaving 90 seconds for transport
|
||||
teardown and receipt delivery.
|
||||
- Worst-case Host112 chain, including five 90-second SSH lock waits: 1080
|
||||
seconds.
|
||||
- Queue child outer deadline: 1680 seconds, leaving 600 seconds after the
|
||||
Host112 chain.
|
||||
- Dispatch client deadline: 1800 seconds, leaving 120 seconds after the queue
|
||||
child deadline.
|
||||
|
||||
The control plane validates these reserves before attempting Host112 recovery.
|
||||
@@ -235,7 +235,13 @@ def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None:
|
||||
assert 'if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt "$EXECUTOR_DEADLINE_SECONDS" ]; then' in source
|
||||
assert 'if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -gt "$MANAGER_VERIFY_TIMEOUT_SECONDS" ]; then' in source
|
||||
assert "TimeoutStartSec=480" in service
|
||||
assert "Invoke-HostSshText $targetHost $applyCommand 590 1" in control
|
||||
assert "$script:Agent99Host112ExecutorBudgetSeconds = 420" in control
|
||||
assert "$script:Agent99Host112ApplyTransportTimeoutSeconds = 510" in control
|
||||
assert "Invoke-AgentHost112SshText $applyCommand $script:Agent99Host112ApplyTransportTimeoutSeconds 1" in control
|
||||
assert "executorReserveSeconds -ge 60" in control
|
||||
assert "queueReserveSeconds -ge 300" in control
|
||||
assert "clientReserveSeconds -ge 120" in control
|
||||
assert '$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)' in control
|
||||
|
||||
|
||||
def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> None:
|
||||
@@ -396,8 +402,8 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
|
||||
assert f'[string]`${parameter} = ""' in bootstrap
|
||||
assert 'reason = "controlled_dispatch_identity_unsafe"' in control
|
||||
assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
|
||||
assert '$before.managerPreflightStatus -eq "ready"' in function
|
||||
assert 'receiptStatus -eq "written"' in function
|
||||
assert '$dryRun.managerPreflightStatus -eq "ready"' in function
|
||||
assert 'receiptStatus -eq "written_immutable"' in function
|
||||
assert '$after.managerVerified' in function
|
||||
assert 'name = "wazuh_manager_service_active"' in function
|
||||
assert 'name = "wazuh_analysisd_process_present"' in function
|
||||
@@ -406,3 +412,94 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
|
||||
assert 'name = "wazuh_manager_api_55000_tcp"' in function
|
||||
assert "host112_readback_transport_unavailable" in function
|
||||
assert 'terminal = "no_write_control_identity_incomplete_or_unsafe"' in function
|
||||
|
||||
|
||||
def test_agent99_manager_success_requires_apply_and_immutable_receipt_chain() -> None:
|
||||
control = read(CONTROL)
|
||||
function = control[control.index("function Invoke-AgentHost112GuestRecovery") :]
|
||||
function = function[: function.index("function Test-HostReachability")]
|
||||
|
||||
assert "$applyTransport -and $applyTransport.ok" in function
|
||||
assert '$applyReceipt.durableReceiptTerminal -eq "verified_healthy"' in function
|
||||
assert "$applyReceipt.managerIndependentVerifierReady" in function
|
||||
assert "$applyReceipt.managerRuntimeWritePerformed" in function
|
||||
assert '[string]$applyReceipt.managerAttemptCount -eq "1"' in function
|
||||
assert "$durableReceiptReadback.receiptPath -eq $applyReceipt.receiptPath" in function
|
||||
assert "$durableReceiptReadback.receiptSha256 -eq $applyReceipt.receiptSha256" in function
|
||||
assert "$durableReceiptReadback.durableReceiptTerminal -eq $applyReceipt.durableReceiptTerminal" in function
|
||||
assert '$durableReceiptReadback.durableReceiptTerminal -eq "verified_healthy"' in function
|
||||
assert "$afterManagerVerifierVerified" in function
|
||||
assert 'afterVerifierRole = "independent_second_verifier_only"' in function
|
||||
assert "$after.ready -and" in function
|
||||
assert "$managerApplyVerified" in function
|
||||
assert "existingReceiptTerminalAccepted" in function
|
||||
assert "immutable_receipt_replay_not_successful" in function
|
||||
assert "host112_failed_apply_cannot_be_washed_green" in read(
|
||||
ROOT / "agent99-synthetic-tests.ps1"
|
||||
)
|
||||
|
||||
|
||||
def test_agent99_uses_correlated_sudo_dry_run_over_canonical_direct_key() -> None:
|
||||
control = read(CONTROL)
|
||||
|
||||
assert "function Invoke-AgentHost112SshText" in control
|
||||
assert 'expectedUser = "kali"' in control
|
||||
assert 'route = "direct_host112_kali"' in control
|
||||
assert "Invoke-SshText -TargetHost $transportContract.targetHost" in control
|
||||
assert "--dry-run --trace-id {0} --run-id {1} --work-item-id {2}" in control
|
||||
assert "Invoke-AgentHost112SshText $dryRunCommand" in control
|
||||
assert "$dryRun.identityMatches" in control
|
||||
assert "$dryRun.managerPreflightPresent" in control
|
||||
assert "$dryRun.transportOk" in control
|
||||
assert "-not $dryRun.durableReceiptPresent" in control
|
||||
|
||||
|
||||
def test_unhealthy_executor_exit_one_is_delivered_readback_not_transport_failure() -> None:
|
||||
control = read(CONTROL)
|
||||
function = control[control.index("function Invoke-AgentHost112GuestRecovery") :]
|
||||
function = function[: function.index("function Test-HostReachability")]
|
||||
synthetic = read(ROOT / "agent99-synthetic-tests.ps1")
|
||||
|
||||
assert function.count('case "$executor_exit" in 0|1) exit 0') == 3
|
||||
assert "$before.executorExitCode -in @(\"0\", \"1\")" in function
|
||||
assert "$before.readbackPresent -and" in function
|
||||
assert "$before.identityMatches -and" in function
|
||||
assert '$beforeTransport = Invoke-AgentHost112SshText $checkCommand' in function
|
||||
assert 'New-Host112SyntheticRow "check" 0 0 "none" 0 "not_requested" 0 "unknown" 1' in synthetic
|
||||
assert "host112_general_candidate_not_blocked_by_healthy_manager" in synthetic
|
||||
|
||||
|
||||
def test_general_guest_candidate_is_not_blocked_by_manager_candidate() -> None:
|
||||
control = read(CONTROL)
|
||||
function = control[control.index("function Invoke-AgentHost112GuestRecovery") :]
|
||||
function = function[: function.index("function Test-HostReachability")]
|
||||
synthetic = read(ROOT / "agent99-synthetic-tests.ps1")
|
||||
|
||||
assert "$generalCandidateRequired = [bool]($beforeVerified -and -not $before.generalReady)" in function
|
||||
assert "$managerCandidateRequired = [bool]($beforeVerified -and -not $before.managerVerified)" in function
|
||||
assert "$generalApplyEligible = [bool]($generalCandidateRequired -and $generalDryRunVerified)" in function
|
||||
assert "$managerApplyEligible = [bool]($managerCandidateRequired -and $managerDryRunVerified)" in function
|
||||
assert "$applyEligible = [bool]($generalApplyEligible -or $managerApplyEligible)" in function
|
||||
assert "managerNoAttemptPreserved" in function
|
||||
assert "host112_general_candidate_not_blocked_by_healthy_manager" in synthetic
|
||||
assert '-not $generalOnly.managerCandidateRequired' in synthetic
|
||||
assert '[string]$generalOnly.applyReceipt.managerAttemptCount -eq "0"' in synthetic
|
||||
assert '-not $generalOnly.applyReceipt.managerRuntimeWritePerformed' in synthetic
|
||||
|
||||
|
||||
def test_deploy_and_bootstrap_canonicalize_host112_direct_transport() -> None:
|
||||
deploy = read(ROOT / "agent99-deploy.ps1")
|
||||
bootstrap = read(BOOTSTRAP)
|
||||
generic_config = read(ROOT / "agent99.config.example.json")
|
||||
host_config = read(ROOT / "agent99.config.99.example.json")
|
||||
|
||||
assert 'Set-AgentProperty $config.sshUsers "192.168.0.112" "kali"' in deploy
|
||||
assert 'name = "host112_canonical_direct_route"' in deploy
|
||||
assert 'Host112 canonical direct SSH config post-verifier failed.' in deploy
|
||||
assert "Ensure-AgentHost112CanonicalSshConfig" in bootstrap
|
||||
assert 'Set-AgentBootstrapProperty $config.sshUsers "192.168.0.112" "kali"' in bootstrap
|
||||
assert 'Host112 canonical direct SSH config bootstrap verifier failed.' in bootstrap
|
||||
for config in (generic_config, host_config):
|
||||
assert '"192.168.0.112": "kali"' in config
|
||||
assert '"directHosts"' in config
|
||||
assert '"192.168.0.112"' in config
|
||||
|
||||
Reference in New Issue
Block a user