fix(recovery): close host112 wazuh manager loop
This commit is contained in:
@@ -188,11 +188,14 @@ param(
|
||||
[ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")]
|
||||
[string]`$Mode = "Status",
|
||||
[switch]`$ControlledApply,
|
||||
[string]`$AutomationRunId = "",
|
||||
[string]`$TraceId = "",
|
||||
[string]`$WorkItemId = "",
|
||||
[switch]`$SuppressAlerts,
|
||||
[string]`$SuppressReason = ""
|
||||
)
|
||||
|
||||
& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir"
|
||||
& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -AutomationRunId `$AutomationRunId -TraceId `$TraceId -WorkItemId `$WorkItemId -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir"
|
||||
"@ | Set-Content -Encoding UTF8 $launcher
|
||||
|
||||
$submitCmd = Join-Path $AgentRoot "agent99-submit-request.cmd"
|
||||
|
||||
@@ -2,6 +2,9 @@ param(
|
||||
[ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")]
|
||||
[string]$Mode = "Status",
|
||||
[switch]$ControlledApply,
|
||||
[string]$AutomationRunId = "",
|
||||
[string]$TraceId = "",
|
||||
[string]$WorkItemId = "",
|
||||
[switch]$SelfTestTelegramCard,
|
||||
[switch]$SelfTestTelegramDelivery,
|
||||
[switch]$SelfTestSensorGate,
|
||||
@@ -4940,6 +4943,12 @@ function Invoke-AgentQueuedCommands {
|
||||
}
|
||||
$hasDispatchIdentity = [bool]($automationRunId -or $traceId -or $workItemId -or $idempotencyKey -or $incidentId -or $projectId -or $sourceFingerprint -or $dispatchRouteId -or $singleFlightKey -or $lockOwner -or $canonicalDigest)
|
||||
$dispatchIdentityIncomplete = [bool](-not $automationRunId -or -not $traceId -or -not $workItemId -or -not $idempotencyKey -or -not $incidentId -or -not $projectId -or -not $sourceFingerprint -or -not $dispatchRouteId -or -not $singleFlightKey -or -not $lockOwner -or -not $canonicalDigest -or -not $approvalIdentityPresent -or -not $executionGeneration)
|
||||
$controlIdentityPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+\-]{0,127}$"
|
||||
$dispatchControlIdentitySafe = [bool](
|
||||
$automationRunId -match $controlIdentityPattern -and
|
||||
$traceId -match $controlIdentityPattern -and
|
||||
$workItemId -match $controlIdentityPattern
|
||||
)
|
||||
if (($controlled -and $modeName -eq "Recover" -and -not $hasDispatchIdentity) -or ($hasDispatchIdentity -and $dispatchIdentityIncomplete)) {
|
||||
$target = Join-Path $failedDir ("failed-" + $file.Name)
|
||||
Move-Item -Force $file.FullName $target
|
||||
@@ -4956,6 +4965,22 @@ function Invoke-AgentQueuedCommands {
|
||||
$results += $result
|
||||
continue
|
||||
}
|
||||
if ($hasDispatchIdentity -and -not $dispatchControlIdentitySafe) {
|
||||
$target = Join-Path $failedDir ("failed-" + $file.Name)
|
||||
Move-Item -Force $file.FullName $target
|
||||
$result = [pscustomobject]@{
|
||||
id = $id
|
||||
mode = $modeName
|
||||
source = $commandSource
|
||||
ok = $false
|
||||
reason = "controlled_dispatch_identity_unsafe"
|
||||
runtimeWritePerformed = $false
|
||||
file = $target
|
||||
}
|
||||
Record-AgentEvent "operator_command_rejected" "warning" "id=$id mode=$modeName reason=controlled_dispatch_identity_unsafe" $result -Alert
|
||||
$results += $result
|
||||
continue
|
||||
}
|
||||
|
||||
$runningPath = Join-Path $queueDir ("running-" + $file.Name)
|
||||
Move-Item -Force $file.FullName $runningPath
|
||||
@@ -4965,6 +4990,9 @@ function Invoke-AgentQueuedCommands {
|
||||
$stderrPath = Join-Path $EvidenceDir "agent99-command-$id.err"
|
||||
$args = @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", $modeName)
|
||||
if ($controlled) { $args += "-ControlledApply" }
|
||||
if ($automationRunId) { $args += @("-AutomationRunId", $automationRunId) }
|
||||
if ($traceId) { $args += @("-TraceId", $traceId) }
|
||||
if ($workItemId) { $args += @("-WorkItemId", $workItemId) }
|
||||
if ($suppressTelegram) { $args += @("-SuppressAlerts", "-SuppressReason", "do_not_page") }
|
||||
$process = Start-Process -FilePath "powershell.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru
|
||||
$completed = $process.WaitForExit(600000)
|
||||
@@ -5351,7 +5379,12 @@ function Invoke-AgentProviderFreshnessTriage {
|
||||
}
|
||||
|
||||
function Convert-AgentHost112GuestReadback {
|
||||
param([object]$Transport)
|
||||
param(
|
||||
[object]$Transport,
|
||||
[string]$ExpectedTraceId,
|
||||
[string]$ExpectedRunId,
|
||||
[string]$ExpectedWorkItemId
|
||||
)
|
||||
|
||||
$values = @{}
|
||||
$text = if ($Transport -and $Transport.PSObject.Properties["output"]) { [string]$Transport.output } else { "" }
|
||||
@@ -5367,19 +5400,61 @@ function Convert-AgentHost112GuestReadback {
|
||||
param([string]$Name)
|
||||
[bool]((& $get $Name "0") -eq "1")
|
||||
}
|
||||
$readbackPresent = [bool](
|
||||
$values.ContainsKey("schema_version") -and
|
||||
$values.ContainsKey("boot_id") -and
|
||||
$values.ContainsKey("guest_ready") -and
|
||||
$values.ContainsKey("manager_preflight_status")
|
||||
)
|
||||
$identityPresent = [bool](
|
||||
$values.ContainsKey("trace_id") -and
|
||||
$values.ContainsKey("run_id") -and
|
||||
$values.ContainsKey("work_item_id")
|
||||
)
|
||||
$identityMatches = [bool](
|
||||
$identityPresent -and
|
||||
((& $get "trace_id") -eq $ExpectedTraceId) -and
|
||||
((& $get "run_id") -eq $ExpectedRunId) -and
|
||||
((& $get "work_item_id") -eq $ExpectedWorkItemId)
|
||||
)
|
||||
$analysisdPresent = [bool](((& $get "wazuh_analysisd_process_count" "0") -match "^[1-9][0-9]*$"))
|
||||
$managerServiceActive = [bool](((& $get "wazuh_manager") -eq "active"))
|
||||
$agentEventPortReady = (& $asBool "wazuh_agent_event_port_1514_ready")
|
||||
$agentEnrollmentPortReady = (& $asBool "wazuh_agent_enrollment_port_1515_tcp")
|
||||
$managerApiPortReady = (& $asBool "wazuh_manager_api_port_55000_tcp")
|
||||
$managerVerified = [bool](
|
||||
$managerServiceActive -and
|
||||
$analysisdPresent -and
|
||||
$agentEventPortReady -and
|
||||
$agentEnrollmentPortReady -and
|
||||
$managerApiPortReady
|
||||
)
|
||||
$checks = @(
|
||||
[pscustomobject]@{ name = "readback_present"; passed = [bool]($values.ContainsKey("boot_id") -and $values.ContainsKey("guest_ready")) },
|
||||
[pscustomobject]@{ name = "transport_ok"; passed = [bool]($Transport -and $Transport.ok) },
|
||||
[pscustomobject]@{ name = "readback_present"; passed = $readbackPresent },
|
||||
[pscustomobject]@{ name = "control_identity_present"; passed = $identityPresent },
|
||||
[pscustomobject]@{ name = "control_identity_matches"; passed = $identityMatches },
|
||||
[pscustomobject]@{ name = "systemd_running"; passed = [bool]((& $get "systemd_state") -eq "running") },
|
||||
[pscustomobject]@{ name = "console_ready"; passed = (& $asBool "console_ready") },
|
||||
[pscustomobject]@{ name = "wazuh_manager_service_active"; passed = $managerServiceActive },
|
||||
[pscustomobject]@{ name = "wazuh_analysisd_process_present"; passed = $analysisdPresent },
|
||||
[pscustomobject]@{ name = "wazuh_agent_event_1514_ready"; passed = $agentEventPortReady },
|
||||
[pscustomobject]@{ name = "wazuh_agent_enrollment_1515_tcp"; passed = $agentEnrollmentPortReady },
|
||||
[pscustomobject]@{ name = "wazuh_manager_api_55000_tcp"; passed = $managerApiPortReady },
|
||||
[pscustomobject]@{ name = "services_ready"; passed = (& $asBool "services_ready") },
|
||||
[pscustomobject]@{ name = "recovery_timer_ready"; passed = (& $asBool "recovery_timer_ready") },
|
||||
[pscustomobject]@{ name = "guest_ready"; passed = (& $asBool "guest_ready") }
|
||||
)
|
||||
$failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_host112_guest_readback_v1"
|
||||
readbackPresent = [bool]($values.ContainsKey("boot_id") -and $values.ContainsKey("guest_ready"))
|
||||
schemaVersion = "agent99_host112_guest_readback_v2"
|
||||
readbackPresent = $readbackPresent
|
||||
ready = [bool]($failedChecks.Count -eq 0)
|
||||
identityPresent = $identityPresent
|
||||
identityMatches = $identityMatches
|
||||
traceId = & $get "trace_id"
|
||||
runId = & $get "run_id"
|
||||
workItemId = & $get "work_item_id"
|
||||
bootId = & $get "boot_id"
|
||||
uptimeSeconds = & $get "uptime_seconds" "-1"
|
||||
systemdState = & $get "systemd_state"
|
||||
@@ -5391,6 +5466,21 @@ function Convert-AgentHost112GuestReadback {
|
||||
xorg = & $get "xorg"
|
||||
wazuhIndexer = & $get "wazuh_indexer"
|
||||
wazuhManager = & $get "wazuh_manager"
|
||||
wazuhManagerResult = & $get "wazuh_manager_result"
|
||||
managerFailedStateObserved = (& $asBool "manager_failed_state_observed")
|
||||
wazuhAnalysisdProcessCount = & $get "wazuh_analysisd_process_count" "0"
|
||||
managerOrphanAnalysisd = (& $asBool "manager_orphan_analysisd")
|
||||
managerMemoryAvailableKb = & $get "manager_memory_available_kb" "0"
|
||||
managerDiskAvailableKb = & $get "manager_disk_available_kb" "0"
|
||||
managerDiskUsedPercent = & $get "manager_disk_used_percent" "100"
|
||||
managerResourceReady = (& $asBool "manager_resource_ready")
|
||||
managerPreflightStatus = & $get "manager_preflight_status"
|
||||
managerVerified = $managerVerified
|
||||
wazuhAgentEventPort1514Tcp = (& $asBool "wazuh_agent_event_port_1514_tcp")
|
||||
wazuhAgentEventPort1514Udp = (& $asBool "wazuh_agent_event_port_1514_udp")
|
||||
wazuhAgentEventPort1514Ready = $agentEventPortReady
|
||||
wazuhAgentEnrollmentPort1515Tcp = $agentEnrollmentPortReady
|
||||
wazuhManagerApiPort55000Tcp = $managerApiPortReady
|
||||
wazuhDashboard = & $get "wazuh_dashboard"
|
||||
filebeat = & $get "filebeat"
|
||||
consoleReady = (& $asBool "console_ready")
|
||||
@@ -5399,6 +5489,17 @@ function Convert-AgentHost112GuestReadback {
|
||||
guestReady = (& $asBool "guest_ready")
|
||||
actionCount = & $get "action_count" "0"
|
||||
actions = & $get "actions" "none"
|
||||
runtimeWritePerformed = (& $asBool "runtime_write_performed")
|
||||
managerRuntimeWritePerformed = (& $asBool "manager_runtime_write_performed")
|
||||
receiptWritePerformed = (& $asBool "receipt_write_performed")
|
||||
receiptStatus = & $get "receipt_status"
|
||||
receiptRef = & $get "receipt_ref" "none"
|
||||
managerAttemptCount = & $get "manager_attempt_count" "0"
|
||||
managerStartExit = & $get "manager_start_exit" "not_attempted"
|
||||
rollbackAttempted = (& $asBool "rollback_attempted")
|
||||
rollbackVerified = (& $asBool "rollback_verified")
|
||||
rollbackTerminal = & $get "rollback_terminal" "not_required"
|
||||
terminal = & $get "terminal" "observed"
|
||||
transportOk = [bool]($Transport -and $Transport.ok)
|
||||
transportExitCode = if ($Transport) { $Transport.exitCode } else { -1 }
|
||||
transportRoute = if ($Transport -and $Transport.PSObject.Properties["route"]) { [string]$Transport.route } else { "unknown" }
|
||||
@@ -5410,42 +5511,149 @@ function Convert-AgentHost112GuestReadback {
|
||||
}
|
||||
|
||||
function Invoke-AgentHost112GuestRecovery {
|
||||
$targetHost = "192.168.0.112"
|
||||
$checkCommand = "/usr/local/sbin/awoooi-host112-guest-readiness --check"
|
||||
$applyCommand = "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply"
|
||||
$beforeTransport = Invoke-HostSshText $targetHost $checkCommand 20 1
|
||||
$before = Convert-AgentHost112GuestReadback $beforeTransport
|
||||
$applyTransport = $null
|
||||
$after = $before
|
||||
$applyAttempted = $false
|
||||
param(
|
||||
[string]$RequestedRunId = "",
|
||||
[string]$RequestedTraceId = "",
|
||||
[string]$RequestedWorkItemId = "",
|
||||
[string]$FallbackRunId = ""
|
||||
)
|
||||
|
||||
if (-not $before.ready -and $ControlledApply) {
|
||||
$applyAttempted = $true
|
||||
Write-AgentLog "CONTROLLED_APPLY host112_guest_recovery failedChecks=$($before.failedChecks -join ',')"
|
||||
$applyTransport = Invoke-HostSshText $targetHost $applyCommand 300 1
|
||||
$afterTransport = Invoke-HostSshText $targetHost $checkCommand 20 1
|
||||
$after = Convert-AgentHost112GuestReadback $afterTransport
|
||||
$targetHost = "192.168.0.112"
|
||||
$identityPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+\-]{0,127}$"
|
||||
$hasRequestedIdentity = [bool]($RequestedRunId -or $RequestedTraceId -or $RequestedWorkItemId)
|
||||
$requestedIdentityComplete = [bool]($RequestedRunId -and $RequestedTraceId -and $RequestedWorkItemId)
|
||||
$effectiveRunId = $RequestedRunId
|
||||
$effectiveTraceId = $RequestedTraceId
|
||||
$effectiveWorkItemId = $RequestedWorkItemId
|
||||
if (-not $hasRequestedIdentity) {
|
||||
$effectiveRunId = if ($FallbackRunId) { $FallbackRunId } else { "host112-observe-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) }
|
||||
$effectiveTraceId = "trace:$effectiveRunId"
|
||||
$effectiveWorkItemId = "HOST112-WAZUH-MANAGER-RECOVERY"
|
||||
}
|
||||
$identitySafe = [bool](
|
||||
(-not $hasRequestedIdentity -or $requestedIdentityComplete) -and
|
||||
$effectiveRunId -match $identityPattern -and
|
||||
$effectiveTraceId -match $identityPattern -and
|
||||
$effectiveWorkItemId -match $identityPattern
|
||||
)
|
||||
if (-not $identitySafe) {
|
||||
$before = Convert-AgentHost112GuestReadback $null $effectiveTraceId $effectiveRunId $effectiveWorkItemId
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_host112_guest_recovery_v2"
|
||||
targetHost = $targetHost
|
||||
traceId = $effectiveTraceId
|
||||
runId = $effectiveRunId
|
||||
workItemId = $effectiveWorkItemId
|
||||
controlledApply = [bool]$ControlledApply
|
||||
applyEligible = $false
|
||||
applyAttempted = $false
|
||||
applyBlockedReason = "control_identity_incomplete_or_unsafe"
|
||||
runtimeWritePerformed = $false
|
||||
changed = $false
|
||||
managerVerified = $false
|
||||
verified = $false
|
||||
receiptVerified = $false
|
||||
terminal = "no_write_control_identity_incomplete_or_unsafe"
|
||||
before = $before
|
||||
after = $before
|
||||
applyReceipt = $null
|
||||
rollback = [pscustomobject]@{ attempted = $false; verified = $false; terminal = "not_required" }
|
||||
verifier = "wazuh_manager_service_process_agent_event_1514_enrollment_1515_and_api_55000"
|
||||
prohibitedActions = @("host_reboot", "vm_power_change", "vm_reset", "secret_read", "firewall_change", "database_write")
|
||||
}
|
||||
}
|
||||
|
||||
$verified = [bool]$after.ready
|
||||
$changed = [bool]($applyAttempted -and $verified -and -not $before.ready)
|
||||
$checkCommand = "/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId"
|
||||
$applyCommand = "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId"
|
||||
$beforeTransport = Invoke-HostSshText $targetHost $checkCommand 30 1
|
||||
$before = Convert-AgentHost112GuestReadback $beforeTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
|
||||
$applyTransport = $null
|
||||
$applyReceipt = $null
|
||||
$after = $before
|
||||
$applyAttempted = $false
|
||||
$receiptVerified = $false
|
||||
$applyEligible = [bool](
|
||||
$before.transportOk -and
|
||||
$before.identityMatches -and
|
||||
-not $before.managerVerified -and
|
||||
$before.managerPreflightStatus -eq "ready"
|
||||
)
|
||||
$applyBlockedReason = if ($before.managerVerified) {
|
||||
"idempotent_already_verified_healthy"
|
||||
} elseif (-not $before.transportOk) {
|
||||
"host112_readback_transport_unavailable"
|
||||
} elseif (-not $before.identityMatches) {
|
||||
"control_identity_readback_mismatch"
|
||||
} elseif ($before.managerPreflightStatus -ne "ready") {
|
||||
[string]$before.managerPreflightStatus
|
||||
} else {
|
||||
"none"
|
||||
}
|
||||
|
||||
if ($ControlledApply -and $applyEligible) {
|
||||
$applyAttempted = $true
|
||||
Write-AgentLog "CONTROLLED_APPLY host112_guest_recovery traceId=$effectiveTraceId runId=$effectiveRunId workItemId=$effectiveWorkItemId preflight=$($before.managerPreflightStatus)"
|
||||
$applyTransport = Invoke-HostSshText $targetHost $applyCommand 590 1
|
||||
$applyReceipt = Convert-AgentHost112GuestReadback $applyTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
|
||||
$receiptVerified = [bool](
|
||||
$applyReceipt.identityMatches -and
|
||||
$applyReceipt.receiptWritePerformed -and
|
||||
$applyReceipt.receiptStatus -eq "written" -and
|
||||
$applyReceipt.receiptRef -ne "none"
|
||||
)
|
||||
$afterTransport = Invoke-HostSshText $targetHost $checkCommand 30 1
|
||||
$after = Convert-AgentHost112GuestReadback $afterTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId
|
||||
}
|
||||
|
||||
$managerVerified = [bool]$after.managerVerified
|
||||
$verified = [bool]($after.ready -and (-not $applyAttempted -or $receiptVerified))
|
||||
$changed = [bool](
|
||||
$applyAttempted -and
|
||||
$applyReceipt -and
|
||||
$applyReceipt.managerRuntimeWritePerformed -and
|
||||
$managerVerified -and
|
||||
$receiptVerified
|
||||
)
|
||||
$terminal = if ($applyAttempted -and $applyReceipt) {
|
||||
[string]$applyReceipt.terminal
|
||||
} elseif ($before.managerVerified) {
|
||||
"idempotent_already_verified_healthy"
|
||||
} elseif ($ControlledApply) {
|
||||
"no_write_$applyBlockedReason"
|
||||
} else {
|
||||
"observed_no_write"
|
||||
}
|
||||
$result = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host112_guest_recovery_v1"
|
||||
schemaVersion = "agent99_host112_guest_recovery_v2"
|
||||
targetHost = $targetHost
|
||||
traceId = $effectiveTraceId
|
||||
runId = $effectiveRunId
|
||||
workItemId = $effectiveWorkItemId
|
||||
controlledApply = [bool]$ControlledApply
|
||||
applyEligible = $applyEligible
|
||||
applyAttempted = $applyAttempted
|
||||
applyBlockedReason = $applyBlockedReason
|
||||
applyExitCode = if ($applyTransport) { $applyTransport.exitCode } else { $null }
|
||||
runtimeWritePerformed = [bool]($applyReceipt -and $applyReceipt.runtimeWritePerformed)
|
||||
changed = $changed
|
||||
managerVerified = $managerVerified
|
||||
verified = $verified
|
||||
receiptVerified = $receiptVerified
|
||||
terminal = $terminal
|
||||
before = $before
|
||||
after = $after
|
||||
rollback = "systemd package backup under /var/lib/awoooi-host112-recovery/backups"
|
||||
verifier = "forced_command_host112_guest_readiness_check"
|
||||
applyReceipt = $applyReceipt
|
||||
rollback = [pscustomobject]@{
|
||||
attempted = [bool]($applyReceipt -and $applyReceipt.rollbackAttempted)
|
||||
verified = [bool]($applyReceipt -and $applyReceipt.rollbackVerified)
|
||||
terminal = if ($applyReceipt) { [string]$applyReceipt.rollbackTerminal } else { "not_required" }
|
||||
}
|
||||
verifier = "wazuh_manager_service_process_agent_event_1514_enrollment_1515_and_api_55000"
|
||||
prohibitedActions = @("host_reboot", "vm_power_change", "vm_reset", "secret_read", "firewall_change", "database_write")
|
||||
}
|
||||
if ($changed) {
|
||||
Record-AgentEvent "host112_guest_recovered" "info" "112 圖形登入、VMware Tools、Wazuh 與 recovery timer 已由 Agent99 修復並驗證。" $result -Alert
|
||||
} elseif ($applyAttempted -and -not $verified) {
|
||||
} elseif ($applyAttempted -and (-not $verified -or -not $receiptVerified)) {
|
||||
Record-AgentEvent "host112_guest_recovery_failed" "critical" "112 受控修復後仍未通過 verifier;保留 failed checks 與 rollback evidence。" $result -Alert
|
||||
}
|
||||
$result
|
||||
@@ -5756,7 +5964,11 @@ $recoverySlo = $null
|
||||
$recoveryCoordinator = $null
|
||||
$recoveryPhases = @()
|
||||
$recoveryStartedAt = if ($Mode -eq "Recover") { Get-Date } else { $null }
|
||||
$recoveryRunId = if ($Mode -eq "Recover") { "agent99-recovery-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) } else { "" }
|
||||
$recoveryRunId = if ($Mode -eq "Recover") {
|
||||
if ($AutomationRunId) { $AutomationRunId } else { "agent99-recovery-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) }
|
||||
} else {
|
||||
""
|
||||
}
|
||||
$recoveryConfig = Get-AgentRecoverySloConfig
|
||||
|
||||
if ($Mode -in @("Status", "Recover")) {
|
||||
@@ -5792,7 +6004,11 @@ if ($Mode -in @("Status", "Recover")) {
|
||||
}
|
||||
|
||||
if ($Mode -in @("Status", "Recover")) {
|
||||
$host112Recovery = Invoke-AgentHost112GuestRecovery
|
||||
$host112Recovery = Invoke-AgentHost112GuestRecovery `
|
||||
-RequestedRunId $AutomationRunId `
|
||||
-RequestedTraceId $TraceId `
|
||||
-RequestedWorkItemId $WorkItemId `
|
||||
-FallbackRunId $recoveryRunId
|
||||
if ($Mode -eq "Recover") {
|
||||
$recoveryPhases += New-AgentRecoveryPhase "host112_guest_readiness" $(if ($host112Recovery.verified) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds)
|
||||
} elseif (-not $host112Recovery.verified -and -not $recoveryTrigger) {
|
||||
|
||||
@@ -126,11 +126,23 @@ def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() ->
|
||||
function = source[source.index("function Convert-AgentHost112GuestReadback") :]
|
||||
function = function[: function.index("function Test-HostReachability")]
|
||||
|
||||
assert '"/usr/local/sbin/awoooi-host112-guest-readiness --check"' in function
|
||||
assert '"sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply"' in function
|
||||
assert '"/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId"' in function
|
||||
assert '"sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId"' in function
|
||||
assert function.count("Invoke-HostSshText $targetHost $checkCommand") == 2
|
||||
assert 'schemaVersion = "agent99_host112_guest_recovery_v1"' in function
|
||||
assert 'verifier = "forced_command_host112_guest_readiness_check"' in function
|
||||
assert 'schemaVersion = "agent99_host112_guest_recovery_v2"' in function
|
||||
assert '$before.managerPreflightStatus -eq "ready"' in function
|
||||
assert '$before.transportOk -and' in function
|
||||
assert '$before.identityMatches -and' in function
|
||||
assert "Invoke-HostSshText $targetHost $applyCommand 590 1" in function
|
||||
assert 'receiptStatus -eq "written"' in function
|
||||
assert 'verifier = "wazuh_manager_service_process_agent_event_1514_enrollment_1515_and_api_55000"' in function
|
||||
for verifier_field in (
|
||||
"wazuhAnalysisdProcessCount",
|
||||
"wazuhAgentEventPort1514Ready",
|
||||
"wazuhAgentEnrollmentPort1515Tcp",
|
||||
"wazuhManagerApiPort55000Tcp",
|
||||
):
|
||||
assert verifier_field in function
|
||||
for forbidden in ("host_reboot", "vm_power_change", "vm_reset", "secret_read"):
|
||||
assert f'"{forbidden}"' in function
|
||||
|
||||
|
||||
@@ -5,8 +5,10 @@ Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply
|
||||
TimeoutStartSec=300
|
||||
ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer
|
||||
# Script phase defaults sum to 500 seconds; the unit owns the 600-second
|
||||
# outer contract while every child command remains independently bounded.
|
||||
TimeoutStartSec=600
|
||||
User=root
|
||||
Group=root
|
||||
NoNewPrivileges=true
|
||||
|
||||
@@ -1,36 +1,186 @@
|
||||
#!/usr/bin/env bash
|
||||
# Host 112 guest readiness and bounded recovery executor.
|
||||
# Version: 2.0
|
||||
# Last modified: 2026-07-14 11:00 (Asia/Taipei)
|
||||
# Last modified by: Codex
|
||||
# Change: add correlated, cooldown-guarded Wazuh manager recovery and an
|
||||
# independent process/port verifier without reboot, VM power, firewall, or
|
||||
# secret access.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
MODE="check"
|
||||
MODE_EXPLICIT=0
|
||||
TRACE_ID=""
|
||||
RUN_ID=""
|
||||
WORK_ITEM_ID=""
|
||||
|
||||
INDEXER_RETRY_AFTER_SECONDS="${HOST112_INDEXER_RETRY_AFTER_SECONDS:-210}"
|
||||
INDEXER_RETRY_COOLDOWN_SECONDS="${HOST112_INDEXER_RETRY_COOLDOWN_SECONDS:-600}"
|
||||
INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-240}"
|
||||
INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-180}"
|
||||
SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-30}"
|
||||
MANAGER_RETRY_AFTER_SECONDS="${HOST112_MANAGER_RETRY_AFTER_SECONDS:-120}"
|
||||
MANAGER_RETRY_COOLDOWN_SECONDS="${HOST112_MANAGER_RETRY_COOLDOWN_SECONDS:-600}"
|
||||
MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}"
|
||||
MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-120}"
|
||||
MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}"
|
||||
MANAGER_VERIFY_INTERVAL_SECONDS="${HOST112_MANAGER_VERIFY_INTERVAL_SECONDS:-3}"
|
||||
MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-45}"
|
||||
MANAGER_MIN_MEMORY_AVAILABLE_KB="${HOST112_MANAGER_MIN_MEMORY_AVAILABLE_KB:-524288}"
|
||||
MANAGER_MIN_DISK_AVAILABLE_KB="${HOST112_MANAGER_MIN_DISK_AVAILABLE_KB:-1048576}"
|
||||
MANAGER_MAX_DISK_USED_PERCENT="${HOST112_MANAGER_MAX_DISK_USED_PERCENT:-95}"
|
||||
STATE_DIR="${HOST112_RECOVERY_STATE_DIR:-/var/lib/awoooi-host112-recovery}"
|
||||
RETRY_MARKER="${HOST112_INDEXER_RETRY_MARKER:-/run/awoooi-host112-indexer-retry.epoch}"
|
||||
INDEXER_RETRY_MARKER="${HOST112_INDEXER_RETRY_MARKER:-/run/awoooi-host112-indexer-retry.epoch}"
|
||||
MANAGER_RETRY_MARKER="${HOST112_MANAGER_RETRY_MARKER:-/run/awoooi-host112-manager-retry.epoch}"
|
||||
MANAGER_LOCK_FILE="${HOST112_MANAGER_LOCK_FILE:-/run/awoooi-host112-manager-recovery.lock}"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: host112-guest-readiness.sh [--check|--apply]
|
||||
Usage:
|
||||
host112-guest-readiness.sh --check [identity]
|
||||
host112-guest-readiness.sh --dry-run --trace-id ID --run-id ID --work-item-id ID
|
||||
host112-guest-readiness.sh --apply --trace-id ID --run-id ID --work-item-id ID
|
||||
|
||||
--check emits a bounded no-secret guest readiness row.
|
||||
Internal systemd mode:
|
||||
host112-guest-readiness.sh --apply-timer
|
||||
|
||||
--check emits a bounded no-secret guest readiness row and never writes.
|
||||
--dry-run evaluates the exact Wazuh manager preflight and never writes.
|
||||
--apply performs only allowlisted idempotent recovery for graphical.target,
|
||||
LightDM, open-vm-tools, SSH and a cooldown-guarded Wazuh Indexer start retry.
|
||||
LightDM, open-vm-tools, SSH, Wazuh Indexer, and one cooldown-guarded Wazuh
|
||||
manager reset/start attempt. The independent verifier requires the manager
|
||||
process plus agent event 1514, enrollment 1515, and manager API 55000 listeners.
|
||||
USAGE
|
||||
}
|
||||
|
||||
set_mode() {
|
||||
local requested="$1"
|
||||
if [ "$MODE_EXPLICIT" -eq 1 ]; then
|
||||
echo "BLOCKER duplicate_mode" >&2
|
||||
exit 64
|
||||
fi
|
||||
MODE="$requested"
|
||||
MODE_EXPLICIT=1
|
||||
}
|
||||
|
||||
require_value() {
|
||||
local flag="$1" value="${2:-}"
|
||||
if [ -z "$value" ]; then
|
||||
echo "BLOCKER missing_value=$flag" >&2
|
||||
exit 64
|
||||
fi
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--check) MODE="check" ;;
|
||||
--apply) MODE="apply" ;;
|
||||
--check) set_mode "check" ;;
|
||||
--dry-run) set_mode "dry_run" ;;
|
||||
--apply) set_mode "apply" ;;
|
||||
--apply-timer) set_mode "timer_apply" ;;
|
||||
--trace-id)
|
||||
require_value "$1" "${2:-}"
|
||||
TRACE_ID="$2"
|
||||
shift
|
||||
;;
|
||||
--run-id)
|
||||
require_value "$1" "${2:-}"
|
||||
RUN_ID="$2"
|
||||
shift
|
||||
;;
|
||||
--work-item-id)
|
||||
require_value "$1" "${2:-}"
|
||||
WORK_ITEM_ID="$2"
|
||||
shift
|
||||
;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "BLOCKER unknown_argument=$1" >&2; exit 64 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
safe_identity() {
|
||||
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$ ]]
|
||||
}
|
||||
|
||||
identity_count=0
|
||||
[ -n "$TRACE_ID" ] && identity_count=$((identity_count + 1))
|
||||
[ -n "$RUN_ID" ] && identity_count=$((identity_count + 1))
|
||||
[ -n "$WORK_ITEM_ID" ] && identity_count=$((identity_count + 1))
|
||||
if [ "$identity_count" -ne 0 ] && [ "$identity_count" -ne 3 ]; then
|
||||
echo "BLOCKER incomplete_control_identity" >&2
|
||||
exit 64
|
||||
fi
|
||||
if [ "$identity_count" -eq 3 ]; then
|
||||
for identity in "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID"; do
|
||||
if ! safe_identity "$identity"; then
|
||||
echo "BLOCKER unsafe_control_identity" >&2
|
||||
exit 64
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if { [ "$MODE" = "dry_run" ] || [ "$MODE" = "apply" ]; } && [ "$identity_count" -ne 3 ]; then
|
||||
echo "BLOCKER control_identity_required" >&2
|
||||
exit 64
|
||||
fi
|
||||
if [ "$MODE" = "timer_apply" ] && [ "$identity_count" -ne 0 ]; then
|
||||
echo "BLOCKER timer_identity_is_system_generated" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
actions=()
|
||||
runtime_write_performed=0
|
||||
manager_runtime_write_performed=0
|
||||
receipt_write_performed=0
|
||||
receipt_status="not_requested"
|
||||
receipt_ref="none"
|
||||
manager_attempt_count=0
|
||||
manager_start_exit=not_attempted
|
||||
manager_terminal="observed"
|
||||
rollback_attempted=0
|
||||
rollback_verified=0
|
||||
rollback_terminal=not_required
|
||||
|
||||
is_uint() {
|
||||
[[ "$1" =~ ^[0-9]+$ ]]
|
||||
}
|
||||
|
||||
for timing_value in \
|
||||
"$INDEXER_START_TIMEOUT_SECONDS" \
|
||||
"$SERVICE_START_TIMEOUT_SECONDS" \
|
||||
"$MANAGER_RESET_TIMEOUT_SECONDS" \
|
||||
"$MANAGER_START_TIMEOUT_SECONDS" \
|
||||
"$MANAGER_VERIFY_TIMEOUT_SECONDS" \
|
||||
"$MANAGER_VERIFY_INTERVAL_SECONDS" \
|
||||
"$MANAGER_ROLLBACK_TIMEOUT_SECONDS"; do
|
||||
if ! is_uint "$timing_value" || [ "$timing_value" -le 0 ]; then
|
||||
echo "BLOCKER invalid_timeout_configuration" >&2
|
||||
exit 64
|
||||
fi
|
||||
done
|
||||
APPLY_PHASE_TIMEOUT_BUDGET_SECONDS=$((
|
||||
(4 * SERVICE_START_TIMEOUT_SECONDS)
|
||||
+ INDEXER_START_TIMEOUT_SECONDS
|
||||
+ MANAGER_RESET_TIMEOUT_SECONDS
|
||||
+ MANAGER_START_TIMEOUT_SECONDS
|
||||
+ MANAGER_VERIFY_TIMEOUT_SECONDS
|
||||
+ MANAGER_ROLLBACK_TIMEOUT_SECONDS
|
||||
))
|
||||
if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt 570 ]; then
|
||||
echo "BLOCKER apply_phase_timeout_budget_exceeds_570_seconds" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
write_epoch_marker() {
|
||||
local marker="$1" epoch="$2" tmp
|
||||
tmp="${marker}.tmp.$$"
|
||||
if printf '%s\n' "$epoch" >"$tmp" \
|
||||
&& chmod 0644 "$tmp" \
|
||||
&& mv "$tmp" "$marker"; then
|
||||
return 0
|
||||
fi
|
||||
rm -f "$tmp" >/dev/null 2>&1 || true
|
||||
return 1
|
||||
}
|
||||
|
||||
unit_active() {
|
||||
local unit="$1" state
|
||||
@@ -44,8 +194,14 @@ unit_enabled() {
|
||||
printf '%s' "${state:-unknown}"
|
||||
}
|
||||
|
||||
unit_property() {
|
||||
local unit="$1" property="$2" state
|
||||
state="$(systemctl show "$unit" -p "$property" --value 2>/dev/null || true)"
|
||||
printf '%s' "${state:-unknown}"
|
||||
}
|
||||
|
||||
unit_loaded() {
|
||||
[ "$(systemctl show "$1" -p LoadState --value 2>/dev/null || true)" = "loaded" ]
|
||||
[ "$(unit_property "$1" LoadState)" = "loaded" ]
|
||||
}
|
||||
|
||||
record_action() {
|
||||
@@ -58,8 +214,9 @@ start_if_inactive() {
|
||||
if [ "$state" = "active" ] || [ "$state" = "activating" ]; then
|
||||
return 0
|
||||
fi
|
||||
if systemctl start "$unit"; then
|
||||
if timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl start "$unit"; then
|
||||
record_action "$action"
|
||||
runtime_write_performed=1
|
||||
else
|
||||
record_action "${action}_failed"
|
||||
fi
|
||||
@@ -73,21 +230,225 @@ enable_if_disabled() {
|
||||
fi
|
||||
if systemctl enable "$unit" >/dev/null; then
|
||||
record_action "$action"
|
||||
runtime_write_performed=1
|
||||
else
|
||||
record_action "${action}_failed"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "$MODE" = "apply" ]; then
|
||||
tcp_port_listening() {
|
||||
local port="$1"
|
||||
ss -H -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$port$"
|
||||
}
|
||||
|
||||
udp_port_listening() {
|
||||
local port="$1"
|
||||
ss -H -lun 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$port$"
|
||||
}
|
||||
|
||||
process_count() {
|
||||
local process_name="$1" count
|
||||
count="$(pgrep -xc "$process_name" 2>/dev/null || true)"
|
||||
printf '%s' "${count:-0}"
|
||||
}
|
||||
|
||||
memory_available_kb() {
|
||||
local available
|
||||
available="$(awk '/^MemAvailable:/ {print $2; exit}' /proc/meminfo 2>/dev/null || true)"
|
||||
printf '%s' "${available:-0}"
|
||||
}
|
||||
|
||||
disk_readback() {
|
||||
local target="/var/ossec" line
|
||||
[ -e "$target" ] || target="/"
|
||||
line="$(df -Pk "$target" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $4 " " $5; exit}')"
|
||||
if [ -n "$line" ]; then
|
||||
printf '%s' "$line"
|
||||
else
|
||||
printf '0 100'
|
||||
fi
|
||||
}
|
||||
|
||||
capture_manager_state() {
|
||||
local disk_values
|
||||
manager_load_state="$(unit_property wazuh-manager.service LoadState)"
|
||||
manager_active_state="$(unit_property wazuh-manager.service ActiveState)"
|
||||
manager_sub_state="$(unit_property wazuh-manager.service SubState)"
|
||||
manager_result="$(unit_property wazuh-manager.service Result)"
|
||||
manager_failed_state_observed=0
|
||||
if [ "$manager_active_state" = "failed" ] \
|
||||
|| { [ "$manager_result" != "success" ] && [ "$manager_result" != "unknown" ]; }; then
|
||||
manager_failed_state_observed=1
|
||||
fi
|
||||
analysisd_count="$(process_count wazuh-analysisd)"
|
||||
event_1514_tcp=0
|
||||
event_1514_udp=0
|
||||
enrollment_1515_tcp=0
|
||||
manager_api_55000_tcp=0
|
||||
tcp_port_listening 1514 && event_1514_tcp=1
|
||||
udp_port_listening 1514 && event_1514_udp=1
|
||||
tcp_port_listening 1515 && enrollment_1515_tcp=1
|
||||
tcp_port_listening 55000 && manager_api_55000_tcp=1
|
||||
manager_event_port_ready=0
|
||||
if [ "$event_1514_tcp" -eq 1 ] || [ "$event_1514_udp" -eq 1 ]; then
|
||||
manager_event_port_ready=1
|
||||
fi
|
||||
manager_orphan_analysisd=0
|
||||
if [ "$manager_active_state" != "active" ] && is_uint "$analysisd_count" && [ "$analysisd_count" -gt 0 ]; then
|
||||
manager_orphan_analysisd=1
|
||||
fi
|
||||
manager_verifier_ready=0
|
||||
if [ "$manager_active_state" = "active" ] \
|
||||
&& is_uint "$analysisd_count" && [ "$analysisd_count" -gt 0 ] \
|
||||
&& [ "$manager_event_port_ready" -eq 1 ] \
|
||||
&& [ "$enrollment_1515_tcp" -eq 1 ] \
|
||||
&& [ "$manager_api_55000_tcp" -eq 1 ]; then
|
||||
manager_verifier_ready=1
|
||||
fi
|
||||
manager_memory_available_kb="$(memory_available_kb)"
|
||||
disk_values="$(disk_readback)"
|
||||
read -r manager_disk_available_kb manager_disk_used_percent <<<"$disk_values"
|
||||
manager_resource_ready=0
|
||||
if is_uint "$manager_memory_available_kb" \
|
||||
&& is_uint "$manager_disk_available_kb" \
|
||||
&& is_uint "$manager_disk_used_percent" \
|
||||
&& [ "$manager_memory_available_kb" -ge "$MANAGER_MIN_MEMORY_AVAILABLE_KB" ] \
|
||||
&& [ "$manager_disk_available_kb" -ge "$MANAGER_MIN_DISK_AVAILABLE_KB" ] \
|
||||
&& [ "$manager_disk_used_percent" -lt "$MANAGER_MAX_DISK_USED_PERCENT" ]; then
|
||||
manager_resource_ready=1
|
||||
fi
|
||||
}
|
||||
|
||||
manager_cooldown_remaining() {
|
||||
local now_epoch="$1" last_retry_epoch=0 elapsed
|
||||
if [ -r "$MANAGER_RETRY_MARKER" ]; then
|
||||
read -r last_retry_epoch <"$MANAGER_RETRY_MARKER" || last_retry_epoch=0
|
||||
fi
|
||||
is_uint "$last_retry_epoch" || last_retry_epoch=0
|
||||
elapsed=$((now_epoch - last_retry_epoch))
|
||||
if [ "$elapsed" -lt 0 ]; then
|
||||
printf '%s' "$MANAGER_RETRY_COOLDOWN_SECONDS"
|
||||
elif [ "$elapsed" -lt "$MANAGER_RETRY_COOLDOWN_SECONDS" ]; then
|
||||
printf '%s' "$((MANAGER_RETRY_COOLDOWN_SECONDS - elapsed))"
|
||||
else
|
||||
printf '0'
|
||||
fi
|
||||
}
|
||||
|
||||
evaluate_manager_preflight() {
|
||||
local now_epoch="$1" uptime_for_retry="$2"
|
||||
manager_cooldown_remaining_seconds="$(manager_cooldown_remaining "$now_epoch")"
|
||||
manager_preflight_status="ready"
|
||||
if ! command -v systemctl >/dev/null 2>&1 \
|
||||
|| ! command -v timeout >/dev/null 2>&1 \
|
||||
|| ! command -v ss >/dev/null 2>&1 \
|
||||
|| ! command -v flock >/dev/null 2>&1; then
|
||||
manager_preflight_status="blocked_required_tool_missing"
|
||||
elif [ "$manager_load_state" != "loaded" ]; then
|
||||
manager_preflight_status="blocked_manager_unit_not_loaded"
|
||||
elif [ "$manager_verifier_ready" -eq 1 ]; then
|
||||
manager_preflight_status="already_verified_healthy"
|
||||
elif [ "$manager_active_state" = "active" ]; then
|
||||
manager_preflight_status="blocked_active_verifier_mismatch"
|
||||
elif [ "$manager_orphan_analysisd" -eq 1 ]; then
|
||||
manager_preflight_status="blocked_orphan_analysisd_no_write"
|
||||
elif [ "$manager_resource_ready" -ne 1 ]; then
|
||||
manager_preflight_status="blocked_resource_preflight_no_write"
|
||||
elif [ "$(unit_active wazuh-indexer.service)" != "active" ]; then
|
||||
manager_preflight_status="blocked_indexer_dependency_no_write"
|
||||
elif ! is_uint "$uptime_for_retry" || [ "$uptime_for_retry" -lt "$MANAGER_RETRY_AFTER_SECONDS" ]; then
|
||||
manager_preflight_status="blocked_boot_settle_no_write"
|
||||
elif [ "$manager_cooldown_remaining_seconds" -gt 0 ]; then
|
||||
manager_preflight_status="blocked_retry_cooldown_no_write"
|
||||
fi
|
||||
}
|
||||
|
||||
wait_for_manager_verifier() {
|
||||
local deadline now
|
||||
deadline=$(( $(date +%s) + MANAGER_VERIFY_TIMEOUT_SECONDS ))
|
||||
while true; do
|
||||
capture_manager_state
|
||||
[ "$manager_verifier_ready" -eq 1 ] && return 0
|
||||
now="$(date +%s)"
|
||||
[ "$now" -ge "$deadline" ] && return 1
|
||||
sleep "$MANAGER_VERIFY_INTERVAL_SECONDS"
|
||||
done
|
||||
}
|
||||
|
||||
rollback_manager_to_inactive() {
|
||||
rollback_attempted=1
|
||||
rollback_terminal="bounded_stop_attempted"
|
||||
if timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service; then
|
||||
record_action "rollback_stop_wazuh_manager"
|
||||
else
|
||||
record_action "rollback_stop_wazuh_manager_failed"
|
||||
fi
|
||||
capture_manager_state
|
||||
if [ "$manager_active_state" != "active" ] \
|
||||
&& is_uint "$analysisd_count" && [ "$analysisd_count" -eq 0 ] \
|
||||
&& [ "$manager_event_port_ready" -eq 0 ] \
|
||||
&& [ "$enrollment_1515_tcp" -eq 0 ] \
|
||||
&& [ "$manager_api_55000_tcp" -eq 0 ]; then
|
||||
rollback_verified=1
|
||||
rollback_terminal="bounded_stop_verified"
|
||||
else
|
||||
rollback_terminal="bounded_stop_not_verified"
|
||||
fi
|
||||
}
|
||||
|
||||
write_public_receipt() {
|
||||
local receipt_name receipt_path tmp generated_at receipt_line
|
||||
receipt_name="$(printf '%s' "$RUN_ID" | tr -c 'A-Za-z0-9._+-' '_')"
|
||||
[ -n "$receipt_name" ] || receipt_name="unknown"
|
||||
if [ "$MODE" = "timer_apply" ]; then
|
||||
receipt_path="$STATE_DIR/last-timer-manager-recovery-receipt.txt"
|
||||
else
|
||||
receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
|
||||
fi
|
||||
generated_at="$(date --iso-8601=seconds)"
|
||||
receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v1 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed manager_runtime_write_performed=$manager_runtime_write_performed rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
|
||||
if install -d -m 0750 "$(dirname "$receipt_path")"; then
|
||||
tmp="${receipt_path}.tmp.$$"
|
||||
if printf '%s\n' "$receipt_line" >"$tmp" \
|
||||
&& chmod 0640 "$tmp" \
|
||||
&& mv "$tmp" "$receipt_path"; then
|
||||
receipt_write_performed=1
|
||||
receipt_status="written"
|
||||
receipt_ref="host112-manager-recovery:$RUN_ID"
|
||||
return 0
|
||||
fi
|
||||
rm -f "$tmp" >/dev/null 2>&1 || true
|
||||
fi
|
||||
receipt_status="write_failed"
|
||||
return 1
|
||||
}
|
||||
|
||||
boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)"
|
||||
uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)"
|
||||
if [ "$MODE" = "timer_apply" ]; then
|
||||
invocation_identity="${INVOCATION_ID:-${boot_id}-$(date +%s)}"
|
||||
invocation_identity="$(printf '%s' "$invocation_identity" | tr -c 'A-Za-z0-9._+-' '-')"
|
||||
TRACE_ID="host112-timer:$invocation_identity"
|
||||
RUN_ID="host112-timer:$invocation_identity"
|
||||
WORK_ITEM_ID="HOST112-WAZUH-MANAGER-RECOVERY"
|
||||
fi
|
||||
|
||||
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
||||
if [ "${EUID:-$(id -u)}" -ne 0 ]; then
|
||||
echo "BLOCKER host112_apply_requires_root" >&2
|
||||
exit 77
|
||||
fi
|
||||
exec 9>"$MANAGER_LOCK_FILE"
|
||||
if ! flock -n 9; then
|
||||
echo "BLOCKER host112_manager_single_flight_busy" >&2
|
||||
exit 75
|
||||
fi
|
||||
|
||||
current_default="$(systemctl get-default 2>/dev/null || true)"
|
||||
if [ "$current_default" != "graphical.target" ]; then
|
||||
if systemctl set-default graphical.target >/dev/null; then
|
||||
record_action "set_default_graphical_target"
|
||||
runtime_write_performed=1
|
||||
else
|
||||
record_action "set_default_graphical_target_failed"
|
||||
fi
|
||||
@@ -103,27 +464,29 @@ if [ "$MODE" = "apply" ]; then
|
||||
record_action "ssh_service_missing"
|
||||
fi
|
||||
|
||||
uptime_for_retry="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)"
|
||||
indexer_state="$(unit_active wazuh-indexer.service)"
|
||||
if unit_loaded wazuh-indexer.service \
|
||||
&& [ "$indexer_state" != "active" ] \
|
||||
&& [ "$indexer_state" != "activating" ] \
|
||||
&& [ "$uptime_for_retry" -ge "$INDEXER_RETRY_AFTER_SECONDS" ]; then
|
||||
&& is_uint "$uptime_seconds" \
|
||||
&& [ "$uptime_seconds" -ge "$INDEXER_RETRY_AFTER_SECONDS" ]; then
|
||||
now_epoch="$(date +%s)"
|
||||
last_retry_epoch=0
|
||||
if [ -r "$RETRY_MARKER" ]; then
|
||||
read -r last_retry_epoch <"$RETRY_MARKER" || last_retry_epoch=0
|
||||
fi
|
||||
if ! [[ "$last_retry_epoch" =~ ^[0-9]+$ ]]; then
|
||||
last_retry_epoch=0
|
||||
if [ -r "$INDEXER_RETRY_MARKER" ]; then
|
||||
read -r last_retry_epoch <"$INDEXER_RETRY_MARKER" || last_retry_epoch=0
|
||||
fi
|
||||
is_uint "$last_retry_epoch" || last_retry_epoch=0
|
||||
if [ $((now_epoch - last_retry_epoch)) -ge "$INDEXER_RETRY_COOLDOWN_SECONDS" ]; then
|
||||
printf '%s\n' "$now_epoch" >"$RETRY_MARKER"
|
||||
systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true
|
||||
if timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then
|
||||
record_action "retry_wazuh_indexer"
|
||||
if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then
|
||||
systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true
|
||||
runtime_write_performed=1
|
||||
if timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then
|
||||
record_action "retry_wazuh_indexer"
|
||||
else
|
||||
record_action "retry_wazuh_indexer_failed"
|
||||
fi
|
||||
else
|
||||
record_action "retry_wazuh_indexer_failed"
|
||||
record_action "wazuh_indexer_marker_write_failed_no_write"
|
||||
fi
|
||||
else
|
||||
record_action "wazuh_indexer_retry_cooldown"
|
||||
@@ -131,8 +494,72 @@ if [ "$MODE" = "apply" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)"
|
||||
uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo unknown)"
|
||||
capture_manager_state
|
||||
before_manager_active="$manager_active_state"
|
||||
before_manager_result="$manager_result"
|
||||
before_manager_orphan="$manager_orphan_analysisd"
|
||||
before_manager_memory="$manager_memory_available_kb"
|
||||
before_manager_disk="$manager_disk_available_kb"
|
||||
before_manager_disk_used="$manager_disk_used_percent"
|
||||
before_event_1514_ready="$manager_event_port_ready"
|
||||
before_enrollment_1515="$enrollment_1515_tcp"
|
||||
before_manager_api_55000="$manager_api_55000_tcp"
|
||||
now_epoch="$(date +%s)"
|
||||
evaluate_manager_preflight "$now_epoch" "$uptime_seconds"
|
||||
|
||||
if [ "$MODE" = "dry_run" ]; then
|
||||
case "$manager_preflight_status" in
|
||||
ready) manager_terminal="dry_run_ready" ;;
|
||||
already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;;
|
||||
*) manager_terminal="dry_run_${manager_preflight_status}" ;;
|
||||
esac
|
||||
elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
||||
case "$manager_preflight_status" in
|
||||
already_verified_healthy)
|
||||
manager_terminal="idempotent_already_verified_healthy"
|
||||
;;
|
||||
ready)
|
||||
if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then
|
||||
manager_attempt_count=1
|
||||
if timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then
|
||||
record_action "reset_failed_wazuh_manager"
|
||||
runtime_write_performed=1
|
||||
manager_runtime_write_performed=1
|
||||
if timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then
|
||||
manager_start_exit=0
|
||||
else
|
||||
manager_start_exit=$?
|
||||
fi
|
||||
if wait_for_manager_verifier; then
|
||||
record_action "start_wazuh_manager_verified"
|
||||
manager_terminal="verified_healthy"
|
||||
else
|
||||
record_action "start_wazuh_manager_verifier_failed"
|
||||
rollback_manager_to_inactive
|
||||
if [ "$rollback_verified" -eq 1 ]; then
|
||||
manager_terminal="rolled_back_verifier_failed"
|
||||
else
|
||||
manager_terminal="rollback_failed_degraded"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
manager_start_exit="reset_failed_no_start"
|
||||
record_action "reset_failed_wazuh_manager_failed_no_start"
|
||||
manager_terminal="no_start_reset_failed"
|
||||
fi
|
||||
else
|
||||
record_action "wazuh_manager_marker_write_failed_no_write"
|
||||
manager_terminal="no_write_cooldown_marker_write_failed"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
manager_terminal="no_write_${manager_preflight_status}"
|
||||
;;
|
||||
esac
|
||||
capture_manager_state
|
||||
write_public_receipt || manager_terminal="${manager_terminal}_receipt_failed"
|
||||
fi
|
||||
|
||||
systemd_state="$(systemctl is-system-running 2>/dev/null || true)"
|
||||
systemd_state="${systemd_state:-unknown}"
|
||||
default_target="$(systemctl get-default 2>/dev/null || true)"
|
||||
@@ -151,9 +578,7 @@ startup_enabled="$(unit_enabled awoooi-host112-guest-recovery.timer)"
|
||||
startup_active="$(unit_active awoooi-host112-guest-recovery.timer)"
|
||||
|
||||
ssh_port_listening=0
|
||||
if ss -H -ltn 2>/dev/null | awk '{print $4}' | grep -Eq '(^|:)22$'; then
|
||||
ssh_port_listening=1
|
||||
fi
|
||||
tcp_port_listening 22 && ssh_port_listening=1
|
||||
ssh_ready=0
|
||||
if [ "$ssh_service" = "active" ] \
|
||||
&& { [ "$ssh_enabled" = "enabled" ] || [ "$ssh_enabled" = "static" ]; } \
|
||||
@@ -178,9 +603,9 @@ fi
|
||||
|
||||
services_ready=0
|
||||
if [ "$wazuh_indexer" = "active" ] \
|
||||
&& [ "$wazuh_manager" = "active" ] \
|
||||
&& [ "$wazuh_dashboard" = "active" ] \
|
||||
&& [ "$filebeat" = "active" ]; then
|
||||
&& [ "$filebeat" = "active" ] \
|
||||
&& [ "$manager_verifier_ready" -eq 1 ]; then
|
||||
services_ready=1
|
||||
fi
|
||||
|
||||
@@ -203,15 +628,30 @@ if [ "${#actions[@]}" -gt 0 ]; then
|
||||
action_text="$(IFS=,; printf '%s' "${actions[*]}")"
|
||||
fi
|
||||
|
||||
readback="boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready action_count=${#actions[@]} actions=$action_text"
|
||||
trace_output="${TRACE_ID:-none}"
|
||||
run_output="${RUN_ID:-none}"
|
||||
work_item_output="${WORK_ITEM_ID:-none}"
|
||||
readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS runtime_write_performed=$runtime_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
|
||||
printf '%s\n' "$readback"
|
||||
|
||||
if [ "$MODE" = "apply" ]; then
|
||||
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
||||
install -d -m 0750 "$STATE_DIR"
|
||||
tmp_readback="${STATE_DIR}/last-readback.txt.tmp.$$"
|
||||
printf 'generated_at=%s mode=%s %s\n' "$(date --iso-8601=seconds)" "$MODE" "$readback" >"$tmp_readback"
|
||||
chmod 0640 "$tmp_readback"
|
||||
mv "$tmp_readback" "${STATE_DIR}/last-readback.txt"
|
||||
if printf 'generated_at=%s %s\n' "$(date --iso-8601=seconds)" "$readback" >"$tmp_readback" \
|
||||
&& chmod 0640 "$tmp_readback" \
|
||||
&& mv "$tmp_readback" "${STATE_DIR}/last-readback.txt"; then
|
||||
:
|
||||
else
|
||||
rm -f "$tmp_readback" >/dev/null 2>&1 || true
|
||||
exit 74
|
||||
fi
|
||||
fi
|
||||
|
||||
[ "$guest_ready" -eq 1 ]
|
||||
if [ "$MODE" = "dry_run" ]; then
|
||||
case "$manager_preflight_status" in
|
||||
ready|already_verified_healthy) exit 0 ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
[ "$guest_ready" -eq 1 ] && [ "$receipt_status" != "write_failed" ]
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
# Host 112 guest recovery installer.
|
||||
# Version: 2.0
|
||||
# Last modified: 2026-07-14 11:00 (Asia/Taipei)
|
||||
# Last modified by: Codex
|
||||
# Change: install and roll back the argument-regex sudo boundary required for
|
||||
# correlated Wazuh manager recovery receipts.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -16,6 +23,7 @@ SERVICE_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.service"
|
||||
TIMER_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.timer"
|
||||
SUDOERS_TARGET="/etc/sudoers.d/awoooi-host112-agent99"
|
||||
STATE_ROOT="/var/lib/awoooi-host112-recovery"
|
||||
SUDOERS_ARGUMENT_REGEX='^--apply --trace-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --run-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --work-item-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$'
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
@@ -44,6 +52,20 @@ while [ "$#" -gt 0 ]; do
|
||||
shift
|
||||
done
|
||||
|
||||
sudo_policy_readback() {
|
||||
if [ "${EUID:-$(id -u)}" -eq 0 ]; then
|
||||
sudo -n -U "$TARGET_USER" -l "$@" 2>/dev/null || true
|
||||
else
|
||||
sudo -n -l "$@" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
sudo_policy_has_nopasswd() {
|
||||
local output
|
||||
output="$(sudo_policy_readback "$@")"
|
||||
printf '%s\n' "$output" | grep -Fq "NOPASSWD: $READINESS_TARGET"
|
||||
}
|
||||
|
||||
check() {
|
||||
echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1"
|
||||
echo "MODE=check"
|
||||
@@ -56,6 +78,32 @@ check() {
|
||||
done
|
||||
echo "TIMER_ENABLED=$(systemctl is-enabled awoooi-host112-guest-recovery.timer 2>/dev/null || true)"
|
||||
echo "TIMER_ACTIVE=$(systemctl is-active awoooi-host112-guest-recovery.timer 2>/dev/null || true)"
|
||||
if [ -f "$SUDOERS_TARGET" ] && visudo -cf "$SUDOERS_TARGET" >/dev/null 2>&1; then
|
||||
echo "SUDOERS_VALID=1"
|
||||
else
|
||||
echo "SUDOERS_VALID=0"
|
||||
fi
|
||||
if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply \
|
||||
--trace-id host112-policy-check \
|
||||
--run-id host112-policy-check \
|
||||
--work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then
|
||||
echo "SUDO_CORRELATED_APPLY_NOPASSWD=1"
|
||||
else
|
||||
echo "SUDO_CORRELATED_APPLY_NOPASSWD=0"
|
||||
fi
|
||||
if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply; then
|
||||
echo "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=1"
|
||||
else
|
||||
echo "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=0"
|
||||
fi
|
||||
if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply \
|
||||
--trace-id 'unsafe value' \
|
||||
--run-id host112-policy-check \
|
||||
--work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then
|
||||
echo "SUDO_UNSAFE_IDENTITY_NOPASSWD=1"
|
||||
else
|
||||
echo "SUDO_UNSAFE_IDENTITY_NOPASSWD=0"
|
||||
fi
|
||||
if [ -x "$READINESS_TARGET" ]; then
|
||||
"$READINESS_TARGET" --check || true
|
||||
fi
|
||||
@@ -152,7 +200,8 @@ install -m 0644 "$TIMER_SOURCE" "$TIMER_TARGET"
|
||||
|
||||
sudoers_tmp="$(mktemp)"
|
||||
trap 'rm -f "$sudoers_tmp"' EXIT
|
||||
printf '%s ALL=(root) NOPASSWD: %s --apply\n' "$TARGET_USER" "$READINESS_TARGET" >"$sudoers_tmp"
|
||||
printf '%s ALL=(root) NOPASSWD: %s %s\n' \
|
||||
"$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_ARGUMENT_REGEX" >"$sudoers_tmp"
|
||||
visudo -cf "$sudoers_tmp" >/dev/null
|
||||
install -m 0440 "$sudoers_tmp" "$SUDOERS_TARGET"
|
||||
|
||||
|
||||
@@ -791,8 +791,8 @@ def source_controls() -> dict[str, bool]:
|
||||
source_file(
|
||||
"scripts/reboot-recovery/awoooi-host112-guest-recovery.service"
|
||||
),
|
||||
"ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply",
|
||||
"TimeoutStartSec=300",
|
||||
"ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer",
|
||||
"TimeoutStartSec=600",
|
||||
),
|
||||
"host_112_guest_recovery_timer_source_present": file_contains(
|
||||
source_file(
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
READINESS = ROOT / "scripts/reboot-recovery/host112-guest-readiness.sh"
|
||||
INSTALLER = ROOT / "scripts/reboot-recovery/install-host112-guest-recovery.sh"
|
||||
SERVICE = ROOT / "scripts/reboot-recovery/awoooi-host112-guest-recovery.service"
|
||||
CONTROL = ROOT / "agent99-control-plane.ps1"
|
||||
BOOTSTRAP = ROOT / "agent99-bootstrap.ps1"
|
||||
|
||||
|
||||
def read(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def isolated_env(tmp_path: Path) -> dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"HOST112_RECOVERY_STATE_DIR": str(tmp_path / "state"),
|
||||
"HOST112_INDEXER_RETRY_MARKER": str(tmp_path / "indexer.epoch"),
|
||||
"HOST112_MANAGER_RETRY_MARKER": str(tmp_path / "manager.epoch"),
|
||||
"HOST112_MANAGER_LOCK_FILE": str(tmp_path / "manager.lock"),
|
||||
}
|
||||
)
|
||||
return env
|
||||
|
||||
|
||||
def test_check_and_dry_run_are_real_no_write_execution_paths(tmp_path: Path) -> None:
|
||||
identity = [
|
||||
"--trace-id",
|
||||
"trace:D037-host112-dry-run",
|
||||
"--run-id",
|
||||
"run:D037-host112-dry-run",
|
||||
"--work-item-id",
|
||||
"D037-WAZUH-HOST112",
|
||||
]
|
||||
env = isolated_env(tmp_path)
|
||||
|
||||
check = subprocess.run(
|
||||
["bash", str(READINESS), "--check", *identity],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
check=False,
|
||||
timeout=20,
|
||||
)
|
||||
dry_run = subprocess.run(
|
||||
["bash", str(READINESS), "--dry-run", *identity],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
check=False,
|
||||
timeout=20,
|
||||
)
|
||||
|
||||
assert check.returncode in {0, 1}
|
||||
assert dry_run.returncode in {0, 1}
|
||||
for result, mode in ((check, "check"), (dry_run, "dry_run")):
|
||||
output = result.stdout + result.stderr
|
||||
assert f"mode={mode}" in output
|
||||
assert "trace_id=trace:D037-host112-dry-run" in output
|
||||
assert "run_id=run:D037-host112-dry-run" in output
|
||||
assert "work_item_id=D037-WAZUH-HOST112" in output
|
||||
assert "runtime_write_performed=0" in output
|
||||
assert "manager_runtime_write_performed=0" in output
|
||||
assert "receipt_write_performed=0" in output
|
||||
assert "manager_preflight_status=" in output
|
||||
assert "apply_phase_timeout_budget_seconds=500" in output
|
||||
assert not (tmp_path / "state").exists()
|
||||
assert not (tmp_path / "indexer.epoch").exists()
|
||||
assert not (tmp_path / "manager.epoch").exists()
|
||||
assert not (tmp_path / "manager.lock").exists()
|
||||
|
||||
|
||||
def test_unsafe_or_incomplete_identity_fails_before_any_state_path(tmp_path: Path) -> None:
|
||||
env = isolated_env(tmp_path)
|
||||
unsafe = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(READINESS),
|
||||
"--dry-run",
|
||||
"--trace-id",
|
||||
"unsafe value",
|
||||
"--run-id",
|
||||
"run-safe",
|
||||
"--work-item-id",
|
||||
"D037-WAZUH-HOST112",
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
check=False,
|
||||
timeout=10,
|
||||
)
|
||||
incomplete = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(READINESS),
|
||||
"--dry-run",
|
||||
"--trace-id",
|
||||
"trace-safe",
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
check=False,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
assert unsafe.returncode == 64
|
||||
assert "BLOCKER unsafe_control_identity" in (unsafe.stdout + unsafe.stderr)
|
||||
assert incomplete.returncode == 64
|
||||
assert "BLOCKER incomplete_control_identity" in (
|
||||
incomplete.stdout + incomplete.stderr
|
||||
)
|
||||
assert not any(tmp_path.iterdir())
|
||||
|
||||
|
||||
def test_manager_preflight_verifier_receipt_and_rollback_are_complete() -> None:
|
||||
source = read(READINESS)
|
||||
|
||||
for field in (
|
||||
"manager_failed_state_observed",
|
||||
"manager_orphan_analysisd",
|
||||
"manager_memory_available_kb",
|
||||
"manager_disk_available_kb",
|
||||
"manager_disk_used_percent",
|
||||
"wazuh_analysisd_process_count",
|
||||
"wazuh_agent_event_port_1514_tcp",
|
||||
"wazuh_agent_event_port_1514_udp",
|
||||
"wazuh_agent_enrollment_port_1515_tcp",
|
||||
"wazuh_manager_api_port_55000_tcp",
|
||||
"manager_independent_verifier_ready",
|
||||
):
|
||||
assert field in source
|
||||
assert 'manager_active_state="$(unit_property wazuh-manager.service ActiveState)"' in source
|
||||
assert 'manager_result="$(unit_property wazuh-manager.service Result)"' in source
|
||||
assert 'analysisd_count="$(process_count wazuh-analysisd)"' in source
|
||||
assert source.count(
|
||||
'timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service'
|
||||
) == 1
|
||||
assert "manager_attempt_count=1" in source
|
||||
assert 'timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service' in source
|
||||
assert 'rollback_terminal="bounded_stop_verified"' in source
|
||||
assert "schema_version=host112_wazuh_manager_recovery_receipt_v1" in source
|
||||
assert "trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID" in source
|
||||
assert 'receipt_status="write_failed"' in source
|
||||
assert "prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" in source
|
||||
|
||||
preflight = source[
|
||||
source.index("evaluate_manager_preflight() {") : source.index(
|
||||
"wait_for_manager_verifier() {"
|
||||
)
|
||||
]
|
||||
assert 'elif [ "$(unit_active wazuh-indexer.service)" != "active" ]; then' in preflight
|
||||
assert 'manager_preflight_status="blocked_indexer_dependency_no_write"' in preflight
|
||||
|
||||
|
||||
def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None:
|
||||
source = read(READINESS)
|
||||
marker_function = source[
|
||||
source.index("write_epoch_marker() {") : source.index("unit_active() {")
|
||||
]
|
||||
assert 'tmp="${marker}.tmp.$$"' in marker_function
|
||||
assert 'printf \'%s\\n\' "$epoch" >"$tmp"' in marker_function
|
||||
assert 'chmod 0644 "$tmp"' in marker_function
|
||||
assert 'mv "$tmp" "$marker"' in marker_function
|
||||
assert "return 1" in marker_function
|
||||
|
||||
manager_start = source.index(
|
||||
' if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then'
|
||||
)
|
||||
manager_end = source.index(" ;;", manager_start)
|
||||
manager_branch = source[manager_start:manager_end]
|
||||
marker_failure_at = manager_branch.index(
|
||||
' else\n record_action "wazuh_manager_marker_write_failed_no_write"'
|
||||
)
|
||||
success = manager_branch[:marker_failure_at]
|
||||
failure = manager_branch[marker_failure_at:]
|
||||
assert "systemctl reset-failed wazuh-manager.service" in success
|
||||
assert "systemctl start wazuh-manager.service" in success
|
||||
assert 'timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service' in success
|
||||
assert "manager_attempt_count=1" in success
|
||||
assert "systemctl reset-failed wazuh-manager.service" not in failure
|
||||
assert "systemctl start wazuh-manager.service" not in failure
|
||||
assert "wazuh_manager_marker_write_failed_no_write" in failure
|
||||
assert 'manager_terminal="no_write_cooldown_marker_write_failed"' in failure
|
||||
|
||||
indexer_start = source.index(
|
||||
' if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then'
|
||||
)
|
||||
indexer_end = source.index(" else\n record_action", indexer_start)
|
||||
indexer_branch = source[indexer_start:indexer_end]
|
||||
indexer_success, indexer_failure = indexer_branch.split(" else\n", maxsplit=1)
|
||||
assert "systemctl reset-failed wazuh-indexer.service" in indexer_success
|
||||
assert "systemctl start wazuh-indexer.service" in indexer_success
|
||||
assert "systemctl reset-failed wazuh-indexer.service" not in indexer_failure
|
||||
assert "systemctl start wazuh-indexer.service" not in indexer_failure
|
||||
assert "wazuh_indexer_marker_write_failed_no_write" in indexer_failure
|
||||
|
||||
|
||||
def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None:
|
||||
source = read(READINESS)
|
||||
service = read(SERVICE)
|
||||
control = read(CONTROL)
|
||||
|
||||
assert 'SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-30}"' in source
|
||||
assert 'INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-180}"' in source
|
||||
assert 'MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-120}"' in source
|
||||
assert 'MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}"' in source
|
||||
assert 'MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}"' in source
|
||||
assert 'MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-45}"' in source
|
||||
assert 'if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt 570 ]; then' in source
|
||||
assert "TimeoutStartSec=600" in service
|
||||
assert "Invoke-HostSshText $targetHost $applyCommand 590 1" in control
|
||||
|
||||
|
||||
def test_installer_uses_argument_regex_and_post_install_policy_readback() -> None:
|
||||
source = read(INSTALLER)
|
||||
|
||||
assert "SUDOERS_ARGUMENT_REGEX='^--apply --trace-id " in source
|
||||
assert "--run-id [A-Za-z0-9]" in source
|
||||
assert "--work-item-id [A-Za-z0-9]" in source
|
||||
assert '"$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_ARGUMENT_REGEX"' in source
|
||||
assert 'visudo -cf "$sudoers_tmp"' in source
|
||||
assert "SUDO_CORRELATED_APPLY_NOPASSWD" in source
|
||||
assert "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD" in source
|
||||
assert "SUDO_UNSAFE_IDENTITY_NOPASSWD" in source
|
||||
assert "--trace-id 'unsafe value'" in source
|
||||
assert "--rollback" in source
|
||||
|
||||
|
||||
def test_exact_sudoers_argument_regex_parses_when_visudo_is_available() -> None:
|
||||
visudo = shutil.which("visudo")
|
||||
if not visudo:
|
||||
return
|
||||
candidate = (
|
||||
"kali ALL=(root) NOPASSWD: "
|
||||
"/usr/local/sbin/awoooi-host112-guest-readiness "
|
||||
"^--apply --trace-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} "
|
||||
"--run-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} "
|
||||
"--work-item-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$\n"
|
||||
)
|
||||
result = subprocess.run(
|
||||
[visudo, "-cf", "-"],
|
||||
input=candidate,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
timeout=10,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
|
||||
def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
|
||||
control = read(CONTROL)
|
||||
bootstrap = read(BOOTSTRAP)
|
||||
function = control[control.index("function Convert-AgentHost112GuestReadback") :]
|
||||
function = function[: function.index("function Test-HostReachability")]
|
||||
|
||||
for parameter in ("AutomationRunId", "TraceId", "WorkItemId"):
|
||||
assert f"[string]${parameter} = \"\"" in control
|
||||
assert f'[string]`${parameter} = ""' in bootstrap
|
||||
assert 'reason = "controlled_dispatch_identity_unsafe"' in control
|
||||
assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
|
||||
assert '$before.managerPreflightStatus -eq "ready"' in function
|
||||
assert 'receiptStatus -eq "written"' in function
|
||||
assert '$after.managerVerified' in function
|
||||
assert 'name = "wazuh_manager_service_active"' in function
|
||||
assert 'name = "wazuh_analysisd_process_present"' in function
|
||||
assert 'name = "wazuh_agent_event_1514_ready"' in function
|
||||
assert 'name = "wazuh_agent_enrollment_1515_tcp"' in function
|
||||
assert 'name = "wazuh_manager_api_55000_tcp"' in function
|
||||
assert "host112_readback_transport_unavailable" in function
|
||||
assert 'terminal = "no_write_control_identity_incomplete_or_unsafe"' in function
|
||||
@@ -107,7 +107,8 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
|
||||
assert "restrict,command=" in host112_installer
|
||||
assert "visudo -cf" in host112_installer
|
||||
assert "--rollback" in host112_installer
|
||||
assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply" in host112_service
|
||||
assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer" in host112_service
|
||||
assert "TimeoutStartSec=600" in host112_service
|
||||
assert "OnBootSec=20s" in host112_timer
|
||||
assert "OnUnitInactiveSec=60s" in host112_timer
|
||||
|
||||
|
||||
Reference in New Issue
Block a user