From bbd479a36ca8b34584cf94b215380436e992ed8d Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 11:04:27 +0800 Subject: [PATCH] fix(recovery): close host112 wazuh manager loop --- agent99-bootstrap.ps1 | 5 +- agent99-control-plane.ps1 | 268 ++++++++- ...nt99_transport_recovery_deploy_contract.py | 20 +- .../awoooi-host112-guest-recovery.service | 6 +- .../host112-guest-readiness.sh | 510 ++++++++++++++++-- .../install-host112-guest-recovery.sh | 51 +- .../reboot-auto-recovery-slo-scorecard.py | 4 +- .../test_host112_manager_recovery_contract.py | 281 ++++++++++ .../test_reboot_p0_operational_contract.py | 3 +- 9 files changed, 1076 insertions(+), 72 deletions(-) create mode 100644 scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index f94933f4d..33be09e6f 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -188,11 +188,14 @@ param( [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")] [string]`$Mode = "Status", [switch]`$ControlledApply, + [string]`$AutomationRunId = "", + [string]`$TraceId = "", + [string]`$WorkItemId = "", [switch]`$SuppressAlerts, [string]`$SuppressReason = "" ) -& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir" +& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -AutomationRunId `$AutomationRunId -TraceId `$TraceId -WorkItemId `$WorkItemId -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir" "@ | Set-Content -Encoding UTF8 $launcher $submitCmd = Join-Path $AgentRoot "agent99-submit-request.cmd" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 22edfae5e..9ad3fd96d 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -2,6 +2,9 @@ param( [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")] [string]$Mode = "Status", [switch]$ControlledApply, + [string]$AutomationRunId = "", + [string]$TraceId = "", + [string]$WorkItemId = "", [switch]$SelfTestTelegramCard, [switch]$SelfTestTelegramDelivery, [switch]$SelfTestSensorGate, @@ -4940,6 +4943,12 @@ function Invoke-AgentQueuedCommands { } $hasDispatchIdentity = [bool]($automationRunId -or $traceId -or $workItemId -or $idempotencyKey -or $incidentId -or $projectId -or $sourceFingerprint -or $dispatchRouteId -or $singleFlightKey -or $lockOwner -or $canonicalDigest) $dispatchIdentityIncomplete = [bool](-not $automationRunId -or -not $traceId -or -not $workItemId -or -not $idempotencyKey -or -not $incidentId -or -not $projectId -or -not $sourceFingerprint -or -not $dispatchRouteId -or -not $singleFlightKey -or -not $lockOwner -or -not $canonicalDigest -or -not $approvalIdentityPresent -or -not $executionGeneration) + $controlIdentityPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+\-]{0,127}$" + $dispatchControlIdentitySafe = [bool]( + $automationRunId -match $controlIdentityPattern -and + $traceId -match $controlIdentityPattern -and + $workItemId -match $controlIdentityPattern + ) if (($controlled -and $modeName -eq "Recover" -and -not $hasDispatchIdentity) -or ($hasDispatchIdentity -and $dispatchIdentityIncomplete)) { $target = Join-Path $failedDir ("failed-" + $file.Name) Move-Item -Force $file.FullName $target @@ -4956,6 +4965,22 @@ function Invoke-AgentQueuedCommands { $results += $result continue } + if ($hasDispatchIdentity -and -not $dispatchControlIdentitySafe) { + $target = Join-Path $failedDir ("failed-" + $file.Name) + Move-Item -Force $file.FullName $target + $result = [pscustomobject]@{ + id = $id + mode = $modeName + source = $commandSource + ok = $false + reason = "controlled_dispatch_identity_unsafe" + runtimeWritePerformed = $false + file = $target + } + Record-AgentEvent "operator_command_rejected" "warning" "id=$id mode=$modeName reason=controlled_dispatch_identity_unsafe" $result -Alert + $results += $result + continue + } $runningPath = Join-Path $queueDir ("running-" + $file.Name) Move-Item -Force $file.FullName $runningPath @@ -4965,6 +4990,9 @@ function Invoke-AgentQueuedCommands { $stderrPath = Join-Path $EvidenceDir "agent99-command-$id.err" $args = @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", $modeName) if ($controlled) { $args += "-ControlledApply" } + if ($automationRunId) { $args += @("-AutomationRunId", $automationRunId) } + if ($traceId) { $args += @("-TraceId", $traceId) } + if ($workItemId) { $args += @("-WorkItemId", $workItemId) } if ($suppressTelegram) { $args += @("-SuppressAlerts", "-SuppressReason", "do_not_page") } $process = Start-Process -FilePath "powershell.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru $completed = $process.WaitForExit(600000) @@ -5351,7 +5379,12 @@ function Invoke-AgentProviderFreshnessTriage { } function Convert-AgentHost112GuestReadback { - param([object]$Transport) + param( + [object]$Transport, + [string]$ExpectedTraceId, + [string]$ExpectedRunId, + [string]$ExpectedWorkItemId + ) $values = @{} $text = if ($Transport -and $Transport.PSObject.Properties["output"]) { [string]$Transport.output } else { "" } @@ -5367,19 +5400,61 @@ function Convert-AgentHost112GuestReadback { param([string]$Name) [bool]((& $get $Name "0") -eq "1") } + $readbackPresent = [bool]( + $values.ContainsKey("schema_version") -and + $values.ContainsKey("boot_id") -and + $values.ContainsKey("guest_ready") -and + $values.ContainsKey("manager_preflight_status") + ) + $identityPresent = [bool]( + $values.ContainsKey("trace_id") -and + $values.ContainsKey("run_id") -and + $values.ContainsKey("work_item_id") + ) + $identityMatches = [bool]( + $identityPresent -and + ((& $get "trace_id") -eq $ExpectedTraceId) -and + ((& $get "run_id") -eq $ExpectedRunId) -and + ((& $get "work_item_id") -eq $ExpectedWorkItemId) + ) + $analysisdPresent = [bool](((& $get "wazuh_analysisd_process_count" "0") -match "^[1-9][0-9]*$")) + $managerServiceActive = [bool](((& $get "wazuh_manager") -eq "active")) + $agentEventPortReady = (& $asBool "wazuh_agent_event_port_1514_ready") + $agentEnrollmentPortReady = (& $asBool "wazuh_agent_enrollment_port_1515_tcp") + $managerApiPortReady = (& $asBool "wazuh_manager_api_port_55000_tcp") + $managerVerified = [bool]( + $managerServiceActive -and + $analysisdPresent -and + $agentEventPortReady -and + $agentEnrollmentPortReady -and + $managerApiPortReady + ) $checks = @( - [pscustomobject]@{ name = "readback_present"; passed = [bool]($values.ContainsKey("boot_id") -and $values.ContainsKey("guest_ready")) }, + [pscustomobject]@{ name = "transport_ok"; passed = [bool]($Transport -and $Transport.ok) }, + [pscustomobject]@{ name = "readback_present"; passed = $readbackPresent }, + [pscustomobject]@{ name = "control_identity_present"; passed = $identityPresent }, + [pscustomobject]@{ name = "control_identity_matches"; passed = $identityMatches }, [pscustomobject]@{ name = "systemd_running"; passed = [bool]((& $get "systemd_state") -eq "running") }, [pscustomobject]@{ name = "console_ready"; passed = (& $asBool "console_ready") }, + [pscustomobject]@{ name = "wazuh_manager_service_active"; passed = $managerServiceActive }, + [pscustomobject]@{ name = "wazuh_analysisd_process_present"; passed = $analysisdPresent }, + [pscustomobject]@{ name = "wazuh_agent_event_1514_ready"; passed = $agentEventPortReady }, + [pscustomobject]@{ name = "wazuh_agent_enrollment_1515_tcp"; passed = $agentEnrollmentPortReady }, + [pscustomobject]@{ name = "wazuh_manager_api_55000_tcp"; passed = $managerApiPortReady }, [pscustomobject]@{ name = "services_ready"; passed = (& $asBool "services_ready") }, [pscustomobject]@{ name = "recovery_timer_ready"; passed = (& $asBool "recovery_timer_ready") }, [pscustomobject]@{ name = "guest_ready"; passed = (& $asBool "guest_ready") } ) $failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name }) [pscustomobject]@{ - schemaVersion = "agent99_host112_guest_readback_v1" - readbackPresent = [bool]($values.ContainsKey("boot_id") -and $values.ContainsKey("guest_ready")) + schemaVersion = "agent99_host112_guest_readback_v2" + readbackPresent = $readbackPresent ready = [bool]($failedChecks.Count -eq 0) + identityPresent = $identityPresent + identityMatches = $identityMatches + traceId = & $get "trace_id" + runId = & $get "run_id" + workItemId = & $get "work_item_id" bootId = & $get "boot_id" uptimeSeconds = & $get "uptime_seconds" "-1" systemdState = & $get "systemd_state" @@ -5391,6 +5466,21 @@ function Convert-AgentHost112GuestReadback { xorg = & $get "xorg" wazuhIndexer = & $get "wazuh_indexer" wazuhManager = & $get "wazuh_manager" + wazuhManagerResult = & $get "wazuh_manager_result" + managerFailedStateObserved = (& $asBool "manager_failed_state_observed") + wazuhAnalysisdProcessCount = & $get "wazuh_analysisd_process_count" "0" + managerOrphanAnalysisd = (& $asBool "manager_orphan_analysisd") + managerMemoryAvailableKb = & $get "manager_memory_available_kb" "0" + managerDiskAvailableKb = & $get "manager_disk_available_kb" "0" + managerDiskUsedPercent = & $get "manager_disk_used_percent" "100" + managerResourceReady = (& $asBool "manager_resource_ready") + managerPreflightStatus = & $get "manager_preflight_status" + managerVerified = $managerVerified + wazuhAgentEventPort1514Tcp = (& $asBool "wazuh_agent_event_port_1514_tcp") + wazuhAgentEventPort1514Udp = (& $asBool "wazuh_agent_event_port_1514_udp") + wazuhAgentEventPort1514Ready = $agentEventPortReady + wazuhAgentEnrollmentPort1515Tcp = $agentEnrollmentPortReady + wazuhManagerApiPort55000Tcp = $managerApiPortReady wazuhDashboard = & $get "wazuh_dashboard" filebeat = & $get "filebeat" consoleReady = (& $asBool "console_ready") @@ -5399,6 +5489,17 @@ function Convert-AgentHost112GuestReadback { guestReady = (& $asBool "guest_ready") actionCount = & $get "action_count" "0" actions = & $get "actions" "none" + runtimeWritePerformed = (& $asBool "runtime_write_performed") + managerRuntimeWritePerformed = (& $asBool "manager_runtime_write_performed") + receiptWritePerformed = (& $asBool "receipt_write_performed") + receiptStatus = & $get "receipt_status" + receiptRef = & $get "receipt_ref" "none" + managerAttemptCount = & $get "manager_attempt_count" "0" + managerStartExit = & $get "manager_start_exit" "not_attempted" + rollbackAttempted = (& $asBool "rollback_attempted") + rollbackVerified = (& $asBool "rollback_verified") + rollbackTerminal = & $get "rollback_terminal" "not_required" + terminal = & $get "terminal" "observed" transportOk = [bool]($Transport -and $Transport.ok) transportExitCode = if ($Transport) { $Transport.exitCode } else { -1 } transportRoute = if ($Transport -and $Transport.PSObject.Properties["route"]) { [string]$Transport.route } else { "unknown" } @@ -5410,42 +5511,149 @@ function Convert-AgentHost112GuestReadback { } function Invoke-AgentHost112GuestRecovery { - $targetHost = "192.168.0.112" - $checkCommand = "/usr/local/sbin/awoooi-host112-guest-readiness --check" - $applyCommand = "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply" - $beforeTransport = Invoke-HostSshText $targetHost $checkCommand 20 1 - $before = Convert-AgentHost112GuestReadback $beforeTransport - $applyTransport = $null - $after = $before - $applyAttempted = $false + param( + [string]$RequestedRunId = "", + [string]$RequestedTraceId = "", + [string]$RequestedWorkItemId = "", + [string]$FallbackRunId = "" + ) - if (-not $before.ready -and $ControlledApply) { - $applyAttempted = $true - Write-AgentLog "CONTROLLED_APPLY host112_guest_recovery failedChecks=$($before.failedChecks -join ',')" - $applyTransport = Invoke-HostSshText $targetHost $applyCommand 300 1 - $afterTransport = Invoke-HostSshText $targetHost $checkCommand 20 1 - $after = Convert-AgentHost112GuestReadback $afterTransport + $targetHost = "192.168.0.112" + $identityPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+\-]{0,127}$" + $hasRequestedIdentity = [bool]($RequestedRunId -or $RequestedTraceId -or $RequestedWorkItemId) + $requestedIdentityComplete = [bool]($RequestedRunId -and $RequestedTraceId -and $RequestedWorkItemId) + $effectiveRunId = $RequestedRunId + $effectiveTraceId = $RequestedTraceId + $effectiveWorkItemId = $RequestedWorkItemId + if (-not $hasRequestedIdentity) { + $effectiveRunId = if ($FallbackRunId) { $FallbackRunId } else { "host112-observe-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) } + $effectiveTraceId = "trace:$effectiveRunId" + $effectiveWorkItemId = "HOST112-WAZUH-MANAGER-RECOVERY" + } + $identitySafe = [bool]( + (-not $hasRequestedIdentity -or $requestedIdentityComplete) -and + $effectiveRunId -match $identityPattern -and + $effectiveTraceId -match $identityPattern -and + $effectiveWorkItemId -match $identityPattern + ) + if (-not $identitySafe) { + $before = Convert-AgentHost112GuestReadback $null $effectiveTraceId $effectiveRunId $effectiveWorkItemId + return [pscustomobject]@{ + schemaVersion = "agent99_host112_guest_recovery_v2" + targetHost = $targetHost + traceId = $effectiveTraceId + runId = $effectiveRunId + workItemId = $effectiveWorkItemId + controlledApply = [bool]$ControlledApply + applyEligible = $false + applyAttempted = $false + applyBlockedReason = "control_identity_incomplete_or_unsafe" + runtimeWritePerformed = $false + changed = $false + managerVerified = $false + verified = $false + receiptVerified = $false + terminal = "no_write_control_identity_incomplete_or_unsafe" + before = $before + after = $before + applyReceipt = $null + rollback = [pscustomobject]@{ attempted = $false; verified = $false; terminal = "not_required" } + verifier = "wazuh_manager_service_process_agent_event_1514_enrollment_1515_and_api_55000" + prohibitedActions = @("host_reboot", "vm_power_change", "vm_reset", "secret_read", "firewall_change", "database_write") + } } - $verified = [bool]$after.ready - $changed = [bool]($applyAttempted -and $verified -and -not $before.ready) + $checkCommand = "/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId" + $applyCommand = "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId" + $beforeTransport = Invoke-HostSshText $targetHost $checkCommand 30 1 + $before = Convert-AgentHost112GuestReadback $beforeTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId + $applyTransport = $null + $applyReceipt = $null + $after = $before + $applyAttempted = $false + $receiptVerified = $false + $applyEligible = [bool]( + $before.transportOk -and + $before.identityMatches -and + -not $before.managerVerified -and + $before.managerPreflightStatus -eq "ready" + ) + $applyBlockedReason = if ($before.managerVerified) { + "idempotent_already_verified_healthy" + } elseif (-not $before.transportOk) { + "host112_readback_transport_unavailable" + } elseif (-not $before.identityMatches) { + "control_identity_readback_mismatch" + } elseif ($before.managerPreflightStatus -ne "ready") { + [string]$before.managerPreflightStatus + } else { + "none" + } + + if ($ControlledApply -and $applyEligible) { + $applyAttempted = $true + Write-AgentLog "CONTROLLED_APPLY host112_guest_recovery traceId=$effectiveTraceId runId=$effectiveRunId workItemId=$effectiveWorkItemId preflight=$($before.managerPreflightStatus)" + $applyTransport = Invoke-HostSshText $targetHost $applyCommand 590 1 + $applyReceipt = Convert-AgentHost112GuestReadback $applyTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId + $receiptVerified = [bool]( + $applyReceipt.identityMatches -and + $applyReceipt.receiptWritePerformed -and + $applyReceipt.receiptStatus -eq "written" -and + $applyReceipt.receiptRef -ne "none" + ) + $afterTransport = Invoke-HostSshText $targetHost $checkCommand 30 1 + $after = Convert-AgentHost112GuestReadback $afterTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId + } + + $managerVerified = [bool]$after.managerVerified + $verified = [bool]($after.ready -and (-not $applyAttempted -or $receiptVerified)) + $changed = [bool]( + $applyAttempted -and + $applyReceipt -and + $applyReceipt.managerRuntimeWritePerformed -and + $managerVerified -and + $receiptVerified + ) + $terminal = if ($applyAttempted -and $applyReceipt) { + [string]$applyReceipt.terminal + } elseif ($before.managerVerified) { + "idempotent_already_verified_healthy" + } elseif ($ControlledApply) { + "no_write_$applyBlockedReason" + } else { + "observed_no_write" + } $result = [pscustomobject]@{ - schemaVersion = "agent99_host112_guest_recovery_v1" + schemaVersion = "agent99_host112_guest_recovery_v2" targetHost = $targetHost + traceId = $effectiveTraceId + runId = $effectiveRunId + workItemId = $effectiveWorkItemId controlledApply = [bool]$ControlledApply + applyEligible = $applyEligible applyAttempted = $applyAttempted + applyBlockedReason = $applyBlockedReason applyExitCode = if ($applyTransport) { $applyTransport.exitCode } else { $null } + runtimeWritePerformed = [bool]($applyReceipt -and $applyReceipt.runtimeWritePerformed) changed = $changed + managerVerified = $managerVerified verified = $verified + receiptVerified = $receiptVerified + terminal = $terminal before = $before after = $after - rollback = "systemd package backup under /var/lib/awoooi-host112-recovery/backups" - verifier = "forced_command_host112_guest_readiness_check" + applyReceipt = $applyReceipt + rollback = [pscustomobject]@{ + attempted = [bool]($applyReceipt -and $applyReceipt.rollbackAttempted) + verified = [bool]($applyReceipt -and $applyReceipt.rollbackVerified) + terminal = if ($applyReceipt) { [string]$applyReceipt.rollbackTerminal } else { "not_required" } + } + verifier = "wazuh_manager_service_process_agent_event_1514_enrollment_1515_and_api_55000" prohibitedActions = @("host_reboot", "vm_power_change", "vm_reset", "secret_read", "firewall_change", "database_write") } if ($changed) { Record-AgentEvent "host112_guest_recovered" "info" "112 圖形登入、VMware Tools、Wazuh 與 recovery timer 已由 Agent99 修復並驗證。" $result -Alert - } elseif ($applyAttempted -and -not $verified) { + } elseif ($applyAttempted -and (-not $verified -or -not $receiptVerified)) { Record-AgentEvent "host112_guest_recovery_failed" "critical" "112 受控修復後仍未通過 verifier;保留 failed checks 與 rollback evidence。" $result -Alert } $result @@ -5756,7 +5964,11 @@ $recoverySlo = $null $recoveryCoordinator = $null $recoveryPhases = @() $recoveryStartedAt = if ($Mode -eq "Recover") { Get-Date } else { $null } -$recoveryRunId = if ($Mode -eq "Recover") { "agent99-recovery-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) } else { "" } +$recoveryRunId = if ($Mode -eq "Recover") { + if ($AutomationRunId) { $AutomationRunId } else { "agent99-recovery-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) } +} else { + "" +} $recoveryConfig = Get-AgentRecoverySloConfig if ($Mode -in @("Status", "Recover")) { @@ -5792,7 +6004,11 @@ if ($Mode -in @("Status", "Recover")) { } if ($Mode -in @("Status", "Recover")) { - $host112Recovery = Invoke-AgentHost112GuestRecovery + $host112Recovery = Invoke-AgentHost112GuestRecovery ` + -RequestedRunId $AutomationRunId ` + -RequestedTraceId $TraceId ` + -RequestedWorkItemId $WorkItemId ` + -FallbackRunId $recoveryRunId if ($Mode -eq "Recover") { $recoveryPhases += New-AgentRecoveryPhase "host112_guest_readiness" $(if ($host112Recovery.verified) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds) } elseif (-not $host112Recovery.verified -and -not $recoveryTrigger) { diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index 7822fdc8a..2b7da6d83 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -126,11 +126,23 @@ def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> function = source[source.index("function Convert-AgentHost112GuestReadback") :] function = function[: function.index("function Test-HostReachability")] - assert '"/usr/local/sbin/awoooi-host112-guest-readiness --check"' in function - assert '"sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply"' in function + assert '"/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId"' in function + assert '"sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId"' in function assert function.count("Invoke-HostSshText $targetHost $checkCommand") == 2 - assert 'schemaVersion = "agent99_host112_guest_recovery_v1"' in function - assert 'verifier = "forced_command_host112_guest_readiness_check"' in function + assert 'schemaVersion = "agent99_host112_guest_recovery_v2"' in function + assert '$before.managerPreflightStatus -eq "ready"' in function + assert '$before.transportOk -and' in function + assert '$before.identityMatches -and' in function + assert "Invoke-HostSshText $targetHost $applyCommand 590 1" in function + assert 'receiptStatus -eq "written"' in function + assert 'verifier = "wazuh_manager_service_process_agent_event_1514_enrollment_1515_and_api_55000"' in function + for verifier_field in ( + "wazuhAnalysisdProcessCount", + "wazuhAgentEventPort1514Ready", + "wazuhAgentEnrollmentPort1515Tcp", + "wazuhManagerApiPort55000Tcp", + ): + assert verifier_field in function for forbidden in ("host_reboot", "vm_power_change", "vm_reset", "secret_read"): assert f'"{forbidden}"' in function diff --git a/scripts/reboot-recovery/awoooi-host112-guest-recovery.service b/scripts/reboot-recovery/awoooi-host112-guest-recovery.service index 9d8fb1209..bc95b3b81 100644 --- a/scripts/reboot-recovery/awoooi-host112-guest-recovery.service +++ b/scripts/reboot-recovery/awoooi-host112-guest-recovery.service @@ -5,8 +5,10 @@ Wants=network-online.target [Service] Type=oneshot -ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply -TimeoutStartSec=300 +ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer +# Script phase defaults sum to 500 seconds; the unit owns the 600-second +# outer contract while every child command remains independently bounded. +TimeoutStartSec=600 User=root Group=root NoNewPrivileges=true diff --git a/scripts/reboot-recovery/host112-guest-readiness.sh b/scripts/reboot-recovery/host112-guest-readiness.sh index 1a0a50ffb..682bead84 100755 --- a/scripts/reboot-recovery/host112-guest-readiness.sh +++ b/scripts/reboot-recovery/host112-guest-readiness.sh @@ -1,36 +1,186 @@ #!/usr/bin/env bash # Host 112 guest readiness and bounded recovery executor. +# Version: 2.0 +# Last modified: 2026-07-14 11:00 (Asia/Taipei) +# Last modified by: Codex +# Change: add correlated, cooldown-guarded Wazuh manager recovery and an +# independent process/port verifier without reboot, VM power, firewall, or +# secret access. set -uo pipefail MODE="check" +MODE_EXPLICIT=0 +TRACE_ID="" +RUN_ID="" +WORK_ITEM_ID="" + INDEXER_RETRY_AFTER_SECONDS="${HOST112_INDEXER_RETRY_AFTER_SECONDS:-210}" INDEXER_RETRY_COOLDOWN_SECONDS="${HOST112_INDEXER_RETRY_COOLDOWN_SECONDS:-600}" -INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-240}" +INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-180}" +SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-30}" +MANAGER_RETRY_AFTER_SECONDS="${HOST112_MANAGER_RETRY_AFTER_SECONDS:-120}" +MANAGER_RETRY_COOLDOWN_SECONDS="${HOST112_MANAGER_RETRY_COOLDOWN_SECONDS:-600}" +MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}" +MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-120}" +MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}" +MANAGER_VERIFY_INTERVAL_SECONDS="${HOST112_MANAGER_VERIFY_INTERVAL_SECONDS:-3}" +MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-45}" +MANAGER_MIN_MEMORY_AVAILABLE_KB="${HOST112_MANAGER_MIN_MEMORY_AVAILABLE_KB:-524288}" +MANAGER_MIN_DISK_AVAILABLE_KB="${HOST112_MANAGER_MIN_DISK_AVAILABLE_KB:-1048576}" +MANAGER_MAX_DISK_USED_PERCENT="${HOST112_MANAGER_MAX_DISK_USED_PERCENT:-95}" STATE_DIR="${HOST112_RECOVERY_STATE_DIR:-/var/lib/awoooi-host112-recovery}" -RETRY_MARKER="${HOST112_INDEXER_RETRY_MARKER:-/run/awoooi-host112-indexer-retry.epoch}" +INDEXER_RETRY_MARKER="${HOST112_INDEXER_RETRY_MARKER:-/run/awoooi-host112-indexer-retry.epoch}" +MANAGER_RETRY_MARKER="${HOST112_MANAGER_RETRY_MARKER:-/run/awoooi-host112-manager-retry.epoch}" +MANAGER_LOCK_FILE="${HOST112_MANAGER_LOCK_FILE:-/run/awoooi-host112-manager-recovery.lock}" usage() { cat <<'USAGE' -Usage: host112-guest-readiness.sh [--check|--apply] +Usage: + host112-guest-readiness.sh --check [identity] + host112-guest-readiness.sh --dry-run --trace-id ID --run-id ID --work-item-id ID + host112-guest-readiness.sh --apply --trace-id ID --run-id ID --work-item-id ID ---check emits a bounded no-secret guest readiness row. +Internal systemd mode: + host112-guest-readiness.sh --apply-timer + +--check emits a bounded no-secret guest readiness row and never writes. +--dry-run evaluates the exact Wazuh manager preflight and never writes. --apply performs only allowlisted idempotent recovery for graphical.target, -LightDM, open-vm-tools, SSH and a cooldown-guarded Wazuh Indexer start retry. +LightDM, open-vm-tools, SSH, Wazuh Indexer, and one cooldown-guarded Wazuh +manager reset/start attempt. The independent verifier requires the manager +process plus agent event 1514, enrollment 1515, and manager API 55000 listeners. USAGE } +set_mode() { + local requested="$1" + if [ "$MODE_EXPLICIT" -eq 1 ]; then + echo "BLOCKER duplicate_mode" >&2 + exit 64 + fi + MODE="$requested" + MODE_EXPLICIT=1 +} + +require_value() { + local flag="$1" value="${2:-}" + if [ -z "$value" ]; then + echo "BLOCKER missing_value=$flag" >&2 + exit 64 + fi +} + while [ "$#" -gt 0 ]; do case "$1" in - --check) MODE="check" ;; - --apply) MODE="apply" ;; + --check) set_mode "check" ;; + --dry-run) set_mode "dry_run" ;; + --apply) set_mode "apply" ;; + --apply-timer) set_mode "timer_apply" ;; + --trace-id) + require_value "$1" "${2:-}" + TRACE_ID="$2" + shift + ;; + --run-id) + require_value "$1" "${2:-}" + RUN_ID="$2" + shift + ;; + --work-item-id) + require_value "$1" "${2:-}" + WORK_ITEM_ID="$2" + shift + ;; -h|--help) usage; exit 0 ;; *) echo "BLOCKER unknown_argument=$1" >&2; exit 64 ;; esac shift done +safe_identity() { + [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$ ]] +} + +identity_count=0 +[ -n "$TRACE_ID" ] && identity_count=$((identity_count + 1)) +[ -n "$RUN_ID" ] && identity_count=$((identity_count + 1)) +[ -n "$WORK_ITEM_ID" ] && identity_count=$((identity_count + 1)) +if [ "$identity_count" -ne 0 ] && [ "$identity_count" -ne 3 ]; then + echo "BLOCKER incomplete_control_identity" >&2 + exit 64 +fi +if [ "$identity_count" -eq 3 ]; then + for identity in "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID"; do + if ! safe_identity "$identity"; then + echo "BLOCKER unsafe_control_identity" >&2 + exit 64 + fi + done +fi +if { [ "$MODE" = "dry_run" ] || [ "$MODE" = "apply" ]; } && [ "$identity_count" -ne 3 ]; then + echo "BLOCKER control_identity_required" >&2 + exit 64 +fi +if [ "$MODE" = "timer_apply" ] && [ "$identity_count" -ne 0 ]; then + echo "BLOCKER timer_identity_is_system_generated" >&2 + exit 64 +fi + actions=() +runtime_write_performed=0 +manager_runtime_write_performed=0 +receipt_write_performed=0 +receipt_status="not_requested" +receipt_ref="none" +manager_attempt_count=0 +manager_start_exit=not_attempted +manager_terminal="observed" +rollback_attempted=0 +rollback_verified=0 +rollback_terminal=not_required + +is_uint() { + [[ "$1" =~ ^[0-9]+$ ]] +} + +for timing_value in \ + "$INDEXER_START_TIMEOUT_SECONDS" \ + "$SERVICE_START_TIMEOUT_SECONDS" \ + "$MANAGER_RESET_TIMEOUT_SECONDS" \ + "$MANAGER_START_TIMEOUT_SECONDS" \ + "$MANAGER_VERIFY_TIMEOUT_SECONDS" \ + "$MANAGER_VERIFY_INTERVAL_SECONDS" \ + "$MANAGER_ROLLBACK_TIMEOUT_SECONDS"; do + if ! is_uint "$timing_value" || [ "$timing_value" -le 0 ]; then + echo "BLOCKER invalid_timeout_configuration" >&2 + exit 64 + fi +done +APPLY_PHASE_TIMEOUT_BUDGET_SECONDS=$(( + (4 * SERVICE_START_TIMEOUT_SECONDS) + + INDEXER_START_TIMEOUT_SECONDS + + MANAGER_RESET_TIMEOUT_SECONDS + + MANAGER_START_TIMEOUT_SECONDS + + MANAGER_VERIFY_TIMEOUT_SECONDS + + MANAGER_ROLLBACK_TIMEOUT_SECONDS +)) +if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt 570 ]; then + echo "BLOCKER apply_phase_timeout_budget_exceeds_570_seconds" >&2 + exit 64 +fi + +write_epoch_marker() { + local marker="$1" epoch="$2" tmp + tmp="${marker}.tmp.$$" + if printf '%s\n' "$epoch" >"$tmp" \ + && chmod 0644 "$tmp" \ + && mv "$tmp" "$marker"; then + return 0 + fi + rm -f "$tmp" >/dev/null 2>&1 || true + return 1 +} unit_active() { local unit="$1" state @@ -44,8 +194,14 @@ unit_enabled() { printf '%s' "${state:-unknown}" } +unit_property() { + local unit="$1" property="$2" state + state="$(systemctl show "$unit" -p "$property" --value 2>/dev/null || true)" + printf '%s' "${state:-unknown}" +} + unit_loaded() { - [ "$(systemctl show "$1" -p LoadState --value 2>/dev/null || true)" = "loaded" ] + [ "$(unit_property "$1" LoadState)" = "loaded" ] } record_action() { @@ -58,8 +214,9 @@ start_if_inactive() { if [ "$state" = "active" ] || [ "$state" = "activating" ]; then return 0 fi - if systemctl start "$unit"; then + if timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl start "$unit"; then record_action "$action" + runtime_write_performed=1 else record_action "${action}_failed" fi @@ -73,21 +230,225 @@ enable_if_disabled() { fi if systemctl enable "$unit" >/dev/null; then record_action "$action" + runtime_write_performed=1 else record_action "${action}_failed" fi } -if [ "$MODE" = "apply" ]; then +tcp_port_listening() { + local port="$1" + ss -H -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$port$" +} + +udp_port_listening() { + local port="$1" + ss -H -lun 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$port$" +} + +process_count() { + local process_name="$1" count + count="$(pgrep -xc "$process_name" 2>/dev/null || true)" + printf '%s' "${count:-0}" +} + +memory_available_kb() { + local available + available="$(awk '/^MemAvailable:/ {print $2; exit}' /proc/meminfo 2>/dev/null || true)" + printf '%s' "${available:-0}" +} + +disk_readback() { + local target="/var/ossec" line + [ -e "$target" ] || target="/" + line="$(df -Pk "$target" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $4 " " $5; exit}')" + if [ -n "$line" ]; then + printf '%s' "$line" + else + printf '0 100' + fi +} + +capture_manager_state() { + local disk_values + manager_load_state="$(unit_property wazuh-manager.service LoadState)" + manager_active_state="$(unit_property wazuh-manager.service ActiveState)" + manager_sub_state="$(unit_property wazuh-manager.service SubState)" + manager_result="$(unit_property wazuh-manager.service Result)" + manager_failed_state_observed=0 + if [ "$manager_active_state" = "failed" ] \ + || { [ "$manager_result" != "success" ] && [ "$manager_result" != "unknown" ]; }; then + manager_failed_state_observed=1 + fi + analysisd_count="$(process_count wazuh-analysisd)" + event_1514_tcp=0 + event_1514_udp=0 + enrollment_1515_tcp=0 + manager_api_55000_tcp=0 + tcp_port_listening 1514 && event_1514_tcp=1 + udp_port_listening 1514 && event_1514_udp=1 + tcp_port_listening 1515 && enrollment_1515_tcp=1 + tcp_port_listening 55000 && manager_api_55000_tcp=1 + manager_event_port_ready=0 + if [ "$event_1514_tcp" -eq 1 ] || [ "$event_1514_udp" -eq 1 ]; then + manager_event_port_ready=1 + fi + manager_orphan_analysisd=0 + if [ "$manager_active_state" != "active" ] && is_uint "$analysisd_count" && [ "$analysisd_count" -gt 0 ]; then + manager_orphan_analysisd=1 + fi + manager_verifier_ready=0 + if [ "$manager_active_state" = "active" ] \ + && is_uint "$analysisd_count" && [ "$analysisd_count" -gt 0 ] \ + && [ "$manager_event_port_ready" -eq 1 ] \ + && [ "$enrollment_1515_tcp" -eq 1 ] \ + && [ "$manager_api_55000_tcp" -eq 1 ]; then + manager_verifier_ready=1 + fi + manager_memory_available_kb="$(memory_available_kb)" + disk_values="$(disk_readback)" + read -r manager_disk_available_kb manager_disk_used_percent <<<"$disk_values" + manager_resource_ready=0 + if is_uint "$manager_memory_available_kb" \ + && is_uint "$manager_disk_available_kb" \ + && is_uint "$manager_disk_used_percent" \ + && [ "$manager_memory_available_kb" -ge "$MANAGER_MIN_MEMORY_AVAILABLE_KB" ] \ + && [ "$manager_disk_available_kb" -ge "$MANAGER_MIN_DISK_AVAILABLE_KB" ] \ + && [ "$manager_disk_used_percent" -lt "$MANAGER_MAX_DISK_USED_PERCENT" ]; then + manager_resource_ready=1 + fi +} + +manager_cooldown_remaining() { + local now_epoch="$1" last_retry_epoch=0 elapsed + if [ -r "$MANAGER_RETRY_MARKER" ]; then + read -r last_retry_epoch <"$MANAGER_RETRY_MARKER" || last_retry_epoch=0 + fi + is_uint "$last_retry_epoch" || last_retry_epoch=0 + elapsed=$((now_epoch - last_retry_epoch)) + if [ "$elapsed" -lt 0 ]; then + printf '%s' "$MANAGER_RETRY_COOLDOWN_SECONDS" + elif [ "$elapsed" -lt "$MANAGER_RETRY_COOLDOWN_SECONDS" ]; then + printf '%s' "$((MANAGER_RETRY_COOLDOWN_SECONDS - elapsed))" + else + printf '0' + fi +} + +evaluate_manager_preflight() { + local now_epoch="$1" uptime_for_retry="$2" + manager_cooldown_remaining_seconds="$(manager_cooldown_remaining "$now_epoch")" + manager_preflight_status="ready" + if ! command -v systemctl >/dev/null 2>&1 \ + || ! command -v timeout >/dev/null 2>&1 \ + || ! command -v ss >/dev/null 2>&1 \ + || ! command -v flock >/dev/null 2>&1; then + manager_preflight_status="blocked_required_tool_missing" + elif [ "$manager_load_state" != "loaded" ]; then + manager_preflight_status="blocked_manager_unit_not_loaded" + elif [ "$manager_verifier_ready" -eq 1 ]; then + manager_preflight_status="already_verified_healthy" + elif [ "$manager_active_state" = "active" ]; then + manager_preflight_status="blocked_active_verifier_mismatch" + elif [ "$manager_orphan_analysisd" -eq 1 ]; then + manager_preflight_status="blocked_orphan_analysisd_no_write" + elif [ "$manager_resource_ready" -ne 1 ]; then + manager_preflight_status="blocked_resource_preflight_no_write" + elif [ "$(unit_active wazuh-indexer.service)" != "active" ]; then + manager_preflight_status="blocked_indexer_dependency_no_write" + elif ! is_uint "$uptime_for_retry" || [ "$uptime_for_retry" -lt "$MANAGER_RETRY_AFTER_SECONDS" ]; then + manager_preflight_status="blocked_boot_settle_no_write" + elif [ "$manager_cooldown_remaining_seconds" -gt 0 ]; then + manager_preflight_status="blocked_retry_cooldown_no_write" + fi +} + +wait_for_manager_verifier() { + local deadline now + deadline=$(( $(date +%s) + MANAGER_VERIFY_TIMEOUT_SECONDS )) + while true; do + capture_manager_state + [ "$manager_verifier_ready" -eq 1 ] && return 0 + now="$(date +%s)" + [ "$now" -ge "$deadline" ] && return 1 + sleep "$MANAGER_VERIFY_INTERVAL_SECONDS" + done +} + +rollback_manager_to_inactive() { + rollback_attempted=1 + rollback_terminal="bounded_stop_attempted" + if timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service; then + record_action "rollback_stop_wazuh_manager" + else + record_action "rollback_stop_wazuh_manager_failed" + fi + capture_manager_state + if [ "$manager_active_state" != "active" ] \ + && is_uint "$analysisd_count" && [ "$analysisd_count" -eq 0 ] \ + && [ "$manager_event_port_ready" -eq 0 ] \ + && [ "$enrollment_1515_tcp" -eq 0 ] \ + && [ "$manager_api_55000_tcp" -eq 0 ]; then + rollback_verified=1 + rollback_terminal="bounded_stop_verified" + else + rollback_terminal="bounded_stop_not_verified" + fi +} + +write_public_receipt() { + local receipt_name receipt_path tmp generated_at receipt_line + receipt_name="$(printf '%s' "$RUN_ID" | tr -c 'A-Za-z0-9._+-' '_')" + [ -n "$receipt_name" ] || receipt_name="unknown" + if [ "$MODE" = "timer_apply" ]; then + receipt_path="$STATE_DIR/last-timer-manager-recovery-receipt.txt" + else + receipt_path="$STATE_DIR/receipts/${receipt_name}.txt" + fi + generated_at="$(date --iso-8601=seconds)" + receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v1 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed manager_runtime_write_performed=$manager_runtime_write_performed rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" + if install -d -m 0750 "$(dirname "$receipt_path")"; then + tmp="${receipt_path}.tmp.$$" + if printf '%s\n' "$receipt_line" >"$tmp" \ + && chmod 0640 "$tmp" \ + && mv "$tmp" "$receipt_path"; then + receipt_write_performed=1 + receipt_status="written" + receipt_ref="host112-manager-recovery:$RUN_ID" + return 0 + fi + rm -f "$tmp" >/dev/null 2>&1 || true + fi + receipt_status="write_failed" + return 1 +} + +boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)" +uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)" +if [ "$MODE" = "timer_apply" ]; then + invocation_identity="${INVOCATION_ID:-${boot_id}-$(date +%s)}" + invocation_identity="$(printf '%s' "$invocation_identity" | tr -c 'A-Za-z0-9._+-' '-')" + TRACE_ID="host112-timer:$invocation_identity" + RUN_ID="host112-timer:$invocation_identity" + WORK_ITEM_ID="HOST112-WAZUH-MANAGER-RECOVERY" +fi + +if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then if [ "${EUID:-$(id -u)}" -ne 0 ]; then echo "BLOCKER host112_apply_requires_root" >&2 exit 77 fi + exec 9>"$MANAGER_LOCK_FILE" + if ! flock -n 9; then + echo "BLOCKER host112_manager_single_flight_busy" >&2 + exit 75 + fi current_default="$(systemctl get-default 2>/dev/null || true)" if [ "$current_default" != "graphical.target" ]; then if systemctl set-default graphical.target >/dev/null; then record_action "set_default_graphical_target" + runtime_write_performed=1 else record_action "set_default_graphical_target_failed" fi @@ -103,27 +464,29 @@ if [ "$MODE" = "apply" ]; then record_action "ssh_service_missing" fi - uptime_for_retry="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)" indexer_state="$(unit_active wazuh-indexer.service)" if unit_loaded wazuh-indexer.service \ && [ "$indexer_state" != "active" ] \ && [ "$indexer_state" != "activating" ] \ - && [ "$uptime_for_retry" -ge "$INDEXER_RETRY_AFTER_SECONDS" ]; then + && is_uint "$uptime_seconds" \ + && [ "$uptime_seconds" -ge "$INDEXER_RETRY_AFTER_SECONDS" ]; then now_epoch="$(date +%s)" last_retry_epoch=0 - if [ -r "$RETRY_MARKER" ]; then - read -r last_retry_epoch <"$RETRY_MARKER" || last_retry_epoch=0 - fi - if ! [[ "$last_retry_epoch" =~ ^[0-9]+$ ]]; then - last_retry_epoch=0 + if [ -r "$INDEXER_RETRY_MARKER" ]; then + read -r last_retry_epoch <"$INDEXER_RETRY_MARKER" || last_retry_epoch=0 fi + is_uint "$last_retry_epoch" || last_retry_epoch=0 if [ $((now_epoch - last_retry_epoch)) -ge "$INDEXER_RETRY_COOLDOWN_SECONDS" ]; then - printf '%s\n' "$now_epoch" >"$RETRY_MARKER" - systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true - if timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then - record_action "retry_wazuh_indexer" + if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then + systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true + runtime_write_performed=1 + if timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then + record_action "retry_wazuh_indexer" + else + record_action "retry_wazuh_indexer_failed" + fi else - record_action "retry_wazuh_indexer_failed" + record_action "wazuh_indexer_marker_write_failed_no_write" fi else record_action "wazuh_indexer_retry_cooldown" @@ -131,8 +494,72 @@ if [ "$MODE" = "apply" ]; then fi fi -boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)" -uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo unknown)" +capture_manager_state +before_manager_active="$manager_active_state" +before_manager_result="$manager_result" +before_manager_orphan="$manager_orphan_analysisd" +before_manager_memory="$manager_memory_available_kb" +before_manager_disk="$manager_disk_available_kb" +before_manager_disk_used="$manager_disk_used_percent" +before_event_1514_ready="$manager_event_port_ready" +before_enrollment_1515="$enrollment_1515_tcp" +before_manager_api_55000="$manager_api_55000_tcp" +now_epoch="$(date +%s)" +evaluate_manager_preflight "$now_epoch" "$uptime_seconds" + +if [ "$MODE" = "dry_run" ]; then + case "$manager_preflight_status" in + ready) manager_terminal="dry_run_ready" ;; + already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;; + *) manager_terminal="dry_run_${manager_preflight_status}" ;; + esac +elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then + case "$manager_preflight_status" in + already_verified_healthy) + manager_terminal="idempotent_already_verified_healthy" + ;; + ready) + if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then + manager_attempt_count=1 + if timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then + record_action "reset_failed_wazuh_manager" + runtime_write_performed=1 + manager_runtime_write_performed=1 + if timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then + manager_start_exit=0 + else + manager_start_exit=$? + fi + if wait_for_manager_verifier; then + record_action "start_wazuh_manager_verified" + manager_terminal="verified_healthy" + else + record_action "start_wazuh_manager_verifier_failed" + rollback_manager_to_inactive + if [ "$rollback_verified" -eq 1 ]; then + manager_terminal="rolled_back_verifier_failed" + else + manager_terminal="rollback_failed_degraded" + fi + fi + else + manager_start_exit="reset_failed_no_start" + record_action "reset_failed_wazuh_manager_failed_no_start" + manager_terminal="no_start_reset_failed" + fi + else + record_action "wazuh_manager_marker_write_failed_no_write" + manager_terminal="no_write_cooldown_marker_write_failed" + fi + ;; + *) + manager_terminal="no_write_${manager_preflight_status}" + ;; + esac + capture_manager_state + write_public_receipt || manager_terminal="${manager_terminal}_receipt_failed" +fi + systemd_state="$(systemctl is-system-running 2>/dev/null || true)" systemd_state="${systemd_state:-unknown}" default_target="$(systemctl get-default 2>/dev/null || true)" @@ -151,9 +578,7 @@ startup_enabled="$(unit_enabled awoooi-host112-guest-recovery.timer)" startup_active="$(unit_active awoooi-host112-guest-recovery.timer)" ssh_port_listening=0 -if ss -H -ltn 2>/dev/null | awk '{print $4}' | grep -Eq '(^|:)22$'; then - ssh_port_listening=1 -fi +tcp_port_listening 22 && ssh_port_listening=1 ssh_ready=0 if [ "$ssh_service" = "active" ] \ && { [ "$ssh_enabled" = "enabled" ] || [ "$ssh_enabled" = "static" ]; } \ @@ -178,9 +603,9 @@ fi services_ready=0 if [ "$wazuh_indexer" = "active" ] \ - && [ "$wazuh_manager" = "active" ] \ && [ "$wazuh_dashboard" = "active" ] \ - && [ "$filebeat" = "active" ]; then + && [ "$filebeat" = "active" ] \ + && [ "$manager_verifier_ready" -eq 1 ]; then services_ready=1 fi @@ -203,15 +628,30 @@ if [ "${#actions[@]}" -gt 0 ]; then action_text="$(IFS=,; printf '%s' "${actions[*]}")" fi -readback="boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready action_count=${#actions[@]} actions=$action_text" +trace_output="${TRACE_ID:-none}" +run_output="${RUN_ID:-none}" +work_item_output="${WORK_ITEM_ID:-none}" +readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS runtime_write_performed=$runtime_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" printf '%s\n' "$readback" -if [ "$MODE" = "apply" ]; then +if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then install -d -m 0750 "$STATE_DIR" tmp_readback="${STATE_DIR}/last-readback.txt.tmp.$$" - printf 'generated_at=%s mode=%s %s\n' "$(date --iso-8601=seconds)" "$MODE" "$readback" >"$tmp_readback" - chmod 0640 "$tmp_readback" - mv "$tmp_readback" "${STATE_DIR}/last-readback.txt" + if printf 'generated_at=%s %s\n' "$(date --iso-8601=seconds)" "$readback" >"$tmp_readback" \ + && chmod 0640 "$tmp_readback" \ + && mv "$tmp_readback" "${STATE_DIR}/last-readback.txt"; then + : + else + rm -f "$tmp_readback" >/dev/null 2>&1 || true + exit 74 + fi fi -[ "$guest_ready" -eq 1 ] +if [ "$MODE" = "dry_run" ]; then + case "$manager_preflight_status" in + ready|already_verified_healthy) exit 0 ;; + *) exit 1 ;; + esac +fi + +[ "$guest_ready" -eq 1 ] && [ "$receipt_status" != "write_failed" ] diff --git a/scripts/reboot-recovery/install-host112-guest-recovery.sh b/scripts/reboot-recovery/install-host112-guest-recovery.sh index c23504007..f5ca4c85c 100755 --- a/scripts/reboot-recovery/install-host112-guest-recovery.sh +++ b/scripts/reboot-recovery/install-host112-guest-recovery.sh @@ -1,4 +1,11 @@ #!/usr/bin/env bash +# Host 112 guest recovery installer. +# Version: 2.0 +# Last modified: 2026-07-14 11:00 (Asia/Taipei) +# Last modified by: Codex +# Change: install and roll back the argument-regex sudo boundary required for +# correlated Wazuh manager recovery receipts. + set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -16,6 +23,7 @@ SERVICE_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.service" TIMER_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.timer" SUDOERS_TARGET="/etc/sudoers.d/awoooi-host112-agent99" STATE_ROOT="/var/lib/awoooi-host112-recovery" +SUDOERS_ARGUMENT_REGEX='^--apply --trace-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --run-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --work-item-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$' usage() { cat <<'USAGE' @@ -44,6 +52,20 @@ while [ "$#" -gt 0 ]; do shift done +sudo_policy_readback() { + if [ "${EUID:-$(id -u)}" -eq 0 ]; then + sudo -n -U "$TARGET_USER" -l "$@" 2>/dev/null || true + else + sudo -n -l "$@" 2>/dev/null || true + fi +} + +sudo_policy_has_nopasswd() { + local output + output="$(sudo_policy_readback "$@")" + printf '%s\n' "$output" | grep -Fq "NOPASSWD: $READINESS_TARGET" +} + check() { echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1" echo "MODE=check" @@ -56,6 +78,32 @@ check() { done echo "TIMER_ENABLED=$(systemctl is-enabled awoooi-host112-guest-recovery.timer 2>/dev/null || true)" echo "TIMER_ACTIVE=$(systemctl is-active awoooi-host112-guest-recovery.timer 2>/dev/null || true)" + if [ -f "$SUDOERS_TARGET" ] && visudo -cf "$SUDOERS_TARGET" >/dev/null 2>&1; then + echo "SUDOERS_VALID=1" + else + echo "SUDOERS_VALID=0" + fi + if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply \ + --trace-id host112-policy-check \ + --run-id host112-policy-check \ + --work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then + echo "SUDO_CORRELATED_APPLY_NOPASSWD=1" + else + echo "SUDO_CORRELATED_APPLY_NOPASSWD=0" + fi + if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply; then + echo "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=1" + else + echo "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=0" + fi + if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply \ + --trace-id 'unsafe value' \ + --run-id host112-policy-check \ + --work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then + echo "SUDO_UNSAFE_IDENTITY_NOPASSWD=1" + else + echo "SUDO_UNSAFE_IDENTITY_NOPASSWD=0" + fi if [ -x "$READINESS_TARGET" ]; then "$READINESS_TARGET" --check || true fi @@ -152,7 +200,8 @@ install -m 0644 "$TIMER_SOURCE" "$TIMER_TARGET" sudoers_tmp="$(mktemp)" trap 'rm -f "$sudoers_tmp"' EXIT -printf '%s ALL=(root) NOPASSWD: %s --apply\n' "$TARGET_USER" "$READINESS_TARGET" >"$sudoers_tmp" +printf '%s ALL=(root) NOPASSWD: %s %s\n' \ + "$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_ARGUMENT_REGEX" >"$sudoers_tmp" visudo -cf "$sudoers_tmp" >/dev/null install -m 0440 "$sudoers_tmp" "$SUDOERS_TARGET" diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index 1ec0582c1..189fffb91 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -791,8 +791,8 @@ def source_controls() -> dict[str, bool]: source_file( "scripts/reboot-recovery/awoooi-host112-guest-recovery.service" ), - "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply", - "TimeoutStartSec=300", + "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer", + "TimeoutStartSec=600", ), "host_112_guest_recovery_timer_source_present": file_contains( source_file( diff --git a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py new file mode 100644 index 000000000..2b9b22b98 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +READINESS = ROOT / "scripts/reboot-recovery/host112-guest-readiness.sh" +INSTALLER = ROOT / "scripts/reboot-recovery/install-host112-guest-recovery.sh" +SERVICE = ROOT / "scripts/reboot-recovery/awoooi-host112-guest-recovery.service" +CONTROL = ROOT / "agent99-control-plane.ps1" +BOOTSTRAP = ROOT / "agent99-bootstrap.ps1" + + +def read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def isolated_env(tmp_path: Path) -> dict[str, str]: + env = os.environ.copy() + env.update( + { + "HOST112_RECOVERY_STATE_DIR": str(tmp_path / "state"), + "HOST112_INDEXER_RETRY_MARKER": str(tmp_path / "indexer.epoch"), + "HOST112_MANAGER_RETRY_MARKER": str(tmp_path / "manager.epoch"), + "HOST112_MANAGER_LOCK_FILE": str(tmp_path / "manager.lock"), + } + ) + return env + + +def test_check_and_dry_run_are_real_no_write_execution_paths(tmp_path: Path) -> None: + identity = [ + "--trace-id", + "trace:D037-host112-dry-run", + "--run-id", + "run:D037-host112-dry-run", + "--work-item-id", + "D037-WAZUH-HOST112", + ] + env = isolated_env(tmp_path) + + check = subprocess.run( + ["bash", str(READINESS), "--check", *identity], + text=True, + capture_output=True, + env=env, + check=False, + timeout=20, + ) + dry_run = subprocess.run( + ["bash", str(READINESS), "--dry-run", *identity], + text=True, + capture_output=True, + env=env, + check=False, + timeout=20, + ) + + assert check.returncode in {0, 1} + assert dry_run.returncode in {0, 1} + for result, mode in ((check, "check"), (dry_run, "dry_run")): + output = result.stdout + result.stderr + assert f"mode={mode}" in output + assert "trace_id=trace:D037-host112-dry-run" in output + assert "run_id=run:D037-host112-dry-run" in output + assert "work_item_id=D037-WAZUH-HOST112" in output + assert "runtime_write_performed=0" in output + assert "manager_runtime_write_performed=0" in output + assert "receipt_write_performed=0" in output + assert "manager_preflight_status=" in output + assert "apply_phase_timeout_budget_seconds=500" in output + assert not (tmp_path / "state").exists() + assert not (tmp_path / "indexer.epoch").exists() + assert not (tmp_path / "manager.epoch").exists() + assert not (tmp_path / "manager.lock").exists() + + +def test_unsafe_or_incomplete_identity_fails_before_any_state_path(tmp_path: Path) -> None: + env = isolated_env(tmp_path) + unsafe = subprocess.run( + [ + "bash", + str(READINESS), + "--dry-run", + "--trace-id", + "unsafe value", + "--run-id", + "run-safe", + "--work-item-id", + "D037-WAZUH-HOST112", + ], + text=True, + capture_output=True, + env=env, + check=False, + timeout=10, + ) + incomplete = subprocess.run( + [ + "bash", + str(READINESS), + "--dry-run", + "--trace-id", + "trace-safe", + ], + text=True, + capture_output=True, + env=env, + check=False, + timeout=10, + ) + + assert unsafe.returncode == 64 + assert "BLOCKER unsafe_control_identity" in (unsafe.stdout + unsafe.stderr) + assert incomplete.returncode == 64 + assert "BLOCKER incomplete_control_identity" in ( + incomplete.stdout + incomplete.stderr + ) + assert not any(tmp_path.iterdir()) + + +def test_manager_preflight_verifier_receipt_and_rollback_are_complete() -> None: + source = read(READINESS) + + for field in ( + "manager_failed_state_observed", + "manager_orphan_analysisd", + "manager_memory_available_kb", + "manager_disk_available_kb", + "manager_disk_used_percent", + "wazuh_analysisd_process_count", + "wazuh_agent_event_port_1514_tcp", + "wazuh_agent_event_port_1514_udp", + "wazuh_agent_enrollment_port_1515_tcp", + "wazuh_manager_api_port_55000_tcp", + "manager_independent_verifier_ready", + ): + assert field in source + assert 'manager_active_state="$(unit_property wazuh-manager.service ActiveState)"' in source + assert 'manager_result="$(unit_property wazuh-manager.service Result)"' in source + assert 'analysisd_count="$(process_count wazuh-analysisd)"' in source + assert source.count( + 'timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service' + ) == 1 + assert "manager_attempt_count=1" in source + assert 'timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service' in source + assert 'rollback_terminal="bounded_stop_verified"' in source + assert "schema_version=host112_wazuh_manager_recovery_receipt_v1" in source + assert "trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID" in source + assert 'receipt_status="write_failed"' in source + assert "prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" in source + + preflight = source[ + source.index("evaluate_manager_preflight() {") : source.index( + "wait_for_manager_verifier() {" + ) + ] + assert 'elif [ "$(unit_active wazuh-indexer.service)" != "active" ]; then' in preflight + assert 'manager_preflight_status="blocked_indexer_dependency_no_write"' in preflight + + +def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None: + source = read(READINESS) + marker_function = source[ + source.index("write_epoch_marker() {") : source.index("unit_active() {") + ] + assert 'tmp="${marker}.tmp.$$"' in marker_function + assert 'printf \'%s\\n\' "$epoch" >"$tmp"' in marker_function + assert 'chmod 0644 "$tmp"' in marker_function + assert 'mv "$tmp" "$marker"' in marker_function + assert "return 1" in marker_function + + manager_start = source.index( + ' if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then' + ) + manager_end = source.index(" ;;", manager_start) + manager_branch = source[manager_start:manager_end] + marker_failure_at = manager_branch.index( + ' else\n record_action "wazuh_manager_marker_write_failed_no_write"' + ) + success = manager_branch[:marker_failure_at] + failure = manager_branch[marker_failure_at:] + assert "systemctl reset-failed wazuh-manager.service" in success + assert "systemctl start wazuh-manager.service" in success + assert 'timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service' in success + assert "manager_attempt_count=1" in success + assert "systemctl reset-failed wazuh-manager.service" not in failure + assert "systemctl start wazuh-manager.service" not in failure + assert "wazuh_manager_marker_write_failed_no_write" in failure + assert 'manager_terminal="no_write_cooldown_marker_write_failed"' in failure + + indexer_start = source.index( + ' if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then' + ) + indexer_end = source.index(" else\n record_action", indexer_start) + indexer_branch = source[indexer_start:indexer_end] + indexer_success, indexer_failure = indexer_branch.split(" else\n", maxsplit=1) + assert "systemctl reset-failed wazuh-indexer.service" in indexer_success + assert "systemctl start wazuh-indexer.service" in indexer_success + assert "systemctl reset-failed wazuh-indexer.service" not in indexer_failure + assert "systemctl start wazuh-indexer.service" not in indexer_failure + assert "wazuh_indexer_marker_write_failed_no_write" in indexer_failure + + +def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None: + source = read(READINESS) + service = read(SERVICE) + control = read(CONTROL) + + assert 'SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-30}"' in source + assert 'INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-180}"' in source + assert 'MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-120}"' in source + assert 'MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}"' in source + assert 'MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}"' in source + assert 'MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-45}"' in source + assert 'if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt 570 ]; then' in source + assert "TimeoutStartSec=600" in service + assert "Invoke-HostSshText $targetHost $applyCommand 590 1" in control + + +def test_installer_uses_argument_regex_and_post_install_policy_readback() -> None: + source = read(INSTALLER) + + assert "SUDOERS_ARGUMENT_REGEX='^--apply --trace-id " in source + assert "--run-id [A-Za-z0-9]" in source + assert "--work-item-id [A-Za-z0-9]" in source + assert '"$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_ARGUMENT_REGEX"' in source + assert 'visudo -cf "$sudoers_tmp"' in source + assert "SUDO_CORRELATED_APPLY_NOPASSWD" in source + assert "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD" in source + assert "SUDO_UNSAFE_IDENTITY_NOPASSWD" in source + assert "--trace-id 'unsafe value'" in source + assert "--rollback" in source + + +def test_exact_sudoers_argument_regex_parses_when_visudo_is_available() -> None: + visudo = shutil.which("visudo") + if not visudo: + return + candidate = ( + "kali ALL=(root) NOPASSWD: " + "/usr/local/sbin/awoooi-host112-guest-readiness " + "^--apply --trace-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} " + "--run-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} " + "--work-item-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$\n" + ) + result = subprocess.run( + [visudo, "-cf", "-"], + input=candidate, + text=True, + capture_output=True, + check=False, + timeout=10, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_agent99_propagates_identity_and_requires_independent_receipt() -> None: + control = read(CONTROL) + bootstrap = read(BOOTSTRAP) + function = control[control.index("function Convert-AgentHost112GuestReadback") :] + function = function[: function.index("function Test-HostReachability")] + + for parameter in ("AutomationRunId", "TraceId", "WorkItemId"): + assert f"[string]${parameter} = \"\"" in control + assert f'[string]`${parameter} = ""' in bootstrap + assert 'reason = "controlled_dispatch_identity_unsafe"' in control + assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function + assert '$before.managerPreflightStatus -eq "ready"' in function + assert 'receiptStatus -eq "written"' in function + assert '$after.managerVerified' in function + assert 'name = "wazuh_manager_service_active"' in function + assert 'name = "wazuh_analysisd_process_present"' in function + assert 'name = "wazuh_agent_event_1514_ready"' in function + assert 'name = "wazuh_agent_enrollment_1515_tcp"' in function + assert 'name = "wazuh_manager_api_55000_tcp"' in function + assert "host112_readback_transport_unavailable" in function + assert 'terminal = "no_write_control_identity_incomplete_or_unsafe"' in function diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 56b76c6cf..585167f62 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -107,7 +107,8 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "restrict,command=" in host112_installer assert "visudo -cf" in host112_installer assert "--rollback" in host112_installer - assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply" in host112_service + assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer" in host112_service + assert "TimeoutStartSec=600" in host112_service assert "OnBootSec=20s" in host112_timer assert "OnUnitInactiveSec=60s" in host112_timer