From c9bb4bbfc6af05c41df7c3af546ba44cbcf45708 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 12:35:15 +0800 Subject: [PATCH] fix(recovery): bind host112 closure artifacts --- agent99-contract-check.ps1 | 2 +- agent99-control-plane.ps1 | 30 +- agent99-synthetic-tests.ps1 | 11 +- ...nt99_transport_recovery_deploy_contract.py | 20 +- ...99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md | 30 +- .../host112-guest-readiness.sh | 447 ++++++++++++++++-- .../install-host112-guest-recovery.sh | 2 +- .../reboot-auto-recovery-slo-scorecard.py | 2 +- .../test_host112_manager_recovery_contract.py | 81 +++- .../test_host112_recovery_runtime_harness.py | 419 ++++++++++++++++ ...install_host112_guest_recovery_contract.py | 1 + .../test_reboot_p0_operational_contract.py | 2 +- 12 files changed, 978 insertions(+), 69 deletions(-) create mode 100644 scripts/reboot-recovery/tests/test_host112_recovery_runtime_harness.py diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index 80e03acf9..d6dce1169 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -87,7 +87,7 @@ Add-Check "recovery:durable_boot_event" ($control.Contains("pendingRecovery") -a Add-Check "recovery:terminal_evidence_budget" ($taskRegistration.Contains('$recoverySettings') -and $taskRegistration.Contains('New-TimeSpan -Minutes 20')) "startup recovery outlives the ten-minute SLO long enough to write terminal evidence" Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present" Add-Check "recovery:full_sop_coordinator" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_scorecard") -and $control.Contains("SCORECARD_FRESH_REBOOT_WINDOW") -and $control.Contains('scope = $recoveryScope')) "Agent99 cannot close reboot recovery without the 110 full-SOP scorecard" -Add-Check "recovery:host112_immutable_apply_closure" ($control.Contains("managerApplyVerified") -and $control.Contains("durableReceiptVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply closure binds transport, immutable receipt and independent verifier" +Add-Check "recovery:host112_immutable_apply_closure" ($control.Contains("managerApplyVerified") -and $control.Contains("durableReceiptVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply closure binds transport, attributed start, immutable receipt/readback and independent verifier" Add-Check "recovery:host112_direct_transport" ($control.Contains("Invoke-AgentHost112SshText") -and $control.Contains('expectedUser = "kali"') -and $control.Contains('route = "direct_host112_kali"') -and $deployer.Contains('name = "host112_canonical_direct_route"') -and $bootstrap.Contains("Ensure-AgentHost112CanonicalSshConfig")) "Host112 recovery uses the canonical 99 direct key route" Add-Check "recovery:host112_timeout_reserve" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120")) "Host112 timeout chain has executor, queue and client reserves" Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index f480c3dbb..150a5503c 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -5626,6 +5626,11 @@ function Convert-AgentHost112GuestReadback { durableReceiptIdentityMatch = (& $asBool "durable_receipt_identity_match") durableReceiptBootMatch = (& $asBool "durable_receipt_boot_match") durableReceiptTerminal = & $get "durable_receipt_terminal" "unknown" + durableReadbackPresent = (& $asBool "durable_readback_present") + durableReadbackIdentityMatch = (& $asBool "durable_readback_identity_match") + durableReadbackBootMatch = (& $asBool "durable_readback_boot_match") + durableReadbackReceiptLinkMatch = (& $asBool "durable_readback_receipt_link_match") + durableReadbackTerminalMatch = (& $asBool "durable_readback_terminal_match") durableReadbackStatus = & $get "durable_readback_status" "unknown" durableReadbackPath = & $get "durable_readback_path" "none" durableReadbackSha256 = & $get "durable_readback_sha256" "none" @@ -5761,7 +5766,13 @@ function Invoke-AgentHost112GuestRecovery { ( $before.durableReceiptIdentityMatch -and $before.durableReceiptBootMatch -and - $before.receiptSha256 -match "^[a-fA-F0-9]{64}$" + $before.receiptSha256 -match "^[a-fA-F0-9]{64}$" -and + $before.durableReadbackPresent -and + $before.durableReadbackIdentityMatch -and + $before.durableReadbackBootMatch -and + $before.durableReadbackReceiptLinkMatch -and + $before.durableReadbackTerminalMatch -and + $before.durableReadbackSha256 -match "^[a-fA-F0-9]{64}$" ) ) $generalCandidateRequired = [bool]($beforeVerified -and -not $before.generalReady) @@ -5864,7 +5875,14 @@ function Invoke-AgentHost112GuestRecovery { $applyReceipt.durableReceiptPresent -and $applyReceipt.durableReceiptIdentityMatch -and $applyReceipt.durableReceiptBootMatch -and - $applyReceipt.durableReceiptTerminal -ne "unknown" + $applyReceipt.durableReceiptTerminal -ne "unknown" -and + $applyReceipt.durableReadbackPresent -and + $applyReceipt.durableReadbackIdentityMatch -and + $applyReceipt.durableReadbackBootMatch -and + $applyReceipt.durableReadbackReceiptLinkMatch -and + $applyReceipt.durableReadbackTerminalMatch -and + $applyReceipt.durableReadbackStatus -eq "written_immutable" -and + $applyReceipt.durableReadbackSha256 -match "^[a-fA-F0-9]{64}$" ) $durableReceiptVerified = [bool]( $applyReceiptCommonVerified -and @@ -5876,6 +5894,13 @@ function Invoke-AgentHost112GuestRecovery { $durableReceiptReadback.durableReceiptPresent -and $durableReceiptReadback.durableReceiptIdentityMatch -and $durableReceiptReadback.durableReceiptBootMatch -and + $durableReceiptReadback.durableReadbackPresent -and + $durableReceiptReadback.durableReadbackIdentityMatch -and + $durableReceiptReadback.durableReadbackBootMatch -and + $durableReceiptReadback.durableReadbackReceiptLinkMatch -and + $durableReceiptReadback.durableReadbackTerminalMatch -and + $durableReceiptReadback.durableReadbackStatus -in @("loaded_immutable_verified", "reused_immutable_verified") -and + $durableReceiptReadback.durableReadbackSha256 -eq $applyReceipt.durableReadbackSha256 -and $durableReceiptReadback.receiptRef -eq $applyReceipt.receiptRef -and $durableReceiptReadback.receiptPath -eq $applyReceipt.receiptPath -and $durableReceiptReadback.receiptSha256 -eq $applyReceipt.receiptSha256 -and @@ -5909,6 +5934,7 @@ function Invoke-AgentHost112GuestRecovery { $applyReceipt.managerIndependentVerifierReady -and $applyReceipt.managerRuntimeWritePerformed -and [string]$applyReceipt.managerAttemptCount -eq "1" -and + [string]$applyReceipt.managerStartExit -eq "0" -and -not $applyReceipt.rollbackAttempted -and $durableReceiptReadback.executorExitCode -eq "0" -and $durableReceiptReadback.durableReceiptTerminal -eq "verified_healthy" -and diff --git a/agent99-synthetic-tests.ps1 b/agent99-synthetic-tests.ps1 index b2d746d0a..ce1b84c53 100644 --- a/agent99-synthetic-tests.ps1 +++ b/agent99-synthetic-tests.ps1 @@ -195,8 +195,9 @@ if ( $receiptRef = if ($DurablePresent -eq 1) { "host112-manager-recovery:fixture" } else { "none" } $receiptPath = if ($DurablePresent -eq 1) { "/var/lib/awoooi-host112-recovery/receipts/fixture.txt" } else { "none" } $receiptSha = if ($DurablePresent -eq 1) { "a" * 64 } else { "none" } + $durableReadbackStatus = if ($DurablePresent -ne 1) { "not_requested" } elseif ($Mode -eq "apply") { "written_immutable" } else { "loaded_immutable_verified" } $preflight = "already_verified_healthy" - "schema_version=host112_guest_recovery_v2 mode=$Mode trace_id=trace:fixture run_id=run:fixture work_item_id=HOST112-FIXTURE boot_id=boot-fixture uptime_seconds=999 systemd_state=running default_target=graphical.target graphical_target=active display_manager=active lightdm=active vmware_tools=active xorg=active wazuh_indexer=active wazuh_manager=active wazuh_manager_result=success wazuh_analysisd_process_count=1 wazuh_dashboard=active filebeat=active ssh_service=active ssh_enabled=enabled ssh_port_listening=1 ssh_ready=1 console_ready=$ConsoleReady services_ready=1 recovery_timer_ready=1 guest_ready=$guestReady manager_preflight_status=$preflight manager_resource_ready=1 wazuh_agent_event_port_1514_ready=1 wazuh_agent_enrollment_port_1515_tcp=1 wazuh_manager_api_port_55000_tcp=1 manager_independent_verifier_ready=1 manager_attempt_count=0 manager_start_exit=not_attempted executor_deadline_seconds=420 runtime_write_performed=$RuntimeWrite state_write_performed=0 manager_marker_write_performed=0 manager_runtime_write_performed=0 receipt_write_performed=$ReceiptWrite receipt_status=$ReceiptStatus receipt_ref=$receiptRef receipt_path=$receiptPath receipt_sha256=$receiptSha receipt_reused=0 durable_receipt_present=$DurablePresent durable_receipt_identity_match=$DurablePresent durable_receipt_boot_match=$DurablePresent durable_receipt_terminal=$DurableTerminal rollback_attempted=0 rollback_verified=0 rollback_terminal=not_required action_count=1 actions=$Actions executor_exit_code=$ExecutorExit" + "schema_version=host112_guest_recovery_v2 mode=$Mode trace_id=trace:fixture run_id=run:fixture work_item_id=HOST112-FIXTURE boot_id=boot-fixture uptime_seconds=999 systemd_state=running default_target=graphical.target graphical_target=active display_manager=active lightdm=active vmware_tools=active xorg=active wazuh_indexer=active wazuh_manager=active wazuh_manager_result=success wazuh_analysisd_process_count=1 wazuh_dashboard=active filebeat=active ssh_service=active ssh_enabled=enabled ssh_port_listening=1 ssh_ready=1 console_ready=$ConsoleReady services_ready=1 recovery_timer_ready=1 guest_ready=$guestReady manager_preflight_status=$preflight manager_resource_ready=1 wazuh_agent_event_port_1514_ready=1 wazuh_agent_enrollment_port_1515_tcp=1 wazuh_manager_api_port_55000_tcp=1 manager_independent_verifier_ready=1 manager_attempt_count=0 manager_start_exit=not_attempted executor_deadline_seconds=420 runtime_write_performed=$RuntimeWrite state_write_performed=0 manager_marker_write_performed=0 manager_runtime_write_performed=0 receipt_write_performed=$ReceiptWrite receipt_status=$ReceiptStatus receipt_ref=$receiptRef receipt_path=$receiptPath receipt_sha256=$receiptSha receipt_reused=0 durable_receipt_present=$DurablePresent durable_receipt_identity_match=$DurablePresent durable_receipt_boot_match=$DurablePresent durable_receipt_terminal=$DurableTerminal durable_readback_present=$DurablePresent durable_readback_identity_match=$DurablePresent durable_readback_boot_match=$DurablePresent durable_readback_receipt_link_match=$DurablePresent durable_readback_terminal_match=$DurablePresent durable_readback_status=$durableReadbackStatus durable_readback_path=$receiptPath.readback durable_readback_sha256=$receiptSha rollback_attempted=0 rollback_verified=0 rollback_terminal=not_required action_count=1 actions=$Actions executor_exit_code=$ExecutorExit" } function Invoke-AgentHost112SshText { param([string]$Command, [int]$TimeoutSeconds, [int]$Retries = 1) @@ -207,9 +208,9 @@ if ( switch ($script:Host112SyntheticCall) { 1 { New-Host112SyntheticRow "check" 0 0 "none" 0 "not_requested" 0 "unknown" 1 } 2 { New-Host112SyntheticRow "dry_run" 0 0 "none" 0 "not_requested" 0 "unknown" 0 } - 3 { New-Host112SyntheticRow "apply" 1 1 "start_lightdm" 1 "written_immutable" 1 "idempotent_already_verified_healthy" 0 } - 4 { New-Host112SyntheticRow "dry_run" 1 0 "none" 0 "not_requested" 1 "idempotent_already_verified_healthy" 0 } - default { New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "idempotent_already_verified_healthy" 0 } + 3 { New-Host112SyntheticRow "apply" 1 1 "start_lightdm" 1 "written_immutable" 1 "verified_healthy" 0 } + 4 { New-Host112SyntheticRow "dry_run" 1 0 "none" 0 "not_requested" 1 "verified_healthy" 0 } + default { New-Host112SyntheticRow "check" 1 0 "none" 0 "not_requested" 1 "verified_healthy" 0 } } } [pscustomobject]@{ @@ -260,7 +261,7 @@ Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_a Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence" Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("recovered_late")) "boot recovery remains durable through first-run termination and late closure" Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present" -Add-SyntheticCheck "recovery:host112_apply_receipt_closure" ($control.Contains("managerApplyVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains("durableReceiptVerified") -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply cannot be closed by a later healthy observation without its immutable receipt" +Add-SyntheticCheck "recovery:host112_apply_receipt_closure" ($control.Contains("managerApplyVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains("durableReceiptVerified") -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply cannot be closed by a later healthy observation without its attributed start and immutable receipt/readback" Add-SyntheticCheck "recovery:host112_candidate_split" ($control.Contains("generalCandidateRequired") -and $control.Contains("managerCandidateRequired") -and $control.Contains("generalApplyEligible") -and $control.Contains("managerApplyEligible") -and $control.Contains("managerNoAttemptPreserved")) "Host112 general guest repair is independent from manager eligibility" Add-SyntheticCheck "recovery:host112_timeout_contract" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120") -and $control.Contains('$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)')) "Host112 executor, SSH lock, queue and client deadlines have explicit reserves" diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index 2b7da6d83..bef686919 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -126,15 +126,21 @@ def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> function = source[source.index("function Convert-AgentHost112GuestReadback") :] function = function[: function.index("function Test-HostReachability")] - assert '"/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId"' in function - assert '"sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id $effectiveTraceId --run-id $effectiveRunId --work-item-id $effectiveWorkItemId"' in function - assert function.count("Invoke-HostSshText $targetHost $checkCommand") == 2 - assert 'schemaVersion = "agent99_host112_guest_recovery_v2"' in function - assert '$before.managerPreflightStatus -eq "ready"' in function + assert "/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id {0} --run-id {1} --work-item-id {2}" in function + assert "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --dry-run --trace-id {0} --run-id {1} --work-item-id {2}" in function + assert "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id {0} --run-id {1} --work-item-id {2}" in function + assert function.count("Invoke-AgentHost112SshText $checkCommand") == 2 + assert function.count("Invoke-AgentHost112SshText $dryRunCommand") == 2 + assert 'schemaVersion = "agent99_host112_guest_recovery_v3"' in function + assert '$dryRun.managerPreflightStatus -eq "ready"' in function assert '$before.transportOk -and' in function assert '$before.identityMatches -and' in function - assert "Invoke-HostSshText $targetHost $applyCommand 590 1" in function - assert 'receiptStatus -eq "written"' in function + assert "Invoke-AgentHost112SshText $applyCommand $script:Agent99Host112ApplyTransportTimeoutSeconds 1" in function + assert 'receiptStatus -eq "written_immutable"' in function + assert "$applyReceipt.durableReadbackReceiptLinkMatch" in function + assert "$applyReceipt.durableReadbackTerminalMatch" in function + assert "$before.durableReadbackReceiptLinkMatch" in function + assert "$before.durableReadbackTerminalMatch" in function assert 'verifier = "wazuh_manager_service_process_agent_event_1514_enrollment_1515_and_api_55000"' in function for verifier_field in ( "wazuhAnalysisdProcessCount", diff --git a/docs/operations/AGENT99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md b/docs/operations/AGENT99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md index e4b2a9002..757e1b5b8 100644 --- a/docs/operations/AGENT99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md +++ b/docs/operations/AGENT99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md @@ -26,11 +26,14 @@ A manager recovery is verified only when all of the following are true for the same apply: 1. The direct apply transport completed and the executor returned healthy. -2. The apply emitted a new immutable receipt with the matching identity and - boot, `terminal=verified_healthy`, independent manager verifier ready, - `manager_runtime_write_performed=1`, and `manager_attempt_count=1`. -3. A second correlated `sudo --dry-run` reloads the durable receipt and returns - the exact same receipt path, SHA-256, identity, boot, and terminal. +2. The complete guest verifier (console, SSH, recovery timer, Wazuh services, + manager process and ports) passes before the apply emits + `terminal=verified_healthy`. A manager-only success cannot freeze a green + receipt while the rest of the guest is degraded. +3. The apply emits a new immutable receipt and immutable readback with matching + identity, boot, terminal, receipt path, receipt SHA-256, and readback + SHA-256. A second correlated `sudo --dry-run` reloads and verifies that exact + receipt/readback pair; it does not manufacture a current-state substitute. 4. A final ordinary `--check` independently verifies current guest and manager runtime health on the same boot. @@ -38,6 +41,23 @@ The last check is evidence only. A timer or external process that makes the manager healthy after a failed apply cannot replace the apply transport or its immutable receipt and therefore cannot turn that run green. +All mutating `systemctl` calls are conservatively classified as a write attempt +before execution. A timeout or non-zero return therefore still produces a +failure receipt instead of entering the no-write path. TERM/INT/HUP handling +remains active through the complete verifier; the final receipt/readback pair is +committed in one short signal-masked section. + +## Timer artifact bound + +The 60-second systemd timer does not create per-invocation immutable files when +it observes no runtime mutation. Repeated no-write observations update one +atomic `timer-observation.latest` aggregate with a state fingerprint and repeat +count. Timer runs that attempt a runtime mutation use a 128-slot rolling ledger +containing at most 128 receipt files and 128 linked readback files. Each slot +records the exact run identity, receipt hash, terminal and explicit rolling +retention policy. Agent99-triggered correlated applies remain immutable and are +not part of this rolling timer ledger. + ## Timeout chain - Host executor absolute deadline: 420 seconds. diff --git a/scripts/reboot-recovery/host112-guest-readiness.sh b/scripts/reboot-recovery/host112-guest-readiness.sh index 64f8864cf..387adb9ca 100755 --- a/scripts/reboot-recovery/host112-guest-readiness.sh +++ b/scripts/reboot-recovery/host112-guest-readiness.sh @@ -147,14 +147,33 @@ durable_receipt_terminal="none" durable_readback_status="not_requested" durable_readback_path="none" durable_readback_sha256="none" +durable_readback_present=0 +durable_readback_identity_match=0 +durable_readback_boot_match=0 +durable_readback_receipt_link_match=0 +durable_readback_terminal_match=0 latest_readback_pointer_written=0 immutable_replay=0 manager_attempt_count=0 manager_start_exit=not_attempted manager_terminal="observed" +run_terminal="observed" rollback_attempted=0 rollback_verified=0 rollback_terminal=not_required +timer_artifact_policy="not_applicable" +timer_ledger_capacity=128 +timer_ledger_slot="none" +timer_observation_status="not_requested" +timer_observation_path="none" +timer_observation_sha256="none" +timer_observation_repeat_count=0 +timer_observation_write_performed=0 +timer_observation_failed=0 +signal_readback_status="not_requested" +signal_readback_path="none" +signal_readback_sha256="none" +action_failure_observed=0 is_uint() { [[ "$1" =~ ^[0-9]+$ ]] @@ -213,7 +232,7 @@ bounded_timeout() { write_epoch_marker() { local marker="$1" epoch="$2" tmp - tmp="${marker}.tmp.$$" + tmp="${marker}.tmp" if printf '%s\n' "$epoch" >"$tmp" \ && chmod 0644 "$tmp" \ && mv "$tmp" "$marker"; then @@ -255,9 +274,12 @@ start_if_inactive() { if [ "$state" = "active" ] || [ "$state" = "activating" ]; then return 0 fi + # A mutating systemctl invocation may partially change runtime state before + # returning non-zero or timing out. Record the attempt before execution so + # those paths can never be classified as no-write timer observations. + runtime_write_performed=1 if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl start "$unit"; then record_action "$action" - runtime_write_performed=1 else record_action "${action}_failed" fi @@ -269,9 +291,10 @@ enable_if_disabled() { if [ "$state" = "enabled" ] || [ "$state" = "static" ]; then return 0 fi + runtime_write_performed=1 + state_write_performed=1 if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl enable "$unit" >/dev/null; then record_action "$action" - runtime_write_performed=1 else record_action "${action}_failed" fi @@ -469,6 +492,21 @@ receipt_token() { ' "$path" 2>/dev/null } +current_action_text() { + if [ "${#actions[@]}" -gt 0 ]; then + IFS=,; printf '%s' "${actions[*]}" + else + printf 'none' + fi +} + +timer_slot_for_identity() { + local checksum + checksum="$(printf '%s\0%s\0%s\0' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" | cksum | awk '{print $1}')" + is_uint "$checksum" || return 1 + printf '%03d' "$((checksum % timer_ledger_capacity))" +} + load_durable_receipt() { local receipt_name stored_trace stored_run stored_work_item stored_boot receipt_name="$(receipt_name_for_identity)" @@ -498,25 +536,97 @@ load_durable_receipt() { return 0 } +load_durable_readback() { + local receipt_name stored_trace stored_run stored_work_item stored_boot + local stored_receipt_path stored_receipt_sha stored_terminal + receipt_name="$(receipt_name_for_identity)" + durable_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt" + [ -f "$durable_readback_path" ] || { + durable_readback_status="missing_for_durable_receipt" + return 1 + } + durable_readback_present=1 + stored_trace="$(receipt_token "$durable_readback_path" trace_id)" + stored_run="$(receipt_token "$durable_readback_path" run_id)" + stored_work_item="$(receipt_token "$durable_readback_path" work_item_id)" + stored_boot="$(receipt_token "$durable_readback_path" boot_id)" + stored_receipt_path="$(receipt_token "$durable_readback_path" receipt_path)" + stored_receipt_sha="$(receipt_token "$durable_readback_path" receipt_sha256)" + stored_terminal="$(receipt_token "$durable_readback_path" durable_receipt_terminal)" + durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')" + if [ "$stored_trace" = "$TRACE_ID" ] \ + && [ "$stored_run" = "$RUN_ID" ] \ + && [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then + durable_readback_identity_match=1 + fi + if [ "$stored_boot" = "$boot_id" ]; then + durable_readback_boot_match=1 + fi + if [ "$stored_receipt_path" = "$receipt_path" ] \ + && [ "$stored_receipt_sha" = "$receipt_sha256" ]; then + durable_readback_receipt_link_match=1 + fi + if [ "$stored_terminal" = "$durable_receipt_terminal" ]; then + durable_readback_terminal_match=1 + fi + if [ "$durable_readback_identity_match" -ne 1 ] \ + || [ "$durable_readback_boot_match" -ne 1 ] \ + || [ "$durable_readback_receipt_link_match" -ne 1 ] \ + || [ "$durable_readback_terminal_match" -ne 1 ]; then + durable_readback_status="immutable_identity_boot_receipt_or_terminal_mismatch" + return 65 + fi + durable_readback_status="loaded_immutable_verified" + return 0 +} + write_public_receipt() { - local receipt_name tmp generated_at receipt_line load_status - if load_durable_receipt; then - receipt_status="reused_immutable" - receipt_reused=1 - return 0 - else - load_status=$? - if [ "$load_status" -eq 65 ]; then - return 1 + local receipt_name tmp generated_at receipt_line load_status receipt_action_text timer_slot + if [ "$MODE" != "timer_apply" ]; then + if load_durable_receipt; then + receipt_status="reused_immutable" + receipt_reused=1 + return 0 + else + load_status=$? + if [ "$load_status" -eq 65 ]; then + return 1 + fi fi fi - receipt_name="$(receipt_name_for_identity)" - receipt_path="$STATE_DIR/receipts/${receipt_name}.txt" + if [ "$MODE" = "timer_apply" ]; then + timer_slot="$(timer_slot_for_identity)" || { + receipt_status="write_failed" + return 1 + } + timer_ledger_slot="$timer_slot" + receipt_name="timer-slot-${timer_slot}" + receipt_path="$STATE_DIR/timer-ledger/${receipt_name}.receipt.txt" + timer_artifact_policy="bounded_ring_128_for_runtime_mutation" + else + receipt_name="$(receipt_name_for_identity)" + receipt_path="$STATE_DIR/receipts/${receipt_name}.txt" + fi generated_at="$(date --iso-8601=seconds)" - receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v2 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" + receipt_action_text="$(current_action_text)" + receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v2 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed action_count=${#actions[@]} actions=$receipt_action_text rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal manager_terminal=$manager_terminal terminal=$run_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" if install -d -m 0750 "$(dirname "$receipt_path")"; then - tmp="${receipt_path}.tmp.$$" - if printf '%s\n' "$receipt_line" >"$tmp" \ + tmp="${receipt_path}.tmp" + if [ "$MODE" = "timer_apply" ] \ + && printf '%s\n' "$receipt_line" >"$tmp" \ + && chmod 0640 "$tmp" \ + && mv "$tmp" "$receipt_path"; then + receipt_write_performed=1 + receipt_status="written_bounded_timer_slot" + receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')" + receipt_ref="host112-timer-ledger:${receipt_name}" + durable_receipt_present=1 + durable_receipt_identity_match=1 + durable_receipt_boot_match=1 + durable_receipt_terminal="$run_terminal" + return 0 + elif [ "$MODE" != "timer_apply" ] \ + && printf '%s\n' "$receipt_line" >"$tmp" \ && chmod 0640 "$tmp" \ && ln "$tmp" "$receipt_path" 2>/dev/null; then rm -f "$tmp" @@ -527,11 +637,11 @@ write_public_receipt() { durable_receipt_present=1 durable_receipt_identity_match=1 durable_receipt_boot_match=1 - durable_receipt_terminal="$manager_terminal" + durable_receipt_terminal="$run_terminal" return 0 fi rm -f "$tmp" >/dev/null 2>&1 || true - if load_durable_receipt; then + if [ "$MODE" != "timer_apply" ] && load_durable_receipt; then receipt_status="reused_immutable" receipt_reused=1 return 0 @@ -541,6 +651,137 @@ write_public_receipt() { return 1 } +write_signal_readback() { + local receipt_name tmp signal_line signal_action_text timer_slot + if [ "$MODE" = "timer_apply" ]; then + timer_slot="$(timer_slot_for_identity)" || { + signal_readback_status="write_failed" + return 1 + } + timer_ledger_slot="$timer_slot" + receipt_name="timer-slot-${timer_slot}" + signal_readback_path="$STATE_DIR/timer-ledger/${receipt_name}.readback.txt" + else + receipt_name="$(receipt_name_for_identity)" + signal_readback_path="$STATE_DIR/readbacks/${receipt_name}.signal.txt" + fi + signal_action_text="$(current_action_text)" + signal_line="schema_version=host112_guest_recovery_signal_readback_v1 generated_at=$(date --iso-8601=seconds) mode=$MODE trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id manager_terminal=$manager_terminal terminal=$run_terminal manager_active=$manager_active_state manager_result=$manager_result manager_orphan_analysisd=$manager_orphan_analysisd manager_verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed action_count=${#actions[@]} actions=$signal_action_text rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal receipt_path=$receipt_path receipt_sha256=$receipt_sha256 prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" + install -d -m 0750 "$(dirname "$signal_readback_path")" || { + signal_readback_status="write_failed" + return 1 + } + if [ "$MODE" != "timer_apply" ] && [ -f "$signal_readback_path" ]; then + if [ "$(receipt_token "$signal_readback_path" trace_id)" != "$TRACE_ID" ] \ + || [ "$(receipt_token "$signal_readback_path" run_id)" != "$RUN_ID" ] \ + || [ "$(receipt_token "$signal_readback_path" work_item_id)" != "$WORK_ITEM_ID" ] \ + || [ "$(receipt_token "$signal_readback_path" boot_id)" != "$boot_id" ] \ + || [ "$(receipt_token "$signal_readback_path" receipt_path)" != "$receipt_path" ] \ + || [ "$(receipt_token "$signal_readback_path" receipt_sha256)" != "$receipt_sha256" ]; then + signal_readback_status="immutable_identity_boot_or_receipt_mismatch" + return 1 + fi + signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')" + signal_readback_status="reused_immutable" + return 0 + fi + tmp="${signal_readback_path}.tmp" + if [ "$MODE" = "timer_apply" ] \ + && printf '%s\n' "$signal_line" >"$tmp" \ + && chmod 0640 "$tmp" \ + && mv "$tmp" "$signal_readback_path"; then + signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')" + signal_readback_status="written_bounded_timer_slot" + return 0 + elif [ "$MODE" != "timer_apply" ] \ + && printf '%s\n' "$signal_line" >"$tmp" \ + && chmod 0640 "$tmp" \ + && ln "$tmp" "$signal_readback_path" 2>/dev/null; then + rm -f "$tmp" + signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')" + signal_readback_status="written_immutable" + return 0 + fi + rm -f "$tmp" >/dev/null 2>&1 || true + signal_readback_status="write_failed" + return 1 +} + +write_bounded_timer_mutation_readback() { + local timer_slot tmp + timer_slot="$(timer_slot_for_identity)" || { + durable_readback_status="write_failed" + return 1 + } + timer_ledger_slot="$timer_slot" + durable_readback_path="$STATE_DIR/timer-ledger/timer-slot-${timer_slot}.readback.txt" + install -d -m 0750 "$(dirname "$durable_readback_path")" || { + durable_readback_status="write_failed" + return 1 + } + tmp="${durable_readback_path}.tmp" + if printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s timer_ledger_slot=%s timer_ledger_capacity=%s retention=rolling_atomic_overwrite durable_readback_status=written_bounded_timer_slot durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \ + "$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" \ + "$timer_ledger_slot" "$timer_ledger_capacity" "$durable_readback_path" >"$tmp" \ + && chmod 0640 "$tmp" \ + && mv "$tmp" "$durable_readback_path"; then + durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')" + durable_readback_status="written_bounded_timer_slot" + durable_readback_present=1 + durable_readback_identity_match=1 + durable_readback_boot_match=1 + durable_readback_receipt_link_match=1 + durable_readback_terminal_match=1 + return 0 + fi + rm -f "$tmp" >/dev/null 2>&1 || true + durable_readback_status="write_failed" + return 1 +} + +write_bounded_timer_observation() { + local tmp now previous_fingerprint previous_repeat previous_first fingerprint first_at observation_line + timer_observation_path="$STATE_DIR/timer-observation.latest" + now="$(date --iso-8601=seconds)" + fingerprint="$(printf '%s\0%s\0%s\0%s\0%s\0%s\0%s\0%s\0' \ + "$boot_id" "$guest_ready" "$run_terminal" "$systemd_state" \ + "$console_ready" "$services_ready" "$ssh_ready" "$recovery_timer_ready" \ + | sha256sum | awk '{print $1}')" + previous_fingerprint="" + previous_repeat=0 + previous_first="$now" + if [ -r "$timer_observation_path" ]; then + previous_fingerprint="$(receipt_token "$timer_observation_path" fingerprint)" + previous_repeat="$(receipt_token "$timer_observation_path" repeat_count)" + previous_first="$(receipt_token "$timer_observation_path" first_observed_at)" + fi + is_uint "$previous_repeat" || previous_repeat=0 + if [ "$previous_fingerprint" = "$fingerprint" ]; then + timer_observation_repeat_count=$((previous_repeat + 1)) + first_at="${previous_first:-$now}" + else + timer_observation_repeat_count=1 + first_at="$now" + fi + observation_line="schema_version=host112_timer_observation_aggregate_v1 first_observed_at=$first_at last_observed_at=$now repeat_count=$timer_observation_repeat_count fingerprint=$fingerprint boot_id=$boot_id guest_ready=$guest_ready manager_terminal=$manager_terminal terminal=$run_terminal systemd_state=$systemd_state console_ready=$console_ready services_ready=$services_ready ssh_ready=$ssh_ready recovery_timer_ready=$recovery_timer_ready runtime_write_performed=0 immutable_artifact_written=0 aggregation=fixed_single_atomic_pointer" + install -d -m 0750 "$STATE_DIR" || { + timer_observation_status="write_failed" + return 1 + } + tmp="${timer_observation_path}.tmp" + if printf '%s\n' "$observation_line" >"$tmp" \ + && chmod 0640 "$tmp" \ + && mv "$tmp" "$timer_observation_path"; then + timer_observation_sha256="$(sha256sum "$timer_observation_path" | awk '{print $1}')" + timer_observation_status="updated_bounded_aggregate" + timer_observation_write_performed=1 + return 0 + fi + rm -f "$tmp" >/dev/null 2>&1 || true + timer_observation_status="write_failed" + return 1 +} + write_immutable_readback() { local receipt_name tmp stored_trace stored_run stored_work_item stored_boot latest_tmp receipt_name="$(receipt_name_for_identity)" @@ -557,14 +798,16 @@ write_immutable_readback() { durable_readback_status="immutable_identity_or_boot_mismatch" return 1 fi - durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')" - durable_readback_status="reused_immutable" - return 0 + if load_durable_readback; then + durable_readback_status="reused_immutable_verified" + return 0 + fi + return 1 fi install -d -m 0750 "$(dirname "$durable_readback_path")" || return 1 - tmp="${durable_readback_path}.tmp.$$" - if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s\n' \ - "$(date --iso-8601=seconds)" "$readback" "$receipt_path" "$receipt_sha256" >"$tmp" \ + tmp="${durable_readback_path}.tmp" + if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s durable_readback_status=written_immutable durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \ + "$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" "$durable_readback_path" >"$tmp" \ || ! chmod 0640 "$tmp" \ || ! ln "$tmp" "$durable_readback_path" 2>/dev/null; then rm -f "$tmp" >/dev/null 2>&1 || true @@ -574,10 +817,15 @@ write_immutable_readback() { rm -f "$tmp" durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')" durable_readback_status="written_immutable" + durable_readback_present=1 + durable_readback_identity_match=1 + durable_readback_boot_match=1 + durable_readback_receipt_link_match=1 + durable_readback_terminal_match=1 # Preserve the historical convenience readback as an explicitly mutable # pointer. It is never used as execution proof; the immutable path/hash is. - latest_tmp="${STATE_DIR}/last-readback.txt.tmp.$$" + latest_tmp="${STATE_DIR}/last-readback.txt.tmp" if printf 'immutable_path=%s immutable_sha256=%s\n' \ "$durable_readback_path" "$durable_readback_sha256" >"$latest_tmp" \ && chmod 0640 "$latest_tmp" \ @@ -600,8 +848,9 @@ if [ "$MODE" = "timer_apply" ]; then WORK_ITEM_ID="HOST112-WAZUH-MANAGER-RECOVERY" fi -if ! command -v sha256sum >/dev/null 2>&1; then - echo "BLOCKER sha256sum_required" >&2 +if ! command -v sha256sum >/dev/null 2>&1 \ + || ! command -v cksum >/dev/null 2>&1; then + echo "BLOCKER sha256sum_and_cksum_required" >&2 exit 69 fi @@ -619,6 +868,12 @@ else exit 65 fi fi +if [ "$durable_receipt_present" -eq 1 ]; then + if ! load_durable_readback; then + echo "BLOCKER durable_readback_missing_or_mismatched_for_receipt" >&2 + exit 65 + fi +fi # Capture a pre-mutation baseline so a TERM/INT/HUP receipt never relies on # unset state, even when interruption lands during the general guest repair. @@ -639,12 +894,18 @@ handle_executor_signal() { local signal_name="$1" trap - TERM INT HUP manager_terminal="interrupted_signal_${signal_name}" + run_terminal="interrupted_signal_${signal_name}" manager_start_exit="interrupted" if [ "$manager_runtime_write_performed" -eq 1 ]; then rollback_manager_to_inactive || true fi capture_manager_state || true - write_public_receipt || true + if [ "$runtime_write_performed" -eq 1 ]; then + if write_public_receipt && write_signal_readback; then + exit 75 + fi + exit 74 + fi exit 75 } @@ -665,9 +926,10 @@ if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \ current_default="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)" if [ "$current_default" != "graphical.target" ]; then + runtime_write_performed=1 + state_write_performed=1 if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl set-default graphical.target >/dev/null; then record_action "set_default_graphical_target" - runtime_write_performed=1 else record_action "set_default_graphical_target_failed" fi @@ -758,8 +1020,13 @@ elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then manager_start_exit=$? fi if wait_for_manager_verifier; then - record_action "start_wazuh_manager_verified" - manager_terminal="verified_healthy" + if [ "$manager_start_exit" = "0" ]; then + record_action "start_wazuh_manager_verified" + manager_terminal="verified_healthy" + else + record_action "wazuh_manager_verifier_healthy_after_nonzero_start_unattributed" + manager_terminal="manager_start_nonzero_verifier_healthy_unattributed" + fi else record_action "start_wazuh_manager_verifier_failed" rollback_manager_to_inactive @@ -784,10 +1051,8 @@ elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then ;; esac capture_manager_state - write_public_receipt || manager_terminal="${manager_terminal}_receipt_failed" fi -trap - TERM INT HUP systemd_state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-system-running 2>/dev/null || true)" systemd_state="${systemd_state:-unknown}" default_target="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)" @@ -852,27 +1117,111 @@ if [ "$console_ready" -eq 1 ] \ guest_ready=1 fi +if [ "${#actions[@]}" -gt 0 ]; then + for action in "${actions[@]}"; do + case "$action" in + *_failed|*_failed_no_start|*_failed_no_write|*_unattributed|*_missing) action_failure_observed=1 ;; + esac + done +fi + +if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then + if [ "$immutable_replay" -eq 1 ]; then + run_terminal="immutable_receipt_replay" + elif [ "$guest_ready" -eq 1 ] \ + && [ "$action_failure_observed" -eq 0 ] \ + && { [ "$manager_terminal" = "verified_healthy" ] \ + || [ "$manager_terminal" = "idempotent_already_verified_healthy" ]; }; then + if [ "$runtime_write_performed" -eq 1 ]; then + run_terminal="verified_healthy" + else + run_terminal="idempotent_already_verified_healthy" + fi + elif [ "$action_failure_observed" -eq 1 ]; then + run_terminal="execution_failed_or_unattributed_${manager_terminal}" + else + run_terminal="guest_verification_failed_${manager_terminal}" + fi +fi + action_text="none" if [ "${#actions[@]}" -gt 0 ]; then action_text="$(IFS=,; printf '%s' "${actions[*]}")" fi +if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then + # Keep signal handling active through the complete guest verifier above. + # Ignore TERM/INT/HUP only for the short atomic receipt/readback commit + # section so a successful receipt can never be separated from its readback. + trap '' TERM INT HUP + if [ "$immutable_replay" -eq 0 ]; then + if [ "$MODE" = "apply" ] || [ "$runtime_write_performed" -eq 1 ]; then + if [ "$MODE" = "timer_apply" ]; then + timer_artifact_policy="bounded_ring_128_for_runtime_mutation" + fi + if ! write_public_receipt; then + run_terminal="${run_terminal}_receipt_failed" + fi + else + receipt_status="timer_no_runtime_mutation_aggregated" + timer_artifact_policy="bounded_mutable_observation_no_immutable_artifact" + fi + fi + if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then + if ! write_bounded_timer_observation; then + timer_observation_failed=1 + fi + durable_readback_status="skipped_bounded_timer_no_runtime_mutation" + fi +fi + trace_output="${TRACE_ID:-none}" run_output="${RUN_ID:-none}" work_item_output="${WORK_ITEM_ID:-none}" -readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" +readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_terminal=$manager_terminal run_terminal=$run_terminal terminal=$run_terminal action_failure_observed=$action_failure_observed manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal durable_readback_present=$durable_readback_present durable_readback_identity_match=$durable_readback_identity_match durable_readback_boot_match=$durable_readback_boot_match durable_readback_receipt_link_match=$durable_readback_receipt_link_match durable_readback_terminal_match=$durable_readback_terminal_match rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal timer_artifact_policy=$timer_artifact_policy timer_ledger_capacity=$timer_ledger_capacity timer_ledger_slot=$timer_ledger_slot timer_observation_status=$timer_observation_status timer_observation_path=$timer_observation_path timer_observation_sha256=$timer_observation_sha256 timer_observation_repeat_count=$timer_observation_repeat_count timer_observation_write_performed=$timer_observation_write_performed signal_readback_status=$signal_readback_status signal_readback_path=$signal_readback_path signal_readback_sha256=$signal_readback_sha256 action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" + +canonical_readback_base() { + printf '%s\n' "$readback" | awk ' + { + separator = "" + for (i = 1; i <= NF; i++) { + if ($i ~ /^durable_readback_(present|identity_match|boot_match|receipt_link_match|terminal_match)=/) { + continue + } + printf "%s%s", separator, $i + separator = " " + } + printf "\n" + } + ' +} + +emit_readback() { + printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s durable_readback_present=%s durable_readback_identity_match=%s durable_readback_boot_match=%s durable_readback_receipt_link_match=%s durable_readback_terminal_match=%s latest_readback_pointer_written=%s\n' \ + "$(canonical_readback_base)" "$durable_readback_status" "$durable_readback_path" \ + "$durable_readback_sha256" "$durable_readback_present" \ + "$durable_readback_identity_match" "$durable_readback_boot_match" \ + "$durable_readback_receipt_link_match" "$durable_readback_terminal_match" \ + "$latest_readback_pointer_written" +} if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then - if ! write_immutable_readback; then - printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s latest_readback_pointer_written=%s\n' \ - "$readback" "$durable_readback_status" "$durable_readback_path" \ - "$durable_readback_sha256" "$latest_readback_pointer_written" + if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then + if [ "$timer_observation_failed" -eq 1 ]; then + emit_readback + exit 74 + fi + elif [ "$MODE" = "timer_apply" ]; then + if ! write_bounded_timer_mutation_readback; then + emit_readback + exit 74 + fi + elif ! write_immutable_readback; then + emit_readback exit 74 fi fi -printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s latest_readback_pointer_written=%s\n' \ - "$readback" "$durable_readback_status" "$durable_readback_path" \ - "$durable_readback_sha256" "$latest_readback_pointer_written" +emit_readback if [ "$MODE" = "dry_run" ]; then case "$manager_preflight_status" in @@ -881,4 +1230,16 @@ if [ "$MODE" = "dry_run" ]; then esac fi -[ "$guest_ready" -eq 1 ] && [ "$receipt_status" != "write_failed" ] +if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then + successful_terminal="$run_terminal" + if [ "$immutable_replay" -eq 1 ]; then + successful_terminal="$durable_receipt_terminal" + fi + [ "$guest_ready" -eq 1 ] \ + && [ "$receipt_status" != "write_failed" ] \ + && { [ "$successful_terminal" = "verified_healthy" ] \ + || [ "$successful_terminal" = "idempotent_already_verified_healthy" ]; } + exit $? +fi + +[ "$guest_ready" -eq 1 ] diff --git a/scripts/reboot-recovery/install-host112-guest-recovery.sh b/scripts/reboot-recovery/install-host112-guest-recovery.sh index 38a750107..8793dc0e5 100755 --- a/scripts/reboot-recovery/install-host112-guest-recovery.sh +++ b/scripts/reboot-recovery/install-host112-guest-recovery.sh @@ -484,7 +484,7 @@ handle_apply_failure() { static_source_preflight() { local source service_state - for tool in bash cmp flock getent install sha256sum ssh-keygen sudo systemctl systemd-analyze timeout visudo; do + for tool in bash cksum cmp flock getent install sha256sum ssh-keygen sudo systemctl systemd-analyze timeout visudo; do command -v "$tool" >/dev/null 2>&1 || { echo "BLOCKER required_tool_missing=$tool" >&2; return 1; } done for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "$CONTROL_PUBLIC_KEY_PATH" "${SCRIPT_DIR}/install-host112-guest-recovery.sh"; do diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index 189fffb91..ee901d502 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -792,7 +792,7 @@ def source_controls() -> dict[str, bool]: "scripts/reboot-recovery/awoooi-host112-guest-recovery.service" ), "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer", - "TimeoutStartSec=600", + "TimeoutStartSec=480", ), "host_112_guest_recovery_timer_source_present": file_contains( source_file( diff --git a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py index ba3223349..8410443b4 100644 --- a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py +++ b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py @@ -149,6 +149,8 @@ def test_manager_preflight_verifier_receipt_and_rollback_are_complete() -> None: 'timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service' ) == 1 assert "manager_attempt_count=1" in source + assert 'if [ "$manager_start_exit" = "0" ]; then' in source + assert 'manager_terminal="manager_start_nonzero_verifier_healthy_unattributed"' in source assert 'timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service' in source assert 'rollback_terminal="bounded_stop_verified"' in source assert "schema_version=host112_wazuh_manager_recovery_receipt_v2" in source @@ -171,7 +173,7 @@ def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None marker_function = source[ source.index("write_epoch_marker() {") : source.index("unit_active() {") ] - assert 'tmp="${marker}.tmp.$$"' in marker_function + assert 'tmp="${marker}.tmp"' in marker_function assert 'printf \'%s\\n\' "$epoch" >"$tmp"' in marker_function assert 'chmod 0644 "$tmp"' in marker_function assert 'mv "$tmp" "$marker"' in marker_function @@ -251,7 +253,15 @@ def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> No assert "printf '%s\\0%s\\0%s\\0'" in source assert 'boot_id=$boot_id' in source assert 'ln "$tmp" "$receipt_path"' in source - assert 'mv "$tmp" "$receipt_path"' not in source + receipt_writer = source[ + source.index("write_public_receipt() {") : source.index( + "write_signal_readback() {", + ) + ] + assert 'if [ "$MODE" = "timer_apply" ]' in receipt_writer + assert 'mv "$tmp" "$receipt_path"' in receipt_writer + assert 'elif [ "$MODE" != "timer_apply" ]' in receipt_writer + assert 'ln "$tmp" "$receipt_path"' in receipt_writer assert 'receipt_status="reused_immutable"' in source assert 'manager_terminal="immutable_receipt_replay"' in source assert "immutable_receipt_identity_or_boot_mismatch" in source @@ -279,6 +289,17 @@ def test_check_reads_exact_immutable_receipt_and_rejects_boot_mismatch( "boot_id=unknown terminal=verified_healthy\n", encoding="utf-8", ) + readback_dir = tmp_path / "state" / "readbacks" + readback_dir.mkdir(parents=True) + durable_readback = readback_dir / f"{run_id}-{digest}.txt" + receipt_sha = hashlib.sha256(receipt.read_bytes()).hexdigest() + durable_readback.write_text( + "schema_version=host112_guest_recovery_v2 " + f"trace_id={trace_id} run_id={run_id} work_item_id={work_item_id} " + "boot_id=unknown durable_receipt_terminal=verified_healthy " + f"receipt_path={receipt} receipt_sha256={receipt_sha}\n", + encoding="utf-8", + ) original = receipt.read_bytes() env = isolated_env(tmp_path) args = [ @@ -301,6 +322,9 @@ def test_check_reads_exact_immutable_receipt_and_rejects_boot_mismatch( assert "durable_receipt_identity_match=1" in matched_output assert "durable_receipt_boot_match=1" in matched_output assert "durable_receipt_terminal=verified_healthy" in matched_output + assert "durable_readback_present=1" in matched_output + assert "durable_readback_receipt_link_match=1" in matched_output + assert "durable_readback_terminal_match=1" in matched_output assert receipt.read_bytes() == original receipt.write_text( @@ -328,12 +352,56 @@ def test_executor_has_absolute_deadline_bounded_reads_and_signal_receipt() -> No assert "trap 'handle_executor_signal TERM' TERM" in source assert "trap 'handle_executor_signal INT' INT" in source assert "trap 'handle_executor_signal HUP' HUP" in source - assert "write_public_receipt || true" in source + assert "if write_public_receipt && write_signal_readback; then" in source + assert "action_count=${#actions[@]} actions=$receipt_action_text" in source + assert "schema_version=host112_guest_recovery_signal_readback_v1" in source + assert 'ln "$tmp" "$signal_readback_path"' in source + assert "signal_readback_status=\"written_immutable\"" in source assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl show' in source assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -ltn' in source assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" df -Pk' in source +def test_timer_noop_uses_one_bounded_aggregate_instead_of_per_minute_artifacts() -> None: + source = read(READINESS) + function = source[ + source.index("write_bounded_timer_observation() {") : source.index( + 'boot_id="$(cat /proc/sys/kernel/random/boot_id', + ) + ] + + assert 'timer_observation_path="$STATE_DIR/timer-observation.latest"' in function + assert "schema_version=host112_timer_observation_aggregate_v1" in function + assert "aggregation=fixed_single_atomic_pointer" in function + assert 'mv "$tmp" "$timer_observation_path"' in function + assert "repeat_count=$timer_observation_repeat_count" in function + assert 'if [ "$MODE" = "apply" ] || [ "$runtime_write_performed" -eq 1 ]; then' in source + assert 'receipt_status="timer_no_runtime_mutation_aggregated"' in source + assert 'durable_readback_status="skipped_bounded_timer_no_runtime_mutation"' in source + assert "timer_ledger_capacity=128" in source + assert "write_bounded_timer_mutation_readback()" in source + assert "retention=rolling_atomic_overwrite" in source + assert 'timer_artifact_policy="bounded_ring_128_for_runtime_mutation"' in source + assert 'timer_artifact_policy="bounded_mutable_observation_no_immutable_artifact"' in source + + +def test_apply_success_is_bound_to_same_run_durable_terminal() -> None: + source = read(READINESS) + terminal_gate = source[source.rindex('if [ "$MODE" = "apply" ]') :] + + assert 'successful_terminal="$run_terminal"' in terminal_gate + assert 'successful_terminal="$durable_receipt_terminal"' in terminal_gate + assert '[ "$successful_terminal" = "verified_healthy" ]' in terminal_gate + assert '[ "$successful_terminal" = "idempotent_already_verified_healthy" ]' in terminal_gate + guest_verifier = source.index('guest_ready=0\nif [ "$console_ready" -eq 1 ]') + receipt_commit = source.index("if ! write_public_receipt; then", guest_verifier) + assert guest_verifier < receipt_commit + assert 'run_terminal="guest_verification_failed_${manager_terminal}"' in source + assert 'run_terminal="execution_failed_or_unattributed_${manager_terminal}"' in source + assert 'action_failure_observed=1' in source + assert "manager_terminal=$manager_terminal terminal=$run_terminal" in source + + def test_general_guest_repair_includes_recovery_timer_independent_of_manager() -> None: source = read(READINESS) general_start = source.index( @@ -424,10 +492,17 @@ def test_agent99_manager_success_requires_apply_and_immutable_receipt_chain() -> assert "$applyReceipt.managerIndependentVerifierReady" in function assert "$applyReceipt.managerRuntimeWritePerformed" in function assert '[string]$applyReceipt.managerAttemptCount -eq "1"' in function + assert '[string]$applyReceipt.managerStartExit -eq "0"' in function assert "$durableReceiptReadback.receiptPath -eq $applyReceipt.receiptPath" in function assert "$durableReceiptReadback.receiptSha256 -eq $applyReceipt.receiptSha256" in function assert "$durableReceiptReadback.durableReceiptTerminal -eq $applyReceipt.durableReceiptTerminal" in function assert '$durableReceiptReadback.durableReceiptTerminal -eq "verified_healthy"' in function + assert "$applyReceipt.durableReadbackReceiptLinkMatch" in function + assert "$applyReceipt.durableReadbackTerminalMatch" in function + assert "$durableReceiptReadback.durableReadbackReceiptLinkMatch" in function + assert "$durableReceiptReadback.durableReadbackTerminalMatch" in function + assert "$before.durableReadbackReceiptLinkMatch" in function + assert "$before.durableReadbackTerminalMatch" in function assert "$afterManagerVerifierVerified" in function assert 'afterVerifierRole = "independent_second_verifier_only"' in function assert "$after.ready -and" in function diff --git a/scripts/reboot-recovery/tests/test_host112_recovery_runtime_harness.py b/scripts/reboot-recovery/tests/test_host112_recovery_runtime_harness.py new file mode 100644 index 000000000..4754e4d7f --- /dev/null +++ b/scripts/reboot-recovery/tests/test_host112_recovery_runtime_harness.py @@ -0,0 +1,419 @@ +from __future__ import annotations + +import os +import signal +import subprocess +import time +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +READINESS = ROOT / "scripts/reboot-recovery/host112-guest-readiness.sh" + + +def _write_executable(path: Path, body: str) -> None: + path.write_text(body, encoding="utf-8") + path.chmod(0o755) + + +def _runtime_harness( + tmp_path: Path, + *, + ledger_capacity: int = 128, +) -> tuple[Path, dict[str, str], Path]: + source = READINESS.read_text(encoding="utf-8") + root_gate = 'if [ "${EUID:-$(id -u)}" -ne 0 ]; then' + assert source.count(root_gate) == 1 + assert source.count("timer_ledger_capacity=128") == 1 + source = source.replace(root_gate, "if false; then", 1) + source = source.replace( + "timer_ledger_capacity=128", + f"timer_ledger_capacity={ledger_capacity}", + 1, + ) + script = tmp_path / "host112-runtime-harness.sh" + _write_executable(script, source) + + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_state = tmp_path / "fake-host" + fake_state.mkdir() + + _write_executable( + fake_bin / "timeout", + """#!/usr/bin/env bash +set -u +while [ "$#" -gt 0 ]; do + case "$1" in + --signal=*|--kill-after=*) shift ;; + *) break ;; + esac +done +[ "$#" -gt 0 ] || exit 64 +shift +"$@" +""", + ) + _write_executable( + fake_bin / "date", + """#!/usr/bin/env bash +case "${1:-}" in + --iso-8601=seconds) printf '%s\n' '2026-07-14T12:00:00+08:00' ;; + +%s) printf '%s\n' '1000' ;; + *) /bin/date "$@" ;; +esac +""", + ) + _write_executable( + fake_bin / "flock", + """#!/usr/bin/env bash +exit 0 +""", + ) + _write_executable( + fake_bin / "mv", + """#!/usr/bin/env bash +set -u +destination="" +for argument in "$@"; do destination="$argument"; done +if [ -n "${FAKE_RECEIPT_MV_MARKER:-}" ] && [[ "$destination" == *.receipt.txt ]]; then + : >"$FAKE_RECEIPT_MV_MARKER" + sleep "${FAKE_RECEIPT_MV_SLEEP_SECONDS:-0}" +fi +/bin/mv "$@" +""", + ) + _write_executable( + fake_bin / "pgrep", + """#!/usr/bin/env bash +case "$*" in + *wazuh-analysisd*) printf '%s\n' '1'; exit 0 ;; + *Xorg*|*Xwayland*) exit 0 ;; + *) exit 1 ;; +esac +""", + ) + _write_executable( + fake_bin / "ss", + """#!/usr/bin/env bash +case "$*" in + *-ltn*) + for port in 22 1514 1515 55000; do + printf 'LISTEN 0 128 0.0.0.0:%s 0.0.0.0:*\n' "$port" + done + ;; + *-lun*) printf '%s\n' 'UNCONN 0 0 0.0.0.0:1514 0.0.0.0:*' ;; +esac +""", + ) + _write_executable( + fake_bin / "df", + """#!/usr/bin/env bash +printf '%s\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on' +printf '%s\n' '/dev/fake 10000000 1000 9999000 1% /' +""", + ) + _write_executable( + fake_bin / "systemctl", + """#!/usr/bin/env bash +set -u +state="${FAKE_HOST_STATE_DIR:?}" +cmd="${1:-}" +[ "$#" -gt 0 ] && shift +case "$cmd" in + show) + unit="${1:-}"; shift || true + property="" + while [ "$#" -gt 0 ]; do + if [ "$1" = "-p" ]; then property="${2:-}"; shift 2; else shift; fi + done + case "$property" in + LoadState) printf '%s\n' 'loaded' ;; + ActiveState) printf '%s\n' 'active' ;; + SubState) printf '%s\n' 'running' ;; + Result) printf '%s\n' 'success' ;; + *) printf '%s\n' 'unknown' ;; + esac + ;; + is-active) + unit="${1:-}" + case "$unit" in + lightdm.service|display-manager.service) + if [ -f "$state/lightdm_active" ]; then printf '%s\n' 'active'; else printf '%s\n' 'inactive'; fi + ;; + *) printf '%s\n' 'active' ;; + esac + ;; + is-enabled) printf '%s\n' 'enabled' ;; + is-system-running) + if [ -n "${FAKE_FINALIZE_MARKER:-}" ]; then + : >"$FAKE_FINALIZE_MARKER" + sleep "${FAKE_FINALIZE_SLEEP_SECONDS:-0}" + fi + printf '%s\n' 'running' + ;; + get-default) printf '%s\n' 'graphical.target' ;; + set-default) + printf 'set-default:%s\n' "${1:-}" >>"$state/mutations.log" + exit "${FAKE_SET_DEFAULT_RC:-0}" + ;; + start) + unit="${1:-}" + printf 'start:%s\n' "$unit" >>"$state/mutations.log" + if [ "$unit" = "lightdm.service" ]; then + if [ "${FAKE_PARTIAL_LIGHTDM_START:-0}" = "1" ]; then : >"$state/lightdm_active"; fi + rc="${FAKE_LIGHTDM_START_RC:-0}" + if [ "$rc" -eq 0 ]; then : >"$state/lightdm_active"; fi + exit "$rc" + fi + ;; + enable) + printf 'enable:%s\n' "${1:-}" >>"$state/mutations.log" + ;; + reset-failed|stop) ;; + *) exit 1 ;; +esac +""", + ) + + env = os.environ.copy() + env.update( + { + "PATH": f"{fake_bin}:{env['PATH']}", + "FAKE_HOST_STATE_DIR": str(fake_state), + "HOST112_RECOVERY_STATE_DIR": str(tmp_path / "state"), + "HOST112_INDEXER_RETRY_MARKER": str(tmp_path / "indexer.epoch"), + "HOST112_MANAGER_RETRY_MARKER": str(tmp_path / "manager.epoch"), + "HOST112_MANAGER_LOCK_FILE": str(tmp_path / "manager.lock"), + } + ) + return script, env, fake_state + + +def _apply(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "bash", + str(script), + "--apply", + "--trace-id", + "trace:runtime-harness", + "--run-id", + "run:runtime-harness", + "--work-item-id", + "HOST112-RUNTIME-HARNESS", + ], + text=True, + capture_output=True, + env=env, + check=False, + timeout=15, + ) + + +def test_failed_general_run_cannot_be_washed_green_by_external_recovery( + tmp_path: Path, +) -> None: + script, env, fake_state = _runtime_harness(tmp_path) + env["FAKE_LIGHTDM_START_RC"] = "124" + env["FAKE_PARTIAL_LIGHTDM_START"] = "1" + + failed = _apply(script, env) + failed_output = failed.stdout + failed.stderr + assert failed.returncode == 1 + assert "runtime_write_performed=1" in failed_output + assert "receipt_status=written_immutable" in failed_output + assert "durable_readback_status=written_immutable" in failed_output + assert ( + "durable_receipt_terminal=execution_failed_or_unattributed_" + "idempotent_already_verified_healthy" + ) in failed_output + assert "guest_ready=1" in failed_output + assert "action_failure_observed=1" in failed_output + + receipt = next((tmp_path / "state" / "receipts").glob("*.txt")) + readback = next((tmp_path / "state" / "readbacks").glob("*.txt")) + receipt_before = receipt.read_bytes() + readback_before = readback.read_bytes() + + (fake_state / "lightdm_active").touch() + env.pop("FAKE_LIGHTDM_START_RC") + env.pop("FAKE_PARTIAL_LIGHTDM_START") + replay = _apply(script, env) + replay_output = replay.stdout + replay.stderr + assert replay.returncode == 1 + assert "guest_ready=1" in replay_output + assert "receipt_status=reused_immutable" in replay_output + assert "durable_readback_status=reused_immutable_verified" in replay_output + assert "terminal=immutable_receipt_replay" in replay_output + assert receipt.read_bytes() == receipt_before + assert readback.read_bytes() == readback_before + + +def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded( + tmp_path: Path, +) -> None: + noop_root = tmp_path / "noop" + noop_root.mkdir() + script, env, fake_state = _runtime_harness(noop_root, ledger_capacity=4) + (fake_state / "lightdm_active").touch() + + for index in range(5): + run_env = env | {"INVOCATION_ID": f"noop-{index}"} + result = subprocess.run( + ["bash", str(script), "--apply-timer"], + text=True, + capture_output=True, + env=run_env, + check=False, + timeout=15, + ) + assert result.returncode == 0, result.stdout + result.stderr + + state = noop_root / "state" + assert [path.relative_to(state) for path in state.rglob("*") if path.is_file()] == [ + Path("timer-observation.latest") + ] + assert "repeat_count=5" in (state / "timer-observation.latest").read_text( + encoding="utf-8" + ) + + mutation_root = tmp_path / "mutation" + mutation_root.mkdir() + script, env, _ = _runtime_harness(mutation_root, ledger_capacity=4) + env["FAKE_LIGHTDM_START_RC"] = "124" + for index in range(12): + run_env = env | {"INVOCATION_ID": f"mutation-{index}"} + result = subprocess.run( + ["bash", str(script), "--apply-timer"], + text=True, + capture_output=True, + env=run_env, + check=False, + timeout=15, + ) + output = result.stdout + result.stderr + assert result.returncode == 1 + assert "runtime_write_performed=1" in output + assert "receipt_status=written_bounded_timer_slot" in output + assert "durable_readback_status=written_bounded_timer_slot" in output + for field in ( + "durable_readback_present", + "durable_readback_identity_match", + "durable_readback_boot_match", + "durable_readback_receipt_link_match", + "durable_readback_terminal_match", + ): + assert f"{field}=1" in output + assert f"{field}=0" not in output + assert "timer_no_runtime_mutation_aggregated" not in output + + ledger_files = [ + path + for path in (mutation_root / "state" / "timer-ledger").iterdir() + if path.is_file() + ] + assert len(ledger_files) <= 8 + assert not (mutation_root / "state" / "receipts").exists() + assert not (mutation_root / "state" / "readbacks").exists() + for readback in (mutation_root / "state" / "timer-ledger").glob( + "*.readback.txt" + ): + stored = readback.read_text(encoding="utf-8") + for field in ( + "durable_readback_present", + "durable_readback_identity_match", + "durable_readback_boot_match", + "durable_readback_receipt_link_match", + "durable_readback_terminal_match", + ): + assert stored.count(f"{field}=1") == 1 + assert f"{field}=0" not in stored + + +def test_timer_sigkill_during_receipt_commit_leaves_only_bounded_slot_temps( + tmp_path: Path, +) -> None: + script, env, _ = _runtime_harness(tmp_path, ledger_capacity=4) + env["FAKE_LIGHTDM_START_RC"] = "124" + + for index in range(12): + marker = tmp_path / f"receipt-mv-{index}.entered" + run_env = env | { + "INVOCATION_ID": f"sigkill-{index}", + "FAKE_RECEIPT_MV_MARKER": str(marker), + "FAKE_RECEIPT_MV_SLEEP_SECONDS": "5", + } + process = subprocess.Popen( + ["bash", str(script), "--apply-timer"], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=run_env, + start_new_session=True, + ) + deadline = time.monotonic() + 8 + while not marker.exists() and time.monotonic() < deadline: + time.sleep(0.01) + assert marker.exists() + os.killpg(process.pid, signal.SIGKILL) + process.communicate(timeout=5) + assert process.returncode == -signal.SIGKILL + + ledger = tmp_path / "state" / "timer-ledger" + files = [path for path in ledger.iterdir() if path.is_file()] + assert len(files) <= 4 + assert all(path.name.endswith(".receipt.txt.tmp") for path in files) + + +def test_signal_during_final_verifier_writes_correlated_failure_chain( + tmp_path: Path, +) -> None: + script, env, _ = _runtime_harness(tmp_path) + marker = tmp_path / "final-verifier.entered" + env.update( + { + "FAKE_FINALIZE_MARKER": str(marker), + "FAKE_FINALIZE_SLEEP_SECONDS": "2", + } + ) + process = subprocess.Popen( + [ + "bash", + str(script), + "--apply", + "--trace-id", + "trace:runtime-harness", + "--run-id", + "run:runtime-harness", + "--work-item-id", + "HOST112-RUNTIME-HARNESS", + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + deadline = time.monotonic() + 8 + while not marker.exists() and time.monotonic() < deadline: + time.sleep(0.02) + assert marker.exists() + os.kill(process.pid, signal.SIGTERM) + stdout, stderr = process.communicate(timeout=12) + assert process.returncode == 75, stdout + stderr + + receipt = next((tmp_path / "state" / "receipts").glob("*.txt")) + signal_readback = next((tmp_path / "state" / "readbacks").glob("*.signal.txt")) + receipt_text = receipt.read_text(encoding="utf-8") + signal_text = signal_readback.read_text(encoding="utf-8") + assert "terminal=interrupted_signal_TERM" in receipt_text + assert "action_count=1 actions=start_lightdm" in receipt_text + assert f"receipt_path={receipt}" in signal_text + assert "terminal=interrupted_signal_TERM" in signal_text + assert "schema_version=host112_guest_recovery_signal_readback_v1" in signal_text + + replay = _apply(script, env | {"FAKE_FINALIZE_MARKER": ""}) + assert replay.returncode == 65 + assert "BLOCKER durable_readback_missing_or_mismatched_for_receipt" in ( + replay.stdout + replay.stderr + ) diff --git a/scripts/reboot-recovery/tests/test_install_host112_guest_recovery_contract.py b/scripts/reboot-recovery/tests/test_install_host112_guest_recovery_contract.py index 2d8903a75..ce0a92738 100644 --- a/scripts/reboot-recovery/tests/test_install_host112_guest_recovery_contract.py +++ b/scripts/reboot-recovery/tests/test_install_host112_guest_recovery_contract.py @@ -108,6 +108,7 @@ def test_static_gate_precedes_timer_activation_and_no_oneshot_is_started() -> No assert 'systemctl enable --now awoooi-host112-guest-recovery.timer' not in text assert 'systemctl start awoooi-host112-guest-recovery.service' not in text assert "grep -Fq 'TimeoutStartSec=480'" in text + assert "for tool in bash cksum cmp flock" in text assert "TimeoutStartSec=600" not in text assert 'INSTALLER_SYNCHRONOUS_RECOVERY_START=0' in text assert 'RECOVERY_EXECUTION_PERFORMED=0' in text diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 585167f62..97d814440 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -108,7 +108,7 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "visudo -cf" in host112_installer assert "--rollback" in host112_installer assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer" in host112_service - assert "TimeoutStartSec=600" in host112_service + assert "TimeoutStartSec=480" in host112_service assert "OnBootSec=20s" in host112_timer assert "OnUnitInactiveSec=60s" in host112_timer