diff --git a/scripts/reboot-recovery/awoooi-host112-guest-recovery.service b/scripts/reboot-recovery/awoooi-host112-guest-recovery.service index bc95b3b81..8f5a7fb2a 100644 --- a/scripts/reboot-recovery/awoooi-host112-guest-recovery.service +++ b/scripts/reboot-recovery/awoooi-host112-guest-recovery.service @@ -6,9 +6,10 @@ Wants=network-online.target [Service] Type=oneshot ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer -# Script phase defaults sum to 500 seconds; the unit owns the 600-second -# outer contract while every child command remains independently bounded. -TimeoutStartSec=600 +# Script mutation defaults sum to 370 seconds under a 420-second absolute +# deadline. The unit leaves a further 60 seconds for signal-safe rollback and +# immutable receipt finalization before systemd may terminate the executor. +TimeoutStartSec=480 User=root Group=root NoNewPrivileges=true diff --git a/scripts/reboot-recovery/host112-guest-readiness.sh b/scripts/reboot-recovery/host112-guest-readiness.sh index 682bead84..64f8864cf 100755 --- a/scripts/reboot-recovery/host112-guest-readiness.sh +++ b/scripts/reboot-recovery/host112-guest-readiness.sh @@ -17,15 +17,17 @@ WORK_ITEM_ID="" INDEXER_RETRY_AFTER_SECONDS="${HOST112_INDEXER_RETRY_AFTER_SECONDS:-210}" INDEXER_RETRY_COOLDOWN_SECONDS="${HOST112_INDEXER_RETRY_COOLDOWN_SECONDS:-600}" -INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-180}" -SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-30}" +INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-90}" +SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-15}" MANAGER_RETRY_AFTER_SECONDS="${HOST112_MANAGER_RETRY_AFTER_SECONDS:-120}" MANAGER_RETRY_COOLDOWN_SECONDS="${HOST112_MANAGER_RETRY_COOLDOWN_SECONDS:-600}" MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}" -MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-120}" +MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-90}" MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}" MANAGER_VERIFY_INTERVAL_SECONDS="${HOST112_MANAGER_VERIFY_INTERVAL_SECONDS:-3}" -MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-45}" +MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-30}" +READ_TIMEOUT_SECONDS="${HOST112_READ_TIMEOUT_SECONDS:-5}" +EXECUTOR_DEADLINE_SECONDS="${HOST112_EXECUTOR_DEADLINE_SECONDS:-420}" MANAGER_MIN_MEMORY_AVAILABLE_KB="${HOST112_MANAGER_MIN_MEMORY_AVAILABLE_KB:-524288}" MANAGER_MIN_DISK_AVAILABLE_KB="${HOST112_MANAGER_MIN_DISK_AVAILABLE_KB:-1048576}" MANAGER_MAX_DISK_USED_PERCENT="${HOST112_MANAGER_MAX_DISK_USED_PERCENT:-95}" @@ -129,10 +131,24 @@ fi actions=() runtime_write_performed=0 +state_write_performed=0 manager_runtime_write_performed=0 +manager_marker_write_performed=0 receipt_write_performed=0 receipt_status="not_requested" receipt_ref="none" +receipt_path="none" +receipt_sha256="none" +receipt_reused=0 +durable_receipt_present=0 +durable_receipt_identity_match=0 +durable_receipt_boot_match=0 +durable_receipt_terminal="none" +durable_readback_status="not_requested" +durable_readback_path="none" +durable_readback_sha256="none" +latest_readback_pointer_written=0 +immutable_replay=0 manager_attempt_count=0 manager_start_exit=not_attempted manager_terminal="observed" @@ -151,24 +167,49 @@ for timing_value in \ "$MANAGER_START_TIMEOUT_SECONDS" \ "$MANAGER_VERIFY_TIMEOUT_SECONDS" \ "$MANAGER_VERIFY_INTERVAL_SECONDS" \ - "$MANAGER_ROLLBACK_TIMEOUT_SECONDS"; do + "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" \ + "$READ_TIMEOUT_SECONDS" \ + "$EXECUTOR_DEADLINE_SECONDS"; do if ! is_uint "$timing_value" || [ "$timing_value" -le 0 ]; then echo "BLOCKER invalid_timeout_configuration" >&2 exit 64 fi done +if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -gt "$MANAGER_VERIFY_TIMEOUT_SECONDS" ]; then + echo "BLOCKER manager_verify_interval_exceeds_timeout" >&2 + exit 64 +fi APPLY_PHASE_TIMEOUT_BUDGET_SECONDS=$(( - (4 * SERVICE_START_TIMEOUT_SECONDS) + (8 * SERVICE_START_TIMEOUT_SECONDS) + INDEXER_START_TIMEOUT_SECONDS - + MANAGER_RESET_TIMEOUT_SECONDS + + (2 * MANAGER_RESET_TIMEOUT_SECONDS) + MANAGER_START_TIMEOUT_SECONDS + MANAGER_VERIFY_TIMEOUT_SECONDS + MANAGER_ROLLBACK_TIMEOUT_SECONDS )) -if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt 570 ]; then - echo "BLOCKER apply_phase_timeout_budget_exceeds_570_seconds" >&2 +if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt "$EXECUTOR_DEADLINE_SECONDS" ]; then + echo "BLOCKER apply_phase_timeout_budget_exceeds_executor_deadline" >&2 exit 64 fi +EXECUTOR_DEADLINE_EPOCH=$(( $(date +%s) + EXECUTOR_DEADLINE_SECONDS )) + +remaining_executor_seconds() { + local now remaining + now="$(date +%s)" + remaining=$((EXECUTOR_DEADLINE_EPOCH - now)) + [ "$remaining" -gt 0 ] || return 1 + printf '%s' "$remaining" +} + +bounded_timeout() { + local requested="$1" remaining effective + shift + remaining="$(remaining_executor_seconds)" || return 124 + effective="$requested" + [ "$effective" -le "$remaining" ] || effective="$remaining" + [ "$effective" -gt 0 ] || return 124 + timeout --signal=TERM --kill-after=2 "$effective" "$@" +} write_epoch_marker() { local marker="$1" epoch="$2" tmp @@ -184,19 +225,19 @@ write_epoch_marker() { unit_active() { local unit="$1" state - state="$(systemctl is-active "$unit" 2>/dev/null || true)" + state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-active "$unit" 2>/dev/null || true)" printf '%s' "${state:-unknown}" } unit_enabled() { local unit="$1" state - state="$(systemctl is-enabled "$unit" 2>/dev/null || true)" + state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-enabled "$unit" 2>/dev/null || true)" printf '%s' "${state:-unknown}" } unit_property() { local unit="$1" property="$2" state - state="$(systemctl show "$unit" -p "$property" --value 2>/dev/null || true)" + state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl show "$unit" -p "$property" --value 2>/dev/null || true)" printf '%s' "${state:-unknown}" } @@ -214,7 +255,7 @@ start_if_inactive() { if [ "$state" = "active" ] || [ "$state" = "activating" ]; then return 0 fi - if timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl start "$unit"; then + if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl start "$unit"; then record_action "$action" runtime_write_performed=1 else @@ -228,7 +269,7 @@ enable_if_disabled() { if [ "$state" = "enabled" ] || [ "$state" = "static" ]; then return 0 fi - if systemctl enable "$unit" >/dev/null; then + if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl enable "$unit" >/dev/null; then record_action "$action" runtime_write_performed=1 else @@ -238,17 +279,19 @@ enable_if_disabled() { tcp_port_listening() { local port="$1" - ss -H -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$port$" + bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -ltn 2>/dev/null \ + | awk '{print $4}' | grep -Eq "(^|:)$port$" } udp_port_listening() { local port="$1" - ss -H -lun 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$port$" + bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -lun 2>/dev/null \ + | awk '{print $4}' | grep -Eq "(^|:)$port$" } process_count() { local process_name="$1" count - count="$(pgrep -xc "$process_name" 2>/dev/null || true)" + count="$(bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -xc "$process_name" 2>/dev/null || true)" printf '%s' "${count:-0}" } @@ -261,7 +304,8 @@ memory_available_kb() { disk_readback() { local target="/var/ossec" line [ -e "$target" ] || target="/" - line="$(df -Pk "$target" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $4 " " $5; exit}')" + line="$(bounded_timeout "$READ_TIMEOUT_SECONDS" df -Pk "$target" 2>/dev/null \ + | awk 'NR==2 {gsub(/%/, "", $5); print $4 " " $5; exit}')" if [ -n "$line" ]; then printf '%s' "$line" else @@ -364,21 +408,27 @@ evaluate_manager_preflight() { } wait_for_manager_verifier() { - local deadline now + local deadline now remaining deadline=$(( $(date +%s) + MANAGER_VERIFY_TIMEOUT_SECONDS )) while true; do capture_manager_state [ "$manager_verifier_ready" -eq 1 ] && return 0 now="$(date +%s)" [ "$now" -ge "$deadline" ] && return 1 - sleep "$MANAGER_VERIFY_INTERVAL_SECONDS" + remaining="$((deadline - now))" + [ "$remaining" -gt 0 ] || return 1 + if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -lt "$remaining" ]; then + sleep "$MANAGER_VERIFY_INTERVAL_SECONDS" + else + sleep "$remaining" + fi done } rollback_manager_to_inactive() { rollback_attempted=1 rollback_terminal="bounded_stop_attempted" - if timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service; then + if bounded_timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service; then record_action "rollback_stop_wazuh_manager" else record_action "rollback_stop_wazuh_manager_failed" @@ -396,33 +446,150 @@ rollback_manager_to_inactive() { fi } -write_public_receipt() { - local receipt_name receipt_path tmp generated_at receipt_line - receipt_name="$(printf '%s' "$RUN_ID" | tr -c 'A-Za-z0-9._+-' '_')" - [ -n "$receipt_name" ] || receipt_name="unknown" - if [ "$MODE" = "timer_apply" ]; then - receipt_path="$STATE_DIR/last-timer-manager-recovery-receipt.txt" - else - receipt_path="$STATE_DIR/receipts/${receipt_name}.txt" +receipt_name_for_identity() { + local identity_hash safe_run + identity_hash="$(printf '%s\0%s\0%s\0' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" | sha256sum | awk '{print $1}')" + safe_run="$(printf '%s' "$RUN_ID" | tr -c 'A-Za-z0-9._+-' '_')" + safe_run="${safe_run:0:48}" + [ -n "$safe_run" ] || safe_run="unknown" + printf '%s-%s' "$safe_run" "$identity_hash" +} + +receipt_token() { + local path="$1" key="$2" + awk -v prefix="${key}=" ' + { + for (i = 1; i <= NF; i++) { + if (index($i, prefix) == 1) { + print substr($i, length(prefix) + 1) + exit + } + } + } + ' "$path" 2>/dev/null +} + +load_durable_receipt() { + local receipt_name stored_trace stored_run stored_work_item stored_boot + receipt_name="$(receipt_name_for_identity)" + receipt_path="$STATE_DIR/receipts/${receipt_name}.txt" + [ -f "$receipt_path" ] || return 1 + durable_receipt_present=1 + stored_trace="$(receipt_token "$receipt_path" trace_id)" + stored_run="$(receipt_token "$receipt_path" run_id)" + stored_work_item="$(receipt_token "$receipt_path" work_item_id)" + stored_boot="$(receipt_token "$receipt_path" boot_id)" + durable_receipt_terminal="$(receipt_token "$receipt_path" terminal)" + receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')" + receipt_ref="host112-manager-recovery:${receipt_name}" + if [ "$stored_trace" = "$TRACE_ID" ] \ + && [ "$stored_run" = "$RUN_ID" ] \ + && [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then + durable_receipt_identity_match=1 fi + if [ "$stored_boot" = "$boot_id" ]; then + durable_receipt_boot_match=1 + fi + if [ "$durable_receipt_identity_match" -ne 1 ] \ + || [ "$durable_receipt_boot_match" -ne 1 ]; then + receipt_status="immutable_identity_or_boot_mismatch" + return 65 + fi + return 0 +} + +write_public_receipt() { + local receipt_name tmp generated_at receipt_line load_status + if load_durable_receipt; then + receipt_status="reused_immutable" + receipt_reused=1 + return 0 + else + load_status=$? + if [ "$load_status" -eq 65 ]; then + return 1 + fi + fi + receipt_name="$(receipt_name_for_identity)" + receipt_path="$STATE_DIR/receipts/${receipt_name}.txt" generated_at="$(date --iso-8601=seconds)" - receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v1 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed manager_runtime_write_performed=$manager_runtime_write_performed rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" + receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v2 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" if install -d -m 0750 "$(dirname "$receipt_path")"; then tmp="${receipt_path}.tmp.$$" if printf '%s\n' "$receipt_line" >"$tmp" \ && chmod 0640 "$tmp" \ - && mv "$tmp" "$receipt_path"; then + && ln "$tmp" "$receipt_path" 2>/dev/null; then + rm -f "$tmp" receipt_write_performed=1 - receipt_status="written" - receipt_ref="host112-manager-recovery:$RUN_ID" + receipt_status="written_immutable" + receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')" + receipt_ref="host112-manager-recovery:${receipt_name}" + durable_receipt_present=1 + durable_receipt_identity_match=1 + durable_receipt_boot_match=1 + durable_receipt_terminal="$manager_terminal" return 0 fi rm -f "$tmp" >/dev/null 2>&1 || true + if load_durable_receipt; then + receipt_status="reused_immutable" + receipt_reused=1 + return 0 + fi fi receipt_status="write_failed" return 1 } +write_immutable_readback() { + local receipt_name tmp stored_trace stored_run stored_work_item stored_boot latest_tmp + receipt_name="$(receipt_name_for_identity)" + durable_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt" + if [ -f "$durable_readback_path" ]; then + stored_trace="$(receipt_token "$durable_readback_path" trace_id)" + stored_run="$(receipt_token "$durable_readback_path" run_id)" + stored_work_item="$(receipt_token "$durable_readback_path" work_item_id)" + stored_boot="$(receipt_token "$durable_readback_path" boot_id)" + if [ "$stored_trace" != "$TRACE_ID" ] \ + || [ "$stored_run" != "$RUN_ID" ] \ + || [ "$stored_work_item" != "$WORK_ITEM_ID" ] \ + || [ "$stored_boot" != "$boot_id" ]; then + durable_readback_status="immutable_identity_or_boot_mismatch" + return 1 + fi + durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')" + durable_readback_status="reused_immutable" + return 0 + fi + install -d -m 0750 "$(dirname "$durable_readback_path")" || return 1 + tmp="${durable_readback_path}.tmp.$$" + if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s\n' \ + "$(date --iso-8601=seconds)" "$readback" "$receipt_path" "$receipt_sha256" >"$tmp" \ + || ! chmod 0640 "$tmp" \ + || ! ln "$tmp" "$durable_readback_path" 2>/dev/null; then + rm -f "$tmp" >/dev/null 2>&1 || true + durable_readback_status="write_failed" + return 1 + fi + rm -f "$tmp" + durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')" + durable_readback_status="written_immutable" + + # Preserve the historical convenience readback as an explicitly mutable + # pointer. It is never used as execution proof; the immutable path/hash is. + latest_tmp="${STATE_DIR}/last-readback.txt.tmp.$$" + if printf 'immutable_path=%s immutable_sha256=%s\n' \ + "$durable_readback_path" "$durable_readback_sha256" >"$latest_tmp" \ + && chmod 0640 "$latest_tmp" \ + && mv "$latest_tmp" "${STATE_DIR}/last-readback.txt"; then + latest_readback_pointer_written=1 + return 0 + fi + rm -f "$latest_tmp" >/dev/null 2>&1 || true + durable_readback_status="immutable_written_latest_pointer_failed" + return 1 +} + boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)" uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)" if [ "$MODE" = "timer_apply" ]; then @@ -433,7 +600,56 @@ if [ "$MODE" = "timer_apply" ]; then WORK_ITEM_ID="HOST112-WAZUH-MANAGER-RECOVERY" fi -if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then +if ! command -v sha256sum >/dev/null 2>&1; then + echo "BLOCKER sha256sum_required" >&2 + exit 69 +fi + +if load_durable_receipt; then + if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then + immutable_replay=1 + receipt_status="reused_immutable" + receipt_reused=1 + manager_terminal="immutable_receipt_replay" + fi +else + load_status=$? + if [ "$load_status" -eq 65 ]; then + echo "BLOCKER immutable_receipt_identity_or_boot_mismatch" >&2 + exit 65 + fi +fi + +# Capture a pre-mutation baseline so a TERM/INT/HUP receipt never relies on +# unset state, even when interruption lands during the general guest repair. +capture_manager_state +before_manager_active="$manager_active_state" +before_manager_result="$manager_result" +before_manager_orphan="$manager_orphan_analysisd" +before_manager_memory="$manager_memory_available_kb" +before_manager_disk="$manager_disk_available_kb" +before_manager_disk_used="$manager_disk_used_percent" +before_event_1514_ready="$manager_event_port_ready" +before_enrollment_1515="$enrollment_1515_tcp" +before_manager_api_55000="$manager_api_55000_tcp" +now_epoch="$(date +%s)" +evaluate_manager_preflight "$now_epoch" "$uptime_seconds" + +handle_executor_signal() { + local signal_name="$1" + trap - TERM INT HUP + manager_terminal="interrupted_signal_${signal_name}" + manager_start_exit="interrupted" + if [ "$manager_runtime_write_performed" -eq 1 ]; then + rollback_manager_to_inactive || true + fi + capture_manager_state || true + write_public_receipt || true + exit 75 +} + +if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \ + && [ "$immutable_replay" -eq 0 ]; then if [ "${EUID:-$(id -u)}" -ne 0 ]; then echo "BLOCKER host112_apply_requires_root" >&2 exit 77 @@ -443,10 +659,13 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then echo "BLOCKER host112_manager_single_flight_busy" >&2 exit 75 fi + trap 'handle_executor_signal TERM' TERM + trap 'handle_executor_signal INT' INT + trap 'handle_executor_signal HUP' HUP - current_default="$(systemctl get-default 2>/dev/null || true)" + current_default="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)" if [ "$current_default" != "graphical.target" ]; then - if systemctl set-default graphical.target >/dev/null; then + if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl set-default graphical.target >/dev/null; then record_action "set_default_graphical_target" runtime_write_performed=1 else @@ -463,6 +682,12 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then else record_action "ssh_service_missing" fi + if unit_loaded awoooi-host112-guest-recovery.timer; then + enable_if_disabled awoooi-host112-guest-recovery.timer enable_recovery_timer + start_if_inactive awoooi-host112-guest-recovery.timer start_recovery_timer + else + record_action "recovery_timer_missing" + fi indexer_state="$(unit_active wazuh-indexer.service)" if unit_loaded wazuh-indexer.service \ @@ -478,9 +703,11 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then is_uint "$last_retry_epoch" || last_retry_epoch=0 if [ $((now_epoch - last_retry_epoch)) -ge "$INDEXER_RETRY_COOLDOWN_SECONDS" ]; then if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then - systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true runtime_write_performed=1 - if timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then + state_write_performed=1 + bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" \ + systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true + if bounded_timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then record_action "retry_wazuh_indexer" else record_action "retry_wazuh_indexer_failed" @@ -495,15 +722,6 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then fi capture_manager_state -before_manager_active="$manager_active_state" -before_manager_result="$manager_result" -before_manager_orphan="$manager_orphan_analysisd" -before_manager_memory="$manager_memory_available_kb" -before_manager_disk="$manager_disk_available_kb" -before_manager_disk_used="$manager_disk_used_percent" -before_event_1514_ready="$manager_event_port_ready" -before_enrollment_1515="$enrollment_1515_tcp" -before_manager_api_55000="$manager_api_55000_tcp" now_epoch="$(date +%s)" evaluate_manager_preflight "$now_epoch" "$uptime_seconds" @@ -513,6 +731,9 @@ if [ "$MODE" = "dry_run" ]; then already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;; *) manager_terminal="dry_run_${manager_preflight_status}" ;; esac +elif { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \ + && [ "$immutable_replay" -eq 1 ]; then + manager_terminal="immutable_receipt_replay" elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then case "$manager_preflight_status" in already_verified_healthy) @@ -520,12 +741,18 @@ elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then ;; ready) if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then + # The cooldown marker is a durable runtime state mutation even if the + # following systemctl operation fails. Account for it immediately so + # no failure path can emit a false no-write receipt. + runtime_write_performed=1 + state_write_performed=1 + manager_marker_write_performed=1 manager_attempt_count=1 - if timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then + if bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then record_action "reset_failed_wazuh_manager" runtime_write_performed=1 manager_runtime_write_performed=1 - if timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then + if bounded_timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then manager_start_exit=0 else manager_start_exit=$? @@ -560,9 +787,10 @@ elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then write_public_receipt || manager_terminal="${manager_terminal}_receipt_failed" fi -systemd_state="$(systemctl is-system-running 2>/dev/null || true)" +trap - TERM INT HUP +systemd_state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-system-running 2>/dev/null || true)" systemd_state="${systemd_state:-unknown}" -default_target="$(systemctl get-default 2>/dev/null || true)" +default_target="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)" default_target="${default_target:-unknown}" graphical_target="$(unit_active graphical.target)" display_manager="$(unit_active display-manager.service)" @@ -587,7 +815,8 @@ if [ "$ssh_service" = "active" ] \ fi xorg="inactive" -if pgrep -x Xorg >/dev/null 2>&1 || pgrep -x Xwayland >/dev/null 2>&1; then +if bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -x Xorg >/dev/null 2>&1 \ + || bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -x Xwayland >/dev/null 2>&1; then xorg="active" fi @@ -631,21 +860,19 @@ fi trace_output="${TRACE_ID:-none}" run_output="${RUN_ID:-none}" work_item_output="${WORK_ITEM_ID:-none}" -readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS runtime_write_performed=$runtime_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" -printf '%s\n' "$readback" +readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then - install -d -m 0750 "$STATE_DIR" - tmp_readback="${STATE_DIR}/last-readback.txt.tmp.$$" - if printf 'generated_at=%s %s\n' "$(date --iso-8601=seconds)" "$readback" >"$tmp_readback" \ - && chmod 0640 "$tmp_readback" \ - && mv "$tmp_readback" "${STATE_DIR}/last-readback.txt"; then - : - else - rm -f "$tmp_readback" >/dev/null 2>&1 || true + if ! write_immutable_readback; then + printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s latest_readback_pointer_written=%s\n' \ + "$readback" "$durable_readback_status" "$durable_readback_path" \ + "$durable_readback_sha256" "$latest_readback_pointer_written" exit 74 fi fi +printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s latest_readback_pointer_written=%s\n' \ + "$readback" "$durable_readback_status" "$durable_readback_path" \ + "$durable_readback_sha256" "$latest_readback_pointer_written" if [ "$MODE" = "dry_run" ]; then case "$manager_preflight_status" in diff --git a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py index 2b9b22b98..b44a43b4a 100644 --- a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py +++ b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import os import shutil import subprocess @@ -71,7 +72,9 @@ def test_check_and_dry_run_are_real_no_write_execution_paths(tmp_path: Path) -> assert "manager_runtime_write_performed=0" in output assert "receipt_write_performed=0" in output assert "manager_preflight_status=" in output - assert "apply_phase_timeout_budget_seconds=500" in output + assert "apply_phase_timeout_budget_seconds=370" in output + assert "executor_deadline_seconds=420" in output + assert "durable_receipt_present=0" in output assert not (tmp_path / "state").exists() assert not (tmp_path / "indexer.epoch").exists() assert not (tmp_path / "manager.epoch").exists() @@ -148,7 +151,8 @@ def test_manager_preflight_verifier_receipt_and_rollback_are_complete() -> None: assert "manager_attempt_count=1" in source assert 'timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service' in source assert 'rollback_terminal="bounded_stop_verified"' in source - assert "schema_version=host112_wazuh_manager_recovery_receipt_v1" in source + assert "schema_version=host112_wazuh_manager_recovery_receipt_v2" in source + assert "boot_id=$boot_id" in source assert "trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID" in source assert 'receipt_status="write_failed"' in source assert "prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" in source @@ -183,14 +187,25 @@ def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None ) success = manager_branch[:marker_failure_at] failure = manager_branch[marker_failure_at:] + marker_success_at = success.index( + ' manager_marker_write_performed=1' + ) + reset_at = success.index("systemctl reset-failed wazuh-manager.service") + assert marker_success_at < reset_at + assert "runtime_write_performed=1" in success[:reset_at] + assert "state_write_performed=1" in success[:reset_at] assert "systemctl reset-failed wazuh-manager.service" in success assert "systemctl start wazuh-manager.service" in success - assert 'timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service' in success + assert 'bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service' in success assert "manager_attempt_count=1" in success assert "systemctl reset-failed wazuh-manager.service" not in failure assert "systemctl start wazuh-manager.service" not in failure assert "wazuh_manager_marker_write_failed_no_write" in failure assert 'manager_terminal="no_write_cooldown_marker_write_failed"' in failure + assert "manager_marker_write_performed=1" not in failure + + assert "state_write_performed=$state_write_performed" in source + assert "manager_marker_write_performed=$manager_marker_write_performed" in source indexer_start = source.index( ' if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then' @@ -210,17 +225,129 @@ def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None: service = read(SERVICE) control = read(CONTROL) - assert 'SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-30}"' in source - assert 'INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-180}"' in source - assert 'MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-120}"' in source + assert 'SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-15}"' in source + assert 'INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-90}"' in source + assert 'MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-90}"' in source assert 'MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}"' in source assert 'MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}"' in source - assert 'MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-45}"' in source - assert 'if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt 570 ]; then' in source - assert "TimeoutStartSec=600" in service + assert 'MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-30}"' in source + assert 'EXECUTOR_DEADLINE_SECONDS="${HOST112_EXECUTOR_DEADLINE_SECONDS:-420}"' in source + assert 'if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt "$EXECUTOR_DEADLINE_SECONDS" ]; then' in source + assert 'if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -gt "$MANAGER_VERIFY_TIMEOUT_SECONDS" ]; then' in source + assert "TimeoutStartSec=480" in service assert "Invoke-HostSshText $targetHost $applyCommand 590 1" in control +def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> None: + source = read(READINESS) + + assert "receipt_name_for_identity()" in source + assert "printf '%s\\0%s\\0%s\\0'" in source + assert 'boot_id=$boot_id' in source + assert 'ln "$tmp" "$receipt_path"' in source + assert 'mv "$tmp" "$receipt_path"' not in source + assert 'receipt_status="reused_immutable"' in source + assert 'manager_terminal="immutable_receipt_replay"' in source + assert "immutable_receipt_identity_or_boot_mismatch" in source + assert "durable_receipt_terminal=$durable_receipt_terminal" in source + assert 'ln "$tmp" "$durable_readback_path"' in source + assert "durable_readback_sha256=%s" in source + assert "latest_readback_pointer_written=%s" in source + + +def test_check_reads_exact_immutable_receipt_and_rejects_boot_mismatch( + tmp_path: Path, +) -> None: + trace_id = "trace-D037-receipt" + run_id = "run-D037-receipt" + work_item_id = "D037-WAZUH-HOST112" + digest = hashlib.sha256( + f"{trace_id}\0{run_id}\0{work_item_id}\0".encode() + ).hexdigest() + receipt_dir = tmp_path / "state" / "receipts" + receipt_dir.mkdir(parents=True) + receipt = receipt_dir / f"{run_id}-{digest}.txt" + receipt.write_text( + "schema_version=host112_wazuh_manager_recovery_receipt_v2 " + f"trace_id={trace_id} run_id={run_id} work_item_id={work_item_id} " + "boot_id=unknown terminal=verified_healthy\n", + encoding="utf-8", + ) + original = receipt.read_bytes() + env = isolated_env(tmp_path) + args = [ + "bash", + str(READINESS), + "--check", + "--trace-id", + trace_id, + "--run-id", + run_id, + "--work-item-id", + work_item_id, + ] + + matched = subprocess.run( + args, text=True, capture_output=True, env=env, check=False, timeout=20 + ) + matched_output = matched.stdout + matched.stderr + assert "durable_receipt_present=1" in matched_output + assert "durable_receipt_identity_match=1" in matched_output + assert "durable_receipt_boot_match=1" in matched_output + assert "durable_receipt_terminal=verified_healthy" in matched_output + assert receipt.read_bytes() == original + + receipt.write_text( + receipt.read_text(encoding="utf-8").replace( + "boot_id=unknown", "boot_id=previous-boot" + ), + encoding="utf-8", + ) + mismatched = subprocess.run( + args, text=True, capture_output=True, env=env, check=False, timeout=20 + ) + assert mismatched.returncode == 65 + assert "immutable_receipt_identity_or_boot_mismatch" in ( + mismatched.stdout + mismatched.stderr + ) + + +def test_executor_has_absolute_deadline_bounded_reads_and_signal_receipt() -> None: + source = read(READINESS) + + assert "remaining_executor_seconds()" in source + assert "bounded_timeout()" in source + assert 'timeout --signal=TERM --kill-after=2 "$effective" "$@"' in source + assert "handle_executor_signal()" in source + assert "trap 'handle_executor_signal TERM' TERM" in source + assert "trap 'handle_executor_signal INT' INT" in source + assert "trap 'handle_executor_signal HUP' HUP" in source + assert "write_public_receipt || true" in source + assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl show' in source + assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -ltn' in source + assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" df -Pk' in source + + +def test_general_guest_repair_includes_recovery_timer_independent_of_manager() -> None: + source = read(READINESS) + general_start = source.index( + ' current_default="$(bounded_timeout "$READ_TIMEOUT_SECONDS"' + ) + manager_start = source.index("\nfi\n\ncapture_manager_state", general_start) + general = source[general_start:manager_start] + + assert "unit_loaded awoooi-host112-guest-recovery.timer" in general + assert ( + "enable_if_disabled awoooi-host112-guest-recovery.timer " + "enable_recovery_timer" + ) in general + assert ( + "start_if_inactive awoooi-host112-guest-recovery.timer " + "start_recovery_timer" + ) in general + assert 'record_action "recovery_timer_missing"' in general + + def test_installer_uses_argument_regex_and_post_install_policy_readback() -> None: source = read(INSTALLER)