#!/usr/bin/env bash # Host 112 guest readiness and bounded recovery executor. # Version: 2.0 # Last modified: 2026-07-14 11:00 (Asia/Taipei) # Last modified by: Codex # Change: add correlated, cooldown-guarded Wazuh manager recovery and an # independent process/port verifier without reboot, VM power, firewall, or # secret access. set -uo pipefail MODE="check" MODE_EXPLICIT=0 TRACE_ID="" RUN_ID="" WORK_ITEM_ID="" INDEXER_RETRY_AFTER_SECONDS="${HOST112_INDEXER_RETRY_AFTER_SECONDS:-210}" INDEXER_RETRY_COOLDOWN_SECONDS="${HOST112_INDEXER_RETRY_COOLDOWN_SECONDS:-600}" INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-90}" SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-15}" MANAGER_RETRY_AFTER_SECONDS="${HOST112_MANAGER_RETRY_AFTER_SECONDS:-120}" MANAGER_RETRY_COOLDOWN_SECONDS="${HOST112_MANAGER_RETRY_COOLDOWN_SECONDS:-600}" MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}" MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-90}" MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}" MANAGER_VERIFY_INTERVAL_SECONDS="${HOST112_MANAGER_VERIFY_INTERVAL_SECONDS:-3}" MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-30}" READ_TIMEOUT_SECONDS="${HOST112_READ_TIMEOUT_SECONDS:-5}" EXECUTOR_DEADLINE_SECONDS="${HOST112_EXECUTOR_DEADLINE_SECONDS:-420}" MANAGER_MIN_MEMORY_AVAILABLE_KB="${HOST112_MANAGER_MIN_MEMORY_AVAILABLE_KB:-524288}" MANAGER_MIN_DISK_AVAILABLE_KB="${HOST112_MANAGER_MIN_DISK_AVAILABLE_KB:-1048576}" MANAGER_MAX_DISK_USED_PERCENT="${HOST112_MANAGER_MAX_DISK_USED_PERCENT:-95}" STATE_DIR="${HOST112_RECOVERY_STATE_DIR:-/var/lib/awoooi-host112-recovery}" INDEXER_RETRY_MARKER="${HOST112_INDEXER_RETRY_MARKER:-/run/awoooi-host112-indexer-retry.epoch}" MANAGER_RETRY_MARKER="${HOST112_MANAGER_RETRY_MARKER:-/run/awoooi-host112-manager-retry.epoch}" MANAGER_LOCK_FILE="${HOST112_MANAGER_LOCK_FILE:-/run/awoooi-host112-manager-recovery.lock}" usage() { cat <<'USAGE' Usage: host112-guest-readiness.sh --check [identity] host112-guest-readiness.sh --dry-run --trace-id ID --run-id ID --work-item-id ID host112-guest-readiness.sh --apply --trace-id ID --run-id ID --work-item-id ID Internal systemd mode: host112-guest-readiness.sh --apply-timer --check emits a bounded no-secret guest readiness row and never writes. --dry-run evaluates the exact Wazuh manager preflight and never writes. --apply performs only allowlisted idempotent recovery for graphical.target, LightDM, open-vm-tools, SSH, Wazuh Indexer, and one cooldown-guarded Wazuh manager reset/start attempt. The independent verifier requires the manager process plus agent event 1514, enrollment 1515, and manager API 55000 listeners. USAGE } set_mode() { local requested="$1" if [ "$MODE_EXPLICIT" -eq 1 ]; then echo "BLOCKER duplicate_mode" >&2 exit 64 fi MODE="$requested" MODE_EXPLICIT=1 } require_value() { local flag="$1" value="${2:-}" if [ -z "$value" ]; then echo "BLOCKER missing_value=$flag" >&2 exit 64 fi } while [ "$#" -gt 0 ]; do case "$1" in --check) set_mode "check" ;; --dry-run) set_mode "dry_run" ;; --apply) set_mode "apply" ;; --apply-timer) set_mode "timer_apply" ;; --trace-id) require_value "$1" "${2:-}" TRACE_ID="$2" shift ;; --run-id) require_value "$1" "${2:-}" RUN_ID="$2" shift ;; --work-item-id) require_value "$1" "${2:-}" WORK_ITEM_ID="$2" shift ;; -h|--help) usage; exit 0 ;; *) echo "BLOCKER unknown_argument=$1" >&2; exit 64 ;; esac shift done safe_identity() { [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$ ]] } identity_count=0 [ -n "$TRACE_ID" ] && identity_count=$((identity_count + 1)) [ -n "$RUN_ID" ] && identity_count=$((identity_count + 1)) [ -n "$WORK_ITEM_ID" ] && identity_count=$((identity_count + 1)) if [ "$identity_count" -ne 0 ] && [ "$identity_count" -ne 3 ]; then echo "BLOCKER incomplete_control_identity" >&2 exit 64 fi if [ "$identity_count" -eq 3 ]; then for identity in "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID"; do if ! safe_identity "$identity"; then echo "BLOCKER unsafe_control_identity" >&2 exit 64 fi done fi if { [ "$MODE" = "dry_run" ] || [ "$MODE" = "apply" ]; } && [ "$identity_count" -ne 3 ]; then echo "BLOCKER control_identity_required" >&2 exit 64 fi if [ "$MODE" = "timer_apply" ] && [ "$identity_count" -ne 0 ]; then echo "BLOCKER timer_identity_is_system_generated" >&2 exit 64 fi actions=() runtime_write_performed=0 state_write_performed=0 manager_runtime_write_performed=0 manager_marker_write_performed=0 receipt_write_performed=0 receipt_status="not_requested" receipt_ref="none" receipt_path="none" receipt_sha256="none" receipt_reused=0 durable_receipt_present=0 durable_receipt_identity_match=0 durable_receipt_boot_match=0 durable_receipt_terminal="none" durable_readback_status="not_requested" durable_readback_path="none" durable_readback_sha256="none" durable_readback_present=0 durable_readback_identity_match=0 durable_readback_boot_match=0 durable_readback_receipt_link_match=0 durable_readback_terminal_match=0 latest_readback_pointer_written=0 immutable_replay=0 manager_attempt_count=0 manager_start_exit=not_attempted manager_terminal="observed" run_terminal="observed" rollback_attempted=0 rollback_verified=0 rollback_terminal=not_required timer_artifact_policy="not_applicable" timer_ledger_capacity=128 timer_ledger_slot="none" timer_observation_status="not_requested" timer_observation_path="none" timer_observation_sha256="none" timer_observation_repeat_count=0 timer_observation_write_performed=0 timer_observation_failed=0 signal_readback_status="not_requested" signal_readback_path="none" signal_readback_sha256="none" action_failure_observed=0 is_uint() { [[ "$1" =~ ^[0-9]+$ ]] } for timing_value in \ "$INDEXER_START_TIMEOUT_SECONDS" \ "$SERVICE_START_TIMEOUT_SECONDS" \ "$MANAGER_RESET_TIMEOUT_SECONDS" \ "$MANAGER_START_TIMEOUT_SECONDS" \ "$MANAGER_VERIFY_TIMEOUT_SECONDS" \ "$MANAGER_VERIFY_INTERVAL_SECONDS" \ "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" \ "$READ_TIMEOUT_SECONDS" \ "$EXECUTOR_DEADLINE_SECONDS"; do if ! is_uint "$timing_value" || [ "$timing_value" -le 0 ]; then echo "BLOCKER invalid_timeout_configuration" >&2 exit 64 fi done if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -gt "$MANAGER_VERIFY_TIMEOUT_SECONDS" ]; then echo "BLOCKER manager_verify_interval_exceeds_timeout" >&2 exit 64 fi APPLY_PHASE_TIMEOUT_BUDGET_SECONDS=$(( (8 * SERVICE_START_TIMEOUT_SECONDS) + INDEXER_START_TIMEOUT_SECONDS + (2 * MANAGER_RESET_TIMEOUT_SECONDS) + MANAGER_START_TIMEOUT_SECONDS + MANAGER_VERIFY_TIMEOUT_SECONDS + MANAGER_ROLLBACK_TIMEOUT_SECONDS )) if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt "$EXECUTOR_DEADLINE_SECONDS" ]; then echo "BLOCKER apply_phase_timeout_budget_exceeds_executor_deadline" >&2 exit 64 fi EXECUTOR_DEADLINE_EPOCH=$(( $(date +%s) + EXECUTOR_DEADLINE_SECONDS )) remaining_executor_seconds() { local now remaining now="$(date +%s)" remaining=$((EXECUTOR_DEADLINE_EPOCH - now)) [ "$remaining" -gt 0 ] || return 1 printf '%s' "$remaining" } bounded_timeout() { local requested="$1" remaining effective shift remaining="$(remaining_executor_seconds)" || return 124 effective="$requested" [ "$effective" -le "$remaining" ] || effective="$remaining" [ "$effective" -gt 0 ] || return 124 timeout --signal=TERM --kill-after=2 "$effective" "$@" } write_epoch_marker() { local marker="$1" epoch="$2" tmp tmp="${marker}.tmp" if printf '%s\n' "$epoch" >"$tmp" \ && chmod 0644 "$tmp" \ && mv "$tmp" "$marker"; then return 0 fi rm -f "$tmp" >/dev/null 2>&1 || true return 1 } unit_active() { local unit="$1" state state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-active "$unit" 2>/dev/null || true)" printf '%s' "${state:-unknown}" } unit_enabled() { local unit="$1" state state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-enabled "$unit" 2>/dev/null || true)" printf '%s' "${state:-unknown}" } unit_property() { local unit="$1" property="$2" state state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl show "$unit" -p "$property" --value 2>/dev/null || true)" printf '%s' "${state:-unknown}" } unit_loaded() { [ "$(unit_property "$1" LoadState)" = "loaded" ] } record_action() { actions+=("$1") } start_if_inactive() { local unit="$1" action="$2" state state="$(unit_active "$unit")" if [ "$state" = "active" ] || [ "$state" = "activating" ]; then return 0 fi # A mutating systemctl invocation may partially change runtime state before # returning non-zero or timing out. Record the attempt before execution so # those paths can never be classified as no-write timer observations. runtime_write_performed=1 if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl start "$unit"; then record_action "$action" else record_action "${action}_failed" fi } enable_if_disabled() { local unit="$1" action="$2" state state="$(unit_enabled "$unit")" if [ "$state" = "enabled" ] || [ "$state" = "static" ]; then return 0 fi runtime_write_performed=1 state_write_performed=1 if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl enable "$unit" >/dev/null; then record_action "$action" else record_action "${action}_failed" fi } tcp_port_listening() { local port="$1" bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -ltn 2>/dev/null \ | awk '{print $4}' | grep -Eq "(^|:)$port$" } udp_port_listening() { local port="$1" bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -lun 2>/dev/null \ | awk '{print $4}' | grep -Eq "(^|:)$port$" } process_count() { local process_name="$1" count count="$(bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -xc "$process_name" 2>/dev/null || true)" printf '%s' "${count:-0}" } memory_available_kb() { local available available="$(awk '/^MemAvailable:/ {print $2; exit}' /proc/meminfo 2>/dev/null || true)" printf '%s' "${available:-0}" } disk_readback() { local target="/var/ossec" line [ -e "$target" ] || target="/" line="$(bounded_timeout "$READ_TIMEOUT_SECONDS" df -Pk "$target" 2>/dev/null \ | awk 'NR==2 {gsub(/%/, "", $5); print $4 " " $5; exit}')" if [ -n "$line" ]; then printf '%s' "$line" else printf '0 100' fi } capture_manager_state() { local disk_values manager_load_state="$(unit_property wazuh-manager.service LoadState)" manager_active_state="$(unit_property wazuh-manager.service ActiveState)" manager_sub_state="$(unit_property wazuh-manager.service SubState)" manager_result="$(unit_property wazuh-manager.service Result)" manager_failed_state_observed=0 if [ "$manager_active_state" = "failed" ] \ || { [ "$manager_result" != "success" ] && [ "$manager_result" != "unknown" ]; }; then manager_failed_state_observed=1 fi analysisd_count="$(process_count wazuh-analysisd)" event_1514_tcp=0 event_1514_udp=0 enrollment_1515_tcp=0 manager_api_55000_tcp=0 tcp_port_listening 1514 && event_1514_tcp=1 udp_port_listening 1514 && event_1514_udp=1 tcp_port_listening 1515 && enrollment_1515_tcp=1 tcp_port_listening 55000 && manager_api_55000_tcp=1 manager_event_port_ready=0 if [ "$event_1514_tcp" -eq 1 ] || [ "$event_1514_udp" -eq 1 ]; then manager_event_port_ready=1 fi manager_orphan_analysisd=0 if [ "$manager_active_state" != "active" ] && is_uint "$analysisd_count" && [ "$analysisd_count" -gt 0 ]; then manager_orphan_analysisd=1 fi manager_verifier_ready=0 if [ "$manager_active_state" = "active" ] \ && is_uint "$analysisd_count" && [ "$analysisd_count" -gt 0 ] \ && [ "$manager_event_port_ready" -eq 1 ] \ && [ "$enrollment_1515_tcp" -eq 1 ] \ && [ "$manager_api_55000_tcp" -eq 1 ]; then manager_verifier_ready=1 fi manager_memory_available_kb="$(memory_available_kb)" disk_values="$(disk_readback)" read -r manager_disk_available_kb manager_disk_used_percent <<<"$disk_values" manager_resource_ready=0 if is_uint "$manager_memory_available_kb" \ && is_uint "$manager_disk_available_kb" \ && is_uint "$manager_disk_used_percent" \ && [ "$manager_memory_available_kb" -ge "$MANAGER_MIN_MEMORY_AVAILABLE_KB" ] \ && [ "$manager_disk_available_kb" -ge "$MANAGER_MIN_DISK_AVAILABLE_KB" ] \ && [ "$manager_disk_used_percent" -lt "$MANAGER_MAX_DISK_USED_PERCENT" ]; then manager_resource_ready=1 fi } manager_cooldown_remaining() { local now_epoch="$1" last_retry_epoch=0 elapsed if [ -r "$MANAGER_RETRY_MARKER" ]; then read -r last_retry_epoch <"$MANAGER_RETRY_MARKER" || last_retry_epoch=0 fi is_uint "$last_retry_epoch" || last_retry_epoch=0 elapsed=$((now_epoch - last_retry_epoch)) if [ "$elapsed" -lt 0 ]; then printf '%s' "$MANAGER_RETRY_COOLDOWN_SECONDS" elif [ "$elapsed" -lt "$MANAGER_RETRY_COOLDOWN_SECONDS" ]; then printf '%s' "$((MANAGER_RETRY_COOLDOWN_SECONDS - elapsed))" else printf '0' fi } evaluate_manager_preflight() { local now_epoch="$1" uptime_for_retry="$2" manager_cooldown_remaining_seconds="$(manager_cooldown_remaining "$now_epoch")" manager_preflight_status="ready" if ! command -v systemctl >/dev/null 2>&1 \ || ! command -v timeout >/dev/null 2>&1 \ || ! command -v ss >/dev/null 2>&1 \ || ! command -v flock >/dev/null 2>&1; then manager_preflight_status="blocked_required_tool_missing" elif [ "$manager_load_state" != "loaded" ]; then manager_preflight_status="blocked_manager_unit_not_loaded" elif [ "$manager_verifier_ready" -eq 1 ]; then manager_preflight_status="already_verified_healthy" elif [ "$manager_active_state" = "active" ]; then manager_preflight_status="blocked_active_verifier_mismatch" elif [ "$manager_orphan_analysisd" -eq 1 ]; then manager_preflight_status="blocked_orphan_analysisd_no_write" elif [ "$manager_resource_ready" -ne 1 ]; then manager_preflight_status="blocked_resource_preflight_no_write" elif [ "$(unit_active wazuh-indexer.service)" != "active" ]; then manager_preflight_status="blocked_indexer_dependency_no_write" elif ! is_uint "$uptime_for_retry" || [ "$uptime_for_retry" -lt "$MANAGER_RETRY_AFTER_SECONDS" ]; then manager_preflight_status="blocked_boot_settle_no_write" elif [ "$manager_cooldown_remaining_seconds" -gt 0 ]; then manager_preflight_status="blocked_retry_cooldown_no_write" fi } wait_for_manager_verifier() { local deadline now remaining deadline=$(( $(date +%s) + MANAGER_VERIFY_TIMEOUT_SECONDS )) while true; do capture_manager_state [ "$manager_verifier_ready" -eq 1 ] && return 0 now="$(date +%s)" [ "$now" -ge "$deadline" ] && return 1 remaining="$((deadline - now))" [ "$remaining" -gt 0 ] || return 1 if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -lt "$remaining" ]; then sleep "$MANAGER_VERIFY_INTERVAL_SECONDS" else sleep "$remaining" fi done } rollback_manager_to_inactive() { rollback_attempted=1 rollback_terminal="bounded_stop_attempted" if bounded_timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service; then record_action "rollback_stop_wazuh_manager" else record_action "rollback_stop_wazuh_manager_failed" fi capture_manager_state if [ "$manager_active_state" != "active" ] \ && is_uint "$analysisd_count" && [ "$analysisd_count" -eq 0 ] \ && [ "$manager_event_port_ready" -eq 0 ] \ && [ "$enrollment_1515_tcp" -eq 0 ] \ && [ "$manager_api_55000_tcp" -eq 0 ]; then rollback_verified=1 rollback_terminal="bounded_stop_verified" else rollback_terminal="bounded_stop_not_verified" fi } receipt_name_for_identity() { local identity_hash safe_run identity_hash="$(printf '%s\0%s\0%s\0' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" | sha256sum | awk '{print $1}')" safe_run="$(printf '%s' "$RUN_ID" | tr -c 'A-Za-z0-9._+-' '_')" safe_run="${safe_run:0:48}" [ -n "$safe_run" ] || safe_run="unknown" printf '%s-%s' "$safe_run" "$identity_hash" } receipt_token() { local path="$1" key="$2" awk -v prefix="${key}=" ' { for (i = 1; i <= NF; i++) { if (index($i, prefix) == 1) { print substr($i, length(prefix) + 1) exit } } } ' "$path" 2>/dev/null } current_action_text() { if [ "${#actions[@]}" -gt 0 ]; then IFS=,; printf '%s' "${actions[*]}" else printf 'none' fi } timer_slot_for_identity() { local checksum checksum="$(printf '%s\0%s\0%s\0' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" | cksum | awk '{print $1}')" is_uint "$checksum" || return 1 printf '%03d' "$((checksum % timer_ledger_capacity))" } load_durable_receipt() { local receipt_name stored_trace stored_run stored_work_item stored_boot receipt_name="$(receipt_name_for_identity)" receipt_path="$STATE_DIR/receipts/${receipt_name}.txt" [ -f "$receipt_path" ] || return 1 durable_receipt_present=1 stored_trace="$(receipt_token "$receipt_path" trace_id)" stored_run="$(receipt_token "$receipt_path" run_id)" stored_work_item="$(receipt_token "$receipt_path" work_item_id)" stored_boot="$(receipt_token "$receipt_path" boot_id)" durable_receipt_terminal="$(receipt_token "$receipt_path" terminal)" receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')" receipt_ref="host112-manager-recovery:${receipt_name}" if [ "$stored_trace" = "$TRACE_ID" ] \ && [ "$stored_run" = "$RUN_ID" ] \ && [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then durable_receipt_identity_match=1 fi if [ "$stored_boot" = "$boot_id" ]; then durable_receipt_boot_match=1 fi if [ "$durable_receipt_identity_match" -ne 1 ] \ || [ "$durable_receipt_boot_match" -ne 1 ]; then receipt_status="immutable_identity_or_boot_mismatch" return 65 fi return 0 } load_durable_readback() { local receipt_name stored_trace stored_run stored_work_item stored_boot local stored_receipt_path stored_receipt_sha stored_terminal receipt_name="$(receipt_name_for_identity)" durable_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt" [ -f "$durable_readback_path" ] || { durable_readback_status="missing_for_durable_receipt" return 1 } durable_readback_present=1 stored_trace="$(receipt_token "$durable_readback_path" trace_id)" stored_run="$(receipt_token "$durable_readback_path" run_id)" stored_work_item="$(receipt_token "$durable_readback_path" work_item_id)" stored_boot="$(receipt_token "$durable_readback_path" boot_id)" stored_receipt_path="$(receipt_token "$durable_readback_path" receipt_path)" stored_receipt_sha="$(receipt_token "$durable_readback_path" receipt_sha256)" stored_terminal="$(receipt_token "$durable_readback_path" durable_receipt_terminal)" durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')" if [ "$stored_trace" = "$TRACE_ID" ] \ && [ "$stored_run" = "$RUN_ID" ] \ && [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then durable_readback_identity_match=1 fi if [ "$stored_boot" = "$boot_id" ]; then durable_readback_boot_match=1 fi if [ "$stored_receipt_path" = "$receipt_path" ] \ && [ "$stored_receipt_sha" = "$receipt_sha256" ]; then durable_readback_receipt_link_match=1 fi if [ "$stored_terminal" = "$durable_receipt_terminal" ]; then durable_readback_terminal_match=1 fi if [ "$durable_readback_identity_match" -ne 1 ] \ || [ "$durable_readback_boot_match" -ne 1 ] \ || [ "$durable_readback_receipt_link_match" -ne 1 ] \ || [ "$durable_readback_terminal_match" -ne 1 ]; then durable_readback_status="immutable_identity_boot_receipt_or_terminal_mismatch" return 65 fi durable_readback_status="loaded_immutable_verified" return 0 } write_public_receipt() { local receipt_name tmp generated_at receipt_line load_status receipt_action_text timer_slot if [ "$MODE" != "timer_apply" ]; then if load_durable_receipt; then receipt_status="reused_immutable" receipt_reused=1 return 0 else load_status=$? if [ "$load_status" -eq 65 ]; then return 1 fi fi fi if [ "$MODE" = "timer_apply" ]; then timer_slot="$(timer_slot_for_identity)" || { receipt_status="write_failed" return 1 } timer_ledger_slot="$timer_slot" receipt_name="timer-slot-${timer_slot}" receipt_path="$STATE_DIR/timer-ledger/${receipt_name}.receipt.txt" timer_artifact_policy="bounded_ring_128_for_runtime_mutation" else receipt_name="$(receipt_name_for_identity)" receipt_path="$STATE_DIR/receipts/${receipt_name}.txt" fi generated_at="$(date --iso-8601=seconds)" receipt_action_text="$(current_action_text)" receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v2 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed action_count=${#actions[@]} actions=$receipt_action_text rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal manager_terminal=$manager_terminal terminal=$run_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" if install -d -m 0750 "$(dirname "$receipt_path")"; then tmp="${receipt_path}.tmp" if [ "$MODE" = "timer_apply" ] \ && printf '%s\n' "$receipt_line" >"$tmp" \ && chmod 0640 "$tmp" \ && mv "$tmp" "$receipt_path"; then receipt_write_performed=1 receipt_status="written_bounded_timer_slot" receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')" receipt_ref="host112-timer-ledger:${receipt_name}" durable_receipt_present=1 durable_receipt_identity_match=1 durable_receipt_boot_match=1 durable_receipt_terminal="$run_terminal" return 0 elif [ "$MODE" != "timer_apply" ] \ && printf '%s\n' "$receipt_line" >"$tmp" \ && chmod 0640 "$tmp" \ && ln "$tmp" "$receipt_path" 2>/dev/null; then rm -f "$tmp" receipt_write_performed=1 receipt_status="written_immutable" receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')" receipt_ref="host112-manager-recovery:${receipt_name}" durable_receipt_present=1 durable_receipt_identity_match=1 durable_receipt_boot_match=1 durable_receipt_terminal="$run_terminal" return 0 fi rm -f "$tmp" >/dev/null 2>&1 || true if [ "$MODE" != "timer_apply" ] && load_durable_receipt; then receipt_status="reused_immutable" receipt_reused=1 return 0 fi fi receipt_status="write_failed" return 1 } write_signal_readback() { local receipt_name tmp signal_line signal_action_text timer_slot if [ "$MODE" = "timer_apply" ]; then timer_slot="$(timer_slot_for_identity)" || { signal_readback_status="write_failed" return 1 } timer_ledger_slot="$timer_slot" receipt_name="timer-slot-${timer_slot}" signal_readback_path="$STATE_DIR/timer-ledger/${receipt_name}.readback.txt" else receipt_name="$(receipt_name_for_identity)" signal_readback_path="$STATE_DIR/readbacks/${receipt_name}.signal.txt" fi signal_action_text="$(current_action_text)" signal_line="schema_version=host112_guest_recovery_signal_readback_v1 generated_at=$(date --iso-8601=seconds) mode=$MODE trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id manager_terminal=$manager_terminal terminal=$run_terminal manager_active=$manager_active_state manager_result=$manager_result manager_orphan_analysisd=$manager_orphan_analysisd manager_verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed action_count=${#actions[@]} actions=$signal_action_text rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal receipt_path=$receipt_path receipt_sha256=$receipt_sha256 prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" install -d -m 0750 "$(dirname "$signal_readback_path")" || { signal_readback_status="write_failed" return 1 } if [ "$MODE" != "timer_apply" ] && [ -f "$signal_readback_path" ]; then if [ "$(receipt_token "$signal_readback_path" trace_id)" != "$TRACE_ID" ] \ || [ "$(receipt_token "$signal_readback_path" run_id)" != "$RUN_ID" ] \ || [ "$(receipt_token "$signal_readback_path" work_item_id)" != "$WORK_ITEM_ID" ] \ || [ "$(receipt_token "$signal_readback_path" boot_id)" != "$boot_id" ] \ || [ "$(receipt_token "$signal_readback_path" receipt_path)" != "$receipt_path" ] \ || [ "$(receipt_token "$signal_readback_path" receipt_sha256)" != "$receipt_sha256" ]; then signal_readback_status="immutable_identity_boot_or_receipt_mismatch" return 1 fi signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')" signal_readback_status="reused_immutable" return 0 fi tmp="${signal_readback_path}.tmp" if [ "$MODE" = "timer_apply" ] \ && printf '%s\n' "$signal_line" >"$tmp" \ && chmod 0640 "$tmp" \ && mv "$tmp" "$signal_readback_path"; then signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')" signal_readback_status="written_bounded_timer_slot" return 0 elif [ "$MODE" != "timer_apply" ] \ && printf '%s\n' "$signal_line" >"$tmp" \ && chmod 0640 "$tmp" \ && ln "$tmp" "$signal_readback_path" 2>/dev/null; then rm -f "$tmp" signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')" signal_readback_status="written_immutable" return 0 fi rm -f "$tmp" >/dev/null 2>&1 || true signal_readback_status="write_failed" return 1 } write_bounded_timer_mutation_readback() { local timer_slot tmp timer_slot="$(timer_slot_for_identity)" || { durable_readback_status="write_failed" return 1 } timer_ledger_slot="$timer_slot" durable_readback_path="$STATE_DIR/timer-ledger/timer-slot-${timer_slot}.readback.txt" install -d -m 0750 "$(dirname "$durable_readback_path")" || { durable_readback_status="write_failed" return 1 } tmp="${durable_readback_path}.tmp" if printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s timer_ledger_slot=%s timer_ledger_capacity=%s retention=rolling_atomic_overwrite durable_readback_status=written_bounded_timer_slot durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \ "$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" \ "$timer_ledger_slot" "$timer_ledger_capacity" "$durable_readback_path" >"$tmp" \ && chmod 0640 "$tmp" \ && mv "$tmp" "$durable_readback_path"; then durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')" durable_readback_status="written_bounded_timer_slot" durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1 return 0 fi rm -f "$tmp" >/dev/null 2>&1 || true durable_readback_status="write_failed" return 1 } write_bounded_timer_observation() { local tmp now previous_fingerprint previous_repeat previous_first fingerprint first_at observation_line timer_observation_path="$STATE_DIR/timer-observation.latest" now="$(date --iso-8601=seconds)" fingerprint="$(printf '%s\0%s\0%s\0%s\0%s\0%s\0%s\0%s\0' \ "$boot_id" "$guest_ready" "$run_terminal" "$systemd_state" \ "$console_ready" "$services_ready" "$ssh_ready" "$recovery_timer_ready" \ | sha256sum | awk '{print $1}')" previous_fingerprint="" previous_repeat=0 previous_first="$now" if [ -r "$timer_observation_path" ]; then previous_fingerprint="$(receipt_token "$timer_observation_path" fingerprint)" previous_repeat="$(receipt_token "$timer_observation_path" repeat_count)" previous_first="$(receipt_token "$timer_observation_path" first_observed_at)" fi is_uint "$previous_repeat" || previous_repeat=0 if [ "$previous_fingerprint" = "$fingerprint" ]; then timer_observation_repeat_count=$((previous_repeat + 1)) first_at="${previous_first:-$now}" else timer_observation_repeat_count=1 first_at="$now" fi observation_line="schema_version=host112_timer_observation_aggregate_v1 first_observed_at=$first_at last_observed_at=$now repeat_count=$timer_observation_repeat_count fingerprint=$fingerprint boot_id=$boot_id guest_ready=$guest_ready manager_terminal=$manager_terminal terminal=$run_terminal systemd_state=$systemd_state console_ready=$console_ready services_ready=$services_ready ssh_ready=$ssh_ready recovery_timer_ready=$recovery_timer_ready runtime_write_performed=0 immutable_artifact_written=0 aggregation=fixed_single_atomic_pointer" install -d -m 0750 "$STATE_DIR" || { timer_observation_status="write_failed" return 1 } tmp="${timer_observation_path}.tmp" if printf '%s\n' "$observation_line" >"$tmp" \ && chmod 0640 "$tmp" \ && mv "$tmp" "$timer_observation_path"; then timer_observation_sha256="$(sha256sum "$timer_observation_path" | awk '{print $1}')" timer_observation_status="updated_bounded_aggregate" timer_observation_write_performed=1 return 0 fi rm -f "$tmp" >/dev/null 2>&1 || true timer_observation_status="write_failed" return 1 } write_immutable_readback() { local receipt_name tmp stored_trace stored_run stored_work_item stored_boot latest_tmp receipt_name="$(receipt_name_for_identity)" durable_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt" if [ -f "$durable_readback_path" ]; then stored_trace="$(receipt_token "$durable_readback_path" trace_id)" stored_run="$(receipt_token "$durable_readback_path" run_id)" stored_work_item="$(receipt_token "$durable_readback_path" work_item_id)" stored_boot="$(receipt_token "$durable_readback_path" boot_id)" if [ "$stored_trace" != "$TRACE_ID" ] \ || [ "$stored_run" != "$RUN_ID" ] \ || [ "$stored_work_item" != "$WORK_ITEM_ID" ] \ || [ "$stored_boot" != "$boot_id" ]; then durable_readback_status="immutable_identity_or_boot_mismatch" return 1 fi if load_durable_readback; then durable_readback_status="reused_immutable_verified" return 0 fi return 1 fi install -d -m 0750 "$(dirname "$durable_readback_path")" || return 1 tmp="${durable_readback_path}.tmp" if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s durable_readback_status=written_immutable durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \ "$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" "$durable_readback_path" >"$tmp" \ || ! chmod 0640 "$tmp" \ || ! ln "$tmp" "$durable_readback_path" 2>/dev/null; then rm -f "$tmp" >/dev/null 2>&1 || true durable_readback_status="write_failed" return 1 fi rm -f "$tmp" durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')" durable_readback_status="written_immutable" durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1 # Preserve the historical convenience readback as an explicitly mutable # pointer. It is never used as execution proof; the immutable path/hash is. latest_tmp="${STATE_DIR}/last-readback.txt.tmp" if printf 'immutable_path=%s immutable_sha256=%s\n' \ "$durable_readback_path" "$durable_readback_sha256" >"$latest_tmp" \ && chmod 0640 "$latest_tmp" \ && mv "$latest_tmp" "${STATE_DIR}/last-readback.txt"; then latest_readback_pointer_written=1 return 0 fi rm -f "$latest_tmp" >/dev/null 2>&1 || true durable_readback_status="immutable_written_latest_pointer_failed" return 1 } boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)" uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)" if [ "$MODE" = "timer_apply" ]; then invocation_identity="${INVOCATION_ID:-${boot_id}-$(date +%s)}" invocation_identity="$(printf '%s' "$invocation_identity" | tr -c 'A-Za-z0-9._+-' '-')" TRACE_ID="host112-timer:$invocation_identity" RUN_ID="host112-timer:$invocation_identity" WORK_ITEM_ID="HOST112-WAZUH-MANAGER-RECOVERY" fi if ! command -v sha256sum >/dev/null 2>&1 \ || ! command -v cksum >/dev/null 2>&1; then echo "BLOCKER sha256sum_and_cksum_required" >&2 exit 69 fi if load_durable_receipt; then if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then immutable_replay=1 receipt_status="reused_immutable" receipt_reused=1 manager_terminal="immutable_receipt_replay" fi else load_status=$? if [ "$load_status" -eq 65 ]; then echo "BLOCKER immutable_receipt_identity_or_boot_mismatch" >&2 exit 65 fi fi if [ "$durable_receipt_present" -eq 1 ]; then if ! load_durable_readback; then echo "BLOCKER durable_readback_missing_or_mismatched_for_receipt" >&2 exit 65 fi fi # Capture a pre-mutation baseline so a TERM/INT/HUP receipt never relies on # unset state, even when interruption lands during the general guest repair. capture_manager_state before_manager_active="$manager_active_state" before_manager_result="$manager_result" before_manager_orphan="$manager_orphan_analysisd" before_manager_memory="$manager_memory_available_kb" before_manager_disk="$manager_disk_available_kb" before_manager_disk_used="$manager_disk_used_percent" before_event_1514_ready="$manager_event_port_ready" before_enrollment_1515="$enrollment_1515_tcp" before_manager_api_55000="$manager_api_55000_tcp" now_epoch="$(date +%s)" evaluate_manager_preflight "$now_epoch" "$uptime_seconds" handle_executor_signal() { local signal_name="$1" trap - TERM INT HUP manager_terminal="interrupted_signal_${signal_name}" run_terminal="interrupted_signal_${signal_name}" manager_start_exit="interrupted" if [ "$manager_runtime_write_performed" -eq 1 ]; then rollback_manager_to_inactive || true fi capture_manager_state || true if [ "$runtime_write_performed" -eq 1 ]; then if write_public_receipt && write_signal_readback; then exit 75 fi exit 74 fi exit 75 } if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \ && [ "$immutable_replay" -eq 0 ]; then if [ "${EUID:-$(id -u)}" -ne 0 ]; then echo "BLOCKER host112_apply_requires_root" >&2 exit 77 fi exec 9>"$MANAGER_LOCK_FILE" if ! flock -n 9; then echo "BLOCKER host112_manager_single_flight_busy" >&2 exit 75 fi trap 'handle_executor_signal TERM' TERM trap 'handle_executor_signal INT' INT trap 'handle_executor_signal HUP' HUP current_default="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)" if [ "$current_default" != "graphical.target" ]; then runtime_write_performed=1 state_write_performed=1 if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl set-default graphical.target >/dev/null; then record_action "set_default_graphical_target" else record_action "set_default_graphical_target_failed" fi fi start_if_inactive graphical.target start_graphical_target start_if_inactive open-vm-tools.service start_vmware_tools start_if_inactive lightdm.service start_lightdm if unit_loaded ssh.service; then enable_if_disabled ssh.service enable_ssh start_if_inactive ssh.service start_ssh else record_action "ssh_service_missing" fi if unit_loaded awoooi-host112-guest-recovery.timer; then enable_if_disabled awoooi-host112-guest-recovery.timer enable_recovery_timer start_if_inactive awoooi-host112-guest-recovery.timer start_recovery_timer else record_action "recovery_timer_missing" fi indexer_state="$(unit_active wazuh-indexer.service)" if unit_loaded wazuh-indexer.service \ && [ "$indexer_state" != "active" ] \ && [ "$indexer_state" != "activating" ] \ && is_uint "$uptime_seconds" \ && [ "$uptime_seconds" -ge "$INDEXER_RETRY_AFTER_SECONDS" ]; then now_epoch="$(date +%s)" last_retry_epoch=0 if [ -r "$INDEXER_RETRY_MARKER" ]; then read -r last_retry_epoch <"$INDEXER_RETRY_MARKER" || last_retry_epoch=0 fi is_uint "$last_retry_epoch" || last_retry_epoch=0 if [ $((now_epoch - last_retry_epoch)) -ge "$INDEXER_RETRY_COOLDOWN_SECONDS" ]; then if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then runtime_write_performed=1 state_write_performed=1 bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" \ systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true if bounded_timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then record_action "retry_wazuh_indexer" else record_action "retry_wazuh_indexer_failed" fi else record_action "wazuh_indexer_marker_write_failed_no_write" fi else record_action "wazuh_indexer_retry_cooldown" fi fi fi capture_manager_state now_epoch="$(date +%s)" evaluate_manager_preflight "$now_epoch" "$uptime_seconds" if [ "$MODE" = "dry_run" ]; then case "$manager_preflight_status" in ready) manager_terminal="dry_run_ready" ;; already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;; *) manager_terminal="dry_run_${manager_preflight_status}" ;; esac elif { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \ && [ "$immutable_replay" -eq 1 ]; then manager_terminal="immutable_receipt_replay" elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then case "$manager_preflight_status" in already_verified_healthy) manager_terminal="idempotent_already_verified_healthy" ;; ready) if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then # The cooldown marker is a durable runtime state mutation even if the # following systemctl operation fails. Account for it immediately so # no failure path can emit a false no-write receipt. runtime_write_performed=1 state_write_performed=1 manager_marker_write_performed=1 manager_attempt_count=1 if bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then record_action "reset_failed_wazuh_manager" runtime_write_performed=1 manager_runtime_write_performed=1 if bounded_timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then manager_start_exit=0 else manager_start_exit=$? fi if wait_for_manager_verifier; then if [ "$manager_start_exit" = "0" ]; then record_action "start_wazuh_manager_verified" manager_terminal="verified_healthy" else record_action "wazuh_manager_verifier_healthy_after_nonzero_start_unattributed" manager_terminal="manager_start_nonzero_verifier_healthy_unattributed" fi else record_action "start_wazuh_manager_verifier_failed" rollback_manager_to_inactive if [ "$rollback_verified" -eq 1 ]; then manager_terminal="rolled_back_verifier_failed" else manager_terminal="rollback_failed_degraded" fi fi else manager_start_exit="reset_failed_no_start" record_action "reset_failed_wazuh_manager_failed_no_start" manager_terminal="no_start_reset_failed" fi else record_action "wazuh_manager_marker_write_failed_no_write" manager_terminal="no_write_cooldown_marker_write_failed" fi ;; *) manager_terminal="no_write_${manager_preflight_status}" ;; esac capture_manager_state fi systemd_state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-system-running 2>/dev/null || true)" systemd_state="${systemd_state:-unknown}" default_target="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)" default_target="${default_target:-unknown}" graphical_target="$(unit_active graphical.target)" display_manager="$(unit_active display-manager.service)" lightdm="$(unit_active lightdm.service)" vmware_tools="$(unit_active open-vm-tools.service)" wazuh_indexer="$(unit_active wazuh-indexer.service)" wazuh_manager="$(unit_active wazuh-manager.service)" wazuh_dashboard="$(unit_active wazuh-dashboard.service)" filebeat="$(unit_active filebeat.service)" ssh_service="$(unit_active ssh.service)" ssh_enabled="$(unit_enabled ssh.service)" startup_enabled="$(unit_enabled awoooi-host112-guest-recovery.timer)" startup_active="$(unit_active awoooi-host112-guest-recovery.timer)" ssh_port_listening=0 tcp_port_listening 22 && ssh_port_listening=1 ssh_ready=0 if [ "$ssh_service" = "active" ] \ && { [ "$ssh_enabled" = "enabled" ] || [ "$ssh_enabled" = "static" ]; } \ && [ "$ssh_port_listening" -eq 1 ]; then ssh_ready=1 fi xorg="inactive" if bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -x Xorg >/dev/null 2>&1 \ || bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -x Xwayland >/dev/null 2>&1; then xorg="active" fi console_ready=0 if [ "$default_target" = "graphical.target" ] \ && [ "$graphical_target" = "active" ] \ && [ "$display_manager" = "active" ] \ && [ "$lightdm" = "active" ] \ && [ "$vmware_tools" = "active" ] \ && [ "$xorg" = "active" ]; then console_ready=1 fi services_ready=0 if [ "$wazuh_indexer" = "active" ] \ && [ "$wazuh_dashboard" = "active" ] \ && [ "$filebeat" = "active" ] \ && [ "$manager_verifier_ready" -eq 1 ]; then services_ready=1 fi recovery_timer_ready=0 if [ "$startup_enabled" = "enabled" ] && [ "$startup_active" = "active" ]; then recovery_timer_ready=1 fi guest_ready=0 if [ "$console_ready" -eq 1 ] \ && [ "$services_ready" -eq 1 ] \ && [ "$ssh_ready" -eq 1 ] \ && [ "$recovery_timer_ready" -eq 1 ] \ && [ "$systemd_state" = "running" ]; then guest_ready=1 fi if [ "${#actions[@]}" -gt 0 ]; then for action in "${actions[@]}"; do case "$action" in *_failed|*_failed_no_start|*_failed_no_write|*_unattributed|*_missing) action_failure_observed=1 ;; esac done fi if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then if [ "$immutable_replay" -eq 1 ]; then run_terminal="immutable_receipt_replay" elif [ "$guest_ready" -eq 1 ] \ && [ "$action_failure_observed" -eq 0 ] \ && { [ "$manager_terminal" = "verified_healthy" ] \ || [ "$manager_terminal" = "idempotent_already_verified_healthy" ]; }; then if [ "$runtime_write_performed" -eq 1 ]; then run_terminal="verified_healthy" else run_terminal="idempotent_already_verified_healthy" fi elif [ "$action_failure_observed" -eq 1 ]; then run_terminal="execution_failed_or_unattributed_${manager_terminal}" else run_terminal="guest_verification_failed_${manager_terminal}" fi fi action_text="none" if [ "${#actions[@]}" -gt 0 ]; then action_text="$(IFS=,; printf '%s' "${actions[*]}")" fi if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then # Keep signal handling active through the complete guest verifier above. # Ignore TERM/INT/HUP only for the short atomic receipt/readback commit # section so a successful receipt can never be separated from its readback. trap '' TERM INT HUP if [ "$immutable_replay" -eq 0 ]; then if [ "$MODE" = "apply" ] || [ "$runtime_write_performed" -eq 1 ]; then if [ "$MODE" = "timer_apply" ]; then timer_artifact_policy="bounded_ring_128_for_runtime_mutation" fi if ! write_public_receipt; then run_terminal="${run_terminal}_receipt_failed" fi else receipt_status="timer_no_runtime_mutation_aggregated" timer_artifact_policy="bounded_mutable_observation_no_immutable_artifact" fi fi if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then if ! write_bounded_timer_observation; then timer_observation_failed=1 fi durable_readback_status="skipped_bounded_timer_no_runtime_mutation" fi fi trace_output="${TRACE_ID:-none}" run_output="${RUN_ID:-none}" work_item_output="${WORK_ITEM_ID:-none}" readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_terminal=$manager_terminal run_terminal=$run_terminal terminal=$run_terminal action_failure_observed=$action_failure_observed manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal durable_readback_present=$durable_readback_present durable_readback_identity_match=$durable_readback_identity_match durable_readback_boot_match=$durable_readback_boot_match durable_readback_receipt_link_match=$durable_readback_receipt_link_match durable_readback_terminal_match=$durable_readback_terminal_match rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal timer_artifact_policy=$timer_artifact_policy timer_ledger_capacity=$timer_ledger_capacity timer_ledger_slot=$timer_ledger_slot timer_observation_status=$timer_observation_status timer_observation_path=$timer_observation_path timer_observation_sha256=$timer_observation_sha256 timer_observation_repeat_count=$timer_observation_repeat_count timer_observation_write_performed=$timer_observation_write_performed signal_readback_status=$signal_readback_status signal_readback_path=$signal_readback_path signal_readback_sha256=$signal_readback_sha256 action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" canonical_readback_base() { printf '%s\n' "$readback" | awk ' { separator = "" for (i = 1; i <= NF; i++) { if ($i ~ /^durable_readback_(present|identity_match|boot_match|receipt_link_match|terminal_match)=/) { continue } printf "%s%s", separator, $i separator = " " } printf "\n" } ' } emit_readback() { printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s durable_readback_present=%s durable_readback_identity_match=%s durable_readback_boot_match=%s durable_readback_receipt_link_match=%s durable_readback_terminal_match=%s latest_readback_pointer_written=%s\n' \ "$(canonical_readback_base)" "$durable_readback_status" "$durable_readback_path" \ "$durable_readback_sha256" "$durable_readback_present" \ "$durable_readback_identity_match" "$durable_readback_boot_match" \ "$durable_readback_receipt_link_match" "$durable_readback_terminal_match" \ "$latest_readback_pointer_written" } if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then if [ "$timer_observation_failed" -eq 1 ]; then emit_readback exit 74 fi elif [ "$MODE" = "timer_apply" ]; then if ! write_bounded_timer_mutation_readback; then emit_readback exit 74 fi elif ! write_immutable_readback; then emit_readback exit 74 fi fi emit_readback if [ "$MODE" = "dry_run" ]; then case "$manager_preflight_status" in ready|already_verified_healthy) exit 0 ;; *) exit 1 ;; esac fi if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then successful_terminal="$run_terminal" if [ "$immutable_replay" -eq 1 ]; then successful_terminal="$durable_receipt_terminal" fi [ "$guest_ready" -eq 1 ] \ && [ "$receipt_status" != "write_failed" ] \ && { [ "$successful_terminal" = "verified_healthy" ] \ || [ "$successful_terminal" = "idempotent_already_verified_healthy" ]; } exit $? fi [ "$guest_ready" -eq 1 ]