wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(recovery): make host112 receipts replay safe

Browse Source
This commit is contained in:
ogt
2026-07-14 11:25:18 +08:00
parent 8bee637466
commit 989862e296
3 changed files with 429 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
scripts/reboot-recovery/awoooi-host112-guest-recovery.service
View File

@@ -6,9 +6,10 @@ Wants=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer
# Script phase defaults sum to 500 seconds; the unit owns the 600-second
# outer contract while every child command remains independently bounded.
TimeoutStartSec=600
# Script mutation defaults sum to 370 seconds under a 420-second absolute
# deadline. The unit leaves a further 60 seconds for signal-safe rollback and
# immutable receipt finalization before systemd may terminate the executor.
TimeoutStartSec=480
User=root
Group=root
NoNewPrivileges=true

351
scripts/reboot-recovery/host112-guest-readiness.sh
View File

@@ -17,15 +17,17 @@ WORK_ITEM_ID=""
INDEXER_RETRY_AFTER_SECONDS="${HOST112_INDEXER_RETRY_AFTER_SECONDS:-210}"
INDEXER_RETRY_COOLDOWN_SECONDS="${HOST112_INDEXER_RETRY_COOLDOWN_SECONDS:-600}"
INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-180}"
SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-30}"
INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-90}"
SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-15}"
MANAGER_RETRY_AFTER_SECONDS="${HOST112_MANAGER_RETRY_AFTER_SECONDS:-120}"
MANAGER_RETRY_COOLDOWN_SECONDS="${HOST112_MANAGER_RETRY_COOLDOWN_SECONDS:-600}"
MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}"
MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-120}"
MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-90}"
MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}"
MANAGER_VERIFY_INTERVAL_SECONDS="${HOST112_MANAGER_VERIFY_INTERVAL_SECONDS:-3}"
MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-45}"
MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-30}"
READ_TIMEOUT_SECONDS="${HOST112_READ_TIMEOUT_SECONDS:-5}"
EXECUTOR_DEADLINE_SECONDS="${HOST112_EXECUTOR_DEADLINE_SECONDS:-420}"
MANAGER_MIN_MEMORY_AVAILABLE_KB="${HOST112_MANAGER_MIN_MEMORY_AVAILABLE_KB:-524288}"
MANAGER_MIN_DISK_AVAILABLE_KB="${HOST112_MANAGER_MIN_DISK_AVAILABLE_KB:-1048576}"
MANAGER_MAX_DISK_USED_PERCENT="${HOST112_MANAGER_MAX_DISK_USED_PERCENT:-95}"
@@ -129,10 +131,24 @@ fi
actions=()
runtime_write_performed=0
state_write_performed=0
manager_runtime_write_performed=0
manager_marker_write_performed=0
receipt_write_performed=0
receipt_status="not_requested"
receipt_ref="none"
receipt_path="none"
receipt_sha256="none"
receipt_reused=0
durable_receipt_present=0
durable_receipt_identity_match=0
durable_receipt_boot_match=0
durable_receipt_terminal="none"
durable_readback_status="not_requested"
durable_readback_path="none"
durable_readback_sha256="none"
latest_readback_pointer_written=0
immutable_replay=0
manager_attempt_count=0
manager_start_exit=not_attempted
manager_terminal="observed"
@@ -151,24 +167,49 @@ for timing_value in \
"$MANAGER_START_TIMEOUT_SECONDS" \
"$MANAGER_VERIFY_TIMEOUT_SECONDS" \
"$MANAGER_VERIFY_INTERVAL_SECONDS" \
"$MANAGER_ROLLBACK_TIMEOUT_SECONDS"; do
"$MANAGER_ROLLBACK_TIMEOUT_SECONDS" \
"$READ_TIMEOUT_SECONDS" \
"$EXECUTOR_DEADLINE_SECONDS"; do
if ! is_uint "$timing_value" || [ "$timing_value" -le 0 ]; then
echo "BLOCKER invalid_timeout_configuration" >&2
exit 64
fi
done
if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -gt "$MANAGER_VERIFY_TIMEOUT_SECONDS" ]; then
echo "BLOCKER manager_verify_interval_exceeds_timeout" >&2
exit 64
fi
APPLY_PHASE_TIMEOUT_BUDGET_SECONDS=$((
(4 * SERVICE_START_TIMEOUT_SECONDS)
(8 * SERVICE_START_TIMEOUT_SECONDS)
+ INDEXER_START_TIMEOUT_SECONDS
+ MANAGER_RESET_TIMEOUT_SECONDS
+ (2 * MANAGER_RESET_TIMEOUT_SECONDS)
+ MANAGER_START_TIMEOUT_SECONDS
+ MANAGER_VERIFY_TIMEOUT_SECONDS
+ MANAGER_ROLLBACK_TIMEOUT_SECONDS
))
if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt 570 ]; then
echo "BLOCKER apply_phase_timeout_budget_exceeds_570_seconds" >&2
if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt "$EXECUTOR_DEADLINE_SECONDS" ]; then
echo "BLOCKER apply_phase_timeout_budget_exceeds_executor_deadline" >&2
exit 64
fi
EXECUTOR_DEADLINE_EPOCH=$(( $(date +%s) + EXECUTOR_DEADLINE_SECONDS ))
remaining_executor_seconds() {
local now remaining
now="$(date +%s)"
remaining=$((EXECUTOR_DEADLINE_EPOCH - now))
[ "$remaining" -gt 0 ] || return 1
printf '%s' "$remaining"
}
bounded_timeout() {
local requested="$1" remaining effective
shift
remaining="$(remaining_executor_seconds)" || return 124
effective="$requested"
[ "$effective" -le "$remaining" ] || effective="$remaining"
[ "$effective" -gt 0 ] || return 124
timeout --signal=TERM --kill-after=2 "$effective" "$@"
}
write_epoch_marker() {
local marker="$1" epoch="$2" tmp
@@ -184,19 +225,19 @@ write_epoch_marker() {
unit_active() {
local unit="$1" state
state="$(systemctl is-active "$unit" 2>/dev/null || true)"
state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-active "$unit" 2>/dev/null || true)"
printf '%s' "${state:-unknown}"
}
unit_enabled() {
local unit="$1" state
state="$(systemctl is-enabled "$unit" 2>/dev/null || true)"
state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-enabled "$unit" 2>/dev/null || true)"
printf '%s' "${state:-unknown}"
}
unit_property() {
local unit="$1" property="$2" state
state="$(systemctl show "$unit" -p "$property" --value 2>/dev/null || true)"
state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl show "$unit" -p "$property" --value 2>/dev/null || true)"
printf '%s' "${state:-unknown}"
}
@@ -214,7 +255,7 @@ start_if_inactive() {
if [ "$state" = "active" ] || [ "$state" = "activating" ]; then
return 0
fi
if timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl start "$unit"; then
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl start "$unit"; then
record_action "$action"
runtime_write_performed=1
else
@@ -228,7 +269,7 @@ enable_if_disabled() {
if [ "$state" = "enabled" ] || [ "$state" = "static" ]; then
return 0
fi
if systemctl enable "$unit" >/dev/null; then
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl enable "$unit" >/dev/null; then
record_action "$action"
runtime_write_performed=1
else
@@ -238,17 +279,19 @@ enable_if_disabled() {
tcp_port_listening() {
local port="$1"
ss -H -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$port$"
bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -ltn 2>/dev/null \
| awk '{print $4}' | grep -Eq "(^|:)$port$"
}
udp_port_listening() {
local port="$1"
ss -H -lun 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$port$"
bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -lun 2>/dev/null \
| awk '{print $4}' | grep -Eq "(^|:)$port$"
}
process_count() {
local process_name="$1" count
count="$(pgrep -xc "$process_name" 2>/dev/null || true)"
count="$(bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -xc "$process_name" 2>/dev/null || true)"
printf '%s' "${count:-0}"
}
@@ -261,7 +304,8 @@ memory_available_kb() {
disk_readback() {
local target="/var/ossec" line
[ -e "$target" ] || target="/"
line="$(df -Pk "$target" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $4 " " $5; exit}')"
line="$(bounded_timeout "$READ_TIMEOUT_SECONDS" df -Pk "$target" 2>/dev/null \
| awk 'NR==2 {gsub(/%/, "", $5); print $4 " " $5; exit}')"
if [ -n "$line" ]; then
printf '%s' "$line"
else
@@ -364,21 +408,27 @@ evaluate_manager_preflight() {
}
wait_for_manager_verifier() {
local deadline now
local deadline now remaining
deadline=$(( $(date +%s) + MANAGER_VERIFY_TIMEOUT_SECONDS ))
while true; do
capture_manager_state
[ "$manager_verifier_ready" -eq 1 ] && return 0
now="$(date +%s)"
[ "$now" -ge "$deadline" ] && return 1
sleep "$MANAGER_VERIFY_INTERVAL_SECONDS"
remaining="$((deadline - now))"
[ "$remaining" -gt 0 ] || return 1
if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -lt "$remaining" ]; then
sleep "$MANAGER_VERIFY_INTERVAL_SECONDS"
else
sleep "$remaining"
fi
done
}
rollback_manager_to_inactive() {
rollback_attempted=1
rollback_terminal="bounded_stop_attempted"
if timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service; then
if bounded_timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service; then
record_action "rollback_stop_wazuh_manager"
else
record_action "rollback_stop_wazuh_manager_failed"
@@ -396,33 +446,150 @@ rollback_manager_to_inactive() {
fi
}
write_public_receipt() {
local receipt_name receipt_path tmp generated_at receipt_line
receipt_name="$(printf '%s' "$RUN_ID" | tr -c 'A-Za-z0-9._+-' '_')"
[ -n "$receipt_name" ] || receipt_name="unknown"
if [ "$MODE" = "timer_apply" ]; then
receipt_path="$STATE_DIR/last-timer-manager-recovery-receipt.txt"
else
receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
receipt_name_for_identity() {
local identity_hash safe_run
identity_hash="$(printf '%s\0%s\0%s\0' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" | sha256sum | awk '{print $1}')"
safe_run="$(printf '%s' "$RUN_ID" | tr -c 'A-Za-z0-9._+-' '_')"
safe_run="${safe_run:0:48}"
[ -n "$safe_run" ] || safe_run="unknown"
printf '%s-%s' "$safe_run" "$identity_hash"
}
receipt_token() {
local path="$1" key="$2"
awk -v prefix="${key}=" '
{
for (i = 1; i <= NF; i++) {
if (index($i, prefix) == 1) {
print substr($i, length(prefix) + 1)
exit
}
}
}
' "$path" 2>/dev/null
}
load_durable_receipt() {
local receipt_name stored_trace stored_run stored_work_item stored_boot
receipt_name="$(receipt_name_for_identity)"
receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
[ -f "$receipt_path" ] || return 1
durable_receipt_present=1
stored_trace="$(receipt_token "$receipt_path" trace_id)"
stored_run="$(receipt_token "$receipt_path" run_id)"
stored_work_item="$(receipt_token "$receipt_path" work_item_id)"
stored_boot="$(receipt_token "$receipt_path" boot_id)"
durable_receipt_terminal="$(receipt_token "$receipt_path" terminal)"
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
receipt_ref="host112-manager-recovery:${receipt_name}"
if [ "$stored_trace" = "$TRACE_ID" ] \
&& [ "$stored_run" = "$RUN_ID" ] \
&& [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then
durable_receipt_identity_match=1
fi
if [ "$stored_boot" = "$boot_id" ]; then
durable_receipt_boot_match=1
fi
if [ "$durable_receipt_identity_match" -ne 1 ] \
|| [ "$durable_receipt_boot_match" -ne 1 ]; then
receipt_status="immutable_identity_or_boot_mismatch"
return 65
fi
return 0
}
write_public_receipt() {
local receipt_name tmp generated_at receipt_line load_status
if load_durable_receipt; then
receipt_status="reused_immutable"
receipt_reused=1
return 0
else
load_status=$?
if [ "$load_status" -eq 65 ]; then
return 1
fi
fi
receipt_name="$(receipt_name_for_identity)"
receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
generated_at="$(date --iso-8601=seconds)"
receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v1 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed manager_runtime_write_performed=$manager_runtime_write_performed rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v2 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
if install -d -m 0750 "$(dirname "$receipt_path")"; then
tmp="${receipt_path}.tmp.$$"
if printf '%s\n' "$receipt_line" >"$tmp" \
&& chmod 0640 "$tmp" \
&& mv "$tmp" "$receipt_path"; then
&& ln "$tmp" "$receipt_path" 2>/dev/null; then
rm -f "$tmp"
receipt_write_performed=1
receipt_status="written"
receipt_ref="host112-manager-recovery:$RUN_ID"
receipt_status="written_immutable"
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
receipt_ref="host112-manager-recovery:${receipt_name}"
durable_receipt_present=1
durable_receipt_identity_match=1
durable_receipt_boot_match=1
durable_receipt_terminal="$manager_terminal"
return 0
fi
rm -f "$tmp" >/dev/null 2>&1 || true
if load_durable_receipt; then
receipt_status="reused_immutable"
receipt_reused=1
return 0
fi
fi
receipt_status="write_failed"
return 1
}
write_immutable_readback() {
local receipt_name tmp stored_trace stored_run stored_work_item stored_boot latest_tmp
receipt_name="$(receipt_name_for_identity)"
durable_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt"
if [ -f "$durable_readback_path" ]; then
stored_trace="$(receipt_token "$durable_readback_path" trace_id)"
stored_run="$(receipt_token "$durable_readback_path" run_id)"
stored_work_item="$(receipt_token "$durable_readback_path" work_item_id)"
stored_boot="$(receipt_token "$durable_readback_path" boot_id)"
if [ "$stored_trace" != "$TRACE_ID" ] \
|| [ "$stored_run" != "$RUN_ID" ] \
|| [ "$stored_work_item" != "$WORK_ITEM_ID" ] \
|| [ "$stored_boot" != "$boot_id" ]; then
durable_readback_status="immutable_identity_or_boot_mismatch"
return 1
fi
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
durable_readback_status="reused_immutable"
return 0
fi
install -d -m 0750 "$(dirname "$durable_readback_path")" || return 1
tmp="${durable_readback_path}.tmp.$$"
if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s\n' \
"$(date --iso-8601=seconds)" "$readback" "$receipt_path" "$receipt_sha256" >"$tmp" \
|| ! chmod 0640 "$tmp" \
|| ! ln "$tmp" "$durable_readback_path" 2>/dev/null; then
rm -f "$tmp" >/dev/null 2>&1 || true
durable_readback_status="write_failed"
return 1
fi
rm -f "$tmp"
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
durable_readback_status="written_immutable"
# Preserve the historical convenience readback as an explicitly mutable
# pointer. It is never used as execution proof; the immutable path/hash is.
latest_tmp="${STATE_DIR}/last-readback.txt.tmp.$$"
if printf 'immutable_path=%s immutable_sha256=%s\n' \
"$durable_readback_path" "$durable_readback_sha256" >"$latest_tmp" \
&& chmod 0640 "$latest_tmp" \
&& mv "$latest_tmp" "${STATE_DIR}/last-readback.txt"; then
latest_readback_pointer_written=1
return 0
fi
rm -f "$latest_tmp" >/dev/null 2>&1 || true
durable_readback_status="immutable_written_latest_pointer_failed"
return 1
}
boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)"
uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)"
if [ "$MODE" = "timer_apply" ]; then
@@ -433,7 +600,56 @@ if [ "$MODE" = "timer_apply" ]; then
WORK_ITEM_ID="HOST112-WAZUH-MANAGER-RECOVERY"
fi
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
if ! command -v sha256sum >/dev/null 2>&1; then
echo "BLOCKER sha256sum_required" >&2
exit 69
fi
if load_durable_receipt; then
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
immutable_replay=1
receipt_status="reused_immutable"
receipt_reused=1
manager_terminal="immutable_receipt_replay"
fi
else
load_status=$?
if [ "$load_status" -eq 65 ]; then
echo "BLOCKER immutable_receipt_identity_or_boot_mismatch" >&2
exit 65
fi
fi
# Capture a pre-mutation baseline so a TERM/INT/HUP receipt never relies on
# unset state, even when interruption lands during the general guest repair.
capture_manager_state
before_manager_active="$manager_active_state"
before_manager_result="$manager_result"
before_manager_orphan="$manager_orphan_analysisd"
before_manager_memory="$manager_memory_available_kb"
before_manager_disk="$manager_disk_available_kb"
before_manager_disk_used="$manager_disk_used_percent"
before_event_1514_ready="$manager_event_port_ready"
before_enrollment_1515="$enrollment_1515_tcp"
before_manager_api_55000="$manager_api_55000_tcp"
now_epoch="$(date +%s)"
evaluate_manager_preflight "$now_epoch" "$uptime_seconds"
handle_executor_signal() {
local signal_name="$1"
trap - TERM INT HUP
manager_terminal="interrupted_signal_${signal_name}"
manager_start_exit="interrupted"
if [ "$manager_runtime_write_performed" -eq 1 ]; then
rollback_manager_to_inactive || true
fi
capture_manager_state || true
write_public_receipt || true
exit 75
}
if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
&& [ "$immutable_replay" -eq 0 ]; then
if [ "${EUID:-$(id -u)}" -ne 0 ]; then
echo "BLOCKER host112_apply_requires_root" >&2
exit 77
@@ -443,10 +659,13 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
echo "BLOCKER host112_manager_single_flight_busy" >&2
exit 75
fi
trap 'handle_executor_signal TERM' TERM
trap 'handle_executor_signal INT' INT
trap 'handle_executor_signal HUP' HUP
current_default="$(systemctl get-default 2>/dev/null || true)"
current_default="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)"
if [ "$current_default" != "graphical.target" ]; then
if systemctl set-default graphical.target >/dev/null; then
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl set-default graphical.target >/dev/null; then
record_action "set_default_graphical_target"
runtime_write_performed=1
else
@@ -463,6 +682,12 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
else
record_action "ssh_service_missing"
fi
if unit_loaded awoooi-host112-guest-recovery.timer; then
enable_if_disabled awoooi-host112-guest-recovery.timer enable_recovery_timer
start_if_inactive awoooi-host112-guest-recovery.timer start_recovery_timer
else
record_action "recovery_timer_missing"
fi
indexer_state="$(unit_active wazuh-indexer.service)"
if unit_loaded wazuh-indexer.service \
@@ -478,9 +703,11 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
is_uint "$last_retry_epoch" || last_retry_epoch=0
if [ $((now_epoch - last_retry_epoch)) -ge "$INDEXER_RETRY_COOLDOWN_SECONDS" ]; then
if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then
systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true
runtime_write_performed=1
if timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then
state_write_performed=1
bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" \
systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true
if bounded_timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then
record_action "retry_wazuh_indexer"
else
record_action "retry_wazuh_indexer_failed"
@@ -495,15 +722,6 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
fi
capture_manager_state
before_manager_active="$manager_active_state"
before_manager_result="$manager_result"
before_manager_orphan="$manager_orphan_analysisd"
before_manager_memory="$manager_memory_available_kb"
before_manager_disk="$manager_disk_available_kb"
before_manager_disk_used="$manager_disk_used_percent"
before_event_1514_ready="$manager_event_port_ready"
before_enrollment_1515="$enrollment_1515_tcp"
before_manager_api_55000="$manager_api_55000_tcp"
now_epoch="$(date +%s)"
evaluate_manager_preflight "$now_epoch" "$uptime_seconds"
@@ -513,6 +731,9 @@ if [ "$MODE" = "dry_run" ]; then
already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;;
*) manager_terminal="dry_run_${manager_preflight_status}" ;;
esac
elif { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
&& [ "$immutable_replay" -eq 1 ]; then
manager_terminal="immutable_receipt_replay"
elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
case "$manager_preflight_status" in
already_verified_healthy)
@@ -520,12 +741,18 @@ elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
;;
ready)
if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then
# The cooldown marker is a durable runtime state mutation even if the
# following systemctl operation fails. Account for it immediately so
# no failure path can emit a false no-write receipt.
runtime_write_performed=1
state_write_performed=1
manager_marker_write_performed=1
manager_attempt_count=1
if timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then
if bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then
record_action "reset_failed_wazuh_manager"
runtime_write_performed=1
manager_runtime_write_performed=1
if timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then
if bounded_timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then
manager_start_exit=0
else
manager_start_exit=$?
@@ -560,9 +787,10 @@ elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
write_public_receipt || manager_terminal="${manager_terminal}_receipt_failed"
fi
systemd_state="$(systemctl is-system-running 2>/dev/null || true)"
trap - TERM INT HUP
systemd_state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-system-running 2>/dev/null || true)"
systemd_state="${systemd_state:-unknown}"
default_target="$(systemctl get-default 2>/dev/null || true)"
default_target="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)"
default_target="${default_target:-unknown}"
graphical_target="$(unit_active graphical.target)"
display_manager="$(unit_active display-manager.service)"
@@ -587,7 +815,8 @@ if [ "$ssh_service" = "active" ] \
fi
xorg="inactive"
if pgrep -x Xorg >/dev/null 2>&1 || pgrep -x Xwayland >/dev/null 2>&1; then
if bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -x Xorg >/dev/null 2>&1 \
|| bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -x Xwayland >/dev/null 2>&1; then
xorg="active"
fi
@@ -631,21 +860,19 @@ fi
trace_output="${TRACE_ID:-none}"
run_output="${RUN_ID:-none}"
work_item_output="${WORK_ITEM_ID:-none}"
readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS runtime_write_performed=$runtime_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
printf '%s\n' "$readback"
readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
install -d -m 0750 "$STATE_DIR"
tmp_readback="${STATE_DIR}/last-readback.txt.tmp.$$"
if printf 'generated_at=%s %s\n' "$(date --iso-8601=seconds)" "$readback" >"$tmp_readback" \
&& chmod 0640 "$tmp_readback" \
&& mv "$tmp_readback" "${STATE_DIR}/last-readback.txt"; then
:
else
rm -f "$tmp_readback" >/dev/null 2>&1 || true
if ! write_immutable_readback; then
printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s latest_readback_pointer_written=%s\n' \
"$readback" "$durable_readback_status" "$durable_readback_path" \
"$durable_readback_sha256" "$latest_readback_pointer_written"
exit 74
fi
fi
printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s latest_readback_pointer_written=%s\n' \
"$readback" "$durable_readback_status" "$durable_readback_path" \
"$durable_readback_sha256" "$latest_readback_pointer_written"
if [ "$MODE" = "dry_run" ]; then
case "$manager_preflight_status" in

145
scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py
View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import hashlib
import os
import shutil
import subprocess
@@ -71,7 +72,9 @@ def test_check_and_dry_run_are_real_no_write_execution_paths(tmp_path: Path) ->
assert "manager_runtime_write_performed=0" in output
assert "receipt_write_performed=0" in output
assert "manager_preflight_status=" in output
assert "apply_phase_timeout_budget_seconds=500" in output
assert "apply_phase_timeout_budget_seconds=370" in output
assert "executor_deadline_seconds=420" in output
assert "durable_receipt_present=0" in output
assert not (tmp_path / "state").exists()
assert not (tmp_path / "indexer.epoch").exists()
assert not (tmp_path / "manager.epoch").exists()
@@ -148,7 +151,8 @@ def test_manager_preflight_verifier_receipt_and_rollback_are_complete() -> None:
assert "manager_attempt_count=1" in source
assert 'timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service' in source
assert 'rollback_terminal="bounded_stop_verified"' in source
assert "schema_version=host112_wazuh_manager_recovery_receipt_v1" in source
assert "schema_version=host112_wazuh_manager_recovery_receipt_v2" in source
assert "boot_id=$boot_id" in source
assert "trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID" in source
assert 'receipt_status="write_failed"' in source
assert "prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" in source
@@ -183,14 +187,25 @@ def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None
)
success = manager_branch[:marker_failure_at]
failure = manager_branch[marker_failure_at:]
marker_success_at = success.index(
' manager_marker_write_performed=1'
)
reset_at = success.index("systemctl reset-failed wazuh-manager.service")
assert marker_success_at < reset_at
assert "runtime_write_performed=1" in success[:reset_at]
assert "state_write_performed=1" in success[:reset_at]
assert "systemctl reset-failed wazuh-manager.service" in success
assert "systemctl start wazuh-manager.service" in success
assert 'timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service' in success
assert 'bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service' in success
assert "manager_attempt_count=1" in success
assert "systemctl reset-failed wazuh-manager.service" not in failure
assert "systemctl start wazuh-manager.service" not in failure
assert "wazuh_manager_marker_write_failed_no_write" in failure
assert 'manager_terminal="no_write_cooldown_marker_write_failed"' in failure
assert "manager_marker_write_performed=1" not in failure
assert "state_write_performed=$state_write_performed" in source
assert "manager_marker_write_performed=$manager_marker_write_performed" in source
indexer_start = source.index(
' if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then'
@@ -210,17 +225,129 @@ def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None:
service = read(SERVICE)
control = read(CONTROL)
assert 'SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-30}"' in source
assert 'INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-180}"' in source
assert 'MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-120}"' in source
assert 'SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-15}"' in source
assert 'INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-90}"' in source
assert 'MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-90}"' in source
assert 'MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}"' in source
assert 'MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}"' in source
assert 'MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-45}"' in source
assert 'if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt 570 ]; then' in source
assert "TimeoutStartSec=600" in service
assert 'MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-30}"' in source
assert 'EXECUTOR_DEADLINE_SECONDS="${HOST112_EXECUTOR_DEADLINE_SECONDS:-420}"' in source
assert 'if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt "$EXECUTOR_DEADLINE_SECONDS" ]; then' in source
assert 'if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -gt "$MANAGER_VERIFY_TIMEOUT_SECONDS" ]; then' in source
assert "TimeoutStartSec=480" in service
assert "Invoke-HostSshText $targetHost $applyCommand 590 1" in control
def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> None:
source = read(READINESS)
assert "receipt_name_for_identity()" in source
assert "printf '%s\\0%s\\0%s\\0'" in source
assert 'boot_id=$boot_id' in source
assert 'ln "$tmp" "$receipt_path"' in source
assert 'mv "$tmp" "$receipt_path"' not in source
assert 'receipt_status="reused_immutable"' in source
assert 'manager_terminal="immutable_receipt_replay"' in source
assert "immutable_receipt_identity_or_boot_mismatch" in source
assert "durable_receipt_terminal=$durable_receipt_terminal" in source
assert 'ln "$tmp" "$durable_readback_path"' in source
assert "durable_readback_sha256=%s" in source
assert "latest_readback_pointer_written=%s" in source
def test_check_reads_exact_immutable_receipt_and_rejects_boot_mismatch(
tmp_path: Path,
) -> None:
trace_id = "trace-D037-receipt"
run_id = "run-D037-receipt"
work_item_id = "D037-WAZUH-HOST112"
digest = hashlib.sha256(
f"{trace_id}\0{run_id}\0{work_item_id}\0".encode()
).hexdigest()
receipt_dir = tmp_path / "state" / "receipts"
receipt_dir.mkdir(parents=True)
receipt = receipt_dir / f"{run_id}-{digest}.txt"
receipt.write_text(
"schema_version=host112_wazuh_manager_recovery_receipt_v2 "
f"trace_id={trace_id} run_id={run_id} work_item_id={work_item_id} "
"boot_id=unknown terminal=verified_healthy\n",
encoding="utf-8",
)
original = receipt.read_bytes()
env = isolated_env(tmp_path)
args = [
"bash",
str(READINESS),
"--check",
"--trace-id",
trace_id,
"--run-id",
run_id,
"--work-item-id",
work_item_id,
]
matched = subprocess.run(
args, text=True, capture_output=True, env=env, check=False, timeout=20
)
matched_output = matched.stdout + matched.stderr
assert "durable_receipt_present=1" in matched_output
assert "durable_receipt_identity_match=1" in matched_output
assert "durable_receipt_boot_match=1" in matched_output
assert "durable_receipt_terminal=verified_healthy" in matched_output
assert receipt.read_bytes() == original
receipt.write_text(
receipt.read_text(encoding="utf-8").replace(
"boot_id=unknown", "boot_id=previous-boot"
),
encoding="utf-8",
)
mismatched = subprocess.run(
args, text=True, capture_output=True, env=env, check=False, timeout=20
)
assert mismatched.returncode == 65
assert "immutable_receipt_identity_or_boot_mismatch" in (
mismatched.stdout + mismatched.stderr
)
def test_executor_has_absolute_deadline_bounded_reads_and_signal_receipt() -> None:
source = read(READINESS)
assert "remaining_executor_seconds()" in source
assert "bounded_timeout()" in source
assert 'timeout --signal=TERM --kill-after=2 "$effective" "$@"' in source
assert "handle_executor_signal()" in source
assert "trap 'handle_executor_signal TERM' TERM" in source
assert "trap 'handle_executor_signal INT' INT" in source
assert "trap 'handle_executor_signal HUP' HUP" in source
assert "write_public_receipt || true" in source
assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl show' in source
assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -ltn' in source
assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" df -Pk' in source
def test_general_guest_repair_includes_recovery_timer_independent_of_manager() -> None:
source = read(READINESS)
general_start = source.index(
' current_default="$(bounded_timeout "$READ_TIMEOUT_SECONDS"'
)
manager_start = source.index("\nfi\n\ncapture_manager_state", general_start)
general = source[general_start:manager_start]
assert "unit_loaded awoooi-host112-guest-recovery.timer" in general
assert (
"enable_if_disabled awoooi-host112-guest-recovery.timer "
"enable_recovery_timer"
) in general
assert (
"start_if_inactive awoooi-host112-guest-recovery.timer "
"start_recovery_timer"
) in general
assert 'record_action "recovery_timer_missing"' in general
def test_installer_uses_argument_regex_and_post_install_policy_readback() -> None:
source = read(INSTALLER)