wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(recovery): bind host112 closure artifacts
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 2m35s
Details
CD Pipeline / build-and-deploy (push) Failing after 7m35s
Details
CD Pipeline / post-deploy-checks (push) Has been skipped
Details
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
Details
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 16s
Details

Browse Source
This commit is contained in:
ogt
2026-07-14 12:35:15 +08:00
parent d7a17e58b2
commit c9bb4bbfc6
12 changed files with 978 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

447
scripts/reboot-recovery/host112-guest-readiness.sh
View File

@@ -147,14 +147,33 @@ durable_receipt_terminal="none"
durable_readback_status="not_requested"
durable_readback_path="none"
durable_readback_sha256="none"
durable_readback_present=0
durable_readback_identity_match=0
durable_readback_boot_match=0
durable_readback_receipt_link_match=0
durable_readback_terminal_match=0
latest_readback_pointer_written=0
immutable_replay=0
manager_attempt_count=0
manager_start_exit=not_attempted
manager_terminal="observed"
run_terminal="observed"
rollback_attempted=0
rollback_verified=0
rollback_terminal=not_required
timer_artifact_policy="not_applicable"
timer_ledger_capacity=128
timer_ledger_slot="none"
timer_observation_status="not_requested"
timer_observation_path="none"
timer_observation_sha256="none"
timer_observation_repeat_count=0
timer_observation_write_performed=0
timer_observation_failed=0
signal_readback_status="not_requested"
signal_readback_path="none"
signal_readback_sha256="none"
action_failure_observed=0
is_uint() {
[[ "$1" =~ ^[0-9]+$ ]]
@@ -213,7 +232,7 @@ bounded_timeout() {
write_epoch_marker() {
local marker="$1" epoch="$2" tmp
tmp="${marker}.tmp.$$"
tmp="${marker}.tmp"
if printf '%s\n' "$epoch" >"$tmp" \
&& chmod 0644 "$tmp" \
&& mv "$tmp" "$marker"; then
@@ -255,9 +274,12 @@ start_if_inactive() {
if [ "$state" = "active" ] || [ "$state" = "activating" ]; then
return 0
fi
# A mutating systemctl invocation may partially change runtime state before
# returning non-zero or timing out. Record the attempt before execution so
# those paths can never be classified as no-write timer observations.
runtime_write_performed=1
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl start "$unit"; then
record_action "$action"
runtime_write_performed=1
else
record_action "${action}_failed"
fi
@@ -269,9 +291,10 @@ enable_if_disabled() {
if [ "$state" = "enabled" ] || [ "$state" = "static" ]; then
return 0
fi
runtime_write_performed=1
state_write_performed=1
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl enable "$unit" >/dev/null; then
record_action "$action"
runtime_write_performed=1
else
record_action "${action}_failed"
fi
@@ -469,6 +492,21 @@ receipt_token() {
' "$path" 2>/dev/null
}
current_action_text() {
if [ "${#actions[@]}" -gt 0 ]; then
IFS=,; printf '%s' "${actions[*]}"
else
printf 'none'
fi
}
timer_slot_for_identity() {
local checksum
checksum="$(printf '%s\0%s\0%s\0' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" | cksum | awk '{print $1}')"
is_uint "$checksum" || return 1
printf '%03d' "$((checksum % timer_ledger_capacity))"
}
load_durable_receipt() {
local receipt_name stored_trace stored_run stored_work_item stored_boot
receipt_name="$(receipt_name_for_identity)"
@@ -498,25 +536,97 @@ load_durable_receipt() {
return 0
}
load_durable_readback() {
local receipt_name stored_trace stored_run stored_work_item stored_boot
local stored_receipt_path stored_receipt_sha stored_terminal
receipt_name="$(receipt_name_for_identity)"
durable_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt"
[ -f "$durable_readback_path" ] || {
durable_readback_status="missing_for_durable_receipt"
return 1
}
durable_readback_present=1
stored_trace="$(receipt_token "$durable_readback_path" trace_id)"
stored_run="$(receipt_token "$durable_readback_path" run_id)"
stored_work_item="$(receipt_token "$durable_readback_path" work_item_id)"
stored_boot="$(receipt_token "$durable_readback_path" boot_id)"
stored_receipt_path="$(receipt_token "$durable_readback_path" receipt_path)"
stored_receipt_sha="$(receipt_token "$durable_readback_path" receipt_sha256)"
stored_terminal="$(receipt_token "$durable_readback_path" durable_receipt_terminal)"
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
if [ "$stored_trace" = "$TRACE_ID" ] \
&& [ "$stored_run" = "$RUN_ID" ] \
&& [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then
durable_readback_identity_match=1
fi
if [ "$stored_boot" = "$boot_id" ]; then
durable_readback_boot_match=1
fi
if [ "$stored_receipt_path" = "$receipt_path" ] \
&& [ "$stored_receipt_sha" = "$receipt_sha256" ]; then
durable_readback_receipt_link_match=1
fi
if [ "$stored_terminal" = "$durable_receipt_terminal" ]; then
durable_readback_terminal_match=1
fi
if [ "$durable_readback_identity_match" -ne 1 ] \
|| [ "$durable_readback_boot_match" -ne 1 ] \
|| [ "$durable_readback_receipt_link_match" -ne 1 ] \
|| [ "$durable_readback_terminal_match" -ne 1 ]; then
durable_readback_status="immutable_identity_boot_receipt_or_terminal_mismatch"
return 65
fi
durable_readback_status="loaded_immutable_verified"
return 0
}
write_public_receipt() {
local receipt_name tmp generated_at receipt_line load_status
if load_durable_receipt; then
receipt_status="reused_immutable"
receipt_reused=1
return 0
else
load_status=$?
if [ "$load_status" -eq 65 ]; then
return 1
local receipt_name tmp generated_at receipt_line load_status receipt_action_text timer_slot
if [ "$MODE" != "timer_apply" ]; then
if load_durable_receipt; then
receipt_status="reused_immutable"
receipt_reused=1
return 0
else
load_status=$?
if [ "$load_status" -eq 65 ]; then
return 1
fi
fi
fi
receipt_name="$(receipt_name_for_identity)"
receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
if [ "$MODE" = "timer_apply" ]; then
timer_slot="$(timer_slot_for_identity)" || {
receipt_status="write_failed"
return 1
}
timer_ledger_slot="$timer_slot"
receipt_name="timer-slot-${timer_slot}"
receipt_path="$STATE_DIR/timer-ledger/${receipt_name}.receipt.txt"
timer_artifact_policy="bounded_ring_128_for_runtime_mutation"
else
receipt_name="$(receipt_name_for_identity)"
receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
fi
generated_at="$(date --iso-8601=seconds)"
receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v2 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
receipt_action_text="$(current_action_text)"
receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v2 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed action_count=${#actions[@]} actions=$receipt_action_text rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal manager_terminal=$manager_terminal terminal=$run_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
if install -d -m 0750 "$(dirname "$receipt_path")"; then
tmp="${receipt_path}.tmp.$$"
if printf '%s\n' "$receipt_line" >"$tmp" \
tmp="${receipt_path}.tmp"
if [ "$MODE" = "timer_apply" ] \
&& printf '%s\n' "$receipt_line" >"$tmp" \
&& chmod 0640 "$tmp" \
&& mv "$tmp" "$receipt_path"; then
receipt_write_performed=1
receipt_status="written_bounded_timer_slot"
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
receipt_ref="host112-timer-ledger:${receipt_name}"
durable_receipt_present=1
durable_receipt_identity_match=1
durable_receipt_boot_match=1
durable_receipt_terminal="$run_terminal"
return 0
elif [ "$MODE" != "timer_apply" ] \
&& printf '%s\n' "$receipt_line" >"$tmp" \
&& chmod 0640 "$tmp" \
&& ln "$tmp" "$receipt_path" 2>/dev/null; then
rm -f "$tmp"
@@ -527,11 +637,11 @@ write_public_receipt() {
durable_receipt_present=1
durable_receipt_identity_match=1
durable_receipt_boot_match=1
durable_receipt_terminal="$manager_terminal"
durable_receipt_terminal="$run_terminal"
return 0
fi
rm -f "$tmp" >/dev/null 2>&1 || true
if load_durable_receipt; then
if [ "$MODE" != "timer_apply" ] && load_durable_receipt; then
receipt_status="reused_immutable"
receipt_reused=1
return 0
@@ -541,6 +651,137 @@ write_public_receipt() {
return 1
}
write_signal_readback() {
local receipt_name tmp signal_line signal_action_text timer_slot
if [ "$MODE" = "timer_apply" ]; then
timer_slot="$(timer_slot_for_identity)" || {
signal_readback_status="write_failed"
return 1
}
timer_ledger_slot="$timer_slot"
receipt_name="timer-slot-${timer_slot}"
signal_readback_path="$STATE_DIR/timer-ledger/${receipt_name}.readback.txt"
else
receipt_name="$(receipt_name_for_identity)"
signal_readback_path="$STATE_DIR/readbacks/${receipt_name}.signal.txt"
fi
signal_action_text="$(current_action_text)"
signal_line="schema_version=host112_guest_recovery_signal_readback_v1 generated_at=$(date --iso-8601=seconds) mode=$MODE trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id manager_terminal=$manager_terminal terminal=$run_terminal manager_active=$manager_active_state manager_result=$manager_result manager_orphan_analysisd=$manager_orphan_analysisd manager_verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed action_count=${#actions[@]} actions=$signal_action_text rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal receipt_path=$receipt_path receipt_sha256=$receipt_sha256 prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
install -d -m 0750 "$(dirname "$signal_readback_path")" || {
signal_readback_status="write_failed"
return 1
}
if [ "$MODE" != "timer_apply" ] && [ -f "$signal_readback_path" ]; then
if [ "$(receipt_token "$signal_readback_path" trace_id)" != "$TRACE_ID" ] \
|| [ "$(receipt_token "$signal_readback_path" run_id)" != "$RUN_ID" ] \
|| [ "$(receipt_token "$signal_readback_path" work_item_id)" != "$WORK_ITEM_ID" ] \
|| [ "$(receipt_token "$signal_readback_path" boot_id)" != "$boot_id" ] \
|| [ "$(receipt_token "$signal_readback_path" receipt_path)" != "$receipt_path" ] \
|| [ "$(receipt_token "$signal_readback_path" receipt_sha256)" != "$receipt_sha256" ]; then
signal_readback_status="immutable_identity_boot_or_receipt_mismatch"
return 1
fi
signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')"
signal_readback_status="reused_immutable"
return 0
fi
tmp="${signal_readback_path}.tmp"
if [ "$MODE" = "timer_apply" ] \
&& printf '%s\n' "$signal_line" >"$tmp" \
&& chmod 0640 "$tmp" \
&& mv "$tmp" "$signal_readback_path"; then
signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')"
signal_readback_status="written_bounded_timer_slot"
return 0
elif [ "$MODE" != "timer_apply" ] \
&& printf '%s\n' "$signal_line" >"$tmp" \
&& chmod 0640 "$tmp" \
&& ln "$tmp" "$signal_readback_path" 2>/dev/null; then
rm -f "$tmp"
signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')"
signal_readback_status="written_immutable"
return 0
fi
rm -f "$tmp" >/dev/null 2>&1 || true
signal_readback_status="write_failed"
return 1
}
write_bounded_timer_mutation_readback() {
local timer_slot tmp
timer_slot="$(timer_slot_for_identity)" || {
durable_readback_status="write_failed"
return 1
}
timer_ledger_slot="$timer_slot"
durable_readback_path="$STATE_DIR/timer-ledger/timer-slot-${timer_slot}.readback.txt"
install -d -m 0750 "$(dirname "$durable_readback_path")" || {
durable_readback_status="write_failed"
return 1
}
tmp="${durable_readback_path}.tmp"
if printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s timer_ledger_slot=%s timer_ledger_capacity=%s retention=rolling_atomic_overwrite durable_readback_status=written_bounded_timer_slot durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \
"$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" \
"$timer_ledger_slot" "$timer_ledger_capacity" "$durable_readback_path" >"$tmp" \
&& chmod 0640 "$tmp" \
&& mv "$tmp" "$durable_readback_path"; then
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
durable_readback_status="written_bounded_timer_slot"
durable_readback_present=1
durable_readback_identity_match=1
durable_readback_boot_match=1
durable_readback_receipt_link_match=1
durable_readback_terminal_match=1
return 0
fi
rm -f "$tmp" >/dev/null 2>&1 || true
durable_readback_status="write_failed"
return 1
}
write_bounded_timer_observation() {
local tmp now previous_fingerprint previous_repeat previous_first fingerprint first_at observation_line
timer_observation_path="$STATE_DIR/timer-observation.latest"
now="$(date --iso-8601=seconds)"
fingerprint="$(printf '%s\0%s\0%s\0%s\0%s\0%s\0%s\0%s\0' \
"$boot_id" "$guest_ready" "$run_terminal" "$systemd_state" \
"$console_ready" "$services_ready" "$ssh_ready" "$recovery_timer_ready" \
| sha256sum | awk '{print $1}')"
previous_fingerprint=""
previous_repeat=0
previous_first="$now"
if [ -r "$timer_observation_path" ]; then
previous_fingerprint="$(receipt_token "$timer_observation_path" fingerprint)"
previous_repeat="$(receipt_token "$timer_observation_path" repeat_count)"
previous_first="$(receipt_token "$timer_observation_path" first_observed_at)"
fi
is_uint "$previous_repeat" || previous_repeat=0
if [ "$previous_fingerprint" = "$fingerprint" ]; then
timer_observation_repeat_count=$((previous_repeat + 1))
first_at="${previous_first:-$now}"
else
timer_observation_repeat_count=1
first_at="$now"
fi
observation_line="schema_version=host112_timer_observation_aggregate_v1 first_observed_at=$first_at last_observed_at=$now repeat_count=$timer_observation_repeat_count fingerprint=$fingerprint boot_id=$boot_id guest_ready=$guest_ready manager_terminal=$manager_terminal terminal=$run_terminal systemd_state=$systemd_state console_ready=$console_ready services_ready=$services_ready ssh_ready=$ssh_ready recovery_timer_ready=$recovery_timer_ready runtime_write_performed=0 immutable_artifact_written=0 aggregation=fixed_single_atomic_pointer"
install -d -m 0750 "$STATE_DIR" || {
timer_observation_status="write_failed"
return 1
}
tmp="${timer_observation_path}.tmp"
if printf '%s\n' "$observation_line" >"$tmp" \
&& chmod 0640 "$tmp" \
&& mv "$tmp" "$timer_observation_path"; then
timer_observation_sha256="$(sha256sum "$timer_observation_path" | awk '{print $1}')"
timer_observation_status="updated_bounded_aggregate"
timer_observation_write_performed=1
return 0
fi
rm -f "$tmp" >/dev/null 2>&1 || true
timer_observation_status="write_failed"
return 1
}
write_immutable_readback() {
local receipt_name tmp stored_trace stored_run stored_work_item stored_boot latest_tmp
receipt_name="$(receipt_name_for_identity)"
@@ -557,14 +798,16 @@ write_immutable_readback() {
durable_readback_status="immutable_identity_or_boot_mismatch"
return 1
fi
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
durable_readback_status="reused_immutable"
return 0
if load_durable_readback; then
durable_readback_status="reused_immutable_verified"
return 0
fi
return 1
fi
install -d -m 0750 "$(dirname "$durable_readback_path")" || return 1
tmp="${durable_readback_path}.tmp.$$"
if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s\n' \
"$(date --iso-8601=seconds)" "$readback" "$receipt_path" "$receipt_sha256" >"$tmp" \
tmp="${durable_readback_path}.tmp"
if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s durable_readback_status=written_immutable durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \
"$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" "$durable_readback_path" >"$tmp" \
|| ! chmod 0640 "$tmp" \
|| ! ln "$tmp" "$durable_readback_path" 2>/dev/null; then
rm -f "$tmp" >/dev/null 2>&1 || true
@@ -574,10 +817,15 @@ write_immutable_readback() {
rm -f "$tmp"
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
durable_readback_status="written_immutable"
durable_readback_present=1
durable_readback_identity_match=1
durable_readback_boot_match=1
durable_readback_receipt_link_match=1
durable_readback_terminal_match=1
# Preserve the historical convenience readback as an explicitly mutable
# pointer. It is never used as execution proof; the immutable path/hash is.
latest_tmp="${STATE_DIR}/last-readback.txt.tmp.$$"
latest_tmp="${STATE_DIR}/last-readback.txt.tmp"
if printf 'immutable_path=%s immutable_sha256=%s\n' \
"$durable_readback_path" "$durable_readback_sha256" >"$latest_tmp" \
&& chmod 0640 "$latest_tmp" \
@@ -600,8 +848,9 @@ if [ "$MODE" = "timer_apply" ]; then
WORK_ITEM_ID="HOST112-WAZUH-MANAGER-RECOVERY"
fi
if ! command -v sha256sum >/dev/null 2>&1; then
echo "BLOCKER sha256sum_required" >&2
if ! command -v sha256sum >/dev/null 2>&1 \
|| ! command -v cksum >/dev/null 2>&1; then
echo "BLOCKER sha256sum_and_cksum_required" >&2
exit 69
fi
@@ -619,6 +868,12 @@ else
exit 65
fi
fi
if [ "$durable_receipt_present" -eq 1 ]; then
if ! load_durable_readback; then
echo "BLOCKER durable_readback_missing_or_mismatched_for_receipt" >&2
exit 65
fi
fi
# Capture a pre-mutation baseline so a TERM/INT/HUP receipt never relies on
# unset state, even when interruption lands during the general guest repair.
@@ -639,12 +894,18 @@ handle_executor_signal() {
local signal_name="$1"
trap - TERM INT HUP
manager_terminal="interrupted_signal_${signal_name}"
run_terminal="interrupted_signal_${signal_name}"
manager_start_exit="interrupted"
if [ "$manager_runtime_write_performed" -eq 1 ]; then
rollback_manager_to_inactive || true
fi
capture_manager_state || true
write_public_receipt || true
if [ "$runtime_write_performed" -eq 1 ]; then
if write_public_receipt && write_signal_readback; then
exit 75
fi
exit 74
fi
exit 75
}
@@ -665,9 +926,10 @@ if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
current_default="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)"
if [ "$current_default" != "graphical.target" ]; then
runtime_write_performed=1
state_write_performed=1
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl set-default graphical.target >/dev/null; then
record_action "set_default_graphical_target"
runtime_write_performed=1
else
record_action "set_default_graphical_target_failed"
fi
@@ -758,8 +1020,13 @@ elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
manager_start_exit=$?
fi
if wait_for_manager_verifier; then
record_action "start_wazuh_manager_verified"
manager_terminal="verified_healthy"
if [ "$manager_start_exit" = "0" ]; then
record_action "start_wazuh_manager_verified"
manager_terminal="verified_healthy"
else
record_action "wazuh_manager_verifier_healthy_after_nonzero_start_unattributed"
manager_terminal="manager_start_nonzero_verifier_healthy_unattributed"
fi
else
record_action "start_wazuh_manager_verifier_failed"
rollback_manager_to_inactive
@@ -784,10 +1051,8 @@ elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
;;
esac
capture_manager_state
write_public_receipt || manager_terminal="${manager_terminal}_receipt_failed"
fi
trap - TERM INT HUP
systemd_state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-system-running 2>/dev/null || true)"
systemd_state="${systemd_state:-unknown}"
default_target="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)"
@@ -852,27 +1117,111 @@ if [ "$console_ready" -eq 1 ] \
guest_ready=1
fi
if [ "${#actions[@]}" -gt 0 ]; then
for action in "${actions[@]}"; do
case "$action" in
*_failed|*_failed_no_start|*_failed_no_write|*_unattributed|*_missing) action_failure_observed=1 ;;
esac
done
fi
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
if [ "$immutable_replay" -eq 1 ]; then
run_terminal="immutable_receipt_replay"
elif [ "$guest_ready" -eq 1 ] \
&& [ "$action_failure_observed" -eq 0 ] \
&& { [ "$manager_terminal" = "verified_healthy" ] \
|| [ "$manager_terminal" = "idempotent_already_verified_healthy" ]; }; then
if [ "$runtime_write_performed" -eq 1 ]; then
run_terminal="verified_healthy"
else
run_terminal="idempotent_already_verified_healthy"
fi
elif [ "$action_failure_observed" -eq 1 ]; then
run_terminal="execution_failed_or_unattributed_${manager_terminal}"
else
run_terminal="guest_verification_failed_${manager_terminal}"
fi
fi
action_text="none"
if [ "${#actions[@]}" -gt 0 ]; then
action_text="$(IFS=,; printf '%s' "${actions[*]}")"
fi
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
# Keep signal handling active through the complete guest verifier above.
# Ignore TERM/INT/HUP only for the short atomic receipt/readback commit
# section so a successful receipt can never be separated from its readback.
trap '' TERM INT HUP
if [ "$immutable_replay" -eq 0 ]; then
if [ "$MODE" = "apply" ] || [ "$runtime_write_performed" -eq 1 ]; then
if [ "$MODE" = "timer_apply" ]; then
timer_artifact_policy="bounded_ring_128_for_runtime_mutation"
fi
if ! write_public_receipt; then
run_terminal="${run_terminal}_receipt_failed"
fi
else
receipt_status="timer_no_runtime_mutation_aggregated"
timer_artifact_policy="bounded_mutable_observation_no_immutable_artifact"
fi
fi
if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then
if ! write_bounded_timer_observation; then
timer_observation_failed=1
fi
durable_readback_status="skipped_bounded_timer_no_runtime_mutation"
fi
fi
trace_output="${TRACE_ID:-none}"
run_output="${RUN_ID:-none}"
work_item_output="${WORK_ITEM_ID:-none}"
readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_terminal=$manager_terminal run_terminal=$run_terminal terminal=$run_terminal action_failure_observed=$action_failure_observed manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal durable_readback_present=$durable_readback_present durable_readback_identity_match=$durable_readback_identity_match durable_readback_boot_match=$durable_readback_boot_match durable_readback_receipt_link_match=$durable_readback_receipt_link_match durable_readback_terminal_match=$durable_readback_terminal_match rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal timer_artifact_policy=$timer_artifact_policy timer_ledger_capacity=$timer_ledger_capacity timer_ledger_slot=$timer_ledger_slot timer_observation_status=$timer_observation_status timer_observation_path=$timer_observation_path timer_observation_sha256=$timer_observation_sha256 timer_observation_repeat_count=$timer_observation_repeat_count timer_observation_write_performed=$timer_observation_write_performed signal_readback_status=$signal_readback_status signal_readback_path=$signal_readback_path signal_readback_sha256=$signal_readback_sha256 action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
canonical_readback_base() {
printf '%s\n' "$readback" | awk '
{
separator = ""
for (i = 1; i <= NF; i++) {
if ($i ~ /^durable_readback_(present|identity_match|boot_match|receipt_link_match|terminal_match)=/) {
continue
}
printf "%s%s", separator, $i
separator = " "
}
printf "\n"
}
'
}
emit_readback() {
printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s durable_readback_present=%s durable_readback_identity_match=%s durable_readback_boot_match=%s durable_readback_receipt_link_match=%s durable_readback_terminal_match=%s latest_readback_pointer_written=%s\n' \
"$(canonical_readback_base)" "$durable_readback_status" "$durable_readback_path" \
"$durable_readback_sha256" "$durable_readback_present" \
"$durable_readback_identity_match" "$durable_readback_boot_match" \
"$durable_readback_receipt_link_match" "$durable_readback_terminal_match" \
"$latest_readback_pointer_written"
}
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
if ! write_immutable_readback; then
printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s latest_readback_pointer_written=%s\n' \
"$readback" "$durable_readback_status" "$durable_readback_path" \
"$durable_readback_sha256" "$latest_readback_pointer_written"
if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then
if [ "$timer_observation_failed" -eq 1 ]; then
emit_readback
exit 74
fi
elif [ "$MODE" = "timer_apply" ]; then
if ! write_bounded_timer_mutation_readback; then
emit_readback
exit 74
fi
elif ! write_immutable_readback; then
emit_readback
exit 74
fi
fi
printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s latest_readback_pointer_written=%s\n' \
"$readback" "$durable_readback_status" "$durable_readback_path" \
"$durable_readback_sha256" "$latest_readback_pointer_written"
emit_readback
if [ "$MODE" = "dry_run" ]; then
case "$manager_preflight_status" in
@@ -881,4 +1230,16 @@ if [ "$MODE" = "dry_run" ]; then
esac
fi
[ "$guest_ready" -eq 1 ] && [ "$receipt_status" != "write_failed" ]
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
successful_terminal="$run_terminal"
if [ "$immutable_replay" -eq 1 ]; then
successful_terminal="$durable_receipt_terminal"
fi
[ "$guest_ready" -eq 1 ] \
&& [ "$receipt_status" != "write_failed" ] \
&& { [ "$successful_terminal" = "verified_healthy" ] \
|| [ "$successful_terminal" = "idempotent_already_verified_healthy" ]; }
exit $?
fi
[ "$guest_ready" -eq 1 ]

2
scripts/reboot-recovery/install-host112-guest-recovery.sh
View File

@@ -484,7 +484,7 @@ handle_apply_failure() {
static_source_preflight() {
local source service_state
for tool in bash cmp flock getent install sha256sum ssh-keygen sudo systemctl systemd-analyze timeout visudo; do
for tool in bash cksum cmp flock getent install sha256sum ssh-keygen sudo systemctl systemd-analyze timeout visudo; do
command -v "$tool" >/dev/null 2>&1 || { echo "BLOCKER required_tool_missing=$tool" >&2; return 1; }
done
for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "$CONTROL_PUBLIC_KEY_PATH" "${SCRIPT_DIR}/install-host112-guest-recovery.sh"; do

2
scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py
View File

@@ -792,7 +792,7 @@ def source_controls() -> dict[str, bool]:
"scripts/reboot-recovery/awoooi-host112-guest-recovery.service"
),
"ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer",
"TimeoutStartSec=600",
"TimeoutStartSec=480",
),
"host_112_guest_recovery_timer_source_present": file_contains(
source_file(

81
scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py
View File

@@ -149,6 +149,8 @@ def test_manager_preflight_verifier_receipt_and_rollback_are_complete() -> None:
'timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service'
) == 1
assert "manager_attempt_count=1" in source
assert 'if [ "$manager_start_exit" = "0" ]; then' in source
assert 'manager_terminal="manager_start_nonzero_verifier_healthy_unattributed"' in source
assert 'timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service' in source
assert 'rollback_terminal="bounded_stop_verified"' in source
assert "schema_version=host112_wazuh_manager_recovery_receipt_v2" in source
@@ -171,7 +173,7 @@ def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None
marker_function = source[
source.index("write_epoch_marker() {") : source.index("unit_active() {")
]
assert 'tmp="${marker}.tmp.$$"' in marker_function
assert 'tmp="${marker}.tmp"' in marker_function
assert 'printf \'%s\\n\' "$epoch" >"$tmp"' in marker_function
assert 'chmod 0644 "$tmp"' in marker_function
assert 'mv "$tmp" "$marker"' in marker_function
@@ -251,7 +253,15 @@ def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> No
assert "printf '%s\\0%s\\0%s\\0'" in source
assert 'boot_id=$boot_id' in source
assert 'ln "$tmp" "$receipt_path"' in source
assert 'mv "$tmp" "$receipt_path"' not in source
receipt_writer = source[
source.index("write_public_receipt() {") : source.index(
"write_signal_readback() {",
)
]
assert 'if [ "$MODE" = "timer_apply" ]' in receipt_writer
assert 'mv "$tmp" "$receipt_path"' in receipt_writer
assert 'elif [ "$MODE" != "timer_apply" ]' in receipt_writer
assert 'ln "$tmp" "$receipt_path"' in receipt_writer
assert 'receipt_status="reused_immutable"' in source
assert 'manager_terminal="immutable_receipt_replay"' in source
assert "immutable_receipt_identity_or_boot_mismatch" in source
@@ -279,6 +289,17 @@ def test_check_reads_exact_immutable_receipt_and_rejects_boot_mismatch(
"boot_id=unknown terminal=verified_healthy\n",
encoding="utf-8",
)
readback_dir = tmp_path / "state" / "readbacks"
readback_dir.mkdir(parents=True)
durable_readback = readback_dir / f"{run_id}-{digest}.txt"
receipt_sha = hashlib.sha256(receipt.read_bytes()).hexdigest()
durable_readback.write_text(
"schema_version=host112_guest_recovery_v2 "
f"trace_id={trace_id} run_id={run_id} work_item_id={work_item_id} "
"boot_id=unknown durable_receipt_terminal=verified_healthy "
f"receipt_path={receipt} receipt_sha256={receipt_sha}\n",
encoding="utf-8",
)
original = receipt.read_bytes()
env = isolated_env(tmp_path)
args = [
@@ -301,6 +322,9 @@ def test_check_reads_exact_immutable_receipt_and_rejects_boot_mismatch(
assert "durable_receipt_identity_match=1" in matched_output
assert "durable_receipt_boot_match=1" in matched_output
assert "durable_receipt_terminal=verified_healthy" in matched_output
assert "durable_readback_present=1" in matched_output
assert "durable_readback_receipt_link_match=1" in matched_output
assert "durable_readback_terminal_match=1" in matched_output
assert receipt.read_bytes() == original
receipt.write_text(
@@ -328,12 +352,56 @@ def test_executor_has_absolute_deadline_bounded_reads_and_signal_receipt() -> No
assert "trap 'handle_executor_signal TERM' TERM" in source
assert "trap 'handle_executor_signal INT' INT" in source
assert "trap 'handle_executor_signal HUP' HUP" in source
assert "write_public_receipt || true" in source
assert "if write_public_receipt && write_signal_readback; then" in source
assert "action_count=${#actions[@]} actions=$receipt_action_text" in source
assert "schema_version=host112_guest_recovery_signal_readback_v1" in source
assert 'ln "$tmp" "$signal_readback_path"' in source
assert "signal_readback_status=\"written_immutable\"" in source
assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl show' in source
assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -ltn' in source
assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" df -Pk' in source
def test_timer_noop_uses_one_bounded_aggregate_instead_of_per_minute_artifacts() -> None:
source = read(READINESS)
function = source[
source.index("write_bounded_timer_observation() {") : source.index(
'boot_id="$(cat /proc/sys/kernel/random/boot_id',
)
]
assert 'timer_observation_path="$STATE_DIR/timer-observation.latest"' in function
assert "schema_version=host112_timer_observation_aggregate_v1" in function
assert "aggregation=fixed_single_atomic_pointer" in function
assert 'mv "$tmp" "$timer_observation_path"' in function
assert "repeat_count=$timer_observation_repeat_count" in function
assert 'if [ "$MODE" = "apply" ] || [ "$runtime_write_performed" -eq 1 ]; then' in source
assert 'receipt_status="timer_no_runtime_mutation_aggregated"' in source
assert 'durable_readback_status="skipped_bounded_timer_no_runtime_mutation"' in source
assert "timer_ledger_capacity=128" in source
assert "write_bounded_timer_mutation_readback()" in source
assert "retention=rolling_atomic_overwrite" in source
assert 'timer_artifact_policy="bounded_ring_128_for_runtime_mutation"' in source
assert 'timer_artifact_policy="bounded_mutable_observation_no_immutable_artifact"' in source
def test_apply_success_is_bound_to_same_run_durable_terminal() -> None:
source = read(READINESS)
terminal_gate = source[source.rindex('if [ "$MODE" = "apply" ]') :]
assert 'successful_terminal="$run_terminal"' in terminal_gate
assert 'successful_terminal="$durable_receipt_terminal"' in terminal_gate
assert '[ "$successful_terminal" = "verified_healthy" ]' in terminal_gate
assert '[ "$successful_terminal" = "idempotent_already_verified_healthy" ]' in terminal_gate
guest_verifier = source.index('guest_ready=0\nif [ "$console_ready" -eq 1 ]')
receipt_commit = source.index("if ! write_public_receipt; then", guest_verifier)
assert guest_verifier < receipt_commit
assert 'run_terminal="guest_verification_failed_${manager_terminal}"' in source
assert 'run_terminal="execution_failed_or_unattributed_${manager_terminal}"' in source
assert 'action_failure_observed=1' in source
assert "manager_terminal=$manager_terminal terminal=$run_terminal" in source
def test_general_guest_repair_includes_recovery_timer_independent_of_manager() -> None:
source = read(READINESS)
general_start = source.index(
@@ -424,10 +492,17 @@ def test_agent99_manager_success_requires_apply_and_immutable_receipt_chain() ->
assert "$applyReceipt.managerIndependentVerifierReady" in function
assert "$applyReceipt.managerRuntimeWritePerformed" in function
assert '[string]$applyReceipt.managerAttemptCount -eq "1"' in function
assert '[string]$applyReceipt.managerStartExit -eq "0"' in function
assert "$durableReceiptReadback.receiptPath -eq $applyReceipt.receiptPath" in function
assert "$durableReceiptReadback.receiptSha256 -eq $applyReceipt.receiptSha256" in function
assert "$durableReceiptReadback.durableReceiptTerminal -eq $applyReceipt.durableReceiptTerminal" in function
assert '$durableReceiptReadback.durableReceiptTerminal -eq "verified_healthy"' in function
assert "$applyReceipt.durableReadbackReceiptLinkMatch" in function
assert "$applyReceipt.durableReadbackTerminalMatch" in function
assert "$durableReceiptReadback.durableReadbackReceiptLinkMatch" in function
assert "$durableReceiptReadback.durableReadbackTerminalMatch" in function
assert "$before.durableReadbackReceiptLinkMatch" in function
assert "$before.durableReadbackTerminalMatch" in function
assert "$afterManagerVerifierVerified" in function
assert 'afterVerifierRole = "independent_second_verifier_only"' in function
assert "$after.ready -and" in function

419
scripts/reboot-recovery/tests/test_host112_recovery_runtime_harness.py Normal file
View File

@@ -0,0 +1,419 @@
from __future__ import annotations
import os
import signal
import subprocess
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
READINESS = ROOT / "scripts/reboot-recovery/host112-guest-readiness.sh"
def _write_executable(path: Path, body: str) -> None:
path.write_text(body, encoding="utf-8")
path.chmod(0o755)
def _runtime_harness(
tmp_path: Path,
*,
ledger_capacity: int = 128,
) -> tuple[Path, dict[str, str], Path]:
source = READINESS.read_text(encoding="utf-8")
root_gate = 'if [ "${EUID:-$(id -u)}" -ne 0 ]; then'
assert source.count(root_gate) == 1
assert source.count("timer_ledger_capacity=128") == 1
source = source.replace(root_gate, "if false; then", 1)
source = source.replace(
"timer_ledger_capacity=128",
f"timer_ledger_capacity={ledger_capacity}",
1,
)
script = tmp_path / "host112-runtime-harness.sh"
_write_executable(script, source)
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
fake_state = tmp_path / "fake-host"
fake_state.mkdir()
_write_executable(
fake_bin / "timeout",
"""#!/usr/bin/env bash
set -u
while [ "$#" -gt 0 ]; do
case "$1" in
--signal=*|--kill-after=*) shift ;;
*) break ;;
esac
done
[ "$#" -gt 0 ] || exit 64
shift
"$@"
""",
)
_write_executable(
fake_bin / "date",
"""#!/usr/bin/env bash
case "${1:-}" in
--iso-8601=seconds) printf '%s\n' '2026-07-14T12:00:00+08:00' ;;
+%s) printf '%s\n' '1000' ;;
*) /bin/date "$@" ;;
esac
""",
)
_write_executable(
fake_bin / "flock",
"""#!/usr/bin/env bash
exit 0
""",
)
_write_executable(
fake_bin / "mv",
"""#!/usr/bin/env bash
set -u
destination=""
for argument in "$@"; do destination="$argument"; done
if [ -n "${FAKE_RECEIPT_MV_MARKER:-}" ] && [[ "$destination" == *.receipt.txt ]]; then
: >"$FAKE_RECEIPT_MV_MARKER"
sleep "${FAKE_RECEIPT_MV_SLEEP_SECONDS:-0}"
fi
/bin/mv "$@"
""",
)
_write_executable(
fake_bin / "pgrep",
"""#!/usr/bin/env bash
case "$*" in
*wazuh-analysisd*) printf '%s\n' '1'; exit 0 ;;
*Xorg*|*Xwayland*) exit 0 ;;
*) exit 1 ;;
esac
""",
)
_write_executable(
fake_bin / "ss",
"""#!/usr/bin/env bash
case "$*" in
*-ltn*)
for port in 22 1514 1515 55000; do
printf 'LISTEN 0 128 0.0.0.0:%s 0.0.0.0:*\n' "$port"
done
;;
*-lun*) printf '%s\n' 'UNCONN 0 0 0.0.0.0:1514 0.0.0.0:*' ;;
esac
""",
)
_write_executable(
fake_bin / "df",
"""#!/usr/bin/env bash
printf '%s\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on'
printf '%s\n' '/dev/fake 10000000 1000 9999000 1% /'
""",
)
_write_executable(
fake_bin / "systemctl",
"""#!/usr/bin/env bash
set -u
state="${FAKE_HOST_STATE_DIR:?}"
cmd="${1:-}"
[ "$#" -gt 0 ] && shift
case "$cmd" in
show)
unit="${1:-}"; shift || true
property=""
while [ "$#" -gt 0 ]; do
if [ "$1" = "-p" ]; then property="${2:-}"; shift 2; else shift; fi
done
case "$property" in
LoadState) printf '%s\n' 'loaded' ;;
ActiveState) printf '%s\n' 'active' ;;
SubState) printf '%s\n' 'running' ;;
Result) printf '%s\n' 'success' ;;
*) printf '%s\n' 'unknown' ;;
esac
;;
is-active)
unit="${1:-}"
case "$unit" in
lightdm.service|display-manager.service)
if [ -f "$state/lightdm_active" ]; then printf '%s\n' 'active'; else printf '%s\n' 'inactive'; fi
;;
*) printf '%s\n' 'active' ;;
esac
;;
is-enabled) printf '%s\n' 'enabled' ;;
is-system-running)
if [ -n "${FAKE_FINALIZE_MARKER:-}" ]; then
: >"$FAKE_FINALIZE_MARKER"
sleep "${FAKE_FINALIZE_SLEEP_SECONDS:-0}"
fi
printf '%s\n' 'running'
;;
get-default) printf '%s\n' 'graphical.target' ;;
set-default)
printf 'set-default:%s\n' "${1:-}" >>"$state/mutations.log"
exit "${FAKE_SET_DEFAULT_RC:-0}"
;;
start)
unit="${1:-}"
printf 'start:%s\n' "$unit" >>"$state/mutations.log"
if [ "$unit" = "lightdm.service" ]; then
if [ "${FAKE_PARTIAL_LIGHTDM_START:-0}" = "1" ]; then : >"$state/lightdm_active"; fi
rc="${FAKE_LIGHTDM_START_RC:-0}"
if [ "$rc" -eq 0 ]; then : >"$state/lightdm_active"; fi
exit "$rc"
fi
;;
enable)
printf 'enable:%s\n' "${1:-}" >>"$state/mutations.log"
;;
reset-failed|stop) ;;
*) exit 1 ;;
esac
""",
)
env = os.environ.copy()
env.update(
{
"PATH": f"{fake_bin}:{env['PATH']}",
"FAKE_HOST_STATE_DIR": str(fake_state),
"HOST112_RECOVERY_STATE_DIR": str(tmp_path / "state"),
"HOST112_INDEXER_RETRY_MARKER": str(tmp_path / "indexer.epoch"),
"HOST112_MANAGER_RETRY_MARKER": str(tmp_path / "manager.epoch"),
"HOST112_MANAGER_LOCK_FILE": str(tmp_path / "manager.lock"),
}
)
return script, env, fake_state
def _apply(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[
"bash",
str(script),
"--apply",
"--trace-id",
"trace:runtime-harness",
"--run-id",
"run:runtime-harness",
"--work-item-id",
"HOST112-RUNTIME-HARNESS",
],
text=True,
capture_output=True,
env=env,
check=False,
timeout=15,
)
def test_failed_general_run_cannot_be_washed_green_by_external_recovery(
tmp_path: Path,
) -> None:
script, env, fake_state = _runtime_harness(tmp_path)
env["FAKE_LIGHTDM_START_RC"] = "124"
env["FAKE_PARTIAL_LIGHTDM_START"] = "1"
failed = _apply(script, env)
failed_output = failed.stdout + failed.stderr
assert failed.returncode == 1
assert "runtime_write_performed=1" in failed_output
assert "receipt_status=written_immutable" in failed_output
assert "durable_readback_status=written_immutable" in failed_output
assert (
"durable_receipt_terminal=execution_failed_or_unattributed_"
"idempotent_already_verified_healthy"
) in failed_output
assert "guest_ready=1" in failed_output
assert "action_failure_observed=1" in failed_output
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
readback = next((tmp_path / "state" / "readbacks").glob("*.txt"))
receipt_before = receipt.read_bytes()
readback_before = readback.read_bytes()
(fake_state / "lightdm_active").touch()
env.pop("FAKE_LIGHTDM_START_RC")
env.pop("FAKE_PARTIAL_LIGHTDM_START")
replay = _apply(script, env)
replay_output = replay.stdout + replay.stderr
assert replay.returncode == 1
assert "guest_ready=1" in replay_output
assert "receipt_status=reused_immutable" in replay_output
assert "durable_readback_status=reused_immutable_verified" in replay_output
assert "terminal=immutable_receipt_replay" in replay_output
assert receipt.read_bytes() == receipt_before
assert readback.read_bytes() == readback_before
def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
tmp_path: Path,
) -> None:
noop_root = tmp_path / "noop"
noop_root.mkdir()
script, env, fake_state = _runtime_harness(noop_root, ledger_capacity=4)
(fake_state / "lightdm_active").touch()
for index in range(5):
run_env = env | {"INVOCATION_ID": f"noop-{index}"}
result = subprocess.run(
["bash", str(script), "--apply-timer"],
text=True,
capture_output=True,
env=run_env,
check=False,
timeout=15,
)
assert result.returncode == 0, result.stdout + result.stderr
state = noop_root / "state"
assert [path.relative_to(state) for path in state.rglob("*") if path.is_file()] == [
Path("timer-observation.latest")
]
assert "repeat_count=5" in (state / "timer-observation.latest").read_text(
encoding="utf-8"
)
mutation_root = tmp_path / "mutation"
mutation_root.mkdir()
script, env, _ = _runtime_harness(mutation_root, ledger_capacity=4)
env["FAKE_LIGHTDM_START_RC"] = "124"
for index in range(12):
run_env = env | {"INVOCATION_ID": f"mutation-{index}"}
result = subprocess.run(
["bash", str(script), "--apply-timer"],
text=True,
capture_output=True,
env=run_env,
check=False,
timeout=15,
)
output = result.stdout + result.stderr
assert result.returncode == 1
assert "runtime_write_performed=1" in output
assert "receipt_status=written_bounded_timer_slot" in output
assert "durable_readback_status=written_bounded_timer_slot" in output
for field in (
"durable_readback_present",
"durable_readback_identity_match",
"durable_readback_boot_match",
"durable_readback_receipt_link_match",
"durable_readback_terminal_match",
):
assert f"{field}=1" in output
assert f"{field}=0" not in output
assert "timer_no_runtime_mutation_aggregated" not in output
ledger_files = [
path
for path in (mutation_root / "state" / "timer-ledger").iterdir()
if path.is_file()
]
assert len(ledger_files) <= 8
assert not (mutation_root / "state" / "receipts").exists()
assert not (mutation_root / "state" / "readbacks").exists()
for readback in (mutation_root / "state" / "timer-ledger").glob(
"*.readback.txt"
):
stored = readback.read_text(encoding="utf-8")
for field in (
"durable_readback_present",
"durable_readback_identity_match",
"durable_readback_boot_match",
"durable_readback_receipt_link_match",
"durable_readback_terminal_match",
):
assert stored.count(f"{field}=1") == 1
assert f"{field}=0" not in stored
def test_timer_sigkill_during_receipt_commit_leaves_only_bounded_slot_temps(
tmp_path: Path,
) -> None:
script, env, _ = _runtime_harness(tmp_path, ledger_capacity=4)
env["FAKE_LIGHTDM_START_RC"] = "124"
for index in range(12):
marker = tmp_path / f"receipt-mv-{index}.entered"
run_env = env | {
"INVOCATION_ID": f"sigkill-{index}",
"FAKE_RECEIPT_MV_MARKER": str(marker),
"FAKE_RECEIPT_MV_SLEEP_SECONDS": "5",
}
process = subprocess.Popen(
["bash", str(script), "--apply-timer"],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=run_env,
start_new_session=True,
)
deadline = time.monotonic() + 8
while not marker.exists() and time.monotonic() < deadline:
time.sleep(0.01)
assert marker.exists()
os.killpg(process.pid, signal.SIGKILL)
process.communicate(timeout=5)
assert process.returncode == -signal.SIGKILL
ledger = tmp_path / "state" / "timer-ledger"
files = [path for path in ledger.iterdir() if path.is_file()]
assert len(files) <= 4
assert all(path.name.endswith(".receipt.txt.tmp") for path in files)
def test_signal_during_final_verifier_writes_correlated_failure_chain(
tmp_path: Path,
) -> None:
script, env, _ = _runtime_harness(tmp_path)
marker = tmp_path / "final-verifier.entered"
env.update(
{
"FAKE_FINALIZE_MARKER": str(marker),
"FAKE_FINALIZE_SLEEP_SECONDS": "2",
}
)
process = subprocess.Popen(
[
"bash",
str(script),
"--apply",
"--trace-id",
"trace:runtime-harness",
"--run-id",
"run:runtime-harness",
"--work-item-id",
"HOST112-RUNTIME-HARNESS",
],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
)
deadline = time.monotonic() + 8
while not marker.exists() and time.monotonic() < deadline:
time.sleep(0.02)
assert marker.exists()
os.kill(process.pid, signal.SIGTERM)
stdout, stderr = process.communicate(timeout=12)
assert process.returncode == 75, stdout + stderr
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
signal_readback = next((tmp_path / "state" / "readbacks").glob("*.signal.txt"))
receipt_text = receipt.read_text(encoding="utf-8")
signal_text = signal_readback.read_text(encoding="utf-8")
assert "terminal=interrupted_signal_TERM" in receipt_text
assert "action_count=1 actions=start_lightdm" in receipt_text
assert f"receipt_path={receipt}" in signal_text
assert "terminal=interrupted_signal_TERM" in signal_text
assert "schema_version=host112_guest_recovery_signal_readback_v1" in signal_text
replay = _apply(script, env | {"FAKE_FINALIZE_MARKER": ""})
assert replay.returncode == 65
assert "BLOCKER durable_readback_missing_or_mismatched_for_receipt" in (
replay.stdout + replay.stderr
)

1
scripts/reboot-recovery/tests/test_install_host112_guest_recovery_contract.py
View File

@@ -108,6 +108,7 @@ def test_static_gate_precedes_timer_activation_and_no_oneshot_is_started() -> No
assert 'systemctl enable --now awoooi-host112-guest-recovery.timer' not in text
assert 'systemctl start awoooi-host112-guest-recovery.service' not in text
assert "grep -Fq 'TimeoutStartSec=480'" in text
assert "for tool in bash cksum cmp flock" in text
assert "TimeoutStartSec=600" not in text
assert 'INSTALLER_SYNCHRONOUS_RECOVERY_START=0' in text
assert 'RECOVERY_EXECUTION_PERFORMED=0' in text

2
scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py
View File

@@ -108,7 +108,7 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
assert "visudo -cf" in host112_installer
assert "--rollback" in host112_installer
assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer" in host112_service
assert "TimeoutStartSec=600" in host112_service
assert "TimeoutStartSec=480" in host112_service
assert "OnBootSec=20s" in host112_timer
assert "OnUnitInactiveSec=60s" in host112_timer