wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 4 Packages Projects Releases Wiki Activity
Files
c9bb4bbfc6af05c41df7c3af546ba44cbcf45708
awoooi/scripts/reboot-recovery/host112-guest-readiness.sh
ogt c9bb4bbfc6
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 2m35s
Details
CD Pipeline / build-and-deploy (push) Failing after 7m35s
Details
CD Pipeline / post-deploy-checks (push) Has been skipped
Details
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
Details
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 16s
Details
fix(recovery): bind host112 closure artifacts
2026-07-14 12:37:28 +08:00

1246 lines
51 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# Host 112 guest readiness and bounded recovery executor.
# Version: 2.0
# Last modified: 2026-07-14 11:00 (Asia/Taipei)
# Last modified by: Codex
# Change: add correlated, cooldown-guarded Wazuh manager recovery and an
# independent process/port verifier without reboot, VM power, firewall, or
# secret access.
set -uo pipefail
MODE="check"
MODE_EXPLICIT=0
TRACE_ID=""
RUN_ID=""
WORK_ITEM_ID=""
INDEXER_RETRY_AFTER_SECONDS="${HOST112_INDEXER_RETRY_AFTER_SECONDS:-210}"
INDEXER_RETRY_COOLDOWN_SECONDS="${HOST112_INDEXER_RETRY_COOLDOWN_SECONDS:-600}"
INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-90}"
SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-15}"
MANAGER_RETRY_AFTER_SECONDS="${HOST112_MANAGER_RETRY_AFTER_SECONDS:-120}"
MANAGER_RETRY_COOLDOWN_SECONDS="${HOST112_MANAGER_RETRY_COOLDOWN_SECONDS:-600}"
MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}"
MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-90}"
MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}"
MANAGER_VERIFY_INTERVAL_SECONDS="${HOST112_MANAGER_VERIFY_INTERVAL_SECONDS:-3}"
MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-30}"
READ_TIMEOUT_SECONDS="${HOST112_READ_TIMEOUT_SECONDS:-5}"
EXECUTOR_DEADLINE_SECONDS="${HOST112_EXECUTOR_DEADLINE_SECONDS:-420}"
MANAGER_MIN_MEMORY_AVAILABLE_KB="${HOST112_MANAGER_MIN_MEMORY_AVAILABLE_KB:-524288}"
MANAGER_MIN_DISK_AVAILABLE_KB="${HOST112_MANAGER_MIN_DISK_AVAILABLE_KB:-1048576}"
MANAGER_MAX_DISK_USED_PERCENT="${HOST112_MANAGER_MAX_DISK_USED_PERCENT:-95}"
STATE_DIR="${HOST112_RECOVERY_STATE_DIR:-/var/lib/awoooi-host112-recovery}"
INDEXER_RETRY_MARKER="${HOST112_INDEXER_RETRY_MARKER:-/run/awoooi-host112-indexer-retry.epoch}"
MANAGER_RETRY_MARKER="${HOST112_MANAGER_RETRY_MARKER:-/run/awoooi-host112-manager-retry.epoch}"
MANAGER_LOCK_FILE="${HOST112_MANAGER_LOCK_FILE:-/run/awoooi-host112-manager-recovery.lock}"
usage() {
cat <<'USAGE'
Usage:
host112-guest-readiness.sh --check [identity]
host112-guest-readiness.sh --dry-run --trace-id ID --run-id ID --work-item-id ID
host112-guest-readiness.sh --apply --trace-id ID --run-id ID --work-item-id ID
Internal systemd mode:
host112-guest-readiness.sh --apply-timer
--check emits a bounded no-secret guest readiness row and never writes.
--dry-run evaluates the exact Wazuh manager preflight and never writes.
--apply performs only allowlisted idempotent recovery for graphical.target,
LightDM, open-vm-tools, SSH, Wazuh Indexer, and one cooldown-guarded Wazuh
manager reset/start attempt. The independent verifier requires the manager
process plus agent event 1514, enrollment 1515, and manager API 55000 listeners.
USAGE
}
set_mode() {
local requested="$1"
if [ "$MODE_EXPLICIT" -eq 1 ]; then
echo "BLOCKER duplicate_mode" >&2
exit 64
fi
MODE="$requested"
MODE_EXPLICIT=1
}
require_value() {
local flag="$1" value="${2:-}"
if [ -z "$value" ]; then
echo "BLOCKER missing_value=$flag" >&2
exit 64
fi
}
while [ "$#" -gt 0 ]; do
case "$1" in
--check) set_mode "check" ;;
--dry-run) set_mode "dry_run" ;;
--apply) set_mode "apply" ;;
--apply-timer) set_mode "timer_apply" ;;
--trace-id)
require_value "$1" "${2:-}"
TRACE_ID="$2"
shift
;;
--run-id)
require_value "$1" "${2:-}"
RUN_ID="$2"
shift
;;
--work-item-id)
require_value "$1" "${2:-}"
WORK_ITEM_ID="$2"
shift
;;
-h|--help) usage; exit 0 ;;
*) echo "BLOCKER unknown_argument=$1" >&2; exit 64 ;;
esac
shift
done
safe_identity() {
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$ ]]
}
identity_count=0
[ -n "$TRACE_ID" ] && identity_count=$((identity_count + 1))
[ -n "$RUN_ID" ] && identity_count=$((identity_count + 1))
[ -n "$WORK_ITEM_ID" ] && identity_count=$((identity_count + 1))
if [ "$identity_count" -ne 0 ] && [ "$identity_count" -ne 3 ]; then
echo "BLOCKER incomplete_control_identity" >&2
exit 64
fi
if [ "$identity_count" -eq 3 ]; then
for identity in "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID"; do
if ! safe_identity "$identity"; then
echo "BLOCKER unsafe_control_identity" >&2
exit 64
fi
done
fi
if { [ "$MODE" = "dry_run" ] || [ "$MODE" = "apply" ]; } && [ "$identity_count" -ne 3 ]; then
echo "BLOCKER control_identity_required" >&2
exit 64
fi
if [ "$MODE" = "timer_apply" ] && [ "$identity_count" -ne 0 ]; then
echo "BLOCKER timer_identity_is_system_generated" >&2
exit 64
fi
actions=()
runtime_write_performed=0
state_write_performed=0
manager_runtime_write_performed=0
manager_marker_write_performed=0
receipt_write_performed=0
receipt_status="not_requested"
receipt_ref="none"
receipt_path="none"
receipt_sha256="none"
receipt_reused=0
durable_receipt_present=0
durable_receipt_identity_match=0
durable_receipt_boot_match=0
durable_receipt_terminal="none"
durable_readback_status="not_requested"
durable_readback_path="none"
durable_readback_sha256="none"
durable_readback_present=0
durable_readback_identity_match=0
durable_readback_boot_match=0
durable_readback_receipt_link_match=0
durable_readback_terminal_match=0
latest_readback_pointer_written=0
immutable_replay=0
manager_attempt_count=0
manager_start_exit=not_attempted
manager_terminal="observed"
run_terminal="observed"
rollback_attempted=0
rollback_verified=0
rollback_terminal=not_required
timer_artifact_policy="not_applicable"
timer_ledger_capacity=128
timer_ledger_slot="none"
timer_observation_status="not_requested"
timer_observation_path="none"
timer_observation_sha256="none"
timer_observation_repeat_count=0
timer_observation_write_performed=0
timer_observation_failed=0
signal_readback_status="not_requested"
signal_readback_path="none"
signal_readback_sha256="none"
action_failure_observed=0
is_uint() {
[[ "$1" =~ ^[0-9]+$ ]]
}
for timing_value in \
"$INDEXER_START_TIMEOUT_SECONDS" \
"$SERVICE_START_TIMEOUT_SECONDS" \
"$MANAGER_RESET_TIMEOUT_SECONDS" \
"$MANAGER_START_TIMEOUT_SECONDS" \
"$MANAGER_VERIFY_TIMEOUT_SECONDS" \
"$MANAGER_VERIFY_INTERVAL_SECONDS" \
"$MANAGER_ROLLBACK_TIMEOUT_SECONDS" \
"$READ_TIMEOUT_SECONDS" \
"$EXECUTOR_DEADLINE_SECONDS"; do
if ! is_uint "$timing_value" || [ "$timing_value" -le 0 ]; then
echo "BLOCKER invalid_timeout_configuration" >&2
exit 64
fi
done
if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -gt "$MANAGER_VERIFY_TIMEOUT_SECONDS" ]; then
echo "BLOCKER manager_verify_interval_exceeds_timeout" >&2
exit 64
fi
APPLY_PHASE_TIMEOUT_BUDGET_SECONDS=$((
(8 * SERVICE_START_TIMEOUT_SECONDS)
+ INDEXER_START_TIMEOUT_SECONDS
+ (2 * MANAGER_RESET_TIMEOUT_SECONDS)
+ MANAGER_START_TIMEOUT_SECONDS
+ MANAGER_VERIFY_TIMEOUT_SECONDS
+ MANAGER_ROLLBACK_TIMEOUT_SECONDS
))
if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt "$EXECUTOR_DEADLINE_SECONDS" ]; then
echo "BLOCKER apply_phase_timeout_budget_exceeds_executor_deadline" >&2
exit 64
fi
EXECUTOR_DEADLINE_EPOCH=$(( $(date +%s) + EXECUTOR_DEADLINE_SECONDS ))
remaining_executor_seconds() {
local now remaining
now="$(date +%s)"
remaining=$((EXECUTOR_DEADLINE_EPOCH - now))
[ "$remaining" -gt 0 ] || return 1
printf '%s' "$remaining"
}
bounded_timeout() {
local requested="$1" remaining effective
shift
remaining="$(remaining_executor_seconds)" || return 124
effective="$requested"
[ "$effective" -le "$remaining" ] || effective="$remaining"
[ "$effective" -gt 0 ] || return 124
timeout --signal=TERM --kill-after=2 "$effective" "$@"
}
write_epoch_marker() {
local marker="$1" epoch="$2" tmp
tmp="${marker}.tmp"
if printf '%s\n' "$epoch" >"$tmp" \
&& chmod 0644 "$tmp" \
&& mv "$tmp" "$marker"; then
return 0
fi
rm -f "$tmp" >/dev/null 2>&1 || true
return 1
}
unit_active() {
local unit="$1" state
state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-active "$unit" 2>/dev/null || true)"
printf '%s' "${state:-unknown}"
}
unit_enabled() {
local unit="$1" state
state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-enabled "$unit" 2>/dev/null || true)"
printf '%s' "${state:-unknown}"
}
unit_property() {
local unit="$1" property="$2" state
state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl show "$unit" -p "$property" --value 2>/dev/null || true)"
printf '%s' "${state:-unknown}"
}
unit_loaded() {
[ "$(unit_property "$1" LoadState)" = "loaded" ]
}
record_action() {
actions+=("$1")
}
start_if_inactive() {
local unit="$1" action="$2" state
state="$(unit_active "$unit")"
if [ "$state" = "active" ] || [ "$state" = "activating" ]; then
return 0
fi
# A mutating systemctl invocation may partially change runtime state before
# returning non-zero or timing out. Record the attempt before execution so
# those paths can never be classified as no-write timer observations.
runtime_write_performed=1
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl start "$unit"; then
record_action "$action"
else
record_action "${action}_failed"
fi
}
enable_if_disabled() {
local unit="$1" action="$2" state
state="$(unit_enabled "$unit")"
if [ "$state" = "enabled" ] || [ "$state" = "static" ]; then
return 0
fi
runtime_write_performed=1
state_write_performed=1
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl enable "$unit" >/dev/null; then
record_action "$action"
else
record_action "${action}_failed"
fi
}
tcp_port_listening() {
local port="$1"
bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -ltn 2>/dev/null \
| awk '{print $4}' | grep -Eq "(^|:)$port$"
}
udp_port_listening() {
local port="$1"
bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -lun 2>/dev/null \
| awk '{print $4}' | grep -Eq "(^|:)$port$"
}
process_count() {
local process_name="$1" count
count="$(bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -xc "$process_name" 2>/dev/null || true)"
printf '%s' "${count:-0}"
}
memory_available_kb() {
local available
available="$(awk '/^MemAvailable:/ {print $2; exit}' /proc/meminfo 2>/dev/null || true)"
printf '%s' "${available:-0}"
}
disk_readback() {
local target="/var/ossec" line
[ -e "$target" ] || target="/"
line="$(bounded_timeout "$READ_TIMEOUT_SECONDS" df -Pk "$target" 2>/dev/null \
| awk 'NR==2 {gsub(/%/, "", $5); print $4 " " $5; exit}')"
if [ -n "$line" ]; then
printf '%s' "$line"
else
printf '0 100'
fi
}
capture_manager_state() {
local disk_values
manager_load_state="$(unit_property wazuh-manager.service LoadState)"
manager_active_state="$(unit_property wazuh-manager.service ActiveState)"
manager_sub_state="$(unit_property wazuh-manager.service SubState)"
manager_result="$(unit_property wazuh-manager.service Result)"
manager_failed_state_observed=0
if [ "$manager_active_state" = "failed" ] \
|| { [ "$manager_result" != "success" ] && [ "$manager_result" != "unknown" ]; }; then
manager_failed_state_observed=1
fi
analysisd_count="$(process_count wazuh-analysisd)"
event_1514_tcp=0
event_1514_udp=0
enrollment_1515_tcp=0
manager_api_55000_tcp=0
tcp_port_listening 1514 && event_1514_tcp=1
udp_port_listening 1514 && event_1514_udp=1
tcp_port_listening 1515 && enrollment_1515_tcp=1
tcp_port_listening 55000 && manager_api_55000_tcp=1
manager_event_port_ready=0
if [ "$event_1514_tcp" -eq 1 ] || [ "$event_1514_udp" -eq 1 ]; then
manager_event_port_ready=1
fi
manager_orphan_analysisd=0
if [ "$manager_active_state" != "active" ] && is_uint "$analysisd_count" && [ "$analysisd_count" -gt 0 ]; then
manager_orphan_analysisd=1
fi
manager_verifier_ready=0
if [ "$manager_active_state" = "active" ] \
&& is_uint "$analysisd_count" && [ "$analysisd_count" -gt 0 ] \
&& [ "$manager_event_port_ready" -eq 1 ] \
&& [ "$enrollment_1515_tcp" -eq 1 ] \
&& [ "$manager_api_55000_tcp" -eq 1 ]; then
manager_verifier_ready=1
fi
manager_memory_available_kb="$(memory_available_kb)"
disk_values="$(disk_readback)"
read -r manager_disk_available_kb manager_disk_used_percent <<<"$disk_values"
manager_resource_ready=0
if is_uint "$manager_memory_available_kb" \
&& is_uint "$manager_disk_available_kb" \
&& is_uint "$manager_disk_used_percent" \
&& [ "$manager_memory_available_kb" -ge "$MANAGER_MIN_MEMORY_AVAILABLE_KB" ] \
&& [ "$manager_disk_available_kb" -ge "$MANAGER_MIN_DISK_AVAILABLE_KB" ] \
&& [ "$manager_disk_used_percent" -lt "$MANAGER_MAX_DISK_USED_PERCENT" ]; then
manager_resource_ready=1
fi
}
manager_cooldown_remaining() {
local now_epoch="$1" last_retry_epoch=0 elapsed
if [ -r "$MANAGER_RETRY_MARKER" ]; then
read -r last_retry_epoch <"$MANAGER_RETRY_MARKER" || last_retry_epoch=0
fi
is_uint "$last_retry_epoch" || last_retry_epoch=0
elapsed=$((now_epoch - last_retry_epoch))
if [ "$elapsed" -lt 0 ]; then
printf '%s' "$MANAGER_RETRY_COOLDOWN_SECONDS"
elif [ "$elapsed" -lt "$MANAGER_RETRY_COOLDOWN_SECONDS" ]; then
printf '%s' "$((MANAGER_RETRY_COOLDOWN_SECONDS - elapsed))"
else
printf '0'
fi
}
evaluate_manager_preflight() {
local now_epoch="$1" uptime_for_retry="$2"
manager_cooldown_remaining_seconds="$(manager_cooldown_remaining "$now_epoch")"
manager_preflight_status="ready"
if ! command -v systemctl >/dev/null 2>&1 \
|| ! command -v timeout >/dev/null 2>&1 \
|| ! command -v ss >/dev/null 2>&1 \
|| ! command -v flock >/dev/null 2>&1; then
manager_preflight_status="blocked_required_tool_missing"
elif [ "$manager_load_state" != "loaded" ]; then
manager_preflight_status="blocked_manager_unit_not_loaded"
elif [ "$manager_verifier_ready" -eq 1 ]; then
manager_preflight_status="already_verified_healthy"
elif [ "$manager_active_state" = "active" ]; then
manager_preflight_status="blocked_active_verifier_mismatch"
elif [ "$manager_orphan_analysisd" -eq 1 ]; then
manager_preflight_status="blocked_orphan_analysisd_no_write"
elif [ "$manager_resource_ready" -ne 1 ]; then
manager_preflight_status="blocked_resource_preflight_no_write"
elif [ "$(unit_active wazuh-indexer.service)" != "active" ]; then
manager_preflight_status="blocked_indexer_dependency_no_write"
elif ! is_uint "$uptime_for_retry" || [ "$uptime_for_retry" -lt "$MANAGER_RETRY_AFTER_SECONDS" ]; then
manager_preflight_status="blocked_boot_settle_no_write"
elif [ "$manager_cooldown_remaining_seconds" -gt 0 ]; then
manager_preflight_status="blocked_retry_cooldown_no_write"
fi
}
wait_for_manager_verifier() {
local deadline now remaining
deadline=$(( $(date +%s) + MANAGER_VERIFY_TIMEOUT_SECONDS ))
while true; do
capture_manager_state
[ "$manager_verifier_ready" -eq 1 ] && return 0
now="$(date +%s)"
[ "$now" -ge "$deadline" ] && return 1
remaining="$((deadline - now))"
[ "$remaining" -gt 0 ] || return 1
if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -lt "$remaining" ]; then
sleep "$MANAGER_VERIFY_INTERVAL_SECONDS"
else
sleep "$remaining"
fi
done
}
rollback_manager_to_inactive() {
rollback_attempted=1
rollback_terminal="bounded_stop_attempted"
if bounded_timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service; then
record_action "rollback_stop_wazuh_manager"
else
record_action "rollback_stop_wazuh_manager_failed"
fi
capture_manager_state
if [ "$manager_active_state" != "active" ] \
&& is_uint "$analysisd_count" && [ "$analysisd_count" -eq 0 ] \
&& [ "$manager_event_port_ready" -eq 0 ] \
&& [ "$enrollment_1515_tcp" -eq 0 ] \
&& [ "$manager_api_55000_tcp" -eq 0 ]; then
rollback_verified=1
rollback_terminal="bounded_stop_verified"
else
rollback_terminal="bounded_stop_not_verified"
fi
}
receipt_name_for_identity() {
local identity_hash safe_run
identity_hash="$(printf '%s\0%s\0%s\0' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" | sha256sum | awk '{print $1}')"
safe_run="$(printf '%s' "$RUN_ID" | tr -c 'A-Za-z0-9._+-' '_')"
safe_run="${safe_run:0:48}"
[ -n "$safe_run" ] || safe_run="unknown"
printf '%s-%s' "$safe_run" "$identity_hash"
}
receipt_token() {
local path="$1" key="$2"
awk -v prefix="${key}=" '
{
for (i = 1; i <= NF; i++) {
if (index($i, prefix) == 1) {
print substr($i, length(prefix) + 1)
exit
}
}
}
' "$path" 2>/dev/null
}
current_action_text() {
if [ "${#actions[@]}" -gt 0 ]; then
IFS=,; printf '%s' "${actions[*]}"
else
printf 'none'
fi
}
timer_slot_for_identity() {
local checksum
checksum="$(printf '%s\0%s\0%s\0' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" | cksum | awk '{print $1}')"
is_uint "$checksum" || return 1
printf '%03d' "$((checksum % timer_ledger_capacity))"
}
load_durable_receipt() {
local receipt_name stored_trace stored_run stored_work_item stored_boot
receipt_name="$(receipt_name_for_identity)"
receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
[ -f "$receipt_path" ] || return 1
durable_receipt_present=1
stored_trace="$(receipt_token "$receipt_path" trace_id)"
stored_run="$(receipt_token "$receipt_path" run_id)"
stored_work_item="$(receipt_token "$receipt_path" work_item_id)"
stored_boot="$(receipt_token "$receipt_path" boot_id)"
durable_receipt_terminal="$(receipt_token "$receipt_path" terminal)"
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
receipt_ref="host112-manager-recovery:${receipt_name}"
if [ "$stored_trace" = "$TRACE_ID" ] \
&& [ "$stored_run" = "$RUN_ID" ] \
&& [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then
durable_receipt_identity_match=1
fi
if [ "$stored_boot" = "$boot_id" ]; then
durable_receipt_boot_match=1
fi
if [ "$durable_receipt_identity_match" -ne 1 ] \
|| [ "$durable_receipt_boot_match" -ne 1 ]; then
receipt_status="immutable_identity_or_boot_mismatch"
return 65
fi
return 0
}
load_durable_readback() {
local receipt_name stored_trace stored_run stored_work_item stored_boot
local stored_receipt_path stored_receipt_sha stored_terminal
receipt_name="$(receipt_name_for_identity)"
durable_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt"
[ -f "$durable_readback_path" ] || {
durable_readback_status="missing_for_durable_receipt"
return 1
}
durable_readback_present=1
stored_trace="$(receipt_token "$durable_readback_path" trace_id)"
stored_run="$(receipt_token "$durable_readback_path" run_id)"
stored_work_item="$(receipt_token "$durable_readback_path" work_item_id)"
stored_boot="$(receipt_token "$durable_readback_path" boot_id)"
stored_receipt_path="$(receipt_token "$durable_readback_path" receipt_path)"
stored_receipt_sha="$(receipt_token "$durable_readback_path" receipt_sha256)"
stored_terminal="$(receipt_token "$durable_readback_path" durable_receipt_terminal)"
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
if [ "$stored_trace" = "$TRACE_ID" ] \
&& [ "$stored_run" = "$RUN_ID" ] \
&& [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then
durable_readback_identity_match=1
fi
if [ "$stored_boot" = "$boot_id" ]; then
durable_readback_boot_match=1
fi
if [ "$stored_receipt_path" = "$receipt_path" ] \
&& [ "$stored_receipt_sha" = "$receipt_sha256" ]; then
durable_readback_receipt_link_match=1
fi
if [ "$stored_terminal" = "$durable_receipt_terminal" ]; then
durable_readback_terminal_match=1
fi
if [ "$durable_readback_identity_match" -ne 1 ] \
|| [ "$durable_readback_boot_match" -ne 1 ] \
|| [ "$durable_readback_receipt_link_match" -ne 1 ] \
|| [ "$durable_readback_terminal_match" -ne 1 ]; then
durable_readback_status="immutable_identity_boot_receipt_or_terminal_mismatch"
return 65
fi
durable_readback_status="loaded_immutable_verified"
return 0
}
write_public_receipt() {
local receipt_name tmp generated_at receipt_line load_status receipt_action_text timer_slot
if [ "$MODE" != "timer_apply" ]; then
if load_durable_receipt; then
receipt_status="reused_immutable"
receipt_reused=1
return 0
else
load_status=$?
if [ "$load_status" -eq 65 ]; then
return 1
fi
fi
fi
if [ "$MODE" = "timer_apply" ]; then
timer_slot="$(timer_slot_for_identity)" || {
receipt_status="write_failed"
return 1
}
timer_ledger_slot="$timer_slot"
receipt_name="timer-slot-${timer_slot}"
receipt_path="$STATE_DIR/timer-ledger/${receipt_name}.receipt.txt"
timer_artifact_policy="bounded_ring_128_for_runtime_mutation"
else
receipt_name="$(receipt_name_for_identity)"
receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
fi
generated_at="$(date --iso-8601=seconds)"
receipt_action_text="$(current_action_text)"
receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v2 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed action_count=${#actions[@]} actions=$receipt_action_text rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal manager_terminal=$manager_terminal terminal=$run_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
if install -d -m 0750 "$(dirname "$receipt_path")"; then
tmp="${receipt_path}.tmp"
if [ "$MODE" = "timer_apply" ] \
&& printf '%s\n' "$receipt_line" >"$tmp" \
&& chmod 0640 "$tmp" \
&& mv "$tmp" "$receipt_path"; then
receipt_write_performed=1
receipt_status="written_bounded_timer_slot"
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
receipt_ref="host112-timer-ledger:${receipt_name}"
durable_receipt_present=1
durable_receipt_identity_match=1
durable_receipt_boot_match=1
durable_receipt_terminal="$run_terminal"
return 0
elif [ "$MODE" != "timer_apply" ] \
&& printf '%s\n' "$receipt_line" >"$tmp" \
&& chmod 0640 "$tmp" \
&& ln "$tmp" "$receipt_path" 2>/dev/null; then
rm -f "$tmp"
receipt_write_performed=1
receipt_status="written_immutable"
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
receipt_ref="host112-manager-recovery:${receipt_name}"
durable_receipt_present=1
durable_receipt_identity_match=1
durable_receipt_boot_match=1
durable_receipt_terminal="$run_terminal"
return 0
fi
rm -f "$tmp" >/dev/null 2>&1 || true
if [ "$MODE" != "timer_apply" ] && load_durable_receipt; then
receipt_status="reused_immutable"
receipt_reused=1
return 0
fi
fi
receipt_status="write_failed"
return 1
}
write_signal_readback() {
local receipt_name tmp signal_line signal_action_text timer_slot
if [ "$MODE" = "timer_apply" ]; then
timer_slot="$(timer_slot_for_identity)" || {
signal_readback_status="write_failed"
return 1
}
timer_ledger_slot="$timer_slot"
receipt_name="timer-slot-${timer_slot}"
signal_readback_path="$STATE_DIR/timer-ledger/${receipt_name}.readback.txt"
else
receipt_name="$(receipt_name_for_identity)"
signal_readback_path="$STATE_DIR/readbacks/${receipt_name}.signal.txt"
fi
signal_action_text="$(current_action_text)"
signal_line="schema_version=host112_guest_recovery_signal_readback_v1 generated_at=$(date --iso-8601=seconds) mode=$MODE trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id manager_terminal=$manager_terminal terminal=$run_terminal manager_active=$manager_active_state manager_result=$manager_result manager_orphan_analysisd=$manager_orphan_analysisd manager_verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed action_count=${#actions[@]} actions=$signal_action_text rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal receipt_path=$receipt_path receipt_sha256=$receipt_sha256 prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
install -d -m 0750 "$(dirname "$signal_readback_path")" || {
signal_readback_status="write_failed"
return 1
}
if [ "$MODE" != "timer_apply" ] && [ -f "$signal_readback_path" ]; then
if [ "$(receipt_token "$signal_readback_path" trace_id)" != "$TRACE_ID" ] \
|| [ "$(receipt_token "$signal_readback_path" run_id)" != "$RUN_ID" ] \
|| [ "$(receipt_token "$signal_readback_path" work_item_id)" != "$WORK_ITEM_ID" ] \
|| [ "$(receipt_token "$signal_readback_path" boot_id)" != "$boot_id" ] \
|| [ "$(receipt_token "$signal_readback_path" receipt_path)" != "$receipt_path" ] \
|| [ "$(receipt_token "$signal_readback_path" receipt_sha256)" != "$receipt_sha256" ]; then
signal_readback_status="immutable_identity_boot_or_receipt_mismatch"
return 1
fi
signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')"
signal_readback_status="reused_immutable"
return 0
fi
tmp="${signal_readback_path}.tmp"
if [ "$MODE" = "timer_apply" ] \
&& printf '%s\n' "$signal_line" >"$tmp" \
&& chmod 0640 "$tmp" \
&& mv "$tmp" "$signal_readback_path"; then
signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')"
signal_readback_status="written_bounded_timer_slot"
return 0
elif [ "$MODE" != "timer_apply" ] \
&& printf '%s\n' "$signal_line" >"$tmp" \
&& chmod 0640 "$tmp" \
&& ln "$tmp" "$signal_readback_path" 2>/dev/null; then
rm -f "$tmp"
signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')"
signal_readback_status="written_immutable"
return 0
fi
rm -f "$tmp" >/dev/null 2>&1 || true
signal_readback_status="write_failed"
return 1
}
write_bounded_timer_mutation_readback() {
local timer_slot tmp
timer_slot="$(timer_slot_for_identity)" || {
durable_readback_status="write_failed"
return 1
}
timer_ledger_slot="$timer_slot"
durable_readback_path="$STATE_DIR/timer-ledger/timer-slot-${timer_slot}.readback.txt"
install -d -m 0750 "$(dirname "$durable_readback_path")" || {
durable_readback_status="write_failed"
return 1
}
tmp="${durable_readback_path}.tmp"
if printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s timer_ledger_slot=%s timer_ledger_capacity=%s retention=rolling_atomic_overwrite durable_readback_status=written_bounded_timer_slot durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \
"$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" \
"$timer_ledger_slot" "$timer_ledger_capacity" "$durable_readback_path" >"$tmp" \
&& chmod 0640 "$tmp" \
&& mv "$tmp" "$durable_readback_path"; then
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
durable_readback_status="written_bounded_timer_slot"
durable_readback_present=1
durable_readback_identity_match=1
durable_readback_boot_match=1
durable_readback_receipt_link_match=1
durable_readback_terminal_match=1
return 0
fi
rm -f "$tmp" >/dev/null 2>&1 || true
durable_readback_status="write_failed"
return 1
}
write_bounded_timer_observation() {
local tmp now previous_fingerprint previous_repeat previous_first fingerprint first_at observation_line
timer_observation_path="$STATE_DIR/timer-observation.latest"
now="$(date --iso-8601=seconds)"
fingerprint="$(printf '%s\0%s\0%s\0%s\0%s\0%s\0%s\0%s\0' \
"$boot_id" "$guest_ready" "$run_terminal" "$systemd_state" \
"$console_ready" "$services_ready" "$ssh_ready" "$recovery_timer_ready" \
| sha256sum | awk '{print $1}')"
previous_fingerprint=""
previous_repeat=0
previous_first="$now"
if [ -r "$timer_observation_path" ]; then
previous_fingerprint="$(receipt_token "$timer_observation_path" fingerprint)"
previous_repeat="$(receipt_token "$timer_observation_path" repeat_count)"
previous_first="$(receipt_token "$timer_observation_path" first_observed_at)"
fi
is_uint "$previous_repeat" || previous_repeat=0
if [ "$previous_fingerprint" = "$fingerprint" ]; then
timer_observation_repeat_count=$((previous_repeat + 1))
first_at="${previous_first:-$now}"
else
timer_observation_repeat_count=1
first_at="$now"
fi
observation_line="schema_version=host112_timer_observation_aggregate_v1 first_observed_at=$first_at last_observed_at=$now repeat_count=$timer_observation_repeat_count fingerprint=$fingerprint boot_id=$boot_id guest_ready=$guest_ready manager_terminal=$manager_terminal terminal=$run_terminal systemd_state=$systemd_state console_ready=$console_ready services_ready=$services_ready ssh_ready=$ssh_ready recovery_timer_ready=$recovery_timer_ready runtime_write_performed=0 immutable_artifact_written=0 aggregation=fixed_single_atomic_pointer"
install -d -m 0750 "$STATE_DIR" || {
timer_observation_status="write_failed"
return 1
}
tmp="${timer_observation_path}.tmp"
if printf '%s\n' "$observation_line" >"$tmp" \
&& chmod 0640 "$tmp" \
&& mv "$tmp" "$timer_observation_path"; then
timer_observation_sha256="$(sha256sum "$timer_observation_path" | awk '{print $1}')"
timer_observation_status="updated_bounded_aggregate"
timer_observation_write_performed=1
return 0
fi
rm -f "$tmp" >/dev/null 2>&1 || true
timer_observation_status="write_failed"
return 1
}
write_immutable_readback() {
local receipt_name tmp stored_trace stored_run stored_work_item stored_boot latest_tmp
receipt_name="$(receipt_name_for_identity)"
durable_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt"
if [ -f "$durable_readback_path" ]; then
stored_trace="$(receipt_token "$durable_readback_path" trace_id)"
stored_run="$(receipt_token "$durable_readback_path" run_id)"
stored_work_item="$(receipt_token "$durable_readback_path" work_item_id)"
stored_boot="$(receipt_token "$durable_readback_path" boot_id)"
if [ "$stored_trace" != "$TRACE_ID" ] \
|| [ "$stored_run" != "$RUN_ID" ] \
|| [ "$stored_work_item" != "$WORK_ITEM_ID" ] \
|| [ "$stored_boot" != "$boot_id" ]; then
durable_readback_status="immutable_identity_or_boot_mismatch"
return 1
fi
if load_durable_readback; then
durable_readback_status="reused_immutable_verified"
return 0
fi
return 1
fi
install -d -m 0750 "$(dirname "$durable_readback_path")" || return 1
tmp="${durable_readback_path}.tmp"
if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s durable_readback_status=written_immutable durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \
"$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" "$durable_readback_path" >"$tmp" \
|| ! chmod 0640 "$tmp" \
|| ! ln "$tmp" "$durable_readback_path" 2>/dev/null; then
rm -f "$tmp" >/dev/null 2>&1 || true
durable_readback_status="write_failed"
return 1
fi
rm -f "$tmp"
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
durable_readback_status="written_immutable"
durable_readback_present=1
durable_readback_identity_match=1
durable_readback_boot_match=1
durable_readback_receipt_link_match=1
durable_readback_terminal_match=1
# Preserve the historical convenience readback as an explicitly mutable
# pointer. It is never used as execution proof; the immutable path/hash is.
latest_tmp="${STATE_DIR}/last-readback.txt.tmp"
if printf 'immutable_path=%s immutable_sha256=%s\n' \
"$durable_readback_path" "$durable_readback_sha256" >"$latest_tmp" \
&& chmod 0640 "$latest_tmp" \
&& mv "$latest_tmp" "${STATE_DIR}/last-readback.txt"; then
latest_readback_pointer_written=1
return 0
fi
rm -f "$latest_tmp" >/dev/null 2>&1 || true
durable_readback_status="immutable_written_latest_pointer_failed"
return 1
}
boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)"
uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)"
if [ "$MODE" = "timer_apply" ]; then
invocation_identity="${INVOCATION_ID:-${boot_id}-$(date +%s)}"
invocation_identity="$(printf '%s' "$invocation_identity" | tr -c 'A-Za-z0-9._+-' '-')"
TRACE_ID="host112-timer:$invocation_identity"
RUN_ID="host112-timer:$invocation_identity"
WORK_ITEM_ID="HOST112-WAZUH-MANAGER-RECOVERY"
fi
if ! command -v sha256sum >/dev/null 2>&1 \
|| ! command -v cksum >/dev/null 2>&1; then
echo "BLOCKER sha256sum_and_cksum_required" >&2
exit 69
fi
if load_durable_receipt; then
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
immutable_replay=1
receipt_status="reused_immutable"
receipt_reused=1
manager_terminal="immutable_receipt_replay"
fi
else
load_status=$?
if [ "$load_status" -eq 65 ]; then
echo "BLOCKER immutable_receipt_identity_or_boot_mismatch" >&2
exit 65
fi
fi
if [ "$durable_receipt_present" -eq 1 ]; then
if ! load_durable_readback; then
echo "BLOCKER durable_readback_missing_or_mismatched_for_receipt" >&2
exit 65
fi
fi
# Capture a pre-mutation baseline so a TERM/INT/HUP receipt never relies on
# unset state, even when interruption lands during the general guest repair.
capture_manager_state
before_manager_active="$manager_active_state"
before_manager_result="$manager_result"
before_manager_orphan="$manager_orphan_analysisd"
before_manager_memory="$manager_memory_available_kb"
before_manager_disk="$manager_disk_available_kb"
before_manager_disk_used="$manager_disk_used_percent"
before_event_1514_ready="$manager_event_port_ready"
before_enrollment_1515="$enrollment_1515_tcp"
before_manager_api_55000="$manager_api_55000_tcp"
now_epoch="$(date +%s)"
evaluate_manager_preflight "$now_epoch" "$uptime_seconds"
handle_executor_signal() {
local signal_name="$1"
trap - TERM INT HUP
manager_terminal="interrupted_signal_${signal_name}"
run_terminal="interrupted_signal_${signal_name}"
manager_start_exit="interrupted"
if [ "$manager_runtime_write_performed" -eq 1 ]; then
rollback_manager_to_inactive || true
fi
capture_manager_state || true
if [ "$runtime_write_performed" -eq 1 ]; then
if write_public_receipt && write_signal_readback; then
exit 75
fi
exit 74
fi
exit 75
}
if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
&& [ "$immutable_replay" -eq 0 ]; then
if [ "${EUID:-$(id -u)}" -ne 0 ]; then
echo "BLOCKER host112_apply_requires_root" >&2
exit 77
fi
exec 9>"$MANAGER_LOCK_FILE"
if ! flock -n 9; then
echo "BLOCKER host112_manager_single_flight_busy" >&2
exit 75
fi
trap 'handle_executor_signal TERM' TERM
trap 'handle_executor_signal INT' INT
trap 'handle_executor_signal HUP' HUP
current_default="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)"
if [ "$current_default" != "graphical.target" ]; then
runtime_write_performed=1
state_write_performed=1
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl set-default graphical.target >/dev/null; then
record_action "set_default_graphical_target"
else
record_action "set_default_graphical_target_failed"
fi
fi
start_if_inactive graphical.target start_graphical_target
start_if_inactive open-vm-tools.service start_vmware_tools
start_if_inactive lightdm.service start_lightdm
if unit_loaded ssh.service; then
enable_if_disabled ssh.service enable_ssh
start_if_inactive ssh.service start_ssh
else
record_action "ssh_service_missing"
fi
if unit_loaded awoooi-host112-guest-recovery.timer; then
enable_if_disabled awoooi-host112-guest-recovery.timer enable_recovery_timer
start_if_inactive awoooi-host112-guest-recovery.timer start_recovery_timer
else
record_action "recovery_timer_missing"
fi
indexer_state="$(unit_active wazuh-indexer.service)"
if unit_loaded wazuh-indexer.service \
&& [ "$indexer_state" != "active" ] \
&& [ "$indexer_state" != "activating" ] \
&& is_uint "$uptime_seconds" \
&& [ "$uptime_seconds" -ge "$INDEXER_RETRY_AFTER_SECONDS" ]; then
now_epoch="$(date +%s)"
last_retry_epoch=0
if [ -r "$INDEXER_RETRY_MARKER" ]; then
read -r last_retry_epoch <"$INDEXER_RETRY_MARKER" || last_retry_epoch=0
fi
is_uint "$last_retry_epoch" || last_retry_epoch=0
if [ $((now_epoch - last_retry_epoch)) -ge "$INDEXER_RETRY_COOLDOWN_SECONDS" ]; then
if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then
runtime_write_performed=1
state_write_performed=1
bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" \
systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true
if bounded_timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then
record_action "retry_wazuh_indexer"
else
record_action "retry_wazuh_indexer_failed"
fi
else
record_action "wazuh_indexer_marker_write_failed_no_write"
fi
else
record_action "wazuh_indexer_retry_cooldown"
fi
fi
fi
capture_manager_state
now_epoch="$(date +%s)"
evaluate_manager_preflight "$now_epoch" "$uptime_seconds"
if [ "$MODE" = "dry_run" ]; then
case "$manager_preflight_status" in
ready) manager_terminal="dry_run_ready" ;;
already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;;
*) manager_terminal="dry_run_${manager_preflight_status}" ;;
esac
elif { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
&& [ "$immutable_replay" -eq 1 ]; then
manager_terminal="immutable_receipt_replay"
elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
case "$manager_preflight_status" in
already_verified_healthy)
manager_terminal="idempotent_already_verified_healthy"
;;
ready)
if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then
# The cooldown marker is a durable runtime state mutation even if the
# following systemctl operation fails. Account for it immediately so
# no failure path can emit a false no-write receipt.
runtime_write_performed=1
state_write_performed=1
manager_marker_write_performed=1
manager_attempt_count=1
if bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then
record_action "reset_failed_wazuh_manager"
runtime_write_performed=1
manager_runtime_write_performed=1
if bounded_timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then
manager_start_exit=0
else
manager_start_exit=$?
fi
if wait_for_manager_verifier; then
if [ "$manager_start_exit" = "0" ]; then
record_action "start_wazuh_manager_verified"
manager_terminal="verified_healthy"
else
record_action "wazuh_manager_verifier_healthy_after_nonzero_start_unattributed"
manager_terminal="manager_start_nonzero_verifier_healthy_unattributed"
fi
else
record_action "start_wazuh_manager_verifier_failed"
rollback_manager_to_inactive
if [ "$rollback_verified" -eq 1 ]; then
manager_terminal="rolled_back_verifier_failed"
else
manager_terminal="rollback_failed_degraded"
fi
fi
else
manager_start_exit="reset_failed_no_start"
record_action "reset_failed_wazuh_manager_failed_no_start"
manager_terminal="no_start_reset_failed"
fi
else
record_action "wazuh_manager_marker_write_failed_no_write"
manager_terminal="no_write_cooldown_marker_write_failed"
fi
;;
*)
manager_terminal="no_write_${manager_preflight_status}"
;;
esac
capture_manager_state
fi
systemd_state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-system-running 2>/dev/null || true)"
systemd_state="${systemd_state:-unknown}"
default_target="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)"
default_target="${default_target:-unknown}"
graphical_target="$(unit_active graphical.target)"
display_manager="$(unit_active display-manager.service)"
lightdm="$(unit_active lightdm.service)"
vmware_tools="$(unit_active open-vm-tools.service)"
wazuh_indexer="$(unit_active wazuh-indexer.service)"
wazuh_manager="$(unit_active wazuh-manager.service)"
wazuh_dashboard="$(unit_active wazuh-dashboard.service)"
filebeat="$(unit_active filebeat.service)"
ssh_service="$(unit_active ssh.service)"
ssh_enabled="$(unit_enabled ssh.service)"
startup_enabled="$(unit_enabled awoooi-host112-guest-recovery.timer)"
startup_active="$(unit_active awoooi-host112-guest-recovery.timer)"
ssh_port_listening=0
tcp_port_listening 22 && ssh_port_listening=1
ssh_ready=0
if [ "$ssh_service" = "active" ] \
&& { [ "$ssh_enabled" = "enabled" ] || [ "$ssh_enabled" = "static" ]; } \
&& [ "$ssh_port_listening" -eq 1 ]; then
ssh_ready=1
fi
xorg="inactive"
if bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -x Xorg >/dev/null 2>&1 \
|| bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -x Xwayland >/dev/null 2>&1; then
xorg="active"
fi
console_ready=0
if [ "$default_target" = "graphical.target" ] \
&& [ "$graphical_target" = "active" ] \
&& [ "$display_manager" = "active" ] \
&& [ "$lightdm" = "active" ] \
&& [ "$vmware_tools" = "active" ] \
&& [ "$xorg" = "active" ]; then
console_ready=1
fi
services_ready=0
if [ "$wazuh_indexer" = "active" ] \
&& [ "$wazuh_dashboard" = "active" ] \
&& [ "$filebeat" = "active" ] \
&& [ "$manager_verifier_ready" -eq 1 ]; then
services_ready=1
fi
recovery_timer_ready=0
if [ "$startup_enabled" = "enabled" ] && [ "$startup_active" = "active" ]; then
recovery_timer_ready=1
fi
guest_ready=0
if [ "$console_ready" -eq 1 ] \
&& [ "$services_ready" -eq 1 ] \
&& [ "$ssh_ready" -eq 1 ] \
&& [ "$recovery_timer_ready" -eq 1 ] \
&& [ "$systemd_state" = "running" ]; then
guest_ready=1
fi
if [ "${#actions[@]}" -gt 0 ]; then
for action in "${actions[@]}"; do
case "$action" in
*_failed|*_failed_no_start|*_failed_no_write|*_unattributed|*_missing) action_failure_observed=1 ;;
esac
done
fi
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
if [ "$immutable_replay" -eq 1 ]; then
run_terminal="immutable_receipt_replay"
elif [ "$guest_ready" -eq 1 ] \
&& [ "$action_failure_observed" -eq 0 ] \
&& { [ "$manager_terminal" = "verified_healthy" ] \
|| [ "$manager_terminal" = "idempotent_already_verified_healthy" ]; }; then
if [ "$runtime_write_performed" -eq 1 ]; then
run_terminal="verified_healthy"
else
run_terminal="idempotent_already_verified_healthy"
fi
elif [ "$action_failure_observed" -eq 1 ]; then
run_terminal="execution_failed_or_unattributed_${manager_terminal}"
else
run_terminal="guest_verification_failed_${manager_terminal}"
fi
fi
action_text="none"
if [ "${#actions[@]}" -gt 0 ]; then
action_text="$(IFS=,; printf '%s' "${actions[*]}")"
fi
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
# Keep signal handling active through the complete guest verifier above.
# Ignore TERM/INT/HUP only for the short atomic receipt/readback commit
# section so a successful receipt can never be separated from its readback.
trap '' TERM INT HUP
if [ "$immutable_replay" -eq 0 ]; then
if [ "$MODE" = "apply" ] || [ "$runtime_write_performed" -eq 1 ]; then
if [ "$MODE" = "timer_apply" ]; then
timer_artifact_policy="bounded_ring_128_for_runtime_mutation"
fi
if ! write_public_receipt; then
run_terminal="${run_terminal}_receipt_failed"
fi
else
receipt_status="timer_no_runtime_mutation_aggregated"
timer_artifact_policy="bounded_mutable_observation_no_immutable_artifact"
fi
fi
if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then
if ! write_bounded_timer_observation; then
timer_observation_failed=1
fi
durable_readback_status="skipped_bounded_timer_no_runtime_mutation"
fi
fi
trace_output="${TRACE_ID:-none}"
run_output="${RUN_ID:-none}"
work_item_output="${WORK_ITEM_ID:-none}"
readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_terminal=$manager_terminal run_terminal=$run_terminal terminal=$run_terminal action_failure_observed=$action_failure_observed manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal durable_readback_present=$durable_readback_present durable_readback_identity_match=$durable_readback_identity_match durable_readback_boot_match=$durable_readback_boot_match durable_readback_receipt_link_match=$durable_readback_receipt_link_match durable_readback_terminal_match=$durable_readback_terminal_match rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal timer_artifact_policy=$timer_artifact_policy timer_ledger_capacity=$timer_ledger_capacity timer_ledger_slot=$timer_ledger_slot timer_observation_status=$timer_observation_status timer_observation_path=$timer_observation_path timer_observation_sha256=$timer_observation_sha256 timer_observation_repeat_count=$timer_observation_repeat_count timer_observation_write_performed=$timer_observation_write_performed signal_readback_status=$signal_readback_status signal_readback_path=$signal_readback_path signal_readback_sha256=$signal_readback_sha256 action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
canonical_readback_base() {
printf '%s\n' "$readback" | awk '
{
separator = ""
for (i = 1; i <= NF; i++) {
if ($i ~ /^durable_readback_(present|identity_match|boot_match|receipt_link_match|terminal_match)=/) {
continue
}
printf "%s%s", separator, $i
separator = " "
}
printf "\n"
}
'
}
emit_readback() {
printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s durable_readback_present=%s durable_readback_identity_match=%s durable_readback_boot_match=%s durable_readback_receipt_link_match=%s durable_readback_terminal_match=%s latest_readback_pointer_written=%s\n' \
"$(canonical_readback_base)" "$durable_readback_status" "$durable_readback_path" \
"$durable_readback_sha256" "$durable_readback_present" \
"$durable_readback_identity_match" "$durable_readback_boot_match" \
"$durable_readback_receipt_link_match" "$durable_readback_terminal_match" \
"$latest_readback_pointer_written"
}
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then
if [ "$timer_observation_failed" -eq 1 ]; then
emit_readback
exit 74
fi
elif [ "$MODE" = "timer_apply" ]; then
if ! write_bounded_timer_mutation_readback; then
emit_readback
exit 74
fi
elif ! write_immutable_readback; then
emit_readback
exit 74
fi
fi
emit_readback
if [ "$MODE" = "dry_run" ]; then
case "$manager_preflight_status" in
ready|already_verified_healthy) exit 0 ;;
*) exit 1 ;;
esac
fi
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
successful_terminal="$run_terminal"
if [ "$immutable_replay" -eq 1 ]; then
successful_terminal="$durable_receipt_terminal"
fi
[ "$guest_ready" -eq 1 ] \
&& [ "$receipt_status" != "write_failed" ] \
&& { [ "$successful_terminal" = "verified_healthy" ] \
|| [ "$successful_terminal" = "idempotent_already_verified_healthy" ]; }
exit $?
fi
[ "$guest_ready" -eq 1 ]
Reference in New Issue View Git Blame Copy Permalink