c9bb4bbfc6
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m35s
CD Pipeline / build-and-deploy (push) Failing after 7m35s
CD Pipeline / post-deploy-checks (push) Has been skipped
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 16s
1246 lines
51 KiB
Bash
Executable File
1246 lines
51 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Host 112 guest readiness and bounded recovery executor.
|
|
# Version: 2.0
|
|
# Last modified: 2026-07-14 11:00 (Asia/Taipei)
|
|
# Last modified by: Codex
|
|
# Change: add correlated, cooldown-guarded Wazuh manager recovery and an
|
|
# independent process/port verifier without reboot, VM power, firewall, or
|
|
# secret access.
|
|
|
|
set -uo pipefail
|
|
|
|
MODE="check"
|
|
MODE_EXPLICIT=0
|
|
TRACE_ID=""
|
|
RUN_ID=""
|
|
WORK_ITEM_ID=""
|
|
|
|
INDEXER_RETRY_AFTER_SECONDS="${HOST112_INDEXER_RETRY_AFTER_SECONDS:-210}"
|
|
INDEXER_RETRY_COOLDOWN_SECONDS="${HOST112_INDEXER_RETRY_COOLDOWN_SECONDS:-600}"
|
|
INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-90}"
|
|
SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-15}"
|
|
MANAGER_RETRY_AFTER_SECONDS="${HOST112_MANAGER_RETRY_AFTER_SECONDS:-120}"
|
|
MANAGER_RETRY_COOLDOWN_SECONDS="${HOST112_MANAGER_RETRY_COOLDOWN_SECONDS:-600}"
|
|
MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}"
|
|
MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-90}"
|
|
MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}"
|
|
MANAGER_VERIFY_INTERVAL_SECONDS="${HOST112_MANAGER_VERIFY_INTERVAL_SECONDS:-3}"
|
|
MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-30}"
|
|
READ_TIMEOUT_SECONDS="${HOST112_READ_TIMEOUT_SECONDS:-5}"
|
|
EXECUTOR_DEADLINE_SECONDS="${HOST112_EXECUTOR_DEADLINE_SECONDS:-420}"
|
|
MANAGER_MIN_MEMORY_AVAILABLE_KB="${HOST112_MANAGER_MIN_MEMORY_AVAILABLE_KB:-524288}"
|
|
MANAGER_MIN_DISK_AVAILABLE_KB="${HOST112_MANAGER_MIN_DISK_AVAILABLE_KB:-1048576}"
|
|
MANAGER_MAX_DISK_USED_PERCENT="${HOST112_MANAGER_MAX_DISK_USED_PERCENT:-95}"
|
|
STATE_DIR="${HOST112_RECOVERY_STATE_DIR:-/var/lib/awoooi-host112-recovery}"
|
|
INDEXER_RETRY_MARKER="${HOST112_INDEXER_RETRY_MARKER:-/run/awoooi-host112-indexer-retry.epoch}"
|
|
MANAGER_RETRY_MARKER="${HOST112_MANAGER_RETRY_MARKER:-/run/awoooi-host112-manager-retry.epoch}"
|
|
MANAGER_LOCK_FILE="${HOST112_MANAGER_LOCK_FILE:-/run/awoooi-host112-manager-recovery.lock}"
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage:
|
|
host112-guest-readiness.sh --check [identity]
|
|
host112-guest-readiness.sh --dry-run --trace-id ID --run-id ID --work-item-id ID
|
|
host112-guest-readiness.sh --apply --trace-id ID --run-id ID --work-item-id ID
|
|
|
|
Internal systemd mode:
|
|
host112-guest-readiness.sh --apply-timer
|
|
|
|
--check emits a bounded no-secret guest readiness row and never writes.
|
|
--dry-run evaluates the exact Wazuh manager preflight and never writes.
|
|
--apply performs only allowlisted idempotent recovery for graphical.target,
|
|
LightDM, open-vm-tools, SSH, Wazuh Indexer, and one cooldown-guarded Wazuh
|
|
manager reset/start attempt. The independent verifier requires the manager
|
|
process plus agent event 1514, enrollment 1515, and manager API 55000 listeners.
|
|
USAGE
|
|
}
|
|
|
|
set_mode() {
|
|
local requested="$1"
|
|
if [ "$MODE_EXPLICIT" -eq 1 ]; then
|
|
echo "BLOCKER duplicate_mode" >&2
|
|
exit 64
|
|
fi
|
|
MODE="$requested"
|
|
MODE_EXPLICIT=1
|
|
}
|
|
|
|
require_value() {
|
|
local flag="$1" value="${2:-}"
|
|
if [ -z "$value" ]; then
|
|
echo "BLOCKER missing_value=$flag" >&2
|
|
exit 64
|
|
fi
|
|
}
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--check) set_mode "check" ;;
|
|
--dry-run) set_mode "dry_run" ;;
|
|
--apply) set_mode "apply" ;;
|
|
--apply-timer) set_mode "timer_apply" ;;
|
|
--trace-id)
|
|
require_value "$1" "${2:-}"
|
|
TRACE_ID="$2"
|
|
shift
|
|
;;
|
|
--run-id)
|
|
require_value "$1" "${2:-}"
|
|
RUN_ID="$2"
|
|
shift
|
|
;;
|
|
--work-item-id)
|
|
require_value "$1" "${2:-}"
|
|
WORK_ITEM_ID="$2"
|
|
shift
|
|
;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) echo "BLOCKER unknown_argument=$1" >&2; exit 64 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
safe_identity() {
|
|
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$ ]]
|
|
}
|
|
|
|
identity_count=0
|
|
[ -n "$TRACE_ID" ] && identity_count=$((identity_count + 1))
|
|
[ -n "$RUN_ID" ] && identity_count=$((identity_count + 1))
|
|
[ -n "$WORK_ITEM_ID" ] && identity_count=$((identity_count + 1))
|
|
if [ "$identity_count" -ne 0 ] && [ "$identity_count" -ne 3 ]; then
|
|
echo "BLOCKER incomplete_control_identity" >&2
|
|
exit 64
|
|
fi
|
|
if [ "$identity_count" -eq 3 ]; then
|
|
for identity in "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID"; do
|
|
if ! safe_identity "$identity"; then
|
|
echo "BLOCKER unsafe_control_identity" >&2
|
|
exit 64
|
|
fi
|
|
done
|
|
fi
|
|
if { [ "$MODE" = "dry_run" ] || [ "$MODE" = "apply" ]; } && [ "$identity_count" -ne 3 ]; then
|
|
echo "BLOCKER control_identity_required" >&2
|
|
exit 64
|
|
fi
|
|
if [ "$MODE" = "timer_apply" ] && [ "$identity_count" -ne 0 ]; then
|
|
echo "BLOCKER timer_identity_is_system_generated" >&2
|
|
exit 64
|
|
fi
|
|
|
|
actions=()
|
|
runtime_write_performed=0
|
|
state_write_performed=0
|
|
manager_runtime_write_performed=0
|
|
manager_marker_write_performed=0
|
|
receipt_write_performed=0
|
|
receipt_status="not_requested"
|
|
receipt_ref="none"
|
|
receipt_path="none"
|
|
receipt_sha256="none"
|
|
receipt_reused=0
|
|
durable_receipt_present=0
|
|
durable_receipt_identity_match=0
|
|
durable_receipt_boot_match=0
|
|
durable_receipt_terminal="none"
|
|
durable_readback_status="not_requested"
|
|
durable_readback_path="none"
|
|
durable_readback_sha256="none"
|
|
durable_readback_present=0
|
|
durable_readback_identity_match=0
|
|
durable_readback_boot_match=0
|
|
durable_readback_receipt_link_match=0
|
|
durable_readback_terminal_match=0
|
|
latest_readback_pointer_written=0
|
|
immutable_replay=0
|
|
manager_attempt_count=0
|
|
manager_start_exit=not_attempted
|
|
manager_terminal="observed"
|
|
run_terminal="observed"
|
|
rollback_attempted=0
|
|
rollback_verified=0
|
|
rollback_terminal=not_required
|
|
timer_artifact_policy="not_applicable"
|
|
timer_ledger_capacity=128
|
|
timer_ledger_slot="none"
|
|
timer_observation_status="not_requested"
|
|
timer_observation_path="none"
|
|
timer_observation_sha256="none"
|
|
timer_observation_repeat_count=0
|
|
timer_observation_write_performed=0
|
|
timer_observation_failed=0
|
|
signal_readback_status="not_requested"
|
|
signal_readback_path="none"
|
|
signal_readback_sha256="none"
|
|
action_failure_observed=0
|
|
|
|
is_uint() {
|
|
[[ "$1" =~ ^[0-9]+$ ]]
|
|
}
|
|
|
|
for timing_value in \
|
|
"$INDEXER_START_TIMEOUT_SECONDS" \
|
|
"$SERVICE_START_TIMEOUT_SECONDS" \
|
|
"$MANAGER_RESET_TIMEOUT_SECONDS" \
|
|
"$MANAGER_START_TIMEOUT_SECONDS" \
|
|
"$MANAGER_VERIFY_TIMEOUT_SECONDS" \
|
|
"$MANAGER_VERIFY_INTERVAL_SECONDS" \
|
|
"$MANAGER_ROLLBACK_TIMEOUT_SECONDS" \
|
|
"$READ_TIMEOUT_SECONDS" \
|
|
"$EXECUTOR_DEADLINE_SECONDS"; do
|
|
if ! is_uint "$timing_value" || [ "$timing_value" -le 0 ]; then
|
|
echo "BLOCKER invalid_timeout_configuration" >&2
|
|
exit 64
|
|
fi
|
|
done
|
|
if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -gt "$MANAGER_VERIFY_TIMEOUT_SECONDS" ]; then
|
|
echo "BLOCKER manager_verify_interval_exceeds_timeout" >&2
|
|
exit 64
|
|
fi
|
|
APPLY_PHASE_TIMEOUT_BUDGET_SECONDS=$((
|
|
(8 * SERVICE_START_TIMEOUT_SECONDS)
|
|
+ INDEXER_START_TIMEOUT_SECONDS
|
|
+ (2 * MANAGER_RESET_TIMEOUT_SECONDS)
|
|
+ MANAGER_START_TIMEOUT_SECONDS
|
|
+ MANAGER_VERIFY_TIMEOUT_SECONDS
|
|
+ MANAGER_ROLLBACK_TIMEOUT_SECONDS
|
|
))
|
|
if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt "$EXECUTOR_DEADLINE_SECONDS" ]; then
|
|
echo "BLOCKER apply_phase_timeout_budget_exceeds_executor_deadline" >&2
|
|
exit 64
|
|
fi
|
|
EXECUTOR_DEADLINE_EPOCH=$(( $(date +%s) + EXECUTOR_DEADLINE_SECONDS ))
|
|
|
|
remaining_executor_seconds() {
|
|
local now remaining
|
|
now="$(date +%s)"
|
|
remaining=$((EXECUTOR_DEADLINE_EPOCH - now))
|
|
[ "$remaining" -gt 0 ] || return 1
|
|
printf '%s' "$remaining"
|
|
}
|
|
|
|
bounded_timeout() {
|
|
local requested="$1" remaining effective
|
|
shift
|
|
remaining="$(remaining_executor_seconds)" || return 124
|
|
effective="$requested"
|
|
[ "$effective" -le "$remaining" ] || effective="$remaining"
|
|
[ "$effective" -gt 0 ] || return 124
|
|
timeout --signal=TERM --kill-after=2 "$effective" "$@"
|
|
}
|
|
|
|
write_epoch_marker() {
|
|
local marker="$1" epoch="$2" tmp
|
|
tmp="${marker}.tmp"
|
|
if printf '%s\n' "$epoch" >"$tmp" \
|
|
&& chmod 0644 "$tmp" \
|
|
&& mv "$tmp" "$marker"; then
|
|
return 0
|
|
fi
|
|
rm -f "$tmp" >/dev/null 2>&1 || true
|
|
return 1
|
|
}
|
|
|
|
unit_active() {
|
|
local unit="$1" state
|
|
state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-active "$unit" 2>/dev/null || true)"
|
|
printf '%s' "${state:-unknown}"
|
|
}
|
|
|
|
unit_enabled() {
|
|
local unit="$1" state
|
|
state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-enabled "$unit" 2>/dev/null || true)"
|
|
printf '%s' "${state:-unknown}"
|
|
}
|
|
|
|
unit_property() {
|
|
local unit="$1" property="$2" state
|
|
state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl show "$unit" -p "$property" --value 2>/dev/null || true)"
|
|
printf '%s' "${state:-unknown}"
|
|
}
|
|
|
|
unit_loaded() {
|
|
[ "$(unit_property "$1" LoadState)" = "loaded" ]
|
|
}
|
|
|
|
record_action() {
|
|
actions+=("$1")
|
|
}
|
|
|
|
start_if_inactive() {
|
|
local unit="$1" action="$2" state
|
|
state="$(unit_active "$unit")"
|
|
if [ "$state" = "active" ] || [ "$state" = "activating" ]; then
|
|
return 0
|
|
fi
|
|
# A mutating systemctl invocation may partially change runtime state before
|
|
# returning non-zero or timing out. Record the attempt before execution so
|
|
# those paths can never be classified as no-write timer observations.
|
|
runtime_write_performed=1
|
|
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl start "$unit"; then
|
|
record_action "$action"
|
|
else
|
|
record_action "${action}_failed"
|
|
fi
|
|
}
|
|
|
|
enable_if_disabled() {
|
|
local unit="$1" action="$2" state
|
|
state="$(unit_enabled "$unit")"
|
|
if [ "$state" = "enabled" ] || [ "$state" = "static" ]; then
|
|
return 0
|
|
fi
|
|
runtime_write_performed=1
|
|
state_write_performed=1
|
|
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl enable "$unit" >/dev/null; then
|
|
record_action "$action"
|
|
else
|
|
record_action "${action}_failed"
|
|
fi
|
|
}
|
|
|
|
tcp_port_listening() {
|
|
local port="$1"
|
|
bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -ltn 2>/dev/null \
|
|
| awk '{print $4}' | grep -Eq "(^|:)$port$"
|
|
}
|
|
|
|
udp_port_listening() {
|
|
local port="$1"
|
|
bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -lun 2>/dev/null \
|
|
| awk '{print $4}' | grep -Eq "(^|:)$port$"
|
|
}
|
|
|
|
process_count() {
|
|
local process_name="$1" count
|
|
count="$(bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -xc "$process_name" 2>/dev/null || true)"
|
|
printf '%s' "${count:-0}"
|
|
}
|
|
|
|
memory_available_kb() {
|
|
local available
|
|
available="$(awk '/^MemAvailable:/ {print $2; exit}' /proc/meminfo 2>/dev/null || true)"
|
|
printf '%s' "${available:-0}"
|
|
}
|
|
|
|
disk_readback() {
|
|
local target="/var/ossec" line
|
|
[ -e "$target" ] || target="/"
|
|
line="$(bounded_timeout "$READ_TIMEOUT_SECONDS" df -Pk "$target" 2>/dev/null \
|
|
| awk 'NR==2 {gsub(/%/, "", $5); print $4 " " $5; exit}')"
|
|
if [ -n "$line" ]; then
|
|
printf '%s' "$line"
|
|
else
|
|
printf '0 100'
|
|
fi
|
|
}
|
|
|
|
capture_manager_state() {
|
|
local disk_values
|
|
manager_load_state="$(unit_property wazuh-manager.service LoadState)"
|
|
manager_active_state="$(unit_property wazuh-manager.service ActiveState)"
|
|
manager_sub_state="$(unit_property wazuh-manager.service SubState)"
|
|
manager_result="$(unit_property wazuh-manager.service Result)"
|
|
manager_failed_state_observed=0
|
|
if [ "$manager_active_state" = "failed" ] \
|
|
|| { [ "$manager_result" != "success" ] && [ "$manager_result" != "unknown" ]; }; then
|
|
manager_failed_state_observed=1
|
|
fi
|
|
analysisd_count="$(process_count wazuh-analysisd)"
|
|
event_1514_tcp=0
|
|
event_1514_udp=0
|
|
enrollment_1515_tcp=0
|
|
manager_api_55000_tcp=0
|
|
tcp_port_listening 1514 && event_1514_tcp=1
|
|
udp_port_listening 1514 && event_1514_udp=1
|
|
tcp_port_listening 1515 && enrollment_1515_tcp=1
|
|
tcp_port_listening 55000 && manager_api_55000_tcp=1
|
|
manager_event_port_ready=0
|
|
if [ "$event_1514_tcp" -eq 1 ] || [ "$event_1514_udp" -eq 1 ]; then
|
|
manager_event_port_ready=1
|
|
fi
|
|
manager_orphan_analysisd=0
|
|
if [ "$manager_active_state" != "active" ] && is_uint "$analysisd_count" && [ "$analysisd_count" -gt 0 ]; then
|
|
manager_orphan_analysisd=1
|
|
fi
|
|
manager_verifier_ready=0
|
|
if [ "$manager_active_state" = "active" ] \
|
|
&& is_uint "$analysisd_count" && [ "$analysisd_count" -gt 0 ] \
|
|
&& [ "$manager_event_port_ready" -eq 1 ] \
|
|
&& [ "$enrollment_1515_tcp" -eq 1 ] \
|
|
&& [ "$manager_api_55000_tcp" -eq 1 ]; then
|
|
manager_verifier_ready=1
|
|
fi
|
|
manager_memory_available_kb="$(memory_available_kb)"
|
|
disk_values="$(disk_readback)"
|
|
read -r manager_disk_available_kb manager_disk_used_percent <<<"$disk_values"
|
|
manager_resource_ready=0
|
|
if is_uint "$manager_memory_available_kb" \
|
|
&& is_uint "$manager_disk_available_kb" \
|
|
&& is_uint "$manager_disk_used_percent" \
|
|
&& [ "$manager_memory_available_kb" -ge "$MANAGER_MIN_MEMORY_AVAILABLE_KB" ] \
|
|
&& [ "$manager_disk_available_kb" -ge "$MANAGER_MIN_DISK_AVAILABLE_KB" ] \
|
|
&& [ "$manager_disk_used_percent" -lt "$MANAGER_MAX_DISK_USED_PERCENT" ]; then
|
|
manager_resource_ready=1
|
|
fi
|
|
}
|
|
|
|
manager_cooldown_remaining() {
|
|
local now_epoch="$1" last_retry_epoch=0 elapsed
|
|
if [ -r "$MANAGER_RETRY_MARKER" ]; then
|
|
read -r last_retry_epoch <"$MANAGER_RETRY_MARKER" || last_retry_epoch=0
|
|
fi
|
|
is_uint "$last_retry_epoch" || last_retry_epoch=0
|
|
elapsed=$((now_epoch - last_retry_epoch))
|
|
if [ "$elapsed" -lt 0 ]; then
|
|
printf '%s' "$MANAGER_RETRY_COOLDOWN_SECONDS"
|
|
elif [ "$elapsed" -lt "$MANAGER_RETRY_COOLDOWN_SECONDS" ]; then
|
|
printf '%s' "$((MANAGER_RETRY_COOLDOWN_SECONDS - elapsed))"
|
|
else
|
|
printf '0'
|
|
fi
|
|
}
|
|
|
|
evaluate_manager_preflight() {
|
|
local now_epoch="$1" uptime_for_retry="$2"
|
|
manager_cooldown_remaining_seconds="$(manager_cooldown_remaining "$now_epoch")"
|
|
manager_preflight_status="ready"
|
|
if ! command -v systemctl >/dev/null 2>&1 \
|
|
|| ! command -v timeout >/dev/null 2>&1 \
|
|
|| ! command -v ss >/dev/null 2>&1 \
|
|
|| ! command -v flock >/dev/null 2>&1; then
|
|
manager_preflight_status="blocked_required_tool_missing"
|
|
elif [ "$manager_load_state" != "loaded" ]; then
|
|
manager_preflight_status="blocked_manager_unit_not_loaded"
|
|
elif [ "$manager_verifier_ready" -eq 1 ]; then
|
|
manager_preflight_status="already_verified_healthy"
|
|
elif [ "$manager_active_state" = "active" ]; then
|
|
manager_preflight_status="blocked_active_verifier_mismatch"
|
|
elif [ "$manager_orphan_analysisd" -eq 1 ]; then
|
|
manager_preflight_status="blocked_orphan_analysisd_no_write"
|
|
elif [ "$manager_resource_ready" -ne 1 ]; then
|
|
manager_preflight_status="blocked_resource_preflight_no_write"
|
|
elif [ "$(unit_active wazuh-indexer.service)" != "active" ]; then
|
|
manager_preflight_status="blocked_indexer_dependency_no_write"
|
|
elif ! is_uint "$uptime_for_retry" || [ "$uptime_for_retry" -lt "$MANAGER_RETRY_AFTER_SECONDS" ]; then
|
|
manager_preflight_status="blocked_boot_settle_no_write"
|
|
elif [ "$manager_cooldown_remaining_seconds" -gt 0 ]; then
|
|
manager_preflight_status="blocked_retry_cooldown_no_write"
|
|
fi
|
|
}
|
|
|
|
wait_for_manager_verifier() {
|
|
local deadline now remaining
|
|
deadline=$(( $(date +%s) + MANAGER_VERIFY_TIMEOUT_SECONDS ))
|
|
while true; do
|
|
capture_manager_state
|
|
[ "$manager_verifier_ready" -eq 1 ] && return 0
|
|
now="$(date +%s)"
|
|
[ "$now" -ge "$deadline" ] && return 1
|
|
remaining="$((deadline - now))"
|
|
[ "$remaining" -gt 0 ] || return 1
|
|
if [ "$MANAGER_VERIFY_INTERVAL_SECONDS" -lt "$remaining" ]; then
|
|
sleep "$MANAGER_VERIFY_INTERVAL_SECONDS"
|
|
else
|
|
sleep "$remaining"
|
|
fi
|
|
done
|
|
}
|
|
|
|
rollback_manager_to_inactive() {
|
|
rollback_attempted=1
|
|
rollback_terminal="bounded_stop_attempted"
|
|
if bounded_timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service; then
|
|
record_action "rollback_stop_wazuh_manager"
|
|
else
|
|
record_action "rollback_stop_wazuh_manager_failed"
|
|
fi
|
|
capture_manager_state
|
|
if [ "$manager_active_state" != "active" ] \
|
|
&& is_uint "$analysisd_count" && [ "$analysisd_count" -eq 0 ] \
|
|
&& [ "$manager_event_port_ready" -eq 0 ] \
|
|
&& [ "$enrollment_1515_tcp" -eq 0 ] \
|
|
&& [ "$manager_api_55000_tcp" -eq 0 ]; then
|
|
rollback_verified=1
|
|
rollback_terminal="bounded_stop_verified"
|
|
else
|
|
rollback_terminal="bounded_stop_not_verified"
|
|
fi
|
|
}
|
|
|
|
receipt_name_for_identity() {
|
|
local identity_hash safe_run
|
|
identity_hash="$(printf '%s\0%s\0%s\0' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" | sha256sum | awk '{print $1}')"
|
|
safe_run="$(printf '%s' "$RUN_ID" | tr -c 'A-Za-z0-9._+-' '_')"
|
|
safe_run="${safe_run:0:48}"
|
|
[ -n "$safe_run" ] || safe_run="unknown"
|
|
printf '%s-%s' "$safe_run" "$identity_hash"
|
|
}
|
|
|
|
receipt_token() {
|
|
local path="$1" key="$2"
|
|
awk -v prefix="${key}=" '
|
|
{
|
|
for (i = 1; i <= NF; i++) {
|
|
if (index($i, prefix) == 1) {
|
|
print substr($i, length(prefix) + 1)
|
|
exit
|
|
}
|
|
}
|
|
}
|
|
' "$path" 2>/dev/null
|
|
}
|
|
|
|
current_action_text() {
|
|
if [ "${#actions[@]}" -gt 0 ]; then
|
|
IFS=,; printf '%s' "${actions[*]}"
|
|
else
|
|
printf 'none'
|
|
fi
|
|
}
|
|
|
|
timer_slot_for_identity() {
|
|
local checksum
|
|
checksum="$(printf '%s\0%s\0%s\0' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" | cksum | awk '{print $1}')"
|
|
is_uint "$checksum" || return 1
|
|
printf '%03d' "$((checksum % timer_ledger_capacity))"
|
|
}
|
|
|
|
load_durable_receipt() {
|
|
local receipt_name stored_trace stored_run stored_work_item stored_boot
|
|
receipt_name="$(receipt_name_for_identity)"
|
|
receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
|
|
[ -f "$receipt_path" ] || return 1
|
|
durable_receipt_present=1
|
|
stored_trace="$(receipt_token "$receipt_path" trace_id)"
|
|
stored_run="$(receipt_token "$receipt_path" run_id)"
|
|
stored_work_item="$(receipt_token "$receipt_path" work_item_id)"
|
|
stored_boot="$(receipt_token "$receipt_path" boot_id)"
|
|
durable_receipt_terminal="$(receipt_token "$receipt_path" terminal)"
|
|
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
|
|
receipt_ref="host112-manager-recovery:${receipt_name}"
|
|
if [ "$stored_trace" = "$TRACE_ID" ] \
|
|
&& [ "$stored_run" = "$RUN_ID" ] \
|
|
&& [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then
|
|
durable_receipt_identity_match=1
|
|
fi
|
|
if [ "$stored_boot" = "$boot_id" ]; then
|
|
durable_receipt_boot_match=1
|
|
fi
|
|
if [ "$durable_receipt_identity_match" -ne 1 ] \
|
|
|| [ "$durable_receipt_boot_match" -ne 1 ]; then
|
|
receipt_status="immutable_identity_or_boot_mismatch"
|
|
return 65
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
load_durable_readback() {
|
|
local receipt_name stored_trace stored_run stored_work_item stored_boot
|
|
local stored_receipt_path stored_receipt_sha stored_terminal
|
|
receipt_name="$(receipt_name_for_identity)"
|
|
durable_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt"
|
|
[ -f "$durable_readback_path" ] || {
|
|
durable_readback_status="missing_for_durable_receipt"
|
|
return 1
|
|
}
|
|
durable_readback_present=1
|
|
stored_trace="$(receipt_token "$durable_readback_path" trace_id)"
|
|
stored_run="$(receipt_token "$durable_readback_path" run_id)"
|
|
stored_work_item="$(receipt_token "$durable_readback_path" work_item_id)"
|
|
stored_boot="$(receipt_token "$durable_readback_path" boot_id)"
|
|
stored_receipt_path="$(receipt_token "$durable_readback_path" receipt_path)"
|
|
stored_receipt_sha="$(receipt_token "$durable_readback_path" receipt_sha256)"
|
|
stored_terminal="$(receipt_token "$durable_readback_path" durable_receipt_terminal)"
|
|
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
|
|
if [ "$stored_trace" = "$TRACE_ID" ] \
|
|
&& [ "$stored_run" = "$RUN_ID" ] \
|
|
&& [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then
|
|
durable_readback_identity_match=1
|
|
fi
|
|
if [ "$stored_boot" = "$boot_id" ]; then
|
|
durable_readback_boot_match=1
|
|
fi
|
|
if [ "$stored_receipt_path" = "$receipt_path" ] \
|
|
&& [ "$stored_receipt_sha" = "$receipt_sha256" ]; then
|
|
durable_readback_receipt_link_match=1
|
|
fi
|
|
if [ "$stored_terminal" = "$durable_receipt_terminal" ]; then
|
|
durable_readback_terminal_match=1
|
|
fi
|
|
if [ "$durable_readback_identity_match" -ne 1 ] \
|
|
|| [ "$durable_readback_boot_match" -ne 1 ] \
|
|
|| [ "$durable_readback_receipt_link_match" -ne 1 ] \
|
|
|| [ "$durable_readback_terminal_match" -ne 1 ]; then
|
|
durable_readback_status="immutable_identity_boot_receipt_or_terminal_mismatch"
|
|
return 65
|
|
fi
|
|
durable_readback_status="loaded_immutable_verified"
|
|
return 0
|
|
}
|
|
|
|
write_public_receipt() {
|
|
local receipt_name tmp generated_at receipt_line load_status receipt_action_text timer_slot
|
|
if [ "$MODE" != "timer_apply" ]; then
|
|
if load_durable_receipt; then
|
|
receipt_status="reused_immutable"
|
|
receipt_reused=1
|
|
return 0
|
|
else
|
|
load_status=$?
|
|
if [ "$load_status" -eq 65 ]; then
|
|
return 1
|
|
fi
|
|
fi
|
|
fi
|
|
if [ "$MODE" = "timer_apply" ]; then
|
|
timer_slot="$(timer_slot_for_identity)" || {
|
|
receipt_status="write_failed"
|
|
return 1
|
|
}
|
|
timer_ledger_slot="$timer_slot"
|
|
receipt_name="timer-slot-${timer_slot}"
|
|
receipt_path="$STATE_DIR/timer-ledger/${receipt_name}.receipt.txt"
|
|
timer_artifact_policy="bounded_ring_128_for_runtime_mutation"
|
|
else
|
|
receipt_name="$(receipt_name_for_identity)"
|
|
receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
|
|
fi
|
|
generated_at="$(date --iso-8601=seconds)"
|
|
receipt_action_text="$(current_action_text)"
|
|
receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v2 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed action_count=${#actions[@]} actions=$receipt_action_text rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal manager_terminal=$manager_terminal terminal=$run_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
|
|
if install -d -m 0750 "$(dirname "$receipt_path")"; then
|
|
tmp="${receipt_path}.tmp"
|
|
if [ "$MODE" = "timer_apply" ] \
|
|
&& printf '%s\n' "$receipt_line" >"$tmp" \
|
|
&& chmod 0640 "$tmp" \
|
|
&& mv "$tmp" "$receipt_path"; then
|
|
receipt_write_performed=1
|
|
receipt_status="written_bounded_timer_slot"
|
|
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
|
|
receipt_ref="host112-timer-ledger:${receipt_name}"
|
|
durable_receipt_present=1
|
|
durable_receipt_identity_match=1
|
|
durable_receipt_boot_match=1
|
|
durable_receipt_terminal="$run_terminal"
|
|
return 0
|
|
elif [ "$MODE" != "timer_apply" ] \
|
|
&& printf '%s\n' "$receipt_line" >"$tmp" \
|
|
&& chmod 0640 "$tmp" \
|
|
&& ln "$tmp" "$receipt_path" 2>/dev/null; then
|
|
rm -f "$tmp"
|
|
receipt_write_performed=1
|
|
receipt_status="written_immutable"
|
|
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
|
|
receipt_ref="host112-manager-recovery:${receipt_name}"
|
|
durable_receipt_present=1
|
|
durable_receipt_identity_match=1
|
|
durable_receipt_boot_match=1
|
|
durable_receipt_terminal="$run_terminal"
|
|
return 0
|
|
fi
|
|
rm -f "$tmp" >/dev/null 2>&1 || true
|
|
if [ "$MODE" != "timer_apply" ] && load_durable_receipt; then
|
|
receipt_status="reused_immutable"
|
|
receipt_reused=1
|
|
return 0
|
|
fi
|
|
fi
|
|
receipt_status="write_failed"
|
|
return 1
|
|
}
|
|
|
|
write_signal_readback() {
|
|
local receipt_name tmp signal_line signal_action_text timer_slot
|
|
if [ "$MODE" = "timer_apply" ]; then
|
|
timer_slot="$(timer_slot_for_identity)" || {
|
|
signal_readback_status="write_failed"
|
|
return 1
|
|
}
|
|
timer_ledger_slot="$timer_slot"
|
|
receipt_name="timer-slot-${timer_slot}"
|
|
signal_readback_path="$STATE_DIR/timer-ledger/${receipt_name}.readback.txt"
|
|
else
|
|
receipt_name="$(receipt_name_for_identity)"
|
|
signal_readback_path="$STATE_DIR/readbacks/${receipt_name}.signal.txt"
|
|
fi
|
|
signal_action_text="$(current_action_text)"
|
|
signal_line="schema_version=host112_guest_recovery_signal_readback_v1 generated_at=$(date --iso-8601=seconds) mode=$MODE trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id manager_terminal=$manager_terminal terminal=$run_terminal manager_active=$manager_active_state manager_result=$manager_result manager_orphan_analysisd=$manager_orphan_analysisd manager_verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed action_count=${#actions[@]} actions=$signal_action_text rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal receipt_path=$receipt_path receipt_sha256=$receipt_sha256 prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
|
|
install -d -m 0750 "$(dirname "$signal_readback_path")" || {
|
|
signal_readback_status="write_failed"
|
|
return 1
|
|
}
|
|
if [ "$MODE" != "timer_apply" ] && [ -f "$signal_readback_path" ]; then
|
|
if [ "$(receipt_token "$signal_readback_path" trace_id)" != "$TRACE_ID" ] \
|
|
|| [ "$(receipt_token "$signal_readback_path" run_id)" != "$RUN_ID" ] \
|
|
|| [ "$(receipt_token "$signal_readback_path" work_item_id)" != "$WORK_ITEM_ID" ] \
|
|
|| [ "$(receipt_token "$signal_readback_path" boot_id)" != "$boot_id" ] \
|
|
|| [ "$(receipt_token "$signal_readback_path" receipt_path)" != "$receipt_path" ] \
|
|
|| [ "$(receipt_token "$signal_readback_path" receipt_sha256)" != "$receipt_sha256" ]; then
|
|
signal_readback_status="immutable_identity_boot_or_receipt_mismatch"
|
|
return 1
|
|
fi
|
|
signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')"
|
|
signal_readback_status="reused_immutable"
|
|
return 0
|
|
fi
|
|
tmp="${signal_readback_path}.tmp"
|
|
if [ "$MODE" = "timer_apply" ] \
|
|
&& printf '%s\n' "$signal_line" >"$tmp" \
|
|
&& chmod 0640 "$tmp" \
|
|
&& mv "$tmp" "$signal_readback_path"; then
|
|
signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')"
|
|
signal_readback_status="written_bounded_timer_slot"
|
|
return 0
|
|
elif [ "$MODE" != "timer_apply" ] \
|
|
&& printf '%s\n' "$signal_line" >"$tmp" \
|
|
&& chmod 0640 "$tmp" \
|
|
&& ln "$tmp" "$signal_readback_path" 2>/dev/null; then
|
|
rm -f "$tmp"
|
|
signal_readback_sha256="$(sha256sum "$signal_readback_path" | awk '{print $1}')"
|
|
signal_readback_status="written_immutable"
|
|
return 0
|
|
fi
|
|
rm -f "$tmp" >/dev/null 2>&1 || true
|
|
signal_readback_status="write_failed"
|
|
return 1
|
|
}
|
|
|
|
write_bounded_timer_mutation_readback() {
|
|
local timer_slot tmp
|
|
timer_slot="$(timer_slot_for_identity)" || {
|
|
durable_readback_status="write_failed"
|
|
return 1
|
|
}
|
|
timer_ledger_slot="$timer_slot"
|
|
durable_readback_path="$STATE_DIR/timer-ledger/timer-slot-${timer_slot}.readback.txt"
|
|
install -d -m 0750 "$(dirname "$durable_readback_path")" || {
|
|
durable_readback_status="write_failed"
|
|
return 1
|
|
}
|
|
tmp="${durable_readback_path}.tmp"
|
|
if printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s timer_ledger_slot=%s timer_ledger_capacity=%s retention=rolling_atomic_overwrite durable_readback_status=written_bounded_timer_slot durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \
|
|
"$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" \
|
|
"$timer_ledger_slot" "$timer_ledger_capacity" "$durable_readback_path" >"$tmp" \
|
|
&& chmod 0640 "$tmp" \
|
|
&& mv "$tmp" "$durable_readback_path"; then
|
|
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
|
|
durable_readback_status="written_bounded_timer_slot"
|
|
durable_readback_present=1
|
|
durable_readback_identity_match=1
|
|
durable_readback_boot_match=1
|
|
durable_readback_receipt_link_match=1
|
|
durable_readback_terminal_match=1
|
|
return 0
|
|
fi
|
|
rm -f "$tmp" >/dev/null 2>&1 || true
|
|
durable_readback_status="write_failed"
|
|
return 1
|
|
}
|
|
|
|
write_bounded_timer_observation() {
|
|
local tmp now previous_fingerprint previous_repeat previous_first fingerprint first_at observation_line
|
|
timer_observation_path="$STATE_DIR/timer-observation.latest"
|
|
now="$(date --iso-8601=seconds)"
|
|
fingerprint="$(printf '%s\0%s\0%s\0%s\0%s\0%s\0%s\0%s\0' \
|
|
"$boot_id" "$guest_ready" "$run_terminal" "$systemd_state" \
|
|
"$console_ready" "$services_ready" "$ssh_ready" "$recovery_timer_ready" \
|
|
| sha256sum | awk '{print $1}')"
|
|
previous_fingerprint=""
|
|
previous_repeat=0
|
|
previous_first="$now"
|
|
if [ -r "$timer_observation_path" ]; then
|
|
previous_fingerprint="$(receipt_token "$timer_observation_path" fingerprint)"
|
|
previous_repeat="$(receipt_token "$timer_observation_path" repeat_count)"
|
|
previous_first="$(receipt_token "$timer_observation_path" first_observed_at)"
|
|
fi
|
|
is_uint "$previous_repeat" || previous_repeat=0
|
|
if [ "$previous_fingerprint" = "$fingerprint" ]; then
|
|
timer_observation_repeat_count=$((previous_repeat + 1))
|
|
first_at="${previous_first:-$now}"
|
|
else
|
|
timer_observation_repeat_count=1
|
|
first_at="$now"
|
|
fi
|
|
observation_line="schema_version=host112_timer_observation_aggregate_v1 first_observed_at=$first_at last_observed_at=$now repeat_count=$timer_observation_repeat_count fingerprint=$fingerprint boot_id=$boot_id guest_ready=$guest_ready manager_terminal=$manager_terminal terminal=$run_terminal systemd_state=$systemd_state console_ready=$console_ready services_ready=$services_ready ssh_ready=$ssh_ready recovery_timer_ready=$recovery_timer_ready runtime_write_performed=0 immutable_artifact_written=0 aggregation=fixed_single_atomic_pointer"
|
|
install -d -m 0750 "$STATE_DIR" || {
|
|
timer_observation_status="write_failed"
|
|
return 1
|
|
}
|
|
tmp="${timer_observation_path}.tmp"
|
|
if printf '%s\n' "$observation_line" >"$tmp" \
|
|
&& chmod 0640 "$tmp" \
|
|
&& mv "$tmp" "$timer_observation_path"; then
|
|
timer_observation_sha256="$(sha256sum "$timer_observation_path" | awk '{print $1}')"
|
|
timer_observation_status="updated_bounded_aggregate"
|
|
timer_observation_write_performed=1
|
|
return 0
|
|
fi
|
|
rm -f "$tmp" >/dev/null 2>&1 || true
|
|
timer_observation_status="write_failed"
|
|
return 1
|
|
}
|
|
|
|
write_immutable_readback() {
|
|
local receipt_name tmp stored_trace stored_run stored_work_item stored_boot latest_tmp
|
|
receipt_name="$(receipt_name_for_identity)"
|
|
durable_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt"
|
|
if [ -f "$durable_readback_path" ]; then
|
|
stored_trace="$(receipt_token "$durable_readback_path" trace_id)"
|
|
stored_run="$(receipt_token "$durable_readback_path" run_id)"
|
|
stored_work_item="$(receipt_token "$durable_readback_path" work_item_id)"
|
|
stored_boot="$(receipt_token "$durable_readback_path" boot_id)"
|
|
if [ "$stored_trace" != "$TRACE_ID" ] \
|
|
|| [ "$stored_run" != "$RUN_ID" ] \
|
|
|| [ "$stored_work_item" != "$WORK_ITEM_ID" ] \
|
|
|| [ "$stored_boot" != "$boot_id" ]; then
|
|
durable_readback_status="immutable_identity_or_boot_mismatch"
|
|
return 1
|
|
fi
|
|
if load_durable_readback; then
|
|
durable_readback_status="reused_immutable_verified"
|
|
return 0
|
|
fi
|
|
return 1
|
|
fi
|
|
install -d -m 0750 "$(dirname "$durable_readback_path")" || return 1
|
|
tmp="${durable_readback_path}.tmp"
|
|
if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s durable_readback_status=written_immutable durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \
|
|
"$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" "$durable_readback_path" >"$tmp" \
|
|
|| ! chmod 0640 "$tmp" \
|
|
|| ! ln "$tmp" "$durable_readback_path" 2>/dev/null; then
|
|
rm -f "$tmp" >/dev/null 2>&1 || true
|
|
durable_readback_status="write_failed"
|
|
return 1
|
|
fi
|
|
rm -f "$tmp"
|
|
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
|
|
durable_readback_status="written_immutable"
|
|
durable_readback_present=1
|
|
durable_readback_identity_match=1
|
|
durable_readback_boot_match=1
|
|
durable_readback_receipt_link_match=1
|
|
durable_readback_terminal_match=1
|
|
|
|
# Preserve the historical convenience readback as an explicitly mutable
|
|
# pointer. It is never used as execution proof; the immutable path/hash is.
|
|
latest_tmp="${STATE_DIR}/last-readback.txt.tmp"
|
|
if printf 'immutable_path=%s immutable_sha256=%s\n' \
|
|
"$durable_readback_path" "$durable_readback_sha256" >"$latest_tmp" \
|
|
&& chmod 0640 "$latest_tmp" \
|
|
&& mv "$latest_tmp" "${STATE_DIR}/last-readback.txt"; then
|
|
latest_readback_pointer_written=1
|
|
return 0
|
|
fi
|
|
rm -f "$latest_tmp" >/dev/null 2>&1 || true
|
|
durable_readback_status="immutable_written_latest_pointer_failed"
|
|
return 1
|
|
}
|
|
|
|
boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)"
|
|
uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)"
|
|
if [ "$MODE" = "timer_apply" ]; then
|
|
invocation_identity="${INVOCATION_ID:-${boot_id}-$(date +%s)}"
|
|
invocation_identity="$(printf '%s' "$invocation_identity" | tr -c 'A-Za-z0-9._+-' '-')"
|
|
TRACE_ID="host112-timer:$invocation_identity"
|
|
RUN_ID="host112-timer:$invocation_identity"
|
|
WORK_ITEM_ID="HOST112-WAZUH-MANAGER-RECOVERY"
|
|
fi
|
|
|
|
if ! command -v sha256sum >/dev/null 2>&1 \
|
|
|| ! command -v cksum >/dev/null 2>&1; then
|
|
echo "BLOCKER sha256sum_and_cksum_required" >&2
|
|
exit 69
|
|
fi
|
|
|
|
if load_durable_receipt; then
|
|
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
|
immutable_replay=1
|
|
receipt_status="reused_immutable"
|
|
receipt_reused=1
|
|
manager_terminal="immutable_receipt_replay"
|
|
fi
|
|
else
|
|
load_status=$?
|
|
if [ "$load_status" -eq 65 ]; then
|
|
echo "BLOCKER immutable_receipt_identity_or_boot_mismatch" >&2
|
|
exit 65
|
|
fi
|
|
fi
|
|
if [ "$durable_receipt_present" -eq 1 ]; then
|
|
if ! load_durable_readback; then
|
|
echo "BLOCKER durable_readback_missing_or_mismatched_for_receipt" >&2
|
|
exit 65
|
|
fi
|
|
fi
|
|
|
|
# Capture a pre-mutation baseline so a TERM/INT/HUP receipt never relies on
|
|
# unset state, even when interruption lands during the general guest repair.
|
|
capture_manager_state
|
|
before_manager_active="$manager_active_state"
|
|
before_manager_result="$manager_result"
|
|
before_manager_orphan="$manager_orphan_analysisd"
|
|
before_manager_memory="$manager_memory_available_kb"
|
|
before_manager_disk="$manager_disk_available_kb"
|
|
before_manager_disk_used="$manager_disk_used_percent"
|
|
before_event_1514_ready="$manager_event_port_ready"
|
|
before_enrollment_1515="$enrollment_1515_tcp"
|
|
before_manager_api_55000="$manager_api_55000_tcp"
|
|
now_epoch="$(date +%s)"
|
|
evaluate_manager_preflight "$now_epoch" "$uptime_seconds"
|
|
|
|
handle_executor_signal() {
|
|
local signal_name="$1"
|
|
trap - TERM INT HUP
|
|
manager_terminal="interrupted_signal_${signal_name}"
|
|
run_terminal="interrupted_signal_${signal_name}"
|
|
manager_start_exit="interrupted"
|
|
if [ "$manager_runtime_write_performed" -eq 1 ]; then
|
|
rollback_manager_to_inactive || true
|
|
fi
|
|
capture_manager_state || true
|
|
if [ "$runtime_write_performed" -eq 1 ]; then
|
|
if write_public_receipt && write_signal_readback; then
|
|
exit 75
|
|
fi
|
|
exit 74
|
|
fi
|
|
exit 75
|
|
}
|
|
|
|
if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
|
|
&& [ "$immutable_replay" -eq 0 ]; then
|
|
if [ "${EUID:-$(id -u)}" -ne 0 ]; then
|
|
echo "BLOCKER host112_apply_requires_root" >&2
|
|
exit 77
|
|
fi
|
|
exec 9>"$MANAGER_LOCK_FILE"
|
|
if ! flock -n 9; then
|
|
echo "BLOCKER host112_manager_single_flight_busy" >&2
|
|
exit 75
|
|
fi
|
|
trap 'handle_executor_signal TERM' TERM
|
|
trap 'handle_executor_signal INT' INT
|
|
trap 'handle_executor_signal HUP' HUP
|
|
|
|
current_default="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)"
|
|
if [ "$current_default" != "graphical.target" ]; then
|
|
runtime_write_performed=1
|
|
state_write_performed=1
|
|
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl set-default graphical.target >/dev/null; then
|
|
record_action "set_default_graphical_target"
|
|
else
|
|
record_action "set_default_graphical_target_failed"
|
|
fi
|
|
fi
|
|
|
|
start_if_inactive graphical.target start_graphical_target
|
|
start_if_inactive open-vm-tools.service start_vmware_tools
|
|
start_if_inactive lightdm.service start_lightdm
|
|
if unit_loaded ssh.service; then
|
|
enable_if_disabled ssh.service enable_ssh
|
|
start_if_inactive ssh.service start_ssh
|
|
else
|
|
record_action "ssh_service_missing"
|
|
fi
|
|
if unit_loaded awoooi-host112-guest-recovery.timer; then
|
|
enable_if_disabled awoooi-host112-guest-recovery.timer enable_recovery_timer
|
|
start_if_inactive awoooi-host112-guest-recovery.timer start_recovery_timer
|
|
else
|
|
record_action "recovery_timer_missing"
|
|
fi
|
|
|
|
indexer_state="$(unit_active wazuh-indexer.service)"
|
|
if unit_loaded wazuh-indexer.service \
|
|
&& [ "$indexer_state" != "active" ] \
|
|
&& [ "$indexer_state" != "activating" ] \
|
|
&& is_uint "$uptime_seconds" \
|
|
&& [ "$uptime_seconds" -ge "$INDEXER_RETRY_AFTER_SECONDS" ]; then
|
|
now_epoch="$(date +%s)"
|
|
last_retry_epoch=0
|
|
if [ -r "$INDEXER_RETRY_MARKER" ]; then
|
|
read -r last_retry_epoch <"$INDEXER_RETRY_MARKER" || last_retry_epoch=0
|
|
fi
|
|
is_uint "$last_retry_epoch" || last_retry_epoch=0
|
|
if [ $((now_epoch - last_retry_epoch)) -ge "$INDEXER_RETRY_COOLDOWN_SECONDS" ]; then
|
|
if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then
|
|
runtime_write_performed=1
|
|
state_write_performed=1
|
|
bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" \
|
|
systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true
|
|
if bounded_timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then
|
|
record_action "retry_wazuh_indexer"
|
|
else
|
|
record_action "retry_wazuh_indexer_failed"
|
|
fi
|
|
else
|
|
record_action "wazuh_indexer_marker_write_failed_no_write"
|
|
fi
|
|
else
|
|
record_action "wazuh_indexer_retry_cooldown"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
capture_manager_state
|
|
now_epoch="$(date +%s)"
|
|
evaluate_manager_preflight "$now_epoch" "$uptime_seconds"
|
|
|
|
if [ "$MODE" = "dry_run" ]; then
|
|
case "$manager_preflight_status" in
|
|
ready) manager_terminal="dry_run_ready" ;;
|
|
already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;;
|
|
*) manager_terminal="dry_run_${manager_preflight_status}" ;;
|
|
esac
|
|
elif { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
|
|
&& [ "$immutable_replay" -eq 1 ]; then
|
|
manager_terminal="immutable_receipt_replay"
|
|
elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
|
case "$manager_preflight_status" in
|
|
already_verified_healthy)
|
|
manager_terminal="idempotent_already_verified_healthy"
|
|
;;
|
|
ready)
|
|
if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then
|
|
# The cooldown marker is a durable runtime state mutation even if the
|
|
# following systemctl operation fails. Account for it immediately so
|
|
# no failure path can emit a false no-write receipt.
|
|
runtime_write_performed=1
|
|
state_write_performed=1
|
|
manager_marker_write_performed=1
|
|
manager_attempt_count=1
|
|
if bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then
|
|
record_action "reset_failed_wazuh_manager"
|
|
runtime_write_performed=1
|
|
manager_runtime_write_performed=1
|
|
if bounded_timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then
|
|
manager_start_exit=0
|
|
else
|
|
manager_start_exit=$?
|
|
fi
|
|
if wait_for_manager_verifier; then
|
|
if [ "$manager_start_exit" = "0" ]; then
|
|
record_action "start_wazuh_manager_verified"
|
|
manager_terminal="verified_healthy"
|
|
else
|
|
record_action "wazuh_manager_verifier_healthy_after_nonzero_start_unattributed"
|
|
manager_terminal="manager_start_nonzero_verifier_healthy_unattributed"
|
|
fi
|
|
else
|
|
record_action "start_wazuh_manager_verifier_failed"
|
|
rollback_manager_to_inactive
|
|
if [ "$rollback_verified" -eq 1 ]; then
|
|
manager_terminal="rolled_back_verifier_failed"
|
|
else
|
|
manager_terminal="rollback_failed_degraded"
|
|
fi
|
|
fi
|
|
else
|
|
manager_start_exit="reset_failed_no_start"
|
|
record_action "reset_failed_wazuh_manager_failed_no_start"
|
|
manager_terminal="no_start_reset_failed"
|
|
fi
|
|
else
|
|
record_action "wazuh_manager_marker_write_failed_no_write"
|
|
manager_terminal="no_write_cooldown_marker_write_failed"
|
|
fi
|
|
;;
|
|
*)
|
|
manager_terminal="no_write_${manager_preflight_status}"
|
|
;;
|
|
esac
|
|
capture_manager_state
|
|
fi
|
|
|
|
systemd_state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-system-running 2>/dev/null || true)"
|
|
systemd_state="${systemd_state:-unknown}"
|
|
default_target="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)"
|
|
default_target="${default_target:-unknown}"
|
|
graphical_target="$(unit_active graphical.target)"
|
|
display_manager="$(unit_active display-manager.service)"
|
|
lightdm="$(unit_active lightdm.service)"
|
|
vmware_tools="$(unit_active open-vm-tools.service)"
|
|
wazuh_indexer="$(unit_active wazuh-indexer.service)"
|
|
wazuh_manager="$(unit_active wazuh-manager.service)"
|
|
wazuh_dashboard="$(unit_active wazuh-dashboard.service)"
|
|
filebeat="$(unit_active filebeat.service)"
|
|
ssh_service="$(unit_active ssh.service)"
|
|
ssh_enabled="$(unit_enabled ssh.service)"
|
|
startup_enabled="$(unit_enabled awoooi-host112-guest-recovery.timer)"
|
|
startup_active="$(unit_active awoooi-host112-guest-recovery.timer)"
|
|
|
|
ssh_port_listening=0
|
|
tcp_port_listening 22 && ssh_port_listening=1
|
|
ssh_ready=0
|
|
if [ "$ssh_service" = "active" ] \
|
|
&& { [ "$ssh_enabled" = "enabled" ] || [ "$ssh_enabled" = "static" ]; } \
|
|
&& [ "$ssh_port_listening" -eq 1 ]; then
|
|
ssh_ready=1
|
|
fi
|
|
|
|
xorg="inactive"
|
|
if bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -x Xorg >/dev/null 2>&1 \
|
|
|| bounded_timeout "$READ_TIMEOUT_SECONDS" pgrep -x Xwayland >/dev/null 2>&1; then
|
|
xorg="active"
|
|
fi
|
|
|
|
console_ready=0
|
|
if [ "$default_target" = "graphical.target" ] \
|
|
&& [ "$graphical_target" = "active" ] \
|
|
&& [ "$display_manager" = "active" ] \
|
|
&& [ "$lightdm" = "active" ] \
|
|
&& [ "$vmware_tools" = "active" ] \
|
|
&& [ "$xorg" = "active" ]; then
|
|
console_ready=1
|
|
fi
|
|
|
|
services_ready=0
|
|
if [ "$wazuh_indexer" = "active" ] \
|
|
&& [ "$wazuh_dashboard" = "active" ] \
|
|
&& [ "$filebeat" = "active" ] \
|
|
&& [ "$manager_verifier_ready" -eq 1 ]; then
|
|
services_ready=1
|
|
fi
|
|
|
|
recovery_timer_ready=0
|
|
if [ "$startup_enabled" = "enabled" ] && [ "$startup_active" = "active" ]; then
|
|
recovery_timer_ready=1
|
|
fi
|
|
|
|
guest_ready=0
|
|
if [ "$console_ready" -eq 1 ] \
|
|
&& [ "$services_ready" -eq 1 ] \
|
|
&& [ "$ssh_ready" -eq 1 ] \
|
|
&& [ "$recovery_timer_ready" -eq 1 ] \
|
|
&& [ "$systemd_state" = "running" ]; then
|
|
guest_ready=1
|
|
fi
|
|
|
|
if [ "${#actions[@]}" -gt 0 ]; then
|
|
for action in "${actions[@]}"; do
|
|
case "$action" in
|
|
*_failed|*_failed_no_start|*_failed_no_write|*_unattributed|*_missing) action_failure_observed=1 ;;
|
|
esac
|
|
done
|
|
fi
|
|
|
|
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
|
if [ "$immutable_replay" -eq 1 ]; then
|
|
run_terminal="immutable_receipt_replay"
|
|
elif [ "$guest_ready" -eq 1 ] \
|
|
&& [ "$action_failure_observed" -eq 0 ] \
|
|
&& { [ "$manager_terminal" = "verified_healthy" ] \
|
|
|| [ "$manager_terminal" = "idempotent_already_verified_healthy" ]; }; then
|
|
if [ "$runtime_write_performed" -eq 1 ]; then
|
|
run_terminal="verified_healthy"
|
|
else
|
|
run_terminal="idempotent_already_verified_healthy"
|
|
fi
|
|
elif [ "$action_failure_observed" -eq 1 ]; then
|
|
run_terminal="execution_failed_or_unattributed_${manager_terminal}"
|
|
else
|
|
run_terminal="guest_verification_failed_${manager_terminal}"
|
|
fi
|
|
fi
|
|
|
|
action_text="none"
|
|
if [ "${#actions[@]}" -gt 0 ]; then
|
|
action_text="$(IFS=,; printf '%s' "${actions[*]}")"
|
|
fi
|
|
|
|
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
|
# Keep signal handling active through the complete guest verifier above.
|
|
# Ignore TERM/INT/HUP only for the short atomic receipt/readback commit
|
|
# section so a successful receipt can never be separated from its readback.
|
|
trap '' TERM INT HUP
|
|
if [ "$immutable_replay" -eq 0 ]; then
|
|
if [ "$MODE" = "apply" ] || [ "$runtime_write_performed" -eq 1 ]; then
|
|
if [ "$MODE" = "timer_apply" ]; then
|
|
timer_artifact_policy="bounded_ring_128_for_runtime_mutation"
|
|
fi
|
|
if ! write_public_receipt; then
|
|
run_terminal="${run_terminal}_receipt_failed"
|
|
fi
|
|
else
|
|
receipt_status="timer_no_runtime_mutation_aggregated"
|
|
timer_artifact_policy="bounded_mutable_observation_no_immutable_artifact"
|
|
fi
|
|
fi
|
|
if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then
|
|
if ! write_bounded_timer_observation; then
|
|
timer_observation_failed=1
|
|
fi
|
|
durable_readback_status="skipped_bounded_timer_no_runtime_mutation"
|
|
fi
|
|
fi
|
|
|
|
trace_output="${TRACE_ID:-none}"
|
|
run_output="${RUN_ID:-none}"
|
|
work_item_output="${WORK_ITEM_ID:-none}"
|
|
readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_terminal=$manager_terminal run_terminal=$run_terminal terminal=$run_terminal action_failure_observed=$action_failure_observed manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal durable_readback_present=$durable_readback_present durable_readback_identity_match=$durable_readback_identity_match durable_readback_boot_match=$durable_readback_boot_match durable_readback_receipt_link_match=$durable_readback_receipt_link_match durable_readback_terminal_match=$durable_readback_terminal_match rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal timer_artifact_policy=$timer_artifact_policy timer_ledger_capacity=$timer_ledger_capacity timer_ledger_slot=$timer_ledger_slot timer_observation_status=$timer_observation_status timer_observation_path=$timer_observation_path timer_observation_sha256=$timer_observation_sha256 timer_observation_repeat_count=$timer_observation_repeat_count timer_observation_write_performed=$timer_observation_write_performed signal_readback_status=$signal_readback_status signal_readback_path=$signal_readback_path signal_readback_sha256=$signal_readback_sha256 action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
|
|
|
|
canonical_readback_base() {
|
|
printf '%s\n' "$readback" | awk '
|
|
{
|
|
separator = ""
|
|
for (i = 1; i <= NF; i++) {
|
|
if ($i ~ /^durable_readback_(present|identity_match|boot_match|receipt_link_match|terminal_match)=/) {
|
|
continue
|
|
}
|
|
printf "%s%s", separator, $i
|
|
separator = " "
|
|
}
|
|
printf "\n"
|
|
}
|
|
'
|
|
}
|
|
|
|
emit_readback() {
|
|
printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s durable_readback_present=%s durable_readback_identity_match=%s durable_readback_boot_match=%s durable_readback_receipt_link_match=%s durable_readback_terminal_match=%s latest_readback_pointer_written=%s\n' \
|
|
"$(canonical_readback_base)" "$durable_readback_status" "$durable_readback_path" \
|
|
"$durable_readback_sha256" "$durable_readback_present" \
|
|
"$durable_readback_identity_match" "$durable_readback_boot_match" \
|
|
"$durable_readback_receipt_link_match" "$durable_readback_terminal_match" \
|
|
"$latest_readback_pointer_written"
|
|
}
|
|
|
|
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
|
if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then
|
|
if [ "$timer_observation_failed" -eq 1 ]; then
|
|
emit_readback
|
|
exit 74
|
|
fi
|
|
elif [ "$MODE" = "timer_apply" ]; then
|
|
if ! write_bounded_timer_mutation_readback; then
|
|
emit_readback
|
|
exit 74
|
|
fi
|
|
elif ! write_immutable_readback; then
|
|
emit_readback
|
|
exit 74
|
|
fi
|
|
fi
|
|
emit_readback
|
|
|
|
if [ "$MODE" = "dry_run" ]; then
|
|
case "$manager_preflight_status" in
|
|
ready|already_verified_healthy) exit 0 ;;
|
|
*) exit 1 ;;
|
|
esac
|
|
fi
|
|
|
|
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
|
successful_terminal="$run_terminal"
|
|
if [ "$immutable_replay" -eq 1 ]; then
|
|
successful_terminal="$durable_receipt_terminal"
|
|
fi
|
|
[ "$guest_ready" -eq 1 ] \
|
|
&& [ "$receipt_status" != "write_failed" ] \
|
|
&& { [ "$successful_terminal" = "verified_healthy" ] \
|
|
|| [ "$successful_terminal" = "idempotent_already_verified_healthy" ]; }
|
|
exit $?
|
|
fi
|
|
|
|
[ "$guest_ready" -eq 1 ]
|