fix(recovery): close host112 wazuh manager loop

This commit is contained in:
ogt
2026-07-14 11:04:27 +08:00
parent 2180993e4a
commit bbd479a36c
9 changed files with 1076 additions and 72 deletions

View File

@@ -5,8 +5,10 @@ Wants=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply
TimeoutStartSec=300
ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer
# Script phase defaults sum to 500 seconds; the unit owns the 600-second
# outer contract while every child command remains independently bounded.
TimeoutStartSec=600
User=root
Group=root
NoNewPrivileges=true

View File

@@ -1,36 +1,186 @@
#!/usr/bin/env bash
# Host 112 guest readiness and bounded recovery executor.
# Version: 2.0
# Last modified: 2026-07-14 11:00 (Asia/Taipei)
# Last modified by: Codex
# Change: add correlated, cooldown-guarded Wazuh manager recovery and an
# independent process/port verifier without reboot, VM power, firewall, or
# secret access.
set -uo pipefail
MODE="check"
MODE_EXPLICIT=0
TRACE_ID=""
RUN_ID=""
WORK_ITEM_ID=""
INDEXER_RETRY_AFTER_SECONDS="${HOST112_INDEXER_RETRY_AFTER_SECONDS:-210}"
INDEXER_RETRY_COOLDOWN_SECONDS="${HOST112_INDEXER_RETRY_COOLDOWN_SECONDS:-600}"
INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-240}"
INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-180}"
SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-30}"
MANAGER_RETRY_AFTER_SECONDS="${HOST112_MANAGER_RETRY_AFTER_SECONDS:-120}"
MANAGER_RETRY_COOLDOWN_SECONDS="${HOST112_MANAGER_RETRY_COOLDOWN_SECONDS:-600}"
MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}"
MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-120}"
MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}"
MANAGER_VERIFY_INTERVAL_SECONDS="${HOST112_MANAGER_VERIFY_INTERVAL_SECONDS:-3}"
MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-45}"
MANAGER_MIN_MEMORY_AVAILABLE_KB="${HOST112_MANAGER_MIN_MEMORY_AVAILABLE_KB:-524288}"
MANAGER_MIN_DISK_AVAILABLE_KB="${HOST112_MANAGER_MIN_DISK_AVAILABLE_KB:-1048576}"
MANAGER_MAX_DISK_USED_PERCENT="${HOST112_MANAGER_MAX_DISK_USED_PERCENT:-95}"
STATE_DIR="${HOST112_RECOVERY_STATE_DIR:-/var/lib/awoooi-host112-recovery}"
RETRY_MARKER="${HOST112_INDEXER_RETRY_MARKER:-/run/awoooi-host112-indexer-retry.epoch}"
INDEXER_RETRY_MARKER="${HOST112_INDEXER_RETRY_MARKER:-/run/awoooi-host112-indexer-retry.epoch}"
MANAGER_RETRY_MARKER="${HOST112_MANAGER_RETRY_MARKER:-/run/awoooi-host112-manager-retry.epoch}"
MANAGER_LOCK_FILE="${HOST112_MANAGER_LOCK_FILE:-/run/awoooi-host112-manager-recovery.lock}"
usage() {
cat <<'USAGE'
Usage: host112-guest-readiness.sh [--check|--apply]
Usage:
host112-guest-readiness.sh --check [identity]
host112-guest-readiness.sh --dry-run --trace-id ID --run-id ID --work-item-id ID
host112-guest-readiness.sh --apply --trace-id ID --run-id ID --work-item-id ID
--check emits a bounded no-secret guest readiness row.
Internal systemd mode:
host112-guest-readiness.sh --apply-timer
--check emits a bounded no-secret guest readiness row and never writes.
--dry-run evaluates the exact Wazuh manager preflight and never writes.
--apply performs only allowlisted idempotent recovery for graphical.target,
LightDM, open-vm-tools, SSH and a cooldown-guarded Wazuh Indexer start retry.
LightDM, open-vm-tools, SSH, Wazuh Indexer, and one cooldown-guarded Wazuh
manager reset/start attempt. The independent verifier requires the manager
process plus agent event 1514, enrollment 1515, and manager API 55000 listeners.
USAGE
}
set_mode() {
local requested="$1"
if [ "$MODE_EXPLICIT" -eq 1 ]; then
echo "BLOCKER duplicate_mode" >&2
exit 64
fi
MODE="$requested"
MODE_EXPLICIT=1
}
require_value() {
local flag="$1" value="${2:-}"
if [ -z "$value" ]; then
echo "BLOCKER missing_value=$flag" >&2
exit 64
fi
}
while [ "$#" -gt 0 ]; do
case "$1" in
--check) MODE="check" ;;
--apply) MODE="apply" ;;
--check) set_mode "check" ;;
--dry-run) set_mode "dry_run" ;;
--apply) set_mode "apply" ;;
--apply-timer) set_mode "timer_apply" ;;
--trace-id)
require_value "$1" "${2:-}"
TRACE_ID="$2"
shift
;;
--run-id)
require_value "$1" "${2:-}"
RUN_ID="$2"
shift
;;
--work-item-id)
require_value "$1" "${2:-}"
WORK_ITEM_ID="$2"
shift
;;
-h|--help) usage; exit 0 ;;
*) echo "BLOCKER unknown_argument=$1" >&2; exit 64 ;;
esac
shift
done
safe_identity() {
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$ ]]
}
identity_count=0
[ -n "$TRACE_ID" ] && identity_count=$((identity_count + 1))
[ -n "$RUN_ID" ] && identity_count=$((identity_count + 1))
[ -n "$WORK_ITEM_ID" ] && identity_count=$((identity_count + 1))
if [ "$identity_count" -ne 0 ] && [ "$identity_count" -ne 3 ]; then
echo "BLOCKER incomplete_control_identity" >&2
exit 64
fi
if [ "$identity_count" -eq 3 ]; then
for identity in "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID"; do
if ! safe_identity "$identity"; then
echo "BLOCKER unsafe_control_identity" >&2
exit 64
fi
done
fi
if { [ "$MODE" = "dry_run" ] || [ "$MODE" = "apply" ]; } && [ "$identity_count" -ne 3 ]; then
echo "BLOCKER control_identity_required" >&2
exit 64
fi
if [ "$MODE" = "timer_apply" ] && [ "$identity_count" -ne 0 ]; then
echo "BLOCKER timer_identity_is_system_generated" >&2
exit 64
fi
actions=()
runtime_write_performed=0
manager_runtime_write_performed=0
receipt_write_performed=0
receipt_status="not_requested"
receipt_ref="none"
manager_attempt_count=0
manager_start_exit=not_attempted
manager_terminal="observed"
rollback_attempted=0
rollback_verified=0
rollback_terminal=not_required
is_uint() {
[[ "$1" =~ ^[0-9]+$ ]]
}
for timing_value in \
"$INDEXER_START_TIMEOUT_SECONDS" \
"$SERVICE_START_TIMEOUT_SECONDS" \
"$MANAGER_RESET_TIMEOUT_SECONDS" \
"$MANAGER_START_TIMEOUT_SECONDS" \
"$MANAGER_VERIFY_TIMEOUT_SECONDS" \
"$MANAGER_VERIFY_INTERVAL_SECONDS" \
"$MANAGER_ROLLBACK_TIMEOUT_SECONDS"; do
if ! is_uint "$timing_value" || [ "$timing_value" -le 0 ]; then
echo "BLOCKER invalid_timeout_configuration" >&2
exit 64
fi
done
APPLY_PHASE_TIMEOUT_BUDGET_SECONDS=$((
(4 * SERVICE_START_TIMEOUT_SECONDS)
+ INDEXER_START_TIMEOUT_SECONDS
+ MANAGER_RESET_TIMEOUT_SECONDS
+ MANAGER_START_TIMEOUT_SECONDS
+ MANAGER_VERIFY_TIMEOUT_SECONDS
+ MANAGER_ROLLBACK_TIMEOUT_SECONDS
))
if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt 570 ]; then
echo "BLOCKER apply_phase_timeout_budget_exceeds_570_seconds" >&2
exit 64
fi
write_epoch_marker() {
local marker="$1" epoch="$2" tmp
tmp="${marker}.tmp.$$"
if printf '%s\n' "$epoch" >"$tmp" \
&& chmod 0644 "$tmp" \
&& mv "$tmp" "$marker"; then
return 0
fi
rm -f "$tmp" >/dev/null 2>&1 || true
return 1
}
unit_active() {
local unit="$1" state
@@ -44,8 +194,14 @@ unit_enabled() {
printf '%s' "${state:-unknown}"
}
unit_property() {
local unit="$1" property="$2" state
state="$(systemctl show "$unit" -p "$property" --value 2>/dev/null || true)"
printf '%s' "${state:-unknown}"
}
unit_loaded() {
[ "$(systemctl show "$1" -p LoadState --value 2>/dev/null || true)" = "loaded" ]
[ "$(unit_property "$1" LoadState)" = "loaded" ]
}
record_action() {
@@ -58,8 +214,9 @@ start_if_inactive() {
if [ "$state" = "active" ] || [ "$state" = "activating" ]; then
return 0
fi
if systemctl start "$unit"; then
if timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl start "$unit"; then
record_action "$action"
runtime_write_performed=1
else
record_action "${action}_failed"
fi
@@ -73,21 +230,225 @@ enable_if_disabled() {
fi
if systemctl enable "$unit" >/dev/null; then
record_action "$action"
runtime_write_performed=1
else
record_action "${action}_failed"
fi
}
if [ "$MODE" = "apply" ]; then
tcp_port_listening() {
local port="$1"
ss -H -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$port$"
}
udp_port_listening() {
local port="$1"
ss -H -lun 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)$port$"
}
process_count() {
local process_name="$1" count
count="$(pgrep -xc "$process_name" 2>/dev/null || true)"
printf '%s' "${count:-0}"
}
memory_available_kb() {
local available
available="$(awk '/^MemAvailable:/ {print $2; exit}' /proc/meminfo 2>/dev/null || true)"
printf '%s' "${available:-0}"
}
disk_readback() {
local target="/var/ossec" line
[ -e "$target" ] || target="/"
line="$(df -Pk "$target" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $4 " " $5; exit}')"
if [ -n "$line" ]; then
printf '%s' "$line"
else
printf '0 100'
fi
}
capture_manager_state() {
local disk_values
manager_load_state="$(unit_property wazuh-manager.service LoadState)"
manager_active_state="$(unit_property wazuh-manager.service ActiveState)"
manager_sub_state="$(unit_property wazuh-manager.service SubState)"
manager_result="$(unit_property wazuh-manager.service Result)"
manager_failed_state_observed=0
if [ "$manager_active_state" = "failed" ] \
|| { [ "$manager_result" != "success" ] && [ "$manager_result" != "unknown" ]; }; then
manager_failed_state_observed=1
fi
analysisd_count="$(process_count wazuh-analysisd)"
event_1514_tcp=0
event_1514_udp=0
enrollment_1515_tcp=0
manager_api_55000_tcp=0
tcp_port_listening 1514 && event_1514_tcp=1
udp_port_listening 1514 && event_1514_udp=1
tcp_port_listening 1515 && enrollment_1515_tcp=1
tcp_port_listening 55000 && manager_api_55000_tcp=1
manager_event_port_ready=0
if [ "$event_1514_tcp" -eq 1 ] || [ "$event_1514_udp" -eq 1 ]; then
manager_event_port_ready=1
fi
manager_orphan_analysisd=0
if [ "$manager_active_state" != "active" ] && is_uint "$analysisd_count" && [ "$analysisd_count" -gt 0 ]; then
manager_orphan_analysisd=1
fi
manager_verifier_ready=0
if [ "$manager_active_state" = "active" ] \
&& is_uint "$analysisd_count" && [ "$analysisd_count" -gt 0 ] \
&& [ "$manager_event_port_ready" -eq 1 ] \
&& [ "$enrollment_1515_tcp" -eq 1 ] \
&& [ "$manager_api_55000_tcp" -eq 1 ]; then
manager_verifier_ready=1
fi
manager_memory_available_kb="$(memory_available_kb)"
disk_values="$(disk_readback)"
read -r manager_disk_available_kb manager_disk_used_percent <<<"$disk_values"
manager_resource_ready=0
if is_uint "$manager_memory_available_kb" \
&& is_uint "$manager_disk_available_kb" \
&& is_uint "$manager_disk_used_percent" \
&& [ "$manager_memory_available_kb" -ge "$MANAGER_MIN_MEMORY_AVAILABLE_KB" ] \
&& [ "$manager_disk_available_kb" -ge "$MANAGER_MIN_DISK_AVAILABLE_KB" ] \
&& [ "$manager_disk_used_percent" -lt "$MANAGER_MAX_DISK_USED_PERCENT" ]; then
manager_resource_ready=1
fi
}
manager_cooldown_remaining() {
local now_epoch="$1" last_retry_epoch=0 elapsed
if [ -r "$MANAGER_RETRY_MARKER" ]; then
read -r last_retry_epoch <"$MANAGER_RETRY_MARKER" || last_retry_epoch=0
fi
is_uint "$last_retry_epoch" || last_retry_epoch=0
elapsed=$((now_epoch - last_retry_epoch))
if [ "$elapsed" -lt 0 ]; then
printf '%s' "$MANAGER_RETRY_COOLDOWN_SECONDS"
elif [ "$elapsed" -lt "$MANAGER_RETRY_COOLDOWN_SECONDS" ]; then
printf '%s' "$((MANAGER_RETRY_COOLDOWN_SECONDS - elapsed))"
else
printf '0'
fi
}
evaluate_manager_preflight() {
local now_epoch="$1" uptime_for_retry="$2"
manager_cooldown_remaining_seconds="$(manager_cooldown_remaining "$now_epoch")"
manager_preflight_status="ready"
if ! command -v systemctl >/dev/null 2>&1 \
|| ! command -v timeout >/dev/null 2>&1 \
|| ! command -v ss >/dev/null 2>&1 \
|| ! command -v flock >/dev/null 2>&1; then
manager_preflight_status="blocked_required_tool_missing"
elif [ "$manager_load_state" != "loaded" ]; then
manager_preflight_status="blocked_manager_unit_not_loaded"
elif [ "$manager_verifier_ready" -eq 1 ]; then
manager_preflight_status="already_verified_healthy"
elif [ "$manager_active_state" = "active" ]; then
manager_preflight_status="blocked_active_verifier_mismatch"
elif [ "$manager_orphan_analysisd" -eq 1 ]; then
manager_preflight_status="blocked_orphan_analysisd_no_write"
elif [ "$manager_resource_ready" -ne 1 ]; then
manager_preflight_status="blocked_resource_preflight_no_write"
elif [ "$(unit_active wazuh-indexer.service)" != "active" ]; then
manager_preflight_status="blocked_indexer_dependency_no_write"
elif ! is_uint "$uptime_for_retry" || [ "$uptime_for_retry" -lt "$MANAGER_RETRY_AFTER_SECONDS" ]; then
manager_preflight_status="blocked_boot_settle_no_write"
elif [ "$manager_cooldown_remaining_seconds" -gt 0 ]; then
manager_preflight_status="blocked_retry_cooldown_no_write"
fi
}
wait_for_manager_verifier() {
local deadline now
deadline=$(( $(date +%s) + MANAGER_VERIFY_TIMEOUT_SECONDS ))
while true; do
capture_manager_state
[ "$manager_verifier_ready" -eq 1 ] && return 0
now="$(date +%s)"
[ "$now" -ge "$deadline" ] && return 1
sleep "$MANAGER_VERIFY_INTERVAL_SECONDS"
done
}
rollback_manager_to_inactive() {
rollback_attempted=1
rollback_terminal="bounded_stop_attempted"
if timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service; then
record_action "rollback_stop_wazuh_manager"
else
record_action "rollback_stop_wazuh_manager_failed"
fi
capture_manager_state
if [ "$manager_active_state" != "active" ] \
&& is_uint "$analysisd_count" && [ "$analysisd_count" -eq 0 ] \
&& [ "$manager_event_port_ready" -eq 0 ] \
&& [ "$enrollment_1515_tcp" -eq 0 ] \
&& [ "$manager_api_55000_tcp" -eq 0 ]; then
rollback_verified=1
rollback_terminal="bounded_stop_verified"
else
rollback_terminal="bounded_stop_not_verified"
fi
}
write_public_receipt() {
local receipt_name receipt_path tmp generated_at receipt_line
receipt_name="$(printf '%s' "$RUN_ID" | tr -c 'A-Za-z0-9._+-' '_')"
[ -n "$receipt_name" ] || receipt_name="unknown"
if [ "$MODE" = "timer_apply" ]; then
receipt_path="$STATE_DIR/last-timer-manager-recovery-receipt.txt"
else
receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
fi
generated_at="$(date --iso-8601=seconds)"
receipt_line="schema_version=host112_wazuh_manager_recovery_receipt_v1 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID before_active=$before_manager_active before_result=$before_manager_result before_orphan_analysisd=$before_manager_orphan before_memory_available_kb=$before_manager_memory before_disk_available_kb=$before_manager_disk before_disk_used_percent=$before_manager_disk_used before_agent_event_1514_ready=$before_event_1514_ready before_agent_enrollment_1515_tcp=$before_enrollment_1515 before_manager_api_55000_tcp=$before_manager_api_55000 preflight=$manager_preflight_status attempt_count=$manager_attempt_count start_exit=$manager_start_exit after_active=$manager_active_state after_result=$manager_result after_orphan_analysisd=$manager_orphan_analysisd after_agent_event_1514_ready=$manager_event_port_ready after_agent_enrollment_1515_tcp=$enrollment_1515_tcp after_manager_api_55000_tcp=$manager_api_55000_tcp verifier_ready=$manager_verifier_ready runtime_write_performed=$runtime_write_performed manager_runtime_write_performed=$manager_runtime_write_performed rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
if install -d -m 0750 "$(dirname "$receipt_path")"; then
tmp="${receipt_path}.tmp.$$"
if printf '%s\n' "$receipt_line" >"$tmp" \
&& chmod 0640 "$tmp" \
&& mv "$tmp" "$receipt_path"; then
receipt_write_performed=1
receipt_status="written"
receipt_ref="host112-manager-recovery:$RUN_ID"
return 0
fi
rm -f "$tmp" >/dev/null 2>&1 || true
fi
receipt_status="write_failed"
return 1
}
boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)"
uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)"
if [ "$MODE" = "timer_apply" ]; then
invocation_identity="${INVOCATION_ID:-${boot_id}-$(date +%s)}"
invocation_identity="$(printf '%s' "$invocation_identity" | tr -c 'A-Za-z0-9._+-' '-')"
TRACE_ID="host112-timer:$invocation_identity"
RUN_ID="host112-timer:$invocation_identity"
WORK_ITEM_ID="HOST112-WAZUH-MANAGER-RECOVERY"
fi
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
if [ "${EUID:-$(id -u)}" -ne 0 ]; then
echo "BLOCKER host112_apply_requires_root" >&2
exit 77
fi
exec 9>"$MANAGER_LOCK_FILE"
if ! flock -n 9; then
echo "BLOCKER host112_manager_single_flight_busy" >&2
exit 75
fi
current_default="$(systemctl get-default 2>/dev/null || true)"
if [ "$current_default" != "graphical.target" ]; then
if systemctl set-default graphical.target >/dev/null; then
record_action "set_default_graphical_target"
runtime_write_performed=1
else
record_action "set_default_graphical_target_failed"
fi
@@ -103,27 +464,29 @@ if [ "$MODE" = "apply" ]; then
record_action "ssh_service_missing"
fi
uptime_for_retry="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)"
indexer_state="$(unit_active wazuh-indexer.service)"
if unit_loaded wazuh-indexer.service \
&& [ "$indexer_state" != "active" ] \
&& [ "$indexer_state" != "activating" ] \
&& [ "$uptime_for_retry" -ge "$INDEXER_RETRY_AFTER_SECONDS" ]; then
&& is_uint "$uptime_seconds" \
&& [ "$uptime_seconds" -ge "$INDEXER_RETRY_AFTER_SECONDS" ]; then
now_epoch="$(date +%s)"
last_retry_epoch=0
if [ -r "$RETRY_MARKER" ]; then
read -r last_retry_epoch <"$RETRY_MARKER" || last_retry_epoch=0
fi
if ! [[ "$last_retry_epoch" =~ ^[0-9]+$ ]]; then
last_retry_epoch=0
if [ -r "$INDEXER_RETRY_MARKER" ]; then
read -r last_retry_epoch <"$INDEXER_RETRY_MARKER" || last_retry_epoch=0
fi
is_uint "$last_retry_epoch" || last_retry_epoch=0
if [ $((now_epoch - last_retry_epoch)) -ge "$INDEXER_RETRY_COOLDOWN_SECONDS" ]; then
printf '%s\n' "$now_epoch" >"$RETRY_MARKER"
systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true
if timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then
record_action "retry_wazuh_indexer"
if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then
systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true
runtime_write_performed=1
if timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then
record_action "retry_wazuh_indexer"
else
record_action "retry_wazuh_indexer_failed"
fi
else
record_action "retry_wazuh_indexer_failed"
record_action "wazuh_indexer_marker_write_failed_no_write"
fi
else
record_action "wazuh_indexer_retry_cooldown"
@@ -131,8 +494,72 @@ if [ "$MODE" = "apply" ]; then
fi
fi
boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)"
uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo unknown)"
capture_manager_state
before_manager_active="$manager_active_state"
before_manager_result="$manager_result"
before_manager_orphan="$manager_orphan_analysisd"
before_manager_memory="$manager_memory_available_kb"
before_manager_disk="$manager_disk_available_kb"
before_manager_disk_used="$manager_disk_used_percent"
before_event_1514_ready="$manager_event_port_ready"
before_enrollment_1515="$enrollment_1515_tcp"
before_manager_api_55000="$manager_api_55000_tcp"
now_epoch="$(date +%s)"
evaluate_manager_preflight "$now_epoch" "$uptime_seconds"
if [ "$MODE" = "dry_run" ]; then
case "$manager_preflight_status" in
ready) manager_terminal="dry_run_ready" ;;
already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;;
*) manager_terminal="dry_run_${manager_preflight_status}" ;;
esac
elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
case "$manager_preflight_status" in
already_verified_healthy)
manager_terminal="idempotent_already_verified_healthy"
;;
ready)
if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then
manager_attempt_count=1
if timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then
record_action "reset_failed_wazuh_manager"
runtime_write_performed=1
manager_runtime_write_performed=1
if timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then
manager_start_exit=0
else
manager_start_exit=$?
fi
if wait_for_manager_verifier; then
record_action "start_wazuh_manager_verified"
manager_terminal="verified_healthy"
else
record_action "start_wazuh_manager_verifier_failed"
rollback_manager_to_inactive
if [ "$rollback_verified" -eq 1 ]; then
manager_terminal="rolled_back_verifier_failed"
else
manager_terminal="rollback_failed_degraded"
fi
fi
else
manager_start_exit="reset_failed_no_start"
record_action "reset_failed_wazuh_manager_failed_no_start"
manager_terminal="no_start_reset_failed"
fi
else
record_action "wazuh_manager_marker_write_failed_no_write"
manager_terminal="no_write_cooldown_marker_write_failed"
fi
;;
*)
manager_terminal="no_write_${manager_preflight_status}"
;;
esac
capture_manager_state
write_public_receipt || manager_terminal="${manager_terminal}_receipt_failed"
fi
systemd_state="$(systemctl is-system-running 2>/dev/null || true)"
systemd_state="${systemd_state:-unknown}"
default_target="$(systemctl get-default 2>/dev/null || true)"
@@ -151,9 +578,7 @@ startup_enabled="$(unit_enabled awoooi-host112-guest-recovery.timer)"
startup_active="$(unit_active awoooi-host112-guest-recovery.timer)"
ssh_port_listening=0
if ss -H -ltn 2>/dev/null | awk '{print $4}' | grep -Eq '(^|:)22$'; then
ssh_port_listening=1
fi
tcp_port_listening 22 && ssh_port_listening=1
ssh_ready=0
if [ "$ssh_service" = "active" ] \
&& { [ "$ssh_enabled" = "enabled" ] || [ "$ssh_enabled" = "static" ]; } \
@@ -178,9 +603,9 @@ fi
services_ready=0
if [ "$wazuh_indexer" = "active" ] \
&& [ "$wazuh_manager" = "active" ] \
&& [ "$wazuh_dashboard" = "active" ] \
&& [ "$filebeat" = "active" ]; then
&& [ "$filebeat" = "active" ] \
&& [ "$manager_verifier_ready" -eq 1 ]; then
services_ready=1
fi
@@ -203,15 +628,30 @@ if [ "${#actions[@]}" -gt 0 ]; then
action_text="$(IFS=,; printf '%s' "${actions[*]}")"
fi
readback="boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready action_count=${#actions[@]} actions=$action_text"
trace_output="${TRACE_ID:-none}"
run_output="${RUN_ID:-none}"
work_item_output="${WORK_ITEM_ID:-none}"
readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS runtime_write_performed=$runtime_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal terminal=$manager_terminal action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
printf '%s\n' "$readback"
if [ "$MODE" = "apply" ]; then
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
install -d -m 0750 "$STATE_DIR"
tmp_readback="${STATE_DIR}/last-readback.txt.tmp.$$"
printf 'generated_at=%s mode=%s %s\n' "$(date --iso-8601=seconds)" "$MODE" "$readback" >"$tmp_readback"
chmod 0640 "$tmp_readback"
mv "$tmp_readback" "${STATE_DIR}/last-readback.txt"
if printf 'generated_at=%s %s\n' "$(date --iso-8601=seconds)" "$readback" >"$tmp_readback" \
&& chmod 0640 "$tmp_readback" \
&& mv "$tmp_readback" "${STATE_DIR}/last-readback.txt"; then
:
else
rm -f "$tmp_readback" >/dev/null 2>&1 || true
exit 74
fi
fi
[ "$guest_ready" -eq 1 ]
if [ "$MODE" = "dry_run" ]; then
case "$manager_preflight_status" in
ready|already_verified_healthy) exit 0 ;;
*) exit 1 ;;
esac
fi
[ "$guest_ready" -eq 1 ] && [ "$receipt_status" != "write_failed" ]

View File

@@ -1,4 +1,11 @@
#!/usr/bin/env bash
# Host 112 guest recovery installer.
# Version: 2.0
# Last modified: 2026-07-14 11:00 (Asia/Taipei)
# Last modified by: Codex
# Change: install and roll back the argument-regex sudo boundary required for
# correlated Wazuh manager recovery receipts.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -16,6 +23,7 @@ SERVICE_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.service"
TIMER_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.timer"
SUDOERS_TARGET="/etc/sudoers.d/awoooi-host112-agent99"
STATE_ROOT="/var/lib/awoooi-host112-recovery"
SUDOERS_ARGUMENT_REGEX='^--apply --trace-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --run-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --work-item-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$'
usage() {
cat <<'USAGE'
@@ -44,6 +52,20 @@ while [ "$#" -gt 0 ]; do
shift
done
sudo_policy_readback() {
if [ "${EUID:-$(id -u)}" -eq 0 ]; then
sudo -n -U "$TARGET_USER" -l "$@" 2>/dev/null || true
else
sudo -n -l "$@" 2>/dev/null || true
fi
}
sudo_policy_has_nopasswd() {
local output
output="$(sudo_policy_readback "$@")"
printf '%s\n' "$output" | grep -Fq "NOPASSWD: $READINESS_TARGET"
}
check() {
echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1"
echo "MODE=check"
@@ -56,6 +78,32 @@ check() {
done
echo "TIMER_ENABLED=$(systemctl is-enabled awoooi-host112-guest-recovery.timer 2>/dev/null || true)"
echo "TIMER_ACTIVE=$(systemctl is-active awoooi-host112-guest-recovery.timer 2>/dev/null || true)"
if [ -f "$SUDOERS_TARGET" ] && visudo -cf "$SUDOERS_TARGET" >/dev/null 2>&1; then
echo "SUDOERS_VALID=1"
else
echo "SUDOERS_VALID=0"
fi
if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply \
--trace-id host112-policy-check \
--run-id host112-policy-check \
--work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then
echo "SUDO_CORRELATED_APPLY_NOPASSWD=1"
else
echo "SUDO_CORRELATED_APPLY_NOPASSWD=0"
fi
if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply; then
echo "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=1"
else
echo "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=0"
fi
if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply \
--trace-id 'unsafe value' \
--run-id host112-policy-check \
--work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then
echo "SUDO_UNSAFE_IDENTITY_NOPASSWD=1"
else
echo "SUDO_UNSAFE_IDENTITY_NOPASSWD=0"
fi
if [ -x "$READINESS_TARGET" ]; then
"$READINESS_TARGET" --check || true
fi
@@ -152,7 +200,8 @@ install -m 0644 "$TIMER_SOURCE" "$TIMER_TARGET"
sudoers_tmp="$(mktemp)"
trap 'rm -f "$sudoers_tmp"' EXIT
printf '%s ALL=(root) NOPASSWD: %s --apply\n' "$TARGET_USER" "$READINESS_TARGET" >"$sudoers_tmp"
printf '%s ALL=(root) NOPASSWD: %s %s\n' \
"$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_ARGUMENT_REGEX" >"$sudoers_tmp"
visudo -cf "$sudoers_tmp" >/dev/null
install -m 0440 "$sudoers_tmp" "$SUDOERS_TARGET"

View File

@@ -791,8 +791,8 @@ def source_controls() -> dict[str, bool]:
source_file(
"scripts/reboot-recovery/awoooi-host112-guest-recovery.service"
),
"ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply",
"TimeoutStartSec=300",
"ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer",
"TimeoutStartSec=600",
),
"host_112_guest_recovery_timer_source_present": file_contains(
source_file(

View File

@@ -0,0 +1,281 @@
from __future__ import annotations
import os
import shutil
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
READINESS = ROOT / "scripts/reboot-recovery/host112-guest-readiness.sh"
INSTALLER = ROOT / "scripts/reboot-recovery/install-host112-guest-recovery.sh"
SERVICE = ROOT / "scripts/reboot-recovery/awoooi-host112-guest-recovery.service"
CONTROL = ROOT / "agent99-control-plane.ps1"
BOOTSTRAP = ROOT / "agent99-bootstrap.ps1"
def read(path: Path) -> str:
return path.read_text(encoding="utf-8")
def isolated_env(tmp_path: Path) -> dict[str, str]:
env = os.environ.copy()
env.update(
{
"HOST112_RECOVERY_STATE_DIR": str(tmp_path / "state"),
"HOST112_INDEXER_RETRY_MARKER": str(tmp_path / "indexer.epoch"),
"HOST112_MANAGER_RETRY_MARKER": str(tmp_path / "manager.epoch"),
"HOST112_MANAGER_LOCK_FILE": str(tmp_path / "manager.lock"),
}
)
return env
def test_check_and_dry_run_are_real_no_write_execution_paths(tmp_path: Path) -> None:
identity = [
"--trace-id",
"trace:D037-host112-dry-run",
"--run-id",
"run:D037-host112-dry-run",
"--work-item-id",
"D037-WAZUH-HOST112",
]
env = isolated_env(tmp_path)
check = subprocess.run(
["bash", str(READINESS), "--check", *identity],
text=True,
capture_output=True,
env=env,
check=False,
timeout=20,
)
dry_run = subprocess.run(
["bash", str(READINESS), "--dry-run", *identity],
text=True,
capture_output=True,
env=env,
check=False,
timeout=20,
)
assert check.returncode in {0, 1}
assert dry_run.returncode in {0, 1}
for result, mode in ((check, "check"), (dry_run, "dry_run")):
output = result.stdout + result.stderr
assert f"mode={mode}" in output
assert "trace_id=trace:D037-host112-dry-run" in output
assert "run_id=run:D037-host112-dry-run" in output
assert "work_item_id=D037-WAZUH-HOST112" in output
assert "runtime_write_performed=0" in output
assert "manager_runtime_write_performed=0" in output
assert "receipt_write_performed=0" in output
assert "manager_preflight_status=" in output
assert "apply_phase_timeout_budget_seconds=500" in output
assert not (tmp_path / "state").exists()
assert not (tmp_path / "indexer.epoch").exists()
assert not (tmp_path / "manager.epoch").exists()
assert not (tmp_path / "manager.lock").exists()
def test_unsafe_or_incomplete_identity_fails_before_any_state_path(tmp_path: Path) -> None:
env = isolated_env(tmp_path)
unsafe = subprocess.run(
[
"bash",
str(READINESS),
"--dry-run",
"--trace-id",
"unsafe value",
"--run-id",
"run-safe",
"--work-item-id",
"D037-WAZUH-HOST112",
],
text=True,
capture_output=True,
env=env,
check=False,
timeout=10,
)
incomplete = subprocess.run(
[
"bash",
str(READINESS),
"--dry-run",
"--trace-id",
"trace-safe",
],
text=True,
capture_output=True,
env=env,
check=False,
timeout=10,
)
assert unsafe.returncode == 64
assert "BLOCKER unsafe_control_identity" in (unsafe.stdout + unsafe.stderr)
assert incomplete.returncode == 64
assert "BLOCKER incomplete_control_identity" in (
incomplete.stdout + incomplete.stderr
)
assert not any(tmp_path.iterdir())
def test_manager_preflight_verifier_receipt_and_rollback_are_complete() -> None:
source = read(READINESS)
for field in (
"manager_failed_state_observed",
"manager_orphan_analysisd",
"manager_memory_available_kb",
"manager_disk_available_kb",
"manager_disk_used_percent",
"wazuh_analysisd_process_count",
"wazuh_agent_event_port_1514_tcp",
"wazuh_agent_event_port_1514_udp",
"wazuh_agent_enrollment_port_1515_tcp",
"wazuh_manager_api_port_55000_tcp",
"manager_independent_verifier_ready",
):
assert field in source
assert 'manager_active_state="$(unit_property wazuh-manager.service ActiveState)"' in source
assert 'manager_result="$(unit_property wazuh-manager.service Result)"' in source
assert 'analysisd_count="$(process_count wazuh-analysisd)"' in source
assert source.count(
'timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service'
) == 1
assert "manager_attempt_count=1" in source
assert 'timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service' in source
assert 'rollback_terminal="bounded_stop_verified"' in source
assert "schema_version=host112_wazuh_manager_recovery_receipt_v1" in source
assert "trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID" in source
assert 'receipt_status="write_failed"' in source
assert "prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" in source
preflight = source[
source.index("evaluate_manager_preflight() {") : source.index(
"wait_for_manager_verifier() {"
)
]
assert 'elif [ "$(unit_active wazuh-indexer.service)" != "active" ]; then' in preflight
assert 'manager_preflight_status="blocked_indexer_dependency_no_write"' in preflight
def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None:
source = read(READINESS)
marker_function = source[
source.index("write_epoch_marker() {") : source.index("unit_active() {")
]
assert 'tmp="${marker}.tmp.$$"' in marker_function
assert 'printf \'%s\\n\' "$epoch" >"$tmp"' in marker_function
assert 'chmod 0644 "$tmp"' in marker_function
assert 'mv "$tmp" "$marker"' in marker_function
assert "return 1" in marker_function
manager_start = source.index(
' if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then'
)
manager_end = source.index(" ;;", manager_start)
manager_branch = source[manager_start:manager_end]
marker_failure_at = manager_branch.index(
' else\n record_action "wazuh_manager_marker_write_failed_no_write"'
)
success = manager_branch[:marker_failure_at]
failure = manager_branch[marker_failure_at:]
assert "systemctl reset-failed wazuh-manager.service" in success
assert "systemctl start wazuh-manager.service" in success
assert 'timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service' in success
assert "manager_attempt_count=1" in success
assert "systemctl reset-failed wazuh-manager.service" not in failure
assert "systemctl start wazuh-manager.service" not in failure
assert "wazuh_manager_marker_write_failed_no_write" in failure
assert 'manager_terminal="no_write_cooldown_marker_write_failed"' in failure
indexer_start = source.index(
' if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then'
)
indexer_end = source.index(" else\n record_action", indexer_start)
indexer_branch = source[indexer_start:indexer_end]
indexer_success, indexer_failure = indexer_branch.split(" else\n", maxsplit=1)
assert "systemctl reset-failed wazuh-indexer.service" in indexer_success
assert "systemctl start wazuh-indexer.service" in indexer_success
assert "systemctl reset-failed wazuh-indexer.service" not in indexer_failure
assert "systemctl start wazuh-indexer.service" not in indexer_failure
assert "wazuh_indexer_marker_write_failed_no_write" in indexer_failure
def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None:
source = read(READINESS)
service = read(SERVICE)
control = read(CONTROL)
assert 'SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-30}"' in source
assert 'INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-180}"' in source
assert 'MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-120}"' in source
assert 'MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}"' in source
assert 'MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}"' in source
assert 'MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-45}"' in source
assert 'if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt 570 ]; then' in source
assert "TimeoutStartSec=600" in service
assert "Invoke-HostSshText $targetHost $applyCommand 590 1" in control
def test_installer_uses_argument_regex_and_post_install_policy_readback() -> None:
source = read(INSTALLER)
assert "SUDOERS_ARGUMENT_REGEX='^--apply --trace-id " in source
assert "--run-id [A-Za-z0-9]" in source
assert "--work-item-id [A-Za-z0-9]" in source
assert '"$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_ARGUMENT_REGEX"' in source
assert 'visudo -cf "$sudoers_tmp"' in source
assert "SUDO_CORRELATED_APPLY_NOPASSWD" in source
assert "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD" in source
assert "SUDO_UNSAFE_IDENTITY_NOPASSWD" in source
assert "--trace-id 'unsafe value'" in source
assert "--rollback" in source
def test_exact_sudoers_argument_regex_parses_when_visudo_is_available() -> None:
visudo = shutil.which("visudo")
if not visudo:
return
candidate = (
"kali ALL=(root) NOPASSWD: "
"/usr/local/sbin/awoooi-host112-guest-readiness "
"^--apply --trace-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} "
"--run-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} "
"--work-item-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$\n"
)
result = subprocess.run(
[visudo, "-cf", "-"],
input=candidate,
text=True,
capture_output=True,
check=False,
timeout=10,
)
assert result.returncode == 0, result.stdout + result.stderr
def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
control = read(CONTROL)
bootstrap = read(BOOTSTRAP)
function = control[control.index("function Convert-AgentHost112GuestReadback") :]
function = function[: function.index("function Test-HostReachability")]
for parameter in ("AutomationRunId", "TraceId", "WorkItemId"):
assert f"[string]${parameter} = \"\"" in control
assert f'[string]`${parameter} = ""' in bootstrap
assert 'reason = "controlled_dispatch_identity_unsafe"' in control
assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
assert '$before.managerPreflightStatus -eq "ready"' in function
assert 'receiptStatus -eq "written"' in function
assert '$after.managerVerified' in function
assert 'name = "wazuh_manager_service_active"' in function
assert 'name = "wazuh_analysisd_process_present"' in function
assert 'name = "wazuh_agent_event_1514_ready"' in function
assert 'name = "wazuh_agent_enrollment_1515_tcp"' in function
assert 'name = "wazuh_manager_api_55000_tcp"' in function
assert "host112_readback_transport_unavailable" in function
assert 'terminal = "no_write_control_identity_incomplete_or_unsafe"' in function

View File

@@ -107,7 +107,8 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
assert "restrict,command=" in host112_installer
assert "visudo -cf" in host112_installer
assert "--rollback" in host112_installer
assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply" in host112_service
assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer" in host112_service
assert "TimeoutStartSec=600" in host112_service
assert "OnBootSec=20s" in host112_timer
assert "OnUnitInactiveSec=60s" in host112_timer