feat(recovery): close host 112 guest readiness
This commit is contained in:
ogt
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=AWOOOI host 112 guest readiness and bounded recovery
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply
|
||||
TimeoutStartSec=300
|
||||
User=root
|
||||
Group=root
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectControlGroups=true
|
||||
RestrictSUIDSGID=true
|
||||
LockPersonality=true
|
||||
12
scripts/reboot-recovery/awoooi-host112-guest-recovery.timer
Normal file
12
scripts/reboot-recovery/awoooi-host112-guest-recovery.timer
Normal file
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=Run AWOOOI host 112 guest recovery after boot and continuously
|
||||
|
||||
[Timer]
|
||||
OnBootSec=20s
|
||||
OnUnitInactiveSec=60s
|
||||
AccuracySec=5s
|
||||
Persistent=true
|
||||
Unit=awoooi-host112-guest-recovery.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
184
scripts/reboot-recovery/host112-guest-readiness.sh
Executable file
184
scripts/reboot-recovery/host112-guest-readiness.sh
Executable file
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env bash
|
||||
# Host 112 guest readiness and bounded recovery executor.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
MODE="check"
|
||||
INDEXER_RETRY_AFTER_SECONDS="${HOST112_INDEXER_RETRY_AFTER_SECONDS:-210}"
|
||||
INDEXER_RETRY_COOLDOWN_SECONDS="${HOST112_INDEXER_RETRY_COOLDOWN_SECONDS:-600}"
|
||||
INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-240}"
|
||||
STATE_DIR="${HOST112_RECOVERY_STATE_DIR:-/var/lib/awoooi-host112-recovery}"
|
||||
RETRY_MARKER="${HOST112_INDEXER_RETRY_MARKER:-/run/awoooi-host112-indexer-retry.epoch}"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: host112-guest-readiness.sh [--check|--apply]
|
||||
|
||||
--check emits a bounded no-secret guest readiness row.
|
||||
--apply performs only allowlisted idempotent recovery for graphical.target,
|
||||
LightDM, open-vm-tools and a cooldown-guarded Wazuh Indexer start retry.
|
||||
USAGE
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--check) MODE="check" ;;
|
||||
--apply) MODE="apply" ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "BLOCKER unknown_argument=$1" >&2; exit 64 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
actions=()
|
||||
|
||||
unit_active() {
|
||||
local unit="$1" state
|
||||
state="$(systemctl is-active "$unit" 2>/dev/null || true)"
|
||||
printf '%s' "${state:-unknown}"
|
||||
}
|
||||
|
||||
unit_enabled() {
|
||||
local unit="$1" state
|
||||
state="$(systemctl is-enabled "$unit" 2>/dev/null || true)"
|
||||
printf '%s' "${state:-unknown}"
|
||||
}
|
||||
|
||||
unit_loaded() {
|
||||
[ "$(systemctl show "$1" -p LoadState --value 2>/dev/null || true)" = "loaded" ]
|
||||
}
|
||||
|
||||
record_action() {
|
||||
actions+=("$1")
|
||||
}
|
||||
|
||||
start_if_inactive() {
|
||||
local unit="$1" action="$2" state
|
||||
state="$(unit_active "$unit")"
|
||||
if [ "$state" = "active" ] || [ "$state" = "activating" ]; then
|
||||
return 0
|
||||
fi
|
||||
if systemctl start "$unit"; then
|
||||
record_action "$action"
|
||||
else
|
||||
record_action "${action}_failed"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "$MODE" = "apply" ]; then
|
||||
if [ "${EUID:-$(id -u)}" -ne 0 ]; then
|
||||
echo "BLOCKER host112_apply_requires_root" >&2
|
||||
exit 77
|
||||
fi
|
||||
|
||||
current_default="$(systemctl get-default 2>/dev/null || true)"
|
||||
if [ "$current_default" != "graphical.target" ]; then
|
||||
if systemctl set-default graphical.target >/dev/null; then
|
||||
record_action "set_default_graphical_target"
|
||||
else
|
||||
record_action "set_default_graphical_target_failed"
|
||||
fi
|
||||
fi
|
||||
|
||||
start_if_inactive graphical.target start_graphical_target
|
||||
start_if_inactive open-vm-tools.service start_vmware_tools
|
||||
start_if_inactive lightdm.service start_lightdm
|
||||
|
||||
uptime_for_retry="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)"
|
||||
indexer_state="$(unit_active wazuh-indexer.service)"
|
||||
if unit_loaded wazuh-indexer.service \
|
||||
&& [ "$indexer_state" != "active" ] \
|
||||
&& [ "$indexer_state" != "activating" ] \
|
||||
&& [ "$uptime_for_retry" -ge "$INDEXER_RETRY_AFTER_SECONDS" ]; then
|
||||
now_epoch="$(date +%s)"
|
||||
last_retry_epoch=0
|
||||
if [ -r "$RETRY_MARKER" ]; then
|
||||
read -r last_retry_epoch <"$RETRY_MARKER" || last_retry_epoch=0
|
||||
fi
|
||||
if ! [[ "$last_retry_epoch" =~ ^[0-9]+$ ]]; then
|
||||
last_retry_epoch=0
|
||||
fi
|
||||
if [ $((now_epoch - last_retry_epoch)) -ge "$INDEXER_RETRY_COOLDOWN_SECONDS" ]; then
|
||||
printf '%s\n' "$now_epoch" >"$RETRY_MARKER"
|
||||
systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true
|
||||
if timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then
|
||||
record_action "retry_wazuh_indexer"
|
||||
else
|
||||
record_action "retry_wazuh_indexer_failed"
|
||||
fi
|
||||
else
|
||||
record_action "wazuh_indexer_retry_cooldown"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)"
|
||||
uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo unknown)"
|
||||
systemd_state="$(systemctl is-system-running 2>/dev/null || true)"
|
||||
systemd_state="${systemd_state:-unknown}"
|
||||
default_target="$(systemctl get-default 2>/dev/null || true)"
|
||||
default_target="${default_target:-unknown}"
|
||||
graphical_target="$(unit_active graphical.target)"
|
||||
display_manager="$(unit_active display-manager.service)"
|
||||
lightdm="$(unit_active lightdm.service)"
|
||||
vmware_tools="$(unit_active open-vm-tools.service)"
|
||||
wazuh_indexer="$(unit_active wazuh-indexer.service)"
|
||||
wazuh_manager="$(unit_active wazuh-manager.service)"
|
||||
wazuh_dashboard="$(unit_active wazuh-dashboard.service)"
|
||||
filebeat="$(unit_active filebeat.service)"
|
||||
startup_enabled="$(unit_enabled awoooi-host112-guest-recovery.timer)"
|
||||
startup_active="$(unit_active awoooi-host112-guest-recovery.timer)"
|
||||
|
||||
xorg="inactive"
|
||||
if pgrep -x Xorg >/dev/null 2>&1 || pgrep -x Xwayland >/dev/null 2>&1; then
|
||||
xorg="active"
|
||||
fi
|
||||
|
||||
console_ready=0
|
||||
if [ "$default_target" = "graphical.target" ] \
|
||||
&& [ "$graphical_target" = "active" ] \
|
||||
&& [ "$display_manager" = "active" ] \
|
||||
&& [ "$lightdm" = "active" ] \
|
||||
&& [ "$vmware_tools" = "active" ] \
|
||||
&& [ "$xorg" = "active" ]; then
|
||||
console_ready=1
|
||||
fi
|
||||
|
||||
services_ready=0
|
||||
if [ "$wazuh_indexer" = "active" ] \
|
||||
&& [ "$wazuh_manager" = "active" ] \
|
||||
&& [ "$wazuh_dashboard" = "active" ] \
|
||||
&& [ "$filebeat" = "active" ]; then
|
||||
services_ready=1
|
||||
fi
|
||||
|
||||
recovery_timer_ready=0
|
||||
if [ "$startup_enabled" = "enabled" ] && [ "$startup_active" = "active" ]; then
|
||||
recovery_timer_ready=1
|
||||
fi
|
||||
|
||||
guest_ready=0
|
||||
if [ "$console_ready" -eq 1 ] \
|
||||
&& [ "$services_ready" -eq 1 ] \
|
||||
&& [ "$recovery_timer_ready" -eq 1 ] \
|
||||
&& [ "$systemd_state" = "running" ]; then
|
||||
guest_ready=1
|
||||
fi
|
||||
|
||||
action_text="none"
|
||||
if [ "${#actions[@]}" -gt 0 ]; then
|
||||
action_text="$(IFS=,; printf '%s' "${actions[*]}")"
|
||||
fi
|
||||
|
||||
readback="boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready action_count=${#actions[@]} actions=$action_text"
|
||||
printf '%s\n' "$readback"
|
||||
|
||||
if [ "$MODE" = "apply" ]; then
|
||||
install -d -m 0750 "$STATE_DIR"
|
||||
tmp_readback="${STATE_DIR}/last-readback.txt.tmp.$$"
|
||||
printf 'generated_at=%s mode=%s %s\n' "$(date --iso-8601=seconds)" "$MODE" "$readback" >"$tmp_readback"
|
||||
chmod 0640 "$tmp_readback"
|
||||
mv "$tmp_readback" "${STATE_DIR}/last-readback.txt"
|
||||
fi
|
||||
|
||||
[ "$guest_ready" -eq 1 ]
|
||||
183
scripts/reboot-recovery/install-host112-guest-recovery.sh
Executable file
183
scripts/reboot-recovery/install-host112-guest-recovery.sh
Executable file
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
MODE="check"
|
||||
ROLLBACK_DIR=""
|
||||
TARGET_USER="${HOST112_TARGET_USER:-kali}"
|
||||
CONTROL_SOURCE_ADDRESS="${HOST112_CONTROL_SOURCE_ADDRESS:-192.168.0.110}"
|
||||
CONTROL_PUBLIC_KEY_PATH="${HOST112_CONTROL_PUBLIC_KEY_PATH:-${SCRIPT_DIR}/control.pub}"
|
||||
READINESS_SOURCE="${SCRIPT_DIR}/host112-guest-readiness.sh"
|
||||
SERVICE_SOURCE="${SCRIPT_DIR}/awoooi-host112-guest-recovery.service"
|
||||
TIMER_SOURCE="${SCRIPT_DIR}/awoooi-host112-guest-recovery.timer"
|
||||
READINESS_TARGET="/usr/local/sbin/awoooi-host112-guest-readiness"
|
||||
INSTALLER_TARGET="/usr/local/sbin/awoooi-install-host112-guest-recovery"
|
||||
SERVICE_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.service"
|
||||
TIMER_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.timer"
|
||||
SUDOERS_TARGET="/etc/sudoers.d/awoooi-host112-agent99"
|
||||
STATE_ROOT="/var/lib/awoooi-host112-recovery"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
install-host112-guest-recovery.sh --check
|
||||
install-host112-guest-recovery.sh --apply
|
||||
install-host112-guest-recovery.sh --rollback BACKUP_DIR
|
||||
|
||||
Apply installs the bounded host-112 recovery timer, an exact-command sudo rule,
|
||||
and a forced-command readback key for host 110. It never reboots the host.
|
||||
USAGE
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--check) MODE="check" ;;
|
||||
--apply) MODE="apply" ;;
|
||||
--rollback)
|
||||
MODE="rollback"
|
||||
shift
|
||||
ROLLBACK_DIR="${1:-}"
|
||||
;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "UNKNOWN_ARG=$1" >&2; usage >&2; exit 64 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
check() {
|
||||
echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1"
|
||||
echo "MODE=check"
|
||||
for path in "$READINESS_TARGET" "$INSTALLER_TARGET" "$SERVICE_TARGET" "$TIMER_TARGET" "$SUDOERS_TARGET"; do
|
||||
if [ -f "$path" ]; then
|
||||
echo "TARGET path=$path present=1 sha256=$(sha256sum "$path" | awk '{print $1}')"
|
||||
else
|
||||
echo "TARGET path=$path present=0 sha256=missing"
|
||||
fi
|
||||
done
|
||||
echo "TIMER_ENABLED=$(systemctl is-enabled awoooi-host112-guest-recovery.timer 2>/dev/null || true)"
|
||||
echo "TIMER_ACTIVE=$(systemctl is-active awoooi-host112-guest-recovery.timer 2>/dev/null || true)"
|
||||
if [ -x "$READINESS_TARGET" ]; then
|
||||
"$READINESS_TARGET" --check || true
|
||||
fi
|
||||
}
|
||||
|
||||
require_root() {
|
||||
if [ "${EUID:-$(id -u)}" -ne 0 ]; then
|
||||
echo "BLOCKER host112_installer_requires_root" >&2
|
||||
exit 77
|
||||
fi
|
||||
}
|
||||
|
||||
backup_path() {
|
||||
local path="$1" backup_dir="$2" relative
|
||||
if [ -e "$path" ]; then
|
||||
relative="${path#/}"
|
||||
install -d "${backup_dir}/rootfs/$(dirname "$relative")"
|
||||
cp -a "$path" "${backup_dir}/rootfs/${relative}"
|
||||
printf 'present\t%s\n' "$path" >>"${backup_dir}/manifest.tsv"
|
||||
else
|
||||
printf 'missing\t%s\n' "$path" >>"${backup_dir}/manifest.tsv"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "$MODE" = "check" ]; then
|
||||
check
|
||||
exit 0
|
||||
fi
|
||||
|
||||
require_root
|
||||
|
||||
if [ "$MODE" = "rollback" ]; then
|
||||
case "$ROLLBACK_DIR" in
|
||||
"$STATE_ROOT"/backups/*) ;;
|
||||
*) echo "BLOCKER invalid_host112_rollback_dir" >&2; exit 64 ;;
|
||||
esac
|
||||
[ -f "$ROLLBACK_DIR/manifest.tsv" ] || { echo "BLOCKER rollback_manifest_missing" >&2; exit 66; }
|
||||
systemctl disable --now awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || true
|
||||
while IFS=$'\t' read -r state path; do
|
||||
[ -n "$path" ] || continue
|
||||
if [ "$state" = "present" ]; then
|
||||
install -d "$(dirname "$path")"
|
||||
cp -a "$ROLLBACK_DIR/rootfs/${path#/}" "$path"
|
||||
else
|
||||
rm -f "$path"
|
||||
fi
|
||||
done <"$ROLLBACK_DIR/manifest.tsv"
|
||||
systemctl daemon-reload
|
||||
if [ -f "$ROLLBACK_DIR/timer_was_enabled" ]; then
|
||||
systemctl enable awoooi-host112-guest-recovery.timer >/dev/null
|
||||
fi
|
||||
if [ -f "$ROLLBACK_DIR/timer_was_active" ]; then
|
||||
systemctl start awoooi-host112-guest-recovery.timer
|
||||
fi
|
||||
echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1"
|
||||
echo "MODE=rollback"
|
||||
echo "ROLLBACK_APPLIED=1"
|
||||
check
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "$CONTROL_PUBLIC_KEY_PATH"; do
|
||||
[ -f "$source" ] || { echo "BLOCKER source_missing=$source" >&2; exit 66; }
|
||||
done
|
||||
|
||||
user_home="$(getent passwd "$TARGET_USER" | cut -d: -f6)"
|
||||
user_group="$(id -gn "$TARGET_USER")"
|
||||
[ -n "$user_home" ] || { echo "BLOCKER target_user_home_missing" >&2; exit 66; }
|
||||
auth_dir="${user_home}/.ssh"
|
||||
auth_file="${auth_dir}/authorized_keys"
|
||||
|
||||
read -r key_type key_blob _ <"$CONTROL_PUBLIC_KEY_PATH"
|
||||
case "$key_type" in
|
||||
ssh-ed25519|ssh-rsa|ecdsa-sha2-*) ;;
|
||||
*) echo "BLOCKER unsupported_control_public_key_type" >&2; exit 65 ;;
|
||||
esac
|
||||
[[ "$key_blob" =~ ^[A-Za-z0-9+/=]+$ ]] || { echo "BLOCKER invalid_control_public_key_blob" >&2; exit 65; }
|
||||
key_fingerprint="$(ssh-keygen -lf "$CONTROL_PUBLIC_KEY_PATH" | awk '{print $2}')"
|
||||
|
||||
run_id="host112-guest-recovery-$(date +%Y%m%dT%H%M%S%z)-$$"
|
||||
backup_dir="${STATE_ROOT}/backups/${run_id}"
|
||||
install -d -m 0750 "$backup_dir"
|
||||
: >"${backup_dir}/manifest.tsv"
|
||||
for path in "$READINESS_TARGET" "$INSTALLER_TARGET" "$SERVICE_TARGET" "$TIMER_TARGET" "$SUDOERS_TARGET" "$auth_file"; do
|
||||
backup_path "$path" "$backup_dir"
|
||||
done
|
||||
systemctl is-enabled awoooi-host112-guest-recovery.timer >/dev/null 2>&1 && touch "$backup_dir/timer_was_enabled" || true
|
||||
systemctl is-active awoooi-host112-guest-recovery.timer >/dev/null 2>&1 && touch "$backup_dir/timer_was_active" || true
|
||||
|
||||
install -m 0755 "$READINESS_SOURCE" "$READINESS_TARGET"
|
||||
install -m 0755 "${SCRIPT_DIR}/install-host112-guest-recovery.sh" "$INSTALLER_TARGET"
|
||||
install -m 0644 "$SERVICE_SOURCE" "$SERVICE_TARGET"
|
||||
install -m 0644 "$TIMER_SOURCE" "$TIMER_TARGET"
|
||||
|
||||
sudoers_tmp="$(mktemp)"
|
||||
trap 'rm -f "$sudoers_tmp"' EXIT
|
||||
printf '%s ALL=(root) NOPASSWD: %s --apply\n' "$TARGET_USER" "$READINESS_TARGET" >"$sudoers_tmp"
|
||||
visudo -cf "$sudoers_tmp" >/dev/null
|
||||
install -m 0440 "$sudoers_tmp" "$SUDOERS_TARGET"
|
||||
|
||||
install -d -m 0700 -o "$TARGET_USER" -g "$user_group" "$auth_dir"
|
||||
if [ ! -f "$auth_file" ]; then
|
||||
install -m 0600 -o "$TARGET_USER" -g "$user_group" /dev/null "$auth_file"
|
||||
fi
|
||||
auth_tmp="$(mktemp "${auth_file}.tmp.XXXXXX")"
|
||||
awk -v blob="$key_blob" 'index($0, blob) == 0 && index($0, "awoooi-host112-probe-110") == 0' "$auth_file" >"$auth_tmp"
|
||||
printf 'from="%s",restrict,command="%s --check" %s %s awoooi-host112-probe-110\n' \
|
||||
"$CONTROL_SOURCE_ADDRESS" "$READINESS_TARGET" "$key_type" "$key_blob" >>"$auth_tmp"
|
||||
chown "$TARGET_USER:$user_group" "$auth_tmp"
|
||||
chmod 0600 "$auth_tmp"
|
||||
mv "$auth_tmp" "$auth_file"
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now awoooi-host112-guest-recovery.timer >/dev/null
|
||||
systemctl start awoooi-host112-guest-recovery.service
|
||||
|
||||
echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1"
|
||||
echo "MODE=apply"
|
||||
echo "RUN_ID=$run_id"
|
||||
echo "BACKUP_DIR=$backup_dir"
|
||||
echo "CONTROL_KEY_FINGERPRINT=$key_fingerprint"
|
||||
echo "REBOOT_PERFORMED=0"
|
||||
echo "SECRET_READ=0"
|
||||
echo "RESTRICTED_CONTROL_KEY_INSTALLED=1"
|
||||
check
|
||||
@@ -22,7 +22,7 @@ HOST_SPECS=(
|
||||
"99=192.168.0.99:vmware-host-autostart"
|
||||
"110=wooo@192.168.0.110:awoooi-startup-110.service"
|
||||
"111=ooo@192.168.0.111:com.momo.ollama111-allow-proxy"
|
||||
"112=192.168.0.112:vm-host-boot"
|
||||
"112=kali@192.168.0.112:awoooi-host112-guest-recovery.timer"
|
||||
"120=wooo@192.168.0.120:k3s.service"
|
||||
"121=wooo@192.168.0.121:k3s.service"
|
||||
"188=ollama@192.168.0.188:awoooi-startup.service"
|
||||
@@ -94,14 +94,19 @@ emit_boot_row() {
|
||||
local systemd_state="$7"
|
||||
local enabled="$8"
|
||||
local active="$9"
|
||||
local details="${10:-}"
|
||||
|
||||
printf 'HOST_BOOT alias=%s target=%s startup_unit=%s reachable=%s boot_id=%s uptime_seconds=%s systemd_state=%s startup_enabled=%s startup_active=%s\n' \
|
||||
printf 'HOST_BOOT alias=%s target=%s startup_unit=%s reachable=%s boot_id=%s uptime_seconds=%s systemd_state=%s startup_enabled=%s startup_active=%s' \
|
||||
"$alias" "$target" "$unit" "$reachable" \
|
||||
"$(escape_value "${boot_id:-unknown}")" \
|
||||
"$(escape_value "${uptime_seconds:-unknown}")" \
|
||||
"$(escape_value "${systemd_state:-unknown}")" \
|
||||
"$(escape_value "${enabled:-unknown}")" \
|
||||
"$(escape_value "${active:-unknown}")"
|
||||
if [[ -n "$details" ]]; then
|
||||
printf ' %s' "$details"
|
||||
fi
|
||||
printf '\n'
|
||||
}
|
||||
|
||||
probe_local_host() {
|
||||
@@ -204,6 +209,39 @@ Write-Output ("boot_id=windows_{0} uptime_seconds={1} systemd_state=windows_ssh
|
||||
"${systemd_state:-windows_ssh}" "${enabled:-unknown}" "${active:-unknown}"
|
||||
}
|
||||
|
||||
probe_host112_guest() {
|
||||
local alias="$1"
|
||||
local target="$2"
|
||||
local unit="$3"
|
||||
local output boot_id uptime_seconds systemd_state enabled active
|
||||
local key value details=""
|
||||
|
||||
output="$(
|
||||
run_with_timeout "$SSH_COMMAND_TIMEOUT_SECONDS" \
|
||||
ssh "${SSH_OPTS[@]}" "$target" ignored-by-forced-command 2>/dev/null
|
||||
)"
|
||||
[[ -n "$output" ]] || return 1
|
||||
|
||||
boot_id="$(sed -n 's/.*boot_id=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
||||
uptime_seconds="$(sed -n 's/.*uptime_seconds=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
||||
systemd_state="$(sed -n 's/.*systemd_state=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
||||
enabled="$(sed -n 's/.*startup_enabled=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
||||
active="$(sed -n 's/.*startup_active=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
||||
[[ -n "$boot_id" && "$uptime_seconds" =~ ^[0-9]+$ ]] || return 1
|
||||
|
||||
for key in \
|
||||
default_target graphical_target display_manager lightdm vmware_tools xorg \
|
||||
wazuh_indexer wazuh_manager wazuh_dashboard filebeat console_ready \
|
||||
services_ready recovery_timer_ready guest_ready action_count actions; do
|
||||
value="$(sed -n "s/.*${key}=\\([^ ]*\\).*/\\1/p" <<<"$output" | tail -n 1)"
|
||||
value="$(escape_value "${value:-unknown}")"
|
||||
details+="${details:+ }${key}=${value}"
|
||||
done
|
||||
|
||||
emit_boot_row "$alias" "$target" "$unit" 1 "$boot_id" "$uptime_seconds" \
|
||||
"${systemd_state:-unknown}" "${enabled:-unknown}" "${active:-unknown}" "$details"
|
||||
}
|
||||
|
||||
probe_host() {
|
||||
local alias="$1"
|
||||
local target="$2"
|
||||
@@ -221,6 +259,10 @@ probe_host() {
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$alias" == "112" ]] && probe_host112_guest "$alias" "$target" "$unit"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$target" != *@* ]]; then
|
||||
if probe_node_exporter "$alias" "$target" "$unit"; then
|
||||
return 0
|
||||
|
||||
@@ -519,6 +519,7 @@ import json, sys
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
windows99 = payload.get("windows99_vmware_autostart") or {}
|
||||
collector = payload.get("windows99_vmware_verify_collector") or {}
|
||||
host112 = (payload.get("host_boot_detection") or {}).get("host112_guest_readiness") or {}
|
||||
|
||||
|
||||
def label(value: object) -> str:
|
||||
@@ -534,6 +535,10 @@ def gauge(name: str, value: bool) -> None:
|
||||
print(f'{name}{{scope="windows99"}} {1 if value else 0}')
|
||||
|
||||
|
||||
def host112_gauge(name: str, value: bool) -> None:
|
||||
print(f'{name}{{scope="host112"}} {1 if value else 0}')
|
||||
|
||||
|
||||
def int_value(value: object, default: int = -1) -> int:
|
||||
try:
|
||||
return int(str(value), 0)
|
||||
@@ -613,6 +618,17 @@ if collector_present:
|
||||
f'{{scope="windows99",status="{label(collector_status)}",'
|
||||
f'safe_next_step="{label(collector_safe_next_step)}"}} 1'
|
||||
)
|
||||
host112_gauge(
|
||||
"awoooi_host112_guest_readback_present",
|
||||
host112.get("readback_present") is True,
|
||||
)
|
||||
host112_gauge("awoooi_host112_console_ready", host112.get("console_ready") is True)
|
||||
host112_gauge("awoooi_host112_services_ready", host112.get("services_ready") is True)
|
||||
host112_gauge(
|
||||
"awoooi_host112_recovery_timer_ready",
|
||||
host112.get("recovery_timer_ready") is True,
|
||||
)
|
||||
host112_gauge("awoooi_host112_guest_ready", host112.get("guest_ready") is True)
|
||||
PY
|
||||
)"
|
||||
active_blocker_metrics="$(python3 - "$scorecard_file" "$TARGET_MINUTES" <<'PY'
|
||||
@@ -701,6 +717,16 @@ awoooi_public_route_unreachable_without_external_l1_fallback{scope="public_route
|
||||
# HELP awoooi_public_maintenance_edge_fallback_ready Whether live edge Nginx maintenance fallback config is installed.
|
||||
# TYPE awoooi_public_maintenance_edge_fallback_ready gauge
|
||||
awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} $public_edge_fallback_ready
|
||||
# HELP awoooi_host112_guest_readback_present Whether host 112 returned its forced-command guest readiness row.
|
||||
# TYPE awoooi_host112_guest_readback_present gauge
|
||||
# HELP awoooi_host112_console_ready Whether host 112 graphical target, LightDM, Xorg and VMware Tools are ready.
|
||||
# TYPE awoooi_host112_console_ready gauge
|
||||
# HELP awoooi_host112_services_ready Whether the host 112 Wazuh stack and Filebeat are ready.
|
||||
# TYPE awoooi_host112_services_ready gauge
|
||||
# HELP awoooi_host112_recovery_timer_ready Whether host 112 bounded recovery timer is enabled and active.
|
||||
# TYPE awoooi_host112_recovery_timer_ready gauge
|
||||
# HELP awoooi_host112_guest_ready Whether host 112 guest, console, services and recovery timer are fully ready.
|
||||
# TYPE awoooi_host112_guest_ready gauge
|
||||
# HELP awoooi_windows99_vmware_readback_present Whether Windows99 VMware verifier stdout was collected.
|
||||
# TYPE awoooi_windows99_vmware_readback_present gauge
|
||||
# HELP awoooi_windows99_vmware_verify_ready Whether Windows99 VMware autostart verifier is fully green.
|
||||
|
||||
@@ -155,6 +155,7 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]:
|
||||
kv = parse_kv(text)
|
||||
vmx_present: dict[str, bool] = {}
|
||||
vm_power: dict[str, dict[str, Any]] = {}
|
||||
vm_processes: dict[str, list[dict[str, Any]]] = {}
|
||||
services: dict[str, dict[str, Any]] = {}
|
||||
policies: dict[str, dict[str, Any]] = {}
|
||||
task: dict[str, Any] = {}
|
||||
@@ -177,6 +178,24 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]:
|
||||
"source": match.group(4) or "vmrun",
|
||||
}
|
||||
continue
|
||||
match = re.match(
|
||||
r"^VM_PROCESS alias=(\S+) present=([01]) process_id=(\S+) "
|
||||
r"session_id=(\S+) owner=(\S+) interactive_session=([01]) "
|
||||
r"ui_truth=(\S+)$",
|
||||
line,
|
||||
)
|
||||
if match:
|
||||
vm_processes.setdefault(match.group(1), []).append(
|
||||
{
|
||||
"present": match.group(2) == "1",
|
||||
"process_id": match.group(3),
|
||||
"session_id": match.group(4),
|
||||
"owner": match.group(5),
|
||||
"interactive_session": match.group(6) == "1",
|
||||
"ui_truth": match.group(7),
|
||||
}
|
||||
)
|
||||
continue
|
||||
match = re.match(
|
||||
r"^VMWARE_SERVICE name=(\S+) present=([01]) status=(\S+) "
|
||||
r"startup_type=(\S+) ok=([01])$",
|
||||
@@ -258,6 +277,15 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]:
|
||||
for alias in required_aliases
|
||||
if not vm_power.get(alias, {}).get("running")
|
||||
)
|
||||
ui_session_isolated_aliases = sorted(
|
||||
alias
|
||||
for alias, rows in vm_processes.items()
|
||||
if any(
|
||||
row.get("present") is True
|
||||
and row.get("interactive_session") is not True
|
||||
for row in rows
|
||||
)
|
||||
)
|
||||
service_blockers = sorted(
|
||||
name
|
||||
for name in ("VMAuthdService", "VMnetDHCP")
|
||||
@@ -328,6 +356,13 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]:
|
||||
"vmware_services": services,
|
||||
"windows_update_policy": policies,
|
||||
"vm_power": vm_power,
|
||||
"vm_processes": vm_processes,
|
||||
"ui_session_isolated_aliases": ui_session_isolated_aliases,
|
||||
"operator_power_truth": (
|
||||
"trust_vmx_process_and_guest_readiness_not_interactive_vmware_ui"
|
||||
if ui_session_isolated_aliases
|
||||
else "vmware_ui_and_runtime_share_interactive_session"
|
||||
),
|
||||
"blockers": blockers,
|
||||
}
|
||||
|
||||
@@ -745,6 +780,35 @@ def source_controls() -> dict[str, bool]:
|
||||
"host_boot_probe_source_present": source_file(
|
||||
"scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh"
|
||||
).exists(),
|
||||
"host_112_guest_readiness_source_present": file_contains(
|
||||
source_file("scripts/reboot-recovery/host112-guest-readiness.sh"),
|
||||
"console_ready=",
|
||||
"services_ready=",
|
||||
"recovery_timer_ready=",
|
||||
"guest_ready=",
|
||||
),
|
||||
"host_112_guest_recovery_service_source_present": file_contains(
|
||||
source_file(
|
||||
"scripts/reboot-recovery/awoooi-host112-guest-recovery.service"
|
||||
),
|
||||
"ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply",
|
||||
"TimeoutStartSec=300",
|
||||
),
|
||||
"host_112_guest_recovery_timer_source_present": file_contains(
|
||||
source_file(
|
||||
"scripts/reboot-recovery/awoooi-host112-guest-recovery.timer"
|
||||
),
|
||||
"OnBootSec=20s",
|
||||
"OnUnitInactiveSec=60s",
|
||||
),
|
||||
"host_112_guest_recovery_installer_source_present": file_contains(
|
||||
source_file(
|
||||
"scripts/reboot-recovery/install-host112-guest-recovery.sh"
|
||||
),
|
||||
"--rollback",
|
||||
"restrict,command=",
|
||||
"visudo -cf",
|
||||
),
|
||||
"reboot_event_detector_source_present": source_file(
|
||||
"scripts/reboot-recovery/reboot-event-detector.py"
|
||||
).exists(),
|
||||
@@ -1419,6 +1483,11 @@ def choose_safe_next_step(
|
||||
"restore_docker_stats_textfile_exporter_then_collect_sanitized_host_"
|
||||
"pressure_no_restart_no_secret_read"
|
||||
)
|
||||
if any(blocker.startswith("host112_") for blocker in blockers):
|
||||
return (
|
||||
"run_host112_guest_recovery_controlled_apply_then_verify_forced_command_"
|
||||
"readback_and_rerun_reboot_scorecard_no_host_reboot"
|
||||
)
|
||||
if "windows99_remote_execution_channel_unavailable" in blockers:
|
||||
return (
|
||||
"restore_windows99_no_secret_management_channel_or_collect_local_console_"
|
||||
@@ -1494,6 +1563,9 @@ def build_required_checks(payload: dict[str, Any]) -> dict[str, bool]:
|
||||
host_boot_detection = payload.get("host_boot_detection")
|
||||
if not isinstance(host_boot_detection, dict):
|
||||
host_boot_detection = {}
|
||||
host112_guest = host_boot_detection.get("host112_guest_readiness")
|
||||
if not isinstance(host112_guest, dict):
|
||||
host112_guest = {}
|
||||
post_reboot_readiness = payload.get("post_reboot_readiness")
|
||||
if not isinstance(post_reboot_readiness, dict):
|
||||
post_reboot_readiness = {}
|
||||
@@ -1522,6 +1594,34 @@ def build_required_checks(payload: dict[str, Any]) -> dict[str, bool]:
|
||||
"required_hosts_reachable": not strings(
|
||||
host_boot_detection.get("unreachable_hosts")
|
||||
),
|
||||
"host112_guest_readback_present": (
|
||||
isinstance(host_boot_detection.get("host112_guest_readiness"), dict)
|
||||
and host_boot_detection.get("host112_guest_readiness", {}).get(
|
||||
"readback_present"
|
||||
)
|
||||
is True
|
||||
),
|
||||
"host112_console_ready": (
|
||||
isinstance(host_boot_detection.get("host112_guest_readiness"), dict)
|
||||
and host_boot_detection.get("host112_guest_readiness", {}).get(
|
||||
"console_ready"
|
||||
)
|
||||
is True
|
||||
),
|
||||
"host112_services_ready": (
|
||||
isinstance(host_boot_detection.get("host112_guest_readiness"), dict)
|
||||
and host_boot_detection.get("host112_guest_readiness", {}).get(
|
||||
"services_ready"
|
||||
)
|
||||
is True
|
||||
),
|
||||
"host112_guest_ready": (
|
||||
isinstance(host_boot_detection.get("host112_guest_readiness"), dict)
|
||||
and host_boot_detection.get("host112_guest_readiness", {}).get(
|
||||
"guest_ready"
|
||||
)
|
||||
is True
|
||||
),
|
||||
"windows99_vmware_autostart_ready": (
|
||||
windows99.get("verify_ready") is True
|
||||
),
|
||||
@@ -1583,6 +1683,11 @@ def reboot_sop_current_phase(active_blockers: list[str], can_claim: bool) -> str
|
||||
"windows99_console_clipboard_unreliable",
|
||||
"windows99_console_focus_unreliable",
|
||||
"windows99_console_verify_output_truncated",
|
||||
"host112_guest_readback_missing",
|
||||
"host112_console_not_ready",
|
||||
"host112_services_not_ready",
|
||||
"host112_recovery_timer_not_ready",
|
||||
"host112_system_state_not_running",
|
||||
}
|
||||
if any(blocker in host_boot_blockers for blocker in active_blockers):
|
||||
return "host_boot_detection_blocked"
|
||||
@@ -1627,6 +1732,11 @@ def reboot_sop_primary_blocker(active_blockers: list[str]) -> str:
|
||||
"all_required_hosts_not_in_10_minute_reboot_window",
|
||||
"host_boot_observation_older_than_target_window",
|
||||
"host_uptime_unknown",
|
||||
"host112_guest_readback_missing",
|
||||
"host112_console_not_ready",
|
||||
"host112_services_not_ready",
|
||||
"host112_recovery_timer_not_ready",
|
||||
"host112_system_state_not_running",
|
||||
"windows99_remote_execution_channel_unavailable",
|
||||
"windows99_vmware_autostart_readback_missing",
|
||||
"windows99_vmrun_missing",
|
||||
@@ -1774,7 +1884,34 @@ def active_blocker_action_row(
|
||||
"force_push_or_ref_delete",
|
||||
]
|
||||
|
||||
if blocker.startswith("windows99_"):
|
||||
if blocker.startswith("host112_"):
|
||||
category = "host112_guest_recovery"
|
||||
owner_lane = "agent99_windows99_and_host112_guest"
|
||||
severity = "critical"
|
||||
next_safe_action = (
|
||||
"run_sudo_n_usr_local_sbin_awoooi_host112_guest_readiness_apply_"
|
||||
"then_verify_forced_command_readback_no_host_reboot"
|
||||
)
|
||||
post_verifier = (
|
||||
"bash scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh "
|
||||
"then_rerun_reboot_auto_recovery_slo_scorecard"
|
||||
)
|
||||
evidence_inputs = [
|
||||
"host_boot_detection.host112_guest_readiness",
|
||||
"host112_guest_recovery_journal",
|
||||
"host112_last_readback",
|
||||
]
|
||||
controlled_apply_mode = "host112_allowlisted_guest_recovery_with_cooldown"
|
||||
forbidden_actions = [
|
||||
"host_reboot",
|
||||
"vm_power_change",
|
||||
"vm_reset",
|
||||
"database_write_or_restore",
|
||||
"secret_value_read",
|
||||
"firewall_change",
|
||||
"force_push_or_ref_delete",
|
||||
]
|
||||
elif blocker.startswith("windows99_"):
|
||||
category = "windows99_vmware_autostart"
|
||||
owner_lane = "windows99_console_or_no_secret_management_channel"
|
||||
next_safe_action = (
|
||||
@@ -1985,6 +2122,9 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
host_boot_detection = payload.get("host_boot_detection")
|
||||
if not isinstance(host_boot_detection, dict):
|
||||
host_boot_detection = {}
|
||||
host112_guest = host_boot_detection.get("host112_guest_readiness")
|
||||
if not isinstance(host112_guest, dict):
|
||||
host112_guest = {}
|
||||
post_reboot_readiness = payload.get("post_reboot_readiness")
|
||||
if not isinstance(post_reboot_readiness, dict):
|
||||
post_reboot_readiness = {}
|
||||
@@ -2082,6 +2222,13 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"unknown_uptime_host_count": len(
|
||||
strings(host_boot_detection.get("unknown_uptime_hosts"))
|
||||
),
|
||||
"host112_guest_readback_present": host112_guest.get("readback_present") is True,
|
||||
"host112_console_ready": host112_guest.get("console_ready") is True,
|
||||
"host112_services_ready": host112_guest.get("services_ready") is True,
|
||||
"host112_recovery_timer_ready": (
|
||||
host112_guest.get("recovery_timer_ready") is True
|
||||
),
|
||||
"host112_guest_ready": host112_guest.get("guest_ready") is True,
|
||||
"post_start_blocked": int_value(post_reboot_readiness.get("post_start_blocked")),
|
||||
"service_green": post_reboot_readiness.get("service_green") is True,
|
||||
"product_data_green": post_reboot_readiness.get("product_data_green") is True,
|
||||
@@ -2259,6 +2406,11 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only,
|
||||
"required_checks": required_checks,
|
||||
"source_controls_present": source_controls_present,
|
||||
"host112_guest_readback_present": rollups["host112_guest_readback_present"],
|
||||
"host112_console_ready": rollups["host112_console_ready"],
|
||||
"host112_services_ready": rollups["host112_services_ready"],
|
||||
"host112_recovery_timer_ready": rollups["host112_recovery_timer_ready"],
|
||||
"host112_guest_ready": rollups["host112_guest_ready"],
|
||||
"windows99_vmware_verify_ready": rollups["windows99_vmware_verify_ready"],
|
||||
"windows99_update_no_auto_reboot_ready": rollups[
|
||||
"windows99_update_no_auto_reboot_ready"
|
||||
@@ -2366,6 +2518,19 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"reboot_auto_recovery_safe_next_step": readback["safe_next_step"],
|
||||
"reboot_auto_recovery_next_safe_action": readback["next_safe_action"],
|
||||
"reboot_auto_recovery_source_controls_present": source_controls_present,
|
||||
"reboot_auto_recovery_host112_guest_readback_present": rollups[
|
||||
"host112_guest_readback_present"
|
||||
],
|
||||
"reboot_auto_recovery_host112_console_ready": rollups[
|
||||
"host112_console_ready"
|
||||
],
|
||||
"reboot_auto_recovery_host112_services_ready": rollups[
|
||||
"host112_services_ready"
|
||||
],
|
||||
"reboot_auto_recovery_host112_recovery_timer_ready": rollups[
|
||||
"host112_recovery_timer_ready"
|
||||
],
|
||||
"reboot_auto_recovery_host112_guest_ready": rollups["host112_guest_ready"],
|
||||
"reboot_auto_recovery_windows99_vmware_verify_ready": rollups[
|
||||
"windows99_vmware_verify_ready"
|
||||
],
|
||||
@@ -2450,6 +2615,30 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]:
|
||||
for row in host_rows
|
||||
if row.get("reachable") and int_value(row.get("uptime_seconds"), -1) < 0
|
||||
)
|
||||
host112_row = next(
|
||||
(row for row in host_rows if str(row.get("alias")) == "112"), {}
|
||||
)
|
||||
host112_guest_readiness = {
|
||||
"readback_present": "guest_ready" in host112_row,
|
||||
"reachable": host112_row.get("reachable") is True,
|
||||
"systemd_state": str(host112_row.get("systemd_state") or "unknown"),
|
||||
"default_target": str(host112_row.get("default_target") or "unknown"),
|
||||
"graphical_target": str(host112_row.get("graphical_target") or "unknown"),
|
||||
"display_manager": str(host112_row.get("display_manager") or "unknown"),
|
||||
"lightdm": str(host112_row.get("lightdm") or "unknown"),
|
||||
"vmware_tools": str(host112_row.get("vmware_tools") or "unknown"),
|
||||
"xorg": str(host112_row.get("xorg") or "unknown"),
|
||||
"wazuh_indexer": str(host112_row.get("wazuh_indexer") or "unknown"),
|
||||
"wazuh_manager": str(host112_row.get("wazuh_manager") or "unknown"),
|
||||
"wazuh_dashboard": str(host112_row.get("wazuh_dashboard") or "unknown"),
|
||||
"filebeat": str(host112_row.get("filebeat") or "unknown"),
|
||||
"console_ready": host112_row.get("console_ready") == "1",
|
||||
"services_ready": host112_row.get("services_ready") == "1",
|
||||
"recovery_timer_ready": host112_row.get("recovery_timer_ready") == "1",
|
||||
"guest_ready": host112_row.get("guest_ready") == "1",
|
||||
"last_action_count": int_value(host112_row.get("action_count"), 0),
|
||||
"last_actions": str(host112_row.get("actions") or "unknown"),
|
||||
}
|
||||
if not host_rows:
|
||||
blockers.append("all_host_reboot_detection_missing")
|
||||
if missing_hosts:
|
||||
@@ -2460,6 +2649,17 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]:
|
||||
blockers.append("host_boot_observation_older_than_target_window")
|
||||
if unknown_uptime_hosts:
|
||||
blockers.append("host_uptime_unknown")
|
||||
if not host112_guest_readiness["readback_present"]:
|
||||
blockers.append("host112_guest_readback_missing")
|
||||
else:
|
||||
if not host112_guest_readiness["console_ready"]:
|
||||
blockers.append("host112_console_not_ready")
|
||||
if not host112_guest_readiness["services_ready"]:
|
||||
blockers.append("host112_services_not_ready")
|
||||
if not host112_guest_readiness["recovery_timer_ready"]:
|
||||
blockers.append("host112_recovery_timer_not_ready")
|
||||
if host112_guest_readiness["systemd_state"] != "running":
|
||||
blockers.append("host112_system_state_not_running")
|
||||
|
||||
reboot_event_missing_hosts = strings(reboot_event.get("missing_hosts"))
|
||||
reboot_event_unreachable_hosts = strings(reboot_event.get("unreachable_hosts"))
|
||||
@@ -2549,6 +2749,7 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]:
|
||||
"stale_hosts": stale_hosts,
|
||||
"unknown_uptime_hosts": unknown_uptime_hosts,
|
||||
"max_observed_uptime_seconds": max_uptime,
|
||||
"host112_guest_readiness": host112_guest_readiness,
|
||||
"host_rows": host_rows,
|
||||
},
|
||||
"reboot_event_detection": {
|
||||
|
||||
@@ -30,7 +30,7 @@ TARGET_HOSTS=99,110,111,112,120,121,188
|
||||
HOST_BOOT alias=99 target=192.168.0.99 startup_unit=vmware-host-autostart reachable=1 boot_id=aa uptime_seconds=100 systemd_state=windows_exporter startup_enabled=enabled startup_active=active
|
||||
HOST_BOOT alias=110 target=wooo@192.168.0.110 startup_unit=awoooi-startup-110.service reachable=1 boot_id=a uptime_seconds=120 systemd_state=running startup_enabled=enabled startup_active=active
|
||||
HOST_BOOT alias=111 target=192.168.0.111 startup_unit=vm-host-boot reachable=1 boot_id=bb uptime_seconds=125 systemd_state=node_exporter startup_enabled=unknown startup_active=unknown
|
||||
HOST_BOOT alias=112 target=192.168.0.112 startup_unit=vm-host-boot reachable=1 boot_id=cc uptime_seconds=128 systemd_state=node_exporter startup_enabled=unknown startup_active=unknown
|
||||
HOST_BOOT alias=112 target=kali@192.168.0.112 startup_unit=awoooi-host112-guest-recovery.timer reachable=1 boot_id=cc uptime_seconds=128 systemd_state=running startup_enabled=enabled startup_active=active default_target=graphical.target graphical_target=active display_manager=active lightdm=active vmware_tools=active xorg=active wazuh_indexer=active wazuh_manager=active wazuh_dashboard=active filebeat=active console_ready=1 services_ready=1 recovery_timer_ready=1 guest_ready=1 action_count=0 actions=none
|
||||
HOST_BOOT alias=120 target=wooo@192.168.0.120 startup_unit=k3s.service reachable=1 boot_id=b uptime_seconds=130 systemd_state=running startup_enabled=enabled startup_active=active
|
||||
HOST_BOOT alias=121 target=wooo@192.168.0.121 startup_unit=k3s.service reachable=1 boot_id=c uptime_seconds=140 systemd_state=running startup_enabled=enabled startup_active=active
|
||||
HOST_BOOT alias=188 target=ollama@192.168.0.188 startup_unit=awoooi-startup.service reachable=1 boot_id=d uptime_seconds=150 systemd_state=running startup_enabled=enabled startup_active=active
|
||||
@@ -86,6 +86,7 @@ VM_POWER alias=188 vmx_present=1 running=1
|
||||
VM_POWER alias=120 vmx_present=1 running=1
|
||||
VM_POWER alias=121 vmx_present=1 running=1
|
||||
VM_POWER alias=112 vmx_present=1 running=1
|
||||
VM_PROCESS alias=112 present=1 process_id=22288 session_id=0 owner=NT_AUTHORITY\\SYSTEM interactive_session=0 ui_truth=runtime_running_gui_session_isolated
|
||||
VMWARE_AUTOSTART_CONFIG_READY=1
|
||||
VMWARE_AUTOSTART_POWER_READY=1
|
||||
WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1
|
||||
@@ -492,6 +493,10 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) -
|
||||
is False
|
||||
)
|
||||
assert payload["required_checks"]["windows99_management_channel_ready"] is True
|
||||
assert payload["required_checks"]["host112_guest_readback_present"] is True
|
||||
assert payload["required_checks"]["host112_console_ready"] is True
|
||||
assert payload["required_checks"]["host112_services_ready"] is True
|
||||
assert payload["required_checks"]["host112_guest_ready"] is True
|
||||
assert payload["required_checks"]["public_maintenance_fallback_runtime_ready"] is True
|
||||
assert payload["rollups"]["source_controls_present"] is True
|
||||
assert payload["rollups"]["public_maintenance_runtime_readback_present"] is True
|
||||
@@ -522,6 +527,10 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) -
|
||||
"host_110_startup_controlled_drain_guarded_autostart_source_present"
|
||||
] is True
|
||||
assert payload["host_boot_detection"]["max_observed_uptime_seconds"] == 150
|
||||
assert payload["host_boot_detection"]["host112_guest_readiness"]["guest_ready"] is True
|
||||
assert payload["rollups"]["host112_console_ready"] is True
|
||||
assert payload["rollups"]["host112_services_ready"] is True
|
||||
assert payload["readback"]["host112_guest_ready"] is True
|
||||
assert payload["windows99_vmware_autostart"]["required_vm_aliases"] == [
|
||||
"111",
|
||||
"112",
|
||||
@@ -530,6 +539,12 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) -
|
||||
"188",
|
||||
]
|
||||
assert payload["windows99_vmware_autostart"]["verify_ready"] is True
|
||||
assert payload["windows99_vmware_autostart"]["ui_session_isolated_aliases"] == [
|
||||
"112"
|
||||
]
|
||||
assert payload["windows99_vmware_autostart"]["operator_power_truth"] == (
|
||||
"trust_vmx_process_and_guest_readiness_not_interactive_vmware_ui"
|
||||
)
|
||||
task = payload["windows99_vmware_autostart"]["scheduled_task"]
|
||||
assert task["principal_user"] == "SYSTEM"
|
||||
assert task["run_level"] == "Highest"
|
||||
@@ -549,6 +564,36 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) -
|
||||
] is True
|
||||
|
||||
|
||||
def test_host112_console_gap_blocks_reboot_completion_with_controlled_action(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
probe = HOST_PROBE_GREEN.replace("xorg=active", "xorg=inactive").replace(
|
||||
"console_ready=1", "console_ready=0"
|
||||
).replace("guest_ready=1", "guest_ready=0")
|
||||
|
||||
payload = run_scorecard(tmp_path, GREEN_SUMMARY, probe=probe)
|
||||
|
||||
assert "host112_console_not_ready" in payload["active_blockers"]
|
||||
assert payload["host_boot_detection"]["host112_guest_readiness"][
|
||||
"console_ready"
|
||||
] is False
|
||||
assert payload["current_phase"] == "host_boot_detection_blocked"
|
||||
assert payload["safe_next_step"] == (
|
||||
"run_host112_guest_recovery_controlled_apply_then_verify_forced_command_"
|
||||
"readback_and_rerun_reboot_scorecard_no_host_reboot"
|
||||
)
|
||||
action = next(
|
||||
item
|
||||
for item in payload["active_blocker_action_matrix"]["items"]
|
||||
if item["blocker"] == "host112_console_not_ready"
|
||||
)
|
||||
assert action["category"] == "host112_guest_recovery"
|
||||
assert action["controlled_apply_mode"] == (
|
||||
"host112_allowlisted_guest_recovery_with_cooldown"
|
||||
)
|
||||
assert "host_reboot" in action["forbidden_actions"]
|
||||
|
||||
|
||||
def test_vm_power_provenance_suffix_is_preserved(tmp_path: Path) -> None:
|
||||
windows99 = "\n".join(
|
||||
f"{line} source=vmware_vmx_process"
|
||||
|
||||
@@ -83,6 +83,29 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
|
||||
assert "systemd_state=windows_ssh" in host_probe
|
||||
assert '[[ "$boot_id" == windows_* && "$uptime_seconds" =~ ^[0-9]+$ ]]' in host_probe
|
||||
assert '"111=ooo@192.168.0.111:com.momo.ollama111-allow-proxy"' in host_probe
|
||||
assert '"112=kali@192.168.0.112:awoooi-host112-guest-recovery.timer"' in host_probe
|
||||
assert "probe_host112_guest()" in host_probe
|
||||
for field in ["console_ready", "services_ready", "recovery_timer_ready", "guest_ready"]:
|
||||
assert field in host_probe
|
||||
|
||||
host112_readiness = read("scripts/reboot-recovery/host112-guest-readiness.sh")
|
||||
host112_installer = read("scripts/reboot-recovery/install-host112-guest-recovery.sh")
|
||||
host112_service = read(
|
||||
"scripts/reboot-recovery/awoooi-host112-guest-recovery.service"
|
||||
)
|
||||
host112_timer = read("scripts/reboot-recovery/awoooi-host112-guest-recovery.timer")
|
||||
assert "systemctl set-default graphical.target" in host112_readiness
|
||||
assert "start_if_inactive lightdm.service" in host112_readiness
|
||||
assert "start_if_inactive open-vm-tools.service" in host112_readiness
|
||||
assert "INDEXER_RETRY_COOLDOWN_SECONDS" in host112_readiness
|
||||
assert "console_ready=$console_ready" in host112_readiness
|
||||
assert "guest_ready=$guest_ready" in host112_readiness
|
||||
assert "restrict,command=" in host112_installer
|
||||
assert "visudo -cf" in host112_installer
|
||||
assert "--rollback" in host112_installer
|
||||
assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply" in host112_service
|
||||
assert "OnBootSec=20s" in host112_timer
|
||||
assert "OnUnitInactiveSec=60s" in host112_timer
|
||||
|
||||
macos_readback = read("scripts/reboot-recovery/macos-host-boot-readback.sh")
|
||||
macos_installer = read("scripts/reboot-recovery/install-macos111-host-boot-readback.sh")
|
||||
@@ -135,6 +158,9 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
|
||||
assert "last_task_result" in windows99
|
||||
assert "start_script_present" in windows99
|
||||
assert "VMWARE_AUTOSTART_TASK_ACTION" in windows99
|
||||
assert "Get-VmwareVmxProcessReadback" in windows99
|
||||
assert "VM_PROCESS alias=$alias" in windows99
|
||||
assert "runtime_running_gui_session_isolated" in windows99
|
||||
|
||||
verify_exit = windows99.index('if ($Mode -eq "Verify")')
|
||||
apply_start = windows99.index("Write-StartupScript -VmMap $vmMap")
|
||||
@@ -387,8 +413,17 @@ def test_reboot_slo_scorecard_fails_closed_on_raw_public_502(tmp_path: Path) ->
|
||||
host_probe = tmp_path / "host-probe.txt"
|
||||
host_probe.write_text(
|
||||
"\n".join(
|
||||
f"HOST_BOOT alias={alias} host=192.168.0.{alias} reachable=1 uptime_seconds=120"
|
||||
for alias in ["99", "110", "111", "112", "120", "121", "188"]
|
||||
[
|
||||
f"HOST_BOOT alias={alias} host=192.168.0.{alias} reachable=1 uptime_seconds=120"
|
||||
for alias in ["99", "110", "111", "120", "121", "188"]
|
||||
]
|
||||
+ [
|
||||
"HOST_BOOT alias=112 host=192.168.0.112 reachable=1 uptime_seconds=120 "
|
||||
"systemd_state=running default_target=graphical.target graphical_target=active "
|
||||
"display_manager=active lightdm=active vmware_tools=active xorg=active "
|
||||
"wazuh_indexer=active wazuh_manager=active wazuh_dashboard=active filebeat=active "
|
||||
"console_ready=1 services_ready=1 recovery_timer_ready=1 guest_ready=1"
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
@@ -187,6 +187,51 @@ function Get-VmwareVmxProcessCommandLines {
|
||||
}
|
||||
}
|
||||
|
||||
function Get-VmwareVmxProcessReadback {
|
||||
param([hashtable]$VmMap)
|
||||
|
||||
$processes = @()
|
||||
try {
|
||||
$processes = @(Get-CimInstance -ClassName Win32_Process -Filter "Name='vmware-vmx.exe'" -ErrorAction SilentlyContinue)
|
||||
} catch {
|
||||
$processes = @()
|
||||
}
|
||||
foreach ($alias in $VmOrder) {
|
||||
$path = "$($VmMap[$alias])"
|
||||
$matches = @($processes | Where-Object {
|
||||
$commandLine = [string]$_.CommandLine
|
||||
$path -and $commandLine -and $commandLine.IndexOf($path, [StringComparison]::OrdinalIgnoreCase) -ge 0
|
||||
})
|
||||
if ($matches.Count -eq 0) {
|
||||
Write-Output "VM_PROCESS alias=$alias present=0 process_id=unknown session_id=unknown owner=unknown interactive_session=0 ui_truth=not_running"
|
||||
continue
|
||||
}
|
||||
foreach ($process in $matches) {
|
||||
$owner = "unknown"
|
||||
try {
|
||||
$ownerResult = Invoke-CimMethod -InputObject $process -MethodName GetOwner -ErrorAction SilentlyContinue
|
||||
if ($ownerResult -and $ownerResult.ReturnValue -eq 0 -and $ownerResult.User) {
|
||||
$owner = if ($ownerResult.Domain) {
|
||||
"$($ownerResult.Domain)\$($ownerResult.User)"
|
||||
} else {
|
||||
"$($ownerResult.User)"
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
$owner = "unknown"
|
||||
}
|
||||
$sessionId = if ($null -ne $process.SessionId) { [int]$process.SessionId } else { -1 }
|
||||
$interactiveSession = [int]($sessionId -gt 0)
|
||||
$uiTruth = if ($interactiveSession -eq 1) {
|
||||
"runtime_running_interactive_session"
|
||||
} else {
|
||||
"runtime_running_gui_session_isolated"
|
||||
}
|
||||
Write-Output "VM_PROCESS alias=$alias present=1 process_id=$($process.ProcessId) session_id=$sessionId owner=$(Format-ReadbackValue $owner) interactive_session=$interactiveSession ui_truth=$uiTruth"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-VmxRunningSource {
|
||||
param(
|
||||
[string]$VmxPath,
|
||||
@@ -350,6 +395,7 @@ Get-VmwareServiceReadback
|
||||
Get-AutostartTaskReadback
|
||||
Get-WindowsUpdatePolicyReadback
|
||||
Get-VmPowerReadback -VmMap $vmMap
|
||||
Get-VmwareVmxProcessReadback -VmMap $vmMap
|
||||
|
||||
$serviceReady = @("VMAuthdService", "VMnetDHCP") | ForEach-Object {
|
||||
$svc = Get-Service -Name $_ -ErrorAction SilentlyContinue
|
||||
|
||||
Reference in New Issue
Block a user