319 lines
11 KiB
Bash
Executable File
319 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Read-only host boot probe for the AWOOOI reboot auto-recovery SLO.
|
|
#
|
|
# It detects whether the P0 hosts are reachable after reboot and records boot_id,
|
|
# uptime, systemd state, and the expected startup unit state. It never restarts,
|
|
# reloads, repairs, or writes host state.
|
|
|
|
set -uo pipefail
|
|
|
|
SSH_OPTS=(
|
|
-o BatchMode=yes
|
|
-o ConnectTimeout="${SSH_CONNECT_TIMEOUT_SECONDS:-6}"
|
|
-o ConnectionAttempts=1
|
|
-o ServerAliveInterval="${SSH_SERVER_ALIVE_INTERVAL_SECONDS:-5}"
|
|
-o ServerAliveCountMax="${SSH_SERVER_ALIVE_COUNT_MAX:-1}"
|
|
)
|
|
SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-15}"
|
|
NODE_EXPORTER_PORT="${NODE_EXPORTER_PORT:-9100}"
|
|
NODE_EXPORTER_TIMEOUT_SECONDS="${NODE_EXPORTER_TIMEOUT_SECONDS:-4}"
|
|
WINDOWS99_SSH_TARGET="${WINDOWS99_SSH_TARGET:-Administrator@192.168.0.99}"
|
|
HOST_SPECS=(
|
|
"99=192.168.0.99:vmware-host-autostart"
|
|
"110=wooo@192.168.0.110:awoooi-startup-110.service"
|
|
"111=ooo@192.168.0.111:com.momo.ollama111-allow-proxy"
|
|
"112=kali@192.168.0.112:awoooi-host112-guest-recovery.timer"
|
|
"120=wooo@192.168.0.120:k3s.service"
|
|
"121=wooo@192.168.0.121:k3s.service"
|
|
"188=ollama@192.168.0.188:awoooi-startup.service"
|
|
)
|
|
|
|
escape_value() {
|
|
printf '%s' "$1" | tr ' \t\n' '___'
|
|
}
|
|
|
|
local_ip_list() {
|
|
{
|
|
hostname -I 2>/dev/null | tr ' ' '\n' || true
|
|
ip -o -4 addr show 2>/dev/null | awk '{split($4,a,"/"); print a[1]}' || true
|
|
ifconfig 2>/dev/null | awk '$1 == "inet" {print $2}' || true
|
|
} | awk 'NF'
|
|
}
|
|
|
|
is_local_target() {
|
|
local target_host="$1"
|
|
local ips
|
|
[[ "$target_host" == "127.0.0.1" || "$target_host" == "localhost" ]] && return 0
|
|
ips="$(local_ip_list)"
|
|
grep -Fxq "$target_host" <<<"$ips"
|
|
}
|
|
|
|
run_with_timeout() {
|
|
local timeout_seconds="$1"
|
|
shift
|
|
if command -v timeout >/dev/null 2>&1; then
|
|
timeout "$timeout_seconds" "$@"
|
|
return $?
|
|
fi
|
|
if command -v python3 >/dev/null 2>&1; then
|
|
python3 - "$timeout_seconds" "$@" <<'PY'
|
|
import subprocess
|
|
import sys
|
|
|
|
timeout_seconds = float(sys.argv[1])
|
|
command = sys.argv[2:]
|
|
try:
|
|
result = subprocess.run(
|
|
command,
|
|
capture_output=True,
|
|
check=False,
|
|
text=True,
|
|
timeout=timeout_seconds,
|
|
)
|
|
except subprocess.TimeoutExpired as exc:
|
|
if isinstance(exc.stdout, str):
|
|
print(exc.stdout, end="")
|
|
sys.exit(124)
|
|
print(result.stdout, end="")
|
|
if result.stderr:
|
|
print(result.stderr, end="", file=sys.stderr)
|
|
sys.exit(result.returncode)
|
|
PY
|
|
return $?
|
|
fi
|
|
"$@"
|
|
}
|
|
|
|
emit_boot_row() {
|
|
local alias="$1"
|
|
local target="$2"
|
|
local unit="$3"
|
|
local reachable="$4"
|
|
local boot_id="$5"
|
|
local uptime_seconds="$6"
|
|
local systemd_state="$7"
|
|
local enabled="$8"
|
|
local active="$9"
|
|
local details="${10:-}"
|
|
|
|
printf 'HOST_BOOT alias=%s target=%s startup_unit=%s reachable=%s boot_id=%s uptime_seconds=%s systemd_state=%s startup_enabled=%s startup_active=%s' \
|
|
"$alias" "$target" "$unit" "$reachable" \
|
|
"$(escape_value "${boot_id:-unknown}")" \
|
|
"$(escape_value "${uptime_seconds:-unknown}")" \
|
|
"$(escape_value "${systemd_state:-unknown}")" \
|
|
"$(escape_value "${enabled:-unknown}")" \
|
|
"$(escape_value "${active:-unknown}")"
|
|
if [[ -n "$details" ]]; then
|
|
printf ' %s' "$details"
|
|
fi
|
|
printf '\n'
|
|
}
|
|
|
|
probe_local_host() {
|
|
local alias="$1"
|
|
local target="$2"
|
|
local unit="$3"
|
|
local boot_id uptime_seconds systemd_state enabled active
|
|
|
|
boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)"
|
|
uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo unknown)"
|
|
systemd_state="$(systemctl is-system-running 2>/dev/null || true)"
|
|
enabled="$(systemctl is-enabled "$unit" 2>/dev/null || true)"
|
|
active="$(systemctl is-active "$unit" 2>/dev/null || true)"
|
|
enabled="${enabled:-unknown}"
|
|
active="${active:-unknown}"
|
|
emit_boot_row "$alias" "$target" "$unit" 1 "$boot_id" "$uptime_seconds" "$systemd_state" "$enabled" "$active"
|
|
}
|
|
|
|
probe_node_exporter() {
|
|
local alias="$1"
|
|
local target="$2"
|
|
local unit="$3"
|
|
local target_host="${target##*@}"
|
|
local metrics boot_time now uptime_seconds
|
|
|
|
if command -v timeout >/dev/null 2>&1; then
|
|
metrics="$(timeout "$NODE_EXPORTER_TIMEOUT_SECONDS" curl -fsS "http://${target_host}:${NODE_EXPORTER_PORT}/metrics" 2>/dev/null || true)"
|
|
else
|
|
metrics="$(curl --max-time "$NODE_EXPORTER_TIMEOUT_SECONDS" -fsS "http://${target_host}:${NODE_EXPORTER_PORT}/metrics" 2>/dev/null || true)"
|
|
fi
|
|
boot_time="$(awk '$1 == "node_boot_time_seconds" {printf "%d", $2; found=1; exit} END {if (!found) print ""}' <<<"$metrics")"
|
|
if [[ -z "$boot_time" ]]; then
|
|
return 1
|
|
fi
|
|
now="$(date +%s)"
|
|
uptime_seconds=$((now - boot_time))
|
|
emit_boot_row "$alias" "$target" "$unit" 1 "node_exporter_${boot_time}" "$uptime_seconds" "node_exporter" "unknown" "unknown"
|
|
}
|
|
|
|
probe_reachable_only() {
|
|
local alias="$1"
|
|
local target="$2"
|
|
local unit="$3"
|
|
local target_host="${target##*@}"
|
|
|
|
if run_with_timeout "${PING_TIMEOUT_SECONDS:-2}" ping -c 1 "$target_host" >/dev/null 2>&1; then
|
|
emit_boot_row "$alias" "$target" "$unit" 1 "reachable_unknown_boot" "unknown" "ping_reachable" "unknown" "unknown"
|
|
return 0
|
|
fi
|
|
|
|
if command -v nc >/dev/null 2>&1; then
|
|
for port in ${BOOT_PROBE_TCP_PORTS:-22 80 443 3389 5985 9100}; do
|
|
if run_with_timeout "${TCP_CONNECT_TIMEOUT_SECONDS:-2}" \
|
|
nc -z -w "${TCP_CONNECT_TIMEOUT_SECONDS:-2}" "$target_host" "$port" >/dev/null 2>&1; then
|
|
emit_boot_row "$alias" "$target" "$unit" 1 "reachable_unknown_boot" "unknown" "tcp_${port}_reachable" "unknown" "unknown"
|
|
return 0
|
|
fi
|
|
done
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
probe_windows_ssh() {
|
|
local alias="$1"
|
|
local target="$2"
|
|
local unit="$3"
|
|
local powershell_payload encoded_payload remote_command output
|
|
local boot_id uptime_seconds systemd_state enabled active
|
|
|
|
command -v iconv >/dev/null 2>&1 || return 1
|
|
command -v base64 >/dev/null 2>&1 || return 1
|
|
|
|
powershell_payload='$ErrorActionPreference = "Stop"
|
|
$boot = (Get-CimInstance Win32_OperatingSystem).LastBootUpTime
|
|
$bootEpoch = ([DateTimeOffset]$boot).ToUnixTimeSeconds()
|
|
$uptimeSeconds = [math]::Floor(((Get-Date) - $boot).TotalSeconds)
|
|
Write-Output ("boot_id=windows_{0} uptime_seconds={1} systemd_state=windows_ssh startup_enabled=unknown startup_active=unknown" -f $bootEpoch, $uptimeSeconds)'
|
|
encoded_payload="$(
|
|
printf '%s' "$powershell_payload" \
|
|
| iconv -f UTF-8 -t UTF-16LE \
|
|
| base64 \
|
|
| tr -d '\r\n'
|
|
)"
|
|
[[ -n "$encoded_payload" ]] || return 1
|
|
remote_command="powershell.exe -NoProfile -NonInteractive -EncodedCommand $encoded_payload"
|
|
output="$(
|
|
run_with_timeout "$SSH_COMMAND_TIMEOUT_SECONDS" \
|
|
ssh "${SSH_OPTS[@]}" "$WINDOWS99_SSH_TARGET" "$remote_command" 2>/dev/null
|
|
)" || return 1
|
|
|
|
boot_id="$(sed -n 's/.*boot_id=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
|
uptime_seconds="$(sed -n 's/.*uptime_seconds=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
|
systemd_state="$(sed -n 's/.*systemd_state=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
|
enabled="$(sed -n 's/.*startup_enabled=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
|
active="$(sed -n 's/.*startup_active=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
|
[[ "$boot_id" == windows_* && "$uptime_seconds" =~ ^[0-9]+$ ]] || return 1
|
|
|
|
emit_boot_row "$alias" "$target" "$unit" 1 "$boot_id" "$uptime_seconds" \
|
|
"${systemd_state:-windows_ssh}" "${enabled:-unknown}" "${active:-unknown}"
|
|
}
|
|
|
|
probe_host112_guest() {
|
|
local alias="$1"
|
|
local target="$2"
|
|
local unit="$3"
|
|
local output boot_id uptime_seconds systemd_state enabled active
|
|
local key value details=""
|
|
|
|
output="$(
|
|
run_with_timeout "$SSH_COMMAND_TIMEOUT_SECONDS" \
|
|
ssh "${SSH_OPTS[@]}" "$target" ignored-by-forced-command 2>/dev/null
|
|
)"
|
|
[[ -n "$output" ]] || return 1
|
|
|
|
boot_id="$(sed -n 's/.*boot_id=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
|
uptime_seconds="$(sed -n 's/.*uptime_seconds=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
|
systemd_state="$(sed -n 's/.*systemd_state=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
|
enabled="$(sed -n 's/.*startup_enabled=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
|
active="$(sed -n 's/.*startup_active=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)"
|
|
[[ -n "$boot_id" && "$uptime_seconds" =~ ^[0-9]+$ ]] || return 1
|
|
|
|
for key in \
|
|
default_target graphical_target display_manager lightdm vmware_tools xorg \
|
|
wazuh_indexer wazuh_manager wazuh_dashboard filebeat console_ready \
|
|
services_ready recovery_timer_ready guest_ready action_count actions; do
|
|
value="$(sed -n "s/.*${key}=\\([^ ]*\\).*/\\1/p" <<<"$output" | tail -n 1)"
|
|
value="$(escape_value "${value:-unknown}")"
|
|
details+="${details:+ }${key}=${value}"
|
|
done
|
|
|
|
emit_boot_row "$alias" "$target" "$unit" 1 "$boot_id" "$uptime_seconds" \
|
|
"${systemd_state:-unknown}" "${enabled:-unknown}" "${active:-unknown}" "$details"
|
|
}
|
|
|
|
probe_host() {
|
|
local alias="$1"
|
|
local target="$2"
|
|
local unit="$3"
|
|
local output boot_id uptime_seconds systemd_state enabled active
|
|
local target_host="${target##*@}"
|
|
local remote_script
|
|
|
|
if is_local_target "$target_host"; then
|
|
probe_local_host "$alias" "$target" "$unit"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$alias" == "99" ]] && probe_windows_ssh "$alias" "$target" "$unit"; then
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$alias" == "112" ]] && probe_host112_guest "$alias" "$target" "$unit"; then
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$target" != *@* ]]; then
|
|
if probe_node_exporter "$alias" "$target" "$unit"; then
|
|
return 0
|
|
fi
|
|
if probe_reachable_only "$alias" "$target" "$unit"; then
|
|
return 0
|
|
fi
|
|
emit_boot_row "$alias" "$target" "$unit" 0 "unknown" "unknown" "unknown" "unknown" "unknown"
|
|
return 0
|
|
fi
|
|
|
|
remote_script="unit='$unit'; \
|
|
boot_id=\$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown); \
|
|
uptime_seconds=\$(awk '{print int(\$1)}' /proc/uptime 2>/dev/null || echo unknown); \
|
|
systemd_state=\$(systemctl is-system-running 2>/dev/null || true); \
|
|
enabled=\$(systemctl is-enabled \"\$unit\" 2>/dev/null || true); \
|
|
active=\$(systemctl is-active \"\$unit\" 2>/dev/null || true); \
|
|
enabled=\${enabled:-unknown}; \
|
|
active=\${active:-unknown}; \
|
|
printf 'boot_id=%s uptime_seconds=%s systemd_state=%s startup_enabled=%s startup_active=%s\n' \"\$boot_id\" \"\$uptime_seconds\" \"\$systemd_state\" \"\$enabled\" \"\$active\" \
|
|
"
|
|
output="$(run_with_timeout "$SSH_COMMAND_TIMEOUT_SECONDS" ssh "${SSH_OPTS[@]}" "$target" "$remote_script" 2>/dev/null)"
|
|
if [[ $? -ne 0 || -z "$output" ]]; then
|
|
if probe_node_exporter "$alias" "$target" "$unit"; then
|
|
return 0
|
|
fi
|
|
if probe_reachable_only "$alias" "$target" "$unit"; then
|
|
return 0
|
|
fi
|
|
emit_boot_row "$alias" "$target" "$unit" 0 "unknown" "unknown" "unknown" "unknown" "unknown"
|
|
return 0
|
|
fi
|
|
|
|
boot_id="$(sed -n 's/.*boot_id=\([^ ]*\).*/\1/p' <<<"$output")"
|
|
uptime_seconds="$(sed -n 's/.*uptime_seconds=\([^ ]*\).*/\1/p' <<<"$output")"
|
|
systemd_state="$(sed -n 's/.*systemd_state=\([^ ]*\).*/\1/p' <<<"$output")"
|
|
enabled="$(sed -n 's/.*startup_enabled=\([^ ]*\).*/\1/p' <<<"$output")"
|
|
active="$(sed -n 's/.*startup_active=\([^ ]*\).*/\1/p' <<<"$output")"
|
|
|
|
emit_boot_row "$alias" "$target" "$unit" 1 "$boot_id" "$uptime_seconds" "$systemd_state" "$enabled" "$active"
|
|
}
|
|
|
|
echo "AWOOOI_REBOOT_AUTO_RECOVERY_HOST_PROBE=1"
|
|
echo "TARGET_HOSTS=99,110,111,112,120,121,188"
|
|
echo "GENERATED_AT=$(date '+%Y-%m-%dT%H:%M:%S%z')"
|
|
|
|
for spec in "${HOST_SPECS[@]}"; do
|
|
alias="${spec%%=*}"
|
|
rest="${spec#*=}"
|
|
target="${rest%%:*}"
|
|
unit="${rest#*:}"
|
|
probe_host "$alias" "$target" "$unit"
|
|
done
|