#!/usr/bin/env bash
# Read-only host boot probe for the AWOOOI reboot auto-recovery SLO.
#
# It detects whether the P0 hosts are reachable after reboot and records boot_id,
# uptime, systemd state, and the expected startup unit state. It never restarts,
# reloads, repairs, or writes host state.

set -uo pipefail

SSH_OPTS=(
  -o BatchMode=yes
  -o ConnectTimeout="${SSH_CONNECT_TIMEOUT_SECONDS:-6}"
  -o ConnectionAttempts=1
  -o ServerAliveInterval="${SSH_SERVER_ALIVE_INTERVAL_SECONDS:-5}"
  -o ServerAliveCountMax="${SSH_SERVER_ALIVE_COUNT_MAX:-1}"
)
SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-15}"
NODE_EXPORTER_PORT="${NODE_EXPORTER_PORT:-9100}"
NODE_EXPORTER_TIMEOUT_SECONDS="${NODE_EXPORTER_TIMEOUT_SECONDS:-4}"
HOST_SPECS=(
  "99=192.168.0.99:vmware-host-autostart"
  "110=wooo@192.168.0.110:awoooi-startup-110.service"
  "111=192.168.0.111:vm-host-boot"
  "112=192.168.0.112:vm-host-boot"
  "120=wooo@192.168.0.120:k3s.service"
  "121=wooo@192.168.0.121:k3s.service"
  "188=ollama@192.168.0.188:awoooi-startup.service"
)

escape_value() {
  printf '%s' "$1" | tr ' \t\n' '___'
}

local_ip_list() {
  {
    hostname -I 2>/dev/null | tr ' ' '\n' || true
    ip -o -4 addr show 2>/dev/null | awk '{split($4,a,"/"); print a[1]}' || true
    ifconfig 2>/dev/null | awk '$1 == "inet" {print $2}' || true
  } | awk 'NF'
}

is_local_target() {
  local target_host="$1"
  local ips
  [[ "$target_host" == "127.0.0.1" || "$target_host" == "localhost" ]] && return 0
  ips="$(local_ip_list)"
  grep -Fxq "$target_host" <<<"$ips"
}

run_with_timeout() {
  local timeout_seconds="$1"
  shift
  if command -v timeout >/dev/null 2>&1; then
    timeout "$timeout_seconds" "$@"
    return $?
  fi
  if command -v python3 >/dev/null 2>&1; then
    python3 - "$timeout_seconds" "$@" <<'PY'
import subprocess
import sys

timeout_seconds = float(sys.argv[1])
command = sys.argv[2:]
try:
    result = subprocess.run(
        command,
        capture_output=True,
        check=False,
        text=True,
        timeout=timeout_seconds,
    )
except subprocess.TimeoutExpired as exc:
    if isinstance(exc.stdout, str):
        print(exc.stdout, end="")
    sys.exit(124)
print(result.stdout, end="")
if result.stderr:
    print(result.stderr, end="", file=sys.stderr)
sys.exit(result.returncode)
PY
    return $?
  fi
  "$@"
}

emit_boot_row() {
  local alias="$1"
  local target="$2"
  local unit="$3"
  local reachable="$4"
  local boot_id="$5"
  local uptime_seconds="$6"
  local systemd_state="$7"
  local enabled="$8"
  local active="$9"

  printf 'HOST_BOOT alias=%s target=%s startup_unit=%s reachable=%s boot_id=%s uptime_seconds=%s systemd_state=%s startup_enabled=%s startup_active=%s\n' \
    "$alias" "$target" "$unit" "$reachable" \
    "$(escape_value "${boot_id:-unknown}")" \
    "$(escape_value "${uptime_seconds:-unknown}")" \
    "$(escape_value "${systemd_state:-unknown}")" \
    "$(escape_value "${enabled:-unknown}")" \
    "$(escape_value "${active:-unknown}")"
}

probe_local_host() {
  local alias="$1"
  local target="$2"
  local unit="$3"
  local boot_id uptime_seconds systemd_state enabled active

  boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)"
  uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo unknown)"
  systemd_state="$(systemctl is-system-running 2>/dev/null || true)"
  enabled="$(systemctl is-enabled "$unit" 2>/dev/null || true)"
  active="$(systemctl is-active "$unit" 2>/dev/null || true)"
  enabled="${enabled:-unknown}"
  active="${active:-unknown}"
  emit_boot_row "$alias" "$target" "$unit" 1 "$boot_id" "$uptime_seconds" "$systemd_state" "$enabled" "$active"
}

probe_node_exporter() {
  local alias="$1"
  local target="$2"
  local unit="$3"
  local target_host="${target##*@}"
  local metrics boot_time now uptime_seconds

  if command -v timeout >/dev/null 2>&1; then
    metrics="$(timeout "$NODE_EXPORTER_TIMEOUT_SECONDS" curl -fsS "http://${target_host}:${NODE_EXPORTER_PORT}/metrics" 2>/dev/null || true)"
  else
    metrics="$(curl --max-time "$NODE_EXPORTER_TIMEOUT_SECONDS" -fsS "http://${target_host}:${NODE_EXPORTER_PORT}/metrics" 2>/dev/null || true)"
  fi
  boot_time="$(awk '$1 == "node_boot_time_seconds" {printf "%d", $2; found=1; exit} END {if (!found) print ""}' <<<"$metrics")"
  if [[ -z "$boot_time" ]]; then
    return 1
  fi
  now="$(date +%s)"
  uptime_seconds=$((now - boot_time))
  emit_boot_row "$alias" "$target" "$unit" 1 "node_exporter_${boot_time}" "$uptime_seconds" "node_exporter" "unknown" "unknown"
}

probe_reachable_only() {
  local alias="$1"
  local target="$2"
  local unit="$3"
  local target_host="${target##*@}"

  if run_with_timeout "${PING_TIMEOUT_SECONDS:-2}" ping -c 1 "$target_host" >/dev/null 2>&1; then
    emit_boot_row "$alias" "$target" "$unit" 1 "reachable_unknown_boot" "unknown" "ping_reachable" "unknown" "unknown"
    return 0
  fi

  if command -v nc >/dev/null 2>&1; then
    for port in ${BOOT_PROBE_TCP_PORTS:-22 80 443 3389 5985 9100}; do
      if run_with_timeout "${TCP_CONNECT_TIMEOUT_SECONDS:-2}" \
        nc -z -w "${TCP_CONNECT_TIMEOUT_SECONDS:-2}" "$target_host" "$port" >/dev/null 2>&1; then
        emit_boot_row "$alias" "$target" "$unit" 1 "reachable_unknown_boot" "unknown" "tcp_${port}_reachable" "unknown" "unknown"
        return 0
      fi
    done
  fi

  return 1
}

probe_host() {
  local alias="$1"
  local target="$2"
  local unit="$3"
  local output boot_id uptime_seconds systemd_state enabled active
  local target_host="${target##*@}"
  local remote_script

  if is_local_target "$target_host"; then
    probe_local_host "$alias" "$target" "$unit"
    return 0
  fi

  if [[ "$target" != *@* ]]; then
    if probe_node_exporter "$alias" "$target" "$unit"; then
      return 0
    fi
    if probe_reachable_only "$alias" "$target" "$unit"; then
      return 0
    fi
    emit_boot_row "$alias" "$target" "$unit" 0 "unknown" "unknown" "unknown" "unknown" "unknown"
    return 0
  fi

  remote_script="unit='$unit'; \
    boot_id=\$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown); \
    uptime_seconds=\$(awk '{print int(\$1)}' /proc/uptime 2>/dev/null || echo unknown); \
    systemd_state=\$(systemctl is-system-running 2>/dev/null || true); \
    enabled=\$(systemctl is-enabled \"\$unit\" 2>/dev/null || true); \
    active=\$(systemctl is-active \"\$unit\" 2>/dev/null || true); \
    enabled=\${enabled:-unknown}; \
    active=\${active:-unknown}; \
    printf 'boot_id=%s uptime_seconds=%s systemd_state=%s startup_enabled=%s startup_active=%s\n' \"\$boot_id\" \"\$uptime_seconds\" \"\$systemd_state\" \"\$enabled\" \"\$active\" \
  "
  output="$(run_with_timeout "$SSH_COMMAND_TIMEOUT_SECONDS" ssh "${SSH_OPTS[@]}" "$target" "$remote_script" 2>/dev/null)"
  if [[ $? -ne 0 || -z "$output" ]]; then
    if probe_node_exporter "$alias" "$target" "$unit"; then
      return 0
    fi
    if probe_reachable_only "$alias" "$target" "$unit"; then
      return 0
    fi
    emit_boot_row "$alias" "$target" "$unit" 0 "unknown" "unknown" "unknown" "unknown" "unknown"
    return 0
  fi

  boot_id="$(sed -n 's/.*boot_id=\([^ ]*\).*/\1/p' <<<"$output")"
  uptime_seconds="$(sed -n 's/.*uptime_seconds=\([^ ]*\).*/\1/p' <<<"$output")"
  systemd_state="$(sed -n 's/.*systemd_state=\([^ ]*\).*/\1/p' <<<"$output")"
  enabled="$(sed -n 's/.*startup_enabled=\([^ ]*\).*/\1/p' <<<"$output")"
  active="$(sed -n 's/.*startup_active=\([^ ]*\).*/\1/p' <<<"$output")"

  emit_boot_row "$alias" "$target" "$unit" 1 "$boot_id" "$uptime_seconds" "$systemd_state" "$enabled" "$active"
}

echo "AWOOOI_REBOOT_AUTO_RECOVERY_HOST_PROBE=1"
echo "TARGET_HOSTS=99,110,111,112,120,121,188"
echo "GENERATED_AT=$(date '+%Y-%m-%dT%H:%M:%S%z')"

for spec in "${HOST_SPECS[@]}"; do
  alias="${spec%%=*}"
  rest="${spec#*=}"
  target="${rest%%:*}"
  unit="${rest#*:}"
  probe_host "$alias" "$target" "$unit"
done
