#!/usr/bin/env bash
# Read-only classifier for 110 SSH publickey authentication stalls.
#
# Run from an operator host. It does not write remote state, does not read
# authorized_keys contents, and does not ask for passwords.

set -uo pipefail

HOST="${HOST:-192.168.0.110}"
PORT="${PORT:-22}"
NODE_EXPORTER_PORT="${NODE_EXPORTER_PORT:-9100}"
CONNECT_TIMEOUT_SECONDS="${CONNECT_TIMEOUT_SECONDS:-4}"
SSH_ATTEMPT_TIMEOUT_SECONDS="${SSH_ATTEMPT_TIMEOUT_SECONDS:-8}"
SSH_COMMAND_PATH_TIMEOUT_SECONDS="${SSH_COMMAND_PATH_TIMEOUT_SECONDS:-20}"
NODE_EXPORTER_TIMEOUT_SECONDS="${NODE_EXPORTER_TIMEOUT_SECONDS:-8}"
USERS=(${USERS:-wooo root git ollama})
COMMAND_PATH_USER="${COMMAND_PATH_USER:-wooo}"

tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/awoooi-110-ssh-auth.XXXXXX")"
trap 'rm -rf "$tmp_dir"' EXIT

ssh_base_opts=(
  -vv
  -p "$PORT"
  -o ConnectTimeout="$CONNECT_TIMEOUT_SECONDS"
  -o ConnectionAttempts=1
  -o BatchMode=yes
  -o StrictHostKeyChecking=accept-new
  -o ServerAliveInterval=2
  -o ServerAliveCountMax=1
)

ssh_command_opts=(
  -p "$PORT"
  -o ConnectTimeout="$CONNECT_TIMEOUT_SECONDS"
  -o ConnectionAttempts=1
  -o BatchMode=yes
  -o StrictHostKeyChecking=accept-new
  -o ServerAliveInterval=2
  -o ServerAliveCountMax=1
  -o PasswordAuthentication=no
  -o PreferredAuthentications=publickey
)

run_timeout() {
  local seconds="$1"
  shift
  if command -v timeout >/dev/null 2>&1; then
    timeout "$seconds" "$@"
    return $?
  fi
  "$@"
}

classify_log() {
  local path="$1"
  if grep -q 'Server accepts key' "$path" && grep -Eiq 'timed out|not responding|Timeout' "$path"; then
    echo "server_accepts_key_then_timeout"
  elif grep -q 'Server accepts key' "$path"; then
    echo "server_accepts_key"
  elif grep -q 'Offering public key' "$path" && grep -Eiq 'timed out|not responding|Timeout' "$path"; then
    echo "publickey_offer_timeout"
  elif grep -q 'Permission denied' "$path"; then
    echo "permission_denied"
  elif grep -Eiq 'timed out|not responding|Timeout' "$path"; then
    echo "preauth_timeout"
  elif grep -q 'Connection established' "$path"; then
    echo "connected_unclassified"
  else
    echo "connection_failed"
  fi
}

probe_node_exporter() {
  local metrics cpu_count load1 load1_per_cpu
  metrics="$(curl -fsS --max-time "$NODE_EXPORTER_TIMEOUT_SECONDS" "http://${HOST}:${NODE_EXPORTER_PORT}/metrics" 2>/dev/null || true)"
  if [[ -z "$metrics" ]]; then
    echo "NODE_EXPORTER=unreachable"
    return
  fi
  echo "NODE_EXPORTER=ok"
  awk '
    $1 == "node_boot_time_seconds" {print "NODE_BOOT_TIME_SECONDS="$2}
    $1 == "node_time_seconds" {node_time=$2; print "NODE_TIME_SECONDS="$2}
    $1 == "node_load1" {print "NODE_LOAD1="$2}
    $1 == "node_load5" {print "NODE_LOAD5="$2}
    $1 == "node_load15" {print "NODE_LOAD15="$2}
    $1 == "node_procs_blocked" {print "NODE_PROCS_BLOCKED="$2}
    $1 == "node_memory_MemAvailable_bytes" {print "NODE_MEM_AVAILABLE_BYTES="$2}
    $1 == "node_memory_MemTotal_bytes" {print "NODE_MEM_TOTAL_BYTES="$2}
    /^node_textfile_mtime_seconds/ && /docker_stats\.prom/ {docker_stats_mtime=$NF}
    END {
      if (docker_stats_mtime != "") {
        print "DOCKER_STATS_TEXTFILE_MTIME_SECONDS=" docker_stats_mtime
        if (node_time != "") {
          age = node_time - docker_stats_mtime
          printf "DOCKER_STATS_TEXTFILE_AGE_SECONDS=%.0f\n", age
          print "DOCKER_STATS_TEXTFILE_FRESHNESS=" (age <= 300 ? "fresh" : "stale")
        }
      } else {
        print "DOCKER_STATS_TEXTFILE_FRESHNESS=missing"
      }
    }
  ' <<<"$metrics"
  cpu_count="$(awk -F'cpu="' '/^node_cpu_seconds_total/ {split($2, a, "\""); seen[a[1]]=1} END {for (cpu in seen) n++; print n+0}' <<<"$metrics")"
  load1="$(awk '$1 == "node_load1" {print $2; exit}' <<<"$metrics")"
  echo "NODE_CPU_COUNT=${cpu_count:-0}"
  if [[ -n "${load1:-}" && "${cpu_count:-0}" -gt 0 ]]; then
    load1_per_cpu="$(awk -v load="$load1" -v cpus="$cpu_count" 'BEGIN {printf "%.2f", load / cpus}')"
    echo "NODE_LOAD1_PER_CPU=$load1_per_cpu"
    awk -v ratio="$load1_per_cpu" 'BEGIN {print "NODE_LOAD_CLASSIFIER=" (ratio > 1.5 ? "high_load" : "load_not_high")}'
  fi
  awk '
    /^systemd_unit_info/ && /unit="(actions\.runner|awoooi-cd-lane)/ {
      line=$0
      unit=line
      active=line
      substate=line
      sub(/^.*unit="/, "", unit)
      sub(/".*$/, "", unit)
      sub(/^.*active_state="/, "", active)
      sub(/".*$/, "", active)
      sub(/^.*sub_state="/, "", substate)
      sub(/".*$/, "", substate)
      classifier=active
      if (active == "scrape_error" && substate ~ /(timed out|timeout)/) {
        classifier="systemctl_show_timeout"
      } else if (active == "scrape_skipped" && substate ~ /systemctl_timeout_budget_exhausted/) {
        classifier="systemctl_timeout_budget_exhausted"
      }
      printf "SYSTEMD_UNIT unit=%s active_state=%s classifier=%s\n", unit, active, classifier
    }
  ' <<<"$metrics"
}

probe_tcp_banner() {
  if command -v nc >/dev/null 2>&1; then
    if run_timeout "$CONNECT_TIMEOUT_SECONDS" nc -v "$HOST" "$PORT" </dev/null >/tmp/awoooi-110-nc.out 2>&1; then
      echo "SSH_PORT=tcp_open"
      sed -n 's/^SSH-/SSH_BANNER=SSH-/p' /tmp/awoooi-110-nc.out | head -1
      return
    fi
  fi
  echo "SSH_PORT=tcp_unconfirmed"
}

probe_user() {
  local user="$1"
  local mode="$2"
  local log="$tmp_dir/${user}-${mode}.log"
  local rc classification

  if [[ "$mode" == "publickey" ]]; then
    run_timeout "$SSH_ATTEMPT_TIMEOUT_SECONDS" \
      ssh "${ssh_base_opts[@]}" \
      -o PasswordAuthentication=no \
      -o PreferredAuthentications=publickey \
      "${user}@${HOST}" 'true' >"$log" 2>&1
    rc=$?
  else
    run_timeout "$SSH_ATTEMPT_TIMEOUT_SECONDS" \
      ssh "${ssh_base_opts[@]}" \
      -o PubkeyAuthentication=no \
      -o PreferredAuthentications=password \
      -o NumberOfPasswordPrompts=0 \
      "${user}@${HOST}" 'true' >"$log" 2>&1
    rc=$?
  fi

  classification="$(classify_log "$log")"
  printf 'SSH_AUTH user=%s mode=%s rc=%s classification=%s\n' "$user" "$mode" "$rc" "$classification"
}

probe_command_path_user() {
  local user="$1"
  local stdout_path="$tmp_dir/${user}-command-path.out"
  local stderr_path="$tmp_dir/${user}-command-path.err"
  local rc classification marker_seen remote_user remote_user_match

  run_timeout "$SSH_COMMAND_PATH_TIMEOUT_SECONDS" \
    ssh "${ssh_command_opts[@]}" \
    "${user}@${HOST}" \
    'printf "AWOOOI_110_COMMAND_PATH_READY=1\n"; printf "remote_user=%s\n" "$(id -un)"' \
    >"$stdout_path" 2>"$stderr_path"
  rc=$?

  marker_seen=false
  remote_user_match=false
  remote_user="$(awk -F= '$1 == "remote_user" {print $2; exit}' "$stdout_path" 2>/dev/null || true)"
  if grep -qx 'AWOOOI_110_COMMAND_PATH_READY=1' "$stdout_path" 2>/dev/null; then
    marker_seen=true
  fi
  if [[ "$remote_user" == "$user" ]]; then
    remote_user_match=true
  fi

  if [[ "$marker_seen" == "true" && "$remote_user_match" == "true" ]]; then
    classification="command_path_ready"
  elif grep -Eiq 'timed out|not responding|Timeout' "$stderr_path"; then
    classification="command_path_timeout"
  elif grep -q 'Permission denied' "$stderr_path"; then
    classification="command_path_permission_denied"
  else
    classification="command_path_unavailable"
  fi

  printf 'SSH_COMMAND_PATH user=%s rc=%s classification=%s marker_seen=%s remote_user_match=%s\n' \
    "$user" "$rc" "$classification" "$marker_seen" "$remote_user_match"
}

echo "AWOOOI_110_SSH_PUBLICKEY_AUTH_DIAGNOSIS"
echo "TARGET=${HOST}:${PORT}"
probe_node_exporter
probe_tcp_banner

for user in "${USERS[@]}"; do
  probe_user "$user" "publickey"
done

probe_command_path_user "$COMMAND_PATH_USER"

for user in "${USERS[@]}"; do
  probe_user "$user" "password_disabled"
done

echo "INTERPRETATION=command_path_ready_overrides_stale_verbose_true_timeout;server_accepts_key_then_timeout_without_command_path_means_check_110_session_pam_account_or_shell_path;publickey_offer_timeout_means_check_110_authorized_keys_permissions_pam_or_account_lookup_path"