fix(recovery): classify 110 ssh publickey auth blocker

scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+# Read-only classifier for 110 SSH publickey authentication stalls.
+#
+# Run from an operator host. It does not write remote state, does not read
+# authorized_keys contents, and does not ask for passwords.
+
+set -uo pipefail
+
+HOST="${HOST:-192.168.0.110}"
+PORT="${PORT:-22}"
+NODE_EXPORTER_PORT="${NODE_EXPORTER_PORT:-9100}"
+CONNECT_TIMEOUT_SECONDS="${CONNECT_TIMEOUT_SECONDS:-4}"
+SSH_ATTEMPT_TIMEOUT_SECONDS="${SSH_ATTEMPT_TIMEOUT_SECONDS:-8}"
+NODE_EXPORTER_TIMEOUT_SECONDS="${NODE_EXPORTER_TIMEOUT_SECONDS:-8}"
+USERS=(${USERS:-wooo root git ollama})
+
+tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/awoooi-110-ssh-auth.XXXXXX")"
+trap 'rm -rf "$tmp_dir"' EXIT
+
+ssh_base_opts=(
+  -vv
+  -p "$PORT"
+  -o ConnectTimeout="$CONNECT_TIMEOUT_SECONDS"
+  -o ConnectionAttempts=1
+  -o BatchMode=yes
+  -o StrictHostKeyChecking=accept-new
+  -o ServerAliveInterval=2
+  -o ServerAliveCountMax=1
+)
+
+run_timeout() {
+  local seconds="$1"
+  shift
+  if command -v timeout >/dev/null 2>&1; then
+    timeout "$seconds" "$@"
+    return $?
+  fi
+  "$@"
+}
+
+classify_log() {
+  local path="$1"
+  if grep -q 'Server accepts key' "$path"; then
+    echo "server_accepts_key"
+  elif grep -q 'Offering public key' "$path" && grep -Eiq 'timed out|not responding|Timeout' "$path"; then
+    echo "publickey_offer_timeout"
+  elif grep -q 'Permission denied' "$path"; then
+    echo "permission_denied"
+  elif grep -Eiq 'timed out|not responding|Timeout' "$path"; then
+    echo "preauth_timeout"
+  elif grep -q 'Connection established' "$path"; then
+    echo "connected_unclassified"
+  else
+    echo "connection_failed"
+  fi
+}
+
+probe_node_exporter() {
+  local metrics cpu_count load1 load1_per_cpu
+  metrics="$(curl -fsS --max-time "$NODE_EXPORTER_TIMEOUT_SECONDS" "http://${HOST}:${NODE_EXPORTER_PORT}/metrics" 2>/dev/null || true)"
+  if [[ -z "$metrics" ]]; then
+    echo "NODE_EXPORTER=unreachable"
+    return
+  fi
+  echo "NODE_EXPORTER=ok"
+  awk '
+    $1 == "node_boot_time_seconds" {print "NODE_BOOT_TIME_SECONDS="$2}
+    $1 == "node_time_seconds" {print "NODE_TIME_SECONDS="$2}
+    $1 == "node_load1" {print "NODE_LOAD1="$2}
+    $1 == "node_load5" {print "NODE_LOAD5="$2}
+    $1 == "node_load15" {print "NODE_LOAD15="$2}
+    $1 == "node_procs_blocked" {print "NODE_PROCS_BLOCKED="$2}
+    $1 == "node_memory_MemAvailable_bytes" {print "NODE_MEM_AVAILABLE_BYTES="$2}
+    $1 == "node_memory_MemTotal_bytes" {print "NODE_MEM_TOTAL_BYTES="$2}
+  ' <<<"$metrics"
+  cpu_count="$(awk -F'cpu="' '/^node_cpu_seconds_total/ {split($2, a, "\""); seen[a[1]]=1} END {for (cpu in seen) n++; print n+0}' <<<"$metrics")"
+  load1="$(awk '$1 == "node_load1" {print $2; exit}' <<<"$metrics")"
+  echo "NODE_CPU_COUNT=${cpu_count:-0}"
+  if [[ -n "${load1:-}" && "${cpu_count:-0}" -gt 0 ]]; then
+    load1_per_cpu="$(awk -v load="$load1" -v cpus="$cpu_count" 'BEGIN {printf "%.2f", load / cpus}')"
+    echo "NODE_LOAD1_PER_CPU=$load1_per_cpu"
+    awk -v ratio="$load1_per_cpu" 'BEGIN {print "NODE_LOAD_CLASSIFIER=" (ratio > 1.5 ? "high_load" : "load_not_high")}'
+  fi
+}
+
+probe_tcp_banner() {
+  if command -v nc >/dev/null 2>&1; then
+    if run_timeout "$CONNECT_TIMEOUT_SECONDS" nc -v "$HOST" "$PORT" </dev/null >/tmp/awoooi-110-nc.out 2>&1; then
+      echo "SSH_PORT=tcp_open"
+      sed -n 's/^SSH-/SSH_BANNER=SSH-/p' /tmp/awoooi-110-nc.out | head -1
+      return
+    fi
+  fi
+  echo "SSH_PORT=tcp_unconfirmed"
+}
+
+probe_user() {
+  local user="$1"
+  local mode="$2"
+  local log="$tmp_dir/${user}-${mode}.log"
+  local rc classification
+
+  if [[ "$mode" == "publickey" ]]; then
+    run_timeout "$SSH_ATTEMPT_TIMEOUT_SECONDS" \
+      ssh "${ssh_base_opts[@]}" \
+      -o PasswordAuthentication=no \
+      -o PreferredAuthentications=publickey \
+      "${user}@${HOST}" 'true' >"$log" 2>&1
+    rc=$?
+  else
+    run_timeout "$SSH_ATTEMPT_TIMEOUT_SECONDS" \
+      ssh "${ssh_base_opts[@]}" \
+      -o PubkeyAuthentication=no \
+      -o PreferredAuthentications=password \
+      -o NumberOfPasswordPrompts=0 \
+      "${user}@${HOST}" 'true' >"$log" 2>&1
+    rc=$?
+  fi
+
+  classification="$(classify_log "$log")"
+  printf 'SSH_AUTH user=%s mode=%s rc=%s classification=%s\n' "$user" "$mode" "$rc" "$classification"
+}
+
+echo "AWOOOI_110_SSH_PUBLICKEY_AUTH_DIAGNOSIS"
+echo "TARGET=${HOST}:${PORT}"
+probe_node_exporter
+probe_tcp_banner
+
+for user in "${USERS[@]}"; do
+  probe_user "$user" "publickey"
+done
+
+for user in "${USERS[@]}"; do
+  probe_user "$user" "password_disabled"
+done
+
+echo "INTERPRETATION=publickey_offer_timeout_on_wooo_means_check_110_authorized_keys_permissions_pam_or_account_lookup_path"

scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# Local-only check/apply helper for 110 SSH publickey authentication stalls.
+#
+# Run on host 110 from a trusted local console or an already working root shell.
+# It never prints authorized_keys contents. Apply only fixes metadata permissions
+# on the target user's home/.ssh/authorized_keys path and runs sshd syntax checks.
+
+set -euo pipefail
+
+TARGET_USER="${TARGET_USER:-wooo}"
+APPLY=0
+RELOAD_SSH="${RELOAD_SSH:-0}"
+
+usage() {
+  cat <<'USAGE'
+Usage: scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh [--check|--apply]
+
+Options:
+  --check   Read-only checks. Default.
+  --apply   Fix ownership/permissions for TARGET_USER home/.ssh/authorized_keys.
+
+Environment:
+  TARGET_USER=wooo   Account to inspect/fix.
+  RELOAD_SSH=0       Set to 1 to run `systemctl reload ssh` after apply.
+
+This script does not print authorized_keys contents and does not create keys.
+USAGE
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --check)
+      APPLY=0
+      ;;
+    --apply)
+      APPLY=1
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown argument: $1" >&2
+      usage >&2
+      exit 64
+      ;;
+  esac
+  shift
+done
+
+require_110_or_explicit() {
+  if [ "${ALLOW_NON_110:-0}" = "1" ]; then
+    return
+  fi
+  if hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx '192.168.0.110'; then
+    return
+  fi
+  echo "BLOCKED not running on 192.168.0.110; set ALLOW_NON_110=1 only for syntax tests" >&2
+  exit 65
+}
+
+stat_path() {
+  local path="$1"
+  if [ -e "$path" ]; then
+    stat -c 'PATH_STATUS path=%n mode=%a owner=%U group=%G type=%F' "$path"
+  else
+    echo "PATH_STATUS path=$path missing=1"
+  fi
+}
+
+check_user() {
+  local user="$1"
+  local home_dir
+  home_dir="$(getent passwd "$user" | awk -F: '{print $6}')"
+  if [ -z "$home_dir" ]; then
+    echo "USER_STATUS user=$user exists=0"
+    return 1
+  fi
+
+  echo "USER_STATUS user=$user exists=1 home=$home_dir"
+  stat_path "$home_dir"
+  stat_path "$home_dir/.ssh"
+  stat_path "$home_dir/.ssh/authorized_keys"
+  if [ -f "$home_dir/.ssh/authorized_keys" ]; then
+    echo "AUTHORIZED_KEYS_STATUS path=$home_dir/.ssh/authorized_keys exists=1 bytes=$(wc -c < "$home_dir/.ssh/authorized_keys" | tr -d ' ') lines=$(wc -l < "$home_dir/.ssh/authorized_keys" | tr -d ' ')"
+  else
+    echo "AUTHORIZED_KEYS_STATUS path=$home_dir/.ssh/authorized_keys exists=0"
+  fi
+}
+
+apply_user_permissions() {
+  local user="$1"
+  local home_dir
+  home_dir="$(getent passwd "$user" | awk -F: '{print $6}')"
+  if [ -z "$home_dir" ]; then
+    echo "BLOCKED target user missing: $user" >&2
+    exit 66
+  fi
+  if [ ! -d "$home_dir/.ssh" ]; then
+    echo "BLOCKED missing $home_dir/.ssh; not creating key material or SSH directories automatically" >&2
+    exit 67
+  fi
+  if [ ! -f "$home_dir/.ssh/authorized_keys" ]; then
+    echo "BLOCKED missing $home_dir/.ssh/authorized_keys; not creating key material automatically" >&2
+    exit 68
+  fi
+
+  chown "$user:$user" "$home_dir/.ssh" "$home_dir/.ssh/authorized_keys"
+  chmod 700 "$home_dir/.ssh"
+  chmod 600 "$home_dir/.ssh/authorized_keys"
+  echo "APPLIED permissions target_user=$user home=$home_dir"
+}
+
+require_110_or_explicit
+
+echo "AWOOOI_110_SSH_PUBLICKEY_AUTH_LOCAL_REPAIR mode=$([ "$APPLY" -eq 1 ] && echo apply || echo check) target_user=$TARGET_USER"
+systemctl is-active ssh 2>/dev/null | sed 's/^/SSH_SERVICE_ACTIVE=/' || true
+sshd -t
+echo "SSHD_CONFIG_SYNTAX=ok"
+check_user "$TARGET_USER"
+
+if [ "$APPLY" -eq 1 ]; then
+  apply_user_permissions "$TARGET_USER"
+  sshd -t
+  echo "SSHD_CONFIG_SYNTAX_AFTER_APPLY=ok"
+  if [ "$RELOAD_SSH" = "1" ]; then
+    systemctl reload ssh
+    echo "SSH_RELOAD=done"
+  else
+    echo "SSH_RELOAD=skipped"
+  fi
+fi

scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py

@@ -10,6 +10,12 @@ RECOVERY_SCORECARD = (
     ROOT / "scripts" / "reboot-recovery" / "full-stack-recovery-scorecard.sh"
 )
 VERIFY_DEPLOY = ROOT / "scripts" / "reboot-recovery" / "verify-cold-start-monitor-deploy.sh"
+SSH_AUTH_DIAGNOSE = (
+    ROOT / "scripts" / "reboot-recovery" / "diagnose-110-ssh-publickey-auth.sh"
+)
+SSH_AUTH_REPAIR = (
+    ROOT / "scripts" / "reboot-recovery" / "repair-110-ssh-publickey-auth-local.sh"
+)
 
 
 def test_full_stack_cold_start_check_bounds_ssh_probes() -> None:
@@ -81,3 +87,35 @@ def test_cold_start_deploy_parity_verifier_bounds_ssh_readback() -> None:
     assert "timeout ${REMOTE_COMMAND_TIMEOUT_SECONDS}s bash -lc" in text
     assert 'remote_read "sha256sum' in text
     assert 'if remote_read "grep -Fq' in text
+
+
+def test_110_ssh_publickey_auth_diagnosis_is_bounded_and_read_only() -> None:
+    text = SSH_AUTH_DIAGNOSE.read_text(encoding="utf-8")
+
+    assert "PasswordAuthentication=no" in text
+    assert "PubkeyAuthentication=no" in text
+    assert "NumberOfPasswordPrompts=0" in text
+    assert "publickey_offer_timeout" in text
+    assert "NODE_EXPORTER=ok" in text
+    assert "NODE_LOAD1_PER_CPU" in text
+    assert "NODE_LOAD_CLASSIFIER" in text
+    assert "cat /home" not in text
+    assert "cat ~/.ssh/authorized_keys" not in text
+    assert "cat \"$home_dir/.ssh/authorized_keys\"" not in text
+    assert "systemctl" not in text
+    assert "chmod" not in text
+    assert "chown" not in text
+
+
+def test_110_ssh_publickey_auth_repair_is_local_and_does_not_print_keys() -> None:
+    text = SSH_AUTH_REPAIR.read_text(encoding="utf-8")
+
+    assert "not running on 192.168.0.110" in text
+    assert "--apply" in text
+    assert "chmod 700" in text
+    assert "chmod 600" in text
+    assert "chown \"$user:$user\"" in text
+    assert "sshd -t" in text
+    assert 'RELOAD_SSH="${RELOAD_SSH:-0}"' in text
+    assert "cat \"$home_dir/.ssh/authorized_keys\"" not in text
+    assert "echo \"$(cat" not in text