c90f1ced2f
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 36s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
227 lines
7.8 KiB
Bash
Executable File
227 lines
7.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Read-only classifier for 110 SSH publickey authentication stalls.
|
|
#
|
|
# Run from an operator host. It does not write remote state, does not read
|
|
# authorized_keys contents, and does not ask for passwords.
|
|
|
|
set -uo pipefail
|
|
|
|
HOST="${HOST:-192.168.0.110}"
|
|
PORT="${PORT:-22}"
|
|
NODE_EXPORTER_PORT="${NODE_EXPORTER_PORT:-9100}"
|
|
CONNECT_TIMEOUT_SECONDS="${CONNECT_TIMEOUT_SECONDS:-4}"
|
|
SSH_ATTEMPT_TIMEOUT_SECONDS="${SSH_ATTEMPT_TIMEOUT_SECONDS:-8}"
|
|
SSH_COMMAND_PATH_TIMEOUT_SECONDS="${SSH_COMMAND_PATH_TIMEOUT_SECONDS:-20}"
|
|
NODE_EXPORTER_TIMEOUT_SECONDS="${NODE_EXPORTER_TIMEOUT_SECONDS:-8}"
|
|
USERS=(${USERS:-wooo root git ollama})
|
|
COMMAND_PATH_USER="${COMMAND_PATH_USER:-wooo}"
|
|
|
|
tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/awoooi-110-ssh-auth.XXXXXX")"
|
|
trap 'rm -rf "$tmp_dir"' EXIT
|
|
|
|
ssh_base_opts=(
|
|
-vv
|
|
-p "$PORT"
|
|
-o ConnectTimeout="$CONNECT_TIMEOUT_SECONDS"
|
|
-o ConnectionAttempts=1
|
|
-o BatchMode=yes
|
|
-o StrictHostKeyChecking=accept-new
|
|
-o ServerAliveInterval=2
|
|
-o ServerAliveCountMax=1
|
|
)
|
|
|
|
ssh_command_opts=(
|
|
-p "$PORT"
|
|
-o ConnectTimeout="$CONNECT_TIMEOUT_SECONDS"
|
|
-o ConnectionAttempts=1
|
|
-o BatchMode=yes
|
|
-o StrictHostKeyChecking=accept-new
|
|
-o ServerAliveInterval=2
|
|
-o ServerAliveCountMax=1
|
|
-o PasswordAuthentication=no
|
|
-o PreferredAuthentications=publickey
|
|
)
|
|
|
|
run_timeout() {
|
|
local seconds="$1"
|
|
shift
|
|
if command -v timeout >/dev/null 2>&1; then
|
|
timeout "$seconds" "$@"
|
|
return $?
|
|
fi
|
|
"$@"
|
|
}
|
|
|
|
classify_log() {
|
|
local path="$1"
|
|
if grep -q 'Server accepts key' "$path" && grep -Eiq 'timed out|not responding|Timeout' "$path"; then
|
|
echo "server_accepts_key_then_timeout"
|
|
elif grep -q 'Server accepts key' "$path"; then
|
|
echo "server_accepts_key"
|
|
elif grep -q 'Offering public key' "$path" && grep -Eiq 'timed out|not responding|Timeout' "$path"; then
|
|
echo "publickey_offer_timeout"
|
|
elif grep -q 'Permission denied' "$path"; then
|
|
echo "permission_denied"
|
|
elif grep -Eiq 'timed out|not responding|Timeout' "$path"; then
|
|
echo "preauth_timeout"
|
|
elif grep -q 'Connection established' "$path"; then
|
|
echo "connected_unclassified"
|
|
else
|
|
echo "connection_failed"
|
|
fi
|
|
}
|
|
|
|
probe_node_exporter() {
|
|
local metrics cpu_count load1 load1_per_cpu
|
|
metrics="$(curl -fsS --max-time "$NODE_EXPORTER_TIMEOUT_SECONDS" "http://${HOST}:${NODE_EXPORTER_PORT}/metrics" 2>/dev/null || true)"
|
|
if [[ -z "$metrics" ]]; then
|
|
echo "NODE_EXPORTER=unreachable"
|
|
return
|
|
fi
|
|
echo "NODE_EXPORTER=ok"
|
|
awk '
|
|
$1 == "node_boot_time_seconds" {print "NODE_BOOT_TIME_SECONDS="$2}
|
|
$1 == "node_time_seconds" {node_time=$2; print "NODE_TIME_SECONDS="$2}
|
|
$1 == "node_load1" {print "NODE_LOAD1="$2}
|
|
$1 == "node_load5" {print "NODE_LOAD5="$2}
|
|
$1 == "node_load15" {print "NODE_LOAD15="$2}
|
|
$1 == "node_procs_blocked" {print "NODE_PROCS_BLOCKED="$2}
|
|
$1 == "node_memory_MemAvailable_bytes" {print "NODE_MEM_AVAILABLE_BYTES="$2}
|
|
$1 == "node_memory_MemTotal_bytes" {print "NODE_MEM_TOTAL_BYTES="$2}
|
|
/^node_textfile_mtime_seconds/ && /docker_stats\.prom/ {docker_stats_mtime=$NF}
|
|
END {
|
|
if (docker_stats_mtime != "") {
|
|
print "DOCKER_STATS_TEXTFILE_MTIME_SECONDS=" docker_stats_mtime
|
|
if (node_time != "") {
|
|
age = node_time - docker_stats_mtime
|
|
printf "DOCKER_STATS_TEXTFILE_AGE_SECONDS=%.0f\n", age
|
|
print "DOCKER_STATS_TEXTFILE_FRESHNESS=" (age <= 300 ? "fresh" : "stale")
|
|
}
|
|
} else {
|
|
print "DOCKER_STATS_TEXTFILE_FRESHNESS=missing"
|
|
}
|
|
}
|
|
' <<<"$metrics"
|
|
cpu_count="$(awk -F'cpu="' '/^node_cpu_seconds_total/ {split($2, a, "\""); seen[a[1]]=1} END {for (cpu in seen) n++; print n+0}' <<<"$metrics")"
|
|
load1="$(awk '$1 == "node_load1" {print $2; exit}' <<<"$metrics")"
|
|
echo "NODE_CPU_COUNT=${cpu_count:-0}"
|
|
if [[ -n "${load1:-}" && "${cpu_count:-0}" -gt 0 ]]; then
|
|
load1_per_cpu="$(awk -v load="$load1" -v cpus="$cpu_count" 'BEGIN {printf "%.2f", load / cpus}')"
|
|
echo "NODE_LOAD1_PER_CPU=$load1_per_cpu"
|
|
awk -v ratio="$load1_per_cpu" 'BEGIN {print "NODE_LOAD_CLASSIFIER=" (ratio > 1.5 ? "high_load" : "load_not_high")}'
|
|
fi
|
|
awk '
|
|
/^systemd_unit_info/ && /unit="(actions\.runner|awoooi-cd-lane)/ {
|
|
line=$0
|
|
unit=line
|
|
active=line
|
|
substate=line
|
|
sub(/^.*unit="/, "", unit)
|
|
sub(/".*$/, "", unit)
|
|
sub(/^.*active_state="/, "", active)
|
|
sub(/".*$/, "", active)
|
|
sub(/^.*sub_state="/, "", substate)
|
|
sub(/".*$/, "", substate)
|
|
classifier=active
|
|
if (active == "scrape_error" && substate ~ /(timed out|timeout)/) {
|
|
classifier="systemctl_show_timeout"
|
|
} else if (active == "scrape_skipped" && substate ~ /systemctl_timeout_budget_exhausted/) {
|
|
classifier="systemctl_timeout_budget_exhausted"
|
|
}
|
|
printf "SYSTEMD_UNIT unit=%s active_state=%s classifier=%s\n", unit, active, classifier
|
|
}
|
|
' <<<"$metrics"
|
|
}
|
|
|
|
probe_tcp_banner() {
|
|
if command -v nc >/dev/null 2>&1; then
|
|
if run_timeout "$CONNECT_TIMEOUT_SECONDS" nc -v "$HOST" "$PORT" </dev/null >/tmp/awoooi-110-nc.out 2>&1; then
|
|
echo "SSH_PORT=tcp_open"
|
|
sed -n 's/^SSH-/SSH_BANNER=SSH-/p' /tmp/awoooi-110-nc.out | head -1
|
|
return
|
|
fi
|
|
fi
|
|
echo "SSH_PORT=tcp_unconfirmed"
|
|
}
|
|
|
|
probe_user() {
|
|
local user="$1"
|
|
local mode="$2"
|
|
local log="$tmp_dir/${user}-${mode}.log"
|
|
local rc classification
|
|
|
|
if [[ "$mode" == "publickey" ]]; then
|
|
run_timeout "$SSH_ATTEMPT_TIMEOUT_SECONDS" \
|
|
ssh "${ssh_base_opts[@]}" \
|
|
-o PasswordAuthentication=no \
|
|
-o PreferredAuthentications=publickey \
|
|
"${user}@${HOST}" 'true' >"$log" 2>&1
|
|
rc=$?
|
|
else
|
|
run_timeout "$SSH_ATTEMPT_TIMEOUT_SECONDS" \
|
|
ssh "${ssh_base_opts[@]}" \
|
|
-o PubkeyAuthentication=no \
|
|
-o PreferredAuthentications=password \
|
|
-o NumberOfPasswordPrompts=0 \
|
|
"${user}@${HOST}" 'true' >"$log" 2>&1
|
|
rc=$?
|
|
fi
|
|
|
|
classification="$(classify_log "$log")"
|
|
printf 'SSH_AUTH user=%s mode=%s rc=%s classification=%s\n' "$user" "$mode" "$rc" "$classification"
|
|
}
|
|
|
|
probe_command_path_user() {
|
|
local user="$1"
|
|
local stdout_path="$tmp_dir/${user}-command-path.out"
|
|
local stderr_path="$tmp_dir/${user}-command-path.err"
|
|
local rc classification marker_seen remote_user remote_user_match
|
|
|
|
run_timeout "$SSH_COMMAND_PATH_TIMEOUT_SECONDS" \
|
|
ssh "${ssh_command_opts[@]}" \
|
|
"${user}@${HOST}" \
|
|
'printf "AWOOOI_110_COMMAND_PATH_READY=1\n"; printf "remote_user=%s\n" "$(id -un)"' \
|
|
>"$stdout_path" 2>"$stderr_path"
|
|
rc=$?
|
|
|
|
marker_seen=false
|
|
remote_user_match=false
|
|
remote_user="$(awk -F= '$1 == "remote_user" {print $2; exit}' "$stdout_path" 2>/dev/null || true)"
|
|
if grep -qx 'AWOOOI_110_COMMAND_PATH_READY=1' "$stdout_path" 2>/dev/null; then
|
|
marker_seen=true
|
|
fi
|
|
if [[ "$remote_user" == "$user" ]]; then
|
|
remote_user_match=true
|
|
fi
|
|
|
|
if [[ "$marker_seen" == "true" && "$remote_user_match" == "true" ]]; then
|
|
classification="command_path_ready"
|
|
elif grep -Eiq 'timed out|not responding|Timeout' "$stderr_path"; then
|
|
classification="command_path_timeout"
|
|
elif grep -q 'Permission denied' "$stderr_path"; then
|
|
classification="command_path_permission_denied"
|
|
else
|
|
classification="command_path_unavailable"
|
|
fi
|
|
|
|
printf 'SSH_COMMAND_PATH user=%s rc=%s classification=%s marker_seen=%s remote_user_match=%s\n' \
|
|
"$user" "$rc" "$classification" "$marker_seen" "$remote_user_match"
|
|
}
|
|
|
|
echo "AWOOOI_110_SSH_PUBLICKEY_AUTH_DIAGNOSIS"
|
|
echo "TARGET=${HOST}:${PORT}"
|
|
probe_node_exporter
|
|
probe_tcp_banner
|
|
|
|
for user in "${USERS[@]}"; do
|
|
probe_user "$user" "publickey"
|
|
done
|
|
|
|
probe_command_path_user "$COMMAND_PATH_USER"
|
|
|
|
for user in "${USERS[@]}"; do
|
|
probe_user "$user" "password_disabled"
|
|
done
|
|
|
|
echo "INTERPRETATION=command_path_ready_overrides_stale_verbose_true_timeout;server_accepts_key_then_timeout_without_command_path_means_check_110_session_pam_account_or_shell_path;publickey_offer_timeout_means_check_110_authorized_keys_permissions_pam_or_account_lookup_path"
|