From 78418babf749a6306e154eb57ef38e8c7a0f1449 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 30 Jun 2026 19:26:37 +0800 Subject: [PATCH] fix(recovery): classify 110 ssh publickey auth blocker --- .gitea/workflows/cd.yaml | 6 + docs/LOGBOOK.md | 11 ++ .../runbooks/REBOOT-POST-START-QUICK-CHECK.md | 3 +- ...oot-cold-start-backup-recovery-workplan.md | 2 +- .../test_cd_controlled_runtime_profile.py | 4 + .../diagnose-110-ssh-publickey-auth.sh | 137 ++++++++++++++++++ .../repair-110-ssh-publickey-auth-local.sh | 132 +++++++++++++++++ .../test_cold_start_monitor_bounded_probes.py | 38 +++++ 8 files changed, 331 insertions(+), 2 deletions(-) create mode 100755 scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh create mode 100755 scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 8c7c7923d..c77154a11 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -502,6 +502,10 @@ jobs: ;; scripts/reboot-recovery/full-stack-recovery-scorecard.sh) ;; + scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh) + ;; + scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh) + ;; scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh) ;; scripts/reboot-recovery/tests/test_dr_escrow_evidence_checklist.py) @@ -685,6 +689,8 @@ jobs: ../../scripts/reboot-recovery/188-host-hygiene-maintenance-checklist.sh \ ../../scripts/reboot-recovery/full-stack-cold-start-check.sh \ ../../scripts/reboot-recovery/full-stack-recovery-scorecard.sh \ + ../../scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh \ + ../../scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh \ ../../scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh \ ../../scripts/backup/gitea-repo-bundle-backup.sh DATABASE_URL="${DATABASE_URL:-postgresql+asyncpg://ci:ci@localhost/ci}" \ diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 6e2c88d07..b5ad857e1 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,14 @@ +## 2026-06-30 — current P0-5 110 SSH publickey auth classifier + +**照主線修正的問題**： +- 110 不再只記為泛化 `SSH timeout`：新增 `scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh`，從外部只讀檢查 node-exporter、SSH banner、publickey auth 與 password-disabled 對照，輸出機器可讀 classifier。 +- live 診斷讀回：node-exporter `ok`、SSH banner `OpenSSH_8.9p1 Ubuntu-3ubuntu0.15`、`NODE_LOAD_CLASSIFIER=high_load`、`NODE_PROCS_BLOCKED=0`、`wooo` publickey `publickey_offer_timeout`、`root` publickey `permission_denied`、`git` / `ollama` `preauth_timeout`。這把 P0-5 收斂到 110 sshd publickey auth / authorized_keys / PAM / account lookup path，並將 load / runner pressure 掛在同一 blocker，不是另開支線。 +- 新增 `scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh`：只能在 110 本機/console 或已可用 root shell 執行；`--check` 只讀檢查 sshd syntax、ssh service、target user、home / `.ssh` / authorized_keys metadata；`--apply` 只修 ownership / `700` / `600` metadata，不建立 key、不印 authorized_keys 內容，預設不 reload ssh。 +- CD controlled-runtime profile 已納入兩支腳本與 focused cold-start contract tests，避免 P0-5 工具只停在本地。 +- P0-4 deploy readback：public Gitea Actions 顯示 `cd.yaml #4040` 對 `fd7c513ca` 為 `Failure`，但未登入的 actions list API 回 `401 token is required`，run jobs API 只能作 limited public evidence；不得宣稱 production 已部署最新。 + +**邊界**：未 SSH 寫 110，未重啟主機，未 restart Docker daemon / host Nginx / K3s / DB / Redis / firewall，未 restore / prune / DB write，未讀 secret / token / raw sessions / SQLite / `.env`，未使用 GitHub / `gh` / GitHub API。 + ## 2026-06-30 — current P0 readback after verified Gitea bundle **目前不能宣稱完成的事實**： diff --git a/docs/runbooks/REBOOT-POST-START-QUICK-CHECK.md b/docs/runbooks/REBOOT-POST-START-QUICK-CHECK.md index 8884d1af5..60546afea 100644 --- a/docs/runbooks/REBOOT-POST-START-QUICK-CHECK.md +++ b/docs/runbooks/REBOOT-POST-START-QUICK-CHECK.md @@ -30,7 +30,7 @@ | Tsenyang 502 | `www.tsenyang.com` / `tsenyang.com` 起初 HTTP 502；188 release `/home/ollama/tsenyang-website-releases/b369ed8` 有 source，但 `tsenyang-website:latest` image 缺失、container 未啟動。 | 使用者可見 502 是 P0；可用現有 release 做 bounded `docker build --quiet -t tsenyang-website:latest .` + `docker compose up -d --no-build`，再以 local 3000 與 public 200 驗證。 | | Tsenyang 修復結果 | image `sha256:e25e0acb...` 建成，container `tsenyang-website` running，local `127.0.0.1:3000`、`www.tsenyang.com`、`tsenyang.com` 均回 200。 | 下次若同樣 502，先查 image/container/release symlink，不先動 Nginx、DNS、DB 或 host reboot。 | | StockPlatform freshness | public `/healthz`、`/api/healthz` 回 200；`/api/v1/system/freshness` 與 `/api/v1/system/ingestion` 回 `status=not_configured`、blocker `postgres_not_ready`。 | service up 不等於資料最新；Stock 必須修 PostgreSQL schema/readiness 或資料 source readback，禁止 fake freshness、manual DB row、restore/prune。 | -| 110 control path | node-exporter 可讀、22 port banner 正常；禁用 publickey 時立即 `Permission denied`，但 publickey path 在送出 key 後 timeout、server 未回 `accepts key`。 | 110 control path 是 P0 blocker；下一步集中查 sshd publickey auth / authorized_keys / PAM / account lookup path，未恢復前不可宣稱 Stock DB/schema、Gitea dump、110 backup completeness 已現場驗證。 | +| 110 control path | `diagnose-110-ssh-publickey-auth.sh` 讀回 node-exporter / SSH banner 正常；`NODE_LOAD_CLASSIFIER=high_load`、`NODE_PROCS_BLOCKED=0`；`wooo` publickey 為 `publickey_offer_timeout`，`root` publickey 為 `permission_denied`，`git` / `ollama` 為 `preauth_timeout`。 | 110 control path 是 P0 blocker；下一步集中查 sshd publickey auth / authorized_keys / PAM / account lookup path，同時追蹤 load / runner pressure 是否拖慢 auth。可在 110 console 跑 `repair-110-ssh-publickey-auth-local.sh --check`，確認後才 `--apply` 修 metadata 權限；未恢復前不可宣稱 Stock DB/schema、Gitea dump、110 backup completeness 已現場驗證。 | | Gitea visibility | `https://gitea.wooo.work/api/v1/version` 回 200 / `1.25.5`；public repo search 只列 4 個 public repos；`stockplatform-v2` public page/API 404，但 direct internal Git remote `http://192.168.0.110:3001/wooo/stockplatform-v2.git` 可 `ls-remote`。 | 不能直接說 repo 全消失；要區分 public visibility、private/auth visibility、內網 Git remote、server-side storage 四層。 | | Gitea backup | 188 `/home/ollama/backup/110/gitea` 起初為空；verified emergency bundle 後，`gitea_repo_mirror_from_110` metric fresh `1`、snapshot_count `19`。 | 子樹 metric 已能看見 emergency bundle，但仍只代表部分 Git history；正式 `gitea dump`、private repo non-interactive backup、repo count 與 restore drill 仍是 P0 blocker。 | | Gitea emergency bundle | 188 已建立 verified emergency Git bundle snapshot `/home/ollama/backup/110/gitea/git-bundles/20260630-190931`，`awoooi`、`ewoooc`、`2026FIFAWorldCup`、`agent-bounty-protocol` bundle verify + `.sha256` 成功；`AwoooGo`、`stockplatform-v2`、`vibework` 因 private repo 需要非互動 credential 而 fail-closed。 | 這只保住部分 repo Git history，不等於 `gitea dump`；Gitea DB、settings、issues、packages、secrets、LFS、private repo completeness 與 restore drill 仍是 P0 blocker。 | @@ -41,6 +41,7 @@ - `backup_from_110 fresh=1` 只能代表整體 rsync 成功；Gitea、Harbor、DB、網站、logs 等 domain 必須有獨立 freshness 或 sample count。 - Gitea repo 疑似消失時，順序固定為 `public search` → `repo page/API` → `GIT_TERMINAL_PROMPT=0 git ls-remote ` → server-side storage / dump / backup readback；不得先 restore 或 delete。 - 使用者可見 502 先啟用 L0 maintenance fallback 或恢復單站 container；若 edge/99/188 整段不可達，才進 L1 external fallback / CDN。 +- 110 SSH timeout 不得再泛化成「主機不可達」；先跑 `scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh`，若 `wooo` 是 `publickey_offer_timeout`，修復順序固定為 110 console/local `repair-110-ssh-publickey-auth-local.sh --check` → `--apply` metadata 權限 → 選擇性 `RELOAD_SSH=1` reload → 外部診斷器重跑。 --- diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index dc6b25cbf..4d6ccc4e3 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -19,7 +19,7 @@ | P0-2 | DONE_THIS_INCIDENT | 使用者可見 502：Tsenyang | `www.tsenyang.com` / `tsenyang.com` 由 502 恢復為 200；188 `tsenyang-website` container running；local `127.0.0.1:3000` 回 200。 | 下次同類 502 先查 release symlink / image / container；不先動 Nginx、DNS、DB、主機重啟。 | | P0-3 | BLOCKED | StockPlatform data freshness | public `/healthz`、`/api/healthz` 回 200；freshness / ingestion 回 `not_configured`、`postgres_not_ready`。 | 恢復 110 control path 後，read-only 查 `/home/wooo/stockplatform-v2` compose / DB schema / migration status；禁止 fake freshness、manual DB rows、restore/prune。 | | P0-4 | BLOCKED | AWOOOI production 版本最新性 | source/Gitea `main=3b65f8762`；production runtime readback 仍為 `7890778b83`，`runtime_build_readback_status=runtime_build_diverges_from_committed_deploy_readback`。 | 補 deploy marker / runtime SHA / endpoint readback 一致；未一致前不可宣稱 AWOOOI 最新。 | -| P0-5 | BLOCKED | 110 control path | node-exporter 可讀、22 port banner 正常；禁用 publickey 時立即 `Permission denied`，但 publickey path 在送出 key 後 timeout、server 未回 `accepts key`。 | 集中查 110 sshd publickey auth / authorized_keys / PAM / account lookup path；恢復 SSH read-only command path 後才能驗證 Stock DB、Gitea dump、110 backup completeness。 | +| P0-5 | BLOCKED | 110 control path | `diagnose-110-ssh-publickey-auth.sh`：node-exporter / SSH banner 正常；`NODE_LOAD_CLASSIFIER=high_load`、`NODE_PROCS_BLOCKED=0`；`wooo` publickey `publickey_offer_timeout`，`root` publickey `permission_denied`，`git` / `ollama` `preauth_timeout`。 | 集中查 110 sshd publickey auth / authorized_keys / PAM / account lookup path，並把 load / runner pressure 視為同一 blocker 的共因；可在 110 console/local 跑 `repair-110-ssh-publickey-auth-local.sh --check` / `--apply` 修 metadata 權限。恢復 SSH read-only command path 後才能驗證 Stock DB、Gitea dump、110 backup completeness。 | | P0-6 | BLOCKED_BACKUP_COMPLETENESS | Gitea repo visibility 與完整備份 | Gitea version API 200；public repo search 只列 4 個 public repo；`stockplatform-v2` public page/API 404，但 internal `git ls-remote` 成功；188 `/home/ollama/backup/110/gitea` 起初為空。已建立 verified emergency bundle `/home/ollama/backup/110/gitea/git-bundles/20260630-190931`：4 個 public/internal repo bundle verify + checksum 成功，`AwoooGo`、`stockplatform-v2`、`vibework` 因 private auth fail-closed。 | 188 `gitea_repo_mirror_from_110` subtree metric / alert 已補；下一步仍是恢復 110 SSH command path 後跑正式 `gitea dump`、private repo 非互動備份、repo count 與 restore drill readback。 | | P0-7 | SOURCE_READY_RUNTIME_BLOCKED | 99 VMware / VM autostart | repo 已有 `windows99-vmware-autostart.ps1`，但 99 SSH/WinRM 尚未可用；VM host 111 仍不可達。 | 恢復 99 可控通道或由 console 套用腳本；完成後讀回 111/188/120/121/112 boot evidence。 | | P0-8 | SOURCE_READY_RUNTIME_BLOCKED | 502 maintenance fallback / Telegram / backup alert | L0/L1 fallback runbook、Nginx snippet、reboot / backup alert rules 已在 source；runtime 尚需部署與外部 L1 provider readback。 | L0 以測試 vhost 驗證 `X-AWOOOI-Fallback`；L1 需外部雲端/CDN probe；Telegram 以脫敏 alert receipt 驗證。 | diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index 526db3a70..3b824eacd 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -339,6 +339,8 @@ def test_reboot_auto_recovery_slo_sources_stay_on_controlled_runtime_profile() - "scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py)", "scripts/reboot-recovery/full-stack-cold-start-check.sh)", "scripts/reboot-recovery/full-stack-recovery-scorecard.sh)", + "scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh)", + "scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh)", "scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh)", "scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py)", "scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py)", @@ -376,6 +378,8 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "../../scripts/reboot-recovery/188-host-hygiene-maintenance-checklist.sh", "../../scripts/reboot-recovery/full-stack-cold-start-check.sh", "../../scripts/reboot-recovery/full-stack-recovery-scorecard.sh", + "../../scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh", + "../../scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh", "../../scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh", "../../scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py", "../../scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py", diff --git a/scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh b/scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh new file mode 100755 index 000000000..f4b70e694 --- /dev/null +++ b/scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# Read-only classifier for 110 SSH publickey authentication stalls. +# +# Run from an operator host. It does not write remote state, does not read +# authorized_keys contents, and does not ask for passwords. + +set -uo pipefail + +HOST="${HOST:-192.168.0.110}" +PORT="${PORT:-22}" +NODE_EXPORTER_PORT="${NODE_EXPORTER_PORT:-9100}" +CONNECT_TIMEOUT_SECONDS="${CONNECT_TIMEOUT_SECONDS:-4}" +SSH_ATTEMPT_TIMEOUT_SECONDS="${SSH_ATTEMPT_TIMEOUT_SECONDS:-8}" +NODE_EXPORTER_TIMEOUT_SECONDS="${NODE_EXPORTER_TIMEOUT_SECONDS:-8}" +USERS=(${USERS:-wooo root git ollama}) + +tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/awoooi-110-ssh-auth.XXXXXX")" +trap 'rm -rf "$tmp_dir"' EXIT + +ssh_base_opts=( + -vv + -p "$PORT" + -o ConnectTimeout="$CONNECT_TIMEOUT_SECONDS" + -o ConnectionAttempts=1 + -o BatchMode=yes + -o StrictHostKeyChecking=accept-new + -o ServerAliveInterval=2 + -o ServerAliveCountMax=1 +) + +run_timeout() { + local seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + return $? + fi + "$@" +} + +classify_log() { + local path="$1" + if grep -q 'Server accepts key' "$path"; then + echo "server_accepts_key" + elif grep -q 'Offering public key' "$path" && grep -Eiq 'timed out|not responding|Timeout' "$path"; then + echo "publickey_offer_timeout" + elif grep -q 'Permission denied' "$path"; then + echo "permission_denied" + elif grep -Eiq 'timed out|not responding|Timeout' "$path"; then + echo "preauth_timeout" + elif grep -q 'Connection established' "$path"; then + echo "connected_unclassified" + else + echo "connection_failed" + fi +} + +probe_node_exporter() { + local metrics cpu_count load1 load1_per_cpu + metrics="$(curl -fsS --max-time "$NODE_EXPORTER_TIMEOUT_SECONDS" "http://${HOST}:${NODE_EXPORTER_PORT}/metrics" 2>/dev/null || true)" + if [[ -z "$metrics" ]]; then + echo "NODE_EXPORTER=unreachable" + return + fi + echo "NODE_EXPORTER=ok" + awk ' + $1 == "node_boot_time_seconds" {print "NODE_BOOT_TIME_SECONDS="$2} + $1 == "node_time_seconds" {print "NODE_TIME_SECONDS="$2} + $1 == "node_load1" {print "NODE_LOAD1="$2} + $1 == "node_load5" {print "NODE_LOAD5="$2} + $1 == "node_load15" {print "NODE_LOAD15="$2} + $1 == "node_procs_blocked" {print "NODE_PROCS_BLOCKED="$2} + $1 == "node_memory_MemAvailable_bytes" {print "NODE_MEM_AVAILABLE_BYTES="$2} + $1 == "node_memory_MemTotal_bytes" {print "NODE_MEM_TOTAL_BYTES="$2} + ' <<<"$metrics" + cpu_count="$(awk -F'cpu="' '/^node_cpu_seconds_total/ {split($2, a, "\""); seen[a[1]]=1} END {for (cpu in seen) n++; print n+0}' <<<"$metrics")" + load1="$(awk '$1 == "node_load1" {print $2; exit}' <<<"$metrics")" + echo "NODE_CPU_COUNT=${cpu_count:-0}" + if [[ -n "${load1:-}" && "${cpu_count:-0}" -gt 0 ]]; then + load1_per_cpu="$(awk -v load="$load1" -v cpus="$cpu_count" 'BEGIN {printf "%.2f", load / cpus}')" + echo "NODE_LOAD1_PER_CPU=$load1_per_cpu" + awk -v ratio="$load1_per_cpu" 'BEGIN {print "NODE_LOAD_CLASSIFIER=" (ratio > 1.5 ? "high_load" : "load_not_high")}' + fi +} + +probe_tcp_banner() { + if command -v nc >/dev/null 2>&1; then + if run_timeout "$CONNECT_TIMEOUT_SECONDS" nc -v "$HOST" "$PORT" /tmp/awoooi-110-nc.out 2>&1; then + echo "SSH_PORT=tcp_open" + sed -n 's/^SSH-/SSH_BANNER=SSH-/p' /tmp/awoooi-110-nc.out | head -1 + return + fi + fi + echo "SSH_PORT=tcp_unconfirmed" +} + +probe_user() { + local user="$1" + local mode="$2" + local log="$tmp_dir/${user}-${mode}.log" + local rc classification + + if [[ "$mode" == "publickey" ]]; then + run_timeout "$SSH_ATTEMPT_TIMEOUT_SECONDS" \ + ssh "${ssh_base_opts[@]}" \ + -o PasswordAuthentication=no \ + -o PreferredAuthentications=publickey \ + "${user}@${HOST}" 'true' >"$log" 2>&1 + rc=$? + else + run_timeout "$SSH_ATTEMPT_TIMEOUT_SECONDS" \ + ssh "${ssh_base_opts[@]}" \ + -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password \ + -o NumberOfPasswordPrompts=0 \ + "${user}@${HOST}" 'true' >"$log" 2>&1 + rc=$? + fi + + classification="$(classify_log "$log")" + printf 'SSH_AUTH user=%s mode=%s rc=%s classification=%s\n' "$user" "$mode" "$rc" "$classification" +} + +echo "AWOOOI_110_SSH_PUBLICKEY_AUTH_DIAGNOSIS" +echo "TARGET=${HOST}:${PORT}" +probe_node_exporter +probe_tcp_banner + +for user in "${USERS[@]}"; do + probe_user "$user" "publickey" +done + +for user in "${USERS[@]}"; do + probe_user "$user" "password_disabled" +done + +echo "INTERPRETATION=publickey_offer_timeout_on_wooo_means_check_110_authorized_keys_permissions_pam_or_account_lookup_path" diff --git a/scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh b/scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh new file mode 100755 index 000000000..9de1d8234 --- /dev/null +++ b/scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Local-only check/apply helper for 110 SSH publickey authentication stalls. +# +# Run on host 110 from a trusted local console or an already working root shell. +# It never prints authorized_keys contents. Apply only fixes metadata permissions +# on the target user's home/.ssh/authorized_keys path and runs sshd syntax checks. + +set -euo pipefail + +TARGET_USER="${TARGET_USER:-wooo}" +APPLY=0 +RELOAD_SSH="${RELOAD_SSH:-0}" + +usage() { + cat <<'USAGE' +Usage: scripts/reboot-recovery/repair-110-ssh-publickey-auth-local.sh [--check|--apply] + +Options: + --check Read-only checks. Default. + --apply Fix ownership/permissions for TARGET_USER home/.ssh/authorized_keys. + +Environment: + TARGET_USER=wooo Account to inspect/fix. + RELOAD_SSH=0 Set to 1 to run `systemctl reload ssh` after apply. + +This script does not print authorized_keys contents and does not create keys. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --check) + APPLY=0 + ;; + --apply) + APPLY=1 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 64 + ;; + esac + shift +done + +require_110_or_explicit() { + if [ "${ALLOW_NON_110:-0}" = "1" ]; then + return + fi + if hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx '192.168.0.110'; then + return + fi + echo "BLOCKED not running on 192.168.0.110; set ALLOW_NON_110=1 only for syntax tests" >&2 + exit 65 +} + +stat_path() { + local path="$1" + if [ -e "$path" ]; then + stat -c 'PATH_STATUS path=%n mode=%a owner=%U group=%G type=%F' "$path" + else + echo "PATH_STATUS path=$path missing=1" + fi +} + +check_user() { + local user="$1" + local home_dir + home_dir="$(getent passwd "$user" | awk -F: '{print $6}')" + if [ -z "$home_dir" ]; then + echo "USER_STATUS user=$user exists=0" + return 1 + fi + + echo "USER_STATUS user=$user exists=1 home=$home_dir" + stat_path "$home_dir" + stat_path "$home_dir/.ssh" + stat_path "$home_dir/.ssh/authorized_keys" + if [ -f "$home_dir/.ssh/authorized_keys" ]; then + echo "AUTHORIZED_KEYS_STATUS path=$home_dir/.ssh/authorized_keys exists=1 bytes=$(wc -c < "$home_dir/.ssh/authorized_keys" | tr -d ' ') lines=$(wc -l < "$home_dir/.ssh/authorized_keys" | tr -d ' ')" + else + echo "AUTHORIZED_KEYS_STATUS path=$home_dir/.ssh/authorized_keys exists=0" + fi +} + +apply_user_permissions() { + local user="$1" + local home_dir + home_dir="$(getent passwd "$user" | awk -F: '{print $6}')" + if [ -z "$home_dir" ]; then + echo "BLOCKED target user missing: $user" >&2 + exit 66 + fi + if [ ! -d "$home_dir/.ssh" ]; then + echo "BLOCKED missing $home_dir/.ssh; not creating key material or SSH directories automatically" >&2 + exit 67 + fi + if [ ! -f "$home_dir/.ssh/authorized_keys" ]; then + echo "BLOCKED missing $home_dir/.ssh/authorized_keys; not creating key material automatically" >&2 + exit 68 + fi + + chown "$user:$user" "$home_dir/.ssh" "$home_dir/.ssh/authorized_keys" + chmod 700 "$home_dir/.ssh" + chmod 600 "$home_dir/.ssh/authorized_keys" + echo "APPLIED permissions target_user=$user home=$home_dir" +} + +require_110_or_explicit + +echo "AWOOOI_110_SSH_PUBLICKEY_AUTH_LOCAL_REPAIR mode=$([ "$APPLY" -eq 1 ] && echo apply || echo check) target_user=$TARGET_USER" +systemctl is-active ssh 2>/dev/null | sed 's/^/SSH_SERVICE_ACTIVE=/' || true +sshd -t +echo "SSHD_CONFIG_SYNTAX=ok" +check_user "$TARGET_USER" + +if [ "$APPLY" -eq 1 ]; then + apply_user_permissions "$TARGET_USER" + sshd -t + echo "SSHD_CONFIG_SYNTAX_AFTER_APPLY=ok" + if [ "$RELOAD_SSH" = "1" ]; then + systemctl reload ssh + echo "SSH_RELOAD=done" + else + echo "SSH_RELOAD=skipped" + fi +fi diff --git a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py index f668b55df..e09c22864 100644 --- a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py +++ b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py @@ -10,6 +10,12 @@ RECOVERY_SCORECARD = ( ROOT / "scripts" / "reboot-recovery" / "full-stack-recovery-scorecard.sh" ) VERIFY_DEPLOY = ROOT / "scripts" / "reboot-recovery" / "verify-cold-start-monitor-deploy.sh" +SSH_AUTH_DIAGNOSE = ( + ROOT / "scripts" / "reboot-recovery" / "diagnose-110-ssh-publickey-auth.sh" +) +SSH_AUTH_REPAIR = ( + ROOT / "scripts" / "reboot-recovery" / "repair-110-ssh-publickey-auth-local.sh" +) def test_full_stack_cold_start_check_bounds_ssh_probes() -> None: @@ -81,3 +87,35 @@ def test_cold_start_deploy_parity_verifier_bounds_ssh_readback() -> None: assert "timeout ${REMOTE_COMMAND_TIMEOUT_SECONDS}s bash -lc" in text assert 'remote_read "sha256sum' in text assert 'if remote_read "grep -Fq' in text + + +def test_110_ssh_publickey_auth_diagnosis_is_bounded_and_read_only() -> None: + text = SSH_AUTH_DIAGNOSE.read_text(encoding="utf-8") + + assert "PasswordAuthentication=no" in text + assert "PubkeyAuthentication=no" in text + assert "NumberOfPasswordPrompts=0" in text + assert "publickey_offer_timeout" in text + assert "NODE_EXPORTER=ok" in text + assert "NODE_LOAD1_PER_CPU" in text + assert "NODE_LOAD_CLASSIFIER" in text + assert "cat /home" not in text + assert "cat ~/.ssh/authorized_keys" not in text + assert "cat \"$home_dir/.ssh/authorized_keys\"" not in text + assert "systemctl" not in text + assert "chmod" not in text + assert "chown" not in text + + +def test_110_ssh_publickey_auth_repair_is_local_and_does_not_print_keys() -> None: + text = SSH_AUTH_REPAIR.read_text(encoding="utf-8") + + assert "not running on 192.168.0.110" in text + assert "--apply" in text + assert "chmod 700" in text + assert "chmod 600" in text + assert "chown \"$user:$user\"" in text + assert "sshd -t" in text + assert 'RELOAD_SSH="${RELOAD_SSH:-0}"' in text + assert "cat \"$home_dir/.ssh/authorized_keys\"" not in text + assert "echo \"$(cat" not in text