awoooi/scripts/reboot-recovery/188-host-hygiene-maintenance-checklist.sh

#!/usr/bin/env bash
# 188 host hygiene 維護前只讀檢查。
# 本腳本不會 restart、reload、renew、reset-failed、restore、pg_resetwal 或修改遠端狀態。

set -uo pipefail

REMOTE_188="${REMOTE_188:-ollama@192.168.0.188}"
REMOTE_110="${REMOTE_110:-wooo@192.168.0.110}"
SSH_BATCH_MODE="${SSH_BATCH_MODE:-yes}"
SSH_STRICT_HOST_KEY_CHECKING="${SSH_STRICT_HOST_KEY_CHECKING:-accept-new}"
SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}"
SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-25}"
NO_COLOR=0

usage() {
  cat <<'USAGE'
Usage: bash scripts/reboot-recovery/188-host-hygiene-maintenance-checklist.sh [--no-color]

Read-only pre-maintenance checklist for host 188 hygiene issues.
It prints evidence for host PostgreSQL, certbot, product containers, backup core,
and route health. It never runs pg_resetwal, certbot renew, reset-failed, restore,
Nginx reload, Docker/systemd restart, or any host write.

Environment:
  REMOTE_188=ollama@192.168.0.188
  REMOTE_110=wooo@192.168.0.110
  SSH_BATCH_MODE=yes
  SSH_STRICT_HOST_KEY_CHECKING=accept-new
  SSH_CONNECT_TIMEOUT=8
  SSH_COMMAND_TIMEOUT_SECONDS=25
USAGE
}

while [ "$#" -gt 0 ]; do
  case "$1" in
    --no-color)
      NO_COLOR=1
      shift
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "Unknown argument: $1" >&2
      usage >&2
      exit 64
      ;;
  esac
done

if [ "$NO_COLOR" = "1" ]; then
  green=""
  yellow=""
  red=""
  blue=""
  reset=""
else
  green="$(printf '\033[32m')"
  yellow="$(printf '\033[33m')"
  red="$(printf '\033[31m')"
  blue="$(printf '\033[34m')"
  reset="$(printf '\033[0m')"
fi

PASS=0
WARN=0
BLOCKED=0
SERVICE_GREEN=1
HOST_HYGIENE_BLOCKED=0

ssh_opts=(
  -o BatchMode="$SSH_BATCH_MODE"
  -o ConnectTimeout="$SSH_CONNECT_TIMEOUT"
  -o ConnectionAttempts=1
  -o ServerAliveInterval=5
  -o ServerAliveCountMax=1
  -o PreferredAuthentications=publickey
  -o NumberOfPasswordPrompts=0
  -o StrictHostKeyChecking="$SSH_STRICT_HOST_KEY_CHECKING"
)

local_ip_list() {
  {
    hostname -I 2>/dev/null | tr ' ' '\n' || true
    ip -o -4 addr show 2>/dev/null | awk '{split($4,a,"/"); print a[1]}' || true
    ifconfig 2>/dev/null | awk '$1 == "inet" {print $2}' || true
  } | awk 'NF'
}

is_local_target() {
  local target="$1"
  local host="${target##*@}"
  local ips
  [[ "$host" == "127.0.0.1" || "$host" == "localhost" ]] && return 0
  ips="$(local_ip_list)"
  grep -Fxq "$host" <<<"$ips"
}

section() {
  printf "\n%s=== %s ===%s\n" "$blue" "$1" "$reset"
}

ok() {
  PASS=$((PASS + 1))
  printf "%sOK%s %s\n" "$green" "$reset" "$*"
}

warn() {
  WARN=$((WARN + 1))
  printf "%sWARN%s %s\n" "$yellow" "$reset" "$*"
}

blocked() {
  BLOCKED=$((BLOCKED + 1))
  HOST_HYGIENE_BLOCKED=1
  printf "%sBLOCKED%s %s\n" "$red" "$reset" "$*"
}

ssh_cmd() {
  local target="$1"
  local command="$2"
  local quoted_command=""
  if is_local_target "$target"; then
    bash -lc "$command"
    return $?
  fi
  printf -v quoted_command '%q' "$command"
  ssh "${ssh_opts[@]}" "$target" \
    "if command -v timeout >/dev/null 2>&1; then timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc ${quoted_command}; else bash -lc ${quoted_command}; fi"
}

echo "AWOOOI 188 host hygiene maintenance checklist"
date '+%Y-%m-%d %H:%M:%S %Z'
echo "Scope: 188 host PostgreSQL 14/main, startup unit, certbot / ACME hygiene."
echo "RUNTIME_ACTION_AUTHORIZED=0"

section "188 host state"
if out=$(ssh_cmd "$REMOTE_188" '
hostname
date --iso-8601=seconds
uptime
echo "SYSTEMD_STATE $(systemctl is-system-running || true)"
systemctl --failed --no-pager --plain || true
df -h /
free -h
' 2>&1); then
  echo "$out"
  ok "188 SSH readable"
  if grep -q 'SYSTEMD_STATE running' <<<"$out"; then
    ok "188 systemd running"
  else
    blocked "188 systemd is not fully running; failed units require owner disposition"
  fi
  grep -q 'postgresql@14-main.service' <<<"$out" && blocked "host PostgreSQL failed unit visible" || ok "host PostgreSQL failed unit not visible"
  grep -q 'certbot.service' <<<"$out" && blocked "apt certbot failed unit visible" || ok "apt certbot failed unit not visible"
  grep -q 'snap.certbot.renew.service' <<<"$out" && blocked "snap certbot renew failed unit visible" || ok "snap certbot renew failed unit not visible"
else
  SERVICE_GREEN=0
  blocked "188 SSH host state unavailable"
  echo "$out"
fi

section "188 product container state"
if out=$(ssh_cmd "$REMOTE_188" '
docker ps --format "{{.Names}}\t{{.Status}}\t{{.Ports}}" 2>/dev/null || true
' 2>&1); then
  echo "$out"
  grep -q '^momo-pro-system[[:space:]].*healthy' <<<"$out" && ok "MOMO Pro container healthy" || { SERVICE_GREEN=0; blocked "MOMO Pro container not confirmed healthy"; }
  grep -q '^momo-db[[:space:]].*Up' <<<"$out" && ok "MOMO DB container up" || { SERVICE_GREEN=0; blocked "MOMO DB container not confirmed up"; }
  grep -q '^vibework-production-web-1[[:space:]].*healthy' <<<"$out" && ok "VibeWork production web container healthy" || warn "VibeWork production web container health not confirmed"
else
  SERVICE_GREEN=0
  blocked "188 docker ps unavailable"
  echo "$out"
fi

section "188 host PostgreSQL 14/main"
if out=$(ssh_cmd "$REMOTE_188" '
pg_lsclusters 2>/dev/null || true
systemctl status postgresql@14-main.service --no-pager || true
echo "PG_ISREADY_LOCAL $(pg_isready -h localhost -p 5432 2>/dev/null || true)"
echo "RECOVERY_CONTAINER $(docker inspect -f "{{.State.Running}} {{.HostConfig.NetworkMode}} {{.HostConfig.RestartPolicy.Name}}" k3s-postgres-recovery 2>/dev/null || echo missing)"
' 2>&1); then
  echo "$out"
  recovery_container_ready=0
  if grep -q '^RECOVERY_CONTAINER true host ' <<<"$out" && grep -q 'PG_ISREADY_LOCAL .*accepting connections' <<<"$out"; then
    recovery_container_ready=1
  fi

  if grep -Eq '^14[[:space:]]+main[[:space:]]+5432[[:space:]]+down' <<<"$out"; then
    if [[ "$recovery_container_ready" -eq 1 ]]; then
      warn "host PostgreSQL cluster 14/main is down, but controlled k3s-postgres-recovery runtime is accepting connections"
    else
      blocked "host PostgreSQL cluster 14/main is down and no controlled recovery runtime was accepted"
    fi
  else
    ok "host PostgreSQL cluster 14/main not reported down"
  fi

  if grep -Eiq 'invalid primary checkpoint record|could not locate a valid checkpoint record|PANIC:' <<<"$out"; then
    if [[ "$recovery_container_ready" -eq 1 ]]; then
      warn "PostgreSQL checkpoint/WAL error remains historical host-cluster evidence; pg_resetwal is still break-glass only"
    else
      blocked "PostgreSQL checkpoint/WAL error detected; pg_resetwal is break-glass only"
    fi
  fi

  if [[ "$recovery_container_ready" -eq 1 ]]; then
    ok "PostgreSQL runtime is provided by k3s-postgres-recovery on host network"
  elif grep -q 'PG_ISREADY_LOCAL .*accepting connections' <<<"$out"; then
    warn "pg_isready accepts on localhost; do not use this alone as host 14/main health"
  fi
else
  blocked "PostgreSQL evidence unavailable"
  echo "$out"
fi

section "188 certbot / ACME"
if out=$(ssh_cmd "$REMOTE_188" '
systemctl show certbot.service snap.certbot.renew.service certbot.timer snap.certbot.renew.timer -p Id -p ActiveState -p SubState -p Result -p UnitFileState --no-pager || true
systemctl list-timers --all --no-pager | grep -i certbot || true
' 2>&1); then
  echo "$out"
  if grep -q 'Id=certbot.service' <<<"$out" && grep -A3 'Id=certbot.service' <<<"$out" | grep -q 'Result=failed'; then
    blocked "apt certbot service currently failed"
  else
    ok "apt certbot service is not currently failed"
  fi
  if grep -q 'Id=snap.certbot.renew.service' <<<"$out" && grep -A3 'Id=snap.certbot.renew.service' <<<"$out" | grep -q 'Result=failed'; then
    blocked "snap certbot renew service currently failed"
  else
    ok "snap certbot renew service is not currently failed"
  fi
  if grep -A4 'Id=certbot.timer' <<<"$out" | grep -q 'UnitFileState=disabled'; then
    ok "legacy apt certbot timer disabled to avoid duplicate renewals"
  else
    warn "legacy apt certbot timer is not disabled"
  fi
  if grep -A4 'Id=snap.certbot.renew.timer' <<<"$out" | grep -q 'ActiveState=active' && grep -A4 'Id=snap.certbot.renew.timer' <<<"$out" | grep -q 'UnitFileState=enabled'; then
    ok "snap certbot renew timer enabled"
  else
    blocked "snap certbot renew timer is not enabled and active"
  fi
else
  blocked "certbot status unavailable"
  echo "$out"
fi

cert_out="$(openssl s_client -servername sentry.wooo.work -connect sentry.wooo.work:443 </dev/null 2>/dev/null | openssl x509 -noout -subject -issuer -dates 2>/dev/null || true)"
printf '%s\n' "$cert_out"
if grep -q 'notAfter=' <<<"$cert_out"; then
  ok "sentry.wooo.work public certificate readable"
else
  warn "sentry.wooo.work public certificate not readable"
fi

section "110 backup / escrow boundary"
if out=$(ssh_cmd "$REMOTE_110" '
test -x /backup/scripts/backup-status.sh && /backup/scripts/backup-status.sh --no-notify --no-refresh
' 2>&1); then
  echo "$out"
  grep -q 'core_blockers=0' <<<"$out" && ok "backup core blockers remain zero" || blocked "backup core blockers not zero"
  grep -q 'escrow_missing=0' <<<"$out" && ok "credential escrow complete" || warn "credential escrow still incomplete; DR_COMPLETE remains blocked"
else
  warn "110 backup status unavailable"
  echo "$out"
fi

section "Public route health"
route_fail=0
for url in \
  https://mo.wooo.work/health \
  https://signoz.wooo.work/ \
  https://vibework.wooo.work/ \
  https://awoooi.wooo.work/api/v1/health
do
  code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || echo 000)"
  printf 'PUBLIC_ROUTE %s %s\n' "$code" "$url"
  case "$code" in
    2??|3??) ;;
    *) route_fail=1 ;;
  esac
done
if [ "$route_fail" -eq 0 ]; then
  ok "critical public routes respond"
else
  SERVICE_GREEN=0
  blocked "one or more critical public routes failed"
fi

section "Maintenance decision tree"
if [ "$SERVICE_GREEN" -eq 1 ] && [ "$HOST_HYGIENE_BLOCKED" -eq 0 ]; then
  cat <<'STEPS'
Current expected outcome:
  SERVICE_GREEN=1
  HOST_HYGIENE_BLOCKED=0
  RESULT=HOST_188_HYGIENE_GREEN

Allowed next step:
  1. Keep this host in the normal post-reboot summary.
  2. Wait for snap certbot timer / ACME-window readback before declaring formal certificate renewal success.
  3. Keep DR credential escrow and Wazuh registry evidence as separate blockers.

Forbidden without separate approval:
  - pg_resetwal
  - DB restore
  - Docker/systemd restart
  - firewall change
  - Wazuh active response or agent re-enroll
STEPS
else
  cat <<'STEPS'
Current expected outcome when 188 service is green but host hygiene is not:
  SERVICE_GREEN=1
  HOST_HYGIENE_BLOCKED=1
  RESULT=SERVICE_GREEN_HOST_HYGIENE_BLOCKED

Allowed next step:
  1. Collect owner disposition for host PostgreSQL 14/main:
     - retired cluster, restore from backup, or break-glass repair.
  2. Collect DNS / ACME / certificate owner evidence before certbot renew.
  3. Open an explicit maintenance window with rollback owner and post-check.

Forbidden without maintenance approval:
  - pg_resetwal
  - systemctl reset-failed
  - certbot renew
  - nginx reload
  - DB restore
  - Docker/systemd restart
  - host file write
STEPS
fi

echo
echo "SERVICE_GREEN=$SERVICE_GREEN"
echo "HOST_HYGIENE_BLOCKED=$HOST_HYGIENE_BLOCKED"
echo "RUNTIME_ACTION_AUTHORIZED=0"
echo "PASS=$PASS WARN=$WARN BLOCKED=$BLOCKED"

if [ "$SERVICE_GREEN" -eq 1 ] && [ "$HOST_HYGIENE_BLOCKED" -eq 1 ]; then
  echo "Result: SERVICE_GREEN_HOST_HYGIENE_BLOCKED. Open maintenance window; do not auto-repair."
  exit 1
fi

if [ "$BLOCKED" -gt 0 ]; then
  echo "Result: BLOCKED. Service or host evidence needs triage."
  exit 2
fi

echo "Result: HOST_188_HYGIENE_GREEN."
exit 0