awoooi/scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
MODE="check"
ROLLBACK_CONF=""

EDGE_HOST="${EDGE_HOST:-192.168.0.188}"
EDGE_USER="${EDGE_USER:-ollama}"
EDGE_TARGET="${EDGE_USER}@${EDGE_HOST}"
REMOTE_CONF="${REMOTE_CONF:-/etc/nginx/sites-enabled/awoooi.wooo.work.conf}"
REMOTE_MAINT_DIR="${REMOTE_MAINT_DIR:-/var/www/maintenance}"
REMOTE_MAINT_FILE="${REMOTE_MAINT_FILE:-${REMOTE_MAINT_DIR}/maintenance.html}"
REMOTE_BACKUP_DIR="${REMOTE_BACKUP_DIR:-/home/ollama/awoooi-edge-maintenance-backups}"
LOCAL_CONF="${LOCAL_CONF:-${ROOT_DIR}/ops/nginx/awoooi.wooo.work.conf}"
LOCAL_MAINT_FILE="${LOCAL_MAINT_FILE:-${ROOT_DIR}/ops/maintenance/maintenance.html}"

SSH_OPTS=(
  -o BatchMode=yes
  -o ConnectTimeout="${SSH_CONNECT_TIMEOUT_SECONDS:-8}"
  -o StrictHostKeyChecking=accept-new
)

usage() {
  cat <<'USAGE'
Usage:
  public-maintenance-edge-fallback-apply.sh --check
  public-maintenance-edge-fallback-apply.sh --apply
  public-maintenance-edge-fallback-apply.sh --rollback <remote-backup-conf>

No secret collection is allowed. --apply requires an existing passwordless sudo
channel on the edge host; otherwise it exits fail-closed before writing.
USAGE
}

while [ "$#" -gt 0 ]; do
  case "$1" in
    --check)
      MODE="check"
      ;;
    --apply)
      MODE="apply"
      ;;
    --rollback)
      MODE="rollback"
      shift
      ROLLBACK_CONF="${1:-}"
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "UNKNOWN_ARG=$1" >&2
      usage >&2
      exit 64
      ;;
  esac
  shift
done

if [ ! -f "$LOCAL_CONF" ]; then
  echo "BLOCKER local_nginx_conf_missing path=$LOCAL_CONF"
  exit 66
fi
if [ ! -f "$LOCAL_MAINT_FILE" ]; then
  echo "BLOCKER local_maintenance_file_missing path=$LOCAL_MAINT_FILE"
  exit 66
fi

remote_check() {
  ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \
    "REMOTE_CONF='$REMOTE_CONF' REMOTE_MAINT_FILE='$REMOTE_MAINT_FILE' bash -s" <<'REMOTE'
set -eu
echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1"
echo "MODE=check"
echo "HOSTNAME=$(hostname 2>/dev/null || true)"
echo "WHOAMI=$(whoami 2>/dev/null || true)"
echo "REMOTE_CONF=${REMOTE_CONF}"
if [ -r "$REMOTE_CONF" ]; then
  echo "REMOTE_CONF_READABLE=1"
else
  echo "REMOTE_CONF_READABLE=0"
fi
if [ -r "$REMOTE_CONF" ]; then
  grep -q 'proxy_intercept_errors on;' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_INTERCEPT=1" || echo "LIVE_CONF_HAS_INTERCEPT=0"
  grep -q 'error_page 502 503 504 =503 /__awoooi-maintenance.html;' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_ERROR_PAGE_503=1" || echo "LIVE_CONF_HAS_ERROR_PAGE_503=0"
  grep -q 'X-AWOOOI-Fallback "local-maintenance"' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_FALLBACK_HEADER=1" || echo "LIVE_CONF_HAS_FALLBACK_HEADER=0"
  grep -q 'location = /__awoooi-maintenance.html' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_INTERNAL_LOCATION=1" || echo "LIVE_CONF_HAS_INTERNAL_LOCATION=0"
  grep -q '192.168.0.125:32334' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_API_VIP=1" || echo "LIVE_CONF_HAS_API_VIP=0"
  grep -q '192.168.0.125:32335' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_WEB_VIP=1" || echo "LIVE_CONF_HAS_WEB_VIP=0"
else
  echo "LIVE_CONF_HAS_INTERCEPT=0"
  echo "LIVE_CONF_HAS_ERROR_PAGE_503=0"
  echo "LIVE_CONF_HAS_FALLBACK_HEADER=0"
  echo "LIVE_CONF_HAS_INTERNAL_LOCATION=0"
  echo "LIVE_CONF_HAS_API_VIP=0"
  echo "LIVE_CONF_HAS_WEB_VIP=0"
fi
if [ -r "$REMOTE_MAINT_FILE" ]; then
  echo "MAINTENANCE_FILE_PRESENT=1"
  grep -q 'AWOOOI 維護模式' "$REMOTE_MAINT_FILE" && echo "MAINTENANCE_FILE_HAS_MARKER=1" || echo "MAINTENANCE_FILE_HAS_MARKER=0"
else
  echo "MAINTENANCE_FILE_PRESENT=0"
  echo "MAINTENANCE_FILE_HAS_MARKER=0"
fi
SUDO_N_READY=$(if sudo -n true >/dev/null 2>&1; then echo 1; else echo 0; fi)
echo "SUDO_N_READY=${SUDO_N_READY}"
REMOTE_CONF_READABLE=$(if [ -r "$REMOTE_CONF" ]; then echo 1; else echo 0; fi)
LIVE_CONF_HAS_INTERCEPT=$(if [ -r "$REMOTE_CONF" ] && grep -q 'proxy_intercept_errors on;' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
LIVE_CONF_HAS_ERROR_PAGE_503=$(if [ -r "$REMOTE_CONF" ] && grep -q 'error_page 502 503 504 =503 /__awoooi-maintenance.html;' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
LIVE_CONF_HAS_FALLBACK_HEADER=$(if [ -r "$REMOTE_CONF" ] && grep -q 'X-AWOOOI-Fallback "local-maintenance"' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
LIVE_CONF_HAS_INTERNAL_LOCATION=$(if [ -r "$REMOTE_CONF" ] && grep -q 'location = /__awoooi-maintenance.html' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
LIVE_CONF_HAS_API_VIP=$(if [ -r "$REMOTE_CONF" ] && grep -q '192.168.0.125:32334' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
LIVE_CONF_HAS_WEB_VIP=$(if [ -r "$REMOTE_CONF" ] && grep -q '192.168.0.125:32335' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
MAINTENANCE_FILE_PRESENT=$(if [ -r "$REMOTE_MAINT_FILE" ]; then echo 1; else echo 0; fi)
MAINTENANCE_FILE_HAS_MARKER=$(if [ -r "$REMOTE_MAINT_FILE" ] && grep -q 'AWOOOI 維護模式' "$REMOTE_MAINT_FILE"; then echo 1; else echo 0; fi)
if [ "$REMOTE_CONF_READABLE" = 1 ] \
  && [ "$LIVE_CONF_HAS_INTERCEPT" = 1 ] \
  && [ "$LIVE_CONF_HAS_ERROR_PAGE_503" = 1 ] \
  && [ "$LIVE_CONF_HAS_FALLBACK_HEADER" = 1 ] \
  && [ "$LIVE_CONF_HAS_INTERNAL_LOCATION" = 1 ] \
  && [ "$LIVE_CONF_HAS_API_VIP" = 1 ] \
  && [ "$LIVE_CONF_HAS_WEB_VIP" = 1 ] \
  && [ "$MAINTENANCE_FILE_PRESENT" = 1 ] \
  && [ "$MAINTENANCE_FILE_HAS_MARKER" = 1 ]; then
  echo "EDGE_FALLBACK_READY=1"
else
  echo "EDGE_FALLBACK_READY=0"
  echo "BLOCKER edge_public_maintenance_live_config_drift"
  if [ "$SUDO_N_READY" = 0 ]; then
    echo "BLOCKER privileged_edge_apply_channel_unavailable"
  fi
fi
REMOTE
}

remote_apply() {
  local tmp_conf="/tmp/awoooi.wooo.work.conf.$$"
  local tmp_maint="/tmp/awoooi-maintenance.$$"
  scp -q "${SSH_OPTS[@]}" "$LOCAL_CONF" "${EDGE_TARGET}:${tmp_conf}"
  scp -q "${SSH_OPTS[@]}" "$LOCAL_MAINT_FILE" "${EDGE_TARGET}:${tmp_maint}"
  ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \
    "REMOTE_CONF='$REMOTE_CONF' REMOTE_MAINT_DIR='$REMOTE_MAINT_DIR' REMOTE_MAINT_FILE='$REMOTE_MAINT_FILE' REMOTE_BACKUP_DIR='$REMOTE_BACKUP_DIR' TMP_CONF='$tmp_conf' TMP_MAINT='$tmp_maint' bash -s" <<'REMOTE'
set -eu
echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1"
echo "MODE=apply"
if ! sudo -n true >/dev/null 2>&1; then
  echo "SUDO_N_READY=0"
  echo "BLOCKER privileged_edge_apply_channel_unavailable"
  rm -f "$TMP_CONF" "$TMP_MAINT"
  exit 75
fi
echo "SUDO_N_READY=1"
mkdir -p "$REMOTE_BACKUP_DIR"
backup="${REMOTE_BACKUP_DIR}/awoooi.wooo.work.conf.before-maintenance.$(date +%Y%m%d%H%M%S)"
if [ -f "$REMOTE_CONF" ]; then
  sudo cp "$REMOTE_CONF" "$backup"
else
  : >"${backup}.missing"
fi
sudo mkdir -p "$REMOTE_MAINT_DIR"
sudo install -m 0644 "$TMP_CONF" "$REMOTE_CONF"
sudo install -m 0644 "$TMP_MAINT" "$REMOTE_MAINT_FILE"
rm -f "$TMP_CONF" "$TMP_MAINT"
if ! sudo nginx -t; then
  echo "BLOCKER nginx_test_failed_restoring_backup"
  if [ -f "$backup" ]; then
    sudo cp "$backup" "$REMOTE_CONF"
    sudo nginx -t || true
  fi
  exit 76
fi
sudo systemctl reload nginx
echo "BACKUP_CONF=$backup"
echo "NGINX_RELOAD_DONE=1"
REMOTE
  remote_check
}

remote_rollback() {
  if [ -z "$ROLLBACK_CONF" ]; then
    echo "BLOCKER rollback_conf_required"
    exit 64
  fi
  ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \
    "REMOTE_CONF='$REMOTE_CONF' ROLLBACK_CONF='$ROLLBACK_CONF' bash -s" <<'REMOTE'
set -eu
echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1"
echo "MODE=rollback"
if ! sudo -n true >/dev/null 2>&1; then
  echo "SUDO_N_READY=0"
  echo "BLOCKER privileged_edge_apply_channel_unavailable"
  exit 75
fi
if [ ! -f "$ROLLBACK_CONF" ]; then
  echo "BLOCKER rollback_conf_not_found path=$ROLLBACK_CONF"
  exit 66
fi
sudo cp "$ROLLBACK_CONF" "$REMOTE_CONF"
sudo nginx -t
sudo systemctl reload nginx
echo "ROLLBACK_DONE=1"
REMOTE
  remote_check
}

case "$MODE" in
  check)
    remote_check
    ;;
  apply)
    remote_apply
    ;;
  rollback)
    remote_rollback
    ;;
  *)
    echo "UNKNOWN_MODE=$MODE" >&2
    exit 64
    ;;
esac