fix(reboot): add edge maintenance fallback apply gate

scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh

@@ -0,0 +1,222 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+MODE="check"
+ROLLBACK_CONF=""
+
+EDGE_HOST="${EDGE_HOST:-192.168.0.188}"
+EDGE_USER="${EDGE_USER:-ollama}"
+EDGE_TARGET="${EDGE_USER}@${EDGE_HOST}"
+REMOTE_CONF="${REMOTE_CONF:-/etc/nginx/sites-enabled/awoooi.wooo.work.conf}"
+REMOTE_MAINT_DIR="${REMOTE_MAINT_DIR:-/var/www/maintenance}"
+REMOTE_MAINT_FILE="${REMOTE_MAINT_FILE:-${REMOTE_MAINT_DIR}/maintenance.html}"
+REMOTE_BACKUP_DIR="${REMOTE_BACKUP_DIR:-/home/ollama/awoooi-edge-maintenance-backups}"
+LOCAL_CONF="${LOCAL_CONF:-${ROOT_DIR}/ops/nginx/awoooi.wooo.work.conf}"
+LOCAL_MAINT_FILE="${LOCAL_MAINT_FILE:-${ROOT_DIR}/ops/maintenance/maintenance.html}"
+
+SSH_OPTS=(
+  -o BatchMode=yes
+  -o ConnectTimeout="${SSH_CONNECT_TIMEOUT_SECONDS:-8}"
+  -o StrictHostKeyChecking=accept-new
+)
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  public-maintenance-edge-fallback-apply.sh --check
+  public-maintenance-edge-fallback-apply.sh --apply
+  public-maintenance-edge-fallback-apply.sh --rollback <remote-backup-conf>
+
+No secret collection is allowed. --apply requires an existing passwordless sudo
+channel on the edge host; otherwise it exits fail-closed before writing.
+USAGE
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --check)
+      MODE="check"
+      ;;
+    --apply)
+      MODE="apply"
+      ;;
+    --rollback)
+      MODE="rollback"
+      shift
+      ROLLBACK_CONF="${1:-}"
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "UNKNOWN_ARG=$1" >&2
+      usage >&2
+      exit 64
+      ;;
+  esac
+  shift
+done
+
+if [ ! -f "$LOCAL_CONF" ]; then
+  echo "BLOCKER local_nginx_conf_missing path=$LOCAL_CONF"
+  exit 66
+fi
+if [ ! -f "$LOCAL_MAINT_FILE" ]; then
+  echo "BLOCKER local_maintenance_file_missing path=$LOCAL_MAINT_FILE"
+  exit 66
+fi
+
+remote_check() {
+  ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \
+    "REMOTE_CONF='$REMOTE_CONF' REMOTE_MAINT_FILE='$REMOTE_MAINT_FILE' bash -s" <<'REMOTE'
+set -eu
+echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1"
+echo "MODE=check"
+echo "HOSTNAME=$(hostname 2>/dev/null || true)"
+echo "WHOAMI=$(whoami 2>/dev/null || true)"
+echo "REMOTE_CONF=${REMOTE_CONF}"
+if [ -r "$REMOTE_CONF" ]; then
+  echo "REMOTE_CONF_READABLE=1"
+else
+  echo "REMOTE_CONF_READABLE=0"
+fi
+if [ -r "$REMOTE_CONF" ]; then
+  grep -q 'proxy_intercept_errors on;' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_INTERCEPT=1" || echo "LIVE_CONF_HAS_INTERCEPT=0"
+  grep -q 'error_page 502 503 504 =503 /__awoooi-maintenance.html;' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_ERROR_PAGE_503=1" || echo "LIVE_CONF_HAS_ERROR_PAGE_503=0"
+  grep -q 'X-AWOOOI-Fallback "local-maintenance"' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_FALLBACK_HEADER=1" || echo "LIVE_CONF_HAS_FALLBACK_HEADER=0"
+  grep -q 'location = /__awoooi-maintenance.html' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_INTERNAL_LOCATION=1" || echo "LIVE_CONF_HAS_INTERNAL_LOCATION=0"
+  grep -q '192.168.0.125:32334' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_API_VIP=1" || echo "LIVE_CONF_HAS_API_VIP=0"
+  grep -q '192.168.0.125:32335' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_WEB_VIP=1" || echo "LIVE_CONF_HAS_WEB_VIP=0"
+else
+  echo "LIVE_CONF_HAS_INTERCEPT=0"
+  echo "LIVE_CONF_HAS_ERROR_PAGE_503=0"
+  echo "LIVE_CONF_HAS_FALLBACK_HEADER=0"
+  echo "LIVE_CONF_HAS_INTERNAL_LOCATION=0"
+  echo "LIVE_CONF_HAS_API_VIP=0"
+  echo "LIVE_CONF_HAS_WEB_VIP=0"
+fi
+if [ -r "$REMOTE_MAINT_FILE" ]; then
+  echo "MAINTENANCE_FILE_PRESENT=1"
+  grep -q 'AWOOOI 維護模式' "$REMOTE_MAINT_FILE" && echo "MAINTENANCE_FILE_HAS_MARKER=1" || echo "MAINTENANCE_FILE_HAS_MARKER=0"
+else
+  echo "MAINTENANCE_FILE_PRESENT=0"
+  echo "MAINTENANCE_FILE_HAS_MARKER=0"
+fi
+if sudo -n true >/dev/null 2>&1; then
+  echo "SUDO_N_READY=1"
+else
+  echo "SUDO_N_READY=0"
+fi
+REMOTE_CONF_READABLE=$(if [ -r "$REMOTE_CONF" ]; then echo 1; else echo 0; fi)
+LIVE_CONF_HAS_INTERCEPT=$(if [ -r "$REMOTE_CONF" ] && grep -q 'proxy_intercept_errors on;' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
+LIVE_CONF_HAS_ERROR_PAGE_503=$(if [ -r "$REMOTE_CONF" ] && grep -q 'error_page 502 503 504 =503 /__awoooi-maintenance.html;' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
+LIVE_CONF_HAS_FALLBACK_HEADER=$(if [ -r "$REMOTE_CONF" ] && grep -q 'X-AWOOOI-Fallback "local-maintenance"' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
+LIVE_CONF_HAS_INTERNAL_LOCATION=$(if [ -r "$REMOTE_CONF" ] && grep -q 'location = /__awoooi-maintenance.html' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
+LIVE_CONF_HAS_API_VIP=$(if [ -r "$REMOTE_CONF" ] && grep -q '192.168.0.125:32334' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
+LIVE_CONF_HAS_WEB_VIP=$(if [ -r "$REMOTE_CONF" ] && grep -q '192.168.0.125:32335' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
+MAINTENANCE_FILE_PRESENT=$(if [ -r "$REMOTE_MAINT_FILE" ]; then echo 1; else echo 0; fi)
+MAINTENANCE_FILE_HAS_MARKER=$(if [ -r "$REMOTE_MAINT_FILE" ] && grep -q 'AWOOOI 維護模式' "$REMOTE_MAINT_FILE"; then echo 1; else echo 0; fi)
+if [ "$REMOTE_CONF_READABLE" = 1 ] \
+  && [ "$LIVE_CONF_HAS_INTERCEPT" = 1 ] \
+  && [ "$LIVE_CONF_HAS_ERROR_PAGE_503" = 1 ] \
+  && [ "$LIVE_CONF_HAS_FALLBACK_HEADER" = 1 ] \
+  && [ "$LIVE_CONF_HAS_INTERNAL_LOCATION" = 1 ] \
+  && [ "$LIVE_CONF_HAS_API_VIP" = 1 ] \
+  && [ "$LIVE_CONF_HAS_WEB_VIP" = 1 ] \
+  && [ "$MAINTENANCE_FILE_PRESENT" = 1 ] \
+  && [ "$MAINTENANCE_FILE_HAS_MARKER" = 1 ]; then
+  echo "EDGE_FALLBACK_READY=1"
+else
+  echo "EDGE_FALLBACK_READY=0"
+  echo "BLOCKER edge_public_maintenance_live_config_drift"
+fi
+REMOTE
+}
+
+remote_apply() {
+  local tmp_conf="/tmp/awoooi.wooo.work.conf.$$"
+  local tmp_maint="/tmp/awoooi-maintenance.$$"
+  scp -q "${SSH_OPTS[@]}" "$LOCAL_CONF" "${EDGE_TARGET}:${tmp_conf}"
+  scp -q "${SSH_OPTS[@]}" "$LOCAL_MAINT_FILE" "${EDGE_TARGET}:${tmp_maint}"
+  ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \
+    "REMOTE_CONF='$REMOTE_CONF' REMOTE_MAINT_DIR='$REMOTE_MAINT_DIR' REMOTE_MAINT_FILE='$REMOTE_MAINT_FILE' REMOTE_BACKUP_DIR='$REMOTE_BACKUP_DIR' TMP_CONF='$tmp_conf' TMP_MAINT='$tmp_maint' bash -s" <<'REMOTE'
+set -eu
+echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1"
+echo "MODE=apply"
+if ! sudo -n true >/dev/null 2>&1; then
+  echo "SUDO_N_READY=0"
+  echo "BLOCKER privileged_edge_apply_channel_unavailable"
+  rm -f "$TMP_CONF" "$TMP_MAINT"
+  exit 75
+fi
+echo "SUDO_N_READY=1"
+mkdir -p "$REMOTE_BACKUP_DIR"
+backup="${REMOTE_BACKUP_DIR}/awoooi.wooo.work.conf.before-maintenance.$(date +%Y%m%d%H%M%S)"
+if [ -f "$REMOTE_CONF" ]; then
+  sudo cp "$REMOTE_CONF" "$backup"
+else
+  : >"${backup}.missing"
+fi
+sudo mkdir -p "$REMOTE_MAINT_DIR"
+sudo install -m 0644 "$TMP_CONF" "$REMOTE_CONF"
+sudo install -m 0644 "$TMP_MAINT" "$REMOTE_MAINT_FILE"
+rm -f "$TMP_CONF" "$TMP_MAINT"
+if ! sudo nginx -t; then
+  echo "BLOCKER nginx_test_failed_restoring_backup"
+  if [ -f "$backup" ]; then
+    sudo cp "$backup" "$REMOTE_CONF"
+    sudo nginx -t || true
+  fi
+  exit 76
+fi
+sudo systemctl reload nginx
+echo "BACKUP_CONF=$backup"
+echo "NGINX_RELOAD_DONE=1"
+REMOTE
+  remote_check
+}
+
+remote_rollback() {
+  if [ -z "$ROLLBACK_CONF" ]; then
+    echo "BLOCKER rollback_conf_required"
+    exit 64
+  fi
+  ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \
+    "REMOTE_CONF='$REMOTE_CONF' ROLLBACK_CONF='$ROLLBACK_CONF' bash -s" <<'REMOTE'
+set -eu
+echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1"
+echo "MODE=rollback"
+if ! sudo -n true >/dev/null 2>&1; then
+  echo "SUDO_N_READY=0"
+  echo "BLOCKER privileged_edge_apply_channel_unavailable"
+  exit 75
+fi
+if [ ! -f "$ROLLBACK_CONF" ]; then
+  echo "BLOCKER rollback_conf_not_found path=$ROLLBACK_CONF"
+  exit 66
+fi
+sudo cp "$ROLLBACK_CONF" "$REMOTE_CONF"
+sudo nginx -t
+sudo systemctl reload nginx
+echo "ROLLBACK_DONE=1"
+REMOTE
+  remote_check
+}
+
+case "$MODE" in
+  check)
+    remote_check
+    ;;
+  apply)
+    remote_apply
+    ;;
+  rollback)
+    remote_rollback
+    ;;
+  *)
+    echo "UNKNOWN_MODE=$MODE" >&2
+    exit 64
+    ;;
+esac

scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py

@@ -144,6 +144,8 @@ def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> Non
         assert "/50x.html" not in conf
 
     ops_conf = read("ops/nginx/awoooi.wooo.work.conf")
+    assert "192.168.0.125:32334" in ops_conf
+    assert "192.168.0.125:32335" in ops_conf
     for marker in [
         "location /api/ {",
         "location ~ ^/api/v1/(dashboard/stream|agent/thinking) {",
@@ -191,6 +193,24 @@ def test_public_maintenance_probe_does_not_count_fallback_5xx_as_raw(
     assert route["raw_5xx_without_fallback"] is False
 
 
+def test_public_maintenance_edge_apply_script_is_guarded_and_reversible() -> None:
+    script = read("scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh")
+
+    assert "MODE=\"check\"" in script
+    assert "MODE=\"apply\"" in script
+    assert "MODE=\"rollback\"" in script
+    assert "sudo -n true" in script
+    assert "BLOCKER privileged_edge_apply_channel_unavailable" in script
+    assert "nginx -t" in script
+    assert "systemctl reload nginx" in script
+    assert "BACKUP_CONF=" in script
+    assert "EDGE_FALLBACK_READY=1" in script
+    assert "EDGE_FALLBACK_READY=0" in script
+    assert "raw_secret" not in script.lower()
+    assert "docker restart" not in script
+    assert "reboot " not in script
+
+
 def test_reboot_slo_scorecard_fails_closed_on_raw_public_502(tmp_path: Path) -> None:
     summary = tmp_path / "summary.txt"
     summary.write_text(