227 lines
8.9 KiB
Bash
Executable File
227 lines
8.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
MODE="check"
|
|
ROLLBACK_CONF=""
|
|
|
|
EDGE_HOST="${EDGE_HOST:-192.168.0.188}"
|
|
EDGE_USER="${EDGE_USER:-ollama}"
|
|
EDGE_TARGET="${EDGE_USER}@${EDGE_HOST}"
|
|
REMOTE_CONF="${REMOTE_CONF:-/etc/nginx/sites-enabled/awoooi.wooo.work.conf}"
|
|
REMOTE_MAINT_DIR="${REMOTE_MAINT_DIR:-/var/www/maintenance}"
|
|
REMOTE_MAINT_FILE="${REMOTE_MAINT_FILE:-${REMOTE_MAINT_DIR}/maintenance.html}"
|
|
REMOTE_BACKUP_DIR="${REMOTE_BACKUP_DIR:-/home/ollama/awoooi-edge-maintenance-backups}"
|
|
LOCAL_CONF="${LOCAL_CONF:-${ROOT_DIR}/ops/nginx/awoooi.wooo.work.conf}"
|
|
LOCAL_MAINT_FILE="${LOCAL_MAINT_FILE:-${ROOT_DIR}/ops/maintenance/maintenance.html}"
|
|
REMOTE_HELPER="${REMOTE_HELPER:-/usr/local/sbin/awoooi-edge-maintenance}"
|
|
|
|
SSH_OPTS=(
|
|
-o BatchMode=yes
|
|
-o ConnectTimeout="${SSH_CONNECT_TIMEOUT_SECONDS:-8}"
|
|
-o StrictHostKeyChecking=accept-new
|
|
)
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage:
|
|
public-maintenance-edge-fallback-apply.sh --check
|
|
public-maintenance-edge-fallback-apply.sh --apply
|
|
public-maintenance-edge-fallback-apply.sh --rollback <remote-backup-conf>
|
|
public-maintenance-edge-fallback-apply.sh --rollback-latest
|
|
|
|
No secret collection is allowed. --apply uses only the root-owned fixed-action
|
|
edge helper; otherwise it exits fail-closed before writing.
|
|
USAGE
|
|
}
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--check)
|
|
MODE="check"
|
|
;;
|
|
--apply)
|
|
MODE="apply"
|
|
;;
|
|
--rollback)
|
|
MODE="rollback"
|
|
shift
|
|
ROLLBACK_CONF="${1:-}"
|
|
;;
|
|
--rollback-latest)
|
|
MODE="rollback"
|
|
ROLLBACK_CONF="latest"
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "UNKNOWN_ARG=$1" >&2
|
|
usage >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
if [ ! -f "$LOCAL_CONF" ]; then
|
|
echo "BLOCKER local_nginx_conf_missing path=$LOCAL_CONF"
|
|
exit 66
|
|
fi
|
|
if [ ! -f "$LOCAL_MAINT_FILE" ]; then
|
|
echo "BLOCKER local_maintenance_file_missing path=$LOCAL_MAINT_FILE"
|
|
exit 66
|
|
fi
|
|
|
|
remote_check() {
|
|
local expected_conf_sha expected_maint_sha
|
|
expected_conf_sha="$(sha256sum "$LOCAL_CONF" | awk '{print $1}')"
|
|
expected_maint_sha="$(sha256sum "$LOCAL_MAINT_FILE" | awk '{print $1}')"
|
|
ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \
|
|
"REMOTE_CONF='$REMOTE_CONF' REMOTE_MAINT_FILE='$REMOTE_MAINT_FILE' REMOTE_HELPER='$REMOTE_HELPER' EXPECTED_CONF_SHA='$expected_conf_sha' EXPECTED_MAINT_SHA='$expected_maint_sha' bash -s" <<'REMOTE'
|
|
set -eu
|
|
echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1"
|
|
echo "MODE=check"
|
|
echo "HOSTNAME=$(hostname 2>/dev/null || true)"
|
|
echo "WHOAMI=$(whoami 2>/dev/null || true)"
|
|
echo "REMOTE_CONF=${REMOTE_CONF}"
|
|
if [ -r "$REMOTE_CONF" ]; then
|
|
echo "REMOTE_CONF_READABLE=1"
|
|
else
|
|
echo "REMOTE_CONF_READABLE=0"
|
|
fi
|
|
if [ -r "$REMOTE_CONF" ]; then
|
|
grep -q 'proxy_intercept_errors on;' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_INTERCEPT=1" || echo "LIVE_CONF_HAS_INTERCEPT=0"
|
|
grep -q 'error_page 502 503 504 =503 /__awoooi-maintenance.html;' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_ERROR_PAGE_503=1" || echo "LIVE_CONF_HAS_ERROR_PAGE_503=0"
|
|
grep -q 'X-AWOOOI-Fallback "local-maintenance"' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_FALLBACK_HEADER=1" || echo "LIVE_CONF_HAS_FALLBACK_HEADER=0"
|
|
grep -q 'location = /__awoooi-maintenance.html' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_INTERNAL_LOCATION=1" || echo "LIVE_CONF_HAS_INTERNAL_LOCATION=0"
|
|
grep -q '192.168.0.125:32334' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_API_VIP=1" || echo "LIVE_CONF_HAS_API_VIP=0"
|
|
grep -q '192.168.0.125:32335' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_WEB_VIP=1" || echo "LIVE_CONF_HAS_WEB_VIP=0"
|
|
else
|
|
echo "LIVE_CONF_HAS_INTERCEPT=0"
|
|
echo "LIVE_CONF_HAS_ERROR_PAGE_503=0"
|
|
echo "LIVE_CONF_HAS_FALLBACK_HEADER=0"
|
|
echo "LIVE_CONF_HAS_INTERNAL_LOCATION=0"
|
|
echo "LIVE_CONF_HAS_API_VIP=0"
|
|
echo "LIVE_CONF_HAS_WEB_VIP=0"
|
|
fi
|
|
if [ -r "$REMOTE_MAINT_FILE" ]; then
|
|
echo "MAINTENANCE_FILE_PRESENT=1"
|
|
grep -q 'AWOOOI 維護模式' "$REMOTE_MAINT_FILE" && echo "MAINTENANCE_FILE_HAS_MARKER=1" || echo "MAINTENANCE_FILE_HAS_MARKER=0"
|
|
else
|
|
echo "MAINTENANCE_FILE_PRESENT=0"
|
|
echo "MAINTENANCE_FILE_HAS_MARKER=0"
|
|
fi
|
|
PRIVILEGED_EXECUTOR_READY=0
|
|
EDGE_EXECUTOR_PAYLOAD_PARITY=0
|
|
CANONICAL_CONF_SHA256=""
|
|
CANONICAL_MAINT_SHA256=""
|
|
helper_status="$(sudo -n "$REMOTE_HELPER" check 2>/dev/null || true)"
|
|
if printf '%s\n' "$helper_status" | grep -q '^EDGE_EXECUTOR_READY=1$'; then
|
|
PRIVILEGED_EXECUTOR_READY=1
|
|
CANONICAL_CONF_SHA256="$(printf '%s\n' "$helper_status" | awk -F= '$1 == "CANONICAL_CONF_SHA256" {print $2; exit}')"
|
|
CANONICAL_MAINT_SHA256="$(printf '%s\n' "$helper_status" | awk -F= '$1 == "CANONICAL_MAINT_SHA256" {print $2; exit}')"
|
|
if [ "$CANONICAL_CONF_SHA256" = "$EXPECTED_CONF_SHA" ] \
|
|
&& [ "$CANONICAL_MAINT_SHA256" = "$EXPECTED_MAINT_SHA" ]; then
|
|
EDGE_EXECUTOR_PAYLOAD_PARITY=1
|
|
fi
|
|
fi
|
|
echo "PRIVILEGED_EXECUTOR_READY=${PRIVILEGED_EXECUTOR_READY}"
|
|
echo "SUDO_N_READY=${PRIVILEGED_EXECUTOR_READY}"
|
|
echo "EDGE_EXECUTOR_PAYLOAD_PARITY=${EDGE_EXECUTOR_PAYLOAD_PARITY}"
|
|
echo "CANONICAL_CONF_SHA256=${CANONICAL_CONF_SHA256}"
|
|
echo "CANONICAL_MAINT_SHA256=${CANONICAL_MAINT_SHA256}"
|
|
REMOTE_CONF_READABLE=$(if [ -r "$REMOTE_CONF" ]; then echo 1; else echo 0; fi)
|
|
LIVE_CONF_HAS_INTERCEPT=$(if [ -r "$REMOTE_CONF" ] && grep -q 'proxy_intercept_errors on;' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
|
|
LIVE_CONF_HAS_ERROR_PAGE_503=$(if [ -r "$REMOTE_CONF" ] && grep -q 'error_page 502 503 504 =503 /__awoooi-maintenance.html;' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
|
|
LIVE_CONF_HAS_FALLBACK_HEADER=$(if [ -r "$REMOTE_CONF" ] && grep -q 'X-AWOOOI-Fallback "local-maintenance"' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
|
|
LIVE_CONF_HAS_INTERNAL_LOCATION=$(if [ -r "$REMOTE_CONF" ] && grep -q 'location = /__awoooi-maintenance.html' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
|
|
LIVE_CONF_HAS_API_VIP=$(if [ -r "$REMOTE_CONF" ] && grep -q '192.168.0.125:32334' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
|
|
LIVE_CONF_HAS_WEB_VIP=$(if [ -r "$REMOTE_CONF" ] && grep -q '192.168.0.125:32335' "$REMOTE_CONF"; then echo 1; else echo 0; fi)
|
|
MAINTENANCE_FILE_PRESENT=$(if [ -r "$REMOTE_MAINT_FILE" ]; then echo 1; else echo 0; fi)
|
|
MAINTENANCE_FILE_HAS_MARKER=$(if [ -r "$REMOTE_MAINT_FILE" ] && grep -q 'AWOOOI 維護模式' "$REMOTE_MAINT_FILE"; then echo 1; else echo 0; fi)
|
|
if [ "$REMOTE_CONF_READABLE" = 1 ] \
|
|
&& [ "$LIVE_CONF_HAS_INTERCEPT" = 1 ] \
|
|
&& [ "$LIVE_CONF_HAS_ERROR_PAGE_503" = 1 ] \
|
|
&& [ "$LIVE_CONF_HAS_FALLBACK_HEADER" = 1 ] \
|
|
&& [ "$LIVE_CONF_HAS_INTERNAL_LOCATION" = 1 ] \
|
|
&& [ "$LIVE_CONF_HAS_API_VIP" = 1 ] \
|
|
&& [ "$LIVE_CONF_HAS_WEB_VIP" = 1 ] \
|
|
&& [ "$MAINTENANCE_FILE_PRESENT" = 1 ] \
|
|
&& [ "$MAINTENANCE_FILE_HAS_MARKER" = 1 ]; then
|
|
echo "EDGE_FALLBACK_READY=1"
|
|
else
|
|
echo "EDGE_FALLBACK_READY=0"
|
|
echo "BLOCKER edge_public_maintenance_live_config_drift"
|
|
if [ "$PRIVILEGED_EXECUTOR_READY" = 0 ]; then
|
|
echo "BLOCKER privileged_edge_apply_channel_unavailable"
|
|
elif [ "$EDGE_EXECUTOR_PAYLOAD_PARITY" = 0 ]; then
|
|
echo "BLOCKER edge_executor_payload_drift"
|
|
fi
|
|
fi
|
|
REMOTE
|
|
}
|
|
|
|
require_executor_payload_parity() {
|
|
local expected_conf_sha expected_maint_sha helper_status remote_conf_sha remote_maint_sha
|
|
expected_conf_sha="$(sha256sum "$LOCAL_CONF" | awk '{print $1}')"
|
|
expected_maint_sha="$(sha256sum "$LOCAL_MAINT_FILE" | awk '{print $1}')"
|
|
if ! helper_status="$(ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" "sudo -n '$REMOTE_HELPER' check")"; then
|
|
echo "BLOCKER privileged_edge_apply_channel_unavailable"
|
|
return 75
|
|
fi
|
|
remote_conf_sha="$(printf '%s\n' "$helper_status" | awk -F= '$1 == "CANONICAL_CONF_SHA256" {print $2; exit}')"
|
|
remote_maint_sha="$(printf '%s\n' "$helper_status" | awk -F= '$1 == "CANONICAL_MAINT_SHA256" {print $2; exit}')"
|
|
if [ "$remote_conf_sha" != "$expected_conf_sha" ] \
|
|
|| [ "$remote_maint_sha" != "$expected_maint_sha" ]; then
|
|
echo "BLOCKER edge_executor_payload_drift"
|
|
return 78
|
|
fi
|
|
}
|
|
|
|
remote_apply() {
|
|
require_executor_payload_parity
|
|
echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1"
|
|
echo "MODE=apply"
|
|
ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" "sudo -n '$REMOTE_HELPER' apply"
|
|
remote_check
|
|
}
|
|
|
|
remote_rollback() {
|
|
if [ -z "$ROLLBACK_CONF" ]; then
|
|
echo "BLOCKER rollback_conf_required"
|
|
exit 64
|
|
fi
|
|
require_executor_payload_parity
|
|
local helper_status last_backup
|
|
helper_status="$(ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" "sudo -n '$REMOTE_HELPER' check")"
|
|
last_backup="$(printf '%s\n' "$helper_status" | awk -F= '$1 == "LAST_BACKUP_DIR" {print $2; exit}')"
|
|
if [ "$ROLLBACK_CONF" != "latest" ] \
|
|
&& [ "$ROLLBACK_CONF" != "$last_backup" ] \
|
|
&& [ "$ROLLBACK_CONF" != "$last_backup/live.conf" ]; then
|
|
echo "BLOCKER rollback_conf_not_latest"
|
|
exit 66
|
|
fi
|
|
echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1"
|
|
echo "MODE=rollback"
|
|
ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" "sudo -n '$REMOTE_HELPER' rollback-latest"
|
|
remote_check
|
|
}
|
|
|
|
case "$MODE" in
|
|
check)
|
|
remote_check
|
|
;;
|
|
apply)
|
|
remote_apply
|
|
;;
|
|
rollback)
|
|
remote_rollback
|
|
;;
|
|
*)
|
|
echo "UNKNOWN_MODE=$MODE" >&2
|
|
exit 64
|
|
;;
|
|
esac
|