awoooi/scripts/reboot-recovery/install-full-host-reboot-orchestrator.sh

#!/usr/bin/env bash
set -Eeuo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
UNIT_NAME="awoooi-full-host-reboot-orchestrator"
SYSTEMD_DIR="${SYSTEMD_DIR:-/etc/systemd/system}"
ENV_DIR="${ENV_DIR:-/etc/awoooi}"
LOG_DIR="${LOG_DIR:-/var/log/awoooi/reboot-recovery}"
INSTALL_ROOT="${INSTALL_ROOT:-/opt/awoooi/reboot-recovery}"
MODE="dry-run"
ENABLE_NOW=0

usage() {
  cat <<'USAGE'
Usage: install-full-host-reboot-orchestrator.sh [--dry-run|--apply|--verify-only|--rollback] [--enable-now]

Installs the AWOOOI full-host reboot recovery orchestrator source and systemd
units. The default is --dry-run. Runtime scheduling requires both --apply and
--enable-now so a source deploy cannot accidentally start host automation.

Options:
  --dry-run       Print the install plan only. This is the default.
  --apply         Stage source files and systemd units, then daemon-reload.
  --verify-only   Read back systemd timer/service/source status without writing.
  --rollback      Disable and remove only this orchestrator unit/timer.
  --enable-now    With --apply, enable and start only the orchestrator timer.
  -h, --help      Show this help.

Forbidden here: reboot, node drain, firewall changes, DB restore/drop/truncate,
Docker prune, secret/session/.env/raw-volume reads, repo/ref destructive ops.
USAGE
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --dry-run)
      MODE="dry-run"
      ;;
    --apply)
      MODE="apply"
      ;;
    --verify-only)
      MODE="verify"
      ;;
    --rollback)
      MODE="rollback"
      ;;
    --enable-now)
      ENABLE_NOW=1
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "Unknown argument: $1" >&2
      usage >&2
      exit 2
      ;;
  esac
  shift
done

cat <<PLAN
AWOOOI_FULL_HOST_REBOOT_ORCHESTRATOR_INSTALL_PLAN=1
mode=${MODE}
enable_now=${ENABLE_NOW}
install_root=${INSTALL_ROOT}
systemd_dir=${SYSTEMD_DIR}
env_dir=${ENV_DIR}
log_dir=${LOG_DIR}
source_root=${ROOT_DIR}
unit=${UNIT_NAME}
apply_command=bash scripts/reboot-recovery/install-full-host-reboot-orchestrator.sh --apply
enable_command=bash scripts/reboot-recovery/install-full-host-reboot-orchestrator.sh --apply --enable-now
verify_command=bash scripts/reboot-recovery/install-full-host-reboot-orchestrator.sh --verify-only
rollback_command=bash scripts/reboot-recovery/install-full-host-reboot-orchestrator.sh --rollback
PLAN

verify_status() {
  cat <<STATUS
FULL_HOST_ORCHESTRATOR_TIMER_ENABLED=$(systemctl is-enabled "${UNIT_NAME}.timer" 2>/dev/null || true)
FULL_HOST_ORCHESTRATOR_TIMER_ACTIVE=$(systemctl is-active "${UNIT_NAME}.timer" 2>/dev/null || true)
FULL_HOST_ORCHESTRATOR_SERVICE_LOAD=$(systemctl show "${UNIT_NAME}.service" -p LoadState --value 2>/dev/null || true)
FULL_HOST_ORCHESTRATOR_SERVICE_RESULT=$(systemctl show "${UNIT_NAME}.service" -p Result --value 2>/dev/null || true)
FULL_HOST_ORCHESTRATOR_SERVICE_LAST_STATUS=$(systemctl show "${UNIT_NAME}.service" -p ExecMainStatus --value 2>/dev/null || true)
FULL_HOST_ORCHESTRATOR_INSTALL_ROOT_PRESENT=$([ -x "${INSTALL_ROOT}/scripts/reboot-recovery/full-host-reboot-orchestrator.sh" ] && echo 1 || echo 0)
FULL_HOST_ORCHESTRATOR_AI_TRIAGE_PRESENT=$([ -x "${INSTALL_ROOT}/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh" ] && echo 1 || echo 0)
FULL_HOST_ORCHESTRATOR_LATEST_ARTIFACT_DIR=$(find "$LOG_DIR" -maxdepth 1 -type d -name '20*' 2>/dev/null | sort | tail -n 1 || true)
STATUS
}

if [[ "$MODE" != "apply" ]]; then
  case "$MODE" in
    dry-run)
      echo "DRY_RUN=1"
      exit 0
      ;;
    verify)
      verify_status
      exit 0
      ;;
    rollback)
      systemctl disable --now "${UNIT_NAME}.timer" >/dev/null 2>&1 || true
      systemctl stop "${UNIT_NAME}.service" >/dev/null 2>&1 || true
      rm -f "${SYSTEMD_DIR}/${UNIT_NAME}.timer" "${SYSTEMD_DIR}/${UNIT_NAME}.service"
      systemctl daemon-reload
      echo "FULL_HOST_ORCHESTRATOR_ROLLBACK_DONE=1"
      verify_status
      exit 0
      ;;
    *)
      printf 'Unsupported mode: %s\n' "$MODE" >&2
      exit 2
      ;;
  esac
fi

install -d "$SYSTEMD_DIR" "$ENV_DIR" "$LOG_DIR" "$INSTALL_ROOT"
rsync -a \
  --exclude '.git' \
  --exclude '__pycache__' \
  --exclude '.pytest_cache' \
  "${ROOT_DIR}/" "${INSTALL_ROOT}/"
install -m 0644 "${SCRIPT_DIR}/${UNIT_NAME}.service" "${SYSTEMD_DIR}/${UNIT_NAME}.service"
install -m 0644 "${SCRIPT_DIR}/${UNIT_NAME}.timer" "${SYSTEMD_DIR}/${UNIT_NAME}.timer"

if [[ ! -f "${ENV_DIR}/reboot-recovery.env" ]]; then
  install -m 0600 /dev/null "${ENV_DIR}/reboot-recovery.env"
  cat >"${ENV_DIR}/reboot-recovery.env" <<'ENV'
# Optional no-secret runtime knobs for full-host reboot recovery.
# Do not place raw tokens in source control. Put host-local secrets only in this root-owned file.
HOSTS="99 110 111 112 120 121 188"
RECOVERY_SLO_SECONDS=600
SSH_TIMEOUT_SECONDS=12
DISABLE_TELEGRAM=1
# TELEGRAM_NOTIFY_SCRIPT=/usr/local/sbin/awoooi-telegram-notify
ENV
fi

systemctl daemon-reload
if [[ "$ENABLE_NOW" -eq 1 ]]; then
  systemctl enable --now "${UNIT_NAME}.timer"
else
  echo "ENABLE_NOW_SKIPPED=1"
fi
verify_status