#!/usr/bin/env bash
# Install the read-only AWOOOI reboot auto-recovery SLO verifier on host 110.
#
# This installer only stages source scripts, enables a systemd timer, starts the
# verifier service once, and reads back metrics/status. It does not reboot hosts
# or restart Docker, Nginx, K3s, PostgreSQL, Redis, firewall, or product units.

set -euo pipefail

HOST="wooo@192.168.0.110"
REMOTE_ROOT="/home/wooo/awoooi-reboot-recovery-slo"
TEXTFILE_DIR="/home/wooo/node_exporter_textfiles"
LOG_DIR="/home/wooo/reboot-recovery"
MODE="install"
RUN_ONCE=1
SSH_CONNECT_TIMEOUT_SECONDS="${SSH_CONNECT_TIMEOUT_SECONDS:-8}"

usage() {
  cat <<'USAGE'
Usage: install-reboot-auto-recovery-slo-110.sh [options]

Installs the AWOOOI reboot auto-recovery 10-minute SLO verifier on host 110.

Options:
  --dry-run              Print the controlled apply plan without writing host state.
  --verify-only          Read back timer/service/metric state without writing.
  --rollback             Disable/remove the verifier timer/service/wrapper.
  --no-run-once          Install and enable the timer without starting the verifier once.
  --host HOST            SSH target. Default: wooo@192.168.0.110
  --remote-root DIR      Remote source root. Default: /home/wooo/awoooi-reboot-recovery-slo
  --textfile-dir DIR     Node exporter textfile dir. Default: /home/wooo/node_exporter_textfiles
  --log-dir DIR          Evidence log dir. Default: /home/wooo/reboot-recovery
  -h, --help             Show this help.

Controlled apply boundaries:
  - allowed: stage repo scripts, install verifier unit/timer, daemon-reload,
    enable/start the verifier timer, start only awoooi-reboot-auto-recovery-slo.service
  - forbidden here: reboot, node drain, firewall changes, destructive DB work,
    service restarts for Docker/Nginx/K3s/PostgreSQL/Redis/product workloads,
    secret/session/auth/.env reads, GitHub usage, force push
USAGE
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --dry-run)
      MODE="dry-run"
      ;;
    --verify-only)
      MODE="verify"
      ;;
    --rollback)
      MODE="rollback"
      ;;
    --no-run-once)
      RUN_ONCE=0
      ;;
    --host)
      HOST="${2:?--host requires a value}"
      shift
      ;;
    --remote-root)
      REMOTE_ROOT="${2:?--remote-root requires a value}"
      shift
      ;;
    --textfile-dir)
      TEXTFILE_DIR="${2:?--textfile-dir requires a value}"
      shift
      ;;
    --log-dir)
      LOG_DIR="${2:?--log-dir requires a value}"
      shift
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      printf 'Unknown argument: %s\n' "$1" >&2
      usage >&2
      exit 2
      ;;
  esac
  shift
done

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
SSH_OPTS=(
  -o BatchMode=yes
  -o ConnectTimeout="$SSH_CONNECT_TIMEOUT_SECONDS"
  -o ConnectionAttempts=1
  -o ServerAliveInterval="${SSH_SERVER_ALIVE_INTERVAL_SECONDS:-5}"
  -o ServerAliveCountMax="${SSH_SERVER_ALIVE_COUNT_MAX:-1}"
  -o NumberOfPasswordPrompts=0
)
UNIT_NAME="awoooi-reboot-auto-recovery-slo"
REMOTE_TARBALL="/tmp/${UNIT_NAME}-$(date '+%Y%m%d%H%M%S').tar.gz"

quote() {
  printf '%q' "$1"
}

print_plan() {
  cat <<PLAN
AWOOOI_REBOOT_AUTO_RECOVERY_SLO_INSTALL_PLAN=1
mode=$MODE
host=$HOST
remote_root=$REMOTE_ROOT
textfile_dir=$TEXTFILE_DIR
log_dir=$LOG_DIR
run_once=$RUN_ONCE
source_of_truth=$ROOT_DIR
target_selector=host_110_systemd_timer:${UNIT_NAME}.timer
dry_run_command=bash scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh --dry-run
rollback_command=bash scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh --rollback
verify_command=bash scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh --verify-only
post_apply_verifier=systemctl_is_enabled_active_timer_and_reboot_auto_recovery_slo_metric_present
PLAN
}

remote_verify_command() {
  cat <<'REMOTE'
set -uo pipefail
unit="awoooi-reboot-auto-recovery-slo"
metric="${TEXTFILE_DIR}/reboot_auto_recovery_slo.prom"
latest_dir="$(find "${LOG_DIR}" -maxdepth 1 -type d -name 'reboot-auto-recovery-slo-*' 2>/dev/null | sort | tail -n 1)"
echo "SLO_TIMER_ENABLED=$(systemctl is-enabled "${unit}.timer" 2>/dev/null || true)"
echo "SLO_TIMER_ACTIVE=$(systemctl is-active "${unit}.timer" 2>/dev/null || true)"
echo "SLO_SERVICE_LOAD=$(systemctl show "${unit}.service" -p LoadState --value 2>/dev/null || true)"
echo "SLO_SERVICE_RESULT=$(systemctl show "${unit}.service" -p Result --value 2>/dev/null || true)"
echo "SLO_SERVICE_LAST_STATUS=$(systemctl show "${unit}.service" -p ExecMainStatus --value 2>/dev/null || true)"
echo "SLO_METRIC_PRESENT=$([ -s "$metric" ] && echo 1 || echo 0)"
echo "SLO_METRIC_PATH=$metric"
if [ -s "$metric" ]; then
  awk 'NF && $1 !~ /^#/ {print "SLO_METRIC_LINE " $0}' "$metric" | tail -20
fi
echo "SLO_LATEST_ARTIFACT_DIR=${latest_dir:-none}"
if [ -n "${latest_dir:-}" ] && [ -s "$latest_dir/scorecard.json" ]; then
  python3 - "$latest_dir/scorecard.json" <<'PY'
import json, sys
payload = json.load(open(sys.argv[1], encoding="utf-8"))
print("SLO_SCORECARD_STATUS=" + str(payload.get("status", "unknown")))
print(
    "SLO_CAN_CLAIM_ALL_SERVICES_RECOVERED_WITHIN_TARGET="
    + ("1" if payload.get("can_claim_all_services_recovered_within_target") else "0")
)
print("SLO_ACTIVE_BLOCKERS=" + ",".join(payload.get("active_blockers") or []))
print(
    "SLO_OBSERVED_HOSTS="
    + ",".join(payload.get("host_boot_detection", {}).get("observed_hosts") or [])
)
PY
fi
REMOTE
}

ssh_remote() {
  ssh "${SSH_OPTS[@]}" "$HOST" \
    "TEXTFILE_DIR=$(quote "$TEXTFILE_DIR") LOG_DIR=$(quote "$LOG_DIR") bash -s"
}

verify_remote() {
  remote_verify_command | ssh_remote
}

rollback_remote() {
  cat <<'REMOTE' | ssh_remote
set -euo pipefail
unit="awoooi-reboot-auto-recovery-slo"
sudo systemctl disable --now "${unit}.timer" >/dev/null 2>&1 || true
sudo systemctl stop "${unit}.service" >/dev/null 2>&1 || true
sudo rm -f "/etc/systemd/system/${unit}.timer" "/etc/systemd/system/${unit}.service"
sudo rm -f "/usr/local/bin/${unit}.sh"
sudo systemctl daemon-reload
echo "SLO_ROLLBACK_DONE=1"
REMOTE
}

install_remote() {
  local run_once_command=":"
  if [[ "$RUN_ONCE" -eq 1 ]]; then
    run_once_command="sudo systemctl start ${UNIT_NAME}.service"
  fi

  cat <<REMOTE | ssh_remote
set -euo pipefail
remote_root=$(quote "$REMOTE_ROOT")
tarball=$(quote "$REMOTE_TARBALL")
textfile_dir=$(quote "$TEXTFILE_DIR")
log_dir=$(quote "$LOG_DIR")
unit=$(quote "$UNIT_NAME")
mkdir -p "\$remote_root" "\$textfile_dir" "\$log_dir"
tar -xzf "\$tarball" -C "\$remote_root"
find "\$remote_root/scripts/reboot-recovery" "\$remote_root/scripts/security" -type f \( -name '*.sh' -o -name '*.py' -o -name '*.awk' \) -exec chmod 0755 {} \;
sudo install -m 0755 "\$remote_root/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh" "/usr/local/bin/\${unit}.sh"
sudo install -m 0644 "\$remote_root/scripts/reboot-recovery/\${unit}.service" "/etc/systemd/system/\${unit}.service"
sudo install -m 0644 "\$remote_root/scripts/reboot-recovery/\${unit}.timer" "/etc/systemd/system/\${unit}.timer"
sudo sed -i "s|^Environment=ROOT_DIR=.*|Environment=ROOT_DIR=\$remote_root|" "/etc/systemd/system/\${unit}.service"
sudo sed -i "s|^Environment=TEXTFILE_DIR=.*|Environment=TEXTFILE_DIR=\$textfile_dir|" "/etc/systemd/system/\${unit}.service"
sudo sed -i "s|^Environment=LOG_DIR=.*|Environment=LOG_DIR=\$log_dir|" "/etc/systemd/system/\${unit}.service"
sudo systemctl daemon-reload
sudo systemctl enable --now "\${unit}.timer"
$run_once_command
rm -f "\$tarball"
echo "SLO_INSTALL_DONE=1"
REMOTE
}

create_payload() {
  local tarball="$1"
  export COPYFILE_DISABLE=1
  tar --no-xattrs -C "$ROOT_DIR" \
    --exclude='*/__pycache__' \
    --exclude='*.pyc' \
    --exclude='.pytest_cache' \
    -czf "$tarball" \
    scripts/reboot-recovery \
    scripts/security \
    ops/maintenance \
    ops/monitoring \
    docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md
}

print_plan

case "$MODE" in
  dry-run)
    cat <<DRYRUN
DRY_RUN=1
would_create_tarball_from=$ROOT_DIR/scripts/reboot-recovery,$ROOT_DIR/scripts/security
would_include_source_gates=$ROOT_DIR/ops/maintenance,$ROOT_DIR/ops/monitoring,$ROOT_DIR/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md
would_copy_tarball_to=$HOST:$REMOTE_TARBALL
would_enable_timer=${UNIT_NAME}.timer
would_start_service_once=$RUN_ONCE
would_write_metric=$TEXTFILE_DIR/reboot_auto_recovery_slo.prom
DRYRUN
    exit 0
    ;;
  verify)
    verify_remote
    exit 0
    ;;
  rollback)
    rollback_remote
    verify_remote
    exit 0
    ;;
  install)
    ;;
  *)
    printf 'Unsupported mode: %s\n' "$MODE" >&2
    exit 2
    ;;
esac

payload_dir="$(mktemp -d "${TMPDIR:-/tmp}/${UNIT_NAME}.XXXXXX")"
payload="${payload_dir}/payload.tar.gz"
trap 'rm -rf "$payload_dir"' EXIT
create_payload "$payload"
scp "${SSH_OPTS[@]}" "$payload" "$HOST:$REMOTE_TARBALL"
install_remote
verify_remote
