awoooi/scripts/reboot-recovery/repair-110-startup-script-stub.sh

#!/usr/bin/env bash
# Restore the real 110 startup recovery script when a runner fail-closed stub
# was accidentally installed at /usr/local/bin/awoooi-startup-110.sh.
#
# This script does not reboot, restart Docker, reload Nginx, touch K3s, DBs,
# firewall, secrets, .env files, sessions, SQLite, or runner registration data.

set -euo pipefail

MODE="verify"
HOST="${HOST:-wooo@192.168.0.110}"
REMOTE_PATH="${REMOTE_PATH:-/usr/local/bin/awoooi-startup-110.sh}"
RESET_FAILED="${RESET_FAILED:-1}"
START_SERVICE=0
SSH_CONNECT_TIMEOUT_SECONDS="${SSH_CONNECT_TIMEOUT_SECONDS:-6}"
SSH_STRICT_HOST_KEY_CHECKING="${SSH_STRICT_HOST_KEY_CHECKING:-accept-new}"
BACKUP_SUFFIX="${BACKUP_SUFFIX:-before-full-startup-restore-$(date +%Y%m%d-%H%M%S)}"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
LOCAL_SCRIPT="$SCRIPT_DIR/awoooi-startup-110.sh"
REMOTE_TMP="/tmp/awoooi-startup-110.sh.restore-${BACKUP_SUFFIX}"

SSH_OPTS=(
  -o BatchMode=yes
  -o ConnectTimeout="$SSH_CONNECT_TIMEOUT_SECONDS"
  -o ConnectionAttempts=1
  -o ServerAliveInterval=5
  -o ServerAliveCountMax=1
  -o StrictHostKeyChecking="$SSH_STRICT_HOST_KEY_CHECKING"
)

usage() {
  cat <<'USAGE'
Usage: repair-110-startup-script-stub.sh [--verify|--dry-run|--apply] [--start-service]

Restores /usr/local/bin/awoooi-startup-110.sh from repo source if the live host
has the runner/CD-lane fail-closed stub installed in its place.

Options:
  --verify         Read live hash, stub marker, and service state. Default.
  --dry-run        Print the controlled apply plan without writing host state.
  --apply          Copy the real startup script, backup the live file, install,
                   daemon-reload, and reset only awoooi-startup-110.service.
  --start-service  With --apply only, start awoooi-startup-110.service after
                   installing. This can run docker compose up checks, so it is
                   intentionally opt-in.
  -h, --help       Show this help.
USAGE
}

while [ "$#" -gt 0 ]; do
  case "$1" in
    --verify)
      MODE="verify"
      ;;
    --dry-run)
      MODE="dry-run"
      ;;
    --apply)
      MODE="apply"
      ;;
    --start-service)
      START_SERVICE=1
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      printf 'Unknown argument: %s\n' "$1" >&2
      usage >&2
      exit 2
      ;;
  esac
  shift
done

if [ ! -s "$LOCAL_SCRIPT" ]; then
  echo "LOCAL_STARTUP_SCRIPT_MISSING path=$LOCAL_SCRIPT" >&2
  exit 66
fi

local_sha256() {
  if command -v sha256sum >/dev/null 2>&1; then
    sha256sum "$LOCAL_SCRIPT" | awk '{print $1}'
  else
    shasum -a 256 "$LOCAL_SCRIPT" | awk '{print $1}'
  fi
}

quote() {
  printf '%q' "$1"
}

remote_readback() {
  local expected_hash="$1"
  local remote_path_q
  remote_path_q="$(quote "$REMOTE_PATH")"
  ssh "${SSH_OPTS[@]}" "$HOST" "EXPECTED_HASH=$(quote "$expected_hash") REMOTE_PATH=$remote_path_q bash -s" <<'REMOTE'
set -uo pipefail
echo "STARTUP_STUB_REPAIR_READBACK=1"
echo "remote_path=$REMOTE_PATH"
if [ -e "$REMOTE_PATH" ]; then
  echo "remote_file_present=1"
  echo "remote_hash=$(sha256sum "$REMOTE_PATH" 2>/dev/null | awk '{print $1}' || true)"
  echo "remote_matches_expected=$([ "$(sha256sum "$REMOTE_PATH" 2>/dev/null | awk '{print $1}' || true)" = "$EXPECTED_HASH" ] && echo 1 || echo 0)"
  if head -20 "$REMOTE_PATH" 2>/dev/null | grep -Fq "runner/CD lane is fail-closed"; then
    echo "remote_is_runner_failclosed_stub=1"
  else
    echo "remote_is_runner_failclosed_stub=0"
  fi
  file "$REMOTE_PATH" 2>/dev/null | sed 's/^/remote_file_type=/'
else
  echo "remote_file_present=0"
  echo "remote_hash="
  echo "remote_matches_expected=0"
  echo "remote_is_runner_failclosed_stub=0"
fi
echo "service_enabled=$(systemctl is-enabled awoooi-startup-110.service 2>/dev/null || true)"
echo "service_active=$(systemctl is-active awoooi-startup-110.service 2>/dev/null || true)"
echo "service_result=$(systemctl show awoooi-startup-110.service -p Result --value 2>/dev/null || true)"
echo "service_exec_status=$(systemctl show awoooi-startup-110.service -p ExecMainStatus --value 2>/dev/null || true)"
echo "system_failed_units=$(systemctl --failed --no-legend --plain 2>/dev/null | wc -l | tr -d ' ')"
REMOTE
}

print_plan() {
  cat <<PLAN
AWOOOI_110_STARTUP_STUB_REPAIR_PLAN=1
mode=$MODE
host=$HOST
remote_path=$REMOTE_PATH
local_script=$LOCAL_SCRIPT
local_hash=$(local_sha256)
backup_path=${REMOTE_PATH}.${BACKUP_SUFFIX}
reset_failed=$RESET_FAILED
start_service=$START_SERVICE
forbidden_actions=reboot,docker_restart,nginx_reload,k3s_change,db_change,firewall_change,secret_read,runner_registration_read
rollback_hint=restore ${REMOTE_PATH}.${BACKUP_SUFFIX} to $REMOTE_PATH and systemctl daemon-reload
PLAN
}

apply_restore() {
  local expected_hash
  local start_command=":"
  expected_hash="$(local_sha256)"

  scp "${SSH_OPTS[@]}" "$LOCAL_SCRIPT" "$HOST:$REMOTE_TMP" >/dev/null

  if [ "$START_SERVICE" -eq 1 ]; then
    start_command="sudo -n systemctl start awoooi-startup-110.service"
  fi

  ssh "${SSH_OPTS[@]}" "$HOST" \
    "REMOTE_TMP=$(quote "$REMOTE_TMP") REMOTE_PATH=$(quote "$REMOTE_PATH") BACKUP_SUFFIX=$(quote "$BACKUP_SUFFIX") RESET_FAILED=$(quote "$RESET_FAILED") EXPECTED_HASH=$(quote "$expected_hash") bash -s" <<REMOTE
set -euo pipefail
if ! sudo -n true >/dev/null 2>&1; then
  echo "BLOCKED sudo_nopasswd_required_for_startup_script_install" >&2
  exit 77
fi
sudo -n chattr -i "\$REMOTE_PATH" "\$(dirname "\$REMOTE_PATH")" >/dev/null 2>&1 || true
if [ -e "\$REMOTE_PATH" ]; then
  sudo -n cp -p "\$REMOTE_PATH" "\$REMOTE_PATH.\$BACKUP_SUFFIX"
fi
sudo -n install -o root -g root -m 0755 "\$REMOTE_TMP" "\$REMOTE_PATH"
remote_hash=\$(sha256sum "\$REMOTE_PATH" | awk '{print \$1}')
if [ "\$remote_hash" != "\$EXPECTED_HASH" ]; then
  echo "BLOCKED installed_hash_mismatch expected=\$EXPECTED_HASH actual=\$remote_hash" >&2
  exit 78
fi
sudo -n systemctl daemon-reload
if [ "\$RESET_FAILED" = "1" ]; then
  sudo -n systemctl reset-failed awoooi-startup-110.service >/dev/null 2>&1 || true
fi
rm -f "\$REMOTE_TMP"
$start_command
REMOTE

  remote_readback "$expected_hash"
}

print_plan

case "$MODE" in
  dry-run)
    echo "DRY_RUN=1"
    exit 0
    ;;
  verify)
    remote_readback "$(local_sha256)"
    ;;
  apply)
    apply_restore
    ;;
  *)
    echo "Unsupported mode: $MODE" >&2
    exit 2
    ;;
esac