#!/usr/bin/env bash
set -Eeuo pipefail

# Full-host reboot recovery orchestrator.
# This is intentionally a coordinator: irreversible actions stay out of scope,
# while bounded service-start/recovery hooks are delegated to audited scripts.

MODE="check"
ARTIFACT_ROOT="${ARTIFACT_ROOT:-/tmp/awoooi-full-host-reboot-recovery}"
RUN_ID="${RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)}"
HOSTS="${HOSTS:-99 110 111 112 120 121 188}"
PUBLIC_ROUTES="${PUBLIC_ROUTES:-https://awoooi.wooo.work/api/v1/health https://awoooi.wooo.work/ https://stock.wooo.work/healthz https://stock.wooo.work/api/healthz https://gitea.wooo.work/ https://harbor.wooo.work/ https://registry.wooo.work/v2/}"
SSH_TIMEOUT_SECONDS="${SSH_TIMEOUT_SECONDS:-12}"
RECOVERY_SLO_SECONDS="${RECOVERY_SLO_SECONDS:-600}"
TELEGRAM_NOTIFY_SCRIPT="${TELEGRAM_NOTIFY_SCRIPT:-}"
DISABLE_TELEGRAM="${DISABLE_TELEGRAM:-0}"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ARTIFACT_DIR="${ARTIFACT_ROOT%/}/${RUN_ID}"
EVENTS_TSV="${ARTIFACT_DIR}/events.tsv"
FINDINGS_TSV="${ARTIFACT_DIR}/findings.tsv"
ACTIONS_TSV="${ARTIFACT_DIR}/actions.tsv"
SUMMARY_TXT="${ARTIFACT_DIR}/summary.txt"

usage() {
  cat <<'USAGE'
Usage:
  full-host-reboot-orchestrator.sh [--check|--apply] [--artifact-dir DIR]

Modes:
  --check   Read-only orchestration and evidence collection.
  --apply   Run controlled apply hooks that are already allowlisted.

Environment:
  HOSTS                    Space-separated host list, default: 99 110 111 112 120 121 188
  PUBLIC_ROUTES            Space-separated route readback list.
  SSH_TIMEOUT_SECONDS      SSH connect/command timeout, default: 12.
  RECOVERY_SLO_SECONDS     Full recovery target, default: 600.
  TELEGRAM_NOTIFY_SCRIPT   Optional no-secret wrapper script for Telegram notification.
  DISABLE_TELEGRAM=1       Suppress Telegram wrapper even when configured.
USAGE
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --check)
      MODE="check"
      shift
      ;;
    --apply)
      MODE="apply"
      shift
      ;;
    --artifact-dir)
      ARTIFACT_DIR="$2"
      ARTIFACT_ROOT="$(dirname "$2")"
      RUN_ID="$(basename "$2")"
      shift 2
      ;;
    --help|-h)
      usage
      exit 0
      ;;
    *)
      echo "Unknown argument: $1" >&2
      usage >&2
      exit 2
      ;;
  esac
done

mkdir -p "$ARTIFACT_DIR"
: >"$EVENTS_TSV"
: >"$FINDINGS_TSV"
: >"$ACTIONS_TSV"

event() {
  local level="$1" component="$2" message="$3"
  printf '%s\t%s\t%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$level" "$component" "$message" | tee -a "$EVENTS_TSV" >/dev/null
}

finding() {
  local severity="$1" component="$2" code="$3" message="$4"
  printf '%s\t%s\t%s\t%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$severity" "$component" "$code" "$message" | tee -a "$FINDINGS_TSV" >/dev/null
}

action() {
  local mode="$1" component="$2" code="$3" message="$4"
  printf '%s\t%s\t%s\t%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$mode" "$component" "$code" "$message" | tee -a "$ACTIONS_TSV" >/dev/null
}

notify() {
  local phase="$1" status="$2" message="$3"
  if [[ "$DISABLE_TELEGRAM" == "1" || -z "$TELEGRAM_NOTIFY_SCRIPT" ]]; then
    event "INFO" "telegram" "notify skipped phase=${phase} status=${status}: ${message}"
    return 0
  fi
  if [[ ! -x "$TELEGRAM_NOTIFY_SCRIPT" ]]; then
    finding "P1" "telegram" "TELEGRAM_WRAPPER_NOT_EXECUTABLE" "$TELEGRAM_NOTIFY_SCRIPT"
    return 0
  fi
  "$TELEGRAM_NOTIFY_SCRIPT" --phase "$phase" --status "$status" --artifact-dir "$ARTIFACT_DIR" --message "$message" || {
    finding "P1" "telegram" "TELEGRAM_NOTIFY_FAILED" "phase=${phase} status=${status}"
  }
}

host_ip() {
  case "$1" in
    99) printf '192.168.0.99' ;;
    110) printf '192.168.0.110' ;;
    111) printf '192.168.0.111' ;;
    112) printf '192.168.0.112' ;;
    120) printf '192.168.0.120' ;;
    121) printf '192.168.0.121' ;;
    188) printf '192.168.0.188' ;;
    *) printf '%s' "$1" ;;
  esac
}

ssh_user() {
  case "$1" in
    188) printf 'ollama' ;;
    *) printf 'wooo' ;;
  esac
}

probe_host_tcp() {
  local host="$1" ip
  ip="$(host_ip "$host")"
  if nc -z -w "$SSH_TIMEOUT_SECONDS" "$ip" 22 >/dev/null 2>&1; then
    event "INFO" "host-${host}" "ssh tcp open ${ip}:22"
    return 0
  fi
  finding "P0" "host-${host}" "HOST_SSH_PORT_DOWN" "${ip}:22 not reachable"
  return 1
}

probe_host_command() {
  local host="$1" ip user output_file
  ip="$(host_ip "$host")"
  user="$(ssh_user "$host")"
  output_file="${ARTIFACT_DIR}/host-${host}-uptime.txt"
  if timeout "$((SSH_TIMEOUT_SECONDS + 5))" ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout="$SSH_TIMEOUT_SECONDS" "${user}@${ip}" 'printf "host=%s\n" "$(hostname)"; uptime -s; uptime' >"$output_file" 2>&1; then
    event "INFO" "host-${host}" "ssh command ok ${user}@${ip}"
    return 0
  fi
  if grep -Eqi 'Permission denied|publickey|authentication' "$output_file"; then
    finding "P0" "host-${host}" "SSH_AUTH_FAILED" "${user}@${ip}"
  elif grep -Eqi 'timed out|Timeout|not responding' "$output_file"; then
    finding "P0" "host-${host}" "SSH_COMMAND_SESSION_TIMEOUT" "${user}@${ip}"
  else
    finding "P0" "host-${host}" "SSH_COMMAND_FAILED" "${user}@${ip}; see ${output_file}"
  fi
  return 1
}

detect_recent_reboot() {
  local host="$1" ip user output_file boot_epoch now_epoch age
  ip="$(host_ip "$host")"
  user="$(ssh_user "$host")"
  output_file="${ARTIFACT_DIR}/host-${host}-boot.txt"
  if ! timeout "$((SSH_TIMEOUT_SECONDS + 5))" ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout="$SSH_TIMEOUT_SECONDS" "${user}@${ip}" 'date +%s; cat /proc/stat | awk "/btime/ {print \$2}"' >"$output_file" 2>&1; then
    event "WARN" "host-${host}" "recent reboot detection unavailable"
    return 0
  fi
  now_epoch="$(sed -n '1p' "$output_file")"
  boot_epoch="$(sed -n '2p' "$output_file")"
  if [[ "$now_epoch" =~ ^[0-9]+$ && "$boot_epoch" =~ ^[0-9]+$ ]]; then
    age=$((now_epoch - boot_epoch))
    printf '%s\t%s\t%s\n' "$host" "$boot_epoch" "$age" >>"${ARTIFACT_DIR}/boot-age.tsv"
    if (( age <= RECOVERY_SLO_SECONDS )); then
      event "INFO" "host-${host}" "recent reboot detected age_seconds=${age}"
    else
      event "INFO" "host-${host}" "host already past reboot window age_seconds=${age}"
    fi
  fi
}

probe_public_routes() {
  local route status output_file safe_name
  for route in $PUBLIC_ROUTES; do
    safe_name="$(printf '%s' "$route" | sed 's#[^A-Za-z0-9._-]#_#g')"
    output_file="${ARTIFACT_DIR}/route-${safe_name}.txt"
    status="$(curl -k -L --max-time 12 -o "$output_file" -w '%{http_code}' "$route" 2>"${output_file}.err" || true)"
    printf '%s\t%s\n' "$route" "$status" >>"${ARTIFACT_DIR}/public-routes.tsv"
    case "$status" in
      200|204|301|302|307|308|401)
        event "INFO" "route" "${route} status=${status}"
        ;;
      502|503|504)
        finding "P0" "route" "PUBLIC_ROUTE_UPSTREAM_FAILED" "${route} status=${status}"
        ;;
      000)
        finding "P0" "route" "PUBLIC_ROUTE_UNREACHABLE" "${route} status=${status}"
        ;;
      *)
        finding "P1" "route" "PUBLIC_ROUTE_UNEXPECTED_STATUS" "${route} status=${status}"
        ;;
    esac
  done
}

run_ai_log_triage() {
  local triage_dir="${ARTIFACT_DIR}/ai-log-triage"
  local rc=0
  if [[ ! -x "${SCRIPT_DIR}/ai-log-triage-auto-recovery.sh" ]]; then
    finding "P0" "recovery" "AI_LOG_TRIAGE_SCRIPT_MISSING" "${SCRIPT_DIR}/ai-log-triage-auto-recovery.sh"
    return 1
  fi
  action "$MODE" "ai-log-triage" "RUN_AI_LOG_TRIAGE" "artifact_dir=${triage_dir}"
  "${SCRIPT_DIR}/ai-log-triage-auto-recovery.sh" "--${MODE}" --artifact-dir "$triage_dir" || rc=$?
  if [[ "$rc" -ne 0 && "$rc" -ne 2 ]]; then
    finding "P0" "ai-log-triage" "AI_LOG_TRIAGE_FAILED" "see ${triage_dir}"
    return 1
  fi
  if [[ "$rc" -eq 2 ]]; then
    event "INFO" "ai-log-triage" "AI_LOG_TRIAGE_REPORTED_FINDINGS rc=2"
  fi
  if [[ -s "${triage_dir}/findings.tsv" ]]; then
    awk -v now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" -F '\t' \
      '{print now "\t" $2 "\tai-log-triage\t" $1 "\thost=" $3 " evidence=" $4 " next_action=" $5}' \
      "${triage_dir}/findings.tsv" >>"$FINDINGS_TSV" || true
  fi
}

run_scorecard() {
  local scorecard_file="${ARTIFACT_DIR}/scorecard.txt"
  if [[ ! -f "${SCRIPT_DIR}/reboot-auto-recovery-slo-scorecard.py" ]]; then
    finding "P0" "scorecard" "SCORECARD_SCRIPT_MISSING" "${SCRIPT_DIR}/reboot-auto-recovery-slo-scorecard.py"
    return 1
  fi
  action "check" "scorecard" "RUN_REBOOT_SCORECARD" "$scorecard_file"
  python3 "${SCRIPT_DIR}/reboot-auto-recovery-slo-scorecard.py" >"$scorecard_file" 2>&1 || {
    finding "P0" "scorecard" "SCORECARD_FAILED" "see ${scorecard_file}"
    return 1
  }
}

run_stock_upstream_gate_hint() {
  local stock_gate="/Users/ogt/codex-workspaces/gitea-restored-worktrees-20260701/stockplatform-v2/scripts/ops/stockplatform-api-upstream-recovery-gate.sh"
  if [[ ! -x "$stock_gate" ]]; then
    event "WARN" "stockplatform" "stock upstream recovery gate not executable or missing: ${stock_gate}"
    return 0
  fi
  action "check" "stockplatform" "VERIFY_STOCK_API_UPSTREAM_GATE" "$stock_gate"
  STOCKPLATFORM_DIRECT_EDGE_URL="${STOCKPLATFORM_DIRECT_EDGE_URL:-http://127.0.0.1:31235}" \
  STOCKPLATFORM_PUBLIC_WEB_URL="${STOCKPLATFORM_PUBLIC_WEB_URL:-https://stock.wooo.work}" \
    "$stock_gate" --verify-only >"${ARTIFACT_DIR}/stockplatform-api-upstream-gate.txt" 2>&1 || {
      finding "P0" "stockplatform" "STOCK_API_UPSTREAM_GATE_BLOCKED" "see ${ARTIFACT_DIR}/stockplatform-api-upstream-gate.txt"
    }
}

write_summary() {
  local p0_count p1_count elapsed status
  elapsed=$(( $(date +%s) - START_EPOCH ))
  p0_count="$(awk -F '\t' '$2=="P0"{c++} END{print c+0}' "$FINDINGS_TSV")"
  p1_count="$(awk -F '\t' '$2=="P1"{c++} END{print c+0}' "$FINDINGS_TSV")"
  if (( p0_count == 0 )); then
    status="RECOVERY_READY_FOR_RUNTIME_CONFIRMATION"
  else
    status="RECOVERY_BLOCKED_P0"
  fi
  {
    printf 'run_id=%s\n' "$RUN_ID"
    printf 'mode=%s\n' "$MODE"
    printf 'status=%s\n' "$status"
    printf 'elapsed_seconds=%s\n' "$elapsed"
    printf 'slo_seconds=%s\n' "$RECOVERY_SLO_SECONDS"
    printf 'p0_count=%s\n' "$p0_count"
    printf 'p1_count=%s\n' "$p1_count"
    printf 'artifact_dir=%s\n' "$ARTIFACT_DIR"
    printf '\n[P0 findings]\n'
    awk -F '\t' '$2=="P0"{print}' "$FINDINGS_TSV" || true
    printf '\n[Actions]\n'
    cat "$ACTIONS_TSV"
  } >"$SUMMARY_TXT"
  cat "$SUMMARY_TXT"
}

START_EPOCH="$(date +%s)"

event "INFO" "orchestrator" "start mode=${MODE} run_id=${RUN_ID}"
notify "start" "running" "full-host reboot recovery started mode=${MODE}"

for host in $HOSTS; do
  probe_host_tcp "$host" || true
done

for host in $HOSTS; do
  probe_host_command "$host" || true
  detect_recent_reboot "$host" || true
done

probe_public_routes
run_ai_log_triage || true
run_scorecard || true
run_stock_upstream_gate_hint || true

write_summary

if awk -F '\t' '$2=="P0"{found=1} END{exit found ? 0 : 1}' "$FINDINGS_TSV"; then
  notify "finish" "blocked" "full-host reboot recovery has P0 blockers; artifact=${ARTIFACT_DIR}"
  exit 1
fi

notify "finish" "ok" "full-host reboot recovery has no P0 blockers; artifact=${ARTIFACT_DIR}"
exit 0