#!/bin/bash
# AWOOOI 重開機自動恢復腳本
# 2026-04-04 ogt: 根據實際事故建立，處理 container / Docker 啟動順序與 K3s Kine 維護。
# 2026-06-26 Codex: PostgreSQL checkpoint/WAL 錯誤改為 fail-closed；
# 不在自動啟動腳本內執行 pg_resetwal，避免資料破壞被誤判成恢復。
# 2026-06-26 Codex: 允許受控 recovery container 提供 14/main runtime；
# 不再因 systemd postgresql@14-main failed 而誤判活 DB 為不可用。
# 2026-06-30 Codex: ClawBot rebuild 改為 bounded / opt-in，避免重開機
# startup oneshot 被單一 image rebuild timeout 拖成 failed。
# 部署位置: /usr/local/bin/awoooi-startup.sh (on 192.168.0.188)
# systemd unit: /etc/systemd/system/awoooi-startup.service

set -uo pipefail
LOG="/var/log/awoooi-startup.log"
exec > >(tee -a "$LOG") 2>&1

log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }

CLAWBOT_UP_TIMEOUT_SECONDS="${CLAWBOT_UP_TIMEOUT_SECONDS:-90}"
CLAWBOT_BUILD_TIMEOUT_SECONDS="${CLAWBOT_BUILD_TIMEOUT_SECONDS:-120}"
CLAWBOT_STARTUP_REBUILD_ALLOWED="${CLAWBOT_STARTUP_REBUILD_ALLOWED:-0}"
CONTAINERD_METADATA_REPAIR_ALLOWED="${CONTAINERD_METADATA_REPAIR_ALLOWED:-0}"
DOCKER_NETWORK_METADATA_REPAIR_ALLOWED="${DOCKER_NETWORK_METADATA_REPAIR_ALLOWED:-0}"
KINE_MAINTENANCE_ALLOWED="${KINE_MAINTENANCE_ALLOWED:-0}"
SIGNOZ_188_START_ALLOWED="${SIGNOZ_188_START_ALLOWED:-0}"

run_bounded() {
    seconds="$1"
    shift
    if command -v timeout >/dev/null 2>&1; then
        timeout "${seconds}s" "$@"
    else
        "$@"
    fi
}

postgres_runtime_ready() {
    if systemctl is-active postgresql@14-main >/dev/null 2>&1; then
        log "✅ PostgreSQL systemd unit active"
        return 0
    fi

    if docker inspect -f '{{.State.Running}} {{.HostConfig.NetworkMode}}' k3s-postgres-recovery 2>/dev/null | grep -q '^true host$'; then
        if pg_isready -h localhost -p 5432 >/dev/null 2>&1; then
            log "✅ PostgreSQL recovery container active on host network"
            return 0
        fi
    fi

    return 1
}

log "=== AWOOOI 啟動序列開始 ==="

# ──────────────────────────────────────────────
# STEP 1: containerd 修復（若損壞）
# ──────────────────────────────────────────────
log "[1/7] 檢查 containerd..."

if ! systemctl is-active containerd >/dev/null 2>&1; then
    log "containerd 未啟動，嘗試啟動..."
    systemctl start containerd || true
    sleep 5
fi

if ! systemctl is-active containerd >/dev/null 2>&1; then
    log "containerd 啟動失敗，檢查 BoltDB 損壞..."
    BOLT_DB="/var/lib/containerd/io.containerd.metadata.v1.bolt/meta.db"
    if [ -f "$BOLT_DB" ]; then
        if [ "$CONTAINERD_METADATA_REPAIR_ALLOWED" != "1" ]; then
            log "❌ containerd metadata repair 未授權；禁止自動備份/刪除 meta.db"
            exit 1
        fi
        log "containerd metadata repair 已授權；先備份再移除損壞的 meta.db..."
        cp "$BOLT_DB" "${BOLT_DB}.bak.$(date +%Y%m%d%H%M%S)"
        rm -f "$BOLT_DB"
    fi
    systemctl start containerd
    sleep 5
    systemctl is-active containerd && log "✅ containerd 修復成功" || { log "❌ containerd 修復失敗"; exit 1; }
else
    log "✅ containerd 已 active"
fi

# ──────────────────────────────────────────────
# STEP 2: Docker 修復（若損壞）
# ──────────────────────────────────────────────
log "[2/7] 檢查 Docker..."

if ! systemctl is-active docker >/dev/null 2>&1; then
    log "Docker 未啟動，嘗試啟動..."
    systemctl start docker || true
    sleep 8
fi

if ! systemctl is-active docker >/dev/null 2>&1; then
    log "Docker 啟動失敗，修復 network BoltDB..."
    NETWORK_DB="/var/lib/docker/network/files/local-kv.db"
    if [ -f "$NETWORK_DB" ]; then
        if [ "$DOCKER_NETWORK_METADATA_REPAIR_ALLOWED" != "1" ]; then
            log "❌ Docker network metadata repair 未授權；禁止自動備份/刪除 local-kv.db"
            exit 1
        fi
        log "Docker network metadata repair 已授權；先備份再移除 local-kv.db..."
        cp "$NETWORK_DB" "${NETWORK_DB}.bak.$(date +%Y%m%d%H%M%S)"
        rm -f "$NETWORK_DB"
    fi
    systemctl restart containerd
    sleep 5
    systemctl start docker
    sleep 8
    systemctl is-active docker && log "✅ Docker 修復成功" || { log "❌ Docker 修復失敗"; exit 1; }
else
    log "✅ Docker 已 active"
fi

# ──────────────────────────────────────────────
# STEP 3: PostgreSQL 啟動檢查（資料層錯誤必須人工 gate）
# ──────────────────────────────────────────────
log "[3/7] 檢查 PostgreSQL..."

if ! postgres_runtime_ready; then
    log "PostgreSQL 未啟動，嘗試啟動..."
    systemctl start postgresql@14-main || true
    sleep 8
fi

if ! postgres_runtime_ready; then
    log "PostgreSQL 啟動失敗，檢查是否屬於 checkpoint/WAL 類資料層錯誤..."
    if journalctl -u postgresql@14-main -n 20 | grep -q "could not locate a valid checkpoint"; then
        log "❌ 偵測到 PostgreSQL checkpoint/WAL 錯誤；禁止自動 pg_resetwal。"
        log "需要 DB owner、備份/restore evidence、maintenance window 與 post-check 後才能人工處理。"
        exit 1
    fi
    postgres_runtime_ready && log "✅ PostgreSQL runtime 可用" || { log "❌ PostgreSQL 修復失敗"; exit 1; }
fi

# 等待 PG 接受連線
log "等待 PostgreSQL 就緒..."
for i in $(seq 1 30); do
    pg_isready -h localhost -p 5432 >/dev/null 2>&1 && break
    sleep 2
done
pg_isready -h localhost -p 5432 && log "✅ PostgreSQL accepting connections" || { log "❌ PostgreSQL 無法接受連線"; exit 1; }

# kine 維護不屬於開機恢復的必要寫入，預設停用。
if [ "$KINE_MAINTENANCE_ALLOWED" = "1" ]; then
    log "kine maintenance 已授權，執行 bounded stale-session cleanup / VACUUM..."
    sudo -u postgres psql -d k3s_datastore -c "
        SELECT pg_terminate_backend(pid)
        FROM pg_stat_activity
        WHERE datname='k3s_datastore' AND pid!=pg_backend_pid() AND state='active'
        AND query_start < now() - interval '5 minutes';
    " 2>/dev/null || true
    sudo -u postgres psql -d k3s_datastore -c "VACUUM ANALYZE kine;" 2>/dev/null \
        && log "✅ kine VACUUM ANALYZE 完成" \
        || log "⚠️  kine VACUUM 未完成"
else
    log "✅ kine maintenance 預設停用，開機恢復不寫入資料庫"
fi

# ──────────────────────────────────────────────
# STEP 4: Redis
# ──────────────────────────────────────────────
log "[4/7] 啟動 Redis..."
systemctl start redis-server || true
sleep 3
redis-cli ping 2>/dev/null | grep -q PONG && log "✅ Redis UP" || log "⚠️  Redis 可能未就緒"

# ──────────────────────────────────────────────
# STEP 5: Ollama
# ──────────────────────────────────────────────
log "[5/7] 啟動 Ollama..."
systemctl start ollama || true
# Ollama 載入模型需要時間，不立刻驗證

# ──────────────────────────────────────────────
# STEP 6: Nginx
# ──────────────────────────────────────────────
log "[6/7] 啟動 Nginx..."
systemctl start nginx || true
systemctl is-active nginx >/dev/null 2>&1 && log "✅ Nginx UP" || log "⚠️  Nginx 未就緒"

# ──────────────────────────────────────────────
# STEP 7: Docker Compose 服務
# ──────────────────────────────────────────────
log "[7/7] 啟動 Docker Compose 服務..."

# SignOz
SIGNOZ_DIR="/home/ollama/signoz/deploy/docker"
if [ "$SIGNOZ_188_START_ALLOWED" != "1" ]; then
    log "✅ 188 legacy SignOz startup 預設停用；正式 SignOz 由 110 runtime 提供"
elif [ -f "$SIGNOZ_DIR/docker-compose.yaml" ]; then
    log "啟動 SignOz..."
    cd "$SIGNOZ_DIR"
    docker compose up -d 2>&1 | tail -3
    log "✅ SignOz 啟動指令已發送"
else
    log "⚠️  找不到 SignOz compose 檔案"
fi

# MinIO（Velero 備份存儲）
# 2026-04-05 Claude Code: 加入 MinIO — 解決 Velero BSL Unavailable 問題
MINIO_DIR="/home/ollama/minio"
if [ -f "$MINIO_DIR/docker-compose.yml" ]; then
    log "啟動 MinIO (Velero 備份存儲)..."
    cd "$MINIO_DIR"
    docker compose up -d 2>&1 | tail -3
    log "✅ MinIO 啟動指令已發送"
else
    log "⚠️  找不到 MinIO compose 檔案"
fi

# ClawBot（依賴 aiops-network）
# 確保 aiops-network 存在（重開機後 external network 需手動建立）
if ! docker network inspect aiops-network >/dev/null 2>&1; then
    log "建立 aiops-network..."
    docker network create aiops-network && log "✅ aiops-network 建立" || log "⚠️ aiops-network 建立失敗"
fi

CLAWBOT_DIR="/home/ollama/clawbot-v5"
if [ -f "$CLAWBOT_DIR/docker-compose.yml" ]; then
    log "啟動 ClawBot..."
    cd "$CLAWBOT_DIR"
    if ! run_bounded "$CLAWBOT_UP_TIMEOUT_SECONDS" docker compose up -d 2>&1 | tee /tmp/clawbot-up.log | grep -q "Started\|Running\|healthy"; then
        log "⚠️  ClawBot 啟動未在 ${CLAWBOT_UP_TIMEOUT_SECONDS}s 內確認；交由 post-check 判定"
        if [ "$CLAWBOT_STARTUP_REBUILD_ALLOWED" = "1" ]; then
            log "ClawBot bounded rebuild 已允許，最多等待 ${CLAWBOT_BUILD_TIMEOUT_SECONDS}s..."
            run_bounded "$CLAWBOT_BUILD_TIMEOUT_SECONDS" docker compose build 2>&1 | tail -5 || log "⚠️  ClawBot bounded rebuild 未完成"
            run_bounded "$CLAWBOT_UP_TIMEOUT_SECONDS" docker compose up -d 2>&1 | tail -3 || log "⚠️  ClawBot rebuild 後啟動仍未確認"
        else
            log "⚠️  ClawBot rebuild 預設停用，避免拖垮 10 分鐘 post-reboot SLO"
        fi
    fi
    if docker inspect -f '{{.State.Running}}' clawbot 2>/dev/null | grep -q '^true$'; then
        log "✅ ClawBot container running"
    else
        log "⚠️  ClawBot container 未確認 running；post-reboot verifier 會標示服務 blocker"
    fi
    log "✅ ClawBot 啟動指令已發送"
else
    log "⚠️  找不到 ClawBot compose 檔案"
fi

# ──────────────────────────────────────────────
# 完成
# ──────────────────────────────────────────────
log "=== AWOOOI 啟動序列完成 ==="
# 最後永遠 exit 0，不讓 systemd 認為失敗
log "K3s 需在 120/121 手動確認啟動（或由 k3s.service 自動啟動）"
log "詳細 SOP: docs/runbooks/REBOOT-RECOVERY-SOP.md"

exit 0