#!/bin/bash
# =============================================================================
# WOOO AIOps - SignOz 備份腳本 (ClickHouse + SQLite)
# 版本: 1.3.0
# 建立日期: 2026-04-05
# 2026-04-05 Claude Code: 新增 SignOz 分散式追蹤備份 — 首席架構師備份審計
# 2026-04-05 Claude Code: v1.1 修正 tar pipeline exit code 處理 + || true
# 2026-07-15 Codex: v1.2 確保任何 exit/signal/failure 都會受控還原 OTEL Collector
# 2026-07-15 Codex: v1.3 加入初始狀態、全 run 有界還原與 success 前獨立 verifier
# =============================================================================

set -euo pipefail

source "$(dirname "$0")/common.sh"

SERVICE="signoz"
LOCAL_REPO="${BACKUP_BASE}/signoz"
DUMP_DIR="/tmp/signoz-backup-$$"
COLLECTOR_NAME="signoz-otel-collector"
COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"
COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"
COLLECTOR_RESTORE_RETRY_DELAY_SECONDS="${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS:-2}"
COLLECTOR_WAS_RUNNING=0
COLLECTOR_RESTORE_PENDING=0
COLLECTOR_RESTORE_ATTEMPTS_USED=0
CLEANUP_COMPLETE=0
RESTORE_FAILURE_NOTIFIED=0

run_collector_docker() {
    timeout "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"
}

collector_running_state() {
    local running
    if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then
        return 1
    fi

    case "${running}" in
        true)
            printf '%s\n' "running"
            ;;
        false)
            printf '%s\n' "stopped"
            ;;
        *)
            return 1
            ;;
    esac
}

collector_is_running() {
    [ "$(collector_running_state)" = "running" ]
}

notify_restore_failure_once() {
    local detail="$1"
    if [ "${RESTORE_FAILURE_NOTIFIED}" -eq 1 ]; then
        return 0
    fi
    RESTORE_FAILURE_NOTIFIED=1
    notify_clawbot "failed" "${SERVICE}" "${detail}" || true
}

restore_collector() {
    if [ "${COLLECTOR_RESTORE_PENDING}" -ne 1 ]; then
        return 0
    fi

    while [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; do
        COLLECTOR_RESTORE_ATTEMPTS_USED=$((COLLECTOR_RESTORE_ATTEMPTS_USED + 1))
        log_info "重啟 ${COLLECTOR_NAME} (${COLLECTOR_RESTORE_ATTEMPTS_USED}/${COLLECTOR_RESTORE_MAX_ATTEMPTS})..."

        if run_collector_docker start "${COLLECTOR_NAME}" >/dev/null 2>&1 \
            && collector_is_running; then
            COLLECTOR_RESTORE_PENDING=0
            return 0
        fi

        log_warn "${COLLECTOR_NAME} 重啟或 running readback 失敗"
        if [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; then
            sleep "${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS}"
        fi
    done

    log_error "${COLLECTOR_NAME} 在有界重試後仍無法恢復"
    return 1
}

verify_collector_final_state() {
    if [ "${COLLECTOR_WAS_RUNNING}" -ne 1 ]; then
        log_info "${COLLECTOR_NAME} 原本即為 stopped，不執行自動啟動"
        return 0
    fi

    if collector_is_running; then
        log_success "${COLLECTOR_NAME} final running verifier 通過"
        return 0
    fi

    log_warn "${COLLECTOR_NAME} final running verifier 首次失敗，使用剩餘有界還原額度"
    COLLECTOR_RESTORE_PENDING=1
    if ! restore_collector; then
        return 1
    fi

    # 與 docker start 的回傳值與 restore 內部 readback 分離，作為 success 前獨立 verifier。
    collector_is_running
}

cleanup() {
    local exit_code=$?

    if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then
        return "${exit_code}"
    fi
    CLEANUP_COMPLETE=1

    # cleanup 期間忽略重複 signal，避免還原與刪除流程重入。
    trap '' HUP INT TERM
    if ! restore_collector; then
        notify_restore_failure_once "SignOz 備份結束時無法恢復 ${COLLECTOR_NAME}"
    fi
    rm -rf "${DUMP_DIR}"
    return "${exit_code}"
}

on_signal() {
    local signal_number="$1"
    exit "$((128 + signal_number))"
}

backup_volume() {
    local volume_name="$1"
    local output_file="$2"
    local extra_exclude="${3:-}"
    log_info "備份 volume: ${volume_name}"
    # 使用 || true 處理 tar 備份運行中 volume 的 exit 1 警告
    if [ -n "${extra_exclude}" ]; then
        docker run --rm -v "${volume_name}:/data" alpine tar czf - "${extra_exclude}" /data 2>/dev/null > "${output_file}" || true
    else
        docker run --rm -v "${volume_name}:/data" alpine tar czf - /data 2>/dev/null > "${output_file}" || true
    fi
    if [ -s "${output_file}" ]; then
        local size=$(du -h "${output_file}" | cut -f1)
        log_success "  Volume ${volume_name} 備份完成 (${size})"
        return 0
    else
        log_error "  Volume ${volume_name} 備份失敗 (空檔案)"
        return 1
    fi
}

main() {
    local start_time=$(date +%s)

    trap cleanup EXIT
    trap 'on_signal 1' HUP
    trap 'on_signal 2' INT
    trap 'on_signal 15' TERM

    log_info "========== 開始 SignOz 備份 =========="
    mkdir -p "${DUMP_DIR}"

    local timestamp=$(date "+%Y%m%d_%H%M%S")

    # Step 1: 只在 Collector 原本 running 時暫停，並在 stop 前 arm 還原。
    local collector_initial_state
    if ! collector_initial_state="$(collector_running_state)"; then
        log_error "無法在有界 timeout 內讀取 ${COLLECTOR_NAME} 初始狀態"
        notify_restore_failure_once "SignOz 備份無法驗證 ${COLLECTOR_NAME} 初始狀態"
        exit 1
    fi

    if [ "${collector_initial_state}" = "running" ]; then
        COLLECTOR_WAS_RUNNING=1
        COLLECTOR_RESTORE_PENDING=1
        log_info "暫停 ${COLLECTOR_NAME} 以確保數據一致性..."
        if ! run_collector_docker stop "${COLLECTOR_NAME}" >/dev/null 2>&1; then
            log_error "${COLLECTOR_NAME} 在有界 timeout 內無法停止"
            notify_restore_failure_once "SignOz 備份無法受控暫停 ${COLLECTOR_NAME}"
            exit 1
        fi
    else
        log_error "${COLLECTOR_NAME} 初始狀態為 stopped，停止備份且不自動啟動"
        notify_restore_failure_once "SignOz 備份發現 ${COLLECTOR_NAME} 原本已停止"
        exit 1
    fi
    docker stop signoz-telemetrystore-migrator 2>/dev/null || true

    # Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積)
    backup_volume "signoz-clickhouse" "${DUMP_DIR}/clickhouse_${timestamp}.tar.gz" "--exclude=/data/tmp" || {
        log_error "ClickHouse volume 備份失敗"
        notify_clawbot "failed" "${SERVICE}" "SignOz ClickHouse 備份失敗"
        exit 1
    }

    # Step 3: 備份 SQLite volume (SignOz metadata)
    backup_volume "signoz-sqlite" "${DUMP_DIR}/sqlite_${timestamp}.tar.gz" || {
        log_warn "SQLite volume 備份失敗，繼續..."
    }

    # Step 4: 在全 run 共用的有界 retry budget 內恢復 Collector。
    if ! restore_collector; then
        notify_restore_failure_once "SignOz 備份無法恢復 ${COLLECTOR_NAME}"
        exit 1
    fi

    # Step 5: 初始化 Restic 倉庫
    if [ ! -d "${LOCAL_REPO}/data" ]; then
        log_info "初始化 Restic 倉庫: ${LOCAL_REPO}"
        restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
            log_error "Restic 倉庫初始化失敗"
            exit 1
        }
    fi

    # Step 6: Restic 備份
    log_info "建立 Restic 備份..."
    local tags=$(build_tags "${SERVICE}")
    restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}"         --password-file "${RESTIC_PASSWORD_FILE}"         ${tags} 2>&1

    local snapshot_id=$(restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | grep -oP '"short_id":"\K[^"]+' | head -1)
    log_success "Restic 備份完成: ${snapshot_id}"

    # Step 7: GFS 清理
    cleanup_old_backups "${LOCAL_REPO}"

    # Step 8: success 前獨立檢查 runtime state；失敗時不得發送 success。
    if ! verify_collector_final_state; then
        notify_restore_failure_once "SignOz 備份完成前 ${COLLECTOR_NAME} final running verifier 失敗"
        exit 1
    fi

    local end_time=$(date +%s)
    local duration=$((end_time - start_time))
    log_success "========== SignOz 備份完成 (${duration}s) =========="
    notify_clawbot "success" "${SERVICE}" "SignOz 備份完成 (ClickHouse+SQLite)" "${duration}"
}

main "$@"