#!/bin/bash
# =============================================================================
# WOOO AIOps - SigNoz ClickHouse native backup orchestrator
# Version: 2.0.0
#
# ClickHouse is backed up online through its native BACKUP engine.  This script
# never reads or archives the live ClickHouse/SQLite Docker volumes.  SigNoz
# metadata/SQLite remains an explicit application-consistent export gap and is
# recorded in the Restic payload without being misreported as completed.
# =============================================================================

set -euo pipefail

source "$(dirname "$0")/common.sh"

SERVICE="signoz"
LOCAL_REPO="${BACKUP_BASE}/signoz"
TMP_ROOT="${SIGNOZ_BACKUP_TMP_ROOT:-/tmp}"
DUMP_DIR=""
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CLICKHOUSE_NATIVE_BACKUP_SCRIPT="${CLICKHOUSE_NATIVE_BACKUP_SCRIPT:-${SCRIPT_DIR}/clickhouse-native-backup.sh}"
COLLECTOR_NAME="${SIGNOZ_COLLECTOR_NAME:-signoz-otel-collector}"
COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"
COLLECTOR_CONTINUITY_POLL_SECONDS="${COLLECTOR_CONTINUITY_POLL_SECONDS:-1}"
TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"
# Destructive Restic forget/prune is a separate critical break-glass workflow.
# The regular and canary backup paths are backup-only and fail closed if a
# caller attempts to re-enable inline retention cleanup.
BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-1}"
OPERATION_LOCK_FILE="${SIGNOZ_BACKUP_OPERATION_LOCK_FILE:-/tmp/awoooi-signoz-backup-operation.lock}"
RESTIC_RECEIPT_ROOT="${SIGNOZ_BACKUP_RECEIPT_ROOT:-/backup/logs/signoz-backup}"
RESTIC_INIT_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_INIT_TIMEOUT_SECONDS:-300}"
RESTIC_BACKUP_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_BACKUP_TIMEOUT_SECONDS:-1800}"
RESTIC_READBACK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_READBACK_TIMEOUT_SECONDS:-120}"
RESTIC_CHECK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_CHECK_TIMEOUT_SECONDS:-1800}"

BACKUP_ID_SEED="$(date -u '+%Y%m%dT%H%M%SZ')-$$"
TRACE_ID="${TRACE_ID:-signoz-backup-${BACKUP_ID_SEED}}"
RUN_ID="${RUN_ID:-signoz-${BACKUP_ID_SEED}}"
WORK_ITEM_ID="${WORK_ITEM_ID:-SIGNOZ-BACKUP}"
export TRACE_ID RUN_ID WORK_ITEM_ID

COLLECTOR_ID_BEFORE=""
COLLECTOR_STARTED_AT_BEFORE=""
COLLECTOR_RESTARTS_BEFORE=""
COLLECTOR_HEALTH_BEFORE=""
COLLECTOR_OBSERVER_PID=""
COLLECTOR_OBSERVATIONS=""
WORKSPACE_CREATED=0
CLEANUP_COMPLETE=0
FAILURE_NOTIFIED=0

run_collector_docker() {
    timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
        "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"
}

collector_identity_state() {
    run_collector_docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \
        "${COLLECTOR_NAME}" 2>/dev/null
}

notify_failure_once() {
    local detail="$1"
    if [ "${FAILURE_NOTIFIED}" -eq 1 ]; then
        return 0
    fi
    FAILURE_NOTIFIED=1
    notify_clawbot "failed" "${SERVICE}" "${detail}" || true
}

observe_collector_continuity() {
    local epoch observation

    while true; do
        epoch="$(date +%s)"
        if observation="$(collector_identity_state)"; then
            printf '%s\t%s\n' "${epoch}" "${observation}" >> "${COLLECTOR_OBSERVATIONS}"
        else
            printf '%s\tunknown\tunknown\tunknown\tunknown\tunknown\n' "${epoch}" >> "${COLLECTOR_OBSERVATIONS}"
        fi
        sleep "${COLLECTOR_CONTINUITY_POLL_SECONDS}"
    done
}

start_collector_observer() {
    : > "${COLLECTOR_OBSERVATIONS}"
    observe_collector_continuity &
    COLLECTOR_OBSERVER_PID=$!
}

stop_collector_observer() {
    if [ -n "${COLLECTOR_OBSERVER_PID}" ] \
        && kill -0 "${COLLECTOR_OBSERVER_PID}" 2>/dev/null; then
        kill -TERM "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true
        wait "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true
    fi
    COLLECTOR_OBSERVER_PID=""
}

verify_collector_continuity() {
    local observation final_id final_running final_started_at final_restarts final_health

    stop_collector_observer
    [ -s "${COLLECTOR_OBSERVATIONS}" ] || {
        log_error "${COLLECTOR_NAME} continuity receipt 為空"
        return 1
    }
    if awk -F '\t' -v expected_id="${COLLECTOR_ID_BEFORE}" \
        -v expected_started="${COLLECTOR_STARTED_AT_BEFORE}" \
        -v expected_restarts="${COLLECTOR_RESTARTS_BEFORE}" \
        -v expected_health="${COLLECTOR_HEALTH_BEFORE}" \
        '$2 == "unknown" || $3 != "true" || $2 != expected_id || \
         $4 != expected_started || $5 != expected_restarts || $6 != expected_health { exit 1 }' \
        "${COLLECTOR_OBSERVATIONS}"; then
        :
    else
        log_error "${COLLECTOR_NAME} 在 native backup window 發生停止或 identity drift"
        return 1
    fi

    observation="$(collector_identity_state)" || return 1
    IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${observation}"
    [ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \
        && [ "${final_running}" = "true" ] \
        && [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \
        && [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \
        && [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ]
}

cleanup() {
    local exit_code=$?

    if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then
        exit "${exit_code}"
    fi
    CLEANUP_COMPLETE=1
    trap - EXIT HUP INT TERM
    set +e
    stop_collector_observer
    if [ "${WORKSPACE_CREATED}" -eq 1 ]; then
        if [ -n "${DUMP_DIR}" ] && [ -d "${DUMP_DIR}" ] \
            && [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \
            && [[ "${DUMP_DIR}" == "${TMP_ROOT%/}"/signoz-backup.* ]]; then
            rm -rf -- "${DUMP_DIR}" || exit_code=90
            [ ! -e "${DUMP_DIR}" ] || exit_code=90
        else
            log_error "拒絕清理不符合本 run identity 的 SigNoz workspace"
            exit_code=90
        fi
    fi
    exit "${exit_code}"
}

on_signal() {
    local signal_number="$1"
    exit "$((128 + signal_number))"
}

create_private_workspace() {
    [[ "${TMP_ROOT}" == /* ]] \
        && [[ ! "${TMP_ROOT}" =~ (^|/)\.\.?(/|$) ]] \
        && [[ "${TMP_ROOT}" != *$'\n'* ]] \
        && [ -d "${TMP_ROOT}" ] && [ -w "${TMP_ROOT}" ] \
        && [ ! -L "${TMP_ROOT}" ] || {
        log_error "SigNoz workspace root 必須是安全、可寫、非 symlink 的絕對目錄"
        return 1
    }

    DUMP_DIR="$(mktemp -d "${TMP_ROOT%/}/signoz-backup.XXXXXXXX")" || return 1
    WORKSPACE_CREATED=1
    [ -d "${DUMP_DIR}" ] && [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \
        && [ -r "${DUMP_DIR}" ] && [ -w "${DUMP_DIR}" ] && [ -x "${DUMP_DIR}" ] || {
        log_error "mktemp 未建立本 run 擁有的 private SigNoz workspace"
        return 1
    }
    chmod 700 "${DUMP_DIR}" || return 1
    COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv"
    log_info "建立 SigNoz private workspace: ${DUMP_DIR}"
}

run_native_clickhouse_backup() {
    local mode="$1"

    [ -f "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
        && [ -x "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
        && [ ! -L "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
        || {
            log_error "ClickHouse native backup adapter 缺失、不可執行或為 symlink"
            return 1
        }

    CLICKHOUSE_NATIVE_OUTPUT_DIR="${DUMP_DIR}" \
        TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
        "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" "${mode}"
}

write_metadata_gap_receipt() {
    cat > "${DUMP_DIR}/signoz-metadata-gap.json" <<EOF
{"trace_id":"${TRACE_ID}","run_id":"${RUN_ID}","work_item_id":"${WORK_ITEM_ID}","asset":"signoz-metadata-sqlite","terminal":"pending_application_consistent_export","raw_volume_read":false,"raw_volume_archive":false,"completion_claim":false}
EOF
    log_warn "SigNoz metadata/SQLite 尚缺 application-consistent export；本 run 未讀取 live volume"
}

write_restic_receipt() {
    local snapshot_id="$1"
    local artifact_hash="$2"
    local manifest_hash="$3"
    local inventory_hash="$4"
    local receipt_dir receipt_tmp receipt_file

    [[ "${RESTIC_RECEIPT_ROOT}" == /* ]] && [ ! -L "${RESTIC_RECEIPT_ROOT}" ] || return 1
    mkdir -p "${RESTIC_RECEIPT_ROOT}"
    receipt_dir="${RESTIC_RECEIPT_ROOT}/${RUN_ID}"
    [ ! -L "${receipt_dir}" ] || return 1
    mkdir -m 700 -p "${receipt_dir}"
    receipt_file="${receipt_dir}/restic-snapshot.json"
    receipt_tmp="${receipt_file}.partial.$$"
    printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","service":"signoz","snapshot_id":"%s","artifact_sha256":"%s","manifest_sha256":"%s","source_inventory_sha256":"%s","repository_scope":"local_same_failure_domain","restic_check_read_data_subset":"1%%","offsite_verified":false,"metadata_sqlite_terminal":"pending_application_consistent_export","completion_claim":"clickhouse_native_only"}\n' \
        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${snapshot_id}" \
        "${artifact_hash}" "${manifest_hash}" "${inventory_hash}" > "${receipt_tmp}"
    chmod 600 "${receipt_tmp}"
    mv "${receipt_tmp}" "${receipt_file}"
}

acquire_operation_lock() {
    case "${OPERATION_LOCK_FILE}" in
        /*) ;;
        *)
            log_error "SigNoz operation lock path 必須是絕對路徑"
            return 1
            ;;
    esac
    [ ! -L "${OPERATION_LOCK_FILE}" ] || {
        log_error "SigNoz operation lock 不得是 symlink"
        return 1
    }
    if [ -e "${OPERATION_LOCK_FILE}" ]; then
        [ -f "${OPERATION_LOCK_FILE}" ] || {
            log_error "SigNoz operation lock 必須是 regular file"
            return 1
        }
    fi
    exec 8>>"${OPERATION_LOCK_FILE}"
    flock -n 8 || {
        log_error "另一個 SigNoz backup/deploy operation 正在執行"
        return 1
    }
    log_info "取得 SigNoz backup/deploy 共用 operation lock"
}

main() {
    local start_time end_time duration snapshot_id snapshot_json tags
    local initial_observation initial_running final_id final_running
    local final_started_at final_restarts final_health
    local artifact_file manifest_file inventory_file artifact_hash manifest_hash inventory_hash

    start_time="$(date +%s)"
    trap cleanup EXIT
    trap 'on_signal 1' HUP
    trap 'on_signal 2' INT
    trap 'on_signal 15' TERM

    log_info "========== 開始 SigNoz ClickHouse native 備份 =========="
    [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" =~ ^[0-9]+([.][0-9]+)?$ ]] \
        && [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0" ]] \
        && [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0.0" ]] || {
        notify_failure_once "COLLECTOR_CONTINUITY_POLL_SECONDS 無效"
        exit 1
    }
    for timeout_value in "${RESTIC_INIT_TIMEOUT_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \
        "${RESTIC_READBACK_TIMEOUT_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}"; do
        [[ "${timeout_value}" =~ ^[0-9]+$ ]] && [ "${timeout_value}" -gt 0 ] || {
            notify_failure_once "Restic timeout contract 無效"
            exit 1
        }
    done
    [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ] || {
        notify_failure_once "inline Restic forget/prune 已停用；僅允許獨立 critical break-glass workflow"
        exit 1
    }
    acquire_operation_lock || {
        notify_failure_once "SigNoz backup/deploy operation lock 取得失敗"
        exit 1
    }
    create_private_workspace || {
        notify_failure_once "建立 private SigNoz backup workspace 失敗"
        exit 1
    }

    # Check-mode must pass before the backup window. It writes only durable
    # receipts and does not submit BACKUP or create a server artifact.
    if ! run_native_clickhouse_backup --check; then
        log_error "ClickHouse native BACKUP check-mode 失敗；沒有 volume tar fallback"
        notify_failure_once "SigNoz ClickHouse native BACKUP runtime contract 未就緒"
        exit 1
    fi

    initial_observation="$(collector_identity_state)" || {
        notify_failure_once "無法讀取 ${COLLECTOR_NAME} 初始 identity/state"
        exit 1
    }
    IFS=$'\t' read -r COLLECTOR_ID_BEFORE initial_running COLLECTOR_STARTED_AT_BEFORE \
        COLLECTOR_RESTARTS_BEFORE COLLECTOR_HEALTH_BEFORE <<< "${initial_observation}"
    [ -n "${COLLECTOR_ID_BEFORE}" ] && [ "${initial_running}" = "true" ] || {
        notify_failure_once "${COLLECTOR_NAME} 初始狀態不是 running"
        exit 1
    }
    [ -n "${COLLECTOR_STARTED_AT_BEFORE}" ] \
        && [[ "${COLLECTOR_RESTARTS_BEFORE}" =~ ^[0-9]+$ ]] \
        && [[ "${COLLECTOR_HEALTH_BEFORE}" =~ ^(healthy|none)$ ]] || {
        notify_failure_once "${COLLECTOR_NAME} 初始 runtime metadata 不完整"
        exit 1
    }
    start_collector_observer

    # Online native backup: no collector stop and no live-volume read.
    if ! run_native_clickhouse_backup --apply; then
        notify_failure_once "SigNoz ClickHouse native BACKUP 失敗"
        exit 1
    fi

    if ! verify_collector_continuity; then
        notify_failure_once "${COLLECTOR_NAME} no-downtime continuity verifier 失敗"
        exit 1
    fi
    log_success "${COLLECTOR_NAME} no-downtime continuity verifier 通過"

    write_metadata_gap_receipt

    artifact_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip' -print)"
    manifest_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip.manifest.json' -print)"
    inventory_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.source-inventory.tsv' -print)"
    [ "$(printf '%s\n' "${artifact_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \
        && [ "$(printf '%s\n' "${manifest_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \
        && [ "$(printf '%s\n' "${inventory_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] || {
        log_error "ClickHouse native artifact set 不唯一或不完整"
        exit 1
    }
    artifact_hash="$(sha256sum "${artifact_file}" | awk '{print $1}')"
    manifest_hash="$(sha256sum "${manifest_file}" | awk '{print $1}')"
    inventory_hash="$(sha256sum "${inventory_file}" | awk '{print $1}')"

    if [ ! -d "${LOCAL_REPO}/data" ]; then
        log_info "初始化 Restic 倉庫: ${LOCAL_REPO}"
        timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_INIT_TIMEOUT_SECONDS}" \
            restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
            log_error "Restic 倉庫初始化失敗"
            exit 1
        }
    fi

    log_info "建立 ClickHouse native artifact 的 Restic 備份..."
    tags="$(build_tags "${SERVICE}")"
    timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \
        restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" \
            --password-file "${RESTIC_PASSWORD_FILE}" ${tags} \
            --tag "trace:${TRACE_ID}" --tag "run:${RUN_ID}" \
            --tag "work-item:${WORK_ITEM_ID}" 2>&1

    snapshot_json="$(timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
        "${RESTIC_READBACK_TIMEOUT_SECONDS}" restic -r "${LOCAL_REPO}" snapshots \
        --tag "run:${RUN_ID}" --latest 1 --json \
        --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null)"
    snapshot_id="$(printf '%s\n' "${snapshot_json}" \
        | sed -n 's/.*"short_id":"\([^"]*\)".*/\1/p' | head -1)"
    [[ "${snapshot_id}" =~ ^[A-Za-z0-9]+$ ]] || {
        log_error "Restic same-run snapshot durable readback 缺失"
        exit 1
    }
    timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}" \
        restic -r "${LOCAL_REPO}" check --read-data-subset=1% \
            --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
        log_error "Restic repository/data-subset verifier 失敗"
        exit 1
    }
    write_restic_receipt "${snapshot_id}" "${artifact_hash}" \
        "${manifest_hash}" "${inventory_hash}" || {
        log_error "Restic durable receipt writeback 失敗"
        exit 1
    }
    log_success "Restic 備份完成: ${snapshot_id}"

    log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)"

    # Final readback is separate from the continuity observer receipt.
    initial_observation="$(collector_identity_state)" || {
        notify_failure_once "${COLLECTOR_NAME} final readback 失敗"
        exit 1
    }
    IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${initial_observation}"
    [ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \
        && [ "${final_running}" = "true" ] \
        && [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \
        && [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \
        && [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ] || {
        notify_failure_once "${COLLECTOR_NAME} final identity/start/restart/health verifier 失敗"
        exit 1
    }

    end_time="$(date +%s)"
    duration=$((end_time - start_time))
    log_success "========== SigNoz ClickHouse native 備份完成 (${duration}s) =========="
    notify_clawbot "warning" "${SERVICE}" \
        "ClickHouse native backup 完成；metadata/SQLite application-consistent export 仍 pending" \
        "${duration}"
}

main "$@"