#!/bin/bash
# =============================================================================
# WOOO AIOps - Gitea 備份腳本
# 版本: 1.1.0
# 建立日期: 2026-03-12
# 2026-05-19 ogt + Codex: 納入 repo/Ansible；離機上傳改由 sync-offsite-backups.sh 統一管控。
# =============================================================================

set -euo pipefail
umask 077

source "$(dirname "$0")/common.sh"

SERVICE="gitea"
GITEA_CONTAINER="gitea"
LOCAL_REPO="${BACKUP_BASE}/gitea"
DUMP_DIR="/tmp/gitea-backup-$$"
TEXTFILE_DIR="${NODE_EXPORTER_TEXTFILE_DIR:-/home/wooo/node_exporter_textfiles}"
TEXTFILE_PATH="${GITEA_FULL_BACKUP_RECEIPT_TEXTFILE:-${TEXTFILE_DIR}/gitea_full_backup_receipt.prom}"
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
GITEA_FULL_BACKUP_COMPONENTS="database repositories app_settings lfs_objects package_registry attachments actions_logs actions_artifacts avatars hooks_custom_assets"
GITEA_RESTORE_DRILL="$(dirname "$0")/gitea-full-backup-restore-drill.sh"
GITEA_BACKUP_LOCK="${GITEA_BACKUP_LOCK:-/tmp/gitea-backup.lock}"
GITEA_STOP_TIMEOUT_SECONDS="${GITEA_STOP_TIMEOUT_SECONDS:-15}"
GITEA_DUMP_TIMEOUT_SECONDS="${GITEA_DUMP_TIMEOUT_SECONDS:-360}"
GITEA_DUMP_KILL_AFTER_SECONDS="${GITEA_DUMP_KILL_AFTER_SECONDS:-10}"
GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS="${GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS:-10}"
GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS="${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS:-3}"
GITEA_START_TIMEOUT_SECONDS="${GITEA_START_TIMEOUT_SECONDS:-60}"
GITEA_MAX_OUTAGE_SECONDS="${GITEA_MAX_OUTAGE_SECONDS:-600}"
GITEA_LOCAL_HEALTH_URL="${GITEA_LOCAL_HEALTH_URL:-http://127.0.0.1:3001/api/healthz}"
TRANSIENT_CONTAINER=""
PRIMARY_STOPPED=0
PRIMARY_RESTART_VERIFIED=0
OUTAGE_STARTED_AT=0
OUTAGE_SECONDS=0

label_escape() {
    printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'
}

write_gitea_full_backup_receipt() {
    local status="$1"
    local timestamp="$2"
    local duration="$3"
    local snapshot_id="${4:-unknown}"
    local consistent_offline_window="${5:-0}"
    local service_restart_verified="${6:-0}"
    local outage_seconds="${7:-0}"
    local structural_drill_performed="${8:-0}"
    local tmp
    local host
    local service_label
    local status_label
    local component
    local ok=0
    local failed=1

    if [ "${status}" = "success" ]; then
        ok=1
        failed=0
    fi

    host="$(label_escape "${HOST_LABEL}")"
    service_label="$(label_escape "${SERVICE}")"
    status_label="$(label_escape "${status}")"

    install -d -m 755 "${TEXTFILE_DIR}" || return 0
    tmp="$(mktemp "${TEXTFILE_PATH}.tmp.XXXXXX")" || return 0
    {
        echo "# HELP awoooi_gitea_full_backup_last_success_timestamp Unix timestamp of the latest successful Gitea full-server backup receipt."
        echo "# TYPE awoooi_gitea_full_backup_last_success_timestamp gauge"
        echo "# HELP awoooi_gitea_full_backup_last_run_failed Whether the latest Gitea full-server backup run failed."
        echo "# TYPE awoooi_gitea_full_backup_last_run_failed gauge"
        echo "# HELP awoooi_gitea_full_backup_duration_seconds Duration of the latest Gitea full-server backup run."
        echo "# TYPE awoooi_gitea_full_backup_duration_seconds gauge"
        echo "# HELP awoooi_gitea_full_backup_dump_file_present Whether the Gitea dump file was produced before restic backup."
        echo "# TYPE awoooi_gitea_full_backup_dump_file_present gauge"
        echo "# HELP awoooi_gitea_full_backup_restic_snapshot_present Whether a restic snapshot id was recorded for the Gitea dump."
        echo "# TYPE awoooi_gitea_full_backup_restic_snapshot_present gauge"
        echo "# HELP awoooi_gitea_full_backup_component_receipt Whether the Gitea full-server dump includes a redacted component receipt."
        echo "# TYPE awoooi_gitea_full_backup_component_receipt gauge"
        echo "# HELP awoooi_gitea_full_backup_restore_drill_performed Whether a non-production Gitea full-backup restore drill receipt is present."
        echo "# TYPE awoooi_gitea_full_backup_restore_drill_performed gauge"
        echo "# HELP awoooi_gitea_full_backup_secret_value_collected Whether this receipt collected secret values."
        echo "# TYPE awoooi_gitea_full_backup_secret_value_collected gauge"
        echo "# HELP awoooi_gitea_full_backup_production_restore_performed Whether this receipt restored anything into production."
        echo "# TYPE awoooi_gitea_full_backup_production_restore_performed gauge"
        echo "# HELP awoooi_gitea_full_backup_consistent_offline_window Whether the dump was created while the primary Gitea service was stopped."
        echo "# TYPE awoooi_gitea_full_backup_consistent_offline_window gauge"
        echo "# HELP awoooi_gitea_full_backup_service_restart_verified Whether Gitea was independently reachable after the bounded offline window."
        echo "# TYPE awoooi_gitea_full_backup_service_restart_verified gauge"
        echo "# HELP awoooi_gitea_full_backup_service_outage_seconds Duration of the bounded Gitea offline backup window."
        echo "# TYPE awoooi_gitea_full_backup_service_outage_seconds gauge"
        echo "awoooi_gitea_full_backup_last_success_timestamp{host=\"${host}\",service=\"${service_label}\"} $([ "${ok}" = "1" ] && echo "${timestamp}" || echo 0)"
        echo "awoooi_gitea_full_backup_last_run_failed{host=\"${host}\",service=\"${service_label}\",status=\"${status_label}\"} ${failed}"
        echo "awoooi_gitea_full_backup_duration_seconds{host=\"${host}\",service=\"${service_label}\"} ${duration}"
        echo "awoooi_gitea_full_backup_dump_file_present{host=\"${host}\",service=\"${service_label}\"} ${ok}"
        echo "awoooi_gitea_full_backup_restic_snapshot_present{host=\"${host}\",service=\"${service_label}\",snapshot_id=\"$(label_escape "${snapshot_id}")\"} ${ok}"
        for component in ${GITEA_FULL_BACKUP_COMPONENTS}; do
            echo "awoooi_gitea_full_backup_component_receipt{host=\"${host}\",service=\"${service_label}\",component=\"$(label_escape "${component}")\",receipt=\"gitea_dump_restic\"} ${ok}"
        done
        echo "awoooi_gitea_full_backup_restore_drill_performed{host=\"${host}\",service=\"${service_label}\"} ${structural_drill_performed}"
        echo "awoooi_gitea_full_backup_secret_value_collected{host=\"${host}\",service=\"${service_label}\"} 0"
        echo "awoooi_gitea_full_backup_production_restore_performed{host=\"${host}\",service=\"${service_label}\"} 0"
        echo "awoooi_gitea_full_backup_consistent_offline_window{host=\"${host}\",service=\"${service_label}\"} ${consistent_offline_window}"
        echo "awoooi_gitea_full_backup_service_restart_verified{host=\"${host}\",service=\"${service_label}\"} ${service_restart_verified}"
        echo "awoooi_gitea_full_backup_service_outage_seconds{host=\"${host}\",service=\"${service_label}\"} ${outage_seconds}"
    } >"${tmp}"
    mv "${tmp}" "${TEXTFILE_PATH}" || return 0
    chmod 0644 "${TEXTFILE_PATH}" || true
}

cleanup() {
    local status=$?
    trap - EXIT HUP INT TERM
    if [ -n "${TRANSIENT_CONTAINER}" ]; then
        if ! remove_transient_container; then
            status=1
        fi
    fi
    if [ "${PRIMARY_STOPPED}" -eq 1 ]; then
        if ! start_and_verify_primary; then
            status=1
        fi
    fi
    rm -rf "${DUMP_DIR}"
    exit "${status}"
}

validate_positive_integer() {
    case "$2" in
        ''|*[!0-9]*|0) log_error "$1 must be a positive integer"; return 64 ;;
    esac
}

require_commands() {
    local command_name configured_outage_bound
    for command_name in curl docker flock python3 restic timeout; do
        if ! command -v "${command_name}" >/dev/null 2>&1; then
            log_error "Required command missing: ${command_name}"
            return 69
        fi
    done
    [ -x "${GITEA_RESTORE_DRILL}" ] || {
        log_error "Gitea structural restore verifier is unavailable"
        return 69
    }
    [ -r "${RESTIC_PASSWORD_FILE}" ] || {
        log_error "Restic password reference is unavailable"
        return 69
    }
    validate_positive_integer GITEA_STOP_TIMEOUT_SECONDS "${GITEA_STOP_TIMEOUT_SECONDS}"
    validate_positive_integer GITEA_DUMP_TIMEOUT_SECONDS "${GITEA_DUMP_TIMEOUT_SECONDS}"
    validate_positive_integer GITEA_DUMP_KILL_AFTER_SECONDS "${GITEA_DUMP_KILL_AFTER_SECONDS}"
    validate_positive_integer GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS "${GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS}"
    validate_positive_integer GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS "${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS}"
    validate_positive_integer GITEA_START_TIMEOUT_SECONDS "${GITEA_START_TIMEOUT_SECONDS}"
    validate_positive_integer GITEA_MAX_OUTAGE_SECONDS "${GITEA_MAX_OUTAGE_SECONDS}"

    configured_outage_bound=$((
        GITEA_STOP_TIMEOUT_SECONDS
        + GITEA_DUMP_TIMEOUT_SECONDS
        + GITEA_DUMP_KILL_AFTER_SECONDS
        + GITEA_START_TIMEOUT_SECONDS
        + (4 * GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS)
        + (4 * GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS)
    ))
    if [ "${configured_outage_bound}" -gt "${GITEA_MAX_OUTAGE_SECONDS}" ]; then
        log_error "Configured Gitea backup outage bound exceeds ${GITEA_MAX_OUTAGE_SECONDS}s"
        return 64
    fi
}

bounded_docker_control() {
    timeout --signal=TERM \
        --kill-after="${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS}s" \
        "${GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS}s" \
        docker "$@"
}

remove_transient_container() {
    local transient="${TRANSIENT_CONTAINER}"
    local remove_status=0

    [ -n "${transient}" ] || return 0
    if ! bounded_docker_control rm -f "${transient}" >/dev/null 2>&1; then
        log_error "Gitea one-shot backup container removal timed out or failed"
        remove_status=1
    fi
    TRANSIENT_CONTAINER=""
    return "${remove_status}"
}

wait_for_primary_health() {
    local deadline=$((SECONDS + GITEA_START_TIMEOUT_SECONDS))
    local health_body
    while [ "${SECONDS}" -lt "${deadline}" ]; do
        if [ "$(bounded_docker_control inspect -f '{{.State.Running}}' "${GITEA_CONTAINER}" 2>/dev/null || true)" = "true" ]; then
            health_body="$(curl -fsS --max-time 5 "${GITEA_LOCAL_HEALTH_URL}" 2>/dev/null || true)"
            if printf '%s' "${health_body}" | grep -Eq '"status"[[:space:]]*:[[:space:]]*"pass"'; then
                return 0
            fi
        fi
        sleep 2
    done
    return 1
}

start_and_verify_primary() {
    local running

    running="$(bounded_docker_control inspect -f '{{.State.Running}}' "${GITEA_CONTAINER}" 2>/dev/null || true)"
    if [ "${running}" != "true" ]; then
        if ! bounded_docker_control start "${GITEA_CONTAINER}" >/dev/null; then
            log_error "Gitea failed to restart after the backup window"
            return 1
        fi
    fi
    if ! wait_for_primary_health; then
        log_error "Gitea post-backup health verification timed out"
        return 1
    fi
    PRIMARY_STOPPED=0
    PRIMARY_RESTART_VERIFIED=1
    if [ "${OUTAGE_STARTED_AT}" -gt 0 ]; then
        OUTAGE_SECONDS=$(($(date +%s) - OUTAGE_STARTED_AT))
    fi
    if [ "${OUTAGE_SECONDS}" -gt "${GITEA_MAX_OUTAGE_SECONDS}" ]; then
        log_error "Gitea backup outage exceeded ${GITEA_MAX_OUTAGE_SECONDS}s"
        return 1
    fi
    log_success "Gitea primary restarted and independently reachable (outage=${OUTAGE_SECONDS}s)"
}

offline_gitea_dump() {
    local image_id network_rows network_name network_count git_uid host_uid timestamp dump_status stop_status transient_cleanup_status

    [ "$(docker inspect -f '{{.State.Running}}' "${GITEA_CONTAINER}" 2>/dev/null || true)" = "true" ] || {
        log_error "Gitea primary is not running before backup"
        return 1
    }
    wait_for_primary_health || {
        log_error "Gitea primary is not reachable before backup"
        return 1
    }

    image_id="$(docker inspect -f '{{.Image}}' "${GITEA_CONTAINER}")"
    [[ "${image_id}" =~ ^sha256:[0-9a-f]{64}$ ]] || {
        log_error "Gitea image identity is invalid"
        return 1
    }
    network_rows="$(docker inspect -f '{{range $name, $value := .NetworkSettings.Networks}}{{$name}}{{"\n"}}{{end}}' "${GITEA_CONTAINER}" | sed '/^$/d')"
    network_count="$(printf '%s\n' "${network_rows}" | sed '/^$/d' | wc -l | tr -d ' ')"
    [ "${network_count}" -eq 1 ] || {
        log_error "Gitea backup requires exactly one fixed Docker network"
        return 1
    }
    network_name="$(printf '%s\n' "${network_rows}" | head -n 1)"
    [[ "${network_name}" =~ ^[A-Za-z0-9_.-]+$ ]] || {
        log_error "Gitea Docker network identity is invalid"
        return 1
    }
    git_uid="$(docker exec -u git "${GITEA_CONTAINER}" id -u)"
    host_uid="$(id -u)"
    [ "${git_uid}" = "${host_uid}" ] || {
        log_error "Gitea dump staging UID does not match the controller UID"
        return 1
    }

    timestamp="$(date +%Y%m%d%H%M%S)"
    TRANSIENT_CONTAINER="agent99-gitea-backup-${timestamp}-$$"
    OUTAGE_STARTED_AT="$(date +%s)"
    PRIMARY_STOPPED=1
    set +e
    timeout --signal=TERM \
        --kill-after="${GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS}s" \
        "$((GITEA_STOP_TIMEOUT_SECONDS + GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS))s" \
        docker stop --time "${GITEA_STOP_TIMEOUT_SECONDS}" "${GITEA_CONTAINER}" >/dev/null
    stop_status=$?
    set -e
    if [ "${stop_status}" -ne 0 ]; then
        log_error "Gitea primary could not enter the bounded offline backup window"
        start_and_verify_primary || true
        return 1
    fi

    set +e
    timeout --signal=TERM \
        --kill-after="${GITEA_DUMP_KILL_AFTER_SECONDS}s" \
        "${GITEA_DUMP_TIMEOUT_SECONDS}s" docker run --pull=never \
        --name "${TRANSIENT_CONTAINER}" \
        --network "${network_name}" \
        --volumes-from "${GITEA_CONTAINER}" \
        --mount "type=bind,src=${DUMP_DIR},dst=/backup-out" \
        --user "${git_uid}" \
        --workdir /backup-out \
        --cpus "${BACKUP_DOCKER_CPUS}" \
        --memory "${BACKUP_DOCKER_MEMORY}" \
        --memory-swap "${BACKUP_DOCKER_MEMORY_SWAP}" \
        --security-opt no-new-privileges:true \
        --entrypoint /usr/local/bin/gitea \
        "${image_id}" dump \
        -c /data/gitea/conf/app.ini \
        -f /backup-out/gitea-dump.zip \
        -t /tmp
    dump_status=$?
    set -e

    transient_cleanup_status=0
    remove_transient_container || transient_cleanup_status=$?

    if ! start_and_verify_primary; then
        return 1
    fi
    if [ "${transient_cleanup_status}" -ne 0 ]; then
        return "${transient_cleanup_status}"
    fi
    if [ "${dump_status}" -ne 0 ]; then
        log_error "Offline Gitea dump failed (exit=${dump_status})"
        return "${dump_status}"
    fi
    [ -s "${DUMP_DIR}/gitea-dump.zip" ] || {
        log_error "Offline Gitea dump did not produce a non-empty archive"
        return 1
    }
    log_success "Gitea consistent offline dump completed"
}

main() {
    local start_time
    local tags
    local snapshot_id
    local duration
    local structural_drill_performed=0

    start_time=$(date +%s)
    trap cleanup EXIT
    trap 'exit 130' INT
    trap 'exit 143' TERM HUP

    log_info "========== 開始 Gitea 備份 =========="
    require_commands
    exec 9>"${GITEA_BACKUP_LOCK}"
    if ! flock -n 9; then
        log_error "Gitea backup is already running"
        return 75
    fi
    install -d -m 700 "${DUMP_DIR}"

    log_info "執行一致性 Gitea offline dump..."
    if ! offline_gitea_dump; then
        duration=$(($(date +%s) - start_time))
        write_gitea_full_backup_receipt "offline_dump_failed" "0" "${duration}" "none" 0 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" 0 || true
        notify_clawbot "failed" "${SERVICE}" "Gitea 一致性備份失敗；服務重啟驗證=${PRIMARY_RESTART_VERIFIED}" "${duration}"
        return 1
    fi

    if ! "${GITEA_RESTORE_DRILL}" --dump-zip "${DUMP_DIR}/gitea-dump.zip"; then
        duration=$(($(date +%s) - start_time))
        write_gitea_full_backup_receipt "structural_drill_failed" "0" "${duration}" "none" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" 0 || true
        log_error "Gitea dump structural verifier failed"
        notify_clawbot "failed" "${SERVICE}" "Gitea 備份結構驗證失敗" "${duration}"
        return 1
    fi
    structural_drill_performed=1

    if [ ! -d "${LOCAL_REPO}/data" ]; then
        log_info "初始化本地 Restic 倉庫..."
        if ! restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}"; then
            duration=$(($(date +%s) - start_time))
            write_gitea_full_backup_receipt "restic_init_failed" "0" "${duration}" "none" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" "${structural_drill_performed}" || true
            log_error "Restic 初始化失敗"
            notify_clawbot "failed" "${SERVICE}" "Gitea Restic 初始化失敗"
            exit 1
        fi
    fi

    tags=$(build_tags "${SERVICE}")
    if ! restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" \
        --password-file "${RESTIC_PASSWORD_FILE}" \
        ${tags}; then
        duration=$(($(date +%s) - start_time))
        write_gitea_full_backup_receipt "restic_backup_failed" "0" "${duration}" "none" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" "${structural_drill_performed}" || true
        log_error "Restic 備份失敗"
        notify_clawbot "failed" "${SERVICE}" "Gitea Restic 備份失敗"
        exit 1
    fi

    snapshot_id=$(restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | grep -oP '"short_id":"\K[^"]+' | head -1 || true)
    log_success "Restic 備份完成: ${snapshot_id:-unknown}"

    log_info "執行 GFS 清理..."
    cleanup_old_backups "${LOCAL_REPO}"

    log_info "Offsite copy is handled by sync-offsite-backups.sh; no direct rclone sync here."

    duration=$(($(date +%s) - start_time))
    write_gitea_full_backup_receipt "success" "$(date +%s)" "${duration}" "${snapshot_id:-unknown}" 1 "${PRIMARY_RESTART_VERIFIED}" "${OUTAGE_SECONDS}" "${structural_drill_performed}" || true
    log_success "========== Gitea 備份完成 (${duration}s) =========="
    notify_clawbot "success" "${SERVICE}" "Gitea 備份完成" "${duration}"
}

main "$@"