awoooi/scripts/backup/backup-awoooi.sh

#!/bin/bash
# =============================================================================
# WOOO AIOps - AWOOOI 資料庫備份腳本
# 2026-04-05 Claude Code: 首席架構師備份審計 — awoooi_prod/dev/k3s_datastore
# 部署位置: /backup/scripts/backup-awoooi.sh (on 192.168.0.110)
# 整合進 backup-all.sh (步驟 4/4)
# =============================================================================

set -euo pipefail

# 載入共用函式
source "$(dirname "$0")/common.sh"

# 配置
SERVICE="awoooi"
AWOOOI_HOST="192.168.0.188"
AWOOOI_DB_USER="awoooi"
AWOOOI_DB_PASS="${AWOOOI_DB_PASS:-}"
AWOOOI_DB_HOST="localhost"
AWOOOI_DB_PORT="5432"
K3S_DB_USER="postgres"
LOCAL_REPO="${BACKUP_BASE}/awoooi"
DUMP_DIR="/tmp/awoooi-backup-$$"
AWOOOI_K8S_HOST="${AWOOOI_K8S_HOST:-192.168.0.120}"
AWOOOI_K8S_HOSTS="${AWOOOI_K8S_HOSTS:-${AWOOOI_K8S_HOST} 192.168.0.121 192.168.0.125}"
AWOOOI_K8S_SECRET_NAME="${AWOOOI_K8S_SECRET_NAME:-awoooi-secrets}"
AWOOOI_K8S_NAMESPACE="${AWOOOI_K8S_NAMESPACE:-awoooi-prod}"
AWOOOI_K8S_DATABASE_URL_KEYS="${AWOOOI_K8S_DATABASE_URL_KEYS:-AWOOOI_BACKUP_DATABASE_URL BACKUP_DATABASE_URL DATABASE_URL}"

resolve_database_url() {
    if [ -n "${AWOOOI_DATABASE_URL:-}" ]; then
        printf '%s\n' "${AWOOOI_DATABASE_URL}"
        return 0
    fi
    if [ -n "${DATABASE_URL:-}" ]; then
        printf '%s\n' "${DATABASE_URL}"
        return 0
    fi

    # Resolve only inside the runtime backup process. The value is never
    # written to logs, source, artifacts, or command output.
    local k8s_host key encoded decoded
    for k8s_host in ${AWOOOI_K8S_HOSTS}; do
        for key in ${AWOOOI_K8S_DATABASE_URL_KEYS}; do
            encoded="$(ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 "wooo@${k8s_host}" \
                "sudo -n kubectl get secret ${AWOOOI_K8S_SECRET_NAME} -n ${AWOOOI_K8S_NAMESPACE} -o jsonpath='{.data.${key}}' 2>/dev/null || kubectl get secret ${AWOOOI_K8S_SECRET_NAME} -n ${AWOOOI_K8S_NAMESPACE} -o jsonpath='{.data.${key}}'" \
                2>/dev/null || true)"
            decoded="$(printf '%s' "${encoded}" | base64 -d 2>/dev/null || true)"
            if [ -n "${decoded}" ]; then
                printf '%s\n' "${decoded}"
                return 0
            fi
        done
    done
    return 1
}

load_database_config() {
    local database_url
    database_url="$(resolve_database_url || true)"
    if [ -z "${database_url}" ]; then
        log_error "無法解析 AWOOOI DATABASE_URL；拒絕使用舊硬編密碼"
        return 1
    fi

    eval "$(
        python3 - 3<<< "${database_url}" <<'PY'
import shlex
from urllib.parse import unquote, urlparse

with open(3) as source:
    url = source.read().strip()
parsed = urlparse(url)

values = {
    "AWOOOI_DB_USER": unquote(parsed.username or "awoooi"),
    "AWOOOI_DB_PASS": unquote(parsed.password or ""),
    "AWOOOI_DB_HOST": parsed.hostname or "localhost",
    "AWOOOI_DB_PORT": str(parsed.port or 5432),
}
for key, value in values.items():
    print(f"{key}={shlex.quote(value)}")
PY
    )"
}

quote_remote() {
    printf "%q" "$1"
}

pgpass_escape() {
    local value="$1"
    value="${value//\\/\\\\}"
    value="${value//:/\\:}"
    printf '%s' "${value}"
}

pgpass_line() {
    local database="$1"
    printf '%s:%s:%s:%s:%s\n' \
        "$(pgpass_escape "${AWOOOI_DB_HOST}")" \
        "$(pgpass_escape "${AWOOOI_DB_PORT}")" \
        "$(pgpass_escape "${database}")" \
        "$(pgpass_escape "${AWOOOI_DB_USER}")" \
        "$(pgpass_escape "${AWOOOI_DB_PASS}")"
}

remote_pgpass_wrapper() {
    local command="$1"
    printf 'umask 077; pgpass=$(mktemp "${TMPDIR:-/tmp}/awoooi-pgpass.XXXXXX") || exit 1; cleanup() { rm -f "$pgpass"; }; trap cleanup EXIT HUP INT TERM; cat > "$pgpass"; PGOPTIONS="-c statement_timeout=0 -c max_parallel_workers_per_gather=0" PGPASSFILE="$pgpass" %s' "${command}"
}

remote_psql_command() {
    local database="$1"
    printf "psql --no-password -U %s -h %s -p %s -d %s -v ON_ERROR_STOP=1" \
        "$(quote_remote "${AWOOOI_DB_USER}")" \
        "$(quote_remote "${AWOOOI_DB_HOST}")" \
        "$(quote_remote "${AWOOOI_DB_PORT}")" \
        "$(quote_remote "${database}")"
}

run_remote_pgpass_command() {
    local database="$1"
    local command="$2"
    pgpass_line "${database}" | ssh "ollama@${AWOOOI_HOST}" "$(remote_pgpass_wrapper "${command}")"
}

logical_backup_preflight() {
    local database="$1"
    local role_flags
    local forced_rls_count
    local role_query
    local forced_rls_query

    role_query="select (rolsuper::int)::text || '|' || (rolbypassrls::int)::text from pg_roles where rolname = current_user;"
    role_flags="$(run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -At -c $(quote_remote "${role_query}")" 2>/dev/null || true)"
    if [ "${role_flags}" != "0|0" ]; then
        log_error "${database} logical backup blocked: backup role must not be superuser or global BYPASSRLS"
        return 78
    fi

    forced_rls_query="
select count(*)
from pg_class c
join pg_namespace n on n.oid = c.relnamespace
where c.relkind in ('r', 'p')
  and c.relforcerowsecurity
  and n.nspname not in ('pg_catalog', 'information_schema');
"
    forced_rls_count="$(run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -At -c $(quote_remote "${forced_rls_query}")" 2>/dev/null || true)"
    if ! [[ "${forced_rls_count}" =~ ^[0-9]+$ ]]; then
        log_error "${database} logical backup blocked: FORCE RLS preflight readback failed"
        return 78
    fi
    if [ "${forced_rls_count}" -gt 0 ]; then
        log_error "${database} logical backup blocked: forced_rls_tables=${forced_rls_count}; physical/WAL backup with isolated restore verifier is required"
        return 78
    fi
    return 0
}

dump_database_fail_closed() {
    local database="$1"
    local output_file="$2"
    local stderr_file="${output_file}.stderr"
    local dump_rc

    logical_backup_preflight "${database}" || return $?

    set +e
    run_remote_pgpass_command "${database}" "pg_dump --no-password \
        -U $(quote_remote "${AWOOOI_DB_USER}") -h $(quote_remote "${AWOOOI_DB_HOST}") -p $(quote_remote "${AWOOOI_DB_PORT}") \
        $(quote_remote "${database}")" >"${output_file}" 2>"${stderr_file}"
    dump_rc=$?
    set -e

    if [ "${dump_rc}" -eq 0 ]; then
        rm -f "${stderr_file}"
        return 0
    fi

    log_error "${database} dump 失敗，pg_dump stderr 尾端如下（已避免輸出 credential）："
    tail -40 "${stderr_file}" | sed -E 's/(password=)[^ ]+/\1REDACTED/g' >&2 || true
    rm -f "${stderr_file}"
    return "${dump_rc}"
}

# 保留策略覆寫（比其他服務更長）
KEEP_DAILY=14      # 14 天每日
KEEP_WEEKLY=8      # 8 週每週
KEEP_MONTHLY=12    # 12 個月每月

main() {
    local start_time=$(date +%s)
    local failed=0

    if ! load_database_config; then
        notify_clawbot "failed" "${SERVICE}" "AWOOOI DB backup credential unavailable"
        exit 1
    fi

    log_info "========== 開始 AWOOOI 資料庫備份 =========="
    mkdir -p "${DUMP_DIR}"

    # Step 1: awoooi_prod dump（核心資料庫：KB/事故/AutoRepair/Drift）
    log_info "Dump awoooi_prod..."
    local timestamp=$(date "+%Y%m%d_%H%M%S")

    if dump_database_fail_closed \
        "awoooi_prod" "${DUMP_DIR}/awoooi_prod_${timestamp}.sql"; then
        local size=$(du -h "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" | cut -f1)
        log_success "awoooi_prod dump 完成 (${size})"
    else
        log_error "awoooi_prod dump 失敗"
        ((failed++))
    fi

    # Step 2: awoooi_dev dump
    log_info "Dump awoooi_dev..."
    if dump_database_fail_closed \
        "awoooi_dev" "${DUMP_DIR}/awoooi_dev_${timestamp}.sql"; then
        local size=$(du -h "${DUMP_DIR}/awoooi_dev_${timestamp}.sql" | cut -f1)
        log_success "awoooi_dev dump 完成 (${size})"
    else
        log_warn "awoooi_dev dump 跳過（可能不存在）"
    fi

    # Step 3: k3s_datastore dump（Kine 後端）
    log_info "Dump k3s_datastore..."
    if dump_database_fail_closed \
        "k3s_datastore" "${DUMP_DIR}/k3s_datastore_${timestamp}.sql"; then
        local size=$(du -h "${DUMP_DIR}/k3s_datastore_${timestamp}.sql" | cut -f1)
        log_success "k3s_datastore dump 完成 (${size})"
    else
        log_warn "k3s_datastore dump 跳過"
    fi

    # 若核心 DB 失敗，中止
    if [ $failed -gt 0 ]; then
        log_error "核心 DB awoooi_prod 備份失敗，中止"
        notify_clawbot "failed" "${SERVICE}" "AWOOOI 核心 DB 備份失敗"
        rm -rf "${DUMP_DIR}"
        exit 1
    fi

    # Step 4: Restic 備份
    log_info "建立 Restic 備份..."
    local tags=$(build_tags "${SERVICE}")

    if [ ! -d "${LOCAL_REPO}/data" ]; then
        log_info "初始化 Restic 倉庫 ${LOCAL_REPO}..."
        restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1
    fi

    restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" \
        --password-file "${RESTIC_PASSWORD_FILE}" \
        ${tags} 2>&1

    local snapshot_id=$(restic -r "${LOCAL_REPO}" snapshots --latest 1 --json \
        --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \
        grep -oP '"short_id":"\K[^"]+' | head -1)
    log_success "Restic 備份完成: ${snapshot_id}"

    # Step 5: GFS 清理（延長保留）
    log_info "執行 GFS 清理 (daily=${KEEP_DAILY} weekly=${KEEP_WEEKLY} monthly=${KEEP_MONTHLY})..."
    restic -r "${LOCAL_REPO}" forget --prune \
        --password-file "${RESTIC_PASSWORD_FILE}" \
        --keep-daily ${KEEP_DAILY} \
        --keep-weekly ${KEEP_WEEKLY} \
        --keep-monthly ${KEEP_MONTHLY} 2>&1
    log_success "GFS 清理完成"

    # Step 6: Offsite 同步交由集中控制器
    #
    # 2026-06-18 Codex: daily service backup must not run a direct offsite
    # sync. `backup-all.sh` and the scheduled offsite controller own the gated
    # rclone sync / verification flow so failed local backups cannot partially
    # publish stale repositories.
    if check_b2_config; then
        log_info "偵測到 legacy B2 設定；略過 service-level 直同步，請使用 sync-offsite-backups.sh"
    fi

    rm -rf "${DUMP_DIR}"

    local end_time=$(date +%s)
    local duration=$((end_time - start_time))
    log_success "========== AWOOOI 備份完成 (${duration}s) =========="
    notify_clawbot "success" "${SERVICE}" "AWOOOI DB 備份完成 (awoooi_prod/dev + k3s)" "${duration}"
}

main "$@"