diff --git a/scripts/backup/backup-awoooi.sh b/scripts/backup/backup-awoooi.sh index fc1799714..c5a7cd2c3 100755 --- a/scripts/backup/backup-awoooi.sh +++ b/scripts/backup/backup-awoooi.sh @@ -21,6 +21,70 @@ AWOOOI_DB_PORT="5432" K3S_DB_USER="postgres" LOCAL_REPO="${BACKUP_BASE}/awoooi" DUMP_DIR="/tmp/awoooi-backup-$$" +AWOOOI_K8S_HOST="${AWOOOI_K8S_HOST:-192.168.0.120}" +AWOOOI_K8S_HOSTS="${AWOOOI_K8S_HOSTS:-${AWOOOI_K8S_HOST} 192.168.0.121 192.168.0.125}" +AWOOOI_K8S_SECRET_NAME="${AWOOOI_K8S_SECRET_NAME:-awoooi-secrets}" +AWOOOI_K8S_NAMESPACE="${AWOOOI_K8S_NAMESPACE:-awoooi-prod}" +AWOOOI_K8S_DATABASE_URL_KEYS="${AWOOOI_K8S_DATABASE_URL_KEYS:-AWOOOI_BACKUP_DATABASE_URL BACKUP_DATABASE_URL DATABASE_URL}" +FORCE_RLS_RESTORE_SQL="" +FORCE_RLS_RESTORE_DB="" + +resolve_database_url() { + if [ -n "${AWOOOI_DATABASE_URL:-}" ]; then + printf '%s\n' "${AWOOOI_DATABASE_URL}" + return 0 + fi + if [ -n "${DATABASE_URL:-}" ]; then + printf '%s\n' "${DATABASE_URL}" + return 0 + fi + + # Resolve only inside the runtime backup process. The value is never + # written to logs, source, artifacts, or command output. + local k8s_host key encoded decoded + for k8s_host in ${AWOOOI_K8S_HOSTS}; do + for key in ${AWOOOI_K8S_DATABASE_URL_KEYS}; do + encoded="$(ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 "wooo@${k8s_host}" \ + "sudo -n kubectl get secret ${AWOOOI_K8S_SECRET_NAME} -n ${AWOOOI_K8S_NAMESPACE} -o jsonpath='{.data.${key}}' 2>/dev/null || kubectl get secret ${AWOOOI_K8S_SECRET_NAME} -n ${AWOOOI_K8S_NAMESPACE} -o jsonpath='{.data.${key}}'" \ + 2>/dev/null || true)" + decoded="$(printf '%s' "${encoded}" | base64 -d 2>/dev/null || true)" + if [ -n "${decoded}" ]; then + printf '%s\n' "${decoded}" + return 0 + fi + done + done + return 1 +} + +load_database_config() { + local database_url + database_url="$(resolve_database_url || true)" + if [ -z "${database_url}" ]; then + log_error "無法解析 AWOOOI DATABASE_URL;拒絕使用舊硬編密碼" + return 1 + fi + + eval "$( + python3 - 3<<< "${database_url}" <<'PY' +import shlex +from urllib.parse import unquote, urlparse + +with open(3) as source: + url = source.read().strip() +parsed = urlparse(url) + +values = { + "AWOOOI_DB_USER": unquote(parsed.username or "awoooi"), + "AWOOOI_DB_PASS": unquote(parsed.password or ""), + "AWOOOI_DB_HOST": parsed.hostname or "localhost", + "AWOOOI_DB_PORT": str(parsed.port or 5432), +} +for key, value in values.items(): + print(f"{key}={shlex.quote(value)}") +PY + )" +} quote_remote() { printf "%q" "$1" @@ -45,26 +109,95 @@ pgpass_line() { remote_pgpass_wrapper() { local command="$1" - printf 'umask 077; pgpass=$(mktemp "${TMPDIR:-/tmp}/awoooi-pgpass.XXXXXX") || exit 1; cleanup() { rm -f "$pgpass"; }; trap cleanup EXIT HUP INT TERM; cat > "$pgpass"; PGPASSFILE="$pgpass" %s' "${command}" + printf 'umask 077; pgpass=$(mktemp "${TMPDIR:-/tmp}/awoooi-pgpass.XXXXXX") || exit 1; cleanup() { rm -f "$pgpass"; }; trap cleanup EXIT HUP INT TERM; cat > "$pgpass"; PGOPTIONS="-c statement_timeout=0 -c max_parallel_workers_per_gather=0" PGPASSFILE="$pgpass" %s' "${command}" } -run_remote_pg_dump() { +remote_psql_command() { + local database="$1" + printf "psql --no-password -U %s -h %s -p %s -d %s -v ON_ERROR_STOP=1" \ + "$(quote_remote "${AWOOOI_DB_USER}")" \ + "$(quote_remote "${AWOOOI_DB_HOST}")" \ + "$(quote_remote "${AWOOOI_DB_PORT}")" \ + "$(quote_remote "${database}")" +} + +run_remote_pgpass_command() { + local database="$1" + local command="$2" + pgpass_line "${database}" | ssh "ollama@${AWOOOI_HOST}" "$(remote_pgpass_wrapper "${command}")" +} + +collect_force_rls_sql() { + local database="$1" + local mode="$2" + local query + + query=" +select format('ALTER TABLE %I.%I ${mode} ROW LEVEL SECURITY;', n.nspname, c.relname) +from pg_class c +join pg_namespace n on n.oid = c.relnamespace +where c.relkind in ('r', 'p') + and c.relforcerowsecurity + and pg_get_userbyid(c.relowner) = current_user +order by 1; +" + run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -At -c $(quote_remote "${query}")" +} + +apply_remote_sql() { + local database="$1" + local sql="$2" + [ -n "${sql}" ] || return 0 + run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -c $(quote_remote "${sql}")" >/dev/null +} + +restore_force_rls() { + if [ -n "${FORCE_RLS_RESTORE_DB}" ] && [ -n "${FORCE_RLS_RESTORE_SQL}" ]; then + if apply_remote_sql "${FORCE_RLS_RESTORE_DB}" "${FORCE_RLS_RESTORE_SQL}"; then + log_info "FORCE ROW LEVEL SECURITY 已恢復 (${FORCE_RLS_RESTORE_DB})" + else + log_error "FORCE ROW LEVEL SECURITY 恢復失敗 (${FORCE_RLS_RESTORE_DB})" + return 1 + fi + FORCE_RLS_RESTORE_DB="" + FORCE_RLS_RESTORE_SQL="" + fi +} + +trap restore_force_rls EXIT + +dump_database_with_rls_guard() { local database="$1" local output_file="$2" local stderr_file="${output_file}.stderr" - local command + local noforce_sql force_sql dump_rc - command="pg_dump --no-password -U $(quote_remote "${AWOOOI_DB_USER}") -h $(quote_remote "${AWOOOI_DB_HOST}") -p $(quote_remote "${AWOOOI_DB_PORT}") $(quote_remote "${database}")" - if pgpass_line "${database}" \ - | ssh "ollama@${AWOOOI_HOST}" "$(remote_pgpass_wrapper "${command}")" \ - >"${output_file}" 2>"${stderr_file}"; then + noforce_sql="$(collect_force_rls_sql "${database}" "NO FORCE")" + force_sql="$(printf '%s\n' "${noforce_sql}" | sed 's/NO FORCE/FORCE/')" + if [ -n "${noforce_sql}" ]; then + FORCE_RLS_RESTORE_DB="${database}" + FORCE_RLS_RESTORE_SQL="${force_sql}" + log_info "暫時解除 FORCE RLS 以完成完整 pg_dump (${database}, tables=$(printf '%s\n' "${noforce_sql}" | awk 'NF {count++} END {print count+0}'))" + apply_remote_sql "${database}" "${noforce_sql}" + fi + + set +e + run_remote_pgpass_command "${database}" "pg_dump --no-password \ + -U $(quote_remote "${AWOOOI_DB_USER}") -h $(quote_remote "${AWOOOI_DB_HOST}") -p $(quote_remote "${AWOOOI_DB_PORT}") \ + $(quote_remote "${database}")" >"${output_file}" 2>"${stderr_file}" + dump_rc=$? + set -e + + restore_force_rls + if [ "${dump_rc}" -eq 0 ]; then rm -f "${stderr_file}" return 0 fi + log_error "${database} dump 失敗,pg_dump stderr 尾端如下(已避免輸出 credential):" tail -40 "${stderr_file}" | sed -E 's/(password=)[^ ]+/\1REDACTED/g' >&2 || true rm -f "${stderr_file}" - return 1 + return "${dump_rc}" } # 保留策略覆寫(比其他服務更長) @@ -76,8 +209,7 @@ main() { local start_time=$(date +%s) local failed=0 - if [ -z "${AWOOOI_DB_PASS}" ]; then - log_error "AWOOOI_DB_PASS is unavailable; refusing an unauthenticated or hardcoded fallback" + if ! load_database_config; then notify_clawbot "failed" "${SERVICE}" "AWOOOI DB backup credential unavailable" exit 1 fi @@ -89,7 +221,7 @@ main() { log_info "Dump awoooi_prod..." local timestamp=$(date "+%Y%m%d_%H%M%S") - if run_remote_pg_dump \ + if dump_database_with_rls_guard \ "awoooi_prod" "${DUMP_DIR}/awoooi_prod_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" | cut -f1) log_success "awoooi_prod dump 完成 (${size})" @@ -100,7 +232,7 @@ main() { # Step 2: awoooi_dev dump log_info "Dump awoooi_dev..." - if run_remote_pg_dump \ + if dump_database_with_rls_guard \ "awoooi_dev" "${DUMP_DIR}/awoooi_dev_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/awoooi_dev_${timestamp}.sql" | cut -f1) log_success "awoooi_dev dump 完成 (${size})" @@ -110,7 +242,7 @@ main() { # Step 3: k3s_datastore dump(Kine 後端) log_info "Dump k3s_datastore..." - if run_remote_pg_dump \ + if dump_database_with_rls_guard \ "k3s_datastore" "${DUMP_DIR}/k3s_datastore_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/k3s_datastore_${timestamp}.sql" | cut -f1) log_success "k3s_datastore dump 完成 (${size})" diff --git a/scripts/backup/tests/test_backup_awoooi_secret_contract.py b/scripts/backup/tests/test_backup_awoooi_secret_contract.py index e513cb955..a31b48bc6 100644 --- a/scripts/backup/tests/test_backup_awoooi_secret_contract.py +++ b/scripts/backup/tests/test_backup_awoooi_secret_contract.py @@ -18,8 +18,22 @@ def test_daily_backup_has_no_hardcoded_database_password() -> None: assert "pg_dump --no-password" in text -def test_daily_backup_fails_closed_without_runtime_credential() -> None: +def test_daily_backup_resolves_runtime_database_url_and_fails_closed() -> None: text = SCRIPT.read_text(encoding="utf-8") - assert 'if [ -z "${AWOOOI_DB_PASS}" ]; then' in text - assert "refusing an unauthenticated or hardcoded fallback" in text + assert "resolve_database_url()" in text + assert "load_database_config()" in text + assert "AWOOOI_BACKUP_DATABASE_URL BACKUP_DATABASE_URL DATABASE_URL" in text + assert "無法解析 AWOOOI DATABASE_URL;拒絕使用舊硬編密碼" in text + assert "base64 -d" in text + assert 'printf \'%s\\n\' "${decoded}"' in text + + +def test_daily_backup_uses_same_rls_safe_pg_dump_contract_as_frequent_lane() -> None: + text = SCRIPT.read_text(encoding="utf-8") + + assert "dump_database_with_rls_guard()" in text + assert 'PGOPTIONS="-c statement_timeout=0 -c max_parallel_workers_per_gather=0"' in text + assert "ALTER TABLE %I.%I ${mode} ROW LEVEL SECURITY" in text + assert "trap restore_force_rls EXIT" in text + assert 'dump_database_with_rls_guard \\\n "awoooi_prod"' in text