From 8c09dc5ca6acbe3e6cb1a6acd88215abd8b7a8bf Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 17:18:25 +0800 Subject: [PATCH] fix(backup): fail closed on forced RLS --- scripts/backup/backup-awoooi-frequent.sh | 73 +++++++----------- scripts/backup/backup-awoooi.sh | 75 ++++++++----------- .../test_backup_awoooi_secret_contract.py | 38 ++++++++-- 3 files changed, 90 insertions(+), 96 deletions(-) diff --git a/scripts/backup/backup-awoooi-frequent.sh b/scripts/backup/backup-awoooi-frequent.sh index 7f6613685..d1ebe6a70 100755 --- a/scripts/backup/backup-awoooi-frequent.sh +++ b/scripts/backup/backup-awoooi-frequent.sh @@ -25,8 +25,6 @@ AWOOOI_K8S_HOSTS="${AWOOOI_K8S_HOSTS:-${AWOOOI_K8S_HOST} 192.168.0.121 192.168.0 AWOOOI_K8S_SECRET_NAME="${AWOOOI_K8S_SECRET_NAME:-awoooi-secrets}" AWOOOI_K8S_NAMESPACE="${AWOOOI_K8S_NAMESPACE:-awoooi-prod}" AWOOOI_K8S_DATABASE_URL_KEYS="${AWOOOI_K8S_DATABASE_URL_KEYS:-AWOOOI_BACKUP_DATABASE_URL BACKUP_DATABASE_URL DATABASE_URL}" -FORCE_RLS_RESTORE_SQL="" -FORCE_RLS_RESTORE_DB="" # 高頻備份保留策略 # 2026-05-19 ogt + Codex: 保留策略統一交給 common.sh。 @@ -137,60 +135,47 @@ latest_restic_snapshot_id() { 2>/dev/null || echo "unknown" } -collect_force_rls_sql() { +logical_backup_preflight() { local database="$1" - local mode="$2" - local query + local role_flags + local forced_rls_count + local role_query + local forced_rls_query - query=" -select format('ALTER TABLE %I.%I ${mode} ROW LEVEL SECURITY;', n.nspname, c.relname) + role_query="select (rolsuper::int)::text || '|' || (rolbypassrls::int)::text from pg_roles where rolname = current_user;" + role_flags="$(run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -At -c $(quote_remote "${role_query}")" 2>/dev/null || true)" + if [ "${role_flags}" != "0|0" ]; then + log_error "${database} logical backup blocked: backup role must not be superuser or global BYPASSRLS" + return 78 + fi + + forced_rls_query=" +select count(*) from pg_class c join pg_namespace n on n.oid = c.relnamespace where c.relkind in ('r', 'p') and c.relforcerowsecurity - and pg_get_userbyid(c.relowner) = current_user -order by 1; + and n.nspname not in ('pg_catalog', 'information_schema'); " - run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -At -c $(quote_remote "${query}")" -} - -apply_remote_sql() { - local database="$1" - local sql="$2" - [ -n "${sql}" ] || return 0 - run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -c $(quote_remote "${sql}") >/dev/null" -} - -restore_force_rls() { - if [ -n "${FORCE_RLS_RESTORE_DB}" ] && [ -n "${FORCE_RLS_RESTORE_SQL}" ]; then - if apply_remote_sql "${FORCE_RLS_RESTORE_DB}" "${FORCE_RLS_RESTORE_SQL}"; then - log_info "FORCE ROW LEVEL SECURITY 已恢復 (${FORCE_RLS_RESTORE_DB})" - else - log_error "FORCE ROW LEVEL SECURITY 恢復失敗 (${FORCE_RLS_RESTORE_DB})" - return 1 - fi - FORCE_RLS_RESTORE_DB="" - FORCE_RLS_RESTORE_SQL="" + forced_rls_count="$(run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -At -c $(quote_remote "${forced_rls_query}")" 2>/dev/null || true)" + if ! [[ "${forced_rls_count}" =~ ^[0-9]+$ ]]; then + log_error "${database} logical backup blocked: FORCE RLS preflight readback failed" + return 78 fi + if [ "${forced_rls_count}" -gt 0 ]; then + log_error "${database} logical backup blocked: forced_rls_tables=${forced_rls_count}; physical/WAL backup with isolated restore verifier is required" + return 78 + fi + return 0 } -trap restore_force_rls EXIT - -dump_database_with_rls_guard() { +dump_database_fail_closed() { local database="$1" local output_file="$2" local stderr_file="${output_file}.stderr" - local noforce_sql force_sql dump_rc + local dump_rc - noforce_sql="$(collect_force_rls_sql "${database}" "NO FORCE")" - force_sql="$(printf '%s\n' "${noforce_sql}" | sed 's/NO FORCE/FORCE/')" - - if [ -n "${noforce_sql}" ]; then - FORCE_RLS_RESTORE_DB="${database}" - FORCE_RLS_RESTORE_SQL="${force_sql}" - log_info "暫時解除 FORCE RLS 以完成完整 pg_dump (${database}, tables=$(printf '%s\n' "${noforce_sql}" | awk 'NF {count++} END {print count+0}'))" - apply_remote_sql "${database}" "${noforce_sql}" - fi + logical_backup_preflight "${database}" || return $? set +e run_remote_pgpass_command "${database}" "pg_dump --no-password \ @@ -199,8 +184,6 @@ dump_database_with_rls_guard() { dump_rc=$? set -e - restore_force_rls - if [ "${dump_rc}" -ne 0 ]; then log_error "${database} dump 失敗，pg_dump stderr 尾端如下（已避免輸出 credential）：" tail -40 "${stderr_file}" | sed -E 's/(password=)[^ ]+/\1REDACTED/g' || true @@ -223,7 +206,7 @@ main() { local timestamp=$(date "+%Y%m%d_%H%M%S") # 只備份 awoooi_prod（高頻核心） - if dump_database_with_rls_guard "awoooi_prod" "${DUMP_DIR}/awoooi_prod_${timestamp}.sql"; then + if dump_database_fail_closed "awoooi_prod" "${DUMP_DIR}/awoooi_prod_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" | cut -f1) log_success "awoooi_prod dump 完成 (${size})" else diff --git a/scripts/backup/backup-awoooi.sh b/scripts/backup/backup-awoooi.sh index c5a7cd2c3..8026f7385 100755 --- a/scripts/backup/backup-awoooi.sh +++ b/scripts/backup/backup-awoooi.sh @@ -26,8 +26,6 @@ AWOOOI_K8S_HOSTS="${AWOOOI_K8S_HOSTS:-${AWOOOI_K8S_HOST} 192.168.0.121 192.168.0 AWOOOI_K8S_SECRET_NAME="${AWOOOI_K8S_SECRET_NAME:-awoooi-secrets}" AWOOOI_K8S_NAMESPACE="${AWOOOI_K8S_NAMESPACE:-awoooi-prod}" AWOOOI_K8S_DATABASE_URL_KEYS="${AWOOOI_K8S_DATABASE_URL_KEYS:-AWOOOI_BACKUP_DATABASE_URL BACKUP_DATABASE_URL DATABASE_URL}" -FORCE_RLS_RESTORE_SQL="" -FORCE_RLS_RESTORE_DB="" resolve_database_url() { if [ -n "${AWOOOI_DATABASE_URL:-}" ]; then @@ -127,59 +125,47 @@ run_remote_pgpass_command() { pgpass_line "${database}" | ssh "ollama@${AWOOOI_HOST}" "$(remote_pgpass_wrapper "${command}")" } -collect_force_rls_sql() { +logical_backup_preflight() { local database="$1" - local mode="$2" - local query + local role_flags + local forced_rls_count + local role_query + local forced_rls_query - query=" -select format('ALTER TABLE %I.%I ${mode} ROW LEVEL SECURITY;', n.nspname, c.relname) + role_query="select (rolsuper::int)::text || '|' || (rolbypassrls::int)::text from pg_roles where rolname = current_user;" + role_flags="$(run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -At -c $(quote_remote "${role_query}")" 2>/dev/null || true)" + if [ "${role_flags}" != "0|0" ]; then + log_error "${database} logical backup blocked: backup role must not be superuser or global BYPASSRLS" + return 78 + fi + + forced_rls_query=" +select count(*) from pg_class c join pg_namespace n on n.oid = c.relnamespace where c.relkind in ('r', 'p') and c.relforcerowsecurity - and pg_get_userbyid(c.relowner) = current_user -order by 1; + and n.nspname not in ('pg_catalog', 'information_schema'); " - run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -At -c $(quote_remote "${query}")" -} - -apply_remote_sql() { - local database="$1" - local sql="$2" - [ -n "${sql}" ] || return 0 - run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -c $(quote_remote "${sql}")" >/dev/null -} - -restore_force_rls() { - if [ -n "${FORCE_RLS_RESTORE_DB}" ] && [ -n "${FORCE_RLS_RESTORE_SQL}" ]; then - if apply_remote_sql "${FORCE_RLS_RESTORE_DB}" "${FORCE_RLS_RESTORE_SQL}"; then - log_info "FORCE ROW LEVEL SECURITY 已恢復 (${FORCE_RLS_RESTORE_DB})" - else - log_error "FORCE ROW LEVEL SECURITY 恢復失敗 (${FORCE_RLS_RESTORE_DB})" - return 1 - fi - FORCE_RLS_RESTORE_DB="" - FORCE_RLS_RESTORE_SQL="" + forced_rls_count="$(run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -At -c $(quote_remote "${forced_rls_query}")" 2>/dev/null || true)" + if ! [[ "${forced_rls_count}" =~ ^[0-9]+$ ]]; then + log_error "${database} logical backup blocked: FORCE RLS preflight readback failed" + return 78 fi + if [ "${forced_rls_count}" -gt 0 ]; then + log_error "${database} logical backup blocked: forced_rls_tables=${forced_rls_count}; physical/WAL backup with isolated restore verifier is required" + return 78 + fi + return 0 } -trap restore_force_rls EXIT - -dump_database_with_rls_guard() { +dump_database_fail_closed() { local database="$1" local output_file="$2" local stderr_file="${output_file}.stderr" - local noforce_sql force_sql dump_rc + local dump_rc - noforce_sql="$(collect_force_rls_sql "${database}" "NO FORCE")" - force_sql="$(printf '%s\n' "${noforce_sql}" | sed 's/NO FORCE/FORCE/')" - if [ -n "${noforce_sql}" ]; then - FORCE_RLS_RESTORE_DB="${database}" - FORCE_RLS_RESTORE_SQL="${force_sql}" - log_info "暫時解除 FORCE RLS 以完成完整 pg_dump (${database}, tables=$(printf '%s\n' "${noforce_sql}" | awk 'NF {count++} END {print count+0}'))" - apply_remote_sql "${database}" "${noforce_sql}" - fi + logical_backup_preflight "${database}" || return $? set +e run_remote_pgpass_command "${database}" "pg_dump --no-password \ @@ -188,7 +174,6 @@ dump_database_with_rls_guard() { dump_rc=$? set -e - restore_force_rls if [ "${dump_rc}" -eq 0 ]; then rm -f "${stderr_file}" return 0 @@ -221,7 +206,7 @@ main() { log_info "Dump awoooi_prod..." local timestamp=$(date "+%Y%m%d_%H%M%S") - if dump_database_with_rls_guard \ + if dump_database_fail_closed \ "awoooi_prod" "${DUMP_DIR}/awoooi_prod_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" | cut -f1) log_success "awoooi_prod dump 完成 (${size})" @@ -232,7 +217,7 @@ main() { # Step 2: awoooi_dev dump log_info "Dump awoooi_dev..." - if dump_database_with_rls_guard \ + if dump_database_fail_closed \ "awoooi_dev" "${DUMP_DIR}/awoooi_dev_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/awoooi_dev_${timestamp}.sql" | cut -f1) log_success "awoooi_dev dump 完成 (${size})" @@ -242,7 +227,7 @@ main() { # Step 3: k3s_datastore dump（Kine 後端） log_info "Dump k3s_datastore..." - if dump_database_with_rls_guard \ + if dump_database_fail_closed \ "k3s_datastore" "${DUMP_DIR}/k3s_datastore_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/k3s_datastore_${timestamp}.sql" | cut -f1) log_success "k3s_datastore dump 完成 (${size})" diff --git a/scripts/backup/tests/test_backup_awoooi_secret_contract.py b/scripts/backup/tests/test_backup_awoooi_secret_contract.py index a31b48bc6..154ec3f28 100644 --- a/scripts/backup/tests/test_backup_awoooi_secret_contract.py +++ b/scripts/backup/tests/test_backup_awoooi_secret_contract.py @@ -5,6 +5,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[3] SCRIPT = ROOT / "scripts" / "backup" / "backup-awoooi.sh" +FREQUENT_SCRIPT = ROOT / "scripts" / "backup" / "backup-awoooi-frequent.sh" def test_daily_backup_has_no_hardcoded_database_password() -> None: @@ -29,11 +30,36 @@ def test_daily_backup_resolves_runtime_database_url_and_fails_closed() -> None: assert 'printf \'%s\\n\' "${decoded}"' in text -def test_daily_backup_uses_same_rls_safe_pg_dump_contract_as_frequent_lane() -> None: +def test_daily_and_frequent_backups_fail_closed_without_mutating_rls() -> None: + for script in (SCRIPT, FREQUENT_SCRIPT): + text = script.read_text(encoding="utf-8") + + assert "logical_backup_preflight()" in text + assert "dump_database_fail_closed()" in text + assert "rolsuper" in text + assert "rolbypassrls" in text + assert "backup role must not be superuser or global BYPASSRLS" in text + assert "forced_rls_tables=${forced_rls_count}" in text + assert "physical/WAL backup with isolated restore verifier is required" in text + assert "return 78" in text + assert 'PGOPTIONS="-c statement_timeout=0 -c max_parallel_workers_per_gather=0"' in text + assert "ALTER TABLE" not in text + assert "NO FORCE" not in text + assert "collect_force_rls_sql" not in text + assert "apply_remote_sql" not in text + assert "restore_force_rls" not in text + + +def test_daily_backup_routes_every_database_through_fail_closed_dump() -> None: text = SCRIPT.read_text(encoding="utf-8") - assert "dump_database_with_rls_guard()" in text - assert 'PGOPTIONS="-c statement_timeout=0 -c max_parallel_workers_per_gather=0"' in text - assert "ALTER TABLE %I.%I ${mode} ROW LEVEL SECURITY" in text - assert "trap restore_force_rls EXIT" in text - assert 'dump_database_with_rls_guard \\\n "awoooi_prod"' in text + assert text.count("if dump_database_fail_closed \\") == 3 + assert 'dump_database_fail_closed \\\n "awoooi_prod"' in text + assert 'dump_database_fail_closed \\\n "awoooi_dev"' in text + assert 'dump_database_fail_closed \\\n "k3s_datastore"' in text + + +def test_frequent_backup_routes_production_database_through_fail_closed_dump() -> None: + text = FREQUENT_SCRIPT.read_text(encoding="utf-8") + + assert 'dump_database_fail_closed "awoooi_prod"' in text