fix(backup): align daily database credential contract
This commit is contained in:
ogt
@@ -21,6 +21,70 @@ AWOOOI_DB_PORT="5432"
|
||||
K3S_DB_USER="postgres"
|
||||
LOCAL_REPO="${BACKUP_BASE}/awoooi"
|
||||
DUMP_DIR="/tmp/awoooi-backup-$$"
|
||||
AWOOOI_K8S_HOST="${AWOOOI_K8S_HOST:-192.168.0.120}"
|
||||
AWOOOI_K8S_HOSTS="${AWOOOI_K8S_HOSTS:-${AWOOOI_K8S_HOST} 192.168.0.121 192.168.0.125}"
|
||||
AWOOOI_K8S_SECRET_NAME="${AWOOOI_K8S_SECRET_NAME:-awoooi-secrets}"
|
||||
AWOOOI_K8S_NAMESPACE="${AWOOOI_K8S_NAMESPACE:-awoooi-prod}"
|
||||
AWOOOI_K8S_DATABASE_URL_KEYS="${AWOOOI_K8S_DATABASE_URL_KEYS:-AWOOOI_BACKUP_DATABASE_URL BACKUP_DATABASE_URL DATABASE_URL}"
|
||||
FORCE_RLS_RESTORE_SQL=""
|
||||
FORCE_RLS_RESTORE_DB=""
|
||||
|
||||
resolve_database_url() {
|
||||
if [ -n "${AWOOOI_DATABASE_URL:-}" ]; then
|
||||
printf '%s\n' "${AWOOOI_DATABASE_URL}"
|
||||
return 0
|
||||
fi
|
||||
if [ -n "${DATABASE_URL:-}" ]; then
|
||||
printf '%s\n' "${DATABASE_URL}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Resolve only inside the runtime backup process. The value is never
|
||||
# written to logs, source, artifacts, or command output.
|
||||
local k8s_host key encoded decoded
|
||||
for k8s_host in ${AWOOOI_K8S_HOSTS}; do
|
||||
for key in ${AWOOOI_K8S_DATABASE_URL_KEYS}; do
|
||||
encoded="$(ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 "wooo@${k8s_host}" \
|
||||
"sudo -n kubectl get secret ${AWOOOI_K8S_SECRET_NAME} -n ${AWOOOI_K8S_NAMESPACE} -o jsonpath='{.data.${key}}' 2>/dev/null || kubectl get secret ${AWOOOI_K8S_SECRET_NAME} -n ${AWOOOI_K8S_NAMESPACE} -o jsonpath='{.data.${key}}'" \
|
||||
2>/dev/null || true)"
|
||||
decoded="$(printf '%s' "${encoded}" | base64 -d 2>/dev/null || true)"
|
||||
if [ -n "${decoded}" ]; then
|
||||
printf '%s\n' "${decoded}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
load_database_config() {
|
||||
local database_url
|
||||
database_url="$(resolve_database_url || true)"
|
||||
if [ -z "${database_url}" ]; then
|
||||
log_error "無法解析 AWOOOI DATABASE_URL；拒絕使用舊硬編密碼"
|
||||
return 1
|
||||
fi
|
||||
|
||||
eval "$(
|
||||
python3 - 3<<< "${database_url}" <<'PY'
|
||||
import shlex
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
with open(3) as source:
|
||||
url = source.read().strip()
|
||||
parsed = urlparse(url)
|
||||
|
||||
values = {
|
||||
"AWOOOI_DB_USER": unquote(parsed.username or "awoooi"),
|
||||
"AWOOOI_DB_PASS": unquote(parsed.password or ""),
|
||||
"AWOOOI_DB_HOST": parsed.hostname or "localhost",
|
||||
"AWOOOI_DB_PORT": str(parsed.port or 5432),
|
||||
}
|
||||
for key, value in values.items():
|
||||
print(f"{key}={shlex.quote(value)}")
|
||||
PY
|
||||
)"
|
||||
}
|
||||
|
||||
quote_remote() {
|
||||
printf "%q" "$1"
|
||||
@@ -45,26 +109,95 @@ pgpass_line() {
|
||||
|
||||
remote_pgpass_wrapper() {
|
||||
local command="$1"
|
||||
printf 'umask 077; pgpass=$(mktemp "${TMPDIR:-/tmp}/awoooi-pgpass.XXXXXX") || exit 1; cleanup() { rm -f "$pgpass"; }; trap cleanup EXIT HUP INT TERM; cat > "$pgpass"; PGPASSFILE="$pgpass" %s' "${command}"
|
||||
printf 'umask 077; pgpass=$(mktemp "${TMPDIR:-/tmp}/awoooi-pgpass.XXXXXX") || exit 1; cleanup() { rm -f "$pgpass"; }; trap cleanup EXIT HUP INT TERM; cat > "$pgpass"; PGOPTIONS="-c statement_timeout=0 -c max_parallel_workers_per_gather=0" PGPASSFILE="$pgpass" %s' "${command}"
|
||||
}
|
||||
|
||||
run_remote_pg_dump() {
|
||||
remote_psql_command() {
|
||||
local database="$1"
|
||||
printf "psql --no-password -U %s -h %s -p %s -d %s -v ON_ERROR_STOP=1" \
|
||||
"$(quote_remote "${AWOOOI_DB_USER}")" \
|
||||
"$(quote_remote "${AWOOOI_DB_HOST}")" \
|
||||
"$(quote_remote "${AWOOOI_DB_PORT}")" \
|
||||
"$(quote_remote "${database}")"
|
||||
}
|
||||
|
||||
run_remote_pgpass_command() {
|
||||
local database="$1"
|
||||
local command="$2"
|
||||
pgpass_line "${database}" | ssh "ollama@${AWOOOI_HOST}" "$(remote_pgpass_wrapper "${command}")"
|
||||
}
|
||||
|
||||
collect_force_rls_sql() {
|
||||
local database="$1"
|
||||
local mode="$2"
|
||||
local query
|
||||
|
||||
query="
|
||||
select format('ALTER TABLE %I.%I ${mode} ROW LEVEL SECURITY;', n.nspname, c.relname)
|
||||
from pg_class c
|
||||
join pg_namespace n on n.oid = c.relnamespace
|
||||
where c.relkind in ('r', 'p')
|
||||
and c.relforcerowsecurity
|
||||
and pg_get_userbyid(c.relowner) = current_user
|
||||
order by 1;
|
||||
"
|
||||
run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -At -c $(quote_remote "${query}")"
|
||||
}
|
||||
|
||||
apply_remote_sql() {
|
||||
local database="$1"
|
||||
local sql="$2"
|
||||
[ -n "${sql}" ] || return 0
|
||||
run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -c $(quote_remote "${sql}")" >/dev/null
|
||||
}
|
||||
|
||||
restore_force_rls() {
|
||||
if [ -n "${FORCE_RLS_RESTORE_DB}" ] && [ -n "${FORCE_RLS_RESTORE_SQL}" ]; then
|
||||
if apply_remote_sql "${FORCE_RLS_RESTORE_DB}" "${FORCE_RLS_RESTORE_SQL}"; then
|
||||
log_info "FORCE ROW LEVEL SECURITY 已恢復 (${FORCE_RLS_RESTORE_DB})"
|
||||
else
|
||||
log_error "FORCE ROW LEVEL SECURITY 恢復失敗 (${FORCE_RLS_RESTORE_DB})"
|
||||
return 1
|
||||
fi
|
||||
FORCE_RLS_RESTORE_DB=""
|
||||
FORCE_RLS_RESTORE_SQL=""
|
||||
fi
|
||||
}
|
||||
|
||||
trap restore_force_rls EXIT
|
||||
|
||||
dump_database_with_rls_guard() {
|
||||
local database="$1"
|
||||
local output_file="$2"
|
||||
local stderr_file="${output_file}.stderr"
|
||||
local command
|
||||
local noforce_sql force_sql dump_rc
|
||||
|
||||
command="pg_dump --no-password -U $(quote_remote "${AWOOOI_DB_USER}") -h $(quote_remote "${AWOOOI_DB_HOST}") -p $(quote_remote "${AWOOOI_DB_PORT}") $(quote_remote "${database}")"
|
||||
if pgpass_line "${database}" \
|
||||
| ssh "ollama@${AWOOOI_HOST}" "$(remote_pgpass_wrapper "${command}")" \
|
||||
>"${output_file}" 2>"${stderr_file}"; then
|
||||
noforce_sql="$(collect_force_rls_sql "${database}" "NO FORCE")"
|
||||
force_sql="$(printf '%s\n' "${noforce_sql}" | sed 's/NO FORCE/FORCE/')"
|
||||
if [ -n "${noforce_sql}" ]; then
|
||||
FORCE_RLS_RESTORE_DB="${database}"
|
||||
FORCE_RLS_RESTORE_SQL="${force_sql}"
|
||||
log_info "暫時解除 FORCE RLS 以完成完整 pg_dump (${database}, tables=$(printf '%s\n' "${noforce_sql}" | awk 'NF {count++} END {print count+0}'))"
|
||||
apply_remote_sql "${database}" "${noforce_sql}"
|
||||
fi
|
||||
|
||||
set +e
|
||||
run_remote_pgpass_command "${database}" "pg_dump --no-password \
|
||||
-U $(quote_remote "${AWOOOI_DB_USER}") -h $(quote_remote "${AWOOOI_DB_HOST}") -p $(quote_remote "${AWOOOI_DB_PORT}") \
|
||||
$(quote_remote "${database}")" >"${output_file}" 2>"${stderr_file}"
|
||||
dump_rc=$?
|
||||
set -e
|
||||
|
||||
restore_force_rls
|
||||
if [ "${dump_rc}" -eq 0 ]; then
|
||||
rm -f "${stderr_file}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_error "${database} dump 失敗，pg_dump stderr 尾端如下（已避免輸出 credential）："
|
||||
tail -40 "${stderr_file}" | sed -E 's/(password=)[^ ]+/\1REDACTED/g' >&2 || true
|
||||
rm -f "${stderr_file}"
|
||||
return 1
|
||||
return "${dump_rc}"
|
||||
}
|
||||
|
||||
# 保留策略覆寫（比其他服務更長）
|
||||
@@ -76,8 +209,7 @@ main() {
|
||||
local start_time=$(date +%s)
|
||||
local failed=0
|
||||
|
||||
if [ -z "${AWOOOI_DB_PASS}" ]; then
|
||||
log_error "AWOOOI_DB_PASS is unavailable; refusing an unauthenticated or hardcoded fallback"
|
||||
if ! load_database_config; then
|
||||
notify_clawbot "failed" "${SERVICE}" "AWOOOI DB backup credential unavailable"
|
||||
exit 1
|
||||
fi
|
||||
@@ -89,7 +221,7 @@ main() {
|
||||
log_info "Dump awoooi_prod..."
|
||||
local timestamp=$(date "+%Y%m%d_%H%M%S")
|
||||
|
||||
if run_remote_pg_dump \
|
||||
if dump_database_with_rls_guard \
|
||||
"awoooi_prod" "${DUMP_DIR}/awoooi_prod_${timestamp}.sql"; then
|
||||
local size=$(du -h "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" | cut -f1)
|
||||
log_success "awoooi_prod dump 完成 (${size})"
|
||||
@@ -100,7 +232,7 @@ main() {
|
||||
|
||||
# Step 2: awoooi_dev dump
|
||||
log_info "Dump awoooi_dev..."
|
||||
if run_remote_pg_dump \
|
||||
if dump_database_with_rls_guard \
|
||||
"awoooi_dev" "${DUMP_DIR}/awoooi_dev_${timestamp}.sql"; then
|
||||
local size=$(du -h "${DUMP_DIR}/awoooi_dev_${timestamp}.sql" | cut -f1)
|
||||
log_success "awoooi_dev dump 完成 (${size})"
|
||||
@@ -110,7 +242,7 @@ main() {
|
||||
|
||||
# Step 3: k3s_datastore dump（Kine 後端）
|
||||
log_info "Dump k3s_datastore..."
|
||||
if run_remote_pg_dump \
|
||||
if dump_database_with_rls_guard \
|
||||
"k3s_datastore" "${DUMP_DIR}/k3s_datastore_${timestamp}.sql"; then
|
||||
local size=$(du -h "${DUMP_DIR}/k3s_datastore_${timestamp}.sql" | cut -f1)
|
||||
log_success "k3s_datastore dump 完成 (${size})"
|
||||
|
||||
@@ -18,8 +18,22 @@ def test_daily_backup_has_no_hardcoded_database_password() -> None:
|
||||
assert "pg_dump --no-password" in text
|
||||
|
||||
|
||||
def test_daily_backup_fails_closed_without_runtime_credential() -> None:
|
||||
def test_daily_backup_resolves_runtime_database_url_and_fails_closed() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert 'if [ -z "${AWOOOI_DB_PASS}" ]; then' in text
|
||||
assert "refusing an unauthenticated or hardcoded fallback" in text
|
||||
assert "resolve_database_url()" in text
|
||||
assert "load_database_config()" in text
|
||||
assert "AWOOOI_BACKUP_DATABASE_URL BACKUP_DATABASE_URL DATABASE_URL" in text
|
||||
assert "無法解析 AWOOOI DATABASE_URL；拒絕使用舊硬編密碼" in text
|
||||
assert "base64 -d" in text
|
||||
assert 'printf \'%s\\n\' "${decoded}"' in text
|
||||
|
||||
|
||||
def test_daily_backup_uses_same_rls_safe_pg_dump_contract_as_frequent_lane() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert "dump_database_with_rls_guard()" in text
|
||||
assert 'PGOPTIONS="-c statement_timeout=0 -c max_parallel_workers_per_gather=0"' in text
|
||||
assert "ALTER TABLE %I.%I ${mode} ROW LEVEL SECURITY" in text
|
||||
assert "trap restore_force_rls EXIT" in text
|
||||
assert 'dump_database_with_rls_guard \\\n "awoooi_prod"' in text
|
||||
|
||||
Reference in New Issue
Block a user