fix(ops): harden 188 startup data recovery gate
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
#!/bin/bash
|
||||
# AWOOOI 重開機自動恢復腳本
|
||||
# 2026-04-04 ogt: 根據實際事故建立,解決 PostgreSQL WAL 損壞 + Docker BoltDB 損壞 + K3s Kine 慢查詢
|
||||
# 2026-04-04 ogt: 根據實際事故建立,處理 container / Docker 啟動順序與 K3s Kine 維護。
|
||||
# 2026-06-26 Codex: PostgreSQL checkpoint/WAL 錯誤改為 fail-closed;
|
||||
# 不在自動啟動腳本內執行 pg_resetwal,避免資料破壞被誤判成恢復。
|
||||
# 部署位置: /usr/local/bin/awoooi-startup.sh (on 192.168.0.188)
|
||||
# systemd unit: /etc/systemd/system/awoooi-startup.service
|
||||
|
||||
@@ -67,7 +69,7 @@ else
|
||||
fi
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# STEP 3: PostgreSQL 修復(若損壞)
|
||||
# STEP 3: PostgreSQL 啟動檢查(資料層錯誤必須人工 gate)
|
||||
# ──────────────────────────────────────────────
|
||||
log "[3/7] 檢查 PostgreSQL..."
|
||||
|
||||
@@ -78,13 +80,11 @@ if ! systemctl is-active postgresql@14-main >/dev/null 2>&1; then
|
||||
fi
|
||||
|
||||
if ! systemctl is-active postgresql@14-main >/dev/null 2>&1; then
|
||||
log "PostgreSQL 啟動失敗,檢查是否 WAL 損壞..."
|
||||
log "PostgreSQL 啟動失敗,檢查是否屬於 checkpoint/WAL 類資料層錯誤..."
|
||||
if journalctl -u postgresql@14-main -n 20 | grep -q "could not locate a valid checkpoint"; then
|
||||
log "⚠️ WAL 損壞!執行 pg_resetwal..."
|
||||
/usr/lib/postgresql/14/bin/pg_resetwal -f /var/lib/postgresql/14/main
|
||||
log "WAL 重置完成,重啟 PostgreSQL..."
|
||||
systemctl start postgresql@14-main
|
||||
sleep 8
|
||||
log "❌ 偵測到 PostgreSQL checkpoint/WAL 錯誤;禁止自動 pg_resetwal。"
|
||||
log "需要 DB owner、備份/restore evidence、maintenance window 與 post-check 後才能人工處理。"
|
||||
exit 1
|
||||
fi
|
||||
systemctl is-active postgresql@14-main && log "✅ PostgreSQL 修復成功" || { log "❌ PostgreSQL 修復失敗"; exit 1; }
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user