fix(ops): harden 188 startup data recovery gate
Some checks failed
Code Review / ai-code-review (push) Successful in 13s
Ansible / Reboot Recovery Contract / validate (push) Has been cancelled

This commit is contained in:
ogt
2026-06-26 06:54:49 +08:00
parent 002410e63d
commit 6250a94b7e
5 changed files with 67 additions and 26 deletions

View File

@@ -1,6 +1,8 @@
#!/bin/bash
# AWOOOI 重開機自動恢復腳本
# 2026-04-04 ogt: 根據實際事故建立,解決 PostgreSQL WAL 損壞 + Docker BoltDB 損壞 + K3s Kine 慢查詢
# 2026-04-04 ogt: 根據實際事故建立,處理 container / Docker 啟動順序與 K3s Kine 維護。
# 2026-06-26 Codex: PostgreSQL checkpoint/WAL 錯誤改為 fail-closed
# 不在自動啟動腳本內執行 pg_resetwal避免資料破壞被誤判成恢復。
# 部署位置: /usr/local/bin/awoooi-startup.sh (on 192.168.0.188)
# systemd unit: /etc/systemd/system/awoooi-startup.service
@@ -67,7 +69,7 @@ else
fi
# ──────────────────────────────────────────────
# STEP 3: PostgreSQL 修復(若損壞
# STEP 3: PostgreSQL 啟動檢查(資料層錯誤必須人工 gate
# ──────────────────────────────────────────────
log "[3/7] 檢查 PostgreSQL..."
@@ -78,13 +80,11 @@ if ! systemctl is-active postgresql@14-main >/dev/null 2>&1; then
fi
if ! systemctl is-active postgresql@14-main >/dev/null 2>&1; then
log "PostgreSQL 啟動失敗,檢查是否 WAL 損壞..."
log "PostgreSQL 啟動失敗,檢查是否屬於 checkpoint/WAL 類資料層錯誤..."
if journalctl -u postgresql@14-main -n 20 | grep -q "could not locate a valid checkpoint"; then
log "⚠️ WAL 損壞!執行 pg_resetwal..."
/usr/lib/postgresql/14/bin/pg_resetwal -f /var/lib/postgresql/14/main
log "WAL 重置完成,重啟 PostgreSQL..."
systemctl start postgresql@14-main
sleep 8
log "❌ 偵測到 PostgreSQL checkpoint/WAL 錯誤;禁止自動 pg_resetwal"
log "需要 DB owner、備份/restore evidence、maintenance window 與 post-check 後才能人工處理。"
exit 1
fi
systemctl is-active postgresql@14-main && log "✅ PostgreSQL 修復成功" || { log "❌ PostgreSQL 修復失敗"; exit 1; }
fi