254 lines
11 KiB
Bash
254 lines
11 KiB
Bash
#!/bin/bash
|
||
# AWOOOI 重開機自動恢復腳本
|
||
# 2026-04-04 ogt: 根據實際事故建立，處理 container / Docker 啟動順序與 K3s Kine 維護。
|
||
# 2026-06-26 Codex: PostgreSQL checkpoint/WAL 錯誤改為 fail-closed；
|
||
# 不在自動啟動腳本內執行 pg_resetwal，避免資料破壞被誤判成恢復。
|
||
# 2026-06-26 Codex: 允許受控 recovery container 提供 14/main runtime；
|
||
# 不再因 systemd postgresql@14-main failed 而誤判活 DB 為不可用。
|
||
# 2026-06-30 Codex: ClawBot rebuild 改為 bounded / opt-in，避免重開機
|
||
# startup oneshot 被單一 image rebuild timeout 拖成 failed。
|
||
# 部署位置: /usr/local/bin/awoooi-startup.sh (on 192.168.0.188)
|
||
# systemd unit: /etc/systemd/system/awoooi-startup.service
|
||
|
||
set -uo pipefail
|
||
LOG="/var/log/awoooi-startup.log"
|
||
exec > >(tee -a "$LOG") 2>&1
|
||
|
||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
||
|
||
CLAWBOT_UP_TIMEOUT_SECONDS="${CLAWBOT_UP_TIMEOUT_SECONDS:-90}"
|
||
CLAWBOT_BUILD_TIMEOUT_SECONDS="${CLAWBOT_BUILD_TIMEOUT_SECONDS:-120}"
|
||
CLAWBOT_STARTUP_REBUILD_ALLOWED="${CLAWBOT_STARTUP_REBUILD_ALLOWED:-0}"
|
||
CONTAINERD_METADATA_REPAIR_ALLOWED="${CONTAINERD_METADATA_REPAIR_ALLOWED:-0}"
|
||
DOCKER_NETWORK_METADATA_REPAIR_ALLOWED="${DOCKER_NETWORK_METADATA_REPAIR_ALLOWED:-0}"
|
||
KINE_MAINTENANCE_ALLOWED="${KINE_MAINTENANCE_ALLOWED:-0}"
|
||
SIGNOZ_188_START_ALLOWED="${SIGNOZ_188_START_ALLOWED:-0}"
|
||
|
||
run_bounded() {
|
||
seconds="$1"
|
||
shift
|
||
if command -v timeout >/dev/null 2>&1; then
|
||
timeout "${seconds}s" "$@"
|
||
else
|
||
"$@"
|
||
fi
|
||
}
|
||
|
||
postgres_runtime_ready() {
|
||
if systemctl is-active postgresql@14-main >/dev/null 2>&1; then
|
||
log "✅ PostgreSQL systemd unit active"
|
||
return 0
|
||
fi
|
||
|
||
if docker inspect -f '{{.State.Running}} {{.HostConfig.NetworkMode}}' k3s-postgres-recovery 2>/dev/null | grep -q '^true host$'; then
|
||
if pg_isready -h localhost -p 5432 >/dev/null 2>&1; then
|
||
log "✅ PostgreSQL recovery container active on host network"
|
||
return 0
|
||
fi
|
||
fi
|
||
|
||
return 1
|
||
}
|
||
|
||
log "=== AWOOOI 啟動序列開始 ==="
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 1: containerd 修復（若損壞）
|
||
# ──────────────────────────────────────────────
|
||
log "[1/7] 檢查 containerd..."
|
||
|
||
if ! systemctl is-active containerd >/dev/null 2>&1; then
|
||
log "containerd 未啟動，嘗試啟動..."
|
||
systemctl start containerd || true
|
||
sleep 5
|
||
fi
|
||
|
||
if ! systemctl is-active containerd >/dev/null 2>&1; then
|
||
log "containerd 啟動失敗，檢查 BoltDB 損壞..."
|
||
BOLT_DB="/var/lib/containerd/io.containerd.metadata.v1.bolt/meta.db"
|
||
if [ -f "$BOLT_DB" ]; then
|
||
if [ "$CONTAINERD_METADATA_REPAIR_ALLOWED" != "1" ]; then
|
||
log "❌ containerd metadata repair 未授權；禁止自動備份/刪除 meta.db"
|
||
exit 1
|
||
fi
|
||
log "containerd metadata repair 已授權；先備份再移除損壞的 meta.db..."
|
||
cp "$BOLT_DB" "${BOLT_DB}.bak.$(date +%Y%m%d%H%M%S)"
|
||
rm -f "$BOLT_DB"
|
||
fi
|
||
systemctl start containerd
|
||
sleep 5
|
||
systemctl is-active containerd && log "✅ containerd 修復成功" || { log "❌ containerd 修復失敗"; exit 1; }
|
||
else
|
||
log "✅ containerd 已 active"
|
||
fi
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 2: Docker 修復（若損壞）
|
||
# ──────────────────────────────────────────────
|
||
log "[2/7] 檢查 Docker..."
|
||
|
||
if ! systemctl is-active docker >/dev/null 2>&1; then
|
||
log "Docker 未啟動，嘗試啟動..."
|
||
systemctl start docker || true
|
||
sleep 8
|
||
fi
|
||
|
||
if ! systemctl is-active docker >/dev/null 2>&1; then
|
||
log "Docker 啟動失敗，修復 network BoltDB..."
|
||
NETWORK_DB="/var/lib/docker/network/files/local-kv.db"
|
||
if [ -f "$NETWORK_DB" ]; then
|
||
if [ "$DOCKER_NETWORK_METADATA_REPAIR_ALLOWED" != "1" ]; then
|
||
log "❌ Docker network metadata repair 未授權；禁止自動備份/刪除 local-kv.db"
|
||
exit 1
|
||
fi
|
||
log "Docker network metadata repair 已授權；先備份再移除 local-kv.db..."
|
||
cp "$NETWORK_DB" "${NETWORK_DB}.bak.$(date +%Y%m%d%H%M%S)"
|
||
rm -f "$NETWORK_DB"
|
||
fi
|
||
systemctl restart containerd
|
||
sleep 5
|
||
systemctl start docker
|
||
sleep 8
|
||
systemctl is-active docker && log "✅ Docker 修復成功" || { log "❌ Docker 修復失敗"; exit 1; }
|
||
else
|
||
log "✅ Docker 已 active"
|
||
fi
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 3: PostgreSQL 啟動檢查（資料層錯誤必須人工 gate）
|
||
# ──────────────────────────────────────────────
|
||
log "[3/7] 檢查 PostgreSQL..."
|
||
|
||
if ! postgres_runtime_ready; then
|
||
log "PostgreSQL 未啟動，嘗試啟動..."
|
||
systemctl start postgresql@14-main || true
|
||
sleep 8
|
||
fi
|
||
|
||
if ! postgres_runtime_ready; then
|
||
log "PostgreSQL 啟動失敗，檢查是否屬於 checkpoint/WAL 類資料層錯誤..."
|
||
if journalctl -u postgresql@14-main -n 20 | grep -q "could not locate a valid checkpoint"; then
|
||
log "❌ 偵測到 PostgreSQL checkpoint/WAL 錯誤；禁止自動 pg_resetwal。"
|
||
log "需要 DB owner、備份/restore evidence、maintenance window 與 post-check 後才能人工處理。"
|
||
exit 1
|
||
fi
|
||
postgres_runtime_ready && log "✅ PostgreSQL runtime 可用" || { log "❌ PostgreSQL 修復失敗"; exit 1; }
|
||
fi
|
||
|
||
# 等待 PG 接受連線
|
||
log "等待 PostgreSQL 就緒..."
|
||
for i in $(seq 1 30); do
|
||
pg_isready -h localhost -p 5432 >/dev/null 2>&1 && break
|
||
sleep 2
|
||
done
|
||
pg_isready -h localhost -p 5432 && log "✅ PostgreSQL accepting connections" || { log "❌ PostgreSQL 無法接受連線"; exit 1; }
|
||
|
||
# kine 維護不屬於開機恢復的必要寫入，預設停用。
|
||
if [ "$KINE_MAINTENANCE_ALLOWED" = "1" ]; then
|
||
log "kine maintenance 已授權，執行 bounded stale-session cleanup / VACUUM..."
|
||
sudo -u postgres psql -d k3s_datastore -c "
|
||
SELECT pg_terminate_backend(pid)
|
||
FROM pg_stat_activity
|
||
WHERE datname='k3s_datastore' AND pid!=pg_backend_pid() AND state='active'
|
||
AND query_start < now() - interval '5 minutes';
|
||
" 2>/dev/null || true
|
||
sudo -u postgres psql -d k3s_datastore -c "VACUUM ANALYZE kine;" 2>/dev/null \
|
||
&& log "✅ kine VACUUM ANALYZE 完成" \
|
||
|| log "⚠️ kine VACUUM 未完成"
|
||
else
|
||
log "✅ kine maintenance 預設停用，開機恢復不寫入資料庫"
|
||
fi
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 4: Redis
|
||
# ──────────────────────────────────────────────
|
||
log "[4/7] 啟動 Redis..."
|
||
systemctl start redis-server || true
|
||
sleep 3
|
||
redis-cli ping 2>/dev/null | grep -q PONG && log "✅ Redis UP" || log "⚠️ Redis 可能未就緒"
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 5: Ollama
|
||
# ──────────────────────────────────────────────
|
||
log "[5/7] 啟動 Ollama..."
|
||
systemctl start ollama || true
|
||
# Ollama 載入模型需要時間，不立刻驗證
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 6: Nginx
|
||
# ──────────────────────────────────────────────
|
||
log "[6/7] 啟動 Nginx..."
|
||
systemctl start nginx || true
|
||
systemctl is-active nginx >/dev/null 2>&1 && log "✅ Nginx UP" || log "⚠️ Nginx 未就緒"
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 7: Docker Compose 服務
|
||
# ──────────────────────────────────────────────
|
||
log "[7/7] 啟動 Docker Compose 服務..."
|
||
|
||
# SignOz
|
||
SIGNOZ_DIR="/home/ollama/signoz/deploy/docker"
|
||
if [ "$SIGNOZ_188_START_ALLOWED" != "1" ]; then
|
||
log "✅ 188 legacy SignOz startup 預設停用；正式 SignOz 由 110 runtime 提供"
|
||
elif [ -f "$SIGNOZ_DIR/docker-compose.yaml" ]; then
|
||
log "啟動 SignOz..."
|
||
cd "$SIGNOZ_DIR"
|
||
docker compose up -d 2>&1 | tail -3
|
||
log "✅ SignOz 啟動指令已發送"
|
||
else
|
||
log "⚠️ 找不到 SignOz compose 檔案"
|
||
fi
|
||
|
||
# MinIO（Velero 備份存儲）
|
||
# 2026-04-05 Claude Code: 加入 MinIO — 解決 Velero BSL Unavailable 問題
|
||
MINIO_DIR="/home/ollama/minio"
|
||
if [ -f "$MINIO_DIR/docker-compose.yml" ]; then
|
||
log "啟動 MinIO (Velero 備份存儲)..."
|
||
cd "$MINIO_DIR"
|
||
docker compose up -d 2>&1 | tail -3
|
||
log "✅ MinIO 啟動指令已發送"
|
||
else
|
||
log "⚠️ 找不到 MinIO compose 檔案"
|
||
fi
|
||
|
||
# ClawBot（依賴 aiops-network）
|
||
# 確保 aiops-network 存在（重開機後 external network 需手動建立）
|
||
if ! docker network inspect aiops-network >/dev/null 2>&1; then
|
||
log "建立 aiops-network..."
|
||
docker network create aiops-network && log "✅ aiops-network 建立" || log "⚠️ aiops-network 建立失敗"
|
||
fi
|
||
|
||
CLAWBOT_DIR="/home/ollama/clawbot-v5"
|
||
if [ -f "$CLAWBOT_DIR/docker-compose.yml" ]; then
|
||
log "啟動 ClawBot..."
|
||
cd "$CLAWBOT_DIR"
|
||
if ! run_bounded "$CLAWBOT_UP_TIMEOUT_SECONDS" docker compose up -d 2>&1 | tee /tmp/clawbot-up.log | grep -q "Started\|Running\|healthy"; then
|
||
log "⚠️ ClawBot 啟動未在 ${CLAWBOT_UP_TIMEOUT_SECONDS}s 內確認；交由 post-check 判定"
|
||
if [ "$CLAWBOT_STARTUP_REBUILD_ALLOWED" = "1" ]; then
|
||
log "ClawBot bounded rebuild 已允許，最多等待 ${CLAWBOT_BUILD_TIMEOUT_SECONDS}s..."
|
||
run_bounded "$CLAWBOT_BUILD_TIMEOUT_SECONDS" docker compose build 2>&1 | tail -5 || log "⚠️ ClawBot bounded rebuild 未完成"
|
||
run_bounded "$CLAWBOT_UP_TIMEOUT_SECONDS" docker compose up -d 2>&1 | tail -3 || log "⚠️ ClawBot rebuild 後啟動仍未確認"
|
||
else
|
||
log "⚠️ ClawBot rebuild 預設停用，避免拖垮 10 分鐘 post-reboot SLO"
|
||
fi
|
||
fi
|
||
if docker inspect -f '{{.State.Running}}' clawbot 2>/dev/null | grep -q '^true$'; then
|
||
log "✅ ClawBot container running"
|
||
else
|
||
log "⚠️ ClawBot container 未確認 running；post-reboot verifier 會標示服務 blocker"
|
||
fi
|
||
log "✅ ClawBot 啟動指令已發送"
|
||
else
|
||
log "⚠️ 找不到 ClawBot compose 檔案"
|
||
fi
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 完成
|
||
# ──────────────────────────────────────────────
|
||
log "=== AWOOOI 啟動序列完成 ==="
|
||
# 最後永遠 exit 0，不讓 systemd 認為失敗
|
||
log "K3s 需在 120/121 手動確認啟動（或由 k3s.service 自動啟動）"
|
||
log "詳細 SOP: docs/runbooks/REBOOT-RECOVERY-SOP.md"
|
||
|
||
exit 0
|