1c3c7279ac
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
235 lines
10 KiB
Bash
235 lines
10 KiB
Bash
#!/bin/bash
|
||
# AWOOOI 重開機自動恢復腳本
|
||
# 2026-04-04 ogt: 根據實際事故建立，處理 container / Docker 啟動順序與 K3s Kine 維護。
|
||
# 2026-06-26 Codex: PostgreSQL checkpoint/WAL 錯誤改為 fail-closed；
|
||
# 不在自動啟動腳本內執行 pg_resetwal，避免資料破壞被誤判成恢復。
|
||
# 2026-06-26 Codex: 允許受控 recovery container 提供 14/main runtime；
|
||
# 不再因 systemd postgresql@14-main failed 而誤判活 DB 為不可用。
|
||
# 2026-06-30 Codex: ClawBot rebuild 改為 bounded / opt-in，避免重開機
|
||
# startup oneshot 被單一 image rebuild timeout 拖成 failed。
|
||
# 部署位置: /usr/local/bin/awoooi-startup.sh (on 192.168.0.188)
|
||
# systemd unit: /etc/systemd/system/awoooi-startup.service
|
||
|
||
set -uo pipefail
|
||
LOG="/var/log/awoooi-startup.log"
|
||
exec > >(tee -a "$LOG") 2>&1
|
||
|
||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
||
|
||
CLAWBOT_UP_TIMEOUT_SECONDS="${CLAWBOT_UP_TIMEOUT_SECONDS:-90}"
|
||
CLAWBOT_BUILD_TIMEOUT_SECONDS="${CLAWBOT_BUILD_TIMEOUT_SECONDS:-120}"
|
||
CLAWBOT_STARTUP_REBUILD_ALLOWED="${CLAWBOT_STARTUP_REBUILD_ALLOWED:-0}"
|
||
|
||
run_bounded() {
|
||
seconds="$1"
|
||
shift
|
||
if command -v timeout >/dev/null 2>&1; then
|
||
timeout "${seconds}s" "$@"
|
||
else
|
||
"$@"
|
||
fi
|
||
}
|
||
|
||
postgres_runtime_ready() {
|
||
if systemctl is-active postgresql@14-main >/dev/null 2>&1; then
|
||
log "✅ PostgreSQL systemd unit active"
|
||
return 0
|
||
fi
|
||
|
||
if docker inspect -f '{{.State.Running}} {{.HostConfig.NetworkMode}}' k3s-postgres-recovery 2>/dev/null | grep -q '^true host$'; then
|
||
if pg_isready -h localhost -p 5432 >/dev/null 2>&1; then
|
||
log "✅ PostgreSQL recovery container active on host network"
|
||
return 0
|
||
fi
|
||
fi
|
||
|
||
return 1
|
||
}
|
||
|
||
log "=== AWOOOI 啟動序列開始 ==="
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 1: containerd 修復（若損壞）
|
||
# ──────────────────────────────────────────────
|
||
log "[1/7] 檢查 containerd..."
|
||
|
||
if ! systemctl is-active containerd >/dev/null 2>&1; then
|
||
log "containerd 未啟動，嘗試啟動..."
|
||
systemctl start containerd || true
|
||
sleep 5
|
||
fi
|
||
|
||
if ! systemctl is-active containerd >/dev/null 2>&1; then
|
||
log "containerd 啟動失敗，檢查 BoltDB 損壞..."
|
||
BOLT_DB="/var/lib/containerd/io.containerd.metadata.v1.bolt/meta.db"
|
||
if [ -f "$BOLT_DB" ]; then
|
||
log "備份並刪除損壞的 meta.db..."
|
||
cp "$BOLT_DB" "${BOLT_DB}.bak.$(date +%Y%m%d%H%M%S)"
|
||
rm -f "$BOLT_DB"
|
||
fi
|
||
systemctl start containerd
|
||
sleep 5
|
||
systemctl is-active containerd && log "✅ containerd 修復成功" || { log "❌ containerd 修復失敗"; exit 1; }
|
||
else
|
||
log "✅ containerd 已 active"
|
||
fi
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 2: Docker 修復（若損壞）
|
||
# ──────────────────────────────────────────────
|
||
log "[2/7] 檢查 Docker..."
|
||
|
||
if ! systemctl is-active docker >/dev/null 2>&1; then
|
||
log "Docker 未啟動，嘗試啟動..."
|
||
systemctl start docker || true
|
||
sleep 8
|
||
fi
|
||
|
||
if ! systemctl is-active docker >/dev/null 2>&1; then
|
||
log "Docker 啟動失敗，修復 network BoltDB..."
|
||
NETWORK_DB="/var/lib/docker/network/files/local-kv.db"
|
||
if [ -f "$NETWORK_DB" ]; then
|
||
log "備份並刪除損壞的 local-kv.db..."
|
||
cp "$NETWORK_DB" "${NETWORK_DB}.bak.$(date +%Y%m%d%H%M%S)"
|
||
rm -f "$NETWORK_DB"
|
||
fi
|
||
systemctl restart containerd
|
||
sleep 5
|
||
systemctl start docker
|
||
sleep 8
|
||
systemctl is-active docker && log "✅ Docker 修復成功" || { log "❌ Docker 修復失敗"; exit 1; }
|
||
else
|
||
log "✅ Docker 已 active"
|
||
fi
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 3: PostgreSQL 啟動檢查（資料層錯誤必須人工 gate）
|
||
# ──────────────────────────────────────────────
|
||
log "[3/7] 檢查 PostgreSQL..."
|
||
|
||
if ! postgres_runtime_ready; then
|
||
log "PostgreSQL 未啟動，嘗試啟動..."
|
||
systemctl start postgresql@14-main || true
|
||
sleep 8
|
||
fi
|
||
|
||
if ! postgres_runtime_ready; then
|
||
log "PostgreSQL 啟動失敗，檢查是否屬於 checkpoint/WAL 類資料層錯誤..."
|
||
if journalctl -u postgresql@14-main -n 20 | grep -q "could not locate a valid checkpoint"; then
|
||
log "❌ 偵測到 PostgreSQL checkpoint/WAL 錯誤；禁止自動 pg_resetwal。"
|
||
log "需要 DB owner、備份/restore evidence、maintenance window 與 post-check 後才能人工處理。"
|
||
exit 1
|
||
fi
|
||
postgres_runtime_ready && log "✅ PostgreSQL runtime 可用" || { log "❌ PostgreSQL 修復失敗"; exit 1; }
|
||
fi
|
||
|
||
# 等待 PG 接受連線
|
||
log "等待 PostgreSQL 就緒..."
|
||
for i in $(seq 1 30); do
|
||
pg_isready -h localhost -p 5432 >/dev/null 2>&1 && break
|
||
sleep 2
|
||
done
|
||
pg_isready -h localhost -p 5432 && log "✅ PostgreSQL accepting connections" || { log "❌ PostgreSQL 無法接受連線"; exit 1; }
|
||
|
||
# kine 表維護（若有 stale 連線或 WAL 剛重置）
|
||
log "維護 k3s_datastore kine 表..."
|
||
sudo -u postgres psql -d k3s_datastore -c "
|
||
SELECT pg_terminate_backend(pid)
|
||
FROM pg_stat_activity
|
||
WHERE datname='k3s_datastore' AND pid!=pg_backend_pid() AND state='active'
|
||
AND query_start < now() - interval '5 minutes';
|
||
" 2>/dev/null || true
|
||
|
||
sudo -u postgres psql -d k3s_datastore -c "VACUUM ANALYZE kine;" 2>/dev/null && log "✅ kine VACUUM ANALYZE 完成" || log "⚠️ kine VACUUM 跳過（可能無法連線）"
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 4: Redis
|
||
# ──────────────────────────────────────────────
|
||
log "[4/7] 啟動 Redis..."
|
||
systemctl start redis-server || true
|
||
sleep 3
|
||
redis-cli ping 2>/dev/null | grep -q PONG && log "✅ Redis UP" || log "⚠️ Redis 可能未就緒"
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 5: Ollama
|
||
# ──────────────────────────────────────────────
|
||
log "[5/7] 啟動 Ollama..."
|
||
systemctl start ollama || true
|
||
# Ollama 載入模型需要時間，不立刻驗證
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 6: Nginx
|
||
# ──────────────────────────────────────────────
|
||
log "[6/7] 啟動 Nginx..."
|
||
systemctl start nginx || true
|
||
systemctl is-active nginx >/dev/null 2>&1 && log "✅ Nginx UP" || log "⚠️ Nginx 未就緒"
|
||
|
||
# ──────────────────────────────────────────────
|
||
# STEP 7: Docker Compose 服務
|
||
# ──────────────────────────────────────────────
|
||
log "[7/7] 啟動 Docker Compose 服務..."
|
||
|
||
# SignOz
|
||
SIGNOZ_DIR="/home/ollama/signoz/deploy/docker"
|
||
if [ -f "$SIGNOZ_DIR/docker-compose.yaml" ]; then
|
||
log "啟動 SignOz..."
|
||
cd "$SIGNOZ_DIR"
|
||
docker compose up -d 2>&1 | tail -3
|
||
log "✅ SignOz 啟動指令已發送"
|
||
else
|
||
log "⚠️ 找不到 SignOz compose 檔案"
|
||
fi
|
||
|
||
# MinIO（Velero 備份存儲）
|
||
# 2026-04-05 Claude Code: 加入 MinIO — 解決 Velero BSL Unavailable 問題
|
||
MINIO_DIR="/home/ollama/minio"
|
||
if [ -f "$MINIO_DIR/docker-compose.yml" ]; then
|
||
log "啟動 MinIO (Velero 備份存儲)..."
|
||
cd "$MINIO_DIR"
|
||
docker compose up -d 2>&1 | tail -3
|
||
log "✅ MinIO 啟動指令已發送"
|
||
else
|
||
log "⚠️ 找不到 MinIO compose 檔案"
|
||
fi
|
||
|
||
# ClawBot（依賴 aiops-network）
|
||
# 確保 aiops-network 存在（重開機後 external network 需手動建立）
|
||
if ! docker network ls | grep -q aiops-network; then
|
||
log "建立 aiops-network..."
|
||
docker network create aiops-network && log "✅ aiops-network 建立" || log "⚠️ aiops-network 建立失敗"
|
||
fi
|
||
|
||
CLAWBOT_DIR="/home/ollama/clawbot-v5"
|
||
if [ -f "$CLAWBOT_DIR/docker-compose.yml" ]; then
|
||
log "啟動 ClawBot..."
|
||
cd "$CLAWBOT_DIR"
|
||
if ! run_bounded "$CLAWBOT_UP_TIMEOUT_SECONDS" docker compose up -d 2>&1 | tee /tmp/clawbot-up.log | grep -q "Started\|Running\|healthy"; then
|
||
log "⚠️ ClawBot 啟動未在 ${CLAWBOT_UP_TIMEOUT_SECONDS}s 內確認；交由 post-check 判定"
|
||
if [ "$CLAWBOT_STARTUP_REBUILD_ALLOWED" = "1" ]; then
|
||
log "ClawBot bounded rebuild 已允許，最多等待 ${CLAWBOT_BUILD_TIMEOUT_SECONDS}s..."
|
||
run_bounded "$CLAWBOT_BUILD_TIMEOUT_SECONDS" docker compose build 2>&1 | tail -5 || log "⚠️ ClawBot bounded rebuild 未完成"
|
||
run_bounded "$CLAWBOT_UP_TIMEOUT_SECONDS" docker compose up -d 2>&1 | tail -3 || log "⚠️ ClawBot rebuild 後啟動仍未確認"
|
||
else
|
||
log "⚠️ ClawBot rebuild 預設停用，避免拖垮 10 分鐘 post-reboot SLO"
|
||
fi
|
||
fi
|
||
if docker inspect -f '{{.State.Running}}' clawbot 2>/dev/null | grep -q '^true$'; then
|
||
log "✅ ClawBot container running"
|
||
else
|
||
log "⚠️ ClawBot container 未確認 running；post-reboot verifier 會標示服務 blocker"
|
||
fi
|
||
log "✅ ClawBot 啟動指令已發送"
|
||
else
|
||
log "⚠️ 找不到 ClawBot compose 檔案"
|
||
fi
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 完成
|
||
# ──────────────────────────────────────────────
|
||
log "=== AWOOOI 啟動序列完成 ==="
|
||
# 最後永遠 exit 0，不讓 systemd 認為失敗
|
||
log "K3s 需在 120/121 手動確認啟動（或由 k3s.service 自動啟動）"
|
||
log "詳細 SOP: docs/runbooks/REBOOT-RECOVERY-SOP.md"
|
||
|
||
exit 0
|