feat(ops): Plan A docker-health-monitor.sh — Docker 容器健康監控自動修復
- 偵測 unhealthy / exited / dead 容器 - 排除清單: DB(PG/Redis)、Gitea、監控棧 - Prometheus/Grafana/Alertmanager exited → docker start (保護 WAL) - 必須三段式通知: Intent→Action→Result (首席架構師裁示) - HMAC-SHA256 簽章 → AWOOOI API /api/v1/webhooks/custom-alert - Fallback: API down → 直接 Telegram Bot API - 冷卻期 300s，防止重複修復 部署: cron */5 * * * * on 192.168.0.110 + 192.168.0.188 設定: /etc/awoooi-ops/secrets.env Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OG T
254
scripts/ops/docker-health-monitor.sh
Executable file
254
scripts/ops/docker-health-monitor.sh
Executable file
@@ -0,0 +1,254 @@
|
||||
#!/usr/bin/env bash
|
||||
# docker-health-monitor.sh
|
||||
# Plan A: Docker 容器健康監控 + 自動修復
|
||||
#
|
||||
# 部署: cron */5 * * * * /opt/awoooi-ops/docker-health-monitor.sh >> /var/log/docker-health-monitor.log 2>&1
|
||||
# 設定: /etc/awoooi-ops/secrets.env
|
||||
# 撰寫: Claude Sonnet 4.6 / 2026-04-08 Asia/Taipei
|
||||
# 首席架構師裁示: Intent→Action→Result 三段式，禁止靜默修復
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ─── 環境載入 ───────────────────────────────────────────────────────────────
|
||||
SECRETS_FILE="/etc/awoooi-ops/secrets.env"
|
||||
if [[ -f "$SECRETS_FILE" ]]; then
|
||||
# shellcheck source=/dev/null
|
||||
source "$SECRETS_FILE"
|
||||
fi
|
||||
|
||||
: "${AWOOOI_API_URL:=https://awoooi.wooo.work}"
|
||||
: "${TELEGRAM_BOT_TOKEN:=}"
|
||||
: "${TELEGRAM_CHAT_ID:=}"
|
||||
: "${WEBHOOK_HMAC_SECRET:=}"
|
||||
: "${COOLDOWN_SECONDS:=300}"
|
||||
: "${LOG_FILE:=/var/log/docker-health-monitor.log}"
|
||||
: "${COOLDOWN_DIR:=/tmp/docker-health-monitor-cooldown}"
|
||||
|
||||
mkdir -p "$COOLDOWN_DIR"
|
||||
|
||||
# ─── 排除清單（禁止自動修復）───────────────────────────────────────────────
|
||||
# 判斷方式: echo ":list:" | grep -q ":name:"
|
||||
# 分類一：資料庫 — 禁止 restart
|
||||
EXCLUDED_DB_LIST=":postgres:momo-db:langfuse-db:harbor-db:sentry-postgres:signoz-clickhouse:"
|
||||
# 分類二：Redis — 禁止 restart
|
||||
EXCLUDED_REDIS_LIST=":redis:harbor-redis:sentry-redis:"
|
||||
# 分類三：監控棧 exited → docker start（保護 WAL）
|
||||
MONITORING_START_ONLY_LIST=":prometheus:grafana:alertmanager:"
|
||||
# 分類四：監控棧 其他 → 僅告警
|
||||
EXCLUDED_MONITORING_LIST=":blackbox-exporter:signoz-otel-collector:"
|
||||
# 分類五：關鍵系統 — 永遠禁止（Gitea restart 會殺活躍 SSH）
|
||||
EXCLUDED_CRITICAL_LIST=":gitea:"
|
||||
|
||||
# ─── 工具函數 ────────────────────────────────────────────────────────────────
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S %z')] $*"
|
||||
}
|
||||
|
||||
in_list() {
|
||||
local name=":${1}:"
|
||||
local list="$2"
|
||||
[[ "$list" == *"$name"* ]]
|
||||
}
|
||||
|
||||
# 計算 HMAC-SHA256 簽章
|
||||
sign_payload() {
|
||||
local payload="$1"
|
||||
printf '%s' "$payload" | openssl dgst -sha256 -hmac "$WEBHOOK_HMAC_SECRET" -binary | xxd -p -c 256
|
||||
}
|
||||
|
||||
# 傳送 Telegram（Fallback：AWOOOI API down 時直接呼叫 Bot API）
|
||||
send_telegram_direct() {
|
||||
local message="$1"
|
||||
if [[ -z "$TELEGRAM_BOT_TOKEN" || -z "$TELEGRAM_CHAT_ID" ]]; then
|
||||
log "WARN: Telegram 未設定，跳過通知"
|
||||
return 0
|
||||
fi
|
||||
curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"chat_id\":\"${TELEGRAM_CHAT_ID}\",\"text\":\"${message}\",\"parse_mode\":\"HTML\"}" \
|
||||
> /dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
# 傳送 AWOOOI Webhook（若失敗則 Fallback 至 Telegram Bot API）
|
||||
send_awoooi_alert() {
|
||||
local title="$1"
|
||||
local message="$2"
|
||||
local severity="${3:-WARNING}"
|
||||
local source="docker-health-monitor"
|
||||
|
||||
local payload
|
||||
payload=$(printf '{"title":"%s","message":"%s","severity":"%s","source":"%s","labels":{"monitor":"docker-health-monitor","plan":"A"}}' \
|
||||
"$title" "$message" "$severity" "$source")
|
||||
|
||||
local timestamp
|
||||
timestamp=$(date -u +%s)
|
||||
local signature
|
||||
signature=$(sign_payload "${timestamp}${payload}")
|
||||
|
||||
local http_code
|
||||
http_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||
-X POST "${AWOOOI_API_URL}/api/v1/webhooks/custom-alert" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-Timestamp: ${timestamp}" \
|
||||
-H "X-Signature: sha256=${signature}" \
|
||||
-d "$payload" \
|
||||
--connect-timeout 10 \
|
||||
--max-time 30 2>/dev/null) || http_code="0"
|
||||
|
||||
if [[ "$http_code" != "200" && "$http_code" != "202" ]]; then
|
||||
log "WARN: AWOOOI API 回應 ${http_code}，Fallback 到 Telegram Bot API"
|
||||
send_telegram_direct "[docker-health-monitor Fallback] ${title} ${message}"
|
||||
fi
|
||||
}
|
||||
|
||||
# 冷卻期檢查（避免同一容器短時間重複修復）
|
||||
is_in_cooldown() {
|
||||
local container="$1"
|
||||
local cooldown_file="${COOLDOWN_DIR}/${container}.cooldown"
|
||||
if [[ -f "$cooldown_file" ]]; then
|
||||
local last_repair
|
||||
last_repair=$(cat "$cooldown_file")
|
||||
local now
|
||||
now=$(date +%s)
|
||||
local elapsed=$(( now - last_repair ))
|
||||
if (( elapsed < COOLDOWN_SECONDS )); then
|
||||
log "COOLDOWN: ${container} 仍在冷卻期 (${elapsed}s / ${COOLDOWN_SECONDS}s)"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
set_cooldown() {
|
||||
local container="$1"
|
||||
date +%s > "${COOLDOWN_DIR}/${container}.cooldown"
|
||||
}
|
||||
|
||||
# ─── 核心：處理不健康容器 ───────────────────────────────────────────────────
|
||||
handle_unhealthy_container() {
|
||||
local container="$1"
|
||||
local status="$2" # unhealthy | exited | dead
|
||||
local hostname
|
||||
hostname=$(hostname)
|
||||
|
||||
log "DETECTED: ${container} 狀態=${status} on ${hostname}"
|
||||
|
||||
# ── 排除清單判斷 ─────────────────────────────────────────────────────────
|
||||
|
||||
if in_list "$container" "$EXCLUDED_CRITICAL_LIST"; then
|
||||
log "SKIP: ${container} 屬於關鍵排除清單 (Gitea)，僅告警"
|
||||
send_awoooi_alert \
|
||||
"[${hostname}] 關鍵服務異常: ${container}" \
|
||||
"容器 ${container} 狀態=${status}。此服務禁止自動修復，請人工處理！" \
|
||||
"CRITICAL"
|
||||
return
|
||||
fi
|
||||
|
||||
if in_list "$container" "$EXCLUDED_DB_LIST"; then
|
||||
log "SKIP: ${container} 屬於資料庫排除清單，僅告警"
|
||||
send_awoooi_alert \
|
||||
"[${hostname}] 資料庫容器異常: ${container}" \
|
||||
"容器 ${container} 狀態=${status}。資料庫禁止自動修復，需人工介入！" \
|
||||
"CRITICAL"
|
||||
return
|
||||
fi
|
||||
|
||||
if in_list "$container" "$EXCLUDED_REDIS_LIST"; then
|
||||
log "SKIP: ${container} 屬於 Redis 排除清單，僅告警"
|
||||
send_awoooi_alert \
|
||||
"[${hostname}] Redis 容器異常: ${container}" \
|
||||
"容器 ${container} 狀態=${status}。Redis 禁止自動修復，需人工介入！" \
|
||||
"CRITICAL"
|
||||
return
|
||||
fi
|
||||
|
||||
if in_list "$container" "$EXCLUDED_MONITORING_LIST"; then
|
||||
log "SKIP: ${container} 屬於監控棧排除清單，僅告警"
|
||||
send_awoooi_alert \
|
||||
"[${hostname}] 監控元件異常: ${container}" \
|
||||
"容器 ${container} 狀態=${status}。請人工處理。" \
|
||||
"WARNING"
|
||||
return
|
||||
fi
|
||||
|
||||
# ── 冷卻期判斷 ────────────────────────────────────────────────────────────
|
||||
if is_in_cooldown "$container"; then
|
||||
log "SKIP: ${container} 在冷卻期內，跳過本次修復"
|
||||
return
|
||||
fi
|
||||
|
||||
# ── 決定修復動作 ─────────────────────────────────────────────────────────
|
||||
local action_cmd="docker restart"
|
||||
local action_desc="docker restart"
|
||||
if in_list "$container" "$MONITORING_START_ONLY_LIST" && [[ "$status" == "exited" ]]; then
|
||||
action_cmd="docker start"
|
||||
action_desc="docker start（保護 WAL，非 restart）"
|
||||
fi
|
||||
|
||||
# ── Phase 1: Intent（決策意圖通知）──────────────────────────────────────
|
||||
log "INTENT: 即將對 ${container} 執行 ${action_desc}"
|
||||
send_awoooi_alert \
|
||||
"[${hostname}] 自動修復 Intent: ${container}" \
|
||||
"偵測到容器 ${container} 狀態=${status}。即將執行 ${action_desc}，2 秒後開始修復。" \
|
||||
"WARNING"
|
||||
|
||||
sleep 2
|
||||
|
||||
# ── Phase 2: Action（執行修復）──────────────────────────────────────────
|
||||
log "ACTION: 執行 ${action_cmd} ${container}"
|
||||
set_cooldown "$container"
|
||||
|
||||
local repair_ok=false
|
||||
if $action_cmd "$container" >> "$LOG_FILE" 2>&1; then
|
||||
repair_ok=true
|
||||
fi
|
||||
|
||||
# ── Phase 3: Result（執行結果通知）──────────────────────────────────────
|
||||
if $repair_ok; then
|
||||
log "RESULT: ${container} 修復成功"
|
||||
send_awoooi_alert \
|
||||
"[${hostname}] 自動修復成功: ${container}" \
|
||||
"容器 ${container} 已透過 ${action_desc} 成功恢復。原狀態=${status}。" \
|
||||
"INFO"
|
||||
else
|
||||
log "RESULT: ${container} 修復失敗！需人工介入"
|
||||
send_awoooi_alert \
|
||||
"[${hostname}] 自動修復失敗: ${container}" \
|
||||
"容器 ${container} 執行 ${action_desc} 失敗！原狀態=${status}。需人工介入！" \
|
||||
"CRITICAL"
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── 主流程 ──────────────────────────────────────────────────────────────────
|
||||
main() {
|
||||
log "===== docker-health-monitor 啟動 (host=$(hostname)) ====="
|
||||
|
||||
# 取得所有容器狀態
|
||||
# docker ps -a 格式: Names / Health / State
|
||||
while IFS=$'\t' read -r name health_status container_status; do
|
||||
[[ -z "$name" ]] && continue
|
||||
|
||||
local needs_repair=false
|
||||
local detected_status=""
|
||||
|
||||
if [[ "$health_status" == "unhealthy" ]]; then
|
||||
needs_repair=true
|
||||
detected_status="unhealthy"
|
||||
elif [[ "$container_status" == "exited" || "$container_status" == "dead" ]]; then
|
||||
needs_repair=true
|
||||
detected_status="$container_status"
|
||||
elif [[ "$health_status" == "starting" ]]; then
|
||||
log "INFO: ${name} health=starting，等待中跳過"
|
||||
continue
|
||||
fi
|
||||
|
||||
if $needs_repair; then
|
||||
handle_unhealthy_container "$name" "$detected_status"
|
||||
fi
|
||||
|
||||
done < <(docker ps -a --format '{{.Names}} {{.Health}} {{.State}}' 2>/dev/null)
|
||||
|
||||
log "===== docker-health-monitor 完成 ====="
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user