From a421d2c5b8c414ef12250430b138d791316cb668 Mon Sep 17 00:00:00 2001 From: OG T Date: Wed, 8 Apr 2026 11:48:39 +0800 Subject: [PATCH] =?UTF-8?q?feat(ops):=20Plan=20A=20docker-health-monitor.s?= =?UTF-8?q?h=20=E2=80=94=20Docker=20=E5=AE=B9=E5=99=A8=E5=81=A5=E5=BA=B7?= =?UTF-8?q?=E7=9B=A3=E6=8E=A7=E8=87=AA=E5=8B=95=E4=BF=AE=E5=BE=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 偵測 unhealthy / exited / dead 容器 - 排除清單: DB(PG/Redis)、Gitea、監控棧 - Prometheus/Grafana/Alertmanager exited → docker start (保護 WAL) - 必須三段式通知: Intent→Action→Result (首席架構師裁示) - HMAC-SHA256 簽章 → AWOOOI API /api/v1/webhooks/custom-alert - Fallback: API down → 直接 Telegram Bot API - 冷卻期 300s，防止重複修復 部署: cron */5 * * * * on 192.168.0.110 + 192.168.0.188 設定: /etc/awoooi-ops/secrets.env Co-Authored-By: Claude Sonnet 4.6 --- scripts/ops/docker-health-monitor.sh | 254 +++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100755 scripts/ops/docker-health-monitor.sh diff --git a/scripts/ops/docker-health-monitor.sh b/scripts/ops/docker-health-monitor.sh new file mode 100755 index 000000000..44493ef13 --- /dev/null +++ b/scripts/ops/docker-health-monitor.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +# docker-health-monitor.sh +# Plan A: Docker 容器健康監控 + 自動修復 +# +# 部署: cron */5 * * * * /opt/awoooi-ops/docker-health-monitor.sh >> /var/log/docker-health-monitor.log 2>&1 +# 設定: /etc/awoooi-ops/secrets.env +# 撰寫: Claude Sonnet 4.6 / 2026-04-08 Asia/Taipei +# 首席架構師裁示: Intent→Action→Result 三段式，禁止靜默修復 + +set -euo pipefail + +# ─── 環境載入 ─────────────────────────────────────────────────────────────── +SECRETS_FILE="/etc/awoooi-ops/secrets.env" +if [[ -f "$SECRETS_FILE" ]]; then + # shellcheck source=/dev/null + source "$SECRETS_FILE" +fi + +: "${AWOOOI_API_URL:=https://awoooi.wooo.work}" +: "${TELEGRAM_BOT_TOKEN:=}" +: "${TELEGRAM_CHAT_ID:=}" +: "${WEBHOOK_HMAC_SECRET:=}" +: "${COOLDOWN_SECONDS:=300}" +: "${LOG_FILE:=/var/log/docker-health-monitor.log}" +: "${COOLDOWN_DIR:=/tmp/docker-health-monitor-cooldown}" + +mkdir -p "$COOLDOWN_DIR" + +# ─── 排除清單（禁止自動修復）─────────────────────────────────────────────── +# 判斷方式: echo ":list:" | grep -q ":name:" +# 分類一：資料庫 — 禁止 restart +EXCLUDED_DB_LIST=":postgres:momo-db:langfuse-db:harbor-db:sentry-postgres:signoz-clickhouse:" +# 分類二：Redis — 禁止 restart +EXCLUDED_REDIS_LIST=":redis:harbor-redis:sentry-redis:" +# 分類三：監控棧 exited → docker start（保護 WAL） +MONITORING_START_ONLY_LIST=":prometheus:grafana:alertmanager:" +# 分類四：監控棧 其他 → 僅告警 +EXCLUDED_MONITORING_LIST=":blackbox-exporter:signoz-otel-collector:" +# 分類五：關鍵系統 — 永遠禁止（Gitea restart 會殺活躍 SSH） +EXCLUDED_CRITICAL_LIST=":gitea:" + +# ─── 工具函數 ──────────────────────────────────────────────────────────────── +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S %z')] $*" +} + +in_list() { + local name=":${1}:" + local list="$2" + [[ "$list" == *"$name"* ]] +} + +# 計算 HMAC-SHA256 簽章 +sign_payload() { + local payload="$1" + printf '%s' "$payload" | openssl dgst -sha256 -hmac "$WEBHOOK_HMAC_SECRET" -binary | xxd -p -c 256 +} + +# 傳送 Telegram（Fallback：AWOOOI API down 時直接呼叫 Bot API） +send_telegram_direct() { + local message="$1" + if [[ -z "$TELEGRAM_BOT_TOKEN" || -z "$TELEGRAM_CHAT_ID" ]]; then + log "WARN: Telegram 未設定，跳過通知" + return 0 + fi + curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ + -H "Content-Type: application/json" \ + -d "{\"chat_id\":\"${TELEGRAM_CHAT_ID}\",\"text\":\"${message}\",\"parse_mode\":\"HTML\"}" \ + > /dev/null 2>&1 || true +} + +# 傳送 AWOOOI Webhook（若失敗則 Fallback 至 Telegram Bot API） +send_awoooi_alert() { + local title="$1" + local message="$2" + local severity="${3:-WARNING}" + local source="docker-health-monitor" + + local payload + payload=$(printf '{"title":"%s","message":"%s","severity":"%s","source":"%s","labels":{"monitor":"docker-health-monitor","plan":"A"}}' \ + "$title" "$message" "$severity" "$source") + + local timestamp + timestamp=$(date -u +%s) + local signature + signature=$(sign_payload "${timestamp}${payload}") + + local http_code + http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + -X POST "${AWOOOI_API_URL}/api/v1/webhooks/custom-alert" \ + -H "Content-Type: application/json" \ + -H "X-Timestamp: ${timestamp}" \ + -H "X-Signature: sha256=${signature}" \ + -d "$payload" \ + --connect-timeout 10 \ + --max-time 30 2>/dev/null) || http_code="0" + + if [[ "$http_code" != "200" && "$http_code" != "202" ]]; then + log "WARN: AWOOOI API 回應 ${http_code}，Fallback 到 Telegram Bot API" + send_telegram_direct "[docker-health-monitor Fallback] ${title} ${message}" + fi +} + +# 冷卻期檢查（避免同一容器短時間重複修復） +is_in_cooldown() { + local container="$1" + local cooldown_file="${COOLDOWN_DIR}/${container}.cooldown" + if [[ -f "$cooldown_file" ]]; then + local last_repair + last_repair=$(cat "$cooldown_file") + local now + now=$(date +%s) + local elapsed=$(( now - last_repair )) + if (( elapsed < COOLDOWN_SECONDS )); then + log "COOLDOWN: ${container} 仍在冷卻期 (${elapsed}s / ${COOLDOWN_SECONDS}s)" + return 0 + fi + fi + return 1 +} + +set_cooldown() { + local container="$1" + date +%s > "${COOLDOWN_DIR}/${container}.cooldown" +} + +# ─── 核心：處理不健康容器 ─────────────────────────────────────────────────── +handle_unhealthy_container() { + local container="$1" + local status="$2" # unhealthy | exited | dead + local hostname + hostname=$(hostname) + + log "DETECTED: ${container} 狀態=${status} on ${hostname}" + + # ── 排除清單判斷 ───────────────────────────────────────────────────────── + + if in_list "$container" "$EXCLUDED_CRITICAL_LIST"; then + log "SKIP: ${container} 屬於關鍵排除清單 (Gitea)，僅告警" + send_awoooi_alert \ + "[${hostname}] 關鍵服務異常: ${container}" \ + "容器 ${container} 狀態=${status}。此服務禁止自動修復，請人工處理！" \ + "CRITICAL" + return + fi + + if in_list "$container" "$EXCLUDED_DB_LIST"; then + log "SKIP: ${container} 屬於資料庫排除清單，僅告警" + send_awoooi_alert \ + "[${hostname}] 資料庫容器異常: ${container}" \ + "容器 ${container} 狀態=${status}。資料庫禁止自動修復，需人工介入！" \ + "CRITICAL" + return + fi + + if in_list "$container" "$EXCLUDED_REDIS_LIST"; then + log "SKIP: ${container} 屬於 Redis 排除清單，僅告警" + send_awoooi_alert \ + "[${hostname}] Redis 容器異常: ${container}" \ + "容器 ${container} 狀態=${status}。Redis 禁止自動修復，需人工介入！" \ + "CRITICAL" + return + fi + + if in_list "$container" "$EXCLUDED_MONITORING_LIST"; then + log "SKIP: ${container} 屬於監控棧排除清單，僅告警" + send_awoooi_alert \ + "[${hostname}] 監控元件異常: ${container}" \ + "容器 ${container} 狀態=${status}。請人工處理。" \ + "WARNING" + return + fi + + # ── 冷卻期判斷 ──────────────────────────────────────────────────────────── + if is_in_cooldown "$container"; then + log "SKIP: ${container} 在冷卻期內，跳過本次修復" + return + fi + + # ── 決定修復動作 ───────────────────────────────────────────────────────── + local action_cmd="docker restart" + local action_desc="docker restart" + if in_list "$container" "$MONITORING_START_ONLY_LIST" && [[ "$status" == "exited" ]]; then + action_cmd="docker start" + action_desc="docker start（保護 WAL，非 restart）" + fi + + # ── Phase 1: Intent（決策意圖通知）────────────────────────────────────── + log "INTENT: 即將對 ${container} 執行 ${action_desc}" + send_awoooi_alert \ + "[${hostname}] 自動修復 Intent: ${container}" \ + "偵測到容器 ${container} 狀態=${status}。即將執行 ${action_desc}，2 秒後開始修復。" \ + "WARNING" + + sleep 2 + + # ── Phase 2: Action（執行修復）────────────────────────────────────────── + log "ACTION: 執行 ${action_cmd} ${container}" + set_cooldown "$container" + + local repair_ok=false + if $action_cmd "$container" >> "$LOG_FILE" 2>&1; then + repair_ok=true + fi + + # ── Phase 3: Result（執行結果通知）────────────────────────────────────── + if $repair_ok; then + log "RESULT: ${container} 修復成功" + send_awoooi_alert \ + "[${hostname}] 自動修復成功: ${container}" \ + "容器 ${container} 已透過 ${action_desc} 成功恢復。原狀態=${status}。" \ + "INFO" + else + log "RESULT: ${container} 修復失敗！需人工介入" + send_awoooi_alert \ + "[${hostname}] 自動修復失敗: ${container}" \ + "容器 ${container} 執行 ${action_desc} 失敗！原狀態=${status}。需人工介入！" \ + "CRITICAL" + fi +} + +# ─── 主流程 ────────────────────────────────────────────────────────────────── +main() { + log "===== docker-health-monitor 啟動 (host=$(hostname)) =====" + + # 取得所有容器狀態 + # docker ps -a 格式: Names / Health / State + while IFS=$'\t' read -r name health_status container_status; do + [[ -z "$name" ]] && continue + + local needs_repair=false + local detected_status="" + + if [[ "$health_status" == "unhealthy" ]]; then + needs_repair=true + detected_status="unhealthy" + elif [[ "$container_status" == "exited" || "$container_status" == "dead" ]]; then + needs_repair=true + detected_status="$container_status" + elif [[ "$health_status" == "starting" ]]; then + log "INFO: ${name} health=starting，等待中跳過" + continue + fi + + if $needs_repair; then + handle_unhealthy_container "$name" "$detected_status" + fi + + done < <(docker ps -a --format '{{.Names}} {{.Health}} {{.State}}' 2>/dev/null) + + log "===== docker-health-monitor 完成 =====" +} + +main "$@"