b191f8e9fe
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 2m6s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
237 lines
8.9 KiB
Bash
Executable File
237 lines
8.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# docker-health-monitor.sh
|
||
# Sprint 5.2 Plan A: 感知 + 自動修復 + 回報
|
||
#
|
||
# 部署: cron */5 * * * * /opt/awoooi-ops/docker-health-monitor.sh >> /var/log/docker-health-monitor.log 2>&1
|
||
# 設定: /etc/awoooi-ops/secrets.env
|
||
# 撰寫: Claude Sonnet 4.6 / 2026-04-09 Asia/Taipei
|
||
#
|
||
# 修復分級 (ADR-060 統帥裁示):
|
||
# 自動 docker restart: 一般應用容器 (非 DB/Redis/監控棧)
|
||
# 自動 docker start: prometheus / grafana / alertmanager (保護 WAL)
|
||
# 僅告警 (禁止重啟): postgres / redis / clickhouse / DB 類
|
||
#
|
||
# 流程: Detect → AutoRepair → Report (Intent→Action→Result 三段)
|
||
|
||
set -euo pipefail
|
||
|
||
# ─── 環境載入 ───────────────────────────────────────────────────────────────
|
||
SECRETS_FILE="/etc/awoooi-ops/secrets.env"
|
||
if [[ -f "$SECRETS_FILE" ]]; then
|
||
# shellcheck source=/dev/null
|
||
source "$SECRETS_FILE"
|
||
fi
|
||
|
||
: "${AWOOOI_API_URL:=https://awoooi.wooo.work}"
|
||
: "${SRE_GROUP_CHAT_ID:=-1003711974679}"
|
||
: "${LOG_FILE:=/var/log/docker-health-monitor.log}"
|
||
: "${SEND_COOLDOWN_SECONDS:=300}"
|
||
: "${ACTION_COOLDOWN_SECONDS:=${SEND_COOLDOWN_SECONDS}}"
|
||
: "${COOLDOWN_DIR:=/tmp/docker-health-monitor-cooldown}"
|
||
: "${EXCLUDE_CONTAINERS:=signoz-telemetrystore-migrator,signoz-clickhouse,signoz-init-clickhouse,gitea-runner,vtuber-web,vtuber-admin,vtuber-api,vtuber-db,bitan-pharmacy-bitan-1}"
|
||
|
||
mkdir -p "$COOLDOWN_DIR"
|
||
|
||
# ─── 禁止自動重啟的容器 (模式匹配) ─────────────────────────────────────────
|
||
# DB / Cache / 監控棧核心 — 僅告警，不自動重啟
|
||
READONLY_PATTERNS=(
|
||
"postgres" "momo-db" "langfuse-db" "harbor-db" "sentry-postgres"
|
||
"redis" "harbor-redis" "sentry-redis" "signoz-clickhouse"
|
||
)
|
||
|
||
# 監控棧容器：用 docker start（非 restart），保護 WAL
|
||
SAFE_START_PATTERNS=(
|
||
"prometheus" "grafana" "alertmanager"
|
||
)
|
||
|
||
# ─── 工具函數 ────────────────────────────────────────────────────────────────
|
||
log() {
|
||
echo "[$(date '+%Y-%m-%d %H:%M:%S %z')] $*"
|
||
}
|
||
|
||
is_in_cooldown() {
|
||
local container="$1"
|
||
local cooldown_file="${COOLDOWN_DIR}/${container}.cooldown"
|
||
if [[ -f "$cooldown_file" ]]; then
|
||
local last_sent now elapsed
|
||
last_sent=$(cat "$cooldown_file")
|
||
now=$(date +%s)
|
||
elapsed=$(( now - last_sent ))
|
||
if (( elapsed < ACTION_COOLDOWN_SECONDS )); then
|
||
log "COOLDOWN: ${container} 距上次處理 ${elapsed}s，跳過（處理冷卻 ${ACTION_COOLDOWN_SECONDS}s）"
|
||
return 0
|
||
fi
|
||
fi
|
||
return 1
|
||
}
|
||
|
||
set_cooldown() {
|
||
local container="$1"
|
||
date +%s > "${COOLDOWN_DIR}/${container}.cooldown"
|
||
}
|
||
|
||
# 判斷容器是否符合模式清單
|
||
matches_pattern() {
|
||
local name="$1"
|
||
shift
|
||
local patterns=("$@")
|
||
for pattern in "${patterns[@]}"; do
|
||
case "$name" in
|
||
*"${pattern}"*) return 0 ;;
|
||
esac
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# ─── 回報到 AWOOOI API ───────────────────────────────────────────────────────
|
||
report_to_awoooi() {
|
||
local container="$1"
|
||
local detected_status="$2" # unhealthy | exited | dead
|
||
local repair_action="$3" # restarted | started | alert_only | failed
|
||
local repair_result="$4" # success | failed | skipped
|
||
local hostname
|
||
hostname=$(hostname)
|
||
local now_ts
|
||
now_ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
||
|
||
# auto_repair label 根據實際動作設定
|
||
local auto_repair_label="false"
|
||
[[ "$repair_result" == "success" ]] && auto_repair_label="true"
|
||
|
||
local payload
|
||
payload=$(cat <<JSON
|
||
{
|
||
"version": "4",
|
||
"groupKey": "docker-health-${hostname}-${container}",
|
||
"status": "firing",
|
||
"alerts": [{
|
||
"status": "firing",
|
||
"labels": {
|
||
"alertname": "DockerContainerUnhealthy",
|
||
"container": "${container}",
|
||
"host": "${hostname}",
|
||
"layer": "docker",
|
||
"severity": "warning",
|
||
"auto_repair": "${auto_repair_label}",
|
||
"repair_action": "${repair_action}",
|
||
"repair_result": "${repair_result}",
|
||
"source": "docker-health-monitor"
|
||
},
|
||
"annotations": {
|
||
"summary": "容器 ${container} 狀態=${detected_status} → 修復=${repair_action}(${repair_result})",
|
||
"description": "主機 ${hostname} | 偵測=${detected_status} | 動作=${repair_action} | 結果=${repair_result}"
|
||
},
|
||
"startsAt": "${now_ts}"
|
||
}]
|
||
}
|
||
JSON
|
||
)
|
||
|
||
local http_code
|
||
http_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
||
-X POST "${AWOOOI_API_URL}/api/v1/webhooks/alertmanager" \
|
||
-H "Content-Type: application/json" \
|
||
-d "$payload" \
|
||
--connect-timeout 10 \
|
||
--max-time 30 2>/dev/null) || http_code="0"
|
||
|
||
if [[ "$http_code" == "200" || "$http_code" == "202" ]]; then
|
||
log "REPORTED: ${container} repair=${repair_action}(${repair_result}) → API (${http_code})"
|
||
else
|
||
log "WARN: API 回應 ${http_code}；direct Telegram fallback disabled，事件保留在本地 log 等 AI log ingestion"
|
||
fi
|
||
}
|
||
|
||
# ─── 自動修復邏輯 ────────────────────────────────────────────────────────────
|
||
attempt_repair() {
|
||
local container="$1"
|
||
local detected_status="$2"
|
||
|
||
# 1. 禁止重啟清單
|
||
if matches_pattern "$container" "${READONLY_PATTERNS[@]}"; then
|
||
log "ALERT_ONLY: ${container} 在禁止重啟清單，僅告警"
|
||
report_to_awoooi "$container" "$detected_status" "alert_only" "skipped"
|
||
return
|
||
fi
|
||
|
||
# 2. 監控棧 — 用 docker start（保護 WAL）
|
||
if matches_pattern "$container" "${SAFE_START_PATTERNS[@]}"; then
|
||
log "SAFE_START: ${container} 屬監控棧，執行 docker start"
|
||
if docker start "$container" >> /dev/null 2>&1; then
|
||
log "SUCCESS: docker start ${container}"
|
||
report_to_awoooi "$container" "$detected_status" "started" "success"
|
||
else
|
||
log "FAILED: docker start ${container}"
|
||
report_to_awoooi "$container" "$detected_status" "started" "failed"
|
||
fi
|
||
return
|
||
fi
|
||
|
||
# 3. 一般容器 — docker restart
|
||
log "AUTO_REPAIR: docker restart ${container}"
|
||
if docker restart "$container" >> /dev/null 2>&1; then
|
||
log "SUCCESS: docker restart ${container}"
|
||
report_to_awoooi "$container" "$detected_status" "restarted" "success"
|
||
else
|
||
log "FAILED: docker restart ${container}"
|
||
report_to_awoooi "$container" "$detected_status" "restarted" "failed"
|
||
fi
|
||
}
|
||
|
||
# ─── 核心：掃描所有容器 ─────────────────────────────────────────────────────
|
||
check_containers() {
|
||
local hostname
|
||
hostname=$(hostname)
|
||
|
||
while IFS=$'\t' read -r container_id container_name state health; do
|
||
[[ -z "$container_name" ]] && continue
|
||
|
||
# 排除清單
|
||
local excluded=false
|
||
IFS=',' read -ra EXCLUDES <<< "$EXCLUDE_CONTAINERS"
|
||
for pattern in "${EXCLUDES[@]}"; do
|
||
pattern="${pattern// /}"
|
||
[[ -z "$pattern" ]] && continue
|
||
# shellcheck disable=SC2254
|
||
case "$container_name" in
|
||
$pattern) excluded=true; break ;;
|
||
esac
|
||
done
|
||
$excluded && continue
|
||
|
||
local needs_action=false
|
||
local detected_status=""
|
||
|
||
if [[ "$state" == "exited" || "$state" == "dead" ]]; then
|
||
needs_action=true
|
||
detected_status="$state"
|
||
fi
|
||
if [[ "$health" == "unhealthy" ]]; then
|
||
needs_action=true
|
||
detected_status="unhealthy"
|
||
fi
|
||
|
||
if $needs_action; then
|
||
log "DETECTED: ${container_name} state=${state} health=${health} on ${hostname}"
|
||
is_in_cooldown "$container_name" && continue
|
||
set_cooldown "$container_name"
|
||
attempt_repair "$container_name" "$detected_status"
|
||
fi
|
||
done < <(docker ps -a --format '{{.ID}}\t{{.Names}}\t{{.State}}\t{{.Status}}' | \
|
||
awk -F'\t' '{
|
||
health = ""
|
||
if ($4 ~ /\(unhealthy\)/) health = "unhealthy"
|
||
else if ($4 ~ /\(healthy\)/) health = "healthy"
|
||
print $1 "\t" $2 "\t" $3 "\t" health
|
||
}')
|
||
}
|
||
|
||
# ─── Main ───────────────────────────────────────────────────────────────────
|
||
main() {
|
||
log "=== docker-health-monitor 啟動 (自動修復模式) on $(hostname) ==="
|
||
check_containers
|
||
log "=== 掃描完成 ==="
|
||
}
|
||
|
||
main "$@"
|