fix(ops): reduce post-reboot notification noise

2026-06-24 06:52:47 +08:00
parent 95f442adab
commit 35a3a59839
6 changed files with 249 additions and 10 deletions
--- a/scripts/ops/deploy-docker-health-monitor.sh
+++ b/scripts/ops/deploy-docker-health-monitor.sh
@@ -88,6 +88,8 @@ AWOOOI_API_URL=https://awoooi.wooo.work
 TELEGRAM_BOT_TOKEN=CHANGE_ME
 SRE_GROUP_CHAT_ID=-1003711974679
 SEND_COOLDOWN_SECONDS=300
+ACTION_COOLDOWN_SECONDS=300
+NOTIFY_COOLDOWN_SECONDS=1800
 SECRETS_TEMPLATE
        echo '  ⚠️  請填寫 /etc/awoooi-ops/secrets.env.template 後重命名為 secrets.env'
    else
--- a/scripts/ops/docker-health-monitor.sh
+++ b/scripts/ops/docker-health-monitor.sh
@@ -25,12 +25,16 @@ fi
 : "${AWOOOI_API_URL:=https://awoooi.wooo.work}"
 : "${TELEGRAM_BOT_TOKEN:=}"
 : "${SRE_GROUP_CHAT_ID:=-1003711974679}"
+: "${TELEGRAM_ALERT_CHAT_ID:=${SRE_GROUP_CHAT_ID:-${TELEGRAM_CHAT_ID:-}}}"
 : "${LOG_FILE:=/var/log/docker-health-monitor.log}"
 : "${SEND_COOLDOWN_SECONDS:=300}"
+: "${ACTION_COOLDOWN_SECONDS:=${SEND_COOLDOWN_SECONDS}}"
+: "${NOTIFY_COOLDOWN_SECONDS:=1800}"
 : "${COOLDOWN_DIR:=/tmp/docker-health-monitor-cooldown}"
-: "${EXCLUDE_CONTAINERS:=signoz-telemetrystore-migrator,signoz-clickhouse,signoz-init-clickhouse,gitea-runner}"
+: "${NOTIFY_COOLDOWN_DIR:=${COOLDOWN_DIR}/notify}"
+: "${EXCLUDE_CONTAINERS:=signoz-telemetrystore-migrator,signoz-clickhouse,signoz-init-clickhouse,gitea-runner,vtuber-web,vtuber-admin,vtuber-api,vtuber-db,bitan-pharmacy-bitan-1}"

-mkdir -p "$COOLDOWN_DIR"
+mkdir -p "$COOLDOWN_DIR" "$NOTIFY_COOLDOWN_DIR"

 # ─── 禁止自動重啟的容器 (模式匹配) ─────────────────────────────────────────
 # DB / Cache / 監控棧核心 — 僅告警，不自動重啟
@@ -57,8 +61,8 @@ is_in_cooldown() {
        last_sent=$(cat "$cooldown_file")
        now=$(date +%s)
        elapsed=$(( now - last_sent ))
-        if (( elapsed < SEND_COOLDOWN_SECONDS )); then
-            log "COOLDOWN: ${container} 距上次處理 ${elapsed}s，跳過（冷卻 ${SEND_COOLDOWN_SECONDS}s）"
+        if (( elapsed < ACTION_COOLDOWN_SECONDS )); then
+            log "COOLDOWN: ${container} 距上次處理 ${elapsed}s，跳過（處理冷卻 ${ACTION_COOLDOWN_SECONDS}s）"
            return 0
        fi
    fi
@@ -70,6 +74,32 @@ set_cooldown() {
    date +%s > "${COOLDOWN_DIR}/${container}.cooldown"
 }

+safe_cooldown_key() {
+    tr -c 'A-Za-z0-9_.-' '_' <<< "$1"
+}
+
+should_send_direct_telegram() {
+    local fingerprint="$1"
+    local cooldown_key
+    local cooldown_file
+    cooldown_key="$(safe_cooldown_key "$fingerprint")"
+    cooldown_file="${NOTIFY_COOLDOWN_DIR}/${cooldown_key}.cooldown"
+
+    if [[ -f "$cooldown_file" ]]; then
+        local last_sent now elapsed
+        last_sent=$(cat "$cooldown_file" 2>/dev/null || echo 0)
+        now=$(date +%s)
+        elapsed=$(( now - last_sent ))
+        if (( elapsed < NOTIFY_COOLDOWN_SECONDS )); then
+            log "TELEGRAM_COOLDOWN: ${fingerprint} ${elapsed}s/${NOTIFY_COOLDOWN_SECONDS}s，跳過直發"
+            return 1
+        fi
+    fi
+
+    date +%s > "$cooldown_file"
+    return 0
+}
+
 # 判斷容器是否符合模式清單
 matches_pattern() {
    local name="$1"
@@ -86,10 +116,10 @@ matches_pattern() {
 # ─── Telegram 直發 Fallback ──────────────────────────────────────────────────
 send_telegram_direct() {
    local message="$1"
-    [[ -z "$TELEGRAM_BOT_TOKEN" || -z "$SRE_GROUP_CHAT_ID" ]] && return 0
+    [[ -z "$TELEGRAM_BOT_TOKEN" || -z "$TELEGRAM_ALERT_CHAT_ID" ]] && return 0
    curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
        -H "Content-Type: application/json" \
-        -d "{\"chat_id\":\"${SRE_GROUP_CHAT_ID}\",\"text\":\"${message}\",\"parse_mode\":\"HTML\"}" \
+        -d "{\"chat_id\":\"${TELEGRAM_ALERT_CHAT_ID}\",\"text\":\"${message}\",\"parse_mode\":\"HTML\"}" \
        > /dev/null 2>&1 || true
 }

@@ -152,7 +182,9 @@ JSON
        local emoji="🔧"
        [[ "$repair_result" == "failed" ]] && emoji="❌"
        [[ "$repair_action" == "alert_only" ]] && emoji="⚠️"
-        send_telegram_direct "${emoji} [docker-health-monitor]&#10;主機: ${hostname}&#10;容器: ${container}&#10;狀態: ${detected_status}&#10;修復: ${repair_action} → ${repair_result}&#10;(API 不可達)"
+        if should_send_direct_telegram "${hostname}:${container}:${detected_status}:${repair_action}:${repair_result}:${http_code}"; then
+            send_telegram_direct "${emoji} [docker-health-monitor]&#10;主機: ${hostname}&#10;容器: ${container}&#10;狀態: ${detected_status}&#10;修復: ${repair_action} → ${repair_result}&#10;(API 不可達)"
+        fi
    fi
 }

--- a/scripts/ops/momo-pro-health-monitor.sh
+++ b/scripts/ops/momo-pro-health-monitor.sh
@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+# MOMO Pro health monitor (110 cron -> 188 app).
+#
+# Public route health is the first source of truth. The 188 local endpoint and
+# container state are secondary evidence only when the public health endpoint
+# fails. This prevents the old reboot-era false positive that checked the 188
+# root path and sent Telegram every five minutes while the real public service
+# had already recovered.
+
+set -euo pipefail
+
+SECRETS_FILE="${SECRETS_FILE:-/etc/awoooi-ops/secrets.env}"
+if [[ -r "$SECRETS_FILE" ]]; then
+    # shellcheck source=/dev/null
+    source "$SECRETS_FILE"
+fi
+
+: "${HEALTH_URL:=https://mo.wooo.work/health}"
+: "${REMOTE_HOST:=192.168.0.188}"
+: "${REMOTE_USER:=ollama}"
+: "${REMOTE_LOCAL_URL:=http://127.0.0.1:5003/health}"
+: "${CONTAINER_NAME:=momo-pro-system}"
+: "${CURL_TIMEOUT:=15}"
+: "${SEND_COOLDOWN_SECONDS:=1800}"
+: "${COOLDOWN_FILE:=/tmp/momo-pro-health-monitor.cooldown}"
+: "${NOTIFY_AWOOI_OPS_SCRIPT:=/home/wooo/awoooi-ops/notify-awoooi-ops.sh}"
+: "${SSH_STRICT_HOST_KEY_CHECKING:=accept-new}"
+
+log() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S %z')] $*"
+}
+
+http_code_for() {
+    local url="$1"
+    curl -kLsS -o /dev/null -w "%{http_code}" \
+        --connect-timeout "$CURL_TIMEOUT" \
+        --max-time "$CURL_TIMEOUT" \
+        "$url" 2>/dev/null || echo "000"
+}
+
+is_success_code() {
+    case "$1" in
+        2*|3*) return 0 ;;
+        *) return 1 ;;
+    esac
+}
+
+in_cooldown() {
+    [[ -f "$COOLDOWN_FILE" ]] || return 1
+    local last now elapsed
+    last=$(cat "$COOLDOWN_FILE" 2>/dev/null || echo 0)
+    now=$(date +%s)
+    elapsed=$(( now - last ))
+    if (( elapsed < SEND_COOLDOWN_SECONDS )); then
+        log "COOLDOWN: last alert ${elapsed}s ago; skip Telegram (${SEND_COOLDOWN_SECONDS}s window)"
+        return 0
+    fi
+    return 1
+}
+
+mark_cooldown() {
+    date +%s > "$COOLDOWN_FILE"
+}
+
+send_ops_alert() {
+    local alertname="$1"
+    local summary="$2"
+    local detail="$3"
+
+    if in_cooldown; then
+        return 0
+    fi
+
+    if [[ ! -x "$NOTIFY_AWOOI_OPS_SCRIPT" ]]; then
+        log "WARN: notify helper not executable: ${NOTIFY_AWOOI_OPS_SCRIPT}"
+        mark_cooldown
+        return 0
+    fi
+
+    AWOOI_OPS_ALERTNAME="$alertname" \
+    AWOOI_OPS_JOB_NAME="MOMO Pro health monitor" \
+    AWOOI_OPS_STATUS="failed" \
+    AWOOI_OPS_SEVERITY="warning" \
+    AWOOI_OPS_SOURCE="momo-pro-health-monitor" \
+    AWOOI_OPS_COMPONENT="momo-pro" \
+    AWOOI_OPS_SUMMARY="$summary" \
+    AWOOI_OPS_DETAIL="$detail" \
+    "$NOTIFY_AWOOI_OPS_SCRIPT" >/dev/null 2>&1 || log "WARN: AWOOI ops notification failed"
+
+    mark_cooldown
+    log "ALERT_REPORTED: cooldown marked"
+}
+
+remote_health_code() {
+    ssh -o ConnectTimeout=5 -o StrictHostKeyChecking="${SSH_STRICT_HOST_KEY_CHECKING}" "${REMOTE_USER}@${REMOTE_HOST}" \
+        "curl -kLsS -o /dev/null -w '%{http_code}' --connect-timeout ${CURL_TIMEOUT} --max-time ${CURL_TIMEOUT} '${REMOTE_LOCAL_URL}' 2>/dev/null || echo 000" \
+        2>/dev/null || echo "ssh_failed"
+}
+
+remote_container_status() {
+    ssh -o ConnectTimeout=5 -o StrictHostKeyChecking="${SSH_STRICT_HOST_KEY_CHECKING}" "${REMOTE_USER}@${REMOTE_HOST}" \
+        "docker inspect -f '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}' '${CONTAINER_NAME}' 2>/dev/null || echo missing" \
+        2>/dev/null || echo "ssh_failed"
+}
+
+main() {
+    log "CHECK: MOMO public health ${HEALTH_URL}"
+    local public_code
+    public_code=$(http_code_for "$HEALTH_URL")
+    if is_success_code "$public_code"; then
+        log "OK: public health ${public_code}; no alert"
+        return 0
+    fi
+
+    log "WARN: public health ${public_code}; checking ${REMOTE_HOST} local app and container"
+    local local_code container_status
+    local_code=$(remote_health_code)
+    container_status=$(remote_container_status)
+
+    if is_success_code "$local_code"; then
+        send_ops_alert \
+            "MomoProPublicRouteUnhealthy" \
+            "MOMO Pro public route unhealthy but 188 local app is OK" \
+            "public=${HEALTH_URL} HTTP ${public_code}; local=${REMOTE_LOCAL_URL} HTTP ${local_code}; container=${CONTAINER_NAME} ${container_status}; likely gateway/proxy path; do not restart blindly."
+    else
+        send_ops_alert \
+            "MomoProHealthFailed" \
+            "MOMO Pro service health failed" \
+            "public=${HEALTH_URL} HTTP ${public_code}; local=${REMOTE_LOCAL_URL} HTTP ${local_code}; container=${CONTAINER_NAME} ${container_status}; host=${REMOTE_HOST}; manual triage required."
+    fi
+    return 2
+}
+
+main "$@"