fix(monitoring): make alert deploy fail closed

scripts/ops/deploy-alerts.sh

@@ -4,7 +4,7 @@
 # 2026-04-05 Claude Code: Sprint 1 自動化部署
 # 用法: bash scripts/ops/deploy-alerts.sh [--dry-run]
 
-set -eo pipefail
+set -euo pipefail
 
 ALERT_RULES_FILE="ops/monitoring/alerts-unified.yml"
 SLO_RULES_FILE="ops/monitoring/slo-rules.yml"
@@ -15,10 +15,59 @@ TARGET_ALERTS_CANONICAL_PATH="/home/wooo/monitoring/alerts-unified.canonical.yml
 TARGET_SLO_PATH="/home/wooo/monitoring/slo-rules.yml"
 TARGET_DRIFT_GUARD_PATH="/home/wooo/scripts/prometheus-rule-drift-guard.sh"
 PROMETHEUS_URL="http://${TARGET_HOST}:9090"
+PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
+PROMETHEUS_RUNTIME_RULES="${PROMETHEUS_RUNTIME_RULES:-/etc/prometheus/alerts.yml}"
 DRY_RUN="${1:-}"
+DEPLOY_STAMP="$(date +%Y%m%d%H%M%S)"
+REMOTE_CANDIDATE_PATH="/tmp/awoooi-alerts-unified.${DEPLOY_STAMP}.yml"
+CONTAINER_CANDIDATE_PATH="/tmp/awoooi-alerts-unified.${DEPLOY_STAMP}.yml"
+TARGET_ALERTS_BACKUP_PATH="${TARGET_ALERTS_PATH}.bak.${DEPLOY_STAMP}"
+TARGET_ALERTS_CANONICAL_BACKUP_PATH="${TARGET_ALERTS_CANONICAL_PATH}.bak.${DEPLOY_STAMP}"
+TARGET_SLO_BACKUP_PATH="${TARGET_SLO_PATH}.bak.${DEPLOY_STAMP}"
+TARGET_DRIFT_GUARD_BACKUP_PATH="${TARGET_DRIFT_GUARD_PATH}.bak.${DEPLOY_STAMP}"
+APPLY_STARTED=0
+DEPLOY_VERIFIED=0
 
 log() { echo "[$(date '+%H:%M:%S')] $*"; }
 
+cleanup_candidate() {
+    ssh wooo@${TARGET_HOST} "rm -f '${REMOTE_CANDIDATE_PATH}'; docker exec -u 0 '${PROMETHEUS_CONTAINER}' rm -f '${CONTAINER_CANDIDATE_PATH}' >/dev/null 2>&1 || true" >/dev/null 2>&1 || true
+}
+
+rollback_deploy() {
+    log "部署驗證失敗，開始還原 ${DEPLOY_STAMP} 前規則"
+    ssh wooo@${TARGET_HOST} "set -euo pipefail
+        cp '${TARGET_ALERTS_BACKUP_PATH}' '${TARGET_ALERTS_PATH}'
+        cp '${TARGET_ALERTS_CANONICAL_BACKUP_PATH}' '${TARGET_ALERTS_CANONICAL_PATH}'
+        cp '${TARGET_SLO_BACKUP_PATH}' '${TARGET_SLO_PATH}'
+        cp '${TARGET_DRIFT_GUARD_BACKUP_PATH}' '${TARGET_DRIFT_GUARD_PATH}'
+        chmod 0755 '${TARGET_DRIFT_GUARD_PATH}'
+        curl -fsS -X POST '${PROMETHEUS_URL}/-/reload' >/dev/null
+        sleep 2
+        curl -fsS '${PROMETHEUS_URL}/-/ready' >/dev/null"
+    log "已還原規則並完成 Prometheus readiness readback"
+}
+
+on_exit() {
+    local rc=$?
+    cleanup_candidate
+    if [ "$rc" -ne 0 ] && [ "$APPLY_STARTED" -eq 1 ] && [ "$DEPLOY_VERIFIED" -eq 0 ]; then
+        rollback_deploy || log "ERROR: 自動 rollback 失敗，保留精確 backup paths 供獨立恢復"
+    fi
+    exit "$rc"
+}
+
+remote_promtool_preflight() {
+    scp "$ALERT_RULES_FILE" "wooo@${TARGET_HOST}:${REMOTE_CANDIDATE_PATH}"
+    ssh wooo@${TARGET_HOST} "set -euo pipefail
+        docker cp '${REMOTE_CANDIDATE_PATH}' '${PROMETHEUS_CONTAINER}:${CONTAINER_CANDIDATE_PATH}'
+        docker exec '${PROMETHEUS_CONTAINER}' promtool check rules '${CONTAINER_CANDIDATE_PATH}'"
+    cleanup_candidate
+    log "✅ remote promtool preflight 通過"
+}
+
+trap on_exit EXIT
+
 file_sha() {
     if command -v sha256sum >/dev/null 2>&1; then
         sha256sum "$1" | awk '{print $1}'
@@ -59,6 +108,7 @@ log "✅ YAML 語法驗證通過"
 
 # Dry run 模式
 if [ "$DRY_RUN" = "--dry-run" ]; then
+    remote_promtool_preflight
     log "DRY RUN: would deploy $ALERT_RULES_FILE to ${TARGET_HOST}:${TARGET_ALERTS_PATH}"
     log "DRY RUN: would deploy $ALERT_RULES_FILE to ${TARGET_HOST}:${TARGET_ALERTS_CANONICAL_PATH}"
     log "DRY RUN: would deploy $SLO_RULES_FILE to ${TARGET_HOST}:${TARGET_SLO_PATH}"
@@ -70,13 +120,17 @@ if [ "$DRY_RUN" = "--dry-run" ]; then
     exit 0
 fi
 
-# 備份現有規則
-ssh wooo@${TARGET_HOST} "\
-    cp ${TARGET_ALERTS_PATH} ${TARGET_ALERTS_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \
-    cp ${TARGET_ALERTS_CANONICAL_PATH} ${TARGET_ALERTS_CANONICAL_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \
-    cp ${TARGET_SLO_PATH} ${TARGET_SLO_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \
-    cp ${TARGET_DRIFT_GUARD_PATH} ${TARGET_DRIFT_GUARD_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true"
+# 使用 production Prometheus 的 promtool 驗證候選規則，通過後才能寫 active path。
+remote_promtool_preflight
+
+# 備份現有規則；任一 source 缺失時 fail closed，不開始 apply。
+ssh wooo@${TARGET_HOST} "set -euo pipefail
+    cp '${TARGET_ALERTS_PATH}' '${TARGET_ALERTS_BACKUP_PATH}'
+    cp '${TARGET_ALERTS_CANONICAL_PATH}' '${TARGET_ALERTS_CANONICAL_BACKUP_PATH}'
+    cp '${TARGET_SLO_PATH}' '${TARGET_SLO_BACKUP_PATH}'
+    cp '${TARGET_DRIFT_GUARD_PATH}' '${TARGET_DRIFT_GUARD_BACKUP_PATH}'"
 log "✅ 現有規則已備份"
+APPLY_STARTED=1
 
 # 部署新規則
 scp "$ALERT_RULES_FILE" wooo@${TARGET_HOST}:${TARGET_ALERTS_PATH}
@@ -112,7 +166,7 @@ fi
 log "✅ 遠端規則 hash 驗證通過"
 
 # Reload Prometheus
-ssh wooo@${TARGET_HOST} "curl -s -X POST ${PROMETHEUS_URL}/-/reload"
+ssh wooo@${TARGET_HOST} "curl -fsS -X POST ${PROMETHEUS_URL}/-/reload >/dev/null"
 sleep 3
 
 # 驗證規則數量
@@ -161,4 +215,16 @@ for rule in "${KEY_SLO_RULES[@]}"; do
     fi
 done
 
+REMOTE_RUNTIME_SHA=$(ssh wooo@${TARGET_HOST} "docker exec '${PROMETHEUS_CONTAINER}' sha256sum '${PROMETHEUS_RUNTIME_RULES}' | awk '{print \$1}'")
+if [ "$LOCAL_ALERTS_SHA" != "$REMOTE_RUNTIME_SHA" ]; then
+    echo "ERROR: Prometheus runtime rules hash 不一致 local=${LOCAL_ALERTS_SHA} runtime=${REMOTE_RUNTIME_SHA}"
+    exit 1
+fi
+log "✅ Prometheus runtime rules hash 與 canonical source 一致"
+
+bash scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh
+log "✅ Cold-start producer / active rules / Agent99 consumer scope verifier 通過"
+
+DEPLOY_VERIFIED=1
+log "Rollback receipts: ${TARGET_ALERTS_BACKUP_PATH}, ${TARGET_ALERTS_CANONICAL_BACKUP_PATH}"
 log "🎉 部署完成！所有關鍵規則已生效"