diff --git a/scripts/ops/deploy-alerts.sh b/scripts/ops/deploy-alerts.sh index ce8eaf0a9..728edc91c 100755 --- a/scripts/ops/deploy-alerts.sh +++ b/scripts/ops/deploy-alerts.sh @@ -4,7 +4,7 @@ # 2026-04-05 Claude Code: Sprint 1 自動化部署 # 用法: bash scripts/ops/deploy-alerts.sh [--dry-run] -set -eo pipefail +set -euo pipefail ALERT_RULES_FILE="ops/monitoring/alerts-unified.yml" SLO_RULES_FILE="ops/monitoring/slo-rules.yml" @@ -15,10 +15,59 @@ TARGET_ALERTS_CANONICAL_PATH="/home/wooo/monitoring/alerts-unified.canonical.yml TARGET_SLO_PATH="/home/wooo/monitoring/slo-rules.yml" TARGET_DRIFT_GUARD_PATH="/home/wooo/scripts/prometheus-rule-drift-guard.sh" PROMETHEUS_URL="http://${TARGET_HOST}:9090" +PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}" +PROMETHEUS_RUNTIME_RULES="${PROMETHEUS_RUNTIME_RULES:-/etc/prometheus/alerts.yml}" DRY_RUN="${1:-}" +DEPLOY_STAMP="$(date +%Y%m%d%H%M%S)" +REMOTE_CANDIDATE_PATH="/tmp/awoooi-alerts-unified.${DEPLOY_STAMP}.yml" +CONTAINER_CANDIDATE_PATH="/tmp/awoooi-alerts-unified.${DEPLOY_STAMP}.yml" +TARGET_ALERTS_BACKUP_PATH="${TARGET_ALERTS_PATH}.bak.${DEPLOY_STAMP}" +TARGET_ALERTS_CANONICAL_BACKUP_PATH="${TARGET_ALERTS_CANONICAL_PATH}.bak.${DEPLOY_STAMP}" +TARGET_SLO_BACKUP_PATH="${TARGET_SLO_PATH}.bak.${DEPLOY_STAMP}" +TARGET_DRIFT_GUARD_BACKUP_PATH="${TARGET_DRIFT_GUARD_PATH}.bak.${DEPLOY_STAMP}" +APPLY_STARTED=0 +DEPLOY_VERIFIED=0 log() { echo "[$(date '+%H:%M:%S')] $*"; } +cleanup_candidate() { + ssh wooo@${TARGET_HOST} "rm -f '${REMOTE_CANDIDATE_PATH}'; docker exec -u 0 '${PROMETHEUS_CONTAINER}' rm -f '${CONTAINER_CANDIDATE_PATH}' >/dev/null 2>&1 || true" >/dev/null 2>&1 || true +} + +rollback_deploy() { + log "部署驗證失敗，開始還原 ${DEPLOY_STAMP} 前規則" + ssh wooo@${TARGET_HOST} "set -euo pipefail + cp '${TARGET_ALERTS_BACKUP_PATH}' '${TARGET_ALERTS_PATH}' + cp '${TARGET_ALERTS_CANONICAL_BACKUP_PATH}' '${TARGET_ALERTS_CANONICAL_PATH}' + cp '${TARGET_SLO_BACKUP_PATH}' '${TARGET_SLO_PATH}' + cp '${TARGET_DRIFT_GUARD_BACKUP_PATH}' '${TARGET_DRIFT_GUARD_PATH}' + chmod 0755 '${TARGET_DRIFT_GUARD_PATH}' + curl -fsS -X POST '${PROMETHEUS_URL}/-/reload' >/dev/null + sleep 2 + curl -fsS '${PROMETHEUS_URL}/-/ready' >/dev/null" + log "已還原規則並完成 Prometheus readiness readback" +} + +on_exit() { + local rc=$? + cleanup_candidate + if [ "$rc" -ne 0 ] && [ "$APPLY_STARTED" -eq 1 ] && [ "$DEPLOY_VERIFIED" -eq 0 ]; then + rollback_deploy || log "ERROR: 自動 rollback 失敗，保留精確 backup paths 供獨立恢復" + fi + exit "$rc" +} + +remote_promtool_preflight() { + scp "$ALERT_RULES_FILE" "wooo@${TARGET_HOST}:${REMOTE_CANDIDATE_PATH}" + ssh wooo@${TARGET_HOST} "set -euo pipefail + docker cp '${REMOTE_CANDIDATE_PATH}' '${PROMETHEUS_CONTAINER}:${CONTAINER_CANDIDATE_PATH}' + docker exec '${PROMETHEUS_CONTAINER}' promtool check rules '${CONTAINER_CANDIDATE_PATH}'" + cleanup_candidate + log "✅ remote promtool preflight 通過" +} + +trap on_exit EXIT + file_sha() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}' @@ -59,6 +108,7 @@ log "✅ YAML 語法驗證通過" # Dry run 模式 if [ "$DRY_RUN" = "--dry-run" ]; then + remote_promtool_preflight log "DRY RUN: would deploy $ALERT_RULES_FILE to ${TARGET_HOST}:${TARGET_ALERTS_PATH}" log "DRY RUN: would deploy $ALERT_RULES_FILE to ${TARGET_HOST}:${TARGET_ALERTS_CANONICAL_PATH}" log "DRY RUN: would deploy $SLO_RULES_FILE to ${TARGET_HOST}:${TARGET_SLO_PATH}" @@ -70,13 +120,17 @@ if [ "$DRY_RUN" = "--dry-run" ]; then exit 0 fi -# 備份現有規則 -ssh wooo@${TARGET_HOST} "\ - cp ${TARGET_ALERTS_PATH} ${TARGET_ALERTS_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \ - cp ${TARGET_ALERTS_CANONICAL_PATH} ${TARGET_ALERTS_CANONICAL_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \ - cp ${TARGET_SLO_PATH} ${TARGET_SLO_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \ - cp ${TARGET_DRIFT_GUARD_PATH} ${TARGET_DRIFT_GUARD_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true" +# 使用 production Prometheus 的 promtool 驗證候選規則，通過後才能寫 active path。 +remote_promtool_preflight + +# 備份現有規則；任一 source 缺失時 fail closed，不開始 apply。 +ssh wooo@${TARGET_HOST} "set -euo pipefail + cp '${TARGET_ALERTS_PATH}' '${TARGET_ALERTS_BACKUP_PATH}' + cp '${TARGET_ALERTS_CANONICAL_PATH}' '${TARGET_ALERTS_CANONICAL_BACKUP_PATH}' + cp '${TARGET_SLO_PATH}' '${TARGET_SLO_BACKUP_PATH}' + cp '${TARGET_DRIFT_GUARD_PATH}' '${TARGET_DRIFT_GUARD_BACKUP_PATH}'" log "✅ 現有規則已備份" +APPLY_STARTED=1 # 部署新規則 scp "$ALERT_RULES_FILE" wooo@${TARGET_HOST}:${TARGET_ALERTS_PATH} @@ -112,7 +166,7 @@ fi log "✅ 遠端規則 hash 驗證通過" # Reload Prometheus -ssh wooo@${TARGET_HOST} "curl -s -X POST ${PROMETHEUS_URL}/-/reload" +ssh wooo@${TARGET_HOST} "curl -fsS -X POST ${PROMETHEUS_URL}/-/reload >/dev/null" sleep 3 # 驗證規則數量 @@ -161,4 +215,16 @@ for rule in "${KEY_SLO_RULES[@]}"; do fi done +REMOTE_RUNTIME_SHA=$(ssh wooo@${TARGET_HOST} "docker exec '${PROMETHEUS_CONTAINER}' sha256sum '${PROMETHEUS_RUNTIME_RULES}' | awk '{print \$1}'") +if [ "$LOCAL_ALERTS_SHA" != "$REMOTE_RUNTIME_SHA" ]; then + echo "ERROR: Prometheus runtime rules hash 不一致 local=${LOCAL_ALERTS_SHA} runtime=${REMOTE_RUNTIME_SHA}" + exit 1 +fi +log "✅ Prometheus runtime rules hash 與 canonical source 一致" + +bash scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh +log "✅ Cold-start producer / active rules / Agent99 consumer scope verifier 通過" + +DEPLOY_VERIFIED=1 +log "Rollback receipts: ${TARGET_ALERTS_BACKUP_PATH}, ${TARGET_ALERTS_CANONICAL_BACKUP_PATH}" log "🎉 部署完成！所有關鍵規則已生效" diff --git a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py index 7e06ecab2..5ed821e5d 100644 --- a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py +++ b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py @@ -20,6 +20,7 @@ RECOVERY_SCORECARD = ( ROOT / "scripts" / "reboot-recovery" / "full-stack-recovery-scorecard.sh" ) VERIFY_DEPLOY = ROOT / "scripts" / "reboot-recovery" / "verify-cold-start-monitor-deploy.sh" +DEPLOY_ALERTS = ROOT / "scripts" / "ops" / "deploy-alerts.sh" REPAIR_STARTUP_STUB = ( ROOT / "scripts" / "reboot-recovery" / "repair-110-startup-script-stub.sh" ) @@ -105,6 +106,20 @@ def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None: assert 'scope="110_120_121_188"' not in ansible_verifier +def test_alert_rule_deploy_is_preflighted_verified_and_rollback_capable() -> None: + deploy = DEPLOY_ALERTS.read_text(encoding="utf-8") + + assert "remote_promtool_preflight" in deploy + assert "promtool check rules" in deploy + assert "trap on_exit EXIT" in deploy + assert "rollback_deploy" in deploy + assert 'APPLY_STARTED=1' in deploy + assert 'DEPLOY_VERIFIED=1' in deploy + assert "PROMETHEUS_RUNTIME_RULES" in deploy + assert "verify-cold-start-monitor-deploy.sh" in deploy + assert "curl -fsS -X POST" in deploy + + def test_cold_start_momo_current_month_handles_no_new_source_without_false_warn() -> None: text = COLD_START_CHECK.read_text(encoding="utf-8")