#!/bin/bash
# scripts/ops/deploy-alerts.sh
# 部署統一告警規則到 110 Prometheus
# 2026-04-05 Claude Code: Sprint 1 自動化部署
# 用法: bash scripts/ops/deploy-alerts.sh [--dry-run]

set -euo pipefail

ALERT_RULES_FILE="ops/monitoring/alerts-unified.yml"
SLO_RULES_FILE="ops/monitoring/slo-rules.yml"
DRIFT_GUARD_SCRIPT="scripts/ops/prometheus-rule-drift-guard.sh"
TARGET_HOST="192.168.0.110"
TARGET_ALERTS_PATH="/home/wooo/monitoring/alerts.yml"
TARGET_ALERTS_CANONICAL_PATH="/home/wooo/monitoring/alerts-unified.canonical.yml"
TARGET_SLO_PATH="/home/wooo/monitoring/slo-rules.yml"
TARGET_DRIFT_GUARD_PATH="/home/wooo/scripts/prometheus-rule-drift-guard.sh"
PROMETHEUS_URL="http://${TARGET_HOST}:9090"
PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
PROMETHEUS_RUNTIME_RULES="${PROMETHEUS_RUNTIME_RULES:-/etc/prometheus/alerts.yml}"
DRY_RUN="${1:-}"
DEPLOY_STAMP="$(date +%Y%m%d%H%M%S)"
REMOTE_CANDIDATE_PATH="/tmp/awoooi-alerts-unified.${DEPLOY_STAMP}.yml"
CONTAINER_CANDIDATE_PATH="/tmp/awoooi-alerts-unified.${DEPLOY_STAMP}.yml"
TARGET_ALERTS_BACKUP_PATH="${TARGET_ALERTS_PATH}.bak.${DEPLOY_STAMP}"
TARGET_ALERTS_CANONICAL_BACKUP_PATH="${TARGET_ALERTS_CANONICAL_PATH}.bak.${DEPLOY_STAMP}"
TARGET_SLO_BACKUP_PATH="${TARGET_SLO_PATH}.bak.${DEPLOY_STAMP}"
TARGET_DRIFT_GUARD_BACKUP_PATH="${TARGET_DRIFT_GUARD_PATH}.bak.${DEPLOY_STAMP}"
APPLY_STARTED=0
DEPLOY_VERIFIED=0

log() { echo "[$(date '+%H:%M:%S')] $*"; }

cleanup_candidate() {
    ssh wooo@${TARGET_HOST} "rm -f '${REMOTE_CANDIDATE_PATH}'; docker exec -u 0 '${PROMETHEUS_CONTAINER}' rm -f '${CONTAINER_CANDIDATE_PATH}' >/dev/null 2>&1 || true" >/dev/null 2>&1 || true
}

rollback_deploy() {
    log "部署驗證失敗，開始還原 ${DEPLOY_STAMP} 前規則"
    ssh wooo@${TARGET_HOST} "set -euo pipefail
        cp '${TARGET_ALERTS_BACKUP_PATH}' '${TARGET_ALERTS_PATH}'
        cp '${TARGET_ALERTS_CANONICAL_BACKUP_PATH}' '${TARGET_ALERTS_CANONICAL_PATH}'
        cp '${TARGET_SLO_BACKUP_PATH}' '${TARGET_SLO_PATH}'
        cp '${TARGET_DRIFT_GUARD_BACKUP_PATH}' '${TARGET_DRIFT_GUARD_PATH}'
        chmod 0755 '${TARGET_DRIFT_GUARD_PATH}'
        curl -fsS -X POST '${PROMETHEUS_URL}/-/reload' >/dev/null
        sleep 2
        curl -fsS '${PROMETHEUS_URL}/-/ready' >/dev/null"
    log "已還原規則並完成 Prometheus readiness readback"
}

on_exit() {
    local rc=$?
    cleanup_candidate
    if [ "$rc" -ne 0 ] && [ "$APPLY_STARTED" -eq 1 ] && [ "$DEPLOY_VERIFIED" -eq 0 ]; then
        rollback_deploy || log "ERROR: 自動 rollback 失敗，保留精確 backup paths 供獨立恢復"
    fi
    exit "$rc"
}

remote_promtool_preflight() {
    scp "$ALERT_RULES_FILE" "wooo@${TARGET_HOST}:${REMOTE_CANDIDATE_PATH}"
    ssh wooo@${TARGET_HOST} "set -euo pipefail
        docker cp '${REMOTE_CANDIDATE_PATH}' '${PROMETHEUS_CONTAINER}:${CONTAINER_CANDIDATE_PATH}'
        docker exec '${PROMETHEUS_CONTAINER}' promtool check rules '${CONTAINER_CANDIDATE_PATH}'"
    cleanup_candidate
    log "✅ remote promtool preflight 通過"
}

trap on_exit EXIT

file_sha() {
    if command -v sha256sum >/dev/null 2>&1; then
        sha256sum "$1" | awk '{print $1}'
    else
        shasum -a 256 "$1" | awk '{print $1}'
    fi
}

remote_file_sha() {
    ssh wooo@${TARGET_HOST} "\
        if command -v sha256sum >/dev/null 2>&1; then \
            sha256sum '$1' | awk '{print \$1}'; \
        else \
            shasum -a 256 '$1' | awk '{print \$1}'; \
        fi"
}

# 確認檔案存在
for file in "$ALERT_RULES_FILE" "$SLO_RULES_FILE" "$DRIFT_GUARD_SCRIPT"; do
    if [ ! -f "$file" ]; then
        echo "ERROR: $file not found"
        exit 1
    fi
done

# 驗證 YAML 語法
for file in "$ALERT_RULES_FILE" "$SLO_RULES_FILE"; do
    if python3 -c "import yaml; yaml.safe_load(open('$file'))" 2>/dev/null; then
        :
    elif ruby -e "require 'yaml'; YAML.load_file('$file')" 2>/dev/null; then
        :
    else
        echo "ERROR: YAML syntax error or no YAML parser available: $file"
        exit 1
    fi
done
log "✅ YAML 語法驗證通過"

# Dry run 模式
if [ "$DRY_RUN" = "--dry-run" ]; then
    remote_promtool_preflight
    log "DRY RUN: would deploy $ALERT_RULES_FILE to ${TARGET_HOST}:${TARGET_ALERTS_PATH}"
    log "DRY RUN: would deploy $ALERT_RULES_FILE to ${TARGET_HOST}:${TARGET_ALERTS_CANONICAL_PATH}"
    log "DRY RUN: would deploy $SLO_RULES_FILE to ${TARGET_HOST}:${TARGET_SLO_PATH}"
    log "DRY RUN: would deploy $DRIFT_GUARD_SCRIPT to ${TARGET_HOST}:${TARGET_DRIFT_GUARD_PATH}"
    ALERT_COUNT=$(grep -c "alert:" "$ALERT_RULES_FILE")
    SLO_RECORD_COUNT=$(grep -c "record:" "$SLO_RULES_FILE")
    SLO_ALERT_COUNT=$(grep -c "alert:" "$SLO_RULES_FILE")
    log "告警規則數量: $ALERT_COUNT 條；SLO recording: $SLO_RECORD_COUNT 條；SLO alerts: $SLO_ALERT_COUNT 條"
    exit 0
fi

# 使用 production Prometheus 的 promtool 驗證候選規則，通過後才能寫 active path。
remote_promtool_preflight

# 備份現有規則；任一 source 缺失時 fail closed，不開始 apply。
ssh wooo@${TARGET_HOST} "set -euo pipefail
    cp '${TARGET_ALERTS_PATH}' '${TARGET_ALERTS_BACKUP_PATH}'
    cp '${TARGET_ALERTS_CANONICAL_PATH}' '${TARGET_ALERTS_CANONICAL_BACKUP_PATH}'
    cp '${TARGET_SLO_PATH}' '${TARGET_SLO_BACKUP_PATH}'
    cp '${TARGET_DRIFT_GUARD_PATH}' '${TARGET_DRIFT_GUARD_BACKUP_PATH}'"
log "✅ 現有規則已備份"
APPLY_STARTED=1

# 部署新規則
scp "$ALERT_RULES_FILE" wooo@${TARGET_HOST}:${TARGET_ALERTS_PATH}
scp "$ALERT_RULES_FILE" wooo@${TARGET_HOST}:${TARGET_ALERTS_CANONICAL_PATH}
scp "$SLO_RULES_FILE" wooo@${TARGET_HOST}:${TARGET_SLO_PATH}
scp "$DRIFT_GUARD_SCRIPT" wooo@${TARGET_HOST}:${TARGET_DRIFT_GUARD_PATH}
ssh wooo@${TARGET_HOST} "chmod 0755 ${TARGET_DRIFT_GUARD_PATH}"
log "✅ 規則已複製到 ${TARGET_HOST}"

LOCAL_ALERTS_SHA="$(file_sha "$ALERT_RULES_FILE")"
REMOTE_ALERTS_SHA="$(remote_file_sha "$TARGET_ALERTS_PATH")"
REMOTE_ALERTS_CANONICAL_SHA="$(remote_file_sha "$TARGET_ALERTS_CANONICAL_PATH")"
LOCAL_SLO_SHA="$(file_sha "$SLO_RULES_FILE")"
REMOTE_SLO_SHA="$(remote_file_sha "$TARGET_SLO_PATH")"
LOCAL_DRIFT_GUARD_SHA="$(file_sha "$DRIFT_GUARD_SCRIPT")"
REMOTE_DRIFT_GUARD_SHA="$(remote_file_sha "$TARGET_DRIFT_GUARD_PATH")"
if [ "$LOCAL_ALERTS_SHA" != "$REMOTE_ALERTS_SHA" ]; then
    echo "ERROR: 遠端 alerts.yml hash 不一致 local=${LOCAL_ALERTS_SHA} remote=${REMOTE_ALERTS_SHA}"
    exit 1
fi
if [ "$LOCAL_ALERTS_SHA" != "$REMOTE_ALERTS_CANONICAL_SHA" ]; then
    echo "ERROR: 遠端 alerts-unified.canonical.yml hash 不一致 local=${LOCAL_ALERTS_SHA} remote=${REMOTE_ALERTS_CANONICAL_SHA}"
    exit 1
fi
if [ "$LOCAL_SLO_SHA" != "$REMOTE_SLO_SHA" ]; then
    echo "ERROR: 遠端 slo-rules.yml hash 不一致 local=${LOCAL_SLO_SHA} remote=${REMOTE_SLO_SHA}"
    exit 1
fi
if [ "$LOCAL_DRIFT_GUARD_SHA" != "$REMOTE_DRIFT_GUARD_SHA" ]; then
    echo "ERROR: 遠端 prometheus-rule-drift-guard.sh hash 不一致 local=${LOCAL_DRIFT_GUARD_SHA} remote=${REMOTE_DRIFT_GUARD_SHA}"
    exit 1
fi
log "✅ 遠端規則 hash 驗證通過"

# Reload Prometheus
ssh wooo@${TARGET_HOST} "curl -fsS -X POST ${PROMETHEUS_URL}/-/reload >/dev/null"
sleep 3

# 驗證規則數量
RULE_COUNT=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules | python3 -c \"import sys,json; r=json.load(sys.stdin); print(sum(len(g['rules']) for g in r['data']['groups']))\"")
log "Prometheus 已載入 ${RULE_COUNT} 條規則"

if [ "$RULE_COUNT" -lt 25 ]; then
    echo "ERROR: 規則數量異常 ($RULE_COUNT < 25)，請檢查"
    exit 1
fi

NO_ALERTS_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'NoAlertsReceived2Hours'), ''))")
if [[ "$NO_ALERTS_QUERY" != *'source="alertmanager"'* ]]; then
    echo "ERROR: NoAlertsReceived2Hours query 未限制 alertmanager 主鏈路: ${NO_ALERTS_QUERY}"
    exit 1
fi
log "✅ NoAlertsReceived2Hours query 已限制 alertmanager 主鏈路"

SOURCE_PROVIDER_STALE_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'SourceProviderIngestionStale'), ''))")
if [[ "$SOURCE_PROVIDER_STALE_QUERY" != *'source=~"sentry|signoz"'* ]]; then
    echo "ERROR: SourceProviderIngestionStale query 未限制 Sentry/SignOz provider freshness: ${SOURCE_PROVIDER_STALE_QUERY}"
    exit 1
fi
log "✅ SourceProviderIngestionStale query 已限制 Sentry/SignOz provider freshness"

# 驗證關鍵規則存在
KEY_RULES=("SentryDown" "HarborDown" "GiteaDown" "OpenClawDown" "AlertmanagerDown" "AlertChainUnhealthy" "SourceProviderIngestionStale")
for rule in "${KEY_RULES[@]}"; do
    EXISTS=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules | python3 -c \"import sys,json; r=json.load(sys.stdin); names=[x['name'] for g in r['data']['groups'] for x in g['rules']]; print('OK' if '$rule' in names else 'MISSING')\"")
    if [ "$EXISTS" = "OK" ]; then
        log "✅ $rule"
    else
        echo "❌ $rule 未找到"
        exit 1
    fi
done

KEY_SLO_RULES=("sli:autonomy_rate:5m" "sli:decision_accuracy:5m" "sli:confidence_calibration:1h" "sli:km_growth_rate:24h" "SLO_KMGrowthRate_Critical")
for rule in "${KEY_SLO_RULES[@]}"; do
    EXISTS=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules | python3 -c \"import sys,json; r=json.load(sys.stdin); names=[x['name'] for g in r['data']['groups'] for x in g['rules']]; print('OK' if '$rule' in names else 'MISSING')\"")
    if [ "$EXISTS" = "OK" ]; then
        log "✅ $rule"
    else
        echo "❌ $rule 未找到"
        exit 1
    fi
done

REMOTE_RUNTIME_SHA=$(ssh wooo@${TARGET_HOST} "docker exec '${PROMETHEUS_CONTAINER}' sha256sum '${PROMETHEUS_RUNTIME_RULES}' | awk '{print \$1}'")
if [ "$LOCAL_ALERTS_SHA" != "$REMOTE_RUNTIME_SHA" ]; then
    echo "ERROR: Prometheus runtime rules hash 不一致 local=${LOCAL_ALERTS_SHA} runtime=${REMOTE_RUNTIME_SHA}"
    exit 1
fi
log "✅ Prometheus runtime rules hash 與 canonical source 一致"

bash scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh
log "✅ Cold-start producer / active rules / Agent99 consumer scope verifier 通過"

DEPLOY_VERIFIED=1
log "Rollback receipts: ${TARGET_ALERTS_BACKUP_PATH}, ${TARGET_ALERTS_CANONICAL_BACKUP_PATH}"
log "🎉 部署完成！所有關鍵規則已生效"