Merge remote-tracking branch 'origin/main' into codex/p0-obs-002-20260715

2026-07-15 02:30:04 +08:00
parent bfb2d2d040 3d8fa0200e
commit 4cf66c2a7e
17 changed files with 524 additions and 63 deletions
--- a/scripts/ops/deploy-alerts.sh
+++ b/scripts/ops/deploy-alerts.sh
@@ -4,7 +4,7 @@
 # 2026-04-05 Claude Code: Sprint 1 自動化部署
 # 用法: bash scripts/ops/deploy-alerts.sh [--dry-run]

-set -eo pipefail
+set -euo pipefail

 ALERT_RULES_FILE="ops/monitoring/alerts-unified.yml"
 SLO_RULES_FILE="ops/monitoring/slo-rules.yml"
@@ -15,10 +15,59 @@ TARGET_ALERTS_CANONICAL_PATH="/home/wooo/monitoring/alerts-unified.canonical.yml
 TARGET_SLO_PATH="/home/wooo/monitoring/slo-rules.yml"
 TARGET_DRIFT_GUARD_PATH="/home/wooo/scripts/prometheus-rule-drift-guard.sh"
 PROMETHEUS_URL="http://${TARGET_HOST}:9090"
+PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
+PROMETHEUS_RUNTIME_RULES="${PROMETHEUS_RUNTIME_RULES:-/etc/prometheus/alerts.yml}"
 DRY_RUN="${1:-}"
+DEPLOY_STAMP="$(date +%Y%m%d%H%M%S)"
+REMOTE_CANDIDATE_PATH="/tmp/awoooi-alerts-unified.${DEPLOY_STAMP}.yml"
+CONTAINER_CANDIDATE_PATH="/tmp/awoooi-alerts-unified.${DEPLOY_STAMP}.yml"
+TARGET_ALERTS_BACKUP_PATH="${TARGET_ALERTS_PATH}.bak.${DEPLOY_STAMP}"
+TARGET_ALERTS_CANONICAL_BACKUP_PATH="${TARGET_ALERTS_CANONICAL_PATH}.bak.${DEPLOY_STAMP}"
+TARGET_SLO_BACKUP_PATH="${TARGET_SLO_PATH}.bak.${DEPLOY_STAMP}"
+TARGET_DRIFT_GUARD_BACKUP_PATH="${TARGET_DRIFT_GUARD_PATH}.bak.${DEPLOY_STAMP}"
+APPLY_STARTED=0
+DEPLOY_VERIFIED=0

 log() { echo "[$(date '+%H:%M:%S')] $*"; }

+cleanup_candidate() {
+    ssh wooo@${TARGET_HOST} "rm -f '${REMOTE_CANDIDATE_PATH}'; docker exec -u 0 '${PROMETHEUS_CONTAINER}' rm -f '${CONTAINER_CANDIDATE_PATH}' >/dev/null 2>&1 || true" >/dev/null 2>&1 || true
+}
+
+rollback_deploy() {
+    log "部署驗證失敗，開始還原 ${DEPLOY_STAMP} 前規則"
+    ssh wooo@${TARGET_HOST} "set -euo pipefail
+        cp '${TARGET_ALERTS_BACKUP_PATH}' '${TARGET_ALERTS_PATH}'
+        cp '${TARGET_ALERTS_CANONICAL_BACKUP_PATH}' '${TARGET_ALERTS_CANONICAL_PATH}'
+        cp '${TARGET_SLO_BACKUP_PATH}' '${TARGET_SLO_PATH}'
+        cp '${TARGET_DRIFT_GUARD_BACKUP_PATH}' '${TARGET_DRIFT_GUARD_PATH}'
+        chmod 0755 '${TARGET_DRIFT_GUARD_PATH}'
+        curl -fsS -X POST '${PROMETHEUS_URL}/-/reload' >/dev/null
+        sleep 2
+        curl -fsS '${PROMETHEUS_URL}/-/ready' >/dev/null"
+    log "已還原規則並完成 Prometheus readiness readback"
+}
+
+on_exit() {
+    local rc=$?
+    cleanup_candidate
+    if [ "$rc" -ne 0 ] && [ "$APPLY_STARTED" -eq 1 ] && [ "$DEPLOY_VERIFIED" -eq 0 ]; then
+        rollback_deploy || log "ERROR: 自動 rollback 失敗，保留精確 backup paths 供獨立恢復"
+    fi
+    exit "$rc"
+}
+
+remote_promtool_preflight() {
+    scp "$ALERT_RULES_FILE" "wooo@${TARGET_HOST}:${REMOTE_CANDIDATE_PATH}"
+    ssh wooo@${TARGET_HOST} "set -euo pipefail
+        docker cp '${REMOTE_CANDIDATE_PATH}' '${PROMETHEUS_CONTAINER}:${CONTAINER_CANDIDATE_PATH}'
+        docker exec '${PROMETHEUS_CONTAINER}' promtool check rules '${CONTAINER_CANDIDATE_PATH}'"
+    cleanup_candidate
+    log "✅ remote promtool preflight 通過"
+}
+
+trap on_exit EXIT
+
 file_sha() {
    if command -v sha256sum >/dev/null 2>&1; then
        sha256sum "$1" | awk '{print $1}'
@@ -59,6 +108,7 @@ log "✅ YAML 語法驗證通過"

 # Dry run 模式
 if [ "$DRY_RUN" = "--dry-run" ]; then
+    remote_promtool_preflight
    log "DRY RUN: would deploy $ALERT_RULES_FILE to ${TARGET_HOST}:${TARGET_ALERTS_PATH}"
    log "DRY RUN: would deploy $ALERT_RULES_FILE to ${TARGET_HOST}:${TARGET_ALERTS_CANONICAL_PATH}"
    log "DRY RUN: would deploy $SLO_RULES_FILE to ${TARGET_HOST}:${TARGET_SLO_PATH}"
@@ -70,13 +120,17 @@ if [ "$DRY_RUN" = "--dry-run" ]; then
    exit 0
 fi

-# 備份現有規則
-ssh wooo@${TARGET_HOST} "\
-    cp ${TARGET_ALERTS_PATH} ${TARGET_ALERTS_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \
-    cp ${TARGET_ALERTS_CANONICAL_PATH} ${TARGET_ALERTS_CANONICAL_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \
-    cp ${TARGET_SLO_PATH} ${TARGET_SLO_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true; \
-    cp ${TARGET_DRIFT_GUARD_PATH} ${TARGET_DRIFT_GUARD_PATH}.bak.\$(date +%Y%m%d%H%M%S) 2>/dev/null || true"
+# 使用 production Prometheus 的 promtool 驗證候選規則，通過後才能寫 active path。
+remote_promtool_preflight
+
+# 備份現有規則；任一 source 缺失時 fail closed，不開始 apply。
+ssh wooo@${TARGET_HOST} "set -euo pipefail
+    cp '${TARGET_ALERTS_PATH}' '${TARGET_ALERTS_BACKUP_PATH}'
+    cp '${TARGET_ALERTS_CANONICAL_PATH}' '${TARGET_ALERTS_CANONICAL_BACKUP_PATH}'
+    cp '${TARGET_SLO_PATH}' '${TARGET_SLO_BACKUP_PATH}'
+    cp '${TARGET_DRIFT_GUARD_PATH}' '${TARGET_DRIFT_GUARD_BACKUP_PATH}'"
 log "✅ 現有規則已備份"
+APPLY_STARTED=1

 # 部署新規則
 scp "$ALERT_RULES_FILE" wooo@${TARGET_HOST}:${TARGET_ALERTS_PATH}
@@ -112,7 +166,7 @@ fi
 log "✅ 遠端規則 hash 驗證通過"

 # Reload Prometheus
-ssh wooo@${TARGET_HOST} "curl -s -X POST ${PROMETHEUS_URL}/-/reload"
+ssh wooo@${TARGET_HOST} "curl -fsS -X POST ${PROMETHEUS_URL}/-/reload >/dev/null"
 sleep 3

 # 驗證規則數量
@@ -161,4 +215,16 @@ for rule in "${KEY_SLO_RULES[@]}"; do
    fi
 done

+REMOTE_RUNTIME_SHA=$(ssh wooo@${TARGET_HOST} "docker exec '${PROMETHEUS_CONTAINER}' sha256sum '${PROMETHEUS_RUNTIME_RULES}' | awk '{print \$1}'")
+if [ "$LOCAL_ALERTS_SHA" != "$REMOTE_RUNTIME_SHA" ]; then
+    echo "ERROR: Prometheus runtime rules hash 不一致 local=${LOCAL_ALERTS_SHA} runtime=${REMOTE_RUNTIME_SHA}"
+    exit 1
+fi
+log "✅ Prometheus runtime rules hash 與 canonical source 一致"
+
+bash scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh
+log "✅ Cold-start producer / active rules / Agent99 consumer scope verifier 通過"
+
+DEPLOY_VERIFIED=1
+log "Rollback receipts: ${TARGET_ALERTS_BACKUP_PATH}, ${TARGET_ALERTS_CANONICAL_BACKUP_PATH}"
 log "🎉 部署完成！所有關鍵規則已生效"
--- a/scripts/ops/recovery-scorecard-contract-check.py
+++ b/scripts/ops/recovery-scorecard-contract-check.py
@@ -23,7 +23,7 @@ else:

 DEFAULT_RULES = Path("ops/monitoring/alerts-unified.yml")
 DEFAULT_BASELINE = Path("ops/reboot-recovery/full-stack-backup-baseline.yml")
-EXPECTED_CORE = 'awoooi_recovery_core_ready{host="110",scope="110_120_121_188"}'
+EXPECTED_CORE = 'awoooi_recovery_core_ready{host="110",scope="110_112_120_121_188"}'
 EXPECTED_DR = 'awoooi_recovery_dr_offsite_ready{host="110"}'


--- a/scripts/reboot-recovery/full-stack-recovery-scorecard.sh
+++ b/scripts/reboot-recovery/full-stack-recovery-scorecard.sh
@@ -151,9 +151,9 @@ echo "AWOOOI full-stack recovery scorecard"
 date '+%Y-%m-%d %H:%M:%S %Z'
 echo

-cold_green="$(bool_metric 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"}')"
-cold_warn="$(metric_value 'awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"}' 2>/dev/null || echo 999)"
-cold_blocked="$(metric_value 'awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"}' 2>/dev/null || echo 999)"
+cold_green="$(bool_metric 'awoooi_cold_start_last_result{host="110",scope="110_112_120_121_188",result="green"}')"
+cold_warn="$(metric_value 'awoooi_cold_start_warn_gates{host="110",scope="110_112_120_121_188"}' 2>/dev/null || echo 999)"
+cold_blocked="$(metric_value 'awoooi_cold_start_blocked_gates{host="110",scope="110_112_120_121_188"}' 2>/dev/null || echo 999)"
 cold_alerts="$(metric_value 'count(ALERTS{alertname=~"ColdStart.*",alertstate="firing"})' 2>/dev/null || echo 999)"

 status_value CORE_COLD_START_GREEN "$cold_green"
--- a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py
+++ b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py
@@ -20,6 +20,7 @@ RECOVERY_SCORECARD = (
    ROOT / "scripts" / "reboot-recovery" / "full-stack-recovery-scorecard.sh"
 )
 VERIFY_DEPLOY = ROOT / "scripts" / "reboot-recovery" / "verify-cold-start-monitor-deploy.sh"
+DEPLOY_ALERTS = ROOT / "scripts" / "ops" / "deploy-alerts.sh"
 REPAIR_STARTUP_STUB = (
    ROOT / "scripts" / "reboot-recovery" / "repair-110-startup-script-stub.sh"
 )
@@ -72,6 +73,14 @@ def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None:
        encoding="utf-8"
    )
    deploy_verifier = VERIFY_DEPLOY.read_text(encoding="utf-8")
+    alerts = (ROOT / "ops" / "monitoring" / "alerts-unified.yml").read_text(
+        encoding="utf-8"
+    )
+    recovery_scorecard = RECOVERY_SCORECARD.read_text(encoding="utf-8")
+    ansible_verifier = (
+        ROOT / "infra" / "ansible" / "roles" / "cold-start-monitor"
+        / "tasks" / "main.yml"
+    ).read_text(encoding="utf-8")

    assert "Scope: 110 / 112 / 120 / 121 / 188." in text
    assert "for host in 110 112 120 121 188; do" in text
@@ -86,6 +95,29 @@ def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None:
    assert 'reason="host_unreachable",target="112"' in exporter
    assert 'reason="guest_readiness_failed",target="112"' in exporter
    assert 'scope="110_112_120_121_188"' in deploy_verifier
+    assert 'base_url + "/api/v1/rules"' in deploy_verifier
+    assert "COLD_START_RULE_SCOPE_OK" in deploy_verifier
+    assert 'stale_scope = \'scope="110_120_121_188"\'' in deploy_verifier
+    assert 'scope="110_112_120_121_188"' in alerts
+    assert 'scope="110_112_120_121_188"' in recovery_scorecard
+    assert 'scope="110_112_120_121_188"' in ansible_verifier
+    assert 'scope="110_120_121_188"' not in alerts
+    assert 'scope="110_120_121_188"' not in recovery_scorecard
+    assert 'scope="110_120_121_188"' not in ansible_verifier
+
+
+def test_alert_rule_deploy_is_preflighted_verified_and_rollback_capable() -> None:
+    deploy = DEPLOY_ALERTS.read_text(encoding="utf-8")
+
+    assert "remote_promtool_preflight" in deploy
+    assert "promtool check rules" in deploy
+    assert "trap on_exit EXIT" in deploy
+    assert "rollback_deploy" in deploy
+    assert 'APPLY_STARTED=1' in deploy
+    assert 'DEPLOY_VERIFIED=1' in deploy
+    assert "PROMETHEUS_RUNTIME_RULES" in deploy
+    assert "verify-cold-start-monitor-deploy.sh" in deploy
+    assert "curl -fsS -X POST" in deploy


 def test_cold_start_momo_current_month_handles_no_new_source_without_false_warn() -> None:
--- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py
+++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py
@@ -15,6 +15,13 @@ def read(path: str) -> str:
    return (ROOT / path).read_text(encoding="utf-8")


+def test_reboot_sop_never_embeds_passworded_sudo_startup() -> None:
+    sop = read("docs/runbooks/REBOOT-RECOVERY-SOP.md")
+
+    assert "sudo -n /usr/local/bin/awoooi-startup-110.sh" in sop
+    assert "| sudo -S /usr/local/bin/awoooi-startup-110.sh" not in sop
+
+
 def load_public_maintenance_probe() -> ModuleType:
    path = ROOT / "scripts/reboot-recovery/public-maintenance-fallback-probe.py"
    spec = importlib.util.spec_from_file_location(
--- a/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh
+++ b/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh
@@ -95,6 +95,63 @@ else:
 PY
 }

+verify_cold_start_rule_scope() {
+  PROMETHEUS_URL="$PROMETHEUS_URL" python3 - <<'PY'
+import json
+import os
+import urllib.request
+
+base_url = os.environ["PROMETHEUS_URL"].rstrip("/")
+canonical_scope = 'scope="110_112_120_121_188"'
+stale_scope = 'scope="110_120_121_188"'
+required_rules = {
+    "awoooi_recovery_core_ready",
+    "ColdStartMonitorMissing",
+    "ColdStartMonitorStale",
+    "ColdStartRecoveryBlocked",
+    "K3sNodeFilesystemErrorGateBlocked",
+    "ColdStartHost120Unreachable",
+    "ColdStartRecoveryDegraded",
+    "ColdStartLastGreenTooOld",
+}
+
+try:
+    payload = json.load(
+        urllib.request.urlopen(base_url + "/api/v1/rules", timeout=8)
+    )
+except Exception as exc:  # noqa: BLE001
+    raise SystemExit(f"BLOCKED cold-start rule scope readback failed: {exc}") from exc
+
+if payload.get("status") != "success":
+    raise SystemExit("BLOCKED cold-start rule scope readback returned non-success")
+
+queries = {}
+for group in payload.get("data", {}).get("groups", []):
+    for rule in group.get("rules", []):
+        name = str(rule.get("name") or "")
+        if name in required_rules:
+            queries[name] = str(rule.get("query") or rule.get("expr") or "")
+
+missing = sorted(required_rules - queries.keys())
+wrong_scope = sorted(
+    name for name, query in queries.items() if canonical_scope not in query
+)
+stale = sorted(name for name, query in queries.items() if stale_scope in query)
+if missing or wrong_scope or stale:
+    raise SystemExit(
+        "BLOCKED cold-start rule scope drift "
+        f"missing={','.join(missing) or 'none'} "
+        f"wrong_scope={','.join(wrong_scope) or 'none'} "
+        f"stale_scope={','.join(stale) or 'none'}"
+    )
+
+print(
+    "COLD_START_RULE_SCOPE_OK "
+    f"scope=110_112_120_121_188 rules={len(required_rules)} stale_scope=0"
+)
+PY
+}
+
 report_runtime_state() {
  remote_read "awk '
    /awoooi_cold_start_monitor_up/ {monitor_up=\$NF}
@@ -139,6 +196,7 @@ require_remote_pattern \
  "110 cold-start monitor exports host112 readiness blocker series"

 report_runtime_state
+verify_cold_start_rule_scope
 report_cold_start_alerts

 echo "COLD_START_MONITOR_DEPLOY_PARITY_OK"