fix(recovery): align host112 cold-start metric scope

scripts/ops/recovery-scorecard-contract-check.py

@@ -23,7 +23,7 @@ else:
 
 DEFAULT_RULES = Path("ops/monitoring/alerts-unified.yml")
 DEFAULT_BASELINE = Path("ops/reboot-recovery/full-stack-backup-baseline.yml")
-EXPECTED_CORE = 'awoooi_recovery_core_ready{host="110",scope="110_120_121_188"}'
+EXPECTED_CORE = 'awoooi_recovery_core_ready{host="110",scope="110_112_120_121_188"}'
 EXPECTED_DR = 'awoooi_recovery_dr_offsite_ready{host="110"}'
 
 

scripts/reboot-recovery/full-stack-recovery-scorecard.sh

@@ -151,9 +151,9 @@ echo "AWOOOI full-stack recovery scorecard"
 date '+%Y-%m-%d %H:%M:%S %Z'
 echo
 
-cold_green="$(bool_metric 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"}')"
-cold_warn="$(metric_value 'awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"}' 2>/dev/null || echo 999)"
-cold_blocked="$(metric_value 'awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"}' 2>/dev/null || echo 999)"
+cold_green="$(bool_metric 'awoooi_cold_start_last_result{host="110",scope="110_112_120_121_188",result="green"}')"
+cold_warn="$(metric_value 'awoooi_cold_start_warn_gates{host="110",scope="110_112_120_121_188"}' 2>/dev/null || echo 999)"
+cold_blocked="$(metric_value 'awoooi_cold_start_blocked_gates{host="110",scope="110_112_120_121_188"}' 2>/dev/null || echo 999)"
 cold_alerts="$(metric_value 'count(ALERTS{alertname=~"ColdStart.*",alertstate="firing"})' 2>/dev/null || echo 999)"
 
 status_value CORE_COLD_START_GREEN "$cold_green"

scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py

@@ -72,6 +72,14 @@ def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None:
         encoding="utf-8"
     )
     deploy_verifier = VERIFY_DEPLOY.read_text(encoding="utf-8")
+    alerts = (ROOT / "ops" / "monitoring" / "alerts-unified.yml").read_text(
+        encoding="utf-8"
+    )
+    recovery_scorecard = RECOVERY_SCORECARD.read_text(encoding="utf-8")
+    ansible_verifier = (
+        ROOT / "infra" / "ansible" / "roles" / "cold-start-monitor"
+        / "tasks" / "main.yml"
+    ).read_text(encoding="utf-8")
 
     assert "Scope: 110 / 112 / 120 / 121 / 188." in text
     assert "for host in 110 112 120 121 188; do" in text
@@ -86,6 +94,15 @@ def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None:
     assert 'reason="host_unreachable",target="112"' in exporter
     assert 'reason="guest_readiness_failed",target="112"' in exporter
     assert 'scope="110_112_120_121_188"' in deploy_verifier
+    assert 'base_url + "/api/v1/rules"' in deploy_verifier
+    assert "COLD_START_RULE_SCOPE_OK" in deploy_verifier
+    assert 'stale_scope = \'scope="110_120_121_188"\'' in deploy_verifier
+    assert 'scope="110_112_120_121_188"' in alerts
+    assert 'scope="110_112_120_121_188"' in recovery_scorecard
+    assert 'scope="110_112_120_121_188"' in ansible_verifier
+    assert 'scope="110_120_121_188"' not in alerts
+    assert 'scope="110_120_121_188"' not in recovery_scorecard
+    assert 'scope="110_120_121_188"' not in ansible_verifier
 
 
 def test_cold_start_momo_current_month_handles_no_new_source_without_false_warn() -> None:

scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh

@@ -95,6 +95,63 @@ else:
 PY
 }
 
+verify_cold_start_rule_scope() {
+  PROMETHEUS_URL="$PROMETHEUS_URL" python3 - <<'PY'
+import json
+import os
+import urllib.request
+
+base_url = os.environ["PROMETHEUS_URL"].rstrip("/")
+canonical_scope = 'scope="110_112_120_121_188"'
+stale_scope = 'scope="110_120_121_188"'
+required_rules = {
+    "awoooi_recovery_core_ready",
+    "ColdStartMonitorMissing",
+    "ColdStartMonitorStale",
+    "ColdStartRecoveryBlocked",
+    "K3sNodeFilesystemErrorGateBlocked",
+    "ColdStartHost120Unreachable",
+    "ColdStartRecoveryDegraded",
+    "ColdStartLastGreenTooOld",
+}
+
+try:
+    payload = json.load(
+        urllib.request.urlopen(base_url + "/api/v1/rules", timeout=8)
+    )
+except Exception as exc:  # noqa: BLE001
+    raise SystemExit(f"BLOCKED cold-start rule scope readback failed: {exc}") from exc
+
+if payload.get("status") != "success":
+    raise SystemExit("BLOCKED cold-start rule scope readback returned non-success")
+
+queries = {}
+for group in payload.get("data", {}).get("groups", []):
+    for rule in group.get("rules", []):
+        name = str(rule.get("name") or "")
+        if name in required_rules:
+            queries[name] = str(rule.get("query") or rule.get("expr") or "")
+
+missing = sorted(required_rules - queries.keys())
+wrong_scope = sorted(
+    name for name, query in queries.items() if canonical_scope not in query
+)
+stale = sorted(name for name, query in queries.items() if stale_scope in query)
+if missing or wrong_scope or stale:
+    raise SystemExit(
+        "BLOCKED cold-start rule scope drift "
+        f"missing={','.join(missing) or 'none'} "
+        f"wrong_scope={','.join(wrong_scope) or 'none'} "
+        f"stale_scope={','.join(stale) or 'none'}"
+    )
+
+print(
+    "COLD_START_RULE_SCOPE_OK "
+    f"scope=110_112_120_121_188 rules={len(required_rules)} stale_scope=0"
+)
+PY
+}
+
 report_runtime_state() {
   remote_read "awk '
     /awoooi_cold_start_monitor_up/ {monitor_up=\$NF}
@@ -139,6 +196,7 @@ require_remote_pattern \
   "110 cold-start monitor exports host112 readiness blocker series"
 
 report_runtime_state
+verify_cold_start_rule_scope
 report_cold_start_alerts
 
 echo "COLD_START_MONITOR_DEPLOY_PARITY_OK"