from __future__ import annotations import re from pathlib import Path import yaml ROOT = Path(__file__).resolve().parents[3] CANONICAL_RULES = ROOT / "ops" / "monitoring" / "alerts-unified.yml" MIRROR_RULES = ROOT / "ops" / "monitoring" / "alerts.yml" K8S_RULES = ROOT / "k8s" / "monitoring" / "alert-chain-monitor.yaml" PROMETHEUS_CONFIG = ROOT / "k8s" / "monitoring" / "prometheus.yml" RECOVERY_PLAYBOOK = ( ROOT / "infra" / "ansible" / "playbooks" / "110-alertmanager-delivery-recovery.yml" ) RUNTIME_SCRAPE_DEPLOY = ROOT / "scripts" / "ops" / "deploy-alertmanager-runtime-scrape.sh" def _alert_rule(path: Path) -> dict: payload = yaml.safe_load(path.read_text(encoding="utf-8")) groups = payload["spec"]["groups"] if "spec" in payload else payload["groups"] return next( rule for group in groups for rule in group.get("rules", []) if rule.get("alert") == "AlertChainBroken_Alertmanager" ) def _normalized_expr(path: Path) -> str: return re.sub(r"\s+", "", _alert_rule(path)["expr"]) def test_alertmanager_delivery_truth_is_scraped_from_single_runtime() -> None: payload = yaml.safe_load(PROMETHEUS_CONFIG.read_text(encoding="utf-8")) job = next( row for row in payload["scrape_configs"] if row.get("job_name") == "alertmanager-runtime" ) assert job["metrics_path"] == "/metrics" assert job["static_configs"] == [ { "targets": ["alertmanager:9093"], "labels": { "host": "110", "service": "alertmanager", "env": "prod", }, } ] assert job["metric_relabel_configs"] == [ { "source_labels": ["__name__", "integration"], "separator": ";", "regex": "alertmanager_notification_requests(_failed)?_total;webhook", "target_label": "receiver_contract", "replacement": "awoooi-webhook", } ] blackbox = next( row for row in payload["scrape_configs"] if row.get("job_name") == "blackbox-tcp" ) assert "192.168.0.110:9093" in blackbox["static_configs"][0]["targets"] def test_alert_chain_rule_uses_monotonic_alertmanager_counters_in_all_sources() -> None: expressions = { _normalized_expr(path) for path in (CANONICAL_RULES, MIRROR_RULES, K8S_RULES) } assert len(expressions) == 1 expression = expressions.pop() assert "alertmanager_notification_requests_failed_total" in expression assert "alertmanager_notification_requests_total" in expression assert 'job="alertmanager-runtime"' in expression assert 'integration="webhook"' in expression assert 'receiver_contract="awoooi-webhook"' in expression assert ">=5" in expression assert "awoooi_webhook_requests_total" not in expression def test_alert_chain_rule_is_exact_asset_controlled_and_not_no_action() -> None: for path in (CANONICAL_RULES, MIRROR_RULES, K8S_RULES): rule = _alert_rule(path) labels = rule["labels"] assert labels["component"] == "alertmanager" assert labels["auto_repair"] == "true" assert labels["alert_category"] == "alert_chain_health" assert rule["for"] == "5m" if path in {CANONICAL_RULES, MIRROR_RULES}: assert labels["layer"] == "host" def test_runtime_scrape_and_recovery_preflight_match_production_metric_schema() -> None: for path in (RECOVERY_PLAYBOOK, RUNTIME_SCRAPE_DEPLOY): source = path.read_text(encoding="utf-8") assert 'integration="webhook"' in source assert "alertmanager_notification_requests_total" in source assert "alertmanager_notification_requests_failed_total" in source runtime_source = RUNTIME_SCRAPE_DEPLOY.read_text(encoding="utf-8") assert "target_label: receiver_contract" in runtime_source assert "replacement: 'awoooi-webhook'" in runtime_source