4af6487ba6
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 2s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m31s
CD Pipeline / build-and-deploy (push) Successful in 15m18s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 22s
CD Pipeline / post-deploy-checks (push) Successful in 5m35s
109 lines
3.9 KiB
Python
109 lines
3.9 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
CANONICAL_RULES = ROOT / "ops" / "monitoring" / "alerts-unified.yml"
|
|
MIRROR_RULES = ROOT / "ops" / "monitoring" / "alerts.yml"
|
|
K8S_RULES = ROOT / "k8s" / "monitoring" / "alert-chain-monitor.yaml"
|
|
PROMETHEUS_CONFIG = ROOT / "k8s" / "monitoring" / "prometheus.yml"
|
|
RECOVERY_PLAYBOOK = (
|
|
ROOT / "infra" / "ansible" / "playbooks" / "110-alertmanager-delivery-recovery.yml"
|
|
)
|
|
RUNTIME_SCRAPE_DEPLOY = ROOT / "scripts" / "ops" / "deploy-alertmanager-runtime-scrape.sh"
|
|
|
|
|
|
def _alert_rule(path: Path) -> dict:
|
|
payload = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
groups = payload["spec"]["groups"] if "spec" in payload else payload["groups"]
|
|
return next(
|
|
rule
|
|
for group in groups
|
|
for rule in group.get("rules", [])
|
|
if rule.get("alert") == "AlertChainBroken_Alertmanager"
|
|
)
|
|
|
|
|
|
def _normalized_expr(path: Path) -> str:
|
|
return re.sub(r"\s+", "", _alert_rule(path)["expr"])
|
|
|
|
|
|
def test_alertmanager_delivery_truth_is_scraped_from_single_runtime() -> None:
|
|
payload = yaml.safe_load(PROMETHEUS_CONFIG.read_text(encoding="utf-8"))
|
|
job = next(
|
|
row
|
|
for row in payload["scrape_configs"]
|
|
if row.get("job_name") == "alertmanager-runtime"
|
|
)
|
|
|
|
assert job["metrics_path"] == "/metrics"
|
|
assert job["static_configs"] == [
|
|
{
|
|
"targets": ["alertmanager:9093"],
|
|
"labels": {
|
|
"host": "110",
|
|
"service": "alertmanager",
|
|
"env": "prod",
|
|
},
|
|
}
|
|
]
|
|
assert job["metric_relabel_configs"] == [
|
|
{
|
|
"source_labels": ["__name__", "integration"],
|
|
"separator": ";",
|
|
"regex": "alertmanager_notification_requests(_failed)?_total;webhook",
|
|
"target_label": "receiver_contract",
|
|
"replacement": "awoooi-webhook",
|
|
}
|
|
]
|
|
blackbox = next(
|
|
row
|
|
for row in payload["scrape_configs"]
|
|
if row.get("job_name") == "blackbox-tcp"
|
|
)
|
|
assert "192.168.0.110:9093" in blackbox["static_configs"][0]["targets"]
|
|
|
|
|
|
def test_alert_chain_rule_uses_monotonic_alertmanager_counters_in_all_sources() -> None:
|
|
expressions = {
|
|
_normalized_expr(path)
|
|
for path in (CANONICAL_RULES, MIRROR_RULES, K8S_RULES)
|
|
}
|
|
|
|
assert len(expressions) == 1
|
|
expression = expressions.pop()
|
|
assert "alertmanager_notification_requests_failed_total" in expression
|
|
assert "alertmanager_notification_requests_total" in expression
|
|
assert 'job="alertmanager-runtime"' in expression
|
|
assert 'integration="webhook"' in expression
|
|
assert 'receiver_contract="awoooi-webhook"' in expression
|
|
assert ">=5" in expression
|
|
assert "awoooi_webhook_requests_total" not in expression
|
|
|
|
|
|
def test_alert_chain_rule_is_exact_asset_controlled_and_not_no_action() -> None:
|
|
for path in (CANONICAL_RULES, MIRROR_RULES, K8S_RULES):
|
|
rule = _alert_rule(path)
|
|
labels = rule["labels"]
|
|
assert labels["component"] == "alertmanager"
|
|
assert labels["auto_repair"] == "true"
|
|
assert labels["alert_category"] == "alert_chain_health"
|
|
assert rule["for"] == "5m"
|
|
if path in {CANONICAL_RULES, MIRROR_RULES}:
|
|
assert labels["layer"] == "host"
|
|
|
|
|
|
def test_runtime_scrape_and_recovery_preflight_match_production_metric_schema() -> None:
|
|
for path in (RECOVERY_PLAYBOOK, RUNTIME_SCRAPE_DEPLOY):
|
|
source = path.read_text(encoding="utf-8")
|
|
assert 'integration="webhook"' in source
|
|
assert "alertmanager_notification_requests_total" in source
|
|
assert "alertmanager_notification_requests_failed_total" in source
|
|
|
|
runtime_source = RUNTIME_SCRAPE_DEPLOY.read_text(encoding="utf-8")
|
|
assert "target_label: receiver_contract" in runtime_source
|
|
assert "replacement: 'awoooi-webhook'" in runtime_source
|