awoooi/scripts/ops/deploy-alertmanager-runtime-scrape.sh

#!/usr/bin/env bash
# Bounded Prometheus scrape-source convergence for Alertmanager delivery truth.

set -euo pipefail

TRACE_ID="${TRACE_ID:?TRACE_ID is required}"
RUN_ID="${RUN_ID:?RUN_ID is required}"
WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"
TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
PROMETHEUS_URL="${PROMETHEUS_URL:-http://192.168.0.110:9090}"
ALERTMANAGER_METRICS_URL="${ALERTMANAGER_METRICS_URL:-http://127.0.0.1:9093/metrics}"
PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
REMOTE_CONFIG="${REMOTE_CONFIG:-/home/wooo/monitoring/prometheus.yml}"
MODE="${1:---dry-run}"

case "${MODE}" in
    apply|--dry-run) ;;
    *) printf 'usage: %s [--dry-run|apply]\n' "$0" >&2; exit 64 ;;
esac

for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
    case "${value}" in
        *[!A-Za-z0-9._-]*|'')
            printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2
            exit 64
            ;;
    esac
done

ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \
    bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
    "${PROMETHEUS_URL}" "${ALERTMANAGER_METRICS_URL}" \
    "${PROMETHEUS_CONTAINER}" "${REMOTE_CONFIG}" <<'REMOTE'
set -euo pipefail

TRACE_ID="$1"
RUN_ID="$2"
WORK_ITEM_ID="$3"
MODE="$4"
PROMETHEUS_URL="$5"
ALERTMANAGER_METRICS_URL="$6"
PROMETHEUS_CONTAINER="$7"
REMOTE_CONFIG="$8"
REMOTE_CANDIDATE="/tmp/awoooi-prometheus-alertmanager-runtime.${RUN_ID}.yml"
CONTAINER_CANDIDATE="/etc/prometheus/.awoooi-alertmanager-runtime.${RUN_ID}.yml"
BACKUP_CONFIG="${REMOTE_CONFIG}.bak.${RUN_ID}"
APPLY_STARTED=0
DEPLOY_VERIFIED=0
SOURCE_SHA=""
CANDIDATE_SHA=""

exec 9>/tmp/awoooi-prometheus-config.lock
if ! flock -n 9; then
    printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=blocked_with_safe_next_action reason=prometheus_config_apply_in_progress\n' \
        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" >&2
    exit 75
fi

cleanup() {
    rm -f "${REMOTE_CANDIDATE}"
    docker exec -u 0 "${PROMETHEUS_CONTAINER}" \
        rm -f "${CONTAINER_CANDIDATE}" >/dev/null 2>&1 || true
}

rollback() {
    local rollback_rc=0
    local current_sha backup_sha restored_sha
    current_sha="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" || rollback_rc=1
    backup_sha="$(sha256sum "${BACKUP_CONFIG}" | awk '{print $1}')" || rollback_rc=1
    if [ "${rollback_rc}" -eq 0 ] && [ "${current_sha}" = "${SOURCE_SHA}" ]; then
        printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=no_write_source_intact source_sha=%s\n' \
            "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${SOURCE_SHA}"
        return 0
    fi
    if [ "${rollback_rc}" -ne 0 ] \
        || [ "${current_sha}" != "${CANDIDATE_SHA}" ] \
        || [ "${backup_sha}" != "${SOURCE_SHA}" ]; then
        rollback_rc=1
    else
        cp "${BACKUP_CONFIG}" "${REMOTE_CONFIG}" || rollback_rc=1
        restored_sha="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" || rollback_rc=1
        [ "${restored_sha}" = "${SOURCE_SHA}" ] || rollback_rc=1
    fi
    if [ "${rollback_rc}" -eq 0 ]; then
        curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null \
            || rollback_rc=1
    fi
    if [ "${rollback_rc}" -eq 0 ]; then
        curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null || rollback_rc=1
    fi
    if [ "${rollback_rc}" -eq 0 ]; then
        printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rolled_back ready=true backup=%s\n' \
            "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
        return 0
    fi
    printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rollback_failed ready=false backup=%s\n' \
        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}" >&2
    return 1
}

on_exit() {
    rc=$?
    trap - EXIT
    set +e
    cleanup
    rollback_rc=0
    if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] \
        && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
        rollback || rollback_rc=$?
    fi
    if [ "${rollback_rc}" -ne 0 ]; then
        exit 70
    fi
    exit "${rc}"
}
trap on_exit EXIT

test -f "${REMOTE_CONFIG}"
test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
METRICS_SNAPSHOT="$(curl -fsS "${ALERTMANAGER_METRICS_URL}")"
grep -Eq '^alertmanager_notification_requests_total\{[^}]*integration="webhook"' <<<"${METRICS_SNAPSHOT}"
grep -Eq '^alertmanager_notification_requests_failed_total\{[^}]*integration="webhook"' <<<"${METRICS_SNAPSHOT}"
SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"

python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY'
from pathlib import Path
import re
import sys

source = Path(sys.argv[1])
candidate = Path(sys.argv[2])
text = source.read_text(encoding="utf-8")

begin = "  # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH\n"
end = "  # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH\n"
block = """  # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH
  - job_name: 'alertmanager-runtime'
    scrape_interval: 30s
    metrics_path: /metrics
    static_configs:
      - targets: ['alertmanager:9093']
        labels:
          host: '110'
          service: 'alertmanager'
          env: 'prod'
    metric_relabel_configs:
      - source_labels: [__name__, integration]
        separator: ';'
        regex: 'alertmanager_notification_requests(_failed)?_total;webhook'
        target_label: receiver_contract
        replacement: 'awoooi-webhook'
  # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH

"""

if text.count(begin) != text.count(end) or text.count(begin) > 1:
    raise SystemExit("alertmanager runtime managed block is ambiguous")
if begin in text:
    pattern = re.escape(begin) + r".*?" + re.escape(end) + r"\n?"
    text, count = re.subn(pattern, block, text, count=1, flags=re.S)
    if count != 1:
        raise SystemExit("managed block replacement failed")
else:
    legacy_pattern = (
        r"(?ms)^  - job_name: ['\"]alertmanager-runtime['\"]\n"
        r".*?(?=^  - job_name:|\Z)"
    )
    legacy_matches = list(re.finditer(legacy_pattern, text))
    if len(legacy_matches) == 1:
        text = re.sub(legacy_pattern, block, text, count=1)
    elif legacy_matches:
        raise SystemExit("legacy alertmanager runtime job is ambiguous")
    else:
        anchors = (
            "  # === AWOOOI API Metrics",
            "  - job_name: 'awoooi-api'\n",
        )
        anchor = next((value for value in anchors if value in text), None)
        if anchor is None or text.count(anchor) != 1:
            raise SystemExit("awoooi-api insertion anchor missing or ambiguous")
        text = text.replace(anchor, block + anchor, 1)

provider_begin = "      # BEGIN AWOOOI OLLAMA CLOUD EDGE PROBES\n"
provider_end = "      # END AWOOOI OLLAMA CLOUD EDGE PROBES\n"
provider_block = """      # BEGIN AWOOOI OLLAMA CLOUD EDGE PROBES
      - targets:
          - http://34.143.170.20:11434/api/tags
        labels:
          service: ollama-cloud-edge
          provider: ollama_gcp_a
          probe_origin: host110
          boundary: cloud_edge_availability_only
      - targets:
          - http://34.21.145.224:11434/api/tags
        labels:
          service: ollama-cloud-edge
          provider: ollama_gcp_b
          probe_origin: host110
          boundary: cloud_edge_availability_only
      # END AWOOOI OLLAMA CLOUD EDGE PROBES
"""
if (
    text.count(provider_begin) != text.count(provider_end)
    or text.count(provider_begin) > 1
):
    raise SystemExit("Ollama cloud edge managed block is ambiguous")
if provider_begin in text:
    provider_pattern = (
        re.escape(provider_begin)
        + r".*?"
        + re.escape(provider_end)
    )
    text, count = re.subn(
        provider_pattern,
        provider_block,
        text,
        count=1,
        flags=re.S,
    )
    if count != 1:
        raise SystemExit("Ollama cloud edge managed block replacement failed")
else:
    blackbox_match = re.search(
        r"(?ms)^  - job_name: ['\"]blackbox-http['\"]\n"
        r".*?(?=^  - job_name:|\Z)",
        text,
    )
    if blackbox_match is None:
        raise SystemExit("blackbox-http job missing")
    blackbox_job = blackbox_match.group(0)
    if "provider: ollama_gcp_" in blackbox_job:
        raise SystemExit("unmanaged Ollama cloud edge targets are ambiguous")
    relabel_anchor = "    relabel_configs:\n"
    if blackbox_job.count(relabel_anchor) != 1:
        raise SystemExit("blackbox-http relabel anchor missing or ambiguous")
    patched_job = blackbox_job.replace(
        relabel_anchor,
        provider_block + relabel_anchor,
        1,
    )
    text = text[: blackbox_match.start()] + patched_job + text[blackbox_match.end() :]

if text.count("job_name: 'alertmanager-runtime'") != 1:
    raise SystemExit("alertmanager runtime job must exist exactly once")
if "targets: ['alertmanager:9093']" not in text:
    raise SystemExit("canonical Alertmanager target missing")
if text.count("provider: ollama_gcp_a") != 1:
    raise SystemExit("canonical GCP-A cloud edge target missing or ambiguous")
if text.count("provider: ollama_gcp_b") != 1:
    raise SystemExit("canonical GCP-B cloud edge target missing or ambiguous")
if "192.168.0.111:11434" in text:
    raise SystemExit("host111 must not be probed from retired host110 boundary")
candidate.write_text(text, encoding="utf-8")
PY

docker cp "${REMOTE_CANDIDATE}" \
    "${PROMETHEUS_CONTAINER}:${CONTAINER_CANDIDATE}" >/dev/null
docker exec "${PROMETHEUS_CONTAINER}" promtool check config "${CONTAINER_CANDIDATE}"

CANDIDATE_SHA="$(sha256sum "${REMOTE_CANDIDATE}" | awk '{print $1}')"
test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s source_sha=%s candidate_sha=%s promtool=pass persistent_writes_performed=0 temporary_validation_artifact=true\n' \
    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
    "${SOURCE_SHA}" "${CANDIDATE_SHA}"

if [ "${MODE}" = "--dry-run" ]; then
    printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=check_pass_no_persistent_write\n' \
        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"
    exit 0
fi

cp -p "${REMOTE_CONFIG}" "${BACKUP_CONFIG}"
test "$(sha256sum "${BACKUP_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
APPLY_STARTED=1
cp "${REMOTE_CANDIDATE}" "${REMOTE_CONFIG}"
test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${CANDIDATE_SHA}"
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=alertmanager_runtime_scrape_converged active_sha=%s backup=%s\n' \
    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${CANDIDATE_SHA}" \
    "${BACKUP_CONFIG}"

target_readback() {
    curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
import json
import sys

targets = json.load(sys.stdin)["data"]["activeTargets"]
matches = [
    target for target in targets
    if target.get("labels", {}).get("job") == "alertmanager-runtime"
]
if len(matches) != 1:
    print(f"{len(matches)}|missing|never")
else:
    target = matches[0]
    print("{}|{}|{}".format(
        len(matches),
        target.get("health", "unknown"),
        target.get("lastScrape", "never"),
    ))
'
}

provider_target_readback() {
    curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
import json
import sys

targets = json.load(sys.stdin)["data"]["activeTargets"]
providers = ("ollama_gcp_a", "ollama_gcp_b")
matches = {
    provider: [
        target
        for target in targets
        if target.get("labels", {}).get("job") == "blackbox-http"
        and target.get("labels", {}).get("provider") == provider
        and target.get("labels", {}).get("probe_origin") == "host110"
    ]
    for provider in providers
}
ready = all(
    len(matches[provider]) == 1
    and matches[provider][0].get("lastScrape", "never") != "never"
    for provider in providers
)
values = []
for provider in providers:
    target = matches[provider][0] if len(matches[provider]) == 1 else {}
    values.extend([
        target.get("lastScrape", "never"),
        target.get("health", "missing"),
    ])
print("|".join(["1" if ready else "0", *values]))
'
}

FIRST_SCRAPE=""
for attempt in $(seq 1 9); do
    IFS='|' read -r count health last_scrape <<<"$(target_readback)"
    printf 'target_probe trace_id=%s run_id=%s attempt=%s count=%s health=%s last_scrape=%s\n' \
        "${TRACE_ID}" "${RUN_ID}" "${attempt}" "${count}" "${health}" \
        "${last_scrape}"
    if [ "${count}" = 1 ] && [ "${health}" = up ] \
        && [ "${last_scrape}" != never ]; then
        FIRST_SCRAPE="${last_scrape}"
        break
    fi
    sleep 10
done
test -n "${FIRST_SCRAPE}"

FIRST_GCP_A_SCRAPE=""
FIRST_GCP_B_SCRAPE=""
for attempt in $(seq 1 9); do
    IFS='|' read -r providers_ready gcp_a_scrape gcp_a_health \
        gcp_b_scrape gcp_b_health <<<"$(provider_target_readback)"
    printf 'provider_target_probe trace_id=%s run_id=%s attempt=%s ready=%s gcp_a_health=%s gcp_b_health=%s\n' \
        "${TRACE_ID}" "${RUN_ID}" "${attempt}" "${providers_ready}" \
        "${gcp_a_health}" "${gcp_b_health}"
    if [ "${providers_ready}" = 1 ]; then
        FIRST_GCP_A_SCRAPE="${gcp_a_scrape}"
        FIRST_GCP_B_SCRAPE="${gcp_b_scrape}"
        break
    fi
    sleep 10
done
test -n "${FIRST_GCP_A_SCRAPE}"
test -n "${FIRST_GCP_B_SCRAPE}"

sleep 31
IFS='|' read -r count health SECOND_SCRAPE <<<"$(target_readback)"
test "${count}" = 1
test "${health}" = up
test "${SECOND_SCRAPE}" != "${FIRST_SCRAPE}"
IFS='|' read -r providers_ready SECOND_GCP_A_SCRAPE gcp_a_health \
    SECOND_GCP_B_SCRAPE gcp_b_health <<<"$(provider_target_readback)"
test "${providers_ready}" = 1
test "${SECOND_GCP_A_SCRAPE}" != "${FIRST_GCP_A_SCRAPE}"
test "${SECOND_GCP_B_SCRAPE}" != "${FIRST_GCP_B_SCRAPE}"

SERIES_COUNT="$(curl -fsS --get \
    --data-urlencode 'query=alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver_contract="awoooi-webhook"}' \
    "${PROMETHEUS_URL}/api/v1/query" | python3 -c '
import json
import sys
print(len(json.load(sys.stdin)["data"]["result"]))
')"
test "${SERIES_COUNT}" = 1

FAILED_SERIES_COUNT="$(curl -fsS --get \
    --data-urlencode 'query=alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver_contract="awoooi-webhook"}' \
    "${PROMETHEUS_URL}/api/v1/query" | python3 -c '
import json
import sys
print(len(json.load(sys.stdin)["data"]["result"]))
')"
test "${FAILED_SERIES_COUNT}" = 1

PROVIDER_SERIES_COUNT="$(curl -fsS --get \
    --data-urlencode 'query=probe_success{job="blackbox-http",provider=~"ollama_gcp_a|ollama_gcp_b",probe_origin="host110"}' \
    "${PROMETHEUS_URL}/api/v1/query" | python3 -c '
import json
import sys
print(len(json.load(sys.stdin)["data"]["result"]))
')"
test "${PROVIDER_SERIES_COUNT}" = 2
test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${CANDIDATE_SHA}"

DEPLOY_VERIFIED=1
printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s terminal=two_scrapes_up first=%s second=%s delivery_series_count=%s failed_series_count=%s provider_series_count=%s gcp_a_health=%s gcp_b_health=%s\n' \
    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${FIRST_SCRAPE}" \
    "${SECOND_SCRAPE}" "${SERIES_COUNT}" "${FAILED_SERIES_COUNT}" \
    "${PROVIDER_SERIES_COUNT}" "${gcp_a_health}" "${gcp_b_health}"
printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=completed backup=%s\n' \
    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
REMOTE