feat(sre): enforce typed controlled automation

2026-07-16 17:32:01 +08:00
parent dfec0c9f7f
commit 541a32a9cb
130 changed files with 14849 additions and 1738 deletions
--- a/scripts/ops/deploy-alertmanager-runtime-scrape.sh
+++ b/scripts/ops/deploy-alertmanager-runtime-scrape.sh
@@ -0,0 +1,413 @@
+#!/usr/bin/env bash
+# Bounded Prometheus scrape-source convergence for Alertmanager delivery truth.
+
+set -euo pipefail
+
+TRACE_ID="${TRACE_ID:?TRACE_ID is required}"
+RUN_ID="${RUN_ID:?RUN_ID is required}"
+WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"
+TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
+PROMETHEUS_URL="${PROMETHEUS_URL:-http://192.168.0.110:9090}"
+ALERTMANAGER_METRICS_URL="${ALERTMANAGER_METRICS_URL:-http://127.0.0.1:9093/metrics}"
+PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
+REMOTE_CONFIG="${REMOTE_CONFIG:-/home/wooo/monitoring/prometheus.yml}"
+MODE="${1:---dry-run}"
+
+case "${MODE}" in
+    apply|--dry-run) ;;
+    *) printf 'usage: %s [--dry-run|apply]\n' "$0" >&2; exit 64 ;;
+esac
+
+for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
+    case "${value}" in
+        *[!A-Za-z0-9._-]*|'')
+            printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2
+            exit 64
+            ;;
+    esac
+done
+
+ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \
+    bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
+    "${PROMETHEUS_URL}" "${ALERTMANAGER_METRICS_URL}" \
+    "${PROMETHEUS_CONTAINER}" "${REMOTE_CONFIG}" <<'REMOTE'
+set -euo pipefail
+
+TRACE_ID="$1"
+RUN_ID="$2"
+WORK_ITEM_ID="$3"
+MODE="$4"
+PROMETHEUS_URL="$5"
+ALERTMANAGER_METRICS_URL="$6"
+PROMETHEUS_CONTAINER="$7"
+REMOTE_CONFIG="$8"
+REMOTE_CANDIDATE="/tmp/awoooi-prometheus-alertmanager-runtime.${RUN_ID}.yml"
+CONTAINER_CANDIDATE="/etc/prometheus/.awoooi-alertmanager-runtime.${RUN_ID}.yml"
+BACKUP_CONFIG="${REMOTE_CONFIG}.bak.${RUN_ID}"
+APPLY_STARTED=0
+DEPLOY_VERIFIED=0
+SOURCE_SHA=""
+CANDIDATE_SHA=""
+
+exec 9>/tmp/awoooi-prometheus-config.lock
+if ! flock -n 9; then
+    printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=blocked_with_safe_next_action reason=prometheus_config_apply_in_progress\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" >&2
+    exit 75
+fi
+
+cleanup() {
+    rm -f "${REMOTE_CANDIDATE}"
+    docker exec -u 0 "${PROMETHEUS_CONTAINER}" \
+        rm -f "${CONTAINER_CANDIDATE}" >/dev/null 2>&1 || true
+}
+
+rollback() {
+    local rollback_rc=0
+    local current_sha backup_sha restored_sha
+    current_sha="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" || rollback_rc=1
+    backup_sha="$(sha256sum "${BACKUP_CONFIG}" | awk '{print $1}')" || rollback_rc=1
+    if [ "${rollback_rc}" -eq 0 ] && [ "${current_sha}" = "${SOURCE_SHA}" ]; then
+        printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=no_write_source_intact source_sha=%s\n' \
+            "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${SOURCE_SHA}"
+        return 0
+    fi
+    if [ "${rollback_rc}" -ne 0 ] \
+        || [ "${current_sha}" != "${CANDIDATE_SHA}" ] \
+        || [ "${backup_sha}" != "${SOURCE_SHA}" ]; then
+        rollback_rc=1
+    else
+        cp "${BACKUP_CONFIG}" "${REMOTE_CONFIG}" || rollback_rc=1
+        restored_sha="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" || rollback_rc=1
+        [ "${restored_sha}" = "${SOURCE_SHA}" ] || rollback_rc=1
+    fi
+    if [ "${rollback_rc}" -eq 0 ]; then
+        curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null \
+            || rollback_rc=1
+    fi
+    if [ "${rollback_rc}" -eq 0 ]; then
+        curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null || rollback_rc=1
+    fi
+    if [ "${rollback_rc}" -eq 0 ]; then
+        printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rolled_back ready=true backup=%s\n' \
+            "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
+        return 0
+    fi
+    printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rollback_failed ready=false backup=%s\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}" >&2
+    return 1
+}
+
+on_exit() {
+    rc=$?
+    trap - EXIT
+    set +e
+    cleanup
+    rollback_rc=0
+    if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] \
+        && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
+        rollback || rollback_rc=$?
+    fi
+    if [ "${rollback_rc}" -ne 0 ]; then
+        exit 70
+    fi
+    exit "${rc}"
+}
+trap on_exit EXIT
+
+test -f "${REMOTE_CONFIG}"
+test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true
+curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
+METRICS_SNAPSHOT="$(curl -fsS "${ALERTMANAGER_METRICS_URL}")"
+grep -Eq '^alertmanager_notification_requests_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}"
+grep -Eq '^alertmanager_notification_requests_failed_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}"
+SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
+
+python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY'
+from pathlib import Path
+import re
+import sys
+
+source = Path(sys.argv[1])
+candidate = Path(sys.argv[2])
+text = source.read_text(encoding="utf-8")
+
+begin = "  # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH\n"
+end = "  # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH\n"
+block = """  # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH
+  - job_name: 'alertmanager-runtime'
+    scrape_interval: 30s
+    metrics_path: /metrics
+    static_configs:
+      - targets: ['alertmanager:9093']
+        labels:
+          host: '110'
+          service: 'alertmanager'
+          env: 'prod'
+  # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH
+
+"""
+
+if text.count(begin) != text.count(end) or text.count(begin) > 1:
+    raise SystemExit("alertmanager runtime managed block is ambiguous")
+if begin in text:
+    pattern = re.escape(begin) + r".*?" + re.escape(end) + r"\n?"
+    text, count = re.subn(pattern, block, text, count=1, flags=re.S)
+    if count != 1:
+        raise SystemExit("managed block replacement failed")
+else:
+    legacy_pattern = (
+        r"(?ms)^  - job_name: ['\"]alertmanager-runtime['\"]\n"
+        r".*?(?=^  - job_name:|\Z)"
+    )
+    legacy_matches = list(re.finditer(legacy_pattern, text))
+    if len(legacy_matches) == 1:
+        text = re.sub(legacy_pattern, block, text, count=1)
+    elif legacy_matches:
+        raise SystemExit("legacy alertmanager runtime job is ambiguous")
+    else:
+        anchors = (
+            "  # === AWOOOI API Metrics",
+            "  - job_name: 'awoooi-api'\n",
+        )
+        anchor = next((value for value in anchors if value in text), None)
+        if anchor is None or text.count(anchor) != 1:
+            raise SystemExit("awoooi-api insertion anchor missing or ambiguous")
+        text = text.replace(anchor, block + anchor, 1)
+
+provider_begin = "      # BEGIN AWOOOI OLLAMA CLOUD EDGE PROBES\n"
+provider_end = "      # END AWOOOI OLLAMA CLOUD EDGE PROBES\n"
+provider_block = """      # BEGIN AWOOOI OLLAMA CLOUD EDGE PROBES
+      - targets:
+          - http://34.143.170.20:11434/api/tags
+        labels:
+          service: ollama-cloud-edge
+          provider: ollama_gcp_a
+          probe_origin: host110
+          boundary: cloud_edge_availability_only
+      - targets:
+          - http://34.21.145.224:11434/api/tags
+        labels:
+          service: ollama-cloud-edge
+          provider: ollama_gcp_b
+          probe_origin: host110
+          boundary: cloud_edge_availability_only
+      # END AWOOOI OLLAMA CLOUD EDGE PROBES
+"""
+if (
+    text.count(provider_begin) != text.count(provider_end)
+    or text.count(provider_begin) > 1
+):
+    raise SystemExit("Ollama cloud edge managed block is ambiguous")
+if provider_begin in text:
+    provider_pattern = (
+        re.escape(provider_begin)
+        + r".*?"
+        + re.escape(provider_end)
+    )
+    text, count = re.subn(
+        provider_pattern,
+        provider_block,
+        text,
+        count=1,
+        flags=re.S,
+    )
+    if count != 1:
+        raise SystemExit("Ollama cloud edge managed block replacement failed")
+else:
+    blackbox_match = re.search(
+        r"(?ms)^  - job_name: ['\"]blackbox-http['\"]\n"
+        r".*?(?=^  - job_name:|\Z)",
+        text,
+    )
+    if blackbox_match is None:
+        raise SystemExit("blackbox-http job missing")
+    blackbox_job = blackbox_match.group(0)
+    if "provider: ollama_gcp_" in blackbox_job:
+        raise SystemExit("unmanaged Ollama cloud edge targets are ambiguous")
+    relabel_anchor = "    relabel_configs:\n"
+    if blackbox_job.count(relabel_anchor) != 1:
+        raise SystemExit("blackbox-http relabel anchor missing or ambiguous")
+    patched_job = blackbox_job.replace(
+        relabel_anchor,
+        provider_block + relabel_anchor,
+        1,
+    )
+    text = text[: blackbox_match.start()] + patched_job + text[blackbox_match.end() :]
+
+if text.count("job_name: 'alertmanager-runtime'") != 1:
+    raise SystemExit("alertmanager runtime job must exist exactly once")
+if "targets: ['alertmanager:9093']" not in text:
+    raise SystemExit("canonical Alertmanager target missing")
+if text.count("provider: ollama_gcp_a") != 1:
+    raise SystemExit("canonical GCP-A cloud edge target missing or ambiguous")
+if text.count("provider: ollama_gcp_b") != 1:
+    raise SystemExit("canonical GCP-B cloud edge target missing or ambiguous")
+if "192.168.0.111:11434" in text:
+    raise SystemExit("host111 must not be probed from retired host110 boundary")
+candidate.write_text(text, encoding="utf-8")
+PY
+
+docker cp "${REMOTE_CANDIDATE}" \
+    "${PROMETHEUS_CONTAINER}:${CONTAINER_CANDIDATE}" >/dev/null
+docker exec "${PROMETHEUS_CONTAINER}" promtool check config "${CONTAINER_CANDIDATE}"
+
+CANDIDATE_SHA="$(sha256sum "${REMOTE_CANDIDATE}" | awk '{print $1}')"
+test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
+printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s source_sha=%s candidate_sha=%s promtool=pass persistent_writes_performed=0 temporary_validation_artifact=true\n' \
+    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
+    "${SOURCE_SHA}" "${CANDIDATE_SHA}"
+
+if [ "${MODE}" = "--dry-run" ]; then
+    printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=check_pass_no_persistent_write\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"
+    exit 0
+fi
+
+cp -p "${REMOTE_CONFIG}" "${BACKUP_CONFIG}"
+test "$(sha256sum "${BACKUP_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
+test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
+APPLY_STARTED=1
+cp "${REMOTE_CANDIDATE}" "${REMOTE_CONFIG}"
+test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${CANDIDATE_SHA}"
+curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
+curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
+printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=alertmanager_runtime_scrape_converged active_sha=%s backup=%s\n' \
+    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${CANDIDATE_SHA}" \
+    "${BACKUP_CONFIG}"
+
+target_readback() {
+    curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
+import json
+import sys
+
+targets = json.load(sys.stdin)["data"]["activeTargets"]
+matches = [
+    target for target in targets
+    if target.get("labels", {}).get("job") == "alertmanager-runtime"
+]
+if len(matches) != 1:
+    print(f"{len(matches)}|missing|never")
+else:
+    target = matches[0]
+    print("{}|{}|{}".format(
+        len(matches),
+        target.get("health", "unknown"),
+        target.get("lastScrape", "never"),
+    ))
+'
+}
+
+provider_target_readback() {
+    curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
+import json
+import sys
+
+targets = json.load(sys.stdin)["data"]["activeTargets"]
+providers = ("ollama_gcp_a", "ollama_gcp_b")
+matches = {
+    provider: [
+        target
+        for target in targets
+        if target.get("labels", {}).get("job") == "blackbox-http"
+        and target.get("labels", {}).get("provider") == provider
+        and target.get("labels", {}).get("probe_origin") == "host110"
+    ]
+    for provider in providers
+}
+ready = all(
+    len(matches[provider]) == 1
+    and matches[provider][0].get("lastScrape", "never") != "never"
+    for provider in providers
+)
+values = []
+for provider in providers:
+    target = matches[provider][0] if len(matches[provider]) == 1 else {}
+    values.extend([
+        target.get("lastScrape", "never"),
+        target.get("health", "missing"),
+    ])
+print("|".join(["1" if ready else "0", *values]))
+'
+}
+
+FIRST_SCRAPE=""
+for attempt in $(seq 1 9); do
+    IFS='|' read -r count health last_scrape <<<"$(target_readback)"
+    printf 'target_probe trace_id=%s run_id=%s attempt=%s count=%s health=%s last_scrape=%s\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${attempt}" "${count}" "${health}" \
+        "${last_scrape}"
+    if [ "${count}" = 1 ] && [ "${health}" = up ] \
+        && [ "${last_scrape}" != never ]; then
+        FIRST_SCRAPE="${last_scrape}"
+        break
+    fi
+    sleep 10
+done
+test -n "${FIRST_SCRAPE}"
+
+FIRST_GCP_A_SCRAPE=""
+FIRST_GCP_B_SCRAPE=""
+for attempt in $(seq 1 9); do
+    IFS='|' read -r providers_ready gcp_a_scrape gcp_a_health \
+        gcp_b_scrape gcp_b_health <<<"$(provider_target_readback)"
+    printf 'provider_target_probe trace_id=%s run_id=%s attempt=%s ready=%s gcp_a_health=%s gcp_b_health=%s\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${attempt}" "${providers_ready}" \
+        "${gcp_a_health}" "${gcp_b_health}"
+    if [ "${providers_ready}" = 1 ]; then
+        FIRST_GCP_A_SCRAPE="${gcp_a_scrape}"
+        FIRST_GCP_B_SCRAPE="${gcp_b_scrape}"
+        break
+    fi
+    sleep 10
+done
+test -n "${FIRST_GCP_A_SCRAPE}"
+test -n "${FIRST_GCP_B_SCRAPE}"
+
+sleep 31
+IFS='|' read -r count health SECOND_SCRAPE <<<"$(target_readback)"
+test "${count}" = 1
+test "${health}" = up
+test "${SECOND_SCRAPE}" != "${FIRST_SCRAPE}"
+IFS='|' read -r providers_ready SECOND_GCP_A_SCRAPE gcp_a_health \
+    SECOND_GCP_B_SCRAPE gcp_b_health <<<"$(provider_target_readback)"
+test "${providers_ready}" = 1
+test "${SECOND_GCP_A_SCRAPE}" != "${FIRST_GCP_A_SCRAPE}"
+test "${SECOND_GCP_B_SCRAPE}" != "${FIRST_GCP_B_SCRAPE}"
+
+SERIES_COUNT="$(curl -fsS --get \
+    --data-urlencode 'query=alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \
+    "${PROMETHEUS_URL}/api/v1/query" | python3 -c '
+import json
+import sys
+print(len(json.load(sys.stdin)["data"]["result"]))
+')"
+test "${SERIES_COUNT}" = 1
+
+FAILED_SERIES_COUNT="$(curl -fsS --get \
+    --data-urlencode 'query=alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \
+    "${PROMETHEUS_URL}/api/v1/query" | python3 -c '
+import json
+import sys
+print(len(json.load(sys.stdin)["data"]["result"]))
+')"
+test "${FAILED_SERIES_COUNT}" = 1
+
+PROVIDER_SERIES_COUNT="$(curl -fsS --get \
+    --data-urlencode 'query=probe_success{job="blackbox-http",provider=~"ollama_gcp_a|ollama_gcp_b",probe_origin="host110"}' \
+    "${PROMETHEUS_URL}/api/v1/query" | python3 -c '
+import json
+import sys
+print(len(json.load(sys.stdin)["data"]["result"]))
+')"
+test "${PROVIDER_SERIES_COUNT}" = 2
+test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${CANDIDATE_SHA}"
+
+DEPLOY_VERIFIED=1
+printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s terminal=two_scrapes_up first=%s second=%s delivery_series_count=%s failed_series_count=%s provider_series_count=%s gcp_a_health=%s gcp_b_health=%s\n' \
+    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${FIRST_SCRAPE}" \
+    "${SECOND_SCRAPE}" "${SERIES_COUNT}" "${FAILED_SERIES_COUNT}" \
+    "${PROVIDER_SERIES_COUNT}" "${gcp_a_health}" "${gcp_b_health}"
+printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=completed backup=%s\n' \
+    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
+REMOTE