fix(alerting): align Alertmanager metric schema

scripts/ops/deploy-alertmanager-runtime-scrape.sh

@@ -119,8 +119,8 @@ test -f "${REMOTE_CONFIG}"
 test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true
 curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
 METRICS_SNAPSHOT="$(curl -fsS "${ALERTMANAGER_METRICS_URL}")"
-grep -Eq '^alertmanager_notification_requests_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}"
-grep -Eq '^alertmanager_notification_requests_failed_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}"
+grep -Eq '^alertmanager_notification_requests_total\{[^}]*integration="webhook"' <<<"${METRICS_SNAPSHOT}"
+grep -Eq '^alertmanager_notification_requests_failed_total\{[^}]*integration="webhook"' <<<"${METRICS_SNAPSHOT}"
 SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
 
 python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY'
@@ -144,6 +144,12 @@ block = """  # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH
           host: '110'
           service: 'alertmanager'
           env: 'prod'
+    metric_relabel_configs:
+      - source_labels: [__name__, integration]
+        separator: ';'
+        regex: 'alertmanager_notification_requests(_failed)?_total;webhook'
+        target_label: receiver_contract
+        replacement: 'awoooi-webhook'
   # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH
 
 """
@@ -376,7 +382,7 @@ test "${SECOND_GCP_A_SCRAPE}" != "${FIRST_GCP_A_SCRAPE}"
 test "${SECOND_GCP_B_SCRAPE}" != "${FIRST_GCP_B_SCRAPE}"
 
 SERIES_COUNT="$(curl -fsS --get \
-    --data-urlencode 'query=alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \
+    --data-urlencode 'query=alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver_contract="awoooi-webhook"}' \
     "${PROMETHEUS_URL}/api/v1/query" | python3 -c '
 import json
 import sys
@@ -385,7 +391,7 @@ print(len(json.load(sys.stdin)["data"]["result"]))
 test "${SERIES_COUNT}" = 1
 
 FAILED_SERIES_COUNT="$(curl -fsS --get \
-    --data-urlencode 'query=alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \
+    --data-urlencode 'query=alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver_contract="awoooi-webhook"}' \
     "${PROMETHEUS_URL}/api/v1/query" | python3 -c '
 import json
 import sys

scripts/ops/deploy-alerts.sh

@@ -188,12 +188,12 @@ log "✅ NoAlertsReceived2Hours query 已限制 alertmanager 主鏈路"
 ALERT_CHAIN_BROKEN_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'AlertChainBroken_Alertmanager'), ''))")
 if [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'alertmanager_notification_requests_failed_total'* ]] \
     || [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'alertmanager_notification_requests_total'* ]] \
-    || [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'receiver="awoooi-webhook"'* ]] \
+    || [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'receiver_contract="awoooi-webhook"'* ]] \
     || [[ "$ALERT_CHAIN_BROKEN_QUERY" == *'awoooi_webhook_requests_total'* ]]; then
     echo "ERROR: AlertChainBroken_Alertmanager 未使用 primary awoooi-webhook monotonic delivery truth: ${ALERT_CHAIN_BROKEN_QUERY}"
     exit 1
 fi
-log "✅ AlertChainBroken_Alertmanager 只使用 primary awoooi-webhook monotonic delivery truth"
+log "✅ AlertChainBroken_Alertmanager 使用唯一 webhook source contract 的 monotonic delivery truth"
 
 SOURCE_PROVIDER_STALE_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'SourceProviderIngestionStale'), ''))")
 if [[ "$SOURCE_PROVIDER_STALE_QUERY" != *'source=~"sentry|signoz"'* ]]; then

scripts/ops/retire-host110-ollama-proxy.sh

@@ -0,0 +1,154 @@
+#!/usr/bin/env bash
+# Retire only the stale host110 Nginx listener that formerly proxied Ollama.
+
+set -euo pipefail
+
+TRACE_ID="${TRACE_ID:?TRACE_ID is required}"
+RUN_ID="${RUN_ID:?RUN_ID is required}"
+WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"
+TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
+MODE="${1:---check}"
+
+case "${MODE}" in
+    apply|--check) ;;
+    *) printf 'usage: %s [--check|apply]\n' "$0" >&2; exit 64 ;;
+esac
+
+for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
+    case "${value}" in
+        *[!A-Za-z0-9._-]*|'')
+            printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2
+            exit 64
+            ;;
+    esac
+done
+
+ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \
+    bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" <<'REMOTE'
+set -euo pipefail
+
+TRACE_ID="$1"
+RUN_ID="$2"
+WORK_ITEM_ID="$3"
+MODE="$4"
+CONFIG_PATH="/etc/nginx/sites-enabled/110-ollama-proxy.conf"
+BACKUP_PATH="/var/backups/awoooi/110-ollama-proxy.conf.${RUN_ID}"
+LOCK_PATH="/tmp/awoooi-host110-ollama-proxy-retirement.lock"
+APPLY_STARTED=0
+VERIFIED=0
+SOURCE_SHA=""
+
+exec 9>"${LOCK_PATH}"
+if ! flock -n 9; then
+    printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=blocked_with_safe_next_action reason=retirement_apply_in_progress\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" >&2
+    exit 75
+fi
+
+rollback() {
+    local rollback_rc=0
+    if [ "${APPLY_STARTED}" -eq 1 ] && [ "${VERIFIED}" -eq 0 ]; then
+        sudo -n test -f "${BACKUP_PATH}" || rollback_rc=1
+        if [ "${rollback_rc}" -eq 0 ]; then
+            sudo -n install -o root -g root -m 0644 \
+                "${BACKUP_PATH}" "${CONFIG_PATH}" || rollback_rc=1
+        fi
+        if [ "${rollback_rc}" -eq 0 ]; then
+            sudo -n nginx -t >/dev/null 2>&1 || rollback_rc=1
+            sudo -n systemctl reload nginx || rollback_rc=1
+        fi
+        if [ "${rollback_rc}" -eq 0 ]; then
+            printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=restored source_sha=%s\n' \
+                "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${SOURCE_SHA}"
+            return 0
+        fi
+        printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rollback_failed backup=%s\n' \
+            "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_PATH}" >&2
+        return 1
+    fi
+}
+
+on_exit() {
+    local rc=$?
+    local rollback_rc=0
+    trap - EXIT
+    set +e
+    if [ "${rc}" -ne 0 ]; then
+        rollback || rollback_rc=$?
+    fi
+    if [ "${rollback_rc}" -ne 0 ]; then
+        exit 70
+    fi
+    exit "${rc}"
+}
+trap on_exit EXIT
+
+sudo -n true
+if ! sudo -n test -e "${CONFIG_PATH}"; then
+    test "$(sudo -n grep -RIl --include='*.conf' --include='nginx.conf' '11435' /etc/nginx 2>/dev/null | wc -l | tr -d ' ')" = "0"
+    test "$(sudo -n ss -H -ltn 'sport = :11435' | wc -l | tr -d ' ')" = "0"
+    test "$(sudo -n ss -H -ltn 'sport = :11434' | wc -l | tr -d ' ')" = "0"
+    test "$(docker ps --format '{{.Names}}' | grep -Eic 'ollama' || true)" = "0"
+    sudo -n systemctl is-active --quiet nginx
+    sudo -n nginx -t >/dev/null 2>&1
+    printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s state=already_retired listener_11435=0 listener_11434=0 ollama_containers=0 stale_config_paths=0 nginx_active=true nginx_test=pass persistent_writes_performed=0\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}"
+    printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=completed_already_converged_no_write\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"
+    exit 0
+fi
+sudo -n test -f "${CONFIG_PATH}"
+test "$(sudo -n stat -c '%F' "${CONFIG_PATH}")" = "regular file"
+test "$(sudo -n stat -c '%U:%G:%a' "${CONFIG_PATH}")" = "root:root:644"
+test "$(sudo -n grep -Ec '^[[:space:]]*listen[[:space:]]+.*11435' "${CONFIG_PATH}")" = "2"
+PROXY_PASS_COUNT="$(sudo -n grep -Ec '^[[:space:]]*proxy_pass[[:space:]]+http://[^;]+:11434' "${CONFIG_PATH}" || true)"
+ALL_PROXY_PASS_COUNT="$(sudo -n grep -Ec '^[[:space:]]*proxy_pass[[:space:]]+' "${CONFIG_PATH}" || true)"
+test "${PROXY_PASS_COUNT}" -ge 1
+test "${PROXY_PASS_COUNT}" = "${ALL_PROXY_PASS_COUNT}"
+test "$(sudo -n ss -H -ltn 'sport = :11435' | wc -l | tr -d ' ')" = "2"
+test "$(sudo -n ss -H -ltn 'sport = :11434' | wc -l | tr -d ' ')" = "0"
+test "$(docker ps --format '{{.Names}}' | grep -Eic 'ollama' || true)" = "0"
+SOURCE_SHA="$(sudo -n sha256sum "${CONFIG_PATH}" | awk '{print $1}')"
+sudo -n nginx -t >/dev/null 2>&1
+
+printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s exact_path=%s source_sha=%s listener_11435=2 listener_11434=0 ollama_containers=0 nginx_test=pass persistent_writes_performed=0\n' \
+    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
+    "${CONFIG_PATH}" "${SOURCE_SHA}"
+
+if [ "${MODE}" = "--check" ]; then
+    printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=check_pass_no_write\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"
+    exit 0
+fi
+
+sudo -n install -d -o root -g root -m 0700 /var/backups/awoooi
+sudo -n install -o root -g root -m 0600 "${CONFIG_PATH}" "${BACKUP_PATH}"
+test "$(sudo -n sha256sum "${BACKUP_PATH}" | awk '{print $1}')" = "${SOURCE_SHA}"
+APPLY_STARTED=1
+sudo -n rm -- "${CONFIG_PATH}"
+sudo -n nginx -t >/dev/null 2>&1
+sudo -n systemctl reload nginx
+
+for _ in $(seq 1 10); do
+    if [ "$(sudo -n ss -H -ltn 'sport = :11435' | wc -l | tr -d ' ')" = "0" ]; then
+        break
+    fi
+    sleep 1
+done
+
+sudo -n test ! -e "${CONFIG_PATH}"
+test "$(sudo -n grep -RIl --include='*.conf' --include='nginx.conf' '11435' /etc/nginx 2>/dev/null | wc -l | tr -d ' ')" = "0"
+test "$(sudo -n ss -H -ltn 'sport = :11435' | wc -l | tr -d ' ')" = "0"
+test "$(sudo -n ss -H -ltn 'sport = :11434' | wc -l | tr -d ' ')" = "0"
+test "$(docker ps --format '{{.Names}}' | grep -Eic 'ollama' || true)" = "0"
+sudo -n systemctl is-active --quiet nginx
+sudo -n nginx -t >/dev/null 2>&1
+VERIFIED=1
+
+printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=retire_host110_ollama_proxy source_sha=%s backup=%s\n' \
+    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${SOURCE_SHA}" "${BACKUP_PATH}"
+printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s listener_11435=0 listener_11434=0 ollama_containers=0 stale_config_paths=0 nginx_active=true nginx_test=pass\n' \
+    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"
+printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=completed\n' \
+    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"
+REMOTE

scripts/ops/tests/test_alertmanager_runtime_scrape_contract.py

@@ -22,8 +22,10 @@ def test_controlled_scrape_apply_has_check_rollback_and_independent_verifier() -
     assert '!= "${SOURCE_SHA}"' in text
     assert "alertmanager_notification_requests_total" in text
     assert "alertmanager_notification_requests_failed_total" in text
-    assert 'receiver="awoooi-webhook"' in text
-    assert text.count('receiver="awoooi-webhook"') >= 4
+    assert 'integration="webhook"' in text
+    assert "target_label: receiver_contract" in text
+    assert "replacement: 'awoooi-webhook'" in text
+    assert text.count('receiver_contract="awoooi-webhook"') >= 2
     assert "two_scrapes_up" in text
     assert "delivery_series_count" in text
 

scripts/ops/tests/test_host110_ollama_proxy_retirement_contract.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "ops" / "retire-host110-ollama-proxy.sh"
+
+
+def test_retirement_is_exact_bounded_and_rollback_capable() -> None:
+    text = SCRIPT.read_text(encoding="utf-8")
+
+    assert 'TRACE_ID="${TRACE_ID:?TRACE_ID is required}"' in text
+    assert 'RUN_ID="${RUN_ID:?RUN_ID is required}"' in text
+    assert 'WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"' in text
+    assert 'CONFIG_PATH="/etc/nginx/sites-enabled/110-ollama-proxy.conf"' in text
+    assert "flock -n 9" in text
+    assert "source_sha=" in text
+    assert "persistent_writes_performed=0" in text
+    assert "rollback_receipt" in text
+    assert "terminal=rollback_failed" in text
+    assert "sudo -n nginx -t" in text
+    assert "sudo -n systemctl reload nginx" in text
+    assert "completed_already_converged_no_write" in text
+    assert "state=already_retired" in text
+
+
+def test_retirement_proves_the_retired_identity_cannot_reappear() -> None:
+    text = SCRIPT.read_text(encoding="utf-8")
+
+    assert "listener_11435=0" in text
+    assert "listener_11434=0" in text
+    assert "ollama_containers=0" in text
+    assert "stale_config_paths=0" in text
+    assert "grep -RIl" in text
+    assert "rm -rf" not in text
+    assert "pkill" not in text
+    assert "killall" not in text