feat(sre): enforce typed controlled automation

2026-07-16 17:32:01 +08:00
parent dfec0c9f7f
commit 541a32a9cb
130 changed files with 14849 additions and 1738 deletions
--- a/scripts/ops/deploy-alertmanager-config.sh
+++ b/scripts/ops/deploy-alertmanager-config.sh
@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 # Render and deploy ops/alertmanager/alertmanager.yml to the 110 Docker Alertmanager.
 #
-# This script keeps the live direct-Telegram emergency route aligned with Git:
+# This script keeps the direct-Telegram route aligned:
 #   - inject Telegram bot token and SRE group chat id from K8s secret or env
 #   - validate with amtool before touching the live config
-#   - back up the live file
-#   - keep the bind-mounted live file inode and readable permissions intact
+#   - create a 0600 rollback copy without a world-readable intermediate
+#   - keep the bind-mounted live file inode and protect it as runtime-UID 0400
+#   - verify container readability before reload; fail closed with secure rollback
 #   - reload Alertmanager with SIGHUP
 #
 # Usage:
@@ -105,14 +106,20 @@ target = Path(sys.argv[2])
 text = template.read_text()
 text = text.replace("TELEGRAM_BOT_TOKEN_PLACEHOLDER", os.environ["TELEGRAM_BOT_TOKEN"])
 text = text.replace("SRE_GROUP_CHAT_ID_PLACEHOLDER", os.environ["SRE_GROUP_CHAT_ID"])
-if "TELEGRAM_BOT_TOKEN_PLACEHOLDER" in text or "SRE_GROUP_CHAT_ID_PLACEHOLDER" in text:
+if any(
+    placeholder in text
+    for placeholder in (
+        "TELEGRAM_BOT_TOKEN_PLACEHOLDER",
+        "SRE_GROUP_CHAT_ID_PLACEHOLDER",
+    )
+):
    raise SystemExit("unreplaced secret placeholder remains in rendered config")
 target.write_text(text)
 PY

 log "Validating rendered config with live Alertmanager amtool on ${TARGET_HOST}"
 ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" \
-  "docker exec -i alertmanager sh -c 'cat >/tmp/alertmanager-rendered.yml && amtool check-config /tmp/alertmanager-rendered.yml'" \
+  "docker exec -i alertmanager amtool check-config /dev/stdin" \
  < "$tmp_rendered"

 if [[ "$DRY_RUN" == "--dry-run" ]]; then
@@ -127,18 +134,115 @@ ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" \
 ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" "bash -s" <<REMOTE
 set -euo pipefail
 target='${TARGET_PATH}'
+staged='/tmp/alertmanager.yml.new'
+container='alertmanager'
+trap 'rm -f "\$staged"' EXIT
+
+[[ -f "\$target" && ! -L "\$target" ]] || {
+  echo 'ERROR: live Alertmanager config must be a regular non-symlink file' >&2
+  exit 1
+}
+[[ -f "\$staged" && ! -L "\$staged" ]] || {
+  echo 'ERROR: staged Alertmanager config must be a regular non-symlink file' >&2
+  exit 1
+}
+sudo -n true
+
+# Read the effective host UID/GID from the container process itself. This also
+# works when Docker user namespaces remap container identities.
+container_pid="\$(docker inspect --format '{{.State.Pid}}' "\$container")"
+[[ "\$container_pid" =~ ^[1-9][0-9]*$ ]] || {
+  echo 'ERROR: Alertmanager container host PID is unavailable' >&2
+  exit 1
+}
+status_path="/proc/\${container_pid}/status"
+[[ -r "\$status_path" ]] || {
+  echo 'ERROR: Alertmanager runtime identity cannot be verified' >&2
+  exit 1
+}
+runtime_uid="\$(awk '/^Uid:/{print \$2; exit}' "\$status_path")"
+runtime_gid="\$(awk '/^Gid:/{print \$2; exit}' "\$status_path")"
+[[ "\$runtime_uid" =~ ^[0-9]+$ && "\$runtime_gid" =~ ^[0-9]+$ ]] || {
+  echo 'ERROR: Alertmanager runtime UID/GID is invalid' >&2
+  exit 1
+}
+
 backup="\${target}.bak.\$(date +%Y%m%d%H%M%S)"
-cp "\$target" "\$backup"
-# Alertmanager bind-mounts a single file. Keep the existing inode instead of mv'ing
-# a replacement over it, then restore readable permissions for the container user.
-cat /tmp/alertmanager.yml.new > "\$target"
-chmod 0644 "\$target"
-rm -f /tmp/alertmanager.yml.new
-docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml
-docker kill -s HUP alertmanager >/dev/null
+operator_uid="\$(id -u)"
+operator_gid="\$(id -g)"
+umask 077
+# install applies 0600 while creating the copy; cp followed by chmod would leave
+# a secret-bearing backup readable during the gap.
+sudo -n install -m 0600 -o "\$operator_uid" -g "\$operator_gid" "\$target" "\$backup"
+[[ "\$(stat -c '%a' "\$backup")" == '600' ]] || {
+  echo 'ERROR: secure Alertmanager rollback copy was not created' >&2
+  exit 1
+}
+
+# Harden legacy copies produced by older versions of this script without
+# reading or printing their contents.
+target_dir="\$(dirname "\$target")"
+target_base="\$(basename "\$target")"
+while IFS= read -r -d '' legacy_backup; do
+  sudo -n chmod 0600 "\$legacy_backup"
+done < <(find "\$target_dir" -maxdepth 1 -type f -name "\${target_base}.bak.*" -print0)
+
+reload_attempted=false
+rollback_secure() {
+  local rc="\${1:-1}"
+  trap - ERR
+  set +e
+  if [[ -f "\$backup" ]]; then
+    # Remove access before restoring bytes so rollback cannot briefly recreate
+    # the legacy world-readable state.
+    sudo -n chmod 0000 "\$target"
+    sudo -n chown "\${runtime_uid}:\${runtime_gid}" "\$target"
+    sudo -n chmod 0400 "\$target"
+    sudo -n sh -c 'cat "\$1" > "\$2"' _ "\$backup" "\$target"
+    rollback_verified=false
+    if docker exec "\$container" sh -ec \
+      'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml' \
+      >/dev/null 2>&1; then
+      rollback_verified=true
+    fi
+    if [[ "\$reload_attempted" == 'true' && "\$rollback_verified" == 'true' ]]; then
+      docker kill -s HUP "\$container" >/dev/null 2>&1
+    fi
+  fi
+  rm -f "\$staged"
+  echo 'ERROR: Alertmanager deploy failed; secure rollback attempted' >&2
+  exit "\$rc"
+}
+trap 'rollback_secure \$?' ERR
+
+# Alertmanager bind-mounts this single file. Preserve its inode and remove all
+# access before changing ownership, so no secret bytes are ever written while
+# legacy group/other read bits remain. Validate the permission model against the
+# running container before replacing the old config bytes.
+sudo -n chmod 0000 "\$target"
+sudo -n chown "\${runtime_uid}:\${runtime_gid}" "\$target"
+sudo -n chmod 0400 "\$target"
+[[ "\$(stat -c '%u' "\$target")" == "\$runtime_uid" ]]
+[[ "\$(stat -c '%g' "\$target")" == "\$runtime_gid" ]]
+[[ "\$(stat -c '%a' "\$target")" == '400' ]]
+docker exec "\$container" sh -ec \
+  'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml'
+
+sudo -n sh -c 'cat "\$1" > "\$2"' _ "\$staged" "\$target"
+rm -f "\$staged"
+[[ "\$(stat -c '%u' "\$target")" == "\$runtime_uid" ]]
+[[ "\$(stat -c '%g' "\$target")" == "\$runtime_gid" ]]
+[[ "\$(stat -c '%a' "\$target")" == '400' ]]
+docker exec "\$container" sh -ec \
+  'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml'
+
+reload_attempted=true
+docker kill -s HUP "\$container" >/dev/null
 sleep 2
-docker inspect alertmanager --format 'status={{.State.Status}} started={{.State.StartedAt}}'
-echo "backup=\$backup"
+docker inspect "\$container" --format 'status={{.State.Status}} started={{.State.StartedAt}}'
+[[ "\$(docker inspect "\$container" --format '{{.State.Status}}')" == 'running' ]]
+trap - ERR
+echo "backup=\$backup config_mode=0400 runtime_owner=\${runtime_uid}:\${runtime_gid}"
 REMOTE

 log "Alertmanager config deployed and reloaded"
--- a/scripts/ops/deploy-alertmanager-runtime-scrape.sh
+++ b/scripts/ops/deploy-alertmanager-runtime-scrape.sh
@@ -0,0 +1,413 @@
+#!/usr/bin/env bash
+# Bounded Prometheus scrape-source convergence for Alertmanager delivery truth.
+
+set -euo pipefail
+
+TRACE_ID="${TRACE_ID:?TRACE_ID is required}"
+RUN_ID="${RUN_ID:?RUN_ID is required}"
+WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"
+TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
+PROMETHEUS_URL="${PROMETHEUS_URL:-http://192.168.0.110:9090}"
+ALERTMANAGER_METRICS_URL="${ALERTMANAGER_METRICS_URL:-http://127.0.0.1:9093/metrics}"
+PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
+REMOTE_CONFIG="${REMOTE_CONFIG:-/home/wooo/monitoring/prometheus.yml}"
+MODE="${1:---dry-run}"
+
+case "${MODE}" in
+    apply|--dry-run) ;;
+    *) printf 'usage: %s [--dry-run|apply]\n' "$0" >&2; exit 64 ;;
+esac
+
+for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
+    case "${value}" in
+        *[!A-Za-z0-9._-]*|'')
+            printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2
+            exit 64
+            ;;
+    esac
+done
+
+ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \
+    bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
+    "${PROMETHEUS_URL}" "${ALERTMANAGER_METRICS_URL}" \
+    "${PROMETHEUS_CONTAINER}" "${REMOTE_CONFIG}" <<'REMOTE'
+set -euo pipefail
+
+TRACE_ID="$1"
+RUN_ID="$2"
+WORK_ITEM_ID="$3"
+MODE="$4"
+PROMETHEUS_URL="$5"
+ALERTMANAGER_METRICS_URL="$6"
+PROMETHEUS_CONTAINER="$7"
+REMOTE_CONFIG="$8"
+REMOTE_CANDIDATE="/tmp/awoooi-prometheus-alertmanager-runtime.${RUN_ID}.yml"
+CONTAINER_CANDIDATE="/etc/prometheus/.awoooi-alertmanager-runtime.${RUN_ID}.yml"
+BACKUP_CONFIG="${REMOTE_CONFIG}.bak.${RUN_ID}"
+APPLY_STARTED=0
+DEPLOY_VERIFIED=0
+SOURCE_SHA=""
+CANDIDATE_SHA=""
+
+exec 9>/tmp/awoooi-prometheus-config.lock
+if ! flock -n 9; then
+    printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=blocked_with_safe_next_action reason=prometheus_config_apply_in_progress\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" >&2
+    exit 75
+fi
+
+cleanup() {
+    rm -f "${REMOTE_CANDIDATE}"
+    docker exec -u 0 "${PROMETHEUS_CONTAINER}" \
+        rm -f "${CONTAINER_CANDIDATE}" >/dev/null 2>&1 || true
+}
+
+rollback() {
+    local rollback_rc=0
+    local current_sha backup_sha restored_sha
+    current_sha="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" || rollback_rc=1
+    backup_sha="$(sha256sum "${BACKUP_CONFIG}" | awk '{print $1}')" || rollback_rc=1
+    if [ "${rollback_rc}" -eq 0 ] && [ "${current_sha}" = "${SOURCE_SHA}" ]; then
+        printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=no_write_source_intact source_sha=%s\n' \
+            "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${SOURCE_SHA}"
+        return 0
+    fi
+    if [ "${rollback_rc}" -ne 0 ] \
+        || [ "${current_sha}" != "${CANDIDATE_SHA}" ] \
+        || [ "${backup_sha}" != "${SOURCE_SHA}" ]; then
+        rollback_rc=1
+    else
+        cp "${BACKUP_CONFIG}" "${REMOTE_CONFIG}" || rollback_rc=1
+        restored_sha="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" || rollback_rc=1
+        [ "${restored_sha}" = "${SOURCE_SHA}" ] || rollback_rc=1
+    fi
+    if [ "${rollback_rc}" -eq 0 ]; then
+        curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null \
+            || rollback_rc=1
+    fi
+    if [ "${rollback_rc}" -eq 0 ]; then
+        curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null || rollback_rc=1
+    fi
+    if [ "${rollback_rc}" -eq 0 ]; then
+        printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rolled_back ready=true backup=%s\n' \
+            "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
+        return 0
+    fi
+    printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rollback_failed ready=false backup=%s\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}" >&2
+    return 1
+}
+
+on_exit() {
+    rc=$?
+    trap - EXIT
+    set +e
+    cleanup
+    rollback_rc=0
+    if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] \
+        && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
+        rollback || rollback_rc=$?
+    fi
+    if [ "${rollback_rc}" -ne 0 ]; then
+        exit 70
+    fi
+    exit "${rc}"
+}
+trap on_exit EXIT
+
+test -f "${REMOTE_CONFIG}"
+test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true
+curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
+METRICS_SNAPSHOT="$(curl -fsS "${ALERTMANAGER_METRICS_URL}")"
+grep -Eq '^alertmanager_notification_requests_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}"
+grep -Eq '^alertmanager_notification_requests_failed_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}"
+SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
+
+python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY'
+from pathlib import Path
+import re
+import sys
+
+source = Path(sys.argv[1])
+candidate = Path(sys.argv[2])
+text = source.read_text(encoding="utf-8")
+
+begin = "  # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH\n"
+end = "  # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH\n"
+block = """  # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH
+  - job_name: 'alertmanager-runtime'
+    scrape_interval: 30s
+    metrics_path: /metrics
+    static_configs:
+      - targets: ['alertmanager:9093']
+        labels:
+          host: '110'
+          service: 'alertmanager'
+          env: 'prod'
+  # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH
+
+"""
+
+if text.count(begin) != text.count(end) or text.count(begin) > 1:
+    raise SystemExit("alertmanager runtime managed block is ambiguous")
+if begin in text:
+    pattern = re.escape(begin) + r".*?" + re.escape(end) + r"\n?"
+    text, count = re.subn(pattern, block, text, count=1, flags=re.S)
+    if count != 1:
+        raise SystemExit("managed block replacement failed")
+else:
+    legacy_pattern = (
+        r"(?ms)^  - job_name: ['\"]alertmanager-runtime['\"]\n"
+        r".*?(?=^  - job_name:|\Z)"
+    )
+    legacy_matches = list(re.finditer(legacy_pattern, text))
+    if len(legacy_matches) == 1:
+        text = re.sub(legacy_pattern, block, text, count=1)
+    elif legacy_matches:
+        raise SystemExit("legacy alertmanager runtime job is ambiguous")
+    else:
+        anchors = (
+            "  # === AWOOOI API Metrics",
+            "  - job_name: 'awoooi-api'\n",
+        )
+        anchor = next((value for value in anchors if value in text), None)
+        if anchor is None or text.count(anchor) != 1:
+            raise SystemExit("awoooi-api insertion anchor missing or ambiguous")
+        text = text.replace(anchor, block + anchor, 1)
+
+provider_begin = "      # BEGIN AWOOOI OLLAMA CLOUD EDGE PROBES\n"
+provider_end = "      # END AWOOOI OLLAMA CLOUD EDGE PROBES\n"
+provider_block = """      # BEGIN AWOOOI OLLAMA CLOUD EDGE PROBES
+      - targets:
+          - http://34.143.170.20:11434/api/tags
+        labels:
+          service: ollama-cloud-edge
+          provider: ollama_gcp_a
+          probe_origin: host110
+          boundary: cloud_edge_availability_only
+      - targets:
+          - http://34.21.145.224:11434/api/tags
+        labels:
+          service: ollama-cloud-edge
+          provider: ollama_gcp_b
+          probe_origin: host110
+          boundary: cloud_edge_availability_only
+      # END AWOOOI OLLAMA CLOUD EDGE PROBES
+"""
+if (
+    text.count(provider_begin) != text.count(provider_end)
+    or text.count(provider_begin) > 1
+):
+    raise SystemExit("Ollama cloud edge managed block is ambiguous")
+if provider_begin in text:
+    provider_pattern = (
+        re.escape(provider_begin)
+        + r".*?"
+        + re.escape(provider_end)
+    )
+    text, count = re.subn(
+        provider_pattern,
+        provider_block,
+        text,
+        count=1,
+        flags=re.S,
+    )
+    if count != 1:
+        raise SystemExit("Ollama cloud edge managed block replacement failed")
+else:
+    blackbox_match = re.search(
+        r"(?ms)^  - job_name: ['\"]blackbox-http['\"]\n"
+        r".*?(?=^  - job_name:|\Z)",
+        text,
+    )
+    if blackbox_match is None:
+        raise SystemExit("blackbox-http job missing")
+    blackbox_job = blackbox_match.group(0)
+    if "provider: ollama_gcp_" in blackbox_job:
+        raise SystemExit("unmanaged Ollama cloud edge targets are ambiguous")
+    relabel_anchor = "    relabel_configs:\n"
+    if blackbox_job.count(relabel_anchor) != 1:
+        raise SystemExit("blackbox-http relabel anchor missing or ambiguous")
+    patched_job = blackbox_job.replace(
+        relabel_anchor,
+        provider_block + relabel_anchor,
+        1,
+    )
+    text = text[: blackbox_match.start()] + patched_job + text[blackbox_match.end() :]
+
+if text.count("job_name: 'alertmanager-runtime'") != 1:
+    raise SystemExit("alertmanager runtime job must exist exactly once")
+if "targets: ['alertmanager:9093']" not in text:
+    raise SystemExit("canonical Alertmanager target missing")
+if text.count("provider: ollama_gcp_a") != 1:
+    raise SystemExit("canonical GCP-A cloud edge target missing or ambiguous")
+if text.count("provider: ollama_gcp_b") != 1:
+    raise SystemExit("canonical GCP-B cloud edge target missing or ambiguous")
+if "192.168.0.111:11434" in text:
+    raise SystemExit("host111 must not be probed from retired host110 boundary")
+candidate.write_text(text, encoding="utf-8")
+PY
+
+docker cp "${REMOTE_CANDIDATE}" \
+    "${PROMETHEUS_CONTAINER}:${CONTAINER_CANDIDATE}" >/dev/null
+docker exec "${PROMETHEUS_CONTAINER}" promtool check config "${CONTAINER_CANDIDATE}"
+
+CANDIDATE_SHA="$(sha256sum "${REMOTE_CANDIDATE}" | awk '{print $1}')"
+test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
+printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s source_sha=%s candidate_sha=%s promtool=pass persistent_writes_performed=0 temporary_validation_artifact=true\n' \
+    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
+    "${SOURCE_SHA}" "${CANDIDATE_SHA}"
+
+if [ "${MODE}" = "--dry-run" ]; then
+    printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=check_pass_no_persistent_write\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"
+    exit 0
+fi
+
+cp -p "${REMOTE_CONFIG}" "${BACKUP_CONFIG}"
+test "$(sha256sum "${BACKUP_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
+test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
+APPLY_STARTED=1
+cp "${REMOTE_CANDIDATE}" "${REMOTE_CONFIG}"
+test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${CANDIDATE_SHA}"
+curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
+curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
+printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=alertmanager_runtime_scrape_converged active_sha=%s backup=%s\n' \
+    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${CANDIDATE_SHA}" \
+    "${BACKUP_CONFIG}"
+
+target_readback() {
+    curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
+import json
+import sys
+
+targets = json.load(sys.stdin)["data"]["activeTargets"]
+matches = [
+    target for target in targets
+    if target.get("labels", {}).get("job") == "alertmanager-runtime"
+]
+if len(matches) != 1:
+    print(f"{len(matches)}|missing|never")
+else:
+    target = matches[0]
+    print("{}|{}|{}".format(
+        len(matches),
+        target.get("health", "unknown"),
+        target.get("lastScrape", "never"),
+    ))
+'
+}
+
+provider_target_readback() {
+    curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
+import json
+import sys
+
+targets = json.load(sys.stdin)["data"]["activeTargets"]
+providers = ("ollama_gcp_a", "ollama_gcp_b")
+matches = {
+    provider: [
+        target
+        for target in targets
+        if target.get("labels", {}).get("job") == "blackbox-http"
+        and target.get("labels", {}).get("provider") == provider
+        and target.get("labels", {}).get("probe_origin") == "host110"
+    ]
+    for provider in providers
+}
+ready = all(
+    len(matches[provider]) == 1
+    and matches[provider][0].get("lastScrape", "never") != "never"
+    for provider in providers
+)
+values = []
+for provider in providers:
+    target = matches[provider][0] if len(matches[provider]) == 1 else {}
+    values.extend([
+        target.get("lastScrape", "never"),
+        target.get("health", "missing"),
+    ])
+print("|".join(["1" if ready else "0", *values]))
+'
+}
+
+FIRST_SCRAPE=""
+for attempt in $(seq 1 9); do
+    IFS='|' read -r count health last_scrape <<<"$(target_readback)"
+    printf 'target_probe trace_id=%s run_id=%s attempt=%s count=%s health=%s last_scrape=%s\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${attempt}" "${count}" "${health}" \
+        "${last_scrape}"
+    if [ "${count}" = 1 ] && [ "${health}" = up ] \
+        && [ "${last_scrape}" != never ]; then
+        FIRST_SCRAPE="${last_scrape}"
+        break
+    fi
+    sleep 10
+done
+test -n "${FIRST_SCRAPE}"
+
+FIRST_GCP_A_SCRAPE=""
+FIRST_GCP_B_SCRAPE=""
+for attempt in $(seq 1 9); do
+    IFS='|' read -r providers_ready gcp_a_scrape gcp_a_health \
+        gcp_b_scrape gcp_b_health <<<"$(provider_target_readback)"
+    printf 'provider_target_probe trace_id=%s run_id=%s attempt=%s ready=%s gcp_a_health=%s gcp_b_health=%s\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${attempt}" "${providers_ready}" \
+        "${gcp_a_health}" "${gcp_b_health}"
+    if [ "${providers_ready}" = 1 ]; then
+        FIRST_GCP_A_SCRAPE="${gcp_a_scrape}"
+        FIRST_GCP_B_SCRAPE="${gcp_b_scrape}"
+        break
+    fi
+    sleep 10
+done
+test -n "${FIRST_GCP_A_SCRAPE}"
+test -n "${FIRST_GCP_B_SCRAPE}"
+
+sleep 31
+IFS='|' read -r count health SECOND_SCRAPE <<<"$(target_readback)"
+test "${count}" = 1
+test "${health}" = up
+test "${SECOND_SCRAPE}" != "${FIRST_SCRAPE}"
+IFS='|' read -r providers_ready SECOND_GCP_A_SCRAPE gcp_a_health \
+    SECOND_GCP_B_SCRAPE gcp_b_health <<<"$(provider_target_readback)"
+test "${providers_ready}" = 1
+test "${SECOND_GCP_A_SCRAPE}" != "${FIRST_GCP_A_SCRAPE}"
+test "${SECOND_GCP_B_SCRAPE}" != "${FIRST_GCP_B_SCRAPE}"
+
+SERIES_COUNT="$(curl -fsS --get \
+    --data-urlencode 'query=alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \
+    "${PROMETHEUS_URL}/api/v1/query" | python3 -c '
+import json
+import sys
+print(len(json.load(sys.stdin)["data"]["result"]))
+')"
+test "${SERIES_COUNT}" = 1
+
+FAILED_SERIES_COUNT="$(curl -fsS --get \
+    --data-urlencode 'query=alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \
+    "${PROMETHEUS_URL}/api/v1/query" | python3 -c '
+import json
+import sys
+print(len(json.load(sys.stdin)["data"]["result"]))
+')"
+test "${FAILED_SERIES_COUNT}" = 1
+
+PROVIDER_SERIES_COUNT="$(curl -fsS --get \
+    --data-urlencode 'query=probe_success{job="blackbox-http",provider=~"ollama_gcp_a|ollama_gcp_b",probe_origin="host110"}' \
+    "${PROMETHEUS_URL}/api/v1/query" | python3 -c '
+import json
+import sys
+print(len(json.load(sys.stdin)["data"]["result"]))
+')"
+test "${PROVIDER_SERIES_COUNT}" = 2
+test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${CANDIDATE_SHA}"
+
+DEPLOY_VERIFIED=1
+printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s terminal=two_scrapes_up first=%s second=%s delivery_series_count=%s failed_series_count=%s provider_series_count=%s gcp_a_health=%s gcp_b_health=%s\n' \
+    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${FIRST_SCRAPE}" \
+    "${SECOND_SCRAPE}" "${SERIES_COUNT}" "${FAILED_SERIES_COUNT}" \
+    "${PROVIDER_SERIES_COUNT}" "${gcp_a_health}" "${gcp_b_health}"
+printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=completed backup=%s\n' \
+    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
+REMOTE
--- a/scripts/ops/deploy-alerts.sh
+++ b/scripts/ops/deploy-alerts.sh
@@ -185,6 +185,16 @@ if [[ "$NO_ALERTS_QUERY" != *'source="alertmanager"'* ]]; then
 fi
 log "✅ NoAlertsReceived2Hours query 已限制 alertmanager 主鏈路"

+ALERT_CHAIN_BROKEN_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'AlertChainBroken_Alertmanager'), ''))")
+if [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'alertmanager_notification_requests_failed_total'* ]] \
+    || [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'alertmanager_notification_requests_total'* ]] \
+    || [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'receiver="awoooi-webhook"'* ]] \
+    || [[ "$ALERT_CHAIN_BROKEN_QUERY" == *'awoooi_webhook_requests_total'* ]]; then
+    echo "ERROR: AlertChainBroken_Alertmanager 未使用 primary awoooi-webhook monotonic delivery truth: ${ALERT_CHAIN_BROKEN_QUERY}"
+    exit 1
+fi
+log "✅ AlertChainBroken_Alertmanager 只使用 primary awoooi-webhook monotonic delivery truth"
+
 SOURCE_PROVIDER_STALE_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'SourceProviderIngestionStale'), ''))")
 if [[ "$SOURCE_PROVIDER_STALE_QUERY" != *'source=~"sentry|signoz"'* ]]; then
    echo "ERROR: SourceProviderIngestionStale query 未限制 Sentry/SignOz provider freshness: ${SOURCE_PROVIDER_STALE_QUERY}"
--- a/scripts/ops/ollama111-fallback-proxy-diagnose.sh
+++ b/scripts/ops/ollama111-fallback-proxy-diagnose.sh
@@ -1,78 +1,35 @@
 #!/usr/bin/env bash
+# Compatibility entrypoint retained for operators; the host110 proxy lane is
+# retired. All probes use the canonical direct provider identities.
 set -euo pipefail

-# Read-only diagnosis for ADR-110 local fallback:
-# API Pod -> 110:11437 Nginx proxy -> 111:11434 Ollama allowlist proxy.
+GCP_A_URL="${GCP_A_URL:-http://34.143.170.20:11434}"
+GCP_B_URL="${GCP_B_URL:-http://34.21.145.224:11434}"
+HOST111_URL="${HOST111_URL:-http://192.168.0.111:11434}"
+PUBLIC_STATUS_URL="${PUBLIC_STATUS_URL:-https://awoooi.wooo.work/api/v1/ai/status}"
+CURL_TIMEOUT="${CURL_TIMEOUT:-8}"

-NAMESPACE="${NAMESPACE:-awoooi-prod}"
-DEPLOYMENT="${DEPLOYMENT:-awoooi-api}"
-PROXY_HOST="${PROXY_HOST:-wooo@192.168.0.110}"
-LOCAL_HOST="${LOCAL_HOST:-ollama-111-gpu}"
-LOCAL_IP="${LOCAL_IP:-192.168.0.111}"
-PROXY_PORT="${PROXY_PORT:-11437}"
-LOCAL_PORT="${LOCAL_PORT:-11434}"
-SSH_TIMEOUT="${SSH_TIMEOUT:-8}"
-CURL_TIMEOUT="${CURL_TIMEOUT:-5}"
-
-section() {
-  printf '\n== %s ==\n' "$1"
+probe() {
+  local provider="$1"
+  local url="$2"
+  local code
+  code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "${CURL_TIMEOUT}" \
+    "${url}/api/tags" 2>/dev/null || true)"
+  case "${code}" in
+    200) printf 'provider=%s boundary=%s status=healthy http_code=200\n' \
+      "${provider}" "$3" ;;
+    *) printf 'provider=%s boundary=%s status=unavailable http_code=%s\n' \
+      "${provider}" "$3" "${code:-000}" ;;
+  esac
 }

-run_local() {
-  set +e
-  "$@"
-  local rc=$?
-  set -e
-  printf 'exit_code=%s\n' "$rc"
-}
+printf 'schema_version=ollama_direct_route_diagnosis_v1 writes_performed=0\n'
+probe ollama_gcp_a "${GCP_A_URL}" cloud
+probe ollama_gcp_b "${GCP_B_URL}" cloud
+probe ollama_local "${HOST111_URL}" local

-section "Production health provider chain"
-run_local curl -sS -m 12 https://awoooi.wooo.work/api/v1/health
-
-section "API pod view"
-set +e
-kubectl -n "${NAMESPACE}" exec -i "deploy/${DEPLOYMENT}" -- sh -lc "
-  set +e
-  echo OLLAMA_URL=\$OLLAMA_URL
-  echo OLLAMA_SECONDARY_URL=\$OLLAMA_SECONDARY_URL
-  echo OLLAMA_FALLBACK_URL=\$OLLAMA_FALLBACK_URL
-  curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://192.168.0.110:${PROXY_PORT}/api/tags | head -60
-"
-printf 'exit_code=%s\n' "$?"
-set -e
-
-section "110 proxy upstream reachability"
-run_local ssh -o BatchMode=yes -o ConnectTimeout="${SSH_TIMEOUT}" "${PROXY_HOST}" "
-  set +e
-  echo route:
-  ip route get ${LOCAL_IP}
-  echo neigh:
-  ip neigh show ${LOCAL_IP}
-  echo ping:
-  ping -c 3 -W 2 ${LOCAL_IP}
-  echo direct_111:
-  curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://${LOCAL_IP}:${LOCAL_PORT}/api/tags | head -60
-  echo proxy_11437:
-  curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://127.0.0.1:${PROXY_PORT}/api/tags | head -60
-  echo nginx_recent_errors:
-  tail -20 /var/log/nginx/ollama-local-error.log 2>/dev/null || true
-"
-
-section "111 host direct SSH view"
-run_local ssh -o BatchMode=yes -o ConnectTimeout="${SSH_TIMEOUT}" "${LOCAL_HOST}" "
-  set +e
-  hostname
-  date
-  curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://127.0.0.1:${LOCAL_PORT}/api/tags | head -60
-  launchctl print gui/501/com.momo.ollama111-allow-proxy 2>/dev/null | head -60 || true
-  pmset -g custom 2>/dev/null | sed -n '1,80p' || true
-"
-
-cat <<'EOF'
-
-Interpretation:
- If 110 shows "ip neigh INCOMPLETE", "Destination Host Unreachable", or "No route to host",
-  the 111 host or LAN path is down. Do not restart API or change provider order.
- If 110 can reach 111 directly but 11437 returns 502, inspect/reload the 110 Nginx proxy.
- If 111 direct SSH works but 127.0.0.1:11434 fails, recover the 111 Ollama/allowlist LaunchAgent.
-EOF
+status_code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "${CURL_TIMEOUT}" \
+  "${PUBLIC_STATUS_URL}" 2>/dev/null || true)"
+printf 'production_route_readback_http=%s\n' "${status_code:-000}"
+printf '%s\n' \
+  'terminal=read_only provider_order=ollama_gcp_a,ollama_gcp_b,ollama_local,claude,gemini host110_transport_selected=false'
--- a/scripts/ops/run-paid-provider-canary.py
+++ b/scripts/ops/run-paid-provider-canary.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Execute the bounded five-lane SRE comparison and paid-provider canary."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import re
+import sys
+from pathlib import Path
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+_API_ROOT = _REPO_ROOT / "apps" / "api"
+if str(_API_ROOT) not in sys.path:
+    sys.path.insert(0, str(_API_ROOT))
+
+
+async def _run_validation(**kwargs):
+    from src.services.paid_provider_canary_validation import (
+        run_paid_provider_canary_validation,
+    )
+
+    return await run_paid_provider_canary_validation(**kwargs)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run-ref", required=True)
+    parser.add_argument("--operator-id", default="codex-controlled-canary")
+    parser.add_argument("--authorization-ref", required=True)
+    args = parser.parse_args()
+    try:
+        receipt = asyncio.run(
+            _run_validation(
+                run_ref=args.run_ref,
+                operator_id=args.operator_id,
+                authorization_ref=args.authorization_ref,
+            )
+        )
+    except BaseException as exc:
+        error_code = str(exc)
+        if not re.fullmatch(r"[a-z][a-z0-9_]{0,119}", error_code):
+            error_code = f"canary_failed_{type(exc).__name__}"
+        print(
+            json.dumps(
+                {
+                    "schema_version": "paid_provider_sre_canary_error_v1",
+                    "status": "blocked_with_safe_next_action",
+                    "error_code": error_code[:120],
+                    "raw_prompt_persisted": False,
+                    "raw_response_persisted": False,
+                    "secret_value_returned": False,
+                },
+                sort_keys=True,
+            )
+        )
+        return 2
+    print(json.dumps(receipt, ensure_ascii=False, sort_keys=True))
+    return 0 if receipt["independent_verifier"]["passed"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
--- a/scripts/ops/tests/test_alertmanager_runtime_scrape_contract.py
+++ b/scripts/ops/tests/test_alertmanager_runtime_scrape_contract.py
@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "ops" / "deploy-alertmanager-runtime-scrape.sh"
+
+
+def test_controlled_scrape_apply_has_check_rollback_and_independent_verifier() -> None:
+    text = SCRIPT.read_text(encoding="utf-8")
+
+    assert 'TRACE_ID="${TRACE_ID:?TRACE_ID is required}"' in text
+    assert 'RUN_ID="${RUN_ID:?RUN_ID is required}"' in text
+    assert 'WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"' in text
+    assert "promtool check config" in text
+    assert "writes_performed=0" in text
+    assert "rollback_receipt" in text
+    assert "terminal=rollback_failed ready=false" in text
+    assert "flock -n 9" in text
+    assert '!= "${CANDIDATE_SHA}"' in text
+    assert '!= "${SOURCE_SHA}"' in text
+    assert "alertmanager_notification_requests_total" in text
+    assert "alertmanager_notification_requests_failed_total" in text
+    assert 'receiver="awoooi-webhook"' in text
+    assert text.count('receiver="awoooi-webhook"') >= 4
+    assert "two_scrapes_up" in text
+    assert "delivery_series_count" in text
+
+
+def test_controlled_scrape_apply_never_replaces_the_full_live_config() -> None:
+    text = SCRIPT.read_text(encoding="utf-8")
+
+    assert "BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH" in text
+    assert "job_name: 'alertmanager-runtime'" in text
+    assert "targets: ['alertmanager:9093']" in text
+    assert "legacy_pattern" in text
+    assert "k8s/monitoring/prometheus.yml" not in text
--- a/scripts/ops/tests/test_alertmanager_webhook_config.py
+++ b/scripts/ops/tests/test_alertmanager_webhook_config.py
@@ -7,6 +7,7 @@ import yaml

 ROOT = Path(__file__).resolve().parents[3]
 ALERTMANAGER_CONFIG = ROOT / "ops" / "alertmanager" / "alertmanager.yml"
+DEPLOY_SCRIPT = ROOT / "scripts" / "ops" / "deploy-alertmanager-config.sh"


 def _config() -> dict:
@@ -37,3 +38,70 @@ def test_emergency_direct_telegram_stays_limited_to_alert_chain() -> None:
    route_text = str(_config()["route"])
    assert "AWOOOIApiDown" in route_text
    assert 'severity="critical"' in route_text
+
+
+def test_emergency_card_is_bounded_and_has_honest_automation_attribution() -> None:
+    receiver = _receiver("telegram-direct")
+    telegram = receiver["telegram_configs"][0]
+    message = telegram["message"]
+
+    assert "SRE BYPASS｜告警主鏈異常" in message
+    assert "Prometheus deterministic rule（未呼叫 LLM）" in message
+    assert "Executor：none" in message
+    assert "未宣稱已修復" in message
+    assert "Verifier：source-specific delivery metric" in message
+    assert "故障主機" not in message
+    telegram_route = next(
+        route
+        for route in _config()["route"]["routes"]
+        if route["receiver"] == "telegram-direct"
+    )
+    assert telegram_route["repeat_interval"] == "2h"
+
+
+def test_alertmanager_has_no_agent99_inbound_receiver_or_secret_first_hop() -> None:
+    config = _config()
+    assert all(
+        item.get("receiver") != "agent99-alert-chain-relay"
+        for item in config["route"]["routes"]
+    )
+    assert all(
+        receiver.get("name") != "agent99-alert-chain-relay"
+        for receiver in config["receivers"]
+    )
+    source = ALERTMANAGER_CONFIG.read_text(encoding="utf-8")
+    assert "192.168.0.99:8787" not in source
+    assert "AGENT99_SRE_RELAY_TOKEN_PLACEHOLDER" not in source
+    assert "authorization:" not in source
+
+
+def test_alertmanager_deploy_does_not_inject_agent99_relay_secret() -> None:
+    source = DEPLOY_SCRIPT.read_text(encoding="utf-8")
+    assert "AGENT99_SRE_ALERT_RELAY_TOKEN" not in source
+    assert 'os.environ["AGENT99_SRE_RELAY_TOKEN"]' not in source
+    assert "AGENT99_SRE_RELAY_TOKEN_PLACEHOLDER" not in source
+    assert "unreplaced secret placeholder remains" in source
+    assert "set -x" not in source
+
+
+def test_secret_bearing_alertmanager_config_and_backups_are_not_world_readable() -> None:
+    source = DEPLOY_SCRIPT.read_text(encoding="utf-8")
+
+    assert "install -m 0600" in source
+    assert 'chmod 0600 "\\$legacy_backup"' in source
+    assert 'chmod 0400 "\\$target"' in source
+    assert 'chmod 0000 "\\$target"' in source
+    assert "chmod 0644" not in source
+    assert "/proc/\\${container_pid}/status" in source
+    assert "runtime_uid" in source
+    assert "runtime_gid" in source
+    assert "test -r /etc/alertmanager/alertmanager.yml" in source
+    assert "rollback_secure" in source
+    assert "secure rollback attempted" in source
+    assert "amtool check-config /etc/alertmanager/alertmanager.yml" in source
+    assert "amtool check-config /dev/stdin" in source
+    assert "/tmp/alertmanager-rendered.yml" not in source
+    assert "trap 'rm -f \"\\$staged\"' EXIT" in source
+    assert source.index('chmod 0000 "\\$target"') < source.index(
+        "sudo -n sh -c 'cat \"\\$1\" > \"\\$2\"'"
+    )