#!/usr/bin/env bash # Deploy only the canonical SigNoz blackbox route to the host110 Prometheus. # The full Prometheus file intentionally remains untouched because production # contains unrelated, independently governed scrape targets. set -euo pipefail TRACE_ID="${TRACE_ID:?TRACE_ID is required}" RUN_ID="${RUN_ID:?RUN_ID is required}" WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}" TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}" PROMETHEUS_URL="${PROMETHEUS_URL:-http://192.168.0.110:9090}" PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}" REMOTE_CONFIG="${REMOTE_CONFIG:-/home/wooo/monitoring/prometheus.yml}" MODE="${1:-apply}" case "${MODE}" in apply|--dry-run) ;; *) printf 'usage: %s [apply|--dry-run]\n' "$0" >&2; exit 64 ;; esac for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do case "${value}" in *[!A-Za-z0-9._-]*|'') printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2 exit 64 ;; esac done ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \ bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \ "${PROMETHEUS_URL}" "${PROMETHEUS_CONTAINER}" "${REMOTE_CONFIG}" <<'REMOTE' set -euo pipefail TRACE_ID="$1" RUN_ID="$2" WORK_ITEM_ID="$3" MODE="$4" PROMETHEUS_URL="$5" PROMETHEUS_CONTAINER="$6" REMOTE_CONFIG="$7" REMOTE_CANDIDATE="/tmp/awoooi-prometheus-signoz.${RUN_ID}.yml" RUNTIME_CANDIDATE="/etc/prometheus/.awoooi-prometheus-signoz.${RUN_ID}.yml" BACKUP_CONFIG="${REMOTE_CONFIG}.bak.${RUN_ID}" APPLY_STARTED=0 DEPLOY_VERIFIED=0 cleanup_candidate() { rm -f "${REMOTE_CANDIDATE}" docker exec -u 0 "${PROMETHEUS_CONTAINER}" \ rm -f "${RUNTIME_CANDIDATE}" >/dev/null 2>&1 || true } rollback_deploy() { cp "${BACKUP_CONFIG}" "${REMOTE_CONFIG}" curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null printf 'rollback_verifier trace_id=%s run_id=%s work_item_id=%s terminal=rolled_back ready=true backup=%s\n' \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}" } on_exit() { local rc=$? trap - EXIT set +e cleanup_candidate if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] \ && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then rollback_deploy fi exit "${rc}" } trap on_exit EXIT test -f "${REMOTE_CONFIG}" test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY' from pathlib import Path import sys source = Path(sys.argv[1]) candidate = Path(sys.argv[2]) text = source.read_text(encoding="utf-8") canonical = " - http://192.168.0.110:8080/api/v1/health\n" legacy = " - 192.168.0.188:3301\n" anchor = " - http://192.168.0.110:3001\n" def block_bounds(payload: str, job: str) -> tuple[int, int]: marker = f' - job_name: "{job}"\n' start = payload.find(marker) if start < 0 or payload.find(marker, start + 1) >= 0: raise SystemExit(f"expected exactly one {job} job") end = payload.find("\n - job_name:", start + len(marker)) return start, len(payload) if end < 0 else end + 1 http_start, http_end = block_bounds(text, "blackbox-http") http_block = text[http_start:http_end] if http_block.count(canonical) > 1 or http_block.count(anchor) != 1: raise SystemExit("canonical HTTP target or insertion anchor is ambiguous") if canonical not in http_block: http_block = http_block.replace(anchor, anchor + canonical, 1) text = text[:http_start] + http_block + text[http_end:] tcp_start, tcp_end = block_bounds(text, "blackbox-tcp") tcp_block = text[tcp_start:tcp_end] if tcp_block.count(legacy) > 1: raise SystemExit("legacy TCP target is ambiguous") tcp_block = tcp_block.replace(legacy, "", 1) text = text[:tcp_start] + tcp_block + text[tcp_end:] http_start, http_end = block_bounds(text, "blackbox-http") tcp_start, tcp_end = block_bounds(text, "blackbox-tcp") if text[http_start:http_end].count(canonical) != 1: raise SystemExit("canonical HTTP target was not normalized exactly once") if legacy in text[tcp_start:tcp_end]: raise SystemExit("legacy TCP target remains after normalization") candidate.write_text(text, encoding="utf-8") PY docker cp "${REMOTE_CANDIDATE}" \ "${PROMETHEUS_CONTAINER}:${RUNTIME_CANDIDATE}" >/dev/null docker exec "${PROMETHEUS_CONTAINER}" promtool check config "${RUNTIME_CANDIDATE}" SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" CANDIDATE_SHA="$(sha256sum "${REMOTE_CANDIDATE}" | awk '{print $1}')" printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s source_sha=%s candidate_sha=%s promtool=pass\n' \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \ "${SOURCE_SHA}" "${CANDIDATE_SHA}" if [ "${MODE}" = "--dry-run" ]; then printf 'terminal trace_id=%s run_id=%s status=check_pass_no_write\n' \ "${TRACE_ID}" "${RUN_ID}" exit 0 fi cp -p "${REMOTE_CONFIG}" "${BACKUP_CONFIG}" APPLY_STARTED=1 cp "${REMOTE_CANDIDATE}" "${REMOTE_CONFIG}" ACTIVE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" RUNTIME_SHA="$(docker exec "${PROMETHEUS_CONTAINER}" \ sha256sum /etc/prometheus/prometheus.yml | awk '{print $1}')" test "${ACTIVE_SHA}" = "${CANDIDATE_SHA}" test "${RUNTIME_SHA}" = "${CANDIDATE_SHA}" curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=target_first_config_apply active_sha=%s runtime_sha=%s ready=true backup=%s\n' \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${ACTIVE_SHA}" \ "${RUNTIME_SHA}" "${BACKUP_CONFIG}" target_state() { curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c ' import json, sys payload = json.load(sys.stdin) targets = payload["data"]["activeTargets"] canonical = [t for t in targets if t.get("labels", {}).get("job") == "blackbox-http" and t.get("labels", {}).get("instance") == "http://192.168.0.110:8080/api/v1/health"] legacy = [t for t in targets if t.get("labels", {}).get("instance") == "192.168.0.188:3301"] if len(canonical) == 1: target = canonical[0] print("{}|{}|{}|{}".format( 1, target.get("health", "unknown"), target.get("lastScrape", "never"), len(legacy), )) else: print(f"{len(canonical)}|missing|never|{len(legacy)}") ' } FIRST_SCRAPE="" for attempt in $(seq 1 9); do IFS='|' read -r count health last_scrape legacy_count <<<"$(target_state)" printf 'target_probe trace_id=%s run_id=%s attempt=%s count=%s health=%s last_scrape=%s legacy_count=%s\n' \ "${TRACE_ID}" "${RUN_ID}" "${attempt}" "${count}" "${health}" \ "${last_scrape}" "${legacy_count}" if [ "${count}" = 1 ] && [ "${health}" = up ] \ && [ "${legacy_count}" = 0 ] && [ "${last_scrape}" != never ]; then FIRST_SCRAPE="${last_scrape}" break fi sleep 10 done test -n "${FIRST_SCRAPE}" sleep 16 IFS='|' read -r count health SECOND_SCRAPE legacy_count <<<"$(target_state)" printf 'target_probe trace_id=%s run_id=%s sample=post_cooldown count=%s health=%s last_scrape=%s legacy_count=%s\n' \ "${TRACE_ID}" "${RUN_ID}" "${count}" "${health}" \ "${SECOND_SCRAPE}" "${legacy_count}" test "${count}" = 1 test "${health}" = up test "${legacy_count}" = 0 test "${SECOND_SCRAPE}" != "${FIRST_SCRAPE}" PROBE_VALUE="$(curl -fsS --get \ --data-urlencode 'query=probe_success{job="blackbox-http",instance="http://192.168.0.110:8080/api/v1/health"}' \ "${PROMETHEUS_URL}/api/v1/query" | python3 -c ' import json, sys result = json.load(sys.stdin)["data"]["result"] print(result[0]["value"][1] if len(result) == 1 else "invalid") ')" test "${PROBE_VALUE}" = 1 DEPLOY_VERIFIED=1 printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s terminal=target_up_two_scrapes first=%s second=%s probe_success=%s legacy_target_count=0\n' \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${FIRST_SCRAPE}" \ "${SECOND_SCRAPE}" "${PROBE_VALUE}" printf 'terminal trace_id=%s run_id=%s status=completed_target_first backup=%s\n' \ "${TRACE_ID}" "${RUN_ID}" "${BACKUP_CONFIG}" REMOTE