219 lines
8.1 KiB
Bash
Executable File
219 lines
8.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Deploy only the canonical SigNoz blackbox route to the host110 Prometheus.
|
|
# The full Prometheus file intentionally remains untouched because production
|
|
# contains unrelated, independently governed scrape targets.
|
|
|
|
set -euo pipefail
|
|
|
|
TRACE_ID="${TRACE_ID:?TRACE_ID is required}"
|
|
RUN_ID="${RUN_ID:?RUN_ID is required}"
|
|
WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"
|
|
TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
|
|
PROMETHEUS_URL="${PROMETHEUS_URL:-http://192.168.0.110:9090}"
|
|
PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
|
|
REMOTE_CONFIG="${REMOTE_CONFIG:-/home/wooo/monitoring/prometheus.yml}"
|
|
MODE="${1:-apply}"
|
|
|
|
case "${MODE}" in
|
|
apply|--dry-run) ;;
|
|
*) printf 'usage: %s [apply|--dry-run]\n' "$0" >&2; exit 64 ;;
|
|
esac
|
|
|
|
for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
|
|
case "${value}" in
|
|
*[!A-Za-z0-9._-]*|'')
|
|
printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
done
|
|
|
|
ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \
|
|
bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
|
|
"${PROMETHEUS_URL}" "${PROMETHEUS_CONTAINER}" "${REMOTE_CONFIG}" <<'REMOTE'
|
|
set -euo pipefail
|
|
|
|
TRACE_ID="$1"
|
|
RUN_ID="$2"
|
|
WORK_ITEM_ID="$3"
|
|
MODE="$4"
|
|
PROMETHEUS_URL="$5"
|
|
PROMETHEUS_CONTAINER="$6"
|
|
REMOTE_CONFIG="$7"
|
|
REMOTE_CANDIDATE="/tmp/awoooi-prometheus-signoz.${RUN_ID}.yml"
|
|
RUNTIME_CANDIDATE="/etc/prometheus/.awoooi-prometheus-signoz.${RUN_ID}.yml"
|
|
BACKUP_CONFIG="${REMOTE_CONFIG}.bak.${RUN_ID}"
|
|
APPLY_STARTED=0
|
|
DEPLOY_VERIFIED=0
|
|
|
|
cleanup_candidate() {
|
|
rm -f "${REMOTE_CANDIDATE}"
|
|
docker exec -u 0 "${PROMETHEUS_CONTAINER}" \
|
|
rm -f "${RUNTIME_CANDIDATE}" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
rollback_deploy() {
|
|
cp "${BACKUP_CONFIG}" "${REMOTE_CONFIG}"
|
|
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
|
|
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
|
|
printf 'rollback_verifier trace_id=%s run_id=%s work_item_id=%s terminal=rolled_back ready=true backup=%s\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
|
|
}
|
|
|
|
on_exit() {
|
|
local rc=$?
|
|
trap - EXIT
|
|
set +e
|
|
cleanup_candidate
|
|
if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] \
|
|
&& [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
|
|
rollback_deploy
|
|
fi
|
|
exit "${rc}"
|
|
}
|
|
trap on_exit EXIT
|
|
|
|
test -f "${REMOTE_CONFIG}"
|
|
test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true
|
|
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
|
|
|
|
python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY'
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
source = Path(sys.argv[1])
|
|
candidate = Path(sys.argv[2])
|
|
text = source.read_text(encoding="utf-8")
|
|
|
|
canonical = " - http://192.168.0.110:8080/api/v1/health\n"
|
|
legacy = " - 192.168.0.188:3301\n"
|
|
anchor = " - http://192.168.0.110:3001\n"
|
|
|
|
|
|
def block_bounds(payload: str, job: str) -> tuple[int, int]:
|
|
marker = f' - job_name: "{job}"\n'
|
|
start = payload.find(marker)
|
|
if start < 0 or payload.find(marker, start + 1) >= 0:
|
|
raise SystemExit(f"expected exactly one {job} job")
|
|
end = payload.find("\n - job_name:", start + len(marker))
|
|
return start, len(payload) if end < 0 else end + 1
|
|
|
|
|
|
http_start, http_end = block_bounds(text, "blackbox-http")
|
|
http_block = text[http_start:http_end]
|
|
if http_block.count(canonical) > 1 or http_block.count(anchor) != 1:
|
|
raise SystemExit("canonical HTTP target or insertion anchor is ambiguous")
|
|
if canonical not in http_block:
|
|
http_block = http_block.replace(anchor, anchor + canonical, 1)
|
|
text = text[:http_start] + http_block + text[http_end:]
|
|
|
|
tcp_start, tcp_end = block_bounds(text, "blackbox-tcp")
|
|
tcp_block = text[tcp_start:tcp_end]
|
|
if tcp_block.count(legacy) > 1:
|
|
raise SystemExit("legacy TCP target is ambiguous")
|
|
tcp_block = tcp_block.replace(legacy, "", 1)
|
|
text = text[:tcp_start] + tcp_block + text[tcp_end:]
|
|
|
|
http_start, http_end = block_bounds(text, "blackbox-http")
|
|
tcp_start, tcp_end = block_bounds(text, "blackbox-tcp")
|
|
if text[http_start:http_end].count(canonical) != 1:
|
|
raise SystemExit("canonical HTTP target was not normalized exactly once")
|
|
if legacy in text[tcp_start:tcp_end]:
|
|
raise SystemExit("legacy TCP target remains after normalization")
|
|
|
|
candidate.write_text(text, encoding="utf-8")
|
|
PY
|
|
|
|
docker cp "${REMOTE_CANDIDATE}" \
|
|
"${PROMETHEUS_CONTAINER}:${RUNTIME_CANDIDATE}" >/dev/null
|
|
docker exec "${PROMETHEUS_CONTAINER}" promtool check config "${RUNTIME_CANDIDATE}"
|
|
|
|
SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
|
|
CANDIDATE_SHA="$(sha256sum "${REMOTE_CANDIDATE}" | awk '{print $1}')"
|
|
printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s source_sha=%s candidate_sha=%s promtool=pass\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
|
|
"${SOURCE_SHA}" "${CANDIDATE_SHA}"
|
|
|
|
if [ "${MODE}" = "--dry-run" ]; then
|
|
printf 'terminal trace_id=%s run_id=%s status=check_pass_no_write\n' \
|
|
"${TRACE_ID}" "${RUN_ID}"
|
|
exit 0
|
|
fi
|
|
|
|
cp -p "${REMOTE_CONFIG}" "${BACKUP_CONFIG}"
|
|
APPLY_STARTED=1
|
|
cp "${REMOTE_CANDIDATE}" "${REMOTE_CONFIG}"
|
|
|
|
ACTIVE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
|
|
RUNTIME_SHA="$(docker exec "${PROMETHEUS_CONTAINER}" \
|
|
sha256sum /etc/prometheus/prometheus.yml | awk '{print $1}')"
|
|
test "${ACTIVE_SHA}" = "${CANDIDATE_SHA}"
|
|
test "${RUNTIME_SHA}" = "${CANDIDATE_SHA}"
|
|
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
|
|
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
|
|
printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=target_first_config_apply active_sha=%s runtime_sha=%s ready=true backup=%s\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${ACTIVE_SHA}" \
|
|
"${RUNTIME_SHA}" "${BACKUP_CONFIG}"
|
|
|
|
target_state() {
|
|
curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
|
|
import json, sys
|
|
payload = json.load(sys.stdin)
|
|
targets = payload["data"]["activeTargets"]
|
|
canonical = [t for t in targets if t.get("labels", {}).get("job") == "blackbox-http" and t.get("labels", {}).get("instance") == "http://192.168.0.110:8080/api/v1/health"]
|
|
legacy = [t for t in targets if t.get("labels", {}).get("instance") == "192.168.0.188:3301"]
|
|
if len(canonical) == 1:
|
|
target = canonical[0]
|
|
print("{}|{}|{}|{}".format(
|
|
1,
|
|
target.get("health", "unknown"),
|
|
target.get("lastScrape", "never"),
|
|
len(legacy),
|
|
))
|
|
else:
|
|
print(f"{len(canonical)}|missing|never|{len(legacy)}")
|
|
'
|
|
}
|
|
|
|
FIRST_SCRAPE=""
|
|
for attempt in $(seq 1 9); do
|
|
IFS='|' read -r count health last_scrape legacy_count <<<"$(target_state)"
|
|
printf 'target_probe trace_id=%s run_id=%s attempt=%s count=%s health=%s last_scrape=%s legacy_count=%s\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${attempt}" "${count}" "${health}" \
|
|
"${last_scrape}" "${legacy_count}"
|
|
if [ "${count}" = 1 ] && [ "${health}" = up ] \
|
|
&& [ "${legacy_count}" = 0 ] && [ "${last_scrape}" != never ]; then
|
|
FIRST_SCRAPE="${last_scrape}"
|
|
break
|
|
fi
|
|
sleep 10
|
|
done
|
|
test -n "${FIRST_SCRAPE}"
|
|
|
|
sleep 16
|
|
IFS='|' read -r count health SECOND_SCRAPE legacy_count <<<"$(target_state)"
|
|
printf 'target_probe trace_id=%s run_id=%s sample=post_cooldown count=%s health=%s last_scrape=%s legacy_count=%s\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${count}" "${health}" \
|
|
"${SECOND_SCRAPE}" "${legacy_count}"
|
|
test "${count}" = 1
|
|
test "${health}" = up
|
|
test "${legacy_count}" = 0
|
|
test "${SECOND_SCRAPE}" != "${FIRST_SCRAPE}"
|
|
|
|
PROBE_VALUE="$(curl -fsS --get \
|
|
--data-urlencode 'query=probe_success{job="blackbox-http",instance="http://192.168.0.110:8080/api/v1/health"}' \
|
|
"${PROMETHEUS_URL}/api/v1/query" | python3 -c '
|
|
import json, sys
|
|
result = json.load(sys.stdin)["data"]["result"]
|
|
print(result[0]["value"][1] if len(result) == 1 else "invalid")
|
|
')"
|
|
test "${PROBE_VALUE}" = 1
|
|
|
|
DEPLOY_VERIFIED=1
|
|
printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s terminal=target_up_two_scrapes first=%s second=%s probe_success=%s legacy_target_count=0\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${FIRST_SCRAPE}" \
|
|
"${SECOND_SCRAPE}" "${PROBE_VALUE}"
|
|
printf 'terminal trace_id=%s run_id=%s status=completed_target_first backup=%s\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${BACKUP_CONFIG}"
|
|
REMOTE
|