awoooi/scripts/reboot-recovery/signoz-110-static-ingest.sh

#!/usr/bin/env bash
# Controlled static-ingestion fallback for the formal SigNoz runtime on host 110.
# It never reads .env, container environment values, secrets, or stateful volumes.

set -euo pipefail

TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
REMOTE_DIR="${REMOTE_DIR:-/home/wooo/signoz/deploy/docker}"
REMOTE_BASE="${REMOTE_BASE:-$REMOTE_DIR/docker-compose.yaml}"
REMOTE_OVERRIDE="${REMOTE_OVERRIDE:-$REMOTE_DIR/docker-compose.awoooi-static-ingest.yaml}"
REMOTE_PROJECT="${REMOTE_PROJECT:-signoz}"
SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}"
TRACE_ID="${TRACE_ID:-P0-OBS-001}"
RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}"
WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}"
MANAGED_PROMOTION_RECEIPT="${MANAGED_PROMOTION_RECEIPT:-}"
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
LOCAL_OVERRIDE="${LOCAL_OVERRIDE:-$ROOT_DIR/ops/signoz/docker-compose.otlp-static.override.yaml}"
SSH_OPTS=(
  -o BatchMode=yes
  -o ConnectTimeout="$SSH_CONNECT_TIMEOUT"
  -o ConnectionAttempts=1
  -o ServerAliveInterval=5
  -o ServerAliveCountMax=1
)

usage() {
  cat <<'USAGE'
Usage: signoz-110-static-ingest.sh [--check|--apply|--status|--rollback]

  --check      Validate the base stack and source-controlled override.
  --apply      Recreate only signoz-otel-collector with the static base config.
  --status     Verify the collector and OTLP listeners.
  --rollback   Restore managed OpAMP only with a validated promotion receipt.

All compose calls use --env-file /dev/null. Stateful services stay untouched.
USAGE
}

mode="${1:---check}"
case "$mode" in
  --check|--apply|--status|--rollback) ;;
  -h|--help) usage; exit 0 ;;
  *) usage >&2; exit 64 ;;
esac

remote() {
  ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "$@"
}

receipt() {
  local phase="$1" result="$2" detail="$3"
  printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-110-otel-ingest","phase":"%s","result":"%s","detail":"%s"}\n' \
    "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail"
}

on_error() {
  local rc=$?
  receipt terminal failed "command_exit_$rc"
  exit "$rc"
}
trap on_error ERR

status_remote() {
  remote "set -eu
container=signoz-otel-collector
running=\$(docker inspect -f '{{.State.Running}}' \"\$container\")
restart_count=\$(docker inspect -f '{{.RestartCount}}' \"\$container\")
cmd=\$(docker inspect -f '{{json .Config.Cmd}}' \"\$container\")
http_code=\$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[]}' http://127.0.0.1:4318/v1/logs || true)
grpc=0
http=0
if docker exec \"\$container\" sh -c \"grep -q ':10DD .* 0A ' /proc/net/tcp /proc/net/tcp6\"; then grpc=1; fi
if docker exec \"\$container\" sh -c \"grep -q ':10DE .* 0A ' /proc/net/tcp /proc/net/tcp6\"; then http=1; fi
printf 'SIGNOZ_STATIC_INGEST_STATUS running=%s restart_count=%s grpc_4317=%s http_4318=%s otlp_http_code=%s command=%s\\n' \"\$running\" \"\$restart_count\" \"\$grpc\" \"\$http\" \"\${http_code:-000}\" \"\$cmd\"
[ \"\$running\" = true ] && [ \"\$grpc\" = 1 ] && [ \"\$http\" = 1 ] && [ \"\${http_code:-000}\" = 200 ]"
}

validate_managed_promotion_receipt() {
  [ -n "$MANAGED_PROMOTION_RECEIPT" ]
  [ -s "$MANAGED_PROMOTION_RECEIPT" ]
  python3 - "$MANAGED_PROMOTION_RECEIPT" "$WORK_ITEM_ID" <<'PY'
import json
import sys

path, work_item_id = sys.argv[1:]
with open(path, encoding="utf-8") as handle:
    payload = json.load(handle)

assert payload.get("schema") == "awoooi_signoz_managed_promotion_v1"
assert payload.get("work_item_id") == work_item_id
assert payload.get("terminal") == "pass"
assert payload.get("organization", {}).get("setup_completed") is True
assert payload.get("opamp", {}).get("nop_pipeline_count") == 0
signals = payload.get("signals", {})
assert all(signals.get(name) is True for name in ("logs", "metrics", "traces"))
PY
}

case "$mode" in
  --check)
    test -s "$LOCAL_OVERRIDE"
    receipt sensor pass "source_override_present"
    remote "set -eu
test -s '$REMOTE_BASE'
docker inspect signoz >/dev/null
docker inspect signoz-clickhouse >/dev/null
docker inspect signoz-zookeeper-1 >/dev/null
docker inspect signoz-otel-collector >/dev/null
cd '$REMOTE_DIR'
docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' config --quiet
printf 'SIGNOZ_STATIC_INGEST_CHECK base_compose=valid env_file=/dev/null collector=%s\\n' \"\$(docker inspect -f '{{.State.Status}}' signoz-otel-collector)\""
    receipt check pass "base_compose_valid_no_env"
    ;;
  --apply)
    test -s "$LOCAL_OVERRIDE"
    receipt decision allow "risk_medium_stateless_collector_only"
    scp "${SSH_OPTS[@]}" "$LOCAL_OVERRIDE" "$TARGET_HOST:$REMOTE_OVERRIDE.tmp"
    remote "set -eu
chmod 0644 '$REMOTE_OVERRIDE.tmp'
mv '$REMOTE_OVERRIDE.tmp' '$REMOTE_OVERRIDE'
cd '$REMOTE_DIR'
docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' -f '$REMOTE_OVERRIDE' config --quiet
printf 'SIGNOZ_STATIC_INGEST_DRY_RUN result=pass env_file=/dev/null override=%s\\n' '$REMOTE_OVERRIDE'"
    receipt check pass "compose_override_valid"
    remote "set -eu
cd '$REMOTE_DIR'
before_id=\$(docker inspect -f '{{.Id}}' signoz-otel-collector)
docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' -f '$REMOTE_OVERRIDE' up -d --no-deps --pull never otel-collector
after_id=\$(docker inspect -f '{{.Id}}' signoz-otel-collector)
printf 'SIGNOZ_STATIC_INGEST_APPLY before_id=%s after_id=%s stateful_services_touched=0\\n' \"\$before_id\" \"\$after_id\""
    receipt execute pass "collector_recreated_stateful_services_untouched"
    status_remote
    receipt verify pass "otlp_4317_4318_listening_empty_request_200"
    ;;
  --status)
    status_remote
    receipt verify pass "runtime_listener_check"
    ;;
  --rollback)
    if ! validate_managed_promotion_receipt; then
      receipt rollback blocked "managed_promotion_receipt_required"
      exit 3
    fi
    receipt decision allow "bounded_rollback_collector_only"
    remote "set -eu
cd '$REMOTE_DIR'
docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' up -d --no-deps --pull never otel-collector
cmd=\$(docker inspect -f '{{json .Config.Cmd}}' signoz-otel-collector)
printf 'SIGNOZ_STATIC_INGEST_ROLLBACK command=%s\\n' \"\$cmd\"
printf '%s' \"\$cmd\" | grep -q -- '--manager-config'"
    receipt rollback pass "validated_managed_opamp_command_restored"
    ;;
esac