fix(observability): restore durable SigNoz ingestion

2026-07-14 19:42:38 +08:00
parent 378d867e9c
commit 582388e701
10 changed files with 578 additions and 4 deletions
--- a/scripts/reboot-recovery/signoz-110-static-ingest.sh
+++ b/scripts/reboot-recovery/signoz-110-static-ingest.sh
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+# Controlled static-ingestion fallback for the formal SigNoz runtime on host 110.
+# It never reads .env, container environment values, secrets, or stateful volumes.
+
+set -euo pipefail
+
+TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
+REMOTE_DIR="${REMOTE_DIR:-/home/wooo/signoz/deploy/docker}"
+REMOTE_BASE="${REMOTE_BASE:-$REMOTE_DIR/docker-compose.yaml}"
+REMOTE_OVERRIDE="${REMOTE_OVERRIDE:-$REMOTE_DIR/docker-compose.awoooi-static-ingest.yaml}"
+REMOTE_PROJECT="${REMOTE_PROJECT:-signoz}"
+SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}"
+TRACE_ID="${TRACE_ID:-P0-OBS-001}"
+RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}"
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+LOCAL_OVERRIDE="${LOCAL_OVERRIDE:-$ROOT_DIR/ops/signoz/docker-compose.otlp-static.override.yaml}"
+SSH_OPTS=(
+  -o BatchMode=yes
+  -o ConnectTimeout="$SSH_CONNECT_TIMEOUT"
+  -o ConnectionAttempts=1
+  -o ServerAliveInterval=5
+  -o ServerAliveCountMax=1
+)
+
+usage() {
+  cat <<'USAGE'
+Usage: signoz-110-static-ingest.sh [--check|--apply|--status|--rollback]
+
+  --check      Validate the base stack and source-controlled override.
+  --apply      Recreate only signoz-otel-collector with the static base config.
+  --status     Verify the collector and OTLP listeners.
+  --rollback   Recreate only the collector from the original OpAMP command.
+
+All compose calls use --env-file /dev/null. Stateful services stay untouched.
+USAGE
+}
+
+mode="${1:---check}"
+case "$mode" in
+  --check|--apply|--status|--rollback) ;;
+  -h|--help) usage; exit 0 ;;
+  *) usage >&2; exit 64 ;;
+esac
+
+remote() {
+  ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "$@"
+}
+
+receipt() {
+  local phase="$1" result="$2" detail="$3"
+  printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-110-otel-ingest","phase":"%s","result":"%s","detail":"%s"}\n' \
+    "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail"
+}
+
+on_error() {
+  local rc=$?
+  receipt terminal failed "command_exit_$rc"
+  exit "$rc"
+}
+trap on_error ERR
+
+status_remote() {
+  remote "set -eu
+container=signoz-otel-collector
+running=\$(docker inspect -f '{{.State.Running}}' \"\$container\")
+restart_count=\$(docker inspect -f '{{.RestartCount}}' \"\$container\")
+cmd=\$(docker inspect -f '{{json .Config.Cmd}}' \"\$container\")
+http_code=\$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[]}' http://127.0.0.1:4318/v1/logs || true)
+grpc=0
+http=0
+if docker exec \"\$container\" sh -c \"grep -q ':10DD .* 0A ' /proc/net/tcp /proc/net/tcp6\"; then grpc=1; fi
+if docker exec \"\$container\" sh -c \"grep -q ':10DE .* 0A ' /proc/net/tcp /proc/net/tcp6\"; then http=1; fi
+printf 'SIGNOZ_STATIC_INGEST_STATUS running=%s restart_count=%s grpc_4317=%s http_4318=%s otlp_http_code=%s command=%s\\n' \"\$running\" \"\$restart_count\" \"\$grpc\" \"\$http\" \"\${http_code:-000}\" \"\$cmd\"
+[ \"\$running\" = true ] && [ \"\$grpc\" = 1 ] && [ \"\$http\" = 1 ] && [ \"\${http_code:-000}\" = 200 ]"
+}
+
+case "$mode" in
+  --check)
+    test -s "$LOCAL_OVERRIDE"
+    receipt sensor pass "source_override_present"
+    remote "set -eu
+test -s '$REMOTE_BASE'
+docker inspect signoz >/dev/null
+docker inspect signoz-clickhouse >/dev/null
+docker inspect signoz-zookeeper-1 >/dev/null
+docker inspect signoz-otel-collector >/dev/null
+cd '$REMOTE_DIR'
+docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' config --quiet
+printf 'SIGNOZ_STATIC_INGEST_CHECK base_compose=valid env_file=/dev/null collector=%s\\n' \"\$(docker inspect -f '{{.State.Status}}' signoz-otel-collector)\""
+    receipt check pass "base_compose_valid_no_env"
+    ;;
+  --apply)
+    test -s "$LOCAL_OVERRIDE"
+    receipt decision allow "risk_medium_stateless_collector_only"
+    scp "${SSH_OPTS[@]}" "$LOCAL_OVERRIDE" "$TARGET_HOST:$REMOTE_OVERRIDE.tmp"
+    remote "set -eu
+chmod 0644 '$REMOTE_OVERRIDE.tmp'
+mv '$REMOTE_OVERRIDE.tmp' '$REMOTE_OVERRIDE'
+cd '$REMOTE_DIR'
+docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' -f '$REMOTE_OVERRIDE' config --quiet
+printf 'SIGNOZ_STATIC_INGEST_DRY_RUN result=pass env_file=/dev/null override=%s\\n' '$REMOTE_OVERRIDE'"
+    receipt check pass "compose_override_valid"
+    remote "set -eu
+cd '$REMOTE_DIR'
+before_id=\$(docker inspect -f '{{.Id}}' signoz-otel-collector)
+docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' -f '$REMOTE_OVERRIDE' up -d --no-deps --pull never otel-collector
+after_id=\$(docker inspect -f '{{.Id}}' signoz-otel-collector)
+printf 'SIGNOZ_STATIC_INGEST_APPLY before_id=%s after_id=%s stateful_services_touched=0\\n' \"\$before_id\" \"\$after_id\""
+    receipt execute pass "collector_recreated_stateful_services_untouched"
+    status_remote
+    receipt verify pass "otlp_4317_4318_listening_empty_request_200"
+    ;;
+  --status)
+    status_remote
+    receipt verify pass "runtime_listener_check"
+    ;;
+  --rollback)
+    receipt decision allow "bounded_rollback_collector_only"
+    remote "set -eu
+cd '$REMOTE_DIR'
+docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' up -d --no-deps --pull never otel-collector
+cmd=\$(docker inspect -f '{{json .Config.Cmd}}' signoz-otel-collector)
+printf 'SIGNOZ_STATIC_INGEST_ROLLBACK command=%s\\n' \"\$cmd\"
+printf '%s' \"\$cmd\" | grep -q -- '--manager-config'"
+    receipt rollback pass "opamp_command_restored"
+    ;;
+esac
--- a/scripts/reboot-recovery/signoz-188-otlp-bridge.sh
+++ b/scripts/reboot-recovery/signoz-188-otlp-bridge.sh
@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+# Temporary, source-controlled OTLP compatibility bridge for K3s collectors.
+# Owner: platform-observability. Expiry: 2026-08-15 Asia/Taipei.
+# Replacement: repoint all producers and policies directly to host 110 OTLP.
+
+set -euo pipefail
+
+TARGET_HOST="${TARGET_HOST:-ollama@192.168.0.188}"
+REMOTE_SCRIPT="${REMOTE_SCRIPT:-/home/ollama/bin/awoooi-signoz-otlp-bridge.sh}"
+LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}"
+LISTEN_PORT="${LISTEN_PORT:-24318}"
+SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"
+UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}"
+UPSTREAM_PORT="${UPSTREAM_PORT:-4318}"
+TRACE_ID="${TRACE_ID:-P0-OBS-001}"
+RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}"
+VERIFY_NODES="${VERIFY_NODES:-wooo@192.168.0.120 wooo@192.168.0.121}"
+SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}"
+SSH_OPTS=(
+  -o BatchMode=yes
+  -o ConnectTimeout="$SSH_CONNECT_TIMEOUT"
+  -o ConnectionAttempts=1
+  -o ServerAliveInterval=5
+  -o ServerAliveCountMax=1
+)
+
+usage() {
+  cat <<'USAGE'
+Usage: signoz-188-otlp-bridge.sh [--check|--apply|--status|--rollback]
+
+Creates a user-level 192.168.0.188:24318 -> 192.168.0.110:4318 bridge.
+Only source addresses 192.168.0.120 and 192.168.0.121 are accepted.
+No sudo, firewall, Nginx, Docker, database, secret, or volume action is used.
+USAGE
+}
+
+mode="${1:---check}"
+case "$mode" in
+  --check|--apply|--status|--rollback) ;;
+  -h|--help) usage; exit 0 ;;
+  *) usage >&2; exit 64 ;;
+esac
+
+remote() {
+  ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "$@"
+}
+
+receipt() {
+  local phase="$1" result="$2" detail="$3"
+  printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-188-otlp-compat-bridge","phase":"%s","result":"%s","detail":"%s"}\n' \
+    "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail"
+}
+
+remote_payload() {
+  cat <<'REMOTE'
+#!/usr/bin/env bash
+set -euo pipefail
+
+LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}"
+LISTEN_PORT="${LISTEN_PORT:-24318}"
+SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"
+UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}"
+UPSTREAM_PORT="${UPSTREAM_PORT:-4318}"
+RUN_DIR="${RUN_DIR:-/home/ollama/run}"
+LOG_DIR="${LOG_DIR:-/home/ollama/log}"
+PID_FILE="$RUN_DIR/awoooi-signoz-otlp-bridge.pid"
+LOG_FILE="$LOG_DIR/awoooi-signoz-otlp-bridge.log"
+
+is_running() {
+  [ -s "$PID_FILE" ] && kill -0 "$(sed -n '1p' "$PID_FILE")" 2>/dev/null
+}
+
+otlp_code() {
+  curl -sS --max-time 5 -o /dev/null -w '%{http_code}' \
+    -H 'Content-Type: application/json' --data-binary '{"resourceLogs":[]}' "$1" 2>/dev/null || true
+}
+
+status() {
+  running=0
+  if is_running; then running=1; fi
+  listen_socket=0
+  if ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "${LISTEN_HOST}:${LISTEN_PORT}$"; then listen_socket=1; fi
+  upstream_code="$(otlp_code "http://${UPSTREAM_HOST}:${UPSTREAM_PORT}/v1/logs")"
+  printf 'SIGNOZ_OTLP_BRIDGE_STATUS running=%s listen=%s:%s listen_socket=%s source_range=%s upstream=%s:%s upstream_code=%s\n' \
+    "$running" "$LISTEN_HOST" "$LISTEN_PORT" "$listen_socket" "$SOURCE_RANGE" "$UPSTREAM_HOST" "$UPSTREAM_PORT" "${upstream_code:-000}"
+  [ "$running" -eq 1 ] && [ "$listen_socket" -eq 1 ] && [ "${upstream_code:-000}" = 200 ]
+}
+
+start() {
+  command -v socat >/dev/null
+  mkdir -p "$RUN_DIR" "$LOG_DIR"
+  if is_running; then status; return 0; fi
+  if ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)${LISTEN_PORT}$"; then
+    echo "SIGNOZ_OTLP_BRIDGE_BLOCKED port_busy=${LISTEN_PORT}" >&2
+    exit 2
+  fi
+  nohup socat "TCP-LISTEN:${LISTEN_PORT},bind=${LISTEN_HOST},reuseaddr,fork,range=${SOURCE_RANGE}" \
+    "TCP:${UPSTREAM_HOST}:${UPSTREAM_PORT}" >>"$LOG_FILE" 2>&1 &
+  echo "$!" >"$PID_FILE"
+  sleep 1
+  status
+}
+
+stop() {
+  if is_running; then kill "$(sed -n '1p' "$PID_FILE")"; fi
+  rm -f "$PID_FILE"
+  echo "SIGNOZ_OTLP_BRIDGE_STOPPED"
+}
+
+case "${1:-status}" in
+  start) start ;;
+  stop) stop ;;
+  status) status ;;
+  *) exit 64 ;;
+esac
+REMOTE
+}
+
+env_prefix="LISTEN_HOST=$LISTEN_HOST LISTEN_PORT=$LISTEN_PORT SOURCE_RANGE=$SOURCE_RANGE UPSTREAM_HOST=$UPSTREAM_HOST UPSTREAM_PORT=$UPSTREAM_PORT"
+
+verify_nodes() {
+  local target code
+  for target in $VERIFY_NODES; do
+    code="$(ssh "${SSH_OPTS[@]}" "$target" "curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[]}' http://${LISTEN_HOST}:${LISTEN_PORT}/v1/logs || true")"
+    printf 'SIGNOZ_OTLP_BRIDGE_NODE_VERIFY node=%s code=%s\n' "$target" "${code:-000}"
+    [ "${code:-000}" = 200 ]
+  done
+}
+
+case "$mode" in
+  --check)
+    receipt sensor pass "canonical_110_upstream_selected"
+    remote "set -eu
+command -v socat >/dev/null
+upstream=\$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[]}' http://${UPSTREAM_HOST}:${UPSTREAM_PORT}/v1/logs || true)
+busy=0
+if ss -ltn 2>/dev/null | awk '{print \$4}' | grep -Eq '(^|:)${LISTEN_PORT}$'; then busy=1; fi
+printf 'SIGNOZ_OTLP_BRIDGE_CHECK upstream_code=%s listen_port_busy=%s source_range=%s\\n' \"\${upstream:-000}\" \"\$busy\" '$SOURCE_RANGE'
+[ \"\${upstream:-000}\" = 200 ]"
+    receipt check pass "upstream_otlp_200"
+    ;;
+  --apply)
+    receipt decision allow "risk_medium_user_level_compatibility_bridge"
+    remote "mkdir -p /home/ollama/bin /home/ollama/run /home/ollama/log; umask 077; tee '$REMOTE_SCRIPT.tmp' >/dev/null" < <(remote_payload)
+    remote "chmod 0755 '$REMOTE_SCRIPT.tmp' && mv '$REMOTE_SCRIPT.tmp' '$REMOTE_SCRIPT'"
+    remote "(crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d'; printf '%s\n' '# AWOOOI SignOz OTLP bridge start' '@reboot $env_prefix $REMOTE_SCRIPT start >/tmp/awoooi-signoz-otlp-bridge.cron.log 2>&1' '# AWOOOI SignOz OTLP bridge end') | crontab -"
+    remote "$env_prefix '$REMOTE_SCRIPT' start"
+    verify_nodes
+    receipt execute pass "bridge_started_allowlist_120_121"
+    receipt verify pass "nodes_120_121_and_upstream_otlp_200"
+    ;;
+  --status)
+    remote "$env_prefix '$REMOTE_SCRIPT' status"
+    verify_nodes
+    receipt verify pass "bridge_runtime_check"
+    ;;
+  --rollback)
+    remote "if [ -x '$REMOTE_SCRIPT' ]; then $env_prefix '$REMOTE_SCRIPT' stop; fi; crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d' | crontab -"
+    receipt rollback pass "bridge_stopped_cron_removed"
+    ;;
+esac
--- a/scripts/reboot-recovery/tests/test_otel_collector_durable_queue.py
+++ b/scripts/reboot-recovery/tests/test_otel_collector_durable_queue.py
@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+
+ROOT = Path(__file__).resolve().parents[3]
+MANIFEST = ROOT / "k8s" / "observability" / "otel-collector-daemonset.yaml"
+
+
+def _documents() -> list[dict]:
+    return [doc for doc in yaml.safe_load_all(MANIFEST.read_text(encoding="utf-8")) if doc]
+
+
+def test_otel_collector_uses_durable_offsets_and_export_queue() -> None:
+    documents = _documents()
+    config_map = next(doc for doc in documents if doc["kind"] == "ConfigMap")
+    config = yaml.safe_load(config_map["data"]["config.yaml"])
+
+    assert config["extensions"]["file_storage"]["directory"] == "/var/lib/otelcol"
+    assert config["receivers"]["filelog"]["storage"] == "file_storage"
+    exporter = config["exporters"]["otlphttp"]
+    assert exporter["retry_on_failure"]["max_elapsed_time"] == "0s"
+    assert exporter["sending_queue"] == {
+        "enabled": True,
+        "num_consumers": 2,
+        "queue_size": 10000,
+        "storage": "file_storage",
+    }
+    assert "file_storage" in config["service"]["extensions"]
+
+
+def test_otel_daemonset_mounts_per_node_storage_and_rolls_one_at_a_time() -> None:
+    daemonset = next(doc for doc in _documents() if doc["kind"] == "DaemonSet")
+    assert daemonset["spec"]["updateStrategy"]["rollingUpdate"]["maxUnavailable"] == 1
+    pod_spec = daemonset["spec"]["template"]["spec"]
+    assert pod_spec["terminationGracePeriodSeconds"] == 60
+
+    storage = next(volume for volume in pod_spec["volumes"] if volume["name"] == "otel-storage")
+    assert storage["hostPath"] == {
+        "path": "/var/lib/awoooi/otelcol",
+        "type": "DirectoryOrCreate",
+    }
+    collector = next(container for container in pod_spec["containers"] if container["name"] == "otel-collector")
+    assert collector["image"] == (
+        "192.168.0.110:5000/awoooi/otel-collector-contrib@"
+        "sha256:8edf386cdd1f4dc4393e34680e298dcc577296279febf243cf122155fa560c39"
+    )
+    assert collector["imagePullPolicy"] == "IfNotPresent"
+    mount = next(volume for volume in collector["volumeMounts"] if volume["name"] == "otel-storage")
+    assert mount["mountPath"] == "/var/lib/otelcol"
--- a/scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py
+++ b/scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py
@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+OVERRIDE = ROOT / "ops" / "signoz" / "docker-compose.otlp-static.override.yaml"
+STATIC_SCRIPT = ROOT / "scripts" / "reboot-recovery" / "signoz-110-static-ingest.sh"
+BRIDGE_SCRIPT = ROOT / "scripts" / "reboot-recovery" / "signoz-188-otlp-bridge.sh"
+VERIFIER = ROOT / "scripts" / "reboot-recovery" / "verify-signoz-otel-ingestion.sh"
+
+
+def test_static_override_disables_broken_opamp_only_for_collector() -> None:
+    text = OVERRIDE.read_text(encoding="utf-8")
+    assert "otel-collector:" in text
+    assert "--config=/etc/otel-collector-config.yaml" in text
+    assert "--manager-config" not in text
+    assert "--copy-path" not in text
+    assert "signoz-clickhouse" not in text
+    assert "zookeeper" not in text
+    assert "2026-08-15" in text
+
+
+def test_static_apply_is_bounded_no_pull_and_never_reads_dotenv() -> None:
+    text = STATIC_SCRIPT.read_text(encoding="utf-8")
+    executable = "\n".join(
+        line for line in text.splitlines() if not line.lstrip().startswith("#")
+    )
+    assert "--env-file /dev/null" in text
+    assert 'REMOTE_PROJECT="${REMOTE_PROJECT:-signoz}"' in text
+    assert "-p '$REMOTE_PROJECT'" in text
+    assert "up -d --no-deps --pull never otel-collector" in text
+    assert "signoz-clickhouse >/dev/null" in text
+    assert "signoz-zookeeper-1 >/dev/null" in text
+    assert "docker compose down" not in executable
+    assert "docker volume" not in executable
+    assert "docker system prune" not in executable
+    assert "github.com" not in text
+
+
+def test_otlp_bridge_is_allowlisted_reversible_and_has_sunset() -> None:
+    text = BRIDGE_SCRIPT.read_text(encoding="utf-8")
+    executable = "\n".join(
+        line for line in text.splitlines() if not line.lstrip().startswith("#")
+    )
+    assert 'LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}"' in text
+    assert 'LISTEN_PORT="${LISTEN_PORT:-24318}"' in text
+    assert 'SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"' in text
+    assert 'UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}"' in text
+    assert 'UPSTREAM_PORT="${UPSTREAM_PORT:-4318}"' in text
+    assert 'VERIFY_NODES="${VERIFY_NODES:-wooo@192.168.0.120 wooo@192.168.0.121}"' in text
+    assert "resourceLogs" in text
+    assert "@reboot" in text
+    assert "verify_nodes" in text
+    assert "--rollback" in text
+    assert "2026-08-15" in text
+    assert "sudo " not in executable
+    assert "systemctl " not in executable
+    assert "iptables " not in executable
+    assert "docker " not in executable
+
+
+def test_ingestion_verifier_binds_queue_bridge_and_clickhouse() -> None:
+    text = VERIFIER.read_text(encoding="utf-8")
+    assert 'EXPECTED_QUEUE_CAPACITY="${EXPECTED_QUEUE_CAPACITY:-10000}"' in text
+    assert "otelcol_exporter_enqueue_failed_log_records" in text
+    assert "otelcol_exporter_send_failed_log_records" in text
+    assert "otelcol_exporter_sent_log_records" in text
+    assert "otelcol_exporter_queue_size" in text
+    assert "192.168.0.188:24318/v1/logs" in text
+    assert "signoz_logs.logs_v2" in text
+    assert "body={marker:String}" in text
+    assert "declare -A" not in text
+    assert "github.com" not in text
--- a/scripts/reboot-recovery/verify-signoz-otel-ingestion.sh
+++ b/scripts/reboot-recovery/verify-signoz-otel-ingestion.sh
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# Independent verifier for K3s -> 188 compatibility bridge -> 110 SigNoz -> ClickHouse.
+# It writes one non-sensitive synthetic log and queries only that exact marker.
+
+set -euo pipefail
+
+TRACE_ID="${TRACE_ID:-P0-OBS-001}"
+RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}"
+WINDOW_SECONDS="${WINDOW_SECONDS:-30}"
+EXPECTED_PODS="${EXPECTED_PODS:-2}"
+EXPECTED_QUEUE_CAPACITY="${EXPECTED_QUEUE_CAPACITY:-10000}"
+SOURCE_NODE="${SOURCE_NODE:-wooo@192.168.0.120}"
+SIGNOZ_HOST="${SIGNOZ_HOST:-wooo@192.168.0.110}"
+BRIDGE_ENDPOINT="${BRIDGE_ENDPOINT:-http://192.168.0.188:24318/v1/logs}"
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ConnectionAttempts=1)
+marker="P0OBS001_${RUN_ID//[^A-Za-z0-9_]/_}_$(date -u +%Y%m%dT%H%M%SZ)"
+pod_list=()
+enqueue_before=()
+failed_before=()
+sent_before=()
+
+receipt() {
+  local phase="$1" result="$2" detail="$3"
+  printf '{"schema":"awoooi_otel_ingestion_verifier_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","phase":"%s","result":"%s","detail":"%s"}\n' \
+    "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail"
+}
+
+on_error() {
+  local rc=$?
+  receipt terminal failed "command_exit_$rc"
+  exit "$rc"
+}
+trap on_error ERR
+
+pods() {
+  kubectl -n observability get pods -l app.kubernetes.io/name=otel-collector \
+    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'
+}
+
+metric() {
+  local pod="$1" name="$2"
+  kubectl get --raw "/api/v1/namespaces/observability/pods/http:${pod}:8888/proxy/metrics" \
+    | awk -v metric="$name" '$1 ~ ("^" metric "\\{") {print $NF; exit}'
+}
+
+while IFS= read -r pod; do
+  [ -n "$pod" ] && pod_list[${#pod_list[@]}]="$pod"
+done < <(pods)
+pod_count="${#pod_list[@]}"
+[ "$pod_count" -eq "$EXPECTED_PODS" ]
+for ((i = 0; i < pod_count; i++)); do
+  pod="${pod_list[$i]}"
+  enqueue_before[$i]="$(metric "$pod" otelcol_exporter_enqueue_failed_log_records)"
+  failed_before[$i]="$(metric "$pod" otelcol_exporter_send_failed_log_records)"
+  sent_before[$i]="$(metric "$pod" otelcol_exporter_sent_log_records)"
+  capacity="$(metric "$pod" otelcol_exporter_queue_capacity)"
+  [ "$capacity" = "$EXPECTED_QUEUE_CAPACITY" ]
+done
+receipt sensor pass "collector_pods_${pod_count}_queue_capacity_${EXPECTED_QUEUE_CAPACITY}"
+
+http_code="$(ssh "${SSH_OPTS[@]}" "$SOURCE_NODE" "ts=\$(date +%s%N); curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\",\"value\":{\"stringValue\":\"awoooi-p0-obs-verifier\"}}]},\"scopeLogs\":[{\"scope\":{\"name\":\"awoooi.obs.verifier\"},\"logRecords\":[{\"timeUnixNano\":\"'\"\$ts\"'\",\"severityNumber\":9,\"severityText\":\"INFO\",\"body\":{\"stringValue\":\"${marker}\"}}]}]}]}]}' '$BRIDGE_ENDPOINT'")"
+[ "$http_code" = 200 ]
+receipt execute pass "synthetic_otlp_http_200_marker_${marker}"
+
+count=0
+for _ in $(seq 1 15); do
+  count="$(ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" "docker exec signoz-clickhouse clickhouse-client --param_marker='${marker}' --query 'SELECT count() FROM signoz_logs.logs_v2 WHERE body={marker:String}'")"
+  [ "$count" -ge 1 ] && break
+  sleep 2
+done
+[ "$count" -eq 1 ]
+receipt verify pass "clickhouse_exact_marker_count_1"
+
+sleep "$WINDOW_SECONDS"
+for ((i = 0; i < pod_count; i++)); do
+  pod="${pod_list[$i]}"
+  error_lines="$(kubectl -n observability logs "$pod" --since="${WINDOW_SECONDS}s" 2>&1 | grep -Ec 'sending queue is full|connection refused|Exporting failed|rejected_items' || true)"
+  [ "$error_lines" -eq 0 ]
+  [ "$(metric "$pod" otelcol_exporter_queue_size)" = 0 ]
+  [ "$(metric "$pod" otelcol_exporter_enqueue_failed_log_records)" = "${enqueue_before[$i]}" ]
+  [ "$(metric "$pod" otelcol_exporter_send_failed_log_records)" = "${failed_before[$i]}" ]
+  sent_after="$(metric "$pod" otelcol_exporter_sent_log_records)"
+  awk -v before="${sent_before[$i]}" -v after="$sent_after" 'BEGIN {exit !(after >= before)}'
+done
+receipt terminal pass "queue_zero_failure_deltas_zero_window_${WINDOW_SECONDS}s"