fix(observability): restore incumbent OTLP gRPC transport

2026-07-15 01:21:57 +08:00
parent b02bc98c73
commit dd46ff8bc5
7 changed files with 489 additions and 1 deletions
--- a/scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh
+++ b/scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh
@@ -0,0 +1,249 @@
+#!/usr/bin/env bash
+# Independently supervised OTLP/gRPC compatibility bridge for P0-OBS-002.
+# It is paired with, but independently rollbackable from, the HTTP bridge.
+
+set -Eeuo pipefail
+
+TARGET_HOST="${TARGET_HOST:-ollama@192.168.0.188}"
+REMOTE_SCRIPT="${REMOTE_SCRIPT:-/home/ollama/bin/awoooi-signoz-otlp-grpc-bridge.sh}"
+REMOTE_UNIT="${REMOTE_UNIT:-/home/ollama/.config/systemd/user/awoooi-signoz-otlp-grpc-bridge.service}"
+REMOTE_UNIT_STAGE="${REMOTE_UNIT_STAGE:-/home/ollama/run/awoooi-signoz-otlp-grpc-bridge.service}"
+REMOTE_SERVICE="${REMOTE_SERVICE:-awoooi-signoz-otlp-grpc-bridge.service}"
+LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}"
+LISTEN_PORT="${LISTEN_PORT:-24317}"
+SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"
+UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}"
+UPSTREAM_PORT="${UPSTREAM_PORT:-4317}"
+VERIFY_NODES="${VERIFY_NODES:-wooo@192.168.0.120 wooo@192.168.0.121}"
+TRACE_ID="${TRACE_ID:-P0-OBS-002}"
+RUN_ID="${RUN_ID:-P0-OBS-002-grpc-bridge-$(date -u +%Y%m%dT%H%M%SZ)}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-002}"
+SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}"
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+LOCAL_UNIT="${LOCAL_UNIT:-$ROOT_DIR/ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service}"
+APPLY_ACTIVE=0
+SSH_OPTS=(
+  -o BatchMode=yes
+  -o ConnectTimeout="$SSH_CONNECT_TIMEOUT"
+  -o ConnectionAttempts=1
+  -o ServerAliveInterval=5
+  -o ServerAliveCountMax=1
+)
+
+usage() {
+  cat <<'USAGE'
+Usage: signoz-188-otlp-grpc-bridge.sh [mode]
+
+  --check       No-write prerequisite and conflict check.
+  --apply       Install and start the user-systemd gRPC bridge.
+  --status      Require enabled, active, allowlisted end-to-end TCP readiness.
+  --rollback    Disable the new gRPC bridge and verify the listener is absent.
+
+This script never changes firewall, Docker, Kubernetes, database, secrets,
+the existing HTTP bridge, or the application OTLP endpoint.
+USAGE
+}
+
+mode="${1:---check}"
+case "$mode" in
+  --check|--apply|--status|--rollback) ;;
+  -h|--help) usage; exit 0 ;;
+  *) usage >&2; exit 64 ;;
+esac
+
+remote() {
+  ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "$@"
+}
+
+receipt() {
+  local phase="$1" result="$2" detail="$3"
+  printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-188-otlp-grpc-compat-bridge","phase":"%s","result":"%s","detail":"%s"}\n' \
+    "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail"
+}
+
+rollback_runtime() {
+  set +e
+  remote "systemctl --user disable --now '$REMOTE_SERVICE' >/dev/null 2>&1 || true; '$REMOTE_SCRIPT' stop >/dev/null 2>&1 || true"
+  remote "! ss -ltn 2>/dev/null | awk '{print \$4}' | grep -Eq '${LISTEN_HOST}:${LISTEN_PORT}$'"
+  local rc=$?
+  set -e
+  if [ "$rc" -eq 0 ]; then
+    receipt rollback pass "grpc_unit_disabled_listener_absent_http_bridge_unchanged"
+  else
+    receipt rollback failed "grpc_listener_still_present"
+  fi
+  return "$rc"
+}
+
+on_error() {
+  local rc=$?
+  trap - ERR
+  if [ "$APPLY_ACTIVE" -eq 1 ]; then
+    rollback_runtime || true
+  fi
+  receipt terminal failed "command_exit_$rc"
+  exit "$rc"
+}
+trap on_error ERR
+
+remote_payload() {
+  cat <<'REMOTE'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}"
+LISTEN_PORT="${LISTEN_PORT:-24317}"
+SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"
+UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}"
+UPSTREAM_PORT="${UPSTREAM_PORT:-4317}"
+RUN_DIR="${RUN_DIR:-/home/ollama/run}"
+LOG_DIR="${LOG_DIR:-/home/ollama/log}"
+PID_FILE="$RUN_DIR/awoooi-signoz-otlp-grpc-bridge.pid"
+LOG_FILE="$LOG_DIR/awoooi-signoz-otlp-grpc-bridge.log"
+
+is_running() {
+  [ -s "$PID_FILE" ] && kill -0 "$(sed -n '1p' "$PID_FILE")" 2>/dev/null
+}
+
+tcp_open() {
+  timeout 3 bash -c "exec 3<>/dev/tcp/$1/$2" >/dev/null 2>&1
+}
+
+listener_present() {
+  ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "${LISTEN_HOST}:${LISTEN_PORT}$"
+}
+
+status() {
+  running=0
+  listener=0
+  upstream=0
+  if is_running; then running=1; fi
+  if listener_present; then listener=1; fi
+  if tcp_open "$UPSTREAM_HOST" "$UPSTREAM_PORT"; then upstream=1; fi
+  printf 'SIGNOZ_OTLP_GRPC_BRIDGE_STATUS running=%s listen=%s:%s listen_socket=%s source_range=%s upstream=%s:%s upstream_tcp=%s\n' \
+    "$running" "$LISTEN_HOST" "$LISTEN_PORT" "$listener" "$SOURCE_RANGE" "$UPSTREAM_HOST" "$UPSTREAM_PORT" "$upstream"
+  [ "$running" -eq 1 ] && [ "$listener" -eq 1 ] && [ "$upstream" -eq 1 ]
+}
+
+run_foreground() {
+  command -v socat >/dev/null
+  mkdir -p "$RUN_DIR" "$LOG_DIR"
+  if listener_present; then
+    echo "SIGNOZ_OTLP_GRPC_BRIDGE_BLOCKED port_busy=$LISTEN_PORT" >&2
+    exit 2
+  fi
+  echo "$$" >"$PID_FILE"
+  exec socat "TCP-LISTEN:${LISTEN_PORT},bind=${LISTEN_HOST},reuseaddr,fork,range=${SOURCE_RANGE}" \
+    "TCP:${UPSTREAM_HOST}:${UPSTREAM_PORT}" >>"$LOG_FILE" 2>&1
+}
+
+stop() {
+  if is_running; then
+    kill "$(sed -n '1p' "$PID_FILE")"
+    for _ in 1 2 3 4 5; do
+      is_running || break
+      sleep 1
+    done
+  fi
+  rm -f "$PID_FILE"
+  echo "SIGNOZ_OTLP_GRPC_BRIDGE_STOPPED"
+}
+
+case "${1:-status}" in
+  check-upstream) tcp_open "$UPSTREAM_HOST" "$UPSTREAM_PORT" ;;
+  run) run_foreground ;;
+  status) status ;;
+  stop) stop ;;
+  *) exit 64 ;;
+esac
+REMOTE
+}
+
+env_prefix="LISTEN_HOST=$LISTEN_HOST LISTEN_PORT=$LISTEN_PORT SOURCE_RANGE=$SOURCE_RANGE UPSTREAM_HOST=$UPSTREAM_HOST UPSTREAM_PORT=$UPSTREAM_PORT"
+
+verify_nodes() {
+  local node rc
+  for node in $VERIFY_NODES; do
+    rc=0
+    ssh "${SSH_OPTS[@]}" "$node" \
+      "timeout 5 bash -c 'exec 3<>/dev/tcp/${LISTEN_HOST}/${LISTEN_PORT}'" >/dev/null 2>&1 || rc=$?
+    printf 'SIGNOZ_OTLP_GRPC_BRIDGE_NODE_VERIFY node=%s tcp_rc=%s\n' "$node" "$rc"
+    [ "$rc" -eq 0 ]
+  done
+}
+
+preflight() {
+  test -s "$LOCAL_UNIT"
+  remote "set -eu
+command -v bash >/dev/null
+command -v socat >/dev/null
+command -v systemctl >/dev/null
+linger=\$(loginctl show-user \"\$USER\" -p Linger --value 2>/dev/null || true)
+upstream=0
+if timeout 3 bash -c 'exec 3<>/dev/tcp/${UPSTREAM_HOST}/${UPSTREAM_PORT}' >/dev/null 2>&1; then upstream=1; fi
+busy=0
+if ss -ltn 2>/dev/null | awk '{print \$4}' | grep -Eq '${LISTEN_HOST}:${LISTEN_PORT}$'; then busy=1; fi
+printf 'SIGNOZ_OTLP_GRPC_BRIDGE_CHECK upstream_tcp=%s listen_port_busy=%s source_range=%s linger=%s\n' \"\$upstream\" \"\$busy\" '$SOURCE_RANGE' \"\${linger:-no}\"
+[ \"\$upstream\" = 1 ] && [ \"\${linger:-}\" = yes ]"
+}
+
+case "$mode" in
+  --check)
+    receipt sensor pass "grpc_gap_source_and_canonical_110_upstream_selected"
+    receipt normalize pass "asset_signoz_188_otlp_grpc_compat_bridge"
+    receipt diff pass "missing_24317_listener_candidate_add_no_endpoint_change"
+    receipt decision allow "candidate_restore_incumbent_otlp_grpc_transport"
+    receipt policy allow "risk_medium_user_systemd_raw_tcp_allowlist_no_firewall_no_secret"
+    preflight
+    receipt check pass "upstream_4317_tcp_linger_yes_unit_source_present"
+    receipt terminal pass "check_mode_no_write"
+    ;;
+  --apply)
+    receipt sensor pass "grpc_gap_source_and_canonical_110_upstream_selected"
+    receipt normalize pass "asset_signoz_188_otlp_grpc_compat_bridge"
+    receipt diff pass "missing_24317_listener_candidate_add_no_endpoint_change"
+    receipt decision allow "restore_incumbent_otlp_grpc_transport_without_route_promotion"
+    receipt policy allow "risk_medium_user_systemd_raw_tcp_allowlist_no_firewall_no_secret"
+    preflight
+    receipt check pass "upstream_4317_tcp_linger_yes_unit_source_present"
+    APPLY_ACTIVE=1
+    remote "mkdir -p /home/ollama/bin /home/ollama/run /home/ollama/log /home/ollama/.config/systemd/user; umask 077; tee '$REMOTE_SCRIPT.tmp' >/dev/null" < <(remote_payload)
+    scp "${SSH_OPTS[@]}" "$LOCAL_UNIT" "$TARGET_HOST:$REMOTE_UNIT_STAGE"
+    remote "set -eu
+chmod 0755 '$REMOTE_SCRIPT.tmp'
+chmod 0644 '$REMOTE_UNIT_STAGE'
+mv '$REMOTE_SCRIPT.tmp' '$REMOTE_SCRIPT'
+systemd-analyze --user verify '$REMOTE_UNIT_STAGE' >/dev/null
+mv '$REMOTE_UNIT_STAGE' '$REMOTE_UNIT'
+systemctl --user daemon-reload
+systemctl --user enable --now '$REMOTE_SERVICE'
+ready=0
+for _ in 1 2 3 4 5 6 7 8 9 10; do
+  if systemctl --user is-active --quiet '$REMOTE_SERVICE' && $env_prefix '$REMOTE_SCRIPT' status; then ready=1; break; fi
+  sleep 1
+done
+[ \"\$ready\" = 1 ]
+systemctl --user is-enabled --quiet '$REMOTE_SERVICE'"
+    verify_nodes
+    APPLY_ACTIVE=0
+    receipt execute pass "grpc_bridge_unit_enabled_active_allowlist_120_121"
+    receipt verify pass "nodes_120_121_listener_and_upstream_4317_tcp_ready"
+    receipt learning partial "application_trace_metric_10m_post_verifier_pending"
+    receipt terminal pass "grpc_transport_restored_post_verifier_pending"
+    ;;
+  --status)
+    remote "set -eu
+systemctl --user is-enabled --quiet '$REMOTE_SERVICE'
+systemctl --user is-active --quiet '$REMOTE_SERVICE'
+$env_prefix '$REMOTE_SCRIPT' status"
+    verify_nodes
+    receipt verify pass "grpc_unit_enabled_active_nodes_120_121_ready"
+    ;;
+  --rollback)
+    receipt decision allow "rollback_new_grpc_bridge_only_keep_http_bridge"
+    APPLY_ACTIVE=1
+    rollback_runtime
+    APPLY_ACTIVE=0
+    receipt terminal pass "grpc_bridge_rolled_back_http_bridge_unchanged"
+    ;;
+esac
--- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py
+++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py
@@ -125,6 +125,43 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
            )
        )
    )
+    grpc_bridge = _read(
+        root, "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh"
+    )
+    grpc_unit = _read(
+        root, "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service"
+    )
+    grpc_verifier = _read(
+        root, "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"
+    )
+    checks["grpc_transport_recovery_source_ready"] = all(
+        value in grpc_bridge
+        for value in (
+            "LISTEN_PORT:-24317",
+            "UPSTREAM_PORT:-4317",
+            "SOURCE_RANGE:-192.168.0.120/31",
+            "systemctl --user enable --now",
+            "--rollback",
+        )
+    ) and all(
+        value in grpc_unit
+        for value in (
+            "LISTEN_PORT=24317",
+            "UPSTREAM_PORT=4317",
+            "SOURCE_RANGE=192.168.0.120/31",
+            "Restart=on-failure",
+            "NoNewPrivileges=true",
+        )
+    ) and all(
+        value in grpc_verifier
+        for value in (
+            "WINDOW_SECONDS:-600",
+            "serviceName={service:String}",
+            "signoz_metrics.samples_v4",
+            "Failed to export traces",
+            "restart_delta_0",
+        )
+    )
    static_recovery = _read(
        root, "scripts/reboot-recovery/signoz-110-static-ingest.sh"
    )
--- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
+++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
@@ -19,6 +19,9 @@ SIGNAL_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-signals.sh"
 PARITY_SNAPSHOT = ROOT / "ops/signoz/p0-obs-002-three-signal-route-parity.yaml"
 OTEL_DAEMONSET = ROOT / "k8s/observability/otel-collector-daemonset.yaml"
 FILELOG_CANARY = ROOT / "ops/signoz/filelog-normalization-canary.yaml"
+GRPC_BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh"
+GRPC_UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service"
+GRPC_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"


 def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
@@ -187,6 +190,68 @@ def test_filelog_canary_is_synthetic_bounded_and_has_no_network_exporter() -> No
    assert container["securityContext"]["readOnlyRootFilesystem"] is True


+def test_grpc_bridge_is_independently_supervised_allowlisted_and_reversible() -> None:
+    bridge = GRPC_BRIDGE.read_text(encoding="utf-8")
+    unit = GRPC_UNIT.read_text(encoding="utf-8")
+    for expected in (
+        'LISTEN_PORT="${LISTEN_PORT:-24317}"',
+        'SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"',
+        'UPSTREAM_PORT="${UPSTREAM_PORT:-4317}"',
+        "--check|--apply|--status|--rollback",
+        "systemctl --user enable --now",
+        "rollback_runtime",
+        "verify_nodes",
+        "no_firewall_no_secret",
+    ):
+        assert expected in bridge
+    for forbidden in ("sudo", "iptables", "nft ", "docker ", "kubectl "):
+        assert forbidden not in bridge
+
+    for expected in (
+        "LISTEN_PORT=24317",
+        "SOURCE_RANGE=192.168.0.120/31",
+        "UPSTREAM_PORT=4317",
+        "ExecStartPre=/home/ollama/bin/awoooi-signoz-otlp-grpc-bridge.sh check-upstream",
+        "Restart=on-failure",
+        "NoNewPrivileges=true",
+        "ProtectSystem=strict",
+        "ProtectHome=read-only",
+    ):
+        assert expected in unit
+    assert "User=" not in unit
+
+
+def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate() -> None:
+    text = GRPC_VERIFIER.read_text(encoding="utf-8")
+    for expected in (
+        'WINDOW_SECONDS="${WINDOW_SECONDS:-600}"',
+        "https://awoooi.wooo.work/api/v1/openapi.json",
+        "serviceName={service:String}",
+        "signoz_metrics.samples_v4",
+        "resource_attrs['service.name']={service:String}",
+        "Failed to export traces",
+        "Failed to export metrics",
+        "restart_delta_0",
+        "real_application_otlp_grpc_trace_metric_freshness_closed",
+    ):
+        assert expected in text
+    assert "SELECT body" not in text
+    assert "SELECT attributes" not in text
+    assert "/v1/traces" not in text
+
+    contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
+    grpc = contract["runtime_observations"]["application_otlp_grpc"]
+    assert grpc["root_cause"] == "missing_incumbent_grpc_bridge"
+    assert grpc["confidence"] == 0.99
+    assert grpc["recent_awoooi_api_trace_count"] == 0
+    assert grpc["http_only_verifier_blind_spot"] is True
+    assert grpc["source_fix"]["check_terminal"] == "pass"
+    assert (
+        grpc["source_fix"]["runtime_apply_status"]
+        == "check_pass_pending_stable_cd_window"
+    )
+
+
 def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None:
    env = os.environ.copy()
    env.pop("MANAGED_PROMOTION_RECEIPT", None)
--- a/scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh
+++ b/scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# Real application OTLP/gRPC post-verifier. It reads aggregates only.
+
+set -Eeuo pipefail
+
+TRACE_ID="${TRACE_ID:-P0-OBS-002}"
+RUN_ID="${RUN_ID:-P0-OBS-002-grpc-runtime-$(date -u +%Y%m%dT%H%M%SZ)}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-002}"
+WINDOW_SECONDS="${WINDOW_SECONDS:-600}"
+EXPECTED_PODS="${EXPECTED_PODS:-2}"
+NAMESPACE="${NAMESPACE:-awoooi-prod}"
+POD_SELECTOR="${POD_SELECTOR:-app=awoooi-api}"
+SERVICE_NAME="${SERVICE_NAME:-awoooi-api}"
+CANARY_URL="${CANARY_URL:-https://awoooi.wooo.work/api/v1/openapi.json}"
+BRIDGE_HOST="${BRIDGE_HOST:-ollama@192.168.0.188}"
+BRIDGE_SERVICE="${BRIDGE_SERVICE:-awoooi-signoz-otlp-grpc-bridge.service}"
+SIGNOZ_HOST="${SIGNOZ_HOST:-wooo@192.168.0.110}"
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ConnectionAttempts=1)
+
+case "$WINDOW_SECONDS" in
+  ''|*[!0-9]*) echo "WINDOW_SECONDS must be an integer" >&2; exit 64 ;;
+esac
+[ "$WINDOW_SECONDS" -ge 60 ]
+
+receipt() {
+  local phase="$1" result="$2" detail="$3"
+  printf '{"schema":"awoooi_otel_grpc_runtime_verifier_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-188-otlp-grpc-compat-bridge","phase":"%s","result":"%s","detail":"%s"}\n' \
+    "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail"
+}
+
+on_error() {
+  local rc=$?
+  receipt rollback required "run_signoz_188_otlp_grpc_bridge_rollback"
+  receipt terminal failed "command_exit_$rc"
+  exit "$rc"
+}
+trap on_error ERR
+
+pods=()
+while IFS= read -r pod; do
+  [ -n "$pod" ] && pods[${#pods[@]}]="$pod"
+done < <(
+  kubectl -n "$NAMESPACE" get pods -l "$POD_SELECTOR" \
+    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'
+)
+[ "${#pods[@]}" -eq "$EXPECTED_PODS" ]
+restart_before=()
+for pod in "${pods[@]}"; do
+  ready="$(kubectl -n "$NAMESPACE" get pod "$pod" -o jsonpath='{.status.containerStatuses[0].ready}')"
+  [ "$ready" = true ]
+  restart_before[${#restart_before[@]}]="$(kubectl -n "$NAMESPACE" get pod "$pod" -o jsonpath='{.status.containerStatuses[0].restartCount}')"
+done
+ssh "${SSH_OPTS[@]}" "$BRIDGE_HOST" "systemctl --user is-enabled --quiet '$BRIDGE_SERVICE' && systemctl --user is-active --quiet '$BRIDGE_SERVICE'"
+receipt sensor pass "api_pods_${EXPECTED_PODS}_ready_grpc_bridge_enabled_active"
+receipt normalize pass "service_${SERVICE_NAME}_aggregate_only"
+receipt diff pass "incumbent_endpoint_unchanged_grpc_transport_present"
+receipt decision allow "one_non_health_synthetic_request_then_bounded_observation"
+receipt policy allow "risk_low_no_secret_no_payload_query_no_route_change"
+receipt check pass "window_${WINDOW_SECONDS}s_restart_baseline_captured"
+
+http_code="$(curl -sS --max-time 15 -o /dev/null -w '%{http_code}' "$CANARY_URL")"
+[ "$http_code" -ge 200 ] && [ "$http_code" -lt 400 ]
+receipt execute pass "openapi_canary_http_${http_code}"
+
+sleep "$WINDOW_SECONDS"
+error_count=0
+for i in "${!pods[@]}"; do
+  pod="${pods[$i]}"
+  restart_after="$(kubectl -n "$NAMESPACE" get pod "$pod" -o jsonpath='{.status.containerStatuses[0].restartCount}')"
+  [ "$restart_after" = "${restart_before[$i]}" ]
+  pod_errors="$(kubectl -n "$NAMESPACE" logs "$pod" --since="${WINDOW_SECONDS}s" 2>&1 | grep -Ec 'Failed to export traces|Failed to export metrics|Connection refused' || true)"
+  error_count=$((error_count + pod_errors))
+done
+[ "$error_count" -eq 0 ]
+
+trace_count="$(ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
+  "docker exec signoz-clickhouse clickhouse-client --param_service='$SERVICE_NAME' --param_window='$WINDOW_SECONDS' --query \"SELECT count() FROM signoz_traces.signoz_index_v3 WHERE serviceName={service:String} AND timestamp >= now() - toIntervalSecond({window:UInt32})\"")"
+metric_count="$(ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
+  "docker exec signoz-clickhouse clickhouse-client --param_service='$SERVICE_NAME' --param_window='$WINDOW_SECONDS' --query \"SELECT count() FROM signoz_metrics.samples_v4 WHERE unix_milli >= toUnixTimestamp64Milli(now64()) - ({window:UInt32} * 1000) AND fingerprint IN (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE resource_attrs['service.name']={service:String})\"")"
+[ "$trace_count" -gt 0 ]
+[ "$metric_count" -gt 0 ]
+receipt verify pass "window_${WINDOW_SECONDS}s_export_errors_0_restart_delta_0_trace_count_${trace_count}_metric_count_${metric_count}"
+receipt rollback not_required "independent_post_verifier_pass"
+receipt learning partial "durable_gitea_main_and_km_writeback_pending"
+receipt terminal pass "real_application_otlp_grpc_trace_metric_freshness_closed"