fix(observability): restore incumbent OTLP gRPC transport

2026-07-15 01:21:57 +08:00
parent b02bc98c73
commit dd46ff8bc5
7 changed files with 489 additions and 1 deletions
--- a/docs/LOGBOOK.md
+++ b/docs/LOGBOOK.md
@@ -8,15 +8,19 @@
 - 產出 `ops/signoz/p0-obs-002-three-signal-route-parity.yaml`；明確把 synthetic route parity 與真實應用 trace freshness、producer queue gate、正式 promotion 分開。
 - 最新 10 分鐘 error fingerprint 證明 normalization noise 是固定成對錯誤：`.120` 為 `cri_parser regex_no_match=156` + `move3 missing_field=156`，`.121` 為 `126 + 126`；同時兩個 collector queue / send_failed 都是 `0`。
 - 第一版 filelog candidate 的 `on_error: send_quiet` 通過 `0.96.0 validate`，但隔離 Pod 在真正 build pipeline 時明確拒絕；兩次 canary 均已刪除且未碰正式 DaemonSet。source 已移除不相容欄位，改為 parser `body matches` 前置條件、保留 malformed 原始 body、以 `awoooi.log.parse_status=fallback_unparsed` 標記，並只在 `attributes.log` 存在時 move body；修正版刻意不做第三次 runtime canary，狀態維持 canary pending。
+- 真實 trace freshness audit 找到 `0.99` 信心根因：Python SDK traces / metrics 使用 gRPC `.188:24317`，但現有相容橋接只有 HTTP `.188:24318 -> .110:4318`；API pod 對 24317 連線拒絕，ClickHouse 最近只有三筆 synthetic verifier span，`awoooi-api=0`。
+- 新增獨立 user-systemd gRPC bridge source `.188:24317 -> .110:4317`，沿用 `.120/31` source allowlist，具備 check / apply / status / rollback；另增 10 分鐘真實應用 trace + metric aggregate、export error=0、restart delta=0 post-verifier。它不改 API endpoint、不改 firewall，且可與 HTTP bridge 分開回滾。

 **已跑驗證**：
- SigNoz / OTEL / Event Exporter / API config 聚焦回歸：`43 passed`；其中 canonical route migration 為 `9 passed`。ruff、py_compile、四支 shell `bash -n`、canonical route preflight、三份 Kubernetes server dry-run、runner pressure guard、Gitea secret transport guard 與 `git diff --check` 通過。
+- SigNoz / OTEL / Event Exporter / API config 聚焦回歸：`45 passed`；其中 canonical route migration 為 `11 passed`。ruff、py_compile、六支 shell `bash -n`、canonical route preflight、三份 Kubernetes server dry-run、runner pressure guard、Gitea secret transport guard 與 `git diff --check` 通過。
+- gRPC bridge no-write check `P0-OBS-002-grpc-bridge-check-20260715T012000+0800` 通過：`.110:4317` upstream TCP ready、`.188:24317` 無既有 listener、user linger=yes、source range 維持 `.120/31`。
 - Gitea feature evidence commit `0523af6bfc9572f1822daef7277ace80d79cb1bb` 已由 SSH exact ref 回讀；這只完成 feature 層 durable acknowledgement，main / CD / production / KM writeback 仍是獨立 pending。
 - 先前 route-specific log replay 已驗證 `.120/.121` HTTP `200`、ClickHouse exact marker `1`、15 秒 queue / failure delta `0`；本次三訊號 direct verifier不把該結果誤宣稱成 producer canary。
 - bridge rollback 後 `.120/.121 -> .188:24318` 回 HTTP `200`，legacy cron 已恢復，live user-systemd unit 已移除 / disabled。

 **仍維持**：
 - `setupCompleted=false`、organization 未初始化、`.110:4317/4318` direct ingress 暴露面尚未收斂、真實應用 traces 不新鮮、filelog normalization source fix 尚未 canary、bridge 尚未由 user-systemd supervision 接管。
+- gRPC bridge 本輪先維持 source-ready / runtime-apply-pending；必須等目前 Gitea CD / API rollout 穩定 2/2 後，才可套用並跑完整 600 秒 post-verifier。HTTP-only synthetic `/v1/traces` 成功不能取代此 gRPC 證據。
 - synthetic 三訊號 parity 不等於正式 producer canary、Wave B promotion、bridge retirement、AI learning writeback 或 production closure；active route 仍是 incumbent。
 - organization/root bootstrap 涉及 durable credential；direct ingress policy 可能涉及 firewall / mTLS。這些維持獨立 high/critical gate，本輪未讀 secret、未改 firewall、未寫 raw DB、未碰 stateful volume。
 - Gitea feature branch / source 測試、main CD、deploy marker、production runtime 與 visible evidence 仍分層追蹤；不得由 feature push 或 synthetic readback 倒推 production 已部署。
--- a/ops/signoz/organization-canonical-route-migration.yaml
+++ b/ops/signoz/organization-canonical-route-migration.yaml
@@ -34,6 +34,8 @@ routes:
    grpc: 192.168.0.188:24317
    http: http://192.168.0.188:24318
    source_range: 192.168.0.120/31
+    http_bridge_status: active_legacy
+    grpc_bridge_status: source_ready_runtime_apply_pending
    expiry: "2026-08-15T23:59:59+08:00"
  challenger:
    canonical_id: signoz-110-otlp
@@ -78,6 +80,7 @@ wave_a_gates:
  challenger_application_allowlisted: true
  challenger_networkpolicy_allowed: true
  bridge_user_systemd_supervision_required: true
+  incumbent_grpc_bridge_required: true
  route_specific_log_replay_required: true
  route_specific_logs_metrics_traces_replay_required: true

@@ -96,6 +99,20 @@ runtime_observations:
  metrics_fresh: true
  traces_fresh: false
  bridge_supervisor: legacy_orphan_process
+  application_otlp_grpc:
+    endpoint: http://192.168.0.188:24317
+    root_cause: missing_incumbent_grpc_bridge
+    confidence: 0.99
+    api_pods_ready_at_audit: 2
+    recent_awoooi_api_trace_count: 0
+    http_only_verifier_blind_spot: true
+    source_fix:
+      bridge: scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh
+      unit: ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service
+      post_verifier: scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh
+      check_run_id: P0-OBS-002-grpc-bridge-check-20260715T012000+0800
+      check_terminal: pass
+      runtime_apply_status: check_pass_pending_stable_cd_window
  filelog_normalization: source_fix_canary_pending
  filelog_normalization_error_readback:
    window: 10m
--- a/ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service
+++ b/ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service
@@ -0,0 +1,31 @@
+[Unit]
+Description=AWOOOI SigNoz OTLP gRPC compatibility bridge
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=simple
+Environment=LISTEN_HOST=192.168.0.188
+Environment=LISTEN_PORT=24317
+Environment=SOURCE_RANGE=192.168.0.120/31
+Environment=UPSTREAM_HOST=192.168.0.110
+Environment=UPSTREAM_PORT=4317
+ExecStartPre=/home/ollama/bin/awoooi-signoz-otlp-grpc-bridge.sh check-upstream
+ExecStart=/home/ollama/bin/awoooi-signoz-otlp-grpc-bridge.sh run
+ExecStopPost=/usr/bin/rm -f /home/ollama/run/awoooi-signoz-otlp-grpc-bridge.pid
+Restart=on-failure
+RestartSec=5s
+TimeoutStartSec=15s
+TimeoutStopSec=15s
+UMask=0077
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=read-only
+ReadWritePaths=/home/ollama/run /home/ollama/log
+LockPersonality=true
+MemoryDenyWriteExecute=true
+RestrictAddressFamilies=AF_INET AF_INET6
+
+[Install]
+WantedBy=default.target
--- a/scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh
+++ b/scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh
@@ -0,0 +1,249 @@
+#!/usr/bin/env bash
+# Independently supervised OTLP/gRPC compatibility bridge for P0-OBS-002.
+# It is paired with, but independently rollbackable from, the HTTP bridge.
+
+set -Eeuo pipefail
+
+TARGET_HOST="${TARGET_HOST:-ollama@192.168.0.188}"
+REMOTE_SCRIPT="${REMOTE_SCRIPT:-/home/ollama/bin/awoooi-signoz-otlp-grpc-bridge.sh}"
+REMOTE_UNIT="${REMOTE_UNIT:-/home/ollama/.config/systemd/user/awoooi-signoz-otlp-grpc-bridge.service}"
+REMOTE_UNIT_STAGE="${REMOTE_UNIT_STAGE:-/home/ollama/run/awoooi-signoz-otlp-grpc-bridge.service}"
+REMOTE_SERVICE="${REMOTE_SERVICE:-awoooi-signoz-otlp-grpc-bridge.service}"
+LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}"
+LISTEN_PORT="${LISTEN_PORT:-24317}"
+SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"
+UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}"
+UPSTREAM_PORT="${UPSTREAM_PORT:-4317}"
+VERIFY_NODES="${VERIFY_NODES:-wooo@192.168.0.120 wooo@192.168.0.121}"
+TRACE_ID="${TRACE_ID:-P0-OBS-002}"
+RUN_ID="${RUN_ID:-P0-OBS-002-grpc-bridge-$(date -u +%Y%m%dT%H%M%SZ)}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-002}"
+SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}"
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+LOCAL_UNIT="${LOCAL_UNIT:-$ROOT_DIR/ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service}"
+APPLY_ACTIVE=0
+SSH_OPTS=(
+  -o BatchMode=yes
+  -o ConnectTimeout="$SSH_CONNECT_TIMEOUT"
+  -o ConnectionAttempts=1
+  -o ServerAliveInterval=5
+  -o ServerAliveCountMax=1
+)
+
+usage() {
+  cat <<'USAGE'
+Usage: signoz-188-otlp-grpc-bridge.sh [mode]
+
+  --check       No-write prerequisite and conflict check.
+  --apply       Install and start the user-systemd gRPC bridge.
+  --status      Require enabled, active, allowlisted end-to-end TCP readiness.
+  --rollback    Disable the new gRPC bridge and verify the listener is absent.
+
+This script never changes firewall, Docker, Kubernetes, database, secrets,
+the existing HTTP bridge, or the application OTLP endpoint.
+USAGE
+}
+
+mode="${1:---check}"
+case "$mode" in
+  --check|--apply|--status|--rollback) ;;
+  -h|--help) usage; exit 0 ;;
+  *) usage >&2; exit 64 ;;
+esac
+
+remote() {
+  ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "$@"
+}
+
+receipt() {
+  local phase="$1" result="$2" detail="$3"
+  printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-188-otlp-grpc-compat-bridge","phase":"%s","result":"%s","detail":"%s"}\n' \
+    "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail"
+}
+
+rollback_runtime() {
+  set +e
+  remote "systemctl --user disable --now '$REMOTE_SERVICE' >/dev/null 2>&1 || true; '$REMOTE_SCRIPT' stop >/dev/null 2>&1 || true"
+  remote "! ss -ltn 2>/dev/null | awk '{print \$4}' | grep -Eq '${LISTEN_HOST}:${LISTEN_PORT}$'"
+  local rc=$?
+  set -e
+  if [ "$rc" -eq 0 ]; then
+    receipt rollback pass "grpc_unit_disabled_listener_absent_http_bridge_unchanged"
+  else
+    receipt rollback failed "grpc_listener_still_present"
+  fi
+  return "$rc"
+}
+
+on_error() {
+  local rc=$?
+  trap - ERR
+  if [ "$APPLY_ACTIVE" -eq 1 ]; then
+    rollback_runtime || true
+  fi
+  receipt terminal failed "command_exit_$rc"
+  exit "$rc"
+}
+trap on_error ERR
+
+remote_payload() {
+  cat <<'REMOTE'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}"
+LISTEN_PORT="${LISTEN_PORT:-24317}"
+SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"
+UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}"
+UPSTREAM_PORT="${UPSTREAM_PORT:-4317}"
+RUN_DIR="${RUN_DIR:-/home/ollama/run}"
+LOG_DIR="${LOG_DIR:-/home/ollama/log}"
+PID_FILE="$RUN_DIR/awoooi-signoz-otlp-grpc-bridge.pid"
+LOG_FILE="$LOG_DIR/awoooi-signoz-otlp-grpc-bridge.log"
+
+is_running() {
+  [ -s "$PID_FILE" ] && kill -0 "$(sed -n '1p' "$PID_FILE")" 2>/dev/null
+}
+
+tcp_open() {
+  timeout 3 bash -c "exec 3<>/dev/tcp/$1/$2" >/dev/null 2>&1
+}
+
+listener_present() {
+  ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "${LISTEN_HOST}:${LISTEN_PORT}$"
+}
+
+status() {
+  running=0
+  listener=0
+  upstream=0
+  if is_running; then running=1; fi
+  if listener_present; then listener=1; fi
+  if tcp_open "$UPSTREAM_HOST" "$UPSTREAM_PORT"; then upstream=1; fi
+  printf 'SIGNOZ_OTLP_GRPC_BRIDGE_STATUS running=%s listen=%s:%s listen_socket=%s source_range=%s upstream=%s:%s upstream_tcp=%s\n' \
+    "$running" "$LISTEN_HOST" "$LISTEN_PORT" "$listener" "$SOURCE_RANGE" "$UPSTREAM_HOST" "$UPSTREAM_PORT" "$upstream"
+  [ "$running" -eq 1 ] && [ "$listener" -eq 1 ] && [ "$upstream" -eq 1 ]
+}
+
+run_foreground() {
+  command -v socat >/dev/null
+  mkdir -p "$RUN_DIR" "$LOG_DIR"
+  if listener_present; then
+    echo "SIGNOZ_OTLP_GRPC_BRIDGE_BLOCKED port_busy=$LISTEN_PORT" >&2
+    exit 2
+  fi
+  echo "$$" >"$PID_FILE"
+  exec socat "TCP-LISTEN:${LISTEN_PORT},bind=${LISTEN_HOST},reuseaddr,fork,range=${SOURCE_RANGE}" \
+    "TCP:${UPSTREAM_HOST}:${UPSTREAM_PORT}" >>"$LOG_FILE" 2>&1
+}
+
+stop() {
+  if is_running; then
+    kill "$(sed -n '1p' "$PID_FILE")"
+    for _ in 1 2 3 4 5; do
+      is_running || break
+      sleep 1
+    done
+  fi
+  rm -f "$PID_FILE"
+  echo "SIGNOZ_OTLP_GRPC_BRIDGE_STOPPED"
+}
+
+case "${1:-status}" in
+  check-upstream) tcp_open "$UPSTREAM_HOST" "$UPSTREAM_PORT" ;;
+  run) run_foreground ;;
+  status) status ;;
+  stop) stop ;;
+  *) exit 64 ;;
+esac
+REMOTE
+}
+
+env_prefix="LISTEN_HOST=$LISTEN_HOST LISTEN_PORT=$LISTEN_PORT SOURCE_RANGE=$SOURCE_RANGE UPSTREAM_HOST=$UPSTREAM_HOST UPSTREAM_PORT=$UPSTREAM_PORT"
+
+verify_nodes() {
+  local node rc
+  for node in $VERIFY_NODES; do
+    rc=0
+    ssh "${SSH_OPTS[@]}" "$node" \
+      "timeout 5 bash -c 'exec 3<>/dev/tcp/${LISTEN_HOST}/${LISTEN_PORT}'" >/dev/null 2>&1 || rc=$?
+    printf 'SIGNOZ_OTLP_GRPC_BRIDGE_NODE_VERIFY node=%s tcp_rc=%s\n' "$node" "$rc"
+    [ "$rc" -eq 0 ]
+  done
+}
+
+preflight() {
+  test -s "$LOCAL_UNIT"
+  remote "set -eu
+command -v bash >/dev/null
+command -v socat >/dev/null
+command -v systemctl >/dev/null
+linger=\$(loginctl show-user \"\$USER\" -p Linger --value 2>/dev/null || true)
+upstream=0
+if timeout 3 bash -c 'exec 3<>/dev/tcp/${UPSTREAM_HOST}/${UPSTREAM_PORT}' >/dev/null 2>&1; then upstream=1; fi
+busy=0
+if ss -ltn 2>/dev/null | awk '{print \$4}' | grep -Eq '${LISTEN_HOST}:${LISTEN_PORT}$'; then busy=1; fi
+printf 'SIGNOZ_OTLP_GRPC_BRIDGE_CHECK upstream_tcp=%s listen_port_busy=%s source_range=%s linger=%s\n' \"\$upstream\" \"\$busy\" '$SOURCE_RANGE' \"\${linger:-no}\"
+[ \"\$upstream\" = 1 ] && [ \"\${linger:-}\" = yes ]"
+}
+
+case "$mode" in
+  --check)
+    receipt sensor pass "grpc_gap_source_and_canonical_110_upstream_selected"
+    receipt normalize pass "asset_signoz_188_otlp_grpc_compat_bridge"
+    receipt diff pass "missing_24317_listener_candidate_add_no_endpoint_change"
+    receipt decision allow "candidate_restore_incumbent_otlp_grpc_transport"
+    receipt policy allow "risk_medium_user_systemd_raw_tcp_allowlist_no_firewall_no_secret"
+    preflight
+    receipt check pass "upstream_4317_tcp_linger_yes_unit_source_present"
+    receipt terminal pass "check_mode_no_write"
+    ;;
+  --apply)
+    receipt sensor pass "grpc_gap_source_and_canonical_110_upstream_selected"
+    receipt normalize pass "asset_signoz_188_otlp_grpc_compat_bridge"
+    receipt diff pass "missing_24317_listener_candidate_add_no_endpoint_change"
+    receipt decision allow "restore_incumbent_otlp_grpc_transport_without_route_promotion"
+    receipt policy allow "risk_medium_user_systemd_raw_tcp_allowlist_no_firewall_no_secret"
+    preflight
+    receipt check pass "upstream_4317_tcp_linger_yes_unit_source_present"
+    APPLY_ACTIVE=1
+    remote "mkdir -p /home/ollama/bin /home/ollama/run /home/ollama/log /home/ollama/.config/systemd/user; umask 077; tee '$REMOTE_SCRIPT.tmp' >/dev/null" < <(remote_payload)
+    scp "${SSH_OPTS[@]}" "$LOCAL_UNIT" "$TARGET_HOST:$REMOTE_UNIT_STAGE"
+    remote "set -eu
+chmod 0755 '$REMOTE_SCRIPT.tmp'
+chmod 0644 '$REMOTE_UNIT_STAGE'
+mv '$REMOTE_SCRIPT.tmp' '$REMOTE_SCRIPT'
+systemd-analyze --user verify '$REMOTE_UNIT_STAGE' >/dev/null
+mv '$REMOTE_UNIT_STAGE' '$REMOTE_UNIT'
+systemctl --user daemon-reload
+systemctl --user enable --now '$REMOTE_SERVICE'
+ready=0
+for _ in 1 2 3 4 5 6 7 8 9 10; do
+  if systemctl --user is-active --quiet '$REMOTE_SERVICE' && $env_prefix '$REMOTE_SCRIPT' status; then ready=1; break; fi
+  sleep 1
+done
+[ \"\$ready\" = 1 ]
+systemctl --user is-enabled --quiet '$REMOTE_SERVICE'"
+    verify_nodes
+    APPLY_ACTIVE=0
+    receipt execute pass "grpc_bridge_unit_enabled_active_allowlist_120_121"
+    receipt verify pass "nodes_120_121_listener_and_upstream_4317_tcp_ready"
+    receipt learning partial "application_trace_metric_10m_post_verifier_pending"
+    receipt terminal pass "grpc_transport_restored_post_verifier_pending"
+    ;;
+  --status)
+    remote "set -eu
+systemctl --user is-enabled --quiet '$REMOTE_SERVICE'
+systemctl --user is-active --quiet '$REMOTE_SERVICE'
+$env_prefix '$REMOTE_SCRIPT' status"
+    verify_nodes
+    receipt verify pass "grpc_unit_enabled_active_nodes_120_121_ready"
+    ;;
+  --rollback)
+    receipt decision allow "rollback_new_grpc_bridge_only_keep_http_bridge"
+    APPLY_ACTIVE=1
+    rollback_runtime
+    APPLY_ACTIVE=0
+    receipt terminal pass "grpc_bridge_rolled_back_http_bridge_unchanged"
+    ;;
+esac
--- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py
+++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py
@@ -125,6 +125,43 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
            )
        )
    )
+    grpc_bridge = _read(
+        root, "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh"
+    )
+    grpc_unit = _read(
+        root, "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service"
+    )
+    grpc_verifier = _read(
+        root, "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"
+    )
+    checks["grpc_transport_recovery_source_ready"] = all(
+        value in grpc_bridge
+        for value in (
+            "LISTEN_PORT:-24317",
+            "UPSTREAM_PORT:-4317",
+            "SOURCE_RANGE:-192.168.0.120/31",
+            "systemctl --user enable --now",
+            "--rollback",
+        )
+    ) and all(
+        value in grpc_unit
+        for value in (
+            "LISTEN_PORT=24317",
+            "UPSTREAM_PORT=4317",
+            "SOURCE_RANGE=192.168.0.120/31",
+            "Restart=on-failure",
+            "NoNewPrivileges=true",
+        )
+    ) and all(
+        value in grpc_verifier
+        for value in (
+            "WINDOW_SECONDS:-600",
+            "serviceName={service:String}",
+            "signoz_metrics.samples_v4",
+            "Failed to export traces",
+            "restart_delta_0",
+        )
+    )
    static_recovery = _read(
        root, "scripts/reboot-recovery/signoz-110-static-ingest.sh"
    )
--- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
+++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
@@ -19,6 +19,9 @@ SIGNAL_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-signals.sh"
 PARITY_SNAPSHOT = ROOT / "ops/signoz/p0-obs-002-three-signal-route-parity.yaml"
 OTEL_DAEMONSET = ROOT / "k8s/observability/otel-collector-daemonset.yaml"
 FILELOG_CANARY = ROOT / "ops/signoz/filelog-normalization-canary.yaml"
+GRPC_BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh"
+GRPC_UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service"
+GRPC_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"


 def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
@@ -187,6 +190,68 @@ def test_filelog_canary_is_synthetic_bounded_and_has_no_network_exporter() -> No
    assert container["securityContext"]["readOnlyRootFilesystem"] is True


+def test_grpc_bridge_is_independently_supervised_allowlisted_and_reversible() -> None:
+    bridge = GRPC_BRIDGE.read_text(encoding="utf-8")
+    unit = GRPC_UNIT.read_text(encoding="utf-8")
+    for expected in (
+        'LISTEN_PORT="${LISTEN_PORT:-24317}"',
+        'SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"',
+        'UPSTREAM_PORT="${UPSTREAM_PORT:-4317}"',
+        "--check|--apply|--status|--rollback",
+        "systemctl --user enable --now",
+        "rollback_runtime",
+        "verify_nodes",
+        "no_firewall_no_secret",
+    ):
+        assert expected in bridge
+    for forbidden in ("sudo", "iptables", "nft ", "docker ", "kubectl "):
+        assert forbidden not in bridge
+
+    for expected in (
+        "LISTEN_PORT=24317",
+        "SOURCE_RANGE=192.168.0.120/31",
+        "UPSTREAM_PORT=4317",
+        "ExecStartPre=/home/ollama/bin/awoooi-signoz-otlp-grpc-bridge.sh check-upstream",
+        "Restart=on-failure",
+        "NoNewPrivileges=true",
+        "ProtectSystem=strict",
+        "ProtectHome=read-only",
+    ):
+        assert expected in unit
+    assert "User=" not in unit
+
+
+def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate() -> None:
+    text = GRPC_VERIFIER.read_text(encoding="utf-8")
+    for expected in (
+        'WINDOW_SECONDS="${WINDOW_SECONDS:-600}"',
+        "https://awoooi.wooo.work/api/v1/openapi.json",
+        "serviceName={service:String}",
+        "signoz_metrics.samples_v4",
+        "resource_attrs['service.name']={service:String}",
+        "Failed to export traces",
+        "Failed to export metrics",
+        "restart_delta_0",
+        "real_application_otlp_grpc_trace_metric_freshness_closed",
+    ):
+        assert expected in text
+    assert "SELECT body" not in text
+    assert "SELECT attributes" not in text
+    assert "/v1/traces" not in text
+
+    contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
+    grpc = contract["runtime_observations"]["application_otlp_grpc"]
+    assert grpc["root_cause"] == "missing_incumbent_grpc_bridge"
+    assert grpc["confidence"] == 0.99
+    assert grpc["recent_awoooi_api_trace_count"] == 0
+    assert grpc["http_only_verifier_blind_spot"] is True
+    assert grpc["source_fix"]["check_terminal"] == "pass"
+    assert (
+        grpc["source_fix"]["runtime_apply_status"]
+        == "check_pass_pending_stable_cd_window"
+    )
+
+
 def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None:
    env = os.environ.copy()
    env.pop("MANAGED_PROMOTION_RECEIPT", None)
--- a/scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh
+++ b/scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# Real application OTLP/gRPC post-verifier. It reads aggregates only.
+
+set -Eeuo pipefail
+
+TRACE_ID="${TRACE_ID:-P0-OBS-002}"
+RUN_ID="${RUN_ID:-P0-OBS-002-grpc-runtime-$(date -u +%Y%m%dT%H%M%SZ)}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-002}"
+WINDOW_SECONDS="${WINDOW_SECONDS:-600}"
+EXPECTED_PODS="${EXPECTED_PODS:-2}"
+NAMESPACE="${NAMESPACE:-awoooi-prod}"
+POD_SELECTOR="${POD_SELECTOR:-app=awoooi-api}"
+SERVICE_NAME="${SERVICE_NAME:-awoooi-api}"
+CANARY_URL="${CANARY_URL:-https://awoooi.wooo.work/api/v1/openapi.json}"
+BRIDGE_HOST="${BRIDGE_HOST:-ollama@192.168.0.188}"
+BRIDGE_SERVICE="${BRIDGE_SERVICE:-awoooi-signoz-otlp-grpc-bridge.service}"
+SIGNOZ_HOST="${SIGNOZ_HOST:-wooo@192.168.0.110}"
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ConnectionAttempts=1)
+
+case "$WINDOW_SECONDS" in
+  ''|*[!0-9]*) echo "WINDOW_SECONDS must be an integer" >&2; exit 64 ;;
+esac
+[ "$WINDOW_SECONDS" -ge 60 ]
+
+receipt() {
+  local phase="$1" result="$2" detail="$3"
+  printf '{"schema":"awoooi_otel_grpc_runtime_verifier_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-188-otlp-grpc-compat-bridge","phase":"%s","result":"%s","detail":"%s"}\n' \
+    "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail"
+}
+
+on_error() {
+  local rc=$?
+  receipt rollback required "run_signoz_188_otlp_grpc_bridge_rollback"
+  receipt terminal failed "command_exit_$rc"
+  exit "$rc"
+}
+trap on_error ERR
+
+pods=()
+while IFS= read -r pod; do
+  [ -n "$pod" ] && pods[${#pods[@]}]="$pod"
+done < <(
+  kubectl -n "$NAMESPACE" get pods -l "$POD_SELECTOR" \
+    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'
+)
+[ "${#pods[@]}" -eq "$EXPECTED_PODS" ]
+restart_before=()
+for pod in "${pods[@]}"; do
+  ready="$(kubectl -n "$NAMESPACE" get pod "$pod" -o jsonpath='{.status.containerStatuses[0].ready}')"
+  [ "$ready" = true ]
+  restart_before[${#restart_before[@]}]="$(kubectl -n "$NAMESPACE" get pod "$pod" -o jsonpath='{.status.containerStatuses[0].restartCount}')"
+done
+ssh "${SSH_OPTS[@]}" "$BRIDGE_HOST" "systemctl --user is-enabled --quiet '$BRIDGE_SERVICE' && systemctl --user is-active --quiet '$BRIDGE_SERVICE'"
+receipt sensor pass "api_pods_${EXPECTED_PODS}_ready_grpc_bridge_enabled_active"
+receipt normalize pass "service_${SERVICE_NAME}_aggregate_only"
+receipt diff pass "incumbent_endpoint_unchanged_grpc_transport_present"
+receipt decision allow "one_non_health_synthetic_request_then_bounded_observation"
+receipt policy allow "risk_low_no_secret_no_payload_query_no_route_change"
+receipt check pass "window_${WINDOW_SECONDS}s_restart_baseline_captured"
+
+http_code="$(curl -sS --max-time 15 -o /dev/null -w '%{http_code}' "$CANARY_URL")"
+[ "$http_code" -ge 200 ] && [ "$http_code" -lt 400 ]
+receipt execute pass "openapi_canary_http_${http_code}"
+
+sleep "$WINDOW_SECONDS"
+error_count=0
+for i in "${!pods[@]}"; do
+  pod="${pods[$i]}"
+  restart_after="$(kubectl -n "$NAMESPACE" get pod "$pod" -o jsonpath='{.status.containerStatuses[0].restartCount}')"
+  [ "$restart_after" = "${restart_before[$i]}" ]
+  pod_errors="$(kubectl -n "$NAMESPACE" logs "$pod" --since="${WINDOW_SECONDS}s" 2>&1 | grep -Ec 'Failed to export traces|Failed to export metrics|Connection refused' || true)"
+  error_count=$((error_count + pod_errors))
+done
+[ "$error_count" -eq 0 ]
+
+trace_count="$(ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
+  "docker exec signoz-clickhouse clickhouse-client --param_service='$SERVICE_NAME' --param_window='$WINDOW_SECONDS' --query \"SELECT count() FROM signoz_traces.signoz_index_v3 WHERE serviceName={service:String} AND timestamp >= now() - toIntervalSecond({window:UInt32})\"")"
+metric_count="$(ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
+  "docker exec signoz-clickhouse clickhouse-client --param_service='$SERVICE_NAME' --param_window='$WINDOW_SECONDS' --query \"SELECT count() FROM signoz_metrics.samples_v4 WHERE unix_milli >= toUnixTimestamp64Milli(now64()) - ({window:UInt32} * 1000) AND fingerprint IN (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE resource_attrs['service.name']={service:String})\"")"
+[ "$trace_count" -gt 0 ]
+[ "$metric_count" -gt 0 ]
+receipt verify pass "window_${WINDOW_SECONDS}s_export_errors_0_restart_delta_0_trace_count_${trace_count}_metric_count_${metric_count}"
+receipt rollback not_required "independent_post_verifier_pass"
+receipt learning partial "durable_gitea_main_and_km_writeback_pending"
+receipt terminal pass "real_application_otlp_grpc_trace_metric_freshness_closed"