fix(observability): restore durable SigNoz ingestion

2026-07-14 19:42:38 +08:00
parent 378d867e9c
commit 582388e701
10 changed files with 578 additions and 4 deletions
--- a/docs/LOGBOOK.md
+++ b/docs/LOGBOOK.md
@@ -1,3 +1,27 @@
+## 2026-07-14 — P0-OBS-001 SigNoz OTLP 收集與持久佇列修復
+
+**完成內容**：
+- 確認正式 SigNoz 已移至 `.110`；根因為 `setupCompleted=false` 且 OpAMP 因缺少 `orgId` 將 logs / metrics / traces pipeline 改為 `nop`，並非 `.188` 主機效能問題。
+- 以 `P0-OBS-001-20260714T191800+0800` 進行受控修復：`.110` 只重建 stateless `signoz-otel-collector`，改用已驗證的 static config；SigNoz UI、ClickHouse、Zookeeper 的 container ID 與啟動時間未變。
+- 新增 `.188:24318 -> .110:4318` 暫時相容橋接，僅允許 `.120/.121` 來源，具備 check / apply / status / rollback 與 `2026-08-15` 到期日。
+- K3s OTEL DaemonSet 新增 `file_storage`、filelog offset 持久化、exporter WAL、`queue_size=10000`、無限時 retry、60 秒 termination grace 與逐節點 rolling update。
+- 將現行 `0.96.0` image 由 K3s cache 直接匯入 Harbor，未對外下載；production 已 pin 至 `192.168.0.110:5000/awoooi/otel-collector-contrib@sha256:8edf386c...`。
+- 新增獨立 ingestion verifier，以同一 run 綁定 K3s queue、`.188` bridge、`.110` listener 與 ClickHouse exact-marker 讀回。
+
+**已跑驗證**：
+- 精準 pytest：`7 passed`；三支 shell 的 `bash -n` 與 `git diff --check` 通過。
+- `kubectl apply --dry-run=server`、exact live digest `file_storage` canary、Harbor digest pull canary 全數通過；canary 顯示 `Extension started: file_storage`、`Initializing new persistent queue`、`Everything is ready`。
+- DaemonSet generation `4 -> 5` 完成逐節點 rollout；兩個 collector 均 `Ready`、`restarts=0`、queue capacity `10000`、queue size `0`。
+- Synthetic OTLP 從 `.120 -> .188:24318 -> .110:4318 -> signoz_logs.logs_v2` 回讀 exact count `1`；verifier terminal 為 `queue_zero_failure_deltas_zero_window_15s`，兩個 production collector 後續連續超過 11 分鐘 exporter error `0`、queue `0`、send-failed `0`。
+
+**仍維持**：
+- `setupCompleted=false` 與 SigNoz organization 未初始化仍是治理缺口；static override 與 `.188` 橋接是可回滾、有到期日的暫時控制，不是 Foundry 遷移完成。
+- 本筆 LOGBOOK 建立時，Gitea normal push、CD terminal 與 main-to-runtime parity 尚待後續證據；不得由 live direct apply 倒推 source 已閉環。
+- 未讀取 secret、token、`.env`、raw sessions、SQLite 或 auth；未使用 GitHub / `gh`；未重啟主機、ClickHouse、Zookeeper、SigNoz UI，也未執行 restore、prune、force push 或防火牆變更。
+
+**下一步**：
+- 將本輪 source 以 normal push 送入 Gitea main，分離追蹤 CD / deploy marker / production readback；在 bridge 到期前將 producer 與 NetworkPolicy 直接遷至 `.110:4318`，再以 replay / shadow / canary 遷移 SigNoz Foundry。
+
 ## 2026-07-09 — P0-006 AI log triage controlled auto-recovery package

 **完成內容**：
--- a/docs/adr/ADR-053-observability-signoz-unified-architecture.md
+++ b/docs/adr/ADR-053-observability-signoz-unified-architecture.md
@@ -7,6 +7,9 @@
 | **決策者** | 統帥 (ogt) |
 | **首席架構師** | Claude Code |
 | **相關 ADR** | ADR-035 (Telegram Alert), ADR-037 (監控增強), ADR-039 (Gitea CI/CD) |
+| **2026-07-14 執行期修正** | `superseded_by=global_product_governance_v2`；`.188` SigNoz 主機配置已取代 |
+| **負責範圍 / 到期日** | `platform-observability` / 暫時 `.188:24318` 橋接於 `2026-08-15 Asia/Taipei` 到期 |
+| **替代方案 / 驗證器** | 正式後端 `.110:4318`；`scripts/reboot-recovery/verify-signoz-otel-ingestion.sh` |

 ---

@@ -39,7 +42,8 @@
 ```
 所有 Log/Event 來源
  → OTEL Collector (DaemonSet, K3s 節點)
-  → 192.168.0.188:24318 (SigNoz OTEL Collector HTTP)
+  → 192.168.0.188:24318 (暫時性來源白名單相容橋接)
+  → 192.168.0.110:4318 (正式 SigNoz OTEL Collector HTTP)
  → ClickHouse (統一儲存)
  → SigNoz UI (統一查詢)

@@ -50,6 +54,9 @@ SigNoz OTEL Collector (Prometheus Receiver)

 注意: 原計畫 Prometheus remote_write 因 SigNoz 不支援 Protobuf 格式而廢棄
      改用 OTEL Collector 內建 prometheus receiver 直接 scrape 關鍵指標
+
+2026-07-14: `.188` 不再承載 SigNoz stateful runtime。相容 bridge 只允許 K3s
+`.120/.121` 來源，並必須在到期日前將所有產生端與 NetworkPolicy 改為直接 `.110`。
 ```

 ---
--- a/k8s/observability/otel-collector-daemonset.yaml
+++ b/k8s/observability/otel-collector-daemonset.yaml
@@ -55,6 +55,12 @@ metadata:
  namespace: observability
 data:
  config.yaml: |
+    extensions:
+      # P0-OBS-001: persist file offsets and exporter requests across Pod restarts.
+      # Per-node WAL is intentionally not backed up; drain it before cleanup/rollback.
+      file_storage:
+        directory: /var/lib/otelcol
+
    receivers:
      # 讀取 K3s 節點上所有 Pod 的 stdout/stderr log
      filelog:
@@ -64,6 +70,7 @@ data:
          # 排除 OTEL Collector 自身 (防遞迴)
          - /var/log/pods/observability_otel-collector*/**/*.log
        start_at: end
+        storage: file_storage
        include_file_path: true
        include_file_name: false
        operators:
@@ -142,12 +149,16 @@ data:
          enabled: true
          initial_interval: 5s
          max_interval: 30s
+          # Never discard a request merely because the backend outage was long.
+          max_elapsed_time: 0s
        sending_queue:
          enabled: true
          num_consumers: 2
-          queue_size: 100
+          queue_size: 10000
+          storage: file_storage

    service:
+      extensions: [file_storage]
      pipelines:
        logs:
          receivers: [filelog]
@@ -164,6 +175,10 @@ metadata:
    app.kubernetes.io/component: log-collector
    phase: o-2
 spec:
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: otel-collector
@@ -174,6 +189,7 @@ spec:
        app.kubernetes.io/component: log-collector
    spec:
      serviceAccountName: otel-collector
+      terminationGracePeriodSeconds: 60
      # observability namespace 設為 privileged PSA
      # 原因: /var/log/pods 是 root:root 0750，filelog receiver 必須 root 讀取
      securityContext:
@@ -181,7 +197,10 @@ spec:
        runAsGroup: 0
      containers:
        - name: otel-collector
-          image: otel/opentelemetry-collector-contrib:0.96.0
+          # P0-OBS-001: exact live 0.96.0 image mirrored from K3s cache to Harbor.
+          # Do not upgrade in the durability patch; use a separate replay/shadow canary.
+          image: 192.168.0.110:5000/awoooi/otel-collector-contrib@sha256:8edf386cdd1f4dc4393e34680e298dcc577296279febf243cf122155fa560c39
+          imagePullPolicy: IfNotPresent
          args:
            - --config=/etc/otel/config.yaml
          env:
@@ -208,6 +227,8 @@ spec:
            # OTEL Collector 需要 tmp 來寫 checkpoint
            - name: tmp
              mountPath: /tmp
+            - name: otel-storage
+              mountPath: /var/lib/otelcol
      volumes:
        - name: config
          configMap:
@@ -219,6 +240,10 @@ spec:
        - name: tmp
          emptyDir:
            sizeLimit: 50Mi
+        - name: otel-storage
+          hostPath:
+            path: /var/lib/awoooi/otelcol
+            type: DirectoryOrCreate
      tolerations:
        # 確保在所有節點上都運行 (包括 control-plane)
        - key: node-role.kubernetes.io/control-plane
--- a/ops/signoz/docker-compose.otlp-static.override.yaml
+++ b/ops/signoz/docker-compose.otlp-static.override.yaml
@@ -0,0 +1,13 @@
+# P0-OBS-001 controlled workaround for SigNoz v0.113.0 setupCompleted=false.
+# Risk: medium. Scope: recreate only the stateless otel-collector container.
+# Why: OpAMP cannot attach an orgId and replaces all signal pipelines with nop.
+# Replacement: initialize the SigNoz organization and migrate the stack to Foundry.
+# Owner: platform-observability. Expiry: 2026-08-15 Asia/Taipei.
+# Rollback: run the base compose file without this override.
+services:
+  otel-collector:
+    entrypoint:
+      - /bin/sh
+    command:
+      - -c
+      - exec /signoz-otel-collector --config=/etc/otel-collector-config.yaml
--- a/ops/signoz/otel-collector-config-phase-o.yaml
+++ b/ops/signoz/otel-collector-config-phase-o.yaml
@@ -3,7 +3,10 @@
 # =============================================================================
 # 建立者: Claude Code (首席架構師)
 # 日期: 2026-04-02 (台北時間)
-# 部署位置: 192.168.0.188:/home/ollama/signoz/deploy/docker/otel-collector-config.yaml
+# superseded_by=global_product_governance_v2
+# Legacy .188 deployment is retired; do not deploy this file to .188.
+# Owner: platform-observability. Replacement: formal .110 static ingestion override.
+# Expiry/verifier: 2026-08-15; scripts/reboot-recovery/verify-signoz-otel-ingestion.sh
 #
 # Phase O-3 新增內容 (與原版差異):
 #   prometheus receiver 新增 scrape jobs:
--- a/scripts/reboot-recovery/signoz-110-static-ingest.sh
+++ b/scripts/reboot-recovery/signoz-110-static-ingest.sh
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+# Controlled static-ingestion fallback for the formal SigNoz runtime on host 110.
+# It never reads .env, container environment values, secrets, or stateful volumes.
+
+set -euo pipefail
+
+TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
+REMOTE_DIR="${REMOTE_DIR:-/home/wooo/signoz/deploy/docker}"
+REMOTE_BASE="${REMOTE_BASE:-$REMOTE_DIR/docker-compose.yaml}"
+REMOTE_OVERRIDE="${REMOTE_OVERRIDE:-$REMOTE_DIR/docker-compose.awoooi-static-ingest.yaml}"
+REMOTE_PROJECT="${REMOTE_PROJECT:-signoz}"
+SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}"
+TRACE_ID="${TRACE_ID:-P0-OBS-001}"
+RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}"
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+LOCAL_OVERRIDE="${LOCAL_OVERRIDE:-$ROOT_DIR/ops/signoz/docker-compose.otlp-static.override.yaml}"
+SSH_OPTS=(
+  -o BatchMode=yes
+  -o ConnectTimeout="$SSH_CONNECT_TIMEOUT"
+  -o ConnectionAttempts=1
+  -o ServerAliveInterval=5
+  -o ServerAliveCountMax=1
+)
+
+usage() {
+  cat <<'USAGE'
+Usage: signoz-110-static-ingest.sh [--check|--apply|--status|--rollback]
+
+  --check      Validate the base stack and source-controlled override.
+  --apply      Recreate only signoz-otel-collector with the static base config.
+  --status     Verify the collector and OTLP listeners.
+  --rollback   Recreate only the collector from the original OpAMP command.
+
+All compose calls use --env-file /dev/null. Stateful services stay untouched.
+USAGE
+}
+
+mode="${1:---check}"
+case "$mode" in
+  --check|--apply|--status|--rollback) ;;
+  -h|--help) usage; exit 0 ;;
+  *) usage >&2; exit 64 ;;
+esac
+
+remote() {
+  ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "$@"
+}
+
+receipt() {
+  local phase="$1" result="$2" detail="$3"
+  printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-110-otel-ingest","phase":"%s","result":"%s","detail":"%s"}\n' \
+    "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail"
+}
+
+on_error() {
+  local rc=$?
+  receipt terminal failed "command_exit_$rc"
+  exit "$rc"
+}
+trap on_error ERR
+
+status_remote() {
+  remote "set -eu
+container=signoz-otel-collector
+running=\$(docker inspect -f '{{.State.Running}}' \"\$container\")
+restart_count=\$(docker inspect -f '{{.RestartCount}}' \"\$container\")
+cmd=\$(docker inspect -f '{{json .Config.Cmd}}' \"\$container\")
+http_code=\$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[]}' http://127.0.0.1:4318/v1/logs || true)
+grpc=0
+http=0
+if docker exec \"\$container\" sh -c \"grep -q ':10DD .* 0A ' /proc/net/tcp /proc/net/tcp6\"; then grpc=1; fi
+if docker exec \"\$container\" sh -c \"grep -q ':10DE .* 0A ' /proc/net/tcp /proc/net/tcp6\"; then http=1; fi
+printf 'SIGNOZ_STATIC_INGEST_STATUS running=%s restart_count=%s grpc_4317=%s http_4318=%s otlp_http_code=%s command=%s\\n' \"\$running\" \"\$restart_count\" \"\$grpc\" \"\$http\" \"\${http_code:-000}\" \"\$cmd\"
+[ \"\$running\" = true ] && [ \"\$grpc\" = 1 ] && [ \"\$http\" = 1 ] && [ \"\${http_code:-000}\" = 200 ]"
+}
+
+case "$mode" in
+  --check)
+    test -s "$LOCAL_OVERRIDE"
+    receipt sensor pass "source_override_present"
+    remote "set -eu
+test -s '$REMOTE_BASE'
+docker inspect signoz >/dev/null
+docker inspect signoz-clickhouse >/dev/null
+docker inspect signoz-zookeeper-1 >/dev/null
+docker inspect signoz-otel-collector >/dev/null
+cd '$REMOTE_DIR'
+docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' config --quiet
+printf 'SIGNOZ_STATIC_INGEST_CHECK base_compose=valid env_file=/dev/null collector=%s\\n' \"\$(docker inspect -f '{{.State.Status}}' signoz-otel-collector)\""
+    receipt check pass "base_compose_valid_no_env"
+    ;;
+  --apply)
+    test -s "$LOCAL_OVERRIDE"
+    receipt decision allow "risk_medium_stateless_collector_only"
+    scp "${SSH_OPTS[@]}" "$LOCAL_OVERRIDE" "$TARGET_HOST:$REMOTE_OVERRIDE.tmp"
+    remote "set -eu
+chmod 0644 '$REMOTE_OVERRIDE.tmp'
+mv '$REMOTE_OVERRIDE.tmp' '$REMOTE_OVERRIDE'
+cd '$REMOTE_DIR'
+docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' -f '$REMOTE_OVERRIDE' config --quiet
+printf 'SIGNOZ_STATIC_INGEST_DRY_RUN result=pass env_file=/dev/null override=%s\\n' '$REMOTE_OVERRIDE'"
+    receipt check pass "compose_override_valid"
+    remote "set -eu
+cd '$REMOTE_DIR'
+before_id=\$(docker inspect -f '{{.Id}}' signoz-otel-collector)
+docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' -f '$REMOTE_OVERRIDE' up -d --no-deps --pull never otel-collector
+after_id=\$(docker inspect -f '{{.Id}}' signoz-otel-collector)
+printf 'SIGNOZ_STATIC_INGEST_APPLY before_id=%s after_id=%s stateful_services_touched=0\\n' \"\$before_id\" \"\$after_id\""
+    receipt execute pass "collector_recreated_stateful_services_untouched"
+    status_remote
+    receipt verify pass "otlp_4317_4318_listening_empty_request_200"
+    ;;
+  --status)
+    status_remote
+    receipt verify pass "runtime_listener_check"
+    ;;
+  --rollback)
+    receipt decision allow "bounded_rollback_collector_only"
+    remote "set -eu
+cd '$REMOTE_DIR'
+docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' up -d --no-deps --pull never otel-collector
+cmd=\$(docker inspect -f '{{json .Config.Cmd}}' signoz-otel-collector)
+printf 'SIGNOZ_STATIC_INGEST_ROLLBACK command=%s\\n' \"\$cmd\"
+printf '%s' \"\$cmd\" | grep -q -- '--manager-config'"
+    receipt rollback pass "opamp_command_restored"
+    ;;
+esac
--- a/scripts/reboot-recovery/signoz-188-otlp-bridge.sh
+++ b/scripts/reboot-recovery/signoz-188-otlp-bridge.sh
@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+# Temporary, source-controlled OTLP compatibility bridge for K3s collectors.
+# Owner: platform-observability. Expiry: 2026-08-15 Asia/Taipei.
+# Replacement: repoint all producers and policies directly to host 110 OTLP.
+
+set -euo pipefail
+
+TARGET_HOST="${TARGET_HOST:-ollama@192.168.0.188}"
+REMOTE_SCRIPT="${REMOTE_SCRIPT:-/home/ollama/bin/awoooi-signoz-otlp-bridge.sh}"
+LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}"
+LISTEN_PORT="${LISTEN_PORT:-24318}"
+SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"
+UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}"
+UPSTREAM_PORT="${UPSTREAM_PORT:-4318}"
+TRACE_ID="${TRACE_ID:-P0-OBS-001}"
+RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}"
+VERIFY_NODES="${VERIFY_NODES:-wooo@192.168.0.120 wooo@192.168.0.121}"
+SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}"
+SSH_OPTS=(
+  -o BatchMode=yes
+  -o ConnectTimeout="$SSH_CONNECT_TIMEOUT"
+  -o ConnectionAttempts=1
+  -o ServerAliveInterval=5
+  -o ServerAliveCountMax=1
+)
+
+usage() {
+  cat <<'USAGE'
+Usage: signoz-188-otlp-bridge.sh [--check|--apply|--status|--rollback]
+
+Creates a user-level 192.168.0.188:24318 -> 192.168.0.110:4318 bridge.
+Only source addresses 192.168.0.120 and 192.168.0.121 are accepted.
+No sudo, firewall, Nginx, Docker, database, secret, or volume action is used.
+USAGE
+}
+
+mode="${1:---check}"
+case "$mode" in
+  --check|--apply|--status|--rollback) ;;
+  -h|--help) usage; exit 0 ;;
+  *) usage >&2; exit 64 ;;
+esac
+
+remote() {
+  ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "$@"
+}
+
+receipt() {
+  local phase="$1" result="$2" detail="$3"
+  printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-188-otlp-compat-bridge","phase":"%s","result":"%s","detail":"%s"}\n' \
+    "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail"
+}
+
+remote_payload() {
+  cat <<'REMOTE'
+#!/usr/bin/env bash
+set -euo pipefail
+
+LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}"
+LISTEN_PORT="${LISTEN_PORT:-24318}"
+SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"
+UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}"
+UPSTREAM_PORT="${UPSTREAM_PORT:-4318}"
+RUN_DIR="${RUN_DIR:-/home/ollama/run}"
+LOG_DIR="${LOG_DIR:-/home/ollama/log}"
+PID_FILE="$RUN_DIR/awoooi-signoz-otlp-bridge.pid"
+LOG_FILE="$LOG_DIR/awoooi-signoz-otlp-bridge.log"
+
+is_running() {
+  [ -s "$PID_FILE" ] && kill -0 "$(sed -n '1p' "$PID_FILE")" 2>/dev/null
+}
+
+otlp_code() {
+  curl -sS --max-time 5 -o /dev/null -w '%{http_code}' \
+    -H 'Content-Type: application/json' --data-binary '{"resourceLogs":[]}' "$1" 2>/dev/null || true
+}
+
+status() {
+  running=0
+  if is_running; then running=1; fi
+  listen_socket=0
+  if ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "${LISTEN_HOST}:${LISTEN_PORT}$"; then listen_socket=1; fi
+  upstream_code="$(otlp_code "http://${UPSTREAM_HOST}:${UPSTREAM_PORT}/v1/logs")"
+  printf 'SIGNOZ_OTLP_BRIDGE_STATUS running=%s listen=%s:%s listen_socket=%s source_range=%s upstream=%s:%s upstream_code=%s\n' \
+    "$running" "$LISTEN_HOST" "$LISTEN_PORT" "$listen_socket" "$SOURCE_RANGE" "$UPSTREAM_HOST" "$UPSTREAM_PORT" "${upstream_code:-000}"
+  [ "$running" -eq 1 ] && [ "$listen_socket" -eq 1 ] && [ "${upstream_code:-000}" = 200 ]
+}
+
+start() {
+  command -v socat >/dev/null
+  mkdir -p "$RUN_DIR" "$LOG_DIR"
+  if is_running; then status; return 0; fi
+  if ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)${LISTEN_PORT}$"; then
+    echo "SIGNOZ_OTLP_BRIDGE_BLOCKED port_busy=${LISTEN_PORT}" >&2
+    exit 2
+  fi
+  nohup socat "TCP-LISTEN:${LISTEN_PORT},bind=${LISTEN_HOST},reuseaddr,fork,range=${SOURCE_RANGE}" \
+    "TCP:${UPSTREAM_HOST}:${UPSTREAM_PORT}" >>"$LOG_FILE" 2>&1 &
+  echo "$!" >"$PID_FILE"
+  sleep 1
+  status
+}
+
+stop() {
+  if is_running; then kill "$(sed -n '1p' "$PID_FILE")"; fi
+  rm -f "$PID_FILE"
+  echo "SIGNOZ_OTLP_BRIDGE_STOPPED"
+}
+
+case "${1:-status}" in
+  start) start ;;
+  stop) stop ;;
+  status) status ;;
+  *) exit 64 ;;
+esac
+REMOTE
+}
+
+env_prefix="LISTEN_HOST=$LISTEN_HOST LISTEN_PORT=$LISTEN_PORT SOURCE_RANGE=$SOURCE_RANGE UPSTREAM_HOST=$UPSTREAM_HOST UPSTREAM_PORT=$UPSTREAM_PORT"
+
+verify_nodes() {
+  local target code
+  for target in $VERIFY_NODES; do
+    code="$(ssh "${SSH_OPTS[@]}" "$target" "curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[]}' http://${LISTEN_HOST}:${LISTEN_PORT}/v1/logs || true")"
+    printf 'SIGNOZ_OTLP_BRIDGE_NODE_VERIFY node=%s code=%s\n' "$target" "${code:-000}"
+    [ "${code:-000}" = 200 ]
+  done
+}
+
+case "$mode" in
+  --check)
+    receipt sensor pass "canonical_110_upstream_selected"
+    remote "set -eu
+command -v socat >/dev/null
+upstream=\$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[]}' http://${UPSTREAM_HOST}:${UPSTREAM_PORT}/v1/logs || true)
+busy=0
+if ss -ltn 2>/dev/null | awk '{print \$4}' | grep -Eq '(^|:)${LISTEN_PORT}$'; then busy=1; fi
+printf 'SIGNOZ_OTLP_BRIDGE_CHECK upstream_code=%s listen_port_busy=%s source_range=%s\\n' \"\${upstream:-000}\" \"\$busy\" '$SOURCE_RANGE'
+[ \"\${upstream:-000}\" = 200 ]"
+    receipt check pass "upstream_otlp_200"
+    ;;
+  --apply)
+    receipt decision allow "risk_medium_user_level_compatibility_bridge"
+    remote "mkdir -p /home/ollama/bin /home/ollama/run /home/ollama/log; umask 077; tee '$REMOTE_SCRIPT.tmp' >/dev/null" < <(remote_payload)
+    remote "chmod 0755 '$REMOTE_SCRIPT.tmp' && mv '$REMOTE_SCRIPT.tmp' '$REMOTE_SCRIPT'"
+    remote "(crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d'; printf '%s\n' '# AWOOOI SignOz OTLP bridge start' '@reboot $env_prefix $REMOTE_SCRIPT start >/tmp/awoooi-signoz-otlp-bridge.cron.log 2>&1' '# AWOOOI SignOz OTLP bridge end') | crontab -"
+    remote "$env_prefix '$REMOTE_SCRIPT' start"
+    verify_nodes
+    receipt execute pass "bridge_started_allowlist_120_121"
+    receipt verify pass "nodes_120_121_and_upstream_otlp_200"
+    ;;
+  --status)
+    remote "$env_prefix '$REMOTE_SCRIPT' status"
+    verify_nodes
+    receipt verify pass "bridge_runtime_check"
+    ;;
+  --rollback)
+    remote "if [ -x '$REMOTE_SCRIPT' ]; then $env_prefix '$REMOTE_SCRIPT' stop; fi; crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d' | crontab -"
+    receipt rollback pass "bridge_stopped_cron_removed"
+    ;;
+esac
--- a/scripts/reboot-recovery/tests/test_otel_collector_durable_queue.py
+++ b/scripts/reboot-recovery/tests/test_otel_collector_durable_queue.py
@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+
+ROOT = Path(__file__).resolve().parents[3]
+MANIFEST = ROOT / "k8s" / "observability" / "otel-collector-daemonset.yaml"
+
+
+def _documents() -> list[dict]:
+    return [doc for doc in yaml.safe_load_all(MANIFEST.read_text(encoding="utf-8")) if doc]
+
+
+def test_otel_collector_uses_durable_offsets_and_export_queue() -> None:
+    documents = _documents()
+    config_map = next(doc for doc in documents if doc["kind"] == "ConfigMap")
+    config = yaml.safe_load(config_map["data"]["config.yaml"])
+
+    assert config["extensions"]["file_storage"]["directory"] == "/var/lib/otelcol"
+    assert config["receivers"]["filelog"]["storage"] == "file_storage"
+    exporter = config["exporters"]["otlphttp"]
+    assert exporter["retry_on_failure"]["max_elapsed_time"] == "0s"
+    assert exporter["sending_queue"] == {
+        "enabled": True,
+        "num_consumers": 2,
+        "queue_size": 10000,
+        "storage": "file_storage",
+    }
+    assert "file_storage" in config["service"]["extensions"]
+
+
+def test_otel_daemonset_mounts_per_node_storage_and_rolls_one_at_a_time() -> None:
+    daemonset = next(doc for doc in _documents() if doc["kind"] == "DaemonSet")
+    assert daemonset["spec"]["updateStrategy"]["rollingUpdate"]["maxUnavailable"] == 1
+    pod_spec = daemonset["spec"]["template"]["spec"]
+    assert pod_spec["terminationGracePeriodSeconds"] == 60
+
+    storage = next(volume for volume in pod_spec["volumes"] if volume["name"] == "otel-storage")
+    assert storage["hostPath"] == {
+        "path": "/var/lib/awoooi/otelcol",
+        "type": "DirectoryOrCreate",
+    }
+    collector = next(container for container in pod_spec["containers"] if container["name"] == "otel-collector")
+    assert collector["image"] == (
+        "192.168.0.110:5000/awoooi/otel-collector-contrib@"
+        "sha256:8edf386cdd1f4dc4393e34680e298dcc577296279febf243cf122155fa560c39"
+    )
+    assert collector["imagePullPolicy"] == "IfNotPresent"
+    mount = next(volume for volume in collector["volumeMounts"] if volume["name"] == "otel-storage")
+    assert mount["mountPath"] == "/var/lib/otelcol"
--- a/scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py
+++ b/scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py
@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+OVERRIDE = ROOT / "ops" / "signoz" / "docker-compose.otlp-static.override.yaml"
+STATIC_SCRIPT = ROOT / "scripts" / "reboot-recovery" / "signoz-110-static-ingest.sh"
+BRIDGE_SCRIPT = ROOT / "scripts" / "reboot-recovery" / "signoz-188-otlp-bridge.sh"
+VERIFIER = ROOT / "scripts" / "reboot-recovery" / "verify-signoz-otel-ingestion.sh"
+
+
+def test_static_override_disables_broken_opamp_only_for_collector() -> None:
+    text = OVERRIDE.read_text(encoding="utf-8")
+    assert "otel-collector:" in text
+    assert "--config=/etc/otel-collector-config.yaml" in text
+    assert "--manager-config" not in text
+    assert "--copy-path" not in text
+    assert "signoz-clickhouse" not in text
+    assert "zookeeper" not in text
+    assert "2026-08-15" in text
+
+
+def test_static_apply_is_bounded_no_pull_and_never_reads_dotenv() -> None:
+    text = STATIC_SCRIPT.read_text(encoding="utf-8")
+    executable = "\n".join(
+        line for line in text.splitlines() if not line.lstrip().startswith("#")
+    )
+    assert "--env-file /dev/null" in text
+    assert 'REMOTE_PROJECT="${REMOTE_PROJECT:-signoz}"' in text
+    assert "-p '$REMOTE_PROJECT'" in text
+    assert "up -d --no-deps --pull never otel-collector" in text
+    assert "signoz-clickhouse >/dev/null" in text
+    assert "signoz-zookeeper-1 >/dev/null" in text
+    assert "docker compose down" not in executable
+    assert "docker volume" not in executable
+    assert "docker system prune" not in executable
+    assert "github.com" not in text
+
+
+def test_otlp_bridge_is_allowlisted_reversible_and_has_sunset() -> None:
+    text = BRIDGE_SCRIPT.read_text(encoding="utf-8")
+    executable = "\n".join(
+        line for line in text.splitlines() if not line.lstrip().startswith("#")
+    )
+    assert 'LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}"' in text
+    assert 'LISTEN_PORT="${LISTEN_PORT:-24318}"' in text
+    assert 'SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"' in text
+    assert 'UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}"' in text
+    assert 'UPSTREAM_PORT="${UPSTREAM_PORT:-4318}"' in text
+    assert 'VERIFY_NODES="${VERIFY_NODES:-wooo@192.168.0.120 wooo@192.168.0.121}"' in text
+    assert "resourceLogs" in text
+    assert "@reboot" in text
+    assert "verify_nodes" in text
+    assert "--rollback" in text
+    assert "2026-08-15" in text
+    assert "sudo " not in executable
+    assert "systemctl " not in executable
+    assert "iptables " not in executable
+    assert "docker " not in executable
+
+
+def test_ingestion_verifier_binds_queue_bridge_and_clickhouse() -> None:
+    text = VERIFIER.read_text(encoding="utf-8")
+    assert 'EXPECTED_QUEUE_CAPACITY="${EXPECTED_QUEUE_CAPACITY:-10000}"' in text
+    assert "otelcol_exporter_enqueue_failed_log_records" in text
+    assert "otelcol_exporter_send_failed_log_records" in text
+    assert "otelcol_exporter_sent_log_records" in text
+    assert "otelcol_exporter_queue_size" in text
+    assert "192.168.0.188:24318/v1/logs" in text
+    assert "signoz_logs.logs_v2" in text
+    assert "body={marker:String}" in text
+    assert "declare -A" not in text
+    assert "github.com" not in text
--- a/scripts/reboot-recovery/verify-signoz-otel-ingestion.sh
+++ b/scripts/reboot-recovery/verify-signoz-otel-ingestion.sh
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# Independent verifier for K3s -> 188 compatibility bridge -> 110 SigNoz -> ClickHouse.
+# It writes one non-sensitive synthetic log and queries only that exact marker.
+
+set -euo pipefail
+
+TRACE_ID="${TRACE_ID:-P0-OBS-001}"
+RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}"
+WINDOW_SECONDS="${WINDOW_SECONDS:-30}"
+EXPECTED_PODS="${EXPECTED_PODS:-2}"
+EXPECTED_QUEUE_CAPACITY="${EXPECTED_QUEUE_CAPACITY:-10000}"
+SOURCE_NODE="${SOURCE_NODE:-wooo@192.168.0.120}"
+SIGNOZ_HOST="${SIGNOZ_HOST:-wooo@192.168.0.110}"
+BRIDGE_ENDPOINT="${BRIDGE_ENDPOINT:-http://192.168.0.188:24318/v1/logs}"
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ConnectionAttempts=1)
+marker="P0OBS001_${RUN_ID//[^A-Za-z0-9_]/_}_$(date -u +%Y%m%dT%H%M%SZ)"
+pod_list=()
+enqueue_before=()
+failed_before=()
+sent_before=()
+
+receipt() {
+  local phase="$1" result="$2" detail="$3"
+  printf '{"schema":"awoooi_otel_ingestion_verifier_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","phase":"%s","result":"%s","detail":"%s"}\n' \
+    "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail"
+}
+
+on_error() {
+  local rc=$?
+  receipt terminal failed "command_exit_$rc"
+  exit "$rc"
+}
+trap on_error ERR
+
+pods() {
+  kubectl -n observability get pods -l app.kubernetes.io/name=otel-collector \
+    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'
+}
+
+metric() {
+  local pod="$1" name="$2"
+  kubectl get --raw "/api/v1/namespaces/observability/pods/http:${pod}:8888/proxy/metrics" \
+    | awk -v metric="$name" '$1 ~ ("^" metric "\\{") {print $NF; exit}'
+}
+
+while IFS= read -r pod; do
+  [ -n "$pod" ] && pod_list[${#pod_list[@]}]="$pod"
+done < <(pods)
+pod_count="${#pod_list[@]}"
+[ "$pod_count" -eq "$EXPECTED_PODS" ]
+for ((i = 0; i < pod_count; i++)); do
+  pod="${pod_list[$i]}"
+  enqueue_before[$i]="$(metric "$pod" otelcol_exporter_enqueue_failed_log_records)"
+  failed_before[$i]="$(metric "$pod" otelcol_exporter_send_failed_log_records)"
+  sent_before[$i]="$(metric "$pod" otelcol_exporter_sent_log_records)"
+  capacity="$(metric "$pod" otelcol_exporter_queue_capacity)"
+  [ "$capacity" = "$EXPECTED_QUEUE_CAPACITY" ]
+done
+receipt sensor pass "collector_pods_${pod_count}_queue_capacity_${EXPECTED_QUEUE_CAPACITY}"
+
+http_code="$(ssh "${SSH_OPTS[@]}" "$SOURCE_NODE" "ts=\$(date +%s%N); curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\",\"value\":{\"stringValue\":\"awoooi-p0-obs-verifier\"}}]},\"scopeLogs\":[{\"scope\":{\"name\":\"awoooi.obs.verifier\"},\"logRecords\":[{\"timeUnixNano\":\"'\"\$ts\"'\",\"severityNumber\":9,\"severityText\":\"INFO\",\"body\":{\"stringValue\":\"${marker}\"}}]}]}]}]}' '$BRIDGE_ENDPOINT'")"
+[ "$http_code" = 200 ]
+receipt execute pass "synthetic_otlp_http_200_marker_${marker}"
+
+count=0
+for _ in $(seq 1 15); do
+  count="$(ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" "docker exec signoz-clickhouse clickhouse-client --param_marker='${marker}' --query 'SELECT count() FROM signoz_logs.logs_v2 WHERE body={marker:String}'")"
+  [ "$count" -ge 1 ] && break
+  sleep 2
+done
+[ "$count" -eq 1 ]
+receipt verify pass "clickhouse_exact_marker_count_1"
+
+sleep "$WINDOW_SECONDS"
+for ((i = 0; i < pod_count; i++)); do
+  pod="${pod_list[$i]}"
+  error_lines="$(kubectl -n observability logs "$pod" --since="${WINDOW_SECONDS}s" 2>&1 | grep -Ec 'sending queue is full|connection refused|Exporting failed|rejected_items' || true)"
+  [ "$error_lines" -eq 0 ]
+  [ "$(metric "$pod" otelcol_exporter_queue_size)" = 0 ]
+  [ "$(metric "$pod" otelcol_exporter_enqueue_failed_log_records)" = "${enqueue_before[$i]}" ]
+  [ "$(metric "$pod" otelcol_exporter_send_failed_log_records)" = "${failed_before[$i]}" ]
+  sent_after="$(metric "$pod" otelcol_exporter_sent_log_records)"
+  awk -v before="${sent_before[$i]}" -v after="$sent_after" 'BEGIN {exit !(after >= before)}'
+done
+receipt terminal pass "queue_zero_failure_deltas_zero_window_${WINDOW_SECONDS}s"