From 582388e7012cd8b0b0784faa0b8d6a7671a7ed42 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 19:42:38 +0800 Subject: [PATCH] fix(observability): restore durable SigNoz ingestion --- docs/LOGBOOK.md | 24 +++ ...servability-signoz-unified-architecture.md | 9 +- .../otel-collector-daemonset.yaml | 29 +++- .../docker-compose.otlp-static.override.yaml | 13 ++ ops/signoz/otel-collector-config-phase-o.yaml | 5 +- .../signoz-110-static-ingest.sh | 128 ++++++++++++++ .../reboot-recovery/signoz-188-otlp-bridge.sh | 162 ++++++++++++++++++ .../test_otel_collector_durable_queue.py | 52 ++++++ .../test_signoz_static_ingest_recovery.py | 74 ++++++++ .../verify-signoz-otel-ingestion.sh | 86 ++++++++++ 10 files changed, 578 insertions(+), 4 deletions(-) create mode 100644 ops/signoz/docker-compose.otlp-static.override.yaml create mode 100755 scripts/reboot-recovery/signoz-110-static-ingest.sh create mode 100755 scripts/reboot-recovery/signoz-188-otlp-bridge.sh create mode 100644 scripts/reboot-recovery/tests/test_otel_collector_durable_queue.py create mode 100644 scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py create mode 100755 scripts/reboot-recovery/verify-signoz-otel-ingestion.sh diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 4901c91bf..fc15d45e3 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,27 @@ +## 2026-07-14 — P0-OBS-001 SigNoz OTLP 收集與持久佇列修復 + +**完成內容**： +- 確認正式 SigNoz 已移至 `.110`；根因為 `setupCompleted=false` 且 OpAMP 因缺少 `orgId` 將 logs / metrics / traces pipeline 改為 `nop`，並非 `.188` 主機效能問題。 +- 以 `P0-OBS-001-20260714T191800+0800` 進行受控修復：`.110` 只重建 stateless `signoz-otel-collector`，改用已驗證的 static config；SigNoz UI、ClickHouse、Zookeeper 的 container ID 與啟動時間未變。 +- 新增 `.188:24318 -> .110:4318` 暫時相容橋接，僅允許 `.120/.121` 來源，具備 check / apply / status / rollback 與 `2026-08-15` 到期日。 +- K3s OTEL DaemonSet 新增 `file_storage`、filelog offset 持久化、exporter WAL、`queue_size=10000`、無限時 retry、60 秒 termination grace 與逐節點 rolling update。 +- 將現行 `0.96.0` image 由 K3s cache 直接匯入 Harbor，未對外下載；production 已 pin 至 `192.168.0.110:5000/awoooi/otel-collector-contrib@sha256:8edf386c...`。 +- 新增獨立 ingestion verifier，以同一 run 綁定 K3s queue、`.188` bridge、`.110` listener 與 ClickHouse exact-marker 讀回。 + +**已跑驗證**： +- 精準 pytest：`7 passed`；三支 shell 的 `bash -n` 與 `git diff --check` 通過。 +- `kubectl apply --dry-run=server`、exact live digest `file_storage` canary、Harbor digest pull canary 全數通過；canary 顯示 `Extension started: file_storage`、`Initializing new persistent queue`、`Everything is ready`。 +- DaemonSet generation `4 -> 5` 完成逐節點 rollout；兩個 collector 均 `Ready`、`restarts=0`、queue capacity `10000`、queue size `0`。 +- Synthetic OTLP 從 `.120 -> .188:24318 -> .110:4318 -> signoz_logs.logs_v2` 回讀 exact count `1`；verifier terminal 為 `queue_zero_failure_deltas_zero_window_15s`，兩個 production collector 後續連續超過 11 分鐘 exporter error `0`、queue `0`、send-failed `0`。 + +**仍維持**： +- `setupCompleted=false` 與 SigNoz organization 未初始化仍是治理缺口；static override 與 `.188` 橋接是可回滾、有到期日的暫時控制，不是 Foundry 遷移完成。 +- 本筆 LOGBOOK 建立時，Gitea normal push、CD terminal 與 main-to-runtime parity 尚待後續證據；不得由 live direct apply 倒推 source 已閉環。 +- 未讀取 secret、token、`.env`、raw sessions、SQLite 或 auth；未使用 GitHub / `gh`；未重啟主機、ClickHouse、Zookeeper、SigNoz UI，也未執行 restore、prune、force push 或防火牆變更。 + +**下一步**： +- 將本輪 source 以 normal push 送入 Gitea main，分離追蹤 CD / deploy marker / production readback；在 bridge 到期前將 producer 與 NetworkPolicy 直接遷至 `.110:4318`，再以 replay / shadow / canary 遷移 SigNoz Foundry。 + ## 2026-07-09 — P0-006 AI log triage controlled auto-recovery package **完成內容**： diff --git a/docs/adr/ADR-053-observability-signoz-unified-architecture.md b/docs/adr/ADR-053-observability-signoz-unified-architecture.md index 30ad8244c..842f81345 100644 --- a/docs/adr/ADR-053-observability-signoz-unified-architecture.md +++ b/docs/adr/ADR-053-observability-signoz-unified-architecture.md @@ -7,6 +7,9 @@ | **決策者** | 統帥 (ogt) | | **首席架構師** | Claude Code | | **相關 ADR** | ADR-035 (Telegram Alert), ADR-037 (監控增強), ADR-039 (Gitea CI/CD) | +| **2026-07-14 執行期修正** | `superseded_by=global_product_governance_v2`；`.188` SigNoz 主機配置已取代 | +| **負責範圍 / 到期日** | `platform-observability` / 暫時 `.188:24318` 橋接於 `2026-08-15 Asia/Taipei` 到期 | +| **替代方案 / 驗證器** | 正式後端 `.110:4318`；`scripts/reboot-recovery/verify-signoz-otel-ingestion.sh` | --- @@ -39,7 +42,8 @@ ``` 所有 Log/Event 來源 → OTEL Collector (DaemonSet, K3s 節點) - → 192.168.0.188:24318 (SigNoz OTEL Collector HTTP) + → 192.168.0.188:24318 (暫時性來源白名單相容橋接) + → 192.168.0.110:4318 (正式 SigNoz OTEL Collector HTTP) → ClickHouse (統一儲存) → SigNoz UI (統一查詢) @@ -50,6 +54,9 @@ SigNoz OTEL Collector (Prometheus Receiver) 注意: 原計畫 Prometheus remote_write 因 SigNoz 不支援 Protobuf 格式而廢棄 改用 OTEL Collector 內建 prometheus receiver 直接 scrape 關鍵指標 + +2026-07-14: `.188` 不再承載 SigNoz stateful runtime。相容 bridge 只允許 K3s +`.120/.121` 來源，並必須在到期日前將所有產生端與 NetworkPolicy 改為直接 `.110`。 ``` --- diff --git a/k8s/observability/otel-collector-daemonset.yaml b/k8s/observability/otel-collector-daemonset.yaml index 5894ca8e0..411939c0f 100644 --- a/k8s/observability/otel-collector-daemonset.yaml +++ b/k8s/observability/otel-collector-daemonset.yaml @@ -55,6 +55,12 @@ metadata: namespace: observability data: config.yaml: | + extensions: + # P0-OBS-001: persist file offsets and exporter requests across Pod restarts. + # Per-node WAL is intentionally not backed up; drain it before cleanup/rollback. + file_storage: + directory: /var/lib/otelcol + receivers: # 讀取 K3s 節點上所有 Pod 的 stdout/stderr log filelog: @@ -64,6 +70,7 @@ data: # 排除 OTEL Collector 自身 (防遞迴) - /var/log/pods/observability_otel-collector*/**/*.log start_at: end + storage: file_storage include_file_path: true include_file_name: false operators: @@ -142,12 +149,16 @@ data: enabled: true initial_interval: 5s max_interval: 30s + # Never discard a request merely because the backend outage was long. + max_elapsed_time: 0s sending_queue: enabled: true num_consumers: 2 - queue_size: 100 + queue_size: 10000 + storage: file_storage service: + extensions: [file_storage] pipelines: logs: receivers: [filelog] @@ -164,6 +175,10 @@ metadata: app.kubernetes.io/component: log-collector phase: o-2 spec: + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: otel-collector @@ -174,6 +189,7 @@ spec: app.kubernetes.io/component: log-collector spec: serviceAccountName: otel-collector + terminationGracePeriodSeconds: 60 # observability namespace 設為 privileged PSA # 原因: /var/log/pods 是 root:root 0750，filelog receiver 必須 root 讀取 securityContext: @@ -181,7 +197,10 @@ spec: runAsGroup: 0 containers: - name: otel-collector - image: otel/opentelemetry-collector-contrib:0.96.0 + # P0-OBS-001: exact live 0.96.0 image mirrored from K3s cache to Harbor. + # Do not upgrade in the durability patch; use a separate replay/shadow canary. + image: 192.168.0.110:5000/awoooi/otel-collector-contrib@sha256:8edf386cdd1f4dc4393e34680e298dcc577296279febf243cf122155fa560c39 + imagePullPolicy: IfNotPresent args: - --config=/etc/otel/config.yaml env: @@ -208,6 +227,8 @@ spec: # OTEL Collector 需要 tmp 來寫 checkpoint - name: tmp mountPath: /tmp + - name: otel-storage + mountPath: /var/lib/otelcol volumes: - name: config configMap: @@ -219,6 +240,10 @@ spec: - name: tmp emptyDir: sizeLimit: 50Mi + - name: otel-storage + hostPath: + path: /var/lib/awoooi/otelcol + type: DirectoryOrCreate tolerations: # 確保在所有節點上都運行 (包括 control-plane) - key: node-role.kubernetes.io/control-plane diff --git a/ops/signoz/docker-compose.otlp-static.override.yaml b/ops/signoz/docker-compose.otlp-static.override.yaml new file mode 100644 index 000000000..682d4f65e --- /dev/null +++ b/ops/signoz/docker-compose.otlp-static.override.yaml @@ -0,0 +1,13 @@ +# P0-OBS-001 controlled workaround for SigNoz v0.113.0 setupCompleted=false. +# Risk: medium. Scope: recreate only the stateless otel-collector container. +# Why: OpAMP cannot attach an orgId and replaces all signal pipelines with nop. +# Replacement: initialize the SigNoz organization and migrate the stack to Foundry. +# Owner: platform-observability. Expiry: 2026-08-15 Asia/Taipei. +# Rollback: run the base compose file without this override. +services: + otel-collector: + entrypoint: + - /bin/sh + command: + - -c + - exec /signoz-otel-collector --config=/etc/otel-collector-config.yaml diff --git a/ops/signoz/otel-collector-config-phase-o.yaml b/ops/signoz/otel-collector-config-phase-o.yaml index 681e3a536..8ca7aeba1 100644 --- a/ops/signoz/otel-collector-config-phase-o.yaml +++ b/ops/signoz/otel-collector-config-phase-o.yaml @@ -3,7 +3,10 @@ # ============================================================================= # 建立者: Claude Code (首席架構師) # 日期: 2026-04-02 (台北時間) -# 部署位置: 192.168.0.188:/home/ollama/signoz/deploy/docker/otel-collector-config.yaml +# superseded_by=global_product_governance_v2 +# Legacy .188 deployment is retired; do not deploy this file to .188. +# Owner: platform-observability. Replacement: formal .110 static ingestion override. +# Expiry/verifier: 2026-08-15; scripts/reboot-recovery/verify-signoz-otel-ingestion.sh # # Phase O-3 新增內容 (與原版差異): # prometheus receiver 新增 scrape jobs: diff --git a/scripts/reboot-recovery/signoz-110-static-ingest.sh b/scripts/reboot-recovery/signoz-110-static-ingest.sh new file mode 100755 index 000000000..0ff9ab0b5 --- /dev/null +++ b/scripts/reboot-recovery/signoz-110-static-ingest.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# Controlled static-ingestion fallback for the formal SigNoz runtime on host 110. +# It never reads .env, container environment values, secrets, or stateful volumes. + +set -euo pipefail + +TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}" +REMOTE_DIR="${REMOTE_DIR:-/home/wooo/signoz/deploy/docker}" +REMOTE_BASE="${REMOTE_BASE:-$REMOTE_DIR/docker-compose.yaml}" +REMOTE_OVERRIDE="${REMOTE_OVERRIDE:-$REMOTE_DIR/docker-compose.awoooi-static-ingest.yaml}" +REMOTE_PROJECT="${REMOTE_PROJECT:-signoz}" +SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}" +TRACE_ID="${TRACE_ID:-P0-OBS-001}" +RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}" +WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +LOCAL_OVERRIDE="${LOCAL_OVERRIDE:-$ROOT_DIR/ops/signoz/docker-compose.otlp-static.override.yaml}" +SSH_OPTS=( + -o BatchMode=yes + -o ConnectTimeout="$SSH_CONNECT_TIMEOUT" + -o ConnectionAttempts=1 + -o ServerAliveInterval=5 + -o ServerAliveCountMax=1 +) + +usage() { + cat <<'USAGE' +Usage: signoz-110-static-ingest.sh [--check|--apply|--status|--rollback] + + --check Validate the base stack and source-controlled override. + --apply Recreate only signoz-otel-collector with the static base config. + --status Verify the collector and OTLP listeners. + --rollback Recreate only the collector from the original OpAMP command. + +All compose calls use --env-file /dev/null. Stateful services stay untouched. +USAGE +} + +mode="${1:---check}" +case "$mode" in + --check|--apply|--status|--rollback) ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; exit 64 ;; +esac + +remote() { + ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "$@" +} + +receipt() { + local phase="$1" result="$2" detail="$3" + printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-110-otel-ingest","phase":"%s","result":"%s","detail":"%s"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail" +} + +on_error() { + local rc=$? + receipt terminal failed "command_exit_$rc" + exit "$rc" +} +trap on_error ERR + +status_remote() { + remote "set -eu +container=signoz-otel-collector +running=\$(docker inspect -f '{{.State.Running}}' \"\$container\") +restart_count=\$(docker inspect -f '{{.RestartCount}}' \"\$container\") +cmd=\$(docker inspect -f '{{json .Config.Cmd}}' \"\$container\") +http_code=\$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[]}' http://127.0.0.1:4318/v1/logs || true) +grpc=0 +http=0 +if docker exec \"\$container\" sh -c \"grep -q ':10DD .* 0A ' /proc/net/tcp /proc/net/tcp6\"; then grpc=1; fi +if docker exec \"\$container\" sh -c \"grep -q ':10DE .* 0A ' /proc/net/tcp /proc/net/tcp6\"; then http=1; fi +printf 'SIGNOZ_STATIC_INGEST_STATUS running=%s restart_count=%s grpc_4317=%s http_4318=%s otlp_http_code=%s command=%s\\n' \"\$running\" \"\$restart_count\" \"\$grpc\" \"\$http\" \"\${http_code:-000}\" \"\$cmd\" +[ \"\$running\" = true ] && [ \"\$grpc\" = 1 ] && [ \"\$http\" = 1 ] && [ \"\${http_code:-000}\" = 200 ]" +} + +case "$mode" in + --check) + test -s "$LOCAL_OVERRIDE" + receipt sensor pass "source_override_present" + remote "set -eu +test -s '$REMOTE_BASE' +docker inspect signoz >/dev/null +docker inspect signoz-clickhouse >/dev/null +docker inspect signoz-zookeeper-1 >/dev/null +docker inspect signoz-otel-collector >/dev/null +cd '$REMOTE_DIR' +docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' config --quiet +printf 'SIGNOZ_STATIC_INGEST_CHECK base_compose=valid env_file=/dev/null collector=%s\\n' \"\$(docker inspect -f '{{.State.Status}}' signoz-otel-collector)\"" + receipt check pass "base_compose_valid_no_env" + ;; + --apply) + test -s "$LOCAL_OVERRIDE" + receipt decision allow "risk_medium_stateless_collector_only" + scp "${SSH_OPTS[@]}" "$LOCAL_OVERRIDE" "$TARGET_HOST:$REMOTE_OVERRIDE.tmp" + remote "set -eu +chmod 0644 '$REMOTE_OVERRIDE.tmp' +mv '$REMOTE_OVERRIDE.tmp' '$REMOTE_OVERRIDE' +cd '$REMOTE_DIR' +docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' -f '$REMOTE_OVERRIDE' config --quiet +printf 'SIGNOZ_STATIC_INGEST_DRY_RUN result=pass env_file=/dev/null override=%s\\n' '$REMOTE_OVERRIDE'" + receipt check pass "compose_override_valid" + remote "set -eu +cd '$REMOTE_DIR' +before_id=\$(docker inspect -f '{{.Id}}' signoz-otel-collector) +docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' -f '$REMOTE_OVERRIDE' up -d --no-deps --pull never otel-collector +after_id=\$(docker inspect -f '{{.Id}}' signoz-otel-collector) +printf 'SIGNOZ_STATIC_INGEST_APPLY before_id=%s after_id=%s stateful_services_touched=0\\n' \"\$before_id\" \"\$after_id\"" + receipt execute pass "collector_recreated_stateful_services_untouched" + status_remote + receipt verify pass "otlp_4317_4318_listening_empty_request_200" + ;; + --status) + status_remote + receipt verify pass "runtime_listener_check" + ;; + --rollback) + receipt decision allow "bounded_rollback_collector_only" + remote "set -eu +cd '$REMOTE_DIR' +docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' up -d --no-deps --pull never otel-collector +cmd=\$(docker inspect -f '{{json .Config.Cmd}}' signoz-otel-collector) +printf 'SIGNOZ_STATIC_INGEST_ROLLBACK command=%s\\n' \"\$cmd\" +printf '%s' \"\$cmd\" | grep -q -- '--manager-config'" + receipt rollback pass "opamp_command_restored" + ;; +esac diff --git a/scripts/reboot-recovery/signoz-188-otlp-bridge.sh b/scripts/reboot-recovery/signoz-188-otlp-bridge.sh new file mode 100755 index 000000000..e31512e85 --- /dev/null +++ b/scripts/reboot-recovery/signoz-188-otlp-bridge.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# Temporary, source-controlled OTLP compatibility bridge for K3s collectors. +# Owner: platform-observability. Expiry: 2026-08-15 Asia/Taipei. +# Replacement: repoint all producers and policies directly to host 110 OTLP. + +set -euo pipefail + +TARGET_HOST="${TARGET_HOST:-ollama@192.168.0.188}" +REMOTE_SCRIPT="${REMOTE_SCRIPT:-/home/ollama/bin/awoooi-signoz-otlp-bridge.sh}" +LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}" +LISTEN_PORT="${LISTEN_PORT:-24318}" +SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}" +UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}" +UPSTREAM_PORT="${UPSTREAM_PORT:-4318}" +TRACE_ID="${TRACE_ID:-P0-OBS-001}" +RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}" +WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}" +VERIFY_NODES="${VERIFY_NODES:-wooo@192.168.0.120 wooo@192.168.0.121}" +SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}" +SSH_OPTS=( + -o BatchMode=yes + -o ConnectTimeout="$SSH_CONNECT_TIMEOUT" + -o ConnectionAttempts=1 + -o ServerAliveInterval=5 + -o ServerAliveCountMax=1 +) + +usage() { + cat <<'USAGE' +Usage: signoz-188-otlp-bridge.sh [--check|--apply|--status|--rollback] + +Creates a user-level 192.168.0.188:24318 -> 192.168.0.110:4318 bridge. +Only source addresses 192.168.0.120 and 192.168.0.121 are accepted. +No sudo, firewall, Nginx, Docker, database, secret, or volume action is used. +USAGE +} + +mode="${1:---check}" +case "$mode" in + --check|--apply|--status|--rollback) ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; exit 64 ;; +esac + +remote() { + ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "$@" +} + +receipt() { + local phase="$1" result="$2" detail="$3" + printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-188-otlp-compat-bridge","phase":"%s","result":"%s","detail":"%s"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail" +} + +remote_payload() { + cat <<'REMOTE' +#!/usr/bin/env bash +set -euo pipefail + +LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}" +LISTEN_PORT="${LISTEN_PORT:-24318}" +SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}" +UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}" +UPSTREAM_PORT="${UPSTREAM_PORT:-4318}" +RUN_DIR="${RUN_DIR:-/home/ollama/run}" +LOG_DIR="${LOG_DIR:-/home/ollama/log}" +PID_FILE="$RUN_DIR/awoooi-signoz-otlp-bridge.pid" +LOG_FILE="$LOG_DIR/awoooi-signoz-otlp-bridge.log" + +is_running() { + [ -s "$PID_FILE" ] && kill -0 "$(sed -n '1p' "$PID_FILE")" 2>/dev/null +} + +otlp_code() { + curl -sS --max-time 5 -o /dev/null -w '%{http_code}' \ + -H 'Content-Type: application/json' --data-binary '{"resourceLogs":[]}' "$1" 2>/dev/null || true +} + +status() { + running=0 + if is_running; then running=1; fi + listen_socket=0 + if ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "${LISTEN_HOST}:${LISTEN_PORT}$"; then listen_socket=1; fi + upstream_code="$(otlp_code "http://${UPSTREAM_HOST}:${UPSTREAM_PORT}/v1/logs")" + printf 'SIGNOZ_OTLP_BRIDGE_STATUS running=%s listen=%s:%s listen_socket=%s source_range=%s upstream=%s:%s upstream_code=%s\n' \ + "$running" "$LISTEN_HOST" "$LISTEN_PORT" "$listen_socket" "$SOURCE_RANGE" "$UPSTREAM_HOST" "$UPSTREAM_PORT" "${upstream_code:-000}" + [ "$running" -eq 1 ] && [ "$listen_socket" -eq 1 ] && [ "${upstream_code:-000}" = 200 ] +} + +start() { + command -v socat >/dev/null + mkdir -p "$RUN_DIR" "$LOG_DIR" + if is_running; then status; return 0; fi + if ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)${LISTEN_PORT}$"; then + echo "SIGNOZ_OTLP_BRIDGE_BLOCKED port_busy=${LISTEN_PORT}" >&2 + exit 2 + fi + nohup socat "TCP-LISTEN:${LISTEN_PORT},bind=${LISTEN_HOST},reuseaddr,fork,range=${SOURCE_RANGE}" \ + "TCP:${UPSTREAM_HOST}:${UPSTREAM_PORT}" >>"$LOG_FILE" 2>&1 & + echo "$!" >"$PID_FILE" + sleep 1 + status +} + +stop() { + if is_running; then kill "$(sed -n '1p' "$PID_FILE")"; fi + rm -f "$PID_FILE" + echo "SIGNOZ_OTLP_BRIDGE_STOPPED" +} + +case "${1:-status}" in + start) start ;; + stop) stop ;; + status) status ;; + *) exit 64 ;; +esac +REMOTE +} + +env_prefix="LISTEN_HOST=$LISTEN_HOST LISTEN_PORT=$LISTEN_PORT SOURCE_RANGE=$SOURCE_RANGE UPSTREAM_HOST=$UPSTREAM_HOST UPSTREAM_PORT=$UPSTREAM_PORT" + +verify_nodes() { + local target code + for target in $VERIFY_NODES; do + code="$(ssh "${SSH_OPTS[@]}" "$target" "curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[]}' http://${LISTEN_HOST}:${LISTEN_PORT}/v1/logs || true")" + printf 'SIGNOZ_OTLP_BRIDGE_NODE_VERIFY node=%s code=%s\n' "$target" "${code:-000}" + [ "${code:-000}" = 200 ] + done +} + +case "$mode" in + --check) + receipt sensor pass "canonical_110_upstream_selected" + remote "set -eu +command -v socat >/dev/null +upstream=\$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[]}' http://${UPSTREAM_HOST}:${UPSTREAM_PORT}/v1/logs || true) +busy=0 +if ss -ltn 2>/dev/null | awk '{print \$4}' | grep -Eq '(^|:)${LISTEN_PORT}$'; then busy=1; fi +printf 'SIGNOZ_OTLP_BRIDGE_CHECK upstream_code=%s listen_port_busy=%s source_range=%s\\n' \"\${upstream:-000}\" \"\$busy\" '$SOURCE_RANGE' +[ \"\${upstream:-000}\" = 200 ]" + receipt check pass "upstream_otlp_200" + ;; + --apply) + receipt decision allow "risk_medium_user_level_compatibility_bridge" + remote "mkdir -p /home/ollama/bin /home/ollama/run /home/ollama/log; umask 077; tee '$REMOTE_SCRIPT.tmp' >/dev/null" < <(remote_payload) + remote "chmod 0755 '$REMOTE_SCRIPT.tmp' && mv '$REMOTE_SCRIPT.tmp' '$REMOTE_SCRIPT'" + remote "(crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d'; printf '%s\n' '# AWOOOI SignOz OTLP bridge start' '@reboot $env_prefix $REMOTE_SCRIPT start >/tmp/awoooi-signoz-otlp-bridge.cron.log 2>&1' '# AWOOOI SignOz OTLP bridge end') | crontab -" + remote "$env_prefix '$REMOTE_SCRIPT' start" + verify_nodes + receipt execute pass "bridge_started_allowlist_120_121" + receipt verify pass "nodes_120_121_and_upstream_otlp_200" + ;; + --status) + remote "$env_prefix '$REMOTE_SCRIPT' status" + verify_nodes + receipt verify pass "bridge_runtime_check" + ;; + --rollback) + remote "if [ -x '$REMOTE_SCRIPT' ]; then $env_prefix '$REMOTE_SCRIPT' stop; fi; crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d' | crontab -" + receipt rollback pass "bridge_stopped_cron_removed" + ;; +esac diff --git a/scripts/reboot-recovery/tests/test_otel_collector_durable_queue.py b/scripts/reboot-recovery/tests/test_otel_collector_durable_queue.py new file mode 100644 index 000000000..d05367e89 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_otel_collector_durable_queue.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[3] +MANIFEST = ROOT / "k8s" / "observability" / "otel-collector-daemonset.yaml" + + +def _documents() -> list[dict]: + return [doc for doc in yaml.safe_load_all(MANIFEST.read_text(encoding="utf-8")) if doc] + + +def test_otel_collector_uses_durable_offsets_and_export_queue() -> None: + documents = _documents() + config_map = next(doc for doc in documents if doc["kind"] == "ConfigMap") + config = yaml.safe_load(config_map["data"]["config.yaml"]) + + assert config["extensions"]["file_storage"]["directory"] == "/var/lib/otelcol" + assert config["receivers"]["filelog"]["storage"] == "file_storage" + exporter = config["exporters"]["otlphttp"] + assert exporter["retry_on_failure"]["max_elapsed_time"] == "0s" + assert exporter["sending_queue"] == { + "enabled": True, + "num_consumers": 2, + "queue_size": 10000, + "storage": "file_storage", + } + assert "file_storage" in config["service"]["extensions"] + + +def test_otel_daemonset_mounts_per_node_storage_and_rolls_one_at_a_time() -> None: + daemonset = next(doc for doc in _documents() if doc["kind"] == "DaemonSet") + assert daemonset["spec"]["updateStrategy"]["rollingUpdate"]["maxUnavailable"] == 1 + pod_spec = daemonset["spec"]["template"]["spec"] + assert pod_spec["terminationGracePeriodSeconds"] == 60 + + storage = next(volume for volume in pod_spec["volumes"] if volume["name"] == "otel-storage") + assert storage["hostPath"] == { + "path": "/var/lib/awoooi/otelcol", + "type": "DirectoryOrCreate", + } + collector = next(container for container in pod_spec["containers"] if container["name"] == "otel-collector") + assert collector["image"] == ( + "192.168.0.110:5000/awoooi/otel-collector-contrib@" + "sha256:8edf386cdd1f4dc4393e34680e298dcc577296279febf243cf122155fa560c39" + ) + assert collector["imagePullPolicy"] == "IfNotPresent" + mount = next(volume for volume in collector["volumeMounts"] if volume["name"] == "otel-storage") + assert mount["mountPath"] == "/var/lib/otelcol" diff --git a/scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py b/scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py new file mode 100644 index 000000000..1956243a0 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +OVERRIDE = ROOT / "ops" / "signoz" / "docker-compose.otlp-static.override.yaml" +STATIC_SCRIPT = ROOT / "scripts" / "reboot-recovery" / "signoz-110-static-ingest.sh" +BRIDGE_SCRIPT = ROOT / "scripts" / "reboot-recovery" / "signoz-188-otlp-bridge.sh" +VERIFIER = ROOT / "scripts" / "reboot-recovery" / "verify-signoz-otel-ingestion.sh" + + +def test_static_override_disables_broken_opamp_only_for_collector() -> None: + text = OVERRIDE.read_text(encoding="utf-8") + assert "otel-collector:" in text + assert "--config=/etc/otel-collector-config.yaml" in text + assert "--manager-config" not in text + assert "--copy-path" not in text + assert "signoz-clickhouse" not in text + assert "zookeeper" not in text + assert "2026-08-15" in text + + +def test_static_apply_is_bounded_no_pull_and_never_reads_dotenv() -> None: + text = STATIC_SCRIPT.read_text(encoding="utf-8") + executable = "\n".join( + line for line in text.splitlines() if not line.lstrip().startswith("#") + ) + assert "--env-file /dev/null" in text + assert 'REMOTE_PROJECT="${REMOTE_PROJECT:-signoz}"' in text + assert "-p '$REMOTE_PROJECT'" in text + assert "up -d --no-deps --pull never otel-collector" in text + assert "signoz-clickhouse >/dev/null" in text + assert "signoz-zookeeper-1 >/dev/null" in text + assert "docker compose down" not in executable + assert "docker volume" not in executable + assert "docker system prune" not in executable + assert "github.com" not in text + + +def test_otlp_bridge_is_allowlisted_reversible_and_has_sunset() -> None: + text = BRIDGE_SCRIPT.read_text(encoding="utf-8") + executable = "\n".join( + line for line in text.splitlines() if not line.lstrip().startswith("#") + ) + assert 'LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}"' in text + assert 'LISTEN_PORT="${LISTEN_PORT:-24318}"' in text + assert 'SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"' in text + assert 'UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}"' in text + assert 'UPSTREAM_PORT="${UPSTREAM_PORT:-4318}"' in text + assert 'VERIFY_NODES="${VERIFY_NODES:-wooo@192.168.0.120 wooo@192.168.0.121}"' in text + assert "resourceLogs" in text + assert "@reboot" in text + assert "verify_nodes" in text + assert "--rollback" in text + assert "2026-08-15" in text + assert "sudo " not in executable + assert "systemctl " not in executable + assert "iptables " not in executable + assert "docker " not in executable + + +def test_ingestion_verifier_binds_queue_bridge_and_clickhouse() -> None: + text = VERIFIER.read_text(encoding="utf-8") + assert 'EXPECTED_QUEUE_CAPACITY="${EXPECTED_QUEUE_CAPACITY:-10000}"' in text + assert "otelcol_exporter_enqueue_failed_log_records" in text + assert "otelcol_exporter_send_failed_log_records" in text + assert "otelcol_exporter_sent_log_records" in text + assert "otelcol_exporter_queue_size" in text + assert "192.168.0.188:24318/v1/logs" in text + assert "signoz_logs.logs_v2" in text + assert "body={marker:String}" in text + assert "declare -A" not in text + assert "github.com" not in text diff --git a/scripts/reboot-recovery/verify-signoz-otel-ingestion.sh b/scripts/reboot-recovery/verify-signoz-otel-ingestion.sh new file mode 100755 index 000000000..ee8d3a402 --- /dev/null +++ b/scripts/reboot-recovery/verify-signoz-otel-ingestion.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Independent verifier for K3s -> 188 compatibility bridge -> 110 SigNoz -> ClickHouse. +# It writes one non-sensitive synthetic log and queries only that exact marker. + +set -euo pipefail + +TRACE_ID="${TRACE_ID:-P0-OBS-001}" +RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}" +WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}" +WINDOW_SECONDS="${WINDOW_SECONDS:-30}" +EXPECTED_PODS="${EXPECTED_PODS:-2}" +EXPECTED_QUEUE_CAPACITY="${EXPECTED_QUEUE_CAPACITY:-10000}" +SOURCE_NODE="${SOURCE_NODE:-wooo@192.168.0.120}" +SIGNOZ_HOST="${SIGNOZ_HOST:-wooo@192.168.0.110}" +BRIDGE_ENDPOINT="${BRIDGE_ENDPOINT:-http://192.168.0.188:24318/v1/logs}" +SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ConnectionAttempts=1) +marker="P0OBS001_${RUN_ID//[^A-Za-z0-9_]/_}_$(date -u +%Y%m%dT%H%M%SZ)" +pod_list=() +enqueue_before=() +failed_before=() +sent_before=() + +receipt() { + local phase="$1" result="$2" detail="$3" + printf '{"schema":"awoooi_otel_ingestion_verifier_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","phase":"%s","result":"%s","detail":"%s"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail" +} + +on_error() { + local rc=$? + receipt terminal failed "command_exit_$rc" + exit "$rc" +} +trap on_error ERR + +pods() { + kubectl -n observability get pods -l app.kubernetes.io/name=otel-collector \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' +} + +metric() { + local pod="$1" name="$2" + kubectl get --raw "/api/v1/namespaces/observability/pods/http:${pod}:8888/proxy/metrics" \ + | awk -v metric="$name" '$1 ~ ("^" metric "\\{") {print $NF; exit}' +} + +while IFS= read -r pod; do + [ -n "$pod" ] && pod_list[${#pod_list[@]}]="$pod" +done < <(pods) +pod_count="${#pod_list[@]}" +[ "$pod_count" -eq "$EXPECTED_PODS" ] +for ((i = 0; i < pod_count; i++)); do + pod="${pod_list[$i]}" + enqueue_before[$i]="$(metric "$pod" otelcol_exporter_enqueue_failed_log_records)" + failed_before[$i]="$(metric "$pod" otelcol_exporter_send_failed_log_records)" + sent_before[$i]="$(metric "$pod" otelcol_exporter_sent_log_records)" + capacity="$(metric "$pod" otelcol_exporter_queue_capacity)" + [ "$capacity" = "$EXPECTED_QUEUE_CAPACITY" ] +done +receipt sensor pass "collector_pods_${pod_count}_queue_capacity_${EXPECTED_QUEUE_CAPACITY}" + +http_code="$(ssh "${SSH_OPTS[@]}" "$SOURCE_NODE" "ts=\$(date +%s%N); curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\",\"value\":{\"stringValue\":\"awoooi-p0-obs-verifier\"}}]},\"scopeLogs\":[{\"scope\":{\"name\":\"awoooi.obs.verifier\"},\"logRecords\":[{\"timeUnixNano\":\"'\"\$ts\"'\",\"severityNumber\":9,\"severityText\":\"INFO\",\"body\":{\"stringValue\":\"${marker}\"}}]}]}]}]}' '$BRIDGE_ENDPOINT'")" +[ "$http_code" = 200 ] +receipt execute pass "synthetic_otlp_http_200_marker_${marker}" + +count=0 +for _ in $(seq 1 15); do + count="$(ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" "docker exec signoz-clickhouse clickhouse-client --param_marker='${marker}' --query 'SELECT count() FROM signoz_logs.logs_v2 WHERE body={marker:String}'")" + [ "$count" -ge 1 ] && break + sleep 2 +done +[ "$count" -eq 1 ] +receipt verify pass "clickhouse_exact_marker_count_1" + +sleep "$WINDOW_SECONDS" +for ((i = 0; i < pod_count; i++)); do + pod="${pod_list[$i]}" + error_lines="$(kubectl -n observability logs "$pod" --since="${WINDOW_SECONDS}s" 2>&1 | grep -Ec 'sending queue is full|connection refused|Exporting failed|rejected_items' || true)" + [ "$error_lines" -eq 0 ] + [ "$(metric "$pod" otelcol_exporter_queue_size)" = 0 ] + [ "$(metric "$pod" otelcol_exporter_enqueue_failed_log_records)" = "${enqueue_before[$i]}" ] + [ "$(metric "$pod" otelcol_exporter_send_failed_log_records)" = "${failed_before[$i]}" ] + sent_after="$(metric "$pod" otelcol_exporter_sent_log_records)" + awk -v before="${sent_before[$i]}" -v after="$sent_after" 'BEGIN {exit !(after >= before)}' +done +receipt terminal pass "queue_zero_failure_deltas_zero_window_${WINDOW_SECONDS}s"