diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 0aeff594e..18f3fbc93 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -8,15 +8,19 @@ - 產出 `ops/signoz/p0-obs-002-three-signal-route-parity.yaml`;明確把 synthetic route parity 與真實應用 trace freshness、producer queue gate、正式 promotion 分開。 - 最新 10 分鐘 error fingerprint 證明 normalization noise 是固定成對錯誤:`.120` 為 `cri_parser regex_no_match=156` + `move3 missing_field=156`,`.121` 為 `126 + 126`;同時兩個 collector queue / send_failed 都是 `0`。 - 第一版 filelog candidate 的 `on_error: send_quiet` 通過 `0.96.0 validate`,但隔離 Pod 在真正 build pipeline 時明確拒絕;兩次 canary 均已刪除且未碰正式 DaemonSet。source 已移除不相容欄位,改為 parser `body matches` 前置條件、保留 malformed 原始 body、以 `awoooi.log.parse_status=fallback_unparsed` 標記,並只在 `attributes.log` 存在時 move body;修正版刻意不做第三次 runtime canary,狀態維持 canary pending。 +- 真實 trace freshness audit 找到 `0.99` 信心根因:Python SDK traces / metrics 使用 gRPC `.188:24317`,但現有相容橋接只有 HTTP `.188:24318 -> .110:4318`;API pod 對 24317 連線拒絕,ClickHouse 最近只有三筆 synthetic verifier span,`awoooi-api=0`。 +- 新增獨立 user-systemd gRPC bridge source `.188:24317 -> .110:4317`,沿用 `.120/31` source allowlist,具備 check / apply / status / rollback;另增 10 分鐘真實應用 trace + metric aggregate、export error=0、restart delta=0 post-verifier。它不改 API endpoint、不改 firewall,且可與 HTTP bridge 分開回滾。 **已跑驗證**: -- SigNoz / OTEL / Event Exporter / API config 聚焦回歸:`43 passed`;其中 canonical route migration 為 `9 passed`。ruff、py_compile、四支 shell `bash -n`、canonical route preflight、三份 Kubernetes server dry-run、runner pressure guard、Gitea secret transport guard 與 `git diff --check` 通過。 +- SigNoz / OTEL / Event Exporter / API config 聚焦回歸:`45 passed`;其中 canonical route migration 為 `11 passed`。ruff、py_compile、六支 shell `bash -n`、canonical route preflight、三份 Kubernetes server dry-run、runner pressure guard、Gitea secret transport guard 與 `git diff --check` 通過。 +- gRPC bridge no-write check `P0-OBS-002-grpc-bridge-check-20260715T012000+0800` 通過:`.110:4317` upstream TCP ready、`.188:24317` 無既有 listener、user linger=yes、source range 維持 `.120/31`。 - Gitea feature evidence commit `0523af6bfc9572f1822daef7277ace80d79cb1bb` 已由 SSH exact ref 回讀;這只完成 feature 層 durable acknowledgement,main / CD / production / KM writeback 仍是獨立 pending。 - 先前 route-specific log replay 已驗證 `.120/.121` HTTP `200`、ClickHouse exact marker `1`、15 秒 queue / failure delta `0`;本次三訊號 direct verifier不把該結果誤宣稱成 producer canary。 - bridge rollback 後 `.120/.121 -> .188:24318` 回 HTTP `200`,legacy cron 已恢復,live user-systemd unit 已移除 / disabled。 **仍維持**: - `setupCompleted=false`、organization 未初始化、`.110:4317/4318` direct ingress 暴露面尚未收斂、真實應用 traces 不新鮮、filelog normalization source fix 尚未 canary、bridge 尚未由 user-systemd supervision 接管。 +- gRPC bridge 本輪先維持 source-ready / runtime-apply-pending;必須等目前 Gitea CD / API rollout 穩定 2/2 後,才可套用並跑完整 600 秒 post-verifier。HTTP-only synthetic `/v1/traces` 成功不能取代此 gRPC 證據。 - synthetic 三訊號 parity 不等於正式 producer canary、Wave B promotion、bridge retirement、AI learning writeback 或 production closure;active route 仍是 incumbent。 - organization/root bootstrap 涉及 durable credential;direct ingress policy 可能涉及 firewall / mTLS。這些維持獨立 high/critical gate,本輪未讀 secret、未改 firewall、未寫 raw DB、未碰 stateful volume。 - Gitea feature branch / source 測試、main CD、deploy marker、production runtime 與 visible evidence 仍分層追蹤;不得由 feature push 或 synthetic readback 倒推 production 已部署。 diff --git a/ops/signoz/organization-canonical-route-migration.yaml b/ops/signoz/organization-canonical-route-migration.yaml index b638ae37e..66dcb8ef0 100644 --- a/ops/signoz/organization-canonical-route-migration.yaml +++ b/ops/signoz/organization-canonical-route-migration.yaml @@ -34,6 +34,8 @@ routes: grpc: 192.168.0.188:24317 http: http://192.168.0.188:24318 source_range: 192.168.0.120/31 + http_bridge_status: active_legacy + grpc_bridge_status: source_ready_runtime_apply_pending expiry: "2026-08-15T23:59:59+08:00" challenger: canonical_id: signoz-110-otlp @@ -78,6 +80,7 @@ wave_a_gates: challenger_application_allowlisted: true challenger_networkpolicy_allowed: true bridge_user_systemd_supervision_required: true + incumbent_grpc_bridge_required: true route_specific_log_replay_required: true route_specific_logs_metrics_traces_replay_required: true @@ -96,6 +99,20 @@ runtime_observations: metrics_fresh: true traces_fresh: false bridge_supervisor: legacy_orphan_process + application_otlp_grpc: + endpoint: http://192.168.0.188:24317 + root_cause: missing_incumbent_grpc_bridge + confidence: 0.99 + api_pods_ready_at_audit: 2 + recent_awoooi_api_trace_count: 0 + http_only_verifier_blind_spot: true + source_fix: + bridge: scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh + unit: ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service + post_verifier: scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh + check_run_id: P0-OBS-002-grpc-bridge-check-20260715T012000+0800 + check_terminal: pass + runtime_apply_status: check_pass_pending_stable_cd_window filelog_normalization: source_fix_canary_pending filelog_normalization_error_readback: window: 10m diff --git a/ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service b/ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service new file mode 100644 index 000000000..4afd07aa7 --- /dev/null +++ b/ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service @@ -0,0 +1,31 @@ +[Unit] +Description=AWOOOI SigNoz OTLP gRPC compatibility bridge +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +Environment=LISTEN_HOST=192.168.0.188 +Environment=LISTEN_PORT=24317 +Environment=SOURCE_RANGE=192.168.0.120/31 +Environment=UPSTREAM_HOST=192.168.0.110 +Environment=UPSTREAM_PORT=4317 +ExecStartPre=/home/ollama/bin/awoooi-signoz-otlp-grpc-bridge.sh check-upstream +ExecStart=/home/ollama/bin/awoooi-signoz-otlp-grpc-bridge.sh run +ExecStopPost=/usr/bin/rm -f /home/ollama/run/awoooi-signoz-otlp-grpc-bridge.pid +Restart=on-failure +RestartSec=5s +TimeoutStartSec=15s +TimeoutStopSec=15s +UMask=0077 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths=/home/ollama/run /home/ollama/log +LockPersonality=true +MemoryDenyWriteExecute=true +RestrictAddressFamilies=AF_INET AF_INET6 + +[Install] +WantedBy=default.target diff --git a/scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh b/scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh new file mode 100755 index 000000000..8bc17be21 --- /dev/null +++ b/scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash +# Independently supervised OTLP/gRPC compatibility bridge for P0-OBS-002. +# It is paired with, but independently rollbackable from, the HTTP bridge. + +set -Eeuo pipefail + +TARGET_HOST="${TARGET_HOST:-ollama@192.168.0.188}" +REMOTE_SCRIPT="${REMOTE_SCRIPT:-/home/ollama/bin/awoooi-signoz-otlp-grpc-bridge.sh}" +REMOTE_UNIT="${REMOTE_UNIT:-/home/ollama/.config/systemd/user/awoooi-signoz-otlp-grpc-bridge.service}" +REMOTE_UNIT_STAGE="${REMOTE_UNIT_STAGE:-/home/ollama/run/awoooi-signoz-otlp-grpc-bridge.service}" +REMOTE_SERVICE="${REMOTE_SERVICE:-awoooi-signoz-otlp-grpc-bridge.service}" +LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}" +LISTEN_PORT="${LISTEN_PORT:-24317}" +SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}" +UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}" +UPSTREAM_PORT="${UPSTREAM_PORT:-4317}" +VERIFY_NODES="${VERIFY_NODES:-wooo@192.168.0.120 wooo@192.168.0.121}" +TRACE_ID="${TRACE_ID:-P0-OBS-002}" +RUN_ID="${RUN_ID:-P0-OBS-002-grpc-bridge-$(date -u +%Y%m%dT%H%M%SZ)}" +WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-002}" +SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +LOCAL_UNIT="${LOCAL_UNIT:-$ROOT_DIR/ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service}" +APPLY_ACTIVE=0 +SSH_OPTS=( + -o BatchMode=yes + -o ConnectTimeout="$SSH_CONNECT_TIMEOUT" + -o ConnectionAttempts=1 + -o ServerAliveInterval=5 + -o ServerAliveCountMax=1 +) + +usage() { + cat <<'USAGE' +Usage: signoz-188-otlp-grpc-bridge.sh [mode] + + --check No-write prerequisite and conflict check. + --apply Install and start the user-systemd gRPC bridge. + --status Require enabled, active, allowlisted end-to-end TCP readiness. + --rollback Disable the new gRPC bridge and verify the listener is absent. + +This script never changes firewall, Docker, Kubernetes, database, secrets, +the existing HTTP bridge, or the application OTLP endpoint. +USAGE +} + +mode="${1:---check}" +case "$mode" in + --check|--apply|--status|--rollback) ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; exit 64 ;; +esac + +remote() { + ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "$@" +} + +receipt() { + local phase="$1" result="$2" detail="$3" + printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-188-otlp-grpc-compat-bridge","phase":"%s","result":"%s","detail":"%s"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail" +} + +rollback_runtime() { + set +e + remote "systemctl --user disable --now '$REMOTE_SERVICE' >/dev/null 2>&1 || true; '$REMOTE_SCRIPT' stop >/dev/null 2>&1 || true" + remote "! ss -ltn 2>/dev/null | awk '{print \$4}' | grep -Eq '${LISTEN_HOST}:${LISTEN_PORT}$'" + local rc=$? + set -e + if [ "$rc" -eq 0 ]; then + receipt rollback pass "grpc_unit_disabled_listener_absent_http_bridge_unchanged" + else + receipt rollback failed "grpc_listener_still_present" + fi + return "$rc" +} + +on_error() { + local rc=$? + trap - ERR + if [ "$APPLY_ACTIVE" -eq 1 ]; then + rollback_runtime || true + fi + receipt terminal failed "command_exit_$rc" + exit "$rc" +} +trap on_error ERR + +remote_payload() { + cat <<'REMOTE' +#!/usr/bin/env bash +set -Eeuo pipefail + +LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}" +LISTEN_PORT="${LISTEN_PORT:-24317}" +SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}" +UPSTREAM_HOST="${UPSTREAM_HOST:-192.168.0.110}" +UPSTREAM_PORT="${UPSTREAM_PORT:-4317}" +RUN_DIR="${RUN_DIR:-/home/ollama/run}" +LOG_DIR="${LOG_DIR:-/home/ollama/log}" +PID_FILE="$RUN_DIR/awoooi-signoz-otlp-grpc-bridge.pid" +LOG_FILE="$LOG_DIR/awoooi-signoz-otlp-grpc-bridge.log" + +is_running() { + [ -s "$PID_FILE" ] && kill -0 "$(sed -n '1p' "$PID_FILE")" 2>/dev/null +} + +tcp_open() { + timeout 3 bash -c "exec 3<>/dev/tcp/$1/$2" >/dev/null 2>&1 +} + +listener_present() { + ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "${LISTEN_HOST}:${LISTEN_PORT}$" +} + +status() { + running=0 + listener=0 + upstream=0 + if is_running; then running=1; fi + if listener_present; then listener=1; fi + if tcp_open "$UPSTREAM_HOST" "$UPSTREAM_PORT"; then upstream=1; fi + printf 'SIGNOZ_OTLP_GRPC_BRIDGE_STATUS running=%s listen=%s:%s listen_socket=%s source_range=%s upstream=%s:%s upstream_tcp=%s\n' \ + "$running" "$LISTEN_HOST" "$LISTEN_PORT" "$listener" "$SOURCE_RANGE" "$UPSTREAM_HOST" "$UPSTREAM_PORT" "$upstream" + [ "$running" -eq 1 ] && [ "$listener" -eq 1 ] && [ "$upstream" -eq 1 ] +} + +run_foreground() { + command -v socat >/dev/null + mkdir -p "$RUN_DIR" "$LOG_DIR" + if listener_present; then + echo "SIGNOZ_OTLP_GRPC_BRIDGE_BLOCKED port_busy=$LISTEN_PORT" >&2 + exit 2 + fi + echo "$$" >"$PID_FILE" + exec socat "TCP-LISTEN:${LISTEN_PORT},bind=${LISTEN_HOST},reuseaddr,fork,range=${SOURCE_RANGE}" \ + "TCP:${UPSTREAM_HOST}:${UPSTREAM_PORT}" >>"$LOG_FILE" 2>&1 +} + +stop() { + if is_running; then + kill "$(sed -n '1p' "$PID_FILE")" + for _ in 1 2 3 4 5; do + is_running || break + sleep 1 + done + fi + rm -f "$PID_FILE" + echo "SIGNOZ_OTLP_GRPC_BRIDGE_STOPPED" +} + +case "${1:-status}" in + check-upstream) tcp_open "$UPSTREAM_HOST" "$UPSTREAM_PORT" ;; + run) run_foreground ;; + status) status ;; + stop) stop ;; + *) exit 64 ;; +esac +REMOTE +} + +env_prefix="LISTEN_HOST=$LISTEN_HOST LISTEN_PORT=$LISTEN_PORT SOURCE_RANGE=$SOURCE_RANGE UPSTREAM_HOST=$UPSTREAM_HOST UPSTREAM_PORT=$UPSTREAM_PORT" + +verify_nodes() { + local node rc + for node in $VERIFY_NODES; do + rc=0 + ssh "${SSH_OPTS[@]}" "$node" \ + "timeout 5 bash -c 'exec 3<>/dev/tcp/${LISTEN_HOST}/${LISTEN_PORT}'" >/dev/null 2>&1 || rc=$? + printf 'SIGNOZ_OTLP_GRPC_BRIDGE_NODE_VERIFY node=%s tcp_rc=%s\n' "$node" "$rc" + [ "$rc" -eq 0 ] + done +} + +preflight() { + test -s "$LOCAL_UNIT" + remote "set -eu +command -v bash >/dev/null +command -v socat >/dev/null +command -v systemctl >/dev/null +linger=\$(loginctl show-user \"\$USER\" -p Linger --value 2>/dev/null || true) +upstream=0 +if timeout 3 bash -c 'exec 3<>/dev/tcp/${UPSTREAM_HOST}/${UPSTREAM_PORT}' >/dev/null 2>&1; then upstream=1; fi +busy=0 +if ss -ltn 2>/dev/null | awk '{print \$4}' | grep -Eq '${LISTEN_HOST}:${LISTEN_PORT}$'; then busy=1; fi +printf 'SIGNOZ_OTLP_GRPC_BRIDGE_CHECK upstream_tcp=%s listen_port_busy=%s source_range=%s linger=%s\n' \"\$upstream\" \"\$busy\" '$SOURCE_RANGE' \"\${linger:-no}\" +[ \"\$upstream\" = 1 ] && [ \"\${linger:-}\" = yes ]" +} + +case "$mode" in + --check) + receipt sensor pass "grpc_gap_source_and_canonical_110_upstream_selected" + receipt normalize pass "asset_signoz_188_otlp_grpc_compat_bridge" + receipt diff pass "missing_24317_listener_candidate_add_no_endpoint_change" + receipt decision allow "candidate_restore_incumbent_otlp_grpc_transport" + receipt policy allow "risk_medium_user_systemd_raw_tcp_allowlist_no_firewall_no_secret" + preflight + receipt check pass "upstream_4317_tcp_linger_yes_unit_source_present" + receipt terminal pass "check_mode_no_write" + ;; + --apply) + receipt sensor pass "grpc_gap_source_and_canonical_110_upstream_selected" + receipt normalize pass "asset_signoz_188_otlp_grpc_compat_bridge" + receipt diff pass "missing_24317_listener_candidate_add_no_endpoint_change" + receipt decision allow "restore_incumbent_otlp_grpc_transport_without_route_promotion" + receipt policy allow "risk_medium_user_systemd_raw_tcp_allowlist_no_firewall_no_secret" + preflight + receipt check pass "upstream_4317_tcp_linger_yes_unit_source_present" + APPLY_ACTIVE=1 + remote "mkdir -p /home/ollama/bin /home/ollama/run /home/ollama/log /home/ollama/.config/systemd/user; umask 077; tee '$REMOTE_SCRIPT.tmp' >/dev/null" < <(remote_payload) + scp "${SSH_OPTS[@]}" "$LOCAL_UNIT" "$TARGET_HOST:$REMOTE_UNIT_STAGE" + remote "set -eu +chmod 0755 '$REMOTE_SCRIPT.tmp' +chmod 0644 '$REMOTE_UNIT_STAGE' +mv '$REMOTE_SCRIPT.tmp' '$REMOTE_SCRIPT' +systemd-analyze --user verify '$REMOTE_UNIT_STAGE' >/dev/null +mv '$REMOTE_UNIT_STAGE' '$REMOTE_UNIT' +systemctl --user daemon-reload +systemctl --user enable --now '$REMOTE_SERVICE' +ready=0 +for _ in 1 2 3 4 5 6 7 8 9 10; do + if systemctl --user is-active --quiet '$REMOTE_SERVICE' && $env_prefix '$REMOTE_SCRIPT' status; then ready=1; break; fi + sleep 1 +done +[ \"\$ready\" = 1 ] +systemctl --user is-enabled --quiet '$REMOTE_SERVICE'" + verify_nodes + APPLY_ACTIVE=0 + receipt execute pass "grpc_bridge_unit_enabled_active_allowlist_120_121" + receipt verify pass "nodes_120_121_listener_and_upstream_4317_tcp_ready" + receipt learning partial "application_trace_metric_10m_post_verifier_pending" + receipt terminal pass "grpc_transport_restored_post_verifier_pending" + ;; + --status) + remote "set -eu +systemctl --user is-enabled --quiet '$REMOTE_SERVICE' +systemctl --user is-active --quiet '$REMOTE_SERVICE' +$env_prefix '$REMOTE_SCRIPT' status" + verify_nodes + receipt verify pass "grpc_unit_enabled_active_nodes_120_121_ready" + ;; + --rollback) + receipt decision allow "rollback_new_grpc_bridge_only_keep_http_bridge" + APPLY_ACTIVE=1 + rollback_runtime + APPLY_ACTIVE=0 + receipt terminal pass "grpc_bridge_rolled_back_http_bridge_unchanged" + ;; +esac diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py index c114926ab..bae9358e8 100644 --- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -125,6 +125,43 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: ) ) ) + grpc_bridge = _read( + root, "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh" + ) + grpc_unit = _read( + root, "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service" + ) + grpc_verifier = _read( + root, "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh" + ) + checks["grpc_transport_recovery_source_ready"] = all( + value in grpc_bridge + for value in ( + "LISTEN_PORT:-24317", + "UPSTREAM_PORT:-4317", + "SOURCE_RANGE:-192.168.0.120/31", + "systemctl --user enable --now", + "--rollback", + ) + ) and all( + value in grpc_unit + for value in ( + "LISTEN_PORT=24317", + "UPSTREAM_PORT=4317", + "SOURCE_RANGE=192.168.0.120/31", + "Restart=on-failure", + "NoNewPrivileges=true", + ) + ) and all( + value in grpc_verifier + for value in ( + "WINDOW_SECONDS:-600", + "serviceName={service:String}", + "signoz_metrics.samples_v4", + "Failed to export traces", + "restart_delta_0", + ) + ) static_recovery = _read( root, "scripts/reboot-recovery/signoz-110-static-ingest.sh" ) diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py index b7044245d..3dbb4d8ed 100644 --- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -19,6 +19,9 @@ SIGNAL_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-signals.sh" PARITY_SNAPSHOT = ROOT / "ops/signoz/p0-obs-002-three-signal-route-parity.yaml" OTEL_DAEMONSET = ROOT / "k8s/observability/otel-collector-daemonset.yaml" FILELOG_CANARY = ROOT / "ops/signoz/filelog-normalization-canary.yaml" +GRPC_BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh" +GRPC_UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service" +GRPC_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh" def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None: @@ -187,6 +190,68 @@ def test_filelog_canary_is_synthetic_bounded_and_has_no_network_exporter() -> No assert container["securityContext"]["readOnlyRootFilesystem"] is True +def test_grpc_bridge_is_independently_supervised_allowlisted_and_reversible() -> None: + bridge = GRPC_BRIDGE.read_text(encoding="utf-8") + unit = GRPC_UNIT.read_text(encoding="utf-8") + for expected in ( + 'LISTEN_PORT="${LISTEN_PORT:-24317}"', + 'SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"', + 'UPSTREAM_PORT="${UPSTREAM_PORT:-4317}"', + "--check|--apply|--status|--rollback", + "systemctl --user enable --now", + "rollback_runtime", + "verify_nodes", + "no_firewall_no_secret", + ): + assert expected in bridge + for forbidden in ("sudo", "iptables", "nft ", "docker ", "kubectl "): + assert forbidden not in bridge + + for expected in ( + "LISTEN_PORT=24317", + "SOURCE_RANGE=192.168.0.120/31", + "UPSTREAM_PORT=4317", + "ExecStartPre=/home/ollama/bin/awoooi-signoz-otlp-grpc-bridge.sh check-upstream", + "Restart=on-failure", + "NoNewPrivileges=true", + "ProtectSystem=strict", + "ProtectHome=read-only", + ): + assert expected in unit + assert "User=" not in unit + + +def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate() -> None: + text = GRPC_VERIFIER.read_text(encoding="utf-8") + for expected in ( + 'WINDOW_SECONDS="${WINDOW_SECONDS:-600}"', + "https://awoooi.wooo.work/api/v1/openapi.json", + "serviceName={service:String}", + "signoz_metrics.samples_v4", + "resource_attrs['service.name']={service:String}", + "Failed to export traces", + "Failed to export metrics", + "restart_delta_0", + "real_application_otlp_grpc_trace_metric_freshness_closed", + ): + assert expected in text + assert "SELECT body" not in text + assert "SELECT attributes" not in text + assert "/v1/traces" not in text + + contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8")) + grpc = contract["runtime_observations"]["application_otlp_grpc"] + assert grpc["root_cause"] == "missing_incumbent_grpc_bridge" + assert grpc["confidence"] == 0.99 + assert grpc["recent_awoooi_api_trace_count"] == 0 + assert grpc["http_only_verifier_blind_spot"] is True + assert grpc["source_fix"]["check_terminal"] == "pass" + assert ( + grpc["source_fix"]["runtime_apply_status"] + == "check_pass_pending_stable_cd_window" + ) + + def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None: env = os.environ.copy() env.pop("MANAGED_PROMOTION_RECEIPT", None) diff --git a/scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh b/scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh new file mode 100755 index 000000000..1ef963b0b --- /dev/null +++ b/scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Real application OTLP/gRPC post-verifier. It reads aggregates only. + +set -Eeuo pipefail + +TRACE_ID="${TRACE_ID:-P0-OBS-002}" +RUN_ID="${RUN_ID:-P0-OBS-002-grpc-runtime-$(date -u +%Y%m%dT%H%M%SZ)}" +WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-002}" +WINDOW_SECONDS="${WINDOW_SECONDS:-600}" +EXPECTED_PODS="${EXPECTED_PODS:-2}" +NAMESPACE="${NAMESPACE:-awoooi-prod}" +POD_SELECTOR="${POD_SELECTOR:-app=awoooi-api}" +SERVICE_NAME="${SERVICE_NAME:-awoooi-api}" +CANARY_URL="${CANARY_URL:-https://awoooi.wooo.work/api/v1/openapi.json}" +BRIDGE_HOST="${BRIDGE_HOST:-ollama@192.168.0.188}" +BRIDGE_SERVICE="${BRIDGE_SERVICE:-awoooi-signoz-otlp-grpc-bridge.service}" +SIGNOZ_HOST="${SIGNOZ_HOST:-wooo@192.168.0.110}" +SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ConnectionAttempts=1) + +case "$WINDOW_SECONDS" in + ''|*[!0-9]*) echo "WINDOW_SECONDS must be an integer" >&2; exit 64 ;; +esac +[ "$WINDOW_SECONDS" -ge 60 ] + +receipt() { + local phase="$1" result="$2" detail="$3" + printf '{"schema":"awoooi_otel_grpc_runtime_verifier_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-188-otlp-grpc-compat-bridge","phase":"%s","result":"%s","detail":"%s"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail" +} + +on_error() { + local rc=$? + receipt rollback required "run_signoz_188_otlp_grpc_bridge_rollback" + receipt terminal failed "command_exit_$rc" + exit "$rc" +} +trap on_error ERR + +pods=() +while IFS= read -r pod; do + [ -n "$pod" ] && pods[${#pods[@]}]="$pod" +done < <( + kubectl -n "$NAMESPACE" get pods -l "$POD_SELECTOR" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' +) +[ "${#pods[@]}" -eq "$EXPECTED_PODS" ] +restart_before=() +for pod in "${pods[@]}"; do + ready="$(kubectl -n "$NAMESPACE" get pod "$pod" -o jsonpath='{.status.containerStatuses[0].ready}')" + [ "$ready" = true ] + restart_before[${#restart_before[@]}]="$(kubectl -n "$NAMESPACE" get pod "$pod" -o jsonpath='{.status.containerStatuses[0].restartCount}')" +done +ssh "${SSH_OPTS[@]}" "$BRIDGE_HOST" "systemctl --user is-enabled --quiet '$BRIDGE_SERVICE' && systemctl --user is-active --quiet '$BRIDGE_SERVICE'" +receipt sensor pass "api_pods_${EXPECTED_PODS}_ready_grpc_bridge_enabled_active" +receipt normalize pass "service_${SERVICE_NAME}_aggregate_only" +receipt diff pass "incumbent_endpoint_unchanged_grpc_transport_present" +receipt decision allow "one_non_health_synthetic_request_then_bounded_observation" +receipt policy allow "risk_low_no_secret_no_payload_query_no_route_change" +receipt check pass "window_${WINDOW_SECONDS}s_restart_baseline_captured" + +http_code="$(curl -sS --max-time 15 -o /dev/null -w '%{http_code}' "$CANARY_URL")" +[ "$http_code" -ge 200 ] && [ "$http_code" -lt 400 ] +receipt execute pass "openapi_canary_http_${http_code}" + +sleep "$WINDOW_SECONDS" +error_count=0 +for i in "${!pods[@]}"; do + pod="${pods[$i]}" + restart_after="$(kubectl -n "$NAMESPACE" get pod "$pod" -o jsonpath='{.status.containerStatuses[0].restartCount}')" + [ "$restart_after" = "${restart_before[$i]}" ] + pod_errors="$(kubectl -n "$NAMESPACE" logs "$pod" --since="${WINDOW_SECONDS}s" 2>&1 | grep -Ec 'Failed to export traces|Failed to export metrics|Connection refused' || true)" + error_count=$((error_count + pod_errors)) +done +[ "$error_count" -eq 0 ] + +trace_count="$(ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \ + "docker exec signoz-clickhouse clickhouse-client --param_service='$SERVICE_NAME' --param_window='$WINDOW_SECONDS' --query \"SELECT count() FROM signoz_traces.signoz_index_v3 WHERE serviceName={service:String} AND timestamp >= now() - toIntervalSecond({window:UInt32})\"")" +metric_count="$(ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \ + "docker exec signoz-clickhouse clickhouse-client --param_service='$SERVICE_NAME' --param_window='$WINDOW_SECONDS' --query \"SELECT count() FROM signoz_metrics.samples_v4 WHERE unix_milli >= toUnixTimestamp64Milli(now64()) - ({window:UInt32} * 1000) AND fingerprint IN (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE resource_attrs['service.name']={service:String})\"")" +[ "$trace_count" -gt 0 ] +[ "$metric_count" -gt 0 ] +receipt verify pass "window_${WINDOW_SECONDS}s_export_errors_0_restart_delta_0_trace_count_${trace_count}_metric_count_${metric_count}" +receipt rollback not_required "independent_post_verifier_pass" +receipt learning partial "durable_gitea_main_and_km_writeback_pending" +receipt terminal pass "real_application_otlp_grpc_trace_metric_freshness_closed"