feat(observability): verify SigNoz three-signal route parity

2026-07-15 01:08:31 +08:00
parent bcc87ccd7e
commit fb0150d6ae
8 changed files with 557 additions and 2 deletions
--- a/docs/LOGBOOK.md
+++ b/docs/LOGBOOK.md
@@ -1,3 +1,28 @@
+## 2026-07-15 — P0-OBS-002 SigNoz canonical route Wave A
+
+**完成內容**：
+- 建立 organization / canonical route machine-readable contract；五個 producer 的 active endpoint 維持 `.188` incumbent，僅加入 `.110` challenger application / NetworkPolicy allow，未切流。
+- 新增 route-scoped logs / metrics / traces verifier。`P0-OBS-002-20260715T005626+0800` 以唯一 synthetic marker 驗證 incumbent bridge、`.120 -> .110` 與 `.121 -> .110` 三條路徑。
+- 三條路徑的 OTLP logs / metrics / traces 都回 HTTP `200`；ClickHouse 對每條路徑各自讀回 log `1`、metric series `1`、metric sample `1`、trace `1`。
+- bridge user-systemd supervisor 受控接管三次均 fail closed 並回滾至 legacy process + reboot cron；第三次 journal 證明 unit 曾啟動，但 executor readiness 判定過早。source 已補 takeover rollback flag 與最長 10 秒 readiness loop，本輪不再進行第四次 runtime 嘗試。
+- 產出 `ops/signoz/p0-obs-002-three-signal-route-parity.yaml`；明確把 synthetic route parity 與真實應用 trace freshness、producer queue gate、正式 promotion 分開。
+- 最新 10 分鐘 error fingerprint 證明 normalization noise 是固定成對錯誤：`.120` 為 `cri_parser regex_no_match=156` + `move3 missing_field=156`，`.121` 為 `126 + 126`；同時兩個 collector queue / send_failed 都是 `0`。
+- 第一版 filelog candidate 的 `on_error: send_quiet` 通過 `0.96.0 validate`，但隔離 Pod 在真正 build pipeline 時明確拒絕；兩次 canary 均已刪除且未碰正式 DaemonSet。source 已移除不相容欄位，改為 parser `body matches` 前置條件、保留 malformed 原始 body、以 `awoooi.log.parse_status=fallback_unparsed` 標記，並只在 `attributes.log` 存在時 move body；修正版刻意不做第三次 runtime canary，狀態維持 canary pending。
+
+**已跑驗證**：
+- SigNoz / OTEL / Event Exporter / API config 聚焦回歸：`43 passed`；其中 canonical route migration 為 `9 passed`。ruff、py_compile、四支 shell `bash -n`、canonical route preflight、三份 Kubernetes server dry-run、runner pressure guard、Gitea secret transport guard 與 `git diff --check` 通過。
+- 先前 route-specific log replay 已驗證 `.120/.121` HTTP `200`、ClickHouse exact marker `1`、15 秒 queue / failure delta `0`；本次三訊號 direct verifier不把該結果誤宣稱成 producer canary。
+- bridge rollback 後 `.120/.121 -> .188:24318` 回 HTTP `200`，legacy cron 已恢復，live user-systemd unit 已移除 / disabled。
+
+**仍維持**：
+- `setupCompleted=false`、organization 未初始化、`.110:4317/4318` direct ingress 暴露面尚未收斂、真實應用 traces 不新鮮、filelog normalization source fix 尚未 canary、bridge 尚未由 user-systemd supervision 接管。
+- synthetic 三訊號 parity 不等於正式 producer canary、Wave B promotion、bridge retirement、AI learning writeback 或 production closure；active route 仍是 incumbent。
+- organization/root bootstrap 涉及 durable credential；direct ingress policy 可能涉及 firewall / mTLS。這些維持獨立 high/critical gate，本輪未讀 secret、未改 firewall、未寫 raw DB、未碰 stateful volume。
+- Gitea feature branch / source 測試、main CD、deploy marker、production runtime 與 visible evidence 仍分層追蹤；不得由 feature push 或 synthetic readback 倒推 production 已部署。
+
+**下一步**：
+- 等 direct ingress policy 與 organization gate 具備受控包後，先做單一 producer canary，要求同一 run 的三訊號、queue/failure delta、normalization error rate、rollback與獨立 post-verifier 全綠；再逐批 promotion，最後才生成 bridge retirement receipt。
+
 ## 2026-07-14 — P0-OBS-001 SigNoz OTLP 收集與持久佇列修復

 **完成內容**：
--- a/k8s/observability/otel-collector-daemonset.yaml
+++ b/k8s/observability/otel-collector-daemonset.yaml
@@ -78,6 +78,9 @@ data:
          - type: regex_parser
            id: cri_parser
            regex: '^(?P<time>[^ ]+) (?P<stream>stdout|stderr) (?P<flags>[^ ]*) (?P<log>.*)$'
+            # Skip malformed/non-CRI input before parsing. It continues through
+            # the fallback path with its original body preserved.
+            if: 'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"'
            timestamp:
              parse_from: attributes.time
              # K3s CRI log: 2026-04-02T13:58:56.472854397+08:00
@@ -99,10 +102,23 @@ data:
          - type: move
            from: attributes.container
            to: resource["k8s.container.name"]
-          # log body 使用解析出的 log 內容
+          - type: add
+            id: mark_cri_parsed
+            field: attributes["awoooi.log.parse_status"]
+            value: parsed_cri
+            if: attributes.log != nil
+          # Parsed CRI entries use the extracted body. Malformed entries retain
+          # their original body instead of generating a second move error.
          - type: move
+            id: move_cri_body
            from: attributes.log
            to: body
+            if: attributes.log != nil
+          - type: add
+            id: mark_cri_fallback
+            field: attributes["awoooi.log.parse_status"]
+            value: fallback_unparsed
+            if: attributes["awoooi.log.parse_status"] == nil

    processors:
      # 批次發送: 降低網路開銷
--- a/ops/signoz/filelog-normalization-canary.yaml
+++ b/ops/signoz/filelog-normalization-canary.yaml
@@ -0,0 +1,98 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: p0-obs-002-filelog-normalization-canary
+  namespace: observability
+  labels:
+    awoooi.work-item: P0-OBS-002
+data:
+  config.yaml: |
+    receivers:
+      filelog:
+        include:
+          - /input/input.log
+        start_at: beginning
+        operators:
+          - type: regex_parser
+            id: cri_parser
+            regex: '^(?P<time>[^ ]+) (?P<stream>stdout|stderr) (?P<flags>[^ ]*) (?P<log>.*)$'
+            if: 'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"'
+            timestamp:
+              parse_from: attributes.time
+              layout_type: gotime
+              layout: '2006-01-02T15:04:05.999999999-07:00'
+          - type: add
+            id: mark_cri_parsed
+            field: attributes["awoooi.log.parse_status"]
+            value: parsed_cri
+            if: attributes.log != nil
+          - type: move
+            id: move_cri_body
+            from: attributes.log
+            to: body
+            if: attributes.log != nil
+          - type: add
+            id: mark_cri_fallback
+            field: attributes["awoooi.log.parse_status"]
+            value: fallback_unparsed
+            if: attributes["awoooi.log.parse_status"] == nil
+    exporters:
+      debug:
+        verbosity: detailed
+    service:
+      pipelines:
+        logs:
+          receivers: [filelog]
+          exporters: [debug]
+  input.log: |
+    2026-07-14T17:01:00.000000000+00:00 stdout F P0OBS_FILELOG_PARSED_CRI
+    P0OBS_FILELOG_FALLBACK_UNPARSED
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: p0-obs-002-filelog-normalization-canary
+  namespace: observability
+  labels:
+    awoooi.work-item: P0-OBS-002
+    app.kubernetes.io/name: p0-obs-002-filelog-normalization-canary
+spec:
+  automountServiceAccountToken: false
+  restartPolicy: Never
+  activeDeadlineSeconds: 90
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 10001
+    runAsGroup: 10001
+    fsGroup: 10001
+    seccompProfile:
+      type: RuntimeDefault
+  containers:
+    - name: otel-collector
+      image: 192.168.0.110:5000/awoooi/otel-collector-contrib@sha256:8edf386cdd1f4dc4393e34680e298dcc577296279febf243cf122155fa560c39
+      imagePullPolicy: IfNotPresent
+      args:
+        - --config=/etc/otel-canary/config.yaml
+      securityContext:
+        allowPrivilegeEscalation: false
+        readOnlyRootFilesystem: true
+        capabilities:
+          drop: ["ALL"]
+      resources:
+        requests:
+          cpu: 10m
+          memory: 32Mi
+        limits:
+          cpu: 100m
+          memory: 128Mi
+      volumeMounts:
+        - name: canary-data
+          mountPath: /etc/otel-canary
+          readOnly: true
+        - name: canary-data
+          mountPath: /input
+          readOnly: true
+  volumes:
+    - name: canary-data
+      configMap:
+        name: p0-obs-002-filelog-normalization-canary
--- a/ops/signoz/organization-canonical-route-migration.yaml
+++ b/ops/signoz/organization-canonical-route-migration.yaml
@@ -79,6 +79,7 @@ wave_a_gates:
  challenger_networkpolicy_allowed: true
  bridge_user_systemd_supervision_required: true
  route_specific_log_replay_required: true
+  route_specific_logs_metrics_traces_replay_required: true

 wave_b_gates:
  organization_managed_collector_gate: independent
@@ -95,7 +96,38 @@ runtime_observations:
  metrics_fresh: true
  traces_fresh: false
  bridge_supervisor: legacy_orphan_process
-  filelog_normalization: degraded
+  filelog_normalization: source_fix_canary_pending
+  filelog_normalization_error_readback:
+    window: 10m
+    observed_at: "2026-07-15T01:01:00+08:00"
+    otel_collector_120:
+      cri_parser_regex_no_match: 156
+      dependent_move_missing_field: 156
+    otel_collector_121:
+      cri_parser_regex_no_match: 126
+      dependent_move_missing_field: 126
+    transport_queue_size: 0
+    transport_send_failed_log_records: 0
+    source_fix:
+      preserves_unparsed_body: true
+      emits_parse_status_attribute: true
+      rejected_on_error_send_quiet_removed: true
+      parser_precondition_added: true
+      runtime_canary_passed: false
+  synthetic_three_signal_route_parity:
+    trace_id: P0-OBS-002-20260715T005626+0800
+    snapshot: ops/signoz/p0-obs-002-three-signal-route-parity.yaml
+    signals:
+      - logs
+      - metrics
+      - traces
+    incumbent_bridge: pass
+    direct_110_from_120: pass
+    direct_110_from_121: pass
+    exact_count_per_signal_and_route: 1
+    real_application_trace_freshness_inferred: false
+    producer_queue_failure_gate_inferred: false
+    promotion_authorized: false

 terminal:
  status: partial_degraded
--- a/ops/signoz/p0-obs-002-three-signal-route-parity.yaml
+++ b/ops/signoz/p0-obs-002-three-signal-route-parity.yaml
@@ -0,0 +1,110 @@
+schema: awoooi_signoz_three_signal_route_parity_v1
+generated_at: "2026-07-15T00:57:00+08:00"
+trace_id: P0-OBS-002-20260715T005626+0800
+work_item_id: P0-OBS-002
+route_parity_terminal: pass
+program_terminal: partial_degraded
+
+controlled_apply_contract:
+  sensor_source_receipt:
+    status: pass
+    refs:
+      - scripts/reboot-recovery/signoz-canonical-route-preflight.py
+      - ops/signoz/organization-canonical-route-migration.yaml
+  normalized_asset_identity:
+    status: pass
+    assets:
+      - canonical_id: signoz-188-otlp-compat-bridge
+        route_id: incumbent_bridge
+        source_node: 192.168.0.120
+      - canonical_id: signoz-110-otlp
+        route_id: direct_110_from_120
+        source_node: 192.168.0.120
+      - canonical_id: signoz-110-otlp
+        route_id: direct_110_from_121
+        source_node: 192.168.0.121
+  source_of_truth_diff:
+    status: pass
+    active_route_before: incumbent
+    active_route_after: incumbent
+    candidate_change: synthetic_route_parity_only
+  ai_decision:
+    status: pass
+    candidate_action: send_unique_synthetic_logs_metrics_traces_then_exact_readback
+    decision: bounded_low_risk_apply_allowed
+  risk_policy_decision:
+    status: pass
+    risk: low
+    synthetic_only: true
+    no_secret: true
+    no_raw_database_read: true
+    no_route_change: true
+    no_firewall_change: true
+  check_mode_receipt:
+    status: pass
+    source_ready: true
+    focused_tests_passed: 6
+    writes_performed: 0
+  idempotent_bounded_execution:
+    status: pass
+    uniqueness: one_marker_per_route_and_run
+    executions:
+      - run_id: P0-OBS-002-three-signal-incumbent-20260715T005626+0800
+        route_id: incumbent_bridge
+        source_node: 192.168.0.120
+        endpoint: http://192.168.0.188:24318
+        marker: P0OBS_SIGNAL_incumbent_bridge_P0_OBS_002_three_signal_incumbent_20260715T005626_0800_20260714T165633Z
+        logs_http: 200
+        metrics_http: 200
+        traces_http: 200
+      - run_id: P0-OBS-002-three-signal-direct120-20260715T005626+0800
+        route_id: direct_110_from_120
+        source_node: 192.168.0.120
+        endpoint: http://192.168.0.110:4318
+        marker: P0OBS_SIGNAL_direct_110_from_120_P0_OBS_002_three_signal_direct120_20260715T005626_0800_20260714T165652Z
+        logs_http: 200
+        metrics_http: 200
+        traces_http: 200
+      - run_id: P0-OBS-002-three-signal-direct121-20260715T005626+0800
+        route_id: direct_110_from_121
+        source_node: 192.168.0.121
+        endpoint: http://192.168.0.110:4318
+        marker: P0OBS_SIGNAL_direct_110_from_121_P0_OBS_002_three_signal_direct121_20260715T005626_0800_20260714T165652Z
+        logs_http: 200
+        metrics_http: 200
+        traces_http: 200
+  independent_post_verifier:
+    status: pass
+    mechanism: clickhouse_exact_synthetic_marker_query
+    logs_v2_exact_count_per_route: 1
+    metric_time_series_v4_exact_count_per_route: 1
+    metric_samples_v4_exact_count_per_route: 1
+    trace_index_v3_exact_count_per_route: 1
+  rollback_no_write_terminal:
+    status: pass
+    rollback_required: false
+    reason: append_only_synthetic_observation_no_active_route_or_business_state_change
+  learning_writeback:
+    status: partial
+    repository_refs:
+      - docs/LOGBOOK.md
+      - ops/signoz/organization-canonical-route-migration.yaml
+    durable_gitea_main_acknowledgement: pending
+    km_rag_mcp_playbook_acknowledgement: pending
+
+promotion:
+  authorized: false
+  real_application_trace_freshness_verified: false
+  producer_queue_failure_same_run_verified: false
+  filelog_normalization_gate_passed: false
+  direct_ingress_policy_gate_passed: false
+
+production_mutations:
+  active_route_changed: false
+  producer_endpoint_changed: false
+  bridge_retired: false
+  organization_initialized: false
+  firewall_changed: false
+  raw_database_write: false
+  stateful_volume_write: false
+  secret_read: false
--- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py
+++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py
@@ -106,8 +106,24 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
    )

    verifier = _read(root, "scripts/reboot-recovery/verify-signoz-otel-ingestion.sh")
+    signal_verifier = _read(
+        root, "scripts/reboot-recovery/verify-signoz-otel-signals.sh"
+    )
    checks["route_specific_verifier"] = (
        "OTLP_ENDPOINT" in verifier and "ROUTE_ID" in verifier
+        and all(
+            value in signal_verifier
+            for value in (
+                'send_signal logs',
+                'send_signal metrics',
+                'send_signal traces',
+                '/v1/$signal',
+                "signoz_logs.logs_v2",
+                "signoz_metrics.time_series_v4",
+                "signoz_metrics.samples_v4",
+                "signoz_traces.signoz_index_v3",
+            )
+        )
    )
    static_recovery = _read(
        root, "scripts/reboot-recovery/signoz-110-static-ingest.sh"
--- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
+++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
@@ -15,6 +15,10 @@ PREFLIGHT = ROOT / "scripts/reboot-recovery/signoz-canonical-route-preflight.py"
 BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-bridge.sh"
 STATIC = ROOT / "scripts/reboot-recovery/signoz-110-static-ingest.sh"
 UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-bridge.service"
+SIGNAL_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-signals.sh"
+PARITY_SNAPSHOT = ROOT / "ops/signoz/p0-obs-002-three-signal-route-parity.yaml"
+OTEL_DAEMONSET = ROOT / "k8s/observability/otel-collector-daemonset.yaml"
+FILELOG_CANARY = ROOT / "ops/signoz/filelog-normalization-canary.yaml"


 def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
@@ -82,6 +86,95 @@ def test_bridge_unit_is_user_scoped_restarting_and_hardened() -> None:
    assert "sudo" not in text


+def test_three_signal_verifier_is_route_scoped_and_exact() -> None:
+    text = SIGNAL_VERIFIER.read_text(encoding="utf-8")
+    for expected in (
+        "OTLP_BASE_ENDPOINT",
+        "ROUTE_ID",
+        "/v1/$signal",
+        "resourceLogs",
+        "resourceMetrics",
+        "resourceSpans",
+        "signoz_logs.logs_v2",
+        "signoz_metrics.time_series_v4",
+        "signoz_metrics.samples_v4",
+        "signoz_traces.signoz_index_v3",
+        "exact_log_1_metric_series_1_metric_sample_1_trace_1",
+    ):
+        assert expected in text
+    assert "github.com" not in text
+    assert "SIGNOZ-API-KEY" not in text
+    assert "Authorization:" not in text
+
+
+def test_three_signal_snapshot_keeps_synthetic_parity_separate_from_promotion() -> None:
+    contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
+    parity = contract["runtime_observations"]["synthetic_three_signal_route_parity"]
+    assert parity["incumbent_bridge"] == "pass"
+    assert parity["direct_110_from_120"] == "pass"
+    assert parity["direct_110_from_121"] == "pass"
+    assert parity["exact_count_per_signal_and_route"] == 1
+    assert parity["real_application_trace_freshness_inferred"] is False
+    assert parity["producer_queue_failure_gate_inferred"] is False
+    assert parity["promotion_authorized"] is False
+
+    snapshot = yaml.safe_load(PARITY_SNAPSHOT.read_text(encoding="utf-8"))
+    assert snapshot["trace_id"] == parity["trace_id"]
+    assert snapshot["route_parity_terminal"] == "pass"
+    assert snapshot["program_terminal"] == "partial_degraded"
+    assert snapshot["controlled_apply_contract"]["learning_writeback"]["status"] == "partial"
+    assert snapshot["promotion"]["authorized"] is False
+    assert snapshot["production_mutations"]["active_route_changed"] is False
+    assert snapshot["production_mutations"]["secret_read"] is False
+
+
+def test_filelog_fallback_preserves_body_and_exposes_normalization_coverage() -> None:
+    text = OTEL_DAEMONSET.read_text(encoding="utf-8")
+    assert "id: cri_parser" in text
+    assert "on_error: send_quiet" not in text
+    assert 'if: \'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"\'' in text
+    assert "id: mark_cri_parsed" in text
+    assert 'value: parsed_cri' in text
+    assert "id: move_cri_body" in text
+    assert "if: attributes.log != nil" in text
+    assert "id: mark_cri_fallback" in text
+    assert 'value: fallback_unparsed' in text
+    assert 'if: attributes["awoooi.log.parse_status"] == nil' in text
+
+    contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
+    observation = contract["runtime_observations"][
+        "filelog_normalization_error_readback"
+    ]
+    assert observation["transport_queue_size"] == 0
+    assert observation["transport_send_failed_log_records"] == 0
+    assert observation["source_fix"]["preserves_unparsed_body"] is True
+    assert observation["source_fix"]["runtime_canary_passed"] is False
+
+
+def test_filelog_canary_is_synthetic_bounded_and_has_no_network_exporter() -> None:
+    text = FILELOG_CANARY.read_text(encoding="utf-8")
+    documents = list(yaml.safe_load_all(text))
+    config_map, pod = documents
+    assert config_map["kind"] == "ConfigMap"
+    assert "P0OBS_FILELOG_PARSED_CRI" in config_map["data"]["input.log"]
+    assert "P0OBS_FILELOG_FALLBACK_UNPARSED" in config_map["data"]["input.log"]
+    config = config_map["data"]["config.yaml"]
+    assert "on_error: send_quiet" not in config
+    assert 'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"' in config
+    assert "fallback_unparsed" in config
+    assert "debug:" in config
+    assert "otlp" not in config.lower()
+
+    spec = pod["spec"]
+    assert spec["automountServiceAccountToken"] is False
+    assert spec["restartPolicy"] == "Never"
+    assert spec["activeDeadlineSeconds"] == 90
+    container = spec["containers"][0]
+    assert "@sha256:" in container["image"]
+    assert container["securityContext"]["allowPrivilegeEscalation"] is False
+    assert container["securityContext"]["readOnlyRootFilesystem"] is True
+
+
 def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None:
    env = os.environ.copy()
    env.pop("MANAGED_PROMOTION_RECEIPT", None)
--- a/scripts/reboot-recovery/verify-signoz-otel-signals.sh
+++ b/scripts/reboot-recovery/verify-signoz-otel-signals.sh
@@ -0,0 +1,165 @@
+#!/usr/bin/env bash
+# Route-scoped OTLP logs/metrics/traces verifier for P0-OBS-002.
+# It writes only unique synthetic markers and reads back only those markers.
+
+set -Eeuo pipefail
+
+TRACE_ID="${TRACE_ID:-P0-OBS-002}"
+RUN_ID="${RUN_ID:-P0-OBS-002-$(date -u +%Y%m%dT%H%M%SZ)}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-002}"
+ROUTE_ID="${ROUTE_ID:-incumbent_bridge}"
+OTLP_BASE_ENDPOINT="${OTLP_BASE_ENDPOINT:-http://192.168.0.188:24318}"
+SOURCE_NODE="${SOURCE_NODE:-wooo@192.168.0.120}"
+SIGNOZ_HOST="${SIGNOZ_HOST:-wooo@192.168.0.110}"
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ConnectionAttempts=1)
+OTLP_BASE_ENDPOINT="${OTLP_BASE_ENDPOINT%/}"
+marker="P0OBS_SIGNAL_${ROUTE_ID//[^A-Za-z0-9_]/_}_${RUN_ID//[^A-Za-z0-9_]/_}_$(date -u +%Y%m%dT%H%M%SZ)"
+trace_id="$(printf '%s' "$marker" | shasum -a 256 | awk '{print substr($1,1,32)}')"
+span_id="${trace_id:0:16}"
+timestamp_ns="$(ssh "${SSH_OPTS[@]}" "$SOURCE_NODE" 'date +%s%N')"
+
+receipt() {
+  local phase="$1" result="$2" detail="$3"
+  printf '{"schema":"awoooi_otel_signal_parity_verifier_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","route_id":"%s","phase":"%s","result":"%s","detail":"%s"}\n' \
+    "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$ROUTE_ID" "$phase" "$result" "$detail"
+}
+
+on_error() {
+  local rc=$?
+  receipt terminal failed "command_exit_$rc"
+  exit "$rc"
+}
+trap on_error ERR
+
+payload() {
+  local signal="$1"
+  python3 - "$signal" "$marker" "$ROUTE_ID" "$trace_id" "$span_id" "$timestamp_ns" <<'PY'
+import json
+import sys
+
+signal, marker, route_id, trace_id, span_id, timestamp = sys.argv[1:]
+resource = {
+    "attributes": [
+        {"key": "service.name", "value": {"stringValue": "awoooi-p0-obs-signal-verifier"}},
+        {"key": "awoooi.route.id", "value": {"stringValue": route_id}},
+    ]
+}
+if signal == "logs":
+    body = {
+        "resourceLogs": [{
+            "resource": resource,
+            "scopeLogs": [{
+                "scope": {"name": "awoooi.obs.signal-verifier"},
+                "logRecords": [{
+                    "timeUnixNano": timestamp,
+                    "severityNumber": 9,
+                    "severityText": "INFO",
+                    "body": {"stringValue": marker},
+                }],
+            }],
+        }]
+    }
+elif signal == "metrics":
+    body = {
+        "resourceMetrics": [{
+            "resource": resource,
+            "scopeMetrics": [{
+                "scope": {"name": "awoooi.obs.signal-verifier"},
+                "metrics": [{
+                    "name": "awoooi_p0_obs_route_canary",
+                    "description": "P0-OBS-002 route canary",
+                    "unit": "1",
+                    "gauge": {"dataPoints": [{
+                        "attributes": [{
+                            "key": "awoooi.marker",
+                            "value": {"stringValue": marker},
+                        }],
+                        "timeUnixNano": timestamp,
+                        "asDouble": 1.0,
+                    }]},
+                }],
+            }],
+        }]
+    }
+elif signal == "traces":
+    body = {
+        "resourceSpans": [{
+            "resource": resource,
+            "scopeSpans": [{
+                "scope": {"name": "awoooi.obs.signal-verifier"},
+                "spans": [{
+                    "traceId": trace_id,
+                    "spanId": span_id,
+                    "name": marker,
+                    "kind": 1,
+                    "startTimeUnixNano": timestamp,
+                    "endTimeUnixNano": str(int(timestamp) + 1_000_000),
+                    "status": {"code": 1},
+                }],
+            }],
+        }]
+    }
+else:
+    raise SystemExit(64)
+print(json.dumps(body, separators=(",", ":")))
+PY
+}
+
+send_signal() {
+  local signal="$1" endpoint code body
+  endpoint="$OTLP_BASE_ENDPOINT/v1/$signal"
+  body="$(payload "$signal")"
+  code="$(printf '%s' "$body" | ssh "${SSH_OPTS[@]}" "$SOURCE_NODE" \
+    "curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary @- '$endpoint'")"
+  printf 'SIGNOZ_SIGNAL_SEND route=%s signal=%s code=%s\n' "$ROUTE_ID" "$signal" "$code"
+  [ "$code" = 200 ]
+}
+
+query_log_count() {
+  ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
+    "docker exec signoz-clickhouse clickhouse-client --param_marker='$marker' --param_route='$ROUTE_ID' --query \"SELECT count() FROM signoz_logs.logs_v2 WHERE body={marker:String} AND resources_string['awoooi.route.id']={route:String}\""
+}
+
+query_metric_series_count() {
+  ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
+    "docker exec signoz-clickhouse clickhouse-client --param_marker='$marker' --param_route='$ROUTE_ID' --query \"SELECT count() FROM signoz_metrics.time_series_v4 WHERE metric_name='awoooi_p0_obs_route_canary' AND attrs['awoooi.marker']={marker:String} AND resource_attrs['awoooi.route.id']={route:String}\""
+}
+
+query_metric_sample_count() {
+  ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
+    "docker exec signoz-clickhouse clickhouse-client --param_marker='$marker' --param_route='$ROUTE_ID' --query \"SELECT count() FROM signoz_metrics.samples_v4 WHERE metric_name='awoooi_p0_obs_route_canary' AND fingerprint IN (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name='awoooi_p0_obs_route_canary' AND attrs['awoooi.marker']={marker:String} AND resource_attrs['awoooi.route.id']={route:String})\""
+}
+
+query_trace_count() {
+  ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
+    "docker exec signoz-clickhouse clickhouse-client --param_marker='$marker' --param_route='$ROUTE_ID' --query \"SELECT count() FROM signoz_traces.signoz_index_v3 WHERE name={marker:String} AND resources_string['awoooi.route.id']={route:String}\""
+}
+
+receipt sensor pass "route_${ROUTE_ID}_endpoint_selected"
+send_signal logs
+send_signal metrics
+send_signal traces
+receipt execute pass "logs_metrics_traces_http_200_marker_${marker}"
+
+log_count=0
+metric_series_count=0
+metric_sample_count=0
+trace_count=0
+for _ in $(seq 1 20); do
+  log_count="$(query_log_count)"
+  metric_series_count="$(query_metric_series_count)"
+  metric_sample_count="$(query_metric_sample_count)"
+  trace_count="$(query_trace_count)"
+  if [ "$log_count" -eq 1 ] && [ "$metric_series_count" -eq 1 ] && \
+     [ "$metric_sample_count" -eq 1 ] && [ "$trace_count" -eq 1 ]; then
+    break
+  fi
+  sleep 2
+done
+
+[ "$log_count" -eq 1 ]
+[ "$metric_series_count" -eq 1 ]
+[ "$metric_sample_count" -eq 1 ]
+[ "$trace_count" -eq 1 ]
+receipt verify pass "exact_log_1_metric_series_1_metric_sample_1_trace_1"
+receipt terminal pass "route_${ROUTE_ID}_three_signal_readback_closed"