feat(observability): verify SigNoz three-signal route parity

2026-07-15 01:08:31 +08:00
parent bcc87ccd7e
commit fb0150d6ae
8 changed files with 557 additions and 2 deletions
--- a/ops/signoz/filelog-normalization-canary.yaml
+++ b/ops/signoz/filelog-normalization-canary.yaml
@@ -0,0 +1,98 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: p0-obs-002-filelog-normalization-canary
+  namespace: observability
+  labels:
+    awoooi.work-item: P0-OBS-002
+data:
+  config.yaml: |
+    receivers:
+      filelog:
+        include:
+          - /input/input.log
+        start_at: beginning
+        operators:
+          - type: regex_parser
+            id: cri_parser
+            regex: '^(?P<time>[^ ]+) (?P<stream>stdout|stderr) (?P<flags>[^ ]*) (?P<log>.*)$'
+            if: 'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"'
+            timestamp:
+              parse_from: attributes.time
+              layout_type: gotime
+              layout: '2006-01-02T15:04:05.999999999-07:00'
+          - type: add
+            id: mark_cri_parsed
+            field: attributes["awoooi.log.parse_status"]
+            value: parsed_cri
+            if: attributes.log != nil
+          - type: move
+            id: move_cri_body
+            from: attributes.log
+            to: body
+            if: attributes.log != nil
+          - type: add
+            id: mark_cri_fallback
+            field: attributes["awoooi.log.parse_status"]
+            value: fallback_unparsed
+            if: attributes["awoooi.log.parse_status"] == nil
+    exporters:
+      debug:
+        verbosity: detailed
+    service:
+      pipelines:
+        logs:
+          receivers: [filelog]
+          exporters: [debug]
+  input.log: |
+    2026-07-14T17:01:00.000000000+00:00 stdout F P0OBS_FILELOG_PARSED_CRI
+    P0OBS_FILELOG_FALLBACK_UNPARSED
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: p0-obs-002-filelog-normalization-canary
+  namespace: observability
+  labels:
+    awoooi.work-item: P0-OBS-002
+    app.kubernetes.io/name: p0-obs-002-filelog-normalization-canary
+spec:
+  automountServiceAccountToken: false
+  restartPolicy: Never
+  activeDeadlineSeconds: 90
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 10001
+    runAsGroup: 10001
+    fsGroup: 10001
+    seccompProfile:
+      type: RuntimeDefault
+  containers:
+    - name: otel-collector
+      image: 192.168.0.110:5000/awoooi/otel-collector-contrib@sha256:8edf386cdd1f4dc4393e34680e298dcc577296279febf243cf122155fa560c39
+      imagePullPolicy: IfNotPresent
+      args:
+        - --config=/etc/otel-canary/config.yaml
+      securityContext:
+        allowPrivilegeEscalation: false
+        readOnlyRootFilesystem: true
+        capabilities:
+          drop: ["ALL"]
+      resources:
+        requests:
+          cpu: 10m
+          memory: 32Mi
+        limits:
+          cpu: 100m
+          memory: 128Mi
+      volumeMounts:
+        - name: canary-data
+          mountPath: /etc/otel-canary
+          readOnly: true
+        - name: canary-data
+          mountPath: /input
+          readOnly: true
+  volumes:
+    - name: canary-data
+      configMap:
+        name: p0-obs-002-filelog-normalization-canary
--- a/ops/signoz/organization-canonical-route-migration.yaml
+++ b/ops/signoz/organization-canonical-route-migration.yaml
@@ -79,6 +79,7 @@ wave_a_gates:
  challenger_networkpolicy_allowed: true
  bridge_user_systemd_supervision_required: true
  route_specific_log_replay_required: true
+  route_specific_logs_metrics_traces_replay_required: true

 wave_b_gates:
  organization_managed_collector_gate: independent
@@ -95,7 +96,38 @@ runtime_observations:
  metrics_fresh: true
  traces_fresh: false
  bridge_supervisor: legacy_orphan_process
-  filelog_normalization: degraded
+  filelog_normalization: source_fix_canary_pending
+  filelog_normalization_error_readback:
+    window: 10m
+    observed_at: "2026-07-15T01:01:00+08:00"
+    otel_collector_120:
+      cri_parser_regex_no_match: 156
+      dependent_move_missing_field: 156
+    otel_collector_121:
+      cri_parser_regex_no_match: 126
+      dependent_move_missing_field: 126
+    transport_queue_size: 0
+    transport_send_failed_log_records: 0
+    source_fix:
+      preserves_unparsed_body: true
+      emits_parse_status_attribute: true
+      rejected_on_error_send_quiet_removed: true
+      parser_precondition_added: true
+      runtime_canary_passed: false
+  synthetic_three_signal_route_parity:
+    trace_id: P0-OBS-002-20260715T005626+0800
+    snapshot: ops/signoz/p0-obs-002-three-signal-route-parity.yaml
+    signals:
+      - logs
+      - metrics
+      - traces
+    incumbent_bridge: pass
+    direct_110_from_120: pass
+    direct_110_from_121: pass
+    exact_count_per_signal_and_route: 1
+    real_application_trace_freshness_inferred: false
+    producer_queue_failure_gate_inferred: false
+    promotion_authorized: false

 terminal:
  status: partial_degraded
--- a/ops/signoz/p0-obs-002-three-signal-route-parity.yaml
+++ b/ops/signoz/p0-obs-002-three-signal-route-parity.yaml
@@ -0,0 +1,110 @@
+schema: awoooi_signoz_three_signal_route_parity_v1
+generated_at: "2026-07-15T00:57:00+08:00"
+trace_id: P0-OBS-002-20260715T005626+0800
+work_item_id: P0-OBS-002
+route_parity_terminal: pass
+program_terminal: partial_degraded
+
+controlled_apply_contract:
+  sensor_source_receipt:
+    status: pass
+    refs:
+      - scripts/reboot-recovery/signoz-canonical-route-preflight.py
+      - ops/signoz/organization-canonical-route-migration.yaml
+  normalized_asset_identity:
+    status: pass
+    assets:
+      - canonical_id: signoz-188-otlp-compat-bridge
+        route_id: incumbent_bridge
+        source_node: 192.168.0.120
+      - canonical_id: signoz-110-otlp
+        route_id: direct_110_from_120
+        source_node: 192.168.0.120
+      - canonical_id: signoz-110-otlp
+        route_id: direct_110_from_121
+        source_node: 192.168.0.121
+  source_of_truth_diff:
+    status: pass
+    active_route_before: incumbent
+    active_route_after: incumbent
+    candidate_change: synthetic_route_parity_only
+  ai_decision:
+    status: pass
+    candidate_action: send_unique_synthetic_logs_metrics_traces_then_exact_readback
+    decision: bounded_low_risk_apply_allowed
+  risk_policy_decision:
+    status: pass
+    risk: low
+    synthetic_only: true
+    no_secret: true
+    no_raw_database_read: true
+    no_route_change: true
+    no_firewall_change: true
+  check_mode_receipt:
+    status: pass
+    source_ready: true
+    focused_tests_passed: 6
+    writes_performed: 0
+  idempotent_bounded_execution:
+    status: pass
+    uniqueness: one_marker_per_route_and_run
+    executions:
+      - run_id: P0-OBS-002-three-signal-incumbent-20260715T005626+0800
+        route_id: incumbent_bridge
+        source_node: 192.168.0.120
+        endpoint: http://192.168.0.188:24318
+        marker: P0OBS_SIGNAL_incumbent_bridge_P0_OBS_002_three_signal_incumbent_20260715T005626_0800_20260714T165633Z
+        logs_http: 200
+        metrics_http: 200
+        traces_http: 200
+      - run_id: P0-OBS-002-three-signal-direct120-20260715T005626+0800
+        route_id: direct_110_from_120
+        source_node: 192.168.0.120
+        endpoint: http://192.168.0.110:4318
+        marker: P0OBS_SIGNAL_direct_110_from_120_P0_OBS_002_three_signal_direct120_20260715T005626_0800_20260714T165652Z
+        logs_http: 200
+        metrics_http: 200
+        traces_http: 200
+      - run_id: P0-OBS-002-three-signal-direct121-20260715T005626+0800
+        route_id: direct_110_from_121
+        source_node: 192.168.0.121
+        endpoint: http://192.168.0.110:4318
+        marker: P0OBS_SIGNAL_direct_110_from_121_P0_OBS_002_three_signal_direct121_20260715T005626_0800_20260714T165652Z
+        logs_http: 200
+        metrics_http: 200
+        traces_http: 200
+  independent_post_verifier:
+    status: pass
+    mechanism: clickhouse_exact_synthetic_marker_query
+    logs_v2_exact_count_per_route: 1
+    metric_time_series_v4_exact_count_per_route: 1
+    metric_samples_v4_exact_count_per_route: 1
+    trace_index_v3_exact_count_per_route: 1
+  rollback_no_write_terminal:
+    status: pass
+    rollback_required: false
+    reason: append_only_synthetic_observation_no_active_route_or_business_state_change
+  learning_writeback:
+    status: partial
+    repository_refs:
+      - docs/LOGBOOK.md
+      - ops/signoz/organization-canonical-route-migration.yaml
+    durable_gitea_main_acknowledgement: pending
+    km_rag_mcp_playbook_acknowledgement: pending
+
+promotion:
+  authorized: false
+  real_application_trace_freshness_verified: false
+  producer_queue_failure_same_run_verified: false
+  filelog_normalization_gate_passed: false
+  direct_ingress_policy_gate_passed: false
+
+production_mutations:
+  active_route_changed: false
+  producer_endpoint_changed: false
+  bridge_retired: false
+  organization_initialized: false
+  firewall_changed: false
+  raw_database_write: false
+  stateful_volume_write: false
+  secret_read: false