wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

feat(observability): verify SigNoz three-signal route parity

Browse Source
This commit is contained in:
ogt
2026-07-15 01:08:31 +08:00
parent bcc87ccd7e
commit fb0150d6ae
8 changed files with 557 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
scripts/reboot-recovery/signoz-canonical-route-preflight.py
View File

@@ -106,8 +106,24 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
)
verifier = _read(root, "scripts/reboot-recovery/verify-signoz-otel-ingestion.sh")
signal_verifier = _read(
root, "scripts/reboot-recovery/verify-signoz-otel-signals.sh"
)
checks["route_specific_verifier"] = (
"OTLP_ENDPOINT" in verifier and "ROUTE_ID" in verifier
and all(
value in signal_verifier
for value in (
'send_signal logs',
'send_signal metrics',
'send_signal traces',
'/v1/$signal',
"signoz_logs.logs_v2",
"signoz_metrics.time_series_v4",
"signoz_metrics.samples_v4",
"signoz_traces.signoz_index_v3",
)
)
)
static_recovery = _read(
root, "scripts/reboot-recovery/signoz-110-static-ingest.sh"

93
scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
View File

@@ -15,6 +15,10 @@ PREFLIGHT = ROOT / "scripts/reboot-recovery/signoz-canonical-route-preflight.py"
BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-bridge.sh"
STATIC = ROOT / "scripts/reboot-recovery/signoz-110-static-ingest.sh"
UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-bridge.service"
SIGNAL_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-signals.sh"
PARITY_SNAPSHOT = ROOT / "ops/signoz/p0-obs-002-three-signal-route-parity.yaml"
OTEL_DAEMONSET = ROOT / "k8s/observability/otel-collector-daemonset.yaml"
FILELOG_CANARY = ROOT / "ops/signoz/filelog-normalization-canary.yaml"
def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
@@ -82,6 +86,95 @@ def test_bridge_unit_is_user_scoped_restarting_and_hardened() -> None:
assert "sudo" not in text
def test_three_signal_verifier_is_route_scoped_and_exact() -> None:
text = SIGNAL_VERIFIER.read_text(encoding="utf-8")
for expected in (
"OTLP_BASE_ENDPOINT",
"ROUTE_ID",
"/v1/$signal",
"resourceLogs",
"resourceMetrics",
"resourceSpans",
"signoz_logs.logs_v2",
"signoz_metrics.time_series_v4",
"signoz_metrics.samples_v4",
"signoz_traces.signoz_index_v3",
"exact_log_1_metric_series_1_metric_sample_1_trace_1",
):
assert expected in text
assert "github.com" not in text
assert "SIGNOZ-API-KEY" not in text
assert "Authorization:" not in text
def test_three_signal_snapshot_keeps_synthetic_parity_separate_from_promotion() -> None:
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
parity = contract["runtime_observations"]["synthetic_three_signal_route_parity"]
assert parity["incumbent_bridge"] == "pass"
assert parity["direct_110_from_120"] == "pass"
assert parity["direct_110_from_121"] == "pass"
assert parity["exact_count_per_signal_and_route"] == 1
assert parity["real_application_trace_freshness_inferred"] is False
assert parity["producer_queue_failure_gate_inferred"] is False
assert parity["promotion_authorized"] is False
snapshot = yaml.safe_load(PARITY_SNAPSHOT.read_text(encoding="utf-8"))
assert snapshot["trace_id"] == parity["trace_id"]
assert snapshot["route_parity_terminal"] == "pass"
assert snapshot["program_terminal"] == "partial_degraded"
assert snapshot["controlled_apply_contract"]["learning_writeback"]["status"] == "partial"
assert snapshot["promotion"]["authorized"] is False
assert snapshot["production_mutations"]["active_route_changed"] is False
assert snapshot["production_mutations"]["secret_read"] is False
def test_filelog_fallback_preserves_body_and_exposes_normalization_coverage() -> None:
text = OTEL_DAEMONSET.read_text(encoding="utf-8")
assert "id: cri_parser" in text
assert "on_error: send_quiet" not in text
assert 'if: \'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"\'' in text
assert "id: mark_cri_parsed" in text
assert 'value: parsed_cri' in text
assert "id: move_cri_body" in text
assert "if: attributes.log != nil" in text
assert "id: mark_cri_fallback" in text
assert 'value: fallback_unparsed' in text
assert 'if: attributes["awoooi.log.parse_status"] == nil' in text
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
observation = contract["runtime_observations"][
"filelog_normalization_error_readback"
]
assert observation["transport_queue_size"] == 0
assert observation["transport_send_failed_log_records"] == 0
assert observation["source_fix"]["preserves_unparsed_body"] is True
assert observation["source_fix"]["runtime_canary_passed"] is False
def test_filelog_canary_is_synthetic_bounded_and_has_no_network_exporter() -> None:
text = FILELOG_CANARY.read_text(encoding="utf-8")
documents = list(yaml.safe_load_all(text))
config_map, pod = documents
assert config_map["kind"] == "ConfigMap"
assert "P0OBS_FILELOG_PARSED_CRI" in config_map["data"]["input.log"]
assert "P0OBS_FILELOG_FALLBACK_UNPARSED" in config_map["data"]["input.log"]
config = config_map["data"]["config.yaml"]
assert "on_error: send_quiet" not in config
assert 'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"' in config
assert "fallback_unparsed" in config
assert "debug:" in config
assert "otlp" not in config.lower()
spec = pod["spec"]
assert spec["automountServiceAccountToken"] is False
assert spec["restartPolicy"] == "Never"
assert spec["activeDeadlineSeconds"] == 90
container = spec["containers"][0]
assert "@sha256:" in container["image"]
assert container["securityContext"]["allowPrivilegeEscalation"] is False
assert container["securityContext"]["readOnlyRootFilesystem"] is True
def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None:
env = os.environ.copy()
env.pop("MANAGED_PROMOTION_RECEIPT", None)

165
scripts/reboot-recovery/verify-signoz-otel-signals.sh Executable file
View File

@@ -0,0 +1,165 @@
#!/usr/bin/env bash
# Route-scoped OTLP logs/metrics/traces verifier for P0-OBS-002.
# It writes only unique synthetic markers and reads back only those markers.
set -Eeuo pipefail
TRACE_ID="${TRACE_ID:-P0-OBS-002}"
RUN_ID="${RUN_ID:-P0-OBS-002-$(date -u +%Y%m%dT%H%M%SZ)}"
WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-002}"
ROUTE_ID="${ROUTE_ID:-incumbent_bridge}"
OTLP_BASE_ENDPOINT="${OTLP_BASE_ENDPOINT:-http://192.168.0.188:24318}"
SOURCE_NODE="${SOURCE_NODE:-wooo@192.168.0.120}"
SIGNOZ_HOST="${SIGNOZ_HOST:-wooo@192.168.0.110}"
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ConnectionAttempts=1)
OTLP_BASE_ENDPOINT="${OTLP_BASE_ENDPOINT%/}"
marker="P0OBS_SIGNAL_${ROUTE_ID//[^A-Za-z0-9_]/_}_${RUN_ID//[^A-Za-z0-9_]/_}_$(date -u +%Y%m%dT%H%M%SZ)"
trace_id="$(printf '%s' "$marker" | shasum -a 256 | awk '{print substr($1,1,32)}')"
span_id="${trace_id:0:16}"
timestamp_ns="$(ssh "${SSH_OPTS[@]}" "$SOURCE_NODE" 'date +%s%N')"
receipt() {
local phase="$1" result="$2" detail="$3"
printf '{"schema":"awoooi_otel_signal_parity_verifier_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","route_id":"%s","phase":"%s","result":"%s","detail":"%s"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$ROUTE_ID" "$phase" "$result" "$detail"
}
on_error() {
local rc=$?
receipt terminal failed "command_exit_$rc"
exit "$rc"
}
trap on_error ERR
payload() {
local signal="$1"
python3 - "$signal" "$marker" "$ROUTE_ID" "$trace_id" "$span_id" "$timestamp_ns" <<'PY'
import json
import sys
signal, marker, route_id, trace_id, span_id, timestamp = sys.argv[1:]
resource = {
"attributes": [
{"key": "service.name", "value": {"stringValue": "awoooi-p0-obs-signal-verifier"}},
{"key": "awoooi.route.id", "value": {"stringValue": route_id}},
]
}
if signal == "logs":
body = {
"resourceLogs": [{
"resource": resource,
"scopeLogs": [{
"scope": {"name": "awoooi.obs.signal-verifier"},
"logRecords": [{
"timeUnixNano": timestamp,
"severityNumber": 9,
"severityText": "INFO",
"body": {"stringValue": marker},
}],
}],
}]
}
elif signal == "metrics":
body = {
"resourceMetrics": [{
"resource": resource,
"scopeMetrics": [{
"scope": {"name": "awoooi.obs.signal-verifier"},
"metrics": [{
"name": "awoooi_p0_obs_route_canary",
"description": "P0-OBS-002 route canary",
"unit": "1",
"gauge": {"dataPoints": [{
"attributes": [{
"key": "awoooi.marker",
"value": {"stringValue": marker},
}],
"timeUnixNano": timestamp,
"asDouble": 1.0,
}]},
}],
}],
}]
}
elif signal == "traces":
body = {
"resourceSpans": [{
"resource": resource,
"scopeSpans": [{
"scope": {"name": "awoooi.obs.signal-verifier"},
"spans": [{
"traceId": trace_id,
"spanId": span_id,
"name": marker,
"kind": 1,
"startTimeUnixNano": timestamp,
"endTimeUnixNano": str(int(timestamp) + 1_000_000),
"status": {"code": 1},
}],
}],
}]
}
else:
raise SystemExit(64)
print(json.dumps(body, separators=(",", ":")))
PY
}
send_signal() {
local signal="$1" endpoint code body
endpoint="$OTLP_BASE_ENDPOINT/v1/$signal"
body="$(payload "$signal")"
code="$(printf '%s' "$body" | ssh "${SSH_OPTS[@]}" "$SOURCE_NODE" \
"curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary @- '$endpoint'")"
printf 'SIGNOZ_SIGNAL_SEND route=%s signal=%s code=%s\n' "$ROUTE_ID" "$signal" "$code"
[ "$code" = 200 ]
}
query_log_count() {
ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
"docker exec signoz-clickhouse clickhouse-client --param_marker='$marker' --param_route='$ROUTE_ID' --query \"SELECT count() FROM signoz_logs.logs_v2 WHERE body={marker:String} AND resources_string['awoooi.route.id']={route:String}\""
}
query_metric_series_count() {
ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
"docker exec signoz-clickhouse clickhouse-client --param_marker='$marker' --param_route='$ROUTE_ID' --query \"SELECT count() FROM signoz_metrics.time_series_v4 WHERE metric_name='awoooi_p0_obs_route_canary' AND attrs['awoooi.marker']={marker:String} AND resource_attrs['awoooi.route.id']={route:String}\""
}
query_metric_sample_count() {
ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
"docker exec signoz-clickhouse clickhouse-client --param_marker='$marker' --param_route='$ROUTE_ID' --query \"SELECT count() FROM signoz_metrics.samples_v4 WHERE metric_name='awoooi_p0_obs_route_canary' AND fingerprint IN (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name='awoooi_p0_obs_route_canary' AND attrs['awoooi.marker']={marker:String} AND resource_attrs['awoooi.route.id']={route:String})\""
}
query_trace_count() {
ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
"docker exec signoz-clickhouse clickhouse-client --param_marker='$marker' --param_route='$ROUTE_ID' --query \"SELECT count() FROM signoz_traces.signoz_index_v3 WHERE name={marker:String} AND resources_string['awoooi.route.id']={route:String}\""
}
receipt sensor pass "route_${ROUTE_ID}_endpoint_selected"
send_signal logs
send_signal metrics
send_signal traces
receipt execute pass "logs_metrics_traces_http_200_marker_${marker}"
log_count=0
metric_series_count=0
metric_sample_count=0
trace_count=0
for _ in $(seq 1 20); do
log_count="$(query_log_count)"
metric_series_count="$(query_metric_series_count)"
metric_sample_count="$(query_metric_sample_count)"
trace_count="$(query_trace_count)"
if [ "$log_count" -eq 1 ] && [ "$metric_series_count" -eq 1 ] && \
[ "$metric_sample_count" -eq 1 ] && [ "$trace_count" -eq 1 ]; then
break
fi
sleep 2
done
[ "$log_count" -eq 1 ]
[ "$metric_series_count" -eq 1 ]
[ "$metric_sample_count" -eq 1 ]
[ "$trace_count" -eq 1 ]
receipt verify pass "exact_log_1_metric_series_1_metric_sample_1_trace_1"
receipt terminal pass "route_${ROUTE_ID}_three_signal_readback_closed"