86 lines
4.4 KiB
Bash
Executable File
86 lines
4.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Real application OTLP/gRPC post-verifier. It reads aggregates only.
|
|
|
|
set -Eeuo pipefail
|
|
|
|
TRACE_ID="${TRACE_ID:-P0-OBS-002}"
|
|
RUN_ID="${RUN_ID:-P0-OBS-002-grpc-runtime-$(date -u +%Y%m%dT%H%M%SZ)}"
|
|
WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-002}"
|
|
WINDOW_SECONDS="${WINDOW_SECONDS:-600}"
|
|
EXPECTED_PODS="${EXPECTED_PODS:-2}"
|
|
NAMESPACE="${NAMESPACE:-awoooi-prod}"
|
|
POD_SELECTOR="${POD_SELECTOR:-app=awoooi-api}"
|
|
SERVICE_NAME="${SERVICE_NAME:-awoooi-api}"
|
|
CANARY_URL="${CANARY_URL:-https://awoooi.wooo.work/api/v1/openapi.json}"
|
|
BRIDGE_HOST="${BRIDGE_HOST:-ollama@192.168.0.188}"
|
|
BRIDGE_SERVICE="${BRIDGE_SERVICE:-awoooi-signoz-otlp-grpc-bridge.service}"
|
|
SIGNOZ_HOST="${SIGNOZ_HOST:-wooo@192.168.0.110}"
|
|
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ConnectionAttempts=1)
|
|
|
|
case "$WINDOW_SECONDS" in
|
|
''|*[!0-9]*) echo "WINDOW_SECONDS must be an integer" >&2; exit 64 ;;
|
|
esac
|
|
[ "$WINDOW_SECONDS" -ge 60 ]
|
|
|
|
receipt() {
|
|
local phase="$1" result="$2" detail="$3"
|
|
printf '{"schema":"awoooi_otel_grpc_runtime_verifier_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"signoz-188-otlp-grpc-compat-bridge","phase":"%s","result":"%s","detail":"%s"}\n' \
|
|
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail"
|
|
}
|
|
|
|
on_error() {
|
|
local rc=$?
|
|
receipt rollback required "run_signoz_188_otlp_grpc_bridge_rollback"
|
|
receipt terminal failed "command_exit_$rc"
|
|
exit "$rc"
|
|
}
|
|
trap on_error ERR
|
|
|
|
pods=()
|
|
while IFS= read -r pod; do
|
|
[ -n "$pod" ] && pods[${#pods[@]}]="$pod"
|
|
done < <(
|
|
kubectl -n "$NAMESPACE" get pods -l "$POD_SELECTOR" \
|
|
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'
|
|
)
|
|
[ "${#pods[@]}" -eq "$EXPECTED_PODS" ]
|
|
restart_before=()
|
|
for pod in "${pods[@]}"; do
|
|
ready="$(kubectl -n "$NAMESPACE" get pod "$pod" -o jsonpath='{.status.containerStatuses[0].ready}')"
|
|
[ "$ready" = true ]
|
|
restart_before[${#restart_before[@]}]="$(kubectl -n "$NAMESPACE" get pod "$pod" -o jsonpath='{.status.containerStatuses[0].restartCount}')"
|
|
done
|
|
ssh "${SSH_OPTS[@]}" "$BRIDGE_HOST" "systemctl --user is-enabled --quiet '$BRIDGE_SERVICE' && systemctl --user is-active --quiet '$BRIDGE_SERVICE'"
|
|
receipt sensor pass "api_pods_${EXPECTED_PODS}_ready_grpc_bridge_enabled_active"
|
|
receipt normalize pass "service_${SERVICE_NAME}_aggregate_only"
|
|
receipt diff pass "incumbent_endpoint_unchanged_grpc_transport_present"
|
|
receipt decision allow "one_non_health_synthetic_request_then_bounded_observation"
|
|
receipt policy allow "risk_low_no_secret_no_payload_query_no_route_change"
|
|
receipt check pass "window_${WINDOW_SECONDS}s_restart_baseline_captured"
|
|
|
|
http_code="$(curl -sS --max-time 15 -o /dev/null -w '%{http_code}' "$CANARY_URL")"
|
|
[ "$http_code" -ge 200 ] && [ "$http_code" -lt 400 ]
|
|
receipt execute pass "openapi_canary_http_${http_code}"
|
|
|
|
sleep "$WINDOW_SECONDS"
|
|
error_count=0
|
|
for i in "${!pods[@]}"; do
|
|
pod="${pods[$i]}"
|
|
restart_after="$(kubectl -n "$NAMESPACE" get pod "$pod" -o jsonpath='{.status.containerStatuses[0].restartCount}')"
|
|
[ "$restart_after" = "${restart_before[$i]}" ]
|
|
pod_errors="$(kubectl -n "$NAMESPACE" logs "$pod" --since="${WINDOW_SECONDS}s" 2>&1 | grep -Ec 'Failed to export traces|Failed to export metrics|Connection refused' || true)"
|
|
error_count=$((error_count + pod_errors))
|
|
done
|
|
[ "$error_count" -eq 0 ]
|
|
|
|
trace_count="$(ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
|
|
"docker exec signoz-clickhouse clickhouse-client --param_service='$SERVICE_NAME' --param_window='$WINDOW_SECONDS' --query \"SELECT count() FROM signoz_traces.signoz_index_v3 WHERE serviceName={service:String} AND timestamp >= now() - toIntervalSecond({window:UInt32})\"")"
|
|
metric_count="$(ssh "${SSH_OPTS[@]}" "$SIGNOZ_HOST" \
|
|
"docker exec signoz-clickhouse clickhouse-client --param_service='$SERVICE_NAME' --param_window='$WINDOW_SECONDS' --query \"SELECT count() FROM signoz_metrics.samples_v4 WHERE unix_milli >= toUnixTimestamp64Milli(now64()) - ({window:UInt32} * 1000) AND fingerprint IN (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE resource_attrs['service.name']={service:String})\"")"
|
|
[ "$trace_count" -gt 0 ]
|
|
[ "$metric_count" -gt 0 ]
|
|
receipt verify pass "window_${WINDOW_SECONDS}s_export_errors_0_restart_delta_0_trace_count_${trace_count}_metric_count_${metric_count}"
|
|
receipt rollback not_required "independent_post_verifier_pass"
|
|
receipt learning partial "durable_gitea_main_and_km_writeback_pending"
|
|
receipt terminal pass "real_application_otlp_grpc_trace_metric_freshness_closed"
|