fix(ops): bound CD DB probes and restore SigNoz collector
This commit is contained in:
@@ -152,6 +152,13 @@ runtime_observations:
|
||||
real_application_trace_freshness_inferred: false
|
||||
producer_queue_failure_gate_inferred: false
|
||||
promotion_authorized: false
|
||||
post_closure_regression:
|
||||
snapshot: ops/signoz/p0-obs-002-post-closure-regression.yaml
|
||||
grpc_bridge_rollback_indicated: false
|
||||
canonical_collector_recovered: true
|
||||
api_runtime_cooldown_closed: false
|
||||
cd_probe_safety_fix_deployed: false
|
||||
backup_collector_restore_guard_deployed: false
|
||||
|
||||
terminal:
|
||||
status: partial_degraded
|
||||
@@ -160,4 +167,7 @@ terminal:
|
||||
- direct_ingress_policy_not_closed
|
||||
- http_bridge_not_supervised
|
||||
- filelog_normalization_degraded
|
||||
- api_runtime_cooldown_pending
|
||||
- cd_probe_safety_fix_not_deployed
|
||||
- backup_collector_restore_guard_not_deployed
|
||||
- wave_b_not_authorized
|
||||
|
||||
93
ops/signoz/p0-obs-002-post-closure-regression.yaml
Normal file
93
ops/signoz/p0-obs-002-post-closure-regression.yaml
Normal file
@@ -0,0 +1,93 @@
|
||||
schema: awoooi_signoz_post_closure_regression_v1
|
||||
work_item_id: P0-OBS-002
|
||||
trace_id: P0-OBS-002
|
||||
observed_at: "2026-07-15T02:20:00+08:00"
|
||||
|
||||
baseline:
|
||||
grpc_runtime_run_id: P0-OBS-002-20260715T013800+0800
|
||||
verifier_window_seconds: 600
|
||||
verifier_terminal: pass
|
||||
api_restart_delta: 0
|
||||
trace_count: 1098
|
||||
metric_sample_count: 60180
|
||||
|
||||
regressions:
|
||||
api_runtime:
|
||||
first_probe_timeout_at: "2026-07-15T02:11:22+08:00"
|
||||
deployed_source_sha: 31de3a709815f33a11ba6de6e78b771b7160a245
|
||||
deploy_marker_sha: f331ae87c27f42ecc96d3cd4e86f30499b5fbfba
|
||||
image_digest: sha256:a99b3b276def966c03be56f7e8e15feb69d5e2029c8d4ec11dd4b839709351b8
|
||||
minimum_ready_replicas_observed: 0
|
||||
expected_replicas: 2
|
||||
restart_count_per_pod_at_recovery: 4
|
||||
last_observed_ready_replicas: 2
|
||||
last_observed_cooldown_closed: false
|
||||
primary_classification: db_connection_budget_and_request_pressure
|
||||
structural_risk_confidence: 0.98
|
||||
direct_application_commit_regression_confidence: 0.12
|
||||
evidence:
|
||||
- too_many_connections_for_awoooi_api_runtime
|
||||
- constant_response_liveness_and_readiness_timed_out
|
||||
- no_oom_or_cgroup_cpu_throttle
|
||||
- fixed_eight_worker_cd_db_probe_can_exhaust_role_limit_twelve
|
||||
canonical_collector:
|
||||
stopped_at: "2026-07-15T02:12:15+08:00"
|
||||
exit_code: 0
|
||||
oom_killed: false
|
||||
trigger: daily_backup_signoz_collector_stop
|
||||
backup_terminal: failed
|
||||
backup_services_succeeded: 3
|
||||
backup_services_total: 9
|
||||
automatic_restore_observed: false
|
||||
|
||||
controlled_recovery:
|
||||
run_id: P0-OBS-002-static-recovery-20260715T021600+0800
|
||||
collector_only: true
|
||||
stateful_services_touched: 0
|
||||
collector_running: true
|
||||
grpc_4317_listening: true
|
||||
http_4318_listening: true
|
||||
empty_otlp_http_status: 200
|
||||
grpc_bridge_enabled_active: true
|
||||
grpc_bridge_restart_count: 0
|
||||
source_nodes_tcp_verified:
|
||||
- 192.168.0.120
|
||||
- 192.168.0.121
|
||||
|
||||
source_candidates:
|
||||
cd_probe:
|
||||
path: .gitea/workflows/cd.yaml
|
||||
connection_budget_precheck_required: true
|
||||
max_workers: 4
|
||||
max_attempts: 1
|
||||
request_timeout_seconds: 5
|
||||
safety_reserve_connections: 4
|
||||
receipt_fields_added: true
|
||||
deployment_status: pending
|
||||
backup_restore_guard:
|
||||
path: scripts/backup/backup-signoz.sh
|
||||
trap_exit_hup_int_term: true
|
||||
restore_armed_before_stop: true
|
||||
bounded_exit_retry: true
|
||||
deployment_status: pending
|
||||
|
||||
scope_boundaries:
|
||||
grpc_bridge_rollback_indicated: false
|
||||
api_manual_rollback_performed: false
|
||||
firewall_changed: false
|
||||
database_changed: false
|
||||
stateful_volume_read: false
|
||||
secret_read: false
|
||||
github_used: false
|
||||
|
||||
terminal:
|
||||
status: partial_degraded
|
||||
blockers:
|
||||
- api_runtime_cooldown_pending
|
||||
- cd_probe_safety_fix_not_deployed
|
||||
- backup_collector_restore_guard_not_deployed
|
||||
- newer_main_cd_nonterminal
|
||||
safe_next_action: >-
|
||||
Merge current Gitea main, validate and deploy the bounded CD probe and backup
|
||||
restore guard, then require a fresh CD terminal plus ten-minute API and OTLP
|
||||
runtime verifier with zero restart and exporter-error deltas.
|
||||
Reference in New Issue
Block a user