fix(ops): bound CD DB probes and restore SigNoz collector

This commit is contained in:
ogt
2026-07-15 02:29:51 +08:00
parent 27736add14
commit bfb2d2d040
11 changed files with 442 additions and 19 deletions

View File

@@ -152,6 +152,13 @@ runtime_observations:
real_application_trace_freshness_inferred: false
producer_queue_failure_gate_inferred: false
promotion_authorized: false
post_closure_regression:
snapshot: ops/signoz/p0-obs-002-post-closure-regression.yaml
grpc_bridge_rollback_indicated: false
canonical_collector_recovered: true
api_runtime_cooldown_closed: false
cd_probe_safety_fix_deployed: false
backup_collector_restore_guard_deployed: false
terminal:
status: partial_degraded
@@ -160,4 +167,7 @@ terminal:
- direct_ingress_policy_not_closed
- http_bridge_not_supervised
- filelog_normalization_degraded
- api_runtime_cooldown_pending
- cd_probe_safety_fix_not_deployed
- backup_collector_restore_guard_not_deployed
- wave_b_not_authorized

View File

@@ -0,0 +1,93 @@
schema: awoooi_signoz_post_closure_regression_v1
work_item_id: P0-OBS-002
trace_id: P0-OBS-002
observed_at: "2026-07-15T02:20:00+08:00"
baseline:
grpc_runtime_run_id: P0-OBS-002-20260715T013800+0800
verifier_window_seconds: 600
verifier_terminal: pass
api_restart_delta: 0
trace_count: 1098
metric_sample_count: 60180
regressions:
api_runtime:
first_probe_timeout_at: "2026-07-15T02:11:22+08:00"
deployed_source_sha: 31de3a709815f33a11ba6de6e78b771b7160a245
deploy_marker_sha: f331ae87c27f42ecc96d3cd4e86f30499b5fbfba
image_digest: sha256:a99b3b276def966c03be56f7e8e15feb69d5e2029c8d4ec11dd4b839709351b8
minimum_ready_replicas_observed: 0
expected_replicas: 2
restart_count_per_pod_at_recovery: 4
last_observed_ready_replicas: 2
last_observed_cooldown_closed: false
primary_classification: db_connection_budget_and_request_pressure
structural_risk_confidence: 0.98
direct_application_commit_regression_confidence: 0.12
evidence:
- too_many_connections_for_awoooi_api_runtime
- constant_response_liveness_and_readiness_timed_out
- no_oom_or_cgroup_cpu_throttle
- fixed_eight_worker_cd_db_probe_can_exhaust_role_limit_twelve
canonical_collector:
stopped_at: "2026-07-15T02:12:15+08:00"
exit_code: 0
oom_killed: false
trigger: daily_backup_signoz_collector_stop
backup_terminal: failed
backup_services_succeeded: 3
backup_services_total: 9
automatic_restore_observed: false
controlled_recovery:
run_id: P0-OBS-002-static-recovery-20260715T021600+0800
collector_only: true
stateful_services_touched: 0
collector_running: true
grpc_4317_listening: true
http_4318_listening: true
empty_otlp_http_status: 200
grpc_bridge_enabled_active: true
grpc_bridge_restart_count: 0
source_nodes_tcp_verified:
- 192.168.0.120
- 192.168.0.121
source_candidates:
cd_probe:
path: .gitea/workflows/cd.yaml
connection_budget_precheck_required: true
max_workers: 4
max_attempts: 1
request_timeout_seconds: 5
safety_reserve_connections: 4
receipt_fields_added: true
deployment_status: pending
backup_restore_guard:
path: scripts/backup/backup-signoz.sh
trap_exit_hup_int_term: true
restore_armed_before_stop: true
bounded_exit_retry: true
deployment_status: pending
scope_boundaries:
grpc_bridge_rollback_indicated: false
api_manual_rollback_performed: false
firewall_changed: false
database_changed: false
stateful_volume_read: false
secret_read: false
github_used: false
terminal:
status: partial_degraded
blockers:
- api_runtime_cooldown_pending
- cd_probe_safety_fix_not_deployed
- backup_collector_restore_guard_not_deployed
- newer_main_cd_nonterminal
safe_next_action: >-
Merge current Gitea main, validate and deploy the bounded CD probe and backup
restore guard, then require a fresh CD terminal plus ten-minute API and OTLP
runtime verifier with zero restart and exporter-error deltas.