wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 4 Packages Projects Releases Wiki Activity

chore(observability): record native backup production verifier

Browse Source
This commit is contained in:
ogt
2026-07-15 10:17:00 +08:00
parent 227189cb91
commit ca8403ce79
4 changed files with 611 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
ops/signoz/organization-canonical-route-migration.yaml
View File

@@ -172,8 +172,10 @@ runtime_observations:
backup_collector_restore_guard_deployed: true
backup_canary_scope: clickhouse_native_only
backup_canary_verified: true
backup_canary_last_run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4
backup_canary_last_terminal: pass
backup_canary_last_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
backup_canary_last_terminal: partial_degraded
backup_canary_runtime_operation_terminal: pass
backup_canary_machine_receipt_verifier_terminal: pass
backup_canary_prior_root_cause: raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract
backup_canary_cooldown_retry_status: superseded_by_online_native_no_stop_contract
clickhouse_native_backup_migration_status: production_closed
@@ -181,10 +183,21 @@ runtime_observations:
clickhouse_native_backup_disk_status: deployed_verified
clickhouse_native_backup_toolchain_run_id: P0-OBS-002-native-toolchain-20260715T001252Z-apply4
clickhouse_native_backup_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4
clickhouse_native_current_source_status: pending_toolchain_deploy_and_runtime_canary
clickhouse_native_current_source_deployed: false
clickhouse_native_current_source_status: deployed_verified
clickhouse_native_current_source_deployed: true
clickhouse_native_current_source_runtime_verifier_terminal: pass
clickhouse_native_current_source_toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6
clickhouse_native_current_source_toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5
clickhouse_native_current_source_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5
clickhouse_native_current_source_toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5
clickhouse_native_current_source_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5
clickhouse_native_current_source_canary_apply_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
clickhouse_native_current_source_wrapper_terminal: partial_degraded
clickhouse_native_current_source_runtime_operation_terminal: pass
clickhouse_native_current_source_machine_receipt_verifier_terminal: pass
clickhouse_native_current_source_artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83
clickhouse_native_restore_terminal: verified
clickhouse_native_restic_snapshot_id: 574e2a1a
clickhouse_native_restic_snapshot_id: 135485ab
clickhouse_native_offsite_status: pending
clickhouse_native_failed_artifact_retention_status: pending_bounded_retention_decision
clickhouse_native_resume_inventory_status: pending_durable_submitted_inventory_contract

181
ops/signoz/p0-obs-002-post-closure-regression.yaml
View File

@@ -364,12 +364,12 @@ controlled_applies:
ruff_format: pass
yaml_xml_parse: pass
current_source:
observed_at: "2026-07-15T09:40:50+08:00"
observed_at: "2026-07-15T09:53:46+08:00"
evidence_scope: committed_current_source
status: pending_toolchain_deploy_and_runtime_canary
deployment_status: pending
current_source_deployed: false
runtime_verifier_terminal: pending
status: deployed_verified
deployment_status: deployed
current_source_deployed: true
runtime_verifier_terminal: pass
historical_production_receipt_preserved: true
expected_file_count: 7
hashes:
@@ -380,13 +380,38 @@ controlled_applies:
canary_wrapper_sha256: e6707c1398e066cbeec5e43a936533682b881b6e2763e21085f85b3de33c5886
backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9
isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b
deployed_source_hashes:
backup_orchestrator_sha256: d1950c6ada4be4696f38dabf904bea3167c89293d7ca300365e62c2831cf4df2
native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a
isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7
inventory_verifier_sha256: c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863
canary_wrapper_sha256: e6707c1398e066cbeec5e43a936533682b881b6e2763e21085f85b3de33c5886
backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9
isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b
drift_from_historical_production_receipt:
- backup_orchestrator_sha256
- canary_wrapper_sha256
- inventory_verifier_sha256
toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6
toolchain_check_terminal: check_pass_no_write
toolchain_check_diff: matching_4_drift_3_absent_0_active_operations_0
toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5
toolchain_apply_terminal: pass
toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5
toolchain_postcheck_terminal: check_pass_no_write
toolchain_postcheck_diff: matching_7_drift_0_absent_0_active_operations_0
toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5
toolchain_postcanary_terminal: check_pass_no_write
toolchain_postcanary_diff: matching_7_drift_0_absent_0_active_operations_0
default_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5
default_canary_check_terminal: check_pass_no_write
runtime_canary_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
wrapper_terminal: partial_degraded
runtime_operation_terminal: pass
nested_machine_receipt_verifier_terminal: pass
source_validation:
backup_restore_deployer_tests_passed: 92
canonical_preflight_tests_passed: 23
canonical_preflight_tests_passed: 24
bash_syntax: pass
ruff: pass
ruff_format: pass
@@ -503,6 +528,109 @@ controlled_applies:
cleanup_verified: true
secret_values_collected: false
closure_writeback: durable_local_receipts_acknowledged
- run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5
check_terminal: check_pass_no_write
wrapper_terminal: partial_degraded
terminal: partial_degraded
runtime_operation:
terminal: pass
scope: clickhouse_native_backup_and_isolated_restore
nested_machine_receipt_verifier:
stage: native_restore_receipt_verifier
terminal: pass
backup:
terminal: BACKUP_CREATED
terminal_sequence:
- check_pass_no_write
- pass
independent_verifier_terminal: pass
operation_identity_hash: d60d3b618798bc1f60eeff4bfcf693ca14c5a1da70150043af90b3fa5e459ec1
operation_status: "BACKUP_CREATED|4885|108006575|99461897|"
file_count: 4885
total_size_bytes: 108006575
compressed_size_bytes: 99461897
artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83
active_operation_count_after: 0
server_artifact_absent_after: true
isolated_restore:
terminal: verified
terminal_sequence:
- check_pass_no_runtime_write
- verified
clickhouse_restore_terminal: RESTORED
independent_post_verifier_terminal: pass
cleanup_terminal: pass
network_scope: internal_only
internal_network_attached: true
staging_network_attached: false
production_network_attached: false
production_restore_performed: false
cleanup_verified: true
database_count: 6
restored_database_count: 6
table_count: 100
restored_table_count: 100
table_engine_schema_manifest_parity: true
check_table_count: 44
check_table_pass_count: 44
critical_nonzero_count: 4
restic:
snapshot_id: 135485ab
snapshot_readback: pass
repository_scope: local_same_failure_domain
offsite_verified: false
sqlite:
terminal: pending_application_consistent_export
independent_post_verifier:
terminal: pass
independent_monitor: pass
deployed_source_hash_drift_count: 0
runtime_drift_count: 0
production_container_count: 4
production_container_identity_unchanged: true
production_container_restart_count_delta: 0
production_containers:
clickhouse:
started_at: "2026-07-14T22:49:13.665350281Z"
restart_count: 0
health: healthy
collector:
started_at: "2026-07-14T22:20:53.725583861Z"
restart_count: 0
health: not_configured
signoz:
started_at: "2026-07-11T08:45:17.179074963Z"
restart_count: 0
health: healthy
zookeeper:
started_at: "2026-07-11T08:45:17.055893064Z"
restart_count: 0
health: healthy
api_http_status: 200
listener_counts:
"4317": 2
"4318": 2
"8080": 2
active_backup_operation_count: 0
restore_container_count: 0
restore_volume_count: 0
restore_network_count: 0
toolchain_stage_residue_count: 0
toolchain_candidate_residue_count: 0
toolchain_rollback_residue_count: 0
backup_process_count: 0
lock_holders:
canary: none
operation: none
toolchain: none
server_artifact_absent: true
exact_ephemeral_container_volume_network_process_tmp_residue_count: 0
cleanup_verified: true
closure_writeback:
terminal: partial_degraded
current_source_runtime_closed: true
overall_program_partial_degraded: true
terminal: production_closed_clickhouse_native_scope
backup_canary:
@@ -599,6 +727,34 @@ completed_verifiers:
restic_snapshot_id: 574e2a1a
independent_verifier: pass
exact_cleanup: pass
clickhouse_native_current_source_toolchain:
status: deployed_verified
current_source_deployed: true
runtime_verifier_terminal: pass
historical_production_receipt_preserved: true
toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6
toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5
toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5
toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5
default_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5
runtime_canary_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
wrapper_terminal: partial_degraded
runtime_operation_terminal: pass
nested_machine_receipt_verifier_terminal: pass
operation_identity_hash: d60d3b618798bc1f60eeff4bfcf693ca14c5a1da70150043af90b3fa5e459ec1
operation_status: "BACKUP_CREATED|4885|108006575|99461897|"
restic_snapshot_id: 135485ab
artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83
offsite_verified: false
sqlite_terminal: pending_application_consistent_export
independent_monitor: pass
api_http_status: 200
production_container_identity_unchanged: true
production_container_restart_count_delta: 0
active_backup_operation_count: 0
runtime_drift_count: 0
residue_count: 0
exact_cleanup: pass
pending_verifiers:
signoz_sqlite_application_consistent_backup:
@@ -611,7 +767,7 @@ pending_verifiers:
signoz_backup_offsite_dr:
status: pending_offsite_snapshot_and_restore_verifier
current_repository_scope: local_same_failure_domain
latest_local_snapshot_id: 574e2a1a
latest_local_snapshot_id: 135485ab
offsite_verified: false
required_preconditions:
- immutable_or_versioned_offsite_target
@@ -641,16 +797,6 @@ pending_verifiers:
- bind_inventory_hash_operation_id_artifact_hash_and_run_identity
- resume_reuses_original_inventory_and_fails_closed_if_missing_or_tampered
- regression_test_with_live_counter_drift_and_zero_duplicate_backup_submits
clickhouse_native_current_source_toolchain:
status: pending_toolchain_deploy_and_runtime_canary
current_source_deployed: false
runtime_verifier_terminal: pending
historical_production_receipt_preserved: true
required_preconditions:
- controlled_toolchain_apply_with_current_source_hash_readback
- same_run_native_backup_and_isolated_restore_canary
- independent_runtime_and_cleanup_verifier
scope_boundaries:
grpc_bridge_rollback_indicated: false
api_manual_rollback_performed: false
@@ -670,7 +816,6 @@ terminal:
- signoz_backup_offsite_dr_pending
- clickhouse_native_failed_artifact_retention_pending
- clickhouse_native_resume_submitted_inventory_pending
- clickhouse_native_current_source_toolchain_pending
- service_registry_runtime_mirror_reconciliation_pending
- github_freeze_legacy_asset_retirement_pending
- monitoring_generator_duplicate_awoooi_api_identity_pending