diff --git a/ops/signoz/organization-canonical-route-migration.yaml b/ops/signoz/organization-canonical-route-migration.yaml index afda9b611..4810622a4 100644 --- a/ops/signoz/organization-canonical-route-migration.yaml +++ b/ops/signoz/organization-canonical-route-migration.yaml @@ -172,8 +172,10 @@ runtime_observations: backup_collector_restore_guard_deployed: true backup_canary_scope: clickhouse_native_only backup_canary_verified: true - backup_canary_last_run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4 - backup_canary_last_terminal: pass + backup_canary_last_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5 + backup_canary_last_terminal: partial_degraded + backup_canary_runtime_operation_terminal: pass + backup_canary_machine_receipt_verifier_terminal: pass backup_canary_prior_root_cause: raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract backup_canary_cooldown_retry_status: superseded_by_online_native_no_stop_contract clickhouse_native_backup_migration_status: production_closed @@ -181,10 +183,21 @@ runtime_observations: clickhouse_native_backup_disk_status: deployed_verified clickhouse_native_backup_toolchain_run_id: P0-OBS-002-native-toolchain-20260715T001252Z-apply4 clickhouse_native_backup_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4 - clickhouse_native_current_source_status: pending_toolchain_deploy_and_runtime_canary - clickhouse_native_current_source_deployed: false + clickhouse_native_current_source_status: deployed_verified + clickhouse_native_current_source_deployed: true + clickhouse_native_current_source_runtime_verifier_terminal: pass + clickhouse_native_current_source_toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6 + clickhouse_native_current_source_toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5 + clickhouse_native_current_source_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5 + clickhouse_native_current_source_toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5 + clickhouse_native_current_source_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5 + clickhouse_native_current_source_canary_apply_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5 + clickhouse_native_current_source_wrapper_terminal: partial_degraded + clickhouse_native_current_source_runtime_operation_terminal: pass + clickhouse_native_current_source_machine_receipt_verifier_terminal: pass + clickhouse_native_current_source_artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83 clickhouse_native_restore_terminal: verified - clickhouse_native_restic_snapshot_id: 574e2a1a + clickhouse_native_restic_snapshot_id: 135485ab clickhouse_native_offsite_status: pending clickhouse_native_failed_artifact_retention_status: pending_bounded_retention_decision clickhouse_native_resume_inventory_status: pending_durable_submitted_inventory_contract diff --git a/ops/signoz/p0-obs-002-post-closure-regression.yaml b/ops/signoz/p0-obs-002-post-closure-regression.yaml index 73f015236..48f1f5f8c 100644 --- a/ops/signoz/p0-obs-002-post-closure-regression.yaml +++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml @@ -364,12 +364,12 @@ controlled_applies: ruff_format: pass yaml_xml_parse: pass current_source: - observed_at: "2026-07-15T09:40:50+08:00" + observed_at: "2026-07-15T09:53:46+08:00" evidence_scope: committed_current_source - status: pending_toolchain_deploy_and_runtime_canary - deployment_status: pending - current_source_deployed: false - runtime_verifier_terminal: pending + status: deployed_verified + deployment_status: deployed + current_source_deployed: true + runtime_verifier_terminal: pass historical_production_receipt_preserved: true expected_file_count: 7 hashes: @@ -380,13 +380,38 @@ controlled_applies: canary_wrapper_sha256: e6707c1398e066cbeec5e43a936533682b881b6e2763e21085f85b3de33c5886 backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9 isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b + deployed_source_hashes: + backup_orchestrator_sha256: d1950c6ada4be4696f38dabf904bea3167c89293d7ca300365e62c2831cf4df2 + native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a + isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7 + inventory_verifier_sha256: c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863 + canary_wrapper_sha256: e6707c1398e066cbeec5e43a936533682b881b6e2763e21085f85b3de33c5886 + backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9 + isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b drift_from_historical_production_receipt: - backup_orchestrator_sha256 - canary_wrapper_sha256 - inventory_verifier_sha256 + toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6 + toolchain_check_terminal: check_pass_no_write + toolchain_check_diff: matching_4_drift_3_absent_0_active_operations_0 + toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5 + toolchain_apply_terminal: pass + toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5 + toolchain_postcheck_terminal: check_pass_no_write + toolchain_postcheck_diff: matching_7_drift_0_absent_0_active_operations_0 + toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5 + toolchain_postcanary_terminal: check_pass_no_write + toolchain_postcanary_diff: matching_7_drift_0_absent_0_active_operations_0 + default_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5 + default_canary_check_terminal: check_pass_no_write + runtime_canary_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5 + wrapper_terminal: partial_degraded + runtime_operation_terminal: pass + nested_machine_receipt_verifier_terminal: pass source_validation: backup_restore_deployer_tests_passed: 92 - canonical_preflight_tests_passed: 23 + canonical_preflight_tests_passed: 24 bash_syntax: pass ruff: pass ruff_format: pass @@ -503,6 +528,109 @@ controlled_applies: cleanup_verified: true secret_values_collected: false closure_writeback: durable_local_receipts_acknowledged + - run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5 + check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5 + check_terminal: check_pass_no_write + wrapper_terminal: partial_degraded + terminal: partial_degraded + runtime_operation: + terminal: pass + scope: clickhouse_native_backup_and_isolated_restore + nested_machine_receipt_verifier: + stage: native_restore_receipt_verifier + terminal: pass + backup: + terminal: BACKUP_CREATED + terminal_sequence: + - check_pass_no_write + - pass + independent_verifier_terminal: pass + operation_identity_hash: d60d3b618798bc1f60eeff4bfcf693ca14c5a1da70150043af90b3fa5e459ec1 + operation_status: "BACKUP_CREATED|4885|108006575|99461897|" + file_count: 4885 + total_size_bytes: 108006575 + compressed_size_bytes: 99461897 + artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83 + active_operation_count_after: 0 + server_artifact_absent_after: true + isolated_restore: + terminal: verified + terminal_sequence: + - check_pass_no_runtime_write + - verified + clickhouse_restore_terminal: RESTORED + independent_post_verifier_terminal: pass + cleanup_terminal: pass + network_scope: internal_only + internal_network_attached: true + staging_network_attached: false + production_network_attached: false + production_restore_performed: false + cleanup_verified: true + database_count: 6 + restored_database_count: 6 + table_count: 100 + restored_table_count: 100 + table_engine_schema_manifest_parity: true + check_table_count: 44 + check_table_pass_count: 44 + critical_nonzero_count: 4 + restic: + snapshot_id: 135485ab + snapshot_readback: pass + repository_scope: local_same_failure_domain + offsite_verified: false + sqlite: + terminal: pending_application_consistent_export + independent_post_verifier: + terminal: pass + independent_monitor: pass + deployed_source_hash_drift_count: 0 + runtime_drift_count: 0 + production_container_count: 4 + production_container_identity_unchanged: true + production_container_restart_count_delta: 0 + production_containers: + clickhouse: + started_at: "2026-07-14T22:49:13.665350281Z" + restart_count: 0 + health: healthy + collector: + started_at: "2026-07-14T22:20:53.725583861Z" + restart_count: 0 + health: not_configured + signoz: + started_at: "2026-07-11T08:45:17.179074963Z" + restart_count: 0 + health: healthy + zookeeper: + started_at: "2026-07-11T08:45:17.055893064Z" + restart_count: 0 + health: healthy + api_http_status: 200 + listener_counts: + "4317": 2 + "4318": 2 + "8080": 2 + active_backup_operation_count: 0 + restore_container_count: 0 + restore_volume_count: 0 + restore_network_count: 0 + toolchain_stage_residue_count: 0 + toolchain_candidate_residue_count: 0 + toolchain_rollback_residue_count: 0 + backup_process_count: 0 + lock_holders: + canary: none + operation: none + toolchain: none + server_artifact_absent: true + exact_ephemeral_container_volume_network_process_tmp_residue_count: 0 + cleanup_verified: true + closure_writeback: + terminal: partial_degraded + current_source_runtime_closed: true + overall_program_partial_degraded: true terminal: production_closed_clickhouse_native_scope backup_canary: @@ -599,6 +727,34 @@ completed_verifiers: restic_snapshot_id: 574e2a1a independent_verifier: pass exact_cleanup: pass + clickhouse_native_current_source_toolchain: + status: deployed_verified + current_source_deployed: true + runtime_verifier_terminal: pass + historical_production_receipt_preserved: true + toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6 + toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5 + toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5 + toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5 + default_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5 + runtime_canary_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5 + wrapper_terminal: partial_degraded + runtime_operation_terminal: pass + nested_machine_receipt_verifier_terminal: pass + operation_identity_hash: d60d3b618798bc1f60eeff4bfcf693ca14c5a1da70150043af90b3fa5e459ec1 + operation_status: "BACKUP_CREATED|4885|108006575|99461897|" + restic_snapshot_id: 135485ab + artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83 + offsite_verified: false + sqlite_terminal: pending_application_consistent_export + independent_monitor: pass + api_http_status: 200 + production_container_identity_unchanged: true + production_container_restart_count_delta: 0 + active_backup_operation_count: 0 + runtime_drift_count: 0 + residue_count: 0 + exact_cleanup: pass pending_verifiers: signoz_sqlite_application_consistent_backup: @@ -611,7 +767,7 @@ pending_verifiers: signoz_backup_offsite_dr: status: pending_offsite_snapshot_and_restore_verifier current_repository_scope: local_same_failure_domain - latest_local_snapshot_id: 574e2a1a + latest_local_snapshot_id: 135485ab offsite_verified: false required_preconditions: - immutable_or_versioned_offsite_target @@ -641,16 +797,6 @@ pending_verifiers: - bind_inventory_hash_operation_id_artifact_hash_and_run_identity - resume_reuses_original_inventory_and_fails_closed_if_missing_or_tampered - regression_test_with_live_counter_drift_and_zero_duplicate_backup_submits - clickhouse_native_current_source_toolchain: - status: pending_toolchain_deploy_and_runtime_canary - current_source_deployed: false - runtime_verifier_terminal: pending - historical_production_receipt_preserved: true - required_preconditions: - - controlled_toolchain_apply_with_current_source_hash_readback - - same_run_native_backup_and_isolated_restore_canary - - independent_runtime_and_cleanup_verifier - scope_boundaries: grpc_bridge_rollback_indicated: false api_manual_rollback_performed: false @@ -670,7 +816,6 @@ terminal: - signoz_backup_offsite_dr_pending - clickhouse_native_failed_artifact_retention_pending - clickhouse_native_resume_submitted_inventory_pending - - clickhouse_native_current_source_toolchain_pending - service_registry_runtime_mirror_reconciliation_pending - github_freeze_legacy_asset_retirement_pending - monitoring_generator_duplicate_awoooi_api_identity_pending diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py index e3d857987..6694cda2a 100644 --- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -31,7 +31,6 @@ EXPECTED_POST_CLOSURE_BLOCKERS = [ "signoz_backup_offsite_dr_pending", "clickhouse_native_failed_artifact_retention_pending", "clickhouse_native_resume_submitted_inventory_pending", - "clickhouse_native_current_source_toolchain_pending", "service_registry_runtime_mirror_reconciliation_pending", "github_freeze_legacy_asset_retirement_pending", "monitoring_generator_duplicate_awoooi_api_identity_pending", @@ -474,16 +473,35 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: native_canary = applies.get("clickhouse_native_backup_restore_canary", {}) native_attempts = native_canary.get("attempts", []) - native_apply2 = native_attempts[1] if len(native_attempts) == 4 else {} - native_apply3 = native_attempts[2] if len(native_attempts) == 4 else {} - native_apply4 = native_attempts[3] if len(native_attempts) == 4 else {} + native_attempts_by_run_id = { + attempt.get("run_id"): attempt + for attempt in native_attempts + if isinstance(attempt, dict) and attempt.get("run_id") + } + native_apply2 = native_attempts_by_run_id.get( + "P0-OBS-002-native-canary-20260714T234436Z-apply2", {} + ) + native_apply3 = native_attempts_by_run_id.get( + "P0-OBS-002-native-canary-20260715T000638Z-apply3", {} + ) + native_apply4 = native_attempts_by_run_id.get( + "P0-OBS-002-native-canary-20260715T001329Z-apply4", {} + ) + native_apply5 = native_attempts_by_run_id.get( + "P0-OBS-002-native-canary-default-20260715T015149Z-apply5", {} + ) native_backup = native_apply4.get("backup", {}) native_restore = native_apply4.get("isolated_restore", {}) native_restic = native_apply4.get("restic", {}) native_post = native_apply4.get("independent_post_verifier", {}) + current_backup = native_apply5.get("backup", {}) + current_restore = native_apply5.get("isolated_restore", {}) + current_restic = native_apply5.get("restic", {}) + current_post = native_apply5.get("independent_post_verifier", {}) toolchain_apply = native_canary.get("toolchain_controlled_apply", {}) historical_native_source = native_canary.get("source", {}) current_native_source = native_canary.get("current_source", {}) + current_source_validation = current_native_source.get("source_validation", {}) historical_source_hashes = { field: historical_native_source.get(field) for field in TOOLCHAIN_SOURCE_PATHS } @@ -511,25 +529,166 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: and declared_current_hashes == actual_current_hashes and declared_historical_drift == actual_historical_drift ) - pending_source_state = ( - current_native_source.get("status") - == "pending_toolchain_deploy_and_runtime_canary" - and current_native_source.get("deployment_status") == "pending" - and current_native_source.get("current_source_deployed") is False - and current_native_source.get("runtime_verifier_terminal") == "pending" - ) deployed_source_state = ( current_native_source.get("status") == "deployed_verified" and current_native_source.get("deployment_status") == "deployed" and current_native_source.get("current_source_deployed") is True and current_native_source.get("runtime_verifier_terminal") == "pass" and current_native_source.get("deployed_source_hashes") == actual_current_hashes - and bool(current_native_source.get("toolchain_apply_run_id")) - and bool(current_native_source.get("runtime_canary_run_id")) + and current_native_source.get("toolchain_check_run_id") + == "P0-OBS-002-native-toolchain-20260715T015040Z-check6" + and current_native_source.get("toolchain_check_terminal") + == "check_pass_no_write" + and current_native_source.get("toolchain_check_diff") + == "matching_4_drift_3_absent_0_active_operations_0" + and current_native_source.get("toolchain_apply_run_id") + == "P0-OBS-002-native-toolchain-20260715T015059Z-apply5" + and current_native_source.get("toolchain_apply_terminal") == "pass" + and current_native_source.get("toolchain_postcheck_run_id") + == "P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5" + and current_native_source.get("toolchain_postcheck_terminal") + == "check_pass_no_write" + and current_native_source.get("toolchain_postcheck_diff") + == "matching_7_drift_0_absent_0_active_operations_0" + and current_native_source.get("toolchain_postcanary_run_id") + == "P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5" + and current_native_source.get("toolchain_postcanary_terminal") + == "check_pass_no_write" + and current_native_source.get("toolchain_postcanary_diff") + == "matching_7_drift_0_absent_0_active_operations_0" + and current_native_source.get("default_canary_check_run_id") + == "P0-OBS-002-native-canary-default-20260715T015131Z-check5" + and current_native_source.get("default_canary_check_terminal") + == "check_pass_no_write" + and current_native_source.get("runtime_canary_run_id") + == "P0-OBS-002-native-canary-default-20260715T015149Z-apply5" + and current_native_source.get("wrapper_terminal") == "partial_degraded" + and current_native_source.get("runtime_operation_terminal") == "pass" + and current_native_source.get("nested_machine_receipt_verifier_terminal") + == "pass" + and current_source_validation.get("backup_restore_deployer_tests_passed") == 92 + and current_source_validation.get("canonical_preflight_tests_passed") == 24 + and current_source_validation.get("bash_syntax") == "pass" + and current_source_validation.get("ruff") == "pass" + and current_source_validation.get("ruff_format") == "pass" + and current_source_validation.get("yaml_parse") == "pass" + and current_source_validation.get("diff_check") == "pass" + and current_source_validation.get("independent_reaudit") == "no_commit_blocker" ) checks["native_backup_current_source_deployment_state_explicit"] = ( current_native_source.get("historical_production_receipt_preserved") is True - and (pending_source_state or deployed_source_state) + and deployed_source_state + ) + checks["native_backup_current_source_runtime_closed"] = ( + native_apply5.get("check_run_id") + == "P0-OBS-002-native-canary-default-20260715T015131Z-check5" + and native_apply5.get("check_terminal") == "check_pass_no_write" + and native_apply5.get("wrapper_terminal") == "partial_degraded" + and native_apply5.get("terminal") == "partial_degraded" + and native_apply5.get("runtime_operation", {}).get("terminal") == "pass" + and native_apply5.get("nested_machine_receipt_verifier", {}).get("stage") + == "native_restore_receipt_verifier" + and native_apply5.get("nested_machine_receipt_verifier", {}).get("terminal") + == "pass" + and current_backup.get("terminal") == "BACKUP_CREATED" + and current_backup.get("terminal_sequence") == ["check_pass_no_write", "pass"] + and current_backup.get("independent_verifier_terminal") == "pass" + and current_backup.get("operation_identity_hash") + == "d60d3b618798bc1f60eeff4bfcf693ca14c5a1da70150043af90b3fa5e459ec1" + and current_backup.get("operation_status") + == "BACKUP_CREATED|4885|108006575|99461897|" + and current_backup.get("file_count") == 4885 + and current_backup.get("total_size_bytes") == 108006575 + and current_backup.get("compressed_size_bytes") == 99461897 + and current_backup.get("artifact_sha256") + == "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83" + and current_backup.get("active_operation_count_after") == 0 + and current_backup.get("server_artifact_absent_after") is True + and current_restore.get("terminal") == "verified" + and current_restore.get("terminal_sequence") + == ["check_pass_no_runtime_write", "verified"] + and current_restore.get("clickhouse_restore_terminal") == "RESTORED" + and current_restore.get("independent_post_verifier_terminal") == "pass" + and current_restore.get("cleanup_terminal") == "pass" + and current_restore.get("network_scope") == "internal_only" + and current_restore.get("internal_network_attached") is True + and current_restore.get("staging_network_attached") is False + and current_restore.get("production_network_attached") is False + and current_restore.get("production_restore_performed") is False + and current_restore.get("cleanup_verified") is True + and current_restore.get("database_count") + == current_restore.get("restored_database_count") + == 6 + and current_restore.get("table_count") + == current_restore.get("restored_table_count") + == 100 + and current_restore.get("table_engine_schema_manifest_parity") is True + and current_restore.get("check_table_count") + == current_restore.get("check_table_pass_count") + == 44 + and current_restore.get("critical_nonzero_count") == 4 + and current_restic.get("snapshot_id") == "135485ab" + and current_restic.get("snapshot_readback") == "pass" + and current_restic.get("repository_scope") == "local_same_failure_domain" + and current_restic.get("offsite_verified") is False + and native_apply5.get("sqlite", {}).get("terminal") + == "pending_application_consistent_export" + and current_post.get("terminal") == "pass" + and current_post.get("independent_monitor") == "pass" + and current_post.get("deployed_source_hash_drift_count") == 0 + and current_post.get("runtime_drift_count") == 0 + and current_post.get("production_container_count") == 4 + and current_post.get("production_container_identity_unchanged") is True + and current_post.get("production_container_restart_count_delta") == 0 + and current_post.get("production_containers") + == { + "clickhouse": { + "started_at": "2026-07-14T22:49:13.665350281Z", + "restart_count": 0, + "health": "healthy", + }, + "collector": { + "started_at": "2026-07-14T22:20:53.725583861Z", + "restart_count": 0, + "health": "not_configured", + }, + "signoz": { + "started_at": "2026-07-11T08:45:17.179074963Z", + "restart_count": 0, + "health": "healthy", + }, + "zookeeper": { + "started_at": "2026-07-11T08:45:17.055893064Z", + "restart_count": 0, + "health": "healthy", + }, + } + and current_post.get("api_http_status") == 200 + and current_post.get("listener_counts") == {"4317": 2, "4318": 2, "8080": 2} + and current_post.get("active_backup_operation_count") == 0 + and current_post.get("restore_container_count") == 0 + and current_post.get("restore_volume_count") == 0 + and current_post.get("restore_network_count") == 0 + and current_post.get("toolchain_stage_residue_count") == 0 + and current_post.get("toolchain_candidate_residue_count") == 0 + and current_post.get("toolchain_rollback_residue_count") == 0 + and current_post.get("backup_process_count") == 0 + and current_post.get("lock_holders") + == {"canary": "none", "operation": "none", "toolchain": "none"} + and current_post.get("server_artifact_absent") is True + and current_post.get( + "exact_ephemeral_container_volume_network_process_tmp_residue_count" + ) + == 0 + and current_post.get("cleanup_verified") is True + and native_apply5.get("closure_writeback", {}).get( + "current_source_runtime_closed" + ) + is True + and native_apply5.get("closure_writeback", {}).get( + "overall_program_partial_degraded" + ) + is True ) checks["native_backup_restore_canary_closed"] = ( native_canary.get("completion_scope") == "clickhouse_native_only" @@ -655,8 +814,10 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: is True ) - native_migration = regression.get("completed_verifiers", {}).get( - "clickhouse_native_backup_migration", {} + completed_verifiers = regression.get("completed_verifiers", {}) + native_migration = completed_verifiers.get("clickhouse_native_backup_migration", {}) + current_source_toolchain = completed_verifiers.get( + "clickhouse_native_current_source_toolchain", {} ) pending_verifiers = regression.get("pending_verifiers", {}) sqlite_backup = pending_verifiers.get( @@ -669,9 +830,6 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: resume_inventory = pending_verifiers.get( "clickhouse_native_resume_submitted_inventory", {} ) - current_source_toolchain = pending_verifiers.get( - "clickhouse_native_current_source_toolchain", {} - ) checks["native_backup_migration_closed_and_remaining_gaps_explicit"] = ( native_migration.get("prior_run_id") == retry2.get("run_id") and native_migration.get("run_id") == native_apply4.get("run_id") @@ -701,12 +859,47 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: and len(resume_inventory.get("required_preconditions", [])) == 4 and current_source_toolchain.get("status") == current_native_source.get("status") - == "pending_toolchain_deploy_and_runtime_canary" - and current_source_toolchain.get("current_source_deployed") is False - and current_source_toolchain.get("runtime_verifier_terminal") == "pending" + == "deployed_verified" + and current_source_toolchain.get("current_source_deployed") is True + and current_source_toolchain.get("runtime_verifier_terminal") == "pass" and current_source_toolchain.get("historical_production_receipt_preserved") is True - and len(current_source_toolchain.get("required_preconditions", [])) == 3 + and current_source_toolchain.get("toolchain_check_run_id") + == current_native_source.get("toolchain_check_run_id") + and current_source_toolchain.get("toolchain_apply_run_id") + == current_native_source.get("toolchain_apply_run_id") + and current_source_toolchain.get("toolchain_postcheck_run_id") + == current_native_source.get("toolchain_postcheck_run_id") + and current_source_toolchain.get("toolchain_postcanary_run_id") + == current_native_source.get("toolchain_postcanary_run_id") + and current_source_toolchain.get("default_canary_check_run_id") + == current_native_source.get("default_canary_check_run_id") + and current_source_toolchain.get("runtime_canary_run_id") + == current_native_source.get("runtime_canary_run_id") + and current_source_toolchain.get("wrapper_terminal") == "partial_degraded" + and current_source_toolchain.get("runtime_operation_terminal") == "pass" + and current_source_toolchain.get("nested_machine_receipt_verifier_terminal") + == "pass" + and current_source_toolchain.get("operation_identity_hash") + == current_backup.get("operation_identity_hash") + and current_source_toolchain.get("operation_status") + == current_backup.get("operation_status") + and current_source_toolchain.get("restic_snapshot_id") == "135485ab" + and current_source_toolchain.get("artifact_sha256") + == "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83" + and current_source_toolchain.get("offsite_verified") is False + and current_source_toolchain.get("sqlite_terminal") + == "pending_application_consistent_export" + and current_source_toolchain.get("independent_monitor") == "pass" + and current_source_toolchain.get("api_http_status") == 200 + and current_source_toolchain.get("production_container_identity_unchanged") + is True + and current_source_toolchain.get("production_container_restart_count_delta") + == 0 + and current_source_toolchain.get("active_backup_operation_count") == 0 + and current_source_toolchain.get("runtime_drift_count") == 0 + and current_source_toolchain.get("residue_count") == 0 + and current_source_toolchain.get("exact_cleanup") == "pass" ) checks["post_closure_contract_mirror_consistent"] = ( @@ -719,8 +912,11 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: and post_closure.get("backup_collector_restore_guard_deployed") is True and post_closure.get("backup_canary_scope") == "clickhouse_native_only" and post_closure.get("backup_canary_verified") is True - and post_closure.get("backup_canary_last_run_id") == native_apply4.get("run_id") - and post_closure.get("backup_canary_last_terminal") == "pass" + and post_closure.get("backup_canary_last_run_id") == native_apply5.get("run_id") + and post_closure.get("backup_canary_last_terminal") == "partial_degraded" + and post_closure.get("backup_canary_runtime_operation_terminal") == "pass" + and post_closure.get("backup_canary_machine_receipt_verifier_terminal") + == "pass" and post_closure.get("backup_canary_cooldown_retry_status") == "superseded_by_online_native_no_stop_contract" and post_closure.get("clickhouse_native_backup_migration_status") @@ -737,8 +933,40 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: == current_native_source.get("status") and post_closure.get("clickhouse_native_current_source_deployed") == current_native_source.get("current_source_deployed") + and post_closure.get( + "clickhouse_native_current_source_runtime_verifier_terminal" + ) + == current_native_source.get("runtime_verifier_terminal") + and post_closure.get("clickhouse_native_current_source_toolchain_check_run_id") + == current_native_source.get("toolchain_check_run_id") + and post_closure.get("clickhouse_native_current_source_toolchain_apply_run_id") + == current_native_source.get("toolchain_apply_run_id") + and post_closure.get( + "clickhouse_native_current_source_toolchain_postcheck_run_id" + ) + == current_native_source.get("toolchain_postcheck_run_id") + and post_closure.get( + "clickhouse_native_current_source_toolchain_postcanary_run_id" + ) + == current_native_source.get("toolchain_postcanary_run_id") + and post_closure.get("clickhouse_native_current_source_canary_check_run_id") + == current_native_source.get("default_canary_check_run_id") + and post_closure.get("clickhouse_native_current_source_canary_apply_run_id") + == current_native_source.get("runtime_canary_run_id") + and post_closure.get("clickhouse_native_current_source_wrapper_terminal") + == "partial_degraded" + and post_closure.get( + "clickhouse_native_current_source_runtime_operation_terminal" + ) + == "pass" + and post_closure.get( + "clickhouse_native_current_source_machine_receipt_verifier_terminal" + ) + == "pass" + and post_closure.get("clickhouse_native_current_source_artifact_sha256") + == "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83" and post_closure.get("clickhouse_native_restore_terminal") == "verified" - and post_closure.get("clickhouse_native_restic_snapshot_id") == "574e2a1a" + and post_closure.get("clickhouse_native_restic_snapshot_id") == "135485ab" and post_closure.get("clickhouse_native_offsite_status") == "pending" and post_closure.get("clickhouse_native_failed_artifact_retention_status") == "pending_bounded_retention_decision" diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py index 463fc9496..410f5e68e 100644 --- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -98,12 +98,9 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None: assert payload["writes_performed"] == 0 assert payload["terminal"] == "pass" assert payload["runtime_status"] == "partial_degraded" - assert ( - payload["backup_toolchain_current_source_status"] - == "pending_toolchain_deploy_and_runtime_canary" - ) - assert payload["backup_toolchain_current_source_deployed"] is False - assert payload["backup_toolchain_runtime_verifier_terminal"] == "pending" + assert payload["backup_toolchain_current_source_status"] == "deployed_verified" + assert payload["backup_toolchain_current_source_deployed"] is True + assert payload["backup_toolchain_runtime_verifier_terminal"] == "pass" assert payload["backup_toolchain_source_drift_fields"] == [ "backup_orchestrator_sha256", "canary_wrapper_sha256", @@ -119,6 +116,7 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None: payload["checks"]["native_backup_current_source_deployment_state_explicit"] is True ) + assert payload["checks"]["native_backup_current_source_runtime_closed"] is True assert payload["checks"]["native_backup_historical_source_receipt_preserved"] assert payload["checks"]["post_release_600s_verifier_consistent"] is True assert ( @@ -602,8 +600,8 @@ def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> No assert canary["toolchain_controlled_apply"]["postcheck_diff"] == ( "matching_7_drift_0_absent_0_active_operations_0" ) - assert len(canary["attempts"]) == 4 - _, apply2, apply3, apply4 = canary["attempts"] + assert len(canary["attempts"]) == 5 + _, apply2, apply3, apply4, apply5 = canary["attempts"] assert apply2["artifact_retention"] == "preserved_failed_run_evidence" assert apply2["cleanup_verified"] is True assert apply3["artifact_retention"] == "preserved_failed_run_evidence" @@ -647,6 +645,54 @@ def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> No ) assert post["cleanup_verified"] is True assert post["secret_values_collected"] is False + + assert apply5["run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015149Z-apply5" + ) + assert apply5["check_run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015131Z-check5" + ) + assert apply5["wrapper_terminal"] == "partial_degraded" + assert apply5["terminal"] == "partial_degraded" + assert apply5["runtime_operation"]["terminal"] == "pass" + assert apply5["nested_machine_receipt_verifier"] == { + "stage": "native_restore_receipt_verifier", + "terminal": "pass", + } + assert apply5["backup"]["terminal"] == "BACKUP_CREATED" + assert apply5["backup"]["file_count"] == 4885 + assert apply5["backup"]["total_size_bytes"] == 108006575 + assert apply5["backup"]["compressed_size_bytes"] == 99461897 + assert apply5["backup"]["artifact_sha256"] == ( + "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83" + ) + current_restore = apply5["isolated_restore"] + assert ( + current_restore["database_count"] + == current_restore["restored_database_count"] + == 6 + ) + assert ( + current_restore["table_count"] == current_restore["restored_table_count"] == 100 + ) + assert ( + current_restore["check_table_count"] + == current_restore["check_table_pass_count"] + == 44 + ) + assert current_restore["critical_nonzero_count"] == 4 + assert apply5["restic"]["snapshot_id"] == "135485ab" + current_post = apply5["independent_post_verifier"] + assert current_post["independent_monitor"] == "pass" + assert current_post["deployed_source_hash_drift_count"] == 0 + assert current_post["runtime_drift_count"] == 0 + assert ( + current_post[ + "exact_ephemeral_container_volume_network_process_tmp_residue_count" + ] + == 0 + ) + assert current_post["cleanup_verified"] is True assert canary["terminal"] == "production_closed_clickhouse_native_scope" @@ -677,16 +723,38 @@ def test_current_toolchain_hashes_are_bound_without_rewriting_production_receipt assert current["hashes"]["inventory_verifier_sha256"] == ( "c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863" ) - assert current["status"] == "pending_toolchain_deploy_and_runtime_canary" - assert current["deployment_status"] == "pending" - assert current["current_source_deployed"] is False - assert current["runtime_verifier_terminal"] == "pending" + assert current["deployed_source_hashes"] == current_hashes + assert current["status"] == "deployed_verified" + assert current["deployment_status"] == "deployed" + assert current["current_source_deployed"] is True + assert current["runtime_verifier_terminal"] == "pass" assert current["historical_production_receipt_preserved"] is True assert current["drift_from_historical_production_receipt"] == [ "backup_orchestrator_sha256", "canary_wrapper_sha256", "inventory_verifier_sha256", ] + assert current["toolchain_check_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015040Z-check6" + ) + assert current["toolchain_apply_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015059Z-apply5" + ) + assert current["toolchain_postcheck_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5" + ) + assert current["toolchain_postcanary_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5" + ) + assert current["default_canary_check_run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015131Z-check5" + ) + assert current["runtime_canary_run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015149Z-apply5" + ) + assert current["wrapper_terminal"] == "partial_degraded" + assert current["runtime_operation_terminal"] == "pass" + assert current["nested_machine_receipt_verifier_terminal"] == "pass" def test_preflight_fails_closed_when_current_toolchain_source_hash_is_stale( @@ -707,10 +775,31 @@ def test_preflight_fails_closed_when_current_toolchain_source_hash_is_stale( assert payload["terminal"] == "failed" assert payload["checks"]["native_backup_current_source_hashes_bound"] is False assert payload["checks"]["native_backup_historical_source_receipt_preserved"] - assert ( - payload["backup_toolchain_current_source_status"] - == "pending_toolchain_deploy_and_runtime_canary" - ) + assert payload["backup_toolchain_current_source_status"] == "deployed_verified" + + +def test_preflight_fails_closed_when_current_runtime_receipt_is_incomplete( + monkeypatch, +) -> None: + module = _load_preflight_module() + real_load_yaml = module._load_yaml + + def load_without_operation_identity(path: Path): + payload = real_load_yaml(path) + if path.name == POST_CLOSURE_REGRESSION.name: + attempts = payload["controlled_applies"][ + "clickhouse_native_backup_restore_canary" + ]["attempts"] + attempts[-1]["backup"].pop("operation_identity_hash") + return payload + + monkeypatch.setattr(module, "_load_yaml", load_without_operation_identity) + payload = module.evaluate(ROOT, CONTRACT) + + assert payload["source_ready"] is False + assert payload["terminal"] == "failed" + assert payload["checks"]["native_backup_current_source_runtime_closed"] is False + assert payload["runtime_status"] == "partial_degraded" def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> None: @@ -759,14 +848,33 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> ] assert resume["same_run_resume_safe"] is False assert len(resume["required_preconditions"]) == 4 - current_source = receipt["pending_verifiers"][ + current_source = receipt["completed_verifiers"][ "clickhouse_native_current_source_toolchain" ] - assert current_source["status"] == "pending_toolchain_deploy_and_runtime_canary" - assert current_source["current_source_deployed"] is False - assert current_source["runtime_verifier_terminal"] == "pending" + assert current_source["status"] == "deployed_verified" + assert current_source["current_source_deployed"] is True + assert current_source["runtime_verifier_terminal"] == "pass" assert current_source["historical_production_receipt_preserved"] is True - assert len(current_source["required_preconditions"]) == 3 + assert current_source["toolchain_apply_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015059Z-apply5" + ) + assert current_source["runtime_canary_run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015149Z-apply5" + ) + assert current_source["wrapper_terminal"] == "partial_degraded" + assert current_source["runtime_operation_terminal"] == "pass" + assert current_source["nested_machine_receipt_verifier_terminal"] == "pass" + assert current_source["restic_snapshot_id"] == "135485ab" + assert current_source["artifact_sha256"] == ( + "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83" + ) + assert current_source["independent_monitor"] == "pass" + assert current_source["runtime_drift_count"] == 0 + assert current_source["residue_count"] == 0 + assert current_source["exact_cleanup"] == "pass" + assert ( + "clickhouse_native_current_source_toolchain" not in receipt["pending_verifiers"] + ) assets = receipt["asset_reconciliation"] assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001" assert assets["source_service_count"] == 27 @@ -797,7 +905,6 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> "signoz_backup_offsite_dr_pending", "clickhouse_native_failed_artifact_retention_pending", "clickhouse_native_resume_submitted_inventory_pending", - "clickhouse_native_current_source_toolchain_pending", "service_registry_runtime_mirror_reconciliation_pending", "github_freeze_legacy_asset_retirement_pending", "monitoring_generator_duplicate_awoooi_api_identity_pending", @@ -818,10 +925,12 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N assert post_closure["backup_collector_restore_guard_deployed"] is True assert post_closure["backup_canary_scope"] == "clickhouse_native_only" assert post_closure["backup_canary_verified"] is True - assert post_closure["backup_canary_last_terminal"] == "pass" + assert post_closure["backup_canary_last_terminal"] == "partial_degraded" assert post_closure["backup_canary_last_run_id"] == ( - "P0-OBS-002-native-canary-20260715T001329Z-apply4" + "P0-OBS-002-native-canary-default-20260715T015149Z-apply5" ) + assert post_closure["backup_canary_runtime_operation_terminal"] == "pass" + assert post_closure["backup_canary_machine_receipt_verifier_terminal"] == "pass" assert post_closure["backup_canary_cooldown_retry_status"] == ( "superseded_by_online_native_no_stop_contract" ) @@ -838,12 +947,51 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N assert post_closure["clickhouse_native_backup_toolchain_postcheck_run_id"] == ( "P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4" ) - assert post_closure["clickhouse_native_current_source_status"] == ( - "pending_toolchain_deploy_and_runtime_canary" + assert ( + post_closure["clickhouse_native_current_source_status"] == "deployed_verified" + ) + assert post_closure["clickhouse_native_current_source_deployed"] is True + assert ( + post_closure["clickhouse_native_current_source_runtime_verifier_terminal"] + == "pass" + ) + assert post_closure["clickhouse_native_current_source_toolchain_check_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015040Z-check6" + ) + assert post_closure["clickhouse_native_current_source_toolchain_apply_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015059Z-apply5" + ) + assert ( + post_closure["clickhouse_native_current_source_toolchain_postcheck_run_id"] + == "P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5" + ) + assert ( + post_closure["clickhouse_native_current_source_toolchain_postcanary_run_id"] + == "P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5" + ) + assert post_closure["clickhouse_native_current_source_canary_check_run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015131Z-check5" + ) + assert post_closure["clickhouse_native_current_source_canary_apply_run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015149Z-apply5" + ) + assert post_closure["clickhouse_native_current_source_wrapper_terminal"] == ( + "partial_degraded" + ) + assert post_closure[ + "clickhouse_native_current_source_runtime_operation_terminal" + ] == ("pass") + assert ( + post_closure[ + "clickhouse_native_current_source_machine_receipt_verifier_terminal" + ] + == "pass" + ) + assert post_closure["clickhouse_native_current_source_artifact_sha256"] == ( + "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83" ) - assert post_closure["clickhouse_native_current_source_deployed"] is False assert post_closure["clickhouse_native_restore_terminal"] == "verified" - assert post_closure["clickhouse_native_restic_snapshot_id"] == "574e2a1a" + assert post_closure["clickhouse_native_restic_snapshot_id"] == "135485ab" assert post_closure["clickhouse_native_offsite_status"] == "pending" assert post_closure["clickhouse_native_failed_artifact_retention_status"] == ( "pending_bounded_retention_decision"