chore(observability): record native backup production verifier
This commit is contained in:
ogt
@@ -172,8 +172,10 @@ runtime_observations:
|
||||
backup_collector_restore_guard_deployed: true
|
||||
backup_canary_scope: clickhouse_native_only
|
||||
backup_canary_verified: true
|
||||
backup_canary_last_run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4
|
||||
backup_canary_last_terminal: pass
|
||||
backup_canary_last_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
|
||||
backup_canary_last_terminal: partial_degraded
|
||||
backup_canary_runtime_operation_terminal: pass
|
||||
backup_canary_machine_receipt_verifier_terminal: pass
|
||||
backup_canary_prior_root_cause: raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract
|
||||
backup_canary_cooldown_retry_status: superseded_by_online_native_no_stop_contract
|
||||
clickhouse_native_backup_migration_status: production_closed
|
||||
@@ -181,10 +183,21 @@ runtime_observations:
|
||||
clickhouse_native_backup_disk_status: deployed_verified
|
||||
clickhouse_native_backup_toolchain_run_id: P0-OBS-002-native-toolchain-20260715T001252Z-apply4
|
||||
clickhouse_native_backup_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4
|
||||
clickhouse_native_current_source_status: pending_toolchain_deploy_and_runtime_canary
|
||||
clickhouse_native_current_source_deployed: false
|
||||
clickhouse_native_current_source_status: deployed_verified
|
||||
clickhouse_native_current_source_deployed: true
|
||||
clickhouse_native_current_source_runtime_verifier_terminal: pass
|
||||
clickhouse_native_current_source_toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6
|
||||
clickhouse_native_current_source_toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5
|
||||
clickhouse_native_current_source_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5
|
||||
clickhouse_native_current_source_toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5
|
||||
clickhouse_native_current_source_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5
|
||||
clickhouse_native_current_source_canary_apply_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
|
||||
clickhouse_native_current_source_wrapper_terminal: partial_degraded
|
||||
clickhouse_native_current_source_runtime_operation_terminal: pass
|
||||
clickhouse_native_current_source_machine_receipt_verifier_terminal: pass
|
||||
clickhouse_native_current_source_artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83
|
||||
clickhouse_native_restore_terminal: verified
|
||||
clickhouse_native_restic_snapshot_id: 574e2a1a
|
||||
clickhouse_native_restic_snapshot_id: 135485ab
|
||||
clickhouse_native_offsite_status: pending
|
||||
clickhouse_native_failed_artifact_retention_status: pending_bounded_retention_decision
|
||||
clickhouse_native_resume_inventory_status: pending_durable_submitted_inventory_contract
|
||||
|
||||
@@ -364,12 +364,12 @@ controlled_applies:
|
||||
ruff_format: pass
|
||||
yaml_xml_parse: pass
|
||||
current_source:
|
||||
observed_at: "2026-07-15T09:40:50+08:00"
|
||||
observed_at: "2026-07-15T09:53:46+08:00"
|
||||
evidence_scope: committed_current_source
|
||||
status: pending_toolchain_deploy_and_runtime_canary
|
||||
deployment_status: pending
|
||||
current_source_deployed: false
|
||||
runtime_verifier_terminal: pending
|
||||
status: deployed_verified
|
||||
deployment_status: deployed
|
||||
current_source_deployed: true
|
||||
runtime_verifier_terminal: pass
|
||||
historical_production_receipt_preserved: true
|
||||
expected_file_count: 7
|
||||
hashes:
|
||||
@@ -380,13 +380,38 @@ controlled_applies:
|
||||
canary_wrapper_sha256: e6707c1398e066cbeec5e43a936533682b881b6e2763e21085f85b3de33c5886
|
||||
backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9
|
||||
isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b
|
||||
deployed_source_hashes:
|
||||
backup_orchestrator_sha256: d1950c6ada4be4696f38dabf904bea3167c89293d7ca300365e62c2831cf4df2
|
||||
native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a
|
||||
isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7
|
||||
inventory_verifier_sha256: c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863
|
||||
canary_wrapper_sha256: e6707c1398e066cbeec5e43a936533682b881b6e2763e21085f85b3de33c5886
|
||||
backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9
|
||||
isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b
|
||||
drift_from_historical_production_receipt:
|
||||
- backup_orchestrator_sha256
|
||||
- canary_wrapper_sha256
|
||||
- inventory_verifier_sha256
|
||||
toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6
|
||||
toolchain_check_terminal: check_pass_no_write
|
||||
toolchain_check_diff: matching_4_drift_3_absent_0_active_operations_0
|
||||
toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5
|
||||
toolchain_apply_terminal: pass
|
||||
toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5
|
||||
toolchain_postcheck_terminal: check_pass_no_write
|
||||
toolchain_postcheck_diff: matching_7_drift_0_absent_0_active_operations_0
|
||||
toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5
|
||||
toolchain_postcanary_terminal: check_pass_no_write
|
||||
toolchain_postcanary_diff: matching_7_drift_0_absent_0_active_operations_0
|
||||
default_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5
|
||||
default_canary_check_terminal: check_pass_no_write
|
||||
runtime_canary_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
|
||||
wrapper_terminal: partial_degraded
|
||||
runtime_operation_terminal: pass
|
||||
nested_machine_receipt_verifier_terminal: pass
|
||||
source_validation:
|
||||
backup_restore_deployer_tests_passed: 92
|
||||
canonical_preflight_tests_passed: 23
|
||||
canonical_preflight_tests_passed: 24
|
||||
bash_syntax: pass
|
||||
ruff: pass
|
||||
ruff_format: pass
|
||||
@@ -503,6 +528,109 @@ controlled_applies:
|
||||
cleanup_verified: true
|
||||
secret_values_collected: false
|
||||
closure_writeback: durable_local_receipts_acknowledged
|
||||
- run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
|
||||
check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5
|
||||
check_terminal: check_pass_no_write
|
||||
wrapper_terminal: partial_degraded
|
||||
terminal: partial_degraded
|
||||
runtime_operation:
|
||||
terminal: pass
|
||||
scope: clickhouse_native_backup_and_isolated_restore
|
||||
nested_machine_receipt_verifier:
|
||||
stage: native_restore_receipt_verifier
|
||||
terminal: pass
|
||||
backup:
|
||||
terminal: BACKUP_CREATED
|
||||
terminal_sequence:
|
||||
- check_pass_no_write
|
||||
- pass
|
||||
independent_verifier_terminal: pass
|
||||
operation_identity_hash: d60d3b618798bc1f60eeff4bfcf693ca14c5a1da70150043af90b3fa5e459ec1
|
||||
operation_status: "BACKUP_CREATED|4885|108006575|99461897|"
|
||||
file_count: 4885
|
||||
total_size_bytes: 108006575
|
||||
compressed_size_bytes: 99461897
|
||||
artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83
|
||||
active_operation_count_after: 0
|
||||
server_artifact_absent_after: true
|
||||
isolated_restore:
|
||||
terminal: verified
|
||||
terminal_sequence:
|
||||
- check_pass_no_runtime_write
|
||||
- verified
|
||||
clickhouse_restore_terminal: RESTORED
|
||||
independent_post_verifier_terminal: pass
|
||||
cleanup_terminal: pass
|
||||
network_scope: internal_only
|
||||
internal_network_attached: true
|
||||
staging_network_attached: false
|
||||
production_network_attached: false
|
||||
production_restore_performed: false
|
||||
cleanup_verified: true
|
||||
database_count: 6
|
||||
restored_database_count: 6
|
||||
table_count: 100
|
||||
restored_table_count: 100
|
||||
table_engine_schema_manifest_parity: true
|
||||
check_table_count: 44
|
||||
check_table_pass_count: 44
|
||||
critical_nonzero_count: 4
|
||||
restic:
|
||||
snapshot_id: 135485ab
|
||||
snapshot_readback: pass
|
||||
repository_scope: local_same_failure_domain
|
||||
offsite_verified: false
|
||||
sqlite:
|
||||
terminal: pending_application_consistent_export
|
||||
independent_post_verifier:
|
||||
terminal: pass
|
||||
independent_monitor: pass
|
||||
deployed_source_hash_drift_count: 0
|
||||
runtime_drift_count: 0
|
||||
production_container_count: 4
|
||||
production_container_identity_unchanged: true
|
||||
production_container_restart_count_delta: 0
|
||||
production_containers:
|
||||
clickhouse:
|
||||
started_at: "2026-07-14T22:49:13.665350281Z"
|
||||
restart_count: 0
|
||||
health: healthy
|
||||
collector:
|
||||
started_at: "2026-07-14T22:20:53.725583861Z"
|
||||
restart_count: 0
|
||||
health: not_configured
|
||||
signoz:
|
||||
started_at: "2026-07-11T08:45:17.179074963Z"
|
||||
restart_count: 0
|
||||
health: healthy
|
||||
zookeeper:
|
||||
started_at: "2026-07-11T08:45:17.055893064Z"
|
||||
restart_count: 0
|
||||
health: healthy
|
||||
api_http_status: 200
|
||||
listener_counts:
|
||||
"4317": 2
|
||||
"4318": 2
|
||||
"8080": 2
|
||||
active_backup_operation_count: 0
|
||||
restore_container_count: 0
|
||||
restore_volume_count: 0
|
||||
restore_network_count: 0
|
||||
toolchain_stage_residue_count: 0
|
||||
toolchain_candidate_residue_count: 0
|
||||
toolchain_rollback_residue_count: 0
|
||||
backup_process_count: 0
|
||||
lock_holders:
|
||||
canary: none
|
||||
operation: none
|
||||
toolchain: none
|
||||
server_artifact_absent: true
|
||||
exact_ephemeral_container_volume_network_process_tmp_residue_count: 0
|
||||
cleanup_verified: true
|
||||
closure_writeback:
|
||||
terminal: partial_degraded
|
||||
current_source_runtime_closed: true
|
||||
overall_program_partial_degraded: true
|
||||
terminal: production_closed_clickhouse_native_scope
|
||||
|
||||
backup_canary:
|
||||
@@ -599,6 +727,34 @@ completed_verifiers:
|
||||
restic_snapshot_id: 574e2a1a
|
||||
independent_verifier: pass
|
||||
exact_cleanup: pass
|
||||
clickhouse_native_current_source_toolchain:
|
||||
status: deployed_verified
|
||||
current_source_deployed: true
|
||||
runtime_verifier_terminal: pass
|
||||
historical_production_receipt_preserved: true
|
||||
toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6
|
||||
toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5
|
||||
toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5
|
||||
toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5
|
||||
default_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5
|
||||
runtime_canary_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
|
||||
wrapper_terminal: partial_degraded
|
||||
runtime_operation_terminal: pass
|
||||
nested_machine_receipt_verifier_terminal: pass
|
||||
operation_identity_hash: d60d3b618798bc1f60eeff4bfcf693ca14c5a1da70150043af90b3fa5e459ec1
|
||||
operation_status: "BACKUP_CREATED|4885|108006575|99461897|"
|
||||
restic_snapshot_id: 135485ab
|
||||
artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83
|
||||
offsite_verified: false
|
||||
sqlite_terminal: pending_application_consistent_export
|
||||
independent_monitor: pass
|
||||
api_http_status: 200
|
||||
production_container_identity_unchanged: true
|
||||
production_container_restart_count_delta: 0
|
||||
active_backup_operation_count: 0
|
||||
runtime_drift_count: 0
|
||||
residue_count: 0
|
||||
exact_cleanup: pass
|
||||
|
||||
pending_verifiers:
|
||||
signoz_sqlite_application_consistent_backup:
|
||||
@@ -611,7 +767,7 @@ pending_verifiers:
|
||||
signoz_backup_offsite_dr:
|
||||
status: pending_offsite_snapshot_and_restore_verifier
|
||||
current_repository_scope: local_same_failure_domain
|
||||
latest_local_snapshot_id: 574e2a1a
|
||||
latest_local_snapshot_id: 135485ab
|
||||
offsite_verified: false
|
||||
required_preconditions:
|
||||
- immutable_or_versioned_offsite_target
|
||||
@@ -641,16 +797,6 @@ pending_verifiers:
|
||||
- bind_inventory_hash_operation_id_artifact_hash_and_run_identity
|
||||
- resume_reuses_original_inventory_and_fails_closed_if_missing_or_tampered
|
||||
- regression_test_with_live_counter_drift_and_zero_duplicate_backup_submits
|
||||
clickhouse_native_current_source_toolchain:
|
||||
status: pending_toolchain_deploy_and_runtime_canary
|
||||
current_source_deployed: false
|
||||
runtime_verifier_terminal: pending
|
||||
historical_production_receipt_preserved: true
|
||||
required_preconditions:
|
||||
- controlled_toolchain_apply_with_current_source_hash_readback
|
||||
- same_run_native_backup_and_isolated_restore_canary
|
||||
- independent_runtime_and_cleanup_verifier
|
||||
|
||||
scope_boundaries:
|
||||
grpc_bridge_rollback_indicated: false
|
||||
api_manual_rollback_performed: false
|
||||
@@ -670,7 +816,6 @@ terminal:
|
||||
- signoz_backup_offsite_dr_pending
|
||||
- clickhouse_native_failed_artifact_retention_pending
|
||||
- clickhouse_native_resume_submitted_inventory_pending
|
||||
- clickhouse_native_current_source_toolchain_pending
|
||||
- service_registry_runtime_mirror_reconciliation_pending
|
||||
- github_freeze_legacy_asset_retirement_pending
|
||||
- monitoring_generator_duplicate_awoooi_api_identity_pending
|
||||
|
||||
@@ -31,7 +31,6 @@ EXPECTED_POST_CLOSURE_BLOCKERS = [
|
||||
"signoz_backup_offsite_dr_pending",
|
||||
"clickhouse_native_failed_artifact_retention_pending",
|
||||
"clickhouse_native_resume_submitted_inventory_pending",
|
||||
"clickhouse_native_current_source_toolchain_pending",
|
||||
"service_registry_runtime_mirror_reconciliation_pending",
|
||||
"github_freeze_legacy_asset_retirement_pending",
|
||||
"monitoring_generator_duplicate_awoooi_api_identity_pending",
|
||||
@@ -474,16 +473,35 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
|
||||
native_canary = applies.get("clickhouse_native_backup_restore_canary", {})
|
||||
native_attempts = native_canary.get("attempts", [])
|
||||
native_apply2 = native_attempts[1] if len(native_attempts) == 4 else {}
|
||||
native_apply3 = native_attempts[2] if len(native_attempts) == 4 else {}
|
||||
native_apply4 = native_attempts[3] if len(native_attempts) == 4 else {}
|
||||
native_attempts_by_run_id = {
|
||||
attempt.get("run_id"): attempt
|
||||
for attempt in native_attempts
|
||||
if isinstance(attempt, dict) and attempt.get("run_id")
|
||||
}
|
||||
native_apply2 = native_attempts_by_run_id.get(
|
||||
"P0-OBS-002-native-canary-20260714T234436Z-apply2", {}
|
||||
)
|
||||
native_apply3 = native_attempts_by_run_id.get(
|
||||
"P0-OBS-002-native-canary-20260715T000638Z-apply3", {}
|
||||
)
|
||||
native_apply4 = native_attempts_by_run_id.get(
|
||||
"P0-OBS-002-native-canary-20260715T001329Z-apply4", {}
|
||||
)
|
||||
native_apply5 = native_attempts_by_run_id.get(
|
||||
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5", {}
|
||||
)
|
||||
native_backup = native_apply4.get("backup", {})
|
||||
native_restore = native_apply4.get("isolated_restore", {})
|
||||
native_restic = native_apply4.get("restic", {})
|
||||
native_post = native_apply4.get("independent_post_verifier", {})
|
||||
current_backup = native_apply5.get("backup", {})
|
||||
current_restore = native_apply5.get("isolated_restore", {})
|
||||
current_restic = native_apply5.get("restic", {})
|
||||
current_post = native_apply5.get("independent_post_verifier", {})
|
||||
toolchain_apply = native_canary.get("toolchain_controlled_apply", {})
|
||||
historical_native_source = native_canary.get("source", {})
|
||||
current_native_source = native_canary.get("current_source", {})
|
||||
current_source_validation = current_native_source.get("source_validation", {})
|
||||
historical_source_hashes = {
|
||||
field: historical_native_source.get(field) for field in TOOLCHAIN_SOURCE_PATHS
|
||||
}
|
||||
@@ -511,25 +529,166 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
and declared_current_hashes == actual_current_hashes
|
||||
and declared_historical_drift == actual_historical_drift
|
||||
)
|
||||
pending_source_state = (
|
||||
current_native_source.get("status")
|
||||
== "pending_toolchain_deploy_and_runtime_canary"
|
||||
and current_native_source.get("deployment_status") == "pending"
|
||||
and current_native_source.get("current_source_deployed") is False
|
||||
and current_native_source.get("runtime_verifier_terminal") == "pending"
|
||||
)
|
||||
deployed_source_state = (
|
||||
current_native_source.get("status") == "deployed_verified"
|
||||
and current_native_source.get("deployment_status") == "deployed"
|
||||
and current_native_source.get("current_source_deployed") is True
|
||||
and current_native_source.get("runtime_verifier_terminal") == "pass"
|
||||
and current_native_source.get("deployed_source_hashes") == actual_current_hashes
|
||||
and bool(current_native_source.get("toolchain_apply_run_id"))
|
||||
and bool(current_native_source.get("runtime_canary_run_id"))
|
||||
and current_native_source.get("toolchain_check_run_id")
|
||||
== "P0-OBS-002-native-toolchain-20260715T015040Z-check6"
|
||||
and current_native_source.get("toolchain_check_terminal")
|
||||
== "check_pass_no_write"
|
||||
and current_native_source.get("toolchain_check_diff")
|
||||
== "matching_4_drift_3_absent_0_active_operations_0"
|
||||
and current_native_source.get("toolchain_apply_run_id")
|
||||
== "P0-OBS-002-native-toolchain-20260715T015059Z-apply5"
|
||||
and current_native_source.get("toolchain_apply_terminal") == "pass"
|
||||
and current_native_source.get("toolchain_postcheck_run_id")
|
||||
== "P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5"
|
||||
and current_native_source.get("toolchain_postcheck_terminal")
|
||||
== "check_pass_no_write"
|
||||
and current_native_source.get("toolchain_postcheck_diff")
|
||||
== "matching_7_drift_0_absent_0_active_operations_0"
|
||||
and current_native_source.get("toolchain_postcanary_run_id")
|
||||
== "P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5"
|
||||
and current_native_source.get("toolchain_postcanary_terminal")
|
||||
== "check_pass_no_write"
|
||||
and current_native_source.get("toolchain_postcanary_diff")
|
||||
== "matching_7_drift_0_absent_0_active_operations_0"
|
||||
and current_native_source.get("default_canary_check_run_id")
|
||||
== "P0-OBS-002-native-canary-default-20260715T015131Z-check5"
|
||||
and current_native_source.get("default_canary_check_terminal")
|
||||
== "check_pass_no_write"
|
||||
and current_native_source.get("runtime_canary_run_id")
|
||||
== "P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
||||
and current_native_source.get("wrapper_terminal") == "partial_degraded"
|
||||
and current_native_source.get("runtime_operation_terminal") == "pass"
|
||||
and current_native_source.get("nested_machine_receipt_verifier_terminal")
|
||||
== "pass"
|
||||
and current_source_validation.get("backup_restore_deployer_tests_passed") == 92
|
||||
and current_source_validation.get("canonical_preflight_tests_passed") == 24
|
||||
and current_source_validation.get("bash_syntax") == "pass"
|
||||
and current_source_validation.get("ruff") == "pass"
|
||||
and current_source_validation.get("ruff_format") == "pass"
|
||||
and current_source_validation.get("yaml_parse") == "pass"
|
||||
and current_source_validation.get("diff_check") == "pass"
|
||||
and current_source_validation.get("independent_reaudit") == "no_commit_blocker"
|
||||
)
|
||||
checks["native_backup_current_source_deployment_state_explicit"] = (
|
||||
current_native_source.get("historical_production_receipt_preserved") is True
|
||||
and (pending_source_state or deployed_source_state)
|
||||
and deployed_source_state
|
||||
)
|
||||
checks["native_backup_current_source_runtime_closed"] = (
|
||||
native_apply5.get("check_run_id")
|
||||
== "P0-OBS-002-native-canary-default-20260715T015131Z-check5"
|
||||
and native_apply5.get("check_terminal") == "check_pass_no_write"
|
||||
and native_apply5.get("wrapper_terminal") == "partial_degraded"
|
||||
and native_apply5.get("terminal") == "partial_degraded"
|
||||
and native_apply5.get("runtime_operation", {}).get("terminal") == "pass"
|
||||
and native_apply5.get("nested_machine_receipt_verifier", {}).get("stage")
|
||||
== "native_restore_receipt_verifier"
|
||||
and native_apply5.get("nested_machine_receipt_verifier", {}).get("terminal")
|
||||
== "pass"
|
||||
and current_backup.get("terminal") == "BACKUP_CREATED"
|
||||
and current_backup.get("terminal_sequence") == ["check_pass_no_write", "pass"]
|
||||
and current_backup.get("independent_verifier_terminal") == "pass"
|
||||
and current_backup.get("operation_identity_hash")
|
||||
== "d60d3b618798bc1f60eeff4bfcf693ca14c5a1da70150043af90b3fa5e459ec1"
|
||||
and current_backup.get("operation_status")
|
||||
== "BACKUP_CREATED|4885|108006575|99461897|"
|
||||
and current_backup.get("file_count") == 4885
|
||||
and current_backup.get("total_size_bytes") == 108006575
|
||||
and current_backup.get("compressed_size_bytes") == 99461897
|
||||
and current_backup.get("artifact_sha256")
|
||||
== "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
|
||||
and current_backup.get("active_operation_count_after") == 0
|
||||
and current_backup.get("server_artifact_absent_after") is True
|
||||
and current_restore.get("terminal") == "verified"
|
||||
and current_restore.get("terminal_sequence")
|
||||
== ["check_pass_no_runtime_write", "verified"]
|
||||
and current_restore.get("clickhouse_restore_terminal") == "RESTORED"
|
||||
and current_restore.get("independent_post_verifier_terminal") == "pass"
|
||||
and current_restore.get("cleanup_terminal") == "pass"
|
||||
and current_restore.get("network_scope") == "internal_only"
|
||||
and current_restore.get("internal_network_attached") is True
|
||||
and current_restore.get("staging_network_attached") is False
|
||||
and current_restore.get("production_network_attached") is False
|
||||
and current_restore.get("production_restore_performed") is False
|
||||
and current_restore.get("cleanup_verified") is True
|
||||
and current_restore.get("database_count")
|
||||
== current_restore.get("restored_database_count")
|
||||
== 6
|
||||
and current_restore.get("table_count")
|
||||
== current_restore.get("restored_table_count")
|
||||
== 100
|
||||
and current_restore.get("table_engine_schema_manifest_parity") is True
|
||||
and current_restore.get("check_table_count")
|
||||
== current_restore.get("check_table_pass_count")
|
||||
== 44
|
||||
and current_restore.get("critical_nonzero_count") == 4
|
||||
and current_restic.get("snapshot_id") == "135485ab"
|
||||
and current_restic.get("snapshot_readback") == "pass"
|
||||
and current_restic.get("repository_scope") == "local_same_failure_domain"
|
||||
and current_restic.get("offsite_verified") is False
|
||||
and native_apply5.get("sqlite", {}).get("terminal")
|
||||
== "pending_application_consistent_export"
|
||||
and current_post.get("terminal") == "pass"
|
||||
and current_post.get("independent_monitor") == "pass"
|
||||
and current_post.get("deployed_source_hash_drift_count") == 0
|
||||
and current_post.get("runtime_drift_count") == 0
|
||||
and current_post.get("production_container_count") == 4
|
||||
and current_post.get("production_container_identity_unchanged") is True
|
||||
and current_post.get("production_container_restart_count_delta") == 0
|
||||
and current_post.get("production_containers")
|
||||
== {
|
||||
"clickhouse": {
|
||||
"started_at": "2026-07-14T22:49:13.665350281Z",
|
||||
"restart_count": 0,
|
||||
"health": "healthy",
|
||||
},
|
||||
"collector": {
|
||||
"started_at": "2026-07-14T22:20:53.725583861Z",
|
||||
"restart_count": 0,
|
||||
"health": "not_configured",
|
||||
},
|
||||
"signoz": {
|
||||
"started_at": "2026-07-11T08:45:17.179074963Z",
|
||||
"restart_count": 0,
|
||||
"health": "healthy",
|
||||
},
|
||||
"zookeeper": {
|
||||
"started_at": "2026-07-11T08:45:17.055893064Z",
|
||||
"restart_count": 0,
|
||||
"health": "healthy",
|
||||
},
|
||||
}
|
||||
and current_post.get("api_http_status") == 200
|
||||
and current_post.get("listener_counts") == {"4317": 2, "4318": 2, "8080": 2}
|
||||
and current_post.get("active_backup_operation_count") == 0
|
||||
and current_post.get("restore_container_count") == 0
|
||||
and current_post.get("restore_volume_count") == 0
|
||||
and current_post.get("restore_network_count") == 0
|
||||
and current_post.get("toolchain_stage_residue_count") == 0
|
||||
and current_post.get("toolchain_candidate_residue_count") == 0
|
||||
and current_post.get("toolchain_rollback_residue_count") == 0
|
||||
and current_post.get("backup_process_count") == 0
|
||||
and current_post.get("lock_holders")
|
||||
== {"canary": "none", "operation": "none", "toolchain": "none"}
|
||||
and current_post.get("server_artifact_absent") is True
|
||||
and current_post.get(
|
||||
"exact_ephemeral_container_volume_network_process_tmp_residue_count"
|
||||
)
|
||||
== 0
|
||||
and current_post.get("cleanup_verified") is True
|
||||
and native_apply5.get("closure_writeback", {}).get(
|
||||
"current_source_runtime_closed"
|
||||
)
|
||||
is True
|
||||
and native_apply5.get("closure_writeback", {}).get(
|
||||
"overall_program_partial_degraded"
|
||||
)
|
||||
is True
|
||||
)
|
||||
checks["native_backup_restore_canary_closed"] = (
|
||||
native_canary.get("completion_scope") == "clickhouse_native_only"
|
||||
@@ -655,8 +814,10 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
is True
|
||||
)
|
||||
|
||||
native_migration = regression.get("completed_verifiers", {}).get(
|
||||
"clickhouse_native_backup_migration", {}
|
||||
completed_verifiers = regression.get("completed_verifiers", {})
|
||||
native_migration = completed_verifiers.get("clickhouse_native_backup_migration", {})
|
||||
current_source_toolchain = completed_verifiers.get(
|
||||
"clickhouse_native_current_source_toolchain", {}
|
||||
)
|
||||
pending_verifiers = regression.get("pending_verifiers", {})
|
||||
sqlite_backup = pending_verifiers.get(
|
||||
@@ -669,9 +830,6 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
resume_inventory = pending_verifiers.get(
|
||||
"clickhouse_native_resume_submitted_inventory", {}
|
||||
)
|
||||
current_source_toolchain = pending_verifiers.get(
|
||||
"clickhouse_native_current_source_toolchain", {}
|
||||
)
|
||||
checks["native_backup_migration_closed_and_remaining_gaps_explicit"] = (
|
||||
native_migration.get("prior_run_id") == retry2.get("run_id")
|
||||
and native_migration.get("run_id") == native_apply4.get("run_id")
|
||||
@@ -701,12 +859,47 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
and len(resume_inventory.get("required_preconditions", [])) == 4
|
||||
and current_source_toolchain.get("status")
|
||||
== current_native_source.get("status")
|
||||
== "pending_toolchain_deploy_and_runtime_canary"
|
||||
and current_source_toolchain.get("current_source_deployed") is False
|
||||
and current_source_toolchain.get("runtime_verifier_terminal") == "pending"
|
||||
== "deployed_verified"
|
||||
and current_source_toolchain.get("current_source_deployed") is True
|
||||
and current_source_toolchain.get("runtime_verifier_terminal") == "pass"
|
||||
and current_source_toolchain.get("historical_production_receipt_preserved")
|
||||
is True
|
||||
and len(current_source_toolchain.get("required_preconditions", [])) == 3
|
||||
and current_source_toolchain.get("toolchain_check_run_id")
|
||||
== current_native_source.get("toolchain_check_run_id")
|
||||
and current_source_toolchain.get("toolchain_apply_run_id")
|
||||
== current_native_source.get("toolchain_apply_run_id")
|
||||
and current_source_toolchain.get("toolchain_postcheck_run_id")
|
||||
== current_native_source.get("toolchain_postcheck_run_id")
|
||||
and current_source_toolchain.get("toolchain_postcanary_run_id")
|
||||
== current_native_source.get("toolchain_postcanary_run_id")
|
||||
and current_source_toolchain.get("default_canary_check_run_id")
|
||||
== current_native_source.get("default_canary_check_run_id")
|
||||
and current_source_toolchain.get("runtime_canary_run_id")
|
||||
== current_native_source.get("runtime_canary_run_id")
|
||||
and current_source_toolchain.get("wrapper_terminal") == "partial_degraded"
|
||||
and current_source_toolchain.get("runtime_operation_terminal") == "pass"
|
||||
and current_source_toolchain.get("nested_machine_receipt_verifier_terminal")
|
||||
== "pass"
|
||||
and current_source_toolchain.get("operation_identity_hash")
|
||||
== current_backup.get("operation_identity_hash")
|
||||
and current_source_toolchain.get("operation_status")
|
||||
== current_backup.get("operation_status")
|
||||
and current_source_toolchain.get("restic_snapshot_id") == "135485ab"
|
||||
and current_source_toolchain.get("artifact_sha256")
|
||||
== "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
|
||||
and current_source_toolchain.get("offsite_verified") is False
|
||||
and current_source_toolchain.get("sqlite_terminal")
|
||||
== "pending_application_consistent_export"
|
||||
and current_source_toolchain.get("independent_monitor") == "pass"
|
||||
and current_source_toolchain.get("api_http_status") == 200
|
||||
and current_source_toolchain.get("production_container_identity_unchanged")
|
||||
is True
|
||||
and current_source_toolchain.get("production_container_restart_count_delta")
|
||||
== 0
|
||||
and current_source_toolchain.get("active_backup_operation_count") == 0
|
||||
and current_source_toolchain.get("runtime_drift_count") == 0
|
||||
and current_source_toolchain.get("residue_count") == 0
|
||||
and current_source_toolchain.get("exact_cleanup") == "pass"
|
||||
)
|
||||
|
||||
checks["post_closure_contract_mirror_consistent"] = (
|
||||
@@ -719,8 +912,11 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
and post_closure.get("backup_collector_restore_guard_deployed") is True
|
||||
and post_closure.get("backup_canary_scope") == "clickhouse_native_only"
|
||||
and post_closure.get("backup_canary_verified") is True
|
||||
and post_closure.get("backup_canary_last_run_id") == native_apply4.get("run_id")
|
||||
and post_closure.get("backup_canary_last_terminal") == "pass"
|
||||
and post_closure.get("backup_canary_last_run_id") == native_apply5.get("run_id")
|
||||
and post_closure.get("backup_canary_last_terminal") == "partial_degraded"
|
||||
and post_closure.get("backup_canary_runtime_operation_terminal") == "pass"
|
||||
and post_closure.get("backup_canary_machine_receipt_verifier_terminal")
|
||||
== "pass"
|
||||
and post_closure.get("backup_canary_cooldown_retry_status")
|
||||
== "superseded_by_online_native_no_stop_contract"
|
||||
and post_closure.get("clickhouse_native_backup_migration_status")
|
||||
@@ -737,8 +933,40 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
== current_native_source.get("status")
|
||||
and post_closure.get("clickhouse_native_current_source_deployed")
|
||||
== current_native_source.get("current_source_deployed")
|
||||
and post_closure.get(
|
||||
"clickhouse_native_current_source_runtime_verifier_terminal"
|
||||
)
|
||||
== current_native_source.get("runtime_verifier_terminal")
|
||||
and post_closure.get("clickhouse_native_current_source_toolchain_check_run_id")
|
||||
== current_native_source.get("toolchain_check_run_id")
|
||||
and post_closure.get("clickhouse_native_current_source_toolchain_apply_run_id")
|
||||
== current_native_source.get("toolchain_apply_run_id")
|
||||
and post_closure.get(
|
||||
"clickhouse_native_current_source_toolchain_postcheck_run_id"
|
||||
)
|
||||
== current_native_source.get("toolchain_postcheck_run_id")
|
||||
and post_closure.get(
|
||||
"clickhouse_native_current_source_toolchain_postcanary_run_id"
|
||||
)
|
||||
== current_native_source.get("toolchain_postcanary_run_id")
|
||||
and post_closure.get("clickhouse_native_current_source_canary_check_run_id")
|
||||
== current_native_source.get("default_canary_check_run_id")
|
||||
and post_closure.get("clickhouse_native_current_source_canary_apply_run_id")
|
||||
== current_native_source.get("runtime_canary_run_id")
|
||||
and post_closure.get("clickhouse_native_current_source_wrapper_terminal")
|
||||
== "partial_degraded"
|
||||
and post_closure.get(
|
||||
"clickhouse_native_current_source_runtime_operation_terminal"
|
||||
)
|
||||
== "pass"
|
||||
and post_closure.get(
|
||||
"clickhouse_native_current_source_machine_receipt_verifier_terminal"
|
||||
)
|
||||
== "pass"
|
||||
and post_closure.get("clickhouse_native_current_source_artifact_sha256")
|
||||
== "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
|
||||
and post_closure.get("clickhouse_native_restore_terminal") == "verified"
|
||||
and post_closure.get("clickhouse_native_restic_snapshot_id") == "574e2a1a"
|
||||
and post_closure.get("clickhouse_native_restic_snapshot_id") == "135485ab"
|
||||
and post_closure.get("clickhouse_native_offsite_status") == "pending"
|
||||
and post_closure.get("clickhouse_native_failed_artifact_retention_status")
|
||||
== "pending_bounded_retention_decision"
|
||||
|
||||
@@ -98,12 +98,9 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
|
||||
assert payload["writes_performed"] == 0
|
||||
assert payload["terminal"] == "pass"
|
||||
assert payload["runtime_status"] == "partial_degraded"
|
||||
assert (
|
||||
payload["backup_toolchain_current_source_status"]
|
||||
== "pending_toolchain_deploy_and_runtime_canary"
|
||||
)
|
||||
assert payload["backup_toolchain_current_source_deployed"] is False
|
||||
assert payload["backup_toolchain_runtime_verifier_terminal"] == "pending"
|
||||
assert payload["backup_toolchain_current_source_status"] == "deployed_verified"
|
||||
assert payload["backup_toolchain_current_source_deployed"] is True
|
||||
assert payload["backup_toolchain_runtime_verifier_terminal"] == "pass"
|
||||
assert payload["backup_toolchain_source_drift_fields"] == [
|
||||
"backup_orchestrator_sha256",
|
||||
"canary_wrapper_sha256",
|
||||
@@ -119,6 +116,7 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
|
||||
payload["checks"]["native_backup_current_source_deployment_state_explicit"]
|
||||
is True
|
||||
)
|
||||
assert payload["checks"]["native_backup_current_source_runtime_closed"] is True
|
||||
assert payload["checks"]["native_backup_historical_source_receipt_preserved"]
|
||||
assert payload["checks"]["post_release_600s_verifier_consistent"] is True
|
||||
assert (
|
||||
@@ -602,8 +600,8 @@ def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> No
|
||||
assert canary["toolchain_controlled_apply"]["postcheck_diff"] == (
|
||||
"matching_7_drift_0_absent_0_active_operations_0"
|
||||
)
|
||||
assert len(canary["attempts"]) == 4
|
||||
_, apply2, apply3, apply4 = canary["attempts"]
|
||||
assert len(canary["attempts"]) == 5
|
||||
_, apply2, apply3, apply4, apply5 = canary["attempts"]
|
||||
assert apply2["artifact_retention"] == "preserved_failed_run_evidence"
|
||||
assert apply2["cleanup_verified"] is True
|
||||
assert apply3["artifact_retention"] == "preserved_failed_run_evidence"
|
||||
@@ -647,6 +645,54 @@ def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> No
|
||||
)
|
||||
assert post["cleanup_verified"] is True
|
||||
assert post["secret_values_collected"] is False
|
||||
|
||||
assert apply5["run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
||||
)
|
||||
assert apply5["check_run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015131Z-check5"
|
||||
)
|
||||
assert apply5["wrapper_terminal"] == "partial_degraded"
|
||||
assert apply5["terminal"] == "partial_degraded"
|
||||
assert apply5["runtime_operation"]["terminal"] == "pass"
|
||||
assert apply5["nested_machine_receipt_verifier"] == {
|
||||
"stage": "native_restore_receipt_verifier",
|
||||
"terminal": "pass",
|
||||
}
|
||||
assert apply5["backup"]["terminal"] == "BACKUP_CREATED"
|
||||
assert apply5["backup"]["file_count"] == 4885
|
||||
assert apply5["backup"]["total_size_bytes"] == 108006575
|
||||
assert apply5["backup"]["compressed_size_bytes"] == 99461897
|
||||
assert apply5["backup"]["artifact_sha256"] == (
|
||||
"08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
|
||||
)
|
||||
current_restore = apply5["isolated_restore"]
|
||||
assert (
|
||||
current_restore["database_count"]
|
||||
== current_restore["restored_database_count"]
|
||||
== 6
|
||||
)
|
||||
assert (
|
||||
current_restore["table_count"] == current_restore["restored_table_count"] == 100
|
||||
)
|
||||
assert (
|
||||
current_restore["check_table_count"]
|
||||
== current_restore["check_table_pass_count"]
|
||||
== 44
|
||||
)
|
||||
assert current_restore["critical_nonzero_count"] == 4
|
||||
assert apply5["restic"]["snapshot_id"] == "135485ab"
|
||||
current_post = apply5["independent_post_verifier"]
|
||||
assert current_post["independent_monitor"] == "pass"
|
||||
assert current_post["deployed_source_hash_drift_count"] == 0
|
||||
assert current_post["runtime_drift_count"] == 0
|
||||
assert (
|
||||
current_post[
|
||||
"exact_ephemeral_container_volume_network_process_tmp_residue_count"
|
||||
]
|
||||
== 0
|
||||
)
|
||||
assert current_post["cleanup_verified"] is True
|
||||
assert canary["terminal"] == "production_closed_clickhouse_native_scope"
|
||||
|
||||
|
||||
@@ -677,16 +723,38 @@ def test_current_toolchain_hashes_are_bound_without_rewriting_production_receipt
|
||||
assert current["hashes"]["inventory_verifier_sha256"] == (
|
||||
"c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863"
|
||||
)
|
||||
assert current["status"] == "pending_toolchain_deploy_and_runtime_canary"
|
||||
assert current["deployment_status"] == "pending"
|
||||
assert current["current_source_deployed"] is False
|
||||
assert current["runtime_verifier_terminal"] == "pending"
|
||||
assert current["deployed_source_hashes"] == current_hashes
|
||||
assert current["status"] == "deployed_verified"
|
||||
assert current["deployment_status"] == "deployed"
|
||||
assert current["current_source_deployed"] is True
|
||||
assert current["runtime_verifier_terminal"] == "pass"
|
||||
assert current["historical_production_receipt_preserved"] is True
|
||||
assert current["drift_from_historical_production_receipt"] == [
|
||||
"backup_orchestrator_sha256",
|
||||
"canary_wrapper_sha256",
|
||||
"inventory_verifier_sha256",
|
||||
]
|
||||
assert current["toolchain_check_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015040Z-check6"
|
||||
)
|
||||
assert current["toolchain_apply_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015059Z-apply5"
|
||||
)
|
||||
assert current["toolchain_postcheck_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5"
|
||||
)
|
||||
assert current["toolchain_postcanary_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5"
|
||||
)
|
||||
assert current["default_canary_check_run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015131Z-check5"
|
||||
)
|
||||
assert current["runtime_canary_run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
||||
)
|
||||
assert current["wrapper_terminal"] == "partial_degraded"
|
||||
assert current["runtime_operation_terminal"] == "pass"
|
||||
assert current["nested_machine_receipt_verifier_terminal"] == "pass"
|
||||
|
||||
|
||||
def test_preflight_fails_closed_when_current_toolchain_source_hash_is_stale(
|
||||
@@ -707,10 +775,31 @@ def test_preflight_fails_closed_when_current_toolchain_source_hash_is_stale(
|
||||
assert payload["terminal"] == "failed"
|
||||
assert payload["checks"]["native_backup_current_source_hashes_bound"] is False
|
||||
assert payload["checks"]["native_backup_historical_source_receipt_preserved"]
|
||||
assert (
|
||||
payload["backup_toolchain_current_source_status"]
|
||||
== "pending_toolchain_deploy_and_runtime_canary"
|
||||
)
|
||||
assert payload["backup_toolchain_current_source_status"] == "deployed_verified"
|
||||
|
||||
|
||||
def test_preflight_fails_closed_when_current_runtime_receipt_is_incomplete(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
module = _load_preflight_module()
|
||||
real_load_yaml = module._load_yaml
|
||||
|
||||
def load_without_operation_identity(path: Path):
|
||||
payload = real_load_yaml(path)
|
||||
if path.name == POST_CLOSURE_REGRESSION.name:
|
||||
attempts = payload["controlled_applies"][
|
||||
"clickhouse_native_backup_restore_canary"
|
||||
]["attempts"]
|
||||
attempts[-1]["backup"].pop("operation_identity_hash")
|
||||
return payload
|
||||
|
||||
monkeypatch.setattr(module, "_load_yaml", load_without_operation_identity)
|
||||
payload = module.evaluate(ROOT, CONTRACT)
|
||||
|
||||
assert payload["source_ready"] is False
|
||||
assert payload["terminal"] == "failed"
|
||||
assert payload["checks"]["native_backup_current_source_runtime_closed"] is False
|
||||
assert payload["runtime_status"] == "partial_degraded"
|
||||
|
||||
|
||||
def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> None:
|
||||
@@ -759,14 +848,33 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() ->
|
||||
]
|
||||
assert resume["same_run_resume_safe"] is False
|
||||
assert len(resume["required_preconditions"]) == 4
|
||||
current_source = receipt["pending_verifiers"][
|
||||
current_source = receipt["completed_verifiers"][
|
||||
"clickhouse_native_current_source_toolchain"
|
||||
]
|
||||
assert current_source["status"] == "pending_toolchain_deploy_and_runtime_canary"
|
||||
assert current_source["current_source_deployed"] is False
|
||||
assert current_source["runtime_verifier_terminal"] == "pending"
|
||||
assert current_source["status"] == "deployed_verified"
|
||||
assert current_source["current_source_deployed"] is True
|
||||
assert current_source["runtime_verifier_terminal"] == "pass"
|
||||
assert current_source["historical_production_receipt_preserved"] is True
|
||||
assert len(current_source["required_preconditions"]) == 3
|
||||
assert current_source["toolchain_apply_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015059Z-apply5"
|
||||
)
|
||||
assert current_source["runtime_canary_run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
||||
)
|
||||
assert current_source["wrapper_terminal"] == "partial_degraded"
|
||||
assert current_source["runtime_operation_terminal"] == "pass"
|
||||
assert current_source["nested_machine_receipt_verifier_terminal"] == "pass"
|
||||
assert current_source["restic_snapshot_id"] == "135485ab"
|
||||
assert current_source["artifact_sha256"] == (
|
||||
"08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
|
||||
)
|
||||
assert current_source["independent_monitor"] == "pass"
|
||||
assert current_source["runtime_drift_count"] == 0
|
||||
assert current_source["residue_count"] == 0
|
||||
assert current_source["exact_cleanup"] == "pass"
|
||||
assert (
|
||||
"clickhouse_native_current_source_toolchain" not in receipt["pending_verifiers"]
|
||||
)
|
||||
assets = receipt["asset_reconciliation"]
|
||||
assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001"
|
||||
assert assets["source_service_count"] == 27
|
||||
@@ -797,7 +905,6 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() ->
|
||||
"signoz_backup_offsite_dr_pending",
|
||||
"clickhouse_native_failed_artifact_retention_pending",
|
||||
"clickhouse_native_resume_submitted_inventory_pending",
|
||||
"clickhouse_native_current_source_toolchain_pending",
|
||||
"service_registry_runtime_mirror_reconciliation_pending",
|
||||
"github_freeze_legacy_asset_retirement_pending",
|
||||
"monitoring_generator_duplicate_awoooi_api_identity_pending",
|
||||
@@ -818,10 +925,12 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N
|
||||
assert post_closure["backup_collector_restore_guard_deployed"] is True
|
||||
assert post_closure["backup_canary_scope"] == "clickhouse_native_only"
|
||||
assert post_closure["backup_canary_verified"] is True
|
||||
assert post_closure["backup_canary_last_terminal"] == "pass"
|
||||
assert post_closure["backup_canary_last_terminal"] == "partial_degraded"
|
||||
assert post_closure["backup_canary_last_run_id"] == (
|
||||
"P0-OBS-002-native-canary-20260715T001329Z-apply4"
|
||||
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
||||
)
|
||||
assert post_closure["backup_canary_runtime_operation_terminal"] == "pass"
|
||||
assert post_closure["backup_canary_machine_receipt_verifier_terminal"] == "pass"
|
||||
assert post_closure["backup_canary_cooldown_retry_status"] == (
|
||||
"superseded_by_online_native_no_stop_contract"
|
||||
)
|
||||
@@ -838,12 +947,51 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N
|
||||
assert post_closure["clickhouse_native_backup_toolchain_postcheck_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_status"] == (
|
||||
"pending_toolchain_deploy_and_runtime_canary"
|
||||
assert (
|
||||
post_closure["clickhouse_native_current_source_status"] == "deployed_verified"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_deployed"] is True
|
||||
assert (
|
||||
post_closure["clickhouse_native_current_source_runtime_verifier_terminal"]
|
||||
== "pass"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_toolchain_check_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015040Z-check6"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_toolchain_apply_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015059Z-apply5"
|
||||
)
|
||||
assert (
|
||||
post_closure["clickhouse_native_current_source_toolchain_postcheck_run_id"]
|
||||
== "P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5"
|
||||
)
|
||||
assert (
|
||||
post_closure["clickhouse_native_current_source_toolchain_postcanary_run_id"]
|
||||
== "P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_canary_check_run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015131Z-check5"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_canary_apply_run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_wrapper_terminal"] == (
|
||||
"partial_degraded"
|
||||
)
|
||||
assert post_closure[
|
||||
"clickhouse_native_current_source_runtime_operation_terminal"
|
||||
] == ("pass")
|
||||
assert (
|
||||
post_closure[
|
||||
"clickhouse_native_current_source_machine_receipt_verifier_terminal"
|
||||
]
|
||||
== "pass"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_artifact_sha256"] == (
|
||||
"08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_deployed"] is False
|
||||
assert post_closure["clickhouse_native_restore_terminal"] == "verified"
|
||||
assert post_closure["clickhouse_native_restic_snapshot_id"] == "574e2a1a"
|
||||
assert post_closure["clickhouse_native_restic_snapshot_id"] == "135485ab"
|
||||
assert post_closure["clickhouse_native_offsite_status"] == "pending"
|
||||
assert post_closure["clickhouse_native_failed_artifact_retention_status"] == (
|
||||
"pending_bounded_retention_decision"
|
||||
|
||||
Reference in New Issue
Block a user