wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

chore(observability): record native backup production verifier

Browse Source
This commit is contained in:
ogt
2026-07-15 10:17:00 +08:00
parent 227189cb91
commit ca8403ce79
4 changed files with 611 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
View File

@@ -98,12 +98,9 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
assert payload["writes_performed"] == 0
assert payload["terminal"] == "pass"
assert payload["runtime_status"] == "partial_degraded"
assert (
payload["backup_toolchain_current_source_status"]
== "pending_toolchain_deploy_and_runtime_canary"
)
assert payload["backup_toolchain_current_source_deployed"] is False
assert payload["backup_toolchain_runtime_verifier_terminal"] == "pending"
assert payload["backup_toolchain_current_source_status"] == "deployed_verified"
assert payload["backup_toolchain_current_source_deployed"] is True
assert payload["backup_toolchain_runtime_verifier_terminal"] == "pass"
assert payload["backup_toolchain_source_drift_fields"] == [
"backup_orchestrator_sha256",
"canary_wrapper_sha256",
@@ -119,6 +116,7 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
payload["checks"]["native_backup_current_source_deployment_state_explicit"]
is True
)
assert payload["checks"]["native_backup_current_source_runtime_closed"] is True
assert payload["checks"]["native_backup_historical_source_receipt_preserved"]
assert payload["checks"]["post_release_600s_verifier_consistent"] is True
assert (
@@ -602,8 +600,8 @@ def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> No
assert canary["toolchain_controlled_apply"]["postcheck_diff"] == (
"matching_7_drift_0_absent_0_active_operations_0"
)
assert len(canary["attempts"]) == 4
_, apply2, apply3, apply4 = canary["attempts"]
assert len(canary["attempts"]) == 5
_, apply2, apply3, apply4, apply5 = canary["attempts"]
assert apply2["artifact_retention"] == "preserved_failed_run_evidence"
assert apply2["cleanup_verified"] is True
assert apply3["artifact_retention"] == "preserved_failed_run_evidence"
@@ -647,6 +645,54 @@ def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> No
)
assert post["cleanup_verified"] is True
assert post["secret_values_collected"] is False
assert apply5["run_id"] == (
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
)
assert apply5["check_run_id"] == (
"P0-OBS-002-native-canary-default-20260715T015131Z-check5"
)
assert apply5["wrapper_terminal"] == "partial_degraded"
assert apply5["terminal"] == "partial_degraded"
assert apply5["runtime_operation"]["terminal"] == "pass"
assert apply5["nested_machine_receipt_verifier"] == {
"stage": "native_restore_receipt_verifier",
"terminal": "pass",
}
assert apply5["backup"]["terminal"] == "BACKUP_CREATED"
assert apply5["backup"]["file_count"] == 4885
assert apply5["backup"]["total_size_bytes"] == 108006575
assert apply5["backup"]["compressed_size_bytes"] == 99461897
assert apply5["backup"]["artifact_sha256"] == (
"08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
)
current_restore = apply5["isolated_restore"]
assert (
current_restore["database_count"]
== current_restore["restored_database_count"]
== 6
)
assert (
current_restore["table_count"] == current_restore["restored_table_count"] == 100
)
assert (
current_restore["check_table_count"]
== current_restore["check_table_pass_count"]
== 44
)
assert current_restore["critical_nonzero_count"] == 4
assert apply5["restic"]["snapshot_id"] == "135485ab"
current_post = apply5["independent_post_verifier"]
assert current_post["independent_monitor"] == "pass"
assert current_post["deployed_source_hash_drift_count"] == 0
assert current_post["runtime_drift_count"] == 0
assert (
current_post[
"exact_ephemeral_container_volume_network_process_tmp_residue_count"
]
== 0
)
assert current_post["cleanup_verified"] is True
assert canary["terminal"] == "production_closed_clickhouse_native_scope"
@@ -677,16 +723,38 @@ def test_current_toolchain_hashes_are_bound_without_rewriting_production_receipt
assert current["hashes"]["inventory_verifier_sha256"] == (
"c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863"
)
assert current["status"] == "pending_toolchain_deploy_and_runtime_canary"
assert current["deployment_status"] == "pending"
assert current["current_source_deployed"] is False
assert current["runtime_verifier_terminal"] == "pending"
assert current["deployed_source_hashes"] == current_hashes
assert current["status"] == "deployed_verified"
assert current["deployment_status"] == "deployed"
assert current["current_source_deployed"] is True
assert current["runtime_verifier_terminal"] == "pass"
assert current["historical_production_receipt_preserved"] is True
assert current["drift_from_historical_production_receipt"] == [
"backup_orchestrator_sha256",
"canary_wrapper_sha256",
"inventory_verifier_sha256",
]
assert current["toolchain_check_run_id"] == (
"P0-OBS-002-native-toolchain-20260715T015040Z-check6"
)
assert current["toolchain_apply_run_id"] == (
"P0-OBS-002-native-toolchain-20260715T015059Z-apply5"
)
assert current["toolchain_postcheck_run_id"] == (
"P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5"
)
assert current["toolchain_postcanary_run_id"] == (
"P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5"
)
assert current["default_canary_check_run_id"] == (
"P0-OBS-002-native-canary-default-20260715T015131Z-check5"
)
assert current["runtime_canary_run_id"] == (
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
)
assert current["wrapper_terminal"] == "partial_degraded"
assert current["runtime_operation_terminal"] == "pass"
assert current["nested_machine_receipt_verifier_terminal"] == "pass"
def test_preflight_fails_closed_when_current_toolchain_source_hash_is_stale(
@@ -707,10 +775,31 @@ def test_preflight_fails_closed_when_current_toolchain_source_hash_is_stale(
assert payload["terminal"] == "failed"
assert payload["checks"]["native_backup_current_source_hashes_bound"] is False
assert payload["checks"]["native_backup_historical_source_receipt_preserved"]
assert (
payload["backup_toolchain_current_source_status"]
== "pending_toolchain_deploy_and_runtime_canary"
)
assert payload["backup_toolchain_current_source_status"] == "deployed_verified"
def test_preflight_fails_closed_when_current_runtime_receipt_is_incomplete(
monkeypatch,
) -> None:
module = _load_preflight_module()
real_load_yaml = module._load_yaml
def load_without_operation_identity(path: Path):
payload = real_load_yaml(path)
if path.name == POST_CLOSURE_REGRESSION.name:
attempts = payload["controlled_applies"][
"clickhouse_native_backup_restore_canary"
]["attempts"]
attempts[-1]["backup"].pop("operation_identity_hash")
return payload
monkeypatch.setattr(module, "_load_yaml", load_without_operation_identity)
payload = module.evaluate(ROOT, CONTRACT)
assert payload["source_ready"] is False
assert payload["terminal"] == "failed"
assert payload["checks"]["native_backup_current_source_runtime_closed"] is False
assert payload["runtime_status"] == "partial_degraded"
def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> None:
@@ -759,14 +848,33 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() ->
]
assert resume["same_run_resume_safe"] is False
assert len(resume["required_preconditions"]) == 4
current_source = receipt["pending_verifiers"][
current_source = receipt["completed_verifiers"][
"clickhouse_native_current_source_toolchain"
]
assert current_source["status"] == "pending_toolchain_deploy_and_runtime_canary"
assert current_source["current_source_deployed"] is False
assert current_source["runtime_verifier_terminal"] == "pending"
assert current_source["status"] == "deployed_verified"
assert current_source["current_source_deployed"] is True
assert current_source["runtime_verifier_terminal"] == "pass"
assert current_source["historical_production_receipt_preserved"] is True
assert len(current_source["required_preconditions"]) == 3
assert current_source["toolchain_apply_run_id"] == (
"P0-OBS-002-native-toolchain-20260715T015059Z-apply5"
)
assert current_source["runtime_canary_run_id"] == (
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
)
assert current_source["wrapper_terminal"] == "partial_degraded"
assert current_source["runtime_operation_terminal"] == "pass"
assert current_source["nested_machine_receipt_verifier_terminal"] == "pass"
assert current_source["restic_snapshot_id"] == "135485ab"
assert current_source["artifact_sha256"] == (
"08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
)
assert current_source["independent_monitor"] == "pass"
assert current_source["runtime_drift_count"] == 0
assert current_source["residue_count"] == 0
assert current_source["exact_cleanup"] == "pass"
assert (
"clickhouse_native_current_source_toolchain" not in receipt["pending_verifiers"]
)
assets = receipt["asset_reconciliation"]
assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001"
assert assets["source_service_count"] == 27
@@ -797,7 +905,6 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() ->
"signoz_backup_offsite_dr_pending",
"clickhouse_native_failed_artifact_retention_pending",
"clickhouse_native_resume_submitted_inventory_pending",
"clickhouse_native_current_source_toolchain_pending",
"service_registry_runtime_mirror_reconciliation_pending",
"github_freeze_legacy_asset_retirement_pending",
"monitoring_generator_duplicate_awoooi_api_identity_pending",
@@ -818,10 +925,12 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N
assert post_closure["backup_collector_restore_guard_deployed"] is True
assert post_closure["backup_canary_scope"] == "clickhouse_native_only"
assert post_closure["backup_canary_verified"] is True
assert post_closure["backup_canary_last_terminal"] == "pass"
assert post_closure["backup_canary_last_terminal"] == "partial_degraded"
assert post_closure["backup_canary_last_run_id"] == (
"P0-OBS-002-native-canary-20260715T001329Z-apply4"
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
)
assert post_closure["backup_canary_runtime_operation_terminal"] == "pass"
assert post_closure["backup_canary_machine_receipt_verifier_terminal"] == "pass"
assert post_closure["backup_canary_cooldown_retry_status"] == (
"superseded_by_online_native_no_stop_contract"
)
@@ -838,12 +947,51 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N
assert post_closure["clickhouse_native_backup_toolchain_postcheck_run_id"] == (
"P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4"
)
assert post_closure["clickhouse_native_current_source_status"] == (
"pending_toolchain_deploy_and_runtime_canary"
assert (
post_closure["clickhouse_native_current_source_status"] == "deployed_verified"
)
assert post_closure["clickhouse_native_current_source_deployed"] is True
assert (
post_closure["clickhouse_native_current_source_runtime_verifier_terminal"]
== "pass"
)
assert post_closure["clickhouse_native_current_source_toolchain_check_run_id"] == (
"P0-OBS-002-native-toolchain-20260715T015040Z-check6"
)
assert post_closure["clickhouse_native_current_source_toolchain_apply_run_id"] == (
"P0-OBS-002-native-toolchain-20260715T015059Z-apply5"
)
assert (
post_closure["clickhouse_native_current_source_toolchain_postcheck_run_id"]
== "P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5"
)
assert (
post_closure["clickhouse_native_current_source_toolchain_postcanary_run_id"]
== "P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5"
)
assert post_closure["clickhouse_native_current_source_canary_check_run_id"] == (
"P0-OBS-002-native-canary-default-20260715T015131Z-check5"
)
assert post_closure["clickhouse_native_current_source_canary_apply_run_id"] == (
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
)
assert post_closure["clickhouse_native_current_source_wrapper_terminal"] == (
"partial_degraded"
)
assert post_closure[
"clickhouse_native_current_source_runtime_operation_terminal"
] == ("pass")
assert (
post_closure[
"clickhouse_native_current_source_machine_receipt_verifier_terminal"
]
== "pass"
)
assert post_closure["clickhouse_native_current_source_artifact_sha256"] == (
"08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
)
assert post_closure["clickhouse_native_current_source_deployed"] is False
assert post_closure["clickhouse_native_restore_terminal"] == "verified"
assert post_closure["clickhouse_native_restic_snapshot_id"] == "574e2a1a"
assert post_closure["clickhouse_native_restic_snapshot_id"] == "135485ab"
assert post_closure["clickhouse_native_offsite_status"] == "pending"
assert post_closure["clickhouse_native_failed_artifact_retention_status"] == (
"pending_bounded_retention_decision"