feat(observability): close native SigNoz backup restore canary

2026-07-15 08:55:08 +08:00
parent 6c1a12e388
commit 322e812d93
44 changed files with 9209 additions and 842 deletions
--- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
+++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
@@ -27,7 +27,9 @@ POST_CLOSURE_REGRESSION = ROOT / "ops/signoz/p0-obs-002-post-closure-regression.

 def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
    payload = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
-    assert payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1"
+    assert (
+        payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1"
+    )
    assert payload["work_item_id"] == "P0-OBS-002"
    assert payload["phase"] == "wave_a_dual_allow"
    assert payload["routes"]["active_route"] == "incumbent"
@@ -43,14 +45,33 @@ def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() ->
            )


-def test_contract_contains_no_credential_values_or_destructive_db_plan() -> None:
-    text = CONTRACT.read_text(encoding="utf-8").lower()
-    assert "signoz_user_root_password:" not in text
-    assert "signoz-api-key:" not in text
-    assert "authorization:" not in text
-    assert "raw_relational_database_update" in text
-    assert "stateful_volume_restore" in text
-    assert "apply_allowed_in_wave_a: false" in text
+def test_contract_and_receipt_contain_no_secret_values_or_github_dependency() -> None:
+    contract_text = CONTRACT.read_text(encoding="utf-8").lower()
+    regression_text = POST_CLOSURE_REGRESSION.read_text(encoding="utf-8").lower()
+    forbidden = (
+        "signoz_user_root_password:",
+        "signoz-api-key:",
+        "authorization:",
+        "password:",
+        "private_key:",
+        "cookie:",
+        "session:",
+        "github.com",
+        "api.github.com",
+        "raw.githubusercontent.com",
+        "codeload.github.com",
+        "ghcr.io",
+    )
+    for text in (contract_text, regression_text):
+        assert all(value not in text for value in forbidden)
+
+    assert "raw_relational_database_update" in contract_text
+    assert "stateful_volume_restore" in contract_text
+    assert "apply_allowed_in_wave_a: false" in contract_text
+
+    regression = yaml.safe_load(regression_text)
+    assert regression["scope_boundaries"]["secret_read"] is False
+    assert regression["scope_boundaries"]["github_used"] is False


 def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
@@ -69,9 +90,31 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
    assert payload["checks"] and all(payload["checks"].values())
    assert payload["checks"]["grpc_apply_waits_for_independent_post_verifier"] is True
    assert payload["checks"]["grpc_contract_runtime_state_consistent"] is True
+    assert payload["checks"]["post_closure_schema_v2"] is True
+    assert payload["checks"]["post_release_600s_verifier_consistent"] is True
+    assert (
+        payload["checks"]["native_backup_migration_closed_and_remaining_gaps_explicit"]
+        is True
+    )
+    assert payload["checks"]["service_registry_runtime_mirror_drift_explicit"]
+    assert payload["checks"]["github_freeze_legacy_asset_drift_explicit"]
+    assert payload["checks"]["monitoring_generator_duplicate_identity_drift_explicit"]
    assert "organization_not_initialized" in payload["blockers"]
-    assert "signoz_control_plane_health_route_not_deployed" in payload["blockers"]
    assert "wave_b_not_authorized" in payload["blockers"]
+    assert payload["blockers"] == [
+        "organization_not_initialized",
+        "direct_ingress_policy_not_closed",
+        "http_bridge_not_supervised",
+        "filelog_normalization_degraded",
+        "wave_b_not_authorized",
+    ]
+    for closed_blocker in (
+        "api_runtime_cooldown_pending",
+        "cd_probe_safety_fix_not_deployed",
+        "backup_collector_restore_guard_not_deployed",
+        "signoz_control_plane_health_route_not_deployed",
+    ):
+        assert closed_blocker not in payload["blockers"]


 def test_bridge_unit_is_user_scoped_restarting_and_hardened() -> None:
@@ -129,7 +172,10 @@ def test_three_signal_snapshot_keeps_synthetic_parity_separate_from_promotion()
    assert snapshot["trace_id"] == parity["trace_id"]
    assert snapshot["route_parity_terminal"] == "pass"
    assert snapshot["program_terminal"] == "partial_degraded"
-    assert snapshot["controlled_apply_contract"]["learning_writeback"]["status"] == "partial"
+    assert (
+        snapshot["controlled_apply_contract"]["learning_writeback"]["status"]
+        == "partial"
+    )
    assert (
        snapshot["controlled_apply_contract"]["learning_writeback"][
            "durable_gitea_feature_acknowledgement"
@@ -151,13 +197,13 @@ def test_filelog_fallback_preserves_body_and_exposes_normalization_coverage() ->
    text = OTEL_DAEMONSET.read_text(encoding="utf-8")
    assert "id: cri_parser" in text
    assert "on_error: send_quiet" not in text
-    assert 'if: \'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"\'' in text
+    assert "if: 'body matches \"^[^ ]+ (stdout|stderr) [^ ]* .*$\"'" in text
    assert "id: mark_cri_parsed" in text
-    assert 'value: parsed_cri' in text
+    assert "value: parsed_cri" in text
    assert "id: move_cri_body" in text
    assert "if: attributes.log != nil" in text
    assert "id: mark_cri_fallback" in text
-    assert 'value: fallback_unparsed' in text
+    assert "value: fallback_unparsed" in text
    assert 'if: attributes["awoooi.log.parse_status"] == nil' in text

    contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
@@ -231,11 +277,10 @@ def test_grpc_apply_terminal_stays_partial_until_independent_runtime_verifier()
    apply_block = bridge.split("  --apply)", 1)[1].split("  --status)", 1)[0]

    learning = (
-        'receipt learning partial '
-        '"application_trace_metric_10m_post_verifier_pending"'
+        'receipt learning partial "application_trace_metric_10m_post_verifier_pending"'
    )
    terminal = (
-        'receipt terminal partial '
+        "receipt terminal partial "
        '"grpc_transport_restored_independent_600s_post_verifier_pending"'
    )
    assert learning in apply_block
@@ -243,9 +288,8 @@ def test_grpc_apply_terminal_stays_partial_until_independent_runtime_verifier()
    assert apply_block.index(learning) < apply_block.index(terminal)
    assert "receipt terminal pass" not in apply_block
    assert (
-        'receipt terminal pass '
-        '"real_application_otlp_grpc_trace_metric_freshness_closed"'
-        in verifier
+        "receipt terminal pass "
+        '"real_application_otlp_grpc_trace_metric_freshness_closed"' in verifier
    )


@@ -266,7 +310,7 @@ def test_contract_grpc_route_state_matches_verified_runtime_evidence() -> None:
    assert grpc["source_fix"]["runtime_verifier_window_seconds"] >= 600


-def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate() -> None:
+def test_grpc_runtime_verifier_uses_real_aggregates_and_ten_minute_gate() -> None:
    text = GRPC_VERIFIER.read_text(encoding="utf-8")
    for expected in (
        'WINDOW_SECONDS="${WINDOW_SECONDS:-600}"',
@@ -293,10 +337,7 @@ def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate(
    assert grpc["recent_awoooi_api_metric_sample_count"] == 60180
    assert grpc["http_only_verifier_blind_spot"] is True
    assert grpc["source_fix"]["check_terminal"] == "pass"
-    assert (
-        grpc["source_fix"]["runtime_apply_status"]
-        == "applied_post_verifier_pass"
-    )
+    assert grpc["source_fix"]["runtime_apply_status"] == "applied_post_verifier_pass"
    assert grpc["source_fix"]["runtime_verifier_terminal"] == "pass"

    receipt = yaml.safe_load(GRPC_RUNTIME_RECEIPT.read_text(encoding="utf-8"))
@@ -312,13 +353,431 @@ def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate(

    regression = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
    assert regression["baseline"]["verifier_terminal"] == "pass"
-    assert regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is False
+    assert (
+        regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"]
+        is True
+    )
    assert regression["controlled_recovery"]["stateful_services_touched"] == 0
    assert regression["controlled_recovery"]["grpc_4317_listening"] is True
    assert regression["scope_boundaries"]["grpc_bridge_rollback_indicated"] is False
    assert regression["terminal"]["status"] == "partial_degraded"


+def test_post_closure_v2_records_cd_5140_and_config_route_patch() -> None:
+    receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+    assert receipt["schema"] == "awoooi_signoz_post_closure_regression_v2"
+
+    cd = receipt["release_receipts"]["gitea_cd_5140"]
+    assert cd["gitea_run_id"] == 5140
+    assert cd["source_sha"] == "e4da6570a645e504cc93647d396f45f167b3172a"
+    assert cd["workflow_terminal"] == "success"
+    assert cd["focused_tests_passed"] == 6
+    assert cd["unit_tests"] == {"passed": 4817, "skipped": 23, "warnings": 69}
+    assert cd["integration_tests_passed"] == 5
+    assert cd["production_readback"]["build_tag_matched"] is True
+    assert cd["production_readback"]["desired_tag_matched"] is True
+    assert cd["production_readback"]["rollout_terminal"] == "success"
+    assert cd["postdeploy"]["alert_chain_terminal"] == "pass"
+    assert cd["postdeploy"]["playwright_passed"] == 5
+    assert cd["cd_probe"]["receipt_provenance"] == "gitea_run_5140_job_log"
+    assert cd["cd_probe"]["restart_free"] is True
+    assert cd["cd_probe"]["workers"] == 4
+    assert cd["cd_probe"]["safety_reserve_connections"] == 4
+
+    route = receipt["controlled_applies"]["config_route_patch"]
+    assert route["run_id"] == ("P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1")
+    assert route["source_commit"] == "9a9d1464a586ba172d1613db5bc285def43eb3e8"
+    assert route["included_in_release_sha"] == cd["source_sha"]
+    assert route["normalized_asset"]["internal_url"] == "http://192.168.0.110:8080"
+    assert route["normalized_asset"]["public_url"] == "https://signoz.wooo.work"
+    assert route["source_truth_diff"]["otlp_producer_route_changed"] is False
+    assert route["execution_receipt"]["deployment_status"] == "deployed_verified"
+    assert route["terminal"] == "applied_post_verifier_pass"
+
+
+def test_prometheus_target_first_receipt_has_independent_verifier() -> None:
+    receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+    route = receipt["controlled_applies"]["prometheus_target_route"]
+    check = route["check_receipt"]
+    execution = route["execution_receipt"]
+    post = route["post_verifier"]
+    independent = route["independent_verifier"]
+
+    assert route["run_id"] == ("P0-OBS-002-prometheus-route-20260714T215106Z-apply1")
+    assert check["promtool"] == "pass"
+    assert check["candidate_sha"] == execution["active_sha"] == execution["runtime_sha"]
+    assert execution["action"] == "target_first_config_apply"
+    assert execution["backup_sha"] == check["source_sha"]
+    assert post["terminal"] == "target_up_two_scrapes"
+    assert post["first_scrape_at"] == "2026-07-15T05:52:38.146935853+08:00"
+    assert post["second_scrape_at"] == "2026-07-15T05:52:53.146935853+08:00"
+    assert post["first_scrape_at"] < post["second_scrape_at"]
+    assert post["target_count"] == 1
+    assert post["target_health"] == "up"
+    assert post["probe_success"] == 1
+    assert post["legacy_target_count"] == 0
+    assert independent["mode"] == "independent_readback"
+    assert independent["active_sha_matches_runtime"] is True
+    assert independent["target_count"] == 1
+    assert independent["target_health"] == "up"
+    assert independent["probe_success"] == 1
+    assert independent["legacy_target_count"] == 0
+    assert independent["terminal"] == "pass"
+    assert route["rollback"] == {"required": False, "available": True}
+    assert route["terminal"] == "completed_target_first"
+
+
+def test_alert_rules_receipt_hashes_and_runtime_rule_match() -> None:
+    receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+    alert = receipt["controlled_applies"]["alert_rules"]
+    source_sha = alert["check_receipt"]["source_sha"]
+    execution = alert["execution_receipt"]
+    verifier = alert["post_verifier"]
+
+    assert alert["run_id"] == "P0-OBS-002-alert-rules-20260714T215328Z-apply1"
+    assert alert["check_receipt"]["promtool"] == "pass"
+    assert source_sha == execution["remote_sha"]
+    assert source_sha == execution["canonical_sha"]
+    assert source_sha == execution["runtime_sha"]
+    assert execution["hashes_match"] is True
+    assert execution["prometheus_ready"] is True
+    assert verifier["window_seconds"] == 130
+    assert verifier["terminal"] == "pass"
+    assert verifier["signoz_rule_count"] == 1
+    assert verifier["signoz_rule_health"] == "ok"
+    assert verifier["signoz_rule_state"] == "inactive"
+    assert verifier["exact_query"] == (
+        '(probe_success{instance="http://192.168.0.110:8080/api/v1/health",'
+        'job="blackbox-http"} == 0) or absent(probe_success{instance="http://192.168.0.110:8080/api/v1/health",'
+        'job="blackbox-http"})'
+    )
+    assert alert["terminal"] == "completed_verified"
+
+
+def test_failed_backup_canary_preserves_failure_and_verified_rollback() -> None:
+    receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+    backup = receipt["controlled_applies"]["backup_canary"]
+    assert backup["guard_source"]["deployment_status"] == "deployed_verified"
+    assert len(backup["attempts"]) == 2
+
+    attempt = backup["attempts"][0]
+    assert attempt["run_id"] == ("P0-OBS-002-backup-canary-20260714T213210Z-retry1")
+    assert attempt["terminal"] == "failed_rolled_back"
+    assert attempt["classification"] == (
+        "failed_before_clickhouse_archive_commit_rollback_verified_no_restic_no_retention"
+    )
+    assert attempt["failed_stage"] == "clickhouse_archive_create"
+    assert attempt["archive_container_exit_code"] == 137
+    assert attempt["root_cause"] == (
+        "docker_health_monitor_restarted_collector_during_intentional_backup_stop"
+    )
+    for false_field in (
+        "consistent_archive_valid",
+        "clickhouse_archive_committed",
+        "sqlite_stage_reached",
+        "restic_stage_reached",
+        "retention_stage_reached",
+        "success_stage_reached",
+    ):
+        assert attempt[false_field] is False
+
+    rollback = attempt["rollback"]
+    assert rollback["collector_running"] is True
+    assert rollback["collector_restart_count"] == 0
+    assert rollback["grpc_4317_listening"] is True
+    assert rollback["http_4318_listening"] is True
+    assert rollback["canary_processes_absent"] is True
+    assert rollback["canary_container_absent"] is True
+    assert rollback["current_temp_dir_absent"] is True
+    assert rollback["partial_artifacts_absent"] is True
+    assert not attempt["terminal"].startswith("completed")
+    assert "learning_recorded" not in attempt
+
+    retry2 = backup["attempts"][1]
+    assert retry2["run_id"] == ("P0-OBS-002-backup-canary-20260714T221230Z-retry2")
+    assert retry2["terminal"] == "failed_rolled_back"
+    assert retry2["backup_script_exit_code"] == 1
+    assert retry2["archive_container_exit_code"] == "unknown_not_durably_captured"
+    assert retry2["archive_stderr_receipt"] == "suppressed_by_legacy_pipeline"
+    assert retry2["root_cause"] == (
+        "raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract"
+    )
+    assert retry2["exact_tar_error_proven"] is False
+    assert retry2["monitor_window"]["cron_intervals_crossed"] == 2
+    assert retry2["monitor_window"]["auto_repair_count"] == 0
+    for false_field in (
+        "consistent_archive_valid",
+        "clickhouse_archive_committed",
+        "sqlite_stage_reached",
+        "restic_stage_reached",
+        "retention_stage_reached",
+        "success_stage_reached",
+    ):
+        assert retry2[false_field] is False
+    retry2_rollback = retry2["rollback"]
+    assert retry2_rollback["cooldown_lease_restored"] is True
+    assert retry2_rollback["collector_running"] is True
+    assert retry2_rollback["collector_restart_count"] == 0
+    assert retry2_rollback["grpc_4317_listening"] is True
+    assert retry2_rollback["http_4318_listening"] is True
+    assert retry2_rollback["current_temp_dir_absent"] is True
+    assert retry2_rollback["partial_artifacts_absent"] is True
+    assert retry2["residual_cleanup_debt"] == []
+
+
+def test_clickhouse_native_backup_disk_preserves_failed_attempt_and_closes_retry() -> (
+    None
+):
+    receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+    native_disk = receipt["controlled_applies"]["clickhouse_native_backup_disk"]
+
+    assert native_disk["source"]["override_sha256"] == (
+        "cba7f4428ebf54f2890f3836d883d611d964b226210f228279206c84596a6488"
+    )
+    assert native_disk["source"]["config_sha256"] == (
+        "c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9"
+    )
+    assert native_disk["check_receipt"]["terminal"] == "check_pass_no_write"
+    assert len(native_disk["attempts"]) == 2
+
+    apply1, retry2 = native_disk["attempts"]
+    assert apply1["terminal"] == "failed_rolled_back"
+    assert apply1["exit_code"] == 2
+    assert apply1["rollback"]["terminal"] == "pass"
+    assert apply1["rollback"]["original_data_volume_preserved"] is True
+    assert apply1["rollback"]["residue_count"] == 0
+
+    assert retry2["run_id"] == (
+        "P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2"
+    )
+    assert retry2["terminal"] == "pass"
+    assert retry2["execution"]["image_digest_unchanged"] is True
+    assert retry2["execution"]["data_volume_unchanged"] is True
+    assert retry2["post_verifier"]["independent"] is True
+    assert retry2["post_verifier"]["server_write_access"] is True
+    assert retry2["post_verifier"]["dependent_restart_count_delta"] == 0
+    assert retry2["post_verifier"]["api_http_status"] == 200
+    assert retry2["post_verifier"]["active_backup_or_restore_count"] == 0
+    assert retry2["post_verifier"]["stage_candidate_process_residue_count"] == 0
+    assert native_disk["terminal"] == "deployed_verified"
+
+
+def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> None:
+    receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+    canary = receipt["controlled_applies"]["clickhouse_native_backup_restore_canary"]
+    assert canary["completion_scope"] == "clickhouse_native_only"
+    assert canary["source"]["post_canary_focused_tests_passed"] == 94
+    assert canary["source"]["post_format_focused_tests_passed"] == 94
+    assert canary["source"]["ruff_format"] == "pass"
+    assert canary["toolchain_controlled_apply"]["apply_terminal"] == "pass"
+    assert canary["toolchain_controlled_apply"]["postcheck_diff"] == (
+        "matching_7_drift_0_absent_0_active_operations_0"
+    )
+    assert len(canary["attempts"]) == 4
+    _, apply2, apply3, apply4 = canary["attempts"]
+    assert apply2["artifact_retention"] == "preserved_failed_run_evidence"
+    assert apply2["cleanup_verified"] is True
+    assert apply3["artifact_retention"] == "preserved_failed_run_evidence"
+    assert apply3["cleanup_verified"] is True
+
+    assert apply4["run_id"] == ("P0-OBS-002-native-canary-20260715T001329Z-apply4")
+    assert apply4["check_terminal"] == "check_pass_no_write"
+    assert apply4["terminal"] == "pass"
+    assert apply4["backup"]["terminal"] == "BACKUP_CREATED"
+    assert apply4["backup"]["error"] == ""
+    assert apply4["backup"]["artifact_sha256"] == (
+        "4fecd3ed99bfdc219b909cc028f3c1f31b8fdd15c1fdccdef18d710365113bdc"
+    )
+
+    restore = apply4["isolated_restore"]
+    assert restore["terminal"] == "verified"
+    assert restore["clickhouse_restore_terminal"] == "RESTORED"
+    assert restore["production_network_attached"] is False
+    assert restore["production_restore_performed"] is False
+    assert restore["source_staged_clickhouse_artifact_sha_match"] is True
+    assert restore["database_count"] == restore["restored_database_count"] == 6
+    assert restore["table_count"] == restore["restored_table_count"] == 100
+    assert restore["table_engine_schema_manifest_parity"] is True
+    assert restore["check_table_count"] == restore["check_table_pass_count"] == 44
+    assert restore["critical_nonzero_count"] == 4
+    assert restore["aggregate_counter_exact_parity_gate"] is False
+
+    restic = apply4["restic"]
+    assert restic["snapshot_id"] == "574e2a1a"
+    assert restic["snapshot_readback"] == "pass"
+    assert restic["repository_scope"] == "local_same_failure_domain"
+    assert restic["offsite_verified"] is False
+
+    post = apply4["independent_post_verifier"]
+    assert post["production_restart_count_delta"] == 0
+    assert post["api_http_status"] == 200
+    assert post["new_server_artifact_removed"] is True
+    assert post["apply2_apply3_preserved_artifact_hashes_unchanged"] is True
+    assert (
+        post["exact_ephemeral_container_volume_network_process_tmp_residue_count"] == 0
+    )
+    assert post["cleanup_verified"] is True
+    assert post["secret_values_collected"] is False
+    assert canary["terminal"] == "production_closed_clickhouse_native_scope"
+
+
+def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> None:
+    receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+    verifier = receipt["post_release_runtime_verifier"]
+    assert verifier["run_id"] == "P0-OBS-002-post-release-20260714T215500Z"
+    assert verifier["window_seconds"] == 600
+    assert verifier["terminal"] == "pass"
+    assert verifier["restart_delta"] == 0
+    assert verifier["exporter_trace_errors"] == 0
+    assert verifier["exporter_metric_errors"] == 0
+    assert verifier["trace_count"] == 4026
+    assert verifier["metric_sample_count"] == 76369
+    assert verifier["api_runtime_cooldown_closed"] is True
+    assert (
+        receipt["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is True
+    )
+
+    closed = receipt["completed_verifiers"]["clickhouse_native_backup_migration"]
+    assert closed["prior_run_id"] == (
+        "P0-OBS-002-backup-canary-20260714T221230Z-retry2"
+    )
+    assert closed["run_id"] == ("P0-OBS-002-native-canary-20260715T001329Z-apply4")
+    assert closed["status"] == "production_closed_clickhouse_native_scope"
+    assert closed["native_backup_terminal"] == "BACKUP_CREATED"
+    assert closed["isolated_restore_terminal"] == "verified"
+    assert closed["restic_snapshot_id"] == "574e2a1a"
+    assert closed["independent_verifier"] == "pass"
+    assert closed["exact_cleanup"] == "pass"
+    sqlite_pending = receipt["pending_verifiers"][
+        "signoz_sqlite_application_consistent_backup"
+    ]
+    assert sqlite_pending["raw_sqlite_read_or_sync_allowed"] is False
+    assert sqlite_pending["sqlite_cli_present_in_runtime"] is False
+    offsite = receipt["pending_verifiers"]["signoz_backup_offsite_dr"]
+    assert offsite["current_repository_scope"] == "local_same_failure_domain"
+    assert offsite["offsite_verified"] is False
+    artifacts = receipt["pending_verifiers"][
+        "clickhouse_native_failed_artifact_retention"
+    ]
+    assert artifacts["destructive_cleanup_authorized"] is False
+    assert artifacts["active_backup_or_restore_count"] == 0
+    assert len(artifacts["preserved_artifacts"]) == 2
+    resume = receipt["pending_verifiers"][
+        "clickhouse_native_resume_submitted_inventory"
+    ]
+    assert resume["same_run_resume_safe"] is False
+    assert len(resume["required_preconditions"]) == 4
+    assets = receipt["asset_reconciliation"]
+    assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001"
+    assert assets["source_service_count"] == 27
+    assert assets["runtime_mirror_service_count"] == 24
+    assert assets["exact_shared_service_count"] == 24
+    assert assets["shared_service_drift_count"] == 0
+    assert assets["source_only_services"] == ["bitan-app", "sentry", "signoz"]
+    assert assets["signoz_clickhouse_exact_match"] is True
+    assert assets["live_signoz_query_host"] == "192.168.0.110"
+    assert assets["source_signoz_host"] == "192.168.0.188"
+    github_drift = assets["github_freeze_legacy_asset_drift"]
+    assert github_drift["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-002"
+    assert github_drift["superseded_by"] == "global_product_governance_v2"
+    assert github_drift["active_github_use_in_this_run"] is False
+    assert len(github_drift["source_references"]) == 3
+    assert github_drift["p0_p1_github_recovery_scheduled"] is False
+    duplicate_identity = assets["monitoring_generator_duplicate_identity_drift"]
+    assert duplicate_identity["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-003"
+    assert duplicate_identity["status"] == "pending_source_deduplication"
+    assert duplicate_identity["awoooi_api_registry_entry_count"] == 2
+    assert duplicate_identity["generated_scrape_config_count"] == 24
+    assert duplicate_identity["generated_awoooi_api_job_count"] == 2
+    assert duplicate_identity["generated_blackbox_target_count"] == 20
+    assert duplicate_identity["runtime_apply_performed"] is False
+    assert receipt["terminal"]["status"] == "partial_degraded"
+    assert receipt["terminal"]["blockers"] == [
+        "signoz_sqlite_application_consistent_backup_pending",
+        "signoz_backup_offsite_dr_pending",
+        "clickhouse_native_failed_artifact_retention_pending",
+        "clickhouse_native_resume_submitted_inventory_pending",
+        "service_registry_runtime_mirror_reconciliation_pending",
+        "github_freeze_legacy_asset_retirement_pending",
+        "monitoring_generator_duplicate_awoooi_api_identity_pending",
+    ]
+
+
+def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> None:
+    contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
+    post_closure = contract["runtime_observations"]["post_closure_regression"]
+    assert post_closure["snapshot_schema"] == (
+        "awoooi_signoz_post_closure_regression_v2"
+    )
+    assert post_closure["gitea_cd_run_id"] == 5140
+    assert post_closure["api_runtime_cooldown_closed"] is True
+    assert post_closure["cd_probe_safety_fix_deployed"] is True
+    assert post_closure["config_route_patch_deployed"] is True
+    assert post_closure["signoz_control_plane_health_route_deployed"] is True
+    assert post_closure["backup_collector_restore_guard_deployed"] is True
+    assert post_closure["backup_canary_scope"] == "clickhouse_native_only"
+    assert post_closure["backup_canary_verified"] is True
+    assert post_closure["backup_canary_last_terminal"] == "pass"
+    assert post_closure["backup_canary_last_run_id"] == (
+        "P0-OBS-002-native-canary-20260715T001329Z-apply4"
+    )
+    assert post_closure["backup_canary_cooldown_retry_status"] == (
+        "superseded_by_online_native_no_stop_contract"
+    )
+    assert post_closure["clickhouse_native_backup_migration_status"] == (
+        "production_closed"
+    )
+    assert post_closure["clickhouse_native_backup_disk_run_id"] == (
+        "P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2"
+    )
+    assert post_closure["clickhouse_native_backup_disk_status"] == "deployed_verified"
+    assert post_closure["clickhouse_native_backup_toolchain_run_id"] == (
+        "P0-OBS-002-native-toolchain-20260715T001252Z-apply4"
+    )
+    assert post_closure["clickhouse_native_backup_toolchain_postcheck_run_id"] == (
+        "P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4"
+    )
+    assert post_closure["clickhouse_native_restore_terminal"] == "verified"
+    assert post_closure["clickhouse_native_restic_snapshot_id"] == "574e2a1a"
+    assert post_closure["clickhouse_native_offsite_status"] == "pending"
+    assert post_closure["clickhouse_native_failed_artifact_retention_status"] == (
+        "pending_bounded_retention_decision"
+    )
+    assert post_closure["clickhouse_native_resume_inventory_status"] == (
+        "pending_durable_submitted_inventory_contract"
+    )
+    assert (
+        post_closure["signoz_sqlite_application_consistent_backup_status"] == "pending"
+    )
+    assert post_closure["service_registry_runtime_mirror_status"] == (
+        "partial_degraded"
+    )
+    assert post_closure["service_registry_runtime_mirror_work_item_id"] == (
+        "P0-OBS-002-ASSET-DRIFT-001"
+    )
+    assert post_closure["github_freeze_legacy_asset_status"] == (
+        "pending_controlled_retirement"
+    )
+    assert post_closure["github_freeze_legacy_asset_work_item_id"] == (
+        "P0-OBS-002-ASSET-DRIFT-002"
+    )
+    assert post_closure["monitoring_generator_duplicate_identity_status"] == (
+        "pending_source_deduplication"
+    )
+    assert post_closure["monitoring_generator_duplicate_identity_work_item_id"] == (
+        "P0-OBS-002-ASSET-DRIFT-003"
+    )
+    assert post_closure["post_release_otlp_grpc_600s_verifier_status"] == "pass"
+    assert contract["terminal"]["status"] == "partial_degraded"
+    assert contract["terminal"]["blockers"] == [
+        "organization_not_initialized",
+        "direct_ingress_policy_not_closed",
+        "http_bridge_not_supervised",
+        "filelog_normalization_degraded",
+        "wave_b_not_authorized",
+    ]
+
+
 def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None:
    env = os.environ.copy()
    env.pop("MANAGED_PROMOTION_RECEIPT", None)