From 3a2829daf34e7354f9539929c0ccffbc86c0b214 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 17:03:12 +0800 Subject: [PATCH] chore(observability): record inactive metadata toolchain --- ...rganization-canonical-route-migration.yaml | 2 +- .../p0-obs-002-post-closure-regression.yaml | 45 ++++++++++++++++++- .../signoz-canonical-route-preflight.py | 42 +++++++++++++++-- .../test_signoz_canonical_route_migration.py | 41 ++++++++++++++--- 4 files changed, 119 insertions(+), 11 deletions(-) diff --git a/ops/signoz/organization-canonical-route-migration.yaml b/ops/signoz/organization-canonical-route-migration.yaml index d0b2a6426..d0c37435c 100644 --- a/ops/signoz/organization-canonical-route-migration.yaml +++ b/ops/signoz/organization-canonical-route-migration.yaml @@ -202,7 +202,7 @@ runtime_observations: clickhouse_native_failed_artifact_retention_status: pending_bounded_retention_decision clickhouse_native_resume_inventory_status: pending_durable_submitted_inventory_contract signoz_sqlite_application_consistent_backup_status: pending - signoz_metadata_export_source_contract_status: implemented_tested_not_deployed + signoz_metadata_export_source_contract_status: deployed_inactive_verified signoz_metadata_export_policy: config/signoz/metadata-export-policy.json signoz_metadata_exporter: scripts/backup/signoz-metadata-export.py signoz_metadata_export_verifier: scripts/backup/verify-signoz-metadata-export.py diff --git a/ops/signoz/p0-obs-002-post-closure-regression.yaml b/ops/signoz/p0-obs-002-post-closure-regression.yaml index 5eff85f3a..0fed82ef1 100644 --- a/ops/signoz/p0-obs-002-post-closure-regression.yaml +++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml @@ -760,7 +760,7 @@ pending_verifiers: status: pending_supported_non_raw_export_or_coordinated_snapshot raw_sqlite_read_or_sync_allowed: false sqlite_cli_present_in_runtime: false - source_contract_status: implemented_tested_not_deployed + source_contract_status: deployed_inactive_verified source_contract: policy: config/signoz/metadata-export-policy.json exporter: scripts/backup/signoz-metadata-export.py @@ -781,6 +781,48 @@ pending_verifiers: controlled_deployer_sha256: 2d0f7ae20ff8919dfc0f6caa1c3e346c0d8127d997ffa46f86a6e92df212e6f4 deployer_tests_sha256: eb6acd4da593f75dbd26a0abfc6674a60221ef7eac91e2f679f8ac75e9eda43e test_terminal: 21_passed + deployment: + status: deployed_inactive_verified + asset_id: host110-signoz-metadata-toolchain + target_host: 192.168.0.110 + target_root: /backup/toolchains/signoz-metadata + bundle_id: bb6d78b67f506072b2e45e92bc1d415364e25852282a9b068900665c578df011 + target_path: /backup/toolchains/signoz-metadata/bb6d78b67f506072b2e45e92bc1d415364e25852282a9b068900665c578df011 + directory_owner: root:root + directory_mode: "0750" + active_pointer_present: false + runtime_export_executed: false + raw_sqlite_read: false + raw_volume_read: false + initial_failed_attempt: + run_id: P0-OBS-002-metadata-toolchain-20260715T161200P0800 + terminal: failed_no_active_pointer_write + failure_reason: protected_regular_root_append_denied + active_pointer_present: false + runtime_export_executed: false + bounded_execution: + run_id: P0-OBS-002-metadata-toolchain-20260715T165600P0800-retry2 + durable_inner_terminal: pass_source_deployed_inactive + outer_terminal: failed_source_deploy_unverified + outer_false_negative_reason: unprivileged_verifier_could_not_traverse_root_owned_parent + active_pointer_present: false + runtime_export_executed: false + independent_root_postcheck: + run_id: P0-OBS-002-metadata-toolchain-rootcheck-20260715T170000P0800 + terminal: check_pass_no_write + remote_state: exact + drift_count: 0 + api_http_status: 200 + active_backup_operation_count: 0 + idempotent_closure: + run_id: P0-OBS-002-metadata-toolchain-idempotent-20260715T170000P0800-retry3 + execution_terminal: idempotent_no_write + independent_post_verifier: pass + terminal: pass_source_deployed_inactive + active_pointer_present: false + runtime_export_executed: false + learning_writeback: partial_degraded + remaining_blocker: authenticated_export_restore_zero_residue_pending portable_assets: - dashboards - alert_rules @@ -805,7 +847,6 @@ pending_verifiers: runtime_restore_terminal: pending_isolated_same_version_target runtime_zero_residue_terminal: pending required_preconditions: - - deploy_exact_source_contract_with_hash_and_mode_readback - trusted_service_account_secret_reference_without_value_readback - authenticated_two_read_stable_production_export - isolated_same_version_restore_semantic_readback diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py index 223d5a303..ca5308d3e 100644 --- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -858,6 +858,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: "signoz_sqlite_application_consistent_backup", {} ) metadata_export_source_contract = sqlite_backup.get("source_contract", {}) + metadata_deployment = metadata_export_source_contract.get("deployment", {}) declared_metadata_export_hashes = metadata_export_source_contract.get("hashes", {}) actual_metadata_export_hashes = _current_metadata_export_source_hashes(root) checks["signoz_metadata_export_source_hashes_bound"] = ( @@ -888,8 +889,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: == "pending_supported_non_raw_export_or_coordinated_snapshot" and sqlite_backup.get("raw_sqlite_read_or_sync_allowed") is False and sqlite_backup.get("sqlite_cli_present_in_runtime") is False - and sqlite_backup.get("source_contract_status") - == "implemented_tested_not_deployed" + and sqlite_backup.get("source_contract_status") == "deployed_inactive_verified" and metadata_export_source_contract.get("policy") == "config/signoz/metadata-export-policy.json" and metadata_export_source_contract.get("exporter") @@ -902,12 +902,48 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: and metadata_export_source_contract.get("raw_volume_read") is False and metadata_export_source_contract.get("secret_value_persistence") is False and metadata_export_source_contract.get("full_sqlite_completion_claim") is False + and metadata_deployment.get("status") == "deployed_inactive_verified" + and metadata_deployment.get("asset_id") == "host110-signoz-metadata-toolchain" + and metadata_deployment.get("target_host") == "192.168.0.110" + and metadata_deployment.get("bundle_id") + == "bb6d78b67f506072b2e45e92bc1d415364e25852282a9b068900665c578df011" + and metadata_deployment.get("directory_owner") == "root:root" + and metadata_deployment.get("directory_mode") == "0750" + and metadata_deployment.get("active_pointer_present") is False + and metadata_deployment.get("runtime_export_executed") is False + and metadata_deployment.get("raw_sqlite_read") is False + and metadata_deployment.get("raw_volume_read") is False + and metadata_deployment.get("initial_failed_attempt", {}).get("terminal") + == "failed_no_active_pointer_write" + and metadata_deployment.get("bounded_execution", {}).get( + "durable_inner_terminal" + ) + == "pass_source_deployed_inactive" + and metadata_deployment.get("bounded_execution", {}).get("outer_terminal") + == "failed_source_deploy_unverified" + and metadata_deployment.get("independent_root_postcheck", {}).get("terminal") + == "check_pass_no_write" + and metadata_deployment.get("independent_root_postcheck", {}).get( + "remote_state" + ) + == "exact" + and metadata_deployment.get("independent_root_postcheck", {}).get("drift_count") + == 0 + and metadata_deployment.get("idempotent_closure", {}).get("execution_terminal") + == "idempotent_no_write" + and metadata_deployment.get("idempotent_closure", {}).get("terminal") + == "pass_source_deployed_inactive" + and metadata_deployment.get("idempotent_closure", {}).get( + "independent_post_verifier" + ) + == "pass" + and metadata_deployment.get("learning_writeback") == "partial_degraded" and sqlite_backup.get("runtime_export_terminal") == "pending_authenticated_stable_export" and sqlite_backup.get("runtime_restore_terminal") == "pending_isolated_same_version_target" and sqlite_backup.get("runtime_zero_residue_terminal") == "pending" - and len(sqlite_backup.get("required_preconditions", [])) == 6 + and len(sqlite_backup.get("required_preconditions", [])) == 5 and offsite_backup.get("status") == "pending_offsite_snapshot_and_restore_verifier" and offsite_backup.get("current_repository_scope") diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py index 287316873..added60fe 100644 --- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -834,9 +834,7 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> ] assert sqlite_pending["raw_sqlite_read_or_sync_allowed"] is False assert sqlite_pending["sqlite_cli_present_in_runtime"] is False - assert sqlite_pending["source_contract_status"] == ( - "implemented_tested_not_deployed" - ) + assert sqlite_pending["source_contract_status"] == ("deployed_inactive_verified") source_contract = sqlite_pending["source_contract"] assert source_contract["policy"] == "config/signoz/metadata-export-policy.json" assert source_contract["exporter"] == ("scripts/backup/signoz-metadata-export.py") @@ -878,6 +876,39 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> for field, relative in metadata_paths.items() } assert source_contract["test_terminal"] == "21_passed" + deployment = source_contract["deployment"] + assert deployment["status"] == "deployed_inactive_verified" + assert deployment["asset_id"] == "host110-signoz-metadata-toolchain" + assert deployment["target_host"] == "192.168.0.110" + assert deployment["bundle_id"] == ( + "bb6d78b67f506072b2e45e92bc1d415364e25852282a9b068900665c578df011" + ) + assert deployment["directory_owner"] == "root:root" + assert deployment["directory_mode"] == "0750" + assert deployment["active_pointer_present"] is False + assert deployment["runtime_export_executed"] is False + assert deployment["initial_failed_attempt"]["terminal"] == ( + "failed_no_active_pointer_write" + ) + assert deployment["bounded_execution"]["durable_inner_terminal"] == ( + "pass_source_deployed_inactive" + ) + assert deployment["bounded_execution"]["outer_terminal"] == ( + "failed_source_deploy_unverified" + ) + assert deployment["independent_root_postcheck"]["terminal"] == ( + "check_pass_no_write" + ) + assert deployment["independent_root_postcheck"]["remote_state"] == "exact" + assert deployment["independent_root_postcheck"]["drift_count"] == 0 + assert deployment["idempotent_closure"]["execution_terminal"] == ( + "idempotent_no_write" + ) + assert deployment["idempotent_closure"]["terminal"] == ( + "pass_source_deployed_inactive" + ) + assert deployment["idempotent_closure"]["independent_post_verifier"] == "pass" + assert deployment["learning_writeback"] == "partial_degraded" assert source_contract["raw_sqlite_read"] is False assert source_contract["raw_volume_read"] is False assert source_contract["secret_value_persistence"] is False @@ -889,7 +920,7 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> "pending_isolated_same_version_target" ) assert sqlite_pending["runtime_zero_residue_terminal"] == "pending" - assert len(sqlite_pending["required_preconditions"]) == 6 + assert len(sqlite_pending["required_preconditions"]) == 5 offsite = receipt["pending_verifiers"]["signoz_backup_offsite_dr"] assert offsite["current_repository_scope"] == "local_same_failure_domain" assert offsite["offsite_verified"] is False @@ -1063,7 +1094,7 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N post_closure["signoz_sqlite_application_consistent_backup_status"] == "pending" ) assert post_closure["signoz_metadata_export_source_contract_status"] == ( - "implemented_tested_not_deployed" + "deployed_inactive_verified" ) assert post_closure["signoz_metadata_export_policy"] == ( "config/signoz/metadata-export-policy.json"