From 7d6db4f2b77ee6612a2d8815153564300ec5672f Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 16:58:03 +0800 Subject: [PATCH 1/2] fix(observability): verify root-owned SigNoz bundle --- ops/signoz/p0-obs-002-post-closure-regression.yaml | 6 +++--- scripts/ops/deploy-signoz-metadata-toolchain.sh | 6 +++++- scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py | 7 +++++++ .../reboot-recovery/signoz-canonical-route-preflight.py | 2 +- .../tests/test_signoz_canonical_route_migration.py | 2 +- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/ops/signoz/p0-obs-002-post-closure-regression.yaml b/ops/signoz/p0-obs-002-post-closure-regression.yaml index 3e1510057..5eff85f3a 100644 --- a/ops/signoz/p0-obs-002-post-closure-regression.yaml +++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml @@ -778,9 +778,9 @@ pending_verifiers: independent_export_verifier_sha256: cbab56dd3919866a96550c1dce46ec67783cd6f00473e4491861e17d1430048c isolated_restore_driver_sha256: 38ce70b3f891a3ce8edfaeb880ccf69d0bb6817c3f801a0aefa0f06341901997 contract_tests_sha256: 474d9dfcfe170c62425356ea9f06ed0dc1629b39215ee80f395f721c7d43a923 - controlled_deployer_sha256: ca64829e8cd850d83b016a6207bf463da586a68a4c78571e7a8734c93deb4766 - deployer_tests_sha256: 2092164f254b7c8b9064f2ffb1859e6dd4213b627992373719b3d9bc8a6ae200 - test_terminal: 20_passed + controlled_deployer_sha256: 2d0f7ae20ff8919dfc0f6caa1c3e346c0d8127d997ffa46f86a6e92df212e6f4 + deployer_tests_sha256: eb6acd4da593f75dbd26a0abfc6674a60221ef7eac91e2f679f8ac75e9eda43e + test_terminal: 21_passed portable_assets: - dashboards - alert_rules diff --git a/scripts/ops/deploy-signoz-metadata-toolchain.sh b/scripts/ops/deploy-signoz-metadata-toolchain.sh index ac447471e..b6519d42b 100755 --- a/scripts/ops/deploy-signoz-metadata-toolchain.sh +++ b/scripts/ops/deploy-signoz-metadata-toolchain.sh @@ -166,7 +166,11 @@ emit ai_decision pass "candidate_additive_immutable_inactive_source_deploy" emit risk_policy_decision pass "medium_no_active_pointer_no_secret_no_raw_sqlite" remote_check() { - bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" bash -s -- \ + # The immutable root is deliberately 0700/root-owned. Run this bounded + # verifier as root so it can distinguish exact from absent without relaxing + # directory permissions. REMOTE_CHECK contains only stat/hash/process/API + # reads and performs no writes. + bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" sudo -n bash -s -- \ "${REMOTE_ROOT}" "${BUNDLE_ID}" "${REMOTE_TIMEOUT_SECONDS}" \ "${KILL_AFTER_SECONDS}" "${SOURCE_HASHES[@]}" <<'REMOTE_CHECK' set -euo pipefail diff --git a/scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py b/scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py index 256f2feec..c2b2840eb 100644 --- a/scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py +++ b/scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py @@ -174,6 +174,13 @@ def test_shared_operation_lock_is_existing_read_only_and_fail_closed() -> None: assert "exec 8>>/tmp/awoooi-signoz-backup-operation.lock" not in source +def test_remote_check_uses_privileged_read_only_visibility() -> None: + source = DEPLOYER.read_text(encoding="utf-8") + remote_check = source.split("remote_check() {", 1)[1].split("\n}", 1)[0] + assert '"${TARGET_HOST}" sudo -n bash -s --' in remote_check + assert "REMOTE_CHECK contains only stat/hash/process/API" in remote_check + + def test_exact_five_file_supply_chain_and_modes_are_fixed() -> None: source = DEPLOYER.read_text(encoding="utf-8") for path in ( diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py index d22a1e528..223d5a303 100644 --- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -865,7 +865,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: == len(METADATA_EXPORT_SOURCE_PATHS) and set(declared_metadata_export_hashes) == set(METADATA_EXPORT_SOURCE_PATHS) and declared_metadata_export_hashes == actual_metadata_export_hashes - and metadata_export_source_contract.get("test_terminal") == "20_passed" + and metadata_export_source_contract.get("test_terminal") == "21_passed" ) offsite_backup = pending_verifiers.get("signoz_backup_offsite_dr", {}) failed_artifacts = pending_verifiers.get( diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py index 8015ff7d8..287316873 100644 --- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -877,7 +877,7 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> field: hashlib.sha256((ROOT / relative).read_bytes()).hexdigest() for field, relative in metadata_paths.items() } - assert source_contract["test_terminal"] == "20_passed" + assert source_contract["test_terminal"] == "21_passed" assert source_contract["raw_sqlite_read"] is False assert source_contract["raw_volume_read"] is False assert source_contract["secret_value_persistence"] is False From 3a2829daf34e7354f9539929c0ccffbc86c0b214 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 17:03:12 +0800 Subject: [PATCH 2/2] chore(observability): record inactive metadata toolchain --- ...rganization-canonical-route-migration.yaml | 2 +- .../p0-obs-002-post-closure-regression.yaml | 45 ++++++++++++++++++- .../signoz-canonical-route-preflight.py | 42 +++++++++++++++-- .../test_signoz_canonical_route_migration.py | 41 ++++++++++++++--- 4 files changed, 119 insertions(+), 11 deletions(-) diff --git a/ops/signoz/organization-canonical-route-migration.yaml b/ops/signoz/organization-canonical-route-migration.yaml index d0b2a6426..d0c37435c 100644 --- a/ops/signoz/organization-canonical-route-migration.yaml +++ b/ops/signoz/organization-canonical-route-migration.yaml @@ -202,7 +202,7 @@ runtime_observations: clickhouse_native_failed_artifact_retention_status: pending_bounded_retention_decision clickhouse_native_resume_inventory_status: pending_durable_submitted_inventory_contract signoz_sqlite_application_consistent_backup_status: pending - signoz_metadata_export_source_contract_status: implemented_tested_not_deployed + signoz_metadata_export_source_contract_status: deployed_inactive_verified signoz_metadata_export_policy: config/signoz/metadata-export-policy.json signoz_metadata_exporter: scripts/backup/signoz-metadata-export.py signoz_metadata_export_verifier: scripts/backup/verify-signoz-metadata-export.py diff --git a/ops/signoz/p0-obs-002-post-closure-regression.yaml b/ops/signoz/p0-obs-002-post-closure-regression.yaml index 5eff85f3a..0fed82ef1 100644 --- a/ops/signoz/p0-obs-002-post-closure-regression.yaml +++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml @@ -760,7 +760,7 @@ pending_verifiers: status: pending_supported_non_raw_export_or_coordinated_snapshot raw_sqlite_read_or_sync_allowed: false sqlite_cli_present_in_runtime: false - source_contract_status: implemented_tested_not_deployed + source_contract_status: deployed_inactive_verified source_contract: policy: config/signoz/metadata-export-policy.json exporter: scripts/backup/signoz-metadata-export.py @@ -781,6 +781,48 @@ pending_verifiers: controlled_deployer_sha256: 2d0f7ae20ff8919dfc0f6caa1c3e346c0d8127d997ffa46f86a6e92df212e6f4 deployer_tests_sha256: eb6acd4da593f75dbd26a0abfc6674a60221ef7eac91e2f679f8ac75e9eda43e test_terminal: 21_passed + deployment: + status: deployed_inactive_verified + asset_id: host110-signoz-metadata-toolchain + target_host: 192.168.0.110 + target_root: /backup/toolchains/signoz-metadata + bundle_id: bb6d78b67f506072b2e45e92bc1d415364e25852282a9b068900665c578df011 + target_path: /backup/toolchains/signoz-metadata/bb6d78b67f506072b2e45e92bc1d415364e25852282a9b068900665c578df011 + directory_owner: root:root + directory_mode: "0750" + active_pointer_present: false + runtime_export_executed: false + raw_sqlite_read: false + raw_volume_read: false + initial_failed_attempt: + run_id: P0-OBS-002-metadata-toolchain-20260715T161200P0800 + terminal: failed_no_active_pointer_write + failure_reason: protected_regular_root_append_denied + active_pointer_present: false + runtime_export_executed: false + bounded_execution: + run_id: P0-OBS-002-metadata-toolchain-20260715T165600P0800-retry2 + durable_inner_terminal: pass_source_deployed_inactive + outer_terminal: failed_source_deploy_unverified + outer_false_negative_reason: unprivileged_verifier_could_not_traverse_root_owned_parent + active_pointer_present: false + runtime_export_executed: false + independent_root_postcheck: + run_id: P0-OBS-002-metadata-toolchain-rootcheck-20260715T170000P0800 + terminal: check_pass_no_write + remote_state: exact + drift_count: 0 + api_http_status: 200 + active_backup_operation_count: 0 + idempotent_closure: + run_id: P0-OBS-002-metadata-toolchain-idempotent-20260715T170000P0800-retry3 + execution_terminal: idempotent_no_write + independent_post_verifier: pass + terminal: pass_source_deployed_inactive + active_pointer_present: false + runtime_export_executed: false + learning_writeback: partial_degraded + remaining_blocker: authenticated_export_restore_zero_residue_pending portable_assets: - dashboards - alert_rules @@ -805,7 +847,6 @@ pending_verifiers: runtime_restore_terminal: pending_isolated_same_version_target runtime_zero_residue_terminal: pending required_preconditions: - - deploy_exact_source_contract_with_hash_and_mode_readback - trusted_service_account_secret_reference_without_value_readback - authenticated_two_read_stable_production_export - isolated_same_version_restore_semantic_readback diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py index 223d5a303..ca5308d3e 100644 --- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -858,6 +858,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: "signoz_sqlite_application_consistent_backup", {} ) metadata_export_source_contract = sqlite_backup.get("source_contract", {}) + metadata_deployment = metadata_export_source_contract.get("deployment", {}) declared_metadata_export_hashes = metadata_export_source_contract.get("hashes", {}) actual_metadata_export_hashes = _current_metadata_export_source_hashes(root) checks["signoz_metadata_export_source_hashes_bound"] = ( @@ -888,8 +889,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: == "pending_supported_non_raw_export_or_coordinated_snapshot" and sqlite_backup.get("raw_sqlite_read_or_sync_allowed") is False and sqlite_backup.get("sqlite_cli_present_in_runtime") is False - and sqlite_backup.get("source_contract_status") - == "implemented_tested_not_deployed" + and sqlite_backup.get("source_contract_status") == "deployed_inactive_verified" and metadata_export_source_contract.get("policy") == "config/signoz/metadata-export-policy.json" and metadata_export_source_contract.get("exporter") @@ -902,12 +902,48 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: and metadata_export_source_contract.get("raw_volume_read") is False and metadata_export_source_contract.get("secret_value_persistence") is False and metadata_export_source_contract.get("full_sqlite_completion_claim") is False + and metadata_deployment.get("status") == "deployed_inactive_verified" + and metadata_deployment.get("asset_id") == "host110-signoz-metadata-toolchain" + and metadata_deployment.get("target_host") == "192.168.0.110" + and metadata_deployment.get("bundle_id") + == "bb6d78b67f506072b2e45e92bc1d415364e25852282a9b068900665c578df011" + and metadata_deployment.get("directory_owner") == "root:root" + and metadata_deployment.get("directory_mode") == "0750" + and metadata_deployment.get("active_pointer_present") is False + and metadata_deployment.get("runtime_export_executed") is False + and metadata_deployment.get("raw_sqlite_read") is False + and metadata_deployment.get("raw_volume_read") is False + and metadata_deployment.get("initial_failed_attempt", {}).get("terminal") + == "failed_no_active_pointer_write" + and metadata_deployment.get("bounded_execution", {}).get( + "durable_inner_terminal" + ) + == "pass_source_deployed_inactive" + and metadata_deployment.get("bounded_execution", {}).get("outer_terminal") + == "failed_source_deploy_unverified" + and metadata_deployment.get("independent_root_postcheck", {}).get("terminal") + == "check_pass_no_write" + and metadata_deployment.get("independent_root_postcheck", {}).get( + "remote_state" + ) + == "exact" + and metadata_deployment.get("independent_root_postcheck", {}).get("drift_count") + == 0 + and metadata_deployment.get("idempotent_closure", {}).get("execution_terminal") + == "idempotent_no_write" + and metadata_deployment.get("idempotent_closure", {}).get("terminal") + == "pass_source_deployed_inactive" + and metadata_deployment.get("idempotent_closure", {}).get( + "independent_post_verifier" + ) + == "pass" + and metadata_deployment.get("learning_writeback") == "partial_degraded" and sqlite_backup.get("runtime_export_terminal") == "pending_authenticated_stable_export" and sqlite_backup.get("runtime_restore_terminal") == "pending_isolated_same_version_target" and sqlite_backup.get("runtime_zero_residue_terminal") == "pending" - and len(sqlite_backup.get("required_preconditions", [])) == 6 + and len(sqlite_backup.get("required_preconditions", [])) == 5 and offsite_backup.get("status") == "pending_offsite_snapshot_and_restore_verifier" and offsite_backup.get("current_repository_scope") diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py index 287316873..added60fe 100644 --- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -834,9 +834,7 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> ] assert sqlite_pending["raw_sqlite_read_or_sync_allowed"] is False assert sqlite_pending["sqlite_cli_present_in_runtime"] is False - assert sqlite_pending["source_contract_status"] == ( - "implemented_tested_not_deployed" - ) + assert sqlite_pending["source_contract_status"] == ("deployed_inactive_verified") source_contract = sqlite_pending["source_contract"] assert source_contract["policy"] == "config/signoz/metadata-export-policy.json" assert source_contract["exporter"] == ("scripts/backup/signoz-metadata-export.py") @@ -878,6 +876,39 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> for field, relative in metadata_paths.items() } assert source_contract["test_terminal"] == "21_passed" + deployment = source_contract["deployment"] + assert deployment["status"] == "deployed_inactive_verified" + assert deployment["asset_id"] == "host110-signoz-metadata-toolchain" + assert deployment["target_host"] == "192.168.0.110" + assert deployment["bundle_id"] == ( + "bb6d78b67f506072b2e45e92bc1d415364e25852282a9b068900665c578df011" + ) + assert deployment["directory_owner"] == "root:root" + assert deployment["directory_mode"] == "0750" + assert deployment["active_pointer_present"] is False + assert deployment["runtime_export_executed"] is False + assert deployment["initial_failed_attempt"]["terminal"] == ( + "failed_no_active_pointer_write" + ) + assert deployment["bounded_execution"]["durable_inner_terminal"] == ( + "pass_source_deployed_inactive" + ) + assert deployment["bounded_execution"]["outer_terminal"] == ( + "failed_source_deploy_unverified" + ) + assert deployment["independent_root_postcheck"]["terminal"] == ( + "check_pass_no_write" + ) + assert deployment["independent_root_postcheck"]["remote_state"] == "exact" + assert deployment["independent_root_postcheck"]["drift_count"] == 0 + assert deployment["idempotent_closure"]["execution_terminal"] == ( + "idempotent_no_write" + ) + assert deployment["idempotent_closure"]["terminal"] == ( + "pass_source_deployed_inactive" + ) + assert deployment["idempotent_closure"]["independent_post_verifier"] == "pass" + assert deployment["learning_writeback"] == "partial_degraded" assert source_contract["raw_sqlite_read"] is False assert source_contract["raw_volume_read"] is False assert source_contract["secret_value_persistence"] is False @@ -889,7 +920,7 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> "pending_isolated_same_version_target" ) assert sqlite_pending["runtime_zero_residue_terminal"] == "pending" - assert len(sqlite_pending["required_preconditions"]) == 6 + assert len(sqlite_pending["required_preconditions"]) == 5 offsite = receipt["pending_verifiers"]["signoz_backup_offsite_dr"] assert offsite["current_repository_scope"] == "local_same_failure_domain" assert offsite["offsite_verified"] is False @@ -1063,7 +1094,7 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N post_closure["signoz_sqlite_application_consistent_backup_status"] == "pending" ) assert post_closure["signoz_metadata_export_source_contract_status"] == ( - "implemented_tested_not_deployed" + "deployed_inactive_verified" ) assert post_closure["signoz_metadata_export_policy"] == ( "config/signoz/metadata-export-policy.json"