From 2b7ddad22b165d77ea95a77ba27893800ac6a80a Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 16:15:59 +0800 Subject: [PATCH] fix(observability): preserve shared SigNoz operation lock --- ops/signoz/p0-obs-002-post-closure-regression.yaml | 6 +++--- scripts/ops/deploy-signoz-metadata-toolchain.sh | 9 ++++++++- .../ops/tests/test_signoz_metadata_toolchain_deploy.py | 9 +++++++++ .../reboot-recovery/signoz-canonical-route-preflight.py | 2 +- .../tests/test_signoz_canonical_route_migration.py | 2 +- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/ops/signoz/p0-obs-002-post-closure-regression.yaml b/ops/signoz/p0-obs-002-post-closure-regression.yaml index 0800683ed..3e1510057 100644 --- a/ops/signoz/p0-obs-002-post-closure-regression.yaml +++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml @@ -778,9 +778,9 @@ pending_verifiers: independent_export_verifier_sha256: cbab56dd3919866a96550c1dce46ec67783cd6f00473e4491861e17d1430048c isolated_restore_driver_sha256: 38ce70b3f891a3ce8edfaeb880ccf69d0bb6817c3f801a0aefa0f06341901997 contract_tests_sha256: 474d9dfcfe170c62425356ea9f06ed0dc1629b39215ee80f395f721c7d43a923 - controlled_deployer_sha256: b3edcb3629aa3069fcecde5812cc4f21e83e324eec89b3c75f61cc73444adb97 - deployer_tests_sha256: cae135b2075d845f13d762c751d9e6de88bf806b37dcd5bd8837487211a20880 - test_terminal: 19_passed + controlled_deployer_sha256: ca64829e8cd850d83b016a6207bf463da586a68a4c78571e7a8734c93deb4766 + deployer_tests_sha256: 2092164f254b7c8b9064f2ffb1859e6dd4213b627992373719b3d9bc8a6ae200 + test_terminal: 20_passed portable_assets: - dashboards - alert_rules diff --git a/scripts/ops/deploy-signoz-metadata-toolchain.sh b/scripts/ops/deploy-signoz-metadata-toolchain.sh index d9981c188..ac447471e 100755 --- a/scripts/ops/deploy-signoz-metadata-toolchain.sh +++ b/scripts/ops/deploy-signoz-metadata-toolchain.sh @@ -395,7 +395,14 @@ mkdir -m 0700 "${RECEIPT_DIR}" "${CANDIDATE_DIR}" : > "${RECEIPT_LOG}" chmod 0600 "${RECEIPT_LOG}" -exec 8>>/tmp/awoooi-signoz-backup-operation.lock +OPERATION_LOCK=/tmp/awoooi-signoz-backup-operation.lock +[ -f "${OPERATION_LOCK}" ] && [ ! -L "${OPERATION_LOCK}" ] +# The shared lock is intentionally owned by the unprivileged backup account. +# Open the existing inode read-only so Linux fs.protected_regular=2 cannot +# reject a root O_CREAT/O_APPEND open in sticky /tmp. flock(2) still provides +# the same exclusive inter-process lock without changing file contents, +# ownership, or mode. Missing/unsafe lock state fails closed. +exec 8<"${OPERATION_LOCK}" flock -n 8 active_operations="$( (pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' || true) \ diff --git a/scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py b/scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py index b2d1d130e..256f2feec 100644 --- a/scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py +++ b/scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py @@ -165,6 +165,15 @@ def test_deployer_has_no_active_pointer_or_destructive_fallback() -> None: assert "ln -s" not in source +def test_shared_operation_lock_is_existing_read_only_and_fail_closed() -> None: + source = DEPLOYER.read_text(encoding="utf-8") + assert "OPERATION_LOCK=/tmp/awoooi-signoz-backup-operation.lock" in source + assert '[ -f "${OPERATION_LOCK}" ] && [ ! -L "${OPERATION_LOCK}" ]' in source + assert 'exec 8<"${OPERATION_LOCK}"' in source + assert "flock -n 8" in source + assert "exec 8>>/tmp/awoooi-signoz-backup-operation.lock" not in source + + def test_exact_five_file_supply_chain_and_modes_are_fixed() -> None: source = DEPLOYER.read_text(encoding="utf-8") for path in ( diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py index fed7e045b..d22a1e528 100644 --- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -865,7 +865,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: == len(METADATA_EXPORT_SOURCE_PATHS) and set(declared_metadata_export_hashes) == set(METADATA_EXPORT_SOURCE_PATHS) and declared_metadata_export_hashes == actual_metadata_export_hashes - and metadata_export_source_contract.get("test_terminal") == "19_passed" + and metadata_export_source_contract.get("test_terminal") == "20_passed" ) offsite_backup = pending_verifiers.get("signoz_backup_offsite_dr", {}) failed_artifacts = pending_verifiers.get( diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py index ba17d042e..8015ff7d8 100644 --- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -877,7 +877,7 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> field: hashlib.sha256((ROOT / relative).read_bytes()).hexdigest() for field, relative in metadata_paths.items() } - assert source_contract["test_terminal"] == "19_passed" + assert source_contract["test_terminal"] == "20_passed" assert source_contract["raw_sqlite_read"] is False assert source_contract["raw_volume_read"] is False assert source_contract["secret_value_persistence"] is False