fix(observability): harden native backup controlled apply

scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py

@@ -3,10 +3,13 @@ from __future__ import annotations
 import os
 from pathlib import Path
 import subprocess
+import time
 
 
 ROOT = Path(__file__).resolve().parents[3]
 DEPLOYER = ROOT / "scripts/ops/deploy-signoz-native-backup-toolchain.sh"
+BACKUP = ROOT / "scripts/backup/backup-signoz.sh"
+CANARY = ROOT / "scripts/backup/run-signoz-backup-canary.sh"
 
 
 def deployer_text() -> str:
@@ -119,6 +122,33 @@ def test_safe_absolute_path_overrides_fail_closed_before_network_access() -> Non
     assert "must not overlap" in result.stderr
 
 
+def test_timeout_overrides_fail_closed_before_network_access() -> None:
+    for key, value, expected in (
+        (
+            "SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS",
+            "0",
+            "positive integer seconds",
+        ),
+        (
+            "SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS",
+            "301",
+            "exceeds bounded maximum",
+        ),
+    ):
+        env = base_env()
+        env[key] = value
+        result = subprocess.run(
+            ["bash", str(DEPLOYER), "--check"],
+            check=False,
+            capture_output=True,
+            text=True,
+            env=env,
+        )
+        assert result.returncode == 64
+        assert expected in result.stderr
+        assert "ssh:" not in result.stderr
+
+
 def test_target_host_is_fixed_to_host110_and_not_environment_overridable() -> None:
     text = deployer_text()
     assert 'TARGET_HOST="wooo@192.168.0.110"' in text
@@ -229,6 +259,70 @@ def test_check_mode_uses_only_ssh_and_never_scp(tmp_path: Path) -> None:
     assert "check_pass_no_write" in result.stdout
 
 
+def test_hung_ssh_is_bounded_and_cannot_claim_check_pass(tmp_path: Path) -> None:
+    fake_bin = tmp_path / "bin"
+    fake_bin.mkdir()
+    ssh = fake_bin / "ssh"
+    ssh.write_text("#!/bin/sh\nsleep 5\n", encoding="utf-8")
+    scp = fake_bin / "scp"
+    scp.write_text("#!/bin/sh\nexit 99\n", encoding="utf-8")
+    ssh.chmod(0o755)
+    scp.chmod(0o755)
+
+    env = base_env()
+    env["PATH"] = f"{fake_bin}:{env['PATH']}"
+    env["SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS"] = "1"
+    env["SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS"] = "1"
+    started = time.monotonic()
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+        timeout=5,
+    )
+    elapsed = time.monotonic() - started
+
+    assert result.returncode == 124
+    assert elapsed < 4
+    assert "check_pass_no_write" not in result.stdout
+
+
+def test_all_ssh_scp_and_docker_calls_use_bounded_wrappers() -> None:
+    text = deployer_text()
+    executable = executable_text()
+
+    assert (
+        'SSH_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS:-300}"'
+        in text
+    )
+    assert (
+        'SCP_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS:-120}"'
+        in text
+    )
+    assert (
+        'DOCKER_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS:-30}"'
+        in text
+    )
+    assert (
+        'TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS:-10}"'
+        in text
+    )
+    assert "bounded_ssh()" in text
+    assert "bounded_scp()" in text
+    assert text.count('ssh "$@"') == 1
+    assert text.count('scp "$@"') == 1
+    assert executable.count("bounded_docker()") == 2
+    assert "$(docker inspect" not in executable
+    assert "$(docker exec" not in executable
+    assert "\n  docker inspect" not in executable
+    assert "\n  docker exec" not in executable
+    assert executable.count("bounded_docker inspect") == 2
+    assert executable.count("bounded_docker exec") == 2
+    assert "timeout --signal=TERM" in executable
+
+
 def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
     text = deployer_text()
     for token in (
@@ -250,18 +344,60 @@ def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
     assert "bounded_execution_not_started_targets_unchanged" in text
 
 
-def test_apply_uses_same_filesystem_candidates_and_atomic_replace() -> None:
+def test_apply_uses_same_filesystem_candidates_and_accurate_replace_receipt() -> None:
     text = deployer_text()
     assert '"${target}.candidate.${RUN_ID}"' in text
     assert (
         'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"'
     ) in text
     assert "deployed-manifest.tsv" in text
-    assert "seven_candidates_atomically_replaced_expected_hashes_and_modes" in text
+    assert (
+        "seven_same_filesystem_candidates_replaced_with_full_rollback_"
+        "expected_hashes_and_modes"
+    ) in text
+    assert "seven_candidates_atomically_replaced" not in text
+    assert "atomically replaces exact targets" not in text
     assert "stage_candidate_residue_0" in text
     assert "verify_residue_zero" in text
 
 
+def test_apply_holds_dual_backup_locks_through_postverify_or_rollback() -> None:
+    text = deployer_text()
+    backup_text = BACKUP.read_text(encoding="utf-8")
+    canary_text = CANARY.read_text(encoding="utf-8")
+    canary_lock = "/tmp/awoooi-signoz-backup-canary.lock"
+    operation_lock = "/tmp/awoooi-signoz-backup-operation.lock"
+
+    assert f'CANARY_LOCK="{canary_lock}"' in text
+    assert f'OPERATION_LOCK="{operation_lock}"' in text
+    assert canary_lock in canary_text
+    assert operation_lock in backup_text
+    canary_acquire = text.index('exec 8>>"${CANARY_LOCK}"')
+    operation_acquire = text.index('exec 7>>"${OPERATION_LOCK}"')
+    active_check = text.index("if pgrep -af", operation_acquire)
+    apply_started = text.index("APPLY_STARTED=1", operation_acquire)
+    target_replace = text.index(
+        'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}"', apply_started
+    )
+    assert canary_acquire < operation_acquire < active_check < apply_started
+    assert apply_started < target_replace
+
+    on_exit = text[text.index("on_exit() {") : text.index("trap on_exit EXIT")]
+    rollback = on_exit.index("rollback_targets")
+    release_operation = on_exit.index("flock -u 7")
+    release_canary = on_exit.index("flock -u 8")
+    assert rollback < release_operation < release_canary
+    assert 'rm -f "${OPERATION_LOCK}"' not in text
+    assert 'rm -f "${CANARY_LOCK}"' not in text
+    assert "another_signoz_backup_holds_operation_lock" in text
+    assert "another_signoz_canary_holds_canary_lock" in text
+    assert (
+        "risk_medium_bounded_ssh_scp_docker_dual_signoz_canary_and_"
+        "backup_operation_locks_"
+        "no_runtime_execution_no_container_change_full_rollback"
+    ) in text
+
+
 def test_runtime_post_verifier_requires_exact_identity_and_lifecycle_parity() -> None:
     text = deployer_text()
     for container in (