diff --git a/agent99-signoz-metadata-executor.ps1 b/agent99-signoz-metadata-executor.ps1 index a3b06bb48..d46c40551 100644 --- a/agent99-signoz-metadata-executor.ps1 +++ b/agent99-signoz-metadata-executor.ps1 @@ -38,11 +38,11 @@ $SignozImageId = "sha256:c17991d10966aedcb0d4d83805a0baf5310f1553d90611629036fcf $SignozContainer = "signoz" $OperationLock = "/tmp/awoooi-signoz-backup-operation.lock" $TargetKillAfterSeconds = 15 -$TargetGuardWorstCaseSeconds = 143 +$TargetGuardWorstCaseSeconds = 95 $CheckVerifyTargetTimeoutSeconds = 90 $ApplyTargetTimeoutSeconds = 720 $CheckVerifyControllerWaitSeconds = 300 -$ApplyControllerWaitSeconds = 960 +$ApplyControllerWaitSeconds = 900 $ControllerTransportReserveSeconds = 30 $TargetProgramTimeoutSeconds = if ($Mode -eq "Apply") { $ApplyTargetTimeoutSeconds } else { $CheckVerifyTargetTimeoutSeconds } $ControllerWaitSeconds = if ($Mode -eq "Apply") { $ApplyControllerWaitSeconds } else { $CheckVerifyControllerWaitSeconds } @@ -171,8 +171,8 @@ function New-Agent99SignozLockedRemoteCommand { "[ ! -e `"$OutputDir`" ] && [ ! -L `"$OutputDir`" ] && " + "[ ! -e `"$RemoteReceipt`" ] && [ ! -L `"$RemoteReceipt`" ]; " } - return ( - "sudo -n /bin/bash -c 'set -euo pipefail; " + + $bashProgram = ( + "set -euo pipefail; " + "[ -f `"$OperationLock`" ] && [ ! -L `"$OperationLock`" ]; " + "exec 8<`"$OperationLock`"; /usr/bin/flock -n 8; " + $artifactGuard + @@ -181,19 +181,16 @@ function New-Agent99SignozLockedRemoteCommand { "[ `"`$(/usr/bin/timeout --kill-after=5s 15s /usr/bin/docker inspect --format={{.State.Running}} $SignozContainer)`" = `"true`" ]; " + "[ `"`$(/usr/bin/timeout --kill-after=5s 15s /usr/bin/curl -sS -o /dev/null -w `"%{http_code}`" $BaseUrl/api/v1/health)`" = `"200`" ]; " + "set +e; active_output=`$(/usr/bin/timeout --kill-after=5s 10s /usr/bin/pgrep -af `"([s]ignoz-metadata-export[.]py|[s]ignoz-metadata-restore-drill[.]py|(^|/|[[:space:]])([b]ackup-signoz|[c]lickhouse-native-backup|[c]lickhouse-native-restore-drill|[r]un-signoz-backup-canary)[.]sh([[:space:]]|$))`"); active_rc=`$?; set -e; " + - "[ `"`$active_rc`" -eq 0 ] || [ `"`$active_rc`" -eq 1 ]; " + - "ancestor_pattern=`$`$; ancestor_cursor=`$PPID; ancestor_complete=0; " + - "for ancestor_step in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16; do " + - "case `"`$ancestor_cursor`" in `"`"|*[!0-9]*) exit 70;; esac; " + - "ancestor_pattern=`"`$ancestor_pattern|`$ancestor_cursor`"; " + - "if [ `"`$ancestor_cursor`" -eq 1 ]; then ancestor_complete=1; break; fi; " + - "ancestor_cursor=`$(/usr/bin/timeout --signal=TERM --kill-after=1s 2s /bin/ps -o ppid= -p `"`$ancestor_cursor`" | /usr/bin/tr -d `"[:space:]`"); " + - "done; [ `"`$ancestor_complete`" -eq 1 ]; " + - "set +e; external_output=`$(/usr/bin/printf `"%s\n`" `"`$active_output`" | /usr/bin/grep -Ev `"^(`$ancestor_pattern)[[:space:]]`"); filter_rc=`$?; set -e; " + - "[ `"`$filter_rc`" -eq 0 ] || [ `"`$filter_rc`" -eq 1 ]; " + - "[ -z `"`$external_output`" ]; " + + "[ `"`$active_rc`" -eq 1 ] && [ -z `"`$active_output`" ]; " + "exec /usr/bin/timeout --signal=TERM --kill-after=$($TargetKillAfterSeconds)s " + - "$($TargetTimeoutSeconds)s $FixedCommand'" + "$($TargetTimeoutSeconds)s $FixedCommand" + ) + $encodedProgram = [Convert]::ToBase64String( + [Text.Encoding]::UTF8.GetBytes($bashProgram) + ) + return ( + "/usr/bin/printf %s $encodedProgram | /usr/bin/base64 -d | " + + "sudo -n /bin/bash" ) } diff --git a/scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py b/scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py index cf0a5a083..c7c87263e 100644 --- a/scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py +++ b/scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py @@ -1,12 +1,8 @@ from __future__ import annotations import re -import shutil -import subprocess from pathlib import Path -import pytest - ROOT = Path(__file__).resolve().parents[3] ENTRYPOINT = ROOT / "agent99-signoz-metadata-executor.ps1" @@ -73,11 +69,11 @@ def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() -> assert '$OperationLock = "/tmp/awoooi-signoz-backup-operation.lock"' in source assert '[ -f `"$OperationLock`" ] && [ ! -L `"$OperationLock`" ]' in source assert 'exec 8<`"$OperationLock`"; /usr/bin/flock -n 8' in source - assert "$TargetGuardWorstCaseSeconds = 143" in source + assert "$TargetGuardWorstCaseSeconds = 95" in source assert "$CheckVerifyTargetTimeoutSeconds = 90" in source assert "$ApplyTargetTimeoutSeconds = 720" in source assert "$CheckVerifyControllerWaitSeconds = 300" in source - assert "$ApplyControllerWaitSeconds = 960" in source + assert "$ApplyControllerWaitSeconds = 900" in source assert "$ControllerTransportReserveSeconds = 30" in source assert ( "New-Agent99SignozLockedRemoteCommand $applyProgram " @@ -101,175 +97,19 @@ def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() -> assert '--kill-after=5s 15s /usr/bin/curl' in source assert '--kill-after=5s 10s /usr/bin/pgrep' in lock_helper assert "active_rc=`$?; set -e" in lock_helper - assert ( - '[ `"`$active_rc`" -eq 0 ] || [ `"`$active_rc`" -eq 1 ]' - in lock_helper - ) - assert "ancestor_pattern=`$`$; ancestor_cursor=`$PPID" in lock_helper - assert ( - "for ancestor_step in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16" - in lock_helper - ) - assert "/bin/ps -o ppid= -p" in lock_helper - assert ( - "/usr/bin/timeout --signal=TERM --kill-after=1s 2s /bin/ps" - in lock_helper - ) - assert "ancestor_complete=1; break" in lock_helper - assert '[ `"`$ancestor_complete`" -eq 1 ]' in lock_helper - assert ( - '/usr/bin/grep -Ev `"^(`$ancestor_pattern)[[:space:]]`"' - in lock_helper - ) - assert "filter_rc=`$?; set -e" in lock_helper - assert ( - '[ `"`$filter_rc`" -eq 0 ] || [ `"`$filter_rc`" -eq 1 ]' - in lock_helper - ) - assert '[ -z `"`$external_output`" ]' in lock_helper - assert lock_helper.index("active_rc=`$?; set -e") < lock_helper.index( - "filter_rc=`$?; set -e" - ) < lock_helper.index('[ -z `"`$external_output`" ]') + assert '[ `"`$active_rc`" -eq 1 ] && [ -z `"`$active_output`" ]' in lock_helper + assert "$bashProgram = (" in lock_helper + assert "[Convert]::ToBase64String(" in lock_helper + assert "[Text.Encoding]::UTF8.GetBytes($bashProgram)" in lock_helper + assert "/usr/bin/printf %s $encodedProgram" in lock_helper + assert "/usr/bin/base64 -d" in lock_helper + assert '"sudo -n /bin/bash"' in lock_helper + assert "sudo -n /bin/bash -c '" not in lock_helper assert "pgrep -af" in lock_helper assert "|| true" not in lock_helper assert '--signal=TERM --kill-after=$($TargetKillAfterSeconds)s' in source -def _render_locked_process_gate() -> str: - source = ENTRYPOINT.read_text(encoding="utf-8") - helper = source[ - source.index("function New-Agent99SignozLockedRemoteCommand") : - source.index("function Get-Agent99RuntimeSourceBinding") - ] - block = helper[ - helper.index(' "set +e; active_output=') : - helper.index(' "exec /usr/bin/timeout') - ] - rendered_parts: list[str] = [] - for line in block.splitlines(): - literal = line.strip() - assert literal.startswith('"') and literal.endswith('" +') - literal = literal[1:-3] - rendered = [] - index = 0 - while index < len(literal): - if literal[index] == "`": - index += 1 - assert index < len(literal) - rendered.append(literal[index]) - index += 1 - rendered_parts.append("".join(rendered)) - rendered_block = "".join(rendered_parts) - gate_start = rendered_block.index( - '[ "$active_rc" -eq 0 ] || [ "$active_rc" -eq 1 ]; ' - ) - return rendered_block[gate_start:] - - -@pytest.mark.parametrize( - ("pgrep_rc", "row_scenario", "expected"), - ( - (1, "empty", True), - (0, "ancestors", True), - (0, "external", False), - (2, "empty", False), - (124, "empty", False), - ), -) -def test_locked_process_guard_rendered_shell_behavior( - pgrep_rc: int, - row_scenario: str, - expected: bool, -) -> None: - timeout_path = shutil.which("timeout") - assert timeout_path is not None - rendered_gate = _render_locked_process_gate().replace( - "/usr/bin/timeout", timeout_path - ) - setup = { - "empty": 'active_output=""; ', - "ancestors": ( - 'grandparent=$(/bin/ps -o ppid= -p "$PPID" | ' - '/usr/bin/tr -d "[:space:]"); ' - 'active_output="$$ bash -c fixed-export\n' - '$PPID sudo bash -c fixed-export\n' - '$grandparent outer-shell fixed-export"; ' - ), - "external": ( - 'active_output="$$ bash -c fixed-export\n' - '$PPID sudo bash -c fixed-export\n' - '99999999 python3 signoz-metadata-export.py --apply"; ' - ), - }[row_scenario] - completed = subprocess.run( - ( - "/bin/bash", - "-c", - f"set -euo pipefail; active_rc={pgrep_rc}; {setup}" - f"{rendered_gate}printf guard_accepted", - ), - check=False, - capture_output=True, - text=True, - timeout=10, - ) - assert (completed.returncode == 0) is expected - assert (completed.stdout == "guard_accepted") is expected - - -def test_locked_process_guard_rendered_filter_error_fails_closed() -> None: - gate = _render_locked_process_gate() - filter_gate = gate[gate.index('[ "$filter_rc" -eq 0 ]') :] - completed = subprocess.run( - ( - "/bin/bash", - "-c", - "set -euo pipefail; filter_rc=2; external_output=\"\"; " - f"{filter_gate}printf guard_accepted", - ), - check=False, - capture_output=True, - text=True, - timeout=10, - ) - assert completed.returncode != 0 - assert completed.stdout == "" - - -@pytest.mark.parametrize( - "replacement", - ( - "/usr/bin/false", - "/usr/bin/printf not-a-pid", - "/usr/bin/printf 99999999", - ), -) -def test_locked_process_guard_ancestor_walk_faults_fail_closed( - replacement: str, -) -> None: - gate = _render_locked_process_gate() - lookup = ( - "/usr/bin/timeout --signal=TERM --kill-after=1s 2s " - '/bin/ps -o ppid= -p "$ancestor_cursor"' - ) - assert gate.count(lookup) == 1 - fault_injected_gate = gate.replace(lookup, replacement) - completed = subprocess.run( - ( - "/bin/bash", - "-c", - "set -euo pipefail; active_rc=1; active_output=\"\"; " - f"{fault_injected_gate}printf guard_accepted", - ), - check=False, - capture_output=True, - text=True, - timeout=10, - ) - assert completed.returncode != 0 - assert completed.stdout == "" - - def test_entrypoint_uses_typed_stdout_without_windows_process_exit_code() -> None: source = ENTRYPOINT.read_text(encoding="utf-8") transport = source[ @@ -360,11 +200,11 @@ def test_entrypoint_typed_readbacks_are_exact_and_preserve_unknown_state() -> No assert "$RemoteResult.completed -ne $true" in source assert "$RemoteResult.timedOut" in source - guard = 143 + guard = 95 kill_after = 15 transport_reserve = 30 assert guard + 90 + kill_after + transport_reserve < 300 - assert guard + 720 + kill_after + transport_reserve < 960 + assert guard + 720 + kill_after + transport_reserve < 900 def test_entrypoint_never_reads_or_persists_credential_value_in_controller() -> None: