wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

feat(signoz): add recoverable Agent99 metadata toolchain route

Browse Source
This commit is contained in:
Your Name
2026-07-22 21:02:02 +08:00
parent d097013a03
commit 2bff3edcff
23 changed files with 3510 additions and 1111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

485
scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh Executable file
View File

@@ -0,0 +1,485 @@
#!/usr/bin/env bash
# Source-bound controller -> Windows99/Agent99 -> Host110 SigNoz toolchain route.
set -euo pipefail
readonly SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
readonly DEFAULT_SOURCE_ROOT="$(CDPATH= cd -- "${SCRIPT_DIR}/../.." && pwd)"
readonly TARGET="Administrator@192.168.0.99"
readonly GITEA_REMOTE="gitea"
readonly GITEA_URL_HTTPS="https://gitea.wooo.work/wooo/awoooi.git"
readonly GITEA_URL_HTTP_INTERNAL="http://192.168.0.110:3000/wooo/awoooi.git"
readonly GITEA_URL_SSH_INTERNAL="ssh://git@192.168.0.110:2222/wooo/awoooi.git"
readonly EXECUTOR_RELATIVE="agent99-signoz-metadata-executor.ps1"
readonly TARGET_RELATIVE="agent99-signoz-toolchain-target.py"
readonly WRAPPER_RELATIVE="scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh"
LABELS=(
"metadata-export-policy.json"
"signoz_metadata_contract.py"
"signoz-metadata-export.py"
"verify-signoz-metadata-export.py"
"signoz-metadata-restore-drill.py"
"signoz-metadata-isolated-restore.py"
"isolated-cluster.xml"
)
TOOLCHAIN_SOURCES=(
"config/signoz/metadata-export-policy.json"
"scripts/backup/signoz_metadata_contract.py"
"scripts/backup/signoz-metadata-export.py"
"scripts/backup/verify-signoz-metadata-export.py"
"scripts/backup/signoz-metadata-restore-drill.py"
"scripts/backup/signoz-metadata-isolated-restore.py"
"ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
)
MODES=("0644" "0644" "0755" "0755" "0755" "0755" "0644")
MODE="check"
MODE_SELECTED=""
TRACE_ID=""
RUN_ID=""
WORK_ITEM_ID=""
ATTEMPT_ID=""
SOURCE_ROOT="${DEFAULT_SOURCE_ROOT}"
SOURCE_REVISION=""
IDENTITY_FILE=""
KNOWN_HOSTS_FILE="${HOME:-}/.ssh/known_hosts"
CONNECT_TIMEOUT_SECONDS="8"
REMOTE_EXECUTION_TIMEOUT_SECONDS="1200"
REMOTE_PAYLOAD_PREPARED="false"
REMOTE_PAYLOAD_PATH_WINDOWS=""
usage() {
printf '%s\n' \
"Usage: $0 [--check|--apply|--verify|--reconcile] [options]" \
"" \
"Fixed route: local fresh Gitea main -> Windows99 Agent99 -> Host110." \
"No local connection to Host110 and no active toolchain pointer are allowed." \
"" \
" --trace-id ID Required controlled-apply trace identity." \
" --run-id ID Required original deployment run identity." \
" --work-item-id P0-OBS-002 Required fixed work item." \
" --attempt-id ID Required for reconcile; optional apply recovery ID." \
" --source-root PATH Checkout root (default: repository root)." \
" --source-revision SHA Must equal freshly fetched gitea/main." \
" --identity-file PATH Optional Windows99 SSH identity path." \
" --known-hosts-file PATH Pinned known_hosts file." \
" --connect-timeout SECONDS 1 through 30 (default: 8)."
}
fail() {
printf 'signoz_toolchain_agent99_error=%s\n' "$1" >&2
exit 64
}
require_value() {
[[ $# -ge 2 && -n "$2" ]] || fail "missing_option_value"
}
while [[ $# -gt 0 ]]; do
case "$1" in
--check|--apply|--verify|--reconcile)
[[ -z "${MODE_SELECTED}" ]] || fail "multiple_mode_flags"
MODE="${1#--}"
MODE_SELECTED="true"
shift
;;
--trace-id)
require_value "$@"; TRACE_ID="$2"; shift 2 ;;
--run-id)
require_value "$@"; RUN_ID="$2"; shift 2 ;;
--work-item-id)
require_value "$@"; WORK_ITEM_ID="$2"; shift 2 ;;
--attempt-id)
require_value "$@"; ATTEMPT_ID="$2"; shift 2 ;;
--source-root)
require_value "$@"; SOURCE_ROOT="$2"; shift 2 ;;
--source-revision)
require_value "$@"; SOURCE_REVISION="$2"; shift 2 ;;
--identity-file)
require_value "$@"; IDENTITY_FILE="$2"; shift 2 ;;
--known-hosts-file)
require_value "$@"; KNOWN_HOSTS_FILE="$2"; shift 2 ;;
--connect-timeout)
require_value "$@"; CONNECT_TIMEOUT_SECONDS="$2"; shift 2 ;;
-h|--help)
usage; exit 0 ;;
*) fail "unsupported_option" ;;
esac
done
readonly SAFE_ID_PATTERN='^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$'
[[ "${TRACE_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "trace_id_missing_or_invalid"
[[ "${RUN_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "run_id_missing_or_invalid"
[[ "${WORK_ITEM_ID}" == "P0-OBS-002" ]] || fail "work_item_id_not_allowlisted"
if [[ "${MODE}" == "reconcile" ]]; then
[[ "${ATTEMPT_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "reconcile_attempt_id_missing_or_invalid"
elif [[ -n "${ATTEMPT_ID}" && "${MODE}" != "apply" ]]; then
fail "attempt_id_not_allowed_for_mode"
fi
[[ "${CONNECT_TIMEOUT_SECONDS}" =~ ^[0-9]+$ ]] || fail "invalid_connect_timeout"
(( CONNECT_TIMEOUT_SECONDS >= 1 && CONNECT_TIMEOUT_SECONDS <= 30 )) || fail "invalid_connect_timeout"
for dependency in git python3 ssh scp mktemp timeout; do
command -v "${dependency}" >/dev/null 2>&1 || fail "required_command_missing_${dependency}"
done
[[ -f "${KNOWN_HOSTS_FILE}" && ! -L "${KNOWN_HOSTS_FILE}" ]] || fail "known_hosts_file_missing_or_unsafe"
if [[ -n "${IDENTITY_FILE}" ]]; then
[[ -f "${IDENTITY_FILE}" && ! -L "${IDENTITY_FILE}" ]] || fail "identity_file_missing_or_unsafe"
fi
[[ -d "${SOURCE_ROOT}" ]] || fail "source_root_missing"
SOURCE_ROOT="$(CDPATH= cd -- "${SOURCE_ROOT}" && pwd)"
GITEA_URL="$(git -C "${SOURCE_ROOT}" remote get-url "${GITEA_REMOTE}" 2>/dev/null)" \
|| fail "gitea_remote_missing"
case "${GITEA_URL}" in
"${GITEA_URL_HTTPS}"|"${GITEA_URL_HTTP_INTERNAL}"|"${GITEA_URL_SSH_INTERNAL}") ;;
*) fail "gitea_remote_not_fixed_internal" ;;
esac
git -C "${SOURCE_ROOT}" fetch --quiet --no-tags "${GITEA_REMOTE}" \
'+refs/heads/main:refs/remotes/gitea/main' || fail "fresh_gitea_main_fetch_failed"
GITEA_MAIN_REVISION="$(git -C "${SOURCE_ROOT}" rev-parse --verify 'refs/remotes/gitea/main^{commit}')" \
|| fail "fresh_gitea_main_revision_unavailable"
GITEA_MAIN_REVISION="$(printf '%s' "${GITEA_MAIN_REVISION}" | tr '[:upper:]' '[:lower:]')"
if [[ -z "${SOURCE_REVISION}" ]]; then
SOURCE_REVISION="${GITEA_MAIN_REVISION}"
fi
SOURCE_REVISION="$(printf '%s' "${SOURCE_REVISION}" | tr '[:upper:]' '[:lower:]')"
[[ "${SOURCE_REVISION}" =~ ^[0-9a-f]{40}$ ]] || fail "source_revision_must_be_full_commit_sha"
[[ "${SOURCE_REVISION}" == "${GITEA_MAIN_REVISION}" ]] || fail "source_revision_is_not_fresh_gitea_main"
SOURCE_BOUND_FILES=(
"${TOOLCHAIN_SOURCES[@]}"
"${EXECUTOR_RELATIVE}"
"${TARGET_RELATIVE}"
"${WRAPPER_RELATIVE}"
)
for relative_path in "${SOURCE_BOUND_FILES[@]}"; do
[[ -f "${SOURCE_ROOT}/${relative_path}" && ! -L "${SOURCE_ROOT}/${relative_path}" ]] \
|| fail "source_bound_file_missing_or_unsafe"
git -C "${SOURCE_ROOT}" ls-files --error-unmatch -- "${relative_path}" >/dev/null 2>&1 \
|| fail "source_bound_file_untracked"
done
git -C "${SOURCE_ROOT}" diff --quiet "${SOURCE_REVISION}" -- "${SOURCE_BOUND_FILES[@]}" \
|| fail "source_bundle_differs_from_revision"
umask 077
WORK_DIR="$(mktemp -d /tmp/awoooi-signoz-toolchain-agent99.XXXXXXXX)"
ENVELOPE_PATH="${WORK_DIR}/toolchain-envelope.json"
REMOTE_STDOUT_PATH="${WORK_DIR}/remote.stdout"
REMOTE_STDERR_PATH="${WORK_DIR}/remote.stderr"
SSH_OPTIONS=(
-T
-o BatchMode=yes
-o PreferredAuthentications=publickey
-o PubkeyAuthentication=yes
-o PasswordAuthentication=no
-o KbdInteractiveAuthentication=no
-o NumberOfPasswordPrompts=0
-o StrictHostKeyChecking=yes
-o "UserKnownHostsFile=${KNOWN_HOSTS_FILE}"
-o "ConnectTimeout=${CONNECT_TIMEOUT_SECONDS}"
-o ConnectionAttempts=1
-o ServerAliveInterval=10
-o ServerAliveCountMax=3
-o LogLevel=ERROR
-o IdentitiesOnly=yes
)
SCP_OPTIONS=("${SSH_OPTIONS[@]:1}")
if [[ -n "${IDENTITY_FILE}" ]]; then
SSH_OPTIONS+=( -i "${IDENTITY_FILE}" )
SCP_OPTIONS+=( -i "${IDENTITY_FILE}" )
fi
make_encoded_command() {
local executor_mode="$1" payload_path="$2" attempt_id="$3"
python3 - "${executor_mode}" "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
"${SOURCE_REVISION}" "${payload_path}" "${attempt_id}" <<'PY'
from __future__ import annotations
import base64
import sys
mode, trace_id, run_id, work_item_id, revision, payload_path, attempt_id = sys.argv[1:]
values = [mode, trace_id, run_id, work_item_id, revision, payload_path, attempt_id]
if any("'" in value or "\r" in value or "\n" in value for value in values):
raise SystemExit("unsafe_remote_command_value")
script = rf'''$ErrorActionPreference = 'Stop'
$executor = 'C:\Wooo\Agent99\bin\agent99-signoz-metadata-executor.ps1'
if (-not (Test-Path -LiteralPath $executor -PathType Leaf)) {{ throw 'executor_missing' }}
$parameters = @{{
Mode = '{mode}'
TraceId = '{trace_id}'
RunId = '{run_id}'
WorkItemId = '{work_item_id}'
SourceRevision = '{revision}'
}}
'''
if mode == "ToolchainApply":
script += rf'''$payloadPath = '{payload_path}'
$envelope = ''
try {{
$item = Get-Item -LiteralPath $payloadPath -ErrorAction Stop
if (($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 -or $item.Length -le 0 -or $item.Length -gt 4194304) {{
throw 'toolchain_envelope_transport_file_invalid'
}}
$envelope = [IO.File]::ReadAllText($payloadPath, [Text.Encoding]::UTF8)
}} finally {{
Remove-Item -LiteralPath $payloadPath -Force -ErrorAction SilentlyContinue
}}
if (Test-Path -LiteralPath $payloadPath) {{ throw 'toolchain_envelope_transport_delete_failed' }}
$parameters.ToolchainEnvelopeJson = $envelope
$parameters.ToolchainEnvelopeDeletedBeforeExecutor = $true
'''
elif mode == "ToolchainReconcile":
script += f"$parameters.AttemptId = '{attempt_id}'\r\n"
script += "& $executor @parameters\r\nexit [int]$LASTEXITCODE\r\n"
encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii")
print(
"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
f"-EncodedCommand {encoded}"
)
PY
}
make_file_command() {
local action="$1" path="$2"
python3 - "${action}" "${path}" <<'PY'
from __future__ import annotations
import base64
import sys
action, path = sys.argv[1:]
if "'" in path or "\r" in path or "\n" in path:
raise SystemExit("unsafe_transport_path")
if action == "prepare":
script = rf'''$ErrorActionPreference='Stop'
$directory='C:\Wooo\Agent99\deploy\transport-inbox'
$path='{path}'
New-Item -ItemType Directory -Force -Path $directory | Out-Null
Remove-Item -LiteralPath $path -Force -ErrorAction SilentlyContinue
if (Test-Path -LiteralPath $path) {{ throw 'stale_transport_payload_cleanup_failed' }}
'''
else:
script = rf'''$ErrorActionPreference='Stop'
$path='{path}'
Remove-Item -LiteralPath $path -Force -ErrorAction SilentlyContinue
if (Test-Path -LiteralPath $path) {{ throw 'transport_payload_cleanup_failed' }}
'''
encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii")
print(
"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
f"-EncodedCommand {encoded}"
)
PY
}
cleanup() {
if [[ "${REMOTE_PAYLOAD_PREPARED}" == "true" && -n "${REMOTE_PAYLOAD_PATH_WINDOWS}" ]]; then
cleanup_command="$(make_file_command cleanup "${REMOTE_PAYLOAD_PATH_WINDOWS}" 2>/dev/null || true)"
if [[ -n "${cleanup_command}" ]]; then
timeout --kill-after=10 60 ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${cleanup_command}" \
>/dev/null 2>&1 || true
fi
fi
rm -rf -- "${WORK_DIR}"
}
trap cleanup EXIT HUP INT TERM
build_envelope() {
python3 - "${SOURCE_ROOT}" "${SOURCE_REVISION}" "${TRACE_ID}" "${RUN_ID}" \
"${WORK_ITEM_ID}" "${ENVELOPE_PATH}" \
"${#TOOLCHAIN_SOURCES[@]}" \
"${TOOLCHAIN_SOURCES[@]}" \
"${LABELS[@]}" \
"${MODES[@]}" <<'PY'
from __future__ import annotations
import base64
import hashlib
import json
import sys
from pathlib import Path
root = Path(sys.argv[1])
revision, trace_id, run_id, work_item_id = sys.argv[2:6]
output = Path(sys.argv[6])
count = int(sys.argv[7])
sources = sys.argv[8 : 8 + count]
labels = sys.argv[8 + count : 8 + 2 * count]
modes = sys.argv[8 + 2 * count :]
if count != 7 or len(set(labels)) != 7 or len(modes) != 7:
raise SystemExit("fixed_toolchain_file_contract_failed")
files = []
manifest_rows = []
hashes = []
for source, label, mode in zip(sources, labels, modes, strict=True):
content = (root / source).read_bytes()
digest = hashlib.sha256(content).hexdigest()
hashes.append(digest)
manifest_rows.append(f"{label}\t{digest}\t{mode}\n")
files.append(
{
"name": label,
"sha256": digest,
"mode": mode,
"contentBase64": base64.b64encode(content).decode("ascii"),
}
)
bundle_id = hashlib.sha256(
"".join(f"{digest}\n" for digest in hashes).encode("ascii")
).hexdigest()
manifest_sha256 = hashlib.sha256("".join(manifest_rows).encode()).hexdigest()
value = {
"schemaVersion": "awoooi_signoz_metadata_toolchain_envelope_v1",
"transport": "fresh_gitea_main_to_windows99_agent99_to_fixed_host110",
"traceId": trace_id,
"runId": run_id,
"workItemId": work_item_id,
"sourceRevision": revision,
"bundleId": bundle_id,
"manifestSha256": manifest_sha256,
"fileCount": 7,
"transportPayloadContainsSecrets": False,
"files": files,
}
output.write_text(json.dumps(value, separators=(",", ":")), encoding="utf-8")
PY
}
invoke_remote() {
local executor_mode="$1" payload_path="$2" attempt_id="$3"
local command
command="$(make_encoded_command "${executor_mode}" "${payload_path}" "${attempt_id}")"
: >"${REMOTE_STDOUT_PATH}"
: >"${REMOTE_STDERR_PATH}"
set +e
timeout --kill-after=20 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}" \
ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${command}" \
>"${REMOTE_STDOUT_PATH}" 2>"${REMOTE_STDERR_PATH}"
LAST_REMOTE_RC=$?
set -e
if [[ ! -s "${REMOTE_STDOUT_PATH}" || $(wc -c <"${REMOTE_STDOUT_PATH}") -gt 65536 ]]; then
LAST_REMOTE_OUTPUT=""
return
fi
LAST_REMOTE_OUTPUT="$(tr -d '\r' <"${REMOTE_STDOUT_PATH}")"
}
validate_receipt() {
local value="$1" executor_mode="$2" allowed_csv="$3"
python3 - "${executor_mode}" "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
"${SOURCE_REVISION}" "${allowed_csv}" "${value}" <<'PY'
from __future__ import annotations
import json
import sys
mode, trace_id, run_id, work_item_id, revision, allowed_csv, raw = sys.argv[1:]
try:
value = json.loads(raw)
except json.JSONDecodeError as exc:
raise SystemExit("agent99_receipt_json_invalid") from exc
allowed = set(allowed_csv.split(","))
required = {
"schemaVersion": "agent99_signoz_metadata_executor_dispatch_v1",
"mode": mode,
"traceId": trace_id,
"runId": run_id,
"workItemId": work_item_id,
"sourceRevision": revision,
"targetHost": "192.168.0.110",
"dispatchPath": "windows99_agent99_to_host110_fixed_metadata_toolchain",
"productionRuntimeMutationPerformed": False,
"completionClaim": False,
}
if any(value.get(key) != expected for key, expected in required.items()):
raise SystemExit("agent99_receipt_identity_or_boundary_invalid")
if value.get("terminal") not in allowed:
raise SystemExit("agent99_receipt_terminal_invalid")
if mode == "ToolchainApply" and (
value.get("toolchainEnvelopeTransportPayloadUsed") is not True
or value.get("toolchainEnvelopeTransportPayloadDeletedBeforeExecutor") is not True
or value.get("toolchainEnvelopeTransportPayloadContainsSecrets") is not False
):
raise SystemExit("agent99_envelope_transport_receipt_invalid")
PY
}
run_checked_mode() {
local executor_mode="$1" payload_path="$2" attempt_id="$3" allowed="$4"
invoke_remote "${executor_mode}" "${payload_path}" "${attempt_id}"
[[ -n "${LAST_REMOTE_OUTPUT}" ]] || fail "${executor_mode}_receipt_missing"
validate_receipt "${LAST_REMOTE_OUTPUT}" "${executor_mode}" "${allowed}" \
|| fail "${executor_mode}_receipt_invalid"
printf '%s\n' "${LAST_REMOTE_OUTPUT}"
}
case "${MODE}" in
check)
run_checked_mode "ToolchainCheck" "" "" "toolchain_check_ready"
[[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_check_failed"
;;
verify)
run_checked_mode "ToolchainVerify" "" "" "toolchain_verify_pass"
[[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_verify_failed"
;;
reconcile)
run_checked_mode "ToolchainReconcile" "" "${ATTEMPT_ID}" \
"toolchain_reconcile_exact_deployment_verified,toolchain_reconcile_zero_owned_residue,toolchain_reconcile_preserved_unverified,toolchain_reconcile_failed_or_unverified"
[[ ${LAST_REMOTE_RC} -eq 0 ]] || exit 75
;;
apply)
run_checked_mode "ToolchainCheck" "" "" "toolchain_check_ready"
[[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_check_failed_before_apply"
build_envelope
PAYLOAD_TOKEN="$(python3 - "${ENVELOPE_PATH}" <<'PY'
import hashlib
import sys
from pathlib import Path
print(hashlib.sha256(Path(sys.argv[1]).read_bytes()).hexdigest()[:24])
PY
)"
REMOTE_PAYLOAD_PATH_WINDOWS="C:\\Wooo\\Agent99\\deploy\\transport-inbox\\signoz-toolchain-${PAYLOAD_TOKEN}.json"
REMOTE_PAYLOAD_PATH_SCP="C:/Wooo/Agent99/deploy/transport-inbox/signoz-toolchain-${PAYLOAD_TOKEN}.json"
prepare_command="$(make_file_command prepare "${REMOTE_PAYLOAD_PATH_WINDOWS}")"
timeout --kill-after=10 60 ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${prepare_command}" \
>/dev/null 2>"${REMOTE_STDERR_PATH}" || fail "remote_payload_prepare_failed"
REMOTE_PAYLOAD_PREPARED="true"
timeout --kill-after=10 120 scp -q "${SCP_OPTIONS[@]}" "${ENVELOPE_PATH}" \
"${TARGET}:${REMOTE_PAYLOAD_PATH_SCP}" >/dev/null 2>"${REMOTE_STDERR_PATH}" \
|| fail "remote_payload_transport_failed"
invoke_remote "ToolchainApply" "${REMOTE_PAYLOAD_PATH_WINDOWS}" ""
apply_output="${LAST_REMOTE_OUTPUT}"
apply_rc="${LAST_REMOTE_RC}"
cleanup_command="$(make_file_command cleanup "${REMOTE_PAYLOAD_PATH_WINDOWS}")"
timeout --kill-after=10 60 ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${cleanup_command}" \
>/dev/null 2>&1 || true
REMOTE_PAYLOAD_PREPARED="false"
if [[ -n "${apply_output}" ]] && validate_receipt "${apply_output}" "ToolchainApply" \
"toolchain_deployed_inactive_verified,toolchain_apply_failed_or_unverified"; then
printf '%s\n' "${apply_output}"
fi
if [[ ${apply_rc} -eq 0 ]]; then
validate_receipt "${apply_output}" "ToolchainApply" "toolchain_deployed_inactive_verified" \
|| fail "toolchain_apply_success_receipt_invalid"
run_checked_mode "ToolchainVerify" "" "" "toolchain_verify_pass"
[[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_verify_failed_after_apply"
exit 0
fi
if [[ -z "${ATTEMPT_ID}" ]]; then
ATTEMPT_ID="reconcile-$(date -u +%Y%m%dT%H%M%SZ)-$$"
fi
[[ "${ATTEMPT_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "generated_reconcile_attempt_id_invalid"
run_checked_mode "ToolchainReconcile" "" "${ATTEMPT_ID}" \
"toolchain_reconcile_exact_deployment_verified,toolchain_reconcile_zero_owned_residue,toolchain_reconcile_preserved_unverified,toolchain_reconcile_failed_or_unverified"
if [[ ${LAST_REMOTE_RC} -eq 0 ]] && [[ "${LAST_REMOTE_OUTPUT}" == *'"terminal":"toolchain_reconcile_exact_deployment_verified"'* ]]; then
run_checked_mode "ToolchainVerify" "" "" "toolchain_verify_pass"
[[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_verify_failed_after_reconcile"
exit 0
fi
exit 75
;;
esac

548
scripts/ops/deploy-signoz-metadata-toolchain.sh
View File

@@ -1,548 +1,8 @@
#!/usr/bin/env bash
# P0-OBS-002 - deploy an immutable, inactive SigNoz metadata toolchain to host110.
#
# The deployer never reads credentials, SQLite, or Docker volumes. It does not
# change an active pointer or invoke an authenticated metadata export. Failed
# candidates are moved into the run receipt directory rather than deleted.
# Historical compatibility entrypoint. Direct controller -> Host110 mutation is retired.
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TARGET_HOST="${SIGNOZ_METADATA_TARGET_HOST:-wooo@192.168.0.110}"
REMOTE_ROOT="${SIGNOZ_METADATA_REMOTE_ROOT:-/backup/toolchains/signoz-metadata}"
RECEIPT_ROOT="${SIGNOZ_METADATA_RECEIPT_ROOT:-/backup/deploy-receipts/signoz-metadata-toolchain}"
STAGE_ROOT="${SIGNOZ_METADATA_STAGE_ROOT:-/tmp}"
SSH_TIMEOUT_SECONDS="${SIGNOZ_METADATA_SSH_TIMEOUT_SECONDS:-120}"
SCP_TIMEOUT_SECONDS="${SIGNOZ_METADATA_SCP_TIMEOUT_SECONDS:-120}"
REMOTE_TIMEOUT_SECONDS="${SIGNOZ_METADATA_REMOTE_TIMEOUT_SECONDS:-60}"
KILL_AFTER_SECONDS="${SIGNOZ_METADATA_KILL_AFTER_SECONDS:-10}"
LOCAL_PYTHON_BIN="${SIGNOZ_METADATA_LOCAL_PYTHON_BIN:-python3.11}"
SSH_IDENTITY_FILE="${SIGNOZ_METADATA_SSH_IDENTITY_FILE:-}"
SSH_KNOWN_HOSTS_FILE="${SIGNOZ_METADATA_KNOWN_HOSTS_FILE:-}"
LABELS=(
metadata-export-policy.json
signoz_metadata_contract.py
signoz-metadata-export.py
verify-signoz-metadata-export.py
signoz-metadata-restore-drill.py
signoz-metadata-isolated-restore.py
isolated-cluster.xml
)
LOCAL_SOURCES=(
"${ROOT_DIR}/config/signoz/metadata-export-policy.json"
"${ROOT_DIR}/scripts/backup/signoz_metadata_contract.py"
"${ROOT_DIR}/scripts/backup/signoz-metadata-export.py"
"${ROOT_DIR}/scripts/backup/verify-signoz-metadata-export.py"
"${ROOT_DIR}/scripts/backup/signoz-metadata-restore-drill.py"
"${ROOT_DIR}/scripts/backup/signoz-metadata-isolated-restore.py"
"${ROOT_DIR}/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
)
MODES=(0644 0644 0755 0755 0755 0755 0644)
SSH_OPTIONS=(
-o BatchMode=yes
-o IdentitiesOnly=yes
-o StrictHostKeyChecking=yes
-o ConnectTimeout=8
-o ConnectionAttempts=1
-o ServerAliveInterval=5
-o ServerAliveCountMax=2
)
if [ -n "${SSH_IDENTITY_FILE}" ]; then
[ -f "${SSH_IDENTITY_FILE}" ] && [ ! -L "${SSH_IDENTITY_FILE}" ] || {
echo "invalid SSH identity file" >&2
exit 64
}
SSH_OPTIONS+=( -i "${SSH_IDENTITY_FILE}" )
fi
if [ -n "${SSH_KNOWN_HOSTS_FILE}" ]; then
[ -f "${SSH_KNOWN_HOSTS_FILE}" ] && [ ! -L "${SSH_KNOWN_HOSTS_FILE}" ] || {
echo "invalid SSH known_hosts file" >&2
exit 64
}
SSH_OPTIONS+=( -o "UserKnownHostsFile=${SSH_KNOWN_HOSTS_FILE}" )
fi
usage() {
cat <<'USAGE'
Usage: deploy-signoz-metadata-toolchain.sh [--check|--apply]
Required environment:
TRACE_ID Path-safe controlled-apply trace identifier
RUN_ID Unique path-safe execution identifier
WORK_ITEM_ID Must be P0-OBS-002
Optional fixed transport paths:
SIGNOZ_METADATA_SSH_IDENTITY_FILE
SIGNOZ_METADATA_KNOWN_HOSTS_FILE
--check validates the seven local sources and performs a no-write host110 diff.
--apply creates one immutable exact-hash directory. It does not create or
change an active pointer and does not run an authenticated metadata export.
USAGE
}
MODE="${1:---check}"
case "${MODE}" in
--check|--apply) ;;
-h|--help) usage; exit 0 ;;
*) usage >&2; exit 64 ;;
esac
[ "$#" -le 1 ] || { usage >&2; exit 64; }
TRACE_ID="${TRACE_ID:-}"
RUN_ID="${RUN_ID:-}"
WORK_ITEM_ID="${WORK_ITEM_ID:-}"
valid_identity() {
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
}
valid_positive_integer() {
[[ "$1" =~ ^[1-9][0-9]*$ ]]
}
safe_absolute_path() {
local path="$1"
[[ "${path}" == /* ]] || return 1
[[ "${path}" != / && "${path}" != */ ]] || return 1
[[ "${path}" =~ ^/[A-Za-z0-9._/-]+$ ]] || return 1
case "/${path#/}/" in
*'//'*|*'/../'*|*'/./'*) return 1 ;;
esac
}
valid_identity "${TRACE_ID}" || { echo "invalid TRACE_ID" >&2; exit 64; }
valid_identity "${RUN_ID}" || { echo "invalid RUN_ID" >&2; exit 64; }
[ "${WORK_ITEM_ID}" = "P0-OBS-002" ] || {
echo "WORK_ITEM_ID must be P0-OBS-002" >&2
exit 64
}
for value in "${SSH_TIMEOUT_SECONDS}" "${SCP_TIMEOUT_SECONDS}" \
"${REMOTE_TIMEOUT_SECONDS}" "${KILL_AFTER_SECONDS}"; do
valid_positive_integer "${value}" || { echo "invalid timeout" >&2; exit 64; }
done
for path in "${REMOTE_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do
safe_absolute_path "${path}" || { echo "unsafe remote path" >&2; exit 64; }
done
bounded_ssh() {
timeout --kill-after="${KILL_AFTER_SECONDS}" "${SSH_TIMEOUT_SECONDS}" ssh "$@"
}
bounded_scp() {
timeout --kill-after="${KILL_AFTER_SECONDS}" "${SCP_TIMEOUT_SECONDS}" scp "$@"
}
hash_file() {
sha256sum "$1" | awk '{print $1}'
}
emit() {
local phase="$1" terminal="$2" detail="$3"
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-metadata-toolchain","phase":"%s","terminal":"%s","detail":"%s"}\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${terminal}" "${detail}"
}
validate_local_sources() {
local source
command -v "${LOCAL_PYTHON_BIN}" >/dev/null 2>&1
"${LOCAL_PYTHON_BIN}" -c 'import sys; assert sys.version_info >= (3, 10)'
for source in "${LOCAL_SOURCES[@]}"; do
[ -f "${source}" ] && [ ! -L "${source}" ] && [ -s "${source}" ]
done
"${LOCAL_PYTHON_BIN}" - "${LOCAL_SOURCES[@]:1}" <<'PY'
from pathlib import Path
import sys
for value in sys.argv[1:]:
path = Path(value)
if path.suffix == ".py":
compile(path.read_text(encoding="utf-8"), str(path), "exec")
PY
PYTHONPATH="${ROOT_DIR}/scripts/backup" "${LOCAL_PYTHON_BIN}" - \
"${LOCAL_SOURCES[0]}" <<'PY'
from pathlib import Path
import sys
from signoz_metadata_contract import load_policy
policy = load_policy(Path(sys.argv[1]))
assert policy["work_item_id"] == "P0-OBS-002"
assert policy["source_runtime"]["raw_database_access_allowed"] is False
assert policy["source_runtime"]["raw_volume_access_allowed"] is False
assert policy["completion_contract"]["full_sqlite_completion_claim_allowed"] is False
PY
}
validate_local_sources
SOURCE_HASHES=()
SOURCE_DETAIL=""
for index in "${!LOCAL_SOURCES[@]}"; do
source_hash="$(hash_file "${LOCAL_SOURCES[index]}")"
SOURCE_HASHES+=("${source_hash}")
SOURCE_DETAIL+="${LABELS[index]}_${source_hash}_"
done
SOURCE_DETAIL="${SOURCE_DETAIL%_}"
BUNDLE_ID="$(printf '%s\n' "${SOURCE_HASHES[@]}" | sha256sum | awk '{print $1}')"
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
STAGE_DIR="${STAGE_ROOT}/awoooi-signoz-metadata-toolchain.${RUN_ID}"
emit sensor_source pass "local_exact_sources_7"
emit normalized_asset_identity pass "bundle_${BUNDLE_ID}"
emit ai_decision pass "candidate_additive_immutable_inactive_source_deploy"
emit risk_policy_decision pass "medium_no_active_pointer_no_secret_no_raw_sqlite"
remote_check() {
# The immutable root is deliberately 0700/root-owned. Run this bounded
# verifier as root so it can distinguish exact from absent without relaxing
# directory permissions. REMOTE_CHECK contains only stat/hash/process/API
# reads and performs no writes.
bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" sudo -n bash -s -- \
"${REMOTE_ROOT}" "${BUNDLE_ID}" "${REMOTE_TIMEOUT_SECONDS}" \
"${KILL_AFTER_SECONDS}" "${SOURCE_HASHES[@]}" <<'REMOTE_CHECK'
set -euo pipefail
REMOTE_ROOT="$1"
BUNDLE_ID="$2"
REMOTE_TIMEOUT_SECONDS="$3"
KILL_AFTER_SECONDS="$4"
shift 4
EXPECTED_HASHES=("$@")
LABELS=(
metadata-export-policy.json
signoz_metadata_contract.py
signoz-metadata-export.py
verify-signoz-metadata-export.py
signoz-metadata-restore-drill.py
signoz-metadata-isolated-restore.py
isolated-cluster.xml
)
MODES=(0644 0644 0755 0755 0755 0755 0644)
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
[ -d /backup ] && [ ! -L /backup ]
[ ! -e "${REMOTE_ROOT}/current" ] && [ ! -L "${REMOTE_ROOT}/current" ]
for command_name in curl docker pgrep python3 sha256sum stat timeout; do
command -v "${command_name}" >/dev/null 2>&1
done
python3 -c 'import sys; assert sys.version_info >= (3, 10)'
container_state="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
"${REMOTE_TIMEOUT_SECONDS}" docker inspect --format \
'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' signoz)"
api_status="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
"${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \
http://127.0.0.1:8080/api/v1/health)"
[ "${api_status}" = "200" ]
active_operations="$(
(pgrep -af '([s]ignoz-metadata-isolated-restore[.]py|(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$))' || true) \
| wc -l | tr -d ' '
)"
[ "${active_operations}" = "0" ]
state=absent
drift=0
if [ -e "${TARGET_DIR}" ] || [ -L "${TARGET_DIR}" ]; then
[ -d "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ] || drift=$((drift + 1))
for index in "${!LABELS[@]}"; do
target="${TARGET_DIR}/${LABELS[index]}"
if [ ! -f "${target}" ] || [ -L "${target}" ]; then
drift=$((drift + 1))
continue
fi
[ "$(sha256sum "${target}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ] || drift=$((drift + 1))
[ "0$(stat -c '%a' "${target}")" = "${MODES[index]}" ] || drift=$((drift + 1))
done
state=exact
[ "${drift}" -eq 0 ] || state=drift
fi
candidate_count=0
for candidate in "${REMOTE_ROOT}"/."${BUNDLE_ID}".candidate.*; do
[ -e "${candidate}" ] || [ -L "${candidate}" ] || continue
candidate_count=$((candidate_count + 1))
done
[ "${candidate_count}" -eq 0 ]
[ "${drift}" -eq 0 ]
printf 'REMOTE_CHECK terminal=check_pass_no_write state=%s drift=%s candidate_count=%s api=%s active_operations=%s container_state_sha256=%s\n' \
"${state}" "${drift}" "${candidate_count}" "${api_status}" "${active_operations}" \
"$(printf '%s' "${container_state}" | sha256sum | awk '{print $1}')"
REMOTE_CHECK
}
quarantine_transport_stage() {
bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" sudo -n bash -s -- \
"${STAGE_DIR}" "${RECEIPT_ROOT}" "${RUN_ID}" <<'REMOTE_QUARANTINE'
set -euo pipefail
stage_dir="$1"
receipt_root="$2"
run_id="$3"
receipt_dir="${receipt_root}/${run_id}"
if [ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then
exit 0
fi
[ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
mkdir -p "${receipt_root}"
if [ ! -e "${receipt_dir}" ] && [ ! -L "${receipt_dir}" ]; then
mkdir -m 0700 "${receipt_dir}"
fi
[ -d "${receipt_dir}" ] && [ ! -L "${receipt_dir}" ]
[ ! -e "${receipt_dir}/transport-stage" ] && [ ! -L "${receipt_dir}/transport-stage" ]
mv "${stage_dir}" "${receipt_dir}/transport-stage"
REMOTE_QUARANTINE
}
if ! REMOTE_CHECK_OUTPUT="$(remote_check)"; then
emit source_of_truth_diff failed "remote_read_only_check_failed"
emit terminal failed_no_write "remote_check_failed"
exit 1
fi
case "${REMOTE_CHECK_OUTPUT}" in
*"terminal=check_pass_no_write state=absent "*) REMOTE_STATE=absent ;;
*"terminal=check_pass_no_write state=exact "*) REMOTE_STATE=exact ;;
*)
emit source_of_truth_diff failed "remote_check_receipt_invalid"
emit terminal failed_no_write "remote_check_receipt_invalid"
exit 1
;;
esac
emit source_of_truth_diff pass "remote_state_${REMOTE_STATE}_drift_0"
if [ "${MODE}" = "--check" ]; then
emit check_mode check_pass_no_write "bundle_${BUNDLE_ID}_remote_state_${REMOTE_STATE}"
emit rollback no_write "active_pointer_unchanged"
emit terminal check_pass_no_write "source_ready_remote_state_${REMOTE_STATE}"
exit 0
fi
if [ "${REMOTE_STATE}" = "exact" ]; then
emit check_mode pass "immutable_bundle_already_exact"
emit bounded_execution idempotent_no_write "bundle_${BUNDLE_ID}"
emit independent_post_verifier pass "remote_exact_hash_mode_api_identity"
emit rollback no_write "active_pointer_absent"
emit learning_writeback partial_degraded "runtime_export_not_executed"
emit terminal pass_source_deployed_inactive "idempotent_runtime_export_pending"
exit 0
fi
if ! bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" bash -s -- \
"${STAGE_DIR}" <<'REMOTE_STAGE'; then
set -euo pipefail
stage_dir="$1"
[ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
stage_root="${stage_dir%/*}"
[ -d "${stage_root}" ] && [ ! -L "${stage_root}" ] && [ -w "${stage_root}" ]
umask 077
mkdir -m 0700 "${stage_dir}"
REMOTE_STAGE
emit bounded_execution failed "transport_stage_create_failed"
emit terminal failed_no_active_pointer_write "stage_create_failed"
exit 1
fi
for index in "${!LOCAL_SOURCES[@]}"; do
if ! bounded_scp -q "${SSH_OPTIONS[@]}" "${LOCAL_SOURCES[index]}" \
"${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}"; then
quarantine_transport_stage >/dev/null 2>&1 || true
emit bounded_execution failed "transport_failed_index_${index}"
emit terminal failed_no_active_pointer_write "transport_stage_quarantine_attempted"
exit 1
fi
done
if ! REMOTE_APPLY_OUTPUT="$(bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" \
sudo -n bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
"${REMOTE_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" "${BUNDLE_ID}" \
"${REMOTE_TIMEOUT_SECONDS}" "${KILL_AFTER_SECONDS}" \
"${SOURCE_HASHES[@]}" <<'REMOTE_APPLY'
set -euo pipefail
TRACE_ID="$1"
RUN_ID="$2"
WORK_ITEM_ID="$3"
REMOTE_ROOT="$4"
RECEIPT_ROOT="$5"
STAGE_DIR="$6"
BUNDLE_ID="$7"
REMOTE_TIMEOUT_SECONDS="$8"
KILL_AFTER_SECONDS="$9"
shift 9
EXPECTED_HASHES=("$@")
LABELS=(
metadata-export-policy.json
signoz_metadata_contract.py
signoz-metadata-export.py
verify-signoz-metadata-export.py
signoz-metadata-restore-drill.py
signoz-metadata-isolated-restore.py
isolated-cluster.xml
)
MODES=(0644 0644 0755 0755 0755 0755 0644)
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
CANDIDATE_DIR="${REMOTE_ROOT}/.${BUNDLE_ID}.candidate.${RUN_ID}"
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
DEPLOYED=0
TERMINAL=failed
emit_remote() {
local phase="$1" terminal="$2" detail="$3"
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-metadata-toolchain","phase":"%s","terminal":"%s","detail":"%s"}\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${terminal}" "${detail}" | tee -a "${RECEIPT_LOG}"
}
finalize() {
local rc=$? quarantine_terminal=not_required
trap - EXIT HUP INT TERM
set +e
if [ -d "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ]; then
if [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]; then
mv "${STAGE_DIR}" "${RECEIPT_DIR}/transport-stage"
fi
if [ "${rc}" -ne 0 ]; then
quarantine_terminal=inactive_candidates_preserved
if [ -d "${CANDIDATE_DIR}" ] && [ ! -L "${CANDIDATE_DIR}" ]; then
mv "${CANDIDATE_DIR}" "${RECEIPT_DIR}/quarantine-candidate"
fi
if [ "${DEPLOYED}" -eq 1 ] && [ -d "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ]; then
mv "${TARGET_DIR}" "${RECEIPT_DIR}/quarantine-target"
fi
fi
emit_remote rollback "${quarantine_terminal}" "active_pointer_never_created"
emit_remote terminal "${TERMINAL}" "exit_${rc}_runtime_export_not_executed"
fi
exit "${rc}"
}
trap finalize EXIT HUP INT TERM
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
[ ! -e "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ]
[ ! -e "${CANDIDATE_DIR}" ] && [ ! -L "${CANDIDATE_DIR}" ]
[ ! -e "${REMOTE_ROOT}/current" ] && [ ! -L "${REMOTE_ROOT}/current" ]
[ ! -e "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ]
python3 -c 'import sys; assert sys.version_info >= (3, 10)'
umask 077
mkdir -p "${REMOTE_ROOT}" "${RECEIPT_ROOT}"
mkdir -m 0700 "${RECEIPT_DIR}" "${CANDIDATE_DIR}"
: > "${RECEIPT_LOG}"
chmod 0600 "${RECEIPT_LOG}"
OPERATION_LOCK=/tmp/awoooi-signoz-backup-operation.lock
[ -f "${OPERATION_LOCK}" ] && [ ! -L "${OPERATION_LOCK}" ]
# The shared lock is intentionally owned by the unprivileged backup account.
# Open the existing inode read-only so Linux fs.protected_regular=2 cannot
# reject a root O_CREAT/O_APPEND open in sticky /tmp. flock(2) still provides
# the same exclusive inter-process lock without changing file contents,
# ownership, or mode. Missing/unsafe lock state fails closed.
exec 8<"${OPERATION_LOCK}"
flock -n 8
active_operations="$(
(pgrep -af '([s]ignoz-metadata-isolated-restore[.]py|(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$))' || true) \
| wc -l | tr -d ' '
)"
[ "${active_operations}" = "0" ]
runtime_before="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
"${REMOTE_TIMEOUT_SECONDS}" docker inspect --format \
'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' signoz)"
api_before="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
"${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \
http://127.0.0.1:8080/api/v1/health)"
[ "${api_before}" = "200" ]
emit_remote sensor_source pass "staged_sources_7_runtime_api_200"
for index in "${!LABELS[@]}"; do
staged="${STAGE_DIR}/${LABELS[index]}"
[ -f "${staged}" ] && [ ! -L "${staged}" ]
[ "$(sha256sum "${staged}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ]
install -o root -g root -m "${MODES[index]}" "${staged}" \
"${CANDIDATE_DIR}/${LABELS[index]}"
done
python3 - "${CANDIDATE_DIR}" <<'PY'
from pathlib import Path
import sys
root = Path(sys.argv[1])
for name in (
"signoz_metadata_contract.py",
"signoz-metadata-export.py",
"verify-signoz-metadata-export.py",
"signoz-metadata-restore-drill.py",
"signoz-metadata-isolated-restore.py",
):
path = root / name
compile(path.read_text(encoding="utf-8"), str(path), "exec")
PY
PYTHONPATH="${CANDIDATE_DIR}" python3 "${CANDIDATE_DIR}/signoz-metadata-export.py" \
--check --base-url https://signoz.invalid \
--output-dir "${RECEIPT_DIR}/offline-output-must-not-exist" \
--policy "${CANDIDATE_DIR}/metadata-export-policy.json" \
--trace-id "${TRACE_ID}" --run-id "${RUN_ID}" \
--work-item-id "${WORK_ITEM_ID}" >/dev/null
[ ! -e "${RECEIPT_DIR}/offline-output-must-not-exist" ]
emit_remote check_mode pass "candidate_hash_mode_compile_policy_check"
mv "${CANDIDATE_DIR}" "${TARGET_DIR}"
chmod 0750 "${TARGET_DIR}"
DEPLOYED=1
for index in "${!LABELS[@]}"; do
target="${TARGET_DIR}/${LABELS[index]}"
[ -f "${target}" ] && [ ! -L "${target}" ]
[ "$(sha256sum "${target}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ]
[ "0$(stat -c '%a' "${target}")" = "${MODES[index]}" ]
printf '%s\t%s\t%s\n' "${LABELS[index]}" "${EXPECTED_HASHES[index]}" "${MODES[index]}" \
>> "${RECEIPT_DIR}/deployed-manifest.tsv"
done
chmod 0600 "${RECEIPT_DIR}/deployed-manifest.tsv"
runtime_after="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
"${REMOTE_TIMEOUT_SECONDS}" docker inspect --format \
'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' signoz)"
api_after="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
"${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \
http://127.0.0.1:8080/api/v1/health)"
[ "${runtime_after}" = "${runtime_before}" ]
[ "${api_after}" = "200" ]
emit_remote bounded_execution pass "immutable_bundle_deployed_active_pointer_0"
emit_remote independent_post_verifier pass "hash_mode_compile_api_identity_unchanged"
emit_remote learning_writeback partial_degraded "authenticated_export_restore_pending"
TERMINAL=pass_source_deployed_inactive
REMOTE_APPLY
)"; then
quarantine_transport_stage >/dev/null 2>&1 || true
emit bounded_execution failed "remote_apply_failed_inactive_candidate_quarantined"
emit terminal failed_no_active_pointer_write "runtime_export_not_executed"
exit 1
fi
case "${REMOTE_APPLY_OUTPUT}" in
*'"phase":"terminal","terminal":"pass_source_deployed_inactive"'*) ;;
*)
emit independent_post_verifier failed "remote_apply_terminal_invalid"
emit terminal failed_no_active_pointer_write "runtime_export_not_executed"
exit 1
;;
esac
if ! POSTCHECK_OUTPUT="$(remote_check)"; then
emit independent_post_verifier failed "remote_postcheck_failed"
emit terminal failed_source_deploy_unverified "active_pointer_0"
exit 1
fi
case "${POSTCHECK_OUTPUT}" in
*"terminal=check_pass_no_write state=exact "*) ;;
*)
emit independent_post_verifier failed "remote_postcheck_receipt_invalid"
emit terminal failed_source_deploy_unverified "active_pointer_0"
exit 1
;;
esac
emit check_mode pass "precheck_absent"
emit bounded_execution pass "immutable_bundle_${BUNDLE_ID}_active_pointer_0"
emit independent_post_verifier pass "postcheck_exact_hash_mode_api_identity"
emit rollback not_required "inactive_additive_bundle"
emit learning_writeback partial_degraded "runtime_export_restore_zero_residue_pending"
emit terminal pass_source_deployed_inactive "runtime_export_not_executed"
printf '%s\n' \
'{"schema":"awoooi_signoz_metadata_toolchain_route_retirement_v1","terminal":"blocked_direct_host110_route_retired","replacement":"scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh","production_mutation_performed":false,"completion_claim":false}'
exit 78

2
scripts/ops/tests/test_agent99_signoz_credential_provisioner.py
View File

@@ -149,7 +149,7 @@ def test_agent99_entrypoint_is_fixed_and_never_transports_secret() -> None:
assert '[ValidateSet("Check", "Apply", "Verify")]' in source
assert '$TargetHost = "192.168.0.110"' in source
assert '$ExpectedWorkItemId = "P0-OBS-002"' in source
assert '$ExpectedRuntimeFileCount = 20' in source
assert '$ExpectedRuntimeFileCount = 21' in source
assert '"sudo -n /usr/bin/python3 - --mode $($Mode.ToLowerInvariant())"' in source
assert "$process.Refresh()" in source
assert 'if (-not $process.HasExited) { throw "target_exit_state_unavailable" }' in source

265
scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py
View File

@@ -4,7 +4,6 @@ import hashlib
import re
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
ENTRYPOINT = ROOT / "agent99-signoz-metadata-executor.ps1"
@@ -20,7 +19,8 @@ def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None:
assert (
'[ValidateSet("Check", "Apply", "Verify", "RestoreCheck", '
'"RestoreApply", "RestoreVerifyCleanup")]' in source
'"RestoreApply", "RestoreVerifyCleanup", "ToolchainCheck", '
'"ToolchainApply", "ToolchainVerify", "ToolchainReconcile")]' in source
)
assert '$CanonicalAsset = "signoz-110-control-plane-metadata"' in source
assert '$TargetHost = "192.168.0.110"' in source
@@ -28,24 +28,26 @@ def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None:
assert '$WorkItemId -ne "P0-OBS-002"' in source
assert '$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"' in source
assert '$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"' in source
assert '$EvidenceRunId = if ($RunId -match $SafeIdPattern)' in source
assert "$EvidenceRunId = if ($RunId -match $SafeIdPattern)" in source
assert "Get-Agent99RuntimeSourceBinding" in source
assert "$result.fileCount -eq 20" in source
assert "$result.fileCount -eq 21" in source
assert "$result.sourceRevision -eq $SourceRevision" in source
assert "$result.entrypointHashMatched" in source
assert '$CredentialFile = "/etc/awoooi/secrets/signoz-metadata-api-key"' in source
assert (
'$BundleId = "ed4e5fbfc5f732b64d6a78f6ab26b90118e1d42a4f38b40d583876445c32e3aa"'
'$BundleId = "b83bfc898cd43cf8ec5688d75ec20231c2faa32de5be3fa886db33038d620dff"'
in source
)
assert re.search(r'\$BundleId = "[0-9a-f]{64}"', source)
assert '$IsolatedRestorePath = "$ToolchainRoot/signoz-metadata-isolated-restore.py"' in source
assert '$IsolatedClusterConfigPath = "$ToolchainRoot/isolated-cluster.xml"' in source
assert '$BaseUrl = "http://127.0.0.1:8080"' in source
assert (
'$OutputDir = "/var/backups/awoooi/signoz-metadata-export-$RunId"'
'$IsolatedRestorePath = "$ToolchainRoot/signoz-metadata-isolated-restore.py"'
in source
)
assert (
'$IsolatedClusterConfigPath = "$ToolchainRoot/isolated-cluster.xml"' in source
)
assert '$BaseUrl = "http://127.0.0.1:8080"' in source
assert '$OutputDir = "/var/backups/awoooi/signoz-metadata-export-$RunId"' in source
assert '$OutputDir = "/backup/signoz-metadata-export-$RunId"' not in source
assert '$SignozImage = "signoz/signoz:v0.113.0"' in source
assert '$SignozContainer = "signoz"' in source
@@ -63,11 +65,13 @@ def test_entrypoint_pins_current_seven_file_toolchain_bundle() -> None:
("ExporterPath", ROOT / "scripts/backup/signoz-metadata-export.py"),
("VerifierPath", ROOT / "scripts/backup/verify-signoz-metadata-export.py"),
("RestoreDriverPath", ROOT / "scripts/backup/signoz-metadata-restore-drill.py"),
("IsolatedRestorePath", ROOT / "scripts/backup/signoz-metadata-isolated-restore.py"),
(
"IsolatedRestorePath",
ROOT / "scripts/backup/signoz-metadata-isolated-restore.py",
),
(
"IsolatedClusterConfigPath",
ROOT
/ "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
ROOT / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
),
)
hashes = [hashlib.sha256(path.read_bytes()).hexdigest() for _, path in rows]
@@ -86,8 +90,9 @@ def test_entrypoint_pins_current_seven_file_toolchain_bundle() -> None:
def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() -> None:
source = ENTRYPOINT.read_text(encoding="utf-8")
lock_helper = source[
source.index("function New-Agent99SignozLockedRemoteCommand") :
source.index("function Get-Agent99RuntimeSourceBinding")
source.index("function New-Agent99SignozLockedRemoteCommand") : source.index(
"function Get-Agent99RuntimeSourceBinding"
)
]
assert 'Start-Process -FilePath "ssh.exe"' in source
@@ -95,9 +100,9 @@ def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() ->
assert '"-o", "IdentitiesOnly=yes"' in source
assert '"-o", "StrictHostKeyChecking=yes"' in source
assert '"-o", "NumberOfPasswordPrompts=0"' in source
assert 'windows99_agent99_to_host110_fixed_metadata_toolchain' in source
assert 'arbitraryCommandAllowed = $false' in source
assert 'crossDomainFallbackAllowed = $false' in source
assert "windows99_agent99_to_host110_fixed_metadata_toolchain" in source
assert "arbitraryCommandAllowed = $false" in source
assert "crossDomainFallbackAllowed = $false" in source
assert "Invoke-Expression" not in source
assert "-EncodedCommand" not in source
assert "cmd.exe" not in source
@@ -128,9 +133,9 @@ def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() ->
)
assert "targetTimeoutFinishesBeforeControllerWait" in source
assert "target_timeout_not_bounded_before_controller_wait" in source
assert '--kill-after=5s 15s /usr/bin/docker inspect' in source
assert '--kill-after=5s 15s /usr/bin/curl' in source
assert '--kill-after=5s 10s /usr/bin/pgrep' in lock_helper
assert "--kill-after=5s 15s /usr/bin/docker inspect" in source
assert "--kill-after=5s 15s /usr/bin/curl" in source
assert "--kill-after=5s 10s /usr/bin/pgrep" in lock_helper
assert "active_rc=`$?; set -e" in lock_helper
assert '[ `"`$active_rc`" -eq 1 ] && [ -z `"`$active_output`" ]' in lock_helper
assert "$bashProgram = (" in lock_helper
@@ -142,49 +147,52 @@ def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() ->
assert "sudo -n /bin/bash -c '" not in lock_helper
assert "pgrep -af" in lock_helper
assert "|| true" not in lock_helper
assert '--signal=TERM --kill-after=$($TargetKillAfterSeconds)s' in source
assert "--signal=TERM --kill-after=$($TargetKillAfterSeconds)s" in source
def test_entrypoint_uses_typed_stdout_without_windows_process_exit_code() -> None:
source = ENTRYPOINT.read_text(encoding="utf-8")
transport = source[
source.index("function Invoke-Agent99SignozFixedRemote") :
source.index("function New-Agent99SignozLockedRemoteCommand")
source.index("function Invoke-Agent99SignozFixedRemote") : source.index(
"function Test-Agent99SignozTransportOutcome"
)
]
assert "$process.ExitCode" not in transport
assert "exitCodeObserved" not in transport
assert "AllowedExitCodes" not in transport
assert "function Test-Agent99SignozTransportOutcome" in transport
assert "$Result.completed -eq $true" in transport
assert "-not $Result.timedOut" in transport
assert '$process.WaitForExit($TimeoutSeconds * 1000)' in transport
assert "function Test-Agent99SignozTransportOutcome" in source
assert "$Result.completed -eq $true" in source
assert "-not $Result.timedOut" in source
assert "$process.WaitForExit($TimeoutSeconds * 1000)" in transport
assert "taskkill.exe /PID $process.Id /T /F" in transport
def test_entrypoint_preflight_has_explicit_credential_and_artifact_state_contracts() -> None:
def test_entrypoint_preflight_has_explicit_credential_and_artifact_state_contracts() -> (
None
):
source = ENTRYPOINT.read_text(encoding="utf-8")
assert 'function New-Agent99SignozPathStateCommand' in source
assert 'agent99_credential:symlink' in source
assert 'agent99_credential:absent' in source
assert 'agent99_credential:present:%u:%a:%s' in source
assert 'agent99_path_state:$ExpectedKind' in source
assert 'agent99_path_state:absent' in source
assert 'agent99_hashes_begin' in source
assert 'agent99_hashes_end' in source
assert '/usr/bin/echo agent99_hashes_begin' in source
assert '/usr/bin/echo agent99_hashes_end' in source
assert 'agent99_hashes_begin\\n' not in source
assert 'agent99_image:' in source
assert 'agent99_runtime:' in source
assert 'agent99_health:' in source
assert 'agent99_active:' in source
assert "function New-Agent99SignozPathStateCommand" in source
assert "agent99_credential:symlink" in source
assert "agent99_credential:absent" in source
assert "agent99_credential:present:%u:%a:%s" in source
assert "agent99_path_state:$ExpectedKind" in source
assert "agent99_path_state:absent" in source
assert "agent99_hashes_begin" in source
assert "agent99_hashes_end" in source
assert "/usr/bin/echo agent99_hashes_begin" in source
assert "/usr/bin/echo agent99_hashes_end" in source
assert "agent99_hashes_begin\\n" not in source
assert "agent99_image:" in source
assert "agent99_runtime:" in source
assert "agent99_health:" in source
assert "agent99_active:" in source
assert '$credentialState -in @("absent", "symlink", "other")' in source
assert 'credentialReadbackVerified = $credentialReadbackVerified' in source
assert 'transportReadbackVerified = $transportReadbackVerified' in source
assert 'outputStateReadbackVerified = $outputStateReadbackVerified' in source
assert 'receiptStateReadbackVerified = $receiptStateReadbackVerified' in source
assert "credentialReadbackVerified = $credentialReadbackVerified" in source
assert "transportReadbackVerified = $transportReadbackVerified" in source
assert "outputStateReadbackVerified = $outputStateReadbackVerified" in source
assert "receiptStateReadbackVerified = $receiptStateReadbackVerified" in source
assert '"credential_reference_readback_unverified"' in source
assert '"preflight_transport_unverified"' in source
assert source.count("Test-Agent99SignozTransportOutcome $verify") == 2
@@ -195,24 +203,29 @@ def test_entrypoint_preflight_has_explicit_credential_and_artifact_state_contrac
def test_entrypoint_typed_readbacks_are_exact_and_preserve_unknown_state() -> None:
source = ENTRYPOINT.read_text(encoding="utf-8")
preflight = source[
source.index("function Get-Agent99SignozPreflight") :
source.index("function Write-Agent99SignozEvidenceCollision")
source.index("function Get-Agent99SignozPreflight") : source.index(
"function Write-Agent99SignozEvidenceCollision"
)
]
main = source[
source.index(
'$preflight = Get-Agent99SignozPreflight '
'-ExpectedArtifactState "absent"'
"$preflight = Get-Agent99SignozPreflight " '-ExpectedArtifactState "absent"'
) :
]
assert "$hashLines.Count -eq $ToolchainFiles.Count + 2" in preflight
assert "$hashLines[0] -eq \"agent99_hashes_begin\"" in preflight
assert '$hashLines[0] -eq "agent99_hashes_begin"' in preflight
assert '$hashLines[$hashLines.Count - 1] -eq "agent99_hashes_end"' in preflight
assert "$actualHashes.ContainsKey($path)" in preflight
assert "-not $ToolchainFiles.Contains($path)" in preflight
assert '"^agent99_image:(sha256:[0-9a-f]{64})$"' in preflight
assert '--format={{.Id}},{{.Image}},{{.Config.Image}},{{.State.Running}}' in preflight
assert '--format=`"{{.Id}}|{{.Image}}|{{.Config.Image}}|{{.State.Running}}`"' not in preflight
assert (
"--format={{.Id}},{{.Image}},{{.Config.Image}},{{.State.Running}}" in preflight
)
assert (
'--format=`"{{.Id}}|{{.Image}}|{{.Config.Image}}|{{.State.Running}}`"'
not in preflight
)
assert '"^agent99_runtime:([0-9a-f]{64}),(sha256:' in preflight
assert '"^agent99_active:([0-9]+)$"' in preflight
assert '[ `"`$active_rc`" -eq 0 ] || [ `"`$active_rc`" -eq 1 ]' in preflight
@@ -258,69 +271,72 @@ def test_entrypoint_typed_readbacks_are_exact_and_preserve_unknown_state() -> No
def test_entrypoint_never_reads_or_persists_credential_value_in_controller() -> None:
source = ENTRYPOINT.read_text(encoding="utf-8")
assert '/usr/bin/stat -c `"agent99_credential:present:%u:%a:%s`" `"$CredentialFile`"' in source
assert 'credentialValueRead = $false' in source
assert 'credentialValueOutput = $false' in source
assert 'secretValueReadByController = $false' in source
assert 'secretValueTransmitted = $false' in source
assert 'secretValuePersistedInEvidence = $false' in source
assert (
'/usr/bin/stat -c `"agent99_credential:present:%u:%a:%s`" `"$CredentialFile`"'
in source
)
assert "credentialValueRead = $false" in source
assert "credentialValueOutput = $false" in source
assert "secretValueReadByController = $false" in source
assert "secretValueTransmitted = $false" in source
assert "secretValuePersistedInEvidence = $false" in source
assert "credentialReferenceDispatchAttempted" in source
assert "$script:ExportCredentialDispatchAttempted = $true" in source
assert "$script:IsolatedApplyDispatchAttempted = $true" in source
assert "credentialReferenceDispatchAttempted = $script:ExportCredentialDispatchAttempted" in source
assert (
"credentialReferenceDispatchAttempted = $script:ExportCredentialDispatchAttempted"
in source
)
assert "credentialReferenceConsumedOnTarget = if" in source
assert "elseif (Test-Agent99SignozReceiptIdentity $ApplyReceipt" in source
assert 'rawSqliteRead = $false' in source
assert 'rawVolumeRead = $false' in source
assert "rawSqliteRead = $false" in source
assert "rawVolumeRead = $false" in source
assert "Get-Content -LiteralPath $CredentialFile" not in source
assert "ReadAllText($CredentialFile" not in source
assert "ReadAllBytes($CredentialFile" not in source
def test_entrypoint_requires_check_apply_independent_verify_and_partial_terminal() -> None:
def test_entrypoint_requires_check_apply_independent_verify_and_partial_terminal() -> (
None
):
source = ENTRYPOINT.read_text(encoding="utf-8")
assert 'Receipt.schema -eq "awoooi_signoz_metadata_receipt_v1"' in source
assert "awoooi_controlled_apply_receipt_v1" not in source
assert '[string]$Receipt.phase -eq $ExpectedPhase' in source
assert "[string]$Receipt.phase -eq $ExpectedPhase" in source
assert (
'Test-Agent99SignozReceiptIdentity $verifyReceipt '
"Test-Agent99SignozReceiptIdentity $verifyReceipt "
'"independent_export_verifier"' in source
)
assert (
'Test-Agent99SignozReceiptIdentity $checkReceipt "terminal"' in source
)
assert (
'Test-Agent99SignozReceiptIdentity $applyReceipt "terminal"' in source
)
assert 'Test-Agent99SignozReceiptIdentity $checkReceipt "terminal"' in source
assert 'Test-Agent99SignozReceiptIdentity $applyReceipt "terminal"' in source
assert "$ExporterPath --check" in source
assert "$ExporterPath --apply" in source
assert '--receipt-file $RemoteReceipt' in source
assert 'pass_export_verified_restore_pending' in source
assert 'export_verified_restore_pending' in source
assert "--receipt-file $RemoteReceipt" in source
assert "pass_export_verified_restore_pending" in source
assert "export_verified_restore_pending" in source
assert 'else { "pending_isolated_same_version_target" }' in source
assert 'else { "pending" }' in source
assert 'completionClaim = $false' in source
assert "completionClaim = $false" in source
assert "controlledApplyPhases = [pscustomobject]$controlledApplyPhases" in source
assert 'credential_reference_absent' in source
assert 'blocked_with_safe_next_action' in source
assert 'apply_failed_artifact_preserved_for_readback' in source
assert 'post_verifier_runtime_or_artifact_drift' in source
assert "credential_reference_absent" in source
assert "blocked_with_safe_next_action" in source
assert "apply_failed_artifact_preserved_for_readback" in source
assert "post_verifier_runtime_or_artifact_drift" in source
def test_restore_modes_are_bounded_distinct_and_cleanup_remains_available() -> None:
source = ENTRYPOINT.read_text(encoding="utf-8")
main_start = source.index("foreach ($identity")
cleanup_start = source.index(
'if ($Mode -eq "RestoreVerifyCleanup")', main_start
)
cleanup_start = source.index('if ($Mode -eq "RestoreVerifyCleanup")', main_start)
restore_apply_start = source.index(
'if ($Mode -in @("RestoreCheck", "RestoreApply"))', cleanup_start
)
export_verify_start = source.index('if ($Mode -eq "Verify")', restore_apply_start)
cleanup_helper = source[
source.index("function New-Agent99SignozCleanupLockedRemoteCommand") :
source.index("function Get-Agent99RuntimeSourceBinding")
source.index(
"function New-Agent99SignozCleanupLockedRemoteCommand"
) : source.index("function Get-Agent99RuntimeSourceBinding")
]
cleanup_branch = source[cleanup_start:restore_apply_start]
restore_apply_branch = source[restore_apply_start:export_verify_start]
@@ -328,18 +344,82 @@ def test_restore_modes_are_bounded_distinct_and_cleanup_remains_available() -> N
assert "Get-Agent99SignozCleanupPreflight" in cleanup_branch
assert "New-Agent99SignozCleanupLockedRemoteCommand" in cleanup_branch
assert "Get-Agent99SignozPreflight" not in cleanup_branch
assert "$OutputDir" not in cleanup_branch
assert "--bundle-dir $OutputDir" in cleanup_branch
assert "$BaseUrl/api/v1/health" not in cleanup_helper
assert "ExpectedArtifactState" not in cleanup_helper
assert "/usr/bin/docker info" in cleanup_helper
assert "--apply-receipt-file $RestoreRemoteReceipt" in cleanup_branch
assert '"independent_restore_verifier"' in cleanup_branch
assert '"portable_metadata_restore_independently_verified_zero_residue"' in cleanup_branch
assert '"portable_metadata_restore_verified_zero_residue_pending_independent_verifier"' in restore_apply_branch
assert (
'"portable_metadata_restore_independently_verified_zero_residue"'
in cleanup_branch
)
assert (
'"portable_metadata_restore_verified_zero_residue_pending_independent_verifier"'
in restore_apply_branch
)
assert cleanup_start < restore_apply_start < export_verify_start
assert "$RestoreCleanupKillAfterSeconds = 600" in source
assert "$RestoreApplyControllerWaitSeconds = 2550" in source
assert "$RestoreCleanupControllerWaitSeconds = 1050" in source
assert '[string]$AttemptId = ""' in source
assert '"$EvidenceRunId-attempt-$EvidenceAttemptId"' in source
assert 'cleanupAttemptId = if ($Mode -eq "RestoreVerifyCleanup")' in source
assert "cleanup_attempt_id_missing_or_invalid" in source
assert "attempt_id_not_allowed_for_mode" in source
def test_toolchain_modes_use_separate_reconcile_attempt_and_never_fall_through() -> (
None
):
source = ENTRYPOINT.read_text(encoding="utf-8")
main_start = source.index("foreach ($identity")
reconcile_start = source.index('if ($Mode -eq "ToolchainReconcile")', main_start)
toolchain_start = source.index(
'if ($Mode -in @("ToolchainCheck", "ToolchainApply", "ToolchainVerify"))',
reconcile_start,
)
restore_start = source.index(
'if ($Mode -eq "RestoreVerifyCleanup")', toolchain_start
)
reconcile_branch = source[reconcile_start:toolchain_start]
toolchain_branch = source[toolchain_start:restore_start]
assert '"$EvidenceRunId-attempt-$EvidenceAttemptId"' in source
assert 'toolchainReconcileAttemptId = if ($Mode -eq "ToolchainReconcile")' in source
assert 'Invoke-Agent99SignozToolchainTarget "reconcile"' in reconcile_branch
assert 'Invoke-Agent99SignozToolchainTarget "reconcile"' not in toolchain_branch
assert '"toolchain_reconcile_exact_deployment_verified"' in reconcile_branch
assert '"toolchain_reconcile_zero_owned_residue"' in reconcile_branch
assert "_toolchain_reconcile_required" in toolchain_branch
assert "Get-Agent99SignozPreflight" not in reconcile_branch
assert "New-Agent99SignozLockedRemoteCommand" not in reconcile_branch
assert "Get-Agent99SignozPreflight" not in toolchain_branch
assert "$ToolchainEnvelopeDeletedBeforeExecutor" in source
assert "toolchainEnvelopeTransportPayloadDeletedBeforeExecutor" in source
assert "productionSourceMutationState" in source
def test_restore_check_failure_terminals_preserve_no_write_semantics() -> None:
source = ENTRYPOINT.read_text(encoding="utf-8")
cleanup_start = source.index(
'if ($Mode -eq "RestoreVerifyCleanup")', source.index("foreach ($identity")
)
restore_start = source.index(
'if ($Mode -in @("RestoreCheck", "RestoreApply"))', cleanup_start
)
restore_end = source.index('if ($Mode -eq "Verify")', restore_start)
restore_branch = source[restore_start:restore_end]
assert restore_branch.count('"restore_check_failed_no_write"') >= 4
assert (
'"restore_check_failed_unexpected_write_requires_reconcile"' in restore_branch
)
assert '"restore_check_wrote_receipt"' in restore_branch
assert (
'if ($Terminal -eq "restore_check_failed_unexpected_write_requires_reconcile") '
'{ "unexpected_receipt_write_requires_reconcile" }' in source
)
def test_entrypoint_reserves_durable_evidence_before_any_remote_dispatch() -> None:
@@ -349,7 +429,7 @@ def test_entrypoint_reserves_durable_evidence_before_any_remote_dispatch() -> No
reservation = main.index("New-Agent99SignozEvidenceReservation")
first_remote = main.index("Get-Agent99RuntimeSourceBinding")
assert reservation < first_remote
assert '[IO.FileMode]::CreateNew' in source
assert "[IO.FileMode]::CreateNew" in source
assert '$EvidenceReservationPath = "$EvidencePath.reserve"' in source
assert "blocked_evidence_identity_collision" in source
assert "evidence_path_or_reservation_already_exists" in source
@@ -379,8 +459,7 @@ def test_entrypoint_is_in_exact_agent99_runtime_bundle() -> None:
ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"
).read_text(encoding="utf-8")
receiver = (
ROOT
/ "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
).read_text(encoding="utf-8")
assert expected in _quoted_array(bootstrap, "$agentFiles = @(", ")")
@@ -388,8 +467,8 @@ def test_entrypoint_is_in_exact_agent99_runtime_bundle() -> None:
assert expected in _quoted_array(contract, "$requiredFiles = @(", ")")
assert expected in _quoted_array(sender, "RUNTIME_FILES=(", ")")
assert expected in _quoted_array(receiver, "$FixedRuntimeFiles = @(", ")")
assert 'expectedRuntimeFileCount": 20' in sender
assert '$runtimeManifest.fileCount -eq 20' in receiver
assert 'expectedRuntimeFileCount": 21' in sender
assert "$runtimeManifest.fileCount -eq 21" in receiver
def test_runtime_contract_check_requires_typed_signoz_transport_readback() -> None:

357
scripts/ops/tests/test_agent99_signoz_toolchain_target.py Normal file
View File

@@ -0,0 +1,357 @@
from __future__ import annotations
import hashlib
import importlib.util
import json
import os
import stat
from pathlib import Path
from types import ModuleType, SimpleNamespace
import pytest
ROOT = Path(__file__).resolve().parents[3]
TARGET = ROOT / "agent99-signoz-toolchain-target.py"
def _load_target() -> ModuleType:
spec = importlib.util.spec_from_file_location(
"agent99_signoz_toolchain_target", TARGET
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _payloads() -> dict[str, bytes]:
policy = {
"work_item_id": "P0-OBS-002",
"source_runtime": {
"raw_database_access_allowed": False,
"raw_volume_access_allowed": False,
},
"completion_contract": {"full_sqlite_completion_claim_allowed": False},
}
values: dict[str, bytes] = {
"metadata-export-policy.json": (
json.dumps(policy, sort_keys=True, separators=(",", ":")) + "\n"
).encode(),
"isolated-cluster.xml": b"<clickhouse><remote_servers/></clickhouse>\n",
}
for name in (
"signoz_metadata_contract.py",
"signoz-metadata-export.py",
"verify-signoz-metadata-export.py",
"signoz-metadata-restore-drill.py",
"signoz-metadata-isolated-restore.py",
):
values[name] = b"VALUE = 'bounded-toolchain-test'\n"
return values
def _manifest(module: ModuleType, values: dict[str, bytes]) -> list[dict[str, str]]:
return [
{
"name": name,
"sha256": hashlib.sha256(values[name]).hexdigest(),
"mode": f"{mode:04o}",
}
for name, mode in zip(module.LABELS, module.MODES, strict=True)
]
def _args(
module: ModuleType,
manifest: list[dict[str, str]],
*,
mode: str,
run_id: str = "run-001",
attempt_id: str = "",
stage: Path | None = None,
) -> SimpleNamespace:
bundle_id = module.sha256_bytes(
"".join(f"{row['sha256']}\n" for row in manifest).encode("ascii")
)
return SimpleNamespace(
mode=mode,
trace_id="trace-001",
run_id=run_id,
work_item_id="P0-OBS-002",
source_revision="1" * 40,
bundle_id=bundle_id,
stage_dir=str(stage) if stage else None,
attempt_id=attempt_id,
)
@pytest.fixture
def target_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
module = _load_target()
uid = os.getuid()
gid = os.getgid()
remote_parent = tmp_path / "backup" / "toolchains"
receipt_parent = tmp_path / "backup" / "deploy-receipts"
remote_parent.mkdir(parents=True)
receipt_parent.mkdir(parents=True)
lock = tmp_path / "operation.lock"
lock.write_bytes(b"")
os.chmod(lock, 0o600)
snapshot = {
"id": "a" * 64,
"image_id": module.SIGNOZ_IMAGE_ID,
"image_ref": module.SIGNOZ_IMAGE_REF,
"running": True,
"started_at": "2026-07-22T00:00:00Z",
"restart_count": 0,
}
monkeypatch.setattr(module, "ROOT_UID", uid)
monkeypatch.setattr(module, "REMOTE_ROOT", remote_parent / "signoz-metadata")
monkeypatch.setattr(
module,
"RECEIPT_ROOT",
receipt_parent / "signoz-metadata-toolchain",
)
module.REMOTE_ROOT.mkdir(mode=0o700)
module.RECEIPT_ROOT.mkdir(mode=0o700)
monkeypatch.setattr(module, "OPERATION_LOCK", lock)
monkeypatch.setattr(module, "production_snapshot", lambda: dict(snapshot))
monkeypatch.setattr(module, "require_health", lambda: None)
monkeypatch.setattr(module, "require_no_active_operations", lambda: None)
monkeypatch.setenv("SUDO_UID", str(uid))
monkeypatch.setenv("SUDO_GID", str(gid))
values = _payloads()
manifest = _manifest(module, values)
return module, values, manifest, snapshot
def _write_tree(
root: Path,
manifest: list[dict[str, str]],
values: dict[str, bytes],
*,
directory_mode: int,
) -> None:
root.mkdir(mode=directory_mode, parents=True)
for row in manifest:
path = root / row["name"]
path.write_bytes(values[row["name"]])
os.chmod(path, int(row["mode"], 8))
def test_target_state_distinguishes_absent_exact_drift_and_active_pointer(
target_env,
) -> None:
module, values, manifest, _snapshot = target_env
args = _args(module, manifest, mode="check")
assert module.target_state(args.bundle_id, manifest) == "absent"
target = module.REMOTE_ROOT / args.bundle_id
_write_tree(target, manifest, values, directory_mode=0o750)
assert module.target_state(args.bundle_id, manifest) == "exact"
(target / manifest[0]["name"]).write_bytes(b"drift")
with pytest.raises(module.TargetError, match="target_file_drift"):
module.target_state(args.bundle_id, manifest)
(target / manifest[0]["name"]).write_bytes(values[manifest[0]["name"]])
os.chmod(target / manifest[0]["name"], int(manifest[0]["mode"], 8))
(module.REMOTE_ROOT / "current").symlink_to(target)
with pytest.raises(module.TargetError, match="active_pointer_must_remain_absent"):
module.target_state(args.bundle_id, manifest)
def test_prepare_creates_private_transport_owned_stage(
target_env, tmp_path: Path
) -> None:
module, _values, manifest, _snapshot = target_env
stage = tmp_path / "transport-stage"
args = _args(module, manifest, mode="prepare", stage=stage)
module.reset_mutation_state()
receipt = module.prepare_stage(args, manifest)
metadata = stage.lstat()
assert receipt["terminal"] == "stage_prepared"
assert metadata.st_uid == os.getuid()
assert stat.S_IMODE(metadata.st_mode) == 0o700
assert receipt["source_write_attempted"] is True
assert receipt["source_write_performed"] is True
def test_apply_writes_receipt_only_after_exact_target_and_live_stage_cleanup(
target_env, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
module, values, manifest, _snapshot = target_env
stage = tmp_path / "transport-stage"
_write_tree(stage, manifest, values, directory_mode=0o700)
args = _args(module, manifest, mode="apply", stage=stage)
original_write = module.write_deployment_receipt
observed: list[tuple[bool, str]] = []
def guarded_write(*call_args, **call_kwargs):
observed.append(
(
os.path.lexists(stage),
module.target_state(args.bundle_id, manifest),
)
)
return original_write(*call_args, **call_kwargs)
monkeypatch.setattr(module, "write_deployment_receipt", guarded_write)
module.reset_mutation_state()
receipt = module.apply_bundle(args, manifest)
assert receipt["terminal"] == "apply_pass_deployed_inactive"
assert observed == [(False, "exact")]
assert not os.path.lexists(stage)
assert not os.path.lexists(
module.REMOTE_ROOT / f".{args.bundle_id}.candidate.{args.run_id}"
)
durable = module.load_deploy_receipt(args)
assert durable is not None
assert durable["deployment_method"] == "created"
assert (
durable["production_continuity_sha256"]
== receipt["production_continuity_sha256"]
)
assert not os.path.lexists(module.REMOTE_ROOT / "current")
def test_reconcile_retries_half_quarantine_and_preserves_shared_target(
target_env, tmp_path: Path
) -> None:
module, values, manifest, _snapshot = target_env
stage = tmp_path / "transport-stage"
args = _args(
module,
manifest,
mode="reconcile",
run_id="run-half",
attempt_id="attempt-002",
stage=stage,
)
receipt_dir = module.RECEIPT_ROOT / args.run_id
receipt_dir.mkdir(mode=0o700, parents=True)
# Model a signal after the stage was moved but before the candidate move.
(receipt_dir / "transport-stage").mkdir(mode=0o700)
candidate = module.REMOTE_ROOT / f".{args.bundle_id}.candidate.{args.run_id}"
candidate.mkdir(mode=0o700, parents=True)
target = module.REMOTE_ROOT / args.bundle_id
_write_tree(target, manifest, values, directory_mode=0o750)
module.reset_mutation_state()
receipt = module.reconcile_apply(args, manifest)
assert receipt["terminal"] == "reconcile_exact_target_preserved_unverified"
assert receipt["owned_live_residue_verified_absent"] is True
assert target.is_dir()
assert (receipt_dir / "transport-stage").is_dir()
assert (receipt_dir / "quarantine-candidate").is_dir()
assert not candidate.exists()
def test_reconcile_finalizes_interrupted_exact_apply_from_durable_intent(
target_env, tmp_path: Path
) -> None:
module, values, manifest, snapshot = target_env
stage = tmp_path / "transport-stage"
args = _args(
module,
manifest,
mode="reconcile",
run_id="run-interrupted",
attempt_id="attempt-003",
stage=stage,
)
receipt_dir = module.RECEIPT_ROOT / args.run_id
receipt_dir.mkdir(mode=0o700, parents=True)
continuity = module.sha256_bytes(module.canonical_bytes(snapshot))
intent = module.apply_intent(args, "absent", continuity)
module.write_private_new(
receipt_dir / "apply-intent.json", module.canonical_bytes(intent), 0o600
)
target = module.REMOTE_ROOT / args.bundle_id
_write_tree(target, manifest, values, directory_mode=0o750)
module.reset_mutation_state()
receipt = module.reconcile_apply(args, manifest)
assert receipt["terminal"] == "reconcile_exact_deployment_verified"
assert receipt["durable_deploy_receipt_valid"] is True
durable = module.load_deploy_receipt(args)
assert durable is not None
assert durable["deployment_method"] == "reconciled_exact"
assert durable["reconcile_attempt_id"] == "attempt-003"
assert target.is_dir()
def test_reconcile_zero_live_residue_is_repeatable_with_new_attempt(
target_env, tmp_path: Path
) -> None:
module, _values, manifest, _snapshot = target_env
stage = tmp_path / "transport-stage"
first = _args(
module,
manifest,
mode="reconcile",
run_id="run-cleanup",
attempt_id="attempt-004",
stage=stage,
)
stage.mkdir(mode=0o700)
module.reset_mutation_state()
first_receipt = module.reconcile_apply(first, manifest)
second = _args(
module,
manifest,
mode="reconcile",
run_id=first.run_id,
attempt_id="attempt-005",
stage=stage,
)
module.reset_mutation_state()
second_receipt = module.reconcile_apply(second, manifest)
assert first_receipt["terminal"] == "reconcile_zero_owned_residue"
assert second_receipt["terminal"] == "reconcile_zero_owned_residue"
assert not stage.exists()
assert (module.RECEIPT_ROOT / first.run_id / "transport-stage").is_dir()
def test_reconcile_fails_closed_on_production_continuity_change(
target_env, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
module, values, manifest, snapshot = target_env
stage = tmp_path / "transport-stage"
args = _args(
module,
manifest,
mode="reconcile",
run_id="run-continuity",
attempt_id="attempt-006",
stage=stage,
)
receipt_dir = module.RECEIPT_ROOT / args.run_id
receipt_dir.mkdir(mode=0o700, parents=True)
intent = module.apply_intent(
args,
"absent",
module.sha256_bytes(module.canonical_bytes(snapshot)),
)
module.write_private_new(
receipt_dir / "apply-intent.json", module.canonical_bytes(intent), 0o600
)
_write_tree(
module.REMOTE_ROOT / args.bundle_id, manifest, values, directory_mode=0o750
)
changed = dict(snapshot)
changed["restart_count"] = 1
monkeypatch.setattr(module, "production_snapshot", lambda: dict(changed))
module.reset_mutation_state()
with pytest.raises(
module.TargetError, match="apply_intent_production_continuity_mismatch"
):
module.reconcile_apply(args, manifest)

8
scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py
View File

@@ -122,7 +122,7 @@ def test_agent99_atomic_bundle_owns_the_dispatch_entrypoint() -> None:
assert "$dbExecutorRecoveryContract" in contract
assert "awoooi_db_bounded_executor_receipt_v2" in contract
assert "Receipt.database_sqlstate" in contract
assert 'expectedRuntimeFileCount": 20' in transport
assert "expectedRuntimeFileCount=20" in transport
assert "expectedRuntimeFileCount -ne 20" in receiver
assert "fileCount = 20" in receiver
assert 'expectedRuntimeFileCount": 21' in transport
assert "expectedRuntimeFileCount=21" in transport
assert "expectedRuntimeFileCount -ne 21" in receiver
assert "fileCount = 21" in receiver

99
scripts/ops/tests/test_signoz_metadata_toolchain_agent99_transport.py Normal file
View File

@@ -0,0 +1,99 @@
from __future__ import annotations
import re
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SENDER = ROOT / "scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh"
def _array(source: str, name: str) -> tuple[str, ...]:
match = re.search(rf"{re.escape(name)}=\(\n(?P<body>.*?)\n\)", source, re.DOTALL)
assert match is not None
return tuple(re.findall(r'"([^"\n]+)"', match.group("body")))
def test_sender_is_syntax_valid_and_routes_only_through_windows99_agent99() -> None:
source = SENDER.read_text(encoding="utf-8")
result = subprocess.run(
["bash", "-n", str(SENDER)],
check=False,
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
assert 'readonly TARGET="Administrator@192.168.0.99"' in source
assert "wooo@192.168.0.110" not in source
assert (
'dispatchPath": "windows99_agent99_to_host110_fixed_metadata_toolchain"'
in source
)
assert "C:\\Wooo\\Agent99\\bin\\agent99-signoz-metadata-executor.ps1" in source
assert "Invoke-Expression" not in source
assert "github.com" not in source
assert "ghcr.io" not in source
def test_sender_binds_exact_fresh_gitea_main_and_all_ten_transport_sources() -> None:
source = SENDER.read_text(encoding="utf-8")
toolchain_sources = _array(source, "TOOLCHAIN_SOURCES")
assert len(toolchain_sources) == 7
assert len(set(toolchain_sources)) == 7
assert 'readonly GITEA_REMOTE="gitea"' in source
assert "+refs/heads/main:refs/remotes/gitea/main" in source
assert 'SOURCE_REVISION}" == "${GITEA_MAIN_REVISION}' in source
assert 'git -C "${SOURCE_ROOT}" diff --quiet "${SOURCE_REVISION}"' in source
assert '"${TOOLCHAIN_SOURCES[@]}"' in source
assert '"${EXECUTOR_RELATIVE}"' in source
assert '"${TARGET_RELATIVE}"' in source
assert '"${WRAPPER_RELATIVE}"' in source
assert "source_bound_file_untracked" in source
assert "source_bundle_differs_from_revision" in source
def test_sender_envelope_is_exact_seven_file_secret_free_and_deleted_before_executor() -> (
None
):
source = SENDER.read_text(encoding="utf-8")
assert '"schemaVersion": "awoooi_signoz_metadata_toolchain_envelope_v1"' in source
assert '"fileCount": 7' in source
assert '"transportPayloadContainsSecrets": False' in source
assert '"contentBase64": base64.b64encode(content).decode("ascii")' in source
assert "fixed_toolchain_file_contract_failed" in source
assert "toolchain_envelope_transport_file_invalid" in source
assert "toolchain_envelope_transport_delete_failed" in source
delete = source.index("Remove-Item -LiteralPath $payloadPath")
invoke = source.index("& $executor @parameters")
assert delete < invoke
assert "$parameters.ToolchainEnvelopeDeletedBeforeExecutor = $true" in source
assert "REMOTE_PAYLOAD_PREPARED" in source
assert "transport_payload_cleanup_failed" in source
def test_sender_has_bounded_check_apply_verify_and_attempt_scoped_reconcile() -> None:
source = SENDER.read_text(encoding="utf-8")
apply_branch = source[source.index(" apply)") :]
for mode in (
"ToolchainCheck",
"ToolchainApply",
"ToolchainVerify",
"ToolchainReconcile",
):
assert mode in source
assert "reconcile_attempt_id_missing_or_invalid" in source
assert "reconcile-$(date -u +%Y%m%dT%H%M%SZ)-$$" in source
assert apply_branch.index('run_checked_mode "ToolchainCheck"') < apply_branch.index(
'invoke_remote "ToolchainApply"'
)
assert apply_branch.index('invoke_remote "ToolchainApply"') < apply_branch.index(
'run_checked_mode "ToolchainVerify"'
)
assert 'run_checked_mode "ToolchainReconcile"' in apply_branch
assert "toolchain_reconcile_zero_owned_residue" in source
assert "exit 75" in source
assert 'timeout --kill-after=20 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}"' in source

295
scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py
View File

@@ -1,267 +1,62 @@
from __future__ import annotations
import json
import os
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
DEPLOYER = ROOT / "scripts" / "ops" / "deploy-signoz-metadata-toolchain.sh"
DEPLOYER = ROOT / "scripts/ops/deploy-signoz-metadata-toolchain.sh"
def _write_executable(path: Path, text: str) -> None:
path.write_text(text, encoding="utf-8")
path.chmod(0o755)
def fake_environment(tmp_path: Path, *, mode: str) -> dict[str, str]:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
event_log = tmp_path / "events.log"
ssh_count = tmp_path / "ssh.count"
event_log.touch()
ssh_count.write_text("0\n", encoding="utf-8")
_write_executable(
fake_bin / "timeout",
"""#!/bin/bash
set -eu
while [[ "${1:-}" == --kill-after=* ]]; do shift; done
shift
exec "$@"
""",
def test_historical_direct_host110_deployer_is_syntax_valid_and_fail_closed() -> None:
syntax = subprocess.run(
["bash", "-n", str(DEPLOYER)],
check=False,
capture_output=True,
text=True,
)
_write_executable(
fake_bin / "ssh",
"""#!/bin/bash
set -eu
count="$(cat "${TEST_SSH_COUNT:?}")"
count=$((count + 1))
printf '%s\n' "$count" > "${TEST_SSH_COUNT:?}"
printf 'ssh %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
cat >/dev/null || true
case "${FAKE_SSH_MODE:?}" in
check_absent)
printf 'REMOTE_CHECK terminal=check_pass_no_write state=absent drift=0 candidate_count=0 api=200 active_operations=0 container_state_sha256=%064d\n' 0
;;
check_drift)
exit 3
;;
apply)
case "$count" in
1)
printf 'REMOTE_CHECK terminal=check_pass_no_write state=absent drift=0 candidate_count=0 api=200 active_operations=0 container_state_sha256=%064d\n' 0
;;
2) ;;
3)
printf '{"phase":"terminal","terminal":"pass_source_deployed_inactive"}\n'
;;
4)
printf 'REMOTE_CHECK terminal=check_pass_no_write state=exact drift=0 candidate_count=0 api=200 active_operations=0 container_state_sha256=%064d\n' 0
;;
*) exit 90 ;;
esac
;;
esac
""",
result = subprocess.run(
["bash", str(DEPLOYER), "--apply"],
check=False,
capture_output=True,
text=True,
)
_write_executable(
fake_bin / "scp",
"""#!/bin/bash
set -eu
printf 'scp %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
""",
)
return {
**os.environ,
"PATH": f"{fake_bin}:{os.environ['PATH']}",
"TEST_EVENT_LOG": str(event_log),
"TEST_SSH_COUNT": str(ssh_count),
"FAKE_SSH_MODE": mode,
"TRACE_ID": "trace-P0-OBS-002-metadata-deploy",
"RUN_ID": f"run-{mode}",
"WORK_ITEM_ID": "P0-OBS-002",
assert syntax.returncode == 0, syntax.stderr
assert result.returncode == 78
receipt = json.loads(result.stdout)
assert receipt == {
"schema": "awoooi_signoz_metadata_toolchain_route_retirement_v1",
"terminal": "blocked_direct_host110_route_retired",
"replacement": "scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh",
"production_mutation_performed": False,
"completion_claim": False,
}
def run_deployer(
tmp_path: Path,
*,
mode: str,
apply: bool,
env_overrides: dict[str, str] | None = None,
) -> subprocess.CompletedProcess[str]:
env = fake_environment(tmp_path, mode=mode)
if env_overrides:
env.update(env_overrides)
return subprocess.run(
["bash", str(DEPLOYER), "--apply" if apply else "--check"],
env=env,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=20,
check=False,
)
def receipt_rows(stdout: str) -> list[dict[str, object]]:
return [json.loads(line) for line in stdout.splitlines() if line.startswith("{")]
def test_help_documents_inactive_immutable_contract() -> None:
result = subprocess.run(
["bash", str(DEPLOYER), "--help"],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=10,
check=False,
)
assert result.returncode == 0
assert "immutable exact-hash directory" in result.stdout
assert "does not create or" in result.stdout
assert "active pointer" in result.stdout
def test_check_mode_is_no_write_and_ready_when_bundle_absent(tmp_path: Path) -> None:
result = run_deployer(tmp_path, mode="check_absent", apply=False)
assert result.returncode == 0, result.stderr
rows = receipt_rows(result.stdout)
assert rows[-1]["terminal"] == "check_pass_no_write"
assert rows[-1]["detail"].endswith("remote_state_absent")
events = (tmp_path / "events.log").read_text(encoding="utf-8").splitlines()
assert len([line for line in events if line.startswith("ssh ")]) == 1
assert not [line for line in events if line.startswith("scp ")]
def test_check_mode_fails_closed_on_remote_drift(tmp_path: Path) -> None:
result = run_deployer(tmp_path, mode="check_drift", apply=False)
assert result.returncode == 1
rows = receipt_rows(result.stdout)
assert rows[-1]["terminal"] == "failed_no_write"
def test_apply_uses_seven_transfers_and_exact_postcheck(tmp_path: Path) -> None:
result = run_deployer(tmp_path, mode="apply", apply=True)
assert result.returncode == 0, result.stderr
rows = receipt_rows(result.stdout)
assert rows[-1]["terminal"] == "pass_source_deployed_inactive"
assert rows[-1]["detail"] == "runtime_export_not_executed"
events = (tmp_path / "events.log").read_text(encoding="utf-8").splitlines()
assert len([line for line in events if line.startswith("scp ")]) == 7
assert len([line for line in events if line.startswith("ssh ")]) == 4
def test_deployer_has_no_active_pointer_or_destructive_fallback() -> None:
def test_historical_entrypoint_contains_no_transport_or_mutation_primitive() -> None:
source = DEPLOYER.read_text(encoding="utf-8")
assert '"${REMOTE_ROOT}/current"' in source
assert "active_pointer_0" in source
assert "pass_source_deployed_inactive" in source
assert "quarantine-candidate" in source
assert "quarantine-target" in source
assert "docker stop" not in source
assert "docker start" not in source
assert "docker restart" not in source
assert "sqlite3" not in source
assert "github.com" not in source.casefold()
assert "curl |" not in source
assert "curl -fsSL" not in source
assert "\nrm " not in source
assert "ln -s" not in source
assert "signoz-metadata-isolated-restore.py" in source
assert "isolated-cluster.xml" in source
assert "local_exact_sources_7" in source
assert "[ \"${#EXPECTED_HASHES[@]}\" -eq 7 ]" in source
def test_transport_propagates_strict_fixed_paths_with_spaces(tmp_path: Path) -> None:
transport_dir = tmp_path / "transport paths"
transport_dir.mkdir()
identity = transport_dir / "agent99 identity"
known_hosts = transport_dir / "known hosts"
identity.write_text("test-only-identity\n", encoding="utf-8")
known_hosts.write_text("test-only-known-host\n", encoding="utf-8")
result = run_deployer(
tmp_path,
mode="check_absent",
apply=False,
env_overrides={
"SIGNOZ_METADATA_SSH_IDENTITY_FILE": str(identity),
"SIGNOZ_METADATA_KNOWN_HOSTS_FILE": str(known_hosts),
},
)
assert result.returncode == 0, result.stderr
event = (tmp_path / "events.log").read_text(encoding="utf-8")
assert "IdentitiesOnly=yes" in event
assert "StrictHostKeyChecking=yes" in event
assert str(identity) in event
assert f"UserKnownHostsFile={known_hosts}" in event
def test_transport_rejects_missing_fixed_path_before_remote_use(tmp_path: Path) -> None:
result = run_deployer(
tmp_path,
mode="check_absent",
apply=False,
env_overrides={
"SIGNOZ_METADATA_SSH_IDENTITY_FILE": str(tmp_path / "missing identity")
},
)
assert result.returncode == 64
assert "invalid SSH identity file" in result.stderr
assert not (tmp_path / "events.log").read_text(encoding="utf-8")
def test_transport_rejects_symlink_fixed_path_before_remote_use(tmp_path: Path) -> None:
target = tmp_path / "known-hosts-target"
target.write_text("test-only-known-host\n", encoding="utf-8")
symlink = tmp_path / "known hosts link"
symlink.symlink_to(target)
result = run_deployer(
tmp_path,
mode="check_absent",
apply=False,
env_overrides={"SIGNOZ_METADATA_KNOWN_HOSTS_FILE": str(symlink)},
)
assert result.returncode == 64
assert "invalid SSH known_hosts file" in result.stderr
assert not (tmp_path / "events.log").read_text(encoding="utf-8")
def test_shared_operation_lock_is_existing_read_only_and_fail_closed() -> None:
source = DEPLOYER.read_text(encoding="utf-8")
assert "OPERATION_LOCK=/tmp/awoooi-signoz-backup-operation.lock" in source
assert '[ -f "${OPERATION_LOCK}" ] && [ ! -L "${OPERATION_LOCK}" ]' in source
assert 'exec 8<"${OPERATION_LOCK}"' in source
assert "flock -n 8" in source
assert "exec 8>>/tmp/awoooi-signoz-backup-operation.lock" not in source
def test_remote_check_uses_privileged_read_only_visibility() -> None:
source = DEPLOYER.read_text(encoding="utf-8")
remote_check = source.split("remote_check() {", 1)[1].split("\n}", 1)[0]
assert '"${TARGET_HOST}" sudo -n bash -s --' in remote_check
assert "REMOTE_CHECK contains only stat/hash/process/API" in remote_check
def test_exact_seven_file_supply_chain_and_modes_are_fixed() -> None:
source = DEPLOYER.read_text(encoding="utf-8")
for path in (
"config/signoz/metadata-export-policy.json",
"scripts/backup/signoz_metadata_contract.py",
"scripts/backup/signoz-metadata-export.py",
"scripts/backup/verify-signoz-metadata-export.py",
"scripts/backup/signoz-metadata-restore-drill.py",
"scripts/backup/signoz-metadata-isolated-restore.py",
"ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
for forbidden in (
"192.168.0.110",
"ssh ",
"scp ",
"docker ",
"sudo ",
"curl ",
"rm ",
"mv ",
"ln ",
):
assert path in source
assert "MODES=(0644 0644 0755 0755 0755 0755 0644)" in source
assert '[ "${#EXPECTED_HASHES[@]}" -eq 7 ]' in source
assert forbidden not in source
assert "blocked_direct_host110_route_retired" in source
assert "production_mutation_performed" in source
assert "completion_claim" in source
def test_historical_entrypoint_points_only_to_agent99_replacement() -> None:
source = DEPLOYER.read_text(encoding="utf-8")
assert "scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh" in source
assert "github.com" not in source.casefold()
assert "ghcr.io" not in source.casefold()