Files
awoooi/scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh

486 lines
19 KiB
Bash
Executable File

#!/usr/bin/env bash
# Source-bound controller -> Windows99/Agent99 -> Host110 SigNoz toolchain route.
set -euo pipefail
readonly SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
readonly DEFAULT_SOURCE_ROOT="$(CDPATH= cd -- "${SCRIPT_DIR}/../.." && pwd)"
readonly TARGET="Administrator@192.168.0.99"
readonly GITEA_REMOTE="gitea"
readonly GITEA_URL_HTTPS="https://gitea.wooo.work/wooo/awoooi.git"
readonly GITEA_URL_HTTP_INTERNAL="http://192.168.0.110:3000/wooo/awoooi.git"
readonly GITEA_URL_SSH_INTERNAL="ssh://git@192.168.0.110:2222/wooo/awoooi.git"
readonly EXECUTOR_RELATIVE="agent99-signoz-metadata-executor.ps1"
readonly TARGET_RELATIVE="agent99-signoz-toolchain-target.py"
readonly WRAPPER_RELATIVE="scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh"
LABELS=(
"metadata-export-policy.json"
"signoz_metadata_contract.py"
"signoz-metadata-export.py"
"verify-signoz-metadata-export.py"
"signoz-metadata-restore-drill.py"
"signoz-metadata-isolated-restore.py"
"isolated-cluster.xml"
)
TOOLCHAIN_SOURCES=(
"config/signoz/metadata-export-policy.json"
"scripts/backup/signoz_metadata_contract.py"
"scripts/backup/signoz-metadata-export.py"
"scripts/backup/verify-signoz-metadata-export.py"
"scripts/backup/signoz-metadata-restore-drill.py"
"scripts/backup/signoz-metadata-isolated-restore.py"
"ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
)
MODES=("0644" "0644" "0755" "0755" "0755" "0755" "0644")
MODE="check"
MODE_SELECTED=""
TRACE_ID=""
RUN_ID=""
WORK_ITEM_ID=""
ATTEMPT_ID=""
SOURCE_ROOT="${DEFAULT_SOURCE_ROOT}"
SOURCE_REVISION=""
IDENTITY_FILE=""
KNOWN_HOSTS_FILE="${HOME:-}/.ssh/known_hosts"
CONNECT_TIMEOUT_SECONDS="8"
REMOTE_EXECUTION_TIMEOUT_SECONDS="1200"
REMOTE_PAYLOAD_PREPARED="false"
REMOTE_PAYLOAD_PATH_WINDOWS=""
usage() {
printf '%s\n' \
"Usage: $0 [--check|--apply|--verify|--reconcile] [options]" \
"" \
"Fixed route: local fresh Gitea main -> Windows99 Agent99 -> Host110." \
"No local connection to Host110 and no active toolchain pointer are allowed." \
"" \
" --trace-id ID Required controlled-apply trace identity." \
" --run-id ID Required original deployment run identity." \
" --work-item-id P0-OBS-002 Required fixed work item." \
" --attempt-id ID Required for reconcile; optional apply recovery ID." \
" --source-root PATH Checkout root (default: repository root)." \
" --source-revision SHA Must equal freshly fetched gitea/main." \
" --identity-file PATH Optional Windows99 SSH identity path." \
" --known-hosts-file PATH Pinned known_hosts file." \
" --connect-timeout SECONDS 1 through 30 (default: 8)."
}
fail() {
printf 'signoz_toolchain_agent99_error=%s\n' "$1" >&2
exit 64
}
require_value() {
[[ $# -ge 2 && -n "$2" ]] || fail "missing_option_value"
}
while [[ $# -gt 0 ]]; do
case "$1" in
--check|--apply|--verify|--reconcile)
[[ -z "${MODE_SELECTED}" ]] || fail "multiple_mode_flags"
MODE="${1#--}"
MODE_SELECTED="true"
shift
;;
--trace-id)
require_value "$@"; TRACE_ID="$2"; shift 2 ;;
--run-id)
require_value "$@"; RUN_ID="$2"; shift 2 ;;
--work-item-id)
require_value "$@"; WORK_ITEM_ID="$2"; shift 2 ;;
--attempt-id)
require_value "$@"; ATTEMPT_ID="$2"; shift 2 ;;
--source-root)
require_value "$@"; SOURCE_ROOT="$2"; shift 2 ;;
--source-revision)
require_value "$@"; SOURCE_REVISION="$2"; shift 2 ;;
--identity-file)
require_value "$@"; IDENTITY_FILE="$2"; shift 2 ;;
--known-hosts-file)
require_value "$@"; KNOWN_HOSTS_FILE="$2"; shift 2 ;;
--connect-timeout)
require_value "$@"; CONNECT_TIMEOUT_SECONDS="$2"; shift 2 ;;
-h|--help)
usage; exit 0 ;;
*) fail "unsupported_option" ;;
esac
done
readonly SAFE_ID_PATTERN='^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$'
[[ "${TRACE_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "trace_id_missing_or_invalid"
[[ "${RUN_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "run_id_missing_or_invalid"
[[ "${WORK_ITEM_ID}" == "P0-OBS-002" ]] || fail "work_item_id_not_allowlisted"
if [[ "${MODE}" == "reconcile" ]]; then
[[ "${ATTEMPT_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "reconcile_attempt_id_missing_or_invalid"
elif [[ -n "${ATTEMPT_ID}" && "${MODE}" != "apply" ]]; then
fail "attempt_id_not_allowed_for_mode"
fi
[[ "${CONNECT_TIMEOUT_SECONDS}" =~ ^[0-9]+$ ]] || fail "invalid_connect_timeout"
(( CONNECT_TIMEOUT_SECONDS >= 1 && CONNECT_TIMEOUT_SECONDS <= 30 )) || fail "invalid_connect_timeout"
for dependency in git python3 ssh scp mktemp timeout; do
command -v "${dependency}" >/dev/null 2>&1 || fail "required_command_missing_${dependency}"
done
[[ -f "${KNOWN_HOSTS_FILE}" && ! -L "${KNOWN_HOSTS_FILE}" ]] || fail "known_hosts_file_missing_or_unsafe"
if [[ -n "${IDENTITY_FILE}" ]]; then
[[ -f "${IDENTITY_FILE}" && ! -L "${IDENTITY_FILE}" ]] || fail "identity_file_missing_or_unsafe"
fi
[[ -d "${SOURCE_ROOT}" ]] || fail "source_root_missing"
SOURCE_ROOT="$(CDPATH= cd -- "${SOURCE_ROOT}" && pwd)"
GITEA_URL="$(git -C "${SOURCE_ROOT}" remote get-url "${GITEA_REMOTE}" 2>/dev/null)" \
|| fail "gitea_remote_missing"
case "${GITEA_URL}" in
"${GITEA_URL_HTTPS}"|"${GITEA_URL_HTTP_INTERNAL}"|"${GITEA_URL_SSH_INTERNAL}") ;;
*) fail "gitea_remote_not_fixed_internal" ;;
esac
git -C "${SOURCE_ROOT}" fetch --quiet --no-tags "${GITEA_REMOTE}" \
'+refs/heads/main:refs/remotes/gitea/main' || fail "fresh_gitea_main_fetch_failed"
GITEA_MAIN_REVISION="$(git -C "${SOURCE_ROOT}" rev-parse --verify 'refs/remotes/gitea/main^{commit}')" \
|| fail "fresh_gitea_main_revision_unavailable"
GITEA_MAIN_REVISION="$(printf '%s' "${GITEA_MAIN_REVISION}" | tr '[:upper:]' '[:lower:]')"
if [[ -z "${SOURCE_REVISION}" ]]; then
SOURCE_REVISION="${GITEA_MAIN_REVISION}"
fi
SOURCE_REVISION="$(printf '%s' "${SOURCE_REVISION}" | tr '[:upper:]' '[:lower:]')"
[[ "${SOURCE_REVISION}" =~ ^[0-9a-f]{40}$ ]] || fail "source_revision_must_be_full_commit_sha"
[[ "${SOURCE_REVISION}" == "${GITEA_MAIN_REVISION}" ]] || fail "source_revision_is_not_fresh_gitea_main"
SOURCE_BOUND_FILES=(
"${TOOLCHAIN_SOURCES[@]}"
"${EXECUTOR_RELATIVE}"
"${TARGET_RELATIVE}"
"${WRAPPER_RELATIVE}"
)
for relative_path in "${SOURCE_BOUND_FILES[@]}"; do
[[ -f "${SOURCE_ROOT}/${relative_path}" && ! -L "${SOURCE_ROOT}/${relative_path}" ]] \
|| fail "source_bound_file_missing_or_unsafe"
git -C "${SOURCE_ROOT}" ls-files --error-unmatch -- "${relative_path}" >/dev/null 2>&1 \
|| fail "source_bound_file_untracked"
done
git -C "${SOURCE_ROOT}" diff --quiet "${SOURCE_REVISION}" -- "${SOURCE_BOUND_FILES[@]}" \
|| fail "source_bundle_differs_from_revision"
umask 077
WORK_DIR="$(mktemp -d /tmp/awoooi-signoz-toolchain-agent99.XXXXXXXX)"
ENVELOPE_PATH="${WORK_DIR}/toolchain-envelope.json"
REMOTE_STDOUT_PATH="${WORK_DIR}/remote.stdout"
REMOTE_STDERR_PATH="${WORK_DIR}/remote.stderr"
SSH_OPTIONS=(
-T
-o BatchMode=yes
-o PreferredAuthentications=publickey
-o PubkeyAuthentication=yes
-o PasswordAuthentication=no
-o KbdInteractiveAuthentication=no
-o NumberOfPasswordPrompts=0
-o StrictHostKeyChecking=yes
-o "UserKnownHostsFile=${KNOWN_HOSTS_FILE}"
-o "ConnectTimeout=${CONNECT_TIMEOUT_SECONDS}"
-o ConnectionAttempts=1
-o ServerAliveInterval=10
-o ServerAliveCountMax=3
-o LogLevel=ERROR
-o IdentitiesOnly=yes
)
SCP_OPTIONS=("${SSH_OPTIONS[@]:1}")
if [[ -n "${IDENTITY_FILE}" ]]; then
SSH_OPTIONS+=( -i "${IDENTITY_FILE}" )
SCP_OPTIONS+=( -i "${IDENTITY_FILE}" )
fi
make_encoded_command() {
local executor_mode="$1" payload_path="$2" attempt_id="$3"
python3 - "${executor_mode}" "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
"${SOURCE_REVISION}" "${payload_path}" "${attempt_id}" <<'PY'
from __future__ import annotations
import base64
import sys
mode, trace_id, run_id, work_item_id, revision, payload_path, attempt_id = sys.argv[1:]
values = [mode, trace_id, run_id, work_item_id, revision, payload_path, attempt_id]
if any("'" in value or "\r" in value or "\n" in value for value in values):
raise SystemExit("unsafe_remote_command_value")
script = rf'''$ErrorActionPreference = 'Stop'
$executor = 'C:\Wooo\Agent99\bin\agent99-signoz-metadata-executor.ps1'
if (-not (Test-Path -LiteralPath $executor -PathType Leaf)) {{ throw 'executor_missing' }}
$parameters = @{{
Mode = '{mode}'
TraceId = '{trace_id}'
RunId = '{run_id}'
WorkItemId = '{work_item_id}'
SourceRevision = '{revision}'
}}
'''
if mode == "ToolchainApply":
script += rf'''$payloadPath = '{payload_path}'
$envelope = ''
try {{
$item = Get-Item -LiteralPath $payloadPath -ErrorAction Stop
if (($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 -or $item.Length -le 0 -or $item.Length -gt 4194304) {{
throw 'toolchain_envelope_transport_file_invalid'
}}
$envelope = [IO.File]::ReadAllText($payloadPath, [Text.Encoding]::UTF8)
}} finally {{
Remove-Item -LiteralPath $payloadPath -Force -ErrorAction SilentlyContinue
}}
if (Test-Path -LiteralPath $payloadPath) {{ throw 'toolchain_envelope_transport_delete_failed' }}
$parameters.ToolchainEnvelopeJson = $envelope
$parameters.ToolchainEnvelopeDeletedBeforeExecutor = $true
'''
elif mode == "ToolchainReconcile":
script += f"$parameters.AttemptId = '{attempt_id}'\r\n"
script += "& $executor @parameters\r\nexit [int]$LASTEXITCODE\r\n"
encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii")
print(
"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
f"-EncodedCommand {encoded}"
)
PY
}
make_file_command() {
local action="$1" path="$2"
python3 - "${action}" "${path}" <<'PY'
from __future__ import annotations
import base64
import sys
action, path = sys.argv[1:]
if "'" in path or "\r" in path or "\n" in path:
raise SystemExit("unsafe_transport_path")
if action == "prepare":
script = rf'''$ErrorActionPreference='Stop'
$directory='C:\Wooo\Agent99\deploy\transport-inbox'
$path='{path}'
New-Item -ItemType Directory -Force -Path $directory | Out-Null
Remove-Item -LiteralPath $path -Force -ErrorAction SilentlyContinue
if (Test-Path -LiteralPath $path) {{ throw 'stale_transport_payload_cleanup_failed' }}
'''
else:
script = rf'''$ErrorActionPreference='Stop'
$path='{path}'
Remove-Item -LiteralPath $path -Force -ErrorAction SilentlyContinue
if (Test-Path -LiteralPath $path) {{ throw 'transport_payload_cleanup_failed' }}
'''
encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii")
print(
"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
f"-EncodedCommand {encoded}"
)
PY
}
cleanup() {
if [[ "${REMOTE_PAYLOAD_PREPARED}" == "true" && -n "${REMOTE_PAYLOAD_PATH_WINDOWS}" ]]; then
cleanup_command="$(make_file_command cleanup "${REMOTE_PAYLOAD_PATH_WINDOWS}" 2>/dev/null || true)"
if [[ -n "${cleanup_command}" ]]; then
timeout --kill-after=10 60 ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${cleanup_command}" \
>/dev/null 2>&1 || true
fi
fi
rm -rf -- "${WORK_DIR}"
}
trap cleanup EXIT HUP INT TERM
build_envelope() {
python3 - "${SOURCE_ROOT}" "${SOURCE_REVISION}" "${TRACE_ID}" "${RUN_ID}" \
"${WORK_ITEM_ID}" "${ENVELOPE_PATH}" \
"${#TOOLCHAIN_SOURCES[@]}" \
"${TOOLCHAIN_SOURCES[@]}" \
"${LABELS[@]}" \
"${MODES[@]}" <<'PY'
from __future__ import annotations
import base64
import hashlib
import json
import sys
from pathlib import Path
root = Path(sys.argv[1])
revision, trace_id, run_id, work_item_id = sys.argv[2:6]
output = Path(sys.argv[6])
count = int(sys.argv[7])
sources = sys.argv[8 : 8 + count]
labels = sys.argv[8 + count : 8 + 2 * count]
modes = sys.argv[8 + 2 * count :]
if count != 7 or len(set(labels)) != 7 or len(modes) != 7:
raise SystemExit("fixed_toolchain_file_contract_failed")
files = []
manifest_rows = []
hashes = []
for source, label, mode in zip(sources, labels, modes, strict=True):
content = (root / source).read_bytes()
digest = hashlib.sha256(content).hexdigest()
hashes.append(digest)
manifest_rows.append(f"{label}\t{digest}\t{mode}\n")
files.append(
{
"name": label,
"sha256": digest,
"mode": mode,
"contentBase64": base64.b64encode(content).decode("ascii"),
}
)
bundle_id = hashlib.sha256(
"".join(f"{digest}\n" for digest in hashes).encode("ascii")
).hexdigest()
manifest_sha256 = hashlib.sha256("".join(manifest_rows).encode()).hexdigest()
value = {
"schemaVersion": "awoooi_signoz_metadata_toolchain_envelope_v1",
"transport": "fresh_gitea_main_to_windows99_agent99_to_fixed_host110",
"traceId": trace_id,
"runId": run_id,
"workItemId": work_item_id,
"sourceRevision": revision,
"bundleId": bundle_id,
"manifestSha256": manifest_sha256,
"fileCount": 7,
"transportPayloadContainsSecrets": False,
"files": files,
}
output.write_text(json.dumps(value, separators=(",", ":")), encoding="utf-8")
PY
}
invoke_remote() {
local executor_mode="$1" payload_path="$2" attempt_id="$3"
local command
command="$(make_encoded_command "${executor_mode}" "${payload_path}" "${attempt_id}")"
: >"${REMOTE_STDOUT_PATH}"
: >"${REMOTE_STDERR_PATH}"
set +e
timeout --kill-after=20 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}" \
ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${command}" \
>"${REMOTE_STDOUT_PATH}" 2>"${REMOTE_STDERR_PATH}"
LAST_REMOTE_RC=$?
set -e
if [[ ! -s "${REMOTE_STDOUT_PATH}" || $(wc -c <"${REMOTE_STDOUT_PATH}") -gt 65536 ]]; then
LAST_REMOTE_OUTPUT=""
return
fi
LAST_REMOTE_OUTPUT="$(tr -d '\r' <"${REMOTE_STDOUT_PATH}")"
}
validate_receipt() {
local value="$1" executor_mode="$2" allowed_csv="$3"
python3 - "${executor_mode}" "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
"${SOURCE_REVISION}" "${allowed_csv}" "${value}" <<'PY'
from __future__ import annotations
import json
import sys
mode, trace_id, run_id, work_item_id, revision, allowed_csv, raw = sys.argv[1:]
try:
value = json.loads(raw)
except json.JSONDecodeError as exc:
raise SystemExit("agent99_receipt_json_invalid") from exc
allowed = set(allowed_csv.split(","))
required = {
"schemaVersion": "agent99_signoz_metadata_executor_dispatch_v1",
"mode": mode,
"traceId": trace_id,
"runId": run_id,
"workItemId": work_item_id,
"sourceRevision": revision,
"targetHost": "192.168.0.110",
"dispatchPath": "windows99_agent99_to_host110_fixed_metadata_toolchain",
"productionRuntimeMutationPerformed": False,
"completionClaim": False,
}
if any(value.get(key) != expected for key, expected in required.items()):
raise SystemExit("agent99_receipt_identity_or_boundary_invalid")
if value.get("terminal") not in allowed:
raise SystemExit("agent99_receipt_terminal_invalid")
if mode == "ToolchainApply" and (
value.get("toolchainEnvelopeTransportPayloadUsed") is not True
or value.get("toolchainEnvelopeTransportPayloadDeletedBeforeExecutor") is not True
or value.get("toolchainEnvelopeTransportPayloadContainsSecrets") is not False
):
raise SystemExit("agent99_envelope_transport_receipt_invalid")
PY
}
run_checked_mode() {
local executor_mode="$1" payload_path="$2" attempt_id="$3" allowed="$4"
invoke_remote "${executor_mode}" "${payload_path}" "${attempt_id}"
[[ -n "${LAST_REMOTE_OUTPUT}" ]] || fail "${executor_mode}_receipt_missing"
validate_receipt "${LAST_REMOTE_OUTPUT}" "${executor_mode}" "${allowed}" \
|| fail "${executor_mode}_receipt_invalid"
printf '%s\n' "${LAST_REMOTE_OUTPUT}"
}
case "${MODE}" in
check)
run_checked_mode "ToolchainCheck" "" "" "toolchain_check_ready"
[[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_check_failed"
;;
verify)
run_checked_mode "ToolchainVerify" "" "" "toolchain_verify_pass"
[[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_verify_failed"
;;
reconcile)
run_checked_mode "ToolchainReconcile" "" "${ATTEMPT_ID}" \
"toolchain_reconcile_exact_deployment_verified,toolchain_reconcile_zero_owned_residue,toolchain_reconcile_preserved_unverified,toolchain_reconcile_failed_or_unverified"
[[ ${LAST_REMOTE_RC} -eq 0 ]] || exit 75
;;
apply)
run_checked_mode "ToolchainCheck" "" "" "toolchain_check_ready"
[[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_check_failed_before_apply"
build_envelope
PAYLOAD_TOKEN="$(python3 - "${ENVELOPE_PATH}" <<'PY'
import hashlib
import sys
from pathlib import Path
print(hashlib.sha256(Path(sys.argv[1]).read_bytes()).hexdigest()[:24])
PY
)"
REMOTE_PAYLOAD_PATH_WINDOWS="C:\\Wooo\\Agent99\\deploy\\transport-inbox\\signoz-toolchain-${PAYLOAD_TOKEN}.json"
REMOTE_PAYLOAD_PATH_SCP="C:/Wooo/Agent99/deploy/transport-inbox/signoz-toolchain-${PAYLOAD_TOKEN}.json"
prepare_command="$(make_file_command prepare "${REMOTE_PAYLOAD_PATH_WINDOWS}")"
timeout --kill-after=10 60 ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${prepare_command}" \
>/dev/null 2>"${REMOTE_STDERR_PATH}" || fail "remote_payload_prepare_failed"
REMOTE_PAYLOAD_PREPARED="true"
timeout --kill-after=10 120 scp -q "${SCP_OPTIONS[@]}" "${ENVELOPE_PATH}" \
"${TARGET}:${REMOTE_PAYLOAD_PATH_SCP}" >/dev/null 2>"${REMOTE_STDERR_PATH}" \
|| fail "remote_payload_transport_failed"
invoke_remote "ToolchainApply" "${REMOTE_PAYLOAD_PATH_WINDOWS}" ""
apply_output="${LAST_REMOTE_OUTPUT}"
apply_rc="${LAST_REMOTE_RC}"
cleanup_command="$(make_file_command cleanup "${REMOTE_PAYLOAD_PATH_WINDOWS}")"
timeout --kill-after=10 60 ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${cleanup_command}" \
>/dev/null 2>&1 || true
REMOTE_PAYLOAD_PREPARED="false"
if [[ -n "${apply_output}" ]] && validate_receipt "${apply_output}" "ToolchainApply" \
"toolchain_deployed_inactive_verified,toolchain_apply_failed_or_unverified"; then
printf '%s\n' "${apply_output}"
fi
if [[ ${apply_rc} -eq 0 ]]; then
validate_receipt "${apply_output}" "ToolchainApply" "toolchain_deployed_inactive_verified" \
|| fail "toolchain_apply_success_receipt_invalid"
run_checked_mode "ToolchainVerify" "" "" "toolchain_verify_pass"
[[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_verify_failed_after_apply"
exit 0
fi
if [[ -z "${ATTEMPT_ID}" ]]; then
ATTEMPT_ID="reconcile-$(date -u +%Y%m%dT%H%M%SZ)-$$"
fi
[[ "${ATTEMPT_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "generated_reconcile_attempt_id_invalid"
run_checked_mode "ToolchainReconcile" "" "${ATTEMPT_ID}" \
"toolchain_reconcile_exact_deployment_verified,toolchain_reconcile_zero_owned_residue,toolchain_reconcile_preserved_unverified,toolchain_reconcile_failed_or_unverified"
if [[ ${LAST_REMOTE_RC} -eq 0 ]] && [[ "${LAST_REMOTE_OUTPUT}" == *'"terminal":"toolchain_reconcile_exact_deployment_verified"'* ]]; then
run_checked_mode "ToolchainVerify" "" "" "toolchain_verify_pass"
[[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_verify_failed_after_reconcile"
exit 0
fi
exit 75
;;
esac