#!/usr/bin/env python3 """Run a bounded SigNoz portable-metadata restore in an isolated Docker stack. The controller creates only deterministic, run-owned Docker resources. It never pulls or builds an image, never joins a production network or volume, and removes its three containers, four volumes, internal network and private workspace on every terminal. Secrets are generated in memory or stored only in mode-0400 files below the private workspace; they are never command-line arguments, stdout fields or receipt fields. """ from __future__ import annotations import argparse import hashlib import json import os import secrets import shutil import signal import socket import stat import string import subprocess # nosec B404 -- fixed executable and argv vectors only import time import urllib.error import urllib.request from dataclasses import dataclass from pathlib import Path from typing import Any, Callable from signoz_metadata_contract import ( DEFAULT_POLICY, ContractError, canonical_json_bytes, load_policy, read_json_object, receipt, require_no_symlink_components, sha256_bytes, validate_identity, validate_new_private_destination, write_new_private_atomic, ) DOCKER = "/usr/bin/docker" PYTHON = "/usr/bin/python3" RESTORE_DRIVER = Path(__file__).with_name("signoz-metadata-restore-drill.py") EXPORT_VERIFIER = Path(__file__).with_name("verify-signoz-metadata-export.py") DEFAULT_CLUSTER_CONFIG = Path(__file__).with_name("isolated-cluster.xml") PRIMARY_LABEL = "awoooi.signoz.metadata.restore.run_id" IDENTITY_LABEL = "com.awoooi.restore.identity" EXPECTED_IMAGES = { "signoz": ( "signoz/signoz:v0.113.0", "sha256:c17991d10966aedcb0d4d83805a0baf5310f1553d90611629036fcfaee8d969b", ), "clickhouse": ( "clickhouse/clickhouse-server:25.5.6", "sha256:8248e5926d7304400e44ecdf0c7181fbde28e6a9b1d647780eca839f34c00cc6", ), "zookeeper": ( "signoz/zookeeper:3.7.1", "sha256:3ab0e8f032ab58f14ac1e929ac40305a4a464cf320bdfe4151d62d6922729ba0", ), } EXPECTED_PRODUCTION = { "signoz": EXPECTED_IMAGES["signoz"], "signoz-clickhouse": EXPECTED_IMAGES["clickhouse"], "signoz-zookeeper-1": EXPECTED_IMAGES["zookeeper"], "signoz-otel-collector": ( "signoz/signoz-otel-collector:v0.144.1", "sha256:47cbed488288df7bac6aab3665275bec6d6e24fe5c545c66914827a891e538d8", ), } EXPECTED_ENTRYPOINT = ["./signoz", "server"] EXPECTED_SERVER_ENV = { "SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN": "tcp://clickhouse:9000", "SIGNOZ_SQLSTORE_SQLITE_PATH": "/var/lib/signoz/signoz.db", } EXPECTED_SERVER_MOUNT = "/var/lib/signoz" PORT_RETRIES = 120 PORT_RETRY_INTERVAL_SECONDS = 1.0 CLICKHOUSE_RETRIES = 90 CLICKHOUSE_RETRY_INTERVAL_SECONDS = 1.0 COMMAND_TIMEOUT_SECONDS = 60 CLEANUP_COMMAND_TIMEOUT_SECONDS = 3 CLEANUP_RECONCILIATION_ATTEMPTS = 5 CLEANUP_STABLE_PASSES = 3 CLEANUP_RECONCILIATION_DELAY_SECONDS = 1.0 class SignalAbort(ContractError): """A handled termination signal that still requires owned cleanup.""" @dataclass(frozen=True) class ResourcePlan: identity: str short_id: str prefix: str canonical_id: str network: str zookeeper_container: str clickhouse_container: str server_container: str zookeeper_data_volume: str zookeeper_log_volume: str clickhouse_data_volume: str server_data_volume: str workspace: Path @property def containers(self) -> tuple[str, ...]: return ( self.server_container, self.clickhouse_container, self.zookeeper_container, ) @property def volumes(self) -> tuple[str, ...]: return ( self.zookeeper_data_volume, self.zookeeper_log_volume, self.clickhouse_data_volume, self.server_data_volume, ) @dataclass(frozen=True) class CommandResult: returncode: int stdout: str Runner = Callable[[list[str], int], CommandResult] def derive_identity(trace_id: str, run_id: str, work_item_id: str) -> str: material = ( trace_id.encode("utf-8") + b"\0" + run_id.encode("utf-8") + b"\0" + work_item_id.encode("utf-8") ) return hashlib.sha256(material).hexdigest() def build_resource_plan( trace_id: str, run_id: str, work_item_id: str, *, workspace_parent: Path = Path("/tmp"), ) -> ResourcePlan: identity = derive_identity(trace_id, run_id, work_item_id) short_id = identity[:16] prefix = f"awoooi-signoz-md-restore-{short_id}" return ResourcePlan( identity=identity, short_id=short_id, prefix=prefix, canonical_id=f"ephemeral-signoz-metadata-restore-{short_id}", network=f"{prefix}-net", zookeeper_container=f"{prefix}-zookeeper", clickhouse_container=f"{prefix}-clickhouse", server_container=f"{prefix}-server", zookeeper_data_volume=f"{prefix}-zk-data", zookeeper_log_volume=f"{prefix}-zk-log", clickhouse_data_volume=f"{prefix}-ch-data", server_data_volume=f"{prefix}-server-data", workspace=workspace_parent / f"{prefix}-private", ) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() mode = parser.add_mutually_exclusive_group(required=True) mode.add_argument("--check", action="store_true") mode.add_argument("--apply", action="store_true") mode.add_argument("--verify-cleanup", action="store_true") parser.add_argument("--bundle-dir") parser.add_argument("--policy", default=str(DEFAULT_POLICY)) parser.add_argument("--cluster-config", default=str(DEFAULT_CLUSTER_CONFIG)) parser.add_argument("--trace-id", required=True) parser.add_argument("--run-id", required=True) parser.add_argument("--work-item-id", default="P0-OBS-002") parser.add_argument("--receipt-file") parser.add_argument("--apply-receipt-file") args = parser.parse_args() if args.apply and not args.receipt_file: parser.error("receipt_file_required_for_restore_apply") return args def default_runner(argv: list[str], timeout: int) -> CommandResult: try: result = subprocess.run( # nosec B603 argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=timeout, check=False, env={ "PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "LANG": "C.UTF-8", "LC_ALL": "C.UTF-8", }, ) except (OSError, subprocess.TimeoutExpired) as exc: raise ContractError("bounded_command_transport_failed") from exc return CommandResult(result.returncode, result.stdout) def run_required( runner: Runner, argv: list[str], *, label: str, timeout: int = COMMAND_TIMEOUT_SECONDS, ) -> str: result = runner(argv, timeout) if result.returncode != 0: raise ContractError(f"{label}_failed") return result.stdout.strip() def run_optional( runner: Runner, argv: list[str], *, timeout: int = COMMAND_TIMEOUT_SECONDS, ) -> CommandResult: return runner(argv, timeout) def validate_restore_policy(policy: dict[str, Any]) -> dict[str, Any]: isolation = policy.get("restore_isolation") if not isinstance(isolation, dict): raise ContractError("restore_isolation_policy_missing") if isolation.get("images") != { key: {"ref": ref, "id": image_id} for key, (ref, image_id) in EXPECTED_IMAGES.items() }: raise ContractError("restore_image_policy_not_exact") if isolation.get("required_image_ref") != EXPECTED_IMAGES["signoz"][0]: raise ContractError("restore_signoz_image_ref_not_exact") if isolation.get("required_image_id") != EXPECTED_IMAGES["signoz"][1]: raise ContractError("restore_signoz_image_id_not_exact") if isolation.get("resource_prefix") != "awoooi-signoz-md-restore-": raise ContractError("restore_resource_prefix_not_exact") if isolation.get("canonical_id_prefix") != ( "ephemeral-signoz-metadata-restore-" ): raise ContractError("restore_canonical_prefix_not_exact") if isolation.get("resource_label_key") != PRIMARY_LABEL: raise ContractError("restore_primary_label_not_exact") if isolation.get("secondary_resource_label_key") != IDENTITY_LABEL: raise ContractError("restore_identity_label_not_exact") if isolation.get("private_workspace_parent") != "/tmp": raise ContractError("restore_workspace_parent_not_exact") if isolation.get("server_container_port") != 8080: raise ContractError("restore_server_port_not_exact") if isolation.get("loopback_publish_host") != "127.0.0.1": raise ContractError("restore_publish_host_not_exact") if any( isolation.get(key) is not expected for key, expected in ( ("image_pull_allowed", False), ("production_network_attachment_allowed", False), ("production_volume_attachment_allowed", False), ("ephemeral_volumes_only", True), ("cleanup_controller_required", True), ("zero_residue_verifier_required", True), ) ): raise ContractError("restore_isolation_boolean_contract_invalid") startup = isolation.get("startup_contract") if not isinstance(startup, dict) or startup.get("reviewed") is not True: raise ContractError("restore_startup_contract_not_reviewed") if startup.get("server_entrypoint") != EXPECTED_ENTRYPOINT: raise ContractError("restore_server_entrypoint_not_exact") if startup.get("server_environment") != EXPECTED_SERVER_ENV: raise ContractError("restore_server_environment_not_exact") if startup.get("server_data_mount_destination") != EXPECTED_SERVER_MOUNT: raise ContractError("restore_server_mount_not_exact") if startup.get("clickhouse_network_alias") != "clickhouse": raise ContractError("restore_clickhouse_alias_not_exact") if startup.get("zookeeper_network_alias") != "restore-zookeeper": raise ContractError("restore_zookeeper_alias_not_exact") key_presence = startup.get("required_production_key_presence") if ( not isinstance(key_presence, list) or len(key_presence) != len(EXPECTED_SERVER_ENV) or sorted(key_presence) != sorted(EXPECTED_SERVER_ENV) ): raise ContractError("restore_production_key_presence_not_exact") if startup.get("production_dsn_class") != "tcp://clickhouse:9000*": raise ContractError("restore_production_dsn_class_not_exact") if startup.get("production_sqlite_path") != EXPECTED_SERVER_ENV[ "SIGNOZ_SQLSTORE_SQLITE_PATH" ]: raise ContractError("restore_production_sqlite_path_not_exact") if startup.get("secret_value_output_allowed") is not False: raise ContractError("restore_secret_output_must_be_forbidden") if startup.get("full_environment_output_allowed") is not False: raise ContractError("restore_full_environment_output_must_be_forbidden") expected_continuity = { key.replace("-", "_"): { "container": key, "image_ref": ref, "image_id": image_id, } for key, (ref, image_id) in EXPECTED_PRODUCTION.items() } # The policy uses semantic keys for the production containers. actual_continuity = isolation.get("production_continuity") if not isinstance(actual_continuity, dict): raise ContractError("restore_production_continuity_missing") normalized_actual = { str(value.get("container", "")).replace("-", "_"): value for value in actual_continuity.values() if isinstance(value, dict) } if normalized_actual != expected_continuity: raise ContractError("restore_production_continuity_not_exact") return isolation def require_regular_cluster_config(path: Path) -> None: try: metadata = path.lstat() except OSError as exc: raise ContractError("isolated_cluster_config_unavailable") from exc if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): raise ContractError("isolated_cluster_config_not_regular") content = path.read_text(encoding="utf-8") for marker in ( "restore-zookeeper", "restore-clickhouse", '', ): if marker not in content: raise ContractError("isolated_cluster_config_contract_invalid") def label_args(plan: ResourcePlan, run_id: str) -> list[str]: return [ "--label", f"{PRIMARY_LABEL}={run_id}", "--label", f"{IDENTITY_LABEL}={plan.identity}", ] def inspect_image_id(runner: Runner, ref: str) -> str: return run_required( runner, [DOCKER, "image", "inspect", "--format={{.Id}}", ref], label="docker_image_inspect", ) def verify_exact_images(runner: Runner) -> dict[str, dict[str, str]]: observed: dict[str, dict[str, str]] = {} for role, (ref, expected_id) in EXPECTED_IMAGES.items(): actual_id = inspect_image_id(runner, ref) if actual_id != expected_id: raise ContractError(f"restore_image_id_mismatch_{role}") observed[role] = {"ref": ref, "id": actual_id} return observed def inspect_json( runner: Runner, kind: str, name: str, *, timeout: int = COMMAND_TIMEOUT_SECONDS, ) -> dict[str, Any] | None: result = run_optional(runner, [DOCKER, kind, "inspect", name], timeout=timeout) if result.returncode != 0: return None try: value = json.loads(result.stdout) except json.JSONDecodeError as exc: raise ContractError(f"docker_{kind}_inspect_invalid") from exc if not isinstance(value, list) or len(value) != 1 or not isinstance(value[0], dict): raise ContractError(f"docker_{kind}_inspect_invalid") return value[0] def production_snapshot( runner: Runner, *, timeout: int = COMMAND_TIMEOUT_SECONDS, ) -> dict[str, dict[str, Any]]: snapshot: dict[str, dict[str, Any]] = {} for container, (expected_ref, expected_image_id) in EXPECTED_PRODUCTION.items(): data = inspect_json(runner, "container", container, timeout=timeout) if data is None: raise ContractError(f"production_container_missing_{container}") config = data.get("Config") state = data.get("State") if not isinstance(config, dict) or not isinstance(state, dict): raise ContractError("production_container_inspect_shape_invalid") row = { "id": data.get("Id"), "image_id": data.get("Image"), "image_ref": config.get("Image"), "running": state.get("Running"), "started_at": state.get("StartedAt"), "restart_count": data.get("RestartCount"), } if ( not isinstance(row["id"], str) or len(row["id"]) != 64 or row["image_id"] != expected_image_id or row["image_ref"] != expected_ref or row["running"] is not True or not isinstance(row["started_at"], str) or not isinstance(row["restart_count"], int) ): raise ContractError(f"production_container_identity_invalid_{container}") snapshot[container] = row return snapshot def image_inventory( runner: Runner, *, timeout: int = COMMAND_TIMEOUT_SECONDS, ) -> list[str]: output = run_required( runner, [ DOCKER, "image", "ls", "--no-trunc", "--format={{.ID}}|{{.Repository}}:{{.Tag}}", ], label="docker_image_inventory", timeout=timeout, ) rows = sorted(line for line in output.splitlines() if line) if not rows: raise ContractError("docker_image_inventory_empty") return rows def require_daemon_readable( runner: Runner, *, timeout: int = COMMAND_TIMEOUT_SECONDS, ) -> None: output = run_required( runner, [DOCKER, "info", "--format={{.ServerVersion}}"], label="docker_daemon_inventory", timeout=timeout, ) if not output or any(character.isspace() for character in output): raise ContractError("docker_daemon_inventory_invalid") def inspect_resource_labels( runner: Runner, kind: str, name: str, *, timeout: int = COMMAND_TIMEOUT_SECONDS, ) -> dict[str, str] | None: data = inspect_json(runner, kind, name, timeout=timeout) if data is None: return None if kind == "container": labels = data.get("Config", {}).get("Labels") else: labels = data.get("Labels") if labels is None: return {} if not isinstance(labels, dict) or not all( isinstance(key, str) and isinstance(value, str) for key, value in labels.items() ): raise ContractError(f"docker_{kind}_labels_invalid") return labels def assert_absent_or_owned( runner: Runner, plan: ResourcePlan, run_id: str, kind: str, name: str, ) -> bool: labels = inspect_resource_labels( runner, kind, name, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS, ) if labels is None: return False if labels.get(PRIMARY_LABEL) != run_id or labels.get(IDENTITY_LABEL) != plan.identity: raise ContractError(f"restore_{kind}_name_collision_not_owned") return True def require_no_preexisting_resources( runner: Runner, plan: ResourcePlan, run_id: str, ) -> None: for name in plan.containers: if assert_absent_or_owned(runner, plan, run_id, "container", name): raise ContractError("restore_owned_container_residue_requires_reconcile") for name in plan.volumes: if assert_absent_or_owned(runner, plan, run_id, "volume", name): raise ContractError("restore_owned_volume_residue_requires_reconcile") if assert_absent_or_owned(runner, plan, run_id, "network", plan.network): raise ContractError("restore_owned_network_residue_requires_reconcile") if plan.workspace.exists() or plan.workspace.is_symlink(): raise ContractError("restore_private_workspace_residue_requires_reconcile") def verify_bundle(args: argparse.Namespace, runner: Runner) -> None: if not args.bundle_dir: raise ContractError("bundle_dir_required") bundle = Path(args.bundle_dir) output = run_required( runner, [ PYTHON, str(EXPORT_VERIFIER), "--bundle-dir", str(bundle), "--policy", str(args.policy), "--expected-trace-id", args.trace_id, "--expected-run-id", args.run_id, "--expected-work-item-id", args.work_item_id, ], label="independent_bundle_verifier", ) try: value = json.loads(output) except json.JSONDecodeError as exc: raise ContractError("independent_bundle_verifier_receipt_invalid") from exc if value.get("terminal") != "pass_export_verified_restore_pending": raise ContractError("independent_bundle_verifier_terminal_invalid") def create_workspace(plan: ResourcePlan) -> None: plan.workspace.mkdir(mode=0o700) metadata = plan.workspace.lstat() if ( not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != os.geteuid() or stat.S_IMODE(metadata.st_mode) != 0o700 ): raise ContractError("restore_workspace_owner_or_mode_invalid") def write_private(path: Path, data: bytes) -> None: write_new_private_atomic(path, data, label="restore_private_material") path.chmod(0o400) def write_state(path: Path, value: dict[str, Any]) -> None: payload = canonical_json_bytes(value) if path.exists(): metadata = path.lstat() if ( stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.geteuid() or stat.S_IMODE(metadata.st_mode) != 0o600 ): raise ContractError("restore_state_owner_or_mode_invalid") temp = path.with_name(f".{path.name}.{os.getpid()}.tmp") write_new_private_atomic(temp, payload, label="restore_state_update") os.replace(temp, path) path.chmod(0o600) else: write_new_private_atomic(path, payload, label="restore_state") def create_resources( runner: Runner, plan: ResourcePlan, run_id: str, cluster_config: Path, ) -> None: labels = label_args(plan, run_id) run_required( runner, [DOCKER, "network", "create", "--internal", *labels, plan.network], label="restore_network_create", ) for name in plan.volumes: run_required( runner, [DOCKER, "volume", "create", *labels, name], label="restore_volume_create", ) zk_ref = EXPECTED_IMAGES["zookeeper"][0] run_required( runner, [ DOCKER, "run", "--detach", "--pull=never", "--restart=no", "--name", plan.zookeeper_container, "--network", plan.network, "--network-alias", "restore-zookeeper", *labels, "--env", "ZOO_MY_ID=1", "--env", "ZOO_SERVERS=server.1=0.0.0.0:2888:3888;2181", "--volume", f"{plan.zookeeper_data_volume}:/data", "--volume", f"{plan.zookeeper_log_volume}:/datalog", zk_ref, ], label="restore_zookeeper_start", ) clickhouse_ref = EXPECTED_IMAGES["clickhouse"][0] run_required( runner, [ DOCKER, "run", "--detach", "--pull=never", "--restart=no", "--name", plan.clickhouse_container, "--network", plan.network, "--network-alias", "clickhouse", "--network-alias", "restore-clickhouse", *labels, "--env", f"CLICKHOUSE_RESTORE_REPLICA=restore-{plan.short_id}", "--env", "CLICKHOUSE_SKIP_USER_SETUP=1", "--volume", f"{plan.clickhouse_data_volume}:/var/lib/clickhouse", "--volume", f"{cluster_config}:/etc/clickhouse-server/config.d/isolated-cluster.xml:ro", clickhouse_ref, ], label="restore_clickhouse_start", ) wait_for_clickhouse(runner, plan) signoz_ref = EXPECTED_IMAGES["signoz"][0] run_required( runner, [ DOCKER, "run", "--detach", "--pull=never", "--restart=no", "--name", plan.server_container, "--network", plan.network, *labels, "--publish", "127.0.0.1::8080", "--env", "SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN=tcp://clickhouse:9000", "--env", "SIGNOZ_SQLSTORE_SQLITE_PATH=/var/lib/signoz/signoz.db", "--volume", f"{plan.server_data_volume}:/var/lib/signoz", signoz_ref, ], label="restore_signoz_start", ) def wait_for_clickhouse(runner: Runner, plan: ResourcePlan) -> None: for attempt in range(CLICKHOUSE_RETRIES): result = run_optional( runner, [ DOCKER, "exec", plan.clickhouse_container, "clickhouse-client", "--query", "SELECT 1", ], timeout=5, ) if result.returncode == 0 and result.stdout.strip() == "1": return if attempt + 1 < CLICKHOUSE_RETRIES: time.sleep(CLICKHOUSE_RETRY_INTERVAL_SECONDS) raise ContractError("isolated_clickhouse_readiness_timeout") def verify_runtime_container_images(runner: Runner, plan: ResourcePlan) -> None: expected = { plan.server_container: EXPECTED_IMAGES["signoz"], plan.clickhouse_container: EXPECTED_IMAGES["clickhouse"], plan.zookeeper_container: EXPECTED_IMAGES["zookeeper"], } for name, (expected_ref, expected_id) in expected.items(): data = inspect_json(runner, "container", name) if data is None: raise ContractError("restore_runtime_container_missing") config = data.get("Config", {}) host_config = data.get("HostConfig", {}) if ( data.get("Image") != expected_id or config.get("Image") != expected_ref or host_config.get("RestartPolicy", {}).get("Name") not in {"", "no"} ): raise ContractError("restore_runtime_container_image_or_restart_drift") def discover_loopback_port(runner: Runner, plan: ResourcePlan) -> int: output = run_required( runner, [DOCKER, "port", plan.server_container, "8080/tcp"], label="restore_listener_discovery", ) lines = [line.strip() for line in output.splitlines() if line.strip()] if len(lines) != 1 or not lines[0].startswith("127.0.0.1:"): raise ContractError("restore_listener_not_loopback_ephemeral") port_text = lines[0].rsplit(":", 1)[-1] if not port_text.isdigit(): raise ContractError("restore_listener_port_invalid") port = int(port_text) if port <= 0 or port > 65535 or port == 8080: raise ContractError("restore_listener_port_not_ephemeral") return port def request_json( url: str, *, method: str = "GET", payload: dict[str, Any] | None = None, authorization: str | None = None, timeout: int = 10, ) -> dict[str, Any]: body = canonical_json_bytes(payload) if payload is not None else None request = urllib.request.Request(url, data=body, method=method) request.add_header("Accept", "application/json") if body is not None: request.add_header("Content-Type", "application/json") if authorization is not None: request.add_header("Authorization", authorization) try: with urllib.request.urlopen(request, timeout=timeout) as response: # nosec B310 if response.status < 200 or response.status >= 300: raise ContractError("isolated_api_unexpected_status") raw = response.read(1024 * 1024 + 1) except (urllib.error.URLError, TimeoutError, OSError) as exc: raise ContractError("isolated_api_request_failed") from exc if len(raw) > 1024 * 1024: raise ContractError("isolated_api_response_too_large") try: value = json.loads(raw) except (UnicodeError, json.JSONDecodeError) as exc: raise ContractError("isolated_api_response_invalid") from exc if not isinstance(value, dict): raise ContractError("isolated_api_response_not_object") return value def wait_for_health_and_version(base_url: str) -> None: last_health_ok = False for _ in range(PORT_RETRIES): try: health = request_json(f"{base_url}/api/v1/health", timeout=2) last_health_ok = health.get("status") == "ok" if last_health_ok: break except ContractError: pass time.sleep(PORT_RETRY_INTERVAL_SECONDS) if not last_health_ok: raise ContractError("isolated_signoz_health_timeout") version = request_json(f"{base_url}/api/v1/version", timeout=5) if version.get("version") != "v0.113.0": raise ContractError("isolated_signoz_version_mismatch") def random_password() -> str: alphabet = string.ascii_letters + string.digits + "-_" return "R!" + "".join(secrets.choice(alphabet) for _ in range(46)) def bootstrap_isolated_session(base_url: str, workspace: Path) -> Path: email = f"restore-{secrets.token_hex(12)}@example.invalid" password = random_password() register = request_json( f"{base_url}/api/v1/register", method="POST", payload={ "email": email, "orgDisplayName": "AWOOOI isolated restore", "password": password, }, ) data = register.get("data") register_body = data if isinstance(data, dict) else register org_id = register_body.get("orgId") if not isinstance(org_id, str) or not org_id: raise ContractError("isolated_registration_org_id_missing") login = request_json( f"{base_url}/api/v2/sessions/email_password", method="POST", payload={"email": email, "password": password, "orgId": org_id}, ) login_data = login.get("data") if not isinstance(login_data, dict): login_data = login access_token = ( login_data.get("accessToken") if isinstance(login_data, dict) else None ) refresh_token = ( login_data.get("refreshToken") if isinstance(login_data, dict) else None ) if ( not isinstance(access_token, str) or len(access_token) < 16 or not isinstance(refresh_token, str) or len(refresh_token) < 16 ): raise ContractError("isolated_session_token_shape_invalid") token_path = workspace / "session-access-token" write_private(token_path, access_token.encode("ascii") + b"\n") return token_path def isolation_receipt_value( args: argparse.Namespace, plan: ResourcePlan, base_url: str, images: dict[str, dict[str, str]], production_before: dict[str, dict[str, Any]], image_inventory_before: list[str], ) -> dict[str, Any]: return { "schema": "awoooi_signoz_metadata_restore_isolation_v1", "trace_id": args.trace_id, "run_id": args.run_id, "work_item_id": args.work_item_id, "target": { "identity": plan.identity, "prefix": plan.prefix, "canonical_id": plan.canonical_id, "base_url_sha256": sha256_bytes(base_url.encode("utf-8")), "image_ref": EXPECTED_IMAGES["signoz"][0], "image_id": EXPECTED_IMAGES["signoz"][1], "images": images, "image_present_locally": True, "image_pull_performed": False, "image_build_performed": False, "production_network_attached": False, "production_volume_attached": False, "ephemeral_volumes_only": True, "cleanup_controller_armed": True, "zero_residue_verifier_armed": True, "network_scope": "loopback_and_internal_only", "resource_label_key": PRIMARY_LABEL, "resource_label_value": args.run_id, "identity_label_key": IDENTITY_LABEL, "identity_label_value": plan.identity, "resources": { "network": plan.network, "containers": list(plan.containers), "volumes": list(plan.volumes), "workspace": str(plan.workspace), }, "listener": {"host": "127.0.0.1", "port": int(base_url.rsplit(":", 1)[1])}, "authentication_mode": "isolated_session_bearer", }, "production_before": production_before, "image_inventory_before": image_inventory_before, } def invoke_restore_driver( args: argparse.Namespace, runner: Runner, base_url: str, isolation_receipt: Path, token_file: Path, mode: str, ) -> dict[str, Any]: argv = [ PYTHON, str(RESTORE_DRIVER), mode, "--bundle-dir", str(args.bundle_dir), "--policy", str(args.policy), "--target-base-url", base_url, "--credential-file", str(token_file), "--auth-mode", "isolated_session_bearer", "--isolation-receipt", str(isolation_receipt), "--trace-id", args.trace_id, "--run-id", args.run_id, "--work-item-id", args.work_item_id, ] output = run_required( runner, argv, label=f"isolated_restore_driver_{mode.removeprefix('--')}", timeout=900, ) try: value = json.loads(output) except json.JSONDecodeError as exc: raise ContractError("isolated_restore_driver_receipt_invalid") from exc expected = ( "check_pass_no_write" if mode == "--check" else "partial_degraded_cleanup_pending" ) if value.get("terminal") != expected: raise ContractError("isolated_restore_driver_terminal_invalid") return value def tcp_connectable(host: str, port: int) -> bool: try: with socket.create_connection((host, port), timeout=2): return True except OSError: return False def remove_owned_resource( runner: Runner, plan: ResourcePlan, run_id: str, kind: str, name: str, ) -> bool: labels = inspect_resource_labels(runner, kind, name) if labels is None: return False if labels.get(PRIMARY_LABEL) != run_id or labels.get(IDENTITY_LABEL) != plan.identity: raise ContractError(f"cleanup_{kind}_ownership_mismatch") if kind == "container": argv = [DOCKER, "rm", "--force", name] elif kind == "volume": argv = [DOCKER, "volume", "rm", name] else: argv = [DOCKER, "network", "rm", name] run_required( runner, argv, label=f"cleanup_{kind}_remove", timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS, ) return True def remove_private_workspace(plan: ResourcePlan) -> None: if not plan.workspace.exists() and not plan.workspace.is_symlink(): return metadata = plan.workspace.lstat() if ( stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != os.geteuid() or stat.S_IMODE(metadata.st_mode) != 0o700 ): raise ContractError("cleanup_workspace_ownership_or_mode_mismatch") shutil.rmtree(plan.workspace) def label_inventory_count( runner: Runner, kind: str, label: str, ) -> int: if kind == "container": argv = [DOCKER, "ps", "-a", "--quiet", "--filter", f"label={label}"] else: argv = [DOCKER, kind, "ls", "--quiet", "--filter", f"label={label}"] output = run_required( runner, argv, label=f"cleanup_{kind}_inventory", timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS, ) return len([line for line in output.splitlines() if line.strip()]) def capture_owned_listener_before_cleanup( runner: Runner, plan: ResourcePlan, run_id: str, ) -> dict[str, Any] | None: data = inspect_json( runner, "container", plan.server_container, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS, ) if data is None: return None labels = data.get("Config", {}).get("Labels") if ( not isinstance(labels, dict) or labels.get(PRIMARY_LABEL) != run_id or labels.get(IDENTITY_LABEL) != plan.identity ): raise ContractError("cleanup_server_listener_ownership_mismatch") ports = data.get("NetworkSettings", {}).get("Ports") bindings = ports.get("8080/tcp") if isinstance(ports, dict) else None if not isinstance(bindings, list) or len(bindings) != 1: raise ContractError("cleanup_server_listener_binding_unknown") binding = bindings[0] if not isinstance(binding, dict): raise ContractError("cleanup_server_listener_binding_invalid") host = binding.get("HostIp") port_text = binding.get("HostPort") if host != "127.0.0.1" or not isinstance(port_text, str) or not port_text.isdigit(): raise ContractError("cleanup_server_listener_not_loopback") port = int(port_text) if port <= 0 or port > 65535: raise ContractError("cleanup_server_listener_port_invalid") return {"host": host, "port": port} def cleanup_and_verify( runner: Runner, plan: ResourcePlan, run_id: str, state: dict[str, Any] | None, ) -> list[str]: baseline_errors: list[str] = [] listener: dict[str, Any] | None = None production_before: dict[str, Any] | None = None image_inventory_before: list[str] | None = None if state is not None: state_listener = state.get("listener") if isinstance(state_listener, dict): listener = state_listener state_production = state.get("production_before") if isinstance(state_production, dict): production_before = state_production state_images = state.get("image_inventory_before") if isinstance(state_images, list) and all( isinstance(item, str) for item in state_images ): image_inventory_before = state_images try: if listener is None: listener = capture_owned_listener_before_cleanup( runner, plan, run_id ) if production_before is None: production_before = production_snapshot( runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS ) if image_inventory_before is None: image_inventory_before = image_inventory( runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS ) except ContractError as exc: baseline_errors.append(str(exc)) stable_passes = 0 last_errors: list[str] = [] for attempt in range(CLEANUP_RECONCILIATION_ATTEMPTS): attempt_errors = list(baseline_errors) removed_during_attempt = False # Re-run removal on every pass. A Docker API request interrupted after # daemon acceptance may materialize a labeled resource after the first # inventory; repeated ownership-gated removal closes that race. for name in plan.containers: try: removed_during_attempt = ( remove_owned_resource(runner, plan, run_id, "container", name) or removed_during_attempt ) except ContractError as exc: attempt_errors.append(str(exc)) for name in plan.volumes: try: removed_during_attempt = ( remove_owned_resource(runner, plan, run_id, "volume", name) or removed_during_attempt ) except ContractError as exc: attempt_errors.append(str(exc)) try: removed_during_attempt = ( remove_owned_resource(runner, plan, run_id, "network", plan.network) or removed_during_attempt ) except ContractError as exc: attempt_errors.append(str(exc)) try: require_daemon_readable( runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS ) for name in plan.containers: if inspect_json( runner, "container", name, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS, ) is not None: attempt_errors.append("cleanup_exact_container_residue") for name in plan.volumes: if inspect_json( runner, "volume", name, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS, ) is not None: attempt_errors.append("cleanup_exact_volume_residue") if inspect_json( runner, "network", plan.network, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS, ) is not None: attempt_errors.append("cleanup_exact_network_residue") for label in ( f"{PRIMARY_LABEL}={run_id}", f"{IDENTITY_LABEL}={plan.identity}", ): for kind in ("container", "volume", "network"): if label_inventory_count(runner, kind, label) != 0: attempt_errors.append(f"cleanup_{kind}_label_residue") except ContractError as exc: attempt_errors.append(str(exc)) if listener is not None: host = listener.get("host") port = listener.get("port") if isinstance(host, str) and isinstance(port, int): if tcp_connectable(host, port): attempt_errors.append("cleanup_tcp_listener_still_present") else: attempt_errors.append("cleanup_listener_state_invalid") if production_before is None or image_inventory_before is None: attempt_errors.append("cleanup_continuity_baseline_unavailable") else: try: if production_snapshot( runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS ) != production_before: attempt_errors.append( "cleanup_production_identity_continuity_failed" ) if image_inventory( runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS ) != image_inventory_before: attempt_errors.append("cleanup_image_inventory_drift") except ContractError as exc: attempt_errors.append(str(exc)) try: remove_private_workspace(plan) except ContractError as exc: attempt_errors.append(str(exc)) if plan.workspace.exists() or plan.workspace.is_symlink(): attempt_errors.append("cleanup_private_workspace_residue") if removed_during_attempt: attempt_errors.append("cleanup_owned_resource_reappeared") last_errors = sorted(set(attempt_errors)) if last_errors: stable_passes = 0 else: stable_passes += 1 if stable_passes >= CLEANUP_STABLE_PASSES: return [] if attempt + 1 < CLEANUP_RECONCILIATION_ATTEMPTS: time.sleep(CLEANUP_RECONCILIATION_DELAY_SECONDS) return sorted(set(last_errors + ["cleanup_stable_zero_residue_not_observed"])) def load_state_if_safe(plan: ResourcePlan) -> dict[str, Any] | None: path = plan.workspace / "state.json" if not path.exists(): return None value = read_json_object(path, label="restore_state") if value.get("schema") != "awoooi_signoz_metadata_restore_state_v1": raise ContractError("restore_state_schema_invalid") if value.get("identity") != plan.identity: raise ContractError("restore_state_identity_mismatch") if not isinstance(value.get("production_before"), dict): raise ContractError("restore_state_production_baseline_invalid") images = value.get("image_inventory_before") if not isinstance(images, list) or not all(isinstance(item, str) for item in images): raise ContractError("restore_state_image_baseline_invalid") listener = value.get("listener") if listener is not None and ( not isinstance(listener, dict) or listener.get("host") != "127.0.0.1" or not isinstance(listener.get("port"), int) or listener["port"] <= 0 or listener["port"] > 65535 ): raise ContractError("restore_state_listener_invalid") return value def run_check( args: argparse.Namespace, runner: Runner, plan: ResourcePlan, policy: dict[str, Any], cluster_config: Path, ) -> dict[str, Any]: validate_restore_policy(policy) require_regular_cluster_config(cluster_config) verify_bundle(args, runner) require_daemon_readable(runner) verify_exact_images(runner) production_snapshot(runner) image_inventory(runner) require_no_preexisting_resources(runner, plan, args.run_id) value = receipt( trace_id=args.trace_id, run_id=args.run_id, work_item_id=args.work_item_id, phase="terminal", terminal="check_pass_no_write", detail="exact_images_startup_contract_production_continuity_and_zero_collision_pass", ) value.update( { "identity": plan.identity, "resourcePrefix": plan.prefix, "resourceWritePerformed": False, "secretValueRead": False, "completionClaim": False, } ) return value def run_apply( args: argparse.Namespace, runner: Runner, plan: ResourcePlan, policy: dict[str, Any], cluster_config: Path, ) -> dict[str, Any]: run_check(args, runner, plan, policy, cluster_config) create_workspace(plan) state_path = plan.workspace / "state.json" images = verify_exact_images(runner) before = production_snapshot(runner) inventory_before = image_inventory(runner) state: dict[str, Any] = { "schema": "awoooi_signoz_metadata_restore_state_v1", "identity": plan.identity, "trace_id": args.trace_id, "run_id": args.run_id, "work_item_id": args.work_item_id, "production_before": before, "image_inventory_before": inventory_before, "listener": None, } write_state(state_path, state) create_resources(runner, plan, args.run_id, cluster_config) verify_runtime_container_images(runner, plan) port = discover_loopback_port(runner, plan) state["listener"] = {"host": "127.0.0.1", "port": port} write_state(state_path, state) base_url = f"http://127.0.0.1:{port}" wait_for_health_and_version(base_url) token_path = bootstrap_isolated_session(base_url, plan.workspace) isolation_path = plan.workspace / "isolation-receipt.json" isolation = isolation_receipt_value( args, plan, base_url, images, before, inventory_before, ) write_new_private_atomic( isolation_path, canonical_json_bytes(isolation), label="isolation_receipt", ) invoke_restore_driver( args, runner, base_url, isolation_path, token_path, "--check", ) restore = invoke_restore_driver( args, runner, base_url, isolation_path, token_path, "--apply", ) value = receipt( trace_id=args.trace_id, run_id=args.run_id, work_item_id=args.work_item_id, phase="terminal", terminal="portable_metadata_restore_verified_cleanup_pending", detail="portable_assets_semantic_readback_pass_cleanup_pending", ) value["restoreSemanticDigest"] = restore.get("portable_semantic_sha256") value["restoredAssetCounts"] = restore.get("restored_asset_counts") value["completionClaim"] = False return value def executor_success( args: argparse.Namespace, plan: ResourcePlan, restore_result: dict[str, Any], ) -> dict[str, Any]: value = receipt( trace_id=args.trace_id, run_id=args.run_id, work_item_id=args.work_item_id, phase="terminal", terminal=( "portable_metadata_restore_verified_zero_residue_" "pending_independent_verifier" ), detail=( "portable_assets_semantic_parity_and_executor_cleanup_verified_" "pending_independent_verifier" ), ) value.update( { "identity": plan.identity, "resourcePrefix": plan.prefix, "completionScope": "portable_assets_only", "portableAssetIds": [ "dashboards", "alert_rules", "explorer_views", ], "completionClaim": False, "executorZeroResidueVerified": True, "independentZeroResidueVerified": False, "fullSqliteCompletionClaim": False, "rolesPreferencesGlobalConfigRestoreClaim": False, "secretBearingAssetsRestoreClaim": False, "productionMutationPerformed": False, "imagePullPerformed": False, "imageBuildPerformed": False, "secretValueInArguments": False, "secretValueInReceipt": False, "restoreSemanticDigest": restore_result.get("restoreSemanticDigest"), "restoredAssetCounts": restore_result.get("restoredAssetCounts"), "zeroResidue": { "exactContainers": 0, "exactVolumes": 0, "exactNetworks": 0, "labelInventory": 0, "tcpListener": 0, "privateWorkspace": 0, }, } ) return value def validate_apply_receipt( path: Path, args: argparse.Namespace, plan: ResourcePlan, ) -> dict[str, Any]: if not path.is_absolute(): raise ContractError("apply_receipt_path_must_be_absolute") require_no_symlink_components(path.parent, label="apply_receipt_parent") try: metadata = path.lstat() except OSError as exc: raise ContractError("apply_receipt_unavailable") from exc if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): raise ContractError("apply_receipt_not_regular") if metadata.st_uid != os.geteuid(): raise ContractError("apply_receipt_owner_mismatch") if stat.S_IMODE(metadata.st_mode) not in {0o400, 0o600}: raise ContractError("apply_receipt_mode_invalid") if metadata.st_size <= 0 or metadata.st_size > 65536: raise ContractError("apply_receipt_size_invalid") value = read_json_object(path, label="apply_receipt") expected = { "schema": "awoooi_signoz_metadata_receipt_v1", "trace_id": args.trace_id, "run_id": args.run_id, "work_item_id": args.work_item_id, "phase": "terminal", "terminal": ( "portable_metadata_restore_verified_zero_residue_" "pending_independent_verifier" ), "identity": plan.identity, "resourcePrefix": plan.prefix, "completionClaim": False, "executorZeroResidueVerified": True, "independentZeroResidueVerified": False, "completionScope": "portable_assets_only", "fullSqliteCompletionClaim": False, "rolesPreferencesGlobalConfigRestoreClaim": False, "secretBearingAssetsRestoreClaim": False, "productionMutationPerformed": False, "imagePullPerformed": False, "imageBuildPerformed": False, "secretValueInArguments": False, "secretValueInReceipt": False, } if any(value.get(key) != expected_value for key, expected_value in expected.items()): raise ContractError("apply_receipt_identity_or_terminal_invalid") digest = value.get("restoreSemanticDigest") counts = value.get("restoredAssetCounts") zero_residue = value.get("zeroResidue") if ( not isinstance(digest, str) or len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest) or not isinstance(counts, dict) or sorted(counts) != ["alert_rules", "dashboards", "explorer_views"] or not all(isinstance(count, int) and count >= 0 for count in counts.values()) or zero_residue != { "exactContainers": 0, "exactVolumes": 0, "exactNetworks": 0, "labelInventory": 0, "tcpListener": 0, "privateWorkspace": 0, } ): raise ContractError("apply_receipt_semantic_evidence_invalid") return value def independent_success( args: argparse.Namespace, plan: ResourcePlan, apply_receipt: dict[str, Any], ) -> dict[str, Any]: value = receipt( trace_id=args.trace_id, run_id=args.run_id, work_item_id=args.work_item_id, phase="independent_restore_verifier", terminal="portable_metadata_restore_independently_verified_zero_residue", detail="immutable_apply_receipt_and_stable_zero_residue_readback_pass", ) value.update( { "verifierIdentity": "signoz_metadata_restore_cleanup_verifier", "applyReceiptValidated": True, "applyReceiptSha256": sha256_bytes( canonical_json_bytes(apply_receipt) ), "identity": plan.identity, "resourcePrefix": plan.prefix, "completionScope": "portable_assets_only", "completionClaim": True, "executorZeroResidueVerified": True, "independentZeroResidueVerified": True, "fullSqliteCompletionClaim": False, "rolesPreferencesGlobalConfigRestoreClaim": False, "secretBearingAssetsRestoreClaim": False, "restoreSemanticDigest": apply_receipt["restoreSemanticDigest"], "restoredAssetCounts": apply_receipt["restoredAssetCounts"], "zeroResidue": { "stablePasses": CLEANUP_STABLE_PASSES, "exactContainers": 0, "exactVolumes": 0, "exactNetworks": 0, "labelInventory": 0, "tcpListener": 0, "privateWorkspace": 0, }, } ) return value def write_terminal(path: Path, value: dict[str, Any]) -> None: write_new_private_atomic(path, canonical_json_bytes(value), label="restore_receipt") def emit(value: dict[str, Any]) -> None: print(json.dumps(value, sort_keys=True, separators=(",", ":"))) def main() -> int: args = parse_args() runner = default_runner plan: ResourcePlan | None = None cleanup_errors: list[str] = [] restore_result: dict[str, Any] | None = None caught_error: str | None = None old_handlers: dict[int, Any] = {} cleanup_reconciliation_completed = False signals_held_until_terminal = False def handle_signal(signum: int, _frame: Any) -> None: # The first handled signal transfers control to finally. Ignore later # handled signals so the owned cleanup cannot be interrupted mid-way. for handled in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM): signal.signal(handled, signal.SIG_IGN) raise SignalAbort(f"signal_{signum}_cleanup_required") def restore_signal_handlers() -> None: for signum, handler in old_handlers.items(): signal.signal(signum, handler) def hold_cleanup_signals() -> None: nonlocal signals_held_until_terminal for handled in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM): signal.signal(handled, signal.SIG_IGN) signals_held_until_terminal = True try: for value, label in ( (args.trace_id, "trace_id"), (args.run_id, "run_id"), (args.work_item_id, "work_item_id"), ): validate_identity(value, label=label) if args.work_item_id != "P0-OBS-002": raise ContractError("work_item_not_allowlisted") if args.receipt_file: validate_new_private_destination( Path(args.receipt_file), label="restore_receipt" ) plan = build_resource_plan(args.trace_id, args.run_id, args.work_item_id) policy = load_policy(Path(args.policy)) cluster_config = Path(args.cluster_config) for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM): old_handlers[signum] = signal.getsignal(signum) signal.signal(signum, handle_signal) if args.check: result = run_check(args, runner, plan, policy, cluster_config) if args.receipt_file: write_terminal(Path(args.receipt_file), result) emit(result) return 0 if args.verify_cleanup: try: state = load_state_if_safe(plan) except (ContractError, OSError, UnicodeError) as exc: state = None cleanup_errors.append(str(exc)) reconciliation_errors = cleanup_and_verify( runner, plan, args.run_id, state ) cleanup_errors.extend(reconciliation_errors) cleanup_reconciliation_completed = True if cleanup_errors: raise ContractError("cleanup_reconciliation_incomplete") if args.apply_receipt_file: apply_receipt = validate_apply_receipt( Path(args.apply_receipt_file), args, plan ) result = independent_success(args, plan, apply_receipt) else: result = receipt( trace_id=args.trace_id, run_id=args.run_id, work_item_id=args.work_item_id, phase="independent_restore_verifier", terminal="pass_zero_residue_verified", detail="deterministic_reconciliation_zero_residue_pass", ) result["completionClaim"] = False result["productionContinuityScope"] = ( "reconciliation_window_only" ) result["restoreCompletionClaim"] = False if args.receipt_file: write_terminal(Path(args.receipt_file), result) emit(result) return 0 restore_result = run_apply(args, runner, plan, policy, cluster_config) except (ContractError, OSError, UnicodeError) as exc: caught_error = str(exc) finally: cleanup_retry_required = bool( plan is not None and ( args.apply or (args.verify_cleanup and not cleanup_reconciliation_completed) ) ) if cleanup_retry_required and plan is not None: # The target timeout sends TERM before its kill-after window. Once # reconciliation starts here, keep all handled signals ignored # through the durable terminal write so cleanup cannot be cut in # half or separated from its receipt. hold_cleanup_signals() try: state = load_state_if_safe(plan) if state is None and args.apply: cleanup_errors.append( "restore_state_missing_for_apply_continuity" ) except (ContractError, OSError, UnicodeError) as exc: state = None cleanup_errors.append(str(exc)) try: reconciliation_errors = cleanup_and_verify( runner, plan, args.run_id, state ) cleanup_errors.extend(reconciliation_errors) cleanup_reconciliation_completed = True except (ContractError, OSError, UnicodeError) as exc: cleanup_errors.append(str(exc)) if plan is not None and args.verify_cleanup and caught_error is not None: # Cleanup may have completed just before a signal or verifier # receipt validation failure. Preserve the failure receipt window # even when no second reconciliation pass is required. hold_cleanup_signals() if not args.apply and not signals_held_until_terminal: restore_signal_handlers() if plan is None: return 1 if caught_error is None and not cleanup_errors and restore_result is not None: result = executor_success(args, plan, restore_result) exit_code = 0 else: error_code = caught_error or "cleanup_reconciliation_incomplete" result = receipt( trace_id=args.trace_id, run_id=args.run_id, work_item_id=args.work_item_id, phase="terminal", terminal=( "failed_zero_residue_verified_no_completion" if not cleanup_errors else "failed_cleanup_reconciliation_required" ), detail=f"isolated_restore_failed_{error_code}", ) result.update( { "completionClaim": False, "fullSqliteCompletionClaim": False, "secretValueInArguments": False, "secretValueInReceipt": False, "zeroResidueVerified": not cleanup_errors, "cleanupErrorCodes": cleanup_errors, } ) exit_code = 1 try: if args.receipt_file: write_terminal(Path(args.receipt_file), result) emit(result) return exit_code except (ContractError, OSError): return 74 finally: # Restore signal behavior only after the durable terminal and stdout # receipt are complete. A second TERM cannot interrupt cleanup or the # write-once receipt persistence window. restore_signal_handlers() if __name__ == "__main__": raise SystemExit(main())