1722 lines
60 KiB
Python
Executable File
1722 lines
60 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Run a bounded SigNoz portable-metadata restore in an isolated Docker stack.
|
|
|
|
The controller creates only deterministic, run-owned Docker resources. It
|
|
never pulls or builds an image, never joins a production network or volume,
|
|
and removes its three containers, four volumes, internal network and private
|
|
workspace on every terminal. Secrets are generated in memory or stored only
|
|
in mode-0400 files below the private workspace; they are never command-line
|
|
arguments, stdout fields or receipt fields.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import secrets
|
|
import shutil
|
|
import signal
|
|
import socket
|
|
import stat
|
|
import string
|
|
import subprocess # nosec B404 -- fixed executable and argv vectors only
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
from signoz_metadata_contract import (
|
|
DEFAULT_POLICY,
|
|
ContractError,
|
|
canonical_json_bytes,
|
|
load_policy,
|
|
read_json_object,
|
|
receipt,
|
|
require_no_symlink_components,
|
|
sha256_bytes,
|
|
validate_identity,
|
|
validate_new_private_destination,
|
|
write_new_private_atomic,
|
|
)
|
|
|
|
|
|
DOCKER = "/usr/bin/docker"
|
|
PYTHON = "/usr/bin/python3"
|
|
RESTORE_DRIVER = Path(__file__).with_name("signoz-metadata-restore-drill.py")
|
|
EXPORT_VERIFIER = Path(__file__).with_name("verify-signoz-metadata-export.py")
|
|
DEFAULT_CLUSTER_CONFIG = Path(__file__).with_name("isolated-cluster.xml")
|
|
PRIMARY_LABEL = "awoooi.signoz.metadata.restore.run_id"
|
|
IDENTITY_LABEL = "com.awoooi.restore.identity"
|
|
EXPECTED_IMAGES = {
|
|
"signoz": (
|
|
"signoz/signoz:v0.113.0",
|
|
"sha256:c17991d10966aedcb0d4d83805a0baf5310f1553d90611629036fcfaee8d969b",
|
|
),
|
|
"clickhouse": (
|
|
"clickhouse/clickhouse-server:25.5.6",
|
|
"sha256:8248e5926d7304400e44ecdf0c7181fbde28e6a9b1d647780eca839f34c00cc6",
|
|
),
|
|
"zookeeper": (
|
|
"signoz/zookeeper:3.7.1",
|
|
"sha256:3ab0e8f032ab58f14ac1e929ac40305a4a464cf320bdfe4151d62d6922729ba0",
|
|
),
|
|
}
|
|
EXPECTED_PRODUCTION = {
|
|
"signoz": EXPECTED_IMAGES["signoz"],
|
|
"signoz-clickhouse": EXPECTED_IMAGES["clickhouse"],
|
|
"signoz-zookeeper-1": EXPECTED_IMAGES["zookeeper"],
|
|
"signoz-otel-collector": (
|
|
"signoz/signoz-otel-collector:v0.144.1",
|
|
"sha256:47cbed488288df7bac6aab3665275bec6d6e24fe5c545c66914827a891e538d8",
|
|
),
|
|
}
|
|
EXPECTED_ENTRYPOINT = ["./signoz", "server"]
|
|
EXPECTED_SERVER_ENV = {
|
|
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN": "tcp://clickhouse:9000",
|
|
"SIGNOZ_SQLSTORE_SQLITE_PATH": "/var/lib/signoz/signoz.db",
|
|
}
|
|
EXPECTED_SERVER_MOUNT = "/var/lib/signoz"
|
|
PORT_RETRIES = 120
|
|
PORT_RETRY_INTERVAL_SECONDS = 1.0
|
|
CLICKHOUSE_RETRIES = 90
|
|
CLICKHOUSE_RETRY_INTERVAL_SECONDS = 1.0
|
|
COMMAND_TIMEOUT_SECONDS = 60
|
|
CLEANUP_COMMAND_TIMEOUT_SECONDS = 3
|
|
CLEANUP_RECONCILIATION_ATTEMPTS = 5
|
|
CLEANUP_STABLE_PASSES = 3
|
|
CLEANUP_RECONCILIATION_DELAY_SECONDS = 1.0
|
|
|
|
|
|
class SignalAbort(ContractError):
|
|
"""A handled termination signal that still requires owned cleanup."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ResourcePlan:
|
|
identity: str
|
|
short_id: str
|
|
prefix: str
|
|
canonical_id: str
|
|
network: str
|
|
zookeeper_container: str
|
|
clickhouse_container: str
|
|
server_container: str
|
|
zookeeper_data_volume: str
|
|
zookeeper_log_volume: str
|
|
clickhouse_data_volume: str
|
|
server_data_volume: str
|
|
workspace: Path
|
|
|
|
@property
|
|
def containers(self) -> tuple[str, ...]:
|
|
return (
|
|
self.server_container,
|
|
self.clickhouse_container,
|
|
self.zookeeper_container,
|
|
)
|
|
|
|
@property
|
|
def volumes(self) -> tuple[str, ...]:
|
|
return (
|
|
self.zookeeper_data_volume,
|
|
self.zookeeper_log_volume,
|
|
self.clickhouse_data_volume,
|
|
self.server_data_volume,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CommandResult:
|
|
returncode: int
|
|
stdout: str
|
|
|
|
|
|
Runner = Callable[[list[str], int], CommandResult]
|
|
|
|
|
|
def derive_identity(trace_id: str, run_id: str, work_item_id: str) -> str:
|
|
material = (
|
|
trace_id.encode("utf-8")
|
|
+ b"\0"
|
|
+ run_id.encode("utf-8")
|
|
+ b"\0"
|
|
+ work_item_id.encode("utf-8")
|
|
)
|
|
return hashlib.sha256(material).hexdigest()
|
|
|
|
|
|
def build_resource_plan(
|
|
trace_id: str,
|
|
run_id: str,
|
|
work_item_id: str,
|
|
*,
|
|
workspace_parent: Path = Path("/tmp"),
|
|
) -> ResourcePlan:
|
|
identity = derive_identity(trace_id, run_id, work_item_id)
|
|
short_id = identity[:16]
|
|
prefix = f"awoooi-signoz-md-restore-{short_id}"
|
|
return ResourcePlan(
|
|
identity=identity,
|
|
short_id=short_id,
|
|
prefix=prefix,
|
|
canonical_id=f"ephemeral-signoz-metadata-restore-{short_id}",
|
|
network=f"{prefix}-net",
|
|
zookeeper_container=f"{prefix}-zookeeper",
|
|
clickhouse_container=f"{prefix}-clickhouse",
|
|
server_container=f"{prefix}-server",
|
|
zookeeper_data_volume=f"{prefix}-zk-data",
|
|
zookeeper_log_volume=f"{prefix}-zk-log",
|
|
clickhouse_data_volume=f"{prefix}-ch-data",
|
|
server_data_volume=f"{prefix}-server-data",
|
|
workspace=workspace_parent / f"{prefix}-private",
|
|
)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser()
|
|
mode = parser.add_mutually_exclusive_group(required=True)
|
|
mode.add_argument("--check", action="store_true")
|
|
mode.add_argument("--apply", action="store_true")
|
|
mode.add_argument("--verify-cleanup", action="store_true")
|
|
parser.add_argument("--bundle-dir")
|
|
parser.add_argument("--policy", default=str(DEFAULT_POLICY))
|
|
parser.add_argument("--cluster-config", default=str(DEFAULT_CLUSTER_CONFIG))
|
|
parser.add_argument("--trace-id", required=True)
|
|
parser.add_argument("--run-id", required=True)
|
|
parser.add_argument("--work-item-id", default="P0-OBS-002")
|
|
parser.add_argument("--receipt-file")
|
|
parser.add_argument("--apply-receipt-file")
|
|
args = parser.parse_args()
|
|
if args.apply and not args.receipt_file:
|
|
parser.error("receipt_file_required_for_restore_apply")
|
|
return args
|
|
|
|
|
|
def default_runner(argv: list[str], timeout: int) -> CommandResult:
|
|
try:
|
|
result = subprocess.run( # nosec B603
|
|
argv,
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
text=True,
|
|
timeout=timeout,
|
|
check=False,
|
|
env={
|
|
"PATH": "/usr/sbin:/usr/bin:/sbin:/bin",
|
|
"LANG": "C.UTF-8",
|
|
"LC_ALL": "C.UTF-8",
|
|
},
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
raise ContractError("bounded_command_transport_failed") from exc
|
|
return CommandResult(result.returncode, result.stdout)
|
|
|
|
|
|
def run_required(
|
|
runner: Runner,
|
|
argv: list[str],
|
|
*,
|
|
label: str,
|
|
timeout: int = COMMAND_TIMEOUT_SECONDS,
|
|
) -> str:
|
|
result = runner(argv, timeout)
|
|
if result.returncode != 0:
|
|
raise ContractError(f"{label}_failed")
|
|
return result.stdout.strip()
|
|
|
|
|
|
def run_optional(
|
|
runner: Runner,
|
|
argv: list[str],
|
|
*,
|
|
timeout: int = COMMAND_TIMEOUT_SECONDS,
|
|
) -> CommandResult:
|
|
return runner(argv, timeout)
|
|
|
|
|
|
def validate_restore_policy(policy: dict[str, Any]) -> dict[str, Any]:
|
|
isolation = policy.get("restore_isolation")
|
|
if not isinstance(isolation, dict):
|
|
raise ContractError("restore_isolation_policy_missing")
|
|
if isolation.get("images") != {
|
|
key: {"ref": ref, "id": image_id}
|
|
for key, (ref, image_id) in EXPECTED_IMAGES.items()
|
|
}:
|
|
raise ContractError("restore_image_policy_not_exact")
|
|
if isolation.get("required_image_ref") != EXPECTED_IMAGES["signoz"][0]:
|
|
raise ContractError("restore_signoz_image_ref_not_exact")
|
|
if isolation.get("required_image_id") != EXPECTED_IMAGES["signoz"][1]:
|
|
raise ContractError("restore_signoz_image_id_not_exact")
|
|
if isolation.get("resource_prefix") != "awoooi-signoz-md-restore-":
|
|
raise ContractError("restore_resource_prefix_not_exact")
|
|
if isolation.get("canonical_id_prefix") != (
|
|
"ephemeral-signoz-metadata-restore-"
|
|
):
|
|
raise ContractError("restore_canonical_prefix_not_exact")
|
|
if isolation.get("resource_label_key") != PRIMARY_LABEL:
|
|
raise ContractError("restore_primary_label_not_exact")
|
|
if isolation.get("secondary_resource_label_key") != IDENTITY_LABEL:
|
|
raise ContractError("restore_identity_label_not_exact")
|
|
if isolation.get("private_workspace_parent") != "/tmp":
|
|
raise ContractError("restore_workspace_parent_not_exact")
|
|
if isolation.get("server_container_port") != 8080:
|
|
raise ContractError("restore_server_port_not_exact")
|
|
if isolation.get("loopback_publish_host") != "127.0.0.1":
|
|
raise ContractError("restore_publish_host_not_exact")
|
|
if any(
|
|
isolation.get(key) is not expected
|
|
for key, expected in (
|
|
("image_pull_allowed", False),
|
|
("production_network_attachment_allowed", False),
|
|
("production_volume_attachment_allowed", False),
|
|
("ephemeral_volumes_only", True),
|
|
("cleanup_controller_required", True),
|
|
("zero_residue_verifier_required", True),
|
|
)
|
|
):
|
|
raise ContractError("restore_isolation_boolean_contract_invalid")
|
|
|
|
startup = isolation.get("startup_contract")
|
|
if not isinstance(startup, dict) or startup.get("reviewed") is not True:
|
|
raise ContractError("restore_startup_contract_not_reviewed")
|
|
if startup.get("server_entrypoint") != EXPECTED_ENTRYPOINT:
|
|
raise ContractError("restore_server_entrypoint_not_exact")
|
|
if startup.get("server_environment") != EXPECTED_SERVER_ENV:
|
|
raise ContractError("restore_server_environment_not_exact")
|
|
if startup.get("server_data_mount_destination") != EXPECTED_SERVER_MOUNT:
|
|
raise ContractError("restore_server_mount_not_exact")
|
|
if startup.get("clickhouse_network_alias") != "clickhouse":
|
|
raise ContractError("restore_clickhouse_alias_not_exact")
|
|
if startup.get("zookeeper_network_alias") != "restore-zookeeper":
|
|
raise ContractError("restore_zookeeper_alias_not_exact")
|
|
key_presence = startup.get("required_production_key_presence")
|
|
if (
|
|
not isinstance(key_presence, list)
|
|
or len(key_presence) != len(EXPECTED_SERVER_ENV)
|
|
or sorted(key_presence) != sorted(EXPECTED_SERVER_ENV)
|
|
):
|
|
raise ContractError("restore_production_key_presence_not_exact")
|
|
if startup.get("production_dsn_class") != "tcp://clickhouse:9000*":
|
|
raise ContractError("restore_production_dsn_class_not_exact")
|
|
if startup.get("production_sqlite_path") != EXPECTED_SERVER_ENV[
|
|
"SIGNOZ_SQLSTORE_SQLITE_PATH"
|
|
]:
|
|
raise ContractError("restore_production_sqlite_path_not_exact")
|
|
if startup.get("secret_value_output_allowed") is not False:
|
|
raise ContractError("restore_secret_output_must_be_forbidden")
|
|
if startup.get("full_environment_output_allowed") is not False:
|
|
raise ContractError("restore_full_environment_output_must_be_forbidden")
|
|
|
|
expected_continuity = {
|
|
key.replace("-", "_"): {
|
|
"container": key,
|
|
"image_ref": ref,
|
|
"image_id": image_id,
|
|
}
|
|
for key, (ref, image_id) in EXPECTED_PRODUCTION.items()
|
|
}
|
|
# The policy uses semantic keys for the production containers.
|
|
actual_continuity = isolation.get("production_continuity")
|
|
if not isinstance(actual_continuity, dict):
|
|
raise ContractError("restore_production_continuity_missing")
|
|
normalized_actual = {
|
|
str(value.get("container", "")).replace("-", "_"): value
|
|
for value in actual_continuity.values()
|
|
if isinstance(value, dict)
|
|
}
|
|
if normalized_actual != expected_continuity:
|
|
raise ContractError("restore_production_continuity_not_exact")
|
|
return isolation
|
|
|
|
|
|
def require_regular_cluster_config(path: Path) -> None:
|
|
try:
|
|
metadata = path.lstat()
|
|
except OSError as exc:
|
|
raise ContractError("isolated_cluster_config_unavailable") from exc
|
|
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
|
|
raise ContractError("isolated_cluster_config_not_regular")
|
|
content = path.read_text(encoding="utf-8")
|
|
for marker in (
|
|
"<host>restore-zookeeper</host>",
|
|
"<host>restore-clickhouse</host>",
|
|
'<replica from_env="CLICKHOUSE_RESTORE_REPLICA"/>',
|
|
):
|
|
if marker not in content:
|
|
raise ContractError("isolated_cluster_config_contract_invalid")
|
|
|
|
|
|
def label_args(plan: ResourcePlan, run_id: str) -> list[str]:
|
|
return [
|
|
"--label",
|
|
f"{PRIMARY_LABEL}={run_id}",
|
|
"--label",
|
|
f"{IDENTITY_LABEL}={plan.identity}",
|
|
]
|
|
|
|
|
|
def inspect_image_id(runner: Runner, ref: str) -> str:
|
|
return run_required(
|
|
runner,
|
|
[DOCKER, "image", "inspect", "--format={{.Id}}", ref],
|
|
label="docker_image_inspect",
|
|
)
|
|
|
|
|
|
def verify_exact_images(runner: Runner) -> dict[str, dict[str, str]]:
|
|
observed: dict[str, dict[str, str]] = {}
|
|
for role, (ref, expected_id) in EXPECTED_IMAGES.items():
|
|
actual_id = inspect_image_id(runner, ref)
|
|
if actual_id != expected_id:
|
|
raise ContractError(f"restore_image_id_mismatch_{role}")
|
|
observed[role] = {"ref": ref, "id": actual_id}
|
|
return observed
|
|
|
|
|
|
def inspect_json(
|
|
runner: Runner,
|
|
kind: str,
|
|
name: str,
|
|
*,
|
|
timeout: int = COMMAND_TIMEOUT_SECONDS,
|
|
) -> dict[str, Any] | None:
|
|
result = run_optional(runner, [DOCKER, kind, "inspect", name], timeout=timeout)
|
|
if result.returncode != 0:
|
|
return None
|
|
try:
|
|
value = json.loads(result.stdout)
|
|
except json.JSONDecodeError as exc:
|
|
raise ContractError(f"docker_{kind}_inspect_invalid") from exc
|
|
if not isinstance(value, list) or len(value) != 1 or not isinstance(value[0], dict):
|
|
raise ContractError(f"docker_{kind}_inspect_invalid")
|
|
return value[0]
|
|
|
|
|
|
def production_snapshot(
|
|
runner: Runner,
|
|
*,
|
|
timeout: int = COMMAND_TIMEOUT_SECONDS,
|
|
) -> dict[str, dict[str, Any]]:
|
|
snapshot: dict[str, dict[str, Any]] = {}
|
|
for container, (expected_ref, expected_image_id) in EXPECTED_PRODUCTION.items():
|
|
data = inspect_json(runner, "container", container, timeout=timeout)
|
|
if data is None:
|
|
raise ContractError(f"production_container_missing_{container}")
|
|
config = data.get("Config")
|
|
state = data.get("State")
|
|
if not isinstance(config, dict) or not isinstance(state, dict):
|
|
raise ContractError("production_container_inspect_shape_invalid")
|
|
row = {
|
|
"id": data.get("Id"),
|
|
"image_id": data.get("Image"),
|
|
"image_ref": config.get("Image"),
|
|
"running": state.get("Running"),
|
|
"started_at": state.get("StartedAt"),
|
|
"restart_count": data.get("RestartCount"),
|
|
}
|
|
if (
|
|
not isinstance(row["id"], str)
|
|
or len(row["id"]) != 64
|
|
or row["image_id"] != expected_image_id
|
|
or row["image_ref"] != expected_ref
|
|
or row["running"] is not True
|
|
or not isinstance(row["started_at"], str)
|
|
or not isinstance(row["restart_count"], int)
|
|
):
|
|
raise ContractError(f"production_container_identity_invalid_{container}")
|
|
snapshot[container] = row
|
|
return snapshot
|
|
|
|
|
|
def image_inventory(
|
|
runner: Runner,
|
|
*,
|
|
timeout: int = COMMAND_TIMEOUT_SECONDS,
|
|
) -> list[str]:
|
|
output = run_required(
|
|
runner,
|
|
[
|
|
DOCKER,
|
|
"image",
|
|
"ls",
|
|
"--no-trunc",
|
|
"--format={{.ID}}|{{.Repository}}:{{.Tag}}",
|
|
],
|
|
label="docker_image_inventory",
|
|
timeout=timeout,
|
|
)
|
|
rows = sorted(line for line in output.splitlines() if line)
|
|
if not rows:
|
|
raise ContractError("docker_image_inventory_empty")
|
|
return rows
|
|
|
|
|
|
def require_daemon_readable(
|
|
runner: Runner,
|
|
*,
|
|
timeout: int = COMMAND_TIMEOUT_SECONDS,
|
|
) -> None:
|
|
output = run_required(
|
|
runner,
|
|
[DOCKER, "info", "--format={{.ServerVersion}}"],
|
|
label="docker_daemon_inventory",
|
|
timeout=timeout,
|
|
)
|
|
if not output or any(character.isspace() for character in output):
|
|
raise ContractError("docker_daemon_inventory_invalid")
|
|
|
|
|
|
def inspect_resource_labels(
|
|
runner: Runner,
|
|
kind: str,
|
|
name: str,
|
|
*,
|
|
timeout: int = COMMAND_TIMEOUT_SECONDS,
|
|
) -> dict[str, str] | None:
|
|
data = inspect_json(runner, kind, name, timeout=timeout)
|
|
if data is None:
|
|
return None
|
|
if kind == "container":
|
|
labels = data.get("Config", {}).get("Labels")
|
|
else:
|
|
labels = data.get("Labels")
|
|
if labels is None:
|
|
return {}
|
|
if not isinstance(labels, dict) or not all(
|
|
isinstance(key, str) and isinstance(value, str)
|
|
for key, value in labels.items()
|
|
):
|
|
raise ContractError(f"docker_{kind}_labels_invalid")
|
|
return labels
|
|
|
|
|
|
def assert_absent_or_owned(
|
|
runner: Runner,
|
|
plan: ResourcePlan,
|
|
run_id: str,
|
|
kind: str,
|
|
name: str,
|
|
) -> bool:
|
|
labels = inspect_resource_labels(
|
|
runner,
|
|
kind,
|
|
name,
|
|
timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS,
|
|
)
|
|
if labels is None:
|
|
return False
|
|
if labels.get(PRIMARY_LABEL) != run_id or labels.get(IDENTITY_LABEL) != plan.identity:
|
|
raise ContractError(f"restore_{kind}_name_collision_not_owned")
|
|
return True
|
|
|
|
|
|
def require_no_preexisting_resources(
|
|
runner: Runner,
|
|
plan: ResourcePlan,
|
|
run_id: str,
|
|
) -> None:
|
|
for name in plan.containers:
|
|
if assert_absent_or_owned(runner, plan, run_id, "container", name):
|
|
raise ContractError("restore_owned_container_residue_requires_reconcile")
|
|
for name in plan.volumes:
|
|
if assert_absent_or_owned(runner, plan, run_id, "volume", name):
|
|
raise ContractError("restore_owned_volume_residue_requires_reconcile")
|
|
if assert_absent_or_owned(runner, plan, run_id, "network", plan.network):
|
|
raise ContractError("restore_owned_network_residue_requires_reconcile")
|
|
if plan.workspace.exists() or plan.workspace.is_symlink():
|
|
raise ContractError("restore_private_workspace_residue_requires_reconcile")
|
|
|
|
|
|
def verify_bundle(args: argparse.Namespace, runner: Runner) -> None:
|
|
if not args.bundle_dir:
|
|
raise ContractError("bundle_dir_required")
|
|
bundle = Path(args.bundle_dir)
|
|
output = run_required(
|
|
runner,
|
|
[
|
|
PYTHON,
|
|
str(EXPORT_VERIFIER),
|
|
"--bundle-dir",
|
|
str(bundle),
|
|
"--policy",
|
|
str(args.policy),
|
|
"--expected-trace-id",
|
|
args.trace_id,
|
|
"--expected-run-id",
|
|
args.run_id,
|
|
"--expected-work-item-id",
|
|
args.work_item_id,
|
|
],
|
|
label="independent_bundle_verifier",
|
|
)
|
|
try:
|
|
value = json.loads(output)
|
|
except json.JSONDecodeError as exc:
|
|
raise ContractError("independent_bundle_verifier_receipt_invalid") from exc
|
|
if value.get("terminal") != "pass_export_verified_restore_pending":
|
|
raise ContractError("independent_bundle_verifier_terminal_invalid")
|
|
|
|
|
|
def create_workspace(plan: ResourcePlan) -> None:
|
|
plan.workspace.mkdir(mode=0o700)
|
|
metadata = plan.workspace.lstat()
|
|
if (
|
|
not stat.S_ISDIR(metadata.st_mode)
|
|
or metadata.st_uid != os.geteuid()
|
|
or stat.S_IMODE(metadata.st_mode) != 0o700
|
|
):
|
|
raise ContractError("restore_workspace_owner_or_mode_invalid")
|
|
|
|
|
|
def write_private(path: Path, data: bytes) -> None:
|
|
write_new_private_atomic(path, data, label="restore_private_material")
|
|
path.chmod(0o400)
|
|
|
|
|
|
def write_state(path: Path, value: dict[str, Any]) -> None:
|
|
payload = canonical_json_bytes(value)
|
|
if path.exists():
|
|
metadata = path.lstat()
|
|
if (
|
|
stat.S_ISLNK(metadata.st_mode)
|
|
or not stat.S_ISREG(metadata.st_mode)
|
|
or metadata.st_uid != os.geteuid()
|
|
or stat.S_IMODE(metadata.st_mode) != 0o600
|
|
):
|
|
raise ContractError("restore_state_owner_or_mode_invalid")
|
|
temp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
|
write_new_private_atomic(temp, payload, label="restore_state_update")
|
|
os.replace(temp, path)
|
|
path.chmod(0o600)
|
|
else:
|
|
write_new_private_atomic(path, payload, label="restore_state")
|
|
|
|
|
|
def create_resources(
|
|
runner: Runner,
|
|
plan: ResourcePlan,
|
|
run_id: str,
|
|
cluster_config: Path,
|
|
) -> None:
|
|
labels = label_args(plan, run_id)
|
|
run_required(
|
|
runner,
|
|
[DOCKER, "network", "create", "--internal", *labels, plan.network],
|
|
label="restore_network_create",
|
|
)
|
|
for name in plan.volumes:
|
|
run_required(
|
|
runner,
|
|
[DOCKER, "volume", "create", *labels, name],
|
|
label="restore_volume_create",
|
|
)
|
|
|
|
zk_ref = EXPECTED_IMAGES["zookeeper"][0]
|
|
run_required(
|
|
runner,
|
|
[
|
|
DOCKER,
|
|
"run",
|
|
"--detach",
|
|
"--pull=never",
|
|
"--restart=no",
|
|
"--name",
|
|
plan.zookeeper_container,
|
|
"--network",
|
|
plan.network,
|
|
"--network-alias",
|
|
"restore-zookeeper",
|
|
*labels,
|
|
"--env",
|
|
"ZOO_MY_ID=1",
|
|
"--env",
|
|
"ZOO_SERVERS=server.1=0.0.0.0:2888:3888;2181",
|
|
"--volume",
|
|
f"{plan.zookeeper_data_volume}:/data",
|
|
"--volume",
|
|
f"{plan.zookeeper_log_volume}:/datalog",
|
|
zk_ref,
|
|
],
|
|
label="restore_zookeeper_start",
|
|
)
|
|
|
|
clickhouse_ref = EXPECTED_IMAGES["clickhouse"][0]
|
|
run_required(
|
|
runner,
|
|
[
|
|
DOCKER,
|
|
"run",
|
|
"--detach",
|
|
"--pull=never",
|
|
"--restart=no",
|
|
"--name",
|
|
plan.clickhouse_container,
|
|
"--network",
|
|
plan.network,
|
|
"--network-alias",
|
|
"clickhouse",
|
|
"--network-alias",
|
|
"restore-clickhouse",
|
|
*labels,
|
|
"--env",
|
|
f"CLICKHOUSE_RESTORE_REPLICA=restore-{plan.short_id}",
|
|
"--env",
|
|
"CLICKHOUSE_SKIP_USER_SETUP=1",
|
|
"--volume",
|
|
f"{plan.clickhouse_data_volume}:/var/lib/clickhouse",
|
|
"--volume",
|
|
f"{cluster_config}:/etc/clickhouse-server/config.d/isolated-cluster.xml:ro",
|
|
clickhouse_ref,
|
|
],
|
|
label="restore_clickhouse_start",
|
|
)
|
|
wait_for_clickhouse(runner, plan)
|
|
|
|
signoz_ref = EXPECTED_IMAGES["signoz"][0]
|
|
run_required(
|
|
runner,
|
|
[
|
|
DOCKER,
|
|
"run",
|
|
"--detach",
|
|
"--pull=never",
|
|
"--restart=no",
|
|
"--name",
|
|
plan.server_container,
|
|
"--network",
|
|
plan.network,
|
|
*labels,
|
|
"--publish",
|
|
"127.0.0.1::8080",
|
|
"--env",
|
|
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN=tcp://clickhouse:9000",
|
|
"--env",
|
|
"SIGNOZ_SQLSTORE_SQLITE_PATH=/var/lib/signoz/signoz.db",
|
|
"--volume",
|
|
f"{plan.server_data_volume}:/var/lib/signoz",
|
|
signoz_ref,
|
|
],
|
|
label="restore_signoz_start",
|
|
)
|
|
|
|
|
|
def wait_for_clickhouse(runner: Runner, plan: ResourcePlan) -> None:
|
|
for attempt in range(CLICKHOUSE_RETRIES):
|
|
result = run_optional(
|
|
runner,
|
|
[
|
|
DOCKER,
|
|
"exec",
|
|
plan.clickhouse_container,
|
|
"clickhouse-client",
|
|
"--query",
|
|
"SELECT 1",
|
|
],
|
|
timeout=5,
|
|
)
|
|
if result.returncode == 0 and result.stdout.strip() == "1":
|
|
return
|
|
if attempt + 1 < CLICKHOUSE_RETRIES:
|
|
time.sleep(CLICKHOUSE_RETRY_INTERVAL_SECONDS)
|
|
raise ContractError("isolated_clickhouse_readiness_timeout")
|
|
|
|
|
|
def verify_runtime_container_images(runner: Runner, plan: ResourcePlan) -> None:
|
|
expected = {
|
|
plan.server_container: EXPECTED_IMAGES["signoz"],
|
|
plan.clickhouse_container: EXPECTED_IMAGES["clickhouse"],
|
|
plan.zookeeper_container: EXPECTED_IMAGES["zookeeper"],
|
|
}
|
|
for name, (expected_ref, expected_id) in expected.items():
|
|
data = inspect_json(runner, "container", name)
|
|
if data is None:
|
|
raise ContractError("restore_runtime_container_missing")
|
|
config = data.get("Config", {})
|
|
host_config = data.get("HostConfig", {})
|
|
if (
|
|
data.get("Image") != expected_id
|
|
or config.get("Image") != expected_ref
|
|
or host_config.get("RestartPolicy", {}).get("Name") not in {"", "no"}
|
|
):
|
|
raise ContractError("restore_runtime_container_image_or_restart_drift")
|
|
|
|
|
|
def discover_loopback_port(runner: Runner, plan: ResourcePlan) -> int:
|
|
output = run_required(
|
|
runner,
|
|
[DOCKER, "port", plan.server_container, "8080/tcp"],
|
|
label="restore_listener_discovery",
|
|
)
|
|
lines = [line.strip() for line in output.splitlines() if line.strip()]
|
|
if len(lines) != 1 or not lines[0].startswith("127.0.0.1:"):
|
|
raise ContractError("restore_listener_not_loopback_ephemeral")
|
|
port_text = lines[0].rsplit(":", 1)[-1]
|
|
if not port_text.isdigit():
|
|
raise ContractError("restore_listener_port_invalid")
|
|
port = int(port_text)
|
|
if port <= 0 or port > 65535 or port == 8080:
|
|
raise ContractError("restore_listener_port_not_ephemeral")
|
|
return port
|
|
|
|
|
|
def request_json(
|
|
url: str,
|
|
*,
|
|
method: str = "GET",
|
|
payload: dict[str, Any] | None = None,
|
|
authorization: str | None = None,
|
|
timeout: int = 10,
|
|
) -> dict[str, Any]:
|
|
body = canonical_json_bytes(payload) if payload is not None else None
|
|
request = urllib.request.Request(url, data=body, method=method)
|
|
request.add_header("Accept", "application/json")
|
|
if body is not None:
|
|
request.add_header("Content-Type", "application/json")
|
|
if authorization is not None:
|
|
request.add_header("Authorization", authorization)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response: # nosec B310
|
|
if response.status < 200 or response.status >= 300:
|
|
raise ContractError("isolated_api_unexpected_status")
|
|
raw = response.read(1024 * 1024 + 1)
|
|
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
|
raise ContractError("isolated_api_request_failed") from exc
|
|
if len(raw) > 1024 * 1024:
|
|
raise ContractError("isolated_api_response_too_large")
|
|
try:
|
|
value = json.loads(raw)
|
|
except (UnicodeError, json.JSONDecodeError) as exc:
|
|
raise ContractError("isolated_api_response_invalid") from exc
|
|
if not isinstance(value, dict):
|
|
raise ContractError("isolated_api_response_not_object")
|
|
return value
|
|
|
|
|
|
def wait_for_health_and_version(base_url: str) -> None:
|
|
last_health_ok = False
|
|
for _ in range(PORT_RETRIES):
|
|
try:
|
|
health = request_json(f"{base_url}/api/v1/health", timeout=2)
|
|
last_health_ok = health.get("status") == "ok"
|
|
if last_health_ok:
|
|
break
|
|
except ContractError:
|
|
pass
|
|
time.sleep(PORT_RETRY_INTERVAL_SECONDS)
|
|
if not last_health_ok:
|
|
raise ContractError("isolated_signoz_health_timeout")
|
|
version = request_json(f"{base_url}/api/v1/version", timeout=5)
|
|
if version.get("version") != "v0.113.0":
|
|
raise ContractError("isolated_signoz_version_mismatch")
|
|
|
|
|
|
def random_password() -> str:
|
|
alphabet = string.ascii_letters + string.digits + "-_"
|
|
return "R!" + "".join(secrets.choice(alphabet) for _ in range(46))
|
|
|
|
|
|
def bootstrap_isolated_session(base_url: str, workspace: Path) -> Path:
|
|
email = f"restore-{secrets.token_hex(12)}@example.invalid"
|
|
password = random_password()
|
|
register = request_json(
|
|
f"{base_url}/api/v1/register",
|
|
method="POST",
|
|
payload={
|
|
"email": email,
|
|
"orgDisplayName": "AWOOOI isolated restore",
|
|
"password": password,
|
|
},
|
|
)
|
|
data = register.get("data")
|
|
register_body = data if isinstance(data, dict) else register
|
|
org_id = register_body.get("orgId")
|
|
if not isinstance(org_id, str) or not org_id:
|
|
raise ContractError("isolated_registration_org_id_missing")
|
|
login = request_json(
|
|
f"{base_url}/api/v2/sessions/email_password",
|
|
method="POST",
|
|
payload={"email": email, "password": password, "orgId": org_id},
|
|
)
|
|
login_data = login.get("data")
|
|
if not isinstance(login_data, dict):
|
|
login_data = login
|
|
access_token = (
|
|
login_data.get("accessToken") if isinstance(login_data, dict) else None
|
|
)
|
|
refresh_token = (
|
|
login_data.get("refreshToken") if isinstance(login_data, dict) else None
|
|
)
|
|
if (
|
|
not isinstance(access_token, str)
|
|
or len(access_token) < 16
|
|
or not isinstance(refresh_token, str)
|
|
or len(refresh_token) < 16
|
|
):
|
|
raise ContractError("isolated_session_token_shape_invalid")
|
|
token_path = workspace / "session-access-token"
|
|
write_private(token_path, access_token.encode("ascii") + b"\n")
|
|
return token_path
|
|
|
|
|
|
def isolation_receipt_value(
|
|
args: argparse.Namespace,
|
|
plan: ResourcePlan,
|
|
base_url: str,
|
|
images: dict[str, dict[str, str]],
|
|
production_before: dict[str, dict[str, Any]],
|
|
image_inventory_before: list[str],
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"schema": "awoooi_signoz_metadata_restore_isolation_v1",
|
|
"trace_id": args.trace_id,
|
|
"run_id": args.run_id,
|
|
"work_item_id": args.work_item_id,
|
|
"target": {
|
|
"identity": plan.identity,
|
|
"prefix": plan.prefix,
|
|
"canonical_id": plan.canonical_id,
|
|
"base_url_sha256": sha256_bytes(base_url.encode("utf-8")),
|
|
"image_ref": EXPECTED_IMAGES["signoz"][0],
|
|
"image_id": EXPECTED_IMAGES["signoz"][1],
|
|
"images": images,
|
|
"image_present_locally": True,
|
|
"image_pull_performed": False,
|
|
"image_build_performed": False,
|
|
"production_network_attached": False,
|
|
"production_volume_attached": False,
|
|
"ephemeral_volumes_only": True,
|
|
"cleanup_controller_armed": True,
|
|
"zero_residue_verifier_armed": True,
|
|
"network_scope": "loopback_and_internal_only",
|
|
"resource_label_key": PRIMARY_LABEL,
|
|
"resource_label_value": args.run_id,
|
|
"identity_label_key": IDENTITY_LABEL,
|
|
"identity_label_value": plan.identity,
|
|
"resources": {
|
|
"network": plan.network,
|
|
"containers": list(plan.containers),
|
|
"volumes": list(plan.volumes),
|
|
"workspace": str(plan.workspace),
|
|
},
|
|
"listener": {"host": "127.0.0.1", "port": int(base_url.rsplit(":", 1)[1])},
|
|
"authentication_mode": "isolated_session_bearer",
|
|
},
|
|
"production_before": production_before,
|
|
"image_inventory_before": image_inventory_before,
|
|
}
|
|
|
|
|
|
def invoke_restore_driver(
|
|
args: argparse.Namespace,
|
|
runner: Runner,
|
|
base_url: str,
|
|
isolation_receipt: Path,
|
|
token_file: Path,
|
|
mode: str,
|
|
) -> dict[str, Any]:
|
|
argv = [
|
|
PYTHON,
|
|
str(RESTORE_DRIVER),
|
|
mode,
|
|
"--bundle-dir",
|
|
str(args.bundle_dir),
|
|
"--policy",
|
|
str(args.policy),
|
|
"--target-base-url",
|
|
base_url,
|
|
"--credential-file",
|
|
str(token_file),
|
|
"--auth-mode",
|
|
"isolated_session_bearer",
|
|
"--isolation-receipt",
|
|
str(isolation_receipt),
|
|
"--trace-id",
|
|
args.trace_id,
|
|
"--run-id",
|
|
args.run_id,
|
|
"--work-item-id",
|
|
args.work_item_id,
|
|
]
|
|
output = run_required(
|
|
runner,
|
|
argv,
|
|
label=f"isolated_restore_driver_{mode.removeprefix('--')}",
|
|
timeout=900,
|
|
)
|
|
try:
|
|
value = json.loads(output)
|
|
except json.JSONDecodeError as exc:
|
|
raise ContractError("isolated_restore_driver_receipt_invalid") from exc
|
|
expected = (
|
|
"check_pass_no_write"
|
|
if mode == "--check"
|
|
else "partial_degraded_cleanup_pending"
|
|
)
|
|
if value.get("terminal") != expected:
|
|
raise ContractError("isolated_restore_driver_terminal_invalid")
|
|
return value
|
|
|
|
|
|
def tcp_connectable(host: str, port: int) -> bool:
|
|
try:
|
|
with socket.create_connection((host, port), timeout=2):
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def remove_owned_resource(
|
|
runner: Runner,
|
|
plan: ResourcePlan,
|
|
run_id: str,
|
|
kind: str,
|
|
name: str,
|
|
) -> bool:
|
|
labels = inspect_resource_labels(runner, kind, name)
|
|
if labels is None:
|
|
return False
|
|
if labels.get(PRIMARY_LABEL) != run_id or labels.get(IDENTITY_LABEL) != plan.identity:
|
|
raise ContractError(f"cleanup_{kind}_ownership_mismatch")
|
|
if kind == "container":
|
|
argv = [DOCKER, "rm", "--force", name]
|
|
elif kind == "volume":
|
|
argv = [DOCKER, "volume", "rm", name]
|
|
else:
|
|
argv = [DOCKER, "network", "rm", name]
|
|
run_required(
|
|
runner,
|
|
argv,
|
|
label=f"cleanup_{kind}_remove",
|
|
timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS,
|
|
)
|
|
return True
|
|
|
|
|
|
def remove_private_workspace(plan: ResourcePlan) -> None:
|
|
if not plan.workspace.exists() and not plan.workspace.is_symlink():
|
|
return
|
|
metadata = plan.workspace.lstat()
|
|
if (
|
|
stat.S_ISLNK(metadata.st_mode)
|
|
or not stat.S_ISDIR(metadata.st_mode)
|
|
or metadata.st_uid != os.geteuid()
|
|
or stat.S_IMODE(metadata.st_mode) != 0o700
|
|
):
|
|
raise ContractError("cleanup_workspace_ownership_or_mode_mismatch")
|
|
shutil.rmtree(plan.workspace)
|
|
|
|
|
|
def label_inventory_count(
|
|
runner: Runner,
|
|
kind: str,
|
|
label: str,
|
|
) -> int:
|
|
if kind == "container":
|
|
argv = [DOCKER, "ps", "-a", "--quiet", "--filter", f"label={label}"]
|
|
else:
|
|
argv = [DOCKER, kind, "ls", "--quiet", "--filter", f"label={label}"]
|
|
output = run_required(
|
|
runner,
|
|
argv,
|
|
label=f"cleanup_{kind}_inventory",
|
|
timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS,
|
|
)
|
|
return len([line for line in output.splitlines() if line.strip()])
|
|
|
|
|
|
def capture_owned_listener_before_cleanup(
|
|
runner: Runner,
|
|
plan: ResourcePlan,
|
|
run_id: str,
|
|
) -> dict[str, Any] | None:
|
|
data = inspect_json(
|
|
runner,
|
|
"container",
|
|
plan.server_container,
|
|
timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS,
|
|
)
|
|
if data is None:
|
|
return None
|
|
labels = data.get("Config", {}).get("Labels")
|
|
if (
|
|
not isinstance(labels, dict)
|
|
or labels.get(PRIMARY_LABEL) != run_id
|
|
or labels.get(IDENTITY_LABEL) != plan.identity
|
|
):
|
|
raise ContractError("cleanup_server_listener_ownership_mismatch")
|
|
ports = data.get("NetworkSettings", {}).get("Ports")
|
|
bindings = ports.get("8080/tcp") if isinstance(ports, dict) else None
|
|
if not isinstance(bindings, list) or len(bindings) != 1:
|
|
raise ContractError("cleanup_server_listener_binding_unknown")
|
|
binding = bindings[0]
|
|
if not isinstance(binding, dict):
|
|
raise ContractError("cleanup_server_listener_binding_invalid")
|
|
host = binding.get("HostIp")
|
|
port_text = binding.get("HostPort")
|
|
if host != "127.0.0.1" or not isinstance(port_text, str) or not port_text.isdigit():
|
|
raise ContractError("cleanup_server_listener_not_loopback")
|
|
port = int(port_text)
|
|
if port <= 0 or port > 65535:
|
|
raise ContractError("cleanup_server_listener_port_invalid")
|
|
return {"host": host, "port": port}
|
|
|
|
|
|
def cleanup_and_verify(
|
|
runner: Runner,
|
|
plan: ResourcePlan,
|
|
run_id: str,
|
|
state: dict[str, Any] | None,
|
|
) -> list[str]:
|
|
baseline_errors: list[str] = []
|
|
listener: dict[str, Any] | None = None
|
|
production_before: dict[str, Any] | None = None
|
|
image_inventory_before: list[str] | None = None
|
|
if state is not None:
|
|
state_listener = state.get("listener")
|
|
if isinstance(state_listener, dict):
|
|
listener = state_listener
|
|
state_production = state.get("production_before")
|
|
if isinstance(state_production, dict):
|
|
production_before = state_production
|
|
state_images = state.get("image_inventory_before")
|
|
if isinstance(state_images, list) and all(
|
|
isinstance(item, str) for item in state_images
|
|
):
|
|
image_inventory_before = state_images
|
|
try:
|
|
if listener is None:
|
|
listener = capture_owned_listener_before_cleanup(
|
|
runner, plan, run_id
|
|
)
|
|
if production_before is None:
|
|
production_before = production_snapshot(
|
|
runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS
|
|
)
|
|
if image_inventory_before is None:
|
|
image_inventory_before = image_inventory(
|
|
runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS
|
|
)
|
|
except ContractError as exc:
|
|
baseline_errors.append(str(exc))
|
|
|
|
stable_passes = 0
|
|
last_errors: list[str] = []
|
|
for attempt in range(CLEANUP_RECONCILIATION_ATTEMPTS):
|
|
attempt_errors = list(baseline_errors)
|
|
removed_during_attempt = False
|
|
|
|
# Re-run removal on every pass. A Docker API request interrupted after
|
|
# daemon acceptance may materialize a labeled resource after the first
|
|
# inventory; repeated ownership-gated removal closes that race.
|
|
for name in plan.containers:
|
|
try:
|
|
removed_during_attempt = (
|
|
remove_owned_resource(runner, plan, run_id, "container", name)
|
|
or removed_during_attempt
|
|
)
|
|
except ContractError as exc:
|
|
attempt_errors.append(str(exc))
|
|
for name in plan.volumes:
|
|
try:
|
|
removed_during_attempt = (
|
|
remove_owned_resource(runner, plan, run_id, "volume", name)
|
|
or removed_during_attempt
|
|
)
|
|
except ContractError as exc:
|
|
attempt_errors.append(str(exc))
|
|
try:
|
|
removed_during_attempt = (
|
|
remove_owned_resource(runner, plan, run_id, "network", plan.network)
|
|
or removed_during_attempt
|
|
)
|
|
except ContractError as exc:
|
|
attempt_errors.append(str(exc))
|
|
|
|
try:
|
|
require_daemon_readable(
|
|
runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS
|
|
)
|
|
for name in plan.containers:
|
|
if inspect_json(
|
|
runner,
|
|
"container",
|
|
name,
|
|
timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS,
|
|
) is not None:
|
|
attempt_errors.append("cleanup_exact_container_residue")
|
|
for name in plan.volumes:
|
|
if inspect_json(
|
|
runner,
|
|
"volume",
|
|
name,
|
|
timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS,
|
|
) is not None:
|
|
attempt_errors.append("cleanup_exact_volume_residue")
|
|
if inspect_json(
|
|
runner,
|
|
"network",
|
|
plan.network,
|
|
timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS,
|
|
) is not None:
|
|
attempt_errors.append("cleanup_exact_network_residue")
|
|
for label in (
|
|
f"{PRIMARY_LABEL}={run_id}",
|
|
f"{IDENTITY_LABEL}={plan.identity}",
|
|
):
|
|
for kind in ("container", "volume", "network"):
|
|
if label_inventory_count(runner, kind, label) != 0:
|
|
attempt_errors.append(f"cleanup_{kind}_label_residue")
|
|
except ContractError as exc:
|
|
attempt_errors.append(str(exc))
|
|
|
|
if listener is not None:
|
|
host = listener.get("host")
|
|
port = listener.get("port")
|
|
if isinstance(host, str) and isinstance(port, int):
|
|
if tcp_connectable(host, port):
|
|
attempt_errors.append("cleanup_tcp_listener_still_present")
|
|
else:
|
|
attempt_errors.append("cleanup_listener_state_invalid")
|
|
if production_before is None or image_inventory_before is None:
|
|
attempt_errors.append("cleanup_continuity_baseline_unavailable")
|
|
else:
|
|
try:
|
|
if production_snapshot(
|
|
runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS
|
|
) != production_before:
|
|
attempt_errors.append(
|
|
"cleanup_production_identity_continuity_failed"
|
|
)
|
|
if image_inventory(
|
|
runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS
|
|
) != image_inventory_before:
|
|
attempt_errors.append("cleanup_image_inventory_drift")
|
|
except ContractError as exc:
|
|
attempt_errors.append(str(exc))
|
|
|
|
try:
|
|
remove_private_workspace(plan)
|
|
except ContractError as exc:
|
|
attempt_errors.append(str(exc))
|
|
if plan.workspace.exists() or plan.workspace.is_symlink():
|
|
attempt_errors.append("cleanup_private_workspace_residue")
|
|
|
|
if removed_during_attempt:
|
|
attempt_errors.append("cleanup_owned_resource_reappeared")
|
|
|
|
last_errors = sorted(set(attempt_errors))
|
|
if last_errors:
|
|
stable_passes = 0
|
|
else:
|
|
stable_passes += 1
|
|
if stable_passes >= CLEANUP_STABLE_PASSES:
|
|
return []
|
|
if attempt + 1 < CLEANUP_RECONCILIATION_ATTEMPTS:
|
|
time.sleep(CLEANUP_RECONCILIATION_DELAY_SECONDS)
|
|
|
|
return sorted(set(last_errors + ["cleanup_stable_zero_residue_not_observed"]))
|
|
|
|
|
|
def load_state_if_safe(plan: ResourcePlan) -> dict[str, Any] | None:
|
|
path = plan.workspace / "state.json"
|
|
if not path.exists():
|
|
return None
|
|
value = read_json_object(path, label="restore_state")
|
|
if value.get("schema") != "awoooi_signoz_metadata_restore_state_v1":
|
|
raise ContractError("restore_state_schema_invalid")
|
|
if value.get("identity") != plan.identity:
|
|
raise ContractError("restore_state_identity_mismatch")
|
|
if not isinstance(value.get("production_before"), dict):
|
|
raise ContractError("restore_state_production_baseline_invalid")
|
|
images = value.get("image_inventory_before")
|
|
if not isinstance(images, list) or not all(isinstance(item, str) for item in images):
|
|
raise ContractError("restore_state_image_baseline_invalid")
|
|
listener = value.get("listener")
|
|
if listener is not None and (
|
|
not isinstance(listener, dict)
|
|
or listener.get("host") != "127.0.0.1"
|
|
or not isinstance(listener.get("port"), int)
|
|
or listener["port"] <= 0
|
|
or listener["port"] > 65535
|
|
):
|
|
raise ContractError("restore_state_listener_invalid")
|
|
return value
|
|
|
|
|
|
def run_check(
|
|
args: argparse.Namespace,
|
|
runner: Runner,
|
|
plan: ResourcePlan,
|
|
policy: dict[str, Any],
|
|
cluster_config: Path,
|
|
) -> dict[str, Any]:
|
|
validate_restore_policy(policy)
|
|
require_regular_cluster_config(cluster_config)
|
|
verify_bundle(args, runner)
|
|
require_daemon_readable(runner)
|
|
verify_exact_images(runner)
|
|
production_snapshot(runner)
|
|
image_inventory(runner)
|
|
require_no_preexisting_resources(runner, plan, args.run_id)
|
|
value = receipt(
|
|
trace_id=args.trace_id,
|
|
run_id=args.run_id,
|
|
work_item_id=args.work_item_id,
|
|
phase="terminal",
|
|
terminal="check_pass_no_write",
|
|
detail="exact_images_startup_contract_production_continuity_and_zero_collision_pass",
|
|
)
|
|
value.update(
|
|
{
|
|
"identity": plan.identity,
|
|
"resourcePrefix": plan.prefix,
|
|
"resourceWritePerformed": False,
|
|
"secretValueRead": False,
|
|
"completionClaim": False,
|
|
}
|
|
)
|
|
return value
|
|
|
|
|
|
def run_apply(
|
|
args: argparse.Namespace,
|
|
runner: Runner,
|
|
plan: ResourcePlan,
|
|
policy: dict[str, Any],
|
|
cluster_config: Path,
|
|
) -> dict[str, Any]:
|
|
run_check(args, runner, plan, policy, cluster_config)
|
|
create_workspace(plan)
|
|
state_path = plan.workspace / "state.json"
|
|
images = verify_exact_images(runner)
|
|
before = production_snapshot(runner)
|
|
inventory_before = image_inventory(runner)
|
|
state: dict[str, Any] = {
|
|
"schema": "awoooi_signoz_metadata_restore_state_v1",
|
|
"identity": plan.identity,
|
|
"trace_id": args.trace_id,
|
|
"run_id": args.run_id,
|
|
"work_item_id": args.work_item_id,
|
|
"production_before": before,
|
|
"image_inventory_before": inventory_before,
|
|
"listener": None,
|
|
}
|
|
write_state(state_path, state)
|
|
create_resources(runner, plan, args.run_id, cluster_config)
|
|
verify_runtime_container_images(runner, plan)
|
|
port = discover_loopback_port(runner, plan)
|
|
state["listener"] = {"host": "127.0.0.1", "port": port}
|
|
write_state(state_path, state)
|
|
base_url = f"http://127.0.0.1:{port}"
|
|
wait_for_health_and_version(base_url)
|
|
token_path = bootstrap_isolated_session(base_url, plan.workspace)
|
|
isolation_path = plan.workspace / "isolation-receipt.json"
|
|
isolation = isolation_receipt_value(
|
|
args,
|
|
plan,
|
|
base_url,
|
|
images,
|
|
before,
|
|
inventory_before,
|
|
)
|
|
write_new_private_atomic(
|
|
isolation_path,
|
|
canonical_json_bytes(isolation),
|
|
label="isolation_receipt",
|
|
)
|
|
invoke_restore_driver(
|
|
args,
|
|
runner,
|
|
base_url,
|
|
isolation_path,
|
|
token_path,
|
|
"--check",
|
|
)
|
|
restore = invoke_restore_driver(
|
|
args,
|
|
runner,
|
|
base_url,
|
|
isolation_path,
|
|
token_path,
|
|
"--apply",
|
|
)
|
|
value = receipt(
|
|
trace_id=args.trace_id,
|
|
run_id=args.run_id,
|
|
work_item_id=args.work_item_id,
|
|
phase="terminal",
|
|
terminal="portable_metadata_restore_verified_cleanup_pending",
|
|
detail="portable_assets_semantic_readback_pass_cleanup_pending",
|
|
)
|
|
value["restoreSemanticDigest"] = restore.get("portable_semantic_sha256")
|
|
value["restoredAssetCounts"] = restore.get("restored_asset_counts")
|
|
value["completionClaim"] = False
|
|
return value
|
|
|
|
|
|
def executor_success(
|
|
args: argparse.Namespace,
|
|
plan: ResourcePlan,
|
|
restore_result: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
value = receipt(
|
|
trace_id=args.trace_id,
|
|
run_id=args.run_id,
|
|
work_item_id=args.work_item_id,
|
|
phase="terminal",
|
|
terminal=(
|
|
"portable_metadata_restore_verified_zero_residue_"
|
|
"pending_independent_verifier"
|
|
),
|
|
detail=(
|
|
"portable_assets_semantic_parity_and_executor_cleanup_verified_"
|
|
"pending_independent_verifier"
|
|
),
|
|
)
|
|
value.update(
|
|
{
|
|
"identity": plan.identity,
|
|
"resourcePrefix": plan.prefix,
|
|
"completionScope": "portable_assets_only",
|
|
"portableAssetIds": [
|
|
"dashboards",
|
|
"alert_rules",
|
|
"explorer_views",
|
|
],
|
|
"completionClaim": False,
|
|
"executorZeroResidueVerified": True,
|
|
"independentZeroResidueVerified": False,
|
|
"fullSqliteCompletionClaim": False,
|
|
"rolesPreferencesGlobalConfigRestoreClaim": False,
|
|
"secretBearingAssetsRestoreClaim": False,
|
|
"productionMutationPerformed": False,
|
|
"imagePullPerformed": False,
|
|
"imageBuildPerformed": False,
|
|
"secretValueInArguments": False,
|
|
"secretValueInReceipt": False,
|
|
"restoreSemanticDigest": restore_result.get("restoreSemanticDigest"),
|
|
"restoredAssetCounts": restore_result.get("restoredAssetCounts"),
|
|
"zeroResidue": {
|
|
"exactContainers": 0,
|
|
"exactVolumes": 0,
|
|
"exactNetworks": 0,
|
|
"labelInventory": 0,
|
|
"tcpListener": 0,
|
|
"privateWorkspace": 0,
|
|
},
|
|
}
|
|
)
|
|
return value
|
|
|
|
|
|
def validate_apply_receipt(
|
|
path: Path,
|
|
args: argparse.Namespace,
|
|
plan: ResourcePlan,
|
|
) -> dict[str, Any]:
|
|
if not path.is_absolute():
|
|
raise ContractError("apply_receipt_path_must_be_absolute")
|
|
require_no_symlink_components(path.parent, label="apply_receipt_parent")
|
|
try:
|
|
metadata = path.lstat()
|
|
except OSError as exc:
|
|
raise ContractError("apply_receipt_unavailable") from exc
|
|
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
|
|
raise ContractError("apply_receipt_not_regular")
|
|
if metadata.st_uid != os.geteuid():
|
|
raise ContractError("apply_receipt_owner_mismatch")
|
|
if stat.S_IMODE(metadata.st_mode) not in {0o400, 0o600}:
|
|
raise ContractError("apply_receipt_mode_invalid")
|
|
if metadata.st_size <= 0 or metadata.st_size > 65536:
|
|
raise ContractError("apply_receipt_size_invalid")
|
|
value = read_json_object(path, label="apply_receipt")
|
|
expected = {
|
|
"schema": "awoooi_signoz_metadata_receipt_v1",
|
|
"trace_id": args.trace_id,
|
|
"run_id": args.run_id,
|
|
"work_item_id": args.work_item_id,
|
|
"phase": "terminal",
|
|
"terminal": (
|
|
"portable_metadata_restore_verified_zero_residue_"
|
|
"pending_independent_verifier"
|
|
),
|
|
"identity": plan.identity,
|
|
"resourcePrefix": plan.prefix,
|
|
"completionClaim": False,
|
|
"executorZeroResidueVerified": True,
|
|
"independentZeroResidueVerified": False,
|
|
"completionScope": "portable_assets_only",
|
|
"fullSqliteCompletionClaim": False,
|
|
"rolesPreferencesGlobalConfigRestoreClaim": False,
|
|
"secretBearingAssetsRestoreClaim": False,
|
|
"productionMutationPerformed": False,
|
|
"imagePullPerformed": False,
|
|
"imageBuildPerformed": False,
|
|
"secretValueInArguments": False,
|
|
"secretValueInReceipt": False,
|
|
}
|
|
if any(value.get(key) != expected_value for key, expected_value in expected.items()):
|
|
raise ContractError("apply_receipt_identity_or_terminal_invalid")
|
|
digest = value.get("restoreSemanticDigest")
|
|
counts = value.get("restoredAssetCounts")
|
|
zero_residue = value.get("zeroResidue")
|
|
if (
|
|
not isinstance(digest, str)
|
|
or len(digest) != 64
|
|
or any(character not in "0123456789abcdef" for character in digest)
|
|
or not isinstance(counts, dict)
|
|
or sorted(counts) != ["alert_rules", "dashboards", "explorer_views"]
|
|
or not all(isinstance(count, int) and count >= 0 for count in counts.values())
|
|
or zero_residue
|
|
!= {
|
|
"exactContainers": 0,
|
|
"exactVolumes": 0,
|
|
"exactNetworks": 0,
|
|
"labelInventory": 0,
|
|
"tcpListener": 0,
|
|
"privateWorkspace": 0,
|
|
}
|
|
):
|
|
raise ContractError("apply_receipt_semantic_evidence_invalid")
|
|
return value
|
|
|
|
|
|
def independent_success(
|
|
args: argparse.Namespace,
|
|
plan: ResourcePlan,
|
|
apply_receipt: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
value = receipt(
|
|
trace_id=args.trace_id,
|
|
run_id=args.run_id,
|
|
work_item_id=args.work_item_id,
|
|
phase="independent_restore_verifier",
|
|
terminal="portable_metadata_restore_independently_verified_zero_residue",
|
|
detail="immutable_apply_receipt_and_stable_zero_residue_readback_pass",
|
|
)
|
|
value.update(
|
|
{
|
|
"verifierIdentity": "signoz_metadata_restore_cleanup_verifier",
|
|
"applyReceiptValidated": True,
|
|
"applyReceiptSha256": sha256_bytes(
|
|
canonical_json_bytes(apply_receipt)
|
|
),
|
|
"identity": plan.identity,
|
|
"resourcePrefix": plan.prefix,
|
|
"completionScope": "portable_assets_only",
|
|
"completionClaim": True,
|
|
"executorZeroResidueVerified": True,
|
|
"independentZeroResidueVerified": True,
|
|
"fullSqliteCompletionClaim": False,
|
|
"rolesPreferencesGlobalConfigRestoreClaim": False,
|
|
"secretBearingAssetsRestoreClaim": False,
|
|
"restoreSemanticDigest": apply_receipt["restoreSemanticDigest"],
|
|
"restoredAssetCounts": apply_receipt["restoredAssetCounts"],
|
|
"zeroResidue": {
|
|
"stablePasses": CLEANUP_STABLE_PASSES,
|
|
"exactContainers": 0,
|
|
"exactVolumes": 0,
|
|
"exactNetworks": 0,
|
|
"labelInventory": 0,
|
|
"tcpListener": 0,
|
|
"privateWorkspace": 0,
|
|
},
|
|
}
|
|
)
|
|
return value
|
|
|
|
|
|
def write_terminal(path: Path, value: dict[str, Any]) -> None:
|
|
write_new_private_atomic(path, canonical_json_bytes(value), label="restore_receipt")
|
|
|
|
|
|
def emit(value: dict[str, Any]) -> None:
|
|
print(json.dumps(value, sort_keys=True, separators=(",", ":")))
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
runner = default_runner
|
|
plan: ResourcePlan | None = None
|
|
cleanup_errors: list[str] = []
|
|
restore_result: dict[str, Any] | None = None
|
|
caught_error: str | None = None
|
|
old_handlers: dict[int, Any] = {}
|
|
cleanup_reconciliation_completed = False
|
|
signals_held_until_terminal = False
|
|
|
|
def handle_signal(signum: int, _frame: Any) -> None:
|
|
# The first handled signal transfers control to finally. Ignore later
|
|
# handled signals so the owned cleanup cannot be interrupted mid-way.
|
|
for handled in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM):
|
|
signal.signal(handled, signal.SIG_IGN)
|
|
raise SignalAbort(f"signal_{signum}_cleanup_required")
|
|
|
|
def restore_signal_handlers() -> None:
|
|
for signum, handler in old_handlers.items():
|
|
signal.signal(signum, handler)
|
|
|
|
def hold_cleanup_signals() -> None:
|
|
nonlocal signals_held_until_terminal
|
|
for handled in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM):
|
|
signal.signal(handled, signal.SIG_IGN)
|
|
signals_held_until_terminal = True
|
|
|
|
try:
|
|
for value, label in (
|
|
(args.trace_id, "trace_id"),
|
|
(args.run_id, "run_id"),
|
|
(args.work_item_id, "work_item_id"),
|
|
):
|
|
validate_identity(value, label=label)
|
|
if args.work_item_id != "P0-OBS-002":
|
|
raise ContractError("work_item_not_allowlisted")
|
|
if args.receipt_file:
|
|
validate_new_private_destination(
|
|
Path(args.receipt_file), label="restore_receipt"
|
|
)
|
|
plan = build_resource_plan(args.trace_id, args.run_id, args.work_item_id)
|
|
policy = load_policy(Path(args.policy))
|
|
cluster_config = Path(args.cluster_config)
|
|
for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM):
|
|
old_handlers[signum] = signal.getsignal(signum)
|
|
signal.signal(signum, handle_signal)
|
|
|
|
if args.check:
|
|
result = run_check(args, runner, plan, policy, cluster_config)
|
|
if args.receipt_file:
|
|
write_terminal(Path(args.receipt_file), result)
|
|
emit(result)
|
|
return 0
|
|
|
|
if args.verify_cleanup:
|
|
try:
|
|
state = load_state_if_safe(plan)
|
|
except (ContractError, OSError, UnicodeError) as exc:
|
|
state = None
|
|
cleanup_errors.append(str(exc))
|
|
reconciliation_errors = cleanup_and_verify(
|
|
runner, plan, args.run_id, state
|
|
)
|
|
cleanup_errors.extend(reconciliation_errors)
|
|
cleanup_reconciliation_completed = True
|
|
if cleanup_errors:
|
|
raise ContractError("cleanup_reconciliation_incomplete")
|
|
if args.apply_receipt_file:
|
|
apply_receipt = validate_apply_receipt(
|
|
Path(args.apply_receipt_file), args, plan
|
|
)
|
|
result = independent_success(args, plan, apply_receipt)
|
|
else:
|
|
result = receipt(
|
|
trace_id=args.trace_id,
|
|
run_id=args.run_id,
|
|
work_item_id=args.work_item_id,
|
|
phase="independent_restore_verifier",
|
|
terminal="pass_zero_residue_verified",
|
|
detail="deterministic_reconciliation_zero_residue_pass",
|
|
)
|
|
result["completionClaim"] = False
|
|
result["productionContinuityScope"] = (
|
|
"reconciliation_window_only"
|
|
)
|
|
result["restoreCompletionClaim"] = False
|
|
if args.receipt_file:
|
|
write_terminal(Path(args.receipt_file), result)
|
|
emit(result)
|
|
return 0
|
|
|
|
restore_result = run_apply(args, runner, plan, policy, cluster_config)
|
|
except (ContractError, OSError, UnicodeError) as exc:
|
|
caught_error = str(exc)
|
|
finally:
|
|
cleanup_retry_required = bool(
|
|
plan is not None
|
|
and (
|
|
args.apply
|
|
or (args.verify_cleanup and not cleanup_reconciliation_completed)
|
|
)
|
|
)
|
|
if cleanup_retry_required and plan is not None:
|
|
# The target timeout sends TERM before its kill-after window. Once
|
|
# reconciliation starts here, keep all handled signals ignored
|
|
# through the durable terminal write so cleanup cannot be cut in
|
|
# half or separated from its receipt.
|
|
hold_cleanup_signals()
|
|
try:
|
|
state = load_state_if_safe(plan)
|
|
if state is None and args.apply:
|
|
cleanup_errors.append(
|
|
"restore_state_missing_for_apply_continuity"
|
|
)
|
|
except (ContractError, OSError, UnicodeError) as exc:
|
|
state = None
|
|
cleanup_errors.append(str(exc))
|
|
try:
|
|
reconciliation_errors = cleanup_and_verify(
|
|
runner, plan, args.run_id, state
|
|
)
|
|
cleanup_errors.extend(reconciliation_errors)
|
|
cleanup_reconciliation_completed = True
|
|
except (ContractError, OSError, UnicodeError) as exc:
|
|
cleanup_errors.append(str(exc))
|
|
if plan is not None and args.verify_cleanup and caught_error is not None:
|
|
# Cleanup may have completed just before a signal or verifier
|
|
# receipt validation failure. Preserve the failure receipt window
|
|
# even when no second reconciliation pass is required.
|
|
hold_cleanup_signals()
|
|
if not args.apply and not signals_held_until_terminal:
|
|
restore_signal_handlers()
|
|
|
|
if plan is None:
|
|
return 1
|
|
if caught_error is None and not cleanup_errors and restore_result is not None:
|
|
result = executor_success(args, plan, restore_result)
|
|
exit_code = 0
|
|
else:
|
|
error_code = caught_error or "cleanup_reconciliation_incomplete"
|
|
result = receipt(
|
|
trace_id=args.trace_id,
|
|
run_id=args.run_id,
|
|
work_item_id=args.work_item_id,
|
|
phase="terminal",
|
|
terminal=(
|
|
"failed_zero_residue_verified_no_completion"
|
|
if not cleanup_errors
|
|
else "failed_cleanup_reconciliation_required"
|
|
),
|
|
detail=f"isolated_restore_failed_{error_code}",
|
|
)
|
|
result.update(
|
|
{
|
|
"completionClaim": False,
|
|
"fullSqliteCompletionClaim": False,
|
|
"secretValueInArguments": False,
|
|
"secretValueInReceipt": False,
|
|
"zeroResidueVerified": not cleanup_errors,
|
|
"cleanupErrorCodes": cleanup_errors,
|
|
}
|
|
)
|
|
exit_code = 1
|
|
try:
|
|
if args.receipt_file:
|
|
write_terminal(Path(args.receipt_file), result)
|
|
emit(result)
|
|
return exit_code
|
|
except (ContractError, OSError):
|
|
return 74
|
|
finally:
|
|
# Restore signal behavior only after the durable terminal and stdout
|
|
# receipt are complete. A second TERM cannot interrupt cleanup or the
|
|
# write-once receipt persistence window.
|
|
restore_signal_handlers()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|