358 lines
12 KiB
Python
358 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import stat
|
|
from pathlib import Path
|
|
from types import ModuleType, SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
TARGET = ROOT / "agent99-signoz-toolchain-target.py"
|
|
|
|
|
|
def _load_target() -> ModuleType:
|
|
spec = importlib.util.spec_from_file_location(
|
|
"agent99_signoz_toolchain_target", TARGET
|
|
)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _payloads() -> dict[str, bytes]:
|
|
policy = {
|
|
"work_item_id": "P0-OBS-002",
|
|
"source_runtime": {
|
|
"raw_database_access_allowed": False,
|
|
"raw_volume_access_allowed": False,
|
|
},
|
|
"completion_contract": {"full_sqlite_completion_claim_allowed": False},
|
|
}
|
|
values: dict[str, bytes] = {
|
|
"metadata-export-policy.json": (
|
|
json.dumps(policy, sort_keys=True, separators=(",", ":")) + "\n"
|
|
).encode(),
|
|
"isolated-cluster.xml": b"<clickhouse><remote_servers/></clickhouse>\n",
|
|
}
|
|
for name in (
|
|
"signoz_metadata_contract.py",
|
|
"signoz-metadata-export.py",
|
|
"verify-signoz-metadata-export.py",
|
|
"signoz-metadata-restore-drill.py",
|
|
"signoz-metadata-isolated-restore.py",
|
|
):
|
|
values[name] = b"VALUE = 'bounded-toolchain-test'\n"
|
|
return values
|
|
|
|
|
|
def _manifest(module: ModuleType, values: dict[str, bytes]) -> list[dict[str, str]]:
|
|
return [
|
|
{
|
|
"name": name,
|
|
"sha256": hashlib.sha256(values[name]).hexdigest(),
|
|
"mode": f"{mode:04o}",
|
|
}
|
|
for name, mode in zip(module.LABELS, module.MODES, strict=True)
|
|
]
|
|
|
|
|
|
def _args(
|
|
module: ModuleType,
|
|
manifest: list[dict[str, str]],
|
|
*,
|
|
mode: str,
|
|
run_id: str = "run-001",
|
|
attempt_id: str = "",
|
|
stage: Path | None = None,
|
|
) -> SimpleNamespace:
|
|
bundle_id = module.sha256_bytes(
|
|
"".join(f"{row['sha256']}\n" for row in manifest).encode("ascii")
|
|
)
|
|
return SimpleNamespace(
|
|
mode=mode,
|
|
trace_id="trace-001",
|
|
run_id=run_id,
|
|
work_item_id="P0-OBS-002",
|
|
source_revision="1" * 40,
|
|
bundle_id=bundle_id,
|
|
stage_dir=str(stage) if stage else None,
|
|
attempt_id=attempt_id,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def target_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
|
module = _load_target()
|
|
uid = os.getuid()
|
|
gid = os.getgid()
|
|
remote_parent = tmp_path / "backup" / "toolchains"
|
|
receipt_parent = tmp_path / "backup" / "deploy-receipts"
|
|
remote_parent.mkdir(parents=True)
|
|
receipt_parent.mkdir(parents=True)
|
|
lock = tmp_path / "operation.lock"
|
|
lock.write_bytes(b"")
|
|
os.chmod(lock, 0o600)
|
|
snapshot = {
|
|
"id": "a" * 64,
|
|
"image_id": module.SIGNOZ_IMAGE_ID,
|
|
"image_ref": module.SIGNOZ_IMAGE_REF,
|
|
"running": True,
|
|
"started_at": "2026-07-22T00:00:00Z",
|
|
"restart_count": 0,
|
|
}
|
|
monkeypatch.setattr(module, "ROOT_UID", uid)
|
|
monkeypatch.setattr(module, "REMOTE_ROOT", remote_parent / "signoz-metadata")
|
|
monkeypatch.setattr(
|
|
module,
|
|
"RECEIPT_ROOT",
|
|
receipt_parent / "signoz-metadata-toolchain",
|
|
)
|
|
module.REMOTE_ROOT.mkdir(mode=0o700)
|
|
module.RECEIPT_ROOT.mkdir(mode=0o700)
|
|
monkeypatch.setattr(module, "OPERATION_LOCK", lock)
|
|
monkeypatch.setattr(module, "production_snapshot", lambda: dict(snapshot))
|
|
monkeypatch.setattr(module, "require_health", lambda: None)
|
|
monkeypatch.setattr(module, "require_no_active_operations", lambda: None)
|
|
monkeypatch.setenv("SUDO_UID", str(uid))
|
|
monkeypatch.setenv("SUDO_GID", str(gid))
|
|
values = _payloads()
|
|
manifest = _manifest(module, values)
|
|
return module, values, manifest, snapshot
|
|
|
|
|
|
def _write_tree(
|
|
root: Path,
|
|
manifest: list[dict[str, str]],
|
|
values: dict[str, bytes],
|
|
*,
|
|
directory_mode: int,
|
|
) -> None:
|
|
root.mkdir(mode=directory_mode, parents=True)
|
|
for row in manifest:
|
|
path = root / row["name"]
|
|
path.write_bytes(values[row["name"]])
|
|
os.chmod(path, int(row["mode"], 8))
|
|
|
|
|
|
def test_target_state_distinguishes_absent_exact_drift_and_active_pointer(
|
|
target_env,
|
|
) -> None:
|
|
module, values, manifest, _snapshot = target_env
|
|
args = _args(module, manifest, mode="check")
|
|
assert module.target_state(args.bundle_id, manifest) == "absent"
|
|
|
|
target = module.REMOTE_ROOT / args.bundle_id
|
|
_write_tree(target, manifest, values, directory_mode=0o750)
|
|
assert module.target_state(args.bundle_id, manifest) == "exact"
|
|
|
|
(target / manifest[0]["name"]).write_bytes(b"drift")
|
|
with pytest.raises(module.TargetError, match="target_file_drift"):
|
|
module.target_state(args.bundle_id, manifest)
|
|
|
|
(target / manifest[0]["name"]).write_bytes(values[manifest[0]["name"]])
|
|
os.chmod(target / manifest[0]["name"], int(manifest[0]["mode"], 8))
|
|
(module.REMOTE_ROOT / "current").symlink_to(target)
|
|
with pytest.raises(module.TargetError, match="active_pointer_must_remain_absent"):
|
|
module.target_state(args.bundle_id, manifest)
|
|
|
|
|
|
def test_prepare_creates_private_transport_owned_stage(
|
|
target_env, tmp_path: Path
|
|
) -> None:
|
|
module, _values, manifest, _snapshot = target_env
|
|
stage = tmp_path / "transport-stage"
|
|
args = _args(module, manifest, mode="prepare", stage=stage)
|
|
|
|
module.reset_mutation_state()
|
|
receipt = module.prepare_stage(args, manifest)
|
|
|
|
metadata = stage.lstat()
|
|
assert receipt["terminal"] == "stage_prepared"
|
|
assert metadata.st_uid == os.getuid()
|
|
assert stat.S_IMODE(metadata.st_mode) == 0o700
|
|
assert receipt["source_write_attempted"] is True
|
|
assert receipt["source_write_performed"] is True
|
|
|
|
|
|
def test_apply_writes_receipt_only_after_exact_target_and_live_stage_cleanup(
|
|
target_env, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
module, values, manifest, _snapshot = target_env
|
|
stage = tmp_path / "transport-stage"
|
|
_write_tree(stage, manifest, values, directory_mode=0o700)
|
|
args = _args(module, manifest, mode="apply", stage=stage)
|
|
original_write = module.write_deployment_receipt
|
|
observed: list[tuple[bool, str]] = []
|
|
|
|
def guarded_write(*call_args, **call_kwargs):
|
|
observed.append(
|
|
(
|
|
os.path.lexists(stage),
|
|
module.target_state(args.bundle_id, manifest),
|
|
)
|
|
)
|
|
return original_write(*call_args, **call_kwargs)
|
|
|
|
monkeypatch.setattr(module, "write_deployment_receipt", guarded_write)
|
|
module.reset_mutation_state()
|
|
receipt = module.apply_bundle(args, manifest)
|
|
|
|
assert receipt["terminal"] == "apply_pass_deployed_inactive"
|
|
assert observed == [(False, "exact")]
|
|
assert not os.path.lexists(stage)
|
|
assert not os.path.lexists(
|
|
module.REMOTE_ROOT / f".{args.bundle_id}.candidate.{args.run_id}"
|
|
)
|
|
durable = module.load_deploy_receipt(args)
|
|
assert durable is not None
|
|
assert durable["deployment_method"] == "created"
|
|
assert (
|
|
durable["production_continuity_sha256"]
|
|
== receipt["production_continuity_sha256"]
|
|
)
|
|
assert not os.path.lexists(module.REMOTE_ROOT / "current")
|
|
|
|
|
|
def test_reconcile_retries_half_quarantine_and_preserves_shared_target(
|
|
target_env, tmp_path: Path
|
|
) -> None:
|
|
module, values, manifest, _snapshot = target_env
|
|
stage = tmp_path / "transport-stage"
|
|
args = _args(
|
|
module,
|
|
manifest,
|
|
mode="reconcile",
|
|
run_id="run-half",
|
|
attempt_id="attempt-002",
|
|
stage=stage,
|
|
)
|
|
receipt_dir = module.RECEIPT_ROOT / args.run_id
|
|
receipt_dir.mkdir(mode=0o700, parents=True)
|
|
# Model a signal after the stage was moved but before the candidate move.
|
|
(receipt_dir / "transport-stage").mkdir(mode=0o700)
|
|
candidate = module.REMOTE_ROOT / f".{args.bundle_id}.candidate.{args.run_id}"
|
|
candidate.mkdir(mode=0o700, parents=True)
|
|
target = module.REMOTE_ROOT / args.bundle_id
|
|
_write_tree(target, manifest, values, directory_mode=0o750)
|
|
|
|
module.reset_mutation_state()
|
|
receipt = module.reconcile_apply(args, manifest)
|
|
|
|
assert receipt["terminal"] == "reconcile_exact_target_preserved_unverified"
|
|
assert receipt["owned_live_residue_verified_absent"] is True
|
|
assert target.is_dir()
|
|
assert (receipt_dir / "transport-stage").is_dir()
|
|
assert (receipt_dir / "quarantine-candidate").is_dir()
|
|
assert not candidate.exists()
|
|
|
|
|
|
def test_reconcile_finalizes_interrupted_exact_apply_from_durable_intent(
|
|
target_env, tmp_path: Path
|
|
) -> None:
|
|
module, values, manifest, snapshot = target_env
|
|
stage = tmp_path / "transport-stage"
|
|
args = _args(
|
|
module,
|
|
manifest,
|
|
mode="reconcile",
|
|
run_id="run-interrupted",
|
|
attempt_id="attempt-003",
|
|
stage=stage,
|
|
)
|
|
receipt_dir = module.RECEIPT_ROOT / args.run_id
|
|
receipt_dir.mkdir(mode=0o700, parents=True)
|
|
continuity = module.sha256_bytes(module.canonical_bytes(snapshot))
|
|
intent = module.apply_intent(args, "absent", continuity)
|
|
module.write_private_new(
|
|
receipt_dir / "apply-intent.json", module.canonical_bytes(intent), 0o600
|
|
)
|
|
target = module.REMOTE_ROOT / args.bundle_id
|
|
_write_tree(target, manifest, values, directory_mode=0o750)
|
|
|
|
module.reset_mutation_state()
|
|
receipt = module.reconcile_apply(args, manifest)
|
|
|
|
assert receipt["terminal"] == "reconcile_exact_deployment_verified"
|
|
assert receipt["durable_deploy_receipt_valid"] is True
|
|
durable = module.load_deploy_receipt(args)
|
|
assert durable is not None
|
|
assert durable["deployment_method"] == "reconciled_exact"
|
|
assert durable["reconcile_attempt_id"] == "attempt-003"
|
|
assert target.is_dir()
|
|
|
|
|
|
def test_reconcile_zero_live_residue_is_repeatable_with_new_attempt(
|
|
target_env, tmp_path: Path
|
|
) -> None:
|
|
module, _values, manifest, _snapshot = target_env
|
|
stage = tmp_path / "transport-stage"
|
|
first = _args(
|
|
module,
|
|
manifest,
|
|
mode="reconcile",
|
|
run_id="run-cleanup",
|
|
attempt_id="attempt-004",
|
|
stage=stage,
|
|
)
|
|
stage.mkdir(mode=0o700)
|
|
|
|
module.reset_mutation_state()
|
|
first_receipt = module.reconcile_apply(first, manifest)
|
|
second = _args(
|
|
module,
|
|
manifest,
|
|
mode="reconcile",
|
|
run_id=first.run_id,
|
|
attempt_id="attempt-005",
|
|
stage=stage,
|
|
)
|
|
module.reset_mutation_state()
|
|
second_receipt = module.reconcile_apply(second, manifest)
|
|
|
|
assert first_receipt["terminal"] == "reconcile_zero_owned_residue"
|
|
assert second_receipt["terminal"] == "reconcile_zero_owned_residue"
|
|
assert not stage.exists()
|
|
assert (module.RECEIPT_ROOT / first.run_id / "transport-stage").is_dir()
|
|
|
|
|
|
def test_reconcile_fails_closed_on_production_continuity_change(
|
|
target_env, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
module, values, manifest, snapshot = target_env
|
|
stage = tmp_path / "transport-stage"
|
|
args = _args(
|
|
module,
|
|
manifest,
|
|
mode="reconcile",
|
|
run_id="run-continuity",
|
|
attempt_id="attempt-006",
|
|
stage=stage,
|
|
)
|
|
receipt_dir = module.RECEIPT_ROOT / args.run_id
|
|
receipt_dir.mkdir(mode=0o700, parents=True)
|
|
intent = module.apply_intent(
|
|
args,
|
|
"absent",
|
|
module.sha256_bytes(module.canonical_bytes(snapshot)),
|
|
)
|
|
module.write_private_new(
|
|
receipt_dir / "apply-intent.json", module.canonical_bytes(intent), 0o600
|
|
)
|
|
_write_tree(
|
|
module.REMOTE_ROOT / args.bundle_id, manifest, values, directory_mode=0o750
|
|
)
|
|
changed = dict(snapshot)
|
|
changed["restart_count"] = 1
|
|
monkeypatch.setattr(module, "production_snapshot", lambda: dict(changed))
|
|
|
|
module.reset_mutation_state()
|
|
with pytest.raises(
|
|
module.TargetError, match="apply_intent_production_continuity_mismatch"
|
|
):
|
|
module.reconcile_apply(args, manifest)
|