393 lines
12 KiB
Python
393 lines
12 KiB
Python
"""Focused tests for trusted same-run Kubernetes rollout verification."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from src.services.kubernetes_controlled_executor import (
|
|
build_kubernetes_controlled_execution_claim,
|
|
)
|
|
from src.services.kubernetes_rollout_verifier import (
|
|
KubernetesRolloutVerifier,
|
|
build_kubernetes_rollout_verifier_receipt,
|
|
build_kubernetes_state_readback_receipt,
|
|
issue_kubernetes_execution_context,
|
|
)
|
|
|
|
RAW_UID = "12345678-1234-4abc-8def-1234567890ab"
|
|
|
|
|
|
def _claim() -> dict:
|
|
route = {
|
|
"schema_version": "typed_domain_target_route_v2",
|
|
"resolution_status": "resolved",
|
|
"route_id": "typed:kubernetes_workload:service-awoooi-api",
|
|
"target_kind": "kubernetes_workload",
|
|
"target_resource": "awoooi-api",
|
|
"source_namespace": "awoooi-prod",
|
|
"canonical_asset_id": "service:awoooi-api",
|
|
"executor": "kubernetes_controlled_executor",
|
|
"verifier": "kubernetes_rollout_verifier",
|
|
"controlled_apply_allowed": True,
|
|
"cross_domain_fallback_allowed": False,
|
|
"identity_evidence": {
|
|
"status": "verified",
|
|
"namespace": "awoooi-prod",
|
|
"kind": "deployment",
|
|
"name": "awoooi-api",
|
|
},
|
|
}
|
|
return build_kubernetes_controlled_execution_claim(
|
|
action=(
|
|
"kubectl rollout restart deployment/awoooi-api "
|
|
"-n awoooi-prod"
|
|
),
|
|
typed_target_route=route,
|
|
incident_id="INC-K8S-001",
|
|
)
|
|
|
|
|
|
def _raw_pre() -> dict:
|
|
return {
|
|
"namespace": "awoooi-prod",
|
|
"kind": "deployment",
|
|
"name": "awoooi-api",
|
|
"uid": RAW_UID,
|
|
"generation": 4,
|
|
"observed_generation": 4,
|
|
"desired_replicas": 2,
|
|
"updated_replicas": 2,
|
|
"ready_replicas": 2,
|
|
"available_replicas": 2,
|
|
"unavailable_replicas": 0,
|
|
}
|
|
|
|
|
|
def _raw_post() -> dict:
|
|
return {
|
|
**_raw_pre(),
|
|
"generation": 5,
|
|
"observed_generation": 5,
|
|
}
|
|
|
|
|
|
def _same_run_states() -> tuple[dict, object, dict, dict]:
|
|
claim = _claim()
|
|
context = issue_kubernetes_execution_context(claim)
|
|
pre = build_kubernetes_state_readback_receipt(
|
|
claim=claim,
|
|
execution_context=context,
|
|
phase="pre",
|
|
raw_state=_raw_pre(),
|
|
observed_at_epoch_ms=1000,
|
|
)
|
|
post = build_kubernetes_state_readback_receipt(
|
|
claim=claim,
|
|
execution_context=context,
|
|
phase="post",
|
|
raw_state=_raw_post(),
|
|
observed_at_epoch_ms=2000,
|
|
)
|
|
return claim, context, pre, post
|
|
|
|
|
|
def test_exact_same_run_rollout_convergence_is_verified_public_safely() -> None:
|
|
claim, context, pre, post = _same_run_states()
|
|
|
|
receipt = build_kubernetes_rollout_verifier_receipt(
|
|
claim=claim,
|
|
pre_state=pre,
|
|
post_state=post,
|
|
execution_context=context,
|
|
)
|
|
|
|
assert receipt["status"] == "verified_success"
|
|
assert receipt["verified"] is True
|
|
assert receipt["blockers"] == []
|
|
assert len(receipt["workload_uid_sha256"]) == 64
|
|
assert receipt["trace_id"].startswith("trace:k8s:")
|
|
assert receipt["run_id"].startswith("run:k8s:")
|
|
assert receipt["stores_raw_uid"] is False
|
|
assert receipt["stores_raw_output"] is False
|
|
assert RAW_UID not in json.dumps(receipt, sort_keys=True)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("target", "patch", "blocker"),
|
|
[
|
|
("pre", {"uid": "other"}, "workload_uid_mismatch"),
|
|
("post", {"generation": 4}, "rollout_generation_not_advanced"),
|
|
("post", {"observed_generation": 4}, "rollout_generation_not_observed"),
|
|
("post", {"ready_replicas": 1}, "rollout_replicas_not_converged"),
|
|
("post", {"unavailable_replicas": 1}, "rollout_has_unavailable_replicas"),
|
|
("post", {"name": "other"}, "post_state_identity_mismatch"),
|
|
("post", {"run_id": "replayed"}, "post_state_run_id_invalid"),
|
|
("post", {"source": "caller_supplied"}, "post_state_source_invalid"),
|
|
("post", {"trusted_server_readback": False}, "post_state_trusted_server_readback_invalid"),
|
|
("post", {"observed_at_epoch_ms": 1000}, "state_readback_order_invalid"),
|
|
],
|
|
)
|
|
def test_rollout_verifier_fails_closed_on_drift_or_fabrication(
|
|
target: str,
|
|
patch: dict,
|
|
blocker: str,
|
|
) -> None:
|
|
claim, context, pre, post = _same_run_states()
|
|
(pre if target == "pre" else post).update(patch)
|
|
|
|
receipt = build_kubernetes_rollout_verifier_receipt(
|
|
claim=claim,
|
|
pre_state=pre,
|
|
post_state=post,
|
|
execution_context=context,
|
|
)
|
|
|
|
assert receipt["status"] == "failed_closed"
|
|
assert receipt["verified"] is False
|
|
assert blocker in receipt["blockers"]
|
|
|
|
|
|
def test_replayed_pre_state_cannot_be_reused_as_post_state() -> None:
|
|
claim, context, pre, _post = _same_run_states()
|
|
replay = {**pre, "phase": "post"}
|
|
|
|
receipt = build_kubernetes_rollout_verifier_receipt(
|
|
claim=claim,
|
|
pre_state=pre,
|
|
post_state=replay,
|
|
execution_context=context,
|
|
)
|
|
|
|
assert receipt["verified"] is False
|
|
assert "state_readback_receipt_replayed" in receipt["blockers"]
|
|
|
|
|
|
def test_state_receipts_cannot_be_replayed_under_another_server_context() -> None:
|
|
claim, first_context, pre, post = _same_run_states()
|
|
second_context = issue_kubernetes_execution_context(claim)
|
|
|
|
receipt = build_kubernetes_rollout_verifier_receipt(
|
|
claim=claim,
|
|
pre_state=pre,
|
|
post_state=post,
|
|
execution_context=second_context,
|
|
)
|
|
|
|
assert second_context is not first_context
|
|
assert receipt["verified"] is False
|
|
assert "pre_state_trace_id_invalid" in receipt["blockers"]
|
|
assert "post_state_execution_context_sha256_invalid" in receipt["blockers"]
|
|
|
|
|
|
def test_malformed_claim_returns_failed_closed_instead_of_raising() -> None:
|
|
receipt = build_kubernetes_rollout_verifier_receipt(
|
|
claim=None,
|
|
pre_state=None,
|
|
post_state=None,
|
|
execution_context=object(),
|
|
)
|
|
|
|
assert receipt["verified"] is False
|
|
assert "controlled_execution_claim_invalid" in receipt["blockers"]
|
|
assert "server_issued_execution_context_invalid" in receipt["blockers"]
|
|
|
|
|
|
class _StateExecutor:
|
|
def __init__(self, states: list[dict]) -> None:
|
|
self.states = list(states)
|
|
|
|
async def read_workload_state(self, *_args) -> dict:
|
|
return self.states.pop(0)
|
|
|
|
|
|
class _HangingStateExecutor:
|
|
async def read_workload_state(self, *_args) -> dict:
|
|
await asyncio.sleep(10)
|
|
return _raw_post()
|
|
|
|
|
|
class _PreThenHangingStateExecutor:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
async def read_workload_state(self, *_args) -> dict:
|
|
self.calls += 1
|
|
if self.calls == 1:
|
|
return _raw_pre()
|
|
await asyncio.sleep(10)
|
|
return _raw_post()
|
|
|
|
|
|
class _ReceiptWriter:
|
|
def __init__(self, fail_statuses: set[str] | None = None) -> None:
|
|
self.fail_statuses = fail_statuses or set()
|
|
self.receipts: list[dict] = []
|
|
|
|
async def write(self, *, claim, receipt) -> bool:
|
|
assert claim["claim_id"] == receipt["claim_id"]
|
|
self.receipts.append(dict(receipt))
|
|
return receipt["status"] not in self.fail_statuses
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runtime_verifier_reads_pre_post_and_requires_durable_writeback() -> None:
|
|
writer = _ReceiptWriter()
|
|
verifier = KubernetesRolloutVerifier(
|
|
state_executor=_StateExecutor([_raw_pre(), _raw_post()]),
|
|
receipt_writer=writer,
|
|
max_attempts=1,
|
|
interval_seconds=0,
|
|
)
|
|
claim = _claim()
|
|
|
|
context, pre = await verifier.prepare(claim)
|
|
lifecycle = await verifier.record_execution_outcome(
|
|
claim=claim,
|
|
execution_context=context,
|
|
runtime_write_outcome="confirmed_applied",
|
|
)
|
|
receipt = await verifier.verify(
|
|
claim=claim,
|
|
execution_context=context,
|
|
pre_state=pre,
|
|
)
|
|
|
|
assert receipt["verified"] is True
|
|
assert receipt["durable_writeback_ack"] is True
|
|
assert lifecycle["durable_writeback_ack"] is True
|
|
assert [item["status"] for item in writer.receipts] == [
|
|
"prepared",
|
|
"applied_pending_verification",
|
|
"verified_success",
|
|
]
|
|
assert RAW_UID not in json.dumps(writer.receipts, sort_keys=True)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_deadline_verifier_can_converge_after_more_than_four_samples() -> None:
|
|
writer = _ReceiptWriter()
|
|
state_executor = _StateExecutor(
|
|
[_raw_pre(), *[_raw_pre() for _ in range(5)], _raw_post()]
|
|
)
|
|
verifier = KubernetesRolloutVerifier(
|
|
state_executor=state_executor,
|
|
receipt_writer=writer,
|
|
max_attempts=8,
|
|
interval_seconds=0,
|
|
deadline_seconds=1,
|
|
write_timeout_seconds=0.05,
|
|
)
|
|
claim = _claim()
|
|
|
|
context, pre = await verifier.prepare(claim)
|
|
await verifier.record_execution_outcome(
|
|
claim=claim,
|
|
execution_context=context,
|
|
runtime_write_outcome="confirmed_applied",
|
|
)
|
|
receipt = await verifier.verify(
|
|
claim=claim,
|
|
execution_context=context,
|
|
pre_state=pre,
|
|
)
|
|
|
|
assert receipt["verified"] is True
|
|
assert state_executor.states == []
|
|
assert writer.receipts[-1]["status"] == "verified_success"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runtime_verifier_fails_closure_when_writeback_is_missing() -> None:
|
|
verifier = KubernetesRolloutVerifier(
|
|
state_executor=_StateExecutor([_raw_pre(), _raw_post()]),
|
|
receipt_writer=_ReceiptWriter(fail_statuses={"verified_success"}),
|
|
max_attempts=1,
|
|
interval_seconds=0,
|
|
)
|
|
claim = _claim()
|
|
|
|
context, pre = await verifier.prepare(claim)
|
|
await verifier.record_execution_outcome(
|
|
claim=claim,
|
|
execution_context=context,
|
|
runtime_write_outcome="confirmed_applied",
|
|
)
|
|
receipt = await verifier.verify(
|
|
claim=claim,
|
|
execution_context=context,
|
|
pre_state=pre,
|
|
)
|
|
|
|
assert receipt["verified"] is False
|
|
assert receipt["durable_writeback_ack"] is False
|
|
assert "durable_verifier_writeback_missing" in receipt["blockers"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pre_state_readback_timeout_happens_before_any_mutation() -> None:
|
|
writer = _ReceiptWriter()
|
|
verifier = KubernetesRolloutVerifier(
|
|
state_executor=_HangingStateExecutor(),
|
|
receipt_writer=writer,
|
|
max_attempts=1,
|
|
interval_seconds=0,
|
|
read_timeout_seconds=0.01,
|
|
)
|
|
|
|
with pytest.raises(TimeoutError):
|
|
await verifier.prepare(_claim())
|
|
assert writer.receipts[-1]["status"] == "terminal_prepare_failed_no_write"
|
|
assert writer.receipts[-1]["runtime_write_performed"] is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prepare_cancellation_persists_terminal_no_write_receipt() -> None:
|
|
writer = _ReceiptWriter()
|
|
verifier = KubernetesRolloutVerifier(
|
|
state_executor=_HangingStateExecutor(),
|
|
receipt_writer=writer,
|
|
read_timeout_seconds=5,
|
|
)
|
|
|
|
task = asyncio.create_task(verifier.prepare(_claim()))
|
|
await asyncio.sleep(0)
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
|
|
assert writer.receipts[-1]["status"] == "terminal_prepare_cancelled_no_write"
|
|
assert writer.receipts[-1]["runtime_write_performed"] is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_state_timeout_persists_failed_closed_receipt() -> None:
|
|
writer = _ReceiptWriter()
|
|
verifier = KubernetesRolloutVerifier(
|
|
state_executor=_PreThenHangingStateExecutor(),
|
|
receipt_writer=writer,
|
|
max_attempts=1,
|
|
interval_seconds=0,
|
|
read_timeout_seconds=0.01,
|
|
)
|
|
claim = _claim()
|
|
|
|
context, pre = await verifier.prepare(claim)
|
|
await verifier.record_execution_outcome(
|
|
claim=claim,
|
|
execution_context=context,
|
|
runtime_write_outcome="unknown_after_dispatch",
|
|
)
|
|
receipt = await verifier.verify(
|
|
claim=claim,
|
|
execution_context=context,
|
|
pre_state=pre,
|
|
)
|
|
|
|
assert receipt["verified"] is False
|
|
assert receipt["durable_writeback_ack"] is True
|
|
assert receipt["blockers"] == ["post_state_readback_unavailable"]
|
|
assert writer.receipts[-1]["status"] == "failed_closed"
|