"""Focused tests for trusted same-run Kubernetes rollout verification.""" from __future__ import annotations import asyncio import json import pytest from src.services.kubernetes_controlled_executor import ( build_kubernetes_controlled_execution_claim, ) from src.services.kubernetes_rollout_verifier import ( KubernetesRolloutVerifier, build_kubernetes_rollout_verifier_receipt, build_kubernetes_state_readback_receipt, issue_kubernetes_execution_context, ) RAW_UID = "12345678-1234-4abc-8def-1234567890ab" def _claim() -> dict: route = { "schema_version": "typed_domain_target_route_v2", "resolution_status": "resolved", "route_id": "typed:kubernetes_workload:service-awoooi-api", "target_kind": "kubernetes_workload", "target_resource": "awoooi-api", "source_namespace": "awoooi-prod", "canonical_asset_id": "service:awoooi-api", "executor": "kubernetes_controlled_executor", "verifier": "kubernetes_rollout_verifier", "controlled_apply_allowed": True, "cross_domain_fallback_allowed": False, "identity_evidence": { "status": "verified", "namespace": "awoooi-prod", "kind": "deployment", "name": "awoooi-api", }, } return build_kubernetes_controlled_execution_claim( action=( "kubectl rollout restart deployment/awoooi-api " "-n awoooi-prod" ), typed_target_route=route, incident_id="INC-K8S-001", ) def _raw_pre() -> dict: return { "namespace": "awoooi-prod", "kind": "deployment", "name": "awoooi-api", "uid": RAW_UID, "generation": 4, "observed_generation": 4, "desired_replicas": 2, "updated_replicas": 2, "ready_replicas": 2, "available_replicas": 2, "unavailable_replicas": 0, } def _raw_post() -> dict: return { **_raw_pre(), "generation": 5, "observed_generation": 5, } def _same_run_states() -> tuple[dict, object, dict, dict]: claim = _claim() context = issue_kubernetes_execution_context(claim) pre = build_kubernetes_state_readback_receipt( claim=claim, execution_context=context, phase="pre", raw_state=_raw_pre(), observed_at_epoch_ms=1000, ) post = build_kubernetes_state_readback_receipt( claim=claim, execution_context=context, phase="post", raw_state=_raw_post(), observed_at_epoch_ms=2000, ) return claim, context, pre, post def test_exact_same_run_rollout_convergence_is_verified_public_safely() -> None: claim, context, pre, post = _same_run_states() receipt = build_kubernetes_rollout_verifier_receipt( claim=claim, pre_state=pre, post_state=post, execution_context=context, ) assert receipt["status"] == "verified_success" assert receipt["verified"] is True assert receipt["blockers"] == [] assert len(receipt["workload_uid_sha256"]) == 64 assert receipt["trace_id"].startswith("trace:k8s:") assert receipt["run_id"].startswith("run:k8s:") assert receipt["stores_raw_uid"] is False assert receipt["stores_raw_output"] is False assert RAW_UID not in json.dumps(receipt, sort_keys=True) @pytest.mark.parametrize( ("target", "patch", "blocker"), [ ("pre", {"uid": "other"}, "workload_uid_mismatch"), ("post", {"generation": 4}, "rollout_generation_not_advanced"), ("post", {"observed_generation": 4}, "rollout_generation_not_observed"), ("post", {"ready_replicas": 1}, "rollout_replicas_not_converged"), ("post", {"unavailable_replicas": 1}, "rollout_has_unavailable_replicas"), ("post", {"name": "other"}, "post_state_identity_mismatch"), ("post", {"run_id": "replayed"}, "post_state_run_id_invalid"), ("post", {"source": "caller_supplied"}, "post_state_source_invalid"), ("post", {"trusted_server_readback": False}, "post_state_trusted_server_readback_invalid"), ("post", {"observed_at_epoch_ms": 1000}, "state_readback_order_invalid"), ], ) def test_rollout_verifier_fails_closed_on_drift_or_fabrication( target: str, patch: dict, blocker: str, ) -> None: claim, context, pre, post = _same_run_states() (pre if target == "pre" else post).update(patch) receipt = build_kubernetes_rollout_verifier_receipt( claim=claim, pre_state=pre, post_state=post, execution_context=context, ) assert receipt["status"] == "failed_closed" assert receipt["verified"] is False assert blocker in receipt["blockers"] def test_replayed_pre_state_cannot_be_reused_as_post_state() -> None: claim, context, pre, _post = _same_run_states() replay = {**pre, "phase": "post"} receipt = build_kubernetes_rollout_verifier_receipt( claim=claim, pre_state=pre, post_state=replay, execution_context=context, ) assert receipt["verified"] is False assert "state_readback_receipt_replayed" in receipt["blockers"] def test_state_receipts_cannot_be_replayed_under_another_server_context() -> None: claim, first_context, pre, post = _same_run_states() second_context = issue_kubernetes_execution_context(claim) receipt = build_kubernetes_rollout_verifier_receipt( claim=claim, pre_state=pre, post_state=post, execution_context=second_context, ) assert second_context is not first_context assert receipt["verified"] is False assert "pre_state_trace_id_invalid" in receipt["blockers"] assert "post_state_execution_context_sha256_invalid" in receipt["blockers"] def test_malformed_claim_returns_failed_closed_instead_of_raising() -> None: receipt = build_kubernetes_rollout_verifier_receipt( claim=None, pre_state=None, post_state=None, execution_context=object(), ) assert receipt["verified"] is False assert "controlled_execution_claim_invalid" in receipt["blockers"] assert "server_issued_execution_context_invalid" in receipt["blockers"] class _StateExecutor: def __init__(self, states: list[dict]) -> None: self.states = list(states) async def read_workload_state(self, *_args) -> dict: return self.states.pop(0) class _HangingStateExecutor: async def read_workload_state(self, *_args) -> dict: await asyncio.sleep(10) return _raw_post() class _PreThenHangingStateExecutor: def __init__(self) -> None: self.calls = 0 async def read_workload_state(self, *_args) -> dict: self.calls += 1 if self.calls == 1: return _raw_pre() await asyncio.sleep(10) return _raw_post() class _ReceiptWriter: def __init__(self, fail_statuses: set[str] | None = None) -> None: self.fail_statuses = fail_statuses or set() self.receipts: list[dict] = [] async def write(self, *, claim, receipt) -> bool: assert claim["claim_id"] == receipt["claim_id"] self.receipts.append(dict(receipt)) return receipt["status"] not in self.fail_statuses @pytest.mark.asyncio async def test_runtime_verifier_reads_pre_post_and_requires_durable_writeback() -> None: writer = _ReceiptWriter() verifier = KubernetesRolloutVerifier( state_executor=_StateExecutor([_raw_pre(), _raw_post()]), receipt_writer=writer, max_attempts=1, interval_seconds=0, ) claim = _claim() context, pre = await verifier.prepare(claim) lifecycle = await verifier.record_execution_outcome( claim=claim, execution_context=context, runtime_write_outcome="confirmed_applied", ) receipt = await verifier.verify( claim=claim, execution_context=context, pre_state=pre, ) assert receipt["verified"] is True assert receipt["durable_writeback_ack"] is True assert lifecycle["durable_writeback_ack"] is True assert [item["status"] for item in writer.receipts] == [ "prepared", "applied_pending_verification", "verified_success", ] assert RAW_UID not in json.dumps(writer.receipts, sort_keys=True) @pytest.mark.asyncio async def test_deadline_verifier_can_converge_after_more_than_four_samples() -> None: writer = _ReceiptWriter() state_executor = _StateExecutor( [_raw_pre(), *[_raw_pre() for _ in range(5)], _raw_post()] ) verifier = KubernetesRolloutVerifier( state_executor=state_executor, receipt_writer=writer, max_attempts=8, interval_seconds=0, deadline_seconds=1, write_timeout_seconds=0.05, ) claim = _claim() context, pre = await verifier.prepare(claim) await verifier.record_execution_outcome( claim=claim, execution_context=context, runtime_write_outcome="confirmed_applied", ) receipt = await verifier.verify( claim=claim, execution_context=context, pre_state=pre, ) assert receipt["verified"] is True assert state_executor.states == [] assert writer.receipts[-1]["status"] == "verified_success" @pytest.mark.asyncio async def test_runtime_verifier_fails_closure_when_writeback_is_missing() -> None: verifier = KubernetesRolloutVerifier( state_executor=_StateExecutor([_raw_pre(), _raw_post()]), receipt_writer=_ReceiptWriter(fail_statuses={"verified_success"}), max_attempts=1, interval_seconds=0, ) claim = _claim() context, pre = await verifier.prepare(claim) await verifier.record_execution_outcome( claim=claim, execution_context=context, runtime_write_outcome="confirmed_applied", ) receipt = await verifier.verify( claim=claim, execution_context=context, pre_state=pre, ) assert receipt["verified"] is False assert receipt["durable_writeback_ack"] is False assert "durable_verifier_writeback_missing" in receipt["blockers"] @pytest.mark.asyncio async def test_pre_state_readback_timeout_happens_before_any_mutation() -> None: writer = _ReceiptWriter() verifier = KubernetesRolloutVerifier( state_executor=_HangingStateExecutor(), receipt_writer=writer, max_attempts=1, interval_seconds=0, read_timeout_seconds=0.01, ) with pytest.raises(TimeoutError): await verifier.prepare(_claim()) assert writer.receipts[-1]["status"] == "terminal_prepare_failed_no_write" assert writer.receipts[-1]["runtime_write_performed"] is False @pytest.mark.asyncio async def test_prepare_cancellation_persists_terminal_no_write_receipt() -> None: writer = _ReceiptWriter() verifier = KubernetesRolloutVerifier( state_executor=_HangingStateExecutor(), receipt_writer=writer, read_timeout_seconds=5, ) task = asyncio.create_task(verifier.prepare(_claim())) await asyncio.sleep(0) task.cancel() with pytest.raises(asyncio.CancelledError): await task assert writer.receipts[-1]["status"] == "terminal_prepare_cancelled_no_write" assert writer.receipts[-1]["runtime_write_performed"] is False @pytest.mark.asyncio async def test_post_state_timeout_persists_failed_closed_receipt() -> None: writer = _ReceiptWriter() verifier = KubernetesRolloutVerifier( state_executor=_PreThenHangingStateExecutor(), receipt_writer=writer, max_attempts=1, interval_seconds=0, read_timeout_seconds=0.01, ) claim = _claim() context, pre = await verifier.prepare(claim) await verifier.record_execution_outcome( claim=claim, execution_context=context, runtime_write_outcome="unknown_after_dispatch", ) receipt = await verifier.verify( claim=claim, execution_context=context, pre_state=pre, ) assert receipt["verified"] is False assert receipt["durable_writeback_ack"] is True assert receipt["blockers"] == ["post_state_readback_unavailable"] assert writer.receipts[-1]["status"] == "failed_closed"