feat(sre): enforce verified kubernetes repairs

apps/api/tests/test_kubernetes_rollout_verifier.py

@@ -0,0 +1,392 @@
+"""Focused tests for trusted same-run Kubernetes rollout verification."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+
+import pytest
+
+from src.services.kubernetes_controlled_executor import (
+    build_kubernetes_controlled_execution_claim,
+)
+from src.services.kubernetes_rollout_verifier import (
+    KubernetesRolloutVerifier,
+    build_kubernetes_rollout_verifier_receipt,
+    build_kubernetes_state_readback_receipt,
+    issue_kubernetes_execution_context,
+)
+
+RAW_UID = "12345678-1234-4abc-8def-1234567890ab"
+
+
+def _claim() -> dict:
+    route = {
+        "schema_version": "typed_domain_target_route_v2",
+        "resolution_status": "resolved",
+        "route_id": "typed:kubernetes_workload:service-awoooi-api",
+        "target_kind": "kubernetes_workload",
+        "target_resource": "awoooi-api",
+        "source_namespace": "awoooi-prod",
+        "canonical_asset_id": "service:awoooi-api",
+        "executor": "kubernetes_controlled_executor",
+        "verifier": "kubernetes_rollout_verifier",
+        "controlled_apply_allowed": True,
+        "cross_domain_fallback_allowed": False,
+        "identity_evidence": {
+            "status": "verified",
+            "namespace": "awoooi-prod",
+            "kind": "deployment",
+            "name": "awoooi-api",
+        },
+    }
+    return build_kubernetes_controlled_execution_claim(
+        action=(
+            "kubectl rollout restart deployment/awoooi-api "
+            "-n awoooi-prod"
+        ),
+        typed_target_route=route,
+        incident_id="INC-K8S-001",
+    )
+
+
+def _raw_pre() -> dict:
+    return {
+        "namespace": "awoooi-prod",
+        "kind": "deployment",
+        "name": "awoooi-api",
+        "uid": RAW_UID,
+        "generation": 4,
+        "observed_generation": 4,
+        "desired_replicas": 2,
+        "updated_replicas": 2,
+        "ready_replicas": 2,
+        "available_replicas": 2,
+        "unavailable_replicas": 0,
+    }
+
+
+def _raw_post() -> dict:
+    return {
+        **_raw_pre(),
+        "generation": 5,
+        "observed_generation": 5,
+    }
+
+
+def _same_run_states() -> tuple[dict, object, dict, dict]:
+    claim = _claim()
+    context = issue_kubernetes_execution_context(claim)
+    pre = build_kubernetes_state_readback_receipt(
+        claim=claim,
+        execution_context=context,
+        phase="pre",
+        raw_state=_raw_pre(),
+        observed_at_epoch_ms=1000,
+    )
+    post = build_kubernetes_state_readback_receipt(
+        claim=claim,
+        execution_context=context,
+        phase="post",
+        raw_state=_raw_post(),
+        observed_at_epoch_ms=2000,
+    )
+    return claim, context, pre, post
+
+
+def test_exact_same_run_rollout_convergence_is_verified_public_safely() -> None:
+    claim, context, pre, post = _same_run_states()
+
+    receipt = build_kubernetes_rollout_verifier_receipt(
+        claim=claim,
+        pre_state=pre,
+        post_state=post,
+        execution_context=context,
+    )
+
+    assert receipt["status"] == "verified_success"
+    assert receipt["verified"] is True
+    assert receipt["blockers"] == []
+    assert len(receipt["workload_uid_sha256"]) == 64
+    assert receipt["trace_id"].startswith("trace:k8s:")
+    assert receipt["run_id"].startswith("run:k8s:")
+    assert receipt["stores_raw_uid"] is False
+    assert receipt["stores_raw_output"] is False
+    assert RAW_UID not in json.dumps(receipt, sort_keys=True)
+
+
+@pytest.mark.parametrize(
+    ("target", "patch", "blocker"),
+    [
+        ("pre", {"uid": "other"}, "workload_uid_mismatch"),
+        ("post", {"generation": 4}, "rollout_generation_not_advanced"),
+        ("post", {"observed_generation": 4}, "rollout_generation_not_observed"),
+        ("post", {"ready_replicas": 1}, "rollout_replicas_not_converged"),
+        ("post", {"unavailable_replicas": 1}, "rollout_has_unavailable_replicas"),
+        ("post", {"name": "other"}, "post_state_identity_mismatch"),
+        ("post", {"run_id": "replayed"}, "post_state_run_id_invalid"),
+        ("post", {"source": "caller_supplied"}, "post_state_source_invalid"),
+        ("post", {"trusted_server_readback": False}, "post_state_trusted_server_readback_invalid"),
+        ("post", {"observed_at_epoch_ms": 1000}, "state_readback_order_invalid"),
+    ],
+)
+def test_rollout_verifier_fails_closed_on_drift_or_fabrication(
+    target: str,
+    patch: dict,
+    blocker: str,
+) -> None:
+    claim, context, pre, post = _same_run_states()
+    (pre if target == "pre" else post).update(patch)
+
+    receipt = build_kubernetes_rollout_verifier_receipt(
+        claim=claim,
+        pre_state=pre,
+        post_state=post,
+        execution_context=context,
+    )
+
+    assert receipt["status"] == "failed_closed"
+    assert receipt["verified"] is False
+    assert blocker in receipt["blockers"]
+
+
+def test_replayed_pre_state_cannot_be_reused_as_post_state() -> None:
+    claim, context, pre, _post = _same_run_states()
+    replay = {**pre, "phase": "post"}
+
+    receipt = build_kubernetes_rollout_verifier_receipt(
+        claim=claim,
+        pre_state=pre,
+        post_state=replay,
+        execution_context=context,
+    )
+
+    assert receipt["verified"] is False
+    assert "state_readback_receipt_replayed" in receipt["blockers"]
+
+
+def test_state_receipts_cannot_be_replayed_under_another_server_context() -> None:
+    claim, first_context, pre, post = _same_run_states()
+    second_context = issue_kubernetes_execution_context(claim)
+
+    receipt = build_kubernetes_rollout_verifier_receipt(
+        claim=claim,
+        pre_state=pre,
+        post_state=post,
+        execution_context=second_context,
+    )
+
+    assert second_context is not first_context
+    assert receipt["verified"] is False
+    assert "pre_state_trace_id_invalid" in receipt["blockers"]
+    assert "post_state_execution_context_sha256_invalid" in receipt["blockers"]
+
+
+def test_malformed_claim_returns_failed_closed_instead_of_raising() -> None:
+    receipt = build_kubernetes_rollout_verifier_receipt(
+        claim=None,
+        pre_state=None,
+        post_state=None,
+        execution_context=object(),
+    )
+
+    assert receipt["verified"] is False
+    assert "controlled_execution_claim_invalid" in receipt["blockers"]
+    assert "server_issued_execution_context_invalid" in receipt["blockers"]
+
+
+class _StateExecutor:
+    def __init__(self, states: list[dict]) -> None:
+        self.states = list(states)
+
+    async def read_workload_state(self, *_args) -> dict:
+        return self.states.pop(0)
+
+
+class _HangingStateExecutor:
+    async def read_workload_state(self, *_args) -> dict:
+        await asyncio.sleep(10)
+        return _raw_post()
+
+
+class _PreThenHangingStateExecutor:
+    def __init__(self) -> None:
+        self.calls = 0
+
+    async def read_workload_state(self, *_args) -> dict:
+        self.calls += 1
+        if self.calls == 1:
+            return _raw_pre()
+        await asyncio.sleep(10)
+        return _raw_post()
+
+
+class _ReceiptWriter:
+    def __init__(self, fail_statuses: set[str] | None = None) -> None:
+        self.fail_statuses = fail_statuses or set()
+        self.receipts: list[dict] = []
+
+    async def write(self, *, claim, receipt) -> bool:
+        assert claim["claim_id"] == receipt["claim_id"]
+        self.receipts.append(dict(receipt))
+        return receipt["status"] not in self.fail_statuses
+
+
+@pytest.mark.asyncio
+async def test_runtime_verifier_reads_pre_post_and_requires_durable_writeback() -> None:
+    writer = _ReceiptWriter()
+    verifier = KubernetesRolloutVerifier(
+        state_executor=_StateExecutor([_raw_pre(), _raw_post()]),
+        receipt_writer=writer,
+        max_attempts=1,
+        interval_seconds=0,
+    )
+    claim = _claim()
+
+    context, pre = await verifier.prepare(claim)
+    lifecycle = await verifier.record_execution_outcome(
+        claim=claim,
+        execution_context=context,
+        runtime_write_outcome="confirmed_applied",
+    )
+    receipt = await verifier.verify(
+        claim=claim,
+        execution_context=context,
+        pre_state=pre,
+    )
+
+    assert receipt["verified"] is True
+    assert receipt["durable_writeback_ack"] is True
+    assert lifecycle["durable_writeback_ack"] is True
+    assert [item["status"] for item in writer.receipts] == [
+        "prepared",
+        "applied_pending_verification",
+        "verified_success",
+    ]
+    assert RAW_UID not in json.dumps(writer.receipts, sort_keys=True)
+
+
+@pytest.mark.asyncio
+async def test_deadline_verifier_can_converge_after_more_than_four_samples() -> None:
+    writer = _ReceiptWriter()
+    state_executor = _StateExecutor(
+        [_raw_pre(), *[_raw_pre() for _ in range(5)], _raw_post()]
+    )
+    verifier = KubernetesRolloutVerifier(
+        state_executor=state_executor,
+        receipt_writer=writer,
+        max_attempts=8,
+        interval_seconds=0,
+        deadline_seconds=1,
+        write_timeout_seconds=0.05,
+    )
+    claim = _claim()
+
+    context, pre = await verifier.prepare(claim)
+    await verifier.record_execution_outcome(
+        claim=claim,
+        execution_context=context,
+        runtime_write_outcome="confirmed_applied",
+    )
+    receipt = await verifier.verify(
+        claim=claim,
+        execution_context=context,
+        pre_state=pre,
+    )
+
+    assert receipt["verified"] is True
+    assert state_executor.states == []
+    assert writer.receipts[-1]["status"] == "verified_success"
+
+
+@pytest.mark.asyncio
+async def test_runtime_verifier_fails_closure_when_writeback_is_missing() -> None:
+    verifier = KubernetesRolloutVerifier(
+        state_executor=_StateExecutor([_raw_pre(), _raw_post()]),
+        receipt_writer=_ReceiptWriter(fail_statuses={"verified_success"}),
+        max_attempts=1,
+        interval_seconds=0,
+    )
+    claim = _claim()
+
+    context, pre = await verifier.prepare(claim)
+    await verifier.record_execution_outcome(
+        claim=claim,
+        execution_context=context,
+        runtime_write_outcome="confirmed_applied",
+    )
+    receipt = await verifier.verify(
+        claim=claim,
+        execution_context=context,
+        pre_state=pre,
+    )
+
+    assert receipt["verified"] is False
+    assert receipt["durable_writeback_ack"] is False
+    assert "durable_verifier_writeback_missing" in receipt["blockers"]
+
+
+@pytest.mark.asyncio
+async def test_pre_state_readback_timeout_happens_before_any_mutation() -> None:
+    writer = _ReceiptWriter()
+    verifier = KubernetesRolloutVerifier(
+        state_executor=_HangingStateExecutor(),
+        receipt_writer=writer,
+        max_attempts=1,
+        interval_seconds=0,
+        read_timeout_seconds=0.01,
+    )
+
+    with pytest.raises(TimeoutError):
+        await verifier.prepare(_claim())
+    assert writer.receipts[-1]["status"] == "terminal_prepare_failed_no_write"
+    assert writer.receipts[-1]["runtime_write_performed"] is False
+
+
+@pytest.mark.asyncio
+async def test_prepare_cancellation_persists_terminal_no_write_receipt() -> None:
+    writer = _ReceiptWriter()
+    verifier = KubernetesRolloutVerifier(
+        state_executor=_HangingStateExecutor(),
+        receipt_writer=writer,
+        read_timeout_seconds=5,
+    )
+
+    task = asyncio.create_task(verifier.prepare(_claim()))
+    await asyncio.sleep(0)
+    task.cancel()
+    with pytest.raises(asyncio.CancelledError):
+        await task
+
+    assert writer.receipts[-1]["status"] == "terminal_prepare_cancelled_no_write"
+    assert writer.receipts[-1]["runtime_write_performed"] is False
+
+
+@pytest.mark.asyncio
+async def test_post_state_timeout_persists_failed_closed_receipt() -> None:
+    writer = _ReceiptWriter()
+    verifier = KubernetesRolloutVerifier(
+        state_executor=_PreThenHangingStateExecutor(),
+        receipt_writer=writer,
+        max_attempts=1,
+        interval_seconds=0,
+        read_timeout_seconds=0.01,
+    )
+    claim = _claim()
+
+    context, pre = await verifier.prepare(claim)
+    await verifier.record_execution_outcome(
+        claim=claim,
+        execution_context=context,
+        runtime_write_outcome="unknown_after_dispatch",
+    )
+    receipt = await verifier.verify(
+        claim=claim,
+        execution_context=context,
+        pre_state=pre,
+    )
+
+    assert receipt["verified"] is False
+    assert receipt["durable_writeback_ack"] is True
+    assert receipt["blockers"] == ["post_state_readback_unavailable"]
+    assert writer.receipts[-1]["status"] == "failed_closed"