"""Independent, durable Kubernetes rollout verification.""" from __future__ import annotations import asyncio import hashlib import time import uuid from collections.abc import Mapping from dataclasses import dataclass from typing import Any import structlog from src.services.executor import ActionExecutor from src.services.kubernetes_controlled_executor import ( KUBERNETES_VERIFIER_ID, validate_kubernetes_controlled_execution_claim, ) logger = structlog.get_logger(__name__) VERIFIER_RECEIPT_SCHEMA_VERSION = "kubernetes_rollout_verifier_receipt_v1" LIFECYCLE_RECEIPT_SCHEMA_VERSION = "kubernetes_rollout_lifecycle_receipt_v1" STATE_RECEIPT_SCHEMA_VERSION = "kubernetes_workload_state_readback_v1" STATE_RECEIPT_SOURCE = "kubernetes_api_independent_readback" STATE_RECEIPT_VERIFIER = "kubernetes_rollout_state_reader" @dataclass(slots=True) class _KubernetesExecutionContext: """Opaque server-issued correlation that model JSON cannot fabricate.""" trace_id: str run_id: str claim_id: str nonce: str lifecycle_required: bool = False prepared_writeback_ack: bool = False applied_pending_writeback_ack: bool = False def issue_kubernetes_execution_context( claim: Mapping[str, Any] | None, ) -> object: valid, reason = validate_kubernetes_controlled_execution_claim(claim) if not valid: raise ValueError(f"kubernetes_execution_claim_invalid:{reason}") assert claim is not None return _KubernetesExecutionContext( trace_id=f"trace:k8s:{uuid.uuid4()}", run_id=f"run:k8s:{uuid.uuid4()}", claim_id=str(claim.get("claim_id") or ""), nonce=uuid.uuid4().hex, ) def _context_digest(context: _KubernetesExecutionContext) -> str: return hashlib.sha256(context.nonce.encode()).hexdigest() def _integer(value: Any) -> int | None: if isinstance(value, bool): return None try: parsed = int(value) except (TypeError, ValueError): return None return parsed if parsed >= 0 else None def build_kubernetes_state_readback_receipt( *, claim: Mapping[str, Any], execution_context: object, phase: str, raw_state: Mapping[str, Any], observed_at_epoch_ms: int | None = None, ) -> dict[str, Any]: """Wrap one direct Kubernetes API observation in trusted same-run metadata.""" if not isinstance(execution_context, _KubernetesExecutionContext): raise ValueError("server_issued_execution_context_required") if phase not in {"pre", "post"}: raise ValueError("state_readback_phase_invalid") valid, reason = validate_kubernetes_controlled_execution_claim(claim) if not valid: raise ValueError(f"controlled_execution_claim_invalid:{reason}") if execution_context.claim_id != str(claim.get("claim_id") or ""): raise ValueError("execution_context_claim_mismatch") if raw_state.get("error"): raise RuntimeError("kubernetes_state_readback_failed") observed_at = observed_at_epoch_ms or time.time_ns() // 1_000_000 return { "schema_version": STATE_RECEIPT_SCHEMA_VERSION, "receipt_id": f"k8s-state:{phase}:{uuid.uuid4()}", "source": STATE_RECEIPT_SOURCE, "verified_by": STATE_RECEIPT_VERIFIER, "trusted_server_readback": True, "phase": phase, "trace_id": execution_context.trace_id, "run_id": execution_context.run_id, "claim_id": execution_context.claim_id, "execution_context_sha256": _context_digest(execution_context), "observed_at_epoch_ms": observed_at, "namespace": str(raw_state.get("namespace") or ""), "kind": str(raw_state.get("kind") or "").lower(), "name": str(raw_state.get("name") or "").lower(), "uid": str(raw_state.get("uid") or ""), "generation": raw_state.get("generation"), "observed_generation": raw_state.get("observed_generation"), "desired_replicas": raw_state.get("desired_replicas"), "updated_replicas": raw_state.get("updated_replicas"), "ready_replicas": raw_state.get("ready_replicas"), "available_replicas": raw_state.get("available_replicas"), "unavailable_replicas": raw_state.get("unavailable_replicas"), } def _state_receipt_blockers( *, state: Mapping[str, Any], phase: str, claim: Mapping[str, Any], context: _KubernetesExecutionContext, ) -> list[str]: blockers: list[str] = [] required = { "schema_version": STATE_RECEIPT_SCHEMA_VERSION, "source": STATE_RECEIPT_SOURCE, "verified_by": STATE_RECEIPT_VERIFIER, "trusted_server_readback": True, "phase": phase, "trace_id": context.trace_id, "run_id": context.run_id, "claim_id": context.claim_id, "execution_context_sha256": _context_digest(context), } for field, expected in required.items(): if state.get(field) != expected: blockers.append(f"{phase}_state_{field}_invalid") if not str(state.get("receipt_id") or ""): blockers.append(f"{phase}_state_receipt_id_missing") expected_identity = ( str(claim.get("namespace") or ""), str(claim.get("workload_kind") or ""), str(claim.get("workload_name") or ""), ) observed_identity = ( str(state.get("namespace") or ""), str(state.get("kind") or ""), str(state.get("name") or ""), ) if observed_identity != expected_identity: blockers.append(f"{phase}_state_identity_mismatch") return blockers def build_kubernetes_rollout_verifier_receipt( *, claim: Mapping[str, Any] | None, pre_state: Mapping[str, Any] | None, post_state: Mapping[str, Any] | None, execution_context: object, ) -> dict[str, Any]: """Verify exact identity, unique same-run readbacks, and rollout convergence.""" blockers: list[str] = [] claim_mapping = claim if isinstance(claim, Mapping) else {} pre = pre_state if isinstance(pre_state, Mapping) else {} post = post_state if isinstance(post_state, Mapping) else {} claim_valid, _claim_reason = validate_kubernetes_controlled_execution_claim(claim) context = ( execution_context if isinstance(execution_context, _KubernetesExecutionContext) else None ) if not claim_valid: blockers.append("controlled_execution_claim_invalid") if context is None: blockers.append("server_issued_execution_context_invalid") elif context.claim_id != str(claim_mapping.get("claim_id") or ""): blockers.append("execution_context_claim_mismatch") elif context.lifecycle_required: if not context.prepared_writeback_ack: blockers.append("prepared_writeback_missing") if not context.applied_pending_writeback_ack: blockers.append("applied_pending_writeback_missing") if claim_valid and context is not None: blockers.extend( _state_receipt_blockers( state=pre, phase="pre", claim=claim_mapping, context=context, ) ) blockers.extend( _state_receipt_blockers( state=post, phase="post", claim=claim_mapping, context=context, ) ) pre_receipt_id = str(pre.get("receipt_id") or "") post_receipt_id = str(post.get("receipt_id") or "") if pre_receipt_id and pre_receipt_id == post_receipt_id: blockers.append("state_readback_receipt_replayed") pre_observed_at = _integer(pre.get("observed_at_epoch_ms")) post_observed_at = _integer(post.get("observed_at_epoch_ms")) if ( pre_observed_at is None or post_observed_at is None or post_observed_at <= pre_observed_at ): blockers.append("state_readback_order_invalid") pre_uid = str(pre.get("uid") or "") post_uid = str(post.get("uid") or "") if not pre_uid or not post_uid or pre_uid != post_uid: blockers.append("workload_uid_mismatch") pre_generation = _integer(pre.get("generation")) generation = _integer(post.get("generation")) observed_generation = _integer(post.get("observed_generation")) desired = _integer(post.get("desired_replicas")) updated = _integer(post.get("updated_replicas")) ready = _integer(post.get("ready_replicas")) available = _integer(post.get("available_replicas")) unavailable = _integer(post.get("unavailable_replicas")) if pre_generation is None or generation is None or generation <= pre_generation: blockers.append("rollout_generation_not_advanced") if generation is None or observed_generation is None or observed_generation < generation: blockers.append("rollout_generation_not_observed") if desired is None or desired < 1: blockers.append("desired_replicas_invalid") elif any(value is None or value < desired for value in (updated, ready, available)): blockers.append("rollout_replicas_not_converged") if unavailable is None or unavailable != 0: blockers.append("rollout_has_unavailable_replicas") blockers = list(dict.fromkeys(blockers)) verified = not blockers uid_digest = hashlib.sha256(post_uid.encode()).hexdigest() if post_uid else None return { "schema_version": VERIFIER_RECEIPT_SCHEMA_VERSION, "receipt_id": f"k8s-rollout:{uuid.uuid4()}", "status": "verified_success" if verified else "failed_closed", "verified": verified, "verifier": KUBERNETES_VERIFIER_ID, "claim_id": str(claim_mapping.get("claim_id") or "") or None, "incident_id": str(claim_mapping.get("incident_id") or "") or None, "trace_id": context.trace_id if context else None, "run_id": context.run_id if context else None, "route_id": str(claim_mapping.get("route_id") or "") or None, "canonical_asset_id": str(claim_mapping.get("canonical_asset_id") or "") or None, "namespace": str(claim_mapping.get("namespace") or "") or None, "workload_kind": str(claim_mapping.get("workload_kind") or "") or None, "workload_name": str(claim_mapping.get("workload_name") or "") or None, "workload_uid_sha256": uid_digest, "pre_state_receipt_id": pre_receipt_id or None, "post_state_receipt_id": post_receipt_id or None, "pre_generation": pre_generation, "generation": generation, "observed_generation": observed_generation, "desired_replicas": desired, "updated_replicas": updated, "ready_replicas": ready, "available_replicas": available, "unavailable_replicas": unavailable, "blockers": blockers, "durable_writeback_ack": False, "stores_raw_uid": False, "stores_raw_output": False, } class KubernetesRolloutReceiptWriter: """Persist the public-safe verifier receipt into incident evidence.""" async def write( self, *, claim: Mapping[str, Any], receipt: Mapping[str, Any], ) -> bool: try: from src.services.evidence_snapshot import EvidenceSnapshot snapshot = EvidenceSnapshot( incident_id=str(claim.get("incident_id") or ""), project_id="awoooi", ) evidence_key = ( "kubernetes_rollout_lifecycle" if receipt.get("schema_version") == LIFECYCLE_RECEIPT_SCHEMA_VERSION else "kubernetes_rollout_verifier" ) snapshot.post_execution_state = {evidence_key: dict(receipt)} if receipt.get("status") in { "prepared", "applied_pending_verification", "write_outcome_unknown_pending_verification", "no_write_pending_verification", }: snapshot.verification_result = "pending" else: snapshot.verification_result = ( "success" if receipt.get("verified") is True else "failed" ) snapshot.sensors_attempted = 1 snapshot.sensors_succeeded = 1 snapshot.mcp_health = {KUBERNETES_VERIFIER_ID: True} snapshot.evidence_summary = ( "[KubernetesRolloutVerifier] exact same-run receipt; " f"claim_id={receipt.get('claim_id')}; status={receipt.get('status')}" ) await snapshot.save() return True except Exception as exc: logger.warning( "kubernetes_rollout_receipt_write_failed", incident_id=str(claim.get("incident_id") or ""), error_type=type(exc).__name__, ) return False class KubernetesRolloutVerifier: """Independent state reader + durable fail-closed verifier.""" def __init__( self, *, state_executor: ActionExecutor | None = None, receipt_writer: KubernetesRolloutReceiptWriter | None = None, max_attempts: int = 12, interval_seconds: float = 4.0, deadline_seconds: float = 45.0, read_timeout_seconds: float = 3.0, write_timeout_seconds: float = 5.0, ) -> None: self._state_executor = state_executor or ActionExecutor() self._receipt_writer = receipt_writer or KubernetesRolloutReceiptWriter() self._max_attempts = max(1, max_attempts) self._interval_seconds = max(0.0, interval_seconds) self._deadline_seconds = max(0.1, deadline_seconds) self._read_timeout_seconds = max(0.1, read_timeout_seconds) self._write_timeout_seconds = max(0.1, write_timeout_seconds) @staticmethod def _bounded_timeout( configured_seconds: float, deadline_monotonic: float | None, ) -> float: if deadline_monotonic is None: return configured_seconds remaining = deadline_monotonic - time.monotonic() if remaining <= 0: raise TimeoutError("kubernetes_execution_deadline_exhausted") return min(configured_seconds, remaining) async def _read_state( self, claim: Mapping[str, Any], *, deadline_monotonic: float | None = None, ) -> dict[str, Any]: return await asyncio.wait_for( self._state_executor.read_workload_state( str(claim.get("workload_kind") or ""), str(claim.get("workload_name") or ""), str(claim.get("namespace") or ""), ), timeout=self._bounded_timeout( self._read_timeout_seconds, deadline_monotonic, ), ) async def _persist_receipt( self, *, claim: Mapping[str, Any], receipt: Mapping[str, Any], deadline_monotonic: float | None = None, ) -> bool: try: return await asyncio.wait_for( self._receipt_writer.write( claim=claim, receipt=receipt, ), timeout=self._bounded_timeout( self._write_timeout_seconds, deadline_monotonic, ), ) except Exception as exc: logger.warning( "kubernetes_rollout_receipt_persist_failed", claim_id=str(claim.get("claim_id") or ""), status=str(receipt.get("status") or ""), error_type=type(exc).__name__, ) return False @staticmethod def _lifecycle_receipt( *, claim: Mapping[str, Any], context: _KubernetesExecutionContext, status: str, runtime_write_outcome: str | None = None, terminal: bool = False, blockers: list[str] | None = None, ) -> dict[str, Any]: return { "schema_version": LIFECYCLE_RECEIPT_SCHEMA_VERSION, "receipt_id": f"k8s-lifecycle:{uuid.uuid4()}", "status": status, "verified": False, "verifier": KUBERNETES_VERIFIER_ID, "claim_id": context.claim_id, "incident_id": str(claim.get("incident_id") or "") or None, "trace_id": context.trace_id, "run_id": context.run_id, "route_id": str(claim.get("route_id") or "") or None, "canonical_asset_id": ( str(claim.get("canonical_asset_id") or "") or None ), "namespace": str(claim.get("namespace") or "") or None, "workload_kind": str(claim.get("workload_kind") or "") or None, "workload_name": str(claim.get("workload_name") or "") or None, "runtime_write_outcome": runtime_write_outcome, "runtime_write_performed": ( runtime_write_outcome == "confirmed_applied" ), "terminal": terminal, "blockers": list(blockers or []), "durable_writeback_ack": True, "stores_raw_uid": False, "stores_raw_output": False, } async def prepare( self, claim: Mapping[str, Any], *, deadline_monotonic: float | None = None, ) -> tuple[object, dict[str, Any]]: context = issue_kubernetes_execution_context(claim) assert isinstance(context, _KubernetesExecutionContext) context.lifecycle_required = True try: raw_state = await self._read_state( claim, deadline_monotonic=deadline_monotonic, ) pre_state = build_kubernetes_state_readback_receipt( claim=claim, execution_context=context, phase="pre", raw_state=raw_state, ) prepared_receipt = self._lifecycle_receipt( claim=claim, context=context, status="prepared", ) context.prepared_writeback_ack = await self._persist_receipt( claim=claim, receipt=prepared_receipt, deadline_monotonic=deadline_monotonic, ) if not context.prepared_writeback_ack: raise RuntimeError("kubernetes_prepared_writeback_missing") return context, pre_state except asyncio.CancelledError: cancelled_receipt = self._lifecycle_receipt( claim=claim, context=context, status="terminal_prepare_cancelled_no_write", runtime_write_outcome="confirmed_no_write", terminal=True, blockers=["prepare_cancelled"], ) await asyncio.shield( self._persist_receipt( claim=claim, receipt=cancelled_receipt, deadline_monotonic=deadline_monotonic, ) ) raise except Exception as exc: failed_receipt = self._lifecycle_receipt( claim=claim, context=context, status="terminal_prepare_failed_no_write", runtime_write_outcome="confirmed_no_write", terminal=True, blockers=[f"prepare_failed:{type(exc).__name__}"], ) await self._persist_receipt( claim=claim, receipt=failed_receipt, deadline_monotonic=deadline_monotonic, ) raise async def record_execution_outcome( self, *, claim: Mapping[str, Any], execution_context: object, runtime_write_outcome: str, deadline_monotonic: float | None = None, ) -> dict[str, Any]: if not isinstance(execution_context, _KubernetesExecutionContext): raise ValueError("server_issued_execution_context_required") if execution_context.claim_id != str(claim.get("claim_id") or ""): raise ValueError("execution_context_claim_mismatch") if runtime_write_outcome == "confirmed_applied": status = "applied_pending_verification" elif runtime_write_outcome == "unknown_after_dispatch": status = "write_outcome_unknown_pending_verification" else: status = "no_write_pending_verification" receipt = self._lifecycle_receipt( claim=claim, context=execution_context, status=status, runtime_write_outcome=runtime_write_outcome, ) execution_context.applied_pending_writeback_ack = await self._persist_receipt( claim=claim, receipt=receipt, deadline_monotonic=deadline_monotonic, ) return { **receipt, "durable_writeback_ack": ( execution_context.applied_pending_writeback_ack ), } async def record_terminal_outcome( self, *, claim: Mapping[str, Any], execution_context: object, status: str, runtime_write_outcome: str, blockers: list[str] | None = None, deadline_monotonic: float | None = None, ) -> dict[str, Any]: if not isinstance(execution_context, _KubernetesExecutionContext): raise ValueError("server_issued_execution_context_required") if execution_context.claim_id != str(claim.get("claim_id") or ""): raise ValueError("execution_context_claim_mismatch") receipt = self._lifecycle_receipt( claim=claim, context=execution_context, status=status, runtime_write_outcome=runtime_write_outcome, terminal=True, blockers=blockers, ) persisted = await self._persist_receipt( claim=claim, receipt=receipt, deadline_monotonic=deadline_monotonic, ) return {**receipt, "durable_writeback_ack": persisted} async def verify( self, *, claim: Mapping[str, Any], execution_context: object, pre_state: Mapping[str, Any], deadline_monotonic: float | None = None, ) -> dict[str, Any]: receipt: dict[str, Any] | None = None deadline = time.monotonic() + self._deadline_seconds if deadline_monotonic is not None: deadline = min(deadline, deadline_monotonic) poll_deadline = deadline - min(self._write_timeout_seconds, 5.0) for attempt in range(self._max_attempts): if attempt and self._interval_seconds: remaining = poll_deadline - time.monotonic() if remaining <= 0: break await asyncio.sleep(min(self._interval_seconds, remaining)) if time.monotonic() >= poll_deadline: break try: raw_state = await self._read_state( claim, deadline_monotonic=poll_deadline, ) pre_observed_at = _integer(pre_state.get("observed_at_epoch_ms")) or 0 post_state = build_kubernetes_state_readback_receipt( claim=claim, execution_context=execution_context, phase="post", raw_state=raw_state, observed_at_epoch_ms=max( time.time_ns() // 1_000_000, pre_observed_at + 1, ), ) except Exception as exc: logger.warning( "kubernetes_rollout_post_readback_failed", claim_id=str(claim.get("claim_id") or ""), error_type=type(exc).__name__, ) continue receipt = build_kubernetes_rollout_verifier_receipt( claim=claim, pre_state=pre_state, post_state=post_state, execution_context=execution_context, ) if receipt["verified"] is True: break if time.monotonic() >= poll_deadline: break if receipt is None: receipt = { "schema_version": VERIFIER_RECEIPT_SCHEMA_VERSION, "receipt_id": f"k8s-rollout:{uuid.uuid4()}", "status": "failed_closed", "verified": False, "verifier": KUBERNETES_VERIFIER_ID, "claim_id": str(claim.get("claim_id") or "") or None, "incident_id": str(claim.get("incident_id") or "") or None, "blockers": ["post_state_readback_unavailable"], "durable_writeback_ack": False, "stores_raw_uid": False, "stores_raw_output": False, } durable_receipt = {**receipt, "durable_writeback_ack": True} persisted = await self._persist_receipt( claim=claim, receipt=durable_receipt, deadline_monotonic=deadline, ) if persisted: return durable_receipt blockers = list(receipt.get("blockers") or []) blockers.append("durable_verifier_writeback_missing") return { **receipt, "status": "failed_closed", "verified": False, "blockers": list(dict.fromkeys(blockers)), "durable_writeback_ack": False, } _verifier: KubernetesRolloutVerifier | None = None def get_kubernetes_rollout_verifier() -> KubernetesRolloutVerifier: global _verifier if _verifier is None: _verifier = KubernetesRolloutVerifier() return _verifier