Files
awoooi/apps/api/src/services/kubernetes_rollout_verifier.py
2026-07-18 21:55:52 +08:00

681 lines
25 KiB
Python

"""Independent, durable Kubernetes rollout verification."""
from __future__ import annotations
import asyncio
import hashlib
import time
import uuid
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any
import structlog
from src.services.executor import ActionExecutor
from src.services.kubernetes_controlled_executor import (
KUBERNETES_VERIFIER_ID,
validate_kubernetes_controlled_execution_claim,
)
logger = structlog.get_logger(__name__)
VERIFIER_RECEIPT_SCHEMA_VERSION = "kubernetes_rollout_verifier_receipt_v1"
LIFECYCLE_RECEIPT_SCHEMA_VERSION = "kubernetes_rollout_lifecycle_receipt_v1"
STATE_RECEIPT_SCHEMA_VERSION = "kubernetes_workload_state_readback_v1"
STATE_RECEIPT_SOURCE = "kubernetes_api_independent_readback"
STATE_RECEIPT_VERIFIER = "kubernetes_rollout_state_reader"
@dataclass(slots=True)
class _KubernetesExecutionContext:
"""Opaque server-issued correlation that model JSON cannot fabricate."""
trace_id: str
run_id: str
claim_id: str
nonce: str
lifecycle_required: bool = False
prepared_writeback_ack: bool = False
applied_pending_writeback_ack: bool = False
def issue_kubernetes_execution_context(
claim: Mapping[str, Any] | None,
) -> object:
valid, reason = validate_kubernetes_controlled_execution_claim(claim)
if not valid:
raise ValueError(f"kubernetes_execution_claim_invalid:{reason}")
assert claim is not None
return _KubernetesExecutionContext(
trace_id=f"trace:k8s:{uuid.uuid4()}",
run_id=f"run:k8s:{uuid.uuid4()}",
claim_id=str(claim.get("claim_id") or ""),
nonce=uuid.uuid4().hex,
)
def _context_digest(context: _KubernetesExecutionContext) -> str:
return hashlib.sha256(context.nonce.encode()).hexdigest()
def _integer(value: Any) -> int | None:
if isinstance(value, bool):
return None
try:
parsed = int(value)
except (TypeError, ValueError):
return None
return parsed if parsed >= 0 else None
def build_kubernetes_state_readback_receipt(
*,
claim: Mapping[str, Any],
execution_context: object,
phase: str,
raw_state: Mapping[str, Any],
observed_at_epoch_ms: int | None = None,
) -> dict[str, Any]:
"""Wrap one direct Kubernetes API observation in trusted same-run metadata."""
if not isinstance(execution_context, _KubernetesExecutionContext):
raise ValueError("server_issued_execution_context_required")
if phase not in {"pre", "post"}:
raise ValueError("state_readback_phase_invalid")
valid, reason = validate_kubernetes_controlled_execution_claim(claim)
if not valid:
raise ValueError(f"controlled_execution_claim_invalid:{reason}")
if execution_context.claim_id != str(claim.get("claim_id") or ""):
raise ValueError("execution_context_claim_mismatch")
if raw_state.get("error"):
raise RuntimeError("kubernetes_state_readback_failed")
observed_at = observed_at_epoch_ms or time.time_ns() // 1_000_000
return {
"schema_version": STATE_RECEIPT_SCHEMA_VERSION,
"receipt_id": f"k8s-state:{phase}:{uuid.uuid4()}",
"source": STATE_RECEIPT_SOURCE,
"verified_by": STATE_RECEIPT_VERIFIER,
"trusted_server_readback": True,
"phase": phase,
"trace_id": execution_context.trace_id,
"run_id": execution_context.run_id,
"claim_id": execution_context.claim_id,
"execution_context_sha256": _context_digest(execution_context),
"observed_at_epoch_ms": observed_at,
"namespace": str(raw_state.get("namespace") or ""),
"kind": str(raw_state.get("kind") or "").lower(),
"name": str(raw_state.get("name") or "").lower(),
"uid": str(raw_state.get("uid") or ""),
"generation": raw_state.get("generation"),
"observed_generation": raw_state.get("observed_generation"),
"desired_replicas": raw_state.get("desired_replicas"),
"updated_replicas": raw_state.get("updated_replicas"),
"ready_replicas": raw_state.get("ready_replicas"),
"available_replicas": raw_state.get("available_replicas"),
"unavailable_replicas": raw_state.get("unavailable_replicas"),
}
def _state_receipt_blockers(
*,
state: Mapping[str, Any],
phase: str,
claim: Mapping[str, Any],
context: _KubernetesExecutionContext,
) -> list[str]:
blockers: list[str] = []
required = {
"schema_version": STATE_RECEIPT_SCHEMA_VERSION,
"source": STATE_RECEIPT_SOURCE,
"verified_by": STATE_RECEIPT_VERIFIER,
"trusted_server_readback": True,
"phase": phase,
"trace_id": context.trace_id,
"run_id": context.run_id,
"claim_id": context.claim_id,
"execution_context_sha256": _context_digest(context),
}
for field, expected in required.items():
if state.get(field) != expected:
blockers.append(f"{phase}_state_{field}_invalid")
if not str(state.get("receipt_id") or ""):
blockers.append(f"{phase}_state_receipt_id_missing")
expected_identity = (
str(claim.get("namespace") or ""),
str(claim.get("workload_kind") or ""),
str(claim.get("workload_name") or ""),
)
observed_identity = (
str(state.get("namespace") or ""),
str(state.get("kind") or ""),
str(state.get("name") or ""),
)
if observed_identity != expected_identity:
blockers.append(f"{phase}_state_identity_mismatch")
return blockers
def build_kubernetes_rollout_verifier_receipt(
*,
claim: Mapping[str, Any] | None,
pre_state: Mapping[str, Any] | None,
post_state: Mapping[str, Any] | None,
execution_context: object,
) -> dict[str, Any]:
"""Verify exact identity, unique same-run readbacks, and rollout convergence."""
blockers: list[str] = []
claim_mapping = claim if isinstance(claim, Mapping) else {}
pre = pre_state if isinstance(pre_state, Mapping) else {}
post = post_state if isinstance(post_state, Mapping) else {}
claim_valid, _claim_reason = validate_kubernetes_controlled_execution_claim(claim)
context = (
execution_context
if isinstance(execution_context, _KubernetesExecutionContext)
else None
)
if not claim_valid:
blockers.append("controlled_execution_claim_invalid")
if context is None:
blockers.append("server_issued_execution_context_invalid")
elif context.claim_id != str(claim_mapping.get("claim_id") or ""):
blockers.append("execution_context_claim_mismatch")
elif context.lifecycle_required:
if not context.prepared_writeback_ack:
blockers.append("prepared_writeback_missing")
if not context.applied_pending_writeback_ack:
blockers.append("applied_pending_writeback_missing")
if claim_valid and context is not None:
blockers.extend(
_state_receipt_blockers(
state=pre,
phase="pre",
claim=claim_mapping,
context=context,
)
)
blockers.extend(
_state_receipt_blockers(
state=post,
phase="post",
claim=claim_mapping,
context=context,
)
)
pre_receipt_id = str(pre.get("receipt_id") or "")
post_receipt_id = str(post.get("receipt_id") or "")
if pre_receipt_id and pre_receipt_id == post_receipt_id:
blockers.append("state_readback_receipt_replayed")
pre_observed_at = _integer(pre.get("observed_at_epoch_ms"))
post_observed_at = _integer(post.get("observed_at_epoch_ms"))
if (
pre_observed_at is None
or post_observed_at is None
or post_observed_at <= pre_observed_at
):
blockers.append("state_readback_order_invalid")
pre_uid = str(pre.get("uid") or "")
post_uid = str(post.get("uid") or "")
if not pre_uid or not post_uid or pre_uid != post_uid:
blockers.append("workload_uid_mismatch")
pre_generation = _integer(pre.get("generation"))
generation = _integer(post.get("generation"))
observed_generation = _integer(post.get("observed_generation"))
desired = _integer(post.get("desired_replicas"))
updated = _integer(post.get("updated_replicas"))
ready = _integer(post.get("ready_replicas"))
available = _integer(post.get("available_replicas"))
unavailable = _integer(post.get("unavailable_replicas"))
if pre_generation is None or generation is None or generation <= pre_generation:
blockers.append("rollout_generation_not_advanced")
if generation is None or observed_generation is None or observed_generation < generation:
blockers.append("rollout_generation_not_observed")
if desired is None or desired < 1:
blockers.append("desired_replicas_invalid")
elif any(value is None or value < desired for value in (updated, ready, available)):
blockers.append("rollout_replicas_not_converged")
if unavailable is None or unavailable != 0:
blockers.append("rollout_has_unavailable_replicas")
blockers = list(dict.fromkeys(blockers))
verified = not blockers
uid_digest = hashlib.sha256(post_uid.encode()).hexdigest() if post_uid else None
return {
"schema_version": VERIFIER_RECEIPT_SCHEMA_VERSION,
"receipt_id": f"k8s-rollout:{uuid.uuid4()}",
"status": "verified_success" if verified else "failed_closed",
"verified": verified,
"verifier": KUBERNETES_VERIFIER_ID,
"claim_id": str(claim_mapping.get("claim_id") or "") or None,
"incident_id": str(claim_mapping.get("incident_id") or "") or None,
"trace_id": context.trace_id if context else None,
"run_id": context.run_id if context else None,
"route_id": str(claim_mapping.get("route_id") or "") or None,
"canonical_asset_id": str(claim_mapping.get("canonical_asset_id") or "") or None,
"namespace": str(claim_mapping.get("namespace") or "") or None,
"workload_kind": str(claim_mapping.get("workload_kind") or "") or None,
"workload_name": str(claim_mapping.get("workload_name") or "") or None,
"workload_uid_sha256": uid_digest,
"pre_state_receipt_id": pre_receipt_id or None,
"post_state_receipt_id": post_receipt_id or None,
"pre_generation": pre_generation,
"generation": generation,
"observed_generation": observed_generation,
"desired_replicas": desired,
"updated_replicas": updated,
"ready_replicas": ready,
"available_replicas": available,
"unavailable_replicas": unavailable,
"blockers": blockers,
"durable_writeback_ack": False,
"stores_raw_uid": False,
"stores_raw_output": False,
}
class KubernetesRolloutReceiptWriter:
"""Persist the public-safe verifier receipt into incident evidence."""
async def write(
self,
*,
claim: Mapping[str, Any],
receipt: Mapping[str, Any],
) -> bool:
try:
from src.services.evidence_snapshot import EvidenceSnapshot
snapshot = EvidenceSnapshot(
incident_id=str(claim.get("incident_id") or ""),
project_id="awoooi",
)
evidence_key = (
"kubernetes_rollout_lifecycle"
if receipt.get("schema_version")
== LIFECYCLE_RECEIPT_SCHEMA_VERSION
else "kubernetes_rollout_verifier"
)
snapshot.post_execution_state = {evidence_key: dict(receipt)}
if receipt.get("status") in {
"prepared",
"applied_pending_verification",
"write_outcome_unknown_pending_verification",
"no_write_pending_verification",
}:
snapshot.verification_result = "pending"
else:
snapshot.verification_result = (
"success" if receipt.get("verified") is True else "failed"
)
snapshot.sensors_attempted = 1
snapshot.sensors_succeeded = 1
snapshot.mcp_health = {KUBERNETES_VERIFIER_ID: True}
snapshot.evidence_summary = (
"[KubernetesRolloutVerifier] exact same-run receipt; "
f"claim_id={receipt.get('claim_id')}; status={receipt.get('status')}"
)
await snapshot.save()
return True
except Exception as exc:
logger.warning(
"kubernetes_rollout_receipt_write_failed",
incident_id=str(claim.get("incident_id") or ""),
error_type=type(exc).__name__,
)
return False
class KubernetesRolloutVerifier:
"""Independent state reader + durable fail-closed verifier."""
def __init__(
self,
*,
state_executor: ActionExecutor | None = None,
receipt_writer: KubernetesRolloutReceiptWriter | None = None,
max_attempts: int = 12,
interval_seconds: float = 4.0,
deadline_seconds: float = 45.0,
read_timeout_seconds: float = 3.0,
write_timeout_seconds: float = 5.0,
) -> None:
self._state_executor = state_executor or ActionExecutor()
self._receipt_writer = receipt_writer or KubernetesRolloutReceiptWriter()
self._max_attempts = max(1, max_attempts)
self._interval_seconds = max(0.0, interval_seconds)
self._deadline_seconds = max(0.1, deadline_seconds)
self._read_timeout_seconds = max(0.1, read_timeout_seconds)
self._write_timeout_seconds = max(0.1, write_timeout_seconds)
@staticmethod
def _bounded_timeout(
configured_seconds: float,
deadline_monotonic: float | None,
) -> float:
if deadline_monotonic is None:
return configured_seconds
remaining = deadline_monotonic - time.monotonic()
if remaining <= 0:
raise TimeoutError("kubernetes_execution_deadline_exhausted")
return min(configured_seconds, remaining)
async def _read_state(
self,
claim: Mapping[str, Any],
*,
deadline_monotonic: float | None = None,
) -> dict[str, Any]:
return await asyncio.wait_for(
self._state_executor.read_workload_state(
str(claim.get("workload_kind") or ""),
str(claim.get("workload_name") or ""),
str(claim.get("namespace") or ""),
),
timeout=self._bounded_timeout(
self._read_timeout_seconds,
deadline_monotonic,
),
)
async def _persist_receipt(
self,
*,
claim: Mapping[str, Any],
receipt: Mapping[str, Any],
deadline_monotonic: float | None = None,
) -> bool:
try:
return await asyncio.wait_for(
self._receipt_writer.write(
claim=claim,
receipt=receipt,
),
timeout=self._bounded_timeout(
self._write_timeout_seconds,
deadline_monotonic,
),
)
except Exception as exc:
logger.warning(
"kubernetes_rollout_receipt_persist_failed",
claim_id=str(claim.get("claim_id") or ""),
status=str(receipt.get("status") or ""),
error_type=type(exc).__name__,
)
return False
@staticmethod
def _lifecycle_receipt(
*,
claim: Mapping[str, Any],
context: _KubernetesExecutionContext,
status: str,
runtime_write_outcome: str | None = None,
terminal: bool = False,
blockers: list[str] | None = None,
) -> dict[str, Any]:
return {
"schema_version": LIFECYCLE_RECEIPT_SCHEMA_VERSION,
"receipt_id": f"k8s-lifecycle:{uuid.uuid4()}",
"status": status,
"verified": False,
"verifier": KUBERNETES_VERIFIER_ID,
"claim_id": context.claim_id,
"incident_id": str(claim.get("incident_id") or "") or None,
"trace_id": context.trace_id,
"run_id": context.run_id,
"route_id": str(claim.get("route_id") or "") or None,
"canonical_asset_id": (
str(claim.get("canonical_asset_id") or "") or None
),
"namespace": str(claim.get("namespace") or "") or None,
"workload_kind": str(claim.get("workload_kind") or "") or None,
"workload_name": str(claim.get("workload_name") or "") or None,
"runtime_write_outcome": runtime_write_outcome,
"runtime_write_performed": (
runtime_write_outcome == "confirmed_applied"
),
"terminal": terminal,
"blockers": list(blockers or []),
"durable_writeback_ack": True,
"stores_raw_uid": False,
"stores_raw_output": False,
}
async def prepare(
self,
claim: Mapping[str, Any],
*,
deadline_monotonic: float | None = None,
) -> tuple[object, dict[str, Any]]:
context = issue_kubernetes_execution_context(claim)
assert isinstance(context, _KubernetesExecutionContext)
context.lifecycle_required = True
try:
raw_state = await self._read_state(
claim,
deadline_monotonic=deadline_monotonic,
)
pre_state = build_kubernetes_state_readback_receipt(
claim=claim,
execution_context=context,
phase="pre",
raw_state=raw_state,
)
prepared_receipt = self._lifecycle_receipt(
claim=claim,
context=context,
status="prepared",
)
context.prepared_writeback_ack = await self._persist_receipt(
claim=claim,
receipt=prepared_receipt,
deadline_monotonic=deadline_monotonic,
)
if not context.prepared_writeback_ack:
raise RuntimeError("kubernetes_prepared_writeback_missing")
return context, pre_state
except asyncio.CancelledError:
cancelled_receipt = self._lifecycle_receipt(
claim=claim,
context=context,
status="terminal_prepare_cancelled_no_write",
runtime_write_outcome="confirmed_no_write",
terminal=True,
blockers=["prepare_cancelled"],
)
await asyncio.shield(
self._persist_receipt(
claim=claim,
receipt=cancelled_receipt,
deadline_monotonic=deadline_monotonic,
)
)
raise
except Exception as exc:
failed_receipt = self._lifecycle_receipt(
claim=claim,
context=context,
status="terminal_prepare_failed_no_write",
runtime_write_outcome="confirmed_no_write",
terminal=True,
blockers=[f"prepare_failed:{type(exc).__name__}"],
)
await self._persist_receipt(
claim=claim,
receipt=failed_receipt,
deadline_monotonic=deadline_monotonic,
)
raise
async def record_execution_outcome(
self,
*,
claim: Mapping[str, Any],
execution_context: object,
runtime_write_outcome: str,
deadline_monotonic: float | None = None,
) -> dict[str, Any]:
if not isinstance(execution_context, _KubernetesExecutionContext):
raise ValueError("server_issued_execution_context_required")
if execution_context.claim_id != str(claim.get("claim_id") or ""):
raise ValueError("execution_context_claim_mismatch")
if runtime_write_outcome == "confirmed_applied":
status = "applied_pending_verification"
elif runtime_write_outcome == "unknown_after_dispatch":
status = "write_outcome_unknown_pending_verification"
else:
status = "no_write_pending_verification"
receipt = self._lifecycle_receipt(
claim=claim,
context=execution_context,
status=status,
runtime_write_outcome=runtime_write_outcome,
)
execution_context.applied_pending_writeback_ack = await self._persist_receipt(
claim=claim,
receipt=receipt,
deadline_monotonic=deadline_monotonic,
)
return {
**receipt,
"durable_writeback_ack": (
execution_context.applied_pending_writeback_ack
),
}
async def record_terminal_outcome(
self,
*,
claim: Mapping[str, Any],
execution_context: object,
status: str,
runtime_write_outcome: str,
blockers: list[str] | None = None,
deadline_monotonic: float | None = None,
) -> dict[str, Any]:
if not isinstance(execution_context, _KubernetesExecutionContext):
raise ValueError("server_issued_execution_context_required")
if execution_context.claim_id != str(claim.get("claim_id") or ""):
raise ValueError("execution_context_claim_mismatch")
receipt = self._lifecycle_receipt(
claim=claim,
context=execution_context,
status=status,
runtime_write_outcome=runtime_write_outcome,
terminal=True,
blockers=blockers,
)
persisted = await self._persist_receipt(
claim=claim,
receipt=receipt,
deadline_monotonic=deadline_monotonic,
)
return {**receipt, "durable_writeback_ack": persisted}
async def verify(
self,
*,
claim: Mapping[str, Any],
execution_context: object,
pre_state: Mapping[str, Any],
deadline_monotonic: float | None = None,
) -> dict[str, Any]:
receipt: dict[str, Any] | None = None
deadline = time.monotonic() + self._deadline_seconds
if deadline_monotonic is not None:
deadline = min(deadline, deadline_monotonic)
poll_deadline = deadline - min(self._write_timeout_seconds, 5.0)
for attempt in range(self._max_attempts):
if attempt and self._interval_seconds:
remaining = poll_deadline - time.monotonic()
if remaining <= 0:
break
await asyncio.sleep(min(self._interval_seconds, remaining))
if time.monotonic() >= poll_deadline:
break
try:
raw_state = await self._read_state(
claim,
deadline_monotonic=poll_deadline,
)
pre_observed_at = _integer(pre_state.get("observed_at_epoch_ms")) or 0
post_state = build_kubernetes_state_readback_receipt(
claim=claim,
execution_context=execution_context,
phase="post",
raw_state=raw_state,
observed_at_epoch_ms=max(
time.time_ns() // 1_000_000,
pre_observed_at + 1,
),
)
except Exception as exc:
logger.warning(
"kubernetes_rollout_post_readback_failed",
claim_id=str(claim.get("claim_id") or ""),
error_type=type(exc).__name__,
)
continue
receipt = build_kubernetes_rollout_verifier_receipt(
claim=claim,
pre_state=pre_state,
post_state=post_state,
execution_context=execution_context,
)
if receipt["verified"] is True:
break
if time.monotonic() >= poll_deadline:
break
if receipt is None:
receipt = {
"schema_version": VERIFIER_RECEIPT_SCHEMA_VERSION,
"receipt_id": f"k8s-rollout:{uuid.uuid4()}",
"status": "failed_closed",
"verified": False,
"verifier": KUBERNETES_VERIFIER_ID,
"claim_id": str(claim.get("claim_id") or "") or None,
"incident_id": str(claim.get("incident_id") or "") or None,
"blockers": ["post_state_readback_unavailable"],
"durable_writeback_ack": False,
"stores_raw_uid": False,
"stores_raw_output": False,
}
durable_receipt = {**receipt, "durable_writeback_ack": True}
persisted = await self._persist_receipt(
claim=claim,
receipt=durable_receipt,
deadline_monotonic=deadline,
)
if persisted:
return durable_receipt
blockers = list(receipt.get("blockers") or [])
blockers.append("durable_verifier_writeback_missing")
return {
**receipt,
"status": "failed_closed",
"verified": False,
"blockers": list(dict.fromkeys(blockers)),
"durable_writeback_ack": False,
}
_verifier: KubernetesRolloutVerifier | None = None
def get_kubernetes_rollout_verifier() -> KubernetesRolloutVerifier:
global _verifier
if _verifier is None:
_verifier = KubernetesRolloutVerifier()
return _verifier