681 lines
25 KiB
Python
681 lines
25 KiB
Python
"""Independent, durable Kubernetes rollout verification."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import time
|
|
import uuid
|
|
from collections.abc import Mapping
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
import structlog
|
|
|
|
from src.services.executor import ActionExecutor
|
|
from src.services.kubernetes_controlled_executor import (
|
|
KUBERNETES_VERIFIER_ID,
|
|
validate_kubernetes_controlled_execution_claim,
|
|
)
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
|
|
VERIFIER_RECEIPT_SCHEMA_VERSION = "kubernetes_rollout_verifier_receipt_v1"
|
|
LIFECYCLE_RECEIPT_SCHEMA_VERSION = "kubernetes_rollout_lifecycle_receipt_v1"
|
|
STATE_RECEIPT_SCHEMA_VERSION = "kubernetes_workload_state_readback_v1"
|
|
STATE_RECEIPT_SOURCE = "kubernetes_api_independent_readback"
|
|
STATE_RECEIPT_VERIFIER = "kubernetes_rollout_state_reader"
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _KubernetesExecutionContext:
|
|
"""Opaque server-issued correlation that model JSON cannot fabricate."""
|
|
|
|
trace_id: str
|
|
run_id: str
|
|
claim_id: str
|
|
nonce: str
|
|
lifecycle_required: bool = False
|
|
prepared_writeback_ack: bool = False
|
|
applied_pending_writeback_ack: bool = False
|
|
|
|
|
|
def issue_kubernetes_execution_context(
|
|
claim: Mapping[str, Any] | None,
|
|
) -> object:
|
|
valid, reason = validate_kubernetes_controlled_execution_claim(claim)
|
|
if not valid:
|
|
raise ValueError(f"kubernetes_execution_claim_invalid:{reason}")
|
|
assert claim is not None
|
|
return _KubernetesExecutionContext(
|
|
trace_id=f"trace:k8s:{uuid.uuid4()}",
|
|
run_id=f"run:k8s:{uuid.uuid4()}",
|
|
claim_id=str(claim.get("claim_id") or ""),
|
|
nonce=uuid.uuid4().hex,
|
|
)
|
|
|
|
|
|
def _context_digest(context: _KubernetesExecutionContext) -> str:
|
|
return hashlib.sha256(context.nonce.encode()).hexdigest()
|
|
|
|
|
|
def _integer(value: Any) -> int | None:
|
|
if isinstance(value, bool):
|
|
return None
|
|
try:
|
|
parsed = int(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return parsed if parsed >= 0 else None
|
|
|
|
|
|
def build_kubernetes_state_readback_receipt(
|
|
*,
|
|
claim: Mapping[str, Any],
|
|
execution_context: object,
|
|
phase: str,
|
|
raw_state: Mapping[str, Any],
|
|
observed_at_epoch_ms: int | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Wrap one direct Kubernetes API observation in trusted same-run metadata."""
|
|
|
|
if not isinstance(execution_context, _KubernetesExecutionContext):
|
|
raise ValueError("server_issued_execution_context_required")
|
|
if phase not in {"pre", "post"}:
|
|
raise ValueError("state_readback_phase_invalid")
|
|
valid, reason = validate_kubernetes_controlled_execution_claim(claim)
|
|
if not valid:
|
|
raise ValueError(f"controlled_execution_claim_invalid:{reason}")
|
|
if execution_context.claim_id != str(claim.get("claim_id") or ""):
|
|
raise ValueError("execution_context_claim_mismatch")
|
|
if raw_state.get("error"):
|
|
raise RuntimeError("kubernetes_state_readback_failed")
|
|
|
|
observed_at = observed_at_epoch_ms or time.time_ns() // 1_000_000
|
|
return {
|
|
"schema_version": STATE_RECEIPT_SCHEMA_VERSION,
|
|
"receipt_id": f"k8s-state:{phase}:{uuid.uuid4()}",
|
|
"source": STATE_RECEIPT_SOURCE,
|
|
"verified_by": STATE_RECEIPT_VERIFIER,
|
|
"trusted_server_readback": True,
|
|
"phase": phase,
|
|
"trace_id": execution_context.trace_id,
|
|
"run_id": execution_context.run_id,
|
|
"claim_id": execution_context.claim_id,
|
|
"execution_context_sha256": _context_digest(execution_context),
|
|
"observed_at_epoch_ms": observed_at,
|
|
"namespace": str(raw_state.get("namespace") or ""),
|
|
"kind": str(raw_state.get("kind") or "").lower(),
|
|
"name": str(raw_state.get("name") or "").lower(),
|
|
"uid": str(raw_state.get("uid") or ""),
|
|
"generation": raw_state.get("generation"),
|
|
"observed_generation": raw_state.get("observed_generation"),
|
|
"desired_replicas": raw_state.get("desired_replicas"),
|
|
"updated_replicas": raw_state.get("updated_replicas"),
|
|
"ready_replicas": raw_state.get("ready_replicas"),
|
|
"available_replicas": raw_state.get("available_replicas"),
|
|
"unavailable_replicas": raw_state.get("unavailable_replicas"),
|
|
}
|
|
|
|
|
|
def _state_receipt_blockers(
|
|
*,
|
|
state: Mapping[str, Any],
|
|
phase: str,
|
|
claim: Mapping[str, Any],
|
|
context: _KubernetesExecutionContext,
|
|
) -> list[str]:
|
|
blockers: list[str] = []
|
|
required = {
|
|
"schema_version": STATE_RECEIPT_SCHEMA_VERSION,
|
|
"source": STATE_RECEIPT_SOURCE,
|
|
"verified_by": STATE_RECEIPT_VERIFIER,
|
|
"trusted_server_readback": True,
|
|
"phase": phase,
|
|
"trace_id": context.trace_id,
|
|
"run_id": context.run_id,
|
|
"claim_id": context.claim_id,
|
|
"execution_context_sha256": _context_digest(context),
|
|
}
|
|
for field, expected in required.items():
|
|
if state.get(field) != expected:
|
|
blockers.append(f"{phase}_state_{field}_invalid")
|
|
if not str(state.get("receipt_id") or ""):
|
|
blockers.append(f"{phase}_state_receipt_id_missing")
|
|
expected_identity = (
|
|
str(claim.get("namespace") or ""),
|
|
str(claim.get("workload_kind") or ""),
|
|
str(claim.get("workload_name") or ""),
|
|
)
|
|
observed_identity = (
|
|
str(state.get("namespace") or ""),
|
|
str(state.get("kind") or ""),
|
|
str(state.get("name") or ""),
|
|
)
|
|
if observed_identity != expected_identity:
|
|
blockers.append(f"{phase}_state_identity_mismatch")
|
|
return blockers
|
|
|
|
|
|
def build_kubernetes_rollout_verifier_receipt(
|
|
*,
|
|
claim: Mapping[str, Any] | None,
|
|
pre_state: Mapping[str, Any] | None,
|
|
post_state: Mapping[str, Any] | None,
|
|
execution_context: object,
|
|
) -> dict[str, Any]:
|
|
"""Verify exact identity, unique same-run readbacks, and rollout convergence."""
|
|
|
|
blockers: list[str] = []
|
|
claim_mapping = claim if isinstance(claim, Mapping) else {}
|
|
pre = pre_state if isinstance(pre_state, Mapping) else {}
|
|
post = post_state if isinstance(post_state, Mapping) else {}
|
|
claim_valid, _claim_reason = validate_kubernetes_controlled_execution_claim(claim)
|
|
context = (
|
|
execution_context
|
|
if isinstance(execution_context, _KubernetesExecutionContext)
|
|
else None
|
|
)
|
|
if not claim_valid:
|
|
blockers.append("controlled_execution_claim_invalid")
|
|
if context is None:
|
|
blockers.append("server_issued_execution_context_invalid")
|
|
elif context.claim_id != str(claim_mapping.get("claim_id") or ""):
|
|
blockers.append("execution_context_claim_mismatch")
|
|
elif context.lifecycle_required:
|
|
if not context.prepared_writeback_ack:
|
|
blockers.append("prepared_writeback_missing")
|
|
if not context.applied_pending_writeback_ack:
|
|
blockers.append("applied_pending_writeback_missing")
|
|
|
|
if claim_valid and context is not None:
|
|
blockers.extend(
|
|
_state_receipt_blockers(
|
|
state=pre,
|
|
phase="pre",
|
|
claim=claim_mapping,
|
|
context=context,
|
|
)
|
|
)
|
|
blockers.extend(
|
|
_state_receipt_blockers(
|
|
state=post,
|
|
phase="post",
|
|
claim=claim_mapping,
|
|
context=context,
|
|
)
|
|
)
|
|
|
|
pre_receipt_id = str(pre.get("receipt_id") or "")
|
|
post_receipt_id = str(post.get("receipt_id") or "")
|
|
if pre_receipt_id and pre_receipt_id == post_receipt_id:
|
|
blockers.append("state_readback_receipt_replayed")
|
|
pre_observed_at = _integer(pre.get("observed_at_epoch_ms"))
|
|
post_observed_at = _integer(post.get("observed_at_epoch_ms"))
|
|
if (
|
|
pre_observed_at is None
|
|
or post_observed_at is None
|
|
or post_observed_at <= pre_observed_at
|
|
):
|
|
blockers.append("state_readback_order_invalid")
|
|
|
|
pre_uid = str(pre.get("uid") or "")
|
|
post_uid = str(post.get("uid") or "")
|
|
if not pre_uid or not post_uid or pre_uid != post_uid:
|
|
blockers.append("workload_uid_mismatch")
|
|
|
|
pre_generation = _integer(pre.get("generation"))
|
|
generation = _integer(post.get("generation"))
|
|
observed_generation = _integer(post.get("observed_generation"))
|
|
desired = _integer(post.get("desired_replicas"))
|
|
updated = _integer(post.get("updated_replicas"))
|
|
ready = _integer(post.get("ready_replicas"))
|
|
available = _integer(post.get("available_replicas"))
|
|
unavailable = _integer(post.get("unavailable_replicas"))
|
|
|
|
if pre_generation is None or generation is None or generation <= pre_generation:
|
|
blockers.append("rollout_generation_not_advanced")
|
|
if generation is None or observed_generation is None or observed_generation < generation:
|
|
blockers.append("rollout_generation_not_observed")
|
|
if desired is None or desired < 1:
|
|
blockers.append("desired_replicas_invalid")
|
|
elif any(value is None or value < desired for value in (updated, ready, available)):
|
|
blockers.append("rollout_replicas_not_converged")
|
|
if unavailable is None or unavailable != 0:
|
|
blockers.append("rollout_has_unavailable_replicas")
|
|
|
|
blockers = list(dict.fromkeys(blockers))
|
|
verified = not blockers
|
|
uid_digest = hashlib.sha256(post_uid.encode()).hexdigest() if post_uid else None
|
|
return {
|
|
"schema_version": VERIFIER_RECEIPT_SCHEMA_VERSION,
|
|
"receipt_id": f"k8s-rollout:{uuid.uuid4()}",
|
|
"status": "verified_success" if verified else "failed_closed",
|
|
"verified": verified,
|
|
"verifier": KUBERNETES_VERIFIER_ID,
|
|
"claim_id": str(claim_mapping.get("claim_id") or "") or None,
|
|
"incident_id": str(claim_mapping.get("incident_id") or "") or None,
|
|
"trace_id": context.trace_id if context else None,
|
|
"run_id": context.run_id if context else None,
|
|
"route_id": str(claim_mapping.get("route_id") or "") or None,
|
|
"canonical_asset_id": str(claim_mapping.get("canonical_asset_id") or "") or None,
|
|
"namespace": str(claim_mapping.get("namespace") or "") or None,
|
|
"workload_kind": str(claim_mapping.get("workload_kind") or "") or None,
|
|
"workload_name": str(claim_mapping.get("workload_name") or "") or None,
|
|
"workload_uid_sha256": uid_digest,
|
|
"pre_state_receipt_id": pre_receipt_id or None,
|
|
"post_state_receipt_id": post_receipt_id or None,
|
|
"pre_generation": pre_generation,
|
|
"generation": generation,
|
|
"observed_generation": observed_generation,
|
|
"desired_replicas": desired,
|
|
"updated_replicas": updated,
|
|
"ready_replicas": ready,
|
|
"available_replicas": available,
|
|
"unavailable_replicas": unavailable,
|
|
"blockers": blockers,
|
|
"durable_writeback_ack": False,
|
|
"stores_raw_uid": False,
|
|
"stores_raw_output": False,
|
|
}
|
|
|
|
|
|
class KubernetesRolloutReceiptWriter:
|
|
"""Persist the public-safe verifier receipt into incident evidence."""
|
|
|
|
async def write(
|
|
self,
|
|
*,
|
|
claim: Mapping[str, Any],
|
|
receipt: Mapping[str, Any],
|
|
) -> bool:
|
|
try:
|
|
from src.services.evidence_snapshot import EvidenceSnapshot
|
|
|
|
snapshot = EvidenceSnapshot(
|
|
incident_id=str(claim.get("incident_id") or ""),
|
|
project_id="awoooi",
|
|
)
|
|
evidence_key = (
|
|
"kubernetes_rollout_lifecycle"
|
|
if receipt.get("schema_version")
|
|
== LIFECYCLE_RECEIPT_SCHEMA_VERSION
|
|
else "kubernetes_rollout_verifier"
|
|
)
|
|
snapshot.post_execution_state = {evidence_key: dict(receipt)}
|
|
if receipt.get("status") in {
|
|
"prepared",
|
|
"applied_pending_verification",
|
|
"write_outcome_unknown_pending_verification",
|
|
"no_write_pending_verification",
|
|
}:
|
|
snapshot.verification_result = "pending"
|
|
else:
|
|
snapshot.verification_result = (
|
|
"success" if receipt.get("verified") is True else "failed"
|
|
)
|
|
snapshot.sensors_attempted = 1
|
|
snapshot.sensors_succeeded = 1
|
|
snapshot.mcp_health = {KUBERNETES_VERIFIER_ID: True}
|
|
snapshot.evidence_summary = (
|
|
"[KubernetesRolloutVerifier] exact same-run receipt; "
|
|
f"claim_id={receipt.get('claim_id')}; status={receipt.get('status')}"
|
|
)
|
|
await snapshot.save()
|
|
return True
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"kubernetes_rollout_receipt_write_failed",
|
|
incident_id=str(claim.get("incident_id") or ""),
|
|
error_type=type(exc).__name__,
|
|
)
|
|
return False
|
|
|
|
|
|
class KubernetesRolloutVerifier:
|
|
"""Independent state reader + durable fail-closed verifier."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
state_executor: ActionExecutor | None = None,
|
|
receipt_writer: KubernetesRolloutReceiptWriter | None = None,
|
|
max_attempts: int = 12,
|
|
interval_seconds: float = 4.0,
|
|
deadline_seconds: float = 45.0,
|
|
read_timeout_seconds: float = 3.0,
|
|
write_timeout_seconds: float = 5.0,
|
|
) -> None:
|
|
self._state_executor = state_executor or ActionExecutor()
|
|
self._receipt_writer = receipt_writer or KubernetesRolloutReceiptWriter()
|
|
self._max_attempts = max(1, max_attempts)
|
|
self._interval_seconds = max(0.0, interval_seconds)
|
|
self._deadline_seconds = max(0.1, deadline_seconds)
|
|
self._read_timeout_seconds = max(0.1, read_timeout_seconds)
|
|
self._write_timeout_seconds = max(0.1, write_timeout_seconds)
|
|
|
|
@staticmethod
|
|
def _bounded_timeout(
|
|
configured_seconds: float,
|
|
deadline_monotonic: float | None,
|
|
) -> float:
|
|
if deadline_monotonic is None:
|
|
return configured_seconds
|
|
remaining = deadline_monotonic - time.monotonic()
|
|
if remaining <= 0:
|
|
raise TimeoutError("kubernetes_execution_deadline_exhausted")
|
|
return min(configured_seconds, remaining)
|
|
|
|
async def _read_state(
|
|
self,
|
|
claim: Mapping[str, Any],
|
|
*,
|
|
deadline_monotonic: float | None = None,
|
|
) -> dict[str, Any]:
|
|
return await asyncio.wait_for(
|
|
self._state_executor.read_workload_state(
|
|
str(claim.get("workload_kind") or ""),
|
|
str(claim.get("workload_name") or ""),
|
|
str(claim.get("namespace") or ""),
|
|
),
|
|
timeout=self._bounded_timeout(
|
|
self._read_timeout_seconds,
|
|
deadline_monotonic,
|
|
),
|
|
)
|
|
|
|
async def _persist_receipt(
|
|
self,
|
|
*,
|
|
claim: Mapping[str, Any],
|
|
receipt: Mapping[str, Any],
|
|
deadline_monotonic: float | None = None,
|
|
) -> bool:
|
|
try:
|
|
return await asyncio.wait_for(
|
|
self._receipt_writer.write(
|
|
claim=claim,
|
|
receipt=receipt,
|
|
),
|
|
timeout=self._bounded_timeout(
|
|
self._write_timeout_seconds,
|
|
deadline_monotonic,
|
|
),
|
|
)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"kubernetes_rollout_receipt_persist_failed",
|
|
claim_id=str(claim.get("claim_id") or ""),
|
|
status=str(receipt.get("status") or ""),
|
|
error_type=type(exc).__name__,
|
|
)
|
|
return False
|
|
|
|
@staticmethod
|
|
def _lifecycle_receipt(
|
|
*,
|
|
claim: Mapping[str, Any],
|
|
context: _KubernetesExecutionContext,
|
|
status: str,
|
|
runtime_write_outcome: str | None = None,
|
|
terminal: bool = False,
|
|
blockers: list[str] | None = None,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": LIFECYCLE_RECEIPT_SCHEMA_VERSION,
|
|
"receipt_id": f"k8s-lifecycle:{uuid.uuid4()}",
|
|
"status": status,
|
|
"verified": False,
|
|
"verifier": KUBERNETES_VERIFIER_ID,
|
|
"claim_id": context.claim_id,
|
|
"incident_id": str(claim.get("incident_id") or "") or None,
|
|
"trace_id": context.trace_id,
|
|
"run_id": context.run_id,
|
|
"route_id": str(claim.get("route_id") or "") or None,
|
|
"canonical_asset_id": (
|
|
str(claim.get("canonical_asset_id") or "") or None
|
|
),
|
|
"namespace": str(claim.get("namespace") or "") or None,
|
|
"workload_kind": str(claim.get("workload_kind") or "") or None,
|
|
"workload_name": str(claim.get("workload_name") or "") or None,
|
|
"runtime_write_outcome": runtime_write_outcome,
|
|
"runtime_write_performed": (
|
|
runtime_write_outcome == "confirmed_applied"
|
|
),
|
|
"terminal": terminal,
|
|
"blockers": list(blockers or []),
|
|
"durable_writeback_ack": True,
|
|
"stores_raw_uid": False,
|
|
"stores_raw_output": False,
|
|
}
|
|
|
|
async def prepare(
|
|
self,
|
|
claim: Mapping[str, Any],
|
|
*,
|
|
deadline_monotonic: float | None = None,
|
|
) -> tuple[object, dict[str, Any]]:
|
|
context = issue_kubernetes_execution_context(claim)
|
|
assert isinstance(context, _KubernetesExecutionContext)
|
|
context.lifecycle_required = True
|
|
try:
|
|
raw_state = await self._read_state(
|
|
claim,
|
|
deadline_monotonic=deadline_monotonic,
|
|
)
|
|
pre_state = build_kubernetes_state_readback_receipt(
|
|
claim=claim,
|
|
execution_context=context,
|
|
phase="pre",
|
|
raw_state=raw_state,
|
|
)
|
|
prepared_receipt = self._lifecycle_receipt(
|
|
claim=claim,
|
|
context=context,
|
|
status="prepared",
|
|
)
|
|
context.prepared_writeback_ack = await self._persist_receipt(
|
|
claim=claim,
|
|
receipt=prepared_receipt,
|
|
deadline_monotonic=deadline_monotonic,
|
|
)
|
|
if not context.prepared_writeback_ack:
|
|
raise RuntimeError("kubernetes_prepared_writeback_missing")
|
|
return context, pre_state
|
|
except asyncio.CancelledError:
|
|
cancelled_receipt = self._lifecycle_receipt(
|
|
claim=claim,
|
|
context=context,
|
|
status="terminal_prepare_cancelled_no_write",
|
|
runtime_write_outcome="confirmed_no_write",
|
|
terminal=True,
|
|
blockers=["prepare_cancelled"],
|
|
)
|
|
await asyncio.shield(
|
|
self._persist_receipt(
|
|
claim=claim,
|
|
receipt=cancelled_receipt,
|
|
deadline_monotonic=deadline_monotonic,
|
|
)
|
|
)
|
|
raise
|
|
except Exception as exc:
|
|
failed_receipt = self._lifecycle_receipt(
|
|
claim=claim,
|
|
context=context,
|
|
status="terminal_prepare_failed_no_write",
|
|
runtime_write_outcome="confirmed_no_write",
|
|
terminal=True,
|
|
blockers=[f"prepare_failed:{type(exc).__name__}"],
|
|
)
|
|
await self._persist_receipt(
|
|
claim=claim,
|
|
receipt=failed_receipt,
|
|
deadline_monotonic=deadline_monotonic,
|
|
)
|
|
raise
|
|
|
|
async def record_execution_outcome(
|
|
self,
|
|
*,
|
|
claim: Mapping[str, Any],
|
|
execution_context: object,
|
|
runtime_write_outcome: str,
|
|
deadline_monotonic: float | None = None,
|
|
) -> dict[str, Any]:
|
|
if not isinstance(execution_context, _KubernetesExecutionContext):
|
|
raise ValueError("server_issued_execution_context_required")
|
|
if execution_context.claim_id != str(claim.get("claim_id") or ""):
|
|
raise ValueError("execution_context_claim_mismatch")
|
|
if runtime_write_outcome == "confirmed_applied":
|
|
status = "applied_pending_verification"
|
|
elif runtime_write_outcome == "unknown_after_dispatch":
|
|
status = "write_outcome_unknown_pending_verification"
|
|
else:
|
|
status = "no_write_pending_verification"
|
|
receipt = self._lifecycle_receipt(
|
|
claim=claim,
|
|
context=execution_context,
|
|
status=status,
|
|
runtime_write_outcome=runtime_write_outcome,
|
|
)
|
|
execution_context.applied_pending_writeback_ack = await self._persist_receipt(
|
|
claim=claim,
|
|
receipt=receipt,
|
|
deadline_monotonic=deadline_monotonic,
|
|
)
|
|
return {
|
|
**receipt,
|
|
"durable_writeback_ack": (
|
|
execution_context.applied_pending_writeback_ack
|
|
),
|
|
}
|
|
|
|
async def record_terminal_outcome(
|
|
self,
|
|
*,
|
|
claim: Mapping[str, Any],
|
|
execution_context: object,
|
|
status: str,
|
|
runtime_write_outcome: str,
|
|
blockers: list[str] | None = None,
|
|
deadline_monotonic: float | None = None,
|
|
) -> dict[str, Any]:
|
|
if not isinstance(execution_context, _KubernetesExecutionContext):
|
|
raise ValueError("server_issued_execution_context_required")
|
|
if execution_context.claim_id != str(claim.get("claim_id") or ""):
|
|
raise ValueError("execution_context_claim_mismatch")
|
|
receipt = self._lifecycle_receipt(
|
|
claim=claim,
|
|
context=execution_context,
|
|
status=status,
|
|
runtime_write_outcome=runtime_write_outcome,
|
|
terminal=True,
|
|
blockers=blockers,
|
|
)
|
|
persisted = await self._persist_receipt(
|
|
claim=claim,
|
|
receipt=receipt,
|
|
deadline_monotonic=deadline_monotonic,
|
|
)
|
|
return {**receipt, "durable_writeback_ack": persisted}
|
|
|
|
async def verify(
|
|
self,
|
|
*,
|
|
claim: Mapping[str, Any],
|
|
execution_context: object,
|
|
pre_state: Mapping[str, Any],
|
|
deadline_monotonic: float | None = None,
|
|
) -> dict[str, Any]:
|
|
receipt: dict[str, Any] | None = None
|
|
deadline = time.monotonic() + self._deadline_seconds
|
|
if deadline_monotonic is not None:
|
|
deadline = min(deadline, deadline_monotonic)
|
|
poll_deadline = deadline - min(self._write_timeout_seconds, 5.0)
|
|
for attempt in range(self._max_attempts):
|
|
if attempt and self._interval_seconds:
|
|
remaining = poll_deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
break
|
|
await asyncio.sleep(min(self._interval_seconds, remaining))
|
|
if time.monotonic() >= poll_deadline:
|
|
break
|
|
try:
|
|
raw_state = await self._read_state(
|
|
claim,
|
|
deadline_monotonic=poll_deadline,
|
|
)
|
|
pre_observed_at = _integer(pre_state.get("observed_at_epoch_ms")) or 0
|
|
post_state = build_kubernetes_state_readback_receipt(
|
|
claim=claim,
|
|
execution_context=execution_context,
|
|
phase="post",
|
|
raw_state=raw_state,
|
|
observed_at_epoch_ms=max(
|
|
time.time_ns() // 1_000_000,
|
|
pre_observed_at + 1,
|
|
),
|
|
)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"kubernetes_rollout_post_readback_failed",
|
|
claim_id=str(claim.get("claim_id") or ""),
|
|
error_type=type(exc).__name__,
|
|
)
|
|
continue
|
|
receipt = build_kubernetes_rollout_verifier_receipt(
|
|
claim=claim,
|
|
pre_state=pre_state,
|
|
post_state=post_state,
|
|
execution_context=execution_context,
|
|
)
|
|
if receipt["verified"] is True:
|
|
break
|
|
if time.monotonic() >= poll_deadline:
|
|
break
|
|
|
|
if receipt is None:
|
|
receipt = {
|
|
"schema_version": VERIFIER_RECEIPT_SCHEMA_VERSION,
|
|
"receipt_id": f"k8s-rollout:{uuid.uuid4()}",
|
|
"status": "failed_closed",
|
|
"verified": False,
|
|
"verifier": KUBERNETES_VERIFIER_ID,
|
|
"claim_id": str(claim.get("claim_id") or "") or None,
|
|
"incident_id": str(claim.get("incident_id") or "") or None,
|
|
"blockers": ["post_state_readback_unavailable"],
|
|
"durable_writeback_ack": False,
|
|
"stores_raw_uid": False,
|
|
"stores_raw_output": False,
|
|
}
|
|
|
|
durable_receipt = {**receipt, "durable_writeback_ack": True}
|
|
persisted = await self._persist_receipt(
|
|
claim=claim,
|
|
receipt=durable_receipt,
|
|
deadline_monotonic=deadline,
|
|
)
|
|
if persisted:
|
|
return durable_receipt
|
|
|
|
blockers = list(receipt.get("blockers") or [])
|
|
blockers.append("durable_verifier_writeback_missing")
|
|
return {
|
|
**receipt,
|
|
"status": "failed_closed",
|
|
"verified": False,
|
|
"blockers": list(dict.fromkeys(blockers)),
|
|
"durable_writeback_ack": False,
|
|
}
|
|
|
|
|
|
_verifier: KubernetesRolloutVerifier | None = None
|
|
|
|
|
|
def get_kubernetes_rollout_verifier() -> KubernetesRolloutVerifier:
|
|
global _verifier
|
|
if _verifier is None:
|
|
_verifier = KubernetesRolloutVerifier()
|
|
return _verifier
|