feat(security): guard VPA image rollout

scripts/security/runtime_image_mirror_controller.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 import argparse
 import hashlib
 import json
+import math
 import re
 import shlex
 import subprocess
@@ -41,6 +42,13 @@ class Workload:
     container_kind: str
 
 
+@dataclass(frozen=True)
+class AvailabilityGuard:
+    minimum_ready_replicas: int
+    maximum_effective_unavailable: int
+    minimum_effective_surge: int
+
+
 @dataclass(frozen=True)
 class ImagePolicy:
     asset_id: str
@@ -50,6 +58,7 @@ class ImagePolicy:
     push_ref: str
     runtime_ref: str
     workload: Workload
+    availability_guard: AvailabilityGuard | None
 
 
 @dataclass(frozen=True)
@@ -112,6 +121,19 @@ def _require_name(value: Any, label: str) -> str:
     return name
 
 
+def _require_nonnegative_int(value: Any, label: str) -> int:
+    if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+        raise ControllerError(f"invalid_{label}")
+    return value
+
+
+def _require_positive_int(value: Any, label: str) -> int:
+    number = _require_nonnegative_int(value, label)
+    if number == 0:
+        raise ControllerError(f"invalid_{label}")
+    return number
+
+
 def _policy_checksum(payload: dict[str, Any]) -> str:
     canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
     return "sha256:" + hashlib.sha256(canonical).hexdigest()
@@ -201,6 +223,27 @@ def load_policy(path: Path) -> Policy:
         if workload_key in workloads:
             raise ControllerError("duplicate_workload_target")
         workloads.add(workload_key)
+        availability_guard = None
+        guard_row = row.get("availability_guard")
+        if guard_row is not None:
+            if not isinstance(guard_row, dict):
+                raise ControllerError("availability_guard_invalid")
+            if workload.kind != "deployment":
+                raise ControllerError("availability_guard_requires_deployment")
+            availability_guard = AvailabilityGuard(
+                minimum_ready_replicas=_require_positive_int(
+                    guard_row.get("minimum_ready_replicas"),
+                    "minimum_ready_replicas",
+                ),
+                maximum_effective_unavailable=_require_nonnegative_int(
+                    guard_row.get("maximum_effective_unavailable"),
+                    "maximum_effective_unavailable",
+                ),
+                minimum_effective_surge=_require_positive_int(
+                    guard_row.get("minimum_effective_surge"),
+                    "minimum_effective_surge",
+                ),
+            )
         images.append(
             ImagePolicy(
                 asset_id=asset_id,
@@ -210,6 +253,7 @@ def load_policy(path: Path) -> Policy:
                 push_ref=push_ref,
                 runtime_ref=runtime_ref,
                 workload=workload,
+                availability_guard=availability_guard,
             )
         )
 
@@ -372,6 +416,7 @@ class RuntimeCache:
         self, policy: Policy | StagingPolicy, ssh_key: Path, known_hosts: Path
     ) -> None:
         self.policy = policy
+        self._verified_sources: dict[tuple[str, str], str] = {}
         self.ssh = [
             "ssh",
             "-i",
@@ -403,6 +448,10 @@ class RuntimeCache:
         return sorted(set(matches))[0]
 
     def verify_source(self, image: ImagePolicy | StagingImage) -> str:
+        cache_key = (image.source_index_digest, image.platform_digest)
+        cached = self._verified_sources.get(cache_key)
+        if cached is not None:
+            return cached
         source_ref = self.source_ref(image.source_index_digest)
         check = _run(self._command("images", "check", f"name=={source_ref}"))
         if "complete" not in (check.stdout or ""):
@@ -423,6 +472,7 @@ class RuntimeCache:
         ]
         if selected != [image.platform_digest]:
             raise ControllerError("runtime_cache_platform_digest_mismatch")
+        self._verified_sources[cache_key] = source_ref
         return source_ref
 
     def source_config_digest(self, platform_digest: str) -> str:
@@ -840,6 +890,92 @@ class Kubernetes:
             raise ControllerError("workload_container_not_found")
         return values[0]
 
+    @staticmethod
+    def _effective_replicas(
+        value: Any, replicas: int, *, round_up: bool, label: str
+    ) -> int:
+        if isinstance(value, bool):
+            raise ControllerError(f"{label}_invalid")
+        if isinstance(value, int) and value >= 0:
+            return value
+        if isinstance(value, str) and value.endswith("%"):
+            try:
+                percent = int(value[:-1])
+            except ValueError as exc:
+                raise ControllerError(f"{label}_invalid") from exc
+            if percent < 0 or percent > 100:
+                raise ControllerError(f"{label}_invalid")
+            scaled = replicas * percent / 100
+            return math.ceil(scaled) if round_up else math.floor(scaled)
+        raise ControllerError(f"{label}_invalid")
+
+    def verify_availability(self, item: ImagePolicy) -> None:
+        guard = item.availability_guard
+        if guard is None:
+            return
+        workload = self.workload(item)
+        metadata = workload.get("metadata", {})
+        spec = workload.get("spec", {})
+        status = workload.get("status", {})
+        generation = metadata.get("generation")
+        observed_generation = status.get("observedGeneration")
+        replicas = spec.get("replicas", 1)
+        if (
+            isinstance(replicas, bool)
+            or not isinstance(replicas, int)
+            or replicas < guard.minimum_ready_replicas
+        ):
+            raise ControllerError("availability_replicas_insufficient")
+        if (
+            isinstance(generation, bool)
+            or not isinstance(generation, int)
+            or isinstance(observed_generation, bool)
+            or not isinstance(observed_generation, int)
+            or observed_generation < generation
+        ):
+            raise ControllerError("availability_generation_not_observed")
+        ready = status.get("readyReplicas", 0)
+        available = status.get("availableReplicas", 0)
+        unavailable = status.get("unavailableReplicas", 0)
+        if (
+            isinstance(ready, bool)
+            or not isinstance(ready, int)
+            or isinstance(available, bool)
+            or not isinstance(available, int)
+            or ready < guard.minimum_ready_replicas
+            or available < guard.minimum_ready_replicas
+        ):
+            raise ControllerError("availability_ready_replicas_insufficient")
+        if (
+            isinstance(unavailable, bool)
+            or not isinstance(unavailable, int)
+            or unavailable > guard.maximum_effective_unavailable
+        ):
+            raise ControllerError("availability_unavailable_replicas_exceeded")
+
+        strategy = spec.get("strategy", {})
+        if not isinstance(strategy, dict) or strategy.get("type") != "RollingUpdate":
+            raise ControllerError("availability_rolling_update_required")
+        rolling = strategy.get("rollingUpdate", {})
+        if not isinstance(rolling, dict):
+            raise ControllerError("availability_rolling_update_invalid")
+        effective_unavailable = self._effective_replicas(
+            rolling.get("maxUnavailable", "25%"),
+            replicas,
+            round_up=False,
+            label="availability_max_unavailable",
+        )
+        effective_surge = self._effective_replicas(
+            rolling.get("maxSurge", "25%"),
+            replicas,
+            round_up=True,
+            label="availability_max_surge",
+        )
+        if effective_unavailable > guard.maximum_effective_unavailable:
+            raise ControllerError("availability_effective_unavailable_exceeded")
+        if effective_surge < guard.minimum_effective_surge:
+            raise ControllerError("availability_effective_surge_insufficient")
+
     def patch_image(self, item: ImagePolicy, image_ref: str, *, dry_run: bool) -> None:
         container_field = self._spec_container_field(item)
         patch = json.dumps(
@@ -994,6 +1130,7 @@ def apply_policy(
     changed: list[ImagePolicy] = []
     for item in policy.images:
         previous[item.asset_id] = kubernetes.current_image(item)
+        kubernetes.verify_availability(item)
         kubernetes.patch_image(item, item.runtime_ref, dry_run=True)
 
     if apply:
@@ -1005,6 +1142,7 @@ def apply_policy(
                 changed.append(item)
             for item in policy.images:
                 kubernetes.rollout(item)
+                kubernetes.verify_availability(item)
         except ControllerError:
             for item in reversed(changed):
                 try:
@@ -1038,6 +1176,12 @@ def apply_policy(
         "ai_candidate_action": "replace_forbidden_registry_with_internal_digest",
         "policy_decision": "controlled_apply_allowed",
         "server_dry_run_passed": True,
+        "availability_guard_count": sum(
+            1 for item in policy.images if item.availability_guard is not None
+        ),
+        "availability_guard_verified_count": sum(
+            1 for item in policy.images if item.availability_guard is not None
+        ),
         "execution_changed_count": len(changed),
         "independent_verifier_passed": verified,
         "verified_workload_count": compliant_count,