feat(security): guard metrics server rollout

scripts/security/runtime_image_mirror_controller.py

@@ -47,6 +47,8 @@ class AvailabilityGuard:
     minimum_ready_replicas: int
     maximum_effective_unavailable: int
     minimum_effective_surge: int
+    enforced_max_unavailable: int | None = None
+    enforced_max_surge: int | None = None
 
 
 @dataclass(frozen=True)
@@ -230,19 +232,46 @@ def load_policy(path: Path) -> Policy:
                 raise ControllerError("availability_guard_invalid")
             if workload.kind != "deployment":
                 raise ControllerError("availability_guard_requires_deployment")
+            minimum_ready_replicas = _require_positive_int(
+                guard_row.get("minimum_ready_replicas"),
+                "minimum_ready_replicas",
+            )
+            maximum_effective_unavailable = _require_nonnegative_int(
+                guard_row.get("maximum_effective_unavailable"),
+                "maximum_effective_unavailable",
+            )
+            minimum_effective_surge = _require_positive_int(
+                guard_row.get("minimum_effective_surge"),
+                "minimum_effective_surge",
+            )
+            enforced_max_unavailable = None
+            enforced_max_surge = None
+            strategy_row = guard_row.get("enforced_strategy")
+            if strategy_row is not None:
+                if not isinstance(strategy_row, dict):
+                    raise ControllerError("availability_enforced_strategy_invalid")
+                enforced_max_unavailable = _require_nonnegative_int(
+                    strategy_row.get("max_unavailable"),
+                    "enforced_max_unavailable",
+                )
+                enforced_max_surge = _require_positive_int(
+                    strategy_row.get("max_surge"),
+                    "enforced_max_surge",
+                )
+                if enforced_max_unavailable > maximum_effective_unavailable:
+                    raise ControllerError(
+                        "availability_enforced_unavailable_exceeds_guard"
+                    )
+                if enforced_max_surge < minimum_effective_surge:
+                    raise ControllerError(
+                        "availability_enforced_surge_below_guard"
+                    )
             availability_guard = AvailabilityGuard(
-                minimum_ready_replicas=_require_positive_int(
-                    guard_row.get("minimum_ready_replicas"),
-                    "minimum_ready_replicas",
-                ),
-                maximum_effective_unavailable=_require_nonnegative_int(
-                    guard_row.get("maximum_effective_unavailable"),
-                    "maximum_effective_unavailable",
-                ),
-                minimum_effective_surge=_require_positive_int(
-                    guard_row.get("minimum_effective_surge"),
-                    "minimum_effective_surge",
-                ),
+                minimum_ready_replicas=minimum_ready_replicas,
+                maximum_effective_unavailable=maximum_effective_unavailable,
+                minimum_effective_surge=minimum_effective_surge,
+                enforced_max_unavailable=enforced_max_unavailable,
+                enforced_max_surge=enforced_max_surge,
             )
         images.append(
             ImagePolicy(
@@ -890,6 +919,29 @@ class Kubernetes:
             raise ControllerError("workload_container_not_found")
         return values[0]
 
+    def current_strategy(self, item: ImagePolicy) -> dict[str, Any]:
+        strategy = self.workload(item).get("spec", {}).get("strategy")
+        if not isinstance(strategy, dict):
+            raise ControllerError("workload_strategy_missing")
+        return strategy
+
+    @staticmethod
+    def enforced_strategy(item: ImagePolicy) -> dict[str, Any] | None:
+        guard = item.availability_guard
+        if (
+            guard is None
+            or guard.enforced_max_unavailable is None
+            or guard.enforced_max_surge is None
+        ):
+            return None
+        return {
+            "type": "RollingUpdate",
+            "rollingUpdate": {
+                "maxUnavailable": guard.enforced_max_unavailable,
+                "maxSurge": guard.enforced_max_surge,
+            },
+        }
+
     @staticmethod
     def _effective_replicas(
         value: Any, replicas: int, *, round_up: bool, label: str
@@ -909,7 +961,12 @@ class Kubernetes:
             return math.ceil(scaled) if round_up else math.floor(scaled)
         raise ControllerError(f"{label}_invalid")
 
-    def verify_availability(self, item: ImagePolicy) -> None:
+    def verify_availability(
+        self,
+        item: ImagePolicy,
+        *,
+        strategy_override: dict[str, Any] | None = None,
+    ) -> None:
         guard = item.availability_guard
         if guard is None:
             return
@@ -953,7 +1010,9 @@ class Kubernetes:
         ):
             raise ControllerError("availability_unavailable_replicas_exceeded")
 
-        strategy = spec.get("strategy", {})
+        strategy = strategy_override if strategy_override is not None else spec.get(
+            "strategy", {}
+        )
         if not isinstance(strategy, dict) or strategy.get("type") != "RollingUpdate":
             raise ControllerError("availability_rolling_update_required")
         rolling = strategy.get("rollingUpdate", {})
@@ -976,20 +1035,28 @@ class Kubernetes:
         if effective_surge < guard.minimum_effective_surge:
             raise ControllerError("availability_effective_surge_insufficient")
 
-    def patch_image(self, item: ImagePolicy, image_ref: str, *, dry_run: bool) -> None:
+    def patch_image(
+        self,
+        item: ImagePolicy,
+        image_ref: str,
+        *,
+        dry_run: bool,
+        strategy: dict[str, Any] | None = None,
+    ) -> None:
         container_field = self._spec_container_field(item)
-        patch = json.dumps(
-            {
+        spec_patch: dict[str, Any] = {
+            "template": {
                 "spec": {
-                    "template": {
-                        "spec": {
-                            container_field: [
-                                {"name": item.workload.container, "image": image_ref}
-                            ]
-                        }
-                    }
+                    container_field: [
+                        {"name": item.workload.container, "image": image_ref}
+                    ]
                 }
-            },
+            }
+        }
+        if strategy is not None:
+            spec_patch["strategy"] = strategy
+        patch = json.dumps(
+            {"spec": spec_patch},
             separators=(",", ":"),
         )
         args = [
@@ -1127,39 +1194,102 @@ def apply_policy(
     mirror_receipt = _read_mirror_receipt(mirror_receipt_path, policy, trace_id)
     kubernetes = Kubernetes(use_sudo=use_sudo, kubeconfig=kubeconfig, server=server)
     previous: dict[str, str] = {}
+    previous_strategies: dict[str, dict[str, Any] | None] = {}
+    desired_strategies: dict[str, dict[str, Any] | None] = {}
+    strategy_diffs: dict[str, bool] = {}
+    requires_changes: dict[str, bool] = {}
     changed: list[ImagePolicy] = []
+    failure_reason_code: str | None = None
+    rollback_performed = False
+    rollback_verified = False
+    rollback_verified_workload_count = 0
     for item in policy.images:
         previous[item.asset_id] = kubernetes.current_image(item)
-        kubernetes.verify_availability(item)
-        kubernetes.patch_image(item, item.runtime_ref, dry_run=True)
+        desired_strategy = kubernetes.enforced_strategy(item)
+        desired_strategies[item.asset_id] = desired_strategy
+        previous_strategy = (
+            kubernetes.current_strategy(item) if desired_strategy is not None else None
+        )
+        previous_strategies[item.asset_id] = previous_strategy
+        strategy_diffs[item.asset_id] = (
+            desired_strategy is not None and previous_strategy != desired_strategy
+        )
+        requires_changes[item.asset_id] = (
+            previous[item.asset_id] != item.runtime_ref
+            or strategy_diffs[item.asset_id]
+        )
+        kubernetes.verify_availability(
+            item,
+            strategy_override=desired_strategy,
+        )
+        kubernetes.patch_image(
+            item,
+            item.runtime_ref,
+            dry_run=True,
+            strategy=desired_strategy,
+        )
 
     if apply:
         try:
             for item in policy.images:
-                if previous[item.asset_id] == item.runtime_ref:
+                if not requires_changes[item.asset_id]:
                     continue
-                kubernetes.patch_image(item, item.runtime_ref, dry_run=False)
+                kubernetes.patch_image(
+                    item,
+                    item.runtime_ref,
+                    dry_run=False,
+                    strategy=desired_strategies[item.asset_id],
+                )
                 changed.append(item)
             for item in policy.images:
                 kubernetes.rollout(item)
                 kubernetes.verify_availability(item)
-        except ControllerError:
+            for item in policy.images:
+                if kubernetes.current_image(item) != item.runtime_ref:
+                    raise ControllerError("workload_image_not_applied")
+                kubernetes.verify_pods(item)
+        except ControllerError as exc:
+            failure_reason_code = str(exc)
+            rollback_performed = bool(changed)
             for item in reversed(changed):
                 try:
-                    kubernetes.patch_image(item, previous[item.asset_id], dry_run=False)
+                    kubernetes.patch_image(
+                        item,
+                        previous[item.asset_id],
+                        dry_run=False,
+                        strategy=previous_strategies[item.asset_id],
+                    )
                     kubernetes.rollout(item)
+                    image_restored = (
+                        kubernetes.current_image(item) == previous[item.asset_id]
+                    )
+                    strategy_restored = (
+                        previous_strategies[item.asset_id] is None
+                        or kubernetes.current_strategy(item)
+                        == previous_strategies[item.asset_id]
+                    )
                 except ControllerError:
-                    pass
-            raise
+                    continue
+                if image_restored and strategy_restored:
+                    rollback_verified_workload_count += 1
+            rollback_verified = (
+                not changed or rollback_verified_workload_count == len(changed)
+            )
 
     pod_count = 0
     compliant_count = 0
     for item in policy.images:
-        if kubernetes.current_image(item) == item.runtime_ref:
-            compliant_count += 1
-            pod_count += kubernetes.verify_pods(item)
+        try:
+            if kubernetes.current_image(item) == item.runtime_ref:
+                compliant_count += 1
+                pod_count += kubernetes.verify_pods(item)
+        except ControllerError as exc:
+            if failure_reason_code is None:
+                failure_reason_code = str(exc)
 
-    verified = compliant_count == len(policy.images)
+    verified = (
+        failure_reason_code is None and compliant_count == len(policy.images)
+    )
     receipt = {
         "schema_version": RECEIPT_SCHEMA_VERSION,
         "trace_id": trace_id,
@@ -1171,7 +1301,7 @@ def apply_policy(
         "sensor_receipt_present": True,
         "normalized_asset_count": len(policy.images),
         "source_of_truth_diff_count": sum(
-            1 for item in policy.images if previous[item.asset_id] != item.runtime_ref
+            1 for item in policy.images if requires_changes[item.asset_id]
         ),
         "ai_candidate_action": "replace_forbidden_registry_with_internal_digest",
         "policy_decision": "controlled_apply_allowed",
@@ -1182,17 +1312,30 @@ def apply_policy(
         "availability_guard_verified_count": sum(
             1 for item in policy.images if item.availability_guard is not None
         ),
+        "availability_strategy_diff_count": sum(strategy_diffs.values()),
+        "availability_strategy_changed_count": sum(
+            1 for item in changed if strategy_diffs[item.asset_id]
+        ),
+        "availability_strategy_verified_count": (
+            sum(1 for value in desired_strategies.values() if value is not None)
+            if apply and verified
+            else 0
+        ),
         "execution_changed_count": len(changed),
         "independent_verifier_passed": verified,
         "verified_workload_count": compliant_count,
         "verified_pod_count": pod_count,
-        "rollback_performed": False,
+        "rollback_performed": rollback_performed,
+        "rollback_verified": rollback_verified,
+        "rollback_verified_workload_count": rollback_verified_workload_count,
+        "no_write_terminal": failure_reason_code is not None and not changed,
+        "failure_reason_code": failure_reason_code,
         "external_pull_allowed": False,
         "mirror_stage_verified": mirror_receipt.get("state") == "verified",
         "durable_writeback": "kubernetes_configmap",
         "state": "verified" if verified else "blocked_with_safe_next_action",
     }
-    if apply and verified:
+    if apply:
         kubernetes.write_receipt(receipt)
     return receipt