feat(automation): harden D037 runtime closure receipts

apps/api/src/services/executor_trust_boundary_readback.py

@@ -8,14 +8,15 @@ import os
 import re
 import time
 from collections.abc import Mapping
+from datetime import UTC, datetime
 from typing import Any
 
 import structlog
 
 logger = structlog.get_logger(__name__)
 
-SCHEMA_VERSION = "awoooi_executor_trust_boundary_readback_v3"
-RECEIPT_SCHEMA_VERSION = "awoooi_executor_boundary_verification_v3"
+SCHEMA_VERSION = "awoooi_executor_trust_boundary_readback_v4"
+RECEIPT_SCHEMA_VERSION = "awoooi_executor_boundary_verification_v4"
 DEFAULT_NAMESPACE = "awoooi-prod"
 DEFAULT_CONFIGMAP_NAME = "awoooi-executor-boundary-verification"
 _CACHE_TTL_SECONDS = 20.0
@@ -59,6 +60,17 @@ _CANONICAL_LEARNING_REQUIRED_TRUE_FIELDS = (
     "learning_receipts_public_safe",
 )
 _DB_WORKLOAD_IDS = ("api", "worker", "broker")
+_DB_WORKLOAD_BUDGETS = {
+    "api": {"connection_limit": 12, "required_connection_budget": 8},
+    "worker": {"connection_limit": 8, "required_connection_budget": 5},
+    "broker": {"connection_limit": 4, "required_connection_budget": 2},
+}
+_DB_IDENTITY_VERIFIER = (
+    "runtime_role_fingerprint_role_limit_null_pool_http_concurrency_and_"
+    "representative_preflight_v4"
+)
+_DB_CAPACITY_RECEIPT_MAX_AGE_SECONDS = 15 * 60
+_DB_CAPACITY_RECEIPT_FUTURE_SKEW_SECONDS = 5 * 60
 _ROLE_FINGERPRINT_PATTERN = re.compile(r"^[0-9a-f]{64}$")
 _cache: tuple[float, dict[str, Any]] | None = None
 
@@ -79,11 +91,40 @@ def _role_fingerprint(value: str | None) -> str | None:
     return normalized if _ROLE_FINGERPRINT_PATTERN.fullmatch(normalized) else None
 
 
+def _parse_utc_timestamp(value: str | None) -> datetime | None:
+    normalized = str(value or "").strip()
+    if not normalized:
+        return None
+    if normalized.endswith("Z"):
+        normalized = f"{normalized[:-1]}+00:00"
+    try:
+        parsed = datetime.fromisoformat(normalized)
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        return None
+    return parsed.astimezone(UTC)
+
+
 def _build_database_identity_boundary(
     receipt: Mapping[str, str],
     *,
     receipt_ref: str,
+    now_utc: datetime,
 ) -> dict[str, Any]:
+    verified_at = receipt.get("verified_at")
+    verified_at_utc = _parse_utc_timestamp(verified_at)
+    receipt_age_seconds = (
+        int((now_utc - verified_at_utc).total_seconds())
+        if verified_at_utc is not None
+        else None
+    )
+    capacity_receipt_fresh = bool(
+        receipt_age_seconds is not None
+        and -_DB_CAPACITY_RECEIPT_FUTURE_SKEW_SECONDS
+        <= receipt_age_seconds
+        <= _DB_CAPACITY_RECEIPT_MAX_AGE_SECONDS
+    )
     workloads: dict[str, dict[str, Any]] = {}
     for workload_id in _DB_WORKLOAD_IDS:
         workloads[workload_id] = {
@@ -99,9 +140,15 @@ def _build_database_identity_boundary(
             "required_connection_budget": _int_or_none(
                 receipt.get(f"{workload_id}_required_connection_budget")
             ),
-            "concurrent_connection_probe_count": _int_or_none(
+            "active_role_connection_count": _int_or_none(
+                receipt.get(f"{workload_id}_active_role_connection_count")
+            ),
+            "available_connection_headroom": _int_or_none(
+                receipt.get(f"{workload_id}_available_connection_headroom")
+            ),
+            "representative_connection_probe_count": _int_or_none(
                 receipt.get(
-                    f"{workload_id}_concurrent_connection_probe_count"
+                    f"{workload_id}_representative_connection_probe_count"
                 )
             ),
             "database_null_pool": _truthy(
@@ -143,23 +190,36 @@ def _build_database_identity_boundary(
         and len(set(role_fingerprints)) == len(_DB_WORKLOAD_IDS)
         and _truthy(receipt.get("distinct_db_role_fingerprints"))
     )
-    monolithic_env_from_absent = bool(
-        workloads["api"]["monolithic_secret_env_from_absent"]
-        and workloads["worker"]["monolithic_secret_env_from_absent"]
+    monolithic_env_from_absent = all(
+        workloads[workload_id]["monolithic_secret_env_from_absent"] is True
+        for workload_id in _DB_WORKLOAD_IDS
     )
     null_pool_verified = all(
         workloads[workload_id]["database_null_pool"] is True
         for workload_id in _DB_WORKLOAD_IDS
     )
     workload_budget_verified = all(
-        isinstance(workloads[workload_id]["connection_limit"], int)
-        and isinstance(workloads[workload_id]["required_connection_budget"], int)
-        and workloads[workload_id]["connection_limit"] >= workloads[workload_id][
-            "required_connection_budget"
-        ]
-        and workloads[workload_id]["required_connection_budget"] > 0
+        workloads[workload_id]["connection_limit"]
+        == _DB_WORKLOAD_BUDGETS[workload_id]["connection_limit"]
+        and workloads[workload_id]["required_connection_budget"]
+        == _DB_WORKLOAD_BUDGETS[workload_id]["required_connection_budget"]
+        and isinstance(
+            workloads[workload_id]["active_role_connection_count"], int
+        )
+        and workloads[workload_id]["active_role_connection_count"] >= 0
+        and isinstance(
+            workloads[workload_id]["available_connection_headroom"], int
+        )
+        and workloads[workload_id]["available_connection_headroom"]
+        == (
+            workloads[workload_id]["connection_limit"]
+            - workloads[workload_id]["active_role_connection_count"]
+        )
+        and workloads[workload_id]["available_connection_headroom"]
+        >= workloads[workload_id]["required_connection_budget"]
         for workload_id in _DB_WORKLOAD_IDS
     )
+    verifier_verified = receipt.get("db_identity_verifier") == _DB_IDENTITY_VERIFIER
     api_http_concurrency_verified = _truthy(
         receipt.get("api_http_concurrency_probe_passed")
     )
@@ -170,12 +230,16 @@ def _build_database_identity_boundary(
                 workloads[workload_id]["required_connection_budget"], int
             )
             and isinstance(
-                workloads[workload_id]["concurrent_connection_probe_count"],
+                workloads[workload_id][
+                    "representative_connection_probe_count"
+                ],
                 int,
             )
             and workloads[workload_id]["required_connection_budget"] > 0
-            and workloads[workload_id]["concurrent_connection_probe_count"]
-            >= workloads[workload_id]["required_connection_budget"]
+            and workloads[workload_id][
+                "representative_connection_probe_count"
+            ]
+            >= 1
             and workloads[workload_id]["table_privilege_gap_count"] == 0
             and workloads[workload_id]["sequence_privilege_gap_count"] == 0
             for workload_id in _DB_WORKLOAD_IDS
@@ -194,6 +258,8 @@ def _build_database_identity_boundary(
         and null_pool_verified
         and workload_budget_verified
         and representative_preflights_verified
+        and verifier_verified
+        and capacity_receipt_fresh
         and _truthy(receipt.get("connection_budget_verified"))
     )
 
@@ -202,12 +268,22 @@ def _build_database_identity_boundary(
         blockers.append("api_monolithic_secret_env_from_present")
     if not workloads["worker"]["monolithic_secret_env_from_absent"]:
         blockers.append("worker_monolithic_secret_env_from_present")
+    if not workloads["broker"]["monolithic_secret_env_from_absent"]:
+        blockers.append("broker_monolithic_secret_env_from_present")
     if not distinct_secret_refs:
         blockers.append("database_secret_refs_not_distinct")
     if not distinct_role_fingerprints:
         blockers.append("database_role_fingerprints_not_distinct")
     if not api_http_concurrency_verified:
         blockers.append("api_http_concurrency_probe_not_verified")
+    if not verifier_verified:
+        blockers.append("database_identity_verifier_not_verified")
+    if verified_at_utc is None:
+        blockers.append("database_capacity_receipt_timestamp_invalid")
+    elif receipt_age_seconds < -_DB_CAPACITY_RECEIPT_FUTURE_SKEW_SECONDS:
+        blockers.append("database_capacity_receipt_timestamp_in_future")
+    elif receipt_age_seconds > _DB_CAPACITY_RECEIPT_MAX_AGE_SECONDS:
+        blockers.append("database_capacity_receipt_stale")
     for workload_id in _DB_WORKLOAD_IDS:
         workload = workloads[workload_id]
         if workload["role_fingerprint"] is None:
@@ -217,11 +293,10 @@ def _build_database_identity_boundary(
         if (
             workload["representative_preflight_passed"] is not True
             or not isinstance(
-                workload["concurrent_connection_probe_count"], int
+                workload["representative_connection_probe_count"], int
             )
             or not isinstance(workload["required_connection_budget"], int)
-            or workload["concurrent_connection_probe_count"]
-            < workload["required_connection_budget"]
+            or workload["representative_connection_probe_count"] < 1
             or workload["table_privilege_gap_count"] != 0
             or workload["sequence_privilege_gap_count"] != 0
         ):
@@ -230,11 +305,17 @@ def _build_database_identity_boundary(
             )
         connection_limit = workload["connection_limit"]
         required_budget = workload["required_connection_budget"]
+        active_role_connections = workload["active_role_connection_count"]
+        available_headroom = workload["available_connection_headroom"]
+        expected_budget = _DB_WORKLOAD_BUDGETS[workload_id]
         if (
-            not isinstance(connection_limit, int)
-            or not isinstance(required_budget, int)
-            or required_budget <= 0
-            or connection_limit < required_budget
+            connection_limit != expected_budget["connection_limit"]
+            or required_budget != expected_budget["required_connection_budget"]
+            or not isinstance(active_role_connections, int)
+            or active_role_connections < 0
+            or not isinstance(available_headroom, int)
+            or available_headroom != connection_limit - active_role_connections
+            or available_headroom < required_budget
         ):
             blockers.append(f"{workload_id}_connection_budget_not_verified")
     if not db_identity_isolated:
@@ -250,8 +331,16 @@ def _build_database_identity_boundary(
         workloads[workload_id]["required_connection_budget"]
         for workload_id in _DB_WORKLOAD_IDS
     ]
+    active_role_connection_counts = [
+        workloads[workload_id]["active_role_connection_count"]
+        for workload_id in _DB_WORKLOAD_IDS
+    ]
+    available_headrooms = [
+        workloads[workload_id]["available_connection_headroom"]
+        for workload_id in _DB_WORKLOAD_IDS
+    ]
     return {
-        "schema_version": "awoooi_workload_db_identity_budget_readback_v2",
+        "schema_version": "awoooi_workload_db_identity_budget_readback_v4",
         "status": "verified_ready" if not blockers else "in_progress",
         "receipt_ref": receipt_ref,
         "verifier": receipt.get("db_identity_verifier") or None,
@@ -265,10 +354,15 @@ def _build_database_identity_boundary(
             "worker_monolithic_secret_env_from_absent": workloads["worker"][
                 "monolithic_secret_env_from_absent"
             ],
+            "broker_monolithic_secret_env_from_absent": workloads["broker"][
+                "monolithic_secret_env_from_absent"
+            ],
             "distinct_database_secret_refs": distinct_secret_refs,
             "distinct_db_role_fingerprints": distinct_role_fingerprints,
             "database_null_pool_verified": null_pool_verified,
             "workload_connection_budgets_verified": workload_budget_verified,
+            "database_identity_verifier_verified": verifier_verified,
+            "database_capacity_receipt_fresh": capacity_receipt_fresh,
             "representative_db_preflights_verified": (
                 representative_preflights_verified
             ),
@@ -287,6 +381,28 @@ def _build_database_identity_boundary(
                 if all(isinstance(value, int) for value in required_budgets)
                 else None
             ),
+            "active_role_connection_total": (
+                sum(active_role_connection_counts)
+                if all(
+                    isinstance(value, int)
+                    for value in active_role_connection_counts
+                )
+                else None
+            ),
+            "available_headroom_total": (
+                sum(available_headrooms)
+                if all(isinstance(value, int) for value in available_headrooms)
+                else None
+            ),
+        },
+        "signal_freshness": {
+            "verified_at": verified_at or None,
+            "age_seconds": receipt_age_seconds,
+            "max_age_seconds": _DB_CAPACITY_RECEIPT_MAX_AGE_SECONDS,
+            "future_clock_skew_seconds": (
+                _DB_CAPACITY_RECEIPT_FUTURE_SKEW_SECONDS
+            ),
+            "fresh": capacity_receipt_fresh,
         },
         "active_blockers": blockers,
         "writes_on_read": False,
@@ -407,6 +523,7 @@ def build_executor_trust_boundary_readback(
     namespace: str = DEFAULT_NAMESPACE,
     configmap_name: str = DEFAULT_CONFIGMAP_NAME,
     error_type: str | None = None,
+    now_utc: datetime | None = None,
 ) -> dict[str, Any]:
     receipt = {str(key): str(value) for key, value in (data or {}).items()}
     receipt_ref = f"configmap:{namespace}/{configmap_name}"
@@ -433,6 +550,7 @@ def build_executor_trust_boundary_readback(
     database_identity_boundary = _build_database_identity_boundary(
         receipt,
         receipt_ref=receipt_ref,
+        now_utc=(now_utc or datetime.now(UTC)).astimezone(UTC),
     )
     autonomous_single_writer_source_boundary = (
         _build_autonomous_single_writer_source_boundary(

apps/api/src/services/reboot_auto_recovery_slo_scorecard.py

@@ -163,7 +163,7 @@ _WINDOWS99_EXPECTED_VMX_CANDIDATES = {
         r"D:\Documents\Virtual Machines\192.168.0.121_Ubuntu_64-bit\192.168.0.121_Ubuntu_64-bit.vmx",
     ],
     "112": [
-        r"D:\Downloads\kali-linux-2025.4-vmware-amd64\kali-linux-2025.4-vmware-amd64.vmwarevm\kali-linux-2025.4-vmware-amd64.vmx",
+        r"S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx",
     ],
 }
 _WINDOWS99_VMX_REPAIR_FORBIDDEN_ACTIONS = [