fix(cd): keep database verifier non-disruptive

.gitea/workflows/cd.yaml

@@ -3705,6 +3705,25 @@ jobs:
             CONNECTION_BUDGET_VERIFIED=true
           fi
 
+          api_restart_total() {
+            $KUBECTL get pods -n awoooi-prod -l app=awoooi-api -o json \
+              | python3 -c '
+          import json
+          import sys
+
+          payload = json.load(sys.stdin)
+          print(sum(
+              int(status.get("restartCount") or 0)
+              for item in payload.get("items", [])
+              for status in (item.get("status", {}).get("containerStatuses") or [])
+              if status.get("name") == "api"
+          ))
+          '
+          }
+          API_RESTART_COUNT_BEFORE=$(api_restart_total)
+          API_DESIRED_REPLICA_COUNT=$($KUBECTL get deployment awoooi-api \
+            -n awoooi-prod -o jsonpath='{.spec.replicas}')
+
           API_HTTP_CONCURRENCY_VERIFIED=$(python3 - <<'PY'
           import concurrent.futures
           import os
@@ -3713,12 +3732,10 @@ jobs:
           import urllib.error
           import urllib.request
 
-          urls = (
-              # Keep the verifier representative and bounded: the connection
-              # concurrency is the subject under test, not a repeated scan of
-              # the default 100-row coverage window.
-              "https://awoooi.wooo.work/api/v1/platform/events/dossier/coverage?project_id=awoooi&limit=1",
-              "https://awoooi.wooo.work/api/v1/platform/cicd/events?project_id=awoooi&limit=1",
+          readiness_url = "https://awoooi.wooo.work/api/v1/health/ready"
+          db_probe_url = (
+              "https://awoooi.wooo.work/api/v1/platform/cicd/events"
+              "?project_id=awoooi&limit=1"
           )
           transient_http_statuses = {408, 425, 429, 500, 502, 503, 504}
 
@@ -3762,33 +3779,23 @@ jobs:
 
           readiness_result = "unavailable"
           for readiness_attempt in range(1, readiness_attempts + 1):
-              readiness_statuses = []
-              readiness_errors = []
-              for url in urls:
-                  status, error_type = fetch_status(url, timeout=10)
-                  readiness_statuses.append(status)
-                  readiness_errors.append(error_type)
-                  if (
-                      status is not None
-                      and status != 200
-                      and status not in transient_http_statuses
-                  ):
-                      raise SystemExit(
-                          "API sequential readiness permanent HTTP status: "
-                          + str(status)
-                      )
-              readiness_result = (
-                  "statuses="
-                  + ",".join(
-                      str(status) if status is not None else "unavailable"
-                      for status in readiness_statuses
-                  )
-                  + ";errors="
-                  + ",".join(readiness_errors)
-              )
-              if readiness_statuses and all(
-                  status == 200 for status in readiness_statuses
+              status, error_type = fetch_status(readiness_url, timeout=10)
+              if (
+                  status is not None
+                  and status != 200
+                  and status not in transient_http_statuses
               ):
+                  raise SystemExit(
+                      "API sequential readiness permanent HTTP status: "
+                      + str(status)
+                  )
+              readiness_result = (
+                  "status="
+                  + (str(status) if status is not None else "unavailable")
+                  + ";error="
+                  + error_type
+              )
+              if status == 200:
                   print(
                       "api_sequential_readiness=ready;attempt="
                       + str(readiness_attempt),
@@ -3811,7 +3818,7 @@ jobs:
 
 
           last_result = "unavailable"
-          concurrency_urls = urls * 6
+          concurrency_urls = (db_probe_url,) * 8
           for attempt in range(1, concurrency_attempts + 1):
               try:
                   with concurrent.futures.ThreadPoolExecutor(
@@ -3850,6 +3857,25 @@ jobs:
           if [ "$API_HTTP_CONCURRENCY_VERIFIED" != "true" ]; then
             CONNECTION_BUDGET_VERIFIED=false
           fi
+          API_POST_PROBE_ROLLOUT_READY=false
+          if $KUBECTL rollout status deployment/awoooi-api -n awoooi-prod \
+              --timeout=120s >/dev/null 2>&1; then
+            API_POST_PROBE_ROLLOUT_READY=true
+          fi
+          API_RESTART_COUNT_AFTER=$(api_restart_total)
+          API_READY_REPLICA_COUNT=$($KUBECTL get deployment awoooi-api \
+            -n awoooi-prod -o jsonpath='{.status.readyReplicas}')
+          API_READY_REPLICA_COUNT=${API_READY_REPLICA_COUNT:-0}
+          API_PROBE_RESTART_FREE=false
+          if [ "$API_RESTART_COUNT_AFTER" -eq "$API_RESTART_COUNT_BEFORE" ]; then
+            API_PROBE_RESTART_FREE=true
+          fi
+          if [ "$API_POST_PROBE_ROLLOUT_READY" != "true" ] \
+            || [ "$API_PROBE_RESTART_FREE" != "true" ] \
+            || [ "$API_READY_REPLICA_COUNT" -ne "$API_DESIRED_REPLICA_COUNT" ]; then
+            echo "API concurrency probe destabilized rollout: ready=${API_READY_REPLICA_COUNT}/${API_DESIRED_REPLICA_COUNT} restarts=${API_RESTART_COUNT_BEFORE}->${API_RESTART_COUNT_AFTER}" >&2
+            exit 1
+          fi
 
           BOUNDARY_VERIFIED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
           $KUBECTL create configmap awoooi-executor-boundary-verification \
@@ -3928,6 +3954,11 @@ jobs:
             --from-literal=broker_sequence_privilege_gap_count="$BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" \
             --from-literal=representative_db_preflights_verified="$REPRESENTATIVE_DB_PREFLIGHTS_VERIFIED" \
             --from-literal=api_http_concurrency_probe_passed="$API_HTTP_CONCURRENCY_VERIFIED" \
+            --from-literal=api_http_probe_restart_free="$API_PROBE_RESTART_FREE" \
+            --from-literal=api_restart_count_before="$API_RESTART_COUNT_BEFORE" \
+            --from-literal=api_restart_count_after="$API_RESTART_COUNT_AFTER" \
+            --from-literal=api_ready_replica_count="$API_READY_REPLICA_COUNT" \
+            --from-literal=api_desired_replica_count="$API_DESIRED_REPLICA_COUNT" \
             --from-literal=api_database_null_pool="$API_DB_NULL_POOL" \
             --from-literal=worker_database_null_pool="$WORKER_DB_NULL_POOL" \
             --from-literal=broker_database_null_pool="$BROKER_DB_NULL_POOL" \

apps/api/tests/test_workload_database_identity_projection.py

@@ -237,6 +237,9 @@ def test_workload_database_bootstrap_keeps_secret_values_off_output() -> None:
     assert "pid <> pg_backend_pid()" in script
     assert "for preflight_attempt in 1 2 3" in script
     assert "preflight_attempts_exhausted=3" in script
+    assert '"connection_headroom_below_required"' in script
+    assert 'error_code = "workload_database_preflight_failed"' in script
+    assert '"error_code": error_code' in script
     assert 'text("SELECT pg_sleep(0.25)")' in script
     assert "api) printf '12'" in script
     assert "worker) printf '8'" in script
@@ -278,8 +281,17 @@ def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None:
     assert "API_PUBLIC_READINESS_ATTEMPTS" in workflow
     assert "API_HTTP_CONCURRENCY_ATTEMPTS" in workflow
     assert "API sequential readiness permanent HTTP status" in workflow
-    assert "events/dossier/coverage?project_id=awoooi&limit=1" in workflow
-    assert "platform/cicd/events?project_id=awoooi&limit=1" in workflow
+    assert "api/v1/health/ready" in workflow
+    assert "events/dossier/coverage?project_id=awoooi&limit=1" not in workflow
+    assert "api/v1/platform/cicd/events" in workflow
+    assert "?project_id=awoooi&limit=1" in workflow
+    assert "concurrency_urls = (db_probe_url,) * 8" in workflow
+    assert "concurrency_urls = urls * 6" not in workflow
+    assert "API_RESTART_COUNT_BEFORE=$(api_restart_total)" in workflow
+    assert "API_RESTART_COUNT_AFTER=$(api_restart_total)" in workflow
+    assert '"$API_PROBE_RESTART_FREE" != "true"' in workflow
+    assert "API concurrency probe destabilized rollout" in workflow
+    assert "api_http_probe_restart_free" in workflow
     assert workflow.index("API_HTTP_CONCURRENCY_VERIFIED=$(python3") < workflow.index(
         "$KUBECTL create configmap awoooi-executor-boundary-verification"
     )
@@ -329,7 +341,7 @@ def test_cd_api_readiness_retries_http_503_then_runs_all_200_concurrency(
     monkeypatch: pytest.MonkeyPatch,
     capsys: pytest.CaptureFixture[str],
 ) -> None:
-    responses = [503, 200, 200, 200, *([200] * 12)]
+    responses = [503, 200, *([200] * 8)]
     calls: list[str] = []
 
     def fake_urlopen(url: str, *, timeout: int) -> _FakeHTTPResponse:
@@ -358,7 +370,10 @@ def test_cd_api_readiness_retries_http_503_then_runs_all_200_concurrency(
     output = capsys.readouterr()
     assert output.out.strip() == "true"
     assert "api_sequential_readiness=ready;attempt=2" in output.err
-    assert len(calls) == 16
+    assert len(calls) == 10
+    assert calls[0].startswith("https://awoooi.wooo.work/api/v1/health/ready|")
+    assert calls[1].startswith("https://awoooi.wooo.work/api/v1/health/ready|")
+    assert all("platform/cicd/events" in call for call in calls[2:])
     assert responses == []
 
 
@@ -392,7 +407,8 @@ def test_cd_api_readiness_exhaustion_stops_before_green_receipt_write(
     output = capsys.readouterr()
     assert output.out == ""
     assert namespace["receipt_write_reached"] is False
-    assert len(calls) == 6
+    assert len(calls) == 3
+    assert all("api/v1/health/ready" in call for call in calls)
 
 
 def test_cd_api_readiness_permanent_http_error_fails_without_retry(

scripts/ops/awoooi-workload-db-identity-bootstrap.sh

@@ -410,10 +410,21 @@ spec:
           try:
               asyncio.run(main())
           except Exception as exc:
+              safe_error_codes = {
+                  "representative_probe_count_invalid",
+                  "connection_limit_mismatch",
+                  "connection_budget_below_required",
+                  "connection_headroom_below_required",
+                  "database_privilege_gap",
+              }
+              error_code = str(exc)
+              if error_code not in safe_error_codes:
+                  error_code = "workload_database_preflight_failed"
               print(json.dumps({
                   "status": "failed",
                   "workload": os.environ.get("WORKLOAD_ID", "unknown"),
                   "error_type": type(exc).__name__,
+                  "error_code": error_code,
                   "secret_value_exposed": False,
                   "secret_value_logged": False,
               }, separators=(",", ":")))