fix(cd): bound post-rollout database verifier

.gitea/workflows/cd.yaml

@@ -3269,11 +3269,37 @@ jobs:
             container="$2"
             representative_probe_count="$3"
             required_connection_budget="$4"
-            $KUBECTL exec -i "deployment/$workload" -n awoooi-prod \
-              -c "$container" -- env \
-              AWOOOI_DB_REPRESENTATIVE_PROBE_COUNT="$representative_probe_count" \
-              AWOOOI_DB_REQUIRED_CONNECTION_BUDGET="$required_connection_budget" \
-              python - <<'PY'
+            last_failure="workload_not_ready"
+            ready_pod_ref=""
+            if ! $KUBECTL rollout status "deployment/$workload" \
+              -n awoooi-prod --timeout=20s >/dev/null 2>&1; then
+              last_failure="deployment_rollout_not_ready"
+            else
+              ready_pod_ref=$(
+                $KUBECTL get pods -n awoooi-prod -l "app=$workload" \
+                  --field-selector=status.phase=Running \
+                  --sort-by=.metadata.creationTimestamp \
+                  -o name 2>/dev/null | tail -n 1
+              ) || ready_pod_ref=""
+              if [ -z "$ready_pod_ref" ]; then
+                last_failure="running_pod_unavailable"
+              elif ! $KUBECTL wait --for=condition=Ready "$ready_pod_ref" \
+                -n awoooi-prod --timeout=10s >/dev/null 2>&1; then
+                last_failure="ready_pod_unavailable"
+              else
+                container_ready=$(
+                  $KUBECTL get "$ready_pod_ref" -n awoooi-prod \
+                    -o jsonpath="{.status.containerStatuses[?(@.name=='$container')].ready}" \
+                    2>/dev/null
+                ) || container_ready=""
+                if [ "$container_ready" != "true" ]; then
+                  last_failure="container_not_ready"
+                elif identity_output=$(
+                  $KUBECTL exec -i "$ready_pod_ref" -n awoooi-prod \
+                    -c "$container" -- env \
+                    AWOOOI_DB_REPRESENTATIVE_PROBE_COUNT="$representative_probe_count" \
+                    AWOOOI_DB_REQUIRED_CONNECTION_BUDGET="$required_connection_budget" \
+                    python - <<'PY'
           import asyncio
           import hashlib
           import json
@@ -3411,9 +3437,40 @@ jobs:
 
           asyncio.run(main())
           PY
+                ); then
+                  identity_json=$(printf '%s\n' "$identity_output" | tail -n 1)
+                  if printf '%s' "$identity_json" | python3 -c '
+          import json
+          import sys
+
+          payload = json.load(sys.stdin)
+          valid = (
+              bool(payload.get("role_fingerprint"))
+              and payload.get("representative_preflight_passed") is True
+              and int(
+                  payload.get("representative_connection_probe_count") or 0
+              ) >= 1
+          )
+          raise SystemExit(0 if valid else 1)
+          ' >/dev/null 2>&1; then
+                    printf '%s\n' "$identity_json"
+                    return 0
+                  fi
+                  last_failure="identity_preflight_not_ready"
+                else
+                  last_failure="kubectl_exec_failed"
+                fi
+              fi
+            fi
+            echo "workload_db_identity_once_failed workload=$workload container=$container reason=$last_failure" >&2
+            return 1
           }
           read_workload_db_identity() {
             workload="$1"
+            retry_backoff_seconds="${AWOOOI_DB_IDENTITY_RETRY_BACKOFF_SECONDS:-5}"
+            case "$retry_backoff_seconds" in
+              ''|*[!0-9]*) retry_backoff_seconds=5 ;;
+            esac
             attempt=1
             while [ "$attempt" -le 3 ]; do
               if identity_output=$(read_workload_db_identity_once "$@"); then
@@ -3422,11 +3479,12 @@ jobs:
               fi
               echo "workload_db_identity_exec_retry=$attempt/3;workload=$workload" >&2
               if [ "$attempt" -lt 3 ]; then
-                sleep $((attempt * 3))
+                sleep "$retry_backoff_seconds"
               fi
               attempt=$((attempt + 1))
             done
             echo "workload_db_identity_exec_exhausted=3;workload=$workload" >&2
+            printf '%s\n' '{"role_fingerprint":"","connection_limit":null,"required_connection_budget":null,"active_role_connection_count":null,"available_connection_headroom":null,"representative_connection_probe_count":0,"representative_preflight_passed":false,"table_privilege_gap_count":null,"sequence_privilege_gap_count":null,"error_type":"WorkloadDbIdentityAttemptsExhausted"}'
             return 1
           }
           json_field() {
@@ -3457,12 +3515,28 @@ jobs:
             fi
           }
 
-          API_DB_IDENTITY=$(read_workload_db_identity awoooi-api api 1 8 \
-            | tail -n 1 || true)
-          WORKER_DB_IDENTITY=$(read_workload_db_identity awoooi-worker worker 1 5 \
-            | tail -n 1 || true)
-          BROKER_DB_IDENTITY=$(read_workload_db_identity \
-            awoooi-ansible-executor-broker broker 1 2 | tail -n 1 || true)
+          if API_DB_IDENTITY=$(read_workload_db_identity awoooi-api api 1 8); then
+            API_DB_IDENTITY_EXIT=0
+          else
+            API_DB_IDENTITY_EXIT=$?
+          fi
+          if WORKER_DB_IDENTITY=$(read_workload_db_identity awoooi-worker worker 1 5); then
+            WORKER_DB_IDENTITY_EXIT=0
+          else
+            WORKER_DB_IDENTITY_EXIT=$?
+          fi
+          if BROKER_DB_IDENTITY=$(read_workload_db_identity \
+            awoooi-ansible-executor-broker broker 1 2); then
+            BROKER_DB_IDENTITY_EXIT=0
+          else
+            BROKER_DB_IDENTITY_EXIT=$?
+          fi
+          if [ "$API_DB_IDENTITY_EXIT" -ne 0 ] \
+            || [ "$WORKER_DB_IDENTITY_EXIT" -ne 0 ] \
+            || [ "$BROKER_DB_IDENTITY_EXIT" -ne 0 ]; then
+            echo "❌ workload DB identity attempts exhausted before receipt write"
+            exit 1
+          fi
           API_DB_ROLE_FINGERPRINT=$(printf '%s' "$API_DB_IDENTITY" \
             | json_field role_fingerprint 2>/dev/null || true)
           WORKER_DB_ROLE_FINGERPRINT=$(printf '%s' "$WORKER_DB_IDENTITY" \
@@ -3633,27 +3707,117 @@ jobs:
 
           API_HTTP_CONCURRENCY_VERIFIED=$(python3 - <<'PY'
           import concurrent.futures
+          import os
+          import sys
           import time
+          import urllib.error
           import urllib.request
 
           urls = (
-              "https://awoooi.wooo.work/api/v1/platform/events/dossier/coverage?project_id=awoooi",
-              "https://awoooi.wooo.work/api/v1/platform/cicd/events?project_id=awoooi",
-          ) * 6
+              # Keep the verifier representative and bounded: the connection
+              # concurrency is the subject under test, not a repeated scan of
+              # the default 100-row coverage window.
+              "https://awoooi.wooo.work/api/v1/platform/events/dossier/coverage?project_id=awoooi&limit=1",
+              "https://awoooi.wooo.work/api/v1/platform/cicd/events?project_id=awoooi&limit=1",
+          )
+          transient_http_statuses = {408, 425, 429, 500, 502, 503, 504}
+
+
+          def bounded_int(
+              name: str,
+              default: int,
+              minimum: int,
+              maximum: int,
+          ) -> int:
+              try:
+                  value = int(os.environ.get(name, str(default)))
+              except (TypeError, ValueError):
+                  value = default
+              return max(minimum, min(value, maximum))
+
+
+          readiness_attempts = bounded_int(
+              "API_PUBLIC_READINESS_ATTEMPTS", 12, 1, 20
+          )
+          readiness_delay_seconds = bounded_int(
+              "API_PUBLIC_READINESS_DELAY_SECONDS", 5, 0, 30
+          )
+          concurrency_attempts = bounded_int(
+              "API_HTTP_CONCURRENCY_ATTEMPTS", 4, 1, 8
+          )
+          concurrency_delay_seconds = bounded_int(
+              "API_HTTP_CONCURRENCY_DELAY_SECONDS", 5, 0, 30
+          )
+
+
+          def fetch_status(url: str, timeout: int) -> tuple[int | None, str]:
+              try:
+                  with urllib.request.urlopen(url, timeout=timeout) as response:
+                      return int(response.status), "response"
+              except urllib.error.HTTPError as exc:
+                  return int(exc.code), "HTTPError"
+              except Exception as exc:
+                  return None, type(exc).__name__
+
+
+          readiness_result = "unavailable"
+          for readiness_attempt in range(1, readiness_attempts + 1):
+              readiness_statuses = []
+              readiness_errors = []
+              for url in urls:
+                  status, error_type = fetch_status(url, timeout=10)
+                  readiness_statuses.append(status)
+                  readiness_errors.append(error_type)
+                  if (
+                      status is not None
+                      and status != 200
+                      and status not in transient_http_statuses
+                  ):
+                      raise SystemExit(
+                          "API sequential readiness permanent HTTP status: "
+                          + str(status)
+                      )
+              readiness_result = (
+                  "statuses="
+                  + ",".join(
+                      str(status) if status is not None else "unavailable"
+                      for status in readiness_statuses
+                  )
+                  + ";errors="
+                  + ",".join(readiness_errors)
+              )
+              if readiness_statuses and all(
+                  status == 200 for status in readiness_statuses
+              ):
+                  print(
+                      "api_sequential_readiness=ready;attempt="
+                      + str(readiness_attempt),
+                      file=sys.stderr,
+                  )
+                  break
+              if readiness_attempt < readiness_attempts:
+                  time.sleep(readiness_delay_seconds)
+          else:
+              raise SystemExit(
+                  "API sequential readiness failed: " + readiness_result
+              )
 
 
           def fetch(url: str) -> int:
-              with urllib.request.urlopen(url, timeout=20) as response:
-                  return int(response.status)
+              status, error_type = fetch_status(url, timeout=20)
+              if status is None:
+                  raise RuntimeError(error_type)
+              return status
 
 
           last_result = "unavailable"
-          for attempt in range(1, 4):
+          concurrency_urls = urls * 6
+          for attempt in range(1, concurrency_attempts + 1):
               try:
                   with concurrent.futures.ThreadPoolExecutor(
                       max_workers=8
                   ) as executor:
-                      statuses = list(executor.map(fetch, urls))
+                      statuses = list(executor.map(fetch, concurrency_urls))
               except Exception as exc:
                   last_result = type(exc).__name__
               else:
@@ -3661,8 +3825,22 @@ jobs:
                   if all(status == 200 for status in statuses):
                       print("true")
                       break
-              if attempt < 3:
-                  time.sleep(5)
+                  permanent_statuses = sorted({
+                      status
+                      for status in statuses
+                      if status != 200
+                      and status not in transient_http_statuses
+                  })
+                  if permanent_statuses:
+                      raise SystemExit(
+                          "API database concurrency probe permanent HTTP "
+                          "status: "
+                          + ",".join(
+                              str(status) for status in permanent_statuses
+                          )
+                      )
+              if attempt < concurrency_attempts:
+                  time.sleep(concurrency_delay_seconds)
           else:
               raise SystemExit(
                   "API database concurrency probe failed: " + last_result