fix(cd): tolerate bounded workload rollout races

.gitea/workflows/cd.yaml

@@ -3949,22 +3949,26 @@ jobs:
           read_workload_db_identity() {
             workload="$1"
             retry_backoff_seconds="${AWOOOI_DB_IDENTITY_RETRY_BACKOFF_SECONDS:-5}"
+            retry_attempts="${AWOOOI_DB_IDENTITY_RETRY_ATTEMPTS:-6}"
             case "$retry_backoff_seconds" in
               ''|*[!0-9]*) retry_backoff_seconds=5 ;;
             esac
+            case "$retry_attempts" in
+              ''|*[!0-9]*|0) retry_attempts=6 ;;
+            esac
             attempt=1
-            while [ "$attempt" -le 3 ]; do
+            while [ "$attempt" -le "$retry_attempts" ]; do
               if identity_output=$(read_workload_db_identity_once "$@"); then
                 printf '%s\n' "$identity_output"
                 return 0
               fi
-              echo "workload_db_identity_exec_retry=$attempt/3;workload=$workload" >&2
-              if [ "$attempt" -lt 3 ]; then
+              echo "workload_db_identity_exec_retry=$attempt/$retry_attempts;workload=$workload" >&2
+              if [ "$attempt" -lt "$retry_attempts" ]; then
                 sleep "$retry_backoff_seconds"
               fi
               attempt=$((attempt + 1))
             done
-            echo "workload_db_identity_exec_exhausted=3;workload=$workload" >&2
+            echo "workload_db_identity_exec_exhausted=$retry_attempts;workload=$workload" >&2
             printf '%s\n' '{"role_fingerprint":"","connection_limit":null,"required_connection_budget":null,"active_role_connection_count":null,"available_connection_headroom":null,"representative_connection_probe_count":0,"representative_preflight_passed":false,"table_privilege_gap_count":null,"sequence_privilege_gap_count":null,"error_type":"WorkloadDbIdentityAttemptsExhausted"}'
             return 1
           }

apps/api/tests/test_signal_worker_ansible_executor_binding.py

@@ -310,9 +310,10 @@ def test_cd_prunes_and_verifies_api_executor_boundary_in_production() -> None:
     assert "read_workload_db_identity awoooi-worker worker 1 5" in workflow
     assert "awoooi-ansible-executor-broker broker 1 2" in workflow
     assert "read_workload_db_identity_once" in workflow
-    assert 'while [ "$attempt" -le 3 ]' in workflow
-    assert "workload_db_identity_exec_retry=$attempt/3" in workflow
-    assert "workload_db_identity_exec_exhausted=3" in workflow
+    assert 'AWOOOI_DB_IDENTITY_RETRY_ATTEMPTS:-6' in workflow
+    assert 'while [ "$attempt" -le "$retry_attempts" ]' in workflow
+    assert "workload_db_identity_exec_retry=$attempt/$retry_attempts" in workflow
+    assert "workload_db_identity_exec_exhausted=$retry_attempts" in workflow
     assert "AWOOOI_DB_REPRESENTATIVE_PROBE_COUNT" in workflow
     assert "AWOOOI_DB_REQUIRED_CONNECTION_BUDGET" in workflow
     assert "api_http_concurrency_probe_passed" in workflow

apps/api/tests/test_workload_database_identity_projection.py

@@ -43,6 +43,7 @@ def _run_workload_db_identity_contract(
     tmp_path: Path,
     *,
     exec_failures: int,
+    retry_attempts: int = 3,
 ) -> tuple[subprocess.CompletedProcess[str], dict[str, int]]:
     state_dir = tmp_path / "state"
     state_dir.mkdir()
@@ -103,6 +104,7 @@ def _run_workload_db_identity_contract(
             "set -e",
             'KUBECTL="$FAKE_KUBECTL"',
             "AWOOOI_DB_IDENTITY_RETRY_BACKOFF_SECONDS=0",
+            f"AWOOOI_DB_IDENTITY_RETRY_ATTEMPTS={retry_attempts}",
             _read_workload_db_identity_shell(),
             "if result=$(read_workload_db_identity " "awoooi-worker worker 1 5); then",
             "  status=0",
@@ -271,7 +273,9 @@ def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None:
     assert "get secret '${secret_name}' -n awoooi-prod -o jsonpath" not in workflow
     assert "get secret awoooi-api-db -n awoooi-prod -o jsonpath" not in workflow
     assert "HEAD^" not in workflow
-    assert 'while [ "$attempt" -le 3 ]' in workflow
+    assert 'AWOOOI_DB_IDENTITY_RETRY_ATTEMPTS:-6' in workflow
+    assert 'while [ "$attempt" -le "$retry_attempts" ]' in workflow
+    assert 'workload_db_identity_exec_retry=$attempt/$retry_attempts' in workflow
     assert 'rollout status "deployment/$workload"' in workflow
     assert 'wait --for=condition=Ready "$ready_pod_ref"' in workflow
     assert 'exec -i "$ready_pod_ref"' in workflow
@@ -358,6 +362,25 @@ def test_cd_workload_db_identity_blocks_after_three_worker_exec_failures(
     assert "workload_db_identity_exec_exhausted=3" in completed.stderr
 
 
+def test_cd_workload_db_identity_survives_extended_rollout_race(
+    tmp_path: Path,
+) -> None:
+    completed, counts = _run_workload_db_identity_contract(
+        tmp_path,
+        exec_failures=3,
+        retry_attempts=6,
+    )
+
+    lines = completed.stdout.strip().splitlines()
+    assert lines[0] == "status=0"
+    payload = json.loads(lines[-1])
+    assert payload["role_fingerprint"] == "worker-fingerprint"
+    assert payload["representative_preflight_passed"] is True
+    assert counts == {"rollout": 4, "get": 8, "wait": 4, "exec": 4}
+    assert "workload_db_identity_exec_retry=3/6" in completed.stderr
+    assert "workload_db_identity_exec_exhausted" not in completed.stderr
+
+
 def test_cd_api_readiness_retries_http_503_then_runs_all_200_concurrency(
     monkeypatch: pytest.MonkeyPatch,
     capsys: pytest.CaptureFixture[str],