diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index d5e07e93d..3edee3b9f 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -3936,7 +3936,7 @@ jobs: and connection_limit >= required_connection_budget and ( available_connection_headroom - >= required_connection_budget + >= representative_probe_count ) ), "table_privilege_gap_count": table_gap_count, @@ -4220,9 +4220,9 @@ jobs: && [ "$API_DB_ACTIVE_ROLE_CONNECTION_COUNT" -ge 0 ] \ && [ "$WORKER_DB_ACTIVE_ROLE_CONNECTION_COUNT" -ge 0 ] \ && [ "$BROKER_DB_ACTIVE_ROLE_CONNECTION_COUNT" -ge 0 ] \ - && [ "$API_DB_AVAILABLE_CONNECTION_HEADROOM" -ge "$API_DB_REQUIRED_CONNECTION_BUDGET" ] \ - && [ "$WORKER_DB_AVAILABLE_CONNECTION_HEADROOM" -ge "$WORKER_DB_REQUIRED_CONNECTION_BUDGET" ] \ - && [ "$BROKER_DB_AVAILABLE_CONNECTION_HEADROOM" -ge "$BROKER_DB_REQUIRED_CONNECTION_BUDGET" ]; then + && [ "$API_DB_AVAILABLE_CONNECTION_HEADROOM" -ge "$API_DB_REPRESENTATIVE_PROBE_COUNT" ] \ + && [ "$WORKER_DB_AVAILABLE_CONNECTION_HEADROOM" -ge "$WORKER_DB_REPRESENTATIVE_PROBE_COUNT" ] \ + && [ "$BROKER_DB_AVAILABLE_CONNECTION_HEADROOM" -ge "$BROKER_DB_REPRESENTATIVE_PROBE_COUNT" ]; then CONNECTION_BUDGET_VERIFIED=true fi @@ -4253,8 +4253,8 @@ jobs: API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE=0 fi API_HTTP_CONCURRENCY_WORKERS=$API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE - if [ "$API_HTTP_CONCURRENCY_WORKERS" -gt 4 ]; then - API_HTTP_CONCURRENCY_WORKERS=4 + if [ "$API_HTTP_CONCURRENCY_WORKERS" -gt "$API_DB_NULL_POOL_CONCURRENCY_LIMIT" ]; then + API_HTTP_CONCURRENCY_WORKERS=$API_DB_NULL_POOL_CONCURRENCY_LIMIT fi API_HTTP_CONCURRENCY_VERIFIED=false API_HTTP_PROBE_SKIP_REASON="" @@ -4300,7 +4300,7 @@ jobs: "API_PUBLIC_READINESS_DELAY_SECONDS", 5, 0, 30 ) concurrency_attempts = bounded_int( - "API_HTTP_CONCURRENCY_ATTEMPTS", 1, 1, 1 + "API_HTTP_CONCURRENCY_ATTEMPTS", 3, 1, 3 ) concurrency_delay_seconds = bounded_int( "API_HTTP_CONCURRENCY_DELAY_SECONDS", 5, 0, 30 @@ -4354,7 +4354,7 @@ jobs: def fetch(url: str) -> int: - status, error_type = fetch_status(url, timeout=5) + status, error_type = fetch_status(url, timeout=10) if status is None: raise RuntimeError(error_type) return status diff --git a/apps/api/src/services/executor_trust_boundary_readback.py b/apps/api/src/services/executor_trust_boundary_readback.py index cd2a9c3a4..4a711471e 100644 --- a/apps/api/src/services/executor_trust_boundary_readback.py +++ b/apps/api/src/services/executor_trust_boundary_readback.py @@ -230,7 +230,7 @@ def _build_database_identity_boundary( and workloads[workload_id]["representative_connection_probe_count"] >= 1 and workloads[workload_id]["available_connection_headroom"] - >= workloads[workload_id]["required_connection_budget"] + >= workloads[workload_id]["representative_connection_probe_count"] for workload_id in _DB_WORKLOAD_IDS ) verifier_verified = receipt.get("db_identity_verifier") == _DB_IDENTITY_VERIFIER @@ -337,7 +337,7 @@ def _build_database_identity_boundary( ) or workload["representative_connection_probe_count"] < 1 or available_headroom - < required_budget + < workload["representative_connection_probe_count"] ): blockers.append(f"{workload_id}_connection_budget_not_verified") if not db_identity_isolated: diff --git a/apps/api/tests/test_executor_trust_boundary_readback.py b/apps/api/tests/test_executor_trust_boundary_readback.py index ab097a745..e5018767b 100644 --- a/apps/api/tests/test_executor_trust_boundary_readback.py +++ b/apps/api/tests/test_executor_trust_boundary_readback.py @@ -384,8 +384,8 @@ def test_executor_trust_boundary_requires_api_null_pool_concurrency_cap() -> Non def test_executor_trust_boundary_does_not_double_count_used_budget() -> None: receipt = _verified_receipt() - receipt["worker_active_role_connection_count"] = "3" - receipt["worker_available_connection_headroom"] = "5" + receipt["worker_active_role_connection_count"] = "7" + receipt["worker_available_connection_headroom"] = "1" readback = build_executor_trust_boundary_readback( receipt, @@ -400,6 +400,24 @@ def test_executor_trust_boundary_does_not_double_count_used_budget() -> None: ) +def test_executor_trust_boundary_requires_representative_probe_headroom() -> None: + receipt = _verified_receipt() + receipt["worker_active_role_connection_count"] = "8" + receipt["worker_available_connection_headroom"] = "0" + + readback = build_executor_trust_boundary_readback( + receipt, + deployed_source_sha="abc123", + ) + + database_boundary = readback["database_identity_boundary"] + assert readback["full_workload_boundary_verified"] is False + assert database_boundary["connection_budget_verified"] is False + assert "worker_connection_budget_not_verified" in ( + database_boundary["active_blockers"] + ) + + def test_executor_trust_boundary_binds_budget_and_verifier_contract() -> None: wrong_budget = _verified_receipt() wrong_budget["worker_required_connection_budget"] = "4" diff --git a/apps/api/tests/test_workload_database_identity_projection.py b/apps/api/tests/test_workload_database_identity_projection.py index b1f84c324..066fe1ea3 100644 --- a/apps/api/tests/test_workload_database_identity_projection.py +++ b/apps/api/tests/test_workload_database_identity_projection.py @@ -269,12 +269,12 @@ def test_workload_database_bootstrap_keeps_secret_values_off_output() -> None: assert "representative_connection_probe_count" in script assert "active_role_connection_count" in script assert "available_connection_headroom" in script - assert "connection_headroom_below_required" in script + assert "connection_headroom_below_probe_requirement" in script assert "FROM pg_stat_activity" in script assert "pid <> pg_backend_pid()" in script assert "for preflight_attempt in 1 2 3" in script assert "preflight_attempts_exhausted=3" in script - assert '"connection_headroom_below_required"' in script + assert '"connection_headroom_below_probe_requirement"' in script assert 'error_code = "workload_database_preflight_failed"' in script assert '"error_code": error_code' in script assert 'text("SELECT pg_sleep(0.25)")' in script @@ -320,30 +320,36 @@ def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None: assert "workload DB identity attempts exhausted before receipt write" in workflow assert "API_PUBLIC_READINESS_ATTEMPTS" in workflow assert "API_HTTP_CONCURRENCY_ATTEMPTS" in workflow + assert '"API_HTTP_CONCURRENCY_ATTEMPTS", 3, 1, 3' in workflow assert "API_HTTP_PROBE_SAFETY_RESERVE=4" in workflow assert '"$API_DB_NULL_POOL_CONCURRENCY_LIMIT" -eq 2' in workflow assert ( - '"$API_DB_AVAILABLE_CONNECTION_HEADROOM" -ge ' - '"$API_DB_REQUIRED_CONNECTION_BUDGET"' - ) in workflow - assert ( - '"$WORKER_DB_AVAILABLE_CONNECTION_HEADROOM" -ge ' - '"$WORKER_DB_REQUIRED_CONNECTION_BUDGET"' - ) in workflow - assert ( - '"$BROKER_DB_AVAILABLE_CONNECTION_HEADROOM" -ge ' - '"$BROKER_DB_REQUIRED_CONNECTION_BUDGET"' - ) in workflow - assert ( - "available_connection_headroom >= required_connection_budget" + "available_connection_headroom >= representative_probe_count" in normalized_workflow ) assert ( - '"$API_DB_AVAILABLE_CONNECTION_HEADROOM" -ge ' - '"$API_DB_REPRESENTATIVE_PROBE_COUNT"' - ) not in workflow + "available_connection_headroom >= required_connection_budget" + not in normalized_workflow + ) + for workload in ("API", "WORKER", "BROKER"): + assert ( + f'"${workload}_DB_AVAILABLE_CONNECTION_HEADROOM" -ge ' + f'"${workload}_DB_REPRESENTATIVE_PROBE_COUNT"' + in workflow + ) + assert ( + f'"${workload}_DB_AVAILABLE_CONNECTION_HEADROOM" -ge ' + f'"${workload}_DB_REQUIRED_CONNECTION_BUDGET"' + not in workflow + ) assert "api_database_null_pool_concurrency_limit" in workflow assert "API_HTTP_CONCURRENCY_WORKERS" in workflow + assert ( + '"$API_HTTP_CONCURRENCY_WORKERS" -gt ' + '"$API_DB_NULL_POOL_CONCURRENCY_LIMIT"' + in workflow + ) + assert "API_HTTP_CONCURRENCY_WORKERS=4" not in workflow assert "API sequential readiness permanent HTTP status" in workflow assert "api/v1/health/ready" in workflow assert "events/dossier/coverage?project_id=awoooi&limit=1" not in workflow diff --git a/scripts/ops/awoooi-workload-db-identity-bootstrap.sh b/scripts/ops/awoooi-workload-db-identity-bootstrap.sh index 49f0ea9fa..a495b3410 100755 --- a/scripts/ops/awoooi-workload-db-identity-bootstrap.sh +++ b/scripts/ops/awoooi-workload-db-identity-bootstrap.sh @@ -382,8 +382,10 @@ spec: raise RuntimeError("connection_limit_mismatch") if observed_limit < required_budget: raise RuntimeError("connection_budget_below_required") - if available_connection_headroom < required_budget: - raise RuntimeError("connection_headroom_below_required") + if available_connection_headroom < representative_probe_count: + raise RuntimeError( + "connection_headroom_below_probe_requirement" + ) if table_gap_count or sequence_gap_count: raise RuntimeError("database_privilege_gap") print(json.dumps({ @@ -414,7 +416,7 @@ spec: "representative_probe_count_invalid", "connection_limit_mismatch", "connection_budget_below_required", - "connection_headroom_below_required", + "connection_headroom_below_probe_requirement", "database_privilege_gap", } error_code = str(exc)