diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 45ca4c838..0210f9dcd 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -3705,6 +3705,25 @@ jobs: CONNECTION_BUDGET_VERIFIED=true fi + api_restart_total() { + $KUBECTL get pods -n awoooi-prod -l app=awoooi-api -o json \ + | python3 -c ' + import json + import sys + + payload = json.load(sys.stdin) + print(sum( + int(status.get("restartCount") or 0) + for item in payload.get("items", []) + for status in (item.get("status", {}).get("containerStatuses") or []) + if status.get("name") == "api" + )) + ' + } + API_RESTART_COUNT_BEFORE=$(api_restart_total) + API_DESIRED_REPLICA_COUNT=$($KUBECTL get deployment awoooi-api \ + -n awoooi-prod -o jsonpath='{.spec.replicas}') + API_HTTP_CONCURRENCY_VERIFIED=$(python3 - <<'PY' import concurrent.futures import os @@ -3713,12 +3732,10 @@ jobs: import urllib.error import urllib.request - urls = ( - # Keep the verifier representative and bounded: the connection - # concurrency is the subject under test, not a repeated scan of - # the default 100-row coverage window. - "https://awoooi.wooo.work/api/v1/platform/events/dossier/coverage?project_id=awoooi&limit=1", - "https://awoooi.wooo.work/api/v1/platform/cicd/events?project_id=awoooi&limit=1", + readiness_url = "https://awoooi.wooo.work/api/v1/health/ready" + db_probe_url = ( + "https://awoooi.wooo.work/api/v1/platform/cicd/events" + "?project_id=awoooi&limit=1" ) transient_http_statuses = {408, 425, 429, 500, 502, 503, 504} @@ -3762,33 +3779,23 @@ jobs: readiness_result = "unavailable" for readiness_attempt in range(1, readiness_attempts + 1): - readiness_statuses = [] - readiness_errors = [] - for url in urls: - status, error_type = fetch_status(url, timeout=10) - readiness_statuses.append(status) - readiness_errors.append(error_type) - if ( - status is not None - and status != 200 - and status not in transient_http_statuses - ): - raise SystemExit( - "API sequential readiness permanent HTTP status: " - + str(status) - ) - readiness_result = ( - "statuses=" - + ",".join( - str(status) if status is not None else "unavailable" - for status in readiness_statuses - ) - + ";errors=" - + ",".join(readiness_errors) - ) - if readiness_statuses and all( - status == 200 for status in readiness_statuses + status, error_type = fetch_status(readiness_url, timeout=10) + if ( + status is not None + and status != 200 + and status not in transient_http_statuses ): + raise SystemExit( + "API sequential readiness permanent HTTP status: " + + str(status) + ) + readiness_result = ( + "status=" + + (str(status) if status is not None else "unavailable") + + ";error=" + + error_type + ) + if status == 200: print( "api_sequential_readiness=ready;attempt=" + str(readiness_attempt), @@ -3811,7 +3818,7 @@ jobs: last_result = "unavailable" - concurrency_urls = urls * 6 + concurrency_urls = (db_probe_url,) * 8 for attempt in range(1, concurrency_attempts + 1): try: with concurrent.futures.ThreadPoolExecutor( @@ -3850,6 +3857,25 @@ jobs: if [ "$API_HTTP_CONCURRENCY_VERIFIED" != "true" ]; then CONNECTION_BUDGET_VERIFIED=false fi + API_POST_PROBE_ROLLOUT_READY=false + if $KUBECTL rollout status deployment/awoooi-api -n awoooi-prod \ + --timeout=120s >/dev/null 2>&1; then + API_POST_PROBE_ROLLOUT_READY=true + fi + API_RESTART_COUNT_AFTER=$(api_restart_total) + API_READY_REPLICA_COUNT=$($KUBECTL get deployment awoooi-api \ + -n awoooi-prod -o jsonpath='{.status.readyReplicas}') + API_READY_REPLICA_COUNT=${API_READY_REPLICA_COUNT:-0} + API_PROBE_RESTART_FREE=false + if [ "$API_RESTART_COUNT_AFTER" -eq "$API_RESTART_COUNT_BEFORE" ]; then + API_PROBE_RESTART_FREE=true + fi + if [ "$API_POST_PROBE_ROLLOUT_READY" != "true" ] \ + || [ "$API_PROBE_RESTART_FREE" != "true" ] \ + || [ "$API_READY_REPLICA_COUNT" -ne "$API_DESIRED_REPLICA_COUNT" ]; then + echo "API concurrency probe destabilized rollout: ready=${API_READY_REPLICA_COUNT}/${API_DESIRED_REPLICA_COUNT} restarts=${API_RESTART_COUNT_BEFORE}->${API_RESTART_COUNT_AFTER}" >&2 + exit 1 + fi BOUNDARY_VERIFIED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ") $KUBECTL create configmap awoooi-executor-boundary-verification \ @@ -3928,6 +3954,11 @@ jobs: --from-literal=broker_sequence_privilege_gap_count="$BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" \ --from-literal=representative_db_preflights_verified="$REPRESENTATIVE_DB_PREFLIGHTS_VERIFIED" \ --from-literal=api_http_concurrency_probe_passed="$API_HTTP_CONCURRENCY_VERIFIED" \ + --from-literal=api_http_probe_restart_free="$API_PROBE_RESTART_FREE" \ + --from-literal=api_restart_count_before="$API_RESTART_COUNT_BEFORE" \ + --from-literal=api_restart_count_after="$API_RESTART_COUNT_AFTER" \ + --from-literal=api_ready_replica_count="$API_READY_REPLICA_COUNT" \ + --from-literal=api_desired_replica_count="$API_DESIRED_REPLICA_COUNT" \ --from-literal=api_database_null_pool="$API_DB_NULL_POOL" \ --from-literal=worker_database_null_pool="$WORKER_DB_NULL_POOL" \ --from-literal=broker_database_null_pool="$BROKER_DB_NULL_POOL" \ diff --git a/apps/api/tests/test_workload_database_identity_projection.py b/apps/api/tests/test_workload_database_identity_projection.py index be6dff27c..90bdf27eb 100644 --- a/apps/api/tests/test_workload_database_identity_projection.py +++ b/apps/api/tests/test_workload_database_identity_projection.py @@ -237,6 +237,9 @@ def test_workload_database_bootstrap_keeps_secret_values_off_output() -> None: assert "pid <> pg_backend_pid()" in script assert "for preflight_attempt in 1 2 3" in script assert "preflight_attempts_exhausted=3" in script + assert '"connection_headroom_below_required"' in script + assert 'error_code = "workload_database_preflight_failed"' in script + assert '"error_code": error_code' in script assert 'text("SELECT pg_sleep(0.25)")' in script assert "api) printf '12'" in script assert "worker) printf '8'" in script @@ -278,8 +281,17 @@ def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None: assert "API_PUBLIC_READINESS_ATTEMPTS" in workflow assert "API_HTTP_CONCURRENCY_ATTEMPTS" in workflow assert "API sequential readiness permanent HTTP status" in workflow - assert "events/dossier/coverage?project_id=awoooi&limit=1" in workflow - assert "platform/cicd/events?project_id=awoooi&limit=1" in workflow + assert "api/v1/health/ready" in workflow + assert "events/dossier/coverage?project_id=awoooi&limit=1" not in workflow + assert "api/v1/platform/cicd/events" in workflow + assert "?project_id=awoooi&limit=1" in workflow + assert "concurrency_urls = (db_probe_url,) * 8" in workflow + assert "concurrency_urls = urls * 6" not in workflow + assert "API_RESTART_COUNT_BEFORE=$(api_restart_total)" in workflow + assert "API_RESTART_COUNT_AFTER=$(api_restart_total)" in workflow + assert '"$API_PROBE_RESTART_FREE" != "true"' in workflow + assert "API concurrency probe destabilized rollout" in workflow + assert "api_http_probe_restart_free" in workflow assert workflow.index("API_HTTP_CONCURRENCY_VERIFIED=$(python3") < workflow.index( "$KUBECTL create configmap awoooi-executor-boundary-verification" ) @@ -329,7 +341,7 @@ def test_cd_api_readiness_retries_http_503_then_runs_all_200_concurrency( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: - responses = [503, 200, 200, 200, *([200] * 12)] + responses = [503, 200, *([200] * 8)] calls: list[str] = [] def fake_urlopen(url: str, *, timeout: int) -> _FakeHTTPResponse: @@ -358,7 +370,10 @@ def test_cd_api_readiness_retries_http_503_then_runs_all_200_concurrency( output = capsys.readouterr() assert output.out.strip() == "true" assert "api_sequential_readiness=ready;attempt=2" in output.err - assert len(calls) == 16 + assert len(calls) == 10 + assert calls[0].startswith("https://awoooi.wooo.work/api/v1/health/ready|") + assert calls[1].startswith("https://awoooi.wooo.work/api/v1/health/ready|") + assert all("platform/cicd/events" in call for call in calls[2:]) assert responses == [] @@ -392,7 +407,8 @@ def test_cd_api_readiness_exhaustion_stops_before_green_receipt_write( output = capsys.readouterr() assert output.out == "" assert namespace["receipt_write_reached"] is False - assert len(calls) == 6 + assert len(calls) == 3 + assert all("api/v1/health/ready" in call for call in calls) def test_cd_api_readiness_permanent_http_error_fails_without_retry( diff --git a/scripts/ops/awoooi-workload-db-identity-bootstrap.sh b/scripts/ops/awoooi-workload-db-identity-bootstrap.sh index 904131826..49f0ea9fa 100755 --- a/scripts/ops/awoooi-workload-db-identity-bootstrap.sh +++ b/scripts/ops/awoooi-workload-db-identity-bootstrap.sh @@ -410,10 +410,21 @@ spec: try: asyncio.run(main()) except Exception as exc: + safe_error_codes = { + "representative_probe_count_invalid", + "connection_limit_mismatch", + "connection_budget_below_required", + "connection_headroom_below_required", + "database_privilege_gap", + } + error_code = str(exc) + if error_code not in safe_error_codes: + error_code = "workload_database_preflight_failed" print(json.dumps({ "status": "failed", "workload": os.environ.get("WORKLOAD_ID", "unknown"), "error_type": type(exc).__name__, + "error_code": error_code, "secret_value_exposed": False, "secret_value_logged": False, }, separators=(",", ":")))