From 0ba0b30f68753dfd5d2da410b1940dfc19aebd0c Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 12:37:24 +0800 Subject: [PATCH] fix(cd): tolerate bounded workload rollout races --- .gitea/workflows/cd.yaml | 12 ++++++--- ..._signal_worker_ansible_executor_binding.py | 7 +++--- ...t_workload_database_identity_projection.py | 25 ++++++++++++++++++- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 11cb27ac2..3d682d36f 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -3949,22 +3949,26 @@ jobs: read_workload_db_identity() { workload="$1" retry_backoff_seconds="${AWOOOI_DB_IDENTITY_RETRY_BACKOFF_SECONDS:-5}" + retry_attempts="${AWOOOI_DB_IDENTITY_RETRY_ATTEMPTS:-6}" case "$retry_backoff_seconds" in ''|*[!0-9]*) retry_backoff_seconds=5 ;; esac + case "$retry_attempts" in + ''|*[!0-9]*|0) retry_attempts=6 ;; + esac attempt=1 - while [ "$attempt" -le 3 ]; do + while [ "$attempt" -le "$retry_attempts" ]; do if identity_output=$(read_workload_db_identity_once "$@"); then printf '%s\n' "$identity_output" return 0 fi - echo "workload_db_identity_exec_retry=$attempt/3;workload=$workload" >&2 - if [ "$attempt" -lt 3 ]; then + echo "workload_db_identity_exec_retry=$attempt/$retry_attempts;workload=$workload" >&2 + if [ "$attempt" -lt "$retry_attempts" ]; then sleep "$retry_backoff_seconds" fi attempt=$((attempt + 1)) done - echo "workload_db_identity_exec_exhausted=3;workload=$workload" >&2 + echo "workload_db_identity_exec_exhausted=$retry_attempts;workload=$workload" >&2 printf '%s\n' '{"role_fingerprint":"","connection_limit":null,"required_connection_budget":null,"active_role_connection_count":null,"available_connection_headroom":null,"representative_connection_probe_count":0,"representative_preflight_passed":false,"table_privilege_gap_count":null,"sequence_privilege_gap_count":null,"error_type":"WorkloadDbIdentityAttemptsExhausted"}' return 1 } diff --git a/apps/api/tests/test_signal_worker_ansible_executor_binding.py b/apps/api/tests/test_signal_worker_ansible_executor_binding.py index 086d00987..139e768d9 100644 --- a/apps/api/tests/test_signal_worker_ansible_executor_binding.py +++ b/apps/api/tests/test_signal_worker_ansible_executor_binding.py @@ -310,9 +310,10 @@ def test_cd_prunes_and_verifies_api_executor_boundary_in_production() -> None: assert "read_workload_db_identity awoooi-worker worker 1 5" in workflow assert "awoooi-ansible-executor-broker broker 1 2" in workflow assert "read_workload_db_identity_once" in workflow - assert 'while [ "$attempt" -le 3 ]' in workflow - assert "workload_db_identity_exec_retry=$attempt/3" in workflow - assert "workload_db_identity_exec_exhausted=3" in workflow + assert 'AWOOOI_DB_IDENTITY_RETRY_ATTEMPTS:-6' in workflow + assert 'while [ "$attempt" -le "$retry_attempts" ]' in workflow + assert "workload_db_identity_exec_retry=$attempt/$retry_attempts" in workflow + assert "workload_db_identity_exec_exhausted=$retry_attempts" in workflow assert "AWOOOI_DB_REPRESENTATIVE_PROBE_COUNT" in workflow assert "AWOOOI_DB_REQUIRED_CONNECTION_BUDGET" in workflow assert "api_http_concurrency_probe_passed" in workflow diff --git a/apps/api/tests/test_workload_database_identity_projection.py b/apps/api/tests/test_workload_database_identity_projection.py index 3e3d7e45c..63b197810 100644 --- a/apps/api/tests/test_workload_database_identity_projection.py +++ b/apps/api/tests/test_workload_database_identity_projection.py @@ -43,6 +43,7 @@ def _run_workload_db_identity_contract( tmp_path: Path, *, exec_failures: int, + retry_attempts: int = 3, ) -> tuple[subprocess.CompletedProcess[str], dict[str, int]]: state_dir = tmp_path / "state" state_dir.mkdir() @@ -103,6 +104,7 @@ def _run_workload_db_identity_contract( "set -e", 'KUBECTL="$FAKE_KUBECTL"', "AWOOOI_DB_IDENTITY_RETRY_BACKOFF_SECONDS=0", + f"AWOOOI_DB_IDENTITY_RETRY_ATTEMPTS={retry_attempts}", _read_workload_db_identity_shell(), "if result=$(read_workload_db_identity " "awoooi-worker worker 1 5); then", " status=0", @@ -271,7 +273,9 @@ def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None: assert "get secret '${secret_name}' -n awoooi-prod -o jsonpath" not in workflow assert "get secret awoooi-api-db -n awoooi-prod -o jsonpath" not in workflow assert "HEAD^" not in workflow - assert 'while [ "$attempt" -le 3 ]' in workflow + assert 'AWOOOI_DB_IDENTITY_RETRY_ATTEMPTS:-6' in workflow + assert 'while [ "$attempt" -le "$retry_attempts" ]' in workflow + assert 'workload_db_identity_exec_retry=$attempt/$retry_attempts' in workflow assert 'rollout status "deployment/$workload"' in workflow assert 'wait --for=condition=Ready "$ready_pod_ref"' in workflow assert 'exec -i "$ready_pod_ref"' in workflow @@ -358,6 +362,25 @@ def test_cd_workload_db_identity_blocks_after_three_worker_exec_failures( assert "workload_db_identity_exec_exhausted=3" in completed.stderr +def test_cd_workload_db_identity_survives_extended_rollout_race( + tmp_path: Path, +) -> None: + completed, counts = _run_workload_db_identity_contract( + tmp_path, + exec_failures=3, + retry_attempts=6, + ) + + lines = completed.stdout.strip().splitlines() + assert lines[0] == "status=0" + payload = json.loads(lines[-1]) + assert payload["role_fingerprint"] == "worker-fingerprint" + assert payload["representative_preflight_passed"] is True + assert counts == {"rollout": 4, "get": 8, "wait": 4, "exec": 4} + assert "workload_db_identity_exec_retry=3/6" in completed.stderr + assert "workload_db_identity_exec_exhausted" not in completed.stderr + + def test_cd_api_readiness_retries_http_503_then_runs_all_200_concurrency( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str],