wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(db): enforce global nullpool session budget

Browse Source
This commit is contained in:
ogt
2026-07-15 15:49:18 +08:00
parent 19d7db93e5
commit f57845ba5e
12 changed files with 709 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
apps/api/tests/test_executor_trust_boundary_readback.py
View File

@@ -92,6 +92,7 @@ def _verified_receipt(source_sha: str = "abc123") -> dict[str, str]:
"representative_db_preflights_verified": "true",
"api_http_concurrency_probe_passed": "true",
"api_database_null_pool": "true",
"api_database_null_pool_concurrency_limit": "2",
"worker_database_null_pool": "true",
"broker_database_null_pool": "true",
"api_monolithic_secret_env_from_absent": "true",
@@ -128,6 +129,9 @@ def test_executor_trust_boundary_requires_live_controls_and_matching_source() ->
assert database_boundary["controls"][
"representative_db_preflights_verified"
] is True
assert database_boundary["controls"][
"api_null_pool_concurrency_limit_verified"
] is True
assert database_boundary["secret_values_read"] is False
assert database_boundary["role_names_exposed"] is False
assert database_boundary["signal_freshness"]["fresh"] is True
@@ -361,6 +365,23 @@ def test_executor_trust_boundary_requires_live_connection_headroom() -> None:
)
def test_executor_trust_boundary_requires_api_null_pool_concurrency_cap() -> None:
receipt = _verified_receipt()
receipt["api_database_null_pool_concurrency_limit"] = "0"
readback = build_executor_trust_boundary_readback(
receipt,
deployed_source_sha="abc123",
)
database_boundary = readback["database_identity_boundary"]
assert readback["full_workload_boundary_verified"] is False
assert database_boundary["connection_budget_verified"] is False
assert "api_null_pool_concurrency_limit_not_verified" in (
database_boundary["active_blockers"]
)
def test_executor_trust_boundary_binds_budget_and_verifier_contract() -> None:
wrong_budget = _verified_receipt()
wrong_budget["worker_required_connection_budget"] = "4"

356
apps/api/tests/test_runtime_bootstrap_guards.py
View File

@@ -225,17 +225,11 @@ def test_get_engine_uses_null_pool_for_bounded_background_processes(monkeypatch)
@pytest.mark.asyncio
async def test_null_pool_session_limiter_serializes_worker_db_sessions(
async def test_null_pool_session_limiter_serializes_raw_factory_sessions(
monkeypatch,
) -> None:
from src.db import base as db_base
assert "async with _database_session_slot()" in inspect.getsource(
db_base.get_db
)
assert "async with _database_session_slot()" in inspect.getsource(
db_base.get_db_context
)
monkeypatch.setattr(db_base.settings, "DATABASE_NULL_POOL", True)
monkeypatch.setattr(
db_base.settings,
@@ -245,17 +239,25 @@ async def test_null_pool_session_limiter_serializes_worker_db_sessions(
monkeypatch.setattr(db_base.settings, "DATABASE_POOL_TIMEOUT_SECONDS", 1.0)
monkeypatch.setattr(db_base, "_null_pool_session_limiter", None)
monkeypatch.setattr(db_base, "_null_pool_session_limiter_size", 0)
monkeypatch.setattr(db_base, "_session_factory", None)
monkeypatch.setattr(db_base, "get_engine", lambda: None)
factory = db_base.get_session_factory()
assert factory.class_ is db_base._BudgetedAsyncSession
assert "_database_session_slot" not in inspect.getsource(db_base.get_db)
assert "_database_session_slot" not in inspect.getsource(db_base.get_db_context)
first_entered = asyncio.Event()
release_first = asyncio.Event()
second_entered = asyncio.Event()
async def first_session() -> None:
async with db_base._database_session_slot():
async with factory():
first_entered.set()
await release_first.wait()
async def second_session() -> None:
async with db_base._database_session_slot():
async with factory():
second_entered.set()
first_task = asyncio.create_task(first_session())
@@ -270,6 +272,342 @@ async def test_null_pool_session_limiter_serializes_worker_db_sessions(
assert second_entered.is_set() is True
@pytest.mark.asyncio
async def test_unit_of_work_shares_raw_factory_cap_without_double_acquire(
monkeypatch,
) -> None:
from sqlalchemy.ext.asyncio import AsyncSession
from src.core.unit_of_work import UnitOfWork
from src.db import base as db_base
monkeypatch.setattr(db_base.settings, "DATABASE_NULL_POOL", True)
monkeypatch.setattr(
db_base.settings,
"DATABASE_NULL_POOL_CONCURRENCY_LIMIT",
1,
)
monkeypatch.setattr(db_base.settings, "DATABASE_POOL_TIMEOUT_SECONDS", 1.0)
monkeypatch.setattr(db_base, "_null_pool_session_limiter", None)
monkeypatch.setattr(db_base, "_null_pool_session_limiter_size", 0)
monkeypatch.setattr(db_base, "_session_factory", None)
monkeypatch.setattr(db_base, "get_engine", lambda: None)
executed_session_ids: list[int] = []
async def fake_execute(
session: AsyncSession,
*_args: object,
**_kwargs: object,
) -> object:
executed_session_ids.append(id(session))
return object()
async def fake_commit(_session: AsyncSession) -> None:
return None
async def fake_close(_session: AsyncSession) -> None:
return None
monkeypatch.setattr(AsyncSession, "execute", fake_execute)
monkeypatch.setattr(AsyncSession, "commit", fake_commit)
monkeypatch.setattr(AsyncSession, "close", fake_close)
factory = db_base.get_session_factory()
raw_entered = asyncio.Event()
release_raw = asyncio.Event()
uow_entered = asyncio.Event()
async def raw_factory_session() -> None:
async with factory():
raw_entered.set()
await release_raw.wait()
async def unit_of_work_session() -> None:
async with UnitOfWork(factory, project_id="awoooi"):
uow_entered.set()
raw_task = asyncio.create_task(raw_factory_session())
await raw_entered.wait()
uow_task = asyncio.create_task(unit_of_work_session())
await asyncio.sleep(0)
assert uow_entered.is_set() is False
assert executed_session_ids == []
release_raw.set()
await asyncio.gather(raw_task, uow_task)
assert uow_entered.is_set() is True
assert len(executed_session_ids) == 1
async with asyncio.timeout(0.5):
async with db_base.get_db_context("awoooi"):
pass
assert len(executed_session_ids) == 2
@pytest.mark.asyncio
async def test_unit_of_work_enter_failure_releases_global_session_cap(
monkeypatch,
) -> None:
from sqlalchemy.ext.asyncio import AsyncSession
from src.core.unit_of_work import UnitOfWork
from src.db import base as db_base
monkeypatch.setattr(db_base.settings, "DATABASE_NULL_POOL", True)
monkeypatch.setattr(
db_base.settings,
"DATABASE_NULL_POOL_CONCURRENCY_LIMIT",
1,
)
monkeypatch.setattr(db_base.settings, "DATABASE_POOL_TIMEOUT_SECONDS", 0.2)
monkeypatch.setattr(db_base, "_null_pool_session_limiter", None)
monkeypatch.setattr(db_base, "_null_pool_session_limiter_size", 0)
monkeypatch.setattr(db_base, "_session_factory", None)
monkeypatch.setattr(db_base, "get_engine", lambda: None)
execute_attempts = 0
async def fake_execute(
_session: AsyncSession,
*_args: object,
**_kwargs: object,
) -> object:
nonlocal execute_attempts
execute_attempts += 1
if execute_attempts == 1:
raise RuntimeError("synthetic_uow_enter_failure")
return object()
async def fake_commit(_session: AsyncSession) -> None:
return None
async def fake_close(_session: AsyncSession) -> None:
return None
monkeypatch.setattr(AsyncSession, "execute", fake_execute)
monkeypatch.setattr(AsyncSession, "commit", fake_commit)
monkeypatch.setattr(AsyncSession, "close", fake_close)
factory = db_base.get_session_factory()
with pytest.raises(RuntimeError, match="synthetic_uow_enter_failure"):
async with UnitOfWork(factory, project_id="awoooi"):
pass
async with asyncio.timeout(0.5):
async with UnitOfWork(factory, project_id="awoooi"):
pass
assert execute_attempts == 2
@pytest.mark.asyncio
async def test_budgeted_transaction_enter_failure_releases_global_session_cap(
monkeypatch,
) -> None:
from sqlalchemy.ext.asyncio import AsyncSession, AsyncSessionTransaction
from src.db import base as db_base
monkeypatch.setattr(db_base.settings, "DATABASE_NULL_POOL", True)
monkeypatch.setattr(
db_base.settings,
"DATABASE_NULL_POOL_CONCURRENCY_LIMIT",
1,
)
monkeypatch.setattr(db_base.settings, "DATABASE_POOL_TIMEOUT_SECONDS", 0.2)
monkeypatch.setattr(db_base, "_null_pool_session_limiter", None)
monkeypatch.setattr(db_base, "_null_pool_session_limiter_size", 0)
monkeypatch.setattr(db_base, "_session_factory", None)
monkeypatch.setattr(db_base, "get_engine", lambda: None)
async def fail_start(
_transaction: AsyncSessionTransaction,
is_ctxmanager: bool = False,
) -> AsyncSessionTransaction:
del is_ctxmanager
raise RuntimeError("synthetic_transaction_enter_failure")
async def fake_close(_session: AsyncSession) -> None:
return None
monkeypatch.setattr(AsyncSessionTransaction, "start", fail_start)
monkeypatch.setattr(AsyncSession, "close", fake_close)
factory = db_base.get_session_factory()
with pytest.raises(RuntimeError, match="synthetic_transaction_enter_failure"):
async with factory.begin():
pass
async with asyncio.timeout(0.5):
async with factory():
pass
@pytest.mark.asyncio
async def test_cancelled_transaction_entry_releases_global_session_cap(
monkeypatch,
) -> None:
from sqlalchemy.ext.asyncio import AsyncSession, AsyncSessionTransaction
from src.db import base as db_base
monkeypatch.setattr(db_base.settings, "DATABASE_NULL_POOL", True)
monkeypatch.setattr(
db_base.settings,
"DATABASE_NULL_POOL_CONCURRENCY_LIMIT",
1,
)
monkeypatch.setattr(db_base.settings, "DATABASE_POOL_TIMEOUT_SECONDS", 0.2)
monkeypatch.setattr(db_base, "_null_pool_session_limiter", None)
monkeypatch.setattr(db_base, "_null_pool_session_limiter_size", 0)
monkeypatch.setattr(db_base, "_session_factory", None)
monkeypatch.setattr(db_base, "get_engine", lambda: None)
transaction_start_entered = asyncio.Event()
never_complete = asyncio.Event()
async def block_start(
_transaction: AsyncSessionTransaction,
is_ctxmanager: bool = False,
) -> AsyncSessionTransaction:
del is_ctxmanager
transaction_start_entered.set()
await never_complete.wait()
raise AssertionError("unreachable")
async def fake_close(_session: AsyncSession) -> None:
return None
monkeypatch.setattr(AsyncSessionTransaction, "start", block_start)
monkeypatch.setattr(AsyncSession, "close", fake_close)
factory = db_base.get_session_factory()
async def enter_transaction() -> None:
async with factory.begin():
pass
transaction_task = asyncio.create_task(enter_transaction())
await transaction_start_entered.wait()
transaction_task.cancel()
with pytest.raises(asyncio.CancelledError):
await transaction_task
async with asyncio.timeout(0.5):
async with factory():
pass
@pytest.mark.asyncio
async def test_cancelled_slot_waiter_does_not_consume_or_over_release_capacity(
monkeypatch,
) -> None:
from sqlalchemy.ext.asyncio import AsyncSession
from src.db import base as db_base
monkeypatch.setattr(db_base.settings, "DATABASE_NULL_POOL", True)
monkeypatch.setattr(
db_base.settings,
"DATABASE_NULL_POOL_CONCURRENCY_LIMIT",
1,
)
monkeypatch.setattr(db_base.settings, "DATABASE_POOL_TIMEOUT_SECONDS", 1.0)
monkeypatch.setattr(db_base, "_null_pool_session_limiter", None)
monkeypatch.setattr(db_base, "_null_pool_session_limiter_size", 0)
monkeypatch.setattr(db_base, "_session_factory", None)
monkeypatch.setattr(db_base, "get_engine", lambda: None)
async def fake_close(_session: AsyncSession) -> None:
return None
monkeypatch.setattr(AsyncSession, "close", fake_close)
factory = db_base.get_session_factory()
first_entered = asyncio.Event()
release_first = asyncio.Event()
async def hold_first_slot() -> None:
async with factory():
first_entered.set()
await release_first.wait()
async def wait_for_slot() -> None:
async with factory():
pass
first_task = asyncio.create_task(hold_first_slot())
await first_entered.wait()
cancelled_waiter = asyncio.create_task(wait_for_slot())
await asyncio.sleep(0)
cancelled_waiter.cancel()
with pytest.raises(asyncio.CancelledError):
await cancelled_waiter
release_first.set()
await first_task
async with asyncio.timeout(0.5):
async with factory():
pass
@pytest.mark.asyncio
async def test_unit_of_work_reentry_fails_without_replacing_active_session(
monkeypatch,
) -> None:
from sqlalchemy.ext.asyncio import AsyncSession
from src.core.unit_of_work import UnitOfWork
from src.db import base as db_base
monkeypatch.setattr(db_base.settings, "DATABASE_NULL_POOL", True)
monkeypatch.setattr(
db_base.settings,
"DATABASE_NULL_POOL_CONCURRENCY_LIMIT",
1,
)
monkeypatch.setattr(db_base.settings, "DATABASE_POOL_TIMEOUT_SECONDS", 0.2)
monkeypatch.setattr(db_base, "_null_pool_session_limiter", None)
monkeypatch.setattr(db_base, "_null_pool_session_limiter_size", 0)
monkeypatch.setattr(db_base, "_session_factory", None)
monkeypatch.setattr(db_base, "get_engine", lambda: None)
async def fake_execute(
_session: AsyncSession,
*_args: object,
**_kwargs: object,
) -> object:
return object()
async def fake_commit(_session: AsyncSession) -> None:
return None
async def fake_close(_session: AsyncSession) -> None:
return None
monkeypatch.setattr(AsyncSession, "execute", fake_execute)
monkeypatch.setattr(AsyncSession, "commit", fake_commit)
monkeypatch.setattr(AsyncSession, "close", fake_close)
factory = db_base.get_session_factory()
unit_of_work = UnitOfWork(factory, project_id="awoooi")
async with unit_of_work:
active_session = unit_of_work.session
with pytest.raises(
RuntimeError,
match="UnitOfWork cannot be entered more than once",
):
await unit_of_work.__aenter__()
assert unit_of_work.session is active_session
async with asyncio.timeout(0.5):
async with factory():
pass
@pytest.mark.asyncio
async def test_init_db_serializes_bootstrap_ddl_with_advisory_lock(monkeypatch):
from src.db import base as db_base

1
apps/api/tests/test_signal_worker_ansible_executor_binding.py
View File

@@ -144,6 +144,7 @@ def test_api_manifest_has_read_only_identity_and_no_ssh_executor_mounts() -> Non
assert pod_spec["serviceAccountName"] == "awoooi-api"
assert env["SSH_MCP_ENABLED"] == "false"
assert env["DATABASE_NULL_POOL"] == "true"
assert env["DATABASE_NULL_POOL_CONCURRENCY_LIMIT"] == "2"
assert env["AWOOOI_API_BACKGROUND_TASKS_ENABLED"] == "false"
assert env["ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER"] == "false"
assert env["ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER"] == "false"

58
apps/api/tests/test_workload_database_identity_projection.py
View File

@@ -196,6 +196,11 @@ def test_runtime_workloads_use_distinct_database_secret_projections() -> None:
"awoooi-broker-db",
}
api_env = _env_by_name(
_deployment_container("k8s/awoooi-prod/06-deployment-api.yaml", "api")
)
assert api_env["DATABASE_NULL_POOL_CONCURRENCY_LIMIT"]["value"] == "2"
def test_api_and_worker_project_shared_secrets_explicitly() -> None:
for path, container_name in (
@@ -217,6 +222,36 @@ def test_api_and_worker_project_shared_secrets_explicitly() -> None:
assert secret_ref["optional"] is True
def test_api_null_pool_cap_preserves_the_verified_role_headroom() -> None:
documents = yaml.safe_load_all(
(
REPO_ROOT / "k8s/awoooi-prod/06-deployment-api.yaml"
).read_text()
)
deployment = next(
document
for document in documents
if document and document.get("kind") == "Deployment"
)
container = next(
item
for item in deployment["spec"]["template"]["spec"]["containers"]
if item["name"] == "api"
)
env = _env_by_name(container)
replica_count = int(deployment["spec"]["replicas"])
per_process_cap = int(
env["DATABASE_NULL_POOL_CONCURRENCY_LIMIT"]["value"]
)
role_connection_limit = 12
required_free_headroom = 8
assert replica_count == 2
assert replica_count * per_process_cap == (
role_connection_limit - required_free_headroom
)
def test_workload_database_bootstrap_keeps_secret_values_off_output() -> None:
script = (
REPO_ROOT / "scripts/ops/awoooi-workload-db-identity-bootstrap.sh"
@@ -261,6 +296,7 @@ def test_workload_database_bootstrap_keeps_secret_values_off_output() -> None:
def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None:
workflow = (REPO_ROOT / ".gitea/workflows/cd.yaml").read_text()
normalized_workflow = " ".join(workflow.split())
assert "WORKLOAD_DB_ROLLBACK_FILE" in workflow
assert "WORKLOAD_DB_SECRET_PREEXISTENCE_FILE" in workflow
@@ -285,6 +321,28 @@ def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None:
assert "API_PUBLIC_READINESS_ATTEMPTS" in workflow
assert "API_HTTP_CONCURRENCY_ATTEMPTS" in workflow
assert "API_HTTP_PROBE_SAFETY_RESERVE=4" in workflow
assert '"$API_DB_NULL_POOL_CONCURRENCY_LIMIT" -eq 2' in workflow
assert (
'"$API_DB_AVAILABLE_CONNECTION_HEADROOM" -ge '
'"$API_DB_REQUIRED_CONNECTION_BUDGET"'
) in workflow
assert (
'"$WORKER_DB_AVAILABLE_CONNECTION_HEADROOM" -ge '
'"$WORKER_DB_REQUIRED_CONNECTION_BUDGET"'
) in workflow
assert (
'"$BROKER_DB_AVAILABLE_CONNECTION_HEADROOM" -ge '
'"$BROKER_DB_REQUIRED_CONNECTION_BUDGET"'
) in workflow
assert (
"available_connection_headroom >= required_connection_budget"
in normalized_workflow
)
assert (
'"$API_DB_AVAILABLE_CONNECTION_HEADROOM" -ge '
'"$API_DB_REPRESENTATIVE_PROBE_COUNT"'
) not in workflow
assert "api_database_null_pool_concurrency_limit" in workflow
assert "API_HTTP_CONCURRENCY_WORKERS" in workflow
assert "API sequential readiness permanent HTTP status" in workflow
assert "api/v1/health/ready" in workflow