fix(agent99): bound DB sessions and expire stale outcomes
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 4m0s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 17s

This commit is contained in:
ogt
2026-07-15 08:47:05 +08:00
parent 47586431c0
commit 0f2ddc0ce8
10 changed files with 569 additions and 37 deletions

View File

@@ -298,6 +298,16 @@ class Settings(BaseSettings):
"processes that share a constrained production role."
),
)
DATABASE_NULL_POOL_CONCURRENCY_LIMIT: int = Field(
default=0,
ge=0,
le=50,
description=(
"Per-process concurrent DB session cap when NullPool is enabled. "
"Zero leaves concurrency unchanged; bounded workers set this to "
"their verified role budget."
),
)
DATABASE_BOOTSTRAP_DDL_ENABLED: bool = Field(
default=True,
description=(

View File

@@ -50,9 +50,54 @@ class Base(DeclarativeBase):
_engine: AsyncEngine | None = None
_session_factory: async_sessionmaker[AsyncSession] | None = None
_null_pool_session_limiter: asyncio.BoundedSemaphore | None = None
_null_pool_session_limiter_size = 0
logger = get_logger("awoooi.db")
def _get_null_pool_session_limiter() -> asyncio.BoundedSemaphore | None:
"""Honor the configured connection budget even when NullPool is active."""
global _null_pool_session_limiter, _null_pool_session_limiter_size
limit = int(settings.DATABASE_NULL_POOL_CONCURRENCY_LIMIT)
if not settings.DATABASE_NULL_POOL or limit <= 0:
return None
if _null_pool_session_limiter is None or _null_pool_session_limiter_size != limit:
_null_pool_session_limiter = asyncio.BoundedSemaphore(limit)
_null_pool_session_limiter_size = limit
return _null_pool_session_limiter
@asynccontextmanager
async def _database_session_slot() -> AsyncGenerator[None, None]:
"""Bound concurrent NullPool sessions without retaining DB connections."""
limiter = _get_null_pool_session_limiter()
if limiter is None:
yield
return
try:
await asyncio.wait_for(
limiter.acquire(),
timeout=settings.DATABASE_POOL_TIMEOUT_SECONDS,
)
except TimeoutError as exc:
logger.warning(
"database_null_pool_session_concurrency_limit_reached",
concurrency_limit=int(
settings.DATABASE_NULL_POOL_CONCURRENCY_LIMIT
),
timeout_seconds=settings.DATABASE_POOL_TIMEOUT_SECONDS,
)
raise SQLAlchemyTimeoutError(
"NullPool database session concurrency limit reached"
) from exc
try:
yield
finally:
limiter.release()
def _raise_unauthorized_db_context(msg: str) -> None:
context = get_current_project_context()
logger.error(
@@ -133,27 +178,28 @@ async def get_db() -> AsyncGenerator[AsyncSession, None]:
...
"""
factory = get_session_factory()
async with factory() as session:
try:
from src.core.context import get_current_project_id
async with _database_session_slot():
async with factory() as session:
try:
from src.core.context import get_current_project_id
# AwoooP Phase 2.3 (2026-05-04 ogt): SET LOCAL app.project_id 讓 RLS Policy 生效
# Fail-Closed RLS: 遇到未授權情境拋出錯誤而非回退到 "awoooi"
pid = get_current_project_id()
if not pid:
_raise_unauthorized_db_context(
"Unauthorized: project_id is missing in context (Fail-Closed RLS)"
# AwoooP Phase 2.3 (2026-05-04 ogt): SET LOCAL app.project_id 讓 RLS Policy 生效
# Fail-Closed RLS: 遇到未授權情境拋出錯誤而非回退到 "awoooi"
pid = get_current_project_id()
if not pid:
_raise_unauthorized_db_context(
"Unauthorized: project_id is missing in context (Fail-Closed RLS)"
)
await session.execute(
text("SELECT set_config('app.project_id', :pid, TRUE)"),
{"pid": pid},
)
await session.execute(
text("SELECT set_config('app.project_id', :pid, TRUE)"),
{"pid": pid},
)
yield session
await session.commit()
except Exception:
await session.rollback()
raise
yield session
await session.commit()
except Exception:
await session.rollback()
raise
@asynccontextmanager
@@ -178,17 +224,18 @@ async def get_db_context(project_id: str | None = None) -> AsyncGenerator[AsyncS
_raise_unauthorized_db_context("Unauthorized: project_id is missing in context (Fail-Closed RLS)")
factory = get_session_factory()
async with factory() as session:
try:
await session.execute(
text("SELECT set_config('app.project_id', :pid, TRUE)"),
{"pid": effective_pid},
)
yield session
await session.commit()
except Exception:
await session.rollback()
raise
async with _database_session_slot():
async with factory() as session:
try:
await session.execute(
text("SELECT set_config('app.project_id', :pid, TRUE)"),
{"pid": effective_pid},
)
yield session
await session.commit()
except Exception:
await session.rollback()
raise
# =============================================================================

View File

@@ -5,6 +5,7 @@ from __future__ import annotations
import asyncio
import hashlib
from collections.abc import Awaitable, Callable
from datetime import UTC, datetime
from typing import Any
import structlog
@@ -54,6 +55,25 @@ LegacyIncidentFetcher = Callable[..., Awaitable[list[dict[str, Any]]]]
Agent99Dispatcher = Callable[..., Awaitable[dict[str, Any]]]
def _dispatch_timeout_elapsed(value: Any) -> bool:
if not isinstance(value, datetime):
return False
normalized = value if value.tzinfo is not None else value.replace(tzinfo=UTC)
return normalized <= datetime.now(UTC)
async def _terminalize_expired_outcome(
ledger: Any,
item: dict[str, Any],
) -> bool:
if not _dispatch_timeout_elapsed(item.get("timeout_at")):
return False
terminal = await ledger.record_outcome_timeout_no_write_terminal(
identity=item["identity"],
)
return terminal.get("receipt_persisted") is True
def _is_bound_no_write_status_outcome(
outcome: dict[str, Any] | None,
*,
@@ -594,6 +614,7 @@ async def reconcile_agent99_controlled_dispatches_once(
"status_reconcile_requested": 0,
"status_reconcile_pending": 0,
"external_no_write_reconciled": 0,
"outcome_timeout_terminalized": 0,
"verifier_written": 0,
"closed": 0,
}
@@ -622,7 +643,10 @@ async def reconcile_agent99_controlled_dispatches_once(
source_receipt.get("resolved") is not True
or not source_receipt_ref
):
counters["source_not_resolved"] += 1
if await _terminalize_expired_outcome(ledger, item):
counters["outcome_timeout_terminalized"] += 1
else:
counters["source_not_resolved"] += 1
continue
outcome = await asyncio.to_thread(
read_agent99_sre_outcome,
@@ -633,6 +657,9 @@ async def reconcile_agent99_controlled_dispatches_once(
identity=identity,
source_receipt_ref=source_receipt_ref,
):
if await _terminalize_expired_outcome(ledger, item):
counters["outcome_timeout_terminalized"] += 1
continue
request_receipt = await asyncio.to_thread(
request_agent99_same_run_status_reconcile,
identity,
@@ -674,6 +701,8 @@ async def reconcile_agent99_controlled_dispatches_once(
str(identity.run_id),
)
if not isinstance(outcome, dict):
if await _terminalize_expired_outcome(ledger, item):
counters["outcome_timeout_terminalized"] += 1
continue
mode = str(outcome.get("mode") or mode)
outcome_contract = (
@@ -707,6 +736,8 @@ async def reconcile_agent99_controlled_dispatches_once(
evidence_refs=evidence_refs,
)
if verifier.get("receipt_persisted") is not True:
if await _terminalize_expired_outcome(ledger, item):
counters["outcome_timeout_terminalized"] += 1
continue
counters["verifier_written"] += 1
receipt = verifier
@@ -735,7 +766,20 @@ async def run_agent99_controlled_dispatch_reconciler_loop() -> None:
error=str(exc),
)
try:
await reconcile_agent99_controlled_dispatches_once()
reconciled = await reconcile_agent99_controlled_dispatches_once()
if any(
reconciled[key] > 0
for key in (
"external_no_write_reconciled",
"outcome_timeout_terminalized",
"verifier_written",
"closed",
)
):
logger.info(
"agent99_controlled_dispatch_reconciler_tick",
**reconciled,
)
except asyncio.CancelledError:
raise
except Exception as exc:

View File

@@ -1752,6 +1752,200 @@ class PostgresAgent99DispatchLedger:
"runtime_closure_verified": False,
}
async def record_outcome_timeout_no_write_terminal(
self,
*,
identity: Agent99DispatchIdentity,
) -> dict[str, Any]:
"""Fail an expired run without replaying transport or claiming closure."""
now = _utc_now_naive()
error_code = "E-AGENT99-OUTCOME-TIMEOUT"
try:
async with get_db_context(identity.project_id) as db:
current_result = await db.execute(
select(
AwoooPRunState.state,
AwoooPRunState.error_detail,
AwoooPRunState.timeout_at,
)
.where(
AwoooPRunState.run_id == identity.run_id,
AwoooPRunState.project_id == identity.project_id,
)
.with_for_update()
)
current = current_result.one_or_none()
envelope = _parse_envelope(
current.error_detail if current is not None else None
)
identity_matches = bool(
isinstance(envelope, dict)
and _stage_identity_matches(
identity,
envelope.get("identity")
if isinstance(envelope.get("identity"), dict)
else {},
)
)
if (
current is not None
and current.state == "failed"
and identity_matches
and envelope.get("outcome_timeout_no_write_terminal") is True
):
return {
**envelope,
"status": "outcome_timeout_no_write_terminal_idempotent",
"receipt_persisted": True,
"runtime_closure_verified": False,
}
if current is None or not identity_matches:
return {
"status": "outcome_timeout_run_identity_invalid_fail_closed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
timeout_at = current.timeout_at
if (
current.state != "waiting_tool"
or timeout_at is None
or timeout_at > now
):
return {
"status": "outcome_timeout_not_due_no_write",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
prior_controlled_apply_authorized = bool(
envelope.get("controlled_apply_authorized") is True
)
prior_runtime_execution_authorized = bool(
envelope.get("runtime_execution_authorized") is True
)
prior_runtime_execution_attempted = bool(
envelope.get("runtime_execution_attempted") is True
)
terminal_receipt = {
"schema_version": "agent99_outcome_timeout_no_write_v1",
"status": "authenticated_outcome_timeout",
**_stage_identity(identity),
"timeout_at": timeout_at.isoformat(),
"terminal_at": now.isoformat(),
"terminalizer_runtime_write_performed": False,
"original_runtime_write_status": (
"unknown_without_authenticated_outcome"
),
"transport_replayed": False,
"incident_resolution_authorized": False,
"learning_writeback_authorized": False,
"telegram_authorized": False,
"stores_raw_evidence": False,
}
envelope.update({
"status": "outcome_timeout_no_write_terminal",
"run_terminal_state": "failed",
"outcome_timeout_no_write_terminal": True,
"prior_controlled_apply_authorized": (
prior_controlled_apply_authorized
),
"prior_runtime_execution_authorized": (
prior_runtime_execution_authorized
),
"runtime_execution_attempted": (
prior_runtime_execution_attempted
),
"controlled_apply_authorized": False,
"runtime_execution_authorized": False,
"terminalizer_runtime_write_performed": False,
"outcome_runtime_write_status": (
"unknown_without_authenticated_outcome"
),
"transport_replayed": False,
"automation_execution_success": False,
"post_verifier_passed": False,
"runtime_closure_verified": False,
"closure_complete": False,
"incident_resolution_authorized": False,
"safe_next_action": (
"await_new_source_recurrence_after_agent99_relay_recovery"
),
"verifier": {
**_stage_identity(identity),
"step_seq": 2,
"status": "failed_authenticated_outcome_timeout",
"post_verifier_passed": False,
"receipt": terminal_receipt,
},
"learning_writeback": {
**_stage_identity(identity),
"step_seq": 3,
"status": "not_applicable_outcome_timeout_no_write",
"required_receipts": [],
"receipt_refs": {},
"stores_raw_evidence": False,
},
})
envelope_json = _stable_json(envelope)
updated = await db.execute(
update(AwoooPRunState)
.where(
AwoooPRunState.run_id == identity.run_id,
AwoooPRunState.project_id == identity.project_id,
AwoooPRunState.state == "waiting_tool",
AwoooPRunState.timeout_at <= now,
)
.values(
state="failed",
output_sha256=_sha256(envelope_json),
error_code=error_code,
error_detail=envelope_json,
heartbeat_at=now,
completed_at=now,
lease_until=None,
next_attempt_at=None,
worker_id=None,
)
.returning(AwoooPRunState.run_id)
)
persisted = updated.scalar_one_or_none() is not None
if persisted:
await db.execute(
update(AwoooPRunStepJournal)
.where(
AwoooPRunStepJournal.run_id == identity.run_id,
AwoooPRunStepJournal.step_seq.in_([2, 3]),
AwoooPRunStepJournal.result_status == "pending",
)
.values(
output_hash=_sha256(_stable_json(terminal_receipt)),
compensation_json=terminal_receipt,
result_status="failed",
error_code=error_code,
was_blocked=True,
block_reason="authenticated_outcome_timeout_no_write",
completed_at=now,
)
)
except Exception as exc:
logger.warning(
"agent99_outcome_timeout_terminal_failed_fail_closed",
incident_id=identity.incident_id,
run_id=str(identity.run_id),
error=str(exc),
)
return {
"status": "outcome_timeout_terminal_persistence_failed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
return {
**envelope,
"receipt_persisted": persisted,
"runtime_closure_verified": False,
}
async def record_learning_writeback(
self,
*,
@@ -2236,6 +2430,7 @@ class PostgresAgent99DispatchLedger:
AwoooPRunState.run_id,
AwoooPRunState.state,
AwoooPRunState.error_detail,
AwoooPRunState.timeout_at,
)
.where(
AwoooPRunState.project_id == (project_id or "awoooi"),
@@ -2309,6 +2504,7 @@ class PostgresAgent99DispatchLedger:
"identity": identity,
"run_state": str(row.state),
"receipt": envelope,
"timeout_at": getattr(row, "timeout_at", None),
})
return items

View File

@@ -101,6 +101,7 @@ async def test_reconciler_consumes_authenticated_outcome_and_closes_same_run(
"status_reconcile_requested": 0,
"status_reconcile_pending": 0,
"external_no_write_reconciled": 0,
"outcome_timeout_terminalized": 0,
"verifier_written": 1,
"closed": 1,
}
@@ -130,6 +131,7 @@ async def test_reconciler_does_not_finalize_without_outcome(monkeypatch) -> None
"status_reconcile_requested": 0,
"status_reconcile_pending": 0,
"external_no_write_reconciled": 0,
"outcome_timeout_terminalized": 0,
"verifier_written": 0,
"closed": 0,
}

View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import json
from copy import deepcopy
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock
@@ -812,6 +813,184 @@ async def test_external_no_write_terminal_rejects_malformed_source_counts(
assert result["receipt_persisted"] is False
def test_dispatch_timeout_elapsed_handles_naive_and_aware_datetimes() -> None:
past = datetime.now(UTC) - timedelta(minutes=1)
future = datetime.now(UTC) + timedelta(minutes=1)
assert job._dispatch_timeout_elapsed(past) is True
assert job._dispatch_timeout_elapsed(past.replace(tzinfo=None)) is True
assert job._dispatch_timeout_elapsed(future) is False
assert job._dispatch_timeout_elapsed(None) is False
@pytest.mark.asyncio
async def test_reconciler_terminalizes_expired_run_without_outcome(
monkeypatch,
) -> None:
identity = build_agent99_dispatch_identity(
project_id="awoooi",
incident_id="INC-BRR-EXPIRED",
source_fingerprint="backup-restore:" + "b" * 64,
route_id="agent99:backup_health:Status",
work_item_id="agent99-dispatch:awoooi:INC-BRR-EXPIRED",
)
class TimeoutLedger(_Ledger):
def __init__(self) -> None:
super().__init__()
self.timeout_calls: list[object] = []
async def list_reconcilable(self, **_kwargs): # type: ignore[no-untyped-def]
return [{
"identity": identity,
"receipt": self.list_receipt,
"timeout_at": datetime.now(UTC) - timedelta(minutes=1),
}]
async def record_outcome_timeout_no_write_terminal(
self,
**kwargs,
): # type: ignore[no-untyped-def]
self.timeout_calls.append(kwargs["identity"])
return {
"status": "outcome_timeout_no_write_terminal",
"receipt_persisted": True,
"runtime_closure_verified": False,
}
ledger = TimeoutLedger()
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: None)
result = await job.reconcile_agent99_controlled_dispatches_once()
assert result["outcome_timeout_terminalized"] == 1
assert result["verifier_written"] == 0
assert ledger.timeout_calls == [identity]
@pytest.mark.asyncio
async def test_outcome_timeout_terminal_does_not_resolve_incident_or_replay(
monkeypatch,
) -> None:
envelope = build_agent99_dispatch_receipt_envelope(
identity=IDENTITY,
dispatch_receipt={
"accepted": True,
"inbox_triggered": True,
"queue_accepted": True,
"dispatch_identity_matched": True,
"delivery_certainty": "delivered",
},
controlled_apply_authorized=True,
)
class TimeoutDB:
def __init__(self) -> None:
self.call = 0
self.statements: list[str] = []
async def execute(self, statement): # type: ignore[no-untyped-def]
self.call += 1
self.statements.append(str(statement))
if self.call == 1:
return _ScalarResult(
row=SimpleNamespace(
state="waiting_tool",
error_detail=json.dumps(envelope),
timeout_at=(
datetime.now(UTC).replace(tzinfo=None)
- timedelta(minutes=1)
),
)
)
if self.call == 2:
return _ScalarResult(value=IDENTITY.run_id)
return _ScalarResult()
db = TimeoutDB()
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(db),
)
result = (
await PostgresAgent99DispatchLedger()
.record_outcome_timeout_no_write_terminal(identity=IDENTITY)
)
assert result["receipt_persisted"] is True
assert result["run_terminal_state"] == "failed"
assert result["prior_controlled_apply_authorized"] is True
assert result["controlled_apply_authorized"] is False
assert result["terminalizer_runtime_write_performed"] is False
assert result["outcome_runtime_write_status"] == (
"unknown_without_authenticated_outcome"
)
assert result["transport_replayed"] is False
assert result["automation_execution_success"] is False
assert result["post_verifier_passed"] is False
assert result["runtime_closure_verified"] is False
assert result["incident_resolution_authorized"] is False
assert result["learning_writeback"]["status"] == (
"not_applicable_outcome_timeout_no_write"
)
sql = "\n".join(db.statements).lower()
assert "incident_records" not in sql
assert "alert_operation_logs" not in sql
assert "telegram" not in sql
@pytest.mark.asyncio
async def test_outcome_timeout_terminal_rejects_unexpired_run(
monkeypatch,
) -> None:
envelope = build_agent99_dispatch_receipt_envelope(
identity=IDENTITY,
dispatch_receipt={
"accepted": True,
"inbox_triggered": True,
"queue_accepted": True,
"dispatch_identity_matched": True,
},
controlled_apply_authorized=False,
)
class FutureTimeoutDB:
def __init__(self) -> None:
self.call = 0
async def execute(self, _statement): # type: ignore[no-untyped-def]
self.call += 1
return _ScalarResult(
row=SimpleNamespace(
state="waiting_tool",
error_detail=json.dumps(envelope),
timeout_at=(
datetime.now(UTC).replace(tzinfo=None)
+ timedelta(minutes=1)
),
)
)
db = FutureTimeoutDB()
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(db),
)
result = (
await PostgresAgent99DispatchLedger()
.record_outcome_timeout_no_write_terminal(identity=IDENTITY)
)
assert result["status"] == "outcome_timeout_not_due_no_write"
assert result["receipt_persisted"] is False
assert db.call == 1
def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() -> None:
root = Path(__file__).resolve().parents[3]
relay = (root / "agent99-sre-alert-relay.ps1").read_text(encoding="utf-8")

View File

@@ -172,3 +172,4 @@ def test_database_pool_budget_defaults_to_production_safe_values():
assert s.DATABASE_MAX_OVERFLOW == 0
assert s.DATABASE_POOL_TIMEOUT_SECONDS == 5.0
assert s.DATABASE_NULL_POOL is False
assert s.DATABASE_NULL_POOL_CONCURRENCY_LIMIT == 0

View File

@@ -8,6 +8,7 @@ Runtime bootstrap guard tests.
from __future__ import annotations
import asyncio
import inspect
from collections.abc import Awaitable
from pathlib import Path
@@ -223,6 +224,52 @@ def test_get_engine_uses_null_pool_for_bounded_background_processes(monkeypatch)
assert "pool_timeout" not in captured
@pytest.mark.asyncio
async def test_null_pool_session_limiter_serializes_worker_db_sessions(
monkeypatch,
) -> None:
from src.db import base as db_base
assert "async with _database_session_slot()" in inspect.getsource(
db_base.get_db
)
assert "async with _database_session_slot()" in inspect.getsource(
db_base.get_db_context
)
monkeypatch.setattr(db_base.settings, "DATABASE_NULL_POOL", True)
monkeypatch.setattr(
db_base.settings,
"DATABASE_NULL_POOL_CONCURRENCY_LIMIT",
1,
)
monkeypatch.setattr(db_base.settings, "DATABASE_POOL_TIMEOUT_SECONDS", 1.0)
monkeypatch.setattr(db_base, "_null_pool_session_limiter", None)
monkeypatch.setattr(db_base, "_null_pool_session_limiter_size", 0)
first_entered = asyncio.Event()
release_first = asyncio.Event()
second_entered = asyncio.Event()
async def first_session() -> None:
async with db_base._database_session_slot():
first_entered.set()
await release_first.wait()
async def second_session() -> None:
async with db_base._database_session_slot():
second_entered.set()
first_task = asyncio.create_task(first_session())
await first_entered.wait()
second_task = asyncio.create_task(second_session())
await asyncio.sleep(0)
assert second_entered.is_set() is False
release_first.set()
await asyncio.gather(first_task, second_task)
assert second_entered.is_set() is True
@pytest.mark.asyncio
async def test_init_db_serializes_bootstrap_ddl_with_advisory_lock(monkeypatch):
from src.db import base as db_base

View File

@@ -91,6 +91,8 @@ def test_worker_manifest_has_no_execution_transport_or_write_loop() -> None:
assert worker_env["MCP_FEDERATION_STARTUP_SLEEP_SECONDS"] == "45"
assert worker_env["MCP_FEDERATION_SOURCE_TIMEOUT_SECONDS"] == "10"
assert worker_env["AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WINDOW_HOURS"] == "168"
assert worker_env["DATABASE_NULL_POOL"] == "true"
assert worker_env["DATABASE_NULL_POOL_CONCURRENCY_LIMIT"] == "1"
def test_execution_broker_is_only_workload_with_ssh_transport() -> None: