fix(api): bound run evidence batch receipts
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m39s
CD Pipeline / build-and-deploy (push) Successful in 10m32s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 13s
CD Pipeline / post-deploy-checks (push) Successful in 2m12s

This commit is contained in:
ogt
2026-07-14 17:05:30 +08:00
parent 1471f05f59
commit bf44f66fb3
2 changed files with 132 additions and 50 deletions

View File

@@ -106,6 +106,8 @@ _REMEDIATION_HISTORY_SCHEMA_VERSIONS = {
"adr100_remediation_dry_run_history_v1",
"adr100_remediation_approval_history_v1",
}
_MAX_REMEDIATION_HISTORY_BATCH_INCIDENTS = _MAX_LIST_CONTEXT_ROWS
_MAX_REMEDIATION_HISTORY_BATCH_ROWS = 5_000
_ADR100_GATE5_PROJECTION_TRIGGER = "adr100_runtime_replay_gate5"
_CALLBACK_REPLY_CACHE_TTL_SECONDS = int(
os.getenv("AWOOOP_CALLBACK_REPLY_CACHE_TTL_SECONDS", "20")
@@ -7617,17 +7619,21 @@ async def _fetch_run_remediation_histories_by_incident(
db: Any | None = None,
) -> tuple[dict[str, list[dict[str, Any]]], dict[str, dict[str, str]]]:
"""Fetch ADR-100 list evidence with one bounded query for all incidents."""
normalized_incident_ids = list(dict.fromkeys(incident_ids))
requested_incident_ids = list(dict.fromkeys(incident_ids))
normalized_incident_ids = requested_incident_ids[
:_MAX_REMEDIATION_HISTORY_BATCH_INCIDENTS
]
if not normalized_incident_ids:
return {}, {}
safe_limit = max(1, min(limit, 200))
per_event_fetch_limit = min(max(safe_limit * 4, 50), 200)
batch_limit = (
requested_batch_limit = (
len(normalized_incident_ids)
* len(_REMEDIATION_HISTORY_EVENT_TYPES)
* per_event_fetch_limit
)
batch_limit = min(requested_batch_limit, _MAX_REMEDIATION_HISTORY_BATCH_ROWS)
event_rank = func.row_number().over(
partition_by=(
AlertOperationLog.incident_id,
@@ -7654,7 +7660,7 @@ async def _fetch_run_remediation_histories_by_incident(
)
.where(ranked_rows.c.event_rank <= per_event_fetch_limit)
.order_by(AlertOperationLog.created_at.desc())
.limit(batch_limit)
.limit(batch_limit + 1)
)
try:
@@ -7665,17 +7671,19 @@ async def _fetch_run_remediation_histories_by_incident(
result = await db.execute(stmt)
rows = list(result.scalars().all())
except Exception as exc:
error = str(exc)
error = f"remediation_history_batch_fetch_failed:{type(exc).__name__}"
logger.warning(
"run_list_remediation_history_batch_fetch_failed",
incident_count=len(normalized_incident_ids),
error=error,
error_type=type(exc).__name__,
)
return {}, {
incident_id: {"incident_id": incident_id, "error": error}
for incident_id in normalized_incident_ids
for incident_id in requested_incident_ids
}
row_limit_reached = len(rows) > batch_limit
rows = rows[:batch_limit]
rows_by_incident: dict[str, list[AlertOperationLog]] = defaultdict(list)
for row in rows:
incident_id = str(getattr(row, "incident_id", "") or "")
@@ -7686,6 +7694,23 @@ async def _fetch_run_remediation_histories_by_incident(
histories_by_incident: dict[str, list[dict[str, Any]]] = {}
errors_by_incident: dict[str, dict[str, str]] = {}
if row_limit_reached:
errors_by_incident.update({
incident_id: {
"incident_id": incident_id,
"error": "remediation_history_batch_row_limit_reached",
}
for incident_id in normalized_incident_ids
})
errors_by_incident.update({
incident_id: {
"incident_id": incident_id,
"error": "remediation_history_batch_incident_limit_reached",
}
for incident_id in requested_incident_ids[
_MAX_REMEDIATION_HISTORY_BATCH_INCIDENTS:
]
})
for incident_id in normalized_incident_ids:
try:
items: list[dict[str, Any]] = []
@@ -7706,11 +7731,11 @@ async def _fetch_run_remediation_histories_by_incident(
break
histories_by_incident[incident_id] = items
except Exception as exc:
error = str(exc)
error = f"remediation_history_row_decode_failed:{type(exc).__name__}"
logger.warning(
"run_list_remediation_history_row_decode_failed",
incident_id=incident_id,
error=error,
error_type=type(exc).__name__,
)
errors_by_incident[incident_id] = {
"incident_id": incident_id,
@@ -7754,9 +7779,16 @@ async def _build_run_remediation_summaries(
db=db,
)
)
legacy_incident_ids = all_incident_ids[
:_MAX_REMEDIATION_HISTORY_BATCH_INCIDENTS
]
legacy_mcp_by_incident = await _fetch_legacy_mcp_by_incident_ids(
all_incident_ids,
limit=min(max(len(all_incident_ids) * _REMEDIATION_HISTORY_LIMIT, 100), 5_000),
legacy_incident_ids,
limit=min(
max(len(legacy_incident_ids) * _REMEDIATION_HISTORY_LIMIT, 100),
5_000,
),
db=db,
)
summaries: dict[UUID, dict[str, Any]] = {}
@@ -7881,19 +7913,24 @@ async def _fetch_legacy_mcp_by_incident_ids(
incident_ids: list[str],
*,
limit: int,
db: Any | None = None,
) -> dict[str, list[dict[str, Any]]]:
"""Fetch legacy/self-built MCP rows for list evidence summaries."""
if not incident_ids:
return {}
async with get_db_context("awoooi") as db:
result = await db.execute(
select(MCPAuditLog)
.where(MCPAuditLog.incident_id.in_(incident_ids))
.order_by(MCPAuditLog.created_at.desc())
.limit(limit)
)
rows = list(result.scalars().all())
stmt = (
select(MCPAuditLog)
.where(MCPAuditLog.incident_id.in_(incident_ids))
.order_by(MCPAuditLog.created_at.desc())
.limit(limit)
)
if db is None:
async with get_db_context("awoooi") as legacy_db:
result = await legacy_db.execute(stmt)
else:
result = await db.execute(stmt)
rows = list(result.scalars().all())
by_incident: dict[str, list[dict[str, Any]]] = defaultdict(list)
for row in rows: