fix(api): bound run evidence batch receipts
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m39s
CD Pipeline / build-and-deploy (push) Successful in 10m32s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 13s
CD Pipeline / post-deploy-checks (push) Successful in 2m12s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m39s
CD Pipeline / build-and-deploy (push) Successful in 10m32s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 13s
CD Pipeline / post-deploy-checks (push) Successful in 2m12s
This commit is contained in:
@@ -106,6 +106,8 @@ _REMEDIATION_HISTORY_SCHEMA_VERSIONS = {
|
||||
"adr100_remediation_dry_run_history_v1",
|
||||
"adr100_remediation_approval_history_v1",
|
||||
}
|
||||
_MAX_REMEDIATION_HISTORY_BATCH_INCIDENTS = _MAX_LIST_CONTEXT_ROWS
|
||||
_MAX_REMEDIATION_HISTORY_BATCH_ROWS = 5_000
|
||||
_ADR100_GATE5_PROJECTION_TRIGGER = "adr100_runtime_replay_gate5"
|
||||
_CALLBACK_REPLY_CACHE_TTL_SECONDS = int(
|
||||
os.getenv("AWOOOP_CALLBACK_REPLY_CACHE_TTL_SECONDS", "20")
|
||||
@@ -7617,17 +7619,21 @@ async def _fetch_run_remediation_histories_by_incident(
|
||||
db: Any | None = None,
|
||||
) -> tuple[dict[str, list[dict[str, Any]]], dict[str, dict[str, str]]]:
|
||||
"""Fetch ADR-100 list evidence with one bounded query for all incidents."""
|
||||
normalized_incident_ids = list(dict.fromkeys(incident_ids))
|
||||
requested_incident_ids = list(dict.fromkeys(incident_ids))
|
||||
normalized_incident_ids = requested_incident_ids[
|
||||
:_MAX_REMEDIATION_HISTORY_BATCH_INCIDENTS
|
||||
]
|
||||
if not normalized_incident_ids:
|
||||
return {}, {}
|
||||
|
||||
safe_limit = max(1, min(limit, 200))
|
||||
per_event_fetch_limit = min(max(safe_limit * 4, 50), 200)
|
||||
batch_limit = (
|
||||
requested_batch_limit = (
|
||||
len(normalized_incident_ids)
|
||||
* len(_REMEDIATION_HISTORY_EVENT_TYPES)
|
||||
* per_event_fetch_limit
|
||||
)
|
||||
batch_limit = min(requested_batch_limit, _MAX_REMEDIATION_HISTORY_BATCH_ROWS)
|
||||
event_rank = func.row_number().over(
|
||||
partition_by=(
|
||||
AlertOperationLog.incident_id,
|
||||
@@ -7654,7 +7660,7 @@ async def _fetch_run_remediation_histories_by_incident(
|
||||
)
|
||||
.where(ranked_rows.c.event_rank <= per_event_fetch_limit)
|
||||
.order_by(AlertOperationLog.created_at.desc())
|
||||
.limit(batch_limit)
|
||||
.limit(batch_limit + 1)
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -7665,17 +7671,19 @@ async def _fetch_run_remediation_histories_by_incident(
|
||||
result = await db.execute(stmt)
|
||||
rows = list(result.scalars().all())
|
||||
except Exception as exc:
|
||||
error = str(exc)
|
||||
error = f"remediation_history_batch_fetch_failed:{type(exc).__name__}"
|
||||
logger.warning(
|
||||
"run_list_remediation_history_batch_fetch_failed",
|
||||
incident_count=len(normalized_incident_ids),
|
||||
error=error,
|
||||
error_type=type(exc).__name__,
|
||||
)
|
||||
return {}, {
|
||||
incident_id: {"incident_id": incident_id, "error": error}
|
||||
for incident_id in normalized_incident_ids
|
||||
for incident_id in requested_incident_ids
|
||||
}
|
||||
|
||||
row_limit_reached = len(rows) > batch_limit
|
||||
rows = rows[:batch_limit]
|
||||
rows_by_incident: dict[str, list[AlertOperationLog]] = defaultdict(list)
|
||||
for row in rows:
|
||||
incident_id = str(getattr(row, "incident_id", "") or "")
|
||||
@@ -7686,6 +7694,23 @@ async def _fetch_run_remediation_histories_by_incident(
|
||||
|
||||
histories_by_incident: dict[str, list[dict[str, Any]]] = {}
|
||||
errors_by_incident: dict[str, dict[str, str]] = {}
|
||||
if row_limit_reached:
|
||||
errors_by_incident.update({
|
||||
incident_id: {
|
||||
"incident_id": incident_id,
|
||||
"error": "remediation_history_batch_row_limit_reached",
|
||||
}
|
||||
for incident_id in normalized_incident_ids
|
||||
})
|
||||
errors_by_incident.update({
|
||||
incident_id: {
|
||||
"incident_id": incident_id,
|
||||
"error": "remediation_history_batch_incident_limit_reached",
|
||||
}
|
||||
for incident_id in requested_incident_ids[
|
||||
_MAX_REMEDIATION_HISTORY_BATCH_INCIDENTS:
|
||||
]
|
||||
})
|
||||
for incident_id in normalized_incident_ids:
|
||||
try:
|
||||
items: list[dict[str, Any]] = []
|
||||
@@ -7706,11 +7731,11 @@ async def _fetch_run_remediation_histories_by_incident(
|
||||
break
|
||||
histories_by_incident[incident_id] = items
|
||||
except Exception as exc:
|
||||
error = str(exc)
|
||||
error = f"remediation_history_row_decode_failed:{type(exc).__name__}"
|
||||
logger.warning(
|
||||
"run_list_remediation_history_row_decode_failed",
|
||||
incident_id=incident_id,
|
||||
error=error,
|
||||
error_type=type(exc).__name__,
|
||||
)
|
||||
errors_by_incident[incident_id] = {
|
||||
"incident_id": incident_id,
|
||||
@@ -7754,9 +7779,16 @@ async def _build_run_remediation_summaries(
|
||||
db=db,
|
||||
)
|
||||
)
|
||||
legacy_incident_ids = all_incident_ids[
|
||||
:_MAX_REMEDIATION_HISTORY_BATCH_INCIDENTS
|
||||
]
|
||||
legacy_mcp_by_incident = await _fetch_legacy_mcp_by_incident_ids(
|
||||
all_incident_ids,
|
||||
limit=min(max(len(all_incident_ids) * _REMEDIATION_HISTORY_LIMIT, 100), 5_000),
|
||||
legacy_incident_ids,
|
||||
limit=min(
|
||||
max(len(legacy_incident_ids) * _REMEDIATION_HISTORY_LIMIT, 100),
|
||||
5_000,
|
||||
),
|
||||
db=db,
|
||||
)
|
||||
|
||||
summaries: dict[UUID, dict[str, Any]] = {}
|
||||
@@ -7881,19 +7913,24 @@ async def _fetch_legacy_mcp_by_incident_ids(
|
||||
incident_ids: list[str],
|
||||
*,
|
||||
limit: int,
|
||||
db: Any | None = None,
|
||||
) -> dict[str, list[dict[str, Any]]]:
|
||||
"""Fetch legacy/self-built MCP rows for list evidence summaries."""
|
||||
if not incident_ids:
|
||||
return {}
|
||||
|
||||
async with get_db_context("awoooi") as db:
|
||||
result = await db.execute(
|
||||
select(MCPAuditLog)
|
||||
.where(MCPAuditLog.incident_id.in_(incident_ids))
|
||||
.order_by(MCPAuditLog.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
rows = list(result.scalars().all())
|
||||
stmt = (
|
||||
select(MCPAuditLog)
|
||||
.where(MCPAuditLog.incident_id.in_(incident_ids))
|
||||
.order_by(MCPAuditLog.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if db is None:
|
||||
async with get_db_context("awoooi") as legacy_db:
|
||||
result = await legacy_db.execute(stmt)
|
||||
else:
|
||||
result = await db.execute(stmt)
|
||||
rows = list(result.scalars().all())
|
||||
|
||||
by_incident: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in rows:
|
||||
|
||||
Reference in New Issue
Block a user