wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(operator): fail closed on missing backup evidence

Browse Source
This commit is contained in:
ogt
2026-07-14 11:38:11 +08:00
parent 989862e296
commit 6c8fdea99f
3 changed files with 348 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

253
apps/api/src/services/platform_operator_service.py
View File

@@ -1515,8 +1515,9 @@ async def list_ai_alert_card_delivery_readback(
FROM awooop_outbound_message m
WHERE {where_sql}
""")
# Bound the page and the Agent99 subset before the lateral lookup. Without
# both materialized sets PostgreSQL can rescan the full run ledger per card.
# Resolve at most one Agent99 row per bounded page candidate. Materializing
# every Agent99 row for the project still grows with ledger history and can
# exhaust the readback timeout even when the requested page is small.
list_sql = text(f"""
WITH candidate_messages AS MATERIALIZED (
SELECT m.*
@@ -1525,18 +1526,56 @@ async def list_ai_alert_card_delivery_readback(
ORDER BY COALESCE(m.sent_at, m.queued_at) DESC, m.message_id DESC
LIMIT :limit OFFSET :offset
),
candidate_refs AS MATERIALIZED (
SELECT
m.message_id,
m.project_id,
NULLIF(
COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
'{{agent99_dispatch_receipt,run_id}}',
''
) AS receipt_run_id,
NULLIF(
COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
'{{agent99_dispatch_receipt,incident_id}}',
''
) AS receipt_incident_id
FROM candidate_messages m
),
agent99_runs AS MATERIALIZED (
SELECT
ars.error_detail,
ars.state,
ars.trace_id,
ars.run_id,
ars.project_id,
ars.trigger_ref,
ars.created_at
FROM awooop_run_state ars
WHERE ars.project_id = :project_id
AND ars.agent_id = 'agent99_controlled_dispatch'
cr.message_id,
matched.error_detail,
matched.state,
matched.trace_id
FROM candidate_refs cr
LEFT JOIN LATERAL (
SELECT
ars.error_detail,
ars.state,
ars.trace_id,
ars.run_id,
ars.created_at
FROM awooop_run_state ars
WHERE ars.project_id = cr.project_id
AND ars.agent_id = 'agent99_controlled_dispatch'
AND (
(
cr.receipt_run_id IS NOT NULL
AND ars.run_id::text = cr.receipt_run_id
)
OR (
cr.receipt_incident_id IS NOT NULL
AND ars.trigger_ref LIKE (
'agent99:' || cr.receipt_incident_id || ':%'
)
)
)
ORDER BY (
ars.run_id::text = COALESCE(cr.receipt_run_id, '')
) DESC, ars.created_at DESC
LIMIT 1
) matched ON TRUE
)
SELECT
m.message_id,
@@ -1566,46 +1605,8 @@ async def list_ai_alert_card_delivery_readback(
LEFT JOIN awooop_run_state r
ON r.project_id = m.project_id
AND r.run_id = m.run_id
LEFT JOIN LATERAL (
SELECT
ars.error_detail,
ars.state,
ars.trace_id
FROM agent99_runs ars
WHERE ars.project_id = m.project_id
AND (
ars.run_id::text = COALESCE(
COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
'{{agent99_dispatch_receipt,run_id}}',
''
)
OR (
COALESCE(
COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
'{{agent99_dispatch_receipt,incident_id}}',
''
) <> ''
AND ars.trigger_ref LIKE (
'agent99:' ||
COALESCE(
COALESCE(
m.source_envelope::jsonb,
'{{}}'::jsonb
) #>> '{{agent99_dispatch_receipt,incident_id}}',
''
) || ':%'
)
)
)
ORDER BY (
ars.run_id::text = COALESCE(
COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
'{{agent99_dispatch_receipt,run_id}}',
''
)
) DESC, ars.created_at DESC
LIMIT 1
) ar ON TRUE
LEFT JOIN agent99_runs ar
ON ar.message_id = m.message_id
ORDER BY COALESCE(m.sent_at, m.queued_at) DESC, m.message_id DESC
""")
@@ -1907,8 +1908,8 @@ async def _load_ai_alert_card_delivery_readback_direct(
""", *args),
timeout=_AI_ALERT_CARD_DIRECT_QUERY_TIMEOUT_SECONDS,
)
# Mirror the pooled-session plan so the pressure fallback does
# not reintroduce a per-card full run-ledger scan.
# Mirror the pooled-session plan: the materialized Agent99 set
# is hard-bounded to one exact match per page candidate.
list_rows = await asyncio.wait_for(
conn.fetch(f"""
WITH candidate_messages AS MATERIALIZED (
@@ -1920,18 +1921,65 @@ async def _load_ai_alert_card_delivery_readback_direct(
m.message_id DESC
LIMIT ${limit_param} OFFSET ${offset_param}
),
candidate_refs AS MATERIALIZED (
SELECT
m.message_id,
m.project_id,
NULLIF(
COALESCE(
m.source_envelope::jsonb,
'{{}}'::jsonb
) #>> '{{agent99_dispatch_receipt,run_id}}',
''
) AS receipt_run_id,
NULLIF(
COALESCE(
m.source_envelope::jsonb,
'{{}}'::jsonb
) #>> '{{agent99_dispatch_receipt,incident_id}}',
''
) AS receipt_incident_id
FROM candidate_messages m
),
agent99_runs AS MATERIALIZED (
SELECT
ars.error_detail,
ars.state,
ars.trace_id,
ars.run_id,
ars.project_id,
ars.trigger_ref,
ars.created_at
FROM awooop_run_state ars
WHERE ars.project_id = $1
AND ars.agent_id = 'agent99_controlled_dispatch'
cr.message_id,
matched.error_detail,
matched.state,
matched.trace_id
FROM candidate_refs cr
LEFT JOIN LATERAL (
SELECT
ars.error_detail,
ars.state,
ars.trace_id,
ars.run_id,
ars.created_at
FROM awooop_run_state ars
WHERE ars.project_id = cr.project_id
AND ars.agent_id = 'agent99_controlled_dispatch'
AND (
(
cr.receipt_run_id IS NOT NULL
AND ars.run_id::text = cr.receipt_run_id
)
OR (
cr.receipt_incident_id IS NOT NULL
AND ars.trigger_ref LIKE (
'agent99:' ||
cr.receipt_incident_id ||
':%'
)
)
)
ORDER BY (
ars.run_id::text = COALESCE(
cr.receipt_run_id,
''
)
) DESC, ars.created_at DESC
LIMIT 1
) matched ON TRUE
)
SELECT
m.message_id,
@@ -1967,52 +2015,8 @@ async def _load_ai_alert_card_delivery_readback_direct(
LEFT JOIN awooop_run_state r
ON r.project_id = m.project_id
AND r.run_id = m.run_id
LEFT JOIN LATERAL (
SELECT
ars.error_detail,
ars.state,
ars.trace_id
FROM agent99_runs ars
WHERE ars.project_id = m.project_id
AND (
ars.run_id::text = COALESCE(
COALESCE(
m.source_envelope::jsonb,
'{{}}'::jsonb
) #>> '{{agent99_dispatch_receipt,run_id}}',
''
)
OR (
COALESCE(
COALESCE(
m.source_envelope::jsonb,
'{{}}'::jsonb
) #>> '{{agent99_dispatch_receipt,incident_id}}',
''
) <> ''
AND ars.trigger_ref LIKE (
'agent99:' ||
COALESCE(
COALESCE(
m.source_envelope::jsonb,
'{{}}'::jsonb
) #>> '{{agent99_dispatch_receipt,incident_id}}',
''
) || ':%'
)
)
)
ORDER BY (
ars.run_id::text = COALESCE(
COALESCE(
m.source_envelope::jsonb,
'{{}}'::jsonb
) #>> '{{agent99_dispatch_receipt,run_id}}',
''
)
) DESC, ars.created_at DESC
LIMIT 1
) ar ON TRUE
LEFT JOIN agent99_runs ar
ON ar.message_id = m.message_id
ORDER BY COALESCE(m.sent_at, m.queued_at) DESC, m.message_id DESC
""", *args, per_page, offset),
timeout=_AI_ALERT_CARD_DIRECT_QUERY_TIMEOUT_SECONDS,
@@ -2198,8 +2202,13 @@ def _ai_alert_card_delivery_source_unavailable_response(
target["status"] = "blocked_by_ai_alert_card_delivery_readback_unavailable"
target["next_action"] = "repair_awooop_ai_alert_card_delivery_readback"
backup_restore_filtered = event_type == BACKUP_RESTORE_EVENT_TYPE
summary.update({
"status": "source_unavailable_ai_controlled_repair_required",
"status": (
"blocked_backup_restore_evidence_missing_source_unavailable"
if backup_restore_filtered
else "source_unavailable_ai_controlled_repair_required"
),
"db_read_status": "unavailable",
"source_unavailable_reason": error_type,
"source_unavailable_next_action": (
@@ -2218,6 +2227,15 @@ def _ai_alert_card_delivery_source_unavailable_response(
"repair_awooop_ai_alert_card_delivery_readback_then_consume_learning_registry"
),
})
if backup_restore_filtered:
summary.update({
"backup_restore_readback_required": True,
"no_false_green_enforced": True,
"backup_restore_next_action": (
"repair_awooop_ai_alert_card_delivery_readback_then_collect_"
"backup_status_freshness_offsite_escrow_restore_and_resolution_receipts"
),
})
learning_registry.update({
"status": "source_unavailable_ai_controlled_repair_required",
"ready_target_count": 0,
@@ -2281,7 +2299,8 @@ def _ai_alert_card_delivery_summary_from_row(
0,
)
backup_restore_policy_applied = bool(
backup_restore_total
backup_restore_filtered
or backup_restore_total
or backup_runtime_gate_excluded_total
or backup_learning_writeback_excluded_total
)
@@ -2291,14 +2310,18 @@ def _ai_alert_card_delivery_summary_from_row(
status_value = "delivery_failure_observed"
elif pending_total > 0:
status_value = "delivery_pending_observed"
elif backup_restore_filtered and total > 0:
status_value = "blocked_backup_restore_evidence_or_verifier"
elif backup_restore_filtered:
status_value = (
"blocked_backup_restore_evidence_missing"
if total == 0
else "blocked_backup_restore_evidence_or_verifier"
)
elif backup_restore_policy_applied and total > 0:
status_value = "observed_effective_backup_policy_projection_applied"
learning_next_action = (
"collect_backup_status_freshness_offsite_escrow_restore_and_resolution_receipts"
if backup_restore_policy_applied and total > 0
if backup_restore_policy_applied
else "none"
if total > 0 and learning_writeback_missing_total == 0
else "ensure_ai_alert_cards_include_event_type_lane_and_delivery_receipt"