From 7a8ca6eab0ca493fc7c6821dddd4a0a3db6c2172 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 14:21:48 +0800 Subject: [PATCH] fix(agent): verify retry incident lifecycle --- .../ai_agent_autonomous_runtime_control.py | 73 +++++++++++++++---- .../awoooi_priority_work_order_readback.py | 7 ++ ...est_ai_agent_autonomous_runtime_control.py | 17 ++++- ...awoooi_priority_work_order_readback_api.py | 4 + 4 files changed, 86 insertions(+), 15 deletions(-) diff --git a/apps/api/src/services/ai_agent_autonomous_runtime_control.py b/apps/api/src/services/ai_agent_autonomous_runtime_control.py index 87ff2b4cc..e10270b50 100644 --- a/apps/api/src/services/ai_agent_autonomous_runtime_control.py +++ b/apps/api/src/services/ai_agent_autonomous_runtime_control.py @@ -4938,6 +4938,9 @@ def _build_autonomous_retry_rollback_terminal_readback( backoff_min_seconds = _int_value(row.get("retry_error_backoff_min_seconds")) backoff_max_seconds = _int_value(row.get("retry_error_backoff_max_seconds")) retry_idempotency_key = str(row.get("retry_idempotency_key") or "") + retry_receipt_incident_id = str( + row.get("retry_receipt_incident_id") or "" + ) terminal_type = str( row.get("terminal_type") or row.get("replay_terminal_disposition") or "" ) @@ -4950,7 +4953,7 @@ def _build_autonomous_retry_rollback_terminal_readback( "apply_automation_run_id", "replay_automation_run_id", "retry_receipt_automation_run_id", - "incident_terminal_automation_run_id", + "incident_lifecycle_automation_run_id", "learning_automation_run_id", "km_automation_run_id", "telegram_automation_run_id", @@ -4961,10 +4964,23 @@ def _build_autonomous_retry_rollback_terminal_readback( and all(retry_run_ids) and set(retry_run_ids) == {automation_run_id} ) + incident_lifecycle_recorded = bool( + row.get("incident_lifecycle_event_id") + and row.get("incident_lifecycle_incident_id") == incident_id + and row.get("incident_lifecycle_event_type") == "EXECUTION_COMPLETED" + and str(row.get("incident_lifecycle_success")).lower() == "false" + and row.get("incident_lifecycle_apply_op_id") == failed_apply_op_id + and row.get("incident_lifecycle_automation_run_id") + == automation_run_id + ) explicit_no_write_terminal = bool( terminal_type.startswith("no_write_replay_") and _bool_value(row.get("verified_no_write_terminal")) - and _bool_value(row.get("incident_no_write_terminal")) + and retry_receipt_incident_id == incident_id + and row.get("retry_receipt_failed_apply_op_id") + == failed_apply_op_id + and row.get("retry_receipt_retry_op_id") == retry_op_id + and incident_lifecycle_recorded ) verified_rollback_terminal = bool( _bool_value(row.get("rollback_performed")) @@ -5016,17 +5032,8 @@ def _build_autonomous_retry_rollback_terminal_readback( ), "incident_terminal_repository_readback": bool( incident_id - and row.get("incident_receipt_schema_version") - == AI_AUTOMATION_STAGE_RECEIPT_SCHEMA_VERSION - and row.get("incident_terminal_schema_version") - == "ansible_incident_terminal_disposition_v1" - and _bool_value(row.get("incident_receipt_durable")) - and _bool_value(row.get("incident_repository_write_acknowledged")) - and _bool_value(row.get("incident_repository_readback_verified")) - and _bool_value(row.get("incident_durable_write_acknowledged")) - and row.get("incident_terminal_apply_op_id") == failed_apply_op_id - and row.get("incident_terminal_retry_op_id") == retry_op_id - and row.get("incident_terminal_type") == terminal_type + and retry_receipt_incident_id == incident_id + and incident_lifecycle_recorded ), "same_run_correlation": same_run_correlation, "km_writeback_same_run": bool( @@ -5077,6 +5084,11 @@ def _build_autonomous_retry_rollback_terminal_readback( if row.get("telegram_message_id") else "" ), + ( + f"alert-operation:{row.get('incident_lifecycle_event_id')}:execution_completed" + if row.get("incident_lifecycle_event_id") + else "" + ), ] return { "schema_version": "awoooi_autonomous_retry_rollback_terminal_v1", @@ -5087,6 +5099,9 @@ def _build_autonomous_retry_rollback_terminal_readback( "failed_apply_op_id": failed_apply_op_id or None, "retry_op_id": retry_op_id or None, "incident_id": incident_id or None, + "incident_lifecycle_event_id": ( + row.get("incident_lifecycle_event_id") or None + ), "catalog_id": row.get("catalog_id"), "terminal_type": terminal_type or None, "incident_status": row.get("incident_status"), @@ -7629,6 +7644,12 @@ _RUNTIME_RETRY_TERMINAL_LATEST_SQL = """ retry_receipt.receipt ->> 'stage_id' AS retry_receipt_stage_id, retry_receipt.receipt ->> 'automation_run_id' AS retry_receipt_automation_run_id, + retry_receipt.receipt ->> 'incident_id' + AS retry_receipt_incident_id, + retry_receipt.receipt #>> '{detail,failed_apply_op_id}' + AS retry_receipt_failed_apply_op_id, + retry_receipt.receipt #>> '{detail,retry_check_mode_op_id}' + AS retry_receipt_retry_op_id, retry_receipt.receipt #>> '{detail,schema_version}' AS retry_terminal_schema_version, retry_receipt.receipt #>> '{detail,terminal_type}' AS terminal_type, @@ -7705,6 +7726,15 @@ _RUNTIME_RETRY_TERMINAL_LATEST_SQL = """ '{automation_terminal,no_write_terminal}' ) AS incident_no_write_terminal, + incident_lifecycle.id::text AS incident_lifecycle_event_id, + incident_lifecycle.incident_id AS incident_lifecycle_incident_id, + incident_lifecycle.event_type::text + AS incident_lifecycle_event_type, + incident_lifecycle.success AS incident_lifecycle_success, + incident_lifecycle.context ->> 'automation_run_id' + AS incident_lifecycle_automation_run_id, + incident_lifecycle.context ->> 'apply_op_id' + AS incident_lifecycle_apply_op_id, learning.op_id::text AS learning_operation_id, learning.status AS learning_status, learning.input ->> 'automation_run_id' AS learning_automation_run_id, @@ -7766,6 +7796,23 @@ _RUNTIME_RETRY_TERMINAL_LATEST_SQL = """ apply.input ->> 'incident_id' ) AND incident.project_id = :project_id + LEFT JOIN LATERAL ( + SELECT lifecycle.* + FROM alert_operation_log lifecycle + WHERE lifecycle.incident_id = coalesce( + apply.incident_id::text, + apply.input ->> 'incident_id' + ) + AND lifecycle.event_type::text = 'EXECUTION_COMPLETED' + AND lifecycle.success IS FALSE + AND lifecycle.context ->> 'apply_op_id' = apply.op_id::text + AND lifecycle.context ->> 'automation_run_id' = coalesce( + replay.input ->> 'automation_run_id', + apply.input ->> 'automation_run_id' + ) + ORDER BY lifecycle.created_at DESC + LIMIT 1 + ) incident_lifecycle ON TRUE LEFT JOIN LATERAL ( SELECT learning_row.* FROM automation_operation_log learning_row diff --git a/apps/api/src/services/awoooi_priority_work_order_readback.py b/apps/api/src/services/awoooi_priority_work_order_readback.py index ff1f3682e..9d9abcd27 100644 --- a/apps/api/src/services/awoooi_priority_work_order_readback.py +++ b/apps/api/src/services/awoooi_priority_work_order_readback.py @@ -7889,6 +7889,12 @@ def _apply_ai_automation_program_ledger(payload: dict[str, Any]) -> None: "incident_id": str( summary.get("ai_agent_retry_rollback_incident_id") or "" ), + "incident_lifecycle_event_id": str( + summary.get( + "ai_agent_retry_rollback_incident_lifecycle_event_id" + ) + or "" + ), "catalog_id": str( summary.get("ai_agent_retry_rollback_catalog_id") or "" ), @@ -8600,6 +8606,7 @@ def apply_ai_automation_live_closure_readbacks( "failed_apply_op_id", "retry_op_id", "incident_id", + "incident_lifecycle_event_id", "catalog_id", "terminal_type", "incident_status", diff --git a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py index d88bc6e01..be4ddb403 100644 --- a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py +++ b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py @@ -46,6 +46,9 @@ def _closed_retry_terminal_row() -> dict[str, object]: "retry_receipt_schema_version": "ai_automation_stage_receipt_v1", "retry_receipt_stage_id": "retry_or_rollback", "retry_receipt_automation_run_id": run_id, + "retry_receipt_incident_id": "INC-RETRY-1", + "retry_receipt_failed_apply_op_id": apply_op_id, + "retry_receipt_retry_op_id": retry_op_id, "retry_terminal_schema_version": "ansible_controlled_retry_terminal_v2", "terminal_type": "no_write_replay_passed_waiting_repair_candidate", "retry_attempt": "1", @@ -77,6 +80,12 @@ def _closed_retry_terminal_row() -> dict[str, object]: "no_write_replay_passed_waiting_repair_candidate" ), "incident_no_write_terminal": "true", + "incident_lifecycle_event_id": "alert-lifecycle-retry-1", + "incident_lifecycle_incident_id": "INC-RETRY-1", + "incident_lifecycle_event_type": "EXECUTION_COMPLETED", + "incident_lifecycle_success": "false", + "incident_lifecycle_automation_run_id": run_id, + "incident_lifecycle_apply_op_id": apply_op_id, "learning_operation_id": "learning-retry-1", "learning_status": "success", "learning_automation_run_id": run_id, @@ -3444,6 +3453,9 @@ def test_retry_rollback_terminal_closes_from_same_run_public_safe_receipts(): assert terminal["status"] == "verified_no_write_terminal" assert terminal["closed"] is True assert terminal["automation_run_id"] == "retry-run-1" + assert terminal["incident_lifecycle_event_id"] == ( + "alert-lifecycle-retry-1" + ) assert terminal["retry_attempt"] == 1 assert terminal["max_retry_attempts"] == 1 assert terminal["retry_backoff_seconds"] == {"min": 15, "max": 30} @@ -3509,9 +3521,9 @@ def test_retry_rollback_terminal_rejects_unbounded_retry_or_unverified_rollback( assert terminal["operation_boundaries"]["verified_rollback_terminal"] is False -def test_retry_rollback_terminal_requires_durable_incident_stage_receipt(): +def test_retry_rollback_terminal_requires_immutable_incident_lifecycle(): row = _closed_retry_terminal_row() - row["incident_receipt_durable"] = "false" + row["incident_lifecycle_event_id"] = None terminal = build_runtime_receipt_readback_from_rows( retry_terminal_latest_rows=[row] @@ -3525,3 +3537,4 @@ def test_retry_rollback_terminal_requires_durable_incident_stage_receipt(): assert "incident_terminal_repository_readback_not_verified" in ( terminal["active_blockers"] ) + assert terminal["operation_boundaries"]["explicit_no_write_terminal"] is False diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index abfa259d2..438e01f89 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -274,6 +274,7 @@ def _autonomous_runtime_control_retry_rollback_closed() -> dict: "failed_apply_op_id": "failed-apply-p0-005", "retry_op_id": "retry-op-p0-005", "incident_id": "INC-P0-005", + "incident_lifecycle_event_id": "alert-lifecycle-p0-005", "catalog_id": "ansible:awoooi-auto-repair-canary", "terminal_type": "no_write_replay_passed_waiting_repair_candidate", "incident_status": "MITIGATING", @@ -2371,6 +2372,9 @@ def test_ai_automation_retry_rollback_terminal_advances_order() -> None: ) assert retry["completion_evidence"]["retry_attempt"] == 1 assert retry["completion_evidence"]["max_retry_attempts"] == 1 + assert retry["completion_evidence"]["incident_lifecycle_event_id"] == ( + "alert-lifecycle-p0-005" + ) assert retry["completion_evidence"]["retry_backoff_seconds"] == { "min": 15, "max": 30,