feat(agent): persist failed retry terminal receipt

apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py

@@ -122,6 +122,7 @@ async def enqueue_missing_ansible_candidates_once(
             "failed_apply_retry_replayed": 0,
             "failed_apply_retry_check_mode_passed": 0,
             "failed_apply_retry_check_mode_failed": 0,
+            "failed_apply_retry_stage_receipt_written": 0,
             "error": None,
         }
 
@@ -141,6 +142,7 @@ async def enqueue_missing_ansible_candidates_once(
         "failed_apply_retry_replayed": 0,
         "failed_apply_retry_check_mode_passed": 0,
         "failed_apply_retry_check_mode_failed": 0,
+        "failed_apply_retry_stage_receipt_written": 0,
         "error": None,
     }
 
@@ -194,6 +196,9 @@ async def enqueue_missing_ansible_candidates_once(
         stats["failed_apply_retry_check_mode_failed"] = int(
             retry_stats.get("check_mode_failed") or 0
         )
+        stats["failed_apply_retry_stage_receipt_written"] = int(
+            retry_stats.get("runtime_stage_receipt_written") or 0
+        )
         if retry_stats.get("error") and not stats["error"]:
             stats["error"] = retry_stats["error"]
     except Exception as exc:

apps/api/src/services/awooop_ansible_check_mode_service.py

@@ -911,6 +911,78 @@ async def _record_runtime_stage_receipts(
         return False
 
 
+async def _record_retry_runtime_stage_receipt(
+    claim: AnsibleCheckModeClaim,
+    result: AnsibleRunResult,
+    *,
+    apply_op_id: str,
+    replay_op_id: str,
+    project_id: str,
+) -> bool:
+    """Append the verified no-write retry terminal to the original apply run."""
+
+    receipt = _runtime_stage_receipt(
+        claim,
+        stage_id="retry_or_rollback",
+        evidence_ref=f"automation_operation_log:{replay_op_id}:dry_run_result",
+        detail={
+            "failed_apply_op_id": apply_op_id,
+            "retry_check_mode_op_id": replay_op_id,
+            "terminal_type": (
+                "no_write_replay_passed_waiting_repair_candidate"
+                if result.returncode == 0
+                else "no_write_replay_failed_waiting_playbook_or_transport_repair"
+            ),
+            "check_mode_replay_performed": True,
+            "check_mode_replay_returncode": result.returncode,
+            "runtime_apply_executed": False,
+            "rollback_performed": False,
+        },
+    )
+    try:
+        async with get_db_context(project_id) as db:
+            updated = await db.execute(
+                text("""
+                    UPDATE automation_operation_log apply
+                    SET input = jsonb_set(
+                        coalesce(apply.input, '{}'::jsonb),
+                        '{runtime_stage_receipts}',
+                        coalesce(
+                            (
+                                SELECT jsonb_agg(existing_receipt)
+                                FROM jsonb_array_elements(
+                                    coalesce(
+                                        apply.input -> 'runtime_stage_receipts',
+                                        '[]'::jsonb
+                                    )
+                                ) AS existing_receipt
+                                WHERE existing_receipt ->> 'stage_id'
+                                    <> 'retry_or_rollback'
+                            ),
+                            '[]'::jsonb
+                        ) || jsonb_build_array(CAST(:receipt AS jsonb)),
+                        true
+                    )
+                    WHERE apply.op_id = CAST(:apply_op_id AS uuid)
+                      AND apply.operation_type = 'ansible_apply_executed'
+                """),
+                {
+                    "receipt": json.dumps(receipt, ensure_ascii=False),
+                    "apply_op_id": apply_op_id,
+                },
+            )
+            return bool(updated.rowcount)
+    except Exception as exc:
+        logger.warning(
+            "ansible_retry_runtime_stage_receipt_write_failed",
+            automation_run_id=_automation_run_id_for_claim(claim),
+            apply_op_id=apply_op_id,
+            replay_op_id=replay_op_id,
+            error=str(exc),
+        )
+        return False
+
+
 async def _record_learning_writeback_receipt(
     claim: AnsibleCheckModeClaim,
     result: AnsibleRunResult,
@@ -1388,6 +1460,7 @@ async def run_failed_apply_check_mode_replay_once(
         "check_mode_passed": 0,
         "check_mode_failed": 0,
         "runtime_apply_executed": 0,
+        "runtime_stage_receipt_written": 0,
         "blockers": [],
         "error": None,
     }
@@ -1585,6 +1658,14 @@ async def run_failed_apply_check_mode_replay_once(
         stats[
             "check_mode_passed" if replay_result.returncode == 0 else "check_mode_failed"
         ] = 1
+        if await _record_retry_runtime_stage_receipt(
+            claim,
+            replay_result,
+            apply_op_id=apply_op_id,
+            replay_op_id=replay_op_id,
+            project_id=project_id,
+        ):
+            stats["runtime_stage_receipt_written"] = 1
         logger.info(
             "ansible_failed_apply_check_mode_replay_completed",
             automation_run_id=automation_run_id,
@@ -1592,6 +1673,9 @@ async def run_failed_apply_check_mode_replay_once(
             replay_op_id=str(replay_op_id),
             returncode=replay_result.returncode,
             runtime_apply_executed=False,
+            runtime_stage_receipt_written=bool(
+                stats["runtime_stage_receipt_written"]
+            ),
         )
     except Exception as exc:
         stats["error"] = f"{type(exc).__name__}: {exc}"[:500]