From 78eade177db3cbbe1a6434c67a0a1827db689b74 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 19:53:55 +0800 Subject: [PATCH] feat(agent): persist failed retry terminal receipt --- .../awooop_ansible_candidate_backfill_job.py | 5 ++ .../awooop_ansible_check_mode_service.py | 84 +++++++++++++++++++ ...t_awooop_ansible_candidate_backfill_job.py | 2 + .../tests/test_awooop_truth_chain_service.py | 8 ++ 4 files changed, 99 insertions(+) diff --git a/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py b/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py index c49ea52a4..a6cf3ba87 100644 --- a/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py +++ b/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py @@ -122,6 +122,7 @@ async def enqueue_missing_ansible_candidates_once( "failed_apply_retry_replayed": 0, "failed_apply_retry_check_mode_passed": 0, "failed_apply_retry_check_mode_failed": 0, + "failed_apply_retry_stage_receipt_written": 0, "error": None, } @@ -141,6 +142,7 @@ async def enqueue_missing_ansible_candidates_once( "failed_apply_retry_replayed": 0, "failed_apply_retry_check_mode_passed": 0, "failed_apply_retry_check_mode_failed": 0, + "failed_apply_retry_stage_receipt_written": 0, "error": None, } @@ -194,6 +196,9 @@ async def enqueue_missing_ansible_candidates_once( stats["failed_apply_retry_check_mode_failed"] = int( retry_stats.get("check_mode_failed") or 0 ) + stats["failed_apply_retry_stage_receipt_written"] = int( + retry_stats.get("runtime_stage_receipt_written") or 0 + ) if retry_stats.get("error") and not stats["error"]: stats["error"] = retry_stats["error"] except Exception as exc: diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index f0462e495..414a84ee6 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -911,6 +911,78 @@ async def _record_runtime_stage_receipts( return False +async def _record_retry_runtime_stage_receipt( + claim: AnsibleCheckModeClaim, + result: AnsibleRunResult, + *, + apply_op_id: str, + replay_op_id: str, + project_id: str, +) -> bool: + """Append the verified no-write retry terminal to the original apply run.""" + + receipt = _runtime_stage_receipt( + claim, + stage_id="retry_or_rollback", + evidence_ref=f"automation_operation_log:{replay_op_id}:dry_run_result", + detail={ + "failed_apply_op_id": apply_op_id, + "retry_check_mode_op_id": replay_op_id, + "terminal_type": ( + "no_write_replay_passed_waiting_repair_candidate" + if result.returncode == 0 + else "no_write_replay_failed_waiting_playbook_or_transport_repair" + ), + "check_mode_replay_performed": True, + "check_mode_replay_returncode": result.returncode, + "runtime_apply_executed": False, + "rollback_performed": False, + }, + ) + try: + async with get_db_context(project_id) as db: + updated = await db.execute( + text(""" + UPDATE automation_operation_log apply + SET input = jsonb_set( + coalesce(apply.input, '{}'::jsonb), + '{runtime_stage_receipts}', + coalesce( + ( + SELECT jsonb_agg(existing_receipt) + FROM jsonb_array_elements( + coalesce( + apply.input -> 'runtime_stage_receipts', + '[]'::jsonb + ) + ) AS existing_receipt + WHERE existing_receipt ->> 'stage_id' + <> 'retry_or_rollback' + ), + '[]'::jsonb + ) || jsonb_build_array(CAST(:receipt AS jsonb)), + true + ) + WHERE apply.op_id = CAST(:apply_op_id AS uuid) + AND apply.operation_type = 'ansible_apply_executed' + """), + { + "receipt": json.dumps(receipt, ensure_ascii=False), + "apply_op_id": apply_op_id, + }, + ) + return bool(updated.rowcount) + except Exception as exc: + logger.warning( + "ansible_retry_runtime_stage_receipt_write_failed", + automation_run_id=_automation_run_id_for_claim(claim), + apply_op_id=apply_op_id, + replay_op_id=replay_op_id, + error=str(exc), + ) + return False + + async def _record_learning_writeback_receipt( claim: AnsibleCheckModeClaim, result: AnsibleRunResult, @@ -1388,6 +1460,7 @@ async def run_failed_apply_check_mode_replay_once( "check_mode_passed": 0, "check_mode_failed": 0, "runtime_apply_executed": 0, + "runtime_stage_receipt_written": 0, "blockers": [], "error": None, } @@ -1585,6 +1658,14 @@ async def run_failed_apply_check_mode_replay_once( stats[ "check_mode_passed" if replay_result.returncode == 0 else "check_mode_failed" ] = 1 + if await _record_retry_runtime_stage_receipt( + claim, + replay_result, + apply_op_id=apply_op_id, + replay_op_id=replay_op_id, + project_id=project_id, + ): + stats["runtime_stage_receipt_written"] = 1 logger.info( "ansible_failed_apply_check_mode_replay_completed", automation_run_id=automation_run_id, @@ -1592,6 +1673,9 @@ async def run_failed_apply_check_mode_replay_once( replay_op_id=str(replay_op_id), returncode=replay_result.returncode, runtime_apply_executed=False, + runtime_stage_receipt_written=bool( + stats["runtime_stage_receipt_written"] + ), ) except Exception as exc: stats["error"] = f"{type(exc).__name__}: {exc}"[:500] diff --git a/apps/api/tests/test_awooop_ansible_candidate_backfill_job.py b/apps/api/tests/test_awooop_ansible_candidate_backfill_job.py index fdcf8455b..be3bfb2af 100644 --- a/apps/api/tests/test_awooop_ansible_candidate_backfill_job.py +++ b/apps/api/tests/test_awooop_ansible_candidate_backfill_job.py @@ -72,6 +72,7 @@ async def test_backfill_enqueues_catalog_matched_incident(monkeypatch: pytest.Mo "replayed": 1, "check_mode_passed": 1, "check_mode_failed": 0, + "runtime_stage_receipt_written": 1, "error": None, } @@ -93,6 +94,7 @@ async def test_backfill_enqueues_catalog_matched_incident(monkeypatch: pytest.Mo assert result["failed_apply_retry_replayed"] == 1 assert result["failed_apply_retry_check_mode_passed"] == 1 assert result["failed_apply_retry_check_mode_failed"] == 0 + assert result["failed_apply_retry_stage_receipt_written"] == 1 assert recorded[0]["decision_path"] == "repair_candidate_controlled_queue" assert recorded[0]["incident"]["incident_id"] == "INC-20260627-NODE110" diff --git a/apps/api/tests/test_awooop_truth_chain_service.py b/apps/api/tests/test_awooop_truth_chain_service.py index 9b1e6306b..1e3859945 100644 --- a/apps/api/tests/test_awooop_truth_chain_service.py +++ b/apps/api/tests/test_awooop_truth_chain_service.py @@ -23,6 +23,7 @@ from src.services.awooop_ansible_check_mode_service import ( _post_apply_verification_result, _record_auto_repair_execution_receipt, _record_learning_writeback_receipt, + _record_retry_runtime_stage_receipt, _record_runtime_stage_receipts, _run_ansible_command, _send_controlled_apply_telegram_receipt, @@ -1793,9 +1794,16 @@ def test_failed_apply_retry_replay_is_no_write_and_idempotent() -> None: assert "controlled_apply_allowed=False" in source assert '"runtime_apply_executed": False' in source assert "retry_requires_repair_and_new_apply_gate" in source + assert "_record_retry_runtime_stage_receipt" in source assert "NOT EXISTS" in source assert "run_controlled_apply_for_claim" not in source + receipt_source = inspect.getsource(_record_retry_runtime_stage_receipt) + assert "_runtime_stage_receipt" in receipt_source + assert 'stage_id="retry_or_rollback"' in receipt_source + assert "jsonb_array_elements" in receipt_source + assert "runtime_apply_executed" in receipt_source + def test_ansible_check_and_apply_rows_persist_canonical_automation_run_id() -> None: finalize_source = inspect.getsource(finalize_check_mode_claim)