wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

feat(agent): persist failed retry terminal receipt

Browse Source
This commit is contained in:
ogt
2026-07-10 19:53:55 +08:00
parent 2a0466703b
commit 78eade177d
4 changed files with 99 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py
View File

@@ -122,6 +122,7 @@ async def enqueue_missing_ansible_candidates_once(
"failed_apply_retry_replayed": 0,
"failed_apply_retry_check_mode_passed": 0,
"failed_apply_retry_check_mode_failed": 0,
"failed_apply_retry_stage_receipt_written": 0,
"error": None,
}
@@ -141,6 +142,7 @@ async def enqueue_missing_ansible_candidates_once(
"failed_apply_retry_replayed": 0,
"failed_apply_retry_check_mode_passed": 0,
"failed_apply_retry_check_mode_failed": 0,
"failed_apply_retry_stage_receipt_written": 0,
"error": None,
}
@@ -194,6 +196,9 @@ async def enqueue_missing_ansible_candidates_once(
stats["failed_apply_retry_check_mode_failed"] = int(
retry_stats.get("check_mode_failed") or 0
)
stats["failed_apply_retry_stage_receipt_written"] = int(
retry_stats.get("runtime_stage_receipt_written") or 0
)
if retry_stats.get("error") and not stats["error"]:
stats["error"] = retry_stats["error"]
except Exception as exc:

84
apps/api/src/services/awooop_ansible_check_mode_service.py
View File

@@ -911,6 +911,78 @@ async def _record_runtime_stage_receipts(
return False
async def _record_retry_runtime_stage_receipt(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
replay_op_id: str,
project_id: str,
) -> bool:
"""Append the verified no-write retry terminal to the original apply run."""
receipt = _runtime_stage_receipt(
claim,
stage_id="retry_or_rollback",
evidence_ref=f"automation_operation_log:{replay_op_id}:dry_run_result",
detail={
"failed_apply_op_id": apply_op_id,
"retry_check_mode_op_id": replay_op_id,
"terminal_type": (
"no_write_replay_passed_waiting_repair_candidate"
if result.returncode == 0
else "no_write_replay_failed_waiting_playbook_or_transport_repair"
),
"check_mode_replay_performed": True,
"check_mode_replay_returncode": result.returncode,
"runtime_apply_executed": False,
"rollback_performed": False,
},
)
try:
async with get_db_context(project_id) as db:
updated = await db.execute(
text("""
UPDATE automation_operation_log apply
SET input = jsonb_set(
coalesce(apply.input, '{}'::jsonb),
'{runtime_stage_receipts}',
coalesce(
(
SELECT jsonb_agg(existing_receipt)
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) AS existing_receipt
WHERE existing_receipt ->> 'stage_id'
<> 'retry_or_rollback'
),
'[]'::jsonb
) || jsonb_build_array(CAST(:receipt AS jsonb)),
true
)
WHERE apply.op_id = CAST(:apply_op_id AS uuid)
AND apply.operation_type = 'ansible_apply_executed'
"""),
{
"receipt": json.dumps(receipt, ensure_ascii=False),
"apply_op_id": apply_op_id,
},
)
return bool(updated.rowcount)
except Exception as exc:
logger.warning(
"ansible_retry_runtime_stage_receipt_write_failed",
automation_run_id=_automation_run_id_for_claim(claim),
apply_op_id=apply_op_id,
replay_op_id=replay_op_id,
error=str(exc),
)
return False
async def _record_learning_writeback_receipt(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
@@ -1388,6 +1460,7 @@ async def run_failed_apply_check_mode_replay_once(
"check_mode_passed": 0,
"check_mode_failed": 0,
"runtime_apply_executed": 0,
"runtime_stage_receipt_written": 0,
"blockers": [],
"error": None,
}
@@ -1585,6 +1658,14 @@ async def run_failed_apply_check_mode_replay_once(
stats[
"check_mode_passed" if replay_result.returncode == 0 else "check_mode_failed"
] = 1
if await _record_retry_runtime_stage_receipt(
claim,
replay_result,
apply_op_id=apply_op_id,
replay_op_id=replay_op_id,
project_id=project_id,
):
stats["runtime_stage_receipt_written"] = 1
logger.info(
"ansible_failed_apply_check_mode_replay_completed",
automation_run_id=automation_run_id,
@@ -1592,6 +1673,9 @@ async def run_failed_apply_check_mode_replay_once(
replay_op_id=str(replay_op_id),
returncode=replay_result.returncode,
runtime_apply_executed=False,
runtime_stage_receipt_written=bool(
stats["runtime_stage_receipt_written"]
),
)
except Exception as exc:
stats["error"] = f"{type(exc).__name__}: {exc}"[:500]

2
apps/api/tests/test_awooop_ansible_candidate_backfill_job.py
View File

@@ -72,6 +72,7 @@ async def test_backfill_enqueues_catalog_matched_incident(monkeypatch: pytest.Mo
"replayed": 1,
"check_mode_passed": 1,
"check_mode_failed": 0,
"runtime_stage_receipt_written": 1,
"error": None,
}
@@ -93,6 +94,7 @@ async def test_backfill_enqueues_catalog_matched_incident(monkeypatch: pytest.Mo
assert result["failed_apply_retry_replayed"] == 1
assert result["failed_apply_retry_check_mode_passed"] == 1
assert result["failed_apply_retry_check_mode_failed"] == 0
assert result["failed_apply_retry_stage_receipt_written"] == 1
assert recorded[0]["decision_path"] == "repair_candidate_controlled_queue"
assert recorded[0]["incident"]["incident_id"] == "INC-20260627-NODE110"

8
apps/api/tests/test_awooop_truth_chain_service.py
View File

@@ -23,6 +23,7 @@ from src.services.awooop_ansible_check_mode_service import (
_post_apply_verification_result,
_record_auto_repair_execution_receipt,
_record_learning_writeback_receipt,
_record_retry_runtime_stage_receipt,
_record_runtime_stage_receipts,
_run_ansible_command,
_send_controlled_apply_telegram_receipt,
@@ -1793,9 +1794,16 @@ def test_failed_apply_retry_replay_is_no_write_and_idempotent() -> None:
assert "controlled_apply_allowed=False" in source
assert '"runtime_apply_executed": False' in source
assert "retry_requires_repair_and_new_apply_gate" in source
assert "_record_retry_runtime_stage_receipt" in source
assert "NOT EXISTS" in source
assert "run_controlled_apply_for_claim" not in source
receipt_source = inspect.getsource(_record_retry_runtime_stage_receipt)
assert "_runtime_stage_receipt" in receipt_source
assert 'stage_id="retry_or_rollback"' in receipt_source
assert "jsonb_array_elements" in receipt_source
assert "runtime_apply_executed" in receipt_source
def test_ansible_check_and_apply_rows_persist_canonical_automation_run_id() -> None:
finalize_source = inspect.getsource(finalize_check_mode_claim)