fix(agent): reconcile missing retry terminal projections
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 3m17s
CD Pipeline / build-and-deploy (push) Successful in 7m55s
CD Pipeline / post-deploy-checks (push) Successful in 2m13s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 18s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 3m17s
CD Pipeline / build-and-deploy (push) Successful in 7m55s
CD Pipeline / post-deploy-checks (push) Successful in 2m13s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 18s
This commit is contained in:
@@ -236,6 +236,21 @@ async def test_broker_reconciles_receipts_before_claiming_fresh_work(
|
||||
"_expire_stale_ansible_execution_capabilities",
|
||||
AsyncMock(return_value=0),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"backfill_missing_retry_terminal_projections_once",
|
||||
AsyncMock(
|
||||
return_value={
|
||||
"scanned": 0,
|
||||
"written": 0,
|
||||
"incident_receipt_written": 0,
|
||||
"lifecycle_written": 0,
|
||||
"verified": 0,
|
||||
"runtime_apply_executed": False,
|
||||
"error": None,
|
||||
}
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"backfill_missing_auto_repair_execution_receipts_once",
|
||||
@@ -263,6 +278,116 @@ async def test_broker_reconciles_receipts_before_claiming_fresh_work(
|
||||
retry_replayer.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_terminal_projection_backfill_writes_and_verifies_no_apply(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
row = {
|
||||
"op_id": "00000000-0000-0000-0000-000000000104",
|
||||
"replay_op_id": "00000000-0000-0000-0000-000000000105",
|
||||
"terminal_type": "no_write_replay_passed_waiting_repair_candidate",
|
||||
"telegram_receipt_acknowledged": True,
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_load_missing_retry_terminal_projection_rows",
|
||||
AsyncMock(return_value=[row]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_claim_from_apply_operation_row",
|
||||
lambda _row: (
|
||||
_claim(),
|
||||
service.AnsibleRunResult(
|
||||
returncode=1,
|
||||
stdout="",
|
||||
stderr="",
|
||||
duration_ms=0,
|
||||
),
|
||||
),
|
||||
)
|
||||
terminal_writer = AsyncMock(
|
||||
return_value={
|
||||
"automation_run_id": "00000000-0000-0000-0000-000000000102",
|
||||
"apply_op_id": row["op_id"],
|
||||
"retry_op_id": row["replay_op_id"],
|
||||
"terminal_type": row["terminal_type"],
|
||||
"repository_readback_verified": True,
|
||||
}
|
||||
)
|
||||
receipt_writer = AsyncMock(return_value=True)
|
||||
lifecycle_writer = AsyncMock(return_value=True)
|
||||
verifier = AsyncMock(return_value=True)
|
||||
monkeypatch.setattr(
|
||||
service, "_record_incident_terminal_disposition", terminal_writer
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_append_runtime_stage_receipts_to_apply", receipt_writer
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_append_alert_lifecycle_receipt", lifecycle_writer
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_verify_retry_terminal_projection_readback", verifier
|
||||
)
|
||||
|
||||
result = await service.backfill_missing_retry_terminal_projections_once()
|
||||
|
||||
assert result == {
|
||||
"scanned": 1,
|
||||
"written": 1,
|
||||
"incident_receipt_written": 1,
|
||||
"lifecycle_written": 1,
|
||||
"verified": 1,
|
||||
"runtime_apply_executed": False,
|
||||
"error": None,
|
||||
}
|
||||
terminal_writer.assert_awaited_once()
|
||||
assert terminal_writer.await_args.kwargs["success_terminal"] is False
|
||||
receipt = receipt_writer.await_args.kwargs["receipts"][0]
|
||||
assert receipt["stage_id"] == "incident_closure"
|
||||
assert receipt["derived_from_durable_chain"] is True
|
||||
assert lifecycle_writer.await_args.args[1] == "EXECUTION_COMPLETED"
|
||||
assert lifecycle_writer.await_args.kwargs["success"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_broker_prioritizes_retry_terminal_projection_backfill(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_expire_stale_ansible_execution_capabilities",
|
||||
AsyncMock(return_value=0),
|
||||
)
|
||||
receipt_backfill = AsyncMock(
|
||||
return_value={
|
||||
"scanned": 1,
|
||||
"written": 0,
|
||||
"incident_closure_written": 1,
|
||||
"telegram_receipt_acknowledged": 0,
|
||||
"retry_terminal_projection_scanned": 1,
|
||||
"retry_terminal_projection_written": 1,
|
||||
"retry_terminal_projection_verified": 1,
|
||||
"retry_terminal_projection_runtime_apply_executed": False,
|
||||
"error": None,
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"backfill_missing_auto_repair_execution_receipts_once",
|
||||
receipt_backfill,
|
||||
)
|
||||
|
||||
result = await service.run_pending_check_modes_once(limit=1)
|
||||
|
||||
assert result["claimed"] == 0
|
||||
assert result["retry_terminal_projection_verified"] == 1
|
||||
assert result["retry_terminal_projection_runtime_apply_executed"] is False
|
||||
assert result["repair_receipt_backfill_priority_tick"] is True
|
||||
receipt_backfill.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_signal_worker_retry_preflight_is_query_only(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
||||
Reference in New Issue
Block a user