fix(agent): reconcile missing retry terminal projections
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 3m17s
CD Pipeline / build-and-deploy (push) Successful in 7m55s
CD Pipeline / post-deploy-checks (push) Successful in 2m13s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 18s

This commit is contained in:
ogt
2026-07-11 15:01:22 +08:00
parent 18c7b7f651
commit 86bba878e4
2 changed files with 490 additions and 0 deletions

View File

@@ -236,6 +236,21 @@ async def test_broker_reconciles_receipts_before_claiming_fresh_work(
"_expire_stale_ansible_execution_capabilities",
AsyncMock(return_value=0),
)
monkeypatch.setattr(
service,
"backfill_missing_retry_terminal_projections_once",
AsyncMock(
return_value={
"scanned": 0,
"written": 0,
"incident_receipt_written": 0,
"lifecycle_written": 0,
"verified": 0,
"runtime_apply_executed": False,
"error": None,
}
),
)
monkeypatch.setattr(
service,
"backfill_missing_auto_repair_execution_receipts_once",
@@ -263,6 +278,116 @@ async def test_broker_reconciles_receipts_before_claiming_fresh_work(
retry_replayer.assert_not_awaited()
@pytest.mark.asyncio
async def test_retry_terminal_projection_backfill_writes_and_verifies_no_apply(
monkeypatch: pytest.MonkeyPatch,
) -> None:
row = {
"op_id": "00000000-0000-0000-0000-000000000104",
"replay_op_id": "00000000-0000-0000-0000-000000000105",
"terminal_type": "no_write_replay_passed_waiting_repair_candidate",
"telegram_receipt_acknowledged": True,
}
monkeypatch.setattr(
service,
"_load_missing_retry_terminal_projection_rows",
AsyncMock(return_value=[row]),
)
monkeypatch.setattr(
service,
"_claim_from_apply_operation_row",
lambda _row: (
_claim(),
service.AnsibleRunResult(
returncode=1,
stdout="",
stderr="",
duration_ms=0,
),
),
)
terminal_writer = AsyncMock(
return_value={
"automation_run_id": "00000000-0000-0000-0000-000000000102",
"apply_op_id": row["op_id"],
"retry_op_id": row["replay_op_id"],
"terminal_type": row["terminal_type"],
"repository_readback_verified": True,
}
)
receipt_writer = AsyncMock(return_value=True)
lifecycle_writer = AsyncMock(return_value=True)
verifier = AsyncMock(return_value=True)
monkeypatch.setattr(
service, "_record_incident_terminal_disposition", terminal_writer
)
monkeypatch.setattr(
service, "_append_runtime_stage_receipts_to_apply", receipt_writer
)
monkeypatch.setattr(
service, "_append_alert_lifecycle_receipt", lifecycle_writer
)
monkeypatch.setattr(
service, "_verify_retry_terminal_projection_readback", verifier
)
result = await service.backfill_missing_retry_terminal_projections_once()
assert result == {
"scanned": 1,
"written": 1,
"incident_receipt_written": 1,
"lifecycle_written": 1,
"verified": 1,
"runtime_apply_executed": False,
"error": None,
}
terminal_writer.assert_awaited_once()
assert terminal_writer.await_args.kwargs["success_terminal"] is False
receipt = receipt_writer.await_args.kwargs["receipts"][0]
assert receipt["stage_id"] == "incident_closure"
assert receipt["derived_from_durable_chain"] is True
assert lifecycle_writer.await_args.args[1] == "EXECUTION_COMPLETED"
assert lifecycle_writer.await_args.kwargs["success"] is False
@pytest.mark.asyncio
async def test_broker_prioritizes_retry_terminal_projection_backfill(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
service,
"_expire_stale_ansible_execution_capabilities",
AsyncMock(return_value=0),
)
receipt_backfill = AsyncMock(
return_value={
"scanned": 1,
"written": 0,
"incident_closure_written": 1,
"telegram_receipt_acknowledged": 0,
"retry_terminal_projection_scanned": 1,
"retry_terminal_projection_written": 1,
"retry_terminal_projection_verified": 1,
"retry_terminal_projection_runtime_apply_executed": False,
"error": None,
}
)
monkeypatch.setattr(
service,
"backfill_missing_auto_repair_execution_receipts_once",
receipt_backfill,
)
result = await service.run_pending_check_modes_once(limit=1)
assert result["claimed"] == 0
assert result["retry_terminal_projection_verified"] == 1
assert result["retry_terminal_projection_runtime_apply_executed"] is False
assert result["repair_receipt_backfill_priority_tick"] is True
receipt_backfill.assert_awaited_once()
@pytest.mark.asyncio
async def test_signal_worker_retry_preflight_is_query_only(
monkeypatch: pytest.MonkeyPatch,