fix(agent): replay stdin boundary failures without apply
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m34s
CD Pipeline / build-and-deploy (push) Successful in 7m28s
CD Pipeline / post-deploy-checks (push) Successful in 2m18s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m34s
CD Pipeline / build-and-deploy (push) Successful in 7m28s
CD Pipeline / post-deploy-checks (push) Successful in 2m18s
This commit is contained in:
@@ -41,6 +41,7 @@ from src.services.awooop_ansible_check_mode_service import (
|
||||
_record_learning_writeback_receipt,
|
||||
_record_retry_runtime_stage_receipt,
|
||||
_record_runtime_stage_receipts,
|
||||
_record_stdin_boundary_replay_terminal,
|
||||
_record_timeline_projection_receipt,
|
||||
_resolve_execution_capability_operation_type,
|
||||
_revoke_ansible_execution_capability,
|
||||
@@ -57,6 +58,7 @@ from src.services.awooop_ansible_check_mode_service import (
|
||||
claim_catalog_drift_failed_check_modes,
|
||||
claim_pending_check_modes,
|
||||
claim_stale_pending_check_modes,
|
||||
claim_stdin_boundary_failed_check_modes,
|
||||
detect_ansible_transport_blockers,
|
||||
finalize_check_mode_claim,
|
||||
recent_ansible_transport_blockers,
|
||||
@@ -2063,6 +2065,30 @@ def test_failed_check_mode_catalog_drift_replays_once() -> None:
|
||||
assert "claim_catalog_drift_failed_check_modes" in run_source
|
||||
|
||||
|
||||
def test_failed_stdin_boundary_checks_replay_once_without_apply() -> None:
|
||||
claim_source = inspect.getsource(
|
||||
claim_stdin_boundary_failed_check_modes
|
||||
)
|
||||
terminal_source = inspect.getsource(
|
||||
_record_stdin_boundary_replay_terminal
|
||||
)
|
||||
run_source = inspect.getsource(run_pending_check_modes_once)
|
||||
|
||||
assert "Ansible requires blocking IO" in claim_source
|
||||
assert "Non-blocking file handles detected" in claim_source
|
||||
assert "FOR UPDATE SKIP LOCKED" in claim_source
|
||||
assert "replay_of_check_mode_op_id" in claim_source
|
||||
assert "historical_stdin_boundary_replay_no_write" in claim_source
|
||||
assert '"controlled_apply_allowed": False' in claim_source
|
||||
assert '"apply_enabled": False' in claim_source
|
||||
assert "stdin_boundary_check_mode_replay_terminal" in terminal_source
|
||||
assert "verified_no_write_terminal" in terminal_source
|
||||
assert "runtime_stage_receipts" in terminal_source
|
||||
assert "runtime_apply_executed" in terminal_source
|
||||
assert "_record_stdin_boundary_replay_terminal" in run_source
|
||||
assert "stdin_boundary_replay_passed_no_write_terminal" in run_source
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_catalog_drift_query_failure_does_not_block_fresh_candidate_claims(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -2116,6 +2142,11 @@ async def test_catalog_drift_query_failure_does_not_block_fresh_candidate_claims
|
||||
no_transport_blockers,
|
||||
)
|
||||
monkeypatch.setattr(service, "claim_stale_pending_check_modes", no_stale_claims)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"claim_stdin_boundary_failed_check_modes",
|
||||
no_stale_claims,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"run_failed_apply_check_mode_replay_once",
|
||||
@@ -2192,6 +2223,11 @@ async def test_execution_broker_runs_retry_then_keeps_fresh_capacity_moving(
|
||||
"claim_stale_pending_check_modes",
|
||||
candidate_claim_continues,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"claim_stdin_boundary_failed_check_modes",
|
||||
candidate_claim_continues,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"claim_catalog_drift_failed_check_modes",
|
||||
@@ -2214,6 +2250,142 @@ async def test_execution_broker_runs_retry_then_keeps_fresh_capacity_moving(
|
||||
assert result["failed_apply_retry_priority_tick"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execution_broker_stdin_replay_never_enters_apply(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from src.services import awooop_ansible_check_mode_service as service
|
||||
|
||||
replay_claim = AnsibleCheckModeClaim(
|
||||
op_id="00000000-0000-0000-0000-000000000302",
|
||||
source_candidate_op_id=(
|
||||
"00000000-0000-0000-0000-000000000301"
|
||||
),
|
||||
incident_id="INC-STDIN-REPLAY",
|
||||
catalog_id="ansible:188-momo-backup-user",
|
||||
playbook_path=(
|
||||
"infra/ansible/playbooks/188-momo-backup-user.yml"
|
||||
),
|
||||
apply_playbook_path=(
|
||||
"infra/ansible/playbooks/188-momo-backup-user.yml"
|
||||
),
|
||||
inventory_hosts=("host_188",),
|
||||
risk_level="low",
|
||||
input_payload={
|
||||
"automation_run_id": (
|
||||
"00000000-0000-0000-0000-000000000301"
|
||||
),
|
||||
"historical_stdin_boundary_replay": True,
|
||||
"replay_of_check_mode_op_id": (
|
||||
"00000000-0000-0000-0000-000000000300"
|
||||
),
|
||||
"controlled_apply_allowed": False,
|
||||
},
|
||||
)
|
||||
|
||||
async def no_expired_capabilities(**_kwargs):
|
||||
return 0
|
||||
|
||||
async def no_transport_blockers(**_kwargs):
|
||||
return []
|
||||
|
||||
async def no_claims(**_kwargs):
|
||||
return []
|
||||
|
||||
async def stdin_replay_claims(**_kwargs):
|
||||
return [replay_claim]
|
||||
|
||||
async def no_failed_apply_retry(**_kwargs):
|
||||
return {
|
||||
"scanned": 0,
|
||||
"replayed": 0,
|
||||
"check_mode_passed": 0,
|
||||
"check_mode_failed": 0,
|
||||
"runtime_stage_receipt_written": 0,
|
||||
"blockers": [],
|
||||
"error": None,
|
||||
}
|
||||
|
||||
async def issue_capability(claim, **_kwargs):
|
||||
return claim, "00000000-0000-0000-0000-000000000303"
|
||||
|
||||
async def revoke_capability(*_args, **_kwargs):
|
||||
return True
|
||||
|
||||
async def successful_check(*_args, **_kwargs):
|
||||
return AnsibleRunResult(
|
||||
returncode=0,
|
||||
stdout="",
|
||||
stderr="",
|
||||
duration_ms=12,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_expire_stale_ansible_execution_capabilities",
|
||||
no_expired_capabilities,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"backfill_missing_auto_repair_execution_receipts_once",
|
||||
AsyncMock(return_value={"scanned": 0, "written": 0, "error": None}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"run_failed_apply_check_mode_replay_once",
|
||||
no_failed_apply_retry,
|
||||
)
|
||||
monkeypatch.setattr(service, "_runtime_blockers", lambda: [])
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"recent_ansible_transport_blockers",
|
||||
no_transport_blockers,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"claim_stale_pending_check_modes",
|
||||
no_claims,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"claim_stdin_boundary_failed_check_modes",
|
||||
stdin_replay_claims,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_issue_ansible_execution_capability",
|
||||
issue_capability,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_revoke_ansible_execution_capability",
|
||||
revoke_capability,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"run_claimed_check_mode",
|
||||
successful_check,
|
||||
)
|
||||
terminal_write = AsyncMock(return_value=True)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_record_stdin_boundary_replay_terminal",
|
||||
terminal_write,
|
||||
)
|
||||
apply = AsyncMock()
|
||||
monkeypatch.setattr(service, "run_controlled_apply_for_claim", apply)
|
||||
|
||||
result = await service.run_pending_check_modes_once(limit=1)
|
||||
|
||||
assert result["claimed"] == 1
|
||||
assert result["stdin_boundary_replayed"] == 1
|
||||
assert result["stdin_boundary_replay_terminal_written"] == 1
|
||||
assert result["apply_completed"] == 0
|
||||
assert result["apply_blocked"] == 1
|
||||
apply.assert_not_awaited()
|
||||
terminal_write.assert_awaited_once()
|
||||
|
||||
|
||||
def test_check_mode_worker_error_backoff_is_short_and_bounded() -> None:
|
||||
from src.jobs import awooop_ansible_check_mode_job as job
|
||||
|
||||
|
||||
Reference in New Issue
Block a user