fix(agent): replay stdin boundary failures without apply
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m34s
CD Pipeline / build-and-deploy (push) Successful in 7m28s
CD Pipeline / post-deploy-checks (push) Successful in 2m18s

This commit is contained in:
ogt
2026-07-14 10:43:00 +08:00
parent 708f976275
commit fc3b08afe3
2 changed files with 641 additions and 1 deletions

View File

@@ -41,6 +41,7 @@ from src.services.awooop_ansible_check_mode_service import (
_record_learning_writeback_receipt,
_record_retry_runtime_stage_receipt,
_record_runtime_stage_receipts,
_record_stdin_boundary_replay_terminal,
_record_timeline_projection_receipt,
_resolve_execution_capability_operation_type,
_revoke_ansible_execution_capability,
@@ -57,6 +58,7 @@ from src.services.awooop_ansible_check_mode_service import (
claim_catalog_drift_failed_check_modes,
claim_pending_check_modes,
claim_stale_pending_check_modes,
claim_stdin_boundary_failed_check_modes,
detect_ansible_transport_blockers,
finalize_check_mode_claim,
recent_ansible_transport_blockers,
@@ -2063,6 +2065,30 @@ def test_failed_check_mode_catalog_drift_replays_once() -> None:
assert "claim_catalog_drift_failed_check_modes" in run_source
def test_failed_stdin_boundary_checks_replay_once_without_apply() -> None:
claim_source = inspect.getsource(
claim_stdin_boundary_failed_check_modes
)
terminal_source = inspect.getsource(
_record_stdin_boundary_replay_terminal
)
run_source = inspect.getsource(run_pending_check_modes_once)
assert "Ansible requires blocking IO" in claim_source
assert "Non-blocking file handles detected" in claim_source
assert "FOR UPDATE SKIP LOCKED" in claim_source
assert "replay_of_check_mode_op_id" in claim_source
assert "historical_stdin_boundary_replay_no_write" in claim_source
assert '"controlled_apply_allowed": False' in claim_source
assert '"apply_enabled": False' in claim_source
assert "stdin_boundary_check_mode_replay_terminal" in terminal_source
assert "verified_no_write_terminal" in terminal_source
assert "runtime_stage_receipts" in terminal_source
assert "runtime_apply_executed" in terminal_source
assert "_record_stdin_boundary_replay_terminal" in run_source
assert "stdin_boundary_replay_passed_no_write_terminal" in run_source
@pytest.mark.asyncio
async def test_catalog_drift_query_failure_does_not_block_fresh_candidate_claims(
monkeypatch: pytest.MonkeyPatch,
@@ -2116,6 +2142,11 @@ async def test_catalog_drift_query_failure_does_not_block_fresh_candidate_claims
no_transport_blockers,
)
monkeypatch.setattr(service, "claim_stale_pending_check_modes", no_stale_claims)
monkeypatch.setattr(
service,
"claim_stdin_boundary_failed_check_modes",
no_stale_claims,
)
monkeypatch.setattr(
service,
"run_failed_apply_check_mode_replay_once",
@@ -2192,6 +2223,11 @@ async def test_execution_broker_runs_retry_then_keeps_fresh_capacity_moving(
"claim_stale_pending_check_modes",
candidate_claim_continues,
)
monkeypatch.setattr(
service,
"claim_stdin_boundary_failed_check_modes",
candidate_claim_continues,
)
monkeypatch.setattr(
service,
"claim_catalog_drift_failed_check_modes",
@@ -2214,6 +2250,142 @@ async def test_execution_broker_runs_retry_then_keeps_fresh_capacity_moving(
assert result["failed_apply_retry_priority_tick"] is True
@pytest.mark.asyncio
async def test_execution_broker_stdin_replay_never_enters_apply(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from src.services import awooop_ansible_check_mode_service as service
replay_claim = AnsibleCheckModeClaim(
op_id="00000000-0000-0000-0000-000000000302",
source_candidate_op_id=(
"00000000-0000-0000-0000-000000000301"
),
incident_id="INC-STDIN-REPLAY",
catalog_id="ansible:188-momo-backup-user",
playbook_path=(
"infra/ansible/playbooks/188-momo-backup-user.yml"
),
apply_playbook_path=(
"infra/ansible/playbooks/188-momo-backup-user.yml"
),
inventory_hosts=("host_188",),
risk_level="low",
input_payload={
"automation_run_id": (
"00000000-0000-0000-0000-000000000301"
),
"historical_stdin_boundary_replay": True,
"replay_of_check_mode_op_id": (
"00000000-0000-0000-0000-000000000300"
),
"controlled_apply_allowed": False,
},
)
async def no_expired_capabilities(**_kwargs):
return 0
async def no_transport_blockers(**_kwargs):
return []
async def no_claims(**_kwargs):
return []
async def stdin_replay_claims(**_kwargs):
return [replay_claim]
async def no_failed_apply_retry(**_kwargs):
return {
"scanned": 0,
"replayed": 0,
"check_mode_passed": 0,
"check_mode_failed": 0,
"runtime_stage_receipt_written": 0,
"blockers": [],
"error": None,
}
async def issue_capability(claim, **_kwargs):
return claim, "00000000-0000-0000-0000-000000000303"
async def revoke_capability(*_args, **_kwargs):
return True
async def successful_check(*_args, **_kwargs):
return AnsibleRunResult(
returncode=0,
stdout="",
stderr="",
duration_ms=12,
)
monkeypatch.setattr(
service,
"_expire_stale_ansible_execution_capabilities",
no_expired_capabilities,
)
monkeypatch.setattr(
service,
"backfill_missing_auto_repair_execution_receipts_once",
AsyncMock(return_value={"scanned": 0, "written": 0, "error": None}),
)
monkeypatch.setattr(
service,
"run_failed_apply_check_mode_replay_once",
no_failed_apply_retry,
)
monkeypatch.setattr(service, "_runtime_blockers", lambda: [])
monkeypatch.setattr(
service,
"recent_ansible_transport_blockers",
no_transport_blockers,
)
monkeypatch.setattr(
service,
"claim_stale_pending_check_modes",
no_claims,
)
monkeypatch.setattr(
service,
"claim_stdin_boundary_failed_check_modes",
stdin_replay_claims,
)
monkeypatch.setattr(
service,
"_issue_ansible_execution_capability",
issue_capability,
)
monkeypatch.setattr(
service,
"_revoke_ansible_execution_capability",
revoke_capability,
)
monkeypatch.setattr(
service,
"run_claimed_check_mode",
successful_check,
)
terminal_write = AsyncMock(return_value=True)
monkeypatch.setattr(
service,
"_record_stdin_boundary_replay_terminal",
terminal_write,
)
apply = AsyncMock()
monkeypatch.setattr(service, "run_controlled_apply_for_claim", apply)
result = await service.run_pending_check_modes_once(limit=1)
assert result["claimed"] == 1
assert result["stdin_boundary_replayed"] == 1
assert result["stdin_boundary_replay_terminal_written"] == 1
assert result["apply_completed"] == 0
assert result["apply_blocked"] == 1
apply.assert_not_awaited()
terminal_write.assert_awaited_once()
def test_check_mode_worker_error_backoff_is_short_and_bounded() -> None:
from src.jobs import awooop_ansible_check_mode_job as job