wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(sre): bind Agent99 closure to typed verifier

Browse Source
This commit is contained in:
Your Name
2026-07-19 00:14:43 +08:00
parent 39685e9a08
commit 373ad979f4
5 changed files with 799 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

320
apps/api/tests/test_agent99_completion_callback_api.py
View File

@@ -17,6 +17,9 @@ from src.api.v1 import agents as agents_api
from src.core.config import settings
from src.models.agent99_completion import Agent99CompletionCallbackRequest
from src.services import agent99_completion_callback as callback_service
from src.services.agent99_controlled_dispatch_ledger import (
build_agent99_dispatch_identity,
)
def payload() -> dict:
@@ -132,6 +135,27 @@ def test_completion_callback_requires_durable_readback(monkeypatch) -> None:
assert response.json()["detail"] == "agent99_completion_durable_readback_failed"
def test_completion_callback_keeps_unverified_closure_pending(monkeypatch) -> None:
monkeypatch.setattr(settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "expected")
async def fake_record(_request):
return {
"ok": False,
"durable_readback": True,
"status": "closure_authority_pending",
}
monkeypatch.setattr(agents_api, "record_agent99_completion_callback", fake_record)
response = app_client().post(
"/api/v1/agents/agent99/completion-callback",
headers={"X-Agent99-Completion-Token": "expected"},
json=payload(),
)
assert response.status_code == 503
assert response.json()["detail"] == "closure_authority_pending"
def test_completion_callback_returns_same_trace_receipt(monkeypatch) -> None:
monkeypatch.setattr(settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "expected")
@@ -168,8 +192,301 @@ class FakeOperationRepository:
return SimpleNamespace(id=f"operation-{len(self.events)}")
def authorize_terminal_callback(monkeypatch) -> None:
async def fake_authority(_payload):
return {
"required": True,
"verified": True,
"blocker": "none",
"typed_domain": "control_plane_recovery",
"route_id": "agent99:host_recovery:Recover",
}
monkeypatch.setattr(
callback_service,
"_load_callback_closure_authority",
fake_authority,
)
def typed_dispatch_callback_contract() -> tuple[
Agent99CompletionCallbackRequest,
dict,
]:
identity = build_agent99_dispatch_identity(
project_id="awoooi",
incident_id="INC-20260711-001",
source_fingerprint="agent99-callback-source",
route_id="agent99_cold_start_recovery",
work_item_id="agent99-incident:INC-20260711-001",
)
verifier_name = "recover_post_condition_v1"
request = Agent99CompletionCallbackRequest.model_validate({
**payload(),
"run_id": str(identity.run_id),
"trace_id": identity.trace_id,
"work_item_id": identity.work_item_id,
"verifier_name": verifier_name,
})
receipt = {
"schema_version": "agent99_controlled_dispatch_receipt_v1",
"status": "verifier_passed_learning_writeback_pending",
"identity": identity.public_dict(),
"run_id": str(identity.run_id),
"trace_id": identity.trace_id,
"run_state": "waiting_tool",
"dispatch_promoted": True,
"controlled_apply_authorized": True,
"post_verifier_passed": True,
"dispatch_scope": {
"schema_version": "agent99_dispatch_scope_v1",
"kind": "host_recovery",
"suggested_mode": "Recover",
"target_resource": "cold-start-gate",
"controlled_apply_requested": True,
"canonical_asset_id": "windows-vmware:host_99",
"typed_domain": "control_plane_recovery",
"executor": "Agent99",
"verifier": verifier_name,
"route_id": identity.route_id,
},
"verifier": {
"schema_version": "agent99_independent_verifier_receipt_v1",
"status": "success",
"run_id": str(identity.run_id),
"trace_id": identity.trace_id,
"work_item_id": identity.work_item_id,
"outcome_state": "resolved",
"transport_ok": True,
"verifier_passed": True,
"source_event_resolved": True,
},
}
return request, receipt
@pytest.mark.asyncio
async def test_callback_closure_authority_requires_exact_typed_ledger(
monkeypatch,
) -> None:
request, dispatch_receipt = typed_dispatch_callback_contract()
async def fake_read(**_kwargs):
return dispatch_receipt
monkeypatch.setattr(
callback_service,
"read_agent99_dispatch_receipt_for_run",
fake_read,
)
authority = await callback_service._load_callback_closure_authority(request)
assert authority == {
"required": True,
"verified": True,
"blocker": "none",
"typed_domain": "control_plane_recovery",
"route_id": "agent99_cold_start_recovery",
}
@pytest.mark.asyncio
async def test_callback_closure_authority_rejects_cross_domain_scope(
monkeypatch,
) -> None:
request, dispatch_receipt = typed_dispatch_callback_contract()
dispatch_receipt["dispatch_scope"]["typed_domain"] = "docker_compose"
async def fake_read(**_kwargs):
return dispatch_receipt
monkeypatch.setattr(
callback_service,
"read_agent99_dispatch_receipt_for_run",
fake_read,
)
authority = await callback_service._load_callback_closure_authority(request)
assert authority["verified"] is False
assert authority["blocker"] == "typed_dispatch_scope_mismatch"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"field",
["transport_ok", "verifier_passed", "source_event_resolved"],
)
async def test_callback_closure_authority_rejects_false_verifier_projection(
monkeypatch,
field: str,
) -> None:
request, dispatch_receipt = typed_dispatch_callback_contract()
request = request.model_copy(update={field: False})
async def fake_read(**_kwargs):
return dispatch_receipt
monkeypatch.setattr(
callback_service,
"read_agent99_dispatch_receipt_for_run",
fake_read,
)
authority = await callback_service._load_callback_closure_authority(request)
assert authority["verified"] is False
assert authority["blocker"] == "callback_verifier_projection_mismatch"
@pytest.mark.asyncio
async def test_completion_service_does_not_resolve_from_caller_booleans(
monkeypatch,
) -> None:
request = Agent99CompletionCallbackRequest.model_validate(payload())
event_id = UUID("3c9e8ee7-fb21-5420-afae-ef928f75a58c")
repository = FakeOperationRepository()
readbacks = iter([
None,
{
"event_id": str(event_id),
"run_id": "5e5b0080-e8c9-53a7-872b-0fc1ce80f11f",
"operation_count": 1,
"execution_count": 1,
"resolved_count": 0,
},
])
async def fake_authority(_payload):
return {
"required": True,
"verified": False,
"blocker": "dispatch_run_not_found",
"typed_domain": "",
"route_id": "",
}
async def fake_readback(**_kwargs):
return next(readbacks)
async def fake_record_event(**kwargs):
assert kwargs["stage"] == "verifying"
assert kwargs["severity"] == "warning"
return event_id
monkeypatch.setattr(
callback_service,
"_load_callback_closure_authority",
fake_authority,
)
monkeypatch.setattr(callback_service, "_load_callback_readback", fake_readback)
monkeypatch.setattr(callback_service, "record_external_alert_event", fake_record_event)
monkeypatch.setattr(
callback_service,
"get_alert_operation_log_repository",
lambda: repository,
)
receipt = await callback_service.record_agent99_completion_callback(request)
assert receipt["ok"] is False
assert receipt["durable_readback"] is True
assert receipt["outcome_state"] == "verifying"
assert receipt["closure_allowed"] is False
assert receipt["closure_blocker"] == "dispatch_run_not_found"
assert [item[0] for item in repository.events] == ["EXECUTION_COMPLETED"]
assert repository.events[0][1]["success"] is False
@pytest.mark.asyncio
async def test_completion_service_retry_adds_only_resolved_receipt(
monkeypatch,
) -> None:
request = Agent99CompletionCallbackRequest.model_validate(payload())
verifying_event_id = UUID("3c9e8ee7-fb21-5420-afae-ef928f75a58c")
resolved_event_id = UUID("5e5b0080-e8c9-53a7-872b-0fc1ce80f11f")
awooop_run_id = "8b877ed3-037f-5463-954b-f4b77a77d786"
repository = FakeOperationRepository()
authority_ready = {"value": False}
readbacks = iter([
None,
{
"event_id": str(verifying_event_id),
"run_id": awooop_run_id,
"operation_count": 1,
"execution_count": 1,
"resolved_count": 0,
},
{
"event_id": None,
"run_id": None,
"operation_count": 1,
"execution_count": 1,
"resolved_count": 0,
},
{
"event_id": str(resolved_event_id),
"run_id": awooop_run_id,
"operation_count": 2,
"execution_count": 1,
"resolved_count": 1,
},
])
event_calls: list[dict] = []
async def fake_authority(_payload):
verified = authority_ready["value"]
return {
"required": True,
"verified": verified,
"blocker": "none" if verified else "independent_verifier_missing",
"typed_domain": "control_plane_recovery" if verified else "",
"route_id": "agent99_cold_start_recovery" if verified else "",
}
async def fake_readback(**_kwargs):
return next(readbacks)
async def fake_record_event(**kwargs):
event_calls.append(kwargs)
return (
resolved_event_id
if kwargs["stage"] == "resolved"
else verifying_event_id
)
monkeypatch.setattr(
callback_service,
"_load_callback_closure_authority",
fake_authority,
)
monkeypatch.setattr(callback_service, "_load_callback_readback", fake_readback)
monkeypatch.setattr(callback_service, "record_external_alert_event", fake_record_event)
monkeypatch.setattr(
callback_service,
"get_alert_operation_log_repository",
lambda: repository,
)
pending = await callback_service.record_agent99_completion_callback(request)
authority_ready["value"] = True
resolved = await callback_service.record_agent99_completion_callback(request)
assert pending["ok"] is False
assert resolved["ok"] is True
assert resolved["duplicate"] is True
assert [item[0] for item in repository.events] == [
"EXECUTION_COMPLETED",
"RESOLVED",
]
assert [call["stage"] for call in event_calls] == ["verifying", "resolved"]
assert [call["is_duplicate"] for call in event_calls] == [False, True]
@pytest.mark.asyncio
async def test_completion_service_writes_event_run_and_operation_receipts(monkeypatch) -> None:
authorize_terminal_callback(monkeypatch)
request = Agent99CompletionCallbackRequest.model_validate(payload())
event_id = UUID("3c9e8ee7-fb21-5420-afae-ef928f75a58c")
awooop_run_id = "5e5b0080-e8c9-53a7-872b-0fc1ce80f11f"
@@ -221,6 +538,7 @@ async def test_completion_service_writes_event_run_and_operation_receipts(monkey
async def test_completion_service_compacts_long_alert_id_for_operation_log(
monkeypatch,
) -> None:
authorize_terminal_callback(monkeypatch)
external_alert_id = "awoooi-agent99-ae36a454-c9eb-5f2d-9e35-be852b3eac9b"
request = Agent99CompletionCallbackRequest.model_validate(
{**payload(), "alert_id": external_alert_id}
@@ -267,6 +585,7 @@ async def test_completion_service_compacts_long_alert_id_for_operation_log(
@pytest.mark.asyncio
async def test_completion_service_is_idempotent(monkeypatch) -> None:
authorize_terminal_callback(monkeypatch)
request = Agent99CompletionCallbackRequest.model_validate(payload())
event_id = UUID("3c9e8ee7-fb21-5420-afae-ef928f75a58c")
existing = {
@@ -304,6 +623,7 @@ async def test_completion_service_is_idempotent(monkeypatch) -> None:
@pytest.mark.asyncio
async def test_completion_service_repairs_partial_terminal_receipt(monkeypatch) -> None:
authorize_terminal_callback(monkeypatch)
request = Agent99CompletionCallbackRequest.model_validate(payload())
event_id = UUID("3c9e8ee7-fb21-5420-afae-ef928f75a58c")
before = {