wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(sre): bind Agent99 closure to typed verifier

Browse Source
This commit is contained in:
Your Name
2026-07-19 00:14:43 +08:00
parent 39685e9a08
commit 373ad979f4
5 changed files with 799 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

320
apps/api/tests/test_agent99_completion_callback_api.py
View File

@@ -17,6 +17,9 @@ from src.api.v1 import agents as agents_api
from src.core.config import settings
from src.models.agent99_completion import Agent99CompletionCallbackRequest
from src.services import agent99_completion_callback as callback_service
from src.services.agent99_controlled_dispatch_ledger import (
build_agent99_dispatch_identity,
)
def payload() -> dict:
@@ -132,6 +135,27 @@ def test_completion_callback_requires_durable_readback(monkeypatch) -> None:
assert response.json()["detail"] == "agent99_completion_durable_readback_failed"
def test_completion_callback_keeps_unverified_closure_pending(monkeypatch) -> None:
monkeypatch.setattr(settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "expected")
async def fake_record(_request):
return {
"ok": False,
"durable_readback": True,
"status": "closure_authority_pending",
}
monkeypatch.setattr(agents_api, "record_agent99_completion_callback", fake_record)
response = app_client().post(
"/api/v1/agents/agent99/completion-callback",
headers={"X-Agent99-Completion-Token": "expected"},
json=payload(),
)
assert response.status_code == 503
assert response.json()["detail"] == "closure_authority_pending"
def test_completion_callback_returns_same_trace_receipt(monkeypatch) -> None:
monkeypatch.setattr(settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "expected")
@@ -168,8 +192,301 @@ class FakeOperationRepository:
return SimpleNamespace(id=f"operation-{len(self.events)}")
def authorize_terminal_callback(monkeypatch) -> None:
async def fake_authority(_payload):
return {
"required": True,
"verified": True,
"blocker": "none",
"typed_domain": "control_plane_recovery",
"route_id": "agent99:host_recovery:Recover",
}
monkeypatch.setattr(
callback_service,
"_load_callback_closure_authority",
fake_authority,
)
def typed_dispatch_callback_contract() -> tuple[
Agent99CompletionCallbackRequest,
dict,
]:
identity = build_agent99_dispatch_identity(
project_id="awoooi",
incident_id="INC-20260711-001",
source_fingerprint="agent99-callback-source",
route_id="agent99_cold_start_recovery",
work_item_id="agent99-incident:INC-20260711-001",
)
verifier_name = "recover_post_condition_v1"
request = Agent99CompletionCallbackRequest.model_validate({
**payload(),
"run_id": str(identity.run_id),
"trace_id": identity.trace_id,
"work_item_id": identity.work_item_id,
"verifier_name": verifier_name,
})
receipt = {
"schema_version": "agent99_controlled_dispatch_receipt_v1",
"status": "verifier_passed_learning_writeback_pending",
"identity": identity.public_dict(),
"run_id": str(identity.run_id),
"trace_id": identity.trace_id,
"run_state": "waiting_tool",
"dispatch_promoted": True,
"controlled_apply_authorized": True,
"post_verifier_passed": True,
"dispatch_scope": {
"schema_version": "agent99_dispatch_scope_v1",
"kind": "host_recovery",
"suggested_mode": "Recover",
"target_resource": "cold-start-gate",
"controlled_apply_requested": True,
"canonical_asset_id": "windows-vmware:host_99",
"typed_domain": "control_plane_recovery",
"executor": "Agent99",
"verifier": verifier_name,
"route_id": identity.route_id,
},
"verifier": {
"schema_version": "agent99_independent_verifier_receipt_v1",
"status": "success",
"run_id": str(identity.run_id),
"trace_id": identity.trace_id,
"work_item_id": identity.work_item_id,
"outcome_state": "resolved",
"transport_ok": True,
"verifier_passed": True,
"source_event_resolved": True,
},
}
return request, receipt
@pytest.mark.asyncio
async def test_callback_closure_authority_requires_exact_typed_ledger(
monkeypatch,
) -> None:
request, dispatch_receipt = typed_dispatch_callback_contract()
async def fake_read(**_kwargs):
return dispatch_receipt
monkeypatch.setattr(
callback_service,
"read_agent99_dispatch_receipt_for_run",
fake_read,
)
authority = await callback_service._load_callback_closure_authority(request)
assert authority == {
"required": True,
"verified": True,
"blocker": "none",
"typed_domain": "control_plane_recovery",
"route_id": "agent99_cold_start_recovery",
}
@pytest.mark.asyncio
async def test_callback_closure_authority_rejects_cross_domain_scope(
monkeypatch,
) -> None:
request, dispatch_receipt = typed_dispatch_callback_contract()
dispatch_receipt["dispatch_scope"]["typed_domain"] = "docker_compose"
async def fake_read(**_kwargs):
return dispatch_receipt
monkeypatch.setattr(
callback_service,
"read_agent99_dispatch_receipt_for_run",
fake_read,
)
authority = await callback_service._load_callback_closure_authority(request)
assert authority["verified"] is False
assert authority["blocker"] == "typed_dispatch_scope_mismatch"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"field",
["transport_ok", "verifier_passed", "source_event_resolved"],
)
async def test_callback_closure_authority_rejects_false_verifier_projection(
monkeypatch,
field: str,
) -> None:
request, dispatch_receipt = typed_dispatch_callback_contract()
request = request.model_copy(update={field: False})
async def fake_read(**_kwargs):
return dispatch_receipt
monkeypatch.setattr(
callback_service,
"read_agent99_dispatch_receipt_for_run",
fake_read,
)
authority = await callback_service._load_callback_closure_authority(request)
assert authority["verified"] is False
assert authority["blocker"] == "callback_verifier_projection_mismatch"
@pytest.mark.asyncio
async def test_completion_service_does_not_resolve_from_caller_booleans(
monkeypatch,
) -> None:
request = Agent99CompletionCallbackRequest.model_validate(payload())
event_id = UUID("3c9e8ee7-fb21-5420-afae-ef928f75a58c")
repository = FakeOperationRepository()
readbacks = iter([
None,
{
"event_id": str(event_id),
"run_id": "5e5b0080-e8c9-53a7-872b-0fc1ce80f11f",
"operation_count": 1,
"execution_count": 1,
"resolved_count": 0,
},
])
async def fake_authority(_payload):
return {
"required": True,
"verified": False,
"blocker": "dispatch_run_not_found",
"typed_domain": "",
"route_id": "",
}
async def fake_readback(**_kwargs):
return next(readbacks)
async def fake_record_event(**kwargs):
assert kwargs["stage"] == "verifying"
assert kwargs["severity"] == "warning"
return event_id
monkeypatch.setattr(
callback_service,
"_load_callback_closure_authority",
fake_authority,
)
monkeypatch.setattr(callback_service, "_load_callback_readback", fake_readback)
monkeypatch.setattr(callback_service, "record_external_alert_event", fake_record_event)
monkeypatch.setattr(
callback_service,
"get_alert_operation_log_repository",
lambda: repository,
)
receipt = await callback_service.record_agent99_completion_callback(request)
assert receipt["ok"] is False
assert receipt["durable_readback"] is True
assert receipt["outcome_state"] == "verifying"
assert receipt["closure_allowed"] is False
assert receipt["closure_blocker"] == "dispatch_run_not_found"
assert [item[0] for item in repository.events] == ["EXECUTION_COMPLETED"]
assert repository.events[0][1]["success"] is False
@pytest.mark.asyncio
async def test_completion_service_retry_adds_only_resolved_receipt(
monkeypatch,
) -> None:
request = Agent99CompletionCallbackRequest.model_validate(payload())
verifying_event_id = UUID("3c9e8ee7-fb21-5420-afae-ef928f75a58c")
resolved_event_id = UUID("5e5b0080-e8c9-53a7-872b-0fc1ce80f11f")
awooop_run_id = "8b877ed3-037f-5463-954b-f4b77a77d786"
repository = FakeOperationRepository()
authority_ready = {"value": False}
readbacks = iter([
None,
{
"event_id": str(verifying_event_id),
"run_id": awooop_run_id,
"operation_count": 1,
"execution_count": 1,
"resolved_count": 0,
},
{
"event_id": None,
"run_id": None,
"operation_count": 1,
"execution_count": 1,
"resolved_count": 0,
},
{
"event_id": str(resolved_event_id),
"run_id": awooop_run_id,
"operation_count": 2,
"execution_count": 1,
"resolved_count": 1,
},
])
event_calls: list[dict] = []
async def fake_authority(_payload):
verified = authority_ready["value"]
return {
"required": True,
"verified": verified,
"blocker": "none" if verified else "independent_verifier_missing",
"typed_domain": "control_plane_recovery" if verified else "",
"route_id": "agent99_cold_start_recovery" if verified else "",
}
async def fake_readback(**_kwargs):
return next(readbacks)
async def fake_record_event(**kwargs):
event_calls.append(kwargs)
return (
resolved_event_id
if kwargs["stage"] == "resolved"
else verifying_event_id
)
monkeypatch.setattr(
callback_service,
"_load_callback_closure_authority",
fake_authority,
)
monkeypatch.setattr(callback_service, "_load_callback_readback", fake_readback)
monkeypatch.setattr(callback_service, "record_external_alert_event", fake_record_event)
monkeypatch.setattr(
callback_service,
"get_alert_operation_log_repository",
lambda: repository,
)
pending = await callback_service.record_agent99_completion_callback(request)
authority_ready["value"] = True
resolved = await callback_service.record_agent99_completion_callback(request)
assert pending["ok"] is False
assert resolved["ok"] is True
assert resolved["duplicate"] is True
assert [item[0] for item in repository.events] == [
"EXECUTION_COMPLETED",
"RESOLVED",
]
assert [call["stage"] for call in event_calls] == ["verifying", "resolved"]
assert [call["is_duplicate"] for call in event_calls] == [False, True]
@pytest.mark.asyncio
async def test_completion_service_writes_event_run_and_operation_receipts(monkeypatch) -> None:
authorize_terminal_callback(monkeypatch)
request = Agent99CompletionCallbackRequest.model_validate(payload())
event_id = UUID("3c9e8ee7-fb21-5420-afae-ef928f75a58c")
awooop_run_id = "5e5b0080-e8c9-53a7-872b-0fc1ce80f11f"
@@ -221,6 +538,7 @@ async def test_completion_service_writes_event_run_and_operation_receipts(monkey
async def test_completion_service_compacts_long_alert_id_for_operation_log(
monkeypatch,
) -> None:
authorize_terminal_callback(monkeypatch)
external_alert_id = "awoooi-agent99-ae36a454-c9eb-5f2d-9e35-be852b3eac9b"
request = Agent99CompletionCallbackRequest.model_validate(
{**payload(), "alert_id": external_alert_id}
@@ -267,6 +585,7 @@ async def test_completion_service_compacts_long_alert_id_for_operation_log(
@pytest.mark.asyncio
async def test_completion_service_is_idempotent(monkeypatch) -> None:
authorize_terminal_callback(monkeypatch)
request = Agent99CompletionCallbackRequest.model_validate(payload())
event_id = UUID("3c9e8ee7-fb21-5420-afae-ef928f75a58c")
existing = {
@@ -304,6 +623,7 @@ async def test_completion_service_is_idempotent(monkeypatch) -> None:
@pytest.mark.asyncio
async def test_completion_service_repairs_partial_terminal_receipt(monkeypatch) -> None:
authorize_terminal_callback(monkeypatch)
request = Agent99CompletionCallbackRequest.model_validate(payload())
event_id = UUID("3c9e8ee7-fb21-5420-afae-ef928f75a58c")
before = {

152
apps/api/tests/test_agent99_controlled_dispatch_p1.py
View File

@@ -45,11 +45,13 @@ def _outcome(identity, *, transport_ok: bool = True) -> dict:
return {
"identity": identity.public_dict(),
"controlledApply": True,
"mode": "Recover",
"outcome": {
"identity": identity.public_dict(),
"schemaVersion": "agent99_outcome_contract_v1",
"state": "resolved",
"transportOk": transport_ok,
"verifierName": "recover_post_condition_v1",
"verifierPassed": True,
"sourceEventResolved": True,
"verifiedAt": "2026-07-11T20:00:00+08:00",
@@ -57,6 +59,30 @@ def _outcome(identity, *, transport_ok: bool = True) -> dict:
}
def _promoted_dispatch(identity, *, authorized: bool = True) -> dict:
return build_agent99_dispatch_receipt_envelope(
identity=identity,
dispatch_receipt={
"kind": "host_recovery",
"suggested_mode": "Recover",
"target_resource": "cold-start-gate",
"controlled_apply_requested": True,
"accepted": True,
"inbox_triggered": True,
"queue_accepted": True,
"dispatch_identity_matched": True,
"delivery_certainty": "delivered",
"dispatch_scope": {
"canonical_asset_id": "windows-vmware:host_99",
"typed_domain": "control_plane_recovery",
"executor": "Agent99",
"verifier": "recover_post_condition_v1",
},
},
controlled_apply_authorized=authorized,
)
def test_pending_inbox_never_promotes_or_authorizes() -> None:
receipt = build_agent99_dispatch_receipt_envelope(
identity=_identity(),
@@ -91,6 +117,45 @@ def test_complete_identity_is_required_and_recomputed() -> None:
parse_agent99_dispatch_identity(mismatched)
@pytest.mark.asyncio
async def test_dispatch_receipt_read_is_exact_run_only(monkeypatch) -> None:
identity = _identity()
ledger = PostgresAgent99DispatchLedger()
assert await ledger.read_for_run(
project_id=identity.project_id,
run_id="free-form-shadow-run",
) is None
envelope = _promoted_dispatch(identity)
class ReadDB:
async def execute(self, _statement):
return _ScalarResult(
row=SimpleNamespace(
run_id=identity.run_id,
state="waiting_tool",
trace_id=identity.trace_id,
error_detail=json.dumps(envelope),
)
)
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(ReadDB()),
)
receipt = await ledger.read_for_run(
project_id=identity.project_id,
run_id=str(identity.run_id),
)
assert receipt is not None
assert receipt["run_id"] == str(identity.run_id)
assert receipt["trace_id"] == identity.trace_id
assert receipt["run_state"] == "waiting_tool"
def test_approval_identity_cannot_alias_run_or_idempotency() -> None:
common = {
"project_id": "awoooi",
@@ -335,15 +400,7 @@ async def test_verifier_requires_transport_and_all_evidence(monkeypatch) -> None
assert missing["status"] == "verifier_evidence_missing_fail_closed"
assert missing["receipt_persisted"] is False
promoted = build_agent99_dispatch_receipt_envelope(
identity=identity,
dispatch_receipt={
"accepted": True,
"inbox_triggered": True,
"delivery_certainty": "delivered",
},
controlled_apply_authorized=True,
)
promoted = _promoted_dispatch(identity)
class VerifierDB:
def __init__(self) -> None:
@@ -397,6 +454,83 @@ async def test_verifier_requires_transport_and_all_evidence(monkeypatch) -> None
assert passed["runtime_closure_verified"] is False
@pytest.mark.asyncio
@pytest.mark.parametrize("mismatch", ["verifier", "mode", "controlled_apply"])
async def test_verifier_rejects_dispatch_scope_mismatch(
monkeypatch,
mismatch: str,
) -> None:
identity = _identity()
promoted = _promoted_dispatch(identity)
outcome = _outcome(identity)
if mismatch == "verifier":
outcome["outcome"]["verifierName"] = "foreign-verifier"
elif mismatch == "mode":
outcome["mode"] = "Status"
else:
outcome["controlledApply"] = False
class ScopeDB:
async def execute(self, _statement):
return _ScalarResult(
row=SimpleNamespace(
state="waiting_tool",
error_detail=json.dumps(promoted),
)
)
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(ScopeDB()),
)
result = await PostgresAgent99DispatchLedger().record_verifier(
identity=identity,
outcome_receipt=outcome,
evidence_refs=_evidence_refs(),
)
assert result["status"] == "verifier_dispatch_scope_mismatch_fail_closed"
assert result["receipt_persisted"] is False
assert result["runtime_closure_verified"] is False
@pytest.mark.asyncio
async def test_verifier_never_upgrades_controlled_apply_from_outcome(
monkeypatch,
) -> None:
identity = _identity()
promoted = _promoted_dispatch(identity, authorized=False)
class ScopeDB:
async def execute(self, _statement):
return _ScalarResult(
row=SimpleNamespace(
state="waiting_tool",
error_detail=json.dumps(promoted),
)
)
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(ScopeDB()),
)
result = await PostgresAgent99DispatchLedger().record_verifier(
identity=identity,
outcome_receipt=_outcome(identity),
evidence_refs=_evidence_refs(),
)
assert result["status"] == (
"verifier_controlled_apply_not_authorized_fail_closed"
)
assert result["receipt_persisted"] is False
assert result["runtime_closure_verified"] is False
@pytest.mark.asyncio
async def test_learning_writeback_is_the_only_same_run_terminal_step(
monkeypatch,