wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 4 Packages Projects Releases Wiki Activity

fix(sre): fail closed typed Agent99 dispatch

Browse Source
This commit is contained in:
Your Name
2026-07-19 03:21:22 +08:00
parent 3223f9c913
commit ba4e5c41a7
7 changed files with 329 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
apps/api/tests/test_agent99_controlled_dispatch_p1.py
View File

@@ -15,12 +15,17 @@ from src.services import agent99_controlled_dispatch_ledger as ledger_module
from src.services import agent99_outcome_ingestion as ingestion_module
from src.services.agent99_controlled_dispatch_ledger import (
PostgresAgent99DispatchLedger,
attach_agent99_dispatch_identity,
build_agent99_dispatch_identity,
build_agent99_dispatch_receipt_envelope,
parse_agent99_dispatch_identity,
record_agent99_learning_writeback,
)
from src.services.agent99_sre_bridge import bridge_alertmanager_to_agent99
from src.services.agent99_sre_bridge import (
_agent99_dispatch_scope_from_payload,
bridge_alertmanager_to_agent99,
build_agent99_sre_alert,
)
def _identity():
@@ -491,6 +496,115 @@ async def test_verifier_requires_transport_and_all_evidence(monkeypatch) -> None
assert passed["runtime_closure_verified"] is False
@pytest.mark.asyncio
async def test_backupcheck_read_only_scope_passes_real_ledger_verifier(
monkeypatch,
) -> None:
identity = build_agent99_dispatch_identity(
project_id="awoooi",
incident_id="INC-20260711-BACKUP",
source_fingerprint="backup-fingerprint",
route_id="agent99:backup_health:BackupCheck",
execution_generation="1",
work_item_id=(
"agent99-dispatch:awoooi:INC-20260711-BACKUP:BackupCheck"
),
)
payload = build_agent99_sre_alert(
alert_id="backup-readback",
alertname="BackupCredentialEscrowEvidenceMissing",
severity="warning",
namespace="awoooi-prod",
target_resource="backup_restore",
message="freshness escrow restore evidence required",
labels={"event_type": "backup_restore_escrow_signal"},
fingerprint="backup-fingerprint",
)
payload = attach_agent99_dispatch_identity(payload, identity)
scope = _agent99_dispatch_scope_from_payload(payload)
assert scope["executor"] == "Agent99"
assert scope["break_glass_executor"] == "backup_restore_break_glass"
assert scope["controlled_apply_requested"] is False
promoted = build_agent99_dispatch_receipt_envelope(
identity=identity,
dispatch_receipt={
"kind": "backup_health",
"suggested_mode": "BackupCheck",
"target_resource": "backup_restore",
"controlled_apply_requested": False,
"accepted": True,
"inbox_triggered": True,
"queue_accepted": True,
"dispatch_identity_matched": True,
"delivery_certainty": "delivered",
"dispatch_scope": scope,
},
controlled_apply_authorized=False,
)
outcome = {
"identity": identity.public_dict(),
"controlledApply": False,
"mode": "BackupCheck",
"outcome": {
"identity": identity.public_dict(),
"schemaVersion": "agent99_outcome_contract_v1",
"state": "resolved",
"transportOk": True,
"verifierName": "backup_restore_readback_verifier",
"verifierPassed": True,
"sourceEventResolved": True,
"verifiedAt": "2026-07-11T20:00:00+08:00",
},
}
class BackupVerifierDB:
def __init__(self) -> None:
self.call = 0
async def execute(self, _statement):
self.call += 1
if self.call == 1:
return _ScalarResult(
row=SimpleNamespace(
state="waiting_tool",
error_detail=json.dumps(promoted),
)
)
if self.call == 2:
return _ScalarResult(identity.run_id)
if self.call == 4:
return _ScalarResult(identity.incident_id)
if self.call in {5, 6}:
return _ScalarResult(identity.run_id)
return _ScalarResult()
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(BackupVerifierDB()),
)
result = await PostgresAgent99DispatchLedger().record_verifier(
identity=identity,
outcome_receipt=outcome,
evidence_refs={
**_evidence_refs(),
"backup_status_evidence_ref": "backup-status:1",
"freshness_evidence_ref": "backup-freshness:1",
"offsite_verify_evidence_ref": "offsite-verify:1",
"escrow_evidence_ref": "escrow-readback:1",
"restore_drill_evidence_ref": "restore-drill:1",
"source_resolution_receipt_ref": "alert-resolved:1",
},
)
assert result["status"] == "verifier_passed_learning_writeback_pending"
assert result["post_verifier_passed"] is True
assert result["controlled_apply_authorized"] is False
assert result["runtime_closure_verified"] is False
@pytest.mark.asyncio
@pytest.mark.parametrize("mismatch", ["verifier", "mode", "controlled_apply"])
async def test_verifier_rejects_dispatch_scope_mismatch(

119
apps/api/tests/test_agent99_sre_bridge.py
View File

@@ -421,6 +421,91 @@ def test_agent99_single_flight_key_does_not_expose_fingerprint() -> None:
assert "private" not in key
@pytest.mark.asyncio
async def test_agent99_bridge_rejects_unknown_read_only_before_claim_or_transport(
monkeypatch,
) -> None:
claimed: list[dict[str, object]] = []
dispatched: list[dict[str, object]] = []
async def unexpected_claim(*_args, **kwargs) -> dict[str, object]:
claimed.append(kwargs)
return {"acquired": True, "reason": "unexpected"}
monkeypatch.setattr(
"src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
unexpected_claim,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda payload: dispatched.append(payload) or {},
)
result = await bridge_alertmanager_to_agent99(
alert_id="unknown-read-only",
alertname="UnknownObservation",
severity="warning",
namespace="node",
target_resource="192.168.0.120",
message="observation only",
fingerprint="unknown-read-only-fingerprint",
)
assert result["status"] == "failed"
assert result["reason"] == "agent99_typed_dispatch_not_allowed"
assert result["dispatchPerformed"] is False
assert result["runtimeClosureVerified"] is False
assert result["typedTargetRoute"]["resolution_status"] == (
"asset_identity_unresolved"
)
assert claimed == []
assert dispatched == []
@pytest.mark.asyncio
async def test_agent99_bridge_rejects_open_circuit_before_claim_or_transport(
monkeypatch,
) -> None:
claimed: list[dict[str, object]] = []
dispatched: list[dict[str, object]] = []
async def unexpected_claim(*_args, **kwargs) -> dict[str, object]:
claimed.append(kwargs)
return {"acquired": True, "reason": "unexpected"}
monkeypatch.setattr(
"src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
unexpected_claim,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda payload: dispatched.append(payload) or {},
)
result = await bridge_alertmanager_to_agent99(
alert_id="agent99-circuit-open",
alertname="Agent99ServiceUnhealthy",
severity="critical",
namespace="windows-control-plane",
target_resource="Agent99",
message="Agent99 control plane health check failed",
labels={"host": "192.168.0.99", "circuit_state": "open"},
fingerprint="agent99-circuit-open-fingerprint",
)
assert result["status"] == "failed"
assert result["reason"] == "agent99_circuit_open_fail_closed"
assert result["dispatchPerformed"] is False
assert result["runtimeClosureVerified"] is False
assert result["circuitState"] == "open"
assert result["repairWorkItemRequired"] is True
assert result["typedTargetRoute"]["canonical_asset_id"] == (
"windows-vmware:host_99"
)
assert claimed == []
assert dispatched == []
@pytest.mark.asyncio
async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) -> None:
async def duplicate(*_args, **_kwargs) -> dict[str, object]:
@@ -446,11 +531,11 @@ async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) ->
result = await bridge_alertmanager_to_agent99(
alert_id="duplicate-alert",
alertname="UnknownObservation",
alertname="Agent99ServiceUnhealthy",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
message="observation only",
namespace="windows-control-plane",
target_resource="Agent99",
message="Agent99 control plane health check failed",
fingerprint="same-fingerprint",
)
@@ -484,11 +569,11 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No
result = await bridge_alertmanager_to_agent99(
alert_id="winner-alert",
alertname="UnknownObservation",
alertname="Agent99ServiceUnhealthy",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
message="observation only",
namespace="windows-control-plane",
target_resource="Agent99",
message="Agent99 control plane health check failed",
fingerprint="winner-fingerprint",
)
@@ -496,7 +581,7 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No
assert result["dispatch"] == "relay"
assert result["dispatchReceipt"]["accepted"] is True
assert result["dispatchReceipt"]["inbox_triggered"] is True
assert result["suggestedMode"] == "Status"
assert result["suggestedMode"] == "SelfCheck"
assert len(dispatched) == 1
@@ -533,11 +618,11 @@ async def test_agent99_bridge_releases_lock_when_receipt_is_not_accepted(
result = await bridge_alertmanager_to_agent99(
alert_id="receipt-rejected",
alertname="UnknownObservation",
alertname="Agent99ServiceUnhealthy",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
message="observation only",
namespace="windows-control-plane",
target_resource="Agent99",
message="Agent99 control plane health check failed",
fingerprint="receipt-rejected-fingerprint",
)
@@ -575,11 +660,11 @@ async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) ->
result = await bridge_alertmanager_to_agent99(
alert_id="transport-failure",
alertname="UnknownObservation",
alertname="Agent99ServiceUnhealthy",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
message="observation only",
namespace="windows-control-plane",
target_resource="Agent99",
message="Agent99 control plane health check failed",
fingerprint="transport-fingerprint",
)

10
apps/api/tests/test_sre_typed_domain_router.py
View File

@@ -557,11 +557,17 @@ def test_backup_restore_is_critical_readback_only() -> None:
)
assert route["target_kind"] == "backup_restore"
assert route["executor"] == "Agent99"
assert route["break_glass_executor"] == "backup_restore_break_glass"
assert route["risk_class"] == "critical"
assert route["critical_break_glass_required"] is True
assert route["controlled_apply_allowed"] is False
assert route["allowed_catalog_ids"] == []
assert route["agent99_bridge"]["dispatch_allowed"] is False
assert route["verifier"] == "backup_restore_readback_verifier"
assert route["agent99_bridge"]["dispatch_allowed"] is True
assert route["agent99_bridge"]["execution_role"] == (
"single_writer_agent99_backup_readback_collector"
)
def test_registry_auto_flag_cannot_override_critical_backup_break_glass() -> None:
@@ -591,6 +597,8 @@ def test_registry_auto_flag_cannot_override_critical_backup_break_glass() -> Non
assert route["resolution_status"] == "resolved"
assert route["target_kind"] == "backup_restore"
assert route["executor"] == "Agent99"
assert route["break_glass_executor"] == "backup_restore_break_glass"
assert route["risk_class"] == "critical"
assert route["critical_break_glass_required"] is True
assert route["controlled_apply_allowed"] is False