wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(sre): fail closed typed Agent99 dispatch

Browse Source
This commit is contained in:
Your Name
2026-07-19 03:21:22 +08:00
parent 3223f9c913
commit ba4e5c41a7
7 changed files with 329 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
apps/api/src/services/agent99_controlled_dispatch_ledger.py
View File

@@ -314,11 +314,12 @@ def build_agent99_dispatch_scope(
canonical_asset_id: Any = "",
typed_domain: Any = "",
executor: Any = "",
break_glass_executor: Any = "",
verifier: Any = "",
) -> dict[str, Any]:
"""Build the bounded typed scope persisted with one durable run."""
return {
scope = {
"schema_version": "agent99_dispatch_scope_v1",
"kind": str(kind or "")[:120],
"suggested_mode": str(suggested_mode or "")[:80],
@@ -332,6 +333,10 @@ def build_agent99_dispatch_scope(
# the same-run source of truth for route binding.
"route_id": identity.route_id,
}
normalized_break_glass_executor = str(break_glass_executor or "")[:160]
if normalized_break_glass_executor:
scope["break_glass_executor"] = normalized_break_glass_executor
return scope
def build_agent99_dispatch_receipt_envelope(
@@ -384,6 +389,7 @@ def build_agent99_dispatch_receipt_envelope(
canonical_asset_id=supplied_scope.get("canonical_asset_id"),
typed_domain=supplied_scope.get("typed_domain"),
executor=supplied_scope.get("executor"),
break_glass_executor=supplied_scope.get("break_glass_executor"),
verifier=supplied_scope.get("verifier"),
)
stage_identity = {
@@ -644,6 +650,9 @@ class PostgresAgent99DispatchLedger:
canonical_asset_id=typed_target_route.get("canonical_asset_id"),
typed_domain=typed_target_route.get("target_kind"),
executor=typed_target_route.get("executor"),
break_glass_executor=typed_target_route.get(
"break_glass_executor"
),
verifier=typed_target_route.get("verifier"),
)
input_payload = {

77
apps/api/src/services/agent99_sre_bridge.py
View File

@@ -919,7 +919,7 @@ def _agent99_dispatch_scope_from_payload(
if isinstance(awoooi.get("agent99DispatchIdentity"), dict)
else {}
)
return {
scope = {
"schema_version": "agent99_dispatch_scope_v1",
"kind": _safe_text(payload.get("kind"), max_length=120),
"suggested_mode": _safe_text(
@@ -941,6 +941,12 @@ def _agent99_dispatch_scope_from_payload(
"verifier": _safe_text(typed_target.get("verifier"), max_length=160),
"route_id": _safe_text(identity.get("route_id"), max_length=240),
}
break_glass_executor = _safe_text(
typed_target.get("break_glass_executor"), max_length=160
)
if break_glass_executor:
scope["break_glass_executor"] = break_glass_executor
return scope
def dispatch_agent99_sre_alert_with_receipt(
@@ -1179,37 +1185,42 @@ async def bridge_alertmanager_to_agent99(
)
suggested_mode = str(payload.get("suggestedMode") or "Status")
mutating = bool(payload.get("controlledApply") is True)
typed_dispatch_allowed = bool(
routing.get("typedDispatchAllowed") is True
typed_target_route = (
routing.get("typedTargetRoute")
if isinstance(routing.get("typedTargetRoute"), dict)
else {}
)
if mutating and not typed_dispatch_allowed:
typed_target_route = (
routing.get("typedTargetRoute")
if isinstance(routing.get("typedTargetRoute"), dict)
else {}
)
source_labels = labels if isinstance(labels, dict) else {}
source_annotations = annotations if isinstance(annotations, dict) else {}
circuit_state = str(
source_labels.get("circuit_state")
or source_annotations.get("circuit_state")
or ""
).strip().lower()
circuit_open_flag = str(
source_labels.get("circuit_open")
or source_annotations.get("circuit_open")
or ""
).strip().lower() in {"1", "true", "yes", "open"}
if circuit_state == "open" or circuit_open_flag:
logger.warning(
"agent99_mutating_dispatch_typed_domain_denied",
"agent99_dispatch_circuit_open_denied",
alert_id=alert_id,
alertname=alertname,
target_kind=typed_target_route.get("target_kind"),
canonical_asset_id=typed_target_route.get(
"canonical_asset_id"
),
drift_work_item_id=typed_target_route.get(
"drift_work_item_id"
),
reason="agent99_typed_dispatch_not_allowed",
target_kind=typed_target_route.get("target_kind"),
reason="agent99_circuit_open_fail_closed",
)
return {
"status": "failed",
"reason": "agent99_typed_dispatch_not_allowed",
"reason": "agent99_circuit_open_fail_closed",
"dispatchPerformed": False,
"runtimeClosureVerified": False,
"circuitState": "open",
"repairWorkItemRequired": True,
"typedTargetRoute": typed_target_route,
"driftWorkItemId": typed_target_route.get(
"drift_work_item_id"
),
}
resolved_route_id = (
str(route_id or "").strip()
@@ -1258,6 +1269,34 @@ async def bridge_alertmanager_to_agent99(
"drift_work_item_id"
),
}
typed_dispatch_allowed = bool(
routing.get("typedDispatchAllowed") is True
)
if not typed_dispatch_allowed:
logger.warning(
"agent99_dispatch_typed_domain_denied",
alert_id=alert_id,
alertname=alertname,
controlled_apply_requested=mutating,
target_kind=typed_target_route.get("target_kind"),
canonical_asset_id=typed_target_route.get(
"canonical_asset_id"
),
drift_work_item_id=typed_target_route.get(
"drift_work_item_id"
),
reason="agent99_typed_dispatch_not_allowed",
)
return {
"status": "failed",
"reason": "agent99_typed_dispatch_not_allowed",
"dispatchPerformed": False,
"runtimeClosureVerified": False,
"typedTargetRoute": typed_target_route,
"driftWorkItemId": typed_target_route.get(
"drift_work_item_id"
),
}
durable_route = bool(
str(incident_id or "").strip()
or str(route_id or "").strip()

26
apps/api/src/services/controlled_alert_target_router.py
View File

@@ -303,6 +303,7 @@ def _typed_route(
drift_work_item_id: str | None = None,
stateful_level: str | None = None,
controlled_apply_allowed: bool | None = None,
break_glass_executor: str | None = None,
execution_role: str = "policy_observer_and_no_secret_dispatch_relay",
) -> dict[str, Any]:
resolved = resolution_status == "resolved"
@@ -314,6 +315,9 @@ def _typed_route(
agent99_dispatch = resolved and target_kind in {
"control_plane_recovery",
"windows_vmware",
# Agent99 is only the read-only evidence collector for this domain;
# backup/restore mutation remains on the critical break-glass executor.
"backup_restore",
}
return {
"schema_version": "typed_domain_target_route_v2",
@@ -328,6 +332,7 @@ def _typed_route(
"host": host,
"stateful_level": stateful_level,
"executor": executor,
"break_glass_executor": break_glass_executor,
"verifier": verifier,
"risk_class": risk_class,
"allowed_catalog_ids": list(allowed_catalog_ids or []),
@@ -573,10 +578,12 @@ def resolve_typed_alert_target(
identity.get("canonical_id") or "data-protection:backup-restore"
),
host=identity.get("host"),
executor="backup_restore_break_glass",
executor="Agent99",
break_glass_executor="backup_restore_break_glass",
verifier="backup_restore_readback_verifier",
risk_class="critical",
stateful_level=identity.get("stateful_level"),
execution_role="single_writer_agent99_backup_readback_collector",
)
if any(marker in text for marker in _WINDOWS_VMWARE_MARKERS):
@@ -784,6 +791,7 @@ def resolve_typed_alert_target(
in {StatefulLevel.BLOCK.value, StatefulLevel.CRITICAL_HITL.value}
else "medium"
)
read_only_backup_route = domain == "backup_restore"
return _typed_route(
resolution_status="resolved",
target_kind=domain,
@@ -791,7 +799,16 @@ def resolve_typed_alert_target(
namespace=namespace,
canonical_asset_id=str(identity["canonical_id"]),
host=identity.get("host"),
executor=str(identity.get("executor") or "") or None,
executor=(
"Agent99"
if read_only_backup_route
else str(identity.get("executor") or "") or None
),
break_glass_executor=(
str(identity.get("executor") or "backup_restore_break_glass")
if read_only_backup_route
else None
),
verifier=str(identity.get("verifier") or "") or None,
risk_class=risk,
allowed_catalog_ids=catalog_ids,
@@ -800,6 +817,11 @@ def resolve_typed_alert_target(
controlled_apply_allowed=bool(
identity.get("controlled_apply_allowed")
),
execution_role=(
"single_writer_agent99_backup_readback_collector"
if read_only_backup_route
else "policy_observer_and_no_secret_dispatch_relay"
),
)
identity_evidence = _kubernetes_identity_evidence(

116
apps/api/tests/test_agent99_controlled_dispatch_p1.py
View File

@@ -15,12 +15,17 @@ from src.services import agent99_controlled_dispatch_ledger as ledger_module
from src.services import agent99_outcome_ingestion as ingestion_module
from src.services.agent99_controlled_dispatch_ledger import (
PostgresAgent99DispatchLedger,
attach_agent99_dispatch_identity,
build_agent99_dispatch_identity,
build_agent99_dispatch_receipt_envelope,
parse_agent99_dispatch_identity,
record_agent99_learning_writeback,
)
from src.services.agent99_sre_bridge import bridge_alertmanager_to_agent99
from src.services.agent99_sre_bridge import (
_agent99_dispatch_scope_from_payload,
bridge_alertmanager_to_agent99,
build_agent99_sre_alert,
)
def _identity():
@@ -491,6 +496,115 @@ async def test_verifier_requires_transport_and_all_evidence(monkeypatch) -> None
assert passed["runtime_closure_verified"] is False
@pytest.mark.asyncio
async def test_backupcheck_read_only_scope_passes_real_ledger_verifier(
monkeypatch,
) -> None:
identity = build_agent99_dispatch_identity(
project_id="awoooi",
incident_id="INC-20260711-BACKUP",
source_fingerprint="backup-fingerprint",
route_id="agent99:backup_health:BackupCheck",
execution_generation="1",
work_item_id=(
"agent99-dispatch:awoooi:INC-20260711-BACKUP:BackupCheck"
),
)
payload = build_agent99_sre_alert(
alert_id="backup-readback",
alertname="BackupCredentialEscrowEvidenceMissing",
severity="warning",
namespace="awoooi-prod",
target_resource="backup_restore",
message="freshness escrow restore evidence required",
labels={"event_type": "backup_restore_escrow_signal"},
fingerprint="backup-fingerprint",
)
payload = attach_agent99_dispatch_identity(payload, identity)
scope = _agent99_dispatch_scope_from_payload(payload)
assert scope["executor"] == "Agent99"
assert scope["break_glass_executor"] == "backup_restore_break_glass"
assert scope["controlled_apply_requested"] is False
promoted = build_agent99_dispatch_receipt_envelope(
identity=identity,
dispatch_receipt={
"kind": "backup_health",
"suggested_mode": "BackupCheck",
"target_resource": "backup_restore",
"controlled_apply_requested": False,
"accepted": True,
"inbox_triggered": True,
"queue_accepted": True,
"dispatch_identity_matched": True,
"delivery_certainty": "delivered",
"dispatch_scope": scope,
},
controlled_apply_authorized=False,
)
outcome = {
"identity": identity.public_dict(),
"controlledApply": False,
"mode": "BackupCheck",
"outcome": {
"identity": identity.public_dict(),
"schemaVersion": "agent99_outcome_contract_v1",
"state": "resolved",
"transportOk": True,
"verifierName": "backup_restore_readback_verifier",
"verifierPassed": True,
"sourceEventResolved": True,
"verifiedAt": "2026-07-11T20:00:00+08:00",
},
}
class BackupVerifierDB:
def __init__(self) -> None:
self.call = 0
async def execute(self, _statement):
self.call += 1
if self.call == 1:
return _ScalarResult(
row=SimpleNamespace(
state="waiting_tool",
error_detail=json.dumps(promoted),
)
)
if self.call == 2:
return _ScalarResult(identity.run_id)
if self.call == 4:
return _ScalarResult(identity.incident_id)
if self.call in {5, 6}:
return _ScalarResult(identity.run_id)
return _ScalarResult()
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(BackupVerifierDB()),
)
result = await PostgresAgent99DispatchLedger().record_verifier(
identity=identity,
outcome_receipt=outcome,
evidence_refs={
**_evidence_refs(),
"backup_status_evidence_ref": "backup-status:1",
"freshness_evidence_ref": "backup-freshness:1",
"offsite_verify_evidence_ref": "offsite-verify:1",
"escrow_evidence_ref": "escrow-readback:1",
"restore_drill_evidence_ref": "restore-drill:1",
"source_resolution_receipt_ref": "alert-resolved:1",
},
)
assert result["status"] == "verifier_passed_learning_writeback_pending"
assert result["post_verifier_passed"] is True
assert result["controlled_apply_authorized"] is False
assert result["runtime_closure_verified"] is False
@pytest.mark.asyncio
@pytest.mark.parametrize("mismatch", ["verifier", "mode", "controlled_apply"])
async def test_verifier_rejects_dispatch_scope_mismatch(

119
apps/api/tests/test_agent99_sre_bridge.py
View File

@@ -421,6 +421,91 @@ def test_agent99_single_flight_key_does_not_expose_fingerprint() -> None:
assert "private" not in key
@pytest.mark.asyncio
async def test_agent99_bridge_rejects_unknown_read_only_before_claim_or_transport(
monkeypatch,
) -> None:
claimed: list[dict[str, object]] = []
dispatched: list[dict[str, object]] = []
async def unexpected_claim(*_args, **kwargs) -> dict[str, object]:
claimed.append(kwargs)
return {"acquired": True, "reason": "unexpected"}
monkeypatch.setattr(
"src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
unexpected_claim,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda payload: dispatched.append(payload) or {},
)
result = await bridge_alertmanager_to_agent99(
alert_id="unknown-read-only",
alertname="UnknownObservation",
severity="warning",
namespace="node",
target_resource="192.168.0.120",
message="observation only",
fingerprint="unknown-read-only-fingerprint",
)
assert result["status"] == "failed"
assert result["reason"] == "agent99_typed_dispatch_not_allowed"
assert result["dispatchPerformed"] is False
assert result["runtimeClosureVerified"] is False
assert result["typedTargetRoute"]["resolution_status"] == (
"asset_identity_unresolved"
)
assert claimed == []
assert dispatched == []
@pytest.mark.asyncio
async def test_agent99_bridge_rejects_open_circuit_before_claim_or_transport(
monkeypatch,
) -> None:
claimed: list[dict[str, object]] = []
dispatched: list[dict[str, object]] = []
async def unexpected_claim(*_args, **kwargs) -> dict[str, object]:
claimed.append(kwargs)
return {"acquired": True, "reason": "unexpected"}
monkeypatch.setattr(
"src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
unexpected_claim,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda payload: dispatched.append(payload) or {},
)
result = await bridge_alertmanager_to_agent99(
alert_id="agent99-circuit-open",
alertname="Agent99ServiceUnhealthy",
severity="critical",
namespace="windows-control-plane",
target_resource="Agent99",
message="Agent99 control plane health check failed",
labels={"host": "192.168.0.99", "circuit_state": "open"},
fingerprint="agent99-circuit-open-fingerprint",
)
assert result["status"] == "failed"
assert result["reason"] == "agent99_circuit_open_fail_closed"
assert result["dispatchPerformed"] is False
assert result["runtimeClosureVerified"] is False
assert result["circuitState"] == "open"
assert result["repairWorkItemRequired"] is True
assert result["typedTargetRoute"]["canonical_asset_id"] == (
"windows-vmware:host_99"
)
assert claimed == []
assert dispatched == []
@pytest.mark.asyncio
async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) -> None:
async def duplicate(*_args, **_kwargs) -> dict[str, object]:
@@ -446,11 +531,11 @@ async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) ->
result = await bridge_alertmanager_to_agent99(
alert_id="duplicate-alert",
alertname="UnknownObservation",
alertname="Agent99ServiceUnhealthy",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
message="observation only",
namespace="windows-control-plane",
target_resource="Agent99",
message="Agent99 control plane health check failed",
fingerprint="same-fingerprint",
)
@@ -484,11 +569,11 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No
result = await bridge_alertmanager_to_agent99(
alert_id="winner-alert",
alertname="UnknownObservation",
alertname="Agent99ServiceUnhealthy",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
message="observation only",
namespace="windows-control-plane",
target_resource="Agent99",
message="Agent99 control plane health check failed",
fingerprint="winner-fingerprint",
)
@@ -496,7 +581,7 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No
assert result["dispatch"] == "relay"
assert result["dispatchReceipt"]["accepted"] is True
assert result["dispatchReceipt"]["inbox_triggered"] is True
assert result["suggestedMode"] == "Status"
assert result["suggestedMode"] == "SelfCheck"
assert len(dispatched) == 1
@@ -533,11 +618,11 @@ async def test_agent99_bridge_releases_lock_when_receipt_is_not_accepted(
result = await bridge_alertmanager_to_agent99(
alert_id="receipt-rejected",
alertname="UnknownObservation",
alertname="Agent99ServiceUnhealthy",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
message="observation only",
namespace="windows-control-plane",
target_resource="Agent99",
message="Agent99 control plane health check failed",
fingerprint="receipt-rejected-fingerprint",
)
@@ -575,11 +660,11 @@ async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) ->
result = await bridge_alertmanager_to_agent99(
alert_id="transport-failure",
alertname="UnknownObservation",
alertname="Agent99ServiceUnhealthy",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
message="observation only",
namespace="windows-control-plane",
target_resource="Agent99",
message="Agent99 control plane health check failed",
fingerprint="transport-fingerprint",
)

10
apps/api/tests/test_sre_typed_domain_router.py
View File

@@ -557,11 +557,17 @@ def test_backup_restore_is_critical_readback_only() -> None:
)
assert route["target_kind"] == "backup_restore"
assert route["executor"] == "Agent99"
assert route["break_glass_executor"] == "backup_restore_break_glass"
assert route["risk_class"] == "critical"
assert route["critical_break_glass_required"] is True
assert route["controlled_apply_allowed"] is False
assert route["allowed_catalog_ids"] == []
assert route["agent99_bridge"]["dispatch_allowed"] is False
assert route["verifier"] == "backup_restore_readback_verifier"
assert route["agent99_bridge"]["dispatch_allowed"] is True
assert route["agent99_bridge"]["execution_role"] == (
"single_writer_agent99_backup_readback_collector"
)
def test_registry_auto_flag_cannot_override_critical_backup_break_glass() -> None:
@@ -591,6 +597,8 @@ def test_registry_auto_flag_cannot_override_critical_backup_break_glass() -> Non
assert route["resolution_status"] == "resolved"
assert route["target_kind"] == "backup_restore"
assert route["executor"] == "Agent99"
assert route["break_glass_executor"] == "backup_restore_break_glass"
assert route["risk_class"] == "critical"
assert route["critical_break_glass_required"] is True
assert route["controlled_apply_allowed"] is False