diff --git a/apps/api/src/services/agent99_controlled_dispatch_ledger.py b/apps/api/src/services/agent99_controlled_dispatch_ledger.py index 558d9d6b1..6656d9c31 100644 --- a/apps/api/src/services/agent99_controlled_dispatch_ledger.py +++ b/apps/api/src/services/agent99_controlled_dispatch_ledger.py @@ -314,11 +314,12 @@ def build_agent99_dispatch_scope( canonical_asset_id: Any = "", typed_domain: Any = "", executor: Any = "", + break_glass_executor: Any = "", verifier: Any = "", ) -> dict[str, Any]: """Build the bounded typed scope persisted with one durable run.""" - return { + scope = { "schema_version": "agent99_dispatch_scope_v1", "kind": str(kind or "")[:120], "suggested_mode": str(suggested_mode or "")[:80], @@ -332,6 +333,10 @@ def build_agent99_dispatch_scope( # the same-run source of truth for route binding. "route_id": identity.route_id, } + normalized_break_glass_executor = str(break_glass_executor or "")[:160] + if normalized_break_glass_executor: + scope["break_glass_executor"] = normalized_break_glass_executor + return scope def build_agent99_dispatch_receipt_envelope( @@ -384,6 +389,7 @@ def build_agent99_dispatch_receipt_envelope( canonical_asset_id=supplied_scope.get("canonical_asset_id"), typed_domain=supplied_scope.get("typed_domain"), executor=supplied_scope.get("executor"), + break_glass_executor=supplied_scope.get("break_glass_executor"), verifier=supplied_scope.get("verifier"), ) stage_identity = { @@ -644,6 +650,9 @@ class PostgresAgent99DispatchLedger: canonical_asset_id=typed_target_route.get("canonical_asset_id"), typed_domain=typed_target_route.get("target_kind"), executor=typed_target_route.get("executor"), + break_glass_executor=typed_target_route.get( + "break_glass_executor" + ), verifier=typed_target_route.get("verifier"), ) input_payload = { diff --git a/apps/api/src/services/agent99_sre_bridge.py b/apps/api/src/services/agent99_sre_bridge.py index 40a1d2da9..e0b362c37 100644 --- a/apps/api/src/services/agent99_sre_bridge.py +++ b/apps/api/src/services/agent99_sre_bridge.py @@ -919,7 +919,7 @@ def _agent99_dispatch_scope_from_payload( if isinstance(awoooi.get("agent99DispatchIdentity"), dict) else {} ) - return { + scope = { "schema_version": "agent99_dispatch_scope_v1", "kind": _safe_text(payload.get("kind"), max_length=120), "suggested_mode": _safe_text( @@ -941,6 +941,12 @@ def _agent99_dispatch_scope_from_payload( "verifier": _safe_text(typed_target.get("verifier"), max_length=160), "route_id": _safe_text(identity.get("route_id"), max_length=240), } + break_glass_executor = _safe_text( + typed_target.get("break_glass_executor"), max_length=160 + ) + if break_glass_executor: + scope["break_glass_executor"] = break_glass_executor + return scope def dispatch_agent99_sre_alert_with_receipt( @@ -1179,37 +1185,42 @@ async def bridge_alertmanager_to_agent99( ) suggested_mode = str(payload.get("suggestedMode") or "Status") mutating = bool(payload.get("controlledApply") is True) - typed_dispatch_allowed = bool( - routing.get("typedDispatchAllowed") is True + typed_target_route = ( + routing.get("typedTargetRoute") + if isinstance(routing.get("typedTargetRoute"), dict) + else {} ) - if mutating and not typed_dispatch_allowed: - typed_target_route = ( - routing.get("typedTargetRoute") - if isinstance(routing.get("typedTargetRoute"), dict) - else {} - ) + source_labels = labels if isinstance(labels, dict) else {} + source_annotations = annotations if isinstance(annotations, dict) else {} + circuit_state = str( + source_labels.get("circuit_state") + or source_annotations.get("circuit_state") + or "" + ).strip().lower() + circuit_open_flag = str( + source_labels.get("circuit_open") + or source_annotations.get("circuit_open") + or "" + ).strip().lower() in {"1", "true", "yes", "open"} + if circuit_state == "open" or circuit_open_flag: logger.warning( - "agent99_mutating_dispatch_typed_domain_denied", + "agent99_dispatch_circuit_open_denied", alert_id=alert_id, alertname=alertname, - target_kind=typed_target_route.get("target_kind"), canonical_asset_id=typed_target_route.get( "canonical_asset_id" ), - drift_work_item_id=typed_target_route.get( - "drift_work_item_id" - ), - reason="agent99_typed_dispatch_not_allowed", + target_kind=typed_target_route.get("target_kind"), + reason="agent99_circuit_open_fail_closed", ) return { "status": "failed", - "reason": "agent99_typed_dispatch_not_allowed", + "reason": "agent99_circuit_open_fail_closed", "dispatchPerformed": False, "runtimeClosureVerified": False, + "circuitState": "open", + "repairWorkItemRequired": True, "typedTargetRoute": typed_target_route, - "driftWorkItemId": typed_target_route.get( - "drift_work_item_id" - ), } resolved_route_id = ( str(route_id or "").strip() @@ -1258,6 +1269,34 @@ async def bridge_alertmanager_to_agent99( "drift_work_item_id" ), } + typed_dispatch_allowed = bool( + routing.get("typedDispatchAllowed") is True + ) + if not typed_dispatch_allowed: + logger.warning( + "agent99_dispatch_typed_domain_denied", + alert_id=alert_id, + alertname=alertname, + controlled_apply_requested=mutating, + target_kind=typed_target_route.get("target_kind"), + canonical_asset_id=typed_target_route.get( + "canonical_asset_id" + ), + drift_work_item_id=typed_target_route.get( + "drift_work_item_id" + ), + reason="agent99_typed_dispatch_not_allowed", + ) + return { + "status": "failed", + "reason": "agent99_typed_dispatch_not_allowed", + "dispatchPerformed": False, + "runtimeClosureVerified": False, + "typedTargetRoute": typed_target_route, + "driftWorkItemId": typed_target_route.get( + "drift_work_item_id" + ), + } durable_route = bool( str(incident_id or "").strip() or str(route_id or "").strip() diff --git a/apps/api/src/services/controlled_alert_target_router.py b/apps/api/src/services/controlled_alert_target_router.py index 101f5ee4a..a6bbb95f1 100644 --- a/apps/api/src/services/controlled_alert_target_router.py +++ b/apps/api/src/services/controlled_alert_target_router.py @@ -303,6 +303,7 @@ def _typed_route( drift_work_item_id: str | None = None, stateful_level: str | None = None, controlled_apply_allowed: bool | None = None, + break_glass_executor: str | None = None, execution_role: str = "policy_observer_and_no_secret_dispatch_relay", ) -> dict[str, Any]: resolved = resolution_status == "resolved" @@ -314,6 +315,9 @@ def _typed_route( agent99_dispatch = resolved and target_kind in { "control_plane_recovery", "windows_vmware", + # Agent99 is only the read-only evidence collector for this domain; + # backup/restore mutation remains on the critical break-glass executor. + "backup_restore", } return { "schema_version": "typed_domain_target_route_v2", @@ -328,6 +332,7 @@ def _typed_route( "host": host, "stateful_level": stateful_level, "executor": executor, + "break_glass_executor": break_glass_executor, "verifier": verifier, "risk_class": risk_class, "allowed_catalog_ids": list(allowed_catalog_ids or []), @@ -573,10 +578,12 @@ def resolve_typed_alert_target( identity.get("canonical_id") or "data-protection:backup-restore" ), host=identity.get("host"), - executor="backup_restore_break_glass", + executor="Agent99", + break_glass_executor="backup_restore_break_glass", verifier="backup_restore_readback_verifier", risk_class="critical", stateful_level=identity.get("stateful_level"), + execution_role="single_writer_agent99_backup_readback_collector", ) if any(marker in text for marker in _WINDOWS_VMWARE_MARKERS): @@ -784,6 +791,7 @@ def resolve_typed_alert_target( in {StatefulLevel.BLOCK.value, StatefulLevel.CRITICAL_HITL.value} else "medium" ) + read_only_backup_route = domain == "backup_restore" return _typed_route( resolution_status="resolved", target_kind=domain, @@ -791,7 +799,16 @@ def resolve_typed_alert_target( namespace=namespace, canonical_asset_id=str(identity["canonical_id"]), host=identity.get("host"), - executor=str(identity.get("executor") or "") or None, + executor=( + "Agent99" + if read_only_backup_route + else str(identity.get("executor") or "") or None + ), + break_glass_executor=( + str(identity.get("executor") or "backup_restore_break_glass") + if read_only_backup_route + else None + ), verifier=str(identity.get("verifier") or "") or None, risk_class=risk, allowed_catalog_ids=catalog_ids, @@ -800,6 +817,11 @@ def resolve_typed_alert_target( controlled_apply_allowed=bool( identity.get("controlled_apply_allowed") ), + execution_role=( + "single_writer_agent99_backup_readback_collector" + if read_only_backup_route + else "policy_observer_and_no_secret_dispatch_relay" + ), ) identity_evidence = _kubernetes_identity_evidence( diff --git a/apps/api/tests/test_agent99_controlled_dispatch_p1.py b/apps/api/tests/test_agent99_controlled_dispatch_p1.py index 36d2853d2..918ce1280 100644 --- a/apps/api/tests/test_agent99_controlled_dispatch_p1.py +++ b/apps/api/tests/test_agent99_controlled_dispatch_p1.py @@ -15,12 +15,17 @@ from src.services import agent99_controlled_dispatch_ledger as ledger_module from src.services import agent99_outcome_ingestion as ingestion_module from src.services.agent99_controlled_dispatch_ledger import ( PostgresAgent99DispatchLedger, + attach_agent99_dispatch_identity, build_agent99_dispatch_identity, build_agent99_dispatch_receipt_envelope, parse_agent99_dispatch_identity, record_agent99_learning_writeback, ) -from src.services.agent99_sre_bridge import bridge_alertmanager_to_agent99 +from src.services.agent99_sre_bridge import ( + _agent99_dispatch_scope_from_payload, + bridge_alertmanager_to_agent99, + build_agent99_sre_alert, +) def _identity(): @@ -491,6 +496,115 @@ async def test_verifier_requires_transport_and_all_evidence(monkeypatch) -> None assert passed["runtime_closure_verified"] is False +@pytest.mark.asyncio +async def test_backupcheck_read_only_scope_passes_real_ledger_verifier( + monkeypatch, +) -> None: + identity = build_agent99_dispatch_identity( + project_id="awoooi", + incident_id="INC-20260711-BACKUP", + source_fingerprint="backup-fingerprint", + route_id="agent99:backup_health:BackupCheck", + execution_generation="1", + work_item_id=( + "agent99-dispatch:awoooi:INC-20260711-BACKUP:BackupCheck" + ), + ) + payload = build_agent99_sre_alert( + alert_id="backup-readback", + alertname="BackupCredentialEscrowEvidenceMissing", + severity="warning", + namespace="awoooi-prod", + target_resource="backup_restore", + message="freshness escrow restore evidence required", + labels={"event_type": "backup_restore_escrow_signal"}, + fingerprint="backup-fingerprint", + ) + payload = attach_agent99_dispatch_identity(payload, identity) + scope = _agent99_dispatch_scope_from_payload(payload) + assert scope["executor"] == "Agent99" + assert scope["break_glass_executor"] == "backup_restore_break_glass" + assert scope["controlled_apply_requested"] is False + + promoted = build_agent99_dispatch_receipt_envelope( + identity=identity, + dispatch_receipt={ + "kind": "backup_health", + "suggested_mode": "BackupCheck", + "target_resource": "backup_restore", + "controlled_apply_requested": False, + "accepted": True, + "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, + "delivery_certainty": "delivered", + "dispatch_scope": scope, + }, + controlled_apply_authorized=False, + ) + outcome = { + "identity": identity.public_dict(), + "controlledApply": False, + "mode": "BackupCheck", + "outcome": { + "identity": identity.public_dict(), + "schemaVersion": "agent99_outcome_contract_v1", + "state": "resolved", + "transportOk": True, + "verifierName": "backup_restore_readback_verifier", + "verifierPassed": True, + "sourceEventResolved": True, + "verifiedAt": "2026-07-11T20:00:00+08:00", + }, + } + + class BackupVerifierDB: + def __init__(self) -> None: + self.call = 0 + + async def execute(self, _statement): + self.call += 1 + if self.call == 1: + return _ScalarResult( + row=SimpleNamespace( + state="waiting_tool", + error_detail=json.dumps(promoted), + ) + ) + if self.call == 2: + return _ScalarResult(identity.run_id) + if self.call == 4: + return _ScalarResult(identity.incident_id) + if self.call in {5, 6}: + return _ScalarResult(identity.run_id) + return _ScalarResult() + + monkeypatch.setattr( + ledger_module, + "get_db_context", + lambda _project_id: _Context(BackupVerifierDB()), + ) + + result = await PostgresAgent99DispatchLedger().record_verifier( + identity=identity, + outcome_receipt=outcome, + evidence_refs={ + **_evidence_refs(), + "backup_status_evidence_ref": "backup-status:1", + "freshness_evidence_ref": "backup-freshness:1", + "offsite_verify_evidence_ref": "offsite-verify:1", + "escrow_evidence_ref": "escrow-readback:1", + "restore_drill_evidence_ref": "restore-drill:1", + "source_resolution_receipt_ref": "alert-resolved:1", + }, + ) + + assert result["status"] == "verifier_passed_learning_writeback_pending" + assert result["post_verifier_passed"] is True + assert result["controlled_apply_authorized"] is False + assert result["runtime_closure_verified"] is False + + @pytest.mark.asyncio @pytest.mark.parametrize("mismatch", ["verifier", "mode", "controlled_apply"]) async def test_verifier_rejects_dispatch_scope_mismatch( diff --git a/apps/api/tests/test_agent99_sre_bridge.py b/apps/api/tests/test_agent99_sre_bridge.py index 7a9ad5c6d..d9dba58a4 100644 --- a/apps/api/tests/test_agent99_sre_bridge.py +++ b/apps/api/tests/test_agent99_sre_bridge.py @@ -421,6 +421,91 @@ def test_agent99_single_flight_key_does_not_expose_fingerprint() -> None: assert "private" not in key +@pytest.mark.asyncio +async def test_agent99_bridge_rejects_unknown_read_only_before_claim_or_transport( + monkeypatch, +) -> None: + claimed: list[dict[str, object]] = [] + dispatched: list[dict[str, object]] = [] + + async def unexpected_claim(*_args, **kwargs) -> dict[str, object]: + claimed.append(kwargs) + return {"acquired": True, "reason": "unexpected"} + + monkeypatch.setattr( + "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight", + unexpected_claim, + ) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", + lambda payload: dispatched.append(payload) or {}, + ) + + result = await bridge_alertmanager_to_agent99( + alert_id="unknown-read-only", + alertname="UnknownObservation", + severity="warning", + namespace="node", + target_resource="192.168.0.120", + message="observation only", + fingerprint="unknown-read-only-fingerprint", + ) + + assert result["status"] == "failed" + assert result["reason"] == "agent99_typed_dispatch_not_allowed" + assert result["dispatchPerformed"] is False + assert result["runtimeClosureVerified"] is False + assert result["typedTargetRoute"]["resolution_status"] == ( + "asset_identity_unresolved" + ) + assert claimed == [] + assert dispatched == [] + + +@pytest.mark.asyncio +async def test_agent99_bridge_rejects_open_circuit_before_claim_or_transport( + monkeypatch, +) -> None: + claimed: list[dict[str, object]] = [] + dispatched: list[dict[str, object]] = [] + + async def unexpected_claim(*_args, **kwargs) -> dict[str, object]: + claimed.append(kwargs) + return {"acquired": True, "reason": "unexpected"} + + monkeypatch.setattr( + "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight", + unexpected_claim, + ) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", + lambda payload: dispatched.append(payload) or {}, + ) + + result = await bridge_alertmanager_to_agent99( + alert_id="agent99-circuit-open", + alertname="Agent99ServiceUnhealthy", + severity="critical", + namespace="windows-control-plane", + target_resource="Agent99", + message="Agent99 control plane health check failed", + labels={"host": "192.168.0.99", "circuit_state": "open"}, + fingerprint="agent99-circuit-open-fingerprint", + ) + + assert result["status"] == "failed" + assert result["reason"] == "agent99_circuit_open_fail_closed" + assert result["dispatchPerformed"] is False + assert result["runtimeClosureVerified"] is False + assert result["circuitState"] == "open" + assert result["repairWorkItemRequired"] is True + assert result["typedTargetRoute"]["canonical_asset_id"] == ( + "windows-vmware:host_99" + ) + assert claimed == [] + assert dispatched == [] + + @pytest.mark.asyncio async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) -> None: async def duplicate(*_args, **_kwargs) -> dict[str, object]: @@ -446,11 +531,11 @@ async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) -> result = await bridge_alertmanager_to_agent99( alert_id="duplicate-alert", - alertname="UnknownObservation", + alertname="Agent99ServiceUnhealthy", severity="critical", - namespace="node", - target_resource="192.168.0.120", - message="observation only", + namespace="windows-control-plane", + target_resource="Agent99", + message="Agent99 control plane health check failed", fingerprint="same-fingerprint", ) @@ -484,11 +569,11 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No result = await bridge_alertmanager_to_agent99( alert_id="winner-alert", - alertname="UnknownObservation", + alertname="Agent99ServiceUnhealthy", severity="critical", - namespace="node", - target_resource="192.168.0.120", - message="observation only", + namespace="windows-control-plane", + target_resource="Agent99", + message="Agent99 control plane health check failed", fingerprint="winner-fingerprint", ) @@ -496,7 +581,7 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No assert result["dispatch"] == "relay" assert result["dispatchReceipt"]["accepted"] is True assert result["dispatchReceipt"]["inbox_triggered"] is True - assert result["suggestedMode"] == "Status" + assert result["suggestedMode"] == "SelfCheck" assert len(dispatched) == 1 @@ -533,11 +618,11 @@ async def test_agent99_bridge_releases_lock_when_receipt_is_not_accepted( result = await bridge_alertmanager_to_agent99( alert_id="receipt-rejected", - alertname="UnknownObservation", + alertname="Agent99ServiceUnhealthy", severity="critical", - namespace="node", - target_resource="192.168.0.120", - message="observation only", + namespace="windows-control-plane", + target_resource="Agent99", + message="Agent99 control plane health check failed", fingerprint="receipt-rejected-fingerprint", ) @@ -575,11 +660,11 @@ async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) -> result = await bridge_alertmanager_to_agent99( alert_id="transport-failure", - alertname="UnknownObservation", + alertname="Agent99ServiceUnhealthy", severity="critical", - namespace="node", - target_resource="192.168.0.120", - message="observation only", + namespace="windows-control-plane", + target_resource="Agent99", + message="Agent99 control plane health check failed", fingerprint="transport-fingerprint", ) diff --git a/apps/api/tests/test_sre_typed_domain_router.py b/apps/api/tests/test_sre_typed_domain_router.py index 1e891721a..a2d57cd17 100644 --- a/apps/api/tests/test_sre_typed_domain_router.py +++ b/apps/api/tests/test_sre_typed_domain_router.py @@ -557,11 +557,17 @@ def test_backup_restore_is_critical_readback_only() -> None: ) assert route["target_kind"] == "backup_restore" + assert route["executor"] == "Agent99" + assert route["break_glass_executor"] == "backup_restore_break_glass" assert route["risk_class"] == "critical" assert route["critical_break_glass_required"] is True assert route["controlled_apply_allowed"] is False assert route["allowed_catalog_ids"] == [] - assert route["agent99_bridge"]["dispatch_allowed"] is False + assert route["verifier"] == "backup_restore_readback_verifier" + assert route["agent99_bridge"]["dispatch_allowed"] is True + assert route["agent99_bridge"]["execution_role"] == ( + "single_writer_agent99_backup_readback_collector" + ) def test_registry_auto_flag_cannot_override_critical_backup_break_glass() -> None: @@ -591,6 +597,8 @@ def test_registry_auto_flag_cannot_override_critical_backup_break_glass() -> Non assert route["resolution_status"] == "resolved" assert route["target_kind"] == "backup_restore" + assert route["executor"] == "Agent99" + assert route["break_glass_executor"] == "backup_restore_break_glass" assert route["risk_class"] == "critical" assert route["critical_break_glass_required"] is True assert route["controlled_apply_allowed"] is False diff --git a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json index d9512e06f..5bcddd744 100644 --- a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json +++ b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json @@ -400,20 +400,30 @@ ], "source_refs": [ "apps/api/src/services/controlled_alert_target_router.py", + "apps/api/src/services/agent99_sre_bridge.py", + "apps/api/src/services/agent99_controlled_dispatch_ledger.py", "apps/api/src/services/awooop_ansible_audit_service.py", "apps/api/src/services/awooop_ansible_check_mode_service.py", "apps/api/src/api/v1/agents.py", + "apps/api/tests/test_agent99_sre_bridge.py", + "apps/api/tests/test_agent99_controlled_dispatch_p1.py", + "apps/api/tests/test_sre_typed_domain_router.py", "ops/monitoring/alerts-unified.yml" ], "executor": "typed policy router", "verifier": "domain/catalog/host/canonical ID scope verifier", "rollback": "revert router source while keeping unknown fail-closed", "exit_condition": "every candidate is exact-domain scoped; circuit-open never selects another host/domain; GCP-A/GCP-B unsanitized prompts and every cloud tool loop fail closed before a network call", + "confirmed_truth": [ + "Agent99 admission rejects unresolved and cross-domain routes for both read-only and controlled-apply events before single-flight claim or transport", + "Explicit circuit_state=open or circuit_open=true is terminal before Agent99 claim/transport and requires a same-domain repair work item instead of fallback", + "BackupCheck persists executor=Agent99 for the exact read-only collector receipt and separately records break_glass_executor=backup_restore_break_glass; controlled apply remains false and the real ledger verifier accepts only the identity-bound BackupCheck evidence set" + ], "runtime_gaps": [ "source tests do not prove production router enforcement", "Agent99 exact-host Alertmanager pull and reduced HTTPS relay are source-implemented but have no host99 production receipt or primary-path outage verifier" ], - "next_action": "run critical override, AlertChainBroken exact-host, cloud sanitized-input/tool-loop fail-closed and full Ansible replay regressions; then verify the same policy in production" + "next_action": "deploy the typed admission gate, then replay one unknown read-only event, one explicit circuit-open event and one BackupCheck event; prove zero Agent99 transport for the first two and an identity-bound read-only receipt for BackupCheck before broader production policy verification" }, { "id": "AIA-SRE-005",