diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index 0318f07c5..6c5bd5d1b 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -640,6 +640,67 @@ async def _try_auto_repair_background( if isinstance(dispatch_result.get("dispatchReceipt"), dict) else {} ) + no_write_terminal = ( + dispatch_result.get("noWriteTerminal") + if isinstance(dispatch_result.get("noWriteTerminal"), dict) + else {} + ) + no_write_verifier_contract = ( + no_write_terminal.get("verifier_contract") + if isinstance(no_write_terminal.get("verifier_contract"), dict) + else {} + ) + no_write_transport = ( + no_write_terminal.get("transport") + if isinstance(no_write_terminal.get("transport"), dict) + else {} + ) + no_write_terminal_verified = bool( + dispatch_result.get("status") == "no_write_terminal" + and dispatch_result.get("dispatchPerformed") is False + and dispatch_result.get("runtimeWritePerformed") is False + and dispatch_result.get("controlledApplyAuthorized") is False + and dispatch_result.get("runtimeClosureVerified") is False + and no_write_terminal.get("schema_version") + == "agent99_cold_start_no_write_terminal_v1" + and no_write_terminal.get("status") == "no_write_terminal" + and no_write_terminal.get("incident_id") == incident_id + and no_write_terminal.get("route_id") == route_id + and no_write_terminal.get("reason") + == "exact_inventory_scope_missing" + and no_write_terminal.get("canonical_asset_id") + == "control-plane:cold-start-gate" + and no_write_terminal.get("source_namespace") == "default" + and no_write_terminal.get("normalized_execution_namespace") + == "host_recovery" + and no_write_terminal.get("kubernetes_namespace_applicable") is False + and no_write_terminal.get("allowed_inventory_hosts") == [] + and no_write_terminal.get("runtime_write_performed") is False + and no_write_terminal.get("agent99_dispatch_performed") is False + and no_write_terminal.get("production_executor_invoked") is False + and no_write_terminal.get("incident_resolution_allowed") is False + and no_write_transport.get("claim_performed") is False + and no_write_transport.get("transport_performed") is False + and no_write_verifier_contract.get("name") + == "cold_start_no_write_terminal_contract" + and no_write_verifier_contract.get("status") + == "ready_for_independent_source_validation" + and no_write_verifier_contract.get("scope") + == "source_no_write_only" + and no_write_verifier_contract.get("runtime_verifier_applicable") is False + ) + no_write_verifier_receipt = ( + { + "schema_version": "cold_start_no_write_source_verifier_receipt_v1", + "verifier": "alert_webhook_no_write_terminal_validator", + "status": "passed_source_contract", + "producer_independent": True, + "runtime_verifier": False, + "runtime_closure_verified": False, + } + if no_write_terminal_verified + else None + ) accepted = bool(dispatch_receipt.get("accepted") is True) inbox_triggered = bool(dispatch_receipt.get("inbox_triggered") is True) dispatch_promoted = bool( @@ -657,7 +718,9 @@ async def _try_auto_repair_background( ) handoff.update({ "status": ( - "agent99_dispatch_accepted_verifier_pending" + "cold_start_no_write_terminal" + if no_write_terminal_verified + else "agent99_dispatch_accepted_verifier_pending" if dispatch_promoted and receipt_persisted else "agent99_dispatch_failed_or_receipt_missing" ), @@ -671,6 +734,8 @@ async def _try_auto_repair_background( "idempotency_key": identity.get("idempotency_key"), "dispatch_receipt": dispatch_receipt or None, "correlated_receipt": correlated_receipt or None, + "no_write_terminal": no_write_terminal or None, + "no_write_verifier_receipt": no_write_verifier_receipt, "receipt_persisted": receipt_persisted, "dispatch_accepted": accepted and receipt_persisted, "inbox_triggered": inbox_triggered and receipt_persisted, @@ -678,7 +743,9 @@ async def _try_auto_repair_background( "runtime_execution_authorized": False, "runtime_closure_verified": False, "active_blockers": ( - [ + [str(no_write_terminal.get("reason"))] + if no_write_terminal_verified + else [ "post_apply_verifier_missing", "km_playbook_writeback_missing", ] @@ -691,7 +758,9 @@ async def _try_auto_repair_background( ] ), "safe_next_action": ( - "run_independent_cold_start_verifier_then_km_playbook_writeback" + str(no_write_terminal.get("next_safe_action")) + if no_write_terminal_verified + else "run_independent_cold_start_verifier_then_km_playbook_writeback" if dispatch_promoted and receipt_persisted else "retry_new_generation_after_dispatch_transport_repair" ), @@ -709,19 +778,22 @@ async def _try_auto_repair_background( handoff["execution_priority"] = execution_priority queued = handoff.get("queued") is True - await op_log.append( + no_write_terminal = handoff.get("status") == "cold_start_no_write_terminal" + operation_receipt = await op_log.append( "AUTO_REPAIR_TRIGGERED", incident_id=incident_id, approval_id=approval_id, actor="single_writer_decision_router", action_detail=( - "controlled_check_mode_queued" + "controlled_no_write_terminal" + if no_write_terminal + else "controlled_check_mode_queued" if queued else "controlled_domain_route_pending_receipt" if handoff.get("single_writer_executor") == "Agent99" else "controlled_queue_blocked" ), - success=queued, + success=None if no_write_terminal else queued, error_message=( None if queued @@ -739,6 +811,10 @@ async def _try_auto_repair_background( "side_effect_performed": bool(handoff.get("side_effect_performed")), "dispatch_receipt": handoff.get("dispatch_receipt"), "correlated_receipt": handoff.get("correlated_receipt"), + "no_write_terminal": handoff.get("no_write_terminal"), + "no_write_verifier_receipt": handoff.get( + "no_write_verifier_receipt" + ), "runtime_closure_verified": False, "safe_next_action": ( handoff.get("safe_next_action") @@ -750,6 +826,24 @@ async def _try_auto_repair_background( ), }, ) + if no_write_terminal: + durable_receipt_id = str( + getattr(operation_receipt, "id", "") or "" + ) + if not durable_receipt_id: + handoff.update( + { + "status": "cold_start_no_write_terminal_receipt_unavailable", + "receipt_persisted": False, + "active_blockers": ["durable_no_write_receipt_unavailable"], + "safe_next_action": ( + "repair_durable_receipt_before_new_recover_candidate" + ), + } + ) + else: + handoff["receipt_persisted"] = True + handoff["no_write_terminal_receipt_id"] = durable_receipt_id logger.info( "auto_repair_routed_to_single_writer_executor", incident_id=incident_id, diff --git a/apps/api/src/services/agent99_sre_bridge.py b/apps/api/src/services/agent99_sre_bridge.py index e0b362c37..a2a477ed6 100644 --- a/apps/api/src/services/agent99_sre_bridge.py +++ b/apps/api/src/services/agent99_sre_bridge.py @@ -34,6 +34,8 @@ AGENT99_CONTROL_PLANE_HEALTH_MODE = "SelfCheck" AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID = ( "agent99:agent99_control_plane_health:SelfCheck" ) +AGENT99_COLD_START_INCIDENT_ID = "INC-20260711-11C751" +AGENT99_COLD_START_RECOVER_ROUTE_ID = "agent99:host_recovery:Recover" AGENT99_DURABLE_ROUTE_KINDS = { AGENT99_CONTROL_PLANE_HEALTH_KIND, "provider_freshness_signal", @@ -265,6 +267,91 @@ def resolve_agent99_durable_route( } +def _cold_start_no_write_terminal( + *, + alert_id: str, + project_id: str, + incident_id: str, + approval_id: str, + fingerprint: str, + route_id: str, + work_item_id: str, + typed_target_route: dict[str, Any], +) -> dict[str, Any]: + """Return a traceable terminal before claim or transport can occur.""" + + receipt_digest = hashlib.sha256( + ( + f"{project_id}:{incident_id}:{approval_id}:{fingerprint}:" + f"{route_id}:exact-inventory-scope-missing" + ).encode() + ).hexdigest() + terminal = { + "schema_version": "agent99_cold_start_no_write_terminal_v1", + "receipt_id": f"cold-start-no-write:{receipt_digest[:24]}", + "status": "no_write_terminal", + "reason": "exact_inventory_scope_missing", + "project_id": project_id, + "incident_id": incident_id, + "approval_id": approval_id, + "source_alert_id": alert_id, + "source_fingerprint": fingerprint, + "route_id": route_id, + "work_item_id": work_item_id or None, + "canonical_asset_id": typed_target_route.get("canonical_asset_id"), + "source_namespace": typed_target_route.get("source_namespace"), + "normalized_execution_namespace": typed_target_route.get( + "normalized_execution_namespace" + ), + "kubernetes_namespace_applicable": False, + "allowed_inventory_hosts": list( + typed_target_route.get("allowed_inventory_hosts") or [] + ), + "playbook": { + "check": "Agent99 Status controlledApply=false", + "apply": "blocked_exact_inventory_scope_missing", + "rollback": "not_applicable_no_runtime_write", + }, + "transport": { + "status": "suppressed_before_single_flight_claim", + "claim_performed": False, + "transport_performed": False, + }, + "verifier_contract": { + "name": "cold_start_no_write_terminal_contract", + "status": "ready_for_independent_source_validation", + "scope": "source_no_write_only", + "runtime_verifier_applicable": False, + "assertions": [ + "normalized_execution_namespace_is_host_recovery", + "kubernetes_namespace_is_not_execution_identity", + "exact_inventory_scope_is_absent", + "single_flight_claim_not_performed", + "agent99_transport_not_performed", + "runtime_write_not_performed", + ], + }, + "runtime_write_performed": False, + "agent99_dispatch_performed": False, + "production_executor_invoked": False, + "incident_resolution_allowed": False, + "next_safe_action": ( + "bind_exact_inventory_scope_before_creating_a_new_recover_candidate" + ), + } + return { + "schemaVersion": "agent99_cold_start_no_write_result_v1", + "status": "no_write_terminal", + "reason": terminal["reason"], + "dispatchPerformed": False, + "runtimeClosureVerified": False, + "runtimeWritePerformed": False, + "controlledApplyAuthorized": False, + "typedTargetRoute": typed_target_route, + "noWriteTerminal": terminal, + } + + def _resolution_policy_for_kind(kind: str) -> str: if kind in { AGENT99_CONTROL_PLANE_HEALTH_KIND, @@ -1226,6 +1313,43 @@ async def bridge_alertmanager_to_agent99( str(route_id or "").strip() or f"agent99:{str(payload.get('kind') or 'monitoring_alert')}:{suggested_mode}" ) + if ( + incident_id == AGENT99_COLD_START_INCIDENT_ID + and resolved_route_id == AGENT99_COLD_START_RECOVER_ROUTE_ID + and mutating + and routing.get("typedDispatchAllowed") is True + and typed_target_route.get("resolution_status") == "resolved" + and typed_target_route.get("target_kind") == "control_plane_recovery" + and typed_target_route.get("canonical_asset_id") + == "control-plane:cold-start-gate" + and typed_target_route.get("executor") == "Agent99" + and typed_target_route.get("verifier") + == "cold_start_independent_scorecard_verifier" + and typed_target_route.get("cross_domain_fallback_allowed") is False + and typed_target_route.get("normalized_execution_namespace") + == "host_recovery" + and typed_target_route.get("source_namespace") == "default" + and typed_target_route.get("kubernetes_namespace_applicable") is False + and typed_target_route.get("controlled_apply_allowed") is False + and typed_target_route.get("allowed_inventory_hosts") == [] + and typed_target_route.get("no_write_terminal_required") is True + ): + logger.info( + "agent99_cold_start_no_write_terminal", + incident_id=incident_id, + route_id=resolved_route_id, + reason="exact_inventory_scope_missing", + ) + return _cold_start_no_write_terminal( + alert_id=alert_id, + project_id=project_id, + incident_id=incident_id, + approval_id=approval_id, + fingerprint=fingerprint, + route_id=resolved_route_id, + work_item_id=work_item_id, + typed_target_route=typed_target_route, + ) if payload.get("kind") == AGENT99_CONTROL_PLANE_HEALTH_KIND: typed_target_route = ( routing.get("typedTargetRoute") diff --git a/apps/api/src/services/controlled_alert_target_router.py b/apps/api/src/services/controlled_alert_target_router.py index a6bbb95f1..8bca0b8c2 100644 --- a/apps/api/src/services/controlled_alert_target_router.py +++ b/apps/api/src/services/controlled_alert_target_router.py @@ -404,9 +404,27 @@ def resolve_typed_alert_target( executor="Agent99", verifier="cold_start_independent_scorecard_verifier", risk_class="high", + controlled_apply_allowed=False, execution_role="single_writer_control_plane_recovery_executor", ) route["route_id"] = "agent99_recover_after_owner_review" + route.update( + { + "source_namespace": legacy_recovery["source_namespace"], + "normalized_execution_namespace": legacy_recovery[ + "normalized_execution_namespace" + ], + "kubernetes_namespace_applicable": False, + "inventory_scope_status": "exact_inventory_scope_unresolved", + "no_write_terminal_required": True, + "no_write_terminal_reason": "exact_inventory_scope_missing", + "check_route": legacy_recovery["check_route"], + "controlled_apply_route": legacy_recovery[ + "controlled_apply_route" + ], + "rollback_route": legacy_recovery["rollback_route"], + } + ) return route if compact_alert in { @@ -962,10 +980,17 @@ def resolve_controlled_alert_target( "schema_version": "controlled_alert_target_route_v1", "target_kind": "control_plane_recovery", "target_resource": target_resource or "cold-start-gate", + "canonical_asset_id": "control-plane:cold-start-gate", "source_namespace": namespace or None, "normalized_execution_namespace": "host_recovery", "kubernetes_namespace_applicable": False, "executor": "Agent99", + "verifier": "cold_start_independent_scorecard_verifier", + "allowed_inventory_hosts": [], + "controlled_apply_allowed": False, + "inventory_scope_status": "exact_inventory_scope_unresolved", + "no_write_terminal_required": True, + "no_write_terminal_reason": "exact_inventory_scope_missing", "route_id": "agent99_recover_after_owner_review", "check_route": "agent99-mode Status controlledApply=false", "controlled_apply_route": "agent99-mode Recover controlledApply=true", @@ -1020,10 +1045,16 @@ def build_controlled_recovery_promotion_contract( }, { "field": "controlled_apply_route", - "status": "ready", + "status": "blocked", "source": "domain_target_normalizer", "value": route["controlled_apply_route"], }, + { + "field": "no_write_terminal_receipt", + "status": "pending", + "source": "alert_operation_log", + "value": "required_before_no_write_terminal_is_durable", + }, { "field": "dispatch_receipt", "status": "pending", @@ -1089,12 +1120,12 @@ def build_controlled_recovery_promotion_contract( "fields": fields, "runtime_execution_authorized": False, "runtime_write_allowed": False, - "controlled_playbook_queue": True, + "controlled_playbook_queue": False, "owner_review_required": False, "owner_review_gate": "auto_waived_for_low_medium_high", "needs_human": False, - "completion_status": "partial", - "completion_blocker": "dispatch_verifier_and_learning_receipts_missing", + "completion_status": "no_write_terminal_pending_durable_receipt", + "completion_blocker": "exact_inventory_scope_missing", "forbidden_operations": [ "kubectl_for_host_control_plane_target", "generic_ansible_keyword_fallback", @@ -1107,12 +1138,16 @@ def build_controlled_recovery_promotion_contract( "controlled_automation_closure": { "schema_version": "controlled_recovery_closure_contract_v1", "trace_id_required": True, - "status": "pending_source_check_dispatch_verify_writeback", + "status": "no_write_terminal_pending_durable_receipt", "stages": [ {"stage": "sensor", "status": "pending", "writes_runtime_state": False}, {"stage": "normalize", "status": "passed", "writes_runtime_state": False}, {"stage": "check", "status": "pending", "writes_runtime_state": False}, - {"stage": "controlled_apply", "status": "pending", "writes_runtime_state": False}, + { + "stage": "controlled_apply", + "status": "blocked_exact_inventory_scope_missing", + "writes_runtime_state": False, + }, {"stage": "verify", "status": "pending", "writes_runtime_state": False}, {"stage": "learn_writeback", "status": "pending", "writes_runtime_state": False}, ], @@ -1128,7 +1163,7 @@ def build_controlled_recovery_promotion_contract( "asset_type": "PlayBook", "asset_id": "agent99:Recover", "owner": "Agent99", - "status": "dispatch_receipt_pending", + "status": "blocked_exact_inventory_scope_missing", }, { "asset_type": "Verifier", @@ -1162,19 +1197,34 @@ def build_controlled_recovery_handoff( ) if contract is None: return None + no_write_required = bool( + contract.get("target_route", {}).get("no_write_terminal_required") is True + ) return { "schema_version": "ai_decision_controlled_executor_handoff_v1", - "status": "agent99_dispatch_receipt_readback_pending", + "status": ( + "cold_start_no_write_terminal_pending_receipt" + if no_write_required + else "agent99_dispatch_receipt_readback_pending" + ), "queued": False, "side_effect_performed": False, "single_writer_executor": "Agent99", "route_id": contract["route_id"], - "active_blockers": [ - "agent99_dispatch_receipt_readback_pending", - "post_apply_verifier_missing", - "km_playbook_writeback_missing", - ], - "safe_next_action": "read_agent99_dispatch_then_run_independent_cold_start_verifier", + "active_blockers": ( + ["exact_inventory_scope_missing"] + if no_write_required + else [ + "agent99_dispatch_receipt_readback_pending", + "post_apply_verifier_missing", + "km_playbook_writeback_missing", + ] + ), + "safe_next_action": ( + "persist_no_write_terminal_without_claim_or_transport" + if no_write_required + else "read_agent99_dispatch_then_run_independent_cold_start_verifier" + ), "repair_candidate_promotion_contract": contract, "runtime_execution_authorized": False, "runtime_write_allowed": False, diff --git a/apps/api/tests/test_agent99_sre_bridge.py b/apps/api/tests/test_agent99_sre_bridge.py index d9dba58a4..e67213b93 100644 --- a/apps/api/tests/test_agent99_sre_bridge.py +++ b/apps/api/tests/test_agent99_sre_bridge.py @@ -815,6 +815,75 @@ def test_agent99_dispatch_identity_is_stable_across_projections() -> None: assert telegram.work_item_id == alertmanager.work_item_id +@pytest.mark.asyncio +async def test_named_cold_start_without_inventory_scope_stops_before_claim( + monkeypatch, +) -> None: + class NoClaimLedger: + async def reserve(self, **_kwargs): # type: ignore[no-untyped-def] + raise AssertionError("no-write terminal must not reserve a run") + + async def unexpected_single_flight(*_args, **_kwargs): # type: ignore[no-untyped-def] + raise AssertionError("no-write terminal must not claim single-flight") + + monkeypatch.setattr( + "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger", + lambda: NoClaimLedger(), + ) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight", + unexpected_single_flight, + ) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", + lambda _payload: (_ for _ in ()).throw( + AssertionError("no-write terminal must not perform transport") + ), + ) + + result = await bridge_alertmanager_to_agent99( + alert_id="alert-11c751", + alertname="ColdStartGateBlocked", + severity="critical", + namespace="default", + target_resource="cold-start-gate", + message="cold-start gate blocked", + labels={"host": "110"}, + fingerprint="source-fingerprint", + project_id="awoooi", + incident_id="INC-20260711-11C751", + approval_id="00000000-0000-0000-0000-000000000751", + route_id="agent99:host_recovery:Recover", + work_item_id="agent99-dispatch:awoooi:INC-20260711-11C751:cold-start", + ) + + assert result["status"] == "no_write_terminal" + assert result["dispatchPerformed"] is False + terminal = result["noWriteTerminal"] + assert terminal["source_alert_id"] == "alert-11c751" + assert terminal["work_item_id"] == ( + "agent99-dispatch:awoooi:INC-20260711-11C751:cold-start" + ) + assert terminal["source_namespace"] == "default" + assert terminal["normalized_execution_namespace"] == "host_recovery" + assert terminal["kubernetes_namespace_applicable"] is False + assert terminal["playbook"] == { + "check": "Agent99 Status controlledApply=false", + "apply": "blocked_exact_inventory_scope_missing", + "rollback": "not_applicable_no_runtime_write", + } + assert terminal["transport"] == { + "status": "suppressed_before_single_flight_claim", + "claim_performed": False, + "transport_performed": False, + } + assert terminal["verifier_contract"]["status"] == ( + "ready_for_independent_source_validation" + ) + assert terminal["verifier_contract"]["runtime_verifier_applicable"] is False + assert terminal["runtime_write_performed"] is False + + @pytest.mark.asyncio async def test_mutating_dispatch_is_deduplicated_by_postgres_identity( monkeypatch, diff --git a/apps/api/tests/test_controlled_alert_target_router.py b/apps/api/tests/test_controlled_alert_target_router.py index ca995018a..9047f3d3f 100644 --- a/apps/api/tests/test_controlled_alert_target_router.py +++ b/apps/api/tests/test_controlled_alert_target_router.py @@ -56,6 +56,10 @@ def test_normalizer_separates_source_namespace_from_execution_domain() -> None: assert route["normalized_execution_namespace"] == "host_recovery" assert route["kubernetes_namespace_applicable"] is False assert route["executor"] == "Agent99" + assert route["canonical_asset_id"] == "control-plane:cold-start-gate" + assert route["allowed_inventory_hosts"] == [] + assert route["controlled_apply_allowed"] is False + assert route["no_write_terminal_required"] is True assert route["runtime_execution_authorized"] is False assert resolve_controlled_alert_target( @@ -76,11 +80,12 @@ def test_cold_start_contract_requires_same_run_receipts_before_closure() -> None assert contract is not None fields = {row["field"]: row for row in contract["fields"]} assert fields["target_selector"]["status"] == "ready" - assert fields["controlled_apply_route"]["status"] == "ready" + assert fields["controlled_apply_route"]["status"] == "blocked" for required in ( "source_sensor_receipt", "check_mode_receipt", "dispatch_receipt", + "no_write_terminal_receipt", "post_apply_verifier", "incident_closure_receipt", "telegram_receipt", @@ -91,7 +96,11 @@ def test_cold_start_contract_requires_same_run_receipts_before_closure() -> None assert contract["runtime_execution_authorized"] is False assert contract["owner_review_required"] is False assert contract["needs_human"] is False - assert contract["completion_status"] == "partial" + assert contract["controlled_playbook_queue"] is False + assert contract["completion_status"] == ( + "no_write_terminal_pending_durable_receipt" + ) + assert contract["completion_blocker"] == "exact_inventory_scope_missing" def test_cold_start_is_excluded_from_generic_ansible_keyword_catalog() -> None: @@ -139,7 +148,7 @@ def test_cold_start_does_not_emit_ansible_candidate_audit() -> None: assert payload is None -def test_handoff_stays_partial_until_dispatch_verifier_and_learning() -> None: +def test_handoff_requires_no_write_terminal_until_inventory_scope_is_exact() -> None: handoff = build_controlled_recovery_handoff( alertname="custom", target_resource="cold-start-gate", @@ -154,44 +163,66 @@ def test_handoff_stays_partial_until_dispatch_verifier_and_learning() -> None: assert handoff["runtime_execution_authorized"] is False assert handoff["owner_review_required"] is False assert handoff["needs_human"] is False - assert handoff["active_blockers"] == [ - "agent99_dispatch_receipt_readback_pending", - "post_apply_verifier_missing", - "km_playbook_writeback_missing", - ] + assert handoff["status"] == "cold_start_no_write_terminal_pending_receipt" + assert handoff["active_blockers"] == ["exact_inventory_scope_missing"] + assert handoff["safe_next_action"] == ( + "persist_no_write_terminal_without_claim_or_transport" + ) @pytest.mark.asyncio -async def test_webhook_router_never_queues_generic_ansible_for_cold_start( +@pytest.mark.parametrize( + "operation_receipt_id", + ["no-write-receipt-11c751", "", None], +) +async def test_webhook_router_persists_named_cold_start_no_write_terminal( monkeypatch: pytest.MonkeyPatch, + operation_receipt_id: str | None, ) -> None: queue = AsyncMock(side_effect=AssertionError("generic Ansible must not be queued")) - append = AsyncMock() + append = AsyncMock( + return_value=( + SimpleNamespace(id=operation_receipt_id) + if operation_receipt_id is not None + else None + ) + ) incident_lookup = AsyncMock(side_effect=AssertionError("incident lookup not needed")) bridge = AsyncMock( return_value={ - "status": "dispatched", - "dispatchPerformed": True, - "identity": { - "run_id": "7cf8fdf7-0966-5ac6-95b2-09e32808b248", - "trace_id": "00-11111111111111111111111111111111-2222222222222222-01", - "work_item_id": "agent99-dispatch:awoooi:INC-20260711-11C751:recovery", - "idempotency_key": "agent99-controlled:stable-key", - }, - "dispatchReceipt": { - "status": "accepted_queue_persisted", - "accepted": True, - "inbox_triggered": True, - "queue_accepted": True, - "dispatch_identity_matched": True, - }, - "correlatedReceipt": { - "status": "dispatch_accepted_verifier_pending", - "receipt_persisted": True, - "runtime_execution_authorized": False, - "runtime_closure_verified": False, - "verifier": {"status": "pending"}, - "learning_writeback": {"status": "pending_verifier"}, + "status": "no_write_terminal", + "dispatchPerformed": False, + "runtimeWritePerformed": False, + "controlledApplyAuthorized": False, + "runtimeClosureVerified": False, + "noWriteTerminal": { + "schema_version": "agent99_cold_start_no_write_terminal_v1", + "status": "no_write_terminal", + "reason": "exact_inventory_scope_missing", + "incident_id": "INC-20260711-11C751", + "route_id": "agent99:host_recovery:Recover", + "canonical_asset_id": "control-plane:cold-start-gate", + "source_namespace": "default", + "normalized_execution_namespace": "host_recovery", + "kubernetes_namespace_applicable": False, + "allowed_inventory_hosts": [], + "runtime_write_performed": False, + "agent99_dispatch_performed": False, + "production_executor_invoked": False, + "incident_resolution_allowed": False, + "transport": { + "claim_performed": False, + "transport_performed": False, + }, + "next_safe_action": ( + "bind_exact_inventory_scope_before_creating_a_new_recover_candidate" + ), + "verifier_contract": { + "name": "cold_start_no_write_terminal_contract", + "status": "ready_for_independent_source_validation", + "scope": "source_no_write_only", + "runtime_verifier_applicable": False, + }, }, } ) @@ -235,19 +266,42 @@ async def test_webhook_router_never_queues_generic_ansible_for_cold_start( "agent99:host_recovery:Recover" ) assert handoff["execution_priority"] == 30 - assert handoff["queued"] is True + assert handoff["queued"] is False + assert handoff["side_effect_performed"] is False + if operation_receipt_id: + assert handoff["status"] == "cold_start_no_write_terminal" + assert handoff["receipt_persisted"] is True + assert handoff["no_write_terminal_receipt_id"] == ( + "no-write-receipt-11c751" + ) + else: + assert handoff["status"] == ( + "cold_start_no_write_terminal_receipt_unavailable" + ) + assert handoff["receipt_persisted"] is False + assert handoff["active_blockers"] == [ + "durable_no_write_receipt_unavailable" + ] assert handoff["runtime_execution_authorized"] is False assert handoff["runtime_closure_verified"] is False - assert handoff["automation_run_id"] == ( - "7cf8fdf7-0966-5ac6-95b2-09e32808b248" - ) append.assert_awaited_once() receipt = append.await_args.kwargs - assert receipt["action_detail"] == "controlled_check_mode_queued" - assert receipt["success"] is True + assert receipt["action_detail"] == "controlled_no_write_terminal" + assert receipt["success"] is None assert receipt["context"]["safe_next_action"] == ( - "run_independent_cold_start_verifier_then_km_playbook_writeback" + "bind_exact_inventory_scope_before_creating_a_new_recover_candidate" ) + assert receipt["context"]["no_write_terminal"]["reason"] == ( + "exact_inventory_scope_missing" + ) + assert receipt["context"]["no_write_verifier_receipt"] == { + "schema_version": "cold_start_no_write_source_verifier_receipt_v1", + "verifier": "alert_webhook_no_write_terminal_validator", + "status": "passed_source_contract", + "producer_independent": True, + "runtime_verifier": False, + "runtime_closure_verified": False, + } assert receipt["context"]["runtime_closure_verified"] is False diff --git a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py index 17b4e01b2..231c99880 100644 --- a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py +++ b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py @@ -155,8 +155,8 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() - "active_or_completed_commitments": 68, "by_status": { "analysis_or_governance_complete": 6, - "source_implemented_runtime_pending": 29, - "in_progress": 31, + "source_implemented_runtime_pending": 30, + "in_progress": 30, "not_started_or_no_current_evidence": 2, "superseded": 2, }, @@ -182,6 +182,12 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() - assert "destination-bound provider acknowledgement" in " ".join( commitments["AIA-CONV-031"]["source_evidence"] ) + assert commitments["AIA-CONV-032"]["status"] == ( + "source_implemented_runtime_pending" + ) + assert "blocks Recover before single-flight claim" in " ".join( + commitments["AIA-CONV-032"]["source_evidence"] + ) assert "Host112" in commitments["AIA-CONV-049"]["title"] assert commitments["AIA-CONV-049"]["status"] == ( "source_implemented_runtime_pending" diff --git a/apps/api/tests/test_sre_typed_domain_router.py b/apps/api/tests/test_sre_typed_domain_router.py index 37d85e547..d6571cb5b 100644 --- a/apps/api/tests/test_sre_typed_domain_router.py +++ b/apps/api/tests/test_sre_typed_domain_router.py @@ -672,6 +672,11 @@ def test_agent99_payload_carries_same_typed_host_operations_contract() -> None: assert payload["awoooi"]["canonicalAssetId"] == ( "control-plane:cold-start-gate" ) + assert route["source_namespace"] == "default" + assert route["normalized_execution_namespace"] == "host_recovery" + assert route["kubernetes_namespace_applicable"] is False + assert route["controlled_apply_allowed"] is False + assert route["no_write_terminal_required"] is True # Generic cold-start remains no-write until the router binds an exact # inventory scope; the Windows inbox rejects an empty scope fail-closed. assert route["host"] is None diff --git a/docs/operations/sre-ai-agent-conversation-commitments.snapshot.json b/docs/operations/sre-ai-agent-conversation-commitments.snapshot.json index 0ea1339ed..f7ab15a50 100644 --- a/docs/operations/sre-ai-agent-conversation-commitments.snapshot.json +++ b/docs/operations/sre-ai-agent-conversation-commitments.snapshot.json @@ -49,7 +49,7 @@ {"id":"AIA-CONV-030","category":"telegram","status":"source_implemented_runtime_pending","title":"對話可啟動受控調查/修復並建立 Codex 開發 work item","linked_work_items":["AIA-SRE-001","AIA-SRE-015","AIA-SRE-017"],"terminal_condition":"討論結果轉為去重 work item；production action 仍走 typed executor/verifier。","source_evidence":["verified controlled_action_request ingress gate","exact canonical asset/domain resolution or asset_identity_unresolved drift item","fingerprint-deduplicated durable internal work-item receipt","deterministic no-provider no-runtime-mutation Telegram response"]}, {"id":"AIA-CONV-031","category":"telegram","status":"source_implemented_runtime_pending","title":"同 fingerprint 去重、聚合、抑噪及 resolved/recovered 更新","linked_work_items":["AIA-SRE-017"],"terminal_condition":"同事件只維護 canonical lifecycle，恢復卡有 destination-bound provider acknowledgement。","source_evidence":["existing firing fingerprint convergence and parent-child alert storm aggregation","single-row durable resolved lifecycle reservation with bounded retry","duplicate resolved webhook suppression before incident or Telegram side effects","recovery card accepted only with destination-bound provider acknowledgement"]}, - {"id":"AIA-CONV-032","category":"named_incident","status":"in_progress","title":"INC-20260711-11C751 cold-start-gate 補 PlayBook、transport、rollback 與 verifier","linked_work_items":["AIA-SRE-004","AIA-SRE-006","AIA-SRE-015"],"terminal_condition":"namespace identity 修正後完成一次 bounded same-run repair 或明確 no-write terminal。"}, + {"id":"AIA-CONV-032","category":"named_incident","status":"source_implemented_runtime_pending","title":"INC-20260711-11C751 cold-start-gate 補 PlayBook、transport、rollback 與 verifier","linked_work_items":["AIA-SRE-004","AIA-SRE-006","AIA-SRE-015"],"terminal_condition":"namespace identity 修正後完成一次 bounded same-run repair 或明確 no-write terminal。","source_evidence":["source namespace separated from normalized host_recovery execution identity","exact inventory scope absence blocks Recover before single-flight claim or Agent99 transport","durable alert operation receipt records the explicit no-write terminal","consumer-validated source no-write verifier receipt with runtime closure false"]}, {"id":"AIA-CONV-033","category":"named_incident","status":"in_progress","title":"INC-20260711-D037E5 修復 stale candidate 阻擋、自動排隊與 durable closure","linked_work_items":["AIA-SRE-007","AIA-SRE-015"],"terminal_condition":"新候選可 claim，apply 不重複，verifier、Telegram 與 learning receipts 同 run 完成。"}, {"id":"AIA-CONV-034","category":"named_incident","status":"in_progress","title":"AWOOOI CPU 高負載與 P99 上升完成 RCA、bounded repair 與 verifier","linked_work_items":["AIA-SRE-004","AIA-SRE-006","AIA-SRE-010"],"terminal_condition":"相關 metrics/logs/changes 對齊 canonical asset，修復後 latency 與資源 verifier 關閉。"}, {"id":"AIA-CONV-035","category":"named_incident","status":"in_progress","title":"Backup/restore escrow 補 freshness、escrow metadata、restore drill 與 DR scorecard","linked_work_items":["AIA-SRE-011","AIA-SRE-016"],"terminal_condition":"read-only evidence 齊全；缺 escrow/restore receipt 時保持 blocked 而非假綠。"}, @@ -98,8 +98,8 @@ "active_or_completed_commitments": 68, "by_status": { "analysis_or_governance_complete": 6, - "source_implemented_runtime_pending": 29, - "in_progress": 31, + "source_implemented_runtime_pending": 30, + "in_progress": 30, "not_started_or_no_current_evidence": 2, "superseded": 2 },