6cf8429d17
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m32s
CD Pipeline / build-and-deploy (push) Successful in 8m2s
CD Pipeline / post-deploy-checks (push) Successful in 2m22s
Restore D037 durable incident readback without weakening database failure semantics. Fence Agent99 dispatch and terminal learning to canonical identities, durable leases, verifier receipts, and reconciler-owned checkpoints. Drain backup and restore legacy receipts in bounded cohorts with strict transport, claim, projection, and no-false-green controls. Keep SSH service refusal visible while separating it from broker network-policy reachability.
243 lines
8.6 KiB
Python
243 lines
8.6 KiB
Python
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock
|
|
|
|
import pytest
|
|
|
|
from src.api.v1 import webhooks
|
|
from src.services.awooop_ansible_audit_service import (
|
|
_catalog_hints,
|
|
build_ansible_decision_audit_payload,
|
|
build_ansible_truth,
|
|
)
|
|
from src.services.controlled_alert_target_router import (
|
|
build_controlled_recovery_handoff,
|
|
build_controlled_recovery_promotion_contract,
|
|
resolve_controlled_alert_target,
|
|
)
|
|
|
|
|
|
def _cold_start_incident() -> dict:
|
|
return {
|
|
"incident_id": "INC-20260711-11C751",
|
|
"project_id": "awoooi",
|
|
"alertname": "ColdStartGateBlocked",
|
|
"alert_category": "general",
|
|
"affected_services": ["cold-start-gate"],
|
|
"signals": [
|
|
{
|
|
"alert_name": "ColdStartGateBlocked",
|
|
"labels": {
|
|
"component": "cold-start-gate",
|
|
"namespace": "default",
|
|
"host": "110",
|
|
},
|
|
"annotations": {
|
|
"summary": (
|
|
"full-stack cold-start recovery blocked; backup and "
|
|
"Docker evidence must be checked"
|
|
)
|
|
},
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
def test_normalizer_separates_source_namespace_from_execution_domain() -> None:
|
|
route = resolve_controlled_alert_target(
|
|
alertname="ColdStartGateBlocked",
|
|
target_resource="cold-start-gate",
|
|
namespace="default",
|
|
)
|
|
|
|
assert route is not None
|
|
assert route["source_namespace"] == "default"
|
|
assert route["normalized_execution_namespace"] == "host_recovery"
|
|
assert route["kubernetes_namespace_applicable"] is False
|
|
assert route["executor"] == "Agent99"
|
|
assert route["runtime_execution_authorized"] is False
|
|
|
|
assert resolve_controlled_alert_target(
|
|
alertname="SentryRpsZero",
|
|
target_resource="sentry",
|
|
namespace="awoooi-prod",
|
|
) is None
|
|
|
|
|
|
def test_cold_start_contract_requires_same_run_receipts_before_closure() -> None:
|
|
contract = build_controlled_recovery_promotion_contract(
|
|
alertname="ColdStartGateBlocked",
|
|
target_resource="cold-start-gate",
|
|
namespace="default",
|
|
incident_id="INC-20260711-11C751",
|
|
)
|
|
|
|
assert contract is not None
|
|
fields = {row["field"]: row for row in contract["fields"]}
|
|
assert fields["target_selector"]["status"] == "ready"
|
|
assert fields["controlled_apply_route"]["status"] == "ready"
|
|
for required in (
|
|
"source_sensor_receipt",
|
|
"check_mode_receipt",
|
|
"dispatch_receipt",
|
|
"post_apply_verifier",
|
|
"incident_closure_receipt",
|
|
"telegram_receipt",
|
|
"km_writeback",
|
|
"playbook_trust_writeback",
|
|
):
|
|
assert fields[required]["status"] == "pending"
|
|
assert contract["runtime_execution_authorized"] is False
|
|
assert contract["owner_review_required"] is False
|
|
assert contract["needs_human"] is False
|
|
assert contract["completion_status"] == "partial"
|
|
|
|
|
|
def test_cold_start_is_excluded_from_generic_ansible_keyword_catalog() -> None:
|
|
hints = _catalog_hints(_cold_start_incident(), None)
|
|
|
|
assert hints["match_mode"] == "domain_controlled_executor_route_v1"
|
|
assert hints["decision_effect"] == "delegate_to_domain_executor"
|
|
assert hints["candidates"] == []
|
|
assert hints["controlled_executor"]["executor"] == "Agent99"
|
|
assert "ansible:110-devops" in hints["unmatched_catalog_ids"]
|
|
assert "ansible:188-momo-backup-user" in hints["unmatched_catalog_ids"]
|
|
|
|
truth = build_ansible_truth([], incident=_cold_start_incident(), drift=None)
|
|
assert truth["not_used_reason"] == "non_kubernetes_control_plane_route"
|
|
assert truth["candidate_catalog"]["controlled_executor"]["route_id"] == (
|
|
"agent99_recover_after_owner_review"
|
|
)
|
|
|
|
|
|
def test_cold_start_does_not_emit_ansible_candidate_audit() -> None:
|
|
incident = SimpleNamespace(
|
|
incident_id="INC-20260711-11C751",
|
|
project_id="awoooi",
|
|
alertname="ColdStartGateBlocked",
|
|
alert_category="general",
|
|
notification_type="TYPE-3",
|
|
severity=SimpleNamespace(value="P3"),
|
|
affected_services=["cold-start-gate"],
|
|
signals=[
|
|
SimpleNamespace(
|
|
alert_name="ColdStartGateBlocked",
|
|
labels={"component": "cold-start-gate", "namespace": "default"},
|
|
annotations={"summary": "full-stack cold-start blocked"},
|
|
)
|
|
],
|
|
)
|
|
|
|
payload = build_ansible_decision_audit_payload(
|
|
incident=incident,
|
|
proposal_data={"source": "test", "risk_level": "low"},
|
|
decision_path="alert_webhook_controlled_router",
|
|
not_used_reason="test",
|
|
)
|
|
|
|
assert payload is None
|
|
|
|
|
|
def test_handoff_stays_partial_until_dispatch_verifier_and_learning() -> None:
|
|
handoff = build_controlled_recovery_handoff(
|
|
alertname="custom",
|
|
target_resource="cold-start-gate",
|
|
namespace="default",
|
|
incident_id="INC-20260711-11C751",
|
|
)
|
|
|
|
assert handoff is not None
|
|
assert handoff["single_writer_executor"] == "Agent99"
|
|
assert handoff["queued"] is False
|
|
assert handoff["side_effect_performed"] is False
|
|
assert handoff["runtime_execution_authorized"] is False
|
|
assert handoff["owner_review_required"] is False
|
|
assert handoff["needs_human"] is False
|
|
assert handoff["active_blockers"] == [
|
|
"agent99_dispatch_receipt_readback_pending",
|
|
"post_apply_verifier_missing",
|
|
"km_playbook_writeback_missing",
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_webhook_router_never_queues_generic_ansible_for_cold_start(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
queue = AsyncMock(side_effect=AssertionError("generic Ansible must not be queued"))
|
|
append = AsyncMock()
|
|
incident_lookup = AsyncMock(side_effect=AssertionError("incident lookup not needed"))
|
|
bridge = AsyncMock(
|
|
return_value={
|
|
"status": "dispatched",
|
|
"dispatchPerformed": True,
|
|
"identity": {
|
|
"run_id": "7cf8fdf7-0966-5ac6-95b2-09e32808b248",
|
|
"trace_id": "00-11111111111111111111111111111111-2222222222222222-01",
|
|
"work_item_id": "agent99-dispatch:awoooi:INC-20260711-11C751:recovery",
|
|
"idempotency_key": "agent99-controlled:stable-key",
|
|
},
|
|
"dispatchReceipt": {
|
|
"status": "accepted_inbox_triggered",
|
|
"accepted": True,
|
|
"inbox_triggered": True,
|
|
},
|
|
"correlatedReceipt": {
|
|
"status": "dispatch_accepted_verifier_pending",
|
|
"receipt_persisted": True,
|
|
"runtime_execution_authorized": False,
|
|
"runtime_closure_verified": False,
|
|
"verifier": {"status": "pending"},
|
|
"learning_writeback": {"status": "pending_verifier"},
|
|
},
|
|
}
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
"src.jobs.awooop_ansible_candidate_backfill_job.enqueue_ai_decision_ansible_candidate",
|
|
queue,
|
|
)
|
|
monkeypatch.setattr(
|
|
"src.repositories.alert_operation_log_repository.get_alert_operation_log_repository",
|
|
lambda: SimpleNamespace(append=append),
|
|
)
|
|
monkeypatch.setattr(
|
|
webhooks,
|
|
"get_incident_service",
|
|
lambda: SimpleNamespace(get_from_working_memory=incident_lookup),
|
|
)
|
|
monkeypatch.setattr(webhooks, "bridge_alertmanager_to_agent99", bridge)
|
|
|
|
handoff = await webhooks._try_auto_repair_background(
|
|
incident_id="INC-20260711-11C751",
|
|
approval_id="00000000-0000-0000-0000-000000000751",
|
|
alert_type="custom",
|
|
target_resource="cold-start-gate",
|
|
namespace="default",
|
|
risk_level="low",
|
|
source_alert_id="alert-11c751",
|
|
source_alertname="ColdStartGateBlocked",
|
|
source_fingerprint="source-fingerprint",
|
|
)
|
|
|
|
queue.assert_not_awaited()
|
|
incident_lookup.assert_not_awaited()
|
|
bridge.assert_awaited_once()
|
|
assert handoff["single_writer_executor"] == "Agent99"
|
|
assert handoff["execution_priority"] == 30
|
|
assert handoff["queued"] is True
|
|
assert handoff["runtime_execution_authorized"] is False
|
|
assert handoff["runtime_closure_verified"] is False
|
|
assert handoff["automation_run_id"] == (
|
|
"7cf8fdf7-0966-5ac6-95b2-09e32808b248"
|
|
)
|
|
append.assert_awaited_once()
|
|
receipt = append.await_args.kwargs
|
|
assert receipt["action_detail"] == "controlled_check_mode_queued"
|
|
assert receipt["success"] is True
|
|
assert receipt["context"]["safe_next_action"] == (
|
|
"run_independent_cold_start_verifier_then_km_playbook_writeback"
|
|
)
|
|
assert receipt["context"]["runtime_closure_verified"] is False
|