fix(automation): enforce durable alert closure
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m32s
CD Pipeline / build-and-deploy (push) Successful in 8m2s
CD Pipeline / post-deploy-checks (push) Successful in 2m22s

Restore D037 durable incident readback without weakening database failure semantics.

Fence Agent99 dispatch and terminal learning to canonical identities, durable leases, verifier receipts, and reconciler-owned checkpoints.

Drain backup and restore legacy receipts in bounded cohorts with strict transport, claim, projection, and no-false-green controls.

Keep SSH service refusal visible while separating it from broker network-policy reachability.
This commit is contained in:
ogt
2026-07-14 09:40:53 +08:00
parent 72a167a322
commit 6cf8429d17
56 changed files with 15100 additions and 227 deletions

View File

@@ -0,0 +1,899 @@
from __future__ import annotations
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from fastapi import HTTPException
from src.api.v1 import webhooks
from src.db.models import IncidentRecord
from src.models.incident import IncidentStatus
from src.services import agent99_controlled_dispatch_ledger as ledger_module
from src.services import agent99_outcome_ingestion as ingestion_module
from src.services.agent99_controlled_dispatch_ledger import (
PostgresAgent99DispatchLedger,
build_agent99_dispatch_identity,
build_agent99_dispatch_receipt_envelope,
parse_agent99_dispatch_identity,
record_agent99_learning_writeback,
)
from src.services.agent99_sre_bridge import bridge_alertmanager_to_agent99
def _identity():
return build_agent99_dispatch_identity(
project_id="awoooi",
incident_id="INC-20260711-11C751",
source_fingerprint="cold-start-source-fingerprint",
route_id="agent99_cold_start_recovery",
execution_generation="1",
work_item_id="agent99-dispatch:awoooi:INC-20260711-11C751:recovery",
)
def _evidence_refs() -> dict[str, str]:
return {
"agent99_outcome_receipt_id": "agent99-outcome:queue-1",
"post_verifier_evidence_ref": "evidence:cold-start-scorecard:1",
"source_event_evidence_ref": "alert:fingerprint:resolved:1",
}
def _outcome(identity, *, transport_ok: bool = True) -> dict:
return {
"identity": identity.public_dict(),
"controlledApply": True,
"outcome": {
"identity": identity.public_dict(),
"schemaVersion": "agent99_outcome_contract_v1",
"state": "resolved",
"transportOk": transport_ok,
"verifierPassed": True,
"sourceEventResolved": True,
"verifiedAt": "2026-07-11T20:00:00+08:00",
},
}
def test_pending_inbox_never_promotes_or_authorizes() -> None:
receipt = build_agent99_dispatch_receipt_envelope(
identity=_identity(),
dispatch_receipt={
"accepted": True,
"inbox_triggered": False,
"delivery_certainty": "unknown",
},
controlled_apply_authorized=True,
)
assert receipt["dispatch_accepted"] is True
assert receipt["inbox_triggered"] is False
assert receipt["dispatch_promoted"] is False
assert receipt["controlled_apply_authorized"] is False
assert receipt["runtime_execution_authorized"] is False
assert receipt["verifier"]["status"] == "blocked_dispatch_not_promoted"
assert receipt["runtime_closure_verified"] is False
def test_complete_identity_is_required_and_recomputed() -> None:
identity = _identity()
assert parse_agent99_dispatch_identity(identity.public_dict()) == identity
missing = identity.public_dict()
missing["trace_id"] = ""
with pytest.raises(ValueError, match="identity_fields_required:trace_id"):
parse_agent99_dispatch_identity(missing)
mismatched = identity.public_dict()
mismatched["run_id"] = "00000000-0000-0000-0000-000000000000"
with pytest.raises(ValueError, match="identity_mismatch:run_id"):
parse_agent99_dispatch_identity(mismatched)
def test_approval_identity_cannot_alias_run_or_idempotency() -> None:
common = {
"project_id": "awoooi",
"incident_id": "INC-20260711-11C751",
"source_fingerprint": "cold-start-source-fingerprint",
"route_id": "agent99_cold_start_recovery",
"execution_generation": "1",
"work_item_id": "agent99-dispatch:awoooi:11c751",
}
first = build_agent99_dispatch_identity(
**common,
approval_id="00000000-0000-0000-0000-000000000751",
)
second = build_agent99_dispatch_identity(
**common,
approval_id="00000000-0000-0000-0000-000000000752",
)
assert first.run_id != second.run_id
assert first.idempotency_key != second.idempotency_key
assert first.public_dict()["canonical_digest"] != (
second.public_dict()["canonical_digest"]
)
assert first.single_flight_key == second.single_flight_key
def test_terminal_incident_status_uses_database_enum_member() -> None:
status_type = IncidentRecord.__table__.c.status.type
assert status_type.enum_class is IncidentStatus
assert IncidentStatus.RESOLVED.name in status_type.enums
assert IncidentStatus.RESOLVED.value == "resolved"
class _ScalarResult:
def __init__(self, value=None, row=None) -> None:
self.value = value
self.row = row
def scalar_one_or_none(self):
return self.value
def one_or_none(self):
return self.row
class _Context:
def __init__(self, db) -> None:
self.db = db
async def __aenter__(self):
return self.db
async def __aexit__(self, *_args):
return False
@pytest.mark.asyncio
async def test_reservation_claim_tokens_are_unique_and_running_is_reconcile_only(
monkeypatch,
) -> None:
identity = _identity()
claim_statements = []
class ReserveDB:
def __init__(self) -> None:
self.call = 0
async def execute(self, statement):
self.call += 1
if self.call == 3:
claim_statements.append(statement)
return _ScalarResult(identity.run_id)
if self.call == 5:
return _ScalarResult(
row=SimpleNamespace(
state="running",
error_detail=json.dumps({"identity": identity.public_dict()}),
)
)
return _ScalarResult()
databases = [ReserveDB(), ReserveDB()]
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(databases.pop(0)),
)
ledger = PostgresAgent99DispatchLedger()
payload = {
"kind": "host_recovery",
"suggestedMode": "Recover",
"controlledApply": True,
"awoooi": {"targetResource": "cold-start-gate"},
}
first = await ledger.reserve(identity=identity, payload=payload)
second = await ledger.reserve(identity=identity, payload=payload)
assert first["claimed"] is True
assert second["claimed"] is True
assert first["claim_token"] != second["claim_token"]
claim_sql = str(claim_statements[0])
assert "awooop_run_state.state = :state_1" in claim_sql
assert "awooop_run_state.next_attempt_at IS NULL" in claim_sql
assert "awooop_run_state.next_attempt_at <=" in claim_sql
assert "awooop_run_state.lease_until <" not in claim_sql
@pytest.mark.asyncio
async def test_stale_worker_cannot_complete_newer_claim(monkeypatch) -> None:
identity = _identity()
class FencedDB:
async def execute(self, _statement):
return _ScalarResult(None)
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(FencedDB()),
)
result = await PostgresAgent99DispatchLedger().complete(
identity=identity,
claim_token="stale-claim-token",
dispatch_receipt={
"accepted": True,
"inbox_triggered": True,
"delivery_certainty": "delivered",
},
controlled_apply_authorized=True,
)
assert result["status"] == "claim_fenced_no_write"
assert result["receipt_persisted"] is False
assert result["runtime_closure_verified"] is False
@pytest.mark.asyncio
async def test_redis_lock_collision_returns_run_to_retryable_pending(
monkeypatch,
) -> None:
identity = _identity()
initial = {
"schema_version": "agent99_controlled_dispatch_receipt_v1",
"identity": identity.public_dict(),
"runtime_closure_verified": False,
}
class RetryDB:
def __init__(self) -> None:
self.call = 0
async def execute(self, _statement):
self.call += 1
if self.call == 1:
return _ScalarResult(
row=SimpleNamespace(
state="running",
worker_id="claim-2",
attempt_count=1,
max_attempts=3,
error_detail=json.dumps(initial),
)
)
if self.call == 2:
return _ScalarResult(identity.run_id)
return _ScalarResult()
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(RetryDB()),
)
result = await PostgresAgent99DispatchLedger().defer_claim_retryable(
identity=identity,
claim_token="claim-2",
reason="single_flight_duplicate",
retry_after_seconds=600,
)
assert result["status"] == "dispatch_lock_retryable"
assert result["receipt_persisted"] is True
assert result["retry_policy"]["retry_same_generation"] is True
assert result["controlled_apply_authorized"] is False
assert result["runtime_closure_verified"] is False
@pytest.mark.asyncio
async def test_verifier_requires_transport_and_all_evidence(monkeypatch) -> None:
identity = _identity()
ledger = PostgresAgent99DispatchLedger()
missing = await ledger.record_verifier(
identity=identity,
outcome_receipt=_outcome(identity),
evidence_refs={"agent99_outcome_receipt_id": "one-ref-only"},
)
assert missing["status"] == "verifier_evidence_missing_fail_closed"
assert missing["receipt_persisted"] is False
promoted = build_agent99_dispatch_receipt_envelope(
identity=identity,
dispatch_receipt={
"accepted": True,
"inbox_triggered": True,
"delivery_certainty": "delivered",
},
controlled_apply_authorized=True,
)
class VerifierDB:
def __init__(self) -> None:
self.call = 0
async def execute(self, _statement):
self.call += 1
if self.call == 1:
return _ScalarResult(
row=SimpleNamespace(
state="waiting_tool",
error_detail=json.dumps(promoted),
)
)
if self.call == 2:
return _ScalarResult(identity.run_id)
if self.call == 4:
return _ScalarResult(identity.incident_id)
if self.call in {5, 6}:
return _ScalarResult(identity.run_id)
return _ScalarResult()
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(VerifierDB()),
)
failed = await ledger.record_verifier(
identity=identity,
outcome_receipt=_outcome(identity, transport_ok=False),
evidence_refs=_evidence_refs(),
)
assert failed["post_verifier_passed"] is False
assert failed["verifier"]["transport_ok"] is False
assert failed["runtime_closure_verified"] is False
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(VerifierDB()),
)
passed = await ledger.record_verifier(
identity=identity,
outcome_receipt=_outcome(identity, transport_ok=True),
evidence_refs=_evidence_refs(),
)
assert passed["post_verifier_passed"] is True
assert passed["verifier"]["transport_ok"] is True
assert passed["status"] == "verifier_passed_learning_writeback_pending"
assert passed["runtime_closure_verified"] is False
@pytest.mark.asyncio
async def test_learning_writeback_is_the_only_same_run_terminal_step(
monkeypatch,
) -> None:
identity = _identity()
ledger = PostgresAgent99DispatchLedger()
verifier_passed = build_agent99_dispatch_receipt_envelope(
identity=identity,
dispatch_receipt={
"accepted": True,
"inbox_triggered": True,
"delivery_certainty": "delivered",
},
controlled_apply_authorized=True,
)
verifier_passed.update({
"status": "verifier_passed_learning_writeback_pending",
"post_verifier_passed": True,
"runtime_closure_verified": False,
"closure_complete": False,
"learning_writeback": {
"status": "pending_writeback",
"receipt_refs": {"km_writeback_ack_id": "km-only"},
},
})
class IncompleteCheckpointDB:
def __init__(self) -> None:
self.call = 0
async def execute(self, _statement):
self.call += 1
if self.call == 1:
return _ScalarResult(
row=SimpleNamespace(
state="waiting_tool",
error_detail=json.dumps(verifier_passed),
)
)
raise AssertionError("terminal tables must not be touched")
incomplete_db = IncompleteCheckpointDB()
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(incomplete_db),
)
incomplete = await ledger.record_learning_writeback(
identity=identity,
receipt_refs={
"telegram_lifecycle_receipt_id": "telegram:fake",
"km_writeback_ack_id": "knowledge:fake",
"playbook_trust_writeback_ack_id": "playbook:fake",
},
)
assert incomplete["status"] == (
"learning_writeback_checkpoint_incomplete_fail_closed"
)
assert incomplete["runtime_closure_verified"] is False
assert incomplete_db.call == 1
checkpoint_refs = {
"telegram_lifecycle_receipt_id": "telegram-1",
"km_writeback_ack_id": "km-1",
"playbook_trust_writeback_ack_id": "playbook-1",
}
verifier_passed["learning_writeback"]["receipt_refs"] = checkpoint_refs
class LearningDB:
def __init__(self) -> None:
self.call = 0
async def execute(self, _statement):
self.call += 1
if self.call == 1:
return _ScalarResult(
row=SimpleNamespace(
state="waiting_tool",
error_detail=json.dumps(verifier_passed),
)
)
if self.call == 2:
return _ScalarResult(SimpleNamespace(outcome={}))
if self.call == 4:
return _ScalarResult(identity.incident_id)
if self.call in {5, 6}:
return _ScalarResult(identity.run_id)
return _ScalarResult()
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(LearningDB()),
)
closed = await ledger.record_learning_writeback(
identity=identity,
receipt_refs=checkpoint_refs,
)
assert closed["status"] == "closed_verified_learning_written"
assert closed["runtime_closure_verified"] is True
assert closed["closure_complete"] is True
@pytest.mark.asyncio
async def test_terminal_post_commit_projection_is_project_scoped(
monkeypatch,
) -> None:
identity = _identity()
incident = SimpleNamespace(incident_id=identity.incident_id)
episodic = AsyncMock(return_value=incident)
working = AsyncMock(return_value=True)
monkeypatch.setattr(
ledger_module._ledger,
"record_learning_writeback",
AsyncMock(
return_value={
"status": "closed_verified_learning_written",
"postgres_terminal_bundle_committed": True,
"redis_projection_pending": True,
"runtime_closure_verified": True,
}
),
)
monkeypatch.setattr(
"src.services.incident_service.get_incident_service",
lambda: SimpleNamespace(
get_from_episodic_memory=episodic,
save_to_working_memory=working,
),
)
result = await record_agent99_learning_writeback(
identity=identity,
receipt_refs={
"telegram_lifecycle_receipt_id": "telegram:1",
"km_writeback_ack_id": "knowledge:1",
"playbook_trust_writeback_ack_id": "playbook:1",
},
)
episodic.assert_awaited_once_with(
identity.incident_id,
project_id=identity.project_id,
)
working.assert_awaited_once_with(incident)
assert result["redis_projection_acknowledged"] is True
assert result["redis_projection_pending"] is False
@pytest.mark.asyncio
async def test_outcome_ingestion_ignores_untrusted_refs_and_cannot_terminalize(
monkeypatch,
) -> None:
identity = _identity()
calls: list[tuple[str, str]] = []
async def verifier(**kwargs):
calls.append(("verifier", str(kwargs["identity"].run_id)))
return {
"status": "verifier_passed_learning_writeback_pending",
"receipt_persisted": True,
"post_verifier_passed": True,
"runtime_closure_verified": False,
}
terminal = AsyncMock(side_effect=AssertionError("terminal bypass invoked"))
latest = AsyncMock(side_effect=AssertionError("terminal readback invoked"))
monkeypatch.setattr(ingestion_module, "record_agent99_verifier_receipt", verifier)
monkeypatch.setattr(
ingestion_module,
"record_agent99_learning_writeback",
terminal,
raising=False,
)
monkeypatch.setattr(
ingestion_module,
"read_agent99_dispatch_receipt",
latest,
raising=False,
)
result = await ingestion_module.ingest_agent99_outcome_receipt({
"identity": identity.public_dict(),
"outcome_receipt": _outcome(identity),
"evidence_refs": _evidence_refs(),
"learning_receipt_refs": {
"telegram_lifecycle_receipt_id": "telegram:fake",
"km_writeback_ack_id": "knowledge:fake",
"playbook_trust_writeback_ack_id": "playbook:fake",
},
})
assert calls == [("verifier", str(identity.run_id))]
terminal.assert_not_awaited()
latest.assert_not_awaited()
assert result["status"] == "verifier_recorded_learning_pending"
assert result["learning_writeback"]["status"] == (
"untrusted_external_receipts_ignored_reconciler_pending"
)
assert result["untrusted_learning_receipt_refs_ignored"] is True
assert result["runtime_closure_verified"] is False
@pytest.mark.asyncio
async def test_authenticated_backup_outcome_merges_all_nine_safe_refs(
monkeypatch,
) -> None:
identity = build_agent99_dispatch_identity(
project_id="awoooi",
incident_id="INC-BRR-20260711-001",
source_fingerprint="backup-source-fingerprint",
route_id="agent99:backup_health:BackupCheck",
work_item_id="agent99-dispatch:awoooi:backup-readback",
)
captured: dict[str, object] = {}
async def verifier(**kwargs):
captured.update(kwargs)
return {
"status": "verifier_passed_learning_writeback_pending",
"receipt_persisted": True,
"post_verifier_passed": True,
"runtime_closure_verified": False,
}
monkeypatch.setattr(ingestion_module, "record_agent99_verifier_receipt", verifier)
result = await ingestion_module.ingest_agent99_outcome_receipt({
"identity": identity.public_dict(),
"evidence_refs": {
"agent99_outcome_receipt_id": "agent99-outcome:backup-1",
"post_verifier_evidence_ref": "backupcheck:verified:1",
"source_event_evidence_ref": "backup-alert:resolved:1",
"untrusted_extra": "must-not-persist",
},
"outcome_receipt": {
"identity": identity.public_dict(),
"mode": "BackupCheck",
"outcome": {
"identity": identity.public_dict(),
"schemaVersion": "agent99_outcome_contract_v1",
"state": "resolved",
"transportOk": True,
"verifierPassed": True,
"sourceEventResolved": True,
"evidenceRefs": {
"backup_status_evidence_ref": "backup-status:run:1",
"freshness_evidence_ref": "backup-freshness:slo:1",
"offsite_verify_evidence_ref": "offsite-verify:run:1",
"escrow_evidence_ref": "escrow-metadata:run:1",
"restore_drill_evidence_ref": "restore-drill:run:1",
"source_resolution_receipt_ref": "source-resolution:run:1",
"token": "must-not-persist",
},
},
},
})
refs = captured["evidence_refs"]
assert set(refs) == {
"agent99_outcome_receipt_id",
"post_verifier_evidence_ref",
"source_event_evidence_ref",
"backup_status_evidence_ref",
"freshness_evidence_ref",
"offsite_verify_evidence_ref",
"escrow_evidence_ref",
"restore_drill_evidence_ref",
"source_resolution_receipt_ref",
}
assert "must-not-persist" not in str(refs)
assert captured["identity"].run_id == identity.run_id
assert result["status"] == "verifier_recorded_learning_pending"
assert result["runtime_closure_verified"] is False
def test_outcome_evidence_ref_filter_rejects_secret_shaped_values() -> None:
refs = ingestion_module._public_evidence_refs(
{
"agent99_outcome_receipt_id": "https://unsafe.invalid/outcome",
"post_verifier_evidence_ref": "verifier:public-ref",
"source_event_evidence_ref": "source:public-ref",
},
{
"outcome": {
"evidenceRefs": {
"backup_status_evidence_ref": "token=must-not-persist",
"freshness_evidence_ref": "freshness:public-ref",
"offsite_verify_evidence_ref": "receipt?token=unsafe",
"escrow_evidence_ref": "../runtime-secret",
"restore_drill_evidence_ref": "Bearer must-not-persist",
}
}
},
)
assert refs == {
"post_verifier_evidence_ref": "verifier:public-ref",
"source_event_evidence_ref": "source:public-ref",
"freshness_evidence_ref": "freshness:public-ref",
}
@pytest.mark.asyncio
async def test_outcome_webhook_requires_shared_token(monkeypatch) -> None:
identity = _identity()
request = webhooks.Agent99OutcomeWebhookPayload(
identity=identity.public_dict(),
outcome_receipt=_outcome(identity),
evidence_refs=_evidence_refs(),
)
monkeypatch.setattr(
webhooks.settings,
"AGENT99_SRE_ALERT_RELAY_TOKEN",
"relay-token",
)
with pytest.raises(HTTPException) as exc_info:
await webhooks.ingest_agent99_outcome_webhook(request, None)
assert exc_info.value.status_code == 401
async def accepted(_payload):
return {
"status": "verifier_recorded_learning_pending",
"accepted": True,
"runtime_closure_verified": False,
}
monkeypatch.setattr(webhooks, "ingest_agent99_outcome_receipt", accepted)
response = await webhooks.ingest_agent99_outcome_webhook(
request,
"relay-token",
)
assert response["accepted"] is True
assert response["runtime_closure_verified"] is False
@pytest.mark.asyncio
async def test_safe_failed_generation_advances_bounded_and_keeps_one_flight_key(
monkeypatch,
) -> None:
identities = []
flight_calls: list[tuple[str, str]] = []
class GenerationLedger:
async def reserve(self, *, identity, payload):
identities.append(identity)
if identity.execution_generation == "1":
return {
"status": "idempotent_failed",
"claimed": False,
"receipt": {
"status": "dispatch_not_delivered_retry_safe",
"retry_policy": {
"safe_to_retry": True,
"requires_generation_advance": True,
"max_generation": 3,
},
},
}
return {
"status": "reserved",
"claimed": True,
"claim_token": "generation-2-claim",
"receipt": None,
}
async def complete(
self,
*,
identity,
claim_token,
dispatch_receipt,
**_kwargs,
):
return {
**build_agent99_dispatch_receipt_envelope(
identity=identity,
dispatch_receipt=dispatch_receipt,
controlled_apply_authorized=True,
),
"receipt_persisted": True,
}
async def mark_delivery_attempt_started(self, *, identity, claim_token):
return {
"status": "dispatch_delivery_unknown_reconcile_only",
"identity": identity.public_dict(),
"receipt_persisted": True,
}
async def acquire(key, owner, **_kwargs):
flight_calls.append((key, owner))
return {"acquired": True, "reason": "acquired"}
monkeypatch.setattr(
"src.services.agent99_sre_bridge.get_agent99_dispatch_ledger",
lambda: GenerationLedger(),
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
acquire,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda _payload: {
"status": "accepted_inbox_triggered",
"transport": "relay",
"accepted": True,
"inbox_triggered": True,
"delivery_certainty": "delivered",
},
)
result = await bridge_alertmanager_to_agent99(
alert_id="recurrence-2",
alertname="ColdStartGateBlocked",
severity="critical",
namespace="host_recovery",
target_resource="cold-start-gate",
message="cold-start gate blocked",
fingerprint="same-source-fingerprint",
incident_id="INC-20260711-11C751",
route_id="agent99_cold_start_recovery",
)
assert result["status"] == "dispatched"
assert [item.execution_generation for item in identities] == ["1", "2"]
assert identities[0].idempotency_key != identities[1].idempotency_key
assert identities[0].single_flight_key == identities[1].single_flight_key
assert flight_calls == [
(identities[1].single_flight_key, "generation-2-claim")
]
@pytest.mark.asyncio
async def test_retry_not_before_does_not_burn_generation_or_transport(
monkeypatch,
) -> None:
identities = []
class CooldownLedger:
async def reserve(self, *, identity, payload): # type: ignore[no-untyped-def]
identities.append(identity)
return {
"status": "retry_not_before",
"claimed": False,
"next_attempt_at": "2026-07-11T20:10:00",
"receipt": {
"status": "dispatch_not_delivered_retry_safe",
"retry_policy": {
"safe_to_retry": True,
"requires_generation_advance": True,
"max_generation": 3,
},
},
}
async def unexpected_flight(*_args, **_kwargs): # type: ignore[no-untyped-def]
raise AssertionError("cooldown must return before Redis or transport")
monkeypatch.setattr(
"src.services.agent99_sre_bridge.get_agent99_dispatch_ledger",
lambda: CooldownLedger(),
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
unexpected_flight,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda _payload: (_ for _ in ()).throw(
AssertionError("cooldown must not dispatch")
),
)
result = await bridge_alertmanager_to_agent99(
alert_id="cooldown-recurrence",
alertname="ColdStartGateBlocked",
severity="critical",
namespace="host_recovery",
target_resource="cold-start-gate",
message="cold-start gate blocked",
fingerprint="same-source-fingerprint",
incident_id="INC-20260711-11C751",
route_id="agent99_cold_start_recovery",
)
assert result["status"] == "retryable"
assert result["reason"] == "durable_retry_not_before"
assert result["nextAttemptAt"] == "2026-07-11T20:10:00"
assert [item.execution_generation for item in identities] == ["1"]
@pytest.mark.asyncio
async def test_webhook_pending_inbox_is_not_queued_or_authorized(monkeypatch) -> None:
append = AsyncMock()
monkeypatch.setattr(
"src.repositories.alert_operation_log_repository.get_alert_operation_log_repository",
lambda: SimpleNamespace(append=append),
)
monkeypatch.setattr(
webhooks,
"bridge_alertmanager_to_agent99",
AsyncMock(
return_value={
"status": "delivery_pending",
"reason": "accepted_without_inbox_triggered",
"dispatchPerformed": True,
"identity": _identity().public_dict(),
"dispatchReceipt": {
"accepted": True,
"inbox_triggered": False,
"status": "accepted_inbox_pending",
},
"correlatedReceipt": {
"receipt_persisted": True,
"dispatch_promoted": False,
"controlled_apply_authorized": False,
"runtime_closure_verified": False,
},
}
),
)
handoff = await webhooks._try_auto_repair_background(
incident_id="INC-20260711-11C751",
approval_id="00000000-0000-0000-0000-000000000751",
alert_type="ColdStartGateBlocked",
target_resource="cold-start-gate",
namespace="host_recovery",
risk_level="low",
source_alert_id="pending-inbox",
source_alertname="ColdStartGateBlocked",
source_fingerprint="same-source-fingerprint",
)
assert handoff["queued"] is False
assert handoff["dispatch_accepted"] is True
assert handoff["inbox_triggered"] is False
assert handoff["dispatch_promoted"] is False
assert handoff["runtime_execution_authorized"] is False
assert handoff["runtime_closure_verified"] is False
assert append.await_args.kwargs["success"] is False