"""Regression tests for callers that must distinguish routing from delivery.""" from __future__ import annotations from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta from types import SimpleNamespace from unittest.mock import AsyncMock import pytest from src.models.incident import Incident, Severity, Signal from src.repositories.alert_operation_log_repository import ALERT_EVENT_TYPES from src.services import decision_manager as decision_module from src.services import report_generation_service as report_module from src.services import telegram_gateway as gateway_module from src.services.ai_rate_limiter import AIRateLimiter from src.services.approval_execution import ApprovalExecutionService from src.services.report_generation_service import ReportGenerationService NO_SEND_RECEIPT = { "ok": True, "_awooop_delivery_status": "blocked_no_egress", "_awooop_provider_send_performed": False, "_awoooi_canonical_route_receipt": { "decision": "blocked", "provider_send_performed": False, }, } _SENT_CHAT_ID = -1001 _SENT_DESTINATION_BINDING = gateway_module._telegram_destination_binding( _SENT_CHAT_ID ) SENT_RECEIPT = { "ok": True, "_awooop_delivery_status": "sent", "_awooop_provider_send_performed": True, "result": {"message_id": 42, "chat": {"id": -1001}}, "_awoooi_canonical_route_receipt": { "schema_version": "telegram_canonical_egress_receipt_v1", "decision": "allowed", "destination_alias": "awoooi_sre_war_room", "destination_binding": _SENT_DESTINATION_BINDING, "sender_bot_alias": "tsenyang_bot", "provider_send_performed": True, }, gateway_module._DELIVERY_CONTEXT_KEY: { "schema_version": "telegram_delivery_context_v1", "method": "sendMessage", "sender_bot_alias": "tsenyang_bot", "destination_alias": "awoooi_sre_war_room", "destination_binding": _SENT_DESTINATION_BINDING, "payload_destination_binding": _SENT_DESTINATION_BINDING, "provider_destination_binding": _SENT_DESTINATION_BINDING, "destination_binding_verified": True, }, } class FakeRedis: def __init__(self) -> None: self.values: dict[str, str] = {} self.set_calls: list[tuple[str, str, int | None]] = [] self.setex_calls: list[tuple[str, int, str]] = [] async def get(self, key: str): return self.values.get(key) async def exists(self, key: str) -> bool: return key in self.values async def set(self, key: str, value: str, ex: int | None = None, **_kwargs): self.values[key] = value self.set_calls.append((key, value, ex)) return True async def setex(self, key: str, ttl: int, value: str) -> bool: self.values[key] = value self.setex_calls.append((key, ttl, value)) return True class FakeLogger: def __init__(self) -> None: self.events: list[tuple[str, str]] = [] def debug(self, event: str, **_kwargs) -> None: self.events.append(("debug", event)) def info(self, event: str, **_kwargs) -> None: self.events.append(("info", event)) def warning(self, event: str, **_kwargs) -> None: self.events.append(("warning", event)) def error(self, event: str, **_kwargs) -> None: self.events.append(("error", event)) @pytest.mark.asyncio @pytest.mark.parametrize("method_name", ["_send_cost_alert", "_send_cost_warning"]) @pytest.mark.parametrize("delivery_result", [NO_SEND_RECEIPT, None]) async def test_cost_alert_dedup_is_written_only_after_affirmative_delivery( monkeypatch, method_name: str, delivery_result: object, ) -> None: redis = FakeRedis() class Gateway: def canonical_destination_chat_id(self, **_route) -> str: return "sre-room" async def send_canonical_message(self, **_payload): return delivery_result monkeypatch.setattr( "src.services.telegram_gateway.get_telegram_gateway", lambda: Gateway(), ) limiter = AIRateLimiter() limiter._redis = redis if method_name == "_send_cost_alert": await limiter._send_cost_alert("gemini", 5.1, 5.0) else: await limiter._send_cost_warning("gemini", 4.1, 4.0) assert redis.set_calls == [] @pytest.mark.asyncio async def test_cost_alert_sets_dedup_after_affirmative_delivery(monkeypatch) -> None: redis = FakeRedis() class Gateway: def canonical_destination_chat_id(self, **_route) -> str: return "sre-room" async def send_canonical_message(self, **_payload): return dict(SENT_RECEIPT) monkeypatch.setattr( "src.services.telegram_gateway.get_telegram_gateway", lambda: Gateway(), ) limiter = AIRateLimiter() limiter._redis = redis await limiter._send_cost_alert("gemini", 5.1, 5.0) assert len(redis.set_calls) == 1 assert redis.set_calls[0][0] == "ai_rate:cost_alert_sent:gemini" @pytest.mark.asyncio async def test_p3_reports_fail_closed_before_collection_when_route_missing( monkeypatch, ) -> None: class Gateway: def canonical_destination_chat_id(self, **_route) -> str: return "" async def send_to_group(self, *_args, **_kwargs): raise AssertionError("provider must not be called without a route") monkeypatch.setattr( "src.services.telegram_gateway.get_telegram_gateway", lambda: Gateway(), ) service = ReportGenerationService() daily = AsyncMock() monthly = AsyncMock() monkeypatch.setattr(service, "collect_daily_kpi", daily) monkeypatch.setattr(service, "collect_report_source_health", monthly) await service.send_daily_report() monthly_result = await service.send_monthly_report() daily.assert_not_awaited() monthly.assert_not_awaited() assert monthly_result is False @pytest.mark.asyncio async def test_postmortem_no_ack_retries_and_verifies_fallback(monkeypatch) -> None: send = AsyncMock(return_value=dict(NO_SEND_RECEIPT)) class Gateway: send_to_group = send monkeypatch.setattr( "src.services.telegram_gateway.get_telegram_gateway", lambda: Gateway(), ) monkeypatch.setattr(report_module.asyncio, "sleep", AsyncMock()) service = ReportGenerationService() monkeypatch.setattr(service, "_persist_postmortem_km", AsyncMock()) now = datetime.now(UTC) await service.trigger_postmortem( incident_id="INC-NO-ACK", title="provider did not acknowledge", created_at=now - timedelta(minutes=12), resolved_at=now, ) # Three bounded attempts plus one simplified fallback; all remain no-send. assert send.await_count == 4 @pytest.mark.asyncio async def test_approval_blocked_receipt_writes_no_send_not_sent(monkeypatch) -> None: redis = FakeRedis() operations: list[dict] = [] class Gateway: def canonical_destination_chat_id(self, **_route) -> str: return "sre-room" async def _send_request(self, *_args, **_kwargs): return dict(SENT_RECEIPT) async def send_canonical_message(self, **_payload): return dict(NO_SEND_RECEIPT) class OperationRepository: async def append(self, event_type: str, **kwargs) -> None: operations.append({"event_type": event_type, **kwargs}) class BrokenDB: async def execute(self, *_args, **_kwargs): raise RuntimeError("expected test DB bypass") @asynccontextmanager async def fake_db_context(): yield BrokenDB() monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) monkeypatch.setattr("src.db.base.get_db_context", fake_db_context) monkeypatch.setattr( "src.services.telegram_gateway.get_telegram_gateway", lambda: Gateway(), ) monkeypatch.setattr( "src.repositories.alert_operation_log_repository.get_alert_operation_log_repository", lambda: OperationRepository(), ) monkeypatch.setattr( "src.services.approval_execution.incident_truth_chain_button_row", lambda _incident_id: None, ) service = ApprovalExecutionService() monkeypatch.setattr( service, "_fetch_operator_outcome_for_result", AsyncMock( return_value={ "state": "completed_verified", "summary_zh": "verified", "needs_human": False, "next_action": "monitor", "notification": {"channels": []}, "execution_result": {}, } ), ) approval = SimpleNamespace( id="approval-no-ack", incident_id="INC-NO-ACK", action="kubectl rollout restart deployment/api", requested_by="operator", ) await service._push_execution_result_to_alert( approval, success=True, error=None, execution_kind="controlled_apply", repair_executed=True, repair_attempted=True, ) assert [row["event_type"] for row in operations] == ["NOTIFICATION_CLASSIFIED"] assert operations[0]["event_type"] in ALERT_EVENT_TYPES assert operations[0]["action_detail"] == "telegram_execution_result_no_send" assert operations[0]["success"] is False assert operations[0]["context"]["execution_success"] is True def _incident() -> Incident: return Incident( incident_id="INC-DECISION-NO-ACK", severity=Severity.P3, signals=[ Signal( alert_name="ContainerPressure", severity=Severity.P3, source="alertmanager", fired_at=datetime.now(UTC), labels={"alertname": "ContainerPressure"}, ) ], affected_services=["api"], ) @pytest.mark.asyncio async def test_decision_no_ack_does_not_set_fingerprint_dedup(monkeypatch) -> None: from src.services.telegram_gateway import NotificationType redis = FakeRedis() class Gateway: async def send_info_notification(self, **_payload): return dict(NO_SEND_RECEIPT) class ApprovalService: async def update_action_by_incident_id(self, **_payload) -> None: return None monkeypatch.setattr(decision_module.settings, "OPENCLAW_TG_BOT_TOKEN", "configured") monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) monkeypatch.setattr( "src.services.telegram_gateway.get_telegram_gateway", lambda: Gateway(), ) monkeypatch.setattr( "src.services.telegram_gateway.classify_notification", lambda **_kwargs: NotificationType.TYPE_1, ) monkeypatch.setattr( "src.services.approval_db.get_approval_service", lambda: ApprovalService(), ) await decision_module._push_decision_to_telegram( _incident(), { "action": "OBSERVE", "description": "bounded observation", "reasoning": "診斷:bounded observation", "risk_level": "low", "confidence": 0.9, "source": "test", }, ) assert not any(key.startswith("telegram_sent:fp:") for key in redis.values) @pytest.mark.asyncio async def test_log_summary_route_preflight_runs_before_llm(monkeypatch) -> None: summarize = AsyncMock(return_value="summary") class Gateway: def canonical_destination_chat_id(self, **_route) -> str: return "" async def send_as_nemotron(self, *_args, **_kwargs): raise AssertionError("provider must not be called without a route") monkeypatch.setattr( "src.services.telegram_gateway.get_telegram_gateway", lambda: Gateway(), ) monkeypatch.setattr( "src.services.log_summary_service.get_log_summary_service", lambda: SimpleNamespace(summarize_with_soft_timeout=summarize), ) await decision_module._send_log_summary(_incident()) summarize.assert_not_awaited() @pytest.mark.asyncio async def test_auto_repair_fallback_requires_affirmative_delivery(monkeypatch) -> None: test_logger = FakeLogger() class Gateway: async def append_incident_update(self, **_payload) -> bool: return False def canonical_destination_chat_id(self, **_route) -> str: return "sre-room" async def send_alert_notification(self, *_args, **_kwargs): return dict(NO_SEND_RECEIPT) class DB: async def execute(self, *_args, **_kwargs) -> None: return None async def commit(self) -> None: return None @asynccontextmanager async def fake_db_context(): yield DB() monkeypatch.setattr(decision_module, "logger", test_logger) monkeypatch.setattr("src.db.base.get_db_context", fake_db_context) monkeypatch.setattr( "src.services.telegram_gateway.get_telegram_gateway", lambda: Gateway(), ) await decision_module._push_auto_repair_result( _incident(), action="bounded observe", success=False, error="verifier_failed", ) event_names = [event for _level, event in test_logger.events] assert "auto_repair_result_no_send" in event_names assert "auto_repair_result_sent" not in event_names