"""Delivery-truth regressions for migrated Telegram callers. 2026-07-14 Codex v1.0 (Asia/Taipei): an adapter may return ``ok=true`` while the canonical egress gate reports ``provider_send_performed=false``. These tests keep that structured no-send receipt from becoming sent logs, counters, or durable dedup state. No Telegram/provider network is used. """ from __future__ import annotations import asyncio import json from contextlib import asynccontextmanager from datetime import UTC, datetime from types import SimpleNamespace from unittest.mock import AsyncMock import pytest from src.jobs import ai_slo_watchdog_job as watchdog_module from src.jobs import capacity_forecaster_job as capacity_module from src.jobs import hermes_rule_quality_job as hermes_module from src.services import converged_alert_recurrence_notifier as recurrence_module from src.services import decision_manager as decision_module from src.services import failover_alerter as failover_module from src.services import k3s_monitor_service as k3s_module from src.services import telegram_gateway as gateway_module from src.services import weekly_report_service as weekly_module NO_PROVIDER_SEND_RECEIPT = { "ok": True, "_awooop_delivery_status": "sent", "_awooop_provider_send_performed": False, "_awoooi_canonical_route_receipt": { "decision": "blocked", "provider_send_performed": False, }, } _SENT_CHAT_ID = -1001 _SENT_DESTINATION_BINDING = gateway_module._telegram_destination_binding( _SENT_CHAT_ID ) SENT_RECEIPT = { "ok": True, "_awooop_delivery_status": "sent", "_awooop_provider_send_performed": True, "result": {"message_id": 42, "chat": {"id": -1001}}, "_awoooi_canonical_route_receipt": { "schema_version": "telegram_canonical_egress_receipt_v1", "decision": "allowed", "destination_alias": "awoooi_sre_war_room", "destination_binding": _SENT_DESTINATION_BINDING, "sender_bot_alias": "tsenyang_bot", "provider_send_performed": True, }, gateway_module._DELIVERY_CONTEXT_KEY: { "schema_version": "telegram_delivery_context_v1", "method": "sendMessage", "sender_bot_alias": "tsenyang_bot", "destination_alias": "awoooi_sre_war_room", "destination_binding": _SENT_DESTINATION_BINDING, "payload_destination_binding": _SENT_DESTINATION_BINDING, "provider_destination_binding": _SENT_DESTINATION_BINDING, "destination_binding_verified": True, }, } class _Redis: def __init__(self) -> None: self.values: dict[str, str] = {} self.deleted: list[str] = [] async def set( self, key: str, value: str, *, ex: int | None = None, nx: bool | None = None, **_kwargs, ) -> bool | None: del ex if nx and key in self.values: return None self.values[key] = value return True async def delete(self, key: str) -> int: self.deleted.append(key) return 1 if self.values.pop(key, None) is not None else 0 async def exists(self, key: str) -> bool: return key in self.values async def get(self, key: str) -> str | None: return self.values.get(key) async def scan(self, _cursor: int, *, match: str, count: int): del match, count return 0, list(self.values) class _Logger: def __init__(self) -> None: self.events: list[tuple[str, str]] = [] def debug(self, event: str, **_kwargs) -> None: self.events.append(("debug", event)) def info(self, event: str, **_kwargs) -> None: self.events.append(("info", event)) def warning(self, event: str, **_kwargs) -> None: self.events.append(("warning", event)) def error(self, event: str, **_kwargs) -> None: self.events.append(("error", event)) def exception(self, event: str, **_kwargs) -> None: self.events.append(("exception", event)) class _Gateway: def __init__(self, receipt: dict) -> None: self.receipt = receipt def canonical_destination_chat_id(self, **_route) -> str: return "canonical-room" async def send_alert_notification(self, *_args, **_kwargs) -> dict: return dict(self.receipt) async def send_meta_alert(self, **_kwargs) -> dict: return dict(self.receipt) async def send_canonical_message(self, **_kwargs) -> dict: return dict(self.receipt) async def send_text(self, *_args, **_kwargs) -> dict: return dict(self.receipt) @pytest.mark.asyncio async def test_failover_commits_dedup_and_sent_log_only_after_provider_ack( monkeypatch: pytest.MonkeyPatch, ) -> None: redis = _Redis() logger = _Logger() gateway = _Gateway(NO_PROVIDER_SEND_RECEIPT) monkeypatch.setattr(failover_module, "logger", logger) monkeypatch.setattr( "src.services.telegram_gateway.get_telegram_gateway", lambda: gateway, ) alerter = failover_module.FailoverAlerter(redis_client=redis) await alerter.alert_failover({"to_provider": "ollama_gcp_b"}) dedup_key = "alert:failover:ollama_gcp_b:dedup" assert dedup_key not in redis.values assert ("info", "failover_alert_sent") not in logger.events assert ("warning", "failover_alert_no_send") in logger.events gateway.receipt = SENT_RECEIPT await alerter.alert_failover({"to_provider": "ollama_gcp_b"}) assert redis.values[dedup_key] == "sent" assert ("info", "failover_alert_sent") in logger.events @pytest.mark.asyncio async def test_watchdog_releases_dedup_and_does_not_log_sent_without_provider_ack( monkeypatch: pytest.MonkeyPatch, ) -> None: from src.services import ai_slo_calculator, flywheel_stats_service, governance_agent class _SloCalculator: async def calculate(self): return SimpleNamespace(any_violated=False, metrics=[]) class _FlywheelStats: async def compute(self): return SimpleNamespace(execution_success_rate=1.0) class _GovernanceAgent: async def check_trust_drift(self, *, emit_alert: bool): assert emit_alert is False return {"drifted": 0, "drift_ratio": 0.0} redis = _Redis() logger = _Logger() monkeypatch.setattr(watchdog_module, "logger", logger) monkeypatch.setattr(watchdog_module, "get_redis", lambda: redis) monkeypatch.setattr( watchdog_module, "_is_grace_active", AsyncMock(return_value=True) ) monkeypatch.setattr( watchdog_module, "_count_pending_no_tg_sent", AsyncMock(return_value=watchdog_module._TG_SILENCE_THRESHOLD), ) monkeypatch.setattr( watchdog_module, "_count_approved_playbooks", AsyncMock(return_value=(1, 1)), ) monkeypatch.setattr( watchdog_module, "_count_pending_stuck_analysis", AsyncMock(return_value=0), ) monkeypatch.setattr(ai_slo_calculator, "AiSloCalculator", _SloCalculator) monkeypatch.setattr(flywheel_stats_service, "FlywheelStatsService", _FlywheelStats) monkeypatch.setattr( governance_agent, "get_governance_agent", lambda: _GovernanceAgent(), ) monkeypatch.setattr( "src.services.telegram_gateway.get_telegram_gateway", lambda: _Gateway(NO_PROVIDER_SEND_RECEIPT), ) await watchdog_module._check_once() assert redis.values == {} assert ("warning", "ai_slo_watchdog_alert_sent") not in logger.events assert ("error", "ai_slo_watchdog_alert_no_send") in logger.events @pytest.mark.asyncio async def test_advisory_callers_reject_ok_true_without_provider_send( monkeypatch: pytest.MonkeyPatch, ) -> None: from src.services import ai_advisory_helpers monkeypatch.setattr( ai_advisory_helpers, "is_snoozed", AsyncMock(return_value=False), ) monkeypatch.setattr( ai_advisory_helpers, "build_ai_advisory_keyboard", lambda **_kwargs: {}, ) monkeypatch.setattr( "src.services.telegram_gateway.get_telegram_gateway", lambda: _Gateway(NO_PROVIDER_SEND_RECEIPT), ) hermes_sent = await hermes_module._send_telegram_summary( [ { "rule_name": "HighNoiseRule", "noise_rate": 0.9, "tp": 1, "fp": 9, } ], ) capacity_sent = await capacity_module._send_telegram_forecast( { "host-a": [ { "reason": "disk trend", "value": 0.9, } ] }, ) assert hermes_sent is False assert capacity_sent is False @pytest.mark.asyncio async def test_report_callers_reject_ok_true_without_provider_send( monkeypatch: pytest.MonkeyPatch, ) -> None: gateway = _Gateway(NO_PROVIDER_SEND_RECEIPT) monkeypatch.setattr(weekly_module, "get_telegram_gateway", lambda: gateway) monkeypatch.setattr(k3s_module, "get_telegram_gateway", lambda: gateway) weekly = weekly_module.WeeklyReportService() monkeypatch.setattr( weekly, "generate_report", AsyncMock( return_value=SimpleNamespace( format=lambda: "weekly", week_range="2026-W29", ) ), ) k3s = k3s_module.K3sMonitorService() monkeypatch.setattr( k3s, "collect_cluster_status", AsyncMock( return_value=SimpleNamespace( format=lambda: "daily", report_date="2026-07-14", ) ), ) assert await weekly.send_weekly_report() is False assert await k3s.send_daily_report() is False @pytest.mark.asyncio async def test_recurrence_releases_reservation_and_never_logs_sent_without_ack( monkeypatch: pytest.MonkeyPatch, ) -> None: redis = _Redis() logger = _Logger() monkeypatch.setattr(recurrence_module, "logger", logger) monkeypatch.setattr(recurrence_module, "get_redis", lambda: redis) monkeypatch.setattr( recurrence_module, "should_notify_converged_alert_recurrence", AsyncMock(return_value=True), ) monkeypatch.setattr( recurrence_module, "get_telegram_gateway", lambda: _Gateway(NO_PROVIDER_SEND_RECEIPT), ) await recurrence_module.notify_converged_alert_recurrence( source="alertmanager", fingerprint="fp-no-provider-send", alertname="ServiceDown", severity="critical", namespace="awoooi-prod", target_resource="api", hit_count=2, incident_id="INC-NO-PROVIDER-SEND", approval_id="approval-1", ) key = recurrence_module.converged_alert_recurrence_key("fp-no-provider-send") assert key in redis.deleted assert ("info", "converged_alert_recurrence_sent") not in logger.events assert ("error", "converged_alert_recurrence_failed") in logger.events @pytest.mark.asyncio @pytest.mark.parametrize("provider_ack, expected_resent", [(False, 0), (True, 1)]) async def test_stale_token_replay_counts_only_awaited_delivery_receipts( monkeypatch: pytest.MonkeyPatch, provider_ack: bool, expected_resent: int, ) -> None: from src.db import base as db_base from src.repositories import incident_repository incident_id = "INC-STALE-DELIVERY-TRUTH" redis = _Redis() redis.values["decision:DEC-STALE"] = json.dumps( { "state": decision_module.DecisionState.READY.value, "incident_id": incident_id, "proposal_data": {"description": "bounded replay"}, "token": "DEC-STALE", } ) incident = SimpleNamespace( incident_id=incident_id, status="open", affected_services=["api"], created_at=datetime.now(UTC), signals=[ SimpleNamespace( labels={"alertname": "ServiceDown"}, alert_name="ServiceDown", annotations={}, ) ], ) @asynccontextmanager async def _db_context(): yield object() class _IncidentRepository: def __init__(self, _db) -> None: pass async def get_by_id(self, requested_id: str): assert requested_id == incident_id return incident push = AsyncMock(return_value=provider_ack) logger = _Logger() monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) monkeypatch.setattr(db_base, "get_db_context", _db_context) monkeypatch.setattr( incident_repository, "IncidentDBRepository", _IncidentRepository, ) monkeypatch.setattr(decision_module, "_push_decision_to_telegram", push) monkeypatch.setattr(decision_module, "logger", logger) monkeypatch.setattr(asyncio, "sleep", AsyncMock()) manager = object.__new__(decision_module.DecisionManager) resent = await manager.resend_stale_ready_tokens() assert resent == expected_resent push.assert_awaited_once() if provider_ack: assert ("info", "stale_ready_token_resent") in logger.events else: assert ("info", "stale_ready_token_resent") not in logger.events assert ("warning", "stale_ready_token_no_send") in logger.events