420 lines
13 KiB
Python
420 lines
13 KiB
Python
"""Delivery-truth regressions for migrated Telegram callers.
|
|
|
|
2026-07-14 Codex v1.0 (Asia/Taipei): an adapter may return ``ok=true`` while
|
|
the canonical egress gate reports ``provider_send_performed=false``. These
|
|
tests keep that structured no-send receipt from becoming sent logs, counters,
|
|
or durable dedup state. No Telegram/provider network is used.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
from contextlib import asynccontextmanager
|
|
from datetime import UTC, datetime
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock
|
|
|
|
import pytest
|
|
|
|
from src.jobs import ai_slo_watchdog_job as watchdog_module
|
|
from src.jobs import capacity_forecaster_job as capacity_module
|
|
from src.jobs import hermes_rule_quality_job as hermes_module
|
|
from src.services import converged_alert_recurrence_notifier as recurrence_module
|
|
from src.services import decision_manager as decision_module
|
|
from src.services import failover_alerter as failover_module
|
|
from src.services import k3s_monitor_service as k3s_module
|
|
from src.services import telegram_gateway as gateway_module
|
|
from src.services import weekly_report_service as weekly_module
|
|
|
|
NO_PROVIDER_SEND_RECEIPT = {
|
|
"ok": True,
|
|
"_awooop_delivery_status": "sent",
|
|
"_awooop_provider_send_performed": False,
|
|
"_awoooi_canonical_route_receipt": {
|
|
"decision": "blocked",
|
|
"provider_send_performed": False,
|
|
},
|
|
}
|
|
|
|
_SENT_CHAT_ID = -1001
|
|
_SENT_DESTINATION_BINDING = gateway_module._telegram_destination_binding(
|
|
_SENT_CHAT_ID
|
|
)
|
|
SENT_RECEIPT = {
|
|
"ok": True,
|
|
"_awooop_delivery_status": "sent",
|
|
"_awooop_provider_send_performed": True,
|
|
"result": {"message_id": 42, "chat": {"id": -1001}},
|
|
"_awoooi_canonical_route_receipt": {
|
|
"schema_version": "telegram_canonical_egress_receipt_v1",
|
|
"decision": "allowed",
|
|
"destination_alias": "awoooi_sre_war_room",
|
|
"destination_binding": _SENT_DESTINATION_BINDING,
|
|
"sender_bot_alias": "tsenyang_bot",
|
|
"provider_send_performed": True,
|
|
},
|
|
gateway_module._DELIVERY_CONTEXT_KEY: {
|
|
"schema_version": "telegram_delivery_context_v1",
|
|
"method": "sendMessage",
|
|
"sender_bot_alias": "tsenyang_bot",
|
|
"destination_alias": "awoooi_sre_war_room",
|
|
"destination_binding": _SENT_DESTINATION_BINDING,
|
|
"payload_destination_binding": _SENT_DESTINATION_BINDING,
|
|
"provider_destination_binding": _SENT_DESTINATION_BINDING,
|
|
"destination_binding_verified": True,
|
|
},
|
|
}
|
|
|
|
|
|
class _Redis:
|
|
def __init__(self) -> None:
|
|
self.values: dict[str, str] = {}
|
|
self.deleted: list[str] = []
|
|
|
|
async def set(
|
|
self,
|
|
key: str,
|
|
value: str,
|
|
*,
|
|
ex: int | None = None,
|
|
nx: bool | None = None,
|
|
**_kwargs,
|
|
) -> bool | None:
|
|
del ex
|
|
if nx and key in self.values:
|
|
return None
|
|
self.values[key] = value
|
|
return True
|
|
|
|
async def delete(self, key: str) -> int:
|
|
self.deleted.append(key)
|
|
return 1 if self.values.pop(key, None) is not None else 0
|
|
|
|
async def exists(self, key: str) -> bool:
|
|
return key in self.values
|
|
|
|
async def get(self, key: str) -> str | None:
|
|
return self.values.get(key)
|
|
|
|
async def scan(self, _cursor: int, *, match: str, count: int):
|
|
del match, count
|
|
return 0, list(self.values)
|
|
|
|
|
|
class _Logger:
|
|
def __init__(self) -> None:
|
|
self.events: list[tuple[str, str]] = []
|
|
|
|
def debug(self, event: str, **_kwargs) -> None:
|
|
self.events.append(("debug", event))
|
|
|
|
def info(self, event: str, **_kwargs) -> None:
|
|
self.events.append(("info", event))
|
|
|
|
def warning(self, event: str, **_kwargs) -> None:
|
|
self.events.append(("warning", event))
|
|
|
|
def error(self, event: str, **_kwargs) -> None:
|
|
self.events.append(("error", event))
|
|
|
|
def exception(self, event: str, **_kwargs) -> None:
|
|
self.events.append(("exception", event))
|
|
|
|
|
|
class _Gateway:
|
|
def __init__(self, receipt: dict) -> None:
|
|
self.receipt = receipt
|
|
|
|
def canonical_destination_chat_id(self, **_route) -> str:
|
|
return "canonical-room"
|
|
|
|
async def send_alert_notification(self, *_args, **_kwargs) -> dict:
|
|
return dict(self.receipt)
|
|
|
|
async def send_meta_alert(self, **_kwargs) -> dict:
|
|
return dict(self.receipt)
|
|
|
|
async def send_canonical_message(self, **_kwargs) -> dict:
|
|
return dict(self.receipt)
|
|
|
|
async def send_text(self, *_args, **_kwargs) -> dict:
|
|
return dict(self.receipt)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_failover_commits_dedup_and_sent_log_only_after_provider_ack(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
redis = _Redis()
|
|
logger = _Logger()
|
|
gateway = _Gateway(NO_PROVIDER_SEND_RECEIPT)
|
|
monkeypatch.setattr(failover_module, "logger", logger)
|
|
monkeypatch.setattr(
|
|
"src.services.telegram_gateway.get_telegram_gateway",
|
|
lambda: gateway,
|
|
)
|
|
alerter = failover_module.FailoverAlerter(redis_client=redis)
|
|
|
|
await alerter.alert_failover({"to_provider": "ollama_gcp_b"})
|
|
|
|
dedup_key = "alert:failover:ollama_gcp_b:dedup"
|
|
assert dedup_key not in redis.values
|
|
assert ("info", "failover_alert_sent") not in logger.events
|
|
assert ("warning", "failover_alert_no_send") in logger.events
|
|
|
|
gateway.receipt = SENT_RECEIPT
|
|
await alerter.alert_failover({"to_provider": "ollama_gcp_b"})
|
|
|
|
assert redis.values[dedup_key] == "sent"
|
|
assert ("info", "failover_alert_sent") in logger.events
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watchdog_releases_dedup_and_does_not_log_sent_without_provider_ack(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
from src.services import ai_slo_calculator, flywheel_stats_service, governance_agent
|
|
|
|
class _SloCalculator:
|
|
async def calculate(self):
|
|
return SimpleNamespace(any_violated=False, metrics=[])
|
|
|
|
class _FlywheelStats:
|
|
async def compute(self):
|
|
return SimpleNamespace(execution_success_rate=1.0)
|
|
|
|
class _GovernanceAgent:
|
|
async def check_trust_drift(self, *, emit_alert: bool):
|
|
assert emit_alert is False
|
|
return {"drifted": 0, "drift_ratio": 0.0}
|
|
|
|
redis = _Redis()
|
|
logger = _Logger()
|
|
monkeypatch.setattr(watchdog_module, "logger", logger)
|
|
monkeypatch.setattr(watchdog_module, "get_redis", lambda: redis)
|
|
monkeypatch.setattr(
|
|
watchdog_module, "_is_grace_active", AsyncMock(return_value=True)
|
|
)
|
|
monkeypatch.setattr(
|
|
watchdog_module,
|
|
"_count_pending_no_tg_sent",
|
|
AsyncMock(return_value=watchdog_module._TG_SILENCE_THRESHOLD),
|
|
)
|
|
monkeypatch.setattr(
|
|
watchdog_module,
|
|
"_count_approved_playbooks",
|
|
AsyncMock(return_value=(1, 1)),
|
|
)
|
|
monkeypatch.setattr(
|
|
watchdog_module,
|
|
"_count_pending_stuck_analysis",
|
|
AsyncMock(return_value=0),
|
|
)
|
|
monkeypatch.setattr(ai_slo_calculator, "AiSloCalculator", _SloCalculator)
|
|
monkeypatch.setattr(flywheel_stats_service, "FlywheelStatsService", _FlywheelStats)
|
|
monkeypatch.setattr(
|
|
governance_agent,
|
|
"get_governance_agent",
|
|
lambda: _GovernanceAgent(),
|
|
)
|
|
monkeypatch.setattr(
|
|
"src.services.telegram_gateway.get_telegram_gateway",
|
|
lambda: _Gateway(NO_PROVIDER_SEND_RECEIPT),
|
|
)
|
|
|
|
await watchdog_module._check_once()
|
|
|
|
assert redis.values == {}
|
|
assert ("warning", "ai_slo_watchdog_alert_sent") not in logger.events
|
|
assert ("error", "ai_slo_watchdog_alert_no_send") in logger.events
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_advisory_callers_reject_ok_true_without_provider_send(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
from src.services import ai_advisory_helpers
|
|
|
|
monkeypatch.setattr(
|
|
ai_advisory_helpers,
|
|
"is_snoozed",
|
|
AsyncMock(return_value=False),
|
|
)
|
|
monkeypatch.setattr(
|
|
ai_advisory_helpers,
|
|
"build_ai_advisory_keyboard",
|
|
lambda **_kwargs: {},
|
|
)
|
|
monkeypatch.setattr(
|
|
"src.services.telegram_gateway.get_telegram_gateway",
|
|
lambda: _Gateway(NO_PROVIDER_SEND_RECEIPT),
|
|
)
|
|
|
|
hermes_sent = await hermes_module._send_telegram_summary(
|
|
[
|
|
{
|
|
"rule_name": "HighNoiseRule",
|
|
"noise_rate": 0.9,
|
|
"tp": 1,
|
|
"fp": 9,
|
|
}
|
|
],
|
|
)
|
|
capacity_sent = await capacity_module._send_telegram_forecast(
|
|
{
|
|
"host-a": [
|
|
{
|
|
"reason": "disk trend",
|
|
"value": 0.9,
|
|
}
|
|
]
|
|
},
|
|
)
|
|
|
|
assert hermes_sent is False
|
|
assert capacity_sent is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_report_callers_reject_ok_true_without_provider_send(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
gateway = _Gateway(NO_PROVIDER_SEND_RECEIPT)
|
|
monkeypatch.setattr(weekly_module, "get_telegram_gateway", lambda: gateway)
|
|
monkeypatch.setattr(k3s_module, "get_telegram_gateway", lambda: gateway)
|
|
|
|
weekly = weekly_module.WeeklyReportService()
|
|
monkeypatch.setattr(
|
|
weekly,
|
|
"generate_report",
|
|
AsyncMock(
|
|
return_value=SimpleNamespace(
|
|
format=lambda: "weekly",
|
|
week_range="2026-W29",
|
|
)
|
|
),
|
|
)
|
|
k3s = k3s_module.K3sMonitorService()
|
|
monkeypatch.setattr(
|
|
k3s,
|
|
"collect_cluster_status",
|
|
AsyncMock(
|
|
return_value=SimpleNamespace(
|
|
format=lambda: "daily",
|
|
report_date="2026-07-14",
|
|
)
|
|
),
|
|
)
|
|
|
|
assert await weekly.send_weekly_report() is False
|
|
assert await k3s.send_daily_report() is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_recurrence_releases_reservation_and_never_logs_sent_without_ack(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
redis = _Redis()
|
|
logger = _Logger()
|
|
monkeypatch.setattr(recurrence_module, "logger", logger)
|
|
monkeypatch.setattr(recurrence_module, "get_redis", lambda: redis)
|
|
monkeypatch.setattr(
|
|
recurrence_module,
|
|
"should_notify_converged_alert_recurrence",
|
|
AsyncMock(return_value=True),
|
|
)
|
|
monkeypatch.setattr(
|
|
recurrence_module,
|
|
"get_telegram_gateway",
|
|
lambda: _Gateway(NO_PROVIDER_SEND_RECEIPT),
|
|
)
|
|
|
|
await recurrence_module.notify_converged_alert_recurrence(
|
|
source="alertmanager",
|
|
fingerprint="fp-no-provider-send",
|
|
alertname="ServiceDown",
|
|
severity="critical",
|
|
namespace="awoooi-prod",
|
|
target_resource="api",
|
|
hit_count=2,
|
|
incident_id="INC-NO-PROVIDER-SEND",
|
|
approval_id="approval-1",
|
|
)
|
|
|
|
key = recurrence_module.converged_alert_recurrence_key("fp-no-provider-send")
|
|
assert key in redis.deleted
|
|
assert ("info", "converged_alert_recurrence_sent") not in logger.events
|
|
assert ("error", "converged_alert_recurrence_failed") in logger.events
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("provider_ack, expected_resent", [(False, 0), (True, 1)])
|
|
async def test_stale_token_replay_counts_only_awaited_delivery_receipts(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
provider_ack: bool,
|
|
expected_resent: int,
|
|
) -> None:
|
|
from src.db import base as db_base
|
|
from src.repositories import incident_repository
|
|
|
|
incident_id = "INC-STALE-DELIVERY-TRUTH"
|
|
redis = _Redis()
|
|
redis.values["decision:DEC-STALE"] = json.dumps(
|
|
{
|
|
"state": decision_module.DecisionState.READY.value,
|
|
"incident_id": incident_id,
|
|
"proposal_data": {"description": "bounded replay"},
|
|
"token": "DEC-STALE",
|
|
}
|
|
)
|
|
incident = SimpleNamespace(
|
|
incident_id=incident_id,
|
|
status="open",
|
|
affected_services=["api"],
|
|
created_at=datetime.now(UTC),
|
|
signals=[
|
|
SimpleNamespace(
|
|
labels={"alertname": "ServiceDown"},
|
|
alert_name="ServiceDown",
|
|
annotations={},
|
|
)
|
|
],
|
|
)
|
|
|
|
@asynccontextmanager
|
|
async def _db_context():
|
|
yield object()
|
|
|
|
class _IncidentRepository:
|
|
def __init__(self, _db) -> None:
|
|
pass
|
|
|
|
async def get_by_id(self, requested_id: str):
|
|
assert requested_id == incident_id
|
|
return incident
|
|
|
|
push = AsyncMock(return_value=provider_ack)
|
|
logger = _Logger()
|
|
monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis)
|
|
monkeypatch.setattr(db_base, "get_db_context", _db_context)
|
|
monkeypatch.setattr(
|
|
incident_repository,
|
|
"IncidentDBRepository",
|
|
_IncidentRepository,
|
|
)
|
|
monkeypatch.setattr(decision_module, "_push_decision_to_telegram", push)
|
|
monkeypatch.setattr(decision_module, "logger", logger)
|
|
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
|
|
|
|
manager = object.__new__(decision_module.DecisionManager)
|
|
resent = await manager.resend_stale_ready_tokens()
|
|
|
|
assert resent == expected_resent
|
|
push.assert_awaited_once()
|
|
if provider_ack:
|
|
assert ("info", "stale_ready_token_resent") in logger.events
|
|
else:
|
|
assert ("info", "stale_ready_token_resent") not in logger.events
|
|
assert ("warning", "stale_ready_token_no_send") in logger.events
|