"""
P0 #4 heartbeat 噪音降頻測試
2026-05-03 Claude Opus 4.7 + 統帥 ogt
驗證 send_heartbeat() 的低噪音邏輯:
健康(無 warnings)→ 不即時推 Telegram,只留 metrics/log/每日摘要
有 warnings 且跟上次同 hash → 跳過
有 warnings 且跟上次不同 → 立即推送
warnings 消失 → 只推一次恢復通知
直接測 telegram_gateway.send_heartbeat(),mock 掉 redis + report + send_to_group。
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
class FakeRedis:
"""模擬 Redis 行為,記錄 set/get/delete 呼叫"""
def __init__(self):
self._store: dict[str, str] = {}
self.set_calls: list[tuple] = []
self.delete_calls: list[str] = []
async def set(self, key: str, value: str, nx: bool = False, ex: int | None = None):
if nx and key in self._store:
return False
self._store[key] = value
self.set_calls.append((key, value, nx, ex))
return True
async def setex(self, key: str, ttl: int, value: str):
self._store[key] = value
self.set_calls.append((key, value, False, ttl))
return True
async def get(self, key: str):
return self._store.get(key)
async def exists(self, key: str):
return key in self._store
async def delete(self, *keys):
for k in keys:
self._store.pop(k, None)
self.delete_calls.append(k)
return len(keys)
def preset(self, key: str, value: str = "1"):
"""測試前預設 key"""
self._store[key] = value
def _make_report(warnings: list[str] | None = None):
"""構造 fake HeartbeatReport"""
from datetime import datetime, timezone
from src.services.heartbeat_report_service import HeartbeatReport
return HeartbeatReport(
timestamp=datetime.now(timezone.utc),
warnings=warnings or [],
)
@pytest.fixture
def sre_group_configured(monkeypatch):
"""Heartbeat 正式推送只能在 AwoooI SRE 戰情室設定存在時成立。"""
from src.services.telegram_gateway import settings
monkeypatch.setattr(settings, "SRE_GROUP_CHAT_ID", "-1003711974679")
@pytest.fixture
def gateway_with_fake_redis():
"""構造 telegram gateway 實例 + 注入 fake redis"""
from src.services.telegram_gateway import TelegramGateway
gw = TelegramGateway.__new__(TelegramGateway) # 跳過 __init__
gw._initialized = True
gw._last_message_time = None
gw.send_to_group = AsyncMock()
gw.send_notification = AsyncMock()
fake_redis = FakeRedis()
return gw, fake_redis
class TestHeartbeatDedup:
"""P0 #4 heartbeat 降頻邏輯"""
@pytest.mark.asyncio
async def test_healthy_first_send_is_suppressed(
self,
gateway_with_fake_redis,
sre_group_configured,
):
"""健康狀態第一次檢查 → 不推 Telegram,只記錄 suppressed marker"""
gw, fake_redis = gateway_with_fake_redis
with patch("src.core.redis_client.get_redis", return_value=fake_redis), \
patch("src.services.heartbeat_report_service.HeartbeatReportService") as MockSvc, \
patch("src.services.heartbeat_report_service.report_to_telegram_html",
return_value="healthy"):
MockSvc.return_value.collect = AsyncMock(return_value=_make_report([]))
result = await gw.send_heartbeat()
assert result is True
assert "heartbeat:healthy_suppressed_last_seen" in fake_redis._store
gw.send_to_group.assert_not_called()
gw.send_notification.assert_not_called()
@pytest.mark.asyncio
async def test_healthy_second_send_stays_suppressed(
self,
gateway_with_fake_redis,
sre_group_configured,
):
"""健康狀態第二次檢查 → 仍不推 Telegram"""
gw, fake_redis = gateway_with_fake_redis
fake_redis.preset("heartbeat:healthy_suppressed_last_seen")
with patch("src.core.redis_client.get_redis", return_value=fake_redis), \
patch("src.services.heartbeat_report_service.HeartbeatReportService") as MockSvc, \
patch("src.services.heartbeat_report_service.report_to_telegram_html",
return_value="healthy"):
MockSvc.return_value.collect = AsyncMock(return_value=_make_report([]))
result = await gw.send_heartbeat()
assert result is True
gw.send_to_group.assert_not_called()
gw.send_notification.assert_not_called()
@pytest.mark.asyncio
async def test_warnings_unchanged_skipped(
self,
gateway_with_fake_redis,
sre_group_configured,
):
"""有 warnings 跟上次同 hash → 跳過"""
gw, fake_redis = gateway_with_fake_redis
warnings = ["Pod api-x Failed", "Redis: down"]
# 預設上次 hash
from src.services.telegram_gateway import _heartbeat_warnings_hash
last_hash = _heartbeat_warnings_hash(warnings)
fake_redis.preset("heartbeat:warnings_hash", last_hash)
with patch("src.core.redis_client.get_redis", return_value=fake_redis), \
patch("src.services.heartbeat_report_service.HeartbeatReportService") as MockSvc, \
patch("src.services.heartbeat_report_service.report_to_telegram_html",
return_value="warnings"):
MockSvc.return_value.collect = AsyncMock(return_value=_make_report(warnings))
result = await gw.send_heartbeat()
assert result is True
gw.send_to_group.assert_not_called()
gw.send_notification.assert_not_called()
@pytest.mark.asyncio
async def test_warnings_with_same_actionable_condition_are_skipped(
self,
gateway_with_fake_redis,
sre_group_configured,
):
"""同一可處置 warning 即使 HTTP / timeout / latency 變動也不重複洗版"""
gw, fake_redis = gateway_with_fake_redis
from src.services.telegram_gateway import _heartbeat_warnings_hash
fake_redis.preset(
"heartbeat:warnings_hash",
_heartbeat_warnings_hash(["Ollama 111 異常: ❌ HTTP 502 124ms"]),
)
with patch("src.core.redis_client.get_redis", return_value=fake_redis), \
patch("src.services.heartbeat_report_service.HeartbeatReportService") as MockSvc, \
patch("src.services.heartbeat_report_service.report_to_telegram_html",
return_value="warnings"):
MockSvc.return_value.collect = AsyncMock(
return_value=_make_report(["Ollama 111 異常: ❌ HTTP 504 236ms"])
)
result = await gw.send_heartbeat()
assert result is True
gw.send_to_group.assert_not_called()
gw.send_notification.assert_not_called()
@pytest.mark.asyncio
async def test_warnings_changed_pushes(
self,
gateway_with_fake_redis,
sre_group_configured,
):
"""有 warnings 但跟上次不同 → 立即推送"""
gw, fake_redis = gateway_with_fake_redis
# 預設舊的 hash(跟新 warnings 不同)
fake_redis.preset("heartbeat:warnings_hash", "old1234567890")
with patch("src.core.redis_client.get_redis", return_value=fake_redis), \
patch("src.services.heartbeat_report_service.HeartbeatReportService") as MockSvc, \
patch("src.services.heartbeat_report_service.report_to_telegram_html",
return_value="new warnings"):
MockSvc.return_value.collect = AsyncMock(
return_value=_make_report(["Pod api-y Failed"]) # 新 warnings
)
result = await gw.send_heartbeat()
assert result is True
gw.send_to_group.assert_called_once()
gw.send_notification.assert_not_called()
@pytest.mark.asyncio
async def test_warnings_to_healthy_clears_warnings_hash(
self,
gateway_with_fake_redis,
sre_group_configured,
):
"""從有事 → 健康:清掉 warnings_hash marker,下次有事可立即推"""
gw, fake_redis = gateway_with_fake_redis
fake_redis.preset("heartbeat:warnings_hash", "old1234567890")
with patch("src.core.redis_client.get_redis", return_value=fake_redis), \
patch("src.services.heartbeat_report_service.HeartbeatReportService") as MockSvc, \
patch("src.services.heartbeat_report_service.report_to_telegram_html",
return_value="healthy"):
MockSvc.return_value.collect = AsyncMock(return_value=_make_report([]))
await gw.send_heartbeat()
assert "heartbeat:warnings_hash" in fake_redis.delete_calls
gw.send_to_group.assert_called_once()
gw.send_notification.assert_not_called()
@pytest.mark.asyncio
async def test_healthy_to_warnings_clears_suppressed_marker(
self,
gateway_with_fake_redis,
sre_group_configured,
):
"""從健康 → 有事:清掉 healthy suppressed marker,並推送新 warning"""
gw, fake_redis = gateway_with_fake_redis
fake_redis.preset("heartbeat:healthy_suppressed_last_seen")
with patch("src.core.redis_client.get_redis", return_value=fake_redis), \
patch("src.services.heartbeat_report_service.HeartbeatReportService") as MockSvc, \
patch("src.services.heartbeat_report_service.report_to_telegram_html",
return_value="warning"):
MockSvc.return_value.collect = AsyncMock(
return_value=_make_report(["Pod api-z Failed"])
)
await gw.send_heartbeat()
assert "heartbeat:healthy_suppressed_last_seen" in fake_redis.delete_calls
assert "heartbeat:warnings_hash" in fake_redis._store