ff8d917247
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
563 lines
18 KiB
Python
563 lines
18 KiB
Python
"""Delivery-truth regressions for gateway and webhook Telegram callers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from contextlib import asynccontextmanager
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock
|
|
|
|
import pytest
|
|
|
|
from src.api.v1 import sentry_webhook, signoz_webhook, telegram, webhooks
|
|
from src.services import telegram_gateway as gateway_module
|
|
from src.services.telegram_gateway import (
|
|
TelegramGateway,
|
|
_telegram_send_delivery_succeeded,
|
|
)
|
|
|
|
NO_SEND_RECEIPT = {
|
|
"ok": True,
|
|
"_awooop_delivery_status": "blocked_no_egress",
|
|
"_awooop_provider_send_performed": False,
|
|
"_awoooi_canonical_route_receipt": {
|
|
"decision": "blocked_no_egress",
|
|
"provider_send_performed": False,
|
|
},
|
|
}
|
|
|
|
_SENT_CHAT_ID = -1001
|
|
_SENT_DESTINATION_BINDING = gateway_module._telegram_destination_binding(
|
|
_SENT_CHAT_ID
|
|
)
|
|
SENT_RECEIPT = {
|
|
"ok": True,
|
|
"_awooop_delivery_status": "sent",
|
|
"_awooop_provider_send_performed": True,
|
|
"result": {"message_id": 42, "chat": {"id": _SENT_CHAT_ID}},
|
|
"_awoooi_canonical_route_receipt": {
|
|
"schema_version": "telegram_canonical_egress_receipt_v1",
|
|
"decision": "allowed",
|
|
"destination_alias": "awoooi_sre_war_room",
|
|
"destination_binding": _SENT_DESTINATION_BINDING,
|
|
"sender_bot_alias": "tsenyang_bot",
|
|
"provider_send_performed": True,
|
|
},
|
|
gateway_module._DELIVERY_CONTEXT_KEY: {
|
|
"schema_version": "telegram_delivery_context_v1",
|
|
"method": "sendMessage",
|
|
"sender_bot_alias": "tsenyang_bot",
|
|
"destination_alias": "awoooi_sre_war_room",
|
|
"destination_binding": _SENT_DESTINATION_BINDING,
|
|
"payload_destination_binding": _SENT_DESTINATION_BINDING,
|
|
"provider_destination_binding": _SENT_DESTINATION_BINDING,
|
|
"destination_binding_verified": True,
|
|
},
|
|
}
|
|
|
|
|
|
class FakeLogger:
|
|
def __init__(self) -> None:
|
|
self.events: list[tuple[str, str]] = []
|
|
|
|
def debug(self, event: str, **_kwargs) -> None:
|
|
self.events.append(("debug", event))
|
|
|
|
def info(self, event: str, **_kwargs) -> None:
|
|
self.events.append(("info", event))
|
|
|
|
def warning(self, event: str, **_kwargs) -> None:
|
|
self.events.append(("warning", event))
|
|
|
|
def error(self, event: str, **_kwargs) -> None:
|
|
self.events.append(("error", event))
|
|
|
|
def exception(self, event: str, **_kwargs) -> None:
|
|
self.events.append(("exception", event))
|
|
|
|
|
|
class FakeRedis:
|
|
def __init__(self, values: dict[str, str] | None = None) -> None:
|
|
self.values = dict(values or {})
|
|
self.set_calls: list[tuple[str, str, dict]] = []
|
|
|
|
async def get(self, key: str):
|
|
return self.values.get(key)
|
|
|
|
async def set(self, key: str, value: str, **kwargs):
|
|
self.set_calls.append((key, value, kwargs))
|
|
if kwargs.get("nx") and key in self.values:
|
|
return False
|
|
self.values[key] = value
|
|
return True
|
|
|
|
async def setex(self, key: str, _ttl: int, value: str):
|
|
self.values[key] = value
|
|
return True
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("receipt", "expected"),
|
|
[
|
|
(SENT_RECEIPT, True),
|
|
(NO_SEND_RECEIPT, False),
|
|
({"ok": True}, False),
|
|
({**SENT_RECEIPT, "ok": 1}, False),
|
|
({**SENT_RECEIPT, "_awooop_delivery_status": ""}, False),
|
|
({**SENT_RECEIPT, "_awooop_provider_send_performed": None}, False),
|
|
({**SENT_RECEIPT, "result": {}}, False),
|
|
({**SENT_RECEIPT, "result": {"message_id": 0}}, False),
|
|
({**SENT_RECEIPT, "result": {"message_id": True}}, False),
|
|
({**SENT_RECEIPT, "result": {"message_id": "42"}}, False),
|
|
({**SENT_RECEIPT, "_awoooi_canonical_route_receipt": {}}, False),
|
|
({**SENT_RECEIPT, gateway_module._DELIVERY_CONTEXT_KEY: {}}, False),
|
|
(
|
|
{
|
|
**SENT_RECEIPT,
|
|
"_awoooi_canonical_route_receipt": {
|
|
"schema_version": "telegram_canonical_egress_receipt_v1",
|
|
"decision": "blocked_no_egress",
|
|
"provider_send_performed": True,
|
|
},
|
|
},
|
|
False,
|
|
),
|
|
(
|
|
{
|
|
**SENT_RECEIPT,
|
|
"_awoooi_canonical_route_receipt": {
|
|
**SENT_RECEIPT["_awoooi_canonical_route_receipt"],
|
|
"sender_bot_alias": "",
|
|
},
|
|
},
|
|
False,
|
|
),
|
|
(
|
|
{
|
|
**SENT_RECEIPT,
|
|
gateway_module._DELIVERY_CONTEXT_KEY: {
|
|
**SENT_RECEIPT[gateway_module._DELIVERY_CONTEXT_KEY],
|
|
"sender_bot_alias": "openclaw_bot",
|
|
},
|
|
},
|
|
False,
|
|
),
|
|
(
|
|
{
|
|
**SENT_RECEIPT,
|
|
"_awoooi_canonical_route_receipt": {
|
|
**SENT_RECEIPT["_awoooi_canonical_route_receipt"],
|
|
"destination_alias": "other_destination",
|
|
},
|
|
},
|
|
False,
|
|
),
|
|
(
|
|
{
|
|
**SENT_RECEIPT,
|
|
"_awoooi_canonical_route_receipt": {
|
|
**SENT_RECEIPT["_awoooi_canonical_route_receipt"],
|
|
"destination_binding": "",
|
|
},
|
|
},
|
|
False,
|
|
),
|
|
(
|
|
{
|
|
**SENT_RECEIPT,
|
|
"result": {"message_id": 42, "chat": {"id": -2002}},
|
|
},
|
|
False,
|
|
),
|
|
],
|
|
)
|
|
def test_delivery_helper_fails_closed_without_complete_canonical_receipt(
|
|
receipt: object,
|
|
expected: bool,
|
|
) -> None:
|
|
assert _telegram_send_delivery_succeeded(receipt) is expected
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_mark_auto_repaired_returns_false_for_structured_no_send(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
redis = FakeRedis({"tg_approval:APR-1": "42"})
|
|
gateway = TelegramGateway()
|
|
|
|
async def send_request(method: str, _payload: dict):
|
|
if method == "sendMessage":
|
|
return dict(NO_SEND_RECEIPT)
|
|
return {"ok": True}
|
|
|
|
monkeypatch.setattr(gateway_module, "get_redis", lambda: redis)
|
|
monkeypatch.setattr(gateway, "_send_request", send_request)
|
|
|
|
assert await gateway.mark_auto_repaired("APR-1", "bounded", 10) is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("method_name", ["incident_update", "grouped_digest"])
|
|
async def test_lifecycle_no_send_does_not_write_dedup_receipt(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
method_name: str,
|
|
) -> None:
|
|
redis = FakeRedis({"tg_msg:INC-1": "42"})
|
|
gateway = TelegramGateway()
|
|
|
|
async def send_request(method: str, _payload: dict):
|
|
if method == "sendMessage":
|
|
return dict(NO_SEND_RECEIPT)
|
|
return {"ok": True}
|
|
|
|
monkeypatch.setattr(gateway_module, "get_redis", lambda: redis)
|
|
monkeypatch.setattr(gateway, "_send_request", send_request)
|
|
|
|
if method_name == "incident_update":
|
|
result = await gateway.append_incident_update("INC-1", "healthy")
|
|
else:
|
|
result = await gateway.append_grouped_alert_digest(
|
|
incident_id="INC-1",
|
|
group_key="group-1",
|
|
digest_text="digest",
|
|
)
|
|
|
|
assert result is False
|
|
assert redis.set_calls == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("method_name", ["incident_update", "grouped_digest"])
|
|
async def test_lifecycle_writes_dedup_only_after_affirmative_ack(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
method_name: str,
|
|
) -> None:
|
|
redis = FakeRedis({"tg_msg:INC-1": "42"})
|
|
gateway = TelegramGateway()
|
|
|
|
async def send_request(method: str, _payload: dict):
|
|
if method == "sendMessage":
|
|
return dict(SENT_RECEIPT)
|
|
return {"ok": True}
|
|
|
|
monkeypatch.setattr(gateway_module, "get_redis", lambda: redis)
|
|
monkeypatch.setattr(gateway, "_send_request", send_request)
|
|
|
|
if method_name == "incident_update":
|
|
result = await gateway.append_incident_update("INC-1", "healthy")
|
|
else:
|
|
result = await gateway.append_grouped_alert_digest(
|
|
incident_id="INC-1",
|
|
group_key="group-1",
|
|
digest_text="digest",
|
|
)
|
|
|
|
assert result is True
|
|
assert len(redis.set_calls) == 1
|
|
assert redis.set_calls[0][1] == "provider_ack"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_analyzing_placeholder_does_not_return_or_log_sent_on_no_send(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
logger = FakeLogger()
|
|
gateway = TelegramGateway()
|
|
monkeypatch.setattr(gateway_module.settings, "OPENCLAW_TG_BOT_TOKEN", "configured")
|
|
monkeypatch.setattr(gateway_module, "logger", logger)
|
|
monkeypatch.setattr(
|
|
gateway,
|
|
"_send_request",
|
|
AsyncMock(return_value=dict(NO_SEND_RECEIPT)),
|
|
)
|
|
|
|
result = await gateway.send_analyzing_placeholder("cpu", "api", "medium")
|
|
|
|
assert result is None
|
|
assert ("info", "analyzing_placeholder_no_send") in logger.events
|
|
assert ("info", "analyzing_placeholder_sent") not in logger.events
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_approval_card_records_failed_and_never_sent_for_no_send(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
logger = FakeLogger()
|
|
gateway = TelegramGateway()
|
|
executed_params: list[dict] = []
|
|
|
|
class FakeDB:
|
|
async def execute(self, _statement, params: dict) -> None:
|
|
executed_params.append(params)
|
|
|
|
@asynccontextmanager
|
|
async def fake_db_context(*_args, **_kwargs):
|
|
yield FakeDB()
|
|
|
|
monkeypatch.setattr(gateway_module, "logger", logger)
|
|
monkeypatch.setattr(
|
|
gateway_module,
|
|
"_fetch_remediation_summary_for_card",
|
|
AsyncMock(return_value=None),
|
|
)
|
|
monkeypatch.setattr(gateway, "_build_inline_keyboard", AsyncMock(return_value={}))
|
|
monkeypatch.setattr(
|
|
gateway,
|
|
"_send_request",
|
|
AsyncMock(return_value=dict(NO_SEND_RECEIPT)),
|
|
)
|
|
monkeypatch.setattr("src.db.base.get_db_context", fake_db_context)
|
|
|
|
result = await gateway.send_approval_card(
|
|
approval_id="APR-1",
|
|
risk_level="medium",
|
|
resource_name="api",
|
|
root_cause="cpu",
|
|
suggested_action="inspect",
|
|
notification_type="TYPE-3",
|
|
)
|
|
|
|
assert result == NO_SEND_RECEIPT
|
|
assert executed_params and executed_params[0]["ds"] == "failed"
|
|
assert ("warning", "telegram_approval_card_no_send") in logger.events
|
|
assert ("info", "telegram_approval_card_sent") not in logger.events
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_html_callback_no_send_exhausts_fallback_and_records_failure(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
gateway = TelegramGateway()
|
|
send_request = AsyncMock(return_value=dict(NO_SEND_RECEIPT))
|
|
record_failure = AsyncMock()
|
|
monkeypatch.setattr(gateway, "_send_request", send_request)
|
|
monkeypatch.setattr(gateway, "_record_callback_reply_failure", record_failure)
|
|
|
|
await gateway._send_html_line_message(
|
|
["<b>reply</b>"],
|
|
failure_context="test",
|
|
inbound_chat_id="room",
|
|
inbound_message_id=42,
|
|
incident_id="INC-1",
|
|
)
|
|
|
|
assert send_request.await_count == 3
|
|
record_failure.assert_awaited_once()
|
|
statuses = [
|
|
call.args[1]
|
|
.get(gateway_module._AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY, {})
|
|
.get("callback_reply", {})
|
|
.get("status")
|
|
for call in send_request.await_args_list
|
|
]
|
|
assert statuses == [
|
|
"callback_reply_attempt",
|
|
"callback_reply_fallback_attempt",
|
|
"callback_reply_rescue_attempt",
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("reply_kind", ["advisory", "category"])
|
|
async def test_interactive_reply_no_send_logs_failure_not_sent(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
reply_kind: str,
|
|
) -> None:
|
|
from src.services import ai_advisory_helpers, callback_dispatcher
|
|
|
|
logger = FakeLogger()
|
|
gateway = TelegramGateway()
|
|
monkeypatch.setattr(gateway_module, "logger", logger)
|
|
monkeypatch.setattr(gateway, "_answer_callback", AsyncMock())
|
|
monkeypatch.setattr(
|
|
gateway,
|
|
"_send_request",
|
|
AsyncMock(return_value=dict(NO_SEND_RECEIPT)),
|
|
)
|
|
|
|
if reply_kind == "advisory":
|
|
monkeypatch.setattr(
|
|
ai_advisory_helpers,
|
|
"handle_ai_advisory_callback",
|
|
AsyncMock(
|
|
return_value={
|
|
"feedback_text": "ack",
|
|
"success": True,
|
|
}
|
|
),
|
|
)
|
|
await gateway._handle_ai_advisory_action(
|
|
"ai_advisory_view",
|
|
"cost:ADV-1",
|
|
"callback-1",
|
|
1,
|
|
"operator",
|
|
{"id": 1},
|
|
inbound_chat_id="room",
|
|
inbound_message_id=42,
|
|
)
|
|
failed_event = "ai_advisory_group_reply_failed"
|
|
sent_event = "ai_advisory_group_reply_sent"
|
|
else:
|
|
|
|
class Repo:
|
|
async def get_by_id(self, _incident_id: str):
|
|
return None
|
|
|
|
monkeypatch.setattr(
|
|
callback_dispatcher,
|
|
"get_action_spec",
|
|
lambda _action: SimpleNamespace(emoji="info"),
|
|
)
|
|
monkeypatch.setattr(
|
|
callback_dispatcher,
|
|
"dispatch_action",
|
|
AsyncMock(
|
|
return_value=SimpleNamespace(
|
|
result_text="result",
|
|
success=True,
|
|
duration_ms=1.0,
|
|
)
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
"src.repositories.incident_repository.get_incident_repository",
|
|
lambda: Repo(),
|
|
)
|
|
await gateway._dispatch_category_action(
|
|
"callback-1",
|
|
"history",
|
|
"INC-1",
|
|
1,
|
|
inbound_chat_id="room",
|
|
inbound_message_id=42,
|
|
)
|
|
failed_event = "category_action_reply_failed"
|
|
sent_event = "category_action_reply_sent"
|
|
|
|
assert ("warning", failed_event) in logger.events
|
|
assert ("info", sent_event) not in logger.events
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
("notification_type", "alert_category"),
|
|
[("generic", ""), ("TYPE-4D", ""), ("TYPE-8M", "alertchain_health")],
|
|
)
|
|
async def test_background_no_send_never_confirms_or_writes_sent(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
notification_type: str,
|
|
alert_category: str,
|
|
) -> None:
|
|
calls: list[dict] = []
|
|
|
|
class Gateway:
|
|
async def send_approval_card(self, **kwargs):
|
|
calls.append(kwargs)
|
|
return dict(NO_SEND_RECEIPT)
|
|
|
|
async def send_drift_card(self, **kwargs):
|
|
calls.append(kwargs)
|
|
return dict(NO_SEND_RECEIPT)
|
|
|
|
async def send_meta_alert(self, **kwargs):
|
|
calls.append(kwargs)
|
|
return dict(NO_SEND_RECEIPT)
|
|
|
|
async def delete_message(self, *_args, **_kwargs):
|
|
raise AssertionError("placeholder must not be deleted before delivery")
|
|
|
|
class ApprovalService:
|
|
async def mark_telegram_confirmed(self, *_args, **_kwargs):
|
|
raise AssertionError("no-send must not be marked confirmed")
|
|
|
|
monkeypatch.setattr(webhooks.settings, "OPENCLAW_TG_BOT_TOKEN", "configured")
|
|
monkeypatch.setattr(webhooks, "get_telegram_gateway", lambda: Gateway())
|
|
monkeypatch.setattr(webhooks, "get_approval_service", lambda: ApprovalService())
|
|
|
|
await webhooks._push_to_telegram_background(
|
|
approval_id="APR-1",
|
|
risk_level="medium",
|
|
resource_name="api",
|
|
root_cause="cpu",
|
|
suggested_action="inspect",
|
|
estimated_downtime="0s",
|
|
notification_type=notification_type,
|
|
alert_category=alert_category,
|
|
fingerprint="fp",
|
|
placeholder_message_id=42,
|
|
)
|
|
|
|
assert len(calls) == 1
|
|
if notification_type == "generic":
|
|
assert calls[0]["notification_type"] == "TYPE-3"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("module_name", ["sentry", "signoz"])
|
|
async def test_unclassified_monitoring_webhooks_fail_closed_and_do_not_false_log_sent(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
module_name: str,
|
|
) -> None:
|
|
logger = FakeLogger()
|
|
captured: list[dict] = []
|
|
|
|
class Gateway:
|
|
async def initialize(self) -> bool:
|
|
return True
|
|
|
|
async def send_approval_card(self, **kwargs):
|
|
captured.append(kwargs)
|
|
return dict(NO_SEND_RECEIPT)
|
|
|
|
if module_name == "sentry":
|
|
monkeypatch.setattr(sentry_webhook, "logger", logger)
|
|
monkeypatch.setattr(sentry_webhook, "get_telegram_gateway", lambda: Gateway())
|
|
await sentry_webhook.send_sentry_telegram_alert(
|
|
{"title": "error", "culprit": "api.py", "level": "error"},
|
|
None,
|
|
"APR-1",
|
|
)
|
|
sent_event = "sentry_telegram_sent"
|
|
no_send_event = "sentry_telegram_no_send"
|
|
else:
|
|
monkeypatch.setattr(signoz_webhook, "logger", logger)
|
|
monkeypatch.setattr(signoz_webhook, "get_telegram_gateway", lambda: Gateway())
|
|
await signoz_webhook.send_signoz_telegram(
|
|
"APR-1",
|
|
"INC-1",
|
|
"HighCPU",
|
|
{"service": "api"},
|
|
{"summary": "cpu"},
|
|
"warning",
|
|
)
|
|
sent_event = "signoz_telegram_sent"
|
|
no_send_event = "signoz_telegram_no_send"
|
|
|
|
assert captured[0]["notification_type"] == "UNKNOWN"
|
|
assert captured[0]["product_id"] is None
|
|
assert captured[0]["signal_family"] is None
|
|
assert captured[0]["route_severity"] is None
|
|
assert ("warning", no_send_event) in logger.events
|
|
assert ("info", sent_event) not in logger.events
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_dev_test_push_returns_false_for_structured_no_send(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
captured: list[dict] = []
|
|
|
|
class Gateway:
|
|
async def send_approval_card(self, **kwargs):
|
|
captured.append(kwargs)
|
|
return dict(NO_SEND_RECEIPT)
|
|
|
|
monkeypatch.setattr(telegram.settings, "ENVIRONMENT", "dev")
|
|
monkeypatch.setattr(telegram, "get_telegram_gateway", lambda: Gateway())
|
|
|
|
response = await telegram.test_push(telegram.TestPushRequest(approval_id="APR-1"))
|
|
|
|
assert response["ok"] is False
|
|
assert response["message"] == "Test push no-send"
|
|
assert captured[0]["notification_type"] == "TYPE-3"
|