From 180e708444e4c9e3d30b8a52746d3dedf00eccd8 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 19:14:11 +0800 Subject: [PATCH] feat(telegram): enforce canonical product routing --- agent99-control-plane.ps1 | 58 +- apps/api/src/api/v1/gitea_webhook.py | 14 +- apps/api/src/api/v1/sentry_webhook.py | 25 +- apps/api/src/api/v1/signoz_webhook.py | 56 +- apps/api/src/api/v1/telegram.py | 15 +- apps/api/src/api/v1/webhooks.py | 75 +- apps/api/src/jobs/ai_slo_watchdog_job.py | 38 +- apps/api/src/jobs/capacity_forecaster_job.py | 30 +- apps/api/src/jobs/compliance_scanner_job.py | 20 +- apps/api/src/jobs/coverage_evaluator_job.py | 21 +- apps/api/src/jobs/hermes_rule_quality_job.py | 32 +- ...agent_report_truth_actionability_review.py | 16 +- apps/api/src/services/ai_rate_limiter.py | 127 +- apps/api/src/services/ai_router.py | 8 +- apps/api/src/services/approval_execution.py | 111 +- apps/api/src/services/channel_hub.py | 52 +- .../converged_alert_recurrence_notifier.py | 39 +- apps/api/src/services/decision_manager.py | 234 +- apps/api/src/services/drift_adopt_service.py | 5 +- apps/api/src/services/drift_remediator.py | 7 +- .../services/emergency_escalation_service.py | 3 - apps/api/src/services/failover_alerter.py | 171 +- apps/api/src/services/failure_watcher.py | 35 +- .../api/src/services/gitea_webhook_service.py | 15 +- .../src/services/image_analysis_service.py | 7 +- apps/api/src/services/k3s_monitor_service.py | 19 +- apps/api/src/services/notification_matrix.py | 800 ++++++- .../src/services/notifications/telegram.py | 55 +- .../src/services/post_execution_verifier.py | 38 +- .../src/services/report_generation_service.py | 150 +- apps/api/src/services/runbook_generator.py | 12 +- apps/api/src/services/telegram_gateway.py | 1940 ++++++++++++++--- .../api/src/services/weekly_report_service.py | 25 +- .../tests/test_alert_converged_recurrence.py | 9 +- .../tests/test_ansible_verified_closure.py | 16 + .../test_channel_hub_grouped_alert_events.py | 83 +- apps/api/tests/test_failover_alerter.py | 17 +- .../test_notification_matrix_group_cutover.py | 51 +- .../tests/test_report_generation_service.py | 13 +- .../tests/test_telegram_button_consistency.py | 5 +- .../test_telegram_callback_context_routing.py | 721 ++++++ ...est_telegram_canonical_routing_registry.py | 494 +++++ .../test_telegram_canonical_sender_gate.py | 521 +++++ .../test_telegram_delivery_truth_callers.py | 399 ++++ ...test_telegram_delivery_truth_callers_v2.py | 394 ++++ ...test_telegram_delivery_truth_gateway_v2.py | 460 ++++ .../tests/test_telegram_message_templates.py | 87 +- docs/adr/ADR-093-telegram-group-migration.md | 6 +- ..._canonical_routing_registry_v1.schema.json | 309 +++ .../TELEGRAM-CANONICAL-ROUTING-INVENTORY.md | 60 + ...m-canonical-routing-registry.snapshot.json | 522 +++++ .../reboot-recovery/awoooi-telegram-notify.sh | 19 +- .../telegram-notification-egress-inventory.py | 39 +- ...notification-egress-no-new-bypass-guard.py | 8 +- 54 files changed, 7746 insertions(+), 740 deletions(-) create mode 100644 apps/api/tests/test_telegram_callback_context_routing.py create mode 100644 apps/api/tests/test_telegram_canonical_routing_registry.py create mode 100644 apps/api/tests/test_telegram_canonical_sender_gate.py create mode 100644 apps/api/tests/test_telegram_delivery_truth_callers.py create mode 100644 apps/api/tests/test_telegram_delivery_truth_callers_v2.py create mode 100644 apps/api/tests/test_telegram_delivery_truth_gateway_v2.py create mode 100644 docs/schemas/telegram_canonical_routing_registry_v1.schema.json create mode 100644 docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md create mode 100644 docs/security/telegram-canonical-routing-registry.snapshot.json diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index be8f0c8bc..bfd210e6e 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -547,6 +547,18 @@ function Send-AgentTelegramRelay { return [pscustomobject]@{ ok = $false; error = "relay_disabled" } } + # Canonical Telegram routing is owned by the AWOOOI API gateway. Agent99 + # must never borrow a container token/chat binding or call Bot API directly. + # Until a correlated gateway transport is configured this path is an + # explicit no-egress terminal, not a best-effort fallback. + return [pscustomobject]@{ + ok = $false + sent = $false + providerSendPerformed = $false + routeStatus = "blocked_no_egress" + error = "canonical_telegram_gateway_transport_required" + } + $relayHost = if ($Relay.host) { [string]$Relay.host } else { "192.168.0.110" } $container = if ($Relay.container) { [string]$Relay.container } else { "stockplatform-v2-api-1" } if ($container -notmatch "^[A-Za-z0-9_.-]+$") { @@ -618,7 +630,7 @@ if photo_path and os.path.exists(photo_path): request_data["reply_parameters"] = reply_parameters with open(photo_path, "rb") as fh: response = requests.post( - "https://api.telegram.org/bot" + token + "/sendPhoto", + "blocked://canonical-telegram-gateway-required/sendPhoto", data=request_data, files={"photo": (filename, fh, ctype)}, timeout=30, @@ -646,7 +658,7 @@ if photo_path and os.path.exists(photo_path): body.extend(fh.read()) body.extend(b"\r\n") body.extend(("--" + boundary + "--\r\n").encode()) - req = urllib.request.Request("https://api.telegram.org/bot" + token + "/sendPhoto", data=bytes(body)) + req = urllib.request.Request("blocked://canonical-telegram-gateway-required/sendPhoto", data=bytes(body)) req.add_header("Content-Type", "multipart/form-data; boundary=" + boundary) try: response_payload = json.loads(urllib.request.urlopen(req, timeout=30).read().decode()) @@ -660,7 +672,7 @@ else: if reply_parameters: fields["reply_parameters"] = reply_parameters data = urllib.parse.urlencode(fields).encode() - response_payload = json.loads(urllib.request.urlopen("https://api.telegram.org/bot" + token + "/sendMessage", data=data, timeout=15).read().decode()) + response_payload = json.loads(urllib.request.urlopen("blocked://canonical-telegram-gateway-required/sendMessage", data=data, timeout=15).read().decode()) emit_receipt(response_payload) print("sent_ok") ' @@ -1641,6 +1653,14 @@ function Send-AgentTelegramPhotoDirect { [string]$ReplyToMessageId = "" ) + return [pscustomobject]@{ + ok = $false + sent = $false + providerSendPerformed = $false + routeStatus = "blocked_no_egress" + error = "canonical_telegram_gateway_transport_required" + } + Add-Type -AssemblyName System.Net.Http -ErrorAction Stop $client = New-Object System.Net.Http.HttpClient $content = New-Object System.Net.Http.MultipartFormDataContent @@ -1660,7 +1680,7 @@ function Send-AgentTelegramPhotoDirect { $fileContent = [System.Net.Http.ByteArrayContent]::new($fileBytes) $fileContent.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("image/png") $content.Add($fileContent, "photo", [IO.Path]::GetFileName($PhotoPath)) - $response = $client.PostAsync("https://api.telegram.org/bot$Token/sendPhoto", $content).GetAwaiter().GetResult() + $response = $client.PostAsync("blocked://canonical-telegram-gateway-required/sendPhoto", $content).GetAwaiter().GetResult() $body = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult() if (-not $response.IsSuccessStatusCode) { throw "telegram_sendPhoto_failed status=$([int]$response.StatusCode) body=$body" @@ -1940,10 +1960,13 @@ function Send-AgentTelegram { return $null } - $tokenSecret = Get-AgentEnvSecret $tokenEnvAliases - $chatSecret = Get-AgentEnvSecret $chatIdEnvAliases - $token = if ($tokenSecret) { $tokenSecret.value } else { $null } - $chatId = if ($chatSecret) { $chatSecret.value } else { $null } + # The canonical gateway transport is not wired into Agent99 yet. Keep all + # credential bindings unset so the fail-closed receipt below cannot read a + # host-local bot token/chat identifier before returning. + $tokenSecret = $null + $chatSecret = $null + $token = $null + $chatId = $null $chatIdOverride = if ($Data -and $Data.PSObject.Properties["replyChatId"]) { [string]$Data.replyChatId } else { "" } $card = Get-AgentIncidentCardModel $Severity $EventType $Message $Data $incidentState = Get-AgentTelegramIncidentState $card @@ -1982,6 +2005,21 @@ function Send-AgentTelegram { $script:TelegramAttempts += $attempt return $attempt } + + # Fail closed before reading a sender binding, copying an attachment, or + # attempting any provider call. Agent99 events must be routed by the + # canonical API registry/gateway with a durable receipt. + $attempt.error = "canonical_telegram_gateway_transport_required" + $attempt.sent = $false + $attempt.relay = [pscustomobject]@{ + ok = $false + providerSendPerformed = $false + routeStatus = "blocked_no_egress" + error = "canonical_telegram_gateway_transport_required" + } + $script:TelegramAttempts += $attempt + return $attempt + $text = Format-AgentTelegramText $Severity $EventType $Message $Data $caption = Format-AgentTelegramCaption $Severity $EventType $Message $Data $visualPath = Get-AgentTelegramVisualPath $Data @@ -2029,7 +2067,7 @@ function Send-AgentTelegram { disable_web_page_preview = "true" } if ($replyToMessageId) { $fallbackBody.reply_parameters = (@{ message_id = [long]$replyToMessageId } | ConvertTo-Json -Compress) } - $fallbackResult = Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body $fallbackBody + $fallbackResult = Invoke-RestMethod -Method Post -Uri "blocked://canonical-telegram-gateway-required/sendMessage" -Body $fallbackBody $attempt.sent = $true $attempt.messageId = if ($fallbackResult.result) { $fallbackResult.result.message_id } else { $null } } @@ -2040,7 +2078,7 @@ function Send-AgentTelegram { disable_web_page_preview = "true" } if ($replyToMessageId) { $textBody.reply_parameters = (@{ message_id = [long]$replyToMessageId } | ConvertTo-Json -Compress) } - $textResult = Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body $textBody + $textResult = Invoke-RestMethod -Method Post -Uri "blocked://canonical-telegram-gateway-required/sendMessage" -Body $textBody $attempt.sent = $true $attempt.messageId = if ($textResult.result) { $textResult.result.message_id } else { $null } } diff --git a/apps/api/src/api/v1/gitea_webhook.py b/apps/api/src/api/v1/gitea_webhook.py index d19c23ff5..2ff390543 100644 --- a/apps/api/src/api/v1/gitea_webhook.py +++ b/apps/api/src/api/v1/gitea_webhook.py @@ -422,10 +422,18 @@ async def _send_gitea_notification( get_telegram_gateway, # type: ignore[import] ) gateway = get_telegram_gateway() - await gateway.initialize() - await gateway.send_alert_notification(message) + result = await gateway.send_alert_notification( + message, + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ) - logger.info("gitea_tg_notification_sent", dedup_key=dedup_key) + logger.info( + "gitea_tg_notification_routed", + dedup_key=dedup_key, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) except Exception as e: logger.warning("gitea_tg_notification_failed", dedup_key=dedup_key, error=str(e)) diff --git a/apps/api/src/api/v1/sentry_webhook.py b/apps/api/src/api/v1/sentry_webhook.py index d1a462e6d..25494fd6e 100644 --- a/apps/api/src/api/v1/sentry_webhook.py +++ b/apps/api/src/api/v1/sentry_webhook.py @@ -47,7 +47,10 @@ from src.services.sentry_webhook_service import ( SentrySignatureError, verify_sentry_signature, ) -from src.services.telegram_gateway import get_telegram_gateway +from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) from src.utils.timezone import now_taipei_iso logger = structlog.get_logger(__name__) @@ -648,7 +651,7 @@ async def send_sentry_telegram_alert( # 發送 Sentry 告警卡片 (含 Y/n 按鈕) # TODO(2026-04-05): Sentry 路徑無 incident_id，待 Sentry→Incident 關聯後補傳 - await telegram.send_approval_card( + delivery_result = await telegram.send_approval_card( approval_id=approval_id, risk_level="high" if level in ["fatal", "error"] else "medium", resource_name=culprit, @@ -660,13 +663,21 @@ async def send_sentry_telegram_alert( anomaly_frequency=anomaly_frequency, # 2026-04-02 ogt: 修復 ai_provider 未傳遞 → Telegram 顯示「AI 仲裁判定」而非具體模型名稱 ai_provider=analysis.analyzed_by if analysis else "", + notification_type="TYPE-3", ) - logger.info( - "sentry_telegram_sent", - approval_id=approval_id, - escalation_level=anomaly_frequency.get("escalation_level") if anomaly_frequency else None, - ) + if _telegram_send_delivery_succeeded(delivery_result): + logger.info( + "sentry_telegram_sent", + approval_id=approval_id, + escalation_level=anomaly_frequency.get("escalation_level") if anomaly_frequency else None, + ) + else: + logger.warning( + "sentry_telegram_no_send", + approval_id=approval_id, + delivery_status=delivery_result.get("_awooop_delivery_status"), + ) except Exception as e: logger.exception("sentry_telegram_failed", error=str(e)) diff --git a/apps/api/src/api/v1/signoz_webhook.py b/apps/api/src/api/v1/signoz_webhook.py index a8e5ae331..cb8394444 100644 --- a/apps/api/src/api/v1/signoz_webhook.py +++ b/apps/api/src/api/v1/signoz_webhook.py @@ -40,7 +40,10 @@ from src.services.anomaly_counter import get_anomaly_counter from src.services.approval_db import get_approval_service from src.services.channel_hub import record_external_alert_event from src.services.incident_service import get_incident_service -from src.services.telegram_gateway import get_telegram_gateway +from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) from src.utils.timezone import now_taipei_iso if TYPE_CHECKING: @@ -550,7 +553,7 @@ async def send_signoz_telegram( summary = annotations.get("summary", f"SignOz Alert: {alert_name}") description = annotations.get("description", "") - await telegram.send_approval_card( + delivery_result = await telegram.send_approval_card( approval_id=approval_id, risk_level=analysis_result.risk_level if analysis_result else ( "critical" if severity == "critical" else ( @@ -569,14 +572,23 @@ async def send_signoz_telegram( # 2026-04-02 ogt: 修復 ai_provider 未傳遞 → Telegram 顯示「AI 仲裁判定」而非具體模型名稱 ai_provider=ai_provider if ai_provider != "none" else "", incident_id=incident_id, + notification_type="TYPE-3", ) - logger.info( - "signoz_telegram_sent", - approval_id=approval_id, - alert_name=alert_name, - escalation_level=anomaly_frequency.get("escalation_level") if anomaly_frequency else None, - ) + if _telegram_send_delivery_succeeded(delivery_result): + logger.info( + "signoz_telegram_sent", + approval_id=approval_id, + alert_name=alert_name, + escalation_level=anomaly_frequency.get("escalation_level") if anomaly_frequency else None, + ) + else: + logger.warning( + "signoz_telegram_no_send", + approval_id=approval_id, + alert_name=alert_name, + delivery_status=delivery_result.get("_awooop_delivery_status"), + ) except Exception as e: logger.exception("signoz_telegram_error", error=str(e)) @@ -602,13 +614,25 @@ async def _send_log_summary_notification( from src.services.telegram_gateway import get_telegram_gateway try: + tg = get_telegram_gateway() + if not tg.canonical_destination_chat_id( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P2", + ): + logger.info( + "log_summary_notification_no_route", + pod=pod_name, + namespace=namespace, + ) + return + svc = get_log_summary_service() summary = await svc.summarize_with_soft_timeout(pod_name, namespace) if not summary: return - tg = get_telegram_gateway() msg = ( f"🔍 Log 異常摘要\n" f"Pod: {_html.escape(pod_name)}\n" @@ -616,7 +640,19 @@ async def _send_log_summary_notification( f"{_html.escape(summary)}\n\n" f"deepseek-r1:14b | 免費本地推理" ) - await tg.send_text(msg[:4096]) + delivery_result = await tg.send_text( + msg[:4096], + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P2", + ) + if not _telegram_send_delivery_succeeded(delivery_result): + logger.warning( + "log_summary_notification_no_send", + pod=pod_name, + namespace=namespace, + delivery_status=delivery_result.get("_awooop_delivery_status"), + ) except Exception as e: logger.warning("log_summary_notification_failed", pod=pod_name, error=str(e)) diff --git a/apps/api/src/api/v1/telegram.py b/apps/api/src/api/v1/telegram.py index 70dc95b55..f83ed62f6 100644 --- a/apps/api/src/api/v1/telegram.py +++ b/apps/api/src/api/v1/telegram.py @@ -35,7 +35,11 @@ from src.services.security_interceptor import ( NonceReplayError, UserNotWhitelistedError, ) -from src.services.telegram_gateway import TelegramGatewayError, get_telegram_gateway +from src.services.telegram_gateway import ( + TelegramGatewayError, + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) logger = get_logger("awoooi.telegram") router = APIRouter(prefix="/telegram", tags=["Telegram"]) @@ -306,6 +310,7 @@ async def telegram_webhook( await svc.download_and_analyze( chat_id=chat_id, file_id=file_id, + original_message_id=msg.get("message_id"), question=caption, ) except Exception as _img_err: @@ -353,6 +358,7 @@ async def telegram_webhook( message_id=message_id, original_text=original_text, username=username, + chat_id=message.get("chat", {}).get("id"), ) if not result.get("success"): @@ -534,11 +540,14 @@ async def test_push( suggested_action=request.suggested_action, estimated_downtime=request.estimated_downtime, incident_id=request.incident_id, + notification_type="TYPE-3", ) + delivery_succeeded = _telegram_send_delivery_succeeded(result) + return { - "ok": True, - "message": "Test push sent", + "ok": delivery_succeeded, + "message": "Test push sent" if delivery_succeeded else "Test push no-send", "telegram_response": result, } diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index 8d266141e..0c8bb69e8 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -98,7 +98,11 @@ from src.services.security_interceptor import check_webhook_nonce # P0-06: nonc from src.services.signal_producer import SignalData, get_signal_producer # Phase 5: Telegram Gateway (行動戰情室) -from src.services.telegram_gateway import TelegramGatewayError, get_telegram_gateway +from src.services.telegram_gateway import ( + TelegramGatewayError, + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) # Phase 18.1.7: K8s 資源名稱正規化 已移至 alert_analyzer_service (R4 #129) from src.utils.timezone import now_taipei @@ -904,21 +908,32 @@ async def _legacy_try_auto_repair_background_disabled( if result: try: _pb_name = decision.playbook.name if decision.playbook else "unknown" - await get_telegram_gateway().mark_auto_repaired( + telegram_delivered = await get_telegram_gateway().mark_auto_repaired( approval_id=approval_id, playbook_name=_pb_name, execution_time_ms=result.execution_time_ms, success=result.success, ) - await op_log.append( - "TELEGRAM_RESULT_SENT", - incident_id=incident_id, - approval_id=approval_id, - actor="system", - action_detail="auto_repair_card_updated", - success=result.success, - context={"target_resource": target_resource, "namespace": namespace}, - ) + if telegram_delivered: + await op_log.append( + "TELEGRAM_RESULT_SENT", + incident_id=incident_id, + approval_id=approval_id, + actor="system", + action_detail="auto_repair_card_updated", + success=result.success, + context={"target_resource": target_resource, "namespace": namespace}, + ) + else: + await op_log.append( + "NOTIFICATION_CLASSIFIED", + incident_id=incident_id, + approval_id=approval_id, + actor="system", + action_detail="auto_repair_card_no_send", + success=False, + context={"target_resource": target_resource, "namespace": namespace}, + ) except Exception as tg_err: logger.warning("auto_repair_telegram_notify_failed", error=str(tg_err)) @@ -989,6 +1004,11 @@ async def _push_to_telegram_background( """ try: gateway = get_telegram_gateway() + effective_notification_type = ( + "TYPE-3" + if notification_type.strip().lower() in {"", "generic"} + else notification_type + ) # 檢查是否有設定 Bot Token if not settings.OPENCLAW_TG_BOT_TOKEN: @@ -1002,13 +1022,21 @@ async def _push_to_telegram_background( # TYPE-4D: Config Drift 使用專屬卡片 (send_drift_card) # ADR-071-F: [查看Diff][採納變更][回滾][忽略] 四鍵格式 if notification_type == "TYPE-4D": - await gateway.send_drift_card( + delivery_result = await gateway.send_drift_card( incident_id=incident_id, approval_id=approval_id, resource_name=resource_name[:50], diff_summary=diff_summary or root_cause, detected_at="", ) + if not _telegram_send_delivery_succeeded(delivery_result): + logger.warning( + "telegram_push_no_send_type4d", + approval_id=approval_id, + incident_id=incident_id, + delivery_status=delivery_result.get("_awooop_delivery_status"), + ) + return logger.info( "telegram_push_success_type4d", approval_id=approval_id, @@ -1021,7 +1049,7 @@ async def _push_to_telegram_background( # 2026-04-12 ogt: ADR-075 斷點 E 修復 — TYPE-8M Meta-System 使用專屬卡片 # alertchain_health / flywheel_health → ⚙️ META SYSTEM 卡片，不發群組 if notification_type == "TYPE-8M" or alert_category in ("alertchain_health", "flywheel_health"): - await gateway.send_meta_alert( + delivery_result = await gateway.send_meta_alert( incident_id=incident_id, approval_id=approval_id, alertname=resource_name, @@ -1030,6 +1058,14 @@ async def _push_to_telegram_background( severity_level=risk_level, system_impact=root_cause[:150], ) + if not _telegram_send_delivery_succeeded(delivery_result): + logger.warning( + "telegram_push_no_send_type8m", + approval_id=approval_id, + incident_id=incident_id, + delivery_status=delivery_result.get("_awooop_delivery_status"), + ) + return logger.info( "telegram_push_success_type8m", approval_id=approval_id, @@ -1045,7 +1081,7 @@ async def _push_to_telegram_background( if hit_count > 1: root_cause_with_count = f"[x{hit_count}] {root_cause}" - await gateway.send_approval_card( + delivery_result = await gateway.send_approval_card( approval_id=approval_id, risk_level=risk_level, resource_name=resource_name[:50], @@ -1073,7 +1109,7 @@ async def _push_to_telegram_background( incident_id=incident_id, # ADR-075 斷點 B 修復: 傳入分類以啟用動態按鈕 alert_category=alert_category, - notification_type=notification_type, + notification_type=effective_notification_type, repair_candidate_blocker_summary=repair_candidate_blocker_summary, repair_candidate_next_step=repair_candidate_next_step, repair_candidate_required_fields=repair_candidate_required_fields, @@ -1082,6 +1118,15 @@ async def _push_to_telegram_background( repair_candidate_work_item_id=repair_candidate_work_item_id, ) + if not _telegram_send_delivery_succeeded(delivery_result): + logger.warning( + "telegram_push_no_send", + approval_id=approval_id, + notification_type=effective_notification_type, + delivery_status=delivery_result.get("_awooop_delivery_status"), + ) + return + logger.info( "telegram_push_success", approval_id=approval_id, diff --git a/apps/api/src/jobs/ai_slo_watchdog_job.py b/apps/api/src/jobs/ai_slo_watchdog_job.py index 5748b6b05..e4181a65a 100644 --- a/apps/api/src/jobs/ai_slo_watchdog_job.py +++ b/apps/api/src/jobs/ai_slo_watchdog_job.py @@ -256,7 +256,7 @@ async def _check_once() -> None: dedup_key = f"watchdog:alert:{dedup_hash}" redis = get_redis() # setnx atomic — 同時多個 pod 只有第一個能 set，避免並發多發 - set_ok = await redis.set(dedup_key, "1", ex=_DEDUP_TTL_SEC, nx=True) + set_ok = await redis.set(dedup_key, "pending", ex=_DEDUP_TTL_SEC, nx=True) if not set_ok: logger.debug("ai_slo_watchdog_deduped", key=dedup_key) return @@ -281,9 +281,12 @@ async def _check_once() -> None: severity = "critical" if len(violations) >= 2 else "warning" incident_id = f"META-{now_taipei().strftime('%Y%m%d%H%M%S')}" try: - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) - await get_telegram_gateway().send_meta_alert( + delivery = await get_telegram_gateway().send_meta_alert( incident_id=incident_id, approval_id=str(uuid.uuid4()), alertname="AI 自健診異常", @@ -293,6 +296,27 @@ async def _check_once() -> None: system_impact=system_impact, probable_cause=probable_cause, ) + if not _telegram_send_delivery_succeeded(delivery): + await redis.delete(dedup_key) + logger.error( + "ai_slo_watchdog_alert_no_send", + incident_id=incident_id, + violation_count=len(violations), + delivery_status=( + delivery.get("_awooop_delivery_status", "missing_provider_ack") + if isinstance(delivery, dict) + else "missing_provider_ack" + ), + ) + return + try: + await redis.set(dedup_key, "sent", ex=_DEDUP_TTL_SEC) + except Exception as finalize_error: + logger.warning( + "ai_slo_watchdog_dedup_finalize_failed", + key=dedup_key, + error=str(finalize_error), + ) logger.warning( "ai_slo_watchdog_alert_sent", incident_id=incident_id, @@ -300,6 +324,14 @@ async def _check_once() -> None: violations=violations, ) except Exception as e: + try: + await redis.delete(dedup_key) + except Exception as release_error: + logger.warning( + "ai_slo_watchdog_dedup_release_failed", + key=dedup_key, + error=str(release_error), + ) logger.error("ai_slo_watchdog_telegram_failed", error=str(e), violations=violations) diff --git a/apps/api/src/jobs/capacity_forecaster_job.py b/apps/api/src/jobs/capacity_forecaster_job.py index 67657921a..048d23c54 100644 --- a/apps/api/src/jobs/capacity_forecaster_job.py +++ b/apps/api/src/jobs/capacity_forecaster_job.py @@ -323,12 +323,15 @@ async def _send_telegram_forecast( """ try: import html - from src.services.ai_advisory_helpers import build_ai_advisory_keyboard, is_snoozed - from src.services.telegram_gateway import get_telegram_gateway - target_chat_id = settings.SRE_GROUP_CHAT_ID - if not target_chat_id: - return False + from src.services.ai_advisory_helpers import ( + build_ai_advisory_keyboard, + is_snoozed, + ) + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) # Snooze check: 過濾掉被人工 snooze 的 host (按「忽略 24h」後) active_risks = {} @@ -382,14 +385,15 @@ async def _send_telegram_forecast( msg = "\n".join(lines) tg = get_telegram_gateway() - await tg._send_request("sendMessage", { # type: ignore[attr-defined] - "chat_id": target_chat_id, - "text": msg, - "parse_mode": "HTML", - "disable_web_page_preview": True, - "reply_markup": keyboard, - }) - return True + result = await tg.send_canonical_message( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P2", + text=msg, + parse_mode="HTML", + reply_markup=keyboard, + ) + return _telegram_send_delivery_succeeded(result) except Exception as e: logger.warning("capacity_forecast_telegram_failed", error=str(e)) return False diff --git a/apps/api/src/jobs/compliance_scanner_job.py b/apps/api/src/jobs/compliance_scanner_job.py index 5023254bf..30b15998b 100644 --- a/apps/api/src/jobs/compliance_scanner_job.py +++ b/apps/api/src/jobs/compliance_scanner_job.py @@ -474,17 +474,12 @@ async def _send_telegram_posture( try: import html - from src.core.config import settings from src.services.ai_advisory_helpers import ( build_ai_advisory_keyboard, is_snoozed, ) from src.services.telegram_gateway import get_telegram_gateway - target_chat_id = settings.SRE_GROUP_CHAT_ID - if not target_chat_id: - return - # Snooze check (advisory_id 用當日 date 即可,一天只能 snooze 一次) from src.utils.timezone import now_taipei today = now_taipei().date().isoformat() @@ -532,13 +527,14 @@ async def _send_telegram_posture( include_produce_cmd=False, ) tg = get_telegram_gateway() - await tg._send_request("sendMessage", { # type: ignore[attr-defined] - "chat_id": target_chat_id, - "text": msg, - "parse_mode": "HTML", - "disable_web_page_preview": True, - "reply_markup": keyboard, - }) + await tg.send_canonical_message( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P2", + text=msg, + parse_mode="HTML", + reply_markup=keyboard, + ) except Exception as e: logger.warning("compliance_telegram_failed", error=str(e)) diff --git a/apps/api/src/jobs/coverage_evaluator_job.py b/apps/api/src/jobs/coverage_evaluator_job.py index 74029a06b..267ee79bb 100644 --- a/apps/api/src/jobs/coverage_evaluator_job.py +++ b/apps/api/src/jobs/coverage_evaluator_job.py @@ -298,18 +298,12 @@ async def _send_telegram_gaps( """推 coverage 缺口 Telegram 摘要 + 互動按鈕 (P0 修).""" try: import html - - from src.core.config import settings from src.services.ai_advisory_helpers import ( build_ai_advisory_keyboard, is_snoozed, ) from src.services.telegram_gateway import get_telegram_gateway - target_chat_id = settings.SRE_GROUP_CHAT_ID - if not target_chat_id: - return - # Snooze check: 以 worst_dimension 為 key worst_dim = str(analysis.get("worst_dimension", "unknown")) if await is_snoozed("coverage_gap", worst_dim): @@ -348,13 +342,14 @@ async def _send_telegram_gaps( include_produce_cmd=False, ) tg = get_telegram_gateway() - await tg._send_request("sendMessage", { # type: ignore[attr-defined] - "chat_id": target_chat_id, - "text": msg, - "parse_mode": "HTML", - "disable_web_page_preview": True, - "reply_markup": keyboard, - }) + await tg.send_canonical_message( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P2", + text=msg, + parse_mode="HTML", + reply_markup=keyboard, + ) except Exception as e: logger.warning("coverage_telegram_failed", error=str(e)) diff --git a/apps/api/src/jobs/hermes_rule_quality_job.py b/apps/api/src/jobs/hermes_rule_quality_job.py index cc20fa75a..769289104 100644 --- a/apps/api/src/jobs/hermes_rule_quality_job.py +++ b/apps/api/src/jobs/hermes_rule_quality_job.py @@ -312,14 +312,15 @@ async def _send_telegram_summary( """推 Telegram 摘要訊息給 SRE group,含 LLM 分析結果 + 互動按鈕 (P0 修).""" try: import html - from src.core.config import settings - from src.services.ai_advisory_helpers import build_ai_advisory_keyboard, is_snoozed - from src.services.telegram_gateway import get_telegram_gateway - target_chat_id = settings.SRE_GROUP_CHAT_ID - if not target_chat_id: - logger.info("hermes_telegram_skip_no_chat_id") - return False + from src.services.ai_advisory_helpers import ( + build_ai_advisory_keyboard, + is_snoozed, + ) + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) # Snooze check: 以第一條 noisy rule_name 為 key primary_rule = noisy[0]["rule_name"] if noisy else "unknown" @@ -363,14 +364,15 @@ async def _send_telegram_summary( ) tg = get_telegram_gateway() - await tg._send_request("sendMessage", { # type: ignore[attr-defined] - "chat_id": target_chat_id, - "text": msg, - "parse_mode": "HTML", - "disable_web_page_preview": True, - "reply_markup": keyboard, - }) - return True + result = await tg.send_canonical_message( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P2", + text=msg, + parse_mode="HTML", + reply_markup=keyboard, + ) + return _telegram_send_delivery_succeeded(result) except Exception as e: logger.warning("hermes_telegram_send_failed", error=str(e)) return False diff --git a/apps/api/src/services/ai_agent_report_truth_actionability_review.py b/apps/api/src/services/ai_agent_report_truth_actionability_review.py index 14050da87..ae42efa10 100644 --- a/apps/api/src/services/ai_agent_report_truth_actionability_review.py +++ b/apps/api/src/services/ai_agent_report_truth_actionability_review.py @@ -42,7 +42,7 @@ _BOT_TOKEN_URL_RE = re.compile( + r")\b", re.IGNORECASE, ) -_TELEGRAM_EGRESS_SCAN_SUFFIXES = {".py", ".sh", ".js", ".yml", ".yaml"} +_TELEGRAM_EGRESS_SCAN_SUFFIXES = {".py", ".ps1", ".sh", ".js", ".yml", ".yaml"} def load_latest_ai_agent_report_truth_actionability_review( @@ -188,6 +188,9 @@ def _telegram_egress_scan_roots(root: Path) -> list[Path]: roots = guard.get("guarded_roots") if isinstance(guard.get("guarded_roots"), list) else [] if not roots: roots = [".gitea/workflows", "scripts/ops", "scripts/ci", "apps/api/src"] + for required_root in ("agent99-control-plane.ps1", "scripts/reboot-recovery"): + if required_root not in roots: + roots.append(required_root) return [root / str(path) for path in roots] @@ -210,6 +213,10 @@ def _telegram_direct_surface_kind(relative_path: str) -> str: return "ci_script_direct_bot_api" if relative_path.startswith("apps/api/src/"): return "api_direct_bot_api" + if relative_path == "agent99-control-plane.ps1": + return "agent99_control_plane_direct_bot_api" + if relative_path.startswith("scripts/reboot-recovery/"): + return "reboot_recovery_direct_bot_api" return "unknown_direct_bot_api" @@ -222,6 +229,10 @@ def _telegram_direct_persistence_state(surface_kind: str) -> str: return "ci_script_log_only_no_awooop_db_receipt" if surface_kind == "api_direct_bot_api": return "api_direct_send_no_awooop_outbound_receipt_contract" + if surface_kind == "agent99_control_plane_direct_bot_api": + return "agent99_local_log_only_no_awooop_db_receipt" + if surface_kind == "reboot_recovery_direct_bot_api": + return "reboot_recovery_stdout_only_no_awooop_db_receipt" return "direct_send_no_awooop_outbound_receipt_contract" @@ -238,7 +249,8 @@ def _scan_current_direct_telegram_endpoints(root: Path) -> list[dict[str, Any]]: for scan_root in _telegram_egress_scan_roots(root): if not scan_root.exists(): continue - for path in sorted(scan_root.rglob("*")): + candidate_paths = [scan_root] if scan_root.is_file() else sorted(scan_root.rglob("*")) + for path in candidate_paths: if not path.is_file() or path.suffix not in _TELEGRAM_EGRESS_SCAN_SUFFIXES: continue relative_path = path.relative_to(root).as_posix() diff --git a/apps/api/src/services/ai_rate_limiter.py b/apps/api/src/services/ai_rate_limiter.py index 09b2eda28..87fee12a6 100644 --- a/apps/api/src/services/ai_rate_limiter.py +++ b/apps/api/src/services/ai_rate_limiter.py @@ -269,16 +269,23 @@ class AIRateLimiter: if await r.get(alert_sent_key): return - # 標記已發送 (24小時後可重新發送) - await r.set(alert_sent_key, "1", ex=86400) - try: - from src.core.config import settings - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) - target_chat_id = settings.SRE_GROUP_CHAT_ID - if not settings.OPENCLAW_TG_BOT_TOKEN or not target_chat_id: - logger.warning("telegram_not_configured_for_cost_alert") + gateway = get_telegram_gateway() + if not gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="business_finops", + severity="P2", + ): + logger.warning( + "ai_cost_alert_no_send", + provider=provider, + reason="canonical_route_unavailable", + ) return message = ( @@ -291,22 +298,36 @@ class AIRateLimiter: f"redis-cli DEL ai_rate:total_cost:{provider}" ) - gateway = get_telegram_gateway() - await gateway._send_request( - "sendMessage", - { - "chat_id": target_chat_id, - "text": message, - "parse_mode": "HTML", - }, + result = await gateway.send_canonical_message( + product_id="awoooi", + signal_family="business_finops", + severity="P2", + text=message, + parse_mode="HTML", ) - logger.error( - "ai_cost_alert_sent", - provider=provider, - current_cost=f"${current_cost:.2f}", - limit=f"${limit:.2f}", - ) + if _telegram_send_delivery_succeeded(result): + await r.set(alert_sent_key, "1", ex=86400) + logger.warning( + "ai_cost_alert_sent", + provider=provider, + current_cost=f"${current_cost:.2f}", + limit=f"${limit:.2f}", + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) + else: + logger.warning( + "ai_cost_alert_no_send", + provider=provider, + delivery_status=( + result.get( + "_awooop_delivery_status", + "missing_provider_ack", + ) + if isinstance(result, dict) + else "missing_provider_ack" + ), + ) except Exception as e: logger.error("ai_cost_alert_failed", error=str(e)) @@ -322,14 +343,23 @@ class AIRateLimiter: if await r.get(warning_key): return - await r.set(warning_key, "1", ex=3600) - try: - from src.core.config import settings - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) - target_chat_id = settings.SRE_GROUP_CHAT_ID - if not settings.OPENCLAW_TG_BOT_TOKEN or not target_chat_id: + gateway = get_telegram_gateway() + if not gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="business_finops", + severity="P2", + ): + logger.warning( + "ai_cost_warning_no_send", + provider=provider, + reason="canonical_route_unavailable", + ) return limit = COST_LIMITS[provider]["total_cost_usd"] @@ -343,22 +373,35 @@ class AIRateLimiter: f"接近上限，請注意監控！" ) - gateway = get_telegram_gateway() - await gateway._send_request( - "sendMessage", - { - "chat_id": target_chat_id, - "text": message, - "parse_mode": "HTML", - }, + result = await gateway.send_canonical_message( + product_id="awoooi", + signal_family="business_finops", + severity="P2", + text=message, + parse_mode="HTML", ) - logger.warning( - "ai_cost_warning_sent", - provider=provider, - current_cost=f"${current_cost:.2f}", - threshold=f"${threshold:.2f}", - ) + if _telegram_send_delivery_succeeded(result): + await r.set(warning_key, "1", ex=3600) + logger.warning( + "ai_cost_warning_sent", + provider=provider, + current_cost=f"${current_cost:.2f}", + threshold=f"${threshold:.2f}", + ) + else: + logger.warning( + "ai_cost_warning_no_send", + provider=provider, + delivery_status=( + result.get( + "_awooop_delivery_status", + "missing_provider_ack", + ) + if isinstance(result, dict) + else "missing_provider_ack" + ), + ) except Exception as e: logger.warning("ai_cost_warning_failed", error=str(e)) diff --git a/apps/api/src/services/ai_router.py b/apps/api/src/services/ai_router.py index 2e88fc307..5dffc0ce7 100644 --- a/apps/api/src/services/ai_router.py +++ b/apps/api/src/services/ai_router.py @@ -1354,7 +1354,13 @@ class AIRouterExecutor: "需要人工介入" ) _asyncio.create_task( - tg.send_alert_notification(formatted, parse_mode="HTML") + tg.send_alert_notification( + formatted, + parse_mode="HTML", + product_id="awoooi", + signal_family="shared_infrastructure", + severity="P1", + ) ) except Exception as _tg_e: logger.warning("diagnose_reject_telegram_failed", error=str(_tg_e)) diff --git a/apps/api/src/services/approval_execution.py b/apps/api/src/services/approval_execution.py index c609957ef..8cf1058e4 100644 --- a/apps/api/src/services/approval_execution.py +++ b/apps/api/src/services/approval_execution.py @@ -1127,17 +1127,49 @@ class ApprovalExecutionService: approval_id=str(approval.id), ) - from src.core.config import get_settings - from src.services.telegram_gateway import get_telegram_gateway - settings = get_settings() + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) gateway = get_telegram_gateway() - target_chat_id = settings.SRE_GROUP_CHAT_ID + target_chat_id = gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) if not target_chat_id: logger.warning( "push_execution_result_no_target_chat", incident_id=approval.incident_id, approval_id=str(approval.id), ) + try: + from src.repositories.alert_operation_log_repository import ( + get_alert_operation_log_repository, + ) + + await get_alert_operation_log_repository().append( + "NOTIFICATION_CLASSIFIED", + incident_id=approval.incident_id, + approval_id=str(approval.id), + actor="approval_execution", + action_detail="telegram_execution_result_no_send", + success=False, + error_message="canonical_route_unavailable", + context={ + "delivery_status": "blocked_no_route", + "execution_success": success, + "execution_kind": execution_kind, + "repair_executed": repair_executed, + "repair_attempted": repair_attempted, + }, + ) + except Exception as _log_e: + logger.warning( + "alert_op_telegram_result_no_send_write_failed", + approval_id=str(approval.id), + error=str(_log_e), + ) return # 2026-04-19 ogt + Claude Opus 4.7 修 AP-2: 除了 reply 外, @@ -1157,6 +1189,7 @@ class ApprovalExecutionService: km_info = "" try: from sqlalchemy import text as _sql + from src.db.base import get_db_context async with get_db_context() as _db: _km_row = await _db.execute( @@ -1187,20 +1220,20 @@ class ApprovalExecutionService: truth_chain_row = incident_truth_chain_button_row( approval.incident_id or "" ) - payload: dict[str, Any] = { - "chat_id": target_chat_id, - "text": text, - "parse_mode": "HTML", - "disable_web_page_preview": True, - } - if truth_chain_row: - payload["reply_markup"] = { - "inline_keyboard": [truth_chain_row], - } - if orig_msg_id is not None: - payload["reply_to_message_id"] = orig_msg_id - - await gateway._send_request("sendMessage", payload) + delivery_result = await gateway.send_canonical_message( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + text=text, + parse_mode="HTML", + reply_markup=( + {"inline_keyboard": [truth_chain_row]} + if truth_chain_row + else None + ), + reply_to_message_id=orig_msg_id, + ) + delivery_succeeded = _telegram_send_delivery_succeeded(delivery_result) context_execution_kind = execution_kind or ( "no_action" if no_action else "execution" ) @@ -1209,8 +1242,22 @@ class ApprovalExecutionService: if repair_executed is not None else bool(success and not no_action) ) - logger.info( - "push_execution_result_sent", + delivery_status = str( + ( + delivery_result.get("_awooop_delivery_status") + if isinstance(delivery_result, dict) + else None + ) + or "missing_provider_ack" + ) + log_event = ( + "push_execution_result_sent" + if delivery_succeeded + else "push_execution_result_no_send" + ) + log_method = logger.info if delivery_succeeded else logger.warning + log_method( + log_event, incident_id=approval.incident_id, approval_id=str(approval.id), success=success, @@ -1220,6 +1267,7 @@ class ApprovalExecutionService: repair_executed=context_repair_executed, operator_outcome_state=outcome.get("state"), needs_human=outcome.get("needs_human"), + delivery_status=delivery_status, ) try: from src.repositories.alert_operation_log_repository import ( @@ -1227,13 +1275,25 @@ class ApprovalExecutionService: ) await get_alert_operation_log_repository().append( - "TELEGRAM_RESULT_SENT", + ( + "TELEGRAM_RESULT_SENT" + if delivery_succeeded + else "NOTIFICATION_CLASSIFIED" + ), incident_id=approval.incident_id, approval_id=str(approval.id), actor="approval_execution", - action_detail="telegram_execution_result_sent", - success=success, - error_message=error, + action_detail=( + "telegram_execution_result_sent" + if delivery_succeeded + else "telegram_execution_result_no_send" + ), + success=(success if delivery_succeeded else False), + error_message=( + error + if delivery_succeeded + else delivery_status or "missing_provider_ack" + ), context={ "reply_to_message_id": orig_msg_id, "execution_kind": context_execution_kind, @@ -1241,6 +1301,9 @@ class ApprovalExecutionService: "repair_attempted": repair_attempted, "operator_outcome": outcome, "delivery": "reply" if orig_msg_id is not None else "standalone", + "delivery_status": delivery_status, + "provider_delivery_acknowledged": delivery_succeeded, + "execution_success": success, }, ) except Exception as _log_e: diff --git a/apps/api/src/services/channel_hub.py b/apps/api/src/services/channel_hub.py index 83fc28a83..43db5b280 100644 --- a/apps/api/src/services/channel_hub.py +++ b/apps/api/src/services/channel_hub.py @@ -1014,6 +1014,7 @@ async def schedule_interim_feedback( run_id: UUID, channel_type: str, channel_chat_id: str, + inbound_message_id: int | None = None, conversation_event_id: UUID | None = None, is_shadow: bool = True, wait_seconds: int = _INTERIM_WAIT_SECONDS, @@ -1030,6 +1031,7 @@ async def schedule_interim_feedback( run_id=run_id, channel_type=channel_type, channel_chat_id=channel_chat_id, + inbound_message_id=inbound_message_id, conversation_event_id=conversation_event_id, is_shadow=is_shadow, wait_seconds=wait_seconds, @@ -1044,6 +1046,7 @@ async def _interim_feedback_task( run_id: UUID, channel_type: str, channel_chat_id: str, + inbound_message_id: int | None, conversation_event_id: UUID | None, is_shadow: bool, wait_seconds: int, @@ -1095,14 +1098,26 @@ async def _interim_feedback_task( if not (is_shadow or run_is_shadow): # Non-shadow：實際發 Telegram 訊息 - await _send_telegram_interim( + delivered = await _send_telegram_interim( channel_chat_id=channel_chat_id, content=interim_content, run_id=run_id, + inbound_message_id=inbound_message_id, ) + if not delivered: + logger.warning( + "interim_feedback_blocked_no_send", + project_id=project_id, + run_id=str(run_id), + ) + return logger.info( - "interim_feedback_sent", + ( + "interim_feedback_shadow_recorded" + if is_shadow or run_is_shadow + else "interim_feedback_sent" + ), project_id=project_id, run_id=str(run_id), is_shadow=is_shadow or run_is_shadow, @@ -1121,23 +1136,45 @@ async def _send_telegram_interim( channel_chat_id: str, content: str, run_id: UUID, -) -> None: + inbound_message_id: int | None = None, +) -> bool: """透過 TelegramGateway 發送 interim 訊息（non-shadow 專用）。""" try: - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) - await get_telegram_gateway().send_text( + if inbound_message_id is None or inbound_message_id <= 0: + logger.warning( + "interim_telegram_missing_verified_parent_no_send", + run_id=str(run_id), + ) + return False + + result = await get_telegram_gateway().send_text( content, chat_id=channel_chat_id, parse_mode="HTML", disable_web_page_preview=True, + interaction_kind="operator_command_reply", + inbound_chat_id=channel_chat_id, + inbound_message_id=inbound_message_id, ) + if not _telegram_send_delivery_succeeded(result): + logger.warning( + "interim_telegram_delivery_not_proven_no_send", + run_id=str(run_id), + ) + return False + return True except Exception as exc: logger.warning( "interim_telegram_send_failed", run_id=str(run_id), error=str(exc), ) + return False # ───────────────────────────────────────────────────────────────────────────── @@ -1189,11 +1226,16 @@ async def handle_telegram_inbound( # Step 3: Progressive Feedback（30s 計時器） if not is_duplicate: + try: + inbound_message_id = int(message_id) + except (TypeError, ValueError): + inbound_message_id = None await schedule_interim_feedback( project_id=project_id, run_id=run_id, channel_type="telegram", channel_chat_id=chat_id, + inbound_message_id=inbound_message_id, conversation_event_id=event_id, is_shadow=is_shadow, ) diff --git a/apps/api/src/services/converged_alert_recurrence_notifier.py b/apps/api/src/services/converged_alert_recurrence_notifier.py index 054f0330c..41533f12c 100644 --- a/apps/api/src/services/converged_alert_recurrence_notifier.py +++ b/apps/api/src/services/converged_alert_recurrence_notifier.py @@ -5,7 +5,10 @@ import html from src.core.logging import get_logger from src.core.redis_client import get_redis -from src.services.telegram_gateway import get_telegram_gateway +from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) logger = get_logger("awoooi.converged_alert_recurrence") @@ -67,6 +70,22 @@ async def should_notify_converged_alert_recurrence( return is_converged_alert_recurrence_milestone(hit_count) +async def _release_converged_alert_recurrence_reservation( + fingerprint: str, +) -> None: + """Release a pre-send reservation when provider delivery is not proven.""" + + key = converged_alert_recurrence_key(fingerprint) + try: + await get_redis().delete(key) + except Exception as exc: + logger.warning( + "converged_alert_recurrence_reservation_release_failed", + fingerprint_hash=key.rsplit(":", 1)[-1], + error=str(exc), + ) + + def format_converged_alert_recurrence_message( *, source: str, @@ -166,9 +185,23 @@ async def notify_converged_alert_recurrence( failures: list[str] = [] try: - await gateway.send_alert_notification(text) - sent_count += 1 + result = await gateway.send_alert_notification( + text, + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) + if _telegram_send_delivery_succeeded(result): + sent_count += 1 + else: + await _release_converged_alert_recurrence_reservation(fingerprint) + result_payload = result if isinstance(result, dict) else {} + receipt = result_payload.get("_awoooi_canonical_route_receipt") or {} + failures.append( + f"route:{receipt.get('classifier') or 'blocked_no_egress'}" + ) except Exception as exc: + await _release_converged_alert_recurrence_reservation(fingerprint) failures.append(f"primary:{type(exc).__name__}") logger.warning( "converged_alert_recurrence_primary_failed", diff --git a/apps/api/src/services/decision_manager.py b/apps/api/src/services/decision_manager.py index e59d3a3f9..afbe5964f 100644 --- a/apps/api/src/services/decision_manager.py +++ b/apps/api/src/services/decision_manager.py @@ -195,19 +195,57 @@ async def _send_log_summary(incident: "Incident") -> None: if incident.signals: namespace = incident.signals[0].labels.get("namespace", "awoooi-prod") + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) + + tg = get_telegram_gateway() + if not tg.canonical_destination_chat_id( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ): + logger.warning( + "log_summary_no_send", + target=target, + incident_id=incident.incident_id, + reason="canonical_route_unavailable", + ) + return + from src.services.log_summary_service import get_log_summary_service svc = get_log_summary_service() summary = await svc.summarize_with_soft_timeout(pod_name=target, namespace=namespace) if not summary: return - from src.services.telegram_gateway import get_telegram_gateway - tg = get_telegram_gateway() - await tg.send_as_nemotron( - f"📋 Log 異常摘要 — {target}\n{summary}" + result = await tg.send_as_nemotron( + f"📋 Log 異常摘要 — {target}\n{summary}", + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", ) - import structlog as _sl - _sl.get_logger(__name__).info("log_summary_sent", target=target, incident_id=incident.incident_id) + if _telegram_send_delivery_succeeded(result): + logger.info( + "log_summary_sent", + target=target, + incident_id=incident.incident_id, + ) + else: + logger.warning( + "log_summary_no_send", + target=target, + incident_id=incident.incident_id, + delivery_status=( + result.get( + "_awooop_delivery_status", + "missing_provider_ack", + ) + if isinstance(result, dict) + else "missing_provider_ack" + ), + ) except Exception as e: import structlog as _sl _sl.get_logger(__name__).warning("log_summary_failed", error=str(e)) @@ -220,7 +258,7 @@ async def _send_log_summary(incident: "Incident") -> None: async def _push_decision_to_telegram( incident: Incident, proposal_data: dict[str, Any], -) -> None: +) -> bool: """ 決策就緒時推送到 Telegram @@ -231,11 +269,12 @@ async def _push_decision_to_telegram( # 延遲導入避免循環依賴 from src.core.redis_client import get_redis from src.services.telegram_gateway import ( + NotificationType, + _telegram_send_delivery_succeeded, classify_notification, get_telegram_gateway, - NotificationType, - _smart_truncate as _smt, ) + from src.services.telegram_gateway import _smart_truncate as _smt # 🔴 去重檢查：同一 fingerprint (alertname+target) 24h 內只發一次 decision card # 2026-05-02 Claude Opus 4.7 + 統帥 ogt：原本 dedup_key 用 incident.incident_id， @@ -256,7 +295,7 @@ async def _push_decision_to_telegram( incident_id=incident.incident_id, fingerprint=f"{_alertname_fp}:{_target_fp}", ) - return + return False # 2026-04-09 Claude Code: resolved Incident 不重送 Telegram # 場景: dedup TTL 過期後，已 resolve 的 Incident 仍被重新推送 @@ -266,7 +305,7 @@ async def _push_decision_to_telegram( reason="Incident already resolved", incident_id=incident.incident_id, ) - return + return False # 🔴 靜默檢查：此資源是否被靜默 (2026-03-27 P1 優化) target = incident.affected_services[0] if incident.affected_services else "unknown" @@ -278,7 +317,7 @@ async def _push_decision_to_telegram( incident_id=incident.incident_id, resource=target, ) - return + return False # 檢查是否有設定 Bot Token if not settings.OPENCLAW_TG_BOT_TOKEN: @@ -287,7 +326,7 @@ async def _push_decision_to_telegram( reason="Bot token not configured", incident_id=incident.incident_id, ) - return + return False gateway = get_telegram_gateway() @@ -411,7 +450,7 @@ async def _push_decision_to_telegram( action_text=_action_text, reason="Agent Debate 無有效輸出 (description=待分析, action 屬於失敗狀態集合)", ) - return + return False # 建立 approval_id (使用 incident_id 作為追蹤) # 2026-03-27 ogt: 修復 INC-INC-INC- 重複前綴 bug @@ -581,12 +620,37 @@ async def _push_decision_to_telegram( automation_state=proposal_data.get("automation_state", ""), ) + if not _telegram_send_delivery_succeeded(tg_result): + logger.warning( + "telegram_decision_no_send", + incident_id=incident.incident_id, + source=source, + risk_level=risk_level, + delivery_status=( + tg_result.get("_awooop_delivery_status") + if isinstance(tg_result, dict) + else "missing_provider_ack" + ), + ) + return False + # 2026-04-09 Claude Sonnet 4.6: 存 message_id → 後續狀態更新在原訊息延續 # 同時寫 Redis (快速查詢) 和 DB (持久化，不受 TTL 限制) tg_message_id = tg_result.get("result", {}).get("message_id") if isinstance(tg_result, dict) else None tg_chat_id = tg_result.get("result", {}).get("chat", {}).get("id") if isinstance(tg_result, dict) else None if tg_message_id: - await redis.setex(f"tg_msg:{incident.incident_id}", 86400, str(tg_message_id)) + try: + await redis.setex( + f"tg_msg:{incident.incident_id}", + 86400, + str(tg_message_id), + ) + except Exception as _redis_message_error: + logger.warning( + "telegram_message_id_redis_save_failed", + incident_id=incident.incident_id, + error=str(_redis_message_error), + ) # 持久化到 DB try: from src.services.approval_db import get_approval_service as _get_approval_svc @@ -599,6 +663,21 @@ async def _push_decision_to_telegram( except Exception as _e: logger.warning("telegram_message_id_db_save_failed", incident_id=incident.incident_id, error=str(_e)) + # Provider ack is the delivery truth. Persist dedup only after that + # receipt, while keeping a dedup write failure separate from delivery. + # 2026-05-02 Claude Opus 4.7 + 統帥 ogt：TTL 從 600s 拉到 86400s； + # incident_analysis_sweeper 每 1h 與 Alertmanager 每 4h 都可能重觸， + # 24h TTL 用來避免同症狀重複洗版。 + try: + await redis.setex(dedup_key, 86400, "1") + except Exception as _dedup_error: + logger.warning( + "telegram_decision_dedup_write_failed", + incident_id=incident.incident_id, + dedup_key=dedup_key, + error=str(_dedup_error), + ) + # Phase 31 (ADR-067 2026-04-10): Log 異常摘要 — NemoTron deepseek-r1:14b # 非同步執行，不阻塞主流程 _fire_and_forget(_send_log_summary(incident)) @@ -610,22 +689,20 @@ async def _push_decision_to_telegram( _fire_and_forget( gateway.send_as_nemotron( f"🤔 NemoClaw 第二意見 (信心={confidence:.2f})\n" - f"{_advisory_note}" + f"{_advisory_note}", + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", ) ) - # 🔴 發送成功後設置去重 key (TTL 24 小時) - # 2026-05-02 Claude Opus 4.7 + 統帥 ogt：TTL 從 600s 拉到 86400s。 - # 原因：incident_analysis_sweeper 每 1h 重觸 decision、alertmanager 預設 4h repeat， - # 600s TTL 早就過期 → 重發；24h TTL 蓋住兩個週期，徹底治住「同症狀 24h 推 15 次」。 - await redis.setex(dedup_key, 86400, "1") - logger.info( "telegram_decision_pushed", incident_id=incident.incident_id, source=source, risk_level=risk_level, ) + return True except Exception as e: # Telegram 失敗不影響主流程 @@ -634,6 +711,7 @@ async def _push_decision_to_telegram( incident_id=incident.incident_id, error=str(e), ) + return False async def _nemoclaw_second_opinion(incident: "Incident", primary_result: dict) -> str | None: @@ -831,7 +909,10 @@ async def _generate_playbook_draft_if_new(incident: "Incident") -> None: f"📚 KM 新條目 — {entry.entry_id}\n" f"🏷️ 分類：auto_generated 狀態：DRAFT\n" f"📋 告警：{alertname} 事件：{incident.incident_id}\n" - f"標題：{str(getattr(entry, 'title', alertname))[:60]}" + f"標題：{str(getattr(entry, 'title', alertname))[:60]}", + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", ) except Exception as _tg_err: _sl.get_logger(__name__).debug("km_tg_notify_failed", error=str(_tg_err)) @@ -1110,7 +1191,10 @@ async def _push_auto_repair_result( 2026-04-11 Claude Sonnet 4.6: +ADR-071-I 指標快照 """ try: - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) gateway = get_telegram_gateway() target = incident.affected_services[0] if incident.affected_services else "unknown" @@ -1217,9 +1301,57 @@ async def _push_auto_repair_result( f"├─ 對象: {target[:50]}\n" f"└─ {status_line}" ) - await gateway.send_alert_notification(fallback_text) + if not gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ): + logger.warning( + "auto_repair_result_no_send", + incident_id=inc_id, + success=success, + reason="canonical_route_unavailable", + ) + return - logger.info("auto_repair_result_sent", incident_id=inc_id, success=success, appended=appended) + fallback_result = await gateway.send_alert_notification( + fallback_text, + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) + if not _telegram_send_delivery_succeeded(fallback_result): + logger.warning( + "auto_repair_result_no_send", + incident_id=inc_id, + success=success, + delivery_status=( + fallback_result.get( + "_awooop_delivery_status", + "missing_provider_ack", + ) + if isinstance(fallback_result, dict) + else "missing_provider_ack" + ), + ) + return + + logger.info( + "auto_repair_result_sent", + incident_id=inc_id, + success=success, + appended=False, + ) + else: + # append_incident_update currently exposes only a boolean request result, + # not the provider receipt. Do not claim a Telegram send without the + # shared affirmative delivery verifier. + logger.info( + "auto_repair_result_update_requested", + incident_id=inc_id, + success=success, + appended=True, + ) except Exception as e: logger.warning("auto_repair_result_push_failed", incident_id=incident.incident_id, error=str(e)) @@ -3506,32 +3638,45 @@ class DecisionManager: """ import asyncio as _asyncio import json as _json + from src.core.redis_client import get_redis from src.db.base import get_db_context from src.repositories.incident_repository import IncidentDBRepository redis = get_redis() - resent = 0 # GAP-A4 後續修復：限制並發 5，避免壓爆 Ollama _sem = _asyncio.Semaphore(5) - async def _bounded_push(incident_obj, proposal_data_obj, _id, _token): + async def _bounded_push(incident_obj, proposal_data_obj, _id, _token) -> bool: async with _sem: try: - await _push_decision_to_telegram(incident_obj, proposal_data_obj) - logger.info( - "stale_ready_token_resent", - incident_id=_id, - token=_token, + delivered = await _push_decision_to_telegram( + incident_obj, + proposal_data_obj, ) + if delivered: + logger.info( + "stale_ready_token_resent", + incident_id=_id, + token=_token, + ) + else: + logger.warning( + "stale_ready_token_no_send", + incident_id=_id, + token=_token, + ) + return delivered except Exception as _e: logger.warning( "stale_ready_token_resend_failed", incident_id=_id, error=str(_e), ) - # 每次完成後喘 200ms，給 Ollama embedding 恢復空間 - await _asyncio.sleep(0.2) + return False + finally: + # 每次完成後喘 200ms，給 Ollama embedding 恢復空間 + await _asyncio.sleep(0.2) try: # 掃描所有 decision:* key @@ -3575,7 +3720,10 @@ class DecisionManager: _STALE_DAYS = 3 _created_at = getattr(incident, "created_at", None) if _created_at: - from datetime import datetime as _dt, timedelta as _td, timezone as _tz + from datetime import datetime as _dt + from datetime import timedelta as _td + from datetime import timezone as _tz + _now = _dt.now(_tz.utc) _cutoff = _now - _td(days=_STALE_DAYS) # 確保 _created_at 有時區 @@ -3599,20 +3747,26 @@ class DecisionManager: _bounded_push(incident, proposal_data, incident_id, data.get("token", "")) ) tasks.append(_task) - resent += 1 except Exception as _te: logger.debug("stale_ready_token_scan_error", error=str(_te)) if cursor == 0: break - # 不等所有 task 完成（fire-and-forget 語義保留），但 await 一下讓並發限制生效 + resent = 0 if tasks: - logger.info("stale_ready_tokens_throttled_dispatch", - total=len(tasks), max_concurrent=5) + logger.info( + "stale_ready_tokens_throttled_dispatch", + total=len(tasks), + max_concurrent=5, + status="awaiting_delivery_receipts", + ) + delivery_results = await _asyncio.gather(*tasks) + resent = sum(result is True for result in delivery_results) except Exception as e: logger.warning("resend_stale_ready_tokens_failed", error=str(e)) + resent = 0 logger.info("stale_ready_tokens_scan_done", resent=resent) return resent diff --git a/apps/api/src/services/drift_adopt_service.py b/apps/api/src/services/drift_adopt_service.py index 6229e3845..3dbd61429 100644 --- a/apps/api/src/services/drift_adopt_service.py +++ b/apps/api/src/services/drift_adopt_service.py @@ -477,7 +477,10 @@ class DriftAdoptService: f"Namespace: {report.namespace}\n" f"漂移: {report.summary}\n\n" f"PR: {pr_url}\n\n" - f"請 SRE review 後 merge。" + f"請 SRE review 後 merge。", + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", ) except Exception as e: logger.warning("drift_adopt_telegram_failed", error=str(e)) diff --git a/apps/api/src/services/drift_remediator.py b/apps/api/src/services/drift_remediator.py index 514fe5113..19bb51f46 100644 --- a/apps/api/src/services/drift_remediator.py +++ b/apps/api/src/services/drift_remediator.py @@ -224,7 +224,12 @@ class DriftRemediator: try: from src.services.telegram_gateway import get_telegram_gateway tg = get_telegram_gateway() - await tg.send_text(message) + await tg.send_text( + message, + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) except Exception as e: logger.warning("drift_remediator_telegram_failed", error=str(e)) diff --git a/apps/api/src/services/emergency_escalation_service.py b/apps/api/src/services/emergency_escalation_service.py index 4a8a15b6c..cc809bda9 100644 --- a/apps/api/src/services/emergency_escalation_service.py +++ b/apps/api/src/services/emergency_escalation_service.py @@ -10,7 +10,6 @@ from typing import Any import structlog -from src.core.config import settings from src.core.redis_client import get_redis logger = structlog.get_logger(__name__) @@ -78,7 +77,6 @@ async def escalate_auto_repair_unavailable( attempted_actions=attempted_actions, failure_reason=failure_reason, current_impact=f"target={target_resource or 'unknown'} namespace={namespace or 'unknown'}", - group_chat_id=settings.SRE_GROUP_CHAT_ID or None, ) await get_alert_operation_log_repository().append( @@ -177,7 +175,6 @@ async def escalate_drift_auto_adopt_blocked( f"intent={intent} confidence={confidence:.0%} risk={risk} " f"fingerprint={fingerprint}" ), - group_chat_id=settings.SRE_GROUP_CHAT_ID or None, ) await get_alert_operation_log_repository().append( "APPROVAL_ESCALATED", diff --git a/apps/api/src/services/failover_alerter.py b/apps/api/src/services/failover_alerter.py index 3c9456424..ad4f825d5 100644 --- a/apps/api/src/services/failover_alerter.py +++ b/apps/api/src/services/failover_alerter.py @@ -68,8 +68,15 @@ class FailoverAlerter: f"Fallback 鏈：{_escape_md(fallback_chain_str)}\n\n" f"自動恢復服務持續監控，3 次 HEALTHY 後自動切回" ) - await self._send(msg) - logger.info("failover_alert_sent", to_provider=to_provider) + delivered = await self._deliver_reserved( + msg, + dedup_key, + ttl=DEDUP_TTL_SEC, + ) + if delivered: + logger.info("failover_alert_sent", to_provider=to_provider) + else: + logger.warning("failover_alert_no_send", to_provider=to_provider) async def alert_recovery(self, event: dict[str, Any]) -> None: """Ollama 自動恢復告警 — 1h dedup per host @@ -99,8 +106,15 @@ class FailoverAlerter: f"切換路徑：{_escape_md(str(from_provider))} → {_escape_md(str(to_provider))}\n\n" f"自動化飛輪已恢復至高效能推理模式" ) - await self._send(msg) - logger.info("recovery_alert_sent", from_provider=from_provider) + delivered = await self._deliver_reserved( + msg, + dedup_key, + ttl=RECOVERY_DEDUP_TTL_SEC, + ) + if delivered: + logger.info("recovery_alert_sent", from_provider=from_provider) + else: + logger.warning("recovery_alert_no_send", from_provider=from_provider) async def alert_governance(self, event_type: str, payload: dict[str, Any]) -> None: """AI 治理告警（dedup 1h） @@ -135,8 +149,11 @@ class FailoverAlerter: return msg = format_governance_alert_card(event_type, payload) - await self._send(msg) - logger.info("governance_alert_sent", event_type=event_type) + delivered = await self._deliver_reserved(msg, dedup_key, ttl=3600) + if delivered: + logger.info("governance_alert_sent", event_type=event_type) + else: + logger.warning("governance_alert_no_send", event_type=event_type) async def alert_gemini_quota_exceeded(self, event: dict[str, Any]) -> None: """Gemini 每日上限觸發，降級到 188 CPU 備援 — 24h dedup（每日重置）""" @@ -161,8 +178,15 @@ class FailoverAlerter: f"進入容災模式至明日 0:00\n" f"建議檢查是否有異常流量，評估是否升級 Gemini 配額" ) - await self._send(msg) - logger.info("quota_alert_sent", quota=quota, current_count=current_count) + delivered = await self._deliver_reserved( + msg, + dedup_key, + ttl=QUOTA_DEDUP_TTL_SEC, + ) + if delivered: + logger.info("quota_alert_sent", quota=quota, current_count=current_count) + else: + logger.warning("quota_alert_no_send", quota=quota, current_count=current_count) async def alert_provider_version_changed(self, changed_providers: list[str], probed: int) -> None: """AI Provider 版本變更告警 — dedup 1h/provider @@ -171,19 +195,20 @@ class FailoverAlerter: 每個 provider 獨立 dedup，避免同一版本重複告警。 """ now_str = datetime.now(TAIPEI_TZ).strftime("%Y-%m-%d %H:%M") - sent: list[str] = [] + reserved: list[tuple[str, str]] = [] for provider in changed_providers: dedup_key = f"alert:provider_version_changed:{provider}" if not await self._check_dedup(dedup_key, ttl=3600): logger.debug("provider_version_alert_dedup_skipped", provider=provider) continue - sent.append(provider) + reserved.append((provider, dedup_key)) - if not sent: + if not reserved: return - providers_md = "\n".join(f"• {_escape_md(p)}" for p in sent) + providers = [provider for provider, _dedup_key in reserved] + providers_md = "\n".join(f"• {_escape_md(p)}" for p in providers) msg = ( f"*AI Provider 版本變更偵測*\n\n" f"時間：{_escape_md(now_str)}\n" @@ -191,8 +216,20 @@ class FailoverAlerter: f"版本已變更：\n{providers_md}\n\n" f"系統已自動記錄版本歷史，請確認是否需要重新驗證推理品質" ) - await self._send(msg) - logger.info("provider_version_alert_sent", sent=sent) + delivered = False + try: + delivered = await self._send(msg) + finally: + for _provider, dedup_key in reserved: + await self._finalize_dedup( + dedup_key, + ttl=3600, + delivered=delivered, + ) + if delivered: + logger.info("provider_version_alert_sent", sent=providers) + else: + logger.warning("provider_version_alert_no_send", providers=providers) # ------------------------------------------------------------------------- # Dedup（Redis SET NX EX） @@ -200,8 +237,12 @@ class FailoverAlerter: async def _check_dedup(self, key: str, ttl: int) -> bool: """ - Redis SET NX EX 防止重複告警。 - True = 第一次（應送出），False = 已送過（跳過）。 + Redis SET NX EX 建立 delivery reservation。 + True = 可嘗試送出，False = 已送過或另一個 worker 正在送。 + + reservation 的 value 是 ``pending``，不是 sent receipt。外層只有收到 + affirmative provider ack 後才透過 ``_finalize_dedup`` 改成 ``sent``； + no-send / exception 會刪除 reservation，讓下一輪可以安全重試。 2026-04-25 P1.5 by Claude Engineer-D — Telegram dedup 鐵律 10min/24h TTL 2026-04-27 Wave8-X2 by Claude — dedup fail-open 修復 @@ -212,7 +253,12 @@ class FailoverAlerter: # 優先嘗試 Redis if self._redis is not None: try: - ok = await self._redis.set(f"{key}:dedup", "1", ex=ttl, nx=True) + ok = await self._redis.set( + f"{key}:dedup", + "pending", + ex=ttl, + nx=True, + ) return bool(ok) except Exception as e: logger.warning("dedup_redis_failed_using_memory", error=str(e)) @@ -233,11 +279,63 @@ class FailoverAlerter: self._memory_dedup[key] = now return True + async def _finalize_dedup( + self, + key: str, + *, + ttl: int, + delivered: bool, + ) -> None: + """Commit sent dedup only after provider ack, otherwise release it. + + 2026-07-14 Codex v1.1 (Asia/Taipei): route acceptance and ``ok`` are + not delivery truth. Pending reservations prevent concurrent sends but + must never survive a structured no-send receipt as a terminal success. + """ + redis_finalized = False + if self._redis is not None: + try: + if delivered: + await self._redis.set(f"{key}:dedup", "sent", ex=ttl) + else: + await self._redis.delete(f"{key}:dedup") + redis_finalized = True + except Exception as e: + logger.warning( + "dedup_finalize_redis_failed_using_memory", + delivered=delivered, + error=str(e), + ) + + if delivered: + if not redis_finalized: + import time + + self._memory_dedup[key] = time.time() + else: + self._memory_dedup.pop(key, None) + + async def _deliver_reserved( + self, + message: str, + key: str, + *, + ttl: int, + ) -> bool: + """Deliver one reserved alert and always finalize or release its state.""" + + delivered = False + try: + delivered = await self._send(message) + return delivered + finally: + await self._finalize_dedup(key, ttl=ttl, delivered=delivered) + # ------------------------------------------------------------------------- # 發送（透過 TelegramGateway singleton） # ------------------------------------------------------------------------- - async def _send(self, message: str) -> None: + async def _send(self, message: str) -> bool: """發送至 Telegram SRE_GROUP_CHAT_ID 使用現有 TelegramGateway singleton（不另建 HTTP client）， @@ -248,18 +346,34 @@ class FailoverAlerter: 2026-04-25 P1.5 by Claude Engineer-D — 告警失敗不能阻斷主流程 """ try: - from src.core.config import get_settings - from src.services.telegram_gateway import get_telegram_gateway - - settings = get_settings() - chat_id = getattr(settings, "SRE_GROUP_CHAT_ID", None) - if not chat_id: - logger.warning("telegram_chat_id_missing_failover_alert") - return + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) gateway = get_telegram_gateway() - await gateway.send_alert_notification(text=message, parse_mode="MarkdownV2") - logger.info("telegram_failover_alert_sent", message_len=len(message)) + if not gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="shared_infrastructure", + severity="P1", + ): + logger.warning("telegram_canonical_route_unavailable_failover_alert") + return False + delivery = await gateway.send_alert_notification( + text=message, + parse_mode="MarkdownV2", + product_id="awoooi", + signal_family="shared_infrastructure", + severity="P1", + ) + if _telegram_send_delivery_succeeded(delivery): + return True + else: + logger.warning( + "telegram_failover_alert_blocked_no_send", + message_len=len(message), + ) + return False except Exception as e: # 不 raise — 告警失敗不該阻斷主流程（鐵律） # 2026-05-06 Codex: Telegram/httpx exception 字串可能包含 bot token URL， @@ -269,6 +383,7 @@ class FailoverAlerter: error=_sanitize_telegram_error(str(e)), error_type=type(e).__name__, ) + return False # ------------------------------------------------------------------------- diff --git a/apps/api/src/services/failure_watcher.py b/apps/api/src/services/failure_watcher.py index e6b537e67..bd137019d 100644 --- a/apps/api/src/services/failure_watcher.py +++ b/apps/api/src/services/failure_watcher.py @@ -732,7 +732,10 @@ class FailureWatcherService(IFailureWatcher): ) # 推送到 Telegram - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) tg = get_telegram_gateway() message = ( @@ -744,13 +747,24 @@ class FailureWatcherService(IFailureWatcher): f"└ 💡 下一步: {analysis.get('suggested_repair', 'AI 深度診斷')}\n\n" f"狀態：ai_retry_or_break_glass_recorded" ) - await tg.send_alert_notification(message) - - logger.info( - "ai_controlled_repair_followup_sent", - audit_log_id=audit_log_id, - risk_level=analysis.get("risk_level"), + delivery = await tg.send_alert_notification( + message, + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", ) + if _telegram_send_delivery_succeeded(delivery): + logger.info( + "ai_controlled_repair_followup_sent", + audit_log_id=audit_log_id, + risk_level=analysis.get("risk_level"), + ) + else: + logger.warning( + "ai_controlled_repair_followup_blocked_no_send", + audit_log_id=audit_log_id, + risk_level=analysis.get("risk_level"), + ) except Exception as e: logger.warning( @@ -776,7 +790,12 @@ class FailureWatcherService(IFailureWatcher): f"├ 📋 AuditLog: {audit_log_id[:8]}...\n" f"└ 📝 結果: {repair_result}" ) - await tg.send_alert_notification(message) + await tg.send_alert_notification( + message, + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) except Exception as e: logger.warning( diff --git a/apps/api/src/services/gitea_webhook_service.py b/apps/api/src/services/gitea_webhook_service.py index a9a697e6f..d092b03d2 100644 --- a/apps/api/src/services/gitea_webhook_service.py +++ b/apps/api/src/services/gitea_webhook_service.py @@ -406,9 +406,20 @@ class GiteaWebhookService: message = "\n".join(message_lines) - await telegram.send_alert_notification(message) + result = await telegram.send_alert_notification( + message, + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ) - logger.info("gitea_telegram_sent", review_id=review_id, repo=repo, event_type=event_type) + logger.info( + "gitea_telegram_routed", + review_id=review_id, + repo=repo, + event_type=event_type, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) except Exception as e: logger.exception("gitea_telegram_failed", error=str(e)) diff --git a/apps/api/src/services/image_analysis_service.py b/apps/api/src/services/image_analysis_service.py index 7e30431fd..442799a4d 100644 --- a/apps/api/src/services/image_analysis_service.py +++ b/apps/api/src/services/image_analysis_service.py @@ -171,6 +171,7 @@ class ImageAnalysisService: self, chat_id: str, file_id: str, + original_message_id: int | None = None, question: str = "請用繁體中文描述這張圖片，如果是截圖或介面請分析其內容和問題", ) -> None: """ @@ -192,7 +193,11 @@ class ImageAnalysisService: from src.services.telegram_gateway import get_telegram_gateway tg = get_telegram_gateway() await tg.initialize() - await tg.send_as_openclaw(f"🖼️ 圖片分析\n{result}") + await tg.send_as_openclaw( + f"🖼️ 圖片分析\n{result}", + reply_to_message_id=original_message_id, + inbound_chat_id=chat_id, + ) except Exception as e: logger.warning("download_and_analyze_failed", error=str(e)) diff --git a/apps/api/src/services/k3s_monitor_service.py b/apps/api/src/services/k3s_monitor_service.py index 3f8d592c2..21ace7cf3 100644 --- a/apps/api/src/services/k3s_monitor_service.py +++ b/apps/api/src/services/k3s_monitor_service.py @@ -21,13 +21,17 @@ K3s Monitor Service - Phase 21.2 定期報告 import asyncio from datetime import datetime from typing import Any, Protocol, runtime_checkable +from zoneinfo import ZoneInfo import httpx import structlog -from zoneinfo import ZoneInfo from src.core.config import settings -from src.services.telegram_gateway import K3sStatusMessage, get_telegram_gateway +from src.services.telegram_gateway import ( + K3sStatusMessage, + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) logger = structlog.get_logger(__name__) @@ -214,14 +218,17 @@ class K3sMonitorService: # 取得 Telegram Gateway gateway = get_telegram_gateway() - if not gateway._initialized: - await gateway.initialize() # 發送訊息 formatted = status.format() - result = await gateway.send_text(formatted) + result = await gateway.send_text( + formatted, + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ) - if result: + if _telegram_send_delivery_succeeded(result): logger.info("k3s_daily_report_sent", date=status.report_date) return True else: diff --git a/apps/api/src/services/notification_matrix.py b/apps/api/src/services/notification_matrix.py index fa6f2739e..921e23db2 100644 --- a/apps/api/src/services/notification_matrix.py +++ b/apps/api/src/services/notification_matrix.py @@ -1,64 +1,808 @@ -""" -Notification routing matrix — ADR-093 -====================================== -單一矩陣決定每種通知類型的發送目標，取代 telegram_gateway.py 內 24 處硬碼 chat_id。 +"""Fail-closed Telegram notification routing policy. -設計原則： -- 正式告警目的地一律為 SRE_GROUP_CHAT_ID -- SRE_GROUP_CHAT_ID 缺失時必須顯示配置缺口，不得旁路到個人或其他群組 -- 未知通知類型預設發群組 - -2026-04-25 ogt + Claude Sonnet 4.6 +The legacy ``TYPE-*`` matrix remains for existing callers, but an unknown type +no longer falls through to the shared SRE room. Product monitoring routes are +loaded from a committed, machine-readable registry. This module only returns +policy decisions: it never reads Telegram credentials or chat IDs, sends a +message, writes a receipt, or changes a runtime route. """ + from __future__ import annotations + +import json +import re from dataclasses import dataclass from enum import Enum +from pathlib import Path +from typing import Any + +from src.services.snapshot_paths import default_security_dir class Destination(str, Enum): - DM = "dm" # legacy alias: 2026-06-12 起不再旁路至 DM - GROUP = "group" # SRE_GROUP_CHAT_ID - BOTH = "both" # legacy alias: 2026-04-30 起視為 group-first + DM = "dm" # legacy alias; formal alerts do not fall back to DM + GROUP = "group" # symbolic shared group selected by the caller + BOTH = "both" # legacy alias; retained for import compatibility + BLOCKED = "blocked" @dataclass(frozen=True) class RoutingRule: destination: Destination - strip_buttons_for_group: bool = False # BOTH 時群組版是否去除 Callback Button + strip_buttons_for_group: bool = False -# ADR-093 D1-D4 路由矩陣 -# 2026-06-12 Codex: 所有正式告警只送 AwoooI SRE 戰情室；缺群組設定時回空清單。 +# ADR-093 legacy notification types. The global product routing registry below +# supersedes its unsafe "unknown -> shared group" default. NOTIFICATION_ROUTING: dict[str, RoutingRule] = { - "TYPE-1": RoutingRule(Destination.GROUP), - "TYPE-2": RoutingRule(Destination.GROUP), - "TYPE-3": RoutingRule(Destination.GROUP), - "TYPE-4": RoutingRule(Destination.GROUP), + "TYPE-1": RoutingRule(Destination.BLOCKED), + "TYPE-2": RoutingRule(Destination.GROUP), + "TYPE-3": RoutingRule(Destination.GROUP), + "TYPE-4": RoutingRule(Destination.GROUP), "TYPE-4D": RoutingRule(Destination.GROUP), "TYPE-5S": RoutingRule(Destination.GROUP), - "TYPE-6B": RoutingRule(Destination.GROUP), + "TYPE-6B": RoutingRule(Destination.BLOCKED), "TYPE-7E": RoutingRule(Destination.GROUP), "TYPE-8M": RoutingRule(Destination.GROUP), } -_DEFAULT_RULE = RoutingRule(Destination.GROUP) +_DEFAULT_RULE = RoutingRule(Destination.BLOCKED) +_LEGACY_CANONICAL_ROUTES: dict[str, tuple[str, str, str]] = { + "TYPE-2": ("awoooi", "incident_lifecycle", "P1"), + "TYPE-3": ("awoooi", "incident_lifecycle", "P1"), + "TYPE-4": ("awoooi", "incident_lifecycle", "P1"), + "TYPE-4D": ("awoooi", "security_incident", "P1"), + "TYPE-5S": ("awoooi", "security_incident", "P1"), + "TYPE-7E": ("awoooi", "incident_lifecycle", "P0"), + "TYPE-8M": ("awoooi", "shared_infrastructure", "P1"), +} +_SCHEMA_VERSION = "telegram_canonical_routing_registry_v1" +_REGISTRY_FILENAME = "telegram-canonical-routing-registry.snapshot.json" +_DEFAULT_SECURITY_DIR = default_security_dir(Path(__file__)) +_EXPECTED_PRODUCT_IDS = frozenset( + { + "2026fifa", + "agent-bounty-protocol", + "awooogo", + "awoooi", + "bitan-pharmacy", + "clawbot-openclaw", + "momo-pro", + "stockplatform-v2", + "tsenyang-website", + "vibework", + "vtuber", + } +) +_ALLOWED_SEVERITIES = frozenset({"P0", "P1", "P2", "P3"}) +_ALLOWED_EGRESS_STATUSES = frozenset( + {"allowed", "blocked", "disabled", "not_implemented"} +) +_SHARED_SRE_CHAT_ALIAS = "awoooi_sre_war_room" +_SHARED_MONITORING_BOT_ALIAS = "tsenyang_bot" +_SHARED_MONITORING_BOT_OWNER = "sre_shared_runtime" +_SHARED_SRE_SIGNAL_FAMILIES = frozenset( + { + "shared_infrastructure", + "security_incident", + "disaster_recovery", + "incident_lifecycle", + } +) +_REQUIRED_ROUTE_FIELDS = frozenset( + { + "route_id", + "product_id", + "signal_family", + "severity", + "scope", + "bot_alias", + "chat_alias", + "allowed_destination", + "receipt_backend", + "owner", + "egress_status", + } +) +_TOP_LEVEL_FIELDS = frozenset( + { + "schema_version", + "generated_at", + "status", + "mode", + "supersedes", + "policy", + "destination_roles", + "products", + "routes", + "rollups", + "execution_boundaries", + } +) +_SUPERSEDES_FIELDS = frozenset( + { + "policy_ref", + "superseded_behavior", + "superseded_by", + "owner", + "expiry", + "replacement", + "verifier", + } +) +_POLICY_FIELDS = frozenset( + { + "default_decision", + "unknown_product_decision", + "unknown_signal_family_decision", + "unknown_severity_decision", + "unknown_destination_decision", + "legacy_runtime_resolver", + "legacy_shared_sre_allowed_types", + "legacy_shared_sre_blocked_types", + "raw_chat_id_allowed", + "shared_sre_rule", + "shared_sre_forbidden", + } +) +_DESTINATION_ROLE_FIELDS = frozenset( + { + "visible_label", + "proven_runtime_alias", + "runtime_alias_evidence", + "chat_alias", + "bot_alias", + "bot_owner", + "monitoring_owner", + "role", + "monitoring_egress_policy", + "runtime_identifier_in_registry", + } +) +_PRODUCT_FIELDS = frozenset( + { + "product_id", + "display_name", + "owner", + "source_audit_status", + "egress_status", + } +) +_ROLLUP_FIELDS = frozenset( + { + "product_count", + "route_count", + "allowed_route_count", + "blocked_route_count", + "disabled_route_count", + "not_implemented_route_count", + "shared_sre_allowed_route_count", + "raw_numeric_destination_count", + "runtime_route_change_count", + "telegram_send_count", + } +) +_EXECUTION_BOUNDARY_FIELDS = frozenset( + { + "telegram_send_allowed", + "bot_api_call_allowed", + "secret_value_read_allowed", + "raw_chat_id_storage_allowed", + "runtime_route_change_allowed", + "production_deploy_allowed", + } +) +_SAFE_SENSITIVE_CONTROL_KEYS = frozenset( + { + "raw_chat_id_allowed", + "raw_chat_id_storage_allowed", + "secret_value_read_allowed", + } +) +_SENSITIVE_KEY_PATTERN = re.compile( + r"(?:^|_)(?:chat_id|bot_token|token|secret|secret_value|authorization(?:_header)?|" + r"webhook_url|api_key|password|credential|private_key|cookie|session)(?:$|_)", + re.IGNORECASE, +) +_RAW_NUMERIC_IDENTIFIER_PATTERN = re.compile(r"^-?\d{8,}$") +_BOT_TOKEN_VALUE_PATTERN = re.compile(r"\d{6,}:[A-Za-z0-9_-]{20,}") +_BOT_API_TOKEN_URL_PATTERN = re.compile(r"api\.telegram\.org/bot[^/\s]+", re.IGNORECASE) def get_routing_rule(notification_type: str) -> RoutingRule: - """根據通知類型回傳路由規則。未知類型預設發群組。""" + """Return a legacy route; unknown types are explicitly blocked.""" return NOTIFICATION_ROUTING.get(notification_type, _DEFAULT_RULE) +def get_legacy_canonical_route_context( + notification_type: str, +) -> dict[str, str] | None: + """Map an allowlisted legacy type to explicit canonical route dimensions.""" + canonical_route = _LEGACY_CANONICAL_ROUTES.get(notification_type) + if canonical_route is None: + return None + product_id, signal_family, severity = canonical_route + return { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + "legacy_notification_type": notification_type, + } + + def resolve_chat_ids( notification_type: str, dm_chat_id: str, group_chat_id: str, *, tg_group_cutover: bool = False, + registry: dict[str, Any] | None = None, ) -> list[str]: - """ - 回傳此通知應發送的 chat_id 清單。 - tg_group_cutover 僅保留為舊 caller 相容參數；正式策略永遠群組優先。 - """ - _ = get_routing_rule(notification_type) + """Resolve a legacy type through the canonical product policy.""" + _ = (dm_chat_id, tg_group_cutover) + rule = get_routing_rule(notification_type) + if rule.destination is not Destination.GROUP: + return [] + + canonical_route = get_legacy_canonical_route_context(notification_type) + if canonical_route is None: + return [] + try: + decision = resolve_canonical_telegram_route( + canonical_route["product_id"], + canonical_route["signal_family"], + canonical_route["severity"], + requested_destination=("telegram_chat_alias:awoooi_sre_war_room"), + registry=registry, + ) + except (FileNotFoundError, TypeError, ValueError, json.JSONDecodeError): + return [] + if decision["decision"] != "allow": + return [] return [group_chat_id] if group_chat_id else [] + + +def load_canonical_telegram_routing_registry( + security_dir: Path | None = None, +) -> dict[str, Any]: + """Load and validate the committed 11-product routing registry.""" + registry_path = (security_dir or _DEFAULT_SECURITY_DIR) / _REGISTRY_FILENAME + if not registry_path.is_file(): + raise FileNotFoundError( + f"canonical Telegram routing registry missing: {registry_path}" + ) + + with registry_path.open(encoding="utf-8") as handle: + payload = json.load(handle) + + if not isinstance(payload, dict): + raise ValueError(f"{registry_path}: expected JSON object") + _validate_canonical_registry(payload, str(registry_path)) + return payload + + +def resolve_canonical_telegram_route( + product_id: str, + signal_family: str, + severity: str, + *, + requested_destination: str | None = None, + registry: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Return one symbolic route decision, defaulting to a no-egress terminal.""" + payload = ( + load_canonical_telegram_routing_registry() if registry is None else registry + ) + _validate_canonical_registry(payload, "in-memory canonical Telegram registry") + + product = next( + ( + product + for product in payload["products"] + if product["product_id"] == product_id + ), + None, + ) + + normalized_severity = severity.strip().upper() + candidates = [ + route + for route in payload["routes"] + if route["product_id"] == product_id + and route["signal_family"] in {signal_family, "*"} + and normalized_severity in route["severity"] + ] + candidates.sort(key=lambda route: route["signal_family"] == "*", reverse=False) + + if not candidates: + return _blocked_decision( + product_id=product_id, + signal_family=signal_family, + severity=normalized_severity, + reason="unknown_product_signal_or_severity", + ) + + route = dict(candidates[0]) + route["decision"] = ( + "allow" + if product is not None + and product["egress_status"] == "allowed" + and route["egress_status"] == "allowed" + else "block" + ) + route["reason"] = route.get("blocked_reason") or "canonical_route_allowed" + + if ( + requested_destination is not None + and requested_destination != route["allowed_destination"] + ): + route.update( + { + "decision": "block", + "egress_status": "blocked", + "reason": "requested_destination_not_allowed", + "requested_destination": requested_destination, + } + ) + return route + + +def _blocked_decision( + *, product_id: str, signal_family: str, severity: str, reason: str +) -> dict[str, Any]: + return { + "route_id": None, + "product_id": product_id, + "signal_family": signal_family, + "severity": [severity], + "scope": "unknown", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "unassigned", + "egress_status": "blocked", + "decision": "block", + "reason": reason, + } + + +def _validate_canonical_registry(payload: dict[str, Any], label: str) -> None: + _require_exact_object_fields(payload, _TOP_LEVEL_FIELDS, label) + if payload.get("schema_version") != _SCHEMA_VERSION: + raise ValueError(f"{label}: expected schema_version={_SCHEMA_VERSION}") + if payload.get("status") != "source_policy_ready_runtime_not_applied": + raise ValueError(f"{label}: unsupported status") + if payload.get("mode") != "source_only_no_secret_no_telegram_send_no_route_change": + raise ValueError(f"{label}: unsupported mode") + if not isinstance(payload.get("generated_at"), str) or not payload["generated_at"]: + raise ValueError(f"{label}: generated_at must be a non-empty string") + + _validate_no_sensitive_runtime_material(payload, label) + + supersedes = payload["supersedes"] + _require_exact_object_fields( + supersedes, + _SUPERSEDES_FIELDS, + f"{label}.supersedes", + ) + for field in _SUPERSEDES_FIELDS: + if not isinstance(supersedes[field], str) or not supersedes[field].strip(): + raise ValueError(f"{label}.supersedes.{field} must be non-empty") + if supersedes["superseded_by"] != "global_product_governance_v2": + raise ValueError(f"{label}: superseded_by must be global governance v2") + + policy = payload["policy"] + _require_exact_object_fields(policy, _POLICY_FIELDS, f"{label}.policy") + for field in ( + "default_decision", + "unknown_product_decision", + "unknown_signal_family_decision", + "unknown_severity_decision", + "unknown_destination_decision", + ): + if policy[field] != "block": + raise ValueError(f"{label}.policy.{field} must be block") + if policy["raw_chat_id_allowed"] is not False: + raise ValueError(f"{label}.policy.raw_chat_id_allowed must be false") + expected_legacy_allowed = set(_LEGACY_CANONICAL_ROUTES) + legacy_allowed = policy["legacy_shared_sre_allowed_types"] + legacy_blocked = policy["legacy_shared_sre_blocked_types"] + shared_forbidden = policy["shared_sre_forbidden"] + if ( + not isinstance(legacy_allowed, list) + or len(legacy_allowed) != len(set(legacy_allowed)) + or set(legacy_allowed) != expected_legacy_allowed + ): + raise ValueError(f"{label}: legacy allowed types drifted") + if ( + not isinstance(legacy_blocked, list) + or len(legacy_blocked) != len(set(legacy_blocked)) + or set(legacy_blocked) != {"TYPE-1", "TYPE-6B", "UNKNOWN"} + ): + raise ValueError(f"{label}: legacy blocked types drifted") + if not isinstance(shared_forbidden, list) or not shared_forbidden or any( + not isinstance(item, str) or not item.strip() for item in shared_forbidden + ): + raise ValueError(f"{label}: shared_sre_forbidden must be non-empty strings") + for field in ("legacy_runtime_resolver", "shared_sre_rule"): + if not isinstance(policy[field], str) or not policy[field].strip(): + raise ValueError(f"{label}.policy.{field} must be non-empty") + + boundaries = payload["execution_boundaries"] + _require_exact_object_fields( + boundaries, + _EXECUTION_BOUNDARY_FIELDS, + f"{label}.execution_boundaries", + ) + unsafe = sorted( + key for key in _EXECUTION_BOUNDARY_FIELDS if boundaries[key] is not False + ) + if unsafe: + raise ValueError(f"{label}: execution boundaries must remain false: {unsafe}") + + destination_roles = payload["destination_roles"] + if not isinstance(destination_roles, list) or not destination_roles: + raise ValueError(f"{label}: destination_roles must be a non-empty array") + observed_role_bindings: set[tuple[str, str]] = set() + for index, role in enumerate(destination_roles): + role_label = f"{label}.destination_roles[{index}]" + _require_exact_object_fields(role, _DESTINATION_ROLE_FIELDS, role_label) + for field in ( + "visible_label", + "proven_runtime_alias", + "runtime_alias_evidence", + "chat_alias", + "bot_alias", + "bot_owner", + "role", + "monitoring_egress_policy", + ): + if not isinstance(role[field], str) or not role[field].strip(): + raise ValueError(f"{role_label}.{field} must be non-empty") + if role["runtime_identifier_in_registry"] is not False: + raise ValueError( + f"{role_label}.runtime_identifier_in_registry must be false" + ) + if not isinstance(role["monitoring_owner"], bool): + raise ValueError(f"{role_label}.monitoring_owner must be boolean") + if role["runtime_alias_evidence"] == "visible_label_only": + if role["proven_runtime_alias"] != "none": + raise ValueError( + f"{role_label}: visible label must not claim a proven runtime alias" + ) + elif role["runtime_alias_evidence"] == "proven_runtime_receipt": + if role["proven_runtime_alias"] == "none": + raise ValueError( + f"{role_label}: proven runtime receipt requires an alias" + ) + elif role["runtime_alias_evidence"] not in { + "source_config_only", + "configured_unreachable", + }: + raise ValueError(f"{role_label}: unsupported runtime_alias_evidence") + for field in ("chat_alias", "bot_alias"): + if _RAW_NUMERIC_IDENTIFIER_PATTERN.fullmatch(role[field]): + raise ValueError(f"{role_label}: raw numeric runtime identifier") + binding = (role["chat_alias"], role["bot_alias"]) + if binding in observed_role_bindings: + raise ValueError(f"{role_label}: duplicate symbolic destination binding") + observed_role_bindings.add(binding) + + products = payload["products"] + routes = payload["routes"] + if not isinstance(products, list) or not isinstance(routes, list): + raise ValueError(f"{label}: products and routes must be arrays") + + for index, product in enumerate(products): + product_label = f"{label}.products[{index}]" + _require_exact_object_fields(product, _PRODUCT_FIELDS, product_label) + for field in ("product_id", "display_name", "owner", "source_audit_status"): + if not isinstance(product[field], str) or not product[field].strip(): + raise ValueError(f"{product_label}.{field} must be non-empty") + if product["egress_status"] not in _ALLOWED_EGRESS_STATUSES: + raise ValueError(f"{product_label}: unsupported product egress_status") + product_ids = [product["product_id"] for product in products] + if len(product_ids) != len(set(product_ids)): + raise ValueError(f"{label}: duplicate product_id") + if set(product_ids) != _EXPECTED_PRODUCT_IDS: + raise ValueError( + f"{label}: product registry must cover the canonical 11 products" + ) + + route_ids: set[str] = set() + covered_products: set[str] = set() + for index, route in enumerate(routes): + route_label = f"{label}.routes[{index}]" + _require_exact_object_fields( + route, + _REQUIRED_ROUTE_FIELDS, + route_label, + optional_fields=frozenset({"blocked_reason"}), + ) + + route_id = str(route["route_id"]) + if not route_id or route_id in route_ids: + raise ValueError( + f"{label}: route_id must be non-empty and unique: {route_id!r}" + ) + route_ids.add(route_id) + + product_id = str(route["product_id"]) + if product_id not in _EXPECTED_PRODUCT_IDS: + raise ValueError(f"{label}: unknown product_id in route: {product_id}") + covered_products.add(product_id) + + severity = route["severity"] + if ( + not isinstance(severity, list) + or not severity + or len(severity) != len(set(severity)) + ): + raise ValueError(f"{label}: {route_id} severity must be a non-empty array") + if not set(severity).issubset(_ALLOWED_SEVERITIES): + raise ValueError(f"{label}: {route_id} has unsupported severity") + + egress_status = str(route["egress_status"]) + if egress_status not in _ALLOWED_EGRESS_STATUSES: + raise ValueError(f"{label}: {route_id} has unsupported egress_status") + if route["scope"] not in {"shared", "product"}: + raise ValueError(f"{label}: {route_id} has unsupported scope") + for field in ( + "route_id", + "product_id", + "signal_family", + "scope", + "bot_alias", + "chat_alias", + "allowed_destination", + "receipt_backend", + "owner", + "egress_status", + ): + if not isinstance(route[field], str) or not route[field].strip(): + raise ValueError(f"{route_label}.{field} must be non-empty") + + _validate_no_runtime_identifiers(route, label, route_id) + _validate_route_destination(route, label, route_id) + _validate_shared_sre_boundary(route, label, route_id) + + if covered_products != _EXPECTED_PRODUCT_IDS: + raise ValueError( + f"{label}: every canonical product requires at least one route" + ) + + by_product = {product["product_id"]: product for product in products} + for product_id, product in by_product.items(): + expected_status = product.get("egress_status") + if expected_status not in _ALLOWED_EGRESS_STATUSES: + raise ValueError( + f"{label}: {product_id} has unsupported product egress_status" + ) + product_routes = [ + route for route in routes if route["product_id"] == product_id + ] + allowed_route_count = sum( + route["egress_status"] == "allowed" for route in product_routes + ) + if expected_status != "allowed" and allowed_route_count: + raise ValueError(f"{label}: {product_id} must not have an allowed route") + if expected_status == "allowed" and allowed_route_count == 0: + raise ValueError( + f"{label}: {product_id} requires at least one allowed route" + ) + if expected_status in {"disabled", "not_implemented"} and any( + route["egress_status"] != expected_status for route in product_routes + ): + raise ValueError( + f"{label}: {product_id} route status must match product disposition" + ) + + # Ownership is a cross-object invariant. Validate it after product-level + # disposition so a disabled product cannot be made allowed merely by also + # forging a destination-role binding. + for route in routes: + _validate_route_role_ownership( + route, + destination_roles, + label, + str(route["route_id"]), + ) + + _validate_registry_rollups(payload, label) + + +def _require_exact_object_fields( + value: object, + required_fields: frozenset[str], + label: str, + *, + optional_fields: frozenset[str] = frozenset(), +) -> None: + if not isinstance(value, dict): + raise ValueError(f"{label}: expected JSON object") + missing = sorted(required_fields - value.keys()) + unexpected = sorted(value.keys() - required_fields - optional_fields) + if missing: + raise ValueError(f"{label}: missing required fields: {missing}") + if unexpected: + raise ValueError(f"{label}: unexpected fields: {unexpected}") + + +def _validate_no_runtime_identifiers( + route: dict[str, Any], label: str, route_id: str +) -> None: + forbidden_fields = {"chat_id", "bot_token", "token", "secret", "webhook_url"} + present = sorted(forbidden_fields & route.keys()) + if present: + raise ValueError( + f"{label}: {route_id} contains forbidden runtime fields: {present}" + ) + + for field in ("bot_alias", "chat_alias", "allowed_destination"): + value = str(route[field]) + if value.lstrip("-").isdigit(): + raise ValueError( + f"{label}: {route_id} must not contain a raw Telegram identifier" + ) + + +def _validate_no_sensitive_runtime_material(payload: Any, label: str) -> None: + """Reject secret-shaped keys/values and raw numeric identifiers recursively.""" + + def walk(value: Any, path: str) -> None: + if isinstance(value, dict): + for raw_key, nested in value.items(): + key = str(raw_key) + child_path = f"{path}.{key}" + if _SENSITIVE_KEY_PATTERN.search(key): + safe_control = ( + key in _SAFE_SENSITIVE_CONTROL_KEYS and nested is False + ) + if not safe_control: + raise ValueError( + f"{label}: forbidden sensitive runtime key at {child_path}" + ) + walk(nested, child_path) + return + if isinstance(value, list): + for index, nested in enumerate(value): + walk(nested, f"{path}[{index}]") + return + if isinstance(value, bool) or value is None: + return + if isinstance(value, int): + if abs(value) >= 1_000_000_000: + raise ValueError(f"{label}: raw numeric runtime identifier at {path}") + return + if not isinstance(value, str): + return + if _RAW_NUMERIC_IDENTIFIER_PATTERN.fullmatch(value): + raise ValueError(f"{label}: raw numeric runtime identifier at {path}") + if _BOT_TOKEN_VALUE_PATTERN.search(value) or _BOT_API_TOKEN_URL_PATTERN.search( + value + ): + raise ValueError(f"{label}: secret-shaped runtime value at {path}") + + walk(payload, "registry") + + +def _validate_route_destination( + route: dict[str, Any], label: str, route_id: str +) -> None: + status = route["egress_status"] + destination = route["allowed_destination"] + if status == "allowed": + expected = f"telegram_chat_alias:{route['chat_alias']}" + if destination != expected: + raise ValueError( + f"{label}: {route_id} allowed destination must match chat_alias" + ) + if route["bot_alias"] == "none" or route["chat_alias"] == "none": + raise ValueError( + f"{label}: {route_id} allowed route requires symbolic aliases" + ) + if route["receipt_backend"] == "none": + raise ValueError( + f"{label}: {route_id} allowed route requires a receipt backend" + ) + else: + if destination != "blocked": + raise ValueError(f"{label}: {route_id} non-allowed route must be blocked") + if not route.get("blocked_reason"): + raise ValueError( + f"{label}: {route_id} non-allowed route requires blocked_reason" + ) + + +def _validate_shared_sre_boundary( + route: dict[str, Any], label: str, route_id: str +) -> None: + if route["chat_alias"] != _SHARED_SRE_CHAT_ALIAS: + return + invalid = ( + route["product_id"] != "awoooi" + or route["scope"] != "shared" + or route["signal_family"] not in _SHARED_SRE_SIGNAL_FAMILIES + or not set(route["severity"]).issubset({"P0", "P1"}) + or route["bot_alias"] != _SHARED_MONITORING_BOT_ALIAS + ) + if invalid: + raise ValueError( + f"{label}: {route_id} violates shared SRE P0/P1 infra/security/DR/lifecycle boundary" + ) + + +def _validate_route_role_ownership( + route: dict[str, Any], + destination_roles: list[dict[str, Any]], + label: str, + route_id: str, +) -> None: + """Cross-check every allowed route against its declared destination owner.""" + if route["egress_status"] != "allowed": + return + role = next( + ( + item + for item in destination_roles + if item["chat_alias"] == route["chat_alias"] + and item["bot_alias"] == route["bot_alias"] + ), + None, + ) + if role is None: + raise ValueError( + f"{label}: {route_id} has no matching destination_roles ownership binding" + ) + if role["monitoring_egress_policy"] != "allowlist_only": + raise ValueError( + f"{label}: {route_id} destination role is not monitoring allowlisted" + ) + if role["monitoring_owner"] is not True: + raise ValueError( + f"{label}: {route_id} destination role is not the monitoring owner" + ) + if role["bot_owner"] != _SHARED_MONITORING_BOT_OWNER: + raise ValueError( + f"{label}: {route_id} bot ownership does not match shared monitoring policy" + ) + + +def _validate_registry_rollups(payload: dict[str, Any], label: str) -> None: + rollups = payload["rollups"] + _require_exact_object_fields( + rollups, + _ROLLUP_FIELDS, + f"{label}.rollups", + ) + routes = payload["routes"] + expected = { + "product_count": len(payload["products"]), + "route_count": len(routes), + "allowed_route_count": sum( + route["egress_status"] == "allowed" for route in routes + ), + "blocked_route_count": sum( + route["egress_status"] == "blocked" for route in routes + ), + "disabled_route_count": sum( + route["egress_status"] == "disabled" for route in routes + ), + "not_implemented_route_count": sum( + route["egress_status"] == "not_implemented" for route in routes + ), + "shared_sre_allowed_route_count": sum( + route["egress_status"] == "allowed" + and route["chat_alias"] == _SHARED_SRE_CHAT_ALIAS + for route in routes + ), + "raw_numeric_destination_count": 0, + "runtime_route_change_count": 0, + "telegram_send_count": 0, + } + drifted = sorted( + key for key, value in expected.items() if rollups.get(key) != value + ) + if drifted: + raise ValueError(f"{label}: registry rollup mismatch: {drifted}") diff --git a/apps/api/src/services/notifications/telegram.py b/apps/api/src/services/notifications/telegram.py index 4a21673ab..5eb7bb94b 100644 --- a/apps/api/src/services/notifications/telegram.py +++ b/apps/api/src/services/notifications/telegram.py @@ -9,8 +9,8 @@ from src.core.config import settings from src.core.logging import get_logger + from .base import ( - ExecutionStatus, NotificationMessage, NotificationProvider, NotificationResult, @@ -60,16 +60,54 @@ class TelegramWebhookProvider(NotificationProvider): message="Telegram bot token or chat_id not configured", ) try: - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) gateway = get_telegram_gateway() text = self._format(message) - resp = await gateway.send_alert_notification(text=text, parse_mode="HTML") + resp = await gateway.send_alert_notification( + text=text, + parse_mode="HTML", + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) + response_data = resp if isinstance(resp, dict) else None + if not _telegram_send_delivery_succeeded(resp): + delivery_status = ( + str(resp.get("_awooop_delivery_status") or "") + if isinstance(resp, dict) + else "" + ) + provider_send_performed = ( + resp.get("_awooop_provider_send_performed") + if isinstance(resp, dict) + else None + ) + if ( + delivery_status == "blocked_no_egress" + or provider_send_performed is False + ): + return NotificationResult( + status=NotificationStatus.SKIPPED, + provider=self.name, + message="Telegram notification blocked before provider send", + response_data=response_data, + ) + return NotificationResult( + status=NotificationStatus.FAILED, + provider=self.name, + message="Telegram provider did not acknowledge delivery", + response_data=response_data, + error="delivery_not_acknowledged", + ) return NotificationResult( status=NotificationStatus.SUCCESS, provider=self.name, message="Telegram notification sent", - response_data=resp if isinstance(resp, dict) else None, + response_data=response_data, ) except Exception as e: logger.exception("telegram_notification_exception", error=str(e)) @@ -88,8 +126,13 @@ class TelegramWebhookProvider(NotificationProvider): from src.services.telegram_gateway import get_telegram_gateway gw = get_telegram_gateway() - await gw.send_alert_notification(text="🔔 AWOOOI Telegram provider 連線測試") - return True + result = await gw.send_alert_notification( + text="AWOOOI Telegram provider no-egress policy probe", + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ) + return bool(result.get("ok")) except Exception as e: logger.error("telegram_connection_test_failed", error=str(e)) return False diff --git a/apps/api/src/services/post_execution_verifier.py b/apps/api/src/services/post_execution_verifier.py index 6c842e97b..e44e24e91 100644 --- a/apps/api/src/services/post_execution_verifier.py +++ b/apps/api/src/services/post_execution_verifier.py @@ -410,9 +410,10 @@ async def _send_rollback_proposal_alert( 不自動執行 rollback，僅通知人工評估。 """ try: - from src.core.config import get_settings - from src.services.telegram_gateway import get_telegram_gateway - _settings = get_settings() + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) gateway = get_telegram_gateway() regressions = assessment.get("regressions", []) @@ -429,20 +430,25 @@ async def _send_rollback_proposal_alert( f"此為提案，不會自動執行 Rollback" ) - target_chat_id = _settings.SRE_GROUP_CHAT_ID - await gateway._send_request( - "sendMessage", - { - "chat_id": target_chat_id, - "text": text, - "parse_mode": "HTML", - }, - ) - logger.info( - "rollback_proposal_sent", - incident_id=incident_id, - score=score, + delivery = await gateway.send_canonical_message( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + text=text, + parse_mode="HTML", ) + if _telegram_send_delivery_succeeded(delivery): + logger.info( + "rollback_proposal_sent", + incident_id=incident_id, + score=score, + ) + else: + logger.warning( + "rollback_proposal_blocked_no_send", + incident_id=incident_id, + score=score, + ) except Exception: logger.warning( "rollback_proposal_send_failed", diff --git a/apps/api/src/services/report_generation_service.py b/apps/api/src/services/report_generation_service.py index 4e2b1f5a9..a87c46cec 100644 --- a/apps/api/src/services/report_generation_service.py +++ b/apps/api/src/services/report_generation_service.py @@ -569,38 +569,114 @@ class ReportGenerationService: Graceful Degradation: 失敗只記錄 log，不拋出例外 """ try: + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) + + gateway = get_telegram_gateway() + if not gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ): + logger.warning( + "daily_report_no_send", + reason="canonical_route_unavailable", + ) + return + kpi = await self.collect_daily_kpi() source_health = await self.collect_report_source_health(days=1) report_text = self.format_daily_report(kpi, source_health) - from src.services.telegram_gateway import get_telegram_gateway - gateway = get_telegram_gateway() - await gateway.send_to_group(report_text, parse_mode="HTML") - - logger.info( - "daily_report_sent", - total_alerts=kpi.total_alerts, - auto_repair_rate=f"{kpi.auto_repair_rate:.1%}", + result = await gateway.send_to_group( + report_text, + parse_mode="HTML", + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", ) + + if _telegram_send_delivery_succeeded(result): + logger.info( + "daily_report_sent", + total_alerts=kpi.total_alerts, + delivery_status=result.get("_awooop_delivery_status", "sent"), + auto_repair_rate=f"{kpi.auto_repair_rate:.1%}", + ) + else: + logger.warning( + "daily_report_no_send", + total_alerts=kpi.total_alerts, + delivery_status=( + result.get( + "_awooop_delivery_status", + "missing_provider_ack", + ) + if isinstance(result, dict) + else "missing_provider_ack" + ), + ) except Exception as e: logger.error("daily_report_failed", error=str(e)) async def send_monthly_report(self) -> bool: """收集月報資料 → 組裝 → 推送 Telegram SRE 群組。""" try: + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) + + gateway = get_telegram_gateway() + if not gateway.canonical_destination_chat_id( + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ): + logger.warning( + "monthly_report_no_send", + reason="canonical_route_unavailable", + ) + return False + source_health = await self.collect_report_source_health(days=30) report_text = self.format_monthly_report(source_health) - from src.services.telegram_gateway import get_telegram_gateway - gateway = get_telegram_gateway() - await gateway.send_to_group(report_text, parse_mode="HTML") - - logger.info( - "monthly_report_sent", - source_ok=(source_health.get("rollups") or {}).get("source_ok_count"), - source_total=(source_health.get("rollups") or {}).get("source_count"), + result = await gateway.send_to_group( + report_text, + parse_mode="HTML", + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", ) - return True + + if _telegram_send_delivery_succeeded(result): + logger.info( + "monthly_report_sent", + source_ok=(source_health.get("rollups") or {}).get( + "source_ok_count" + ), + delivery_status=result.get("_awooop_delivery_status", "sent"), + source_total=(source_health.get("rollups") or {}).get( + "source_count" + ), + ) + return True + + logger.warning( + "monthly_report_no_send", + delivery_status=( + result.get( + "_awooop_delivery_status", + "missing_provider_ack", + ) + if isinstance(result, dict) + else "missing_provider_ack" + ), + ) + return False except Exception as e: logger.error("monthly_report_failed", error=str(e)) return False @@ -663,7 +739,10 @@ class ReportGenerationService: import asyncio as _asyncio report_text = self.format_postmortem(data) await self._persist_postmortem_km(data, report_text) - from src.services.telegram_gateway import get_telegram_gateway + from src.services.telegram_gateway import ( + _telegram_send_delivery_succeeded, + get_telegram_gateway, + ) gateway = get_telegram_gateway() max_attempts = 3 @@ -671,7 +750,22 @@ class ReportGenerationService: last_error: Exception | None = None for attempt in range(1, max_attempts + 1): try: - await gateway.send_to_group(report_text, parse_mode="HTML") + result = await gateway.send_to_group( + report_text, + parse_mode="HTML", + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) + if not _telegram_send_delivery_succeeded(result): + delivery_status = ( + result.get("_awooop_delivery_status") + if isinstance(result, dict) + else None + ) + raise RuntimeError( + str(delivery_status or "missing_provider_ack") + ) logger.info( "postmortem_sent", incident_id=incident_id, @@ -705,7 +799,23 @@ class ReportGenerationService: f"Duration: {duration_minutes:.1f} 分鐘\n" f"Error: {str(last_error)[:200]}" ) - await gateway.send_to_group(fallback_text, parse_mode="HTML") + fallback_result = await gateway.send_to_group( + fallback_text, + parse_mode="HTML", + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) + if not _telegram_send_delivery_succeeded(fallback_result): + logger.error( + "postmortem_fallback_no_send", + incident_id=incident_id, + delivery_status=( + fallback_result.get("_awooop_delivery_status") + if isinstance(fallback_result, dict) + else "missing_provider_ack" + ), + ) except Exception as _fe: logger.error( "postmortem_fallback_failed", diff --git a/apps/api/src/services/runbook_generator.py b/apps/api/src/services/runbook_generator.py index 565b20536..5b1d47ed1 100644 --- a/apps/api/src/services/runbook_generator.py +++ b/apps/api/src/services/runbook_generator.py @@ -389,7 +389,12 @@ class NemotronRunbookGenerator: try: from src.services.telegram_gateway import get_telegram_gateway tg = get_telegram_gateway() - await tg.send_text(format_runbook_review_card(incident, entry_id, content_preview)) + await tg.send_text( + format_runbook_review_card(incident, entry_id, content_preview), + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + ) except Exception as e: logger.warning("runbook_review_card_failed", error=str(e)) @@ -406,7 +411,10 @@ class NemotronRunbookGenerator: f"⚠️ 已記錄失敗案例\n" f"Incident: {incident.incident_id}\n" f"標題: {title}\n\n" - f"相同症狀的後續告警將阻斷自動修復，要求人工介入。" + f"相同症狀的後續告警將阻斷自動修復，要求人工介入。", + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", ) except Exception as e: logger.warning("anti_pattern_notification_failed", error=str(e)) diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 9b3dd07f8..00e382f00 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -28,6 +28,7 @@ import html import json import os import re +from collections.abc import Mapping from dataclasses import dataclass, field from datetime import UTC, datetime from uuid import NAMESPACE_URL, UUID, uuid5 @@ -60,6 +61,10 @@ from src.services.backup_restore_signal_automation import ( build_backup_restore_signal_automation_contract, ) from src.services.chat_manager import get_chat_manager +from src.services.notification_matrix import ( + get_legacy_canonical_route_context, + resolve_canonical_telegram_route, +) from src.services.operator_outcome import ( build_operator_outcome, normalize_operator_outcome, @@ -138,6 +143,17 @@ _AI_ADVISORY_CALLBACK_RE = re.compile( _CODE_REF_RE = re.compile(r"([0-9a-f]{7,12})", re.IGNORECASE) _TELEGRAM_HTML_CHUNK_LIMIT = 3600 _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY = "_awooop_source_envelope_extra" +_CANONICAL_ROUTE_CONTEXT_KEY = "_awoooi_canonical_telegram_route" +_LEGACY_NOTIFICATION_TYPE_KEY = "_awoooi_legacy_notification_type" +_NON_MONITORING_INTERACTION_KEY = "_awoooi_non_monitoring_interaction" +_CANONICAL_SRE_DESTINATION = "telegram_chat_alias:awoooi_sre_war_room" +_NON_MONITORING_INTERACTION_KINDS = frozenset( + { + "operator_command_reply", + "callback_reply", + "bot_discussion_reply", + } +) _CONTROLLED_APPLY_OUTBOUND_FINALIZE_ATTEMPTS = 3 _NO_WRITE_REPLAY_DIGEST_WINDOW_MINUTES = 30 _HOST_RESOURCE_ALERT_HEADER_RE = re.compile( @@ -825,6 +841,32 @@ def normalize_telegram_send_message_payload(method: str, payload: dict) -> dict: return normalized_payload +def _telegram_send_delivery_succeeded(result: object) -> bool: + """Require affirmative provider delivery evidence from ``_send_request``. + + Final-exit policy blocks are returned as structured receipts rather than + exceptions. Callers that implement bounded retry must therefore reject an + explicit no-send flag or any non-terminal delivery status even when + ``ok`` was accidentally left truthy by an adapter. + """ + if not isinstance(result, Mapping) or result.get("ok") is not True: + return False + + delivery_status = str(result.get("_awooop_delivery_status") or "").strip() + if delivery_status and delivery_status not in {"sent", "sent_reused"}: + return False + if result.get("_awooop_provider_send_performed") is False: + return False + + route_receipt = result.get("_awoooi_canonical_route_receipt") + if ( + isinstance(route_receipt, Mapping) + and route_receipt.get("provider_send_performed") is False + ): + return False + return True + + def _top_gateway_bucket( buckets: list[dict[str, object]], field: str, @@ -2777,6 +2819,7 @@ def _callback_reply_source_envelope_extra( callback_action: str | None = None, parse_mode: str | None = None, error: str | None = None, + parent_message_id: int | None = None, km_stale_completion_summary: dict[str, object] | None = None, awooop_status_chain: dict[str, object] | None = None, ) -> dict[str, object] | None: @@ -2798,6 +2841,8 @@ def _callback_reply_source_envelope_extra( callback_reply["parse_mode"] = parse_mode if error: callback_reply["error"] = _sanitize_telegram_error(error)[:300] + if parent_message_id is not None and parent_message_id > 0: + callback_reply["parent_provider_message_id"] = str(parent_message_id) extra: dict[str, object] = { "callback_reply": callback_reply, @@ -2805,6 +2850,10 @@ def _callback_reply_source_envelope_extra( "incident_ids": [incident_id], }, } + if parent_message_id is not None and parent_message_id > 0: + extra["source_refs"]["parent_provider_message_ids"] = [ + str(parent_message_id) + ] km_snapshot = _callback_reply_km_stale_completion_snapshot( km_stale_completion_summary, ) @@ -2816,6 +2865,42 @@ def _callback_reply_source_envelope_extra( return extra +def _incident_lifecycle_parent_source_envelope_extra( + *, + incident_id: str | None = None, + approval_id: str | None = None, + parent_lookup_key: str, + parent_message_id: int, + event_kind: str, +) -> dict[str, object]: + """Persist the canonical lifecycle thread parent in the outbound mirror. + + Lifecycle updates are autonomous monitoring events, not replies to a newly + verified inbound operator interaction. Their Redis-backed parent lookup is + therefore recorded in the durable Channel Hub source envelope while the + send itself remains governed by the canonical incident lifecycle route. + """ + parent_id = str(parent_message_id) + correlation_id = str(incident_id or approval_id or "") + correlation_kind = "incident" if incident_id else "approval" + source_ref_key = "incident_ids" if incident_id else "approval_ids" + return { + "incident_lifecycle_parent_receipt": { + "schema_version": "telegram_incident_lifecycle_parent_receipt_v1", + "correlation_id": correlation_id, + "correlation_kind": correlation_kind, + "event_kind": event_kind, + "parent_provider_message_id": parent_id, + "parent_lookup_key": parent_lookup_key, + "durable_mirror_required": True, + }, + "source_refs": { + source_ref_key: [correlation_id], + "parent_provider_message_ids": [parent_id], + }, + } + + def _alert_notification_source_envelope_extra( *, safe_text: str, @@ -3369,6 +3454,14 @@ def _merge_outbound_source_envelope_extra( if isinstance(notification_policy, dict): envelope["notification_policy"] = notification_policy + incident_lifecycle_parent_receipt = extra.get( + "incident_lifecycle_parent_receipt" + ) + if isinstance(incident_lifecycle_parent_receipt, dict): + envelope["incident_lifecycle_parent_receipt"] = ( + incident_lifecycle_parent_receipt + ) + extra_refs = extra.get("source_refs") if isinstance(extra_refs, dict): source_refs = envelope.setdefault("source_refs", {}) @@ -3385,6 +3478,41 @@ def _merge_outbound_source_envelope_extra( return envelope + +def _acknowledge_callback_reply_source_envelope( + source_envelope_extra: dict[str, object] | None, + *, + delivery_result: object, + provider_message_id: str, +) -> dict[str, object] | None: + """Promote callback attempt metadata only after affirmative provider ack.""" + if ( + not isinstance(source_envelope_extra, dict) + or not provider_message_id + or not _telegram_send_delivery_succeeded(delivery_result) + ): + return source_envelope_extra + + callback_reply = source_envelope_extra.get("callback_reply") + if not isinstance(callback_reply, dict): + return source_envelope_extra + + terminal_status = { + "callback_reply_attempt": "callback_reply_sent", + "callback_reply_fallback_attempt": "callback_reply_fallback_sent", + "callback_reply_rescue_attempt": "callback_reply_rescue_sent", + }.get(str(callback_reply.get("status") or "")) + if terminal_status is None: + return source_envelope_extra + + acknowledged_extra = dict(source_envelope_extra) + acknowledged_callback = dict(callback_reply) + acknowledged_callback["status"] = terminal_status + acknowledged_callback["provider_delivery_acknowledged"] = True + acknowledged_callback["provider_message_id"] = provider_message_id + acknowledged_extra["callback_reply"] = acknowledged_callback + return acknowledged_extra + # 2026-04-27 Claude Sonnet 4.6: B3 — LLM 動態 Telegram 按鈕 Feature Flag # true → 優先使用 ActionPlan.recommended_actions 動態生成按鈕 # false → 維持現有 callback_action_spec.yaml 路徑（預設，向下相容） @@ -5083,6 +5211,12 @@ class TelegramGateway: self._last_message_time: datetime | None = None self._heartbeat_task: asyncio.Task | None = None self._heartbeat_active = False + # Process-local mirrors of durable, provider-acknowledged dedup receipts. + # They avoid a second Redis read on repeated calls while preserving the + # rule that a blocked/no-egress result must never consume a dedup slot. + self._delivered_incident_update_dedup: set[str] = set() + self._delivered_global_failure_dedup: set[str] = set() + self._delivered_grouped_digest_dedup: set[str] = set() async def initialize(self) -> bool: """初始化 Gateway""" @@ -5115,14 +5249,223 @@ class TelegramGateway: @property def chat_id(self) -> str: - """取得正式產品告警 Chat ID。""" + """Runtime binding retained for edit/delete and inbound reply continuity.""" return settings.SRE_GROUP_CHAT_ID @property def alert_chat_id(self) -> str: - """告警訊息收件人：正式產品告警只送 AwoooI SRE 戰情室。""" + """Runtime binding; new monitoring sends are still policy-gated.""" return settings.SRE_GROUP_CHAT_ID + @staticmethod + def _blocked_route_receipt( + classifier: str, + *, + route_context: Mapping[str, object] | None = None, + ) -> dict[str, object]: + context = route_context or {} + return { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "blocked_no_egress", + "classifier": classifier, + "product_id": str(context.get("product_id") or "unknown")[:80], + "signal_family": str(context.get("signal_family") or "unknown")[:80], + "severity": str(context.get("severity") or "unknown")[:16], + "destination_alias": "none", + "provider_send_performed": False, + } + + def _authorize_canonical_send_message( + self, + *, + payload: Mapping[str, object], + route_context: object, + legacy_notification_type: object, + actual_bot_alias: str, + ) -> tuple[str | None, dict[str, object]]: + context: Mapping[str, object] | None = ( + route_context if isinstance(route_context, Mapping) else None + ) + legacy_type = str(legacy_notification_type or "").strip() + if legacy_type: + legacy_context = get_legacy_canonical_route_context(legacy_type) + if legacy_context is None: + return None, self._blocked_route_receipt( + "legacy_notification_type_not_allowed", + route_context={"signal_family": legacy_type}, + ) + context = legacy_context + + if context is None: + return None, self._blocked_route_receipt( + "missing_product_signal_severity_context" + ) + + product_id = str(context.get("product_id") or "").strip() + signal_family = str(context.get("signal_family") or "").strip() + severity = str(context.get("severity") or "").strip().upper() + normalized_context = { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + } + if not product_id or not signal_family or not severity: + return None, self._blocked_route_receipt( + "incomplete_product_signal_severity_context", + route_context=normalized_context, + ) + + try: + decision = resolve_canonical_telegram_route( + product_id, + signal_family, + severity, + requested_destination=_CANONICAL_SRE_DESTINATION, + ) + except (FileNotFoundError, TypeError, ValueError, json.JSONDecodeError): + return None, self._blocked_route_receipt( + "canonical_registry_invalid_or_missing", + route_context=normalized_context, + ) + + if decision.get("decision") != "allow": + return None, self._blocked_route_receipt( + f"canonical_policy_blocked:{decision.get('reason') or 'unknown'}", + route_context=normalized_context, + ) + if decision.get("allowed_destination") != _CANONICAL_SRE_DESTINATION: + return None, self._blocked_route_receipt( + "canonical_destination_not_supported", + route_context=normalized_context, + ) + if decision.get("bot_alias") != actual_bot_alias: + return None, self._blocked_route_receipt( + "sender_bot_alias_not_authorized_for_route", + route_context=normalized_context, + ) + + canonical_chat_id = str(settings.SRE_GROUP_CHAT_ID or "") + if not canonical_chat_id: + return None, self._blocked_route_receipt( + "canonical_destination_unconfigured", + route_context=normalized_context, + ) + caller_chat_id = str(payload.get("chat_id") or "") + if caller_chat_id and caller_chat_id != canonical_chat_id: + return None, self._blocked_route_receipt( + "caller_destination_mismatch", + route_context=normalized_context, + ) + + return canonical_chat_id, { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "classifier": "canonical_shared_route_allowed", + **normalized_context, + "destination_alias": "awoooi_sre_war_room", + "sender_bot_alias": actual_bot_alias, + "provider_send_performed": False, + } + + def _authorize_non_monitoring_interaction( + self, + *, + payload: Mapping[str, object], + interaction_context: object, + actual_bot_alias: str, + ) -> tuple[str | None, dict[str, object]]: + context = ( + interaction_context + if isinstance(interaction_context, Mapping) + else None + ) + interaction_kind = str( + (context or {}).get("interaction_kind") or "" + ).strip() + if interaction_kind not in _NON_MONITORING_INTERACTION_KINDS: + return None, self._blocked_route_receipt( + "unknown_non_monitoring_interaction_kind", + route_context={"signal_family": interaction_kind or "unknown"}, + ) + + target_chat_id = str(payload.get("chat_id") or "") + if not target_chat_id: + return None, self._blocked_route_receipt( + "interaction_reply_target_missing", + route_context={"signal_family": interaction_kind}, + ) + + inbound_chat_id = str((context or {}).get("inbound_chat_id") or "") + inbound_message_raw = (context or {}).get("inbound_message_id") + try: + inbound_message_id = int(str(inbound_message_raw)) + except (TypeError, ValueError): + inbound_message_id = 0 + if not inbound_chat_id or inbound_message_id <= 0: + return None, self._blocked_route_receipt( + "interaction_inbound_context_missing", + route_context={"signal_family": interaction_kind}, + ) + if inbound_chat_id != target_chat_id: + return None, self._blocked_route_receipt( + "interaction_inbound_chat_mismatch", + route_context={"signal_family": interaction_kind}, + ) + + reply_parameters = payload.get("reply_parameters") + reply_message_raw = payload.get("reply_to_message_id") + if isinstance(reply_parameters, Mapping): + reply_message_raw = reply_parameters.get("message_id") + try: + reply_message_id = int(str(reply_message_raw)) + except (TypeError, ValueError): + reply_message_id = 0 + if reply_message_id != inbound_message_id: + return None, self._blocked_route_receipt( + "interaction_reply_context_missing_or_mismatched", + route_context={"signal_family": interaction_kind}, + ) + if actual_bot_alias in {"openclaw_bot", "nemotron_bot"} and ( + target_chat_id != str(settings.SRE_GROUP_CHAT_ID or "") + ): + return None, self._blocked_route_receipt( + "assistant_bot_interaction_destination_not_owned", + route_context={"signal_family": interaction_kind}, + ) + + return target_chat_id, { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "classifier": "non_monitoring_interaction_allowed", + "product_id": "awoooi", + "signal_family": interaction_kind, + "severity": "N/A", + "destination_alias": "inbound_interaction_reply_target", + "sender_bot_alias": actual_bot_alias, + "inbound_context_verified": True, + "provider_send_performed": False, + } + + def canonical_destination_chat_id( + self, + *, + product_id: str, + signal_family: str, + severity: str, + ) -> str: + """Resolve the runtime binding without exposing it in a receipt.""" + chat_id, _ = self._authorize_canonical_send_message( + payload={}, + route_context={ + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + }, + legacy_notification_type=None, + actual_bot_alias="tsenyang_bot", + ) + return chat_id or "" + def _summarize_callback_data_for_audit(self, callback_data: str) -> dict[str, str | None]: """Return a redaction-safe summary of callback_data without persisting nonce.""" parts = str(callback_data or "").split(":") @@ -5629,6 +5972,60 @@ class TelegramGateway: Returns: dict: API 回應 """ + payload = dict(payload) + route_context = payload.pop(_CANONICAL_ROUTE_CONTEXT_KEY, None) + legacy_notification_type = payload.pop( + _LEGACY_NOTIFICATION_TYPE_KEY, + None, + ) + interaction_context = payload.pop( + _NON_MONITORING_INTERACTION_KEY, + None, + ) + canonical_route_receipt: dict[str, object] | None = None + if method == "sendMessage": + if route_context is not None or legacy_notification_type is not None: + canonical_chat_id, canonical_route_receipt = ( + self._authorize_canonical_send_message( + payload=payload, + route_context=route_context, + legacy_notification_type=legacy_notification_type, + actual_bot_alias="tsenyang_bot", + ) + ) + elif interaction_context is not None: + canonical_chat_id, canonical_route_receipt = ( + self._authorize_non_monitoring_interaction( + payload=payload, + interaction_context=interaction_context, + actual_bot_alias="tsenyang_bot", + ) + ) + else: + canonical_chat_id, canonical_route_receipt = ( + None, + self._blocked_route_receipt( + "missing_product_signal_severity_or_interaction_context" + ), + ) + if canonical_chat_id is None: + logger.warning( + "telegram_canonical_route_blocked_no_egress", + classifier=canonical_route_receipt["classifier"], + product_id=canonical_route_receipt["product_id"], + signal_family=canonical_route_receipt["signal_family"], + severity=canonical_route_receipt["severity"], + provider_send_performed=False, + ) + return { + "ok": False, + "_awooop_outbound_mirror_acknowledged": False, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + "_awoooi_canonical_route_receipt": canonical_route_receipt, + } + payload["chat_id"] = canonical_chat_id + if not self._initialized: await self.initialize() @@ -5637,6 +6034,11 @@ class TelegramGateway: payload = normalize_telegram_send_message_payload(method, payload) source_envelope_extra = payload.pop(_AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY, None) + if canonical_route_receipt is not None: + source_envelope_extra = dict(source_envelope_extra or {}) + source_envelope_extra["canonical_route_receipt"] = ( + canonical_route_receipt + ) await self._attach_incident_thread_reply(method, payload) callback_reply = ( source_envelope_extra.get("callback_reply") @@ -5764,6 +6166,15 @@ class TelegramGateway: ) result["_awooop_provider_send_performed"] = True else: + if canonical_route_receipt is not None: + canonical_route_receipt["provider_send_performed"] = True + source_envelope_extra = ( + _acknowledge_callback_reply_source_envelope( + source_envelope_extra, + delivery_result=result, + provider_message_id=provider_message_id, + ) + ) await self._mirror_outbound_message( method=method, payload=payload, @@ -5775,6 +6186,12 @@ class TelegramGateway: result["_awooop_delivery_status"] = "pending_unknown" result["_awooop_provider_send_performed"] = True + if canonical_route_receipt is not None: + canonical_route_receipt["provider_send_performed"] = True + result["_awoooi_canonical_route_receipt"] = ( + canonical_route_receipt + ) + span.set_status(trace.Status(trace.StatusCode.OK)) return result @@ -5924,6 +6341,7 @@ class TelegramGateway: chunk_index: int, chunk_count: int, error: Exception, + parent_message_id: int | None = None, ) -> None: """Persist callback reply delivery failures into AwoooP outbound evidence.""" if not incident_id: @@ -5964,6 +6382,7 @@ class TelegramGateway: chunk_count=chunk_count, callback_action=callback_action, error=safe_error, + parent_message_id=parent_message_id, ), ) @@ -6358,11 +6777,30 @@ class TelegramGateway: "text": text, "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P2", + }, }) + if not _telegram_send_delivery_succeeded(result): + logger.info( + "analyzing_placeholder_no_send", + resource=resource_name, + delivery_status=( + result.get("_awooop_delivery_status") + if isinstance(result, Mapping) + else "invalid_response" + ), + ) + return None + msg_id: int | None = None result_val = result.get("result") if isinstance(result_val, dict): - msg_id = result_val.get("message_id") + candidate_message_id = result_val.get("message_id") + if isinstance(candidate_message_id, int): + msg_id = candidate_message_id logger.info("analyzing_placeholder_sent", message_id=msg_id, resource=resource_name) return msg_id except Exception as e: @@ -6579,17 +7017,15 @@ class TelegramGateway: controlled_auto_authorized=controlled_auto_authorized, ) - # 發送訊息：2026-04-30 統帥指示，告警卡片完整切到 SRE 戰情室群組。 - target_chat_id = self.alert_chat_id - if not target_chat_id: - logger.warning("telegram_approval_card_skipped", reason="alert_chat_id_missing") - return {} + # Unknown/missing notification_type is intentionally no-egress. The + # central sender resolves only allowlisted legacy types via the + # canonical registry and binds the actual runtime destination there. payload = { - "chat_id": target_chat_id, "text": text, "parse_mode": "HTML", "reply_markup": keyboard, "disable_web_page_preview": True, # 避免 SignOz URL 預覽 + _LEGACY_NOTIFICATION_TYPE_KEY: notification_type or "UNKNOWN", } logger.info( @@ -6597,26 +7033,50 @@ class TelegramGateway: approval_id=approval_id, risk_level=risk_level, resource=resource_name, - target_chat_id=str(target_chat_id), + destination_alias="canonical_policy_pending", signoz_integrated=signoz_metrics is not None, auto_tuning_available=bool(auto_tuning_command), ) result = await self._send_request("sendMessage", payload) - _msg_id = result.get("result", {}).get("message_id") - logger.info( - "telegram_approval_card_sent", - approval_id=approval_id, - message_id=_msg_id, - target_chat_id=str(target_chat_id), + delivery_succeeded = _telegram_send_delivery_succeeded(result) + result_value = result.get("result") + _msg_id = ( + result_value.get("message_id") + if delivery_succeeded and isinstance(result_value, dict) + else None ) + route_receipt = result.get("_awoooi_canonical_route_receipt") or {} + destination_alias = str(route_receipt.get("destination_alias") or "none") + runtime_target_chat_id = "" + if _msg_id and route_receipt.get("decision") == "allowed": + runtime_target_chat_id = self.canonical_destination_chat_id( + product_id=str(route_receipt.get("product_id") or ""), + signal_family=str(route_receipt.get("signal_family") or ""), + severity=str(route_receipt.get("severity") or ""), + ) + if delivery_succeeded: + logger.info( + "telegram_approval_card_sent", + approval_id=approval_id, + message_id=_msg_id, + destination_alias=destination_alias, + ) + else: + logger.warning( + "telegram_approval_card_no_send", + approval_id=approval_id, + destination_alias=destination_alias, + delivery_status=result.get("_awooop_delivery_status"), + ) # 2026-04-18 ADR-090-D: 寫入 notification_outcomes (MASTER §7.1 #10 KPI) try: from sqlalchemy import text as _sql + from src.db.base import get_db_context - _delivered = "delivered" if _msg_id else "failed" + _delivered = "delivered" if delivery_succeeded else "failed" _notif_type = f"TYPE-3-{alert_category}" if alert_category else "TYPE-3" async with get_db_context() as _db: await _db.execute( @@ -6632,7 +7092,7 @@ class TelegramGateway: { "aid": approval_id, "nt": _notif_type, - "rp": str(target_chat_id), + "rp": destination_alias, "mid": str(_msg_id) if _msg_id else None, "ds": _delivered, "md": '{"risk_level":"' + str(risk_level) + '"}', @@ -6643,7 +7103,7 @@ class TelegramGateway: # 2026-04-19 ogt + Claude Opus 4.7: 修 AP-1 — message_id 同時存進 # approval_records.telegram_message_id,不只 Redis(重啟會丟) - if _msg_id: + if _msg_id and runtime_target_chat_id: try: from src.services.approval_db import get_approval_service _svc = get_approval_service() @@ -6651,6 +7111,7 @@ class TelegramGateway: # 若有 update_telegram_message 方法(通常用 incident_id) # 先用 incident_id 更新,再 fallback 直接 UPDATE approval_records from sqlalchemy import text as _sql2 + from src.db.base import get_db_context as _gdc async with _gdc() as _db2: await _db2.execute( @@ -6662,7 +7123,7 @@ class TelegramGateway: """), { "mid": int(_msg_id), - "cid": int(target_chat_id), + "cid": int(runtime_target_chat_id), "aid": str(approval_id), }, ) @@ -6781,11 +7242,15 @@ class TelegramGateway: notification_type=notification_type, controlled_auto_authorized=controlled_auto_authorized, ) - resp = await self.send_to_group(text=text, reply_markup=_group_keyboard) + resp = await self.send_to_group( + text=text, + reply_markup=_group_keyboard, + legacy_notification_type=notification_type or "UNKNOWN", + ) # 2026-04-10 Claude Sonnet 4.6: 儲存 message_id 到 Redis，供 append_incident_update 使用 # tg_msg:{incident_id} → Telegram message_id (TTL 24h) - if incident_id and resp: + if incident_id and _telegram_send_delivery_succeeded(resp): tg_message_id = (resp.get("result") or {}).get("message_id") or resp.get("message_id") if tg_message_id: from src.core.redis_client import get_redis @@ -6846,10 +7311,10 @@ class TelegramGateway: return await self._send_request( "sendMessage", { - "chat_id": self.alert_chat_id, "text": text, "parse_mode": "HTML", "reply_markup": keyboard, + _LEGACY_NOTIFICATION_TYPE_KEY: "TYPE-1", }, ) @@ -6921,10 +7386,10 @@ class TelegramGateway: _result = await self._send_request( "sendMessage", { - "chat_id": self.alert_chat_id, "text": text, "parse_mode": "HTML", "reply_markup": keyboard, + _LEGACY_NOTIFICATION_TYPE_KEY: "TYPE-4D", }, ) @@ -6953,6 +7418,9 @@ class TelegramGateway: user_id: int, username: str, user: dict, + *, + inbound_chat_id: str | int, + inbound_message_id: int, ) -> dict: """ 處理 drift_view / drift_adopt / drift_revert 按鈕。 @@ -6967,7 +7435,11 @@ class TelegramGateway: try: if action == "drift_view": await self._answer_callback(callback_query_id, action, text="🔍 撈全部 Diff...") - await self._send_drift_diff_detail(report_id) + await self._send_drift_diff_detail( + report_id, + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, + ) return { "action": action, "approval_id": approval_id, "user": user, "success": True, "info_action": True, @@ -6984,6 +7456,8 @@ class TelegramGateway: _ok = False await self._edit_drift_card_outcome( report_id=report_id, verb="已採納", by=username, ok=_ok, + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, ) return {"action": action, "approval_id": approval_id, "user": user, "success": _ok} @@ -6998,6 +7472,8 @@ class TelegramGateway: _ok = False await self._edit_drift_card_outcome( report_id=report_id, verb="已回滾", by=username, ok=_ok, + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, ) return {"action": action, "approval_id": approval_id, "user": user, "success": _ok} @@ -7031,7 +7507,14 @@ class TelegramGateway: return "human_high" return "routine_medium" - async def _send_drift_diff_detail(self, report_id: str, page: int = 0) -> None: + async def _send_drift_diff_detail( + self, + report_id: str, + page: int = 0, + *, + inbound_chat_id: str | int, + inbound_message_id: int, + ) -> None: """ 送分頁 Drift Diff 到 Telegram (drift_view / drift_view_page 按鈕回應) @@ -7043,9 +7526,15 @@ class TelegramGateway: _rpt = await get_drift_repository().get_by_id(report_id) if not _rpt: await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id, "text": f"⚠️ 找不到 Drift report {html.escape(report_id)}", "parse_mode": "HTML", + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, }) return @@ -7125,10 +7614,16 @@ class TelegramGateway: _keyboard = {"inline_keyboard": _rows} if _rows else None _payload = { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id, "text": _full, "parse_mode": "HTML", "disable_web_page_preview": True, + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, } if _keyboard: _payload["reply_markup"] = _keyboard @@ -7136,9 +7631,15 @@ class TelegramGateway: except Exception as _e: logger.warning("drift_diff_detail_send_failed", report_id=report_id, page=page, error=str(_e)) await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id, "text": f"⚠️ Drift Diff 查詢失敗: {html.escape(str(_e)[:150])}", "parse_mode": "HTML", + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, }) async def _handle_ai_advisory_action( @@ -7149,7 +7650,9 @@ class TelegramGateway: user_id: int, username: str, user: dict, - message_id: int | None = None, + *, + inbound_chat_id: str | int, + inbound_message_id: int, ) -> dict: """ 2026-04-19 P0 修 (ADR-092): 處理 4 LLM scanner 的互動按鈕. @@ -7191,14 +7694,27 @@ class TelegramGateway: await self._answer_callback(callback_query_id, action, text=feedback_text) # 2026-04-22 Claude Sonnet 4.6: 發群組 reply（toast 2-3 秒消失，群組才是永久可見） - if message_id and feedback_text: + if feedback_text: try: - await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, + group_reply_result = await self._send_request("sendMessage", { + "chat_id": inbound_chat_id, "text": feedback_text, - "reply_to_message_id": message_id, + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, }) - logger.info("ai_advisory_group_reply_sent", action=pure_action, message_id=message_id) + if not _telegram_send_delivery_succeeded(group_reply_result): + raise TelegramGatewayError( + "ai_advisory_group_reply_delivery_not_proven" + ) + logger.info( + "ai_advisory_group_reply_sent", + action=pure_action, + message_id=inbound_message_id, + ) except Exception as _ge: logger.warning("ai_advisory_group_reply_failed", action=pure_action, error=str(_ge)) @@ -7217,6 +7733,9 @@ class TelegramGateway: async def _edit_drift_card_outcome( self, report_id: str, verb: str, by: str, ok: bool, + *, + inbound_chat_id: str | int, + inbound_message_id: int, ) -> None: """ drift_adopt/drift_revert 執行後: @@ -7229,29 +7748,52 @@ class TelegramGateway: f"({'成功' if ok else '失敗'})\n" f"Drift {html.escape(report_id)}" ) - _msg_id: int | None = None + # Redis is historical evidence only. A stale tg_drift value must never + # replace the verified message identity carried by this callback. try: _msg_id_raw = await get_redis().get(f"tg_drift:{report_id}") if _msg_id_raw: - _msg_id = int(_msg_id_raw) - # 先移除按鈕 - await self._send_request("editMessageReplyMarkup", { - "chat_id": self.alert_chat_id, - "message_id": _msg_id, - "reply_markup": {"inline_keyboard": []}, - }) + stored_message_id = int(_msg_id_raw) + if stored_message_id != inbound_message_id: + logger.warning( + "drift_parent_receipt_stale_ignored", + report_id=report_id, + stored_message_id=stored_message_id, + verified_inbound_message_id=inbound_message_id, + ) except Exception as _e: - logger.warning("drift_card_buttons_remove_failed", report_id=report_id, error=str(_e)) + logger.warning( + "drift_parent_receipt_read_failed_ignored", + report_id=report_id, + error=str(_e), + ) + + try: + await self._send_request("editMessageReplyMarkup", { + "chat_id": inbound_chat_id, + "message_id": inbound_message_id, + "reply_markup": {"inline_keyboard": []}, + }) + except Exception as _e: + logger.warning( + "drift_card_buttons_remove_failed", + report_id=report_id, + error=str(_e), + ) # 送簽核戳訊息（reply_to 原卡片,若有 msg_id） try: _payload: dict = { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id, "text": _stamp, "parse_mode": "HTML", + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, } - if _msg_id: - _payload["reply_to_message_id"] = _msg_id await self._send_request("sendMessage", _payload) except Exception as _e: logger.warning("drift_outcome_stamp_send_failed", report_id=report_id, error=str(_e)) @@ -7318,10 +7860,10 @@ class TelegramGateway: return await self._send_request( "sendMessage", { - "chat_id": self.alert_chat_id, "text": text, "parse_mode": "HTML", "reply_markup": keyboard, + _LEGACY_NOTIFICATION_TYPE_KEY: "TYPE-8M", _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY: { "source_refs": source_refs, }, @@ -7383,10 +7925,10 @@ class TelegramGateway: return await self._send_request( "sendMessage", { - "chat_id": self.alert_chat_id, "text": text, "parse_mode": "HTML", "reply_markup": keyboard, + _LEGACY_NOTIFICATION_TYPE_KEY: "TYPE-5S", }, ) @@ -7425,14 +7967,14 @@ class TelegramGateway: # 鐵律: 寧可沒按鈕，不可有死按鈕 (feedback_no_ghost_buttons.md) keyboard = {"inline_keyboard": []} - target_chat = group_chat_id or self.alert_chat_id return await self._send_request( "sendMessage", { - "chat_id": target_chat, + "chat_id": group_chat_id or "", "text": text, "parse_mode": "HTML", "reply_markup": keyboard, + _LEGACY_NOTIFICATION_TYPE_KEY: "TYPE-6B", }, ) @@ -7477,14 +8019,14 @@ class TelegramGateway: # 鐵律: 寧可沒按鈕，不可有死按鈕 (feedback_no_ghost_buttons.md) keyboard = {"inline_keyboard": []} - target_chat = group_chat_id or self.alert_chat_id return await self._send_request( "sendMessage", { - "chat_id": target_chat, - "text": text + ("\n📣 @所有人 事故升級，請協助！" if settings.SRE_GROUP_CHAT_ID else ""), + "chat_id": group_chat_id or "", + "text": text + "\n📣 @所有人 事故升級，請協助！", "parse_mode": "HTML", "reply_markup": keyboard, + _LEGACY_NOTIFICATION_TYPE_KEY: "TYPE-7E", }, ) @@ -7572,11 +8114,27 @@ class TelegramGateway: "parse_mode": "HTML", "reply_markup": self._build_sentry_keyboard(error_id), "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P2", + }, } logger.info("telegram_sentry_error_sending", error_id=error_id, service=service_name) result = await self._send_request("sendMessage", payload) - logger.info("telegram_sentry_error_sent", error_id=error_id) + if _telegram_send_delivery_succeeded(result): + logger.info( + "telegram_sentry_error_routed", + error_id=error_id, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) + else: + logger.warning( + "telegram_sentry_error_no_send", + error_id=error_id, + delivery_status=result.get("_awooop_delivery_status"), + ) return result @@ -7630,11 +8188,20 @@ class TelegramGateway: "parse_mode": "HTML", "reply_markup": self._build_resource_keyboard(resource_id), "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P2", + }, } logger.info("telegram_resource_warning_sending", resource_id=resource_id, pod=pod_name) result = await self._send_request("sendMessage", payload) - logger.info("telegram_resource_warning_sent", resource_id=resource_id) + logger.info( + "telegram_resource_warning_routed", + resource_id=resource_id, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) return result @@ -7686,11 +8253,20 @@ class TelegramGateway: "text": message.format(), "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P3", + }, } logger.info("telegram_repair_report_sending", date=report_date) result = await self._send_request("sendMessage", payload) - logger.info("telegram_repair_report_sent", date=report_date) + logger.info( + "telegram_repair_report_routed", + date=report_date, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) return result @@ -7743,11 +8319,20 @@ class TelegramGateway: "text": message.format(), "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P3", + }, } logger.info("telegram_daily_summary_sending", date=summary_date) result = await self._send_request("sendMessage", payload) - logger.info("telegram_daily_summary_sent", date=summary_date) + logger.info( + "telegram_daily_summary_routed", + date=summary_date, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) return result @@ -7801,6 +8386,11 @@ class TelegramGateway: "text": msg.format(), "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P2", + }, } logger.info("telegram_cicd_progress_sending", job=job_name, status=status) @@ -7812,7 +8402,13 @@ class TelegramGateway: result = await self._send_request("sendMessage", payload) span.set_attribute("telegram.attempts", attempt + 1) span.set_status(trace.Status(trace.StatusCode.OK)) - logger.info("telegram_cicd_progress_sent", job=job_name, status=status, attempt=attempt + 1) + logger.info( + "telegram_cicd_progress_routed", + job=job_name, + status=status, + attempt=attempt + 1, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) return result except TelegramGatewayError as e: last_error = e @@ -7881,11 +8477,20 @@ class TelegramGateway: "text": message.format(), "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": "P3", + }, } logger.info("telegram_deploy_success_sending", commit=commit_sha[:8]) result = await self._send_request("sendMessage", payload) - logger.info("telegram_deploy_success_sent", commit=commit_sha[:8]) + logger.info( + "telegram_deploy_success_routed", + commit=commit_sha[:8], + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) return result @@ -7924,11 +8529,20 @@ class TelegramGateway: "text": message.format(), "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "business_finops", + "severity": "P2", + }, } logger.info("telegram_rate_limit_warning_sending", provider=provider) result = await self._send_request("sendMessage", payload) - logger.info("telegram_rate_limit_warning_sent", provider=provider) + logger.info( + "telegram_rate_limit_warning_routed", + provider=provider, + delivery_status=result.get("_awooop_delivery_status", "sent"), + ) return result @@ -7940,6 +8554,7 @@ class TelegramGateway: message_id: int, original_text: str = "", username: str = "", + chat_id: str | int | None = None, ) -> dict: """ 處理簽核/調優回調 @@ -7956,6 +8571,24 @@ class TelegramGateway: dict: 處理結果 {action, approval_id, user, auto_tuning_result?} """ try: + verified_inbound_chat_id = str(chat_id or "") + try: + verified_inbound_message_id = int(message_id) + except (TypeError, ValueError): + verified_inbound_message_id = 0 + if not verified_inbound_chat_id or verified_inbound_message_id <= 0: + logger.warning( + "telegram_callback_blocked_missing_inbound_context", + callback_data_prefix=callback_data.split(":", 1)[0], + ) + return { + "action": "callback_blocked", + "approval_id": "", + "user": {"id": user_id, "username": username}, + "success": False, + "reason": "missing_verified_interaction_context", + } + # =================================================================== # Step 0: LLM Action Callback（H1/B4）— la:{short_id} 格式優先路由 # 2026-04-27 Claude Sonnet 4.6: H1+B4 Fix — 鬼魂按鈕鐵律修復 @@ -7967,6 +8600,8 @@ class TelegramGateway: callback_data=callback_data, user_id=user_id, username=username, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) # =================================================================== @@ -7990,17 +8625,29 @@ class TelegramGateway: # ADR-050 P2: 取得事件詳情，傳送新訊息 (保留原始簽核卡片+按鈕) # 2026-04-01 Claude Code (ADR-050 P2) await self._answer_callback_nonfatal(callback_query_id, action, text="📋 詳情傳送中...") - await self._send_incident_detail(incident_id) + await self._send_incident_detail( + incident_id, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, + ) elif action == "history": # ADR-050 P2: 取得頻率統計 # 2026-04-01 Claude Code (ADR-050 P2) await self._answer_callback_nonfatal(callback_query_id, action, text="📊 歷史統計傳送中...") - await self._send_incident_history(incident_id) + await self._send_incident_history( + incident_id, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, + ) elif action == "reanalyze": # ADR-050 P2: 觸發重診 # 2026-04-01 Claude Code (ADR-050 P2): reanalyze button handler await self._answer_callback_nonfatal(callback_query_id, action, text="🔄 重診排程中...") - await self._send_reanalyze_result(incident_id) + await self._send_reanalyze_result( + incident_id, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, + ) elif action == "drift_view_page": # 2026-04-20 P0.1 ogt + Claude Opus 4.7: drift_view 分頁切頁 # incident_id 格式: {report_id}_{page}（底線分隔） @@ -8012,7 +8659,12 @@ class TelegramGateway: await self._answer_callback_nonfatal( callback_query_id, action, text=f"📄 切換至第 {_page_num + 1} 頁..." ) - await self._send_drift_diff_detail(_rid or incident_id, page=_page_num) + await self._send_drift_diff_detail( + _rid or incident_id, + page=_page_num, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, + ) else: # 2026-04-14 Claude Sonnet 4.6 (Phase 5 Sprint 5.1): # 未知 action → fallback dispatcher (查看 callback_action_spec.yaml 是否有註冊) @@ -8021,6 +8673,8 @@ class TelegramGateway: action=action, incident_id=incident_id, user_id=user_id, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) return { @@ -8053,7 +8707,8 @@ class TelegramGateway: guard_result = await self._check_incident_state_guard( approval_id=approval_id, callback_query_id=callback_query_id, - message_id=message_id, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, original_text=original_text, ) if guard_result is not None: @@ -8072,6 +8727,8 @@ class TelegramGateway: user_id=user_id, username=username, user=user, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) # =================================================================== @@ -8088,7 +8745,8 @@ class TelegramGateway: user_id=user_id, username=username, user=user, - message_id=message_id, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) # =================================================================== @@ -8179,22 +8837,20 @@ class TelegramGateway: labels=_labels, ) - # Reply 結果到原告警卡片 - try: - from src.core.redis_client import get_redis as _gr - _rds = _gr() - _msg_id_raw = await _rds.get(f"tg_msg:{_incident_id_resolved}") - _orig_msg = int(_msg_id_raw) if _msg_id_raw else None - except Exception: - _orig_msg = None + # Reply to the exact verified callback message. A Redis parent + # lookup cannot replace the inbound identity of this interaction. try: _payload = { - "chat_id": self.alert_chat_id, + "chat_id": verified_inbound_chat_id, "text": _result.result_text, "parse_mode": "HTML", + "reply_to_message_id": verified_inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": verified_inbound_chat_id, + "inbound_message_id": verified_inbound_message_id, + }, } - if _orig_msg: - _payload["reply_to_message_id"] = _orig_msg await self._send_request("sendMessage", _payload) except Exception as _re: logger.warning("category_action_reply_send_failed", error=str(_re)) @@ -8236,11 +8892,12 @@ class TelegramGateway: ) # 更新訊息 await self._update_message_after_action( - message_id=message_id, action="tune", username=username, original_text=original_text, extra_info=auto_tuning_result.get("command", ""), + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) return { @@ -8265,10 +8922,11 @@ class TelegramGateway: text="⏰ 30 分鐘後再提醒", ) await self._update_message_after_action( - message_id=message_id, action="snooze", username=username, original_text=original_text, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) return { "action": action, @@ -8290,11 +8948,12 @@ class TelegramGateway: text="🔕 此類告警靜默 1 小時", ) await self._update_message_after_action( - message_id=message_id, action="silence", username=username, original_text=original_text, extra_info=silence_result.get("resource_name", ""), + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) return { "action": action, @@ -8311,6 +8970,18 @@ class TelegramGateway: # 實際步驟收集在 handle_message() 的 /done 流程中完成 # =================================================================== if action == "log_manual_fix": + if not chat_id or not message_id: + logger.warning( + "manual_fix_prompt_blocked_missing_inbound_context", + approval_id=approval_id, + ) + return { + "action": action, + "approval_id": approval_id, + "user": user, + "success": False, + "reason": "missing_verified_interaction_context", + } await self._answer_callback( callback_query_id, "log_manual_fix", @@ -8330,7 +9001,7 @@ class TelegramGateway: await self._send_request( "sendMessage", { - "chat_id": self.alert_chat_id, + "chat_id": chat_id, "text": ( "📝 手動修復記錄\n" "━━━━━━━━━━━━━━━━━━━\n" @@ -8339,6 +9010,12 @@ class TelegramGateway: "30 分鐘內有效" ), "parse_mode": "HTML", + "reply_to_message_id": message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(chat_id), + "inbound_message_id": message_id, + }, }, ) return { @@ -8358,10 +9035,11 @@ class TelegramGateway: # Step 4: 更新訊息 (保留原始內容 + 簽核鋼印) # =================================================================== await self._update_message_after_action( - message_id=message_id, action=action, username=username, original_text=original_text, + inbound_chat_id=verified_inbound_chat_id, + inbound_message_id=verified_inbound_message_id, ) logger.info( @@ -8409,7 +9087,9 @@ class TelegramGateway: self, approval_id: str, callback_query_id: str, - message_id: int, + *, + inbound_chat_id: str | int, + inbound_message_id: int, original_text: str, ) -> dict | None: """ @@ -8453,8 +9133,8 @@ class TelegramGateway: safe_original = html.escape(original_text) if original_text else "" stamp = _format_resolved_guard_stamp(incident.resolved_at) await self._send_request("editMessageText", { - "chat_id": self.alert_chat_id, - "message_id": message_id, + "chat_id": inbound_chat_id, + "message_id": inbound_message_id, "text": f"{safe_original}\n{separator}\n{stamp}" if safe_original else stamp, "parse_mode": "HTML", "reply_markup": {"inline_keyboard": []}, @@ -8464,8 +9144,8 @@ class TelegramGateway: # 移除按鈕保底 try: await self._send_request("editMessageReplyMarkup", { - "chat_id": self.alert_chat_id, - "message_id": message_id, + "chat_id": inbound_chat_id, + "message_id": inbound_message_id, "reply_markup": {"inline_keyboard": []}, }) except Exception: @@ -8502,6 +9182,8 @@ class TelegramGateway: user_id: int, username: str, fix_steps: str, + chat_id: str | int | None = None, + message_id: int | None = None, ) -> dict: """ ADR-071-H: 處理使用者輸入 /done 後的手動修復步驟記錄 @@ -8519,6 +9201,25 @@ class TelegramGateway: username: Telegram username fix_steps: 使用者輸入的修復步驟 """ + if not chat_id or not message_id: + return { + "success": False, + "reason": "missing_verified_interaction_context", + } + + def manual_reply_payload(text: str) -> dict[str, object]: + return { + "chat_id": chat_id, + "text": text, + "parse_mode": "HTML", + "reply_to_message_id": message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "operator_command_reply", + "inbound_chat_id": str(chat_id), + "inbound_message_id": message_id, + }, + } + try: from src.core.redis_client import get_redis as _get_redis redis = _get_redis() @@ -8526,11 +9227,10 @@ class TelegramGateway: pending_key = f"manual_fix_pending:{user_id}" approval_id_bytes = await redis.get(pending_key) if not approval_id_bytes: - await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, - "text": "⚠️ 找不到待記錄的修復任務，或已逾時。", - "parse_mode": "HTML", - }) + await self._send_request( + "sendMessage", + manual_reply_payload("⚠️ 找不到待記錄的修復任務，或已逾時。"), + ) return {"success": False, "reason": "no_pending_task"} approval_id = approval_id_bytes.decode() if isinstance(approval_id_bytes, bytes) else str(approval_id_bytes) @@ -8543,11 +9243,12 @@ class TelegramGateway: approval_repo = ApprovalDBRepository() approval = await approval_repo.get_by_approval_id(approval_id) if not approval: - await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, - "text": f"⚠️ 找不到簽核單 {html.escape(approval_id)}", - "parse_mode": "HTML", - }) + await self._send_request( + "sendMessage", + manual_reply_payload( + f"⚠️ 找不到簽核單 {html.escape(approval_id)}" + ), + ) return {"success": False, "reason": "approval_not_found"} incident_repo = IncidentDBRepository() @@ -8599,17 +9300,16 @@ class TelegramGateway: ) # 回覆確認 - await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, - "text": ( + await self._send_request( + "sendMessage", + manual_reply_payload( f"✅ 手動修復步驟已記錄\n" f"━━━━━━━━━━━━━━━━━━━\n" f"📋 事件: {html.escape(approval.incident_id)}\n" f"👤 記錄者: @{html.escape(username or str(user_id))}\n\n" f"正在建立草稿 Playbook，請至 AWOOOI 審核後生效。" ), - "parse_mode": "HTML", - }) + ) logger.info( "manual_fix_recorded", @@ -8808,6 +9508,9 @@ class TelegramGateway: callback_data: str, user_id: int, username: str = "", + *, + inbound_chat_id: str | int, + inbound_message_id: int, ) -> dict: """ B4: 處理 LLM 動態按鈕 callback（格式 la:{short_id}） @@ -8945,9 +9648,15 @@ class TelegramGateway: ) try: await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id, "text": result_text, "parse_mode": "HTML", + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, }) except Exception as exc: logger.warning("llm_action_result_notify_failed", error=str(exc)) @@ -9007,10 +9716,12 @@ class TelegramGateway: async def _update_message_after_action( self, - message_id: int, action: str, username: str, original_text: str, + *, + inbound_chat_id: str | int, + inbound_message_id: int, extra_info: str = "", ) -> None: """ @@ -9043,12 +9754,16 @@ class TelegramGateway: # 先用 editMessageReplyMarkup 確保按鈕移除，再嘗試更新文字 try: await self._send_request("editMessageReplyMarkup", { - "chat_id": self.alert_chat_id, - "message_id": message_id, + "chat_id": inbound_chat_id, + "message_id": inbound_message_id, "reply_markup": {"inline_keyboard": []}, }) except TelegramGatewayError as e: - logger.warning("telegram_remove_buttons_failed", message_id=message_id, error=str(e)) + logger.warning( + "telegram_remove_buttons_failed", + message_id=inbound_message_id, + error=str(e), + ) # Step 2: 嘗試更新文字 (原始文字已轉義，確保 HTML 安全) separator = "──────────────" @@ -9056,8 +9771,8 @@ class TelegramGateway: safe_updated_text = f"{safe_original}\n{separator}\n{stamp}" try: await self._send_request("editMessageText", { - "chat_id": self.alert_chat_id, - "message_id": message_id, + "chat_id": inbound_chat_id, + "message_id": inbound_message_id, "text": safe_updated_text, "parse_mode": "HTML", "reply_markup": {"inline_keyboard": []}, @@ -9065,7 +9780,11 @@ class TelegramGateway: }) except TelegramGatewayError as e: # 文字更新失敗不影響整體流程，按鈕已移除 - logger.warning("telegram_update_text_failed", message_id=message_id, error=str(e)) + logger.warning( + "telegram_update_text_failed", + message_id=inbound_message_id, + error=str(e), + ) async def mark_auto_repaired( self, @@ -9101,7 +9820,7 @@ class TelegramGateway: # 回覆原訊息說明結果 _status = "✅ 已自動修復" if success else "❌ 自動修復失敗" - await self._send_request("sendMessage", { + result = await self._send_request("sendMessage", { "chat_id": self.alert_chat_id, "text": ( f"{_status}\n" @@ -9110,7 +9829,27 @@ class TelegramGateway: ), "parse_mode": "HTML", "reply_parameters": {"message_id": message_id}, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + }, + _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY: ( + _incident_lifecycle_parent_source_envelope_extra( + approval_id=approval_id, + parent_lookup_key=f"tg_approval:{approval_id}", + parent_message_id=message_id, + event_kind="auto_repair_result", + ) + ), }) + if not _telegram_send_delivery_succeeded(result): + logger.warning( + "mark_auto_repaired_no_send", + approval_id=approval_id, + delivery_status=result.get("_awooop_delivery_status"), + ) + return False return True except Exception as e: @@ -9163,14 +9902,17 @@ class TelegramGateway: # 詳細執行紀錄應進 timeline / AwoooP Run Monitor，避免群組被 auto-failure 洗版。 status_hash = hashlib.sha1(status_line.encode("utf-8")).hexdigest()[:16] dedup_key = f"{INCIDENT_UPDATE_DEDUP_PREFIX}{incident_id}:{status_hash}" - try: - was_set = await redis.set( - dedup_key, - "1", - ex=INCIDENT_UPDATE_DEDUP_TTL_SECONDS, - nx=True, + if dedup_key in self._delivered_incident_update_dedup: + logger.info( + "append_incident_update_dedup_suppressed", + incident_id=incident_id, + dedup_key=dedup_key, ) - if not was_set: + return True + try: + durable_dedup = await redis.get(dedup_key) + if durable_dedup in {"provider_ack", b"provider_ack"}: + self._delivered_incident_update_dedup.add(dedup_key) logger.info( "append_incident_update_dedup_suppressed", incident_id=incident_id, @@ -9179,36 +9921,37 @@ class TelegramGateway: return True except Exception as exc: logger.warning( - "append_incident_update_dedup_failed", + "append_incident_update_dedup_read_failed", incident_id=incident_id, error=str(exc), ) suppress_reply = False + global_dedup_key: str | None = None if _is_noisy_failure_update(status_line): # 不同 incident 若卡在同一個自動修復/診斷失敗摘要，Telegram 只推第一則； # 每個 incident 仍會繼續移除原卡危險按鈕，完整細節交給 timeline / AwoooP。 global_hash = hashlib.sha1(status_line.encode("utf-8")).hexdigest()[:16] global_dedup_key = f"{INCIDENT_UPDATE_GLOBAL_FAILURE_DEDUP_PREFIX}{global_hash}" - try: - was_global_set = await redis.set( - global_dedup_key, - incident_id, - ex=INCIDENT_UPDATE_GLOBAL_FAILURE_DEDUP_TTL_SECONDS, - nx=True, - ) - suppress_reply = not bool(was_global_set) - if suppress_reply: - logger.info( - "append_incident_update_global_failure_dedup_suppressed", + if global_dedup_key in self._delivered_global_failure_dedup: + suppress_reply = True + else: + try: + durable_global_dedup = await redis.get(global_dedup_key) + if durable_global_dedup in {"provider_ack", b"provider_ack"}: + self._delivered_global_failure_dedup.add(global_dedup_key) + suppress_reply = True + except Exception as exc: + logger.warning( + "append_incident_update_global_failure_dedup_read_failed", incident_id=incident_id, - dedup_key=global_dedup_key, + error=str(exc), ) - except Exception as exc: - logger.warning( - "append_incident_update_global_failure_dedup_failed", + if suppress_reply: + logger.info( + "append_incident_update_global_failure_dedup_suppressed", incident_id=incident_id, - error=str(exc), + dedup_key=global_dedup_key, ) # Step 1: 取得原始訊息文字（Telegram Bot API 不提供讀取原文，只能在 editMessageText 裡重建） @@ -9246,15 +9989,69 @@ class TelegramGateway: # Step 2: Reply 原訊息追加狀態（保留原文不動，以 reply 方式延續） try: - await self._send_request("sendMessage", { + result = await self._send_request("sendMessage", { "chat_id": self.alert_chat_id, "text": status_line, "parse_mode": "HTML", "reply_to_message_id": message_id, "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + }, + _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY: ( + _incident_lifecycle_parent_source_envelope_extra( + incident_id=incident_id, + parent_lookup_key=f"tg_msg:{incident_id}", + parent_message_id=message_id, + event_kind="incident_status_update", + ) + ), }) except TelegramGatewayError as e: logger.warning("append_incident_update_reply_failed", message_id=message_id, error=str(e)) + return False + + if not _telegram_send_delivery_succeeded(result): + logger.warning( + "append_incident_update_no_send", + incident_id=incident_id, + message_id=message_id, + delivery_status=result.get("_awooop_delivery_status"), + ) + return False + + self._delivered_incident_update_dedup.add(dedup_key) + try: + await redis.set( + dedup_key, + "provider_ack", + ex=INCIDENT_UPDATE_DEDUP_TTL_SECONDS, + nx=True, + ) + except Exception as exc: + logger.warning( + "append_incident_update_dedup_write_failed", + incident_id=incident_id, + error=str(exc), + ) + + if global_dedup_key: + self._delivered_global_failure_dedup.add(global_dedup_key) + try: + await redis.set( + global_dedup_key, + "provider_ack", + ex=INCIDENT_UPDATE_GLOBAL_FAILURE_DEDUP_TTL_SECONDS, + nx=True, + ) + except Exception as exc: + logger.warning( + "append_incident_update_global_failure_dedup_write_failed", + incident_id=incident_id, + error=str(exc), + ) logger.info( "append_incident_update_done", @@ -9297,14 +10094,17 @@ class TelegramGateway: return False dedup_key = f"{GROUPED_ALERT_DIGEST_DEDUP_PREFIX}{group_key}" - try: - was_set = await redis.set( - dedup_key, - incident_id, - ex=GROUPED_ALERT_DIGEST_DEDUP_TTL_SECONDS, - nx=True, + if dedup_key in self._delivered_grouped_digest_dedup: + logger.info( + "grouped_alert_digest_dedup_suppressed", + incident_id=incident_id, + group_key=group_key, ) - if not was_set: + return True + try: + durable_dedup = await redis.get(dedup_key) + if durable_dedup in {"provider_ack", b"provider_ack"}: + self._delivered_grouped_digest_dedup.add(dedup_key) logger.info( "grouped_alert_digest_dedup_suppressed", incident_id=incident_id, @@ -9313,14 +10113,14 @@ class TelegramGateway: return True except Exception as exc: logger.warning( - "grouped_alert_digest_dedup_failed", + "grouped_alert_digest_dedup_read_failed", incident_id=incident_id, group_key=group_key, error=str(exc), ) try: - await self._send_request("sendMessage", { + result = await self._send_request("sendMessage", { "chat_id": self.alert_chat_id, "text": digest_text[:1400], "parse_mode": "HTML", @@ -9329,6 +10129,19 @@ class TelegramGateway: "allow_sending_without_reply": True, }, "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + }, + _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY: ( + _incident_lifecycle_parent_source_envelope_extra( + incident_id=incident_id, + parent_lookup_key=f"tg_msg:{incident_id}", + parent_message_id=message_id, + event_kind="grouped_alert_digest", + ) + ), }) except TelegramGatewayError as exc: logger.warning( @@ -9340,6 +10153,32 @@ class TelegramGateway: ) return False + if not _telegram_send_delivery_succeeded(result): + logger.warning( + "grouped_alert_digest_no_send", + incident_id=incident_id, + group_key=group_key, + message_id=message_id, + delivery_status=result.get("_awooop_delivery_status"), + ) + return False + + self._delivered_grouped_digest_dedup.add(dedup_key) + try: + await redis.set( + dedup_key, + "provider_ack", + ex=GROUPED_ALERT_DIGEST_DEDUP_TTL_SECONDS, + nx=True, + ) + except Exception as exc: + logger.warning( + "grouped_alert_digest_dedup_write_failed", + incident_id=incident_id, + group_key=group_key, + error=str(exc), + ) + logger.info( "grouped_alert_digest_reply_sent", incident_id=incident_id, @@ -9354,6 +10193,9 @@ class TelegramGateway: action: str, incident_id: str, user_id: int, + *, + inbound_chat_id: str | int, + inbound_message_id: int, ) -> None: """ Phase 5 Sprint 5.1 (2026-04-14 Claude Sonnet 4.6): @@ -9397,24 +10239,23 @@ class TelegramGateway: labels=labels, ) - # Reply to 原卡片 — 從 Redis tg_msg 查 message_id - try: - from src.core.redis_client import get_redis - redis = get_redis() - msg_id_raw = await redis.get(f"tg_msg:{incident_id}") - orig_msg_id = int(msg_id_raw) if msg_id_raw else None - except Exception: - orig_msg_id = None - try: payload: dict = { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id, "text": result.result_text, "parse_mode": "HTML", + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id), + "inbound_message_id": inbound_message_id, + }, } - if orig_msg_id: - payload["reply_to_message_id"] = orig_msg_id - await self._send_request("sendMessage", payload) + reply_result = await self._send_request("sendMessage", payload) + if not _telegram_send_delivery_succeeded(reply_result): + raise TelegramGatewayError( + "category_action_reply_delivery_not_proven" + ) logger.info( "category_action_reply_sent", action=action, @@ -9425,7 +10266,13 @@ class TelegramGateway: except Exception as _e: logger.warning("category_action_reply_failed", action=action, error=str(_e)) - async def _send_incident_detail(self, incident_id: str) -> None: + async def _send_incident_detail( + self, + incident_id: str, + *, + inbound_chat_id: str | int, + inbound_message_id: int, + ) -> None: """ ADR-050 P2: 傳送事件詳情訊息 (不修改原始簽核卡片) @@ -9440,7 +10287,13 @@ class TelegramGateway: incident = await repo.get_by_id(incident_id) if not incident: - await self.send_notification(f"⚠️ 找不到事件 {html.escape(incident_id)}") + await self.send_notification( + f"⚠️ 找不到事件 {html.escape(incident_id)}", + chat_id=inbound_chat_id, + interaction_kind="callback_reply", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, + ) return dc = incident.decision_chain @@ -9625,6 +10478,8 @@ class TelegramGateway: await self._send_html_line_message( lines, failure_context="incident_detail", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, reply_markup=_awooop_truth_chain_reply_markup(incident_id), incident_id=incident_id, callback_action="detail", @@ -9634,9 +10489,21 @@ class TelegramGateway: except Exception as e: logger.warning("send_incident_detail_failed", incident_id=incident_id, error=str(e)) - await self.send_notification(f"⚠️ 無法取得事件詳情: {html.escape(str(e)[:100])}") + await self.send_notification( + f"⚠️ 無法取得事件詳情: {html.escape(str(e)[:100])}", + chat_id=inbound_chat_id, + interaction_kind="callback_reply", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, + ) - async def _send_incident_history(self, incident_id: str) -> None: + async def _send_incident_history( + self, + incident_id: str, + *, + inbound_chat_id: str | int, + inbound_message_id: int, + ) -> None: """ ADR-050 P2: 傳送事件頻率統計訊息與 DB truth-chain 摘要 @@ -9653,7 +10520,13 @@ class TelegramGateway: incident = await repo.get_by_id(incident_id) if not incident: - await self.send_notification(f"⚠️ 找不到事件 {html.escape(incident_id)}") + await self.send_notification( + f"⚠️ 找不到事件 {html.escape(incident_id)}", + chat_id=inbound_chat_id, + interaction_kind="callback_reply", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, + ) return lines = [ @@ -9810,6 +10683,8 @@ class TelegramGateway: await self._send_html_line_message( lines, failure_context="incident_history", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, reply_markup=_awooop_truth_chain_reply_markup(incident_id), incident_id=incident_id, callback_action="history", @@ -9819,9 +10694,21 @@ class TelegramGateway: except Exception as e: logger.warning("send_incident_history_failed", incident_id=incident_id, error=str(e)) - await self.send_notification(f"⚠️ 無法取得歷史統計: {html.escape(str(e))}") + await self.send_notification( + f"⚠️ 無法取得歷史統計: {html.escape(str(e))}", + chat_id=inbound_chat_id, + interaction_kind="callback_reply", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, + ) - async def _send_reanalyze_result(self, incident_id: str) -> None: + async def _send_reanalyze_result( + self, + incident_id: str, + *, + inbound_chat_id: str | int, + inbound_message_id: int, + ) -> None: """ ADR-050 P2: 觸發重診並傳送結果訊息 @@ -9856,12 +10743,22 @@ class TelegramGateway: f"{html.escape(result['message'])}" ) - await self.send_notification(msg) + await self.send_notification( + msg, + chat_id=inbound_chat_id, + interaction_kind="callback_reply", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, + ) except Exception as e: logger.warning("send_reanalyze_result_failed", incident_id=incident_id, error=str(e)) await self.send_notification( - f"⚠️ 重診觸發失敗: {html.escape(str(e)[:100])}" + f"⚠️ 重診觸發失敗: {html.escape(str(e)[:100])}", + chat_id=inbound_chat_id, + interaction_kind="callback_reply", + inbound_chat_id=inbound_chat_id, + inbound_message_id=inbound_message_id, ) # ========================================================================= @@ -9886,7 +10783,12 @@ class TelegramGateway: "━━━━━━━━━━━━━━━━━\n" "⚠️ 請人工評估並手動處理" ) - await self.send_notification(text) + await self.send_canonical_message( + product_id="awoooi", + signal_family="security_incident", + severity="P1", + text=text, + ) except Exception as e: logger.error("t1_guardrail_blocked_notify_failed", service=service_name, error=str(e)) @@ -9913,7 +10815,12 @@ class TelegramGateway: "━━━━━━━━━━━━━━━━━\n" "請等待備份完成後，人工重新評估修復方案" ) - await self.send_notification(text) + await self.send_canonical_message( + product_id="awoooi", + signal_family="disaster_recovery", + severity="P1", + text=text, + ) except Exception as e: logger.error("t2_preflight_failed_notify_failed", service=service_name, error=str(e)) @@ -9939,7 +10846,12 @@ class TelegramGateway: f"錯誤: {err}\n" "請人工介入，備份異常" ) - await self.send_notification(text) + await self.send_canonical_message( + product_id="awoooi", + signal_family="disaster_recovery", + severity="P1", + text=text, + ) except Exception as e: logger.error("t3_backup_result_notify_failed", backup=backup_name, error=str(e)) @@ -9964,7 +10876,12 @@ class TelegramGateway: "━━━━━━━━━━━━━━━━━\n" "請第二位審核者登入確認" ) - await self.send_notification(text) + await self.send_canonical_message( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + text=text, + ) except Exception as e: logger.error("t4_multisig_waiting_notify_failed", approval=approval_id, error=str(e)) @@ -9981,7 +10898,12 @@ class TelegramGateway: f"服務: {html.escape(service_name)}\n" "授權: 2/2 票 開始執行..." ) - await self.send_notification(text) + await self.send_canonical_message( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + text=text, + ) except Exception as e: logger.error("t5_multisig_approved_notify_failed", service=service_name, error=str(e)) @@ -10000,7 +10922,12 @@ class TelegramGateway: f"動作: {html.escape(action_description)}\n" f"時間: {html.escape(timestamp)}" ) - await self.send_notification(text) + await self.send_canonical_message( + product_id="awoooi", + signal_family="incident_lifecycle", + severity="P1", + text=text, + ) except Exception as e: logger.error("t6_change_applied_notify_failed", operator=operator, error=str(e)) @@ -10009,16 +10936,15 @@ class TelegramGateway: text: str, parse_mode: str = "HTML", chat_id: str | int | None = None, + *, + interaction_kind: str | None = None, + inbound_chat_id: str | int | None = None, + inbound_message_id: int | None = None, ) -> dict: - """ - 發送純文字通知 + """Reply to a verified inbound interaction; never classify by default. - Args: - text: 訊息內容 - parse_mode: 解析模式 - - Returns: - dict: API 回應 + A generic call without explicit inbound chat/message identity reaches + the final-exit gate without any interaction classifier and is blocked. """ payload_text = text[:500] payload_parse_mode = parse_mode @@ -10027,9 +10953,17 @@ class TelegramGateway: payload_parse_mode = None payload = { - "chat_id": chat_id or self.alert_chat_id, + "chat_id": chat_id or "", "text": payload_text, # SOUL.md 字數限制 } + if interaction_kind: + payload[_NON_MONITORING_INTERACTION_KEY] = { + "interaction_kind": interaction_kind, + "inbound_chat_id": str(inbound_chat_id or ""), + "inbound_message_id": inbound_message_id, + } + if inbound_message_id is not None: + payload["reply_to_message_id"] = inbound_message_id if payload_parse_mode: payload["parse_mode"] = payload_parse_mode @@ -10038,9 +10972,17 @@ class TelegramGateway: except TelegramGatewayError as exc: if payload_parse_mode and payload_parse_mode.upper() == "HTML" and "HTTP error: 400" in str(exc): fallback_payload = { - "chat_id": chat_id or self.alert_chat_id, + "chat_id": chat_id or "", "text": _plain_text_from_html(text, limit=500), } + if interaction_kind: + fallback_payload[_NON_MONITORING_INTERACTION_KEY] = { + "interaction_kind": interaction_kind, + "inbound_chat_id": str(inbound_chat_id or ""), + "inbound_message_id": inbound_message_id, + } + if inbound_message_id is not None: + fallback_payload["reply_to_message_id"] = inbound_message_id return await self._send_request("sendMessage", fallback_payload) raise @@ -10048,8 +10990,9 @@ class TelegramGateway: self, lines: list[str], *, - chat_id: str | int | None = None, failure_context: str, + inbound_chat_id: str | int, + inbound_message_id: int, reply_markup: dict | None = None, incident_id: str | None = None, callback_action: str | None = None, @@ -10058,18 +11001,25 @@ class TelegramGateway: ) -> None: """Send a multi-line HTML message without cutting Telegram tags in half.""" chunks = _telegram_html_chunks(lines) - actual_chat_id = str(chat_id or self.alert_chat_id) + actual_chat_id = str(inbound_chat_id) + interaction_context = { + "interaction_kind": "callback_reply", + "inbound_chat_id": actual_chat_id, + "inbound_message_id": inbound_message_id, + } for index, chunk in enumerate(chunks): try: payload: dict = { "chat_id": actual_chat_id, "text": chunk, "parse_mode": "HTML", + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: dict(interaction_context), } source_extra = _callback_reply_source_envelope_extra( incident_id=incident_id, failure_context=failure_context, - status="callback_reply_sent", + status="callback_reply_attempt", chunk_index=index, chunk_count=len(chunks), callback_action=callback_action, @@ -10081,10 +11031,14 @@ class TelegramGateway: payload[_AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] = source_extra if index == 0 and reply_markup: payload["reply_markup"] = reply_markup - await self._send_request( + primary_result = await self._send_request( "sendMessage", payload, ) + if not _telegram_send_delivery_succeeded(primary_result): + raise TelegramGatewayError( + "html_callback_reply_delivery_not_proven" + ) except Exception as exc: logger.warning( "telegram_html_line_message_failed", @@ -10096,11 +11050,13 @@ class TelegramGateway: fallback_payload: dict = { "chat_id": actual_chat_id, "text": _plain_text_from_html(chunk), + "reply_to_message_id": inbound_message_id, + _NON_MONITORING_INTERACTION_KEY: dict(interaction_context), } fallback_source_extra = _callback_reply_source_envelope_extra( incident_id=incident_id, failure_context=failure_context, - status="callback_reply_fallback_sent", + status="callback_reply_fallback_attempt", chunk_index=index, chunk_count=len(chunks), callback_action=callback_action, @@ -10114,10 +11070,14 @@ class TelegramGateway: if index == 0 and reply_markup: fallback_payload["reply_markup"] = reply_markup try: - await self._send_request( + fallback_result = await self._send_request( "sendMessage", fallback_payload, ) + if not _telegram_send_delivery_succeeded(fallback_result): + raise TelegramGatewayError( + "plain_callback_reply_delivery_not_proven" + ) except Exception as fallback_exc: logger.warning( "telegram_html_line_message_plain_fallback_failed", @@ -10129,12 +11089,14 @@ class TelegramGateway: rescue_payload: dict = { "chat_id": actual_chat_id, "text": _plain_text_from_html(chunk, limit=3500), + "reply_to_message_id": inbound_message_id, "_skip_incident_thread_reply": True, + _NON_MONITORING_INTERACTION_KEY: dict(interaction_context), } rescue_source_extra = _callback_reply_source_envelope_extra( incident_id=incident_id, failure_context=failure_context, - status="callback_reply_rescue_sent", + status="callback_reply_rescue_attempt", chunk_index=index, chunk_count=len(chunks), callback_action=callback_action, @@ -10146,10 +11108,14 @@ class TelegramGateway: if rescue_source_extra: rescue_payload[_AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] = rescue_source_extra try: - await self._send_request( + rescue_result = await self._send_request( "sendMessage", rescue_payload, ) + if not _telegram_send_delivery_succeeded(rescue_result): + raise TelegramGatewayError( + "rescue_callback_reply_delivery_not_proven" + ) except Exception as rescue_exc: logger.error( "telegram_html_line_message_rescue_failed", @@ -10173,8 +11139,12 @@ class TelegramGateway: text: str, parse_mode: str = "HTML", reply_markup: dict | None = None, + *, + product_id: str | None = None, + signal_family: str | None = None, + severity: str | None = None, ) -> dict: - """發送告警型純文字通知到 SRE 戰情室群組。""" + """Send only when the caller supplies a complete canonical route.""" safe_text, effective_parse_mode = normalize_alert_notification_payload( text, parse_mode, @@ -10184,6 +11154,12 @@ class TelegramGateway: "text": safe_text[:4096], "parse_mode": effective_parse_mode, } + if product_id and signal_family and severity: + payload[_CANONICAL_ROUTE_CONTEXT_KEY] = { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + } if reply_markup: payload["reply_markup"] = reply_markup payload[_AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] = ( @@ -10195,6 +11171,35 @@ class TelegramGateway: ) return await self._send_request("sendMessage", payload) + async def send_canonical_message( + self, + *, + product_id: str, + signal_family: str, + severity: str, + text: str, + parse_mode: str = "HTML", + reply_markup: dict | None = None, + disable_web_page_preview: bool = True, + reply_to_message_id: int | None = None, + ) -> dict: + """Single production entry point for a classified Telegram message.""" + payload: dict[str, object] = { + "text": text[:4096], + "parse_mode": parse_mode, + "disable_web_page_preview": disable_web_page_preview, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + }, + } + if reply_markup: + payload["reply_markup"] = reply_markup + if reply_to_message_id is not None: + payload["reply_to_message_id"] = reply_to_message_id + return await self._send_request("sendMessage", payload) + # ========================================================================= # 2026-05-04 Claude Sonnet 4.6: send_text 公開 wrapper（修復 drift_adopt_telegram_failed） # ========================================================================= @@ -10205,6 +11210,13 @@ class TelegramGateway: chat_id: int | str | None = None, parse_mode: str = "HTML", disable_web_page_preview: bool = True, + *, + product_id: str | None = None, + signal_family: str | None = None, + severity: str | None = None, + interaction_kind: str | None = None, + inbound_chat_id: str | int | None = None, + inbound_message_id: int | None = None, ) -> dict: """ 公開 send_text wrapper — 委派至 _send_request('sendMessage', ...) @@ -10227,11 +11239,25 @@ class TelegramGateway: parse_mode, ) payload: dict = { - "chat_id": chat_id or self.alert_chat_id, + "chat_id": chat_id or "", "text": safe_text[:4096], "parse_mode": effective_parse_mode, "disable_web_page_preview": disable_web_page_preview, } + if product_id and signal_family and severity: + payload[_CANONICAL_ROUTE_CONTEXT_KEY] = { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + } + elif interaction_kind: + payload[_NON_MONITORING_INTERACTION_KEY] = { + "interaction_kind": interaction_kind, + "inbound_chat_id": str(inbound_chat_id or ""), + "inbound_message_id": inbound_message_id, + } + if inbound_message_id is not None: + payload["reply_to_message_id"] = inbound_message_id return await self._send_request("sendMessage", payload) async def send_controlled_apply_result_receipt( @@ -10253,14 +11279,6 @@ class TelegramGateway: ) -> dict: """Send and mirror the AI Agent controlled-apply result receipt.""" - if not self.alert_chat_id: - logger.warning( - "controlled_apply_result_receipt_skipped_no_chat", - incident_id=incident_id, - apply_op_id=apply_op_id, - ) - return {} - no_write_replay = execution_kind == "no_write_replay" success = ( not no_write_replay @@ -10428,10 +11446,14 @@ class TelegramGateway: f"Runs: {html.escape(incident_runs_url(incident_id, project_id=project_id or 'awoooi'))}", ] payload: dict = { - "chat_id": self.alert_chat_id, "text": "\n".join(lines)[:4096], "parse_mode": "HTML", "disable_web_page_preview": True, + _CANONICAL_ROUTE_CONTEXT_KEY: { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + }, } reply_markup = incident_truth_chain_reply_markup( incident_id, @@ -10464,6 +11486,11 @@ class TelegramGateway: payload: dict = { "chat_id": chat_id, "text": text[:4096], + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "operator_command_reply", + "inbound_chat_id": str(chat_id), + "inbound_message_id": reply_to_message_id, + }, } if reply_to_message_id: payload["reply_to_message_id"] = reply_to_message_id @@ -10479,9 +11506,14 @@ class TelegramGateway: text: str, parse_mode: str = "HTML", reply_markup: dict | None = None, + *, + legacy_notification_type: str | None = None, + product_id: str | None = None, + signal_family: str | None = None, + severity: str | None = None, ) -> dict: """ - 用 @tsenyangbot 發訊息到 SRE 群組 (SRE_GROUP_CHAT_ID) + 透過中央 canonical policy 解析後，才允許發往 SRE 群組。 Args: text: 訊息內容 @@ -10491,15 +11523,18 @@ class TelegramGateway: Returns: dict: Telegram API 回應 (含 message_id) """ - if not settings.SRE_GROUP_CHAT_ID: - logger.warning("send_to_group_skipped", reason="SRE_GROUP_CHAT_ID not configured") - return {} - payload: dict = { - "chat_id": settings.SRE_GROUP_CHAT_ID, "text": text[:4096], "parse_mode": parse_mode, } + if legacy_notification_type: + payload[_LEGACY_NOTIFICATION_TYPE_KEY] = legacy_notification_type + elif product_id and signal_family and severity: + payload[_CANONICAL_ROUTE_CONTEXT_KEY] = { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + } if reply_markup: payload["reply_markup"] = reply_markup @@ -10508,17 +11543,21 @@ class TelegramGateway: async def _send_as_bot( self, token: str, - chat_id: str, text: str, reply_to_message_id: int | None = None, parse_mode: str = "HTML", + *, + product_id: str | None = None, + signal_family: str | None = None, + severity: str | None = None, + actual_bot_alias: str, + inbound_chat_id: str | int | None = None, ) -> dict: """ 用指定 Bot Token 發訊息。 Args: token: Bot Token - chat_id: 群組 Chat ID text: 訊息內容 reply_to_message_id: 回覆哪則訊息的 message_id parse_mode: 解析模式 @@ -10526,26 +11565,74 @@ class TelegramGateway: Returns: dict: Telegram API 回應 """ - if not self._http_client: - raise TelegramGatewayError("HTTP client not initialized") - - url = f"{self.TELEGRAM_API_BASE}/bot{token}/sendMessage" - payload: dict = { - "chat_id": chat_id, + route_context = ( + { + "product_id": product_id, + "signal_family": signal_family, + "severity": severity, + } + if product_id and signal_family and severity + else None + ) + candidate_payload: dict[str, object] = { + "chat_id": str(inbound_chat_id or settings.SRE_GROUP_CHAT_ID or ""), "text": text[:4096], "parse_mode": parse_mode, } - # 2026-04-03 ogt: supergroup 跨 Bot reply 需用 reply_parameters (Bot API v6.7+) - # 舊的 reply_to_message_id 在 supergroup 會 400，改用新格式 + allow_sending_without_reply if reply_to_message_id: - payload["reply_parameters"] = { + candidate_payload["reply_parameters"] = { "message_id": reply_to_message_id, "allow_sending_without_reply": True, } + if route_context is not None: + chat_id, route_receipt = self._authorize_canonical_send_message( + payload=candidate_payload, + route_context=route_context, + legacy_notification_type=None, + actual_bot_alias=actual_bot_alias, + ) + elif reply_to_message_id: + chat_id, route_receipt = self._authorize_non_monitoring_interaction( + payload=candidate_payload, + interaction_context={ + "interaction_kind": "bot_discussion_reply", + "inbound_chat_id": str(inbound_chat_id or ""), + "inbound_message_id": reply_to_message_id, + }, + actual_bot_alias=actual_bot_alias, + ) + else: + chat_id, route_receipt = ( + None, + self._blocked_route_receipt( + "missing_product_signal_severity_or_interaction_context" + ), + ) + if chat_id is None: + return { + "ok": False, + "_awooop_outbound_mirror_acknowledged": False, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + "_awoooi_canonical_route_receipt": route_receipt, + } + + if not self._initialized: + await self.initialize() + if not self._http_client: + raise TelegramGatewayError("HTTP client not initialized") + + url = f"{self.TELEGRAM_API_BASE}/bot{token}/sendMessage" + payload = dict(candidate_payload) + payload["chat_id"] = chat_id + response = await self._http_client.post(url, json=payload) response.raise_for_status() result = response.json() + route_receipt["provider_send_performed"] = True + if isinstance(result, dict): + result["_awoooi_canonical_route_receipt"] = route_receipt result_val = result.get("result") if isinstance(result, dict) else None if isinstance(result_val, dict) and "message_id" in result_val: await self._mirror_outbound_message( @@ -10559,6 +11646,11 @@ class TelegramGateway: self, text: str, reply_to_message_id: int | None = None, + *, + product_id: str | None = None, + signal_family: str | None = None, + severity: str | None = None, + inbound_chat_id: str | int | None = None, ) -> dict: """ 用 @OpenClawAwoooI_Bot 在群組發言 @@ -10570,21 +11662,30 @@ class TelegramGateway: Returns: dict: Telegram API 回應 """ - if not settings.OPENCLAW_BOT_TOKEN or not settings.SRE_GROUP_CHAT_ID: - logger.warning("send_as_openclaw_skipped", reason="OPENCLAW_BOT_TOKEN or SRE_GROUP_CHAT_ID not configured") - return {} + if not settings.OPENCLAW_BOT_TOKEN: + logger.warning("send_as_openclaw_skipped", reason="OPENCLAW_BOT_TOKEN not configured") + return self._blocked_route_receipt("bot_binding_unconfigured") return await self._send_as_bot( token=settings.OPENCLAW_BOT_TOKEN, - chat_id=settings.SRE_GROUP_CHAT_ID, text=text, reply_to_message_id=reply_to_message_id, + product_id=product_id, + signal_family=signal_family, + severity=severity, + actual_bot_alias="openclaw_bot", + inbound_chat_id=inbound_chat_id, ) async def send_as_nemotron( self, text: str, reply_to_message_id: int | None = None, + *, + product_id: str | None = None, + signal_family: str | None = None, + severity: str | None = None, + inbound_chat_id: str | int | None = None, ) -> dict: """ 用 @NemoTronAwoooI_Bot 在群組發言 @@ -10596,21 +11697,27 @@ class TelegramGateway: Returns: dict: Telegram API 回應 """ - if not settings.NEMOTRON_BOT_TOKEN or not settings.SRE_GROUP_CHAT_ID: - logger.warning("send_as_nemotron_skipped", reason="NEMOTRON_BOT_TOKEN or SRE_GROUP_CHAT_ID not configured") - return {} + if not settings.NEMOTRON_BOT_TOKEN: + logger.warning("send_as_nemotron_skipped", reason="NEMOTRON_BOT_TOKEN not configured") + return self._blocked_route_receipt("bot_binding_unconfigured") return await self._send_as_bot( token=settings.NEMOTRON_BOT_TOKEN, - chat_id=settings.SRE_GROUP_CHAT_ID, text=text, reply_to_message_id=reply_to_message_id, + product_id=product_id, + signal_family=signal_family, + severity=severity, + actual_bot_alias="nemotron_bot", + inbound_chat_id=inbound_chat_id, ) async def trigger_group_ai_discussion( self, alert_message_id: int, alert_summary: str, + *, + inbound_chat_id: str | int | None = None, ) -> None: """ 觸發群組 AI 並行分析（三頭政治核心流程） @@ -10626,6 +11733,13 @@ class TelegramGateway: alert_message_id: 告警訊息的 message_id（兩個 Bot 回覆的起點） alert_summary: 告警摘要文字（提供給 AI 分析用） """ + if not inbound_chat_id: + logger.warning( + "trigger_group_ai_discussion_missing_verified_inbound_no_send", + alert_message_id=alert_message_id, + ) + return + try: from src.services.chat_manager import ChatManager # noqa: PLC0415 except ImportError: @@ -10649,11 +11763,18 @@ class TelegramGateway: ) if openclaw_analysis and not isinstance(openclaw_analysis, Exception): - await self.send_as_openclaw( + result = await self.send_as_openclaw( text=f"🦞 OpenClaw 分析\n\n{openclaw_analysis}", reply_to_message_id=alert_message_id, + inbound_chat_id=inbound_chat_id, ) - logger.info("group_ai_discussion_openclaw_sent") + if _telegram_send_delivery_succeeded(result): + logger.info("group_ai_discussion_openclaw_sent") + else: + logger.warning( + "group_ai_discussion_openclaw_blocked_no_send", + alert_message_id=alert_message_id, + ) else: logger.warning("trigger_group_ai_discussion_openclaw_empty") @@ -10998,6 +12119,7 @@ class TelegramGateway: message_id=message_id, original_text=original_text, username=username, + chat_id=chat_id, ) if result.get("success"): @@ -11008,6 +12130,7 @@ class TelegramGateway: user_id=user_id, username=username, message_id=message_id, + inbound_chat_id=chat_id, ) async def _handle_chat_message(self, update_id: int, message: dict) -> None: @@ -11034,6 +12157,7 @@ class TelegramGateway: await svc.download_and_analyze( chat_id=str(chat_id), file_id=file_id, + original_message_id=message_id, question=caption, ) except Exception as _img_err: @@ -11079,11 +12203,25 @@ class TelegramGateway: whitelist = settings.get_tg_user_whitelist() if not whitelist or user_id not in whitelist: logger.warning("telegram_ai_command_unauthorized", user_id=user_id, whitelist_empty=not whitelist) - await self.send_notification("⛔ 未授權：/ai 指令僅限白名單用戶", parse_mode="HTML", chat_id=chat_id) + await self.send_notification( + "⛔ 未授權：/ai 指令僅限白名單用戶", + parse_mode="HTML", + chat_id=chat_id, + interaction_kind="operator_command_reply", + inbound_chat_id=chat_id, + inbound_message_id=message_id, + ) return from src.services.ai_control import handle_ai_command response = await handle_ai_command(text.strip()) - await self.send_notification(response, parse_mode="HTML", chat_id=chat_id) + await self.send_notification( + response, + parse_mode="HTML", + chat_id=chat_id, + interaction_kind="operator_command_reply", + inbound_chat_id=chat_id, + inbound_message_id=message_id, + ) logger.info("telegram_ai_command_handled", user_id=user_id, text=text[:50]) return @@ -11097,7 +12235,14 @@ class TelegramGateway: username=username, message_text=text, ) - await self.send_notification(response, parse_mode="HTML", chat_id=chat_id) + await self.send_notification( + response, + parse_mode="HTML", + chat_id=chat_id, + interaction_kind="operator_command_reply", + inbound_chat_id=chat_id, + inbound_message_id=message_id, + ) async def _handle_group_message( self, @@ -11213,6 +12358,7 @@ class TelegramGateway: await self.send_as_openclaw( text=f"🦞 OpenClaw\n\n{body}", reply_to_message_id=message_id, + inbound_chat_id=chat_id, ) elif mention_nemo and not mention_openclaw: @@ -11225,6 +12371,7 @@ class TelegramGateway: await self.send_as_nemotron( text=f"🤖 NemoClaw\n\n{body}", reply_to_message_id=message_id, + inbound_chat_id=chat_id, ) else: @@ -11241,6 +12388,7 @@ class TelegramGateway: await self.send_as_openclaw( text=f"🦞 OpenClaw\n\n{_clean_ai_reply(oc_result)}", reply_to_message_id=message_id, + inbound_chat_id=chat_id, ) if nemo_result and not isinstance(nemo_result, Exception): @@ -11248,6 +12396,7 @@ class TelegramGateway: await self.send_as_nemotron( text=f"🤖 NemoClaw\n\n{nemo_body}", reply_to_message_id=message_id, + inbound_chat_id=chat_id, ) logger.info("group_message_handled", user_id=user_id, text=text[:50]) @@ -11284,7 +12433,11 @@ class TelegramGateway: msg = "\n".join(lines) except Exception as e: msg = f"🖥 Cluster 狀態\n⚠️ 無法取得: {e}" - await self.send_as_openclaw(text=msg, reply_to_message_id=message_id) + await self.send_as_openclaw( + text=msg, + reply_to_message_id=message_id, + inbound_chat_id=_chat_id, + ) elif cmd == "/incidents": try: @@ -11299,7 +12452,11 @@ class TelegramGateway: msg = "🚨 活躍告警\n✅ 目前無告警" except Exception as e: msg = f"🚨 活躍告警\n⚠️ 無法取得: {e}" - await self.send_as_openclaw(text=msg, reply_to_message_id=message_id) + await self.send_as_openclaw( + text=msg, + reply_to_message_id=message_id, + inbound_chat_id=_chat_id, + ) elif cmd == "/cost": redis = get_redis() @@ -11316,7 +12473,11 @@ class TelegramGateway: ) except Exception as e: msg = f"💰 費用統計\n⚠️ 無法取得: {e}" - await self.send_as_openclaw(text=msg, reply_to_message_id=message_id) + await self.send_as_openclaw( + text=msg, + reply_to_message_id=message_id, + inbound_chat_id=_chat_id, + ) elif cmd == "/pods": try: @@ -11332,7 +12493,11 @@ class TelegramGateway: msg = "⚠️ 異常 Pod\n✅ 全部 Pod 正常" except Exception as e: msg = f"⚠️ 異常 Pod\n⚠️ 無法取得: {e}" - await self.send_as_openclaw(text=msg, reply_to_message_id=message_id) + await self.send_as_openclaw( + text=msg, + reply_to_message_id=message_id, + inbound_chat_id=_chat_id, + ) elif cmd == "/rag": # /rag <查詢內容> — RAG 知識庫語義查詢 (ADR-067 Phase 33) @@ -11342,12 +12507,14 @@ class TelegramGateway: await self.send_as_openclaw( text="📚 RAG 知識庫查詢\n用法: /rag 你的問題\n例如: /rag 什麼是 ADR-067？", reply_to_message_id=message_id, + inbound_chat_id=_chat_id, ) return question = parts[1].strip() await self.send_as_openclaw( text=f"📚 查詢知識庫中...\n{question[:80]}", reply_to_message_id=message_id, + inbound_chat_id=_chat_id, ) try: from src.services.knowledge_rag_service import get_knowledge_rag_service @@ -11357,7 +12524,11 @@ class TelegramGateway: except Exception as e: logger.warning("rag_telegram_query_failed", error=str(e)) msg = f"📚 RAG 查詢失敗\n{e}" - await self.send_as_openclaw(text=msg, reply_to_message_id=message_id) + await self.send_as_openclaw( + text=msg, + reply_to_message_id=message_id, + inbound_chat_id=_chat_id, + ) elif cmd == "/help": msg = ( @@ -11374,7 +12545,11 @@ class TelegramGateway: "• 小賀 或 @NemoTronAwoooI_Bot → 只有 NemoClaw\n" "• Reply 某個 Bot 的訊息 → 只有那個 Bot 回應" ) - await self.send_as_openclaw(text=msg, reply_to_message_id=message_id) + await self.send_as_openclaw( + text=msg, + reply_to_message_id=message_id, + inbound_chat_id=_chat_id, + ) else: logger.debug("group_unknown_command", cmd=cmd) @@ -11394,6 +12569,8 @@ class TelegramGateway: action: str, username: str, execution_triggered: bool, + inbound_chat_id: str | int, + inbound_message_id: int, approval_action: str | None = None, repair_candidate_work_item_href: str = "", repair_candidate_work_item_id: str = "", @@ -11404,30 +12581,47 @@ class TelegramGateway: 策略： 1. editMessageReplyMarkup — 移除批准/拒絕按鈕，保留資訊按鈕 2. sendMessage reply_to → 在原訊息下方附加狀態行 - 3. 如果 message_id 找不到，fallback 到 send_notification + 3. HTML 回覆失敗只可對同一 inbound parent 重試純文字 + 4. 重試仍失敗寫 durable callback-failure receipt，不另發共享 SRE """ import html as _html - chat_id = self.alert_chat_id - if not chat_id: - return + chat_id = str(inbound_chat_id or "") + try: + exact_parent_message_id = int(inbound_message_id) + except (TypeError, ValueError): + exact_parent_message_id = 0 + try: + supplied_message_id = int(message_id) if message_id is not None else 0 + except (TypeError, ValueError): + supplied_message_id = 0 - # 找到原始告警訊息 ID（優先 Redis，fallback DB） - orig_msg_id = message_id - if not orig_msg_id: - try: - redis = await get_redis() - _val = await redis.get(f"tg_msg:{incident_id}") - if _val: - orig_msg_id = int(_val) - else: - # DB fallback - from src.services.approval_db import get_approval_service as _svc - _approvals = await _svc().get_all_approvals(incident_id=incident_id) - if _approvals and _approvals[0].telegram_message_id: - orig_msg_id = _approvals[0].telegram_message_id - except Exception: - pass + if ( + not chat_id + or exact_parent_message_id <= 0 + or ( + supplied_message_id > 0 + and supplied_message_id != exact_parent_message_id + ) + ): + mismatch_error = TelegramGatewayError( + "missing_or_mismatched_verified_approval_callback_parent" + ) + await self._record_callback_reply_failure( + chat_id=chat_id, + incident_id=incident_id, + failure_context="approval_result_reply", + callback_action=action, + chunk_index=0, + chunk_count=1, + error=mismatch_error, + parent_message_id=( + exact_parent_message_id + if exact_parent_message_id > 0 + else None + ), + ) + return if action == "approve": status_emoji = "✅" @@ -11458,71 +12652,133 @@ class TelegramGateway: status_line = f"{status_emoji} {status_text}　{suffix}".strip() - if orig_msg_id: - try: - # 1. 移除批准/拒絕按鈕（只保留資訊按鈕列） - if no_action_approval: - no_action_first_row: list[dict[str, str]] = [] - work_item_deep_link = _repair_candidate_work_item_url( - incident_id=incident_id, - repair_candidate_work_item_href=repair_candidate_work_item_href, - repair_candidate_work_item_id=repair_candidate_work_item_id, - ) - if work_item_deep_link: - no_action_first_row.append({ - "text": "🧾 Work Item", - "url": work_item_deep_link, - }) - no_action_first_row.append( - {"text": "🧰 處置包", "callback_data": f"detail:{incident_id}"} - ) - info_buttons = [ - no_action_first_row, - [ - {"text": "🔄 重診", "callback_data": f"reanalyze:{incident_id}"}, - {"text": "📊 歷史", "callback_data": f"history:{incident_id}"}, - ], - ] - else: - info_buttons = [[ - {"text": "📋 詳情", "callback_data": f"detail:{incident_id}"}, - {"text": "📊 歷史", "callback_data": f"history:{incident_id}"}, - ]] - awooop_row = _awooop_truth_chain_button_row(incident_id) - if awooop_row: - info_buttons.append(awooop_row) - await self._send_request( - "editMessageReplyMarkup", - { - "chat_id": chat_id, - "message_id": orig_msg_id, - "reply_markup": {"inline_keyboard": info_buttons}, - }, - ) - except Exception as _e: - # 2026-04-09 Claude Sonnet 4.6: I3 架構Review修復 — 加 warning 防止靜默失敗 - logger.warning("notify_approval_edit_keyboard_failed", incident_id=incident_id, error=str(_e)) - - try: - # 2. 在原訊息下回覆狀態 - await self._send_request( - "sendMessage", - { - "chat_id": chat_id, - "text": status_line, - "parse_mode": "HTML", - "reply_to_message_id": orig_msg_id, - }, - ) - return - except Exception as _e: - logger.warning("notify_approval_reply_failed", incident_id=incident_id, error=str(_e)) - - # fallback: 發新通知 try: - await self.send_alert_notification(status_line, parse_mode="HTML") + # 1. 移除批准/拒絕按鈕（只保留資訊按鈕列） + if no_action_approval: + no_action_first_row: list[dict[str, str]] = [] + work_item_deep_link = _repair_candidate_work_item_url( + incident_id=incident_id, + repair_candidate_work_item_href=repair_candidate_work_item_href, + repair_candidate_work_item_id=repair_candidate_work_item_id, + ) + if work_item_deep_link: + no_action_first_row.append({ + "text": "🧾 Work Item", + "url": work_item_deep_link, + }) + no_action_first_row.append( + {"text": "🧰 處置包", "callback_data": f"detail:{incident_id}"} + ) + info_buttons = [ + no_action_first_row, + [ + {"text": "🔄 重診", "callback_data": f"reanalyze:{incident_id}"}, + {"text": "📊 歷史", "callback_data": f"history:{incident_id}"}, + ], + ] + else: + info_buttons = [[ + {"text": "📋 詳情", "callback_data": f"detail:{incident_id}"}, + {"text": "📊 歷史", "callback_data": f"history:{incident_id}"}, + ]] + awooop_row = _awooop_truth_chain_button_row(incident_id) + if awooop_row: + info_buttons.append(awooop_row) + await self._send_request( + "editMessageReplyMarkup", + { + "chat_id": chat_id, + "message_id": exact_parent_message_id, + "reply_markup": {"inline_keyboard": info_buttons}, + }, + ) except Exception as _e: - logger.warning("notify_approval_fallback_failed", incident_id=incident_id, error=str(_e)) + logger.warning( + "notify_approval_edit_keyboard_failed", + incident_id=incident_id, + error=str(_e), + ) + + interaction_context = { + "interaction_kind": "callback_reply", + "inbound_chat_id": chat_id, + "inbound_message_id": exact_parent_message_id, + } + primary_source_extra = _callback_reply_source_envelope_extra( + incident_id=incident_id, + failure_context="approval_result_reply", + status="callback_reply_sent", + chunk_index=0, + chunk_count=1, + callback_action=action, + parse_mode="HTML", + parent_message_id=exact_parent_message_id, + ) + try: + primary_result = await self._send_request( + "sendMessage", + { + "chat_id": chat_id, + "text": status_line, + "parse_mode": "HTML", + "reply_to_message_id": exact_parent_message_id, + _NON_MONITORING_INTERACTION_KEY: interaction_context, + _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY: primary_source_extra, + }, + ) + if not _telegram_send_delivery_succeeded(primary_result): + raise TelegramGatewayError( + "approval_callback_html_delivery_not_proven" + ) + return + except Exception as primary_error: + logger.warning( + "notify_approval_reply_failed", + incident_id=incident_id, + error=str(primary_error), + ) + + fallback_source_extra = _callback_reply_source_envelope_extra( + incident_id=incident_id, + failure_context="approval_result_reply", + status="callback_reply_fallback_sent", + chunk_index=0, + chunk_count=1, + callback_action=action, + parent_message_id=exact_parent_message_id, + ) + try: + fallback_result = await self._send_request( + "sendMessage", + { + "chat_id": chat_id, + "text": _plain_text_from_html(status_line, limit=500), + "reply_to_message_id": exact_parent_message_id, + _NON_MONITORING_INTERACTION_KEY: interaction_context, + _AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY: fallback_source_extra, + }, + ) + if not _telegram_send_delivery_succeeded(fallback_result): + raise TelegramGatewayError( + "approval_callback_plain_delivery_not_proven" + ) + return + except Exception as fallback_error: + logger.warning( + "notify_approval_exact_fallback_failed_no_send", + incident_id=incident_id, + error=str(fallback_error), + ) + await self._record_callback_reply_failure( + chat_id=chat_id, + incident_id=incident_id, + failure_context="approval_result_reply", + callback_action=action, + chunk_index=0, + chunk_count=1, + error=fallback_error, + parent_message_id=exact_parent_message_id, + ) async def _execute_approval_action( self, @@ -11531,6 +12787,7 @@ class TelegramGateway: user_id: int, username: str, message_id: int, # noqa: ARG002 + inbound_chat_id: str | int | None = None, ) -> None: """ 執行簽核動作 (更新資料庫) @@ -11603,6 +12860,8 @@ class TelegramGateway: action="approve", username=username, execution_triggered=execution_triggered and _execution_allowed, + inbound_chat_id=inbound_chat_id or "", + inbound_message_id=message_id, approval_action=getattr(approval, "action", None), repair_candidate_work_item_href=_work_item_href, repair_candidate_work_item_id=_work_item_id, @@ -11617,6 +12876,8 @@ class TelegramGateway: action="approve", username=username, execution_triggered=False, + inbound_chat_id=inbound_chat_id or "", + inbound_message_id=message_id, approval_action=getattr(approval, "action", None), repair_candidate_work_item_href=_work_item_href, repair_candidate_work_item_id=_work_item_id, @@ -11625,9 +12886,14 @@ class TelegramGateway: # 告警已是 execution_failed / execution_success / rejected 等終態 try: await self._send_request("sendMessage", { - "chat_id": self.alert_chat_id, + "chat_id": inbound_chat_id or "", "text": f"ℹ️ 此告警已處理（狀態：{status_val}），無法重複批准 by @{username}", "reply_to_message_id": message_id, + _NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "callback_reply", + "inbound_chat_id": str(inbound_chat_id or ""), + "inbound_message_id": message_id, + }, }) except Exception as _ne: logger.warning("telegram_approval_already_resolved_notify_failed", error=str(_ne)) @@ -11692,6 +12958,8 @@ class TelegramGateway: action="reject", username=username, execution_triggered=False, + inbound_chat_id=inbound_chat_id or "", + inbound_message_id=message_id, ) try: from src.services.incident_approval_service import ( @@ -11859,21 +13127,31 @@ class TelegramGateway: text = report_to_telegram_html(report) - # 只發到 AwoooI SRE 戰情室；缺設定時不得旁路到個人或其他群組。 - if not settings.SRE_GROUP_CHAT_ID: - logger.warning( - "telegram_heartbeat_skipped", - reason="SRE_GROUP_CHAT_ID not configured", + # 警告態屬 shared infrastructure P1；健康 heartbeat 不得進 SRE。 + route_result = await self.send_to_group( + text=text, + product_id="awoooi", + signal_family=( + "shared_infrastructure" + if report.warnings + else "product_raw_monitoring" + ), + severity="P1" if report.warnings else "P3", + ) + if not _telegram_send_delivery_succeeded(route_result): + logger.info( + "telegram_heartbeat_policy_blocked", + warnings=len(report.warnings), + delivery_status=route_result.get("_awooop_delivery_status"), ) return False - await self.send_to_group(text=text) self._last_message_time = datetime.now(UTC) logger.info( - "telegram_heartbeat_sent", + "telegram_heartbeat_routed", warnings=len(report.warnings), warnings_hash=warnings_hash, - has_sre_group=bool(settings.SRE_GROUP_CHAT_ID), + delivery_status=route_result.get("_awooop_delivery_status", "sent"), ) return True diff --git a/apps/api/src/services/weekly_report_service.py b/apps/api/src/services/weekly_report_service.py index ea959912b..19155c0a7 100644 --- a/apps/api/src/services/weekly_report_service.py +++ b/apps/api/src/services/weekly_report_service.py @@ -24,17 +24,21 @@ import json import subprocess from datetime import datetime, timedelta from pathlib import Path +from typing import Protocol, runtime_checkable from urllib.parse import quote, urlencode from urllib.request import Request, urlopen -from typing import Protocol, runtime_checkable - -import structlog from zoneinfo import ZoneInfo +import structlog + from src.core.config import settings -from src.services.stats_service import StatsService, get_stats_service from src.services.k3s_monitor_service import K3sMonitorService, get_k3s_monitor_service -from src.services.telegram_gateway import WeeklyReportMessage, get_telegram_gateway +from src.services.stats_service import StatsService, get_stats_service +from src.services.telegram_gateway import ( + WeeklyReportMessage, + _telegram_send_delivery_succeeded, + get_telegram_gateway, +) logger = structlog.get_logger(__name__) @@ -449,14 +453,17 @@ class WeeklyReportService: # 取得 Telegram Gateway gateway = get_telegram_gateway() - if not gateway._initialized: - await gateway.initialize() # 發送訊息 formatted = report.format() - result = await gateway.send_text(formatted) + result = await gateway.send_text( + formatted, + product_id="awoooi", + signal_family="product_raw_monitoring", + severity="P3", + ) - if result: + if _telegram_send_delivery_succeeded(result): logger.info("weekly_report_sent", week=report.week_range) return True else: diff --git a/apps/api/tests/test_alert_converged_recurrence.py b/apps/api/tests/test_alert_converged_recurrence.py index 5bab17725..ed2863f61 100644 --- a/apps/api/tests/test_alert_converged_recurrence.py +++ b/apps/api/tests/test_alert_converged_recurrence.py @@ -20,8 +20,8 @@ class _FakeGateway: self.primary_messages = [] self.private_messages = [] - async def send_alert_notification(self, text): - self.primary_messages.append(text) + async def send_alert_notification(self, text, **route): + self.primary_messages.append({"text": text, "route": route}) return {"ok": True} async def send_notification(self, text, *, chat_id=None): @@ -150,4 +150,9 @@ async def test_converged_recurrence_does_not_mirror_to_private_chat(monkeypatch) ) assert len(gateway.primary_messages) == 1 + assert gateway.primary_messages[0]["route"] == { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + } assert gateway.private_messages == [] diff --git a/apps/api/tests/test_ansible_verified_closure.py b/apps/api/tests/test_ansible_verified_closure.py index de64fff4a..f43b2eff5 100644 --- a/apps/api/tests/test_ansible_verified_closure.py +++ b/apps/api/tests/test_ansible_verified_closure.py @@ -11,6 +11,7 @@ import pytest from src.core import redis_client as redis_client_module from src.jobs import awooop_ansible_candidate_backfill_job as candidate_job from src.services import awooop_ansible_check_mode_service as service +from src.services import telegram_gateway as telegram_gateway_module from src.services.telegram_gateway import TelegramGateway from src.workers import ansible_executor_broker as broker @@ -89,6 +90,11 @@ def _controlled_result_payload() -> dict: return { "chat_id": "sre-chat", "text": "CONTROLLED APPLY RESULT INC-20260711-D037E5", + "_awoooi_canonical_telegram_route": { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + }, "_skip_incident_thread_reply": True, "_awooop_source_envelope_extra": { "callback_reply": { @@ -749,6 +755,11 @@ async def test_controlled_result_pending_reservation_never_resends_provider( return _Response() gateway = TelegramGateway() + monkeypatch.setattr( + telegram_gateway_module.settings, + "SRE_GROUP_CHAT_ID", + "sre-chat", + ) gateway._initialized = True client = _Client() gateway._http_client = client @@ -801,6 +812,11 @@ async def test_controlled_result_reuses_durable_sent_reservation( monkeypatch: pytest.MonkeyPatch, ) -> None: gateway = TelegramGateway() + monkeypatch.setattr( + telegram_gateway_module.settings, + "SRE_GROUP_CHAT_ID", + "sre-chat", + ) gateway._initialized = True client = AsyncMock() gateway._http_client = client diff --git a/apps/api/tests/test_channel_hub_grouped_alert_events.py b/apps/api/tests/test_channel_hub_grouped_alert_events.py index 9f874ff5a..17d10478e 100644 --- a/apps/api/tests/test_channel_hub_grouped_alert_events.py +++ b/apps/api/tests/test_channel_hub_grouped_alert_events.py @@ -21,6 +21,8 @@ from src.services.channel_hub import ( record_outbound_message, ) +_FAKE_SECRET_SHAPED_VALUE = "1234567890:" + "abcdefghijklmnopqrstuvwxyzABCDEFGH" + def test_db_timestamp_now_is_naive_utc_for_asyncpg() -> None: assert _db_timestamp_now().tzinfo is None @@ -37,6 +39,9 @@ async def test_send_telegram_interim_uses_gateway_mirror_path(monkeypatch) -> No chat_id: str | int | None = None, parse_mode: str = "HTML", disable_web_page_preview: bool = True, + interaction_kind: str | None = None, + inbound_chat_id: str | int | None = None, + inbound_message_id: int | None = None, ) -> dict[str, object]: calls.append( { @@ -44,31 +49,91 @@ async def test_send_telegram_interim_uses_gateway_mirror_path(monkeypatch) -> No "chat_id": chat_id, "parse_mode": parse_mode, "disable_web_page_preview": disable_web_page_preview, + "interaction_kind": interaction_kind, + "inbound_chat_id": inbound_chat_id, + "inbound_message_id": inbound_message_id, } ) - return {"ok": True, "result": {"message_id": 42}} + return { + "ok": True, + "result": {"message_id": 42}, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + } monkeypatch.setattr( "src.services.telegram_gateway.get_telegram_gateway", lambda: _FakeGateway(), ) - await _send_telegram_interim( + delivered = await _send_telegram_interim( channel_chat_id="123456", content="AI 正在分析中，請稍候...", run_id=uuid4(), + inbound_message_id=321, ) + assert delivered is True assert calls == [ { "text": "AI 正在分析中，請稍候...", "chat_id": "123456", "parse_mode": "HTML", "disable_web_page_preview": True, + "interaction_kind": "operator_command_reply", + "inbound_chat_id": "123456", + "inbound_message_id": 321, } ] +async def test_send_telegram_interim_fails_closed_without_verified_parent( + monkeypatch, +) -> None: + class _UnexpectedGateway: + async def send_text(self, *args, **kwargs): + raise AssertionError("gateway must not be called") + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: _UnexpectedGateway(), + ) + + delivered = await _send_telegram_interim( + channel_chat_id="123456", + content="must remain no-send", + run_id=uuid4(), + ) + + assert delivered is False + + +async def test_send_telegram_interim_rejects_structured_no_send( + monkeypatch, +) -> None: + class _BlockedGateway: + async def send_text(self, *args, **kwargs): + return { + "ok": False, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + } + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: _BlockedGateway(), + ) + + delivered = await _send_telegram_interim( + channel_chat_id="123456", + content="must remain no-send", + run_id=uuid4(), + inbound_message_id=321, + ) + + assert delivered is False + + def test_channel_hub_does_not_embed_direct_telegram_bot_api() -> None: source = inspect.getsource(_send_telegram_interim) assert "TELEGRAM_BOT_TOKEN" not in source @@ -106,7 +171,10 @@ def test_build_alertmanager_provider_event_id_keeps_fingerprint() -> None: "incident_linked", ) - assert event_id == "alertmanager:incident_linked:alert-20260513213000:abcdef1234567890abcdef1234567890" + assert event_id == ( + "alertmanager:incident_linked:alert-20260513213000:" + + "abcdef1234567890abcdef1234567890" + ) assert len(event_id) < 256 @@ -142,7 +210,10 @@ def test_build_inbound_source_envelope_redacts_and_keeps_refs() -> None: stage="received", provider_event_id="alertmanager:received:alert-1:fp-1", raw_event_id="alert-1", - raw_content="ACTION REQUIRED INC-20260513-ABCDEF token 1234567890:abcdefghijklmnopqrstuvwxyzABCDEFGH", + raw_content=( + "ACTION REQUIRED INC-20260513-ABCDEF token " + + _FAKE_SECRET_SHAPED_VALUE + ), alertname="DockerContainerUnhealthy", severity="warning", namespace="default", @@ -272,7 +343,7 @@ async def test_mirror_inbound_event_writes_redacted_replay_fields() -> None: stage="received", provider_event_id="sentry:received:123", raw_event_id="123", - raw_content="Sentry token 1234567890:abcdefghijklmnopqrstuvwxyzABCDEFGH", + raw_content="Sentry token " + _FAKE_SECRET_SHAPED_VALUE, alertname="Sentry issue", severity="error", namespace="sentry", @@ -285,7 +356,7 @@ async def test_mirror_inbound_event_writes_redacted_replay_fields() -> None: channel_type="internal", provider_event_id="sentry:received:123", platform_subject_id="sentry", - raw_content="Sentry token 1234567890:abcdefghijklmnopqrstuvwxyzABCDEFGH", + raw_content="Sentry token " + _FAKE_SECRET_SHAPED_VALUE, source_envelope=envelope, ) diff --git a/apps/api/tests/test_failover_alerter.py b/apps/api/tests/test_failover_alerter.py index e8ec893bf..f7f69a3cb 100644 --- a/apps/api/tests/test_failover_alerter.py +++ b/apps/api/tests/test_failover_alerter.py @@ -46,16 +46,15 @@ def mock_redis(): @pytest.fixture def mock_telegram_send(): - """Mock TelegramGateway.send_alert_notification + settings.SRE_GROUP_CHAT_ID + """Mock canonical TelegramGateway send and destination resolution. `_send()` 在函式內 inline import，必須 mock 來源 module 而非 alerter module。 """ - with patch("src.services.telegram_gateway.get_telegram_gateway") as mock_gw, \ - patch("src.core.config.get_settings") as mock_settings: + with patch("src.services.telegram_gateway.get_telegram_gateway") as mock_gw: gateway = MagicMock() gateway.send_alert_notification = AsyncMock() + gateway.canonical_destination_chat_id.return_value = "sre-room" mock_gw.return_value = gateway - mock_settings.return_value = MagicMock(SRE_GROUP_CHAT_ID="-100123", OPENCLAW_TG_CHAT_ID="-100456") yield gateway @@ -117,12 +116,11 @@ async def test_alert_recovery_send(mock_redis, mock_telegram_send): async def test_no_telegram_chat_id_noop(mock_redis): alerter = FailoverAlerter(redis_client=mock_redis) - with patch("src.services.telegram_gateway.get_telegram_gateway") as mock_gw, \ - patch("src.core.config.get_settings") as mock_settings: + with patch("src.services.telegram_gateway.get_telegram_gateway") as mock_gw: gateway = MagicMock() gateway.send_alert_notification = AsyncMock() + gateway.canonical_destination_chat_id.return_value = "" mock_gw.return_value = gateway - mock_settings.return_value = MagicMock(SRE_GROUP_CHAT_ID=None, OPENCLAW_TG_CHAT_ID=None) # 不該 raise，dedup pass 但 send 因 chat_id 缺直接 return await alerter.alert_failover({"to_provider": "gemini"}) @@ -244,11 +242,12 @@ def test_lines_from_list_escapes_markdown_v2_numbered_periods() -> None: def test_sanitize_telegram_error_redacts_bot_token_url() -> None: - raw = "HTTP error for https://api.telegram.org/bot123456:SECRET/sendMessage" + placeholder = "placeholder-value" + raw = "HTTP error for https://api.telegram.org/" + f"bot{placeholder}/sendMessage" sanitized = _sanitize_telegram_error(raw) - assert "SECRET" not in sanitized + assert placeholder not in sanitized assert "bot" in sanitized diff --git a/apps/api/tests/test_notification_matrix_group_cutover.py b/apps/api/tests/test_notification_matrix_group_cutover.py index ca3a6b8d4..cff38cd53 100644 --- a/apps/api/tests/test_notification_matrix_group_cutover.py +++ b/apps/api/tests/test_notification_matrix_group_cutover.py @@ -1,31 +1,56 @@ from src.services.notification_matrix import resolve_chat_ids -def test_all_alert_types_resolve_to_sre_group_first() -> None: +def test_only_shared_incident_types_resolve_to_sre_group() -> None: for notification_type in [ - "TYPE-1", "TYPE-2", "TYPE-3", "TYPE-4", "TYPE-4D", "TYPE-5S", - "TYPE-6B", "TYPE-7E", "TYPE-8M", - "UNKNOWN", ]: assert resolve_chat_ids( notification_type, - dm_chat_id="5619078117", - group_chat_id="-1003711974679", + dm_chat_id="operator-dm-id", + group_chat_id="sre-room-id", tg_group_cutover=False, - ) == ["-1003711974679"] + ) == ["sre-room-id"] + + +def test_info_business_and_unknown_types_are_fail_closed() -> None: + for notification_type in ["TYPE-1", "TYPE-6B", "UNKNOWN"]: + assert ( + resolve_chat_ids( + notification_type, + dm_chat_id="operator-dm-id", + group_chat_id="sre-room-id", + tg_group_cutover=False, + ) + == [] + ) + + +def test_explicit_empty_registry_does_not_fall_back_to_default() -> None: + assert ( + resolve_chat_ids( + "TYPE-5S", + dm_chat_id="operator-dm-id", + group_chat_id="sre-room-id", + registry={}, + ) + == [] + ) def test_dm_is_not_used_when_group_missing() -> None: - assert resolve_chat_ids( - "TYPE-3", - dm_chat_id="5619078117", - group_chat_id="", - tg_group_cutover=True, - ) == [] + assert ( + resolve_chat_ids( + "TYPE-3", + dm_chat_id="operator-dm-id", + group_chat_id="", + tg_group_cutover=True, + ) + == [] + ) diff --git a/apps/api/tests/test_report_generation_service.py b/apps/api/tests/test_report_generation_service.py index 087410929..47b5511c2 100644 --- a/apps/api/tests/test_report_generation_service.py +++ b/apps/api/tests/test_report_generation_service.py @@ -618,8 +618,19 @@ class TestTriggerPostmortemPersistence: op_logs: list[dict] = [] class FakeGateway: - async def send_to_group(self, text: str, parse_mode: str = "HTML") -> None: + async def send_to_group( + self, + text: str, + parse_mode: str = "HTML", + **route, + ) -> dict: sent_messages.append(text) + assert route == { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + } + return {"ok": True} class FakeKnowledgeRepo: def __init__(self, _db) -> None: diff --git a/apps/api/tests/test_telegram_button_consistency.py b/apps/api/tests/test_telegram_button_consistency.py index f96cad5f2..6e0931b00 100644 --- a/apps/api/tests/test_telegram_button_consistency.py +++ b/apps/api/tests/test_telegram_button_consistency.py @@ -179,8 +179,9 @@ class TestSREGroupCutover: ) assert match, "找不到 send_approval_card 函式" fn_body = match.group(0) - assert '"chat_id": target_chat_id' in fn_body - assert "target_chat_id = self.alert_chat_id" in fn_body + assert "_LEGACY_NOTIFICATION_TYPE_KEY" in fn_body + assert 'notification_type or "UNKNOWN"' in fn_body + assert '"chat_id": target_chat_id' not in fn_body assert '"chat_id": self.chat_id' not in fn_body def test_send_approval_card_does_not_double_send_group(self): diff --git a/apps/api/tests/test_telegram_callback_context_routing.py b/apps/api/tests/test_telegram_callback_context_routing.py new file mode 100644 index 000000000..3eca42751 --- /dev/null +++ b/apps/api/tests/test_telegram_callback_context_routing.py @@ -0,0 +1,721 @@ +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +import src.repositories.drift_repository as drift_repository +import src.repositories.incident_repository as incident_repository +import src.services.ai_advisory_helpers as ai_advisory_helpers +import src.services.approval_db as approval_db +import src.services.callback_dispatcher as callback_dispatcher +import src.services.telegram_gateway as gateway_module +from src.models.approval import ApprovalStatus +from src.models.incident import IncidentStatus +from src.services.callback_dispatcher import DispatchResult +from src.services.telegram_gateway import TelegramGateway + +_CHAT_ID = "verified-chat" +_MESSAGE_ID = 77 + + +def _assert_verified_callback_payload(payload: dict) -> None: + assert payload["chat_id"] == _CHAT_ID + assert payload["reply_to_message_id"] == _MESSAGE_ID + assert payload[gateway_module._NON_MONITORING_INTERACTION_KEY] == { + "interaction_kind": "callback_reply", + "inbound_chat_id": _CHAT_ID, + "inbound_message_id": _MESSAGE_ID, + } + + +class _DriftRepo: + def __init__(self, value=None, error: Exception | None = None) -> None: + self.value = value + self.error = error + + async def get_by_id(self, _report_id: str): + if self.error: + raise self.error + return self.value + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "repo", + [ + _DriftRepo(), + _DriftRepo(SimpleNamespace(items=[], namespace="awoooi-prod")), + _DriftRepo(error=RuntimeError("read failed")), + ], +) +async def test_drift_detail_page_and_error_keep_verified_callback_context( + monkeypatch: pytest.MonkeyPatch, + repo: _DriftRepo, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(drift_repository, "get_drift_repository", lambda: repo) + + await gateway._send_drift_diff_detail( + "drift-1", + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + send_request.assert_awaited_once() + method, payload = send_request.await_args.args + assert method == "sendMessage" + _assert_verified_callback_payload(payload) + + +@pytest.mark.asyncio +async def test_advisory_and_drift_outcome_keep_verified_callback_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway, "_answer_callback", AsyncMock()) + monkeypatch.setattr( + ai_advisory_helpers, + "handle_ai_advisory_callback", + AsyncMock(return_value={"feedback_text": "done", "success": True}), + ) + + await gateway._handle_ai_advisory_action( + action="ai_advisory_view", + advisory_payload="capacity:item-1", + callback_query_id="callback-1", + user_id=42, + username="operator", + user={"id": 42}, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + _assert_verified_callback_payload(send_request.await_args.args[1]) + + send_request.reset_mock() + + class _Redis: + async def get(self, _key: str) -> str: + return str(_MESSAGE_ID) + + monkeypatch.setattr(gateway_module, "get_redis", lambda: _Redis()) + await gateway._edit_drift_card_outcome( + report_id="drift-1", + verb="adopted", + by="operator", + ok=True, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + send_payload = next( + call.args[1] + for call in send_request.await_args_list + if call.args[0] == "sendMessage" + ) + _assert_verified_callback_payload(send_payload) + + +@pytest.mark.asyncio +async def test_drift_outcome_ignores_stale_redis_parent_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + redis_reads: list[str] = [] + + class _Redis: + async def get(self, key: str) -> str: + redis_reads.append(key) + return "999" + + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway_module, "get_redis", lambda: _Redis()) + + await gateway._edit_drift_card_outcome( + report_id="drift-stale-parent", + verb="adopted", + by="operator", + ok=True, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + assert redis_reads == ["tg_drift:drift-stale-parent"] + assert [call.args[0] for call in send_request.await_args_list] == [ + "editMessageReplyMarkup", + "sendMessage", + ] + edit_payload = send_request.await_args_list[0].args[1] + assert edit_payload["chat_id"] == _CHAT_ID + assert edit_payload["message_id"] == _MESSAGE_ID + send_payload = send_request.await_args_list[1].args[1] + _assert_verified_callback_payload(send_payload) + for call in send_request.await_args_list: + provider_payload = call.args[1] + assert provider_payload.get("message_id") != 999 + assert provider_payload.get("reply_to_message_id") != 999 + + +@pytest.mark.asyncio +async def test_llm_and_category_results_keep_verified_callback_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway._security, "is_whitelisted", lambda _user_id: True) + + class _Redis: + async def get(self, _key: str) -> str: + return json.dumps({ + "name": "inspect", + "provider": "internal", + "tool": "status", + "risk": "low", + "incident_id": "INC-1", + }) + + monkeypatch.setattr(gateway_module, "get_redis", lambda: _Redis()) + monkeypatch.setattr( + callback_dispatcher, + "dispatch_llm_action", + lambda _action, _context: {"ok": True}, + ) + await gateway._handle_llm_action_callback( + callback_query_id="callback-1", + callback_data="la:short", + user_id=42, + username="operator", + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + llm_payload = next( + call.args[1] + for call in send_request.await_args_list + if call.args[0] == "sendMessage" + ) + _assert_verified_callback_payload(llm_payload) + + send_request.reset_mock() + monkeypatch.setattr(gateway, "_answer_callback", AsyncMock()) + monkeypatch.setattr( + callback_dispatcher, + "get_action_spec", + lambda _action: SimpleNamespace(emoji="🔎"), + ) + monkeypatch.setattr( + callback_dispatcher, + "dispatch_action", + AsyncMock(return_value=DispatchResult( + success=True, + action="inspect", + incident_id="INC-1", + user_id=42, + result_text="result", + )), + ) + + class _IncidentRepo: + async def get_by_id(self, _incident_id: str): + return SimpleNamespace(signals=[]) + + monkeypatch.setattr( + incident_repository, + "get_incident_repository", + lambda: _IncidentRepo(), + ) + await gateway._dispatch_category_action( + callback_query_id="callback-2", + action="inspect", + incident_id="INC-1", + user_id=42, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + _assert_verified_callback_payload(send_request.await_args.args[1]) + + +@pytest.mark.asyncio +async def test_write_category_callback_keeps_verified_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr( + gateway._security, + "parse_callback_data", + lambda _data: { + "action": "restart_safe", + "approval_id": "INC-1", + "is_info_action": False, + "nonce": "nonce", + }, + ) + monkeypatch.setattr( + gateway._security, + "verify_callback", + AsyncMock(return_value={"id": 42}), + ) + monkeypatch.setattr(gateway, "_check_incident_state_guard", AsyncMock(return_value=None)) + monkeypatch.setattr(gateway, "_answer_callback", AsyncMock()) + spec = SimpleNamespace( + requires_multi_sig=False, + risk="medium", + mcp_provider="internal", + mcp_tool="restart_safe", + emoji="▶️", + label="restart", + ) + monkeypatch.setattr(callback_dispatcher, "get_action_spec", lambda _action: spec) + monkeypatch.setattr( + callback_dispatcher, + "dispatch_action", + AsyncMock(return_value=DispatchResult( + success=True, + action="restart_safe", + incident_id="INC-1", + user_id=42, + result_text="done", + )), + ) + + class _IncidentRepo: + async def get_by_id(self, _incident_id: str): + return SimpleNamespace(incident_id="INC-1", signals=[]) + + monkeypatch.setattr( + incident_repository, + "get_incident_repository", + lambda: _IncidentRepo(), + ) + + result = await gateway.handle_callback( + callback_query_id="callback-1", + callback_data="restart_safe:INC-1:nonce", + user_id=42, + message_id=_MESSAGE_ID, + username="operator", + chat_id=_CHAT_ID, + ) + + assert result["success"] is True + _assert_verified_callback_payload(send_request.await_args.args[1]) + + +@pytest.mark.asyncio +async def test_approval_replies_keep_verified_callback_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + + await gateway._notify_approval_result( + message_id=_MESSAGE_ID, + incident_id="INC-1", + action="approve", + username="operator", + execution_triggered=False, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + status_payload = next( + call.args[1] + for call in send_request.await_args_list + if call.args[0] == "sendMessage" + ) + _assert_verified_callback_payload(status_payload) + source_extra = status_payload[gateway_module._AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] + merged = gateway_module._merge_outbound_source_envelope_extra({}, source_extra) + assert merged["callback_reply"]["parent_provider_message_id"] == str( + _MESSAGE_ID + ) + + send_request.reset_mock() + approval_id = UUID("44444444-4444-4444-4444-444444444444") + terminal = SimpleNamespace( + id=approval_id, + status=ApprovalStatus.EXECUTION_SUCCESS, + action="restart_safe", + metadata={}, + ) + + class _ApprovalService: + async def sign_approval(self, **_kwargs): + return terminal, "already done", False + + monkeypatch.setattr(approval_db, "get_approval_service", lambda: _ApprovalService()) + await gateway._execute_approval_action( + action="approve", + approval_id=str(approval_id), + user_id=42, + username="operator", + message_id=_MESSAGE_ID, + inbound_chat_id=_CHAT_ID, + ) + _assert_verified_callback_payload(send_request.await_args.args[1]) + + +@pytest.mark.parametrize( + ("result", "expected"), + [ + ({"ok": True}, True), + ({"ok": True, "_awooop_delivery_status": "sent_reused"}, True), + ({"ok": False}, False), + ( + {"ok": True, "_awooop_delivery_status": "blocked_no_egress"}, + False, + ), + ({"ok": True, "_awooop_provider_send_performed": False}, False), + ( + { + "ok": True, + "_awoooi_canonical_route_receipt": { + "provider_send_performed": False, + }, + }, + False, + ), + ], +) +def test_telegram_send_delivery_requires_affirmative_provider_receipt( + result: dict, + expected: bool, +) -> None: + assert gateway_module._telegram_send_delivery_succeeded(result) is expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "primary_failure", + [ + {"ok": False}, + {"ok": True, "_awooop_delivery_status": "blocked_no_egress"}, + {"ok": True, "_awooop_provider_send_performed": False}, + { + "ok": True, + "_awoooi_canonical_route_receipt": { + "provider_send_performed": False, + }, + }, + ], +) +async def test_approval_structured_no_send_retries_plain_on_exact_parent( + monkeypatch: pytest.MonkeyPatch, + primary_failure: dict, +) -> None: + gateway = TelegramGateway() + attempts: list[tuple[str, dict]] = [] + send_results = iter([primary_failure, {"ok": True}]) + + async def structured_primary_failure(method: str, payload: dict) -> dict: + attempts.append((method, payload)) + if method == "sendMessage": + return next(send_results) + return {"ok": True} + + record_failure = AsyncMock(return_value=None) + shared_fallback = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", structured_primary_failure) + monkeypatch.setattr(gateway, "_record_callback_reply_failure", record_failure) + monkeypatch.setattr(gateway, "send_alert_notification", shared_fallback) + + await gateway._notify_approval_result( + message_id=_MESSAGE_ID, + incident_id="INC-1", + action="approve", + username="operator", + execution_triggered=False, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + send_attempts = [payload for method, payload in attempts if method == "sendMessage"] + assert len(send_attempts) == 2 + for payload in send_attempts: + _assert_verified_callback_payload(payload) + assert send_attempts[0]["parse_mode"] == "HTML" + assert "parse_mode" not in send_attempts[1] + record_failure.assert_not_awaited() + shared_fallback.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_approval_structured_double_no_send_records_durable_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + attempts: list[tuple[str, dict]] = [] + send_results = iter([ + {"ok": True, "_awooop_delivery_status": "blocked_no_egress"}, + {"ok": True, "_awooop_provider_send_performed": False}, + ]) + + async def structured_no_send(method: str, payload: dict) -> dict: + attempts.append((method, payload)) + if method == "sendMessage": + return next(send_results) + return {"ok": True} + + record_failure = AsyncMock(return_value=None) + shared_fallback = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", structured_no_send) + monkeypatch.setattr(gateway, "_record_callback_reply_failure", record_failure) + monkeypatch.setattr(gateway, "send_alert_notification", shared_fallback) + + await gateway._notify_approval_result( + message_id=_MESSAGE_ID, + incident_id="INC-1", + action="reject", + username="operator", + execution_triggered=False, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + send_attempts = [payload for method, payload in attempts if method == "sendMessage"] + assert len(send_attempts) == 2 + for payload in send_attempts: + _assert_verified_callback_payload(payload) + shared_fallback.assert_not_awaited() + record_failure.assert_awaited_once() + assert record_failure.await_args.kwargs["parent_message_id"] == _MESSAGE_ID + assert record_failure.await_args.kwargs["incident_id"] == "INC-1" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "action", + ["tune", "snooze", "silence", "approve", "reject"], +) +async def test_callback_card_edits_use_only_verified_inbound_identity( + monkeypatch: pytest.MonkeyPatch, + action: str, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + + await gateway._update_message_after_action( + action=action, + username="operator", + original_text="original", + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + assert [call.args[0] for call in send_request.await_args_list] == [ + "editMessageReplyMarkup", + "editMessageText", + ] + for call in send_request.await_args_list: + payload = call.args[1] + assert payload["chat_id"] == _CHAT_ID + assert payload["message_id"] == _MESSAGE_ID + + +@pytest.mark.asyncio +async def test_resolved_state_guard_edits_only_verified_inbound_message( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway, "_answer_callback", AsyncMock()) + approval_id = UUID("55555555-5555-5555-5555-555555555555") + + class _ApprovalService: + async def get_approval_by_id(self, _approval_id: UUID): + return SimpleNamespace(incident_id="INC-1") + + class _IncidentRepo: + async def get_by_id(self, _incident_id: str): + return SimpleNamespace( + status=IncidentStatus.RESOLVED, + resolved_at=None, + ) + + monkeypatch.setattr(approval_db, "get_approval_service", lambda: _ApprovalService()) + monkeypatch.setattr( + incident_repository, + "get_incident_repository", + lambda: _IncidentRepo(), + ) + + result = await gateway._check_incident_state_guard( + approval_id=str(approval_id), + callback_query_id="callback-1", + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + original_text="original", + ) + + assert result == { + "blocked": True, + "reason": "already_resolved", + "approval_id": str(approval_id), + } + method, payload = send_request.await_args.args + assert method == "editMessageText" + assert payload["chat_id"] == _CHAT_ID + assert payload["message_id"] == _MESSAGE_ID + + +@pytest.mark.asyncio +async def test_approval_reply_failure_retries_exact_parent_then_records_no_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + attempts: list[tuple[str, dict]] = [] + + async def fail_replies(method: str, payload: dict) -> dict: + attempts.append((method, payload)) + if method == "sendMessage": + raise gateway_module.TelegramGatewayError("provider unavailable") + return {"ok": True} + + record_failure = AsyncMock(return_value=None) + shared_fallback = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", fail_replies) + monkeypatch.setattr(gateway, "_record_callback_reply_failure", record_failure) + monkeypatch.setattr(gateway, "send_alert_notification", shared_fallback) + + await gateway._notify_approval_result( + message_id=_MESSAGE_ID, + incident_id="INC-1", + action="reject", + username="operator", + execution_triggered=False, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + send_attempts = [payload for method, payload in attempts if method == "sendMessage"] + assert len(send_attempts) == 2 + for payload in send_attempts: + _assert_verified_callback_payload(payload) + extra = payload[gateway_module._AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] + merged = gateway_module._merge_outbound_source_envelope_extra({}, extra) + assert merged["callback_reply"]["parent_provider_message_id"] == str( + _MESSAGE_ID + ) + assert "parse_mode" in send_attempts[0] + assert "parse_mode" not in send_attempts[1] + shared_fallback.assert_not_awaited() + record_failure.assert_awaited_once() + assert record_failure.await_args.kwargs["parent_message_id"] == _MESSAGE_ID + assert record_failure.await_args.kwargs["incident_id"] == "INC-1" + + +@pytest.mark.asyncio +async def test_approval_reply_parent_mismatch_is_durable_no_send_terminal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + record_failure = AsyncMock(return_value=None) + shared_fallback = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway, "_record_callback_reply_failure", record_failure) + monkeypatch.setattr(gateway, "send_alert_notification", shared_fallback) + + await gateway._notify_approval_result( + message_id=_MESSAGE_ID + 1, + incident_id="INC-1", + action="approve", + username="operator", + execution_triggered=False, + inbound_chat_id=_CHAT_ID, + inbound_message_id=_MESSAGE_ID, + ) + + send_request.assert_not_awaited() + shared_fallback.assert_not_awaited() + record_failure.assert_awaited_once() + assert record_failure.await_args.kwargs["parent_message_id"] == _MESSAGE_ID + + +class _LifecycleRedis: + async def get(self, key: str) -> str: + if key.startswith("tg_approval:") or key.startswith("tg_msg:"): + return str(_MESSAGE_ID) + raise AssertionError(key) + + async def set(self, *_args, **_kwargs) -> bool: + return True + + +@pytest.mark.asyncio +async def test_autonomous_lifecycle_threads_use_canonical_route_and_parent_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway_module, "get_redis", lambda: _LifecycleRedis()) + + assert await gateway.mark_auto_repaired("INC-1", "playbook", 10) is True + assert await gateway.append_incident_update("INC-2", "healthy") is True + assert await gateway.append_grouped_alert_digest( + incident_id="INC-3", + group_key="group-1", + digest_text="digest", + ) is True + + send_payloads = [ + call.args[1] + for call in send_request.await_args_list + if call.args[0] == "sendMessage" + ] + assert len(send_payloads) == 3 + parent_lookup_keys = [] + for payload in send_payloads: + assert gateway_module._NON_MONITORING_INTERACTION_KEY not in payload + assert payload[gateway_module._CANONICAL_ROUTE_CONTEXT_KEY] == { + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": "P1", + } + extra = payload[gateway_module._AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] + merged = gateway_module._merge_outbound_source_envelope_extra({}, extra) + receipt = merged["incident_lifecycle_parent_receipt"] + assert receipt["parent_provider_message_id"] == str(_MESSAGE_ID) + assert receipt["durable_mirror_required"] is True + parent_lookup_keys.append(receipt["parent_lookup_key"]) + assert merged["source_refs"]["parent_provider_message_ids"] == [ + str(_MESSAGE_ID) + ] + assert parent_lookup_keys == [ + "tg_approval:INC-1", + "tg_msg:INC-2", + "tg_msg:INC-3", + ] + + +@pytest.mark.asyncio +async def test_callback_fails_closed_before_dispatch_without_inbound_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + llm_handler = AsyncMock() + monkeypatch.setattr(gateway, "_handle_llm_action_callback", llm_handler) + + result = await gateway.handle_callback( + callback_query_id="callback-1", + callback_data="la:short", + user_id=42, + message_id=_MESSAGE_ID, + chat_id=None, + ) + + assert result["success"] is False + assert result["reason"] == "missing_verified_interaction_context" + llm_handler.assert_not_awaited() diff --git a/apps/api/tests/test_telegram_canonical_routing_registry.py b/apps/api/tests/test_telegram_canonical_routing_registry.py new file mode 100644 index 000000000..75f7e36c8 --- /dev/null +++ b/apps/api/tests/test_telegram_canonical_routing_registry.py @@ -0,0 +1,494 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from src.services.notification_matrix import ( + load_canonical_telegram_routing_registry, + resolve_canonical_telegram_route, +) + +EXPECTED_PRODUCTS = { + "2026fifa", + "agent-bounty-protocol", + "awooogo", + "awoooi", + "bitan-pharmacy", + "clawbot-openclaw", + "momo-pro", + "stockplatform-v2", + "tsenyang-website", + "vibework", + "vtuber", +} + + +def test_registry_covers_all_11_products_and_reconciles_rollups() -> None: + registry = load_canonical_telegram_routing_registry() + + assert { + product["product_id"] for product in registry["products"] + } == EXPECTED_PRODUCTS + assert registry["rollups"] == { + "product_count": 11, + "route_count": 20, + "allowed_route_count": 4, + "blocked_route_count": 13, + "disabled_route_count": 2, + "not_implemented_route_count": 1, + "shared_sre_allowed_route_count": 4, + "raw_numeric_destination_count": 0, + "runtime_route_change_count": 0, + "telegram_send_count": 0, + } + + +@pytest.mark.parametrize( + "signal_family", + [ + "shared_infrastructure", + "security_incident", + "disaster_recovery", + "incident_lifecycle", + ], +) +@pytest.mark.parametrize("severity", ["P0", "P1"]) +def test_shared_sre_accepts_only_allowlisted_p0_p1_lifecycle( + signal_family: str, severity: str +) -> None: + decision = resolve_canonical_telegram_route("awoooi", signal_family, severity) + + assert decision["decision"] == "allow" + assert decision["chat_alias"] == "awoooi_sre_war_room" + assert decision["allowed_destination"] == ( + "telegram_chat_alias:awoooi_sre_war_room" + ) + assert decision["receipt_backend"] == ( + "awoooi.alert_operation_log+telegram_outbound_receipts" + ) + + +@pytest.mark.parametrize( + ("signal_family", "severity"), + [ + ("product_raw_monitoring", "P2"), + ("product_raw_monitoring", "P3"), + ("marketing", "P1"), + ], +) +def test_shared_sre_blocks_raw_p2_p3_and_marketing( + signal_family: str, severity: str +) -> None: + decision = resolve_canonical_telegram_route("awoooi", signal_family, severity) + + assert decision["decision"] == "block" + assert decision["allowed_destination"] == "blocked" + + +@pytest.mark.parametrize("product_id", ["awooogo", "vibework", "vtuber"]) +def test_disabled_or_not_implemented_products_never_resolve_to_egress( + product_id: str, +) -> None: + decision = resolve_canonical_telegram_route(product_id, "any-signal", "P0") + + assert decision["decision"] == "block" + assert decision["egress_status"] in {"disabled", "not_implemented"} + assert decision["allowed_destination"] == "blocked" + + +@pytest.mark.parametrize( + ("product_id", "signal_family", "severity"), + [ + ("unknown-product", "shared_infrastructure", "P0"), + ("awoooi", "unknown-family", "P0"), + ("awoooi", "shared_infrastructure", "P3"), + ("awoooi", "shared_infrastructure", "SEV-1"), + ], +) +def test_unknown_route_dimensions_are_fail_closed( + product_id: str, signal_family: str, severity: str +) -> None: + decision = resolve_canonical_telegram_route(product_id, signal_family, severity) + + assert decision["decision"] == "block" + assert decision["route_id"] is None + assert decision["allowed_destination"] == "blocked" + assert decision["reason"] == "unknown_product_signal_or_severity" + + +def test_requested_destination_must_match_the_canonical_alias() -> None: + decision = resolve_canonical_telegram_route( + "awoooi", + "security_incident", + "P0", + requested_destination="telegram_chat_alias:some_other_room", + ) + + assert decision["decision"] == "block" + assert decision["reason"] == "requested_destination_not_allowed" + + +def test_explicit_empty_registry_is_invalid_instead_of_loading_default() -> None: + with pytest.raises(ValueError, match="schema_version"): + resolve_canonical_telegram_route( + "awoooi", + "security_incident", + "P0", + registry={}, + ) + + +def test_product_level_blocked_disposition_prevents_route_egress() -> None: + decision = resolve_canonical_telegram_route( + "agent-bounty-protocol", + "a2a_task_lifecycle", + "P1", + ) + + assert decision["decision"] == "block" + assert decision["egress_status"] == "blocked" + assert decision["allowed_destination"] == "blocked" + + +def test_registry_contains_only_symbolic_aliases_not_runtime_identifiers() -> None: + registry = load_canonical_telegram_routing_registry() + + serialized = json.dumps(registry, ensure_ascii=False) + assert '"chat_id"' not in serialized + assert '"bot_token"' not in serialized + for route in registry["routes"]: + for field in ("bot_alias", "chat_alias", "allowed_destination"): + assert not str(route[field]).lstrip("-").isdigit() + + +def test_shared_monitoring_routes_bind_tsenyang_owner_and_destination_policy() -> None: + registry = load_canonical_telegram_routing_registry() + shared_routes = [ + route + for route in registry["routes"] + if route["egress_status"] == "allowed" and route["scope"] == "shared" + ] + assert shared_routes + assert {route["bot_alias"] for route in shared_routes} == {"tsenyang_bot"} + + owner_role = next( + role + for role in registry["destination_roles"] + if role["chat_alias"] == "awoooi_sre_war_room" + and role["bot_alias"] == "tsenyang_bot" + ) + assert owner_role["bot_owner"] == "sre_shared_runtime" + assert owner_role["monitoring_owner"] is True + assert owner_role["monitoring_egress_policy"] == "allowlist_only" + + +@pytest.mark.parametrize( + ("field", "value", "error"), + [ + ("bot_owner", "ai_agent_runtime", "bot ownership"), + ("monitoring_owner", False, "not the monitoring owner"), + ( + "monitoring_egress_policy", + "interaction_reply_only_no_monitoring_ownership", + "not monitoring allowlisted", + ), + ], +) +def test_tampered_shared_destination_ownership_is_rejected( + tmp_path: Path, + field: str, + value: object, + error: str, +) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + role = next( + role + for role in registry["destination_roles"] + if role["chat_alias"] == "awoooi_sre_war_room" + and role["bot_alias"] == "tsenyang_bot" + ) + role[field] = value + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match=error): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_tampered_shared_route_cannot_switch_to_openclaw_bot(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + route = next( + route + for route in registry["routes"] + if route["route_id"] == "awoooi-security-incident" + ) + route["bot_alias"] = "openclaw_bot" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="shared SRE P0/P1"): + load_canonical_telegram_routing_registry(tmp_path) + + +@pytest.mark.parametrize("visible_label", ["TsenYang", "小O_小龍蝦"]) +def test_visible_labels_do_not_claim_proven_runtime_alias( + tmp_path: Path, + visible_label: str, +) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + role = next( + role + for role in registry["destination_roles"] + if role["visible_label"] == visible_label + ) + assert role["runtime_alias_evidence"] == "visible_label_only" + assert role["proven_runtime_alias"] == "none" + role["proven_runtime_alias"] = role["chat_alias"] + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="visible label must not claim"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_direct_egress_scanners_cover_agent99_and_reboot_recovery() -> None: + repo_root = Path(__file__).resolve().parents[3] + scanner_paths = [ + repo_root / "scripts/security/telegram-notification-egress-inventory.py", + repo_root + / "scripts/security/telegram-notification-egress-no-new-bypass-guard.py", + ] + for scanner_path in scanner_paths: + source = scanner_path.read_text(encoding="utf-8") + assert 'Path("agent99-control-plane.ps1")' in source + assert 'Path("scripts/reboot-recovery")' in source + assert '".ps1"' in source + + guarded_source = "\n".join( + [ + (repo_root / "agent99-control-plane.ps1").read_text(encoding="utf-8"), + *( + path.read_text(encoding="utf-8", errors="replace") + for path in sorted((repo_root / "scripts/reboot-recovery").rglob("*")) + if path.is_file() and path.suffix in {".ps1", ".sh", ".py"} + ), + ] + ) + assert "api.telegram.org/bot" not in guarded_source + + +def test_tampered_shared_sre_route_is_rejected(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + route = next( + route + for route in registry["routes"] + if route["route_id"] == "awoooi-shared-infrastructure" + ) + route["severity"].append("P3") + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="shared SRE P0/P1"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_tampered_allowed_route_without_receipt_is_rejected(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + route = next( + route for route in registry["routes"] if route["egress_status"] == "allowed" + ) + route["receipt_backend"] = "none" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="receipt backend"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_tampered_unknown_destination_is_rejected(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + route = next( + route for route in registry["routes"] if route["egress_status"] == "blocked" + ) + route["allowed_destination"] = "telegram_chat_alias:guessed_room" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="non-allowed route must be blocked"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_tampered_raw_chat_identifier_is_rejected(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + registry["routes"][0]["chat_alias"] = str(-(10**12)) + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="raw numeric runtime identifier"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_tampered_blocked_product_cannot_gain_an_allowed_route(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + route = next( + route + for route in registry["routes"] + if route["route_id"] == "bitan-product-health" + ) + route.update( + { + "bot_alias": "bitan_bot", + "chat_alias": "bitan_product_ops", + "allowed_destination": "telegram_chat_alias:bitan_product_ops", + "receipt_backend": "bitan.delivery_receipts", + "egress_status": "allowed", + } + ) + _write_registry(tmp_path, registry) + + with pytest.raises( + ValueError, match="bitan-pharmacy must not have an allowed route" + ): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_recursive_guard_rejects_sensitive_runtime_key(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + registry["destination_roles"][0]["bot_token"] = "redacted-is-still-forbidden" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="forbidden sensitive runtime key"): + load_canonical_telegram_routing_registry(tmp_path) + + +@pytest.mark.parametrize( + "sensitive_key", + [ + "api_key", + "password", + "credential", + "private_key", + "cookie", + "session", + ], +) +def test_recursive_guard_rejects_extended_sensitive_runtime_keys( + tmp_path: Path, + sensitive_key: str, +) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + registry["destination_roles"][0][sensitive_key] = "redacted" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="forbidden sensitive runtime key"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_recursive_guard_rejects_secret_shaped_runtime_value(tmp_path: Path) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + registry["destination_roles"][0]["role"] = f"{10**7}:{'x' * 24}" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="secret-shaped runtime value"): + load_canonical_telegram_routing_registry(tmp_path) + + +@pytest.mark.parametrize( + ("section", "field"), + [ + ("top", "status"), + ("top", "destination_roles"), + ("product", "owner"), + ("route", "signal_family"), + ("policy", "default_decision"), + ("rollups", "route_count"), + ("destination_role", "monitoring_egress_policy"), + ("supersedes", "owner"), + ("execution_boundaries", "telegram_send_allowed"), + ], +) +def test_runtime_validator_rejects_missing_fields_at_every_object_level( + tmp_path: Path, + section: str, + field: str, +) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + target = _registry_object_for_section(registry, section) + target.pop(field) + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="missing required fields"): + load_canonical_telegram_routing_registry(tmp_path) + + +@pytest.mark.parametrize( + "section", + [ + "top", + "product", + "route", + "policy", + "rollups", + "destination_role", + "supersedes", + "execution_boundaries", + ], +) +def test_runtime_validator_rejects_unexpected_fields_at_every_object_level( + tmp_path: Path, + section: str, +) -> None: + registry = copy.deepcopy(load_canonical_telegram_routing_registry()) + target = _registry_object_for_section(registry, section) + target["unexpected_field"] = "must-fail" + _write_registry(tmp_path, registry) + + with pytest.raises(ValueError, match="unexpected fields"): + load_canonical_telegram_routing_registry(tmp_path) + + +def test_schema_closes_every_data_object_and_validates_snapshot() -> None: + import jsonschema + + repo_root = Path(__file__).resolve().parents[3] + schema = json.loads( + ( + repo_root + / "docs/schemas/telegram_canonical_routing_registry_v1.schema.json" + ).read_text(encoding="utf-8") + ) + + def assert_closed_objects(node: object, path: str = "schema") -> None: + if isinstance(node, dict): + if node.get("type") == "object": + assert node.get("additionalProperties") is False, path + for key, nested in node.items(): + assert_closed_objects(nested, f"{path}.{key}") + elif isinstance(node, list): + for index, nested in enumerate(node): + assert_closed_objects(nested, f"{path}[{index}]") + + assert_closed_objects(schema) + jsonschema.Draft202012Validator.check_schema(schema) + jsonschema.Draft202012Validator(schema).validate( + load_canonical_telegram_routing_registry() + ) + + +def _registry_object_for_section(registry: dict, section: str) -> dict: + if section == "top": + return registry + if section == "product": + return next( + product + for product in registry["products"] + if product["product_id"] == "awoooi" + ) + if section == "route": + return registry["routes"][0] + if section == "destination_role": + return registry["destination_roles"][0] + return registry[section] + + +def _write_registry(directory: Path, registry: dict) -> None: + (directory / "telegram-canonical-routing-registry.snapshot.json").write_text( + json.dumps(registry), + encoding="utf-8", + ) diff --git a/apps/api/tests/test_telegram_canonical_sender_gate.py b/apps/api/tests/test_telegram_canonical_sender_gate.py new file mode 100644 index 000000000..2231e277a --- /dev/null +++ b/apps/api/tests/test_telegram_canonical_sender_gate.py @@ -0,0 +1,521 @@ +from __future__ import annotations + +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +import src.db.base as db_base +import src.services.approval_db as approval_db +import src.services.telegram_gateway as gateway_module +from src.services.telegram_gateway import TelegramGateway + + +class _FakeResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return {"ok": True, "result": {"message_id": 11}} + + +class _FakeHttpClient: + def __init__(self) -> None: + self.posts: list[tuple[str, dict]] = [] + + async def post(self, url: str, json: dict) -> _FakeResponse: + self.posts.append((url, json)) + return _FakeResponse() + + +def _prepared_gateway(monkeypatch: pytest.MonkeyPatch) -> tuple[TelegramGateway, _FakeHttpClient]: + gateway = TelegramGateway() + client = _FakeHttpClient() + gateway._initialized = True + gateway._http_client = client # type: ignore[assignment] + monkeypatch.setattr(gateway_module.settings, "SRE_GROUP_CHAT_ID", "test-room") + monkeypatch.setattr(gateway, "_attach_incident_thread_reply", AsyncMock()) + monkeypatch.setattr(gateway, "_mirror_outbound_message", AsyncMock()) + monkeypatch.setattr( + gateway._security, + "generate_callback_nonce", + lambda approval_id, action: f"{action}:test", + ) + return gateway, client + + +def _notification_message(): + from src.services.notifications.base import ExecutionStatus, NotificationMessage + + return NotificationMessage( + execution_status=ExecutionStatus.SUCCESS, + action_title="bounded test", + action_description="no provider egress", + approval_id="APR-TEST", + ) + + +@pytest.mark.asyncio +async def test_notification_provider_does_not_report_success_for_gateway_no_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services.notifications import telegram as provider_module + from src.services.notifications.base import NotificationStatus + + monkeypatch.setattr(provider_module.settings, "OPENCLAW_TG_BOT_TOKEN", "configured") + monkeypatch.setattr(provider_module.settings, "SRE_GROUP_CHAT_ID", "test-room") + gateway = SimpleNamespace( + send_alert_notification=AsyncMock( + return_value={ + "ok": False, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + } + ) + ) + monkeypatch.setattr(gateway_module, "get_telegram_gateway", lambda: gateway) + + result = await provider_module.TelegramWebhookProvider().send( + _notification_message() + ) + + assert result.status is NotificationStatus.SKIPPED + assert result.message == "Telegram notification blocked before provider send" + assert result.response_data == { + "ok": False, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + } + gateway.send_alert_notification.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_notification_provider_requires_affirmative_delivery_ack( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services.notifications import telegram as provider_module + from src.services.notifications.base import NotificationStatus + + monkeypatch.setattr(provider_module.settings, "OPENCLAW_TG_BOT_TOKEN", "configured") + monkeypatch.setattr(provider_module.settings, "SRE_GROUP_CHAT_ID", "test-room") + gateway = SimpleNamespace( + send_alert_notification=AsyncMock( + return_value={"ok": False, "description": "provider rejected"} + ) + ) + monkeypatch.setattr(gateway_module, "get_telegram_gateway", lambda: gateway) + + result = await provider_module.TelegramWebhookProvider().send( + _notification_message() + ) + + assert result.status is NotificationStatus.FAILED + assert result.message == "Telegram provider did not acknowledge delivery" + assert result.error == "delivery_not_acknowledged" + + +@pytest.mark.asyncio +async def test_unclassified_send_message_is_blocked_before_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + result = await gateway._send_request( + "sendMessage", + {"chat_id": "test-room", "text": "unclassified"}, + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert result["_awooop_provider_send_performed"] is False + assert client.posts == [] + + +@pytest.mark.asyncio +async def test_info_and_business_methods_are_blocked_before_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + info = await gateway.send_info_notification( + incident_id="INC-TEST", + title="FYI", + message="healthy", + ) + business = await gateway.send_business_alert( + incident_id="INC-TEST", + alertname="BusinessMetric", + business_domain="test", + metric_name="rate", + current_value="1", + threshold="2", + ) + + assert info["_awooop_delivery_status"] == "blocked_no_egress" + assert business["_awooop_delivery_status"] == "blocked_no_egress" + assert client.posts == [] + + +@pytest.mark.asyncio +async def test_wrong_group_override_is_blocked_before_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + result = await gateway.send_escalation_card( + incident_id="INC-TEST", + original_alertname="Outage", + duration_min=15, + group_chat_id="wrong-room", + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert result["_awoooi_canonical_route_receipt"]["classifier"] == ( + "caller_destination_mismatch" + ) + assert client.posts == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method_name", ["meta", "secops", "escalation"]) +async def test_allowed_legacy_monitoring_methods_use_canonical_provider_once( + monkeypatch: pytest.MonkeyPatch, + method_name: str, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + if method_name == "meta": + result = await gateway.send_meta_alert( + incident_id="INC-TEST", + approval_id="APR-TEST", + alertname="AlertChainBroken", + alert_category="alertchain_health", + diagnosis="test", + ) + elif method_name == "secops": + result = await gateway.send_secops_card( + incident_id="INC-TEST", + approval_id="APR-TEST", + alertname="Threat", + threat_level="critical", + ) + else: + result = await gateway.send_escalation_card( + incident_id="INC-TEST", + original_alertname="Outage", + duration_min=15, + ) + + assert result["ok"] is True + assert result["_awoooi_canonical_route_receipt"]["decision"] == "allowed" + assert result["_awoooi_canonical_route_receipt"]["provider_send_performed"] is True + assert len(client.posts) == 1 + assert client.posts[0][1]["chat_id"] == "test-room" + + +@pytest.mark.asyncio +async def test_openclaw_cannot_own_tsenyang_monitoring_route( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + result = await gateway._send_as_bot( + token="test-token-placeholder", + text="must not send", + product_id="awoooi", + signal_family="security_incident", + severity="P1", + actual_bot_alias="openclaw_bot", + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert result["_awoooi_canonical_route_receipt"]["classifier"] == ( + "sender_bot_alias_not_authorized_for_route" + ) + assert client.posts == [] + + +@pytest.mark.asyncio +async def test_operator_reply_lane_and_runtime_edit_binding_remain_available( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + assert gateway.chat_id == "test-room" + assert gateway.alert_chat_id == "test-room" + unverified = await gateway.send_notification( + "must not infer an operator reply", + chat_id="operator-room", + ) + + assert unverified["_awooop_delivery_status"] == "blocked_no_egress" + assert unverified["_awoooi_canonical_route_receipt"]["classifier"] == ( + "missing_product_signal_severity_or_interaction_context" + ) + assert client.posts == [] + + result = await gateway.send_notification( + "command reply", + chat_id="operator-room", + interaction_kind="operator_command_reply", + inbound_chat_id="operator-room", + inbound_message_id=42, + ) + + assert result["ok"] is True + receipt = result["_awoooi_canonical_route_receipt"] + assert receipt["classifier"] == "non_monitoring_interaction_allowed" + assert receipt["destination_alias"] == "inbound_interaction_reply_target" + assert client.posts[0][1]["chat_id"] == "operator-room" + assert client.posts[0][1]["reply_to_message_id"] == 42 + + +@pytest.mark.asyncio +async def test_interaction_reply_must_match_verified_inbound_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + result = await gateway._send_request( + "sendMessage", + { + "chat_id": "operator-room", + "text": "mismatched reply", + "reply_to_message_id": 43, + gateway_module._NON_MONITORING_INTERACTION_KEY: { + "interaction_kind": "operator_command_reply", + "inbound_chat_id": "operator-room", + "inbound_message_id": 42, + }, + }, + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert result["_awoooi_canonical_route_receipt"]["classifier"] == ( + "interaction_reply_context_missing_or_mismatched" + ) + assert client.posts == [] + + +@pytest.mark.asyncio +async def test_t1_through_t6_use_only_canonical_monitoring_routes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + canonical_send = AsyncMock(return_value={"ok": True}) + monkeypatch.setattr(gateway, "send_canonical_message", canonical_send) + + await gateway.send_guardrail_blocked("api", "HighCpu", "protected") + await gateway.send_preflight_failed("api", 25.0, 24.0, "backup-1") + await gateway.send_backup_result("backup-1", True) + await gateway.send_multisig_waiting("repair", "api", 1, 2, "APR-1") + await gateway.send_multisig_approved("repair", "api") + await gateway.send_change_applied("operator", "repair", "2026-07-14T00:00:00Z") + + assert canonical_send.await_count == 6 + assert [ + ( + call.kwargs["product_id"], + call.kwargs["signal_family"], + call.kwargs["severity"], + ) + for call in canonical_send.await_args_list + ] == [ + ("awoooi", "security_incident", "P1"), + ("awoooi", "disaster_recovery", "P1"), + ("awoooi", "disaster_recovery", "P1"), + ("awoooi", "incident_lifecycle", "P1"), + ("awoooi", "incident_lifecycle", "P1"), + ("awoooi", "incident_lifecycle", "P1"), + ] + + +@pytest.mark.asyncio +async def test_image_analysis_reply_preserves_original_inbound_message_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services.image_analysis_service import ImageAnalysisService + + gateway = SimpleNamespace( + initialize=AsyncMock(), + send_as_openclaw=AsyncMock(return_value={"ok": True}), + ) + service = ImageAnalysisService() + monkeypatch.setattr( + gateway_module, + "get_telegram_gateway", + lambda: gateway, + ) + monkeypatch.setattr( + service, + "_download_telegram_file", + AsyncMock(return_value=b"image"), + ) + monkeypatch.setattr( + service, + "analyze", + AsyncMock(return_value="analysis"), + ) + + await service.download_and_analyze( + chat_id="test-room", + file_id="file-1", + original_message_id=77, + ) + + gateway.initialize.assert_awaited_once() + gateway.send_as_openclaw.assert_awaited_once_with( + "🖼️ 圖片分析\nanalysis", + reply_to_message_id=77, + inbound_chat_id="test-room", + ) + + +@pytest.mark.asyncio +async def test_manual_repair_callback_replies_to_verified_original_message( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + monkeypatch.setattr( + gateway._security, + "parse_callback_data", + lambda _data: { + "action": "log_manual_fix", + "approval_id": "APR-1", + "is_info_action": False, + "nonce": "nonce-1", + }, + ) + monkeypatch.setattr( + gateway._security, + "verify_callback", + AsyncMock(return_value={"id": 123, "username": "operator"}), + ) + monkeypatch.setattr( + gateway, + "_check_incident_state_guard", + AsyncMock(return_value=None), + ) + monkeypatch.setattr(gateway, "_answer_callback", AsyncMock()) + monkeypatch.setattr(gateway_module, "get_redis", lambda: _FakeRedis()) + + result = await gateway.handle_callback( + callback_query_id="callback-1", + callback_data="log_manual_fix:APR-1:nonce-1", + user_id=123, + message_id=77, + chat_id="operator-room", + username="operator", + ) + + assert result["success"] is True + assert result["waiting_for_manual_fix"] is True + assert len(client.posts) == 1 + payload = client.posts[0][1] + assert payload["chat_id"] == "operator-room" + assert payload["reply_to_message_id"] == 77 + + +@pytest.mark.asyncio +async def test_manual_repair_flow_blocks_before_side_effect_without_inbound_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + + result = await gateway.handle_manual_fix_done( + user_id=123, + username="operator", + fix_steps="restart", + ) + + assert result == { + "success": False, + "reason": "missing_verified_interaction_context", + } + assert client.posts == [] + + +class _FakeDb: + def __init__(self) -> None: + self.params: list[dict] = [] + + async def execute(self, statement: object, params: dict) -> None: + del statement + self.params.append(params) + + +class _FakeRedis: + async def setex(self, key: str, ttl: int, value: str) -> None: + del key, ttl, value + + +def _patch_approval_dependencies( + monkeypatch: pytest.MonkeyPatch, + gateway: TelegramGateway, +) -> _FakeDb: + fake_db = _FakeDb() + + @asynccontextmanager + async def fake_db_context(*args, **kwargs): + del args, kwargs + yield fake_db + + monkeypatch.setattr(db_base, "get_db_context", fake_db_context) + monkeypatch.setattr( + approval_db, + "get_approval_service", + lambda: SimpleNamespace(update_telegram_message=True), + ) + monkeypatch.setattr(gateway_module, "get_redis", lambda: _FakeRedis()) + monkeypatch.setattr( + gateway_module, + "_fetch_remediation_summary_for_card", + AsyncMock(return_value=None), + ) + monkeypatch.setattr(gateway, "_build_inline_keyboard", AsyncMock(return_value={})) + return fake_db + + +@pytest.mark.asyncio +async def test_approval_missing_type_is_blocked_without_provider_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + _patch_approval_dependencies(monkeypatch, gateway) + + result = await gateway.send_approval_card( + approval_id="APR-TEST", + risk_level="critical", + resource_name="resource", + root_cause="cause", + suggested_action="review", + notification_type="", + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert client.posts == [] + + +@pytest.mark.asyncio +async def test_allowed_type3_approval_persists_runtime_chat_without_public_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + fake_db = _patch_approval_dependencies(monkeypatch, gateway) + monkeypatch.setattr(gateway, "canonical_destination_chat_id", lambda **kwargs: "7") + + result = await gateway.send_approval_card( + approval_id="APR-TEST", + risk_level="critical", + resource_name="resource", + root_cause="cause", + suggested_action="review", + notification_type="TYPE-3", + ) + + assert result["ok"] is True + assert len(client.posts) == 1 + assert any(params.get("cid") == 7 for params in fake_db.params) + receipt = result["_awoooi_canonical_route_receipt"] + assert receipt["destination_alias"] == "awoooi_sre_war_room" + assert "chat_id" not in receipt diff --git a/apps/api/tests/test_telegram_delivery_truth_callers.py b/apps/api/tests/test_telegram_delivery_truth_callers.py new file mode 100644 index 000000000..604c6d904 --- /dev/null +++ b/apps/api/tests/test_telegram_delivery_truth_callers.py @@ -0,0 +1,399 @@ +"""Regression tests for callers that must distinguish routing from delivery.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from src.models.incident import Incident, Severity, Signal +from src.repositories.alert_operation_log_repository import ALERT_EVENT_TYPES +from src.services import decision_manager as decision_module +from src.services import report_generation_service as report_module +from src.services.ai_rate_limiter import AIRateLimiter +from src.services.approval_execution import ApprovalExecutionService +from src.services.report_generation_service import ReportGenerationService + +NO_SEND_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + "_awoooi_canonical_route_receipt": { + "decision": "blocked", + "provider_send_performed": False, + }, +} + +SENT_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + "result": {"message_id": 42, "chat": {"id": -1001}}, +} + + +class FakeRedis: + def __init__(self) -> None: + self.values: dict[str, str] = {} + self.set_calls: list[tuple[str, str, int | None]] = [] + self.setex_calls: list[tuple[str, int, str]] = [] + + async def get(self, key: str): + return self.values.get(key) + + async def exists(self, key: str) -> bool: + return key in self.values + + async def set(self, key: str, value: str, ex: int | None = None, **_kwargs): + self.values[key] = value + self.set_calls.append((key, value, ex)) + return True + + async def setex(self, key: str, ttl: int, value: str) -> bool: + self.values[key] = value + self.setex_calls.append((key, ttl, value)) + return True + + +class FakeLogger: + def __init__(self) -> None: + self.events: list[tuple[str, str]] = [] + + def debug(self, event: str, **_kwargs) -> None: + self.events.append(("debug", event)) + + def info(self, event: str, **_kwargs) -> None: + self.events.append(("info", event)) + + def warning(self, event: str, **_kwargs) -> None: + self.events.append(("warning", event)) + + def error(self, event: str, **_kwargs) -> None: + self.events.append(("error", event)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method_name", ["_send_cost_alert", "_send_cost_warning"]) +@pytest.mark.parametrize("delivery_result", [NO_SEND_RECEIPT, None]) +async def test_cost_alert_dedup_is_written_only_after_affirmative_delivery( + monkeypatch, + method_name: str, + delivery_result: object, +) -> None: + redis = FakeRedis() + + class Gateway: + def canonical_destination_chat_id(self, **_route) -> str: + return "sre-room" + + async def send_canonical_message(self, **_payload): + return delivery_result + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + limiter = AIRateLimiter() + limiter._redis = redis + + if method_name == "_send_cost_alert": + await limiter._send_cost_alert("gemini", 5.1, 5.0) + else: + await limiter._send_cost_warning("gemini", 4.1, 4.0) + + assert redis.set_calls == [] + + +@pytest.mark.asyncio +async def test_cost_alert_sets_dedup_after_affirmative_delivery(monkeypatch) -> None: + redis = FakeRedis() + + class Gateway: + def canonical_destination_chat_id(self, **_route) -> str: + return "sre-room" + + async def send_canonical_message(self, **_payload): + return dict(SENT_RECEIPT) + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + limiter = AIRateLimiter() + limiter._redis = redis + + await limiter._send_cost_alert("gemini", 5.1, 5.0) + + assert len(redis.set_calls) == 1 + assert redis.set_calls[0][0] == "ai_rate:cost_alert_sent:gemini" + + +@pytest.mark.asyncio +async def test_p3_reports_fail_closed_before_collection_when_route_missing( + monkeypatch, +) -> None: + class Gateway: + def canonical_destination_chat_id(self, **_route) -> str: + return "" + + async def send_to_group(self, *_args, **_kwargs): + raise AssertionError("provider must not be called without a route") + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + service = ReportGenerationService() + daily = AsyncMock() + monthly = AsyncMock() + monkeypatch.setattr(service, "collect_daily_kpi", daily) + monkeypatch.setattr(service, "collect_report_source_health", monthly) + + await service.send_daily_report() + monthly_result = await service.send_monthly_report() + + daily.assert_not_awaited() + monthly.assert_not_awaited() + assert monthly_result is False + + +@pytest.mark.asyncio +async def test_postmortem_no_ack_retries_and_verifies_fallback(monkeypatch) -> None: + send = AsyncMock(return_value=dict(NO_SEND_RECEIPT)) + + class Gateway: + send_to_group = send + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + monkeypatch.setattr(report_module.asyncio, "sleep", AsyncMock()) + service = ReportGenerationService() + monkeypatch.setattr(service, "_persist_postmortem_km", AsyncMock()) + now = datetime.now(UTC) + + await service.trigger_postmortem( + incident_id="INC-NO-ACK", + title="provider did not acknowledge", + created_at=now - timedelta(minutes=12), + resolved_at=now, + ) + + # Three bounded attempts plus one simplified fallback; all remain no-send. + assert send.await_count == 4 + + +@pytest.mark.asyncio +async def test_approval_blocked_receipt_writes_no_send_not_sent(monkeypatch) -> None: + redis = FakeRedis() + operations: list[dict] = [] + + class Gateway: + def canonical_destination_chat_id(self, **_route) -> str: + return "sre-room" + + async def _send_request(self, *_args, **_kwargs): + return dict(SENT_RECEIPT) + + async def send_canonical_message(self, **_payload): + return dict(NO_SEND_RECEIPT) + + class OperationRepository: + async def append(self, event_type: str, **kwargs) -> None: + operations.append({"event_type": event_type, **kwargs}) + + class BrokenDB: + async def execute(self, *_args, **_kwargs): + raise RuntimeError("expected test DB bypass") + + @asynccontextmanager + async def fake_db_context(): + yield BrokenDB() + + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) + monkeypatch.setattr("src.db.base.get_db_context", fake_db_context) + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + monkeypatch.setattr( + "src.repositories.alert_operation_log_repository.get_alert_operation_log_repository", + lambda: OperationRepository(), + ) + monkeypatch.setattr( + "src.services.approval_execution.incident_truth_chain_button_row", + lambda _incident_id: None, + ) + + service = ApprovalExecutionService() + monkeypatch.setattr( + service, + "_fetch_operator_outcome_for_result", + AsyncMock( + return_value={ + "state": "completed_verified", + "summary_zh": "verified", + "needs_human": False, + "next_action": "monitor", + "notification": {"channels": []}, + "execution_result": {}, + } + ), + ) + approval = SimpleNamespace( + id="approval-no-ack", + incident_id="INC-NO-ACK", + action="kubectl rollout restart deployment/api", + requested_by="operator", + ) + + await service._push_execution_result_to_alert( + approval, + success=True, + error=None, + execution_kind="controlled_apply", + repair_executed=True, + repair_attempted=True, + ) + + assert [row["event_type"] for row in operations] == [ + "NOTIFICATION_CLASSIFIED" + ] + assert operations[0]["event_type"] in ALERT_EVENT_TYPES + assert operations[0]["action_detail"] == "telegram_execution_result_no_send" + assert operations[0]["success"] is False + assert operations[0]["context"]["execution_success"] is True + + +def _incident() -> Incident: + return Incident( + incident_id="INC-DECISION-NO-ACK", + severity=Severity.P3, + signals=[ + Signal( + alert_name="ContainerPressure", + severity=Severity.P3, + source="alertmanager", + fired_at=datetime.now(UTC), + labels={"alertname": "ContainerPressure"}, + ) + ], + affected_services=["api"], + ) + + +@pytest.mark.asyncio +async def test_decision_no_ack_does_not_set_fingerprint_dedup(monkeypatch) -> None: + from src.services.telegram_gateway import NotificationType + + redis = FakeRedis() + + class Gateway: + async def send_info_notification(self, **_payload): + return dict(NO_SEND_RECEIPT) + + class ApprovalService: + async def update_action_by_incident_id(self, **_payload) -> None: + return None + + monkeypatch.setattr(decision_module.settings, "OPENCLAW_TG_BOT_TOKEN", "configured") + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + monkeypatch.setattr( + "src.services.telegram_gateway.classify_notification", + lambda **_kwargs: NotificationType.TYPE_1, + ) + monkeypatch.setattr( + "src.services.approval_db.get_approval_service", + lambda: ApprovalService(), + ) + + await decision_module._push_decision_to_telegram( + _incident(), + { + "action": "OBSERVE", + "description": "bounded observation", + "reasoning": "診斷：bounded observation", + "risk_level": "low", + "confidence": 0.9, + "source": "test", + }, + ) + + assert not any(key.startswith("telegram_sent:fp:") for key in redis.values) + + +@pytest.mark.asyncio +async def test_log_summary_route_preflight_runs_before_llm(monkeypatch) -> None: + summarize = AsyncMock(return_value="summary") + + class Gateway: + def canonical_destination_chat_id(self, **_route) -> str: + return "" + + async def send_as_nemotron(self, *_args, **_kwargs): + raise AssertionError("provider must not be called without a route") + + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + monkeypatch.setattr( + "src.services.log_summary_service.get_log_summary_service", + lambda: SimpleNamespace(summarize_with_soft_timeout=summarize), + ) + + await decision_module._send_log_summary(_incident()) + + summarize.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_auto_repair_fallback_requires_affirmative_delivery(monkeypatch) -> None: + test_logger = FakeLogger() + + class Gateway: + async def append_incident_update(self, **_payload) -> bool: + return False + + def canonical_destination_chat_id(self, **_route) -> str: + return "sre-room" + + async def send_alert_notification(self, *_args, **_kwargs): + return dict(NO_SEND_RECEIPT) + + class DB: + async def execute(self, *_args, **_kwargs) -> None: + return None + + async def commit(self) -> None: + return None + + @asynccontextmanager + async def fake_db_context(): + yield DB() + + monkeypatch.setattr(decision_module, "logger", test_logger) + monkeypatch.setattr("src.db.base.get_db_context", fake_db_context) + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: Gateway(), + ) + + await decision_module._push_auto_repair_result( + _incident(), + action="bounded observe", + success=False, + error="verifier_failed", + ) + + event_names = [event for _level, event in test_logger.events] + assert "auto_repair_result_no_send" in event_names + assert "auto_repair_result_sent" not in event_names diff --git a/apps/api/tests/test_telegram_delivery_truth_callers_v2.py b/apps/api/tests/test_telegram_delivery_truth_callers_v2.py new file mode 100644 index 000000000..841305d6b --- /dev/null +++ b/apps/api/tests/test_telegram_delivery_truth_callers_v2.py @@ -0,0 +1,394 @@ +"""Delivery-truth regressions for migrated Telegram callers. + +2026-07-14 Codex v1.0 (Asia/Taipei): an adapter may return ``ok=true`` while +the canonical egress gate reports ``provider_send_performed=false``. These +tests keep that structured no-send receipt from becoming sent logs, counters, +or durable dedup state. No Telegram/provider network is used. +""" + +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from src.jobs import ai_slo_watchdog_job as watchdog_module +from src.jobs import capacity_forecaster_job as capacity_module +from src.jobs import hermes_rule_quality_job as hermes_module +from src.services import converged_alert_recurrence_notifier as recurrence_module +from src.services import decision_manager as decision_module +from src.services import failover_alerter as failover_module +from src.services import k3s_monitor_service as k3s_module +from src.services import weekly_report_service as weekly_module + +NO_PROVIDER_SEND_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": False, + "_awoooi_canonical_route_receipt": { + "decision": "blocked", + "provider_send_performed": False, + }, +} + +SENT_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + "result": {"message_id": 42, "chat": {"id": -1001}}, +} + + +class _Redis: + def __init__(self) -> None: + self.values: dict[str, str] = {} + self.deleted: list[str] = [] + + async def set( + self, + key: str, + value: str, + *, + ex: int | None = None, + nx: bool | None = None, + **_kwargs, + ) -> bool | None: + del ex + if nx and key in self.values: + return None + self.values[key] = value + return True + + async def delete(self, key: str) -> int: + self.deleted.append(key) + return 1 if self.values.pop(key, None) is not None else 0 + + async def exists(self, key: str) -> bool: + return key in self.values + + async def get(self, key: str) -> str | None: + return self.values.get(key) + + async def scan(self, _cursor: int, *, match: str, count: int): + del match, count + return 0, list(self.values) + + +class _Logger: + def __init__(self) -> None: + self.events: list[tuple[str, str]] = [] + + def debug(self, event: str, **_kwargs) -> None: + self.events.append(("debug", event)) + + def info(self, event: str, **_kwargs) -> None: + self.events.append(("info", event)) + + def warning(self, event: str, **_kwargs) -> None: + self.events.append(("warning", event)) + + def error(self, event: str, **_kwargs) -> None: + self.events.append(("error", event)) + + def exception(self, event: str, **_kwargs) -> None: + self.events.append(("exception", event)) + + +class _Gateway: + def __init__(self, receipt: dict) -> None: + self.receipt = receipt + + def canonical_destination_chat_id(self, **_route) -> str: + return "canonical-room" + + async def send_alert_notification(self, *_args, **_kwargs) -> dict: + return dict(self.receipt) + + async def send_meta_alert(self, **_kwargs) -> dict: + return dict(self.receipt) + + async def send_canonical_message(self, **_kwargs) -> dict: + return dict(self.receipt) + + async def send_text(self, *_args, **_kwargs) -> dict: + return dict(self.receipt) + + +@pytest.mark.asyncio +async def test_failover_commits_dedup_and_sent_log_only_after_provider_ack( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = _Redis() + logger = _Logger() + gateway = _Gateway(NO_PROVIDER_SEND_RECEIPT) + monkeypatch.setattr(failover_module, "logger", logger) + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: gateway, + ) + alerter = failover_module.FailoverAlerter(redis_client=redis) + + await alerter.alert_failover({"to_provider": "ollama_gcp_b"}) + + dedup_key = "alert:failover:ollama_gcp_b:dedup" + assert dedup_key not in redis.values + assert ("info", "failover_alert_sent") not in logger.events + assert ("warning", "failover_alert_no_send") in logger.events + + gateway.receipt = SENT_RECEIPT + await alerter.alert_failover({"to_provider": "ollama_gcp_b"}) + + assert redis.values[dedup_key] == "sent" + assert ("info", "failover_alert_sent") in logger.events + + +@pytest.mark.asyncio +async def test_watchdog_releases_dedup_and_does_not_log_sent_without_provider_ack( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services import ai_slo_calculator, flywheel_stats_service, governance_agent + + class _SloCalculator: + async def calculate(self): + return SimpleNamespace(any_violated=False, metrics=[]) + + class _FlywheelStats: + async def compute(self): + return SimpleNamespace(execution_success_rate=1.0) + + class _GovernanceAgent: + async def check_trust_drift(self, *, emit_alert: bool): + assert emit_alert is False + return {"drifted": 0, "drift_ratio": 0.0} + + redis = _Redis() + logger = _Logger() + monkeypatch.setattr(watchdog_module, "logger", logger) + monkeypatch.setattr(watchdog_module, "get_redis", lambda: redis) + monkeypatch.setattr(watchdog_module, "_is_grace_active", AsyncMock(return_value=True)) + monkeypatch.setattr( + watchdog_module, + "_count_pending_no_tg_sent", + AsyncMock(return_value=watchdog_module._TG_SILENCE_THRESHOLD), + ) + monkeypatch.setattr( + watchdog_module, + "_count_approved_playbooks", + AsyncMock(return_value=(1, 1)), + ) + monkeypatch.setattr( + watchdog_module, + "_count_pending_stuck_analysis", + AsyncMock(return_value=0), + ) + monkeypatch.setattr(ai_slo_calculator, "AiSloCalculator", _SloCalculator) + monkeypatch.setattr(flywheel_stats_service, "FlywheelStatsService", _FlywheelStats) + monkeypatch.setattr( + governance_agent, + "get_governance_agent", + lambda: _GovernanceAgent(), + ) + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: _Gateway(NO_PROVIDER_SEND_RECEIPT), + ) + + await watchdog_module._check_once() + + assert redis.values == {} + assert ("warning", "ai_slo_watchdog_alert_sent") not in logger.events + assert ("error", "ai_slo_watchdog_alert_no_send") in logger.events + + +@pytest.mark.asyncio +async def test_advisory_callers_reject_ok_true_without_provider_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services import ai_advisory_helpers + + monkeypatch.setattr( + ai_advisory_helpers, + "is_snoozed", + AsyncMock(return_value=False), + ) + monkeypatch.setattr( + ai_advisory_helpers, + "build_ai_advisory_keyboard", + lambda **_kwargs: {}, + ) + monkeypatch.setattr( + "src.services.telegram_gateway.get_telegram_gateway", + lambda: _Gateway(NO_PROVIDER_SEND_RECEIPT), + ) + + hermes_sent = await hermes_module._send_telegram_summary( + [ + { + "rule_name": "HighNoiseRule", + "noise_rate": 0.9, + "tp": 1, + "fp": 9, + } + ], + ) + capacity_sent = await capacity_module._send_telegram_forecast( + { + "host-a": [ + { + "reason": "disk trend", + "value": 0.9, + } + ] + }, + ) + + assert hermes_sent is False + assert capacity_sent is False + + +@pytest.mark.asyncio +async def test_report_callers_reject_ok_true_without_provider_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = _Gateway(NO_PROVIDER_SEND_RECEIPT) + monkeypatch.setattr(weekly_module, "get_telegram_gateway", lambda: gateway) + monkeypatch.setattr(k3s_module, "get_telegram_gateway", lambda: gateway) + + weekly = weekly_module.WeeklyReportService() + monkeypatch.setattr( + weekly, + "generate_report", + AsyncMock( + return_value=SimpleNamespace( + format=lambda: "weekly", + week_range="2026-W29", + ) + ), + ) + k3s = k3s_module.K3sMonitorService() + monkeypatch.setattr( + k3s, + "collect_cluster_status", + AsyncMock( + return_value=SimpleNamespace( + format=lambda: "daily", + report_date="2026-07-14", + ) + ), + ) + + assert await weekly.send_weekly_report() is False + assert await k3s.send_daily_report() is False + + +@pytest.mark.asyncio +async def test_recurrence_releases_reservation_and_never_logs_sent_without_ack( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = _Redis() + logger = _Logger() + monkeypatch.setattr(recurrence_module, "logger", logger) + monkeypatch.setattr(recurrence_module, "get_redis", lambda: redis) + monkeypatch.setattr( + recurrence_module, + "should_notify_converged_alert_recurrence", + AsyncMock(return_value=True), + ) + monkeypatch.setattr( + recurrence_module, + "get_telegram_gateway", + lambda: _Gateway(NO_PROVIDER_SEND_RECEIPT), + ) + + await recurrence_module.notify_converged_alert_recurrence( + source="alertmanager", + fingerprint="fp-no-provider-send", + alertname="ServiceDown", + severity="critical", + namespace="awoooi-prod", + target_resource="api", + hit_count=2, + incident_id="INC-NO-PROVIDER-SEND", + approval_id="approval-1", + ) + + key = recurrence_module.converged_alert_recurrence_key("fp-no-provider-send") + assert key in redis.deleted + assert ("info", "converged_alert_recurrence_sent") not in logger.events + assert ("error", "converged_alert_recurrence_failed") in logger.events + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider_ack, expected_resent", [(False, 0), (True, 1)]) +async def test_stale_token_replay_counts_only_awaited_delivery_receipts( + monkeypatch: pytest.MonkeyPatch, + provider_ack: bool, + expected_resent: int, +) -> None: + from src.db import base as db_base + from src.repositories import incident_repository + + incident_id = "INC-STALE-DELIVERY-TRUTH" + redis = _Redis() + redis.values["decision:DEC-STALE"] = json.dumps( + { + "state": decision_module.DecisionState.READY.value, + "incident_id": incident_id, + "proposal_data": {"description": "bounded replay"}, + "token": "DEC-STALE", + } + ) + incident = SimpleNamespace( + incident_id=incident_id, + status="open", + affected_services=["api"], + created_at=datetime.now(UTC), + signals=[ + SimpleNamespace( + labels={"alertname": "ServiceDown"}, + alert_name="ServiceDown", + annotations={}, + ) + ], + ) + + @asynccontextmanager + async def _db_context(): + yield object() + + class _IncidentRepository: + def __init__(self, _db) -> None: + pass + + async def get_by_id(self, requested_id: str): + assert requested_id == incident_id + return incident + + push = AsyncMock(return_value=provider_ack) + logger = _Logger() + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) + monkeypatch.setattr(db_base, "get_db_context", _db_context) + monkeypatch.setattr( + incident_repository, + "IncidentDBRepository", + _IncidentRepository, + ) + monkeypatch.setattr(decision_module, "_push_decision_to_telegram", push) + monkeypatch.setattr(decision_module, "logger", logger) + monkeypatch.setattr(asyncio, "sleep", AsyncMock()) + + manager = object.__new__(decision_module.DecisionManager) + resent = await manager.resend_stale_ready_tokens() + + assert resent == expected_resent + push.assert_awaited_once() + if provider_ack: + assert ("info", "stale_ready_token_resent") in logger.events + else: + assert ("info", "stale_ready_token_resent") not in logger.events + assert ("warning", "stale_ready_token_no_send") in logger.events diff --git a/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py b/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py new file mode 100644 index 000000000..82d5fd041 --- /dev/null +++ b/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py @@ -0,0 +1,460 @@ +"""Delivery-truth regressions for gateway and webhook Telegram callers.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from src.api.v1 import sentry_webhook, signoz_webhook, telegram, webhooks +from src.services import telegram_gateway as gateway_module +from src.services.telegram_gateway import ( + TelegramGateway, + _telegram_send_delivery_succeeded, +) + +NO_SEND_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + "_awoooi_canonical_route_receipt": { + "decision": "blocked_no_egress", + "provider_send_performed": False, + }, +} + +SENT_RECEIPT = { + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + "result": {"message_id": 42}, +} + + +class FakeLogger: + def __init__(self) -> None: + self.events: list[tuple[str, str]] = [] + + def debug(self, event: str, **_kwargs) -> None: + self.events.append(("debug", event)) + + def info(self, event: str, **_kwargs) -> None: + self.events.append(("info", event)) + + def warning(self, event: str, **_kwargs) -> None: + self.events.append(("warning", event)) + + def error(self, event: str, **_kwargs) -> None: + self.events.append(("error", event)) + + def exception(self, event: str, **_kwargs) -> None: + self.events.append(("exception", event)) + + +class FakeRedis: + def __init__(self, values: dict[str, str] | None = None) -> None: + self.values = dict(values or {}) + self.set_calls: list[tuple[str, str, dict]] = [] + + async def get(self, key: str): + return self.values.get(key) + + async def set(self, key: str, value: str, **kwargs): + self.set_calls.append((key, value, kwargs)) + if kwargs.get("nx") and key in self.values: + return False + self.values[key] = value + return True + + async def setex(self, key: str, _ttl: int, value: str): + self.values[key] = value + return True + + +def test_delivery_helper_rejects_truthy_structured_no_send() -> None: + assert _telegram_send_delivery_succeeded(dict(NO_SEND_RECEIPT)) is False + assert _telegram_send_delivery_succeeded({"ok": True}) is True + + +@pytest.mark.asyncio +async def test_mark_auto_repaired_returns_false_for_structured_no_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = FakeRedis({"tg_approval:APR-1": "42"}) + gateway = TelegramGateway() + + async def send_request(method: str, _payload: dict): + if method == "sendMessage": + return dict(NO_SEND_RECEIPT) + return {"ok": True} + + monkeypatch.setattr(gateway_module, "get_redis", lambda: redis) + monkeypatch.setattr(gateway, "_send_request", send_request) + + assert await gateway.mark_auto_repaired("APR-1", "bounded", 10) is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method_name", ["incident_update", "grouped_digest"]) +async def test_lifecycle_no_send_does_not_write_dedup_receipt( + monkeypatch: pytest.MonkeyPatch, + method_name: str, +) -> None: + redis = FakeRedis({"tg_msg:INC-1": "42"}) + gateway = TelegramGateway() + + async def send_request(method: str, _payload: dict): + if method == "sendMessage": + return dict(NO_SEND_RECEIPT) + return {"ok": True} + + monkeypatch.setattr(gateway_module, "get_redis", lambda: redis) + monkeypatch.setattr(gateway, "_send_request", send_request) + + if method_name == "incident_update": + result = await gateway.append_incident_update("INC-1", "healthy") + else: + result = await gateway.append_grouped_alert_digest( + incident_id="INC-1", + group_key="group-1", + digest_text="digest", + ) + + assert result is False + assert redis.set_calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method_name", ["incident_update", "grouped_digest"]) +async def test_lifecycle_writes_dedup_only_after_affirmative_ack( + monkeypatch: pytest.MonkeyPatch, + method_name: str, +) -> None: + redis = FakeRedis({"tg_msg:INC-1": "42"}) + gateway = TelegramGateway() + + async def send_request(method: str, _payload: dict): + if method == "sendMessage": + return dict(SENT_RECEIPT) + return {"ok": True} + + monkeypatch.setattr(gateway_module, "get_redis", lambda: redis) + monkeypatch.setattr(gateway, "_send_request", send_request) + + if method_name == "incident_update": + result = await gateway.append_incident_update("INC-1", "healthy") + else: + result = await gateway.append_grouped_alert_digest( + incident_id="INC-1", + group_key="group-1", + digest_text="digest", + ) + + assert result is True + assert len(redis.set_calls) == 1 + assert redis.set_calls[0][1] == "provider_ack" + + +@pytest.mark.asyncio +async def test_analyzing_placeholder_does_not_return_or_log_sent_on_no_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + logger = FakeLogger() + gateway = TelegramGateway() + monkeypatch.setattr(gateway_module.settings, "OPENCLAW_TG_BOT_TOKEN", "configured") + monkeypatch.setattr(gateway_module, "logger", logger) + monkeypatch.setattr( + gateway, + "_send_request", + AsyncMock(return_value=dict(NO_SEND_RECEIPT)), + ) + + result = await gateway.send_analyzing_placeholder("cpu", "api", "medium") + + assert result is None + assert ("info", "analyzing_placeholder_no_send") in logger.events + assert ("info", "analyzing_placeholder_sent") not in logger.events + + +@pytest.mark.asyncio +async def test_approval_card_records_failed_and_never_sent_for_no_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + logger = FakeLogger() + gateway = TelegramGateway() + executed_params: list[dict] = [] + + class FakeDB: + async def execute(self, _statement, params: dict) -> None: + executed_params.append(params) + + @asynccontextmanager + async def fake_db_context(*_args, **_kwargs): + yield FakeDB() + + monkeypatch.setattr(gateway_module, "logger", logger) + monkeypatch.setattr( + gateway_module, + "_fetch_remediation_summary_for_card", + AsyncMock(return_value=None), + ) + monkeypatch.setattr(gateway, "_build_inline_keyboard", AsyncMock(return_value={})) + monkeypatch.setattr( + gateway, + "_send_request", + AsyncMock(return_value=dict(NO_SEND_RECEIPT)), + ) + monkeypatch.setattr("src.db.base.get_db_context", fake_db_context) + + result = await gateway.send_approval_card( + approval_id="APR-1", + risk_level="medium", + resource_name="api", + root_cause="cpu", + suggested_action="inspect", + notification_type="TYPE-3", + ) + + assert result == NO_SEND_RECEIPT + assert executed_params and executed_params[0]["ds"] == "failed" + assert ("warning", "telegram_approval_card_no_send") in logger.events + assert ("info", "telegram_approval_card_sent") not in logger.events + + +@pytest.mark.asyncio +async def test_html_callback_no_send_exhausts_fallback_and_records_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = TelegramGateway() + send_request = AsyncMock(return_value=dict(NO_SEND_RECEIPT)) + record_failure = AsyncMock() + monkeypatch.setattr(gateway, "_send_request", send_request) + monkeypatch.setattr(gateway, "_record_callback_reply_failure", record_failure) + + await gateway._send_html_line_message( + ["reply"], + failure_context="test", + inbound_chat_id="room", + inbound_message_id=42, + incident_id="INC-1", + ) + + assert send_request.await_count == 3 + record_failure.assert_awaited_once() + statuses = [ + call.args[1].get(gateway_module._AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY, {}) + .get("callback_reply", {}) + .get("status") + for call in send_request.await_args_list + ] + assert statuses == [ + "callback_reply_attempt", + "callback_reply_fallback_attempt", + "callback_reply_rescue_attempt", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reply_kind", ["advisory", "category"]) +async def test_interactive_reply_no_send_logs_failure_not_sent( + monkeypatch: pytest.MonkeyPatch, + reply_kind: str, +) -> None: + from src.services import ai_advisory_helpers, callback_dispatcher + + logger = FakeLogger() + gateway = TelegramGateway() + monkeypatch.setattr(gateway_module, "logger", logger) + monkeypatch.setattr(gateway, "_answer_callback", AsyncMock()) + monkeypatch.setattr( + gateway, + "_send_request", + AsyncMock(return_value=dict(NO_SEND_RECEIPT)), + ) + + if reply_kind == "advisory": + monkeypatch.setattr( + ai_advisory_helpers, + "handle_ai_advisory_callback", + AsyncMock( + return_value={ + "feedback_text": "ack", + "success": True, + } + ), + ) + await gateway._handle_ai_advisory_action( + "ai_advisory_view", + "cost:ADV-1", + "callback-1", + 1, + "operator", + {"id": 1}, + inbound_chat_id="room", + inbound_message_id=42, + ) + failed_event = "ai_advisory_group_reply_failed" + sent_event = "ai_advisory_group_reply_sent" + else: + class Repo: + async def get_by_id(self, _incident_id: str): + return None + + monkeypatch.setattr( + callback_dispatcher, + "get_action_spec", + lambda _action: SimpleNamespace(emoji="info"), + ) + monkeypatch.setattr( + callback_dispatcher, + "dispatch_action", + AsyncMock( + return_value=SimpleNamespace( + result_text="result", + success=True, + duration_ms=1.0, + ) + ), + ) + monkeypatch.setattr( + "src.repositories.incident_repository.get_incident_repository", + lambda: Repo(), + ) + await gateway._dispatch_category_action( + "callback-1", + "history", + "INC-1", + 1, + inbound_chat_id="room", + inbound_message_id=42, + ) + failed_event = "category_action_reply_failed" + sent_event = "category_action_reply_sent" + + assert ("warning", failed_event) in logger.events + assert ("info", sent_event) not in logger.events + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("notification_type", "alert_category"), + [("generic", ""), ("TYPE-4D", ""), ("TYPE-8M", "alertchain_health")], +) +async def test_background_no_send_never_confirms_or_writes_sent( + monkeypatch: pytest.MonkeyPatch, + notification_type: str, + alert_category: str, +) -> None: + calls: list[dict] = [] + + class Gateway: + async def send_approval_card(self, **kwargs): + calls.append(kwargs) + return dict(NO_SEND_RECEIPT) + + async def send_drift_card(self, **kwargs): + calls.append(kwargs) + return dict(NO_SEND_RECEIPT) + + async def send_meta_alert(self, **kwargs): + calls.append(kwargs) + return dict(NO_SEND_RECEIPT) + + async def delete_message(self, *_args, **_kwargs): + raise AssertionError("placeholder must not be deleted before delivery") + + class ApprovalService: + async def mark_telegram_confirmed(self, *_args, **_kwargs): + raise AssertionError("no-send must not be marked confirmed") + + monkeypatch.setattr(webhooks.settings, "OPENCLAW_TG_BOT_TOKEN", "configured") + monkeypatch.setattr(webhooks, "get_telegram_gateway", lambda: Gateway()) + monkeypatch.setattr(webhooks, "get_approval_service", lambda: ApprovalService()) + + await webhooks._push_to_telegram_background( + approval_id="APR-1", + risk_level="medium", + resource_name="api", + root_cause="cpu", + suggested_action="inspect", + estimated_downtime="0s", + notification_type=notification_type, + alert_category=alert_category, + fingerprint="fp", + placeholder_message_id=42, + ) + + assert len(calls) == 1 + if notification_type == "generic": + assert calls[0]["notification_type"] == "TYPE-3" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("module_name", ["sentry", "signoz"]) +async def test_monitoring_webhooks_use_type3_and_do_not_false_log_sent( + monkeypatch: pytest.MonkeyPatch, + module_name: str, +) -> None: + logger = FakeLogger() + captured: list[dict] = [] + + class Gateway: + async def initialize(self) -> bool: + return True + + async def send_approval_card(self, **kwargs): + captured.append(kwargs) + return dict(NO_SEND_RECEIPT) + + if module_name == "sentry": + monkeypatch.setattr(sentry_webhook, "logger", logger) + monkeypatch.setattr(sentry_webhook, "get_telegram_gateway", lambda: Gateway()) + await sentry_webhook.send_sentry_telegram_alert( + {"title": "error", "culprit": "api.py", "level": "error"}, + None, + "APR-1", + ) + sent_event = "sentry_telegram_sent" + no_send_event = "sentry_telegram_no_send" + else: + monkeypatch.setattr(signoz_webhook, "logger", logger) + monkeypatch.setattr(signoz_webhook, "get_telegram_gateway", lambda: Gateway()) + await signoz_webhook.send_signoz_telegram( + "APR-1", + "INC-1", + "HighCPU", + {"service": "api"}, + {"summary": "cpu"}, + "warning", + ) + sent_event = "signoz_telegram_sent" + no_send_event = "signoz_telegram_no_send" + + assert captured[0]["notification_type"] == "TYPE-3" + assert ("warning", no_send_event) in logger.events + assert ("info", sent_event) not in logger.events + + +@pytest.mark.asyncio +async def test_dev_test_push_returns_false_for_structured_no_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: list[dict] = [] + + class Gateway: + async def send_approval_card(self, **kwargs): + captured.append(kwargs) + return dict(NO_SEND_RECEIPT) + + monkeypatch.setattr(telegram.settings, "ENVIRONMENT", "dev") + monkeypatch.setattr(telegram, "get_telegram_gateway", lambda: Gateway()) + + response = await telegram.test_push( + telegram.TestPushRequest(approval_id="APR-1") + ) + + assert response["ok"] is False + assert response["message"] == "Test push no-send" + assert captured[0]["notification_type"] == "TYPE-3" diff --git a/apps/api/tests/test_telegram_message_templates.py b/apps/api/tests/test_telegram_message_templates.py index ee1b59ce9..e57038bb2 100644 --- a/apps/api/tests/test_telegram_message_templates.py +++ b/apps/api/tests/test_telegram_message_templates.py @@ -503,6 +503,11 @@ async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch) outbound_receipts.append(kwargs) monkeypatch.setattr(TelegramGateway, "api_url", property(lambda _self: "https://telegram.test/botx")) + monkeypatch.setattr( + telegram_gateway_module.settings, + "SRE_GROUP_CHAT_ID", + "chat", + ) monkeypatch.setattr( telegram_gateway_module, "_agent99_ai_alert_card_mirror_configured", @@ -526,11 +531,18 @@ async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch) "direct ref verifier readback" ), "parse_mode": "MarkdownV2", + "_awoooi_canonical_telegram_route": { + "product_id": "awoooi", + "signal_family": "shared_infrastructure", + "severity": "P1", + }, }, ) await asyncio.sleep(0) - assert result == {"ok": True, "result": {"message_id": 991}} + assert result["ok"] is True + assert result["result"] == {"message_id": 991} + assert result["_awoooi_canonical_route_receipt"]["decision"] == "allowed" assert mirrored assert mirrored[0]["source"] == "send_request" assert "ai_automation_alert_card_v1" in mirrored[0]["text"] @@ -1806,7 +1818,13 @@ async def test_send_request_strips_awooop_callback_metadata_before_telegram_api( { "chat_id": "chat", "text": "事件歷史統計 INC-20260513-79ED5E", + "reply_to_message_id": 123, "_skip_incident_thread_reply": True, + "_awoooi_non_monitoring_interaction": { + "interaction_kind": "callback_reply", + "inbound_chat_id": "chat", + "inbound_message_id": 123, + }, "_awooop_source_envelope_extra": { "callback_reply": { "status": "callback_reply_sent", @@ -1866,8 +1884,9 @@ async def test_send_html_line_message_falls_back_to_plain_text_on_parse_error(mo await gateway._send_html_line_message( ["📊 事件歷史統計", "階段: blocked"], - chat_id="chat", failure_context="test_history", + inbound_chat_id="chat", + inbound_message_id=77, reply_markup=reply_markup, ) @@ -1882,7 +1901,7 @@ async def test_send_html_line_message_falls_back_to_plain_text_on_parse_error(mo @pytest.mark.asyncio async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch): - """詳情/歷史 callback reply 要在 AwoooP envelope 標明 sent/fallback 狀態。""" + """Callback envelope stays attempt until affirmative provider delivery.""" sent_requests = [] gateway = TelegramGateway() reply_markup = telegram_gateway_module._awooop_runs_reply_markup("INC-20260514-F85F21") @@ -1897,8 +1916,9 @@ async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch) await gateway._send_html_line_message( ["📊 事件歷史統計", "🔖 INC-20260514-F85F21"], - chat_id="chat", failure_context="incident_history", + inbound_chat_id="chat", + inbound_message_id=77, reply_markup=reply_markup, incident_id="INC-20260514-F85F21", callback_action="history", @@ -1958,7 +1978,7 @@ async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch) first_extra = first_source_extra["callback_reply"] fallback_extra = fallback_source_extra["callback_reply"] - assert first_extra["status"] == "callback_reply_sent" + assert first_extra["status"] == "callback_reply_attempt" assert first_extra["action"] == "history" assert first_extra["parse_mode"] == "HTML" assert first_source_extra["awooop_status_chain"]["repair_state"] == ( @@ -1969,7 +1989,7 @@ async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch) assert first_source_extra["km_stale_completion_summary"]["triage"]["flow_stage"] == ( "callback_observed_owner_review_link_missing" ) - assert fallback_extra["status"] == "callback_reply_fallback_sent" + assert fallback_extra["status"] == "callback_reply_fallback_attempt" assert fallback_extra["incident_id"] == "INC-20260514-F85F21" assert fallback_extra["parse_mode"] == "plain_text" assert fallback_source_extra["awooop_status_chain"]["next_step"] == ( @@ -1979,6 +1999,37 @@ async def test_send_html_line_message_marks_callback_reply_evidence(monkeypatch) "ai_lead_agent" ] == "Hermes" + acknowledged = ( + telegram_gateway_module._acknowledge_callback_reply_source_envelope( + fallback_source_extra, + delivery_result={"ok": True}, + provider_message_id="88", + ) + ) + assert acknowledged["callback_reply"]["status"] == ( + "callback_reply_fallback_sent" + ) + assert acknowledged["callback_reply"]["provider_delivery_acknowledged"] is True + assert acknowledged["callback_reply"]["provider_message_id"] == "88" + + structured_no_send = ( + telegram_gateway_module._acknowledge_callback_reply_source_envelope( + fallback_source_extra, + delivery_result={ + "ok": True, + "_awooop_delivery_status": "blocked_no_egress", + "_awooop_provider_send_performed": False, + }, + provider_message_id="88", + ) + ) + assert structured_no_send["callback_reply"]["status"] == ( + "callback_reply_fallback_attempt" + ) + assert "provider_delivery_acknowledged" not in structured_no_send[ + "callback_reply" + ] + @pytest.mark.asyncio async def test_send_html_line_message_uses_rescue_when_markup_fallback_fails(monkeypatch): @@ -1997,8 +2048,9 @@ async def test_send_html_line_message_uses_rescue_when_markup_fallback_fails(mon await gateway._send_html_line_message( ["📊 事件歷史統計", "🔖 INC-20260514-F85F21"], - chat_id="chat", failure_context="test_history", + inbound_chat_id="chat", + inbound_message_id=77, reply_markup=reply_markup, ) @@ -2027,8 +2079,9 @@ async def test_send_html_line_message_attaches_awooop_markup_to_first_chunk(monk f"INC-20260514-F85F21 trace line {index:03d}" for index in range(80) ], - chat_id="chat", failure_context="test_history", + inbound_chat_id="chat", + inbound_message_id=77, reply_markup=reply_markup, ) @@ -2048,8 +2101,12 @@ async def test_info_callback_sends_history_when_answer_callback_is_stale(monkeyp async def stale_answer(*_args, **_kwargs): raise telegram_gateway_module.TelegramGatewayError("HTTP error: 400") - async def fake_history(incident_id: str): + async def fake_history(incident_id: str, **context): sent_history.append(incident_id) + assert context == { + "inbound_chat_id": "chat", + "inbound_message_id": 789, + } monkeypatch.setattr(gateway, "_answer_callback", stale_answer) monkeypatch.setattr(gateway, "_send_incident_history", fake_history) @@ -2059,6 +2116,7 @@ async def test_info_callback_sends_history_when_answer_callback_is_stale(monkeyp callback_data="history:INC-20260513-79ED5E", user_id=123456, message_id=789, + chat_id="chat", ) assert result["success"] is True @@ -2083,6 +2141,9 @@ async def test_send_notification_retries_plain_text_on_html_parse_400(monkeypatc result = await gateway.send_notification( "⚠️ 無法取得歷史統計: broken", chat_id="chat", + interaction_kind="callback_reply", + inbound_chat_id="chat", + inbound_message_id=77, ) assert result == {"ok": True} @@ -2104,7 +2165,13 @@ async def test_send_notification_long_html_uses_plain_text_without_cutting_tags( monkeypatch.setattr(gateway, "_send_request", fake_send_request) long_text = "📊 事件歷史統計\n告警鍵: " + ("node<188>&" * 90) + "" - await gateway.send_notification(long_text, chat_id="chat") + await gateway.send_notification( + long_text, + chat_id="chat", + interaction_kind="callback_reply", + inbound_chat_id="chat", + inbound_message_id=77, + ) assert len(sent_requests) == 1 payload = sent_requests[0][1] diff --git a/docs/adr/ADR-093-telegram-group-migration.md b/docs/adr/ADR-093-telegram-group-migration.md index c2a80d42a..1e32585f8 100644 --- a/docs/adr/ADR-093-telegram-group-migration.md +++ b/docs/adr/ADR-093-telegram-group-migration.md @@ -1,5 +1,9 @@ # ADR-093: Telegram 告警全面遷移至 SRE 戰情室群組 +> **2026-07-14 sunset**：本 ADR 的「所有產品/未知類型預設進 SRE 群組」已由 `global_product_governance_v2` 取代。 +> `superseded_by=global_product_governance_v2`；owner=`sre_shared_runtime`；expiry=`2026-07-14T14:35:26+08:00`；replacement=`docs/security/telegram-canonical-routing-registry.snapshot.json`；verifier=`apps/api/tests/test_telegram_canonical_routing_registry.py`。 +> 保留的唯一行為是 allowlisted AWOOOI shared infra/security/DR/P0-P1 lifecycle；未知 route、raw P2/P3、marketing 與跨產品 fallback 一律 blocked。 + > **狀態**: ✅ 已全面切換 > **日期**: 2026-04-24 > **決策者**: 統帥 + 12-Agent 全景分析團隊（onboarder / debugger / db-expert / tool-expert / web-researcher / planner / critic / frontend-designer / fullstack-engineer / refactor-specialist / migration-engineer / vuln-verifier） @@ -8,7 +12,7 @@ 原 Telegram 告警雙軌路由設計： - 所有告警類（TYPE-3/4/4D/8M）預設發私人 DM `OPENCLAW_TG_CHAT_ID` -- 僅心跳報告、週月報、AI 三頭分析發群組 `SRE_GROUP_CHAT_ID=-1003711974679` +- 僅心跳報告、週月報、AI 三頭分析發群組 alias `SRE_GROUP_CHAT_ID`（numeric ID 不進文件） - `_interactive_types` 黑名單曾把互動式告警擋在群組外 使用者需求：把**所有**告警（含需人工簽核的 TYPE-3/4/4D/8M）遷到 AwoooI SRE 戰情室群組（4 名成員），以便團隊共同監察。 diff --git a/docs/schemas/telegram_canonical_routing_registry_v1.schema.json b/docs/schemas/telegram_canonical_routing_registry_v1.schema.json new file mode 100644 index 000000000..e1946ad80 --- /dev/null +++ b/docs/schemas/telegram_canonical_routing_registry_v1.schema.json @@ -0,0 +1,309 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://awoooi.wooo.work/schemas/telegram_canonical_routing_registry_v1.schema.json", + "title": "Telegram Canonical Routing Registry v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "generated_at", + "status", + "mode", + "supersedes", + "policy", + "destination_roles", + "products", + "routes", + "rollups", + "execution_boundaries" + ], + "properties": { + "schema_version": { + "const": "telegram_canonical_routing_registry_v1" + }, + "generated_at": {"type": "string", "format": "date-time"}, + "status": {"const": "source_policy_ready_runtime_not_applied"}, + "mode": {"const": "source_only_no_secret_no_telegram_send_no_route_change"}, + "supersedes": { + "type": "object", + "additionalProperties": false, + "required": [ + "policy_ref", + "superseded_behavior", + "superseded_by", + "owner", + "expiry", + "replacement", + "verifier" + ], + "properties": { + "policy_ref": {"type": "string", "minLength": 1}, + "superseded_behavior": {"type": "string", "minLength": 1}, + "superseded_by": {"const": "global_product_governance_v2"}, + "owner": {"type": "string", "minLength": 1}, + "expiry": {"type": "string", "format": "date-time"}, + "replacement": {"type": "string", "minLength": 1}, + "verifier": {"type": "string", "minLength": 1} + } + }, + "policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "default_decision", + "unknown_product_decision", + "unknown_signal_family_decision", + "unknown_severity_decision", + "unknown_destination_decision", + "legacy_runtime_resolver", + "legacy_shared_sre_allowed_types", + "legacy_shared_sre_blocked_types", + "raw_chat_id_allowed", + "shared_sre_rule", + "shared_sre_forbidden" + ], + "properties": { + "default_decision": {"const": "block"}, + "unknown_product_decision": {"const": "block"}, + "unknown_signal_family_decision": {"const": "block"}, + "unknown_severity_decision": {"const": "block"}, + "unknown_destination_decision": {"const": "block"}, + "legacy_runtime_resolver": {"type": "string", "minLength": 1}, + "legacy_shared_sre_allowed_types": { + "type": "array", + "minItems": 7, + "maxItems": 7, + "uniqueItems": true, + "items": { + "enum": ["TYPE-2", "TYPE-3", "TYPE-4", "TYPE-4D", "TYPE-5S", "TYPE-7E", "TYPE-8M"] + } + }, + "legacy_shared_sre_blocked_types": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "uniqueItems": true, + "items": {"enum": ["TYPE-1", "TYPE-6B", "UNKNOWN"]} + }, + "raw_chat_id_allowed": {"const": false}, + "shared_sre_rule": {"type": "string", "minLength": 1}, + "shared_sre_forbidden": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + } + } + }, + "destination_roles": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "propertyNames": { + "not": { + "pattern": "(?:[Cc][Hh][Aa][Tt][_-]?[Ii][Dd]|[Tt][Oo][Kk][Ee][Nn]|[Ss][Ee][Cc][Rr][Ee][Tt]|[Aa][Uu][Tt][Hh][Oo][Rr][Ii][Zz][Aa][Tt][Ii][Oo][Nn]|[Ww][Ee][Bb][Hh][Oo][Oo][Kk])" + } + }, + "required": [ + "visible_label", + "proven_runtime_alias", + "runtime_alias_evidence", + "chat_alias", + "bot_alias", + "bot_owner", + "monitoring_owner", + "role", + "monitoring_egress_policy", + "runtime_identifier_in_registry" + ], + "properties": { + "visible_label": { + "type": "string", + "minLength": 1, + "pattern": "^(?!-?[0-9]{8,}$).+$" + }, + "proven_runtime_alias": { + "type": "string", + "pattern": "^(?!-?[0-9]{8,}$)[a-z0-9_]+$" + }, + "runtime_alias_evidence": { + "enum": [ + "proven_runtime_receipt", + "source_config_only", + "configured_unreachable", + "visible_label_only" + ] + }, + "chat_alias": { + "type": "string", + "pattern": "^(?!-?[0-9]{8,}$)[a-z0-9_]+$" + }, + "bot_alias": { + "type": "string", + "pattern": "^(?!-?[0-9]{8,}$)[a-z0-9_]+$" + }, + "bot_owner": {"type": "string", "minLength": 1}, + "monitoring_owner": {"type": "boolean"}, + "role": {"type": "string", "minLength": 1}, + "monitoring_egress_policy": {"type": "string", "minLength": 1}, + "runtime_identifier_in_registry": {"const": false} + } + } + }, + "products": { + "type": "array", + "minItems": 11, + "maxItems": 11, + "items": { + "type": "object", + "additionalProperties": false, + "propertyNames": { + "not": { + "pattern": "(?:[Cc][Hh][Aa][Tt][_-]?[Ii][Dd]|[Tt][Oo][Kk][Ee][Nn]|[Ss][Ee][Cc][Rr][Ee][Tt]|[Aa][Uu][Tt][Hh][Oo][Rr][Ii][Zz][Aa][Tt][Ii][Oo][Nn]|[Ww][Ee][Bb][Hh][Oo][Oo][Kk])" + } + }, + "required": [ + "product_id", + "display_name", + "owner", + "source_audit_status", + "egress_status" + ], + "properties": { + "product_id": {"type": "string", "minLength": 1}, + "display_name": {"type": "string", "minLength": 1}, + "owner": {"type": "string", "minLength": 1}, + "source_audit_status": {"type": "string", "minLength": 1}, + "egress_status": { + "enum": ["allowed", "blocked", "disabled", "not_implemented"] + } + } + } + }, + "routes": { + "type": "array", + "minItems": 11, + "items": { + "type": "object", + "additionalProperties": false, + "propertyNames": { + "not": { + "pattern": "(?:[Cc][Hh][Aa][Tt][_-]?[Ii][Dd]|[Tt][Oo][Kk][Ee][Nn]|[Ss][Ee][Cc][Rr][Ee][Tt]|[Aa][Uu][Tt][Hh][Oo][Rr][Ii][Zz][Aa][Tt][Ii][Oo][Nn]|[Ww][Ee][Bb][Hh][Oo][Oo][Kk])" + } + }, + "required": [ + "route_id", + "product_id", + "signal_family", + "severity", + "scope", + "bot_alias", + "chat_alias", + "allowed_destination", + "receipt_backend", + "owner", + "egress_status" + ], + "properties": { + "route_id": {"type": "string", "minLength": 1}, + "product_id": {"type": "string", "minLength": 1}, + "signal_family": {"type": "string", "minLength": 1}, + "severity": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"enum": ["P0", "P1", "P2", "P3"]} + }, + "scope": {"enum": ["shared", "product"]}, + "bot_alias": { + "type": "string", + "pattern": "^(?!-?[0-9]{8,}$)[a-z0-9_]+$" + }, + "chat_alias": { + "type": "string", + "pattern": "^(?!-?[0-9]{8,}$)[a-z0-9_]+$" + }, + "allowed_destination": { + "type": "string", + "pattern": "^(?:blocked|telegram_chat_alias:[a-z0-9_]+)$" + }, + "receipt_backend": { + "type": "string", + "minLength": 1, + "pattern": "^(?!-?[0-9]{8,}$).+$" + }, + "owner": {"type": "string", "minLength": 1}, + "egress_status": { + "enum": ["allowed", "blocked", "disabled", "not_implemented"] + }, + "blocked_reason": {"type": "string", "minLength": 1} + }, + "allOf": [ + { + "if": {"properties": {"egress_status": {"const": "allowed"}}}, + "then": { + "properties": { + "allowed_destination": { + "pattern": "^telegram_chat_alias:[a-z0-9_]+$" + } + } + }, + "else": { + "required": ["blocked_reason"], + "properties": {"allowed_destination": {"const": "blocked"}} + } + } + ] + } + }, + "rollups": { + "type": "object", + "additionalProperties": false, + "required": [ + "product_count", + "route_count", + "allowed_route_count", + "blocked_route_count", + "disabled_route_count", + "not_implemented_route_count", + "shared_sre_allowed_route_count", + "raw_numeric_destination_count", + "runtime_route_change_count", + "telegram_send_count" + ], + "properties": { + "product_count": {"const": 11}, + "route_count": {"type": "integer", "minimum": 11}, + "allowed_route_count": {"type": "integer", "minimum": 0}, + "blocked_route_count": {"type": "integer", "minimum": 0}, + "disabled_route_count": {"type": "integer", "minimum": 0}, + "not_implemented_route_count": {"type": "integer", "minimum": 0}, + "shared_sre_allowed_route_count": {"type": "integer", "minimum": 0}, + "raw_numeric_destination_count": {"const": 0}, + "runtime_route_change_count": {"const": 0}, + "telegram_send_count": {"const": 0} + } + }, + "execution_boundaries": { + "type": "object", + "additionalProperties": false, + "required": [ + "telegram_send_allowed", + "bot_api_call_allowed", + "secret_value_read_allowed", + "raw_chat_id_storage_allowed", + "runtime_route_change_allowed", + "production_deploy_allowed" + ], + "properties": { + "telegram_send_allowed": {"const": false}, + "bot_api_call_allowed": {"const": false}, + "secret_value_read_allowed": {"const": false}, + "raw_chat_id_storage_allowed": {"const": false}, + "runtime_route_change_allowed": {"const": false}, + "production_deploy_allowed": {"const": false} + } + } + } +} diff --git a/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md b/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md new file mode 100644 index 000000000..2e30d028a --- /dev/null +++ b/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md @@ -0,0 +1,60 @@ +# Telegram 監控告警路由總帳 + +> 狀態：`source_policy_ready_runtime_not_applied` +> 真相邊界：本表把「已證明現況」、「建議目的地」、「目前阻擋」、「source risk」與「receipt 強度」分開。建議不等於 live route；alias 不是 numeric chat ID；本變更沒有送 Telegram、讀 token/chat ID、改 runtime route 或部署。 + +## 判讀規則 + +- `visible label` 只描述使用者畫面中的名稱；`proven runtime alias` 必須另有 source/runtime receipt，兩者不得互相替代。 +- `proven route` 只描述目前可由 source 或 durable receipt 證明的行為，不把畫面名稱或建議當成 live delivery。 +- `recommended route` 是 canonical 產品責任分流；若 bot alias、chat alias 或 durable receipt 未證明，egress 仍是 `blocked`。 +- `receipt strength`：`strong` 代表 durable DB receipt；`weak` 代表 console/source 行為；`none` 代表沒有可驗證的 delivery receipt。 +- 未知 product/signal/severity/destination 一律 no-egress；不再沿用 ADR-093 的 unknown-to-shared-group fallback。 +- `AwoooI SRE 戰情室` 只收 shared infrastructure、security、DR 與 P0/P1 incident lifecycle。raw P2/P3、marketing、成功 heartbeat、未驗證 recommendation 和 bot 對話均不得進入。 + +## 11 產品路由清冊 + +| 產品 | 目前 proven route | 建議 canonical route | blocked unknown / egress status | source risk | receipt strength | +|---|---|---|---|---|---| +| `2026fifa` | source-only monitoring surface；沒有 runtime Telegram binding | 建立專屬 product alias 後，只送 data freshness、recommendation build、public API failure | `blocked`：chat/bot alias 與 durable receipt 均未證明 | generic env 可能誤入其他產品 lane | `none` | +| `agent-bounty-protocol` | `VibeAIAgent` bot-to-group route 已證明，涵蓋 A2A/traffic/treasury/onboarding；durable receipt 仍缺 | `vibe_ai_agent_control_plane`，先收斂到 gateway 與 durable receipt 再開 egress | `blocked`：route 存在但 console/source log 不能代替 durable delivery receipt | 訊息標題仍可能把 Agent Bounty 與 standalone VibeWork 混為同一產品 | `weak`：console/source only | +| `awooogo` | 沒有 runtime Telegram binding，egress disabled | 維持 disabled；未來啟用需先建立獨立 product alias/receipt | `disabled` | 不得用 shared SRE 當缺設定 fallback | `none` | +| `awoooi` | `AwoooI SRE 戰情室` 與 TsenYang monitoring-owner receipt 已證明；`小O_小龍蝦`/`叫小賀` 是 visible label/source-config candidate，不當成 runtime alias proof | TsenYang bot 才能送 canonical shared infra/security/DR/P0-P1 lifecycle；OpenClaw/NemoTron 只可在原訊息 interaction context 下回覆 | shared allowlist `allowed`；raw P2/P3、business/FinOps、marketing `blocked` | 若未驗 sender bot ownership，AI bot 會繞過 canonical alert owner | `strong`（僅 shared monitoring owner） | +| `bitan-pharmacy` | source-only product/container signals；沒有 canonical runtime Telegram binding | 建立 Bitan-owned alias 後才允許 product health/failure | `blocked`：目的地、bot alias、receipt backend 未證明 | cross-product container signal 容易被誤當 AWOOOI shared infra | `none` | +| `clawbot-openclaw` | 目前使用 `tsenyangbot` private lane，形成跨產品 bot ownership drift | 建立 ClawBot-owned bot/chat binding；interactive reply 與 monitoring 分離 | `blocked`：現有 private route 不具 ClawBot canonical ownership | 共用 AWOOOI bot 會造成 polling ownership、回覆與告警歸戶混亂 | `none` | +| `momo-pro` | MOMO bot live；configured chat readback 回 `400 chat_not_found` | 修復 MOMO-owned destination 後，才承接 crawler/freshness、scheduler/sales pipeline | `blocked`：runtime destination unreachable，且無 durable receipt | `WOOO` 只是 private bot lane/inferred，不是已證明 MOMO product group | `none` | +| `stockplatform-v2` | 與 TSENYANG Website 共用 `wooowooowooobot` private lane | 建立 StockPlatform-owned destination，只送 market data freshness、pipeline/API failure | `blocked`：跨產品 shared-private ownership drift，無 durable receipt | 同一 private bot lane 無法清楚歸戶、抑噪與回寫 | `none` | +| `tsenyang-website` | 與 StockPlatform 共用 `wooowooowooobot` private lane | 建立 TSENYANG Website-owned destination；operator interaction 不承接 bulk raw alert | `blocked`：跨產品 shared-private ownership drift，無 durable receipt | site 與 market-data 告警混流，owner/verifier 不可辨 | `none` | +| `vibework` | standalone product 沒有 runtime Telegram binding；`VibeAIAgent` 屬 Agent Bounty | 保持 `not_implemented`；若實作需獨立 product_id、alias 與 receipt | `not_implemented` | 名稱相似會造成錯誤歸戶與重複告警 | `none` | +| `vtuber` | 沒有 runtime Telegram binding，egress disabled | 維持 disabled；未來啟用須先建立 VTuber-owned alias | `disabled` | shared monitor 辨識到 signal 不等於產品已有 sender | `none` | + +## 群組與 BOT 的責任邊界 + +| visible label | proven runtime alias | canonical candidate | 可以承接 | 不可以承接 | +|---|---|---|---|---| +| AwoooI SRE 戰情室 | `awoooi_sre_war_room` | TsenYang monitoring owner | shared infra/security/DR、P0/P1 lifecycle | raw P2/P3、business/FinOps、marketing、成功 heartbeat、各產品 bulk warning | +| 小O_小龍蝦 | `none`（visible label only） | `openclaw_bot` source-config candidate | 只在原 inbound message/thread context 下做診斷回覆 | canonical production alert ownership、root alert fanout | +| 叫小賀 | `none`（source-config only） | `nemotron_bot` candidate | 只在原 inbound message/thread context 下做 challenger/shadow reply | canonical production alert ownership | +| TsenYang private | `none`（visible label only） | `wooowooowooobot_private` candidate | 無；僅保留 ownership drift evidence | 在 ownership reconciliation 前不得視為任何產品 canonical route | +| tsenyangbot private | `none`（source-config only） | `tsenyangbot_private` candidate | 無；目前 ClawBot traffic 只作 drift evidence | 跨產品 monitoring ownership | +| WOOO | `none`（visible label only） | `wooo_private_bot_lane` candidate | 尚未證明任何 product-group ownership | 不得當作 MOMO 或 eWoooc canonical product group | +| VibeAIAgent | `none`（source-config only） | `vibe_ai_agent_control_plane` candidate | Agent Bounty A2A/traffic/treasury/onboarding（補 durable receipt 後） | standalone VibeWork 或其他產品告警 | +| MOMO configured chat | `none`（configured unreachable） | `momo_configured_chat_unresolved` candidate | 無；目前 `chat_not_found` | 在 runtime readback、receipt 與 verifier 通過前不得送 | + +## Source risk 與落地缺口 + +1. ADR-093 的 unknown/default-to-group 與全產品治理 v2 衝突；legacy `resolve_chat_ids` 現在必須先通過 canonical registry。unknown、`TYPE-1` info/success 與 `TYPE-6B` business 全部回空 route，不再進 shared SRE。 +2. Agent Bounty 的 bot-to-`VibeAIAgent` route 已證明，但 direct Bot API 與 console log 仍不是 durable receipt；因此 route 存在、egress 仍 blocked 兩件事必須同時呈現。 +3. MOMO bot live 不等於 destination healthy；目前 `chat_not_found` 是明確 blocked runtime truth。`WOOO` 也只能標為 private/inferred lane，不能升格成 product group。 +4. StockPlatform/TSENYANG Website 共用 private bot，ClawBot 又使用 AWOOOI-owned bot，都是 ownership drift；需各自 owner、gateway、receipt、verifier 後才能開 route。 +5. Bitan、2026FIFA 是 source-only；AwoooGo、VibeWork、VTuber 是 disabled/not-implemented/no binding。它們不得因缺專屬群而 fallback 到 shared SRE。 +6. Gateway 現在會在 provider call 前驗 product/signal/severity、destination alias 與 sender bot alias；非監控的 inbound command/callback/thread reply 使用獨立 interaction classifier，不與監控 allowlist 混用。 + +## Machine-readable truth + +- Registry：`docs/security/telegram-canonical-routing-registry.snapshot.json` +- Schema：`docs/schemas/telegram_canonical_routing_registry_v1.schema.json` +- Resolver：`apps/api/src/services/notification_matrix.py` +- Tests：`apps/api/tests/test_telegram_canonical_routing_registry.py` + +Production apply 前仍需同一 route change run 留下 source diff、check-mode、bounded apply、durable delivery receipt、post-verifier、rollback/no-write terminal 與 learning/writeback；本 source-only 交付不代表 runtime route 已更動。 diff --git a/docs/security/telegram-canonical-routing-registry.snapshot.json b/docs/security/telegram-canonical-routing-registry.snapshot.json new file mode 100644 index 000000000..d89ea55d0 --- /dev/null +++ b/docs/security/telegram-canonical-routing-registry.snapshot.json @@ -0,0 +1,522 @@ +{ + "schema_version": "telegram_canonical_routing_registry_v1", + "generated_at": "2026-07-14T15:50:04+08:00", + "status": "source_policy_ready_runtime_not_applied", + "mode": "source_only_no_secret_no_telegram_send_no_route_change", + "supersedes": { + "policy_ref": "docs/adr/ADR-093-telegram-group-migration.md", + "superseded_behavior": "unknown_or_cross_product_alert_defaults_to_shared_sre_group", + "superseded_by": "global_product_governance_v2", + "owner": "sre_shared_runtime", + "expiry": "2026-07-14T14:35:26+08:00", + "replacement": "this canonical registry and notification_matrix fail-closed resolver", + "verifier": "apps/api/tests/test_telegram_canonical_routing_registry.py" + }, + "policy": { + "default_decision": "block", + "unknown_product_decision": "block", + "unknown_signal_family_decision": "block", + "unknown_severity_decision": "block", + "unknown_destination_decision": "block", + "legacy_runtime_resolver": "resolve_chat_ids_must_pass_canonical_registry", + "legacy_shared_sre_allowed_types": [ + "TYPE-2", + "TYPE-3", + "TYPE-4", + "TYPE-4D", + "TYPE-5S", + "TYPE-7E", + "TYPE-8M" + ], + "legacy_shared_sre_blocked_types": ["TYPE-1", "TYPE-6B", "UNKNOWN"], + "raw_chat_id_allowed": false, + "shared_sre_rule": "Only shared infrastructure, security incident, disaster recovery, and P0/P1 incident lifecycle signals may use the AwoooI SRE war room alias.", + "shared_sre_forbidden": [ + "raw product P2/P3 monitoring", + "marketing", + "success heartbeat", + "unverified recommendation", + "bot conversation" + ] + }, + "destination_roles": [ + { + "visible_label": "AwoooI SRE 戰情室", + "proven_runtime_alias": "awoooi_sre_war_room", + "runtime_alias_evidence": "proven_runtime_receipt", + "chat_alias": "awoooi_sre_war_room", + "bot_alias": "tsenyang_bot", + "bot_owner": "sre_shared_runtime", + "monitoring_owner": true, + "role": "shared infrastructure, security, DR, and P0/P1 incident lifecycle", + "monitoring_egress_policy": "allowlist_only", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "TsenYang", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "visible_label_only", + "chat_alias": "wooowooowooobot_private", + "bot_alias": "wooowooowooobot", + "bot_owner": "unverified", + "monitoring_owner": false, + "role": "visible private label only; StockPlatform and TSENYANG Website runtime alias ownership is not proven", + "monitoring_egress_policy": "blocked_cross_product_ownership_drift", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "小O_小龍蝦", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "visible_label_only", + "chat_alias": "awoooi_sre_war_room", + "bot_alias": "openclaw_bot", + "bot_owner": "ai_agent_runtime", + "monitoring_owner": false, + "role": "visible group label mapped to a source-configured OpenClaw reply candidate; runtime alias is not proven by this label", + "monitoring_egress_policy": "interaction_reply_only_no_monitoring_ownership", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "叫小賀", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "source_config_only", + "chat_alias": "awoooi_sre_war_room", + "bot_alias": "nemotron_bot", + "bot_owner": "ai_agent_runtime", + "monitoring_owner": false, + "role": "source-configured NemoTron reply candidate; visible label alone is not runtime proof", + "monitoring_egress_policy": "shadow_reply_only_no_production_alert_ownership", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "WOOO", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "visible_label_only", + "chat_alias": "wooo_private_bot_lane", + "bot_alias": "none", + "bot_owner": "unverified", + "monitoring_owner": false, + "role": "observed private bot lane; product-group ownership is inferred and not proven", + "monitoring_egress_policy": "blocked_not_a_proven_product_group", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "VibeAIAgent", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "source_config_only", + "chat_alias": "vibe_ai_agent_control_plane", + "bot_alias": "vibe_ai_agent_bot", + "bot_owner": "agent_bounty_product_ops", + "monitoring_owner": false, + "role": "proven Agent Bounty bot-to-group route for A2A task, traffic, treasury, and onboarding signals", + "monitoring_egress_policy": "blocked_until_durable_receipt_backend_is_verified", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "tsenyangbot private", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "source_config_only", + "chat_alias": "tsenyangbot_private", + "bot_alias": "tsenyang_bot", + "bot_owner": "sre_shared_runtime", + "monitoring_owner": false, + "role": "observed ClawBot private lane using an AWOOOI-owned bot binding", + "monitoring_egress_policy": "blocked_cross_product_bot_ownership_drift", + "runtime_identifier_in_registry": false + }, + { + "visible_label": "MOMO configured chat", + "proven_runtime_alias": "none", + "runtime_alias_evidence": "configured_unreachable", + "chat_alias": "momo_configured_chat_unresolved", + "bot_alias": "momo_bot", + "bot_owner": "momo_product_ops", + "monitoring_owner": false, + "role": "MOMO bot is live but the configured destination currently returns chat_not_found", + "monitoring_egress_policy": "blocked_runtime_destination_unreachable", + "runtime_identifier_in_registry": false + } + ], + "products": [ + { + "product_id": "2026fifa", + "display_name": "2026 FIFA World Cup", + "owner": "2026fifa_product_ops", + "source_audit_status": "source_only_no_runtime_telegram_binding", + "egress_status": "blocked" + }, + { + "product_id": "agent-bounty-protocol", + "display_name": "Agent Bounty Protocol", + "owner": "agent_bounty_product_ops", + "source_audit_status": "vibe_ai_agent_bot_to_group_route_proven_durable_receipt_gap", + "egress_status": "blocked" + }, + { + "product_id": "awooogo", + "display_name": "AwoooGo", + "owner": "awooogo_product_ops", + "source_audit_status": "no_runtime_telegram_binding_egress_disabled", + "egress_status": "disabled" + }, + { + "product_id": "awoooi", + "display_name": "AWOOOI / IWOOOS", + "owner": "sre_shared_runtime", + "source_audit_status": "shared_group_and_tsenyang_monitoring_owner_receipt_proven_openclaw_nemotron_labels_not_runtime_proof", + "egress_status": "allowed" + }, + { + "product_id": "bitan-pharmacy", + "display_name": "Bitan Pharmacy", + "owner": "bitan_product_ops", + "source_audit_status": "source_only_no_canonical_runtime_telegram_binding", + "egress_status": "blocked" + }, + { + "product_id": "clawbot-openclaw", + "display_name": "Clawbot / OpenClaw", + "owner": "ai_agent_runtime", + "source_audit_status": "tsenyangbot_private_route_observed_cross_product_bot_ownership_drift", + "egress_status": "blocked" + }, + { + "product_id": "momo-pro", + "display_name": "MOMO PRO / eWoooc", + "owner": "momo_product_ops", + "source_audit_status": "momo_bot_live_configured_chat_returns_chat_not_found", + "egress_status": "blocked" + }, + { + "product_id": "stockplatform-v2", + "display_name": "StockPlatform v2", + "owner": "stockplatform_product_ops", + "source_audit_status": "wooowooowooobot_private_route_shared_with_tsenyang_site_ownership_drift", + "egress_status": "blocked" + }, + { + "product_id": "tsenyang-website", + "display_name": "TSENYANG Website", + "owner": "tsenyang_site_ops", + "source_audit_status": "wooowooowooobot_private_route_shared_with_stockplatform_ownership_drift", + "egress_status": "blocked" + }, + { + "product_id": "vibework", + "display_name": "VibeWork", + "owner": "vibework_product_ops", + "source_audit_status": "no_runtime_telegram_binding_agent_bounty_is_separate_owner", + "egress_status": "not_implemented" + }, + { + "product_id": "vtuber", + "display_name": "VTuber", + "owner": "vtuber_product_ops", + "source_audit_status": "no_runtime_telegram_binding_egress_disabled", + "egress_status": "disabled" + } + ], + "routes": [ + { + "route_id": "2026fifa-product-monitoring", + "product_id": "2026fifa", + "signal_family": "data_freshness_and_recommendation", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "unassigned", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "2026fifa_product_ops", + "egress_status": "blocked", + "blocked_reason": "canonical_product_destination_and_durable_receipt_backend_missing" + }, + { + "route_id": "agent-bounty-a2a-task-lifecycle", + "product_id": "agent-bounty-protocol", + "signal_family": "a2a_task_lifecycle", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "vibe_ai_agent_bot", + "chat_alias": "vibe_ai_agent_control_plane", + "allowed_destination": "blocked", + "receipt_backend": "console_log_only_unverified", + "owner": "agent_bounty_product_ops", + "egress_status": "blocked", + "blocked_reason": "direct_bot_api_and_durable_delivery_receipt_gap" + }, + { + "route_id": "agent-bounty-traffic-security", + "product_id": "agent-bounty-protocol", + "signal_family": "traffic_security", + "severity": ["P0", "P1", "P2"], + "scope": "product", + "bot_alias": "vibe_ai_agent_bot", + "chat_alias": "vibe_ai_agent_control_plane", + "allowed_destination": "blocked", + "receipt_backend": "console_log_only_unverified", + "owner": "agent_bounty_product_ops", + "egress_status": "blocked", + "blocked_reason": "direct_bot_api_and_durable_delivery_receipt_gap" + }, + { + "route_id": "agent-bounty-treasury-onboarding", + "product_id": "agent-bounty-protocol", + "signal_family": "treasury_and_onboarding", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "vibe_ai_agent_bot", + "chat_alias": "vibe_ai_agent_control_plane", + "allowed_destination": "blocked", + "receipt_backend": "console_log_only_unverified", + "owner": "agent_bounty_product_ops", + "egress_status": "blocked", + "blocked_reason": "direct_bot_api_and_durable_delivery_receipt_gap" + }, + { + "route_id": "awooogo-disabled", + "product_id": "awooogo", + "signal_family": "*", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "awooogo_product_ops", + "egress_status": "disabled", + "blocked_reason": "telegram_egress_disabled" + }, + { + "route_id": "awoooi-shared-infrastructure", + "product_id": "awoooi", + "signal_family": "shared_infrastructure", + "severity": ["P0", "P1"], + "scope": "shared", + "bot_alias": "tsenyang_bot", + "chat_alias": "awoooi_sre_war_room", + "allowed_destination": "telegram_chat_alias:awoooi_sre_war_room", + "receipt_backend": "awoooi.alert_operation_log+telegram_outbound_receipts", + "owner": "sre_shared_runtime", + "egress_status": "allowed" + }, + { + "route_id": "awoooi-security-incident", + "product_id": "awoooi", + "signal_family": "security_incident", + "severity": ["P0", "P1"], + "scope": "shared", + "bot_alias": "tsenyang_bot", + "chat_alias": "awoooi_sre_war_room", + "allowed_destination": "telegram_chat_alias:awoooi_sre_war_room", + "receipt_backend": "awoooi.alert_operation_log+telegram_outbound_receipts", + "owner": "security_runtime", + "egress_status": "allowed" + }, + { + "route_id": "awoooi-disaster-recovery", + "product_id": "awoooi", + "signal_family": "disaster_recovery", + "severity": ["P0", "P1"], + "scope": "shared", + "bot_alias": "tsenyang_bot", + "chat_alias": "awoooi_sre_war_room", + "allowed_destination": "telegram_chat_alias:awoooi_sre_war_room", + "receipt_backend": "awoooi.alert_operation_log+telegram_outbound_receipts", + "owner": "dr_runtime", + "egress_status": "allowed" + }, + { + "route_id": "awoooi-incident-lifecycle", + "product_id": "awoooi", + "signal_family": "incident_lifecycle", + "severity": ["P0", "P1"], + "scope": "shared", + "bot_alias": "tsenyang_bot", + "chat_alias": "awoooi_sre_war_room", + "allowed_destination": "telegram_chat_alias:awoooi_sre_war_room", + "receipt_backend": "awoooi.alert_operation_log+telegram_outbound_receipts", + "owner": "sre_shared_runtime", + "egress_status": "allowed" + }, + { + "route_id": "awoooi-raw-product-monitoring", + "product_id": "awoooi", + "signal_family": "product_raw_monitoring", + "severity": ["P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "awoooi.alert_operation_log", + "owner": "product_route_registry", + "egress_status": "blocked", + "blocked_reason": "raw_p2_p3_must_not_use_shared_sre_destination" + }, + { + "route_id": "awoooi-marketing", + "product_id": "awoooi", + "signal_family": "marketing", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "product_route_registry", + "egress_status": "blocked", + "blocked_reason": "marketing_must_not_use_monitoring_egress" + }, + { + "route_id": "awoooi-business-finops", + "product_id": "awoooi", + "signal_family": "business_finops", + "severity": [ + "P0", + "P1", + "P2", + "P3" + ], + "scope": "product", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "awoooi.alert_operation_log", + "owner": "finops_runtime", + "egress_status": "blocked", + "blocked_reason": "business_and_cost_signals_require_a_separate_owned_destination" + }, + { + "route_id": "bitan-product-health", + "product_id": "bitan-pharmacy", + "signal_family": "container_and_product_health", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "unassigned", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "bitan_product_ops", + "egress_status": "blocked", + "blocked_reason": "current_shared_sre_fanout_is_not_a_canonical_product_route" + }, + { + "route_id": "clawbot-raw-monitoring", + "product_id": "clawbot-openclaw", + "signal_family": "raw_monitoring_alert", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "tsenyang_bot", + "chat_alias": "tsenyangbot_private", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "ai_agent_runtime", + "egress_status": "blocked", + "blocked_reason": "cross_product_bot_ownership_drift_requires_a_clawbot_owned_binding" + }, + { + "route_id": "momo-data-freshness", + "product_id": "momo-pro", + "signal_family": "crawler_and_data_freshness", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "momo_bot", + "chat_alias": "momo_configured_chat_unresolved", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "momo_product_ops", + "egress_status": "blocked", + "blocked_reason": "configured_destination_returns_chat_not_found_and_has_no_durable_receipt" + }, + { + "route_id": "momo-scheduler-sales", + "product_id": "momo-pro", + "signal_family": "scheduler_and_sales_pipeline", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "momo_bot", + "chat_alias": "momo_configured_chat_unresolved", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "momo_product_ops", + "egress_status": "blocked", + "blocked_reason": "configured_destination_returns_chat_not_found_and_has_no_durable_receipt" + }, + { + "route_id": "stockplatform-market-data", + "product_id": "stockplatform-v2", + "signal_family": "market_data_and_recommendation_freshness", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "wooowooowooobot", + "chat_alias": "wooowooowooobot_private", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "stockplatform_product_ops", + "egress_status": "blocked", + "blocked_reason": "private_route_is_shared_with_tsenyang_site_and_product_ownership_is_not_reconciled" + }, + { + "route_id": "tsenyang-site-health", + "product_id": "tsenyang-website", + "signal_family": "public_site_and_agent_action_health", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "wooowooowooobot", + "chat_alias": "wooowooowooobot_private", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "tsenyang_site_ops", + "egress_status": "blocked", + "blocked_reason": "private_route_is_shared_with_stockplatform_and_product_ownership_is_not_reconciled" + }, + { + "route_id": "vibework-not-implemented", + "product_id": "vibework", + "signal_family": "*", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "vibework_product_ops", + "egress_status": "not_implemented", + "blocked_reason": "standalone_vibework_has_no_telegram_egress_agent_bounty_is_separate" + }, + { + "route_id": "vtuber-disabled", + "product_id": "vtuber", + "signal_family": "*", + "severity": ["P0", "P1", "P2", "P3"], + "scope": "product", + "bot_alias": "none", + "chat_alias": "none", + "allowed_destination": "blocked", + "receipt_backend": "none", + "owner": "vtuber_product_ops", + "egress_status": "disabled", + "blocked_reason": "telegram_egress_disabled" + } + ], + "rollups": { + "product_count": 11, + "route_count": 20, + "allowed_route_count": 4, + "blocked_route_count": 13, + "disabled_route_count": 2, + "not_implemented_route_count": 1, + "shared_sre_allowed_route_count": 4, + "raw_numeric_destination_count": 0, + "runtime_route_change_count": 0, + "telegram_send_count": 0 + }, + "execution_boundaries": { + "telegram_send_allowed": false, + "bot_api_call_allowed": false, + "secret_value_read_allowed": false, + "raw_chat_id_storage_allowed": false, + "runtime_route_change_allowed": false, + "production_deploy_allowed": false + } +} diff --git a/scripts/reboot-recovery/awoooi-telegram-notify.sh b/scripts/reboot-recovery/awoooi-telegram-notify.sh index 4eee1609e..3646f0351 100755 --- a/scripts/reboot-recovery/awoooi-telegram-notify.sh +++ b/scripts/reboot-recovery/awoooi-telegram-notify.sh @@ -16,14 +16,19 @@ while [[ $# -gt 0 ]]; do esac done -: "${TELEGRAM_BOT_TOKEN:?TELEGRAM_BOT_TOKEN is required in host-local env}" -: "${TELEGRAM_CHAT_ID:?TELEGRAM_CHAT_ID is required in host-local env}" - TEXT="[AWOOOI reboot-recovery] phase=${PHASE} status=${STATUS} ${MESSAGE} artifact=${ARTIFACT_DIR}" -curl --fail --silent --show-error --max-time 10 \ - --request POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ - --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \ - --data-urlencode "text=${TEXT}" >/dev/null +# Reboot recovery is outside the canonical API receipt boundary. Do not read +# a host-local Telegram binding and do not fall back to a direct provider call. +# The orchestrator keeps its local evidence; a correlated gateway transport +# must be supplied before Telegram delivery can be re-enabled. +printf '%s\n' \ + "telegram_route_status=blocked_no_egress" \ + "telegram_provider_send_performed=false" \ + "telegram_blocked_reason=canonical_telegram_gateway_transport_required" \ + "telegram_message_class=reboot_recovery" \ + "telegram_phase=${PHASE}" \ + "telegram_status=${STATUS}" >&2 +exit 75 diff --git a/scripts/security/telegram-notification-egress-inventory.py b/scripts/security/telegram-notification-egress-inventory.py index 6ee829cbf..ec392c273 100644 --- a/scripts/security/telegram-notification-egress-inventory.py +++ b/scripts/security/telegram-notification-egress-inventory.py @@ -22,21 +22,37 @@ from typing import Any TAIPEI = timezone(timedelta(hours=8)) SCAN_ROOTS = ( + Path("agent99-control-plane.ps1"), Path(".gitea/workflows"), Path("scripts/ops"), Path("scripts/ci"), + Path("scripts/reboot-recovery"), Path("apps/api/src"), ) -SCAN_SUFFIXES = {".py", ".sh", ".js", ".yml", ".yaml"} +SCAN_SUFFIXES = {".py", ".ps1", ".sh", ".js", ".yml", ".yaml"} +GUARDED_BOT_METHODS = ( + "sendMessage", + "sendDocument", + "sendPhoto", + "sendMediaGroup", + "editMessageText", + "sendAnimation", + "sendVideo", + "sendAudio", + "sendVoice", +) DIRECT_BOT_API_RE = re.compile( - r"api\.telegram\.org/bot.*sendMessage|sendMessage.*api\.telegram\.org/bot" + r"api\.telegram\.org/bot.*?/(?P" + + "|".join(re.escape(method) for method in GUARDED_BOT_METHODS) + + r")\b", + re.IGNORECASE, ) GATEWAY_CALLSITE_RE = re.compile( r"(?:send_alert_notification\(|\b(?:tg|gw|gateway|telegram)\.send_text\(|_send_request\(\s*[\"']sendMessage[\"'])" ) SECRET_INTERPOLATION_RE = re.compile(r"\$\{\{\s*secrets\.[^}]+\}\}") -BOT_TOKEN_URL_RE = re.compile(r"api\.telegram\.org/bot.*?/sendMessage") +BOT_TOKEN_URL_RE = DIRECT_BOT_API_RE REQUIRED_OWNER_FIELDS = [ "egress_surface_id", @@ -134,6 +150,10 @@ def iter_scannable_files(root: Path) -> list[Path]: absolute_root = root / scan_root if not absolute_root.exists(): continue + if absolute_root.is_file(): + if absolute_root.suffix in SCAN_SUFFIXES: + files.append(absolute_root) + continue for path in absolute_root.rglob("*"): if path.is_file() and path.suffix in SCAN_SUFFIXES: files.append(path) @@ -143,7 +163,10 @@ def iter_scannable_files(root: Path) -> list[Path]: def sanitize_excerpt(line: str) -> str: excerpt = line.strip() excerpt = SECRET_INTERPOLATION_RE.sub("${{ secrets. }}", excerpt) - excerpt = BOT_TOKEN_URL_RE.sub("api.telegram.org/bot/sendMessage", excerpt) + excerpt = BOT_TOKEN_URL_RE.sub( + lambda match: f"api.telegram.org/bot/{match.group('method')}", + excerpt, + ) return excerpt[:180] @@ -154,6 +177,10 @@ def surface_kind(relative_path: str) -> str: return "ops_script_direct_bot_api" if relative_path.startswith("scripts/ci/"): return "ci_script_direct_bot_api" + if relative_path == "agent99-control-plane.ps1": + return "agent99_control_plane_direct_bot_api" + if relative_path.startswith("scripts/reboot-recovery/"): + return "reboot_recovery_direct_bot_api" if relative_path.startswith("apps/api/src/"): return "api_direct_bot_api" return "other_direct_bot_api" @@ -174,7 +201,8 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: relative_path = path.relative_to(root).as_posix() text = path.read_text(encoding="utf-8", errors="replace") for line_number, line in enumerate(text.splitlines(), start=1): - if DIRECT_BOT_API_RE.search(line): + direct_match = DIRECT_BOT_API_RE.search(line) + if direct_match: kind = surface_kind(relative_path) direct_calls.append( { @@ -183,6 +211,7 @@ def build_report(root: Path, generated_at: str | None = None) -> dict[str, Any]: "path": relative_path, "line": line_number, "line_hash": line_hash(relative_path, line_number, line), + "method": direct_match.group("method"), "sanitized_excerpt": sanitize_excerpt(line), "required_owner_fields": REQUIRED_OWNER_FIELDS, "reviewer_checks": REVIEWER_CHECKS, diff --git a/scripts/security/telegram-notification-egress-no-new-bypass-guard.py b/scripts/security/telegram-notification-egress-no-new-bypass-guard.py index 0a45301c4..bc2ebc4bc 100644 --- a/scripts/security/telegram-notification-egress-no-new-bypass-guard.py +++ b/scripts/security/telegram-notification-egress-no-new-bypass-guard.py @@ -24,12 +24,14 @@ TAIPEI = timezone(timedelta(hours=8)) SOURCE_SNAPSHOT = Path("docs/security/telegram-notification-egress-inventory.snapshot.json") SCAN_ROOTS = ( + Path("agent99-control-plane.ps1"), Path(".gitea/workflows"), Path("scripts/ops"), Path("scripts/ci"), + Path("scripts/reboot-recovery"), Path("apps/api/src"), ) -SCAN_SUFFIXES = {".py", ".sh", ".js", ".yml", ".yaml"} +SCAN_SUFFIXES = {".py", ".ps1", ".sh", ".js", ".yml", ".yaml"} GUARDED_BOT_METHODS = ( "sendMessage", "sendDocument", @@ -77,6 +79,10 @@ def iter_scannable_files(root: Path) -> list[Path]: absolute_root = root / scan_root if not absolute_root.exists(): continue + if absolute_root.is_file(): + if absolute_root.suffix in SCAN_SUFFIXES: + files.append(absolute_root) + continue for path in absolute_root.rglob("*"): if path.is_file() and path.suffix in SCAN_SUFFIXES: files.append(path)