feat(telegram): enforce canonical product routing

apps/api/src/services/decision_manager.py

@@ -195,19 +195,57 @@ async def _send_log_summary(incident: "Incident") -> None:
         if incident.signals:
             namespace = incident.signals[0].labels.get("namespace", "awoooi-prod")
 
+        from src.services.telegram_gateway import (
+            _telegram_send_delivery_succeeded,
+            get_telegram_gateway,
+        )
+
+        tg = get_telegram_gateway()
+        if not tg.canonical_destination_chat_id(
+            product_id="awoooi",
+            signal_family="product_raw_monitoring",
+            severity="P3",
+        ):
+            logger.warning(
+                "log_summary_no_send",
+                target=target,
+                incident_id=incident.incident_id,
+                reason="canonical_route_unavailable",
+            )
+            return
+
         from src.services.log_summary_service import get_log_summary_service
         svc = get_log_summary_service()
         summary = await svc.summarize_with_soft_timeout(pod_name=target, namespace=namespace)
         if not summary:
             return
 
-        from src.services.telegram_gateway import get_telegram_gateway
-        tg = get_telegram_gateway()
-        await tg.send_as_nemotron(
-            f"📋 <b>Log 異常摘要</b> — <code>{target}</code>\n{summary}"
+        result = await tg.send_as_nemotron(
+            f"📋 <b>Log 異常摘要</b> — <code>{target}</code>\n{summary}",
+            product_id="awoooi",
+            signal_family="product_raw_monitoring",
+            severity="P3",
         )
-        import structlog as _sl
-        _sl.get_logger(__name__).info("log_summary_sent", target=target, incident_id=incident.incident_id)
+        if _telegram_send_delivery_succeeded(result):
+            logger.info(
+                "log_summary_sent",
+                target=target,
+                incident_id=incident.incident_id,
+            )
+        else:
+            logger.warning(
+                "log_summary_no_send",
+                target=target,
+                incident_id=incident.incident_id,
+                delivery_status=(
+                    result.get(
+                        "_awooop_delivery_status",
+                        "missing_provider_ack",
+                    )
+                    if isinstance(result, dict)
+                    else "missing_provider_ack"
+                ),
+            )
     except Exception as e:
         import structlog as _sl
         _sl.get_logger(__name__).warning("log_summary_failed", error=str(e))
@@ -220,7 +258,7 @@ async def _send_log_summary(incident: "Incident") -> None:
 async def _push_decision_to_telegram(
     incident: Incident,
     proposal_data: dict[str, Any],
-) -> None:
+) -> bool:
     """
     決策就緒時推送到 Telegram
 
@@ -231,11 +269,12 @@ async def _push_decision_to_telegram(
         # 延遲導入避免循環依賴
         from src.core.redis_client import get_redis
         from src.services.telegram_gateway import (
+            NotificationType,
+            _telegram_send_delivery_succeeded,
             classify_notification,
             get_telegram_gateway,
-            NotificationType,
-            _smart_truncate as _smt,
         )
+        from src.services.telegram_gateway import _smart_truncate as _smt
 
         # 🔴 去重檢查：同一 fingerprint (alertname+target) 24h 內只發一次 decision card
         # 2026-05-02 Claude Opus 4.7 + 統帥 ogt：原本 dedup_key 用 incident.incident_id，
@@ -256,7 +295,7 @@ async def _push_decision_to_telegram(
                 incident_id=incident.incident_id,
                 fingerprint=f"{_alertname_fp}:{_target_fp}",
             )
-            return
+            return False
 
         # 2026-04-09 Claude Code: resolved Incident 不重送 Telegram
         # 場景: dedup TTL 過期後，已 resolve 的 Incident 仍被重新推送
@@ -266,7 +305,7 @@ async def _push_decision_to_telegram(
                 reason="Incident already resolved",
                 incident_id=incident.incident_id,
             )
-            return
+            return False
 
         # 🔴 靜默檢查：此資源是否被靜默 (2026-03-27 P1 優化)
         target = incident.affected_services[0] if incident.affected_services else "unknown"
@@ -278,7 +317,7 @@ async def _push_decision_to_telegram(
                 incident_id=incident.incident_id,
                 resource=target,
             )
-            return
+            return False
 
         # 檢查是否有設定 Bot Token
         if not settings.OPENCLAW_TG_BOT_TOKEN:
@@ -287,7 +326,7 @@ async def _push_decision_to_telegram(
                 reason="Bot token not configured",
                 incident_id=incident.incident_id,
             )
-            return
+            return False
 
         gateway = get_telegram_gateway()
 
@@ -411,7 +450,7 @@ async def _push_decision_to_telegram(
                 action_text=_action_text,
                 reason="Agent Debate 無有效輸出 (description=待分析, action 屬於失敗狀態集合)",
             )
-            return
+            return False
 
         # 建立 approval_id (使用 incident_id 作為追蹤)
         # 2026-03-27 ogt: 修復 INC-INC-INC- 重複前綴 bug
@@ -581,12 +620,37 @@ async def _push_decision_to_telegram(
                 automation_state=proposal_data.get("automation_state", ""),
             )
 
+        if not _telegram_send_delivery_succeeded(tg_result):
+            logger.warning(
+                "telegram_decision_no_send",
+                incident_id=incident.incident_id,
+                source=source,
+                risk_level=risk_level,
+                delivery_status=(
+                    tg_result.get("_awooop_delivery_status")
+                    if isinstance(tg_result, dict)
+                    else "missing_provider_ack"
+                ),
+            )
+            return False
+
         # 2026-04-09 Claude Sonnet 4.6: 存 message_id → 後續狀態更新在原訊息延續
         # 同時寫 Redis (快速查詢) 和 DB (持久化，不受 TTL 限制)
         tg_message_id = tg_result.get("result", {}).get("message_id") if isinstance(tg_result, dict) else None
         tg_chat_id = tg_result.get("result", {}).get("chat", {}).get("id") if isinstance(tg_result, dict) else None
         if tg_message_id:
-            await redis.setex(f"tg_msg:{incident.incident_id}", 86400, str(tg_message_id))
+            try:
+                await redis.setex(
+                    f"tg_msg:{incident.incident_id}",
+                    86400,
+                    str(tg_message_id),
+                )
+            except Exception as _redis_message_error:
+                logger.warning(
+                    "telegram_message_id_redis_save_failed",
+                    incident_id=incident.incident_id,
+                    error=str(_redis_message_error),
+                )
             # 持久化到 DB
             try:
                 from src.services.approval_db import get_approval_service as _get_approval_svc
@@ -599,6 +663,21 @@ async def _push_decision_to_telegram(
             except Exception as _e:
                 logger.warning("telegram_message_id_db_save_failed", incident_id=incident.incident_id, error=str(_e))
 
+        # Provider ack is the delivery truth.  Persist dedup only after that
+        # receipt, while keeping a dedup write failure separate from delivery.
+        # 2026-05-02 Claude Opus 4.7 + 統帥 ogt：TTL 從 600s 拉到 86400s；
+        # incident_analysis_sweeper 每 1h 與 Alertmanager 每 4h 都可能重觸，
+        # 24h TTL 用來避免同症狀重複洗版。
+        try:
+            await redis.setex(dedup_key, 86400, "1")
+        except Exception as _dedup_error:
+            logger.warning(
+                "telegram_decision_dedup_write_failed",
+                incident_id=incident.incident_id,
+                dedup_key=dedup_key,
+                error=str(_dedup_error),
+            )
+
         # Phase 31 (ADR-067 2026-04-10): Log 異常摘要 — NemoTron deepseek-r1:14b
         # 非同步執行，不阻塞主流程
         _fire_and_forget(_send_log_summary(incident))
@@ -610,22 +689,20 @@ async def _push_decision_to_telegram(
             _fire_and_forget(
                 gateway.send_as_nemotron(
                     f"🤔 <b>NemoClaw 第二意見</b> (信心={confidence:.2f})\n"
-                    f"<i>{_advisory_note}</i>"
+                    f"<i>{_advisory_note}</i>",
+                    product_id="awoooi",
+                    signal_family="product_raw_monitoring",
+                    severity="P3",
                 )
             )
 
-        # 🔴 發送成功後設置去重 key (TTL 24 小時)
-        # 2026-05-02 Claude Opus 4.7 + 統帥 ogt：TTL 從 600s 拉到 86400s。
-        # 原因：incident_analysis_sweeper 每 1h 重觸 decision、alertmanager 預設 4h repeat，
-        # 600s TTL 早就過期 → 重發；24h TTL 蓋住兩個週期，徹底治住「同症狀 24h 推 15 次」。
-        await redis.setex(dedup_key, 86400, "1")
-
         logger.info(
             "telegram_decision_pushed",
             incident_id=incident.incident_id,
             source=source,
             risk_level=risk_level,
         )
+        return True
 
     except Exception as e:
         # Telegram 失敗不影響主流程
@@ -634,6 +711,7 @@ async def _push_decision_to_telegram(
             incident_id=incident.incident_id,
             error=str(e),
         )
+        return False
 
 
 async def _nemoclaw_second_opinion(incident: "Incident", primary_result: dict) -> str | None:
@@ -831,7 +909,10 @@ async def _generate_playbook_draft_if_new(incident: "Incident") -> None:
                 f"📚 <b>KM 新條目</b> — <code>{entry.entry_id}</code>\n"
                 f"🏷️ 分類：<b>auto_generated</b>  狀態：DRAFT\n"
                 f"📋 告警：<code>{alertname}</code>  事件：<code>{incident.incident_id}</code>\n"
-                f"標題：{str(getattr(entry, 'title', alertname))[:60]}"
+                f"標題：{str(getattr(entry, 'title', alertname))[:60]}",
+                product_id="awoooi",
+                signal_family="product_raw_monitoring",
+                severity="P3",
             )
         except Exception as _tg_err:
             _sl.get_logger(__name__).debug("km_tg_notify_failed", error=str(_tg_err))
@@ -1110,7 +1191,10 @@ async def _push_auto_repair_result(
     2026-04-11 Claude Sonnet 4.6: +ADR-071-I 指標快照
     """
     try:
-        from src.services.telegram_gateway import get_telegram_gateway
+        from src.services.telegram_gateway import (
+            _telegram_send_delivery_succeeded,
+            get_telegram_gateway,
+        )
 
         gateway = get_telegram_gateway()
         target = incident.affected_services[0] if incident.affected_services else "unknown"
@@ -1217,9 +1301,57 @@ async def _push_auto_repair_result(
                 f"├─ 對象: <code>{target[:50]}</code>\n"
                 f"└─ {status_line}"
             )
-            await gateway.send_alert_notification(fallback_text)
+            if not gateway.canonical_destination_chat_id(
+                product_id="awoooi",
+                signal_family="incident_lifecycle",
+                severity="P1",
+            ):
+                logger.warning(
+                    "auto_repair_result_no_send",
+                    incident_id=inc_id,
+                    success=success,
+                    reason="canonical_route_unavailable",
+                )
+                return
 
-        logger.info("auto_repair_result_sent", incident_id=inc_id, success=success, appended=appended)
+            fallback_result = await gateway.send_alert_notification(
+                fallback_text,
+                product_id="awoooi",
+                signal_family="incident_lifecycle",
+                severity="P1",
+            )
+            if not _telegram_send_delivery_succeeded(fallback_result):
+                logger.warning(
+                    "auto_repair_result_no_send",
+                    incident_id=inc_id,
+                    success=success,
+                    delivery_status=(
+                        fallback_result.get(
+                            "_awooop_delivery_status",
+                            "missing_provider_ack",
+                        )
+                        if isinstance(fallback_result, dict)
+                        else "missing_provider_ack"
+                    ),
+                )
+                return
+
+            logger.info(
+                "auto_repair_result_sent",
+                incident_id=inc_id,
+                success=success,
+                appended=False,
+            )
+        else:
+            # append_incident_update currently exposes only a boolean request result,
+            # not the provider receipt.  Do not claim a Telegram send without the
+            # shared affirmative delivery verifier.
+            logger.info(
+                "auto_repair_result_update_requested",
+                incident_id=inc_id,
+                success=success,
+                appended=True,
+            )
 
     except Exception as e:
         logger.warning("auto_repair_result_push_failed", incident_id=incident.incident_id, error=str(e))
@@ -3506,32 +3638,45 @@ class DecisionManager:
         """
         import asyncio as _asyncio
         import json as _json
+
         from src.core.redis_client import get_redis
         from src.db.base import get_db_context
         from src.repositories.incident_repository import IncidentDBRepository
 
         redis = get_redis()
-        resent = 0
         # GAP-A4 後續修復：限制並發 5，避免壓爆 Ollama
         _sem = _asyncio.Semaphore(5)
 
-        async def _bounded_push(incident_obj, proposal_data_obj, _id, _token):
+        async def _bounded_push(incident_obj, proposal_data_obj, _id, _token) -> bool:
             async with _sem:
                 try:
-                    await _push_decision_to_telegram(incident_obj, proposal_data_obj)
-                    logger.info(
-                        "stale_ready_token_resent",
-                        incident_id=_id,
-                        token=_token,
+                    delivered = await _push_decision_to_telegram(
+                        incident_obj,
+                        proposal_data_obj,
                     )
+                    if delivered:
+                        logger.info(
+                            "stale_ready_token_resent",
+                            incident_id=_id,
+                            token=_token,
+                        )
+                    else:
+                        logger.warning(
+                            "stale_ready_token_no_send",
+                            incident_id=_id,
+                            token=_token,
+                        )
+                    return delivered
                 except Exception as _e:
                     logger.warning(
                         "stale_ready_token_resend_failed",
                         incident_id=_id,
                         error=str(_e),
                     )
-                # 每次完成後喘 200ms，給 Ollama embedding 恢復空間
-                await _asyncio.sleep(0.2)
+                    return False
+                finally:
+                    # 每次完成後喘 200ms，給 Ollama embedding 恢復空間
+                    await _asyncio.sleep(0.2)
 
         try:
             # 掃描所有 decision:* key
@@ -3575,7 +3720,10 @@ class DecisionManager:
                         _STALE_DAYS = 3
                         _created_at = getattr(incident, "created_at", None)
                         if _created_at:
-                            from datetime import datetime as _dt, timedelta as _td, timezone as _tz
+                            from datetime import datetime as _dt
+                            from datetime import timedelta as _td
+                            from datetime import timezone as _tz
+
                             _now = _dt.now(_tz.utc)
                             _cutoff = _now - _td(days=_STALE_DAYS)
                             # 確保 _created_at 有時區
@@ -3599,20 +3747,26 @@ class DecisionManager:
                             _bounded_push(incident, proposal_data, incident_id, data.get("token", ""))
                         )
                         tasks.append(_task)
-                        resent += 1
                     except Exception as _te:
                         logger.debug("stale_ready_token_scan_error", error=str(_te))
 
                 if cursor == 0:
                     break
 
-            # 不等所有 task 完成（fire-and-forget 語義保留），但 await 一下讓並發限制生效
+            resent = 0
             if tasks:
-                logger.info("stale_ready_tokens_throttled_dispatch",
-                            total=len(tasks), max_concurrent=5)
+                logger.info(
+                    "stale_ready_tokens_throttled_dispatch",
+                    total=len(tasks),
+                    max_concurrent=5,
+                    status="awaiting_delivery_receipts",
+                )
+                delivery_results = await _asyncio.gather(*tasks)
+                resent = sum(result is True for result in delivery_results)
 
         except Exception as e:
             logger.warning("resend_stale_ready_tokens_failed", error=str(e))
+            resent = 0
 
         logger.info("stale_ready_tokens_scan_done", resent=resent)
         return resent