From 475309915541da2f5f42c291ab5004d198d85d59 Mon Sep 17 00:00:00 2001
From: OG T <ogt@WOOOMacMiniM4.local>
Date: Wed, 6 May 2026 13:38:33 +0800
Subject: [PATCH] fix(alertmanager): send direct alerts to sre group

---
 docs/LOGBOOK.md                               | 42 +++++++++++++++++++
 ...AI-AUTONOMOUS-FLYWHEEL-INTEGRATION-PLAN.md |  8 ++--
 ops/alertmanager/alertmanager.yml             |  8 ++--
 3 files changed, 50 insertions(+), 8 deletions(-)

diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md
index f4f03c12c..c6dc74ab0 100644
--- a/docs/LOGBOOK.md
+++ b/docs/LOGBOOK.md
@@ -3581,3 +3581,45 @@ DockerContainerCpuSustainedHigh -> PB-20260505-F4197B
 
 - 部署後觀察 24h：`approval_records.matched_playbook_id IS NOT NULL` 必須從 0 開始增加。
 - 若 guard 擋下大量 LLM 動作，下一步不是放寬 guard，而是讓 PreDecision/MCP 先收 evidence，再產生 domain-correct SSH/K8s action。
+
+---
+
+## 2026-05-06（台北）— Alertmanager 旁路改送 SRE 群組 + Sentry Snuba 修復
+
+**觸發**：Telegram 收到 `🚨 [Alertmanager Fallback] DockerContainerRestartSpike`，且訊息發到 OpenClaw 私訊/機器人對話；同一時間 AWOOOI 心跳正常，表示 fallback 旁路不是「API 離線才觸發」，而是 Alertmanager critical route 的 direct Telegram 旁路。
+
+### 已修正
+
+| 範圍 | 結果 |
+|------|------|
+| Alertmanager live config | `/home/wooo/monitoring/alertmanager.yml` 的 `telegram-direct.chat_id` 已從 `OPENCLAW_TG_CHAT_ID` 切到 `SRE_GROUP_CHAT_ID`，並以 HUP reload |
+| Alertmanager repo template | `ops/alertmanager/alertmanager.yml` 改用 `SRE_GROUP_CHAT_ID_PLACEHOLDER`，避免後續部署回退到私訊 |
+| Sentry / Snuba schema | 110 上 `/opt/sentry` 執行 `docker compose run --rm snuba-api bootstrap --force`，補齊 ClickHouse Snuba tables |
+| Kafka offset | `ingest-consumer` 的 `ingest-events:0` 與 `generic-metrics-consumer` 的 `ingest-performance-metrics:0` reset 到 latest，修正 `OffsetOutOfRange` |
+
+### 驗證
+
+```text
+docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml
+# SUCCESS
+
+Alertmanager telegram-direct chat_id
+# group/supergroup, suffix=74679
+
+ClickHouse tables
+# system.tables default count = 83
+# default.errors_local = 1
+# default.transactions_local = 1
+# default.metrics_raw_v2_local = 1
+
+Sentry consumers after reset
+# events-consumer healthy
+# generic-metrics-consumer healthy
+# snuba-errors/metrics/transactions consumers healthy
+# recent 45s logs: no OffsetOutOfRange / UNKNOWN_TABLE / ERROR markers
+```
+
+### 注意
+
+- `DockerContainerRestartSpike` 使用 15 分鐘窗口，已發生的 restart spike 會在 Prometheus 窗口過去後退火；修復完成後短時間內 `ALERTS{alertname="DockerContainerRestartSpike"}` 仍可能暫時為 firing。
+- 目前 Alertmanager 的 `telegram-direct` 仍是 critical 旁路，不是真正「webhook failure conditional fallback」。後續應改成只由 AWOOOI API health / AlertChainDown 類告警觸發，避免 critical 告警在主鏈正常時雙路徑通知。
diff --git a/docs/awooop/AWOOOI-AWOOOP-AI-AUTONOMOUS-FLYWHEEL-INTEGRATION-PLAN.md b/docs/awooop/AWOOOI-AWOOOP-AI-AUTONOMOUS-FLYWHEEL-INTEGRATION-PLAN.md
index 4a49b4eee..801956437 100644
--- a/docs/awooop/AWOOOI-AWOOOP-AI-AUTONOMOUS-FLYWHEEL-INTEGRATION-PLAN.md
+++ b/docs/awooop/AWOOOI-AWOOOP-AI-AUTONOMOUS-FLYWHEEL-INTEGRATION-PLAN.md
@@ -293,7 +293,7 @@ For AI routing releases, also verify:
 
 ## 11. Immediate Next Items
 
-1. Finish live deployment verification for `682c0b99` AwoooP route fix.
-2. Create Wave 1 implementation checklist from this document.
-3. Start with MCP Gateway bypass and platform approval authentication, because both directly affect safety and auditability.
-4. Keep GCP-A/GCP-B/111 Ollama routing verification in every alert-path release until EffectivePolicy becomes authoritative.
+1. Convert Alertmanager `telegram-direct` from "all critical bypass" to a true alert-chain/API-health-only emergency route; the 2026-05-06 hotfix moved it to `SRE_GROUP_CHAT_ID`, but it still duplicates critical alerts by design.
+2. Continue Wave 1 with MCP Gateway bypass and MCP audit completeness, because production callers can still route around the gateway.
+3. Keep GCP-A/GCP-B/111 Ollama routing verification in every alert-path release until EffectivePolicy becomes authoritative.
+4. Add a Sentry/Snuba post-reboot health gate: ClickHouse table existence, Snuba migration status, and Kafka consumer offsets must be part of cold-start validation.
diff --git a/ops/alertmanager/alertmanager.yml b/ops/alertmanager/alertmanager.yml
index 03e30cd1a..e851e31ab 100644
--- a/ops/alertmanager/alertmanager.yml
+++ b/ops/alertmanager/alertmanager.yml
@@ -4,8 +4,8 @@
 # 修正後: http://192.168.0.121:32334/api/v1/webhooks/alertmanager (AWOOOI API，複數，正確)
 # 根據 feedback_alertmanager_awoooi_flow.md 鐵律
 # 2026-04-09 Claude Sonnet 4.6 Asia/Taipei: 新增 Telegram Fallback (ADR-035)
-# 架構: awoooi-webhook (主路徑) + telegram-direct (fallback，獨立路由)
-# 當 AWOOOI API 無法回應時，critical 告警直接送 Telegram Bot API
+# 架構: awoooi-webhook (主路徑) + telegram-direct (critical 旁路，獨立路由)
+# critical 旁路必須送 AwoooI SRE 戰情室；OPENCLAW_TG_CHAT_ID 只允許作缺值時的 fail-soft fallback
 # ⚠️  bot_token/chat_id 部署時由 secrets 替換，此檔為模板
 #
 # 2026-04-29 ogt + Claude Opus 4.7: P1-4 新版語法升級 + 因果抑制擴展
@@ -33,7 +33,7 @@ route:
         - severity="critical"
       receiver: 'awoooi-webhook'
       group_wait: 10s
-      # continue:true 讓 critical 同時送 telegram-direct (fallback)
+      # continue:true 讓 critical 同時送 telegram-direct 旁路（SRE 群組）
       continue: true
     - matchers:
         - severity="critical"
@@ -66,7 +66,7 @@ receivers:
   - name: 'telegram-direct'
     telegram_configs:
       - bot_token: 'TELEGRAM_BOT_TOKEN_PLACEHOLDER'
-        chat_id: TELEGRAM_CHAT_ID_PLACEHOLDER
+        chat_id: SRE_GROUP_CHAT_ID_PLACEHOLDER
         parse_mode: 'HTML'
         message: |
           🚨 <b>[Alertmanager Fallback]</b>