diff --git a/apps/api/src/services/decision_manager.py b/apps/api/src/services/decision_manager.py index 4fe41353d..c2804f221 100644 --- a/apps/api/src/services/decision_manager.py +++ b/apps/api/src/services/decision_manager.py @@ -102,8 +102,9 @@ async def _push_decision_to_telegram( action = proposal_data.get("action", proposal_data.get("kubectl_command", "")) description = proposal_data.get("description", "") reasoning = proposal_data.get("reasoning", "") - confidence = proposal_data.get("confidence", 0.75) + confidence = proposal_data.get("confidence", 0.0) # 🔴 預設 0.0 表示未經 AI 分析 source = proposal_data.get("source", "unknown") + ai_provider = proposal_data.get("provider", "") # 2026-03-29 ogt: AI 模型來源 # 建立 approval_id (使用 incident_id 作為追蹤) # 2026-03-27 ogt: 修復 INC-INC-INC- 重複前綴 bug @@ -119,6 +120,7 @@ async def _push_decision_to_telegram( primary_responsibility="INFRA", confidence=confidence, namespace=incident.signals[0].labels.get("namespace", "default") if incident.signals else "default", + ai_provider=ai_provider, # 2026-03-29 ogt: 顯示 AI 模型來源 ) # 🔴 發送成功後設置去重 key (TTL 10 分鐘) @@ -222,16 +224,19 @@ def expert_analyze(incident: Incident) -> dict[str, Any]: rule = EXPERT_RULES[matched_rule] + # 2026-03-29 ogt: Expert System 不應該假裝有高信心分數 + # 設為 0.0 強制標記為規則匹配，而非 AI 仲裁 return { "source": "expert_system", "action": rule["action"].format(target=target), "description": rule["description"], "risk_level": rule["risk_level"], - "reasoning": rule["reasoning"], - "confidence": 0.75, # Expert System 固定信心分數 + "reasoning": f"[規則匹配] {rule['reasoning']}", # 明確標示來源 + "confidence": 0.0, # 🔴 規則匹配不是 AI 仲裁，信心度設 0 "kubectl_command": rule["action"].format(target=target), "matched_rule": matched_rule, "from_cache": False, + "is_rule_based": True, # 新增標記 } diff --git a/apps/api/src/services/openclaw.py b/apps/api/src/services/openclaw.py index 7b33be2e7..410d92aaa 100644 --- a/apps/api/src/services/openclaw.py +++ b/apps/api/src/services/openclaw.py @@ -520,10 +520,11 @@ class OpenClawService: ) # 根據告警類型生成專業 RCA + 仲裁 + # 🔴 2026-03-29 ogt: Mock 響應必須標示為規則匹配，不是 AI 仲裁 if "oom" in message.lower() or "memory" in alert_type.lower(): mock_response = { "action_title": f"刪除異常 Pod {target} (OOMKilled)", - "description": f"🤖 AI 仲裁: {target} 發生 OOMKilled，根因為 JVM Heap 配置與 K8s memory limit 不匹配或存在記憶體洩漏。{signoz_summary}", + "description": f"⚙️ 規則匹配: {target} 發生 OOMKilled，根因為 JVM Heap 配置與 K8s memory limit 不匹配或存在記憶體洩漏。{signoz_summary}", "suggested_action": "DELETE_POD", "kubectl_command": f"kubectl delete pod {target} -n {namespace}", "target_resource": target, @@ -550,9 +551,9 @@ class OpenClawService: "kubectl_or_config": f"kubectl autoscale deployment {target.rsplit('-', 2)[0]} --memory-percent=80 --min=2 --max=5 -n {namespace}" } ], - "reasoning": f"🤖 Pod OOMKilled 後 ReplicaSet 將自動重建，但需同步修正資源配置防止復發。{signoz_correlation}", + "reasoning": f"⚙️ Pod OOMKilled 後 ReplicaSet 將自動重建，但需同步修正資源配置防止復發。{signoz_correlation}", "deviation_analysis": f"Memory 使用率 {metrics.get('memory_percent', 99)}%，超出基準線 60% 達 +6.5σ", - "confidence": 0.88, + "confidence": 0.0, # 🔴 規則匹配，非 AI 仲裁 "affected_services": [target, "api-gateway"], "signoz_correlation": signoz_correlation, } @@ -564,7 +565,7 @@ class OpenClawService: mock_response = { "action_title": f"擴展 {target} 副本數 + 啟用 HPA", - "description": f"🤖 AI 仲裁: {target} CPU 使用率過高，根因為流量突增或計算密集任務未配置自動擴展。{rps_context}", + "description": f"⚙️ 規則匹配: {target} CPU 使用率過高，根因為流量突增或計算密集任務未配置自動擴展。{rps_context}", "suggested_action": "SCALE_DEPLOYMENT", "kubectl_command": tuning["command"], "target_resource": target, @@ -591,16 +592,16 @@ class OpenClawService: "kubectl_or_config": f"kubectl set resources deployment/{target} --requests=cpu=500m --limits=cpu=2000m -n {namespace}" } ], - "reasoning": f"🤖 水平擴展可即時分散負載，同時建議配置 HPA 防止復發。{signoz_correlation}", + "reasoning": f"[規則匹配] 水平擴展可即時分散負載，同時建議配置 HPA 防止復發。{signoz_correlation}", "deviation_analysis": f"CPU 使用率 {metrics.get('cpu_percent', 95)}%，超出基準線 50% 達 +4.5σ", - "confidence": 0.92, + "confidence": 0.0, # 🔴 規則匹配不是 AI 仲裁，信心度設 0 "affected_services": [target], "signoz_correlation": signoz_correlation, } elif "http" in alert_type.lower() or "5xx" in message.lower() or "502" in message.lower(): mock_response = { "action_title": f"重啟 {target} + 檢查上游服務", - "description": f"🤖 AI 仲裁: {target} 產生 HTTP 5xx 錯誤，可能為應用程式例外或上游服務不可達。{signoz_summary}", + "description": f"⚙️ 規則匹配: {target} 產生 HTTP 5xx 錯誤，可能為應用程式例外或上游服務不可達。{signoz_summary}", "suggested_action": "RESTART_DEPLOYMENT", "kubectl_command": f"kubectl rollout restart deployment/{target} -n {namespace}", "target_resource": target, @@ -627,9 +628,9 @@ class OpenClawService: "kubectl_or_config": "# 檢查 Redis 連線池配置，建議 maxTotal=50" } ], - "reasoning": f"🤖 HTTP 錯誤需協同排查，先重啟恢復服務同時通知相關團隊。{signoz_correlation}", + "reasoning": f"[規則匹配] HTTP 錯誤需協同排查，先重啟恢復服務同時通知相關團隊。{signoz_correlation}", "deviation_analysis": "錯誤率 5%，超出基準線 0.1% 達 +50σ", - "confidence": 0.65, + "confidence": 0.0, # 🔴 規則匹配不是 AI 仲裁，信心度設 0 "affected_services": [target, "nginx-ingress", "upstream-api"], "signoz_correlation": signoz_correlation, } @@ -637,7 +638,7 @@ class OpenClawService: # 通用異常處理 mock_response = { "action_title": f"重新啟動 {target} 服務", - "description": f"🤖 AI 仲裁: {target} 發生異常: {message[:80]}。需進一步診斷確認根因。{signoz_summary}", + "description": f"⚙️ 規則匹配: {target} 發生異常: {message[:80]}。需進一步診斷確認根因。{signoz_summary}", "suggested_action": "RESTART_DEPLOYMENT", "kubectl_command": f"kubectl rollout restart deployment/{target} -n {namespace}", "target_resource": target, @@ -659,9 +660,9 @@ class OpenClawService: "kubectl_or_config": tuning["command"], } ], - "reasoning": f"🤖 根據告警 {alert_type} 先重啟恢復服務，同時安排深入診斷。{signoz_correlation}", + "reasoning": f"[規則匹配] 根據告警 {alert_type} 先重啟恢復服務，同時安排深入診斷。{signoz_correlation}", "deviation_analysis": "監控指標顯示異常偏離基準線", - "confidence": 0.70, + "confidence": 0.0, # 🔴 規則匹配不是 AI 仲裁，信心度設 0 "affected_services": [target], "signoz_correlation": signoz_correlation, } diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 3ecab1b83..596f8fd83 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -142,6 +142,10 @@ class TelegramMessage: # 2026-03-29 ogt: AI Token/Cost 追蹤 ai_tokens: int = 0 # LLM Token 使用量 ai_cost: float = 0.0 # LLM 成本 (USD) + # 2026-03-29 ogt: ADR-037 異常頻率統計 + anomaly_frequency: dict | None = None # AnomalyCounter 統計 + # 2026-03-29 ogt: AI Provider 來源顯示 + ai_provider: str = "" # ollama/gemini/claude/expert_system/mock def format(self) -> str: """ @@ -198,6 +202,43 @@ class TelegramMessage: if self.ai_tokens > 0 or self.ai_cost > 0: ai_cost_display = f"💰 Tokens: {self.ai_tokens:,} / ${self.ai_cost:.4f}\n" + # 2026-03-29 ogt: ADR-037 異常頻率顯示 + frequency_block = "" + if self.anomaly_frequency and self.anomaly_frequency.get("count_24h", 0) > 1: + freq = self.anomaly_frequency + escalation_emoji = { + None: "", + "REPEAT": "⚠️", + "ESCALATE": "🔴", + "PERMANENT_FIX": "🚨", + }.get(freq.get("escalation_level"), "") + + frequency_block = ( + f"━━━━━━━━━━━━━━━━━━━\n" + f"📊 頻率統計 {escalation_emoji}\n" + f"├ 1h: {freq.get('count_1h', 0)} 次\n" + f"├ 24h: {freq.get('count_24h', 0)} 次\n" + f"└ 修復: {freq.get('auto_repair_count', 0)} 次\n" + ) + if freq.get("escalation_level"): + frequency_block += f"🔺 升級: {freq['escalation_level']}\n" + + # 2026-03-29 ogt: 根據 confidence + ai_provider 動態顯示來源 + # confidence > 0 = 真實 AI 分析, confidence == 0 = 規則匹配/降級 + if self.confidence > 0 and self.ai_provider: + # 顯示具體 AI 模型 + provider_names = { + "ollama": "Ollama", + "gemini": "Gemini", + "claude": "Claude", + } + provider_display = provider_names.get(self.ai_provider.lower(), self.ai_provider.upper()) + source_label = f"🤖 {provider_display} 仲裁" + elif self.confidence > 0: + source_label = "🤖 AI 仲裁判定" + else: + source_label = "⚙️ 規則匹配" + # 組裝訊息 message = ( f"═══════════════════════════\n" @@ -206,11 +247,12 @@ class TelegramMessage: f"📋 {html.escape(incident_id)}\n" f"🎯 資源: {safe_resource}\n" f"━━━━━━━━━━━━━━━━━━━\n" - f"🤖 AI 仲裁判定\n" + f"{source_label}\n" f"👥 責任: {resp_display}\n" f"📊 信心: {conf_emoji} {confidence_pct}%\n" f"{ai_cost_display}" f"💡 原因: {safe_root_cause}\n" + f"{frequency_block}" f"{signoz_block}" f"━━━━━━━━━━━━━━━━━━━\n" f"🔧 建議: {safe_action}\n" @@ -437,11 +479,16 @@ class TelegramGateway: # 2026-03-29 ogt: AI Token/Cost 追蹤 ai_tokens: int = 0, ai_cost: float = 0.0, + # 2026-03-29 ogt: ADR-037 異常頻率統計 + anomaly_frequency: dict | None = None, + # 2026-03-29 ogt: AI Provider 來源顯示 + ai_provider: str = "", ) -> dict: """ 推送待簽核卡片到 Telegram (v7.0 含 SignOz 整合) SOUL.md 4.1 + AI 仲裁 + SignOz 訊息格式 + Phase 21 (ADR-037): 含異常頻率統計 Args: approval_id: 簽核單 ID @@ -456,6 +503,7 @@ class TelegramGateway: signoz_*: SignOz Gold Metrics signoz_trace_url: 動態時間參數的 Trace URL auto_tuning_command: kubectl 調優指令 + anomaly_frequency: 異常頻率統計 (ADR-037) Returns: dict: Telegram API 回應 @@ -475,7 +523,7 @@ class TelegramGateway: trace_url=signoz_trace_url, ) - # 建立訊息結構 (含 AI 仲裁 + SignOz + Token/Cost) + # 建立訊息結構 (含 AI 仲裁 + SignOz + Token/Cost + 頻率統計) message = TelegramMessage( status_emoji=emoji, risk_level=risk_level.upper(), @@ -493,6 +541,10 @@ class TelegramGateway: # 2026-03-29 ogt: AI Token/Cost 追蹤 ai_tokens=ai_tokens, ai_cost=ai_cost, + # 2026-03-29 ogt: ADR-037 異常頻率統計 + anomaly_frequency=anomaly_frequency, + # 2026-03-29 ogt: AI Provider 來源顯示 + ai_provider=ai_provider, ) # 格式化訊息 diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index b8bd0bf85..4953a9ab7 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -5,28 +5,66 @@ --- -## 📍 當前狀態 (2026-03-29 11:25 台北) +## 📍 當前狀態 (2026-03-29 14:10 台北) | 項目 | 狀態 | |------|------| -| **當前 Phase** | ✅ **Phase 21 Wave A-D 全部完成** (ADR-037 監控增強) | +| **當前 Phase** | ✅ **Phase 21 Wave A-D 全部完成** + 📋 **Phase 22 戰略規劃完成** | | **Day** | Day 12 | | **K3s 版本** | v1.34.5+k3s1 (mon + mon1) | | **叢集健康** | ✅ **所有 Pod 正常運行** | | **K3s 優化** | ✅ **全部完成 + P1-P3 + PSS** | | **K-MON** | ✅ **監控整合** (VIP/Velero/SignOz/Sentry 告警) | -| **K3 HPA** | ✅ **API/Web 2-6 自動擴展** (P3 maxReplicas 調升) | +| **K3 HPA** | ✅ **API/Web 2-6 自動擴展（Worker HPA 待 Wave 1 後部署）** | | **K4 Kured** | ✅ **自動重啟 (02:00-04:00 維護窗口)** | | **K4 Descheduler** | ✅ **負載均衡 (每 2 小時, threshold 30%)** | -| **K4.3 PSS** | ✅ **Pod Security Standards (6 Namespace labels)** 🆕 | +| **K4.3 PSS** | ✅ **Pod Security Standards (6 Namespace labels)** | | **K-HA** | ✅ **雙 Control-Plane (120+121) + PostgreSQL Datastore** | | **VIP** | ✅ **192.168.0.125 (keepalived + CI/CD 整合)** | | **kube-state-metrics** | ✅ **v2.10.1 @ :30888 + NPD 告警整合** | -| **Grafana Dashboard** | ✅ **K3s Cluster Overview (9 panels)** 🆕 | +| **Grafana Dashboard** | ✅ **K3s Cluster Overview (9 panels)** | | **ArgoCD** | ✅ **ApplicationSet CRD 修復** | | **告警狀態** | ✅ **0 個告警觸發** | -| **首席架構師審查** | ✅ **Wave A-D: 194/200 (97%) OUTSTANDING** 🆕🆕🆕 | +| **首席架構師審查** | ✅ **Wave A-D: 194/200 (97%) OUTSTANDING** | | **模組化合規** | ✅ **100% 通過** | +| **⚠️ Wave 1 待執行** | 🔴 **8 個 P0 安全網代碼修復（XCLAIM + Circuit Breaker + Semaphore 等）** | + +## 📋 Phase 22: 全維度盤點暨戰略規劃 (2026-03-29 完成) + +### 完成項目 + +| 類型 | 文件 | 說明 | +|------|------|------| +| 📊 稽核報告 | `docs/AWOOOI_COMPREHENSIVE_AUDIT_2026Q1.md` | 全維度盤點 10 章，87/100 | +| 🏗️ 設計提案 | `docs/proposals/MONITORING_ARCHITECTURE_DEEP_DIVE.md` | 監控三義分離 | +| 🏗️ 設計提案 | `docs/proposals/INTEGRATION_ARCHITECTURE_MASTER.md` | 整合架構主藍圖 | +| ⚔️ 風險矩陣 | `docs/proposals/ARCHITECTURAL_RISK_WAR_GAME.md` | 6+4+代碼確認深度沙盤（第 1-6 章）| +| 📋 最終排程 | `docs/proposals/MASTER_EXECUTION_SCHEDULE.md` | 最終工作排程（供統帥審核）🆕 | +| 📘 RunBook | `docs/runbooks/RUNBOOK-PHASE-D-SENTRY-COMMENT.md` | Sentry Comment + 補丁 | +| 📘 RunBook | `docs/runbooks/RUNBOOK-PHASE-E-SIGNOZ-WEBHOOK.md` | SignOz Webhook 部署 | +| 📘 RunBook | `docs/runbooks/RUNBOOK-WORKER-HPA.md` | HPA + XCLAIM 前置 + terminationGracePeriodSeconds | +| 📘 RunBook | `docs/runbooks/RUNBOOK-E2E-CI-SCHEDULE.md` | E2E CI + Docker Baseline 強制規格 | +| 📘 RunBook | `docs/runbooks/RUNBOOK-FRONTEND-UIUX-SOVEREIGNTY.md` | 前端主權 + release/v1.x 前置 | +| 📐 ADR | `docs/adr/ADR-038-openclaw-concurrency-governance.md` | OpenClaw Semaphore + Circuit Breaker 🆕 | +| 📐 ADR | `docs/adr/ADR-039-global-autorepair-governance.md` | 全域修復熔斷 + StatefulSet 黑名單 🆕 | + +### 確認待執行的 P0 安全網（Wave 0 + Wave 1） + +**Wave 0（統帥手動，今天）**： +- `0.1` Redis AOF + eviction policy 確認（5 min） +- `0.2` Alertmanager 備援 Telegram 路徑確認（10 min） +- `0.3` release/v1.x 穩定分支建立（5 min）+ GitHub Protected Branch +- `0.4` SENTRY_AUTH_TOKEN 存在確認（3 min） + +**Wave 1（代碼 PR，原子性，2026-03-30~04-01）**： +- `core/circuit_breaker.py`（ADR-038：Semaphore + Circuit Breaker） +- `services/global_repair_cooldown.py`（ADR-039：全域熔斷） +- `anomaly_counter.py`：Redis 故障 Graceful Degrade +- `signal_worker.py`：XCLAIM + Active Sweeper + stop() 75s +- `auto_repair_service.py`：StatefulSet 黑名單 + 全域熔斷整合 +- `sentry_webhook.py` + `signoz_webhook.py`：雙層保護整合 +- `incident_service.py`：Global Incident Debounce +- `08-deployment-worker.yaml`：terminationGracePeriodSeconds: 90 ---