From 9dfecc4d1b12db59fc26c5ff794397e81444aba8 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 6 May 2026 22:03:19 +0800 Subject: [PATCH] fix(telegram): separate ssh diagnosis from repair failures --- apps/api/src/services/decision_manager.py | 19 ++++++---- apps/api/src/services/telegram_gateway.py | 8 ++++ ...t_decision_manager_docker_prune_routing.py | 34 +++++++++++++++++ .../tests/test_telegram_message_templates.py | 38 ++++++++++++++++++- docs/LOGBOOK.md | 16 ++++++++ 5 files changed, 107 insertions(+), 8 deletions(-) diff --git a/apps/api/src/services/decision_manager.py b/apps/api/src/services/decision_manager.py index ebf27ef39..e98d5df2a 100644 --- a/apps/api/src/services/decision_manager.py +++ b/apps/api/src/services/decision_manager.py @@ -576,6 +576,7 @@ async def _push_decision_to_telegram( alert_category=_alert_category, notification_type=_notification_type, playbook_name=_playbook_name, + automation_state=proposal_data.get("automation_state", ""), ) # 2026-04-09 Claude Sonnet 4.6: 存 message_id → 後續狀態更新在原訊息延續 @@ -3519,6 +3520,8 @@ class DecisionManager: token.proposal_data["decision_state"] = DecisionState.READY.value token.proposal_data["auto_executed"] = False token.proposal_data["mcp_all_failed"] = True + if _tool == "ssh_diagnose": + token.proposal_data["automation_state"] = "diagnosis_failed_manual_required" await self._save_token(token) _fire_and_forget( _escalate_decision_auto_repair_unavailable( @@ -3528,14 +3531,15 @@ class DecisionManager: attempted_actions=f"decision_manager._ssh_execute -> {_tool}", ) ) - _fire_and_forget( - _push_auto_repair_result( - incident, - action, - success=False, - error=token.error, + if _tool != "ssh_diagnose": + _fire_and_forget( + _push_auto_repair_result( + incident, + action, + success=False, + error=token.error, + ) ) - ) _fire_and_forget(_push_decision_to_telegram(incident, token.proposal_data)) return @@ -3545,6 +3549,7 @@ class DecisionManager: token.proposal_data["auto_executed"] = False token.proposal_data["ssh_diagnosis_collected"] = True token.proposal_data["ssh_diagnosis_preview"] = output_preview + token.proposal_data["automation_state"] = "diagnosis_collected_manual_required" await self._save_token(token) _fire_and_forget( _escalate_decision_auto_repair_unavailable( diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 32500e15f..a1101a5d3 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -215,6 +215,7 @@ class TelegramMessage: # 2026-04-16 ogt + Claude Sonnet 4.6: 告警分類與修復鏈路顯示 (ADR-076) alert_category: str = "" # host/k8s/database/service/external_site/secops 等 playbook_name: str = "" # 匹配到的 Playbook 名稱（空字串=規則匹配） + automation_state: str = "" # diagnosis_collected_manual_required / diagnosis_failed_manual_required # ========================================================================== # Phase 22: Nemotron 協作欄位 (ADR-044) @@ -273,7 +274,12 @@ class TelegramMessage: mode = self._automation_mode() action = (self.suggested_action or "").upper() text = f"{self.root_cause} {self.suggested_action}".lower() + state = (self.automation_state or "").lower() + if state == "diagnosis_collected_manual_required": + return "🔎 AI 已完成只讀診斷，需人工判斷" + if state == "diagnosis_failed_manual_required": + return "🔴 AI 診斷工具失敗，需人工排查" if mode == "llm_timeout_manual_gate": return "🔴 AI 分析超時，需人工排查" if action in {"NO_ACTION", "待分析", ""} or "invalid_target" in text: @@ -1874,6 +1880,7 @@ class TelegramGateway: notification_type: str = "", # 2026-04-16 ogt + Claude Sonnet 4.6: 修復鏈路顯示 (ADR-076) playbook_name: str = "", + automation_state: str = "", ) -> dict: """ 推送待簽核卡片到 Telegram (v7.0 含 SignOz 整合) @@ -1950,6 +1957,7 @@ class TelegramGateway: # 2026-04-16 ogt + Claude Sonnet 4.6: 修復鏈路顯示 (ADR-076) alert_category=alert_category, playbook_name=playbook_name, + automation_state=automation_state, ) # 格式化訊息 — Phase 22: 如果 Nemotron 啟用，使用雙軌格式 diff --git a/apps/api/tests/test_decision_manager_docker_prune_routing.py b/apps/api/tests/test_decision_manager_docker_prune_routing.py index 2f0d2e6a7..749f813ff 100644 --- a/apps/api/tests/test_decision_manager_docker_prune_routing.py +++ b/apps/api/tests/test_decision_manager_docker_prune_routing.py @@ -134,3 +134,37 @@ class TestDockerPruneActionRouting: target="unknown", ) assert manager._ssh.execute.call_args.kwargs["tool_name"] == "ssh_diagnose" + + @pytest.mark.asyncio + async def test_failed_diagnose_does_not_emit_auto_repair_failed(self, manager, monkeypatch): + """ssh_diagnose 是只讀診斷，失敗時不可標成自動修復失敗。""" + incident = _fake_incident() + token = _fake_token() + auto_result_calls = [] + + manager._ssh.execute = AsyncMock( + return_value=MCPToolResult( + success=False, + execution_id="exec-fail", + error="diagnosis connection failed", + ) + ) + monkeypatch.setattr( + "src.services.decision_manager._push_auto_repair_result", + lambda *args, **kwargs: auto_result_calls.append((args, kwargs)), + ) + monkeypatch.setattr( + "src.services.decision_manager._escalate_decision_auto_repair_unavailable", + lambda *args, **kwargs: None, + ) + + await manager._ssh_execute( + incident=incident, + token=token, + action="ssh 192.168.0.110 'df -h && free -h'", + target="unknown", + ) + + assert manager._ssh.execute.call_args.kwargs["tool_name"] == "ssh_diagnose" + assert token.proposal_data["automation_state"] == "diagnosis_failed_manual_required" + assert auto_result_calls == [] diff --git a/apps/api/tests/test_telegram_message_templates.py b/apps/api/tests/test_telegram_message_templates.py index 26cabf6f9..9d3104d18 100644 --- a/apps/api/tests/test_telegram_message_templates.py +++ b/apps/api/tests/test_telegram_message_templates.py @@ -15,8 +15,8 @@ from src.services.telegram_gateway import ( RepairReportMessage, ResourceWarnMessage, SentryErrorMessage, - TelegramMessage, TelegramGateway, + TelegramMessage, ) @@ -84,6 +84,42 @@ class TestTelegramMessageFormat: assert "處置狀態" in result assert "AI 無可安全執行動作，需人工判斷" in result + def test_telegram_message_diagnosis_state_is_not_auto_repair(self): + """SSH 只讀診斷 lane 不得被顯示成自動修復。""" + msg = TelegramMessage( + status_emoji="⚠️", + risk_level="MEDIUM", + resource_name="node-110", + root_cause="SSH diagnosis collected", + suggested_action="ssh 192.168.0.110 'uptime'", + estimated_downtime="unknown", + approval_id="INC-20260506-DIAG", + automation_state="diagnosis_collected_manual_required", + ) + + result = msg.format() + + assert "AI 已完成只讀診斷，需人工判斷" in result + assert "AI 自動修復失敗" not in result + + def test_telegram_message_diagnosis_failure_state(self): + """SSH 診斷工具失敗必須標成診斷失敗，而不是修復失敗。""" + msg = TelegramMessage( + status_emoji="🚨", + risk_level="CRITICAL", + resource_name="node-110", + root_cause="SSH MCP execution failed", + suggested_action="ssh 192.168.0.110 'uptime'", + estimated_downtime="unknown", + approval_id="INC-20260506-DIAGFAIL", + automation_state="diagnosis_failed_manual_required", + ) + + result = msg.format() + + assert "AI 診斷工具失敗，需人工排查" in result + assert "AI 自動修復失敗" not in result + def test_telegram_message_with_token_cost(self): """測試含 Token/Cost 的訊息""" msg = TelegramMessage( diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 726f7e76a..ee4dddfa7 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,19 @@ +## 2026-05-06 | Telegram 將 SSH 診斷 lane 與自動修復 lane 分離 + +**背景**：戰情室截圖中 `ssh_diagnose` 這類只讀主機診斷失敗時，也會出現 `[AUTO] AI 自動修復失敗，已升級人工介入`。這會讓值班者誤以為系統已嘗試修復且修復失敗；實際上它只是「診斷工具失敗」或「診斷已完成但沒有安全修復動作」。 + +**本次修補**： +- `TelegramMessage` 新增 `automation_state`，讓第一屏「處置狀態」能顯示 `AI 已完成只讀診斷，需人工判斷` 或 `AI 診斷工具失敗，需人工排查`。 +- `decision_manager._ssh_execute()` 對 `ssh_diagnose` 成功/失敗分支寫入 `automation_state`。 +- `ssh_diagnose` 失敗不再呼叫 `_push_auto_repair_result(... success=False)`，避免把診斷失敗回覆成自動修復失敗。 +- 修復建議、人工審批與真正寫入型 SSH 操作仍維持原路徑。 + +**驗證**： +- `python -m py_compile apps/api/src/services/telegram_gateway.py apps/api/src/services/decision_manager.py apps/api/tests/test_telegram_message_templates.py apps/api/tests/test_decision_manager_docker_prune_routing.py` +- `pytest tests/test_telegram_message_templates.py tests/test_decision_manager_docker_prune_routing.py tests/test_ssh_provider_tools.py -q` → 31 passed。 +- `ruff check tests/test_telegram_message_templates.py tests/test_decision_manager_docker_prune_routing.py` → All checks passed。 +- 注意：`telegram_gateway.py` 全檔 ruff 仍會掃到既有 import order、bare except、單行 if 等歷史債；本輪未在 6000+ 行 gateway 巨檔做無關機械清理。 + ## 2026-05-06 | SSH MCP 連線參數硬化，修復 `%d format` 導致主機診斷全失敗 **背景**：SRE 戰情室與 production log 顯示 host-layer MCP 工具（`ssh_get_top_processes`、`ssh_get_swap_info`、`ssh_diagnose` 等）全數失敗，錯誤為 `%d format: a real number is required, not str`。這讓主機告警無法取得感官證據，後續 AI 只能降級，並在 Telegram 中重複出現「AI 自動修復失敗，已升級人工介入」。