From 770586dd85cddb8fff7f133d915a8c062b8fda33 Mon Sep 17 00:00:00 2001 From: OG T Date: Tue, 31 Mar 2026 12:10:52 +0800 Subject: [PATCH] =?UTF-8?q?feat(api):=20Phase=2018.3=20K8s=20Executor=20?= =?UTF-8?q?=E6=95=B4=E5=90=88=E8=87=AA=E5=8B=95=E4=BF=AE=E5=BE=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2026-03-31 Claude Code (統帥批准) 新增功能: - execute_auto_repair() 實際執行 K8s 操作 - restart_deployment: rollout restart - restart_pod: 刪除 Pod 觸發重建 - clear_cache: 安全清理 Redis 快取 安全機制: - _check_repair_cooldown(): 防止修復風暴 - 同一資源 5 分鐘內最多修復 3 次 - 超過限制升級為 MEDIUM 風險 - Redis 計數器 + 自動過期 修復閉環完整流程: 執行失敗 → FailureWatcher → AI 分析 → 風險評估 ├─ LOW + 冷卻期內 → 自動修復 → 揭露通知 └─ MEDIUM/CRITICAL 或超限 → Telegram 請求授權 Co-Authored-By: Claude Opus 4.5 --- apps/api/src/repositories/interfaces.py | 5 + apps/api/src/services/failure_watcher.py | 171 ++++++++++++++++++++--- 2 files changed, 159 insertions(+), 17 deletions(-) diff --git a/apps/api/src/repositories/interfaces.py b/apps/api/src/repositories/interfaces.py index 94c4b9d80..d79675c0d 100644 --- a/apps/api/src/repositories/interfaces.py +++ b/apps/api/src/repositories/interfaces.py @@ -508,13 +508,18 @@ class IFailureWatcher(Protocol): self, audit_log_id: str, repair_strategy: str, + failure_data: dict | None = None, ) -> tuple[bool, str]: """ 執行自動修復 (僅限 LOW 風險) + Phase 18.3: K8s Executor 整合 + 2026-03-31 Claude Code (統帥批准) + Args: audit_log_id: 原始失敗的 AuditLog ID repair_strategy: 修復策略 + failure_data: 失敗詳情 (含 target_resource, namespace) Returns: (success, result_message) diff --git a/apps/api/src/services/failure_watcher.py b/apps/api/src/services/failure_watcher.py index 96fd46616..477c7b0a5 100644 --- a/apps/api/src/services/failure_watcher.py +++ b/apps/api/src/services/failure_watcher.py @@ -165,11 +165,29 @@ class FailureWatcherService(IFailureWatcher): "analysis": analysis, } + # Phase 18.3: 安全檢查 - 防止修復風暴 + can_auto_repair = await self._check_repair_cooldown( + target_resource=target_resource, + namespace=failure_data.get("namespace", "awoooi"), + ) + + if not can_auto_repair: + # 超過冷卻期限制，升級為 MEDIUM + logger.info( + "repair_cooldown_escalate", + audit_log_id=audit_log_id, + target_resource=target_resource, + original_risk_level=risk_level, + ) + risk_level = "MEDIUM" + result["risk_level"] = "MEDIUM" + if risk_level == "LOW" and RISK_LEVELS["LOW"]["auto_repair"]: - # 自動修復 + # 自動修復 (Phase 18.3: 傳入完整 failure_data) success, repair_result = await self.execute_auto_repair( audit_log_id=audit_log_id, repair_strategy=analysis["suggested_repair"], + failure_data=failure_data, ) result["repair_attempted"] = True result["repair_result"] = repair_result @@ -260,13 +278,17 @@ class FailureWatcherService(IFailureWatcher): self, audit_log_id: str, repair_strategy: str, + failure_data: dict | None = None, ) -> tuple[bool, str]: """ 執行自動修復 (僅限 LOW 風險) - 目前支援: - - restart_pod: 重啟 Pod - - restart_deployment: 重啟 Deployment + Phase 18.3: K8s Executor 整合 + 2026-03-31 Claude Code (統帥批准) + + 支援操作: + - restart_deployment: 重啟 Deployment (rollout restart) + - restart_pod: 刪除 Pod 觸發重建 - clear_cache: 清理 Redis 快取 Returns: @@ -276,24 +298,90 @@ class FailureWatcherService(IFailureWatcher): "auto_repair_executing", audit_log_id=audit_log_id, strategy=repair_strategy, + target=failure_data.get("target_resource") if failure_data else None, ) try: - # 解析修復策略 - if "restart" in repair_strategy.lower(): - # 目前 Phase 18.2 只記錄，實際執行待 18.3 整合 - logger.info( - "auto_repair_logged", - audit_log_id=audit_log_id, - strategy=repair_strategy, - status="logged_for_future_execution", - ) - return True, f"已記錄修復策略: {repair_strategy} (Phase 18.3 實際執行)" + # 解析目標資源 + target_resource = failure_data.get("target_resource", "") if failure_data else "" + namespace = failure_data.get("namespace", "awoooi") if failure_data else "awoooi" + + # 解析資源類型和名稱 (格式: "deployment/api" 或 "pod/api-xxx") + resource_type = "" + resource_name = "" + if "/" in target_resource: + parts = target_resource.split("/", 1) + resource_type = parts[0].lower() + resource_name = parts[1] if len(parts) > 1 else "" + + # ===================================================================== + # Phase 18.3: 實際執行 K8s 修復操作 + # ===================================================================== + + if "restart" in repair_strategy.lower() and resource_name: + from src.services.executor import get_executor + + executor = get_executor() + + if resource_type == "deployment": + # 重啟 Deployment + result = await executor.restart_deployment( + name=resource_name, + namespace=namespace, + ) + if result.success: + logger.info( + "auto_repair_deployment_restarted", + audit_log_id=audit_log_id, + deployment=resource_name, + namespace=namespace, + ) + return True, f"✅ Deployment {resource_name} 已重啟" + else: + return False, f"❌ 重啟失敗: {result.message}" + + elif resource_type == "pod": + # 刪除 Pod 觸發重建 + result = await executor.delete_pod( + name=resource_name, + namespace=namespace, + ) + if result.success: + logger.info( + "auto_repair_pod_deleted", + audit_log_id=audit_log_id, + pod=resource_name, + namespace=namespace, + ) + return True, f"✅ Pod {resource_name} 已刪除，等待重建" + else: + return False, f"❌ 刪除失敗: {result.message}" + + else: + # 未知資源類型，記錄但不執行 + logger.warning( + "auto_repair_unknown_resource_type", + audit_log_id=audit_log_id, + resource_type=resource_type, + resource_name=resource_name, + ) + return False, f"未知資源類型: {resource_type}" elif "clear_cache" in repair_strategy.lower(): - # 清理 Redis 快取 (危險操作，暫不自動執行) - # TODO Phase 18.3: 實作安全的快取清理 - return True, "快取清理已排程 (需手動確認)" + # 清理 Redis 快取 (只清理特定前綴) + redis = get_redis() + # 安全清理: 只清理 cache 前綴 + keys = await redis.keys("awoooi:cache:*") + if keys: + await redis.delete(*keys) + logger.info( + "auto_repair_cache_cleared", + audit_log_id=audit_log_id, + keys_deleted=len(keys), + ) + return True, f"✅ 已清理 {len(keys)} 個快取 key" + else: + return True, "ℹ️ 無快取需清理" else: return False, f"未知修復策略: {repair_strategy}" @@ -351,6 +439,55 @@ class FailureWatcherService(IFailureWatcher): } return suggestions.get(classification, "需人工分析") + async def _check_repair_cooldown( + self, + target_resource: str, + namespace: str, + ) -> bool: + """ + 檢查修復冷卻期 - 防止修復風暴 + + Phase 18.3: 安全機制 + 2026-03-31 Claude Code (統帥批准) + + 規則: + - 同一資源 5 分鐘內最多修復 3 次 + - 超過則升級為 MEDIUM 風險，請求人工授權 + + Returns: + True 如果可以自動修復，False 如果超過限制 + """ + try: + redis = get_redis() + cooldown_key = f"awoooi:repair_cooldown:{namespace}:{target_resource}" + + # 檢查修復次數 + repair_count = await redis.get(cooldown_key) + if repair_count and int(repair_count) >= MAX_AUTO_REPAIR_RETRIES: + logger.warning( + "repair_cooldown_exceeded", + target_resource=target_resource, + namespace=namespace, + repair_count=int(repair_count), + max_retries=MAX_AUTO_REPAIR_RETRIES, + ) + return False + + # 增加計數並設置 5 分鐘過期 + await redis.incr(cooldown_key) + await redis.expire(cooldown_key, 300) # 5 分鐘 + + return True + + except Exception as e: + logger.warning( + "repair_cooldown_check_error", + target_resource=target_resource, + error=str(e), + ) + # 檢查失敗時，保守起見返回 True 允許修復 + return True + async def _llm_analyze( self, error_message: str,