feat(adr-076): 戰術 B 四大 Task 全部完成 — 告警聚合+重試+自動報告
All checks were successful
CD Pipeline / build-and-deploy (push) Successful in 17m34s
All checks were successful
CD Pipeline / build-and-deploy (push) Successful in 17m34s
Task 2: AlertGroupingService — Redis 5分鐘滑動視窗,防告警風暴 - apps/api/src/services/alert_grouping_service.py (新增) - webhooks.py 整合:指紋生成後/LLM前短路子告警 - Threshold=3,Graceful Degradation,16 tests Task 3: approval_execution.py 執行失敗重試 - MAX_RETRY=2, RETRY_DELAY_SECONDS=30 - _is_transient_error() 瞬態/永久分類,永久錯誤不重試 - Timeline 記錄重試進度,成功後標注重試次數,29 tests Task 4: report_generation_service.py 自動報告 - 日度巡檢報告:每日 08:00 台北時間,Telegram SRE 群組推送 - Postmortem:Incident resolved + duration > 10 分鐘自動觸發 - main.py lifespan 掛載 run_daily_report_loop(),30 tests 測試: 600 → 675 通過 (+75),0 failed Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -10,10 +10,17 @@ Approval Execution Service - Phase 16 R4.2 瘦身 Router 抽取
|
||||
- NotificationManager: 發送通知
|
||||
- Phase 7.6: Playbook 自動萃取
|
||||
|
||||
版本: v1.1
|
||||
版本: v1.2
|
||||
建立: 2026-03-25 (台北時區)
|
||||
更新: 2026-03-26 (Phase 7.6 自動萃取)
|
||||
更新: 2026-04-14 (ADR-076 Task 3: 執行失敗重試機制 — Claude Haiku 4.5 Asia/Taipei)
|
||||
建立者: Claude Code (Phase 16 R4.2)
|
||||
|
||||
重試設計 (ADR-076):
|
||||
- MAX_RETRY = 2 次(共最多 3 次嘗試)
|
||||
- RETRY_DELAY_SECONDS = 30 秒
|
||||
- 只重試瞬態錯誤(connection refused, timeout, i/o error 等)
|
||||
- 永久性錯誤(not found, permission denied, already exists)不重試
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -39,12 +46,67 @@ class ApprovalExecutionService:
|
||||
|
||||
職責:
|
||||
1. 解析操作類型
|
||||
2. 呼叫 K8s Executor 執行
|
||||
2. 呼叫 K8s Executor 執行(含重試)
|
||||
3. 更新資料庫狀態
|
||||
4. 記錄 Timeline 事件
|
||||
5. 發送通知
|
||||
"""
|
||||
|
||||
# ADR-076 Task 3: 重試常數
|
||||
MAX_RETRY: int = 2
|
||||
RETRY_DELAY_SECONDS: int = 30
|
||||
|
||||
# 瞬態錯誤關鍵字(小寫比對),符合任一 → 可重試
|
||||
_TRANSIENT_ERROR_KEYWORDS: tuple[str, ...] = (
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"timeout",
|
||||
"timed out",
|
||||
"i/o error",
|
||||
"io error",
|
||||
"temporary failure",
|
||||
"service unavailable",
|
||||
"too many requests",
|
||||
"dial tcp",
|
||||
"eof",
|
||||
)
|
||||
|
||||
# 永久性錯誤關鍵字(小寫比對),符合任一 → 不重試
|
||||
_PERMANENT_ERROR_KEYWORDS: tuple[str, ...] = (
|
||||
"not found",
|
||||
"forbidden",
|
||||
"permission denied",
|
||||
"unauthorized",
|
||||
"already exists",
|
||||
"invalid",
|
||||
"immutable",
|
||||
"destructive",
|
||||
"blocked",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _is_transient_error(cls, error_message: str | None) -> bool:
|
||||
"""
|
||||
判斷執行錯誤是否為瞬態(可重試)
|
||||
|
||||
優先檢查永久性錯誤(比瞬態錯誤有更高的優先順序),
|
||||
避免 "connection refused (not found)" 這類混合訊息誤判。
|
||||
|
||||
Args:
|
||||
error_message: 執行錯誤訊息
|
||||
|
||||
Returns:
|
||||
True 表示可重試,False 表示永久失敗
|
||||
"""
|
||||
if not error_message:
|
||||
return False
|
||||
lower = error_message.lower()
|
||||
# 永久性錯誤 → 不重試
|
||||
if any(kw in lower for kw in cls._PERMANENT_ERROR_KEYWORDS):
|
||||
return False
|
||||
# 瞬態錯誤 → 可重試
|
||||
return any(kw in lower for kw in cls._TRANSIENT_ERROR_KEYWORDS)
|
||||
|
||||
async def execute_approved_action(self, approval: ApprovalRequest) -> None:
|
||||
"""
|
||||
背景執行已批准的操作
|
||||
@@ -104,7 +166,8 @@ class ApprovalExecutionService:
|
||||
)
|
||||
return
|
||||
|
||||
# Execute with audit
|
||||
# ADR-076 Task 3: 執行失敗重試機制
|
||||
# 瞬態錯誤 (connection refused, timeout 等) 自動重試,最多 MAX_RETRY 次
|
||||
executor = get_executor()
|
||||
result = await executor.execute_with_audit(
|
||||
approval=approval,
|
||||
@@ -113,10 +176,48 @@ class ApprovalExecutionService:
|
||||
namespace=namespace,
|
||||
)
|
||||
|
||||
attempt = 1
|
||||
while not result.success and attempt <= self.MAX_RETRY:
|
||||
if not self._is_transient_error(result.error):
|
||||
logger.info(
|
||||
"execution_retry_skipped_permanent_error",
|
||||
approval_id=str(approval.id),
|
||||
attempt=attempt,
|
||||
error=result.error,
|
||||
)
|
||||
break
|
||||
|
||||
logger.warning(
|
||||
"execution_retry_transient_error",
|
||||
approval_id=str(approval.id),
|
||||
attempt=attempt,
|
||||
max_retry=self.MAX_RETRY,
|
||||
error=result.error,
|
||||
delay_seconds=self.RETRY_DELAY_SECONDS,
|
||||
)
|
||||
await timeline.add_event(
|
||||
event_type="exec",
|
||||
status="warning",
|
||||
title=f"⚠️ 執行失敗,{self.RETRY_DELAY_SECONDS}s 後重試 ({attempt}/{self.MAX_RETRY})",
|
||||
description=f"Error: {result.error}",
|
||||
actor="leWOOOgo",
|
||||
actor_role="executor",
|
||||
approval_id=str(approval.id),
|
||||
)
|
||||
await asyncio.sleep(self.RETRY_DELAY_SECONDS)
|
||||
result = await executor.execute_with_audit(
|
||||
approval=approval,
|
||||
operation_type=operation_type,
|
||||
resource_name=resource_name,
|
||||
namespace=namespace,
|
||||
)
|
||||
attempt += 1
|
||||
|
||||
# Phase 5: 更新資料庫狀態
|
||||
await service.update_execution_status(approval.id, success=result.success)
|
||||
|
||||
# Update approval status based on result
|
||||
total_attempts = attempt # attempt 在重試迴圈後為最終嘗試次數
|
||||
if result.success:
|
||||
logger.info(
|
||||
"background_execution_success",
|
||||
@@ -125,11 +226,13 @@ class ApprovalExecutionService:
|
||||
target=resource_name,
|
||||
namespace=namespace,
|
||||
duration_ms=result.duration_ms,
|
||||
total_attempts=total_attempts,
|
||||
)
|
||||
retry_note = f" (重試 {total_attempts - 1} 次後成功)" if total_attempts > 1 else ""
|
||||
await timeline.add_event(
|
||||
event_type="exec",
|
||||
status="success",
|
||||
title=f"✅ K8s 執行成功: {operation_type.value}",
|
||||
title=f"✅ K8s 執行成功: {operation_type.value}{retry_note}",
|
||||
description=f"Target: {resource_name} @ {namespace} ({result.duration_ms}ms)",
|
||||
actor="leWOOOgo",
|
||||
actor_role="executor",
|
||||
|
||||
Reference in New Issue
Block a user