#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ 系統告警路由模組 處理 Prometheus Alertmanager 的 Webhook 通知 並執行自動排查和修復 """ import hmac import os import json import subprocess import time import re from datetime import datetime, timezone, timedelta from flask import Blueprint, request, jsonify, current_app from functools import wraps import logging import requests logger = logging.getLogger(__name__) # 容器名稱驗證正則表達式 (只允許字母、數字、連字號、底線、點) CONTAINER_NAME_PATTERN = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9._-]*$') def is_valid_container_name(name: str) -> bool: """驗證容器名稱是否安全 (防止命令注入)""" if not name or len(name) > 128: return False return bool(CONTAINER_NAME_PATTERN.match(name)) # 豁免 CSRF 的路由列表 CSRF_EXEMPT_ROUTES = [ '/api/alert/webhook', '/api/alert/test', '/api/alert/fix' ] # 時區設定 TAIPEI_TZ = timezone(timedelta(hours=8)) # Blueprint 定義 alert_bp = Blueprint('alert', __name__) @alert_bp.before_app_request def csrf_exempt_for_alerts(): """豁免告警相關路由的 CSRF 檢查""" if request.path in CSRF_EXEMPT_ROUTES: # 標記此請求豁免 CSRF request.csrf_exempt = True # 告警 Webhook 認證 ALERT_WEBHOOK_USER = os.getenv('ALERT_WEBHOOK_USER', 'alertmanager') ALERT_WEBHOOK_PASSWORD = os.getenv('ALERT_WEBHOOK_PASSWORD', '') if not ALERT_WEBHOOK_PASSWORD: logger.warning( '[SECURITY] ALERT_WEBHOOK_PASSWORD is not set. ' 'All webhook requests will be rejected. Set this environment variable.' ) # Telegram 設定 TELEGRAM_BOT_TOKEN = os.getenv('TELEGRAM_BOT_TOKEN', '') TELEGRAM_CHAT_IDS = json.loads(os.getenv('TELEGRAM_CHAT_IDS', '[]')) # 告警歷史記錄(記憶體快取) _alert_history = [] _MAX_ALERT_HISTORY = 100 # 自動修復設定 AUTO_FIX_ENABLED = os.getenv('AUTO_FIX_ENABLED', 'true').lower() == 'true' AUTO_FIX_COOLDOWN = 300 # 同一容器 5 分鐘內不重複修復 _last_fix_time = {} # issue_type allowlist(防止任意值傳入 auto_fix_container) ALLOWED_ISSUE_TYPES = frozenset({ 'memory', 'container_memory', 'cpu', 'container_cpu', }) ALLOWED_AUTO_FIX_CONTAINERS = frozenset({ 'momo-pro-system', 'momo-scheduler', 'momo-telegram-bot', }) PROTECTED_CONTAINERS = frozenset({ 'momo-db', 'momo-postgres', 'momo-prometheus', 'momo-grafana', }) def check_alert_auth(f): """告警 Webhook 認證裝飾器(fail-secure:密碼未設定時拒絕所有請求)""" @wraps(f) def decorated(*args, **kwargs): # fail-secure: 若密碼未設定,拒絕一切存取 if not ALERT_WEBHOOK_PASSWORD: return jsonify({'error': 'Server misconfiguration: auth not configured'}), 503 auth = request.authorization if not auth: return jsonify({'error': 'Missing authentication'}), 401 if ( not hmac.compare_digest(auth.username or '', ALERT_WEBHOOK_USER) or not hmac.compare_digest(auth.password or '', ALERT_WEBHOOK_PASSWORD) ): return jsonify({'error': 'Invalid credentials'}), 401 return f(*args, **kwargs) return decorated def send_telegram_message(message: str, parse_mode: str = 'HTML') -> bool: """ 發送 Telegram 訊息 ADR-019 Phase 5: 改走 EventRouter 統一入口,享 retry + JSONL queue replay。 parse_mode 參數保留向後相容(EventRouter 內部統一 HTML,舊 caller 若指定 其他 mode 會被忽略;此 route 原本就只用 HTML,影響面=0)。 """ if not TELEGRAM_BOT_TOKEN or not TELEGRAM_CHAT_IDS: logger.warning("[Alert] Telegram 未設定,無法發送告警") return False try: from services.event_router import dispatch_sync result = dispatch_sync(event={ "event_type": "alertmanager_webhook", "severity": "alert", "source": "Alertmanager.Webhook", "title": "Alertmanager 告警", "summary": message[:400], "status": "received", "payload": {"raw_message": message, "parse_mode": parse_mode}, }, admin_chat_ids=TELEGRAM_CHAT_IDS) delivered = bool(result.get("delivered")) if not delivered: logger.error(f"[Alert] EventRouter 告警未送達: {result.get('errors')}") else: logger.info(f"[Alert] EventRouter 告警已派送 sent={result.get('sent')}") return delivered except Exception as e: logger.error(f"[Alert] EventRouter dispatch 失敗: {e}") return False def format_alert_message(alert: dict, status: str) -> str: """格式化告警訊息""" labels = alert.get('labels', {}) annotations = alert.get('annotations', {}) severity = labels.get('severity', 'unknown') alertname = labels.get('alertname', 'Unknown') category = labels.get('category', '') # 嚴重程度圖示 severity_icons = { 'critical': '🔴', 'warning': '🟡', 'info': '🔵' } icon = severity_icons.get(severity, '⚪') # 狀態圖示 status_icon = '🚨' if status == 'firing' else '✅' status_text = '觸發' if status == 'firing' else '恢復' # 時間 now = datetime.now(TAIPEI_TZ).strftime('%Y-%m-%d %H:%M:%S') # 取得詳細資訊 summary = annotations.get('summary', alertname) description = annotations.get('description', '') value = annotations.get('value', '') container = annotations.get('container', labels.get('name', '')) # 組裝訊息 message = f""" {status_icon} 系統告警{status_text} {icon} 告警名稱:{summary} 嚴重程度:{severity.upper()} 類別:{category} """ if container: message += f"容器:{container}\n" if value: message += f"當前值:{value}\n" if description: message += f"\n詳細說明:\n{description}\n" message += f"\n時間:{now}" return message.strip() def analyze_high_load() -> dict: """分析系統高負載原因""" result = { 'timestamp': datetime.now(TAIPEI_TZ).isoformat(), 'high_cpu_containers': [], 'high_memory_containers': [], 'system_info': {}, 'recommendations': [] } try: # 取得所有容器的資源使用情況 (使用列表參數避免 shell 注入) cmd = ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"] proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30) # nosec B603 if proc.returncode == 0: for line in proc.stdout.strip().split('\n'): if not line: continue parts = line.split('\t') if len(parts) >= 4: name = parts[0] cpu = float(parts[1].replace('%', '')) mem_usage = parts[2] mem_pct = float(parts[3].replace('%', '')) if cpu > 50: result['high_cpu_containers'].append({ 'name': name, 'cpu_percent': cpu, 'memory_usage': mem_usage }) if mem_pct > 50: result['high_memory_containers'].append({ 'name': name, 'memory_percent': mem_pct, 'memory_usage': mem_usage }) # 排序 result['high_cpu_containers'].sort(key=lambda x: x['cpu_percent'], reverse=True) result['high_memory_containers'].sort(key=lambda x: x['memory_percent'], reverse=True) # 生成建議 for container in result['high_cpu_containers'][:3]: result['recommendations'].append( f"容器 {container['name']} CPU 使用率 {container['cpu_percent']:.1f}%,建議檢查或重啟" ) for container in result['high_memory_containers'][:3]: result['recommendations'].append( f"容器 {container['name']} 記憶體使用 {container['memory_usage']},可能有記憶體洩漏" ) except Exception as e: logger.error(f"[Alert] 分析高負載失敗: {e}") result['error'] = str(e) return result def auto_fix_container(container_name: str, issue_type: str) -> dict: """自動修復容器問題""" result = { 'container': container_name, 'issue_type': issue_type, 'action': None, 'success': False, 'message': '' } # 安全性檢查: 驗證容器名稱格式 (防止命令注入) if not is_valid_container_name(container_name): result['message'] = "無效的容器名稱格式" return result # 檢查冷卻時間 key = f"{container_name}_{issue_type}" now = time.time() if key in _last_fix_time: elapsed = now - _last_fix_time[key] if elapsed < AUTO_FIX_COOLDOWN: result['message'] = f"冷卻中,{int(AUTO_FIX_COOLDOWN - elapsed)} 秒後可再次修復" return result try: # 排除關鍵容器 if container_name in PROTECTED_CONTAINERS: result['message'] = "此為受保護的關鍵容器,不自動重啟" return result if container_name not in ALLOWED_AUTO_FIX_CONTAINERS: result['message'] = "容器不在 EwoooC 自動修復白名單" return result # 根據問題類型決定修復動作 if issue_type in ['container_memory', 'memory']: # 記憶體問題 - 重啟容器 (使用列表參數避免 shell 注入) result['action'] = 'restart' cmd = ["docker", "restart", container_name] proc = subprocess.run(cmd, capture_output=True, text=True, timeout=60) # nosec B603 if proc.returncode == 0: result['success'] = True result['message'] = f"已重啟容器 {container_name}" _last_fix_time[key] = now else: result['message'] = f"重啟失敗: {proc.stderr}" elif issue_type in ['container_cpu', 'cpu']: # CPU 問題 - 先分析,如果持續則重啟 # 取得容器內的高 CPU 進程 (使用列表參數避免 shell 注入) cmd = ["docker", "exec", container_name, "ps", "aux", "--sort=-%cpu"] proc = subprocess.run(cmd, capture_output=True, text=True, timeout=30) # nosec B603 # 取前 5 行 output_lines = proc.stdout.strip().split('\n')[:5] if proc.returncode == 0: result['action'] = 'analyze' result['message'] = f"容器內 CPU 使用情況:\n" + '\n'.join(output_lines) result['success'] = True else: # 如果無法分析,嘗試重啟 result['action'] = 'restart' cmd = ["docker", "restart", container_name] proc = subprocess.run(cmd, capture_output=True, text=True, timeout=60) # nosec B603 result['success'] = proc.returncode == 0 result['message'] = f"已重啟容器 {container_name}" if result['success'] else f"重啟失敗: {proc.stderr}" if result['success']: _last_fix_time[key] = now except subprocess.TimeoutExpired: result['message'] = "操作超時" except Exception as e: result['message'] = f"修復錯誤: {str(e)}" return result @alert_bp.route('/api/alert/webhook', methods=['POST']) @check_alert_auth def alert_webhook(): """ 接收 Alertmanager Webhook 通知 自動分析問題並嘗試修復 """ try: data = request.get_json() if not data: return jsonify({'error': 'No data received'}), 400 status = data.get('status', 'unknown') alerts = data.get('alerts', []) logger.info(f"[Alert] 收到告警 Webhook: status={status}, alerts={len(alerts)}") processed_alerts = [] fix_results = [] for alert in alerts: alert_status = alert.get('status', status) labels = alert.get('labels', {}) category = labels.get('category', '') severity = labels.get('severity', '') container_name = alert.get('annotations', {}).get('container', labels.get('name', '')) # 格式化並發送 Telegram 通知 message = format_alert_message(alert, alert_status) send_telegram_message(message) # 記錄告警歷史 alert_record = { 'timestamp': datetime.now(TAIPEI_TZ).isoformat(), 'status': alert_status, 'alertname': labels.get('alertname'), 'severity': severity, 'category': category, 'container': container_name, 'message': message } _alert_history.append(alert_record) if len(_alert_history) > _MAX_ALERT_HISTORY: _alert_history.pop(0) processed_alerts.append(alert_record) # 如果是觸發狀態且啟用自動修復 if alert_status == 'firing' and AUTO_FIX_ENABLED: # 分析問題 if severity in ['warning', 'critical']: analysis = analyze_high_load() # 發送分析報告 if analysis.get('recommendations'): analysis_msg = "📊 系統分析報告\n\n" for rec in analysis['recommendations'][:5]: analysis_msg += f"• {rec}\n" send_telegram_message(analysis_msg) # 自動修復容器問題 if container_name and category in ['container_cpu', 'container_memory']: fix_result = auto_fix_container(container_name, category) fix_results.append(fix_result) # 發送修復結果通知 if fix_result['action']: fix_msg = f"🔧 自動修復\n\n" fix_msg += f"容器: {fix_result['container']}\n" fix_msg += f"動作: {fix_result['action']}\n" fix_msg += f"結果: {'成功 ✅' if fix_result['success'] else '失敗 ❌'}\n" fix_msg += f"說明: {fix_result['message']}" send_telegram_message(fix_msg) return jsonify({ 'success': True, 'processed': len(processed_alerts), 'fix_results': fix_results }) except Exception as e: logger.error(f"[Alert] Webhook 處理錯誤: {e}") return jsonify({'error': str(e)}), 500 @alert_bp.route('/api/alert/history') @check_alert_auth def get_alert_history(): """取得告警歷史記錄(需認證,防止系統資訊外洩)""" limit = request.args.get('limit', 50, type=int) # 限制 limit 範圍,防止記憶體 DoS limit = max(1, min(limit, 200)) return jsonify({ 'success': True, 'data': _alert_history[-limit:][::-1] # 最新的在前 }) @alert_bp.route('/api/alert/analyze') @check_alert_auth def analyze_system(): """手動觸發系統分析(需認證,防止系統資訊外洩)""" analysis = analyze_high_load() return jsonify({ 'success': True, 'data': analysis }) @alert_bp.route('/api/alert/fix', methods=['POST']) @check_alert_auth def manual_fix(): """手動觸發修復(需認證 + issue_type allowlist)""" data = request.get_json() or {} container_name = data.get('container') issue_type = data.get('issue_type', 'memory') if not container_name: return jsonify({'error': 'Missing container name'}), 400 # Security: issue_type 只允許已知有效值 if issue_type not in ALLOWED_ISSUE_TYPES: return jsonify({ 'error': f'Invalid issue_type: {issue_type}. ' f'Allowed: {sorted(ALLOWED_ISSUE_TYPES)}' }), 400 result = auto_fix_container(container_name, issue_type) return jsonify({ 'success': result['success'], 'data': result }) @alert_bp.route('/api/alert/test', methods=['POST']) @check_alert_auth def test_alert(): """測試告警通知""" message = """ 🧪 測試告警 這是一則測試告警訊息。 如果您收到此訊息,表示告警系統正常運作。 時間:{} """.format(datetime.now(TAIPEI_TZ).strftime('%Y-%m-%d %H:%M:%S')) success = send_telegram_message(message) return jsonify({ 'success': success, 'message': '測試訊息已發送' if success else '發送失敗,請檢查 Telegram 設定' }) @alert_bp.route('/api/alert/status') @check_alert_auth def alert_status(): """取得告警系統狀態""" return jsonify({ 'success': True, 'data': { 'telegram_configured': bool(TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_IDS), 'auto_fix_enabled': AUTO_FIX_ENABLED, 'auto_fix_cooldown': AUTO_FIX_COOLDOWN, 'alert_history_count': len(_alert_history), 'last_fix_times': _last_fix_time } })