""" Log Summary Service - Phase 31 ================================ 職責:擷取 K8s Pod log,用 deepseek-r1:14b 生成繁中異常摘要 設計邊界: - 只輸出摘要文字,不做 RCA、不生成修復指令 - 5s 軟超時:超過則回傳 None,主流程不阻塞 - LLM 硬超時 180s(deepseek-r1:14b 慢但準) - Redis 快取:log_summary:{pod}:{date} TTL 24h,同 pod 同天不重複呼叫 - 敏感資料遮蔽:Bearer token、密碼 regex → [REDACTED] - 複用 K8sDiagnosticsService 取得 logs 版本: v1.0 建立: 2026-04-10 (台北時區) 建立者: Claude Code (Phase 31 ADR-067) """ from __future__ import annotations import asyncio import re from datetime import UTC, datetime import httpx import structlog from src.core.redis_client import get_redis from src.services.model_registry import get_model logger = structlog.get_logger(__name__) # ============================================================ # 設定 # ============================================================ # 2026-05-19 Codex: 所有 Ollama workload 固定 GCP-A → GCP-B → 111。 def _get_ollama_endpoints(): from src.services.ollama_endpoint_resolver import resolve_ollama_order return resolve_ollama_order("deep_rca") # D1 集中化 2026-04-11: 從 models.json providers.ollama.models.log_anomaly 讀取 SUMMARY_MODEL = get_model("ollama", "log_anomaly") LLM_TIMEOUT = 180.0 # deepseek-r1 硬超時 SOFT_TIMEOUT = 5.0 # 主流程軟超時(超過回 None) LOG_TAIL_LINES = 100 ANOMALY_TAIL_LINES = 50 # 只取最後 50 行異常行送 LLM CACHE_TTL = 86400 # 24 小時 CACHE_PREFIX = "log_summary:" # 異常關鍵字 pattern _ANOMALY_PATTERN = re.compile( r"(ERROR|FATAL|Exception|Traceback|OOMKilled|panic|CRITICAL|WARN|WARNING)", re.IGNORECASE, ) # 敏感資料遮蔽 pattern _SENSITIVE_PATTERNS = [ (re.compile(r"Bearer\s+[A-Za-z0-9+/=._-]{10,}", re.IGNORECASE), "Bearer [REDACTED]"), (re.compile(r"password[=:\s]+\S+", re.IGNORECASE), "password=[REDACTED]"), (re.compile(r"token[=:\s]+[A-Za-z0-9+/=._-]{10,}", re.IGNORECASE), "token=[REDACTED]"), (re.compile(r"secret[=:\s]+\S+", re.IGNORECASE), "secret=[REDACTED]"), ] # ============================================================ # Prompt # ============================================================ _SUMMARY_PROMPT = """你是 AWOOOI SRE 維運助理,請分析以下 K8s Pod log 片段。 ## Pod: {pod_name} (namespace: {namespace}) ## Log 異常片段(最近 {line_count} 行): {log_snippet} ## 任務 用繁體中文,3 行以內,說明: 1. 異常的主要原因是什麼 2. 影響程度(例如 OOM Killed、連線失敗、程式錯誤) 3. 建議立即查看哪個方向 只輸出摘要文字,不要標題或 markdown 格式。 """ class LogSummaryService: """ Pod Log 異常摘要服務 職責邊界: ✅ 擷取 Pod log(複用 K8sDiagnosticsService) ✅ 過濾異常行 + 敏感資料遮蔽 ✅ deepseek-r1:14b 生成繁中摘要(180s timeout) ✅ Redis 快取(24h TTL) ❌ 不做 RCA ❌ 不生成修復指令 ❌ 不阻塞告警主流程(軟超時 5s) """ async def summarize( self, pod_name: str, namespace: str = "awoooi-prod", ) -> str | None: """ 取得 Pod log 異常摘要 Returns: str: 繁中摘要文字(3 行內) None: 快取命中、無異常、或超時 """ cache_key = self._cache_key(pod_name) redis = await get_redis() cached = await redis.get(cache_key) if cached: logger.debug("log_summary_cache_hit", pod=pod_name) return cached.decode() if isinstance(cached, bytes) else cached raw_logs = await self._fetch_logs(pod_name, namespace) if not raw_logs: return None anomaly_lines = self._extract_anomaly_lines(raw_logs) if not anomaly_lines: logger.debug("log_summary_no_anomaly", pod=pod_name) return None summary = await self._call_llm(pod_name, namespace, anomaly_lines) if summary: await redis.set(cache_key, summary, ex=CACHE_TTL) logger.info("log_summary_generated", pod=pod_name, lines=len(anomaly_lines)) return summary async def summarize_with_soft_timeout( self, pod_name: str, namespace: str = "awoooi-prod", ) -> str | None: """ 帶 5s 軟超時的摘要取得 用於告警主流程:超過軟超時回傳 None,不阻塞 Telegram 推送 LLM 繼續在背景跑,結果寫入 Redis 快取供下次使用 """ try: return await asyncio.wait_for( self.summarize(pod_name, namespace), timeout=SOFT_TIMEOUT, ) except TimeoutError: logger.info( "log_summary_soft_timeout", pod=pod_name, soft_timeout=SOFT_TIMEOUT, ) # 繼續在背景跑,結果存 Redis 備用 asyncio.create_task(self.summarize(pod_name, namespace)) return None # -------------------------------------------------------- # Private helpers # -------------------------------------------------------- def _cache_key(self, pod_name: str) -> str: date_str = datetime.now(UTC).strftime("%Y-%m-%d") return f"{CACHE_PREFIX}{pod_name}:{date_str}" async def _fetch_logs(self, pod_name: str, namespace: str) -> str | None: """複用 K8sDiagnosticsService 取得 logs""" try: from src.services.k8s_diagnostics import K8sDiagnosticsService svc = K8sDiagnosticsService(default_namespace=namespace) diag = await svc.collect_diagnostics( pod_name=pod_name, namespace=namespace, log_tail_lines=LOG_TAIL_LINES, include_previous_logs=False, ) return diag.logs or None except Exception as e: logger.warning("log_summary_fetch_failed", pod=pod_name, error=str(e)) return None def _extract_anomaly_lines(self, raw_logs: str) -> list[str]: """過濾異常行 + 敏感資料遮蔽,取最後 ANOMALY_TAIL_LINES 行""" lines = raw_logs.splitlines() anomaly = [line for line in lines if _ANOMALY_PATTERN.search(line)] # 取最後 N 行 anomaly = anomaly[-ANOMALY_TAIL_LINES:] # 遮蔽敏感資料 result = [] for line in anomaly: for pattern, replacement in _SENSITIVE_PATTERNS: line = pattern.sub(replacement, line) result.append(line) return result async def _call_llm( self, pod_name: str, namespace: str, anomaly_lines: list[str], ) -> str | None: """呼叫 deepseek-r1:14b 生成摘要""" log_snippet = "\n".join(anomaly_lines) prompt = _SUMMARY_PROMPT.format( pod_name=pod_name, namespace=namespace, line_count=len(anomaly_lines), log_snippet=log_snippet[:3000], # 避免超出 context ) async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client: for endpoint in _get_ollama_endpoints(): if not endpoint.url: continue try: resp = await client.post( f"{endpoint.url}/api/generate", json={ "model": SUMMARY_MODEL, "prompt": prompt, "stream": False, # Ollama 0.24 returns deepseek-r1 text in `thinking` by default. # Existing callers read `response`, so force final-answer mode. "think": False, "options": {"temperature": 0.1, "num_predict": 200}, }, ) resp.raise_for_status() data = resp.json() raw = data.get("response", "").strip() # 過濾 deepseek-r1 的 ... 推理區塊 text = re.sub(r".*?", "", raw, flags=re.DOTALL).strip() return text or raw or None except httpx.TimeoutException: logger.warning( "log_summary_llm_timeout", model=SUMMARY_MODEL, provider=endpoint.provider_name, ) except Exception as e: logger.warning( "log_summary_llm_error", provider=endpoint.provider_name, error=str(e), ) return None # ============================================================ # Singleton # ============================================================ _log_summary_service: LogSummaryService | None = None def get_log_summary_service() -> LogSummaryService: global _log_summary_service if _log_summary_service is None: _log_summary_service = LogSummaryService() return _log_summary_service