""" Runbook Generator - Phase 25 P1 Knowledge Auto-Harvesting ========================================================== 修復後自動生成 Runbook(成功)或 Anti-Pattern(失敗) 透過唯一 production AI chain 生成,結果沉澱至 KM 知識庫 設計原則: - 非阻塞:asyncio.create_task() 呼叫,絕不影響 AutoRepair 主流程 - 失敗靜默:生成失敗只記 log,不拋例外 - DRAFT/PUBLISHED:成功 → DRAFT(AI controlled review queue),失敗 → PUBLISHED(直接封鎖) 版本: v2.0 建立: 2026-04-04 (台北時區) 建立者: ogt (首席架構師設計) + Claude Code (實作) 關聯設計: docs/superpowers/specs/2026-04-04-nemotron-active-defense-design.md 方向一 變更紀錄: | 版本 | 日期 | 執行者 | 變更內容 | |------|------|--------|----------| | v1.0 | 2026-04-04 | Claude Code | 初始佔位(使用 generate() 但介面不存在) | | v1.1 | 2026-04-04 | ogt (首席架構師) | 接入舊版 cloud provider 介面;新增 Minimal fallback(已取代) | | v2.0 | 2026-07-14 | Codex | 移除 direct NVIDIA;統一 GCP-A → GCP-B → 111 → Gemini 與原子費控 receipt | """ from __future__ import annotations import html import json import re from typing import TYPE_CHECKING import structlog from src.models.knowledge import ( EntrySource, EntryStatus, EntryType, KnowledgeEntryCreate, ) if TYPE_CHECKING: from src.models.incident import Incident from src.models.playbook import Playbook from src.services.auto_repair_service import AutoRepairResult logger = structlog.get_logger(__name__) _CARD_MAX_LEN = 3600 _SECTION_RE = re.compile(r"^#{1,6}\s+(?P.+?)\s*$") _BULLET_RE = re.compile(r"^\s*(?:[-*]|\d+[.)])\s*") _CORRELATION_SANITIZE_RE = re.compile(r"[^A-Za-z0-9._:/-]+") def _html(text: object) -> str: return html.escape(str(text), quote=False) def _shorten(text: object, limit: int = 120) -> str: compact = " ".join(str(text or "").split()) if len(compact) <= limit: return compact return compact[: max(0, limit - 1)].rstrip() + "…" def _clean_preview_line(line: str) -> str: line = _SECTION_RE.sub("", line.strip()) line = _BULLET_RE.sub("", line).strip() line = line.replace("`", "") return " ".join(line.split()) def _section_preview(content: str, title_keyword: str, *, fallback: str) -> str: """從 Markdown 內容抽一行可讀摘要,避免把整段 Runbook 原文丟進 Telegram。""" lines = str(content or "").splitlines() in_section = False for raw_line in lines: line = raw_line.strip() if not line: continue heading = _SECTION_RE.match(line) if heading: in_section = title_keyword in heading.group("title") continue if not in_section: continue preview = _clean_preview_line(line) if preview: return _shorten(preview, 120) return fallback def _step_preview(content: str) -> str: preview = _section_preview(content, "執行", fallback="待 controlled review 的 Runbook 執行步驟") if any(token in preview for token in ("{host}", "{target}", "Unsupported scheme", "Invalid component name")): return "含 placeholder 或不支援的執行步驟,需 AI 修補 / break-glass 後才能發布" return _shorten(preview, 120) def _correlation_id(prefix: str, value: object) -> str: """Build a bounded receipt-safe identifier without copying arbitrary text.""" candidate = f"{prefix}:{value}" candidate = _CORRELATION_SANITIZE_RE.sub("-", candidate).strip("-./:") if not candidate or not candidate[0].isalnum(): candidate = prefix return candidate[:160] def _generated_document(raw_response: str) -> str | None: """Extract the allowlisted document field from JSON-mode provider output.""" raw = str(raw_response or "").strip() if not raw: return None if raw.startswith("```"): lines = raw.splitlines() if lines and lines[0].strip().lower() in {"```", "```json"}: lines = lines[1:] if lines and lines[-1].strip() == "```": lines = lines[:-1] raw = "\n".join(lines).strip() try: payload = json.loads(raw) except json.JSONDecodeError: # Compatibility with an Ollama model that ignored JSON mode. Only a # structured Markdown document is accepted; arbitrary prose is not. return raw if raw.startswith("## ") else None if isinstance(payload, str): return payload.strip() or None if not isinstance(payload, dict): return None content = payload.get("content") return content.strip() if isinstance(content, str) and content.strip() else None def format_runbook_review_card( incident: object, entry_id: str, content: str, ) -> str: """格式化 Telegram Runbook 審核卡片。 2026-05-07 Codex — 將純文字 Markdown preview 改成治理卡片,讓 SRE 能快速判斷知識狀態、受影響服務與審核重點。 """ incident_id = getattr(incident, "incident_id", "unknown") services = ", ".join(getattr(incident, "affected_services", None) or []) or "unknown" symptom = _section_preview(content, "症狀", fallback=f"Incident {incident_id} 的修復知識待審核") step = _step_preview(content) message = ( "📄 <b>RUNBOOK REVIEW|待審核</b>\n" "──────────────────────\n" f"📋 Incident:<code>{_html(incident_id)}</code>\n" f"🧩 受影響服務:<code>{_html(services)}</code>\n" "🧠 知識狀態:<b>DRAFT|AI controlled review</b>\n" f"🗂️ Entry ID:<code>{_html(entry_id)}</code>\n\n" "🧾 <b>內容摘要</b>\n" f"├ 症狀:{_html(symptom)}\n" f"└ 執行:{_html(step)}\n\n" "✅ <b>審核重點</b>\n" "1. 確認步驟可重跑,且不含 placeholder / 不支援 scheme\n" "2. 補齊適用條件、rollback 與驗證方式\n\n" "🔎 AwoooP:知識庫 / Runbook Review" ) return message[:_CARD_MAX_LEN] class NemotronRunbookGenerator: """ Runbook 自動生成器。 類名只為舊 import 相容;production 執行不再呼叫 Nemotron/NVIDIA。 職責: - 成功修復 → AUTO_RUNBOOK (DRAFT) + Telegram 審核 card - 失敗修復 → ANTI_PATTERN (PUBLISHED) + Telegram 通知 leWOOOgo 積木化: - 呼叫 KnowledgeService(不直接存 DB) - 呼叫 AIRouterExecutor,唯一順序 GCP-A → GCP-B → host111 → Gemini - Gemini 只能透過 provider 內的 durable disable-state 與原子費控 receipt """ _RUNBOOK_SYSTEM = ( "你是 AWOOOI 平台的 SRE Runbook 撰寫專家。" "根據提供的 Incident 與修復結果,用繁體中文生成完整結構化 Runbook。" ) _ANTI_PATTERN_SYSTEM = ( "你是 AWOOOI 平台的故障分析專家。" "根據失敗的修復嘗試,用繁體中文生成失敗案例記錄,幫助未來避免重蹈覆轍。" ) async def generate_runbook( self, incident: Incident, playbook: Playbook, result: AutoRepairResult, symptoms_hash: str, ) -> None: """ 成功修復後生成 AUTO_RUNBOOK(fire-and-forget,呼叫方不等待) Args: incident: 觸發的 Incident playbook: 執行的 Playbook result: 執行結果(success=True) symptoms_hash: SymptomPattern.compute_hash() 的 hash """ try: content = await self._generate_runbook_content(incident, playbook, result) if not content: return from src.services.knowledge_service import get_knowledge_service ks = get_knowledge_service() entry_data = KnowledgeEntryCreate( title=f"[AUTO] {incident.incident_id} — {playbook.name}", content=content, entry_type=EntryType.AUTO_RUNBOOK, category="ai_system", tags=list(incident.affected_services or []) + ["auto_runbook", "ai_route"], source=EntrySource.AI_EXTRACTED, status=EntryStatus.DRAFT, related_incident_id=incident.incident_id, related_playbook_id=playbook.playbook_id, symptoms_hash=symptoms_hash, created_by="ai_route_runbook_generator", ) entry = await ks.create_entry(entry_data) logger.info( "auto_runbook_created", incident_id=incident.incident_id, entry_id=entry.id, playbook_id=playbook.playbook_id, ) await self._push_runbook_review_card(incident, entry.id, content) except Exception as e: logger.error( "runbook_generation_failed", incident_id=incident.incident_id, error=str(e), ) async def generate_anti_pattern( self, incident: Incident, playbook: Playbook, result: AutoRepairResult, symptoms_hash: str, ) -> None: """ 失敗修復後生成 ANTI_PATTERN(fire-and-forget,直接 PUBLISHED) Args: incident: 觸發的 Incident playbook: 嘗試執行的 Playbook result: 執行結果(success=False) symptoms_hash: SymptomPattern.compute_hash() 的 hash """ try: content = await self._generate_anti_pattern_content(incident, playbook, result) if not content: return from src.services.knowledge_service import get_knowledge_service ks = get_knowledge_service() title = f"[FAIL] {incident.incident_id} — {playbook.name}" entry_data = KnowledgeEntryCreate( title=title, content=content, entry_type=EntryType.ANTI_PATTERN, category="failure_cases", tags=list(incident.affected_services or []) + ["anti_pattern", "failure"], source=EntrySource.AI_EXTRACTED, status=EntryStatus.PUBLISHED, # 直接發布,無需審核 related_incident_id=incident.incident_id, related_playbook_id=playbook.playbook_id, symptoms_hash=symptoms_hash, created_by="nemotron_runbook_generator", ) entry = await ks.create_entry(entry_data) logger.info( "anti_pattern_created", incident_id=incident.incident_id, entry_id=entry.id, symptoms_hash=symptoms_hash, ) await self._push_anti_pattern_notification(incident, title) except Exception as e: logger.error( "anti_pattern_generation_failed", incident_id=incident.incident_id, error=str(e), ) # ========================================================================= # Private # ========================================================================= async def _execute_generation( self, *, prompt: str, incident: Incident, playbook: Playbook, generation_kind: str, ) -> str | None: """Execute one background generation through the global provider chain.""" from src.services.ai_provider_policy import PRODUCTION_PROVIDER_ORDER from src.services.ai_router import get_ai_executor incident_id = getattr(incident, "incident_id", "unknown") playbook_id = getattr(playbook, "playbook_id", "unknown") context = { "task_type": "runbook_generation", "intent_hint": "maintenance", "enforce_ollama_first": True, "alert_requires_ollama_before_cloud": True, "trace_id": _correlation_id("incident", incident_id), "run_id": _correlation_id( "runbook", f"{incident_id}:{generation_kind}", ), "work_item_id": _correlation_id("playbook", playbook_id), } result = await get_ai_executor().execute( prompt=prompt, provider_order=list(PRODUCTION_PROVIDER_ORDER), context=context, cache_ttl=3600, ) if result.success: content = _generated_document(result.raw_response) if content: logger.info( "runbook_ai_generation_ok", generation_kind=generation_kind, provider=result.provider, route=list(PRODUCTION_PROVIDER_ORDER), tokens=result.tokens, cost_usd=result.cost_usd, ) return content error_code = str(result.error or "invalid_provider_document").split(":", 1)[0][ :80 ] logger.warning( "runbook_ai_generation_blocked", generation_kind=generation_kind, provider=result.provider, error_code=error_code, route=list(PRODUCTION_PROVIDER_ORDER), ) return None async def _generate_runbook_content( self, incident: Incident, playbook: Playbook, result: AutoRepairResult, ) -> str: """透過唯一 production chain 生成 9 段 Runbook。""" prompt = ( f"[SYSTEM]{self._RUNBOOK_SYSTEM}\n\n" f"## Incident 資訊\n" f"- ID: {incident.incident_id}\n" f"- 受影響服務: {', '.join(incident.affected_services or [])}\n" f"- 嚴重度: {incident.severity.value if incident.severity else 'unknown'}\n\n" f"## 執行的 Playbook\n" f"- 名稱: {playbook.name}\n" f"- 執行步驟:\n" + "\n".join(f" {s}" for s in result.executed_steps[:5]) + f"\n\n## 執行結果\n- 狀態: 成功,耗時 {result.execution_time_ms}ms\n\n" "請生成包含以下 9 段的 Runbook(Markdown 格式):\n" "1. ## 症狀描述\n2. ## 根因分析\n3. ## 執行步驟\n" "4. ## 驗證步驟\n5. ## 注意事項\n6. ## 影響範圍\n" "7. ## 相關 Incident\n8. ## 下次預防建議\n9. ## 適用條件\n\n" '只回傳 JSON:{"content":"完整 Markdown Runbook"}' ) try: content = await self._execute_generation( prompt=prompt, incident=incident, playbook=playbook, generation_kind="runbook", ) if content: return content except Exception as e: logger.warning( "runbook_ai_route_failed", error_type=type(e).__name__, ) # Paid fallback blocked / all free lanes unavailable: no write to a # provider is retried outside the governed chain. return self._build_minimal_runbook(incident, playbook, result) async def _generate_anti_pattern_content( self, incident: Incident, playbook: Playbook, result: AutoRepairResult, ) -> str: """透過唯一 production chain 生成失敗案例。""" prompt = ( f"[SYSTEM]{self._ANTI_PATTERN_SYSTEM}\n\n" f"## Incident 資訊\n" f"- ID: {incident.incident_id}\n" f"- 受影響服務: {', '.join(incident.affected_services or [])}\n\n" f"## 嘗試的 Playbook\n- 名稱: {playbook.name}\n\n" f"## 失敗原因\n{result.error or '執行中發生未知異常'}\n\n" "請生成失敗案例文件(Markdown 格式),包含:\n" "## 症狀描述\n## 嘗試的修復方案\n## 失敗原因分析\n" "## 已知不適用條件\n## 替代方案建議\n\n" '只回傳 JSON:{"content":"完整 Markdown Anti-Pattern"}' ) try: content = await self._execute_generation( prompt=prompt, incident=incident, playbook=playbook, generation_kind="anti_pattern", ) if content: return content except Exception as e: logger.warning( "anti_pattern_ai_route_failed", error_type=type(e).__name__, ) return self._build_minimal_anti_pattern(incident, playbook, result) def _build_minimal_runbook( self, incident: Incident, playbook: Playbook, result: AutoRepairResult, ) -> str: """AI production chain 不可用時的基本 Runbook fallback。""" steps = "\n".join(f"- {s}" for s in result.executed_steps) return ( f"## 症狀描述\nIncident {incident.incident_id}," f"受影響服務:{', '.join(incident.affected_services or [])}\n\n" f"## 執行步驟\n{steps}\n\n" f"## 執行結果\n成功,耗時 {result.execution_time_ms}ms\n\n" "*本文件由系統自動生成(AI route fallback),建議透過 controlled review 補充。*" ) def _build_minimal_anti_pattern( self, incident: Incident, playbook: Playbook, result: AutoRepairResult, ) -> str: """AI production chain 不可用時的基本 Anti-Pattern fallback。""" return ( f"## 症狀描述\nIncident {incident.incident_id}," f"受影響服務:{', '.join(incident.affected_services or [])}\n\n" f"## 失敗原因\n{result.error or '執行中發生異常'}\n\n" f"## 已知不適用條件\nPlaybook '{playbook.name}' 在此症狀下失敗,請勿自動重試。\n\n" "*本文件由系統自動生成(AI route fallback)。*" ) async def _push_runbook_review_card( self, incident: Incident, entry_id: str, content_preview: str, ) -> None: """推送 Runbook 審核 card 到 Telegram""" try: from src.services.telegram_gateway import get_telegram_gateway tg = get_telegram_gateway() await tg.send_text( format_runbook_review_card(incident, entry_id, content_preview), product_id="awoooi", signal_family="incident_lifecycle", severity="P1", ) except Exception as e: logger.warning("runbook_review_card_failed", error=str(e)) async def _push_anti_pattern_notification( self, incident: Incident, title: str, ) -> None: """推送 Anti-Pattern 已記錄通知到 Telegram""" try: from src.services.telegram_gateway import get_telegram_gateway tg = get_telegram_gateway() await tg.send_text( f"⚠️ <b>已記錄失敗案例</b>\n" f"Incident: <code>{incident.incident_id}</code>\n" f"標題: {title}\n\n" f"相同症狀的後續告警將阻斷自動修復,要求人工介入。", product_id="awoooi", signal_family="incident_lifecycle", severity="P1", ) except Exception as e: logger.warning("anti_pattern_notification_failed", error=str(e)) # ============================================================================= # 單例管理 # ============================================================================= _generator: NemotronRunbookGenerator | None = None def get_runbook_generator() -> NemotronRunbookGenerator: global _generator if _generator is None: _generator = NemotronRunbookGenerator() return _generator