""" Runbook Generator - Phase 25 P1 Knowledge Auto-Harvesting ========================================================== 修復後自動生成 Runbook(成功)或 Anti-Pattern(失敗) 透過唯一 production AI chain 生成,結果沉澱至 KM 知識庫 設計原則: - 非阻塞:asyncio.create_task() 呼叫,絕不影響 AutoRepair 主流程 - 失敗靜默:生成失敗只記 log,不拋例外 - DRAFT/PUBLISHED:成功 → DRAFT(AI controlled review queue),失敗 → PUBLISHED(直接封鎖) 版本: v2.0 建立: 2026-04-04 (台北時區) 建立者: ogt (首席架構師設計) + Claude Code (實作) 關聯設計: docs/superpowers/specs/2026-04-04-nemotron-active-defense-design.md 方向一 變更紀錄: | 版本 | 日期 | 執行者 | 變更內容 | |------|------|--------|----------| | v1.0 | 2026-04-04 | Claude Code | 初始佔位(使用 generate() 但介面不存在) | | v1.1 | 2026-04-04 | ogt (首席架構師) | 接入舊版 cloud provider 介面;新增 Minimal fallback(已取代) | | v2.0 | 2026-07-14 | Codex | 移除 direct NVIDIA;統一 GCP-A → GCP-B → 111 → Gemini 與原子費控 receipt | """ from __future__ import annotations import html import json import re from typing import TYPE_CHECKING import structlog from src.models.knowledge import ( EntrySource, EntryStatus, EntryType, KnowledgeEntryCreate, ) if TYPE_CHECKING: from src.models.incident import Incident from src.models.playbook import Playbook from src.services.auto_repair_service import AutoRepairResult logger = structlog.get_logger(__name__) _CARD_MAX_LEN = 3600 _SECTION_RE = re.compile(r"^#{1,6}\s+(?P
{_html(incident_id)}\n"
f"🧩 受影響服務:{_html(services)}\n"
"🧠 知識狀態:DRAFT|AI controlled review\n"
f"🗂️ Entry ID:{_html(entry_id)}\n\n"
"🧾 內容摘要\n"
f"├ 症狀:{_html(symptom)}\n"
f"└ 執行:{_html(step)}\n\n"
"✅ 審核重點\n"
"1. 確認步驟可重跑,且不含 placeholder / 不支援 scheme\n"
"2. 補齊適用條件、rollback 與驗證方式\n\n"
"🔎 AwoooP:知識庫 / Runbook Review"
)
return message[:_CARD_MAX_LEN]
class NemotronRunbookGenerator:
"""
Runbook 自動生成器。
類名只為舊 import 相容;production 執行不再呼叫 Nemotron/NVIDIA。
職責:
- 成功修復 → AUTO_RUNBOOK (DRAFT) + Telegram 審核 card
- 失敗修復 → ANTI_PATTERN (PUBLISHED) + Telegram 通知
leWOOOgo 積木化:
- 呼叫 KnowledgeService(不直接存 DB)
- 呼叫 AIRouterExecutor,唯一順序 GCP-A → GCP-B → host111 → Gemini
- Gemini 只能透過 provider 內的 durable disable-state 與原子費控 receipt
"""
_RUNBOOK_SYSTEM = (
"你是 AWOOOI 平台的 SRE Runbook 撰寫專家。"
"根據提供的 Incident 與修復結果,用繁體中文生成完整結構化 Runbook。"
)
_ANTI_PATTERN_SYSTEM = (
"你是 AWOOOI 平台的故障分析專家。"
"根據失敗的修復嘗試,用繁體中文生成失敗案例記錄,幫助未來避免重蹈覆轍。"
)
async def generate_runbook(
self,
incident: Incident,
playbook: Playbook,
result: AutoRepairResult,
symptoms_hash: str,
) -> None:
"""
成功修復後生成 AUTO_RUNBOOK(fire-and-forget,呼叫方不等待)
Args:
incident: 觸發的 Incident
playbook: 執行的 Playbook
result: 執行結果(success=True)
symptoms_hash: SymptomPattern.compute_hash() 的 hash
"""
try:
content = await self._generate_runbook_content(incident, playbook, result)
if not content:
return
from src.services.knowledge_service import get_knowledge_service
ks = get_knowledge_service()
entry_data = KnowledgeEntryCreate(
title=f"[AUTO] {incident.incident_id} — {playbook.name}",
content=content,
entry_type=EntryType.AUTO_RUNBOOK,
category="ai_system",
tags=list(incident.affected_services or []) + ["auto_runbook", "ai_route"],
source=EntrySource.AI_EXTRACTED,
status=EntryStatus.DRAFT,
related_incident_id=incident.incident_id,
related_playbook_id=playbook.playbook_id,
symptoms_hash=symptoms_hash,
created_by="ai_route_runbook_generator",
)
entry = await ks.create_entry(entry_data)
logger.info(
"auto_runbook_created",
incident_id=incident.incident_id,
entry_id=entry.id,
playbook_id=playbook.playbook_id,
)
await self._push_runbook_review_card(incident, entry.id, content)
except Exception as e:
logger.error(
"runbook_generation_failed",
incident_id=incident.incident_id,
error=str(e),
)
async def generate_anti_pattern(
self,
incident: Incident,
playbook: Playbook,
result: AutoRepairResult,
symptoms_hash: str,
) -> None:
"""
失敗修復後生成 ANTI_PATTERN(fire-and-forget,直接 PUBLISHED)
Args:
incident: 觸發的 Incident
playbook: 嘗試執行的 Playbook
result: 執行結果(success=False)
symptoms_hash: SymptomPattern.compute_hash() 的 hash
"""
try:
content = await self._generate_anti_pattern_content(incident, playbook, result)
if not content:
return
from src.services.knowledge_service import get_knowledge_service
ks = get_knowledge_service()
title = f"[FAIL] {incident.incident_id} — {playbook.name}"
entry_data = KnowledgeEntryCreate(
title=title,
content=content,
entry_type=EntryType.ANTI_PATTERN,
category="failure_cases",
tags=list(incident.affected_services or []) + ["anti_pattern", "failure"],
source=EntrySource.AI_EXTRACTED,
status=EntryStatus.PUBLISHED, # 直接發布,無需審核
related_incident_id=incident.incident_id,
related_playbook_id=playbook.playbook_id,
symptoms_hash=symptoms_hash,
created_by="nemotron_runbook_generator",
)
entry = await ks.create_entry(entry_data)
logger.info(
"anti_pattern_created",
incident_id=incident.incident_id,
entry_id=entry.id,
symptoms_hash=symptoms_hash,
)
await self._push_anti_pattern_notification(incident, title)
except Exception as e:
logger.error(
"anti_pattern_generation_failed",
incident_id=incident.incident_id,
error=str(e),
)
# =========================================================================
# Private
# =========================================================================
async def _execute_generation(
self,
*,
prompt: str,
incident: Incident,
playbook: Playbook,
generation_kind: str,
) -> str | None:
"""Execute one background generation through the global provider chain."""
from src.services.ai_provider_policy import PRODUCTION_PROVIDER_ORDER
from src.services.ai_router import get_ai_executor
incident_id = getattr(incident, "incident_id", "unknown")
playbook_id = getattr(playbook, "playbook_id", "unknown")
context = {
"task_type": "runbook_generation",
"intent_hint": "maintenance",
"enforce_ollama_first": True,
"alert_requires_ollama_before_cloud": True,
"trace_id": _correlation_id("incident", incident_id),
"run_id": _correlation_id(
"runbook",
f"{incident_id}:{generation_kind}",
),
"work_item_id": _correlation_id("playbook", playbook_id),
}
result = await get_ai_executor().execute(
prompt=prompt,
provider_order=list(PRODUCTION_PROVIDER_ORDER),
context=context,
cache_ttl=3600,
)
if result.success:
content = _generated_document(result.raw_response)
if content:
logger.info(
"runbook_ai_generation_ok",
generation_kind=generation_kind,
provider=result.provider,
route=list(PRODUCTION_PROVIDER_ORDER),
tokens=result.tokens,
cost_usd=result.cost_usd,
)
return content
error_code = str(result.error or "invalid_provider_document").split(":", 1)[0][
:80
]
logger.warning(
"runbook_ai_generation_blocked",
generation_kind=generation_kind,
provider=result.provider,
error_code=error_code,
route=list(PRODUCTION_PROVIDER_ORDER),
)
return None
async def _generate_runbook_content(
self,
incident: Incident,
playbook: Playbook,
result: AutoRepairResult,
) -> str:
"""透過唯一 production chain 生成 9 段 Runbook。"""
prompt = (
f"[SYSTEM]{self._RUNBOOK_SYSTEM}\n\n"
f"## Incident 資訊\n"
f"- ID: {incident.incident_id}\n"
f"- 受影響服務: {', '.join(incident.affected_services or [])}\n"
f"- 嚴重度: {incident.severity.value if incident.severity else 'unknown'}\n\n"
f"## 執行的 Playbook\n"
f"- 名稱: {playbook.name}\n"
f"- 執行步驟:\n"
+ "\n".join(f" {s}" for s in result.executed_steps[:5])
+ f"\n\n## 執行結果\n- 狀態: 成功,耗時 {result.execution_time_ms}ms\n\n"
"請生成包含以下 9 段的 Runbook(Markdown 格式):\n"
"1. ## 症狀描述\n2. ## 根因分析\n3. ## 執行步驟\n"
"4. ## 驗證步驟\n5. ## 注意事項\n6. ## 影響範圍\n"
"7. ## 相關 Incident\n8. ## 下次預防建議\n9. ## 適用條件\n\n"
'只回傳 JSON:{"content":"完整 Markdown Runbook"}'
)
try:
content = await self._execute_generation(
prompt=prompt,
incident=incident,
playbook=playbook,
generation_kind="runbook",
)
if content:
return content
except Exception as e:
logger.warning(
"runbook_ai_route_failed",
error_type=type(e).__name__,
)
# Paid fallback blocked / all free lanes unavailable: no write to a
# provider is retried outside the governed chain.
return self._build_minimal_runbook(incident, playbook, result)
async def _generate_anti_pattern_content(
self,
incident: Incident,
playbook: Playbook,
result: AutoRepairResult,
) -> str:
"""透過唯一 production chain 生成失敗案例。"""
prompt = (
f"[SYSTEM]{self._ANTI_PATTERN_SYSTEM}\n\n"
f"## Incident 資訊\n"
f"- ID: {incident.incident_id}\n"
f"- 受影響服務: {', '.join(incident.affected_services or [])}\n\n"
f"## 嘗試的 Playbook\n- 名稱: {playbook.name}\n\n"
f"## 失敗原因\n{result.error or '執行中發生未知異常'}\n\n"
"請生成失敗案例文件(Markdown 格式),包含:\n"
"## 症狀描述\n## 嘗試的修復方案\n## 失敗原因分析\n"
"## 已知不適用條件\n## 替代方案建議\n\n"
'只回傳 JSON:{"content":"完整 Markdown Anti-Pattern"}'
)
try:
content = await self._execute_generation(
prompt=prompt,
incident=incident,
playbook=playbook,
generation_kind="anti_pattern",
)
if content:
return content
except Exception as e:
logger.warning(
"anti_pattern_ai_route_failed",
error_type=type(e).__name__,
)
return self._build_minimal_anti_pattern(incident, playbook, result)
def _build_minimal_runbook(
self,
incident: Incident,
playbook: Playbook,
result: AutoRepairResult,
) -> str:
"""AI production chain 不可用時的基本 Runbook fallback。"""
steps = "\n".join(f"- {s}" for s in result.executed_steps)
return (
f"## 症狀描述\nIncident {incident.incident_id},"
f"受影響服務:{', '.join(incident.affected_services or [])}\n\n"
f"## 執行步驟\n{steps}\n\n"
f"## 執行結果\n成功,耗時 {result.execution_time_ms}ms\n\n"
"*本文件由系統自動生成(AI route fallback),建議透過 controlled review 補充。*"
)
def _build_minimal_anti_pattern(
self,
incident: Incident,
playbook: Playbook,
result: AutoRepairResult,
) -> str:
"""AI production chain 不可用時的基本 Anti-Pattern fallback。"""
return (
f"## 症狀描述\nIncident {incident.incident_id},"
f"受影響服務:{', '.join(incident.affected_services or [])}\n\n"
f"## 失敗原因\n{result.error or '執行中發生異常'}\n\n"
f"## 已知不適用條件\nPlaybook '{playbook.name}' 在此症狀下失敗,請勿自動重試。\n\n"
"*本文件由系統自動生成(AI route fallback)。*"
)
async def _push_runbook_review_card(
self,
incident: Incident,
entry_id: str,
content_preview: str,
) -> None:
"""推送 Runbook 審核 card 到 Telegram"""
try:
from src.services.telegram_gateway import get_telegram_gateway
tg = get_telegram_gateway()
await tg.send_text(
format_runbook_review_card(incident, entry_id, content_preview),
product_id="awoooi",
signal_family="incident_lifecycle",
severity="P1",
)
except Exception as e:
logger.warning("runbook_review_card_failed", error=str(e))
async def _push_anti_pattern_notification(
self,
incident: Incident,
title: str,
) -> None:
"""推送 Anti-Pattern 已記錄通知到 Telegram"""
try:
from src.services.telegram_gateway import get_telegram_gateway
tg = get_telegram_gateway()
await tg.send_text(
f"⚠️ 已記錄失敗案例\n"
f"Incident: {incident.incident_id}\n"
f"標題: {title}\n\n"
f"相同症狀的後續告警將阻斷自動修復,要求人工介入。",
product_id="awoooi",
signal_family="incident_lifecycle",
severity="P1",
)
except Exception as e:
logger.warning("anti_pattern_notification_failed", error=str(e))
# =============================================================================
# 單例管理
# =============================================================================
_generator: NemotronRunbookGenerator | None = None
def get_runbook_generator() -> NemotronRunbookGenerator:
global _generator
if _generator is None:
_generator = NemotronRunbookGenerator()
return _generator