533 lines
20 KiB
Python
533 lines
20 KiB
Python
"""
|
||
Runbook Generator - Phase 25 P1 Knowledge Auto-Harvesting
|
||
==========================================================
|
||
修復後自動生成 Runbook（成功）或 Anti-Pattern（失敗）
|
||
透過唯一 production AI chain 生成，結果沉澱至 KM 知識庫
|
||
|
||
設計原則:
|
||
- 非阻塞：asyncio.create_task() 呼叫，絕不影響 AutoRepair 主流程
|
||
- 失敗靜默：生成失敗只記 log，不拋例外
|
||
- DRAFT/PUBLISHED：成功 → DRAFT（AI controlled review queue），失敗 → PUBLISHED（直接封鎖）
|
||
|
||
版本: v2.0
|
||
建立: 2026-04-04 (台北時區)
|
||
建立者: ogt (首席架構師設計) + Claude Code (實作)
|
||
關聯設計: docs/superpowers/specs/2026-04-04-nemotron-active-defense-design.md 方向一
|
||
|
||
變更紀錄:
|
||
| 版本 | 日期 | 執行者 | 變更內容 |
|
||
|------|------|--------|----------|
|
||
| v1.0 | 2026-04-04 | Claude Code | 初始佔位（使用 generate() 但介面不存在） |
|
||
| v1.1 | 2026-04-04 | ogt (首席架構師) | 接入舊版 cloud provider 介面；新增 Minimal fallback（已取代） |
|
||
| v2.0 | 2026-07-14 | Codex | 移除 direct NVIDIA；統一 GCP-A → GCP-B → 111 → Gemini 與原子費控 receipt |
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import html
|
||
import json
|
||
import re
|
||
from typing import TYPE_CHECKING
|
||
|
||
import structlog
|
||
|
||
from src.models.knowledge import (
|
||
EntrySource,
|
||
EntryStatus,
|
||
EntryType,
|
||
KnowledgeEntryCreate,
|
||
)
|
||
|
||
if TYPE_CHECKING:
|
||
from src.models.incident import Incident
|
||
from src.models.playbook import Playbook
|
||
from src.services.auto_repair_service import AutoRepairResult
|
||
|
||
logger = structlog.get_logger(__name__)
|
||
|
||
_CARD_MAX_LEN = 3600
|
||
_SECTION_RE = re.compile(r"^#{1,6}\s+(?P<title>.+?)\s*$")
|
||
_BULLET_RE = re.compile(r"^\s*(?:[-*]|\d+[.)])\s*")
|
||
_CORRELATION_SANITIZE_RE = re.compile(r"[^A-Za-z0-9._:/-]+")
|
||
|
||
|
||
def _html(text: object) -> str:
|
||
return html.escape(str(text), quote=False)
|
||
|
||
|
||
def _shorten(text: object, limit: int = 120) -> str:
|
||
compact = " ".join(str(text or "").split())
|
||
if len(compact) <= limit:
|
||
return compact
|
||
return compact[: max(0, limit - 1)].rstrip() + "…"
|
||
|
||
|
||
def _clean_preview_line(line: str) -> str:
|
||
line = _SECTION_RE.sub("", line.strip())
|
||
line = _BULLET_RE.sub("", line).strip()
|
||
line = line.replace("`", "")
|
||
return " ".join(line.split())
|
||
|
||
|
||
def _section_preview(content: str, title_keyword: str, *, fallback: str) -> str:
|
||
"""從 Markdown 內容抽一行可讀摘要，避免把整段 Runbook 原文丟進 Telegram。"""
|
||
lines = str(content or "").splitlines()
|
||
in_section = False
|
||
for raw_line in lines:
|
||
line = raw_line.strip()
|
||
if not line:
|
||
continue
|
||
heading = _SECTION_RE.match(line)
|
||
if heading:
|
||
in_section = title_keyword in heading.group("title")
|
||
continue
|
||
if not in_section:
|
||
continue
|
||
preview = _clean_preview_line(line)
|
||
if preview:
|
||
return _shorten(preview, 120)
|
||
return fallback
|
||
|
||
|
||
def _step_preview(content: str) -> str:
|
||
preview = _section_preview(content, "執行", fallback="待 controlled review 的 Runbook 執行步驟")
|
||
if any(token in preview for token in ("{host}", "{target}", "Unsupported scheme", "Invalid component name")):
|
||
return "含 placeholder 或不支援的執行步驟，需 AI 修補 / break-glass 後才能發布"
|
||
return _shorten(preview, 120)
|
||
|
||
|
||
def _correlation_id(prefix: str, value: object) -> str:
|
||
"""Build a bounded receipt-safe identifier without copying arbitrary text."""
|
||
candidate = f"{prefix}:{value}"
|
||
candidate = _CORRELATION_SANITIZE_RE.sub("-", candidate).strip("-./:")
|
||
if not candidate or not candidate[0].isalnum():
|
||
candidate = prefix
|
||
return candidate[:160]
|
||
|
||
|
||
def _generated_document(raw_response: str) -> str | None:
|
||
"""Extract the allowlisted document field from JSON-mode provider output."""
|
||
raw = str(raw_response or "").strip()
|
||
if not raw:
|
||
return None
|
||
if raw.startswith("```"):
|
||
lines = raw.splitlines()
|
||
if lines and lines[0].strip().lower() in {"```", "```json"}:
|
||
lines = lines[1:]
|
||
if lines and lines[-1].strip() == "```":
|
||
lines = lines[:-1]
|
||
raw = "\n".join(lines).strip()
|
||
|
||
try:
|
||
payload = json.loads(raw)
|
||
except json.JSONDecodeError:
|
||
# Compatibility with an Ollama model that ignored JSON mode. Only a
|
||
# structured Markdown document is accepted; arbitrary prose is not.
|
||
return raw if raw.startswith("## ") else None
|
||
|
||
if isinstance(payload, str):
|
||
return payload.strip() or None
|
||
if not isinstance(payload, dict):
|
||
return None
|
||
content = payload.get("content")
|
||
return content.strip() if isinstance(content, str) and content.strip() else None
|
||
|
||
|
||
def format_runbook_review_card(
|
||
incident: object,
|
||
entry_id: str,
|
||
content: str,
|
||
) -> str:
|
||
"""格式化 Telegram Runbook 審核卡片。
|
||
|
||
2026-05-07 Codex — 將純文字 Markdown preview 改成治理卡片，讓 SRE
|
||
能快速判斷知識狀態、受影響服務與審核重點。
|
||
"""
|
||
incident_id = getattr(incident, "incident_id", "unknown")
|
||
services = ", ".join(getattr(incident, "affected_services", None) or []) or "unknown"
|
||
symptom = _section_preview(content, "症狀", fallback=f"Incident {incident_id} 的修復知識待審核")
|
||
step = _step_preview(content)
|
||
|
||
message = (
|
||
"📄 <b>RUNBOOK REVIEW｜待審核</b>\n"
|
||
"──────────────────────\n"
|
||
f"📋 Incident：<code>{_html(incident_id)}</code>\n"
|
||
f"🧩 受影響服務：<code>{_html(services)}</code>\n"
|
||
"🧠 知識狀態：<b>DRAFT｜AI controlled review</b>\n"
|
||
f"🗂️ Entry ID：<code>{_html(entry_id)}</code>\n\n"
|
||
"🧾 <b>內容摘要</b>\n"
|
||
f"├ 症狀：{_html(symptom)}\n"
|
||
f"└ 執行：{_html(step)}\n\n"
|
||
"✅ <b>審核重點</b>\n"
|
||
"1. 確認步驟可重跑，且不含 placeholder / 不支援 scheme\n"
|
||
"2. 補齊適用條件、rollback 與驗證方式\n\n"
|
||
"🔎 AwoooP：知識庫 / Runbook Review"
|
||
)
|
||
return message[:_CARD_MAX_LEN]
|
||
|
||
|
||
class NemotronRunbookGenerator:
|
||
"""
|
||
Runbook 自動生成器。
|
||
|
||
類名只為舊 import 相容；production 執行不再呼叫 Nemotron/NVIDIA。
|
||
|
||
職責：
|
||
- 成功修復 → AUTO_RUNBOOK (DRAFT) + Telegram 審核 card
|
||
- 失敗修復 → ANTI_PATTERN (PUBLISHED) + Telegram 通知
|
||
|
||
leWOOOgo 積木化：
|
||
- 呼叫 KnowledgeService（不直接存 DB）
|
||
- 呼叫 AIRouterExecutor，唯一順序 GCP-A → GCP-B → host111 → Gemini
|
||
- Gemini 只能透過 provider 內的 durable disable-state 與原子費控 receipt
|
||
"""
|
||
|
||
_RUNBOOK_SYSTEM = (
|
||
"你是 AWOOOI 平台的 SRE Runbook 撰寫專家。"
|
||
"根據提供的 Incident 與修復結果，用繁體中文生成完整結構化 Runbook。"
|
||
)
|
||
|
||
_ANTI_PATTERN_SYSTEM = (
|
||
"你是 AWOOOI 平台的故障分析專家。"
|
||
"根據失敗的修復嘗試，用繁體中文生成失敗案例記錄，幫助未來避免重蹈覆轍。"
|
||
)
|
||
|
||
async def generate_runbook(
|
||
self,
|
||
incident: Incident,
|
||
playbook: Playbook,
|
||
result: AutoRepairResult,
|
||
symptoms_hash: str,
|
||
) -> None:
|
||
"""
|
||
成功修復後生成 AUTO_RUNBOOK（fire-and-forget，呼叫方不等待）
|
||
|
||
Args:
|
||
incident: 觸發的 Incident
|
||
playbook: 執行的 Playbook
|
||
result: 執行結果（success=True）
|
||
symptoms_hash: SymptomPattern.compute_hash() 的 hash
|
||
"""
|
||
try:
|
||
content = await self._generate_runbook_content(incident, playbook, result)
|
||
if not content:
|
||
return
|
||
|
||
from src.services.knowledge_service import get_knowledge_service
|
||
ks = get_knowledge_service()
|
||
|
||
entry_data = KnowledgeEntryCreate(
|
||
title=f"[AUTO] {incident.incident_id} — {playbook.name}",
|
||
content=content,
|
||
entry_type=EntryType.AUTO_RUNBOOK,
|
||
category="ai_system",
|
||
tags=list(incident.affected_services or []) + ["auto_runbook", "ai_route"],
|
||
source=EntrySource.AI_EXTRACTED,
|
||
status=EntryStatus.DRAFT,
|
||
related_incident_id=incident.incident_id,
|
||
related_playbook_id=playbook.playbook_id,
|
||
symptoms_hash=symptoms_hash,
|
||
created_by="ai_route_runbook_generator",
|
||
)
|
||
|
||
entry = await ks.create_entry(entry_data)
|
||
|
||
logger.info(
|
||
"auto_runbook_created",
|
||
incident_id=incident.incident_id,
|
||
entry_id=entry.id,
|
||
playbook_id=playbook.playbook_id,
|
||
)
|
||
|
||
await self._push_runbook_review_card(incident, entry.id, content)
|
||
|
||
except Exception as e:
|
||
logger.error(
|
||
"runbook_generation_failed",
|
||
incident_id=incident.incident_id,
|
||
error=str(e),
|
||
)
|
||
|
||
async def generate_anti_pattern(
|
||
self,
|
||
incident: Incident,
|
||
playbook: Playbook,
|
||
result: AutoRepairResult,
|
||
symptoms_hash: str,
|
||
) -> None:
|
||
"""
|
||
失敗修復後生成 ANTI_PATTERN（fire-and-forget，直接 PUBLISHED）
|
||
|
||
Args:
|
||
incident: 觸發的 Incident
|
||
playbook: 嘗試執行的 Playbook
|
||
result: 執行結果（success=False）
|
||
symptoms_hash: SymptomPattern.compute_hash() 的 hash
|
||
"""
|
||
try:
|
||
content = await self._generate_anti_pattern_content(incident, playbook, result)
|
||
if not content:
|
||
return
|
||
|
||
from src.services.knowledge_service import get_knowledge_service
|
||
ks = get_knowledge_service()
|
||
|
||
title = f"[FAIL] {incident.incident_id} — {playbook.name}"
|
||
entry_data = KnowledgeEntryCreate(
|
||
title=title,
|
||
content=content,
|
||
entry_type=EntryType.ANTI_PATTERN,
|
||
category="failure_cases",
|
||
tags=list(incident.affected_services or []) + ["anti_pattern", "failure"],
|
||
source=EntrySource.AI_EXTRACTED,
|
||
status=EntryStatus.PUBLISHED, # 直接發布，無需審核
|
||
related_incident_id=incident.incident_id,
|
||
related_playbook_id=playbook.playbook_id,
|
||
symptoms_hash=symptoms_hash,
|
||
created_by="nemotron_runbook_generator",
|
||
)
|
||
|
||
entry = await ks.create_entry(entry_data)
|
||
|
||
logger.info(
|
||
"anti_pattern_created",
|
||
incident_id=incident.incident_id,
|
||
entry_id=entry.id,
|
||
symptoms_hash=symptoms_hash,
|
||
)
|
||
|
||
await self._push_anti_pattern_notification(incident, title)
|
||
|
||
except Exception as e:
|
||
logger.error(
|
||
"anti_pattern_generation_failed",
|
||
incident_id=incident.incident_id,
|
||
error=str(e),
|
||
)
|
||
|
||
# =========================================================================
|
||
# Private
|
||
# =========================================================================
|
||
|
||
async def _execute_generation(
|
||
self,
|
||
*,
|
||
prompt: str,
|
||
incident: Incident,
|
||
playbook: Playbook,
|
||
generation_kind: str,
|
||
) -> str | None:
|
||
"""Execute one background generation through the global provider chain."""
|
||
from src.services.ai_provider_policy import PRODUCTION_PROVIDER_ORDER
|
||
from src.services.ai_router import get_ai_executor
|
||
|
||
incident_id = getattr(incident, "incident_id", "unknown")
|
||
playbook_id = getattr(playbook, "playbook_id", "unknown")
|
||
context = {
|
||
"task_type": "runbook_generation",
|
||
"intent_hint": "maintenance",
|
||
"enforce_ollama_first": True,
|
||
"alert_requires_ollama_before_cloud": True,
|
||
"trace_id": _correlation_id("incident", incident_id),
|
||
"run_id": _correlation_id(
|
||
"runbook",
|
||
f"{incident_id}:{generation_kind}",
|
||
),
|
||
"work_item_id": _correlation_id("playbook", playbook_id),
|
||
}
|
||
result = await get_ai_executor().execute(
|
||
prompt=prompt,
|
||
provider_order=list(PRODUCTION_PROVIDER_ORDER),
|
||
context=context,
|
||
cache_ttl=3600,
|
||
)
|
||
if result.success:
|
||
content = _generated_document(result.raw_response)
|
||
if content:
|
||
logger.info(
|
||
"runbook_ai_generation_ok",
|
||
generation_kind=generation_kind,
|
||
provider=result.provider,
|
||
route=list(PRODUCTION_PROVIDER_ORDER),
|
||
tokens=result.tokens,
|
||
cost_usd=result.cost_usd,
|
||
)
|
||
return content
|
||
|
||
error_code = str(result.error or "invalid_provider_document").split(":", 1)[0][
|
||
:80
|
||
]
|
||
logger.warning(
|
||
"runbook_ai_generation_blocked",
|
||
generation_kind=generation_kind,
|
||
provider=result.provider,
|
||
error_code=error_code,
|
||
route=list(PRODUCTION_PROVIDER_ORDER),
|
||
)
|
||
return None
|
||
|
||
async def _generate_runbook_content(
|
||
self,
|
||
incident: Incident,
|
||
playbook: Playbook,
|
||
result: AutoRepairResult,
|
||
) -> str:
|
||
"""透過唯一 production chain 生成 9 段 Runbook。"""
|
||
prompt = (
|
||
f"[SYSTEM]{self._RUNBOOK_SYSTEM}\n\n"
|
||
f"## Incident 資訊\n"
|
||
f"- ID: {incident.incident_id}\n"
|
||
f"- 受影響服務: {', '.join(incident.affected_services or [])}\n"
|
||
f"- 嚴重度: {incident.severity.value if incident.severity else 'unknown'}\n\n"
|
||
f"## 執行的 Playbook\n"
|
||
f"- 名稱: {playbook.name}\n"
|
||
f"- 執行步驟:\n"
|
||
+ "\n".join(f" {s}" for s in result.executed_steps[:5])
|
||
+ f"\n\n## 執行結果\n- 狀態: 成功，耗時 {result.execution_time_ms}ms\n\n"
|
||
"請生成包含以下 9 段的 Runbook（Markdown 格式）：\n"
|
||
"1. ## 症狀描述\n2. ## 根因分析\n3. ## 執行步驟\n"
|
||
"4. ## 驗證步驟\n5. ## 注意事項\n6. ## 影響範圍\n"
|
||
"7. ## 相關 Incident\n8. ## 下次預防建議\n9. ## 適用條件\n\n"
|
||
'只回傳 JSON：{"content":"完整 Markdown Runbook"}'
|
||
)
|
||
try:
|
||
content = await self._execute_generation(
|
||
prompt=prompt,
|
||
incident=incident,
|
||
playbook=playbook,
|
||
generation_kind="runbook",
|
||
)
|
||
if content:
|
||
return content
|
||
except Exception as e:
|
||
logger.warning(
|
||
"runbook_ai_route_failed",
|
||
error_type=type(e).__name__,
|
||
)
|
||
|
||
# Paid fallback blocked / all free lanes unavailable: no write to a
|
||
# provider is retried outside the governed chain.
|
||
return self._build_minimal_runbook(incident, playbook, result)
|
||
|
||
async def _generate_anti_pattern_content(
|
||
self,
|
||
incident: Incident,
|
||
playbook: Playbook,
|
||
result: AutoRepairResult,
|
||
) -> str:
|
||
"""透過唯一 production chain 生成失敗案例。"""
|
||
prompt = (
|
||
f"[SYSTEM]{self._ANTI_PATTERN_SYSTEM}\n\n"
|
||
f"## Incident 資訊\n"
|
||
f"- ID: {incident.incident_id}\n"
|
||
f"- 受影響服務: {', '.join(incident.affected_services or [])}\n\n"
|
||
f"## 嘗試的 Playbook\n- 名稱: {playbook.name}\n\n"
|
||
f"## 失敗原因\n{result.error or '執行中發生未知異常'}\n\n"
|
||
"請生成失敗案例文件（Markdown 格式），包含：\n"
|
||
"## 症狀描述\n## 嘗試的修復方案\n## 失敗原因分析\n"
|
||
"## 已知不適用條件\n## 替代方案建議\n\n"
|
||
'只回傳 JSON：{"content":"完整 Markdown Anti-Pattern"}'
|
||
)
|
||
|
||
try:
|
||
content = await self._execute_generation(
|
||
prompt=prompt,
|
||
incident=incident,
|
||
playbook=playbook,
|
||
generation_kind="anti_pattern",
|
||
)
|
||
if content:
|
||
return content
|
||
except Exception as e:
|
||
logger.warning(
|
||
"anti_pattern_ai_route_failed",
|
||
error_type=type(e).__name__,
|
||
)
|
||
|
||
return self._build_minimal_anti_pattern(incident, playbook, result)
|
||
|
||
def _build_minimal_runbook(
|
||
self,
|
||
incident: Incident,
|
||
playbook: Playbook,
|
||
result: AutoRepairResult,
|
||
) -> str:
|
||
"""AI production chain 不可用時的基本 Runbook fallback。"""
|
||
steps = "\n".join(f"- {s}" for s in result.executed_steps)
|
||
return (
|
||
f"## 症狀描述\nIncident {incident.incident_id}，"
|
||
f"受影響服務：{', '.join(incident.affected_services or [])}\n\n"
|
||
f"## 執行步驟\n{steps}\n\n"
|
||
f"## 執行結果\n成功，耗時 {result.execution_time_ms}ms\n\n"
|
||
"*本文件由系統自動生成（AI route fallback），建議透過 controlled review 補充。*"
|
||
)
|
||
|
||
def _build_minimal_anti_pattern(
|
||
self,
|
||
incident: Incident,
|
||
playbook: Playbook,
|
||
result: AutoRepairResult,
|
||
) -> str:
|
||
"""AI production chain 不可用時的基本 Anti-Pattern fallback。"""
|
||
return (
|
||
f"## 症狀描述\nIncident {incident.incident_id}，"
|
||
f"受影響服務：{', '.join(incident.affected_services or [])}\n\n"
|
||
f"## 失敗原因\n{result.error or '執行中發生異常'}\n\n"
|
||
f"## 已知不適用條件\nPlaybook '{playbook.name}' 在此症狀下失敗，請勿自動重試。\n\n"
|
||
"*本文件由系統自動生成（AI route fallback）。*"
|
||
)
|
||
|
||
async def _push_runbook_review_card(
|
||
self,
|
||
incident: Incident,
|
||
entry_id: str,
|
||
content_preview: str,
|
||
) -> None:
|
||
"""推送 Runbook 審核 card 到 Telegram"""
|
||
try:
|
||
from src.services.telegram_gateway import get_telegram_gateway
|
||
tg = get_telegram_gateway()
|
||
await tg.send_text(
|
||
format_runbook_review_card(incident, entry_id, content_preview),
|
||
product_id="awoooi",
|
||
signal_family="incident_lifecycle",
|
||
severity="P1",
|
||
)
|
||
except Exception as e:
|
||
logger.warning("runbook_review_card_failed", error=str(e))
|
||
|
||
async def _push_anti_pattern_notification(
|
||
self,
|
||
incident: Incident,
|
||
title: str,
|
||
) -> None:
|
||
"""推送 Anti-Pattern 已記錄通知到 Telegram"""
|
||
try:
|
||
from src.services.telegram_gateway import get_telegram_gateway
|
||
tg = get_telegram_gateway()
|
||
await tg.send_text(
|
||
f"⚠️ <b>已記錄失敗案例</b>\n"
|
||
f"Incident: <code>{incident.incident_id}</code>\n"
|
||
f"標題: {title}\n\n"
|
||
f"相同症狀的後續告警將阻斷自動修復，要求人工介入。",
|
||
product_id="awoooi",
|
||
signal_family="incident_lifecycle",
|
||
severity="P1",
|
||
)
|
||
except Exception as e:
|
||
logger.warning("anti_pattern_notification_failed", error=str(e))
|
||
|
||
|
||
# =============================================================================
|
||
# 單例管理
|
||
# =============================================================================
|
||
|
||
_generator: NemotronRunbookGenerator | None = None
|
||
|
||
|
||
def get_runbook_generator() -> NemotronRunbookGenerator:
|
||
global _generator
|
||
if _generator is None:
|
||
_generator = NemotronRunbookGenerator()
|
||
return _generator
|