修復 P1 違規: - Line 29: Service 直接 import Redis → Repository Pattern - Line 156: 自建 httpx.AsyncClient → DI 注入 變更: - 新增 IEmbeddingCacheRepository Protocol (interfaces.py) - 新增 EmbeddingCacheRepository 實作 (embedding_repository.py) - PlaybookRAGService 改用 DI 注入 http_client + embedding_cache - get_playbook_rag_service() 改為 async factory - PlaybookService 改用 lazy initialization Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
639 lines
21 KiB
Python
639 lines
21 KiB
Python
"""
|
|
Playbook Service - #7 Playbook 萃取
|
|
===================================
|
|
Playbook 業務邏輯層
|
|
|
|
Phase 7.3: Service 實作
|
|
Phase 3 ADR-030: RAG 向量搜尋整合
|
|
建立時間: 2026-03-26 (台北時區)
|
|
建立者: Claude Code (#7 Playbook 萃取)
|
|
|
|
遵循 leWOOOgo 積木化原則:
|
|
- Service 層只依賴 Repository Interface
|
|
- 不直接存取 Redis/DB
|
|
- 封裝所有業務邏輯
|
|
"""
|
|
|
|
from typing import Protocol
|
|
|
|
import structlog
|
|
|
|
from src.models.incident import Incident, IncidentStatus
|
|
from src.models.playbook import (
|
|
ActionType,
|
|
Playbook,
|
|
PlaybookRecommendation,
|
|
PlaybookSource,
|
|
PlaybookStatus,
|
|
RepairStep,
|
|
RiskLevel,
|
|
SymptomPattern,
|
|
)
|
|
from src.repositories.interfaces import IPlaybookRepository
|
|
from src.repositories.playbook_repository import get_playbook_repository
|
|
from src.services.playbook_rag import get_playbook_rag_service
|
|
from src.utils.timezone import now_taipei
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
|
|
|
|
class IPlaybookService(Protocol):
|
|
"""Playbook Service Interface"""
|
|
|
|
async def extract_from_incident(
|
|
self,
|
|
incident: Incident,
|
|
auto_approve: bool = False,
|
|
) -> Playbook | None:
|
|
"""從成功案例萃取 Playbook"""
|
|
...
|
|
|
|
async def get_recommendations(
|
|
self,
|
|
symptoms: SymptomPattern,
|
|
top_k: int = 3,
|
|
) -> list[PlaybookRecommendation]:
|
|
"""取得 Playbook 推薦"""
|
|
...
|
|
|
|
async def approve(
|
|
self,
|
|
playbook_id: str,
|
|
approved_by: str,
|
|
notes: str | None = None,
|
|
) -> Playbook | None:
|
|
"""核准 Playbook"""
|
|
...
|
|
|
|
async def record_execution(
|
|
self,
|
|
playbook_id: str,
|
|
success: bool,
|
|
) -> bool:
|
|
"""記錄 Playbook 執行結果"""
|
|
...
|
|
|
|
|
|
class PlaybookService:
|
|
"""
|
|
Playbook Service 實作
|
|
|
|
職責:
|
|
- 從 Incident 萃取 Playbook
|
|
- 提供 Playbook 推薦 (混合搜尋: Jaccard + RAG)
|
|
- 管理 Playbook 生命週期
|
|
- 維護向量索引
|
|
"""
|
|
|
|
def __init__(self, repository: IPlaybookRepository | None = None):
|
|
self._repository = repository or get_playbook_repository()
|
|
# 2026-03-27 ogt: RAG Service 改為 lazy initialization (async factory)
|
|
self._rag_service = None
|
|
|
|
async def _get_rag_service(self):
|
|
"""Lazy initialization for RAG service (2026-03-27 async factory)"""
|
|
if self._rag_service is None:
|
|
self._rag_service = await get_playbook_rag_service()
|
|
return self._rag_service
|
|
|
|
# === Core Operations ===
|
|
|
|
async def extract_from_incident(
|
|
self,
|
|
incident: Incident,
|
|
auto_approve: bool = False,
|
|
) -> Playbook | None:
|
|
"""
|
|
從成功案例萃取 Playbook
|
|
|
|
前置條件:
|
|
- Incident 狀態為 RESOLVED 或 CLOSED
|
|
- outcome.execution_success == True
|
|
- outcome.effectiveness_score >= 4
|
|
|
|
Args:
|
|
incident: 來源 Incident
|
|
auto_approve: 是否自動核准 (僅限高信心度)
|
|
|
|
Returns:
|
|
Playbook | None
|
|
"""
|
|
# 1. 驗證前置條件
|
|
if incident.status not in [IncidentStatus.RESOLVED, IncidentStatus.CLOSED]:
|
|
logger.warning(
|
|
"playbook_extract_invalid_status",
|
|
incident_id=incident.incident_id,
|
|
status=incident.status,
|
|
)
|
|
return None
|
|
|
|
if not incident.outcome or not incident.outcome.execution_success:
|
|
logger.warning(
|
|
"playbook_extract_no_successful_outcome",
|
|
incident_id=incident.incident_id,
|
|
)
|
|
return None
|
|
|
|
effectiveness = incident.outcome.effectiveness_score or 0
|
|
if effectiveness < 4:
|
|
logger.info(
|
|
"playbook_extract_low_effectiveness",
|
|
incident_id=incident.incident_id,
|
|
effectiveness=effectiveness,
|
|
)
|
|
return None
|
|
|
|
# 2. 萃取症狀模式
|
|
symptom_pattern = self._extract_symptom_pattern(incident)
|
|
|
|
# 3. 萃取修復步驟
|
|
repair_steps = self._extract_repair_steps(incident)
|
|
|
|
# 4. 計算信心度
|
|
confidence = self._calculate_confidence(incident, effectiveness)
|
|
|
|
# 5. 生成名稱和描述
|
|
name = self._generate_name(incident)
|
|
description = self._generate_description(incident)
|
|
|
|
# 6. 建立 Playbook
|
|
playbook = Playbook(
|
|
name=name,
|
|
description=description,
|
|
status=PlaybookStatus.APPROVED if auto_approve and confidence >= 0.9 else PlaybookStatus.DRAFT,
|
|
source=PlaybookSource.EXTRACTED,
|
|
symptom_pattern=symptom_pattern,
|
|
repair_steps=repair_steps,
|
|
source_incident_ids=[incident.incident_id],
|
|
ai_confidence=confidence,
|
|
tags=self._extract_tags(incident),
|
|
)
|
|
|
|
# 7. 儲存
|
|
playbook = await self._repository.create(playbook)
|
|
|
|
# 8. ADR-030 Phase 3: 建立向量索引 (非阻塞,失敗不影響主流程)
|
|
import asyncio
|
|
asyncio.create_task(self._index_playbook_async(playbook))
|
|
|
|
logger.info(
|
|
"playbook_extracted",
|
|
playbook_id=playbook.playbook_id,
|
|
incident_id=incident.incident_id,
|
|
confidence=confidence,
|
|
auto_approved=playbook.status == PlaybookStatus.APPROVED,
|
|
)
|
|
|
|
return playbook
|
|
|
|
async def _index_playbook_async(self, playbook: Playbook) -> None:
|
|
"""非同步建立 Playbook 向量索引 (ADR-030 Phase 3)"""
|
|
try:
|
|
rag_service = await self._get_rag_service()
|
|
success = await rag_service.index_playbook(playbook)
|
|
if success:
|
|
logger.debug(
|
|
"playbook_indexed",
|
|
playbook_id=playbook.playbook_id,
|
|
)
|
|
except Exception as e:
|
|
logger.warning(
|
|
"playbook_index_failed",
|
|
playbook_id=playbook.playbook_id,
|
|
error=str(e),
|
|
)
|
|
|
|
async def get_recommendations(
|
|
self,
|
|
symptoms: SymptomPattern,
|
|
top_k: int = 3,
|
|
use_rag: bool = True,
|
|
) -> list[PlaybookRecommendation]:
|
|
"""
|
|
取得 Playbook 推薦
|
|
|
|
ADR-030 Phase 3 策略:
|
|
1. Jaccard 精確匹配 (Repository)
|
|
2. RAG 向量語意搜尋 (可選)
|
|
3. 混合排序 (Jaccard 40% + Vector 60%)
|
|
4. 按 similarity_score * success_rate 排序
|
|
"""
|
|
# Step 1: Jaccard 精確匹配
|
|
similar_playbooks = await self._repository.find_by_symptoms(
|
|
symptoms=symptoms,
|
|
top_k=top_k * 2, # 多取一些用於後續過濾
|
|
min_similarity=0.4,
|
|
)
|
|
|
|
jaccard_results = [(pb.playbook_id, sim) for pb, sim in similar_playbooks]
|
|
playbook_map = {pb.playbook_id: pb for pb, _ in similar_playbooks}
|
|
|
|
# Step 2: RAG 混合搜尋 (如果啟用)
|
|
if use_rag and symptoms.alert_names:
|
|
try:
|
|
rag_service = await self._get_rag_service()
|
|
hybrid_matches = await rag_service.hybrid_search(
|
|
symptoms=symptoms,
|
|
jaccard_results=jaccard_results,
|
|
top_k=top_k * 2,
|
|
vector_weight=0.6,
|
|
jaccard_weight=0.4,
|
|
)
|
|
|
|
# 補充 playbook_map (RAG 可能找到 Jaccard 沒找到的)
|
|
for match in hybrid_matches:
|
|
if match.playbook_id not in playbook_map:
|
|
pb = await self._repository.get_by_id(match.playbook_id)
|
|
if pb:
|
|
playbook_map[match.playbook_id] = pb
|
|
|
|
# 使用混合結果
|
|
final_results = [
|
|
(playbook_map[m.playbook_id], m.similarity_score)
|
|
for m in hybrid_matches
|
|
if m.playbook_id in playbook_map
|
|
]
|
|
|
|
logger.info(
|
|
"playbook_recommendation_hybrid",
|
|
jaccard_count=len(jaccard_results),
|
|
hybrid_count=len(final_results),
|
|
)
|
|
except Exception as e:
|
|
# RAG 失敗時 fallback 到純 Jaccard
|
|
logger.warning(
|
|
"playbook_rag_fallback",
|
|
error=str(e),
|
|
)
|
|
final_results = similar_playbooks
|
|
else:
|
|
final_results = similar_playbooks
|
|
|
|
if not final_results:
|
|
return []
|
|
|
|
# Step 3: 建立推薦列表
|
|
recommendations: list[PlaybookRecommendation] = []
|
|
|
|
for playbook, similarity in final_results:
|
|
# 找出匹配的症狀
|
|
matched_symptoms = self._find_matched_symptoms(symptoms, playbook.symptom_pattern)
|
|
|
|
# 生成推薦原因
|
|
reason = self._generate_recommendation_reason(
|
|
playbook,
|
|
similarity,
|
|
matched_symptoms,
|
|
)
|
|
|
|
recommendations.append(
|
|
PlaybookRecommendation(
|
|
playbook=playbook,
|
|
similarity_score=similarity,
|
|
matched_symptoms=matched_symptoms,
|
|
reason=reason,
|
|
)
|
|
)
|
|
|
|
# Step 4: 按綜合分數排序 (similarity * success_rate)
|
|
recommendations.sort(
|
|
key=lambda r: r.similarity_score * (0.5 + 0.5 * r.playbook.success_rate),
|
|
reverse=True,
|
|
)
|
|
|
|
return recommendations[:top_k]
|
|
|
|
async def approve(
|
|
self,
|
|
playbook_id: str,
|
|
approved_by: str,
|
|
notes: str | None = None,
|
|
) -> Playbook | None:
|
|
"""核准 Playbook"""
|
|
playbook = await self._repository.get_by_id(playbook_id)
|
|
if not playbook:
|
|
return None
|
|
|
|
if playbook.status != PlaybookStatus.DRAFT:
|
|
logger.warning(
|
|
"playbook_approve_invalid_status",
|
|
playbook_id=playbook_id,
|
|
current_status=playbook.status,
|
|
)
|
|
return None
|
|
|
|
playbook.status = PlaybookStatus.APPROVED
|
|
playbook.approved_by = approved_by
|
|
playbook.approved_at = now_taipei()
|
|
if notes:
|
|
playbook.notes = notes
|
|
|
|
updated = await self._repository.update(playbook)
|
|
|
|
if updated:
|
|
logger.info(
|
|
"playbook_approved",
|
|
playbook_id=playbook_id,
|
|
approved_by=approved_by,
|
|
)
|
|
|
|
return updated
|
|
|
|
async def record_execution(
|
|
self,
|
|
playbook_id: str,
|
|
success: bool,
|
|
) -> bool:
|
|
"""記錄 Playbook 執行結果"""
|
|
return await self._repository.update_stats(playbook_id, success)
|
|
|
|
# === CRUD Proxies ===
|
|
|
|
async def get_by_id(self, playbook_id: str) -> Playbook | None:
|
|
"""取得 Playbook"""
|
|
return await self._repository.get_by_id(playbook_id)
|
|
|
|
async def list_playbooks(
|
|
self,
|
|
status: PlaybookStatus | None = None,
|
|
tags: list[str] | None = None,
|
|
limit: int = 20,
|
|
offset: int = 0,
|
|
) -> tuple[list[Playbook], int]:
|
|
"""列出 Playbooks"""
|
|
return await self._repository.list_playbooks(
|
|
status=status,
|
|
tags=tags,
|
|
limit=limit,
|
|
offset=offset,
|
|
)
|
|
|
|
async def update(self, playbook: Playbook) -> Playbook | None:
|
|
"""更新 Playbook"""
|
|
return await self._repository.update(playbook)
|
|
|
|
async def update_with_validation(
|
|
self,
|
|
playbook_id: str,
|
|
update_data: dict,
|
|
) -> Playbook | None:
|
|
"""
|
|
更新 Playbook (含驗證)
|
|
|
|
Phase 8 P1 修復: 從 Router 層移至 Service 層進行驗證
|
|
|
|
驗證規則:
|
|
- 禁止直接修改 playbook_id
|
|
- 禁止反向狀態轉換 (APPROVED → DRAFT)
|
|
- 統計欄位 (success_count, failure_count) 只能透過 record_execution 更新
|
|
|
|
Args:
|
|
playbook_id: Playbook ID
|
|
update_data: 要更新的欄位 (dict)
|
|
|
|
Returns:
|
|
更新後的 Playbook 或 None
|
|
"""
|
|
playbook = await self._repository.get_by_id(playbook_id)
|
|
if not playbook:
|
|
return None
|
|
|
|
# 禁止修改的欄位
|
|
forbidden_fields = {
|
|
"playbook_id",
|
|
"created_at",
|
|
"success_count",
|
|
"failure_count",
|
|
"last_used_at",
|
|
}
|
|
|
|
for field in forbidden_fields:
|
|
if field in update_data:
|
|
logger.warning(
|
|
"playbook_update_forbidden_field",
|
|
playbook_id=playbook_id,
|
|
field=field,
|
|
)
|
|
del update_data[field]
|
|
|
|
# 狀態轉換驗證
|
|
if "status" in update_data:
|
|
new_status = update_data["status"]
|
|
current_status = playbook.status
|
|
|
|
# 允許的轉換: DRAFT → APPROVED, APPROVED → DEPRECATED
|
|
# 禁止: APPROVED → DRAFT, DEPRECATED → 任何
|
|
if current_status == PlaybookStatus.DEPRECATED:
|
|
logger.warning(
|
|
"playbook_update_deprecated_status",
|
|
playbook_id=playbook_id,
|
|
)
|
|
return None
|
|
|
|
if (
|
|
current_status == PlaybookStatus.APPROVED
|
|
and new_status == PlaybookStatus.DRAFT
|
|
):
|
|
logger.warning(
|
|
"playbook_update_invalid_status_transition",
|
|
playbook_id=playbook_id,
|
|
from_status=current_status.value,
|
|
to_status=new_status,
|
|
)
|
|
return None
|
|
|
|
# 應用更新
|
|
for field, value in update_data.items():
|
|
if value is not None and hasattr(playbook, field):
|
|
setattr(playbook, field, value)
|
|
|
|
return await self._repository.update(playbook)
|
|
|
|
async def delete(self, playbook_id: str) -> bool:
|
|
"""刪除 Playbook (軟刪除)"""
|
|
return await self._repository.delete(playbook_id)
|
|
|
|
# === Private Helpers ===
|
|
|
|
def _extract_symptom_pattern(self, incident: Incident) -> SymptomPattern:
|
|
"""從 Incident 萃取症狀模式"""
|
|
alert_names = [s.alert_name for s in incident.signals] if incident.signals else []
|
|
keywords = []
|
|
|
|
# 從 annotations 提取關鍵字
|
|
for signal in incident.signals or []:
|
|
if signal.annotations:
|
|
for value in signal.annotations.values():
|
|
if isinstance(value, str) and len(value) < 50:
|
|
keywords.append(value)
|
|
|
|
return SymptomPattern(
|
|
alert_names=alert_names,
|
|
affected_services=incident.affected_services or [],
|
|
severity_range=[incident.severity.value] if incident.severity else ["P2"],
|
|
keywords=keywords[:10], # 最多 10 個關鍵字
|
|
)
|
|
|
|
def _extract_repair_steps(self, incident: Incident) -> list[RepairStep]:
|
|
"""從 Incident 萃取修復步驟"""
|
|
steps: list[RepairStep] = []
|
|
step_number = 1
|
|
|
|
# 從 decision_chain.reasoning_steps 提取 kubectl 命令
|
|
if incident.decision_chain and incident.decision_chain.reasoning_steps:
|
|
for reasoning in incident.decision_chain.reasoning_steps:
|
|
# 尋找包含 kubectl 的步驟
|
|
if "kubectl" in reasoning.lower():
|
|
# 嘗試提取 kubectl 命令
|
|
import re
|
|
kubectl_match = re.search(r"kubectl\s+\S+.*", reasoning)
|
|
if kubectl_match:
|
|
steps.append(
|
|
RepairStep(
|
|
step_number=step_number,
|
|
action_type=ActionType.KUBECTL,
|
|
command=kubectl_match.group(0).strip(),
|
|
risk_level=RiskLevel.MEDIUM,
|
|
)
|
|
)
|
|
step_number += 1
|
|
|
|
# 如果沒有從 reasoning_steps 取得,嘗試從 learning_notes 取得
|
|
if not steps and incident.outcome and incident.outcome.learning_notes:
|
|
notes = incident.outcome.learning_notes
|
|
if "kubectl" in notes.lower():
|
|
steps.append(
|
|
RepairStep(
|
|
step_number=1,
|
|
action_type=ActionType.KUBECTL,
|
|
command=notes,
|
|
risk_level=RiskLevel.MEDIUM,
|
|
)
|
|
)
|
|
|
|
return steps
|
|
|
|
def _calculate_confidence(self, incident: Incident, effectiveness: int) -> float:
|
|
"""計算 AI 萃取信心度"""
|
|
base_score = 0.5
|
|
|
|
# effectiveness 貢獻 (4-5 → 0.2-0.4)
|
|
effectiveness_bonus = (effectiveness - 3) * 0.2
|
|
|
|
# 有 decision_chain 加分
|
|
if incident.decision_chain and incident.decision_chain.reasoning_steps:
|
|
base_score += 0.1
|
|
|
|
# 有多個 signals 加分 (更多資料)
|
|
if incident.signals and len(incident.signals) >= 2:
|
|
base_score += 0.05
|
|
|
|
return min(base_score + effectiveness_bonus, 1.0)
|
|
|
|
def _generate_name(self, incident: Incident) -> str:
|
|
"""生成 Playbook 名稱"""
|
|
alert_name = incident.signals[0].alert_name if incident.signals else "Unknown"
|
|
services = incident.affected_services[:2] if incident.affected_services else []
|
|
service_str = "/".join(services) if services else "system"
|
|
|
|
return f"{alert_name} - {service_str} 修復劇本"
|
|
|
|
def _generate_description(self, incident: Incident) -> str:
|
|
"""生成 Playbook 描述"""
|
|
parts = []
|
|
|
|
if incident.signals:
|
|
parts.append(f"觸發告警: {incident.signals[0].alert_name}")
|
|
|
|
if incident.affected_services:
|
|
parts.append(f"影響服務: {', '.join(incident.affected_services)}")
|
|
|
|
# 從 decision_chain.hypothesis 取得 AI 分析結果
|
|
if incident.decision_chain and incident.decision_chain.hypothesis:
|
|
parts.append(f"AI 分析: {incident.decision_chain.hypothesis[:100]}")
|
|
|
|
return ". ".join(parts) if parts else "從成功案例自動萃取的修復劇本"
|
|
|
|
def _extract_tags(self, incident: Incident) -> list[str]:
|
|
"""萃取標籤"""
|
|
tags: set[str] = set()
|
|
|
|
# 從服務名稱提取
|
|
for service in incident.affected_services or []:
|
|
tags.add(service.lower())
|
|
|
|
# 從告警名稱提取類型
|
|
if incident.signals:
|
|
for signal in incident.signals:
|
|
if "cpu" in signal.alert_name.lower():
|
|
tags.add("cpu")
|
|
if "memory" in signal.alert_name.lower():
|
|
tags.add("memory")
|
|
if "pod" in signal.alert_name.lower():
|
|
tags.add("kubernetes")
|
|
if "network" in signal.alert_name.lower():
|
|
tags.add("network")
|
|
|
|
return list(tags)[:10]
|
|
|
|
def _find_matched_symptoms(
|
|
self,
|
|
query: SymptomPattern,
|
|
playbook_pattern: SymptomPattern,
|
|
) -> list[str]:
|
|
"""找出匹配的症狀"""
|
|
matched = []
|
|
|
|
# 匹配的告警
|
|
alert_matches = set(query.alert_names) & set(playbook_pattern.alert_names)
|
|
for alert in alert_matches:
|
|
matched.append(f"Alert: {alert}")
|
|
|
|
# 匹配的服務
|
|
service_matches = set(query.affected_services) & set(playbook_pattern.affected_services)
|
|
for service in service_matches:
|
|
matched.append(f"Service: {service}")
|
|
|
|
# 匹配的嚴重度
|
|
if set(query.severity_range) & set(playbook_pattern.severity_range):
|
|
matched.append(f"Severity: {query.severity_range[0]}")
|
|
|
|
return matched
|
|
|
|
def _generate_recommendation_reason(
|
|
self,
|
|
playbook: Playbook,
|
|
similarity: float,
|
|
matched_symptoms: list[str],
|
|
) -> str:
|
|
"""生成推薦原因"""
|
|
parts = []
|
|
|
|
parts.append(f"相似度 {similarity:.0%}")
|
|
|
|
if playbook.success_rate > 0:
|
|
parts.append(f"成功率 {playbook.success_rate:.0%}")
|
|
|
|
if playbook.total_executions > 0:
|
|
parts.append(f"已執行 {playbook.total_executions} 次")
|
|
|
|
if matched_symptoms:
|
|
parts.append(f"匹配: {', '.join(matched_symptoms[:3])}")
|
|
|
|
return ". ".join(parts)
|
|
|
|
|
|
# =============================================================================
|
|
# Singleton
|
|
# =============================================================================
|
|
|
|
_service: PlaybookService | None = None
|
|
|
|
|
|
def get_playbook_service() -> IPlaybookService:
|
|
"""取得 PlaybookService 單例"""
|
|
global _service
|
|
if _service is None:
|
|
_service = PlaybookService()
|
|
return _service
|