""" LLM Playbook Generator - ADR-104 T1/T2/T6 ========================================= 從成功修復案例生成可治理的 Playbook 草稿。 設計重點: - 只用 local/provider pool 順序(GCP-A -> 111 local),避免新增雲端成本。 - LLM 產出必須經 Pydantic + action_parser 安全收斂。 - 不直接 APPROVED;先 DRAFT/REVIEW,再交治理 job 晉級。 """ from __future__ import annotations import json import re from collections.abc import Awaitable, Callable from dataclasses import dataclass from typing import Any import structlog from pydantic import BaseModel, Field, field_validator from src.models.incident import Incident, IncidentStatus from src.models.playbook import ( ActionType, Playbook, PlaybookSource, PlaybookStatus, RepairStep, RiskLevel, SymptomPattern, ) from src.services.action_parser import kubectl_safety_reason logger = structlog.get_logger(__name__) LLMCallable = Callable[[str, dict[str, Any]], Awaitable[tuple[str, str, bool]]] class GeneratedRepairStep(BaseModel): """LLM repair step contract.""" action_type: str = Field(default="manual") command: str = Field(default="") expected_result: str | None = None rollback_command: str | None = None risk_level: str = Field(default="MEDIUM") @field_validator("risk_level", mode="before") @classmethod def normalize_risk(cls, value: object) -> str: risk = str(value or "MEDIUM").upper() return risk if risk in {"LOW", "MEDIUM", "HIGH", "CRITICAL"} else "MEDIUM" class GeneratedPlaybookPayload(BaseModel): """Strict JSON shape expected from the local LLM.""" name: str = Field(min_length=1, max_length=256) description: str = Field(min_length=1, max_length=2000) alert_names: list[str] = Field(default_factory=list) affected_services: list[str] = Field(default_factory=list) severity_range: list[str] = Field(default_factory=lambda: ["P2"]) keywords: list[str] = Field(default_factory=list) repair_steps: list[GeneratedRepairStep] = Field(default_factory=list) estimated_duration_minutes: int = Field(default=5, ge=1, le=480) confidence: float = Field(default=0.5, ge=0.0, le=1.0) tags: list[str] = Field(default_factory=list) notes: str | None = None @dataclass class PlaybookGenerationResult: """Generator result plus provenance for timeline/KM/metrics.""" playbook: Playbook | None outcome: str provider: str reason: str = "" def _extract_json_object(text: str) -> dict[str, Any] | None: """Parse a JSON object from an LLM response.""" text = (text or "").strip() if not text: return None try: data = json.loads(text) return data if isinstance(data, dict) else None except json.JSONDecodeError: pass fenced = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL) if fenced: try: data = json.loads(fenced.group(1)) return data if isinstance(data, dict) else None except json.JSONDecodeError: return None start = text.find("{") end = text.rfind("}") if start >= 0 and end > start: try: data = json.loads(text[start : end + 1]) return data if isinstance(data, dict) else None except json.JSONDecodeError: return None return None def _safe_risk(value: str) -> RiskLevel: try: return RiskLevel(value.upper()) except ValueError: return RiskLevel.MEDIUM def _manual_step(step_number: int, command: str, reason: str) -> RepairStep: command_preview = command.strip()[:240] or "未提供命令" return RepairStep( step_number=step_number, action_type=ActionType.MANUAL, command=f"人工審核 LLM 建議: {command_preview}", expected_result=reason, requires_approval=True, risk_level=RiskLevel.HIGH, ) class LLMPlaybookGenerator: """Generate Playbook drafts from resolved incidents using local AI.""" def __init__( self, playbook_service: Any | None = None, llm_callable: LLMCallable | None = None, ) -> None: self._playbook_service = playbook_service self._llm_callable = llm_callable async def generate_from_incident( self, incident: Incident, action: str | None = None, persist: bool = True, ) -> PlaybookGenerationResult: """Generate and optionally persist a governed Playbook draft.""" if incident.status not in (IncidentStatus.RESOLVED, IncidentStatus.CLOSED): return self._record(None, "skipped", "none", "incident_not_resolved") if not incident.outcome or incident.outcome.execution_success is not True: return self._record(None, "skipped", "none", "execution_not_successful") prompt = self._build_prompt(incident, action) context = { "incident_id": incident.incident_id, "intent_hint": "playbook_generation", "task_type": "force_local", "alert_type": self._first_alert_name(incident), "target_resource": ",".join(incident.affected_services or []), } raw, provider, success = await self._call_local_llm(prompt, context) payload = self._parse_payload(raw) if success else None if payload is None: fallback = self._deterministic_playbook(incident, action) if fallback and persist: fallback = await self._service().create(fallback) return self._record(fallback, "fallback", provider, "llm_payload_invalid") playbook = self._build_playbook(incident, payload, provider) if not playbook.repair_steps: playbook.repair_steps = self._deterministic_steps(incident, action) if not playbook.repair_steps: playbook.repair_steps = [ _manual_step(1, action or "未提供修復動作", "LLM 未產生可執行安全步驟") ] playbook.status = PlaybookStatus.DRAFT if persist: playbook = await self._persist_with_lineage(playbook) return self._record(playbook, "success", provider, "") async def _persist_with_lineage(self, playbook: Playbook) -> Playbook: """Create a new lineage version when a close approved Playbook exists.""" try: recommendations = await self._service().get_recommendations( symptoms=playbook.symptom_pattern, top_k=1, use_rag=False, ) if recommendations and recommendations[0].similarity_score >= 0.85: base = recommendations[0].playbook created = await self._service().create_new_version( base_playbook_id=base.playbook_id, candidate=playbook, reason="ADR-104 local LLM generated improved Playbook from successful incident", ) if created is not None: return created except Exception as exc: logger.warning("playbook_generation_lineage_fallback", error=str(exc)) return await self._service().create(playbook) async def _call_local_llm( self, prompt: str, context: dict[str, Any], ) -> tuple[str, str, bool]: if self._llm_callable is not None: return await self._llm_callable(prompt, context) try: from src.services.ai_router import get_ai_executor executor = get_ai_executor() result = await executor.execute( prompt=prompt, provider_order=["ollama", "ollama_local"], context=context, cache_ttl=86400, require_local=True, ) return result.raw_response, result.provider, result.success except Exception as exc: logger.warning("playbook_generation_llm_failed", error=str(exc)) return "", "local_ai_error", False def _parse_payload(self, raw: str) -> GeneratedPlaybookPayload | None: data = _extract_json_object(raw) if data is None: return None try: return GeneratedPlaybookPayload.model_validate(data) except Exception as exc: logger.warning("playbook_generation_payload_invalid", error=str(exc)) return None def _build_playbook( self, incident: Incident, payload: GeneratedPlaybookPayload, provider: str, ) -> Playbook: steps = self._sanitize_steps(payload.repair_steps) confidence = payload.confidence status = PlaybookStatus.REVIEW if confidence >= 0.75 and steps else PlaybookStatus.DRAFT alert_names = payload.alert_names or [self._first_alert_name(incident)] affected = payload.affected_services or list(incident.affected_services or []) severity = payload.severity_range or ([incident.severity.value] if incident.severity else ["P2"]) notes = payload.notes or "" provenance = f"Generated by {provider} from {incident.incident_id}" notes = f"{notes}\n{provenance}".strip() return Playbook( name=payload.name, description=payload.description, status=status, source=PlaybookSource.LLM_GENERATED, symptom_pattern=SymptomPattern( alert_names=[x for x in alert_names if x], affected_services=affected, severity_range=severity, keywords=payload.keywords[:10], ), repair_steps=steps, estimated_duration_minutes=payload.estimated_duration_minutes, source_incident_ids=[incident.incident_id], ai_confidence=confidence, trust_score=0.3, tags=[*payload.tags[:8], "llm_generated", provider], notes=notes, ) def _sanitize_steps(self, steps: list[GeneratedRepairStep]) -> list[RepairStep]: sanitized: list[RepairStep] = [] for raw_step in steps[:8]: command = raw_step.command.strip() if not command: continue step_number = len(sanitized) + 1 action_type = raw_step.action_type.strip().lower() if command.startswith("kubectl") or action_type == "kubectl": safety_reason = kubectl_safety_reason(command) if safety_reason is not None: sanitized.append(_manual_step(step_number, command, safety_reason)) continue sanitized.append( RepairStep( step_number=step_number, action_type=ActionType.KUBECTL, command=command, expected_result=raw_step.expected_result, rollback_command=raw_step.rollback_command, requires_approval=_safe_risk(raw_step.risk_level) in (RiskLevel.HIGH, RiskLevel.CRITICAL), risk_level=_safe_risk(raw_step.risk_level), ) ) continue if action_type == "ssh_command" or command.startswith("ssh "): sanitized.append( RepairStep( step_number=step_number, action_type=ActionType.SSH_COMMAND, command=command, expected_result=raw_step.expected_result, rollback_command=raw_step.rollback_command, requires_approval=True, risk_level=max(_safe_risk(raw_step.risk_level), RiskLevel.MEDIUM, key=lambda r: list(RiskLevel).index(r)), ) ) continue sanitized.append(_manual_step(step_number, command, "non_kubectl_step_requires_review")) return sanitized def _deterministic_playbook(self, incident: Incident, action: str | None) -> Playbook | None: steps = self._deterministic_steps(incident, action) if not steps: return None alert_name = self._first_alert_name(incident) or "Unknown" return Playbook( name=f"{alert_name} - AI 生成 fallback Playbook", description="LLM 產出不可解析時,從成功執行動作建立的保守 Playbook 草稿", status=PlaybookStatus.DRAFT, source=PlaybookSource.LLM_GENERATED, symptom_pattern=SymptomPattern( alert_names=[alert_name] if alert_name else [], affected_services=list(incident.affected_services or []), severity_range=[incident.severity.value] if incident.severity else ["P2"], ), repair_steps=steps, source_incident_ids=[incident.incident_id], ai_confidence=0.45, tags=["llm_generated", "fallback"], notes=f"Generated deterministically after local LLM parse failure for {incident.incident_id}", ) def _deterministic_steps(self, incident: Incident, action: str | None) -> list[RepairStep]: command = (action or "").strip() if not command and incident.outcome and incident.outcome.learning_notes: command = incident.outcome.learning_notes.strip() if not command: return [] if command.startswith("kubectl"): safety_reason = kubectl_safety_reason(command) if safety_reason is None: return [ RepairStep( step_number=1, action_type=ActionType.KUBECTL, command=command, requires_approval=False, risk_level=RiskLevel.MEDIUM, ) ] return [_manual_step(1, command, safety_reason)] if command.startswith("ssh "): return [ RepairStep( step_number=1, action_type=ActionType.SSH_COMMAND, command=command, requires_approval=True, risk_level=RiskLevel.MEDIUM, ) ] return [_manual_step(1, command, "unknown_action_type")] def _build_prompt(self, incident: Incident, action: str | None) -> str: signals = [ { "alert_name": signal.alert_name, "severity": signal.severity.value, "labels": signal.labels, "annotations": signal.annotations, } for signal in incident.signals[:5] ] context = { "incident_id": incident.incident_id, "severity": incident.severity.value, "affected_services": incident.affected_services, "signals": signals, "hypothesis": incident.decision_chain.hypothesis if incident.decision_chain else "", "reasoning_steps": incident.decision_chain.reasoning_steps if incident.decision_chain else [], "successful_action": action or (incident.outcome.learning_notes if incident.outcome else ""), "effectiveness_score": incident.outcome.effectiveness_score if incident.outcome else None, } return ( "你是 AWOOOI ADR-104 Playbook Generator,由 OpenClaw/Hermes/NemoTron/ElephantAlpha 的角色視角共同產出。" "請只輸出 JSON object,不要 markdown。任何破壞性命令必須改成 manual 步驟。\n" "JSON schema: {name, description, alert_names, affected_services, severity_range, keywords, " "repair_steps:[{action_type, command, expected_result, rollback_command, risk_level}], " "estimated_duration_minutes, confidence, tags, notes}.\n" f"Incident context:\n{json.dumps(context, ensure_ascii=False, default=str)}" ) def _first_alert_name(self, incident: Incident) -> str: return incident.signals[0].alert_name if incident.signals else "" def _service(self) -> Any: if self._playbook_service is None: from src.services.playbook_service import get_playbook_service self._playbook_service = get_playbook_service() return self._playbook_service def _record( self, playbook: Playbook | None, outcome: str, provider: str, reason: str, ) -> PlaybookGenerationResult: try: from src.core.metrics import observe_playbook_status, record_playbook_generation source = provider or "none" record_playbook_generation(outcome=outcome, source=source) if playbook is not None: observe_playbook_status(status=playbook.status.value, source=source) except Exception as exc: logger.debug("playbook_generation_metric_failed", error=str(exc)) return PlaybookGenerationResult(playbook=playbook, outcome=outcome, provider=provider, reason=reason) _generator: LLMPlaybookGenerator | None = None def get_playbook_generator() -> LLMPlaybookGenerator: """Return global LLM Playbook generator.""" global _generator if _generator is None: _generator = LLMPlaybookGenerator() return _generator