feat(playbook): generate drafts with local llm

apps/api/src/services/learning_service.py

@@ -439,6 +439,7 @@ class LearningService:
         try:
             from src.repositories.incident_repository import get_incident_repository
             from src.services.playbook_service import get_playbook_service
+            from src.core.config import settings
 
             # 取得 Incident
             repo = get_incident_repository()
@@ -451,6 +452,26 @@ class LearningService:
             if incident.status not in [IncidentStatus.RESOLVED, IncidentStatus.CLOSED]:
                 return None
 
+            # ADR-104: 優先用本地 LLM 生成可治理 Playbook。失敗時服務內會降級成
+            # deterministic fallback；feature flag 關閉才回到舊萃取路徑。
+            if settings.ENABLE_LLM_PLAYBOOK_GENERATION:
+                from src.services.playbook_generator import get_playbook_generator
+
+                generated = await get_playbook_generator().generate_from_incident(
+                    incident=incident,
+                    action=action,
+                    persist=True,
+                )
+                if generated.playbook:
+                    logger.info(
+                        "playbook_llm_generated",
+                        incident_id=incident_id,
+                        playbook_id=generated.playbook.playbook_id,
+                        outcome=generated.outcome,
+                        provider=generated.provider,
+                    )
+                    return generated.playbook.playbook_id
+
             # 萃取 Playbook
             service = get_playbook_service()
             playbook = await service.extract_from_incident(

apps/api/src/services/playbook_generator.py

@@ -0,0 +1,418 @@
+"""
+LLM Playbook Generator - ADR-104 T1/T2/T6
+=========================================
+從成功修復案例生成可治理的 Playbook 草稿。
+
+設計重點:
+- 只用 local provider 順序（Ollama 111 -> Ollama 188），避免新增雲端成本。
+- LLM 產出必須經 Pydantic + action_parser 安全收斂。
+- 不直接 APPROVED；先 DRAFT/REVIEW，再交治理 job 晉級。
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from typing import Any
+
+import structlog
+from pydantic import BaseModel, Field, field_validator
+
+from src.models.incident import Incident, IncidentStatus
+from src.models.playbook import (
+    ActionType,
+    Playbook,
+    PlaybookSource,
+    PlaybookStatus,
+    RepairStep,
+    RiskLevel,
+    SymptomPattern,
+)
+from src.services.action_parser import is_safe_kubectl_action
+from src.services.action_parser import kubectl_safety_reason
+
+logger = structlog.get_logger(__name__)
+
+LLMCallable = Callable[[str, dict[str, Any]], Awaitable[tuple[str, str, bool]]]
+
+
+class GeneratedRepairStep(BaseModel):
+    """LLM repair step contract."""
+
+    action_type: str = Field(default="manual")
+    command: str = Field(default="")
+    expected_result: str | None = None
+    rollback_command: str | None = None
+    risk_level: str = Field(default="MEDIUM")
+
+    @field_validator("risk_level", mode="before")
+    @classmethod
+    def normalize_risk(cls, value: object) -> str:
+        risk = str(value or "MEDIUM").upper()
+        return risk if risk in {"LOW", "MEDIUM", "HIGH", "CRITICAL"} else "MEDIUM"
+
+
+class GeneratedPlaybookPayload(BaseModel):
+    """Strict JSON shape expected from the local LLM."""
+
+    name: str = Field(min_length=1, max_length=256)
+    description: str = Field(min_length=1, max_length=2000)
+    alert_names: list[str] = Field(default_factory=list)
+    affected_services: list[str] = Field(default_factory=list)
+    severity_range: list[str] = Field(default_factory=lambda: ["P2"])
+    keywords: list[str] = Field(default_factory=list)
+    repair_steps: list[GeneratedRepairStep] = Field(default_factory=list)
+    estimated_duration_minutes: int = Field(default=5, ge=1, le=480)
+    confidence: float = Field(default=0.5, ge=0.0, le=1.0)
+    tags: list[str] = Field(default_factory=list)
+    notes: str | None = None
+
+
+@dataclass
+class PlaybookGenerationResult:
+    """Generator result plus provenance for timeline/KM/metrics."""
+
+    playbook: Playbook | None
+    outcome: str
+    provider: str
+    reason: str = ""
+
+
+def _extract_json_object(text: str) -> dict[str, Any] | None:
+    """Parse a JSON object from an LLM response."""
+    text = (text or "").strip()
+    if not text:
+        return None
+    try:
+        data = json.loads(text)
+        return data if isinstance(data, dict) else None
+    except json.JSONDecodeError:
+        pass
+
+    fenced = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL)
+    if fenced:
+        try:
+            data = json.loads(fenced.group(1))
+            return data if isinstance(data, dict) else None
+        except json.JSONDecodeError:
+            return None
+
+    start = text.find("{")
+    end = text.rfind("}")
+    if start >= 0 and end > start:
+        try:
+            data = json.loads(text[start : end + 1])
+            return data if isinstance(data, dict) else None
+        except json.JSONDecodeError:
+            return None
+    return None
+
+
+def _safe_risk(value: str) -> RiskLevel:
+    try:
+        return RiskLevel(value.upper())
+    except ValueError:
+        return RiskLevel.MEDIUM
+
+
+def _manual_step(step_number: int, command: str, reason: str) -> RepairStep:
+    command_preview = command.strip()[:240] or "未提供命令"
+    return RepairStep(
+        step_number=step_number,
+        action_type=ActionType.MANUAL,
+        command=f"人工審核 LLM 建議: {command_preview}",
+        expected_result=reason,
+        requires_approval=True,
+        risk_level=RiskLevel.HIGH,
+    )
+
+
+class LLMPlaybookGenerator:
+    """Generate Playbook drafts from resolved incidents using local AI."""
+
+    def __init__(
+        self,
+        playbook_service: Any | None = None,
+        llm_callable: LLMCallable | None = None,
+    ) -> None:
+        self._playbook_service = playbook_service
+        self._llm_callable = llm_callable
+
+    async def generate_from_incident(
+        self,
+        incident: Incident,
+        action: str | None = None,
+        persist: bool = True,
+    ) -> PlaybookGenerationResult:
+        """Generate and optionally persist a governed Playbook draft."""
+        if incident.status not in (IncidentStatus.RESOLVED, IncidentStatus.CLOSED):
+            return self._record(None, "skipped", "none", "incident_not_resolved")
+        if not incident.outcome or incident.outcome.execution_success is not True:
+            return self._record(None, "skipped", "none", "execution_not_successful")
+
+        prompt = self._build_prompt(incident, action)
+        context = {
+            "incident_id": incident.incident_id,
+            "intent_hint": "playbook_generation",
+            "task_type": "force_local",
+            "alert_type": self._first_alert_name(incident),
+            "target_resource": ",".join(incident.affected_services or []),
+        }
+
+        raw, provider, success = await self._call_local_llm(prompt, context)
+        payload = self._parse_payload(raw) if success else None
+        if payload is None:
+            fallback = self._deterministic_playbook(incident, action)
+            if fallback and persist:
+                fallback = await self._service().create(fallback)
+            return self._record(fallback, "fallback", provider, "llm_payload_invalid")
+
+        playbook = self._build_playbook(incident, payload, provider)
+        if not playbook.repair_steps:
+            playbook.repair_steps = self._deterministic_steps(incident, action)
+        if not playbook.repair_steps:
+            playbook.repair_steps = [
+                _manual_step(1, action or "未提供修復動作", "LLM 未產生可執行安全步驟")
+            ]
+            playbook.status = PlaybookStatus.DRAFT
+
+        if persist:
+            playbook = await self._service().create(playbook)
+
+        return self._record(playbook, "success", provider, "")
+
+    async def _call_local_llm(
+        self,
+        prompt: str,
+        context: dict[str, Any],
+    ) -> tuple[str, str, bool]:
+        if self._llm_callable is not None:
+            return await self._llm_callable(prompt, context)
+
+        try:
+            from src.services.ai_router import get_ai_executor
+
+            executor = get_ai_executor()
+            result = await executor.execute(
+                prompt=prompt,
+                provider_order=["ollama", "ollama_188"],
+                context=context,
+                cache_ttl=86400,
+                require_local=True,
+            )
+            return result.raw_response, result.provider, result.success
+        except Exception as exc:
+            logger.warning("playbook_generation_llm_failed", error=str(exc))
+            return "", "local_ai_error", False
+
+    def _parse_payload(self, raw: str) -> GeneratedPlaybookPayload | None:
+        data = _extract_json_object(raw)
+        if data is None:
+            return None
+        try:
+            return GeneratedPlaybookPayload.model_validate(data)
+        except Exception as exc:
+            logger.warning("playbook_generation_payload_invalid", error=str(exc))
+            return None
+
+    def _build_playbook(
+        self,
+        incident: Incident,
+        payload: GeneratedPlaybookPayload,
+        provider: str,
+    ) -> Playbook:
+        steps = self._sanitize_steps(payload.repair_steps)
+        confidence = payload.confidence
+        status = PlaybookStatus.REVIEW if confidence >= 0.75 and steps else PlaybookStatus.DRAFT
+
+        alert_names = payload.alert_names or [self._first_alert_name(incident)]
+        affected = payload.affected_services or list(incident.affected_services or [])
+        severity = payload.severity_range or ([incident.severity.value] if incident.severity else ["P2"])
+
+        notes = payload.notes or ""
+        provenance = f"Generated by {provider} from {incident.incident_id}"
+        notes = f"{notes}\n{provenance}".strip()
+
+        return Playbook(
+            name=payload.name,
+            description=payload.description,
+            status=status,
+            source=PlaybookSource.LLM_GENERATED,
+            symptom_pattern=SymptomPattern(
+                alert_names=[x for x in alert_names if x],
+                affected_services=affected,
+                severity_range=severity,
+                keywords=payload.keywords[:10],
+            ),
+            repair_steps=steps,
+            estimated_duration_minutes=payload.estimated_duration_minutes,
+            source_incident_ids=[incident.incident_id],
+            ai_confidence=confidence,
+            trust_score=0.3,
+            tags=[*payload.tags[:8], "llm_generated", provider],
+            notes=notes,
+        )
+
+    def _sanitize_steps(self, steps: list[GeneratedRepairStep]) -> list[RepairStep]:
+        sanitized: list[RepairStep] = []
+        for raw_step in steps[:8]:
+            command = raw_step.command.strip()
+            if not command:
+                continue
+            step_number = len(sanitized) + 1
+            action_type = raw_step.action_type.strip().lower()
+            if command.startswith("kubectl") or action_type == "kubectl":
+                safety_reason = kubectl_safety_reason(command)
+                if safety_reason is not None:
+                    sanitized.append(_manual_step(step_number, command, safety_reason))
+                    continue
+                sanitized.append(
+                    RepairStep(
+                        step_number=step_number,
+                        action_type=ActionType.KUBECTL,
+                        command=command,
+                        expected_result=raw_step.expected_result,
+                        rollback_command=raw_step.rollback_command,
+                        requires_approval=_safe_risk(raw_step.risk_level) in (RiskLevel.HIGH, RiskLevel.CRITICAL),
+                        risk_level=_safe_risk(raw_step.risk_level),
+                    )
+                )
+                continue
+            if action_type == "ssh_command" or command.startswith("ssh "):
+                sanitized.append(
+                    RepairStep(
+                        step_number=step_number,
+                        action_type=ActionType.SSH_COMMAND,
+                        command=command,
+                        expected_result=raw_step.expected_result,
+                        rollback_command=raw_step.rollback_command,
+                        requires_approval=True,
+                        risk_level=max(_safe_risk(raw_step.risk_level), RiskLevel.MEDIUM, key=lambda r: list(RiskLevel).index(r)),
+                    )
+                )
+                continue
+            sanitized.append(_manual_step(step_number, command, "non_kubectl_step_requires_review"))
+        return sanitized
+
+    def _deterministic_playbook(self, incident: Incident, action: str | None) -> Playbook | None:
+        steps = self._deterministic_steps(incident, action)
+        if not steps:
+            return None
+        alert_name = self._first_alert_name(incident) or "Unknown"
+        return Playbook(
+            name=f"{alert_name} - AI 生成 fallback Playbook",
+            description="LLM 產出不可解析時，從成功執行動作建立的保守 Playbook 草稿",
+            status=PlaybookStatus.DRAFT,
+            source=PlaybookSource.LLM_GENERATED,
+            symptom_pattern=SymptomPattern(
+                alert_names=[alert_name] if alert_name else [],
+                affected_services=list(incident.affected_services or []),
+                severity_range=[incident.severity.value] if incident.severity else ["P2"],
+            ),
+            repair_steps=steps,
+            source_incident_ids=[incident.incident_id],
+            ai_confidence=0.45,
+            tags=["llm_generated", "fallback"],
+            notes=f"Generated deterministically after local LLM parse failure for {incident.incident_id}",
+        )
+
+    def _deterministic_steps(self, incident: Incident, action: str | None) -> list[RepairStep]:
+        command = (action or "").strip()
+        if not command and incident.outcome and incident.outcome.learning_notes:
+            command = incident.outcome.learning_notes.strip()
+        if not command:
+            return []
+        if command.startswith("kubectl"):
+            safety_reason = kubectl_safety_reason(command)
+            if safety_reason is None:
+                return [
+                    RepairStep(
+                        step_number=1,
+                        action_type=ActionType.KUBECTL,
+                        command=command,
+                        requires_approval=False,
+                        risk_level=RiskLevel.MEDIUM,
+                    )
+                ]
+            return [_manual_step(1, command, safety_reason)]
+        if command.startswith("ssh "):
+            return [
+                RepairStep(
+                    step_number=1,
+                    action_type=ActionType.SSH_COMMAND,
+                    command=command,
+                    requires_approval=True,
+                    risk_level=RiskLevel.MEDIUM,
+                )
+            ]
+        return [_manual_step(1, command, "unknown_action_type")]
+
+    def _build_prompt(self, incident: Incident, action: str | None) -> str:
+        signals = [
+            {
+                "alert_name": signal.alert_name,
+                "severity": signal.severity.value,
+                "labels": signal.labels,
+                "annotations": signal.annotations,
+            }
+            for signal in incident.signals[:5]
+        ]
+        context = {
+            "incident_id": incident.incident_id,
+            "severity": incident.severity.value,
+            "affected_services": incident.affected_services,
+            "signals": signals,
+            "hypothesis": incident.decision_chain.hypothesis if incident.decision_chain else "",
+            "reasoning_steps": incident.decision_chain.reasoning_steps if incident.decision_chain else [],
+            "successful_action": action or (incident.outcome.learning_notes if incident.outcome else ""),
+            "effectiveness_score": incident.outcome.effectiveness_score if incident.outcome else None,
+        }
+        return (
+            "你是 AWOOOI ADR-104 Playbook Generator，由 OpenClaw/Hermes/NemoTron/ElephantAlpha 的角色視角共同產出。"
+            "請只輸出 JSON object，不要 markdown。任何破壞性命令必須改成 manual 步驟。\n"
+            "JSON schema: {name, description, alert_names, affected_services, severity_range, keywords, "
+            "repair_steps:[{action_type, command, expected_result, rollback_command, risk_level}], "
+            "estimated_duration_minutes, confidence, tags, notes}.\n"
+            f"Incident context:\n{json.dumps(context, ensure_ascii=False, default=str)}"
+        )
+
+    def _first_alert_name(self, incident: Incident) -> str:
+        return incident.signals[0].alert_name if incident.signals else ""
+
+    def _service(self) -> Any:
+        if self._playbook_service is None:
+            from src.services.playbook_service import get_playbook_service
+
+            self._playbook_service = get_playbook_service()
+        return self._playbook_service
+
+    def _record(
+        self,
+        playbook: Playbook | None,
+        outcome: str,
+        provider: str,
+        reason: str,
+    ) -> PlaybookGenerationResult:
+        try:
+            from src.core.metrics import observe_playbook_status, record_playbook_generation
+
+            source = provider or "none"
+            record_playbook_generation(outcome=outcome, source=source)
+            if playbook is not None:
+                observe_playbook_status(status=playbook.status.value, source=source)
+        except Exception as exc:
+            logger.debug("playbook_generation_metric_failed", error=str(exc))
+        return PlaybookGenerationResult(playbook=playbook, outcome=outcome, provider=provider, reason=reason)
+
+
+_generator: LLMPlaybookGenerator | None = None
+
+
+def get_playbook_generator() -> LLMPlaybookGenerator:
+    """Return global LLM Playbook generator."""
+    global _generator
+    if _generator is None:
+        _generator = LLMPlaybookGenerator()
+    return _generator