awoooi/apps/api/src/services/auto_repair_service.py

"""
Auto Repair Service - #8 自動升級決策
=====================================
高品質 Playbook 自動修復執行

Phase 8: 自動化層實作
建立時間: 2026-03-26 17:30 (台北時區)
建立者: Claude Code (#8 自動升級決策)

遵循 leWOOOgo 積木化原則:
- Service 層只依賴 Repository/Service Interface
- 不直接存取 Redis/DB
- 封裝所有自動修復邏輯

觸發條件 (AND):
1. 有匹配的高品質 Playbook (is_high_quality = True)
2. Playbook 中的動作風險等級 <= MEDIUM
3. Incident 嚴重度 <= P2

安全邊界:
- HIGH/CRITICAL 風險動作永遠需要人工審核
- P0/P1 嚴重度 Incident 需要人工確認
"""

from dataclasses import dataclass
from typing import Protocol

import structlog

from src.models.incident import Incident, Severity
from src.models.playbook import (
    ActionType,
    Playbook,
    RiskLevel,
    SymptomPattern,
)
from src.services.anomaly_counter import AnomalyFrequency, get_anomaly_counter
from src.services.executor import get_executor
from src.services.global_repair_cooldown import (
    check_global_repair_cooldown,
    record_global_repair_action,
)
from src.services.playbook_service import IPlaybookService, get_playbook_service

logger = structlog.get_logger(__name__)


# =============================================================================
# Types
# =============================================================================


@dataclass
class AutoRepairDecision:
    """自動修復決策結果"""

    can_auto_repair: bool
    playbook: Playbook | None = None
    reason: str = ""
    risk_level: RiskLevel = RiskLevel.MEDIUM
    blocked_by: str | None = None  # 阻擋原因 (如 HIGH_RISK, P1_SEVERITY)


@dataclass
class AutoRepairResult:
    """自動修復執行結果"""

    success: bool
    playbook_id: str
    incident_id: str
    executed_steps: list[str]
    error: str | None = None
    execution_time_ms: int = 0


# =============================================================================
# Auto Repair Service Interface
# =============================================================================


class IAutoRepairService(Protocol):
    """自動修復服務介面"""

    async def evaluate_auto_repair(
        self,
        incident: Incident,
    ) -> AutoRepairDecision:
        """
        評估是否可自動修復

        Args:
            incident: 待處理的 Incident

        Returns:
            AutoRepairDecision: 決策結果
        """
        ...

    async def execute_auto_repair(
        self,
        incident: Incident,
        playbook: Playbook,
    ) -> AutoRepairResult:
        """
        執行自動修復

        Args:
            incident: 待處理的 Incident
            playbook: 要執行的 Playbook

        Returns:
            AutoRepairResult: 執行結果
        """
        ...


# =============================================================================
# Auto Repair Service Implementation
# =============================================================================


class AutoRepairService:
    """
    自動修復服務實作

    職責:
    - 評估 Incident 是否可自動修復
    - 執行高品質 Playbook
    - 更新執行統計
    """

    # === 安全邊界常數 ===
    MAX_AUTO_REPAIR_RISK = RiskLevel.MEDIUM  # 最高允許自動修復的風險等級
    MAX_AUTO_REPAIR_SEVERITY = Severity.P2  # 最高允許自動修復的嚴重度
    MIN_SIMILARITY_SCORE = 0.7  # 最低相似度門檻

    def __init__(
        self,
        playbook_service: IPlaybookService | None = None,
    ):
        self._playbook_service = playbook_service or get_playbook_service()

    async def evaluate_auto_repair(
        self,
        incident: Incident,
    ) -> AutoRepairDecision:
        """
        評估是否可自動修復

        決策流程:
        1. 檢查 Incident 嚴重度 (P0/P1 需人工)
        2. 從 Playbook 找匹配項
        3. 檢查 Playbook 是否為高品質
        4. 檢查動作風險等級
        """
        logger.info(
            "auto_repair_evaluate_start",
            incident_id=incident.incident_id,
            severity=incident.severity.value if incident.severity else None,
        )

        # 0. 全域熔斷檢查（ADR-039 最優先）
        can_repair, cooldown_reason = await check_global_repair_cooldown(
            incident_id=incident.incident_id,
            affected_services=incident.affected_services or [],
        )
        if not can_repair:
            logger.warning(
                "auto_repair_blocked_global_cooldown",
                incident_id=incident.incident_id,
                reason=cooldown_reason,
            )
            return AutoRepairDecision(
                can_auto_repair=False,
                reason=cooldown_reason,
                blocked_by="GLOBAL_GUARDRAIL",
            )

        # 1. 檢查 Incident 嚴重度
        if incident.severity and incident.severity.value in ["P0", "P1"]:
            logger.info(
                "auto_repair_blocked_severity",
                incident_id=incident.incident_id,
                severity=incident.severity.value,
            )
            return AutoRepairDecision(
                can_auto_repair=False,
                reason=f"Incident 嚴重度 {incident.severity.value} 需要人工審核",
                blocked_by="HIGH_SEVERITY",
            )

        # 2. 提取症狀模式
        symptoms = self._extract_symptoms(incident)

        # 3. 找匹配的 Playbook
        recommendations = await self._playbook_service.get_recommendations(
            symptoms=symptoms,
            top_k=3,
        )

        if not recommendations:
            logger.info(
                "auto_repair_no_playbook_match",
                incident_id=incident.incident_id,
            )
            return AutoRepairDecision(
                can_auto_repair=False,
                reason="未找到匹配的 Playbook",
                blocked_by="NO_MATCH",
            )

        # 4. 檢查最佳匹配
        best_match = recommendations[0]

        # 相似度檢查
        if best_match.similarity_score < self.MIN_SIMILARITY_SCORE:
            return AutoRepairDecision(
                can_auto_repair=False,
                playbook=best_match.playbook,
                reason=f"相似度 {best_match.similarity_score:.0%} 低於門檻 {self.MIN_SIMILARITY_SCORE:.0%}",
                blocked_by="LOW_SIMILARITY",
            )

        # 高品質檢查
        if not best_match.playbook.is_high_quality:
            return AutoRepairDecision(
                can_auto_repair=False,
                playbook=best_match.playbook,
                reason=f"Playbook 尚未達到高品質標準 (成功率: {best_match.playbook.success_rate:.0%}, 執行次數: {best_match.playbook.total_executions})",
                blocked_by="NOT_HIGH_QUALITY",
            )

        # 5. 檢查動作風險等級
        max_risk = self._get_max_risk_level(best_match.playbook)

        if self._risk_exceeds_threshold(max_risk):
            return AutoRepairDecision(
                can_auto_repair=False,
                playbook=best_match.playbook,
                reason=f"Playbook 包含 {max_risk.value} 風險動作，需要人工審核",
                risk_level=max_risk,
                blocked_by="HIGH_RISK",
            )

        # 6. 可以自動修復
        logger.info(
            "auto_repair_approved",
            incident_id=incident.incident_id,
            playbook_id=best_match.playbook.playbook_id,
            similarity=best_match.similarity_score,
            success_rate=best_match.playbook.success_rate,
        )

        return AutoRepairDecision(
            can_auto_repair=True,
            playbook=best_match.playbook,
            reason=f"匹配高品質 Playbook: {best_match.playbook.name} (成功率 {best_match.playbook.success_rate:.0%})",
            risk_level=max_risk,
        )

    async def execute_auto_repair(
        self,
        incident: Incident,
        playbook: Playbook,
    ) -> AutoRepairResult:
        """
        執行自動修復

        流程:
        1. 依序執行 Playbook 中的 repair_steps
        2. 記錄執行結果
        3. 更新 Playbook 統計
        """
        import time

        start_time = time.perf_counter()
        executed_steps: list[str] = []

        logger.info(
            "auto_repair_execute_start",
            incident_id=incident.incident_id,
            playbook_id=playbook.playbook_id,
            steps_count=len(playbook.repair_steps),
        )

        # ADR-039: 記錄全域修復計數（用於熔斷檢查）
        await record_global_repair_action()

        try:
            # 執行每個步驟
            for step in playbook.repair_steps:
                # 安全檢查: 跳過高風險步驟
                if self._risk_exceeds_threshold(step.risk_level):
                    logger.warning(
                        "auto_repair_skip_high_risk_step",
                        step_number=step.step_number,
                        risk_level=step.risk_level.value,
                    )
                    continue

                # 執行步驟
                step_result = await self._execute_step(incident, step)
                executed_steps.append(
                    f"Step {step.step_number}: {step.command[:50]}... -> {step_result}"
                )

            # 更新 Playbook 統計
            await self._playbook_service.record_execution(
                playbook_id=playbook.playbook_id,
                success=True,
            )

            execution_time = int((time.perf_counter() - start_time) * 1000)

            logger.info(
                "auto_repair_execute_success",
                incident_id=incident.incident_id,
                playbook_id=playbook.playbook_id,
                executed_steps=len(executed_steps),
                execution_time_ms=execution_time,
            )

            return AutoRepairResult(
                success=True,
                playbook_id=playbook.playbook_id,
                incident_id=incident.incident_id,
                executed_steps=executed_steps,
                execution_time_ms=execution_time,
            )

        except Exception as e:
            # 更新失敗統計
            await self._playbook_service.record_execution(
                playbook_id=playbook.playbook_id,
                success=False,
            )

            execution_time = int((time.perf_counter() - start_time) * 1000)

            logger.error(
                "auto_repair_execute_failed",
                incident_id=incident.incident_id,
                playbook_id=playbook.playbook_id,
                error=str(e),
            )

            return AutoRepairResult(
                success=False,
                playbook_id=playbook.playbook_id,
                incident_id=incident.incident_id,
                executed_steps=executed_steps,
                error=str(e),
                execution_time_ms=execution_time,
            )

    # === Private Helpers ===

    def _extract_symptoms(self, incident: Incident) -> SymptomPattern:
        """從 Incident 提取症狀模式"""
        alert_names = []
        keywords = []

        if incident.signals:
            for signal in incident.signals:
                alert_names.append(signal.alert_name)
                # 從 annotations 提取關鍵字
                if signal.annotations:
                    for value in signal.annotations.values():
                        if isinstance(value, str) and len(value) < 50:
                            keywords.append(value)

        return SymptomPattern(
            alert_names=alert_names,
            affected_services=incident.affected_services or [],
            severity_range=[incident.severity.value] if incident.severity else ["P2"],
            keywords=keywords[:10],
        )

    def _get_max_risk_level(self, playbook: Playbook) -> RiskLevel:
        """取得 Playbook 中最高的風險等級"""
        risk_order = {
            RiskLevel.LOW: 0,
            RiskLevel.MEDIUM: 1,
            RiskLevel.HIGH: 2,
            RiskLevel.CRITICAL: 3,
        }

        max_risk = RiskLevel.LOW
        for step in playbook.repair_steps:
            if risk_order.get(step.risk_level, 0) > risk_order.get(max_risk, 0):
                max_risk = step.risk_level

        return max_risk

    def _risk_exceeds_threshold(self, risk: RiskLevel) -> bool:
        """檢查風險是否超過自動修復門檻"""
        high_risks = {RiskLevel.HIGH, RiskLevel.CRITICAL}
        return risk in high_risks

    async def _execute_step(self, incident: Incident, step) -> str:
        """
        執行單一修復步驟

        目前整合:
        - kubectl 命令: 透過 ActionExecutor
        - script: 透過 subprocess
        - manual: 跳過 (需人工)
        """
        if step.action_type == ActionType.MANUAL:
            return "SKIPPED (manual step)"

        if step.action_type == ActionType.KUBECTL:
            # 整合 ActionExecutor
            try:
                executor = get_executor()

                # 替換 {target} 為實際目標
                command = step.command
                if incident.affected_services:
                    command = command.replace("{target}", incident.affected_services[0])

                result = await executor.execute_kubectl_command(command)
                return "SUCCESS" if result.success else f"FAILED: {result.error}"

            except ImportError:
                logger.warning("action_executor_not_available")
                return "SKIPPED (executor not available)"

        return "UNKNOWN_ACTION_TYPE"

    # === ADR-037: Tier-based Repair (2026-03-29) ===

    # Tier 分級動作映射
    TIER_ACTIONS = {
        1: ["restart_pod", "restart_container"],  # 臨時修復
        2: ["scale_up", "increase_memory", "adjust_limits"],  # 緩解修復
        3: ["apply_hotfix", "update_config", "patch_deployment"],  # 根因修復
        4: ["create_issue", "notify_team", "schedule_fix"],  # 架構修復
    }

    async def determine_repair_tier(
        self,
        anomaly_key: str,
        frequency: AnomalyFrequency,
    ) -> int:
        """
        根據頻率決定修復 Tier (ADR-037)

        統帥指示 (2026-03-29):
        - "重啟只是治標，不是治本！太常發生的異常必須徹底解決"
        - 根據異常頻率和修復歷史決定應該嘗試的修復層級

        Returns:
            1: 臨時修復 (重啟)
            2: 緩解修復 (擴容)
            3: 根因修復 (配置變更)
            4: 架構修復 (需開發)
        """
        # 取得修復歷史
        counter = get_anomaly_counter()
        stats = await counter.get_all_repair_stats(anomaly_key)

        # 計算重啟次數
        restart_count = stats.get("restart_pod", {}).get("total", 0)
        restart_count += stats.get("restart_container", {}).get("total", 0)

        # Tier 決策邏輯
        if frequency.permanent_fix_applied:
            # 已有永久修復但仍出問題 → 需架構級修復
            logger.info(
                "tier_decision",
                anomaly_key=anomaly_key,
                tier=4,
                reason="permanent_fix_still_failing",
            )
            return 4

        if frequency.escalation_level == "PERMANENT_FIX":
            # 24h 內 ≥10 次 → 根因修復
            logger.info(
                "tier_decision",
                anomaly_key=anomaly_key,
                tier=3,
                reason="escalation_permanent_fix",
            )
            return 3

        if frequency.escalation_level == "ESCALATE":
            # 24h 內 ≥5 次 → 緩解修復
            logger.info(
                "tier_decision",
                anomaly_key=anomaly_key,
                tier=2,
                reason="escalation_escalate",
            )
            return 2

        if restart_count >= 2:
            # 已重啟 2 次 → 升級到緩解
            logger.info(
                "tier_decision",
                anomaly_key=anomaly_key,
                tier=2,
                reason=f"restart_count_{restart_count}",
            )
            return 2

        # 預設臨時修復
        return 1

    def get_tier_actions(self, tier: int) -> list[str]:
        """
        根據 Tier 返回可用修復動作 (ADR-037)
        """
        return self.TIER_ACTIONS.get(tier, self.TIER_ACTIONS[1])

    async def record_repair_result(
        self,
        anomaly_key: str,
        action: str,
        success: bool,
        tier: int = 1,
    ) -> None:
        """
        記錄修復結果到 AnomalyCounter (ADR-037)

        Args:
            anomaly_key: 異常 key
            action: 修復動作
            success: 是否成功
            tier: 修復 Tier
        """
        counter = get_anomaly_counter()
        await counter.record_repair_attempt(anomaly_key, action, success)

        # 如果是 Tier 3 永久修復成功，標記已套用
        if tier >= 3 and success:
            await counter.mark_permanent_fix_applied(
                anomaly_key=anomaly_key,
                fix_description=f"Tier {tier} repair: {action}",
            )

        logger.info(
            "repair_result_recorded",
            anomaly_key=anomaly_key,
            action=action,
            success=success,
            tier=tier,
        )


# =============================================================================
# Singleton
# =============================================================================

_service: AutoRepairService | None = None


def get_auto_repair_service() -> IAutoRepairService:
    """取得 AutoRepairService 單例"""
    global _service
    if _service is None:
        _service = AutoRepairService()
    return _service


def set_auto_repair_service(service: AutoRepairService | None) -> None:
    """注入 AutoRepairService 實例 (用於 DI 或測試)"""
    global _service
    _service = service