""" AWOOOI AIOps Phase 6 — AI SLO 計算器(決策品質自我監控) ========================================================= 職責:滾動計算三大 AI 決策品質 SLO;違反閾值時寫入 ai_governance_events, 供 decision_manager 自我降級邏輯讀取。 三大 SLO(MASTER §3.6 ADR-087): SLO-1 auto_execute_success_rate > 85% (7d 滾動) SLO-2 human_override_rate < 20% (7d 滾動) SLO-3 verifier_false_neg_rate < 5% (7d 滾動,proxy: 2h 內重複告警) 設計原則: 1. 純讀 + 純寫分離 — calculate() 只讀 DB,save_event() 只寫 DB 2. 計算失敗 → 保守:假設 SLO 違反,寫 violation 事件 3. 所有結果快取 Redis(key: ai:slo:latest, TTL 5min),避免高頻查 DB 4. 不自動解決舊 violation — resolved 只能人工或下次「全部通過」時補填 ADR-087: AI 自我治理閉環 2026-04-15 ogt + Claude Sonnet 4.6(亞太): Phase 6 初始建立 """ from __future__ import annotations import json from dataclasses import dataclass, field from datetime import timedelta import structlog from sqlalchemy import func, select, text from src.db.base import get_session_factory from src.db.models import AiGovernanceEvent, AutoRepairExecution, ApprovalRecord from src.utils.timezone import now_taipei logger = structlog.get_logger(__name__) # ───────────────────────────────────────────────────────────────────────────── # SLO 閾值(MASTER §3.6 鐵律,修改前需 ADR-087 更新) # ───────────────────────────────────────────────────────────────────────────── SLO_AUTO_SUCCESS_MIN: float = 0.85 # auto_execute 成功率下限 SLO_OVERRIDE_RATE_MAX: float = 0.20 # 人工推翻率上限 SLO_FALSE_NEG_MAX: float = 0.05 # verifier false negative 上限 SLO_WINDOW_DAYS: int = 7 # 滾動視窗(天) SLO_MIN_SAMPLES: int = 5 # 最少樣本數,低於此不計算(資料不足) REDIS_KEY = "ai:slo:latest" REDIS_TTL_SEC = 300 # 5 分鐘快取 # ───────────────────────────────────────────────────────────────────────────── # Data Types # ───────────────────────────────────────────────────────────────────────────── @dataclass class SloMetric: """單一 SLO 指標""" name: str value: float | None # None = 樣本不足,跳過 threshold: float direction: str # "above" = 需高於閾值 / "below" = 需低於閾值 sample_count: int violated: bool # 是否違反(None → False,不觸發降級) @property def label(self) -> str: if self.value is None: return f"{self.name}: N/A(樣本 {self.sample_count} < {SLO_MIN_SAMPLES})" pct = f"{self.value:.1%}" thr = f"{self.threshold:.0%}" op = ">" if self.direction == "above" else "<" status = "❌ 違反" if self.violated else "✅ 合規" return f"{self.name}: {pct} (需 {op}{thr}) {status}" @dataclass class SloReport: """完整 SLO 計算報告""" metrics: list[SloMetric] = field(default_factory=list) any_violated: bool = False calculated_at: str = field(default_factory=lambda: now_taipei().isoformat()) window_days: int = SLO_WINDOW_DAYS def to_dict(self) -> dict: return { "calculated_at": self.calculated_at, "window_days": self.window_days, "any_violated": self.any_violated, "metrics": [ { "name": m.name, "value": m.value, "threshold": m.threshold, "direction": m.direction, "sample_count": m.sample_count, "violated": m.violated, "label": m.label, } for m in self.metrics ], } # ───────────────────────────────────────────────────────────────────────────── # Main Service # ───────────────────────────────────────────────────────────────────────────── class AiSloCalculator: """ AI 決策品質 SLO 計算器 Usage: calc = AiSloCalculator() report = await calc.calculate() if report.any_violated: await calc.save_violation_event(report) """ async def calculate(self) -> SloReport: """ 計算三大 SLO 指標(7d 滾動視窗)。 Returns: SloReport(計算失敗時保守回傳 any_violated=True) """ try: since = now_taipei() - timedelta(days=SLO_WINDOW_DAYS) async with get_session_factory()() as session: slo1 = await self._calc_auto_success_rate(session, since) slo2 = await self._calc_human_override_rate(session, since) slo3 = await self._calc_false_neg_rate(session, since) metrics = [slo1, slo2, slo3] any_violated = any(m.violated for m in metrics) report = SloReport( metrics=metrics, any_violated=any_violated, ) logger.info( "slo_calculated", any_violated=any_violated, slo1=slo1.value, slo2=slo2.value, slo3=slo3.value, ) return report except Exception as e: logger.error("slo_calculation_error", error=str(e)) # 保守:計算失敗 → 假設違反 violated_metric = SloMetric( name="calculation_error", value=None, threshold=0.0, direction="above", sample_count=0, violated=True, ) return SloReport( metrics=[violated_metric], any_violated=True, ) async def get_cached_report(self) -> SloReport | None: """從 Redis 讀取最近一次 SLO 報告(5min 快取)。""" try: from src.core.redis_client import get_redis redis = get_redis() raw = await redis.get(REDIS_KEY) if raw: data = json.loads(raw) metrics = [ SloMetric( name=m["name"], value=m["value"], threshold=m["threshold"], direction=m["direction"], sample_count=m["sample_count"], violated=m["violated"], ) for m in data.get("metrics", []) ] return SloReport( metrics=metrics, any_violated=data.get("any_violated", False), calculated_at=data.get("calculated_at", ""), window_days=data.get("window_days", SLO_WINDOW_DAYS), ) except Exception as e: logger.warning("slo_cache_read_error", error=str(e)) return None async def cache_report(self, report: SloReport) -> None: """將 SLO 報告存入 Redis 快取(TTL 5min)。""" try: from src.core.redis_client import get_redis redis = get_redis() await redis.set(REDIS_KEY, json.dumps(report.to_dict()), ex=REDIS_TTL_SEC) except Exception as e: logger.warning("slo_cache_write_error", error=str(e)) async def save_violation_event(self, report: SloReport) -> None: """ 將 SLO 違反寫入 ai_governance_events。 只在 any_violated=True 時呼叫。不管舊違反是否解決。 """ try: async with get_session_factory()() as session: event = AiGovernanceEvent( event_type="slo_violation", details=report.to_dict(), resolved=False, ) session.add(event) await session.commit() logger.warning( "slo_violation_recorded", violated_metrics=[m.name for m in report.metrics if m.violated], ) except Exception as e: logger.error("slo_violation_save_error", error=str(e)) async def run(self) -> SloReport: """ 完整執行:計算 → 快取 → 如違反則寫事件。 Returns: SloReport """ report = await self.calculate() await self.cache_report(report) if report.any_violated: await self.save_violation_event(report) return report # ────────────────────────────────────────────────────────────────────────── # Private: SLO 計算方法 # ────────────────────────────────────────────────────────────────────────── async def _calc_auto_success_rate(self, session, since) -> SloMetric: """SLO-1: auto_repair_executions 7d 成功率。""" try: total_q = await session.execute( select(func.count()).where( AutoRepairExecution.created_at >= since ) ) total: int = total_q.scalar() or 0 if total < SLO_MIN_SAMPLES: return SloMetric( name="auto_execute_success_rate", value=None, threshold=SLO_AUTO_SUCCESS_MIN, direction="above", sample_count=total, violated=False, ) success_q = await session.execute( select(func.count()).where( AutoRepairExecution.created_at >= since, AutoRepairExecution.success.is_(True), ) ) success: int = success_q.scalar() or 0 rate = success / total return SloMetric( name="auto_execute_success_rate", value=rate, threshold=SLO_AUTO_SUCCESS_MIN, direction="above", sample_count=total, violated=rate < SLO_AUTO_SUCCESS_MIN, ) except Exception as e: logger.warning("slo1_calc_error", error=str(e)) return SloMetric( name="auto_execute_success_rate", value=None, threshold=SLO_AUTO_SUCCESS_MIN, direction="above", sample_count=0, violated=False, ) async def _calc_human_override_rate(self, session, since) -> SloMetric: """ SLO-2: 人工推翻率 = AI 提案被 rejected / 總 AI 提案。 rejected = approval_records.status = 'rejected' AI 提案 = requested_by LIKE 'ai_%' or 'system' """ try: ai_q = await session.execute( select(func.count()).where( ApprovalRecord.created_at >= since, ) ) total: int = ai_q.scalar() or 0 if total < SLO_MIN_SAMPLES: return SloMetric( name="human_override_rate", value=None, threshold=SLO_OVERRIDE_RATE_MAX, direction="below", sample_count=total, violated=False, ) rejected_q = await session.execute( select(func.count()).where( ApprovalRecord.created_at >= since, ApprovalRecord.status == "rejected", ) ) rejected: int = rejected_q.scalar() or 0 rate = rejected / total return SloMetric( name="human_override_rate", value=rate, threshold=SLO_OVERRIDE_RATE_MAX, direction="below", sample_count=total, violated=rate > SLO_OVERRIDE_RATE_MAX, ) except Exception as e: logger.warning("slo2_calc_error", error=str(e)) return SloMetric( name="human_override_rate", value=None, threshold=SLO_OVERRIDE_RATE_MAX, direction="below", sample_count=0, violated=False, ) async def _calc_false_neg_rate(self, session, since) -> SloMetric: """ SLO-3: Verifier false negative(代理指標)。 計算方式:auto_repair 執行後 2 小時內同 incident_id 再次出現 在 auto_repair_executions 中(= 修好了又壞 = verifier 誤判為成功)。 使用 SQL window function: - 找出 success=True 的執行 - 計算同 incident_id 下是否有後續 failed 執行在 2h 內 """ try: result = await session.execute( text(""" WITH success_runs AS ( SELECT incident_id, created_at FROM auto_repair_executions WHERE success = TRUE AND created_at >= :since ), false_negs AS ( SELECT DISTINCT s.incident_id FROM success_runs s JOIN auto_repair_executions f ON f.incident_id = s.incident_id AND f.success = FALSE AND f.created_at > s.created_at AND f.created_at <= s.created_at + INTERVAL '2 hours' ) SELECT (SELECT COUNT(*) FROM success_runs) AS total_success, (SELECT COUNT(*) FROM false_negs) AS false_neg_count """), {"since": since}, ) row = result.fetchone() total_success: int = row[0] if row else 0 false_neg: int = row[1] if row else 0 if total_success < SLO_MIN_SAMPLES: return SloMetric( name="verifier_false_neg_rate", value=None, threshold=SLO_FALSE_NEG_MAX, direction="below", sample_count=total_success, violated=False, ) rate = false_neg / total_success return SloMetric( name="verifier_false_neg_rate", value=rate, threshold=SLO_FALSE_NEG_MAX, direction="below", sample_count=total_success, violated=rate > SLO_FALSE_NEG_MAX, ) except Exception as e: logger.warning("slo3_calc_error", error=str(e)) return SloMetric( name="verifier_false_neg_rate", value=None, threshold=SLO_FALSE_NEG_MAX, direction="below", sample_count=0, violated=False, ) # ───────────────────────────────────────────────────────────────────────────── # Singleton # ───────────────────────────────────────────────────────────────────────────── _calculator: AiSloCalculator | None = None def get_ai_slo_calculator() -> AiSloCalculator: global _calculator if _calculator is None: _calculator = AiSloCalculator() return _calculator