fab65e7d7a
Some checks are pending
CD Pipeline / build-and-deploy (push) Has started running
根因：find_by_fingerprint 的 PENDING 匹配條件無時間上限， 2026-04-12 建立的 3 筆 PENDING approval records（hit=77/30/17） 持續吃掉所有同指紋告警，造成 2+ 小時 Telegram 靜音。 修正（approval_db.py）： - PENDING_TTL_HOURS = 24：PENDING 記錄逾 24h 不再收斂新告警 - 原本：OR(status=PENDING, created_at>=30min前) - 修正：OR(PENDING AND created_at>=24h前, created_at>=30min前) 緊急修復：kubectl exec 直接將 7 筆過期 PENDING 記錄設為 expired， 即時恢復 Telegram 告警流（不等部署）。 Phase 6 AI 自我治理閉環（ADR-087）： - feat(db): 新增 ai_governance_events 表 + 3 個 index（base.py + models.py） - feat(svc): ai_slo_calculator.py — 7d 滾動 SLO（success/override/false_neg） - feat(svc): trust_drift_detector.py — Playbook 信任度極端偏態偵測 - feat(job): kb_rot_cleaner.py — K8s API/Prom metric/老舊 incident_case 腐爛清理 - feat(svc): decision_manager.py — 自我降級守衛（SLO 違反 → 提高門檻/保守模式） 2026-04-15 ogt + Claude Sonnet 4.6（亞太） Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
419 lines
16 KiB
Python
419 lines
16 KiB
Python
"""
|
||
AWOOOI AIOps Phase 6 — AI SLO 計算器（決策品質自我監控）
|
||
=========================================================
|
||
職責：滾動計算三大 AI 決策品質 SLO；違反閾值時寫入 ai_governance_events，
|
||
供 decision_manager 自我降級邏輯讀取。
|
||
|
||
三大 SLO（MASTER §3.6 ADR-087）：
|
||
SLO-1 auto_execute_success_rate > 85% （7d 滾動）
|
||
SLO-2 human_override_rate < 20% （7d 滾動）
|
||
SLO-3 verifier_false_neg_rate < 5% （7d 滾動，proxy: 2h 內重複告警）
|
||
|
||
設計原則：
|
||
1. 純讀 + 純寫分離 — calculate() 只讀 DB，save_event() 只寫 DB
|
||
2. 計算失敗 → 保守：假設 SLO 違反，寫 violation 事件
|
||
3. 所有結果快取 Redis（key: ai:slo:latest, TTL 5min），避免高頻查 DB
|
||
4. 不自動解決舊 violation — resolved 只能人工或下次「全部通過」時補填
|
||
|
||
ADR-087: AI 自我治理閉環
|
||
2026-04-15 ogt + Claude Sonnet 4.6（亞太）: Phase 6 初始建立
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from dataclasses import dataclass, field
|
||
from datetime import timedelta
|
||
|
||
import structlog
|
||
from sqlalchemy import func, select, text
|
||
|
||
from src.db.base import get_session_factory
|
||
from src.db.models import AiGovernanceEvent, AutoRepairExecution, ApprovalRecord
|
||
from src.utils.timezone import now_taipei
|
||
|
||
logger = structlog.get_logger(__name__)
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# SLO 閾值（MASTER §3.6 鐵律，修改前需 ADR-087 更新）
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
SLO_AUTO_SUCCESS_MIN: float = 0.85 # auto_execute 成功率下限
|
||
SLO_OVERRIDE_RATE_MAX: float = 0.20 # 人工推翻率上限
|
||
SLO_FALSE_NEG_MAX: float = 0.05 # verifier false negative 上限
|
||
|
||
SLO_WINDOW_DAYS: int = 7 # 滾動視窗（天）
|
||
SLO_MIN_SAMPLES: int = 5 # 最少樣本數，低於此不計算（資料不足）
|
||
|
||
REDIS_KEY = "ai:slo:latest"
|
||
REDIS_TTL_SEC = 300 # 5 分鐘快取
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Data Types
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
@dataclass
|
||
class SloMetric:
|
||
"""單一 SLO 指標"""
|
||
name: str
|
||
value: float | None # None = 樣本不足，跳過
|
||
threshold: float
|
||
direction: str # "above" = 需高於閾值 / "below" = 需低於閾值
|
||
sample_count: int
|
||
violated: bool # 是否違反（None → False，不觸發降級）
|
||
|
||
@property
|
||
def label(self) -> str:
|
||
if self.value is None:
|
||
return f"{self.name}: N/A（樣本 {self.sample_count} < {SLO_MIN_SAMPLES}）"
|
||
pct = f"{self.value:.1%}"
|
||
thr = f"{self.threshold:.0%}"
|
||
op = ">" if self.direction == "above" else "<"
|
||
status = "❌ 違反" if self.violated else "✅ 合規"
|
||
return f"{self.name}: {pct} （需 {op}{thr}） {status}"
|
||
|
||
|
||
@dataclass
|
||
class SloReport:
|
||
"""完整 SLO 計算報告"""
|
||
metrics: list[SloMetric] = field(default_factory=list)
|
||
any_violated: bool = False
|
||
calculated_at: str = field(default_factory=lambda: now_taipei().isoformat())
|
||
window_days: int = SLO_WINDOW_DAYS
|
||
|
||
def to_dict(self) -> dict:
|
||
return {
|
||
"calculated_at": self.calculated_at,
|
||
"window_days": self.window_days,
|
||
"any_violated": self.any_violated,
|
||
"metrics": [
|
||
{
|
||
"name": m.name,
|
||
"value": m.value,
|
||
"threshold": m.threshold,
|
||
"direction": m.direction,
|
||
"sample_count": m.sample_count,
|
||
"violated": m.violated,
|
||
"label": m.label,
|
||
}
|
||
for m in self.metrics
|
||
],
|
||
}
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Main Service
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
class AiSloCalculator:
|
||
"""
|
||
AI 決策品質 SLO 計算器
|
||
|
||
Usage:
|
||
calc = AiSloCalculator()
|
||
report = await calc.calculate()
|
||
if report.any_violated:
|
||
await calc.save_violation_event(report)
|
||
"""
|
||
|
||
async def calculate(self) -> SloReport:
|
||
"""
|
||
計算三大 SLO 指標（7d 滾動視窗）。
|
||
|
||
Returns:
|
||
SloReport（計算失敗時保守回傳 any_violated=True）
|
||
"""
|
||
try:
|
||
since = now_taipei() - timedelta(days=SLO_WINDOW_DAYS)
|
||
|
||
async with get_session_factory()() as session:
|
||
slo1 = await self._calc_auto_success_rate(session, since)
|
||
slo2 = await self._calc_human_override_rate(session, since)
|
||
slo3 = await self._calc_false_neg_rate(session, since)
|
||
|
||
metrics = [slo1, slo2, slo3]
|
||
any_violated = any(m.violated for m in metrics)
|
||
|
||
report = SloReport(
|
||
metrics=metrics,
|
||
any_violated=any_violated,
|
||
)
|
||
|
||
logger.info(
|
||
"slo_calculated",
|
||
any_violated=any_violated,
|
||
slo1=slo1.value,
|
||
slo2=slo2.value,
|
||
slo3=slo3.value,
|
||
)
|
||
return report
|
||
|
||
except Exception as e:
|
||
logger.error("slo_calculation_error", error=str(e))
|
||
# 保守：計算失敗 → 假設違反
|
||
violated_metric = SloMetric(
|
||
name="calculation_error",
|
||
value=None,
|
||
threshold=0.0,
|
||
direction="above",
|
||
sample_count=0,
|
||
violated=True,
|
||
)
|
||
return SloReport(
|
||
metrics=[violated_metric],
|
||
any_violated=True,
|
||
)
|
||
|
||
async def get_cached_report(self) -> SloReport | None:
|
||
"""從 Redis 讀取最近一次 SLO 報告（5min 快取）。"""
|
||
try:
|
||
from src.core.redis_client import get_redis
|
||
redis = get_redis()
|
||
raw = await redis.get(REDIS_KEY)
|
||
if raw:
|
||
data = json.loads(raw)
|
||
metrics = [
|
||
SloMetric(
|
||
name=m["name"],
|
||
value=m["value"],
|
||
threshold=m["threshold"],
|
||
direction=m["direction"],
|
||
sample_count=m["sample_count"],
|
||
violated=m["violated"],
|
||
)
|
||
for m in data.get("metrics", [])
|
||
]
|
||
return SloReport(
|
||
metrics=metrics,
|
||
any_violated=data.get("any_violated", False),
|
||
calculated_at=data.get("calculated_at", ""),
|
||
window_days=data.get("window_days", SLO_WINDOW_DAYS),
|
||
)
|
||
except Exception as e:
|
||
logger.warning("slo_cache_read_error", error=str(e))
|
||
return None
|
||
|
||
async def cache_report(self, report: SloReport) -> None:
|
||
"""將 SLO 報告存入 Redis 快取（TTL 5min）。"""
|
||
try:
|
||
from src.core.redis_client import get_redis
|
||
redis = get_redis()
|
||
await redis.set(REDIS_KEY, json.dumps(report.to_dict()), ex=REDIS_TTL_SEC)
|
||
except Exception as e:
|
||
logger.warning("slo_cache_write_error", error=str(e))
|
||
|
||
async def save_violation_event(self, report: SloReport) -> None:
|
||
"""
|
||
將 SLO 違反寫入 ai_governance_events。
|
||
|
||
只在 any_violated=True 時呼叫。不管舊違反是否解決。
|
||
"""
|
||
try:
|
||
async with get_session_factory()() as session:
|
||
event = AiGovernanceEvent(
|
||
event_type="slo_violation",
|
||
details=report.to_dict(),
|
||
resolved=False,
|
||
)
|
||
session.add(event)
|
||
await session.commit()
|
||
logger.warning(
|
||
"slo_violation_recorded",
|
||
violated_metrics=[m.name for m in report.metrics if m.violated],
|
||
)
|
||
except Exception as e:
|
||
logger.error("slo_violation_save_error", error=str(e))
|
||
|
||
async def run(self) -> SloReport:
|
||
"""
|
||
完整執行：計算 → 快取 → 如違反則寫事件。
|
||
|
||
Returns:
|
||
SloReport
|
||
"""
|
||
report = await self.calculate()
|
||
await self.cache_report(report)
|
||
if report.any_violated:
|
||
await self.save_violation_event(report)
|
||
return report
|
||
|
||
# ──────────────────────────────────────────────────────────────────────────
|
||
# Private: SLO 計算方法
|
||
# ──────────────────────────────────────────────────────────────────────────
|
||
|
||
async def _calc_auto_success_rate(self, session, since) -> SloMetric:
|
||
"""SLO-1: auto_repair_executions 7d 成功率。"""
|
||
try:
|
||
total_q = await session.execute(
|
||
select(func.count()).where(
|
||
AutoRepairExecution.created_at >= since
|
||
)
|
||
)
|
||
total: int = total_q.scalar() or 0
|
||
|
||
if total < SLO_MIN_SAMPLES:
|
||
return SloMetric(
|
||
name="auto_execute_success_rate",
|
||
value=None,
|
||
threshold=SLO_AUTO_SUCCESS_MIN,
|
||
direction="above",
|
||
sample_count=total,
|
||
violated=False,
|
||
)
|
||
|
||
success_q = await session.execute(
|
||
select(func.count()).where(
|
||
AutoRepairExecution.created_at >= since,
|
||
AutoRepairExecution.success.is_(True),
|
||
)
|
||
)
|
||
success: int = success_q.scalar() or 0
|
||
rate = success / total
|
||
|
||
return SloMetric(
|
||
name="auto_execute_success_rate",
|
||
value=rate,
|
||
threshold=SLO_AUTO_SUCCESS_MIN,
|
||
direction="above",
|
||
sample_count=total,
|
||
violated=rate < SLO_AUTO_SUCCESS_MIN,
|
||
)
|
||
except Exception as e:
|
||
logger.warning("slo1_calc_error", error=str(e))
|
||
return SloMetric(
|
||
name="auto_execute_success_rate",
|
||
value=None, threshold=SLO_AUTO_SUCCESS_MIN,
|
||
direction="above", sample_count=0, violated=False,
|
||
)
|
||
|
||
async def _calc_human_override_rate(self, session, since) -> SloMetric:
|
||
"""
|
||
SLO-2: 人工推翻率 = AI 提案被 rejected / 總 AI 提案。
|
||
|
||
rejected = approval_records.status = 'rejected'
|
||
AI 提案 = requested_by LIKE 'ai_%' or 'system'
|
||
"""
|
||
try:
|
||
ai_q = await session.execute(
|
||
select(func.count()).where(
|
||
ApprovalRecord.created_at >= since,
|
||
)
|
||
)
|
||
total: int = ai_q.scalar() or 0
|
||
|
||
if total < SLO_MIN_SAMPLES:
|
||
return SloMetric(
|
||
name="human_override_rate",
|
||
value=None,
|
||
threshold=SLO_OVERRIDE_RATE_MAX,
|
||
direction="below",
|
||
sample_count=total,
|
||
violated=False,
|
||
)
|
||
|
||
rejected_q = await session.execute(
|
||
select(func.count()).where(
|
||
ApprovalRecord.created_at >= since,
|
||
ApprovalRecord.status == "rejected",
|
||
)
|
||
)
|
||
rejected: int = rejected_q.scalar() or 0
|
||
rate = rejected / total
|
||
|
||
return SloMetric(
|
||
name="human_override_rate",
|
||
value=rate,
|
||
threshold=SLO_OVERRIDE_RATE_MAX,
|
||
direction="below",
|
||
sample_count=total,
|
||
violated=rate > SLO_OVERRIDE_RATE_MAX,
|
||
)
|
||
except Exception as e:
|
||
logger.warning("slo2_calc_error", error=str(e))
|
||
return SloMetric(
|
||
name="human_override_rate",
|
||
value=None, threshold=SLO_OVERRIDE_RATE_MAX,
|
||
direction="below", sample_count=0, violated=False,
|
||
)
|
||
|
||
async def _calc_false_neg_rate(self, session, since) -> SloMetric:
|
||
"""
|
||
SLO-3: Verifier false negative（代理指標）。
|
||
|
||
計算方式：auto_repair 執行後 2 小時內同 incident_id 再次出現
|
||
在 auto_repair_executions 中（= 修好了又壞 = verifier 誤判為成功）。
|
||
|
||
使用 SQL window function：
|
||
- 找出 success=True 的執行
|
||
- 計算同 incident_id 下是否有後續 failed 執行在 2h 內
|
||
"""
|
||
try:
|
||
result = await session.execute(
|
||
text("""
|
||
WITH success_runs AS (
|
||
SELECT incident_id, created_at
|
||
FROM auto_repair_executions
|
||
WHERE success = TRUE
|
||
AND created_at >= :since
|
||
),
|
||
false_negs AS (
|
||
SELECT DISTINCT s.incident_id
|
||
FROM success_runs s
|
||
JOIN auto_repair_executions f
|
||
ON f.incident_id = s.incident_id
|
||
AND f.success = FALSE
|
||
AND f.created_at > s.created_at
|
||
AND f.created_at <= s.created_at + INTERVAL '2 hours'
|
||
)
|
||
SELECT
|
||
(SELECT COUNT(*) FROM success_runs) AS total_success,
|
||
(SELECT COUNT(*) FROM false_negs) AS false_neg_count
|
||
"""),
|
||
{"since": since},
|
||
)
|
||
row = result.fetchone()
|
||
total_success: int = row[0] if row else 0
|
||
false_neg: int = row[1] if row else 0
|
||
|
||
if total_success < SLO_MIN_SAMPLES:
|
||
return SloMetric(
|
||
name="verifier_false_neg_rate",
|
||
value=None,
|
||
threshold=SLO_FALSE_NEG_MAX,
|
||
direction="below",
|
||
sample_count=total_success,
|
||
violated=False,
|
||
)
|
||
|
||
rate = false_neg / total_success
|
||
return SloMetric(
|
||
name="verifier_false_neg_rate",
|
||
value=rate,
|
||
threshold=SLO_FALSE_NEG_MAX,
|
||
direction="below",
|
||
sample_count=total_success,
|
||
violated=rate > SLO_FALSE_NEG_MAX,
|
||
)
|
||
except Exception as e:
|
||
logger.warning("slo3_calc_error", error=str(e))
|
||
return SloMetric(
|
||
name="verifier_false_neg_rate",
|
||
value=None, threshold=SLO_FALSE_NEG_MAX,
|
||
direction="below", sample_count=0, violated=False,
|
||
)
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Singleton
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
_calculator: AiSloCalculator | None = None
|
||
|
||
|
||
def get_ai_slo_calculator() -> AiSloCalculator:
|
||
global _calculator
|
||
if _calculator is None:
|
||
_calculator = AiSloCalculator()
|
||
return _calculator
|