awoooi/apps/api/src/services/governance_agent.py

"""AI 自我治理 Agent

四項自檢，每 1 小時執行一次：
1. trust_drift         — Playbook trust_score < 0.2 → 告警建議廢棄
2. knowledge_degradation — KM 7 天未更新 > 20% 總量 → 告警知識衰退
3. llm_hallucination   — 近 100 筆 evidence verification_result=failed 比例 > 10%
4. execution_blast_radius — 近 100 筆 auto_repair_executions.success=False 比例 > 15%

所有 check 互相隔離（try/except），任一失敗不阻斷其他項目。

2026-04-26 P2.2 by Claude
"""

from __future__ import annotations

import asyncio
from datetime import timedelta
from typing import Any

import structlog
from sqlalchemy import func, select

from src.db.base import get_db_context
from src.db.models import (
    AiGovernanceEvent,
    AutoRepairExecution,
    IncidentEvidence,
    KnowledgeEntryRecord,
    PlaybookRecord,
)
from src.models.knowledge import EntryStatus
from src.utils.timezone import now_taipei

logger = structlog.get_logger(__name__)

# =============================================================================
# 閾值常數
# =============================================================================
TRUST_DRIFT_THRESHOLD = 0.2          # playbook trust_score 低於此值 → 告警
KM_STALE_DAYS = 7                     # 知識條目超過幾天未更新視為陳舊
KM_STALE_RATIO = 0.20                 # 陳舊比例超過此值 → 告警
HALLUCINATION_RATE_THRESHOLD = 0.10   # LLM verification failed 比例超過此值 → 告警
EXECUTION_FAIL_RATE_THRESHOLD = 0.15  # 執行失敗比例超過此值 → 告警
RECENT_LIMIT = 100                    # 最近幾筆做統計


# =============================================================================
# GovernanceAgent
# =============================================================================

class GovernanceAgent:
    """AI 自我治理 Agent — 4 項自檢 + 1h 排程

    2026-04-26 P2.2 by Claude
    """

    def __init__(self, alerter=None) -> None:
        # alerter: FailoverAlerter instance（可注入，預設從 singleton 取得）
        self._alerter = alerter

    # =========================================================================
    # 1. Playbook 信任度漂移
    # =========================================================================

    async def check_trust_drift(self) -> dict[str, Any]:
        """Playbook trust_score < 0.2 → 告警建議廢棄

        2026-04-26 P2.2 by Claude
        """
        async with get_db_context() as db:
            result = await db.execute(
                select(PlaybookRecord).where(
                    PlaybookRecord.status.not_in(["deprecated", "archived"])
                )
            )
            all_records = result.scalars().all()

        total = len(all_records)
        drifted = [r for r in all_records if float(r.trust_score) < TRUST_DRIFT_THRESHOLD]
        drifted_ids = [r.playbook_id for r in drifted[:10]]

        if drifted:
            await self._alert(
                "trust_drift",
                {
                    "drifted_count": len(drifted),
                    "total_playbooks": total,
                    "playbook_ids": drifted_ids,
                    "threshold": TRUST_DRIFT_THRESHOLD,
                },
            )

        logger.info(
            "governance_trust_drift_checked",
            total=total,
            drifted=len(drifted),
        )
        return {"checked": total, "drifted": len(drifted)}

    # =========================================================================
    # 2. 知識庫衰退
    # =========================================================================

    async def check_knowledge_degradation(self) -> dict[str, Any]:
        """KM 7 天未更新 > 20% 總量 → 告警知識衰退

        2026-04-26 P2.2 by Claude
        """
        stale_cutoff = now_taipei() - timedelta(days=KM_STALE_DAYS)

        async with get_db_context() as db:
            # 非 archived 總數
            total_result = await db.execute(
                select(func.count()).select_from(KnowledgeEntryRecord).where(
                    KnowledgeEntryRecord.status != EntryStatus.ARCHIVED
                )
            )
            total = total_result.scalar() or 0

            # 7 天內未更新（updated_at < cutoff）且非 archived
            stale_result = await db.execute(
                select(func.count()).select_from(KnowledgeEntryRecord).where(
                    KnowledgeEntryRecord.status != EntryStatus.ARCHIVED,
                    KnowledgeEntryRecord.updated_at < stale_cutoff,
                )
            )
            stale = stale_result.scalar() or 0

        ratio = stale / total if total > 0 else 0.0

        if total > 0 and ratio > KM_STALE_RATIO:
            await self._alert(
                "knowledge_degradation",
                {
                    "stale_count": stale,
                    "total_count": total,
                    "stale_ratio": round(ratio, 3),
                    "threshold": KM_STALE_RATIO,
                    "stale_days": KM_STALE_DAYS,
                },
            )

        logger.info(
            "governance_knowledge_degradation_checked",
            total=total,
            stale=stale,
            ratio=round(ratio, 3),
        )
        return {"total": total, "stale": stale, "ratio": round(ratio, 3)}

    # =========================================================================
    # 3. LLM 幻覺率
    # =========================================================================

    async def check_llm_hallucination(self) -> dict[str, Any]:
        """最近 100 筆 IncidentEvidence verification_result=failed 比例 > 10% → 告警

        verification_result 可能值：success / degraded / failed / timeout
        只有 'failed' 視為幻覺（LLM 判斷錯誤導致執行後驗證失敗）

        2026-04-26 P2.2 by Claude
        """
        async with get_db_context() as db:
            # 取最近 RECENT_LIMIT 筆有 verification_result 的記錄
            result = await db.execute(
                select(IncidentEvidence.verification_result)
                .where(IncidentEvidence.verification_result.is_not(None))
                .order_by(IncidentEvidence.collected_at.desc())
                .limit(RECENT_LIMIT)
            )
            rows = result.scalars().all()

        total = len(rows)
        if total == 0:
            logger.info("governance_hallucination_checked", total=0, rate=0.0)
            return {"total": 0, "failed": 0, "rate": 0.0}

        failed = sum(1 for r in rows if r == "failed")
        rate = failed / total

        if rate > HALLUCINATION_RATE_THRESHOLD:
            await self._alert(
                "llm_hallucination",
                {
                    "failed_count": failed,
                    "total_checked": total,
                    "hallucination_rate": round(rate, 3),
                    "threshold": HALLUCINATION_RATE_THRESHOLD,
                },
            )

        logger.info(
            "governance_hallucination_checked",
            total=total,
            failed=failed,
            rate=round(rate, 3),
        )
        return {"total": total, "failed": failed, "rate": round(rate, 3)}

    # =========================================================================
    # 4. 執行失敗率 (Blast Radius)
    # =========================================================================

    async def check_execution_blast_radius(self) -> dict[str, Any]:
        """最近 100 筆 AutoRepairExecution.success=False 比例 > 15% → 告警

        2026-04-26 P2.2 by Claude
        """
        async with get_db_context() as db:
            result = await db.execute(
                select(AutoRepairExecution.success)
                .order_by(AutoRepairExecution.created_at.desc())
                .limit(RECENT_LIMIT)
            )
            rows = result.scalars().all()

        total = len(rows)
        if total == 0:
            logger.info("governance_blast_radius_checked", total=0, rate=0.0)
            return {"total": 0, "failed": 0, "rate": 0.0}

        failed = sum(1 for r in rows if not r)
        rate = failed / total

        if rate > EXECUTION_FAIL_RATE_THRESHOLD:
            await self._alert(
                "execution_blast_radius",
                {
                    "failed_count": failed,
                    "total_executions": total,
                    "failure_rate": round(rate, 3),
                    "threshold": EXECUTION_FAIL_RATE_THRESHOLD,
                },
            )

        logger.info(
            "governance_blast_radius_checked",
            total=total,
            failed=failed,
            rate=round(rate, 3),
        )
        return {"total": total, "failed": failed, "rate": round(rate, 3)}

    # =========================================================================
    # 全跑（exception 隔離）
    # =========================================================================

    async def run_self_check(self) -> dict[str, Any]:
        """4 項全跑，每項獨立 try/except 隔離，任一失敗不影響其他項目

        2026-04-26 P2.2 by Claude
        """
        results: dict[str, Any] = {}
        checks = [
            ("trust_drift", self.check_trust_drift),
            ("knowledge_degradation", self.check_knowledge_degradation),
            ("llm_hallucination", self.check_llm_hallucination),
            ("execution_blast_radius", self.check_execution_blast_radius),
        ]

        for check_name, check_func in checks:
            try:
                results[check_name] = await check_func()
            except Exception as e:
                logger.exception(
                    "governance_check_failed",
                    check=check_name,
                    error=str(e),
                )
                results[check_name] = {"error": str(e)}

        # 2026-04-27 Wave8-X3 by Claude — B8 全失敗聚合告警
        # ≥3 項失敗代表治理機制本身故障，必須送出緊急告警
        failed_checks = [k for k, v in results.items() if isinstance(v, dict) and "error" in v]
        if len(failed_checks) >= 3:
            try:
                await self._alert(
                    "governance_self_failure",
                    {
                        "failed_checks": failed_checks,
                        "total_checks": 4,
                        "errors": {k: results[k].get("error") for k in failed_checks},
                    },
                )
            except Exception:
                logger.exception("governance_self_failure_alert_failed")

        logger.info("governance_self_check_complete", results=results)
        return results

    # =========================================================================
    # 告警輸出
    # =========================================================================

    async def _alert(self, event_type: str, payload: dict[str, Any]) -> None:
        """structlog 告警 + PG 持久化 + Telegram 推送（via FailoverAlerter）

        2026-04-26 P2.2 by Claude
        2026-04-26 P2-DB-Fix by Claude — db-expert P0 三修（P0.1）: 補 PG 寫入 ai_governance_events
        ADR-085 鐵律：AI 學習成果不可存 Cache，必須落地 PG
        """
        # 1. 寫 PG（ADR-085 鐵律 — 失敗不阻斷主流程）
        try:
            from sqlalchemy import insert as _sa_insert
            async with get_db_context() as db:
                await db.execute(
                    _sa_insert(AiGovernanceEvent).values(
                        event_type=event_type,
                        details=payload,
                    )
                )
                await db.commit()
        except Exception as _pg_err:
            logger.warning("governance_pg_write_failed", error=str(_pg_err))

        # 2. structlog（保留既有行為）
        logger.warning("governance_alert", event_type=event_type, **payload)

        # Lazy import：延遲到實際呼叫時才取 alerter，避免啟動時循環依賴
        alerter = self._alerter
        if alerter is None:
            try:
                from src.services.failover_alerter import get_failover_alerter
                alerter = get_failover_alerter()
            except Exception as e:
                logger.warning("governance_alerter_get_failed", error=str(e))
                return

        try:
            await alerter.alert_governance(event_type, payload)
        except Exception as e:
            logger.warning("governance_telegram_alert_failed", error=str(e))


# =============================================================================
# Singleton + 排程迴圈
# =============================================================================

_agent: GovernanceAgent | None = None


def get_governance_agent() -> GovernanceAgent:
    """取得 GovernanceAgent singleton

    2026-04-26 P2.2 by Claude
    """
    global _agent
    if _agent is None:
        _agent = GovernanceAgent()
    return _agent


def reset_governance_agent() -> None:
    """重置 singleton（測試用）

    2026-04-26 P2.2 by Claude
    """
    global _agent
    _agent = None


async def run_governance_loop(interval_seconds: int = 3600) -> None:
    """每 1 小時執行一次 GovernanceAgent.run_self_check()

    沿用 main.py 的 asyncio.create_task + sleep 迴圈模式（無 APScheduler）。
    coalesce 效果：每次 sleep interval_seconds，不堆積多次執行。

    2026-04-26 P2.2 by Claude
    """
    agent = get_governance_agent()
    while True:
        try:
            await agent.run_self_check()
        except Exception as e:
            logger.warning("governance_loop_error", error=str(e))
        await asyncio.sleep(interval_seconds)