ewoooc/routes/admin_observability_routes.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
routes/admin_observability_routes.py
Operation Ollama-First v5.0 / Phase 27 — Admin Observability Dashboard

提供 admin 介面看戰役累積的觀測資料：
  /observability/ai_calls              — ai_calls 即時查詢（含篩選 / 圖表）
  /observability/promotion_review      — Phase 28 PromotionGate 待審核列表
  /observability/quality_trend         — Phase 25 caller 反饋趨勢
  /observability/host_health           — 三主機 Ollama + MCP 健康度

設計原則：
- 純讀（除了 promotion approve/reject 是 mutation）
- 失敗安全：DB 失敗回空清單 + 警告 banner
- 每頁 100 筆分頁，無限捲動
- 不暴露 secret / prompt 原文
"""

from datetime import datetime, timedelta
from flask import Blueprint, render_template, request, jsonify
from sqlalchemy import text as sa_text

from auth import login_required, get_current_user
from database.manager import get_session


admin_observability_bp = Blueprint(
    'admin_observability',
    __name__,
    url_prefix='/observability',
)


# ─────────────────────────────────────────────────────────────────────────────
# /observability/overview — Phase 45 總覽（單頁聚合 6 項 KPI）
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/')
@admin_observability_bp.route('/overview')
@login_required
def observability_overview():
    """Phase 45 — 觀測台總覽：一頁式聚合 6 個 sub-page 的關鍵 KPI。

    對應 Phase 44 daily Telegram summary 的 web 版本，做為 sidebar 入口頁。
    所有區塊失敗安全：個別 query 失敗只跳過該卡片，不擋整頁渲染。
    """
    from datetime import datetime as _dt
    today = _dt.now()
    month_start = _dt(today.year, today.month, 1)
    summary = {}

    session = get_session()
    try:
        # 三主機 24h 在線率
        try:
            host_rows = session.execute(
                sa_text("""
                    SELECT host_label, COUNT(*) AS total,
                           COUNT(*) FILTER (WHERE healthy) AS up,
                           COALESCE(AVG(response_ms) FILTER (WHERE healthy), 0) AS avg_ms
                    FROM host_health_probes
                    WHERE probed_at >= NOW() - INTERVAL '24 hours'
                    GROUP BY host_label ORDER BY host_label
                """),
            ).fetchall()
            summary['hosts'] = [
                {
                    'label': r[0],
                    'total': int(r[1] or 0),
                    'up': int(r[2] or 0),
                    'avg_ms': int(r[3] or 0),
                    'uptime_pct': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0,
                }
                for r in host_rows
            ]
        except Exception:
            summary['hosts'] = []

        # AI 呼叫 24h
        try:
            ai = session.execute(
                sa_text("""
                    SELECT COUNT(*), COALESCE(SUM(input_tokens + output_tokens), 0),
                           COALESCE(SUM(cost_usd), 0),
                           COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')),
                           COUNT(*) FILTER (WHERE rag_hit),
                           COUNT(*) FILTER (WHERE cache_hit)
                    FROM ai_calls
                    WHERE called_at >= NOW() - INTERVAL '24 hours'
                """),
            ).fetchone()
            total = int(ai[0] or 0)
            summary['ai_calls'] = {
                'total': total,
                'tokens': int(ai[1] or 0),
                'cost_24h': float(ai[2] or 0),
                'errors': int(ai[3] or 0),
                'rag_hits': int(ai[4] or 0),
                'cache_hits': int(ai[5] or 0),
                'error_rate': (float(ai[3] or 0) / total * 100) if total else 0,
                'rag_rate': (float(ai[4] or 0) / total * 100) if total else 0,
                'cache_rate': (float(ai[5] or 0) / total * 100) if total else 0,
            }
        except Exception:
            summary['ai_calls'] = {}

        # 當月成本
        try:
            month_cost = session.execute(
                sa_text("SELECT COALESCE(SUM(cost_usd), 0) FROM ai_calls WHERE called_at >= :ms"),
                {'ms': month_start},
            ).fetchone()[0]
            summary['month_cost'] = float(month_cost or 0)
        except Exception:
            summary['month_cost'] = 0

        # 預算 over 80%
        try:
            budgets = session.execute(
                sa_text("""
                    SELECT b.period, b.provider, b.budget_usd, b.alert_pct,
                           COALESCE((
                               SELECT SUM(cost_usd) FROM ai_calls
                               WHERE called_at >= :ms
                                 AND (b.provider IS NULL OR provider = b.provider)
                           ), 0) AS spent
                    FROM ai_call_budgets b
                """),
                {'ms': month_start},
            ).fetchall()
            over_threshold = []
            for r in budgets:
                budget = float(r[2] or 0)
                spent = float(r[4] or 0)
                ratio = spent / budget if budget > 0 else 0
                threshold = float(r[3] or 80) / 100
                if ratio >= threshold:
                    over_threshold.append({
                        'period': r[0], 'provider': r[1] or '(全部)',
                        'spent': spent, 'budget': budget, 'ratio': ratio,
                    })
            summary['budget_alerts'] = over_threshold
        except Exception:
            summary['budget_alerts'] = []

        # 待審 + 蒸餾池
        try:
            ep_pending = session.execute(
                sa_text("SELECT COUNT(*) FROM learning_episodes WHERE promotion_status = 'awaiting_review' AND reviewed_at IS NULL"),
            ).fetchone()[0]
            ep_total_30d = session.execute(
                sa_text("SELECT COUNT(*) FROM learning_episodes WHERE created_at >= NOW() - INTERVAL '30 days'"),
            ).fetchone()[0]
            ep_approved_30d = session.execute(
                sa_text("SELECT COUNT(*) FROM learning_episodes WHERE created_at >= NOW() - INTERVAL '30 days' AND promotion_status = 'approved'"),
            ).fetchone()[0]
            summary['episodes'] = {
                'pending': int(ep_pending or 0),
                'total_30d': int(ep_total_30d or 0),
                'approved_30d': int(ep_approved_30d or 0),
                'approval_rate': (float(ep_approved_30d or 0) / float(ep_total_30d) * 100) if ep_total_30d else 0,
            }
        except Exception:
            summary['episodes'] = {}

        # PPT 視覺審核 7d
        try:
            ppt = session.execute(
                sa_text("""
                    SELECT COUNT(*),
                           COUNT(*) FILTER (WHERE audit_status='passed'),
                           COUNT(*) FILTER (WHERE audit_status='failed')
                    FROM ppt_audit_results
                    WHERE audited_at >= NOW() - INTERVAL '7 days'
                """),
            ).fetchone()
            ppt_total = int(ppt[0] or 0)
            summary['ppt'] = {
                'total': ppt_total,
                'passed': int(ppt[1] or 0),
                'failed': int(ppt[2] or 0),
                'pass_rate': (float(ppt[1] or 0) / ppt_total * 100) if ppt_total else 0,
            }
        except Exception:
            summary['ppt'] = {}

        # AIOps 7d
        try:
            inc = session.execute(
                sa_text("""
                    SELECT COUNT(*),
                           COUNT(*) FILTER (WHERE status='open'),
                           COUNT(*) FILTER (WHERE severity IN ('P0','P1'))
                    FROM incidents
                    WHERE created_at >= NOW() - INTERVAL '7 days'
                """),
            ).fetchone()
            heal = session.execute(
                sa_text("""
                    SELECT COUNT(*),
                           COUNT(*) FILTER (WHERE result='success')
                    FROM heal_logs
                    WHERE created_at >= NOW() - INTERVAL '7 days'
                """),
            ).fetchone()
            summary['aiops'] = {
                'incidents_total': int(inc[0] or 0),
                'incidents_open': int(inc[1] or 0),
                'incidents_p0_p1': int(inc[2] or 0),
                'heals_total': int(heal[0] or 0),
                'heals_success': int(heal[1] or 0),
                'heal_rate': (float(heal[1] or 0) / float(heal[0]) * 100) if heal[0] else 0,
            }
        except Exception:
            summary['aiops'] = {}

        # MCP 24h
        try:
            mcp = session.execute(
                sa_text("""
                    SELECT COUNT(*), COUNT(DISTINCT server),
                           COUNT(*) FILTER (WHERE cache_hit),
                           COALESCE(SUM(cost_usd), 0)
                    FROM mcp_calls
                    WHERE called_at >= NOW() - INTERVAL '24 hours'
                """),
            ).fetchone()
            mcp_total = int(mcp[0] or 0)
            summary['mcp'] = {
                'total': mcp_total,
                'servers': int(mcp[1] or 0),
                'cache_hits': int(mcp[2] or 0),
                'cost': float(mcp[3] or 0),
                'cache_rate': (float(mcp[2] or 0) / mcp_total * 100) if mcp_total else 0,
            }
        except Exception:
            summary['mcp'] = {}
    finally:
        session.close()

    return render_template(
        'admin/observability_overview.html',
        active_page='obs_overview',
        summary=summary,
        today=today.strftime('%Y-%m-%d'),
    )


# ─────────────────────────────────────────────────────────────────────────────
# /observability/ai_calls — Phase 27 主入口
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/ai_calls')
@login_required
def ai_calls_dashboard():
    """ai_calls 表觀測 dashboard（24h 預設視窗）"""
    hours = int(request.args.get('hours', '24'))
    caller_filter = request.args.get('caller', '').strip()
    provider_filter = request.args.get('provider', '').strip()

    since = datetime.now() - timedelta(hours=hours)
    session = get_session()
    try:
        # 1. 總覽
        summary = session.execute(
            sa_text("""
                SELECT
                  COUNT(*) AS total_calls,
                  COALESCE(SUM(input_tokens + output_tokens), 0) AS total_tokens,
                  COALESCE(SUM(cost_usd), 0) AS total_cost,
                  COALESCE(AVG(duration_ms), 0) AS avg_duration,
                  COUNT(*) FILTER (WHERE status = 'ok') AS ok_calls,
                  COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')) AS error_calls,
                  COUNT(*) FILTER (WHERE rag_hit) AS rag_hits,
                  COUNT(*) FILTER (WHERE cache_hit) AS cache_hits
                FROM ai_calls
                WHERE called_at >= :since
            """),
            {'since': since},
        ).fetchone()

        # 2. by provider
        by_provider = session.execute(
            sa_text("""
                SELECT provider, COUNT(*) AS calls,
                       COALESCE(SUM(input_tokens + output_tokens), 0) AS tokens,
                       COALESCE(SUM(cost_usd), 0) AS cost
                FROM ai_calls
                WHERE called_at >= :since
                GROUP BY provider
                ORDER BY tokens DESC
            """),
            {'since': since},
        ).fetchall()

        # 3. TOP 100 calls — Phase 33 Critic HIGH #2 修補：
        # 改用固定 SQL + 全綁參數，移除 f-string 動態 WHERE 拼接（防後人不慎注入）
        recent = session.execute(
            sa_text("""
                SELECT id, called_at, caller, provider, model,
                       input_tokens, output_tokens, duration_ms, status,
                       cost_usd, cache_hit, rag_hit
                FROM ai_calls
                WHERE called_at >= :since
                  AND (:caller_f = '' OR caller = :caller_f)
                  AND (:provider_f = '' OR provider = :provider_f)
                ORDER BY called_at DESC
                LIMIT 100
            """),
            {
                'since': since,
                'caller_f': caller_filter,
                'provider_f': provider_filter,
            },
        ).fetchall()

        # 4. caller 列表（給篩選 dropdown）
        callers = session.execute(
            sa_text("""
                SELECT DISTINCT caller FROM ai_calls
                WHERE called_at >= :since ORDER BY caller
            """),
            {'since': since},
        ).fetchall()

        # 5. Phase 39 D-3: caller × RAG 命中率 × MCP 編排率（跨表 JOIN）
        # 展現「AI 自動化專業」核心：每個 caller 多大比例走了 RAG / MCP
        caller_richness = session.execute(
            sa_text("""
                SELECT a.caller,
                       COUNT(*) AS total_calls,
                       COUNT(*) FILTER (WHERE a.rag_hit) AS rag_hits,
                       COUNT(DISTINCT m.request_id) AS mcp_orchestrated,
                       COALESCE(AVG(rl.feedback_score) FILTER (WHERE rl.feedback_score IS NOT NULL), 0)
                           AS avg_rag_feedback,
                       COUNT(rl.feedback_score) AS feedback_count
                FROM ai_calls a
                LEFT JOIN mcp_calls m
                       ON m.request_id = a.request_id
                      AND m.called_at >= :since
                LEFT JOIN rag_query_log rl
                       ON rl.caller = a.caller
                      AND rl.queried_at >= :since
                WHERE a.called_at >= :since
                GROUP BY a.caller
                HAVING COUNT(*) >= 5
                ORDER BY total_calls DESC
                LIMIT 12
            """),
            {'since': since},
        ).fetchall()

        return render_template(
            'admin/ai_calls_dashboard.html',
            active_page='obs_ai_calls',
            hours=hours,
            caller_filter=caller_filter,
            provider_filter=provider_filter,
            summary={
                'total_calls': int(summary[0] or 0),
                'total_tokens': int(summary[1] or 0),
                'total_cost': float(summary[2] or 0),
                'avg_duration': int(summary[3] or 0),
                'ok_calls': int(summary[4] or 0),
                'error_calls': int(summary[5] or 0),
                'rag_hits': int(summary[6] or 0),
                'cache_hits': int(summary[7] or 0),
            },
            by_provider=[
                {'provider': r[0], 'calls': int(r[1] or 0),
                 'tokens': int(r[2] or 0), 'cost': float(r[3] or 0)}
                for r in by_provider
            ],
            recent=[
                {'id': r[0], 'called_at': r[1].strftime('%H:%M:%S'),
                 'caller': r[2], 'provider': r[3], 'model': r[4],
                 'in_tokens': int(r[5] or 0), 'out_tokens': int(r[6] or 0),
                 'duration_ms': int(r[7] or 0), 'status': r[8],
                 'cost': float(r[9] or 0), 'cache_hit': bool(r[10]),
                 'rag_hit': bool(r[11])}
                for r in recent
            ],
            callers=[r[0] for r in callers],
            caller_richness=[
                {
                    'caller': r[0],
                    'total_calls': int(r[1] or 0),
                    'rag_hits': int(r[2] or 0),
                    'mcp_orchestrated': int(r[3] or 0),
                    'avg_rag_feedback': round(float(r[4] or 0), 2),
                    'feedback_count': int(r[5] or 0),
                    'rag_hit_rate': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0,
                    'mcp_rate': (float(r[3] or 0) / float(r[1]) * 100) if r[1] else 0,
                }
                for r in caller_richness
            ],
            error=None,
        )
    except Exception as e:
        return render_template(
            'admin/ai_calls_dashboard.html',
            active_page='obs_ai_calls',
            hours=hours, caller_filter=caller_filter,
            provider_filter=provider_filter,
            summary={}, by_provider=[], recent=[], callers=[], caller_richness=[],
            error=f'查詢失敗: {type(e).__name__}: {str(e)[:200]}',
        )
    finally:
        session.close()


# ─────────────────────────────────────────────────────────────────────────────
# /observability/promotion_review — Phase 28 PromotionGate 待審核列表
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/promotion_review')
@login_required
def promotion_review_list():
    """awaiting_review episodes 列表（24h 內 reviewed_at IS NULL）

    Phase 39（D-1）：每筆 episode 自動跑 RAG 找 Top 3 相似已晉升 ai_insights，
    輔助人工判斷晉升價值。RAG fail-safe：失敗則 similar_insights=[]，不擋頁面。
    """
    session = get_session()
    try:
        rows = session.execute(
            sa_text("""
                SELECT id, created_at, episode_type, source_table, source_id,
                       distilled_text, quality_score, weight, promotion_status
                FROM learning_episodes
                WHERE promotion_status = 'awaiting_review'
                  AND reviewed_at IS NULL
                ORDER BY weight DESC, created_at ASC
                LIMIT 50
            """),
        ).fetchall()

        # ai_insights 全表大小（給「晉升後 KB 增長」對照）
        kb_size = 0
        try:
            kb_row = session.execute(
                sa_text("SELECT COUNT(*) FROM ai_insights"),
            ).fetchone()
            kb_size = int(kb_row[0] or 0)
        except Exception:
            pass

        episodes = [
            {'id': r[0], 'created_at': r[1].strftime('%Y-%m-%d %H:%M'),
             'episode_type': r[2], 'source_table': r[3], 'source_id': r[4],
             'distilled_text': (r[5] or '')[:600],
             'quality_score': float(r[6] or 0),
             'weight': float(r[7] or 0),
             'status': r[8],
             'similar_insights': []}
            for r in rows
        ]

        # Phase 39 D-1：對每筆 episode 跑 RAG 找 Top 3 相似已晉升
        try:
            from services.rag_service import rag_service
            for ep in episodes:
                try:
                    rag_result = rag_service.query(
                        text=ep['distilled_text'][:500],
                        caller='admin_promotion_review',
                        top_k=3,
                        threshold=0.7,
                    )
                    ep['similar_insights'] = [
                        {
                            'id': h.get('id'),
                            'insight_type': h.get('insight_type'),
                            'content': (h.get('content') or '')[:180],
                            'similarity': round(float(h.get('similarity', 0)), 3),
                            'created_at': h.get('created_at').strftime('%Y-%m-%d')
                                          if h.get('created_at') else '',
                        }
                        for h in rag_result.hits[:3]
                    ]
                except Exception:
                    pass  # 單筆 RAG 失敗不影響其餘
        except Exception:
            pass  # rag_service import 失敗（feature flag OFF）→ 略過

        return render_template(
            'admin/promotion_review.html',
            active_page='obs_promotion_review',
            episodes=episodes,
            kb_size=kb_size,
            error=None,
        )
    except Exception as e:
        return render_template(
            'admin/promotion_review.html',
            active_page='obs_promotion_review',
            episodes=[],
            kb_size=0,
            error=f'查詢失敗: {type(e).__name__}: {str(e)[:200]}',
        )
    finally:
        session.close()


@admin_observability_bp.route('/promotion_review/approve/<int:episode_id>', methods=['POST'])
@login_required
def promotion_review_approve(episode_id: int):
    """Web 介面「通過」按鈕 — 等同於 Telegram pg_ok callback"""
    try:
        from services.learning_pipeline import promotion_gate, hash_human_approver
        # 從 Flask session 取（已過 @login_required）— 不信任 client header
        user = get_current_user() or {}
        username = user.get('username', 'web_admin')
        approver_hash = hash_human_approver(username)
        insight_id = promotion_gate.promote(episode_id)
        if insight_id:
            return jsonify({'ok': True, 'insight_id': insight_id, 'approver': approver_hash})
        return jsonify({'ok': False, 'error': 'promote failed'}), 500
    except Exception as e:
        return jsonify({'ok': False, 'error': str(e)[:200]}), 500


@admin_observability_bp.route('/promotion_review/reject/<int:episode_id>', methods=['POST'])
@login_required
def promotion_review_reject(episode_id: int):
    """Web 介面「拒絕」按鈕"""
    try:
        from services.learning_pipeline import promotion_gate
        ok = promotion_gate.reject(episode_id, 'rejected_human', detail='web admin reject')
        return jsonify({'ok': ok})
    except Exception as e:
        return jsonify({'ok': False, 'error': str(e)[:200]}), 500


# ─────────────────────────────────────────────────────────────────────────────
# /observability/quality_trend — Phase 25 caller 反饋趨勢視覺化
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/quality_trend')
@login_required
def quality_trend_dashboard():
    """caller × feedback 趨勢（30 日窗格）"""
    days = int(request.args.get('days', '30'))
    try:
        from services.feedback_quality_tracker import (
            compute_caller_quality_trend, get_caller_recommendations,
        )
        trends = compute_caller_quality_trend(days=days)
        recommendations = get_caller_recommendations(days=days)

        # 排序：avg_score 升序（最差先看）
        sorted_trends = sorted(
            trends.items(),
            key=lambda kv: kv[1].get('avg_score', 5),
        )

        # Phase 40 D-6: learning_episodes 各 status 分布（蒸餾池飽和度）
        episode_distribution = {}
        try:
            session = get_session()
            try:
                rows = session.execute(
                    sa_text("""
                        SELECT promotion_status, COUNT(*) AS cnt
                        FROM learning_episodes
                        WHERE created_at >= NOW() - INTERVAL ':days days'
                        GROUP BY promotion_status
                    """).bindparams(days=days),
                ).fetchall() if False else session.execute(
                    sa_text(f"""
                        SELECT promotion_status, COUNT(*) AS cnt
                        FROM learning_episodes
                        WHERE created_at >= NOW() - INTERVAL '{int(days)} days'
                        GROUP BY promotion_status
                    """),
                ).fetchall()
                episode_distribution = {r[0]: int(r[1] or 0) for r in rows}
            finally:
                session.close()
        except Exception:
            pass

        # Phase 40 D-6: 對最差 3 名 caller 跑 RAG 找根因建議
        rag_root_causes = []
        try:
            from services.rag_service import rag_service
            worst_3 = sorted_trends[:3] if len(sorted_trends) >= 3 else sorted_trends
            for caller, info in worst_3:
                if info.get('avg_score', 5) < 3.0 and info.get('total_feedback', 0) >= 3:
                    try:
                        q = (
                            f"caller {caller} 反饋分數低 平均 "
                            f"{info.get('avg_score', 0):.1f}/5 應採取什麼根因排查"
                        )
                        rag_result = rag_service.query(
                            text=q,
                            caller='admin_quality_trend',
                            top_k=2,
                            threshold=0.6,
                        )
                        if rag_result.hits:
                            rag_root_causes.append({
                                'caller': caller,
                                'avg_score': info.get('avg_score', 0),
                                'feedback_n': info.get('total_feedback', 0),
                                'hits': [
                                    {
                                        'id': h.get('id'),
                                        'insight_type': h.get('insight_type'),
                                        'content': (h.get('content') or '')[:200],
                                        'similarity': round(float(h.get('similarity', 0)), 3),
                                    }
                                    for h in rag_result.hits[:2]
                                ],
                            })
                    except Exception:
                        pass
        except Exception:
            pass

        return render_template(
            'admin/quality_trend.html',
            active_page='obs_quality_trend',
            days=days,
            trends=[(c, info) for c, info in sorted_trends],
            recommendations=recommendations,
            episode_distribution=episode_distribution,
            rag_root_causes=rag_root_causes,
            error=None,
        )
    except Exception as e:
        return render_template(
            'admin/quality_trend.html',
            active_page='obs_quality_trend',
            days=days, trends=[], recommendations=[],
            episode_distribution={}, rag_root_causes=[],
            error=f'查詢失敗: {type(e).__name__}: {str(e)[:200]}',
        )


# ─────────────────────────────────────────────────────────────────────────────
# /observability/budget — Phase 29 預算管理 + 手動 throttle
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/budget')
@login_required
def budget_dashboard():
    """ai_call_budgets 編輯 + 當月 spent 即時對比"""
    from datetime import datetime as _dt
    today = _dt.now()
    month_start = _dt(today.year, today.month, 1)

    session = get_session()
    try:
        budgets = session.execute(
            sa_text("""
                SELECT id, period, provider, budget_usd, alert_pct, updated_at
                FROM ai_call_budgets
                ORDER BY period, provider NULLS FIRST
            """),
        ).fetchall()

        spent_rows = session.execute(
            sa_text("""
                SELECT provider, COALESCE(SUM(cost_usd), 0) AS spent
                FROM ai_calls
                WHERE called_at >= :ms
                GROUP BY provider
            """),
            {'ms': month_start},
        ).fetchall()
        spent_map = {r[0]: float(r[1] or 0) for r in spent_rows}

        # throttle 狀態
        throttle_state = {}
        try:
            from services.cost_throttle_service import get_throttle_state
            throttle_state = get_throttle_state()
        except Exception:
            pass

        rows = []
        for b in budgets:
            provider = b[2]  # 可能 None（全供應商總額）
            spent = spent_map.get(provider, 0.0) if provider else sum(spent_map.values())
            budget_usd = float(b[3] or 0)
            ratio = (spent / budget_usd) if budget_usd > 0 else 0
            rows.append({
                'id': b[0], 'period': b[1], 'provider': provider or '(all)',
                'budget_usd': budget_usd, 'alert_pct': int(b[4] or 80),
                'spent': spent, 'ratio': ratio,
                'throttled': throttle_state.get(provider, {}).get('throttled', False) if provider else False,
                'updated_at': b[5].strftime('%Y-%m-%d %H:%M') if b[5] else '-',
            })

        # Phase 39 D-4: RAG 自動建議策略（針對超 80% 的 row）
        budget_strategies = []
        over_threshold_rows = [r for r in rows if r.get('ratio', 0) >= 0.8]
        if over_threshold_rows:
            try:
                from services.rag_service import rag_service
                top_breach = max(over_threshold_rows, key=lambda r: r.get('ratio', 0))
                query_text = (
                    f"預算超出 alert_pct provider={top_breach['provider']} "
                    f"ratio={int(top_breach['ratio']*100)}% 應採取什麼節流策略"
                )
                rag_result = rag_service.query(
                    text=query_text,
                    caller='admin_budget_dashboard',
                    top_k=3,
                    threshold=0.65,
                )
                budget_strategies = [
                    {
                        'id': h.get('id'),
                        'insight_type': h.get('insight_type'),
                        'content': (h.get('content') or '')[:240],
                        'similarity': round(float(h.get('similarity', 0)), 3),
                    }
                    for h in rag_result.hits[:3]
                ]
            except Exception:
                pass

        return render_template(
            'admin/budget.html',
            active_page='obs_budget',
            rows=rows,
            budget_strategies=budget_strategies,
            error=None,
        )
    except Exception as e:
        return render_template('admin/budget.html', active_page='obs_budget', rows=[],
                                budget_strategies=[],
                                error=f'查詢失敗: {type(e).__name__}: {str(e)[:200]}')
    finally:
        session.close()


@admin_observability_bp.route('/ai_calls/trigger_code_review', methods=['POST'])
@login_required
def ai_calls_trigger_code_review():
    """Phase 40 D-7 (L2 自動化)：對高錯誤率時段觸發 Code Review Pipeline。

    用途：admin 在觀測台看到某 caller 錯誤率飆高時，一鍵觸發 5-step
         pipeline (read→hermes_scan→openclaw_summary→ea_decision→nemoton_act)
         在 daemon thread 自動審查最近 commit 變更檔案，找出可能的 regression。
    """
    try:
        import subprocess
        import threading
        from services.code_review_pipeline_service import CodeReviewPipeline

        # 取最新 commit + 變更檔案
        commit_sha = subprocess.check_output(
            ['git', 'rev-parse', 'HEAD'], stderr=subprocess.DEVNULL,
        ).decode().strip()
        changed = subprocess.check_output(
            ['git', 'diff-tree', '--no-commit-id', '--name-only', '-r', commit_sha],
            stderr=subprocess.DEVNULL,
        ).decode().strip().split('\n')
        changed = [f for f in changed if f]

        if not changed:
            return jsonify({'ok': False, 'error': '最新 commit 無變更檔案'}), 400

        pipeline = CodeReviewPipeline(
            commit_sha=commit_sha,
            changed_files=changed,
            branch='main',
            deploy_type='manual_observability',
        )
        threading.Thread(target=pipeline.run, daemon=True).start()
        return jsonify({
            'ok': True,
            'pipeline_id': pipeline.pipeline_id,
            'commit_sha': commit_sha[:8],
            'changed_files_count': len(changed),
            'message': f'已觸發 Code Review (pipeline_id={pipeline.pipeline_id}) 在背景執行，'
                       f'5 step 完成後會推 Telegram 通知。',
        })
    except Exception as e:
        return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500


@admin_observability_bp.route('/ppt_audit/trigger_aider_heal', methods=['POST'])
@login_required
def ppt_audit_trigger_aider_heal():
    """Phase 40 D-8 (L2 自動化)：對失敗 PPT audit 觸發 AiderHeal 修 generator。

    用途：admin 在觀測台看到 PPT vision audit 連續失敗時，一鍵觸發 AiderHeal
         自動修 services/ppt_generator.py（或對應 template generator），
         結果會 git push 到 main 觸發 CD 自動部署。
    """
    try:
        from services.aider_heal_executor import execute_code_fix
        data = request.json or {}
        error_msg = (data.get('error_msg') or '').strip()
        pptx_filename = (data.get('pptx_filename') or '').strip()
        if not error_msg:
            return jsonify({'ok': False, 'error': '需提供 error_msg'}), 400

        # 構造 context 給 AiderHeal
        context = {
            'error_type': 'ppt_vision_audit_failure',
            'error_message': error_msg[:500],
            'target_file': 'services/ppt_generator.py',
            'pptx_filename': pptx_filename,
            'triggered_by': 'admin_observability',
        }
        result = execute_code_fix(context)
        return jsonify({
            'ok': bool(getattr(result, 'success', False)),
            'action': getattr(result, 'action', None),
            'message': getattr(result, 'message', '') or '已派出 AiderHeal',
        })
    except Exception as e:
        return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500


@admin_observability_bp.route('/host_health/trigger_autoheal', methods=['POST'])
@login_required
def host_health_trigger_autoheal():
    """Phase 40 D-9 (L2 自動化)：對掛掉的主機觸發 AutoHeal playbook。

    用途：admin 看到某台 Ollama 主機標記 unhealthy 時一鍵觸發 AutoHeal
         (ADR-013) 跑對應 playbook（DOCKER_RESTART / SSH_CMD / ALERT_ONLY）。

    安全：只能對已標記 unhealthy 的 host 觸發；不接受任意 host URL（防 SSRF）。
    """
    try:
        data = request.json or {}
        host_label = (data.get('host_label') or '').strip()
        from services.auto_heal_service import auto_heal_service
        from services.ollama_service import _is_unhealthy, OLLAMA_HOST_PRIMARY, OLLAMA_HOST_SECONDARY, OLLAMA_HOST_FALLBACK

        # 白名單對應
        host_map = {
            'Primary (GCP)': OLLAMA_HOST_PRIMARY,
            'Secondary (GCP)': OLLAMA_HOST_SECONDARY,
            'Fallback (111)': OLLAMA_HOST_FALLBACK,
        }
        host_url = host_map.get(host_label)
        if not host_url:
            return jsonify({'ok': False, 'error': f'未知 host_label: {host_label}'}), 400

        if not _is_unhealthy(host_url):
            return jsonify({
                'ok': False,
                'error': f'{host_label} 目前未標記異常，無需 AutoHeal',
            }), 400

        result = auto_heal_service.handle_exception(
            error_type='ollama_unhealthy',
            context={
                'host_label': host_label,
                'host_url': host_url,
                'error_message': f'Ollama host {host_label} ({host_url}) marked unhealthy',
                'triggered_by': 'admin_observability',
            },
        )
        return jsonify({
            'ok': bool(getattr(result, 'success', False)),
            'action': getattr(result, 'action', None),
            'message': getattr(result, 'message', '') or 'AutoHeal 已派遣',
        })
    except Exception as e:
        return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500


@admin_observability_bp.route('/budget/force_throttle', methods=['POST'])
@login_required
def budget_force_throttle():
    """Phase 39 D-4 (L2 自動化)：立即強制執行 cost_throttle evaluate（不等 hourly cron）。

    用途：admin 在觀測台看到 ratio 飆超 110% 時不需等下次 cron，
         直接點按鈕強制 re-evaluate 三主機 throttle 狀態（claude→gemini fallback 立即生效）。
    """
    try:
        from services.cost_throttle_service import (
            evaluate_throttle_status, is_cost_throttle_enabled,
        )
        if not is_cost_throttle_enabled():
            return jsonify({
                'ok': False,
                'error': 'COST_THROTTLE_ENABLED=false（先設環境變數）',
            }), 400
        new_state = evaluate_throttle_status()
        throttled = [p for p, s in new_state.items() if s.get('throttled')]
        return jsonify({
            'ok': True,
            'throttled_providers': throttled,
            'state': new_state,
            'message': f'已立即重算 throttle 狀態，被節流的 provider：{throttled or "（無）"}',
        })
    except Exception as e:
        return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500


@admin_observability_bp.route('/budget/update/<int:budget_id>', methods=['POST'])
@login_required
def budget_update(budget_id: int):
    """更新 budget_usd / alert_pct"""
    try:
        new_budget = float(request.json.get('budget_usd'))
        new_alert = int(request.json.get('alert_pct', 80))
        if new_budget <= 0 or not (1 <= new_alert <= 100):
            return jsonify({'ok': False, 'error': 'invalid range'}), 400

        session = get_session()
        try:
            session.execute(
                sa_text("""
                    UPDATE ai_call_budgets
                    SET budget_usd = :b, alert_pct = :a, updated_at = NOW()
                    WHERE id = :id
                """),
                {'b': new_budget, 'a': new_alert, 'id': budget_id},
            )
            session.commit()
            return jsonify({'ok': True})
        finally:
            session.close()
    except Exception as e:
        return jsonify({'ok': False, 'error': str(e)[:200]}), 500


# ─────────────────────────────────────────────────────────────────────────────
# /observability/ppt_audit_history — Phase 29 PPT 視覺審核歷史
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/ppt_audit_history')
@login_required
def ppt_audit_history():
    """掃 reports/ 目錄列近 7 日 .pptx 檔 + 從 ppt_audit_results 表讀 audit 歷史（Phase 38）"""
    import os
    import time
    reports_dir = 'reports'
    files = []
    audit_records = []
    error = None

    try:
        if not os.path.isdir(reports_dir):
            error = f'{reports_dir} 目錄不存在'
        else:
            cutoff = time.time() - 7 * 86400
            for f in os.listdir(reports_dir):
                if not f.lower().endswith('.pptx'):
                    continue
                full = os.path.join(reports_dir, f)
                # symlink 防護：reports/ 內不接受 symlink，避免目錄逃逸（Critic MEDIUM #2）
                if os.path.islink(full):
                    continue
                try:
                    mtime = os.path.getmtime(full)
                    if mtime >= cutoff:
                        files.append({
                            'name': f,
                            'size_kb': round(os.path.getsize(full) / 1024, 1),
                            'mtime': datetime.fromtimestamp(mtime).strftime('%Y-%m-%d %H:%M'),
                            'mtime_ts': mtime,
                        })
                except OSError:
                    continue
            files.sort(key=lambda x: x['mtime_ts'], reverse=True)
    except Exception as e:
        error = f'{type(e).__name__}: {str(e)[:200]}'

    # Phase 38：讀過去 7 日 audit 歷史
    try:
        session = get_session()
        try:
            audit_rows = session.execute(
                sa_text("""
                    SELECT audited_at, pptx_filename, audit_status,
                           issues_count, confidence, duration_ms, error_msg
                    FROM ppt_audit_results
                    WHERE audited_at >= NOW() - INTERVAL '7 days'
                    ORDER BY audited_at DESC
                    LIMIT 100
                """),
            ).fetchall()
            audit_records = [
                {
                    'audited_at': r[0].strftime('%Y-%m-%d %H:%M'),
                    'pptx_filename': r[1],
                    'audit_status': r[2],
                    'issues_count': int(r[3] or 0),
                    'confidence': float(r[4] or 0),
                    'duration_ms': int(r[5] or 0),
                    'error_msg': r[6],
                }
                for r in audit_rows
            ]
        finally:
            session.close()
    except Exception:
        pass  # 表可能尚未 migration，失敗安全

    # PPT vision 啟用狀態
    try:
        from services.ppt_vision_service import is_ppt_vision_enabled
        vision_enabled = is_ppt_vision_enabled()
    except Exception:
        vision_enabled = False

    # Phase 41 E-2: 對最近 3 筆 failed audit 跑 RAG 找相似修法
    rag_fixes = []
    failed_records = [r for r in audit_records if r.get('audit_status') in ('failed', 'error')][:3]
    if failed_records:
        try:
            from services.rag_service import rag_service
            for fr in failed_records:
                try:
                    err_text = fr.get('error_msg') or 'PPT vision audit failed'
                    rag_result = rag_service.query(
                        text=f"PPT 視覺審核失敗 {err_text[:200]} 怎麼修",
                        caller='admin_ppt_audit',
                        top_k=2,
                        threshold=0.6,
                    )
                    if rag_result.hits:
                        rag_fixes.append({
                            'pptx_filename': fr.get('pptx_filename'),
                            'audited_at': fr.get('audited_at'),
                            'error_msg': (err_text or '')[:160],
                            'hits': [
                                {
                                    'id': h.get('id'),
                                    'insight_type': h.get('insight_type'),
                                    'content': (h.get('content') or '')[:200],
                                    'similarity': round(float(h.get('similarity', 0)), 3),
                                }
                                for h in rag_result.hits[:2]
                            ],
                        })
                except Exception:
                    pass
        except Exception:
            pass

    return render_template(
        'admin/ppt_audit_history.html',
        active_page='obs_ppt_audit',
        files=files,
        audit_records=audit_records,
        rag_fixes=rag_fixes,
        vision_enabled=vision_enabled,
        error=error,
    )


# ─────────────────────────────────────────────────────────────────────────────
# /observability/host_health — 三主機 + MCP 健康度
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/host_health')
@login_required
def host_health_dashboard():
    """三主機 Ollama + 4 個 MCP server 即時健康（同時寫入 host_health_probes 留歷史）"""
    import time as _time
    ollama_hosts = []
    probe_records = []  # 收集本次 probe 結果以批次寫 DB
    try:
        from services.ollama_service import (
            OLLAMA_HOST_PRIMARY, OLLAMA_HOST_SECONDARY, OLLAMA_HOST_FALLBACK,
            _is_unhealthy, _unhealthy_marks,
        )
        import requests as _r
        for label, host in [
            ('Primary (GCP)', OLLAMA_HOST_PRIMARY),
            ('Secondary (GCP)', OLLAMA_HOST_SECONDARY),
            ('Fallback (111)', OLLAMA_HOST_FALLBACK),
        ]:
            entry = {'label': label, 'host': host, 'healthy': False,
                     'unhealthy_mark': _is_unhealthy(host), 'models': []}
            t0 = _time.monotonic()
            err = None
            try:
                resp = _r.get(f"{host.rstrip('/')}/api/tags", timeout=3)
                if resp.status_code == 200:
                    entry['healthy'] = True
                    entry['models'] = [
                        m.get('name', '') for m in resp.json().get('models', [])
                    ][:15]
                else:
                    err = f"HTTP {resp.status_code}"
            except Exception as e:
                err = f"{type(e).__name__}: {str(e)[:200]}"
            response_ms = int((_time.monotonic() - t0) * 1000)
            probe_records.append({
                'host_label': label, 'host_url': host, 'healthy': entry['healthy'],
                'unhealthy_mark': entry['unhealthy_mark'],
                'models_count': len(entry['models']), 'response_ms': response_ms,
                'error_msg': err,
            })
            ollama_hosts.append(entry)
    except Exception:
        pass

    # Phase 38：寫入 host_health_probes 留歷史（失敗安全，不擋頁面渲染）
    if probe_records:
        try:
            _session = get_session()
            try:
                for rec in probe_records:
                    _session.execute(
                        sa_text("""
                            INSERT INTO host_health_probes
                                (host_label, host_url, healthy, unhealthy_mark,
                                 models_count, response_ms, error_msg)
                            VALUES
                                (:host_label, :host_url, :healthy, :unhealthy_mark,
                                 :models_count, :response_ms, :error_msg)
                        """),
                        rec,
                    )
                _session.commit()
            finally:
                _session.close()
        except Exception:
            pass  # DB 寫入失敗不影響頁面顯示

    # MCP server 健康
    mcp_status = {}
    try:
        from services.mcp_router import mcp_router
        mcp_status = mcp_router.health_check()
    except Exception:
        pass

    # cost throttle 狀態
    throttle_state = {}
    try:
        from services.cost_throttle_service import get_throttle_state
        throttle_state = get_throttle_state()
    except Exception:
        pass

    # Phase 38：讀過去 24h 三主機健康歷史（給趨勢卡片）
    health_history = []
    mcp_24h = []  # Phase 39 D-2: MCP 24h 各 server 工作量
    aiops_summary = {}  # Phase 39 D-5: incidents + heal_logs 7d 統計
    try:
        _session2 = get_session()
        try:
            history_rows = _session2.execute(
                sa_text("""
                    SELECT host_label,
                           COUNT(*) FILTER (WHERE healthy)            AS up_count,
                           COUNT(*) FILTER (WHERE NOT healthy)        AS down_count,
                           COALESCE(AVG(response_ms) FILTER (WHERE healthy), 0) AS avg_ms,
                           COUNT(*) AS total
                    FROM host_health_probes
                    WHERE probed_at >= NOW() - INTERVAL '24 hours'
                    GROUP BY host_label
                    ORDER BY host_label
                """),
            ).fetchall()
            health_history = [
                {
                    'host_label': r[0],
                    'up_count': int(r[1] or 0),
                    'down_count': int(r[2] or 0),
                    'avg_ms': int(r[3] or 0),
                    'total': int(r[4] or 0),
                    'uptime_pct': (float(r[1] or 0) / float(r[4]) * 100) if r[4] else 0,
                }
                for r in history_rows
            ]

            # Phase 39 D-5：incidents + heal_logs 過去 7d 統計
            try:
                inc_rows = _session2.execute(
                    sa_text("""
                        SELECT
                            COUNT(*) AS total_incidents,
                            COUNT(*) FILTER (WHERE status = 'open') AS open_count,
                            COUNT(*) FILTER (WHERE status = 'resolved') AS resolved_count,
                            COUNT(*) FILTER (WHERE severity = 'P0') AS p0_count,
                            COUNT(*) FILTER (WHERE severity = 'P1') AS p1_count
                        FROM incidents
                        WHERE created_at >= NOW() - INTERVAL '7 days'
                    """),
                ).fetchone()
                heal_rows = _session2.execute(
                    sa_text("""
                        SELECT
                            COUNT(*) AS total_heals,
                            COUNT(*) FILTER (WHERE result = 'success') AS heal_success,
                            COUNT(*) FILTER (WHERE result = 'failed') AS heal_failed,
                            COALESCE(AVG(duration_ms) FILTER (WHERE result = 'success'), 0) AS avg_ms
                        FROM heal_logs
                        WHERE created_at >= NOW() - INTERVAL '7 days'
                    """),
                ).fetchone()
                aiops_summary = {
                    'incidents_total': int(inc_rows[0] or 0),
                    'incidents_open': int(inc_rows[1] or 0),
                    'incidents_resolved': int(inc_rows[2] or 0),
                    'incidents_p0': int(inc_rows[3] or 0),
                    'incidents_p1': int(inc_rows[4] or 0),
                    'heals_total': int(heal_rows[0] or 0),
                    'heals_success': int(heal_rows[1] or 0),
                    'heals_failed': int(heal_rows[2] or 0),
                    'heals_avg_ms': int(heal_rows[3] or 0),
                    'heal_success_rate': (
                        float(heal_rows[1] or 0) / float(heal_rows[0]) * 100
                    ) if heal_rows[0] else 0,
                }
            except Exception:
                aiops_summary = {}

            # Phase 39 D-2：MCP 24h 工作量（每個 server）
            mcp_rows = _session2.execute(
                sa_text("""
                    SELECT server,
                           COUNT(*) AS total_calls,
                           COUNT(*) FILTER (WHERE status = 'ok') AS ok_calls,
                           COUNT(*) FILTER (WHERE cache_hit) AS cache_hits,
                           COALESCE(SUM(cost_usd), 0) AS total_cost,
                           COALESCE(AVG(duration_ms), 0) AS avg_ms,
                           COUNT(DISTINCT tool) AS tools_used
                    FROM mcp_calls
                    WHERE called_at >= NOW() - INTERVAL '24 hours'
                    GROUP BY server
                    ORDER BY total_calls DESC
                """),
            ).fetchall()
            mcp_24h = [
                {
                    'server': r[0],
                    'total_calls': int(r[1] or 0),
                    'ok_calls': int(r[2] or 0),
                    'cache_hits': int(r[3] or 0),
                    'total_cost': float(r[4] or 0),
                    'avg_ms': int(r[5] or 0),
                    'tools_used': int(r[6] or 0),
                    'success_rate': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0,
                    'cache_rate': (float(r[3] or 0) / float(r[1]) * 100) if r[1] else 0,
                }
                for r in mcp_rows
            ]
        finally:
            _session2.close()
    except Exception:
        pass  # 表可能尚未 migration，失敗安全

    return render_template(
        'admin/host_health.html',
        active_page='obs_host_health',
        ollama_hosts=ollama_hosts,
        mcp_status=mcp_status,
        throttle_state=throttle_state,
        health_history=health_history,
        mcp_24h=mcp_24h,
        aiops_summary=aiops_summary,
    )