#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
routes/admin_observability_routes.py
Operation Ollama-First v5.0 / Phase 27 — Admin Observability Dashboard

提供 admin 介面看戰役累積的觀測資料：
  /observability/ai_calls              — ai_calls 即時查詢（含篩選 / 圖表）
  /observability/promotion_review      — Phase 28 PromotionGate 待審核列表
  /observability/quality_trend         — Phase 25 caller 反饋趨勢
  /observability/host_health           — 三主機 Ollama + MCP 健康度

設計原則：
- 純讀（除了 promotion approve/reject 是 mutation）
- 失敗安全：DB 失敗回空清單 + 警告 banner
- 每頁 100 筆分頁，無限捲動
- 不暴露 secret / prompt 原文
"""

import logging
from datetime import datetime, timedelta
from flask import Blueprint, render_template, request, jsonify
from sqlalchemy import text as sa_text

from auth import login_required, get_current_user
from database.manager import get_session

logger = logging.getLogger(__name__)


admin_observability_bp = Blueprint(
    'admin_observability',
    __name__,
    url_prefix='/observability',
)


# ─────────────────────────────────────────────────────────────────────────────
# /observability/overview — Phase 45 總覽（單頁聚合 6 項 KPI）
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/')
@admin_observability_bp.route('/overview')
@login_required
def observability_overview():
    """Phase 45 — 觀測台總覽：一頁式聚合 6 個 sub-page 的關鍵 KPI。

    對應 Phase 44 daily Telegram summary 的 web 版本，做為 sidebar 入口頁。
    所有區塊失敗安全：個別 query 失敗只跳過該卡片，不擋整頁渲染。
    """
    from datetime import datetime as _dt
    today = _dt.now()
    month_start = _dt(today.year, today.month, 1)
    summary = {}

    session = get_session()
    try:
        # 三主機 24h 在線率
        try:
            host_rows = session.execute(
                sa_text("""
                    SELECT host_label, COUNT(*) AS total,
                           COUNT(*) FILTER (WHERE healthy) AS up,
                           COALESCE(AVG(response_ms) FILTER (WHERE healthy), 0) AS avg_ms
                    FROM host_health_probes
                    WHERE probed_at >= NOW() - INTERVAL '24 hours'
                    GROUP BY host_label ORDER BY host_label
                """),
            ).fetchall()
            summary['hosts'] = [
                {
                    'label': r[0],
                    'total': int(r[1] or 0),
                    'up': int(r[2] or 0),
                    'avg_ms': int(r[3] or 0),
                    'uptime_pct': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0,
                }
                for r in host_rows
            ]
        except Exception:
            summary['hosts'] = []

        # AI 呼叫 24h
        try:
            ai = session.execute(
                sa_text("""
                    SELECT COUNT(*), COALESCE(SUM(input_tokens + output_tokens), 0),
                           COALESCE(SUM(cost_usd), 0),
                           COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')),
                           COUNT(*) FILTER (WHERE rag_hit),
                           COUNT(*) FILTER (WHERE cache_hit)
                    FROM ai_calls
                    WHERE called_at >= NOW() - INTERVAL '24 hours'
                """),
            ).fetchone()
            total = int(ai[0] or 0)
            summary['ai_calls'] = {
                'total': total,
                'tokens': int(ai[1] or 0),
                'cost_24h': float(ai[2] or 0),
                'errors': int(ai[3] or 0),
                'rag_hits': int(ai[4] or 0),
                'cache_hits': int(ai[5] or 0),
                'error_rate': (float(ai[3] or 0) / total * 100) if total else 0,
                'rag_rate': (float(ai[4] or 0) / total * 100) if total else 0,
                'cache_rate': (float(ai[5] or 0) / total * 100) if total else 0,
            }
        except Exception:
            summary['ai_calls'] = {}

        # 當月成本
        try:
            month_cost = session.execute(
                sa_text("SELECT COALESCE(SUM(cost_usd), 0) FROM ai_calls WHERE called_at >= :ms"),
                {'ms': month_start},
            ).fetchone()[0]
            summary['month_cost'] = float(month_cost or 0)
        except Exception:
            summary['month_cost'] = 0

        # 預算 over 80%
        try:
            budgets = session.execute(
                sa_text("""
                    SELECT b.period, b.provider, b.budget_usd, b.alert_pct,
                           COALESCE((
                               SELECT SUM(cost_usd) FROM ai_calls
                               WHERE called_at >= :ms
                                 AND (b.provider IS NULL OR provider = b.provider)
                           ), 0) AS spent
                    FROM ai_call_budgets b
                """),
                {'ms': month_start},
            ).fetchall()
            over_threshold = []
            for r in budgets:
                budget = float(r[2] or 0)
                spent = float(r[4] or 0)
                ratio = spent / budget if budget > 0 else 0
                threshold = float(r[3] or 80) / 100
                if ratio >= threshold:
                    over_threshold.append({
                        'period': r[0], 'provider': r[1] or '(全部)',
                        'spent': spent, 'budget': budget, 'ratio': ratio,
                    })
            summary['budget_alerts'] = over_threshold
        except Exception:
            summary['budget_alerts'] = []

        # 待審 + 蒸餾池
        try:
            ep_pending = session.execute(
                sa_text("SELECT COUNT(*) FROM learning_episodes WHERE promotion_status = 'awaiting_review' AND reviewed_at IS NULL"),
            ).fetchone()[0]
            ep_total_30d = session.execute(
                sa_text("SELECT COUNT(*) FROM learning_episodes WHERE created_at >= NOW() - INTERVAL '30 days'"),
            ).fetchone()[0]
            ep_approved_30d = session.execute(
                sa_text("SELECT COUNT(*) FROM learning_episodes WHERE created_at >= NOW() - INTERVAL '30 days' AND promotion_status = 'approved'"),
            ).fetchone()[0]
            summary['episodes'] = {
                'pending': int(ep_pending or 0),
                'total_30d': int(ep_total_30d or 0),
                'approved_30d': int(ep_approved_30d or 0),
                'approval_rate': (float(ep_approved_30d or 0) / float(ep_total_30d) * 100) if ep_total_30d else 0,
            }
        except Exception:
            summary['episodes'] = {}

        # PPT 視覺審核 7d
        try:
            ppt = session.execute(
                sa_text("""
                    SELECT COUNT(*),
                           COUNT(*) FILTER (WHERE audit_status='passed'),
                           COUNT(*) FILTER (WHERE audit_status='failed')
                    FROM ppt_audit_results
                    WHERE audited_at >= NOW() - INTERVAL '7 days'
                """),
            ).fetchone()
            ppt_total = int(ppt[0] or 0)
            summary['ppt'] = {
                'total': ppt_total,
                'passed': int(ppt[1] or 0),
                'failed': int(ppt[2] or 0),
                'pass_rate': (float(ppt[1] or 0) / ppt_total * 100) if ppt_total else 0,
            }
        except Exception:
            summary['ppt'] = {}

        # AIOps 7d
        try:
            inc = session.execute(
                sa_text("""
                    SELECT COUNT(*),
                           COUNT(*) FILTER (WHERE status='open'),
                           COUNT(*) FILTER (WHERE severity IN ('P0','P1'))
                    FROM incidents
                    WHERE created_at >= NOW() - INTERVAL '7 days'
                """),
            ).fetchone()
            heal = session.execute(
                sa_text("""
                    SELECT COUNT(*),
                           COUNT(*) FILTER (WHERE result='success')
                    FROM heal_logs
                    WHERE created_at >= NOW() - INTERVAL '7 days'
                """),
            ).fetchone()
            summary['aiops'] = {
                'incidents_total': int(inc[0] or 0),
                'incidents_open': int(inc[1] or 0),
                'incidents_p0_p1': int(inc[2] or 0),
                'heals_total': int(heal[0] or 0),
                'heals_success': int(heal[1] or 0),
                'heal_rate': (float(heal[1] or 0) / float(heal[0]) * 100) if heal[0] else 0,
            }
        except Exception:
            summary['aiops'] = {}

        # MCP 24h
        try:
            mcp = session.execute(
                sa_text("""
                    SELECT COUNT(*), COUNT(DISTINCT server),
                           COUNT(*) FILTER (WHERE cache_hit),
                           COALESCE(SUM(cost_usd), 0)
                    FROM mcp_calls
                    WHERE called_at >= NOW() - INTERVAL '24 hours'
                """),
            ).fetchone()
            mcp_total = int(mcp[0] or 0)
            summary['mcp'] = {
                'total': mcp_total,
                'servers': int(mcp[1] or 0),
                'cache_hits': int(mcp[2] or 0),
                'cost': float(mcp[3] or 0),
                'cache_rate': (float(mcp[2] or 0) / mcp_total * 100) if mcp_total else 0,
            }
        except Exception:
            summary['mcp'] = {}
    finally:
        session.close()

    # Phase 51 O-3: 24h 三主機健康 sparkline 資料（每小時 bucket × 3 host）
    host_sparkline = {}
    try:
        s_sp = get_session()
        try:
            sp_rows = s_sp.execute(
                sa_text("""
                    SELECT host_label,
                           date_trunc('hour', probed_at) AS hr,
                           COUNT(*) AS total,
                           COUNT(*) FILTER (WHERE healthy) AS up
                    FROM host_health_probes
                    WHERE probed_at >= NOW() - INTERVAL '24 hours'
                    GROUP BY host_label, hr
                    ORDER BY host_label, hr ASC
                """),
            ).fetchall()
            for r in sp_rows:
                label, hr, total, up = r[0], r[1], int(r[2] or 0), int(r[3] or 0)
                if label not in host_sparkline:
                    host_sparkline[label] = {'hours': [], 'uptime_pct': []}
                host_sparkline[label]['hours'].append(
                    hr.strftime('%H:00') if hr else ''
                )
                host_sparkline[label]['uptime_pct'].append(
                    (up / total * 100) if total else 0
                )
        finally:
            s_sp.close()
    except Exception:
        pass

    return render_template(
        'admin/observability_overview.html',
        active_page='obs_overview',
        summary=summary,
        host_sparkline=host_sparkline,
        today=today.strftime('%Y-%m-%d'),
    )


# ─────────────────────────────────────────────────────────────────────────────
# /observability/rag_queries — Phase 51 RAG 召回詳情
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/rag_queries')
@login_required
def rag_queries_dashboard():
    """Phase 51 — RAG 召回詳情：每筆 query 的命中、saved_call、反饋。

    補完 RAG 觀測深度：之前只看 caller 級命中率，現在看每筆查詢的真實內容。
    """
    hours = int(request.args.get('hours', '24'))
    caller_filter = request.args.get('caller', '').strip()
    saved_only = request.args.get('saved_only', '').strip() == '1'

    session = get_session()
    try:
        rag_query_log_exists = bool(session.execute(
            sa_text("SELECT to_regclass('public.rag_query_log') IS NOT NULL")
        ).scalar())
        if not rag_query_log_exists:
            return render_template(
                'admin/rag_queries.html',
                active_page='obs_rag_queries',
                hours=hours,
                caller_filter=caller_filter,
                saved_only=saved_only,
                summary={},
                callers=[],
                by_caller=[],
                queries=[],
                error='rag_query_log 尚未建立，RAG 召回資料待接入。',
            )

        # 整體統計
        summary_row = session.execute(
            sa_text("""
                SELECT COUNT(*) AS total,
                       COUNT(*) FILTER (WHERE saved_call) AS saved,
                       COUNT(*) FILTER (WHERE hit_count > 0) AS with_hits,
                       COALESCE(AVG(hit_count), 0) AS avg_hits,
                       COALESCE(AVG(feedback_score) FILTER (WHERE feedback_score IS NOT NULL), 0) AS avg_score,
                       COUNT(*) FILTER (WHERE feedback_score IS NOT NULL) AS feedback_count,
                       COUNT(DISTINCT caller) AS distinct_callers
                FROM rag_query_log
                WHERE queried_at >= NOW() - (:h * INTERVAL '1 hour')
            """),
            {'h': hours},
        ).fetchone()
        total = int(summary_row[0] or 0)
        saved = int(summary_row[1] or 0)
        with_hits = int(summary_row[2] or 0)
        summary = {
            'total': total,
            'saved': saved,
            'with_hits': with_hits,
            'no_hits': total - with_hits,
            'avg_hits': round(float(summary_row[3] or 0), 2),
            'avg_score': round(float(summary_row[4] or 0), 2),
            'feedback_count': int(summary_row[5] or 0),
            'distinct_callers': int(summary_row[6] or 0),
            'saved_rate': (float(saved) / total * 100) if total else 0,
            'hit_rate': (float(with_hits) / total * 100) if total else 0,
        }

        # caller 列表（dropdown）
        callers = session.execute(
            sa_text("""
                SELECT DISTINCT caller FROM rag_query_log
                WHERE queried_at >= NOW() - (:h * INTERVAL '1 hour')
                ORDER BY caller
            """),
            {'h': hours},
        ).fetchall()
        caller_list = [r[0] for r in callers]

        # by caller 統計
        by_caller = session.execute(
            sa_text("""
                SELECT caller,
                       COUNT(*) AS total,
                       COUNT(*) FILTER (WHERE saved_call) AS saved,
                       COUNT(*) FILTER (WHERE hit_count > 0) AS with_hits,
                       COALESCE(AVG(feedback_score) FILTER (WHERE feedback_score IS NOT NULL), 0) AS avg_score,
                       COUNT(*) FILTER (WHERE feedback_score IS NOT NULL) AS fb_count
                FROM rag_query_log
                WHERE queried_at >= NOW() - (:h * INTERVAL '1 hour')
                GROUP BY caller
                ORDER BY total DESC
            """),
            {'h': hours},
        ).fetchall()

        # 最近 50 筆查詢（套 caller filter + saved_only）
        params = {'h': hours, 'caller_f': caller_filter}
        recent_queries = session.execute(
            sa_text(f"""
                SELECT id, queried_at, caller, LEFT(query_text, 200) AS qtext,
                       top_k, threshold, hit_count, used_results, saved_call,
                       feedback_score, request_id
                FROM rag_query_log
                WHERE queried_at >= NOW() - (:h * INTERVAL '1 hour')
                  AND (:caller_f = '' OR caller = :caller_f)
                  {"AND saved_call = TRUE" if saved_only else ""}
                ORDER BY queried_at DESC
                LIMIT 50
            """),
            params,
        ).fetchall()
        queries = []
        for r in recent_queries:
            used_ids = list(r[7]) if r[7] else []
            queries.append({
                'id': int(r[0]),
                'queried_at': r[1].strftime('%Y-%m-%d %H:%M:%S') if r[1] else '',
                'caller': r[2],
                'query_text': r[3] or '',
                'top_k': int(r[4] or 0),
                'threshold': round(float(r[5] or 0), 3),
                'hit_count': int(r[6] or 0),
                'used_results': used_ids,
                'saved_call': bool(r[8]),
                'feedback_score': int(r[9]) if r[9] is not None else None,
                'request_id': r[10],
            })

        return render_template(
            'admin/rag_queries.html',
            active_page='obs_rag_queries',
            hours=hours,
            caller_filter=caller_filter,
            saved_only=saved_only,
            summary=summary,
            callers=caller_list,
            by_caller=[
                {
                    'caller': r[0], 'total': int(r[1] or 0),
                    'saved': int(r[2] or 0), 'with_hits': int(r[3] or 0),
                    'avg_score': round(float(r[4] or 0), 2),
                    'fb_count': int(r[5] or 0),
                    'saved_rate': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0,
                    'hit_rate': (float(r[3] or 0) / float(r[1]) * 100) if r[1] else 0,
                }
                for r in by_caller
            ],
            queries=queries,
            error=None,
        )
    except Exception as e:
        return render_template(
            'admin/rag_queries.html',
            active_page='obs_rag_queries', hours=hours,
            caller_filter=caller_filter, saved_only=saved_only,
            summary={}, callers=[], by_caller=[], queries=[],
            error='RAG 召回資料暫時不可用，已切換安全空狀態。',
        )
    finally:
        session.close()


@admin_observability_bp.route('/rag_queries/<int:query_id>/hits', methods=['GET'])
@login_required
def rag_query_hits(query_id: int):
    """Phase 51 — JSON API：回傳單筆 query 的 hits 詳細內容（給 modal 展開）。"""
    try:
        session = get_session()
        try:
            row = session.execute(
                sa_text("""
                    SELECT id, query_text, used_results, hit_count, threshold
                    FROM rag_query_log WHERE id = :id
                """),
                {'id': query_id},
            ).fetchone()
            if not row:
                return jsonify({'ok': False, 'error': 'not found'}), 404

            used_ids = list(row[2]) if row[2] else []
            hits = []
            if used_ids:
                rows = session.execute(
                    sa_text("""
                        SELECT id, insight_type, period, product_sku,
                               LEFT(content, 300) AS preview, created_at
                        FROM ai_insights
                        WHERE id = ANY(:ids)
                        ORDER BY created_at DESC
                    """),
                    {'ids': used_ids},
                ).fetchall()
                hits = [
                    {
                        'id': int(h[0]),
                        'insight_type': h[1],
                        'period': h[2],
                        'product_sku': h[3],
                        'content': h[4] or '',
                        'created_at': h[5].strftime('%Y-%m-%d') if h[5] else '',
                    }
                    for h in rows
                ]
            return jsonify({
                'ok': True,
                'query_id': query_id,
                'query_text': row[1],
                'hit_count': int(row[3] or 0),
                'threshold': round(float(row[4] or 0), 3),
                'hits': hits,
            })
        finally:
            session.close()
    except Exception as e:
        return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500


# ─────────────────────────────────────────────────────────────────────────────
# /observability/business_intel — Phase 48 商業面 × AI 編排
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/business_intel')
@login_required
def business_intel_dashboard():
    """Phase 48 — 商業面 × AI 編排：把 AI 觀測台延伸到商業層級。

    展現「AI 在做什麼生意」：
    - ai_price_recommendations × competitor_prices: AI 看到什麼定價機會
    - action_plans × action_outcomes: 計畫到 verdict 的閉環
    - competitor_match_attempts: 競品比對失敗追蹤
    """
    days = int(request.args.get('days', '7'))
    session = get_session()
    try:
        # 1. ai_price_recommendations 30d 總覽
        rec_summary = session.execute(
            sa_text(f"""
                SELECT strategy, COUNT(*) AS cnt,
                       COALESCE(AVG(confidence), 0) AS avg_conf,
                       COALESCE(AVG(gap_pct), 0) AS avg_gap_pct,
                       COALESCE(AVG(sales_7d_delta), 0) AS avg_sales_delta
                FROM ai_price_recommendations
                WHERE created_at >= NOW() - INTERVAL '{int(days)} days'
                GROUP BY strategy ORDER BY cnt DESC
            """),
        ).fetchall()
        rec_by_strategy = [
            {
                'strategy': r[0], 'count': int(r[1] or 0),
                'avg_confidence': round(float(r[2] or 0), 3),
                'avg_gap_pct': round(float(r[3] or 0), 2),
                'avg_sales_delta': round(float(r[4] or 0), 2),
            }
            for r in rec_summary
        ]

        # 2. ai_price_recommendations 最近 20 筆詳細
        latest_recs = session.execute(
            sa_text("""
                SELECT id, sku, LEFT(name, 50), strategy, confidence,
                       momo_price, pchome_price, gap_pct, sales_7d_delta,
                       LEFT(reason, 120), created_at
                FROM ai_price_recommendations
                ORDER BY created_at DESC LIMIT 20
            """),
        ).fetchall()
        latest_recommendations = [
            {
                'id': r[0], 'sku': r[1], 'name': r[2], 'strategy': r[3],
                'confidence': round(float(r[4] or 0), 3),
                'momo_price': float(r[5] or 0),
                'pchome_price': float(r[6] or 0) if r[6] else None,
                'gap_pct': round(float(r[7] or 0), 2),
                'sales_delta': round(float(r[8] or 0), 2) if r[8] is not None else None,
                'reason': r[9] or '',
                'created_at': r[10].strftime('%m-%d %H:%M') if r[10] else '',
            }
            for r in latest_recs
        ]

        # 3. action_plans × action_outcomes 閉環（30d）
        closed_loops = session.execute(
            sa_text(f"""
                SELECT p.id, p.sku, p.plan_type, p.status,
                       p.created_by, p.created_at, p.executed_at,
                       o.verdict, o.metric_type, o.before_val, o.after_val
                FROM action_plans p
                LEFT JOIN action_outcomes o ON o.plan_id = p.id
                WHERE p.created_at >= NOW() - INTERVAL '{int(days)} days'
                ORDER BY p.created_at DESC LIMIT 25
            """),
        ).fetchall()
        loop_records = []
        for r in closed_loops:
            before = float(r[9]) if r[9] is not None else None
            after = float(r[10]) if r[10] is not None else None
            change_pct = None
            if before and before != 0 and after is not None:
                change_pct = (after - before) / abs(before) * 100
            loop_records.append({
                'plan_id': r[0], 'sku': r[1], 'plan_type': r[2],
                'status': r[3], 'created_by': r[4],
                'created_at': r[5].strftime('%m-%d %H:%M') if r[5] else '',
                'executed_at': r[6].strftime('%m-%d %H:%M') if r[6] else None,
                'verdict': r[7], 'metric_type': r[8],
                'before': before, 'after': after, 'change_pct': change_pct,
            })

        # 4. action_outcomes verdict 統計
        verdict_summary = session.execute(
            sa_text(f"""
                SELECT verdict, COUNT(*) AS cnt,
                       AVG(after_val - before_val) AS avg_delta
                FROM action_outcomes
                WHERE created_at >= NOW() - INTERVAL '{int(days)} days'
                  AND before_val IS NOT NULL AND after_val IS NOT NULL
                GROUP BY verdict ORDER BY cnt DESC
            """),
        ).fetchall()
        verdict_stats = [
            {
                'verdict': r[0] or 'unknown', 'count': int(r[1] or 0),
                'avg_delta': round(float(r[2] or 0), 2),
            }
            for r in verdict_summary
        ]

        # 5. competitor_match_attempts 失敗統計（30d）
        match_attempts = session.execute(
            sa_text(f"""
                SELECT attempt_status, COUNT(*) AS cnt,
                       COALESCE(AVG(candidate_count), 0) AS avg_candidates,
                       COALESCE(AVG(best_match_score), 0) AS avg_score
                FROM competitor_match_attempts
                WHERE attempted_at >= NOW() - INTERVAL '{int(days)} days'
                GROUP BY attempt_status ORDER BY cnt DESC
            """),
        ).fetchall()
        match_stats = [
            {
                'status': r[0], 'count': int(r[1] or 0),
                'avg_candidates': round(float(r[2] or 0), 1),
                'avg_score': round(float(r[3] or 0), 3),
            }
            for r in match_attempts
        ]

        # 6. competitor_prices 24h 變動 TOP 10
        recent_competitor = session.execute(
            sa_text("""
                SELECT cph.sku, cph.competitor_product_name, cph.price,
                       cph.momo_price, cph.discount_pct, cph.match_score,
                       cph.crawled_at
                FROM competitor_price_history cph
                WHERE cph.crawled_at >= NOW() - INTERVAL '24 hours'
                  AND cph.match_score >= 0.7
                ORDER BY cph.crawled_at DESC LIMIT 12
            """),
        ).fetchall()
        recent_competitor_prices = [
            {
                'sku': r[0],
                'product_name': (r[1] or '')[:50],
                'pchome_price': float(r[2] or 0),
                'momo_price': float(r[3] or 0) if r[3] else None,
                'discount_pct': int(r[4]) if r[4] else None,
                'match_score': round(float(r[5] or 0), 3),
                'gap': (float(r[3]) - float(r[2])) if (r[2] and r[3]) else None,
                'crawled_at': r[6].strftime('%m-%d %H:%M') if r[6] else '',
            }
            for r in recent_competitor
        ]

        # 7. 高 confidence 但未 follow-through (recommendation 沒對應 action_plan)
        unfollowed = session.execute(
            sa_text(f"""
                SELECT COUNT(*)
                FROM ai_price_recommendations r
                WHERE r.created_at >= NOW() - INTERVAL '{int(days)} days'
                  AND r.confidence >= 0.7
                  AND NOT EXISTS (
                      SELECT 1 FROM action_plans p
                      WHERE p.sku = r.sku
                        AND p.created_at >= r.created_at
                        AND p.created_at < r.created_at + INTERVAL '7 days'
                  )
            """),
        ).fetchone()
        unfollowed_count = int(unfollowed[0] or 0) if unfollowed else 0

        return render_template(
            'admin/business_intel.html',
            active_page='obs_business_intel',
            days=days,
            rec_by_strategy=rec_by_strategy,
            latest_recommendations=latest_recommendations,
            loop_records=loop_records,
            verdict_stats=verdict_stats,
            match_stats=match_stats,
            recent_competitor_prices=recent_competitor_prices,
            unfollowed_count=unfollowed_count,
            error=None,
        )
    except Exception as e:
        return render_template(
            'admin/business_intel.html',
            active_page='obs_business_intel', days=days,
            rec_by_strategy=[], latest_recommendations=[], loop_records=[],
            verdict_stats=[], match_stats=[], recent_competitor_prices=[],
            unfollowed_count=0,
            error='商業 AI 資料暫時不可用，已切換安全空狀態。',
        )
    finally:
        session.close()


# ─────────────────────────────────────────────────────────────────────────────
# /observability/agent_orchestration — Phase 46 編排矩陣
# ─────────────────────────────────────────────────────────────────────────────

# caller → agent 歸類規則（同 services/* 各 agent 真實 caller 值）
_AGENT_CALLER_GROUPS = {
    'openclaw': [
        'openclaw_qa', 'openclaw_daily', 'openclaw_daily_insight',
        'openclaw_meta', 'openclaw_monthly', 'openclaw_weekly',
        'openclaw_bot_main', 'openclaw_bot_gemini', 'openclaw_bot_nim',
        'sales_copy', 'code_review_openclaw',
    ],
    'hermes': [
        'hermes_analyst', 'hermes_intent', 'code_review_hermes',
    ],
    'nemotron': [
        'nemotron_dispatch',
    ],
    'elephant_alpha': [
        'ea_engine', 'code_review_elephant',
    ],
}
_AGENT_LABELS = {
    'openclaw': ('🤖 OpenClaw', '主編排者 / Bot 對話 / 報告生成'),
    'hermes': ('🔍 Hermes', '價格/程式碼分析師'),
    'nemotron': ('🧬 NemoTron', '任務 dispatcher'),
    'elephant_alpha': ('🐘 ElephantAlpha', '自主決策引擎'),
}

# Provider → 類別歸類
_PROVIDER_TIER = {
    'gcp_ollama': 'ollama_local',
    'ollama_secondary': 'ollama_local',
    'ollama_111': 'ollama_local',
    'ollama_other': 'ollama_local',
    'gemini': 'paid_external',
    'claude': 'paid_external',
    'nim': 'paid_external',
    'nim_via_elephant': 'paid_external',
    'openrouter': 'paid_external',
}


@admin_observability_bp.route('/agent_orchestration')
@login_required
def agent_orchestration_dashboard():
    """Phase 46 — 4 Agent × Models × MCP × RAG 編排矩陣

    展現「組合發揮」：每個 agent 在 24h 內如何調用 Ollama/Gemini，
    搭配 MCP tool（外部 + 內部 mcp_collector），與 RAG 知識庫的協作。

    資料來源：ai_calls × mcp_calls × rag_query_log 三表跨 JOIN + caller 分組。
    """
    hours = int(request.args.get('hours', '24'))
    session = get_session()
    try:
        # 1. 整體統計
        overall = session.execute(
            sa_text("""
                SELECT COUNT(*),
                       COALESCE(SUM(cost_usd), 0),
                       COUNT(*) FILTER (WHERE provider IN ('gemini','claude','nim','openrouter','nim_via_elephant')),
                       COUNT(*) FILTER (WHERE provider IN ('gcp_ollama','ollama_secondary','ollama_111','ollama_other')),
                       COUNT(*) FILTER (WHERE rag_hit),
                       COALESCE(SUM(input_tokens + output_tokens), 0)
                FROM ai_calls
                WHERE called_at >= NOW() - (:h * INTERVAL '1 hour')
            """),
            {'h': hours},
        ).fetchone()
        total_calls = int(overall[0] or 0)
        total_cost = float(overall[1] or 0)
        paid_calls = int(overall[2] or 0)
        local_calls = int(overall[3] or 0)
        rag_hits = int(overall[4] or 0)
        total_tokens = int(overall[5] or 0)
        mcp_calls_table_exists = bool(session.execute(
            sa_text("SELECT to_regclass('public.mcp_calls') IS NOT NULL")
        ).scalar())

        # 2. 每個 agent group 的細節
        agent_matrix = []
        for agent_key, callers in _AGENT_CALLER_GROUPS.items():
            ag_row = session.execute(
                sa_text("""
                    SELECT COUNT(*) AS calls,
                           COALESCE(SUM(input_tokens + output_tokens), 0) AS tokens,
                           COALESCE(SUM(cost_usd), 0) AS cost,
                           COUNT(*) FILTER (WHERE rag_hit) AS rag_hits,
                           COUNT(*) FILTER (WHERE provider IN ('gcp_ollama','ollama_secondary','ollama_111','ollama_other')) AS ollama,
                           COUNT(*) FILTER (WHERE provider = 'gcp_ollama') AS ollama_gcp_a,
                           COUNT(*) FILTER (WHERE provider = 'ollama_secondary') AS ollama_gcp_b,
                           COUNT(*) FILTER (WHERE provider = 'ollama_111') AS ollama_111,
                           COUNT(*) FILTER (WHERE provider = 'gemini') AS gemini,
                           COUNT(*) FILTER (WHERE provider IN ('claude','nim','openrouter','nim_via_elephant')) AS other_paid,
                           COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')) AS errors,
                           COALESCE(AVG(duration_ms), 0) AS avg_ms
                    FROM ai_calls
                    WHERE called_at >= NOW() - (:h * INTERVAL '1 hour')
                      AND caller = ANY(:callers)
                """),
                {'h': hours, 'callers': callers},
            ).fetchone()
            calls = int(ag_row[0] or 0)
            if calls == 0:
                # 沒呼叫也佔位顯示
                agent_matrix.append({
                    'key': agent_key, 'label': _AGENT_LABELS[agent_key][0],
                    'desc': _AGENT_LABELS[agent_key][1],
                    'calls': 0, 'tokens': 0, 'cost': 0,
                    'rag_hits': 0, 'rag_rate': 0,
                    'ollama_pct': 0, 'gemini_pct': 0, 'paid_pct': 0,
                    'ollama_gcp_a': 0, 'ollama_gcp_b': 0, 'ollama_111': 0,
                    'gemini': 0, 'other_paid': 0,
                    'errors': 0, 'error_rate': 0,
                    'avg_ms': 0, 'mcp_calls': 0, 'mcp_rate': 0,
                    'callers_in_group': callers,
                })
                continue

            # MCP 編排率（透過 request_id 串接）。mcp_calls 尚未 migration 時安全降級為 0。
            if mcp_calls_table_exists:
                mcp_count = session.execute(
                    sa_text("""
                        SELECT COUNT(DISTINCT a.request_id)
                        FROM ai_calls a
                        INNER JOIN mcp_calls m ON m.request_id = a.request_id
                        WHERE a.called_at >= NOW() - (:h * INTERVAL '1 hour')
                          AND a.caller = ANY(:callers)
                          AND a.request_id IS NOT NULL
                    """),
                    {'h': hours, 'callers': callers},
                ).fetchone()[0] or 0
            else:
                mcp_count = 0

            errors = int(ag_row[10] or 0)
            ollama = int(ag_row[4] or 0)
            gemini = int(ag_row[8] or 0)
            other_paid = int(ag_row[9] or 0)

            agent_matrix.append({
                'key': agent_key,
                'label': _AGENT_LABELS[agent_key][0],
                'desc': _AGENT_LABELS[agent_key][1],
                'calls': calls,
                'tokens': int(ag_row[1] or 0),
                'cost': float(ag_row[2] or 0),
                'rag_hits': int(ag_row[3] or 0),
                'rag_rate': (float(ag_row[3] or 0) / calls * 100) if calls else 0,
                'ollama': ollama, 'ollama_pct': (ollama / calls * 100) if calls else 0,
                'ollama_gcp_a': int(ag_row[5] or 0),
                'ollama_gcp_b': int(ag_row[6] or 0),
                'ollama_111': int(ag_row[7] or 0),
                'gemini': gemini, 'gemini_pct': (gemini / calls * 100) if calls else 0,
                'other_paid': other_paid,
                'paid_pct': ((gemini + other_paid) / calls * 100) if calls else 0,
                'errors': errors, 'error_rate': (errors / calls * 100) if calls else 0,
                'avg_ms': int(ag_row[11] or 0),
                'mcp_calls': int(mcp_count),
                'mcp_rate': (float(mcp_count) / calls * 100) if calls else 0,
                'callers_in_group': callers,
            })

        # 3. MCP server 24h 工作量（同 host_health 邏輯）
        if mcp_calls_table_exists:
            mcp_servers = session.execute(
                sa_text("""
                    SELECT server, caller, COUNT(*) AS calls,
                           COUNT(*) FILTER (WHERE cache_hit) AS cache_hits,
                           COALESCE(SUM(cost_usd), 0) AS cost
                    FROM mcp_calls
                    WHERE called_at >= NOW() - (:h * INTERVAL '1 hour')
                    GROUP BY server, caller
                    ORDER BY calls DESC
                    LIMIT 30
                """),
                {'h': hours},
            ).fetchall()
            mcp_matrix = [
                {
                    'server': r[0], 'caller': r[1],
                    'calls': int(r[2] or 0),
                    'cache_hits': int(r[3] or 0),
                    'cost': float(r[4] or 0),
                    'cache_rate': (float(r[3] or 0) / float(r[2]) * 100) if r[2] else 0,
                }
                for r in mcp_servers
            ]
        else:
            mcp_matrix = []

        # 4. 自動編排建議（rule-based 提案）
        recommendations = []
        if not mcp_calls_table_exists:
            recommendations.append({
                'severity': 'med', 'agent': 'MCP 觀測',
                'finding': 'mcp_calls 尚未建立，MCP 編排率目前以 0 顯示',
                'suggestion': '執行 Phase 10.7 migration 後，本頁會自動接回 MCP server/caller 矩陣',
            })
        for ag in agent_matrix:
            if ag['calls'] == 0:
                continue
            # 規則 1：付費比例 > 50% 且 ollama 比例 < 20% → 建議切 Hermes-first
            if ag['paid_pct'] > 50 and ag['ollama_pct'] < 20:
                recommendations.append({
                    'severity': 'high', 'agent': ag['label'],
                    'finding': f"付費 LLM 比例 {ag['paid_pct']:.0f}%（cost ${ag['cost']:.2f}）",
                    'suggestion': '改用 Hermes-first 短路機制：先試 Ollama 三主機 5s timeout，0 hits 才 escalate Gemini',
                })
            # 規則 2：錯誤率 > 10% → 建議跑 code review
            if ag['error_rate'] > 10:
                recommendations.append({
                    'severity': 'high', 'agent': ag['label'],
                    'finding': f"錯誤率 {ag['error_rate']:.1f}%（{ag['errors']}/{ag['calls']}）",
                    'suggestion': '觸發 Code Review Pipeline 找 regression（ai_calls 觀測台一鍵）',
                })
            # 規則 3：MCP 編排率 < 5% 但 calls 多 → 建議擴大 MCP 使用
            if mcp_calls_table_exists and ag['mcp_rate'] < 5 and ag['calls'] > 50:
                recommendations.append({
                    'severity': 'med', 'agent': ag['label'],
                    'finding': f"MCP 編排率僅 {ag['mcp_rate']:.1f}%，未善用外部工具",
                    'suggestion': '考慮加 MCP omnisearch / firecrawl 補強事實查證鏈',
                })
            # 規則 4：RAG 命中率高（≥40%）但有 saved_call=False 的多 → 提醒 feedback
            if ag['rag_rate'] >= 40 and ag['rag_hits'] >= 20:
                recommendations.append({
                    'severity': 'low', 'agent': ag['label'],
                    'finding': f"RAG 命中率 {ag['rag_rate']:.1f}%（{ag['rag_hits']} hits）— 知識庫貢獻度高",
                    'suggestion': '推 Telegram inline button 收集 feedback_score 強化 promotion gate',
                })
            # 規則 5：111 fallback 比例 > 20% → 警示
            if ag['calls'] > 0 and ag['ollama_111'] / max(ag['calls'], 1) > 0.20:
                fb_pct = ag['ollama_111'] / ag['calls'] * 100
                recommendations.append({
                    'severity': 'med', 'agent': ag['label'],
                    'finding': f"111 fallback 比例 {fb_pct:.0f}%（GCP 兩台不可達？）",
                    'suggestion': '檢查 mo.wooo.work/observability/host_health AIOps incidents',
                })

        return render_template(
            'admin/agent_orchestration.html',
            active_page='obs_agent_orchestration',
            hours=hours,
            agent_matrix=agent_matrix,
            mcp_matrix=mcp_matrix,
            recommendations=recommendations,
            overall={
                'total_calls': total_calls,
                'total_cost': total_cost,
                'total_tokens': total_tokens,
                'paid_calls': paid_calls,
                'local_calls': local_calls,
                'rag_hits': rag_hits,
                'paid_pct': (paid_calls / total_calls * 100) if total_calls else 0,
                'local_pct': (local_calls / total_calls * 100) if total_calls else 0,
                'rag_rate': (rag_hits / total_calls * 100) if total_calls else 0,
            },
            error=None,
        )
    except Exception as e:
        return render_template(
            'admin/agent_orchestration.html',
            active_page='obs_agent_orchestration', hours=hours,
            agent_matrix=[], mcp_matrix=[], recommendations=[], overall={},
            error='資料查詢暫時不可用，已切換安全空狀態。',
        )
    finally:
        session.close()


# ─────────────────────────────────────────────────────────────────────────────
# /observability/ai_calls — Phase 27 主入口
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/ai_calls')
@login_required
def ai_calls_dashboard():
    """ai_calls 表觀測 dashboard（24h 預設視窗）"""
    hours = int(request.args.get('hours', '24'))
    caller_filter = request.args.get('caller', '').strip()
    provider_filter = request.args.get('provider', '').strip()

    since = datetime.now() - timedelta(hours=hours)
    session = get_session()
    try:
        # 1. 總覽
        summary = session.execute(
            sa_text("""
                SELECT
                  COUNT(*) AS total_calls,
                  COALESCE(SUM(input_tokens + output_tokens), 0) AS total_tokens,
                  COALESCE(SUM(cost_usd), 0) AS total_cost,
                  COALESCE(AVG(duration_ms), 0) AS avg_duration,
                  COUNT(*) FILTER (WHERE status = 'ok') AS ok_calls,
                  COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')) AS error_calls,
                  COUNT(*) FILTER (WHERE rag_hit) AS rag_hits,
                  COUNT(*) FILTER (WHERE cache_hit) AS cache_hits
                FROM ai_calls
                WHERE called_at >= :since
            """),
            {'since': since},
        ).fetchone()

        # 2. by provider
        by_provider = session.execute(
            sa_text("""
                SELECT provider, COUNT(*) AS calls,
                       COALESCE(SUM(input_tokens + output_tokens), 0) AS tokens,
                       COALESCE(SUM(cost_usd), 0) AS cost
                FROM ai_calls
                WHERE called_at >= :since
                GROUP BY provider
                ORDER BY tokens DESC
            """),
            {'since': since},
        ).fetchall()

        # 3. TOP 100 calls — Phase 33 Critic HIGH #2 修補：
        # 改用固定 SQL + 全綁參數，移除 f-string 動態 WHERE 拼接（防後人不慎注入）
        recent = session.execute(
            sa_text("""
                SELECT id, called_at, caller, provider, model,
                       input_tokens, output_tokens, duration_ms, status,
                       cost_usd, cache_hit, rag_hit
                FROM ai_calls
                WHERE called_at >= :since
                  AND (:caller_f = '' OR caller = :caller_f)
                  AND (:provider_f = '' OR provider = :provider_f)
                ORDER BY called_at DESC
                LIMIT 100
            """),
            {
                'since': since,
                'caller_f': caller_filter,
                'provider_f': provider_filter,
            },
        ).fetchall()

        # 4. caller 列表（給篩選 dropdown）
        callers = session.execute(
            sa_text("""
                SELECT DISTINCT caller FROM ai_calls
                WHERE called_at >= :since ORDER BY caller
            """),
            {'since': since},
        ).fetchall()

        # 5b. Phase 47 K-2: by model 細分（不只 provider，到實際 model）
        by_model = session.execute(
            sa_text("""
                SELECT model, provider, COUNT(*) AS calls,
                       COALESCE(SUM(input_tokens + output_tokens), 0) AS tokens,
                       COALESCE(SUM(cost_usd), 0) AS cost,
                       COALESCE(AVG(duration_ms), 0) AS avg_ms,
                       COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')) AS errors
                FROM ai_calls
                WHERE called_at >= :since
                  AND model IS NOT NULL AND model != ''
                GROUP BY model, provider
                ORDER BY calls DESC
                LIMIT 15
            """),
            {'since': since},
        ).fetchall()

        # 5c. Phase 47 K-2: hourly 呼叫量趨勢（24 個 bucket）
        hourly_trend = session.execute(
            sa_text("""
                SELECT date_trunc('hour', called_at) AS hr,
                       COUNT(*) AS calls,
                       COALESCE(SUM(cost_usd), 0) AS cost,
                       COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only')) AS errors
                FROM ai_calls
                WHERE called_at >= NOW() - INTERVAL '24 hours'
                GROUP BY hr ORDER BY hr ASC
            """),
        ).fetchall()

        # 5d. Phase 47 K-2: agent_context 最近 10 筆（OpenClaw/Hermes 對話上下文）
        recent_contexts = session.execute(
            sa_text("""
                SELECT created_at, agent_name, context_key, ttl_minutes,
                       LEFT(context_val, 120) AS preview
                FROM agent_context
                ORDER BY created_at DESC LIMIT 10
            """),
        ).fetchall()

        # 5. Phase 39 D-3: caller × RAG 命中率 × MCP 編排率（跨表 JOIN）
        # mcp_calls / rag_query_log 尚未 migration 時安全降級，不曝露 DB exception。
        mcp_calls_table_exists = bool(session.execute(
            sa_text("SELECT to_regclass('public.mcp_calls') IS NOT NULL")
        ).scalar())
        rag_query_log_exists = bool(session.execute(
            sa_text("SELECT to_regclass('public.rag_query_log') IS NOT NULL")
        ).scalar())
        if mcp_calls_table_exists and rag_query_log_exists:
            caller_richness = session.execute(
                sa_text("""
                    SELECT a.caller,
                           COUNT(*) AS total_calls,
                           COUNT(*) FILTER (WHERE a.rag_hit) AS rag_hits,
                           COUNT(DISTINCT m.request_id) AS mcp_orchestrated,
                           COALESCE(AVG(rl.feedback_score) FILTER (WHERE rl.feedback_score IS NOT NULL), 0)
                               AS avg_rag_feedback,
                           COUNT(rl.feedback_score) AS feedback_count
                    FROM ai_calls a
                    LEFT JOIN mcp_calls m
                           ON m.request_id = a.request_id
                          AND m.called_at >= :since
                    LEFT JOIN rag_query_log rl
                           ON rl.caller = a.caller
                          AND rl.queried_at >= :since
                    WHERE a.called_at >= :since
                    GROUP BY a.caller
                    HAVING COUNT(*) >= 5
                    ORDER BY total_calls DESC
                    LIMIT 12
                """),
                {'since': since},
            ).fetchall()
        else:
            caller_richness = session.execute(
                sa_text("""
                    SELECT caller,
                           COUNT(*) AS total_calls,
                           COUNT(*) FILTER (WHERE rag_hit) AS rag_hits
                    FROM ai_calls
                    WHERE called_at >= :since
                    GROUP BY caller
                    HAVING COUNT(*) >= 5
                    ORDER BY total_calls DESC
                    LIMIT 12
                """),
                {'since': since},
            ).fetchall()

        return render_template(
            'admin/ai_calls_dashboard.html',
            active_page='obs_ai_calls',
            hours=hours,
            caller_filter=caller_filter,
            provider_filter=provider_filter,
            summary={
                'total_calls': int(summary[0] or 0),
                'total_tokens': int(summary[1] or 0),
                'total_cost': float(summary[2] or 0),
                'avg_duration': int(summary[3] or 0),
                'ok_calls': int(summary[4] or 0),
                'error_calls': int(summary[5] or 0),
                'rag_hits': int(summary[6] or 0),
                'cache_hits': int(summary[7] or 0),
            },
            by_provider=[
                {'provider': r[0], 'calls': int(r[1] or 0),
                 'tokens': int(r[2] or 0), 'cost': float(r[3] or 0)}
                for r in by_provider
            ],
            recent=[
                {'id': r[0], 'called_at': r[1].strftime('%H:%M:%S'),
                 'caller': r[2], 'provider': r[3], 'model': r[4],
                 'in_tokens': int(r[5] or 0), 'out_tokens': int(r[6] or 0),
                 'duration_ms': int(r[7] or 0), 'status': r[8],
                 'cost': float(r[9] or 0), 'cache_hit': bool(r[10]),
                 'rag_hit': bool(r[11])}
                for r in recent
            ],
            callers=[r[0] for r in callers],
            by_model=[
                {
                    'model': r[0], 'provider': r[1],
                    'calls': int(r[2] or 0), 'tokens': int(r[3] or 0),
                    'cost': float(r[4] or 0), 'avg_ms': int(r[5] or 0),
                    'errors': int(r[6] or 0),
                }
                for r in by_model
            ],
            hourly_trend=[
                {
                    'hour': r[0].strftime('%H:%M') if r[0] else '',
                    'calls': int(r[1] or 0),
                    'cost': float(r[2] or 0),
                    'errors': int(r[3] or 0),
                }
                for r in hourly_trend
            ],
            recent_contexts=[
                {
                    'created_at': r[0].strftime('%Y-%m-%d %H:%M') if r[0] else '',
                    'agent_name': r[1], 'context_key': r[2],
                    'ttl_minutes': int(r[3] or 0),
                    'preview': r[4] or '',
                }
                for r in recent_contexts
            ],
            caller_richness=[
                {
                    'caller': r[0],
                    'total_calls': int(r[1] or 0),
                    'rag_hits': int(r[2] or 0),
                    'mcp_orchestrated': int(r[3] or 0) if mcp_calls_table_exists and rag_query_log_exists else 0,
                    'avg_rag_feedback': round(float(r[4] or 0), 2) if mcp_calls_table_exists and rag_query_log_exists else 0,
                    'feedback_count': int(r[5] or 0) if mcp_calls_table_exists and rag_query_log_exists else 0,
                    'rag_hit_rate': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0,
                    'mcp_rate': (float(r[3] or 0) / float(r[1]) * 100) if mcp_calls_table_exists and rag_query_log_exists and r[1] else 0,
                }
                for r in caller_richness
            ],
            error=None,
        )
    except Exception as e:
        return render_template(
            'admin/ai_calls_dashboard.html',
            active_page='obs_ai_calls',
            hours=hours, caller_filter=caller_filter,
            provider_filter=provider_filter,
            summary={}, by_provider=[], recent=[], callers=[], caller_richness=[],
            by_model=[], hourly_trend=[], recent_contexts=[],
            error='AI 呼叫資料暫時不可用，已切換安全空狀態。',
        )
    finally:
        session.close()


# ─────────────────────────────────────────────────────────────────────────────
# /observability/promotion_review — Phase 28 PromotionGate 待審核列表
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/promotion_review')
@login_required
def promotion_review_list():
    """awaiting_review episodes 列表（24h 內 reviewed_at IS NULL）

    Phase 39（D-1）：每筆 episode 自動跑 RAG 找 Top 3 相似已晉升 ai_insights，
    輔助人工判斷晉升價值。RAG fail-safe：失敗則 similar_insights=[]，不擋頁面。
    """
    session = get_session()
    try:
        rows = session.execute(
            sa_text("""
                SELECT id, created_at, episode_type, source_table, source_id,
                       distilled_text, quality_score, weight, promotion_status
                FROM learning_episodes
                WHERE promotion_status = 'awaiting_review'
                  AND reviewed_at IS NULL
                ORDER BY weight DESC, created_at ASC
                LIMIT 50
            """),
        ).fetchall()

        # ai_insights 全表大小（給「晉升後 KB 增長」對照）
        kb_size = 0
        try:
            kb_row = session.execute(
                sa_text("SELECT COUNT(*) FROM ai_insights"),
            ).fetchone()
            kb_size = int(kb_row[0] or 0)
        except Exception:
            pass

        episodes = [
            {'id': r[0], 'created_at': r[1].strftime('%Y-%m-%d %H:%M'),
             'episode_type': r[2], 'source_table': r[3], 'source_id': r[4],
             'distilled_text': (r[5] or '')[:600],
             'quality_score': float(r[6] or 0),
             'weight': float(r[7] or 0),
             'status': r[8],
             'similar_insights': []}
            for r in rows
        ]

        # Phase 39 D-1：對每筆 episode 跑 RAG 找 Top 3 相似已晉升
        try:
            from services.rag_service import rag_service
            for ep in episodes:
                try:
                    rag_result = rag_service.query(
                        text=ep['distilled_text'][:500],
                        caller='admin_promotion_review',
                        top_k=3,
                        threshold=0.7,
                    )
                    ep['similar_insights'] = [
                        {
                            'id': h.get('id'),
                            'insight_type': h.get('insight_type'),
                            'content': (h.get('content') or '')[:180],
                            'similarity': round(float(h.get('similarity', 0)), 3),
                            'created_at': h.get('created_at').strftime('%Y-%m-%d')
                                          if h.get('created_at') else '',
                        }
                        for h in rag_result.hits[:3]
                    ]
                except Exception:
                    logger.debug(
                        "Promotion review similar-insight lookup failed for episode_id=%s",
                        ep.get('id'),
                        exc_info=True,
                    )
        except Exception:
            logger.debug("Promotion review RAG service unavailable; skipping similar insights", exc_info=True)

        # Phase 47 K-4: 蒸餾池 status 分布（30d）
        ep_distribution = session.execute(
            sa_text("""
                SELECT promotion_status, COUNT(*) AS cnt
                FROM learning_episodes
                WHERE created_at >= NOW() - INTERVAL '30 days'
                GROUP BY promotion_status ORDER BY cnt DESC
            """),
        ).fetchall()
        episode_distribution_30d = {r[0]: int(r[1] or 0) for r in ep_distribution}

        # Phase 47 K-4: ai_insights 最近 10 筆已晉升（type/created_at 視覺）
        latest_insights = session.execute(
            sa_text("""
                SELECT id, insight_type, period, product_sku, created_at,
                       LEFT(content, 160) AS preview
                FROM ai_insights
                ORDER BY created_at DESC LIMIT 10
            """),
        ).fetchall()

        # Phase 47 K-4: agent_strategy_weights TOP 12（OpenClaw 學習權重）
        strategy_weights = session.execute(
            sa_text("""
                SELECT strategy_key, weight, success_cnt, fail_cnt, updated_at
                FROM agent_strategy_weights
                ORDER BY (success_cnt + fail_cnt) DESC
                LIMIT 12
            """),
        ).fetchall()

        return render_template(
            'admin/promotion_review.html',
            active_page='obs_promotion_review',
            episodes=episodes,
            kb_size=kb_size,
            episode_distribution_30d=episode_distribution_30d,
            latest_insights=[
                {
                    'id': r[0], 'insight_type': r[1], 'period': r[2],
                    'product_sku': r[3],
                    'created_at': r[4].strftime('%Y-%m-%d %H:%M') if r[4] else '',
                    'preview': r[5] or '',
                }
                for r in latest_insights
            ],
            strategy_weights=[
                {
                    'strategy_key': r[0], 'weight': float(r[1] or 0),
                    'success': int(r[2] or 0), 'fail': int(r[3] or 0),
                    'updated_at': r[4].strftime('%Y-%m-%d') if r[4] else '',
                    'success_rate': (
                        float(r[2] or 0) / float((r[2] or 0) + (r[3] or 0)) * 100
                    ) if ((r[2] or 0) + (r[3] or 0)) > 0 else 0,
                }
                for r in strategy_weights
            ],
            error=None,
        )
    except Exception as e:
        return render_template(
            'admin/promotion_review.html',
            active_page='obs_promotion_review',
            episodes=[],
            kb_size=0,
            episode_distribution_30d={},
            latest_insights=[],
            strategy_weights=[],
            error='RAG 晉升資料暫時不可用，已切換安全空狀態。',
        )
    finally:
        session.close()


@admin_observability_bp.route('/promotion_review/approve/<int:episode_id>', methods=['POST'])
@login_required
def promotion_review_approve(episode_id: int):
    """Web 介面「通過」按鈕 — 等同於 Telegram pg_ok callback"""
    try:
        from services.learning_pipeline import promotion_gate, hash_human_approver
        # 從 Flask session 取（已過 @login_required）— 不信任 client header
        user = get_current_user() or {}
        username = user.get('username', 'web_admin')
        approver_hash = hash_human_approver(username)
        insight_id = promotion_gate.promote(
            episode_id,
            human_approver=approver_hash,
        )
        if insight_id:
            return jsonify({'ok': True, 'insight_id': insight_id, 'approver': approver_hash})
        return jsonify({'ok': False, 'error': 'promote failed'}), 500
    except Exception as e:
        return jsonify({'ok': False, 'error': str(e)[:200]}), 500


@admin_observability_bp.route('/promotion_review/reject/<int:episode_id>', methods=['POST'])
@login_required
def promotion_review_reject(episode_id: int):
    """Web 介面「拒絕」按鈕"""
    try:
        from services.learning_pipeline import promotion_gate, hash_human_approver
        user = get_current_user() or {}
        username = user.get('username', 'web_admin')
        approver_hash = hash_human_approver(username)
        ok = promotion_gate.reject(
            episode_id,
            'rejected_human',
            detail='web admin reject',
            human_approver=approver_hash,
        )
        return jsonify({'ok': ok})
    except Exception as e:
        return jsonify({'ok': False, 'error': str(e)[:200]}), 500


# ─────────────────────────────────────────────────────────────────────────────
# /observability/quality_trend — Phase 25 caller 反饋趨勢視覺化
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/quality_trend')
@login_required
def quality_trend_dashboard():
    """caller × feedback 趨勢（30 日窗格）"""
    days = int(request.args.get('days', '30'))
    try:
        from services.feedback_quality_tracker import (
            compute_caller_quality_trend, get_caller_recommendations,
        )
        trends = compute_caller_quality_trend(days=days)
        recommendations = get_caller_recommendations(days=days)

        # 排序：avg_score 升序（最差先看）
        sorted_trends = sorted(
            trends.items(),
            key=lambda kv: kv[1].get('avg_score', 5),
        )

        # Phase 40 D-6: learning_episodes 各 status 分布（蒸餾池飽和度）
        episode_distribution = {}
        try:
            session = get_session()
            try:
                rows = session.execute(
                    sa_text("""
                        SELECT promotion_status, COUNT(*) AS cnt
                        FROM learning_episodes
                        WHERE created_at >= NOW() - INTERVAL ':days days'
                        GROUP BY promotion_status
                    """).bindparams(days=days),
                ).fetchall() if False else session.execute(
                    sa_text(f"""
                        SELECT promotion_status, COUNT(*) AS cnt
                        FROM learning_episodes
                        WHERE created_at >= NOW() - INTERVAL '{int(days)} days'
                        GROUP BY promotion_status
                    """),
                ).fetchall()
                episode_distribution = {r[0]: int(r[1] or 0) for r in rows}
            finally:
                session.close()
        except Exception:
            pass

        # Phase 40 D-6: 對最差 3 名 caller 跑 RAG 找根因建議
        rag_root_causes = []
        try:
            from services.rag_service import rag_service
            worst_3 = sorted_trends[:3] if len(sorted_trends) >= 3 else sorted_trends
            for caller, info in worst_3:
                if info.get('avg_score', 5) < 3.0 and info.get('total_feedback', 0) >= 3:
                    try:
                        q = (
                            f"caller {caller} 反饋分數低 平均 "
                            f"{info.get('avg_score', 0):.1f}/5 應採取什麼根因排查"
                        )
                        rag_result = rag_service.query(
                            text=q,
                            caller='admin_quality_trend',
                            top_k=2,
                            threshold=0.6,
                        )
                        if rag_result.hits:
                            rag_root_causes.append({
                                'caller': caller,
                                'avg_score': info.get('avg_score', 0),
                                'feedback_n': info.get('total_feedback', 0),
                                'hits': [
                                    {
                                        'id': h.get('id'),
                                        'insight_type': h.get('insight_type'),
                                        'content': (h.get('content') or '')[:200],
                                        'similarity': round(float(h.get('similarity', 0)), 3),
                                    }
                                    for h in rag_result.hits[:2]
                                ],
                            })
                    except Exception:
                        pass
        except Exception:
            pass

        # Phase 47 K-5: action_outcomes verdict 統計（ADR-012 閉環學習結果）
        action_outcomes_stats = []
        action_plans_status = []
        rag_overall_dist = []
        try:
            session = get_session()
            try:
                # action_outcomes verdict 分布（30d）
                ao_rows = session.execute(
                    sa_text(f"""
                        SELECT verdict, COUNT(*) AS cnt
                        FROM action_outcomes
                        WHERE created_at >= NOW() - INTERVAL '{int(days)} days'
                        GROUP BY verdict ORDER BY cnt DESC
                    """),
                ).fetchall()
                action_outcomes_stats = [{'verdict': r[0] or 'unknown', 'count': int(r[1] or 0)} for r in ao_rows]

                # action_plans status 分布（30d）
                ap_rows = session.execute(
                    sa_text(f"""
                        SELECT status, plan_type, COUNT(*) AS cnt
                        FROM action_plans
                        WHERE created_at >= NOW() - INTERVAL '{int(days)} days'
                        GROUP BY status, plan_type ORDER BY cnt DESC
                    """),
                ).fetchall()
                action_plans_status = [
                    {'status': r[0], 'plan_type': r[1] or 'misc', 'count': int(r[2] or 0)}
                    for r in ap_rows
                ]

                # rag_query_log 整體 feedback 分布（不只 caller-level，整體）
                rag_dist_rows = session.execute(
                    sa_text(f"""
                        SELECT feedback_score, COUNT(*) AS cnt
                        FROM rag_query_log
                        WHERE queried_at >= NOW() - INTERVAL '{int(days)} days'
                          AND feedback_score IS NOT NULL
                        GROUP BY feedback_score ORDER BY feedback_score
                    """),
                ).fetchall()
                rag_overall_dist = [{'score': int(r[0] or 0), 'count': int(r[1] or 0)} for r in rag_dist_rows]
            finally:
                session.close()
        except Exception:
            pass

        return render_template(
            'admin/quality_trend.html',
            active_page='obs_quality_trend',
            days=days,
            trends=[(c, info) for c, info in sorted_trends],
            recommendations=recommendations,
            episode_distribution=episode_distribution,
            rag_root_causes=rag_root_causes,
            action_outcomes_stats=action_outcomes_stats,
            action_plans_status=action_plans_status,
            rag_overall_dist=rag_overall_dist,
            error=None,
        )
    except Exception as e:
        return render_template(
            'admin/quality_trend.html',
            active_page='obs_quality_trend',
            days=days, trends=[], recommendations=[],
            episode_distribution={}, rag_root_causes=[],
            action_outcomes_stats=[], action_plans_status=[], rag_overall_dist=[],
            error='AI 品質資料暫時不可用，已切換安全空狀態。',
        )


# ─────────────────────────────────────────────────────────────────────────────
# /observability/budget — Phase 29 預算管理 + 手動 throttle
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/budget')
@login_required
def budget_dashboard():
    """ai_call_budgets 編輯 + 當月 spent 即時對比"""
    from datetime import datetime as _dt
    today = _dt.now()
    month_start = _dt(today.year, today.month, 1)

    session = get_session()
    try:
        ai_call_budgets_exists = bool(session.execute(
            sa_text("SELECT to_regclass('public.ai_call_budgets') IS NOT NULL")
        ).scalar())
        if ai_call_budgets_exists:
            budgets = session.execute(
                sa_text("""
                    SELECT id, period, provider, budget_usd, alert_pct, updated_at
                    FROM ai_call_budgets
                    ORDER BY period, provider NULLS FIRST
                """),
            ).fetchall()
        else:
            budgets = []

        spent_rows = session.execute(
            sa_text("""
                SELECT provider, COALESCE(SUM(cost_usd), 0) AS spent
                FROM ai_calls
                WHERE called_at >= :ms
                GROUP BY provider
            """),
            {'ms': month_start},
        ).fetchall()
        spent_map = {r[0]: float(r[1] or 0) for r in spent_rows}

        # throttle 狀態
        throttle_state = {}
        try:
            from services.cost_throttle_service import get_throttle_state
            throttle_state = get_throttle_state()
        except Exception:
            pass

        rows = []
        for b in budgets:
            provider = b[2]  # 可能 None（全供應商總額）
            spent = spent_map.get(provider, 0.0) if provider else sum(spent_map.values())
            budget_usd = float(b[3] or 0)
            ratio = (spent / budget_usd) if budget_usd > 0 else 0
            rows.append({
                'id': b[0], 'period': b[1], 'provider': provider or '(all)',
                'budget_usd': budget_usd, 'alert_pct': int(b[4] or 80),
                'spent': spent, 'ratio': ratio,
                'throttled': throttle_state.get(provider, {}).get('throttled', False) if provider else False,
                'updated_at': b[5].strftime('%Y-%m-%d %H:%M') if b[5] else '-',
            })

        # Phase 47 K-3: 30d daily cost trend by provider
        cost_30d = session.execute(
            sa_text("""
                SELECT date_trunc('day', called_at)::date AS d,
                       provider, COALESCE(SUM(cost_usd), 0) AS cost
                FROM ai_calls
                WHERE called_at >= NOW() - INTERVAL '30 days'
                GROUP BY d, provider
                ORDER BY d DESC, cost DESC
            """),
        ).fetchall()
        cost_trend_30d = []
        for r in cost_30d:
            cost_trend_30d.append({
                'date': r[0].strftime('%m-%d') if r[0] else '',
                'provider': r[1],
                'cost': float(r[2] or 0),
            })

        # Phase 55 S-3: 當月各 provider cost 分布（給圓餅圖用）
        provider_cost_month = session.execute(
            sa_text("""
                SELECT provider, COALESCE(SUM(cost_usd), 0) AS cost
                FROM ai_calls
                WHERE called_at >= :ms AND cost_usd > 0
                GROUP BY provider ORDER BY cost DESC
            """),
            {'ms': month_start},
        ).fetchall()

        # Phase 47 K-3: top 5 cost-burning caller (當月)
        top_cost_callers = session.execute(
            sa_text("""
                SELECT caller, COUNT(*) AS calls,
                       COALESCE(SUM(cost_usd), 0) AS cost,
                       COALESCE(SUM(input_tokens + output_tokens), 0) AS tokens
                FROM ai_calls
                WHERE called_at >= :ms
                  AND cost_usd > 0
                GROUP BY caller
                ORDER BY cost DESC LIMIT 5
            """),
            {'ms': month_start},
        ).fetchall()

        # Phase 47 K-3: ai_price_recommendations 7d 統計
        price_rec_7d = session.execute(
            sa_text("""
                SELECT strategy, COUNT(*) AS cnt,
                       COALESCE(AVG(confidence), 0) AS avg_conf
                FROM ai_price_recommendations
                WHERE created_at >= NOW() - INTERVAL '7 days'
                GROUP BY strategy ORDER BY cnt DESC
            """),
        ).fetchall()

        # Phase 39 D-4: RAG 自動建議策略（針對超 80% 的 row）
        budget_strategies = []
        over_threshold_rows = [r for r in rows if r.get('ratio', 0) >= 0.8]
        if over_threshold_rows:
            try:
                from services.rag_service import rag_service
                top_breach = max(over_threshold_rows, key=lambda r: r.get('ratio', 0))
                query_text = (
                    f"預算超出 alert_pct provider={top_breach['provider']} "
                    f"ratio={int(top_breach['ratio']*100)}% 應採取什麼節流策略"
                )
                rag_result = rag_service.query(
                    text=query_text,
                    caller='admin_budget_dashboard',
                    top_k=3,
                    threshold=0.65,
                )
                budget_strategies = [
                    {
                        'id': h.get('id'),
                        'insight_type': h.get('insight_type'),
                        'content': (h.get('content') or '')[:240],
                        'similarity': round(float(h.get('similarity', 0)), 3),
                    }
                    for h in rag_result.hits[:3]
                ]
            except Exception:
                pass

        return render_template(
            'admin/budget.html',
            active_page='obs_budget',
            rows=rows,
            budget_strategies=budget_strategies,
            cost_trend_30d=cost_trend_30d,
            top_cost_callers=[
                {
                    'caller': r[0], 'calls': int(r[1] or 0),
                    'cost': float(r[2] or 0), 'tokens': int(r[3] or 0),
                }
                for r in top_cost_callers
            ],
            provider_cost_month=[
                {'provider': r[0], 'cost': float(r[1] or 0)}
                for r in provider_cost_month
            ],
            price_rec_7d=[
                {
                    'strategy': r[0], 'count': int(r[1] or 0),
                    'avg_confidence': round(float(r[2] or 0), 3),
                }
                for r in price_rec_7d
            ],
            error=None,
        )
    except Exception as e:
        return render_template('admin/budget.html', active_page='obs_budget', rows=[],
                                budget_strategies=[], cost_trend_30d=[],
                                top_cost_callers=[], price_rec_7d=[],
                                provider_cost_month=[],
                                error='預算資料暫時不可用，已切換安全空狀態。')
    finally:
        session.close()


@admin_observability_bp.route('/ai_calls/trigger_code_review', methods=['POST'])
@login_required
def ai_calls_trigger_code_review():
    """Phase 40 D-7 (L2 自動化)：對高錯誤率時段觸發 Code Review Pipeline。

    用途：admin 在觀測台看到某 caller 錯誤率飆高時，一鍵觸發 5-step
         pipeline (read→hermes_scan→openclaw_summary→ea_decision→nemoton_act)
         在 daemon thread 自動審查最近 commit 變更檔案，找出可能的 regression。
    """
    try:
        import subprocess
        import threading
        from services.code_review_pipeline_service import CodeReviewPipeline

        # 取最新 commit + 變更檔案
        commit_sha = subprocess.check_output(
            ['git', 'rev-parse', 'HEAD'], stderr=subprocess.DEVNULL,
        ).decode().strip()
        changed = subprocess.check_output(
            ['git', 'diff-tree', '--no-commit-id', '--name-only', '-r', commit_sha],
            stderr=subprocess.DEVNULL,
        ).decode().strip().split('\n')
        changed = [f for f in changed if f]

        if not changed:
            return jsonify({'ok': False, 'error': '最新 commit 無變更檔案'}), 400

        pipeline = CodeReviewPipeline(
            commit_sha=commit_sha,
            changed_files=changed,
            branch='main',
            deploy_type='manual_observability',
        )
        threading.Thread(target=pipeline.run, daemon=True).start()
        return jsonify({
            'ok': True,
            'pipeline_id': pipeline.pipeline_id,
            'commit_sha': commit_sha[:8],
            'changed_files_count': len(changed),
            'message': f'已觸發 Code Review (pipeline_id={pipeline.pipeline_id}) 在背景執行，'
                       f'5 step 完成後會推 Telegram 通知。',
        })
    except Exception as e:
        return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500


@admin_observability_bp.route('/ppt_audit/trigger_aider_heal', methods=['POST'])
@login_required
def ppt_audit_trigger_aider_heal():
    """Phase 40 D-8 (L2 自動化)：對失敗 PPT audit 觸發 AiderHeal 修 generator。

    用途：admin 在觀測台看到 PPT vision audit 連續失敗時，一鍵觸發 AiderHeal
         自動修 services/ppt_generator.py（或對應 template generator），
         結果會 git push 到 main 觸發 CD 自動部署。
    """
    try:
        from services.aider_heal_executor import execute_code_fix
        data = request.json or {}
        error_msg = (data.get('error_msg') or '').strip()
        pptx_filename = (data.get('pptx_filename') or '').strip()
        if not error_msg:
            return jsonify({'ok': False, 'error': '需提供 error_msg'}), 400

        # 構造 context 給 AiderHeal
        context = {
            'error_type': 'ppt_vision_audit_failure',
            'error_message': error_msg[:500],
            'target_file': 'services/ppt_generator.py',
            'pptx_filename': pptx_filename,
            'triggered_by': 'admin_observability',
        }
        result = execute_code_fix(context)
        return jsonify({
            'ok': bool(getattr(result, 'success', False)),
            'action': getattr(result, 'action', None),
            'message': getattr(result, 'message', '') or '已派出 AiderHeal',
        })
    except Exception as e:
        return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500


@admin_observability_bp.route('/api/health_indicator')
@login_required
def health_indicator_api():
    """Phase 52 P-1：給 topbar 觀測台 indicator 用的輕量 JSON API。

    回傳當前是否有「需要關注」的事件：
    - 三主機掛掉
    - 待審 episode > 0
    - 過去 1h 錯誤率 ≥ 30%
    - 預算 ≥ 90%
    """
    try:
        session = get_session()
        try:
            # 三主機最新狀態
            host_unhealthy = 0
            try:
                rows = session.execute(
                    sa_text("""
                        WITH latest AS (
                            SELECT host_label,
                                   FIRST_VALUE(healthy) OVER (
                                       PARTITION BY host_label ORDER BY probed_at DESC
                                   ) AS healthy
                            FROM host_health_probes
                            WHERE probed_at >= NOW() - INTERVAL '1 hour'
                        )
                        SELECT host_label, BOOL_AND(NOT healthy) AS down
                        FROM latest
                        GROUP BY host_label
                    """),
                ).fetchall()
                host_unhealthy = sum(1 for r in rows if r[1])
            except Exception:
                pass

            # 待審 episode
            ep_pending = 0
            try:
                ep_pending = int(session.execute(
                    sa_text("SELECT COUNT(*) FROM learning_episodes WHERE promotion_status = 'awaiting_review' AND reviewed_at IS NULL"),
                ).fetchone()[0] or 0)
            except Exception:
                pass

            # 1h 錯誤率
            error_rate = 0
            try:
                row = session.execute(
                    sa_text("""
                        SELECT COUNT(*),
                               COUNT(*) FILTER (WHERE status NOT IN ('ok','cache_only'))
                        FROM ai_calls WHERE called_at >= NOW() - INTERVAL '1 hour'
                    """),
                ).fetchone()
                total = int(row[0] or 0)
                errs = int(row[1] or 0)
                error_rate = (errs / total * 100) if total > 20 else 0
            except Exception:
                pass

            # 預算告警（任一 ≥ 90%）
            budget_alert = False
            try:
                from datetime import datetime as _dt
                today = _dt.now()
                ms = _dt(today.year, today.month, 1)
                bgs = session.execute(
                    sa_text("""
                        SELECT b.budget_usd,
                               COALESCE((SELECT SUM(cost_usd) FROM ai_calls
                                         WHERE called_at >= :ms
                                           AND (b.provider IS NULL OR provider = b.provider)), 0) AS spent
                        FROM ai_call_budgets b
                    """),
                    {'ms': ms},
                ).fetchall()
                for budget, spent in bgs:
                    if budget and float(budget) > 0 and float(spent) / float(budget) >= 0.9:
                        budget_alert = True
                        break
            except Exception:
                pass

            alert_count = (
                host_unhealthy
                + (1 if ep_pending > 0 else 0)
                + (1 if error_rate >= 30 else 0)
                + (1 if budget_alert else 0)
            )
            return jsonify({
                'ok': True,
                'alert_count': alert_count,
                'host_unhealthy': host_unhealthy,
                'ep_pending': ep_pending,
                'error_rate_high': error_rate >= 30,
                'budget_alert': budget_alert,
                'tooltip': _build_indicator_tooltip(host_unhealthy, ep_pending, error_rate, budget_alert),
            })
        finally:
            session.close()
    except Exception as e:
        return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500


def _build_indicator_tooltip(host_unhealthy, ep_pending, error_rate, budget_alert) -> str:
    parts = []
    if host_unhealthy:
        parts.append(f"{host_unhealthy} 主機異常")
    if ep_pending > 0:
        parts.append(f"{ep_pending} 待審")
    if error_rate >= 30:
        parts.append(f"錯誤率 {error_rate:.0f}%")
    if budget_alert:
        parts.append("預算 ≥ 90%")
    if not parts:
        return "AI 觀測台（一切正常）"
    return "AI 觀測台 — " + " / ".join(parts)


def _latest_host_probe_unhealthy(host_label: str, window_minutes: int = 30) -> bool:
    """查 DB 最新 host_health_probe，作為 AutoHeal 按鈕的真實狀態來源。

    `_is_unhealthy()` 只代表 Ollama client 在 30 秒 TTL 內的記憶體標記；
    scheduler / 頁面 probe 寫入的是 `host_health_probes`。L2 AutoHeal 入口
    必須接受 DB 最新探針異常，避免 Telegram 或 Web 顯示主機已掛、按鈕卻拒絕執行。
    """
    if not host_label:
        return False
    session = get_session()
    try:
        row = session.execute(
            sa_text("""
                SELECT healthy
                FROM host_health_probes
                WHERE host_label = :label
                  AND probed_at >= NOW() - (:minutes || ' minutes')::interval
                ORDER BY probed_at DESC
                LIMIT 1
            """),
            {'label': host_label, 'minutes': int(window_minutes)},
        ).fetchone()
        return bool(row is not None and row[0] is False)
    except Exception:
        return False
    finally:
        session.close()


@admin_observability_bp.route('/playbooks/toggle/<int:playbook_id>', methods=['POST'])
@login_required
def playbook_toggle(playbook_id: int):
    """Phase 50 N-3：一鍵啟用/停用 playbook（is_active 翻轉）。

    用途：在 host_health 觀測台直接管理 AutoHeal playbook，
         不需 SSH 188 改 DB。
    """
    try:
        session = get_session()
        try:
            row = session.execute(
                sa_text("SELECT id, name, is_active FROM playbooks WHERE id = :id"),
                {'id': playbook_id},
            ).fetchone()
            if not row:
                return jsonify({'ok': False, 'error': f'playbook #{playbook_id} 不存在'}), 404
            new_active = not bool(row[2])
            session.execute(
                sa_text("UPDATE playbooks SET is_active = :a, updated_at = NOW() WHERE id = :id"),
                {'a': new_active, 'id': playbook_id},
            )
            session.commit()
            return jsonify({
                'ok': True,
                'playbook_id': playbook_id,
                'name': row[1],
                'is_active': new_active,
                'message': f'Playbook 「{row[1]}」已{"啟用" if new_active else "停用"}',
            })
        finally:
            session.close()
    except Exception as e:
        return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500


@admin_observability_bp.route('/host_health/trigger_autoheal', methods=['POST'])
@login_required
def host_health_trigger_autoheal():
    """Phase 40 D-9 (L2 自動化)：對掛掉的主機觸發 AutoHeal playbook。

    用途：admin 看到某台 Ollama 主機標記 unhealthy 時一鍵觸發 AutoHeal
         (ADR-013) 跑對應 playbook（DOCKER_RESTART / SSH_CMD / ALERT_ONLY）。

    安全：只能對已標記 unhealthy 的 host 觸發；不接受任意 host URL（防 SSRF）。
    """
    try:
        data = request.json or {}
        host_label = (data.get('host_label') or '').strip()
        from services.auto_heal_service import auto_heal_service
        from services.ollama_service import _is_unhealthy, OLLAMA_HOST_PRIMARY, OLLAMA_HOST_SECONDARY, OLLAMA_HOST_FALLBACK

        # 白名單對應
        host_map = {
            'Primary (GCP)': OLLAMA_HOST_PRIMARY,
            'Secondary (GCP)': OLLAMA_HOST_SECONDARY,
            'Fallback (111)': OLLAMA_HOST_FALLBACK,
        }
        host_url = host_map.get(host_label)
        if not host_url:
            return jsonify({'ok': False, 'error': f'未知 host_label: {host_label}'}), 400

        if not (_is_unhealthy(host_url) or _latest_host_probe_unhealthy(host_label)):
            return jsonify({
                'ok': False,
                'error': f'{host_label} 目前未標記異常，無需 AutoHeal',
            }), 400

        result = auto_heal_service.handle_exception(
            error_type='ollama_unhealthy',
            context={
                'host_label': host_label,
                'host_url': host_url,
                'error_message': f'Ollama host {host_label} ({host_url}) marked unhealthy',
                'triggered_by': 'admin_observability',
            },
        )
        return jsonify({
            'ok': bool(getattr(result, 'success', False)),
            'action': getattr(result, 'action', None),
            'message': getattr(result, 'message', '') or 'AutoHeal 已派遣',
        })
    except Exception as e:
        return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500


@admin_observability_bp.route('/budget/force_throttle', methods=['POST'])
@login_required
def budget_force_throttle():
    """Phase 39 D-4 (L2 自動化)：立即強制執行 cost_throttle evaluate（不等 hourly cron）。

    用途：admin 在觀測台看到 ratio 飆超 110% 時不需等下次 cron，
         直接點按鈕強制 re-evaluate 三主機 throttle 狀態（claude→gemini fallback 立即生效）。
    """
    try:
        from services.cost_throttle_service import (
            evaluate_throttle_status, is_cost_throttle_enabled,
        )
        if not is_cost_throttle_enabled():
            return jsonify({
                'ok': False,
                'error': 'COST_THROTTLE_ENABLED=false（先設環境變數）',
            }), 400
        new_state = evaluate_throttle_status()
        throttled = [p for p, s in new_state.items() if s.get('throttled')]
        return jsonify({
            'ok': True,
            'throttled_providers': throttled,
            'state': new_state,
            'message': f'已立即重算 throttle 狀態，被節流的 provider：{throttled or "（無）"}',
        })
    except Exception as e:
        return jsonify({'ok': False, 'error': f'{type(e).__name__}: {str(e)[:200]}'}), 500


@admin_observability_bp.route('/budget/update/<int:budget_id>', methods=['POST'])
@login_required
def budget_update(budget_id: int):
    """更新 budget_usd / alert_pct"""
    try:
        new_budget = float(request.json.get('budget_usd'))
        new_alert = int(request.json.get('alert_pct', 80))
        if new_budget <= 0 or not (1 <= new_alert <= 100):
            return jsonify({'ok': False, 'error': 'invalid range'}), 400

        session = get_session()
        try:
            session.execute(
                sa_text("""
                    UPDATE ai_call_budgets
                    SET budget_usd = :b, alert_pct = :a, updated_at = NOW()
                    WHERE id = :id
                """),
                {'b': new_budget, 'a': new_alert, 'id': budget_id},
            )
            session.commit()
            return jsonify({'ok': True})
        finally:
            session.close()
    except Exception as e:
        return jsonify({'ok': False, 'error': str(e)[:200]}), 500


# ─────────────────────────────────────────────────────────────────────────────
# /observability/ppt_audit_history — Phase 29 PPT 視覺審核歷史
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/ppt_audit_history')
@login_required
def ppt_audit_history():
    """掃 reports/ 目錄列近 7 日 .pptx 檔 + 從 ppt_audit_results 表讀 audit 歷史（Phase 38）"""
    import os
    import time
    reports_dir = 'reports'
    files = []
    audit_records = []
    error = None

    try:
        if not os.path.isdir(reports_dir):
            error = f'{reports_dir} 目錄不存在'
        else:
            cutoff = time.time() - 7 * 86400
            for f in os.listdir(reports_dir):
                if not f.lower().endswith('.pptx'):
                    continue
                full = os.path.join(reports_dir, f)
                # symlink 防護：reports/ 內不接受 symlink，避免目錄逃逸（Critic MEDIUM #2）
                if os.path.islink(full):
                    continue
                try:
                    mtime = os.path.getmtime(full)
                    if mtime >= cutoff:
                        files.append({
                            'name': f,
                            'size_kb': round(os.path.getsize(full) / 1024, 1),
                            'mtime': datetime.fromtimestamp(mtime).strftime('%Y-%m-%d %H:%M'),
                            'mtime_ts': mtime,
                        })
                except OSError:
                    continue
            files.sort(key=lambda x: x['mtime_ts'], reverse=True)
    except Exception as e:
        error = f'{type(e).__name__}: {str(e)[:200]}'

    # Phase 38：讀過去 7 日 audit 歷史
    try:
        session = get_session()
        try:
            audit_rows = session.execute(
                sa_text("""
                    SELECT audited_at, pptx_filename, audit_status,
                           issues_count, confidence, duration_ms, error_msg
                    FROM ppt_audit_results
                    WHERE audited_at >= NOW() - INTERVAL '7 days'
                    ORDER BY audited_at DESC
                    LIMIT 100
                """),
            ).fetchall()
            audit_records = [
                {
                    'audited_at': r[0].strftime('%Y-%m-%d %H:%M'),
                    'pptx_filename': r[1],
                    'audit_status': r[2],
                    'issues_count': int(r[3] or 0),
                    'confidence': float(r[4] or 0),
                    'duration_ms': int(r[5] or 0),
                    'error_msg': r[6],
                }
                for r in audit_rows
            ]
        finally:
            session.close()
    except Exception:
        logger.debug("PPT audit history table unavailable; rendering empty audit history", exc_info=True)

    # PPT vision 啟用狀態
    try:
        from services.ppt_vision_service import is_ppt_vision_enabled
        vision_enabled = is_ppt_vision_enabled()
    except Exception:
        vision_enabled = False

    # Phase 47 K-6: 30d 統計 + top failure files
    audit_30d_stats = {}
    top_failure_files = []
    try:
        s_ppt = get_session()
        try:
            stat_row = s_ppt.execute(
                sa_text("""
                    SELECT COUNT(*),
                           COUNT(*) FILTER (WHERE audit_status = 'passed'),
                           COUNT(*) FILTER (WHERE audit_status = 'failed'),
                           COUNT(*) FILTER (WHERE audit_status = 'skipped'),
                           COUNT(*) FILTER (WHERE audit_status = 'error'),
                           COALESCE(AVG(confidence) FILTER (WHERE audit_status = 'passed'), 0),
                           COALESCE(SUM(issues_count), 0)
                    FROM ppt_audit_results
                    WHERE audited_at >= NOW() - INTERVAL '30 days'
                """),
            ).fetchone()
            total_30d = int(stat_row[0] or 0)
            audit_30d_stats = {
                'total': total_30d,
                'passed': int(stat_row[1] or 0),
                'failed': int(stat_row[2] or 0),
                'skipped': int(stat_row[3] or 0),
                'error': int(stat_row[4] or 0),
                'avg_confidence': round(float(stat_row[5] or 0), 3),
                'total_issues': int(stat_row[6] or 0),
                'pass_rate': (float(stat_row[1] or 0) / total_30d * 100) if total_30d else 0,
            }

            top_fail_rows = s_ppt.execute(
                sa_text("""
                    SELECT pptx_filename, COUNT(*) AS attempts,
                           SUM(issues_count) AS total_issues,
                           MAX(audited_at) AS last_audit
                    FROM ppt_audit_results
                    WHERE audit_status IN ('failed', 'error')
                      AND audited_at >= NOW() - INTERVAL '30 days'
                    GROUP BY pptx_filename
                    ORDER BY attempts DESC, total_issues DESC LIMIT 10
                """),
            ).fetchall()
            top_failure_files = [
                {
                    'filename': r[0], 'attempts': int(r[1] or 0),
                    'total_issues': int(r[2] or 0),
                    'last_audit': r[3].strftime('%Y-%m-%d %H:%M') if r[3] else '',
                }
                for r in top_fail_rows
            ]
        finally:
            s_ppt.close()
    except Exception:
        pass

    # Phase 41 E-2: 對最近 3 筆 failed audit 跑 RAG 找相似修法
    rag_fixes = []
    failed_records = [r for r in audit_records if r.get('audit_status') in ('failed', 'error')][:3]
    if failed_records:
        try:
            from services.rag_service import rag_service
            for fr in failed_records:
                try:
                    err_text = fr.get('error_msg') or 'PPT vision audit failed'
                    rag_result = rag_service.query(
                        text=f"PPT 視覺審核失敗 {err_text[:200]} 怎麼修",
                        caller='admin_ppt_audit',
                        top_k=2,
                        threshold=0.6,
                    )
                    if rag_result.hits:
                        rag_fixes.append({
                            'pptx_filename': fr.get('pptx_filename'),
                            'audited_at': fr.get('audited_at'),
                            'error_msg': (err_text or '')[:160],
                            'hits': [
                                {
                                    'id': h.get('id'),
                                    'insight_type': h.get('insight_type'),
                                    'content': (h.get('content') or '')[:200],
                                    'similarity': round(float(h.get('similarity', 0)), 3),
                                }
                                for h in rag_result.hits[:2]
                            ],
                        })
                except Exception:
                    pass
        except Exception:
            pass

    return render_template(
        'admin/ppt_audit_history.html',
        active_page='obs_ppt_audit',
        files=files,
        audit_records=audit_records,
        rag_fixes=rag_fixes,
        audit_30d_stats=audit_30d_stats,
        top_failure_files=top_failure_files,
        vision_enabled=vision_enabled,
        error=error,
    )


# ─────────────────────────────────────────────────────────────────────────────
# /observability/host_health — 三主機 + MCP 健康度
# ─────────────────────────────────────────────────────────────────────────────

@admin_observability_bp.route('/host_health')
@login_required
def host_health_dashboard():
    """三主機 Ollama + 4 個 MCP server 即時健康（同時寫入 host_health_probes 留歷史）"""
    import time as _time
    ollama_hosts = []
    probe_records = []  # 收集本次 probe 結果以批次寫 DB
    try:
        from services.ollama_service import (
            OLLAMA_HOST_PRIMARY, OLLAMA_HOST_SECONDARY, OLLAMA_HOST_FALLBACK,
            _is_unhealthy, _unhealthy_marks,
        )
        import requests as _r
        for label, host in [
            ('Primary (GCP)', OLLAMA_HOST_PRIMARY),
            ('Secondary (GCP)', OLLAMA_HOST_SECONDARY),
            ('Fallback (111)', OLLAMA_HOST_FALLBACK),
        ]:
            entry = {'label': label, 'host': host, 'healthy': False,
                     'unhealthy_mark': _is_unhealthy(host), 'models': []}
            t0 = _time.monotonic()
            err = None
            try:
                resp = _r.get(f"{host.rstrip('/')}/api/tags", timeout=3)
                if resp.status_code == 200:
                    entry['healthy'] = True
                    entry['models'] = [
                        m.get('name', '') for m in resp.json().get('models', [])
                    ][:15]
                else:
                    err = f"HTTP {resp.status_code}"
            except Exception as e:
                err = f"{type(e).__name__}: {str(e)[:200]}"
            response_ms = int((_time.monotonic() - t0) * 1000)
            probe_records.append({
                'host_label': label, 'host_url': host, 'healthy': entry['healthy'],
                'unhealthy_mark': entry['unhealthy_mark'],
                'models_count': len(entry['models']), 'response_ms': response_ms,
                'error_msg': err,
            })
            ollama_hosts.append(entry)
    except Exception:
        pass

    # Phase 38：寫入 host_health_probes 留歷史（失敗安全，不擋頁面渲染）
    if probe_records:
        try:
            _session = get_session()
            try:
                for rec in probe_records:
                    _session.execute(
                        sa_text("""
                            INSERT INTO host_health_probes
                                (host_label, host_url, healthy, unhealthy_mark,
                                 models_count, response_ms, error_msg)
                            VALUES
                                (:host_label, :host_url, :healthy, :unhealthy_mark,
                                 :models_count, :response_ms, :error_msg)
                        """),
                        rec,
                    )
                _session.commit()
            finally:
                _session.close()
        except Exception:
            logger.warning("Failed to persist host health probe records", exc_info=True)

    # MCP server 健康
    mcp_status = {}
    try:
        from services.mcp_router import mcp_router
        mcp_status = mcp_router.health_check()
    except Exception:
        pass

    # cost throttle 狀態
    throttle_state = {}
    try:
        from services.cost_throttle_service import get_throttle_state
        throttle_state = get_throttle_state()
    except Exception:
        pass

    # Phase 38：讀過去 24h 三主機健康歷史（給趨勢卡片）
    health_history = []
    mcp_24h = []  # Phase 39 D-2: MCP 24h 各 server 工作量
    aiops_summary = {}  # Phase 39 D-5: incidents + heal_logs 7d 統計
    try:
        _session2 = get_session()
        try:
            history_rows = _session2.execute(
                sa_text("""
                    SELECT host_label,
                           COUNT(*) FILTER (WHERE healthy)            AS up_count,
                           COUNT(*) FILTER (WHERE NOT healthy)        AS down_count,
                           COALESCE(AVG(response_ms) FILTER (WHERE healthy), 0) AS avg_ms,
                           COUNT(*) AS total
                    FROM host_health_probes
                    WHERE probed_at >= NOW() - INTERVAL '24 hours'
                    GROUP BY host_label
                    ORDER BY host_label
                """),
            ).fetchall()
            health_history = [
                {
                    'host_label': r[0],
                    'up_count': int(r[1] or 0),
                    'down_count': int(r[2] or 0),
                    'avg_ms': int(r[3] or 0),
                    'total': int(r[4] or 0),
                    'uptime_pct': (float(r[1] or 0) / float(r[4]) * 100) if r[4] else 0,
                }
                for r in history_rows
            ]

            # Phase 39 D-5：incidents + heal_logs 過去 7d 統計
            try:
                inc_rows = _session2.execute(
                    sa_text("""
                        SELECT
                            COUNT(*) AS total_incidents,
                            COUNT(*) FILTER (WHERE status = 'open') AS open_count,
                            COUNT(*) FILTER (WHERE status = 'resolved') AS resolved_count,
                            COUNT(*) FILTER (WHERE severity = 'P0') AS p0_count,
                            COUNT(*) FILTER (WHERE severity = 'P1') AS p1_count
                        FROM incidents
                        WHERE created_at >= NOW() - INTERVAL '7 days'
                    """),
                ).fetchone()
                heal_rows = _session2.execute(
                    sa_text("""
                        SELECT
                            COUNT(*) AS total_heals,
                            COUNT(*) FILTER (WHERE result = 'success') AS heal_success,
                            COUNT(*) FILTER (WHERE result = 'failed') AS heal_failed,
                            COALESCE(AVG(duration_ms) FILTER (WHERE result = 'success'), 0) AS avg_ms
                        FROM heal_logs
                        WHERE created_at >= NOW() - INTERVAL '7 days'
                    """),
                ).fetchone()
                aiops_summary = {
                    'incidents_total': int(inc_rows[0] or 0),
                    'incidents_open': int(inc_rows[1] or 0),
                    'incidents_resolved': int(inc_rows[2] or 0),
                    'incidents_p0': int(inc_rows[3] or 0),
                    'incidents_p1': int(inc_rows[4] or 0),
                    'heals_total': int(heal_rows[0] or 0),
                    'heals_success': int(heal_rows[1] or 0),
                    'heals_failed': int(heal_rows[2] or 0),
                    'heals_avg_ms': int(heal_rows[3] or 0),
                    'heal_success_rate': (
                        float(heal_rows[1] or 0) / float(heal_rows[0]) * 100
                    ) if heal_rows[0] else 0,
                }

                # Phase 54 R-3: heal 7d daily success rate sparkline
                heal_daily = _session2.execute(
                    sa_text("""
                        SELECT date_trunc('day', created_at)::date AS d,
                               COUNT(*) AS total,
                               COUNT(*) FILTER (WHERE result = 'success') AS ok
                        FROM heal_logs
                        WHERE created_at >= NOW() - INTERVAL '7 days'
                        GROUP BY d ORDER BY d ASC
                    """),
                ).fetchall()
                aiops_summary['heal_sparkline'] = [
                    {
                        'date': r[0].strftime('%m-%d') if r[0] else '',
                        'total': int(r[1] or 0),
                        'ok': int(r[2] or 0),
                        'rate': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0,
                    }
                    for r in heal_daily
                ]
            except Exception:
                aiops_summary = {}

            # Phase 39 D-2：MCP 24h 工作量（每個 server）
            mcp_rows = _session2.execute(
                sa_text("""
                    SELECT server,
                           COUNT(*) AS total_calls,
                           COUNT(*) FILTER (WHERE status = 'ok') AS ok_calls,
                           COUNT(*) FILTER (WHERE cache_hit) AS cache_hits,
                           COALESCE(SUM(cost_usd), 0) AS total_cost,
                           COALESCE(AVG(duration_ms), 0) AS avg_ms,
                           COUNT(DISTINCT tool) AS tools_used
                    FROM mcp_calls
                    WHERE called_at >= NOW() - INTERVAL '24 hours'
                    GROUP BY server
                    ORDER BY total_calls DESC
                """),
            ).fetchall()
            mcp_24h = [
                {
                    'server': r[0],
                    'total_calls': int(r[1] or 0),
                    'ok_calls': int(r[2] or 0),
                    'cache_hits': int(r[3] or 0),
                    'total_cost': float(r[4] or 0),
                    'avg_ms': int(r[5] or 0),
                    'tools_used': int(r[6] or 0),
                    'success_rate': (float(r[2] or 0) / float(r[1]) * 100) if r[1] else 0,
                    'cache_rate': (float(r[3] or 0) / float(r[1]) * 100) if r[1] else 0,
                }
                for r in mcp_rows
            ]
        finally:
            _session2.close()
    except Exception:
        logger.debug("MCP calls table unavailable; rendering empty MCP 24h summary", exc_info=True)

    # Phase 47 K-1: incidents + heal_logs 詳細列表 + playbooks 排行 + backup + embed queue
    recent_incidents = []
    recent_heals = []
    playbook_ranking = []
    backup_history = []
    embed_queue_pending = 0
    embed_queue_failed = 0
    try:
        s3 = get_session()
        try:
            inc_rows = s3.execute(
                sa_text("""
                    SELECT id, created_at, task_name, error_type, severity,
                           status, error_message, retry_count, resolved_at
                    FROM incidents
                    ORDER BY created_at DESC LIMIT 10
                """),
            ).fetchall()
            recent_incidents = [
                {
                    'id': r[0], 'created_at': r[1].strftime('%Y-%m-%d %H:%M'),
                    'task_name': r[2], 'error_type': r[3], 'severity': r[4],
                    'status': r[5], 'error_message': (r[6] or '')[:200],
                    'retry_count': int(r[7] or 0),
                    'resolved_at': r[8].strftime('%Y-%m-%d %H:%M') if r[8] else None,
                }
                for r in inc_rows
            ]
            heal_rows = s3.execute(
                sa_text("""
                    SELECT h.id, h.created_at, h.action_type, h.result,
                           h.duration_ms, h.action_detail, h.incident_id,
                           i.error_type
                    FROM heal_logs h
                    LEFT JOIN incidents i ON i.id = h.incident_id
                    ORDER BY h.created_at DESC LIMIT 10
                """),
            ).fetchall()
            recent_heals = [
                {
                    'id': r[0], 'created_at': r[1].strftime('%Y-%m-%d %H:%M'),
                    'action_type': r[2], 'result': r[3],
                    'duration_ms': int(r[4] or 0),
                    'action_detail': (r[5] or '')[:160],
                    'incident_id': r[6], 'error_type': r[7],
                }
                for r in heal_rows
            ]

            # playbooks 庫排行（success_count + fail_count + 是否 active）
            pb_rows = s3.execute(
                sa_text("""
                    SELECT id, name, error_type, action_type, severity_min,
                           success_count, fail_count, is_active, cooldown_min
                    FROM playbooks
                    ORDER BY (success_count + fail_count) DESC, success_count DESC
                    LIMIT 12
                """),
            ).fetchall()
            playbook_ranking = [
                {
                    'id': int(r[0]),
                    'name': r[1], 'error_type': r[2], 'action_type': r[3],
                    'severity': r[4], 'success': int(r[5] or 0),
                    'fail': int(r[6] or 0), 'is_active': bool(r[7]),
                    'cooldown_min': int(r[8] or 0),
                    'success_rate': (
                        float(r[5] or 0) / float((r[5] or 0) + (r[6] or 0)) * 100
                    ) if ((r[5] or 0) + (r[6] or 0)) > 0 else 0,
                }
                for r in pb_rows
            ]

            # backup_log 7d 歷史
            bk_rows = s3.execute(
                sa_text("""
                    SELECT created_at, backup_type, status, file_size_bytes,
                           duration_seconds, error_message
                    FROM backup_log
                    WHERE created_at >= NOW() - INTERVAL '7 days'
                    ORDER BY created_at DESC LIMIT 10
                """),
            ).fetchall()
            backup_history = [
                {
                    'created_at': r[0].strftime('%Y-%m-%d %H:%M'),
                    'backup_type': r[1], 'status': r[2],
                    'size_mb': round(float(r[3] or 0) / (1024 * 1024), 1),
                    'duration_s': round(float(r[4] or 0), 1),
                    'error': (r[5] or '')[:120],
                }
                for r in bk_rows
            ]

            # embedding_retry_queue pending / failed
            embed_q = s3.execute(
                sa_text("""
                    SELECT
                        COUNT(*) FILTER (WHERE status = 'pending'),
                        COUNT(*) FILTER (WHERE status = 'failed')
                    FROM embedding_retry_queue
                """),
            ).fetchone()
            embed_queue_pending = int(embed_q[0] or 0)
            embed_queue_failed = int(embed_q[1] or 0)
        finally:
            s3.close()
    except Exception:
        pass

    return render_template(
        'admin/host_health.html',
        active_page='obs_host_health',
        ollama_hosts=ollama_hosts,
        mcp_status=mcp_status,
        throttle_state=throttle_state,
        health_history=health_history,
        mcp_24h=mcp_24h,
        aiops_summary=aiops_summary,
        recent_incidents=recent_incidents,
        recent_heals=recent_heals,
        playbook_ranking=playbook_ranking,
        backup_history=backup_history,
        embed_queue_pending=embed_queue_pending,
        embed_queue_failed=embed_queue_failed,
    )