ewoooc/templates/admin/host_health.html

{% extends "ewoooc_base.html" %}

{% block title %}主機健康監控{% endblock %}

{% block ewooo_content %}
<div class="container-fluid mt-3">
  <h2 class="mb-3"><i class="fas fa-heartbeat me-2"></i>主機健康監控
    <small class="text-muted">三主機 Ollama + MCP + 成本節流即時狀態</small>
  </h2>

  <!-- Ollama 三主機 -->
  <div class="card mb-3">
    <div class="card-header"><strong><i class="fas fa-server me-2"></i>Ollama 三主機（HTTP /api/tags 即時 probe）</strong></div>
    <div class="card-body p-0">
      <table class="table mb-0">
        <thead class="table-light">
          <tr><th>角色</th><th>主機</th><th>HTTP 健康</th><th>異常標記</th><th>已載入模型</th><th>動作</th></tr>
        </thead>
        <tbody>
          {% for h in ollama_hosts %}
            <tr>
              <td><strong>{{ h.label }}</strong></td>
              <td><code>{{ h.host }}</code></td>
              <td>
                {% if h.healthy %}
                  <span class="badge bg-success"><i class="fas fa-check me-1"></i>HTTP 正常</span>
                {% else %}
                  <span class="badge bg-danger"><i class="fas fa-times me-1"></i>離線</span>
                {% endif %}
              </td>
              <td>
                {% if h.unhealthy_mark %}
                  <span class="badge bg-warning"><i class="fas fa-exclamation-triangle me-1"></i>已標記異常（30 秒）</span>
                {% else %}
                  <span class="badge bg-light text-dark">—</span>
                {% endif %}
              </td>
              <td>
                {% for m in h.models %}
                  <span class="badge bg-info text-dark me-1">{{ m }}</span>
                {% endfor %}
                {% if not h.models %}<small class="text-muted">無 / 未連線</small>{% endif %}
              </td>
              <td>
                {% if h.unhealthy_mark or not h.healthy %}
                  <button class="btn btn-sm btn-outline-danger"
                          onclick="triggerAutoHeal({{ h.label|tojson }})">
                    <i class="fas fa-band-aid me-1"></i>AutoHeal
                  </button>
                {% else %}
                  <small class="text-muted">—</small>
                {% endif %}
              </td>
            </tr>
          {% endfor %}
        </tbody>
      </table>
    </div>
  </div>

  <!-- MCP servers -->
  <div class="card mb-3">
    <div class="card-header"><strong><i class="fas fa-plug me-2"></i>MCP 服務（Phase 10/10.5）</strong></div>
    <div class="card-body p-0">
      <table class="table mb-0">
        <thead class="table-light">
          <tr><th>服務名稱</th><th>狀態</th></tr>
        </thead>
        <tbody>
          {% for server, healthy in mcp_status.items() %}
            <tr>
              <td><code>{{ server }}</code></td>
              <td>
                {% if healthy %}
                  <span class="badge bg-success"><i class="fas fa-check me-1"></i>正常</span>
                {% else %}
                  <span class="badge bg-secondary">— 未啟用 / 離線</span>
                {% endif %}
              </td>
            </tr>
          {% else %}
            <tr><td colspan="2" class="text-muted small">MCP_ROUTER_ENABLED=false 或 mcp-stack 未部署</td></tr>
          {% endfor %}
        </tbody>
      </table>
    </div>
  </div>

  <!-- Cost Throttle 狀態（Phase 20） -->
  <div class="card mb-3">
    <div class="card-header"><strong><i class="fas fa-dollar-sign me-2"></i>成本節流狀態（Phase 20）</strong></div>
    <div class="card-body p-0">
      {% if throttle_state %}
        <table class="table mb-0">
          <thead class="table-light">
            <tr><th>供應商</th><th>已花費</th><th>預算</th><th>月底推估</th><th>使用率</th><th>狀態</th></tr>
          </thead>
          <tbody>
            {% for provider, info in throttle_state.items() %}
              <tr {% if info.throttled %}class="table-warning"{% endif %}>
                <td><code>{{ provider }}</code></td>
                <td>${{ "%.2f"|format(info.spent) }}</td>
                <td>${{ "%.2f"|format(info.budget) }}</td>
                <td>${{ "%.2f"|format(info.projected) }}</td>
                <td>{{ "%.0f"|format(info.ratio * 100) }}%</td>
                <td>
                  {% if info.throttled %}
                    <span class="badge bg-danger"><i class="fas fa-exclamation-triangle me-1"></i>已節流</span>
                  {% else %}
                    <span class="badge bg-success"><i class="fas fa-check me-1"></i>正常</span>
                  {% endif %}
                </td>
              </tr>
            {% endfor %}
          </tbody>
        </table>
      {% else %}
        <p class="text-muted m-3 small">
          COST_THROTTLE_ENABLED=false 或尚未首次評估（每小時 cron 執行）
        </p>
      {% endif %}
    </div>
  </div>

  <!-- AIOps 7d 摘要（Phase 39 D-5 新增） -->
  {% if aiops_summary %}
  <div class="card mb-3" style="border-left: 4px solid #0d6efd;">
    <div class="card-header">
      <strong><i class="fas fa-shield-virus me-2"></i>AIOps 自癒系統 7 日摘要</strong>
      <small class="text-muted">資料來源：incidents + heal_logs（ADR-013 AutoHeal 閉環）</small>
    </div>
    <div class="card-body">
      <div class="row g-2">
        <div class="col-md-2 col-sm-4">
          <div class="border rounded p-2 text-center">
            <small class="text-muted d-block">總事件</small>
            <strong style="font-size: 1.4em;">{{ aiops_summary.incidents_total }}</strong>
          </div>
        </div>
        <div class="col-md-2 col-sm-4">
          <div class="border rounded p-2 text-center">
            <small class="text-muted d-block">未解決</small>
            <strong class="{% if aiops_summary.incidents_open > 0 %}text-danger{% endif %}" style="font-size: 1.4em;">
              {{ aiops_summary.incidents_open }}
            </strong>
          </div>
        </div>
        <div class="col-md-2 col-sm-4">
          <div class="border rounded p-2 text-center">
            <small class="text-muted d-block">已解決</small>
            <strong class="text-success" style="font-size: 1.4em;">{{ aiops_summary.incidents_resolved }}</strong>
          </div>
        </div>
        <div class="col-md-2 col-sm-4">
          <div class="border rounded p-2 text-center">
            <small class="text-muted d-block">P0/P1</small>
            <strong class="{% if (aiops_summary.incidents_p0 + aiops_summary.incidents_p1) > 0 %}text-danger{% endif %}" style="font-size: 1.4em;">
              {{ aiops_summary.incidents_p0 + aiops_summary.incidents_p1 }}
            </strong>
          </div>
        </div>
        <div class="col-md-2 col-sm-4">
          <div class="border rounded p-2 text-center">
            <small class="text-muted d-block">自癒成功率</small>
            <strong class="{% if aiops_summary.heal_success_rate >= 80 %}text-success{% elif aiops_summary.heal_success_rate >= 50 %}text-warning{% else %}text-danger{% endif %}" style="font-size: 1.4em;">
              {{ "%.0f"|format(aiops_summary.heal_success_rate) }}%
            </strong>
          </div>
        </div>
        <div class="col-md-2 col-sm-4">
          <div class="border rounded p-2 text-center">
            <small class="text-muted d-block">平均自癒耗時</small>
            <strong style="font-size: 1.4em;">{{ aiops_summary.heals_avg_ms }} ms</strong>
          </div>
        </div>
      </div>
      <div class="mt-2 small text-muted">
        <i class="fas fa-info-circle me-1"></i>
        7d 共 {{ aiops_summary.heals_total }} 次自癒嘗試
        （成功 {{ aiops_summary.heals_success }} · 失敗 {{ aiops_summary.heals_failed }}）
      </div>
    </div>
  </div>
  {% endif %}

  <!-- MCP 24h 工作量（Phase 39 D-2 新增） -->
  {% if mcp_24h %}
  <div class="card mb-3">
    <div class="card-header"><strong><i class="fas fa-bolt me-2"></i>MCP 服務 24h 工作量</strong>
      <small class="text-muted">資料來源：mcp_calls 表 — 展現 AI×MCP 編排規模</small>
    </div>
    <div class="card-body p-0">
      <table class="table mb-0">
        <thead class="table-light">
          <tr>
            <th>服務</th>
            <th class="text-end">呼叫次數</th>
            <th class="text-end">成功率</th>
            <th class="text-end">快取命中率</th>
            <th class="text-end">使用 Tool 數</th>
            <th class="text-end">平均耗時</th>
            <th class="text-end">成本 (USD)</th>
          </tr>
        </thead>
        <tbody>
          {% for s in mcp_24h %}
            <tr>
              <td><code>{{ s.server }}</code></td>
              <td class="text-end">{{ "{:,}".format(s.total_calls) }}</td>
              <td class="text-end">
                <strong class="{% if s.success_rate >= 95 %}text-success{% elif s.success_rate >= 80 %}text-warning{% else %}text-danger{% endif %}">
                  {{ "%.1f"|format(s.success_rate) }}%
                </strong>
              </td>
              <td class="text-end">
                <span class="{% if s.cache_rate >= 30 %}text-success{% endif %}">
                  {{ "%.1f"|format(s.cache_rate) }}%
                </span>
              </td>
              <td class="text-end">{{ s.tools_used }}</td>
              <td class="text-end">{{ s.avg_ms }} ms</td>
              <td class="text-end">${{ "%.4f"|format(s.total_cost) }}</td>
            </tr>
          {% endfor %}
        </tbody>
      </table>
    </div>
  </div>
  {% endif %}

  <!-- 過去 24h 健康趨勢（Phase 38 新增） -->
  {% if health_history %}
  <div class="card mb-3">
    <div class="card-header"><strong><i class="fas fa-chart-line me-2"></i>過去 24 小時健康趨勢</strong>
      <small class="text-muted">資料來源：host_health_probes（每次刷新自動寫入）</small>
    </div>
    <div class="card-body p-0">
      <table class="table mb-0">
        <thead class="table-light">
          <tr>
            <th>角色</th>
            <th class="text-end">總探針次數</th>
            <th class="text-end">正常次數</th>
            <th class="text-end">離線次數</th>
            <th class="text-end">在線率</th>
            <th class="text-end">平均回應 ms</th>
          </tr>
        </thead>
        <tbody>
          {% for h in health_history %}
            <tr>
              <td><strong>{{ h.host_label }}</strong></td>
              <td class="text-end">{{ h.total }}</td>
              <td class="text-end text-success">{{ h.up_count }}</td>
              <td class="text-end text-danger">{{ h.down_count }}</td>
              <td class="text-end">
                <strong class="{% if h.uptime_pct >= 99 %}text-success{% elif h.uptime_pct >= 90 %}text-warning{% else %}text-danger{% endif %}">
                  {{ "%.1f"|format(h.uptime_pct) }}%
                </strong>
              </td>
              <td class="text-end">{{ h.avg_ms }}</td>
            </tr>
          {% endfor %}
        </tbody>
      </table>
    </div>
  </div>
  {% endif %}

  <!-- Phase 47 K-1: 最近 10 筆 Incidents 詳細 -->
  <div class="card mb-3">
    <div class="card-header">
      <strong><i class="fas fa-fire me-2"></i>最近 10 筆 Incidents</strong>
      <small class="text-muted">資料來源：incidents（ADR-013 AutoHeal 觸發紀錄）</small>
    </div>
    <div class="card-body p-0">
      {% if recent_incidents %}
        <table class="table table-sm mb-0" style="font-size: 0.9em;">
          <thead class="table-light">
            <tr>
              <th>建立時間</th><th>任務</th><th>錯誤類型</th>
              <th>嚴重度</th><th>狀態</th><th>重試</th>
              <th>錯誤訊息</th><th>解決時間</th>
            </tr>
          </thead>
          <tbody>
            {% for i in recent_incidents %}
              <tr>
                <td><small>{{ i.created_at }}</small></td>
                <td><code>{{ i.task_name }}</code></td>
                <td><span class="badge bg-secondary">{{ i.error_type }}</span></td>
                <td>
                  {% if i.severity in ('P0', 'P1') %}<span class="badge bg-danger">{{ i.severity }}</span>
                  {% elif i.severity == 'P2' %}<span class="badge bg-warning">{{ i.severity }}</span>
                  {% else %}<span class="badge bg-info text-dark">{{ i.severity }}</span>{% endif %}
                </td>
                <td>
                  {% if i.status == 'open' %}<span class="badge bg-danger">未解決</span>
                  {% elif i.status == 'resolved' %}<span class="badge bg-success">已解決</span>
                  {% else %}<span class="badge bg-light text-dark">{{ i.status }}</span>{% endif %}
                </td>
                <td class="text-end">{{ i.retry_count }}</td>
                <td><small class="text-muted">{{ i.error_message }}</small></td>
                <td><small>{{ i.resolved_at or '—' }}</small></td>
              </tr>
            {% endfor %}
          </tbody>
        </table>
      {% else %}
        <div class="text-muted text-center p-3 small">
          <i class="fas fa-shield-check me-1"></i>尚無 incident 紀錄（即系統尚未觸發過 AutoHeal）
        </div>
      {% endif %}
    </div>
  </div>

  <!-- Phase 47 K-1: 最近 10 筆 Heal Logs 詳細 -->
  <div class="card mb-3">
    <div class="card-header">
      <strong><i class="fas fa-band-aid me-2"></i>最近 10 筆 Heal Logs</strong>
      <small class="text-muted">資料來源：heal_logs（自癒 playbook 執行歷史）</small>
    </div>
    <div class="card-body p-0">
      {% if recent_heals %}
        <table class="table table-sm mb-0" style="font-size: 0.9em;">
          <thead class="table-light">
            <tr>
              <th>時間</th><th>動作</th><th>結果</th>
              <th class="text-end">耗時 ms</th><th>關聯 Incident</th><th>細節</th>
            </tr>
          </thead>
          <tbody>
            {% for h in recent_heals %}
              <tr>
                <td><small>{{ h.created_at }}</small></td>
                <td><span class="badge bg-info text-dark">{{ h.action_type or '—' }}</span></td>
                <td>
                  {% if h.result == 'success' %}<span class="badge bg-success"><i class="fas fa-check me-1"></i>成功</span>
                  {% elif h.result == 'failed' %}<span class="badge bg-danger"><i class="fas fa-times me-1"></i>失敗</span>
                  {% elif h.result == 'skipped' %}<span class="badge bg-secondary">跳過</span>
                  {% else %}<span class="badge bg-light text-dark">{{ h.result }}</span>{% endif %}
                </td>
                <td class="text-end">{{ h.duration_ms }}</td>
                <td><small>#{{ h.incident_id }} · <code>{{ h.error_type or '—' }}</code></small></td>
                <td><small class="text-muted">{{ h.action_detail }}</small></td>
              </tr>
            {% endfor %}
          </tbody>
        </table>
      {% else %}
        <div class="text-muted text-center p-3 small">
          <i class="fas fa-info-circle me-1"></i>尚無 heal log（一鍵 AutoHeal 觸發後將累積）
        </div>
      {% endif %}
    </div>
  </div>

  <!-- Phase 47 K-1: AutoHeal Playbook 庫排行 -->
  <div class="card mb-3">
    <div class="card-header">
      <strong><i class="fas fa-book-medical me-2"></i>AutoHeal Playbook 庫排行</strong>
      <small class="text-muted">資料來源：playbooks（success/fail 累計，按執行次數排序）</small>
    </div>
    <div class="card-body p-0">
      {% if playbook_ranking %}
      <table class="table table-sm mb-0" style="font-size: 0.9em;">
        <thead class="table-light">
          <tr>
            <th>Playbook</th><th>錯誤類型</th><th>動作</th>
            <th>嚴重度</th>
            <th class="text-end">成功</th><th class="text-end">失敗</th>
            <th class="text-end">成功率</th>
            <th>狀態</th><th class="text-end">冷卻 min</th><th>切換</th>
          </tr>
        </thead>
        <tbody>
          {% for p in playbook_ranking %}
          <tr>
            <td><strong>{{ p.name }}</strong></td>
            <td><code>{{ p.error_type }}</code></td>
            <td><span class="badge bg-info text-dark">{{ p.action_type }}</span></td>
            <td>{{ p.severity }}</td>
            <td class="text-end text-success">{{ p.success }}</td>
            <td class="text-end text-danger">{{ p.fail }}</td>
            <td class="text-end">
              {% if (p.success + p.fail) > 0 %}
                <strong class="{% if p.success_rate >= 80 %}text-success{% elif p.success_rate >= 50 %}text-warning{% else %}text-danger{% endif %}">
                  {{ "%.0f"|format(p.success_rate) }}%
                </strong>
              {% else %}<small class="text-muted">—</small>{% endif %}
            </td>
            <td>
              {% if p.is_active %}
                <span class="badge bg-success">啟用</span>
              {% else %}
                <span class="badge bg-secondary">停用</span>
              {% endif %}
            </td>
            <td class="text-end">{{ p.cooldown_min }}</td>
            <td>
              <button class="btn btn-sm {% if p.is_active %}btn-outline-secondary{% else %}btn-outline-success{% endif %}"
                      onclick="togglePlaybook({{ p.id }}, {{ p.name|tojson }})">
                {% if p.is_active %}<i class="fas fa-pause me-1"></i>停用{% else %}<i class="fas fa-play me-1"></i>啟用{% endif %}
              </button>
            </td>
          </tr>
          {% endfor %}
        </tbody>
      </table>
      {% else %}
        <div class="text-muted text-center p-3 small">
          <i class="fas fa-info-circle me-1"></i>尚無 playbook 資料（migration 013 + 020 是否已跑？）
        </div>
      {% endif %}
    </div>
  </div>

  <!-- Phase 47 K-1: 備份歷史 7d -->
  <div class="card mb-3">
    <div class="card-header">
      <strong><i class="fas fa-database me-2"></i>備份歷史（過去 7 日）</strong>
      <small class="text-muted">資料來源：backup_log</small>
    </div>
    <div class="card-body p-0">
      {% if backup_history %}
      <table class="table table-sm mb-0" style="font-size: 0.9em;">
        <thead class="table-light">
          <tr>
            <th>時間</th><th>類型</th><th>狀態</th>
            <th class="text-end">大小 (MB)</th><th class="text-end">耗時 s</th>
            <th>錯誤</th>
          </tr>
        </thead>
        <tbody>
          {% for b in backup_history %}
          <tr>
            <td><small>{{ b.created_at }}</small></td>
            <td><span class="badge bg-secondary">{{ b.backup_type }}</span></td>
            <td>
              {% if b.status == 'success' %}<span class="badge bg-success"><i class="fas fa-check me-1"></i>成功</span>
              {% elif b.status == 'failed' %}<span class="badge bg-danger"><i class="fas fa-times me-1"></i>失敗</span>
              {% else %}<span class="badge bg-warning">{{ b.status }}</span>{% endif %}
            </td>
            <td class="text-end">{{ "{:,}".format(b.size_mb) }}</td>
            <td class="text-end">{{ b.duration_s }}</td>
            <td><small class="text-muted">{{ b.error }}</small></td>
          </tr>
          {% endfor %}
        </tbody>
      </table>
      {% else %}
        <div class="text-muted text-center p-3 small">
          <i class="fas fa-info-circle me-1"></i>過去 7 日無備份紀錄（每日 03:00 cron 執行）
        </div>
      {% endif %}
    </div>
  </div>

  <!-- Phase 47 K-1: Embedding 重試佇列 -->
  {% if embed_queue_pending > 0 or embed_queue_failed > 0 %}
  <div class="alert alert-warning mb-3">
    <strong><i class="fas fa-cog me-1"></i>Embedding 重試佇列：</strong>
    待處理 <strong>{{ embed_queue_pending }}</strong> 筆 ·
    失敗 <strong>{{ embed_queue_failed }}</strong> 筆
    <small class="d-block text-muted mt-1">
      資料來源：embedding_retry_queue — 卡住的 embedding 工作，需檢查 Ollama 健康
    </small>
  </div>
  {% endif %}

  <p class="text-muted mt-3"><small>
    <i class="fas fa-robot me-1"></i>Operation Ollama-First v5.0 / Phase 47 — 主機健康監控
    （8 表深挖：host_health_probes / mcp_calls / incidents / heal_logs / playbooks / backup_log / embedding_retry_queue / ai_call_budgets）
  </small></p>
</div>

<script>
async function togglePlaybook(id, name) {
  if (!confirm(`切換 Playbook 「${name}」狀態？`)) return;
  try {
    const r = await fetch(`/observability/playbooks/toggle/${id}`, {method: 'POST'});
    const d = await r.json();
    if (d.ok) {
      alert(`✅ ${d.message}`);
      window.location.reload();
    } else {
      alert('❌ ' + (d.error || '切換失敗'));
    }
  } catch (e) {
    alert('Error: ' + e);
  }
}

async function triggerAutoHeal(hostLabel) {
  if (!confirm(`觸發 AutoHeal？\n\n主機：${hostLabel}\n\n會跑對應 ADR-013 playbook（DOCKER_RESTART / SSH_CMD / ALERT_ONLY）並寫入 incidents 表。`)) return;
  try {
    const r = await fetch('/observability/host_health/trigger_autoheal', {
      method: 'POST',
      headers: {'Content-Type': 'application/json'},
      body: JSON.stringify({host_label: hostLabel}),
    });
    const d = await r.json();
    if (d.ok) {
      alert(`✅ AutoHeal 已派出\n動作：${d.action || '—'}\n訊息：${d.message || ''}`);
      window.location.reload();
    } else {
      alert('❌ ' + (d.error || d.message || '觸發失敗'));
    }
  } catch (e) {
    alert('Error: ' + e);
  }
}
</script>
{% endblock %}