From 8bc086af580497345d5c95b7f9e5a51adc58b001 Mon Sep 17 00:00:00 2001 From: OG T Date: Fri, 3 Apr 2026 00:36:59 +0800 Subject: [PATCH] =?UTF-8?q?feat(infra):=20=E5=AE=8C=E6=95=B4=E7=9B=A3?= =?UTF-8?q?=E6=8E=A7=E5=B7=A5=E5=85=B7=20+=20=E4=B8=BB=E6=A9=9F=E6=9C=8D?= =?UTF-8?q?=E5=8B=99=E6=B8=85=E5=96=AE=20+=20K3s=20Cluster=20=E7=AA=81?= =?UTF-8?q?=E9=A1=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 監控工具 (6個): - 加入 Grafana (110:3002), Sentry (110:9000), Langfuse (110:3100) - 保留 Prometheus, SigNoz, Gitea 基礎架構: - 靜態服務目錄 HOST_CATALOG：每台主機完整服務+Port+說明 - K3s Server #2 (121) 補靜態卡 (API 未回傳) - K3s Cluster HA 獨立藍色區塊，☸ 標題 + VIP 資訊 - 所有服務含 Port 號與功能描述 Co-Authored-By: Claude Sonnet 4.6 --- apps/api/src/api/v1/monitoring.py | 178 +++++++++++------ apps/web/src/app/[locale]/page.tsx | 108 ++++++++-- apps/web/src/components/infra/host-grid.tsx | 206 +++++++++++++------- 3 files changed, 353 insertions(+), 139 deletions(-) diff --git a/apps/api/src/api/v1/monitoring.py b/apps/api/src/api/v1/monitoring.py index 3ccfb75bd..a374f12b2 100644 --- a/apps/api/src/api/v1/monitoring.py +++ b/apps/api/src/api/v1/monitoring.py @@ -1,12 +1,15 @@ """ Monitoring Status API ===================== -探測各監控工具狀態：Grafana / Prometheus / SigNoz / Gitea +探測所有可觀測性工具狀態： + Grafana / Prometheus / Sentry / Langfuse / SigNoz / Gitea 所有探測從後端發出，不暴露內網 IP 給前端。 +Grafana: 110:3002 (Docker 3002→3000) 建立時間: 2026-04-03 (台北時區) 建立者: Claude Code +更新時間: 2026-04-03 加入 Grafana(3002) / Sentry / Langfuse """ import asyncio @@ -21,70 +24,52 @@ logger = get_logger(__name__) router = APIRouter(prefix="/monitoring", tags=["Monitoring"]) -# ============================================================================= -# Internal service endpoints (backend-only) -# ============================================================================= - -SERVICES = { - "grafana": { - "base": "http://192.168.0.188:3000", - "health": "/api/health", - "build": "/api/frontend/settings", - }, - "prometheus": { - "base": "http://192.168.0.110:9090", - "health": "/-/healthy", - "build": "/api/v1/status/buildinfo", - "rules": "/api/v1/rules", - }, - "signoz": { - "base": "http://192.168.0.188:3301", - "health": "/api/v1/health", - }, - "gitea": { - "base": "http://192.168.0.110:3001", - "health": "/-/readiness", - }, -} - TIMEOUT = 3.0 +# ============================================================================= +# Probes +# ============================================================================= + async def _probe_grafana(client: httpx.AsyncClient) -> dict: - base = SERVICES["grafana"]["base"] + base = "http://192.168.0.110:3002" try: r = await client.get(f"{base}/api/health", timeout=TIMEOUT) if r.status_code == 200: data = r.json() - version = data.get("version", "—") - # Try to get dashboard count - dash_r = await client.get(f"{base}/api/search?type=dash-db", timeout=TIMEOUT) - dash_count = len(dash_r.json()) if dash_r.status_code == 200 else None + version = data.get("version") + dash_r = await client.get( + f"{base}/api/search?type=dash-db", + headers={"X-Grafana-Org-Id": "1"}, + timeout=TIMEOUT, + ) + dash_count = len(dash_r.json()) if dash_r.status_code == 200 and isinstance(dash_r.json(), list) else None return { "name": "Grafana", "status": "up", "version": version, - "stats": f"面板 {dash_count} 個" if dash_count is not None else None, - "description": "監控面板 · 指標視覺化", + "stats": f"面板 {dash_count} 個" if dash_count is not None else "監控面板", + "description": "指標視覺化 · Dashboard", } except Exception as e: logger.warning("grafana_probe_failed", error=str(e)) - return {"name": "Grafana", "status": "down", "version": None, "stats": None, "description": "監控面板 · 指標視覺化"} + return { + "name": "Grafana", "status": "down", "version": None, + "stats": None, "description": "指標視覺化 · Dashboard", + } async def _probe_prometheus(client: httpx.AsyncClient) -> dict: - base = SERVICES["prometheus"]["base"] + base = "http://192.168.0.110:9090" try: health_r = await client.get(f"{base}/-/healthy", timeout=TIMEOUT) if health_r.status_code == 200: - # Get build info build_r = await client.get(f"{base}/api/v1/status/buildinfo", timeout=TIMEOUT) version = None if build_r.status_code == 200: version = build_r.json().get("data", {}).get("version") - # Get rules count rules_r = await client.get(f"{base}/api/v1/rules", timeout=TIMEOUT) - rules_count = None + rules_count = 0 firing_count = 0 if rules_r.status_code == 200: groups = rules_r.json().get("data", {}).get("groups", []) @@ -93,62 +78,139 @@ async def _probe_prometheus(client: httpx.AsyncClient) -> dict: 1 for g in groups for r in g.get("rules", []) if r.get("state") == "firing" ) - stats_parts = [] - if rules_count is not None: - stats_parts.append(f"規則 {rules_count} 條") + stats_parts = [f"規則 {rules_count} 條"] if firing_count > 0: stats_parts.append(f"{firing_count} 觸發") return { "name": "Prometheus", "status": "up", "version": version, - "stats": " · ".join(stats_parts) if stats_parts else None, + "stats": " · ".join(stats_parts), "description": "時序資料庫 · 告警規則", "firing_count": firing_count, } except Exception as e: logger.warning("prometheus_probe_failed", error=str(e)) - return {"name": "Prometheus", "status": "down", "version": None, "stats": None, "description": "時序資料庫 · 告警規則", "firing_count": 0} + return { + "name": "Prometheus", "status": "down", "version": None, + "stats": None, "description": "時序資料庫 · 告警規則", "firing_count": 0, + } + + +async def _probe_sentry(client: httpx.AsyncClient) -> dict: + base = "http://192.168.0.110:9000" + try: + r = await client.get(f"{base}/_health/", timeout=TIMEOUT) + if r.status_code == 200 and r.text.strip() == "ok": + ver_r = await client.get(f"{base}/api/0/", timeout=TIMEOUT) + version = None + if ver_r.status_code == 200: + raw = ver_r.json().get("version") + if isinstance(raw, dict): + version = raw.get("version") + elif raw: + version = str(raw) + return { + "name": "Sentry", + "status": "up", + "version": version, + "stats": "Error Tracking · Issue", + "description": "錯誤追蹤 · Issue 管理", + } + except Exception as e: + logger.warning("sentry_probe_failed", error=str(e)) + return { + "name": "Sentry", "status": "down", "version": None, + "stats": None, "description": "錯誤追蹤 · Issue 管理", + } + + +async def _probe_langfuse(client: httpx.AsyncClient) -> dict: + base = "http://192.168.0.110:3100" + try: + r = await client.get(f"{base}/api/public/health", timeout=TIMEOUT) + if r.status_code == 200: + data = r.json() + version = data.get("version") + return { + "name": "Langfuse", + "status": "up", + "version": version, + "stats": "LLM Tracing · AI 觀測", + "description": "LLM 追蹤 · AI 成本監控", + } + except Exception as e: + logger.warning("langfuse_probe_failed", error=str(e)) + return { + "name": "Langfuse", "status": "down", "version": None, + "stats": None, "description": "LLM 追蹤 · AI 成本監控", + } async def _probe_signoz(client: httpx.AsyncClient) -> dict: - base = SERVICES["signoz"]["base"] + base = "http://192.168.0.188:3301" try: r = await client.get(f"{base}/api/v1/health", timeout=TIMEOUT) if r.status_code == 200: - return {"name": "SigNoz", "status": "up", "version": None, "stats": "APM · 追蹤 · 日誌", "description": "可觀測性平台"} + return { + "name": "SigNoz", + "status": "up", + "version": None, + "stats": "APM · Trace · Log", + "description": "可觀測性平台 · OTEL", + } except Exception as e: logger.warning("signoz_probe_failed", error=str(e)) - # Fallback: try root try: r2 = await client.get(f"{base}/", timeout=TIMEOUT) if r2.status_code in (200, 301, 302): - return {"name": "SigNoz", "status": "up", "version": None, "stats": "APM · 追蹤 · 日誌", "description": "可觀測性平台"} + return { + "name": "SigNoz", "status": "up", "version": None, + "stats": "APM · Trace · Log", "description": "可觀測性平台 · OTEL", + } except Exception: pass - return {"name": "SigNoz", "status": "down", "version": None, "stats": None, "description": "可觀測性平台"} + return { + "name": "SigNoz", "status": "down", "version": None, + "stats": None, "description": "可觀測性平台 · OTEL", + } async def _probe_gitea(client: httpx.AsyncClient) -> dict: - base = SERVICES["gitea"]["base"] + base = "http://192.168.0.110:3001" try: r = await client.get(f"{base}/-/readiness", timeout=TIMEOUT) if r.status_code == 200: - # Get version from API ver_r = await client.get(f"{base}/api/v1/version", timeout=TIMEOUT) version = None if ver_r.status_code == 200: version = ver_r.json().get("version") - return {"name": "Gitea", "status": "up", "version": version, "stats": "CI/CD · Git 倉庫", "description": "代碼倉庫 · Pipeline"} + return { + "name": "Gitea", + "status": "up", + "version": version, + "stats": "CI/CD · Git · Mirror", + "description": "代碼倉庫 · Pipeline", + } except Exception as e: logger.warning("gitea_probe_failed", error=str(e)) - return {"name": "Gitea", "status": "down", "version": None, "stats": None, "description": "代碼倉庫 · Pipeline"} + return { + "name": "Gitea", "status": "down", "version": None, + "stats": None, "description": "代碼倉庫 · Pipeline", + } +# ============================================================================= +# Router +# ============================================================================= + @router.get("/status") async def get_monitoring_status() -> dict: """ - 並行探測所有監控工具狀態 + 並行探測所有可觀測性工具狀態 + + 工具清單: Grafana(3002) / Prometheus / Sentry / Langfuse / SigNoz / Gitea + 注意: Loki 未安裝 (ADR: SigNoz 統一派) Returns: dict with tools list, each containing name/status/version/stats/description @@ -157,18 +219,22 @@ async def get_monitoring_status() -> dict: results = await asyncio.gather( _probe_grafana(client), _probe_prometheus(client), + _probe_sentry(client), + _probe_langfuse(client), _probe_signoz(client), _probe_gitea(client), return_exceptions=True, ) + now = datetime.now(UTC).isoformat() tools = [] for r in results: if isinstance(r, Exception): + logger.error("monitoring_probe_exception", error=str(r)) continue - tools.append({**r, "checked_at": datetime.now(UTC).isoformat()}) + tools.append({**r, "checked_at": now}) return { "tools": tools, - "checked_at": datetime.now(UTC).isoformat(), + "checked_at": now, } diff --git a/apps/web/src/app/[locale]/page.tsx b/apps/web/src/app/[locale]/page.tsx index c441aad17..d055ff4af 100644 --- a/apps/web/src/app/[locale]/page.tsx +++ b/apps/web/src/app/[locale]/page.tsx @@ -19,7 +19,7 @@ import { useIncidents } from '@/hooks/useIncidents' import { useHosts, useDashboardStore } from '@/stores/dashboard.store' import { IncidentCard } from '@/components/incident' import { OpenClawPanel } from '@/components/ai/openclaw-panel' -import { HostGrid, type HostInfo } from '@/components/infra/host-grid' +import { HostGrid, type HostInfo, type HostService } from '@/components/infra/host-grid' import { AppLayout } from '@/components/layout' const API_BASE = process.env.NEXT_PUBLIC_API_URL ?? '' @@ -153,6 +153,91 @@ function MonitoringTools() { // Main Page // ============================================================================= +// ============================================================================= +// Static Host Service Catalog +// 定義每台主機完整服務清單（API 只回傳部分，此處補全靜態資訊） +// ============================================================================= + +const HOST_CATALOG: Record = { + '192.168.0.110': { + services: [ + { name: 'Harbor', healthy: false, port: 5000, description: 'Container Registry' }, + { name: 'Gitea', healthy: false, port: 3001, description: 'Git · CI/CD' }, + { name: 'Sentry', healthy: false, port: 9000, description: 'Error Tracking' }, + { name: 'Langfuse', healthy: false, port: 3100, description: 'LLM Tracing' }, + { name: 'Grafana', healthy: false, port: 3002, description: '監控面板' }, + { name: 'Prometheus', healthy: false, port: 9090, description: '告警規則' }, + ], + }, + '192.168.0.112': { + services: [ + { name: 'Scanner API', healthy: false, port: 8080, description: '漏洞掃描' }, + ], + }, + '192.168.0.120': { + isK3s: true, + role: 'Control Plane #1', + services: [ + { name: 'K3s API', healthy: false, port: 6443, description: 'kubectl', isK3s: true }, + { name: 'Traefik', healthy: false, description: 'Ingress', isK3s: true }, + { name: 'awoooi-prod', healthy: false, description: 'Namespace', isK3s: true }, + { name: 'keepalived', healthy: false, description: 'VIP MASTER', isK3s: true }, + ], + }, + '192.168.0.121': { + isK3s: true, + role: 'Control Plane #2 (HA)', + services: [ + { name: 'K3s API', healthy: false, port: 6443, description: 'kubectl', isK3s: true }, + { name: 'API', healthy: false, port: 32334, description: 'NodePort', isK3s: true }, + { name: 'Web', healthy: false, port: 32335, description: 'NodePort', isK3s: true }, + { name: 'keepalived', healthy: false, description: 'VIP BACKUP', isK3s: true }, + ], + }, + '192.168.0.188': { + services: [ + { name: 'Nginx', healthy: false, port: 443, description: 'Reverse Proxy' }, + { name: 'PostgreSQL', healthy: false, port: 5432, description: 'K3s Datastore' }, + { name: 'Redis', healthy: false, port: 6380, description: 'Cache' }, + { name: 'Ollama', healthy: false, port: 11434, description: 'LLM' }, + { name: 'OpenClaw', healthy: false, port: 8088, description: 'AI Agent' }, + { name: 'SigNoz', healthy: false, port: 3301, description: 'APM · OTEL' }, + ], + }, +} + +/** 合併 API 動態健康狀態 + 靜態服務清單 */ +function buildHostInfo( + ip: string, + hostname: string, + cpuPct: number | null, + ramPct: number | null, + dynamicServices: { name: string; status: string }[], +): HostInfo { + const catalog = HOST_CATALOG[ip] + const services: HostService[] = catalog + ? catalog.services.map(s => { + const dyn = dynamicServices.find(d => d.name.toLowerCase() === s.name.toLowerCase()) + return { + ...s, + healthy: dyn ? (dyn.status === 'up' || dyn.status === 'healthy') : false, + } + }) + : dynamicServices.map(s => ({ + name: s.name, + healthy: s.status === 'up' || s.status === 'healthy', + })) + return { + hostname, + ip, + cpuPct, + ramPct, + services, + isK3s: catalog?.isK3s, + role: catalog?.role, + } +} + export default function Home({ params }: { params: { locale: string } }) { const tDashboard = useTranslations('dashboard') const tCommon = useTranslations('common') @@ -496,16 +581,17 @@ export default function Home({ params }: { params: { locale: string } }) {
{tDashboard('infrastructure')}
- ({ - hostname: h.name, - ip: h.ip, - cpuPct: h.metrics?.cpu_percent ?? null, - ramPct: h.metrics?.memory_percent ?? null, - services: h.services.map(s => ({ - name: s.name, - healthy: s.status === 'up' || s.status === 'healthy', - })), - }))} /> + { + const apiHosts = hosts.map(h => + buildHostInfo(h.ip, h.name, h.metrics?.cpu_percent ?? null, h.metrics?.memory_percent ?? null, h.services) + ) + // K3s #2 (121) 若 API 未回傳，補靜態卡 + const has121 = apiHosts.some(h => h.ip === '192.168.0.121') + if (!has121) { + apiHosts.push(buildHostInfo('192.168.0.121', 'K3s Server #2', null, null, [])) + } + return apiHosts + })()} /> {/* 監控工具 */} diff --git a/apps/web/src/components/infra/host-grid.tsx b/apps/web/src/components/infra/host-grid.tsx index 90212f3a8..4e8450ce3 100644 --- a/apps/web/src/components/infra/host-grid.tsx +++ b/apps/web/src/components/infra/host-grid.tsx @@ -1,16 +1,21 @@ 'use client' /** - * HostGrid - 2×2 主機緊湊 Grid + * HostGrid - 主機完整服務 Grid * ================================ - * 每格：hostname + IP + CPU/RAM + 3條關鍵服務 mini 列 - * hover 橘紅邊框，異常指標用警示色 + * 每張卡顯示：hostname + IP + CPU/RAM + 完整服務列表 + * K3s 主機: 藍色標記、K3s 徽章、獨立 Cluster 區塊 * 統帥鐵律：無數據顯示 "--"，禁止假數據 + * + * @updated 2026-04-03 Claude Code — 完整服務清單 + K3s Cluster 突顯 */ export interface HostService { name: string healthy: boolean + port?: number + description?: string + isK3s?: boolean } export interface HostInfo { @@ -19,6 +24,8 @@ export interface HostInfo { cpuPct: number | null ramPct: number | null services: HostService[] + isK3s?: boolean + role?: string } export interface HostGridProps { @@ -26,84 +33,139 @@ export interface HostGridProps { } function MetricBar({ value, warn = 80, critical = 90 }: { value: number | null; warn?: number; critical?: number }) { - if (value === null) return -- + if (value === null) return -- const color = value >= critical ? '#cc2200' : value >= warn ? '#F59E0B' : '#22C55E' + return {value}% +} + +function ServiceRow({ svc }: { svc: HostService }) { + const isK3s = svc.isK3s ?? false + const dotColor = isK3s ? '#4f8ef7' : svc.healthy ? '#22C55E' : '#b0ad9f' return ( - {value}% +
+
+ + {svc.name} + + {svc.port && ( + :{svc.port} + )} + {svc.description && ( + + {svc.description} + + )} +
+ ) +} + +function HostCard({ host }: { host: HostInfo }) { + const isK3s = host.isK3s ?? false + const borderColor = isK3s ? '#93c5fd' : '#e0ddd4' + const headerBg = isK3s ? '#eff6ff' : '#f5f4ed' + const accentColor = isK3s ? '#3b7de8' : '#d97757' + + return ( +
{ (e.currentTarget as HTMLDivElement).style.borderColor = accentColor }} + onMouseLeave={e => { (e.currentTarget as HTMLDivElement).style.borderColor = borderColor }} + > + {/* Header */} +
+
+
+
+ + {host.hostname} + + {isK3s && ( + + K3s + + )} +
+ {host.ip} +
+
+ CPU + RAM + {host.role && ( + {host.role} + )} +
+
+ + {/* Services */} +
+ {host.services.length === 0 ? ( + -- + ) : ( +
+ {host.services.map((svc, i) => )} +
+ )} +
+
) } export function HostGrid({ hosts }: HostGridProps) { - // 補足至 4 格（不足時顯示空格） - const slots = [...hosts, ...Array(Math.max(0, 4 - hosts.length)).fill(null)] + if (hosts.length === 0) { + return
--
+ } + + const normalHosts = hosts.filter(h => !h.isK3s) + const k3sHosts = hosts.filter(h => h.isK3s) return ( -
- {slots.map((host: HostInfo | null, idx) => ( -
host && ((e.currentTarget as HTMLDivElement).style.borderColor = '#d97757')} - onMouseLeave={e => host && ((e.currentTarget as HTMLDivElement).style.borderColor = '#e0ddd4')} - > - {host ? ( - <> - {/* Hostname + IP */} -
-
-
- - {host.hostname} - -
- {host.ip} -
- - {/* CPU / RAM */} -
- - CPU - - - RAM - -
- - {/* 3 關鍵服務 mini 列 */} -
- {host.services.slice(0, 3).map((svc, si) => ( -
-
- - {svc.name} - -
- ))} -
- - ) : ( -
- -- -
- )} +
+ {/* Normal hosts — 2-col grid */} + {normalHosts.length > 0 && ( +
+ {normalHosts.map((host, i) => )}
- ))} + )} + + {/* K3s Cluster — full-width highlighted block */} + {k3sHosts.length > 0 && ( +
+ {/* Cluster banner */} +
+ + ☸ K3S CLUSTER (HA) + + + VIP 192.168.0.125 · kubectl :6443 · Web :32335 · API :32334 + +
+
+ {k3sHosts.map((host, i) => )} +
+
+ )}
) }