541a32a9cb
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m38s
CD Pipeline / build-and-deploy (push) Failing after 24m19s
CD Pipeline / post-deploy-checks (push) Has been skipped
305 lines
10 KiB
JSON
305 lines
10 KiB
JSON
{
|
||
"__inputs": [],
|
||
"__requires": [
|
||
{
|
||
"type": "grafana",
|
||
"id": "grafana",
|
||
"name": "Grafana",
|
||
"version": "10.0.0"
|
||
},
|
||
{
|
||
"type": "panel",
|
||
"id": "stat",
|
||
"name": "Stat",
|
||
"version": ""
|
||
},
|
||
{
|
||
"type": "panel",
|
||
"id": "timeseries",
|
||
"name": "Time series",
|
||
"version": ""
|
||
},
|
||
{
|
||
"type": "panel",
|
||
"id": "piechart",
|
||
"name": "Pie chart",
|
||
"version": ""
|
||
},
|
||
{
|
||
"type": "panel",
|
||
"id": "barchart",
|
||
"name": "Bar chart",
|
||
"version": ""
|
||
}
|
||
],
|
||
"annotations": {
|
||
"list": []
|
||
},
|
||
"description": "Ollama 容災監控 — GCP-A/B cloud-edge probe truth、推理延遲、AI 路由分布、Failover/Recovery 觸發。Host111 由 K3s host120/121 independent verifier 補證，尚未取得 runtime receipt 時不得假綠。",
|
||
"editable": true,
|
||
"fiscalYearStartMonth": 0,
|
||
"graphTooltip": 1,
|
||
"id": null,
|
||
"links": [],
|
||
"panels": [
|
||
{
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "${datasource}"
|
||
},
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"color": {
|
||
"mode": "thresholds"
|
||
},
|
||
"mappings": [],
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [
|
||
{ "color": "red", "value": null },
|
||
{ "color": "yellow", "value": 50 },
|
||
{ "color": "green", "value": 100 }
|
||
]
|
||
},
|
||
"unit": "percent",
|
||
"min": 0,
|
||
"max": 100
|
||
},
|
||
"overrides": []
|
||
},
|
||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 },
|
||
"id": 1,
|
||
"options": {
|
||
"colorMode": "background",
|
||
"graphMode": "none",
|
||
"justifyMode": "center",
|
||
"orientation": "auto",
|
||
"reduceOptions": {
|
||
"calcs": ["lastNotNull"],
|
||
"fields": "",
|
||
"values": false
|
||
},
|
||
"textMode": "auto"
|
||
},
|
||
"title": "GCP-A/B Cloud Edge 可用性",
|
||
"description": "GCP-A/B 僅顯示 host110 blackbox /api/tags 的 probe_success；absent series 明確轉為 0，避免 blackbox-http 其他網站健康時假綠。\n\nHost111 不從已退役的 host110 proxy/probe；狀態為 independent K3s verifier pending，需 host120/121 verifier runtime receipt 才能另行標綠。",
|
||
"targets": [
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"expr": "100 * (max by (provider) (probe_success{job=\"blackbox-http\",provider=\"ollama_gcp_a\",probe_origin=\"host110\",boundary=\"cloud_edge_availability_only\"}) or label_replace(0 * absent(probe_success{job=\"blackbox-http\",provider=\"ollama_gcp_a\",probe_origin=\"host110\",boundary=\"cloud_edge_availability_only\"}), \"provider\", \"ollama_gcp_a\", \"\", \"\"))",
|
||
"legendFormat": "GCP-A",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "${datasource}"
|
||
},
|
||
"expr": "100 * (max by (provider) (probe_success{job=\"blackbox-http\",provider=\"ollama_gcp_b\",probe_origin=\"host110\",boundary=\"cloud_edge_availability_only\"}) or label_replace(0 * absent(probe_success{job=\"blackbox-http\",provider=\"ollama_gcp_b\",probe_origin=\"host110\",boundary=\"cloud_edge_availability_only\"}), \"provider\", \"ollama_gcp_b\", \"\", \"\"))",
|
||
"legendFormat": "GCP-B",
|
||
"refId": "B"
|
||
}
|
||
],
|
||
"type": "stat"
|
||
},
|
||
{
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "${datasource}"
|
||
},
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"color": { "mode": "palette-classic" },
|
||
"custom": {
|
||
"axisCenteredZero": false,
|
||
"axisColorMode": "text",
|
||
"axisLabel": "延遲 (秒)",
|
||
"axisPlacement": "auto",
|
||
"barAlignment": 0,
|
||
"drawStyle": "line",
|
||
"fillOpacity": 10,
|
||
"gradientMode": "none",
|
||
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||
"lineInterpolation": "smooth",
|
||
"lineWidth": 2,
|
||
"pointSize": 5,
|
||
"scaleDistribution": { "type": "linear" },
|
||
"showPoints": "never",
|
||
"spanNulls": false
|
||
},
|
||
"mappings": [],
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [
|
||
{ "color": "green", "value": null },
|
||
{ "color": "yellow", "value": 10 },
|
||
{ "color": "red", "value": 30 }
|
||
]
|
||
},
|
||
"unit": "s"
|
||
},
|
||
"overrides": []
|
||
},
|
||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
|
||
"id": 2,
|
||
"options": {
|
||
"legend": {
|
||
"calcs": ["lastNotNull", "max"],
|
||
"displayMode": "table",
|
||
"placement": "bottom"
|
||
},
|
||
"tooltip": { "mode": "multi", "sort": "none" }
|
||
},
|
||
"title": "推理延遲 P50 / P99",
|
||
"description": "histogram_quantile(0.5/0.99, rate(ollama_inference_duration_seconds_bucket[5m]))\n- P50 > 10s = SLOW 門檻\n- P99 > 30s = DEGRADED 門檻，觸發 failover\n\n⚠️ BACKLOG: ollama_inference_duration_seconds_bucket 尚未暴露，面板會顯示 No Data 直到 Part 3 backlog 補完",
|
||
"targets": [
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"expr": "histogram_quantile(0.5, rate(ollama_inference_duration_seconds_bucket[5m]))",
|
||
"legendFormat": "P50",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"expr": "histogram_quantile(0.99, rate(ollama_inference_duration_seconds_bucket[5m]))",
|
||
"legendFormat": "P99",
|
||
"refId": "B"
|
||
}
|
||
],
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "${datasource}"
|
||
},
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"color": { "mode": "palette-classic" },
|
||
"custom": {
|
||
"hideFrom": { "legend": false, "tooltip": false, "viz": false }
|
||
},
|
||
"mappings": [],
|
||
"unit": "reqps"
|
||
},
|
||
"overrides": []
|
||
},
|
||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 6 },
|
||
"id": 3,
|
||
"options": {
|
||
"displayLabels": ["name", "percent"],
|
||
"legend": {
|
||
"displayMode": "table",
|
||
"placement": "right",
|
||
"values": ["value", "percent"]
|
||
},
|
||
"pieType": "pie",
|
||
"tooltip": { "mode": "single", "sort": "none" }
|
||
},
|
||
"title": "AI Provider 路由分布",
|
||
"description": "sum by (provider) (rate(ai_router_selected_provider_total[5m]))\n- 正常狀態: ollama / ollama_gcp_a 佔大多數\n- failover 中: ollama_gcp_b / ollama_local / gemini 比例上升\n- 全走 gemini = Ollama provider pool 完全 offline\n\n資料來源: OLLAMA_FAILOVER_TRIGGERED_TOTAL + AI_ROUTER_PROVIDER_TOTAL (src/core/metrics.py)",
|
||
"targets": [
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"expr": "sum by (provider) (rate(ai_router_selected_provider_total[5m]))",
|
||
"legendFormat": "{{ provider }}",
|
||
"refId": "A"
|
||
}
|
||
],
|
||
"type": "piechart"
|
||
},
|
||
{
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "${datasource}"
|
||
},
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"color": { "mode": "palette-classic" },
|
||
"custom": {
|
||
"axisLabel": "次數/小時",
|
||
"fillOpacity": 80,
|
||
"gradientMode": "none",
|
||
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||
"lineWidth": 1
|
||
},
|
||
"mappings": [],
|
||
"unit": "short"
|
||
},
|
||
"overrides": [
|
||
{
|
||
"matcher": { "id": "byName", "options": "Failover" },
|
||
"properties": [
|
||
{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }
|
||
]
|
||
},
|
||
{
|
||
"matcher": { "id": "byName", "options": "Recovery" },
|
||
"properties": [
|
||
{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }
|
||
]
|
||
}
|
||
]
|
||
},
|
||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 6 },
|
||
"id": 4,
|
||
"options": {
|
||
"barRadius": 0.05,
|
||
"barWidth": 0.7,
|
||
"groupWidth": 0.7,
|
||
"legend": {
|
||
"calcs": ["sum"],
|
||
"displayMode": "list",
|
||
"placement": "bottom"
|
||
},
|
||
"orientation": "auto",
|
||
"tooltip": { "mode": "multi", "sort": "none" },
|
||
"xTickLabelRotation": 0
|
||
},
|
||
"title": "Failover / Recovery 觸發次數",
|
||
"description": "橘色 = ollama_failover_triggered_total (切離 111)\n綠色 = ollama_recovery_triggered_total (切回 111)\n\n正常狀態兩條都接近 0。\n橘升後緊跟綠升 = auto recovery 正常工作。\n橘升但綠不升 = OllamaRecoveryStuck，看 alert。\n\n資料來源: src/core/metrics.py OLLAMA_FAILOVER_TRIGGERED_TOTAL / OLLAMA_RECOVERY_TRIGGERED_TOTAL",
|
||
"targets": [
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"expr": "sum(rate(ollama_failover_triggered_total[1h])) * 3600",
|
||
"legendFormat": "Failover",
|
||
"refId": "A"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"expr": "sum(rate(ollama_recovery_triggered_total[1h])) * 3600",
|
||
"legendFormat": "Recovery",
|
||
"refId": "B"
|
||
}
|
||
],
|
||
"type": "barchart"
|
||
}
|
||
],
|
||
"refresh": "30s",
|
||
"schemaVersion": 39,
|
||
"tags": ["ollama", "failover", "aiops", "p2.3"],
|
||
"templating": {
|
||
"list": [
|
||
{
|
||
"current": {},
|
||
"hide": 0,
|
||
"includeAll": false,
|
||
"label": "Datasource",
|
||
"multi": false,
|
||
"name": "datasource",
|
||
"options": [],
|
||
"query": "prometheus",
|
||
"refresh": 1,
|
||
"regex": "",
|
||
"type": "datasource"
|
||
}
|
||
]
|
||
},
|
||
"time": { "from": "now-3h", "to": "now" },
|
||
"timepicker": {},
|
||
"timezone": "Asia/Taipei",
|
||
"title": "Ollama 容災監控",
|
||
"uid": "ollama-failover-p23",
|
||
"version": 1
|
||
}
|