feat(p3.2-tests+ci-schema): model_version 測試 + CI test_schema 對齊 + Grafana SLO Dashboard
Some checks failed
CD Pipeline / build-and-deploy (push) Failing after 1m20s

P3.2 配套測試 + CI 環境同步 + ADR-100 Grafana 視覺化:

CI test_schema 補齊(解 1162-1172 阻塞之延伸):
- setup_test_schema.sql 加 ai_provider_version_history 表
- 對齊 production p3_2_provider_version_history.sql(已 K8s exec 上線)

新增測試 (636 行):
- test_model_version_probe.py (387) — Provider 探測單元測試
- test_model_version_tracker.py (249) — Tracker 整合測試
  · 4 個 DB-dependent tests 標 @pytest.mark.integration
  · 15 unit + 4 integration(unit step 跳過 integration class)

新增配套:
- ai-slo-dashboard.json (496 行) — Grafana 儀表板
  · 對應 ADR-100 SLO 規則的 4 大面板:
    自主修復成功率 / 飛輪閉環延遲 / 治理事件 / Provider 健康度

修改:
- governance_agent.py +122 行 — SLO 指標暴露 + retrieve metric 整合

Tests: 15 passed (probe + tracker unit), 4 deselected (integration class)

Production 部署狀態:
- p2_decision_fusion_columns.sql  K8s exec 完成(commit c58bdd0c)
- p3_2_provider_version_history.sql  K8s exec 完成(this commit)
- 兩個 production migration 都已上線,CI test_schema 同步補齊

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Your Name
2026-04-27 14:57:16 +08:00
parent 025a493f06
commit ed205489c1
5 changed files with 1263 additions and 3 deletions

View File

@@ -0,0 +1,496 @@
{
"__inputs": [],
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "10.0.0"
},
{
"type": "panel",
"id": "timeseries",
"name": "Time series",
"version": ""
},
{
"type": "panel",
"id": "gauge",
"name": "Gauge",
"version": ""
},
{
"type": "panel",
"id": "barchart",
"name": "Bar chart",
"version": ""
},
{
"type": "panel",
"id": "table",
"name": "Table",
"version": ""
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
}
],
"annotations": {
"list": []
},
"description": "AI 自主化飛輪 SLO Dashboard — 自主化率/決策準確率/信心校準/KM 增長率 | ADR-100 2026-04-27 P3.4 台北時區",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"refresh": "60s",
"schemaVersion": 39,
"tags": ["slo", "ai", "autonomous", "flywheel"],
"templating": {
"list": [
{
"current": {},
"hide": 0,
"includeAll": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"type": "datasource"
}
]
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {},
"timezone": "Asia/Taipei",
"title": "AI 自主化飛輪 SLO",
"uid": "ai-autonomous-slo-v1",
"version": 1,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "比率",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false
},
"mappings": [],
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "yellow", "value": 0.7 },
{ "color": "green", "value": 0.8 }
]
},
"unit": "percentunit"
},
"overrides": [
{
"matcher": { "id": "byName", "options": "SLO 目標 80%" },
"properties": [
{ "id": "custom.lineStyle", "value": { "dash": [10, 5], "fill": "dash" } },
{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } },
{ "id": "custom.lineWidth", "value": 1 }
]
}
]
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
"id": 1,
"options": {
"legend": { "calcs": ["lastNotNull", "min"], "displayMode": "list", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "none" }
},
"title": "SLO 1 — 自主化率24h 趨勢)",
"description": "SLI = auto_executed / all_operations5m rate\n目標 SLO ≥ 80%\n橙色虛線 = 80% 閾值",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"expr": "sli:autonomy_rate:5m",
"legendFormat": "自主化率",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"expr": "vector(0.80)",
"legendFormat": "SLO 目標 80%",
"refId": "B"
}
],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "比率",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false
},
"mappings": [],
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "yellow", "value": 0.85 },
{ "color": "green", "value": 0.9 }
]
},
"unit": "percentunit"
},
"overrides": [
{
"matcher": { "id": "byName", "options": "SLO 目標 90%" },
"properties": [
{ "id": "custom.lineStyle", "value": { "dash": [10, 5], "fill": "dash" } },
{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } },
{ "id": "custom.lineWidth", "value": 1 }
]
}
]
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
"id": 2,
"options": {
"legend": { "calcs": ["lastNotNull", "min"], "displayMode": "list", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "none" }
},
"title": "SLO 2 — 決策準確率24h 趨勢)",
"description": "SLI = verifier_success / auto_executed5m rate\n目標 SLO ≥ 90%\n橙色虛線 = 90% 閾值",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"expr": "sli:decision_accuracy:5m",
"legendFormat": "決策準確率",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"expr": "vector(0.90)",
"legendFormat": "SLO 目標 90%",
"refId": "B"
}
],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "yellow", "value": 0.7 },
{ "color": "green", "value": 0.8 }
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 6, "x": 0, "y": 8 },
"id": 3,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showThresholdLabels": true,
"showThresholdMarkers": true
},
"title": "SLO 3 — 信心校準(當前值)",
"description": "SLI = high_confidence_success / high_confidence_total1h 滑動窗口)\n目標 SLO ≥ 80%(綠線)\n≥ 0.8 = 綠色0.7~0.8 = 黃色,< 0.7 = 紅色",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"expr": "sli:confidence_calibration:1h",
"legendFormat": "信心校準",
"refId": "A"
}
],
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"fillOpacity": 70,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineWidth": 1
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "yellow", "value": 5 },
{ "color": "green", "value": 20 }
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 6, "x": 6, "y": 8 },
"id": 4,
"options": {
"barRadius": 0,
"barWidth": 0.8,
"colorByField": "Value",
"fullHighlight": false,
"groupWidth": 0.7,
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom" },
"orientation": "auto",
"showValue": "always",
"stacking": "none",
"tooltip": { "mode": "single", "sort": "none" },
"xTickLabelRotation": 0,
"xTickLabelSpacing": 0
},
"title": "SLO 4 — KM 增長率7d 每日新增)",
"description": "SLI = increase(knowledge_entries_total[24h])\n目標 SLO ≥ 20 筆/day綠色\n5~20 = 黃色,< 5 = 紅色(疑似 KM 鏈斷裂)",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"expr": "sli:km_growth_rate:24h",
"legendFormat": "KM 增長/day",
"refId": "A"
}
],
"type": "barchart"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [
{
"options": { "from": 0, "to": 0, "result": { "color": "red", "text": "已耗盡" } },
"type": "range"
}
],
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "yellow", "value": 0.25 },
{ "color": "green", "value": 0.5 }
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
"id": 5,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "horizontal",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"title": "Error Budget Remaining4 SLO",
"description": "剩餘 error budget 比例1 = 100% 剩餘0 = 已耗盡)\n- SLO 3 信心校準 budget 計算:(1 - SLI) / 0.20\n- SLO 1/2 用 5m rate 估算\n- 顯示: 各 SLO 剩餘預算 %",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"expr": "clamp(slo:autonomy_rate:error_budget_remaining, 0, 1)",
"legendFormat": "SLO1 自主化率",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"expr": "clamp(slo:decision_accuracy:error_budget_remaining, 0, 1)",
"legendFormat": "SLO2 決策準確率",
"refId": "B"
},
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"expr": "clamp(slo:confidence_calibration:error_budget_remaining, 0, 1)",
"legendFormat": "SLO3 信心校準",
"refId": "C"
}
],
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"custom": {
"align": "auto",
"cellOptions": { "type": "auto" },
"filterable": false,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 1 },
{ "color": "red", "value": 2 }
]
}
},
"overrides": [
{
"matcher": { "id": "byName", "options": "alertname" },
"properties": [{ "id": "custom.width", "value": 300 }]
},
{
"matcher": { "id": "byName", "options": "severity" },
"properties": [
{ "id": "custom.width", "value": 100 },
{
"id": "mappings",
"value": [
{ "options": { "critical": { "color": "red", "index": 0 } }, "type": "value" },
{ "options": { "warning": { "color": "yellow", "index": 1 } }, "type": "value" },
{ "options": { "info": { "color": "blue", "index": 2 } }, "type": "value" }
]
},
{ "id": "custom.cellOptions", "value": { "type": "color-background" } }
]
},
{
"matcher": { "id": "byName", "options": "slo_name" },
"properties": [{ "id": "custom.width", "value": 200 }]
},
{
"matcher": { "id": "byName", "options": "burn_window" },
"properties": [{ "id": "custom.width", "value": 100 }]
}
]
},
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
"id": 6,
"options": {
"cellHeight": "sm",
"footer": { "countRows": false, "enablePagination": false, "fields": "", "reducer": ["sum"], "show": false },
"showHeader": true,
"sortBy": [{ "desc": true, "displayName": "severity" }]
},
"title": "Burn Rate Alerts當前觸發",
"description": "列出當前觸發中的 SLO burn rate alerts\n按 severity 排序critical > warning > info\n空白 = 所有 SLO 健康",
"targets": [
{
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"expr": "ALERTS{slo_name=~\".+\", alertstate=\"firing\"}",
"format": "table",
"instant": true,
"legendFormat": "",
"refId": "A"
}
],
"transformations": [
{
"id": "filterFieldsByName",
"options": {
"include": {
"names": ["alertname", "severity", "slo_name", "burn_window", "team", "alertstate"]
}
}
},
{
"id": "sortBy",
"options": {
"fields": [{ "desc": true, "displayName": "severity" }]
}
}
],
"type": "table"
}
]
}