feat(p3.2-tests+ci-schema): model_version 測試 + CI test_schema 對齊 + Grafana SLO Dashboard
Some checks failed
CD Pipeline / build-and-deploy (push) Failing after 1m20s
Some checks failed
CD Pipeline / build-and-deploy (push) Failing after 1m20s
P3.2 配套測試 + CI 環境同步 + ADR-100 Grafana 視覺化：
CI test_schema 補齊（解 1162-1172 阻塞之延伸）:
- setup_test_schema.sql 加 ai_provider_version_history 表
- 對齊 production p3_2_provider_version_history.sql（已 K8s exec 上線）
新增測試 (636 行):
- test_model_version_probe.py (387) — Provider 探測單元測試
- test_model_version_tracker.py (249) — Tracker 整合測試
· 4 個 DB-dependent tests 標 @pytest.mark.integration
· 15 unit + 4 integration（unit step 跳過 integration class）
新增配套:
- ai-slo-dashboard.json (496 行) — Grafana 儀表板
· 對應 ADR-100 SLO 規則的 4 大面板：
自主修復成功率 / 飛輪閉環延遲 / 治理事件 / Provider 健康度
修改:
- governance_agent.py +122 行 — SLO 指標暴露 + retrieve metric 整合
Tests: 15 passed (probe + tracker unit), 4 deselected (integration class)
Production 部署狀態:
- p2_decision_fusion_columns.sql ✅ K8s exec 完成（commit c58bdd0c）
- p3_2_provider_version_history.sql ✅ K8s exec 完成（this commit）
- 兩個 production migration 都已上線，CI test_schema 同步補齊
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Your Name
@@ -5,10 +5,12 @@
|
||||
2. knowledge_degradation — KM 7 天未更新 > 20% 總量 → 告警知識衰退
|
||||
3. llm_hallucination — 近 100 筆 evidence verification_result=failed 比例 > 10%
|
||||
4. execution_blast_radius — 近 100 筆 auto_repair_executions.success=False 比例 > 15%
|
||||
5. slo_compliance — 4 個 SLO 合規性檢查（ADR-100），違反時降級飛輪行為
|
||||
|
||||
所有 check 互相隔離（try/except），任一失敗不阻斷其他項目。
|
||||
|
||||
2026-04-26 P2.2 by Claude
|
||||
2026-04-27 P3.4 by Claude — 新增 SLO 合規性自檢（ADR-100）
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -49,9 +51,13 @@ RECENT_LIMIT = 100 # 最近幾筆做統計
|
||||
# =============================================================================
|
||||
|
||||
class GovernanceAgent:
|
||||
"""AI 自我治理 Agent — 4 項自檢 + 1h 排程
|
||||
"""AI 自我治理 Agent — 5 項自檢 + 1h 排程
|
||||
|
||||
1-4: trust_drift / knowledge_degradation / llm_hallucination / execution_blast_radius
|
||||
5: slo_compliance（ADR-100 SLO 合規性）
|
||||
|
||||
2026-04-26 P2.2 by Claude
|
||||
2026-04-27 P3.4 by Claude — 加入第 5 項 slo_compliance
|
||||
"""
|
||||
|
||||
def __init__(self, alerter=None) -> None:
|
||||
@@ -241,14 +247,123 @@ class GovernanceAgent:
|
||||
)
|
||||
return {"total": total, "failed": failed, "rate": round(rate, 3)}
|
||||
|
||||
# =========================================================================
|
||||
# 5. SLO 合規性（ADR-100）
|
||||
# =========================================================================
|
||||
|
||||
async def check_slo_compliance(self) -> dict[str, Any]:
|
||||
"""SLO 4 項合規性檢查 — 違反時降級飛輪行為
|
||||
|
||||
從 Prometheus Recording rules 讀取 SLI 值，
|
||||
與硬紅線閾值比對，違反時呼叫 _alert() 寫 PG + 推 Telegram。
|
||||
|
||||
SLO 1 自主化率: sli:autonomy_rate:5m 硬紅線 < 0.70
|
||||
SLO 2 決策準確率: sli:decision_accuracy:5m 硬紅線 < 0.85
|
||||
SLO 3 信心校準: sli:confidence_calibration:1h 硬紅線 < 0.70
|
||||
SLO 4 KM 增長率: sli:km_growth_rate:24h 硬紅線 < 5
|
||||
|
||||
2026-04-27 P3.4 by Claude — AI SLO（ADR-100）
|
||||
"""
|
||||
import httpx
|
||||
|
||||
from src.core.config import settings
|
||||
|
||||
prom_url = getattr(settings, "PROMETHEUS_URL", "http://prometheus.observability.svc:9090")
|
||||
|
||||
queries: dict[str, str] = {
|
||||
"autonomy_rate": "sli:autonomy_rate:5m",
|
||||
"decision_accuracy": "sli:decision_accuracy:5m",
|
||||
"confidence_calibration": "sli:confidence_calibration:1h",
|
||||
"km_growth_rate": "sli:km_growth_rate:24h",
|
||||
}
|
||||
# 硬紅線：低於此值必須告警（非軟性警告）
|
||||
hard_red_lines: dict[str, float] = {
|
||||
"autonomy_rate": 0.70,
|
||||
"decision_accuracy": 0.85,
|
||||
"confidence_calibration": 0.70,
|
||||
"km_growth_rate": 5.0,
|
||||
}
|
||||
# SLO 目標值（供日誌記錄）
|
||||
slo_targets: dict[str, float] = {
|
||||
"autonomy_rate": 0.80,
|
||||
"decision_accuracy": 0.90,
|
||||
"confidence_calibration": 0.80,
|
||||
"km_growth_rate": 20.0,
|
||||
}
|
||||
|
||||
results: dict[str, Any] = {}
|
||||
|
||||
async with httpx.AsyncClient(timeout=5.0) as client:
|
||||
for name, query in queries.items():
|
||||
try:
|
||||
resp = await client.get(
|
||||
f"{prom_url}/api/v1/query",
|
||||
params={"query": query},
|
||||
)
|
||||
data = resp.json()
|
||||
if data.get("status") == "success":
|
||||
result_list = data.get("data", {}).get("result", [])
|
||||
value = float(result_list[0]["value"][1]) if result_list else 0.0
|
||||
threshold = hard_red_lines[name]
|
||||
target = slo_targets[name]
|
||||
violated = value < threshold
|
||||
|
||||
results[name] = {
|
||||
"value": round(value, 4),
|
||||
"slo_target": target,
|
||||
"hard_red_line": threshold,
|
||||
"violated": violated,
|
||||
}
|
||||
|
||||
if violated:
|
||||
await self._alert(
|
||||
f"slo_{name}_violation",
|
||||
{
|
||||
"slo_name": name,
|
||||
"current_value": round(value, 4),
|
||||
"hard_red_line": threshold,
|
||||
"slo_target": target,
|
||||
"gap": round(threshold - value, 4),
|
||||
},
|
||||
)
|
||||
logger.warning(
|
||||
"governance_slo_violated",
|
||||
slo=name,
|
||||
value=round(value, 4),
|
||||
hard_red_line=threshold,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"governance_slo_ok",
|
||||
slo=name,
|
||||
value=round(value, 4),
|
||||
target=target,
|
||||
)
|
||||
else:
|
||||
results[name] = {"error": "prometheus_query_failed", "status": data.get("status")}
|
||||
logger.warning(
|
||||
"governance_slo_prometheus_error",
|
||||
slo=name,
|
||||
query=query,
|
||||
response_status=data.get("status"),
|
||||
)
|
||||
except Exception as e:
|
||||
results[name] = {"error": str(e)}
|
||||
logger.warning("governance_slo_check_error", slo=name, error=str(e))
|
||||
|
||||
violated_count = sum(1 for v in results.values() if isinstance(v, dict) and v.get("violated"))
|
||||
logger.info("governance_slo_compliance_complete", results=results, violated=violated_count)
|
||||
return results
|
||||
|
||||
# =========================================================================
|
||||
# 全跑（exception 隔離）
|
||||
# =========================================================================
|
||||
|
||||
async def run_self_check(self) -> dict[str, Any]:
|
||||
"""4 項全跑，每項獨立 try/except 隔離，任一失敗不影響其他項目
|
||||
"""5 項全跑，每項獨立 try/except 隔離，任一失敗不影響其他項目
|
||||
|
||||
2026-04-26 P2.2 by Claude
|
||||
2026-04-27 P3.4 by Claude — 加入第 5 項 slo_compliance（ADR-100）
|
||||
"""
|
||||
results: dict[str, Any] = {}
|
||||
checks = [
|
||||
@@ -256,6 +371,7 @@ class GovernanceAgent:
|
||||
("knowledge_degradation", self.check_knowledge_degradation),
|
||||
("llm_hallucination", self.check_llm_hallucination),
|
||||
("execution_blast_radius", self.check_execution_blast_radius),
|
||||
("slo_compliance", self.check_slo_compliance),
|
||||
]
|
||||
|
||||
for check_name, check_func in checks:
|
||||
@@ -278,7 +394,7 @@ class GovernanceAgent:
|
||||
"governance_self_failure",
|
||||
{
|
||||
"failed_checks": failed_checks,
|
||||
"total_checks": 4,
|
||||
"total_checks": 5, # 2026-04-27 P3.4 by Claude — 加入 slo_compliance 後共 5 項
|
||||
"errors": {k: results[k].get("error") for k in failed_checks},
|
||||
},
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user