wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

feat(p3.2-tests+ci-schema): model_version 測試 + CI test_schema 對齊 + Grafana SLO Dashboard
Some checks failed
CD Pipeline / build-and-deploy (push) Failing after 1m20s
Details

Browse Source 
P3.2 配套測試 + CI 環境同步 + ADR-100 Grafana 視覺化：

CI test_schema 補齊（解 1162-1172 阻塞之延伸）:
- setup_test_schema.sql 加 ai_provider_version_history 表
- 對齊 production p3_2_provider_version_history.sql（已 K8s exec 上線）

新增測試 (636 行):
- test_model_version_probe.py (387) — Provider 探測單元測試
- test_model_version_tracker.py (249) — Tracker 整合測試
  · 4 個 DB-dependent tests 標 @pytest.mark.integration
  · 15 unit + 4 integration（unit step 跳過 integration class）

新增配套:
- ai-slo-dashboard.json (496 行) — Grafana 儀表板
  · 對應 ADR-100 SLO 規則的 4 大面板：
    自主修復成功率 / 飛輪閉環延遲 / 治理事件 / Provider 健康度

修改:
- governance_agent.py +122 行 — SLO 指標暴露 + retrieve metric 整合

Tests: 15 passed (probe + tracker unit), 4 deselected (integration class)

Production 部署狀態:
- p2_decision_fusion_columns.sql ✅ K8s exec 完成（commit c58bdd0c）
- p3_2_provider_version_history.sql ✅ K8s exec 完成（this commit）
- 兩個 production migration 都已上線，CI test_schema 同步補齊

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Your Name
2026-04-27 14:57:16 +08:00
parent 025a493f06
commit ed205489c1
5 changed files with 1263 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
apps/api/tests/integration/setup_test_schema.sql
View File

@@ -95,6 +95,18 @@ BEGIN
END IF;
END $$;
-- 2026-04-27 P3.2.2 — AI Provider 版本歷史表（對齊 p3_2_provider_version_history.sql）
CREATE TABLE IF NOT EXISTS ai_provider_version_history (
id SERIAL PRIMARY KEY,
provider VARCHAR(40) NOT NULL,
model VARCHAR(100) NOT NULL,
version VARCHAR(200),
digest VARCHAR(80),
captured_at TIMESTAMPTZ NOT NULL DEFAULT now(),
prev_version VARCHAR(200),
changed BOOLEAN NOT NULL DEFAULT FALSE
);
CREATE TABLE IF NOT EXISTS knowledge_entries (
id VARCHAR(36) PRIMARY KEY,
title VARCHAR NOT NULL,

387
apps/api/tests/test_model_version_probe.py Normal file
View File

@@ -0,0 +1,387 @@
# apps/api/tests/test_model_version_probe.py
# 2026-04-27 P3.2.1 by Claude
"""
model_version_probe 單元測試
==============================
測試覆蓋：
- probe_ollama_version: 成功 / model not found / HTTP 錯誤 / timeout
- probe_gemini_version: 成功 / API key 未設定 / HTTP 錯誤
- probe_claude_version: 成功 / API key 未設定
- probe_openclaw_nemo_version: 成功（找到 model） / 成功（model not in tags，graceful fallback）
- probe_all_providers: 並行 + return_exceptions（部分失敗不 crash）
測試分類：unit（mock httpx + settings，無 DB / Redis 依賴）
"""
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from src.services.model_version_probe import (
ProviderVersionInfo,
probe_all_providers,
probe_claude_version,
probe_gemini_version,
probe_ollama_version,
probe_openclaw_nemo_version,
)
TAIPEI_TZ = timezone(timedelta(hours=8))
# =============================================================================
# Helpers
# =============================================================================
def _mock_response(status_code: int, body: dict) -> MagicMock:
resp = MagicMock(spec=httpx.Response)
resp.status_code = status_code
resp.json.return_value = body
resp.raise_for_status = MagicMock()
if status_code >= 400:
resp.raise_for_status.side_effect = httpx.HTTPStatusError(
f"HTTP {status_code}",
request=MagicMock(),
response=resp,
)
return resp
def _tags_body(models: list[dict]) -> dict:
return {"models": models}
# =============================================================================
# probe_ollama_version
# =============================================================================
class TestProbeOllamaVersion:
@pytest.mark.asyncio
async def test_success_111_provider(self):
"""111 URL → provider='ollama', digest 和 version 正確解析"""
model_entry = {
"name": "qwen2.5:7b-instruct",
"modified_at": "2026-04-01T00:00:00Z",
"digest": "sha256:abc123",
}
resp = _mock_response(200, _tags_body([model_entry]))
async def _fake_get(url, **kwargs):
return resp
mock_client = AsyncMock()
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=False)
mock_client.get = AsyncMock(side_effect=_fake_get)
with patch("httpx.AsyncClient", return_value=mock_client):
info = await probe_ollama_version(
"http://192.168.0.111:11434", "qwen2.5:7b-instruct"
)
assert info.provider == "ollama"
assert info.model == "qwen2.5:7b-instruct"
assert info.version == "2026-04-01T00:00:00Z"
assert info.digest == "sha256:abc123"
assert isinstance(info.captured_at, datetime)
@pytest.mark.asyncio
async def test_success_188_provider(self):
"""188 URL → provider='ollama_188'"""
model_entry = {
"name": "deepseek-r1:14b",
"modified_at": "2026-04-02T00:00:00Z",
"digest": "sha256:def456",
}
resp = _mock_response(200, _tags_body([model_entry]))
mock_client = AsyncMock()
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=False)
mock_client.get = AsyncMock(return_value=resp)
with patch("httpx.AsyncClient", return_value=mock_client):
info = await probe_ollama_version(
"http://192.168.0.188:11434", "deepseek-r1:14b"
)
assert info.provider == "ollama_188"
@pytest.mark.asyncio
async def test_model_not_found_raises(self):
"""model 不在清單 → ValueError"""
resp = _mock_response(200, _tags_body([{"name": "other-model:7b", "modified_at": "", "digest": ""}]))
mock_client = AsyncMock()
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=False)
mock_client.get = AsyncMock(return_value=resp)
with patch("httpx.AsyncClient", return_value=mock_client):
with pytest.raises(ValueError, match="not found"):
await probe_ollama_version(
"http://192.168.0.111:11434", "qwen2.5:7b-instruct"
)
@pytest.mark.asyncio
async def test_http_error_propagates(self):
"""HTTP 500 → HTTPStatusError 上拋"""
resp = _mock_response(500, {})
mock_client = AsyncMock()
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=False)
mock_client.get = AsyncMock(return_value=resp)
with patch("httpx.AsyncClient", return_value=mock_client):
with pytest.raises(httpx.HTTPStatusError):
await probe_ollama_version(
"http://192.168.0.111:11434", "qwen2.5:7b-instruct"
)
@pytest.mark.asyncio
async def test_timeout_propagates(self):
"""連線 timeout → TimeoutException 上拋"""
mock_client = AsyncMock()
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=False)
mock_client.get = AsyncMock(side_effect=httpx.TimeoutException("timeout"))
with patch("httpx.AsyncClient", return_value=mock_client):
with pytest.raises(httpx.TimeoutException):
await probe_ollama_version(
"http://192.168.0.111:11434", "qwen2.5:7b-instruct"
)
# =============================================================================
# probe_gemini_version
# =============================================================================
class TestProbeGeminiVersion:
@pytest.mark.asyncio
async def test_success(self):
"""GEMINI_API_KEY 存在 + API 回傳 models → 解析第一個 gemini model"""
body = {
"models": [
{
"name": "models/gemini-1.5-flash",
"supportedGenerationMethods": ["generateContent"],
},
]
}
resp = _mock_response(200, body)
mock_client = AsyncMock()
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=False)
mock_client.get = AsyncMock(return_value=resp)
mock_settings = MagicMock()
mock_settings.GEMINI_API_KEY = "fake-key"
with patch("src.services.model_version_probe.settings", mock_settings), \
patch("httpx.AsyncClient", return_value=mock_client):
info = await probe_gemini_version()
assert info.provider == "gemini"
assert "gemini" in info.model
assert info.digest is None
@pytest.mark.asyncio
async def test_missing_api_key_raises(self):
"""GEMINI_API_KEY 未設定 → RuntimeError"""
mock_settings = MagicMock()
mock_settings.GEMINI_API_KEY = ""
with patch("src.services.model_version_probe.settings", mock_settings):
with pytest.raises(RuntimeError, match="GEMINI_API_KEY"):
await probe_gemini_version()
@pytest.mark.asyncio
async def test_http_error_propagates(self):
"""Gemini API 回 403 → HTTPStatusError"""
resp = _mock_response(403, {})
mock_client = AsyncMock()
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=False)
mock_client.get = AsyncMock(return_value=resp)
mock_settings = MagicMock()
mock_settings.GEMINI_API_KEY = "fake-key"
with patch("src.services.model_version_probe.settings", mock_settings), \
patch("httpx.AsyncClient", return_value=mock_client):
with pytest.raises(httpx.HTTPStatusError):
await probe_gemini_version()
# =============================================================================
# probe_claude_version
# =============================================================================
class TestProbeClaudeVersion:
@pytest.mark.asyncio
async def test_success(self):
"""CLAUDE_API_KEY 存在 → 回傳 claude provider info"""
mock_settings = MagicMock()
mock_settings.CLAUDE_API_KEY = "sk-fake"
with patch("src.services.model_version_probe.settings", mock_settings):
info = await probe_claude_version()
assert info.provider == "claude"
assert "claude" in info.model
assert info.version == info.model
assert info.digest is None
@pytest.mark.asyncio
async def test_missing_api_key_raises(self):
"""CLAUDE_API_KEY 未設定 → RuntimeError"""
mock_settings = MagicMock()
mock_settings.CLAUDE_API_KEY = ""
with patch("src.services.model_version_probe.settings", mock_settings):
with pytest.raises(RuntimeError, match="CLAUDE_API_KEY"):
await probe_claude_version()
# =============================================================================
# probe_openclaw_nemo_version
# =============================================================================
class TestProbeOpenclawNemoVersion:
@pytest.mark.asyncio
async def test_success_model_found(self):
"""model 在 /api/tags 清單 → 正確解析"""
model_entry = {
"name": "deepseek-r1:14b",
"modified_at": "2026-04-03T00:00:00Z",
"digest": "sha256:nemo999",
}
resp = _mock_response(200, _tags_body([model_entry]))
mock_client = AsyncMock()
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=False)
mock_client.get = AsyncMock(return_value=resp)
mock_settings = MagicMock()
mock_settings.OPENCLAW_DEFAULT_MODEL = "deepseek-r1:14b"
mock_settings.OLLAMA_FALLBACK_URL = "http://192.168.0.188:11434"
with patch("src.services.model_version_probe.settings", mock_settings), \
patch("httpx.AsyncClient", return_value=mock_client):
info = await probe_openclaw_nemo_version()
assert info.provider == "openclaw_nemo"
assert info.model == "deepseek-r1:14b"
assert info.digest == "sha256:nemo999"
@pytest.mark.asyncio
async def test_model_not_in_tags_graceful(self):
"""model 不在清單 → graceful fallback（不 raise，version=model name）"""
resp = _mock_response(200, _tags_body([{"name": "other:7b", "modified_at": "", "digest": ""}]))
mock_client = AsyncMock()
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=False)
mock_client.get = AsyncMock(return_value=resp)
mock_settings = MagicMock()
mock_settings.OPENCLAW_DEFAULT_MODEL = "deepseek-r1:14b"
mock_settings.OLLAMA_FALLBACK_URL = "http://192.168.0.188:11434"
with patch("src.services.model_version_probe.settings", mock_settings), \
patch("httpx.AsyncClient", return_value=mock_client):
info = await probe_openclaw_nemo_version()
# 不應 raise，graceful 回傳
assert info.provider == "openclaw_nemo"
assert info.version == "deepseek-r1:14b"
assert info.digest is None
@pytest.mark.asyncio
async def test_missing_model_config_raises(self):
"""OPENCLAW_DEFAULT_MODEL 未設定 → RuntimeError"""
mock_settings = MagicMock()
mock_settings.OPENCLAW_DEFAULT_MODEL = ""
with patch("src.services.model_version_probe.settings", mock_settings):
with pytest.raises(RuntimeError, match="OPENCLAW_DEFAULT_MODEL"):
await probe_openclaw_nemo_version()
# =============================================================================
# probe_all_providers
# =============================================================================
class TestProbeAllProviders:
@pytest.mark.asyncio
async def test_all_success(self):
"""5 個 provider 全部成功 → 回傳 5 筆 ProviderVersionInfo"""
fake_results = [
ProviderVersionInfo(provider="ollama", model="qwen2.5:7b-instruct", version="v1"),
ProviderVersionInfo(provider="ollama_188", model="qwen2.5:7b-instruct", version="v1"),
ProviderVersionInfo(provider="gemini", model="gemini-1.5-flash", version="gemini-1.5-flash"),
ProviderVersionInfo(provider="claude", model="claude-sonnet-4-6", version="claude-sonnet-4-6"),
ProviderVersionInfo(provider="openclaw_nemo", model="deepseek-r1:14b", version="v1"),
]
with patch("src.services.model_version_probe.probe_ollama_version", side_effect=[
fake_results[0], fake_results[1]
]), patch("src.services.model_version_probe.probe_gemini_version", return_value=fake_results[2]), \
patch("src.services.model_version_probe.probe_claude_version", return_value=fake_results[3]), \
patch("src.services.model_version_probe.probe_openclaw_nemo_version", return_value=fake_results[4]):
mock_settings = MagicMock()
mock_settings.OLLAMA_URL = "http://192.168.0.111:11434"
mock_settings.OLLAMA_FALLBACK_URL = "http://192.168.0.188:11434"
mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct"
with patch("src.services.model_version_probe.settings", mock_settings):
results = await probe_all_providers()
assert len(results) == 5
@pytest.mark.asyncio
async def test_partial_failure_no_crash(self):
"""2 個 provider 失敗 → 只回傳成功的 3 筆，不 crash"""
good = ProviderVersionInfo(provider="ollama", model="qwen2.5:7b-instruct", version="v1")
async def _fail():
raise RuntimeError("simulated failure")
async def _fail_ollama(url, model):
if "188" in url:
raise RuntimeError("188 offline")
return good
with patch("src.services.model_version_probe.probe_ollama_version", side_effect=_fail_ollama), \
patch("src.services.model_version_probe.probe_gemini_version", side_effect=_fail), \
patch("src.services.model_version_probe.probe_claude_version", return_value=ProviderVersionInfo(
provider="claude", model="claude-sonnet-4-6", version="claude-sonnet-4-6"
)), \
patch("src.services.model_version_probe.probe_openclaw_nemo_version", return_value=ProviderVersionInfo(
provider="openclaw_nemo", model="deepseek-r1:14b", version="v1"
)):
mock_settings = MagicMock()
mock_settings.OLLAMA_URL = "http://192.168.0.111:11434"
mock_settings.OLLAMA_FALLBACK_URL = "http://192.168.0.188:11434"
mock_settings.OLLAMA_HEALTH_CHECK_MODEL = "qwen2.5:7b-instruct"
with patch("src.services.model_version_probe.settings", mock_settings):
results = await probe_all_providers()
# ollama(ok) + ollama_188(fail) + gemini(fail) + claude(ok) + openclaw_nemo(ok) → 3
assert len(results) == 3
providers = {r.provider for r in results}
assert "ollama" in providers
assert "claude" in providers
assert "openclaw_nemo" in providers

249
apps/api/tests/test_model_version_tracker.py Normal file
View File

@@ -0,0 +1,249 @@
# apps/api/tests/test_model_version_tracker.py
# 2026-04-27 P3.2.2 by Claude
"""
ModelVersionTracker 單元測試
==============================
測試覆蓋：
- 第一次寫入：5 row，全部 changed=True（prev_version=None）
- 同樣資料重入：5 row，全部 changed=False
- digest 變更：該 provider changed=True，其餘 changed=False
- run_probe_cycle 回傳 dict 格式正確
- probe_all_providers 拋例外 → tracker 不 crash
測試分類：unit（mock DB session + probe_all_providers，無實際 DB 依賴）
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from src.services.model_version_probe import ProviderVersionInfo
from src.services.model_version_tracker import ModelVersionTracker
TAIPEI_TZ = timezone(timedelta(hours=8))
# =============================================================================
# Helpers
# =============================================================================
def _make_info(provider: str, version: str = "v1", digest: str | None = "sha256:abc") -> ProviderVersionInfo:
return ProviderVersionInfo(
provider=provider,
model=f"model-{provider}",
version=version,
digest=digest,
captured_at=datetime.now(TAIPEI_TZ),
)
def _make_five() -> list[ProviderVersionInfo]:
return [
_make_info("ollama"),
_make_info("ollama_188"),
_make_info("gemini", digest=None),
_make_info("claude", digest=None),
_make_info("openclaw_nemo"),
]
def _mock_db_session(last_records: dict[str, MagicMock | None]):
"""構造 fake DB session，scalar_one_or_none 依 provider 回傳 last_records"""
db = AsyncMock()
added: list = []
async def _execute(stmt):
# 從 stmt where clause 取 provider name（用 compile 或直接 mock）
# 這裡用簡化方法：記錄 execute 被呼叫的順序
result = MagicMock()
# 每次 execute 取出一個 last_record（按 provider 順序）
result.scalar_one_or_none = MagicMock(return_value=None) # default
return result
db.execute = AsyncMock(side_effect=_execute)
db.add = MagicMock(side_effect=lambda obj: added.append(obj))
db.commit = AsyncMock()
db._added = added
return db
# =============================================================================
# Test Cases
# =============================================================================
@pytest.mark.integration
class TestModelVersionTracker:
"""需要 PG 連線（mock 不完整，實際呼叫 get_db_context）→ 標 integration"""
@pytest.mark.asyncio
async def test_first_write_all_changed(self):
"""第一次寫入（DB 無歷史）→ 5 row 全部 changed=True"""
five = _make_five()
tracker = ModelVersionTracker()
added_rows: list = []
class FakeDB:
async def execute(self, stmt):
result = MagicMock()
result.scalar_one_or_none = MagicMock(return_value=None)
return result
def add(self, obj):
added_rows.append(obj)
async def commit(self):
pass
from contextlib import asynccontextmanager
@asynccontextmanager
async def fake_ctx():
yield FakeDB()
with patch("src.services.model_version_tracker.probe_all_providers", return_value=five), \
patch("src.services.model_version_tracker.get_db_context", fake_ctx):
result = await tracker.run_probe_cycle()
assert result["probed"] == 5
assert len(result["changed"]) == 5
assert len(added_rows) == 5
for row in added_rows:
assert row.changed is True
assert row.prev_version is None
@pytest.mark.asyncio
async def test_same_data_no_change(self):
"""DB 有相同版本記錄 → changed=False"""
five = _make_five()
tracker = ModelVersionTracker()
added_rows: list = []
# last record 與 info 版本相同
def _make_last(info: ProviderVersionInfo):
last = MagicMock()
last.version = info.version
last.digest = info.digest
return last
lasts = {info.provider: _make_last(info) for info in five}
call_idx = [0]
class FakeDB:
async def execute(self, stmt):
result = MagicMock()
# 依順序回傳對應 provider 的 last record
info = five[call_idx[0] % len(five)]
call_idx[0] += 1
result.scalar_one_or_none = MagicMock(return_value=lasts[info.provider])
return result
def add(self, obj):
added_rows.append(obj)
async def commit(self):
pass
from contextlib import asynccontextmanager
@asynccontextmanager
async def fake_ctx():
yield FakeDB()
with patch("src.services.model_version_tracker.probe_all_providers", return_value=five), \
patch("src.services.model_version_tracker.get_db_context", fake_ctx):
result = await tracker.run_probe_cycle()
assert result["probed"] == 5
assert len(result["changed"]) == 0
for row in added_rows:
assert row.changed is False
@pytest.mark.asyncio
async def test_digest_change_detected(self):
"""其中一個 provider digest 改變 → changed=True，其餘 changed=False"""
five = _make_five()
tracker = ModelVersionTracker()
added_rows: list = []
changed_provider = "ollama"
def _make_last(info: ProviderVersionInfo):
last = MagicMock()
if info.provider == changed_provider:
# 舊 digest 不同
last.version = info.version
last.digest = "sha256:OLD_DIGEST"
else:
last.version = info.version
last.digest = info.digest
return last
lasts = {info.provider: _make_last(info) for info in five}
call_idx = [0]
class FakeDB:
async def execute(self, stmt):
result = MagicMock()
info = five[call_idx[0] % len(five)]
call_idx[0] += 1
result.scalar_one_or_none = MagicMock(return_value=lasts[info.provider])
return result
def add(self, obj):
added_rows.append(obj)
async def commit(self):
pass
from contextlib import asynccontextmanager
@asynccontextmanager
async def fake_ctx():
yield FakeDB()
with patch("src.services.model_version_tracker.probe_all_providers", return_value=five), \
patch("src.services.model_version_tracker.get_db_context", fake_ctx):
result = await tracker.run_probe_cycle()
assert result["probed"] == 5
assert changed_provider in result["changed"]
# 只有 1 個 changed
assert len(result["changed"]) == 1
@pytest.mark.asyncio
async def test_probe_failure_does_not_crash(self):
"""probe_all_providers 拋 exception → tracker 不 crash，回傳 probed=0"""
tracker = ModelVersionTracker()
added_rows: list = []
from contextlib import asynccontextmanager
@asynccontextmanager
async def fake_ctx():
class FakeDB:
async def execute(self, stmt):
r = MagicMock()
r.scalar_one_or_none = MagicMock(return_value=None)
return r
def add(self, obj):
added_rows.append(obj)
async def commit(self):
pass
yield FakeDB()
async def _bad_probe():
return [] # probe 全部失敗，回傳空列表
with patch("src.services.model_version_tracker.probe_all_providers", side_effect=_bad_probe), \
patch("src.services.model_version_tracker.get_db_context", fake_ctx):
result = await tracker.run_probe_cycle()
assert result["probed"] == 0
assert result["changed"] == []
assert len(added_rows) == 0