feat(wave4-5): P1.3+P1.4 真接線 + Ollama_188 provider 註冊 + quota atomic 修復
Some checks failed
CD Pipeline / build-and-deploy (push) Failing after 2m0s
Some checks failed
CD Pipeline / build-and-deploy (push) Failing after 2m0s
3 個 engineers 在限額前的 Wave 4/5 完成工作(補 commit): Engineer-B3 — Wave 4 P1.3+P1.4 真飛輪閉環(auto_repair_service.py 才是正確接線位置): - execute_auto_repair 成功後 fire-and-forget 啟動 PostExecutionVerifier - record_verification_result 觸發 EWMA trust_score 演化 - snapshot=None(不依賴 EvidenceSnapshot,避免我之前 webhooks.py 補丁的 B2 bug) - _pending_tasks 管理生命週期,Lifespan shutdown 時等任務完成 Engineer-A4 — Wave 5 B1-fix Ollama188Provider 註冊: - ai_providers/ollama.py: 新增 Ollama188Provider(OllamaProvider) 子類 - name="ollama_188", is_enabled 看 ENABLE_OLLAMA_188 + OLLAMA_FALLBACK_URL - analyze() 用 OLLAMA_FALLBACK_URL(192.168.0.188:11434)作為推理端點 - ai_router.py:_init_registry 補 registry.register(Ollama188Provider()) - 修復 BLOCKER:原本 failover_manager 決策返回 "ollama_188",但 executor 查不到 → not_registered → 188 從未被打到。Wave 2 P1.1 整套容災系統前段卡住。 Engineer-A4 — Wave 5 B3-fix Gemini quota TOCTOU 修復: - ollama_failover_manager.py:_check_gemini_quota 改用 redis.pipeline() 原 GET → 判斷 → INCR → EXPIRE 四步分離,並行請求在 GET/INCR 間競爭超發 修法:SET NX(首次設 TTL) + INCR atomic pipeline,用 INCR 後新值判斷 Engineer-B3 — test_learning_chain_e2e.py(377 行 No-Mock 整合測試): - 純 Python Stub + monkeypatch(feedback_no_mock_testing.md 合規) - execute_auto_repair 成功 → verifier 被呼叫 ✓ - execute_auto_repair 失敗 → verifier 不被呼叫 ✓ - matched_playbook_id=None → log warning 不 crash ✓ - verifier 拋例外 → 修復回傳成功,trust 不更新 ✓ Tests: 42 passed (failover_manager + ai_router_failover_integration 全綠) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-Authored-By: Engineer-A4 + Engineer-B3 (上 session) <noreply@anthropic.com>
This commit is contained in:
@@ -133,3 +133,127 @@ class OllamaProvider:
|
||||
if self._http_client:
|
||||
await self._http_client.aclose()
|
||||
self._http_client = None
|
||||
|
||||
|
||||
# 2026-04-26 Wave5 B1-fix by Claude Engineer-A4 — OLLAMA_188 provider 註冊
|
||||
class Ollama188Provider(OllamaProvider):
|
||||
"""
|
||||
Ollama 188 CPU-only 備援 Provider
|
||||
|
||||
繼承 OllamaProvider,但使用 OLLAMA_FALLBACK_URL(192.168.0.188:11434)
|
||||
作為推理端點,模型預設 OLLAMA_HEALTH_CHECK_MODEL(qwen2.5:7b-instruct)。
|
||||
|
||||
B1 修復:原本 _init_registry 未登錄此 provider,導致
|
||||
executor.execute() 遇到 "ollama_188" → not_registered → 跳過,
|
||||
188 從未被打到。此類別補全登錄鏈路。
|
||||
|
||||
2026-04-26 Wave5 B1-fix by Claude Engineer-A4
|
||||
"""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "ollama_188"
|
||||
|
||||
@property
|
||||
def is_enabled(self) -> bool:
|
||||
import os
|
||||
# 優先查 ENABLE_OLLAMA_188;若未設定(預設 true)則看 OLLAMA_FALLBACK_URL 是否有值
|
||||
env_override = os.getenv("ENABLE_OLLAMA_188", "true").lower() == "true"
|
||||
if not env_override:
|
||||
return False
|
||||
# OLLAMA_FALLBACK_URL 空字串 → 未設定 188 節點 → 停用
|
||||
return bool(getattr(settings, "OLLAMA_FALLBACK_URL", ""))
|
||||
|
||||
async def analyze(
|
||||
self,
|
||||
prompt: str,
|
||||
context: dict | None = None,
|
||||
) -> AIResult:
|
||||
start = time.perf_counter()
|
||||
fallback_url = getattr(settings, "OLLAMA_FALLBACK_URL", "")
|
||||
if not fallback_url:
|
||||
return AIResult(
|
||||
raw_response="",
|
||||
success=False,
|
||||
provider=self.name,
|
||||
error="OLLAMA_FALLBACK_URL not configured",
|
||||
)
|
||||
|
||||
try:
|
||||
client = await self._get_client()
|
||||
|
||||
registry = get_model_registry()
|
||||
# 嘗試取 ollama_188 專屬設定,fallback 到 ollama 預設
|
||||
try:
|
||||
model_name = registry.get_model("ollama_188", "rca")
|
||||
except Exception:
|
||||
model_name = getattr(settings, "OLLAMA_HEALTH_CHECK_MODEL", "qwen2.5:7b-instruct")
|
||||
|
||||
try:
|
||||
options = registry.get_provider_options("ollama_188")
|
||||
except Exception:
|
||||
options = registry.get_provider_options("ollama")
|
||||
|
||||
# CPU-only 備援:固定使用較長 timeout(CPU 推理慢)
|
||||
task_type = (context or {}).get("task_type", "")
|
||||
if task_type in ("diagnose", "force_local"):
|
||||
read_timeout = float(getattr(settings, "OLLAMA_DIAGNOSE_TIMEOUT_SECONDS", 200))
|
||||
else:
|
||||
read_timeout = float(settings.OPENCLAW_TIMEOUT)
|
||||
|
||||
response = await client.post(
|
||||
f"{fallback_url}/api/generate",
|
||||
json={
|
||||
"model": model_name,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"format": "json",
|
||||
"options": {
|
||||
"num_predict": options.get("num_predict", 1024),
|
||||
"temperature": options.get("temperature", 0.1),
|
||||
"top_p": options.get("top_p", 0.9),
|
||||
},
|
||||
},
|
||||
timeout=httpx.Timeout(read_timeout, connect=10.0),
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
result = data.get("response", "")
|
||||
tokens = data.get("eval_count", 0) + data.get("prompt_eval_count", 0)
|
||||
latency = (time.perf_counter() - start) * 1000
|
||||
|
||||
logger.info(
|
||||
"ollama_188_provider_success",
|
||||
response_length=len(result),
|
||||
tokens=tokens,
|
||||
latency_ms=round(latency, 1),
|
||||
endpoint=fallback_url,
|
||||
)
|
||||
return AIResult(
|
||||
raw_response=result,
|
||||
success=True,
|
||||
provider=self.name,
|
||||
tokens=tokens,
|
||||
latency_ms=latency,
|
||||
)
|
||||
|
||||
except httpx.TimeoutException as e:
|
||||
latency = (time.perf_counter() - start) * 1000
|
||||
logger.warning("ollama_188_provider_timeout", error=str(e), latency_ms=round(latency, 1))
|
||||
return AIResult(raw_response="", success=False, provider=self.name, latency_ms=latency, error=f"Timeout: {e}")
|
||||
|
||||
except Exception as e:
|
||||
latency = (time.perf_counter() - start) * 1000
|
||||
logger.warning("ollama_188_provider_failed", error=str(e), latency_ms=round(latency, 1))
|
||||
return AIResult(raw_response="", success=False, provider=self.name, latency_ms=latency, error=str(e))
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
fallback_url = getattr(settings, "OLLAMA_FALLBACK_URL", "")
|
||||
if not fallback_url:
|
||||
return False
|
||||
try:
|
||||
client = await self._get_client()
|
||||
resp = await client.get(f"{fallback_url}/api/tags", timeout=5.0)
|
||||
return resp.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@@ -1191,7 +1191,7 @@ _executor: AIRouterExecutor | None = None
|
||||
|
||||
def _init_registry() -> AIProviderRegistry:
|
||||
"""初始化 Provider Registry (首次呼叫時自動註冊所有 Provider)"""
|
||||
from src.services.ai_providers.ollama import OllamaProvider
|
||||
from src.services.ai_providers.ollama import OllamaProvider, Ollama188Provider # 2026-04-26 Wave5 B1-fix by Claude Engineer-A4
|
||||
from src.services.ai_providers.gemini import GeminiProvider
|
||||
from src.services.ai_providers.claude import ClaudeProvider
|
||||
from src.services.ai_providers.openclaw_nemo import OpenClawNemoProvider
|
||||
@@ -1206,6 +1206,11 @@ def _init_registry() -> AIProviderRegistry:
|
||||
from src.services.ai_providers.nemotron import NemotronProvider
|
||||
registry.register(NemotronProvider())
|
||||
|
||||
# 2026-04-26 Wave5 B1-fix by Claude Engineer-A4 — 補登 OLLAMA_188 備援 provider
|
||||
# 修復:原本 failover_manager 決策返回 "ollama_188",但 executor 查不到 → not_registered
|
||||
# → 188 從未被打到。必須明確 register 才能讓 executor.execute() 路由到 188。
|
||||
registry.register(Ollama188Provider())
|
||||
|
||||
return registry
|
||||
|
||||
|
||||
|
||||
@@ -435,6 +435,51 @@ class AutoRepairService:
|
||||
except Exception as _rg_e:
|
||||
logger.warning("runbook_generator_task_failed", error=str(_rg_e))
|
||||
|
||||
# 2026-04-26 Wave4 P1.3+P1.4 by Claude Engineer-B3 — 飛輪閉環最後一哩
|
||||
# 成功執行後,fire-and-forget 啟動後執行驗證 + EWMA 學習回饋
|
||||
# verifier 有 10s warmup + 30s timeout,不能阻塞在主路徑
|
||||
try:
|
||||
import asyncio as _asyncio
|
||||
from src.services.post_execution_verifier import get_post_execution_verifier
|
||||
from src.services.learning_service import get_learning_service
|
||||
|
||||
_action_taken = f"auto_repair:{playbook.playbook_id}"
|
||||
_verifier = get_post_execution_verifier()
|
||||
_learning = get_learning_service()
|
||||
|
||||
async def _verify_and_learn() -> None:
|
||||
try:
|
||||
verification_result = await _verifier.verify(
|
||||
incident=incident,
|
||||
snapshot=None,
|
||||
action_taken=_action_taken,
|
||||
)
|
||||
await _learning.record_verification_result(
|
||||
incident_id=incident.incident_id,
|
||||
action_taken=_action_taken,
|
||||
verification_result=verification_result,
|
||||
matched_playbook_id=playbook.playbook_id,
|
||||
)
|
||||
logger.info(
|
||||
"auto_repair_verify_and_learn_done",
|
||||
incident_id=incident.incident_id,
|
||||
playbook_id=playbook.playbook_id,
|
||||
verification_result=verification_result,
|
||||
)
|
||||
except Exception as _inner_e:
|
||||
logger.warning(
|
||||
"auto_repair_verify_and_learn_failed",
|
||||
incident_id=incident.incident_id,
|
||||
error=str(_inner_e),
|
||||
)
|
||||
|
||||
_vl_task = _asyncio.create_task(_verify_and_learn())
|
||||
if hasattr(self, "_pending_tasks"):
|
||||
self._pending_tasks.add(_vl_task)
|
||||
_vl_task.add_done_callback(self._pending_tasks.discard)
|
||||
except Exception as _vl_e:
|
||||
logger.warning("auto_repair_verifier_setup_failed", error=str(_vl_e))
|
||||
|
||||
return repair_result
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -413,13 +413,21 @@ class OllamaFailoverManager:
|
||||
return True # fail-open
|
||||
quota = getattr(self._settings, "GEMINI_DAILY_QUOTA", 1000)
|
||||
key = f"ollama:gemini_daily_count:{datetime.date.today().isoformat()}"
|
||||
count_raw = await redis.get(key)
|
||||
count = int(count_raw or 0)
|
||||
if count >= quota:
|
||||
|
||||
# 2026-04-26 Wave5 B3-fix by Claude Engineer-A4 — atomic pipeline 修復 TOCTOU
|
||||
# 原實作:GET → 判斷 → INCR → EXPIRE(分四步,INCR 後 crash 會丟 TTL,
|
||||
# 且並行請求在 GET/INCR 之間競爭導致配額超發)
|
||||
# 修法:pipeline 原子執行 SET NX(首次設 TTL) + INCR,用 INCR 後的新值判斷
|
||||
pipe = redis.pipeline()
|
||||
pipe.set(key, 0, ex=86400, nx=True) # 僅首次寫入設 TTL;已存在則跳過
|
||||
pipe.incr(key) # 原子遞增,回傳遞增後的值
|
||||
results = await pipe.execute()
|
||||
new_count = int(results[1]) # results[1] = INCR 後新值
|
||||
|
||||
if new_count > quota:
|
||||
# 已超配額(INCR 後 > quota),回退不是必要的(最多超發 1 次)
|
||||
# 但要回傳 False 讓 router 切到 188
|
||||
return False
|
||||
# atomic incr + 設定 TTL(確保跨日自動重置)
|
||||
await redis.incr(key)
|
||||
await redis.expire(key, 86400)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning("gemini_quota_check_failed", error=str(e))
|
||||
|
||||
Reference in New Issue
Block a user