fix(ai): require 111 before alert cloud fallback
All checks were successful
CD Pipeline / tests (push) Successful in 54s
Code Review / ai-code-review (push) Successful in 10s
CD Pipeline / build-and-deploy (push) Successful in 3m21s
CD Pipeline / post-deploy-checks (push) Successful in 2m2s

This commit is contained in:
Your Name
2026-05-06 00:05:38 +08:00
parent 23932773ef
commit 2aa31c205a
8 changed files with 184 additions and 27 deletions

View File

@@ -522,11 +522,11 @@ class Settings(BaseSettings):
),
)
ALERT_OLLAMA_MODEL: str = Field(
default="gemma3:4b",
default="qwen3:14b",
description=(
"Ollama model used for incident/alert diagnosis. Keep this separate "
"from the general RCA model so alert cards stay on the fast local "
"lane before Gemini backup is considered."
"Ollama model used for incident/alert deep diagnosis. Alert cards "
"may wait for this model; Gemini remains a backup after GCP-A, "
"GCP-B, and 111 fail."
),
)
# 2026-03-29 ogt: ADR-036 Nemotron Tool Calling 整合

View File

@@ -29,7 +29,7 @@ from src.services.model_registry import get_model_registry
logger = structlog.get_logger(__name__)
settings = get_settings()
_GCP_SAFE_MODELS = {
_GCP_LIGHTWEIGHT_MODELS = {
"gemma3:4b",
}
@@ -54,26 +54,34 @@ def _resolve_model_for_endpoint(
context: dict | None,
) -> str:
"""
Keep GCP-A/B on the fast alert model unless explicitly allowed.
Keep non-diagnosis calls from polluting the GCP diagnosis lane.
The GCP hosts currently expose CPU-only Ollama. Loading 7B/14B/32B models on
that lane blocks synchronous alerts long enough to fall through to Gemini.
Heavy/deep workloads must use 111 or the future AwoooP Inference Gateway.
GCP-A/B are allowed to run the deep incident diagnosis model because the
alert goal is correctness and resolution, not the fastest Telegram card.
Accidental non-diagnosis workloads still fall back to the lightweight health
model so embedding/Hermes/background calls cannot occupy the same lane.
"""
model_name = requested_model.strip()
context = context or {}
allow_gcp_heavy = bool(context.get("allow_gcp_heavy_model"))
task_type = str(context.get("task_type") or context.get("intent_hint") or "").lower()
is_deep_diagnosis = task_type in {"diagnose", "alert_deep", "incident_diagnosis"}
if _is_gcp_alert_lane(endpoint_url) and not allow_gcp_heavy and model_name not in _GCP_SAFE_MODELS:
alert_model = str(getattr(settings, "ALERT_OLLAMA_MODEL", "gemma3:4b")).strip() or "gemma3:4b"
if (
_is_gcp_alert_lane(endpoint_url)
and not allow_gcp_heavy
and not is_deep_diagnosis
and model_name not in _GCP_LIGHTWEIGHT_MODELS
):
fallback_model = str(getattr(settings, "OLLAMA_HEALTH_CHECK_MODEL", "gemma3:4b")).strip() or "gemma3:4b"
logger.warning(
"ollama_gcp_heavy_model_coerced",
"ollama_gcp_non_diagnosis_model_coerced",
endpoint=endpoint_url,
requested_model=model_name,
safe_model=alert_model,
task_type=context.get("task_type"),
safe_model=fallback_model,
task_type=task_type,
)
return alert_model
return fallback_model
return model_name

View File

@@ -1142,6 +1142,10 @@ class AIRouterExecutor:
_lf_trace_ctx = None
errors: list[str] = []
attempted_providers: set[str] = set()
alert_requires_ollama_before_cloud = bool(
(context or {}).get("alert_requires_ollama_before_cloud")
)
# 2026-04-27 Claude Sonnet 4.6: A2 INC-20260425 — DIAGNOSE fallback metric 追蹤
# 透過 context.get("intent_hint") 判斷是否為 DIAGNOSE避免改動 execute() 簽名
@@ -1191,13 +1195,31 @@ class AIRouterExecutor:
errors.append(f"{provider_name}: privacy_skip(non_local)")
continue
if alert_requires_ollama_before_cloud and provider.privacy_level == "cloud":
if "ollama_local" not in attempted_providers:
errors.append(f"{provider_name}: blocked_until_ollama_local_attempted")
logger.warning(
"ai_router_cloud_blocked_until_ollama_local_attempted",
provider=provider_name,
provider_order=provider_order,
attempted_providers=sorted(attempted_providers),
)
continue
# 閘門 1: Circuit Breaker (per-provider, C2 修復)
cb = self._get_circuit_breaker(provider_name)
if cb.is_open():
errors.append(f"{provider_name}: circuit_open")
logger.warning("ai_router_circuit_open", provider=provider_name)
# 2026-04-27 Claude Sonnet 4.6: F6 — circuit_open 不設 _last_attempted_provider未嘗試
continue
if alert_requires_ollama_before_cloud and provider_name.startswith("ollama"):
logger.warning(
"ai_router_alert_ollama_circuit_bypassed",
provider=provider_name,
reason="alert_requires_ollama_before_cloud",
)
else:
errors.append(f"{provider_name}: circuit_open")
logger.warning("ai_router_circuit_open", provider=provider_name)
# 2026-04-27 Claude Sonnet 4.6: F6 — circuit_open 不設 _last_attempted_provider未嘗試
continue
# 閘門 2: Rate Limiter
# 2026-04-02 Claude Code: Phase 24 B3 + C1 修復 — Rate Limiter (含 openclaw_nemo)
@@ -1217,6 +1239,7 @@ class AIRouterExecutor:
sem = self._get_semaphore(provider_name)
async with sem:
try:
attempted_providers.add(provider_name)
result = await provider.analyze(prompt, context)
if result.success:

View File

@@ -535,7 +535,7 @@ class OpenClawService:
registry = get_model_registry()
model_name = registry.get_model("ollama", "rca")
if ollama_only:
model_name = getattr(settings, "ALERT_OLLAMA_MODEL", "gemma3:4b")
model_name = getattr(settings, "ALERT_OLLAMA_MODEL", "qwen3:14b")
options = registry.get_provider_options("ollama")
timeout_seconds = max(
float(settings.OPENCLAW_TIMEOUT),
@@ -1145,7 +1145,9 @@ class OpenClawService:
if decision.intent == IntentType.DIAGNOSE:
exec_context["task_type"] = "diagnose"
if self._is_incident_alert_context(alert_context):
exec_context["ollama_model"] = getattr(settings, "ALERT_OLLAMA_MODEL", "gemma3:4b")
exec_context["ollama_model"] = getattr(settings, "ALERT_OLLAMA_MODEL", "qwen3:14b")
exec_context["allow_gcp_heavy_model"] = True
exec_context["alert_requires_ollama_before_cloud"] = True
result = await executor.execute(
prompt=prompt,