fix(ai): recover canonical Ollama primary safely

apps/api/src/services/ollama_auto_recovery.py

@@ -31,6 +31,7 @@ from __future__ import annotations
 
 import asyncio
 import datetime
+import time
 from datetime import timedelta, timezone
 from typing import Any, Protocol, runtime_checkable
 
@@ -117,6 +118,11 @@ class OllamaAutoRecoveryService:
         # 修法：判斷改用 _is_ollama_primary()，比對 startswith("ollama_")
         self._current_primary: str = "ollama_gcp_a"   # fallback_manager 傳入的 provider_name
         self._consecutive_healthy: int = 0
+        self._state_lock = asyncio.Lock()
+        self._provider_transition_epoch: int = 0
+        self._provider_transition_completed_epoch: int = 0
+        self._last_observation_generation: int = 0
+        self._last_preferred_recovery_completed_at_monotonic: float = 0.0
         self._task: asyncio.Task | None = None
 
     # -------------------------------------------------------------------------
@@ -151,8 +157,9 @@ class OllamaAutoRecoveryService:
             current_primary=self._current_primary,
         )
 
-        # 若 primary 不是 Ollama，立刻評估一次（不等 30s interval）
-        if not self._is_ollama_primary(self._current_primary):
+        # 若目前不在 preferred GCP-A lane，立刻評估一次（不等 30s interval）。
+        # GCP-B / Local 雖是 Ollama，仍是 degraded fallback，必須持續追蹤 GCP-A。
+        if not self._is_preferred_primary(self._current_primary):
             try:
                 await self._check_and_recover()
             except Exception:
@@ -199,35 +206,81 @@ class OllamaAutoRecoveryService:
     # 狀態控制（供整合層設定當前 primary，例如 failover 觸發時更新）
     # -------------------------------------------------------------------------
 
-    async def set_current_primary(self, provider: str) -> None:
+    async def set_current_primary(
+        self,
+        provider: str,
+        *,
+        observation_generation: int | None = None,
+        observation_started_at_monotonic: float | None = None,
+    ) -> bool:
         """
         通知 recovery service 當前路由 primary 是哪個 provider。
         OllamaFailoverManager.select_provider() 觸發 failover 後呼叫此方法更新狀態。
+        回傳 True 代表 observation 已接受（含 provider 未變）；generation 或
+        recovery barrier 判定為 stale 時回傳 False，讓 manager 抑制舊告警。
 
         # 2026-04-25 critic-fix Part2 H5+H6 by Claude Engineer-C2
         # 改為 async，切換時同步寫 Redis 持久化（跨重啟恢復）
         """
-        if self._current_primary != provider:
-            logger.info(
-                "ollama_auto_recovery_primary_changed",
-                service="ollama_auto_recovery",
-                from_primary=self._current_primary,
-                to_primary=provider,
-            )
-            self._current_primary = provider
-            # Redis 持久化（跨重啟恢復）
-            await self._persist_primary(provider)
+        async with self._state_lock:
+            if observation_generation is not None:
+                if observation_generation <= self._last_observation_generation:
+                    logger.info(
+                        "ollama_auto_recovery_stale_generation_ignored",
+                        service="ollama_auto_recovery",
+                        provider=provider,
+                        observation_generation=observation_generation,
+                        last_observation_generation=(
+                            self._last_observation_generation
+                        ),
+                    )
+                    return False
+                self._last_observation_generation = observation_generation
 
-        # 2026-05-04 ogt: B5/B6 修復 — 判斷改用 _is_ollama_primary()
-        # 原設計：provider != "ollama"，但 callback 傳 "ollama_gcp_a" → 永遠觸發 tracking
-        if not self._is_ollama_primary(provider):
-            # 切換到非 Ollama（gemini/nemotron/claude）→ 重置 counter，開始監控恢復
+            if (
+                observation_started_at_monotonic is not None
+                and observation_started_at_monotonic
+                <= self._last_preferred_recovery_completed_at_monotonic
+            ):
+                logger.info(
+                    "ollama_auto_recovery_stale_observation_ignored",
+                    service="ollama_auto_recovery",
+                    provider=provider,
+                    observation_generation=observation_generation,
+                )
+                return False
+
+            await self._set_current_primary_locked(provider)
+            return True
+
+    async def _set_current_primary_locked(self, provider: str) -> bool:
+        """Apply one serialized provider transition and persist its exact lane."""
+        if self._current_primary == provider:
+            return False
+        logger.info(
+            "ollama_auto_recovery_primary_changed",
+            service="ollama_auto_recovery",
+            from_primary=self._current_primary,
+            to_primary=provider,
+        )
+        self._current_primary = provider
+        self._provider_transition_epoch += 1
+        transition_epoch = self._provider_transition_epoch
+        if not self._is_preferred_primary(provider):
+            # 新切到 fallback lane 時重置 counter；相同 lane 的重複
+            # observation 不得抹掉已累積的 GCP-A stable window。
             self._consecutive_healthy = 0
+        try:
+            await self._persist_primary(provider)
+        finally:
+            self._provider_transition_completed_epoch = transition_epoch
+        if not self._is_preferred_primary(provider):
             logger.info(
                 "ollama_auto_recovery_tracking_started",
                 service="ollama_auto_recovery",
                 current_primary=provider,
             )
+        return True
 
     # -------------------------------------------------------------------------
     # Redis 持久化（跨重啟恢復）
@@ -249,8 +302,10 @@ class OllamaAutoRecoveryService:
         try:
             from src.core.redis_client import get_redis
             redis = get_redis()
-            await redis.set(self._REDIS_PRIMARY_KEY, primary)
-            await redis.set(self._REDIS_PRIMARY_KEY_LEGACY, primary)  # Phase A 雙寫
+            await redis.mset({
+                self._REDIS_PRIMARY_KEY: primary,
+                self._REDIS_PRIMARY_KEY_LEGACY: primary,
+            })
         except Exception as e:
             logger.warning(
                 "ollama_auto_recovery_persist_failed",
@@ -308,6 +363,11 @@ class OllamaAutoRecoveryService:
         """Return true for both legacy "ollama" and ADR-110 provider names."""
         return provider == "ollama" or provider.startswith("ollama_")
 
+    @staticmethod
+    def _is_preferred_primary(provider: str) -> bool:
+        """Return whether routing already points at the canonical GCP-A lane."""
+        return provider in {"ollama", "ollama_gcp_a"}
+
     # -------------------------------------------------------------------------
     # 背景監控迴圈
     # -------------------------------------------------------------------------
@@ -336,6 +396,27 @@ class OllamaAutoRecoveryService:
         抽取為獨立方法，方便單元測試。
         """
         host = self._settings.OLLAMA_URL
+        probe_invoked_epoch = self._provider_transition_epoch
+        probe_invoked_completed_epoch = (
+            self._provider_transition_completed_epoch
+        )
+
+        async with self._state_lock:
+            if (
+                probe_invoked_epoch != probe_invoked_completed_epoch
+                or self._provider_transition_epoch != probe_invoked_epoch
+            ):
+                logger.info(
+                    "ollama_auto_recovery_transition_probe_ignored",
+                    service="ollama_auto_recovery",
+                    probe_invoked_epoch=probe_invoked_epoch,
+                    probe_invoked_completed_epoch=(
+                        probe_invoked_completed_epoch
+                    ),
+                    current_epoch=self._provider_transition_epoch,
+                )
+                return
+            probe_epoch = self._provider_transition_epoch
 
         try:
             health = await self._monitor.check(host)
@@ -345,45 +426,72 @@ class OllamaAutoRecoveryService:
                 service="ollama_auto_recovery",
                 host=host,
             )
-            self._consecutive_healthy = 0
+            async with self._state_lock:
+                if self._provider_transition_epoch == probe_epoch:
+                    self._consecutive_healthy = 0
             return
 
         # 2026-05-04 ogt: SLOW 視為可用（GCP 高負載 10-30s 仍優於 Gemini quota 限制）
         # 原設計只接受 HEALTHY，GCP 若因負載偏高落入 SLOW 區會永久卡在 Gemini
         is_usable = health.status in (HealthStatus.HEALTHY, HealthStatus.SLOW)
 
-        if is_usable:
-            self._consecutive_healthy += 1
-            logger.debug(
-                "ollama_auto_recovery_healthy_tick",
-                service="ollama_auto_recovery",
-                consecutive=self._consecutive_healthy,
-                required=self._stable_count_required,
-                current_primary=self._current_primary,
-                actual_status=health.status.value,
-            )
-
-            if (
-                self._consecutive_healthy >= self._stable_count_required
-                and not self._is_ollama_primary(self._current_primary)
-            ):
-                await self._switch_back_to_ollama()
-        else:
-            # DEGRADED / OFFLINE → counter 歸零，繼續等
-            if self._consecutive_healthy > 0:
-                logger.debug(
-                    "ollama_auto_recovery_counter_reset",
+        should_recover = False
+        async with self._state_lock:
+            if self._provider_transition_epoch != probe_epoch:
+                logger.info(
+                    "ollama_auto_recovery_stale_health_probe_ignored",
                     service="ollama_auto_recovery",
-                    previous_count=self._consecutive_healthy,
-                    status=health.status.value,
+                    probe_epoch=probe_epoch,
+                    current_epoch=self._provider_transition_epoch,
+                    current_primary=self._current_primary,
                 )
-            self._consecutive_healthy = 0
+                return
+
+            if self._is_preferred_primary(self._current_primary):
+                self._consecutive_healthy = 0
+                return
+
+            if is_usable:
+                self._consecutive_healthy += 1
+                logger.debug(
+                    "ollama_auto_recovery_healthy_tick",
+                    service="ollama_auto_recovery",
+                    consecutive=self._consecutive_healthy,
+                    required=self._stable_count_required,
+                    current_primary=self._current_primary,
+                    actual_status=health.status.value,
+                )
+
+                should_recover = (
+                    self._consecutive_healthy
+                    >= self._stable_count_required
+                    and not self._is_preferred_primary(self._current_primary)
+                )
+            else:
+                # DEGRADED / OFFLINE → counter 歸零，繼續等
+                if self._consecutive_healthy > 0:
+                    logger.debug(
+                        "ollama_auto_recovery_counter_reset",
+                        service="ollama_auto_recovery",
+                        previous_count=self._consecutive_healthy,
+                        status=health.status.value,
+                    )
+                self._consecutive_healthy = 0
+
+        if should_recover:
+            await self._switch_back_to_ollama(
+                expected_provider_epoch=probe_epoch,
+            )
 
     # -------------------------------------------------------------------------
     # 切回 Ollama 111
     # -------------------------------------------------------------------------
 
-    async def _switch_back_to_ollama(self) -> None:
+    async def _switch_back_to_ollama(
+        self,
+        *,
+        expected_provider_epoch: int | None = None,
+    ) -> None:
         """
         切回 Ollama 111：
         1. 更新 _current_primary
@@ -392,34 +500,79 @@ class OllamaAutoRecoveryService:
         4. Telegram 通知（若 alerter 已設定）
         5. structlog audit
         """
-        from_provider = self._current_primary
-        self._current_primary = "ollama"
-        stable_count = self._consecutive_healthy
-        # counter 不立即清零：保留供 audit，下次 loop 開始後自然計數
-        self._consecutive_healthy = 0
+        async with self._state_lock:
+            if (
+                expected_provider_epoch is not None
+                and self._provider_transition_epoch
+                != expected_provider_epoch
+            ):
+                logger.info(
+                    "ollama_auto_recovery_stale_switch_ignored",
+                    service="ollama_auto_recovery",
+                    expected_provider_epoch=expected_provider_epoch,
+                    current_epoch=self._provider_transition_epoch,
+                    current_primary=self._current_primary,
+                )
+                return
+            if self._is_preferred_primary(self._current_primary):
+                self._consecutive_healthy = 0
+                return
+            if self._consecutive_healthy < self._stable_count_required:
+                logger.debug(
+                    "ollama_auto_recovery_debounce_pending",
+                    service="ollama_auto_recovery",
+                    consecutive=self._consecutive_healthy,
+                    required=self._stable_count_required,
+                    current_primary=self._current_primary,
+                )
+                return
+            from_provider = self._current_primary
+            stable_count = self._consecutive_healthy
+            self._consecutive_healthy = 0
+            await self._set_current_primary_locked("ollama_gcp_a")
 
-        # 取台北時間（標準庫保證 +8，禁 UTC）
-        # 2026-04-25 critic-fix Part2 B4 by Claude Engineer-C2
-        recovery_time = datetime.datetime.now(TAIPEI_TZ)
+            # 取台北時間（標準庫保證 +8，禁 UTC）
+            # 2026-04-25 critic-fix Part2 B4 by Claude Engineer-C2
+            recovery_time = datetime.datetime.now(TAIPEI_TZ)
 
-        # 清空 failover manager 快取（讓下次 select_provider 重新評估）
-        try:
-            await self._failover_manager.clear_cache()
-        except Exception:
-            logger.exception(
-                "ollama_auto_recovery_clear_cache_error",
-                service="ollama_auto_recovery",
+            # 清空 failover manager 快取（讓下次 select_provider 重新評估）
+            try:
+                await self._failover_manager.clear_cache()
+            except Exception:
+                logger.exception(
+                    "ollama_auto_recovery_clear_cache_error",
+                    service="ollama_auto_recovery",
+                )
+
+            # 通知 failover_manager
+            try:
+                self._failover_manager.notify_recovery("ollama_gcp_a")
+            except Exception:
+                logger.exception(
+                    "ollama_auto_recovery_notify_error",
+                    service="ollama_auto_recovery",
+                )
+            self._last_preferred_recovery_completed_at_monotonic = (
+                time.monotonic()
+            )
+            await self._emit_recovery_side_effects_locked(
+                from_provider=from_provider,
+                stable_count=stable_count,
+                recovery_time=recovery_time,
             )
 
-        # 通知 failover_manager
-        try:
-            # 2026-05-03 ogt: ADR-110 — provider name 改為 "ollama_gcp_a"（GCP-A Primary）
-            self._failover_manager.notify_recovery("ollama_gcp_a")
-        except Exception:
-            logger.exception(
-                "ollama_auto_recovery_notify_error",
-                service="ollama_auto_recovery",
-            )
+    async def _emit_recovery_side_effects_locked(
+        self,
+        *,
+        from_provider: str,
+        stable_count: int,
+        recovery_time: datetime.datetime,
+    ) -> None:
+        """Emit one serialized recovery notification, metric, and audit event.
+
+        The caller holds ``_state_lock`` so a newer fallback cannot become
+        canonical while this recovery lifecycle notification is in flight.
+        """
 
         # Telegram 通知（P1.5 Engineer-D 接入）
         # 2026-04-25 P1.5 by Claude Engineer-D — 接 FailoverAlerter，self._alerter 優先，
@@ -442,7 +595,7 @@ class OllamaAutoRecoveryService:
                 _provider_label = "Ollama"
             await alerter.alert_recovery({
                 "from": from_provider,
-                "to": "ollama",
+                "to": "ollama_gcp_a",
                 "recovered_host": f"{_provider_label} {_recovered_url}",
                 "stable_count": stable_count,
                 "recovery_time": recovery_time.isoformat(),

apps/api/src/services/ollama_failover_manager.py

@@ -31,6 +31,8 @@ from __future__ import annotations
 
 import asyncio
 import datetime
+import inspect
+import time
 from dataclasses import dataclass
 from datetime import timedelta, timezone
 
@@ -188,7 +190,11 @@ class OllamaFailoverManager:
         # recovery_callback: async callable(provider_name: str) → None
         # OllamaAutoRecoveryService.set_current_primary 在 failover 時被通知，
         # 避免重啟後 _current_primary 停留在 "ollama" 而永不啟動恢復監控
-        self._recovery_callback = recovery_callback
+        self._recovery_callback = None
+        self._recovery_callback_accepts_observation_metadata = False
+        self.set_recovery_callback(recovery_callback)
+        self._recovery_observation_generation = 0
+        self._recovery_side_effect_lock = asyncio.Lock()
 
     # -------------------------------------------------------------------------
     # Public API
@@ -218,6 +224,15 @@ class OllamaFailoverManager:
         Returns:
             OllamaRoutingResult
         """
+        recovery_observation_generation: int | None = None
+        recovery_observation_started_at_monotonic: float | None = None
+        if emit_side_effects:
+            self._recovery_observation_generation += 1
+            recovery_observation_generation = (
+                self._recovery_observation_generation
+            )
+            recovery_observation_started_at_monotonic = time.monotonic()
+
         # 2026-05-04 ogt: 改用語意中性名稱 primary/secondary/tertiary，
         # 避免 gcp_a/gcp_b/local 與實際 URL 脫鉤造成 log 誤導
         url_primary = self._settings.OLLAMA_URL  # canonical GCP-A endpoint
@@ -284,12 +299,6 @@ class OllamaFailoverManager:
         # Provider selection never consumes paid quota. Claude/Gemini providers
         # each reserve requests/tokens/cost atomically immediately before calls.
 
-        # Status/readback probes must not emit failover metrics, Telegram, or
-        # recovery-state writes. Production action callers retain the existing
-        # behavior and must own the explicit side-effect boundary.
-        if emit_side_effects:
-            await self._write_failover_audit(result)
-
         def _status(report: HealthReport | None) -> str:
             return report.status.value if report else "not_checked"
 
@@ -310,22 +319,14 @@ class OllamaFailoverManager:
             side_effects_emitted=emit_side_effects,
         )
 
-        # Recovery tracks the first health-observed fallback candidate, while
-        # the selected execution primary remains fixed at GCP-A.
-        # 2026-04-25 critic-fix Part2 H5+H6 by Claude Engineer-C2
-        if emit_side_effects and self._recovery_callback is not None:
-            try:
-                recovery_observation = (
-                    result.observed_fallback.provider_name
-                    if result.observed_fallback
-                    else result.primary.provider_name
-                )
-                await self._recovery_callback(recovery_observation)
-            except Exception as e:
-                logger.warning(
-                    "ollama_failover_recovery_callback_failed",
-                    error=str(e),
-                )
+        if emit_side_effects:
+            await self._emit_recovery_observation_side_effects(
+                result,
+                observation_generation=recovery_observation_generation,
+                observation_started_at_monotonic=(
+                    recovery_observation_started_at_monotonic
+                ),
+            )
 
         return result
 
@@ -533,14 +534,108 @@ class OllamaFailoverManager:
     # Recovery API（供 OllamaAutoRecoveryService 呼叫）
     # -------------------------------------------------------------------------
 
+    async def _emit_recovery_observation_side_effects(
+        self,
+        result: OllamaRoutingResult,
+        *,
+        observation_generation: int | None,
+        observation_started_at_monotonic: float | None,
+    ) -> None:
+        """Serialize one accepted provider observation and its alert lifecycle."""
+        if (
+            observation_generation is None
+            or observation_started_at_monotonic is None
+        ):
+            logger.warning(
+                "ollama_failover_observation_metadata_missing",
+                service="ollama_failover",
+            )
+            return
+
+        async with self._recovery_side_effect_lock:
+            if observation_generation != self._recovery_observation_generation:
+                logger.info(
+                    "ollama_failover_stale_observation_side_effects_ignored",
+                    service="ollama_failover",
+                    observation_generation=observation_generation,
+                    latest_generation=self._recovery_observation_generation,
+                )
+                return
+
+            # Recovery tracks the first health-observed fallback candidate,
+            # while the selected execution primary remains fixed at GCP-A.
+            recovery_callback = self._recovery_callback
+            callback_accepted = True
+            if recovery_callback is not None:
+                recovery_observation = (
+                    result.observed_fallback.provider_name
+                    if result.observed_fallback
+                    else result.primary.provider_name
+                )
+                try:
+                    if self._recovery_callback_accepts_observation_metadata:
+                        callback_result = await recovery_callback(
+                            recovery_observation,
+                            observation_generation=observation_generation,
+                            observation_started_at_monotonic=(
+                                observation_started_at_monotonic
+                            ),
+                        )
+                    else:
+                        callback_result = await recovery_callback(
+                            recovery_observation
+                        )
+                    callback_accepted = callback_result is not False
+                except Exception as e:
+                    callback_accepted = False
+                    logger.warning(
+                        "ollama_failover_recovery_callback_failed",
+                        error=str(e),
+                    )
+
+            if not callback_accepted:
+                logger.info(
+                    "ollama_failover_rejected_observation_alert_suppressed",
+                    service="ollama_failover",
+                    observation_generation=observation_generation,
+                )
+                return
+
+            # State acceptance is ordered before recipient-visible degraded
+            # audit/Telegram, and concurrent selections cannot overtake it.
+            await self._write_failover_audit(result)
+
     def set_recovery_callback(self, callback) -> None:
         """
         設定 recovery callback（供 lifespan wiring 使用）。
-        callback signature: async (provider_name: str) -> None
+        Legacy callback signature remains async (provider_name: str) -> None.
+        Metadata-capable callbacks may additionally declare both
+        observation_generation and observation_started_at_monotonic keyword
+        parameters, or accept arbitrary keyword arguments. Returning literal
+        False rejects the observation and suppresses its degraded alert;
+        legacy None remains accepted.
 
         # 2026-04-25 P1.2 by Claude Engineer-A2 — failover 整合到 ai_router + lifespan
         """
         self._recovery_callback = callback
+        self._recovery_callback_accepts_observation_metadata = False
+        if callback is None:
+            return
+        try:
+            parameters = inspect.signature(callback).parameters.values()
+        except (TypeError, ValueError):
+            return
+        parameter_names = {parameter.name for parameter in parameters}
+        self._recovery_callback_accepts_observation_metadata = (
+            {
+                "observation_generation",
+                "observation_started_at_monotonic",
+            }.issubset(parameter_names)
+            or any(
+                parameter.kind is inspect.Parameter.VAR_KEYWORD
+                for parameter in parameters
+            )
+        )
 
     async def clear_cache(self) -> None:
         """