fix(ai): recover canonical Ollama primary safely

apps/api/tests/test_ollama_auto_recovery.py

@@ -22,7 +22,8 @@ OllamaAutoRecoveryService 單元測試 - P1.1d
 from __future__ import annotations
 
 import asyncio
-from unittest.mock import AsyncMock, MagicMock, call, patch
+import time
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
@@ -147,6 +148,25 @@ class TestCheckAndRecover:
 
         mock_switch.assert_awaited_once()
 
+    @pytest.mark.asyncio
+    async def test_gcp_b_fallback_tracks_gcp_a_recovery(self):
+        """GCP-B fallback must not disable the preferred GCP-A recovery probe."""
+        svc, mock_monitor, _ = _make_service(
+            current_primary="ollama_gcp_b",
+            stable_count=3,
+        )
+        mock_monitor.check.return_value = _make_health(HealthStatus.HEALTHY)
+        svc._consecutive_healthy = 2
+
+        with patch.object(
+            svc,
+            "_switch_back_to_ollama",
+            new_callable=AsyncMock,
+        ) as mock_switch:
+            await svc._check_and_recover()
+
+        mock_switch.assert_awaited_once()
+
     @pytest.mark.asyncio
     async def test_two_healthy_not_yet_switch(self):
         """連續 2 次 HEALTHY（未達 3 次門檻）→ 不觸發切回"""
@@ -235,8 +255,8 @@ class TestDebounce:
         for _ in range(3):
             await svc._check_and_recover()
 
-        # 觸發後 current_primary 應切回 ollama
-        assert svc.current_primary == "ollama"
+        # 觸發後 current_primary 應切回 canonical GCP-A lane
+        assert svc.current_primary == "ollama_gcp_a"
         mock_failover.clear_cache.assert_awaited_once()
         # 2026-05-03 ogt: ADR-110 — notify_recovery 改為 "ollama_gcp_a"（GCP-A Primary）
         mock_failover.notify_recovery.assert_called_once_with("ollama_gcp_a")
@@ -251,20 +271,38 @@ class TestSwitchBackToOllama:
     """_switch_back_to_ollama 切回邏輯"""
 
     @pytest.mark.asyncio
-    async def test_sets_current_primary_to_ollama(self):
+    async def test_sets_current_primary_to_gcp_a(self):
         svc, _, _ = _make_service(current_primary="gemini")
+        svc._consecutive_healthy = svc._stable_count_required
         await svc._switch_back_to_ollama()
-        assert svc.current_primary == "ollama"
+        assert svc.current_primary == "ollama_gcp_a"
+
+    @pytest.mark.asyncio
+    async def test_persists_exact_gcp_a_identity_after_recovery(self):
+        svc, _, _ = _make_service(current_primary="ollama_gcp_b")
+        svc._consecutive_healthy = svc._stable_count_required
+
+        with patch.object(
+            svc,
+            "_persist_primary",
+            new_callable=AsyncMock,
+        ) as persist_primary:
+            await svc._switch_back_to_ollama()
+
+        assert svc.current_primary == "ollama_gcp_a"
+        persist_primary.assert_awaited_once_with("ollama_gcp_a")
 
     @pytest.mark.asyncio
     async def test_calls_clear_cache(self):
         svc, _, mock_failover = _make_service(current_primary="gemini")
+        svc._consecutive_healthy = svc._stable_count_required
         await svc._switch_back_to_ollama()
         mock_failover.clear_cache.assert_awaited_once()
 
     @pytest.mark.asyncio
     async def test_calls_notify_recovery(self):
         svc, _, mock_failover = _make_service(current_primary="gemini")
+        svc._consecutive_healthy = svc._stable_count_required
         await svc._switch_back_to_ollama()
         # 2026-05-03 ogt: ADR-110 — notify_recovery 改為 "ollama_gcp_a"（GCP-A Primary）
         mock_failover.notify_recovery.assert_called_once_with("ollama_gcp_a")
@@ -282,28 +320,30 @@ class TestSwitchBackToOllama:
         mock_alerter.alert_recovery.assert_awaited_once()
         payload = mock_alerter.alert_recovery.call_args[0][0]
         assert payload["from"] == "gemini"
-        assert payload["to"] == "ollama"  # 2026-05-03 ogt: ADR-110 — "to" 改為 "ollama"（不再指定 111）
+        assert payload["to"] == "ollama_gcp_a"
         assert payload["stable_count"] == 3
 
     @pytest.mark.asyncio
     async def test_no_alerter_does_not_crash(self):
         """alerter = None → 不 crash"""
         svc, _, _ = _make_service(current_primary="gemini", alerter=None)
+        svc._consecutive_healthy = svc._stable_count_required
         # 不應 raise
         await svc._switch_back_to_ollama()
-        assert svc.current_primary == "ollama"
+        assert svc.current_primary == "ollama_gcp_a"
 
     @pytest.mark.asyncio
     async def test_clear_cache_failure_does_not_crash(self):
         """clear_cache 失敗 → 靜默繼續，不 crash"""
         svc, _, mock_failover = _make_service(current_primary="gemini")
         mock_failover.clear_cache.side_effect = RuntimeError("Redis down")
+        svc._consecutive_healthy = svc._stable_count_required
 
         # 不應 raise
         await svc._switch_back_to_ollama()
 
         # 儘管 clear_cache 失敗，current_primary 應已更新
-        assert svc.current_primary == "ollama"
+        assert svc.current_primary == "ollama_gcp_a"
 
     @pytest.mark.asyncio
     async def test_alerter_failure_does_not_crash(self):
@@ -311,11 +351,83 @@ class TestSwitchBackToOllama:
         mock_alerter = AsyncMock()
         mock_alerter.alert_recovery = AsyncMock(side_effect=RuntimeError("TG timeout"))
         svc, _, _ = _make_service(current_primary="gemini", alerter=mock_alerter)
+        svc._consecutive_healthy = svc._stable_count_required
 
         # 不應 raise
         await svc._switch_back_to_ollama()
 
-        assert svc.current_primary == "ollama"
+        assert svc.current_primary == "ollama_gcp_a"
+
+    @pytest.mark.asyncio
+    async def test_does_not_switch_before_stability_threshold(self):
+        mock_alerter = AsyncMock()
+        svc, _, mock_failover = _make_service(
+            current_primary="ollama_gcp_b",
+            stable_count=3,
+            alerter=mock_alerter,
+        )
+        svc._consecutive_healthy = 2
+
+        await svc._switch_back_to_ollama()
+
+        assert svc.current_primary == "ollama_gcp_b"
+        assert svc.consecutive_healthy == 2
+        mock_failover.clear_cache.assert_not_awaited()
+        mock_failover.notify_recovery.assert_not_called()
+        mock_alerter.alert_recovery.assert_not_awaited()
+
+    @pytest.mark.asyncio
+    async def test_orders_newer_fallback_after_recovery_notification(self):
+        recovery_alert_started = asyncio.Event()
+        release_recovery_alert = asyncio.Event()
+        observed_primary: list[tuple[str, str]] = []
+
+        async def blocking_recovery_alert(payload):
+            observed_primary.append(("started", svc.current_primary))
+            recovery_alert_started.set()
+            await release_recovery_alert.wait()
+            observed_primary.append(("completed", svc.current_primary))
+
+        mock_alerter = AsyncMock()
+        mock_alerter.alert_recovery = AsyncMock(
+            side_effect=blocking_recovery_alert,
+        )
+        svc, _, _ = _make_service(
+            current_primary="ollama_gcp_b",
+            stable_count=3,
+            alerter=mock_alerter,
+        )
+        svc._consecutive_healthy = svc._stable_count_required
+
+        with patch.object(
+            svc,
+            "_persist_primary",
+            new_callable=AsyncMock,
+        ):
+            recovery_task = asyncio.create_task(
+                svc._switch_back_to_ollama()
+            )
+            await recovery_alert_started.wait()
+            fallback_task = asyncio.create_task(
+                svc.set_current_primary(
+                    "ollama_local",
+                    observation_generation=1,
+                    observation_started_at_monotonic=time.monotonic(),
+                )
+            )
+            await asyncio.sleep(0)
+            fallback_waited_for_recovery = fallback_task.done() is False
+            primary_during_recovery_alert = svc.current_primary
+            release_recovery_alert.set()
+            await asyncio.gather(recovery_task, fallback_task)
+
+        assert fallback_waited_for_recovery is True
+        assert primary_during_recovery_alert == "ollama_gcp_a"
+        assert observed_primary == [
+            ("started", "ollama_gcp_a"),
+            ("completed", "ollama_gcp_a"),
+        ]
+        assert svc.current_primary == "ollama_local"
 
 
 # =============================================================================
@@ -423,6 +535,162 @@ class TestStateManagement:
 
         assert svc.consecutive_healthy == 0
 
+    @pytest.mark.asyncio
+    async def test_set_current_primary_to_gcp_b_resets_recovery_counter(self):
+        svc, _, _ = _make_service(current_primary="ollama_gcp_a")
+        svc._consecutive_healthy = 5
+
+        with patch("src.core.redis_client.get_redis", side_effect=RuntimeError("no redis")):
+            await svc.set_current_primary("ollama_gcp_b")
+
+        assert svc.current_primary == "ollama_gcp_b"
+        assert svc.consecutive_healthy == 0
+
+    @pytest.mark.asyncio
+    async def test_same_gcp_b_observation_preserves_recovery_counter(self):
+        svc, _, _ = _make_service(current_primary="ollama_gcp_b")
+        svc._consecutive_healthy = 2
+        mock_redis = AsyncMock()
+
+        with patch("src.core.redis_client.get_redis", return_value=mock_redis):
+            await svc.set_current_primary("ollama_gcp_b")
+
+        assert svc.consecutive_healthy == 2
+        mock_redis.set.assert_not_awaited()
+
+    @pytest.mark.asyncio
+    async def test_stale_fallback_observation_cannot_reopen_recovered_primary(self):
+        mock_alerter = AsyncMock()
+        svc, _, _ = _make_service(
+            current_primary="ollama_gcp_b",
+            alerter=mock_alerter,
+        )
+        svc._consecutive_healthy = svc._stable_count_required
+        stale_observation_started_at = time.monotonic()
+
+        with patch.object(
+            svc,
+            "_persist_primary",
+            new_callable=AsyncMock,
+        ) as persist_primary:
+            await svc._switch_back_to_ollama()
+            await svc.set_current_primary(
+                "ollama_gcp_b",
+                observation_generation=1,
+                observation_started_at_monotonic=stale_observation_started_at,
+            )
+
+        assert svc.current_primary == "ollama_gcp_a"
+        persist_primary.assert_awaited_once_with("ollama_gcp_a")
+        mock_alerter.alert_recovery.assert_awaited_once()
+
+    @pytest.mark.asyncio
+    async def test_inflight_stale_fallback_waits_for_atomic_recovery_transition(self):
+        mock_alerter = AsyncMock()
+        svc, _, _ = _make_service(
+            current_primary="ollama_gcp_b",
+            alerter=mock_alerter,
+        )
+        svc._consecutive_healthy = svc._stable_count_required
+        stale_observation_started_at = time.monotonic()
+        recovery_persist_started = asyncio.Event()
+        release_recovery_persist = asyncio.Event()
+
+        async def blocking_persist(provider: str) -> None:
+            assert provider == "ollama_gcp_a"
+            recovery_persist_started.set()
+            await release_recovery_persist.wait()
+
+        with patch.object(
+            svc,
+            "_persist_primary",
+            side_effect=blocking_persist,
+        ) as persist_primary:
+            recovery_task = asyncio.create_task(svc._switch_back_to_ollama())
+            await recovery_persist_started.wait()
+            stale_callback_task = asyncio.create_task(
+                svc.set_current_primary(
+                    "ollama_gcp_b",
+                    observation_generation=1,
+                    observation_started_at_monotonic=(
+                        stale_observation_started_at
+                    ),
+                )
+            )
+            await asyncio.sleep(0)
+            release_recovery_persist.set()
+            await asyncio.gather(recovery_task, stale_callback_task)
+
+        assert svc.current_primary == "ollama_gcp_a"
+        assert svc.consecutive_healthy == 0
+        persist_primary.assert_awaited_once_with("ollama_gcp_a")
+        mock_alerter.alert_recovery.assert_awaited_once()
+
+    @pytest.mark.asyncio
+    async def test_newer_local_transition_invalidates_inflight_health_tick(self):
+        mock_alerter = AsyncMock()
+        svc, mock_monitor, _ = _make_service(
+            current_primary="ollama_gcp_b",
+            stable_count=3,
+            alerter=mock_alerter,
+        )
+        svc._consecutive_healthy = 2
+        mock_monitor.check.return_value = _make_health(HealthStatus.HEALTHY)
+        local_persist_started = asyncio.Event()
+        release_local_persist = asyncio.Event()
+
+        async def blocking_persist(provider: str) -> None:
+            if provider == "ollama_local":
+                local_persist_started.set()
+                await release_local_persist.wait()
+
+        with patch.object(
+            svc,
+            "_persist_primary",
+            side_effect=blocking_persist,
+        ) as persist_primary:
+            local_transition = asyncio.create_task(
+                svc.set_current_primary(
+                    "ollama_local",
+                    observation_generation=1,
+                    observation_started_at_monotonic=time.monotonic(),
+                )
+            )
+            await local_persist_started.wait()
+            inflight_health_tick = asyncio.create_task(
+                svc._check_and_recover()
+            )
+            await asyncio.sleep(0)
+            release_local_persist.set()
+            await asyncio.gather(local_transition, inflight_health_tick)
+
+        assert svc.current_primary == "ollama_local"
+        assert svc.consecutive_healthy == 0
+        persist_primary.assert_awaited_once_with("ollama_local")
+        mock_alerter.alert_recovery.assert_not_awaited()
+
+    @pytest.mark.asyncio
+    async def test_older_observation_generation_cannot_overwrite_newer_lane(self):
+        svc, _, _ = _make_service(current_primary="ollama_gcp_a")
+        with patch.object(
+            svc,
+            "_persist_primary",
+            new_callable=AsyncMock,
+        ) as persist_primary:
+            await svc.set_current_primary(
+                "ollama_gcp_b",
+                observation_generation=2,
+                observation_started_at_monotonic=time.monotonic(),
+            )
+            await svc.set_current_primary(
+                "ollama_local",
+                observation_generation=1,
+                observation_started_at_monotonic=time.monotonic(),
+            )
+
+        assert svc.current_primary == "ollama_gcp_b"
+        persist_primary.assert_awaited_once_with("ollama_gcp_b")
+
     @pytest.mark.asyncio
     async def test_set_current_primary_to_ollama_no_counter_reset(self):
         """切換到 ollama（正常路由）→ counter 不重置（直接標記）"""
@@ -451,7 +719,7 @@ class TestRedisPersistence:
 
     @pytest.mark.asyncio
     async def test_set_current_primary_persists_to_redis(self):
-        """set_current_primary("gemini") → Phase A 雙寫新舊 Redis key"""
+        """set_current_primary("gemini") atomically persists both Redis keys."""
         svc, _, _ = _make_service(current_primary="ollama")
         mock_redis = AsyncMock()
         mock_redis.set = AsyncMock()
@@ -459,12 +727,13 @@ class TestRedisPersistence:
         with patch("src.core.redis_client.get_redis", return_value=mock_redis):
             await svc.set_current_primary("gemini")
 
-        mock_redis.set.assert_has_awaits(
-            [
-                call("platform:ollama:current_primary", "gemini"),
-                call("ollama:current_primary", "gemini"),
-            ]
+        mock_redis.mset.assert_awaited_once_with(
+            {
+                "platform:ollama:current_primary": "gemini",
+                "ollama:current_primary": "gemini",
+            }
         )
+        mock_redis.set.assert_not_awaited()
 
     @pytest.mark.asyncio
     async def test_set_current_primary_same_value_no_redis_write(self):
@@ -552,6 +821,26 @@ class TestRedisPersistence:
             finally:
                 await svc.stop()
 
+    @pytest.mark.asyncio
+    async def test_start_immediate_check_when_primary_is_gcp_b_fallback(self):
+        svc, _, _ = _make_service(current_primary="ollama_gcp_a")
+        mock_redis = AsyncMock()
+        mock_redis.get = AsyncMock(return_value=b"ollama_gcp_b")
+
+        with patch.object(
+            svc,
+            "_check_and_recover",
+            new_callable=AsyncMock,
+        ) as immediate_check, patch(
+            "src.core.redis_client.get_redis",
+            return_value=mock_redis,
+        ):
+            try:
+                await svc.start()
+                immediate_check.assert_awaited_once()
+            finally:
+                await svc.stop()
+
     @pytest.mark.asyncio
     async def test_start_no_immediate_check_when_primary_is_ollama(self):
         """start() 時 primary=ollama（正常狀態）→ 不立刻執行 check"""

apps/api/tests/test_ollama_failover_manager.py

@@ -22,7 +22,8 @@ OllamaFailoverManager 單元測試 - P1.1c v4.0
 
 from __future__ import annotations
 
-from unittest.mock import AsyncMock, MagicMock, patch
+import asyncio
+from unittest.mock import ANY, AsyncMock, MagicMock, patch
 
 import pytest
 
@@ -383,7 +384,7 @@ class TestSelectProvider:
             local_status=HealthStatus.OFFLINE,
         )
         recovery_callback = AsyncMock()
-        manager._recovery_callback = recovery_callback
+        manager.set_recovery_callback(recovery_callback)
         audit = AsyncMock()
 
         with patch.object(manager, "_write_failover_audit", audit):
@@ -402,14 +403,156 @@ class TestSelectProvider:
             local_status=HealthStatus.OFFLINE,
         )
         recovery_callback = AsyncMock()
-        manager._recovery_callback = recovery_callback
+        manager.set_recovery_callback(recovery_callback)
         audit = AsyncMock()
 
         with patch.object(manager, "_write_failover_audit", audit):
             result = await manager.select_provider(emit_side_effects=True)
 
         audit.assert_awaited_once_with(result)
-        recovery_callback.assert_awaited_once_with("ollama_gcp_b")
+        recovery_callback.assert_awaited_once_with(
+            "ollama_gcp_b",
+            observation_generation=1,
+            observation_started_at_monotonic=ANY,
+        )
+
+    @pytest.mark.asyncio
+    async def test_legacy_one_argument_recovery_callback_remains_supported(self):
+        manager, _ = self._make_three_layer_mock(
+            gcp_a_status=HealthStatus.OFFLINE,
+            gcp_b_status=HealthStatus.HEALTHY,
+            local_status=HealthStatus.OFFLINE,
+        )
+        observed_providers: list[str] = []
+
+        async def legacy_callback(provider_name: str) -> None:
+            observed_providers.append(provider_name)
+
+        manager.set_recovery_callback(legacy_callback)
+
+        with patch.object(
+            manager,
+            "_write_failover_audit",
+            new_callable=AsyncMock,
+        ):
+            await manager.select_provider(emit_side_effects=True)
+
+        assert observed_providers == ["ollama_gcp_b"]
+
+    @pytest.mark.asyncio
+    async def test_recovery_state_callback_precedes_degraded_alert_audit(self):
+        manager, _ = self._make_three_layer_mock(
+            gcp_a_status=HealthStatus.OFFLINE,
+            gcp_b_status=HealthStatus.HEALTHY,
+            local_status=HealthStatus.OFFLINE,
+        )
+        event_order: list[str] = []
+
+        async def recovery_callback(provider_name: str, **metadata) -> None:
+            event_order.append(f"state:{provider_name}")
+
+        async def degraded_audit(result) -> None:
+            event_order.append(
+                f"alert:{result.observed_fallback.provider_name}"
+            )
+
+        manager.set_recovery_callback(recovery_callback)
+
+        with patch.object(
+            manager,
+            "_write_failover_audit",
+            side_effect=degraded_audit,
+        ):
+            await manager.select_provider(emit_side_effects=True)
+
+        assert event_order == [
+            "state:ollama_gcp_b",
+            "alert:ollama_gcp_b",
+        ]
+
+    @pytest.mark.asyncio
+    async def test_rejected_recovery_observation_suppresses_degraded_alert(self):
+        manager, _ = self._make_three_layer_mock(
+            gcp_a_status=HealthStatus.OFFLINE,
+            gcp_b_status=HealthStatus.HEALTHY,
+            local_status=HealthStatus.OFFLINE,
+        )
+
+        async def stale_callback(provider_name: str, **metadata) -> bool:
+            return False
+
+        manager.set_recovery_callback(stale_callback)
+        audit = AsyncMock()
+
+        with patch.object(manager, "_write_failover_audit", audit):
+            await manager.select_provider(emit_side_effects=True)
+
+        audit.assert_not_awaited()
+
+    @pytest.mark.asyncio
+    async def test_overtaken_selection_cannot_emit_stale_degraded_alert(self):
+        manager = _make_manager()
+        older_secondary_started = asyncio.Event()
+        release_older_secondary = asyncio.Event()
+        newer_selection_started = False
+
+        async def ordered_health_check(url: str) -> HealthReport:
+            if url == URL_GCP_A:
+                return _make_health(HealthStatus.OFFLINE, url)
+            if url == URL_GCP_B and not newer_selection_started:
+                older_secondary_started.set()
+                await release_older_secondary.wait()
+                return _make_health(HealthStatus.HEALTHY, url)
+            if url == URL_GCP_B:
+                return _make_health(HealthStatus.OFFLINE, url)
+            return _make_health(
+                HealthStatus.HEALTHY
+                if newer_selection_started
+                else HealthStatus.OFFLINE,
+                url,
+            )
+
+        manager._monitor.check = AsyncMock(side_effect=ordered_health_check)
+        last_accepted_generation = 0
+
+        async def generation_callback(
+            provider_name: str,
+            *,
+            observation_generation: int,
+            observation_started_at_monotonic: float,
+        ) -> bool:
+            nonlocal last_accepted_generation
+            if observation_generation <= last_accepted_generation:
+                return False
+            last_accepted_generation = observation_generation
+            return True
+
+        manager.set_recovery_callback(generation_callback)
+        alerted_providers: list[str] = []
+
+        async def capture_audit(result: OllamaRoutingResult) -> None:
+            alerted_providers.append(result.observed_fallback.provider_name)
+
+        with patch.object(
+            manager,
+            "_write_failover_audit",
+            side_effect=capture_audit,
+        ):
+            older_selection = asyncio.create_task(
+                manager.select_provider(emit_side_effects=True),
+                name="older-provider-selection",
+            )
+            await older_secondary_started.wait()
+            newer_selection_started = True
+            newer_selection = asyncio.create_task(
+                manager.select_provider(emit_side_effects=True),
+                name="newer-provider-selection",
+            )
+            await newer_selection
+            release_older_secondary.set()
+            await older_selection
+
+        assert alerted_providers == ["ollama_local"]
 
     @pytest.mark.asyncio
     async def test_select_provider_gcp_a_healthy_primary_ollama(self):