加入 Ollama GCP failover 診斷與 unhealthy skip

tests/test_ollama_retry_chain.py

@@ -440,6 +440,84 @@ def test_embedding_fallback_disabled_opens_short_gcp_failure_circuit():
     assert oss._embedding_gcp_failure_circuit['blocked_until'] > 0
 
 
+def test_embedding_health_label_maps_direct_and_proxy_gcp_hosts():
+    """host_health skip 要對齊 scheduler 寫入的 host_label。"""
+    from services import ollama_service as oss
+
+    assert oss._host_label_for_embedding_health("http://34.143.170.20:11434") == "Primary (GCP)"
+    assert oss._host_label_for_embedding_health("http://192.168.0.110:11435") == "Primary (GCP)"
+    assert oss._host_label_for_embedding_health("http://34.21.145.224:11434") == "Secondary (GCP)"
+    assert oss._host_label_for_embedding_health("http://192.168.0.110:11436") == "Secondary (GCP)"
+    assert oss._host_label_for_embedding_health("http://192.168.0.111:11434") == ""
+
+
+def test_recent_embedding_host_unhealthy_reads_fresh_host_health_probe(monkeypatch):
+    """最新 host_health_probes 若顯示 GCP embedding runtime unhealthy，背景 embedding 可先跳過。"""
+    from datetime import datetime
+    from services import ollama_service as oss
+
+    class FakeResult:
+        def fetchone(self):
+            return (False, "EmbedProbe ReadTimeout", datetime.now())
+
+    class FakeSession:
+        def execute(self, *args, **kwargs):
+            return FakeResult()
+
+        def close(self):
+            pass
+
+    monkeypatch.setenv("OLLAMA_EMBED_HOST_HEALTH_SKIP_ENABLED", "true")
+    monkeypatch.setenv("OLLAMA_EMBED_HOST_HEALTH_SKIP_WINDOW_MINUTES", "20")
+    monkeypatch.setattr("database.manager.get_session", lambda: FakeSession())
+
+    assert oss._recent_embedding_host_unhealthy(oss.OLLAMA_HOST_SECONDARY) is True
+
+
+def test_recent_embedding_host_unhealthy_fails_open_when_db_is_unavailable(monkeypatch):
+    """host health 查詢失敗不可阻斷 embedding；最多回到原本網路 retry。"""
+    from services import ollama_service as oss
+
+    monkeypatch.setattr(
+        "database.manager.get_session",
+        lambda: (_ for _ in ()).throw(RuntimeError("db down")),
+    )
+
+    assert oss._recent_embedding_host_unhealthy(oss.OLLAMA_HOST_SECONDARY) is False
+
+
+def test_embedding_fallback_disabled_skips_recent_unhealthy_gcp_hosts():
+    """背景 embedding 會直接跳過 host_health 最近標成 unhealthy 的 GCP，不打 111。"""
+    from services import ollama_service as oss
+    from services.ollama_service import OllamaService
+
+    svc = OllamaService()
+
+    def fake_recent_unhealthy(host):
+        return host in {oss.OLLAMA_HOST_PRIMARY, oss.OLLAMA_HOST_SECONDARY}
+
+    with patch('services.ollama_service.EMBED_GCP_FAILURE_COOLDOWN_SEC', 60), \
+         patch('services.ollama_service.resolve_ollama_host', side_effect=[
+             oss.OLLAMA_HOST_PRIMARY,
+             oss.OLLAMA_HOST_PRIMARY,
+             oss.OLLAMA_HOST_FALLBACK,
+         ]), \
+         patch('services.ollama_service._recent_embedding_host_unhealthy', side_effect=fake_recent_unhealthy), \
+         patch.dict('os.environ', {}, clear=False), \
+         patch('services.ollama_service.requests.post') as mock_post:
+        import os
+        os.environ.pop('EMBEDDING_HOST', None)
+        vec = svc.generate_embedding('test text', allow_111_fallback=False)
+
+    assert vec == []
+    mock_post.assert_not_called()
+    assert oss._embedding_gcp_failure_circuit['blocked_until'] > 0
+    assert oss._embedding_gcp_failure_circuit['tried'] == (
+        oss.OLLAMA_HOST_PRIMARY,
+        oss.OLLAMA_HOST_SECONDARY,
+    )
+
+
 def test_embedding_ignores_111_embedding_host_when_fallback_disabled():
     """EMBEDDING_HOST 若誤設 111，背景 embedding 仍回 GCP resolver，不直接棄跑。"""
     from services import ollama_service as oss