fix: defer embedding work during gcp backoff
All checks were successful
CD Pipeline / deploy (push) Successful in 1m10s

This commit is contained in:
OoO
2026-06-18 12:06:24 +08:00
parent 56cd883148
commit 9bafa73ffc
7 changed files with 145 additions and 13 deletions

View File

@@ -100,6 +100,62 @@ def test_process_one_embedding_writes_signature(monkeypatch):
assert target_updates[0][1]["sig"] == get_embedding_signature(model="bge-m3:latest", dim=1024)
def test_process_one_embedding_defers_without_attempt_when_gcp_circuit_open(monkeypatch):
import services.openclaw_learning_service as learning
executed = []
class Session:
def execute(self, stmt, params=None):
executed.append((str(stmt), params or {}))
def commit(self):
pass
def rollback(self):
pass
def close(self):
pass
monkeypatch.setattr(learning, "get_session", lambda: Session())
monkeypatch.setattr(
learning.ollama_service,
"generate_embedding",
lambda text, model="bge-m3:latest", **_kwargs: [],
)
monkeypatch.setattr(learning, "is_embedding_gcp_circuit_open", lambda: True)
monkeypatch.setattr(learning, "embedding_gcp_circuit_remaining_seconds", lambda: 42.0)
ok = learning._process_one_embedding(
row_id=7,
target_table="learning_episodes",
target_id=42,
text_content="測試內容",
model="bge-m3:latest",
)
assert ok is False
assert not any("attempts = attempts + 1" in stmt for stmt, _ in executed)
defer_updates = [item for item in executed if "SET status='pending'" in item[0]]
assert defer_updates
assert "不扣 attempts" in defer_updates[0][1]["err"]
def test_claim_pending_embeddings_pauses_when_gcp_circuit_open(monkeypatch):
import services.openclaw_learning_service as learning
monkeypatch.setattr(learning, "is_embedding_gcp_circuit_open", lambda: True)
monkeypatch.setattr(learning, "embedding_gcp_circuit_remaining_seconds", lambda: 30.0)
monkeypatch.setattr(
learning,
"get_session",
lambda: (_ for _ in ()).throw(AssertionError("熔斷中不應 claim DB rows")),
)
assert learning._claim_pending_embeddings(limit=3, max_attempts=5) == []
def test_claim_pending_embeddings_uses_skip_locked(monkeypatch):
import services.openclaw_learning_service as learning

View File

@@ -482,6 +482,8 @@ def test_embedding_fallback_disabled_opens_short_gcp_failure_circuit():
assert second == []
assert posted_hosts == [oss.OLLAMA_HOST_PRIMARY, oss.OLLAMA_HOST_SECONDARY]
assert oss._embedding_gcp_failure_circuit['blocked_until'] > 0
assert oss.is_embedding_gcp_circuit_open() is True
assert oss.embedding_gcp_circuit_remaining_seconds() > 0
def test_embedding_health_label_maps_direct_and_proxy_gcp_hosts():