feat(ai): add bounded model-aware decision fallback

tests/test_nemotron_decision_canary_service.py

@@ -3,6 +3,8 @@ import json
 import subprocess
 import sys
 
+import pytest
+
 
 class _Response:
     def __init__(self, payload, status_code=200):
@@ -19,12 +21,18 @@ class _Response:
         return self._payload
 
 
-def _tags(service, *, digest=None):
+def _tags(service, *, model=None, digest=None):
+    resolved_model = model or service.NEMOTRON_OLLAMA_MODEL
+    resolved_digest = digest or (
+        service.NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST
+        if resolved_model == service.NEMOTRON_OLLAMA_FALLBACK_MODEL
+        else service.NEMOTRON_OLLAMA_EXPECTED_DIGEST
+    )
     return {
         "models": [{
-            "name": service.NEMOTRON_OLLAMA_MODEL,
-            "model": service.NEMOTRON_OLLAMA_MODEL,
-            "digest": digest or service.NEMOTRON_OLLAMA_EXPECTED_DIGEST,
+            "name": resolved_model,
+            "model": resolved_model,
+            "digest": resolved_digest,
             "details": {
                 "parameter_size": "14.8B",
                 "quantization_level": "Q4_K_M",
@@ -33,6 +41,22 @@ def _tags(service, *, digest=None):
     }
 
 
+def _valid_canary_tool_call():
+    return {
+        "function": {
+            "name": "trigger_price_alert",
+            "arguments": {
+                "sku": "CANARY-NEMO-001",
+                "name": "NemoTron 受控決策驗證品",
+                "gap_pct": 22.4,
+                "sales_delta": -35.0,
+                "action": "建立價格告警候選",
+                "confidence": 0.91,
+            },
+        }
+    }
+
+
 def test_nemotron_canary_readback_never_calls_network(tmp_path, monkeypatch):
     from services import nemotron_decision_canary_service as service
 
@@ -113,7 +137,8 @@ def test_nemotron_canary_executes_decision_only_on_verified_secondary(
     assert posted[0][0].endswith("/api/chat")
     assert posted[0][1]["keep_alive"] == 0
     assert posted[0][1]["think"] is False
-    assert posted[0][2] == 45
+    assert posted[0][1]["options"]["num_predict"] == 2048
+    assert posted[0][2] == pytest.approx(45, abs=0.01)
     assert payload["latest_execution"]["canary_passed"] is True
 
     acknowledged = service.acknowledge_nemotron_decision_canary(
@@ -157,12 +182,263 @@ def test_nemotron_canary_blocks_model_digest_drift_before_model_call(
 
     assert payload["status"] == "canary_failed"
     assert payload["execution"]["model_call_performed"] is False
-    assert payload["execution"]["error"] == (
-        "no_approved_gcp_host_with_expected_nemotron_digest"
+    assert payload["execution"]["error"] == "all_approved_model_candidates_failed"
+    assert payload["execution"]["attempt_count"] == 3
+    assert all(
+        attempt["status"] == "identity_failed"
+        for attempt in payload["execution"]["runtime_attempts"]
     )
     assert model_calls == []
 
 
+def test_nemotron_canary_malformed_primary_tags_falls_through_secondary(
+    tmp_path, monkeypatch
+):
+    from services import nemotron_decision_canary_service as service
+
+    def fake_get(url, timeout):
+        if service.OLLAMA_HOST_PRIMARY in url:
+            return _Response({
+                "models": [{
+                    "name": service.NEMOTRON_OLLAMA_MODEL,
+                    "model": service.NEMOTRON_OLLAMA_MODEL,
+                    "digest": service.NEMOTRON_OLLAMA_EXPECTED_DIGEST,
+                    "details": [],
+                }]
+            })
+        return _Response(_tags(service))
+
+    monkeypatch.setattr(service.requests, "get", fake_get)
+    monkeypatch.setattr(
+        service.requests,
+        "post",
+        lambda *_args, **_kwargs: _Response({
+            "message": {
+                "role": "assistant",
+                "content": "",
+                "tool_calls": [_valid_canary_tool_call()],
+            }
+        }),
+    )
+
+    payload = service.run_nemotron_decision_canary(
+        output_root=tmp_path,
+        execute=True,
+        write_receipt=False,
+        timeout_sec=45,
+    )
+
+    execution = payload["execution"]
+    assert payload["status"] == "canary_passed"
+    assert execution["selected_host"] == service.OLLAMA_HOST_SECONDARY
+    assert execution["runtime_attempts"][0]["status"] == "identity_failed"
+    assert execution["runtime_attempts"][0]["error"] == "malformed_model_details"
+
+
+def test_nemotron_canary_uses_exact_digest_111_fallback_after_gcp_timeout(
+    tmp_path, monkeypatch
+):
+    import requests
+    from services import nemotron_decision_canary_service as service
+    from services.ollama_service import OLLAMA_HOST_FALLBACK
+
+    def fake_get(url, timeout):
+        assert timeout == service.DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC
+        if service.OLLAMA_HOST_PRIMARY in url:
+            raise requests.ConnectionError("primary unavailable")
+        if service.OLLAMA_HOST_SECONDARY in url:
+            return _Response(_tags(service))
+        if OLLAMA_HOST_FALLBACK in url:
+            return _Response(
+                _tags(service, model=service.NEMOTRON_OLLAMA_FALLBACK_MODEL)
+            )
+        raise AssertionError(f"unexpected identity URL: {url}")
+
+    posted = []
+
+    def fake_post(url, json, timeout):
+        posted.append((url, json, timeout))
+        if service.OLLAMA_HOST_SECONDARY in url:
+            raise requests.Timeout("secondary saturated")
+        return _Response({
+            "message": {
+                "role": "assistant",
+                "content": "",
+                "tool_calls": [{
+                    "function": {
+                        "name": "trigger_price_alert",
+                        "arguments": {
+                            "sku": "CANARY-NEMO-001",
+                            "name": "NemoTron 受控決策驗證品",
+                            "gap_pct": 22.4,
+                            "sales_delta": -35.0,
+                            "action": "建立價格告警候選",
+                            "confidence": 0.91,
+                        },
+                    }
+                }],
+            },
+            "prompt_eval_count": 100,
+            "eval_count": 20,
+        })
+
+    monkeypatch.setattr(service.requests, "get", fake_get)
+    monkeypatch.setattr(service.requests, "post", fake_post)
+
+    payload = service.run_nemotron_decision_canary(
+        output_root=tmp_path,
+        execute=True,
+        write_receipt=True,
+        timeout_sec=90,
+        run_id="nemotron-canary-fallback-test-001",
+    )
+
+    assert payload["status"] == "canary_passed_degraded_fallback"
+    execution = payload["execution"]
+    assert execution["selected_host"] == OLLAMA_HOST_FALLBACK
+    assert execution["model"] == service.NEMOTRON_OLLAMA_FALLBACK_MODEL
+    assert execution["fallback_used"] is True
+    assert execution["degraded_runtime"] is True
+    assert execution["selected_candidate_label"] == "ollama_111_fallback"
+    assert execution["attempt_count"] == 3
+    assert posted[0][2] == 60
+    assert posted[1][2] == 45
+    assert posted[1][1]["model"] == service.NEMOTRON_OLLAMA_FALLBACK_MODEL
+    assert posted[1][1]["options"]["num_ctx"] == 4096
+    assert posted[1][1]["options"]["num_predict"] == 512
+    assert payload["next_machine_action"] == "restore_gcp_decision_runtime_capacity"
+    assert payload["latest_execution"]["fallback_used"] is True
+
+
+def test_nemotron_canary_timeout_is_one_total_deadline(tmp_path, monkeypatch):
+    import requests
+    from services import nemotron_decision_canary_service as service
+
+    clock = {"now": 50.0}
+    monkeypatch.setattr(service.time, "monotonic", lambda: clock["now"])
+    monkeypatch.setattr(
+        service,
+        "inspect_nemotron_model_identity",
+        lambda candidate, **_kwargs: {
+            **candidate.as_public_dict(),
+            "ok": True,
+            "digest": candidate.expected_digest,
+            "digest_matches": True,
+            "elapsed_ms": 0,
+            "error": None,
+        },
+    )
+    posted = []
+
+    def timeout_post(url, json, timeout):
+        posted.append((url, timeout))
+        clock["now"] += float(timeout)
+        raise requests.Timeout("simulated timeout")
+
+    monkeypatch.setattr(service.requests, "post", timeout_post)
+
+    payload = service.run_nemotron_decision_canary(
+        output_root=tmp_path,
+        execute=True,
+        write_receipt=False,
+        timeout_sec=30,
+    )
+
+    execution = payload["execution"]
+    assert payload["status"] == "canary_failed"
+    assert execution["error"] == "decision_total_deadline_exhausted"
+    assert execution["deadline_exhausted"] is True
+    assert execution["total_elapsed_ms"] == 30000
+    assert len(posted) == 1
+    assert posted[0][1] == 30
+    assert execution["attempt_count"] == 2
+
+
+@pytest.mark.parametrize(
+    ("tool_calls", "error_fragment"),
+    [
+        (
+            [
+                _valid_canary_tool_call(),
+                {
+                    "function": {
+                        "name": "delete_product",
+                        "arguments": {"sku": "CANARY-NEMO-001"},
+                    }
+                },
+            ],
+            "tool_call_count_mismatch",
+        ),
+        (
+            [_valid_canary_tool_call(), _valid_canary_tool_call()],
+            "tool_call_count_mismatch",
+        ),
+        (
+            [{
+                "function": {
+                    "name": "delete_product",
+                    "arguments": {"sku": "CANARY-NEMO-001"},
+                }
+            }],
+            "unknown_tool",
+        ),
+    ],
+)
+def test_nemotron_canary_rejects_extra_duplicate_or_unknown_tool_calls(
+    tmp_path, monkeypatch, tool_calls, error_fragment
+):
+    from services import nemotron_decision_canary_service as service
+    from services.ollama_service import OLLAMA_HOST_FALLBACK
+
+    def fake_get(url, timeout):
+        model = (
+            service.NEMOTRON_OLLAMA_FALLBACK_MODEL
+            if OLLAMA_HOST_FALLBACK in url
+            else service.NEMOTRON_OLLAMA_MODEL
+        )
+        return _Response(_tags(service, model=model))
+
+    monkeypatch.setattr(service.requests, "get", fake_get)
+    monkeypatch.setattr(
+        service.requests,
+        "post",
+        lambda *_args, **_kwargs: _Response({
+            "message": {
+                "role": "assistant",
+                "content": "",
+                "tool_calls": tool_calls,
+            }
+        }),
+    )
+
+    payload = service.run_nemotron_decision_canary(
+        output_root=tmp_path,
+        execute=True,
+        write_receipt=False,
+        timeout_sec=90,
+    )
+
+    execution = payload["execution"]
+    assert payload["status"] == "canary_failed"
+    assert execution["canary_passed"] is False
+    assert execution["tool_execution_count"] == 0
+    assert execution["database_call_performed"] is False
+    model_attempts = [
+        attempt
+        for attempt in execution["runtime_attempts"]
+        if attempt["model_call_performed"]
+    ]
+    assert len(model_attempts) == 3
+    assert all(
+        attempt["checks"]["decision_contract_valid"] is False
+        for attempt in model_attempts
+    )
+    assert all(
+        error_fragment in str(attempt.get("contract_error") or "")
+        for attempt in model_attempts
+    )
+
+
 def test_nemotron_canary_route_is_read_only():
     from routes import system_public_routes as routes
 

tests/test_nemotron_decision_envelope.py

@@ -62,7 +62,10 @@ def test_send_telegram_attaches_decision_envelope_to_event_router(monkeypatch):
         "severity": "P2",
         "guardrails": {"can_auto_execute": False},
     }
-    module.NemotronDispatcher()._send_telegram("hello price alert", decision_envelope=envelope)
+    outcome = module.NemotronDispatcher()._send_telegram(
+        "hello price alert",
+        decision_envelope=envelope,
+    )
 
     assert len(captured) == 1
     event = captured[0]
@@ -71,6 +74,25 @@ def test_send_telegram_attaches_decision_envelope_to_event_router(monkeypatch):
     assert event["decision_envelope"] == envelope
     assert event["payload"]["decision_envelope"] == envelope
     assert event["payload"]["raw_message"] == "hello price alert"
+    assert outcome["durable"] is False
+
+
+def test_send_telegram_requires_delivery_or_durable_queue(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    fake_event_router = types.ModuleType("services.event_router")
+    fake_event_router.dispatch_sync = lambda event: {
+        "delivered": False,
+        "queued": False,
+        "errors": ["telegram unavailable"],
+    }
+    monkeypatch.setitem(sys.modules, "services.event_router", fake_event_router)
+
+    outcome = module.NemotronDispatcher()._send_telegram("test")
+
+    assert outcome["durable"] is False
+    assert outcome["delivered"] is False
+    assert outcome["queued"] is False
 
 
 def test_exec_trigger_price_alert_persists_decision_envelope(monkeypatch):
@@ -82,15 +104,18 @@ def test_exec_trigger_price_alert_persists_decision_envelope(monkeypatch):
     monkeypatch.setattr(
         dispatcher,
         "_send_telegram",
-        lambda message, decision_envelope=None: sent.append((message, decision_envelope)),
+        lambda message, decision_envelope=None: (
+            sent.append((message, decision_envelope))
+            or {"durable": True, "delivered": True, "queued": False}
+        ),
     )
     monkeypatch.setattr(
         dispatcher,
         "_sink_insight_to_km",
-        lambda *args, **kwargs: stored.append((args, kwargs)),
+        lambda *args, **kwargs: (stored.append((args, kwargs)) or True),
     )
 
-    dispatcher._exec_trigger_price_alert(
+    outcome = dispatcher._exec_trigger_price_alert(
         sku="SKU-1",
         name="測試商品",
         gap_pct=12.5,
@@ -115,3 +140,4 @@ def test_exec_trigger_price_alert_persists_decision_envelope(monkeypatch):
     assert envelope["subject"]["competitor_product_id"] == "PC-1"
     assert stored
     assert stored[0][1]["metadata"]["decision_envelope"] == envelope
+    assert outcome["durable"] is True

tests/test_nemotron_dispatch_reservation_service.py

@@ -0,0 +1,281 @@
+import json
+import multiprocessing
+from pathlib import Path
+import subprocess
+import sys
+
+
+def _process_reserve(state_path, start_event, result_queue):
+    from services.nemotron_dispatch_reservation_service import (
+        SharedFileReservationStore,
+    )
+
+    store = SharedFileReservationStore(
+        state_path,
+        committed_ttl_sec=3600,
+        lease_ttl_sec=60,
+    )
+    start_event.wait(timeout=5)
+    result_queue.put(store.reserve("SKU-PROCESS-SHARED"))
+
+
+def test_shared_store_coordinates_independent_instances_and_ttl(tmp_path):
+    from services.nemotron_dispatch_reservation_service import (
+        SharedFileReservationStore,
+    )
+
+    clock = {"now": 1000.0}
+    path = tmp_path / "dedupe" / "state.json"
+    first = SharedFileReservationStore(
+        path,
+        committed_ttl_sec=100,
+        lease_ttl_sec=20,
+        clock=lambda: clock["now"],
+    )
+    second = SharedFileReservationStore(
+        path,
+        committed_ttl_sec=100,
+        lease_ttl_sec=20,
+        clock=lambda: clock["now"],
+    )
+
+    token = first.reserve("SKU-SHARED")
+    assert token
+    assert second.reserve("SKU-SHARED") is None
+    assert second.is_duplicate("SKU-SHARED") is True
+    assert "SKU-SHARED" not in path.read_text(encoding="utf-8")
+    assert second.release("SKU-SHARED", "stale-token") is False
+    assert first.refresh("SKU-SHARED", token) is True
+    assert first.commit("SKU-SHARED", token) is True
+    assert second.reserve("SKU-SHARED") is None
+
+    clock["now"] += 101
+    next_token = second.reserve("SKU-SHARED")
+    assert next_token and next_token != token
+    assert second.release("SKU-SHARED", next_token) is True
+
+
+def test_begin_side_effect_persists_extended_quarantine(tmp_path):
+    from services.nemotron_dispatch_reservation_service import (
+        SharedFileReservationStore,
+    )
+
+    clock = {"now": 1000.0}
+    path = tmp_path / "state.json"
+    first = SharedFileReservationStore(
+        path,
+        committed_ttl_sec=100,
+        lease_ttl_sec=20,
+        clock=lambda: clock["now"],
+    )
+    second = SharedFileReservationStore(
+        path,
+        committed_ttl_sec=100,
+        lease_ttl_sec=20,
+        clock=lambda: clock["now"],
+    )
+
+    token = first.reserve("SKU-SIDE-EFFECT")
+    assert token
+    assert first.begin_side_effect("SKU-SIDE-EFFECT", token) is True
+
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    lease = next(iter(payload["leases"].values()))
+    assert lease == {
+        "token": token,
+        "expires_at": 1100.0,
+        "phase": "side_effect_started",
+    }
+
+    clock["now"] += 21
+    assert second.reserve("SKU-SIDE-EFFECT") is None
+    clock["now"] += 80
+    assert second.reserve("SKU-SIDE-EFFECT")
+
+
+def test_shared_store_is_atomic_across_processes(tmp_path):
+    state_path = str(tmp_path / "process" / "state.json")
+    context = multiprocessing.get_context("fork")
+    start_event = context.Event()
+    result_queue = context.Queue()
+    processes = [
+        context.Process(
+            target=_process_reserve,
+            args=(state_path, start_event, result_queue),
+        )
+        for _index in range(6)
+    ]
+    for process in processes:
+        process.start()
+    start_event.set()
+    tokens = [result_queue.get(timeout=10) for _process in processes]
+    for process in processes:
+        process.join(timeout=10)
+        assert process.exitcode == 0
+
+    owners = [token for token in tokens if token]
+    assert len(owners) == 1
+
+
+def test_shared_store_corruption_fails_closed(tmp_path):
+    from services.nemotron_dispatch_reservation_service import (
+        SharedFileReservationStore,
+    )
+
+    path = tmp_path / "state.json"
+    path.write_text("{not-json", encoding="utf-8")
+    store = SharedFileReservationStore(
+        path,
+        committed_ttl_sec=3600,
+        lease_ttl_sec=900,
+    )
+
+    assert store.reserve("SKU-CORRUPT") is None
+    assert store.refresh("SKU-CORRUPT", "token") is False
+    assert store.commit("SKU-CORRUPT", "token") is False
+    assert store.release("SKU-CORRUPT", "token") is False
+    assert store.is_duplicate("SKU-CORRUPT") is True
+    assert path.read_text(encoding="utf-8") == "{not-json"
+
+
+def test_atomic_write_fsyncs_file_and_parent_directory(tmp_path, monkeypatch):
+    from services import nemotron_dispatch_reservation_service as service
+
+    fsynced = []
+    real_fsync = service.os.fsync
+
+    def record_fsync(fd):
+        fsynced.append(fd)
+        return real_fsync(fd)
+
+    monkeypatch.setattr(service.os, "fsync", record_fsync)
+    store = service.SharedFileReservationStore(
+        tmp_path / "nested" / "state.json",
+        committed_ttl_sec=3600,
+        lease_ttl_sec=900,
+    )
+
+    assert store.reserve("SKU-FSYNC")
+    assert len(fsynced) >= 2
+
+
+def test_missing_process_shared_lock_fails_closed(monkeypatch, tmp_path):
+    from services import nemotron_dispatch_reservation_service as service
+
+    monkeypatch.setattr(service, "fcntl", None)
+    monkeypatch.delenv("FLASK_ENV", raising=False)
+    monkeypatch.delenv("USE_POSTGRESQL", raising=False)
+
+    assert service.build_production_shared_reservation_store(
+        committed_ttl_sec=3600,
+        lease_ttl_sec=900,
+    ) is None
+
+    store = service.SharedFileReservationStore(
+        tmp_path / "state.json",
+        committed_ttl_sec=3600,
+        lease_ttl_sec=900,
+    )
+    assert store.reserve("SKU-NO-FCNTL") is None
+    assert store.is_duplicate("SKU-NO-FCNTL") is True
+
+
+def test_production_builder_forces_canonical_shared_data_path(monkeypatch):
+    from services import nemotron_dispatch_reservation_service as service
+
+    monkeypatch.setenv("FLASK_ENV", "production")
+    store = service.build_production_shared_reservation_store(
+        committed_ttl_sec=3600,
+        lease_ttl_sec=900,
+    )
+
+    assert store is not None
+    assert store.state_path == Path(
+        "/app/data/ai_automation/nemotron_dispatch_dedupe_state.json"
+    )
+    assert store.lock_path == Path(
+        "/app/data/ai_automation/nemotron_dispatch_dedupe_state.json.lock"
+    )
+
+
+def test_postgres_runtime_cannot_fall_back_to_process_local_memory(monkeypatch):
+    from services import nemotron_dispatch_reservation_service as service
+
+    monkeypatch.delenv("FLASK_ENV", raising=False)
+    monkeypatch.setenv("USE_POSTGRESQL", "true")
+
+    assert service.production_shared_reservation_required() is True
+    assert service.build_production_shared_reservation_store(
+        committed_ttl_sec=3600,
+        lease_ttl_sec=900,
+    ) is not None
+
+
+def test_shared_state_contains_only_hashed_sku_and_timing_metadata(tmp_path):
+    from services.nemotron_dispatch_reservation_service import (
+        SharedFileReservationStore,
+    )
+
+    path = tmp_path / "state.json"
+    store = SharedFileReservationStore(
+        path,
+        committed_ttl_sec=3600,
+        lease_ttl_sec=900,
+    )
+    token = store.reserve("RAW-SKU-MUST-NOT-APPEAR")
+    payload = json.loads(path.read_text(encoding="utf-8"))
+
+    assert token
+    assert "RAW-SKU-MUST-NOT-APPEAR" not in path.read_text(encoding="utf-8")
+    assert set(payload) == {"version", "leases", "committed"}
+    assert len(next(iter(payload["leases"]))) == 64
+
+
+def test_production_containers_share_data_mount_and_force_shared_backend():
+    compose = (
+        Path(__file__).resolve().parents[1] / "docker-compose.yml"
+    ).read_text(encoding="utf-8")
+
+    assert compose.count("- ./data:/app/data") >= 3
+    assert compose.count("- FLASK_ENV=production") >= 3
+
+
+def test_shared_reservation_canary_has_no_business_side_effects(tmp_path):
+    from services.nemotron_dispatch_reservation_service import (
+        run_shared_reservation_canary,
+    )
+
+    payload = run_shared_reservation_canary(
+        state_path=tmp_path / "canary" / "state.json",
+    )
+
+    assert payload["success"] is True
+    assert payload["check_pass_count"] == payload["check_count"] == 6
+    assert payload["controlled_apply"] == {
+        "writes_shared_reservation_state": True,
+        "terminal_state_clear": True,
+        "writes_database": False,
+        "sends_telegram": False,
+        "executes_business_tool": False,
+        "reads_secret": False,
+    }
+
+
+def test_shared_reservation_canary_cli_is_machine_readable(tmp_path):
+    completed = subprocess.run(
+        [
+            sys.executable,
+            "scripts/ops/run_nemotron_dispatch_reservation_canary.py",
+            "--state-path",
+            str(tmp_path / "cli" / "state.json"),
+        ],
+        cwd=Path(__file__).resolve().parents[1],
+        capture_output=True,
+        check=False,
+        text=True,
+    )
+
+    assert completed.returncode == 0, completed.stderr
+    payload = json.loads(completed.stdout)
+    assert payload["status"] == "shared_reservation_canary_passed"
+    assert payload["controlled_apply"]["writes_database"] is False

tests/test_nemotron_fallback.py

@@ -9,9 +9,11 @@ def _force_legacy_nim_first(monkeypatch):
     import services.nemoton_dispatcher_service as module
 
     module._ALERT_CACHE.clear()
+    module._ALERT_INFLIGHT.clear()
     monkeypatch.setattr(module, "NEMOTRON_OLLAMA_FIRST", False)
     yield
     module._ALERT_CACHE.clear()
+    module._ALERT_INFLIGHT.clear()
 
 
 @dataclass
@@ -33,6 +35,7 @@ def _patch_execution_methods(monkeypatch, dispatcher):
     def record(kind):
         def _inner(*args, **kwargs):
             calls.append({"kind": kind, "args": args, "kwargs": kwargs})
+            return {"durable": True}
         return _inner
 
     monkeypatch.setattr(dispatcher, "_exec_trigger_price_alert", record("price_alert"))
@@ -56,6 +59,207 @@ def test_dispatch_falls_back_to_hermes_rules_without_nim_api_key(monkeypatch):
     assert calls[0]["kind"] == "price_alert"
 
 
+def test_forced_review_failure_is_counted_as_skipped(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    monkeypatch.setattr(module, "NIM_API_KEY", "")
+    dispatcher = module.NemotronDispatcher()
+    monkeypatch.setattr(
+        dispatcher,
+        "_exec_flag_for_human_review",
+        lambda *_args, **_kwargs: {"durable": False},
+    )
+    threat = FakeThreat("SKU-FORCED", "斷崖品")
+    threat.sales_7d_delta_pct = -100.0
+
+    result = dispatcher.dispatch([threat], hermes_stats={"duration_sec": 1})
+
+    assert result["dispatched"] == 0
+    assert result["skipped"] == 1
+    assert len(result["errors"]) == 1
+    assert "durable_side_effect_not_confirmed" in result["errors"][0]
+
+
+def test_hermes_failure_is_counted_as_skipped(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    monkeypatch.setattr(module, "NIM_API_KEY", "")
+    dispatcher = module.NemotronDispatcher()
+    monkeypatch.setattr(
+        dispatcher,
+        "_exec_trigger_price_alert",
+        lambda *_args, **_kwargs: {"durable": False},
+    )
+
+    result = dispatcher.dispatch(
+        [FakeThreat("SKU-HERMES-FAIL", "告警品")],
+        hermes_stats={"duration_sec": 1},
+    )
+
+    assert result["dispatched"] == 0
+    assert result["skipped"] == 1
+    assert len(result["errors"]) == 1
+    assert "durable_side_effect_not_confirmed" in result["errors"][0]
+
+
+def test_forced_review_commit_failure_keeps_durable_result_quarantined(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    monkeypatch.setattr(module, "NIM_API_KEY", "")
+    dispatcher = module.NemotronDispatcher()
+    monkeypatch.setattr(
+        dispatcher,
+        "_exec_flag_for_human_review",
+        lambda *_args, **_kwargs: {"durable": True},
+    )
+    monkeypatch.setattr(module, "_mark_alert_dispatched", lambda *_args: False)
+    released = []
+    monkeypatch.setattr(
+        module,
+        "_release_alert_reservation",
+        lambda *args: (released.append(args) or True),
+    )
+    threat = FakeThreat("SKU-FORCED-COMMIT", "斷崖品")
+    threat.sales_7d_delta_pct = -100.0
+
+    result = dispatcher.dispatch([threat], hermes_stats={"duration_sec": 1})
+
+    assert result["dispatched"] == 1
+    assert result["skipped"] == 0
+    assert result["errors"] == [
+        "dedupe_commit_unverified:flag_for_human_review(SKU-FORCED-COMMIT)"
+    ]
+    assert released == []
+
+
+def test_hermes_commit_failure_keeps_durable_result_quarantined(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    monkeypatch.setattr(module, "NIM_API_KEY", "")
+    dispatcher = module.NemotronDispatcher()
+    _patch_execution_methods(monkeypatch, dispatcher)
+    monkeypatch.setattr(module, "_mark_alert_dispatched", lambda *_args: False)
+    released = []
+    monkeypatch.setattr(
+        module,
+        "_release_alert_reservation",
+        lambda *args: (released.append(args) or True),
+    )
+
+    result = dispatcher.dispatch(
+        [FakeThreat("SKU-HERMES-COMMIT", "告警品")],
+        hermes_stats={"duration_sec": 1},
+    )
+
+    assert result["dispatched"] == 1
+    assert result["skipped"] == 0
+    assert result["errors"] == [
+        "dedupe_commit_unverified:hermes_rule_fallback(SKU-HERMES-COMMIT)"
+    ]
+    assert released == []
+
+
+def test_mixed_batch_cleanup_never_releases_forced_durable_quarantine(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    monkeypatch.setattr(module, "NIM_API_KEY", "test-key")
+    monkeypatch.setattr(module, "_check_nim_quota", lambda: True)
+    dispatcher = module.NemotronDispatcher()
+    _patch_execution_methods(monkeypatch, dispatcher)
+    forced = FakeThreat("SKU-FORCED-MIXED", "斷崖品")
+    forced.sales_7d_delta_pct = -100.0
+    candidate = FakeThreat("SKU-NIM-MIXED", "模型候選")
+    monkeypatch.setattr(
+        dispatcher,
+        "_call_nim",
+        lambda _threats, **_kwargs: (
+            [{
+                "tool": "trigger_price_alert",
+                "args": {
+                    "sku": candidate.sku,
+                    "name": candidate.name,
+                    "gap_pct": candidate.gap_pct,
+                    "sales_delta": candidate.sales_7d_delta_pct,
+                    "action": candidate.recommended_action,
+                    "confidence": candidate.confidence,
+                },
+            }],
+            {"provider": "nim", "total_tokens": 1},
+        ),
+    )
+    real_mark = module._mark_alert_dispatched
+    monkeypatch.setattr(
+        module,
+        "_mark_alert_dispatched",
+        lambda sku, token: (
+            False if str(sku) == forced.sku else real_mark(sku, token)
+        ),
+    )
+
+    result = dispatcher.dispatch(
+        [forced, candidate],
+        hermes_stats={"duration_sec": 1},
+    )
+
+    assert result["dispatched"] == 2
+    assert result["skipped"] == 0
+    assert result["errors"] == [
+        "dedupe_commit_unverified:flag_for_human_review(SKU-FORCED-MIXED)"
+    ]
+    assert module._is_duplicate_alert(forced.sku) is True
+    assert module._is_duplicate_alert(candidate.sku) is True
+
+
+def test_mixed_batch_counts_failed_model_candidate_as_skipped(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    monkeypatch.setattr(module, "NIM_API_KEY", "test-key")
+    monkeypatch.setattr(module, "_check_nim_quota", lambda: True)
+    dispatcher = module.NemotronDispatcher()
+    forced = FakeThreat("SKU-FORCED-COUNT", "斷崖品")
+    forced.sales_7d_delta_pct = -100.0
+    candidate = FakeThreat("SKU-NIM-COUNT", "模型候選")
+    monkeypatch.setattr(
+        dispatcher,
+        "_exec_flag_for_human_review",
+        lambda *_args, **_kwargs: {"durable": True},
+    )
+    monkeypatch.setattr(
+        dispatcher,
+        "_exec_trigger_price_alert",
+        lambda *_args, **_kwargs: {"durable": False},
+    )
+    monkeypatch.setattr(
+        dispatcher,
+        "_call_nim",
+        lambda _threats, **_kwargs: (
+            [{
+                "tool": "trigger_price_alert",
+                "args": {
+                    "sku": candidate.sku,
+                    "name": candidate.name,
+                    "gap_pct": candidate.gap_pct,
+                    "sales_delta": candidate.sales_7d_delta_pct,
+                    "action": candidate.recommended_action,
+                    "confidence": candidate.confidence,
+                },
+            }],
+            {"provider": "nim", "total_tokens": 1},
+        ),
+    )
+
+    result = dispatcher.dispatch(
+        [forced, candidate],
+        hermes_stats={"duration_sec": 1},
+    )
+
+    assert result["dispatched"] == 1
+    assert result["skipped"] == 1
+    assert len(result["errors"]) == 1
+    assert "durable_side_effect_not_confirmed" in result["errors"][0]
+    assert result["dispatched"] + result["skipped"] == 2
+
+
 def test_dispatch_routes_non_exact_match_to_human_review(monkeypatch):
     import services.nemoton_dispatcher_service as module
 
@@ -83,7 +287,7 @@ def test_dispatch_falls_back_to_hermes_rules_on_nim_timeout(monkeypatch):
     monkeypatch.setattr(module, "_check_nim_quota", lambda: True)
     dispatcher = module.NemotronDispatcher()
     calls = _patch_execution_methods(monkeypatch, dispatcher)
-    monkeypatch.setattr(dispatcher, "_call_nim", lambda threats: (_ for _ in ()).throw(requests.Timeout("timeout")))
+    monkeypatch.setattr(dispatcher, "_call_nim", lambda threats, **_kwargs: (_ for _ in ()).throw(requests.Timeout("timeout")))
 
     result = dispatcher.dispatch([FakeThreat("SKU-2", "測試品")], hermes_stats={"duration_sec": 1})
 
@@ -101,7 +305,7 @@ def test_dispatch_falls_back_to_hermes_rules_on_zero_tool_calls(monkeypatch):
     monkeypatch.setattr(module, "_check_nim_quota", lambda: True)
     dispatcher = module.NemotronDispatcher()
     calls = _patch_execution_methods(monkeypatch, dispatcher)
-    monkeypatch.setattr(dispatcher, "_call_nim", lambda threats: ([], {"total_tokens": 10}))
+    monkeypatch.setattr(dispatcher, "_call_nim", lambda threats, **_kwargs: ([], {"total_tokens": 10}))
 
     result = dispatcher.dispatch([FakeThreat("SKU-3", "測試品")], hermes_stats={"duration_sec": 1})
 

tests/test_nemotron_qwen3_compat.py

@@ -39,6 +39,20 @@ class FakeThreat:
     sales_7d_prev_amount: float = 120000.0
 
 
+def _valid_price_alert_call(sku="SKU-Q1"):
+    return {
+        "tool": "trigger_price_alert",
+        "args": {
+            "sku": sku,
+            "name": "qwen3 測試品",
+            "gap_pct": 22.4,
+            "sales_delta": -35.0,
+            "action": "跟進降價",
+            "confidence": 0.85,
+        },
+    }
+
+
 class _FakeResp:
     def __init__(self, payload: dict, status: int = 200):
         self._payload = payload
@@ -59,6 +73,7 @@ def _noop_log_ai_call(*args, **kwargs):
     class _Ctx:
         def set_tokens(self, **_kw): pass
         def set_provider(self, *_a, **_kw): pass
+        def set_model(self, *_a, **_kw): pass
         def set_error(self, *_a, **_kw): pass
         def fallback_to_caller(self, *_a, **_kw): pass
         def set_cache_hit(self, *_a, **_kw): pass
@@ -76,11 +91,13 @@ def _reset_global_state():
     import services.nemoton_dispatcher_service as _nem
     import services.ollama_service as _oss
     _nem._ALERT_CACHE.clear()
+    _nem._ALERT_INFLIGHT.clear()
     _oss._unhealthy_marks.clear()
     _oss._resolved_host_cache['host'] = None
     _oss._resolved_host_cache['ts'] = 0
     yield
     _nem._ALERT_CACHE.clear()
+    _nem._ALERT_INFLIGHT.clear()
     _oss._unhealthy_marks.clear()
     _oss._resolved_host_cache['host'] = None
     _oss._resolved_host_cache['ts'] = 0
@@ -93,6 +110,7 @@ def _patch_execution_methods(monkeypatch, dispatcher):
     def record(kind):
         def _inner(*args, **kwargs):
             calls.append({"kind": kind, "args": args, "kwargs": kwargs})
+            return {"durable": True}
         return _inner
 
     monkeypatch.setattr(dispatcher, "_exec_trigger_price_alert", record("price_alert"))
@@ -102,14 +120,27 @@ def _patch_execution_methods(monkeypatch, dispatcher):
 
 
 def _enable_qwen3_path(monkeypatch, module):
-    """打開 NEMOTRON_OLLAMA_FIRST + 旁路 mcp/log_ai_call/resolve_host 等副作用"""
+    """打開 Ollama-first，並隔離 MCP、logger 與 model identity 網路副作用。"""
     monkeypatch.setattr(module, "NEMOTRON_OLLAMA_FIRST", True)
     monkeypatch.setattr(module, "log_ai_call", _noop_log_ai_call)
     monkeypatch.setattr(module, "build_mcp_context", lambda: "MCP-MOCK")
     # 確保即使未被呼叫，import 路徑可解析
     import services.ollama_service as ollama_module
+    import services.nemotron_runtime_candidate_service as runtime_module
+
     monkeypatch.setattr(ollama_module, "resolve_ollama_host", lambda: "http://34.87.90.216:11434")
     monkeypatch.setattr(ollama_module, "mark_unhealthy", lambda *a, **kw: None)
+    monkeypatch.setattr(
+        runtime_module,
+        "inspect_nemotron_model_identity",
+        lambda candidate, **_kwargs: {
+            **candidate.as_public_dict(),
+            "ok": True,
+            "digest": candidate.expected_digest,
+            "digest_matches": True,
+            "error": None,
+        },
+    )
 
 
 # ─────────────────────────────────────────────────────────────
@@ -178,12 +209,7 @@ def test_qwen3_retries_secondary_when_primary_chat_fails(monkeypatch):
     import services.ollama_service as ollama_module
 
     _enable_qwen3_path(monkeypatch, module)
-    hosts = iter([
-        "http://34.87.90.216:11434",
-        "http://34.21.145.224:11434",
-    ])
     marked = []
-    monkeypatch.setattr(ollama_module, "resolve_ollama_host", lambda: next(hosts))
     monkeypatch.setattr(ollama_module, "mark_unhealthy", lambda host: marked.append(host))
 
     fake_body = {
@@ -227,6 +253,519 @@ def test_qwen3_retries_secondary_when_primary_chat_fails(monkeypatch):
     assert calls[0]["kind"] == "price_alert"
 
 
+def test_qwen3_uses_bounded_111_model_after_both_gcp_candidates_fail(monkeypatch):
+    """兩台 GCP 都逾時時，正式 dispatcher 應改用 exact-digest 111 qwen3:8b。"""
+    import requests
+    import services.nemoton_dispatcher_service as module
+
+    _enable_qwen3_path(monkeypatch, module)
+    posted = []
+    fake_body = {
+        "message": {
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [{
+                "function": {
+                    "name": "trigger_price_alert",
+                    "arguments": {
+                        "sku": "SKU-Q1",
+                        "name": "qwen3 測試品",
+                        "gap_pct": 22.4,
+                        "sales_delta": -35.0,
+                        "action": "跟進降價",
+                        "confidence": 0.85,
+                    },
+                }
+            }],
+        },
+        "prompt_eval_count": 180,
+        "eval_count": 36,
+    }
+
+    def fake_post(url, json, timeout):
+        posted.append((url, json, timeout))
+        if len(posted) <= 2:
+            raise requests.Timeout("GCP saturated")
+        return _FakeResp(fake_body)
+
+    monkeypatch.setattr(module.requests, "post", fake_post)
+    dispatcher = module.NemotronDispatcher()
+    calls = _patch_execution_methods(monkeypatch, dispatcher)
+
+    result = dispatcher.dispatch([FakeThreat()], hermes_stats={"duration_sec": 1.0})
+
+    stats = result["nim_stats"]
+    assert result["dispatched"] == 1
+    assert stats["provider"] == "ollama_111"
+    assert stats["model"] == "qwen3:8b"
+    assert stats["candidate_label"] == "ollama_111_fallback"
+    assert stats["fallback_used"] is True
+    assert [attempt["status"] for attempt in stats["attempted_candidates"]] == [
+        "failed",
+        "failed",
+        "selected",
+    ]
+    assert posted[2][1]["think"] is False
+    assert posted[2][1]["options"]["num_predict"] == 512
+    assert posted[2][2] == 45
+    assert calls[0]["kind"] == "price_alert"
+    assert "qwen3:8b (111 備援)" in calls[0]["kwargs"]["footprint"]
+    assert "降級備援" in calls[0]["kwargs"]["footprint"]
+
+    structured = module._build_footprint_json(None, stats)
+    assert structured["dispatcher"]["platform"] == "Ollama"
+    assert structured["dispatcher"]["model"] == "qwen3:8b"
+    assert structured["dispatcher"]["fallback_used"] is True
+    assert [
+        attempt["label"]
+        for attempt in structured["dispatcher"]["candidate_attempts"]
+    ] == ["gcp_primary", "gcp_secondary", "ollama_111_fallback"]
+    assert all(
+        "host" not in attempt and "error" not in attempt
+        for attempt in structured["dispatcher"]["candidate_attempts"]
+    )
+
+
+def test_invalid_primary_tool_contract_retries_verified_secondary(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    _enable_qwen3_path(monkeypatch, module)
+    valid_body = {
+        "message": {
+            "tool_calls": [{"function": {
+                "name": "trigger_price_alert",
+                "arguments": _valid_price_alert_call()["args"],
+            }}],
+            "content": "",
+        },
+        "prompt_eval_count": 100,
+        "eval_count": 20,
+    }
+    invalid_body = {
+        "message": {
+            "tool_calls": [{"function": {
+                "name": "delete_product",
+                "arguments": {"sku": "SKU-Q1"},
+            }}],
+            "content": "",
+        },
+    }
+    responses = [_FakeResp(invalid_body), _FakeResp(valid_body)]
+    monkeypatch.setattr(
+        module.requests,
+        "post",
+        lambda *_args, **_kwargs: responses.pop(0),
+    )
+    dispatcher = module.NemotronDispatcher()
+    calls = _patch_execution_methods(monkeypatch, dispatcher)
+
+    result = dispatcher.dispatch([FakeThreat()], hermes_stats={"duration_sec": 1.0})
+
+    attempts = result["nim_stats"]["attempted_candidates"]
+    assert [attempt["status"] for attempt in attempts] == ["failed", "selected"]
+    assert "unknown_tool:delete_product" in attempts[0]["error"]
+    assert result["nim_stats"]["provider"] == "ollama_secondary"
+    assert result["dispatched"] == 1
+    assert calls[0]["kind"] == "price_alert"
+
+
+def test_malformed_primary_tags_schema_falls_through_to_verified_secondary(
+    monkeypatch,
+):
+    import services.nemoton_dispatcher_service as module
+    import services.nemotron_runtime_candidate_service as runtime_module
+    import services.ollama_service as ollama_module
+
+    monkeypatch.setattr(module, "NEMOTRON_OLLAMA_FIRST", True)
+    monkeypatch.setattr(module, "log_ai_call", _noop_log_ai_call)
+    monkeypatch.setattr(module, "build_mcp_context", lambda: "MCP-MOCK")
+    monkeypatch.setattr(ollama_module, "mark_unhealthy", lambda *_args: None)
+
+    def fake_get(url, timeout):
+        if runtime_module.OLLAMA_HOST_PRIMARY in url:
+            return _FakeResp({
+                "models": [
+                    None,
+                    {
+                        "name": runtime_module.PRIMARY_MODEL,
+                        "model": runtime_module.PRIMARY_MODEL,
+                        "digest": runtime_module.PRIMARY_EXPECTED_DIGEST,
+                        "details": {},
+                    },
+                ]
+            })
+        return _FakeResp({
+            "models": [{
+                "name": runtime_module.PRIMARY_MODEL,
+                "model": runtime_module.PRIMARY_MODEL,
+                "digest": runtime_module.PRIMARY_EXPECTED_DIGEST,
+                "details": {
+                    "parameter_size": "14.8B",
+                    "quantization_level": "Q4_K_M",
+                },
+            }]
+        })
+
+    valid_body = {
+        "message": {
+            "tool_calls": [{"function": {
+                "name": "trigger_price_alert",
+                "arguments": _valid_price_alert_call()["args"],
+            }}],
+            "content": "",
+        },
+        "prompt_eval_count": 100,
+        "eval_count": 20,
+    }
+    monkeypatch.setattr(module.requests, "get", fake_get)
+    monkeypatch.setattr(
+        module.requests,
+        "post",
+        lambda *_args, **_kwargs: _FakeResp(valid_body),
+    )
+    dispatcher = module.NemotronDispatcher()
+    calls = _patch_execution_methods(monkeypatch, dispatcher)
+
+    result = dispatcher.dispatch([FakeThreat()], hermes_stats={"duration_sec": 1})
+
+    attempts = result["nim_stats"]["attempted_candidates"]
+    assert [attempt["status"] for attempt in attempts] == [
+        "identity_failed",
+        "selected",
+    ]
+    assert attempts[0]["error"] == "malformed_models_schema"
+    assert result["nim_stats"]["provider"] == "ollama_secondary"
+    assert result["dispatched"] == 1
+    assert calls[0]["kind"] == "price_alert"
+
+
+@pytest.mark.parametrize(
+    ("tool_calls", "threats", "error_fragment"),
+    [
+        ([{"tool": "delete_product", "args": {"sku": "SKU-Q1"}}], [FakeThreat()], "unknown_tool"),
+        ([_valid_price_alert_call("OTHER")], [FakeThreat()], "sku_not_in_request"),
+        ([{"tool": "trigger_price_alert", "args": {"sku": "SKU-Q1"}}], [FakeThreat()], "missing_required_args"),
+        ([_valid_price_alert_call()], [FakeThreat(), FakeThreat(sku="SKU-Q2")], "tool_call_count_mismatch"),
+        (
+            [_valid_price_alert_call(), _valid_price_alert_call()],
+            [FakeThreat(), FakeThreat(sku="SKU-Q2")],
+            "duplicate_sku_tool_call",
+        ),
+    ],
+)
+def test_tool_contract_rejects_unbound_incomplete_or_duplicate_calls(
+    tool_calls, threats, error_fragment
+):
+    import services.nemoton_dispatcher_service as module
+
+    with pytest.raises(module.ToolCallContractError, match=error_fragment):
+        module._validate_tool_call_contract(tool_calls, threats)
+
+
+def test_failed_handler_does_not_poison_alert_dedupe(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    dispatcher = module.NemotronDispatcher()
+    monkeypatch.setattr(
+        dispatcher,
+        "_exec_trigger_price_alert",
+        lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("send failed")),
+    )
+    first_token = module._reserve_alert("SKU-Q1")
+    assert first_token
+    first = dispatcher._execute_tool_calls(
+        tool_calls=[_valid_price_alert_call()],
+        threats=[FakeThreat()],
+        hermes_stats=None,
+        nim_stats={"provider": "nim", "total_tokens": 1},
+        reservation_tokens={"SKU-Q1": first_token},
+    )
+
+    assert first["dispatched"] == 0
+    assert module._is_duplicate_alert("SKU-Q1") is False
+
+    sent = []
+    monkeypatch.setattr(
+        dispatcher,
+        "_exec_trigger_price_alert",
+        lambda **kwargs: (sent.append(kwargs) or {"durable": True}),
+    )
+    second_token = module._reserve_alert("SKU-Q1")
+    assert second_token
+    second = dispatcher._execute_tool_calls(
+        tool_calls=[_valid_price_alert_call()],
+        threats=[FakeThreat()],
+        hermes_stats=None,
+        nim_stats={"provider": "nim", "total_tokens": 1},
+        reservation_tokens={"SKU-Q1": second_token},
+    )
+
+    assert second["dispatched"] == 1
+    assert sent
+    assert module._is_duplicate_alert("SKU-Q1") is True
+
+
+def test_non_durable_alert_outcome_is_failed_and_releases_reservation(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    dispatcher = module.NemotronDispatcher()
+    monkeypatch.setattr(
+        dispatcher,
+        "_send_telegram",
+        lambda *_args, **_kwargs: {
+            "durable": False,
+            "delivered": False,
+            "queued": False,
+        },
+    )
+    monkeypatch.setattr(dispatcher, "_sink_insight_to_km", lambda **_kwargs: True)
+    token = module._reserve_alert("SKU-Q1")
+    assert token
+
+    result = dispatcher._execute_tool_calls(
+        tool_calls=[_valid_price_alert_call()],
+        threats=[FakeThreat()],
+        hermes_stats=None,
+        nim_stats={"provider": "nim", "total_tokens": 1},
+        reservation_tokens={"SKU-Q1": token},
+    )
+
+    assert result["dispatched"] == 0
+    assert result["skipped"] == 1
+    assert "durable_side_effect_not_confirmed" in result["errors"][0]
+    assert module._is_duplicate_alert("SKU-Q1") is False
+
+
+def test_recommendation_requires_db_write_or_durable_notification(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    dispatcher = module.NemotronDispatcher(engine=None)
+    monkeypatch.setattr(
+        dispatcher,
+        "_send_telegram",
+        lambda *_args, **_kwargs: {"durable": False},
+    )
+    monkeypatch.setattr(dispatcher, "_sink_insight_to_km", lambda **_kwargs: True)
+    token = module._reserve_alert("SKU-Q1")
+    assert token
+
+    result = dispatcher._execute_tool_calls(
+        tool_calls=[{
+            "tool": "add_to_recommendation",
+            "args": {
+                "sku": "SKU-Q1",
+                "name": "qwen3 測試品",
+                "reason": "具價格競爭力",
+                "confidence": 0.85,
+            },
+        }],
+        threats=[FakeThreat()],
+        hermes_stats=None,
+        nim_stats={"provider": "nim", "total_tokens": 1},
+        reservation_tokens={"SKU-Q1": token},
+    )
+
+    assert result["dispatched"] == 0
+    assert "durable_side_effect_not_confirmed" in result["errors"][0]
+    assert module._is_duplicate_alert("SKU-Q1") is False
+
+
+def test_km_route_requires_durable_insight_write(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    dispatcher = module.NemotronDispatcher()
+    monkeypatch.setattr(dispatcher, "_sink_insight_to_km", lambda **_kwargs: False)
+    token = module._reserve_alert("SKU-Q1")
+    assert token
+
+    result = dispatcher._execute_tool_calls(
+        tool_calls=[{
+            "tool": "route_to_km",
+            "args": {
+                "sku": "SKU-Q1",
+                "name": "qwen3 測試品",
+                "km_domain": "price_competition",
+                "summary": "競價洞察",
+                "confidence": 0.85,
+            },
+        }],
+        threats=[FakeThreat()],
+        hermes_stats=None,
+        nim_stats={"provider": "nim", "total_tokens": 1},
+        reservation_tokens={"SKU-Q1": token},
+    )
+
+    assert result["dispatched"] == 0
+    assert "durable_side_effect_not_confirmed" in result["errors"][0]
+    assert module._is_duplicate_alert("SKU-Q1") is False
+
+
+def test_alert_reservation_is_atomic_and_releasable():
+    import services.nemoton_dispatcher_service as module
+
+    first_token = module._reserve_alert("SKU-Q1")
+    assert first_token
+    assert module._reserve_alert("SKU-Q1") is None
+    assert module._release_alert_reservation("SKU-Q1", first_token) is True
+    second_token = module._reserve_alert("SKU-Q1")
+    assert second_token and second_token != first_token
+    assert module._mark_alert_dispatched("SKU-Q1", second_token) is True
+    assert module._reserve_alert("SKU-Q1") is None
+
+
+def test_expired_reservation_token_cannot_release_or_commit_new_owner(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    clock = {"now": 1000.0}
+    monkeypatch.setattr(module.time, "time", lambda: clock["now"])
+    first_token = module._reserve_alert("SKU-Q1")
+    assert first_token
+
+    clock["now"] += module._ALERT_INFLIGHT_TTL_SEC + 1
+    assert module._mark_alert_dispatched("SKU-Q1", first_token) is False
+    second_token = module._reserve_alert("SKU-Q1")
+    assert second_token and second_token != first_token
+
+    assert module._release_alert_reservation("SKU-Q1", first_token) is False
+    assert module._mark_alert_dispatched("SKU-Q1", first_token) is False
+    assert module._refresh_alert_reservation("SKU-Q1", first_token) is False
+    assert module._refresh_alert_reservation("SKU-Q1", second_token) is True
+    assert module._mark_alert_dispatched("SKU-Q1", second_token) is True
+    assert module._is_duplicate_alert("SKU-Q1") is True
+
+
+def test_concurrent_reservation_has_exactly_one_owner():
+    from concurrent.futures import ThreadPoolExecutor
+    from threading import Barrier
+
+    import services.nemoton_dispatcher_service as module
+
+    barrier = Barrier(8)
+
+    def reserve_once():
+        barrier.wait()
+        return module._reserve_alert("SKU-CONCURRENT")
+
+    with ThreadPoolExecutor(max_workers=8) as pool:
+        tokens = list(pool.map(lambda _index: reserve_once(), range(8)))
+
+    owners = [token for token in tokens if token is not None]
+    assert len(owners) == 1
+    assert module._release_alert_reservation("SKU-CONCURRENT", owners[0]) is True
+
+
+def test_side_effect_boundary_extends_lease_through_slow_handler(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    clock = {"now": 5000.0}
+    monkeypatch.setattr(module.time, "time", lambda: clock["now"])
+    old_token = module._reserve_alert("SKU-Q1")
+    assert old_token
+    new_owner = {}
+    dispatcher = module.NemotronDispatcher()
+
+    def delayed_handler(**_kwargs):
+        clock["now"] += module._ALERT_INFLIGHT_TTL_SEC + 1
+        new_owner["token"] = module._reserve_alert("SKU-Q1")
+        return {"durable": True}
+
+    monkeypatch.setattr(dispatcher, "_exec_trigger_price_alert", delayed_handler)
+    result = dispatcher._execute_tool_calls(
+        tool_calls=[_valid_price_alert_call()],
+        threats=[FakeThreat()],
+        hermes_stats=None,
+        nim_stats={"provider": "nim", "total_tokens": 1},
+        reservation_tokens={"SKU-Q1": old_token},
+    )
+
+    assert new_owner["token"] is None
+    assert result["dispatched"] == 1
+    assert result["errors"] == []
+    assert module._is_duplicate_alert("SKU-Q1") is True
+
+
+def test_durable_tool_commit_failure_never_releases_quarantine(monkeypatch):
+    import services.nemoton_dispatcher_service as module
+
+    dispatcher = module.NemotronDispatcher()
+    monkeypatch.setattr(
+        dispatcher,
+        "_exec_trigger_price_alert",
+        lambda **_kwargs: {"durable": True},
+    )
+    token = module._reserve_alert("SKU-Q1")
+    released = []
+    monkeypatch.setattr(module, "_mark_alert_dispatched", lambda *_args: False)
+    monkeypatch.setattr(
+        module,
+        "_release_alert_reservation",
+        lambda *args: (released.append(args) or True),
+    )
+
+    result = dispatcher._execute_tool_calls(
+        tool_calls=[_valid_price_alert_call()],
+        threats=[FakeThreat()],
+        hermes_stats=None,
+        nim_stats={"provider": "nim", "total_tokens": 1},
+        reservation_tokens={"SKU-Q1": token},
+    )
+
+    assert result["dispatched"] == 1
+    assert result["skipped"] == 0
+    assert result["errors"] == [
+        "dedupe_commit_unverified:trigger_price_alert(SKU-Q1)"
+    ]
+    assert released == []
+    assert module._is_duplicate_alert("SKU-Q1") is True
+
+
+def test_dispatch_total_deadline_bounds_ollama_and_nim_chain(monkeypatch):
+    import requests
+    import services.nemoton_dispatcher_service as module
+
+    _enable_qwen3_path(monkeypatch, module)
+    monkeypatch.setattr(module, "NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC", 30)
+    monkeypatch.setattr(module, "NIM_API_KEY", "fake-key")
+    monkeypatch.setattr(module, "_check_nim_quota", lambda: True)
+    clock = {"now": 100.0}
+    monkeypatch.setattr(module.time, "monotonic", lambda: clock["now"])
+    posted = []
+
+    def timeout_post(url, json=None, timeout=None, **_kwargs):
+        posted.append((url, timeout))
+        clock["now"] += float(timeout or 0)
+        raise requests.Timeout("simulated timeout")
+
+    monkeypatch.setattr(module.requests, "post", timeout_post)
+    dispatcher = module.NemotronDispatcher()
+    calls = _patch_execution_methods(monkeypatch, dispatcher)
+
+    result = dispatcher.dispatch([FakeThreat()], hermes_stats={"duration_sec": 1.0})
+
+    upstream = result["nim_stats"]["upstream_ollama_failure"]
+    assert len(posted) == 1
+    assert posted[0][1] == 30
+    assert clock["now"] == 130.0
+    assert upstream["deadline_exhausted"] is True
+    assert upstream["attempt_count"] == 2
+    assert result["nim_stats"]["provider"] == "hermes_rule_engine"
+    assert "Ollama 候選鏈" in calls[0]["kwargs"]["footprint"]
+    assert "gcp_primary | primary | qwen3:14b | failed | digest=ok | Timeout" in (
+        calls[0]["kwargs"]["footprint"]
+    )
+    structured = module._build_footprint_json(None, result["nim_stats"])
+    attempts = structured["dispatcher"]["candidate_attempts"]
+    assert attempts[0] == {
+        "label": "gcp_primary",
+        "tier": "primary",
+        "model": "qwen3:14b",
+        "status": "failed",
+        "digest_matches": True,
+        "error_class": "Timeout",
+    }
+    assert result["dispatched"] == 1
+
+
 # ─────────────────────────────────────────────────────────────
 # T2. qwen3 沒回 tool_calls 但 content 含 JSON list → fallback 解析
 # ─────────────────────────────────────────────────────────────
@@ -252,7 +791,7 @@ def test_qwen3_content_only_fallback_parsing(monkeypatch):
     calls = _patch_execution_methods(monkeypatch, dispatcher)
     monkeypatch.setattr(
         dispatcher, "_call_nim",
-        lambda threats: (_ for _ in ()).throw(AssertionError("NIM 不應被呼叫")),
+        lambda threats, **_kwargs: (_ for _ in ()).throw(AssertionError("NIM 不應被呼叫")),
     )
 
     result = dispatcher.dispatch([FakeThreat(confidence=0.45)], hermes_stats={"duration_sec": 1.0})
@@ -325,7 +864,7 @@ def test_qwen3_connection_error_falls_back_to_nim(monkeypatch):
     calls = _patch_execution_methods(monkeypatch, dispatcher)
     nim_invoked = {"v": False}
 
-    def _fake_nim(threats):
+    def _fake_nim(threats, **_kwargs):
         nim_invoked["v"] = True
         return (
             [{
@@ -369,7 +908,7 @@ def test_qwen3_and_nim_both_fail_falls_back_to_hermes_rules(monkeypatch):
     # 攔 _call_nim 也擲 timeout
     monkeypatch.setattr(
         dispatcher, "_call_nim",
-        lambda threats: (_ for _ in ()).throw(requests.Timeout("NIM timeout")),
+        lambda threats, **_kwargs: (_ for _ in ()).throw(requests.Timeout("NIM timeout")),
     )
 
     # 攔住規則引擎內部呼叫的 _exec_*，記錄 concern / reason 文字驗證 🟡 標記
@@ -392,6 +931,7 @@ def test_qwen3_and_nim_both_fail_falls_back_to_hermes_rules(monkeypatch):
              'revenue_loss_7d', 'recommended_price'],
             args, kwargs)
         captured.append(("human_review", merged))
+        return {"durable": True}
 
     def record_alert(*args, **kwargs):
         merged = _merge_positional(
@@ -400,9 +940,11 @@ def test_qwen3_and_nim_both_fail_falls_back_to_hermes_rules(monkeypatch):
              'revenue_loss_7d', 'recommended_price'],
             args, kwargs)
         captured.append(("price_alert", merged))
+        return {"durable": True}
 
     def record_reco(*args, **kwargs):
         captured.append(("recommendation", kwargs))
+        return {"durable": True}
 
     monkeypatch.setattr(dispatcher, "_exec_flag_for_human_review", record_review)
     monkeypatch.setattr(dispatcher, "_exec_trigger_price_alert", record_alert)
@@ -448,7 +990,7 @@ def test_flag_false_preserves_nim_first_emergency_path(monkeypatch):
     calls = _patch_execution_methods(monkeypatch, dispatcher)
     monkeypatch.setattr(
         dispatcher, "_call_nim",
-        lambda threats: (
+        lambda threats, **_kwargs: (
             [{
                 "tool": "trigger_price_alert",
                 "args": {

tests/test_nemotron_runtime_candidate_service.py

@@ -0,0 +1,205 @@
+import json
+import os
+import subprocess
+import sys
+
+import pytest
+import requests
+
+
+class _Response:
+    def __init__(self, payload):
+        self._payload = payload
+
+    def raise_for_status(self):
+        return None
+
+    def json(self):
+        return self._payload
+
+
+def test_candidate_chain_is_gcp_first_with_exact_digest_111_fallback():
+    from services import nemotron_runtime_candidate_service as service
+
+    candidates = service.build_nemotron_runtime_candidates()
+
+    assert [candidate.label for candidate in candidates] == [
+        "gcp_primary",
+        "gcp_secondary",
+        "ollama_111_fallback",
+    ]
+    assert [candidate.model for candidate in candidates] == [
+        "qwen3:14b",
+        "qwen3:14b",
+        "qwen3:8b",
+    ]
+    assert candidates[0].is_fallback is False
+    assert candidates[2].is_fallback is True
+    assert candidates[0].request_timeout_sec == 60
+    assert candidates[2].request_timeout_sec == 45
+    assert candidates[0].num_predict == 2048
+    assert candidates[2].num_predict == 512
+    assert candidates[0].num_ctx is None
+    assert candidates[2].num_ctx == 4096
+
+
+def test_model_identity_fails_closed_on_digest_drift():
+    from services import nemotron_runtime_candidate_service as service
+
+    candidate = service.build_nemotron_runtime_candidates()[0]
+    identity = service.inspect_nemotron_model_identity(
+        candidate,
+        request_get=lambda *_args, **_kwargs: _Response({
+            "models": [{
+                "name": candidate.model,
+                "model": candidate.model,
+                "digest": "deadbeef",
+                "details": {},
+            }]
+        }),
+    )
+
+    assert identity["ok"] is False
+    assert identity["digest_matches"] is False
+    assert identity["error"] == "model_digest_mismatch"
+
+
+def test_model_identity_reports_network_failure_without_loading_model():
+    from services import nemotron_runtime_candidate_service as service
+
+    candidate = service.build_nemotron_runtime_candidates()[2]
+
+    def fail(*_args, **_kwargs):
+        raise requests.Timeout("offline")
+
+    identity = service.inspect_nemotron_model_identity(
+        candidate,
+        request_get=fail,
+    )
+
+    assert identity["ok"] is False
+    assert identity["digest"] is None
+    assert identity["error"].startswith("Timeout: offline")
+
+
+@pytest.mark.parametrize(
+    ("payload", "error_fragment"),
+    [
+        (None, "malformed_tags_payload"),
+        ({"models": {}}, "malformed_models_schema"),
+        ({"models": [None]}, "malformed_models_schema"),
+        ({"models": [{"name": "other", "details": None}]}, "malformed_model_details"),
+    ],
+)
+def test_model_identity_fails_closed_on_malformed_tags_schema(
+    payload, error_fragment
+):
+    from services import nemotron_runtime_candidate_service as service
+
+    candidate = service.build_nemotron_runtime_candidates()[0]
+    identity = service.inspect_nemotron_model_identity(
+        candidate,
+        request_get=lambda *_args, **_kwargs: _Response(payload),
+    )
+
+    assert identity["ok"] is False
+    assert identity["digest_matches"] is False
+    assert error_fragment in str(identity["error"])
+
+
+def test_model_identity_rejects_malformed_entry_mixed_with_valid_target():
+    from services import nemotron_runtime_candidate_service as service
+
+    candidate = service.build_nemotron_runtime_candidates()[0]
+    identity = service.inspect_nemotron_model_identity(
+        candidate,
+        request_get=lambda *_args, **_kwargs: _Response({
+            "models": [
+                None,
+                {
+                    "name": candidate.model,
+                    "model": candidate.model,
+                    "digest": candidate.expected_digest,
+                    "details": {},
+                },
+            ]
+        }),
+    )
+
+    assert identity["ok"] is False
+    assert identity["digest_matches"] is False
+    assert identity["error"] == "malformed_models_schema"
+
+
+def test_candidate_attempt_timeout_honors_env_but_111_resources_stay_fixed():
+    env = os.environ.copy()
+    env.update({
+        "NEMOTRON_111_ATTEMPT_TIMEOUT_SEC": "33",
+        "OLLAMA_111_NUM_CTX": "2048",
+        "OLLAMA_111_NUM_PREDICT": "256",
+    })
+    completed = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "import json; "
+                "from services.nemotron_runtime_candidate_service import "
+                "build_nemotron_runtime_candidates; "
+                "c=build_nemotron_runtime_candidates()[-1]; "
+                "print(json.dumps(c.as_public_dict(), sort_keys=True))"
+            ),
+        ],
+        cwd=os.path.dirname(os.path.dirname(__file__)),
+        env=env,
+        capture_output=True,
+        check=False,
+        text=True,
+    )
+
+    assert completed.returncode == 0, completed.stderr
+    fallback = json.loads(completed.stdout.strip())
+    assert fallback["request_timeout_sec"] == 33
+    assert fallback["num_ctx"] == 4096
+    assert fallback["num_predict"] == 512
+
+
+@pytest.mark.parametrize(
+    ("num_ctx", "num_predict"),
+    [
+        ("99999", "99999"),
+        ("-7", "-9"),
+        ("not-a-number", "invalid"),
+    ],
+)
+def test_candidate_resource_contract_cannot_be_weakened_by_environment(
+    num_ctx, num_predict
+):
+    env = os.environ.copy()
+    env.update({
+        "OLLAMA_111_NUM_CTX": num_ctx,
+        "OLLAMA_111_NUM_PREDICT": num_predict,
+    })
+    completed = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "import json; "
+                "from services.nemotron_runtime_candidate_service import "
+                "build_nemotron_runtime_candidates; "
+                "c=build_nemotron_runtime_candidates()[-1]; "
+                "print(json.dumps(c.as_public_dict(), sort_keys=True))"
+            ),
+        ],
+        cwd=os.path.dirname(os.path.dirname(__file__)),
+        env=env,
+        capture_output=True,
+        check=False,
+        text=True,
+    )
+
+    assert completed.returncode == 0, completed.stderr
+    fallback = json.loads(completed.stdout.strip())
+    assert fallback["num_ctx"] == 4096
+    assert fallback["num_predict"] == 512

tests/test_phase3f_cleanup_contracts.py

@@ -132,8 +132,15 @@ def test_env_example_documents_runtime_and_ai_automation_variables():
         "N8N_USER",
         "N8N_WEBHOOK_BASE_URL",
         "NEMOTRON_OLLAMA_FIRST",
+        "NEMOTRON_OLLAMA_EXPECTED_DIGEST",
+        "NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST",
+        "NEMOTRON_OLLAMA_FALLBACK_MODEL",
         "NEMOTRON_OLLAMA_MODEL",
         "NEMOTRON_OLLAMA_TIMEOUT",
+        "NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC",
+        "NEMOTRON_111_ATTEMPT_TIMEOUT_SEC",
+        "NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC",
+        "NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC",
         "OLLAMA_EMBED_TIMEOUT",
         "OPENCLAW_ADMIN_USER_IDS",
         "OPENCLAW_AGENT_DISPATCH",

tests/test_qwen3_runtime_usage.py

@@ -7,6 +7,7 @@ ROOT = Path(__file__).resolve().parents[1]
 def test_qwen3_is_active_runtime_model_not_unused_ollama_weight():
     openclaw_source = (ROOT / "services" / "openclaw_strategist_service.py").read_text(encoding="utf-8")
     nemotron_source = (ROOT / "services" / "nemoton_dispatcher_service.py").read_text(encoding="utf-8")
+    candidate_source = (ROOT / "services" / "nemotron_runtime_candidate_service.py").read_text(encoding="utf-8")
     router_source = (ROOT / "services" / "llm_model_router.py").read_text(encoding="utf-8")
 
     assert "OPENCLAW_QA_OLLAMA_MODEL = os.getenv('OPENCLAW_QA_OLLAMA_MODEL', 'qwen3:14b')" in openclaw_source
@@ -16,7 +17,11 @@ def test_qwen3_is_active_runtime_model_not_unused_ollama_weight():
     assert "OPENCLAW_STRATEGY_OLLAMA_KEEP_ALIVE = os.getenv('OPENCLAW_STRATEGY_OLLAMA_KEEP_ALIVE', '5m')" in openclaw_source
     assert "keep_alive=OPENCLAW_STRATEGY_OLLAMA_KEEP_ALIVE" in openclaw_source
     assert 'keep_alive="24h"' not in openclaw_source
-    assert 'NEMOTRON_OLLAMA_MODEL = os.getenv("NEMOTRON_OLLAMA_MODEL", "qwen3:14b")' in nemotron_source
+    assert "NEMOTRON_OLLAMA_MODEL = PRIMARY_MODEL" in nemotron_source
     assert "def _call_qwen3_dispatch(" in nemotron_source
-    assert "for _attempt in range(3):" in nemotron_source
+    assert "for candidate in build_nemotron_runtime_candidates(" in nemotron_source
+    assert "inspect_nemotron_model_identity(" in nemotron_source
+    assert '"qwen3:14b"' in candidate_source
+    assert '"qwen3:8b"' in candidate_source
+    assert 'label="ollama_111_fallback"' in candidate_source
     assert "'qwen3:14b'" in router_source