feat(ai): add bounded model-aware decision fallback

services/nemotron_decision_canary_service.py

@@ -1,8 +1,9 @@
 """Decision-only NemoTron runtime canary.
 
 The canary exercises the production qwen3 tool-calling payload against an
-approved GCP Ollama host, validates the returned decision, and stops before any
-tool, Telegram, database, price, or insight write is allowed.
+exact-digest GCP-first candidate chain with a bounded 111 fallback, validates
+the returned decision, and stops before any tool, Telegram, database, price, or
+insight write is allowed.
 """
 
 from __future__ import annotations
@@ -18,24 +19,30 @@ from typing import Any, Mapping
 
 import requests
 
+from services.nemotron_runtime_candidate_service import (
+    FALLBACK_EXPECTED_DIGEST,
+    FALLBACK_MODEL,
+    IDENTITY_TIMEOUT_SEC,
+    PRIMARY_EXPECTED_DIGEST,
+    PRIMARY_MODEL,
+    NemotronRuntimeCandidate,
+    build_nemotron_runtime_candidates,
+    inspect_nemotron_model_identity,
+)
 from services.ollama_service import OLLAMA_HOST_PRIMARY, OLLAMA_HOST_SECONDARY
 
 
 POLICY = "controlled_nemotron_decision_only_canary_v1"
 CANARY_VERSION = "nemotron_decision_only_canary_v1"
-NEMOTRON_OLLAMA_MODEL = os.getenv("NEMOTRON_OLLAMA_MODEL", "qwen3:14b")
-NEMOTRON_OLLAMA_EXPECTED_DIGEST = os.getenv(
-    "NEMOTRON_OLLAMA_EXPECTED_DIGEST",
-    "bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8",
-).strip().lower()
+NEMOTRON_OLLAMA_MODEL = PRIMARY_MODEL
+NEMOTRON_OLLAMA_EXPECTED_DIGEST = PRIMARY_EXPECTED_DIGEST
+NEMOTRON_OLLAMA_FALLBACK_MODEL = FALLBACK_MODEL
+NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST = FALLBACK_EXPECTED_DIGEST
 DEFAULT_TIMEOUT_SEC = max(
     30,
     min(int(os.getenv("NEMOTRON_DECISION_CANARY_TIMEOUT_SEC", "300")), 600),
 )
-DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC = max(
-    2,
-    min(int(os.getenv("NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC", "10")), 30),
-)
+DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC = IDENTITY_TIMEOUT_SEC
 DEFAULT_MAX_AGE_HOURS = max(
     1,
     min(int(os.getenv("NEMOTRON_DECISION_CANARY_MAX_AGE_HOURS", "26")), 168),
@@ -86,52 +93,21 @@ def _parse_datetime(value: Any) -> datetime | None:
 
 
 def _model_identity(host: str) -> dict[str, Any]:
-    started = time.monotonic()
-    try:
-        response = requests.get(
-            f"{host.rstrip('/')}/api/tags",
-            timeout=DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC,
-        )
-        response.raise_for_status()
-        models = response.json().get("models") or []
-    except Exception as exc:
-        return {
-            "ok": False,
-            "host": host,
-            "model": NEMOTRON_OLLAMA_MODEL,
-            "digest": None,
-            "digest_matches": False,
-            "elapsed_ms": round((time.monotonic() - started) * 1000),
-            "error": f"{type(exc).__name__}: {str(exc)[:180]}",
-        }
-
-    target = NEMOTRON_OLLAMA_MODEL
-    target_without_latest = target.removesuffix(":latest")
-    matched: Mapping[str, Any] = {}
-    for item in models:
-        names = {
-            str(item.get("name") or "").strip(),
-            str(item.get("model") or "").strip(),
-        }
-        if target in names or target_without_latest in names:
-            matched = item
-            break
-    digest = str(matched.get("digest") or "").strip().lower()
-    digest_matches = bool(digest) and digest == NEMOTRON_OLLAMA_EXPECTED_DIGEST
-    return {
-        "ok": bool(matched) and digest_matches,
-        "host": host,
-        "model": target,
-        "digest": digest or None,
-        "expected_digest": NEMOTRON_OLLAMA_EXPECTED_DIGEST,
-        "digest_matches": digest_matches,
-        "parameter_size": str((matched.get("details") or {}).get("parameter_size") or ""),
-        "quantization_level": str(
-            (matched.get("details") or {}).get("quantization_level") or ""
-        ),
-        "elapsed_ms": round((time.monotonic() - started) * 1000),
-        "error": None if matched else f"model_not_found:{target}",
-    }
+    candidate = NemotronRuntimeCandidate(
+        label="compatibility_identity",
+        tier="primary",
+        host=host.rstrip("/"),
+        model=NEMOTRON_OLLAMA_MODEL,
+        expected_digest=NEMOTRON_OLLAMA_EXPECTED_DIGEST,
+        request_timeout_sec=DEFAULT_TIMEOUT_SEC,
+        num_predict=160,
+        num_ctx=None,
+    )
+    return inspect_nemotron_model_identity(
+        candidate,
+        request_get=requests.get,
+        timeout_sec=DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC,
+    )
 
 
 def _latest_execution(root: Path, *, now: datetime | None = None) -> dict[str, Any]:
@@ -165,7 +141,12 @@ def _latest_execution(root: Path, *, now: datetime | None = None) -> dict[str, A
         "canary_passed": payload.get("canary_passed") is True,
         "selected_host": payload.get("selected_host"),
         "model": payload.get("model"),
+        "expected_digest": payload.get("expected_digest"),
         "observed_digest": payload.get("observed_digest"),
+        "selected_candidate_label": payload.get("selected_candidate_label"),
+        "fallback_used": payload.get("fallback_used") is True,
+        "degraded_runtime": payload.get("degraded_runtime") is True,
+        "attempt_count": int(payload.get("attempt_count") or 0),
         "decision_tool": payload.get("decision_tool"),
         "decision_sku": payload.get("decision_sku"),
         "model_elapsed_ms": payload.get("model_elapsed_ms"),
@@ -283,67 +264,204 @@ def run_nemotron_decision_canary(
             ),
         }
 
-    host_preflights = [
-        _model_identity(OLLAMA_HOST_PRIMARY),
-        _model_identity(OLLAMA_HOST_SECONDARY),
-    ]
-    selected = next((item for item in host_preflights if item.get("ok")), None)
+    from services.nemoton_dispatcher_service import (
+        ToolCallContractError,
+        _parse_content_fallback,
+        _parse_tool_calls_struct,
+        _remaining_timeout,
+        _validate_tool_call_contract,
+        build_qwen3_dispatch_payload,
+    )
+
+    candidates = build_nemotron_runtime_candidates(
+        primary_model=NEMOTRON_OLLAMA_MODEL,
+        primary_expected_digest=NEMOTRON_OLLAMA_EXPECTED_DIGEST,
+        fallback_model=NEMOTRON_OLLAMA_FALLBACK_MODEL,
+        fallback_expected_digest=NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST,
+    )
+    timeout_cap = max(30, min(int(timeout_sec or DEFAULT_TIMEOUT_SEC), 600))
+    canary_started = time.monotonic()
+    deadline_monotonic = canary_started + timeout_cap
+    host_preflights: list[dict[str, Any]] = []
+    runtime_attempts: list[dict[str, Any]] = []
+    selected: dict[str, Any] | None = None
+    selected_candidate: NemotronRuntimeCandidate | None = None
+    post_identity: dict[str, Any] = {}
     model_call_performed = False
     model_elapsed_ms = 0
     decisions: list[dict[str, Any]] = []
     decision_format = "none"
     error: str | None = None
-    if selected is None:
-        error = "no_approved_gcp_host_with_expected_nemotron_digest"
-    else:
-        from services.nemoton_dispatcher_service import (
-            _parse_content_fallback,
-            _parse_tool_calls_struct,
-            build_qwen3_dispatch_payload,
+    expected_tool = "trigger_price_alert"
+    threat = _SyntheticThreat()
+    for candidate in candidates:
+        attempt: dict[str, Any] = {
+            "candidate": candidate.as_public_dict(),
+            "identity": {},
+            "model_call_performed": False,
+            "model_elapsed_ms": 0,
+            "decision_format": "none",
+            "decision_count": 0,
+            "status": "identity_failed",
+            "error": None,
+        }
+        try:
+            identity_timeout = _remaining_timeout(
+                deadline_monotonic,
+                DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC,
+            )
+        except requests.Timeout as exc:
+            attempt["status"] = "deadline_exhausted"
+            attempt["error"] = str(exc)
+            runtime_attempts.append(attempt)
+            error = str(exc)
+            break
+        identity = inspect_nemotron_model_identity(
+            candidate,
+            request_get=requests.get,
+            timeout_sec=identity_timeout,
         )
+        host_preflights.append(identity)
+        attempt["identity"] = identity
+        attempt["error"] = identity.get("error")
+        if identity.get("ok") is not True:
+            runtime_attempts.append(attempt)
+            error = str(identity.get("error") or "model identity failed")
+            continue
 
-        threat = _SyntheticThreat()
         payload = build_qwen3_dispatch_payload(
             [threat],
             mcp_context="controlled decision-only canary; no live market data",
-            num_predict=160,
+            model=candidate.model,
+            num_predict=candidate.num_predict,
+            num_ctx=candidate.num_ctx,
             keep_alive=0,
         )
-        payload["think"] = False
         started = time.monotonic()
         model_call_performed = True
+        attempt["model_call_performed"] = True
+        candidate_decisions: list[dict[str, Any]] = []
+        candidate_decision_format = "none"
+        candidate_error: str | None = None
+        candidate_contract_error: str | None = None
+        candidate_contract_valid = False
         try:
+            request_timeout = _remaining_timeout(
+                deadline_monotonic,
+                candidate.request_timeout_sec,
+            )
+            attempt["request_timeout_sec"] = round(request_timeout, 3)
             response = requests.post(
-                f"{str(selected['host']).rstrip('/')}/api/chat",
+                f"{candidate.host}/api/chat",
                 json=payload,
-                timeout=max(30, min(int(timeout_sec or DEFAULT_TIMEOUT_SEC), 600)),
+                timeout=request_timeout,
             )
             response.raise_for_status()
             body = response.json()
             message = body.get("message") or {}
-            decisions = _parse_tool_calls_struct(message.get("tool_calls") or [])
-            if decisions:
-                decision_format = "tool_calls"
+            candidate_decisions = _parse_tool_calls_struct(
+                message.get("tool_calls") or []
+            )
+            if candidate_decisions:
+                candidate_decision_format = "tool_calls"
             else:
-                decisions = _parse_content_fallback(str(message.get("content") or ""))
-                if decisions:
-                    decision_format = "content_fallback"
+                candidate_decisions = _parse_content_fallback(
+                    str(message.get("content") or "")
+                )
+                if candidate_decisions:
+                    candidate_decision_format = "content_fallback"
+            try:
+                candidate_decisions = _validate_tool_call_contract(
+                    candidate_decisions,
+                    [threat],
+                )
+                candidate_contract_valid = True
+            except ToolCallContractError as exc:
+                candidate_contract_error = str(exc)[:240]
         except Exception as exc:
-            error = f"{type(exc).__name__}: {str(exc)[:240]}"
+            candidate_error = f"{type(exc).__name__}: {str(exc)[:240]}"
         finally:
-            model_elapsed_ms = round((time.monotonic() - started) * 1000)
+            elapsed_ms = round((time.monotonic() - started) * 1000)
+            model_elapsed_ms += elapsed_ms
+            attempt["model_elapsed_ms"] = elapsed_ms
+
+        candidate_decision = candidate_decisions[0] if candidate_decisions else {}
+        candidate_args = (
+            candidate_decision.get("args")
+            if isinstance(candidate_decision.get("args"), dict)
+            else {}
+        )
+        candidate_tool = str(candidate_decision.get("tool") or "")
+        candidate_sku = str((candidate_args or {}).get("sku") or "")
+        candidate_post_identity: dict[str, Any] = {}
+        if candidate_error is None:
+            try:
+                post_identity_timeout = _remaining_timeout(
+                    deadline_monotonic,
+                    DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC,
+                )
+                candidate_post_identity = inspect_nemotron_model_identity(
+                    candidate,
+                    request_get=requests.get,
+                    timeout_sec=post_identity_timeout,
+                )
+            except requests.Timeout as exc:
+                candidate_error = f"{type(exc).__name__}: {str(exc)[:240]}"
+        candidate_checks = {
+            "approved_host_selected": True,
+            "expected_digest_verified_before": identity.get("digest_matches") is True,
+            "model_call_completed": candidate_error is None,
+            "decision_present": bool(candidate_decisions),
+            "decision_contract_valid": candidate_contract_valid,
+            "decision_tool_allowed": candidate_tool in APPROVED_TOOLS,
+            "expected_decision_tool_selected": candidate_tool == expected_tool,
+            "synthetic_sku_preserved": candidate_sku == _SyntheticThreat.sku,
+            "expected_digest_stable_after": (
+                candidate_post_identity.get("digest_matches") is True
+            ),
+            "tool_execution_absent": True,
+            "database_call_absent": True,
+            "telegram_send_absent": True,
+        }
+        attempt.update({
+            "decision_format": candidate_decision_format,
+            "decision_count": len(candidate_decisions),
+            "decision_tool": candidate_tool or None,
+            "decision_sku": candidate_sku or None,
+            "post_model_identity": candidate_post_identity,
+            "checks": candidate_checks,
+            "status": "passed" if all(candidate_checks.values()) else "failed",
+            "error": candidate_error or candidate_contract_error,
+            "contract_error": candidate_contract_error,
+        })
+        runtime_attempts.append(attempt)
+        if all(candidate_checks.values()):
+            selected = identity
+            selected_candidate = candidate
+            post_identity = candidate_post_identity
+            decisions = candidate_decisions
+            decision_format = candidate_decision_format
+            error = None
+            break
+        error = candidate_error or "candidate decision contract failed"
+
+    if selected_candidate is None:
+        error = (
+            "decision_total_deadline_exhausted"
+            if time.monotonic() >= deadline_monotonic
+            else "all_approved_model_candidates_failed"
+        )
 
     decision = decisions[0] if decisions else {}
     decision_tool = str(decision.get("tool") or "")
     decision_args = decision.get("args") if isinstance(decision.get("args"), dict) else {}
     decision_sku = str((decision_args or {}).get("sku") or "")
-    expected_tool = "trigger_price_alert"
-    post_identity = _model_identity(str(selected["host"])) if selected else {}
     checks = {
-        "approved_host_selected": selected is not None,
+        "approved_host_selected": selected_candidate is not None,
         "expected_digest_verified_before": bool(selected and selected.get("digest_matches")),
-        "model_call_completed": model_call_performed and error is None,
+        "model_call_completed": selected_candidate is not None and error is None,
         "decision_present": bool(decisions),
+        "decision_contract_valid": selected_candidate is not None,
         "decision_tool_allowed": decision_tool in APPROVED_TOOLS,
         "expected_decision_tool_selected": decision_tool == expected_tool,
         "synthetic_sku_preserved": decision_sku == _SyntheticThreat.sku,
@@ -353,7 +471,22 @@ def run_nemotron_decision_canary(
         "telegram_send_absent": True,
     }
     canary_passed = all(checks.values())
-    status = "canary_passed" if canary_passed else "canary_failed"
+    fallback_used = bool(selected_candidate and selected_candidate.is_fallback)
+    status = (
+        "canary_passed_degraded_fallback"
+        if canary_passed and fallback_used
+        else "canary_passed"
+        if canary_passed
+        else "canary_failed"
+    )
+    selected_model = (
+        selected_candidate.model if selected_candidate else NEMOTRON_OLLAMA_MODEL
+    )
+    selected_expected_digest = (
+        selected_candidate.expected_digest
+        if selected_candidate
+        else NEMOTRON_OLLAMA_EXPECTED_DIGEST
+    )
     execution_receipt: dict[str, Any] = {
         "policy": POLICY,
         "canary_version": CANARY_VERSION,
@@ -361,14 +494,24 @@ def run_nemotron_decision_canary(
         "run_identity": run_identity,
         "status": status,
         "canary_passed": canary_passed,
-        "model": NEMOTRON_OLLAMA_MODEL,
-        "expected_digest": NEMOTRON_OLLAMA_EXPECTED_DIGEST,
+        "model": selected_model,
+        "expected_digest": selected_expected_digest,
         "observed_digest": selected.get("digest") if selected else None,
         "selected_host": selected.get("host") if selected else None,
+        "selected_candidate_label": (
+            selected_candidate.label if selected_candidate else None
+        ),
+        "fallback_used": fallback_used,
+        "degraded_runtime": fallback_used,
         "host_preflights": host_preflights,
+        "runtime_attempts": runtime_attempts,
+        "attempt_count": len(runtime_attempts),
         "post_model_identity": post_identity,
         "model_call_performed": model_call_performed,
         "model_elapsed_ms": model_elapsed_ms,
+        "timeout_budget_sec": timeout_cap,
+        "total_elapsed_ms": round((time.monotonic() - canary_started) * 1000),
+        "deadline_exhausted": time.monotonic() >= deadline_monotonic,
         "decision_format": decision_format,
         "decision_count": len(decisions),
         "decision_tool": decision_tool or None,
@@ -388,8 +531,8 @@ def run_nemotron_decision_canary(
         "error": error,
         "rollback_terminal": "decision_only_no_tool_or_data_write",
         "source_of_truth_diff": {
-            "expected_model": NEMOTRON_OLLAMA_MODEL,
-            "expected_digest": NEMOTRON_OLLAMA_EXPECTED_DIGEST,
+            "expected_model": selected_model,
+            "expected_digest": selected_expected_digest,
             "observed_digest": selected.get("digest") if selected else None,
             "expected_decision_tool": expected_tool,
             "observed_decision_tool": decision_tool or None,
@@ -400,7 +543,7 @@ def run_nemotron_decision_canary(
             "source_of_truth_diff_recorded": True,
             "ai_candidate_decision_recorded": bool(decisions),
             "risk_policy_decision": "medium_decision_only_no_tool_execution",
-            "check_mode_passed": bool(selected),
+            "check_mode_passed": canary_passed,
             "bounded_execution_performed": model_call_performed,
             "independent_verifier_passed": canary_passed,
             "rollback_or_no_write_terminal": "decision_only_no_tool_or_data_write",
@@ -444,7 +587,9 @@ def run_nemotron_decision_canary(
             "rollback_terminal": "decision_only_no_tool_or_data_write",
         },
         "next_machine_action": (
-            "continue_scheduled_nemotron_decision_canary"
+            "restore_gcp_decision_runtime_capacity"
+            if canary_passed and fallback_used
+            else "continue_scheduled_nemotron_decision_canary"
             if canary_passed
             else "repair_nemotron_model_runtime_then_retry_decision_only_canary"
         ),

services/nemotron_dispatch_reservation_service.py

@@ -0,0 +1,376 @@
+"""Process-shared reservation store for NemoTron dispatch side effects."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+import time
+import uuid
+from pathlib import Path
+from typing import Callable
+
+try:
+    import fcntl
+except ImportError:  # pragma: no cover - exercised by compatibility monkeypatch
+    fcntl = None
+
+
+logger = logging.getLogger(__name__)
+
+STATE_VERSION = 1
+PRODUCTION_STATE_PATH = Path(
+    "/app/data/ai_automation/nemotron_dispatch_dedupe_state.json"
+)
+
+
+def production_shared_reservation_required() -> bool:
+    return (
+        os.getenv("FLASK_ENV", "").strip().lower() in {"prod", "production"}
+        or os.getenv("USE_POSTGRESQL", "").strip().lower() == "true"
+    )
+
+
+def _sku_key(sku: str) -> str:
+    value = str(sku or "").strip()
+    if not value:
+        return ""
+    return hashlib.sha256(f"nemotron-dispatch:{value}".encode("utf-8")).hexdigest()
+
+
+class ReservationStateError(RuntimeError):
+    pass
+
+
+class SharedFileReservationStore:
+    """Atomic lease/commit state shared by processes using the same bind mount."""
+
+    def __init__(
+        self,
+        state_path: str | Path,
+        *,
+        committed_ttl_sec: int,
+        lease_ttl_sec: int,
+        clock: Callable[[], float] | None = None,
+    ):
+        self.state_path = Path(state_path)
+        self.lock_path = self.state_path.with_suffix(self.state_path.suffix + ".lock")
+        self.committed_ttl_sec = max(1, int(committed_ttl_sec))
+        self.lease_ttl_sec = max(1, int(lease_ttl_sec))
+        self.clock = clock or time.time
+
+    @staticmethod
+    def _empty_state() -> dict:
+        return {"version": STATE_VERSION, "leases": {}, "committed": {}}
+
+    def _read_state(self) -> dict:
+        if not self.state_path.exists():
+            return self._empty_state()
+        try:
+            payload = json.loads(self.state_path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as exc:
+            raise ReservationStateError("reservation_state_unreadable") from exc
+        if not isinstance(payload, dict) or payload.get("version") != STATE_VERSION:
+            raise ReservationStateError("reservation_state_schema_invalid")
+        leases = payload.get("leases")
+        committed = payload.get("committed")
+        if not isinstance(leases, dict) or not isinstance(committed, dict):
+            raise ReservationStateError("reservation_state_schema_invalid")
+        for lease in leases.values():
+            if (
+                not isinstance(lease, dict)
+                or not isinstance(lease.get("token"), str)
+                or not isinstance(lease.get("expires_at"), (int, float))
+                or lease.get("phase", "reserved") not in {
+                    "reserved",
+                    "side_effect_started",
+                }
+            ):
+                raise ReservationStateError("reservation_lease_schema_invalid")
+        for marker in committed.values():
+            if (
+                not isinstance(marker, dict)
+                or not isinstance(marker.get("until"), (int, float))
+            ):
+                raise ReservationStateError("reservation_commit_schema_invalid")
+        return payload
+
+    def _write_state(self, payload: dict) -> None:
+        temporary = self.state_path.with_name(
+            f".{self.state_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
+        )
+        data = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+        try:
+            with temporary.open("w", encoding="utf-8") as handle:
+                handle.write(data)
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.chmod(temporary, 0o640)
+            os.replace(temporary, self.state_path)
+            directory_fd = os.open(str(self.state_path.parent), os.O_RDONLY)
+            try:
+                os.fsync(directory_fd)
+            finally:
+                os.close(directory_fd)
+        finally:
+            try:
+                temporary.unlink(missing_ok=True)
+            except OSError:
+                pass
+
+    @staticmethod
+    def _cleanup(payload: dict, now: float) -> bool:
+        changed = False
+        for key in [
+            key
+            for key, lease in payload["leases"].items()
+            if float(lease["expires_at"]) <= now
+        ]:
+            del payload["leases"][key]
+            changed = True
+        for key in [
+            key
+            for key, marker in payload["committed"].items()
+            if float(marker["until"]) <= now
+        ]:
+            del payload["committed"][key]
+            changed = True
+        return changed
+
+    def _locked(self, operation):
+        if fcntl is None:
+            raise ReservationStateError("process_shared_lock_unavailable")
+        self.state_path.parent.mkdir(mode=0o750, parents=True, exist_ok=True)
+        with self.lock_path.open("a+", encoding="utf-8") as lock_handle:
+            os.chmod(self.lock_path, 0o640)
+            fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX)
+            try:
+                payload = self._read_state()
+                now = float(self.clock())
+                cleanup_changed = self._cleanup(payload, now)
+                result, operation_changed = operation(payload, now)
+                if cleanup_changed or operation_changed:
+                    self._write_state(payload)
+                return result
+            finally:
+                fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN)
+
+    def reserve(self, sku: str) -> str | None:
+        key = _sku_key(sku)
+        if not key:
+            return None
+
+        def operation(payload, now):
+            if key in payload["leases"] or key in payload["committed"]:
+                return None, False
+            token = uuid.uuid4().hex
+            payload["leases"][key] = {
+                "token": token,
+                "expires_at": now + self.lease_ttl_sec,
+                "phase": "reserved",
+            }
+            return token, True
+
+        try:
+            return self._locked(operation)
+        except Exception as exc:
+            logger.error(
+                "[NemotronReservation] shared reserve failed closed: %s",
+                type(exc).__name__,
+            )
+            return None
+
+    def refresh(self, sku: str, token: str) -> bool:
+        key = _sku_key(sku)
+
+        def operation(payload, now):
+            lease = payload["leases"].get(key)
+            if not lease or lease.get("token") != str(token):
+                return False, False
+            ttl = (
+                self.committed_ttl_sec
+                if lease.get("phase") == "side_effect_started"
+                else self.lease_ttl_sec
+            )
+            lease["expires_at"] = now + ttl
+            return True, True
+
+        try:
+            return bool(key and self._locked(operation))
+        except Exception as exc:
+            logger.error(
+                "[NemotronReservation] shared refresh failed closed: %s",
+                type(exc).__name__,
+            )
+            return False
+
+    def begin_side_effect(self, sku: str, token: str) -> bool:
+        """Persist the at-most-once boundary before invoking an external effect."""
+        key = _sku_key(sku)
+
+        def operation(payload, now):
+            lease = payload["leases"].get(key)
+            if not lease or lease.get("token") != str(token):
+                return False, False
+            lease["phase"] = "side_effect_started"
+            lease["expires_at"] = now + self.committed_ttl_sec
+            return True, True
+
+        try:
+            return bool(key and self._locked(operation))
+        except Exception as exc:
+            logger.error(
+                "[NemotronReservation] shared side-effect boundary failed closed: %s",
+                type(exc).__name__,
+            )
+            return False
+
+    def commit(self, sku: str, token: str) -> bool:
+        key = _sku_key(sku)
+
+        def operation(payload, now):
+            lease = payload["leases"].get(key)
+            if not lease or lease.get("token") != str(token):
+                return False, False
+            del payload["leases"][key]
+            payload["committed"][key] = {
+                "until": now + self.committed_ttl_sec,
+            }
+            return True, True
+
+        try:
+            return bool(key and self._locked(operation))
+        except Exception as exc:
+            logger.error(
+                "[NemotronReservation] shared commit failed closed: %s",
+                type(exc).__name__,
+            )
+            return False
+
+    def release(self, sku: str, token: str | None) -> bool:
+        key = _sku_key(sku)
+
+        def operation(payload, _now):
+            lease = payload["leases"].get(key)
+            if not lease or lease.get("token") != str(token or ""):
+                return False, False
+            del payload["leases"][key]
+            return True, True
+
+        try:
+            return bool(key and self._locked(operation))
+        except Exception as exc:
+            logger.error(
+                "[NemotronReservation] shared release failed closed: %s",
+                type(exc).__name__,
+            )
+            return False
+
+    def is_duplicate(self, sku: str) -> bool:
+        key = _sku_key(sku)
+        if not key:
+            return True
+
+        def operation(payload, _now):
+            return (
+                key in payload["leases"] or key in payload["committed"],
+                False,
+            )
+
+        try:
+            return bool(self._locked(operation))
+        except Exception as exc:
+            logger.error(
+                "[NemotronReservation] shared read failed closed: %s",
+                type(exc).__name__,
+            )
+            return True
+
+
+def build_production_shared_reservation_store(
+    *,
+    committed_ttl_sec: int,
+    lease_ttl_sec: int,
+) -> SharedFileReservationStore | None:
+    if not production_shared_reservation_required():
+        return None
+    return SharedFileReservationStore(
+        PRODUCTION_STATE_PATH,
+        committed_ttl_sec=committed_ttl_sec,
+        lease_ttl_sec=lease_ttl_sec,
+    )
+
+
+def run_shared_reservation_canary(
+    *,
+    state_path: str | Path = PRODUCTION_STATE_PATH,
+    committed_ttl_sec: int = 4 * 3600,
+    lease_ttl_sec: int = 15 * 60,
+) -> dict:
+    """Exercise shared ownership semantics without business side effects."""
+    first = SharedFileReservationStore(
+        state_path,
+        committed_ttl_sec=committed_ttl_sec,
+        lease_ttl_sec=lease_ttl_sec,
+    )
+    second = SharedFileReservationStore(
+        state_path,
+        committed_ttl_sec=committed_ttl_sec,
+        lease_ttl_sec=lease_ttl_sec,
+    )
+    synthetic_sku = f"CANARY-NEMOTRON-RESERVATION-{uuid.uuid4().hex}"
+    token = first.reserve(synthetic_sku)
+    competing_token = second.reserve(synthetic_sku) if token else None
+    stale_release_rejected = (
+        second.release(synthetic_sku, "stale-canary-token") is False
+        if token
+        else False
+    )
+    side_effect_boundary_persisted = (
+        first.begin_side_effect(synthetic_sku, token) if token else False
+    )
+    owner_release_succeeded = (
+        first.release(synthetic_sku, token) if token else False
+    )
+    terminal_clear = (
+        first.is_duplicate(synthetic_sku) is False
+        if owner_release_succeeded
+        else False
+    )
+    checks = {
+        "first_owner_reserved": bool(token),
+        "competing_owner_rejected": competing_token is None,
+        "stale_token_release_rejected": stale_release_rejected,
+        "side_effect_boundary_persisted": side_effect_boundary_persisted,
+        "owner_release_succeeded": owner_release_succeeded,
+        "terminal_state_clear": terminal_clear,
+    }
+    success = all(checks.values())
+    return {
+        "success": success,
+        "status": "shared_reservation_canary_passed" if success else "shared_reservation_canary_failed",
+        "policy": "nemotron_process_shared_reservation_v1",
+        "state_path": str(Path(state_path)),
+        "checks": checks,
+        "check_count": len(checks),
+        "check_pass_count": sum(checks.values()),
+        "controlled_apply": {
+            "writes_shared_reservation_state": True,
+            "terminal_state_clear": terminal_clear,
+            "writes_database": False,
+            "sends_telegram": False,
+            "executes_business_tool": False,
+            "reads_secret": False,
+        },
+    }
+
+
+__all__ = [
+    "PRODUCTION_STATE_PATH",
+    "ReservationStateError",
+    "SharedFileReservationStore",
+    "build_production_shared_reservation_store",
+    "production_shared_reservation_required",
+    "run_shared_reservation_canary",
+]

services/nemotron_runtime_candidate_service.py

@@ -0,0 +1,234 @@
+"""Digest-locked Ollama candidates for the decision agent runtime."""
+
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import asdict, dataclass
+from typing import Any, Callable, Mapping
+
+import requests
+
+from services.ollama_service import (
+    OLLAMA_HOST_FALLBACK,
+    OLLAMA_HOST_PRIMARY,
+    OLLAMA_HOST_SECONDARY,
+)
+
+
+PRIMARY_MODEL = os.getenv("NEMOTRON_OLLAMA_MODEL", "qwen3:14b").strip()
+PRIMARY_EXPECTED_DIGEST = os.getenv(
+    "NEMOTRON_OLLAMA_EXPECTED_DIGEST",
+    "bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8",
+).strip().lower()
+FALLBACK_MODEL = os.getenv(
+    "NEMOTRON_OLLAMA_FALLBACK_MODEL",
+    "qwen3:8b",
+).strip()
+FALLBACK_EXPECTED_DIGEST = os.getenv(
+    "NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST",
+    "500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41",
+).strip().lower()
+NEMOTRON_FALLBACK_NUM_CTX = 4096
+NEMOTRON_FALLBACK_NUM_PREDICT = 512
+
+
+def _bounded_int_env(name: str, default: int, minimum: int, maximum: int) -> int:
+    try:
+        value = int(os.getenv(name, str(default)))
+    except (TypeError, ValueError):
+        value = default
+    return max(minimum, min(value, maximum))
+
+
+IDENTITY_TIMEOUT_SEC = _bounded_int_env(
+    "NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC", 10, 2, 30
+)
+GCP_ATTEMPT_TIMEOUT_SEC = _bounded_int_env(
+    "NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC", 60, 30, 180
+)
+FALLBACK_ATTEMPT_TIMEOUT_SEC = _bounded_int_env(
+    "NEMOTRON_111_ATTEMPT_TIMEOUT_SEC", 45, 20, 120
+)
+
+
+@dataclass(frozen=True)
+class NemotronRuntimeCandidate:
+    label: str
+    tier: str
+    host: str
+    model: str
+    expected_digest: str
+    request_timeout_sec: int
+    num_predict: int
+    num_ctx: int | None
+
+    @property
+    def is_fallback(self) -> bool:
+        return self.tier == "fallback"
+
+    def as_public_dict(self) -> dict[str, Any]:
+        payload = asdict(self)
+        payload["is_fallback"] = self.is_fallback
+        return payload
+
+
+def build_nemotron_runtime_candidates(
+    *,
+    primary_model: str | None = None,
+    primary_expected_digest: str | None = None,
+    fallback_model: str | None = None,
+    fallback_expected_digest: str | None = None,
+) -> tuple[NemotronRuntimeCandidate, ...]:
+    """Return the approved GCP-first, 111-final candidate chain."""
+    resolved_primary_model = str(primary_model or PRIMARY_MODEL).strip()
+    resolved_primary_digest = str(
+        primary_expected_digest or PRIMARY_EXPECTED_DIGEST
+    ).strip().lower()
+    resolved_fallback_model = str(fallback_model or FALLBACK_MODEL).strip()
+    resolved_fallback_digest = str(
+        fallback_expected_digest or FALLBACK_EXPECTED_DIGEST
+    ).strip().lower()
+    return (
+        NemotronRuntimeCandidate(
+            label="gcp_primary",
+            tier="primary",
+            host=OLLAMA_HOST_PRIMARY.rstrip("/"),
+            model=resolved_primary_model,
+            expected_digest=resolved_primary_digest,
+            request_timeout_sec=GCP_ATTEMPT_TIMEOUT_SEC,
+            num_predict=2048,
+            num_ctx=None,
+        ),
+        NemotronRuntimeCandidate(
+            label="gcp_secondary",
+            tier="secondary",
+            host=OLLAMA_HOST_SECONDARY.rstrip("/"),
+            model=resolved_primary_model,
+            expected_digest=resolved_primary_digest,
+            request_timeout_sec=GCP_ATTEMPT_TIMEOUT_SEC,
+            num_predict=2048,
+            num_ctx=None,
+        ),
+        NemotronRuntimeCandidate(
+            label="ollama_111_fallback",
+            tier="fallback",
+            host=OLLAMA_HOST_FALLBACK.rstrip("/"),
+            model=resolved_fallback_model,
+            expected_digest=resolved_fallback_digest,
+            request_timeout_sec=FALLBACK_ATTEMPT_TIMEOUT_SEC,
+            num_predict=NEMOTRON_FALLBACK_NUM_PREDICT,
+            num_ctx=NEMOTRON_FALLBACK_NUM_CTX,
+        ),
+    )
+
+
+def inspect_nemotron_model_identity(
+    candidate: NemotronRuntimeCandidate,
+    *,
+    request_get: Callable[..., Any] | None = None,
+    timeout_sec: int | None = None,
+) -> dict[str, Any]:
+    """Verify an exact model digest without loading the model."""
+    started = time.monotonic()
+    getter = request_get or requests.get
+    timeout = max(0.1, min(float(timeout_sec or IDENTITY_TIMEOUT_SEC), 30.0))
+    try:
+        response = getter(f"{candidate.host}/api/tags", timeout=timeout)
+        response.raise_for_status()
+        payload = response.json()
+        if not isinstance(payload, Mapping):
+            raise ValueError("malformed_tags_payload")
+        models = payload.get("models", [])
+        if models is None:
+            models = []
+        if not isinstance(models, list):
+            raise ValueError("malformed_models_schema")
+    except Exception as exc:
+        return {
+            **candidate.as_public_dict(),
+            "ok": False,
+            "digest": None,
+            "digest_matches": False,
+            "elapsed_ms": round((time.monotonic() - started) * 1000),
+            "error": f"{type(exc).__name__}: {str(exc)[:180]}",
+        }
+
+    target = candidate.model
+    target_without_latest = target.removesuffix(":latest")
+    matched: Mapping[str, Any] = {}
+    if any(not isinstance(item, Mapping) for item in models):
+        return {
+            **candidate.as_public_dict(),
+            "ok": False,
+            "digest": None,
+            "digest_matches": False,
+            "parameter_size": "",
+            "quantization_level": "",
+            "elapsed_ms": round((time.monotonic() - started) * 1000),
+            "error": "malformed_models_schema",
+        }
+    if any(
+        "details" in item and not isinstance(item.get("details"), Mapping)
+        for item in models
+    ):
+        return {
+            **candidate.as_public_dict(),
+            "ok": False,
+            "digest": None,
+            "digest_matches": False,
+            "parameter_size": "",
+            "quantization_level": "",
+            "elapsed_ms": round((time.monotonic() - started) * 1000),
+            "error": "malformed_model_details",
+        }
+    for item in models:
+        names = {
+            str(item.get("name") or "").strip(),
+            str(item.get("model") or "").strip(),
+        }
+        if target in names or target_without_latest in names:
+            matched = item
+            break
+    digest = str(matched.get("digest") or "").strip().lower()
+    raw_details = matched.get("details", {}) if matched else {}
+    malformed_details = bool(matched) and not isinstance(raw_details, Mapping)
+    details = raw_details if isinstance(raw_details, Mapping) else {}
+    digest_matches = (
+        bool(digest)
+        and digest == candidate.expected_digest
+        and not malformed_details
+    )
+    error = None
+    if not matched:
+        error = f"model_not_found:{target}"
+    elif malformed_details:
+        error = "malformed_model_details"
+    elif not digest_matches:
+        error = "model_digest_mismatch"
+    return {
+        **candidate.as_public_dict(),
+        "ok": bool(matched) and digest_matches,
+        "digest": digest or None,
+        "digest_matches": digest_matches,
+        "parameter_size": str(details.get("parameter_size") or ""),
+        "quantization_level": str(details.get("quantization_level") or ""),
+        "elapsed_ms": round((time.monotonic() - started) * 1000),
+        "error": error,
+    }
+
+
+__all__ = [
+    "FALLBACK_ATTEMPT_TIMEOUT_SEC",
+    "FALLBACK_EXPECTED_DIGEST",
+    "FALLBACK_MODEL",
+    "GCP_ATTEMPT_TIMEOUT_SEC",
+    "IDENTITY_TIMEOUT_SEC",
+    "NemotronRuntimeCandidate",
+    "NEMOTRON_FALLBACK_NUM_CTX",
+    "NEMOTRON_FALLBACK_NUM_PREDICT",
+    "PRIMARY_EXPECTED_DIGEST",
+    "PRIMARY_MODEL",
+    "build_nemotron_runtime_candidates",
+    "inspect_nemotron_model_identity",
+]

services/ollama_service.py

@@ -47,6 +47,14 @@ def approved_ollama_env(name: str, default: str = '') -> str:
     return default
 
 
+def _bounded_int_env(name: str, default: int, minimum: int, maximum: int) -> int:
+    try:
+        value = int(os.getenv(name, str(default)))
+    except (TypeError, ValueError):
+        value = default
+    return max(minimum, min(value, maximum))
+
+
 # Ollama 設定 - 僅允許 GCP-A → GCP-B → 111 三主機
 OLLAMA_HOST_PRIMARY   = approved_ollama_env('OLLAMA_HOST_PRIMARY',   'http://34.87.90.216:11434')
 OLLAMA_HOST_SECONDARY = approved_ollama_env('OLLAMA_HOST_SECONDARY', 'http://34.21.145.224:11434')
@@ -66,8 +74,8 @@ EMBED_GCP_FAILURE_COOLDOWN_SEC = int(os.getenv('OLLAMA_EMBED_GCP_FAILURE_COOLDOW
 EMBED_GCP_FAILURE_NOTICE_SEC = int(os.getenv('OLLAMA_EMBED_GCP_FAILURE_NOTICE_SEC', '30'))
 FALLBACK_111_KEEP_ALIVE = os.getenv('OLLAMA_111_KEEP_ALIVE', '5m')
 FALLBACK_111_MAX_TIMEOUT = int(os.getenv('OLLAMA_111_MAX_TIMEOUT', '20'))
-FALLBACK_111_NUM_CTX = int(os.getenv('OLLAMA_111_NUM_CTX', '4096'))
-FALLBACK_111_NUM_PREDICT = int(os.getenv('OLLAMA_111_NUM_PREDICT', '512'))
+FALLBACK_111_NUM_CTX = _bounded_int_env('OLLAMA_111_NUM_CTX', 4096, 512, 4096)
+FALLBACK_111_NUM_PREDICT = _bounded_int_env('OLLAMA_111_NUM_PREDICT', 512, 32, 512)
 FALLBACK_111_MODEL = os.getenv('OLLAMA_111_MODEL_FALLBACK', 'llama3.2:latest')
 FALLBACK_111_CIRCUIT_CACHE_SEC = int(os.getenv('OLLAMA_111_CIRCUIT_CACHE_SEC', '60'))
 FALLBACK_111_MODEL_PATTERNS = tuple(