""" NeMo/Nemotron Contract-Tuned Smoke Gate ======================================= Evaluates whether a short external runner smoke is safe to expand into a full 50-record replay. This gate is local-only and uses aggregate runner reports. """ from __future__ import annotations from dataclasses import dataclass, field from typing import Any from src.services.agent_nemotron_replay_adapter import ( NEMOTRON_CANDIDATE_ID, NEMOTRON_CONTRACT_TUNED_VARIANT_ID, ) SMOKE_GATE_SCHEMA_VERSION = "agent_nemotron_contract_tuned_smoke_gate_v1" DEFAULT_MINIMUM_RECORDS = 5 DEFAULT_LATENCY_BUDGET_MS = 45_000.0 @dataclass(frozen=True) class NemotronContractTunedSmokeGateReport: """Decision report for expanding a tuned smoke into full replay.""" approved_for_full_replay: bool decision: str model: str minimum_records: int = DEFAULT_MINIMUM_RECORDS latency_budget_ms: float = DEFAULT_LATENCY_BUDGET_MS gates: dict[str, bool] = field(default_factory=dict) failures: list[str] = field(default_factory=list) runner_summary: dict[str, Any] = field(default_factory=dict) source_reports: dict[str, str] = field(default_factory=dict) def to_dict(self) -> dict[str, Any]: return { "schema_version": SMOKE_GATE_SCHEMA_VERSION, "candidate_id": NEMOTRON_CANDIDATE_ID, "candidate_variant_id": NEMOTRON_CONTRACT_TUNED_VARIANT_ID, "approved_for_full_replay": self.approved_for_full_replay, "decision": self.decision, "model": self.model, "minimum_records": self.minimum_records, "latency_budget_ms": self.latency_budget_ms, "gates": dict(self.gates), "failures": list(self.failures), "runner_summary": dict(self.runner_summary), "source_reports": dict(self.source_reports), } def evaluate_nemotron_contract_tuned_smoke_gate( *, runner_report: dict[str, Any], source_reports: dict[str, str] | None = None, minimum_records: int = DEFAULT_MINIMUM_RECORDS, latency_budget_ms: float = DEFAULT_LATENCY_BUDGET_MS, ) -> NemotronContractTunedSmokeGateReport: """Evaluate if a tuned smoke may expand to the full replay pack.""" failures: list[str] = [] gates: dict[str, bool] = {} def gate(name: str, passed: bool, failure: str) -> None: gates[name] = bool(passed) if not passed: failures.append(failure) requests = int(runner_report.get("requests") or 0) results = int(runner_report.get("results") or 0) p95_latency_ms = float(runner_report.get("p95_latency_ms") or 0.0) gate("runner_valid", runner_report.get("valid") is True, "runner_invalid") gate( "candidate_variant_is_contract_tuned_v1", runner_report.get("candidate_variant_id") == NEMOTRON_CONTRACT_TUNED_VARIANT_ID, "candidate_variant_mismatch", ) gate( "minimum_records_met", requests >= minimum_records and results >= minimum_records, "minimum_records_not_met", ) gate( "all_requests_returned_results", requests == results and requests > 0, "requests_results_mismatch", ) gate( "no_external_errors", int(runner_report.get("external_error_records") or 0) == 0, "external_errors_present", ) gate( "no_fallbacks", int(runner_report.get("fallback_used_records") or 0) == 0, "fallbacks_present", ) gate( "trace_complete", int(runner_report.get("trace_incomplete_records") or 0) == 0, "trace_incomplete_records_present", ) gate( "latency_budget_met", p95_latency_ms <= latency_budget_ms, "latency_budget_exceeded", ) approved = not failures return NemotronContractTunedSmokeGateReport( approved_for_full_replay=approved, decision="approved_for_full_replay" if approved else "blocked", model=str(runner_report.get("model") or ""), minimum_records=minimum_records, latency_budget_ms=latency_budget_ms, gates=gates, failures=failures, runner_summary={ "requests": requests, "results": results, "valid": bool(runner_report.get("valid")), "external_error_records": int( runner_report.get("external_error_records") or 0 ), "fallback_used_records": int( runner_report.get("fallback_used_records") or 0 ), "trace_incomplete_records": int( runner_report.get("trace_incomplete_records") or 0 ), "retry_used_records": int(runner_report.get("retry_used_records") or 0), "avg_latency_ms": float(runner_report.get("avg_latency_ms") or 0.0), "p95_latency_ms": p95_latency_ms, }, source_reports=dict(source_reports or {}), )