139 lines
4.8 KiB
Python
139 lines
4.8 KiB
Python
"""
|
|
NeMo/Nemotron Contract-Tuned Smoke Gate
|
|
=======================================
|
|
|
|
Evaluates whether a short external runner smoke is safe to expand into a full
|
|
50-record replay. This gate is local-only and uses aggregate runner reports.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from src.services.agent_nemotron_replay_adapter import (
|
|
NEMOTRON_CANDIDATE_ID,
|
|
NEMOTRON_CONTRACT_TUNED_VARIANT_ID,
|
|
)
|
|
|
|
SMOKE_GATE_SCHEMA_VERSION = "agent_nemotron_contract_tuned_smoke_gate_v1"
|
|
DEFAULT_MINIMUM_RECORDS = 5
|
|
DEFAULT_LATENCY_BUDGET_MS = 45_000.0
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class NemotronContractTunedSmokeGateReport:
|
|
"""Decision report for expanding a tuned smoke into full replay."""
|
|
|
|
approved_for_full_replay: bool
|
|
decision: str
|
|
model: str
|
|
minimum_records: int = DEFAULT_MINIMUM_RECORDS
|
|
latency_budget_ms: float = DEFAULT_LATENCY_BUDGET_MS
|
|
gates: dict[str, bool] = field(default_factory=dict)
|
|
failures: list[str] = field(default_factory=list)
|
|
runner_summary: dict[str, Any] = field(default_factory=dict)
|
|
source_reports: dict[str, str] = field(default_factory=dict)
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": SMOKE_GATE_SCHEMA_VERSION,
|
|
"candidate_id": NEMOTRON_CANDIDATE_ID,
|
|
"candidate_variant_id": NEMOTRON_CONTRACT_TUNED_VARIANT_ID,
|
|
"approved_for_full_replay": self.approved_for_full_replay,
|
|
"decision": self.decision,
|
|
"model": self.model,
|
|
"minimum_records": self.minimum_records,
|
|
"latency_budget_ms": self.latency_budget_ms,
|
|
"gates": dict(self.gates),
|
|
"failures": list(self.failures),
|
|
"runner_summary": dict(self.runner_summary),
|
|
"source_reports": dict(self.source_reports),
|
|
}
|
|
|
|
|
|
def evaluate_nemotron_contract_tuned_smoke_gate(
|
|
*,
|
|
runner_report: dict[str, Any],
|
|
source_reports: dict[str, str] | None = None,
|
|
minimum_records: int = DEFAULT_MINIMUM_RECORDS,
|
|
latency_budget_ms: float = DEFAULT_LATENCY_BUDGET_MS,
|
|
) -> NemotronContractTunedSmokeGateReport:
|
|
"""Evaluate if a tuned smoke may expand to the full replay pack."""
|
|
failures: list[str] = []
|
|
gates: dict[str, bool] = {}
|
|
|
|
def gate(name: str, passed: bool, failure: str) -> None:
|
|
gates[name] = bool(passed)
|
|
if not passed:
|
|
failures.append(failure)
|
|
|
|
requests = int(runner_report.get("requests") or 0)
|
|
results = int(runner_report.get("results") or 0)
|
|
p95_latency_ms = float(runner_report.get("p95_latency_ms") or 0.0)
|
|
gate("runner_valid", runner_report.get("valid") is True, "runner_invalid")
|
|
gate(
|
|
"candidate_variant_is_contract_tuned_v1",
|
|
runner_report.get("candidate_variant_id") == NEMOTRON_CONTRACT_TUNED_VARIANT_ID,
|
|
"candidate_variant_mismatch",
|
|
)
|
|
gate(
|
|
"minimum_records_met",
|
|
requests >= minimum_records and results >= minimum_records,
|
|
"minimum_records_not_met",
|
|
)
|
|
gate(
|
|
"all_requests_returned_results",
|
|
requests == results and requests > 0,
|
|
"requests_results_mismatch",
|
|
)
|
|
gate(
|
|
"no_external_errors",
|
|
int(runner_report.get("external_error_records") or 0) == 0,
|
|
"external_errors_present",
|
|
)
|
|
gate(
|
|
"no_fallbacks",
|
|
int(runner_report.get("fallback_used_records") or 0) == 0,
|
|
"fallbacks_present",
|
|
)
|
|
gate(
|
|
"trace_complete",
|
|
int(runner_report.get("trace_incomplete_records") or 0) == 0,
|
|
"trace_incomplete_records_present",
|
|
)
|
|
gate(
|
|
"latency_budget_met",
|
|
p95_latency_ms <= latency_budget_ms,
|
|
"latency_budget_exceeded",
|
|
)
|
|
|
|
approved = not failures
|
|
return NemotronContractTunedSmokeGateReport(
|
|
approved_for_full_replay=approved,
|
|
decision="approved_for_full_replay" if approved else "blocked",
|
|
model=str(runner_report.get("model") or ""),
|
|
minimum_records=minimum_records,
|
|
latency_budget_ms=latency_budget_ms,
|
|
gates=gates,
|
|
failures=failures,
|
|
runner_summary={
|
|
"requests": requests,
|
|
"results": results,
|
|
"valid": bool(runner_report.get("valid")),
|
|
"external_error_records": int(
|
|
runner_report.get("external_error_records") or 0
|
|
),
|
|
"fallback_used_records": int(
|
|
runner_report.get("fallback_used_records") or 0
|
|
),
|
|
"trace_incomplete_records": int(
|
|
runner_report.get("trace_incomplete_records") or 0
|
|
),
|
|
"retry_used_records": int(runner_report.get("retry_used_records") or 0),
|
|
"avg_latency_ms": float(runner_report.get("avg_latency_ms") or 0.0),
|
|
"p95_latency_ms": p95_latency_ms,
|
|
},
|
|
source_reports=dict(source_reports or {}),
|
|
)
|