138 lines
5.2 KiB
JSON
138 lines
5.2 KiB
JSON
{
|
|
"schema_version": "agent_nemotron_contract_tuned_smoke_matrix_v1",
|
|
"generated_at": "2026-06-02T10:27:22+08:00",
|
|
"candidate_id": "nemo_nemotron_fabric",
|
|
"candidate_variant_id": "nemo_nemotron_fabric_contract_tuned_v1",
|
|
"run_id": "nemotron-replay-prod-20260602095438",
|
|
"latency_budget_ms": 45000,
|
|
"full_replay_allowed": false,
|
|
"decision": "all_tested_nemotron_smokes_blocked_before_full_replay",
|
|
"tested_models": [
|
|
{
|
|
"model": "nvidia/nemotron-3-super-120b-a12b",
|
|
"tested_at": "2026-06-01",
|
|
"requests": 5,
|
|
"results": 5,
|
|
"runner_valid": true,
|
|
"external_error_records": 0,
|
|
"fallback_used_records": 0,
|
|
"trace_incomplete_records": 0,
|
|
"retry_used_records": 1,
|
|
"avg_latency_ms": 213890.3999,
|
|
"p95_latency_ms": 374591.0851,
|
|
"smoke_gate_decision": "blocked",
|
|
"blocking_failures": [
|
|
"latency_budget_exceeded"
|
|
],
|
|
"runner_report": "docs/evaluations/agent_nemotron_contract_tuned_smoke_external_runner_report_2026-06-01.json",
|
|
"smoke_gate": "docs/evaluations/agent_nemotron_contract_tuned_smoke_gate_2026-06-01.json"
|
|
},
|
|
{
|
|
"model": "nvidia/nvidia-nemotron-nano-9b-v2",
|
|
"tested_at": "2026-06-02",
|
|
"requests": 5,
|
|
"results": 5,
|
|
"runner_valid": true,
|
|
"external_error_records": 0,
|
|
"fallback_used_records": 5,
|
|
"trace_incomplete_records": 5,
|
|
"retry_used_records": 0,
|
|
"avg_latency_ms": 60103.0275,
|
|
"p95_latency_ms": 60108.6491,
|
|
"smoke_gate_decision": "blocked",
|
|
"blocking_failures": [
|
|
"fallbacks_present",
|
|
"trace_incomplete_records_present",
|
|
"latency_budget_exceeded"
|
|
],
|
|
"runner_report": "docs/evaluations/agent_nemotron_contract_tuned_nano9b_smoke_external_runner_report_2026-06-02.json",
|
|
"smoke_gate": "docs/evaluations/agent_nemotron_contract_tuned_nano9b_smoke_gate_2026-06-02.json"
|
|
},
|
|
{
|
|
"model": "nvidia/nemotron-mini-4b-instruct",
|
|
"tested_at": "2026-06-02",
|
|
"requests": 5,
|
|
"results": 5,
|
|
"runner_valid": false,
|
|
"external_error_records": 5,
|
|
"fallback_used_records": 5,
|
|
"trace_incomplete_records": 5,
|
|
"retry_used_records": 0,
|
|
"avg_latency_ms": 527.5488,
|
|
"p95_latency_ms": 681.8552,
|
|
"smoke_gate_decision": "blocked",
|
|
"blocking_failures": [
|
|
"runner_invalid",
|
|
"external_errors_present",
|
|
"fallbacks_present",
|
|
"trace_incomplete_records_present"
|
|
],
|
|
"runner_report": "docs/evaluations/agent_nemotron_contract_tuned_mini4b_smoke_external_runner_report_2026-06-02.json",
|
|
"smoke_gate": "docs/evaluations/agent_nemotron_contract_tuned_mini4b_smoke_gate_2026-06-02.json"
|
|
},
|
|
{
|
|
"model": "nvidia/nemotron-3-nano-30b-a3b",
|
|
"tested_at": "2026-06-02",
|
|
"requests": 5,
|
|
"results": 5,
|
|
"runner_valid": false,
|
|
"external_error_records": 4,
|
|
"fallback_used_records": 4,
|
|
"trace_incomplete_records": 4,
|
|
"retry_used_records": 5,
|
|
"avg_latency_ms": 8836.9188,
|
|
"p95_latency_ms": 11180.4184,
|
|
"smoke_gate_decision": "blocked",
|
|
"blocking_failures": [
|
|
"runner_invalid",
|
|
"external_errors_present",
|
|
"fallbacks_present",
|
|
"trace_incomplete_records_present"
|
|
],
|
|
"runner_report": "docs/evaluations/agent_nemotron_contract_tuned_nemotron3nano30b_smoke_external_runner_report_2026-06-02.json",
|
|
"smoke_gate": "docs/evaluations/agent_nemotron_contract_tuned_nemotron3nano30b_smoke_gate_2026-06-02.json"
|
|
},
|
|
{
|
|
"model": "nvidia/llama-3.3-nemotron-super-49b-v1.5",
|
|
"tested_at": "2026-06-02",
|
|
"requests": 5,
|
|
"results": 5,
|
|
"runner_valid": true,
|
|
"external_error_records": 0,
|
|
"fallback_used_records": 0,
|
|
"trace_incomplete_records": 0,
|
|
"retry_used_records": 2,
|
|
"avg_latency_ms": 40121.8494,
|
|
"p95_latency_ms": 67191.2835,
|
|
"smoke_gate_decision": "blocked",
|
|
"blocking_failures": [
|
|
"latency_budget_exceeded"
|
|
],
|
|
"runner_report": "docs/evaluations/agent_nemotron_contract_tuned_49b_v15_smoke_external_runner_report_2026-06-02.json",
|
|
"smoke_gate": "docs/evaluations/agent_nemotron_contract_tuned_49b_v15_smoke_gate_2026-06-02.json"
|
|
}
|
|
],
|
|
"best_observed_models": {
|
|
"best_contract_reliability": "nvidia/llama-3.3-nemotron-super-49b-v1.5",
|
|
"best_latency": "nvidia/nemotron-mini-4b-instruct",
|
|
"best_balanced_candidate": "nvidia/llama-3.3-nemotron-super-49b-v1.5",
|
|
"best_balanced_limit": "p95 latency still exceeds the 45s smoke-gate budget"
|
|
},
|
|
"professional_decision": {
|
|
"may_replace_openclaw": false,
|
|
"may_enter_shadow": false,
|
|
"may_enter_canary": false,
|
|
"may_run_full_50_replay": false,
|
|
"recommended_role": [
|
|
"offline specialist",
|
|
"agent-fabric evaluator",
|
|
"NIM runtime candidate after stricter JSON enforcement or latency reduction"
|
|
],
|
|
"next_safe_steps": [
|
|
"Do not run full replay until a Nemotron-family model passes the 5-record smoke gate.",
|
|
"For Nemotron 3 Nano 30B, investigate stricter structured-output enforcement before another smoke.",
|
|
"For Nemotron 49B v1.5, investigate latency reduction before another smoke."
|
|
]
|
|
}
|
|
}
|