76 lines
3.2 KiB
JSON
76 lines
3.2 KiB
JSON
{
|
|
"schema_version": "agent_nemotron_replay_finalizer_smoke_v1",
|
|
"generated_at": "2026-06-01T18:20:00+08:00",
|
|
"source": "local deterministic sample finalizer smoke; no external NIM/API/LLM calls",
|
|
"candidate_id": "nemo_nemotron_fabric",
|
|
"external_calls_performed": false,
|
|
"raw_artifacts_committed": false,
|
|
"command": "apps/api/.venv/bin/python scripts/agents/nemotron-finalize-replay.py --requests /tmp/nemotron-finalizer-request.sample.jsonl --external-results docs/evaluations/examples/agent_nemotron_external_result.sample.jsonl --inputs docs/evaluations/examples/agent_replay_candidate_input.sample.jsonl --fixtures docs/evaluations/examples/agent_replay_fixture.sample.jsonl --baseline docs/evaluations/examples/agent_replacement_replay.sample.jsonl --output-prefix /tmp/nemotron-finalizer-sample --target-stage shadow",
|
|
"exit_code": 2,
|
|
"decision": "blocked",
|
|
"approved": false,
|
|
"expected_block_reason": "sample smoke has 1 replay incident and cannot satisfy the 50-incident promotion threshold",
|
|
"failures": [
|
|
"scorecard_not_eligible_for_canary",
|
|
"sample_too_small:1<50"
|
|
],
|
|
"import_report": {
|
|
"valid": true,
|
|
"external_results": 1,
|
|
"imported_results": 1,
|
|
"requests": 1,
|
|
"missing_results": [],
|
|
"unexpected_results": [],
|
|
"duplicate_results": [],
|
|
"external_error_records": 0,
|
|
"fallback_used_records": 0,
|
|
"incomplete_trace_records": 0,
|
|
"model_distribution": {
|
|
"nvidia/nemotron-mini-4b-instruct": 1
|
|
}
|
|
},
|
|
"contract_report": {
|
|
"valid": true,
|
|
"inputs": 1,
|
|
"results": 1,
|
|
"failures": []
|
|
},
|
|
"pipeline_report": {
|
|
"contract_valid": true,
|
|
"normalized_records": 1,
|
|
"graded_records": 1,
|
|
"label_grading_applied": true,
|
|
"scorecard_written": true,
|
|
"baseline_records": 1,
|
|
"ignored_nonbaseline_records": 1
|
|
},
|
|
"promotion_gate": {
|
|
"import_report_provided": true,
|
|
"import_report_valid": true,
|
|
"candidate_result_error_records": 0,
|
|
"not_replacement_evidence_records": 0,
|
|
"contract_probe_records": 0,
|
|
"beats_baseline": true,
|
|
"hard_gates_pass": true,
|
|
"eligible_for_canary": false
|
|
},
|
|
"local_artifact_paths": {
|
|
"request_pack": "/tmp/nemotron-finalizer-request.sample.jsonl",
|
|
"candidate_raw": "/tmp/nemotron-finalizer-sample-candidate-raw.jsonl",
|
|
"import_report": "/tmp/nemotron-finalizer-sample-import-report.json",
|
|
"contract_report": "/tmp/nemotron-finalizer-sample-contract-report.json",
|
|
"normalized_output": "/tmp/nemotron-finalizer-sample-candidate-normalized.jsonl",
|
|
"graded_output": "/tmp/nemotron-finalizer-sample-candidate-graded.jsonl",
|
|
"grading_report": "/tmp/nemotron-finalizer-sample-grading-report.json",
|
|
"scorecard": "/tmp/nemotron-finalizer-sample-scorecard.json",
|
|
"promotion_gate": "/tmp/nemotron-finalizer-sample-promotion-gate.json",
|
|
"summary": "/tmp/nemotron-finalizer-sample-finalizer-summary.json"
|
|
},
|
|
"notes": [
|
|
"This smoke proves the finalizer wires import report evidence into the promotion gate.",
|
|
"The import, contract, normalization, grading, scoring, and promotion gate steps all executed locally.",
|
|
"The sample is intentionally blocked because it has only one replay incident.",
|
|
"Raw JSONL artifacts remain local operator artifacts and are not committed."
|
|
]
|
|
}
|