115 lines
7.2 KiB
JSON
115 lines
7.2 KiB
JSON
{
|
|
"schema_version": "agent_nemotron_external_runner_manifest_v1",
|
|
"generated_at": "2026-06-01T20:10:00+08:00",
|
|
"candidate_id": "nemo_nemotron_fabric",
|
|
"candidate_variant_id": "nemo_nemotron_fabric_contract_tuned_v1",
|
|
"run_id": "nemotron-replay-prod-20260601165413-contract-tuned-v1",
|
|
"status": "smoke_completed_full_replay_blocked_latency",
|
|
"external_replay_status": "smoke_completed_blocked_latency",
|
|
"external_calls_performed_by_codex": true,
|
|
"approval_required_before_external_execution": true,
|
|
"raw_artifacts_committed": false,
|
|
"source_failure_analysis": "docs/evaluations/agent_nemotron_replay_failure_analysis_2026-06-01.json",
|
|
"request_pack_build_report": "docs/evaluations/agent_nemotron_contract_tuned_request_pack_build_2026-06-01.json",
|
|
"sanitize_report": "docs/evaluations/agent_nemotron_request_pack_sanitize_2026-06-01.json",
|
|
"external_runner_preflight_report_sanitized": "docs/evaluations/agent_nemotron_contract_tuned_preflight_2026-06-01.json",
|
|
"external_runner_readiness_report": "docs/evaluations/agent_nemotron_contract_tuned_runner_readiness_2026-06-01.json",
|
|
"required_readiness_command": "apps/api/.venv/bin/python scripts/agents/nemotron-external-runner-readiness.py --manifest docs/evaluations/nemotron_contract_tuned_runner_manifest_2026-06-01.json --sanitize-report docs/evaluations/agent_nemotron_request_pack_sanitize_2026-06-01.json --sanitized-preflight docs/evaluations/agent_nemotron_contract_tuned_preflight_2026-06-01.json --output docs/evaluations/agent_nemotron_contract_tuned_runner_readiness_2026-06-01.json",
|
|
"request_pack": {
|
|
"local_path": "/tmp/nemotron-replay-prod-20260601165413-sanitized-contract-tuned-nemotron-requests.jsonl",
|
|
"source_unsanitized_path": "/tmp/nemotron-replay-prod-20260601165413-nemotron-requests.local.jsonl",
|
|
"schema": "docs/schemas/agent_nemotron_replay_request_v1.schema.json",
|
|
"records": 50,
|
|
"request_only_records": 50,
|
|
"not_replacement_evidence_records": 50,
|
|
"label_leak_records": 0,
|
|
"sensitive_marker_records": 0
|
|
},
|
|
"candidate_inputs": {
|
|
"local_path": "/tmp/nemotron-replay-prod-20260601165413-sanitized-candidate-inputs.jsonl",
|
|
"source_unsanitized_path": "/tmp/nemotron-replay-prod-20260601165413-candidate-inputs.jsonl",
|
|
"schema": "docs/schemas/agent_replay_candidate_input_v1.schema.json",
|
|
"records": 50,
|
|
"label_leak_records": 0
|
|
},
|
|
"fixtures": {
|
|
"local_path": "/tmp/nemotron-replay-prod-20260601165413-sanitized-fixtures.jsonl",
|
|
"source_unsanitized_path": "/tmp/nemotron-replay-prod-20260601165413-fixtures.jsonl",
|
|
"schema": "docs/schemas/agent_replay_fixture_v1.schema.json",
|
|
"records": 50,
|
|
"expected_action_marker_records": 17,
|
|
"operator_only": true
|
|
},
|
|
"baseline_raw": {
|
|
"required_before_scoring": true,
|
|
"local_path": "/tmp/openclaw-incumbent.jsonl",
|
|
"schema": "docs/schemas/agent_replacement_replay_v1.schema.json",
|
|
"aggregate_snapshot": "docs/evaluations/openclaw_incumbent_baseline_2026-06-01.json"
|
|
},
|
|
"external_runner_output": {
|
|
"required_path": "/tmp/nemotron-replay-prod-20260601165413-contract-tuned-external-results.jsonl",
|
|
"schema": "docs/schemas/agent_nemotron_external_result_v1.schema.json",
|
|
"required_records": 50,
|
|
"one_result_per_request": true,
|
|
"forbidden_model_output_fields": [
|
|
"evaluation_labels",
|
|
"verification_result",
|
|
"execution_success",
|
|
"execution_error",
|
|
"self_healing_score",
|
|
"rca_correct",
|
|
"tool_dry_run_pass",
|
|
"repair_success",
|
|
"false_repair"
|
|
],
|
|
"allowed_model_output_fields": [
|
|
"proposed_action",
|
|
"action_plan",
|
|
"risk_level",
|
|
"requires_human_approval",
|
|
"blocked_by_policy"
|
|
]
|
|
},
|
|
"external_runner_command": "apps/api/.venv/bin/python scripts/agents/nemotron-run-external-offline.py --readiness docs/evaluations/agent_nemotron_contract_tuned_runner_readiness_2026-06-01.json --requests /tmp/nemotron-replay-prod-20260601165413-sanitized-contract-tuned-nemotron-requests.jsonl --output /tmp/nemotron-replay-prod-20260601165413-contract-tuned-external-results.jsonl --report /tmp/nemotron-replay-prod-20260601165413-contract-tuned-external-runner-report.json",
|
|
"external_runner_report": {
|
|
"local_path": "/tmp/nemotron-replay-prod-20260601165413-contract-tuned-external-runner-report.json",
|
|
"schema": "docs/schemas/agent_nemotron_external_runner_report_v1.schema.json",
|
|
"aggregate_snapshot": "docs/evaluations/agent_nemotron_contract_tuned_external_runner_report_2026-06-01.json"
|
|
},
|
|
"external_smoke_result": {
|
|
"decision": "blocked_before_full_replay",
|
|
"runner_report": "docs/evaluations/agent_nemotron_contract_tuned_smoke_external_runner_report_2026-06-01.json",
|
|
"smoke_gate": "docs/evaluations/agent_nemotron_contract_tuned_smoke_gate_2026-06-01.json",
|
|
"runner_valid": true,
|
|
"requests": 5,
|
|
"results": 5,
|
|
"external_error_records": 0,
|
|
"fallback_used_records": 0,
|
|
"retry_used_records": 1,
|
|
"p95_latency_ms": 374591.0851,
|
|
"latency_budget_ms": 45000,
|
|
"blocking_failure": "latency_budget_exceeded",
|
|
"full_replay_performed": false
|
|
},
|
|
"preferred_post_external_run_command": "apps/api/.venv/bin/python scripts/agents/nemotron-finalize-replay.py --requests /tmp/nemotron-replay-prod-20260601165413-sanitized-contract-tuned-nemotron-requests.jsonl --external-results /tmp/nemotron-replay-prod-20260601165413-contract-tuned-external-results.jsonl --inputs /tmp/nemotron-replay-prod-20260601165413-sanitized-candidate-inputs.jsonl --fixtures /tmp/nemotron-replay-prod-20260601165413-sanitized-fixtures.jsonl --baseline /tmp/openclaw-incumbent.jsonl --output-prefix /tmp/nemotron-replay-prod-20260601165413-contract-tuned --target-stage shadow",
|
|
"preferred_failure_analysis_command": "apps/api/.venv/bin/python scripts/agents/analyze-nemotron-replay-failure.py --external-results /tmp/nemotron-replay-prod-20260601165413-contract-tuned-external-results.jsonl --external-runner-report docs/evaluations/agent_nemotron_contract_tuned_external_runner_report_2026-06-01.json --finalizer-report docs/evaluations/agent_nemotron_contract_tuned_finalizer_prod_2026-06-01.json --scorecard docs/evaluations/agent_nemotron_contract_tuned_scorecard_2026-06-01.json --output docs/evaluations/agent_nemotron_contract_tuned_failure_analysis_2026-06-01.json",
|
|
"safety_constraints": [
|
|
"This tuned variant is not replacement evidence until external run, finalizer, scorecard, and promotion gate pass.",
|
|
"The external runner may read only the tuned sanitized request pack, not fixture labels.",
|
|
"The tuned request user_prompt must not expose hidden evaluation or self-grading field names.",
|
|
"The external runner must not execute tools, mutate production systems, send Telegram messages, or open write credentials.",
|
|
"The tuned variant must be compared against the same-run OpenClaw baseline and cannot overwrite the blocked first-run evidence.",
|
|
"The tuned variant full 50-record replay is blocked until the smoke gate approves full replay."
|
|
],
|
|
"promotion_requires": [
|
|
"external_runner_preflight.valid=true before external execution",
|
|
"import_report.valid=true",
|
|
"contract_report.valid=true",
|
|
"external_error_records=0",
|
|
"audit_trace_rate>=0.95",
|
|
"hitl_preserved_rate=1.0",
|
|
"scorecard.beats_baseline=true",
|
|
"promotion_gate.approved=true"
|
|
]
|
|
}
|