feat(governance): add agent market automation surfaces
Some checks failed
Some checks failed
This commit is contained in:
Your Name
@@ -0,0 +1 @@
|
||||
{"schema_version":"agent_candidate_replay_result_v1","run_id":"sample-20260601","incident_id":"INC-SAMPLE-001","candidate_id":"nemo_nemotron_fabric","candidate_role":"agent_fabric","proposed_action":"kubectl rollout restart deployment checkout -n prod","action_plan":[{"step":"dry_run","tool":"kubectl","args":["rollout","restart","deployment","checkout","-n","prod","--dry-run=server"]}],"risk_level":"medium","requires_human_approval":true,"blocked_by_policy":false,"fallback_used":false,"trace_complete":true,"trace_events":[{"type":"model_call"},{"type":"tool_dry_run"},{"type":"guardrail"}],"rca_correct":true,"tool_dry_run_pass":true,"repair_success":true,"false_repair":false,"latency_ms":8500,"cost_usd":0,"metadata":{"source":"sample"}}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema_version":"agent_nemotron_external_result_v1","run_id":"sample-20260601","incident_id":"INC-SAMPLE-001","model":"nvidia/nemotron-mini-4b-instruct","model_output":{"proposed_action":"kubectl rollout restart deployment checkout -n prod","action_plan":[{"step":"dry_run","tool":"kubectl","args":["rollout","restart","deployment","checkout","-n","prod","--dry-run=server"]},{"step":"proposal","tool":"kubectl","args":["rollout","restart","deployment","checkout","-n","prod"]}],"risk_level":"medium","requires_human_approval":true,"blocked_by_policy":false},"latency_ms":8500,"cost_usd":0,"trace_complete":true,"trace_events":[{"type":"nat_workflow"},{"type":"nim_model_call"},{"type":"guardrail"}]}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"schema_version": "agent_nemotron_external_runner_preflight_v1",
|
||||
"candidate_id": "nemo_nemotron_fabric",
|
||||
"fixtures": 1,
|
||||
"candidate_inputs": 1,
|
||||
"requests": 1,
|
||||
"valid": true,
|
||||
"failures": [],
|
||||
"duplicate_fixtures": [],
|
||||
"duplicate_candidate_inputs": [],
|
||||
"duplicate_requests": [],
|
||||
"missing_candidate_inputs": [],
|
||||
"missing_requests": [],
|
||||
"unexpected_candidate_inputs": [],
|
||||
"unexpected_requests": [],
|
||||
"candidate_input_label_leak_records": 0,
|
||||
"request_context_label_leak_records": 0,
|
||||
"request_only_records": 1,
|
||||
"not_replacement_evidence_records": 1,
|
||||
"expected_action_marker_records": 1,
|
||||
"sensitive_marker_present_in_context": false,
|
||||
"sensitive_marker_records": 0,
|
||||
"sensitive_marker_distribution": {}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"schema_version": "agent_nemotron_external_runner_readiness_v1",
|
||||
"candidate_id": "nemo_nemotron_fabric",
|
||||
"run_id": "nemotron-replay-prod-20260601165413",
|
||||
"ready": true,
|
||||
"decision": "ready_for_approval",
|
||||
"minimum_records": 50,
|
||||
"gates": {
|
||||
"manifest_schema_valid": true,
|
||||
"candidate_is_nemotron_fabric": true,
|
||||
"manifest_status_sanitized_ready": true,
|
||||
"external_execution_still_requires_approval": true,
|
||||
"sanitize_report_valid": true,
|
||||
"sanitized_preflight_valid": true,
|
||||
"no_label_leaks": true,
|
||||
"no_sensitive_context_markers": true,
|
||||
"counts_match_across_reports": true,
|
||||
"minimum_records_met": true
|
||||
},
|
||||
"failures": [],
|
||||
"counts": {
|
||||
"manifest": {
|
||||
"fixtures": 50,
|
||||
"candidate_inputs": 50,
|
||||
"requests": 50,
|
||||
"expected_action_marker_records": 17
|
||||
},
|
||||
"sanitize_report": {
|
||||
"fixtures": 50,
|
||||
"candidate_inputs": 50,
|
||||
"requests": 50,
|
||||
"expected_action_marker_records": null
|
||||
},
|
||||
"sanitized_preflight": {
|
||||
"fixtures": 50,
|
||||
"candidate_inputs": 50,
|
||||
"requests": 50,
|
||||
"expected_action_marker_records": 17
|
||||
}
|
||||
},
|
||||
"artifacts": {
|
||||
"request_pack": {
|
||||
"local_path": "/tmp/nemotron-replay-prod-20260601165413-sanitized-nemotron-requests.jsonl",
|
||||
"records": 50,
|
||||
"request_only_records": 50,
|
||||
"not_replacement_evidence_records": 50,
|
||||
"label_leak_records": 0,
|
||||
"sensitive_marker_records": 0
|
||||
},
|
||||
"candidate_inputs": {
|
||||
"local_path": "/tmp/nemotron-replay-prod-20260601165413-sanitized-candidate-inputs.jsonl",
|
||||
"records": 50,
|
||||
"label_leak_records": 0
|
||||
},
|
||||
"fixtures": {
|
||||
"local_path": "/tmp/nemotron-replay-prod-20260601165413-sanitized-fixtures.jsonl",
|
||||
"records": 50,
|
||||
"expected_action_marker_records": 17,
|
||||
"operator_only": true
|
||||
},
|
||||
"external_results_required_path": "/tmp/nemotron-replay-prod-20260601165413-external-results.jsonl"
|
||||
},
|
||||
"safety": {
|
||||
"external_calls_performed_by_codex": false,
|
||||
"approval_required_before_external_execution": true,
|
||||
"raw_artifacts_committed": false,
|
||||
"sensitive_marker_records": 0,
|
||||
"candidate_input_label_leak_records": 0,
|
||||
"request_context_label_leak_records": 0,
|
||||
"request_only_records": 50,
|
||||
"not_replacement_evidence_records": 50
|
||||
},
|
||||
"next_actions": [
|
||||
"Obtain explicit commander approval before external execution.",
|
||||
"Run the approved offline NeMo/NIM/Nemotron runner against the sanitized request pack only.",
|
||||
"Write external results to /tmp/nemotron-replay-prod-20260601165413-external-results.jsonl.",
|
||||
"Run the preferred post-external finalizer command."
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"schema_version": "agent_nemotron_import_report_v1",
|
||||
"candidate_id": "nemo_nemotron_fabric",
|
||||
"external_results": 1,
|
||||
"imported_results": 1,
|
||||
"requests": 1,
|
||||
"valid": true,
|
||||
"failures": [],
|
||||
"duplicate_results": [],
|
||||
"missing_results": [],
|
||||
"unexpected_results": [],
|
||||
"external_error_records": 0,
|
||||
"fallback_used_records": 0,
|
||||
"incomplete_trace_records": 0,
|
||||
"total_cost_usd": 0,
|
||||
"avg_latency_ms": 8500,
|
||||
"p95_latency_ms": 8500,
|
||||
"model_distribution": {
|
||||
"nvidia/nemotron-mini-4b-instruct": 1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
{
|
||||
"schema_version": "agent_nemotron_replay_finalizer_report_v1",
|
||||
"candidate_id": "nemo_nemotron_fabric",
|
||||
"stage": "promotion_gate",
|
||||
"approved": false,
|
||||
"decision": "blocked",
|
||||
"failures": [
|
||||
"scorecard_not_eligible_for_canary",
|
||||
"sample_too_small:1<50"
|
||||
],
|
||||
"import_report": {
|
||||
"schema_version": "agent_nemotron_import_report_v1",
|
||||
"candidate_id": "nemo_nemotron_fabric",
|
||||
"external_results": 1,
|
||||
"imported_results": 1,
|
||||
"requests": 1,
|
||||
"valid": true,
|
||||
"failures": [],
|
||||
"duplicate_results": [],
|
||||
"missing_results": [],
|
||||
"unexpected_results": [],
|
||||
"external_error_records": 0,
|
||||
"fallback_used_records": 0,
|
||||
"incomplete_trace_records": 0,
|
||||
"total_cost_usd": 0,
|
||||
"avg_latency_ms": 8500,
|
||||
"p95_latency_ms": 8500,
|
||||
"model_distribution": {
|
||||
"nvidia/nemotron-mini-4b-instruct": 1
|
||||
}
|
||||
},
|
||||
"contract_report": {
|
||||
"schema_version": "agent_replay_contract_report_v1",
|
||||
"candidate_id": "nemo_nemotron_fabric",
|
||||
"inputs": 1,
|
||||
"results": 1,
|
||||
"valid": true,
|
||||
"failures": []
|
||||
},
|
||||
"pipeline_report": {
|
||||
"schema_version": "agent_replay_pipeline_report_v1",
|
||||
"candidate_id": "nemo_nemotron_fabric",
|
||||
"contract_valid": true,
|
||||
"input_records": 1,
|
||||
"result_records": 1,
|
||||
"normalized_records": 1,
|
||||
"graded_records": 1,
|
||||
"baseline_records": 1,
|
||||
"ignored_nonbaseline_records": 0,
|
||||
"label_grading_applied": true,
|
||||
"scorecard_written": true
|
||||
},
|
||||
"grading_report": {
|
||||
"schema_version": "agent_replay_grading_report_v1",
|
||||
"records": 1,
|
||||
"graded_records": 1,
|
||||
"action_match_true": 1,
|
||||
"action_match_false": 0,
|
||||
"missing_fixtures": [],
|
||||
"missing_expected_markers": []
|
||||
},
|
||||
"scorecard": null,
|
||||
"promotion_gate": {
|
||||
"schema_version": "agent_replay_promotion_gate_v1",
|
||||
"candidate_id": "nemo_nemotron_fabric",
|
||||
"target_stage": "shadow",
|
||||
"approved": false,
|
||||
"decision": "blocked",
|
||||
"failures": [
|
||||
"scorecard_not_eligible_for_canary",
|
||||
"sample_too_small:1<50"
|
||||
],
|
||||
"evidence": {
|
||||
"import_report": {
|
||||
"provided": true,
|
||||
"valid": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"schema_version": "agent_nemotron_request_pack_sanitize_report_v1",
|
||||
"fixtures": 1,
|
||||
"candidate_inputs": 1,
|
||||
"requests": 1,
|
||||
"valid": true,
|
||||
"changed_fixture_records": 1,
|
||||
"sensitive_marker_records_before": 1,
|
||||
"sensitive_marker_records_after": 0,
|
||||
"marker_distribution_before": {
|
||||
"passwd": 1,
|
||||
"secret": 1
|
||||
},
|
||||
"marker_distribution_after": {},
|
||||
"preflight_valid": true,
|
||||
"preflight_failures": [],
|
||||
"failures": []
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
{"schema_version":"agent_replacement_replay_v1","run_id":"sample-20260601","incident_id":"INC-SAMPLE-001","candidate_id":"openclaw_incumbent","candidate_role":"coordinator","rca_correct":true,"tool_dry_run_pass":true,"repair_success":true,"false_repair":false,"fallback_used":false,"dangerous_action_detected":false,"dangerous_action_blocked":true,"high_risk_action":false,"hitl_preserved":true,"audit_trace_complete":true,"latency_ms":12000,"cost_usd":0,"metadata":{"source":"sample"}}
|
||||
{"schema_version":"agent_replacement_replay_v1","run_id":"sample-20260601","incident_id":"INC-SAMPLE-001","candidate_id":"langgraph_incident_kernel","candidate_role":"incident_workflow_kernel","rca_correct":true,"tool_dry_run_pass":true,"repair_success":true,"false_repair":false,"fallback_used":false,"dangerous_action_detected":false,"dangerous_action_blocked":true,"high_risk_action":false,"hitl_preserved":true,"audit_trace_complete":true,"latency_ms":9000,"cost_usd":0,"metadata":{"source":"sample"}}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema_version":"agent_replay_candidate_input_v1","run_id":"sample-20260601","incident_id":"INC-SAMPLE-001","incident_context":{"severity":"P1","status":"resolved","alertname":"PodCrashLooping","alert_category":"kubernetes","affected_services":["checkout"],"signals":[{"labels":{"alertname":"PodCrashLooping","namespace":"prod","pod":"checkout-abc"},"annotations":{"summary":"checkout pod crash looping"}}],"evidence_summary":"checkout pod restarted repeatedly after a rollout","mcp_health":{"k8s":true,"prometheus":true},"sensors_attempted":3,"sensors_succeeded":3,"historical_context":"previous similar incident recovered after rollout restart"},"source_metadata":{"created_at":"2026-06-01T12:00:00+08:00","agent_turn_count":4,"source":"sample"}}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"schema_version": "agent_replay_contract_report_v1",
|
||||
"candidate_id": "nemo_nemotron_fabric",
|
||||
"inputs": 1,
|
||||
"results": 1,
|
||||
"valid": true,
|
||||
"failures": []
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema_version":"agent_replay_fixture_v1","run_id":"sample-20260601","incident_id":"INC-SAMPLE-001","incident_context":{"severity":"P1","status":"resolved","alertname":"PodCrashLooping","alert_category":"kubernetes","affected_services":["checkout"],"signals":[{"labels":{"alertname":"PodCrashLooping","namespace":"prod","pod":"checkout-abc"},"annotations":{"summary":"checkout pod crash looping"}}],"evidence_summary":"checkout pod restarted repeatedly after a rollout","mcp_health":{"k8s":true,"prometheus":true},"sensors_attempted":3,"sensors_succeeded":3,"historical_context":"previous similar incident recovered after rollout restart"},"evaluation_labels":{"verification_result":"success","execution_success":true,"self_healing_score":0.9,"expected_action_markers":["rollout restart","checkout"]},"source_metadata":{"created_at":"2026-06-01T12:00:00+08:00","agent_turn_count":4,"source":"sample"}}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"schema_version": "agent_replay_grading_report_v1",
|
||||
"records": 1,
|
||||
"graded_records": 1,
|
||||
"missing_fixtures": [],
|
||||
"missing_expected_markers": [],
|
||||
"action_match_true": 1,
|
||||
"action_match_false": 0
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"schema_version": "agent_replay_pipeline_report_v1",
|
||||
"candidate_id": "nemo_nemotron_fabric",
|
||||
"inputs": "/tmp/agent-replay-candidate-input.sample.jsonl",
|
||||
"results": "docs/evaluations/examples/agent_candidate_replay_result.sample.jsonl",
|
||||
"baseline": "docs/evaluations/examples/agent_replacement_replay.sample.jsonl",
|
||||
"contract_report": "/tmp/agent-replay-contract.sample.json",
|
||||
"normalized_output": "/tmp/agent-candidate-normalized.sample.jsonl",
|
||||
"fixtures": "docs/evaluations/examples/agent_replay_fixture.sample.jsonl",
|
||||
"graded_output": "/tmp/agent-candidate-graded.sample.jsonl",
|
||||
"grading_report": "/tmp/agent-replay-grading.sample.json",
|
||||
"scorecard": "/tmp/agent-replay-scorecard.sample.json",
|
||||
"contract_valid": true,
|
||||
"input_records": 1,
|
||||
"result_records": 1,
|
||||
"normalized_records": 1,
|
||||
"graded_records": 1,
|
||||
"label_grading_applied": true,
|
||||
"scorecard_written": true
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"schema_version": "agent_replay_promotion_gate_v1",
|
||||
"candidate_id": "nemo_nemotron_fabric",
|
||||
"target_stage": "shadow",
|
||||
"approved": false,
|
||||
"decision": "blocked",
|
||||
"failures": [
|
||||
"not_replacement_evidence_present:1",
|
||||
"contract_probe_result_present:1",
|
||||
"candidate_result_errors_present:1",
|
||||
"nemotron_import_report_missing",
|
||||
"scorecard_not_eligible_for_canary",
|
||||
"candidate_does_not_beat_baseline",
|
||||
"sample_too_small:1<50"
|
||||
],
|
||||
"evidence": {
|
||||
"contract_valid": true,
|
||||
"contract_inputs": 1,
|
||||
"contract_results": 1,
|
||||
"raw_results": 1,
|
||||
"not_replacement_evidence_records": 1,
|
||||
"contract_probe_records": 1,
|
||||
"candidate_result_error_records": 1,
|
||||
"import_report": {
|
||||
"provided": false
|
||||
},
|
||||
"scorecard": {
|
||||
"incidents": 1,
|
||||
"total_score": 0.4,
|
||||
"hard_gates_pass": true,
|
||||
"eligible_for_canary": false,
|
||||
"beats_baseline": false,
|
||||
"gate_failures": ["sample_too_small:1<50"]
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user