235 lines
9.0 KiB
JSON
235 lines
9.0 KiB
JSON
{
|
|
"generated_at": "2026-06-02T04:07:18.592087+00:00",
|
|
"inputs": {
|
|
"candidate_registry_schema_version": "agent_replacement_candidates_v1",
|
|
"scorecard_schema_version": "agent_market_capability_scorecard_v1",
|
|
"scorecard_scoring_version": "market_capability_v1",
|
|
"watch_report_generated_at": "2026-06-02T03:49:10.165431+00:00",
|
|
"watch_report_mode": "live",
|
|
"watch_summary": {
|
|
"candidate_count": 7,
|
|
"changed_candidates": 2,
|
|
"failure_count": 0,
|
|
"integration_queue_count": 2,
|
|
"source_count": 20,
|
|
"watch_only_candidates": 5
|
|
}
|
|
},
|
|
"policy": {
|
|
"paid_api_calls_approved": false,
|
|
"production_changes_approved": false,
|
|
"raw_external_pages_committed": false,
|
|
"replacement_decision_allowed": false,
|
|
"sdk_installation_approved": false,
|
|
"shadow_or_canary_approved": false
|
|
},
|
|
"reviews": [
|
|
{
|
|
"approval_boundary": {
|
|
"approved_for_paid_api_calls": false,
|
|
"approved_for_sdk_install": false,
|
|
"approved_for_shadow_or_canary": false,
|
|
"requires_cost_approval": true,
|
|
"requires_dependency_approval": true
|
|
},
|
|
"candidate_id": "nemo_nemotron_fabric",
|
|
"decision": "do_not_integrate_refresh_evidence_then_smoke_gate",
|
|
"display_name": "NVIDIA NeMo Agent Toolkit + Nemotron Fabric",
|
|
"market_score": {
|
|
"beats_baseline_capability": true,
|
|
"gaps": [
|
|
"code_remediation_fit"
|
|
],
|
|
"known": true,
|
|
"rank": 3,
|
|
"replay_priority": "p0_replay",
|
|
"risks": [
|
|
"Needs AWOOOI-specific HITL and dangerous-action policy integration.",
|
|
"GPU/NIM operating cost must be compared against current local inference."
|
|
],
|
|
"strengths": [
|
|
"observability_tracing",
|
|
"evaluation_harness",
|
|
"mcp_tool_ecosystem",
|
|
"local_private_deploy",
|
|
"awoooi_integration_fit"
|
|
],
|
|
"total_score": 0.8033
|
|
},
|
|
"market_watch": {
|
|
"changed_sources": [
|
|
{
|
|
"change_basis": "version_or_content_hash_changed",
|
|
"content_hash": "fd8fbe0acb2737726d98c77c",
|
|
"error": null,
|
|
"http_status": 200,
|
|
"published_at": null,
|
|
"source_id": "nvidia_build_models",
|
|
"status": "ok",
|
|
"type": "docs",
|
|
"url": "https://build.nvidia.com/models",
|
|
"version": null
|
|
}
|
|
],
|
|
"decision": "changed_requires_replay_readiness_review",
|
|
"recommended_actions": [
|
|
"refresh_market_capability_evidence",
|
|
"refresh_or_create_no_cost_adapter",
|
|
"run_offline_replay_before_shadow",
|
|
"do_not_promote_without_promotion_gate"
|
|
]
|
|
},
|
|
"readiness": {
|
|
"allowed_next_gate": "refresh_source_evidence_then_5_record_smoke_only",
|
|
"reason": "Nemotron smoke/replay evidence blocks full replay, shadow, and canary.",
|
|
"stage": "blocked_existing_replay_evidence"
|
|
},
|
|
"recommendations": [
|
|
"refresh_market_capability_evidence_from_changed_primary_sources",
|
|
"do_not_replace_openclaw_from_market_watch_signal",
|
|
"do_not_enter_shadow_or_canary_without_offline_replay_promotion_gate",
|
|
"keep_candidate_as_offline_specialist_or_evaluator",
|
|
"rerun_only_5_record_smoke_after_a_specific_runtime_or_model_hypothesis",
|
|
"do_not_run_full_50_replay_until_smoke_gate_passes",
|
|
"cost_boundary_review_required",
|
|
"dependency_boundary_review_required",
|
|
"candidate_role_scope:agent_fabric_tool_model_evaluator"
|
|
],
|
|
"registry_status": {
|
|
"current_decision": "all_contract_tuned_nemotron_smokes_blocked_before_full_replay",
|
|
"evaluation_priority": "must_test",
|
|
"latest_replay_summary": null,
|
|
"latest_smoke_gate": "docs/evaluations/agent_nemotron_contract_tuned_49b_v15_smoke_gate_2026-06-02.json",
|
|
"latest_smoke_matrix": "docs/evaluations/agent_nemotron_contract_tuned_smoke_matrix_2026-06-02.json",
|
|
"latest_smoke_model": "nvidia/llama-3.3-nemotron-super-49b-v1.5",
|
|
"next_variant_id": "nemo_nemotron_fabric_contract_tuned_v1",
|
|
"next_variant_stage": "blocked_before_full_replay_all_tested_smokes",
|
|
"required_stage": "offline_replay",
|
|
"role": "agent_fabric_tool_model_evaluator"
|
|
},
|
|
"unblock_conditions": [
|
|
"changed_sources_reviewed_by_operator",
|
|
"market_scorecard_refreshed_if_primary_sources_changed_semantically",
|
|
"no_sdk_install_without_dependency_approval",
|
|
"no_paid_provider_use_without_cost_and_data_boundary_approval",
|
|
"5_record_smoke_gate_passes",
|
|
"latency_and_output_contract_blockers_resolved",
|
|
"cost_approval_recorded"
|
|
]
|
|
},
|
|
{
|
|
"approval_boundary": {
|
|
"approved_for_paid_api_calls": false,
|
|
"approved_for_sdk_install": false,
|
|
"approved_for_shadow_or_canary": false,
|
|
"requires_cost_approval": true,
|
|
"requires_dependency_approval": true
|
|
},
|
|
"candidate_id": "claude_agent_sdk_remediator",
|
|
"decision": "do_not_integrate_refresh_replay_gate",
|
|
"display_name": "Claude Agent SDK Remediator",
|
|
"market_score": {
|
|
"beats_baseline_capability": true,
|
|
"gaps": [
|
|
"evaluation_harness",
|
|
"local_private_deploy"
|
|
],
|
|
"known": true,
|
|
"rank": 5,
|
|
"replay_priority": "p0_replay",
|
|
"risks": [
|
|
"Best fit is code and DevOps remediation, not necessarily central incident arbitration.",
|
|
"API cost, subscription separation, and vendor boundary must be validated."
|
|
],
|
|
"strengths": [
|
|
"human_in_loop",
|
|
"tool_guardrails",
|
|
"mcp_tool_ecosystem",
|
|
"code_remediation_fit"
|
|
],
|
|
"total_score": 0.7533
|
|
},
|
|
"market_watch": {
|
|
"changed_sources": [
|
|
{
|
|
"change_basis": "version_or_content_hash_changed",
|
|
"content_hash": "4b2b5807eb03fbc03616f198",
|
|
"error": null,
|
|
"http_status": 200,
|
|
"published_at": null,
|
|
"source_id": "claude_agent_sdk_docs",
|
|
"status": "ok",
|
|
"type": "docs",
|
|
"url": "https://platform.claude.com/docs/en/agent-sdk/agent-loop",
|
|
"version": null
|
|
},
|
|
{
|
|
"change_basis": "version_or_content_hash_changed",
|
|
"content_hash": "d5af8907bbca468ea3f694d9",
|
|
"error": null,
|
|
"http_status": 200,
|
|
"published_at": null,
|
|
"source_id": "anthropic_api_docs",
|
|
"status": "ok",
|
|
"type": "docs",
|
|
"url": "https://platform.claude.com/docs/en/home",
|
|
"version": null
|
|
}
|
|
],
|
|
"decision": "changed_requires_replay_readiness_review",
|
|
"recommended_actions": [
|
|
"refresh_market_capability_evidence",
|
|
"refresh_or_create_no_cost_adapter",
|
|
"run_offline_replay_before_shadow",
|
|
"do_not_promote_without_promotion_gate"
|
|
]
|
|
},
|
|
"readiness": {
|
|
"allowed_next_gate": "refresh_scorecard_then_offline_replay_or_promotion_gate",
|
|
"reason": "Candidate has an offline replay summary and must re-enter promotion gate after evidence refresh.",
|
|
"stage": "has_offline_replay_summary"
|
|
},
|
|
"recommendations": [
|
|
"refresh_market_capability_evidence_from_changed_primary_sources",
|
|
"do_not_replace_openclaw_from_market_watch_signal",
|
|
"do_not_enter_shadow_or_canary_without_offline_replay_promotion_gate",
|
|
"rerun_same_contract_offline_replay_before_promotion_gate",
|
|
"cost_boundary_review_required",
|
|
"dependency_boundary_review_required",
|
|
"candidate_role_scope:devops_code_remediation_agent"
|
|
],
|
|
"registry_status": {
|
|
"current_decision": "deterministic_offline_remediator_blocked_does_not_beat_openclaw",
|
|
"evaluation_priority": "must_test",
|
|
"latest_replay_summary": "docs/evaluations/agent_claude_remediator_replay_summary_2026-06-02.json",
|
|
"latest_smoke_gate": null,
|
|
"latest_smoke_matrix": null,
|
|
"latest_smoke_model": null,
|
|
"next_variant_id": null,
|
|
"next_variant_stage": null,
|
|
"required_stage": "offline_replay",
|
|
"role": "devops_code_remediation_agent"
|
|
},
|
|
"unblock_conditions": [
|
|
"changed_sources_reviewed_by_operator",
|
|
"market_scorecard_refreshed_if_primary_sources_changed_semantically",
|
|
"no_sdk_install_without_dependency_approval",
|
|
"no_paid_provider_use_without_cost_and_data_boundary_approval",
|
|
"offline_adapter_contract_valid",
|
|
"50_record_hidden_label_replay_beats_openclaw_baseline",
|
|
"cost_approval_recorded"
|
|
]
|
|
}
|
|
],
|
|
"schema_version": "agent_market_integration_review_v1",
|
|
"summary": {
|
|
"blocked_from_integration": 2,
|
|
"production_changes_approved": 0,
|
|
"requires_cost_approval": 2,
|
|
"requires_dependency_approval": 2,
|
|
"reviewed_candidates": 2,
|
|
"shadow_or_canary_approved": 0,
|
|
"source_failures": 0
|
|
}
|
|
}
|