293 lines
11 KiB
JSON
293 lines
11 KiB
JSON
{
|
|
"schema_version": "agent_market_capability_evidence_v1",
|
|
"updated_at": "2026-06-01",
|
|
"baseline_candidate_id": "openclaw_incumbent",
|
|
"scoring_version": "market_capability_v1",
|
|
"dimensions": {
|
|
"durable_execution": 0.15,
|
|
"human_in_loop": 0.14,
|
|
"tool_guardrails": 0.14,
|
|
"observability_tracing": 0.12,
|
|
"evaluation_harness": 0.12,
|
|
"mcp_tool_ecosystem": 0.1,
|
|
"local_private_deploy": 0.08,
|
|
"code_remediation_fit": 0.08,
|
|
"awoooi_integration_fit": 0.07
|
|
},
|
|
"candidates": [
|
|
{
|
|
"candidate_id": "openclaw_incumbent",
|
|
"display_name": "OpenClaw incumbent",
|
|
"evaluation_priority": "baseline",
|
|
"capabilities": {
|
|
"durable_execution": 1,
|
|
"human_in_loop": 3,
|
|
"tool_guardrails": 2,
|
|
"observability_tracing": 2,
|
|
"evaluation_harness": 1,
|
|
"mcp_tool_ecosystem": 2,
|
|
"local_private_deploy": 3,
|
|
"code_remediation_fit": 1,
|
|
"awoooi_integration_fit": 3
|
|
},
|
|
"official_sources": [
|
|
{
|
|
"title": "AWOOOI incumbent baseline snapshot",
|
|
"url": "docs/evaluations/openclaw_incumbent_baseline_2026-06-01.json",
|
|
"evidence": "Current production baseline and local integration evidence."
|
|
}
|
|
],
|
|
"risks": [
|
|
"Current baseline failed the false repair hard gate.",
|
|
"Evaluation harness and durable execution are weaker than several market frameworks."
|
|
]
|
|
},
|
|
{
|
|
"candidate_id": "openai_agents_sdk_coordinator",
|
|
"display_name": "OpenAI Agents SDK Coordinator",
|
|
"evaluation_priority": "must_test",
|
|
"capabilities": {
|
|
"durable_execution": 2,
|
|
"human_in_loop": 3,
|
|
"tool_guardrails": 3,
|
|
"observability_tracing": 3,
|
|
"evaluation_harness": 3,
|
|
"mcp_tool_ecosystem": 3,
|
|
"local_private_deploy": 1,
|
|
"code_remediation_fit": 2,
|
|
"awoooi_integration_fit": 3
|
|
},
|
|
"official_sources": [
|
|
{
|
|
"title": "OpenAI Agents SDK tracing",
|
|
"url": "https://openai.github.io/openai-agents-python/tracing/",
|
|
"evidence": "Built-in tracing covers agent runs, model generations, tool calls, handoffs, guardrails, and custom events."
|
|
},
|
|
{
|
|
"title": "OpenAI Agents SDK guardrails",
|
|
"url": "https://openai.github.io/openai-agents-js/guides/guardrails",
|
|
"evidence": "Tool guardrails can validate or block custom tool calls before and after execution."
|
|
}
|
|
],
|
|
"risks": [
|
|
"Cloud dependency and sensitive trace handling must pass AWOOOI privacy gates.",
|
|
"Built-in hosted execution tools need separate guardrail validation."
|
|
]
|
|
},
|
|
{
|
|
"candidate_id": "nemo_nemotron_fabric",
|
|
"display_name": "NVIDIA NeMo Agent Toolkit + Nemotron Fabric",
|
|
"evaluation_priority": "must_test",
|
|
"capabilities": {
|
|
"durable_execution": 2,
|
|
"human_in_loop": 2,
|
|
"tool_guardrails": 2,
|
|
"observability_tracing": 3,
|
|
"evaluation_harness": 3,
|
|
"mcp_tool_ecosystem": 3,
|
|
"local_private_deploy": 3,
|
|
"code_remediation_fit": 1,
|
|
"awoooi_integration_fit": 3
|
|
},
|
|
"official_sources": [
|
|
{
|
|
"title": "NVIDIA NeMo Agent Toolkit overview",
|
|
"url": "https://docs.nvidia.com/nemo/agent-toolkit/latest/index.html",
|
|
"evidence": "Framework-agnostic agent toolkit with profiling, observability, evaluation, and MCP support."
|
|
},
|
|
{
|
|
"title": "NVIDIA NeMo Agent Toolkit evaluation",
|
|
"url": "https://docs.nvidia.com/nemo/agent-toolkit/latest/workflows/evaluate.html",
|
|
"evidence": "nat eval produces workflow outputs, evaluator outputs, profiling metrics, and request traces."
|
|
}
|
|
],
|
|
"risks": [
|
|
"Needs AWOOOI-specific HITL and dangerous-action policy integration.",
|
|
"GPU/NIM operating cost must be compared against current local inference."
|
|
]
|
|
},
|
|
{
|
|
"candidate_id": "microsoft_agent_framework",
|
|
"display_name": "Microsoft Agent Framework",
|
|
"evaluation_priority": "can_test",
|
|
"capabilities": {
|
|
"durable_execution": 3,
|
|
"human_in_loop": 3,
|
|
"tool_guardrails": 2,
|
|
"observability_tracing": 3,
|
|
"evaluation_harness": 2,
|
|
"mcp_tool_ecosystem": 3,
|
|
"local_private_deploy": 2,
|
|
"code_remediation_fit": 1,
|
|
"awoooi_integration_fit": 2
|
|
},
|
|
"official_sources": [
|
|
{
|
|
"title": "Microsoft Agent Framework overview",
|
|
"url": "https://learn.microsoft.com/en-us/agent-framework/overview/",
|
|
"evidence": "Combines agents, graph workflows, session state, middleware, telemetry, MCP clients, checkpointing, and HITL."
|
|
}
|
|
],
|
|
"risks": [
|
|
"Public preview status and Microsoft ecosystem fit must be assessed.",
|
|
"Python/FastAPI/K8s integration cost is likely higher than LangGraph or NeMo."
|
|
]
|
|
},
|
|
{
|
|
"candidate_id": "langgraph_incident_kernel",
|
|
"display_name": "LangGraph Incident Kernel",
|
|
"evaluation_priority": "must_test",
|
|
"capabilities": {
|
|
"durable_execution": 3,
|
|
"human_in_loop": 3,
|
|
"tool_guardrails": 2,
|
|
"observability_tracing": 2,
|
|
"evaluation_harness": 2,
|
|
"mcp_tool_ecosystem": 2,
|
|
"local_private_deploy": 3,
|
|
"code_remediation_fit": 1,
|
|
"awoooi_integration_fit": 3
|
|
},
|
|
"official_sources": [
|
|
{
|
|
"title": "LangGraph persistence",
|
|
"url": "https://docs.langchain.com/oss/python/langgraph/persistence",
|
|
"evidence": "Checkpoint persistence supports human-in-the-loop, memory, time travel debugging, and fault-tolerant execution."
|
|
},
|
|
{
|
|
"title": "LangGraph interrupts",
|
|
"url": "https://docs.langchain.com/oss/python/langgraph/human-in-the-loop",
|
|
"evidence": "Interrupts pause graph execution and resume through persisted graph state."
|
|
}
|
|
],
|
|
"risks": [
|
|
"It is a workflow kernel, not a smarter model by itself.",
|
|
"Tool safety and evaluation metrics must be implemented by AWOOOI adapters."
|
|
]
|
|
},
|
|
{
|
|
"candidate_id": "claude_agent_sdk_remediator",
|
|
"display_name": "Claude Agent SDK Remediator",
|
|
"evaluation_priority": "must_test",
|
|
"capabilities": {
|
|
"durable_execution": 2,
|
|
"human_in_loop": 3,
|
|
"tool_guardrails": 3,
|
|
"observability_tracing": 2,
|
|
"evaluation_harness": 1,
|
|
"mcp_tool_ecosystem": 3,
|
|
"local_private_deploy": 1,
|
|
"code_remediation_fit": 3,
|
|
"awoooi_integration_fit": 2
|
|
},
|
|
"official_sources": [
|
|
{
|
|
"title": "Claude Agent SDK loop",
|
|
"url": "https://platform.claude.com/docs/en/agent-sdk/agent-loop",
|
|
"evidence": "Embeds Claude Code's autonomous agent loop with programmatic control over tools, permissions, cost limits, and output."
|
|
},
|
|
{
|
|
"title": "Claude Agent SDK overview",
|
|
"url": "https://docs.claude.com/es/api/agent-sdk/overview",
|
|
"evidence": "SDK exposes context management, file operations, code execution, MCP, permissions, sessions, and monitoring."
|
|
}
|
|
],
|
|
"risks": [
|
|
"Best fit is code and DevOps remediation, not necessarily central incident arbitration.",
|
|
"API cost, subscription separation, and vendor boundary must be validated."
|
|
]
|
|
},
|
|
{
|
|
"candidate_id": "claude_managed_agents_sandbox",
|
|
"display_name": "Claude Managed Agents Sandbox",
|
|
"evaluation_priority": "can_test",
|
|
"capabilities": {
|
|
"durable_execution": 3,
|
|
"human_in_loop": 2,
|
|
"tool_guardrails": 3,
|
|
"observability_tracing": 2,
|
|
"evaluation_harness": 1,
|
|
"mcp_tool_ecosystem": 2,
|
|
"local_private_deploy": 2,
|
|
"code_remediation_fit": 3,
|
|
"awoooi_integration_fit": 2
|
|
},
|
|
"official_sources": [
|
|
{
|
|
"title": "Claude Managed Agents quickstart",
|
|
"url": "https://platform.claude.com/docs/en/managed-agents/quickstart",
|
|
"evidence": "Defines agents, environments, sessions, events, and pre-built agent tools for autonomous sessions."
|
|
}
|
|
],
|
|
"risks": [
|
|
"Managed service and beta header make it less suitable as the first AWOOOI core replacement.",
|
|
"Sandbox placement, data retention, and cost must be reviewed before shadow mode."
|
|
]
|
|
},
|
|
{
|
|
"candidate_id": "google_adk_stack",
|
|
"display_name": "Google Agent Development Kit Stack",
|
|
"evaluation_priority": "can_test",
|
|
"capabilities": {
|
|
"durable_execution": 3,
|
|
"human_in_loop": 2,
|
|
"tool_guardrails": 2,
|
|
"observability_tracing": 2,
|
|
"evaluation_harness": 3,
|
|
"mcp_tool_ecosystem": 2,
|
|
"local_private_deploy": 2,
|
|
"code_remediation_fit": 1,
|
|
"awoooi_integration_fit": 2
|
|
},
|
|
"official_sources": [
|
|
{
|
|
"title": "Google ADK technical overview",
|
|
"url": "https://google.github.io/adk-docs/get-started/about/",
|
|
"evidence": "ADK includes session management, state, events, memory, artifacts, evaluation, and developer UI."
|
|
},
|
|
{
|
|
"title": "Google ADK sessions",
|
|
"url": "https://google.github.io/adk-docs/sessions/session/",
|
|
"evidence": "Runner retrieves sessions and exposes state/events to agents."
|
|
}
|
|
],
|
|
"risks": [
|
|
"Gemini/Vertex ecosystem dependency must be justified against current local-first policy.",
|
|
"AIOps tool safety and rollback gates still need AWOOOI-specific implementation."
|
|
]
|
|
},
|
|
{
|
|
"candidate_id": "crewai_flows_crews",
|
|
"display_name": "CrewAI Flows + Crews",
|
|
"evaluation_priority": "secondary",
|
|
"capabilities": {
|
|
"durable_execution": 2,
|
|
"human_in_loop": 2,
|
|
"tool_guardrails": 2,
|
|
"observability_tracing": 2,
|
|
"evaluation_harness": 1,
|
|
"mcp_tool_ecosystem": 2,
|
|
"local_private_deploy": 3,
|
|
"code_remediation_fit": 1,
|
|
"awoooi_integration_fit": 1
|
|
},
|
|
"official_sources": [
|
|
{
|
|
"title": "CrewAI documentation",
|
|
"url": "https://docs.crewai.com/",
|
|
"evidence": "Docs describe agents, crews, flows, guardrails, memory, knowledge, and observability."
|
|
},
|
|
{
|
|
"title": "CrewAI Flows",
|
|
"url": "https://www.crewai.com/crewai-flows",
|
|
"evidence": "Flows coordinate tasks and crews with structured, event-driven workflows and state management."
|
|
}
|
|
],
|
|
"risks": [
|
|
"Better for rapid automation teams than high-risk production AIOps core.",
|
|
"Durability, strict audit, and permission boundary must be proven in replay."
|
|
]
|
|
}
|
|
]
|
|
}
|