Files
awoooi/docs/evaluations/ai_provider_route_matrix_2026-06-05.json

389 lines
16 KiB
JSON
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"schema_version": "ai_provider_route_matrix_v1",
"generated_at": "2026-07-14T18:30:00+08:00",
"program_status": {
"overall_completion_percent": 100,
"current_priority": "P1",
"current_task_id": "P1-004",
"next_task_id": "P1-005",
"read_only_mode": true
},
"source_refs": [
"docs/schemas/ai_provider_route_matrix_v1.schema.json",
"docs/HARD_RULES.md#cost-change-approval",
"docs/HARD_RULES.md#iwooos-security-governance",
"apps/api/src/services/ai_router.py",
"apps/api/src/services/ollama_endpoint_resolver.py",
"apps/api/src/services/ollama_failover_manager.py",
"apps/api/src/services/openclaw.py",
"apps/api/src/services/platform_operator_service.py",
"apps/api/src/services/ai_providers/openclaw_nemo.py",
"apps/api/src/services/ai_providers/nemotron.py",
"apps/api/src/services/ai_providers/permissions.py",
"apps/api/models.json",
"k8s/awoooi-prod/04-configmap.yaml",
"k8s/awoooi-prod/06-deployment-api.yaml",
"docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md",
"docs/ai/AI_AGENT_AUTOMATION_WORKLIST_2026-06-04.md"
],
"rollups": {
"total_routes": 7,
"by_kind": {
"ai_router_core": 1,
"ollama_failover": 1,
"alert_governance_lane": 1,
"openclaw_nemo": 1,
"nemotron_candidate": 1,
"paid_cloud_fallback": 1,
"legacy_registry": 1
},
"by_status": {
"verified": 5,
"action_required": 1,
"blocked": 1
},
"by_route_gate": {
"route_preserved": 5,
"review_required": 1,
"candidate_blocked": 1
},
"route_ids_requiring_action": [
"openclaw_nemo_rca_lane"
],
"candidate_gate_ids_requiring_approval": [
"nemotron_replay_gate",
"paid_provider_call_gate",
"provider_switch_gate"
],
"source_gap_ids": [
"ai_router_comment_drift",
"legacy_model_registry_order_gap",
"machine_backlog_next_task_gap"
],
"read_only_denials_total": 12,
"provider_switch_allowed_count": 0,
"paid_api_call_allowed_count": 0,
"shadow_or_canary_allowed_count": 0,
"runtime_route_change_allowed_count": 0
},
"provider_routes": [
{
"route_id": "ai_router_execution_policy",
"display_name": "AI Router 執行決策核心",
"kind": "ai_router_core",
"status": "verified",
"risk_level": "critical",
"route_gate": "route_preserved",
"evidence_status": "committed_source",
"current_policy": "AIRouter 的 intent、risk、complexity 只選 Ollama model所有 production generation 共用 GCP-A → GCP-B → host111 → Gemini 四跳,不得直選 Claude / NVIDIA / Nemo / OpenClaw。",
"provider_order": [
"ollama_gcp_a",
"ollama_gcp_b",
"ollama_local",
"gemini"
],
"fallback_policy": "所有 risk / complexity / tool / background / code-review / runbook 路徑共用單一 production chainGemini disabled 時不得被重新加入。",
"evidence_refs": [
"apps/api/src/services/ai_router.py"
],
"next_action": "只讀顯示路由矩陣;任何 provider override 或 fallback chain 變更都需另開批准包。"
},
{
"route_id": "ollama_global_endpoint_order",
"display_name": "Ollama GCP-A → GCP-B → 111",
"kind": "ollama_failover",
"status": "verified",
"risk_level": "critical",
"route_gate": "route_preserved",
"evidence_status": "committed_manifest",
"current_policy": "生產 ConfigMap 與 deployment env 維持 OLLAMA_URL、OLLAMA_SECONDARY_URL、OLLAMA_FALLBACK_URLpolicy order 是 GCP-A → GCP-B → 111再到 Gemini final fallback。",
"provider_order": [
"ollama_gcp_a",
"ollama_gcp_b",
"ollama_local",
"gemini"
],
"fallback_policy": "OllamaFailoverManager 只在 primary 不健康時檢查後兩層SLOW Ollama 仍優先於直接燒 Gemini quota。",
"evidence_refs": [
"apps/api/src/services/ollama_endpoint_resolver.py",
"apps/api/src/services/ollama_failover_manager.py",
"k8s/awoooi-prod/04-configmap.yaml",
"k8s/awoooi-prod/06-deployment-api.yaml"
],
"next_action": "後續 P1-005 可只讀比對 health 缺口;不得在本任務改 OLLAMA_* 或重啟任何 host / pod。"
},
{
"route_id": "alert_ai_ollama_first_lane",
"display_name": "告警 AI Ollama-first Lane",
"kind": "alert_governance_lane",
"status": "verified",
"risk_level": "critical",
"route_gate": "route_preserved",
"evidence_status": "committed_source",
"current_policy": "告警與 AI governance lane 固定走 GCP-A / GCP-B / host111cloud fallback 只允許 Gemini final hop。",
"provider_order": [
"ollama_gcp_a",
"ollama_gcp_b",
"ollama_local",
"gemini"
],
"fallback_policy": "ALERT_AI_ENFORCE_OLLAMA_FIRST=trueALERT_AI_ALLOW_CLOUD_FALLBACK=true 只代表既有 final fallback不代表新增付費呼叫批准。",
"evidence_refs": [
"apps/api/src/services/openclaw.py",
"k8s/awoooi-prod/04-configmap.yaml",
"k8s/awoooi-prod/06-deployment-api.yaml"
],
"next_action": "在 AwoooP Runs / timeline 補 route evidence 時只能讀取既有 route status不得提高 Gemini 呼叫頻率。"
},
{
"route_id": "openclaw_nemo_rca_lane",
"display_name": "OpenClaw Nemo RCA Lane",
"kind": "openclaw_nemo",
"status": "action_required",
"risk_level": "high",
"route_gate": "review_required",
"evidence_status": "committed_source",
"current_policy": "OpenClawNemoProvider 僅保留 non-executable shadow metadata不得成為 production RCA provider。",
"provider_order": [
"ollama_gcp_a",
"ollama_gcp_b",
"ollama_local",
"gemini"
],
"fallback_policy": "Production fallback 不含 OpenClaw/Nemo/NVIDIA/Claude候選只可離線 replay。",
"evidence_refs": [
"apps/api/src/services/ai_providers/openclaw_nemo.py",
"apps/api/src/services/ai_router.py",
"docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md"
],
"next_action": "後續以 replay / shadow / canary 證據比較候選;驗收前不得宣稱 OpenClaw 可被取代。"
},
{
"route_id": "nemotron_tool_calling_candidate",
"display_name": "Nemotron Tool Calling 候選",
"kind": "nemotron_candidate",
"status": "blocked",
"risk_level": "critical",
"route_gate": "candidate_blocked",
"evidence_status": "blocked_replay_recorded",
"current_policy": "NemotronProvider 可作 tool calling / 離線 evaluator但最近 replay / fast-model smoke 仍未過 latency 與品質 gate不得進 production route、shadow 或 canary。",
"provider_order": [
"nemotron"
],
"fallback_policy": "只保留離線 smoke / replay / sanitized request pack任何 NVIDIA/NIM live 呼叫需另有批准與成本邊界。",
"evidence_refs": [
"apps/api/src/services/ai_providers/nemotron.py",
"apps/api/src/services/ai_providers/permissions.py",
"docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md",
"docs/ai/AI_AGENT_AUTOMATION_WORKLIST_2026-06-04.md"
],
"next_action": "P3 才刷新 primary source evidence 與 5 筆 smoke不得把 P1-004 matrix 當作 Nemotron 升級批准。"
},
{
"route_id": "gemini_final_fallback_policy",
"display_name": "Gemini Final Fallback",
"kind": "paid_cloud_fallback",
"status": "verified",
"risk_level": "critical",
"route_gate": "route_preserved",
"evidence_status": "committed_source",
"current_policy": "Gemini 是三層 Ollama 之後唯一 final paid fallbackconfigured、enabled、authenticated、verified 與 cost guard 必須分開顯示。",
"provider_order": [
"ollama_gcp_a",
"ollama_gcp_b",
"ollama_local",
"gemini"
],
"fallback_policy": "Gemini 只能由原子 requests/tokens/cost reservation 放行disabled 不重加unknown model/pricing 必須 durable blocked receipt。",
"evidence_refs": [
"docs/HARD_RULES.md#cost-change-approval",
"apps/api/src/services/openclaw.py",
"apps/api/src/services/ollama_failover_manager.py",
"apps/api/models.json"
],
"next_action": "P2-004 才能提出成本與 fallback 優化;本階段保持現狀。"
},
{
"route_id": "legacy_models_json_policy",
"display_name": "legacy models.json 路由註記",
"kind": "legacy_registry",
"status": "verified",
"risk_level": "medium",
"route_gate": "route_preserved",
"evidence_status": "committed_source",
"current_policy": "apps/api/models.json 已發布四跳 production routeClaude/NVIDIA/Nemo/OpenClaw 僅保留 non-executable shadow metadata。",
"provider_order": [
"ollama_gcp_a",
"ollama_gcp_b",
"ollama_local",
"gemini"
],
"fallback_policy": "Registry、ConfigMap、AI Router source 與 runtime readback 必須一致source-only 仍不得宣稱 production closure。",
"evidence_refs": [
"apps/api/models.json",
"k8s/awoooi-prod/04-configmap.yaml",
"k8s/awoooi-prod/06-deployment-api.yaml"
],
"next_action": "持續以 route validator 阻擋 provider insertionruntime closure 仍需 CD 與 production readback。"
}
],
"candidate_gates": [
{
"gate_id": "provider_switch_gate",
"display_name": "Provider 切換 Gate",
"status": "production_change_blocked",
"approval_required": true,
"summary": "provider switch、production routing change、USE_AI_ROUTER 切換與 fallback order 修改全部需要獨立人工批准。",
"evidence_refs": [
"docs/HARD_RULES.md#iwooos-security-governance",
"docs/HARD_RULES.md#cost-change-approval"
],
"next_action": "若未來要切 provider需先準備 replay、shadow/canary、成本與 rollback 批准包。"
},
{
"gate_id": "nemotron_replay_gate",
"display_name": "Nemotron Replay Gate",
"status": "blocked_by_evidence",
"approval_required": true,
"summary": "Nemotron 目前僅保留為離線 specialist / evaluator不得取代 OpenClaw 或進 shadow/canary。",
"evidence_refs": [
"docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md",
"docs/ai/AI_AGENT_AUTOMATION_WORKLIST_2026-06-04.md"
],
"next_action": "P3 refresh source evidence 後,僅能先做 5 筆 smoke。"
},
{
"gate_id": "paid_provider_call_gate",
"display_name": "付費 Provider 呼叫 Gate",
"status": "cost_approval_required",
"approval_required": true,
"summary": "Gemini / Claude / NVIDIA external call、quota 或頻率提升都需先做成本影響與人工批准。",
"evidence_refs": [
"docs/HARD_RULES.md#cost-change-approval",
"apps/api/models.json"
],
"next_action": "P1-004 不做 live paid callP2-004 才能整理成本優化提案。"
}
],
"source_gaps": [
{
"gap_id": "ai_router_comment_drift",
"display_name": "AI Router 註解與現況 drift",
"status": "resolved_in_source",
"severity": "low",
"summary": "ai_router.py 現行說明已明確標示舊版路由 supersededprovider 固定四跳。",
"evidence_refs": [
"apps/api/src/services/ai_router.py"
],
"next_action": "維持路由負向測試,阻擋舊 provider 重新進入 executable chain。"
},
{
"gap_id": "legacy_model_registry_order_gap",
"display_name": "models.json 與 runtime env 判讀差異",
"status": "resolved_in_source",
"severity": "low",
"summary": "models.json、ConfigMap 與 AI Router 已共用四跳 provider identityendpoint URL 仍以 runtime env 與 production readback 為準。",
"evidence_refs": [
"apps/api/models.json",
"k8s/awoooi-prod/06-deployment-api.yaml"
],
"next_action": "維持 source truth precedenceruntime receipt / env / AI Router / registry / proposal。"
},
{
"gap_id": "machine_backlog_next_task_gap",
"display_name": "P1-005 / P1-006 機器待辦同步缺口",
"status": "proposal_required",
"severity": "low",
"summary": "工作清單 MD 列出 P1-005 / P1-006但目前 committed backlog JSON 仍未納入這兩項P1-004 完成後需在下一階段對齊。",
"evidence_refs": [
"docs/ai/AI_AGENT_AUTOMATION_WORKLIST_2026-06-04.md",
"docs/evaluations/ai_agent_automation_backlog_2026-06-04.json"
],
"next_action": "P1-005 啟動前先決定是否補 machine backlog item避免 UI next_task 跟 MD 不一致。"
}
],
"latest_observations": [
{
"observation_id": "prod_manifest_ai_router_enabled",
"status": "verified",
"summary": "生產 manifest committed env 顯示 USE_AI_ROUTER=trueOLLAMA_* 三層 route 仍經 110 proxyGemini 僅為三層失敗後備援。",
"evidence_refs": [
"k8s/awoooi-prod/04-configmap.yaml",
"k8s/awoooi-prod/06-deployment-api.yaml"
]
},
{
"observation_id": "operator_route_status_exists",
"status": "verified",
"summary": "platform operator service 已有只讀 ai-route-statusP1-004 不新增 live probe 或 paid probe。",
"evidence_refs": [
"apps/api/src/services/platform_operator_service.py",
"apps/api/src/api/v1/platform/operator_runs.py"
]
},
{
"observation_id": "nemotron_candidate_blocked",
"status": "blocked",
"summary": "MASTER 記錄 Nemotron replay / fast-model smoke 未達取代門檻,只能保留為離線候選。",
"evidence_refs": [
"docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md"
]
}
],
"operator_contract": {
"display_mode": "read_only_ai_provider_route_matrix",
"must_not_interpret_as": [
"provider 切換批准",
"production routing change 批准",
"USE_AI_ROUTER 開關變更批准",
"Gemini / Claude / NVIDIA 付費呼叫批准",
"OpenClaw 取代或降級批准",
"Nemotron 進 shadow / canary 批准",
"fallback order 修改批准",
"Ollama endpoint / ConfigMap 修改批准",
"Secret payload 已讀取或可輸出",
"外部 live probe 或 benchmark 批准",
"workflow / deploy / reload 觸發批准",
"runtime execution 授權"
],
"secret_display_policy": "只顯示 env var 名稱與 redacted metadata不得輸出 API key、token、header 或 payload。",
"provider_switch_policy": "P1-004 只能盤點與顯示;切換 provider 需 replay、shadow/canary、成本、rollback 與獨立人工批准。",
"cost_policy": "新增或提高 Gemini / Claude / NVIDIA 呼叫皆需 cost approval本 snapshot 不呼叫任何 paid provider。",
"runtime_policy": "UI/API 可見不等於 runtime route 已授權active runtime gate 維持 0。"
},
"operation_boundaries": {
"read_only_api_allowed": true,
"provider_switch_allowed": false,
"production_routing_change_allowed": false,
"use_ai_router_toggle_allowed": false,
"fallback_order_change_allowed": false,
"ollama_endpoint_change_allowed": false,
"paid_api_call_allowed": false,
"paid_api_frequency_increase_allowed": false,
"external_provider_probe_allowed": false,
"live_benchmark_allowed": false,
"shadow_or_canary_allowed": false,
"openclaw_replacement_allowed": false,
"nemotron_shadow_allowed": false,
"gemini_direct_call_allowed": false,
"secret_read_allowed": false,
"secret_plaintext_allowed": false,
"notification_send_allowed": false,
"workflow_trigger_allowed": false,
"deploy_trigger_allowed": false,
"reload_trigger_allowed": false,
"runtime_execution_allowed": false
},
"approval_boundaries": {
"provider_switch_approved": false,
"production_routing_change_approved": false,
"cost_change_approved": false,
"shadow_or_canary_approved": false,
"external_provider_call_approved": false,
"openclaw_replacement_approved": false,
"nemotron_replay_approved": false,
"secret_access_approved": false,
"runtime_execution_approved": false
}
}