wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity
Files
c9bb4bbfc6af05c41df7c3af546ba44cbcf45708
awoooi/docs/evaluations/ai_agent_critic_reviewer_result_capture_2026-06-13.json
Your Name f2b7e8d66e
Some checks failed
Code Review / ai-code-review (push) Successful in 13s
Details
CD Pipeline / tests (push) Successful in 1m39s
Details
Ansible / Reboot Recovery Contract / validate (push) Has been cancelled
Details
CD Pipeline / post-deploy-checks (push) Has been cancelled
Details
CD Pipeline / build-and-deploy (push) Has been cancelled
Details
fix(web): 收斂治理頁繁中文案
2026-06-19 02:59:46 +08:00

429 lines
16 KiB
JSON
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"schema_version": "ai_agent_critic_reviewer_result_capture_v1",
"generated_at": "2026-06-13T01:46:00+08:00",
"program_status": {
"overall_completion_percent": 100,
"current_priority": "P2",
"current_task_id": "P2-105",
"next_task_id": "P2-106",
"read_only_mode": true,
"runtime_authority": "critic_reviewer_result_capture_contract_only_no_live_write",
"status_note": "P2-105 將 P2-104 的 approved 後 learning 缺口收斂成 critic / reviewer score 與 result capture 契約；本快照只定義評分欄位、結果捕捉欄位、promotion gate 與候選路由，不寫 learning、不更新 PlayBook trust、不送 Telegram。"
},
"source_refs": [
"docs/evaluations/ai_agent_matched_playbook_learning_gap_2026-06-13.json",
"apps/api/src/services/learning_service.py",
"apps/api/src/services/approval_execution.py",
"apps/api/src/services/post_execution_verifier.py",
"apps/api/src/services/telegram_gateway.py",
"docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md",
"docs/ai/AI_AGENT_AUTOMATION_WORKLIST_2026-06-04.md",
"docs/ai/AI_AGENT_INTERACTION_LEARNING_PROOF_2026-06-11.md"
],
"prior_readback": {
"source_schema_version": "ai_agent_matched_playbook_learning_gap_v1",
"readback_at": "2026-06-13T00:48:00+08:00",
"approval_24h_total": 66,
"approval_24h_matched": 66,
"approved_without_execution_meta_24h": 63,
"pending_with_matched_24h": 2,
"execution_failed_with_matched_24h": 1,
"playbook_updated_24h": 0,
"readback_note": "P2-104 已證明 matched_playbook_id 不是主缺口；P2-105 的主缺口是批准前缺 critic / reviewer score，批准後缺可審核 result capture。"
},
"score_truth": {
"p2_104_gap_loaded": true,
"critic_reviewer_score_required": true,
"result_capture_required": true,
"playbook_trust_candidate_required": true,
"owner_review_required_before_write": true,
"post_write_verifier_required": true,
"runtime_critic_score_enabled": false,
"runtime_reviewer_score_enabled": false,
"runtime_result_capture_enabled": false,
"runtime_learning_write_enabled": false,
"playbook_trust_write_enabled": false,
"gateway_queue_write_enabled": false,
"telegram_send_enabled": false,
"production_write_enabled": false,
"secret_value_read_enabled": false,
"destructive_operation_enabled": false,
"work_window_transcript_display_allowed": false,
"approval_24h_total": 66,
"approved_without_execution_meta_24h": 63,
"execution_failed_with_matched_24h": 1,
"critic_runtime_score_count_24h": 0,
"reviewer_runtime_score_count_24h": 0,
"result_capture_runtime_write_count_24h": 0,
"learning_write_count_24h": 0,
"playbook_trust_write_count_24h": 0,
"gateway_queue_write_count_24h": 0,
"telegram_send_count_24h": 0,
"production_write_count_24h": 0,
"secret_value_read_count_24h": 0,
"destructive_operation_count_24h": 0,
"truth_note": "P2-105 只讓統帥看見 score/result-capture 如何被要求與審查；任何 runtime score 寫入、learning write、PlayBook trust update、Telegram send 與 production write 都仍為 0。"
},
"agent_scorecards": [
{
"scorecard_id": "scorecard_openclaw_critic_decision_quality",
"display_name": "OpenClaw Critic 決策品質評分",
"owner_agent": "openclaw",
"role": "critic",
"status": "ready_for_owner_review",
"risk_tier": "high",
"minimum_score": 80,
"required_fields": [
"approval_id",
"incident_id",
"matched_playbook_id",
"hypothesis_count",
"challenge_count",
"unsafe_action_detected",
"score_0_100",
"verdict"
],
"failure_if_missing": "缺 Critic 反方挑戰時，AI Agent 可能把看似合理但根因錯誤的批准結果寫成 learning。",
"runtime_score_enabled": false,
"evidence_hash": "sha256:6767676767676767676767676767676767676767676767676767676767676767"
},
{
"scorecard_id": "scorecard_openclaw_reviewer_safety_verdict",
"display_name": "OpenClaw Reviewer 安全裁決",
"owner_agent": "openclaw",
"role": "reviewer",
"status": "ready_for_owner_review",
"risk_tier": "critical",
"minimum_score": 90,
"required_fields": [
"approval_id",
"risk_level",
"hard_rules_hit",
"dry_run_evidence_hash",
"rollback_owner",
"safety_verdict"
],
"failure_if_missing": "缺 Reviewer 安全裁決時，不得把 approved approval 升級成自動處理或信任度更新。",
"runtime_score_enabled": false,
"evidence_hash": "sha256:7878787878787878787878787878787878787878787878787878787878787878"
},
{
"scorecard_id": "scorecard_hermes_redaction_operator_report",
"display_name": "Hermes redaction / operator report",
"owner_agent": "hermes",
"role": "reporter",
"status": "ready_for_owner_review",
"risk_tier": "medium",
"minimum_score": 85,
"required_fields": [
"redacted_summary",
"blocked_fields",
"allowed_display_fields",
"operator_next_action",
"evidence_hash"
],
"failure_if_missing": "缺 redaction 報告時，前端或 Telegram 可能誤顯示未脫敏提示內容、私有推理內容、機密明文或未脫敏背景內容。",
"runtime_score_enabled": false,
"evidence_hash": "sha256:8989898989898989898989898989898989898989898989898989898989898989"
},
{
"scorecard_id": "scorecard_nemotron_failure_candidate_verifier",
"display_name": "NemoTron failure candidate verifier",
"owner_agent": "nemotron",
"role": "verifier",
"status": "ready_for_owner_review",
"risk_tier": "high",
"minimum_score": 80,
"required_fields": [
"approval_id",
"failure_reason",
"result_state",
"verifier_plan",
"no_write_replay_hash",
"promotion_recommendation"
],
"failure_if_missing": "EXECUTION_FAILED 只能成為 failure learning 候選；未驗證前不得污染 PlayBook trust。",
"runtime_score_enabled": false,
"evidence_hash": "sha256:9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a"
},
{
"scorecard_id": "scorecard_coordinator_disagreement_gate",
"display_name": "Coordinator disagreement gate",
"owner_agent": "openclaw",
"role": "coordinator",
"status": "blocked_by_policy",
"risk_tier": "critical",
"minimum_score": 95,
"required_fields": [
"critic_verdict",
"reviewer_verdict",
"disagreement_reason",
"human_gate_required",
"promotion_decision"
],
"failure_if_missing": "Critic / Reviewer 不一致時必須 hold；Agent 不得互相自行批准 production route。",
"runtime_score_enabled": false,
"evidence_hash": "sha256:abababababababababababababababababababababababababababababababab"
}
],
"result_capture_contracts": [
{
"contract_id": "capture_approved_execution_result",
"display_name": "APPROVED result capture",
"owner_agent": "openclaw",
"result_state": "approved_without_execution_meta",
"status": "needs_owner_review",
"risk_tier": "high",
"count_24h": 63,
"required_fields": [
"approval_id",
"incident_id",
"matched_playbook_id",
"execution_kind",
"result_state",
"operator_outcome",
"verifier_result"
],
"write_enabled": false,
"runtime_writer_enabled": false,
"blocker_summary": "63 筆 approved 已有 matched id，但缺 execution result；未補 result capture 前不可寫 learning。",
"evidence_hash": "sha256:bcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc"
},
{
"contract_id": "capture_execution_failed_candidate",
"display_name": "EXECUTION_FAILED failure candidate",
"owner_agent": "nemotron",
"result_state": "execution_failed",
"status": "needs_owner_review",
"risk_tier": "high",
"count_24h": 1,
"required_fields": [
"approval_id",
"matched_playbook_id",
"failure_reason",
"repair_attempted",
"repair_executed",
"negative_learning_weight",
"verifier_result"
],
"write_enabled": false,
"runtime_writer_enabled": false,
"blocker_summary": "失敗案例可做負向 learning 候選，但需 verifier 與 owner review。",
"evidence_hash": "sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd"
},
{
"contract_id": "capture_pending_human_gate",
"display_name": "PENDING human gate capture",
"owner_agent": "hermes",
"result_state": "pending_human_gate",
"status": "blocked_by_policy",
"risk_tier": "medium",
"count_24h": 2,
"required_fields": [
"approval_id",
"matched_playbook_id",
"pending_reason",
"human_decision_required",
"expires_at"
],
"write_enabled": false,
"runtime_writer_enabled": false,
"blocker_summary": "pending 尚未有人工結果，不可轉成 learning 或 trust 更新。",
"evidence_hash": "sha256:dededededededededededededededededededededededededededededededede"
},
{
"contract_id": "capture_noop_manual_resolution",
"display_name": "no-op / manual resolution capture",
"owner_agent": "hermes",
"result_state": "manual_or_noop",
"status": "ready",
"risk_tier": "medium",
"count_24h": 0,
"required_fields": [
"incident_id",
"approval_id",
"manual_resolution_note",
"noop_reason",
"operator_verified"
],
"write_enabled": false,
"runtime_writer_enabled": false,
"blocker_summary": "no-op 與人工處理需被區分，否則 learning 會誤把未執行視為成功。",
"evidence_hash": "sha256:efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef"
},
{
"contract_id": "capture_post_write_verifier_receipt",
"display_name": "post-write verifier receipt",
"owner_agent": "nemotron",
"result_state": "verifier_receipt",
"status": "blocked_by_policy",
"risk_tier": "critical",
"count_24h": 0,
"required_fields": [
"learning_candidate_id",
"pre_state_hash",
"post_state_hash",
"verifier_result",
"rollback_decision"
],
"write_enabled": false,
"runtime_writer_enabled": false,
"blocker_summary": "未有 post-write verifier 前，PlayBook trust 不可被 runtime 更新。",
"evidence_hash": "sha256:f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0"
}
],
"promotion_gates": [
{
"gate_id": "gate_minimum_critic_reviewer_scores",
"display_name": "minimum critic / reviewer scores",
"owner_agent": "openclaw",
"status": "needs_owner_review",
"required_before": "learning candidate promotion",
"failure_if_missing": "低分或缺分的結果不得進入 learning writeback。",
"creates_runtime_write": false
},
{
"gate_id": "gate_disagreement_human_hold",
"display_name": "disagreement human hold",
"owner_agent": "openclaw",
"status": "blocked_by_policy",
"required_before": "any auto promotion",
"failure_if_missing": "Critic / Reviewer 不一致時必須人工 hold，不得由 Agent 自行批准。",
"creates_runtime_write": false
},
{
"gate_id": "gate_result_capture_payload_complete",
"display_name": "result capture payload complete",
"owner_agent": "hermes",
"status": "needs_owner_review",
"required_before": "learning_service payload",
"failure_if_missing": "缺 approval_id / incident_id / matched_playbook_id / result_state 時不得寫 learning。",
"creates_runtime_write": false
},
{
"gate_id": "gate_redaction_no_private_context",
"display_name": "redaction no private context",
"owner_agent": "hermes",
"status": "ready",
"required_before": "governance UI / Telegram digest",
"failure_if_missing": "不得顯示未脫敏提示內容、私有推理內容、secret、原始 Telegram 載荷或未脫敏背景內容。",
"creates_runtime_write": false
},
{
"gate_id": "gate_post_write_verifier",
"display_name": "post-write verifier",
"owner_agent": "nemotron",
"status": "blocked_by_policy",
"required_before": "PlayBook trust update",
"failure_if_missing": "未驗證前的 success / failure 不得改變 PlayBook trust。",
"creates_runtime_write": false
},
{
"gate_id": "gate_telegram_operator_digest",
"display_name": "Telegram operator digest",
"owner_agent": "hermes",
"status": "blocked_by_policy",
"required_before": "operator approval loop",
"failure_if_missing": "沒有 SRE 戰情室 digest / receipt 前，不得宣稱統帥已看過 learning 候選。",
"creates_runtime_write": false
}
],
"candidate_routes": [
{
"route_id": "route_approved_to_result_capture",
"display_name": "approved -> result capture",
"owner_agent": "openclaw",
"candidate_count_24h": 63,
"status": "ready_for_owner_review",
"risk_tier": "high",
"next_gate": "先補 critic / reviewer score 與 result_state，再進 owner-approved learning dry-run。",
"write_enabled": false,
"evidence_hash": "sha256:1111111111111111111111111111111111111111111111111111111111111111"
},
{
"route_id": "route_failed_to_negative_learning_candidate",
"display_name": "failed -> negative learning candidate",
"owner_agent": "nemotron",
"candidate_count_24h": 1,
"status": "ready_for_owner_review",
"risk_tier": "high",
"next_gate": "只產生 failure candidate；需 verifier 才能進 trust 更新。",
"write_enabled": false,
"evidence_hash": "sha256:2222222222222222222222222222222222222222222222222222222222222222"
},
{
"route_id": "route_pending_to_human_gate",
"display_name": "pending -> human gate",
"owner_agent": "hermes",
"candidate_count_24h": 2,
"status": "blocked_by_policy",
"risk_tier": "medium",
"next_gate": "等待人工審核；未批准前不可寫 learning。",
"write_enabled": false,
"evidence_hash": "sha256:3333333333333333333333333333333333333333333333333333333333333333"
},
{
"route_id": "route_score_ready_to_playbook_trust_hold",
"display_name": "score ready -> PlayBook trust hold",
"owner_agent": "openclaw",
"candidate_count_24h": 0,
"status": "blocked_by_policy",
"risk_tier": "critical",
"next_gate": "P2-106 才能產生 owner-approved 無寫入 dry-run；本段仍不更新 trust。",
"write_enabled": false,
"evidence_hash": "sha256:4444444444444444444444444444444444444444444444444444444444444444"
}
],
"display_redaction_contract": {
"redaction_required": true,
"raw_prompt_display_allowed": false,
"private_reasoning_display_allowed": false,
"secret_value_display_allowed": false,
"raw_telegram_payload_display_allowed": false,
"work_window_transcript_display_allowed": false,
"allowed_display_fields": [
"schema_version",
"generated_at",
"program_status",
"prior_readback",
"score_truth",
"agent_scorecards",
"result_capture_contracts",
"promotion_gates",
"candidate_routes",
"rollups"
],
"blocked_display_fields": [
"未脫敏提示內容",
"私有推理內容",
"推理鏈內容",
"機密明文",
"原始 Telegram 載荷",
"未脫敏操作背景",
"未脫敏私有背景"
]
},
"rollups": {
"scorecard_count": 5,
"result_capture_contract_count": 5,
"promotion_gate_count": 6,
"candidate_route_count": 4,
"approval_24h_total": 66,
"approved_without_execution_meta_24h": 63,
"execution_failed_with_matched_24h": 1,
"pending_with_matched_24h": 2,
"blocked_gate_count": 3,
"owner_review_gate_count": 2,
"runtime_critic_score_count": 0,
"runtime_reviewer_score_count": 0,
"result_capture_runtime_write_count": 0,
"learning_write_count": 0,
"playbook_trust_write_count": 0,
"gateway_queue_write_count": 0,
"telegram_send_count": 0,
"production_write_count": 0,
"secret_value_read_count": 0,
"destructive_operation_count": 0
}
}
Reference in New Issue View Git Blame Copy Permalink