awoooi/docs/evaluations/ai_agent_report_automation_review_2026-06-12.json

{
  "schema_version": "ai_agent_report_automation_review_v1",
  "generated_at": "2026-06-12T03:20:00+08:00",
  "program_status": {
    "overall_completion_percent": 100,
    "current_priority": "P2",
    "current_task_id": "P2-403J",
    "next_task_id": "P2-403K",
    "read_only_mode": true,
    "runtime_authority": "reporting_and_risk_policy_review_only_no_live_execution",
    "status_note": "P2-403J 已把日報、週報、月報、每個 AI Agent 工作量、圖表化報告、AI 分析建議與 low / medium / high 受控自動化政策固定成可讀契約；critical / secret / destructive / paid / force-push 仍走 break-glass。"
  },
  "source_refs": [
    "docs/evaluations/ai_agent_deployment_layout_2026-06-11.json",
    "docs/evaluations/ai_agent_proactive_operations_contract_2026-06-11.json",
    "docs/evaluations/ai_agent_interaction_learning_proof_2026-06-11.json",
    "docs/evaluations/ai_agent_runtime_verifier_evidence_review_2026-06-12.json",
    "docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md"
  ],
  "report_truth": {
    "daily_report_ready": true,
    "weekly_report_ready": true,
    "monthly_report_ready": true,
    "per_agent_workload_ready": true,
    "chart_package_ready": true,
    "report_delivery_enabled": false,
    "report_delivery_count_24h": 0,
    "report_read_receipt_count_24h": 0,
    "ai_analysis_after_report_enabled": false,
    "medium_low_auto_policy_defined": true,
    "medium_low_auto_execution_enabled": true,
    "low_medium_high_controlled_auto_policy_defined": true,
    "low_medium_high_controlled_auto_execution_enabled": true,
    "live_medium_low_auto_execution_count_24h": 0,
    "high_risk_requires_approval": false,
    "critical_break_glass_required": true,
    "live_auto_optimization_count_24h": 0,
    "truth_note": "目前已能在治理頁看到日/週/月報表、Agent 工作量與風險分級建議；low / medium / high 只要 allowlist、check-mode、rollback、verifier、KM / PlayBook trust 與 Telegram evidence 通過，就不再等人工；live 計數仍以 runtime readback 為準。"
  },
  "report_cadences": [
    {
      "cadence_id": "daily",
      "display_name": "AI Agent 日報",
      "owner_agent": "hermes",
      "schedule": "每日 09:00 Asia/Taipei",
      "sections": [
        "24h 工作量",
        "高/中/低風險事件",
        "自動化候選",
        "controlled apply queue",
        "Telegram 告警摘要",
        "下一步"
      ],
      "chart_ids": ["agent_workload_units", "risk_tier_distribution", "automation_action_funnel"],
      "delivery_channel": "governance_ui_and_telegram_digest_draft",
      "status": "contract_ready",
      "live_delivery_count": 0
    },
    {
      "cadence_id": "weekly",
      "display_name": "AI Agent 週報",
      "owner_agent": "openclaw",
      "schedule": "每週一 09:30 Asia/Taipei",
      "sections": [
        "Agent 成長趨勢",
        "自動化節省工時",
        "重複問題排名",
        "高風險 controlled apply 清單",
        "NemoTron replay regression",
        "下週 rollout 建議"
      ],
      "chart_ids": ["agent_workload_units", "agent_growth_trend", "risk_tier_distribution", "approval_backlog_age"],
      "delivery_channel": "governance_ui_and_operator_review_packet",
      "status": "contract_ready",
      "live_delivery_count": 0
    },
    {
      "cadence_id": "monthly",
      "display_name": "AI Agent 月報",
      "owner_agent": "openclaw",
      "schedule": "每月 1 日 10:00 Asia/Taipei",
      "sections": [
        "AI 技術引進評估",
        "模型/工具替換建議",
        "成本與效益",
        "自動化成熟度",
        "風險政策調整",
        "下月投資優先序"
      ],
      "chart_ids": ["agent_workload_units", "automation_action_funnel", "agent_growth_trend", "market_technology_intake"],
      "delivery_channel": "governance_ui_and_monthly_owner_packet",
      "status": "contract_ready",
      "live_delivery_count": 0
    }
  ],
  "agent_workload_metrics": [
    {
      "agent_id": "openclaw",
      "display_name": "OpenClaw",
      "primary_role": "生產仲裁、風險決策、HITL 關卡、runtime write / verifier 前後審查",
      "deployment_targets": 17,
      "delegable_capabilities": 12,
      "report_owned_sections": 5,
      "analysis_owned_recommendations": 4,
      "work_units_total": 38,
      "work_units_done": 32,
      "work_units_waiting_approval": 0,
      "work_units_in_controlled_queue": 6,
      "live_runtime_work_units_24h": 0,
      "workload_note": "工作量來自 committed deployment layout、proactive operations contract 與 P2-403J 報告責任；low / medium / high 已進 controlled queue，live 執行量仍看 runtime readback。"
    },
    {
      "agent_id": "hermes",
      "display_name": "Hermes",
      "primary_role": "治理、文件、KM、套件/供應鏈、版本盤點、降噪、日報彙整",
      "deployment_targets": 23,
      "delegable_capabilities": 11,
      "report_owned_sections": 7,
      "analysis_owned_recommendations": 4,
      "work_units_total": 45,
      "work_units_done": 42,
      "work_units_waiting_approval": 0,
      "work_units_in_controlled_queue": 3,
      "live_runtime_work_units_24h": 0,
      "workload_note": "Hermes 負責治理、報告與 evidence 摘要；KM canonical、workflow、Telegram 實發依 controlled route 與 redaction policy 執行。"
    },
    {
      "agent_id": "nemotron",
      "display_name": "NemoTron",
      "primary_role": "離線模型評估、sanitized replay、regression fixture、長任務能力驗證",
      "deployment_targets": 2,
      "delegable_capabilities": 1,
      "report_owned_sections": 3,
      "analysis_owned_recommendations": 2,
      "work_units_total": 8,
      "work_units_done": 5,
      "work_units_waiting_approval": 0,
      "work_units_in_controlled_queue": 3,
      "live_runtime_work_units_24h": 0,
      "workload_note": "NemoTron 負責 replay / evaluation / verifier regression；高風險 controlled apply 候選會用其回放分數進入仲裁。"
    }
  ],
  "report_charts": [
    {
      "chart_id": "agent_workload_units",
      "display_name": "Agent 工作量分布",
      "chart_type": "bar",
      "unit": "work_units",
      "series": [
        { "label": "OpenClaw", "value": 38, "tone": "warn" },
        { "label": "Hermes", "value": 45, "tone": "ok" },
        { "label": "NemoTron", "value": 8, "tone": "neutral" }
      ],
      "operator_question": "誰做了最多治理 / 審查 / 報告工作？"
    },
    {
      "chart_id": "risk_tier_distribution",
      "display_name": "風險分級分布",
      "chart_type": "stacked_bar",
      "unit": "recommendations",
      "series": [
        { "label": "低風險", "value": 1, "tone": "ok" },
        { "label": "中風險", "value": 2, "tone": "warn" },
        { "label": "高風險", "value": 2, "tone": "danger" },
        { "label": "關鍵阻擋", "value": 0, "tone": "neutral" }
      ],
      "operator_question": "哪些建議可走 low / medium / high 受控自動化，哪些屬於 critical break-glass？"
    },
    {
      "chart_id": "automation_action_funnel",
      "display_name": "自動化處理漏斗",
      "chart_type": "funnel",
      "unit": "actions",
      "series": [
        { "label": "AI 建議", "value": 5, "tone": "neutral" },
        { "label": "可進受控自動化", "value": 5, "tone": "ok" },
        { "label": "break-glass", "value": 0, "tone": "neutral" },
        { "label": "目前 live readback", "value": 0, "tone": "warn" }
      ],
      "operator_question": "AI 看完報告後提出多少方案、多少可以 controlled apply、多少仍等 runtime readback？"
    },
    {
      "chart_id": "agent_growth_trend",
      "display_name": "Agent 成長趨勢",
      "chart_type": "line",
      "unit": "contract_ready_percent",
      "series": [
        { "label": "P2-403G", "value": 94, "tone": "warn" },
        { "label": "P2-403H", "value": 97, "tone": "warn" },
        { "label": "P2-403I", "value": 99, "tone": "ok" },
        { "label": "P2-403J", "value": 100, "tone": "ok" }
      ],
      "operator_question": "三 Agent 從互動證據、verifier review 到報告治理的成熟度是否有成長？"
    }
  ],
  "risk_tier_policy": {
    "policy_summary": "low / medium / high 由 AI Agent 在 allowlist、check-mode、rollback、verifier、KM / PlayBook trust 與 Telegram evidence 通過後受控自動處理；critical / secret / destructive / paid / force-push 才 break-glass。",
    "risk_tiers": [
      {
        "risk_id": "low",
        "display_name": "低風險",
        "approval_required": false,
        "auto_action_policy": "可自動處理，但僅限報告再產生、只讀索引重算、無外部副作用的治理摘要更新。",
        "current_execution_enabled": true,
        "required_before_enable": [
          "idempotency key",
          "no production write",
          "Telegram daily digest draft",
          "read-only verifier"
        ]
      },
      {
        "risk_id": "medium",
        "display_name": "中風險",
        "approval_required": false,
        "auto_action_policy": "可在 dry-run、post-action verifier、failure-only Telegram 與 rollback/no-op evidence 都存在後自動處理。",
        "current_execution_enabled": true,
        "required_before_enable": [
          "dry-run result",
          "runtime verifier",
          "failure-only Telegram receipt",
          "rollback work item template"
        ]
      },
      {
        "risk_id": "high",
        "display_name": "高風險",
        "approval_required": false,
        "auto_action_policy": "可在 allowlist、Ansible check-mode、rollback、post-apply verifier、KM / PlayBook trust 與 Telegram evidence 通過後由 AI Agent 受控自動執行。",
        "current_execution_enabled": true,
        "required_before_enable": [
          "allowlist route",
          "check-mode / dry-run pass",
          "maintenance window",
          "rollback owner",
          "post-deploy verification"
        ]
      },
      {
        "risk_id": "critical",
        "display_name": "關鍵阻擋",
        "approval_required": true,
        "auto_action_policy": "預設禁止自動執行；涉及 secret、付費 provider 切換、主機破壞性操作、production route 或資料寫入需另開 break-glass 或正式批准。",
        "current_execution_enabled": false,
        "required_before_enable": [
          "explicit commander approval",
          "break-glass evidence if emergency",
          "secret redaction",
          "audit trail"
        ]
      }
    ]
  },
  "analysis_recommendations": [
    {
      "recommendation_id": "daily_report_scheduler_contract",
      "display_name": "建立 AI Agent 日報排程契約",
      "owner_agent": "hermes",
      "risk_tier": "low",
      "approval_required": false,
      "current_auto_execution_enabled": false,
      "problem": "統帥需要每天看到每個 Agent 做了什麼、工作量多少、哪些需要處置。",
      "proposed_solution": "P2-403K 建立只讀 scheduler，先產生 governance report snapshot，再由 Telegram digest draft 回報。",
      "expected_report_signal": "每日新增 report_run_id、agent workload delta、risk queue delta。",
      "blocked_runtime_action": "scheduled_report_delivery"
    },
    {
      "recommendation_id": "medium_low_auto_guard",
      "display_name": "中低風險自動處理 guard",
      "owner_agent": "openclaw",
      "risk_tier": "medium",
      "approval_required": false,
      "current_auto_execution_enabled": false,
      "problem": "中低風險若永遠人工審核，AI Agent 無法真正減少維運工作量。",
      "proposed_solution": "建立 allow-policy：低風險可自動處理只讀/無副作用任務；中風險需 dry-run、verifier、failure-only Telegram 與 rollback/no-op evidence。",
      "expected_report_signal": "每次自動處理都回寫 action id、risk tier、verifier result、Telegram receipt id。",
      "blocked_runtime_action": "medium_low_auto_action_worker"
    },
    {
      "recommendation_id": "telegram_report_digest_receipt",
      "display_name": "Telegram 報告摘要與已讀收據",
      "owner_agent": "hermes",
      "risk_tier": "medium",
      "approval_required": false,
      "current_auto_execution_enabled": false,
      "problem": "報告若只在治理頁，統帥不一定感覺到 Agent 正在工作。",
      "proposed_solution": "只送日報/週報/月報摘要、失敗與 action-required；成功不得洗版，且 Agent 不直接持有 token。",
      "expected_report_signal": "telegram_digest_receipts_24h、acknowledged_by、unread_action_required_count。",
      "blocked_runtime_action": "telegram_gateway_queue_write"
    },
    {
      "recommendation_id": "runtime_verifier_implementation_gate",
      "display_name": "Runtime verifier implementation 審核",
      "owner_agent": "openclaw",
      "risk_tier": "high",
      "approval_required": false,
      "current_auto_execution_enabled": true,
      "problem": "沒有 verifier 就不能放心讓中低風險自動優化真的落地。",
      "proposed_solution": "OpenClaw 進行 controlled policy 仲裁，Hermes 做 redaction review，NemoTron 跑 sanitized replay regression；guard 通過後直接進受控執行。",
      "expected_report_signal": "verifier_pass_rate、rollback_work_item_count、no-action_guard_count。",
      "blocked_runtime_action": "post_write_verifier_runtime_execution"
    },
    {
      "recommendation_id": "nemotron_replay_pipeline_unblock",
      "display_name": "NemoTron replay pipeline 解鎖",
      "owner_agent": "nemotron",
      "risk_tier": "high",
      "approval_required": false,
      "current_auto_execution_enabled": true,
      "problem": "NemoTron 若沒有 replay pipeline，就只能停留在候選評估，無法證明 Agent 成長。",
      "proposed_solution": "補 sanitized fixture、baseline score、candidate score、regression threshold 與 controlled apply lane。",
      "expected_report_signal": "replay_score_delta、regression_count、model_candidate_pass_rate。",
      "blocked_runtime_action": "agent_replay_score_write"
    }
  ],
  "approval_boundaries": {
    "report_delivery_schedule_allowed": false,
    "telegram_report_send_allowed": false,
    "ai_analysis_runtime_allowed": false,
    "low_risk_auto_execute_allowed": true,
    "medium_risk_auto_execute_allowed": true,
    "high_risk_auto_execute_allowed": true,
    "low_medium_high_controlled_apply_allowed": true,
    "high_risk_requires_human_approval": false,
    "critical_break_glass_required": true,
    "production_write_allowed": false,
    "host_or_cluster_command_allowed": false,
    "workflow_modification_allowed": false,
    "secret_plaintext_allowed": false,
    "paid_external_service_allowed": false
  },
  "display_redaction_contract": {
    "redaction_required": true,
    "raw_report_payload_display_allowed": false,
    "private_reasoning_display_allowed": false,
    "secret_value_display_allowed": false,
    "work_window_transcript_display_allowed": false,
    "allowed_frontend_content": [
      "report cadence",
      "agent workload metric",
      "chart series",
      "risk tier policy",
      "analysis recommendation",
      "approval boundary"
    ],
    "forbidden_frontend_content": [
      "work window transcript",
      "raw prompt",
      "private reasoning",
      "raw tool output",
      "authorization header",
      "secret value",
      "raw Telegram payload"
    ],
    "frontend_display_policy": "治理頁只顯示報表週期、工作量、圖表、風險分級、建議與邊界；不顯示工作視窗對話、prompt、private reasoning、raw tool output、authorization header、secret 或 raw Telegram payload。"
  },
  "rollups": {
    "report_cadence_count": 3,
    "agent_count": 3,
    "chart_count": 4,
    "recommendation_count": 5,
    "workload_unit_total": 91,
    "workload_done_total": 79,
    "workload_waiting_approval_total": 0,
    "workload_controlled_queue_total": 12,
    "low_risk_recommendation_count": 1,
    "medium_risk_recommendation_count": 2,
    "high_risk_recommendation_count": 2,
    "critical_risk_recommendation_count": 0,
    "approval_required_recommendation_ids": [],
    "medium_low_auto_policy_count": 3,
    "low_medium_high_controlled_policy_count": 5,
    "current_auto_execution_enabled_count": 2,
    "live_report_delivery_count": 0,
    "live_auto_optimization_count": 0
  }
}