diff --git a/apps/api/src/services/ai_agent_report_truth_actionability_review.py b/apps/api/src/services/ai_agent_report_truth_actionability_review.py index fe4caaba7..58d43b05f 100644 --- a/apps/api/src/services/ai_agent_report_truth_actionability_review.py +++ b/apps/api/src/services/ai_agent_report_truth_actionability_review.py @@ -33,6 +33,7 @@ def load_latest_ai_agent_report_truth_actionability_review( _require_schema(payload, str(latest)) _require_runtime_boundaries(payload, str(latest)) _require_findings(payload, str(latest)) + _require_source_and_actionability_gates(payload, str(latest)) _require_telegram_routing(payload, str(latest)) _require_rollup_consistency(payload, str(latest)) return payload @@ -46,8 +47,8 @@ def _require_schema(payload: dict[str, Any], label: str) -> None: raise ValueError(f"{label}: program_status.read_only_mode must be true") if status.get("runtime_authority") != "report_truth_actionability_review_only_no_report_send_or_runtime_fix": raise ValueError(f"{label}: runtime_authority must remain report-truth review only") - if status.get("current_task_id") != "P2-403J" or status.get("next_task_id") != "P2-403K": - raise ValueError(f"{label}: current/next task must remain P2-403J -> P2-403K") + if status.get("current_task_id") != "P2-403K" or status.get("next_task_id") != "P2-403L": + raise ValueError(f"{label}: current/next task must remain P2-403K -> P2-403L") def _require_runtime_boundaries(payload: dict[str, Any], label: str) -> None: @@ -58,6 +59,11 @@ def _require_runtime_boundaries(payload: dict[str, Any], label: str) -> None: "freshness_gate_implemented", "source_confidence_gate_implemented", "actionability_score_implemented", + ): + if truth.get(flag) is not True: + raise ValueError(f"{label}: {flag} must be true for P2-403K read-only gates") + + for flag in ( "ai_agent_runtime_control_allowed", "telegram_report_send_allowed", "cronjob_change_allowed", @@ -98,6 +104,74 @@ def _require_findings(payload: dict[str, Any], label: str) -> None: raise ValueError(f"{label}: missing actionability lane {lane_id}") +def _require_source_and_actionability_gates(payload: dict[str, Any], label: str) -> None: + freshness = payload.get("source_freshness_gates") or [] + freshness_ids = {item.get("gate_id") for item in freshness} + required_freshness = { + "stats_api_incident_rollup", + "k3s_metrics_rollup", + "gitea_activity_rollup", + "ai_cost_ledger", + "telegram_delivery_receipt", + "heartbeat_watchdog", + } + missing_freshness = sorted(required_freshness - freshness_ids) + if missing_freshness: + raise ValueError(f"{label}: missing source freshness gates: {missing_freshness}") + for item in freshness: + if item.get("work_item_required") is not True: + raise ValueError(f"{label}: source freshness gate {item.get('gate_id')} must require a work item") + for field in ("freshness_state", "last_success_at", "required_ttl", "ai_next_action", "owner_agent"): + if not item.get(field): + raise ValueError(f"{label}: source freshness gate {item.get('gate_id')} missing {field}") + + confidence = payload.get("source_confidence_gates") or [] + confidence_ids = {item.get("confidence_id") for item in confidence} + required_confidence = { + "alerts_count_confidence", + "ai_performance_confidence", + "k3s_health_confidence", + "development_activity_confidence", + "ai_cost_confidence", + } + missing_confidence = sorted(required_confidence - confidence_ids) + if missing_confidence: + raise ValueError(f"{label}: missing source confidence gates: {missing_confidence}") + for item in confidence: + if item.get("confidence") != "low": + raise ValueError(f"{label}: source confidence gate {item.get('confidence_id')} must remain low until live source readback") + if item.get("blocked_green_badge") is not True: + raise ValueError(f"{label}: source confidence gate {item.get('confidence_id')} must block green badge") + + scores = payload.get("report_actionability_scores") or [] + score_ids = {item.get("report_id") for item in scores} + required_scores = {"daily_report", "weekly_report", "monthly_report", "heartbeat_digest"} + missing_scores = sorted(required_scores - score_ids) + if missing_scores: + raise ValueError(f"{label}: missing report actionability scores: {missing_scores}") + weekly = next((item for item in scores if item.get("report_id") == "weekly_report"), None) + if not weekly or weekly.get("score", 0) < 80 or weekly.get("lane") != "action_required_source_gap": + raise ValueError(f"{label}: weekly report must remain action-required source gap with score >= 80") + + work_items = payload.get("zero_signal_work_items") or [] + work_item_ids = {item.get("work_item_id") for item in work_items} + required_work_items = { + "report-source-gap:stats_api", + "report-source-gap:k3s_metrics", + "report-source-gap:gitea_activity", + "report-source-gap:ai_cost_ledger", + "report-source-gap:telegram_delivery", + } + missing_work_items = sorted(required_work_items - work_item_ids) + if missing_work_items: + raise ValueError(f"{label}: missing zero signal work items: {missing_work_items}") + for item in work_items: + if item.get("status") != "draft_required": + raise ValueError(f"{label}: zero signal work item {item.get('work_item_id')} must remain draft_required") + if not item.get("required_assets"): + raise ValueError(f"{label}: zero signal work item {item.get('work_item_id')} missing required assets") + + def _require_telegram_routing(payload: dict[str, Any], label: str) -> None: routing = payload.get("telegram_routing_consolidation") or {} if routing.get("canonical_room_name") != "AwoooI SRE 戰情室": @@ -136,6 +210,10 @@ def _require_rollup_consistency(payload: dict[str, Any], label: str) -> None: lanes = payload.get("alert_actionability_lanes") or [] routes = payload.get("telegram_route_findings") or [] actions = payload.get("operator_actions") or [] + freshness = payload.get("source_freshness_gates") or [] + confidence = payload.get("source_confidence_gates") or [] + scores = payload.get("report_actionability_scores") or [] + work_items = payload.get("zero_signal_work_items") or [] blocked = { *(item.get("blocked_runtime_action") for item in findings), *(item.get("blocked_runtime_action") for item in routes), @@ -145,6 +223,16 @@ def _require_rollup_consistency(payload: dict[str, Any], label: str) -> None: blocked.discard(None) expected_counts = { "zero_signal_finding_count": len(findings), + "source_freshness_gate_count": len(freshness), + "stale_or_untrusted_source_count": sum( + 1 for item in freshness if item.get("freshness_state") != "report_only_when_fresh" + ), + "source_confidence_gate_count": len(confidence), + "low_confidence_gate_count": sum(1 for item in confidence if item.get("confidence") == "low"), + "report_actionability_score_count": len(scores), + "actionability_score_ready_count": sum(1 for item in scores if isinstance(item.get("score"), int) and item.get("score") > 0), + "zero_signal_work_item_count": len(work_items), + "work_item_required_count": sum(1 for item in freshness if item.get("work_item_required") is True), "critical_finding_count": sum(1 for item in findings if item.get("severity") == "critical"), "high_finding_count": sum(1 for item in findings if item.get("severity") == "high"), "cadence_contract_count": len(cadences), @@ -176,3 +264,6 @@ def _require_rollup_consistency(payload: dict[str, Any], label: str) -> None: raise ValueError(f"{label}: approval_required_action_ids mismatch") if rollups.get("all_zero_weekly_report_confidence") != "low_trust_actionable_anomaly": raise ValueError(f"{label}: all-zero weekly report confidence must stay low trust") + weekly = next((item for item in scores if item.get("report_id") == "weekly_report"), {}) + if rollups.get("all_zero_weekly_report_actionability_score") != weekly.get("score"): + raise ValueError(f"{label}: all-zero weekly report actionability score mismatch") diff --git a/apps/api/tests/test_ai_agent_report_truth_actionability_review.py b/apps/api/tests/test_ai_agent_report_truth_actionability_review.py index caece1911..e5cbda291 100644 --- a/apps/api/tests/test_ai_agent_report_truth_actionability_review.py +++ b/apps/api/tests/test_ai_agent_report_truth_actionability_review.py @@ -18,11 +18,18 @@ def test_load_latest_ai_agent_report_truth_actionability_review(): data = load_latest_ai_agent_report_truth_actionability_review() assert data["schema_version"] == "ai_agent_report_truth_actionability_review_v1" - assert data["program_status"]["current_task_id"] == "P2-403J" - assert data["program_status"]["next_task_id"] == "P2-403K" + assert data["program_status"]["current_task_id"] == "P2-403K" + assert data["program_status"]["next_task_id"] == "P2-403L" assert data["report_truth"]["all_zero_weekly_report_is_actionable_anomaly"] is True - assert data["report_truth"]["freshness_gate_implemented"] is False + assert data["report_truth"]["freshness_gate_implemented"] is True + assert data["report_truth"]["source_confidence_gate_implemented"] is True + assert data["report_truth"]["actionability_score_implemented"] is True assert data["report_truth"]["telegram_report_send_allowed"] is False + assert data["rollups"]["source_freshness_gate_count"] == len(data["source_freshness_gates"]) + assert data["rollups"]["source_confidence_gate_count"] == len(data["source_confidence_gates"]) + assert data["rollups"]["report_actionability_score_count"] == len(data["report_actionability_scores"]) + assert data["rollups"]["zero_signal_work_item_count"] == len(data["zero_signal_work_items"]) + assert data["rollups"]["all_zero_weekly_report_actionability_score"] == 82 assert data["rollups"]["zero_signal_finding_count"] == len(data["zero_signal_findings"]) assert data["rollups"]["missing_cadence_contract_count"] == 1 assert data["telegram_routing_consolidation"]["canonical_room_name"] == "AwoooI SRE 戰情室" @@ -52,6 +59,36 @@ def test_rejects_runtime_report_send_allowed(tmp_path): load_latest_ai_agent_report_truth_actionability_review(tmp_path) +def test_rejects_missing_source_freshness_gate(tmp_path): + data = load_latest_ai_agent_report_truth_actionability_review() + bad = copy.deepcopy(data) + bad["source_freshness_gates"] = [ + gate for gate in bad["source_freshness_gates"] if gate["gate_id"] != "stats_api_incident_rollup" + ] + bad["rollups"]["source_freshness_gate_count"] = len(bad["source_freshness_gates"]) + bad["rollups"]["stale_or_untrusted_source_count"] = sum( + 1 for gate in bad["source_freshness_gates"] if gate["freshness_state"] != "report_only_when_fresh" + ) + bad["rollups"]["work_item_required_count"] = len(bad["source_freshness_gates"]) + _write_snapshot(tmp_path, bad) + + with pytest.raises(ValueError, match="stats_api_incident_rollup"): + load_latest_ai_agent_report_truth_actionability_review(tmp_path) + + +def test_rejects_weekly_score_below_action_required_threshold(tmp_path): + data = load_latest_ai_agent_report_truth_actionability_review() + bad = copy.deepcopy(data) + for item in bad["report_actionability_scores"]: + if item["report_id"] == "weekly_report": + item["score"] = 40 + bad["rollups"]["all_zero_weekly_report_actionability_score"] = 40 + _write_snapshot(tmp_path, bad) + + with pytest.raises(ValueError, match="weekly report"): + load_latest_ai_agent_report_truth_actionability_review(tmp_path) + + def test_rejects_missing_heartbeat_noise_lane(tmp_path): data = load_latest_ai_agent_report_truth_actionability_review() bad = copy.deepcopy(data) diff --git a/apps/api/tests/test_ai_agent_report_truth_actionability_review_api.py b/apps/api/tests/test_ai_agent_report_truth_actionability_review_api.py index ae8a1e48d..409d50c39 100644 --- a/apps/api/tests/test_ai_agent_report_truth_actionability_review_api.py +++ b/apps/api/tests/test_ai_agent_report_truth_actionability_review_api.py @@ -10,11 +10,20 @@ def test_get_ai_agent_report_truth_actionability_review_api(): assert response.status_code == 200 data = response.json() assert data["schema_version"] == "ai_agent_report_truth_actionability_review_v1" - assert data["program_status"]["current_task_id"] == "P2-403J" - assert data["program_status"]["next_task_id"] == "P2-403K" + assert data["program_status"]["current_task_id"] == "P2-403K" + assert data["program_status"]["next_task_id"] == "P2-403L" assert data["report_truth"]["all_zero_weekly_report_is_actionable_anomaly"] is True - assert data["report_truth"]["freshness_gate_implemented"] is False + assert data["report_truth"]["freshness_gate_implemented"] is True + assert data["report_truth"]["source_confidence_gate_implemented"] is True + assert data["report_truth"]["actionability_score_implemented"] is True assert data["report_truth"]["telegram_report_send_allowed"] is False + assert data["rollups"]["source_freshness_gate_count"] == 6 + assert data["rollups"]["stale_or_untrusted_source_count"] == 5 + assert data["rollups"]["source_confidence_gate_count"] == 5 + assert data["rollups"]["low_confidence_gate_count"] == 5 + assert data["rollups"]["report_actionability_score_count"] == 4 + assert data["rollups"]["zero_signal_work_item_count"] == 5 + assert data["rollups"]["all_zero_weekly_report_actionability_score"] == 82 assert data["rollups"]["zero_signal_finding_count"] == 5 assert data["rollups"]["critical_finding_count"] == 1 assert data["rollups"]["high_finding_count"] == 3 diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index c556f45ba..8745ad48d 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -4337,15 +4337,24 @@ } }, "reportTruthActionabilityReview": { - "title": "P2-403J 報表真相與告警有效性", + "title": "P2-403K 報表真相與告警有效性", "source": "{generated} · {current} → {next}", - "truthTitle": "報表真相", - "telegramTitle": "AwoooI SRE 戰情室路由", - "policyTitle": "收斂與批准邊界", - "metrics": { - "overall": "P2-403J 進度", - "findings": "真相缺口", - "critical": "Critical", + "truthTitle": "報表真相", + "telegramTitle": "AwoooI SRE 戰情室路由", + "policyTitle": "收斂與批准邊界", + "freshnessRadarTitle": "來源 freshness 雷達", + "actionabilityScoreTitle": "報表可處置分數", + "confidenceGateTitle": "低信任欄位", + "workItemQueueTitle": "全 0 工作項隊列", + "metrics": { + "overall": "P2-403K 進度", + "weeklyScore": "週報處置分數", + "sourceFreshness": "來源 Gate", + "staleSources": "不可信來源", + "lowConfidence": "低信任欄位", + "zeroWorkItems": "待建工作項", + "findings": "真相缺口", + "critical": "Critical", "cadences": "日週月", "missingCadence": "缺契約", "telegramRoutes": "TG 旁路", @@ -4362,11 +4371,13 @@ "otherRoutes": "其他群組允許: {value}", "routeChange": "路由變更允許: {value}" }, - "labels": { - "canonicalRoom": "唯一戰情室: {room}", - "legacyRoutesDetail": "direct send / legacy chat / multi bot 必須收斂" - } - }, + "labels": { + "canonicalRoom": "唯一戰情室: {room}", + "legacyRoutesDetail": "direct send / legacy chat / multi bot 必須收斂", + "ttl": "TTL: {value}", + "requiredAssets": "必填資產 {count} 項" + } + }, "reportAutomationReview": { "title": "P2-403J 日週月報與風險自動化 Review", "source": "{generated} · {current} → {next}", diff --git a/apps/web/messages/zh-TW.json b/apps/web/messages/zh-TW.json index c556f45ba..8745ad48d 100644 --- a/apps/web/messages/zh-TW.json +++ b/apps/web/messages/zh-TW.json @@ -4337,15 +4337,24 @@ } }, "reportTruthActionabilityReview": { - "title": "P2-403J 報表真相與告警有效性", + "title": "P2-403K 報表真相與告警有效性", "source": "{generated} · {current} → {next}", - "truthTitle": "報表真相", - "telegramTitle": "AwoooI SRE 戰情室路由", - "policyTitle": "收斂與批准邊界", - "metrics": { - "overall": "P2-403J 進度", - "findings": "真相缺口", - "critical": "Critical", + "truthTitle": "報表真相", + "telegramTitle": "AwoooI SRE 戰情室路由", + "policyTitle": "收斂與批准邊界", + "freshnessRadarTitle": "來源 freshness 雷達", + "actionabilityScoreTitle": "報表可處置分數", + "confidenceGateTitle": "低信任欄位", + "workItemQueueTitle": "全 0 工作項隊列", + "metrics": { + "overall": "P2-403K 進度", + "weeklyScore": "週報處置分數", + "sourceFreshness": "來源 Gate", + "staleSources": "不可信來源", + "lowConfidence": "低信任欄位", + "zeroWorkItems": "待建工作項", + "findings": "真相缺口", + "critical": "Critical", "cadences": "日週月", "missingCadence": "缺契約", "telegramRoutes": "TG 旁路", @@ -4362,11 +4371,13 @@ "otherRoutes": "其他群組允許: {value}", "routeChange": "路由變更允許: {value}" }, - "labels": { - "canonicalRoom": "唯一戰情室: {room}", - "legacyRoutesDetail": "direct send / legacy chat / multi bot 必須收斂" - } - }, + "labels": { + "canonicalRoom": "唯一戰情室: {room}", + "legacyRoutesDetail": "direct send / legacy chat / multi bot 必須收斂", + "ttl": "TTL: {value}", + "requiredAssets": "必填資產 {count} 項" + } + }, "reportAutomationReview": { "title": "P2-403J 日週月報與風險自動化 Review", "source": "{generated} · {current} → {next}", diff --git a/apps/web/src/app/[locale]/governance/tabs/automation-inventory-tab.tsx b/apps/web/src/app/[locale]/governance/tabs/automation-inventory-tab.tsx index ec00e78b8..d599fa246 100644 --- a/apps/web/src/app/[locale]/governance/tabs/automation-inventory-tab.tsx +++ b/apps/web/src/app/[locale]/governance/tabs/automation-inventory-tab.tsx @@ -2030,6 +2030,45 @@ export function AutomationInventoryTab() { }) }, [reportTruthActionabilityReview]) + const visibleReportFreshnessGates = useMemo(() => { + if (!reportTruthActionabilityReview) return [] + const priority = { + stale_or_untrusted: 0, + synthetic_or_unbound: 1, + source_unavailable_in_container: 2, + missing_source_contract: 3, + route_fragmented_needs_canonical_readback: 4, + report_only_when_fresh: 5, + } as Record + return [...reportTruthActionabilityReview.source_freshness_gates] + .sort((a, b) => { + const left = priority[a.freshness_state] ?? 6 + const right = priority[b.freshness_state] ?? 6 + if (left !== right) return left - right + return a.gate_id.localeCompare(b.gate_id) + }) + .slice(0, 6) + }, [reportTruthActionabilityReview]) + + const visibleReportConfidenceGates = useMemo(() => { + if (!reportTruthActionabilityReview) return [] + return [...reportTruthActionabilityReview.source_confidence_gates] + .sort((a, b) => a.score - b.score) + .slice(0, 5) + }, [reportTruthActionabilityReview]) + + const visibleReportActionabilityScores = useMemo(() => { + if (!reportTruthActionabilityReview) return [] + return [...reportTruthActionabilityReview.report_actionability_scores] + .sort((a, b) => b.score - a.score) + .slice(0, 4) + }, [reportTruthActionabilityReview]) + + const visibleZeroSignalWorkItems = useMemo(() => { + if (!reportTruthActionabilityReview) return [] + return [...reportTruthActionabilityReview.zero_signal_work_items].slice(0, 5) + }, [reportTruthActionabilityReview]) + const visibleOwnerDryRunGates = useMemo(() => { if (!ownerDryRunPackage) return [] const priority = { approval_required: 0, approved_for_fixture_only: 1, fixture_only: 2, ready: 3 } as Record @@ -3746,11 +3785,11 @@ export function AutomationInventoryTab() { + resultCaptureReleaseDecisionOwnerResponseAcceptanceGate.rollups.production_write_count ) const reportTruthOverall = reportTruthActionabilityReview.program_status.overall_completion_percent - const reportTruthFindings = reportTruthActionabilityReview.rollups.zero_signal_finding_count - const reportTruthCritical = reportTruthActionabilityReview.rollups.critical_finding_count - const reportTruthCadences = reportTruthActionabilityReview.rollups.cadence_contract_count - const reportTruthMissingCadence = reportTruthActionabilityReview.rollups.missing_cadence_contract_count - const reportTruthRouteFindings = reportTruthActionabilityReview.rollups.telegram_route_finding_count + const reportTruthFreshnessGates = reportTruthActionabilityReview.rollups.source_freshness_gate_count + const reportTruthStaleSources = reportTruthActionabilityReview.rollups.stale_or_untrusted_source_count + const reportTruthLowConfidence = reportTruthActionabilityReview.rollups.low_confidence_gate_count + const reportTruthZeroWorkItems = reportTruthActionabilityReview.rollups.zero_signal_work_item_count + const reportTruthWeeklyScore = reportTruthActionabilityReview.rollups.all_zero_weekly_report_actionability_score const reportTruthLegacyRoutes = reportTruthActionabilityReview.rollups.legacy_or_direct_route_count const reportTruthApprovals = reportTruthActionabilityReview.rollups.approval_required_action_ids.length const reportTruthBlockedActions = reportTruthActionabilityReview.rollups.blocked_runtime_action_count @@ -13092,11 +13131,104 @@ export function AutomationInventoryTab() {
} /> - } /> - } /> - } /> - } /> - } /> + } /> + } /> + } /> + } /> + } /> +
+ +
+
+ {t('reportTruthActionabilityReview.freshnessRadarTitle')} +
+ {visibleReportFreshnessGates.map(gate => ( +
+
+ + {gate.display_name} + + +
+ + + {t('reportTruthActionabilityReview.labels.ttl', { value: gate.required_ttl })} + + + {gate.ai_next_action} + +
+ ))} +
+
+ +
+ {t('reportTruthActionabilityReview.actionabilityScoreTitle')} +
+ {visibleReportActionabilityScores.map(score => ( +
+
+
+ + {score.display_name} + + +
+
+
= 80 ? '#d97757' : score.score >= 50 ? '#f59e0b' : '#64727a' }} /> +
+ + {score.next_action} + +
+ = 80 ? '#d97757' : '#141413', textAlign: 'right' }}> + {score.score} + +
+ ))} +
+
+
+ +
+
+ {t('reportTruthActionabilityReview.confidenceGateTitle')} + {visibleReportConfidenceGates.map(gate => ( +
+
+ + {gate.display_name} + + + {gate.display_policy} + +
+ + {gate.score} + +
+ ))} +
+ +
+ {t('reportTruthActionabilityReview.workItemQueueTitle')} +
+ {visibleZeroSignalWorkItems.map(item => ( +
+ + {item.work_item_id} + +
+ + +
+ + {t('reportTruthActionabilityReview.labels.requiredAssets', { count: item.required_assets.length })} + +
+ ))} +
+
diff --git a/apps/web/src/lib/api-client.ts b/apps/web/src/lib/api-client.ts index e98c92892..681359adb 100644 --- a/apps/web/src/lib/api-client.ts +++ b/apps/web/src/lib/api-client.ts @@ -2770,14 +2770,52 @@ export interface AiAgentReportTruthActionabilityReviewSnapshot { daily_report_contract_present: boolean weekly_report_contract_present: boolean monthly_report_contract_present: false - freshness_gate_implemented: false - source_confidence_gate_implemented: false - actionability_score_implemented: false + freshness_gate_implemented: true + source_confidence_gate_implemented: true + actionability_score_implemented: true ai_agent_runtime_control_allowed: false telegram_report_send_allowed: false cronjob_change_allowed: false truth_note: string } + source_freshness_gates: Array<{ + gate_id: string + display_name: string + freshness_state: string + last_success_at: string + required_ttl: string + evidence_ref: string + operator_impact: string + ai_next_action: string + owner_agent: 'openclaw' | 'hermes' | 'nemotron' + work_item_required: true + }> + source_confidence_gates: Array<{ + confidence_id: string + display_name: string + confidence: 'low' + score: number + reason: string + display_policy: string + blocked_green_badge: true + owner_agent: 'openclaw' | 'hermes' | 'nemotron' + }> + report_actionability_scores: Array<{ + report_id: string + display_name: string + score: number + lane: string + send_policy: string + next_action: string + owner_agent: 'openclaw' | 'hermes' | 'nemotron' + }> + zero_signal_work_items: Array<{ + work_item_id: string + asset_type: string + status: 'draft_required' + source_gate_id: string + required_assets: string[] + }> zero_signal_findings: Array<{ finding_id: string display_name: string @@ -2837,6 +2875,14 @@ export interface AiAgentReportTruthActionabilityReviewSnapshot { approval_boundaries: Record rollups: { zero_signal_finding_count: number + source_freshness_gate_count: number + stale_or_untrusted_source_count: number + source_confidence_gate_count: number + low_confidence_gate_count: number + report_actionability_score_count: number + actionability_score_ready_count: number + zero_signal_work_item_count: number + work_item_required_count: number critical_finding_count: number high_finding_count: number cadence_contract_count: number @@ -2848,6 +2894,7 @@ export interface AiAgentReportTruthActionabilityReviewSnapshot { approval_required_action_ids: string[] blocked_runtime_action_count: number all_zero_weekly_report_confidence: 'low_trust_actionable_anomaly' + all_zero_weekly_report_actionability_score: number } } diff --git a/docs/evaluations/ai_agent_report_truth_actionability_review_2026-06-12.json b/docs/evaluations/ai_agent_report_truth_actionability_review_2026-06-12.json index d6e4941ee..64102b4a9 100644 --- a/docs/evaluations/ai_agent_report_truth_actionability_review_2026-06-12.json +++ b/docs/evaluations/ai_agent_report_truth_actionability_review_2026-06-12.json @@ -2,13 +2,13 @@ "schema_version": "ai_agent_report_truth_actionability_review_v1", "generated_at": "2026-06-12T10:48:00+08:00", "program_status": { - "overall_completion_percent": 99, + "overall_completion_percent": 100, "current_priority": "P2", - "current_task_id": "P2-403J", - "next_task_id": "P2-403K", + "current_task_id": "P2-403K", + "next_task_id": "P2-403L", "read_only_mode": true, "runtime_authority": "report_truth_actionability_review_only_no_report_send_or_runtime_fix", - "status_note": "P2-403J 將日報、週報、月報、心跳與告警可處置性固定成真相審查契約;目前只揭露資料可信度與下一步接管缺口,不發 Telegram、不修改 CronJob、不啟動修復。" + "status_note": "P2-403K 將日報、週報、月報、心跳與告警可處置性升級為來源新鮮度、資料信任度與可處置分數看板;目前只揭露接管缺口與工作項隊列,不發 Telegram、不修改 CronJob、不啟動修復。" }, "source_refs": [ "apps/api/src/services/weekly_report_service.py", @@ -29,14 +29,215 @@ "daily_report_contract_present": true, "weekly_report_contract_present": true, "monthly_report_contract_present": false, - "freshness_gate_implemented": false, - "source_confidence_gate_implemented": false, - "actionability_score_implemented": false, + "freshness_gate_implemented": true, + "source_confidence_gate_implemented": true, + "actionability_score_implemented": true, "ai_agent_runtime_control_allowed": false, "telegram_report_send_allowed": false, "cronjob_change_allowed": false, - "truth_note": "目前已確認週報全 0 不能視為正常;weekly report 有多個資料源失敗後降級成 0 / 硬編值的路徑,必須先建立 freshness、source confidence 與 actionability gate。" + "truth_note": "週報全 0 已固定判定為資料鏈路異常;P2-403K 已把 freshness、source confidence 與 actionability score 做成只讀 gate,讓 operator 直接看到哪個來源不可信、該產生哪個工作項。" }, + "source_freshness_gates": [ + { + "gate_id": "stats_api_incident_rollup", + "display_name": "Stats / Incident 統計", + "freshness_state": "stale_or_untrusted", + "last_success_at": "not_recorded", + "required_ttl": "15m for alert summary / 24h for daily report / 7d for weekly rollup", + "evidence_ref": "apps/api/src/services/weekly_report_service.py#get_incident_summary", + "operator_impact": "告警總數與解決率可能因查詢失敗被顯示為 0。", + "ai_next_action": "建立 report-source-gap:stats_api,補 last_success_at、query_error、row_count 與 verifier。", + "owner_agent": "openclaw", + "work_item_required": true + }, + { + "gate_id": "k3s_metrics_rollup", + "display_name": "K3s / Prometheus 指標", + "freshness_state": "synthetic_or_unbound", + "last_success_at": "not_recorded", + "required_ttl": "5m for uptime / restart / HPA evidence", + "evidence_ref": "apps/api/src/services/weekly_report_service.py#get_k3s_health", + "operator_impact": "Uptime 99.90%、Pod 重啟 0、HPA 0 可能是固定值,不代表叢集健康。", + "ai_next_action": "建立 report-source-gap:k3s_metrics,綁定 Prometheus / K8s read model 與 freshness verifier。", + "owner_agent": "hermes", + "work_item_required": true + }, + { + "gate_id": "gitea_activity_rollup", + "display_name": "Gitea commit / deploy 活動", + "freshness_state": "source_unavailable_in_container", + "last_success_at": "not_recorded", + "required_ttl": "24h for daily / 7d for weekly", + "evidence_ref": "apps/api/src/services/weekly_report_service.py#get_git_stats", + "operator_impact": "Commits 與部署次數可能因容器沒有 git history 而顯示 0。", + "ai_next_action": "建立 report-source-gap:gitea_activity,改接 Gitea workflow / deploy marker readback。", + "owner_agent": "openclaw", + "work_item_required": true + }, + { + "gate_id": "ai_cost_ledger", + "display_name": "AI 成本 / Tokens", + "freshness_state": "missing_source_contract", + "last_success_at": "not_recorded", + "required_ttl": "24h for daily / 7d for weekly / 30d for monthly", + "evidence_ref": "apps/api/src/services/weekly_report_service.py#generate_report", + "operator_impact": "費用 $0.00 與 Tokens 0 目前不能代表沒有成本。", + "ai_next_action": "建立 report-source-gap:ai_cost_ledger,補 provider token/cost ledger 與缺資料紅燈。", + "owner_agent": "hermes", + "work_item_required": true + }, + { + "gate_id": "telegram_delivery_receipt", + "display_name": "Telegram delivery receipt", + "freshness_state": "route_fragmented_needs_canonical_readback", + "last_success_at": "not_recorded", + "required_ttl": "per send attempt / 1h route drift review", + "evidence_ref": "apps/api/src/services/telegram_gateway.py", + "operator_impact": "報表與告警可能分散到不同 Bot 或群組,導致 SRE 戰情室看不到。", + "ai_next_action": "建立 report-source-gap:telegram_delivery,所有正式告警只讀對齊 AwoooI SRE 戰情室 receipt。", + "owner_agent": "openclaw", + "work_item_required": true + }, + { + "gate_id": "heartbeat_watchdog", + "display_name": "Watchdog / DeadMansSwitch 心跳", + "freshness_state": "report_only_when_fresh", + "last_success_at": "source_specific", + "required_ttl": "source-specific heartbeat interval + dedupe window", + "evidence_ref": "apps/api/src/core/constants.py", + "operator_impact": "正常心跳不應洗版;心跳過期才應升級為資料來源異常。", + "ai_next_action": "建立 heartbeat actionability gate:fresh=摘要、stale=failure-only、impact=action-required。", + "owner_agent": "nemotron", + "work_item_required": true + } + ], + "source_confidence_gates": [ + { + "confidence_id": "alerts_count_confidence", + "display_name": "告警統計可信度", + "confidence": "low", + "score": 20, + "reason": "告警統計失敗會降級為 0,且缺 last_success_at / query_error。", + "display_policy": "禁止顯示綠燈;必須標示 DATA MISSING 或 LOW TRUST。", + "blocked_green_badge": true, + "owner_agent": "openclaw" + }, + { + "confidence_id": "ai_performance_confidence", + "display_name": "AI 效能可信度", + "confidence": "low", + "score": 15, + "reason": "提案數、執行數、成功率、平均回應缺少執行與 verifier 來源交叉驗證。", + "display_policy": "顯示為待補來源,不得當作 AI 自動化成效。", + "blocked_green_badge": true, + "owner_agent": "openclaw" + }, + { + "confidence_id": "k3s_health_confidence", + "display_name": "K3s 健康可信度", + "confidence": "low", + "score": 30, + "reason": "K3s uptime / HPA / restart 欄位仍有固定值與未綁來源問題。", + "display_policy": "顯示為只讀缺口;需 Prometheus/K8s freshness 後才可變綠。", + "blocked_green_badge": true, + "owner_agent": "hermes" + }, + { + "confidence_id": "development_activity_confidence", + "display_name": "開發活動可信度", + "confidence": "low", + "score": 25, + "reason": "git log 容器路徑失敗會回 0,尚未改為 Gitea readback。", + "display_policy": "以 source_unavailable 顯示,禁止當作沒有部署。", + "blocked_green_badge": true, + "owner_agent": "openclaw" + }, + { + "confidence_id": "ai_cost_confidence", + "display_name": "AI 成本可信度", + "confidence": "low", + "score": 0, + "reason": "尚無正式成本 / Token ledger contract。", + "display_policy": "顯示為缺資料;不允許 $0.00 綠燈。", + "blocked_green_badge": true, + "owner_agent": "hermes" + } + ], + "report_actionability_scores": [ + { + "report_id": "daily_report", + "display_name": "日報", + "score": 58, + "lane": "source_gap_review", + "send_policy": "send_with_low_trust_banner_only_after_source_status_visible", + "next_action": "補 24h incident / repair / KM / PlayBook freshness,缺資料時建立 work item。", + "owner_agent": "hermes" + }, + { + "report_id": "weekly_report", + "display_name": "週報", + "score": 82, + "lane": "action_required_source_gap", + "send_policy": "block_green_all_zero; include data quality block", + "next_action": "全 0 週報直接建立 report truth failure work item,不得當正常週報。", + "owner_agent": "openclaw" + }, + { + "report_id": "monthly_report", + "display_name": "月報", + "score": 35, + "lane": "contract_missing", + "send_policy": "do_not_send_until_monthly_truth_contract_exists", + "next_action": "建立 30d trend / SLO / cost / noise reduction / learning delta 契約。", + "owner_agent": "hermes" + }, + { + "report_id": "heartbeat_digest", + "display_name": "心跳摘要", + "score": 68, + "lane": "noise_reduction", + "send_policy": "normal heartbeat summary only; stale source failure-only", + "next_action": "把 Watchdog、NoAlertsReceived、provider heartbeat 分為摘要、來源過期、需處置。", + "owner_agent": "nemotron" + } + ], + "zero_signal_work_items": [ + { + "work_item_id": "report-source-gap:stats_api", + "asset_type": "source_freshness", + "status": "draft_required", + "source_gate_id": "stats_api_incident_rollup", + "required_assets": ["query_error", "last_success_at", "row_count", "verifier_plan"] + }, + { + "work_item_id": "report-source-gap:k3s_metrics", + "asset_type": "metrics_binding", + "status": "draft_required", + "source_gate_id": "k3s_metrics_rollup", + "required_assets": ["prometheus_query", "k8s_read_model", "freshness_ttl", "verifier_plan"] + }, + { + "work_item_id": "report-source-gap:gitea_activity", + "asset_type": "gitea_readback", + "status": "draft_required", + "source_gate_id": "gitea_activity_rollup", + "required_assets": ["workflow_run_query", "deploy_marker_query", "fallback_policy", "verifier_plan"] + }, + { + "work_item_id": "report-source-gap:ai_cost_ledger", + "asset_type": "cost_ledger", + "status": "draft_required", + "source_gate_id": "ai_cost_ledger", + "required_assets": ["provider_usage_source", "token_counter", "cost_mapping", "owner_review"] + }, + { + "work_item_id": "report-source-gap:telegram_delivery", + "asset_type": "delivery_receipt", + "status": "draft_required", + "source_gate_id": "telegram_delivery_receipt", + "required_assets": ["canonical_room_receipt", "route_inventory", "dedupe_fingerprint", "verifier_plan"] + } + ], "zero_signal_findings": [ { "finding_id": "weekly_stats_failure_becomes_zero", @@ -270,6 +471,14 @@ }, "rollups": { "zero_signal_finding_count": 5, + "source_freshness_gate_count": 6, + "stale_or_untrusted_source_count": 5, + "source_confidence_gate_count": 5, + "low_confidence_gate_count": 5, + "report_actionability_score_count": 4, + "actionability_score_ready_count": 4, + "zero_signal_work_item_count": 5, + "work_item_required_count": 6, "critical_finding_count": 1, "high_finding_count": 3, "cadence_contract_count": 3, @@ -284,6 +493,7 @@ "create_report_truth_work_items" ], "blocked_runtime_action_count": 28, - "all_zero_weekly_report_confidence": "low_trust_actionable_anomaly" + "all_zero_weekly_report_confidence": "low_trust_actionable_anomaly", + "all_zero_weekly_report_actionability_score": 82 } } diff --git a/docs/schemas/ai_agent_report_truth_actionability_review_v1.schema.json b/docs/schemas/ai_agent_report_truth_actionability_review_v1.schema.json index dfdb4f65f..bce2db77d 100644 --- a/docs/schemas/ai_agent_report_truth_actionability_review_v1.schema.json +++ b/docs/schemas/ai_agent_report_truth_actionability_review_v1.schema.json @@ -9,6 +9,10 @@ "program_status", "source_refs", "report_truth", + "source_freshness_gates", + "source_confidence_gates", + "report_actionability_scores", + "zero_signal_work_items", "zero_signal_findings", "report_cadence_contracts", "alert_actionability_lanes", @@ -25,10 +29,10 @@ "type": "object", "required": ["overall_completion_percent", "current_priority", "current_task_id", "next_task_id", "read_only_mode", "runtime_authority", "status_note"], "properties": { - "overall_completion_percent": { "const": 99 }, + "overall_completion_percent": { "const": 100 }, "current_priority": { "enum": ["P0", "P1", "P2", "P3"] }, - "current_task_id": { "const": "P2-403J" }, - "next_task_id": { "const": "P2-403K" }, + "current_task_id": { "const": "P2-403K" }, + "next_task_id": { "const": "P2-403L" }, "read_only_mode": { "const": true }, "runtime_authority": { "const": "report_truth_actionability_review_only_no_report_send_or_runtime_fix" }, "status_note": { "type": "string" } @@ -58,9 +62,9 @@ "daily_report_contract_present": { "type": "boolean" }, "weekly_report_contract_present": { "type": "boolean" }, "monthly_report_contract_present": { "const": false }, - "freshness_gate_implemented": { "const": false }, - "source_confidence_gate_implemented": { "const": false }, - "actionability_score_implemented": { "const": false }, + "freshness_gate_implemented": { "const": true }, + "source_confidence_gate_implemented": { "const": true }, + "actionability_score_implemented": { "const": true }, "ai_agent_runtime_control_allowed": { "const": false }, "telegram_report_send_allowed": { "const": false }, "cronjob_change_allowed": { "const": false }, @@ -68,6 +72,80 @@ }, "additionalProperties": false }, + "source_freshness_gates": { + "type": "array", + "minItems": 6, + "items": { + "type": "object", + "required": ["gate_id", "display_name", "freshness_state", "last_success_at", "required_ttl", "evidence_ref", "operator_impact", "ai_next_action", "owner_agent", "work_item_required"], + "properties": { + "gate_id": { "type": "string" }, + "display_name": { "type": "string" }, + "freshness_state": { "type": "string" }, + "last_success_at": { "type": "string" }, + "required_ttl": { "type": "string" }, + "evidence_ref": { "type": "string" }, + "operator_impact": { "type": "string" }, + "ai_next_action": { "type": "string" }, + "owner_agent": { "enum": ["openclaw", "hermes", "nemotron"] }, + "work_item_required": { "const": true } + }, + "additionalProperties": false + } + }, + "source_confidence_gates": { + "type": "array", + "minItems": 5, + "items": { + "type": "object", + "required": ["confidence_id", "display_name", "confidence", "score", "reason", "display_policy", "blocked_green_badge", "owner_agent"], + "properties": { + "confidence_id": { "type": "string" }, + "display_name": { "type": "string" }, + "confidence": { "const": "low" }, + "score": { "type": "integer", "minimum": 0, "maximum": 100 }, + "reason": { "type": "string" }, + "display_policy": { "type": "string" }, + "blocked_green_badge": { "const": true }, + "owner_agent": { "enum": ["openclaw", "hermes", "nemotron"] } + }, + "additionalProperties": false + } + }, + "report_actionability_scores": { + "type": "array", + "minItems": 4, + "items": { + "type": "object", + "required": ["report_id", "display_name", "score", "lane", "send_policy", "next_action", "owner_agent"], + "properties": { + "report_id": { "type": "string" }, + "display_name": { "type": "string" }, + "score": { "type": "integer", "minimum": 0, "maximum": 100 }, + "lane": { "type": "string" }, + "send_policy": { "type": "string" }, + "next_action": { "type": "string" }, + "owner_agent": { "enum": ["openclaw", "hermes", "nemotron"] } + }, + "additionalProperties": false + } + }, + "zero_signal_work_items": { + "type": "array", + "minItems": 5, + "items": { + "type": "object", + "required": ["work_item_id", "asset_type", "status", "source_gate_id", "required_assets"], + "properties": { + "work_item_id": { "type": "string" }, + "asset_type": { "type": "string" }, + "status": { "const": "draft_required" }, + "source_gate_id": { "type": "string" }, + "required_assets": { "type": "array", "items": { "type": "string" }, "minItems": 1 } + }, + "additionalProperties": false + } + }, "zero_signal_findings": { "type": "array", "minItems": 1 }, "report_cadence_contracts": { "type": "array", "minItems": 3 }, "alert_actionability_lanes": { "type": "array", "minItems": 1 }, @@ -102,6 +180,14 @@ "type": "object", "required": [ "zero_signal_finding_count", + "source_freshness_gate_count", + "stale_or_untrusted_source_count", + "source_confidence_gate_count", + "low_confidence_gate_count", + "report_actionability_score_count", + "actionability_score_ready_count", + "zero_signal_work_item_count", + "work_item_required_count", "critical_finding_count", "high_finding_count", "cadence_contract_count", @@ -112,10 +198,19 @@ "operator_action_count", "approval_required_action_ids", "blocked_runtime_action_count", - "all_zero_weekly_report_confidence" + "all_zero_weekly_report_confidence", + "all_zero_weekly_report_actionability_score" ], "properties": { "zero_signal_finding_count": { "type": "integer", "minimum": 1 }, + "source_freshness_gate_count": { "type": "integer", "minimum": 6 }, + "stale_or_untrusted_source_count": { "type": "integer", "minimum": 1 }, + "source_confidence_gate_count": { "type": "integer", "minimum": 5 }, + "low_confidence_gate_count": { "type": "integer", "minimum": 1 }, + "report_actionability_score_count": { "type": "integer", "minimum": 4 }, + "actionability_score_ready_count": { "type": "integer", "minimum": 4 }, + "zero_signal_work_item_count": { "type": "integer", "minimum": 5 }, + "work_item_required_count": { "type": "integer", "minimum": 1 }, "critical_finding_count": { "type": "integer", "minimum": 1 }, "high_finding_count": { "type": "integer", "minimum": 1 }, "cadence_contract_count": { "type": "integer", "minimum": 3 }, @@ -126,7 +221,8 @@ "operator_action_count": { "type": "integer", "minimum": 1 }, "approval_required_action_ids": { "type": "array", "items": { "type": "string" } }, "blocked_runtime_action_count": { "type": "integer", "minimum": 1 }, - "all_zero_weekly_report_confidence": { "const": "low_trust_actionable_anomaly" } + "all_zero_weekly_report_confidence": { "const": "low_trust_actionable_anomaly" }, + "all_zero_weekly_report_actionability_score": { "type": "integer", "minimum": 80, "maximum": 100 } }, "additionalProperties": false }