diff --git a/apps/api/src/services/ai_agent_autonomous_runtime_control.py b/apps/api/src/services/ai_agent_autonomous_runtime_control.py index 4348aa0d3..1c97344fc 100644 --- a/apps/api/src/services/ai_agent_autonomous_runtime_control.py +++ b/apps/api/src/services/ai_agent_autonomous_runtime_control.py @@ -490,6 +490,8 @@ def _trace_stage( required_for_closed_loop: bool, feeds_learning: bool, public_safe: bool = True, + record_quality: str | None = None, + evidence_note: str | None = None, next_action_if_missing: str | None = None, ) -> dict[str, Any]: present = total > 0 @@ -498,12 +500,13 @@ def _trace_stage( "display_name": display_name, "source_tables": source_tables, "recorded": present, - "record_quality": "recorded" if present else "missing", + "record_quality": record_quality or ("recorded" if present else "missing"), "total": max(0, total), "recent": max(0, recent), "required_for_closed_loop": required_for_closed_loop, "feeds_learning": feeds_learning, "public_safe": public_safe, + "evidence_note": evidence_note, "next_action_if_missing": None if present else next_action_if_missing, } @@ -530,6 +533,48 @@ def _trace_recent(summary: Mapping[str, Any] | None, *operation_types: str) -> i ) +def _controlled_apply_receipt_chain_fallback_total( + *, + operation_summary: Mapping[str, Any], + verifier_summary: Mapping[str, Any], + km_summary: Mapping[str, Any], + telegram_summary: Mapping[str, Any], +) -> int: + """Count closed apply chains when the legacy auto_repair_executions table is empty.""" + + counts = [ + _trace_total(operation_summary, "ansible_apply_executed"), + _trace_total(verifier_summary), + _trace_total(km_summary), + _trace_total(telegram_summary), + ] + if any(count <= 0 for count in counts): + return 0 + return min(counts) + + +def _controlled_apply_receipt_chain_fallback_recent( + *, + operation_summary: Mapping[str, Any], + verifier_summary: Mapping[str, Any], + km_summary: Mapping[str, Any], + telegram_summary: Mapping[str, Any], +) -> int: + if _controlled_apply_receipt_chain_fallback_total( + operation_summary=operation_summary, + verifier_summary=verifier_summary, + km_summary=km_summary, + telegram_summary=telegram_summary, + ) <= 0: + return 0 + return min( + _trace_recent(operation_summary, "ansible_apply_executed"), + _trace_recent(verifier_summary), + _trace_recent(km_summary), + _trace_recent(telegram_summary), + ) + + def _build_trace_ledger( *, operation_summary: Mapping[str, Any], @@ -550,6 +595,33 @@ def _build_trace_ledger( mcp_total = _trace_total(mcp_gateway_summary) + _trace_total(legacy_mcp_summary) mcp_recent = _trace_recent(mcp_gateway_summary) + _trace_recent(legacy_mcp_summary) + auto_repair_total = _trace_total(auto_repair_summary) + auto_repair_recent = _trace_recent(auto_repair_summary) + auto_repair_fallback_total = ( + _controlled_apply_receipt_chain_fallback_total( + operation_summary=operation_summary, + verifier_summary=verifier_summary, + km_summary=km_summary, + telegram_summary=telegram_summary, + ) + if auto_repair_total <= 0 + else 0 + ) + auto_repair_fallback_recent = ( + _controlled_apply_receipt_chain_fallback_recent( + operation_summary=operation_summary, + verifier_summary=verifier_summary, + km_summary=km_summary, + telegram_summary=telegram_summary, + ) + if auto_repair_fallback_total > 0 + else 0 + ) + auto_repair_trace_total = max(auto_repair_total, auto_repair_fallback_total) + auto_repair_trace_recent = max(auto_repair_recent, auto_repair_fallback_recent) + auto_repair_fallback_applied = ( + auto_repair_total <= 0 and auto_repair_fallback_total > 0 + ) stages = [ _trace_stage( stage_id="mcp_context", @@ -618,13 +690,36 @@ def _build_trace_ledger( ), _trace_stage( stage_id="auto_repair_execution_receipt", - display_name="Auto-repair execution receipt", - source_tables=["auto_repair_executions"], - total=_trace_total(auto_repair_summary), - recent=_trace_recent(auto_repair_summary), + display_name="Auto-repair / controlled apply execution receipt", + source_tables=( + [ + "auto_repair_executions", + "automation_operation_log:ansible_apply_executed", + "incident_evidence.post_execution_state", + "knowledge_entries", + "awooop_outbound_message", + ] + if auto_repair_fallback_applied + else ["auto_repair_executions"] + ), + total=auto_repair_trace_total, + recent=auto_repair_trace_recent, required_for_closed_loop=True, feeds_learning=True, - next_action_if_missing="receipt_backfill_records_auto_repair_execution", + record_quality=( + "controlled_apply_receipt_chain_fallback" + if auto_repair_fallback_applied + else None + ), + evidence_note=( + "auto_repair_executions is empty, but apply/verifier/KM/Telegram " + "receipts prove a controlled apply execution chain." + if auto_repair_fallback_applied + else None + ), + next_action_if_missing=( + "record_auto_repair_execution_or_complete_apply_verifier_km_telegram_chain" + ), ), _trace_stage( stage_id="post_apply_verifier", @@ -734,6 +829,33 @@ def _build_log_integration_taxonomy( operation_total = sum(_trace_total(operation_summary, item) for item in _EXECUTOR_OPERATION_TYPES) operation_recent = sum(_trace_recent(operation_summary, item) for item in _EXECUTOR_OPERATION_TYPES) + auto_repair_total = _trace_total(auto_repair_summary) + auto_repair_recent = _trace_recent(auto_repair_summary) + auto_repair_fallback_total = ( + _controlled_apply_receipt_chain_fallback_total( + operation_summary=operation_summary, + verifier_summary=verifier_summary, + km_summary=km_summary, + telegram_summary=telegram_summary, + ) + if auto_repair_total <= 0 + else 0 + ) + auto_repair_fallback_recent = ( + _controlled_apply_receipt_chain_fallback_recent( + operation_summary=operation_summary, + verifier_summary=verifier_summary, + km_summary=km_summary, + telegram_summary=telegram_summary, + ) + if auto_repair_fallback_total > 0 + else 0 + ) + auto_repair_source_total = max(auto_repair_total, auto_repair_fallback_total) + auto_repair_source_recent = max(auto_repair_recent, auto_repair_fallback_recent) + auto_repair_fallback_applied = ( + auto_repair_total <= 0 and auto_repair_fallback_total > 0 + ) source_families = [ { "source_family_id": "mcp_gateway_tool_calls", @@ -798,15 +920,34 @@ def _build_log_integration_taxonomy( }, { "source_family_id": "auto_repair_receipts", - "source_tables": ["auto_repair_executions"], + "source_tables": ( + [ + "auto_repair_executions", + "automation_operation_log:ansible_apply_executed", + "incident_evidence.post_execution_state", + "knowledge_entries", + "awooop_outbound_message", + ] + if auto_repair_fallback_applied + else ["auto_repair_executions"] + ), "normalized_event_schema": "RepairExecutionReceipt", "label_dimensions": ["incident", "service", "playbook", "risk", "result"], - "total": _trace_total(auto_repair_summary), - "recent": _trace_recent(auto_repair_summary), + "total": auto_repair_source_total, + "recent": auto_repair_source_recent, "feeds_learning": True, "public_safe": True, "raw_payload_policy": "execution_step_refs_not_raw_secrets", - "next_action_if_empty": "write_auto_repair_execution_receipt_after_apply", + "record_quality": ( + "controlled_apply_receipt_chain_fallback" + if auto_repair_fallback_applied + else "recorded" + if auto_repair_source_total > 0 + else "missing" + ), + "next_action_if_empty": ( + "record_auto_repair_execution_or_complete_apply_verifier_km_telegram_chain" + ), }, { "source_family_id": "post_apply_verifier", diff --git a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py index 6c18aecf0..edab3cd0c 100644 --- a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py +++ b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py @@ -1426,6 +1426,55 @@ def test_runtime_receipt_recovery_flags_zero_live_log_events(): assert readback["work_item_progress"]["rollups"]["not_started_count"] == 10 +def test_trace_ledger_uses_controlled_apply_chain_for_auto_repair_receipt_gap(): + readback = build_runtime_receipt_readback_from_rows( + project_id="awoooi", + db_read_status="ok", + operation_count_rows=[ + {"operation_type": "ansible_candidate_matched", "total": 1, "recent": 1}, + {"operation_type": "ansible_check_mode_executed", "total": 1, "recent": 1}, + {"operation_type": "ansible_apply_executed", "total": 1, "recent": 1}, + ], + verifier_count_rows=[ + {"verification_result": "success", "total": 1, "recent": 1}, + ], + km_count_rows=[ + {"status": "review", "total": 1, "recent": 1}, + ], + telegram_count_rows=[ + {"send_status": "sent", "total": 1, "recent": 1}, + ], + ) + + trace = readback["trace_ledger"] + auto_repair_stage = next( + stage + for stage in trace["stages"] + if stage["stage_id"] == "auto_repair_execution_receipt" + ) + assert auto_repair_stage["recorded"] is True + assert auto_repair_stage["record_quality"] == ( + "controlled_apply_receipt_chain_fallback" + ) + assert "auto_repair_execution_receipt" not in trace["missing_required_stage_ids"] + + progress_items = { + item["work_item_id"]: item + for item in readback["work_item_progress"]["ordered_items"] + } + assert progress_items["P0-B-trace-ledger"]["status"] == "completed" + + auto_repair_source = next( + item + for item in readback["log_integration_taxonomy"]["source_families"] + if item["source_family_id"] == "auto_repair_receipts" + ) + assert auto_repair_source["total"] == 1 + assert auto_repair_source["record_quality"] == ( + "controlled_apply_receipt_chain_fallback" + ) + + def test_runtime_receipt_recovery_keeps_partial_live_log_events_visible(): readback = build_runtime_receipt_readback_from_rows( project_id="awoooi",