diff --git a/apps/api/src/services/ai_agent_autonomous_runtime_control.py b/apps/api/src/services/ai_agent_autonomous_runtime_control.py index 472aaae30..e5a94d1eb 100644 --- a/apps/api/src/services/ai_agent_autonomous_runtime_control.py +++ b/apps/api/src/services/ai_agent_autonomous_runtime_control.py @@ -844,6 +844,95 @@ def _build_log_integration_taxonomy( } +def _build_runtime_receipt_readback_recovery( + *, + db_read_status: str, + log_integration_taxonomy: Mapping[str, Any], + error_type: str | None, +) -> dict[str, Any]: + """Expose the next controlled action when live receipt/log evidence is absent.""" + + taxonomy_rollups = log_integration_taxonomy.get("rollups") + if not isinstance(taxonomy_rollups, Mapping): + taxonomy_rollups = {} + source_families = log_integration_taxonomy.get("source_families") + if not isinstance(source_families, list): + source_families = [] + inactive_source_family_ids = [ + str(source.get("source_family_id")) + for source in source_families + if isinstance(source, Mapping) and _int_value(source.get("total")) <= 0 + ] + classified_event_total = _int_value(taxonomy_rollups.get("classified_event_total")) + recent_classified_event_total = _int_value( + taxonomy_rollups.get("recent_classified_event_total") + ) + db_unavailable = db_read_status != "ok" + no_live_log_events = classified_event_total <= 0 + if db_unavailable: + status = "blocked_runtime_receipt_db_readback_unavailable" + safe_next_action_id = ( + "repair_runtime_receipt_db_readback_then_rerun_log_integration_taxonomy" + ) + safe_next_action_stage = "runtime_receipt_db_context_or_schema_recovery" + safe_next_action = ( + "Restore the API readback path for automation_operation_log, KM, MCP, " + "timeline, PlayBook, and alert evidence counters, then rerun the " + "runtime-control endpoint readback." + ) + blocker_fields = ["db_read_status"] + elif no_live_log_events: + status = "blocked_live_log_source_family_events_missing" + safe_next_action_id = ( + "connect_service_tool_package_logs_to_trace_ledger_then_rerun_taxonomy" + ) + safe_next_action_stage = "log_source_ingestion_and_labeling_required" + safe_next_action = ( + "Route sanitized service, package, tool, MCP, executor, verifier, KM, " + "PlayBook, and notification receipts into the trace ledger with labels." + ) + blocker_fields = ["classified_event_total", "active_source_family_count"] + else: + status = "completed_live_runtime_receipts_observed" + safe_next_action_id = "keep_log_learning_loop_monitored" + safe_next_action_stage = "monitoring" + safe_next_action = "Keep the LOG -> KM/RAG/MCP/PlayBook learning loop monitored." + blocker_fields = [] + + return { + "schema_version": "ai_agent_runtime_receipt_readback_recovery_v1", + "status": status, + "db_read_status": db_read_status, + "error_type": error_type, + "safe_next_action_id": safe_next_action_id, + "safe_next_action_stage": safe_next_action_stage, + "safe_next_action": safe_next_action, + "post_recovery_verifier": ( + "GET /api/v1/agents/agent-autonomous-runtime-control and require " + "runtime_receipt_readback.db_read_status=ok plus active_source_family_count>0" + ), + "classified_event_total": classified_event_total, + "recent_classified_event_total": recent_classified_event_total, + "source_family_count": _int_value(taxonomy_rollups.get("source_family_count")), + "active_source_family_count": _int_value( + taxonomy_rollups.get("active_source_family_count") + ), + "inactive_source_family_ids": inactive_source_family_ids, + "blocker_fields": blocker_fields, + "operation_boundaries": { + "readback_only": True, + "km_write_performed": False, + "rag_index_write_performed": False, + "playbook_trust_write_performed": False, + "mcp_tool_call_performed": False, + "runtime_repair_performed": False, + "raw_log_payload_persisted": False, + "secret_value_collection_allowed": False, + "github_api_used": False, + }, + } + + def _decision_wiring_stage( *, stage_id: str, @@ -1938,6 +2027,8 @@ def _build_work_item_progress( if not isinstance(source_families, list): source_families = [] inactive_source_count = _int_value(taxonomy_rollups.get("inactive_source_family_count")) + active_source_count = _int_value(taxonomy_rollups.get("active_source_family_count")) + classified_event_total = _int_value(taxonomy_rollups.get("classified_event_total")) missing_required = trace_ledger.get("missing_required_stage_ids") if not isinstance(missing_required, list): missing_required = [] @@ -2033,10 +2124,12 @@ def _build_work_item_progress( and multi_product_taxonomy.get("status") == "completed" and multi_product_missing == 0 ) + runtime_truth_complete = db_read_status == "ok" and classified_event_total > 0 deployed_readback_complete = ( db_read_status == "ok" and trace_ledger.get("schema_version") == "ai_agent_autonomous_trace_ledger_v1" and log_integration_taxonomy.get("schema_version") == "ai_agent_log_integration_taxonomy_v1" + and classified_event_total > 0 ) ordered_items = [ @@ -2044,8 +2137,23 @@ def _build_work_item_progress( "work_item_id": "P0-A-runtime-truth", "priority": "P0-A", "title": "Controlled apply runtime truth readback", - "status": "completed", + "status": ( + "completed" + if runtime_truth_complete + else "blocked" + if db_read_status != "ok" + else "in_progress" + ), "exit_criteria": "production API reports db_read_status=ok and live executor receipts", + "blocker": None + if runtime_truth_complete + else ( + "runtime_receipt_db_readback_unavailable" + if db_read_status != "ok" + else "live_log_classified_event_total_zero" + ), + "db_read_status": db_read_status, + "classified_event_total": classified_event_total, }, { "work_item_id": "P0-B-trace-ledger", @@ -2060,6 +2168,8 @@ def _build_work_item_progress( "title": "Project/product/site/service/package/tool log taxonomy", "status": "completed", "exit_criteria": "log_integration_taxonomy lists source families, labels, and public-safety policy", + "active_source_family_count": active_source_count, + "classified_event_total": classified_event_total, }, { "work_item_id": "P0-D-ui-visibility", @@ -3021,6 +3131,11 @@ def build_runtime_receipt_readback_from_rows( timeline_summary=timeline_summary, playbook_trust_summary=playbook_trust_summary, ) + runtime_receipt_readback_recovery = _build_runtime_receipt_readback_recovery( + db_read_status=db_read_status, + log_integration_taxonomy=log_integration_taxonomy, + error_type=error_type, + ) agent_decision_wiring = _build_agent_decision_wiring( operation_summary=operation_summary, verifier_summary=verifier_summary, @@ -3196,6 +3311,7 @@ def build_runtime_receipt_readback_from_rows( "autonomous_execution_loop_ledger": loop_ledger, "trace_ledger": trace_ledger, "log_integration_taxonomy": log_integration_taxonomy, + "runtime_receipt_readback_recovery": runtime_receipt_readback_recovery, "log_controlled_writeback_executor": log_controlled_writeback_executor, "log_controlled_writeback_consumer": dict(log_controlled_writeback_consumer), "agent_decision_wiring": agent_decision_wiring, @@ -3258,6 +3374,9 @@ def _attach_runtime_receipt_readback( log_dispatch_summary = ( operation_counts.get(LOG_CONTROLLED_WRITEBACK_DISPATCH_OPERATION_TYPE) or {} ) + runtime_recovery = readback.get("runtime_receipt_readback_recovery") + if not isinstance(runtime_recovery, Mapping): + runtime_recovery = {} rollups.update({ "live_ansible_apply_executed_count": _int_value( readback.get("ansible_apply_executed", {}).get("total") @@ -3345,6 +3464,15 @@ def _attach_runtime_receipt_readback( "recent_classified_event_total" ) ), + "live_runtime_receipt_recovery_status": runtime_recovery.get("status") or "", + "live_runtime_receipt_safe_next_action_id": runtime_recovery.get( + "safe_next_action_id" + ) + or "", + "live_runtime_receipt_safe_next_action_stage": runtime_recovery.get( + "safe_next_action_stage" + ) + or "", "live_log_controlled_writeback_executor_batch_count": _int_value( log_executor_rollups.get("execution_batch_count") ), diff --git a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py index 531d00423..f3e1ee0eb 100644 --- a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py +++ b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py @@ -339,6 +339,17 @@ def test_ai_agent_autonomous_runtime_control_exposes_reports_and_executor_receip assert data["rollups"]["direct_bot_api_allowed_count"] == 0 assert data["rollups"]["legacy_policy_overridden_count"] >= 4 assert data["runtime_receipt_readback"]["db_read_status"] == "not_queried" + recovery = data["runtime_receipt_readback"]["runtime_receipt_readback_recovery"] + assert recovery["status"] == "blocked_runtime_receipt_db_readback_unavailable" + assert recovery["safe_next_action_id"] == ( + "repair_runtime_receipt_db_readback_then_rerun_log_integration_taxonomy" + ) + assert data["rollups"]["live_runtime_receipt_recovery_status"] == ( + "blocked_runtime_receipt_db_readback_unavailable" + ) + assert data["rollups"]["live_runtime_receipt_safe_next_action_id"] == ( + "repair_runtime_receipt_db_readback_then_rerun_log_integration_taxonomy" + ) _assert_log_controlled_writeback_executor( data["runtime_receipt_readback"]["log_controlled_writeback_executor"] ) @@ -380,6 +391,14 @@ def test_ai_agent_autonomous_runtime_control_exposes_reports_and_executor_receip ] == 7 ) + progress_items = { + item["work_item_id"]: item + for item in data["runtime_receipt_readback"]["work_item_progress"]["ordered_items"] + } + assert progress_items["P0-A-runtime-truth"]["status"] == "blocked" + assert progress_items["P0-A-runtime-truth"]["blocker"] == ( + "runtime_receipt_db_readback_unavailable" + ) def test_ai_agent_autonomous_runtime_control_exposes_internal_control_loop(): @@ -635,6 +654,11 @@ def test_runtime_receipt_readback_summarizes_live_executor_closure_rows(): assert readback["db_read_status"] == "ok" assert readback["writes_on_read"] is False + recovery = readback["runtime_receipt_readback_recovery"] + assert recovery["status"] == "completed_live_runtime_receipts_observed" + assert recovery["classified_event_total"] > 0 + assert recovery["active_source_family_count"] == 10 + assert recovery["inactive_source_family_ids"] == [] assert readback["ansible_apply_executed"]["total"] == 1 assert ( readback["ansible_operations"]["counts"]["log_controlled_writeback_dispatched"][ @@ -903,6 +927,8 @@ def test_runtime_receipt_readback_summarizes_live_executor_closure_rows(): "P2-B-multi-product-expansion", ] assert progress["ordered_items"][4]["status"] == "completed" + assert progress["ordered_items"][0]["status"] == "completed" + assert progress["ordered_items"][0]["classified_event_total"] > 0 assert progress["ordered_items"][5]["status"] == "completed" assert progress["ordered_items"][6]["status"] == "completed" assert progress["ordered_items"][7]["status"] == "completed" @@ -929,6 +955,32 @@ def test_runtime_receipt_readback_summarizes_live_executor_closure_rows(): assert progress["rollups"]["pending_count"] == 0 +def test_runtime_receipt_recovery_flags_zero_live_log_events(): + readback = build_runtime_receipt_readback_from_rows( + project_id="awoooi", + db_read_status="ok", + ) + + recovery = readback["runtime_receipt_readback_recovery"] + assert recovery["status"] == "blocked_live_log_source_family_events_missing" + assert recovery["safe_next_action_id"] == ( + "connect_service_tool_package_logs_to_trace_ledger_then_rerun_taxonomy" + ) + assert recovery["active_source_family_count"] == 0 + assert len(recovery["inactive_source_family_ids"]) == 10 + + progress_items = { + item["work_item_id"]: item + for item in readback["work_item_progress"]["ordered_items"] + } + assert progress_items["P0-A-runtime-truth"]["status"] == "in_progress" + assert progress_items["P0-A-runtime-truth"]["blocker"] == ( + "live_log_classified_event_total_zero" + ) + assert progress_items["P1-A-ingestion-coverage"]["status"] == "in_progress" + assert readback["work_item_progress"]["rollups"]["not_started_count"] == 10 + + def test_runtime_receipt_work_items_use_learning_receipts_without_latest_telegram_gate(): apply_op_id = "2f8ef5c8-fd4e-4950-99e9-dc9e61150cab" incident_id = "INC-20260629-LEARNING"