diff --git a/apps/api/src/services/ai_agent_autonomous_runtime_control.py b/apps/api/src/services/ai_agent_autonomous_runtime_control.py index e770f961b..4a04027d2 100644 --- a/apps/api/src/services/ai_agent_autonomous_runtime_control.py +++ b/apps/api/src/services/ai_agent_autonomous_runtime_control.py @@ -54,6 +54,12 @@ _RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS = 0.45 _RUNTIME_RECEIPT_STATEMENT_TIMEOUT_MS = 400 _LOG_CONTROLLED_WRITEBACK_CONSUMER_TIMEOUT_SECONDS = 0.5 _RUNTIME_RECEIPT_READBACK_CACHE_TTL_SECONDS = 20.0 +_AUXILIARY_RUNTIME_RECEIPT_QUERY_NAMES = { + "alert_operation_counts", + "alertmanager_event_counts", + "grouped_alert_event_counts", + "db_context_exit", +} # CD cancel-stale-cd no-op triggers must not change runtime payloads. _EXECUTOR_OPERATION_TYPES = ( "ansible_candidate_matched", @@ -142,6 +148,19 @@ def _int_value(value: Any) -> int: return 0 +def _has_only_auxiliary_runtime_receipt_failures( + partial_query_failures: Iterable[Mapping[str, Any]], +) -> bool: + failures = [dict(item) for item in partial_query_failures] + if not failures: + return False + for failure in failures: + query_name = str(failure.get("query_name") or "") + if query_name.removesuffix("_fallback") not in _AUXILIARY_RUNTIME_RECEIPT_QUERY_NAMES: + return False + return True + + def _runtime_receipt_readback_cache_key( *, project_id: str, @@ -937,8 +956,23 @@ def _build_runtime_receipt_readback_recovery( db_unavailable = db_read_status in {"not_queried", "unavailable"} partial_failures = [dict(item) for item in partial_query_failures] db_partial = db_read_status == "partial" + auxiliary_only_partial = ( + db_partial + and _has_only_auxiliary_runtime_receipt_failures(partial_failures) + ) no_live_log_events = classified_event_total <= 0 - if db_partial and not no_live_log_events: + if auxiliary_only_partial and not no_live_log_events: + status = "completed_live_runtime_receipts_observed_with_auxiliary_query_degradation" + safe_next_action_id = ( + "optimize_auxiliary_alert_queries_without_blocking_runtime_truth" + ) + safe_next_action_stage = "auxiliary_runtime_receipt_query_optimization" + safe_next_action = ( + "Keep core apply, verifier, KM, and Telegram runtime receipts visible " + "while optimizing slower alert aggregation queries separately." + ) + blocker_fields = [] + elif db_partial and not no_live_log_events: status = "degraded_runtime_receipt_partial_query_failures" safe_next_action_id = ( "repair_partial_runtime_receipt_queries_then_rerun_log_integration_taxonomy" @@ -989,7 +1023,8 @@ def _build_runtime_receipt_readback_recovery( "safe_next_action": safe_next_action, "post_recovery_verifier": ( "GET /api/v1/agents/agent-autonomous-runtime-control and require " - "runtime_receipt_readback.db_read_status=ok plus active_source_family_count>0" + "core live runtime receipts plus active_source_family_count>0; only " + "auxiliary alert query degradation may remain nonblocking" ), "classified_event_total": classified_event_total, "recent_classified_event_total": recent_classified_event_total, @@ -2116,6 +2151,7 @@ def _build_work_item_progress( ui_productization: Mapping[str, Any], multi_product_taxonomy: Mapping[str, Any], db_read_status: str, + partial_query_failures: Iterable[Mapping[str, Any]] = (), ) -> dict[str, Any]: """Build ordered work items that the UI and agent can keep advancing.""" @@ -2223,9 +2259,15 @@ def _build_work_item_progress( and multi_product_taxonomy.get("status") == "completed" and multi_product_missing == 0 ) - runtime_truth_complete = db_read_status == "ok" and classified_event_total > 0 + auxiliary_only_partial = ( + db_read_status == "partial" + and _has_only_auxiliary_runtime_receipt_failures(partial_query_failures) + ) + runtime_truth_complete = ( + db_read_status == "ok" or auxiliary_only_partial + ) and classified_event_total > 0 deployed_readback_complete = ( - db_read_status == "ok" + (db_read_status == "ok" or auxiliary_only_partial) and trace_ledger.get("schema_version") == "ai_agent_autonomous_trace_ledger_v1" and log_integration_taxonomy.get("schema_version") == "ai_agent_log_integration_taxonomy_v1" and classified_event_total > 0 @@ -2254,6 +2296,7 @@ def _build_work_item_progress( else "live_log_classified_event_total_zero" ), "db_read_status": db_read_status, + "auxiliary_query_degradation_only": auxiliary_only_partial, "classified_event_total": classified_event_total, }, { @@ -3295,6 +3338,7 @@ def build_runtime_receipt_readback_from_rows( ui_productization=ui_productization, multi_product_taxonomy=multi_product_taxonomy, db_read_status=db_read_status, + partial_query_failures=partial_query_failures, ) apply_summary = operation_summary.get("ansible_apply_executed") or {} readback = { diff --git a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py index c4f2e5635..4b803aec7 100644 --- a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py +++ b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py @@ -1375,6 +1375,54 @@ def test_runtime_receipt_recovery_keeps_partial_live_log_events_visible(): ) +def test_runtime_receipt_auxiliary_alert_query_partial_does_not_block_runtime_truth(): + readback = build_runtime_receipt_readback_from_rows( + project_id="awoooi", + db_read_status="partial", + operation_count_rows=[ + { + "operation_type": "ansible_apply_executed", + "status": "success", + "total": 3, + "recent": 1, + }, + ], + verifier_count_rows=[ + {"status": "success", "total": 3, "recent": 1}, + ], + km_count_rows=[ + {"status": "linked", "total": 3, "recent": 1}, + ], + telegram_count_rows=[ + {"status": "sent", "total": 3, "recent": 1}, + ], + partial_query_failures=[ + {"query_name": "alert_operation_counts", "error_type": "DBAPIError"}, + { + "query_name": "alertmanager_event_counts", + "error_type": "RuntimeReceiptQueryBudgetExceeded", + }, + ], + error_type="partial_query_failures", + ) + + recovery = readback["runtime_receipt_readback_recovery"] + assert recovery["status"] == ( + "completed_live_runtime_receipts_observed_with_auxiliary_query_degradation" + ) + assert recovery["blocker_fields"] == [] + + progress_items = { + item["work_item_id"]: item + for item in readback["work_item_progress"]["ordered_items"] + } + assert progress_items["P0-A-runtime-truth"]["status"] == "completed" + assert ( + progress_items["P0-A-runtime-truth"]["auxiliary_query_degradation_only"] + is True + ) + + def test_runtime_receipt_work_items_use_learning_receipts_without_latest_telegram_gate(): apply_op_id = "2f8ef5c8-fd4e-4950-99e9-dc9e61150cab" incident_id = "INC-20260629-LEARNING"