fix(agents): keep auxiliary receipt lag nonblocking
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m7s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m7s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
This commit is contained in:
@@ -54,6 +54,12 @@ _RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS = 0.45
|
||||
_RUNTIME_RECEIPT_STATEMENT_TIMEOUT_MS = 400
|
||||
_LOG_CONTROLLED_WRITEBACK_CONSUMER_TIMEOUT_SECONDS = 0.5
|
||||
_RUNTIME_RECEIPT_READBACK_CACHE_TTL_SECONDS = 20.0
|
||||
_AUXILIARY_RUNTIME_RECEIPT_QUERY_NAMES = {
|
||||
"alert_operation_counts",
|
||||
"alertmanager_event_counts",
|
||||
"grouped_alert_event_counts",
|
||||
"db_context_exit",
|
||||
}
|
||||
# CD cancel-stale-cd no-op triggers must not change runtime payloads.
|
||||
_EXECUTOR_OPERATION_TYPES = (
|
||||
"ansible_candidate_matched",
|
||||
@@ -142,6 +148,19 @@ def _int_value(value: Any) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _has_only_auxiliary_runtime_receipt_failures(
|
||||
partial_query_failures: Iterable[Mapping[str, Any]],
|
||||
) -> bool:
|
||||
failures = [dict(item) for item in partial_query_failures]
|
||||
if not failures:
|
||||
return False
|
||||
for failure in failures:
|
||||
query_name = str(failure.get("query_name") or "")
|
||||
if query_name.removesuffix("_fallback") not in _AUXILIARY_RUNTIME_RECEIPT_QUERY_NAMES:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _runtime_receipt_readback_cache_key(
|
||||
*,
|
||||
project_id: str,
|
||||
@@ -937,8 +956,23 @@ def _build_runtime_receipt_readback_recovery(
|
||||
db_unavailable = db_read_status in {"not_queried", "unavailable"}
|
||||
partial_failures = [dict(item) for item in partial_query_failures]
|
||||
db_partial = db_read_status == "partial"
|
||||
auxiliary_only_partial = (
|
||||
db_partial
|
||||
and _has_only_auxiliary_runtime_receipt_failures(partial_failures)
|
||||
)
|
||||
no_live_log_events = classified_event_total <= 0
|
||||
if db_partial and not no_live_log_events:
|
||||
if auxiliary_only_partial and not no_live_log_events:
|
||||
status = "completed_live_runtime_receipts_observed_with_auxiliary_query_degradation"
|
||||
safe_next_action_id = (
|
||||
"optimize_auxiliary_alert_queries_without_blocking_runtime_truth"
|
||||
)
|
||||
safe_next_action_stage = "auxiliary_runtime_receipt_query_optimization"
|
||||
safe_next_action = (
|
||||
"Keep core apply, verifier, KM, and Telegram runtime receipts visible "
|
||||
"while optimizing slower alert aggregation queries separately."
|
||||
)
|
||||
blocker_fields = []
|
||||
elif db_partial and not no_live_log_events:
|
||||
status = "degraded_runtime_receipt_partial_query_failures"
|
||||
safe_next_action_id = (
|
||||
"repair_partial_runtime_receipt_queries_then_rerun_log_integration_taxonomy"
|
||||
@@ -989,7 +1023,8 @@ def _build_runtime_receipt_readback_recovery(
|
||||
"safe_next_action": safe_next_action,
|
||||
"post_recovery_verifier": (
|
||||
"GET /api/v1/agents/agent-autonomous-runtime-control and require "
|
||||
"runtime_receipt_readback.db_read_status=ok plus active_source_family_count>0"
|
||||
"core live runtime receipts plus active_source_family_count>0; only "
|
||||
"auxiliary alert query degradation may remain nonblocking"
|
||||
),
|
||||
"classified_event_total": classified_event_total,
|
||||
"recent_classified_event_total": recent_classified_event_total,
|
||||
@@ -2116,6 +2151,7 @@ def _build_work_item_progress(
|
||||
ui_productization: Mapping[str, Any],
|
||||
multi_product_taxonomy: Mapping[str, Any],
|
||||
db_read_status: str,
|
||||
partial_query_failures: Iterable[Mapping[str, Any]] = (),
|
||||
) -> dict[str, Any]:
|
||||
"""Build ordered work items that the UI and agent can keep advancing."""
|
||||
|
||||
@@ -2223,9 +2259,15 @@ def _build_work_item_progress(
|
||||
and multi_product_taxonomy.get("status") == "completed"
|
||||
and multi_product_missing == 0
|
||||
)
|
||||
runtime_truth_complete = db_read_status == "ok" and classified_event_total > 0
|
||||
auxiliary_only_partial = (
|
||||
db_read_status == "partial"
|
||||
and _has_only_auxiliary_runtime_receipt_failures(partial_query_failures)
|
||||
)
|
||||
runtime_truth_complete = (
|
||||
db_read_status == "ok" or auxiliary_only_partial
|
||||
) and classified_event_total > 0
|
||||
deployed_readback_complete = (
|
||||
db_read_status == "ok"
|
||||
(db_read_status == "ok" or auxiliary_only_partial)
|
||||
and trace_ledger.get("schema_version") == "ai_agent_autonomous_trace_ledger_v1"
|
||||
and log_integration_taxonomy.get("schema_version") == "ai_agent_log_integration_taxonomy_v1"
|
||||
and classified_event_total > 0
|
||||
@@ -2254,6 +2296,7 @@ def _build_work_item_progress(
|
||||
else "live_log_classified_event_total_zero"
|
||||
),
|
||||
"db_read_status": db_read_status,
|
||||
"auxiliary_query_degradation_only": auxiliary_only_partial,
|
||||
"classified_event_total": classified_event_total,
|
||||
},
|
||||
{
|
||||
@@ -3295,6 +3338,7 @@ def build_runtime_receipt_readback_from_rows(
|
||||
ui_productization=ui_productization,
|
||||
multi_product_taxonomy=multi_product_taxonomy,
|
||||
db_read_status=db_read_status,
|
||||
partial_query_failures=partial_query_failures,
|
||||
)
|
||||
apply_summary = operation_summary.get("ansible_apply_executed") or {}
|
||||
readback = {
|
||||
|
||||
@@ -1375,6 +1375,54 @@ def test_runtime_receipt_recovery_keeps_partial_live_log_events_visible():
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_receipt_auxiliary_alert_query_partial_does_not_block_runtime_truth():
|
||||
readback = build_runtime_receipt_readback_from_rows(
|
||||
project_id="awoooi",
|
||||
db_read_status="partial",
|
||||
operation_count_rows=[
|
||||
{
|
||||
"operation_type": "ansible_apply_executed",
|
||||
"status": "success",
|
||||
"total": 3,
|
||||
"recent": 1,
|
||||
},
|
||||
],
|
||||
verifier_count_rows=[
|
||||
{"status": "success", "total": 3, "recent": 1},
|
||||
],
|
||||
km_count_rows=[
|
||||
{"status": "linked", "total": 3, "recent": 1},
|
||||
],
|
||||
telegram_count_rows=[
|
||||
{"status": "sent", "total": 3, "recent": 1},
|
||||
],
|
||||
partial_query_failures=[
|
||||
{"query_name": "alert_operation_counts", "error_type": "DBAPIError"},
|
||||
{
|
||||
"query_name": "alertmanager_event_counts",
|
||||
"error_type": "RuntimeReceiptQueryBudgetExceeded",
|
||||
},
|
||||
],
|
||||
error_type="partial_query_failures",
|
||||
)
|
||||
|
||||
recovery = readback["runtime_receipt_readback_recovery"]
|
||||
assert recovery["status"] == (
|
||||
"completed_live_runtime_receipts_observed_with_auxiliary_query_degradation"
|
||||
)
|
||||
assert recovery["blocker_fields"] == []
|
||||
|
||||
progress_items = {
|
||||
item["work_item_id"]: item
|
||||
for item in readback["work_item_progress"]["ordered_items"]
|
||||
}
|
||||
assert progress_items["P0-A-runtime-truth"]["status"] == "completed"
|
||||
assert (
|
||||
progress_items["P0-A-runtime-truth"]["auxiliary_query_degradation_only"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_receipt_work_items_use_learning_receipts_without_latest_telegram_gate():
|
||||
apply_op_id = "2f8ef5c8-fd4e-4950-99e9-dc9e61150cab"
|
||||
incident_id = "INC-20260629-LEARNING"
|
||||
|
||||
Reference in New Issue
Block a user