fix(agents): keep auxiliary receipt lag nonblocking
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m7s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled

This commit is contained in:
ogt
2026-07-09 19:09:18 +08:00
parent 9d24b945ab
commit 2983f755b2
2 changed files with 96 additions and 4 deletions

View File

@@ -54,6 +54,12 @@ _RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS = 0.45
_RUNTIME_RECEIPT_STATEMENT_TIMEOUT_MS = 400
_LOG_CONTROLLED_WRITEBACK_CONSUMER_TIMEOUT_SECONDS = 0.5
_RUNTIME_RECEIPT_READBACK_CACHE_TTL_SECONDS = 20.0
_AUXILIARY_RUNTIME_RECEIPT_QUERY_NAMES = {
"alert_operation_counts",
"alertmanager_event_counts",
"grouped_alert_event_counts",
"db_context_exit",
}
# CD cancel-stale-cd no-op triggers must not change runtime payloads.
_EXECUTOR_OPERATION_TYPES = (
"ansible_candidate_matched",
@@ -142,6 +148,19 @@ def _int_value(value: Any) -> int:
return 0
def _has_only_auxiliary_runtime_receipt_failures(
partial_query_failures: Iterable[Mapping[str, Any]],
) -> bool:
failures = [dict(item) for item in partial_query_failures]
if not failures:
return False
for failure in failures:
query_name = str(failure.get("query_name") or "")
if query_name.removesuffix("_fallback") not in _AUXILIARY_RUNTIME_RECEIPT_QUERY_NAMES:
return False
return True
def _runtime_receipt_readback_cache_key(
*,
project_id: str,
@@ -937,8 +956,23 @@ def _build_runtime_receipt_readback_recovery(
db_unavailable = db_read_status in {"not_queried", "unavailable"}
partial_failures = [dict(item) for item in partial_query_failures]
db_partial = db_read_status == "partial"
auxiliary_only_partial = (
db_partial
and _has_only_auxiliary_runtime_receipt_failures(partial_failures)
)
no_live_log_events = classified_event_total <= 0
if db_partial and not no_live_log_events:
if auxiliary_only_partial and not no_live_log_events:
status = "completed_live_runtime_receipts_observed_with_auxiliary_query_degradation"
safe_next_action_id = (
"optimize_auxiliary_alert_queries_without_blocking_runtime_truth"
)
safe_next_action_stage = "auxiliary_runtime_receipt_query_optimization"
safe_next_action = (
"Keep core apply, verifier, KM, and Telegram runtime receipts visible "
"while optimizing slower alert aggregation queries separately."
)
blocker_fields = []
elif db_partial and not no_live_log_events:
status = "degraded_runtime_receipt_partial_query_failures"
safe_next_action_id = (
"repair_partial_runtime_receipt_queries_then_rerun_log_integration_taxonomy"
@@ -989,7 +1023,8 @@ def _build_runtime_receipt_readback_recovery(
"safe_next_action": safe_next_action,
"post_recovery_verifier": (
"GET /api/v1/agents/agent-autonomous-runtime-control and require "
"runtime_receipt_readback.db_read_status=ok plus active_source_family_count>0"
"core live runtime receipts plus active_source_family_count>0; only "
"auxiliary alert query degradation may remain nonblocking"
),
"classified_event_total": classified_event_total,
"recent_classified_event_total": recent_classified_event_total,
@@ -2116,6 +2151,7 @@ def _build_work_item_progress(
ui_productization: Mapping[str, Any],
multi_product_taxonomy: Mapping[str, Any],
db_read_status: str,
partial_query_failures: Iterable[Mapping[str, Any]] = (),
) -> dict[str, Any]:
"""Build ordered work items that the UI and agent can keep advancing."""
@@ -2223,9 +2259,15 @@ def _build_work_item_progress(
and multi_product_taxonomy.get("status") == "completed"
and multi_product_missing == 0
)
runtime_truth_complete = db_read_status == "ok" and classified_event_total > 0
auxiliary_only_partial = (
db_read_status == "partial"
and _has_only_auxiliary_runtime_receipt_failures(partial_query_failures)
)
runtime_truth_complete = (
db_read_status == "ok" or auxiliary_only_partial
) and classified_event_total > 0
deployed_readback_complete = (
db_read_status == "ok"
(db_read_status == "ok" or auxiliary_only_partial)
and trace_ledger.get("schema_version") == "ai_agent_autonomous_trace_ledger_v1"
and log_integration_taxonomy.get("schema_version") == "ai_agent_log_integration_taxonomy_v1"
and classified_event_total > 0
@@ -2254,6 +2296,7 @@ def _build_work_item_progress(
else "live_log_classified_event_total_zero"
),
"db_read_status": db_read_status,
"auxiliary_query_degradation_only": auxiliary_only_partial,
"classified_event_total": classified_event_total,
},
{
@@ -3295,6 +3338,7 @@ def build_runtime_receipt_readback_from_rows(
ui_productization=ui_productization,
multi_product_taxonomy=multi_product_taxonomy,
db_read_status=db_read_status,
partial_query_failures=partial_query_failures,
)
apply_summary = operation_summary.get("ansible_apply_executed") or {}
readback = {

View File

@@ -1375,6 +1375,54 @@ def test_runtime_receipt_recovery_keeps_partial_live_log_events_visible():
)
def test_runtime_receipt_auxiliary_alert_query_partial_does_not_block_runtime_truth():
readback = build_runtime_receipt_readback_from_rows(
project_id="awoooi",
db_read_status="partial",
operation_count_rows=[
{
"operation_type": "ansible_apply_executed",
"status": "success",
"total": 3,
"recent": 1,
},
],
verifier_count_rows=[
{"status": "success", "total": 3, "recent": 1},
],
km_count_rows=[
{"status": "linked", "total": 3, "recent": 1},
],
telegram_count_rows=[
{"status": "sent", "total": 3, "recent": 1},
],
partial_query_failures=[
{"query_name": "alert_operation_counts", "error_type": "DBAPIError"},
{
"query_name": "alertmanager_event_counts",
"error_type": "RuntimeReceiptQueryBudgetExceeded",
},
],
error_type="partial_query_failures",
)
recovery = readback["runtime_receipt_readback_recovery"]
assert recovery["status"] == (
"completed_live_runtime_receipts_observed_with_auxiliary_query_degradation"
)
assert recovery["blocker_fields"] == []
progress_items = {
item["work_item_id"]: item
for item in readback["work_item_progress"]["ordered_items"]
}
assert progress_items["P0-A-runtime-truth"]["status"] == "completed"
assert (
progress_items["P0-A-runtime-truth"]["auxiliary_query_degradation_only"]
is True
)
def test_runtime_receipt_work_items_use_learning_receipts_without_latest_telegram_gate():
apply_op_id = "2f8ef5c8-fd4e-4950-99e9-dc9e61150cab"
incident_id = "INC-20260629-LEARNING"