From aba8ff474e20395f9cfc07f12eff321c360622f9 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 15:36:37 +0800 Subject: [PATCH] fix(agent): prioritize live closure readback --- apps/api/src/api/v1/agents.py | 18 ++++++++++--- .../ai_agent_autonomous_runtime_control.py | 26 ++++++++++++------- ...est_ai_agent_autonomous_runtime_control.py | 17 ++++++++++++ ...awoooi_priority_work_order_readback_api.py | 13 ++++++++++ 4 files changed, 61 insertions(+), 13 deletions(-) diff --git a/apps/api/src/api/v1/agents.py b/apps/api/src/api/v1/agents.py index 4e9820634..5eda603d9 100644 --- a/apps/api/src/api/v1/agents.py +++ b/apps/api/src/api/v1/agents.py @@ -522,7 +522,7 @@ from src.services.windows99_vmx_source_repair_package import ( router = APIRouter(prefix="/agents", tags=["Agent Teams"]) logger = get_logger("awoooi.agents") -_PRIORITY_WORK_ORDER_LIVE_READBACK_TIMEOUT_SECONDS = 7.0 +_PRIORITY_WORK_ORDER_LIVE_READBACK_TIMEOUT_SECONDS = 12.0 # ============================================================================= @@ -1223,11 +1223,15 @@ async def get_awoooi_priority_work_order_readback() -> dict[str, Any]: async def _build_awoooi_priority_work_order_readback_with_live_overlays() -> dict[str, Any]: + # The AI runtime closure is the authoritative program-order input. Resolve + # it before lower-priority overlays compete for the bounded DB pool. + autonomous_runtime_control = ( + await build_ai_agent_autonomous_runtime_control_with_live_readback() + ) ( gitea_bundle_readback, stockplatform_runtime, reboot_slo, - autonomous_runtime_control, harbor_recovery, log_executor, controlled_cd_lane_metrics, @@ -1238,7 +1242,6 @@ async def _build_awoooi_priority_work_order_readback_with_live_overlays() -> dic asyncio.to_thread(load_latest_gitea_repo_bundle_backup_readback), asyncio.to_thread(load_latest_stockplatform_public_api_runtime_readback), asyncio.to_thread(load_latest_reboot_auto_recovery_slo_scorecard), - build_ai_agent_autonomous_runtime_control_with_live_readback(), asyncio.to_thread(load_latest_harbor_registry_controlled_recovery_preflight), asyncio.to_thread(load_latest_ai_agent_log_controlled_writeback_executor_readback), asyncio.to_thread(load_controlled_cd_lane_live_metric_readback), @@ -1280,6 +1283,15 @@ async def _build_awoooi_priority_work_order_readback_with_live_overlays() -> dic telegram_alert_context_verifier_readback=telegram_alert_context_verifier, autonomous_runtime_control=autonomous_runtime_control, ) + summary = payload.setdefault("summary", {}) + rollups = payload.setdefault("rollups", {}) + mainline = payload.setdefault("mainline_execution_state", {}) + summary["priority_work_order_live_overlay_status"] = "ready" + summary["priority_work_order_live_overlay_timeout_seconds"] = ( + _PRIORITY_WORK_ORDER_LIVE_READBACK_TIMEOUT_SECONDS + ) + rollups["priority_work_order_live_overlay_degraded"] = False + mainline["priority_work_order_live_overlay_degraded"] = False return payload diff --git a/apps/api/src/services/ai_agent_autonomous_runtime_control.py b/apps/api/src/services/ai_agent_autonomous_runtime_control.py index 14d73509a..7d07dce23 100644 --- a/apps/api/src/services/ai_agent_autonomous_runtime_control.py +++ b/apps/api/src/services/ai_agent_autonomous_runtime_control.py @@ -60,9 +60,9 @@ _LIVE_RUNTIME_RECEIPT_TIMEOUT_SECONDS = 8.0 _RUNTIME_RECEIPT_LOCK_TIMEOUT_SECONDS = 0.25 _RUNTIME_RECEIPT_DB_CONTEXT_TIMEOUT_SECONDS = 1.5 _RUNTIME_RECEIPT_DB_CONTEXT_EXIT_TIMEOUT_SECONDS = 0.25 -_RUNTIME_RECEIPT_QUERY_BUDGET_SECONDS = 2.0 -_RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS = 0.45 -_RUNTIME_RECEIPT_STATEMENT_TIMEOUT_MS = 400 +_RUNTIME_RECEIPT_QUERY_BUDGET_SECONDS = 3.0 +_RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS = 0.75 +_RUNTIME_RECEIPT_STATEMENT_TIMEOUT_MS = 700 _LOG_CONTROLLED_WRITEBACK_CONSUMER_TIMEOUT_SECONDS = 4.0 _LOG_CONTROLLED_WRITEBACK_CONSUMER_READBACK_CACHE_TTL_SECONDS = 30.0 _RUNTIME_RECEIPT_READBACK_CACHE_TTL_SECONDS = 20.0 @@ -6600,18 +6600,14 @@ async def _load_ai_agent_autonomous_runtime_receipt_readback_uncached( ) return [] - operation_counts = await _safe_rows( - "operation_counts", - _RUNTIME_OPERATION_COUNTS_SQL, + retry_terminal_latest = await _safe_rows( + "retry_terminal_latest", + _RUNTIME_RETRY_TERMINAL_LATEST_SQL, ) operation_latest = await _safe_rows( "operation_latest", _RUNTIME_OPERATION_LATEST_SQL, ) - retry_terminal_latest = await _safe_rows( - "retry_terminal_latest", - _RUNTIME_RETRY_TERMINAL_LATEST_SQL, - ) for chain_pass in range(2): missing_chain_refs = _missing_runtime_operation_chain_ref_ids( operation_latest @@ -6636,6 +6632,10 @@ async def _load_ai_agent_autonomous_runtime_receipt_readback_uncached( operation_latest, operation_chain_rows, ) + operation_counts = await _safe_rows( + "operation_counts", + _RUNTIME_OPERATION_COUNTS_SQL, + ) auto_repair_counts = await _safe_rows( "auto_repair_counts", _RUNTIME_AUTO_REPAIR_COUNTS_SQL, @@ -7401,6 +7401,9 @@ _RUNTIME_OPERATION_LATEST_SQL = """ FROM automation_operation_log WHERE operation_type = 'ansible_apply_executed' AND status IN ('success', 'failed') + AND created_at >= NOW() - ( + :lookback_hours * INTERVAL '1 hour' + ) ORDER BY created_at DESC LIMIT 1 ) @@ -7479,6 +7482,9 @@ _RUNTIME_OPERATION_LATEST_SQL = """ 'ansible_executor_capability_expired', 'log_controlled_writeback_dispatched' ) + AND operation_row.created_at >= NOW() - ( + :lookback_hours * INTERVAL '1 hour' + ) ORDER BY CASE WHEN operation_row.op_id::text IN ( diff --git a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py index a184cc45a..abc631753 100644 --- a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py +++ b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py @@ -1,4 +1,5 @@ import asyncio +import inspect import pytest @@ -3504,6 +3505,22 @@ def test_retry_rollback_terminal_fails_closed_on_run_mismatch_or_missing_proof() assert "stderr_feed_back" not in retry_sql +def test_runtime_receipt_queries_prioritize_closure_and_bound_latest_window(): + source = inspect.getsource( + runtime_control_module._load_ai_agent_autonomous_runtime_receipt_readback_uncached + ) + + assert source.index('"retry_terminal_latest"') < source.index( + '"operation_latest"' + ) + assert source.index('"operation_latest"') < source.index( + '"operation_counts"' + ) + assert ":lookback_hours * INTERVAL '1 hour'" in ( + runtime_control_module._RUNTIME_OPERATION_LATEST_SQL + ) + + def test_retry_rollback_terminal_rejects_unbounded_retry_or_unverified_rollback(): row = _closed_retry_terminal_row() row["max_retry_attempts"] = "99" diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index cc90e5ec0..e9e131303 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -2,6 +2,7 @@ from __future__ import annotations # ruff: noqa: E402 import asyncio +import inspect import json import os import urllib.error @@ -2731,6 +2732,18 @@ def test_awoooi_priority_work_order_readback_endpoint_returns_static_fallback_on assert "priority_work_order_live_overlay_timeout" in data["active_blockers"] +def test_priority_live_overlay_resolves_ai_runtime_before_secondary_sources() -> None: + source = inspect.getsource( + agents._build_awoooi_priority_work_order_readback_with_live_overlays + ) + + runtime_call = ( + "await build_ai_agent_autonomous_runtime_control_with_live_readback()" + ) + assert source.index(runtime_call) < source.index("await asyncio.gather(") + assert 'priority_work_order_live_overlay_status"] = "ready"' in source + + def test_awoooi_priority_work_order_readback_endpoint_redacts_ai_loop_queue( monkeypatch: pytest.MonkeyPatch, tmp_path: Path,