From 60ee187bd982a58ab354e3c570ae3d375d7d5113 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 05:30:42 +0800 Subject: [PATCH] fix(sre): bound Telegram receipt readbacks --- .../src/services/platform_operator_service.py | 6 +- ...gram_alert_monitoring_coverage_readback.py | 256 ++++++++++++++++-- .../test_awooop_operator_timeline_labels.py | 8 +- ..._alert_monitoring_coverage_readback_api.py | 246 ++++++++++++++++- ...rolled-automation-work-items.snapshot.json | 6 +- 5 files changed, 495 insertions(+), 27 deletions(-) diff --git a/apps/api/src/services/platform_operator_service.py b/apps/api/src/services/platform_operator_service.py index 7edb8c3a1..89e1185a2 100644 --- a/apps/api/src/services/platform_operator_service.py +++ b/apps/api/src/services/platform_operator_service.py @@ -1395,6 +1395,7 @@ async def list_ai_alert_card_delivery_readback( page: int = 1, per_page: int = 20, refresh: bool = False, + cache_write: bool = True, ) -> dict[str, Any]: """Read-only AwoooP delivery readback for AI automation alert cards.""" normalized_project_id = project_id or "awoooi" @@ -1810,9 +1811,12 @@ async def list_ai_alert_card_delivery_readback( page=normalized_page, per_page=normalized_per_page, total=summary["total"], - cache_status="miss", + cache_status="miss" if cache_write else "write_disabled_read_only", cache_ttl_seconds=_AI_ALERT_CARD_CACHE_TTL_SECONDS, ) + if not cache_write: + response["summary"]["cache_write_status"] = "disabled_read_only" + return response try: return await store_operator_summary_async( "ai_alert_card_delivery_readback", diff --git a/apps/api/src/services/telegram_alert_monitoring_coverage_readback.py b/apps/api/src/services/telegram_alert_monitoring_coverage_readback.py index a03a6c80b..ca51bc934 100644 --- a/apps/api/src/services/telegram_alert_monitoring_coverage_readback.py +++ b/apps/api/src/services/telegram_alert_monitoring_coverage_readback.py @@ -13,7 +13,7 @@ from __future__ import annotations import asyncio import json -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from pathlib import Path from typing import Any @@ -31,6 +31,7 @@ from src.services.telegram_alert_ai_automation_matrix import ( load_latest_telegram_alert_ai_automation_matrix, ) from src.services.telegram_alert_learning_context_post_apply_verifier import ( + build_telegram_alert_learning_context_post_apply_verifier, load_latest_telegram_alert_learning_context_post_apply_verifier, ) @@ -41,6 +42,17 @@ _RUNTIME_DIRECT_CONNECT_TIMEOUT_SECONDS = 1.5 _RUNTIME_DIRECT_QUERY_TIMEOUT_SECONDS = 2.0 _RUNTIME_DIRECT_CLOSE_TIMEOUT_SECONDS = 0.25 _RUNTIME_DIRECT_STATEMENT_TIMEOUT_MS = 1500 +_PRELUDE_STAGE_TIMEOUT_SECONDS = 0.75 +_PRELUDE_STAGE_COUNT = 2 +_SERIAL_DB_STAGE_TIMEOUT_SECONDS = 2.0 +_SERIAL_DB_STAGE_COUNT = 4 +_SERIAL_DB_TOTAL_BUDGET_SECONDS = ( + _SERIAL_DB_STAGE_TIMEOUT_SECONDS * _SERIAL_DB_STAGE_COUNT +) +_FULL_READBACK_TOTAL_BUDGET_SECONDS = ( + _PRELUDE_STAGE_TIMEOUT_SECONDS * _PRELUDE_STAGE_COUNT + + _SERIAL_DB_TOTAL_BUDGET_SECONDS +) _MONITORING_LIVE_RECEIPT_OPERATION_TYPE = "telegram_monitoring_live_receipt_applied" _MONITORING_LIVE_RECEIPT_EXECUTOR_ROUTE = "ai_agent_monitoring_live_receipt_consumer" _MONITORING_INVENTORY = "monitoring-alerting-observability-inventory.snapshot.json" @@ -316,6 +328,136 @@ _REQUIRED_TAG_DIMENSIONS = ( logger = get_logger(__name__) +async def _load_prelude_stage( + *, + stage: str, + loader: Callable[[], Awaitable[dict[str, Any]]], +) -> dict[str, Any]: + try: + return await asyncio.wait_for( + loader(), + timeout=_PRELUDE_STAGE_TIMEOUT_SECONDS, + ) + except TimeoutError: + logger.warning( + "telegram_alert_monitoring_prelude_stage_timeout", + stage=stage, + timeout_seconds=_PRELUDE_STAGE_TIMEOUT_SECONDS, + total_budget_seconds=_FULL_READBACK_TOTAL_BUDGET_SECONDS, + ) + return _prelude_timeout_payload(stage) + + +def _prelude_timeout_payload(stage: str) -> dict[str, Any]: + if stage == "matrix": + return { + "schema_version": "telegram_alert_ai_automation_matrix_v1", + "status": "telegram_alert_ai_automation_matrix_unavailable", + "summary": { + "telegram_alert_surface_count": 0, + "known_direct_send_gap_count": 0, + "db_or_log_receipt_ready_surface_count": 0, + "ai_route_ready_surface_count": 0, + "controlled_queue_ready_surface_count": 0, + "post_verifier_ready_surface_count": 0, + "learning_writeback_ready_surface_count": 0, + "error_type": "TimeoutError", + }, + } + if stage == "source_contract": + return { + "alertmanager_alert_log_append_present": False, + "alert_operation_log_telegram_sent_present": False, + "alert_operation_log_repository_present": False, + "telegram_gateway_outbound_mirror_present": False, + "telegram_gateway_outbound_record_present": False, + "ai_alert_card_delivery_readback_present": False, + "ai_alert_card_learning_refs_present": False, + "post_apply_verifier_present": False, + "awooop_coverage_surface_present": False, + } + raise ValueError(f"unsupported prelude stage: {stage}") + + +async def _load_serial_db_stage( + *, + stage: str, + loader: Callable[[], Awaitable[dict[str, Any]]], +) -> dict[str, Any]: + try: + return await asyncio.wait_for( + loader(), + timeout=_SERIAL_DB_STAGE_TIMEOUT_SECONDS, + ) + except TimeoutError: + logger.warning( + "telegram_alert_monitoring_serial_db_stage_timeout", + stage=stage, + timeout_seconds=_SERIAL_DB_STAGE_TIMEOUT_SECONDS, + total_budget_seconds=_SERIAL_DB_TOTAL_BUDGET_SECONDS, + ) + return _serial_db_timeout_payload(stage) + + +def _serial_db_timeout_payload(stage: str) -> dict[str, Any]: + if stage == "consumer": + return { + "schema_version": "ai_agent_log_controlled_writeback_consumer_readback_v1", + "status": "blocked_waiting_controlled_writeback_consumer_db_readback", + "rollups": { + "controlled_consumer_readback_ready": False, + "telegram_alert_learning_context_readback_ready": False, + "telegram_alert_learning_context_receipt_count": 0, + "telegram_alert_learning_ai_agent_context_receipt_count": 0, + "error_type": "TimeoutError", + }, + "telegram_alert_learning_context": { + "status": "blocked_telegram_alert_learning_registry_readback_unavailable", + "active_blockers": ["serial_db_stage_timeout"], + }, + } + if stage == "live_receipt": + return { + "status": "unavailable", + "summary": { + "readback_source": "unavailable", + "row_count": 0, + "ready_row_count": 0, + "accepted_surface_count": 0, + "error_type": "TimeoutError", + }, + "rows": [], + } + if stage == "ai_alert_card": + return { + "status": "unavailable", + "summary": { + "total": 0, + "learning_writeback_ready_total": 0, + "db_read_status": "unavailable", + "error_type": "TimeoutError", + }, + } + if stage == "runtime_log": + return { + "status": "unavailable", + "summary": { + "alert_operation_log_total_7d": 0, + "telegram_sent_event_count_7d": 0, + "km_converted_event_count_7d": 0, + "playbook_draft_event_count_7d": 0, + "automation_triggered_run_count_7d": 0, + "automation_closed_run_count_7d": 0, + "automation_open_run_count_7d": 0, + "automation_runtime_closure_percent": 0.0, + "automation_runtime_closure_ready": False, + "error_type": "TimeoutError", + }, + "event_type_counts_7d": {}, + } + raise ValueError(f"unsupported serial DB stage: {stage}") + + async def load_latest_telegram_alert_monitoring_coverage_readback( *, project_id: str = DEFAULT_PROJECT_ID, @@ -342,21 +484,40 @@ async def load_latest_telegram_alert_monitoring_coverage_readback( ) _require_safe_boundaries(monitoring_inventory, monitoring_readback_plan) - ( - matrix, - consumer_readback, - monitoring_live_receipt_apply, - source_contract, - ) = await asyncio.gather( - _load_telegram_matrix_readback(project_id=project_id), - _load_log_controlled_writeback_consumer_readback(project_id=project_id), - _load_monitoring_live_receipt_apply_readback(project_id=project_id), - asyncio.to_thread(_inspect_source_contract, repo_root), + matrix = await _load_prelude_stage( + stage="matrix", + loader=lambda: _load_telegram_matrix_readback(project_id=project_id), ) - verifier, ai_alert_readback, runtime_log_readback = await asyncio.gather( - _load_post_apply_verifier_readback(project_id=project_id), - _load_ai_alert_card_delivery(project_id=project_id), - _load_runtime_log_readback(project_id=project_id), + source_contract = await _load_prelude_stage( + stage="source_contract", + loader=lambda: asyncio.to_thread(_inspect_source_contract, repo_root), + ) + # These readbacks share the constrained API database role in production. + # Keep them serial so one operator read does not consume several role slots + # and turn otherwise healthy metadata into simultaneous timeout fallbacks. + consumer_readback = await _load_serial_db_stage( + stage="consumer", + loader=lambda: _load_log_controlled_writeback_consumer_readback( + project_id=project_id + ), + ) + monitoring_live_receipt_apply = await _load_serial_db_stage( + stage="live_receipt", + loader=lambda: _load_monitoring_live_receipt_apply_readback( + project_id=project_id + ), + ) + verifier = build_telegram_alert_learning_context_post_apply_verifier( + consumer_readback=consumer_readback, + project_id=project_id, + ) + ai_alert_readback = await _load_serial_db_stage( + stage="ai_alert_card", + loader=lambda: _load_ai_alert_card_delivery(project_id=project_id), + ) + runtime_log_readback = await _load_serial_db_stage( + stage="runtime_log", + loader=lambda: _load_runtime_log_readback(project_id=project_id), ) return build_telegram_alert_monitoring_coverage_readback( @@ -1130,6 +1291,7 @@ async def _load_ai_alert_card_delivery(*, project_id: str) -> dict[str, Any]: page=1, per_page=6, refresh=True, + cache_write=False, ), timeout=_LIVE_READBACK_TIMEOUT_SECONDS, ) @@ -2121,28 +2283,82 @@ def _ai_controlled_gap_queue( }, }) continue + execution_contract = _gap_execution_contract(blocker) queue.append({ "work_item_id": f"CIR-P0-TG-001-{index:02d}", "parent_work_item_id": "CIR-P0-TG-001", "priority": "P0" if index <= 4 else "P1", "blocker": blocker, - "status": "queued_ai_controlled_apply", - "owner_review_required_for_low_medium_high": False, - "critical_break_glass_required": True, + **execution_contract, "target_selector": _gap_target_selector(blocker), "controlled_next_action": _gap_next_action(blocker), "post_verifier": ( "/api/v1/agents/telegram-alert-monitoring-coverage-readback" ), "sample_surfaces": gap_samples[:3] if blocker.startswith("monitoring_") else [], + }) + return queue + + +def _gap_execution_contract(blocker: str) -> dict[str, Any]: + if blocker in { + "runtime_alert_operation_log_db_readback_unavailable", + "awooop_ai_alert_card_delivery_db_readback_unavailable", + }: + return { + "status": "ready_ai_read_only_retry", + "risk_level": "low", + "execution_mode": "automatic_bounded_readback_retry", + "executor_route": "telegram_monitoring_readback_service", + "ai_controlled_execution_allowed": True, + "owner_review_required_for_low_medium_high": False, + "critical_break_glass_required": False, "operation_boundaries": { + "database_read_only": True, "requires_secret": False, "requires_runtime_send": False, + "runtime_write_performed": False, "stores_raw_payload": False, "destructive_action": False, }, - }) - return queue + } + if blocker == "runtime_ai_automation_trigger_receipt_missing" or blocker.startswith( + "runtime_ai_automation_lifecycle_open" + ): + return { + "status": "blocked_critical_break_glass_required", + "risk_level": "critical", + "execution_mode": "controlled_runtime_canary_or_lifecycle_resume", + "executor_route": "single_controlled_executor", + "ai_controlled_execution_allowed": False, + "owner_review_required_for_low_medium_high": False, + "critical_break_glass_required": True, + "operation_boundaries": { + "database_read_only": False, + "requires_secret": False, + "requires_runtime_send": True, + "runtime_write_performed": False, + "stores_raw_payload": False, + "destructive_action": False, + }, + } + return { + "status": "queued_ai_controlled_apply", + "risk_level": "high", + "execution_mode": "controlled_gap_closure", + "executor_route": "single_controlled_executor", + "ai_controlled_execution_allowed": False, + "owner_review_required_for_low_medium_high": False, + "critical_break_glass_required": True, + "operation_boundaries": { + "database_read_only": False, + "requires_secret": False, + "requires_runtime_send": False, + "runtime_write_performed": False, + "stores_raw_payload": False, + "destructive_action": False, + }, + } def _coverage_matrix( diff --git a/apps/api/tests/test_awooop_operator_timeline_labels.py b/apps/api/tests/test_awooop_operator_timeline_labels.py index fb5067442..b93b49f8d 100644 --- a/apps/api/tests/test_awooop_operator_timeline_labels.py +++ b/apps/api/tests/test_awooop_operator_timeline_labels.py @@ -1752,8 +1752,8 @@ async def test_unfiltered_query_selects_declared_effective_and_dispatch_receipts async def __aexit__(self, exc_type, exc, tb): return False - async def passthrough_store(_kind, _key, value, **_kwargs): - return value + async def forbidden_store(*_args, **_kwargs): + raise AssertionError("read-only delivery readback must not write cache") monkeypatch.setattr( platform_operator_service, @@ -1763,7 +1763,7 @@ async def test_unfiltered_query_selects_declared_effective_and_dispatch_receipts monkeypatch.setattr( platform_operator_service, "store_operator_summary_async", - passthrough_store, + forbidden_store, ) payload = await platform_operator_service.list_ai_alert_card_delivery_readback( @@ -1773,6 +1773,7 @@ async def test_unfiltered_query_selects_declared_effective_and_dispatch_receipts page=1, per_page=5, refresh=True, + cache_write=False, ) summary_query = next(query for query in queries if "COUNT(*) AS total" in query) @@ -1790,6 +1791,7 @@ async def test_unfiltered_query_selects_declared_effective_and_dispatch_receipts assert "LEFT JOIN agent99_runs ar" in list_query assert "ON ar.message_id = m.message_id" in list_query assert "FROM agent99_runs ars" not in list_query + assert payload["summary"]["cache_write_status"] == "disabled_read_only" assert list_query.index("LIMIT :limit OFFSET :offset") < list_query.index( "candidate_refs AS MATERIALIZED" ) diff --git a/apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py b/apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py index 95eb291ca..d4bfdd6a0 100644 --- a/apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py +++ b/apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py @@ -471,6 +471,62 @@ def test_telegram_alert_monitoring_coverage_blocks_open_runtime_lifecycle(): pipeline = {stage["stage_id"]: stage for stage in payload["alert_receipt_pipeline"]} assert pipeline["runtime_ai_automation_lifecycle_closure"]["ready"] is False assert pipeline["runtime_ai_automation_lifecycle_closure"]["gap_count"] == 7 + lifecycle_candidate = next( + item + for item in payload["ai_controlled_gap_queue"] + if item["blocker"] == "runtime_ai_automation_lifecycle_open:7" + ) + assert lifecycle_candidate["status"] == "blocked_critical_break_glass_required" + assert lifecycle_candidate["risk_level"] == "critical" + assert lifecycle_candidate["critical_break_glass_required"] is True + assert lifecycle_candidate["operation_boundaries"]["requires_runtime_send"] is True + + +def test_telegram_alert_monitoring_gap_queue_separates_read_only_and_runtime_write(): + payload = _payload( + runtime_log_readback={ + "status": "unavailable", + "summary": { + "alert_operation_log_total_7d": 0, + "automation_triggered_run_count_7d": 0, + "automation_closed_run_count_7d": 0, + "automation_open_run_count_7d": 0, + "automation_runtime_closure_percent": 0.0, + "automation_runtime_closure_ready": False, + }, + "event_type_counts_7d": {}, + }, + ai_alert_card_delivery_readback={ + "status": "unavailable", + "summary": { + "total": 0, + "learning_writeback_ready_total": 0, + "db_read_status": "unavailable", + }, + }, + monitoring_live_receipt_apply_readback=_metadata_live_receipt_apply_readback(), + ) + queue = {item["blocker"]: item for item in payload["ai_controlled_gap_queue"]} + + for blocker in ( + "runtime_alert_operation_log_db_readback_unavailable", + "awooop_ai_alert_card_delivery_db_readback_unavailable", + ): + candidate = queue[blocker] + assert candidate["status"] == "ready_ai_read_only_retry" + assert candidate["risk_level"] == "low" + assert candidate["execution_mode"] == "automatic_bounded_readback_retry" + assert candidate["ai_controlled_execution_allowed"] is True + assert candidate["critical_break_glass_required"] is False + assert candidate["operation_boundaries"]["database_read_only"] is True + assert candidate["operation_boundaries"]["requires_runtime_send"] is False + + runtime_canary = queue["runtime_ai_automation_trigger_receipt_missing"] + assert runtime_canary["status"] == "blocked_critical_break_glass_required" + assert runtime_canary["risk_level"] == "critical" + assert runtime_canary["ai_controlled_execution_allowed"] is False + assert runtime_canary["critical_break_glass_required"] is True + assert runtime_canary["operation_boundaries"]["requires_runtime_send"] is True def test_telegram_alert_monitoring_coverage_blocks_partial_surface_contracts(): @@ -668,6 +724,189 @@ def test_telegram_alert_monitoring_coverage_endpoint(monkeypatch): assert data["operation_boundaries"]["bot_api_call_performed"] is False +@pytest.mark.asyncio +async def test_telegram_alert_monitoring_live_loader_serializes_db_readbacks( + monkeypatch: pytest.MonkeyPatch, +): + active_db_reads = 0 + max_active_db_reads = 0 + read_order: list[str] = [] + + async def db_read(name: str, payload: dict) -> dict: + nonlocal active_db_reads, max_active_db_reads + active_db_reads += 1 + max_active_db_reads = max(max_active_db_reads, active_db_reads) + read_order.append(f"{name}:start") + await asyncio.sleep(0) + read_order.append(f"{name}:end") + active_db_reads -= 1 + return payload + + async def matrix_readback(*, project_id: str) -> dict: + assert project_id == "awoooi" + return { + "schema_version": "telegram_alert_ai_automation_matrix_v1", + "status": "telegram_alert_ai_automation_matrix_ready", + "summary": { + "telegram_alert_surface_count": 9, + "known_direct_send_gap_count": 0, + "db_or_log_receipt_ready_surface_count": 9, + "ai_route_ready_surface_count": 9, + "controlled_queue_ready_surface_count": 9, + "post_verifier_ready_surface_count": 9, + "learning_writeback_ready_surface_count": 9, + }, + } + + async def consumer_readback(*, project_id: str) -> dict: + assert project_id == "awoooi" + return await db_read("consumer", _consumer_context_readback()) + + async def live_receipt_readback(*, project_id: str) -> dict: + assert project_id == "awoooi" + return await db_read("live_receipt", _metadata_live_receipt_apply_readback()) + + async def ai_card_readback(*, project_id: str) -> dict: + assert project_id == "awoooi" + return await db_read( + "ai_card", + { + "status": "ok", + "summary": {"total": 6, "learning_writeback_ready_total": 6}, + }, + ) + + async def runtime_log_readback(*, project_id: str) -> dict: + assert project_id == "awoooi" + return await db_read( + "runtime_log", + { + "status": "ok", + "summary": { + "alert_operation_log_total_7d": 4, + "automation_triggered_run_count_7d": 1, + "automation_closed_run_count_7d": 1, + "automation_open_run_count_7d": 0, + "automation_runtime_closure_percent": 100.0, + "automation_runtime_closure_ready": True, + }, + "event_type_counts_7d": {"ALERT_RECEIVED": 1}, + }, + ) + + monkeypatch.setattr( + coverage_service, "_load_telegram_matrix_readback", matrix_readback + ) + monkeypatch.setattr( + coverage_service, + "_load_log_controlled_writeback_consumer_readback", + consumer_readback, + ) + monkeypatch.setattr( + coverage_service, + "_load_monitoring_live_receipt_apply_readback", + live_receipt_readback, + ) + monkeypatch.setattr( + coverage_service, "_load_ai_alert_card_delivery", ai_card_readback + ) + monkeypatch.setattr( + coverage_service, "_load_runtime_log_readback", runtime_log_readback + ) + + payload = ( + await coverage_service.load_latest_telegram_alert_monitoring_coverage_readback() + ) + + assert max_active_db_reads == 1 + assert read_order == [ + "consumer:start", + "consumer:end", + "live_receipt:start", + "live_receipt:end", + "ai_card:start", + "ai_card:end", + "runtime_log:start", + "runtime_log:end", + ] + assert payload["summary"]["runtime_automation_closure_ready"] is True + + +@pytest.mark.asyncio +async def test_telegram_alert_monitoring_serial_db_budget_is_below_ui_deadline( + monkeypatch: pytest.MonkeyPatch, +): + async def slow_stage() -> dict: + await asyncio.sleep(1) + return {"status": "unexpected"} + + monkeypatch.setattr(coverage_service, "_SERIAL_DB_STAGE_TIMEOUT_SECONDS", 0.01) + started_at = asyncio.get_running_loop().time() + results = [ + await coverage_service._load_serial_db_stage( + stage=stage, + loader=slow_stage, + ) + for stage in ("consumer", "live_receipt", "ai_alert_card", "runtime_log") + ] + elapsed_seconds = asyncio.get_running_loop().time() - started_at + + assert coverage_service._SERIAL_DB_TOTAL_BUDGET_SECONDS == 8.0 + assert coverage_service._FULL_READBACK_TOTAL_BUDGET_SECONDS == 9.5 + assert coverage_service._FULL_READBACK_TOTAL_BUDGET_SECONDS < 12.0 + assert elapsed_seconds < 0.2 + assert [result["status"] for result in results] == [ + "blocked_waiting_controlled_writeback_consumer_db_readback", + "unavailable", + "unavailable", + "unavailable", + ] + + +@pytest.mark.asyncio +async def test_telegram_alert_monitoring_full_loader_honors_shared_deadline( + monkeypatch: pytest.MonkeyPatch, +): + async def slow_stage(**_kwargs) -> dict: + await asyncio.sleep(1) + return {"status": "unexpected"} + + monkeypatch.setattr(coverage_service, "_PRELUDE_STAGE_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(coverage_service, "_SERIAL_DB_STAGE_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr( + coverage_service, "_load_telegram_matrix_readback", slow_stage + ) + monkeypatch.setattr( + coverage_service, + "_load_log_controlled_writeback_consumer_readback", + slow_stage, + ) + monkeypatch.setattr( + coverage_service, + "_load_monitoring_live_receipt_apply_readback", + slow_stage, + ) + monkeypatch.setattr( + coverage_service, "_load_ai_alert_card_delivery", slow_stage + ) + monkeypatch.setattr(coverage_service, "_load_runtime_log_readback", slow_stage) + monkeypatch.setattr( + coverage_service, "_inspect_source_contract", lambda _repo_root: _source_contract() + ) + + started_at = asyncio.get_running_loop().time() + payload = ( + await coverage_service.load_latest_telegram_alert_monitoring_coverage_readback() + ) + elapsed_seconds = asyncio.get_running_loop().time() - started_at + + assert elapsed_seconds < 0.2 + assert payload["status"] == ( + "blocked_telegram_alert_monitoring_coverage_gaps_present" + ) + assert payload["summary"]["runtime_automation_closure_ready"] is False + + @pytest.mark.asyncio async def test_telegram_alert_monitoring_live_readbacks_are_bounded( monkeypatch: pytest.MonkeyPatch, @@ -855,7 +1094,10 @@ def test_runtime_lifecycle_sql_accepts_only_durable_failed_terminals() -> None: async def test_telegram_alert_monitoring_marks_ai_alert_cards_source_unavailable( monkeypatch: pytest.MonkeyPatch, ): - async def source_unavailable_ai_alert_cards(**_kwargs): + call_kwargs: dict = {} + + async def source_unavailable_ai_alert_cards(**kwargs): + call_kwargs.update(kwargs) return ( platform_operator_service ._ai_alert_card_delivery_source_unavailable_response( @@ -877,6 +1119,8 @@ async def test_telegram_alert_monitoring_marks_ai_alert_cards_source_unavailable ai_cards = await coverage_service._load_ai_alert_card_delivery(project_id="awoooi") assert ai_cards["status"] == "unavailable" + assert call_kwargs["refresh"] is True + assert call_kwargs["cache_write"] is False assert ai_cards["summary"]["db_read_status"] == "unavailable" assert ai_cards["learning_registry"]["status"] == ( "source_unavailable_ai_controlled_repair_required" diff --git a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json index e1f3f38a0..2328241b2 100644 --- a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json +++ b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json @@ -902,12 +902,14 @@ "unknown product, signal, severity or requested destination fails closed and active source contains no raw numeric Telegram destination default", "Telegram automation cards distinguish durable AI/provider, Agent, executor, verifier and learning receipts from configured-but-unobserved components", "Agent99 same-run closure is two-phase: an ApprovalRecord fingerprint/hit-count/last-seen recurrence fence must still match, the incident resolution commits first, the recovered Telegram card must receive a durable destination-bound provider acknowledgement, and only then may the run claim runtime_closure_verified", - "information and controlled-action buttons are bound to registered callback actions instead of display-only ghost buttons" + "information and controlled-action buttons are bound to registered callback actions instead of display-only ghost buttons", + "Telegram monitoring coverage source now limits its complete readback pipeline to a 9.5-second fail-closed budget, serializes four database stages to one connection at a time, disables cache writes for the AI-card coverage read, classifies bounded DB retries as low-risk read-only, and preserves break-glass for runtime canary or lifecycle resume" ], "runtime_gaps": [ "the host99 Agent99 independent pull/reduced relay has no production deployment, freshness, dedupe or controlled-repair receipt", "production Alertmanager exposes integration-only notification counters; the bounded check now passes after adding a scrape-time receiver_contract label, but apply and alert resolution remain pending", - "the recipient-visible Agent99 recovered-card closure contract is source-tested only; no production Telegram provider acknowledgement or same-run runtime closure receipt exists yet" + "the recipient-visible Agent99 recovered-card closure contract is source-tested only; no production Telegram provider acknowledgement or same-run runtime closure receipt exists yet", + "production runtime e614f061f7 still reports both alert-operation-log and AI-card-delivery DB readbacks unavailable with zero 7-day automation lifecycle receipts; the bounded serial readback fix is source-only until a successful deploy marker" ], "next_action": "deploy one exact source revision through the authorized release lane, then verify receiver-contract freshness, host99 Agent99 exact-host polling, dedupe, callback execution, current firing alert resolution and recipient-visible same-run delivery receipts; source tests must not be counted as runtime closure" },