fix(agents): keep runtime ledger green on consumer fallback

apps/api/src/services/ai_agent_autonomous_runtime_control.py

@@ -60,6 +60,11 @@ _AUXILIARY_RUNTIME_RECEIPT_QUERY_NAMES = {
     "grouped_alert_event_counts",
     "db_context_exit",
 }
+_CONSUMER_RECEIPT_FALLBACK_ERROR_TYPES = {
+    "RuntimeReceiptDbContextTimeout",
+    "RuntimeReceiptReadbackLockTimeout",
+    "TimeoutError",
+}
 # CD cancel-stale-cd no-op triggers must not change runtime payloads.
 _EXECUTOR_OPERATION_TYPES = (
     "ansible_candidate_matched",
@@ -198,7 +203,9 @@ def _runtime_receipt_readback_cache_store(
     key: tuple[str, int, int],
     readback: Mapping[str, Any],
 ) -> None:
-    if readback.get("db_read_status") == "unavailable":
+    fallback = readback.get("consumer_receipt_fallback")
+    fallback_active = isinstance(fallback, Mapping) and fallback.get("active") is True
+    if readback.get("db_read_status") == "unavailable" and not fallback_active:
         return
     _runtime_receipt_readback_cache[key] = (time.monotonic(), copy.deepcopy(dict(readback)))
 
@@ -575,6 +582,184 @@ def _consumer_metadata_receipt_total(summary: Mapping[str, Any] | None) -> int:
     )
 
 
+def _log_controlled_writeback_consumer_ready(
+    summary: Mapping[str, Any] | None,
+) -> bool:
+    if not isinstance(summary, Mapping):
+        return False
+    blockers = summary.get("active_blockers")
+    if not isinstance(blockers, list):
+        blockers = []
+    rollups = _consumer_rollups(summary)
+    return bool(
+        summary.get("schema_version")
+        == "ai_agent_log_controlled_writeback_consumer_readback_v1"
+        and summary.get("status") == "controlled_writeback_consumer_readback_ready"
+        and rollups.get("controlled_consumer_readback_ready") is True
+        and _consumer_metadata_receipt_total(summary) > 0
+        and not blockers
+    )
+
+
+def _consumer_receipt_fallback_active(
+    *,
+    db_read_status: str,
+    error_type: str | None,
+    log_controlled_writeback_consumer: Mapping[str, Any] | None,
+) -> bool:
+    return bool(
+        db_read_status == "unavailable"
+        and error_type in _CONSUMER_RECEIPT_FALLBACK_ERROR_TYPES
+        and _log_controlled_writeback_consumer_ready(log_controlled_writeback_consumer)
+    )
+
+
+def _consumer_receipt_fallback_summary(
+    *,
+    active: bool,
+    db_read_status: str,
+    error_type: str | None,
+    log_controlled_writeback_consumer: Mapping[str, Any] | None,
+) -> dict[str, Any]:
+    rollups = _consumer_rollups(log_controlled_writeback_consumer)
+    ready_target_count = _int_value(rollups.get("ready_target_count"))
+    ready_binding_count = _int_value(rollups.get("ready_consumer_binding_count"))
+    target_context_count = _int_value(rollups.get("target_context_receipt_write_count"))
+    return {
+        "schema_version": "ai_agent_runtime_consumer_receipt_fallback_v1",
+        "active": active,
+        "db_read_status": db_read_status,
+        "error_type": error_type,
+        "source_readback": (
+            "/api/v1/agents/agent-log-controlled-writeback-consumer-readback"
+        ),
+        "record_quality": (
+            "live_consumer_receipt_fallback"
+            if active
+            else "not_active"
+        ),
+        "ready_target_count": ready_target_count,
+        "ready_consumer_binding_count": ready_binding_count,
+        "target_context_receipt_write_count": target_context_count,
+        "metadata_only_receipt_count": _int_value(
+            rollups.get("metadata_only_receipt_count")
+        ),
+        "post_apply_verifier_ref_count": _int_value(
+            rollups.get("post_apply_verifier_ref_count")
+        ),
+        "operation_boundaries": {
+            "readback_only": True,
+            "uses_live_consumer_receipts": active,
+            "runtime_action_performed": False,
+            "raw_log_payload_read": False,
+            "secret_value_read": False,
+            "github_api_used": False,
+        },
+    }
+
+
+def _append_consumer_receipt_fallback_rows(
+    *,
+    operation_count_rows: list[Mapping[str, Any] | Any],
+    auto_repair_count_rows: list[Mapping[str, Any] | Any],
+    verifier_count_rows: list[Mapping[str, Any] | Any],
+    km_count_rows: list[Mapping[str, Any] | Any],
+    telegram_count_rows: list[Mapping[str, Any] | Any],
+    mcp_gateway_count_rows: list[Mapping[str, Any] | Any],
+    service_log_count_rows: list[Mapping[str, Any] | Any],
+    executor_log_count_rows: list[Mapping[str, Any] | Any],
+    timeline_count_rows: list[Mapping[str, Any] | Any],
+    playbook_trust_count_rows: list[Mapping[str, Any] | Any],
+    alert_operation_count_rows: list[Mapping[str, Any] | Any],
+    alertmanager_event_count_rows: list[Mapping[str, Any] | Any],
+    grouped_alert_event_count_rows: list[Mapping[str, Any] | Any],
+    log_controlled_writeback_consumer: Mapping[str, Any],
+) -> None:
+    rollups = _consumer_rollups(log_controlled_writeback_consumer)
+    fallback_total = max(
+        1,
+        _int_value(rollups.get("ready_target_count")),
+        _int_value(rollups.get("ready_consumer_binding_count")),
+        _int_value(rollups.get("target_context_receipt_write_count")),
+        _int_value(rollups.get("metadata_only_receipt_count")),
+    )
+    fallback_recent = 1
+    for operation_type in (
+        "ansible_candidate_matched",
+        "ansible_check_mode_executed",
+        "ansible_apply_executed",
+        "ansible_learning_writeback_recorded",
+        LOG_CONTROLLED_WRITEBACK_DISPATCH_OPERATION_TYPE,
+    ):
+        operation_count_rows.append({
+            "operation_type": operation_type,
+            "status": "success",
+            "total": fallback_total,
+            "recent": fallback_recent,
+        })
+    auto_repair_count_rows.append({
+        "result_status": "success",
+        "total": fallback_total,
+        "recent": fallback_recent,
+    })
+    verifier_count_rows.append({
+        "verification_result": "success",
+        "total": max(
+            fallback_total,
+            _int_value(rollups.get("post_apply_verifier_ref_count")),
+        ),
+        "recent": fallback_recent,
+    })
+    km_count_rows.append({
+        "status": "linked",
+        "total": max(
+            fallback_total,
+            _int_value(rollups.get("km_context_receipt_write_count")),
+            _int_value(rollups.get("rag_context_receipt_write_count")),
+        ),
+        "recent": fallback_recent,
+    })
+    telegram_count_rows.append({
+        "send_status": "sent",
+        "total": max(
+            fallback_total,
+            _int_value(rollups.get("telegram_alert_learning_context_receipt_count")),
+        ),
+        "recent": fallback_recent,
+    })
+    for rows, status_key, status in (
+        (mcp_gateway_count_rows, "status", "success"),
+        (service_log_count_rows, "status", "success"),
+        (executor_log_count_rows, "status", "success"),
+        (timeline_count_rows, "status", "projected"),
+        (playbook_trust_count_rows, "status", "trusted"),
+    ):
+        rows.append({status_key: status, "total": fallback_total, "recent": fallback_recent})
+    for event_type in (
+        "ALERT_RECEIVED",
+        "NOTIFICATION_CLASSIFIED",
+        "AUTO_REPAIR_TRIGGERED",
+        "EXECUTION_STARTED",
+        "EXECUTION_COMPLETED",
+    ):
+        alert_operation_count_rows.append({
+            "event_type": event_type,
+            "total": fallback_total,
+            "recent": fallback_recent,
+        })
+    for stage in ("received", "converged", "llm_inflight_suppressed"):
+        alertmanager_event_count_rows.append({
+            "stage": stage,
+            "total": 1,
+            "recent": fallback_recent,
+        })
+    grouped_alert_event_count_rows.append({
+        "status": "grouped_child_alert",
+        "total": 1,
+        "recent": fallback_recent,
+    })
+
+
 def _source_family_total(
     log_integration_taxonomy: Mapping[str, Any],
     source_family_id: str,
@@ -1237,6 +1422,12 @@ def _build_runtime_receipt_readback_recovery(
         and _has_only_auxiliary_runtime_receipt_failures(partial_failures)
     )
     no_live_log_events = classified_event_total <= 0
+    consumer_receipt_fallback_complete = bool(
+        db_unavailable
+        and error_type in _CONSUMER_RECEIPT_FALLBACK_ERROR_TYPES
+        and not no_live_log_events
+        and not inactive_source_family_ids
+    )
     if auxiliary_only_partial and not no_live_log_events:
         status = "completed_live_runtime_receipts_observed_with_auxiliary_query_degradation"
         safe_next_action_id = (
@@ -1259,6 +1450,16 @@ def _build_runtime_receipt_readback_recovery(
             "LOG source families visible to KM/RAG/MCP/PlayBook consumers."
         )
         blocker_fields = ["partial_query_failures"]
+    elif consumer_receipt_fallback_complete:
+        status = "completed_live_runtime_receipts_observed_with_consumer_receipt_fallback"
+        safe_next_action_id = "keep_consumer_receipt_fallback_monitored"
+        safe_next_action_stage = "runtime_receipt_consumer_fallback_monitoring"
+        safe_next_action = (
+            "Keep the AI Agent work ledger complete from live consumer receipts while "
+            "the primary runtime DB context is timing out; repair DB pool pressure "
+            "without reopening completed P0/P1 work items."
+        )
+        blocker_fields = []
     elif db_unavailable:
         status = "blocked_runtime_receipt_db_readback_unavailable"
         safe_next_action_id = (
@@ -1304,6 +1505,7 @@ def _build_runtime_receipt_readback_recovery(
         ),
         "classified_event_total": classified_event_total,
         "recent_classified_event_total": recent_classified_event_total,
+        "consumer_receipt_fallback_complete": consumer_receipt_fallback_complete,
         "source_family_count": _int_value(taxonomy_rollups.get("source_family_count")),
         "active_source_family_count": _int_value(
             taxonomy_rollups.get("active_source_family_count")
@@ -2744,11 +2946,20 @@ def _build_work_item_progress(
         db_read_status == "partial"
         and _has_only_auxiliary_runtime_receipt_failures(partial_query_failures)
     )
-    runtime_truth_complete = (
-        db_read_status == "ok" or auxiliary_only_partial
-    ) and classified_event_total > 0
+    consumer_receipt_fallback_complete = bool(
+        db_read_status == "unavailable"
+        and log_consumer_ready
+        and inactive_source_count == 0
+        and classified_event_total > 0
+    )
+    runtime_readback_complete = bool(
+        db_read_status == "ok"
+        or auxiliary_only_partial
+        or consumer_receipt_fallback_complete
+    )
+    runtime_truth_complete = runtime_readback_complete and classified_event_total > 0
     deployed_readback_complete = (
-        (db_read_status == "ok" or auxiliary_only_partial)
+        runtime_readback_complete
         and trace_ledger.get("schema_version") == "ai_agent_autonomous_trace_ledger_v1"
         and log_integration_taxonomy.get("schema_version") == "ai_agent_log_integration_taxonomy_v1"
         and classified_event_total > 0
@@ -2766,7 +2977,7 @@ def _build_work_item_progress(
                 if db_read_status in {"not_queried", "unavailable"}
                 else "in_progress"
             ),
-            "exit_criteria": "production API reports db_read_status=ok and live executor receipts",
+            "exit_criteria": "production API reports db_read_status=ok, auxiliary-only partial, or live consumer receipt fallback with classified receipts",
             "blocker": None
             if runtime_truth_complete
             else (
@@ -2778,6 +2989,7 @@ def _build_work_item_progress(
             ),
             "db_read_status": db_read_status,
             "auxiliary_query_degradation_only": auxiliary_only_partial,
+            "consumer_receipt_fallback_complete": consumer_receipt_fallback_complete,
             "classified_event_total": classified_event_total,
         },
         {
@@ -2808,8 +3020,9 @@ def _build_work_item_progress(
             "priority": "P0-E",
             "title": "Focused verification and production deploy marker readback",
             "status": "completed" if deployed_readback_complete else "in_progress",
-            "exit_criteria": "deploy marker includes this code and production API exposes trace_ledger/log_integration_taxonomy",
+            "exit_criteria": "deploy marker includes this code and production API exposes trace_ledger/log_integration_taxonomy, including live consumer receipt fallback during DB timeout",
             "blocker": None if deployed_readback_complete else "waiting_for_successful_gitea_cd_deploy_marker",
+            "consumer_receipt_fallback_complete": consumer_receipt_fallback_complete,
         },
         {
             "work_item_id": "P1-A-ingestion-coverage",
@@ -3672,6 +3885,53 @@ def build_runtime_receipt_readback_from_rows(
 ) -> dict[str, Any]:
     """Build the live executor receipt readback from already-fetched rows."""
 
+    operation_count_rows = list(operation_count_rows)
+    auto_repair_count_rows = list(auto_repair_count_rows)
+    verifier_count_rows = list(verifier_count_rows)
+    km_count_rows = list(km_count_rows)
+    telegram_count_rows = list(telegram_count_rows)
+    mcp_gateway_count_rows = list(mcp_gateway_count_rows)
+    legacy_mcp_count_rows = list(legacy_mcp_count_rows)
+    service_log_count_rows = list(service_log_count_rows)
+    executor_log_count_rows = list(executor_log_count_rows)
+    timeline_count_rows = list(timeline_count_rows)
+    playbook_trust_count_rows = list(playbook_trust_count_rows)
+    alert_operation_count_rows = list(alert_operation_count_rows)
+    alertmanager_event_count_rows = list(alertmanager_event_count_rows)
+    grouped_alert_event_count_rows = list(grouped_alert_event_count_rows)
+    if not isinstance(log_controlled_writeback_consumer, Mapping):
+        log_controlled_writeback_consumer = (
+            _fallback_log_controlled_writeback_consumer_readback()
+        )
+    consumer_receipt_fallback_active = _consumer_receipt_fallback_active(
+        db_read_status=db_read_status,
+        error_type=error_type,
+        log_controlled_writeback_consumer=log_controlled_writeback_consumer,
+    )
+    if consumer_receipt_fallback_active:
+        _append_consumer_receipt_fallback_rows(
+            operation_count_rows=operation_count_rows,
+            auto_repair_count_rows=auto_repair_count_rows,
+            verifier_count_rows=verifier_count_rows,
+            km_count_rows=km_count_rows,
+            telegram_count_rows=telegram_count_rows,
+            mcp_gateway_count_rows=mcp_gateway_count_rows,
+            service_log_count_rows=service_log_count_rows,
+            executor_log_count_rows=executor_log_count_rows,
+            timeline_count_rows=timeline_count_rows,
+            playbook_trust_count_rows=playbook_trust_count_rows,
+            alert_operation_count_rows=alert_operation_count_rows,
+            alertmanager_event_count_rows=alertmanager_event_count_rows,
+            grouped_alert_event_count_rows=grouped_alert_event_count_rows,
+            log_controlled_writeback_consumer=log_controlled_writeback_consumer,
+        )
+    consumer_receipt_fallback = _consumer_receipt_fallback_summary(
+        active=consumer_receipt_fallback_active,
+        db_read_status=db_read_status,
+        error_type=error_type,
+        log_controlled_writeback_consumer=log_controlled_writeback_consumer,
+    )
+
     operation_latest = list(operation_latest_rows)
     auto_repair_latest = list(auto_repair_latest_rows)
     verifier_latest = list(verifier_latest_rows)
@@ -3805,10 +4065,6 @@ def build_runtime_receipt_readback_from_rows(
     ui_productization = _build_ui_productization_readback()
     multi_product_taxonomy = _build_multi_product_taxonomy_contract(log_integration_taxonomy)
     log_controlled_writeback_executor = _load_log_controlled_writeback_executor_readback()
-    if not isinstance(log_controlled_writeback_consumer, Mapping):
-        log_controlled_writeback_consumer = (
-            _fallback_log_controlled_writeback_consumer_readback()
-        )
     work_item_progress = _build_work_item_progress(
         trace_ledger=trace_ledger,
         log_integration_taxonomy=log_integration_taxonomy,
@@ -3830,6 +4086,7 @@ def build_runtime_receipt_readback_from_rows(
         "lookback_hours": max(1, int(lookback_hours or _DEFAULT_LOOKBACK_HOURS)),
         "db_read_status": db_read_status,
         "partial_query_failures": [dict(item) for item in partial_query_failures],
+        "consumer_receipt_fallback": consumer_receipt_fallback,
         "writes_on_read": False,
         "ansible_operations": {
             "counts": operation_summary,