From c28c2b1598c58a2427f545f8d2e69806ca62371c Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 05:10:29 +0800 Subject: [PATCH 01/27] docs(sre): record failed release runtime truth --- ...e_k3s_controlled_automation_work_items_api.py | 4 +++- ...ontrolled-automation-work-items.snapshot.json | 16 ++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py index 67552e8cc..06a20e9c3 100644 --- a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py +++ b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py @@ -123,7 +123,9 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() - "runtime_closed_items": 0, "program_completion_percent": 0, "asset_coverage_status": "partial", - "runtime_closure_status": "not_started_for_this_program_sha", + "runtime_closure_status": ( + "blocked_cd_5384_failure_runtime_build_e614f061f7" + ), } diff --git a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json index e761a09dd..e1f3f38a0 100644 --- a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json +++ b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json @@ -2,8 +2,8 @@ "schema_version": "sre_k3s_controlled_automation_work_items_v1", "governance_version": "global_product_governance_v2", "program_id": "AIA-SRE-P0-20260715", - "generated_at": "2026-07-17T02:00:46+08:00", - "status": "phase_3_paid_provider_first_canary_failed_safe_rollback_verified", + "generated_at": "2026-07-19T05:07:28+08:00", + "status": "phase_6_release_blocked_cd_5384_failed_runtime_unchanged", "scope_complete": false, "current_p0": { "id": "AIA-SRE-013", @@ -941,14 +941,18 @@ "confirmed_truth": [ "the local release candidate is merged with the latest observed gitea/main and passes the CD-equivalent non-integration API gate with 5666 passed and 23 skipped, including the 977-test changed-file suite; 4 focused web tests, web typecheck and JSON/YAML parsing also pass", "stale Telegram tests that required unobserved OpenClaw/NemoTron/ElephantAlpha labels, a promotional check/apply/verify/writeback string or an unreceipted 88 percent confidence were replaced by no-false-AI receipt and actual stage-state contracts; receipt-backed provider, Agent99 executor, verifier and KM/MCP evidence remain visible", - "the Solver, RecommendedAction schema and callback registry now agree that host/systemd actions use the controlled provider and host_ansible_reconcile typed route instead of generic SSH fallback" + "the Solver, RecommendedAction schema and callback registry now agree that host/systemd actions use the controlled provider and host_ansible_reconcile typed route instead of generic SSH fallback", + "exact source SHA c0afb10861cbb76bc232fad3bbeea0d675cd9b80 was fast-forward pushed once to gitea/main and created the single CD run 5384; no empty commit, retry carrier or force push was used", + "CD run 5384 reached terminal Failure at 2026-07-19T05:05:37+08:00; gitea/main remained c0afb10861cbb76bc232fad3bbeea0d675cd9b80 with no deploy-marker successor", + "production delivery-closure readback generated at 2026-07-19T05:07:28+08:00 still reported runtime build e614f061f781c6e20a3964875d7ab3c28cf91190, so the candidate is not production-visible" ], "runtime_gaps": [ "no program item is runtime closed for this source SHA", "gitea-native exact target is missing while legacy GitHub exporter runtime drift remains", - "local source/test readiness is complete, but no normal push, CD terminal, deploy marker, production runtime or visible UI/Telegram evidence exists for this candidate" + "the exact candidate has a terminal failed CD but no deploy marker, production runtime match, visible UI/Telegram receipt or same-run KM/RAG/MCP/PlayBook acknowledgement", + "authenticated CD logs are unavailable in this lane, so the failed step and sanitized final error remain missing; another carrier would be an unproven retry" ], - "next_action": "hand the exact local candidate SHA to the authorized release lane for one normal push after shared-writer coordination, then require terminal CD plus production API/runtime/UI/Telegram readback and exact gitea-native/legacy-exporter verification" + "next_action": "the CI/CD owner must capture CD 5384 failed-step evidence and either reproduce a deterministic failure locally or prove a transient runner fault before one corrected carrier; after a successful deploy marker, require production API/runtime/UI/Telegram readback plus same-run KM/RAG/MCP/PlayBook acknowledgements" } ], "rollups": { @@ -965,7 +969,7 @@ "runtime_closed_items": 0, "program_completion_percent": 0, "asset_coverage_status": "partial", - "runtime_closure_status": "not_started_for_this_program_sha" + "runtime_closure_status": "blocked_cd_5384_failure_runtime_build_e614f061f7" }, "completion_contract": { "required_stages": [ From 60ee187bd982a58ab354e3c570ae3d375d7d5113 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 05:30:42 +0800 Subject: [PATCH 02/27] fix(sre): bound Telegram receipt readbacks --- .../src/services/platform_operator_service.py | 6 +- ...gram_alert_monitoring_coverage_readback.py | 256 ++++++++++++++++-- .../test_awooop_operator_timeline_labels.py | 8 +- ..._alert_monitoring_coverage_readback_api.py | 246 ++++++++++++++++- ...rolled-automation-work-items.snapshot.json | 6 +- 5 files changed, 495 insertions(+), 27 deletions(-) diff --git a/apps/api/src/services/platform_operator_service.py b/apps/api/src/services/platform_operator_service.py index 7edb8c3a1..89e1185a2 100644 --- a/apps/api/src/services/platform_operator_service.py +++ b/apps/api/src/services/platform_operator_service.py @@ -1395,6 +1395,7 @@ async def list_ai_alert_card_delivery_readback( page: int = 1, per_page: int = 20, refresh: bool = False, + cache_write: bool = True, ) -> dict[str, Any]: """Read-only AwoooP delivery readback for AI automation alert cards.""" normalized_project_id = project_id or "awoooi" @@ -1810,9 +1811,12 @@ async def list_ai_alert_card_delivery_readback( page=normalized_page, per_page=normalized_per_page, total=summary["total"], - cache_status="miss", + cache_status="miss" if cache_write else "write_disabled_read_only", cache_ttl_seconds=_AI_ALERT_CARD_CACHE_TTL_SECONDS, ) + if not cache_write: + response["summary"]["cache_write_status"] = "disabled_read_only" + return response try: return await store_operator_summary_async( "ai_alert_card_delivery_readback", diff --git a/apps/api/src/services/telegram_alert_monitoring_coverage_readback.py b/apps/api/src/services/telegram_alert_monitoring_coverage_readback.py index a03a6c80b..ca51bc934 100644 --- a/apps/api/src/services/telegram_alert_monitoring_coverage_readback.py +++ b/apps/api/src/services/telegram_alert_monitoring_coverage_readback.py @@ -13,7 +13,7 @@ from __future__ import annotations import asyncio import json -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from pathlib import Path from typing import Any @@ -31,6 +31,7 @@ from src.services.telegram_alert_ai_automation_matrix import ( load_latest_telegram_alert_ai_automation_matrix, ) from src.services.telegram_alert_learning_context_post_apply_verifier import ( + build_telegram_alert_learning_context_post_apply_verifier, load_latest_telegram_alert_learning_context_post_apply_verifier, ) @@ -41,6 +42,17 @@ _RUNTIME_DIRECT_CONNECT_TIMEOUT_SECONDS = 1.5 _RUNTIME_DIRECT_QUERY_TIMEOUT_SECONDS = 2.0 _RUNTIME_DIRECT_CLOSE_TIMEOUT_SECONDS = 0.25 _RUNTIME_DIRECT_STATEMENT_TIMEOUT_MS = 1500 +_PRELUDE_STAGE_TIMEOUT_SECONDS = 0.75 +_PRELUDE_STAGE_COUNT = 2 +_SERIAL_DB_STAGE_TIMEOUT_SECONDS = 2.0 +_SERIAL_DB_STAGE_COUNT = 4 +_SERIAL_DB_TOTAL_BUDGET_SECONDS = ( + _SERIAL_DB_STAGE_TIMEOUT_SECONDS * _SERIAL_DB_STAGE_COUNT +) +_FULL_READBACK_TOTAL_BUDGET_SECONDS = ( + _PRELUDE_STAGE_TIMEOUT_SECONDS * _PRELUDE_STAGE_COUNT + + _SERIAL_DB_TOTAL_BUDGET_SECONDS +) _MONITORING_LIVE_RECEIPT_OPERATION_TYPE = "telegram_monitoring_live_receipt_applied" _MONITORING_LIVE_RECEIPT_EXECUTOR_ROUTE = "ai_agent_monitoring_live_receipt_consumer" _MONITORING_INVENTORY = "monitoring-alerting-observability-inventory.snapshot.json" @@ -316,6 +328,136 @@ _REQUIRED_TAG_DIMENSIONS = ( logger = get_logger(__name__) +async def _load_prelude_stage( + *, + stage: str, + loader: Callable[[], Awaitable[dict[str, Any]]], +) -> dict[str, Any]: + try: + return await asyncio.wait_for( + loader(), + timeout=_PRELUDE_STAGE_TIMEOUT_SECONDS, + ) + except TimeoutError: + logger.warning( + "telegram_alert_monitoring_prelude_stage_timeout", + stage=stage, + timeout_seconds=_PRELUDE_STAGE_TIMEOUT_SECONDS, + total_budget_seconds=_FULL_READBACK_TOTAL_BUDGET_SECONDS, + ) + return _prelude_timeout_payload(stage) + + +def _prelude_timeout_payload(stage: str) -> dict[str, Any]: + if stage == "matrix": + return { + "schema_version": "telegram_alert_ai_automation_matrix_v1", + "status": "telegram_alert_ai_automation_matrix_unavailable", + "summary": { + "telegram_alert_surface_count": 0, + "known_direct_send_gap_count": 0, + "db_or_log_receipt_ready_surface_count": 0, + "ai_route_ready_surface_count": 0, + "controlled_queue_ready_surface_count": 0, + "post_verifier_ready_surface_count": 0, + "learning_writeback_ready_surface_count": 0, + "error_type": "TimeoutError", + }, + } + if stage == "source_contract": + return { + "alertmanager_alert_log_append_present": False, + "alert_operation_log_telegram_sent_present": False, + "alert_operation_log_repository_present": False, + "telegram_gateway_outbound_mirror_present": False, + "telegram_gateway_outbound_record_present": False, + "ai_alert_card_delivery_readback_present": False, + "ai_alert_card_learning_refs_present": False, + "post_apply_verifier_present": False, + "awooop_coverage_surface_present": False, + } + raise ValueError(f"unsupported prelude stage: {stage}") + + +async def _load_serial_db_stage( + *, + stage: str, + loader: Callable[[], Awaitable[dict[str, Any]]], +) -> dict[str, Any]: + try: + return await asyncio.wait_for( + loader(), + timeout=_SERIAL_DB_STAGE_TIMEOUT_SECONDS, + ) + except TimeoutError: + logger.warning( + "telegram_alert_monitoring_serial_db_stage_timeout", + stage=stage, + timeout_seconds=_SERIAL_DB_STAGE_TIMEOUT_SECONDS, + total_budget_seconds=_SERIAL_DB_TOTAL_BUDGET_SECONDS, + ) + return _serial_db_timeout_payload(stage) + + +def _serial_db_timeout_payload(stage: str) -> dict[str, Any]: + if stage == "consumer": + return { + "schema_version": "ai_agent_log_controlled_writeback_consumer_readback_v1", + "status": "blocked_waiting_controlled_writeback_consumer_db_readback", + "rollups": { + "controlled_consumer_readback_ready": False, + "telegram_alert_learning_context_readback_ready": False, + "telegram_alert_learning_context_receipt_count": 0, + "telegram_alert_learning_ai_agent_context_receipt_count": 0, + "error_type": "TimeoutError", + }, + "telegram_alert_learning_context": { + "status": "blocked_telegram_alert_learning_registry_readback_unavailable", + "active_blockers": ["serial_db_stage_timeout"], + }, + } + if stage == "live_receipt": + return { + "status": "unavailable", + "summary": { + "readback_source": "unavailable", + "row_count": 0, + "ready_row_count": 0, + "accepted_surface_count": 0, + "error_type": "TimeoutError", + }, + "rows": [], + } + if stage == "ai_alert_card": + return { + "status": "unavailable", + "summary": { + "total": 0, + "learning_writeback_ready_total": 0, + "db_read_status": "unavailable", + "error_type": "TimeoutError", + }, + } + if stage == "runtime_log": + return { + "status": "unavailable", + "summary": { + "alert_operation_log_total_7d": 0, + "telegram_sent_event_count_7d": 0, + "km_converted_event_count_7d": 0, + "playbook_draft_event_count_7d": 0, + "automation_triggered_run_count_7d": 0, + "automation_closed_run_count_7d": 0, + "automation_open_run_count_7d": 0, + "automation_runtime_closure_percent": 0.0, + "automation_runtime_closure_ready": False, + "error_type": "TimeoutError", + }, + "event_type_counts_7d": {}, + } + raise ValueError(f"unsupported serial DB stage: {stage}") + + async def load_latest_telegram_alert_monitoring_coverage_readback( *, project_id: str = DEFAULT_PROJECT_ID, @@ -342,21 +484,40 @@ async def load_latest_telegram_alert_monitoring_coverage_readback( ) _require_safe_boundaries(monitoring_inventory, monitoring_readback_plan) - ( - matrix, - consumer_readback, - monitoring_live_receipt_apply, - source_contract, - ) = await asyncio.gather( - _load_telegram_matrix_readback(project_id=project_id), - _load_log_controlled_writeback_consumer_readback(project_id=project_id), - _load_monitoring_live_receipt_apply_readback(project_id=project_id), - asyncio.to_thread(_inspect_source_contract, repo_root), + matrix = await _load_prelude_stage( + stage="matrix", + loader=lambda: _load_telegram_matrix_readback(project_id=project_id), ) - verifier, ai_alert_readback, runtime_log_readback = await asyncio.gather( - _load_post_apply_verifier_readback(project_id=project_id), - _load_ai_alert_card_delivery(project_id=project_id), - _load_runtime_log_readback(project_id=project_id), + source_contract = await _load_prelude_stage( + stage="source_contract", + loader=lambda: asyncio.to_thread(_inspect_source_contract, repo_root), + ) + # These readbacks share the constrained API database role in production. + # Keep them serial so one operator read does not consume several role slots + # and turn otherwise healthy metadata into simultaneous timeout fallbacks. + consumer_readback = await _load_serial_db_stage( + stage="consumer", + loader=lambda: _load_log_controlled_writeback_consumer_readback( + project_id=project_id + ), + ) + monitoring_live_receipt_apply = await _load_serial_db_stage( + stage="live_receipt", + loader=lambda: _load_monitoring_live_receipt_apply_readback( + project_id=project_id + ), + ) + verifier = build_telegram_alert_learning_context_post_apply_verifier( + consumer_readback=consumer_readback, + project_id=project_id, + ) + ai_alert_readback = await _load_serial_db_stage( + stage="ai_alert_card", + loader=lambda: _load_ai_alert_card_delivery(project_id=project_id), + ) + runtime_log_readback = await _load_serial_db_stage( + stage="runtime_log", + loader=lambda: _load_runtime_log_readback(project_id=project_id), ) return build_telegram_alert_monitoring_coverage_readback( @@ -1130,6 +1291,7 @@ async def _load_ai_alert_card_delivery(*, project_id: str) -> dict[str, Any]: page=1, per_page=6, refresh=True, + cache_write=False, ), timeout=_LIVE_READBACK_TIMEOUT_SECONDS, ) @@ -2121,28 +2283,82 @@ def _ai_controlled_gap_queue( }, }) continue + execution_contract = _gap_execution_contract(blocker) queue.append({ "work_item_id": f"CIR-P0-TG-001-{index:02d}", "parent_work_item_id": "CIR-P0-TG-001", "priority": "P0" if index <= 4 else "P1", "blocker": blocker, - "status": "queued_ai_controlled_apply", - "owner_review_required_for_low_medium_high": False, - "critical_break_glass_required": True, + **execution_contract, "target_selector": _gap_target_selector(blocker), "controlled_next_action": _gap_next_action(blocker), "post_verifier": ( "/api/v1/agents/telegram-alert-monitoring-coverage-readback" ), "sample_surfaces": gap_samples[:3] if blocker.startswith("monitoring_") else [], + }) + return queue + + +def _gap_execution_contract(blocker: str) -> dict[str, Any]: + if blocker in { + "runtime_alert_operation_log_db_readback_unavailable", + "awooop_ai_alert_card_delivery_db_readback_unavailable", + }: + return { + "status": "ready_ai_read_only_retry", + "risk_level": "low", + "execution_mode": "automatic_bounded_readback_retry", + "executor_route": "telegram_monitoring_readback_service", + "ai_controlled_execution_allowed": True, + "owner_review_required_for_low_medium_high": False, + "critical_break_glass_required": False, "operation_boundaries": { + "database_read_only": True, "requires_secret": False, "requires_runtime_send": False, + "runtime_write_performed": False, "stores_raw_payload": False, "destructive_action": False, }, - }) - return queue + } + if blocker == "runtime_ai_automation_trigger_receipt_missing" or blocker.startswith( + "runtime_ai_automation_lifecycle_open" + ): + return { + "status": "blocked_critical_break_glass_required", + "risk_level": "critical", + "execution_mode": "controlled_runtime_canary_or_lifecycle_resume", + "executor_route": "single_controlled_executor", + "ai_controlled_execution_allowed": False, + "owner_review_required_for_low_medium_high": False, + "critical_break_glass_required": True, + "operation_boundaries": { + "database_read_only": False, + "requires_secret": False, + "requires_runtime_send": True, + "runtime_write_performed": False, + "stores_raw_payload": False, + "destructive_action": False, + }, + } + return { + "status": "queued_ai_controlled_apply", + "risk_level": "high", + "execution_mode": "controlled_gap_closure", + "executor_route": "single_controlled_executor", + "ai_controlled_execution_allowed": False, + "owner_review_required_for_low_medium_high": False, + "critical_break_glass_required": True, + "operation_boundaries": { + "database_read_only": False, + "requires_secret": False, + "requires_runtime_send": False, + "runtime_write_performed": False, + "stores_raw_payload": False, + "destructive_action": False, + }, + } def _coverage_matrix( diff --git a/apps/api/tests/test_awooop_operator_timeline_labels.py b/apps/api/tests/test_awooop_operator_timeline_labels.py index fb5067442..b93b49f8d 100644 --- a/apps/api/tests/test_awooop_operator_timeline_labels.py +++ b/apps/api/tests/test_awooop_operator_timeline_labels.py @@ -1752,8 +1752,8 @@ async def test_unfiltered_query_selects_declared_effective_and_dispatch_receipts async def __aexit__(self, exc_type, exc, tb): return False - async def passthrough_store(_kind, _key, value, **_kwargs): - return value + async def forbidden_store(*_args, **_kwargs): + raise AssertionError("read-only delivery readback must not write cache") monkeypatch.setattr( platform_operator_service, @@ -1763,7 +1763,7 @@ async def test_unfiltered_query_selects_declared_effective_and_dispatch_receipts monkeypatch.setattr( platform_operator_service, "store_operator_summary_async", - passthrough_store, + forbidden_store, ) payload = await platform_operator_service.list_ai_alert_card_delivery_readback( @@ -1773,6 +1773,7 @@ async def test_unfiltered_query_selects_declared_effective_and_dispatch_receipts page=1, per_page=5, refresh=True, + cache_write=False, ) summary_query = next(query for query in queries if "COUNT(*) AS total" in query) @@ -1790,6 +1791,7 @@ async def test_unfiltered_query_selects_declared_effective_and_dispatch_receipts assert "LEFT JOIN agent99_runs ar" in list_query assert "ON ar.message_id = m.message_id" in list_query assert "FROM agent99_runs ars" not in list_query + assert payload["summary"]["cache_write_status"] == "disabled_read_only" assert list_query.index("LIMIT :limit OFFSET :offset") < list_query.index( "candidate_refs AS MATERIALIZED" ) diff --git a/apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py b/apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py index 95eb291ca..d4bfdd6a0 100644 --- a/apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py +++ b/apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py @@ -471,6 +471,62 @@ def test_telegram_alert_monitoring_coverage_blocks_open_runtime_lifecycle(): pipeline = {stage["stage_id"]: stage for stage in payload["alert_receipt_pipeline"]} assert pipeline["runtime_ai_automation_lifecycle_closure"]["ready"] is False assert pipeline["runtime_ai_automation_lifecycle_closure"]["gap_count"] == 7 + lifecycle_candidate = next( + item + for item in payload["ai_controlled_gap_queue"] + if item["blocker"] == "runtime_ai_automation_lifecycle_open:7" + ) + assert lifecycle_candidate["status"] == "blocked_critical_break_glass_required" + assert lifecycle_candidate["risk_level"] == "critical" + assert lifecycle_candidate["critical_break_glass_required"] is True + assert lifecycle_candidate["operation_boundaries"]["requires_runtime_send"] is True + + +def test_telegram_alert_monitoring_gap_queue_separates_read_only_and_runtime_write(): + payload = _payload( + runtime_log_readback={ + "status": "unavailable", + "summary": { + "alert_operation_log_total_7d": 0, + "automation_triggered_run_count_7d": 0, + "automation_closed_run_count_7d": 0, + "automation_open_run_count_7d": 0, + "automation_runtime_closure_percent": 0.0, + "automation_runtime_closure_ready": False, + }, + "event_type_counts_7d": {}, + }, + ai_alert_card_delivery_readback={ + "status": "unavailable", + "summary": { + "total": 0, + "learning_writeback_ready_total": 0, + "db_read_status": "unavailable", + }, + }, + monitoring_live_receipt_apply_readback=_metadata_live_receipt_apply_readback(), + ) + queue = {item["blocker"]: item for item in payload["ai_controlled_gap_queue"]} + + for blocker in ( + "runtime_alert_operation_log_db_readback_unavailable", + "awooop_ai_alert_card_delivery_db_readback_unavailable", + ): + candidate = queue[blocker] + assert candidate["status"] == "ready_ai_read_only_retry" + assert candidate["risk_level"] == "low" + assert candidate["execution_mode"] == "automatic_bounded_readback_retry" + assert candidate["ai_controlled_execution_allowed"] is True + assert candidate["critical_break_glass_required"] is False + assert candidate["operation_boundaries"]["database_read_only"] is True + assert candidate["operation_boundaries"]["requires_runtime_send"] is False + + runtime_canary = queue["runtime_ai_automation_trigger_receipt_missing"] + assert runtime_canary["status"] == "blocked_critical_break_glass_required" + assert runtime_canary["risk_level"] == "critical" + assert runtime_canary["ai_controlled_execution_allowed"] is False + assert runtime_canary["critical_break_glass_required"] is True + assert runtime_canary["operation_boundaries"]["requires_runtime_send"] is True def test_telegram_alert_monitoring_coverage_blocks_partial_surface_contracts(): @@ -668,6 +724,189 @@ def test_telegram_alert_monitoring_coverage_endpoint(monkeypatch): assert data["operation_boundaries"]["bot_api_call_performed"] is False +@pytest.mark.asyncio +async def test_telegram_alert_monitoring_live_loader_serializes_db_readbacks( + monkeypatch: pytest.MonkeyPatch, +): + active_db_reads = 0 + max_active_db_reads = 0 + read_order: list[str] = [] + + async def db_read(name: str, payload: dict) -> dict: + nonlocal active_db_reads, max_active_db_reads + active_db_reads += 1 + max_active_db_reads = max(max_active_db_reads, active_db_reads) + read_order.append(f"{name}:start") + await asyncio.sleep(0) + read_order.append(f"{name}:end") + active_db_reads -= 1 + return payload + + async def matrix_readback(*, project_id: str) -> dict: + assert project_id == "awoooi" + return { + "schema_version": "telegram_alert_ai_automation_matrix_v1", + "status": "telegram_alert_ai_automation_matrix_ready", + "summary": { + "telegram_alert_surface_count": 9, + "known_direct_send_gap_count": 0, + "db_or_log_receipt_ready_surface_count": 9, + "ai_route_ready_surface_count": 9, + "controlled_queue_ready_surface_count": 9, + "post_verifier_ready_surface_count": 9, + "learning_writeback_ready_surface_count": 9, + }, + } + + async def consumer_readback(*, project_id: str) -> dict: + assert project_id == "awoooi" + return await db_read("consumer", _consumer_context_readback()) + + async def live_receipt_readback(*, project_id: str) -> dict: + assert project_id == "awoooi" + return await db_read("live_receipt", _metadata_live_receipt_apply_readback()) + + async def ai_card_readback(*, project_id: str) -> dict: + assert project_id == "awoooi" + return await db_read( + "ai_card", + { + "status": "ok", + "summary": {"total": 6, "learning_writeback_ready_total": 6}, + }, + ) + + async def runtime_log_readback(*, project_id: str) -> dict: + assert project_id == "awoooi" + return await db_read( + "runtime_log", + { + "status": "ok", + "summary": { + "alert_operation_log_total_7d": 4, + "automation_triggered_run_count_7d": 1, + "automation_closed_run_count_7d": 1, + "automation_open_run_count_7d": 0, + "automation_runtime_closure_percent": 100.0, + "automation_runtime_closure_ready": True, + }, + "event_type_counts_7d": {"ALERT_RECEIVED": 1}, + }, + ) + + monkeypatch.setattr( + coverage_service, "_load_telegram_matrix_readback", matrix_readback + ) + monkeypatch.setattr( + coverage_service, + "_load_log_controlled_writeback_consumer_readback", + consumer_readback, + ) + monkeypatch.setattr( + coverage_service, + "_load_monitoring_live_receipt_apply_readback", + live_receipt_readback, + ) + monkeypatch.setattr( + coverage_service, "_load_ai_alert_card_delivery", ai_card_readback + ) + monkeypatch.setattr( + coverage_service, "_load_runtime_log_readback", runtime_log_readback + ) + + payload = ( + await coverage_service.load_latest_telegram_alert_monitoring_coverage_readback() + ) + + assert max_active_db_reads == 1 + assert read_order == [ + "consumer:start", + "consumer:end", + "live_receipt:start", + "live_receipt:end", + "ai_card:start", + "ai_card:end", + "runtime_log:start", + "runtime_log:end", + ] + assert payload["summary"]["runtime_automation_closure_ready"] is True + + +@pytest.mark.asyncio +async def test_telegram_alert_monitoring_serial_db_budget_is_below_ui_deadline( + monkeypatch: pytest.MonkeyPatch, +): + async def slow_stage() -> dict: + await asyncio.sleep(1) + return {"status": "unexpected"} + + monkeypatch.setattr(coverage_service, "_SERIAL_DB_STAGE_TIMEOUT_SECONDS", 0.01) + started_at = asyncio.get_running_loop().time() + results = [ + await coverage_service._load_serial_db_stage( + stage=stage, + loader=slow_stage, + ) + for stage in ("consumer", "live_receipt", "ai_alert_card", "runtime_log") + ] + elapsed_seconds = asyncio.get_running_loop().time() - started_at + + assert coverage_service._SERIAL_DB_TOTAL_BUDGET_SECONDS == 8.0 + assert coverage_service._FULL_READBACK_TOTAL_BUDGET_SECONDS == 9.5 + assert coverage_service._FULL_READBACK_TOTAL_BUDGET_SECONDS < 12.0 + assert elapsed_seconds < 0.2 + assert [result["status"] for result in results] == [ + "blocked_waiting_controlled_writeback_consumer_db_readback", + "unavailable", + "unavailable", + "unavailable", + ] + + +@pytest.mark.asyncio +async def test_telegram_alert_monitoring_full_loader_honors_shared_deadline( + monkeypatch: pytest.MonkeyPatch, +): + async def slow_stage(**_kwargs) -> dict: + await asyncio.sleep(1) + return {"status": "unexpected"} + + monkeypatch.setattr(coverage_service, "_PRELUDE_STAGE_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(coverage_service, "_SERIAL_DB_STAGE_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr( + coverage_service, "_load_telegram_matrix_readback", slow_stage + ) + monkeypatch.setattr( + coverage_service, + "_load_log_controlled_writeback_consumer_readback", + slow_stage, + ) + monkeypatch.setattr( + coverage_service, + "_load_monitoring_live_receipt_apply_readback", + slow_stage, + ) + monkeypatch.setattr( + coverage_service, "_load_ai_alert_card_delivery", slow_stage + ) + monkeypatch.setattr(coverage_service, "_load_runtime_log_readback", slow_stage) + monkeypatch.setattr( + coverage_service, "_inspect_source_contract", lambda _repo_root: _source_contract() + ) + + started_at = asyncio.get_running_loop().time() + payload = ( + await coverage_service.load_latest_telegram_alert_monitoring_coverage_readback() + ) + elapsed_seconds = asyncio.get_running_loop().time() - started_at + + assert elapsed_seconds < 0.2 + assert payload["status"] == ( + "blocked_telegram_alert_monitoring_coverage_gaps_present" + ) + assert payload["summary"]["runtime_automation_closure_ready"] is False + + @pytest.mark.asyncio async def test_telegram_alert_monitoring_live_readbacks_are_bounded( monkeypatch: pytest.MonkeyPatch, @@ -855,7 +1094,10 @@ def test_runtime_lifecycle_sql_accepts_only_durable_failed_terminals() -> None: async def test_telegram_alert_monitoring_marks_ai_alert_cards_source_unavailable( monkeypatch: pytest.MonkeyPatch, ): - async def source_unavailable_ai_alert_cards(**_kwargs): + call_kwargs: dict = {} + + async def source_unavailable_ai_alert_cards(**kwargs): + call_kwargs.update(kwargs) return ( platform_operator_service ._ai_alert_card_delivery_source_unavailable_response( @@ -877,6 +1119,8 @@ async def test_telegram_alert_monitoring_marks_ai_alert_cards_source_unavailable ai_cards = await coverage_service._load_ai_alert_card_delivery(project_id="awoooi") assert ai_cards["status"] == "unavailable" + assert call_kwargs["refresh"] is True + assert call_kwargs["cache_write"] is False assert ai_cards["summary"]["db_read_status"] == "unavailable" assert ai_cards["learning_registry"]["status"] == ( "source_unavailable_ai_controlled_repair_required" diff --git a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json index e1f3f38a0..2328241b2 100644 --- a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json +++ b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json @@ -902,12 +902,14 @@ "unknown product, signal, severity or requested destination fails closed and active source contains no raw numeric Telegram destination default", "Telegram automation cards distinguish durable AI/provider, Agent, executor, verifier and learning receipts from configured-but-unobserved components", "Agent99 same-run closure is two-phase: an ApprovalRecord fingerprint/hit-count/last-seen recurrence fence must still match, the incident resolution commits first, the recovered Telegram card must receive a durable destination-bound provider acknowledgement, and only then may the run claim runtime_closure_verified", - "information and controlled-action buttons are bound to registered callback actions instead of display-only ghost buttons" + "information and controlled-action buttons are bound to registered callback actions instead of display-only ghost buttons", + "Telegram monitoring coverage source now limits its complete readback pipeline to a 9.5-second fail-closed budget, serializes four database stages to one connection at a time, disables cache writes for the AI-card coverage read, classifies bounded DB retries as low-risk read-only, and preserves break-glass for runtime canary or lifecycle resume" ], "runtime_gaps": [ "the host99 Agent99 independent pull/reduced relay has no production deployment, freshness, dedupe or controlled-repair receipt", "production Alertmanager exposes integration-only notification counters; the bounded check now passes after adding a scrape-time receiver_contract label, but apply and alert resolution remain pending", - "the recipient-visible Agent99 recovered-card closure contract is source-tested only; no production Telegram provider acknowledgement or same-run runtime closure receipt exists yet" + "the recipient-visible Agent99 recovered-card closure contract is source-tested only; no production Telegram provider acknowledgement or same-run runtime closure receipt exists yet", + "production runtime e614f061f7 still reports both alert-operation-log and AI-card-delivery DB readbacks unavailable with zero 7-day automation lifecycle receipts; the bounded serial readback fix is source-only until a successful deploy marker" ], "next_action": "deploy one exact source revision through the authorized release lane, then verify receiver-contract freshness, host99 Agent99 exact-host polling, dedupe, callback execution, current firing alert resolution and recipient-visible same-run delivery receipts; source tests must not be counted as runtime closure" }, From eafaa20f334238fda54aeb79bb9eb3e3e5ccb69f Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 05:45:08 +0800 Subject: [PATCH 03/27] fix(sre): verify Telegram learning receipts --- ..._controlled_writeback_consumer_readback.py | 41 ++++- ...rt_learning_context_post_apply_verifier.py | 132 ++++++++++++++-- ...trolled_writeback_consumer_readback_api.py | 48 ++++++ ...earning_context_post_apply_verifier_api.py | 148 ++++++++++++++++++ ...rolled-automation-work-items.snapshot.json | 10 +- 5 files changed, 361 insertions(+), 18 deletions(-) diff --git a/apps/api/src/services/ai_agent_log_controlled_writeback_consumer_readback.py b/apps/api/src/services/ai_agent_log_controlled_writeback_consumer_readback.py index 5f3758950..94ffefde8 100644 --- a/apps/api/src/services/ai_agent_log_controlled_writeback_consumer_readback.py +++ b/apps/api/src/services/ai_agent_log_controlled_writeback_consumer_readback.py @@ -586,7 +586,9 @@ def _build_consumer_readback( "verifier_consumer_binding_count": _target_count(bindings, "verifier"), "ai_agent_consumer_binding_count": _target_count(bindings, "ai_agent"), "target_context_receipt_write_count": sum( - 1 for binding in bindings if binding["target_write_performed"] is True + 1 + for binding in bindings + if _binding_has_verified_target_context_write(binding) ), "km_context_receipt_write_count": _target_write_count(bindings, "km"), "rag_context_receipt_write_count": _target_write_count(bindings, "rag"), @@ -895,7 +897,7 @@ def _target_rollups(bindings: list[dict[str, Any]]) -> list[dict[str, Any]]: if binding["target"] == target ), "target_write_performed": any( - binding["target_write_performed"] is True + _binding_has_verified_target_context_write(binding) for binding in bindings if binding["target"] == target ), @@ -912,7 +914,23 @@ def _target_write_count(bindings: list[dict[str, Any]], target: str) -> int: return sum( 1 for binding in bindings - if binding["target"] == target and binding["target_write_performed"] is True + if binding["target"] == target + and _binding_has_verified_target_context_write(binding) + ) + + +def _binding_has_verified_target_context_write( + binding: Mapping[str, Any], +) -> bool: + receipt = _dict(binding.get("target_write_receipt")) + consumer_receipt_id = str(receipt.get("consumer_receipt_id") or "") + return bool( + binding.get("target_write_performed") is True + and consumer_receipt_id + and consumer_receipt_id == str(binding.get("consumer_receipt_id") or "") + and receipt.get("status") == "success" + and receipt.get("target_context_receipt_write_performed") is True + and receipt.get("runtime_target_write_performed") is True ) @@ -1050,6 +1068,7 @@ def _telegram_alert_context_receipts( status = str(binding.get("status") or "") ready = status == "ready_for_consumer_context" and bool(ref) receipts.append({ + "receipt_kind": "alert_card_learning_registry", "receipt_id": ( "telegram_alert_learning_context::" f"{message_id or 'unknown-message'}::{target}" @@ -1172,7 +1191,7 @@ def _telegram_alert_learning_context_with_consumer_fallback( binding for binding in bindings if binding.get("status") == "ready_for_consumer_context" - and binding.get("target_write_performed") is True + and _binding_has_verified_target_context_write(binding) and binding.get("raw_payload_included") is False and binding.get("target") in _TARGETS ] @@ -1237,6 +1256,7 @@ def _telegram_alert_context_receipt_from_consumer_binding( ) verifier_refs = _strings(binding.get("post_apply_verifier_refs")) return { + "receipt_kind": "log_controlled_writeback_consumer", "receipt_id": ( "telegram_alert_learning_context_fallback::" f"{source_receipt_id or target}" @@ -1256,7 +1276,18 @@ def _telegram_alert_context_receipt_from_consumer_binding( "lane": "metadata_feedback_controlled_writeback", "status": "ready_for_ai_loop_context", "metadata_only": True, - "target_write_performed": True, + "target_write_performed": binding.get("target_write_performed") is True, + "consumer_write_receipt": { + "consumer_receipt_id": source_receipt_id, + "status": str(consumer_receipt.get("status") or ""), + "target_context_receipt_write_performed": ( + consumer_receipt.get("target_context_receipt_write_performed") is True + ), + "runtime_target_write_performed": ( + consumer_receipt.get("runtime_target_write_performed") is True + ), + }, + "verifier_write_performed": False, "raw_payload_included": False, "target_selector": { "project_id": project_id, diff --git a/apps/api/src/services/telegram_alert_learning_context_post_apply_verifier.py b/apps/api/src/services/telegram_alert_learning_context_post_apply_verifier.py index 5b4d84f7a..51904a489 100644 --- a/apps/api/src/services/telegram_alert_learning_context_post_apply_verifier.py +++ b/apps/api/src/services/telegram_alert_learning_context_post_apply_verifier.py @@ -18,19 +18,34 @@ from src.services.ai_agent_log_controlled_writeback_consumer_readback import ( SCHEMA_VERSION = "telegram_alert_learning_context_post_apply_verifier_v1" DEFAULT_PROJECT_ID = "awoooi" _TARGETS = ("km", "rag", "playbook", "mcp", "verifier", "ai_agent") -_REQUIRED_TARGET_SELECTOR_FIELDS = ( +_ALERT_CARD_RECEIPT_KIND = "alert_card_learning_registry" +_CONSUMER_RECEIPT_KIND = "log_controlled_writeback_consumer" +_ALERT_CARD_TARGET_SELECTOR_FIELDS = ( "project_id", "run_id", "message_id", "target", "source_ref", ) -_REQUIRED_CHECKS = ( +_CONSUMER_TARGET_SELECTOR_FIELDS = ( + "project_id", + "dispatch_receipt_id", + "consumer_receipt_id", + "target", + "source_ref", +) +_ALERT_CARD_REQUIRED_CHECKS = ( "ai_alert_card_delivery_receipt_present", "learning_registry_binding_ready", "metadata_only_raw_payload_absent", "ai_agent_context_ref_present", ) +_CONSUMER_REQUIRED_CHECKS = ( + "consumer_binding_ready", + "consumer_apply_receipt_present", + "metadata_only_raw_payload_absent", + "post_apply_verifier_refs_present", +) async def load_latest_telegram_alert_learning_context_post_apply_verifier( @@ -163,6 +178,11 @@ def build_telegram_alert_learning_context_post_apply_verifier( "post_apply_verifier_ref_count": sum( len(item["post_apply_verifier_refs"]) for item in verifier_results ), + "verified_target_context_write_count": sum( + 1 + for item in verifier_results + if item["verified_target_context_write_performed"] is True + ), "metadata_only_receipt_count": sum( 1 for item in verifier_results if item["metadata_only"] is True ), @@ -200,27 +220,54 @@ def build_telegram_alert_learning_context_post_apply_verifier( def _verify_context_receipt(receipt: dict[str, Any]) -> dict[str, Any]: target = str(receipt.get("target") or "") + receipt_kind = _receipt_kind(receipt) + receipt_kind_supported = receipt_kind in { + _ALERT_CARD_RECEIPT_KIND, + _CONSUMER_RECEIPT_KIND, + } + consumer_receipt = receipt_kind == _CONSUMER_RECEIPT_KIND target_selector = _dict(receipt.get("target_selector")) source_diff = _dict(receipt.get("source_of_truth_diff")) check_mode = _dict(receipt.get("check_mode")) rollback = _dict(receipt.get("rollback")) post_apply_verifier = _dict(receipt.get("post_apply_verifier")) verifier_refs = _strings(post_apply_verifier.get("verifier_refs")) + required_target_selector_fields = ( + _CONSUMER_TARGET_SELECTOR_FIELDS + if consumer_receipt + else _ALERT_CARD_TARGET_SELECTOR_FIELDS + ) + required_checks = ( + _CONSUMER_REQUIRED_CHECKS + if consumer_receipt + else _ALERT_CARD_REQUIRED_CHECKS + ) target_selector_missing = [ - key for key in _REQUIRED_TARGET_SELECTOR_FIELDS if not target_selector.get(key) + key for key in required_target_selector_fields if not target_selector.get(key) ] checks = _strings(check_mode.get("checks")) - missing_checks = [check for check in _REQUIRED_CHECKS if check not in checks] + missing_checks = [check for check in required_checks if check not in checks] + source_diff_verified = _source_diff_verified( + source_diff, + target, + receipt_kind=receipt_kind, + ) + target_write_contract_verified = _target_write_contract_verified( + receipt, + receipt_kind=receipt_kind, + ) failed_checks: list[str] = [] if target not in _TARGETS: failed_checks.append("target_in_allowed_set") + if not receipt_kind_supported: + failed_checks.append("receipt_kind_supported") if receipt.get("status") != "ready_for_ai_loop_context": failed_checks.append("receipt_ready_for_ai_loop_context") if target_selector_missing: failed_checks.append("target_selector_required_fields_present") - if not _source_diff_verified(source_diff, target): + if not source_diff_verified: failed_checks.append("source_of_truth_diff_verified") if check_mode.get("enabled") is not True or missing_checks: failed_checks.append("check_mode_required_checks_present") @@ -232,13 +279,21 @@ def _verify_context_receipt(receipt: dict[str, Any]) -> dict[str, Any]: failed_checks.append("metadata_only_receipt") if receipt.get("raw_payload_included") is not False: failed_checks.append("raw_payload_absent") - if receipt.get("target_write_performed") is not False: - failed_checks.append("target_write_not_performed_by_verifier") + target_write_check = ( + "consumer_write_receipt_verified" + if consumer_receipt + else "target_write_not_performed_by_verifier" + ) + if not target_write_contract_verified: + failed_checks.append(target_write_check) + if receipt.get("verifier_write_performed") is True: + failed_checks.append("verifier_write_not_performed") if not receipt.get("ai_agent_context_ref"): failed_checks.append("ai_agent_context_ref_present") passed_checks = [ "target_in_allowed_set", + "receipt_kind_supported", "receipt_ready_for_ai_loop_context", "target_selector_required_fields_present", "source_of_truth_diff_verified", @@ -247,7 +302,8 @@ def _verify_context_receipt(receipt: dict[str, Any]) -> dict[str, Any]: "post_apply_verifier_refs_present", "metadata_only_receipt", "raw_payload_absent", - "target_write_not_performed_by_verifier", + target_write_check, + "verifier_write_not_performed", "ai_agent_context_ref_present", ] passed_checks = [check for check in passed_checks if check not in failed_checks] @@ -258,6 +314,7 @@ def _verify_context_receipt(receipt: dict[str, Any]) -> dict[str, Any]: "source_run_id": str(receipt.get("source_run_id") or ""), "source_message_id": str(receipt.get("source_message_id") or ""), "work_item_id": str(receipt.get("work_item_id") or "CIR-P0-TG-001"), + "receipt_kind": receipt_kind, "target": target, "consumer_surface": str(receipt.get("consumer_surface") or ""), "verifier_status": "passed" if not failed_checks else "failed", @@ -265,13 +322,19 @@ def _verify_context_receipt(receipt: dict[str, Any]) -> dict[str, Any]: "failed_checks": failed_checks, "target_selector_verified": not target_selector_missing, "target_selector_missing_fields": target_selector_missing, - "source_of_truth_diff_verified": _source_diff_verified(source_diff, target), + "source_of_truth_diff_verified": source_diff_verified, "check_mode_verified": check_mode.get("enabled") is True and not missing_checks, "check_mode_missing_checks": missing_checks, "rollback_verified": ( rollback.get("required") is True and bool(rollback.get("rollback_ref")) ), "post_apply_verifier_refs": verifier_refs, + "target_write_contract_verified": target_write_contract_verified, + "verified_target_context_write_performed": ( + receipt.get("target_write_performed") is True + and target_write_contract_verified + ), + "verifier_write_performed": False, "metadata_only": receipt.get("metadata_only") is True, "raw_payload_included": receipt.get("raw_payload_included") is True, "runtime_target_write_performed": False, @@ -310,7 +373,36 @@ def _active_blockers( return blockers -def _source_diff_verified(source_diff: dict[str, Any], target: str) -> bool: +def _receipt_kind(receipt: Mapping[str, Any]) -> str: + explicit = str(receipt.get("receipt_kind") or "") + if explicit: + return explicit + receipt_id = str(receipt.get("receipt_id") or "") + target_selector = _dict(receipt.get("target_selector")) + if receipt_id.startswith("telegram_alert_learning_context_fallback::") or ( + target_selector.get("dispatch_receipt_id") + and target_selector.get("consumer_receipt_id") + ): + return _CONSUMER_RECEIPT_KIND + return _ALERT_CARD_RECEIPT_KIND + + +def _source_diff_verified( + source_diff: dict[str, Any], + target: str, + *, + receipt_kind: str, +) -> bool: + if receipt_kind == _CONSUMER_RECEIPT_KIND: + return bool( + source_diff.get("current_state") + == "telegram_alert_registry_unavailable_or_not_ready" + and source_diff.get("desired_state") + == "ai_loop_agent_context_receipt_available" + and source_diff.get("delta_kind") + == f"log_controlled_writeback_{target}_context_fallback" + and source_diff.get("raw_payload_included") is False + ) return bool( source_diff.get("current_state") == "telegram_alert_learning_registry_readable" and source_diff.get("desired_state") @@ -320,6 +412,26 @@ def _source_diff_verified(source_diff: dict[str, Any], target: str) -> bool: ) +def _target_write_contract_verified( + receipt: Mapping[str, Any], + *, + receipt_kind: str, +) -> bool: + if receipt.get("verifier_write_performed") is True: + return False + if receipt_kind != _CONSUMER_RECEIPT_KIND: + return receipt.get("target_write_performed") is False + consumer_write_receipt = _dict(receipt.get("consumer_write_receipt")) + return bool( + receipt.get("target_write_performed") is True + and consumer_write_receipt.get("consumer_receipt_id") + and consumer_write_receipt.get("status") == "success" + and consumer_write_receipt.get("target_context_receipt_write_performed") + is True + and consumer_write_receipt.get("runtime_target_write_performed") is True + ) + + def _dict(value: Any) -> dict[str, Any]: return value if isinstance(value, dict) else {} diff --git a/apps/api/tests/test_ai_agent_log_controlled_writeback_consumer_readback_api.py b/apps/api/tests/test_ai_agent_log_controlled_writeback_consumer_readback_api.py index 044c19d59..7046b2daa 100644 --- a/apps/api/tests/test_ai_agent_log_controlled_writeback_consumer_readback_api.py +++ b/apps/api/tests/test_ai_agent_log_controlled_writeback_consumer_readback_api.py @@ -452,10 +452,57 @@ async def test_log_controlled_writeback_consumer_maps_ready_bindings_when_alert_ receipt["status"] == "ready_for_ai_loop_context" for receipt in context["context_receipts"] ) + assert all( + receipt["receipt_kind"] == "log_controlled_writeback_consumer" + and receipt["target_write_performed"] is True + and receipt["consumer_write_receipt"]["status"] == "success" + and receipt["consumer_write_receipt"][ + "target_context_receipt_write_performed" + ] + is True + and receipt["consumer_write_receipt"]["runtime_target_write_performed"] + is True + and receipt["verifier_write_performed"] is False + for receipt in context["context_receipts"] + ) assert context["operation_boundaries"]["runtime_target_write_performed"] is True assert context["operation_boundaries"]["raw_payload_included"] is False +@pytest.mark.asyncio +async def test_log_controlled_writeback_consumer_fallback_rejects_unverified_write( + monkeypatch, +): + consumer_rows = _consumer_receipt_rows() + consumer_rows[0]["runtime_target_write_performed"] = "false" + fake_db = _FakeDb(_ledger_rows(), consumer_rows) + monkeypatch.setattr( + consumer_module, + "get_db_context", + lambda project_id: _FakeContext(fake_db), + ) + + async def alert_registry_timeout(**_kwargs): + raise TimeoutError("alert registry unavailable") + + monkeypatch.setattr( + consumer_module, + "list_ai_alert_card_delivery_readback", + alert_registry_timeout, + ) + + payload = await load_latest_ai_agent_log_controlled_writeback_consumer_readback() + + assert payload["rollups"]["target_context_receipt_write_count"] == 5 + assert payload["rollups"]["runtime_target_write_performed"] is False + assert payload["operation_boundaries"]["runtime_target_write_performed"] is False + context = payload["telegram_alert_learning_context"] + assert context["status"] == ( + "blocked_telegram_alert_learning_registry_readback_unavailable" + ) + assert context["context_receipt_count"] == 0 + + def test_log_controlled_writeback_consumer_direct_rows_parse_json_text(): dispatch_row = _ledger_rows()[0] consumer_row = _consumer_receipt_rows()[0] @@ -621,6 +668,7 @@ def _assert_consumer_readback(payload: dict): if receipt["target"] == "ai_agent" ][0] assert ai_agent_receipt["status"] == "ready_for_ai_loop_context" + assert ai_agent_receipt["receipt_kind"] == "alert_card_learning_registry" assert ai_agent_receipt["work_item_id"] == "CIR-P0-TG-001" assert ai_agent_receipt["metadata_only"] is True assert ai_agent_receipt["target_write_performed"] is False diff --git a/apps/api/tests/test_telegram_alert_learning_context_post_apply_verifier_api.py b/apps/api/tests/test_telegram_alert_learning_context_post_apply_verifier_api.py index 326c69540..816108adb 100644 --- a/apps/api/tests/test_telegram_alert_learning_context_post_apply_verifier_api.py +++ b/apps/api/tests/test_telegram_alert_learning_context_post_apply_verifier_api.py @@ -80,6 +80,92 @@ def _context_receipts() -> list[dict]: ] +def _fallback_context_receipts() -> list[dict]: + targets = [ + ("km", "km_consumer_context"), + ("rag", "rag_consumer_context"), + ("playbook", "playbook_consumer_context"), + ("mcp", "mcp_consumer_context"), + ("verifier", "verifier_consumer_context"), + ("ai_agent", "ai_agent_consumer_context"), + ] + return [ + { + "receipt_kind": "log_controlled_writeback_consumer", + "receipt_id": ( + "telegram_alert_learning_context_fallback::" + f"log_controlled_writeback_consumed::{target}" + ), + "source_receipt_id": f"log_controlled_writeback_consumed::{target}", + "source_run_id": "", + "source_message_id": "", + "work_item_id": "CIR-P0-TG-001", + "target": target, + "consumer_surface": surface, + "source_ref": f"log_controlled_writeback_dispatched::{target}", + "ai_agent_context_ref": ( + "ai-agent://awoooi/log-controlled-writeback/" + f"log_controlled_writeback_consumed::{target}" + ), + "status": "ready_for_ai_loop_context", + "metadata_only": True, + "target_write_performed": True, + "consumer_write_receipt": { + "consumer_receipt_id": ( + f"log_controlled_writeback_consumed::{target}" + ), + "status": "success", + "target_context_receipt_write_performed": True, + "runtime_target_write_performed": True, + }, + "verifier_write_performed": False, + "raw_payload_included": False, + "target_selector": { + "project_id": "awoooi", + "dispatch_receipt_id": ( + f"log_controlled_writeback_dispatched::{target}" + ), + "consumer_receipt_id": ( + f"log_controlled_writeback_consumed::{target}" + ), + "target": target, + "source_ref": f"log_controlled_writeback_dispatched::{target}", + }, + "source_of_truth_diff": { + "current_state": ( + "telegram_alert_registry_unavailable_or_not_ready" + ), + "desired_state": "ai_loop_agent_context_receipt_available", + "delta_kind": ( + f"log_controlled_writeback_{target}_context_fallback" + ), + "raw_payload_included": False, + }, + "check_mode": { + "enabled": True, + "checks": [ + "consumer_binding_ready", + "consumer_apply_receipt_present", + "metadata_only_raw_payload_absent", + "post_apply_verifier_refs_present", + ], + }, + "rollback": { + "required": True, + "rollback_ref": ( + "rollback://telegram-alert-learning-context-fallback/" + f"log_controlled_writeback_consumed::{target}" + ), + }, + "post_apply_verifier": { + "required": True, + "verifier_refs": [f"post-write-verifier://{target}"], + }, + } + for target, surface in targets + ] + + def _consumer_readback(*, receipts: list[dict] | None = None) -> dict: return { "schema_version": "ai_agent_log_controlled_writeback_consumer_readback_v1", @@ -123,6 +209,7 @@ def test_telegram_alert_learning_context_post_apply_verifier_passes_receipts(): assert rollups["verified_context_receipt_count"] == 6 assert rollups["verified_target_count"] == 6 assert rollups["verified_ai_agent_context_receipt_count"] == 1 + assert rollups["verified_target_context_write_count"] == 0 assert rollups["raw_payload_included_count"] == 0 assert rollups["runtime_write_performed_count"] == 0 @@ -141,6 +228,48 @@ def test_telegram_alert_learning_context_post_apply_verifier_passes_receipts(): assert "raw Telegram payload" not in serialized +def test_telegram_alert_learning_context_post_apply_verifier_passes_fallback_receipts(): + payload = build_telegram_alert_learning_context_post_apply_verifier( + consumer_readback=_consumer_readback(receipts=_fallback_context_receipts()) + ) + + assert payload["status"] == ( + "telegram_alert_learning_context_post_apply_verified" + ) + assert payload["active_blockers"] == [] + assert payload["rollups"]["verified_context_receipt_count"] == 6 + assert payload["rollups"]["verified_target_context_write_count"] == 6 + assert { + result["receipt_kind"] for result in payload["verifier_results"] + } == {"log_controlled_writeback_consumer"} + assert all( + result["target_write_contract_verified"] is True + and result["verified_target_context_write_performed"] is True + and result["verifier_write_performed"] is False + for result in payload["verifier_results"] + ) + + +def test_telegram_alert_learning_context_post_apply_verifier_blocks_bad_fallback_receipt(): + receipts = _fallback_context_receipts() + receipts[0]["consumer_write_receipt"]["runtime_target_write_performed"] = False + + payload = build_telegram_alert_learning_context_post_apply_verifier( + consumer_readback=_consumer_readback(receipts=receipts) + ) + + assert payload["status"] == ( + "blocked_telegram_alert_learning_context_post_apply_verifier" + ) + assert payload["rollups"]["failed_context_receipt_count"] == 1 + failed = [ + result + for result in payload["verifier_results"] + if result["verifier_status"] == "failed" + ] + assert failed[0]["failed_checks"] == ["consumer_write_receipt_verified"] + + def test_telegram_alert_learning_context_post_apply_verifier_blocks_bad_receipt(): receipts = _context_receipts() receipts[0]["rollback"] = {"required": True, "rollback_ref": ""} @@ -159,6 +288,25 @@ def test_telegram_alert_learning_context_post_apply_verifier_blocks_bad_receipt( ) +def test_telegram_alert_learning_context_post_apply_verifier_rejects_unknown_kind(): + receipts = _context_receipts() + receipts[0]["receipt_kind"] = "unsupported_learning_receipt" + + payload = build_telegram_alert_learning_context_post_apply_verifier( + consumer_readback=_consumer_readback(receipts=receipts) + ) + + failed = [ + result + for result in payload["verifier_results"] + if result["verifier_status"] == "failed" + ] + assert payload["status"] == ( + "blocked_telegram_alert_learning_context_post_apply_verifier" + ) + assert failed[0]["failed_checks"] == ["receipt_kind_supported"] + + def test_telegram_alert_learning_context_post_apply_verifier_endpoint(monkeypatch): async def fake_loader(): return build_telegram_alert_learning_context_post_apply_verifier( diff --git a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json index 2328241b2..e1338d30f 100644 --- a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json +++ b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json @@ -640,6 +640,8 @@ "apps/api/src/services/agent99_controlled_dispatch_ledger.py", "apps/api/src/services/agent99_telegram_lifecycle.py", "apps/api/src/services/agent99_public_receipts.py", + "apps/api/src/services/ai_agent_log_controlled_writeback_consumer_readback.py", + "apps/api/src/services/telegram_alert_learning_context_post_apply_verifier.py", "apps/api/src/repositories/knowledge_repository.py" ], "executor": "backup_restore_break_glass", @@ -822,12 +824,14 @@ "confirmed_truth": [ "source and focused tests bind typed verifier outcomes to same-run Telegram, KM, RAG, MCP, PlayBook and backup DR scorecard acknowledgements", "Backup/restore recipient-visible recovered cards name Agent99 BackupCheck as the read-only executor and backup_restore_readback_verifier as the typed independent verifier; apply remains not_applicable and the final durable closure receipt is required", - "BackupCheck learning assets are reconciled idempotently as read-only DR evidence rather than controlled repair" + "BackupCheck learning assets are reconciled idempotently as read-only DR evidence rather than controlled repair", + "the Telegram learning verifier now distinguishes alert-card registry receipts from log-controlled consumer receipts, verifies the latter only with an exact successful consumer write receipt, and keeps verifier runtime writes false" ], "runtime_gaps": [ - "no exact deployed source, production same-run receipt or recipient-visible Telegram readback is recorded for this revision" + "production exposed six fallback context receipts but the deployed verifier rejected all six under the alert-card-only contract; this source revision fixes that mismatch but still has no production same-run receipt", + "release c0afb10861cbb76bc232fad3bbeea0d675cd9b80 reached Gitea CD 5384 terminal failure without a deploy marker, so production remained on e614f061f781c6e20a3964875d7ab3c28cf91190" ], - "next_action": "deploy one exact source revision and read back one same-run Telegram/KM/RAG/MCP/PlayBook/DR closure receipt; source evidence must not be counted as runtime closure" + "next_action": "after the separately owned CI/CD blocker is repaired, integrate this exact tested receipt-contract revision once and read back one same-run Telegram/KM/RAG/MCP/PlayBook/DR closure receipt; source evidence must not be counted as runtime closure" }, { "id": "AIA-SRE-016", From 56e36c472c8f8524c4dfb1ed017d3f4275f67d80 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 05:56:17 +0800 Subject: [PATCH 04/27] fix(sre): require verified learning closure --- ...gram_alert_monitoring_coverage_readback.py | 178 ++++++++++++------ ..._alert_monitoring_coverage_readback_api.py | 151 ++++++++++++--- ...rolled-automation-work-items.snapshot.json | 6 +- 3 files changed, 246 insertions(+), 89 deletions(-) diff --git a/apps/api/src/services/telegram_alert_monitoring_coverage_readback.py b/apps/api/src/services/telegram_alert_monitoring_coverage_readback.py index ca51bc934..bdbcc8258 100644 --- a/apps/api/src/services/telegram_alert_monitoring_coverage_readback.py +++ b/apps/api/src/services/telegram_alert_monitoring_coverage_readback.py @@ -69,6 +69,15 @@ _ALERT_LOG_EVENT_TYPES = ( "EXECUTION_STARTED", "EXECUTION_COMPLETED", ) +_AI_LOOP_LEARNING_TARGETS = ( + "km", + "rag", + "playbook", + "mcp", + "verifier", + "ai_agent", +) +_AI_LOOP_LEARNING_TARGET_COUNT = len(_AI_LOOP_LEARNING_TARGETS) _RUNTIME_LIFECYCLE_SQL = """ WITH run_lifecycle AS ( SELECT @@ -582,7 +591,9 @@ async def _load_post_apply_verifier_readback(*, project_id: str) -> dict[str, An "rollups": { "telegram_alert_learning_context_post_apply_verifier_ready": False, "verified_context_receipt_count": 0, + "target_count": 0, "verified_target_count": 0, + "verified_target_context_write_count": 0, "verified_ai_agent_context_receipt_count": 0, "error_type": type(exc).__name__, }, @@ -937,13 +948,10 @@ def build_telegram_alert_monitoring_coverage_readback( effective_ai_alert_total = max(ai_alert_total, _int(ai_loop_context["receipt_count"])) effective_ai_alert_ready_total = max( ai_alert_ready_total, - _int(ai_loop_context["receipt_count"]), + _int(ai_loop_context["receipt_count"]) + if ai_loop_context["ready"] + else 0, ) - effective_ai_alert_summary = { - **ai_alert_summary, - "total": effective_ai_alert_total, - "learning_writeback_ready_total": effective_ai_alert_ready_total, - } runtime_event_type_counts = { str(key): _int(value) for key, value in _dict( @@ -960,7 +968,7 @@ def build_telegram_alert_monitoring_coverage_readback( readback_summary=readback_summary, matrix_summary=matrix_summary, verifier_rollups=verifier_rollups, - ai_alert_summary=effective_ai_alert_summary, + ai_alert_summary=ai_alert_summary, runtime_summary=runtime_summary, source_contract=source_contract, runtime_db_ok=runtime_db_ok, @@ -973,10 +981,11 @@ def build_telegram_alert_monitoring_coverage_readback( direct_gap_count=direct_gap_count, runtime_db_ok=runtime_db_ok, ai_alert_db_ok=ai_alert_db_ok, - verifier_ready=effective_verifier_ready, + post_apply_verifier_ready=verifier_ready, + durable_learning_writeback_ready=effective_verifier_ready, source_contract=source_contract, - ai_alert_total=effective_ai_alert_total, - ai_alert_ready_total=effective_ai_alert_ready_total, + ai_alert_total=ai_alert_total, + ai_alert_ready_total=ai_alert_ready_total, matrix_db_receipt_gap_count=matrix_db_receipt_gap_count, matrix_ai_route_gap_count=matrix_ai_route_gap_count, matrix_controlled_queue_gap_count=matrix_controlled_queue_gap_count, @@ -1059,6 +1068,7 @@ def build_telegram_alert_monitoring_coverage_readback( effective_verifier_ready ), "km_rag_mcp_playbook_ai_agent_context_ready": effective_verifier_ready, + "durable_learning_writeback_verified": effective_verifier_ready, "km_rag_mcp_playbook_ai_agent_context_source": str( ai_loop_context["source"] ), @@ -1183,10 +1193,14 @@ def build_telegram_alert_monitoring_coverage_readback( verifier_rollups.get("verified_context_receipt_count") ), "verified_target_count": _int(verifier_rollups.get("verified_target_count")), + "verified_target_context_write_count": _int( + ai_loop_context.get("verified_target_context_write_count") + ), "verified_ai_agent_context_receipt_count": _int( verifier_rollups.get("verified_ai_agent_context_receipt_count") ), "ai_loop_context_ready": effective_verifier_ready, + "durable_learning_writeback_ready": effective_verifier_ready, "ai_loop_context_source": str(ai_loop_context["source"]), "ai_loop_context_fallback_used": bool(ai_loop_context["fallback_used"]), "ai_loop_context_receipt_count": _int(ai_loop_context["receipt_count"]), @@ -1337,6 +1351,13 @@ def _effective_ai_loop_context( consumer_context: Mapping[str, Any], ) -> dict[str, Any]: verifier_receipt_count = _int(verifier_rollups.get("verified_context_receipt_count")) + verifier_target_count = _int(verifier_rollups.get("target_count")) + verifier_verified_target_count = _int( + verifier_rollups.get("verified_target_count") + ) + verifier_target_write_count = _int( + verifier_rollups.get("verified_target_context_write_count") + ) verifier_ai_agent_count = _int( verifier_rollups.get("verified_ai_agent_context_receipt_count") ) @@ -1368,61 +1389,56 @@ def _effective_ai_loop_context( and consumer_target_count > 0 and consumer_ready_target_count >= consumer_target_count ) + required_target_count = _AI_LOOP_LEARNING_TARGET_COUNT + durable_verifier_ready = bool( + verifier_ready + and verifier_target_count == required_target_count + and verifier_verified_target_count == required_target_count + and verifier_target_write_count == required_target_count + and verifier_ai_agent_count > 0 + ) - if verifier_ready: + if durable_verifier_ready: return { "ready": True, "source": "post_apply_verifier", "fallback_used": False, - "receipt_count": max( - verifier_receipt_count, - ai_alert_ready_total, - consumer_receipt_count, - ), - "ai_agent_context_receipt_count": max( - verifier_ai_agent_count, - consumer_ai_agent_count, - ), - "ready_target_count": max( - _int(verifier_rollups.get("verified_target_count")), - consumer_ready_target_count, - ), - "target_count": max(consumer_target_count, 0), + "receipt_count": verifier_receipt_count, + "ai_agent_context_receipt_count": verifier_ai_agent_count, + "ready_target_count": verifier_verified_target_count, + "target_count": required_target_count, + "verified_target_context_write_count": verifier_target_write_count, + "durable_learning_writeback_ready": True, } - if consumer_targets_ready: - return { - "ready": True, - "source": "log_controlled_writeback_consumer", - "fallback_used": True, - "receipt_count": max(consumer_receipt_count, ai_alert_ready_total), - "ai_agent_context_receipt_count": consumer_ai_agent_count, - "ready_target_count": consumer_ready_target_count, - "target_count": consumer_target_count, - } - - if ai_alert_total > 0 and ai_alert_ready_total >= ai_alert_total: - return { - "ready": True, - "source": "ai_alert_card_learning_writeback_refs", - "fallback_used": True, - "receipt_count": ai_alert_ready_total, - "ai_agent_context_receipt_count": max(consumer_ai_agent_count, 1), - "ready_target_count": max(consumer_ready_target_count, 1), - "target_count": max(consumer_target_count, 1), - } + source = "unavailable" + if verifier_ready: + source = "post_apply_verifier_without_durable_target_writes" + elif consumer_targets_ready: + source = "unverified_log_controlled_writeback_consumer" + elif ai_alert_total > 0 and ai_alert_ready_total >= ai_alert_total: + source = "unverified_ai_alert_card_learning_writeback_refs" return { "ready": False, - "source": "unavailable", + "source": source, "fallback_used": False, - "receipt_count": max(verifier_receipt_count, ai_alert_ready_total), + "receipt_count": max( + verifier_receipt_count, + consumer_receipt_count, + ai_alert_ready_total, + ), "ai_agent_context_receipt_count": max( verifier_ai_agent_count, consumer_ai_agent_count, ), - "ready_target_count": consumer_ready_target_count, - "target_count": consumer_target_count, + "ready_target_count": max( + verifier_verified_target_count, + consumer_ready_target_count, + ), + "target_count": required_target_count, + "verified_target_context_write_count": verifier_target_write_count, + "durable_learning_writeback_ready": False, "ai_alert_total": ai_alert_total, } @@ -2322,6 +2338,42 @@ def _gap_execution_contract(blocker: str) -> dict[str, Any]: "destructive_action": False, }, } + if blocker == "telegram_alert_ai_loop_post_apply_verifier_not_ready": + return { + "status": "ready_ai_read_only_retry", + "risk_level": "low", + "execution_mode": "automatic_bounded_verifier_readback", + "executor_route": "telegram_alert_learning_context_post_apply_verifier", + "ai_controlled_execution_allowed": True, + "owner_review_required_for_low_medium_high": False, + "critical_break_glass_required": False, + "operation_boundaries": { + "database_read_only": True, + "requires_secret": False, + "requires_runtime_send": False, + "runtime_write_performed": False, + "stores_raw_payload": False, + "destructive_action": False, + }, + } + if blocker == "telegram_alert_ai_loop_durable_learning_writeback_not_verified": + return { + "status": "ready_ai_controlled_metadata_writeback", + "risk_level": "medium", + "execution_mode": "controlled_metadata_writeback_then_verify", + "executor_route": "ai_agent_metadata_writeback_executor", + "ai_controlled_execution_allowed": True, + "owner_review_required_for_low_medium_high": False, + "critical_break_glass_required": False, + "operation_boundaries": { + "database_read_only": False, + "requires_secret": False, + "requires_runtime_send": False, + "runtime_write_performed": False, + "stores_raw_payload": False, + "destructive_action": False, + }, + } if blocker == "runtime_ai_automation_trigger_receipt_missing" or blocker.startswith( "runtime_ai_automation_lifecycle_open" ): @@ -2384,7 +2436,6 @@ def _coverage_matrix( ai_alert_ready_total = _int(ai_alert_summary.get("learning_writeback_ready_total")) ai_loop_source = str(ai_loop_context.get("source") or "unavailable") ai_loop_receipt_count = _int(ai_loop_context.get("receipt_count")) - ai_loop_fallback_used = bool(ai_loop_context.get("fallback_used")) return [ { "surface_id": "monitoring_inventory_static_scope", @@ -2486,9 +2537,7 @@ def _coverage_matrix( { "surface_id": "telegram_alert_ai_loop_context_verifier", "status": ( - "consumer_context_fallback_ready" - if ai_loop_fallback_used and verifier_ready - else "verified_context_ready" + "verified_durable_context_ready" if verifier_ready else "verifier_not_ready" ), @@ -2496,10 +2545,12 @@ def _coverage_matrix( "db_or_log_receipt": "ready_context_receipt_source", "ai_route": "ready_ai_agent_reusable_context", "controlled_queue": "ready_metadata_only_post_apply_verifier", - "post_verifier": "ready" - if not ai_loop_fallback_used - else "consumer_context_fallback", - "learning_writeback": "ready_km_rag_mcp_playbook_ai_agent_targets", + "post_verifier": "ready" if verifier_ready else "waiting_independent_verifier", + "learning_writeback": ( + "ready_verified_km_rag_mcp_playbook_ai_agent_target_writes" + if verifier_ready + else "waiting_durable_target_write_receipts" + ), "gap_count": 0 if verifier_ready else 1, "evidence_refs": [ "/api/v1/agents/telegram-alert-learning-context-post-apply-verifier", @@ -2537,7 +2588,8 @@ def _active_blockers( direct_gap_count: int, runtime_db_ok: bool, ai_alert_db_ok: bool, - verifier_ready: bool, + post_apply_verifier_ready: bool, + durable_learning_writeback_ready: bool, source_contract: Mapping[str, Any], ai_alert_total: int, ai_alert_ready_total: int, @@ -2582,8 +2634,12 @@ def _active_blockers( "runtime_ai_automation_lifecycle_open:" f"{max(runtime_open_run_count, 1)}" ) - if not verifier_ready: + if not post_apply_verifier_ready: blockers.append("telegram_alert_ai_loop_post_apply_verifier_not_ready") + elif not durable_learning_writeback_ready: + blockers.append( + "telegram_alert_ai_loop_durable_learning_writeback_not_verified" + ) blockers.extend(_source_contract_blockers(source_contract)) return _unique(blockers) @@ -2835,6 +2891,8 @@ def _gap_next_action(blocker: str) -> str: return "populate_km_rag_mcp_playbook_ai_agent_learning_refs" if blocker == "telegram_alert_ai_loop_post_apply_verifier_not_ready": return "run_ai_loop_context_post_apply_verifier_readback" + if blocker == "telegram_alert_ai_loop_durable_learning_writeback_not_verified": + return "consume_target_context_receipts_then_run_independent_verifier" if blocker.startswith("source_contract_missing"): return "restore_operator_visible_source_contract_marker" return "continue_ai_controlled_gap_closure" diff --git a/apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py b/apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py index d4bfdd6a0..3b1c79320 100644 --- a/apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py +++ b/apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py @@ -46,7 +46,9 @@ def _payload( "rollups": { "telegram_alert_learning_context_post_apply_verifier_ready": True, "verified_context_receipt_count": 36, + "target_count": 6, "verified_target_count": 6, + "verified_target_context_write_count": 6, "verified_ai_agent_context_receipt_count": 6, }, } @@ -367,7 +369,7 @@ def test_telegram_alert_monitoring_coverage_readback_surfaces_live_gaps(): matrix = {item["surface_id"]: item for item in payload["coverage_matrix"]} assert matrix["telegram_gateway_outbound_mirror"]["gap_count"] == 0 assert matrix["telegram_alert_ai_loop_context_verifier"]["status"] == ( - "verified_context_ready" + "verified_durable_context_ready" ) assert matrix["monitoring_inventory_static_scope"]["gap_count"] == 60 assert payload["operation_boundaries"]["telegram_send_performed"] is False @@ -563,7 +565,7 @@ def test_telegram_alert_monitoring_coverage_blocks_partial_surface_contracts(): ] -def test_telegram_alert_monitoring_coverage_uses_consumer_context_fallback(): +def test_telegram_alert_monitoring_coverage_rejects_unverified_consumer_context(): payload = _payload( post_apply_verifier={ "schema_version": "telegram_alert_learning_context_post_apply_verifier_v1", @@ -571,7 +573,9 @@ def test_telegram_alert_monitoring_coverage_uses_consumer_context_fallback(): "rollups": { "telegram_alert_learning_context_post_apply_verifier_ready": False, "verified_context_receipt_count": 0, + "target_count": 6, "verified_target_count": 0, + "verified_target_context_write_count": 0, "verified_ai_agent_context_receipt_count": 0, }, }, @@ -588,61 +592,64 @@ def test_telegram_alert_monitoring_coverage_uses_consumer_context_fallback(): summary = payload["summary"] assert summary["ai_alert_card_db_readback_ok"] is False - assert summary["ai_loop_context_ready"] is True - assert summary["ai_loop_context_source"] == "log_controlled_writeback_consumer" - assert summary["ai_loop_context_fallback_used"] is True + assert summary["ai_loop_context_ready"] is False + assert summary["durable_learning_writeback_ready"] is False + assert summary["ai_loop_context_source"] == ( + "unverified_log_controlled_writeback_consumer" + ) + assert summary["ai_loop_context_fallback_used"] is False assert summary["ai_loop_context_receipt_count"] == 6 assert summary["ai_loop_ai_agent_context_receipt_count"] == 1 - assert summary["km_rag_mcp_playbook_ai_agent_context_ready"] is True + assert summary["km_rag_mcp_playbook_ai_agent_context_ready"] is False assert ( summary["km_rag_mcp_playbook_ai_agent_context_source"] - == "log_controlled_writeback_consumer" + == "unverified_log_controlled_writeback_consumer" ) assert summary["effective_ai_alert_context_receipt_total"] == 6 - assert summary["effective_ai_alert_context_ready_total"] == 6 + assert summary["effective_ai_alert_context_ready_total"] == 0 assert summary["log_controlled_consumer_readback_ready"] is True - assert summary["telegram_monitoring_audit_completion_percent"] == 81.8 + assert summary["telegram_monitoring_audit_completion_percent"] < 100.0 assert ( payload["operator_answer"][ "all_verified_ai_alert_context_receipts_reusable_by_ai_agent" ] - is True + is False ) assert ( payload["operator_answer"]["km_rag_mcp_playbook_ai_agent_context_ready"] - is True + is False ) assert ( payload["operator_answer"]["km_rag_mcp_playbook_ai_agent_context_source"] - == "log_controlled_writeback_consumer" + == "unverified_log_controlled_writeback_consumer" ) assert ( "awooop_ai_alert_card_delivery_db_readback_unavailable" in payload["active_blockers"] ) - assert "awooop_ai_alert_card_delivery_receipts_missing" not in payload[ + assert "awooop_ai_alert_card_delivery_receipts_missing" in payload[ "active_blockers" ] - assert "awooop_ai_alert_card_learning_writeback_refs_missing" not in payload[ + assert "awooop_ai_alert_card_learning_writeback_refs_missing" in payload[ "active_blockers" ] - assert "telegram_alert_ai_loop_post_apply_verifier_not_ready" not in payload[ + assert "telegram_alert_ai_loop_post_apply_verifier_not_ready" in payload[ "active_blockers" ] pipeline = {stage["stage_id"]: stage for stage in payload["alert_receipt_pipeline"]} - assert pipeline["km_rag_mcp_playbook_ai_agent_learning_refs"]["ready"] is True + assert pipeline["km_rag_mcp_playbook_ai_agent_learning_refs"]["ready"] is False matrix = {item["surface_id"]: item for item in payload["coverage_matrix"]} assert matrix["telegram_alert_ai_loop_context_verifier"]["status"] == ( - "consumer_context_fallback_ready" + "verifier_not_ready" ) assert matrix["telegram_alert_ai_loop_context_verifier"][ "ai_loop_context_source" - ] == "log_controlled_writeback_consumer" + ] == "unverified_log_controlled_writeback_consumer" -def test_telegram_alert_monitoring_coverage_uses_ai_alert_card_learning_refs_fallback(): +def test_telegram_alert_monitoring_coverage_rejects_unverified_ai_alert_refs(): payload = _payload( post_apply_verifier={ "schema_version": "telegram_alert_learning_context_post_apply_verifier_v1", @@ -650,7 +657,9 @@ def test_telegram_alert_monitoring_coverage_uses_ai_alert_card_learning_refs_fal "rollups": { "telegram_alert_learning_context_post_apply_verifier_ready": False, "verified_context_receipt_count": 0, + "target_count": 0, "verified_target_count": 0, + "verified_target_context_write_count": 0, "verified_ai_agent_context_receipt_count": 0, }, }, @@ -680,20 +689,108 @@ def test_telegram_alert_monitoring_coverage_uses_ai_alert_card_learning_refs_fal ) summary = payload["summary"] - assert payload["status"] == "telegram_alert_monitoring_coverage_ready" - assert summary["ai_loop_context_ready"] is True - assert summary["ai_loop_context_source"] == "ai_alert_card_learning_writeback_refs" - assert summary["ai_loop_context_fallback_used"] is True + assert payload["status"] == ( + "blocked_telegram_alert_monitoring_coverage_gaps_present" + ) + assert summary["ai_loop_context_ready"] is False + assert summary["ai_loop_context_source"] == ( + "unverified_ai_alert_card_learning_writeback_refs" + ) + assert summary["ai_loop_context_fallback_used"] is False assert summary["effective_ai_alert_context_receipt_total"] == 444 assert summary["effective_ai_alert_context_ready_total"] == 444 - assert summary["telegram_monitoring_audit_completion_percent"] == 100.0 + assert summary["telegram_monitoring_audit_completion_percent"] < 100.0 + assert "telegram_alert_ai_loop_post_apply_verifier_not_ready" in payload[ + "active_blockers" + ] + assert ( + payload["operator_answer"]["km_rag_mcp_playbook_ai_agent_context_source"] + == "unverified_ai_alert_card_learning_writeback_refs" + ) + + +def test_telegram_alert_monitoring_coverage_requires_durable_target_writes(): + payload = _payload( + post_apply_verifier={ + "schema_version": ( + "telegram_alert_learning_context_post_apply_verifier_v1" + ), + "status": "telegram_alert_learning_context_post_apply_verified", + "rollups": { + "telegram_alert_learning_context_post_apply_verifier_ready": True, + "verified_context_receipt_count": 6, + "target_count": 6, + "verified_target_count": 6, + "verified_target_context_write_count": 0, + "verified_ai_agent_context_receipt_count": 1, + }, + }, + monitoring_live_receipt_apply_readback=( + _metadata_live_receipt_apply_readback() + ), + ) + + summary = payload["summary"] + assert payload["status"] == ( + "blocked_telegram_alert_monitoring_coverage_gaps_present" + ) + assert summary["ai_loop_context_ready"] is False + assert summary["durable_learning_writeback_ready"] is False + assert summary["verified_target_context_write_count"] == 0 + assert summary["ai_loop_context_source"] == ( + "post_apply_verifier_without_durable_target_writes" + ) + assert ( + "telegram_alert_ai_loop_durable_learning_writeback_not_verified" + in payload["active_blockers"] + ) assert "telegram_alert_ai_loop_post_apply_verifier_not_ready" not in payload[ "active_blockers" ] - assert payload["active_blockers"] == [] + queue = {item["blocker"]: item for item in payload["ai_controlled_gap_queue"]} + candidate = queue[ + "telegram_alert_ai_loop_durable_learning_writeback_not_verified" + ] + assert candidate["risk_level"] == "medium" + assert candidate["ai_controlled_execution_allowed"] is True + assert candidate["critical_break_glass_required"] is False + assert candidate["executor_route"] == "ai_agent_metadata_writeback_executor" + + +@pytest.mark.parametrize( + ("target_count", "verified_target_count", "verified_write_count"), + ((5, 5, 5), (6, 6, 7)), +) +def test_telegram_alert_monitoring_coverage_requires_exact_six_target_writes( + target_count: int, + verified_target_count: int, + verified_write_count: int, +): + payload = _payload( + post_apply_verifier={ + "schema_version": ( + "telegram_alert_learning_context_post_apply_verifier_v1" + ), + "status": "telegram_alert_learning_context_post_apply_verified", + "rollups": { + "telegram_alert_learning_context_post_apply_verifier_ready": True, + "verified_context_receipt_count": verified_target_count, + "target_count": target_count, + "verified_target_count": verified_target_count, + "verified_target_context_write_count": verified_write_count, + "verified_ai_agent_context_receipt_count": 1, + }, + }, + monitoring_live_receipt_apply_readback=( + _metadata_live_receipt_apply_readback() + ), + ) + + assert payload["summary"]["durable_learning_writeback_ready"] is False + assert payload["operator_answer"]["durable_learning_writeback_verified"] is False assert ( - payload["operator_answer"]["km_rag_mcp_playbook_ai_agent_context_source"] - == "ai_alert_card_learning_writeback_refs" + "telegram_alert_ai_loop_durable_learning_writeback_not_verified" + in payload["active_blockers"] ) diff --git a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json index e1338d30f..d01997a11 100644 --- a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json +++ b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json @@ -642,6 +642,7 @@ "apps/api/src/services/agent99_public_receipts.py", "apps/api/src/services/ai_agent_log_controlled_writeback_consumer_readback.py", "apps/api/src/services/telegram_alert_learning_context_post_apply_verifier.py", + "apps/api/src/services/telegram_alert_monitoring_coverage_readback.py", "apps/api/src/repositories/knowledge_repository.py" ], "executor": "backup_restore_break_glass", @@ -825,10 +826,11 @@ "source and focused tests bind typed verifier outcomes to same-run Telegram, KM, RAG, MCP, PlayBook and backup DR scorecard acknowledgements", "Backup/restore recipient-visible recovered cards name Agent99 BackupCheck as the read-only executor and backup_restore_readback_verifier as the typed independent verifier; apply remains not_applicable and the final durable closure receipt is required", "BackupCheck learning assets are reconciled idempotently as read-only DR evidence rather than controlled repair", - "the Telegram learning verifier now distinguishes alert-card registry receipts from log-controlled consumer receipts, verifies the latter only with an exact successful consumer write receipt, and keeps verifier runtime writes false" + "the Telegram learning verifier now distinguishes alert-card registry receipts from log-controlled consumer receipts, verifies the latter only with an exact successful consumer write receipt, and keeps verifier runtime writes false", + "the monitoring coverage readback no longer lets consumer candidates or alert-card metadata refs bypass the independent verifier; learning readiness requires every declared target write plus an AI Agent context receipt to be verified" ], "runtime_gaps": [ - "production exposed six fallback context receipts but the deployed verifier rejected all six under the alert-card-only contract; this source revision fixes that mismatch but still has no production same-run receipt", + "production exposed six fallback context receipts but the deployed verifier rejected all six under the alert-card-only contract, while the deployed coverage could still treat unverified fallback metadata as ready; this source revision fixes both false-negative and false-green paths but still has no production same-run receipt", "release c0afb10861cbb76bc232fad3bbeea0d675cd9b80 reached Gitea CD 5384 terminal failure without a deploy marker, so production remained on e614f061f781c6e20a3964875d7ab3c28cf91190" ], "next_action": "after the separately owned CI/CD blocker is repaired, integrate this exact tested receipt-contract revision once and read back one same-run Telegram/KM/RAG/MCP/PlayBook/DR closure receipt; source evidence must not be counted as runtime closure" From 70112d7f8429e833408c73e2bd6b92a553d593d5 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 06:08:41 +0800 Subject: [PATCH 05/27] feat(awooop): surface durable learning proof --- apps/web/messages/en.json | 30 +++ apps/web/messages/zh-TW.json | 30 +++ .../src/app/[locale]/awooop/alerts/page.tsx | 245 +++++++++++++++--- 3 files changed, 275 insertions(+), 30 deletions(-) diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index 539ad8ba5..ad6b8cff2 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -12557,6 +12557,36 @@ "queueDetail": "No open gap", "completion": "Audit Complete", "completionDetail": "Telegram monitoring audit" + }, + "learningProof": { + "title": "Learning closure evidence", + "subtitle": "KM / RAG / MCP / PlayBook / Verifier / AI Agent closes only after the Independent Verifier confirms exactly six durable target writes.", + "source": "Evidence source", + "blocker": "Current blocker", + "states": { + "verified": "6/6 durable verified", + "candidate": "Candidate present, not verified", + "blocked": "No verifiable receipt" + }, + "metrics": { + "durableWrites": "Durable writes", + "verifiedTargets": "Verified targets", + "aiAgentReceipts": "AI Agent receipts" + }, + "sources": { + "verified": "Independent Verifier", + "metadataOnly": "Verifier has metadata only; durable writes are missing", + "consumerCandidate": "Consumer receipt candidate, not independently verified", + "alertCardCandidate": "Alert-card learning refs candidate, not independently verified", + "unavailable": "No evidence source available" + }, + "blockers": { + "loading": "Reading Independent Verifier and durable write receipts.", + "unavailable": "Coverage readback is unavailable; this must not be treated as no blocker.", + "verifier": "Independent Verifier has not passed; candidate receipts cannot replace verification.", + "durableWrites": "Exactly 6/6 durable target-write receipts have not been verified.", + "none": "No learning-closure blocker." + } } } }, diff --git a/apps/web/messages/zh-TW.json b/apps/web/messages/zh-TW.json index 60818f432..12efd4ec4 100644 --- a/apps/web/messages/zh-TW.json +++ b/apps/web/messages/zh-TW.json @@ -12557,6 +12557,36 @@ "queueDetail": "沒有待補 gap", "completion": "盤點完成", "completionDetail": "Telegram monitoring audit" + }, + "learningProof": { + "title": "Learning closure 證據", + "subtitle": "只有 Independent Verifier 精確確認六個 durable target writes，才算 KM / RAG / MCP / PlayBook / Verifier / AI Agent 閉環。", + "source": "證據來源", + "blocker": "目前卡點", + "states": { + "verified": "6/6 durable verified", + "candidate": "候選存在，尚未驗證", + "blocked": "尚無可驗證 receipt" + }, + "metrics": { + "durableWrites": "Durable writes", + "verifiedTargets": "已驗證目標", + "aiAgentReceipts": "AI Agent receipts" + }, + "sources": { + "verified": "Independent Verifier", + "metadataOnly": "Verifier 僅有 metadata，尚無 durable writes", + "consumerCandidate": "Consumer receipt 候選，尚未經獨立驗證", + "alertCardCandidate": "Alert-card learning refs 候選，尚未經獨立驗證", + "unavailable": "尚無可用證據來源" + }, + "blockers": { + "loading": "正在讀取 Independent Verifier 與 durable write receipts。", + "unavailable": "Coverage readback 無法取得；不得視為沒有卡點。", + "verifier": "Independent Verifier 尚未通過；候選 receipt 不可替代驗證。", + "durableWrites": "尚未取得精確 6/6 durable target-write receipts。", + "none": "沒有 learning closure 卡點。" + } } } }, diff --git a/apps/web/src/app/[locale]/awooop/alerts/page.tsx b/apps/web/src/app/[locale]/awooop/alerts/page.tsx index 77e066371..f2c97b892 100644 --- a/apps/web/src/app/[locale]/awooop/alerts/page.tsx +++ b/apps/web/src/app/[locale]/awooop/alerts/page.tsx @@ -26,10 +26,13 @@ type TelegramMonitoringCoveragePayload = { active_blockers?: string[] | null; operator_answer?: { all_telegram_monitoring_alerts_fully_audited?: boolean | null; - all_telegram_monitoring_alert_surfaces_have_db_or_log_receipt?: boolean | null; + all_telegram_monitoring_alert_surfaces_have_db_or_log_receipt?: + | boolean + | null; all_telegram_monitoring_alerts_ai_agent_automation_ready?: boolean | null; all_known_telegram_egress_routes_controlled?: boolean | null; km_rag_mcp_playbook_ai_agent_context_ready?: boolean | null; + durable_learning_writeback_verified?: boolean | null; manual_default_terminal_state_allowed?: boolean | null; critical_break_glass_exception_only?: boolean | null; } | null; @@ -43,6 +46,12 @@ type TelegramMonitoringCoveragePayload = { runtime_km_converted_event_count_7d?: number | null; effective_ai_alert_context_receipt_total?: number | null; effective_ai_alert_context_ready_total?: number | null; + verified_target_count?: number | null; + verified_target_context_write_count?: number | null; + verified_ai_agent_context_receipt_count?: number | null; + ai_loop_context_receipt_count?: number | null; + ai_loop_context_source?: string | null; + durable_learning_writeback_ready?: boolean | null; telegram_monitoring_audit_completion_percent?: number | null; ai_controlled_gap_queue_count?: number | null; full_coverage_ready?: boolean | null; @@ -50,6 +59,7 @@ type TelegramMonitoringCoveragePayload = { }; const API_BASE = getRuntimeApiBaseUrl(); +const LEARNING_TARGET_COUNT = 6; const NAV_ITEMS = [ { @@ -98,7 +108,9 @@ function StatCard({
-

{label}

+

+ {label} +

{value}

@@ -108,20 +120,24 @@ function StatCard({ "flex h-8 w-8 shrink-0 items-center justify-center border", tone === "ok" && "border-[#9bc7a4] bg-[#f0faf2] text-[#17602a]", tone === "warn" && "border-[#d9b36f] bg-[#fff7e8] text-[#8a5a08]", - tone === "neutral" && "border-[#d8d3c7] bg-[#faf9f3] text-[#5f5b52]", + tone === "neutral" && + "border-[#d8d3c7] bg-[#faf9f3] text-[#5f5b52]", )} >
-

{detail}

+

+ {detail} +

); } export default function AwoooPAlertsPage() { const t = useTranslations("awooop.alerts.aiLoop"); - const [coverage, setCoverage] = useState(null); + const [coverage, setCoverage] = + useState(null); const [coverageLoading, setCoverageLoading] = useState(true); const refreshCoverage = useCallback(async () => { @@ -137,7 +153,9 @@ export default function AwoooPAlertsPage() { }, ); if (response.ok) { - setCoverage((await response.json()) as TelegramMonitoringCoveragePayload); + setCoverage( + (await response.json()) as TelegramMonitoringCoveragePayload, + ); } } catch { setCoverage(null); @@ -157,8 +175,37 @@ export default function AwoooPAlertsPage() { const answer = coverage?.operator_answer ?? {}; const blockers = coverage?.active_blockers ?? []; const ready = - summary.full_coverage_ready === true - || coverage?.status === "telegram_alert_monitoring_coverage_ready"; + summary.full_coverage_ready === true || + coverage?.status === "telegram_alert_monitoring_coverage_ready"; + const durableLearningReady = + summary.durable_learning_writeback_ready === true && + answer.durable_learning_writeback_verified === true; + const aiLoopSource = summary.ai_loop_context_source ?? "unavailable"; + const aiLoopSourceLabel = + aiLoopSource === "post_apply_verifier" + ? t("learningProof.sources.verified") + : aiLoopSource === "post_apply_verifier_without_durable_target_writes" + ? t("learningProof.sources.metadataOnly") + : aiLoopSource === "unverified_log_controlled_writeback_consumer" + ? t("learningProof.sources.consumerCandidate") + : aiLoopSource === "unverified_ai_alert_card_learning_writeback_refs" + ? t("learningProof.sources.alertCardCandidate") + : t("learningProof.sources.unavailable"); + const learningBlocker = + blockers.find((blocker) => blocker.startsWith("telegram_alert_ai_loop_")) ?? + blockers[0] ?? + ""; + const learningBlockerLabel = coverageLoading + ? t("learningProof.blockers.loading") + : coverage == null + ? t("learningProof.blockers.unavailable") + : learningBlocker === + "telegram_alert_ai_loop_post_apply_verifier_not_ready" + ? t("learningProof.blockers.verifier") + : learningBlocker === + "telegram_alert_ai_loop_durable_learning_writeback_not_verified" + ? t("learningProof.blockers.durableWrites") + : learningBlocker || t("learningProof.blockers.none"); const surfaceValue = coverageLoading ? "-- / --" : `${formatNumber(summary.monitoring_live_evidence_received_count)} / ${formatNumber(summary.monitoring_surface_count)}`; @@ -174,7 +221,7 @@ export default function AwoooPAlertsPage() { detail: t("summary.coverageDetail", { gaps: formatNumber(summary.monitoring_live_receipt_gap_count), }), - tone: ready ? "ok" as const : "warn" as const, + tone: ready ? ("ok" as const) : ("warn" as const), icon: ShieldCheck, }, { @@ -187,7 +234,7 @@ export default function AwoooPAlertsPage() { telegram: formatNumber(summary.runtime_telegram_sent_event_count_7d), km: formatNumber(summary.runtime_km_converted_event_count_7d), }), - tone: ready ? "ok" as const : "neutral" as const, + tone: ready ? ("ok" as const) : ("neutral" as const), icon: Database, }, { @@ -195,23 +242,33 @@ export default function AwoooPAlertsPage() { label: t("summary.aiContext"), value: aiContextValue, detail: t("summary.aiContextDetail"), - tone: answer.km_rag_mcp_playbook_ai_agent_context_ready ? "ok" as const : "warn" as const, + tone: answer.km_rag_mcp_playbook_ai_agent_context_ready + ? ("ok" as const) + : ("warn" as const), icon: Bot, }, { key: "direct", label: t("summary.direct"), - value: coverageLoading ? "--" : formatNumber(summary.known_direct_send_gap_count), + value: coverageLoading + ? "--" + : formatNumber(summary.known_direct_send_gap_count), detail: t("summary.directDetail"), - tone: answer.all_known_telegram_egress_routes_controlled ? "ok" as const : "warn" as const, + tone: answer.all_known_telegram_egress_routes_controlled + ? ("ok" as const) + : ("warn" as const), icon: Send, }, { key: "queue", label: t("summary.queue"), - value: coverageLoading ? "--" : formatNumber(summary.ai_controlled_gap_queue_count), - detail: blockers[0] ?? t("summary.queueDetail"), - tone: blockers.length === 0 ? "ok" as const : "warn" as const, + value: coverageLoading + ? "--" + : formatNumber(summary.ai_controlled_gap_queue_count), + detail: blockers[0]?.startsWith("telegram_alert_ai_loop_") + ? learningBlockerLabel + : (blockers[0] ?? t("summary.queueDetail")), + tone: blockers.length === 0 ? ("ok" as const) : ("warn" as const), icon: TriangleAlert, }, { @@ -221,11 +278,21 @@ export default function AwoooPAlertsPage() { ? "--" : formatPercent(summary.telegram_monitoring_audit_completion_percent), detail: t("summary.completionDetail"), - tone: ready ? "ok" as const : "neutral" as const, + tone: ready ? ("ok" as const) : ("neutral" as const), icon: Gauge, }, ], - [answer, blockers, coverageLoading, ready, summary, surfaceValue, aiContextValue, t], + [ + answer, + blockers, + coverageLoading, + ready, + summary, + surfaceValue, + aiContextValue, + learningBlockerLabel, + t, + ], ); return ( @@ -255,7 +322,10 @@ export default function AwoooPAlertsPage() {

-