wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(sre): require verified learning closure

Browse Source
This commit is contained in:
Your Name
2026-07-19 05:56:17 +08:00
parent eafaa20f33
commit 56e36c472c
3 changed files with 246 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

178
apps/api/src/services/telegram_alert_monitoring_coverage_readback.py
View File

@@ -69,6 +69,15 @@ _ALERT_LOG_EVENT_TYPES = (
"EXECUTION_STARTED",
"EXECUTION_COMPLETED",
)
_AI_LOOP_LEARNING_TARGETS = (
"km",
"rag",
"playbook",
"mcp",
"verifier",
"ai_agent",
)
_AI_LOOP_LEARNING_TARGET_COUNT = len(_AI_LOOP_LEARNING_TARGETS)
_RUNTIME_LIFECYCLE_SQL = """
WITH run_lifecycle AS (
SELECT
@@ -582,7 +591,9 @@ async def _load_post_apply_verifier_readback(*, project_id: str) -> dict[str, An
"rollups": {
"telegram_alert_learning_context_post_apply_verifier_ready": False,
"verified_context_receipt_count": 0,
"target_count": 0,
"verified_target_count": 0,
"verified_target_context_write_count": 0,
"verified_ai_agent_context_receipt_count": 0,
"error_type": type(exc).__name__,
},
@@ -937,13 +948,10 @@ def build_telegram_alert_monitoring_coverage_readback(
effective_ai_alert_total = max(ai_alert_total, _int(ai_loop_context["receipt_count"]))
effective_ai_alert_ready_total = max(
ai_alert_ready_total,
_int(ai_loop_context["receipt_count"]),
_int(ai_loop_context["receipt_count"])
if ai_loop_context["ready"]
else 0,
)
effective_ai_alert_summary = {
**ai_alert_summary,
"total": effective_ai_alert_total,
"learning_writeback_ready_total": effective_ai_alert_ready_total,
}
runtime_event_type_counts = {
str(key): _int(value)
for key, value in _dict(
@@ -960,7 +968,7 @@ def build_telegram_alert_monitoring_coverage_readback(
readback_summary=readback_summary,
matrix_summary=matrix_summary,
verifier_rollups=verifier_rollups,
ai_alert_summary=effective_ai_alert_summary,
ai_alert_summary=ai_alert_summary,
runtime_summary=runtime_summary,
source_contract=source_contract,
runtime_db_ok=runtime_db_ok,
@@ -973,10 +981,11 @@ def build_telegram_alert_monitoring_coverage_readback(
direct_gap_count=direct_gap_count,
runtime_db_ok=runtime_db_ok,
ai_alert_db_ok=ai_alert_db_ok,
verifier_ready=effective_verifier_ready,
post_apply_verifier_ready=verifier_ready,
durable_learning_writeback_ready=effective_verifier_ready,
source_contract=source_contract,
ai_alert_total=effective_ai_alert_total,
ai_alert_ready_total=effective_ai_alert_ready_total,
ai_alert_total=ai_alert_total,
ai_alert_ready_total=ai_alert_ready_total,
matrix_db_receipt_gap_count=matrix_db_receipt_gap_count,
matrix_ai_route_gap_count=matrix_ai_route_gap_count,
matrix_controlled_queue_gap_count=matrix_controlled_queue_gap_count,
@@ -1059,6 +1068,7 @@ def build_telegram_alert_monitoring_coverage_readback(
effective_verifier_ready
),
"km_rag_mcp_playbook_ai_agent_context_ready": effective_verifier_ready,
"durable_learning_writeback_verified": effective_verifier_ready,
"km_rag_mcp_playbook_ai_agent_context_source": str(
ai_loop_context["source"]
),
@@ -1183,10 +1193,14 @@ def build_telegram_alert_monitoring_coverage_readback(
verifier_rollups.get("verified_context_receipt_count")
),
"verified_target_count": _int(verifier_rollups.get("verified_target_count")),
"verified_target_context_write_count": _int(
ai_loop_context.get("verified_target_context_write_count")
),
"verified_ai_agent_context_receipt_count": _int(
verifier_rollups.get("verified_ai_agent_context_receipt_count")
),
"ai_loop_context_ready": effective_verifier_ready,
"durable_learning_writeback_ready": effective_verifier_ready,
"ai_loop_context_source": str(ai_loop_context["source"]),
"ai_loop_context_fallback_used": bool(ai_loop_context["fallback_used"]),
"ai_loop_context_receipt_count": _int(ai_loop_context["receipt_count"]),
@@ -1337,6 +1351,13 @@ def _effective_ai_loop_context(
consumer_context: Mapping[str, Any],
) -> dict[str, Any]:
verifier_receipt_count = _int(verifier_rollups.get("verified_context_receipt_count"))
verifier_target_count = _int(verifier_rollups.get("target_count"))
verifier_verified_target_count = _int(
verifier_rollups.get("verified_target_count")
)
verifier_target_write_count = _int(
verifier_rollups.get("verified_target_context_write_count")
)
verifier_ai_agent_count = _int(
verifier_rollups.get("verified_ai_agent_context_receipt_count")
)
@@ -1368,61 +1389,56 @@ def _effective_ai_loop_context(
and consumer_target_count > 0
and consumer_ready_target_count >= consumer_target_count
)
required_target_count = _AI_LOOP_LEARNING_TARGET_COUNT
durable_verifier_ready = bool(
verifier_ready
and verifier_target_count == required_target_count
and verifier_verified_target_count == required_target_count
and verifier_target_write_count == required_target_count
and verifier_ai_agent_count > 0
)
if verifier_ready:
if durable_verifier_ready:
return {
"ready": True,
"source": "post_apply_verifier",
"fallback_used": False,
"receipt_count": max(
verifier_receipt_count,
ai_alert_ready_total,
consumer_receipt_count,
),
"ai_agent_context_receipt_count": max(
verifier_ai_agent_count,
consumer_ai_agent_count,
),
"ready_target_count": max(
_int(verifier_rollups.get("verified_target_count")),
consumer_ready_target_count,
),
"target_count": max(consumer_target_count, 0),
"receipt_count": verifier_receipt_count,
"ai_agent_context_receipt_count": verifier_ai_agent_count,
"ready_target_count": verifier_verified_target_count,
"target_count": required_target_count,
"verified_target_context_write_count": verifier_target_write_count,
"durable_learning_writeback_ready": True,
}
if consumer_targets_ready:
return {
"ready": True,
"source": "log_controlled_writeback_consumer",
"fallback_used": True,
"receipt_count": max(consumer_receipt_count, ai_alert_ready_total),
"ai_agent_context_receipt_count": consumer_ai_agent_count,
"ready_target_count": consumer_ready_target_count,
"target_count": consumer_target_count,
}
if ai_alert_total > 0 and ai_alert_ready_total >= ai_alert_total:
return {
"ready": True,
"source": "ai_alert_card_learning_writeback_refs",
"fallback_used": True,
"receipt_count": ai_alert_ready_total,
"ai_agent_context_receipt_count": max(consumer_ai_agent_count, 1),
"ready_target_count": max(consumer_ready_target_count, 1),
"target_count": max(consumer_target_count, 1),
}
source = "unavailable"
if verifier_ready:
source = "post_apply_verifier_without_durable_target_writes"
elif consumer_targets_ready:
source = "unverified_log_controlled_writeback_consumer"
elif ai_alert_total > 0 and ai_alert_ready_total >= ai_alert_total:
source = "unverified_ai_alert_card_learning_writeback_refs"
return {
"ready": False,
"source": "unavailable",
"source": source,
"fallback_used": False,
"receipt_count": max(verifier_receipt_count, ai_alert_ready_total),
"receipt_count": max(
verifier_receipt_count,
consumer_receipt_count,
ai_alert_ready_total,
),
"ai_agent_context_receipt_count": max(
verifier_ai_agent_count,
consumer_ai_agent_count,
),
"ready_target_count": consumer_ready_target_count,
"target_count": consumer_target_count,
"ready_target_count": max(
verifier_verified_target_count,
consumer_ready_target_count,
),
"target_count": required_target_count,
"verified_target_context_write_count": verifier_target_write_count,
"durable_learning_writeback_ready": False,
"ai_alert_total": ai_alert_total,
}
@@ -2322,6 +2338,42 @@ def _gap_execution_contract(blocker: str) -> dict[str, Any]:
"destructive_action": False,
},
}
if blocker == "telegram_alert_ai_loop_post_apply_verifier_not_ready":
return {
"status": "ready_ai_read_only_retry",
"risk_level": "low",
"execution_mode": "automatic_bounded_verifier_readback",
"executor_route": "telegram_alert_learning_context_post_apply_verifier",
"ai_controlled_execution_allowed": True,
"owner_review_required_for_low_medium_high": False,
"critical_break_glass_required": False,
"operation_boundaries": {
"database_read_only": True,
"requires_secret": False,
"requires_runtime_send": False,
"runtime_write_performed": False,
"stores_raw_payload": False,
"destructive_action": False,
},
}
if blocker == "telegram_alert_ai_loop_durable_learning_writeback_not_verified":
return {
"status": "ready_ai_controlled_metadata_writeback",
"risk_level": "medium",
"execution_mode": "controlled_metadata_writeback_then_verify",
"executor_route": "ai_agent_metadata_writeback_executor",
"ai_controlled_execution_allowed": True,
"owner_review_required_for_low_medium_high": False,
"critical_break_glass_required": False,
"operation_boundaries": {
"database_read_only": False,
"requires_secret": False,
"requires_runtime_send": False,
"runtime_write_performed": False,
"stores_raw_payload": False,
"destructive_action": False,
},
}
if blocker == "runtime_ai_automation_trigger_receipt_missing" or blocker.startswith(
"runtime_ai_automation_lifecycle_open"
):
@@ -2384,7 +2436,6 @@ def _coverage_matrix(
ai_alert_ready_total = _int(ai_alert_summary.get("learning_writeback_ready_total"))
ai_loop_source = str(ai_loop_context.get("source") or "unavailable")
ai_loop_receipt_count = _int(ai_loop_context.get("receipt_count"))
ai_loop_fallback_used = bool(ai_loop_context.get("fallback_used"))
return [
{
"surface_id": "monitoring_inventory_static_scope",
@@ -2486,9 +2537,7 @@ def _coverage_matrix(
{
"surface_id": "telegram_alert_ai_loop_context_verifier",
"status": (
"consumer_context_fallback_ready"
if ai_loop_fallback_used and verifier_ready
else "verified_context_ready"
"verified_durable_context_ready"
if verifier_ready
else "verifier_not_ready"
),
@@ -2496,10 +2545,12 @@ def _coverage_matrix(
"db_or_log_receipt": "ready_context_receipt_source",
"ai_route": "ready_ai_agent_reusable_context",
"controlled_queue": "ready_metadata_only_post_apply_verifier",
"post_verifier": "ready"
if not ai_loop_fallback_used
else "consumer_context_fallback",
"learning_writeback": "ready_km_rag_mcp_playbook_ai_agent_targets",
"post_verifier": "ready" if verifier_ready else "waiting_independent_verifier",
"learning_writeback": (
"ready_verified_km_rag_mcp_playbook_ai_agent_target_writes"
if verifier_ready
else "waiting_durable_target_write_receipts"
),
"gap_count": 0 if verifier_ready else 1,
"evidence_refs": [
"/api/v1/agents/telegram-alert-learning-context-post-apply-verifier",
@@ -2537,7 +2588,8 @@ def _active_blockers(
direct_gap_count: int,
runtime_db_ok: bool,
ai_alert_db_ok: bool,
verifier_ready: bool,
post_apply_verifier_ready: bool,
durable_learning_writeback_ready: bool,
source_contract: Mapping[str, Any],
ai_alert_total: int,
ai_alert_ready_total: int,
@@ -2582,8 +2634,12 @@ def _active_blockers(
"runtime_ai_automation_lifecycle_open:"
f"{max(runtime_open_run_count, 1)}"
)
if not verifier_ready:
if not post_apply_verifier_ready:
blockers.append("telegram_alert_ai_loop_post_apply_verifier_not_ready")
elif not durable_learning_writeback_ready:
blockers.append(
"telegram_alert_ai_loop_durable_learning_writeback_not_verified"
)
blockers.extend(_source_contract_blockers(source_contract))
return _unique(blockers)
@@ -2835,6 +2891,8 @@ def _gap_next_action(blocker: str) -> str:
return "populate_km_rag_mcp_playbook_ai_agent_learning_refs"
if blocker == "telegram_alert_ai_loop_post_apply_verifier_not_ready":
return "run_ai_loop_context_post_apply_verifier_readback"
if blocker == "telegram_alert_ai_loop_durable_learning_writeback_not_verified":
return "consume_target_context_receipts_then_run_independent_verifier"
if blocker.startswith("source_contract_missing"):
return "restore_operator_visible_source_contract_marker"
return "continue_ai_controlled_gap_closure"

151
apps/api/tests/test_telegram_alert_monitoring_coverage_readback_api.py
View File

@@ -46,7 +46,9 @@ def _payload(
"rollups": {
"telegram_alert_learning_context_post_apply_verifier_ready": True,
"verified_context_receipt_count": 36,
"target_count": 6,
"verified_target_count": 6,
"verified_target_context_write_count": 6,
"verified_ai_agent_context_receipt_count": 6,
},
}
@@ -367,7 +369,7 @@ def test_telegram_alert_monitoring_coverage_readback_surfaces_live_gaps():
matrix = {item["surface_id"]: item for item in payload["coverage_matrix"]}
assert matrix["telegram_gateway_outbound_mirror"]["gap_count"] == 0
assert matrix["telegram_alert_ai_loop_context_verifier"]["status"] == (
"verified_context_ready"
"verified_durable_context_ready"
)
assert matrix["monitoring_inventory_static_scope"]["gap_count"] == 60
assert payload["operation_boundaries"]["telegram_send_performed"] is False
@@ -563,7 +565,7 @@ def test_telegram_alert_monitoring_coverage_blocks_partial_surface_contracts():
]
def test_telegram_alert_monitoring_coverage_uses_consumer_context_fallback():
def test_telegram_alert_monitoring_coverage_rejects_unverified_consumer_context():
payload = _payload(
post_apply_verifier={
"schema_version": "telegram_alert_learning_context_post_apply_verifier_v1",
@@ -571,7 +573,9 @@ def test_telegram_alert_monitoring_coverage_uses_consumer_context_fallback():
"rollups": {
"telegram_alert_learning_context_post_apply_verifier_ready": False,
"verified_context_receipt_count": 0,
"target_count": 6,
"verified_target_count": 0,
"verified_target_context_write_count": 0,
"verified_ai_agent_context_receipt_count": 0,
},
},
@@ -588,61 +592,64 @@ def test_telegram_alert_monitoring_coverage_uses_consumer_context_fallback():
summary = payload["summary"]
assert summary["ai_alert_card_db_readback_ok"] is False
assert summary["ai_loop_context_ready"] is True
assert summary["ai_loop_context_source"] == "log_controlled_writeback_consumer"
assert summary["ai_loop_context_fallback_used"] is True
assert summary["ai_loop_context_ready"] is False
assert summary["durable_learning_writeback_ready"] is False
assert summary["ai_loop_context_source"] == (
"unverified_log_controlled_writeback_consumer"
)
assert summary["ai_loop_context_fallback_used"] is False
assert summary["ai_loop_context_receipt_count"] == 6
assert summary["ai_loop_ai_agent_context_receipt_count"] == 1
assert summary["km_rag_mcp_playbook_ai_agent_context_ready"] is True
assert summary["km_rag_mcp_playbook_ai_agent_context_ready"] is False
assert (
summary["km_rag_mcp_playbook_ai_agent_context_source"]
== "log_controlled_writeback_consumer"
== "unverified_log_controlled_writeback_consumer"
)
assert summary["effective_ai_alert_context_receipt_total"] == 6
assert summary["effective_ai_alert_context_ready_total"] == 6
assert summary["effective_ai_alert_context_ready_total"] == 0
assert summary["log_controlled_consumer_readback_ready"] is True
assert summary["telegram_monitoring_audit_completion_percent"] == 81.8
assert summary["telegram_monitoring_audit_completion_percent"] < 100.0
assert (
payload["operator_answer"][
"all_verified_ai_alert_context_receipts_reusable_by_ai_agent"
]
is True
is False
)
assert (
payload["operator_answer"]["km_rag_mcp_playbook_ai_agent_context_ready"]
is True
is False
)
assert (
payload["operator_answer"]["km_rag_mcp_playbook_ai_agent_context_source"]
== "log_controlled_writeback_consumer"
== "unverified_log_controlled_writeback_consumer"
)
assert (
"awooop_ai_alert_card_delivery_db_readback_unavailable"
in payload["active_blockers"]
)
assert "awooop_ai_alert_card_delivery_receipts_missing" not in payload[
assert "awooop_ai_alert_card_delivery_receipts_missing" in payload[
"active_blockers"
]
assert "awooop_ai_alert_card_learning_writeback_refs_missing" not in payload[
assert "awooop_ai_alert_card_learning_writeback_refs_missing" in payload[
"active_blockers"
]
assert "telegram_alert_ai_loop_post_apply_verifier_not_ready" not in payload[
assert "telegram_alert_ai_loop_post_apply_verifier_not_ready" in payload[
"active_blockers"
]
pipeline = {stage["stage_id"]: stage for stage in payload["alert_receipt_pipeline"]}
assert pipeline["km_rag_mcp_playbook_ai_agent_learning_refs"]["ready"] is True
assert pipeline["km_rag_mcp_playbook_ai_agent_learning_refs"]["ready"] is False
matrix = {item["surface_id"]: item for item in payload["coverage_matrix"]}
assert matrix["telegram_alert_ai_loop_context_verifier"]["status"] == (
"consumer_context_fallback_ready"
"verifier_not_ready"
)
assert matrix["telegram_alert_ai_loop_context_verifier"][
"ai_loop_context_source"
] == "log_controlled_writeback_consumer"
] == "unverified_log_controlled_writeback_consumer"
def test_telegram_alert_monitoring_coverage_uses_ai_alert_card_learning_refs_fallback():
def test_telegram_alert_monitoring_coverage_rejects_unverified_ai_alert_refs():
payload = _payload(
post_apply_verifier={
"schema_version": "telegram_alert_learning_context_post_apply_verifier_v1",
@@ -650,7 +657,9 @@ def test_telegram_alert_monitoring_coverage_uses_ai_alert_card_learning_refs_fal
"rollups": {
"telegram_alert_learning_context_post_apply_verifier_ready": False,
"verified_context_receipt_count": 0,
"target_count": 0,
"verified_target_count": 0,
"verified_target_context_write_count": 0,
"verified_ai_agent_context_receipt_count": 0,
},
},
@@ -680,20 +689,108 @@ def test_telegram_alert_monitoring_coverage_uses_ai_alert_card_learning_refs_fal
)
summary = payload["summary"]
assert payload["status"] == "telegram_alert_monitoring_coverage_ready"
assert summary["ai_loop_context_ready"] is True
assert summary["ai_loop_context_source"] == "ai_alert_card_learning_writeback_refs"
assert summary["ai_loop_context_fallback_used"] is True
assert payload["status"] == (
"blocked_telegram_alert_monitoring_coverage_gaps_present"
)
assert summary["ai_loop_context_ready"] is False
assert summary["ai_loop_context_source"] == (
"unverified_ai_alert_card_learning_writeback_refs"
)
assert summary["ai_loop_context_fallback_used"] is False
assert summary["effective_ai_alert_context_receipt_total"] == 444
assert summary["effective_ai_alert_context_ready_total"] == 444
assert summary["telegram_monitoring_audit_completion_percent"] == 100.0
assert summary["telegram_monitoring_audit_completion_percent"] < 100.0
assert "telegram_alert_ai_loop_post_apply_verifier_not_ready" in payload[
"active_blockers"
]
assert (
payload["operator_answer"]["km_rag_mcp_playbook_ai_agent_context_source"]
== "unverified_ai_alert_card_learning_writeback_refs"
)
def test_telegram_alert_monitoring_coverage_requires_durable_target_writes():
payload = _payload(
post_apply_verifier={
"schema_version": (
"telegram_alert_learning_context_post_apply_verifier_v1"
),
"status": "telegram_alert_learning_context_post_apply_verified",
"rollups": {
"telegram_alert_learning_context_post_apply_verifier_ready": True,
"verified_context_receipt_count": 6,
"target_count": 6,
"verified_target_count": 6,
"verified_target_context_write_count": 0,
"verified_ai_agent_context_receipt_count": 1,
},
},
monitoring_live_receipt_apply_readback=(
_metadata_live_receipt_apply_readback()
),
)
summary = payload["summary"]
assert payload["status"] == (
"blocked_telegram_alert_monitoring_coverage_gaps_present"
)
assert summary["ai_loop_context_ready"] is False
assert summary["durable_learning_writeback_ready"] is False
assert summary["verified_target_context_write_count"] == 0
assert summary["ai_loop_context_source"] == (
"post_apply_verifier_without_durable_target_writes"
)
assert (
"telegram_alert_ai_loop_durable_learning_writeback_not_verified"
in payload["active_blockers"]
)
assert "telegram_alert_ai_loop_post_apply_verifier_not_ready" not in payload[
"active_blockers"
]
assert payload["active_blockers"] == []
queue = {item["blocker"]: item for item in payload["ai_controlled_gap_queue"]}
candidate = queue[
"telegram_alert_ai_loop_durable_learning_writeback_not_verified"
]
assert candidate["risk_level"] == "medium"
assert candidate["ai_controlled_execution_allowed"] is True
assert candidate["critical_break_glass_required"] is False
assert candidate["executor_route"] == "ai_agent_metadata_writeback_executor"
@pytest.mark.parametrize(
("target_count", "verified_target_count", "verified_write_count"),
((5, 5, 5), (6, 6, 7)),
)
def test_telegram_alert_monitoring_coverage_requires_exact_six_target_writes(
target_count: int,
verified_target_count: int,
verified_write_count: int,
):
payload = _payload(
post_apply_verifier={
"schema_version": (
"telegram_alert_learning_context_post_apply_verifier_v1"
),
"status": "telegram_alert_learning_context_post_apply_verified",
"rollups": {
"telegram_alert_learning_context_post_apply_verifier_ready": True,
"verified_context_receipt_count": verified_target_count,
"target_count": target_count,
"verified_target_count": verified_target_count,
"verified_target_context_write_count": verified_write_count,
"verified_ai_agent_context_receipt_count": 1,
},
},
monitoring_live_receipt_apply_readback=(
_metadata_live_receipt_apply_readback()
),
)
assert payload["summary"]["durable_learning_writeback_ready"] is False
assert payload["operator_answer"]["durable_learning_writeback_verified"] is False
assert (
payload["operator_answer"]["km_rag_mcp_playbook_ai_agent_context_source"]
== "ai_alert_card_learning_writeback_refs"
"telegram_alert_ai_loop_durable_learning_writeback_not_verified"
in payload["active_blockers"]
)