feat(telegram): audit monitoring alert coverage
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 2m8s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 2m8s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
This commit is contained in:
@@ -472,6 +472,9 @@ from src.services.telegram_alert_ai_automation_matrix import (
|
||||
from src.services.telegram_alert_learning_context_post_apply_verifier import (
|
||||
load_latest_telegram_alert_learning_context_post_apply_verifier,
|
||||
)
|
||||
from src.services.telegram_alert_monitoring_coverage_readback import (
|
||||
load_latest_telegram_alert_monitoring_coverage_readback,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/agents", tags=["Agent Teams"])
|
||||
logger = get_logger("awoooi.agents")
|
||||
@@ -2499,6 +2502,39 @@ async def get_telegram_alert_learning_context_post_apply_verifier() -> dict[str,
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/telegram-alert-monitoring-coverage-readback",
|
||||
response_model=dict[str, Any],
|
||||
summary="取得 Telegram monitoring alert DB/log 與 AI automation coverage readback",
|
||||
description=(
|
||||
"整合 monitoring inventory、alert_operation_log、AwoooP outbound mirror、"
|
||||
"Telegram AI automation matrix 與 AI Loop post-apply verifier,回答所有 "
|
||||
"Telegram 監控告警是否有 DB/log receipt、AI route、controlled queue、"
|
||||
"KM/RAG/MCP/PlayBook/AI Agent context。此端點只讀 metadata,不送 Telegram、"
|
||||
"不呼叫 Bot API、不改 receiver、不觸發 workflow、不保存 raw payload、不讀 secret。"
|
||||
),
|
||||
)
|
||||
async def get_telegram_alert_monitoring_coverage_readback() -> dict[str, Any]:
|
||||
"""Read Telegram monitoring alert coverage for AI automation."""
|
||||
try:
|
||||
payload = await load_latest_telegram_alert_monitoring_coverage_readback()
|
||||
return redact_public_lan_topology(payload)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
except (json.JSONDecodeError, ValueError) as exc:
|
||||
logger.error(
|
||||
"telegram_alert_monitoring_coverage_readback_invalid",
|
||||
error=str(exc),
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Telegram monitoring alert coverage readback 無效",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.post(
|
||||
"/agent-log-controlled-writeback-consumer-apply",
|
||||
response_model=dict[str, Any],
|
||||
|
||||
@@ -0,0 +1,863 @@
|
||||
"""Telegram monitoring alert DB/log and AI automation coverage readback.
|
||||
|
||||
This service answers the operator question: for monitoring alerts that reach
|
||||
Telegram, do we have a DB/log receipt, an AI-controlled route, a controlled
|
||||
queue boundary, and reusable KM/RAG/MCP/PlayBook/AI Agent context?
|
||||
|
||||
It is metadata-only. It never sends Telegram, calls Bot API, changes receiver
|
||||
routes, stores raw alert payloads, reads secrets, or triggers workflow/runtime
|
||||
actions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from src.core.logging import get_logger
|
||||
from src.db.base import get_db_context
|
||||
from src.services.platform_operator_service import list_ai_alert_card_delivery_readback
|
||||
from src.services.snapshot_paths import default_security_dir, resolve_repo_root
|
||||
from src.services.telegram_alert_ai_automation_matrix import (
|
||||
load_latest_telegram_alert_ai_automation_matrix,
|
||||
)
|
||||
from src.services.telegram_alert_learning_context_post_apply_verifier import (
|
||||
load_latest_telegram_alert_learning_context_post_apply_verifier,
|
||||
)
|
||||
|
||||
SCHEMA_VERSION = "telegram_alert_monitoring_coverage_readback_v1"
|
||||
DEFAULT_PROJECT_ID = "awoooi"
|
||||
_MONITORING_INVENTORY = "monitoring-alerting-observability-inventory.snapshot.json"
|
||||
_MONITORING_POST_INCIDENT_READBACK = "monitoring-post-incident-readback-plan.snapshot.json"
|
||||
_ALERT_LOG_EVENT_TYPES = (
|
||||
"ALERT_RECEIVED",
|
||||
"TELEGRAM_SENT",
|
||||
"TELEGRAM_RESULT_SENT",
|
||||
"NOTIFICATION_CLASSIFIED",
|
||||
"KM_CONVERTED",
|
||||
"PLAYBOOK_DRAFT_CREATED",
|
||||
"GUARDRAIL_BLOCKED",
|
||||
"AUTO_REPAIR_TRIGGERED",
|
||||
"EXECUTION_STARTED",
|
||||
"EXECUTION_COMPLETED",
|
||||
)
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
async def load_latest_telegram_alert_monitoring_coverage_readback(
|
||||
*,
|
||||
project_id: str = DEFAULT_PROJECT_ID,
|
||||
) -> dict[str, Any]:
|
||||
"""Return the current Telegram monitoring alert coverage readback."""
|
||||
|
||||
anchor = Path(__file__)
|
||||
security_dir = default_security_dir(anchor)
|
||||
repo_root = resolve_repo_root(anchor)
|
||||
|
||||
monitoring_inventory = _load_required_json(security_dir / _MONITORING_INVENTORY)
|
||||
monitoring_readback_plan = _load_required_json(
|
||||
security_dir / _MONITORING_POST_INCIDENT_READBACK
|
||||
)
|
||||
_require_schema(
|
||||
monitoring_inventory,
|
||||
"monitoring_alerting_observability_inventory_v1",
|
||||
_MONITORING_INVENTORY,
|
||||
)
|
||||
_require_schema(
|
||||
monitoring_readback_plan,
|
||||
"monitoring_post_incident_readback_plan_v1",
|
||||
_MONITORING_POST_INCIDENT_READBACK,
|
||||
)
|
||||
_require_safe_boundaries(monitoring_inventory, monitoring_readback_plan)
|
||||
|
||||
matrix = await asyncio.to_thread(load_latest_telegram_alert_ai_automation_matrix)
|
||||
verifier = await load_latest_telegram_alert_learning_context_post_apply_verifier(
|
||||
project_id=project_id
|
||||
)
|
||||
ai_alert_readback = await _load_ai_alert_card_delivery(project_id=project_id)
|
||||
runtime_log_readback = await _load_runtime_log_readback(project_id=project_id)
|
||||
source_contract = _inspect_source_contract(repo_root)
|
||||
|
||||
return build_telegram_alert_monitoring_coverage_readback(
|
||||
project_id=project_id,
|
||||
monitoring_inventory=monitoring_inventory,
|
||||
monitoring_readback_plan=monitoring_readback_plan,
|
||||
telegram_matrix=matrix,
|
||||
post_apply_verifier=verifier,
|
||||
ai_alert_card_delivery_readback=ai_alert_readback,
|
||||
runtime_log_readback=runtime_log_readback,
|
||||
source_contract=source_contract,
|
||||
)
|
||||
|
||||
|
||||
def build_telegram_alert_monitoring_coverage_readback(
|
||||
*,
|
||||
project_id: str,
|
||||
monitoring_inventory: Mapping[str, Any],
|
||||
monitoring_readback_plan: Mapping[str, Any],
|
||||
telegram_matrix: Mapping[str, Any],
|
||||
post_apply_verifier: Mapping[str, Any],
|
||||
ai_alert_card_delivery_readback: Mapping[str, Any],
|
||||
runtime_log_readback: Mapping[str, Any],
|
||||
source_contract: Mapping[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Build deterministic coverage from static inventory and live readbacks."""
|
||||
|
||||
inventory_summary = _dict(monitoring_inventory.get("summary"))
|
||||
readback_summary = _dict(monitoring_readback_plan.get("summary"))
|
||||
matrix_summary = _dict(telegram_matrix.get("summary"))
|
||||
verifier_rollups = _dict(post_apply_verifier.get("rollups"))
|
||||
ai_alert_summary = _dict(ai_alert_card_delivery_readback.get("summary"))
|
||||
runtime_summary = _dict(runtime_log_readback.get("summary"))
|
||||
|
||||
monitoring_surface_count = _int(inventory_summary.get("surface_count"))
|
||||
live_evidence_received_count = _int(
|
||||
inventory_summary.get("live_evidence_received_count")
|
||||
)
|
||||
monitoring_live_gap_count = max(
|
||||
monitoring_surface_count - live_evidence_received_count,
|
||||
0,
|
||||
)
|
||||
direct_gap_count = _int(matrix_summary.get("known_direct_send_gap_count"))
|
||||
matrix_ready = telegram_matrix.get("status") == "telegram_alert_ai_automation_matrix_ready"
|
||||
verifier_ready = bool(
|
||||
verifier_rollups.get(
|
||||
"telegram_alert_learning_context_post_apply_verifier_ready"
|
||||
)
|
||||
)
|
||||
runtime_db_ok = runtime_log_readback.get("status") == "ok"
|
||||
ai_alert_db_ok = ai_alert_card_delivery_readback.get("status") == "ok"
|
||||
source_ready = not _source_contract_blockers(source_contract)
|
||||
ai_alert_total = _int(ai_alert_summary.get("total"))
|
||||
ai_alert_ready_total = _int(
|
||||
ai_alert_summary.get("learning_writeback_ready_total")
|
||||
)
|
||||
|
||||
coverage_matrix = _coverage_matrix(
|
||||
project_id=project_id,
|
||||
monitoring_surface_count=monitoring_surface_count,
|
||||
live_evidence_received_count=live_evidence_received_count,
|
||||
monitoring_live_gap_count=monitoring_live_gap_count,
|
||||
inventory_summary=inventory_summary,
|
||||
readback_summary=readback_summary,
|
||||
matrix_summary=matrix_summary,
|
||||
verifier_rollups=verifier_rollups,
|
||||
ai_alert_summary=ai_alert_summary,
|
||||
runtime_summary=runtime_summary,
|
||||
source_contract=source_contract,
|
||||
runtime_db_ok=runtime_db_ok,
|
||||
ai_alert_db_ok=ai_alert_db_ok,
|
||||
verifier_ready=verifier_ready,
|
||||
)
|
||||
active_blockers = _active_blockers(
|
||||
monitoring_live_gap_count=monitoring_live_gap_count,
|
||||
direct_gap_count=direct_gap_count,
|
||||
runtime_db_ok=runtime_db_ok,
|
||||
ai_alert_db_ok=ai_alert_db_ok,
|
||||
verifier_ready=verifier_ready,
|
||||
source_contract=source_contract,
|
||||
ai_alert_total=ai_alert_total,
|
||||
ai_alert_ready_total=ai_alert_ready_total,
|
||||
)
|
||||
ready = not active_blockers
|
||||
|
||||
return {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"priority": "P0-TELEGRAM-MONITORING-AI-AUTOMATION-COVERAGE",
|
||||
"priority_work_item_id": "CIR-P0-TG-001",
|
||||
"project_id": project_id,
|
||||
"status": (
|
||||
"telegram_alert_monitoring_coverage_ready"
|
||||
if ready
|
||||
else "blocked_telegram_alert_monitoring_coverage_gaps_present"
|
||||
),
|
||||
"mode": "metadata_only_db_log_ai_context_readback",
|
||||
"operator_answer": {
|
||||
"all_known_telegram_egress_routes_controlled": direct_gap_count == 0,
|
||||
"all_known_telegram_egress_routes_have_db_or_log_receipt": (
|
||||
direct_gap_count == 0 and ai_alert_db_ok and runtime_db_ok
|
||||
),
|
||||
"all_verified_ai_alert_context_receipts_reusable_by_ai_agent": verifier_ready,
|
||||
"all_monitoring_inventory_surfaces_have_accepted_live_receipt": (
|
||||
monitoring_live_gap_count == 0
|
||||
),
|
||||
"manual_default_terminal_state_allowed": False,
|
||||
"critical_break_glass_exception_only": True,
|
||||
},
|
||||
"summary": {
|
||||
"monitoring_surface_count": monitoring_surface_count,
|
||||
"monitoring_source_exists_count": _int(
|
||||
inventory_summary.get("source_exists_count")
|
||||
),
|
||||
"monitoring_live_evidence_received_count": live_evidence_received_count,
|
||||
"monitoring_live_receipt_gap_count": monitoring_live_gap_count,
|
||||
"monitoring_readback_candidate_count": _int(
|
||||
readback_summary.get("readback_candidate_count")
|
||||
),
|
||||
"alert_rule_surface_count": _int(
|
||||
inventory_summary.get("alert_rule_surface_count")
|
||||
),
|
||||
"telegram_surface_count": _int(
|
||||
inventory_summary.get("telegram_surface_count")
|
||||
),
|
||||
"telegram_alert_matrix_surface_count": _int(
|
||||
matrix_summary.get("telegram_alert_surface_count")
|
||||
),
|
||||
"known_direct_send_gap_count": direct_gap_count,
|
||||
"db_or_log_receipt_ready_surface_count": _int(
|
||||
matrix_summary.get("db_or_log_receipt_ready_surface_count")
|
||||
),
|
||||
"ai_route_ready_surface_count": _int(
|
||||
matrix_summary.get("ai_route_ready_surface_count")
|
||||
),
|
||||
"controlled_queue_ready_surface_count": _int(
|
||||
matrix_summary.get("controlled_queue_ready_surface_count")
|
||||
),
|
||||
"post_verifier_ready_surface_count": _int(
|
||||
matrix_summary.get("post_verifier_ready_surface_count")
|
||||
),
|
||||
"learning_writeback_ready_surface_count": _int(
|
||||
matrix_summary.get("learning_writeback_ready_surface_count")
|
||||
),
|
||||
"ai_alert_card_delivery_total": ai_alert_total,
|
||||
"ai_alert_card_learning_writeback_ready_total": ai_alert_ready_total,
|
||||
"runtime_alert_operation_log_total_7d": _int(
|
||||
runtime_summary.get("alert_operation_log_total_7d")
|
||||
),
|
||||
"runtime_telegram_sent_event_count_7d": _int(
|
||||
runtime_summary.get("telegram_sent_event_count_7d")
|
||||
),
|
||||
"runtime_km_converted_event_count_7d": _int(
|
||||
runtime_summary.get("km_converted_event_count_7d")
|
||||
),
|
||||
"runtime_playbook_draft_event_count_7d": _int(
|
||||
runtime_summary.get("playbook_draft_event_count_7d")
|
||||
),
|
||||
"verified_context_receipt_count": _int(
|
||||
verifier_rollups.get("verified_context_receipt_count")
|
||||
),
|
||||
"verified_target_count": _int(verifier_rollups.get("verified_target_count")),
|
||||
"verified_ai_agent_context_receipt_count": _int(
|
||||
verifier_rollups.get("verified_ai_agent_context_receipt_count")
|
||||
),
|
||||
"source_contract_ready": source_ready,
|
||||
"runtime_db_readback_ok": runtime_db_ok,
|
||||
"ai_alert_card_db_readback_ok": ai_alert_db_ok,
|
||||
"full_coverage_ready": ready,
|
||||
},
|
||||
"coverage_matrix": coverage_matrix,
|
||||
"active_blockers": active_blockers,
|
||||
"next_controlled_actions": _next_controlled_actions(active_blockers),
|
||||
"controlled_apply_packet": _controlled_apply_packet(
|
||||
project_id=project_id,
|
||||
active_blockers=active_blockers,
|
||||
monitoring_live_gap_count=monitoring_live_gap_count,
|
||||
direct_gap_count=direct_gap_count,
|
||||
),
|
||||
"source_refs": {
|
||||
"monitoring_inventory": f"docs/security/{_MONITORING_INVENTORY}",
|
||||
"monitoring_post_incident_readback": (
|
||||
f"docs/security/{_MONITORING_POST_INCIDENT_READBACK}"
|
||||
),
|
||||
"telegram_matrix": "/api/v1/agents/telegram-alert-ai-automation-matrix",
|
||||
"post_apply_verifier": (
|
||||
"/api/v1/agents/telegram-alert-learning-context-post-apply-verifier"
|
||||
),
|
||||
"ai_alert_card_delivery": "/api/v1/platform/runs/ai-alert-cards",
|
||||
"alert_operation_log": "/api/v1/alert-operation-logs",
|
||||
},
|
||||
"operation_boundaries": {
|
||||
"metadata_read_performed": True,
|
||||
"runtime_db_read_performed": runtime_db_ok,
|
||||
"telegram_send_performed": False,
|
||||
"bot_api_call_performed": False,
|
||||
"gateway_queue_write_performed": False,
|
||||
"receiver_route_changed": False,
|
||||
"alertmanager_reload_performed": False,
|
||||
"runtime_write_performed": False,
|
||||
"km_write_performed": False,
|
||||
"rag_index_write_performed": False,
|
||||
"playbook_trust_write_performed": False,
|
||||
"mcp_tool_call_performed": False,
|
||||
"workflow_dispatch_performed": False,
|
||||
"raw_payload_included": False,
|
||||
"raw_alert_payload_stored": False,
|
||||
"secret_value_read": False,
|
||||
"github_api_used": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def _load_ai_alert_card_delivery(*, project_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
payload = await asyncio.wait_for(
|
||||
list_ai_alert_card_delivery_readback(
|
||||
project_id=project_id,
|
||||
page=1,
|
||||
per_page=6,
|
||||
refresh=True,
|
||||
),
|
||||
timeout=8.0,
|
||||
)
|
||||
summary = _dict(payload.get("summary"))
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": summary,
|
||||
"total": _int(payload.get("total")),
|
||||
}
|
||||
except Exception as exc: # pragma: no cover - live DB pressure
|
||||
logger.warning(
|
||||
"telegram_alert_monitoring_ai_alert_card_readback_unavailable",
|
||||
project_id=project_id,
|
||||
error_type=type(exc).__name__,
|
||||
)
|
||||
return {
|
||||
"status": "unavailable",
|
||||
"summary": {
|
||||
"total": 0,
|
||||
"learning_writeback_ready_total": 0,
|
||||
"db_read_status": "unavailable",
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def _load_runtime_log_readback(*, project_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
_load_runtime_log_readback_once(project_id=project_id),
|
||||
timeout=8.0,
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - live DB pressure
|
||||
logger.warning(
|
||||
"telegram_alert_monitoring_runtime_log_readback_unavailable",
|
||||
project_id=project_id,
|
||||
error_type=type(exc).__name__,
|
||||
)
|
||||
return {
|
||||
"status": "unavailable",
|
||||
"summary": {
|
||||
"alert_operation_log_total_7d": 0,
|
||||
"telegram_sent_event_count_7d": 0,
|
||||
"km_converted_event_count_7d": 0,
|
||||
"playbook_draft_event_count_7d": 0,
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
"event_type_counts_7d": {},
|
||||
}
|
||||
|
||||
|
||||
async def _load_runtime_log_readback_once(*, project_id: str) -> dict[str, Any]:
|
||||
event_types_sql = ", ".join(f"'{event_type}'" for event_type in _ALERT_LOG_EVENT_TYPES)
|
||||
async with get_db_context(project_id) as db:
|
||||
await db.execute(text("SET LOCAL statement_timeout = '5000ms'"), {})
|
||||
event_result = await db.execute(
|
||||
text(f"""
|
||||
SELECT
|
||||
event_type,
|
||||
COUNT(*) AS count,
|
||||
MAX(created_at) AS latest_at
|
||||
FROM alert_operation_log
|
||||
WHERE created_at >= NOW() - INTERVAL '7 days'
|
||||
AND event_type IN ({event_types_sql})
|
||||
GROUP BY event_type
|
||||
"""),
|
||||
)
|
||||
outbound_result = await db.execute(
|
||||
text("""
|
||||
SELECT
|
||||
COUNT(*) AS outbound_telegram_total,
|
||||
COUNT(*) FILTER (WHERE send_status = 'sent') AS outbound_sent_total,
|
||||
COUNT(*) FILTER (
|
||||
WHERE source_envelope ? 'ai_automation_alert_card'
|
||||
) AS outbound_ai_alert_card_total,
|
||||
COUNT(*) FILTER (
|
||||
WHERE source_envelope ? 'source_refs'
|
||||
) AS outbound_source_refs_total,
|
||||
MAX(COALESCE(sent_at, queued_at)) AS latest_outbound_at
|
||||
FROM awooop_outbound_message
|
||||
WHERE project_id = :project_id
|
||||
AND channel_type = 'telegram'
|
||||
AND queued_at >= NOW() - INTERVAL '7 days'
|
||||
"""),
|
||||
{"project_id": project_id},
|
||||
)
|
||||
|
||||
event_rows = [_dict(row) for row in event_result.mappings().all()]
|
||||
event_counts = {str(row["event_type"]): _int(row["count"]) for row in event_rows}
|
||||
latest_event_at = max(
|
||||
[str(row.get("latest_at")) for row in event_rows if row.get("latest_at")],
|
||||
default=None,
|
||||
)
|
||||
outbound_row = _dict(outbound_result.mappings().first() or {})
|
||||
total = sum(event_counts.values())
|
||||
return {
|
||||
"status": "ok",
|
||||
"summary": {
|
||||
"alert_operation_log_total_7d": total,
|
||||
"telegram_sent_event_count_7d": event_counts.get("TELEGRAM_SENT", 0),
|
||||
"telegram_result_sent_event_count_7d": event_counts.get(
|
||||
"TELEGRAM_RESULT_SENT",
|
||||
0,
|
||||
),
|
||||
"km_converted_event_count_7d": event_counts.get("KM_CONVERTED", 0),
|
||||
"playbook_draft_event_count_7d": event_counts.get(
|
||||
"PLAYBOOK_DRAFT_CREATED",
|
||||
0,
|
||||
),
|
||||
"notification_classified_event_count_7d": event_counts.get(
|
||||
"NOTIFICATION_CLASSIFIED",
|
||||
0,
|
||||
),
|
||||
"outbound_telegram_total_7d": _int(
|
||||
outbound_row.get("outbound_telegram_total")
|
||||
),
|
||||
"outbound_telegram_sent_total_7d": _int(
|
||||
outbound_row.get("outbound_sent_total")
|
||||
),
|
||||
"outbound_ai_alert_card_total_7d": _int(
|
||||
outbound_row.get("outbound_ai_alert_card_total")
|
||||
),
|
||||
"outbound_source_refs_total_7d": _int(
|
||||
outbound_row.get("outbound_source_refs_total")
|
||||
),
|
||||
"latest_alert_operation_log_at": latest_event_at,
|
||||
"latest_outbound_telegram_at": str(outbound_row.get("latest_outbound_at"))
|
||||
if outbound_row.get("latest_outbound_at")
|
||||
else None,
|
||||
},
|
||||
"event_type_counts_7d": event_counts,
|
||||
}
|
||||
|
||||
|
||||
def _coverage_matrix(
|
||||
*,
|
||||
project_id: str,
|
||||
monitoring_surface_count: int,
|
||||
live_evidence_received_count: int,
|
||||
monitoring_live_gap_count: int,
|
||||
inventory_summary: Mapping[str, Any],
|
||||
readback_summary: Mapping[str, Any],
|
||||
matrix_summary: Mapping[str, Any],
|
||||
verifier_rollups: Mapping[str, Any],
|
||||
ai_alert_summary: Mapping[str, Any],
|
||||
runtime_summary: Mapping[str, Any],
|
||||
source_contract: Mapping[str, Any],
|
||||
runtime_db_ok: bool,
|
||||
ai_alert_db_ok: bool,
|
||||
verifier_ready: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
direct_gap_count = _int(matrix_summary.get("known_direct_send_gap_count"))
|
||||
ai_alert_total = _int(ai_alert_summary.get("total"))
|
||||
ai_alert_ready_total = _int(ai_alert_summary.get("learning_writeback_ready_total"))
|
||||
return [
|
||||
{
|
||||
"surface_id": "monitoring_inventory_static_scope",
|
||||
"status": (
|
||||
"live_receipt_gaps_present"
|
||||
if monitoring_live_gap_count
|
||||
else "all_monitoring_surfaces_have_live_receipts"
|
||||
),
|
||||
"known_item_count": monitoring_surface_count,
|
||||
"db_or_log_receipt": (
|
||||
"partial_waiting_live_receipts"
|
||||
if monitoring_live_gap_count
|
||||
else "ready_live_receipts_accepted"
|
||||
),
|
||||
"ai_route": "ready_inventory_can_route_to_ai_triage",
|
||||
"controlled_queue": "ready_metadata_only_no_reload_no_send",
|
||||
"post_verifier": "monitoring_post_incident_readback_plan",
|
||||
"learning_writeback": "waiting_live_receipt_context"
|
||||
if monitoring_live_gap_count
|
||||
else "ready_live_receipt_context",
|
||||
"gap_count": monitoring_live_gap_count,
|
||||
"evidence_refs": [
|
||||
f"docs/security/{_MONITORING_INVENTORY}",
|
||||
f"docs/security/{_MONITORING_POST_INCIDENT_READBACK}",
|
||||
],
|
||||
},
|
||||
{
|
||||
"surface_id": "alertmanager_to_alert_operation_log",
|
||||
"status": (
|
||||
"source_contract_present"
|
||||
if source_contract.get("alertmanager_alert_log_append_present")
|
||||
else "source_contract_missing"
|
||||
),
|
||||
"known_item_count": _int(inventory_summary.get("alert_rule_surface_count")),
|
||||
"db_or_log_receipt": "ready_alert_operation_log_event_sourcing",
|
||||
"ai_route": "ready_rule_engine_openclaw_triage",
|
||||
"controlled_queue": "ready_background_telegram_push_no_webhook_block",
|
||||
"post_verifier": "runtime_log_readback",
|
||||
"learning_writeback": "ready_km_and_playbook_event_types",
|
||||
"gap_count": 0
|
||||
if source_contract.get("alertmanager_alert_log_append_present")
|
||||
else 1,
|
||||
"evidence_refs": [
|
||||
"apps/api/src/api/v1/webhooks.py",
|
||||
"apps/api/src/repositories/alert_operation_log_repository.py",
|
||||
],
|
||||
},
|
||||
{
|
||||
"surface_id": "telegram_gateway_outbound_mirror",
|
||||
"status": "controlled_gateway_receipt_path_ready"
|
||||
if direct_gap_count == 0
|
||||
else "direct_send_gaps_present",
|
||||
"known_item_count": _int(
|
||||
matrix_summary.get("gateway_normalized_callsite_count")
|
||||
),
|
||||
"db_or_log_receipt": "ready_awooop_outbound_message_mirror",
|
||||
"ai_route": "ready_ai_automation_alert_card_mirror",
|
||||
"controlled_queue": "ready_telegram_gateway_only",
|
||||
"post_verifier": "telegram_egress_inventory_direct_gap_scan",
|
||||
"learning_writeback": "ready_alert_card_learning_refs",
|
||||
"gap_count": direct_gap_count,
|
||||
"evidence_refs": [
|
||||
"apps/api/src/services/telegram_gateway.py",
|
||||
"docs/security/telegram-notification-egress-inventory.snapshot.json",
|
||||
],
|
||||
},
|
||||
{
|
||||
"surface_id": "awooop_ai_alert_card_delivery_readback",
|
||||
"status": "db_readback_ready" if ai_alert_db_ok else "db_readback_unavailable",
|
||||
"known_item_count": ai_alert_total,
|
||||
"db_or_log_receipt": "ready_ai_alert_card_delivery_receipts"
|
||||
if ai_alert_db_ok
|
||||
else "waiting_ai_alert_card_delivery_db_readback",
|
||||
"ai_route": "ready_ai_alert_card_lane_target",
|
||||
"controlled_queue": "ready_delivery_receipt_readback_only",
|
||||
"post_verifier": "ready_learning_writeback_summary",
|
||||
"learning_writeback": "ready"
|
||||
if ai_alert_ready_total > 0
|
||||
else "waiting_ai_alert_card_learning_refs",
|
||||
"gap_count": 0 if ai_alert_db_ok and ai_alert_ready_total > 0 else 1,
|
||||
"evidence_refs": ["/api/v1/platform/runs/ai-alert-cards"],
|
||||
},
|
||||
{
|
||||
"surface_id": "runtime_alert_operation_log_stats",
|
||||
"status": "db_readback_ready" if runtime_db_ok else "db_readback_unavailable",
|
||||
"known_item_count": _int(
|
||||
runtime_summary.get("alert_operation_log_total_7d")
|
||||
),
|
||||
"db_or_log_receipt": "ready_alert_operation_log_stats"
|
||||
if runtime_db_ok
|
||||
else "waiting_alert_operation_log_db_readback",
|
||||
"ai_route": "ready_event_type_to_ai_lane",
|
||||
"controlled_queue": "ready_metadata_stats_no_payload",
|
||||
"post_verifier": "bounded_statement_timeout_readback",
|
||||
"learning_writeback": "ready_km_playbook_event_type_rollups",
|
||||
"gap_count": 0 if runtime_db_ok else 1,
|
||||
"evidence_refs": ["/api/v1/alert-operation-logs/stats"],
|
||||
},
|
||||
{
|
||||
"surface_id": "telegram_alert_ai_loop_context_verifier",
|
||||
"status": "verified_context_ready" if verifier_ready else "verifier_not_ready",
|
||||
"known_item_count": _int(
|
||||
verifier_rollups.get("verified_context_receipt_count")
|
||||
),
|
||||
"db_or_log_receipt": "ready_context_receipt_source",
|
||||
"ai_route": "ready_ai_agent_reusable_context",
|
||||
"controlled_queue": "ready_metadata_only_post_apply_verifier",
|
||||
"post_verifier": "ready",
|
||||
"learning_writeback": "ready_km_rag_mcp_playbook_ai_agent_targets",
|
||||
"gap_count": 0 if verifier_ready else 1,
|
||||
"evidence_refs": [
|
||||
"/api/v1/agents/telegram-alert-learning-context-post-apply-verifier"
|
||||
],
|
||||
},
|
||||
{
|
||||
"surface_id": "awooop_operator_surfaces",
|
||||
"status": "operator_visible_source_contract_present"
|
||||
if source_contract.get("awooop_coverage_surface_present")
|
||||
else "operator_surface_missing",
|
||||
"known_item_count": 3,
|
||||
"db_or_log_receipt": "ready_reads_coverage_endpoint",
|
||||
"ai_route": "ready_runs_work_items_alerts_visible",
|
||||
"controlled_queue": "ready_no_send_ui_readback",
|
||||
"post_verifier": "desktop_mobile_smoke_required_after_deploy",
|
||||
"learning_writeback": "ready_operator_observable_receipts",
|
||||
"gap_count": 0
|
||||
if source_contract.get("awooop_coverage_surface_present")
|
||||
else 1,
|
||||
"evidence_refs": [
|
||||
"apps/web/src/components/awooop/autonomous-runtime-receipt-panel.tsx",
|
||||
"/zh-TW/awooop/runs",
|
||||
"/zh-TW/awooop/work-items",
|
||||
"/zh-TW/awooop/alerts",
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _active_blockers(
|
||||
*,
|
||||
monitoring_live_gap_count: int,
|
||||
direct_gap_count: int,
|
||||
runtime_db_ok: bool,
|
||||
ai_alert_db_ok: bool,
|
||||
verifier_ready: bool,
|
||||
source_contract: Mapping[str, Any],
|
||||
ai_alert_total: int,
|
||||
ai_alert_ready_total: int,
|
||||
) -> list[str]:
|
||||
blockers = []
|
||||
if monitoring_live_gap_count:
|
||||
blockers.append(f"monitoring_live_receipt_gap:{monitoring_live_gap_count}")
|
||||
if direct_gap_count:
|
||||
blockers.append(f"telegram_direct_send_gap:{direct_gap_count}")
|
||||
if not runtime_db_ok:
|
||||
blockers.append("runtime_alert_operation_log_db_readback_unavailable")
|
||||
if not ai_alert_db_ok:
|
||||
blockers.append("awooop_ai_alert_card_delivery_db_readback_unavailable")
|
||||
if ai_alert_total == 0:
|
||||
blockers.append("awooop_ai_alert_card_delivery_receipts_missing")
|
||||
if ai_alert_ready_total == 0:
|
||||
blockers.append("awooop_ai_alert_card_learning_writeback_refs_missing")
|
||||
if not verifier_ready:
|
||||
blockers.append("telegram_alert_ai_loop_post_apply_verifier_not_ready")
|
||||
blockers.extend(_source_contract_blockers(source_contract))
|
||||
return _unique(blockers)
|
||||
|
||||
|
||||
def _next_controlled_actions(active_blockers: list[str]) -> list[dict[str, Any]]:
|
||||
if not active_blockers:
|
||||
return [
|
||||
{
|
||||
"action_id": "keep_monitoring_telegram_ai_loop_coverage_green",
|
||||
"priority": "P0",
|
||||
"requires_secret": False,
|
||||
"requires_runtime_send": False,
|
||||
"post_verifier": "coverage readback remains ready and smoke stays green",
|
||||
}
|
||||
]
|
||||
return [
|
||||
{
|
||||
"action_id": "ingest_monitoring_surface_live_receipts_to_db_log",
|
||||
"priority": "P0",
|
||||
"scope": "monitoring inventory -> alert_operation_log / awooop_outbound_message",
|
||||
"requires_secret": False,
|
||||
"requires_runtime_send": False,
|
||||
"post_verifier": "monitoring_live_receipt_gap_count decreases without raw payload storage",
|
||||
},
|
||||
{
|
||||
"action_id": "tag_and_cluster_monitoring_alert_logs_for_ai_agent",
|
||||
"priority": "P0",
|
||||
"scope": "alert_operation_log and outbound mirror metadata",
|
||||
"requires_secret": False,
|
||||
"requires_runtime_send": False,
|
||||
"post_verifier": "each alert row exposes product/service/tool/severity/lane/fingerprint/cluster_id",
|
||||
},
|
||||
{
|
||||
"action_id": "write_metadata_receipts_to_km_rag_mcp_playbook_context",
|
||||
"priority": "P0",
|
||||
"scope": "KM / RAG / MCP / PlayBook / verifier / AI Agent context",
|
||||
"requires_secret": False,
|
||||
"requires_runtime_send": False,
|
||||
"post_verifier": "post-apply verifier reports verified targets and raw_payload_included_count=0",
|
||||
},
|
||||
{
|
||||
"action_id": "surface_coverage_gaps_in_awooop_runs_work_items_alerts",
|
||||
"priority": "P0",
|
||||
"scope": "AwoooP operator shared receipt panel",
|
||||
"requires_secret": False,
|
||||
"requires_runtime_send": False,
|
||||
"post_verifier": "desktop/mobile smoke shows coverage cards without manual-default terminal state",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _controlled_apply_packet(
|
||||
*,
|
||||
project_id: str,
|
||||
active_blockers: list[str],
|
||||
monitoring_live_gap_count: int,
|
||||
direct_gap_count: int,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"mode": "ai_controlled_metadata_coverage_apply_packet",
|
||||
"controlled_apply_allowed_for_low_medium_high": True,
|
||||
"owner_review_required_for_low_medium_high": False,
|
||||
"critical_break_glass_required": True,
|
||||
"target_selector": {
|
||||
"project_id": project_id,
|
||||
"work_item_id": "CIR-P0-TG-001",
|
||||
"target_surfaces": [
|
||||
"alert_operation_log",
|
||||
"awooop_outbound_message",
|
||||
"km",
|
||||
"rag",
|
||||
"mcp",
|
||||
"playbook",
|
||||
"ai_agent_context",
|
||||
],
|
||||
},
|
||||
"source_of_truth_diff": {
|
||||
"current_state": "partial_monitoring_live_receipt_coverage"
|
||||
if active_blockers
|
||||
else "monitoring_telegram_ai_loop_coverage_ready",
|
||||
"desired_state": "all_monitoring_alerts_metadata_receipted_and_ai_reusable",
|
||||
"monitoring_live_gap_count": monitoring_live_gap_count,
|
||||
"direct_telegram_gap_count": direct_gap_count,
|
||||
},
|
||||
"check_mode": {
|
||||
"enabled": True,
|
||||
"checks": [
|
||||
"direct_bot_api_gap_count_zero",
|
||||
"db_log_receipt_readback",
|
||||
"metadata_only_raw_payload_absent",
|
||||
"km_rag_mcp_playbook_refs_present",
|
||||
"post_apply_verifier_ready",
|
||||
],
|
||||
},
|
||||
"rollback": {
|
||||
"required": True,
|
||||
"strategy": "supersede_metadata_receipt_projection_without_deleting_source_events",
|
||||
"rollback_ref": f"rollback://{project_id}/telegram-monitoring-coverage/CIR-P0-TG-001",
|
||||
},
|
||||
"post_apply_verifier": {
|
||||
"required": True,
|
||||
"verifier_endpoint": (
|
||||
"/api/v1/agents/telegram-alert-monitoring-coverage-readback"
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _inspect_source_contract(repo_root: Path) -> dict[str, bool]:
|
||||
checks = {
|
||||
"alertmanager_alert_log_append_present": (
|
||||
"apps/api/src/api/v1/webhooks.py",
|
||||
"get_alert_operation_log_repository",
|
||||
),
|
||||
"alert_operation_log_telegram_sent_present": (
|
||||
"apps/api/src/api/v1/webhooks.py",
|
||||
'"TELEGRAM_SENT"',
|
||||
),
|
||||
"alert_operation_log_repository_present": (
|
||||
"apps/api/src/repositories/alert_operation_log_repository.py",
|
||||
"ALERT_EVENT_TYPES",
|
||||
),
|
||||
"telegram_gateway_outbound_mirror_present": (
|
||||
"apps/api/src/services/telegram_gateway.py",
|
||||
"_mirror_outbound_message",
|
||||
),
|
||||
"telegram_gateway_outbound_record_present": (
|
||||
"apps/api/src/services/telegram_gateway.py",
|
||||
"record_outbound_message",
|
||||
),
|
||||
"ai_alert_card_delivery_readback_present": (
|
||||
"apps/api/src/services/platform_operator_service.py",
|
||||
"list_ai_alert_card_delivery_readback",
|
||||
),
|
||||
"ai_alert_card_learning_refs_present": (
|
||||
"apps/api/src/services/platform_operator_service.py",
|
||||
"ai_alert_card_learning_writeback_refs_v1",
|
||||
),
|
||||
"post_apply_verifier_present": (
|
||||
"apps/api/src/services/telegram_alert_learning_context_post_apply_verifier.py",
|
||||
"telegram_alert_learning_context_post_apply_verifier_v1",
|
||||
),
|
||||
"awooop_coverage_surface_present": (
|
||||
"apps/web/src/components/awooop/autonomous-runtime-receipt-panel.tsx",
|
||||
"telegram-alert-monitoring-coverage-readback",
|
||||
),
|
||||
}
|
||||
result: dict[str, bool] = {}
|
||||
for key, (relative_path, marker) in checks.items():
|
||||
text_value = _read_source_contract_text(repo_root, relative_path)
|
||||
result[key] = marker in text_value
|
||||
return result
|
||||
|
||||
|
||||
def _source_contract_blockers(source_contract: Mapping[str, Any]) -> list[str]:
|
||||
return [
|
||||
f"source_contract_missing:{key}"
|
||||
for key, value in source_contract.items()
|
||||
if value is not True
|
||||
]
|
||||
|
||||
|
||||
def _load_required_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"required monitoring coverage source missing: {path}")
|
||||
with path.open(encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"{path}: expected JSON object")
|
||||
return payload
|
||||
|
||||
|
||||
def _require_schema(payload: Mapping[str, Any], expected: str, label: str) -> None:
|
||||
if payload.get("schema_version") != expected:
|
||||
raise ValueError(
|
||||
f"{label}: expected schema_version={expected}, got {payload.get('schema_version')!r}"
|
||||
)
|
||||
|
||||
|
||||
def _require_safe_boundaries(*payloads: Mapping[str, Any]) -> None:
|
||||
forbidden_true_keys = {
|
||||
"telegram_send_authorized",
|
||||
"bot_api_call_authorized",
|
||||
"runtime_execution_authorized",
|
||||
"runtime_gate",
|
||||
"runtime_gate_open",
|
||||
"production_write_authorized",
|
||||
"raw_alert_payload_storage_allowed",
|
||||
"raw_payload_storage_allowed",
|
||||
"secret_value_collection_allowed",
|
||||
"alertmanager_reload_authorized",
|
||||
"receiver_route_change_authorized",
|
||||
"workflow_dispatch_authorized",
|
||||
}
|
||||
for payload in payloads:
|
||||
for section_key in ("operation_boundaries", "execution_boundaries"):
|
||||
section = payload.get(section_key)
|
||||
if not isinstance(section, Mapping):
|
||||
continue
|
||||
unsafe = sorted(
|
||||
key
|
||||
for key in forbidden_true_keys
|
||||
if section.get(key) not in (None, False)
|
||||
)
|
||||
if unsafe:
|
||||
raise ValueError(
|
||||
f"{payload.get('schema_version')}.{section_key}: unsafe true flags {unsafe}"
|
||||
)
|
||||
|
||||
|
||||
def _read_source_contract_text(repo_root: Path, relative_path: str) -> str:
|
||||
candidates = [repo_root / relative_path]
|
||||
api_prefix = "apps/api/"
|
||||
if relative_path.startswith(api_prefix):
|
||||
candidates.append(repo_root / relative_path.removeprefix(api_prefix))
|
||||
|
||||
for path in candidates:
|
||||
if path.exists():
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
return ""
|
||||
|
||||
|
||||
def _dict(value: Any) -> dict[str, Any]:
|
||||
return dict(value) if isinstance(value, Mapping) else {}
|
||||
|
||||
|
||||
def _int(value: Any) -> int:
|
||||
try:
|
||||
return int(value or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _unique(values: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
unique: list[str] = []
|
||||
for value in values:
|
||||
if value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
unique.append(value)
|
||||
return unique
|
||||
@@ -0,0 +1,218 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from src.api.v1 import agents
|
||||
from src.api.v1.agents import router
|
||||
from src.services.telegram_alert_monitoring_coverage_readback import (
|
||||
_inspect_source_contract,
|
||||
build_telegram_alert_monitoring_coverage_readback,
|
||||
)
|
||||
|
||||
|
||||
def _source_contract() -> dict[str, bool]:
|
||||
return {
|
||||
"alertmanager_alert_log_append_present": True,
|
||||
"alert_operation_log_telegram_sent_present": True,
|
||||
"alert_operation_log_repository_present": True,
|
||||
"telegram_gateway_outbound_mirror_present": True,
|
||||
"telegram_gateway_outbound_record_present": True,
|
||||
"ai_alert_card_delivery_readback_present": True,
|
||||
"ai_alert_card_learning_refs_present": True,
|
||||
"post_apply_verifier_present": True,
|
||||
"awooop_coverage_surface_present": True,
|
||||
}
|
||||
|
||||
|
||||
def _payload() -> dict:
|
||||
return build_telegram_alert_monitoring_coverage_readback(
|
||||
project_id="awoooi",
|
||||
monitoring_inventory={
|
||||
"schema_version": "monitoring_alerting_observability_inventory_v1",
|
||||
"summary": {
|
||||
"surface_count": 60,
|
||||
"source_exists_count": 60,
|
||||
"live_evidence_received_count": 0,
|
||||
"alert_rule_surface_count": 13,
|
||||
"telegram_surface_count": 3,
|
||||
},
|
||||
},
|
||||
monitoring_readback_plan={
|
||||
"schema_version": "monitoring_post_incident_readback_plan_v1",
|
||||
"summary": {
|
||||
"readback_candidate_count": 60,
|
||||
},
|
||||
},
|
||||
telegram_matrix={
|
||||
"schema_version": "telegram_alert_ai_automation_matrix_v1",
|
||||
"status": "telegram_alert_ai_automation_matrix_ready",
|
||||
"summary": {
|
||||
"telegram_alert_surface_count": 9,
|
||||
"known_direct_send_gap_count": 0,
|
||||
"gateway_normalized_callsite_count": 57,
|
||||
"db_or_log_receipt_ready_surface_count": 9,
|
||||
"ai_route_ready_surface_count": 9,
|
||||
"controlled_queue_ready_surface_count": 9,
|
||||
"post_verifier_ready_surface_count": 9,
|
||||
"learning_writeback_ready_surface_count": 9,
|
||||
},
|
||||
},
|
||||
post_apply_verifier={
|
||||
"schema_version": "telegram_alert_learning_context_post_apply_verifier_v1",
|
||||
"status": "telegram_alert_learning_context_post_apply_verified",
|
||||
"rollups": {
|
||||
"telegram_alert_learning_context_post_apply_verifier_ready": True,
|
||||
"verified_context_receipt_count": 36,
|
||||
"verified_target_count": 6,
|
||||
"verified_ai_agent_context_receipt_count": 6,
|
||||
},
|
||||
},
|
||||
ai_alert_card_delivery_readback={
|
||||
"status": "ok",
|
||||
"summary": {
|
||||
"total": 36,
|
||||
"learning_writeback_ready_total": 36,
|
||||
},
|
||||
},
|
||||
runtime_log_readback={
|
||||
"status": "ok",
|
||||
"summary": {
|
||||
"alert_operation_log_total_7d": 41,
|
||||
"telegram_sent_event_count_7d": 8,
|
||||
"km_converted_event_count_7d": 4,
|
||||
"playbook_draft_event_count_7d": 3,
|
||||
},
|
||||
},
|
||||
source_contract=_source_contract(),
|
||||
)
|
||||
|
||||
|
||||
def test_telegram_alert_monitoring_coverage_readback_surfaces_live_gaps():
|
||||
payload = _payload()
|
||||
|
||||
assert payload["schema_version"] == (
|
||||
"telegram_alert_monitoring_coverage_readback_v1"
|
||||
)
|
||||
assert payload["priority_work_item_id"] == "CIR-P0-TG-001"
|
||||
assert payload["status"] == (
|
||||
"blocked_telegram_alert_monitoring_coverage_gaps_present"
|
||||
)
|
||||
assert payload["operator_answer"]["all_known_telegram_egress_routes_controlled"] is True
|
||||
assert (
|
||||
payload["operator_answer"][
|
||||
"all_known_telegram_egress_routes_have_db_or_log_receipt"
|
||||
]
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
payload["operator_answer"][
|
||||
"all_verified_ai_alert_context_receipts_reusable_by_ai_agent"
|
||||
]
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
payload["operator_answer"][
|
||||
"all_monitoring_inventory_surfaces_have_accepted_live_receipt"
|
||||
]
|
||||
is False
|
||||
)
|
||||
assert payload["operator_answer"]["manual_default_terminal_state_allowed"] is False
|
||||
|
||||
summary = payload["summary"]
|
||||
assert summary["monitoring_surface_count"] == 60
|
||||
assert summary["monitoring_live_receipt_gap_count"] == 60
|
||||
assert summary["known_direct_send_gap_count"] == 0
|
||||
assert summary["ai_alert_card_delivery_total"] == 36
|
||||
assert summary["verified_context_receipt_count"] == 36
|
||||
assert summary["verified_target_count"] == 6
|
||||
assert summary["runtime_telegram_sent_event_count_7d"] == 8
|
||||
assert summary["source_contract_ready"] is True
|
||||
assert "monitoring_live_receipt_gap:60" in payload["active_blockers"]
|
||||
|
||||
matrix = {item["surface_id"]: item for item in payload["coverage_matrix"]}
|
||||
assert matrix["telegram_gateway_outbound_mirror"]["gap_count"] == 0
|
||||
assert matrix["telegram_alert_ai_loop_context_verifier"]["status"] == (
|
||||
"verified_context_ready"
|
||||
)
|
||||
assert matrix["monitoring_inventory_static_scope"]["gap_count"] == 60
|
||||
assert payload["operation_boundaries"]["telegram_send_performed"] is False
|
||||
assert payload["operation_boundaries"]["raw_alert_payload_stored"] is False
|
||||
assert payload["operation_boundaries"]["secret_value_read"] is False
|
||||
|
||||
serialized = json.dumps(payload, ensure_ascii=False)
|
||||
for marker in [
|
||||
"TELEGRAM_BOT_TOKEN",
|
||||
"authorization_header",
|
||||
"raw Telegram payload",
|
||||
"批准!繼續",
|
||||
"My request for Codex",
|
||||
]:
|
||||
assert marker not in serialized
|
||||
|
||||
|
||||
def test_telegram_alert_monitoring_coverage_endpoint(monkeypatch):
|
||||
async def fake_loader():
|
||||
return _payload()
|
||||
|
||||
monkeypatch.setattr(
|
||||
agents,
|
||||
"load_latest_telegram_alert_monitoring_coverage_readback",
|
||||
fake_loader,
|
||||
)
|
||||
app = FastAPI()
|
||||
app.include_router(router, prefix="/api/v1")
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get(
|
||||
"/api/v1/agents/telegram-alert-monitoring-coverage-readback"
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["schema_version"] == (
|
||||
"telegram_alert_monitoring_coverage_readback_v1"
|
||||
)
|
||||
assert data["summary"]["monitoring_live_receipt_gap_count"] == 60
|
||||
assert data["summary"]["known_direct_send_gap_count"] == 0
|
||||
assert data["operation_boundaries"]["bot_api_call_performed"] is False
|
||||
|
||||
|
||||
def test_telegram_alert_monitoring_coverage_source_contract(tmp_path):
|
||||
(tmp_path / "apps/api/src/api/v1").mkdir(parents=True)
|
||||
(tmp_path / "apps/api/src/repositories").mkdir(parents=True)
|
||||
(tmp_path / "apps/api/src/services").mkdir(parents=True)
|
||||
(tmp_path / "apps/web/src/components/awooop").mkdir(parents=True)
|
||||
(tmp_path / "apps/api/src/api/v1/webhooks.py").write_text(
|
||||
"get_alert_operation_log_repository\n"
|
||||
'"TELEGRAM_SENT"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "apps/api/src/repositories/alert_operation_log_repository.py").write_text(
|
||||
"ALERT_EVENT_TYPES\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "apps/api/src/services/telegram_gateway.py").write_text(
|
||||
"_mirror_outbound_message\n"
|
||||
"record_outbound_message\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "apps/api/src/services/platform_operator_service.py").write_text(
|
||||
"list_ai_alert_card_delivery_readback\n"
|
||||
"ai_alert_card_learning_writeback_refs_v1\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "apps/api/src/services/telegram_alert_learning_context_post_apply_verifier.py").write_text(
|
||||
"telegram_alert_learning_context_post_apply_verifier_v1\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "apps/web/src/components/awooop/autonomous-runtime-receipt-panel.tsx").write_text(
|
||||
"telegram-alert-monitoring-coverage-readback\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
proof = _inspect_source_contract(tmp_path)
|
||||
|
||||
assert proof == _source_contract()
|
||||
@@ -11843,6 +11843,24 @@
|
||||
"blockers": "Active blockers",
|
||||
"noBlockers": "No active blocker"
|
||||
},
|
||||
"monitoringCoverage": {
|
||||
"title": "Telegram monitoring coverage",
|
||||
"subtitle": "Monitoring alerts aligned to DB/log receipts, AI routes, controlled queue, and KM/RAG/MCP/PlayBook context.",
|
||||
"status": "Coverage status",
|
||||
"statusDetail": "coverage endpoint readback",
|
||||
"direct": "Direct gaps",
|
||||
"directDetail": "gateway controlled: {ready}",
|
||||
"monitoring": "Live receipts",
|
||||
"monitoringDetail": "gap {gap}",
|
||||
"delivery": "AI alert delivery",
|
||||
"deliveryDetail": "learning refs ready / delivery receipts",
|
||||
"verifier": "AI context",
|
||||
"verifierDetail": "verified receipts {receipts}",
|
||||
"logs": "7d alert log",
|
||||
"logsDetail": "TG {telegram} / KM {km} / PB {playbook}",
|
||||
"blockers": "Active blockers",
|
||||
"noBlockers": "No active blocker"
|
||||
},
|
||||
"recent": "近 24h {count}",
|
||||
"missing": "缺 {count} 節點",
|
||||
"closedDetail": "required stages ok",
|
||||
|
||||
@@ -11843,6 +11843,24 @@
|
||||
"blockers": "Active blockers",
|
||||
"noBlockers": "無 active blocker"
|
||||
},
|
||||
"monitoringCoverage": {
|
||||
"title": "Telegram monitoring coverage",
|
||||
"subtitle": "監控告警對齊 DB/log receipt、AI route、controlled queue 與 KM/RAG/MCP/PlayBook context。",
|
||||
"status": "Coverage status",
|
||||
"statusDetail": "coverage endpoint readback",
|
||||
"direct": "Direct gaps",
|
||||
"directDetail": "gateway controlled:{ready}",
|
||||
"monitoring": "Live receipts",
|
||||
"monitoringDetail": "gap {gap}",
|
||||
"delivery": "AI alert delivery",
|
||||
"deliveryDetail": "learning refs ready / delivery receipts",
|
||||
"verifier": "AI context",
|
||||
"verifierDetail": "verified receipts {receipts}",
|
||||
"logs": "7d alert log",
|
||||
"logsDetail": "TG {telegram} / KM {km} / PB {playbook}",
|
||||
"blockers": "Active blockers",
|
||||
"noBlockers": "無 active blocker"
|
||||
},
|
||||
"recent": "近 24h {count}",
|
||||
"missing": "缺 {count} 節點",
|
||||
"closedDetail": "required stages ok",
|
||||
|
||||
@@ -280,6 +280,47 @@ type TelegramPostApplyVerifierPayload = {
|
||||
} | null;
|
||||
};
|
||||
|
||||
type TelegramMonitoringCoveragePayload = {
|
||||
status?: string | null;
|
||||
active_blockers?: string[] | null;
|
||||
operator_answer?: {
|
||||
all_known_telegram_egress_routes_controlled?: boolean | null;
|
||||
all_known_telegram_egress_routes_have_db_or_log_receipt?: boolean | null;
|
||||
all_verified_ai_alert_context_receipts_reusable_by_ai_agent?: boolean | null;
|
||||
all_monitoring_inventory_surfaces_have_accepted_live_receipt?: boolean | null;
|
||||
manual_default_terminal_state_allowed?: boolean | null;
|
||||
} | null;
|
||||
summary?: {
|
||||
monitoring_surface_count?: number | null;
|
||||
monitoring_live_evidence_received_count?: number | null;
|
||||
monitoring_live_receipt_gap_count?: number | null;
|
||||
known_direct_send_gap_count?: number | null;
|
||||
db_or_log_receipt_ready_surface_count?: number | null;
|
||||
ai_route_ready_surface_count?: number | null;
|
||||
controlled_queue_ready_surface_count?: number | null;
|
||||
ai_alert_card_delivery_total?: number | null;
|
||||
ai_alert_card_learning_writeback_ready_total?: number | null;
|
||||
runtime_alert_operation_log_total_7d?: number | null;
|
||||
runtime_telegram_sent_event_count_7d?: number | null;
|
||||
runtime_km_converted_event_count_7d?: number | null;
|
||||
runtime_playbook_draft_event_count_7d?: number | null;
|
||||
verified_context_receipt_count?: number | null;
|
||||
verified_target_count?: number | null;
|
||||
source_contract_ready?: boolean | null;
|
||||
runtime_db_readback_ok?: boolean | null;
|
||||
ai_alert_card_db_readback_ok?: boolean | null;
|
||||
full_coverage_ready?: boolean | null;
|
||||
} | null;
|
||||
operation_boundaries?: {
|
||||
telegram_send_performed?: boolean | null;
|
||||
bot_api_call_performed?: boolean | null;
|
||||
runtime_write_performed?: boolean | null;
|
||||
raw_alert_payload_stored?: boolean | null;
|
||||
secret_value_read?: boolean | null;
|
||||
github_api_used?: boolean | null;
|
||||
} | null;
|
||||
};
|
||||
|
||||
type PanelMode = "full" | "compact";
|
||||
type Tone = "ok" | "warn" | "neutral";
|
||||
type WorkFilter = "all" | "completed" | "active" | "pending" | "blocked";
|
||||
@@ -421,6 +462,23 @@ async function fetchTelegramPostApplyVerifier(): Promise<TelegramPostApplyVerifi
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchTelegramMonitoringCoverage(): Promise<TelegramMonitoringCoveragePayload | null> {
|
||||
const controller = new AbortController();
|
||||
const timeout = window.setTimeout(() => controller.abort(), 12_000);
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/api/v1/agents/telegram-alert-monitoring-coverage-readback`, {
|
||||
cache: "no-store",
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
return (await response.json()) as TelegramMonitoringCoveragePayload;
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
window.clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
export function AutonomousRuntimeReceiptPanel({
|
||||
mode = "full",
|
||||
}: {
|
||||
@@ -432,6 +490,7 @@ export function AutonomousRuntimeReceiptPanel({
|
||||
const [priorityPayload, setPriorityPayload] = useState<PriorityWorkOrderPayload | null>(null);
|
||||
const [consumerPayload, setConsumerPayload] = useState<LogConsumerReadbackPayload | null>(null);
|
||||
const [telegramVerifierPayload, setTelegramVerifierPayload] = useState<TelegramPostApplyVerifierPayload | null>(null);
|
||||
const [telegramCoveragePayload, setTelegramCoveragePayload] = useState<TelegramMonitoringCoveragePayload | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState(false);
|
||||
const [updatedAt, setUpdatedAt] = useState<Date | null>(null);
|
||||
@@ -439,16 +498,18 @@ export function AutonomousRuntimeReceiptPanel({
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
setLoading(true);
|
||||
const [next, priority, consumer, telegramVerifier] = await Promise.all([
|
||||
const [next, priority, consumer, telegramVerifier, telegramCoverage] = await Promise.all([
|
||||
fetchRuntimeControl(),
|
||||
fetchPriorityWorkOrder(),
|
||||
fetchLogConsumerReadback(),
|
||||
fetchTelegramPostApplyVerifier(),
|
||||
fetchTelegramMonitoringCoverage(),
|
||||
]);
|
||||
setPayload(next);
|
||||
setPriorityPayload(priority);
|
||||
setConsumerPayload(consumer);
|
||||
setTelegramVerifierPayload(telegramVerifier);
|
||||
setTelegramCoveragePayload(telegramCoverage);
|
||||
setError(next === null);
|
||||
setUpdatedAt(next ? new Date() : null);
|
||||
setLoading(false);
|
||||
@@ -464,6 +525,7 @@ export function AutonomousRuntimeReceiptPanel({
|
||||
const hasPriorityReadback = priorityPayload !== null;
|
||||
const hasConsumerReadback = consumerPayload !== null;
|
||||
const hasTelegramVerifierReadback = telegramVerifierPayload !== null;
|
||||
const hasTelegramCoverageReadback = telegramCoveragePayload !== null;
|
||||
const readback = payload?.runtime_receipt_readback;
|
||||
const ledger = readback?.autonomous_execution_loop_ledger;
|
||||
const traceLedger = readback?.trace_ledger;
|
||||
@@ -485,11 +547,17 @@ export function AutonomousRuntimeReceiptPanel({
|
||||
const consumerBlockers = consumerPayload?.active_blockers ?? [];
|
||||
const telegramVerifierRollups = telegramVerifierPayload?.rollups ?? {};
|
||||
const telegramVerifierBlockers = telegramVerifierPayload?.active_blockers ?? [];
|
||||
const telegramCoverageSummary = telegramCoveragePayload?.summary ?? {};
|
||||
const telegramCoverageAnswers = telegramCoveragePayload?.operator_answer ?? {};
|
||||
const telegramCoverageBlockers = telegramCoveragePayload?.active_blockers ?? [];
|
||||
const consumerReady = consumerRollups.controlled_consumer_readback_ready === true
|
||||
|| consumerPayload?.status === "controlled_writeback_consumer_readback_ready";
|
||||
const telegramPostApplyVerifierReady =
|
||||
telegramVerifierRollups.telegram_alert_learning_context_post_apply_verifier_ready === true
|
||||
|| telegramVerifierPayload?.status === "telegram_alert_learning_context_post_apply_verified";
|
||||
const telegramMonitoringCoverageReady =
|
||||
telegramCoverageSummary.full_coverage_ready === true
|
||||
|| telegramCoveragePayload?.status === "telegram_alert_monitoring_coverage_ready";
|
||||
const runtimeTargetWritePerformed = consumerRollups.runtime_target_write_performed === true
|
||||
|| consumerPayload?.controlled_consume?.runtime_target_write_performed === true
|
||||
|| consumerPayload?.operation_boundaries?.runtime_target_write_performed === true
|
||||
@@ -1056,6 +1124,140 @@ export function AutonomousRuntimeReceiptPanel({
|
||||
: telegramVerifierBlockers.length > 0 ? "warn" as Tone : "ok" as Tone,
|
||||
},
|
||||
];
|
||||
const telegramCoverageCards = [
|
||||
{
|
||||
key: "status",
|
||||
label: t("monitoringCoverage.status"),
|
||||
value: !hasTelegramCoverageReadback
|
||||
? t("states.loading")
|
||||
: telegramMonitoringCoverageReady
|
||||
? t("states.closed")
|
||||
: t("states.open"),
|
||||
detail: t("monitoringCoverage.statusDetail"),
|
||||
icon: ShieldCheck,
|
||||
tone: !hasTelegramCoverageReadback
|
||||
? "neutral" as Tone
|
||||
: telegramMonitoringCoverageReady ? "ok" as Tone : "warn" as Tone,
|
||||
},
|
||||
{
|
||||
key: "direct",
|
||||
label: t("monitoringCoverage.direct"),
|
||||
value: readbackNumber(
|
||||
telegramCoverageSummary.known_direct_send_gap_count,
|
||||
hasTelegramCoverageReadback
|
||||
),
|
||||
detail: t("monitoringCoverage.directDetail", {
|
||||
ready: telegramCoverageAnswers.all_known_telegram_egress_routes_controlled
|
||||
? t("proof.ok")
|
||||
: t("states.open"),
|
||||
}),
|
||||
icon: Send,
|
||||
tone: !hasTelegramCoverageReadback
|
||||
? "neutral" as Tone
|
||||
: toNumber(telegramCoverageSummary.known_direct_send_gap_count) === 0
|
||||
? "ok" as Tone
|
||||
: "warn" as Tone,
|
||||
},
|
||||
{
|
||||
key: "monitoring",
|
||||
label: t("monitoringCoverage.monitoring"),
|
||||
value: readbackRatio(
|
||||
telegramCoverageSummary.monitoring_live_evidence_received_count,
|
||||
telegramCoverageSummary.monitoring_surface_count,
|
||||
hasTelegramCoverageReadback
|
||||
),
|
||||
detail: t("monitoringCoverage.monitoringDetail", {
|
||||
gap: readbackNumber(
|
||||
telegramCoverageSummary.monitoring_live_receipt_gap_count,
|
||||
hasTelegramCoverageReadback
|
||||
),
|
||||
}),
|
||||
icon: Bell,
|
||||
tone: !hasTelegramCoverageReadback
|
||||
? "neutral" as Tone
|
||||
: toNumber(telegramCoverageSummary.monitoring_live_receipt_gap_count) === 0
|
||||
? "ok" as Tone
|
||||
: "warn" as Tone,
|
||||
},
|
||||
{
|
||||
key: "delivery",
|
||||
label: t("monitoringCoverage.delivery"),
|
||||
value: readbackRatio(
|
||||
telegramCoverageSummary.ai_alert_card_learning_writeback_ready_total,
|
||||
telegramCoverageSummary.ai_alert_card_delivery_total,
|
||||
hasTelegramCoverageReadback
|
||||
),
|
||||
detail: t("monitoringCoverage.deliveryDetail"),
|
||||
icon: Database,
|
||||
tone: !hasTelegramCoverageReadback
|
||||
? "neutral" as Tone
|
||||
: telegramCoverageSummary.ai_alert_card_db_readback_ok === true
|
||||
&& toNumber(telegramCoverageSummary.ai_alert_card_learning_writeback_ready_total) > 0
|
||||
? "ok" as Tone
|
||||
: "warn" as Tone,
|
||||
},
|
||||
{
|
||||
key: "verifier",
|
||||
label: t("monitoringCoverage.verifier"),
|
||||
value: readbackRatio(
|
||||
telegramCoverageSummary.verified_target_count,
|
||||
6,
|
||||
hasTelegramCoverageReadback
|
||||
),
|
||||
detail: t("monitoringCoverage.verifierDetail", {
|
||||
receipts: readbackNumber(
|
||||
telegramCoverageSummary.verified_context_receipt_count,
|
||||
hasTelegramCoverageReadback
|
||||
),
|
||||
}),
|
||||
icon: Bot,
|
||||
tone: !hasTelegramCoverageReadback
|
||||
? "neutral" as Tone
|
||||
: telegramCoverageAnswers.all_verified_ai_alert_context_receipts_reusable_by_ai_agent
|
||||
? "ok" as Tone
|
||||
: "warn" as Tone,
|
||||
},
|
||||
{
|
||||
key: "logs",
|
||||
label: t("monitoringCoverage.logs"),
|
||||
value: readbackNumber(
|
||||
telegramCoverageSummary.runtime_alert_operation_log_total_7d,
|
||||
hasTelegramCoverageReadback
|
||||
),
|
||||
detail: t("monitoringCoverage.logsDetail", {
|
||||
telegram: readbackNumber(
|
||||
telegramCoverageSummary.runtime_telegram_sent_event_count_7d,
|
||||
hasTelegramCoverageReadback
|
||||
),
|
||||
km: readbackNumber(
|
||||
telegramCoverageSummary.runtime_km_converted_event_count_7d,
|
||||
hasTelegramCoverageReadback
|
||||
),
|
||||
playbook: readbackNumber(
|
||||
telegramCoverageSummary.runtime_playbook_draft_event_count_7d,
|
||||
hasTelegramCoverageReadback
|
||||
),
|
||||
}),
|
||||
icon: Activity,
|
||||
tone: !hasTelegramCoverageReadback
|
||||
? "neutral" as Tone
|
||||
: telegramCoverageSummary.runtime_db_readback_ok === true ? "ok" as Tone : "warn" as Tone,
|
||||
},
|
||||
{
|
||||
key: "blockers",
|
||||
label: t("monitoringCoverage.blockers"),
|
||||
value: readbackNumber(telegramCoverageBlockers.length, hasTelegramCoverageReadback),
|
||||
detail: !hasTelegramCoverageReadback
|
||||
? t("states.loading")
|
||||
: telegramCoverageBlockers.length > 0
|
||||
? telegramCoverageBlockers[0]
|
||||
: t("monitoringCoverage.noBlockers"),
|
||||
icon: TriangleAlert,
|
||||
tone: !hasTelegramCoverageReadback
|
||||
? "neutral" as Tone
|
||||
: telegramCoverageBlockers.length > 0 ? "warn" as Tone : "ok" as Tone,
|
||||
},
|
||||
];
|
||||
const visibleWorkItems = orderedWorkItems.filter((item) => matchesWorkFilter(item, workFilter));
|
||||
const orderedCompleted = orderedWorkItems.filter((item) => item.status === "completed").length;
|
||||
const orderedActive = orderedWorkItems.filter((item) => (
|
||||
@@ -1249,6 +1451,54 @@ export function AutonomousRuntimeReceiptPanel({
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
data-testid="telegram-alert-monitoring-coverage-readback"
|
||||
className="border-t border-[#e0ddd4] bg-white"
|
||||
>
|
||||
<div className="flex flex-wrap items-start justify-between gap-3 px-4 py-3">
|
||||
<div className="min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
<Bell className="h-4 w-4 text-[#87867f]" aria-hidden="true" />
|
||||
<h4 className="text-sm font-semibold text-[#141413]">{t("monitoringCoverage.title")}</h4>
|
||||
</div>
|
||||
<p className="mt-1 text-xs leading-5 text-[#5f5b52]">
|
||||
{t("monitoringCoverage.subtitle")}
|
||||
</p>
|
||||
</div>
|
||||
<span className={cn("inline-flex border px-2 py-0.5 text-xs font-semibold", toneClass(
|
||||
!hasTelegramCoverageReadback
|
||||
? "neutral"
|
||||
: telegramMonitoringCoverageReady ? "ok" : "warn"
|
||||
))}>
|
||||
{!hasTelegramCoverageReadback
|
||||
? t("states.loading")
|
||||
: telegramCoveragePayload?.status ?? "--"}
|
||||
</span>
|
||||
</div>
|
||||
<div className="grid gap-px border-t border-[#e0ddd4] bg-[#e0ddd4] md:grid-cols-3 xl:grid-cols-7">
|
||||
{telegramCoverageCards.map((card) => {
|
||||
const Icon = card.icon;
|
||||
return (
|
||||
<div key={card.key} className="bg-white px-4 py-3">
|
||||
<div className="flex items-start justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<p className="text-xs font-semibold text-[#77736a]">{card.label}</p>
|
||||
<p className="mt-2 truncate font-mono text-lg font-semibold text-[#141413]">
|
||||
{card.value}
|
||||
</p>
|
||||
</div>
|
||||
<span className={cn("flex h-8 w-8 shrink-0 items-center justify-center border", toneClass(card.tone))}>
|
||||
<Icon className="h-4 w-4" aria-hidden="true" />
|
||||
</span>
|
||||
</div>
|
||||
<p className="mt-1 line-clamp-2 text-xs leading-5 text-[#5f5b52]">
|
||||
{card.detail}
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-px bg-[#e0ddd4] md:grid-cols-5 xl:grid-cols-12">
|
||||
|
||||
Reference in New Issue
Block a user