234 lines
8.1 KiB
Python
234 lines
8.1 KiB
Python
# services/event_router.py
|
||
#
|
||
# ADR-012 §③: 單一入口 dispatch(event) — L0/L1/L2 分流
|
||
# W2-C: L2 優先走 Elephant Alpha Orchestrator；EA 不可用時 fallback AIOrchestrator
|
||
#
|
||
import asyncio
|
||
import logging
|
||
import threading
|
||
import traceback
|
||
import time
|
||
from typing import Any, Dict, Optional
|
||
|
||
from services.ai_orchestrator import AIOrchestrator
|
||
from services.telegram_templates import send_telegram_with_result, triaged_alert
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
async def _handle_l1(event: Dict[str, Any], session_id: str) -> Dict[str, Any]:
|
||
"""L1: semantic translation + reason analysis (Hermes)."""
|
||
orchestrator = AIOrchestrator()
|
||
return await orchestrator.handle_l1(event, session_id)
|
||
|
||
|
||
async def _handle_l2(event: Dict[str, Any], session_id: str) -> Dict[str, Any]:
|
||
"""
|
||
L2: W2-C — EA Orchestrator 優先（動態路由 256K ctx）；
|
||
EA 不可用（API key 未設或連線失敗）時 fallback AIOrchestrator。
|
||
ADR-012: audit trail 由 EA._log_decision + triaged_alert 雙寫保證。
|
||
"""
|
||
try:
|
||
from services.elephant_service import elephant_service
|
||
from services.elephant_alpha_orchestrator import elephant_orchestrator
|
||
|
||
# 護欄：EA API key 未設定則直接 fallback，不嘗試連線
|
||
if not elephant_service.api_key:
|
||
raise RuntimeError("OPENROUTER_API_KEY not configured, using fallback")
|
||
|
||
# 護欄：連線快取確認（W3-A cache 300s，不會每次都 ping）
|
||
if not elephant_service.check_connection():
|
||
raise RuntimeError("EA connection unavailable, using fallback")
|
||
|
||
decision = await elephant_orchestrator.analyze_and_coordinate({
|
||
"event": event,
|
||
"tier": "L2",
|
||
"session_id": session_id,
|
||
"urgency": "high",
|
||
"complexity": "medium",
|
||
"task_type": event.get("event_type", "general_analysis"),
|
||
})
|
||
|
||
return {
|
||
"source": "elephant_alpha",
|
||
"priority": decision.priority,
|
||
"confidence": decision.confidence,
|
||
"execution_plan": decision.execution_plan,
|
||
"agents_required": decision.agents_required,
|
||
"reasoning": decision.reasoning,
|
||
}
|
||
|
||
except Exception as e:
|
||
logger.warning(f"[EventRouter] EA L2 failed ({e}), fallback → AIOrchestrator")
|
||
orchestrator = AIOrchestrator()
|
||
return await orchestrator.handle_l2(event, session_id)
|
||
|
||
|
||
async def _handle_l0(event: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""L0: return raw event (compatibility/monitoring)."""
|
||
return {"status": "ok", "echo": event.get("event_type")}
|
||
|
||
|
||
async def dispatch(event: Dict[str, Any], admin_chat_ids: Optional[list] = None) -> Dict[str, Any]:
|
||
"""
|
||
Main event routing entry (ADR-012 §③ — 唯一入口).
|
||
Output format compatible with routes/bot_api_routes.
|
||
"""
|
||
tier = _classify(event)
|
||
session_id = f"evt:{event.get('event_type')}:{event.get('source', 'unknown')}"
|
||
started_at = time.perf_counter()
|
||
|
||
try:
|
||
if tier == "L0":
|
||
result = await _handle_l0(event)
|
||
elif tier == "L1":
|
||
result = await _handle_l1(event, session_id)
|
||
elif tier == "L2":
|
||
result = await _handle_l2(event, session_id)
|
||
else:
|
||
result = await _handle_l0(event)
|
||
|
||
message, reply_markup = _build_telegram_message(event, tier, result)
|
||
send_result = send_telegram_with_result(message, chat_ids=admin_chat_ids, reply_markup=reply_markup)
|
||
latency_ms = int((time.perf_counter() - started_at) * 1000)
|
||
|
||
return {
|
||
"tier": tier,
|
||
"sent": send_result["sent"],
|
||
"errors": send_result["errors"],
|
||
"latency_ms": latency_ms,
|
||
"payload": result,
|
||
"delivered": send_result["ok"],
|
||
}
|
||
except Exception as e:
|
||
logger.exception(f"[EventRouter] dispatch failed: {e}")
|
||
return {
|
||
"tier": tier,
|
||
"sent": 0,
|
||
"errors": [str(e)],
|
||
"latency_ms": int((time.perf_counter() - started_at) * 1000),
|
||
"payload": None,
|
||
"delivered": False,
|
||
}
|
||
|
||
|
||
def _run_coroutine_in_thread(coro) -> Dict[str, Any]:
|
||
result = {}
|
||
|
||
def runner():
|
||
try:
|
||
result["value"] = asyncio.run(coro)
|
||
except Exception as e:
|
||
result["value"] = {
|
||
"tier": "unknown",
|
||
"sent": 0,
|
||
"errors": [str(e)],
|
||
"latency_ms": 0,
|
||
"payload": None,
|
||
}
|
||
|
||
thread = threading.Thread(target=runner, daemon=True)
|
||
thread.start()
|
||
thread.join(timeout=15)
|
||
if thread.is_alive():
|
||
return {
|
||
"tier": "unknown",
|
||
"sent": 0,
|
||
"errors": ["dispatch_sync timed out"],
|
||
"latency_ms": 15000,
|
||
"payload": None,
|
||
}
|
||
return result["value"]
|
||
|
||
|
||
def dispatch_sync(event: Dict[str, Any], admin_chat_ids: Optional[list] = None) -> Dict[str, Any]:
|
||
"""同步環境使用的 EventRouter 入口。"""
|
||
try:
|
||
asyncio.get_running_loop()
|
||
except RuntimeError:
|
||
return asyncio.run(dispatch(event, admin_chat_ids=admin_chat_ids))
|
||
return _run_coroutine_in_thread(dispatch(event, admin_chat_ids=admin_chat_ids))
|
||
|
||
|
||
def notify_failure(
|
||
task_name: str,
|
||
error: Exception,
|
||
*,
|
||
source: Optional[str] = None,
|
||
event_type: str = "scheduler_task_failure",
|
||
priority: str = "P2",
|
||
title: Optional[str] = None,
|
||
trace: Optional[str] = None,
|
||
payload: Optional[Dict[str, Any]] = None,
|
||
) -> Dict[str, Any]:
|
||
"""排程/背景任務失敗的同步通知 helper。"""
|
||
severity = "alert" if priority in {"P1", "P2"} else "warning"
|
||
event = {
|
||
"source": source or f"Scheduler.{task_name}",
|
||
"event_type": event_type,
|
||
"severity": severity,
|
||
"title": title or f"{task_name} 任務異常",
|
||
"status": "任務失敗",
|
||
"impact": f"{priority} - 背景任務需要檢查",
|
||
"summary": str(error)[:200],
|
||
"trace": trace or "".join(traceback.format_exception(type(error), error, error.__traceback__)),
|
||
"payload": {"task_name": task_name, **(payload or {})},
|
||
}
|
||
return dispatch_sync(event)
|
||
|
||
|
||
def _classify(event: Dict[str, Any]) -> str:
|
||
sev = event.get("severity", "info")
|
||
has_trace = bool(event.get("trace"))
|
||
event_type = event.get("event_type", "")
|
||
|
||
if sev in ("info", "success"):
|
||
return "L0"
|
||
if sev == "warning":
|
||
return "L1" if has_trace else "L0"
|
||
if sev == "alert":
|
||
if event_type in {"price_threat", "db_connection_error", "crawler_timeout",
|
||
"nim_quota_exhausted", "embedding_failure"}:
|
||
return "L2"
|
||
return "L1"
|
||
return "L0"
|
||
|
||
|
||
def _build_telegram_message(event: Dict[str, Any], tier: str, result: Optional[Dict[str, Any]]) -> tuple[str, Optional[Dict[str, Any]]]:
|
||
if tier == "L0":
|
||
title = event.get("title") or event.get("event_type", "system_event")
|
||
summary = event.get("summary") or event.get("status") or "系統事件"
|
||
body = event.get("impact") or event.get("source") or ""
|
||
message = (
|
||
f"ℹ️ <b>{title}</b>\n"
|
||
f"━━━━━━━━━━━━━━━━━━━━\n"
|
||
f"{summary}\n"
|
||
f"{body}\n"
|
||
)
|
||
return message, None
|
||
|
||
ai_summary = ""
|
||
ai_cause = None
|
||
ai_actions = None
|
||
ai_executed = None
|
||
|
||
if isinstance(result, dict):
|
||
ai_summary = (
|
||
result.get("summary")
|
||
or result.get("reasoning")
|
||
or result.get("message")
|
||
or str(result)[:400]
|
||
)
|
||
ai_cause = result.get("cause") or result.get("root_cause")
|
||
ai_actions = result.get("suggested_actions") or result.get("execution_plan")
|
||
ai_executed = result.get("executed_actions")
|
||
|
||
return triaged_alert(
|
||
event,
|
||
tier_label=tier,
|
||
ai_summary=ai_summary,
|
||
ai_cause=ai_cause,
|
||
ai_actions=ai_actions if isinstance(ai_actions, list) else ([str(ai_actions)] if ai_actions else None),
|
||
ai_executed=ai_executed if isinstance(ai_executed, list) else ([str(ai_executed)] if ai_executed else None),
|
||
)
|