新增 AI 自動化 Smoke Dashboard

services/ai_automation_smoke_service.py

@@ -0,0 +1,210 @@
+"""Smoke checks for the four-agent AI automation control plane.
+
+The checks are read-only and intentionally avoid outbound network calls. They
+are meant for a fast dashboard/API sanity check, not for deep production probes.
+"""
+
+from __future__ import annotations
+
+import os
+from datetime import datetime
+from typing import Any, Dict, List
+
+from sqlalchemy import text
+
+from config import SYSTEM_VERSION
+from database.manager import get_session
+
+
+STATUS_RANK = {"ok": 0, "warning": 1, "critical": 2}
+
+
+def _check(name: str, status: str, summary: str, details: Dict[str, Any] | None = None) -> Dict[str, Any]:
+    return {
+        "name": name,
+        "status": status,
+        "summary": summary,
+        "details": details or {},
+    }
+
+
+def _count_jsonl_lines(path: str) -> int:
+    try:
+        with open(path, "r", encoding="utf-8") as fh:
+            return sum(1 for line in fh if line.strip())
+    except FileNotFoundError:
+        return 0
+
+
+def _event_router_check() -> Dict[str, Any]:
+    try:
+        from services import event_router
+        from services.ai_automation_metrics import snapshot
+
+        queue_count = _count_jsonl_lines(event_router._QUEUE_PATH)
+        metrics = snapshot()
+        dispatch_total = sum(
+            value for (metric, _labels), value in metrics.get("counters", {}).items()
+            if metric == "event_router_dispatch_total"
+        )
+        status = "warning" if queue_count else "ok"
+        summary = "EventRouter 可用，通知 queue 乾淨" if status == "ok" else "EventRouter 可用，但有待回放通知"
+        return _check(
+            "EventRouter 通知鏈",
+            status,
+            summary,
+            {
+                "dispatch_sync": callable(getattr(event_router, "dispatch_sync", None)),
+                "notify_failure": callable(getattr(event_router, "notify_failure", None)),
+                "queued_deliveries": queue_count,
+                "dispatch_metric_total": dispatch_total,
+            },
+        )
+    except Exception as exc:
+        return _check("EventRouter 通知鏈", "critical", f"EventRouter smoke 失敗：{exc}")
+
+
+def _autoheal_check() -> Dict[str, Any]:
+    try:
+        import services.auto_heal_service as autoheal
+
+        protected = set(getattr(autoheal, "_PROTECTED_CONTAINERS", set()))
+        required = {"momo-db", "momo-postgres"}
+        missing = sorted(required - protected)
+        allowed_actions = sorted(getattr(autoheal, "_ALLOWED_ACTION_TYPES", set()))
+        status = "critical" if missing else "ok"
+        summary = "AutoHeal 保護資料庫容器，安全邊界存在" if status == "ok" else "AutoHeal protected resource 缺漏"
+        return _check(
+            "AutoHeal 安全邊界",
+            status,
+            summary,
+            {
+                "protected_containers": sorted(protected),
+                "missing_required_protection": missing,
+                "allowed_actions": allowed_actions,
+            },
+        )
+    except Exception as exc:
+        return _check("AutoHeal 安全邊界", "critical", f"AutoHeal smoke 失敗：{exc}")
+
+
+def _nemotron_check() -> Dict[str, Any]:
+    try:
+        import services.nemoton_dispatcher_service as nemotron
+
+        dispatcher_cls = getattr(nemotron, "NemotronDispatcherService", None)
+        fallback_ready = bool(dispatcher_cls and hasattr(dispatcher_cls, "_hermes_rule_fallback"))
+        api_key_configured = bool(getattr(nemotron, "NIM_API_KEY", ""))
+        call_count = getattr(nemotron, "_nim_call_count", {}).get("count", 0)
+        daily_limit = getattr(nemotron, "NIM_DAILY_LIMIT", 80)
+        if not fallback_ready:
+            status = "critical"
+            summary = "NemoTron Hermes fallback 缺失"
+        elif not api_key_configured:
+            status = "warning"
+            summary = "NemoTron API key 未設定，目前會走 Hermes fallback"
+        elif call_count >= daily_limit:
+            status = "warning"
+            summary = "NemoTron 配額已達上限，會走 Hermes fallback"
+        else:
+            status = "ok"
+            summary = "NemoTron 與 Hermes fallback 機制可用"
+        return _check(
+            "NemoTron fallback",
+            status,
+            summary,
+            {
+                "fallback_ready": fallback_ready,
+                "api_key_configured": api_key_configured,
+                "call_count": call_count,
+                "daily_limit": daily_limit,
+            },
+        )
+    except Exception as exc:
+        return _check("NemoTron fallback", "critical", f"NemoTron smoke 失敗：{exc}")
+
+
+def _embedding_queue_check() -> Dict[str, Any]:
+    session = None
+    try:
+        session = get_session()
+        rows = session.execute(
+            text("SELECT status, COUNT(*) AS count FROM embedding_retry_queue GROUP BY status")
+        ).fetchall()
+        counts = {str(row._mapping["status"]): int(row._mapping["count"]) for row in rows}
+        pending = counts.get("pending", 0)
+        processing = counts.get("processing", 0)
+        if pending > 1000 or processing > 200:
+            status = "warning"
+            summary = "OpenClaw embedding queue backlog 偏高"
+        else:
+            status = "ok"
+            summary = "OpenClaw embedding queue 可讀取且 backlog 正常"
+        return _check(
+            "OpenClaw embedding queue",
+            status,
+            summary,
+            {"counts": counts, "pending": pending, "processing": processing},
+        )
+    except Exception as exc:
+        return _check(
+            "OpenClaw embedding queue",
+            "warning",
+            f"Embedding queue 無法讀取，可能是 DB 離線或 migration 未套用：{exc}",
+        )
+    finally:
+        if session is not None:
+            session.close()
+
+
+def _elephant_hitl_check() -> Dict[str, Any]:
+    try:
+        from services.elephant_alpha_autonomous_engine import ElephantAlphaAutonomousEngine
+
+        has_hitl = hasattr(ElephantAlphaAutonomousEngine, "_escalate_to_human")
+        has_timeout_guard = hasattr(ElephantAlphaAutonomousEngine, "_run_with_timeout")
+        api_key_configured = bool(os.getenv("OPENROUTER_API_KEY") or os.getenv("NVIDIA_API_KEY"))
+        if not has_hitl or not has_timeout_guard:
+            status = "critical"
+            summary = "ElephantAlpha HITL 或 timeout guard 缺失"
+        elif not api_key_configured:
+            status = "warning"
+            summary = "ElephantAlpha HITL 程式可用，但 API key 未設定"
+        else:
+            status = "ok"
+            summary = "ElephantAlpha HITL 與 timeout guard 可用"
+        return _check(
+            "ElephantAlpha HITL",
+            status,
+            summary,
+            {
+                "hitl_method": has_hitl,
+                "timeout_guard": has_timeout_guard,
+                "api_key_configured": api_key_configured,
+            },
+        )
+    except Exception as exc:
+        return _check("ElephantAlpha HITL", "critical", f"ElephantAlpha smoke 失敗：{exc}")
+
+
+def collect_ai_automation_smoke() -> Dict[str, Any]:
+    checks: List[Dict[str, Any]] = [
+        _event_router_check(),
+        _autoheal_check(),
+        _nemotron_check(),
+        _embedding_queue_check(),
+        _elephant_hitl_check(),
+    ]
+    worst = max(checks, key=lambda item: STATUS_RANK.get(item["status"], 2))["status"]
+    return {
+        "status": worst,
+        "version": SYSTEM_VERSION,
+        "generated_at": datetime.now().isoformat(timespec="seconds"),
+        "checks": checks,
+        "summary": {
+            "ok": sum(1 for item in checks if item["status"] == "ok"),
+            "warning": sum(1 for item in checks if item["status"] == "warning"),
+            "critical": sum(1 for item in checks if item["status"] == "critical"),
+            "total": len(checks),
+        },
+    }