feat(auto-repair): route ssh diagnostics through mcp gateway

apps/api/src/services/post_execution_verifier.py

@@ -34,6 +34,7 @@ MASTER §3.1 L6×D1
 from __future__ import annotations
 
 import asyncio
+import re
 from typing import TYPE_CHECKING, Any
 
 import structlog
@@ -236,6 +237,7 @@ class PostExecutionVerifier:
             "pod_name": labels.get("pod", labels.get("name", "")),
             "deployment": labels.get("deployment", ""),
             "host": labels.get("instance", "").split(":")[0] or labels.get("host", ""),
+            "query": _build_prometheus_query(alertname, labels),
         }
 
         async def _call_one(reg) -> tuple[str, Any]:
@@ -501,16 +503,95 @@ def _get_incident_id(incident: "Incident") -> str:
 
 def _get_alertname(incident: "Incident") -> str:
     if incident.signals:
-        return incident.signals[0].labels.get("alertname", "")
+        signal = incident.signals[0]
+        labels = signal.labels or {}
+        return labels.get("alertname", "") or getattr(signal, "alert_name", "")
     return ""
 
 
 def _get_labels(incident: "Incident") -> dict[str, Any]:
     if incident.signals:
-        return incident.signals[0].labels
+        return incident.signals[0].labels or {}
     return {}
 
 
+def _build_prometheus_query(alertname: str, labels: dict[str, Any]) -> str:
+    """Build a non-empty PromQL probe for post-execution metric sensors."""
+
+    alert = (alertname or "").lower()
+    namespace = _safe_label_value(str(labels.get("namespace") or "awoooi-prod"))
+    pod_name = _safe_label_value(str(labels.get("pod") or labels.get("name") or ""))
+    host = _safe_label_value(str(labels.get("host") or _short_instance(labels.get("instance")) or ""))
+    container = _safe_label_value(str(
+        labels.get("container_name")
+        or labels.get("container")
+        or labels.get("name")
+        or ""
+    ))
+
+    if alert.startswith("dockercontainer"):
+        selector = _selector({
+            "host": host,
+            "container_name": container,
+        })
+        if "memory" in alert:
+            return (
+                f"docker_container_memory_usage_bytes{selector} / "
+                f"docker_container_memory_limit_bytes{selector}"
+            )
+        if "restart" in alert:
+            return (
+                f"increase(docker_container_inspect_restart_count{selector}[15m]) "
+                f"or increase(docker_container_restart_count{selector}[15m])"
+            )
+        if "cpu" in alert:
+            return f"docker_container_cpu_cores{selector}"
+        return f"docker_container_info{selector}"
+
+    if any(key in alert for key in ("host", "node")):
+        instance_selector = _selector({"instance": str(labels.get("instance") or "")})
+        if "memory" in alert or "oom" in alert:
+            return f"node_memory_MemAvailable_bytes{instance_selector}"
+        if "disk" in alert or "storage" in alert:
+            return f"node_filesystem_avail_bytes{instance_selector}"
+        if "load" in alert or "cpu" in alert:
+            return f"node_load5{instance_selector}"
+        return f"up{instance_selector}"
+
+    pod_filter = f',pod=~"{pod_name}.*"' if pod_name else ""
+    if any(key in alert for key in ("memory", "mem", "oom")):
+        return (
+            f'avg(container_memory_working_set_bytes{{namespace="{namespace}"{pod_filter}}}) '
+            "/ 1048576"
+        )
+    if any(key in alert for key in ("cpu", "load", "throttl")):
+        return f'avg(rate(container_cpu_usage_seconds_total{{namespace="{namespace}"{pod_filter}}}[5m]))'
+    if any(key in alert for key in ("crash", "restart", "backoff")):
+        return f'sum(increase(kube_pod_container_status_restarts_total{{namespace="{namespace}"}}[15m]))'
+    if any(key in alert for key in ("http", "error", "5xx", "probe", "down", "unhealthy")):
+        return "1 - avg(probe_success)"
+    return f'up{{namespace="{namespace}"}}'
+
+
+def _selector(labels: dict[str, str]) -> str:
+    parts = [f'{key}="{value}"' for key, value in labels.items() if value]
+    return "{" + ",".join(parts) + "}" if parts else ""
+
+
+def _short_instance(instance: Any) -> str:
+    raw = str(instance or "")
+    if raw.count(":") == 1:
+        return raw.rsplit(":", 1)[0]
+    return raw
+
+
+def _safe_label_value(value: str) -> str:
+    cleaned = (value or "").strip()
+    if re.fullmatch(r"[a-zA-Z0-9_.:-]{1,128}", cleaned):
+        return cleaned
+    return ""
+
+
 async def _update_snapshot(
     snapshot: EvidenceSnapshot,
     post_state: dict[str, Any],