feat(agent99): bridge sre alerts to windows relay
This commit is contained in:
ogt
@@ -56,6 +56,7 @@ from src.services.alertmanager_llm_guard import (
|
||||
from src.services.approval_db import get_approval_service
|
||||
from src.services.auto_approve import get_auto_approve_policy
|
||||
from src.services.auto_repair_service import AutoRepairService
|
||||
from src.services.agent99_sre_bridge import bridge_alertmanager_to_agent99
|
||||
from src.services.channel_hub import (
|
||||
record_alertmanager_event,
|
||||
record_grouped_alert_event,
|
||||
@@ -2849,6 +2850,21 @@ async def alertmanager_webhook(
|
||||
labels=dict(alert.labels) if alert.labels else {},
|
||||
annotations=dict(alert.annotations) if alert.annotations else {},
|
||||
)
|
||||
background_tasks.add_task(
|
||||
bridge_alertmanager_to_agent99,
|
||||
alert_id=alert_id,
|
||||
alertname=alertname,
|
||||
severity=severity,
|
||||
namespace=namespace,
|
||||
target_resource=target_resource,
|
||||
message=message,
|
||||
labels=dict(alert.labels) if alert.labels else {},
|
||||
annotations=dict(alert.annotations) if alert.annotations else {},
|
||||
fingerprint=fingerprint,
|
||||
alert_category=alert_category,
|
||||
notification_type=notification_type,
|
||||
source_url=alert.generatorURL,
|
||||
)
|
||||
|
||||
# ==========================================================================
|
||||
# ADR-076: 告警聚合引擎 — 5 分鐘滑動視窗，防止告警風暴
|
||||
|
||||
@@ -925,6 +925,28 @@ class Settings(BaseSettings):
|
||||
default="",
|
||||
description="Telegram Webhook Secret Token（setWebhook 設定的同一值）",
|
||||
)
|
||||
AGENT99_SRE_ALERT_INBOX_PATH: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"Optional mounted path for Agent99 SRE alert inbox. Empty disables the "
|
||||
"fail-open Alertmanager to Agent99 file bridge."
|
||||
),
|
||||
)
|
||||
AGENT99_SRE_ALERT_RELAY_URL: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"Optional Agent99 SRE alert relay endpoint. When set, Alertmanager "
|
||||
"events are posted to Agent99 before falling back to the mounted file inbox."
|
||||
),
|
||||
)
|
||||
AGENT99_SRE_ALERT_RELAY_TOKEN: str = Field(
|
||||
default="",
|
||||
description="Optional shared token for Agent99 SRE alert relay POSTs.",
|
||||
)
|
||||
AGENT99_SRE_ALERT_RELAY_TIMEOUT_SECONDS: float = Field(
|
||||
default=3.0,
|
||||
description="Timeout for fail-open Agent99 SRE alert relay POSTs.",
|
||||
)
|
||||
# 2026-04-24 Claude Sonnet 4.6 (ADR-095 WS4): Hermes NL 自然語言閘道
|
||||
# false=不啟用（預設），true=啟用 @mention 問答（需 ANTHROPIC_API_KEY）
|
||||
HERMES_NL_ENABLED: bool = Field(
|
||||
|
||||
357
apps/api/src/services/agent99_sre_bridge.py
Normal file
357
apps/api/src/services/agent99_sre_bridge.py
Normal file
@@ -0,0 +1,357 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from src.core.config import settings
|
||||
from src.core.logging import get_logger
|
||||
from src.utils.timezone import now_taipei
|
||||
|
||||
logger = get_logger("awoooi.agent99_sre_bridge")
|
||||
|
||||
_SENSITIVE_KEY_RE = re.compile(
|
||||
r"(token|secret|password|passwd|authorization|cookie|session|private[_-]?key)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_SAFE_TOKEN_RE = re.compile(r"[^A-Za-z0-9_.-]+")
|
||||
|
||||
|
||||
def _safe_text(value: Any, *, max_length: int = 1200) -> str:
|
||||
text = str(value or "")
|
||||
text = " ".join(text.split())
|
||||
if len(text) > max_length:
|
||||
return text[: max_length - 14] + "...(truncated)"
|
||||
return text
|
||||
|
||||
|
||||
def _safe_label_value(key: str, value: Any) -> str:
|
||||
if _SENSITIVE_KEY_RE.search(key):
|
||||
return "[redacted]"
|
||||
return _safe_text(value, max_length=240)
|
||||
|
||||
|
||||
def _safe_file_token(value: str) -> str:
|
||||
safe = _SAFE_TOKEN_RE.sub("_", value).strip("._-")
|
||||
return (safe or "alert")[:96]
|
||||
|
||||
|
||||
def _safe_url_for_log(value: str) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
parsed = urllib.parse.urlsplit(value)
|
||||
return urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", ""))
|
||||
|
||||
|
||||
def _alert_text(
|
||||
*,
|
||||
alertname: str,
|
||||
severity: str,
|
||||
target_resource: str,
|
||||
namespace: str,
|
||||
message: str,
|
||||
labels: dict[str, Any],
|
||||
annotations: dict[str, Any],
|
||||
) -> str:
|
||||
return " ".join(
|
||||
[
|
||||
alertname,
|
||||
severity,
|
||||
target_resource,
|
||||
namespace,
|
||||
message,
|
||||
" ".join(f"{k}={v}" for k, v in labels.items()),
|
||||
" ".join(f"{k}={v}" for k, v in annotations.items()),
|
||||
]
|
||||
).lower()
|
||||
|
||||
|
||||
def resolve_agent99_alert_kind(
|
||||
*,
|
||||
alertname: str,
|
||||
severity: str,
|
||||
target_resource: str,
|
||||
namespace: str,
|
||||
message: str,
|
||||
labels: dict[str, Any] | None = None,
|
||||
annotations: dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
text = _alert_text(
|
||||
alertname=alertname,
|
||||
severity=severity,
|
||||
target_resource=target_resource,
|
||||
namespace=namespace,
|
||||
message=message,
|
||||
labels=labels or {},
|
||||
annotations=annotations or {},
|
||||
)
|
||||
|
||||
if re.search(
|
||||
r"provider_freshness_signal|source_provider_freshness_triage|"
|
||||
r"raw_signal_normalized|controlled_runtime_candidate|freshness|"
|
||||
r"last_seen|last seen|ingestion",
|
||||
text,
|
||||
):
|
||||
return "provider_freshness_signal"
|
||||
if re.search(r"502|site_502|public_route|website|route_down|nginx upstream", text):
|
||||
return "public_route_502"
|
||||
if re.search(r"cpu|load|memory|mem_|disk|pressure|performance", text):
|
||||
return "performance_pressure"
|
||||
if re.search(r"backup|snapshot|restore drill|cron", text):
|
||||
return "backup_health"
|
||||
if re.search(r"harbor|registry", text):
|
||||
return "harbor_registry"
|
||||
if re.search(r"k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment", text):
|
||||
return "awoooi_k3s"
|
||||
if re.search(r"host_down|hostdown|vm_not_running|reboot|restart|powered_off", text):
|
||||
return "host_recovery"
|
||||
return "monitoring_alert"
|
||||
|
||||
|
||||
def _suggested_mode_for_kind(kind: str) -> str | None:
|
||||
return {
|
||||
"provider_freshness_signal": "ProviderFreshness",
|
||||
"public_route_502": "Recover",
|
||||
"performance_pressure": "Perf",
|
||||
"backup_health": "BackupCheck",
|
||||
"harbor_registry": "HarborRepair",
|
||||
"awoooi_k3s": "AwoooRepair",
|
||||
"host_recovery": "Recover",
|
||||
}.get(kind)
|
||||
|
||||
|
||||
def build_agent99_sre_alert(
|
||||
*,
|
||||
alert_id: str,
|
||||
alertname: str,
|
||||
severity: str,
|
||||
namespace: str,
|
||||
target_resource: str,
|
||||
message: str,
|
||||
labels: dict[str, Any] | None = None,
|
||||
annotations: dict[str, Any] | None = None,
|
||||
fingerprint: str = "",
|
||||
alert_category: str = "",
|
||||
notification_type: str = "",
|
||||
source_url: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
labels = labels or {}
|
||||
annotations = annotations or {}
|
||||
normalized_severity = (severity or "warning").lower()
|
||||
kind = resolve_agent99_alert_kind(
|
||||
alertname=alertname,
|
||||
severity=normalized_severity,
|
||||
target_resource=target_resource,
|
||||
namespace=namespace,
|
||||
message=message,
|
||||
labels=labels,
|
||||
annotations=annotations,
|
||||
)
|
||||
suggested_mode = _suggested_mode_for_kind(kind)
|
||||
host = str(labels.get("host") or labels.get("node") or labels.get("instance") or "")
|
||||
if ":" in host:
|
||||
host = host.split(":", 1)[0]
|
||||
service = str(
|
||||
labels.get("service")
|
||||
or labels.get("app")
|
||||
or labels.get("component")
|
||||
or target_resource
|
||||
or "awoooi"
|
||||
)
|
||||
safe_labels = [
|
||||
"alertmanager",
|
||||
"awoooi-api",
|
||||
f"alertname={_safe_label_value('alertname', alertname)}",
|
||||
f"namespace={_safe_label_value('namespace', namespace)}",
|
||||
f"target={_safe_label_value('target', target_resource)}",
|
||||
]
|
||||
if fingerprint:
|
||||
safe_labels.append(f"fingerprint={_safe_label_value('fingerprint', fingerprint)}")
|
||||
for key, value in sorted(labels.items()):
|
||||
safe_labels.append(f"{_safe_text(key, max_length=80)}={_safe_label_value(key, value)}")
|
||||
if kind == "provider_freshness_signal":
|
||||
safe_labels.extend(["provider-freshness", "no-false-green"])
|
||||
|
||||
instruction = None
|
||||
if kind == "provider_freshness_signal":
|
||||
instruction = (
|
||||
"Provider freshness candidate required; require providerWindow,lastSeen,"
|
||||
"directReference,verifierReadback; mark missing,expired,timeout,"
|
||||
"no_false_green; do not switch provider, call paid model, edit env, or reload."
|
||||
)
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"id": f"awoooi-alertmanager-{_safe_file_token(alert_id)}",
|
||||
"source": "awoooi-api-alertmanager",
|
||||
"severity": normalized_severity,
|
||||
"kind": kind,
|
||||
"service": _safe_text(service, max_length=160),
|
||||
"host": _safe_text(host, max_length=160),
|
||||
"title": _safe_text(f"Alertmanager {alertname}", max_length=240),
|
||||
"message": _safe_text(message, max_length=2000),
|
||||
"labels": safe_labels,
|
||||
"force": True,
|
||||
"controlledApply": True,
|
||||
"createdAt": now_taipei().isoformat(),
|
||||
"awoooi": {
|
||||
"alertId": _safe_text(alert_id, max_length=160),
|
||||
"alertname": _safe_text(alertname, max_length=160),
|
||||
"namespace": _safe_text(namespace, max_length=160),
|
||||
"targetResource": _safe_text(target_resource, max_length=240),
|
||||
"fingerprint": _safe_text(fingerprint, max_length=160),
|
||||
"alertCategory": _safe_text(alert_category, max_length=160),
|
||||
"notificationType": _safe_text(notification_type, max_length=160),
|
||||
"sourceUrl": _safe_text(source_url or "", max_length=600),
|
||||
},
|
||||
}
|
||||
if suggested_mode:
|
||||
payload["suggestedMode"] = suggested_mode
|
||||
if instruction:
|
||||
payload["instruction"] = instruction
|
||||
return payload
|
||||
|
||||
|
||||
def write_agent99_sre_alert(
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
inbox_path: str | None = None,
|
||||
) -> Path | None:
|
||||
target_dir_text = inbox_path if inbox_path is not None else settings.AGENT99_SRE_ALERT_INBOX_PATH
|
||||
if not target_dir_text:
|
||||
logger.info(
|
||||
"agent99_sre_bridge_disabled",
|
||||
reason="AGENT99_SRE_ALERT_INBOX_PATH not configured",
|
||||
alert_id=payload.get("id"),
|
||||
kind=payload.get("kind"),
|
||||
)
|
||||
return None
|
||||
|
||||
target_dir = Path(target_dir_text)
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
alert_id = _safe_file_token(str(payload.get("id") or uuid.uuid4().hex))
|
||||
final_path = target_dir / f"{alert_id}.json"
|
||||
tmp_path = target_dir / f".{alert_id}.{uuid.uuid4().hex}.tmp"
|
||||
tmp_path.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
tmp_path.replace(final_path)
|
||||
logger.info(
|
||||
"agent99_sre_alert_written",
|
||||
path=str(final_path),
|
||||
alert_id=payload.get("id"),
|
||||
kind=payload.get("kind"),
|
||||
)
|
||||
return final_path
|
||||
|
||||
|
||||
def post_agent99_sre_alert(
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
relay_url: str | None = None,
|
||||
relay_token: str | None = None,
|
||||
timeout_seconds: float | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
target_url = relay_url if relay_url is not None else settings.AGENT99_SRE_ALERT_RELAY_URL
|
||||
if not target_url:
|
||||
return None
|
||||
|
||||
token = relay_token if relay_token is not None else settings.AGENT99_SRE_ALERT_RELAY_TOKEN
|
||||
timeout = (
|
||||
timeout_seconds
|
||||
if timeout_seconds is not None
|
||||
else settings.AGENT99_SRE_ALERT_RELAY_TIMEOUT_SECONDS
|
||||
)
|
||||
data = json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
headers = {
|
||||
"Content-Type": "application/json; charset=utf-8",
|
||||
"User-Agent": "awoooi-agent99-sre-bridge/1",
|
||||
}
|
||||
if token:
|
||||
headers["X-Agent99-Relay-Token"] = token
|
||||
|
||||
request = urllib.request.Request(target_url, data=data, headers=headers, method="POST")
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
status = int(response.getcode())
|
||||
body = response.read(4096).decode("utf-8", errors="replace")
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read(4096).decode("utf-8", errors="replace")
|
||||
raise RuntimeError(f"Agent99 relay HTTP {exc.code}: {body[:300]}") from exc
|
||||
|
||||
if status >= 400:
|
||||
raise RuntimeError(f"Agent99 relay HTTP {status}: {body[:300]}")
|
||||
|
||||
logger.info(
|
||||
"agent99_sre_alert_relay_posted",
|
||||
relay_url=_safe_url_for_log(target_url),
|
||||
alert_id=payload.get("id"),
|
||||
kind=payload.get("kind"),
|
||||
status=status,
|
||||
)
|
||||
return {"status": status, "body": body}
|
||||
|
||||
|
||||
def dispatch_agent99_sre_alert(payload: dict[str, Any]) -> str:
|
||||
if settings.AGENT99_SRE_ALERT_RELAY_URL:
|
||||
post_agent99_sre_alert(payload)
|
||||
return "relay"
|
||||
|
||||
if settings.AGENT99_SRE_ALERT_INBOX_PATH:
|
||||
written = write_agent99_sre_alert(payload)
|
||||
if written is not None:
|
||||
return "file"
|
||||
|
||||
logger.info(
|
||||
"agent99_sre_bridge_disabled",
|
||||
reason="no relay URL or mounted inbox path configured",
|
||||
alert_id=payload.get("id"),
|
||||
kind=payload.get("kind"),
|
||||
)
|
||||
return "disabled"
|
||||
|
||||
|
||||
async def bridge_alertmanager_to_agent99(
|
||||
*,
|
||||
alert_id: str,
|
||||
alertname: str,
|
||||
severity: str,
|
||||
namespace: str,
|
||||
target_resource: str,
|
||||
message: str,
|
||||
labels: dict[str, Any] | None = None,
|
||||
annotations: dict[str, Any] | None = None,
|
||||
fingerprint: str = "",
|
||||
alert_category: str = "",
|
||||
notification_type: str = "",
|
||||
source_url: str | None = None,
|
||||
) -> None:
|
||||
try:
|
||||
payload = build_agent99_sre_alert(
|
||||
alert_id=alert_id,
|
||||
alertname=alertname,
|
||||
severity=severity,
|
||||
namespace=namespace,
|
||||
target_resource=target_resource,
|
||||
message=message,
|
||||
labels=labels,
|
||||
annotations=annotations,
|
||||
fingerprint=fingerprint,
|
||||
alert_category=alert_category,
|
||||
notification_type=notification_type,
|
||||
source_url=source_url,
|
||||
)
|
||||
await asyncio.to_thread(dispatch_agent99_sre_alert, payload)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"agent99_sre_bridge_failed_fail_open",
|
||||
alert_id=alert_id,
|
||||
alertname=alertname,
|
||||
error=str(exc),
|
||||
)
|
||||
Reference in New Issue
Block a user