358 lines
11 KiB
Python
358 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import re
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from src.core.config import settings
|
|
from src.core.logging import get_logger
|
|
from src.utils.timezone import now_taipei
|
|
|
|
logger = get_logger("awoooi.agent99_sre_bridge")
|
|
|
|
_SENSITIVE_KEY_RE = re.compile(
|
|
r"(token|secret|password|passwd|authorization|cookie|session|private[_-]?key)",
|
|
re.IGNORECASE,
|
|
)
|
|
_SAFE_TOKEN_RE = re.compile(r"[^A-Za-z0-9_.-]+")
|
|
|
|
|
|
def _safe_text(value: Any, *, max_length: int = 1200) -> str:
|
|
text = str(value or "")
|
|
text = " ".join(text.split())
|
|
if len(text) > max_length:
|
|
return text[: max_length - 14] + "...(truncated)"
|
|
return text
|
|
|
|
|
|
def _safe_label_value(key: str, value: Any) -> str:
|
|
if _SENSITIVE_KEY_RE.search(key):
|
|
return "[redacted]"
|
|
return _safe_text(value, max_length=240)
|
|
|
|
|
|
def _safe_file_token(value: str) -> str:
|
|
safe = _SAFE_TOKEN_RE.sub("_", value).strip("._-")
|
|
return (safe or "alert")[:96]
|
|
|
|
|
|
def _safe_url_for_log(value: str) -> str:
|
|
if not value:
|
|
return ""
|
|
parsed = urllib.parse.urlsplit(value)
|
|
return urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", ""))
|
|
|
|
|
|
def _alert_text(
|
|
*,
|
|
alertname: str,
|
|
severity: str,
|
|
target_resource: str,
|
|
namespace: str,
|
|
message: str,
|
|
labels: dict[str, Any],
|
|
annotations: dict[str, Any],
|
|
) -> str:
|
|
return " ".join(
|
|
[
|
|
alertname,
|
|
severity,
|
|
target_resource,
|
|
namespace,
|
|
message,
|
|
" ".join(f"{k}={v}" for k, v in labels.items()),
|
|
" ".join(f"{k}={v}" for k, v in annotations.items()),
|
|
]
|
|
).lower()
|
|
|
|
|
|
def resolve_agent99_alert_kind(
|
|
*,
|
|
alertname: str,
|
|
severity: str,
|
|
target_resource: str,
|
|
namespace: str,
|
|
message: str,
|
|
labels: dict[str, Any] | None = None,
|
|
annotations: dict[str, Any] | None = None,
|
|
) -> str:
|
|
text = _alert_text(
|
|
alertname=alertname,
|
|
severity=severity,
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
message=message,
|
|
labels=labels or {},
|
|
annotations=annotations or {},
|
|
)
|
|
|
|
if re.search(
|
|
r"provider_freshness_signal|source_provider_freshness_triage|"
|
|
r"raw_signal_normalized|controlled_runtime_candidate|freshness|"
|
|
r"last_seen|last seen|ingestion",
|
|
text,
|
|
):
|
|
return "provider_freshness_signal"
|
|
if re.search(r"502|site_502|public_route|website|route_down|nginx upstream", text):
|
|
return "public_route_502"
|
|
if re.search(r"cpu|load|memory|mem_|disk|pressure|performance", text):
|
|
return "performance_pressure"
|
|
if re.search(r"backup|snapshot|restore drill|cron", text):
|
|
return "backup_health"
|
|
if re.search(r"harbor|registry", text):
|
|
return "harbor_registry"
|
|
if re.search(r"k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment", text):
|
|
return "awoooi_k3s"
|
|
if re.search(r"host_down|hostdown|vm_not_running|reboot|restart|powered_off", text):
|
|
return "host_recovery"
|
|
return "monitoring_alert"
|
|
|
|
|
|
def _suggested_mode_for_kind(kind: str) -> str | None:
|
|
return {
|
|
"provider_freshness_signal": "ProviderFreshness",
|
|
"public_route_502": "Recover",
|
|
"performance_pressure": "Perf",
|
|
"backup_health": "BackupCheck",
|
|
"harbor_registry": "HarborRepair",
|
|
"awoooi_k3s": "AwoooRepair",
|
|
"host_recovery": "Recover",
|
|
}.get(kind)
|
|
|
|
|
|
def build_agent99_sre_alert(
|
|
*,
|
|
alert_id: str,
|
|
alertname: str,
|
|
severity: str,
|
|
namespace: str,
|
|
target_resource: str,
|
|
message: str,
|
|
labels: dict[str, Any] | None = None,
|
|
annotations: dict[str, Any] | None = None,
|
|
fingerprint: str = "",
|
|
alert_category: str = "",
|
|
notification_type: str = "",
|
|
source_url: str | None = None,
|
|
) -> dict[str, Any]:
|
|
labels = labels or {}
|
|
annotations = annotations or {}
|
|
normalized_severity = (severity or "warning").lower()
|
|
kind = resolve_agent99_alert_kind(
|
|
alertname=alertname,
|
|
severity=normalized_severity,
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
message=message,
|
|
labels=labels,
|
|
annotations=annotations,
|
|
)
|
|
suggested_mode = _suggested_mode_for_kind(kind)
|
|
host = str(labels.get("host") or labels.get("node") or labels.get("instance") or "")
|
|
if ":" in host:
|
|
host = host.split(":", 1)[0]
|
|
service = str(
|
|
labels.get("service")
|
|
or labels.get("app")
|
|
or labels.get("component")
|
|
or target_resource
|
|
or "awoooi"
|
|
)
|
|
safe_labels = [
|
|
"alertmanager",
|
|
"awoooi-api",
|
|
f"alertname={_safe_label_value('alertname', alertname)}",
|
|
f"namespace={_safe_label_value('namespace', namespace)}",
|
|
f"target={_safe_label_value('target', target_resource)}",
|
|
]
|
|
if fingerprint:
|
|
safe_labels.append(f"fingerprint={_safe_label_value('fingerprint', fingerprint)}")
|
|
for key, value in sorted(labels.items()):
|
|
safe_labels.append(f"{_safe_text(key, max_length=80)}={_safe_label_value(key, value)}")
|
|
if kind == "provider_freshness_signal":
|
|
safe_labels.extend(["provider-freshness", "no-false-green"])
|
|
|
|
instruction = None
|
|
if kind == "provider_freshness_signal":
|
|
instruction = (
|
|
"Provider freshness candidate required; require providerWindow,lastSeen,"
|
|
"directReference,verifierReadback; mark missing,expired,timeout,"
|
|
"no_false_green; do not switch provider, call paid model, edit env, or reload."
|
|
)
|
|
|
|
payload: dict[str, Any] = {
|
|
"id": f"awoooi-alertmanager-{_safe_file_token(alert_id)}",
|
|
"source": "awoooi-api-alertmanager",
|
|
"severity": normalized_severity,
|
|
"kind": kind,
|
|
"service": _safe_text(service, max_length=160),
|
|
"host": _safe_text(host, max_length=160),
|
|
"title": _safe_text(f"Alertmanager {alertname}", max_length=240),
|
|
"message": _safe_text(message, max_length=2000),
|
|
"labels": safe_labels,
|
|
"force": True,
|
|
"controlledApply": True,
|
|
"createdAt": now_taipei().isoformat(),
|
|
"awoooi": {
|
|
"alertId": _safe_text(alert_id, max_length=160),
|
|
"alertname": _safe_text(alertname, max_length=160),
|
|
"namespace": _safe_text(namespace, max_length=160),
|
|
"targetResource": _safe_text(target_resource, max_length=240),
|
|
"fingerprint": _safe_text(fingerprint, max_length=160),
|
|
"alertCategory": _safe_text(alert_category, max_length=160),
|
|
"notificationType": _safe_text(notification_type, max_length=160),
|
|
"sourceUrl": _safe_text(source_url or "", max_length=600),
|
|
},
|
|
}
|
|
if suggested_mode:
|
|
payload["suggestedMode"] = suggested_mode
|
|
if instruction:
|
|
payload["instruction"] = instruction
|
|
return payload
|
|
|
|
|
|
def write_agent99_sre_alert(
|
|
payload: dict[str, Any],
|
|
*,
|
|
inbox_path: str | None = None,
|
|
) -> Path | None:
|
|
target_dir_text = inbox_path if inbox_path is not None else settings.AGENT99_SRE_ALERT_INBOX_PATH
|
|
if not target_dir_text:
|
|
logger.info(
|
|
"agent99_sre_bridge_disabled",
|
|
reason="AGENT99_SRE_ALERT_INBOX_PATH not configured",
|
|
alert_id=payload.get("id"),
|
|
kind=payload.get("kind"),
|
|
)
|
|
return None
|
|
|
|
target_dir = Path(target_dir_text)
|
|
target_dir.mkdir(parents=True, exist_ok=True)
|
|
alert_id = _safe_file_token(str(payload.get("id") or uuid.uuid4().hex))
|
|
final_path = target_dir / f"{alert_id}.json"
|
|
tmp_path = target_dir / f".{alert_id}.{uuid.uuid4().hex}.tmp"
|
|
tmp_path.write_text(
|
|
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True),
|
|
encoding="utf-8",
|
|
)
|
|
tmp_path.replace(final_path)
|
|
logger.info(
|
|
"agent99_sre_alert_written",
|
|
path=str(final_path),
|
|
alert_id=payload.get("id"),
|
|
kind=payload.get("kind"),
|
|
)
|
|
return final_path
|
|
|
|
|
|
def post_agent99_sre_alert(
|
|
payload: dict[str, Any],
|
|
*,
|
|
relay_url: str | None = None,
|
|
relay_token: str | None = None,
|
|
timeout_seconds: float | None = None,
|
|
) -> dict[str, Any] | None:
|
|
target_url = relay_url if relay_url is not None else settings.AGENT99_SRE_ALERT_RELAY_URL
|
|
if not target_url:
|
|
return None
|
|
|
|
token = relay_token if relay_token is not None else settings.AGENT99_SRE_ALERT_RELAY_TOKEN
|
|
timeout = (
|
|
timeout_seconds
|
|
if timeout_seconds is not None
|
|
else settings.AGENT99_SRE_ALERT_RELAY_TIMEOUT_SECONDS
|
|
)
|
|
data = json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
|
headers = {
|
|
"Content-Type": "application/json; charset=utf-8",
|
|
"User-Agent": "awoooi-agent99-sre-bridge/1",
|
|
}
|
|
if token:
|
|
headers["X-Agent99-Relay-Token"] = token
|
|
|
|
request = urllib.request.Request(target_url, data=data, headers=headers, method="POST")
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
status = int(response.getcode())
|
|
body = response.read(4096).decode("utf-8", errors="replace")
|
|
except urllib.error.HTTPError as exc:
|
|
body = exc.read(4096).decode("utf-8", errors="replace")
|
|
raise RuntimeError(f"Agent99 relay HTTP {exc.code}: {body[:300]}") from exc
|
|
|
|
if status >= 400:
|
|
raise RuntimeError(f"Agent99 relay HTTP {status}: {body[:300]}")
|
|
|
|
logger.info(
|
|
"agent99_sre_alert_relay_posted",
|
|
relay_url=_safe_url_for_log(target_url),
|
|
alert_id=payload.get("id"),
|
|
kind=payload.get("kind"),
|
|
status=status,
|
|
)
|
|
return {"status": status, "body": body}
|
|
|
|
|
|
def dispatch_agent99_sre_alert(payload: dict[str, Any]) -> str:
|
|
if settings.AGENT99_SRE_ALERT_RELAY_URL:
|
|
post_agent99_sre_alert(payload)
|
|
return "relay"
|
|
|
|
if settings.AGENT99_SRE_ALERT_INBOX_PATH:
|
|
written = write_agent99_sre_alert(payload)
|
|
if written is not None:
|
|
return "file"
|
|
|
|
logger.info(
|
|
"agent99_sre_bridge_disabled",
|
|
reason="no relay URL or mounted inbox path configured",
|
|
alert_id=payload.get("id"),
|
|
kind=payload.get("kind"),
|
|
)
|
|
return "disabled"
|
|
|
|
|
|
async def bridge_alertmanager_to_agent99(
|
|
*,
|
|
alert_id: str,
|
|
alertname: str,
|
|
severity: str,
|
|
namespace: str,
|
|
target_resource: str,
|
|
message: str,
|
|
labels: dict[str, Any] | None = None,
|
|
annotations: dict[str, Any] | None = None,
|
|
fingerprint: str = "",
|
|
alert_category: str = "",
|
|
notification_type: str = "",
|
|
source_url: str | None = None,
|
|
) -> None:
|
|
try:
|
|
payload = build_agent99_sre_alert(
|
|
alert_id=alert_id,
|
|
alertname=alertname,
|
|
severity=severity,
|
|
namespace=namespace,
|
|
target_resource=target_resource,
|
|
message=message,
|
|
labels=labels,
|
|
annotations=annotations,
|
|
fingerprint=fingerprint,
|
|
alert_category=alert_category,
|
|
notification_type=notification_type,
|
|
source_url=source_url,
|
|
)
|
|
await asyncio.to_thread(dispatch_agent99_sre_alert, payload)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"agent99_sre_bridge_failed_fail_open",
|
|
alert_id=alert_id,
|
|
alertname=alertname,
|
|
error=str(exc),
|
|
)
|