29072e4004
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m8s
CD Pipeline / build-and-deploy (push) Successful in 13m33s
CD Pipeline / post-deploy-checks (push) Has been cancelled
1321 lines
47 KiB
Python
1321 lines
47 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from src.core.config import settings
|
|
from src.core.logging import get_logger
|
|
from src.services.agent99_controlled_dispatch_ledger import (
|
|
attach_agent99_dispatch_identity,
|
|
build_agent99_dispatch_identity,
|
|
get_agent99_dispatch_ledger,
|
|
parse_agent99_dispatch_identity,
|
|
)
|
|
from src.services.agent99_public_receipts import (
|
|
extract_agent99_outcome_evidence_refs,
|
|
normalize_agent99_public_receipt_ref,
|
|
)
|
|
from src.utils.timezone import now_taipei
|
|
|
|
logger = get_logger("awoooi.agent99_sre_bridge")
|
|
|
|
AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS = 600
|
|
AGENT99_DURABLE_ROUTE_KINDS = {
|
|
"provider_freshness_signal",
|
|
"public_route_502",
|
|
"performance_pressure",
|
|
"backup_health",
|
|
"security_health",
|
|
"harbor_registry",
|
|
"awoooi_k3s",
|
|
"host_recovery",
|
|
}
|
|
|
|
_SENSITIVE_KEY_RE = re.compile(
|
|
r"(token|secret|password|passwd|authorization|cookie|session|private[_-]?key)",
|
|
re.IGNORECASE,
|
|
)
|
|
_SAFE_TOKEN_RE = re.compile(r"[^A-Za-z0-9_.-]+")
|
|
|
|
|
|
def _safe_text(value: Any, *, max_length: int = 1200) -> str:
|
|
text = str(value or "")
|
|
text = " ".join(text.split())
|
|
if len(text) > max_length:
|
|
return text[: max_length - 14] + "...(truncated)"
|
|
return text
|
|
|
|
|
|
def _safe_label_value(key: str, value: Any) -> str:
|
|
if _SENSITIVE_KEY_RE.search(key):
|
|
return "[redacted]"
|
|
return _safe_text(value, max_length=240)
|
|
|
|
|
|
def _safe_file_token(value: str) -> str:
|
|
safe = _SAFE_TOKEN_RE.sub("_", value).strip("._-")
|
|
return (safe or "alert")[:96]
|
|
|
|
|
|
def _safe_url_for_log(value: str) -> str:
|
|
if not value:
|
|
return ""
|
|
parsed = urllib.parse.urlsplit(value)
|
|
return urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", ""))
|
|
|
|
|
|
def _alert_text(
|
|
*,
|
|
alertname: str,
|
|
severity: str,
|
|
target_resource: str,
|
|
namespace: str,
|
|
message: str,
|
|
labels: dict[str, Any],
|
|
annotations: dict[str, Any],
|
|
) -> str:
|
|
return " ".join(
|
|
[
|
|
alertname,
|
|
severity,
|
|
target_resource,
|
|
namespace,
|
|
message,
|
|
" ".join(f"{k}={v}" for k, v in labels.items()),
|
|
" ".join(f"{k}={v}" for k, v in annotations.items()),
|
|
]
|
|
).lower()
|
|
|
|
|
|
def resolve_agent99_alert_kind(
|
|
*,
|
|
alertname: str,
|
|
severity: str,
|
|
target_resource: str,
|
|
namespace: str,
|
|
message: str,
|
|
labels: dict[str, Any] | None = None,
|
|
annotations: dict[str, Any] | None = None,
|
|
) -> str:
|
|
text = _alert_text(
|
|
alertname=alertname,
|
|
severity=severity,
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
message=message,
|
|
labels=labels or {},
|
|
annotations=annotations or {},
|
|
)
|
|
|
|
# Explicit backup identities win over generic freshness words. Backup
|
|
# failure cards intentionally mention freshness, but must stay in the
|
|
# read-only BackupCheck lane with its stricter no-delete/no-restore policy.
|
|
backup_identity = " ".join(
|
|
[
|
|
alertname,
|
|
target_resource,
|
|
str((labels or {}).get("event_type") or ""),
|
|
str((labels or {}).get("lane") or ""),
|
|
]
|
|
).lower()
|
|
if re.search(
|
|
r"backup_restore_escrow|backup[-_ ]restore|backupcredentialescrow|"
|
|
r"backupaggregaterun|backupconfigcapture|escrow_missing|"
|
|
r"restore[_ -]drill|restic|rclone|offsite",
|
|
backup_identity,
|
|
):
|
|
return "backup_health"
|
|
|
|
if re.search(
|
|
r"host112guest(?:readbackmissing|notready)|host112_guest|"
|
|
r"awoooi_host112_|host 112.*(?:guest|console|lightdm|vmware tools)",
|
|
text,
|
|
):
|
|
return "host112_guest_recovery"
|
|
if re.search(
|
|
r"provider_freshness_signal|source_provider_freshness_triage|"
|
|
r"raw_signal_normalized|controlled_runtime_candidate|freshness|"
|
|
r"last_seen|last seen|ingestion",
|
|
text,
|
|
):
|
|
return "provider_freshness_signal"
|
|
if re.search(
|
|
r"wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|"
|
|
r"privilege|vulnerability|\bcve\b|\bioc\b|suspicious|unauthorized",
|
|
text,
|
|
):
|
|
return "security_health"
|
|
if re.search(r"backup|snapshot|restore drill|restore_drill|backup cron", text):
|
|
return "backup_health"
|
|
if re.search(
|
|
r"postgres|postgresql|mysql|mariadb|database|db_connection|"
|
|
r"pg_isready|connection pool|replication lag",
|
|
text,
|
|
):
|
|
return "database_health"
|
|
if re.search(
|
|
r"alertchain|alert_chain|alert chain|alertmanager.*(broken|down|failed)|"
|
|
r"monitoring pipeline|notification pipeline",
|
|
text,
|
|
):
|
|
return "alert_chain_health"
|
|
if re.search(
|
|
r"reboot-auto-recovery|reboot_auto_recovery|reboot recovery|"
|
|
r"cold[-_ ]start(?:[-_ ]gate)?|full[-_ ]stack[-_ ]cold[-_ ]start|"
|
|
r"host_down|hostdown|host_reboot|vm_not_running|powered_off|"
|
|
r"all_required_hosts_not_in_10_minute_reboot_window",
|
|
text,
|
|
):
|
|
return "host_recovery"
|
|
if re.search(r"502|site_502|public_route|website|route_down|nginx upstream", text):
|
|
return "public_route_502"
|
|
if re.search(r"cpu|load|memory|mem_|disk|pressure|performance", text):
|
|
return "performance_pressure"
|
|
if re.search(r"harbor|registry", text):
|
|
return "harbor_registry"
|
|
if re.search(
|
|
r"\bci/?cd\b|\bcicd\b|build-and-deploy|build_and_deploy|"
|
|
r"deploy(?:ment)?[_ -](?:failed|failure)|build[_ -](?:failed|failure)|"
|
|
r"部署失敗|構建失敗",
|
|
text,
|
|
):
|
|
return "cicd_failure"
|
|
if re.search(
|
|
r"k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment", text
|
|
):
|
|
return "awoooi_k3s"
|
|
if re.search(
|
|
r"missing_key|e_ai_missing_key|application_ai_config_error|"
|
|
r"locale guard|ai support",
|
|
text,
|
|
):
|
|
return "application_ai_config_error"
|
|
return "monitoring_alert"
|
|
|
|
|
|
def _suggested_mode_for_kind(kind: str) -> str | None:
|
|
return {
|
|
"provider_freshness_signal": "ProviderFreshness",
|
|
"public_route_502": "Recover",
|
|
"performance_pressure": "Perf",
|
|
"backup_health": "BackupCheck",
|
|
"security_health": "SecurityTriage",
|
|
"database_health": "Status",
|
|
"alert_chain_health": "Status",
|
|
"harbor_registry": "HarborRepair",
|
|
"awoooi_k3s": "AwoooRepair",
|
|
"host_recovery": "Recover",
|
|
"host112_guest_recovery": "Recover",
|
|
"application_ai_config_error": "Status",
|
|
"cicd_failure": "Status",
|
|
"monitoring_alert": "Status",
|
|
}.get(kind)
|
|
|
|
|
|
def resolve_agent99_durable_route(
|
|
*,
|
|
alertname: str,
|
|
severity: str,
|
|
target_resource: str,
|
|
namespace: str,
|
|
message: str,
|
|
labels: dict[str, Any] | None = None,
|
|
annotations: dict[str, Any] | None = None,
|
|
) -> dict[str, str] | None:
|
|
"""Return the single-owner route for Agent99-actionable domains."""
|
|
|
|
kind = resolve_agent99_alert_kind(
|
|
alertname=alertname,
|
|
severity=severity,
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
message=message,
|
|
labels=labels,
|
|
annotations=annotations,
|
|
)
|
|
if kind not in AGENT99_DURABLE_ROUTE_KINDS:
|
|
return None
|
|
suggested_mode = _suggested_mode_for_kind(kind) or "Status"
|
|
return {
|
|
"kind": kind,
|
|
"suggested_mode": suggested_mode,
|
|
"route_id": f"agent99:{kind}:{suggested_mode}",
|
|
}
|
|
|
|
|
|
def _resolution_policy_for_kind(kind: str) -> str:
|
|
if kind in {
|
|
"provider_freshness_signal",
|
|
"public_route_502",
|
|
"performance_pressure",
|
|
"backup_health",
|
|
"harbor_registry",
|
|
"awoooi_k3s",
|
|
"host_recovery",
|
|
"host112_guest_recovery",
|
|
}:
|
|
return "mode_verifier_can_resolve"
|
|
return "external_source_receipt_required"
|
|
|
|
|
|
def _agent99_route_group_key(
|
|
*,
|
|
kind: str,
|
|
target_resource: str,
|
|
host: str,
|
|
) -> str:
|
|
parts = (
|
|
kind.strip().lower() or "monitoring_alert",
|
|
target_resource.strip().lower() or "unknown",
|
|
host.strip().lower() or "no-host",
|
|
)
|
|
return ":".join(_safe_file_token(part) for part in parts)
|
|
|
|
|
|
def agent99_sre_single_flight_key(fingerprint: str) -> str:
|
|
digest = hashlib.sha256(fingerprint.encode("utf-8")).hexdigest()
|
|
return f"agent99:sre_dispatch:single_flight:{digest}"
|
|
|
|
|
|
async def try_acquire_agent99_sre_single_flight(
|
|
fingerprint: str,
|
|
alert_id: str,
|
|
*,
|
|
ttl_seconds: int = AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS,
|
|
) -> bool:
|
|
"""Compatibility wrapper for read-only/non-mutating callers."""
|
|
|
|
decision = await acquire_agent99_sre_single_flight(
|
|
fingerprint,
|
|
alert_id,
|
|
ttl_seconds=ttl_seconds,
|
|
fail_closed=False,
|
|
)
|
|
return bool(decision["acquired"])
|
|
|
|
|
|
async def acquire_agent99_sre_single_flight(
|
|
fingerprint: str,
|
|
alert_id: str,
|
|
*,
|
|
ttl_seconds: int = AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS,
|
|
fail_closed: bool,
|
|
) -> dict[str, Any]:
|
|
"""Acquire the shared Redis lock with explicit failure semantics.
|
|
|
|
Mutating modes pass ``fail_closed=True``. Redis unavailability therefore
|
|
produces zero dispatches instead of silently allowing two Recover actions.
|
|
"""
|
|
|
|
if not fingerprint:
|
|
return {
|
|
"acquired": not fail_closed,
|
|
"reason": (
|
|
"missing_idempotency_identity_fail_closed"
|
|
if fail_closed
|
|
else "missing_identity_readonly_allowed"
|
|
),
|
|
}
|
|
try:
|
|
from src.core.redis_client import get_redis
|
|
|
|
redis = get_redis()
|
|
acquired = await redis.set(
|
|
agent99_sre_single_flight_key(fingerprint),
|
|
alert_id,
|
|
ex=ttl_seconds,
|
|
nx=True,
|
|
)
|
|
return {
|
|
"acquired": bool(acquired),
|
|
"reason": "acquired" if acquired else "single_flight_duplicate",
|
|
}
|
|
except Exception as exc:
|
|
logger.warning(
|
|
(
|
|
"agent99_sre_single_flight_failed_fail_closed"
|
|
if fail_closed
|
|
else "agent99_sre_single_flight_failed_readonly_fail_open"
|
|
),
|
|
fingerprint=fingerprint,
|
|
alert_id=alert_id,
|
|
error=str(exc),
|
|
)
|
|
return {
|
|
"acquired": not fail_closed,
|
|
"reason": (
|
|
"redis_unavailable_fail_closed"
|
|
if fail_closed
|
|
else "redis_unavailable_readonly_fail_open"
|
|
),
|
|
}
|
|
|
|
|
|
async def release_agent99_sre_single_flight(fingerprint: str, alert_id: str) -> None:
|
|
"""Release only the caller-owned lock when transport did not accept the alert."""
|
|
|
|
if not fingerprint:
|
|
return
|
|
try:
|
|
from src.core.redis_client import get_redis
|
|
|
|
redis = get_redis()
|
|
await redis.eval(
|
|
"""
|
|
if redis.call('get', KEYS[1]) == ARGV[1] then
|
|
return redis.call('del', KEYS[1])
|
|
end
|
|
return 0
|
|
""",
|
|
1,
|
|
agent99_sre_single_flight_key(fingerprint),
|
|
alert_id,
|
|
)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"agent99_sre_single_flight_release_failed",
|
|
fingerprint=fingerprint,
|
|
alert_id=alert_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
|
|
def build_agent99_sre_alert(
|
|
*,
|
|
alert_id: str,
|
|
alertname: str,
|
|
severity: str,
|
|
namespace: str,
|
|
target_resource: str,
|
|
message: str,
|
|
labels: dict[str, Any] | None = None,
|
|
annotations: dict[str, Any] | None = None,
|
|
fingerprint: str = "",
|
|
alert_category: str = "",
|
|
notification_type: str = "",
|
|
source_url: str | None = None,
|
|
) -> dict[str, Any]:
|
|
labels = labels or {}
|
|
annotations = annotations or {}
|
|
normalized_severity = (severity or "warning").lower()
|
|
kind = resolve_agent99_alert_kind(
|
|
alertname=alertname,
|
|
severity=normalized_severity,
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
message=message,
|
|
labels=labels,
|
|
annotations=annotations,
|
|
)
|
|
suggested_mode = _suggested_mode_for_kind(kind)
|
|
host = str(labels.get("host") or labels.get("node") or labels.get("instance") or "")
|
|
if ":" in host:
|
|
host = host.split(":", 1)[0]
|
|
service = str(
|
|
labels.get("service")
|
|
or labels.get("app")
|
|
or labels.get("component")
|
|
or target_resource
|
|
or "awoooi"
|
|
)
|
|
route_group_key = _agent99_route_group_key(
|
|
kind=kind,
|
|
target_resource=target_resource,
|
|
host=host,
|
|
)
|
|
safe_labels = [
|
|
"alertmanager",
|
|
"awoooi-api",
|
|
f"alertname={_safe_label_value('alertname', alertname)}",
|
|
f"namespace={_safe_label_value('namespace', namespace)}",
|
|
f"target={_safe_label_value('target', target_resource)}",
|
|
]
|
|
if fingerprint:
|
|
safe_labels.append(
|
|
f"fingerprint={_safe_label_value('fingerprint', fingerprint)}"
|
|
)
|
|
for key, value in sorted(labels.items()):
|
|
safe_labels.append(
|
|
f"{_safe_text(key, max_length=80)}={_safe_label_value(key, value)}"
|
|
)
|
|
if kind == "provider_freshness_signal":
|
|
safe_labels.extend(["provider-freshness", "no-false-green"])
|
|
elif kind == "backup_health":
|
|
safe_labels.extend(["backup-readback", "no-false-green", "read-only"])
|
|
|
|
instruction = None
|
|
if kind == "provider_freshness_signal":
|
|
instruction = (
|
|
"Provider freshness candidate required; require providerWindow,lastSeen,"
|
|
"directReference,verifierReadback; mark missing,expired,timeout,"
|
|
"no_false_green; do not switch provider, call paid model, edit env, or reload."
|
|
)
|
|
elif kind == "backup_health":
|
|
instruction = (
|
|
"BackupCheck readback candidate only; collect backup status, freshness, "
|
|
"offsite verification, escrow missing count, restore drill and source "
|
|
"resolution receipts; no_false_green; do not run backup/restore, prune, "
|
|
"remote delete, change retention, write escrow markers, or read secrets."
|
|
)
|
|
elif kind == "host112_guest_recovery":
|
|
instruction = (
|
|
"Run the allowlisted host 112 guest recovery; verify graphical.target, "
|
|
"LightDM, VMware Tools, Wazuh services and recovery timer; do not reboot "
|
|
"the host, change VM power, reset the VM, read secrets or change firewall."
|
|
)
|
|
|
|
payload: dict[str, Any] = {
|
|
"id": f"awoooi-alertmanager-{_safe_file_token(alert_id)}",
|
|
"source": "awoooi-api-alertmanager",
|
|
"severity": normalized_severity,
|
|
"kind": kind,
|
|
"service": _safe_text(service, max_length=160),
|
|
"host": _safe_text(host, max_length=160),
|
|
"title": _safe_text(f"Alertmanager {alertname}", max_length=240),
|
|
"message": _safe_text(message, max_length=2000),
|
|
"labels": safe_labels,
|
|
"force": True,
|
|
"controlledApply": suggested_mode
|
|
in {
|
|
"Recover",
|
|
"StartVMs",
|
|
"HarborRepair",
|
|
"AwoooRepair",
|
|
"Perf",
|
|
"LoadShed",
|
|
},
|
|
"createdAt": now_taipei().isoformat(),
|
|
"routing": {
|
|
"schemaVersion": "agent99_alert_route_v1",
|
|
"kind": kind,
|
|
"suggestedMode": suggested_mode or "Status",
|
|
"routeSource": "structured_kind",
|
|
"groupKey": route_group_key,
|
|
"correlationKey": route_group_key,
|
|
"sourceFingerprint": _safe_text(fingerprint, max_length=160),
|
|
"singleFlightWindowSeconds": AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS,
|
|
},
|
|
"resolution": {
|
|
"schemaVersion": "agent99_source_event_resolution_v1",
|
|
"policy": _resolution_policy_for_kind(kind),
|
|
"required": True,
|
|
},
|
|
"awoooi": {
|
|
"alertId": _safe_text(alert_id, max_length=160),
|
|
"alertname": _safe_text(alertname, max_length=160),
|
|
"namespace": _safe_text(namespace, max_length=160),
|
|
"targetResource": _safe_text(target_resource, max_length=240),
|
|
"fingerprint": _safe_text(fingerprint, max_length=160),
|
|
"alertCategory": _safe_text(alert_category, max_length=160),
|
|
"notificationType": _safe_text(notification_type, max_length=160),
|
|
"sourceUrl": _safe_text(source_url or "", max_length=600),
|
|
},
|
|
}
|
|
if suggested_mode:
|
|
payload["suggestedMode"] = suggested_mode
|
|
if instruction:
|
|
payload["instruction"] = instruction
|
|
return payload
|
|
|
|
|
|
def write_agent99_sre_alert(
|
|
payload: dict[str, Any],
|
|
*,
|
|
inbox_path: str | None = None,
|
|
) -> Path | None:
|
|
target_dir_text = (
|
|
inbox_path if inbox_path is not None else settings.AGENT99_SRE_ALERT_INBOX_PATH
|
|
)
|
|
if not target_dir_text:
|
|
logger.info(
|
|
"agent99_sre_bridge_disabled",
|
|
reason="AGENT99_SRE_ALERT_INBOX_PATH not configured",
|
|
alert_id=payload.get("id"),
|
|
kind=payload.get("kind"),
|
|
)
|
|
return None
|
|
|
|
target_dir = Path(target_dir_text)
|
|
target_dir.mkdir(parents=True, exist_ok=True)
|
|
alert_id = _safe_file_token(str(payload.get("id") or uuid.uuid4().hex))
|
|
final_path = target_dir / f"{alert_id}.json"
|
|
tmp_path = target_dir / f".{alert_id}.{uuid.uuid4().hex}.tmp"
|
|
tmp_path.write_text(
|
|
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True),
|
|
encoding="utf-8",
|
|
)
|
|
tmp_path.replace(final_path)
|
|
logger.info(
|
|
"agent99_sre_alert_written",
|
|
path=str(final_path),
|
|
alert_id=payload.get("id"),
|
|
kind=payload.get("kind"),
|
|
)
|
|
return final_path
|
|
|
|
|
|
def post_agent99_sre_alert(
|
|
payload: dict[str, Any],
|
|
*,
|
|
relay_url: str | None = None,
|
|
relay_token: str | None = None,
|
|
timeout_seconds: float | None = None,
|
|
) -> dict[str, Any] | None:
|
|
target_url = (
|
|
relay_url if relay_url is not None else settings.AGENT99_SRE_ALERT_RELAY_URL
|
|
)
|
|
if not target_url:
|
|
return None
|
|
|
|
token = (
|
|
relay_token
|
|
if relay_token is not None
|
|
else settings.AGENT99_SRE_ALERT_RELAY_TOKEN
|
|
)
|
|
timeout = (
|
|
timeout_seconds
|
|
if timeout_seconds is not None
|
|
else settings.AGENT99_SRE_ALERT_RELAY_TIMEOUT_SECONDS
|
|
)
|
|
data = json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
|
headers = {
|
|
"Content-Type": "application/json; charset=utf-8",
|
|
"User-Agent": "awoooi-agent99-sre-bridge/1",
|
|
}
|
|
if token:
|
|
headers["X-Agent99-Relay-Token"] = token
|
|
|
|
request = urllib.request.Request(
|
|
target_url, data=data, headers=headers, method="POST"
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
status = int(response.getcode())
|
|
body = response.read(4096).decode("utf-8", errors="replace")
|
|
except urllib.error.HTTPError as exc:
|
|
body = exc.read(4096).decode("utf-8", errors="replace")
|
|
raise RuntimeError(f"Agent99 relay HTTP {exc.code}: {body[:300]}") from exc
|
|
|
|
if status >= 400:
|
|
raise RuntimeError(f"Agent99 relay HTTP {status}: {body[:300]}")
|
|
|
|
logger.info(
|
|
"agent99_sre_alert_relay_posted",
|
|
relay_url=_safe_url_for_log(target_url),
|
|
alert_id=payload.get("id"),
|
|
kind=payload.get("kind"),
|
|
status=status,
|
|
)
|
|
return {"status": status, "body": body}
|
|
|
|
|
|
def read_agent99_sre_outcome(
|
|
run_id: str,
|
|
*,
|
|
relay_url: str | None = None,
|
|
relay_token: str | None = None,
|
|
timeout_seconds: float | None = None,
|
|
) -> dict[str, Any] | None:
|
|
"""Read one public-safe Agent99 outcome from the authenticated relay."""
|
|
|
|
target_url = (
|
|
relay_url if relay_url is not None else settings.AGENT99_SRE_ALERT_RELAY_URL
|
|
)
|
|
token = (
|
|
relay_token
|
|
if relay_token is not None
|
|
else settings.AGENT99_SRE_ALERT_RELAY_TOKEN
|
|
)
|
|
if not target_url or not token:
|
|
return None
|
|
timeout = (
|
|
timeout_seconds
|
|
if timeout_seconds is not None
|
|
else settings.AGENT99_SRE_ALERT_RELAY_TIMEOUT_SECONDS
|
|
)
|
|
parsed = urllib.parse.urlsplit(target_url)
|
|
query = urllib.parse.urlencode({"run_id": str(run_id)})
|
|
readback_url = urllib.parse.urlunsplit(
|
|
(parsed.scheme, parsed.netloc, parsed.path, query, "")
|
|
)
|
|
request = urllib.request.Request(
|
|
readback_url,
|
|
headers={
|
|
"Accept": "application/json",
|
|
"User-Agent": "awoooi-agent99-outcome-reconciler/1",
|
|
"X-Agent99-Relay-Token": token,
|
|
},
|
|
method="GET",
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
if int(response.getcode()) != 200:
|
|
return None
|
|
body = response.read(65536).decode("utf-8", errors="replace")
|
|
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as exc:
|
|
logger.info(
|
|
"agent99_outcome_readback_pending",
|
|
run_id=str(run_id),
|
|
error_type=type(exc).__name__,
|
|
)
|
|
return None
|
|
try:
|
|
payload = json.loads(body)
|
|
except json.JSONDecodeError:
|
|
return None
|
|
if not isinstance(payload, dict) or payload.get("found") is not True:
|
|
return None
|
|
if str(payload.get("automationRunId") or "") != str(run_id):
|
|
return None
|
|
raw_identity = (
|
|
payload.get("identity")
|
|
if isinstance(payload.get("identity"), dict)
|
|
else {}
|
|
)
|
|
outcome = payload.get("outcome") if isinstance(payload.get("outcome"), dict) else {}
|
|
nested_identity = (
|
|
outcome.get("identity")
|
|
if isinstance(outcome.get("identity"), dict)
|
|
else {}
|
|
)
|
|
try:
|
|
identity = parse_agent99_dispatch_identity(raw_identity)
|
|
nested = parse_agent99_dispatch_identity(nested_identity)
|
|
except ValueError:
|
|
return None
|
|
if (
|
|
identity != nested
|
|
or str(identity.run_id) != str(run_id)
|
|
or identity.public_dict() != nested.public_dict()
|
|
):
|
|
return None
|
|
safe_evidence_refs = extract_agent99_outcome_evidence_refs(payload)
|
|
source_event_evidence = normalize_agent99_public_receipt_ref(
|
|
outcome.get("sourceEventEvidence")
|
|
)
|
|
expected_evidence_receipt_id = (
|
|
f"same-run-status-{identity.run_id}-"
|
|
f"{identity.public_dict()['canonical_digest'][:12]}"
|
|
)
|
|
evidence_receipt_id = str(payload.get("evidenceReceiptId") or "")
|
|
if evidence_receipt_id != expected_evidence_receipt_id:
|
|
evidence_receipt_id = ""
|
|
same_run_contract_fields_complete = bool(
|
|
payload.get("sameRunContractFieldsComplete") is True
|
|
and all(
|
|
key in payload
|
|
for key in (
|
|
"automationRunId",
|
|
"traceId",
|
|
"workItemId",
|
|
"projectId",
|
|
"incidentId",
|
|
"routeId",
|
|
"executionGeneration",
|
|
"approvalId",
|
|
"evidenceReceiptId",
|
|
"controlledApply",
|
|
"reconcileOnly",
|
|
"runtimeWritePerformed",
|
|
"mode",
|
|
)
|
|
)
|
|
and all(
|
|
isinstance(payload.get(key), str)
|
|
for key in (
|
|
"automationRunId",
|
|
"traceId",
|
|
"workItemId",
|
|
"projectId",
|
|
"incidentId",
|
|
"routeId",
|
|
"executionGeneration",
|
|
"approvalId",
|
|
"evidenceReceiptId",
|
|
"mode",
|
|
)
|
|
)
|
|
and isinstance(payload.get("controlledApply"), bool)
|
|
and isinstance(payload.get("reconcileOnly"), bool)
|
|
and isinstance(payload.get("runtimeWritePerformed"), bool)
|
|
and all(
|
|
key in outcome
|
|
for key in (
|
|
"schemaVersion",
|
|
"state",
|
|
"resolved",
|
|
"transportOk",
|
|
"verifierName",
|
|
"verifierPassed",
|
|
"sourceEventResolved",
|
|
"sourceEventEvidence",
|
|
"identity",
|
|
)
|
|
)
|
|
and all(
|
|
isinstance(outcome.get(key), str)
|
|
for key in (
|
|
"schemaVersion",
|
|
"state",
|
|
"verifierName",
|
|
"sourceEventEvidence",
|
|
)
|
|
)
|
|
and isinstance(outcome.get("resolved"), bool)
|
|
and isinstance(outcome.get("transportOk"), bool)
|
|
and isinstance(outcome.get("verifierPassed"), bool)
|
|
and isinstance(outcome.get("sourceEventResolved"), bool)
|
|
)
|
|
return {
|
|
"schemaVersion": "agent99_outcome_readback_v1",
|
|
"automationRunId": str(payload.get("automationRunId") or ""),
|
|
"traceId": str(payload.get("traceId") or ""),
|
|
"workItemId": str(payload.get("workItemId") or ""),
|
|
"projectId": str(payload.get("projectId") or ""),
|
|
"routeId": str(payload.get("routeId") or ""),
|
|
"idempotencyKey": str(payload.get("idempotencyKey") or ""),
|
|
"executionGeneration": str(payload.get("executionGeneration") or "1"),
|
|
"incidentId": str(payload.get("incidentId") or ""),
|
|
"approvalId": str(payload.get("approvalId") or ""),
|
|
"evidenceReceiptId": evidence_receipt_id,
|
|
"sameRunContractFieldsComplete": same_run_contract_fields_complete,
|
|
"identity": identity.public_dict(),
|
|
"controlledApply": bool(payload.get("controlledApply") is True),
|
|
"reconcileOnly": bool(payload.get("reconcileOnly") is True),
|
|
"runtimeWritePerformed": bool(
|
|
payload.get("runtimeWritePerformed") is True
|
|
),
|
|
"mode": str(payload.get("mode") or ""),
|
|
"outcome": {
|
|
"schemaVersion": str(outcome.get("schemaVersion") or ""),
|
|
"state": str(outcome.get("state") or "unknown"),
|
|
"resolved": bool(outcome.get("resolved") is True),
|
|
"transportOk": bool(outcome.get("transportOk") is True),
|
|
"verifierName": str(outcome.get("verifierName") or ""),
|
|
"verifierPassed": bool(outcome.get("verifierPassed") is True),
|
|
"sourceEventResolved": bool(
|
|
outcome.get("sourceEventResolved") is True
|
|
),
|
|
"sourceEventEvidence": source_event_evidence or "",
|
|
"failedChecks": [
|
|
str(value)[:120]
|
|
for value in (outcome.get("failedChecks") or [])[:32]
|
|
],
|
|
"checks": [
|
|
{
|
|
"name": str(check.get("name") or "")[:120],
|
|
"required": bool(check.get("required") is True),
|
|
"passed": bool(check.get("passed") is True),
|
|
}
|
|
for check in (outcome.get("checks") or [])[:32]
|
|
if isinstance(check, dict)
|
|
],
|
|
"identity": identity.public_dict(),
|
|
"evidenceRefs": safe_evidence_refs,
|
|
"verifiedAt": str(outcome.get("verifiedAt") or "")[:64],
|
|
},
|
|
"storesRawEvidence": False,
|
|
}
|
|
|
|
|
|
def dispatch_agent99_sre_alert(payload: dict[str, Any]) -> str:
|
|
receipt = dispatch_agent99_sre_alert_with_receipt(payload)
|
|
return str(receipt["transport"])
|
|
|
|
|
|
def dispatch_agent99_sre_alert_with_receipt(
|
|
payload: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
"""Dispatch one alert and retain only public-safe acceptance evidence."""
|
|
|
|
alert_id = _safe_text(payload.get("id"), max_length=160)
|
|
kind = _safe_text(payload.get("kind"), max_length=120)
|
|
awoooi = (
|
|
payload.get("awoooi")
|
|
if isinstance(payload.get("awoooi"), dict)
|
|
else {}
|
|
)
|
|
target_resource = _safe_text(awoooi.get("targetResource"), max_length=240)
|
|
suggested_mode = _safe_text(payload.get("suggestedMode"), max_length=80)
|
|
controlled_apply_requested = bool(payload.get("controlledApply") is True)
|
|
if settings.AGENT99_SRE_ALERT_RELAY_URL:
|
|
raw_receipt = post_agent99_sre_alert(payload) or {}
|
|
response_payload: dict[str, Any] = {}
|
|
try:
|
|
parsed = json.loads(str(raw_receipt.get("body") or ""))
|
|
if isinstance(parsed, dict):
|
|
response_payload = parsed
|
|
except json.JSONDecodeError:
|
|
response_payload = {}
|
|
http_status = int(raw_receipt.get("status") or 0)
|
|
accepted = bool(200 <= http_status < 300 and response_payload.get("ok") is True)
|
|
inbox_triggered = bool(response_payload.get("inboxTriggered") is True)
|
|
return {
|
|
"schema_version": "agent99_sre_dispatch_receipt_v1",
|
|
"status": (
|
|
"accepted_inbox_triggered"
|
|
if accepted and inbox_triggered
|
|
else "accepted_inbox_pending"
|
|
if accepted
|
|
else "relay_response_not_accepted"
|
|
),
|
|
"transport": "relay",
|
|
"alert_id": alert_id,
|
|
"kind": kind,
|
|
"target_resource": target_resource,
|
|
"suggested_mode": suggested_mode,
|
|
"controlled_apply_requested": controlled_apply_requested,
|
|
"http_status": http_status,
|
|
"accepted": accepted,
|
|
"inbox_triggered": inbox_triggered,
|
|
"delivery_certainty": (
|
|
"delivered"
|
|
if accepted and inbox_triggered
|
|
else "unknown"
|
|
if accepted
|
|
else "not_delivered"
|
|
),
|
|
"stores_raw_response": False,
|
|
}
|
|
|
|
if settings.AGENT99_SRE_ALERT_INBOX_PATH:
|
|
written = write_agent99_sre_alert(payload)
|
|
if written is not None:
|
|
return {
|
|
"schema_version": "agent99_sre_dispatch_receipt_v1",
|
|
"status": "file_written",
|
|
"transport": "file",
|
|
"alert_id": alert_id,
|
|
"kind": kind,
|
|
"target_resource": target_resource,
|
|
"suggested_mode": suggested_mode,
|
|
"controlled_apply_requested": controlled_apply_requested,
|
|
"accepted": True,
|
|
"inbox_triggered": False,
|
|
# A file write is not proof that the Agent99 inbox consumed it.
|
|
"delivery_certainty": "unknown",
|
|
"stores_raw_response": False,
|
|
}
|
|
|
|
logger.info(
|
|
"agent99_sre_bridge_disabled",
|
|
reason="no relay URL or mounted inbox path configured",
|
|
alert_id=payload.get("id"),
|
|
kind=payload.get("kind"),
|
|
)
|
|
return {
|
|
"schema_version": "agent99_sre_dispatch_receipt_v1",
|
|
"status": "bridge_disabled",
|
|
"transport": "disabled",
|
|
"alert_id": alert_id,
|
|
"kind": kind,
|
|
"target_resource": target_resource,
|
|
"suggested_mode": suggested_mode,
|
|
"controlled_apply_requested": controlled_apply_requested,
|
|
"accepted": False,
|
|
"inbox_triggered": False,
|
|
"delivery_certainty": "not_delivered",
|
|
"stores_raw_response": False,
|
|
}
|
|
|
|
|
|
async def bridge_alertmanager_to_agent99(
|
|
*,
|
|
alert_id: str,
|
|
alertname: str,
|
|
severity: str,
|
|
namespace: str,
|
|
target_resource: str,
|
|
message: str,
|
|
labels: dict[str, Any] | None = None,
|
|
annotations: dict[str, Any] | None = None,
|
|
fingerprint: str = "",
|
|
alert_category: str = "",
|
|
notification_type: str = "",
|
|
source_url: str | None = None,
|
|
project_id: str = "awoooi",
|
|
incident_id: str = "",
|
|
approval_id: str = "",
|
|
route_id: str = "",
|
|
work_item_id: str = "",
|
|
execution_generation: str = "1",
|
|
) -> dict[str, Any]:
|
|
single_flight_identity = ""
|
|
single_flight_owner = alert_id
|
|
single_flight_acquired = False
|
|
delivery_attempt_started = False
|
|
dispatch_identity = None
|
|
claim_token = ""
|
|
ledger = get_agent99_dispatch_ledger()
|
|
try:
|
|
payload = build_agent99_sre_alert(
|
|
alert_id=alert_id,
|
|
alertname=alertname,
|
|
severity=severity,
|
|
namespace=namespace,
|
|
target_resource=target_resource,
|
|
message=message,
|
|
labels=labels,
|
|
annotations=annotations,
|
|
fingerprint=fingerprint,
|
|
alert_category=alert_category,
|
|
notification_type=notification_type,
|
|
source_url=source_url,
|
|
)
|
|
routing = (
|
|
payload.get("routing") if isinstance(payload.get("routing"), dict) else {}
|
|
)
|
|
suggested_mode = str(payload.get("suggestedMode") or "Status")
|
|
mutating = bool(payload.get("controlledApply") is True)
|
|
resolved_route_id = (
|
|
str(route_id or "").strip()
|
|
or f"agent99:{str(payload.get('kind') or 'monitoring_alert')}:{suggested_mode}"
|
|
)
|
|
durable_route = bool(
|
|
str(incident_id or "").strip()
|
|
or str(route_id or "").strip()
|
|
or str(work_item_id or "").strip()
|
|
)
|
|
if mutating and not durable_route:
|
|
logger.warning(
|
|
"agent99_mutating_dispatch_identity_missing_fail_closed",
|
|
alert_id=alert_id,
|
|
incident_id=incident_id,
|
|
reason="agent99_dispatch_incident_id_required",
|
|
)
|
|
return {
|
|
"status": "failed",
|
|
"reason": "agent99_dispatch_incident_id_required",
|
|
"dispatchPerformed": False,
|
|
"runtimeClosureVerified": False,
|
|
}
|
|
if durable_route:
|
|
try:
|
|
generation = max(1, int(str(execution_generation or "1")))
|
|
except ValueError:
|
|
generation = 1
|
|
generation = min(generation, 3)
|
|
base_payload = payload
|
|
while True:
|
|
try:
|
|
dispatch_identity = build_agent99_dispatch_identity(
|
|
project_id=project_id,
|
|
incident_id=incident_id,
|
|
source_fingerprint=fingerprint,
|
|
route_id=resolved_route_id,
|
|
execution_generation=str(generation),
|
|
approval_id=approval_id,
|
|
work_item_id=work_item_id,
|
|
)
|
|
except ValueError as exc:
|
|
logger.warning(
|
|
"agent99_durable_dispatch_identity_missing_fail_closed",
|
|
alert_id=alert_id,
|
|
incident_id=incident_id,
|
|
reason=str(exc),
|
|
)
|
|
return {
|
|
"status": "failed",
|
|
"reason": str(exc),
|
|
"dispatchPerformed": False,
|
|
"runtimeClosureVerified": False,
|
|
}
|
|
payload = attach_agent99_dispatch_identity(
|
|
base_payload,
|
|
dispatch_identity,
|
|
)
|
|
# Durable structured identity makes the inbox actionable.
|
|
# Mutating routes must not bypass routing checks; read-only
|
|
# BackupCheck retains controlledApply=false/no-write policy.
|
|
if mutating:
|
|
payload["force"] = False
|
|
reservation = await ledger.reserve(
|
|
identity=dispatch_identity,
|
|
payload=payload,
|
|
)
|
|
if reservation.get("claimed") is True:
|
|
claim_token = str(reservation.get("claim_token") or "")
|
|
if not claim_token:
|
|
return {
|
|
"status": "failed",
|
|
"reason": "agent99_dispatch_claim_token_missing",
|
|
"dispatchPerformed": False,
|
|
"identity": dispatch_identity.public_dict(),
|
|
"runtimeClosureVerified": False,
|
|
}
|
|
break
|
|
|
|
existing_receipt = reservation.get("receipt")
|
|
retry_policy = (
|
|
existing_receipt.get("retry_policy")
|
|
if isinstance(existing_receipt, dict)
|
|
and isinstance(existing_receipt.get("retry_policy"), dict)
|
|
else {}
|
|
)
|
|
max_generation = min(
|
|
3,
|
|
int(retry_policy.get("max_generation") or 3),
|
|
)
|
|
if reservation.get("status") == "retry_not_before":
|
|
return {
|
|
"status": "retryable",
|
|
"reason": "durable_retry_not_before",
|
|
"dispatchPerformed": False,
|
|
"nextAttemptAt": reservation.get("next_attempt_at"),
|
|
"correlatedReceipt": existing_receipt,
|
|
"identity": dispatch_identity.public_dict(),
|
|
"runtimeClosureVerified": False,
|
|
}
|
|
if (
|
|
retry_policy.get("safe_to_retry") is True
|
|
and generation < max_generation
|
|
):
|
|
generation += 1
|
|
continue
|
|
|
|
dispatch_receipt = (
|
|
existing_receipt.get("dispatch_receipt")
|
|
if isinstance(existing_receipt, dict)
|
|
and isinstance(existing_receipt.get("dispatch_receipt"), dict)
|
|
else {}
|
|
)
|
|
accepted = bool(dispatch_receipt.get("accepted") is True)
|
|
inbox_triggered = bool(
|
|
dispatch_receipt.get("inbox_triggered") is True
|
|
)
|
|
dispatch_promoted = bool(accepted and inbox_triggered)
|
|
reconcile_only = bool(
|
|
retry_policy.get("reconcile_only") is True
|
|
or reservation.get("status") == "idempotent_running"
|
|
or (
|
|
isinstance(existing_receipt, dict)
|
|
and existing_receipt.get("status")
|
|
== "dispatch_delivery_unknown_reconcile_only"
|
|
)
|
|
)
|
|
return {
|
|
"status": (
|
|
"deduplicated"
|
|
if dispatch_promoted
|
|
else "delivery_unknown"
|
|
if reconcile_only
|
|
else "suppressed"
|
|
),
|
|
"reason": str(
|
|
reservation.get("status") or "pg_idempotency_hit"
|
|
),
|
|
"dispatchPerformed": False,
|
|
"dispatchReceipt": dispatch_receipt or None,
|
|
"correlatedReceipt": existing_receipt,
|
|
"identity": dispatch_identity.public_dict(),
|
|
"runtimeClosureVerified": False,
|
|
}
|
|
# Generation retries share one Redis flight key; otherwise a new
|
|
# generation could race an older still-live transport lock.
|
|
single_flight_identity = dispatch_identity.single_flight_key
|
|
single_flight_owner = claim_token
|
|
else:
|
|
single_flight_identity = fingerprint or str(
|
|
routing.get("correlationKey") or ""
|
|
)
|
|
|
|
lock_decision = await acquire_agent99_sre_single_flight(
|
|
single_flight_identity,
|
|
single_flight_owner,
|
|
fail_closed=durable_route,
|
|
)
|
|
single_flight_acquired = bool(lock_decision.get("acquired") is True)
|
|
if not single_flight_acquired:
|
|
logger.info(
|
|
"agent99_sre_dispatch_single_flight_suppressed",
|
|
alert_id=alert_id,
|
|
alertname=alertname,
|
|
fingerprint=single_flight_identity,
|
|
group_key=routing.get("groupKey"),
|
|
reason=lock_decision.get("reason"),
|
|
)
|
|
correlated_receipt = None
|
|
if dispatch_identity is not None:
|
|
correlated_receipt = await ledger.defer_claim_retryable(
|
|
identity=dispatch_identity,
|
|
claim_token=claim_token,
|
|
reason=str(
|
|
lock_decision.get("reason") or "single_flight_duplicate"
|
|
),
|
|
retry_after_seconds=AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS,
|
|
)
|
|
return {
|
|
"status": "retryable" if dispatch_identity is not None else "suppressed",
|
|
"reason": str(lock_decision.get("reason") or "single_flight_duplicate"),
|
|
"fingerprint": single_flight_identity,
|
|
"groupKey": routing.get("groupKey"),
|
|
"dispatchPerformed": False,
|
|
"correlatedReceipt": correlated_receipt,
|
|
"identity": (
|
|
dispatch_identity.public_dict()
|
|
if dispatch_identity is not None
|
|
else None
|
|
),
|
|
"runtimeClosureVerified": False,
|
|
}
|
|
if dispatch_identity is not None:
|
|
delivery_fence = await ledger.mark_delivery_attempt_started(
|
|
identity=dispatch_identity,
|
|
claim_token=claim_token,
|
|
)
|
|
if delivery_fence.get("receipt_persisted") is not True:
|
|
await release_agent99_sre_single_flight(
|
|
single_flight_identity,
|
|
single_flight_owner,
|
|
)
|
|
single_flight_acquired = False
|
|
return {
|
|
"status": "failed",
|
|
"reason": "delivery_attempt_fence_persistence_failed",
|
|
"dispatchPerformed": False,
|
|
"correlatedReceipt": delivery_fence,
|
|
"identity": dispatch_identity.public_dict(),
|
|
"runtimeClosureVerified": False,
|
|
}
|
|
delivery_attempt_started = True
|
|
dispatch_receipt = await asyncio.to_thread(
|
|
dispatch_agent99_sre_alert_with_receipt, payload
|
|
)
|
|
correlated_receipt = None
|
|
if dispatch_identity is not None:
|
|
correlated_receipt = await ledger.complete(
|
|
identity=dispatch_identity,
|
|
claim_token=claim_token,
|
|
dispatch_receipt=dispatch_receipt,
|
|
controlled_apply_authorized=mutating,
|
|
)
|
|
accepted = bool(dispatch_receipt.get("accepted") is True)
|
|
inbox_triggered = bool(dispatch_receipt.get("inbox_triggered") is True)
|
|
dispatch_promoted = bool(accepted and inbox_triggered)
|
|
delivery_certainty = str(
|
|
dispatch_receipt.get("delivery_certainty") or "unknown"
|
|
)
|
|
if not accepted and delivery_certainty == "not_delivered":
|
|
await release_agent99_sre_single_flight(
|
|
single_flight_identity,
|
|
single_flight_owner,
|
|
)
|
|
single_flight_acquired = False
|
|
return {
|
|
"status": "failed",
|
|
"reason": dispatch_receipt.get("status") or "dispatch_not_accepted",
|
|
"dispatchReceipt": dispatch_receipt,
|
|
"correlatedReceipt": correlated_receipt,
|
|
"fingerprint": single_flight_identity,
|
|
"groupKey": routing.get("groupKey"),
|
|
"dispatchPerformed": True,
|
|
"identity": (
|
|
dispatch_identity.public_dict()
|
|
if dispatch_identity is not None
|
|
else None
|
|
),
|
|
"runtimeClosureVerified": False,
|
|
}
|
|
if (
|
|
dispatch_identity is not None
|
|
and not isinstance(correlated_receipt, dict)
|
|
) or (
|
|
isinstance(correlated_receipt, dict)
|
|
and correlated_receipt.get("receipt_persisted") is not True
|
|
):
|
|
return {
|
|
"status": "failed",
|
|
"reason": "dispatch_receipt_persistence_failed",
|
|
"dispatchReceipt": dispatch_receipt,
|
|
"correlatedReceipt": correlated_receipt,
|
|
"fingerprint": single_flight_identity,
|
|
"groupKey": routing.get("groupKey"),
|
|
"dispatchPerformed": True,
|
|
"identity": dispatch_identity.public_dict(),
|
|
"runtimeClosureVerified": False,
|
|
}
|
|
if not dispatch_promoted:
|
|
return {
|
|
"status": "delivery_pending",
|
|
"reason": "accepted_without_inbox_triggered",
|
|
"dispatch": dispatch_receipt.get("transport"),
|
|
"dispatchReceipt": dispatch_receipt,
|
|
"correlatedReceipt": correlated_receipt,
|
|
"fingerprint": single_flight_identity,
|
|
"groupKey": routing.get("groupKey"),
|
|
"dispatchPerformed": True,
|
|
"identity": (
|
|
dispatch_identity.public_dict()
|
|
if dispatch_identity is not None
|
|
else None
|
|
),
|
|
"runtimeClosureVerified": False,
|
|
}
|
|
return {
|
|
"status": "dispatched",
|
|
"dispatch": dispatch_receipt.get("transport"),
|
|
"dispatchReceipt": dispatch_receipt,
|
|
"correlatedReceipt": correlated_receipt,
|
|
"fingerprint": single_flight_identity,
|
|
"groupKey": routing.get("groupKey"),
|
|
"kind": payload.get("kind"),
|
|
"suggestedMode": payload.get("suggestedMode"),
|
|
"dispatchPerformed": True,
|
|
"identity": (
|
|
dispatch_identity.public_dict()
|
|
if dispatch_identity is not None
|
|
else None
|
|
),
|
|
"runtimeClosureVerified": False,
|
|
}
|
|
except Exception as exc:
|
|
if single_flight_acquired and not delivery_attempt_started:
|
|
await release_agent99_sre_single_flight(
|
|
single_flight_identity,
|
|
single_flight_owner,
|
|
)
|
|
correlated_receipt = None
|
|
if dispatch_identity is not None:
|
|
correlated_receipt = await ledger.complete(
|
|
identity=dispatch_identity,
|
|
claim_token=claim_token,
|
|
dispatch_receipt={
|
|
"schema_version": "agent99_sre_dispatch_receipt_v1",
|
|
"status": "transport_exception",
|
|
"transport": "unknown",
|
|
"alert_id": str(dispatch_identity.run_id),
|
|
"kind": "host_recovery",
|
|
"accepted": False,
|
|
"inbox_triggered": False,
|
|
"delivery_certainty": "unknown",
|
|
"stores_raw_response": False,
|
|
},
|
|
controlled_apply_authorized=mutating,
|
|
)
|
|
logger.warning(
|
|
"agent99_sre_bridge_failed_fail_closed",
|
|
alert_id=alert_id,
|
|
alertname=alertname,
|
|
error=str(exc),
|
|
)
|
|
return {
|
|
"status": "failed",
|
|
"reason": type(exc).__name__,
|
|
"dispatchPerformed": False,
|
|
"correlatedReceipt": correlated_receipt,
|
|
"identity": (
|
|
dispatch_identity.public_dict()
|
|
if dispatch_identity is not None
|
|
else None
|
|
),
|
|
"runtimeClosureVerified": False,
|
|
}
|