676 lines
25 KiB
Python
676 lines
25 KiB
Python
"""Authenticated reduced-envelope ingress for Alertmanager delivery failures.
|
||
|
||
The primary Alertmanager webhook cannot be the only producer of its own repair
|
||
candidate: when that delivery path is broken, calling it again is a catch-22.
|
||
Agent99 therefore polls the exact host-110 Alertmanager API without credentials,
|
||
reduces only the exact firing alert, then sends this authenticated no-secret
|
||
envelope to AWOOOI. This service binds one deterministic approval/Incident/run
|
||
identity and queues the exact host-110 Alertmanager Ansible candidate. Agent99
|
||
is transport and coordination only; it never executes the Linux repair.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import html
|
||
import json
|
||
from collections.abc import Awaitable, Callable, Mapping
|
||
from datetime import UTC, datetime
|
||
from typing import Any, Literal
|
||
from uuid import NAMESPACE_URL, UUID, uuid5
|
||
|
||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||
|
||
from src.core.logging import get_logger
|
||
from src.models.approval import (
|
||
ApprovalRequestCreate,
|
||
BlastRadius,
|
||
DataImpact,
|
||
DryRunCheck,
|
||
RiskLevel,
|
||
)
|
||
from src.services.approval_db import get_approval_service
|
||
from src.services.channel_hub import record_alertmanager_event
|
||
from src.services.incident_service import (
|
||
create_incident_for_approval,
|
||
get_incident_service,
|
||
)
|
||
|
||
logger = get_logger("awoooi.alert_chain_emergency_ingress")
|
||
|
||
ALERT_CHAIN_EMERGENCY_SCHEMA_VERSION = (
|
||
"agent99_alertmanager_emergency_relay_v1"
|
||
)
|
||
ALERT_CHAIN_EMERGENCY_OWNER_SCHEMA_VERSION = (
|
||
"alert_chain_emergency_ingress_owner_v1"
|
||
)
|
||
ALERT_CHAIN_ALERTNAME = "AlertChainBroken_Alertmanager"
|
||
ALERT_CHAIN_CATALOG_ID = "ansible:110-alertmanager-delivery-recovery"
|
||
ALERT_CHAIN_CANONICAL_ASSET_ID = "container:host_110:alertmanager"
|
||
ALERT_CHAIN_EXECUTOR = "host_ansible_executor"
|
||
ALERT_CHAIN_AGENT99_ROLE = "relay_coordination_only"
|
||
ALERT_CHAIN_APPROVAL_FINGERPRINT_DOMAIN = (
|
||
"awoooi.alert_chain_emergency.approval.v1"
|
||
)
|
||
|
||
|
||
class AlertChainEmergencyRelayRequest(BaseModel):
|
||
"""Bounded relay envelope; arbitrary labels, annotations and logs are forbidden."""
|
||
|
||
model_config = ConfigDict(extra="forbid")
|
||
|
||
schema_version: Literal["agent99_alertmanager_emergency_relay_v1"]
|
||
relay_receipt_id: str = Field(
|
||
pattern=r"^agent99-alertchain-[0-9a-f]{64}$",
|
||
max_length=83,
|
||
)
|
||
source_host: Literal["192.168.0.110"]
|
||
source_url: Literal["http://192.168.0.110:9093/api/v2/alerts"]
|
||
source_fingerprint: str = Field(
|
||
pattern=r"^[0-9a-f]{8,64}$",
|
||
max_length=64,
|
||
)
|
||
source_started_at: datetime
|
||
alertname: Literal["AlertChainBroken_Alertmanager"]
|
||
status: Literal["firing"]
|
||
severity: Literal["critical"]
|
||
target_resource: Literal["alertmanager"]
|
||
namespace: Literal["monitoring"]
|
||
agent99_role: Literal["relay_coordination_only"]
|
||
executor: Literal["host_ansible_executor"]
|
||
catalog_id: Literal["ansible:110-alertmanager-delivery-recovery"]
|
||
runtime_write_performed: Literal[False]
|
||
raw_payload_stored: Literal[False]
|
||
|
||
@field_validator("source_started_at")
|
||
@classmethod
|
||
def require_timezone(cls, value: datetime) -> datetime:
|
||
if value.tzinfo is None:
|
||
raise ValueError("source_started_at_timezone_required")
|
||
return value.astimezone(UTC)
|
||
|
||
|
||
def _canonical_json(value: Any) -> str:
|
||
return json.dumps(
|
||
value,
|
||
ensure_ascii=False,
|
||
sort_keys=True,
|
||
separators=(",", ":"),
|
||
)
|
||
|
||
|
||
def build_alert_chain_emergency_owner(
|
||
payload: AlertChainEmergencyRelayRequest,
|
||
*,
|
||
project_id: str = "awoooi",
|
||
) -> dict[str, Any]:
|
||
"""Build deterministic IDs for one exact Alertmanager firing episode."""
|
||
|
||
project = str(project_id or "awoooi").strip() or "awoooi"
|
||
occurrence = {
|
||
"source_host": payload.source_host,
|
||
"source_url": payload.source_url,
|
||
"source_fingerprint": payload.source_fingerprint,
|
||
"source_started_at": payload.source_started_at.isoformat(),
|
||
"relay_receipt_id": payload.relay_receipt_id,
|
||
}
|
||
owner_identity = {
|
||
"schema_version": ALERT_CHAIN_EMERGENCY_OWNER_SCHEMA_VERSION,
|
||
"project_id": project,
|
||
"alertname": ALERT_CHAIN_ALERTNAME,
|
||
"canonical_asset_id": ALERT_CHAIN_CANONICAL_ASSET_ID,
|
||
"source_occurrence": occurrence,
|
||
}
|
||
owner_key = _canonical_json(owner_identity)
|
||
digest = hashlib.sha256(owner_key.encode("utf-8")).hexdigest()
|
||
run_id = str(uuid5(NAMESPACE_URL, f"{owner_key}:run"))
|
||
return {
|
||
**owner_identity,
|
||
"owner_identity": owner_identity,
|
||
"owner_key_sha256": digest,
|
||
"approval_id": str(uuid5(NAMESPACE_URL, f"{owner_key}:approval")),
|
||
"incident_id": f"INC-ACR-{digest[:16].upper()}",
|
||
"run_id": run_id,
|
||
"trace_id": run_id,
|
||
"work_item_id": f"alertchain:{project}:{digest[:32]}",
|
||
"source_receipt_ref": payload.relay_receipt_id,
|
||
"source_fingerprint": payload.source_fingerprint,
|
||
"source_started_at": payload.source_started_at.isoformat(),
|
||
"agent99_role": ALERT_CHAIN_AGENT99_ROLE,
|
||
"executor": ALERT_CHAIN_EXECUTOR,
|
||
"catalog_id": ALERT_CHAIN_CATALOG_ID,
|
||
"canonical_asset_id": ALERT_CHAIN_CANONICAL_ASSET_ID,
|
||
"runtime_write_performed": False,
|
||
"runtime_closure_verified": False,
|
||
}
|
||
|
||
|
||
def _approval_storage_fingerprint(owner: dict[str, Any]) -> str:
|
||
payload = (
|
||
f"{ALERT_CHAIN_APPROVAL_FINGERPRINT_DOMAIN}\x1f"
|
||
f"{_canonical_json(owner['owner_identity'])}"
|
||
)
|
||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def _approval_matches_owner(approval: Any, owner: dict[str, Any]) -> bool:
|
||
metadata = getattr(approval, "metadata", None)
|
||
return bool(
|
||
str(getattr(approval, "id", "") or "") == owner["approval_id"]
|
||
and isinstance(metadata, dict)
|
||
and metadata.get("schema_version")
|
||
== ALERT_CHAIN_EMERGENCY_OWNER_SCHEMA_VERSION
|
||
and metadata.get("owner_identity") == owner["owner_identity"]
|
||
and metadata.get("owner_key_sha256") == owner["owner_key_sha256"]
|
||
and metadata.get("run_id") == owner["run_id"]
|
||
and metadata.get("trace_id") == owner["trace_id"]
|
||
and metadata.get("work_item_id") == owner["work_item_id"]
|
||
and metadata.get("agent99_role") == ALERT_CHAIN_AGENT99_ROLE
|
||
and metadata.get("executor") == ALERT_CHAIN_EXECUTOR
|
||
and metadata.get("catalog_id") == ALERT_CHAIN_CATALOG_ID
|
||
and metadata.get("runtime_write_performed") is False
|
||
)
|
||
|
||
|
||
def _approval_request(owner: dict[str, Any]) -> ApprovalRequestCreate:
|
||
return ApprovalRequestCreate(
|
||
action=(
|
||
"ANSIBLE_CONTROLLED_CANDIDATE "
|
||
"ansible:110-alertmanager-delivery-recovery"
|
||
),
|
||
description=(
|
||
"Authenticated Agent99 poll relay created the exact "
|
||
"Alertmanager delivery recovery candidate. Agent99 remains relay-only; "
|
||
"the Linux executor is the bounded host Ansible broker."
|
||
),
|
||
risk_level=RiskLevel.MEDIUM,
|
||
blast_radius=BlastRadius(
|
||
affected_pods=0,
|
||
estimated_downtime="0-30s",
|
||
related_services=["alertmanager"],
|
||
data_impact=DataImpact.WRITE,
|
||
),
|
||
dry_run_checks=[
|
||
DryRunCheck(
|
||
name="exact-host read-only poll ingress",
|
||
passed=True,
|
||
message="unauthenticated first hop; authenticated reduced outbound",
|
||
),
|
||
DryRunCheck(
|
||
name="typed executor boundary",
|
||
passed=True,
|
||
message=(
|
||
"Agent99 relay-only; host_ansible_executor owns Linux apply"
|
||
),
|
||
),
|
||
DryRunCheck(
|
||
name="no-secret envelope",
|
||
passed=True,
|
||
message="raw labels, annotations, logs and credentials rejected",
|
||
),
|
||
],
|
||
requested_by="Agent99 Alertmanager emergency relay",
|
||
incident_id=owner["incident_id"],
|
||
metadata={
|
||
**owner,
|
||
"preallocated_approval_id": owner["approval_id"],
|
||
"owner_policy": "global_product_governance_v2",
|
||
"candidate_only": True,
|
||
"controlled_apply_allowed": True,
|
||
},
|
||
)
|
||
|
||
|
||
def _incident_payload_for_candidate(
|
||
incident: Any,
|
||
*,
|
||
owner: dict[str, Any],
|
||
) -> dict[str, Any]:
|
||
if hasattr(incident, "model_dump"):
|
||
payload = incident.model_dump(mode="json")
|
||
elif isinstance(incident, dict):
|
||
payload = dict(incident)
|
||
else:
|
||
payload = {}
|
||
payload.update(
|
||
{
|
||
"incident_id": owner["incident_id"],
|
||
"project_id": owner["project_id"],
|
||
"alertname": ALERT_CHAIN_ALERTNAME,
|
||
"alert_category": "alert_chain_health",
|
||
"notification_type": "TYPE-3",
|
||
"trace_id": owner["trace_id"],
|
||
"run_id": owner["run_id"],
|
||
"work_item_id": owner["work_item_id"],
|
||
"source_receipt_ref": owner["source_receipt_ref"],
|
||
"asset_scope_aliases": [
|
||
ALERT_CHAIN_CANONICAL_ASSET_ID,
|
||
"host_110",
|
||
"alertmanager",
|
||
],
|
||
}
|
||
)
|
||
return payload
|
||
|
||
|
||
def _alert_chain_candidate_queued_message(
|
||
*,
|
||
owner: Mapping[str, Any],
|
||
) -> str:
|
||
"""Render a compact, truthful queue receipt for the SRE war room."""
|
||
|
||
incident_id = html.escape(str(owner["incident_id"]))
|
||
run_id = html.escape(str(owner["run_id"]))
|
||
return (
|
||
"🚨 <b>[ALERT-CHAIN | CRITICAL]</b>\n"
|
||
"<code>ai_automation_alert_card_v1</code>\n\n"
|
||
"<b>Alertmanager 告警鏈路中斷</b>\n"
|
||
f"├ 資產：<code>{ALERT_CHAIN_CANONICAL_ASSET_ID}</code>\n"
|
||
"├ 現況：修復候選已持久化並排入受控佇列\n"
|
||
f"├ Incident：<code>{incident_id}</code>\n"
|
||
f"└ Run：<code>{run_id}</code>\n\n"
|
||
"<b>AI Agent 動作</b>\n"
|
||
"├ Agent99：已完成只讀旁路探測與驗證轉送\n"
|
||
"├ Policy：已用 deterministic typed-domain policy 選定執行器\n"
|
||
"├ Host Ansible：已排入 check → bounded apply → verify\n"
|
||
"└ LLM：未介入此緊急路由（不偽造 Ollama/Claude/Gemini 執行）\n\n"
|
||
"⏭ <b>自動後續</b>：獨立驗證 → Incident closure "
|
||
"→ KM/RAG/MCP/PlayBook\n"
|
||
"<code>runtime_write_gate=controlled</code>"
|
||
)
|
||
|
||
|
||
async def _send_alert_chain_candidate_queued_telegram_receipt(
|
||
*,
|
||
owner: Mapping[str, Any],
|
||
project_id: str,
|
||
) -> dict[str, Any]:
|
||
"""Send the queue receipt and reduce provider evidence to a safe receipt."""
|
||
|
||
from src.services.awooop_deeplinks import incident_truth_chain_reply_markup
|
||
from src.services.telegram_gateway import (
|
||
_telegram_send_delivery_succeeded,
|
||
get_telegram_gateway,
|
||
)
|
||
|
||
delivery = await get_telegram_gateway().send_alert_notification(
|
||
_alert_chain_candidate_queued_message(owner=owner),
|
||
product_id=project_id,
|
||
signal_family="incident_lifecycle",
|
||
severity="P1",
|
||
reply_markup=incident_truth_chain_reply_markup(
|
||
str(owner["incident_id"]),
|
||
project_id=project_id,
|
||
),
|
||
)
|
||
provider_result = (
|
||
delivery.get("result")
|
||
if isinstance(delivery, Mapping)
|
||
and isinstance(delivery.get("result"), Mapping)
|
||
else {}
|
||
)
|
||
route_receipt = (
|
||
delivery.get("_awoooi_canonical_route_receipt")
|
||
if isinstance(delivery, Mapping)
|
||
and isinstance(
|
||
delivery.get("_awoooi_canonical_route_receipt"), Mapping
|
||
)
|
||
else {}
|
||
)
|
||
return {
|
||
"schema_version": "alert_chain_telegram_queue_receipt_v1",
|
||
"attempted": True,
|
||
"durable_acknowledged": _telegram_send_delivery_succeeded(delivery),
|
||
"delivery_status": str(
|
||
delivery.get("_awooop_delivery_status") or "unknown"
|
||
)
|
||
if isinstance(delivery, Mapping)
|
||
else "invalid_response",
|
||
"provider_message_id": str(provider_result.get("message_id") or ""),
|
||
"destination_alias": str(route_receipt.get("destination_alias") or ""),
|
||
"classification": "alert_chain_health",
|
||
"agent99_activity": "read_only_relay_completed",
|
||
"executor_activity": "host_ansible_candidate_queued",
|
||
"model_activity": "not_invoked_deterministic_policy",
|
||
}
|
||
|
||
|
||
async def process_alert_chain_emergency_relay(
|
||
payload: AlertChainEmergencyRelayRequest,
|
||
*,
|
||
project_id: str = "awoooi",
|
||
approval_service: Any | None = None,
|
||
incident_service: Any | None = None,
|
||
incident_creator: Callable[..., Awaitable[str]] = (
|
||
create_incident_for_approval
|
||
),
|
||
candidate_enqueuer: Callable[..., Awaitable[dict[str, Any]]] | None = None,
|
||
event_recorder: Callable[..., Awaitable[Any]] = record_alertmanager_event,
|
||
telegram_notifier: Callable[..., Awaitable[dict[str, Any]]] | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Persist the deterministic owner and queue the exact Ansible candidate."""
|
||
|
||
if candidate_enqueuer is None:
|
||
from src.jobs.awooop_ansible_candidate_backfill_job import (
|
||
enqueue_ai_decision_ansible_candidate,
|
||
)
|
||
|
||
candidate_enqueuer = enqueue_ai_decision_ansible_candidate
|
||
if telegram_notifier is None:
|
||
telegram_notifier = _send_alert_chain_candidate_queued_telegram_receipt
|
||
|
||
owner = build_alert_chain_emergency_owner(payload, project_id=project_id)
|
||
approvals = approval_service or get_approval_service()
|
||
incidents = incident_service or get_incident_service()
|
||
approval_uuid = UUID(owner["approval_id"])
|
||
|
||
approval = await approvals.get_approval(approval_uuid)
|
||
created = False
|
||
if approval is not None and not _approval_matches_owner(approval, owner):
|
||
return {
|
||
"schema_version": "alert_chain_emergency_ingress_result_v1",
|
||
"status": "durable_owner_identity_mismatch_no_candidate",
|
||
"candidate_persisted": False,
|
||
"candidate_queued": False,
|
||
"runtime_write_performed": False,
|
||
"runtime_closure_verified": False,
|
||
}
|
||
if approval is None:
|
||
try:
|
||
approval = await approvals.create_approval_with_fingerprint(
|
||
request=_approval_request(owner),
|
||
fingerprint=_approval_storage_fingerprint(owner),
|
||
)
|
||
created = True
|
||
except Exception:
|
||
approval = await approvals.get_approval(approval_uuid)
|
||
if approval is None:
|
||
raise
|
||
else:
|
||
incremented = await approvals.increment_hit_count(approval_uuid)
|
||
if incremented is not None:
|
||
approval = incremented
|
||
|
||
if not _approval_matches_owner(approval, owner):
|
||
return {
|
||
"schema_version": "alert_chain_emergency_ingress_result_v1",
|
||
"status": "durable_owner_readback_mismatch_no_candidate",
|
||
"candidate_persisted": False,
|
||
"candidate_queued": False,
|
||
"runtime_write_performed": False,
|
||
"runtime_closure_verified": False,
|
||
}
|
||
|
||
canonical_incident = await incidents.get_for_readback(
|
||
owner["incident_id"],
|
||
project_id=project_id,
|
||
)
|
||
if canonical_incident is None:
|
||
await incident_creator(
|
||
approval_id=owner["approval_id"],
|
||
risk_level="medium",
|
||
target_resource="alertmanager",
|
||
namespace="monitoring",
|
||
alert_type="custom",
|
||
message=(
|
||
"Alertmanager main webhook delivery failure received by the "
|
||
"authenticated Agent99 secondary machine ingress"
|
||
),
|
||
source="alertmanager",
|
||
alertname=ALERT_CHAIN_ALERTNAME,
|
||
alert_labels={
|
||
"alertname": ALERT_CHAIN_ALERTNAME,
|
||
"severity": "critical",
|
||
"component": "alertmanager",
|
||
"host": "110",
|
||
"alert_category": "alert_chain_health",
|
||
"fingerprint": payload.source_fingerprint,
|
||
"relay_receipt_id": payload.relay_receipt_id,
|
||
"canonical_asset_id": ALERT_CHAIN_CANONICAL_ASSET_ID,
|
||
},
|
||
notification_type="TYPE-3",
|
||
alert_category="alert_chain_health",
|
||
canonical_incident_id=owner["incident_id"],
|
||
)
|
||
await approvals.update_incident_id(
|
||
approval_uuid,
|
||
owner["incident_id"],
|
||
)
|
||
canonical_incident = await incidents.get_for_readback(
|
||
owner["incident_id"],
|
||
project_id=project_id,
|
||
)
|
||
|
||
durable_incident = bool(
|
||
canonical_incident is not None
|
||
and getattr(canonical_incident, "persisted_to_pg", False) is True
|
||
)
|
||
bound_approval = await approvals.get_approval(approval_uuid)
|
||
durable_binding = bool(
|
||
bound_approval is not None
|
||
and _approval_matches_owner(bound_approval, owner)
|
||
and str(getattr(bound_approval, "incident_id", "") or "")
|
||
== owner["incident_id"]
|
||
)
|
||
if not durable_incident or not durable_binding:
|
||
return {
|
||
"schema_version": "alert_chain_emergency_ingress_result_v1",
|
||
"status": "durable_incident_binding_missing_no_candidate",
|
||
"candidate_persisted": False,
|
||
"candidate_queued": False,
|
||
"identity": {
|
||
key: owner[key]
|
||
for key in ("run_id", "trace_id", "work_item_id")
|
||
},
|
||
"runtime_write_performed": False,
|
||
"runtime_closure_verified": False,
|
||
}
|
||
|
||
candidate_incident = _incident_payload_for_candidate(
|
||
canonical_incident,
|
||
owner=owner,
|
||
)
|
||
enqueue_receipt = await candidate_enqueuer(
|
||
incident=candidate_incident,
|
||
proposal_data={
|
||
"source": "alert_webhook_controlled_router",
|
||
"risk_level": "medium",
|
||
"action": "enqueue_allowlisted_ansible_check_mode",
|
||
"alert_type": "custom",
|
||
"alertname": ALERT_CHAIN_ALERTNAME,
|
||
"namespace": "monitoring",
|
||
"approval_id": owner["approval_id"],
|
||
"execution_priority": 20,
|
||
"source_fingerprint": payload.source_fingerprint,
|
||
"source_occurrence_id": payload.relay_receipt_id,
|
||
"source_received_at": payload.source_started_at.isoformat(),
|
||
"source_recurrence_verified": True,
|
||
"source_recurrence_source": "agent99_secondary_machine_ingress",
|
||
"source_recurrence_anchor": payload.relay_receipt_id,
|
||
"source_recurrence_durable_readback_verified": True,
|
||
"source_canonical_asset_id": ALERT_CHAIN_CANONICAL_ASSET_ID,
|
||
},
|
||
project_id=project_id,
|
||
automation_run_id=owner["run_id"],
|
||
)
|
||
queued = bool(
|
||
enqueue_receipt.get("queued") is True
|
||
and str(enqueue_receipt.get("automation_run_id") or "")
|
||
== owner["run_id"]
|
||
and str(enqueue_receipt.get("trace_id") or "")
|
||
== owner["trace_id"]
|
||
and str(enqueue_receipt.get("work_item_id") or "")
|
||
== owner["work_item_id"]
|
||
)
|
||
|
||
try:
|
||
await event_recorder(
|
||
project_id=project_id,
|
||
alert_id=payload.relay_receipt_id,
|
||
alertname=ALERT_CHAIN_ALERTNAME,
|
||
severity="critical",
|
||
namespace="monitoring",
|
||
target_resource="alertmanager",
|
||
fingerprint=payload.source_fingerprint,
|
||
stage=(
|
||
"secondary_ingress_ansible_candidate_queued"
|
||
if queued
|
||
else "secondary_ingress_candidate_queue_blocked"
|
||
),
|
||
notification_type="TYPE-3",
|
||
alert_category="alert_chain_health",
|
||
incident_id=owner["incident_id"],
|
||
approval_id=owner["approval_id"],
|
||
repeat_count=int(getattr(bound_approval, "hit_count", 1) or 1),
|
||
labels={
|
||
"alertname": ALERT_CHAIN_ALERTNAME,
|
||
"component": "alertmanager",
|
||
"canonical_asset_id": ALERT_CHAIN_CANONICAL_ASSET_ID,
|
||
"fingerprint": payload.source_fingerprint,
|
||
},
|
||
annotations={
|
||
"relay_role": ALERT_CHAIN_AGENT99_ROLE,
|
||
"executor": ALERT_CHAIN_EXECUTOR,
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"alert_chain_secondary_ingress_event_record_failed",
|
||
error_type=type(exc).__name__,
|
||
incident_id=owner["incident_id"],
|
||
)
|
||
|
||
telegram_receipt: dict[str, Any] = {
|
||
"schema_version": "alert_chain_telegram_queue_receipt_v1",
|
||
"attempted": False,
|
||
"durable_acknowledged": False,
|
||
"delivery_status": "candidate_not_queued",
|
||
"provider_message_id": "",
|
||
"destination_alias": "",
|
||
"classification": "alert_chain_health",
|
||
"agent99_activity": "read_only_relay_completed",
|
||
"executor_activity": (
|
||
"host_ansible_candidate_queued"
|
||
if queued
|
||
else "host_ansible_candidate_not_queued"
|
||
),
|
||
"model_activity": "not_invoked_deterministic_policy",
|
||
}
|
||
if queued:
|
||
try:
|
||
notification_result = await telegram_notifier(
|
||
owner=owner,
|
||
project_id=project_id,
|
||
)
|
||
if isinstance(notification_result, Mapping):
|
||
delivery_fields = (
|
||
"attempted",
|
||
"durable_acknowledged",
|
||
"delivery_status",
|
||
"provider_message_id",
|
||
"destination_alias",
|
||
)
|
||
telegram_receipt.update(
|
||
{
|
||
key: notification_result.get(key)
|
||
for key in delivery_fields
|
||
if key in notification_result
|
||
}
|
||
)
|
||
except Exception as exc:
|
||
telegram_receipt.update(
|
||
{
|
||
"attempted": True,
|
||
"durable_acknowledged": False,
|
||
"delivery_status": f"failed:{type(exc).__name__}",
|
||
}
|
||
)
|
||
logger.warning(
|
||
"alert_chain_candidate_queued_telegram_receipt_failed",
|
||
error_type=type(exc).__name__,
|
||
incident_id=owner["incident_id"],
|
||
)
|
||
|
||
telegram_acknowledged = bool(
|
||
queued
|
||
and telegram_receipt.get("attempted") is True
|
||
and telegram_receipt.get("durable_acknowledged") is True
|
||
)
|
||
queued_blockers = [
|
||
"independent_post_verifier_pending",
|
||
"incident_closure_and_learning_writeback_pending",
|
||
]
|
||
if queued and not telegram_acknowledged:
|
||
queued_blockers.insert(0, "telegram_durable_queue_receipt_missing")
|
||
|
||
return {
|
||
"schema_version": "alert_chain_emergency_ingress_result_v1",
|
||
"status": (
|
||
(
|
||
"ansible_candidate_queued_verifier_pending"
|
||
if telegram_acknowledged
|
||
else "ansible_candidate_queued_telegram_receipt_pending"
|
||
)
|
||
if queued
|
||
else str(
|
||
enqueue_receipt.get("status")
|
||
or "ansible_candidate_queue_not_acknowledged"
|
||
)
|
||
),
|
||
"candidate_owner_created": created,
|
||
"candidate_persisted": True,
|
||
"candidate_queued": queued,
|
||
"identity": {
|
||
key: owner[key]
|
||
for key in (
|
||
"approval_id",
|
||
"incident_id",
|
||
"run_id",
|
||
"trace_id",
|
||
"work_item_id",
|
||
"source_receipt_ref",
|
||
)
|
||
},
|
||
"agent99_role": ALERT_CHAIN_AGENT99_ROLE,
|
||
"executor": ALERT_CHAIN_EXECUTOR,
|
||
"catalog_id": ALERT_CHAIN_CATALOG_ID,
|
||
"canonical_asset_id": ALERT_CHAIN_CANONICAL_ASSET_ID,
|
||
"enqueue_receipt": {
|
||
"status": enqueue_receipt.get("status"),
|
||
"automation_run_id": enqueue_receipt.get("automation_run_id"),
|
||
"queued": enqueue_receipt.get("queued") is True,
|
||
"side_effect_performed": bool(
|
||
enqueue_receipt.get("side_effect_performed")
|
||
),
|
||
"active_blockers": [
|
||
str(value)[:160]
|
||
for value in enqueue_receipt.get("active_blockers") or []
|
||
][:8],
|
||
},
|
||
"telegram_receipt": telegram_receipt,
|
||
"alert_to_telegram_receipt_verified": telegram_acknowledged,
|
||
"runtime_write_performed": False,
|
||
"runtime_closure_verified": False,
|
||
"active_blockers": (
|
||
queued_blockers
|
||
if queued
|
||
else [
|
||
str(value)[:160]
|
||
for value in enqueue_receipt.get("active_blockers") or [
|
||
"ansible_candidate_queue_not_acknowledged"
|
||
]
|
||
][:8]
|
||
),
|
||
"safe_next_action": (
|
||
(
|
||
"ansible_worker_check_apply_verify_writeback_same_run"
|
||
if telegram_acknowledged
|
||
else (
|
||
"ansible_worker_continue_same_run_and_retry_telegram_receipt"
|
||
)
|
||
)
|
||
if queued
|
||
else "retry_same_relay_receipt_after_candidate_queue_blocker"
|
||
),
|
||
}
|