"""Signal-worker observation bridge for AwoooP truth-chain. The Redis signal worker predates AwoooP channel events. Incidents created from that path used to stop at "source persisted" plus a seed timeline event, which made operator history look like the alert was received but never observed. This service records the non-Telegram path as first-class AwoooP evidence without claiming that any auto-repair was executed. """ from __future__ import annotations import asyncio import json import time from typing import TYPE_CHECKING, Any import structlog from sqlalchemy import text from src.db.base import get_db_context from src.services.channel_hub import ( _db_timestamp_now, build_external_alert_provider_event_id, build_external_alert_run_id, build_inbound_source_envelope, ensure_completed_shadow_run, mirror_inbound_event, record_outbound_message, ) from src.services.evidence_snapshot import EvidenceSnapshot if TYPE_CHECKING: from src.models.incident import Incident logger = structlog.get_logger(__name__) PROJECT_ID = "awoooi" SIGNAL_OBSERVATION_TIMEOUT_SEC = 12.0 def _to_text(value: Any, default: str = "") -> str: if value is None: return default if isinstance(value, bytes): return value.decode(errors="replace") text_value = str(value) return text_value if text_value else default def _as_dict(value: Any) -> dict[str, Any]: if isinstance(value, dict): return dict(value) if value in (None, ""): return {} if isinstance(value, bytes): value = value.decode(errors="replace") if isinstance(value, str): try: parsed = json.loads(value) except json.JSONDecodeError: return {} return dict(parsed) if isinstance(parsed, dict) else {} return {} def _first_signal(incident: "Incident") -> Any | None: signals = getattr(incident, "signals", []) or [] return signals[0] if signals else None def _severity_value(value: Any) -> str: return _to_text(getattr(value, "value", value), "warning") def _incident_id(incident: "Incident") -> str: return _to_text(getattr(incident, "incident_id", None), "unknown") def _alertname(incident: "Incident", signal_data: dict[str, Any]) -> str: signal = _first_signal(incident) labels = _as_dict(getattr(signal, "labels", None)) return ( _to_text(signal_data.get("alert_name")) or _to_text(getattr(signal, "alert_name", None)) or _to_text(labels.get("alertname")) or "unknown" ) def _annotations(incident: "Incident", signal_data: dict[str, Any]) -> dict[str, Any]: signal = _first_signal(incident) merged = _as_dict(signal_data.get("annotations")) merged.update(_as_dict(getattr(signal, "annotations", None))) return merged def _labels(incident: "Incident", signal_data: dict[str, Any]) -> dict[str, Any]: signal = _first_signal(incident) merged = _as_dict(signal_data.get("labels")) merged.update(_as_dict(getattr(signal, "labels", None))) affected_services = getattr(incident, "affected_services", []) or [] target = ( _to_text(signal_data.get("target")) or _to_text(merged.get("target")) or (_to_text(affected_services[0]) if affected_services else "") ) sensor_ip = _to_text(signal_data.get("sensor_ip")) sensor_host = _to_text(signal_data.get("sensor_host")) namespace = _to_text(signal_data.get("namespace"), "default") if target: merged.setdefault("target", target) if namespace: merged.setdefault("namespace", namespace) if sensor_ip: merged.setdefault("sensor_ip", sensor_ip) if sensor_host: merged.setdefault("sensor_host", sensor_host) if not merged.get("host") and (sensor_ip or target): merged["host"] = sensor_ip or target alertname = _alertname(incident, signal_data) if alertname: merged.setdefault("alertname", alertname) merged.setdefault("source", _to_text(signal_data.get("source"), "signal_worker")) return merged def enrich_incident_for_signal_observation( incident: "Incident", signal_data: dict[str, Any], ) -> dict[str, Any]: """Add signal-worker context back to the local incident used by PDI.""" labels = _labels(incident, signal_data) signal = _first_signal(incident) if signal is not None: try: signal.labels = labels except Exception: logger.debug( "signal_observation_label_enrich_skipped", incident_id=_incident_id(incident), ) return labels def _fingerprint(incident: "Incident", signal_data: dict[str, Any]) -> str: signal = _first_signal(incident) return ( _to_text(signal_data.get("fingerprint")) or _to_text(getattr(signal, "fingerprint", None)) or "no-fingerprint" ) def build_signal_worker_provider_event_id(message_id: Any, fingerprint: str) -> str: raw_event_id = f"{_to_text(message_id, 'unknown')}:{fingerprint[:16]}" return build_external_alert_provider_event_id( "signal-worker", raw_event_id, "received", ) def format_signal_worker_event_content( *, incident_id: str, alertname: str, severity: str, namespace: str, target: str, fingerprint: str, message_id: str, signal_source: str, ) -> str: """Render a redaction-safe internal event preview.""" return "\n".join( [ "Signal worker inbound received", f"Incident: {incident_id}", f"Alert: {alertname}", f"Severity: {severity}", f"Namespace: {namespace or 'default'}", f"Target: {target or '-'}", f"Fingerprint: {fingerprint}", f"Redis Message: {message_id}", f"Source: {signal_source or 'signal_worker'}", "Telegram: not sent (internal shadow mirror)", ] ) async def _record_signal_channel_history( *, incident: "Incident", signal_data: dict[str, Any], message_id: Any, labels: dict[str, Any], ) -> None: incident_id = _incident_id(incident) alertname = _alertname(incident, signal_data) severity = _severity_value( signal_data.get("severity") or getattr(incident, "severity", None) ) namespace = _to_text(labels.get("namespace"), "default") target = _to_text(labels.get("target")) or ",".join( _to_text(value) for value in (getattr(incident, "affected_services", []) or []) ) fingerprint = _fingerprint(incident, signal_data) signal_source = _to_text(signal_data.get("source"), "signal_worker") safe_message_id = _to_text(message_id, "unknown") provider_event_id = build_signal_worker_provider_event_id(safe_message_id, fingerprint) run_id = build_external_alert_run_id(PROJECT_ID, provider_event_id) annotations = _annotations(incident, signal_data) content = format_signal_worker_event_content( incident_id=incident_id, alertname=alertname, severity=severity, namespace=namespace, target=target, fingerprint=fingerprint, message_id=safe_message_id, signal_source=signal_source, ) envelope = build_inbound_source_envelope( provider="signal-worker", stage="received", provider_event_id=provider_event_id, raw_event_id=safe_message_id, raw_content=content, alertname=alertname, severity=severity, namespace=namespace, target_resource=target, fingerprint=fingerprint, incident_id=incident_id, labels=labels, annotations=annotations, extra={ "redis_stream": "awoooi:signals", "signal_source": signal_source, "sensor_host": labels.get("sensor_host"), "sensor_ip": labels.get("sensor_ip"), }, ) async with get_db_context(PROJECT_ID) as db: await ensure_completed_shadow_run( db, project_id=PROJECT_ID, run_id=run_id, agent_id="legacy-signal-worker", trigger_type="signal_worker_inbound", trigger_ref=provider_event_id, input_payload={ "incident_id": incident_id, "alertname": alertname, "severity": severity, "namespace": namespace, "target": target, "fingerprint": fingerprint, "message_id": safe_message_id, }, ) conversation_event_id = await mirror_inbound_event( db, project_id=PROJECT_ID, channel_type="internal", provider_event_id=provider_event_id, platform_subject_id="signal-worker", channel_user_id=signal_source, channel_chat_id=f"signal-worker:{namespace or 'default'}", content_type="text", raw_content=content, source_envelope=envelope, provider_ts=_db_timestamp_now(), run_id=run_id, ) exists = ( await db.execute( text( """ SELECT 1 FROM awooop_outbound_message WHERE project_id = :project_id AND triggered_by_state = 'signal_worker_observed' AND coalesce(source_envelope #> '{source_refs,incident_ids}', '[]'::jsonb) ? :incident_id LIMIT 1 """ ), {"project_id": PROJECT_ID, "incident_id": incident_id}, ) ).first() if exists: return outbound_envelope = { "schema_version": "signal_worker_observation_outbound_v1", "source_refs": { "incident_ids": [incident_id], "alert_ids": [provider_event_id, safe_message_id], "fingerprints": [fingerprint], }, "automation_stage": "observed_not_executed", "truth_note": "internal shadow message only; no auto-repair execution claimed", } await record_outbound_message( db, project_id=PROJECT_ID, run_id=run_id, conversation_event_id=conversation_event_id, channel_type="internal", channel_chat_id=f"signal-worker:{namespace or 'default'}", message_type="interim", content=( f"AwoooP signal-worker observation recorded\n" f"Incident: {incident_id}\n" f"Alert: {alertname}\n" "Stage: observed_not_executed\n" "Telegram: not sent" ), source_envelope=outbound_envelope, provider_message_id=provider_event_id[:64], send_status="shadow", triggered_by_state="signal_worker_observed", is_shadow=True, ) async def _evidence_counts(incident_id: str) -> tuple[int, int]: async with get_db_context(PROJECT_ID) as db: row = ( await db.execute( text( """ SELECT count(*)::int AS total, coalesce(sum(sensors_succeeded), 0)::int AS succeeded FROM incident_evidence WHERE incident_id = :incident_id """ ), {"incident_id": incident_id}, ) ).mappings().first() if not row: return 0, 0 return int(row["total"] or 0), int(row["succeeded"] or 0) async def _record_raw_signal_evidence_if_needed( *, incident: "Incident", signal_data: dict[str, Any], message_id: Any, labels: dict[str, Any], pdi_duration_ms: int | None, ) -> bool: incident_id = _incident_id(incident) try: _total, succeeded = await _evidence_counts(incident_id) except Exception as exc: logger.warning( "signal_observation_evidence_count_failed", incident_id=incident_id, error=str(exc), ) succeeded = 0 if succeeded > 0: return False alertname = _alertname(incident, signal_data) severity = _severity_value( signal_data.get("severity") or getattr(incident, "severity", None) ) annotations = _annotations(incident, signal_data) snapshot = EvidenceSnapshot(incident_id=incident_id) snapshot.alert_info = { "alert_name": alertname, "severity": severity, "affected_services": getattr(incident, "affected_services", []) or [], "labels": labels, "annotations": annotations, "source": _to_text(signal_data.get("source"), "signal_worker"), "incident_id": incident_id, "redis_message_id": _to_text(message_id, "unknown"), } snapshot.metrics_snapshot = { "source": "signal_worker_raw_event", "error_count": labels.get("error_count"), "sensor_host": labels.get("sensor_host"), "sensor_ip": labels.get("sensor_ip"), "target": labels.get("target"), } snapshot.historical_context = ( "Raw signal-worker observation captured for truth-chain continuity. " "This is evidence only; no auto-repair execution was performed." ) snapshot.mcp_health = {"signal_worker_raw_event": True} snapshot.sensors_attempted = 1 snapshot.sensors_succeeded = 1 snapshot.collection_duration_ms = pdi_duration_ms or 0 snapshot.evidence_summary = snapshot.build_summary() await snapshot.save() return True async def _add_observation_timeline( *, incident: "Incident", signal_data: dict[str, Any], labels: dict[str, Any], raw_evidence_created: bool, ) -> None: try: from src.services.approval_db import get_timeline_service await get_timeline_service().add_event( event_type="agent", status="success", title=f"Signal worker observation recorded: {_alertname(incident, signal_data)}", description=json.dumps( { "stage": "observed_not_executed", "raw_evidence_created": raw_evidence_created, "target": labels.get("target"), "sensor_host": labels.get("sensor_host"), "sensor_ip": labels.get("sensor_ip"), "telegram": "not_sent_internal_shadow", }, ensure_ascii=False, ), actor="signal_worker", actor_role="signal_worker", risk_level=_severity_value(getattr(incident, "severity", None)), incident_id=_incident_id(incident), ) except Exception as exc: logger.warning( "signal_observation_timeline_failed", incident_id=_incident_id(incident), error=str(exc), ) async def _run_pre_decision_investigation(incident: "Incident") -> int | None: started = time.monotonic() try: await _ensure_signal_observation_mcp_runtime() from src.services.pre_decision_investigator import get_pre_decision_investigator await asyncio.wait_for( get_pre_decision_investigator().investigate(incident), timeout=SIGNAL_OBSERVATION_TIMEOUT_SEC, ) return int((time.monotonic() - started) * 1000) except asyncio.TimeoutError: logger.warning( "signal_observation_pdi_timeout", incident_id=_incident_id(incident), timeout_sec=SIGNAL_OBSERVATION_TIMEOUT_SEC, ) return int((time.monotonic() - started) * 1000) except Exception as exc: logger.warning( "signal_observation_pdi_failed", incident_id=_incident_id(incident), error=str(exc), ) return int((time.monotonic() - started) * 1000) async def _ensure_signal_observation_mcp_runtime() -> None: """Make one-off signal observation runs use the same MCP runtime as API startup.""" try: from src.plugins.mcp.providers import register_all_providers from src.plugins.mcp.registry import get_provider_registry from src.services.mcp_tool_registry import get_mcp_tool_registry, init_mcp_tool_registry provider_registry = get_provider_registry() if len(provider_registry) == 0: register_all_providers() logger.info("signal_observation_mcp_providers_registered") tool_registry = get_mcp_tool_registry() if tool_registry.tool_count == 0: await init_mcp_tool_registry() logger.info( "signal_observation_mcp_tool_registry_initialized", providers=tool_registry.provider_count, tools=tool_registry.tool_count, ) except Exception as exc: logger.warning( "signal_observation_mcp_runtime_init_failed", error=str(exc), ) async def record_signal_worker_observation( incident: "Incident", signal_data: dict[str, Any], message_id: Any, ) -> dict[str, Any]: """Record signal-worker incidents as observable AwoooP events. Returns a small status payload for logs/tests. All called operations are best-effort from the worker perspective; callers may log failures but should not convert them into signal-processing failures. """ labels = enrich_incident_for_signal_observation(incident, signal_data) await _record_signal_channel_history( incident=incident, signal_data=signal_data, message_id=message_id, labels=labels, ) pdi_duration_ms = await _run_pre_decision_investigation(incident) raw_evidence_created = await _record_raw_signal_evidence_if_needed( incident=incident, signal_data=signal_data, message_id=message_id, labels=labels, pdi_duration_ms=pdi_duration_ms, ) await _add_observation_timeline( incident=incident, signal_data=signal_data, labels=labels, raw_evidence_created=raw_evidence_created, ) return { "incident_id": _incident_id(incident), "alertname": _alertname(incident, signal_data), "raw_evidence_created": raw_evidence_created, "pdi_duration_ms": pdi_duration_ms, }