1185 lines
43 KiB
Python
1185 lines
43 KiB
Python
"""Domain-aware routing for non-Kubernetes controlled alert targets.
|
|
|
|
The Alertmanager namespace label is a source attribute, not proof that every
|
|
alert is a Kubernetes workload. Cold-start/reboot recovery is a host control
|
|
plane lane. It must never fall through to a guessed ``kubectl`` action or the
|
|
generic Ansible keyword catalog merely because the source payload mentions a
|
|
host, backup, or container.
|
|
|
|
This module is deliberately pure. It only builds machine-readable routing
|
|
and receipt contracts; it does not dispatch Agent99, run a command, or write
|
|
runtime state.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import re
|
|
from collections.abc import Mapping
|
|
from typing import Any
|
|
|
|
from src.services.ai_provider_policy import PRODUCTION_PROVIDER_ORDER
|
|
from src.services.service_registry import (
|
|
ServiceRegistryClient,
|
|
StatefulLevel,
|
|
get_service_registry,
|
|
)
|
|
|
|
_RECOVERY_MARKERS = (
|
|
"cold-start",
|
|
"cold_start",
|
|
"cold start",
|
|
"full-stack-cold-start",
|
|
"full_stack_cold_start",
|
|
"full stack cold start",
|
|
"reboot-auto-recovery",
|
|
"reboot_auto_recovery",
|
|
"reboot auto recovery",
|
|
)
|
|
|
|
_BACKUP_RESTORE_MARKERS = (
|
|
"backup",
|
|
"restore",
|
|
"escrow",
|
|
"offsite",
|
|
"retention",
|
|
)
|
|
_WINDOWS_VMWARE_MARKERS = (
|
|
"agent99",
|
|
"windows99",
|
|
"windows 99",
|
|
"vmware",
|
|
"vmx",
|
|
)
|
|
_AGENT99_CONTROL_PLANE_HEALTH_ALERTS = {"agent99serviceunhealthy"}
|
|
_TELEGRAM_CALLBACK_FORWARDER_ALERTS = {
|
|
"telegramcallbackingressunverified",
|
|
"telegramcallbackforwarderdrift",
|
|
}
|
|
_DISK_ALERTS = {
|
|
"hostoutofdiskspace",
|
|
"hostdiskusagehigh",
|
|
"hostdiskusagecritical",
|
|
"hostdiskwillfillin24hours",
|
|
}
|
|
_NODE_EXPORTER_ALERTS = {
|
|
"nodeexporterdown",
|
|
"nodeexporterscrapedown",
|
|
"nodeexporterunhealthy",
|
|
}
|
|
_ALERTMANAGER_DELIVERY_ALERTS = {"alertchainbrokenalertmanager"}
|
|
_PROVIDER_FRESHNESS_ALERTS = {
|
|
"providerfreshnesssignal",
|
|
"sourceprovideringestionstale",
|
|
}
|
|
_OLLAMA_LOCAL_RUNTIME_ALERTS = {
|
|
"ollamalocalunavailable",
|
|
"ollama111unavailable",
|
|
}
|
|
_KUBERNETES_KINDS = {
|
|
"cronjob",
|
|
"daemonset",
|
|
"deployment",
|
|
"job",
|
|
"pod",
|
|
"statefulset",
|
|
}
|
|
_KUBERNETES_RUNTIME_UID = re.compile(
|
|
r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
|
|
)
|
|
_KUBERNETES_RUNTIME_IDENTITY_SOURCE = "kubernetes_api_uid_readback"
|
|
_HOSTS = {
|
|
"99": ("host_99", "192.168.0.99"),
|
|
"110": ("host_110", "192.168.0.110"),
|
|
"111": ("host_111", "192.168.0.111"),
|
|
"112": ("host_112", "192.168.0.112"),
|
|
"120": ("host_120", "192.168.0.120"),
|
|
"121": ("host_121", "192.168.0.121"),
|
|
"188": ("host_188", "192.168.0.188"),
|
|
}
|
|
|
|
|
|
def _text_token(value: str | None) -> str:
|
|
return " ".join(str(value or "").strip().lower().split())
|
|
|
|
|
|
def _asset_token(value: str) -> str:
|
|
normalized = re.sub(r"[^a-z0-9_.-]+", "-", value.lower()).strip("-.")
|
|
return normalized[:96] or "unbound"
|
|
|
|
|
|
def _compact_token(value: str | None) -> str:
|
|
return re.sub(r"[^a-z0-9]+", "", _text_token(value))
|
|
|
|
|
|
def _kubernetes_drift_work_item_id(*values: str | None) -> str:
|
|
key = "|".join(_text_token(value) for value in values) or "missing-target"
|
|
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:12].upper()
|
|
return f"AIA-ASSET-DRIFT-{digest}"
|
|
|
|
|
|
def _kubernetes_identity_evidence(
|
|
*,
|
|
identity: Mapping[str, Any],
|
|
labels: Mapping[str, Any],
|
|
namespace: str,
|
|
target_resource: str,
|
|
runtime_identity_evidence: Mapping[str, Any] | None,
|
|
) -> dict[str, Any]:
|
|
"""Verify a Kubernetes identity without trusting a workload-kind label alone."""
|
|
|
|
kind = _text_token(
|
|
str(
|
|
labels.get("workload_kind")
|
|
or labels.get("kubernetes_kind")
|
|
or labels.get("kind")
|
|
or ""
|
|
)
|
|
)
|
|
name = _text_token(
|
|
str(
|
|
labels.get("workload_name")
|
|
or labels.get(kind)
|
|
or labels.get("deployment")
|
|
or labels.get("statefulset")
|
|
or labels.get("daemonset")
|
|
or labels.get("cronjob")
|
|
or labels.get("job")
|
|
or labels.get("pod")
|
|
or ""
|
|
)
|
|
)
|
|
label_namespace = _text_token(str(labels.get("namespace") or ""))
|
|
requested_namespace = _text_token(namespace)
|
|
target_name = _text_token(target_resource)
|
|
registry_name = _text_token(str(identity.get("service_name") or ""))
|
|
registry_exact = bool(
|
|
identity.get("resolution_status") == "resolved"
|
|
and identity.get("asset_domain") == "kubernetes_workload"
|
|
and requested_namespace
|
|
and label_namespace == requested_namespace
|
|
and kind in _KUBERNETES_KINDS
|
|
and name
|
|
and name == target_name
|
|
and (not registry_name or registry_name == target_name)
|
|
)
|
|
|
|
runtime_receipt = (
|
|
runtime_identity_evidence
|
|
if isinstance(runtime_identity_evidence, Mapping)
|
|
else {}
|
|
)
|
|
runtime_uid = _text_token(str(runtime_receipt.get("runtime_uid") or ""))
|
|
runtime_namespace = _text_token(
|
|
str(runtime_receipt.get("namespace") or "")
|
|
)
|
|
runtime_kind = _text_token(str(runtime_receipt.get("kind") or ""))
|
|
runtime_name = _text_token(str(runtime_receipt.get("name") or ""))
|
|
runtime_source = _text_token(
|
|
str(runtime_receipt.get("source") or "")
|
|
)
|
|
runtime_status = _text_token(
|
|
str(runtime_receipt.get("status") or "")
|
|
)
|
|
runtime_uid_verified = bool(
|
|
runtime_receipt.get("schema_version")
|
|
== "kubernetes_runtime_identity_receipt_v1"
|
|
and bool(
|
|
_KUBERNETES_RUNTIME_UID.fullmatch(runtime_uid)
|
|
)
|
|
and runtime_source == _KUBERNETES_RUNTIME_IDENTITY_SOURCE
|
|
and runtime_status == "verified"
|
|
and runtime_receipt.get("durable_readback_ack") is True
|
|
and runtime_receipt.get("verified_by")
|
|
== "kubernetes_runtime_identity_verifier"
|
|
and all(
|
|
str(runtime_receipt.get(field) or "").strip()
|
|
for field in (
|
|
"receipt_id",
|
|
"trace_id",
|
|
"run_id",
|
|
"work_item_id",
|
|
)
|
|
)
|
|
and requested_namespace
|
|
and runtime_namespace == requested_namespace
|
|
and runtime_kind in _KUBERNETES_KINDS
|
|
and runtime_name
|
|
and runtime_name == target_name
|
|
and (not kind or runtime_kind == kind)
|
|
and (not name or runtime_name == name)
|
|
)
|
|
|
|
method = (
|
|
"canonical_registry_namespace_kind_name_exact"
|
|
if registry_exact
|
|
else "kubernetes_runtime_uid_readback"
|
|
if runtime_uid_verified
|
|
else "unverified"
|
|
)
|
|
evidence: dict[str, Any] = {
|
|
"schema_version": "kubernetes_asset_identity_evidence_v1",
|
|
"status": "verified" if registry_exact or runtime_uid_verified else "unverified",
|
|
"method": method,
|
|
"namespace": requested_namespace or None,
|
|
"kind": kind or runtime_kind or None,
|
|
"name": name or runtime_name or target_name or None,
|
|
"registry_identity_exact": registry_exact,
|
|
"runtime_uid_verified": runtime_uid_verified,
|
|
"required": [
|
|
"canonical_registry_identity_plus_exact_namespace_kind_name",
|
|
"or_kubernetes_api_runtime_uid_readback",
|
|
],
|
|
}
|
|
if runtime_uid_verified:
|
|
evidence["runtime_uid_sha256"] = hashlib.sha256(
|
|
runtime_uid.encode("utf-8")
|
|
).hexdigest()
|
|
evidence["runtime_identity_source"] = _KUBERNETES_RUNTIME_IDENTITY_SOURCE
|
|
return evidence
|
|
|
|
|
|
def _host_scope(*values: str | None) -> tuple[str, str] | None:
|
|
"""Resolve only an explicit, known host identity."""
|
|
|
|
for value in values:
|
|
normalized = _text_token(value)
|
|
for suffix, scope in _HOSTS.items():
|
|
patterns = (
|
|
rf"(?:^|[^0-9]){suffix}(?:[^0-9]|$)",
|
|
rf"192\.168\.0\.{suffix}(?:[^0-9]|$)",
|
|
rf"host[_-]?{suffix}(?:[^0-9]|$)",
|
|
)
|
|
if any(re.search(pattern, normalized) for pattern in patterns):
|
|
return scope
|
|
return None
|
|
|
|
|
|
def _agent99_bridge(*, execution_role: str, dispatch_allowed: bool) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "agent99_host_operations_bridge_v1",
|
|
"integration_status": "contract_required_runtime_receipt_pending",
|
|
"execution_role": execution_role,
|
|
"dispatch_allowed": dispatch_allowed,
|
|
"dispatch_identity": "agent99_controlled_dispatch_identity_v1",
|
|
"command_envelope": "agent99_controlled_dispatch_input_v1",
|
|
"dispatch_receipt": "agent99_controlled_dispatch_receipt_v1",
|
|
"outcome_contract": "agent99_outcome_contract_v1",
|
|
"completion_callback": "/api/v1/agents/agent99/completion-callback",
|
|
"required_correlation": ["trace_id", "run_id", "work_item_id"],
|
|
"required_controls": [
|
|
"canonical_asset_id",
|
|
"typed_domain",
|
|
"allowlisted_command_id",
|
|
"ttl",
|
|
"idempotency_key",
|
|
"check_receipt",
|
|
"bounded_apply_receipt",
|
|
"independent_verifier_receipt",
|
|
],
|
|
"forbidden": [
|
|
"arbitrary_command",
|
|
"secret_read",
|
|
"cross_domain_fallback",
|
|
"unresolved_asset_dispatch",
|
|
"mark_resolved_without_callback_and_verifier",
|
|
],
|
|
}
|
|
|
|
|
|
def _typed_route(
|
|
*,
|
|
resolution_status: str,
|
|
target_kind: str,
|
|
target_resource: str,
|
|
namespace: str,
|
|
canonical_asset_id: str | None,
|
|
host: str | None,
|
|
executor: str | None,
|
|
verifier: str | None,
|
|
risk_class: str,
|
|
allowed_catalog_ids: list[str] | None = None,
|
|
allowed_inventory_hosts: list[str] | None = None,
|
|
drift_work_item_id: str | None = None,
|
|
stateful_level: str | None = None,
|
|
controlled_apply_allowed: bool | None = None,
|
|
break_glass_executor: str | None = None,
|
|
execution_role: str = "policy_observer_and_no_secret_dispatch_relay",
|
|
) -> dict[str, Any]:
|
|
resolved = resolution_status == "resolved"
|
|
apply_allowed = (
|
|
resolved and risk_class != "critical"
|
|
if controlled_apply_allowed is None
|
|
else resolved and risk_class != "critical" and controlled_apply_allowed
|
|
)
|
|
agent99_dispatch = resolved and target_kind in {
|
|
"control_plane_recovery",
|
|
"windows_vmware",
|
|
# Agent99 is only the read-only evidence collector for this domain;
|
|
# backup/restore mutation remains on the critical break-glass executor.
|
|
"backup_restore",
|
|
}
|
|
return {
|
|
"schema_version": "typed_domain_target_route_v2",
|
|
"resolution_status": resolution_status,
|
|
"route_id": (
|
|
f"typed:{target_kind}:{_asset_token(canonical_asset_id or target_resource)}"
|
|
),
|
|
"target_kind": target_kind,
|
|
"target_resource": target_resource,
|
|
"canonical_asset_id": canonical_asset_id,
|
|
"source_namespace": namespace or None,
|
|
"host": host,
|
|
"stateful_level": stateful_level,
|
|
"executor": executor,
|
|
"break_glass_executor": break_glass_executor,
|
|
"verifier": verifier,
|
|
"risk_class": risk_class,
|
|
"allowed_catalog_ids": list(allowed_catalog_ids or []),
|
|
"allowed_inventory_hosts": list(allowed_inventory_hosts or []),
|
|
"controlled_apply_allowed": apply_allowed,
|
|
"critical_break_glass_required": risk_class == "critical",
|
|
"cross_domain_fallback_allowed": False,
|
|
"circuit_open_behavior": "stay_in_same_domain_and_create_repair_work_item",
|
|
"drift_work_item_id": drift_work_item_id,
|
|
"investigator_contract": {
|
|
"active": "PreDecisionInvestigator",
|
|
"holmesgpt": "shadow_pending_internal_immutable_artifact",
|
|
"required_evidence": [
|
|
"source_receipt",
|
|
"canonical_asset_identity",
|
|
"metrics_logs_traces",
|
|
"recent_change_and_related_run",
|
|
],
|
|
},
|
|
"reasoning_contract": {
|
|
"rca": "ollama_provider_chain",
|
|
"provider_order": list(PRODUCTION_PROVIDER_ORDER),
|
|
"critic": "gemini_final_fallback_explicit_cost_gate",
|
|
"paid_call_allowed": False,
|
|
},
|
|
"agent99_bridge": _agent99_bridge(
|
|
execution_role=execution_role,
|
|
dispatch_allowed=agent99_dispatch,
|
|
),
|
|
}
|
|
|
|
|
|
def resolve_typed_alert_target(
|
|
*,
|
|
alertname: str,
|
|
target_resource: str,
|
|
namespace: str,
|
|
labels: Mapping[str, Any] | None = None,
|
|
alert_category: str = "",
|
|
registry: ServiceRegistryClient | None = None,
|
|
runtime_identity_evidence: Mapping[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Resolve exactly one domain route; unknown assets never fall back AUTO."""
|
|
|
|
labels = labels or {}
|
|
registry = registry or get_service_registry()
|
|
text = " ".join(
|
|
(
|
|
_text_token(alertname),
|
|
_text_token(target_resource),
|
|
_text_token(alert_category),
|
|
)
|
|
)
|
|
compact_alert = _compact_token(alertname)
|
|
|
|
legacy_recovery = resolve_controlled_alert_target(
|
|
alertname=alertname,
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
)
|
|
if legacy_recovery is not None:
|
|
route = _typed_route(
|
|
resolution_status="resolved",
|
|
target_kind="control_plane_recovery",
|
|
target_resource=target_resource or "cold-start-gate",
|
|
namespace=namespace,
|
|
canonical_asset_id="control-plane:cold-start-gate",
|
|
host=None,
|
|
executor="Agent99",
|
|
verifier="cold_start_independent_scorecard_verifier",
|
|
risk_class="high",
|
|
execution_role="single_writer_control_plane_recovery_executor",
|
|
)
|
|
route["route_id"] = "agent99_recover_after_owner_review"
|
|
return route
|
|
|
|
if compact_alert in {
|
|
"host112guestreadbackmissing",
|
|
"host112guestnotready",
|
|
} or "host112guest" in _compact_token(target_resource):
|
|
return _typed_route(
|
|
resolution_status="resolved",
|
|
target_kind="control_plane_recovery",
|
|
target_resource=target_resource or "host112-guest-readiness",
|
|
namespace=namespace,
|
|
canonical_asset_id="host-guest:host_112",
|
|
host=_HOSTS["112"][1],
|
|
executor="Agent99",
|
|
verifier="host112_guest_independent_readiness_verifier",
|
|
risk_class="high",
|
|
allowed_inventory_hosts=["host_112"],
|
|
execution_role="single_writer_host112_guest_recovery_orchestrator",
|
|
)
|
|
|
|
if compact_alert in _ALERTMANAGER_DELIVERY_ALERTS:
|
|
return _typed_route(
|
|
resolution_status="resolved",
|
|
target_kind="docker_container",
|
|
target_resource="alertmanager",
|
|
namespace=namespace,
|
|
canonical_asset_id="container:host_110:alertmanager",
|
|
host=_HOSTS["110"][1],
|
|
executor="host_ansible_executor",
|
|
verifier="alert_chain_independent_delivery_verifier",
|
|
risk_class="medium",
|
|
allowed_catalog_ids=["ansible:110-alertmanager-delivery-recovery"],
|
|
allowed_inventory_hosts=["host_110"],
|
|
execution_role="single_writer_alert_chain_container_executor",
|
|
)
|
|
|
|
if (
|
|
compact_alert in _PROVIDER_FRESHNESS_ALERTS
|
|
or "source_provider_freshness_triage" in text
|
|
or "alertchain_provider_freshness" in text
|
|
):
|
|
provider = _text_token(
|
|
str(
|
|
labels.get("source")
|
|
or labels.get("provider")
|
|
or labels.get("service")
|
|
or ""
|
|
)
|
|
)
|
|
if provider not in {"sentry", "signoz"}:
|
|
provider = "sentry-signoz"
|
|
route = _typed_route(
|
|
resolution_status="resolved",
|
|
target_kind="observability_source_ingestion",
|
|
target_resource=target_resource or "source-provider-ingestion",
|
|
namespace=namespace,
|
|
canonical_asset_id=f"observability-provider:{provider}",
|
|
host=None,
|
|
executor="provider_freshness_readonly_executor",
|
|
verifier="provider_freshness_independent_verifier",
|
|
risk_class="low",
|
|
controlled_apply_allowed=True,
|
|
execution_role=(
|
|
"single_writer_observability_source_ingestion_readonly_executor"
|
|
),
|
|
)
|
|
route["provider_freshness_contract"] = {
|
|
"schema_version": "provider_freshness_typed_route_v1",
|
|
"execution_mode": "read_only",
|
|
"required_providers": ["sentry", "signoz"],
|
|
"allowed_actions": ["provider_specific_coverage_readback"],
|
|
"mutating_actions_allowed": False,
|
|
"generic_recurrence_can_resolve": False,
|
|
}
|
|
return route
|
|
|
|
if compact_alert in _OLLAMA_LOCAL_RUNTIME_ALERTS:
|
|
return _typed_route(
|
|
resolution_status="resolved",
|
|
target_kind="host_launchagent",
|
|
target_resource="ollama-local",
|
|
namespace=namespace,
|
|
canonical_asset_id="ai-provider:ollama_local",
|
|
host=_HOSTS["111"][1],
|
|
executor="host_ansible_executor",
|
|
verifier="ollama111_k3s_path_independent_verifier",
|
|
risk_class="medium",
|
|
allowed_catalog_ids=["ansible:111-ollama-fallback"],
|
|
allowed_inventory_hosts=["host_111", "host_120", "host_121"],
|
|
execution_role="single_writer_host111_launchagent_executor",
|
|
)
|
|
|
|
if compact_alert in _AGENT99_CONTROL_PLANE_HEALTH_ALERTS:
|
|
# Agent99's own health signal is a read-only control-plane route. It
|
|
# is never permission to recover a guest, power a VM, reboot a host,
|
|
# or fall through to an arbitrary Windows command. An explicit host
|
|
# identity that is not host_99 is drift and must fail closed.
|
|
explicit_host_scope = _host_scope(
|
|
str(labels.get("host") or ""),
|
|
str(labels.get("instance") or ""),
|
|
str(labels.get("node") or ""),
|
|
target_resource,
|
|
)
|
|
if (
|
|
explicit_host_scope is not None
|
|
and explicit_host_scope != _HOSTS["99"]
|
|
):
|
|
return _typed_route(
|
|
resolution_status="asset_identity_unresolved",
|
|
target_kind="unknown",
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
canonical_asset_id=None,
|
|
host=None,
|
|
executor=None,
|
|
verifier=None,
|
|
risk_class="high",
|
|
drift_work_item_id=_kubernetes_drift_work_item_id(
|
|
alertname,
|
|
target_resource,
|
|
explicit_host_scope[0],
|
|
),
|
|
stateful_level=StatefulLevel.UNRESOLVED.value,
|
|
)
|
|
return _typed_route(
|
|
resolution_status="resolved",
|
|
target_kind="windows_vmware",
|
|
target_resource=target_resource or "Agent99",
|
|
namespace=namespace,
|
|
canonical_asset_id="windows-vmware:host_99",
|
|
host=_HOSTS["99"][1],
|
|
executor="Agent99",
|
|
verifier="agent99_control_plane_selfcheck_independent_verifier",
|
|
risk_class="high",
|
|
allowed_inventory_hosts=["host_99"],
|
|
controlled_apply_allowed=False,
|
|
execution_role="single_writer_agent99_control_plane_selfcheck_executor",
|
|
)
|
|
|
|
if any(marker in text for marker in _BACKUP_RESTORE_MARKERS):
|
|
identity = registry.resolve_identity(target_resource)
|
|
generic_backup_signal = _compact_token(target_resource) in {
|
|
"",
|
|
"backuprestore",
|
|
}
|
|
if (
|
|
identity.get("resolution_status") != "resolved"
|
|
and not generic_backup_signal
|
|
):
|
|
return _typed_route(
|
|
resolution_status="asset_identity_unresolved",
|
|
target_kind="unknown",
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
canonical_asset_id=None,
|
|
host=None,
|
|
executor=None,
|
|
verifier=None,
|
|
risk_class="critical",
|
|
drift_work_item_id=identity["drift_work_item_id"],
|
|
stateful_level=StatefulLevel.UNRESOLVED.value,
|
|
)
|
|
return _typed_route(
|
|
resolution_status="resolved",
|
|
target_kind="backup_restore",
|
|
target_resource=target_resource or "backup_restore",
|
|
namespace=namespace,
|
|
canonical_asset_id=(
|
|
identity.get("canonical_id") or "data-protection:backup-restore"
|
|
),
|
|
host=identity.get("host"),
|
|
executor="Agent99",
|
|
break_glass_executor="backup_restore_break_glass",
|
|
verifier="backup_restore_readback_verifier",
|
|
risk_class="critical",
|
|
stateful_level=identity.get("stateful_level"),
|
|
execution_role="single_writer_agent99_backup_readback_collector",
|
|
)
|
|
|
|
if any(marker in text for marker in _WINDOWS_VMWARE_MARKERS):
|
|
host_scope = _host_scope(
|
|
str(labels.get("host") or ""),
|
|
str(labels.get("instance") or ""),
|
|
target_resource,
|
|
alertname,
|
|
)
|
|
explicit_agent99_target = any(
|
|
marker in _text_token(target_resource)
|
|
for marker in ("agent99", "windows99", "windows 99")
|
|
)
|
|
if host_scope is None and explicit_agent99_target:
|
|
host_scope = _HOSTS["99"]
|
|
if host_scope is None:
|
|
identity = registry.resolve_identity(target_resource or alertname)
|
|
return _typed_route(
|
|
resolution_status="asset_identity_unresolved",
|
|
target_kind="unknown",
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
canonical_asset_id=None,
|
|
host=None,
|
|
executor=None,
|
|
verifier=None,
|
|
risk_class="high",
|
|
drift_work_item_id=identity["drift_work_item_id"],
|
|
stateful_level=StatefulLevel.UNRESOLVED.value,
|
|
)
|
|
return _typed_route(
|
|
resolution_status="resolved",
|
|
target_kind="windows_vmware",
|
|
target_resource=target_resource or host_scope[0],
|
|
namespace=namespace,
|
|
canonical_asset_id=f"windows-vmware:{host_scope[0]}",
|
|
host=host_scope[1],
|
|
executor="Agent99",
|
|
verifier="agent99_independent_runtime_verifier",
|
|
risk_class="high",
|
|
allowed_inventory_hosts=[host_scope[0]],
|
|
execution_role="single_writer_windows_vmware_executor",
|
|
)
|
|
|
|
if compact_alert == "wazuhalertmanagerintegrationconvergence":
|
|
identity = registry.resolve_identity(target_resource)
|
|
inventory_host = _host_scope(str(identity.get("host") or ""))
|
|
if (
|
|
identity.get("resolution_status") == "resolved"
|
|
and identity.get("canonical_id") == "service:wazuh-manager"
|
|
and inventory_host is not None
|
|
and inventory_host[0] == "host_112"
|
|
):
|
|
return _typed_route(
|
|
resolution_status="resolved",
|
|
target_kind="host_systemd",
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
canonical_asset_id="service:wazuh-manager",
|
|
host=inventory_host[1],
|
|
executor="host_ansible_executor",
|
|
verifier="wazuh_alert_ingress_independent_verifier",
|
|
risk_class="high",
|
|
allowed_catalog_ids=[
|
|
"ansible:wazuh-alertmanager-integration"
|
|
],
|
|
allowed_inventory_hosts=["host_112"],
|
|
stateful_level=str(identity.get("stateful_level") or ""),
|
|
)
|
|
|
|
host_scope = _host_scope(
|
|
str(labels.get("host") or ""),
|
|
str(labels.get("instance") or ""),
|
|
str(labels.get("node") or ""),
|
|
target_resource,
|
|
alertname,
|
|
)
|
|
host110_pressure_alert = compact_alert.startswith(
|
|
"host110sustainedmoderatepressure"
|
|
)
|
|
allowed_catalog_ids: list[str] = []
|
|
if compact_alert in _DISK_ALERTS and host_scope is not None:
|
|
if host_scope[0] in {"host_110", "host_188"}:
|
|
allowed_catalog_ids = [
|
|
f"ansible:{host_scope[0].removeprefix('host_')}-disk-pressure"
|
|
]
|
|
elif compact_alert in _NODE_EXPORTER_ALERTS and host_scope is not None:
|
|
if host_scope[0] == "host_110":
|
|
allowed_catalog_ids = ["ansible:110-devops"]
|
|
elif (
|
|
host110_pressure_alert
|
|
and host_scope is not None
|
|
and host_scope[0] == "host_110"
|
|
):
|
|
allowed_catalog_ids = ["ansible:110-host-pressure-readonly"]
|
|
elif (
|
|
"wazuh" in text
|
|
and host_scope is not None
|
|
and host_scope[0] == "host_112"
|
|
):
|
|
allowed_catalog_ids = ["ansible:wazuh-manager-posture-readback"]
|
|
|
|
if host_scope is not None and (
|
|
compact_alert in _DISK_ALERTS
|
|
or compact_alert in _NODE_EXPORTER_ALERTS
|
|
or host110_pressure_alert
|
|
or "wazuh" in text
|
|
):
|
|
return _typed_route(
|
|
resolution_status="resolved",
|
|
target_kind="host_systemd",
|
|
target_resource=target_resource or host_scope[0],
|
|
namespace=namespace,
|
|
canonical_asset_id=f"host:{host_scope[1]}",
|
|
host=host_scope[1],
|
|
executor="host_ansible_executor",
|
|
verifier="host_runtime_independent_verifier",
|
|
risk_class="medium",
|
|
allowed_catalog_ids=allowed_catalog_ids,
|
|
allowed_inventory_hosts=[host_scope[0]],
|
|
)
|
|
|
|
identity = registry.resolve_identity(target_resource)
|
|
if identity["resolution_status"] == "resolved":
|
|
domain = str(identity["asset_domain"])
|
|
if domain == "kubernetes_workload":
|
|
identity_evidence = _kubernetes_identity_evidence(
|
|
identity=identity,
|
|
labels=labels,
|
|
namespace=namespace,
|
|
target_resource=target_resource,
|
|
runtime_identity_evidence=runtime_identity_evidence,
|
|
)
|
|
if identity_evidence["status"] != "verified":
|
|
route = _typed_route(
|
|
resolution_status="asset_identity_unresolved",
|
|
target_kind="unknown",
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
canonical_asset_id=None,
|
|
host=None,
|
|
executor=None,
|
|
verifier=None,
|
|
risk_class="high",
|
|
drift_work_item_id=_kubernetes_drift_work_item_id(
|
|
namespace,
|
|
target_resource,
|
|
str(labels.get("workload_kind") or ""),
|
|
),
|
|
stateful_level=StatefulLevel.UNRESOLVED.value,
|
|
)
|
|
route["resolution_reason"] = (
|
|
"kubernetes_identity_evidence_missing_or_mismatch"
|
|
)
|
|
route["identity_evidence"] = identity_evidence
|
|
return route
|
|
route = _typed_route(
|
|
resolution_status="resolved",
|
|
target_kind="kubernetes_workload",
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
canonical_asset_id=str(identity["canonical_id"]),
|
|
host="k3s",
|
|
executor=str(identity.get("executor") or "") or None,
|
|
verifier=str(identity.get("verifier") or "") or None,
|
|
risk_class="medium",
|
|
stateful_level=str(identity.get("stateful_level") or ""),
|
|
controlled_apply_allowed=bool(
|
|
identity.get("controlled_apply_allowed")
|
|
),
|
|
)
|
|
route["identity_evidence"] = identity_evidence
|
|
return route
|
|
inventory_host = _host_scope(str(identity.get("host") or ""))
|
|
catalog_ids = list(identity.get("allowed_catalog_ids") or [])
|
|
exact_sentry_consumer = (
|
|
_text_token(target_resource)
|
|
== "sentry-self-hosted-snuba-profiling-functions-consumer-1"
|
|
)
|
|
exact_registry_catalog_target = bool(
|
|
domain == "host_systemd"
|
|
and _text_token(target_resource)
|
|
== _text_token(str(identity.get("service_name") or ""))
|
|
)
|
|
exact_openclaw_compose_target = bool(
|
|
domain == "docker_compose_container"
|
|
and identity.get("canonical_id") == "service:openclaw:host188"
|
|
and compact_alert in _TELEGRAM_CALLBACK_FORWARDER_ALERTS
|
|
and _text_token(target_resource)
|
|
in {"openclaw", "clawbot", "clawbot.service"}
|
|
)
|
|
if not (
|
|
exact_sentry_consumer
|
|
or exact_registry_catalog_target
|
|
or exact_openclaw_compose_target
|
|
):
|
|
catalog_ids = []
|
|
stateful_level = str(identity.get("stateful_level") or "")
|
|
risk = (
|
|
"critical"
|
|
if domain in {"backup_restore"}
|
|
else "high"
|
|
if domain in {"database", "storage"}
|
|
or stateful_level
|
|
in {StatefulLevel.BLOCK.value, StatefulLevel.CRITICAL_HITL.value}
|
|
else "medium"
|
|
)
|
|
read_only_backup_route = domain == "backup_restore"
|
|
return _typed_route(
|
|
resolution_status="resolved",
|
|
target_kind=domain,
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
canonical_asset_id=str(identity["canonical_id"]),
|
|
host=identity.get("host"),
|
|
executor=(
|
|
"Agent99"
|
|
if read_only_backup_route
|
|
else str(identity.get("executor") or "") or None
|
|
),
|
|
break_glass_executor=(
|
|
str(identity.get("executor") or "backup_restore_break_glass")
|
|
if read_only_backup_route
|
|
else None
|
|
),
|
|
verifier=str(identity.get("verifier") or "") or None,
|
|
risk_class=risk,
|
|
allowed_catalog_ids=catalog_ids,
|
|
allowed_inventory_hosts=[inventory_host[0]] if inventory_host else [],
|
|
stateful_level=stateful_level,
|
|
controlled_apply_allowed=bool(
|
|
identity.get("controlled_apply_allowed")
|
|
),
|
|
execution_role=(
|
|
"single_writer_agent99_backup_readback_collector"
|
|
if read_only_backup_route
|
|
else "policy_observer_and_no_secret_dispatch_relay"
|
|
),
|
|
)
|
|
|
|
identity_evidence = _kubernetes_identity_evidence(
|
|
identity=identity,
|
|
labels=labels,
|
|
namespace=namespace,
|
|
target_resource=target_resource,
|
|
runtime_identity_evidence=runtime_identity_evidence,
|
|
)
|
|
if identity_evidence["runtime_uid_verified"] is True:
|
|
kubernetes_kind = str(identity_evidence["kind"])
|
|
route = _typed_route(
|
|
resolution_status="resolved",
|
|
target_kind="kubernetes_workload",
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
canonical_asset_id=(
|
|
f"k8s:{namespace or 'unbound'}:{kubernetes_kind}:"
|
|
f"{_asset_token(target_resource)}"
|
|
),
|
|
host="k3s",
|
|
executor="kubernetes_controlled_executor",
|
|
verifier="kubernetes_rollout_verifier",
|
|
risk_class="medium",
|
|
)
|
|
route["identity_evidence"] = identity_evidence
|
|
return route
|
|
|
|
route = _typed_route(
|
|
resolution_status="asset_identity_unresolved",
|
|
target_kind="unknown",
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
canonical_asset_id=None,
|
|
host=None,
|
|
executor=None,
|
|
verifier=None,
|
|
risk_class="high",
|
|
drift_work_item_id=identity["drift_work_item_id"],
|
|
stateful_level=StatefulLevel.UNRESOLVED.value,
|
|
)
|
|
if any(
|
|
_text_token(str(labels.get(key) or ""))
|
|
for key in ("workload_kind", "kubernetes_kind", "runtime_uid")
|
|
):
|
|
route["resolution_reason"] = (
|
|
"kubernetes_identity_evidence_missing_or_mismatch"
|
|
)
|
|
route["identity_evidence"] = identity_evidence
|
|
return route
|
|
|
|
|
|
def resolve_typed_incident_target(
|
|
incident: Mapping[str, Any] | None,
|
|
*,
|
|
registry: ServiceRegistryClient | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Extract public incident fields and resolve one canonical typed route."""
|
|
|
|
incident = incident or {}
|
|
signals = [
|
|
signal
|
|
for signal in incident.get("signals") or []
|
|
if isinstance(signal, Mapping)
|
|
]
|
|
first_signal = signals[0] if signals else {}
|
|
labels = (
|
|
first_signal.get("labels")
|
|
if isinstance(first_signal.get("labels"), Mapping)
|
|
else {}
|
|
)
|
|
runtime_identity_evidence = (
|
|
first_signal.get("runtime_identity_evidence")
|
|
if isinstance(first_signal.get("runtime_identity_evidence"), Mapping)
|
|
else None
|
|
)
|
|
alertname = str(
|
|
incident.get("alertname")
|
|
or first_signal.get("alert_name")
|
|
or labels.get("alertname")
|
|
or ""
|
|
)
|
|
registry = registry or get_service_registry()
|
|
candidates = [
|
|
str(incident.get("target_resource") or ""),
|
|
*[
|
|
str(value)
|
|
for value in incident.get("affected_services") or []
|
|
if str(value).strip()
|
|
],
|
|
*[
|
|
str(labels.get(key) or "")
|
|
for key in (
|
|
"container_name",
|
|
"container",
|
|
"component",
|
|
"service",
|
|
"deployment",
|
|
"pod",
|
|
"target_resource",
|
|
)
|
|
],
|
|
]
|
|
target_resource = next((value for value in candidates if value.strip()), "")
|
|
for candidate in candidates:
|
|
if candidate and registry.get_service(candidate) is not None:
|
|
target_resource = candidate
|
|
break
|
|
return resolve_typed_alert_target(
|
|
alertname=alertname,
|
|
target_resource=target_resource,
|
|
namespace=str(labels.get("namespace") or ""),
|
|
labels=labels,
|
|
alert_category=str(incident.get("alert_category") or ""),
|
|
registry=registry,
|
|
runtime_identity_evidence=runtime_identity_evidence,
|
|
)
|
|
|
|
|
|
def resolve_controlled_alert_target(
|
|
*,
|
|
alertname: str,
|
|
target_resource: str,
|
|
namespace: str,
|
|
) -> dict[str, Any] | None:
|
|
"""Return the dedicated executor route for a known non-K8s target."""
|
|
|
|
text = " ".join(
|
|
(
|
|
_text_token(alertname),
|
|
_text_token(target_resource),
|
|
)
|
|
)
|
|
if not any(marker in text for marker in _RECOVERY_MARKERS):
|
|
return None
|
|
|
|
return {
|
|
"schema_version": "controlled_alert_target_route_v1",
|
|
"target_kind": "control_plane_recovery",
|
|
"target_resource": target_resource or "cold-start-gate",
|
|
"source_namespace": namespace or None,
|
|
"normalized_execution_namespace": "host_recovery",
|
|
"kubernetes_namespace_applicable": False,
|
|
"executor": "Agent99",
|
|
"route_id": "agent99_recover_after_owner_review",
|
|
"check_route": "agent99-mode Status controlledApply=false",
|
|
"controlled_apply_route": "agent99-mode Recover controlledApply=true",
|
|
"rollback_route": "agent99-mode Status controlledApply=false",
|
|
"runtime_execution_authorized": False,
|
|
"runtime_write_allowed": False,
|
|
}
|
|
|
|
|
|
def build_controlled_recovery_promotion_contract(
|
|
*,
|
|
alertname: str,
|
|
target_resource: str,
|
|
namespace: str,
|
|
incident_id: str = "",
|
|
) -> dict[str, Any] | None:
|
|
"""Build the no-false-green check/apply/verifier/learning contract."""
|
|
|
|
route = resolve_controlled_alert_target(
|
|
alertname=alertname,
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
)
|
|
if route is None:
|
|
return None
|
|
|
|
source_id = _asset_token(incident_id or f"{alertname}:{target_resource}")
|
|
fields = [
|
|
{
|
|
"field": "target_selector",
|
|
"status": "ready",
|
|
"source": "domain_target_normalizer",
|
|
"value": {
|
|
"target_kind": route["target_kind"],
|
|
"target_resource": route["target_resource"],
|
|
"normalized_execution_namespace": route[
|
|
"normalized_execution_namespace"
|
|
],
|
|
},
|
|
},
|
|
{
|
|
"field": "source_sensor_receipt",
|
|
"status": "pending",
|
|
"source": "alertmanager+cold_start_scorecard",
|
|
"value": "require_current_firing_or_recurrence_and_scorecard_ref",
|
|
},
|
|
{
|
|
"field": "check_mode_receipt",
|
|
"status": "pending",
|
|
"source": "Agent99 Status + read-only cold-start sensor",
|
|
"value": route["check_route"],
|
|
},
|
|
{
|
|
"field": "controlled_apply_route",
|
|
"status": "ready",
|
|
"source": "domain_target_normalizer",
|
|
"value": route["controlled_apply_route"],
|
|
},
|
|
{
|
|
"field": "dispatch_receipt",
|
|
"status": "pending",
|
|
"source": "agent99_sre_dispatch_receipt_v1",
|
|
"value": "accepted_and_inbox_triggered_required",
|
|
},
|
|
{
|
|
"field": "rollback_route",
|
|
"status": "ready",
|
|
"source": "domain_target_normalizer",
|
|
"value": route["rollback_route"],
|
|
},
|
|
{
|
|
"field": "post_apply_verifier",
|
|
"status": "pending",
|
|
"source": "Agent99 outcome + cold-start scorecard",
|
|
"value": [
|
|
"agent99_recover_outcome_terminal_readback",
|
|
"full_stack_cold_start_scorecard_rerun",
|
|
"required_hosts_harbor_k3s_routes_green",
|
|
"original_fingerprint_resolved_or_recurrence_decreased",
|
|
],
|
|
},
|
|
{
|
|
"field": "incident_closure_receipt",
|
|
"status": "pending",
|
|
"source": "incident_timeline",
|
|
"value": "terminal_only_after_independent_verifier",
|
|
},
|
|
{
|
|
"field": "telegram_receipt",
|
|
"status": "pending",
|
|
"source": "telegram_outbound_mirror",
|
|
"value": "lifecycle_result_delivery_ack_required",
|
|
},
|
|
{
|
|
"field": "km_writeback",
|
|
"status": "pending",
|
|
"source": "Hermes",
|
|
"value": f"km:cold-start-recovery:{source_id}",
|
|
},
|
|
{
|
|
"field": "playbook_trust_writeback",
|
|
"status": "pending",
|
|
"source": "OpenClaw/NemoTron",
|
|
"value": f"playbook-trust:cold-start-recovery:{source_id}",
|
|
},
|
|
]
|
|
ready_fields = [row["field"] for row in fields if row["status"] == "ready"]
|
|
blocked_fields = [row["field"] for row in fields if row["status"] != "ready"]
|
|
return {
|
|
"schema_version": "repair_candidate_promotion_contract_v1",
|
|
"status": "controlled_route_candidate_pending_receipts",
|
|
"route_id": route["route_id"],
|
|
"executor": route["executor"],
|
|
"incident_id": incident_id or None,
|
|
"target_route": route,
|
|
"ready_count": len(ready_fields),
|
|
"total_count": len(fields),
|
|
"blocked_count": len(blocked_fields),
|
|
"ready_fields": ready_fields,
|
|
"blocked_fields": blocked_fields,
|
|
"fields": fields,
|
|
"runtime_execution_authorized": False,
|
|
"runtime_write_allowed": False,
|
|
"controlled_playbook_queue": True,
|
|
"owner_review_required": False,
|
|
"owner_review_gate": "auto_waived_for_low_medium_high",
|
|
"needs_human": False,
|
|
"completion_status": "partial",
|
|
"completion_blocker": "dispatch_verifier_and_learning_receipts_missing",
|
|
"forbidden_operations": [
|
|
"kubectl_for_host_control_plane_target",
|
|
"generic_ansible_keyword_fallback",
|
|
"host_reboot",
|
|
"node_drain",
|
|
"backup_restore",
|
|
"secret_read",
|
|
"mark_resolved_without_post_verifier",
|
|
],
|
|
"controlled_automation_closure": {
|
|
"schema_version": "controlled_recovery_closure_contract_v1",
|
|
"trace_id_required": True,
|
|
"status": "pending_source_check_dispatch_verify_writeback",
|
|
"stages": [
|
|
{"stage": "sensor", "status": "pending", "writes_runtime_state": False},
|
|
{"stage": "normalize", "status": "passed", "writes_runtime_state": False},
|
|
{"stage": "check", "status": "pending", "writes_runtime_state": False},
|
|
{"stage": "controlled_apply", "status": "pending", "writes_runtime_state": False},
|
|
{"stage": "verify", "status": "pending", "writes_runtime_state": False},
|
|
{"stage": "learn_writeback", "status": "pending", "writes_runtime_state": False},
|
|
],
|
|
"required_receipts": [row["field"] for row in fields],
|
|
"automation_asset_ledger": [
|
|
{
|
|
"asset_type": "Sensor",
|
|
"asset_id": "scripts/reboot-recovery/full-stack-cold-start-check.sh",
|
|
"owner": "ColdStartMonitor",
|
|
"status": "read_only_receipt_pending",
|
|
},
|
|
{
|
|
"asset_type": "PlayBook",
|
|
"asset_id": "agent99:Recover",
|
|
"owner": "Agent99",
|
|
"status": "dispatch_receipt_pending",
|
|
},
|
|
{
|
|
"asset_type": "Verifier",
|
|
"asset_id": f"verifier:cold-start-recovery:{source_id}",
|
|
"owner": "PostExecutionVerifier",
|
|
"status": "pending",
|
|
},
|
|
{
|
|
"asset_type": "KM",
|
|
"asset_id": f"km:cold-start-recovery:{source_id}",
|
|
"owner": "Hermes",
|
|
"status": "pending_after_verifier",
|
|
},
|
|
],
|
|
},
|
|
}
|
|
|
|
|
|
def build_controlled_recovery_handoff(
|
|
*,
|
|
alertname: str,
|
|
target_resource: str,
|
|
namespace: str,
|
|
incident_id: str = "",
|
|
) -> dict[str, Any] | None:
|
|
contract = build_controlled_recovery_promotion_contract(
|
|
alertname=alertname,
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
incident_id=incident_id,
|
|
)
|
|
if contract is None:
|
|
return None
|
|
return {
|
|
"schema_version": "ai_decision_controlled_executor_handoff_v1",
|
|
"status": "agent99_dispatch_receipt_readback_pending",
|
|
"queued": False,
|
|
"side_effect_performed": False,
|
|
"single_writer_executor": "Agent99",
|
|
"route_id": contract["route_id"],
|
|
"active_blockers": [
|
|
"agent99_dispatch_receipt_readback_pending",
|
|
"post_apply_verifier_missing",
|
|
"km_playbook_writeback_missing",
|
|
],
|
|
"safe_next_action": "read_agent99_dispatch_then_run_independent_cold_start_verifier",
|
|
"repair_candidate_promotion_contract": contract,
|
|
"runtime_execution_authorized": False,
|
|
"runtime_write_allowed": False,
|
|
"owner_review_required": False,
|
|
"owner_review_gate": "auto_waived_for_low_medium_high",
|
|
"needs_human": False,
|
|
}
|