Files
awoooi/apps/api/src/services/awooop_ansible_audit_service.py
ogt 08f3a70ce6
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 4m41s
CD Pipeline / build-and-deploy (push) Successful in 15m36s
CD Pipeline / post-deploy-checks (push) Successful in 4m19s
fix(sre): close bounded runtime contract gaps
2026-07-16 21:21:43 +08:00

2221 lines
87 KiB
Python

"""AwoooP Ansible audit helpers.
This module exposes the Ansible audit contract and repo-known playbook catalog.
Catalog rows are the first hard boundary for AI Agent controlled apply: only
allowlisted playbooks with check-mode support can move from dry-run to apply.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
from typing import Any
from uuid import UUID, uuid4
import structlog
from sqlalchemy import text
from src.db.base import get_db_context
from src.services.controlled_alert_target_router import (
resolve_controlled_alert_target,
resolve_typed_incident_target,
)
logger = structlog.get_logger(__name__)
_TYPED_ROUTE_GENERATION_PREFIX = "typed-route-v2"
_RECURRENCE_REQUIRED_PROPOSAL_SOURCES = frozenset(
{
"alert_webhook_controlled_router",
"truth_chain_candidate_backfill",
"iwooos_wazuh_manager_posture_scheduler",
"iwooos_wazuh_alert_ingress_scheduler",
}
)
def _source_fingerprint(incident: dict[str, Any], proposal: dict[str, Any]) -> str:
"""Return the current alert fingerprint without inventing one."""
explicit = str(proposal.get("source_fingerprint") or "").strip()
if explicit:
return explicit
for signal in incident.get("signals") or []:
if not isinstance(signal, dict):
continue
labels = signal.get("labels")
if not isinstance(labels, dict):
continue
fingerprint = str(
labels.get("fingerprint") or labels.get("alert_fingerprint") or ""
).strip()
if fingerprint:
return fingerprint
return ""
def _source_recurrence_receipt(
incident: dict[str, Any],
proposal: dict[str, Any],
) -> dict[str, Any]:
"""Build a fail-closed receipt proving this is not a historical-only replay."""
explicit_fingerprint = str(
proposal.get("source_fingerprint") or ""
).strip()
fingerprint = explicit_fingerprint or _source_fingerprint(incident, proposal)
proposal_source = str(proposal.get("source") or "").strip()
occurrence_id = str(proposal.get("source_occurrence_id") or "").strip()
verified = bool(
explicit_fingerprint
and occurrence_id
and proposal.get("source_recurrence_verified") is True
)
return {
"schema_version": "alert_source_recurrence_receipt_v1",
"verified": verified,
"fingerprint": fingerprint or None,
"occurrence_id": occurrence_id or None,
"observed_at": str(proposal.get("source_received_at") or "") or None,
"identity_anchor": str(
proposal.get("source_recurrence_anchor") or ""
)
or None,
"canonical_asset_id": str(
proposal.get("source_canonical_asset_id") or ""
)
or None,
"source": (
"current_alert_webhook"
if proposal_source == "alert_webhook_controlled_router"
else str(proposal.get("source_recurrence_source") or "") or None
),
"durable_readback_verified": (
proposal.get("source_recurrence_durable_readback_verified") is True
),
}
def build_typed_target_route_generation(
*,
typed_target_route: dict[str, Any],
executor_candidates: list[dict[str, Any]],
) -> str:
"""Hash only semantic target scope, not deploy SHA or volatile evidence."""
identity = {
"schema_version": str(typed_target_route.get("schema_version") or ""),
"resolution_status": str(
typed_target_route.get("resolution_status") or ""
),
"canonical_asset_id": str(
typed_target_route.get("canonical_asset_id") or ""
),
"typed_domain": str(typed_target_route.get("target_kind") or ""),
"executor": str(typed_target_route.get("executor") or ""),
"cross_domain_fallback_allowed": typed_target_route.get(
"cross_domain_fallback_allowed"
),
"candidates": sorted(
(
{
"catalog_id": str(row.get("catalog_id") or ""),
"catalog_revision": str(row.get("catalog_revision") or ""),
"canonical_asset_id": str(
row.get("canonical_asset_id") or ""
),
"typed_domain": str(row.get("typed_domain") or ""),
"inventory_hosts": sorted(
str(host) for host in row.get("inventory_hosts") or []
),
}
for row in executor_candidates
),
key=lambda row: (
row["catalog_id"],
row["catalog_revision"],
row["inventory_hosts"],
),
),
}
canonical = json.dumps(
identity,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:24]
return f"{_TYPED_ROUTE_GENERATION_PREFIX}:{digest}"
def _automation_operation_log_incident_id(value: str) -> int | None:
# ADR-090 keeps external INC-* ids in input JSON; the DB column is BIGINT.
normalized = str(value or "").strip()
return int(normalized) if normalized.isdigit() else None
ANSIBLE_OPERATION_TYPES = frozenset({
"ansible_candidate_matched",
"ansible_check_mode_executed",
"ansible_apply_executed",
"ansible_learning_writeback_recorded",
"ansible_rollback_executed",
"ansible_execution_skipped",
})
def verified_ansible_candidate_terminal_sql(candidate_alias: str) -> str:
"""Return the shared SQL predicate for a safely terminal candidate."""
if re.fullmatch(r"[a-z_][a-z0-9_]*", candidate_alias) is None:
raise ValueError("invalid_candidate_sql_alias")
candidate = candidate_alias
incident_id = (
f"coalesce({candidate}.incident_id::text, "
f"{candidate}.input ->> 'incident_id')"
)
automation_run_id = (
f"coalesce(nullif({candidate}.input ->> 'automation_run_id', ''), "
f"{candidate}.op_id::text)"
)
template = """
EXISTS (
SELECT 1
FROM automation_operation_log direct_terminal
WHERE direct_terminal.parent_op_id = __CANDIDATE__.op_id
AND direct_terminal.operation_type = 'ansible_execution_skipped'
AND direct_terminal.status = 'dry_run'
AND direct_terminal.dry_run_result ->> 'skipped' = 'true'
AND coalesce(
direct_terminal.dry_run_result ->> 'apply_executed',
'false'
) = 'false'
)
OR EXISTS (
SELECT 1
FROM automation_operation_log check_mode
JOIN automation_operation_log apply
ON apply.parent_op_id = check_mode.op_id
AND apply.operation_type = 'ansible_apply_executed'
AND apply.status = 'failed'
JOIN automation_operation_log replay
ON replay.parent_op_id = apply.op_id
AND replay.operation_type = 'ansible_execution_skipped'
AND replay.status IN ('success', 'failed')
AND replay.input ->> 'execution_mode'
= 'controlled_retry_check_mode_replay'
AND replay.output ->> 'runtime_apply_executed' = 'false'
WHERE check_mode.parent_op_id = __CANDIDATE__.op_id
AND check_mode.operation_type = 'ansible_check_mode_executed'
AND check_mode.status = 'success'
AND check_mode.input ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
AND coalesce(
check_mode.incident_id::text,
check_mode.input ->> 'incident_id'
) = __INCIDENT_ID__
AND apply.input ->> 'automation_run_id' = __AUTOMATION_RUN_ID__
AND coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
) = __INCIDENT_ID__
AND replay.input ->> 'automation_run_id' = __AUTOMATION_RUN_ID__
AND coalesce(
replay.incident_id::text,
replay.input ->> 'incident_id'
) = __INCIDENT_ID__
AND replay.output ->> 'terminal_disposition' = CASE
WHEN replay.status = 'success'
THEN 'no_write_replay_passed_waiting_repair_candidate'
ELSE 'no_write_replay_failed_waiting_playbook_or_transport_repair'
END
AND EXISTS (
SELECT 1
FROM incident_evidence verifier
WHERE verifier.incident_id = __INCIDENT_ID__
AND verifier.post_execution_state ->> 'apply_op_id'
= apply.op_id::text
AND verifier.verification_result IN ('failed', 'timeout')
)
AND EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) receipt(value)
WHERE receipt.value ->> 'stage_id' = 'retry_or_rollback'
AND receipt.value ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
AND receipt.value #>> '{detail,failed_apply_op_id}'
= apply.op_id::text
AND receipt.value #>> '{detail,retry_check_mode_op_id}'
= replay.op_id::text
AND receipt.value #>> '{detail,terminal_type}'
= replay.output ->> 'terminal_disposition'
AND receipt.value #>> '{detail,verified_no_write_terminal}'
= 'true'
AND receipt.value #>> '{detail,runtime_apply_executed}'
= 'false'
AND receipt.value ->> 'durable_receipt' = 'true'
)
AND EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) closure_receipt(value)
WHERE closure_receipt.value ->> 'stage_id'
= 'incident_closure'
AND closure_receipt.value ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
AND closure_receipt.value #>> '{detail,apply_op_id}'
= apply.op_id::text
AND closure_receipt.value #>> '{detail,retry_op_id}'
= replay.op_id::text
AND closure_receipt.value #>> '{detail,terminal_type}'
= replay.output ->> 'terminal_disposition'
AND closure_receipt.value #>>
'{detail,no_write_terminal}' = 'true'
AND closure_receipt.value #>>
'{detail,proposal_executed}' = 'true'
AND closure_receipt.value #>>
'{detail,execution_success}' = 'false'
AND closure_receipt.value #>>
'{detail,telegram_receipt_acknowledged}' = 'true'
AND closure_receipt.value #>>
'{detail,repository_readback_verified}' = 'true'
AND closure_receipt.value ->> 'durable_receipt' = 'true'
)
AND EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = __INCIDENT_ID__
AND lifecycle.event_type::text = 'EXECUTION_COMPLETED'
AND lifecycle.success IS FALSE
AND lifecycle.context ->> 'apply_op_id'
= apply.op_id::text
AND lifecycle.context ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
)
)
OR EXISTS (
SELECT 1
FROM automation_operation_log failed_check
JOIN automation_operation_log replay_check
ON replay_check.parent_op_id = failed_check.parent_op_id
AND replay_check.operation_type = 'ansible_check_mode_executed'
AND replay_check.status IN ('success', 'failed')
AND replay_check.input ->> 'execution_mode'
= 'stdin_boundary_check_mode_replay'
AND failed_check.op_id = CAST(
NULLIF(
replay_check.input
->> 'replay_of_check_mode_op_id',
''
) AS uuid
)
JOIN automation_operation_log terminal
ON terminal.parent_op_id = replay_check.op_id
AND terminal.operation_type = 'ansible_execution_skipped'
AND terminal.status IN ('success', 'failed')
AND terminal.input ->> 'execution_mode'
= 'stdin_boundary_check_mode_replay_terminal'
AND terminal.output ->> 'runtime_apply_executed' = 'false'
WHERE failed_check.parent_op_id = __CANDIDATE__.op_id
AND failed_check.operation_type = 'ansible_check_mode_executed'
AND failed_check.status = 'failed'
AND failed_check.input ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
AND coalesce(
failed_check.incident_id::text,
failed_check.input ->> 'incident_id'
) = __INCIDENT_ID__
AND replay_check.input ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
AND coalesce(
replay_check.incident_id::text,
replay_check.input ->> 'incident_id'
) = __INCIDENT_ID__
AND terminal.input ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
AND terminal.input ->> 'retry_of_check_mode_op_id'
= failed_check.op_id::text
AND coalesce(
terminal.incident_id::text,
terminal.input ->> 'incident_id'
) = __INCIDENT_ID__
AND (
SELECT COUNT(DISTINCT receipt.value ->> 'stage_id')
FROM jsonb_array_elements(
coalesce(
terminal.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) receipt(value)
WHERE receipt.value ->> 'stage_id' IN (
'mcp_context',
'service_log_evidence',
'normalized_asset_identity',
'source_truth_diff',
'risk_policy_decision',
'executor_log_projection',
'retry_or_rollback',
'km_playbook_writeback',
'rag_writeback',
'playbook_trust',
'timeline_projection'
)
AND receipt.value ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
AND receipt.value ->> 'durable_receipt' = 'true'
) = 11
AND EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
terminal.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) receipt(value)
WHERE receipt.value ->> 'stage_id' = 'retry_or_rollback'
AND receipt.value #>>
'{detail,retry_terminal_op_id}'
= terminal.op_id::text
AND receipt.value #>>
'{detail,verified_no_write_terminal}' = 'true'
AND receipt.value #>>
'{detail,runtime_apply_executed}' = 'false'
AND receipt.value #>>
'{detail,repository_readback_verified}' = 'true'
AND receipt.value ->> 'durable_receipt' = 'true'
)
AND EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
terminal.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) receipt(value)
WHERE receipt.value ->> 'stage_id' = 'incident_closure'
AND receipt.value #>> '{detail,terminal_op_id}'
= terminal.op_id::text
AND receipt.value #>> '{detail,no_write_terminal}'
= 'true'
AND receipt.value #>>
'{detail,telegram_receipt_acknowledged}' = 'true'
AND receipt.value #>>
'{detail,learning_writeback_acknowledged}' = 'true'
AND receipt.value #>>
'{detail,incident_projection_readback_verified}'
= 'true'
AND receipt.value #>>
'{detail,projection_readback_verified}' = 'true'
AND receipt.value #>>
'{detail,repository_readback_verified}' = 'true'
AND receipt.value ->> 'durable_receipt' = 'true'
)
AND EXISTS (
SELECT 1
FROM awooop_outbound_message outbound
WHERE outbound.channel_type = 'telegram'
AND (
(
outbound.send_status = 'sent'
AND outbound.provider_message_id IS NOT NULL
)
OR (
outbound.send_status = 'shadow'
AND outbound.provider_message_id IS NULL
AND outbound.source_envelope #>>
'{notification_policy,disposition}'
= 'suppressed'
)
)
AND outbound.source_envelope #>>
'{callback_reply,action}'
= 'controlled_apply_result'
AND outbound.source_envelope #>>
'{callback_reply,execution_kind}'
= 'no_write_replay'
AND outbound.source_envelope #>>
'{callback_reply,automation_run_id}'
= __AUTOMATION_RUN_ID__
AND outbound.source_envelope #>>
'{callback_reply,incident_id}' = __INCIDENT_ID__
AND outbound.source_envelope #>>
'{callback_reply,apply_op_id}'
= terminal.op_id::text
)
AND EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = __INCIDENT_ID__
AND lifecycle.event_type::text = 'EXECUTION_COMPLETED'
AND lifecycle.success IS FALSE
AND lifecycle.context ->> 'apply_op_id'
= terminal.op_id::text
AND lifecycle.context ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
)
AND EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = __INCIDENT_ID__
AND (
lifecycle.event_type::text
= 'TELEGRAM_RESULT_SENT'
OR (
lifecycle.event_type::text
= 'NOTIFICATION_CLASSIFIED'
AND lifecycle.action_detail
= 'stdin_boundary_no_write_result_receipt_suppressed'
)
)
AND lifecycle.context ->> 'apply_op_id'
= terminal.op_id::text
AND lifecycle.context ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
)
)
"""
return (
template.replace("__CANDIDATE__", candidate)
.replace("__INCIDENT_ID__", incident_id)
.replace("__AUTOMATION_RUN_ID__", automation_run_id)
)
_CATALOG: tuple[dict[str, Any], ...] = (
{
"catalog_id": "ansible:awoooi-auto-repair-canary",
"playbook_path": "infra/ansible/playbooks/awoooi-auto-repair-canary.yml",
"check_mode_playbook_path": "infra/ansible/playbooks/awoooi-auto-repair-canary.yml",
"inventory_hosts": ["host_121"],
"domains": ["awoooi", "auto_repair", "k3s", "controlled_canary"],
"keywords": [
"awooopautorepaircanaryt16",
"awoooi-auto-repair-canary",
"auto repair canary",
"controlled canary",
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "low",
},
{
"catalog_id": "ansible:wazuh-manager-posture-readback",
"playbook_path": "infra/ansible/playbooks/wazuh-manager-posture-readback.yml",
"check_mode_playbook_path": "infra/ansible/playbooks/wazuh-manager-posture-readback.yml",
"inventory_hosts": ["host_112"],
"domains": ["wazuh", "siem", "manager_posture", "security"],
"keywords": [
"wazuh",
"wazuhmanager",
"wazuh-manager",
"siem",
"fim",
"security manager",
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "low",
"no_write_only": True,
},
{
"catalog_id": "ansible:wazuh-alertmanager-integration",
"playbook_path": (
"infra/ansible/playbooks/wazuh-alertmanager-integration.yml"
),
"check_mode_playbook_path": (
"infra/ansible/playbooks/wazuh-alertmanager-integration.yml"
),
"inventory_hosts": ["host_112"],
"domains": [
"wazuh",
"siem",
"security_event_ingress",
"alertmanager",
],
"keywords": [
"wazuhalertmanagerintegrationconvergence",
"wazuh alertmanager integration convergence",
"custom-awoooi-alertmanager",
"security event ingress",
],
"activation_keywords": [
"wazuhalertmanagerintegrationconvergence",
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "high",
"catalog_revision": "2026-07-15-wazuh-alert-ingress-v2",
},
{
"catalog_id": "ansible:110-alertmanager-delivery-recovery",
"playbook_path": (
"infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml"
),
"check_mode_playbook_path": (
"infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml"
),
"inventory_hosts": ["host_110"],
"domains": ["alert_chain_health", "alertmanager", "docker_container"],
"keywords": [
"alertchainbrokenalertmanager",
"alertchainbroken_alertmanager",
"alertmanager delivery",
],
"activation_keywords": ["alertchainbrokenalertmanager"],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "medium",
"canonical_asset_id": "container:host_110:alertmanager",
"catalog_revision": "2026-07-16-alertmanager-exact-container-v1",
},
{
"catalog_id": "ansible:111-ollama-fallback",
"playbook_path": "infra/ansible/playbooks/111-ollama-fallback.yml",
"check_mode_playbook_path": (
"infra/ansible/playbooks/111-ollama-fallback.yml"
),
"inventory_hosts": ["host_111", "host_120", "host_121"],
"domains": ["ai_provider", "ollama", "host_launchagent"],
"keywords": [
"ollamalocalunavailable",
"ollama111unavailable",
"ollama-local",
"ollama 111",
],
"activation_keywords": [
"ollamalocalunavailable",
"ollama111unavailable",
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "medium",
"canonical_asset_id": "ai-provider:ollama_local",
"catalog_revision": "2026-07-16-host111-launchagent-k3s-path-v2",
},
{
"catalog_id": "ansible:110-host-pressure-readonly",
"playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml",
"check_mode_playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml",
"inventory_hosts": ["host_110"],
"domains": ["host_pressure", "gitea", "runner", "postgres", "systemd_control_plane"],
"keywords": [
"host110sustainedmoderatepressure",
"host-pressure-controller",
"sustained pressure",
"host_resource",
"gitea_service",
"systemd_control_plane",
],
"activation_keywords": [
"host110sustainedmoderatepressure",
"host-pressure-controller",
"sustained pressure",
],
"supports_check_mode": True,
"auto_apply_enabled": False,
"approval_required": False,
"risk_level": "low",
"no_write_only": True,
"catalog_revision": "2026-07-14-host110-inputs-v2",
},
{
"catalog_id": "ansible:110-disk-pressure",
"playbook_path": (
"infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml"
),
"check_mode_playbook_path": (
"infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml"
),
"inventory_hosts": ["host_110"],
"domains": ["disk_pressure", "docker_artifacts", "host_110"],
"keywords": [
"hostoutofdiskspace",
"hostdiskusagehigh",
"hostdiskusagecritical",
"hostdiskwillfillin24hours",
"disk_full",
"node-exporter-110",
"192.168.0.110",
],
"activation_keyword_groups": [
[
"hostoutofdiskspace",
"hostdiskusagehigh",
"hostdiskusagecritical",
"hostdiskwillfillin24hours",
],
["node-exporter-110", "192.168.0.110", "host_110", "host110"],
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "medium",
"catalog_revision": "2026-07-14-disk-pressure-bounded-v1",
},
{
"catalog_id": "ansible:188-disk-pressure",
"playbook_path": (
"infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml"
),
"check_mode_playbook_path": (
"infra/ansible/playbooks/host-disk-pressure-bounded-cleanup.yml"
),
"inventory_hosts": ["host_188"],
"domains": ["disk_pressure", "docker_artifacts", "host_188"],
"keywords": [
"hostoutofdiskspace",
"hostdiskusagehigh",
"hostdiskusagecritical",
"hostdiskwillfillin24hours",
"disk_full",
"node-exporter-188",
"192.168.0.188",
],
"activation_keyword_groups": [
[
"hostoutofdiskspace",
"hostdiskusagehigh",
"hostdiskusagecritical",
"hostdiskwillfillin24hours",
],
["node-exporter-188", "192.168.0.188", "host_188", "host188"],
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "medium",
"catalog_revision": "2026-07-14-disk-pressure-bounded-v1",
},
{
"catalog_id": "ansible:110-devops",
"playbook_path": "infra/ansible/playbooks/110-node-exporter-repair.yml",
"check_mode_playbook_path": "infra/ansible/playbooks/110-node-exporter-repair.yml",
"inventory_hosts": ["host_110"],
"domains": ["monitoring", "prometheus", "node_exporter", "host_110"],
"keywords": [
"node-exporter-110",
"node exporter 110",
"nodeexporterdown",
"node_exporter",
"node-exporter",
],
"activation_keyword_groups": [
[
"nodeexporterdown",
"nodeexporterscrapedown",
"node_exporter_down",
],
["node-exporter-110", "192.168.0.110", "host_110", "host110"],
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "low",
"catalog_revision": "2026-07-14-node-exporter-bounded-v5",
},
{
"catalog_id": "ansible:110-sentry-profiling-consumer-recovery",
"playbook_path": (
"infra/ansible/playbooks/110-sentry-profiling-consumer-recovery.yml"
),
"check_mode_playbook_path": (
"infra/ansible/playbooks/110-sentry-profiling-consumer-recovery.yml"
),
"inventory_hosts": ["host_110"],
"domains": ["docker_container", "sentry", "host_110"],
"keywords": [
"dockercontainerunhealthy",
"sentry-self-hosted-snuba-profiling-functions-consumer-1",
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "medium",
"canonical_asset_id": "service:sentry",
"catalog_revision": "2026-07-15-sentry-exact-container-v1",
},
{
"catalog_id": "ansible:110-devops-full-convergence",
"playbook_path": "infra/ansible/playbooks/110-devops.yml",
"check_mode_playbook_path": "infra/ansible/playbooks/110-devops.yml",
"inventory_hosts": ["host_110"],
"domains": [
"swap",
"harbor",
"sentry",
"gitea",
"langfuse",
"bitan",
"runner",
"keepalived",
"nginx",
"backup",
],
"keywords": [
"110-devops-full-convergence",
"harbor",
"sentry",
"langfuse",
"bitan",
"keepalived",
"github-runner",
],
"supports_check_mode": True,
"auto_apply_enabled": False,
"approval_required": False,
"risk_level": "high",
"no_write_only": True,
"catalog_revision": "2026-07-14-full-convergence-quarantined-v1",
},
{
"catalog_id": "ansible:188-momo-backup-user",
"playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml",
"check_mode_playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml",
"inventory_hosts": ["host_188"],
"domains": ["momo_backup", "postgresql", "cron", "awooop_notification"],
"keywords": [
"188",
"momo",
"momopostgresbackupfailed",
"postgres",
"postgresql",
"backup",
"momo pg_backup",
"momo postgres backup",
"pg_backup",
"momo-pg-backup",
"cron",
"crontab",
],
"activation_keywords": [
"momo",
"momopostgresbackupfailed",
"backup",
"momo pg_backup",
"momo postgres backup",
"pg_backup",
"momo-pg-backup",
"cron",
"crontab",
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "low",
},
{
"catalog_id": "ansible:188-ai-web",
"playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
"check_mode_playbook_path": "infra/ansible/playbooks/188-ai-web-readonly.yml",
"inventory_hosts": ["host_188"],
"domains": ["docker", "momo_backup", "signoz", "minio", "litellm", "n8n", "open_webui", "nginx"],
"keywords": [
"188",
"docker",
"container",
"dockercontainerunhealthy",
"momo",
"backup",
"postgresql",
"pg_backup",
"signoz",
"minio",
"litellm",
"n8n",
"open-webui",
"openwebui",
"docker-registry",
],
"activation_keywords": [
"dockercontainerunhealthy",
"signoz",
"minio",
"litellm",
"n8n",
"open-webui",
"openwebui",
"docker-registry",
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "medium",
},
{
"catalog_id": "ansible:nginx-sync",
"playbook_path": "infra/ansible/playbooks/nginx-sync.yml",
"check_mode_playbook_path": "infra/ansible/playbooks/nginx-sync-readonly.yml",
"inventory_hosts": ["host_188"],
"domains": ["nginx", "proxy", "tls"],
"keywords": ["nginx", "proxy", "tls", "cert", "502", "upstream"],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "high",
},
{
"catalog_id": "ansible:restore-password-auth",
"playbook_path": "infra/ansible/playbooks/restore-password-auth.yml",
"inventory_hosts": ["host_110", "host_120", "host_121", "host_188"],
"domains": ["ssh", "password_auth"],
"keywords": ["ssh", "passwordauthentication", "password auth", "login", "auth"],
"supports_check_mode": False,
"auto_apply_enabled": False,
"approval_required": True,
"risk_level": "high",
"break_glass_required": True,
},
)
def get_ansible_catalog_item(catalog_id: str) -> dict[str, Any] | None:
"""Return one repo-known Ansible catalog item without exposing mutability."""
for item in _CATALOG:
if item["catalog_id"] == catalog_id:
return dict(item)
return None
def list_ansible_catalog() -> tuple[dict[str, Any], ...]:
"""Return public-safe copies of the controlled Ansible catalog rows."""
return tuple(dict(row) for row in _CATALOG)
def _get(row: dict[str, Any], key: str) -> Any:
return row.get(key)
def _tags(row: dict[str, Any]) -> list[str]:
raw = _get(row, "tags")
if isinstance(raw, list):
return [str(item).lower() for item in raw]
if isinstance(raw, str):
return [part.strip().lower() for part in raw.split(",") if part.strip()]
return []
def _first_present(row: dict[str, Any], keys: tuple[str, ...]) -> Any:
for key in keys:
value = _get(row, key)
if value not in (None, ""):
return value
return None
def _json_object(row: dict[str, Any], key: str) -> dict[str, Any]:
raw = _get(row, key)
if isinstance(raw, dict):
return raw
if isinstance(raw, str) and raw.strip():
try:
parsed = json.loads(raw)
except json.JSONDecodeError:
return {}
return parsed if isinstance(parsed, dict) else {}
return {}
def _json_value(row: dict[str, Any], container_key: str, value_key: str) -> Any:
return _json_object(row, container_key).get(value_key)
def _first_present_with_json(row: dict[str, Any], keys: tuple[str, ...], json_keys: tuple[tuple[str, str], ...] = ()) -> Any:
value = _first_present(row, keys)
if value not in (None, ""):
return value
for container_key, value_key in json_keys:
value = _json_value(row, container_key, value_key)
if value not in (None, ""):
return value
return None
def _bool_or_none(value: Any) -> bool | None:
if isinstance(value, bool):
return value
if value in (None, ""):
return None
normalized = str(value).strip().lower()
if normalized in {"1", "true", "yes", "y", "on"}:
return True
if normalized in {"0", "false", "no", "n", "off"}:
return False
return None
def _int_or_none(value: Any) -> int | None:
if value in (None, ""):
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _is_controlled_apply_record(row: dict[str, Any]) -> bool:
if not row:
return False
if row.get("approval_source"):
return True
if str(row.get("execution_mode") or "").strip().lower() == "controlled_apply":
return True
if "controlled_apply" in _tags(row):
return True
return str(row.get("actor") or "").strip().lower() == "ansible_controlled_apply_worker"
def _is_ansible_operation(row: dict[str, Any]) -> bool:
operation_type = str(_get(row, "operation_type") or "").lower()
if operation_type in ANSIBLE_OPERATION_TYPES:
return True
if "ansible" in _tags(row):
return True
executor = str(
_first_present(
row,
(
"input_executor",
"input_execution_backend",
"output_executor",
"output_execution_backend",
),
)
or ""
).lower()
if executor == "ansible":
return True
playbook_path = str(
_first_present(row, ("input_playbook_path", "output_playbook_path", "input_ansible_playbook_path", "output_ansible_playbook_path"))
or ""
).lower()
return "infra/ansible/" in playbook_path or playbook_path.endswith(".yml") and "ansible" in playbook_path
def _ansible_record(row: dict[str, Any]) -> dict[str, Any]:
return {
"op_id": _get(row, "op_id"),
"parent_op_id": _get(row, "parent_op_id"),
"operation_type": _get(row, "operation_type"),
"status": _get(row, "status"),
"actor": _get(row, "actor"),
"catalog_id": _first_present(row, ("input_catalog_id", "output_catalog_id")),
"playbook_id": _first_present(row, ("input_playbook_id", "output_playbook_id")),
"playbook_path": _first_present(
row,
("input_playbook_path", "output_playbook_path", "input_ansible_playbook_path", "output_ansible_playbook_path"),
),
"execution_mode": _first_present(row, ("input_execution_mode", "output_execution_mode")),
"check_mode": _first_present(row, ("input_check_mode", "output_check_mode")),
"check_mode_executed": _first_present_with_json(
row,
("input_check_mode_executed", "output_check_mode_executed", "dry_run_check_mode_executed"),
(("dry_run_result", "check_mode_executed"),),
),
"apply_enabled": _first_present(row, ("input_apply_enabled", "output_apply_enabled")),
"apply_executed": _first_present_with_json(
row,
("input_apply_executed", "output_apply_executed", "dry_run_apply_executed"),
(("dry_run_result", "apply_executed"),),
),
"approval_source": _first_present(row, ("input_approval_source", "output_approval_source")),
"returncode": _first_present_with_json(
row,
("output_returncode", "input_returncode", "dry_run_returncode"),
(("dry_run_result", "returncode"),),
),
"not_used_reason": _first_present(row, ("input_not_used_reason", "output_not_used_reason")),
"dry_run_result": _get(row, "dry_run_result"),
"error": _get(row, "error"),
"duration_ms": _get(row, "duration_ms"),
"tags": _get(row, "tags"),
"created_at": _get(row, "created_at"),
}
def summarize_ansible_execution(records: list[dict[str, Any]]) -> dict[str, Any]:
"""Summarize durable Ansible audit rows for Telegram and Operator UI."""
typed_records = [row for row in records if isinstance(row, dict)]
check_mode_total = 0
apply_total = 0
rollback_total = 0
pending_check_mode_total = 0
applied_success_total = 0
terminal_check_mode_parent_ids = {
str(row.get("parent_op_id"))
for row in typed_records
if str(row.get("operation_type") or "") in {
"ansible_check_mode_executed",
"ansible_execution_skipped",
}
and row.get("parent_op_id")
}
for row in typed_records:
operation_type = str(row.get("operation_type") or "")
status = str(row.get("status") or "").lower()
if operation_type == "ansible_check_mode_executed" and status != "pending":
check_mode_total += 1
elif operation_type == "ansible_apply_executed":
apply_total += 1
if status in {"success", "completed", "executed", "applied"}:
applied_success_total += 1
elif operation_type == "ansible_rollback_executed":
rollback_total += 1
elif (
operation_type == "ansible_candidate_matched"
and str(row.get("op_id")) not in terminal_check_mode_parent_ids
):
pending_check_mode_total += 1
latest_record = typed_records[0] if typed_records else {}
latest_apply = next(
(row for row in typed_records if row.get("operation_type") == "ansible_apply_executed"),
{},
)
latest_check = next(
(row for row in typed_records if row.get("operation_type") == "ansible_check_mode_executed"),
{},
)
focused = latest_apply or latest_check or latest_record
returncode = _int_or_none(focused.get("returncode"))
approval_source = focused.get("approval_source")
return {
"check_mode_total": check_mode_total,
"apply_total": apply_total,
"rollback_total": rollback_total,
"pending_check_mode_total": pending_check_mode_total,
"applied_success_total": applied_success_total,
"applied": applied_success_total > 0,
"controlled_apply": bool(latest_apply) and _is_controlled_apply_record(latest_apply),
"latest_operation_type": focused.get("operation_type"),
"latest_status": focused.get("status"),
"latest_catalog_id": focused.get("catalog_id"),
"latest_playbook_path": focused.get("playbook_path"),
"latest_execution_mode": focused.get("execution_mode"),
"latest_check_mode": _bool_or_none(focused.get("check_mode")),
"latest_check_mode_executed": _bool_or_none(focused.get("check_mode_executed")),
"latest_apply_enabled": _bool_or_none(focused.get("apply_enabled")),
"latest_apply_executed": _bool_or_none(focused.get("apply_executed")),
"latest_returncode": returncode if returncode is not None else focused.get("returncode"),
"approval_source": approval_source,
"latest_actor": focused.get("actor"),
"latest_op_id": focused.get("op_id"),
"latest_parent_op_id": focused.get("parent_op_id"),
}
def _flatten_text(value: Any, pieces: list[str], remaining: int = 80) -> int:
if remaining <= 0 or value is None:
return remaining
if isinstance(value, dict):
for key, item in value.items():
remaining = _flatten_text(key, pieces, remaining)
remaining = _flatten_text(item, pieces, remaining)
if remaining <= 0:
break
return remaining
if isinstance(value, list):
for item in value:
remaining = _flatten_text(item, pieces, remaining)
if remaining <= 0:
break
return remaining
pieces.append(str(value).lower())
return remaining - 1
def _source_haystack(incident: dict[str, Any] | None, drift: dict[str, Any] | None) -> str:
pieces: list[str] = []
_flatten_text(incident, pieces)
_flatten_text(drift, pieces)
return " ".join(pieces)
def _domain_controlled_executor_route(
incident: dict[str, Any] | None,
) -> dict[str, Any] | None:
"""Keep host control-plane alerts out of the generic Ansible catalog."""
if not isinstance(incident, dict):
return None
signals = [
signal
for signal in incident.get("signals") or []
if isinstance(signal, dict)
]
first_signal = signals[0] if signals else {}
labels = (
first_signal.get("labels")
if isinstance(first_signal.get("labels"), dict)
else {}
)
alertname = str(
incident.get("alertname")
or first_signal.get("alert_name")
or labels.get("alertname")
or ""
)
affected_services = [
str(value)
for value in incident.get("affected_services") or []
if str(value).strip()
]
target_resource = str(
affected_services[0]
if affected_services
else labels.get("component")
or labels.get("target_resource")
or ""
)
namespace = str(labels.get("namespace") or "")
return resolve_controlled_alert_target(
alertname=alertname,
target_resource=target_resource,
namespace=namespace,
)
def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None) -> dict[str, Any]:
typed_route = resolve_typed_incident_target(incident)
allowed_catalog_ids = {
str(value)
for value in typed_route.get("allowed_catalog_ids") or []
if str(value)
}
candidates: list[dict[str, Any]] = []
unmatched: list[str] = []
for item in _CATALOG:
catalog_id = str(item["catalog_id"])
if catalog_id not in allowed_catalog_ids:
unmatched.append(catalog_id)
continue
public_item = {
key: value
for key, value in item.items()
if key
in {
"catalog_id",
"playbook_path",
"inventory_hosts",
"domains",
"supports_check_mode",
"check_mode_playbook_path",
"auto_apply_enabled",
"approval_required",
"risk_level",
"no_write_only",
"catalog_revision",
}
}
allowed_hosts = {
str(value)
for value in typed_route.get("allowed_inventory_hosts") or []
if str(value)
}
catalog_hosts = {
str(value)
for value in item.get("inventory_hosts") or []
if str(value)
}
if not catalog_hosts or catalog_hosts != allowed_hosts:
unmatched.append(catalog_id)
continue
candidates.append(
{
**public_item,
"match_score": 100,
"matched_keywords": ["typed_domain_exact_match"],
"semantic_match": True,
"matched_activation_keywords": [],
"matched_activation_keyword_groups": [],
"canonical_asset_id": typed_route.get("canonical_asset_id"),
"typed_domain": typed_route.get("target_kind"),
"target_scope_validated": True,
}
)
candidates.sort(key=lambda row: str(row["catalog_id"]))
resolution_status = str(typed_route.get("resolution_status") or "")
if resolution_status == "asset_identity_unresolved":
decision_effect = "create_asset_drift_work_item"
not_used_reason = "asset_identity_unresolved_no_fallback"
elif typed_route.get("executor") != "host_ansible_executor":
decision_effect = "delegate_to_domain_executor"
not_used_reason = "typed_non_ansible_domain_route"
elif not candidates:
decision_effect = "create_domain_playbook_work_item"
not_used_reason = "typed_domain_has_no_exact_bounded_playbook"
else:
decision_effect = "bounded_domain_candidate"
not_used_reason = None
result = {
"match_mode": "typed_domain_router_v2",
"decision_effect": decision_effect,
"available_count": len(_CATALOG),
"candidates": candidates,
"unmatched_catalog_ids": sorted(set(unmatched)),
"typed_target_route": typed_route,
"cross_domain_fallback_allowed": False,
}
if not_used_reason:
result["not_used_reason"] = not_used_reason
if decision_effect == "delegate_to_domain_executor":
result["controlled_executor"] = typed_route
if resolution_status == "asset_identity_unresolved":
result["drift_work_item_id"] = typed_route.get("drift_work_item_id")
return result
def _legacy_catalog_hints(
incident: dict[str, Any] | None,
drift: dict[str, Any] | None,
) -> dict[str, Any]:
"""Retained for offline comparison only; runtime routing uses v2 above."""
controlled_route = _domain_controlled_executor_route(incident)
if controlled_route is not None:
return {
"match_mode": "domain_controlled_executor_route_v1",
"decision_effect": "delegate_to_domain_executor",
"available_count": len(_CATALOG),
"candidates": [],
"unmatched_catalog_ids": [str(item["catalog_id"]) for item in _CATALOG],
"controlled_executor": controlled_route,
"not_used_reason": "non_kubernetes_control_plane_route",
}
haystack = _source_haystack(incident, drift)
candidates: list[dict[str, Any]] = []
unmatched: list[str] = []
for item in _CATALOG:
matched = [keyword for keyword in item["keywords"] if keyword in haystack]
activation_keywords = item.get("activation_keywords") or []
matched_activation_keywords = [
keyword for keyword in activation_keywords if keyword in haystack
]
activation_keyword_groups = item.get("activation_keyword_groups") or []
matched_activation_keyword_groups = [
[keyword for keyword in group if keyword in haystack]
for group in activation_keyword_groups
]
semantic_match = (
(not activation_keywords or bool(matched_activation_keywords))
and all(matched_activation_keyword_groups)
)
public_item = {
key: value
for key, value in item.items()
if key
in {
"catalog_id",
"playbook_path",
"inventory_hosts",
"domains",
"supports_check_mode",
"check_mode_playbook_path",
"auto_apply_enabled",
"approval_required",
"risk_level",
"no_write_only",
}
}
if matched and semantic_match:
candidates.append({
**public_item,
"match_score": len(matched),
"matched_keywords": matched,
"semantic_match": True,
"matched_activation_keywords": matched_activation_keywords,
"matched_activation_keyword_groups": (
matched_activation_keyword_groups
),
})
else:
unmatched.append(item["catalog_id"])
candidates.sort(key=lambda row: (-int(row["match_score"]), str(row["catalog_id"])))
return {
"match_mode": "static_catalog_keyword_hint_v1",
"decision_effect": "none",
"available_count": len(_CATALOG),
"candidates": candidates,
"unmatched_catalog_ids": unmatched,
}
def get_ansible_catalog_candidates(
incident: dict[str, Any] | None,
drift: dict[str, Any] | None = None,
) -> list[dict[str, Any]]:
"""Return current semantically matched catalog rows for replay routing."""
hints = _catalog_hints(incident, drift)
candidates = hints.get("candidates")
return [dict(row) for row in candidates or [] if isinstance(row, dict)]
def build_ansible_truth(
automation_ops: list[dict[str, Any]],
*,
incident: dict[str, Any] | None,
drift: dict[str, Any] | None,
) -> dict[str, Any]:
"""Build the truth-chain Ansible section from audited facts and catalog hints."""
records = [_ansible_record(row) for row in automation_ops if _is_ansible_operation(row)]
summary = summarize_ansible_execution(records)
candidate_catalog = _catalog_hints(incident, drift)
return {
"considered": bool(records),
"records": records,
"summary": summary,
"audit_contract": {
"schema_version": "ansible_executor_audit_v1",
"operation_types": sorted(ANSIBLE_OPERATION_TYPES),
"required_audit_fields": [
"incident_id",
"operation_type",
"status",
"actor",
"input.executor",
"input.catalog_id",
"input.execution_mode",
"input.playbook_path",
"input.check_mode",
"input.approval_source",
"output.returncode",
"output.not_used_reason",
"dry_run_result",
],
"default_execution_mode": "catalog/dry-run audit only until approval execution is explicitly wired",
},
"candidate_catalog": candidate_catalog,
"not_used_reason": (
None
if records
else candidate_catalog.get("not_used_reason")
or "no automation_operation_log row with Ansible operation type, tag, or executor backend for this source"
),
}
def _incident_public_dict(incident: Any) -> dict[str, Any]:
if incident is None:
return {}
if isinstance(incident, dict):
return incident
severity = getattr(incident, "severity", None)
signals_payload: list[dict[str, Any]] = []
for signal in getattr(incident, "signals", None) or []:
signals_payload.append({
"alert_name": getattr(signal, "alert_name", None),
"labels": getattr(signal, "labels", None) or {},
"annotations": getattr(signal, "annotations", None) or {},
})
return {
"incident_id": getattr(incident, "incident_id", None),
"project_id": getattr(incident, "project_id", None),
"status": getattr(getattr(incident, "status", None), "value", None),
"alertname": getattr(incident, "alertname", None),
"alert_category": getattr(incident, "alert_category", None),
"notification_type": getattr(incident, "notification_type", None),
"severity": getattr(severity, "value", severity),
"affected_services": getattr(incident, "affected_services", None) or [],
"signals": signals_payload,
}
_CANONICAL_INCIDENT_ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,30}$")
_CANONICAL_PROJECT_ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,64}$")
def _canonical_incident_status(value: Any) -> str:
normalized = str(getattr(value, "value", value) or "").strip().upper()
return (
normalized
if normalized
in {"INVESTIGATING", "MITIGATING", "RESOLVED", "CLOSED", "ESCALATED"}
else "INVESTIGATING"
)
def _canonical_incident_severity(value: Any) -> str:
normalized = str(getattr(value, "value", value) or "").strip().upper()
return {
"CRITICAL": "P0",
"HIGH": "P1",
"WARNING": "P2",
"MEDIUM": "P2",
"LOW": "P3",
"INFO": "P3",
}.get(
normalized,
normalized if normalized in {"P0", "P1", "P2", "P3"} else "P2",
)
def _canonical_incident_ledger_params(
incident_payload: dict[str, Any],
*,
automation_run_id: str,
) -> dict[str, Any] | None:
incident_id = str(incident_payload.get("incident_id") or "").strip()
project_id = str(incident_payload.get("project_id") or "awoooi").strip()
if (
_CANONICAL_INCIDENT_ID_RE.fullmatch(incident_id) is None
or _CANONICAL_PROJECT_ID_RE.fullmatch(project_id) is None
):
return None
signals = [
signal
for signal in incident_payload.get("signals") or []
if isinstance(signal, dict)
][:20]
affected_services = [
str(value)[:100]
for value in incident_payload.get("affected_services") or []
if str(value).strip()
][:50]
notification_type = str(
incident_payload.get("notification_type") or ""
).strip()
if len(notification_type) > 10:
notification_type = ""
decision_chain = {
"schema_version": "ansible_candidate_incident_ingress_v1",
"trace_id": str(
incident_payload.get("trace_id") or automation_run_id
),
"run_id": str(incident_payload.get("run_id") or automation_run_id),
"work_item_id": str(incident_payload.get("work_item_id") or ""),
"source_receipt_ref": str(
incident_payload.get("source_receipt_ref") or ""
),
"asset_scope_aliases": [
str(value)[:100]
for value in incident_payload.get("asset_scope_aliases") or []
if str(value).strip()
][:20],
"normalized_asset_identity": True,
"candidate_runtime_apply_executed": False,
}
return {
"incident_id": incident_id,
"project_id": project_id,
"status": _canonical_incident_status(incident_payload.get("status")),
"severity": _canonical_incident_severity(
incident_payload.get("severity")
),
"signals": json.dumps(signals, ensure_ascii=False),
"affected_services": json.dumps(
affected_services,
ensure_ascii=False,
),
"decision_chain": json.dumps(decision_chain, ensure_ascii=False),
"alertname": str(incident_payload.get("alertname") or "")[:100]
or None,
"notification_type": notification_type or None,
"alert_category": str(
incident_payload.get("alert_category") or ""
)[:50]
or None,
}
async def _ensure_ansible_incident_ledger_with_db(
db: Any,
incident_payload: dict[str, Any],
*,
automation_run_id: str,
) -> dict[str, bool]:
params = _canonical_incident_ledger_params(
incident_payload,
automation_run_id=automation_run_id,
)
if params is None:
return {"ready": False, "inserted": False}
result = await db.execute(
text("""
WITH inserted AS (
INSERT INTO incidents (
incident_id,
project_id,
status,
severity,
signals,
affected_services,
decision_chain,
proposal_ids,
alertname,
notification_type,
alert_category,
created_at,
updated_at,
ttl_days,
vectorized
) VALUES (
:incident_id,
:project_id,
CAST(:status AS incidentstatus),
CAST(:severity AS severity),
CAST(:signals AS json),
CAST(:affected_services AS json),
CAST(:decision_chain AS json),
CAST('[]' AS json),
:alertname,
:notification_type,
:alert_category,
NOW(),
NOW(),
7,
FALSE
)
ON CONFLICT (incident_id) DO NOTHING
RETURNING incident_id
)
SELECT
EXISTS (SELECT 1 FROM inserted) AS inserted,
EXISTS (
SELECT 1
FROM incidents incident
WHERE incident.incident_id = :incident_id
AND incident.project_id = :project_id
) AS ready
"""),
params,
)
row = result.mappings().one_or_none()
return {
"ready": bool(row and row.get("ready") is True),
"inserted": bool(row and row.get("inserted") is True),
}
async def ensure_ansible_incident_ledger(
*,
incident: Any,
automation_run_id: str,
project_id: str | None = None,
) -> dict[str, bool]:
"""Ensure a controlled candidate has a durable canonical incident row."""
incident_payload = dict(_incident_public_dict(incident))
if project_id:
incident_payload["project_id"] = project_id
effective_project_id = str(
incident_payload.get("project_id") or "awoooi"
)
async with get_db_context(effective_project_id) as db:
return await _ensure_ansible_incident_ledger_with_db(
db,
incident_payload,
automation_run_id=automation_run_id,
)
def build_ansible_decision_audit_payload(
*,
incident: Any,
proposal_data: dict[str, Any],
decision_path: str,
not_used_reason: str,
) -> dict[str, Any] | None:
"""Return an AOL payload when Ansible has catalog candidates for a decision."""
incident_payload = _incident_public_dict(incident)
hints = _catalog_hints(incident_payload, None)
candidates = hints.get("candidates") or []
if not candidates:
return None
incident_id = str(incident_payload.get("incident_id") or "")
project_id = str(incident_payload.get("project_id") or "awoooi")
affected_services = [
str(value)
for value in incident_payload.get("affected_services") or []
if str(value).strip()
]
namespaces = sorted({
str((signal.get("labels") or {}).get("namespace") or "").strip()
for signal in incident_payload.get("signals") or []
if isinstance(signal, dict)
and str((signal.get("labels") or {}).get("namespace") or "").strip()
})
proposal_risk_level = str(proposal_data.get("risk_level") or "").strip().lower()
execution_priority = proposal_data.get("execution_priority", 100)
if not isinstance(execution_priority, int) or isinstance(execution_priority, bool):
execution_priority = 100
execution_priority = max(0, min(execution_priority, 100))
typed_target_route = hints["typed_target_route"]
executor_candidates = [
{
"catalog_id": row["catalog_id"],
"playbook_path": row["playbook_path"],
"check_mode_playbook_path": row.get("check_mode_playbook_path"),
"inventory_hosts": row["inventory_hosts"],
"supports_check_mode": row["supports_check_mode"],
"auto_apply_enabled": row["auto_apply_enabled"],
"approval_required": row["approval_required"],
"risk_level": row["risk_level"],
"catalog_revision": row.get("catalog_revision"),
"canonical_asset_id": row.get("canonical_asset_id"),
"typed_domain": row.get("typed_domain"),
"target_scope_validated": row.get("target_scope_validated"),
"match_score": row["match_score"],
"matched_keywords": row["matched_keywords"],
"semantic_match": row["semantic_match"],
"matched_activation_keywords": row[
"matched_activation_keywords"
],
}
for row in candidates[:5]
]
target_route_generation = build_typed_target_route_generation(
typed_target_route=typed_target_route,
executor_candidates=executor_candidates,
)
source_recurrence = _source_recurrence_receipt(
incident_payload,
proposal_data,
)
occurrence_identity = "|".join(
str(source_recurrence.get(key) or "")
for key in ("fingerprint", "occurrence_id", "observed_at")
)
occurrence_generation = hashlib.sha256(
occurrence_identity.encode("utf-8")
).hexdigest()[:20]
idempotency_key = (
f"ansible-candidate:{project_id}:{incident_id}:"
f"{target_route_generation}:{occurrence_generation}"
)
input_payload = {
"incident_id": incident_id,
"project_id": project_id,
"trace_id": str(incident_payload.get("trace_id") or ""),
"run_id": str(incident_payload.get("run_id") or ""),
"work_item_id": str(incident_payload.get("work_item_id") or ""),
"run_namespace": str(incident_payload.get("run_namespace") or ""),
"source_receipt_ref": str(incident_payload.get("source_receipt_ref") or ""),
"asset_scope_aliases": [
str(alias)
for alias in incident_payload.get("asset_scope_aliases") or []
if isinstance(alias, str) and alias
][:20],
"executor": "ansible",
"execution_backend": "ansible",
"single_writer_executor": "awoooi-ansible-executor-broker",
"execution_priority": execution_priority,
"decision_path": decision_path,
"idempotency_key": idempotency_key,
"candidate_lane_lock_key": f"ansible-candidate:{project_id}:{incident_id}",
"target_route_generation": target_route_generation,
"source_occurrence_generation": occurrence_generation,
"source_recurrence": source_recurrence,
"router_source_sha": os.getenv("AWOOOI_BUILD_COMMIT_SHA", "")
.strip()
.lower(),
"check_mode": True,
"apply_enabled": False,
"approval_required": proposal_risk_level == "critical",
"candidate_catalog_schema": hints["match_mode"],
"typed_target_route": typed_target_route,
"cross_domain_fallback_allowed": False,
"executor_candidates": executor_candidates,
"proposal_source": proposal_data.get("source", ""),
"approval_id": str(proposal_data.get("approval_id") or ""),
"proposal_risk_level": proposal_risk_level,
"proposal_action_preview": str(
proposal_data.get("action")
or proposal_data.get("kubectl_command")
or ""
)[:240],
"target_selector": {
"schema_version": "ai_decision_target_selector_v2",
"project_id": project_id,
"incident_id": incident_id,
"canonical_asset_id": typed_target_route.get("canonical_asset_id"),
"typed_domain": typed_target_route.get("target_kind"),
"resolution_status": typed_target_route.get("resolution_status"),
"affected_services": affected_services,
"namespaces": namespaces,
"catalog_ids": [str(row["catalog_id"]) for row in candidates[:5]],
"inventory_hosts": sorted({
str(host)
for row in candidates[:5]
for host in row.get("inventory_hosts") or []
}),
"cross_domain_fallback_allowed": False,
},
"source_truth_diff": {
"required_before_apply": True,
"producer": "awooop_ansible_check_mode_worker",
"receipt_status": "pending_check_mode",
},
"risk_policy_decision": {
"risk_level": proposal_risk_level,
"controlled_apply_candidate": proposal_risk_level in {"low", "medium", "high"},
"critical_break_glass_required": proposal_risk_level == "critical",
},
}
controlled_queue = decision_path == "repair_candidate_controlled_queue"
output_payload = {
"not_used_reason": not_used_reason,
"decision_effect": "check_mode_queue_ready" if controlled_queue else "audit_only",
"next_required_step": (
"awooop_ansible_check_mode_worker_claims_candidate"
if controlled_queue
else "wire approval_execution to Ansible check-mode before apply"
),
}
return {
"operation_type": "ansible_candidate_matched",
"status": "dry_run",
"input": input_payload,
"output": output_payload,
"dry_run_result": {
"check_mode_executed": False,
"candidate_count": len(candidates),
"reason": not_used_reason,
},
"tags": ["ansible", "decision", "candidate", "check_mode_pending"],
}
async def preflight_ansible_candidate_generation(
*,
incident: Any,
proposal_data: dict[str, Any],
decision_path: str,
not_used_reason: str,
project_id: str = "awoooi",
) -> dict[str, Any]:
"""Avoid expensive evidence collection for an already-current generation."""
payload = build_ansible_decision_audit_payload(
incident=incident,
proposal_data=proposal_data,
decision_path=decision_path,
not_used_reason=not_used_reason,
)
if payload is None:
return {
"status": "no_allowlisted_ansible_candidate",
"should_collect_evidence": False,
}
incident_id = str(payload["input"].get("incident_id") or "")
generation = str(
payload["input"].get("target_route_generation") or ""
)
occurrence = str(
payload["input"].get("source_occurrence_generation") or ""
)
source_recurrence = payload["input"].get("source_recurrence") or {}
proposal_source = str(payload["input"].get("proposal_source") or "")
if (
proposal_source in _RECURRENCE_REQUIRED_PROPOSAL_SOURCES
and source_recurrence.get("verified") is not True
):
return {
"status": "current_source_recurrence_not_verified",
"should_collect_evidence": False,
"target_route_generation": generation,
}
terminal_sql = verified_ansible_candidate_terminal_sql("candidate")
try:
async with get_db_context(project_id) as db:
result = await db.execute(
text(f"""
SELECT
candidate.op_id::text AS op_id,
candidate.input,
({terminal_sql}) AS is_terminal,
EXISTS (
SELECT 1
FROM automation_operation_log child
WHERE child.parent_op_id = candidate.op_id
AND child.status IN ('pending', 'running')
) OR EXISTS (
SELECT 1
FROM automation_operation_log check_mode
JOIN automation_operation_log apply
ON apply.parent_op_id = check_mode.op_id
AND apply.operation_type = 'ansible_apply_executed'
AND apply.status IN ('pending', 'running')
WHERE check_mode.parent_op_id = candidate.op_id
AND check_mode.operation_type
= 'ansible_check_mode_executed'
) AS has_pending_child,
EXISTS (
SELECT 1
FROM automation_operation_log check_mode
JOIN automation_operation_log apply
ON apply.parent_op_id = check_mode.op_id
AND apply.operation_type = 'ansible_apply_executed'
WHERE check_mode.parent_op_id = candidate.op_id
AND check_mode.operation_type
= 'ansible_check_mode_executed'
) AS has_apply
FROM automation_operation_log candidate
WHERE candidate.operation_type = 'ansible_candidate_matched'
AND coalesce(
candidate.incident_id::text,
candidate.input ->> 'incident_id'
) = :incident_id
AND candidate.input ->> 'executor' = 'ansible'
AND candidate.input ->> 'decision_path'
= 'repair_candidate_controlled_queue'
ORDER BY candidate.created_at DESC, candidate.op_id DESC
LIMIT 1
"""),
{"incident_id": incident_id},
)
latest = result.mappings().first()
except Exception as exc:
logger.warning(
"ansible_candidate_generation_preflight_failed",
incident_id=incident_id,
error_type=type(exc).__name__,
)
return {
"status": "generation_preflight_unavailable",
"should_collect_evidence": False,
}
if latest is None:
return {
"status": "missing_candidate_generation",
"should_collect_evidence": True,
"target_route_generation": generation,
}
latest_input = _json_object(dict(latest), "input")
same_generation = str(
latest_input.get("target_route_generation") or ""
) == generation
same_occurrence = str(
latest_input.get("source_occurrence_generation") or ""
) == occurrence
is_terminal = latest.get("is_terminal") is True
if same_generation and (not is_terminal or same_occurrence):
return {
"status": "existing_current_generation",
"should_collect_evidence": False,
"target_route_generation": generation,
"existing_candidate_op_id": latest.get("op_id"),
}
if not is_terminal and latest.get("has_apply") is True:
return {
"status": "generation_reconciliation_blocked_has_apply",
"should_collect_evidence": False,
"target_route_generation": generation,
"existing_candidate_op_id": latest.get("op_id"),
}
if not is_terminal and latest.get("has_pending_child") is True:
return {
"status": "generation_reconciliation_deferred_pending_child",
"should_collect_evidence": False,
"target_route_generation": generation,
"existing_candidate_op_id": latest.get("op_id"),
}
return {
"status": (
"fresh_occurrence_same_generation"
if same_generation
else "generation_reconciliation_required"
),
"should_collect_evidence": True,
"target_route_generation": generation,
"existing_candidate_op_id": latest.get("op_id"),
}
async def record_ansible_decision_audit(
*,
incident: Any,
proposal_data: dict[str, Any],
decision_path: str,
not_used_reason: str,
automation_run_id: str | None = None,
) -> bool:
"""Write a best-effort Ansible candidate audit row for one decision."""
payload = build_ansible_decision_audit_payload(
incident=incident,
proposal_data=proposal_data,
decision_path=decision_path,
not_used_reason=not_used_reason,
)
if payload is None:
return False
incident_id = payload["input"]["incident_id"]
project_id = str(payload["input"].get("project_id") or "awoooi")
try:
candidate_op_id = str(UUID(automation_run_id)) if automation_run_id else str(uuid4())
except (TypeError, ValueError):
candidate_op_id = str(uuid4())
payload["input"]["automation_run_id"] = candidate_op_id
incident_payload = dict(_incident_public_dict(incident))
incident_payload["project_id"] = project_id
terminal_candidate_sql = verified_ansible_candidate_terminal_sql("candidate")
target_route_generation = str(
payload["input"].get("target_route_generation") or ""
)
source_recurrence = payload["input"].get("source_recurrence") or {}
proposal_source = str(payload["input"].get("proposal_source") or "")
if (
proposal_source in _RECURRENCE_REQUIRED_PROPOSAL_SOURCES
and source_recurrence.get("verified") is not True
):
logger.warning(
"ansible_candidate_write_blocked",
incident_id=incident_id,
reason="current_source_recurrence_not_verified",
)
return False
try:
async with get_db_context(str(project_id)) as db:
await db.execute(
text("""
SELECT pg_advisory_xact_lock(
hashtextextended(:idempotency_key, 0)
)
"""),
{
"idempotency_key": payload["input"][
"candidate_lane_lock_key"
]
},
)
incident_ledger = await _ensure_ansible_incident_ledger_with_db(
db,
incident_payload,
automation_run_id=candidate_op_id,
)
if incident_ledger["ready"] is not True:
logger.warning(
"ansible_candidate_incident_ledger_missing",
incident_id=incident_id,
project_id=project_id,
automation_run_id=candidate_op_id,
)
return False
latest_result = await db.execute(
text(f"""
SELECT
candidate.op_id::text AS op_id,
candidate.input,
({terminal_candidate_sql}) AS is_terminal,
EXISTS (
SELECT 1
FROM automation_operation_log child
WHERE child.parent_op_id = candidate.op_id
AND child.status IN ('pending', 'running')
) OR EXISTS (
SELECT 1
FROM automation_operation_log check_mode
JOIN automation_operation_log apply
ON apply.parent_op_id = check_mode.op_id
AND apply.operation_type = 'ansible_apply_executed'
AND apply.status IN ('pending', 'running')
WHERE check_mode.parent_op_id = candidate.op_id
AND check_mode.operation_type
= 'ansible_check_mode_executed'
) AS has_pending_child,
EXISTS (
SELECT 1
FROM automation_operation_log check_mode
JOIN automation_operation_log apply
ON apply.parent_op_id = check_mode.op_id
AND apply.operation_type = 'ansible_apply_executed'
WHERE check_mode.parent_op_id = candidate.op_id
AND check_mode.operation_type
= 'ansible_check_mode_executed'
) AS has_apply
FROM automation_operation_log candidate
WHERE candidate.operation_type = 'ansible_candidate_matched'
AND coalesce(
candidate.incident_id::text,
candidate.input ->> 'incident_id'
) = :incident_id
AND candidate.input ->> 'executor' = 'ansible'
AND candidate.input ->> 'decision_path'
= 'repair_candidate_controlled_queue'
ORDER BY candidate.created_at DESC, candidate.op_id DESC
LIMIT 1
"""),
{"incident_id": incident_id},
)
latest = latest_result.mappings().first()
if latest is not None:
latest_input = _json_object(dict(latest), "input")
latest_generation = str(
latest_input.get("target_route_generation") or ""
)
latest_is_terminal = latest.get("is_terminal") is True
same_generation = latest_generation == target_route_generation
same_occurrence = str(
latest_input.get("source_occurrence_generation") or ""
) == str(
payload["input"].get("source_occurrence_generation") or ""
)
if same_generation and (
not latest_is_terminal or same_occurrence
):
return False
if source_recurrence.get("verified") is not True:
logger.warning(
"ansible_typed_route_supersession_blocked",
incident_id=incident_id,
reason="current_source_recurrence_not_verified",
previous_candidate_op_id=latest.get("op_id"),
)
return False
if not latest_is_terminal and not same_generation:
if latest.get("has_apply") is True:
logger.warning(
"ansible_typed_route_supersession_blocked",
incident_id=incident_id,
reason="previous_candidate_has_apply",
previous_candidate_op_id=latest.get("op_id"),
)
return False
if latest.get("has_pending_child") is True:
logger.info(
"ansible_typed_route_supersession_deferred",
incident_id=incident_id,
reason="previous_candidate_has_pending_child",
previous_candidate_op_id=latest.get("op_id"),
)
return False
await db.execute(
text("""
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, dry_run_result,
parent_op_id, tags
) VALUES (
'ansible_execution_skipped',
'typed_route_reconciler',
'dry_run',
:incident_db_id,
CAST(:input AS jsonb),
CAST(:output AS jsonb),
CAST(:dry_run_result AS jsonb),
CAST(:parent_op_id AS uuid),
:tags
)
"""),
{
"incident_db_id": _automation_operation_log_incident_id(
incident_id
),
"input": json.dumps(
{
"incident_id": incident_id,
"automation_run_id": str(
latest_input.get("automation_run_id")
or latest.get("op_id")
or ""
),
"execution_mode": "typed_route_supersession",
"previous_target_route_generation": (
latest_generation or None
),
"superseded_by_target_route_generation": (
target_route_generation
),
"superseded_by_candidate_op_id": candidate_op_id,
"source_recurrence": source_recurrence,
"cross_domain_fallback_allowed": False,
},
ensure_ascii=False,
),
"output": json.dumps(
{
"reason": "typed_route_superseded",
"terminal_disposition": (
"typed_route_superseded_no_write"
),
"runtime_apply_executed": False,
},
ensure_ascii=False,
),
"dry_run_result": json.dumps(
{
"skipped": True,
"apply_executed": False,
"runtime_apply_executed": False,
"supersession_recorded": True,
},
ensure_ascii=False,
),
"parent_op_id": str(latest.get("op_id") or ""),
"tags": [
"ansible",
"no_write",
"typed_route_superseded",
],
},
)
inserted = await db.execute(
text("""
INSERT INTO automation_operation_log (
op_id, operation_type, actor, status, incident_id,
input, output, dry_run_result, tags
) VALUES (
CAST(:op_id AS uuid),
:operation_type,
'decision_manager',
:status,
:incident_db_id,
CAST(:input AS jsonb),
CAST(:output AS jsonb),
CAST(:dry_run_result AS jsonb),
:tags
)
ON CONFLICT (op_id) DO NOTHING
RETURNING op_id::text
"""),
{
"operation_type": payload["operation_type"],
"op_id": candidate_op_id,
"status": payload["status"],
"incident_id": incident_id,
"incident_db_id": _automation_operation_log_incident_id(incident_id),
"input": json.dumps(payload["input"], ensure_ascii=False),
"output": json.dumps(payload["output"], ensure_ascii=False),
"dry_run_result": json.dumps(payload["dry_run_result"], ensure_ascii=False),
"tags": payload["tags"],
},
)
return inserted.scalar() is not None
except Exception as exc:
logger.warning(
"ansible_decision_audit_write_failed",
incident_id=incident_id,
error=str(exc),
)
return False