Files
awoooi/apps/api/src/services/awooop_ansible_audit_service.py
Your Name 876ba030d8
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 16s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
fix(awooop): allow ansible learning receipt rows
2026-06-29 17:26:55 +08:00

652 lines
23 KiB
Python

"""AwoooP Ansible audit helpers.
This module exposes the Ansible audit contract and repo-known playbook catalog.
Catalog rows are the first hard boundary for AI Agent controlled apply: only
allowlisted playbooks with check-mode support can move from dry-run to apply.
"""
from __future__ import annotations
import json
from typing import Any
import structlog
from sqlalchemy import text
from src.db.base import get_db_context
logger = structlog.get_logger(__name__)
def _automation_operation_log_incident_id(value: str) -> int | None:
# ADR-090 keeps external INC-* ids in input JSON; the DB column is BIGINT.
normalized = str(value or "").strip()
return int(normalized) if normalized.isdigit() else None
ANSIBLE_OPERATION_TYPES = frozenset({
"ansible_candidate_matched",
"ansible_check_mode_executed",
"ansible_apply_executed",
"ansible_learning_writeback_recorded",
"ansible_rollback_executed",
"ansible_execution_skipped",
})
_CATALOG: tuple[dict[str, Any], ...] = (
{
"catalog_id": "ansible:110-devops",
"playbook_path": "infra/ansible/playbooks/110-devops.yml",
"inventory_hosts": ["host_110"],
"domains": ["swap", "harbor", "sentry", "gitea", "langfuse", "bitan", "runner", "keepalived", "nginx"],
"keywords": [
"110",
"docker",
"container",
"dockercontainerunhealthy",
"swap",
"harbor",
"sentry",
"gitea",
"langfuse",
"bitan",
"runner",
"github-runner",
"keepalived",
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "medium",
},
{
"catalog_id": "ansible:188-momo-backup-user",
"playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml",
"check_mode_playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml",
"inventory_hosts": ["host_188"],
"domains": ["momo_backup", "postgresql", "cron", "awooop_notification"],
"keywords": [
"188",
"momo",
"momopostgresbackupfailed",
"postgres",
"postgresql",
"backup",
"momo pg_backup",
"momo postgres backup",
"pg_backup",
"momo-pg-backup",
"cron",
"crontab",
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "low",
},
{
"catalog_id": "ansible:188-ai-web",
"playbook_path": "infra/ansible/playbooks/188-ai-web.yml",
"check_mode_playbook_path": "infra/ansible/playbooks/188-ai-web-readonly.yml",
"inventory_hosts": ["host_188"],
"domains": ["docker", "momo_backup", "signoz", "minio", "litellm", "n8n", "open_webui", "nginx"],
"keywords": [
"188",
"docker",
"container",
"dockercontainerunhealthy",
"momo",
"backup",
"postgresql",
"pg_backup",
"signoz",
"minio",
"litellm",
"n8n",
"open-webui",
"openwebui",
"docker-registry",
],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "medium",
},
{
"catalog_id": "ansible:nginx-sync",
"playbook_path": "infra/ansible/playbooks/nginx-sync.yml",
"inventory_hosts": ["host_110", "host_188"],
"domains": ["nginx", "proxy", "ollama_proxy", "tls"],
"keywords": ["nginx", "proxy", "ollama", "gcp", "tls", "cert", "502", "upstream"],
"supports_check_mode": True,
"auto_apply_enabled": True,
"approval_required": False,
"risk_level": "high",
},
{
"catalog_id": "ansible:restore-password-auth",
"playbook_path": "infra/ansible/playbooks/restore-password-auth.yml",
"inventory_hosts": ["host_110", "host_120", "host_121", "host_188"],
"domains": ["ssh", "password_auth"],
"keywords": ["ssh", "passwordauthentication", "password auth", "login", "auth"],
"supports_check_mode": False,
"auto_apply_enabled": False,
"approval_required": True,
"risk_level": "high",
"break_glass_required": True,
},
)
def get_ansible_catalog_item(catalog_id: str) -> dict[str, Any] | None:
"""Return one repo-known Ansible catalog item without exposing mutability."""
for item in _CATALOG:
if item["catalog_id"] == catalog_id:
return dict(item)
return None
def _get(row: dict[str, Any], key: str) -> Any:
return row.get(key)
def _tags(row: dict[str, Any]) -> list[str]:
raw = _get(row, "tags")
if isinstance(raw, list):
return [str(item).lower() for item in raw]
if isinstance(raw, str):
return [part.strip().lower() for part in raw.split(",") if part.strip()]
return []
def _first_present(row: dict[str, Any], keys: tuple[str, ...]) -> Any:
for key in keys:
value = _get(row, key)
if value not in (None, ""):
return value
return None
def _json_object(row: dict[str, Any], key: str) -> dict[str, Any]:
raw = _get(row, key)
if isinstance(raw, dict):
return raw
if isinstance(raw, str) and raw.strip():
try:
parsed = json.loads(raw)
except json.JSONDecodeError:
return {}
return parsed if isinstance(parsed, dict) else {}
return {}
def _json_value(row: dict[str, Any], container_key: str, value_key: str) -> Any:
return _json_object(row, container_key).get(value_key)
def _first_present_with_json(row: dict[str, Any], keys: tuple[str, ...], json_keys: tuple[tuple[str, str], ...] = ()) -> Any:
value = _first_present(row, keys)
if value not in (None, ""):
return value
for container_key, value_key in json_keys:
value = _json_value(row, container_key, value_key)
if value not in (None, ""):
return value
return None
def _bool_or_none(value: Any) -> bool | None:
if isinstance(value, bool):
return value
if value in (None, ""):
return None
normalized = str(value).strip().lower()
if normalized in {"1", "true", "yes", "y", "on"}:
return True
if normalized in {"0", "false", "no", "n", "off"}:
return False
return None
def _int_or_none(value: Any) -> int | None:
if value in (None, ""):
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _is_controlled_apply_record(row: dict[str, Any]) -> bool:
if not row:
return False
if row.get("approval_source"):
return True
if str(row.get("execution_mode") or "").strip().lower() == "controlled_apply":
return True
if "controlled_apply" in _tags(row):
return True
return str(row.get("actor") or "").strip().lower() == "ansible_controlled_apply_worker"
def _is_ansible_operation(row: dict[str, Any]) -> bool:
operation_type = str(_get(row, "operation_type") or "").lower()
if operation_type in ANSIBLE_OPERATION_TYPES:
return True
if "ansible" in _tags(row):
return True
executor = str(
_first_present(
row,
(
"input_executor",
"input_execution_backend",
"output_executor",
"output_execution_backend",
),
)
or ""
).lower()
if executor == "ansible":
return True
playbook_path = str(
_first_present(row, ("input_playbook_path", "output_playbook_path", "input_ansible_playbook_path", "output_ansible_playbook_path"))
or ""
).lower()
return "infra/ansible/" in playbook_path or playbook_path.endswith(".yml") and "ansible" in playbook_path
def _ansible_record(row: dict[str, Any]) -> dict[str, Any]:
return {
"op_id": _get(row, "op_id"),
"parent_op_id": _get(row, "parent_op_id"),
"operation_type": _get(row, "operation_type"),
"status": _get(row, "status"),
"actor": _get(row, "actor"),
"catalog_id": _first_present(row, ("input_catalog_id", "output_catalog_id")),
"playbook_id": _first_present(row, ("input_playbook_id", "output_playbook_id")),
"playbook_path": _first_present(
row,
("input_playbook_path", "output_playbook_path", "input_ansible_playbook_path", "output_ansible_playbook_path"),
),
"execution_mode": _first_present(row, ("input_execution_mode", "output_execution_mode")),
"check_mode": _first_present(row, ("input_check_mode", "output_check_mode")),
"check_mode_executed": _first_present_with_json(
row,
("input_check_mode_executed", "output_check_mode_executed", "dry_run_check_mode_executed"),
(("dry_run_result", "check_mode_executed"),),
),
"apply_enabled": _first_present(row, ("input_apply_enabled", "output_apply_enabled")),
"apply_executed": _first_present_with_json(
row,
("input_apply_executed", "output_apply_executed", "dry_run_apply_executed"),
(("dry_run_result", "apply_executed"),),
),
"approval_source": _first_present(row, ("input_approval_source", "output_approval_source")),
"returncode": _first_present_with_json(
row,
("output_returncode", "input_returncode", "dry_run_returncode"),
(("dry_run_result", "returncode"),),
),
"not_used_reason": _first_present(row, ("input_not_used_reason", "output_not_used_reason")),
"dry_run_result": _get(row, "dry_run_result"),
"error": _get(row, "error"),
"duration_ms": _get(row, "duration_ms"),
"tags": _get(row, "tags"),
"created_at": _get(row, "created_at"),
}
def summarize_ansible_execution(records: list[dict[str, Any]]) -> dict[str, Any]:
"""Summarize durable Ansible audit rows for Telegram and Operator UI."""
typed_records = [row for row in records if isinstance(row, dict)]
check_mode_total = 0
apply_total = 0
rollback_total = 0
pending_check_mode_total = 0
applied_success_total = 0
terminal_check_mode_parent_ids = {
str(row.get("parent_op_id"))
for row in typed_records
if str(row.get("operation_type") or "") in {
"ansible_check_mode_executed",
"ansible_execution_skipped",
}
and row.get("parent_op_id")
}
for row in typed_records:
operation_type = str(row.get("operation_type") or "")
status = str(row.get("status") or "").lower()
if operation_type == "ansible_check_mode_executed" and status != "pending":
check_mode_total += 1
elif operation_type == "ansible_apply_executed":
apply_total += 1
if status in {"success", "completed", "executed", "applied"}:
applied_success_total += 1
elif operation_type == "ansible_rollback_executed":
rollback_total += 1
elif (
operation_type == "ansible_candidate_matched"
and str(row.get("op_id")) not in terminal_check_mode_parent_ids
):
pending_check_mode_total += 1
latest_record = typed_records[0] if typed_records else {}
latest_apply = next(
(row for row in typed_records if row.get("operation_type") == "ansible_apply_executed"),
{},
)
latest_check = next(
(row for row in typed_records if row.get("operation_type") == "ansible_check_mode_executed"),
{},
)
focused = latest_apply or latest_check or latest_record
returncode = _int_or_none(focused.get("returncode"))
approval_source = focused.get("approval_source")
return {
"check_mode_total": check_mode_total,
"apply_total": apply_total,
"rollback_total": rollback_total,
"pending_check_mode_total": pending_check_mode_total,
"applied_success_total": applied_success_total,
"applied": applied_success_total > 0,
"controlled_apply": bool(latest_apply) and _is_controlled_apply_record(latest_apply),
"latest_operation_type": focused.get("operation_type"),
"latest_status": focused.get("status"),
"latest_catalog_id": focused.get("catalog_id"),
"latest_playbook_path": focused.get("playbook_path"),
"latest_execution_mode": focused.get("execution_mode"),
"latest_check_mode": _bool_or_none(focused.get("check_mode")),
"latest_check_mode_executed": _bool_or_none(focused.get("check_mode_executed")),
"latest_apply_enabled": _bool_or_none(focused.get("apply_enabled")),
"latest_apply_executed": _bool_or_none(focused.get("apply_executed")),
"latest_returncode": returncode if returncode is not None else focused.get("returncode"),
"approval_source": approval_source,
"latest_actor": focused.get("actor"),
"latest_op_id": focused.get("op_id"),
"latest_parent_op_id": focused.get("parent_op_id"),
}
def _flatten_text(value: Any, pieces: list[str], remaining: int = 80) -> int:
if remaining <= 0 or value is None:
return remaining
if isinstance(value, dict):
for key, item in value.items():
remaining = _flatten_text(key, pieces, remaining)
remaining = _flatten_text(item, pieces, remaining)
if remaining <= 0:
break
return remaining
if isinstance(value, list):
for item in value:
remaining = _flatten_text(item, pieces, remaining)
if remaining <= 0:
break
return remaining
pieces.append(str(value).lower())
return remaining - 1
def _source_haystack(incident: dict[str, Any] | None, drift: dict[str, Any] | None) -> str:
pieces: list[str] = []
_flatten_text(incident, pieces)
_flatten_text(drift, pieces)
return " ".join(pieces)
def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None) -> dict[str, Any]:
haystack = _source_haystack(incident, drift)
candidates: list[dict[str, Any]] = []
unmatched: list[str] = []
for item in _CATALOG:
matched = [keyword for keyword in item["keywords"] if keyword in haystack]
public_item = {
key: value
for key, value in item.items()
if key
in {
"catalog_id",
"playbook_path",
"inventory_hosts",
"domains",
"supports_check_mode",
"check_mode_playbook_path",
"auto_apply_enabled",
"approval_required",
"risk_level",
}
}
if matched:
candidates.append({
**public_item,
"match_score": len(matched),
"matched_keywords": matched,
})
else:
unmatched.append(item["catalog_id"])
candidates.sort(key=lambda row: (-int(row["match_score"]), str(row["catalog_id"])))
return {
"match_mode": "static_catalog_keyword_hint_v1",
"decision_effect": "none",
"available_count": len(_CATALOG),
"candidates": candidates,
"unmatched_catalog_ids": unmatched,
}
def build_ansible_truth(
automation_ops: list[dict[str, Any]],
*,
incident: dict[str, Any] | None,
drift: dict[str, Any] | None,
) -> dict[str, Any]:
"""Build the truth-chain Ansible section from audited facts and catalog hints."""
records = [_ansible_record(row) for row in automation_ops if _is_ansible_operation(row)]
summary = summarize_ansible_execution(records)
return {
"considered": bool(records),
"records": records,
"summary": summary,
"audit_contract": {
"schema_version": "ansible_executor_audit_v1",
"operation_types": sorted(ANSIBLE_OPERATION_TYPES),
"required_audit_fields": [
"incident_id",
"operation_type",
"status",
"actor",
"input.executor",
"input.catalog_id",
"input.execution_mode",
"input.playbook_path",
"input.check_mode",
"input.approval_source",
"output.returncode",
"output.not_used_reason",
"dry_run_result",
],
"default_execution_mode": "catalog/dry-run audit only until approval execution is explicitly wired",
},
"candidate_catalog": _catalog_hints(incident, drift),
"not_used_reason": (
None
if records
else "no automation_operation_log row with Ansible operation type, tag, or executor backend for this source"
),
}
def _incident_public_dict(incident: Any) -> dict[str, Any]:
if incident is None:
return {}
if isinstance(incident, dict):
return incident
severity = getattr(incident, "severity", None)
signals_payload: list[dict[str, Any]] = []
for signal in getattr(incident, "signals", None) or []:
signals_payload.append({
"alert_name": getattr(signal, "alert_name", None),
"labels": getattr(signal, "labels", None) or {},
"annotations": getattr(signal, "annotations", None) or {},
})
return {
"incident_id": getattr(incident, "incident_id", None),
"project_id": getattr(incident, "project_id", None),
"alertname": getattr(incident, "alertname", None),
"alert_category": getattr(incident, "alert_category", None),
"notification_type": getattr(incident, "notification_type", None),
"severity": getattr(severity, "value", severity),
"affected_services": getattr(incident, "affected_services", None) or [],
"signals": signals_payload,
}
def build_ansible_decision_audit_payload(
*,
incident: Any,
proposal_data: dict[str, Any],
decision_path: str,
not_used_reason: str,
) -> dict[str, Any] | None:
"""Return an AOL payload when Ansible has catalog candidates for a decision."""
incident_payload = _incident_public_dict(incident)
hints = _catalog_hints(incident_payload, None)
candidates = hints.get("candidates") or []
if not candidates:
return None
incident_id = str(incident_payload.get("incident_id") or "")
input_payload = {
"incident_id": incident_id,
"executor": "ansible",
"execution_backend": "ansible",
"decision_path": decision_path,
"check_mode": True,
"apply_enabled": False,
"approval_required": True,
"candidate_catalog_schema": hints["match_mode"],
"executor_candidates": [
{
"catalog_id": row["catalog_id"],
"playbook_path": row["playbook_path"],
"check_mode_playbook_path": row.get("check_mode_playbook_path"),
"inventory_hosts": row["inventory_hosts"],
"supports_check_mode": row["supports_check_mode"],
"auto_apply_enabled": row["auto_apply_enabled"],
"approval_required": row["approval_required"],
"risk_level": row["risk_level"],
"match_score": row["match_score"],
"matched_keywords": row["matched_keywords"],
}
for row in candidates[:5]
],
"proposal_source": proposal_data.get("source", ""),
"proposal_risk_level": proposal_data.get("risk_level", ""),
"proposal_action_preview": str(
proposal_data.get("action")
or proposal_data.get("kubectl_command")
or ""
)[:240],
}
controlled_queue = decision_path == "repair_candidate_controlled_queue"
output_payload = {
"not_used_reason": not_used_reason,
"decision_effect": "check_mode_queue_ready" if controlled_queue else "audit_only",
"next_required_step": (
"awooop_ansible_check_mode_worker_claims_candidate"
if controlled_queue
else "wire approval_execution to Ansible check-mode before apply"
),
}
return {
"operation_type": "ansible_candidate_matched",
"status": "dry_run",
"input": input_payload,
"output": output_payload,
"dry_run_result": {
"check_mode_executed": False,
"candidate_count": len(candidates),
"reason": not_used_reason,
},
"tags": ["ansible", "decision", "candidate", "check_mode_pending"],
}
async def record_ansible_decision_audit(
*,
incident: Any,
proposal_data: dict[str, Any],
decision_path: str,
not_used_reason: str,
) -> bool:
"""Write a best-effort Ansible candidate audit row for one decision."""
payload = build_ansible_decision_audit_payload(
incident=incident,
proposal_data=proposal_data,
decision_path=decision_path,
not_used_reason=not_used_reason,
)
if payload is None:
return False
incident_id = payload["input"]["incident_id"]
project_id = getattr(incident, "project_id", None) or "awoooi"
try:
async with get_db_context(str(project_id)) as db:
existing = await db.execute(
text("""
SELECT op_id
FROM automation_operation_log
WHERE operation_type = 'ansible_candidate_matched'
AND coalesce(incident_id::text, input ->> 'incident_id') = :incident_id
AND input ->> 'executor' = 'ansible'
LIMIT 1
"""),
{"incident_id": incident_id},
)
if existing.scalar() is not None:
return False
await db.execute(
text("""
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, dry_run_result, tags
) VALUES (
:operation_type,
'decision_manager',
:status,
:incident_db_id,
CAST(:input AS jsonb),
CAST(:output AS jsonb),
CAST(:dry_run_result AS jsonb),
:tags
)
"""),
{
"operation_type": payload["operation_type"],
"status": payload["status"],
"incident_id": incident_id,
"incident_db_id": _automation_operation_log_incident_id(incident_id),
"input": json.dumps(payload["input"], ensure_ascii=False),
"output": json.dumps(payload["output"], ensure_ascii=False),
"dry_run_result": json.dumps(payload["dry_run_result"], ensure_ascii=False),
"tags": payload["tags"],
},
)
return True
except Exception as exc:
logger.warning(
"ansible_decision_audit_write_failed",
incident_id=incident_id,
error=str(exc),
)
return False