1440 lines
55 KiB
Python
1440 lines
55 KiB
Python
"""AwoooP Ansible controlled executor.
|
|
|
|
This service claims pending ``ansible_candidate_matched`` AOL rows, runs
|
|
``ansible-playbook --check --diff``, and for allowlisted low/medium/high risk
|
|
catalog rows performs controlled apply after the dry-run passes.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import structlog
|
|
from sqlalchemy import text
|
|
|
|
from src.core.config import settings
|
|
from src.db.base import get_db_context
|
|
from src.services.awooop_ansible_audit_service import get_ansible_catalog_item
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
|
|
_SAFE_HOST_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
|
|
_PLAYBOOK_PREFIX = Path("infra/ansible/playbooks")
|
|
_STDOUT_LIMIT = 20_000
|
|
_STDERR_LIMIT = 12_000
|
|
FORCED_COMMAND_BLOCKER = "ansible_repair_ssh_forced_command_denies_ansible_bootstrap"
|
|
REPAIR_FORCED_COMMAND_KEY_PATH = Path("/etc/repair-ssh/id_ed25519")
|
|
REPAIR_FORCED_COMMAND_KNOWN_HOSTS_PATH = Path("/etc/repair-known-hosts/known_hosts")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AnsibleCheckModeClaim:
|
|
op_id: str
|
|
source_candidate_op_id: str
|
|
incident_id: str
|
|
catalog_id: str
|
|
playbook_path: str
|
|
apply_playbook_path: str
|
|
inventory_hosts: tuple[str, ...]
|
|
risk_level: str
|
|
input_payload: dict[str, Any]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AnsibleCommandSpec:
|
|
command: list[str]
|
|
cwd: Path
|
|
env: dict[str, str]
|
|
playbook_abs_path: Path
|
|
inventory_abs_path: Path
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AnsibleRunResult:
|
|
returncode: int
|
|
stdout: str
|
|
stderr: str
|
|
duration_ms: int
|
|
timed_out: bool = False
|
|
|
|
|
|
def _tail(text_value: str, limit: int) -> str:
|
|
if len(text_value) <= limit:
|
|
return text_value
|
|
return text_value[-limit:]
|
|
|
|
|
|
def _json_loads(value: Any) -> dict[str, Any]:
|
|
if isinstance(value, dict):
|
|
return value
|
|
if isinstance(value, str):
|
|
try:
|
|
parsed = json.loads(value)
|
|
except json.JSONDecodeError:
|
|
return {}
|
|
return parsed if isinstance(parsed, dict) else {}
|
|
return {}
|
|
|
|
|
|
def _incident_id_from_payload(payload: dict[str, Any]) -> str:
|
|
return str(payload.get("incident_id") or "").strip()
|
|
|
|
|
|
def _automation_operation_log_incident_id(value: str) -> int | None:
|
|
# ADR-090 keeps external INC-* ids in input JSON; the DB column is BIGINT.
|
|
normalized = str(value or "").strip()
|
|
return int(normalized) if normalized.isdigit() else None
|
|
|
|
|
|
def _check_mode_ssh_key_path() -> Path:
|
|
return Path(settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH)
|
|
|
|
|
|
def _check_mode_known_hosts_path() -> Path:
|
|
return Path(settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH)
|
|
|
|
|
|
def _uses_repair_forced_command_transport(key_path: Path | None = None) -> bool:
|
|
return (key_path or _check_mode_ssh_key_path()) == REPAIR_FORCED_COMMAND_KEY_PATH
|
|
|
|
|
|
def detect_ansible_transport_blockers(*values: Any) -> list[str]:
|
|
combined = " ".join(str(value or "") for value in values)
|
|
blockers: list[str] = []
|
|
if "REPAIR_DENIED:invalid_command" in combined:
|
|
blockers.append(FORCED_COMMAND_BLOCKER)
|
|
return blockers
|
|
|
|
|
|
def _playbook_roots(module_path: Path | None = None) -> list[Path]:
|
|
resolved_module_path = (module_path or Path(__file__)).resolve()
|
|
return [
|
|
Path("/app/infra/ansible"),
|
|
Path.cwd() / "infra" / "ansible",
|
|
*(parent / "infra" / "ansible" for parent in resolved_module_path.parents),
|
|
]
|
|
|
|
|
|
def _runtime_blockers(
|
|
*,
|
|
playbook_root: Path | None = None,
|
|
check_mode_ssh_key_path: Path | None = None,
|
|
check_mode_known_hosts_path: Path | None = None,
|
|
) -> list[str]:
|
|
root = playbook_root or next((path for path in _playbook_roots() if path.exists()), None)
|
|
ssh_key_path = check_mode_ssh_key_path or _check_mode_ssh_key_path()
|
|
known_hosts_path = check_mode_known_hosts_path or _check_mode_known_hosts_path()
|
|
blockers: list[str] = []
|
|
if shutil.which("ansible-playbook") is None:
|
|
blockers.append("ansible_playbook_binary_missing")
|
|
if root is None:
|
|
blockers.append("ansible_playbook_catalog_missing")
|
|
elif not (root / "inventory" / "hosts.yml").exists():
|
|
blockers.append("ansible_inventory_missing")
|
|
if not ssh_key_path.is_file() or not os.access(ssh_key_path, os.R_OK):
|
|
blockers.append("ansible_check_mode_ssh_key_missing")
|
|
if not known_hosts_path.is_file() or not os.access(known_hosts_path, os.R_OK):
|
|
blockers.append("ansible_check_mode_known_hosts_missing")
|
|
return blockers
|
|
|
|
|
|
def _safe_candidate(input_payload: dict[str, Any]) -> dict[str, Any]:
|
|
candidates = input_payload.get("executor_candidates")
|
|
if not isinstance(candidates, list) or not candidates:
|
|
raise ValueError("missing_executor_candidates")
|
|
|
|
for candidate in candidates:
|
|
if not isinstance(candidate, dict):
|
|
continue
|
|
catalog_id = str(candidate.get("catalog_id") or "")
|
|
catalog_item = get_ansible_catalog_item(catalog_id)
|
|
if not catalog_item:
|
|
continue
|
|
if catalog_item.get("supports_check_mode") is not True:
|
|
continue
|
|
catalog_playbook_path = str(catalog_item.get("playbook_path") or "")
|
|
candidate_playbook_path = str(candidate.get("playbook_path") or catalog_playbook_path)
|
|
check_mode_playbook_path = str(
|
|
candidate.get("check_mode_playbook_path")
|
|
or catalog_item.get("check_mode_playbook_path")
|
|
or catalog_playbook_path
|
|
)
|
|
if candidate_playbook_path not in {catalog_playbook_path, check_mode_playbook_path}:
|
|
continue
|
|
inventory_hosts = candidate.get("inventory_hosts") or catalog_item.get("inventory_hosts") or []
|
|
if (
|
|
isinstance(inventory_hosts, list)
|
|
and inventory_hosts
|
|
and all(isinstance(host, str) and _SAFE_HOST_RE.fullmatch(host) for host in inventory_hosts)
|
|
):
|
|
return {
|
|
"catalog_id": catalog_id,
|
|
"playbook_path": check_mode_playbook_path,
|
|
"catalog_playbook_path": catalog_playbook_path,
|
|
"source_candidate_playbook_path": candidate_playbook_path,
|
|
"check_mode_playbook_path": check_mode_playbook_path,
|
|
"inventory_hosts": tuple(inventory_hosts),
|
|
"risk_level": str(candidate.get("risk_level") or catalog_item.get("risk_level") or ""),
|
|
"auto_apply_enabled": bool(catalog_item.get("auto_apply_enabled") is True),
|
|
"approval_required": bool(catalog_item.get("approval_required") is True),
|
|
"break_glass_required": bool(catalog_item.get("break_glass_required") is True),
|
|
}
|
|
raise ValueError("no_safe_check_mode_candidate")
|
|
|
|
|
|
def _allowed_controlled_apply_risks() -> set[str]:
|
|
raw = str(settings.AWOOOP_ANSIBLE_CONTROLLED_APPLY_ALLOWED_RISK_LEVELS or "")
|
|
return {item.strip().lower() for item in raw.split(",") if item.strip()}
|
|
|
|
|
|
def _controlled_apply_allowed(candidate: dict[str, Any]) -> tuple[bool, str | None]:
|
|
risk = str(candidate.get("risk_level") or "").strip().lower()
|
|
if settings.ENABLE_AWOOOP_ANSIBLE_CONTROLLED_APPLY is not True:
|
|
return False, "controlled_apply_disabled_by_config"
|
|
if candidate.get("break_glass_required") is True:
|
|
return False, "break_glass_required"
|
|
if candidate.get("auto_apply_enabled") is not True:
|
|
return False, "catalog_auto_apply_disabled"
|
|
if candidate.get("approval_required") is True:
|
|
return False, "catalog_still_requires_approval"
|
|
if risk not in _allowed_controlled_apply_risks():
|
|
return False, f"risk_level_not_allowed:{risk or 'unknown'}"
|
|
return True, None
|
|
|
|
|
|
def build_ansible_check_mode_claim_input(
|
|
*,
|
|
source_candidate_op_id: str,
|
|
candidate_input: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
safe = _safe_candidate(candidate_input)
|
|
incident_id = _incident_id_from_payload(candidate_input)
|
|
controlled_apply_allowed, controlled_apply_blocker = _controlled_apply_allowed(safe)
|
|
return {
|
|
"incident_id": incident_id,
|
|
"executor": "ansible",
|
|
"execution_backend": "ansible",
|
|
"execution_mode": "check_mode",
|
|
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
|
|
"check_mode": True,
|
|
"diff": True,
|
|
"apply_enabled": controlled_apply_allowed,
|
|
"approval_required_before_apply": not controlled_apply_allowed,
|
|
"controlled_apply_allowed": controlled_apply_allowed,
|
|
"controlled_apply_blocker": controlled_apply_blocker,
|
|
"source_candidate_op_id": source_candidate_op_id,
|
|
"catalog_id": safe["catalog_id"],
|
|
"playbook_path": safe["playbook_path"],
|
|
"apply_playbook_path": safe["catalog_playbook_path"],
|
|
"catalog_playbook_path": safe["catalog_playbook_path"],
|
|
"source_candidate_playbook_path": safe["source_candidate_playbook_path"],
|
|
"check_mode_playbook_path": safe["check_mode_playbook_path"],
|
|
"inventory_hosts": list(safe["inventory_hosts"]),
|
|
"risk_level": safe["risk_level"],
|
|
}
|
|
|
|
|
|
def _resolve_playbook_path(playbook_root: Path, playbook_path: str) -> Path:
|
|
relative = Path(playbook_path)
|
|
if relative.is_absolute() or not str(relative).startswith(str(_PLAYBOOK_PREFIX)):
|
|
raise ValueError("unsafe_playbook_path")
|
|
repo_root = playbook_root.parent.parent
|
|
resolved = (repo_root / relative).resolve()
|
|
allowed_root = (repo_root / _PLAYBOOK_PREFIX).resolve()
|
|
if allowed_root not in resolved.parents:
|
|
raise ValueError("playbook_outside_catalog")
|
|
if resolved.suffix not in {".yml", ".yaml"} or not resolved.exists():
|
|
raise ValueError("playbook_not_found")
|
|
return resolved
|
|
|
|
|
|
def _ansible_command_env(playbook_root: Path) -> dict[str, str]:
|
|
roles_path = str((playbook_root / "roles").resolve())
|
|
existing_roles_path = os.environ.get("ANSIBLE_ROLES_PATH")
|
|
if existing_roles_path:
|
|
roles_path = os.pathsep.join([roles_path, existing_roles_path])
|
|
return {
|
|
**os.environ,
|
|
"ANSIBLE_HOST_KEY_CHECKING": "true",
|
|
"ANSIBLE_RETRY_FILES_ENABLED": "false",
|
|
"ANSIBLE_ROLES_PATH": roles_path,
|
|
}
|
|
|
|
|
|
def build_ansible_check_mode_command(
|
|
*,
|
|
playbook_path: str,
|
|
inventory_hosts: tuple[str, ...],
|
|
playbook_root: Path | None = None,
|
|
check_mode_ssh_key_path: Path | None = None,
|
|
check_mode_known_hosts_path: Path | None = None,
|
|
) -> AnsibleCommandSpec:
|
|
root = playbook_root or next((path for path in _playbook_roots() if path.exists()), None)
|
|
if root is None:
|
|
raise ValueError("ansible_playbook_catalog_missing")
|
|
inventory_path = (root / "inventory" / "hosts.yml").resolve()
|
|
if not inventory_path.exists():
|
|
raise ValueError("ansible_inventory_missing")
|
|
if not inventory_hosts or not all(_SAFE_HOST_RE.fullmatch(host) for host in inventory_hosts):
|
|
raise ValueError("unsafe_inventory_hosts")
|
|
|
|
playbook_abs = _resolve_playbook_path(root, playbook_path)
|
|
ssh_key_path = check_mode_ssh_key_path or _check_mode_ssh_key_path()
|
|
known_hosts_path = check_mode_known_hosts_path or _check_mode_known_hosts_path()
|
|
ssh_common_args = (
|
|
f"-o UserKnownHostsFile={known_hosts_path} "
|
|
"-o IdentitiesOnly=yes -o BatchMode=yes"
|
|
)
|
|
extra_vars = {
|
|
"ansible_ssh_private_key_file": str(ssh_key_path),
|
|
"ansible_ssh_common_args": ssh_common_args,
|
|
}
|
|
command = [
|
|
"ansible-playbook",
|
|
"-i",
|
|
str(inventory_path),
|
|
str(playbook_abs),
|
|
"--check",
|
|
"--diff",
|
|
"--limit",
|
|
",".join(inventory_hosts),
|
|
"--extra-vars",
|
|
json.dumps(extra_vars, ensure_ascii=False, separators=(",", ":")),
|
|
]
|
|
env = _ansible_command_env(root)
|
|
return AnsibleCommandSpec(
|
|
command=command,
|
|
cwd=root,
|
|
env=env,
|
|
playbook_abs_path=playbook_abs,
|
|
inventory_abs_path=inventory_path,
|
|
)
|
|
|
|
|
|
def build_ansible_apply_command(
|
|
*,
|
|
playbook_path: str,
|
|
inventory_hosts: tuple[str, ...],
|
|
playbook_root: Path | None = None,
|
|
check_mode_ssh_key_path: Path | None = None,
|
|
check_mode_known_hosts_path: Path | None = None,
|
|
) -> AnsibleCommandSpec:
|
|
"""Build the controlled apply command after check-mode has passed."""
|
|
|
|
root = playbook_root or next((path for path in _playbook_roots() if path.exists()), None)
|
|
if root is None:
|
|
raise ValueError("ansible_playbook_catalog_missing")
|
|
inventory_path = (root / "inventory" / "hosts.yml").resolve()
|
|
if not inventory_path.exists():
|
|
raise ValueError("ansible_inventory_missing")
|
|
if not inventory_hosts or not all(_SAFE_HOST_RE.fullmatch(host) for host in inventory_hosts):
|
|
raise ValueError("unsafe_inventory_hosts")
|
|
|
|
playbook_abs = _resolve_playbook_path(root, playbook_path)
|
|
ssh_key_path = check_mode_ssh_key_path or _check_mode_ssh_key_path()
|
|
known_hosts_path = check_mode_known_hosts_path or _check_mode_known_hosts_path()
|
|
ssh_common_args = (
|
|
f"-o UserKnownHostsFile={known_hosts_path} "
|
|
"-o IdentitiesOnly=yes -o BatchMode=yes"
|
|
)
|
|
extra_vars = {
|
|
"ansible_ssh_private_key_file": str(ssh_key_path),
|
|
"ansible_ssh_common_args": ssh_common_args,
|
|
}
|
|
command = [
|
|
"ansible-playbook",
|
|
"-i",
|
|
str(inventory_path),
|
|
str(playbook_abs),
|
|
"--diff",
|
|
"--limit",
|
|
",".join(inventory_hosts),
|
|
"--extra-vars",
|
|
json.dumps(extra_vars, ensure_ascii=False, separators=(",", ":")),
|
|
]
|
|
env = _ansible_command_env(root)
|
|
return AnsibleCommandSpec(
|
|
command=command,
|
|
cwd=root,
|
|
env=env,
|
|
playbook_abs_path=playbook_abs,
|
|
inventory_abs_path=inventory_path,
|
|
)
|
|
|
|
|
|
async def _run_ansible_command(spec: AnsibleCommandSpec, *, timeout_seconds: int) -> AnsibleRunResult:
|
|
started = time.monotonic()
|
|
process = await asyncio.create_subprocess_exec(
|
|
*spec.command,
|
|
cwd=str(spec.cwd),
|
|
env=spec.env,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
timed_out = False
|
|
try:
|
|
stdout_bytes, stderr_bytes = await asyncio.wait_for(
|
|
process.communicate(),
|
|
timeout=timeout_seconds,
|
|
)
|
|
except TimeoutError:
|
|
timed_out = True
|
|
process.kill()
|
|
stdout_bytes, stderr_bytes = await process.communicate()
|
|
duration_ms = int((time.monotonic() - started) * 1000)
|
|
return AnsibleRunResult(
|
|
returncode=124 if timed_out else int(process.returncode or 0),
|
|
stdout=stdout_bytes.decode("utf-8", "replace"),
|
|
stderr=stderr_bytes.decode("utf-8", "replace"),
|
|
duration_ms=duration_ms,
|
|
timed_out=timed_out,
|
|
)
|
|
|
|
|
|
def _build_result_payload(
|
|
result: AnsibleRunResult,
|
|
*,
|
|
controlled_apply_allowed: bool = False,
|
|
controlled_apply_blocker: str | None = None,
|
|
) -> tuple[str, dict[str, Any], dict[str, Any], str | None]:
|
|
status = "success" if result.returncode == 0 else "failed"
|
|
stdout_tail = _tail(result.stdout, _STDOUT_LIMIT)
|
|
stderr_tail = _tail(result.stderr, _STDERR_LIMIT)
|
|
output = {
|
|
"executor": "ansible",
|
|
"execution_mode": "check_mode",
|
|
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
|
|
"ssh_key_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH,
|
|
"known_hosts_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH,
|
|
"check_mode": True,
|
|
"apply_enabled": controlled_apply_allowed,
|
|
"approval_required_before_apply": not controlled_apply_allowed,
|
|
"controlled_apply_allowed": controlled_apply_allowed,
|
|
"controlled_apply_blocker": controlled_apply_blocker,
|
|
"returncode": result.returncode,
|
|
"timed_out": result.timed_out,
|
|
"stdout_tail": stdout_tail,
|
|
"stderr_tail": stderr_tail,
|
|
"next_required_step": (
|
|
"controlled_apply_queued"
|
|
if result.returncode == 0 and controlled_apply_allowed
|
|
else "ai_playbook_or_transport_repair_required"
|
|
if result.returncode != 0
|
|
else "controlled_apply_blocked"
|
|
),
|
|
}
|
|
dry_run_result = {
|
|
"check_mode_executed": True,
|
|
"apply_executed": False,
|
|
"safe_to_apply_without_approval": controlled_apply_allowed,
|
|
"controlled_apply_allowed": controlled_apply_allowed,
|
|
"controlled_apply_blocker": controlled_apply_blocker,
|
|
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
|
|
"ssh_key_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH,
|
|
"known_hosts_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH,
|
|
"returncode": result.returncode,
|
|
"timed_out": result.timed_out,
|
|
"stdout_tail": stdout_tail,
|
|
"stderr_tail": stderr_tail,
|
|
}
|
|
error = None if result.returncode == 0 else (stderr_tail or f"ansible_check_mode_failed_rc_{result.returncode}")
|
|
return status, output, dry_run_result, error
|
|
|
|
|
|
def _build_apply_result_payload(result: AnsibleRunResult) -> tuple[str, dict[str, Any], dict[str, Any], str | None]:
|
|
status = "success" if result.returncode == 0 else "failed"
|
|
stdout_tail = _tail(result.stdout, _STDOUT_LIMIT)
|
|
stderr_tail = _tail(result.stderr, _STDERR_LIMIT)
|
|
output = {
|
|
"executor": "ansible",
|
|
"execution_mode": "controlled_apply",
|
|
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
|
|
"ssh_key_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH,
|
|
"known_hosts_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH,
|
|
"check_mode": False,
|
|
"apply_enabled": True,
|
|
"controlled_apply_executed": True,
|
|
"returncode": result.returncode,
|
|
"timed_out": result.timed_out,
|
|
"stdout_tail": stdout_tail,
|
|
"stderr_tail": stderr_tail,
|
|
"next_required_step": (
|
|
"post_apply_verifier_and_learning_writeback"
|
|
if result.returncode == 0
|
|
else "ai_rollback_or_playbook_repair_required"
|
|
),
|
|
}
|
|
dry_run_result = {
|
|
"check_mode_executed_before_apply": True,
|
|
"apply_executed": True,
|
|
"controlled_apply_executed": True,
|
|
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
|
|
"ssh_key_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH,
|
|
"known_hosts_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH,
|
|
"returncode": result.returncode,
|
|
"timed_out": result.timed_out,
|
|
"stdout_tail": stdout_tail,
|
|
"stderr_tail": stderr_tail,
|
|
}
|
|
error = None if result.returncode == 0 else (stderr_tail or f"ansible_apply_failed_rc_{result.returncode}")
|
|
return status, output, dry_run_result, error
|
|
|
|
|
|
def _build_auto_repair_execution_receipt(
|
|
claim: AnsibleCheckModeClaim,
|
|
result: AnsibleRunResult,
|
|
*,
|
|
apply_op_id: str,
|
|
) -> dict[str, Any]:
|
|
success = result.returncode == 0
|
|
return {
|
|
"incident_id": claim.incident_id,
|
|
"playbook_id": str(claim.catalog_id or "")[:36] or "ansible",
|
|
"playbook_name": f"Ansible controlled apply: {claim.apply_playbook_path}"[:200],
|
|
"success": success,
|
|
"executed_steps": [
|
|
f"candidate:{claim.source_candidate_op_id}",
|
|
f"check_mode:{claim.op_id}",
|
|
f"apply:{apply_op_id}",
|
|
f"catalog:{claim.catalog_id}",
|
|
f"returncode:{result.returncode}",
|
|
],
|
|
"error_message": None if success else _tail(result.stderr or result.stdout, 2000),
|
|
"triggered_by": "ansible_controlled_apply",
|
|
"similarity_score": None,
|
|
"risk_level": str(claim.risk_level or ""),
|
|
"execution_time_ms": result.duration_ms,
|
|
}
|
|
|
|
|
|
def _int_from_value(value: Any, *, default: int = 1) -> int:
|
|
if value in (None, ""):
|
|
return default
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
def _claim_from_apply_operation_row(row: dict[str, Any]) -> tuple[AnsibleCheckModeClaim, AnsibleRunResult] | None:
|
|
input_payload = _json_loads(row.get("input"))
|
|
output_payload = _json_loads(row.get("output"))
|
|
dry_run_result = _json_loads(row.get("dry_run_result"))
|
|
catalog_id = str(input_payload.get("catalog_id") or "")
|
|
if not catalog_id:
|
|
return None
|
|
catalog_item = get_ansible_catalog_item(catalog_id) or {}
|
|
incident_id = str(row.get("incident_id") or input_payload.get("incident_id") or "")
|
|
if not incident_id:
|
|
return None
|
|
inventory_hosts = input_payload.get("inventory_hosts") or catalog_item.get("inventory_hosts") or []
|
|
if not isinstance(inventory_hosts, list) or not inventory_hosts:
|
|
return None
|
|
apply_playbook_path = str(
|
|
input_payload.get("apply_playbook_path")
|
|
or input_payload.get("playbook_path")
|
|
or catalog_item.get("playbook_path")
|
|
or ""
|
|
)
|
|
check_mode_playbook_path = str(
|
|
input_payload.get("check_mode_playbook_path")
|
|
or input_payload.get("playbook_path")
|
|
or apply_playbook_path
|
|
)
|
|
if not apply_playbook_path:
|
|
return None
|
|
returncode = _int_from_value(
|
|
output_payload.get("returncode", dry_run_result.get("returncode")),
|
|
default=0 if str(row.get("status") or "").lower() == "success" else 1,
|
|
)
|
|
claim = AnsibleCheckModeClaim(
|
|
op_id=str(input_payload.get("check_mode_op_id") or row.get("parent_op_id") or ""),
|
|
source_candidate_op_id=str(input_payload.get("source_candidate_op_id") or ""),
|
|
incident_id=incident_id,
|
|
catalog_id=catalog_id,
|
|
playbook_path=check_mode_playbook_path,
|
|
apply_playbook_path=apply_playbook_path,
|
|
inventory_hosts=tuple(str(host) for host in inventory_hosts),
|
|
risk_level=str(input_payload.get("risk_level") or catalog_item.get("risk_level") or ""),
|
|
input_payload=input_payload,
|
|
)
|
|
result = AnsibleRunResult(
|
|
returncode=returncode,
|
|
stdout=str(output_payload.get("stdout_tail") or ""),
|
|
stderr=str(row.get("error") or output_payload.get("stderr_tail") or ""),
|
|
duration_ms=_int_from_value(row.get("duration_ms"), default=0),
|
|
timed_out=bool(output_payload.get("timed_out") or dry_run_result.get("timed_out") or False),
|
|
)
|
|
return claim, result
|
|
|
|
|
|
async def _record_auto_repair_execution_receipt(
|
|
claim: AnsibleCheckModeClaim,
|
|
result: AnsibleRunResult,
|
|
*,
|
|
apply_op_id: str,
|
|
project_id: str,
|
|
) -> bool:
|
|
receipt = _build_auto_repair_execution_receipt(
|
|
claim,
|
|
result,
|
|
apply_op_id=apply_op_id,
|
|
)
|
|
try:
|
|
async with get_db_context(project_id) as db:
|
|
inserted = await db.execute(
|
|
text("""
|
|
INSERT INTO auto_repair_executions (
|
|
incident_id,
|
|
playbook_id,
|
|
playbook_name,
|
|
success,
|
|
executed_steps,
|
|
error_message,
|
|
triggered_by,
|
|
similarity_score,
|
|
risk_level,
|
|
execution_time_ms
|
|
)
|
|
SELECT
|
|
CAST(:incident_id AS varchar(30)),
|
|
CAST(:playbook_id AS varchar(36)),
|
|
CAST(:playbook_name AS varchar(200)),
|
|
:success,
|
|
CAST(:executed_steps AS jsonb),
|
|
:error_message,
|
|
CAST(:triggered_by AS varchar(50)),
|
|
:similarity_score,
|
|
CAST(:risk_level AS varchar(20)),
|
|
:execution_time_ms
|
|
WHERE NOT EXISTS (
|
|
SELECT 1
|
|
FROM auto_repair_executions existing
|
|
WHERE existing.incident_id = CAST(:incident_id AS varchar(30))
|
|
AND existing.triggered_by = CAST(:triggered_by AS varchar(50))
|
|
AND existing.executed_steps::text LIKE CAST(:apply_op_id_needle AS text)
|
|
)
|
|
RETURNING id
|
|
"""),
|
|
{
|
|
**receipt,
|
|
"executed_steps": json.dumps(receipt["executed_steps"], ensure_ascii=False),
|
|
"apply_op_id_needle": f"%{apply_op_id}%",
|
|
},
|
|
)
|
|
return inserted.scalar() is not None
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"ansible_auto_repair_execution_receipt_failed",
|
|
incident_id=claim.incident_id,
|
|
catalog_id=claim.catalog_id,
|
|
apply_op_id=apply_op_id,
|
|
error=str(exc),
|
|
)
|
|
return False
|
|
|
|
|
|
def _post_apply_verification_result(result: AnsibleRunResult) -> str:
|
|
if result.timed_out:
|
|
return "timeout"
|
|
return "success" if result.returncode == 0 else "failed"
|
|
|
|
|
|
def _post_apply_km_path_type(apply_op_id: str) -> str:
|
|
return f"ansible_apply_receipt:{str(apply_op_id or '')[:8]}"
|
|
|
|
|
|
def _post_apply_action_label(claim: AnsibleCheckModeClaim, *, apply_op_id: str) -> str:
|
|
return (
|
|
f"ansible_controlled_apply:{claim.catalog_id}:"
|
|
f"{claim.apply_playbook_path}:apply_op={apply_op_id}"
|
|
)
|
|
|
|
|
|
async def _record_post_apply_verifier_and_learning(
|
|
claim: AnsibleCheckModeClaim,
|
|
result: AnsibleRunResult,
|
|
*,
|
|
apply_op_id: str,
|
|
project_id: str,
|
|
) -> dict[str, bool]:
|
|
"""Persist post-apply verifier evidence and KM learning for Ansible apply."""
|
|
|
|
verification_result = _post_apply_verification_result(result)
|
|
action_label = _post_apply_action_label(claim, apply_op_id=apply_op_id)
|
|
post_state = {
|
|
"schema_version": "ansible_apply_result_verifier_v1",
|
|
"verifier": "ansible_apply_result_verifier",
|
|
"apply_op_id": apply_op_id,
|
|
"catalog_id": claim.catalog_id,
|
|
"playbook_path": claim.apply_playbook_path,
|
|
"returncode": result.returncode,
|
|
"timed_out": result.timed_out,
|
|
"action_taken": action_label,
|
|
"next_required_step": (
|
|
"km_playbook_trust_writeback"
|
|
if verification_result == "success"
|
|
else "rollback_or_playbook_repair_candidate"
|
|
),
|
|
}
|
|
evidence_summary = (
|
|
"AI Agent Ansible controlled apply verifier: "
|
|
f"incident={claim.incident_id}; catalog={claim.catalog_id}; "
|
|
f"result={verification_result}; returncode={result.returncode}; apply_op={apply_op_id}"
|
|
)
|
|
status = {"verification": False, "learning": False}
|
|
|
|
try:
|
|
async with get_db_context(project_id) as db:
|
|
inserted = await db.execute(
|
|
text("""
|
|
INSERT INTO incident_evidence (
|
|
id,
|
|
incident_id,
|
|
matched_playbook_id,
|
|
schema_version,
|
|
mcp_health,
|
|
collection_duration_ms,
|
|
sensors_attempted,
|
|
sensors_succeeded,
|
|
evidence_summary,
|
|
post_execution_state,
|
|
verification_result
|
|
)
|
|
SELECT
|
|
gen_random_uuid()::text,
|
|
CAST(:incident_id AS varchar(30)),
|
|
CAST(:matched_playbook_id AS varchar(36)),
|
|
'v1',
|
|
CAST(:mcp_health AS jsonb),
|
|
:collection_duration_ms,
|
|
1,
|
|
:sensors_succeeded,
|
|
:evidence_summary,
|
|
CAST(:post_execution_state AS jsonb),
|
|
CAST(:verification_result AS varchar(20))
|
|
WHERE NOT EXISTS (
|
|
SELECT 1
|
|
FROM incident_evidence existing
|
|
WHERE existing.incident_id = CAST(:incident_id AS varchar(30))
|
|
AND existing.post_execution_state ->> 'apply_op_id' = CAST(:apply_op_id AS text)
|
|
)
|
|
RETURNING id
|
|
"""),
|
|
{
|
|
"incident_id": claim.incident_id,
|
|
"matched_playbook_id": str(claim.catalog_id or "")[:36],
|
|
"mcp_health": json.dumps({
|
|
"ansible_apply_result_verifier": verification_result == "success"
|
|
}, ensure_ascii=False),
|
|
"collection_duration_ms": max(0, int(result.duration_ms or 0)),
|
|
"sensors_succeeded": 1 if verification_result == "success" else 0,
|
|
"evidence_summary": evidence_summary,
|
|
"post_execution_state": json.dumps(post_state, ensure_ascii=False),
|
|
"verification_result": verification_result,
|
|
"apply_op_id": apply_op_id,
|
|
},
|
|
)
|
|
status["verification"] = inserted.scalar() is not None
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"ansible_post_apply_verifier_receipt_failed",
|
|
incident_id=claim.incident_id,
|
|
catalog_id=claim.catalog_id,
|
|
apply_op_id=apply_op_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
try:
|
|
from src.models.knowledge import (
|
|
EntrySource,
|
|
EntryStatus,
|
|
EntryType,
|
|
KnowledgeEntryCreate,
|
|
)
|
|
from src.repositories.knowledge_repository import KnowledgeDBRepository
|
|
|
|
async with get_db_context(project_id) as db:
|
|
repo = KnowledgeDBRepository(db)
|
|
await repo.create(
|
|
KnowledgeEntryCreate(
|
|
title=f"AI 自動修復沉澱：{claim.incident_id}",
|
|
content=(
|
|
"AI Agent 已完成 Ansible controlled apply 並寫入驗證摘要。\n\n"
|
|
f"- Incident: {claim.incident_id}\n"
|
|
f"- Catalog: {claim.catalog_id}\n"
|
|
f"- PlayBook: {claim.apply_playbook_path}\n"
|
|
f"- Apply operation: {apply_op_id}\n"
|
|
f"- Verification result: {verification_result}\n"
|
|
f"- Return code: {result.returncode}\n"
|
|
f"- Next step: {post_state['next_required_step']}\n"
|
|
),
|
|
entry_type=EntryType.INCIDENT_CASE,
|
|
category="AI自動化/Ansible受控修復",
|
|
tags=[
|
|
"ai_auto_repair",
|
|
"ansible_controlled_apply",
|
|
verification_result,
|
|
str(claim.catalog_id or ""),
|
|
],
|
|
source=EntrySource.AI_EXTRACTED,
|
|
status=EntryStatus.REVIEW,
|
|
related_incident_id=claim.incident_id,
|
|
related_playbook_id=str(claim.catalog_id or "")[:36] or None,
|
|
path_type=_post_apply_km_path_type(apply_op_id),
|
|
created_by="ai_agent_ansible_worker",
|
|
)
|
|
)
|
|
status["learning"] = True
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"ansible_post_apply_learning_writeback_failed",
|
|
incident_id=claim.incident_id,
|
|
catalog_id=claim.catalog_id,
|
|
apply_op_id=apply_op_id,
|
|
error=str(exc),
|
|
)
|
|
return status
|
|
|
|
|
|
async def _send_controlled_apply_telegram_receipt(
|
|
claim: AnsibleCheckModeClaim,
|
|
result: AnsibleRunResult,
|
|
*,
|
|
apply_op_id: str,
|
|
writeback: dict[str, bool],
|
|
project_id: str,
|
|
) -> bool:
|
|
try:
|
|
from src.services.telegram_gateway import get_telegram_gateway
|
|
|
|
response = await get_telegram_gateway().send_controlled_apply_result_receipt(
|
|
incident_id=claim.incident_id,
|
|
catalog_id=claim.catalog_id,
|
|
apply_op_id=apply_op_id,
|
|
playbook_path=claim.apply_playbook_path,
|
|
verification_result=_post_apply_verification_result(result),
|
|
returncode=result.returncode,
|
|
duration_ms=result.duration_ms,
|
|
verifier_written=bool(writeback.get("verification")),
|
|
learning_written=bool(writeback.get("learning")),
|
|
project_id=project_id,
|
|
)
|
|
return bool(response)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"ansible_controlled_apply_telegram_receipt_failed",
|
|
incident_id=claim.incident_id,
|
|
catalog_id=claim.catalog_id,
|
|
apply_op_id=apply_op_id,
|
|
error=str(exc),
|
|
)
|
|
return False
|
|
|
|
|
|
async def backfill_missing_auto_repair_execution_receipts_once(
|
|
*,
|
|
project_id: str = "awoooi",
|
|
window_hours: int = 24,
|
|
limit: int = 10,
|
|
) -> dict[str, Any]:
|
|
"""Create auto_repair_executions receipts for existing Ansible apply rows."""
|
|
|
|
stats = {
|
|
"scanned": 0,
|
|
"written": 0,
|
|
"verification_written": 0,
|
|
"learning_written": 0,
|
|
"skipped": 0,
|
|
"error": None,
|
|
}
|
|
try:
|
|
async with get_db_context(project_id) as db:
|
|
await db.execute(text("SET LOCAL statement_timeout = '5000ms'"))
|
|
result = await db.execute(
|
|
text("""
|
|
SELECT
|
|
apply.op_id::text AS op_id,
|
|
apply.parent_op_id::text AS parent_op_id,
|
|
coalesce(apply.incident_id::text, apply.input ->> 'incident_id') AS incident_id,
|
|
apply.input,
|
|
apply.output,
|
|
apply.dry_run_result,
|
|
apply.error,
|
|
apply.duration_ms,
|
|
apply.status,
|
|
apply.created_at
|
|
FROM automation_operation_log apply
|
|
WHERE apply.operation_type = 'ansible_apply_executed'
|
|
AND apply.created_at >= NOW() - (:window_hours * INTERVAL '1 hour')
|
|
AND (
|
|
NOT EXISTS (
|
|
SELECT 1
|
|
FROM auto_repair_executions existing
|
|
WHERE existing.created_at >= NOW() - (:window_hours * INTERVAL '1 hour')
|
|
AND existing.triggered_by = 'ansible_controlled_apply'
|
|
AND existing.executed_steps::text LIKE '%' || apply.op_id::text || '%'
|
|
)
|
|
OR NOT EXISTS (
|
|
SELECT 1
|
|
FROM incident_evidence evidence
|
|
WHERE evidence.incident_id = coalesce(
|
|
apply.incident_id::text,
|
|
apply.input ->> 'incident_id'
|
|
)
|
|
AND evidence.post_execution_state ->> 'apply_op_id' = apply.op_id::text
|
|
)
|
|
OR NOT EXISTS (
|
|
SELECT 1
|
|
FROM knowledge_entries km
|
|
WHERE km.related_incident_id = coalesce(
|
|
apply.incident_id::text,
|
|
apply.input ->> 'incident_id'
|
|
)
|
|
AND km.path_type = 'ansible_apply_receipt:' || left(apply.op_id::text, 8)
|
|
)
|
|
)
|
|
ORDER BY apply.created_at DESC
|
|
LIMIT :limit
|
|
"""),
|
|
{"window_hours": max(1, window_hours), "limit": max(1, limit)},
|
|
)
|
|
rows = [dict(row) for row in result.mappings().all()]
|
|
stats["scanned"] = len(rows)
|
|
for row in rows:
|
|
reconstructed = _claim_from_apply_operation_row(row)
|
|
if reconstructed is None:
|
|
stats["skipped"] += 1
|
|
continue
|
|
claim, result = reconstructed
|
|
if await _record_auto_repair_execution_receipt(
|
|
claim,
|
|
result,
|
|
apply_op_id=str(row.get("op_id") or ""),
|
|
project_id=project_id,
|
|
):
|
|
stats["written"] += 1
|
|
else:
|
|
stats["skipped"] += 1
|
|
writeback = await _record_post_apply_verifier_and_learning(
|
|
claim,
|
|
result,
|
|
apply_op_id=str(row.get("op_id") or ""),
|
|
project_id=project_id,
|
|
)
|
|
if writeback.get("verification"):
|
|
stats["verification_written"] += 1
|
|
if writeback.get("learning"):
|
|
stats["learning_written"] += 1
|
|
except Exception as exc:
|
|
stats["error"] = f"{type(exc).__name__}: {exc}"[:500]
|
|
logger.warning("ansible_auto_repair_execution_receipt_backfill_failed", **stats)
|
|
return stats
|
|
|
|
|
|
async def claim_pending_check_modes(
|
|
*,
|
|
project_id: str = "awoooi",
|
|
limit: int = 1,
|
|
candidate_max_age_hours: int | None = None,
|
|
) -> list[AnsibleCheckModeClaim]:
|
|
"""Claim pending Ansible candidates by inserting pending check-mode rows."""
|
|
|
|
claims: list[AnsibleCheckModeClaim] = []
|
|
max_age_hours = candidate_max_age_hours or settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS
|
|
async with get_db_context(project_id) as db:
|
|
result = await db.execute(
|
|
text("""
|
|
SELECT
|
|
candidate.op_id,
|
|
candidate.input
|
|
FROM automation_operation_log candidate
|
|
WHERE candidate.operation_type = 'ansible_candidate_matched'
|
|
AND candidate.status = 'dry_run'
|
|
AND candidate.input ->> 'executor' = 'ansible'
|
|
AND candidate.created_at >= NOW() - (:candidate_max_age_hours * INTERVAL '1 hour')
|
|
AND COALESCE((candidate.dry_run_result ->> 'check_mode_executed')::boolean, false) = false
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM automation_operation_log existing
|
|
WHERE existing.parent_op_id = candidate.op_id
|
|
AND existing.operation_type IN (
|
|
'ansible_check_mode_executed',
|
|
'ansible_execution_skipped'
|
|
)
|
|
)
|
|
ORDER BY candidate.created_at ASC
|
|
LIMIT :limit
|
|
FOR UPDATE SKIP LOCKED
|
|
"""),
|
|
{
|
|
"limit": max(1, limit),
|
|
"candidate_max_age_hours": max(1, max_age_hours),
|
|
},
|
|
)
|
|
rows = result.mappings().all()
|
|
for row in rows:
|
|
source_op_id = str(row["op_id"])
|
|
candidate_input = _json_loads(row["input"])
|
|
try:
|
|
claim_input = build_ansible_check_mode_claim_input(
|
|
source_candidate_op_id=source_op_id,
|
|
candidate_input=candidate_input,
|
|
)
|
|
except ValueError as exc:
|
|
await _insert_skipped_candidate(
|
|
db,
|
|
source_candidate_op_id=source_op_id,
|
|
candidate_input=candidate_input,
|
|
reason=str(exc),
|
|
)
|
|
continue
|
|
inserted = await db.execute(
|
|
text("""
|
|
INSERT INTO automation_operation_log (
|
|
operation_type, actor, status, incident_id,
|
|
input, output, dry_run_result,
|
|
parent_op_id, tags
|
|
) VALUES (
|
|
'ansible_check_mode_executed',
|
|
'ansible_check_mode_worker',
|
|
'pending',
|
|
:incident_db_id,
|
|
CAST(:input AS jsonb),
|
|
'{}'::jsonb,
|
|
CAST(:dry_run_result AS jsonb),
|
|
CAST(:parent_op_id AS uuid),
|
|
:tags
|
|
)
|
|
RETURNING op_id
|
|
"""),
|
|
{
|
|
"incident_id": _incident_id_from_payload(claim_input),
|
|
"incident_db_id": _automation_operation_log_incident_id(
|
|
_incident_id_from_payload(claim_input)
|
|
),
|
|
"input": json.dumps(claim_input, ensure_ascii=False),
|
|
"dry_run_result": json.dumps({
|
|
"check_mode_executed": False,
|
|
"apply_executed": False,
|
|
"claim_state": "claimed",
|
|
"controlled_apply_allowed": bool(
|
|
claim_input.get("controlled_apply_allowed")
|
|
),
|
|
"controlled_apply_blocker": claim_input.get("controlled_apply_blocker"),
|
|
}, ensure_ascii=False),
|
|
"parent_op_id": source_op_id,
|
|
"tags": [
|
|
"ansible",
|
|
"check_mode",
|
|
"pending",
|
|
"controlled_apply_allowed"
|
|
if claim_input.get("controlled_apply_allowed")
|
|
else "controlled_apply_blocked",
|
|
],
|
|
},
|
|
)
|
|
op_id = str(inserted.scalar_one())
|
|
claims.append(
|
|
AnsibleCheckModeClaim(
|
|
op_id=op_id,
|
|
source_candidate_op_id=source_op_id,
|
|
incident_id=str(claim_input.get("incident_id") or ""),
|
|
catalog_id=str(claim_input["catalog_id"]),
|
|
playbook_path=str(claim_input["playbook_path"]),
|
|
apply_playbook_path=str(claim_input["apply_playbook_path"]),
|
|
inventory_hosts=tuple(str(host) for host in claim_input["inventory_hosts"]),
|
|
risk_level=str(claim_input.get("risk_level") or ""),
|
|
input_payload=claim_input,
|
|
)
|
|
)
|
|
return claims
|
|
|
|
|
|
async def recent_ansible_transport_blockers(
|
|
*,
|
|
project_id: str = "awoooi",
|
|
cooldown_seconds: int | None = None,
|
|
include_repair_forced_command_blocker: bool | None = None,
|
|
) -> list[str]:
|
|
"""Return transport blockers observed from recent failed check-mode rows."""
|
|
|
|
cooldown = cooldown_seconds or settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_COOLDOWN_SECONDS
|
|
include_forced_blocker = (
|
|
_uses_repair_forced_command_transport()
|
|
if include_repair_forced_command_blocker is None
|
|
else include_repair_forced_command_blocker
|
|
)
|
|
async with get_db_context(project_id) as db:
|
|
result = await db.execute(
|
|
text("""
|
|
SELECT
|
|
coalesce(output::text, '') AS output_text,
|
|
coalesce(dry_run_result::text, '') AS dry_run_text,
|
|
coalesce(error, '') AS error_text,
|
|
coalesce(stderr_feed_back, '') AS stderr_text
|
|
FROM automation_operation_log
|
|
WHERE operation_type = 'ansible_check_mode_executed'
|
|
AND status = 'failed'
|
|
AND created_at >= NOW() - (:cooldown_seconds * INTERVAL '1 second')
|
|
ORDER BY created_at DESC
|
|
LIMIT 20
|
|
"""),
|
|
{"cooldown_seconds": max(60, cooldown)},
|
|
)
|
|
blockers: set[str] = set()
|
|
for row in result.mappings().all():
|
|
detected = detect_ansible_transport_blockers(
|
|
row.get("output_text"),
|
|
row.get("dry_run_text"),
|
|
row.get("error_text"),
|
|
row.get("stderr_text"),
|
|
)
|
|
blockers.update(
|
|
blocker
|
|
for blocker in detected
|
|
if blocker != FORCED_COMMAND_BLOCKER or include_forced_blocker
|
|
)
|
|
return sorted(blockers)
|
|
|
|
|
|
async def _insert_skipped_candidate(
|
|
db: Any,
|
|
*,
|
|
source_candidate_op_id: str,
|
|
candidate_input: dict[str, Any],
|
|
reason: str,
|
|
) -> None:
|
|
input_payload = {
|
|
"incident_id": _incident_id_from_payload(candidate_input),
|
|
"executor": "ansible",
|
|
"execution_backend": "ansible",
|
|
"execution_mode": "check_mode",
|
|
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
|
|
"check_mode": True,
|
|
"apply_enabled": False,
|
|
"source_candidate_op_id": source_candidate_op_id,
|
|
"not_used_reason": reason,
|
|
}
|
|
await db.execute(
|
|
text("""
|
|
INSERT INTO automation_operation_log (
|
|
operation_type, actor, status, incident_id,
|
|
input, output, dry_run_result,
|
|
parent_op_id, tags
|
|
) VALUES (
|
|
'ansible_execution_skipped',
|
|
'ansible_check_mode_worker',
|
|
'dry_run',
|
|
:incident_db_id,
|
|
CAST(:input AS jsonb),
|
|
CAST(:output AS jsonb),
|
|
CAST(:dry_run_result AS jsonb),
|
|
CAST(:parent_op_id AS uuid),
|
|
:tags
|
|
)
|
|
"""),
|
|
{
|
|
"incident_id": _incident_id_from_payload(input_payload),
|
|
"incident_db_id": _automation_operation_log_incident_id(
|
|
_incident_id_from_payload(input_payload)
|
|
),
|
|
"input": json.dumps(input_payload, ensure_ascii=False),
|
|
"output": json.dumps({
|
|
"not_used_reason": reason,
|
|
"decision_effect": "skipped_before_runtime",
|
|
}, ensure_ascii=False),
|
|
"dry_run_result": json.dumps({
|
|
"check_mode_executed": False,
|
|
"apply_executed": False,
|
|
"skipped": True,
|
|
"reason": reason,
|
|
}, ensure_ascii=False),
|
|
"parent_op_id": source_candidate_op_id,
|
|
"tags": ["ansible", "check_mode", "skipped", "apply_locked"],
|
|
},
|
|
)
|
|
|
|
|
|
async def finalize_check_mode_claim(
|
|
claim: AnsibleCheckModeClaim,
|
|
result: AnsibleRunResult,
|
|
*,
|
|
project_id: str = "awoooi",
|
|
) -> None:
|
|
controlled_apply_allowed = bool(claim.input_payload.get("controlled_apply_allowed"))
|
|
controlled_apply_blocker = (
|
|
str(claim.input_payload.get("controlled_apply_blocker"))
|
|
if claim.input_payload.get("controlled_apply_blocker")
|
|
else None
|
|
)
|
|
status, output, dry_run_result, error = _build_result_payload(
|
|
result,
|
|
controlled_apply_allowed=controlled_apply_allowed,
|
|
controlled_apply_blocker=controlled_apply_blocker,
|
|
)
|
|
async with get_db_context(project_id) as db:
|
|
await db.execute(
|
|
text("""
|
|
UPDATE automation_operation_log
|
|
SET status = :status,
|
|
output = CAST(:output AS jsonb),
|
|
dry_run_result = CAST(:dry_run_result AS jsonb),
|
|
error = :error,
|
|
duration_ms = :duration_ms,
|
|
stderr_feed_back = :stderr
|
|
WHERE op_id = CAST(:op_id AS uuid)
|
|
"""),
|
|
{
|
|
"status": status,
|
|
"output": json.dumps(output, ensure_ascii=False),
|
|
"dry_run_result": json.dumps(dry_run_result, ensure_ascii=False),
|
|
"error": _tail(error or "", 2000) or None,
|
|
"duration_ms": result.duration_ms,
|
|
"stderr": _tail(result.stderr, _STDERR_LIMIT),
|
|
"op_id": claim.op_id,
|
|
},
|
|
)
|
|
|
|
|
|
async def run_claimed_check_mode(
|
|
claim: AnsibleCheckModeClaim,
|
|
*,
|
|
timeout_seconds: int,
|
|
project_id: str = "awoooi",
|
|
) -> AnsibleRunResult:
|
|
try:
|
|
spec = build_ansible_check_mode_command(
|
|
playbook_path=claim.playbook_path,
|
|
inventory_hosts=claim.inventory_hosts,
|
|
)
|
|
result = await _run_ansible_command(spec, timeout_seconds=timeout_seconds)
|
|
except Exception as exc:
|
|
result = AnsibleRunResult(
|
|
returncode=1,
|
|
stdout="",
|
|
stderr=f"ansible_check_mode_runtime_error: {exc}",
|
|
duration_ms=0,
|
|
)
|
|
await finalize_check_mode_claim(claim, result, project_id=project_id)
|
|
logger.info(
|
|
"ansible_check_mode_claim_completed",
|
|
op_id=claim.op_id,
|
|
source_candidate_op_id=claim.source_candidate_op_id,
|
|
incident_id=claim.incident_id,
|
|
catalog_id=claim.catalog_id,
|
|
returncode=result.returncode,
|
|
timed_out=result.timed_out,
|
|
)
|
|
return result
|
|
|
|
|
|
async def run_controlled_apply_for_claim(
|
|
claim: AnsibleCheckModeClaim,
|
|
*,
|
|
timeout_seconds: int,
|
|
project_id: str = "awoooi",
|
|
) -> AnsibleRunResult | None:
|
|
"""Execute apply for an allowlisted claim after check-mode succeeds."""
|
|
|
|
if claim.input_payload.get("controlled_apply_allowed") is not True:
|
|
logger.info(
|
|
"ansible_controlled_apply_blocked",
|
|
op_id=claim.op_id,
|
|
source_candidate_op_id=claim.source_candidate_op_id,
|
|
incident_id=claim.incident_id,
|
|
catalog_id=claim.catalog_id,
|
|
blocker=claim.input_payload.get("controlled_apply_blocker"),
|
|
)
|
|
return None
|
|
|
|
async with get_db_context(project_id) as db:
|
|
inserted = await db.execute(
|
|
text("""
|
|
INSERT INTO automation_operation_log (
|
|
operation_type, actor, status, incident_id,
|
|
input, output, dry_run_result,
|
|
parent_op_id, tags
|
|
) VALUES (
|
|
'ansible_apply_executed',
|
|
'ansible_controlled_apply_worker',
|
|
'pending',
|
|
:incident_db_id,
|
|
CAST(:input AS jsonb),
|
|
'{}'::jsonb,
|
|
CAST(:dry_run_result AS jsonb),
|
|
CAST(:parent_op_id AS uuid),
|
|
:tags
|
|
)
|
|
RETURNING op_id
|
|
"""),
|
|
{
|
|
"incident_id": claim.incident_id,
|
|
"incident_db_id": _automation_operation_log_incident_id(claim.incident_id),
|
|
"input": json.dumps({
|
|
**claim.input_payload,
|
|
"execution_mode": "controlled_apply",
|
|
"check_mode_op_id": claim.op_id,
|
|
"playbook_path": claim.apply_playbook_path,
|
|
"check_mode_playbook_path": claim.playbook_path,
|
|
"apply_playbook_path": claim.apply_playbook_path,
|
|
"check_mode": False,
|
|
"apply_enabled": True,
|
|
"approval_required_before_apply": False,
|
|
"controlled_apply_allowed": True,
|
|
}, ensure_ascii=False),
|
|
"dry_run_result": json.dumps({
|
|
"check_mode_executed_before_apply": True,
|
|
"apply_executed": False,
|
|
"claim_state": "claimed",
|
|
"source_check_mode_op_id": claim.op_id,
|
|
}, ensure_ascii=False),
|
|
"parent_op_id": claim.op_id,
|
|
"tags": [
|
|
"ansible",
|
|
"controlled_apply",
|
|
str(claim.risk_level or "unknown").lower(),
|
|
"ai_agent_auto_execution",
|
|
],
|
|
},
|
|
)
|
|
apply_op_id = str(inserted.scalar_one())
|
|
|
|
try:
|
|
spec = build_ansible_apply_command(
|
|
playbook_path=claim.apply_playbook_path,
|
|
inventory_hosts=claim.inventory_hosts,
|
|
)
|
|
result = await _run_ansible_command(spec, timeout_seconds=timeout_seconds)
|
|
except Exception as exc:
|
|
result = AnsibleRunResult(
|
|
returncode=1,
|
|
stdout="",
|
|
stderr=f"ansible_controlled_apply_runtime_error: {exc}",
|
|
duration_ms=0,
|
|
)
|
|
|
|
status, output, dry_run_result, error = _build_apply_result_payload(result)
|
|
async with get_db_context(project_id) as db:
|
|
await db.execute(
|
|
text("""
|
|
UPDATE automation_operation_log
|
|
SET status = :status,
|
|
output = CAST(:output AS jsonb),
|
|
dry_run_result = CAST(:dry_run_result AS jsonb),
|
|
error = :error,
|
|
duration_ms = :duration_ms,
|
|
stderr_feed_back = :stderr
|
|
WHERE op_id = CAST(:op_id AS uuid)
|
|
"""),
|
|
{
|
|
"status": status,
|
|
"output": json.dumps(output, ensure_ascii=False),
|
|
"dry_run_result": json.dumps(dry_run_result, ensure_ascii=False),
|
|
"error": _tail(error or "", 2000) or None,
|
|
"duration_ms": result.duration_ms,
|
|
"stderr": _tail(result.stderr, _STDERR_LIMIT),
|
|
"op_id": apply_op_id,
|
|
},
|
|
)
|
|
receipt_written = await _record_auto_repair_execution_receipt(
|
|
claim,
|
|
result,
|
|
apply_op_id=apply_op_id,
|
|
project_id=project_id,
|
|
)
|
|
writeback = await _record_post_apply_verifier_and_learning(
|
|
claim,
|
|
result,
|
|
apply_op_id=apply_op_id,
|
|
project_id=project_id,
|
|
)
|
|
telegram_receipt_sent = await _send_controlled_apply_telegram_receipt(
|
|
claim,
|
|
result,
|
|
apply_op_id=apply_op_id,
|
|
writeback=writeback,
|
|
project_id=project_id,
|
|
)
|
|
|
|
logger.info(
|
|
"ansible_controlled_apply_completed",
|
|
op_id=apply_op_id,
|
|
check_mode_op_id=claim.op_id,
|
|
source_candidate_op_id=claim.source_candidate_op_id,
|
|
incident_id=claim.incident_id,
|
|
catalog_id=claim.catalog_id,
|
|
returncode=result.returncode,
|
|
timed_out=result.timed_out,
|
|
auto_repair_receipt_written=receipt_written,
|
|
post_apply_verification_written=writeback.get("verification"),
|
|
post_apply_learning_written=writeback.get("learning"),
|
|
telegram_receipt_sent=telegram_receipt_sent,
|
|
)
|
|
return result
|
|
|
|
|
|
async def run_pending_check_modes_once(
|
|
*,
|
|
project_id: str = "awoooi",
|
|
limit: int = 1,
|
|
timeout_seconds: int | None = None,
|
|
) -> dict[str, Any]:
|
|
blockers = _runtime_blockers()
|
|
if blockers:
|
|
logger.warning("ansible_check_mode_runtime_blocked", blockers=blockers)
|
|
return {"claimed": 0, "completed": 0, "failed": 0, "blockers": blockers}
|
|
transport_blockers = await recent_ansible_transport_blockers(project_id=project_id)
|
|
if transport_blockers:
|
|
logger.warning("ansible_check_mode_transport_blocked", blockers=transport_blockers)
|
|
return {"claimed": 0, "completed": 0, "failed": 0, "blockers": transport_blockers}
|
|
|
|
claims = await claim_pending_check_modes(
|
|
project_id=project_id,
|
|
limit=limit,
|
|
candidate_max_age_hours=settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS,
|
|
)
|
|
completed = 0
|
|
failed = 0
|
|
apply_completed = 0
|
|
apply_failed = 0
|
|
apply_blocked = 0
|
|
for claim in claims:
|
|
result = await run_claimed_check_mode(
|
|
claim,
|
|
timeout_seconds=timeout_seconds or settings.AWOOOP_ANSIBLE_CHECK_MODE_TIMEOUT_SECONDS,
|
|
project_id=project_id,
|
|
)
|
|
completed += 1
|
|
if result.returncode != 0:
|
|
failed += 1
|
|
continue
|
|
apply_result = await run_controlled_apply_for_claim(
|
|
claim,
|
|
timeout_seconds=settings.AWOOOP_ANSIBLE_CONTROLLED_APPLY_TIMEOUT_SECONDS,
|
|
project_id=project_id,
|
|
)
|
|
if apply_result is None:
|
|
apply_blocked += 1
|
|
else:
|
|
apply_completed += 1
|
|
if apply_result.returncode != 0:
|
|
apply_failed += 1
|
|
return {
|
|
"claimed": len(claims),
|
|
"completed": completed,
|
|
"failed": failed,
|
|
"apply_completed": apply_completed,
|
|
"apply_failed": apply_failed,
|
|
"apply_blocked": apply_blocked,
|
|
"blockers": [],
|
|
}
|