f615ac506e
Some checks failed
Ansible Lint / lint (push) Successful in 32s
CD Pipeline / tests (push) Successful in 1m16s
Code Review / ai-code-review (push) Successful in 14s
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
662 lines
24 KiB
Python
662 lines
24 KiB
Python
"""Safe Ansible check-mode executor for AwoooP truth-chain evidence.
|
|
|
|
This service is deliberately dry-run only. It claims pending
|
|
``ansible_candidate_matched`` AOL rows, runs ``ansible-playbook --check --diff``,
|
|
and writes the result back as ``ansible_check_mode_executed``. It never enables
|
|
apply and never writes auto_repair_executions.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import structlog
|
|
from sqlalchemy import text
|
|
|
|
from src.core.config import settings
|
|
from src.db.base import get_db_context
|
|
from src.services.awooop_ansible_audit_service import get_ansible_catalog_item
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
|
|
_SAFE_HOST_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
|
|
_PLAYBOOK_PREFIX = Path("infra/ansible/playbooks")
|
|
_STDOUT_LIMIT = 20_000
|
|
_STDERR_LIMIT = 12_000
|
|
FORCED_COMMAND_BLOCKER = "ansible_repair_ssh_forced_command_denies_ansible_bootstrap"
|
|
REPAIR_FORCED_COMMAND_KEY_PATH = Path("/etc/repair-ssh/id_ed25519")
|
|
REPAIR_FORCED_COMMAND_KNOWN_HOSTS_PATH = Path("/etc/repair-known-hosts/known_hosts")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AnsibleCheckModeClaim:
|
|
op_id: str
|
|
source_candidate_op_id: str
|
|
incident_id: str
|
|
catalog_id: str
|
|
playbook_path: str
|
|
inventory_hosts: tuple[str, ...]
|
|
input_payload: dict[str, Any]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AnsibleCommandSpec:
|
|
command: list[str]
|
|
cwd: Path
|
|
env: dict[str, str]
|
|
playbook_abs_path: Path
|
|
inventory_abs_path: Path
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AnsibleRunResult:
|
|
returncode: int
|
|
stdout: str
|
|
stderr: str
|
|
duration_ms: int
|
|
timed_out: bool = False
|
|
|
|
|
|
def _tail(text_value: str, limit: int) -> str:
|
|
if len(text_value) <= limit:
|
|
return text_value
|
|
return text_value[-limit:]
|
|
|
|
|
|
def _json_loads(value: Any) -> dict[str, Any]:
|
|
if isinstance(value, dict):
|
|
return value
|
|
if isinstance(value, str):
|
|
try:
|
|
parsed = json.loads(value)
|
|
except json.JSONDecodeError:
|
|
return {}
|
|
return parsed if isinstance(parsed, dict) else {}
|
|
return {}
|
|
|
|
|
|
def _incident_id_from_payload(payload: dict[str, Any]) -> str:
|
|
return str(payload.get("incident_id") or "").strip()
|
|
|
|
|
|
def _automation_operation_log_incident_id(value: str) -> int | None:
|
|
# ADR-090 keeps external INC-* ids in input JSON; the DB column is BIGINT.
|
|
normalized = str(value or "").strip()
|
|
return int(normalized) if normalized.isdigit() else None
|
|
|
|
|
|
def _check_mode_ssh_key_path() -> Path:
|
|
return Path(settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH)
|
|
|
|
|
|
def _check_mode_known_hosts_path() -> Path:
|
|
return Path(settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH)
|
|
|
|
|
|
def _uses_repair_forced_command_transport(key_path: Path | None = None) -> bool:
|
|
return (key_path or _check_mode_ssh_key_path()) == REPAIR_FORCED_COMMAND_KEY_PATH
|
|
|
|
|
|
def detect_ansible_transport_blockers(*values: Any) -> list[str]:
|
|
combined = " ".join(str(value or "") for value in values)
|
|
blockers: list[str] = []
|
|
if "REPAIR_DENIED:invalid_command" in combined:
|
|
blockers.append(FORCED_COMMAND_BLOCKER)
|
|
return blockers
|
|
|
|
|
|
def _playbook_roots(module_path: Path | None = None) -> list[Path]:
|
|
resolved_module_path = (module_path or Path(__file__)).resolve()
|
|
return [
|
|
Path("/app/infra/ansible"),
|
|
Path.cwd() / "infra" / "ansible",
|
|
*(parent / "infra" / "ansible" for parent in resolved_module_path.parents),
|
|
]
|
|
|
|
|
|
def _runtime_blockers(
|
|
*,
|
|
playbook_root: Path | None = None,
|
|
check_mode_ssh_key_path: Path | None = None,
|
|
check_mode_known_hosts_path: Path | None = None,
|
|
) -> list[str]:
|
|
root = playbook_root or next((path for path in _playbook_roots() if path.exists()), None)
|
|
ssh_key_path = check_mode_ssh_key_path or _check_mode_ssh_key_path()
|
|
known_hosts_path = check_mode_known_hosts_path or _check_mode_known_hosts_path()
|
|
blockers: list[str] = []
|
|
if shutil.which("ansible-playbook") is None:
|
|
blockers.append("ansible_playbook_binary_missing")
|
|
if root is None:
|
|
blockers.append("ansible_playbook_catalog_missing")
|
|
elif not (root / "inventory" / "hosts.yml").exists():
|
|
blockers.append("ansible_inventory_missing")
|
|
if not ssh_key_path.is_file() or not os.access(ssh_key_path, os.R_OK):
|
|
blockers.append("ansible_check_mode_ssh_key_missing")
|
|
if not known_hosts_path.is_file() or not os.access(known_hosts_path, os.R_OK):
|
|
blockers.append("ansible_check_mode_known_hosts_missing")
|
|
return blockers
|
|
|
|
|
|
def _safe_candidate(input_payload: dict[str, Any]) -> dict[str, Any]:
|
|
candidates = input_payload.get("executor_candidates")
|
|
if not isinstance(candidates, list) or not candidates:
|
|
raise ValueError("missing_executor_candidates")
|
|
|
|
for candidate in candidates:
|
|
if not isinstance(candidate, dict):
|
|
continue
|
|
catalog_id = str(candidate.get("catalog_id") or "")
|
|
catalog_item = get_ansible_catalog_item(catalog_id)
|
|
if not catalog_item:
|
|
continue
|
|
if catalog_item.get("supports_check_mode") is not True:
|
|
continue
|
|
if catalog_item.get("auto_apply_enabled") is True:
|
|
continue
|
|
catalog_playbook_path = str(catalog_item.get("playbook_path") or "")
|
|
candidate_playbook_path = str(candidate.get("playbook_path") or catalog_playbook_path)
|
|
check_mode_playbook_path = str(
|
|
candidate.get("check_mode_playbook_path")
|
|
or catalog_item.get("check_mode_playbook_path")
|
|
or catalog_playbook_path
|
|
)
|
|
if candidate_playbook_path not in {catalog_playbook_path, check_mode_playbook_path}:
|
|
continue
|
|
inventory_hosts = candidate.get("inventory_hosts") or catalog_item.get("inventory_hosts") or []
|
|
if (
|
|
isinstance(inventory_hosts, list)
|
|
and inventory_hosts
|
|
and all(isinstance(host, str) and _SAFE_HOST_RE.fullmatch(host) for host in inventory_hosts)
|
|
):
|
|
return {
|
|
"catalog_id": catalog_id,
|
|
"playbook_path": check_mode_playbook_path,
|
|
"catalog_playbook_path": catalog_playbook_path,
|
|
"source_candidate_playbook_path": candidate_playbook_path,
|
|
"check_mode_playbook_path": check_mode_playbook_path,
|
|
"inventory_hosts": tuple(inventory_hosts),
|
|
"risk_level": str(candidate.get("risk_level") or catalog_item.get("risk_level") or ""),
|
|
}
|
|
raise ValueError("no_safe_check_mode_candidate")
|
|
|
|
|
|
def build_ansible_check_mode_claim_input(
|
|
*,
|
|
source_candidate_op_id: str,
|
|
candidate_input: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
safe = _safe_candidate(candidate_input)
|
|
incident_id = _incident_id_from_payload(candidate_input)
|
|
return {
|
|
"incident_id": incident_id,
|
|
"executor": "ansible",
|
|
"execution_backend": "ansible",
|
|
"execution_mode": "check_mode",
|
|
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
|
|
"check_mode": True,
|
|
"diff": True,
|
|
"apply_enabled": False,
|
|
"approval_required_before_apply": True,
|
|
"source_candidate_op_id": source_candidate_op_id,
|
|
"catalog_id": safe["catalog_id"],
|
|
"playbook_path": safe["playbook_path"],
|
|
"catalog_playbook_path": safe["catalog_playbook_path"],
|
|
"source_candidate_playbook_path": safe["source_candidate_playbook_path"],
|
|
"check_mode_playbook_path": safe["check_mode_playbook_path"],
|
|
"inventory_hosts": list(safe["inventory_hosts"]),
|
|
"risk_level": safe["risk_level"],
|
|
}
|
|
|
|
|
|
def _resolve_playbook_path(playbook_root: Path, playbook_path: str) -> Path:
|
|
relative = Path(playbook_path)
|
|
if relative.is_absolute() or not str(relative).startswith(str(_PLAYBOOK_PREFIX)):
|
|
raise ValueError("unsafe_playbook_path")
|
|
repo_root = playbook_root.parent.parent
|
|
resolved = (repo_root / relative).resolve()
|
|
allowed_root = (repo_root / _PLAYBOOK_PREFIX).resolve()
|
|
if allowed_root not in resolved.parents:
|
|
raise ValueError("playbook_outside_catalog")
|
|
if resolved.suffix not in {".yml", ".yaml"} or not resolved.exists():
|
|
raise ValueError("playbook_not_found")
|
|
return resolved
|
|
|
|
|
|
def build_ansible_check_mode_command(
|
|
*,
|
|
playbook_path: str,
|
|
inventory_hosts: tuple[str, ...],
|
|
playbook_root: Path | None = None,
|
|
check_mode_ssh_key_path: Path | None = None,
|
|
check_mode_known_hosts_path: Path | None = None,
|
|
) -> AnsibleCommandSpec:
|
|
root = playbook_root or next((path for path in _playbook_roots() if path.exists()), None)
|
|
if root is None:
|
|
raise ValueError("ansible_playbook_catalog_missing")
|
|
inventory_path = (root / "inventory" / "hosts.yml").resolve()
|
|
if not inventory_path.exists():
|
|
raise ValueError("ansible_inventory_missing")
|
|
if not inventory_hosts or not all(_SAFE_HOST_RE.fullmatch(host) for host in inventory_hosts):
|
|
raise ValueError("unsafe_inventory_hosts")
|
|
|
|
playbook_abs = _resolve_playbook_path(root, playbook_path)
|
|
ssh_key_path = check_mode_ssh_key_path or _check_mode_ssh_key_path()
|
|
known_hosts_path = check_mode_known_hosts_path or _check_mode_known_hosts_path()
|
|
ssh_common_args = (
|
|
f"-o UserKnownHostsFile={known_hosts_path} "
|
|
"-o IdentitiesOnly=yes -o BatchMode=yes"
|
|
)
|
|
extra_vars = {
|
|
"ansible_ssh_private_key_file": str(ssh_key_path),
|
|
"ansible_ssh_common_args": ssh_common_args,
|
|
}
|
|
command = [
|
|
"ansible-playbook",
|
|
"-i",
|
|
str(inventory_path),
|
|
str(playbook_abs),
|
|
"--check",
|
|
"--diff",
|
|
"--limit",
|
|
",".join(inventory_hosts),
|
|
"--extra-vars",
|
|
json.dumps(extra_vars, ensure_ascii=False, separators=(",", ":")),
|
|
]
|
|
env = {
|
|
**os.environ,
|
|
"ANSIBLE_HOST_KEY_CHECKING": "true",
|
|
"ANSIBLE_RETRY_FILES_ENABLED": "false",
|
|
}
|
|
return AnsibleCommandSpec(
|
|
command=command,
|
|
cwd=root,
|
|
env=env,
|
|
playbook_abs_path=playbook_abs,
|
|
inventory_abs_path=inventory_path,
|
|
)
|
|
|
|
|
|
async def _run_ansible_command(spec: AnsibleCommandSpec, *, timeout_seconds: int) -> AnsibleRunResult:
|
|
started = time.monotonic()
|
|
process = await asyncio.create_subprocess_exec(
|
|
*spec.command,
|
|
cwd=str(spec.cwd),
|
|
env=spec.env,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
timed_out = False
|
|
try:
|
|
stdout_bytes, stderr_bytes = await asyncio.wait_for(
|
|
process.communicate(),
|
|
timeout=timeout_seconds,
|
|
)
|
|
except TimeoutError:
|
|
timed_out = True
|
|
process.kill()
|
|
stdout_bytes, stderr_bytes = await process.communicate()
|
|
duration_ms = int((time.monotonic() - started) * 1000)
|
|
return AnsibleRunResult(
|
|
returncode=124 if timed_out else int(process.returncode or 0),
|
|
stdout=stdout_bytes.decode("utf-8", "replace"),
|
|
stderr=stderr_bytes.decode("utf-8", "replace"),
|
|
duration_ms=duration_ms,
|
|
timed_out=timed_out,
|
|
)
|
|
|
|
|
|
def _build_result_payload(result: AnsibleRunResult) -> tuple[str, dict[str, Any], dict[str, Any], str | None]:
|
|
status = "success" if result.returncode == 0 else "failed"
|
|
stdout_tail = _tail(result.stdout, _STDOUT_LIMIT)
|
|
stderr_tail = _tail(result.stderr, _STDERR_LIMIT)
|
|
output = {
|
|
"executor": "ansible",
|
|
"execution_mode": "check_mode",
|
|
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
|
|
"ssh_key_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH,
|
|
"known_hosts_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH,
|
|
"check_mode": True,
|
|
"apply_enabled": False,
|
|
"approval_required_before_apply": True,
|
|
"returncode": result.returncode,
|
|
"timed_out": result.timed_out,
|
|
"stdout_tail": stdout_tail,
|
|
"stderr_tail": stderr_tail,
|
|
"next_required_step": "approval_required_before_ansible_apply",
|
|
}
|
|
dry_run_result = {
|
|
"check_mode_executed": True,
|
|
"apply_executed": False,
|
|
"safe_to_apply_without_approval": False,
|
|
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
|
|
"ssh_key_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH,
|
|
"known_hosts_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH,
|
|
"returncode": result.returncode,
|
|
"timed_out": result.timed_out,
|
|
"stdout_tail": stdout_tail,
|
|
"stderr_tail": stderr_tail,
|
|
}
|
|
error = None if result.returncode == 0 else (stderr_tail or f"ansible_check_mode_failed_rc_{result.returncode}")
|
|
return status, output, dry_run_result, error
|
|
|
|
|
|
async def claim_pending_check_modes(
|
|
*,
|
|
project_id: str = "awoooi",
|
|
limit: int = 1,
|
|
candidate_max_age_hours: int | None = None,
|
|
) -> list[AnsibleCheckModeClaim]:
|
|
"""Claim pending Ansible candidates by inserting pending check-mode rows."""
|
|
|
|
claims: list[AnsibleCheckModeClaim] = []
|
|
max_age_hours = candidate_max_age_hours or settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS
|
|
async with get_db_context(project_id) as db:
|
|
result = await db.execute(
|
|
text("""
|
|
SELECT
|
|
candidate.op_id,
|
|
candidate.input
|
|
FROM automation_operation_log candidate
|
|
WHERE candidate.operation_type = 'ansible_candidate_matched'
|
|
AND candidate.status = 'dry_run'
|
|
AND candidate.input ->> 'executor' = 'ansible'
|
|
AND candidate.created_at >= NOW() - (:candidate_max_age_hours * INTERVAL '1 hour')
|
|
AND COALESCE((candidate.dry_run_result ->> 'check_mode_executed')::boolean, false) = false
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM automation_operation_log existing
|
|
WHERE existing.parent_op_id = candidate.op_id
|
|
AND existing.operation_type IN (
|
|
'ansible_check_mode_executed',
|
|
'ansible_execution_skipped'
|
|
)
|
|
)
|
|
ORDER BY candidate.created_at ASC
|
|
LIMIT :limit
|
|
FOR UPDATE SKIP LOCKED
|
|
"""),
|
|
{
|
|
"limit": max(1, limit),
|
|
"candidate_max_age_hours": max(1, max_age_hours),
|
|
},
|
|
)
|
|
rows = result.mappings().all()
|
|
for row in rows:
|
|
source_op_id = str(row["op_id"])
|
|
candidate_input = _json_loads(row["input"])
|
|
try:
|
|
claim_input = build_ansible_check_mode_claim_input(
|
|
source_candidate_op_id=source_op_id,
|
|
candidate_input=candidate_input,
|
|
)
|
|
except ValueError as exc:
|
|
await _insert_skipped_candidate(
|
|
db,
|
|
source_candidate_op_id=source_op_id,
|
|
candidate_input=candidate_input,
|
|
reason=str(exc),
|
|
)
|
|
continue
|
|
inserted = await db.execute(
|
|
text("""
|
|
INSERT INTO automation_operation_log (
|
|
operation_type, actor, status, incident_id,
|
|
input, output, dry_run_result,
|
|
parent_op_id, tags
|
|
) VALUES (
|
|
'ansible_check_mode_executed',
|
|
'ansible_check_mode_worker',
|
|
'pending',
|
|
:incident_db_id,
|
|
CAST(:input AS jsonb),
|
|
'{}'::jsonb,
|
|
CAST(:dry_run_result AS jsonb),
|
|
CAST(:parent_op_id AS uuid),
|
|
:tags
|
|
)
|
|
RETURNING op_id
|
|
"""),
|
|
{
|
|
"incident_id": _incident_id_from_payload(claim_input),
|
|
"incident_db_id": _automation_operation_log_incident_id(
|
|
_incident_id_from_payload(claim_input)
|
|
),
|
|
"input": json.dumps(claim_input, ensure_ascii=False),
|
|
"dry_run_result": json.dumps({
|
|
"check_mode_executed": False,
|
|
"apply_executed": False,
|
|
"claim_state": "claimed",
|
|
}, ensure_ascii=False),
|
|
"parent_op_id": source_op_id,
|
|
"tags": ["ansible", "check_mode", "pending", "apply_locked"],
|
|
},
|
|
)
|
|
op_id = str(inserted.scalar_one())
|
|
claims.append(
|
|
AnsibleCheckModeClaim(
|
|
op_id=op_id,
|
|
source_candidate_op_id=source_op_id,
|
|
incident_id=str(claim_input.get("incident_id") or ""),
|
|
catalog_id=str(claim_input["catalog_id"]),
|
|
playbook_path=str(claim_input["playbook_path"]),
|
|
inventory_hosts=tuple(str(host) for host in claim_input["inventory_hosts"]),
|
|
input_payload=claim_input,
|
|
)
|
|
)
|
|
return claims
|
|
|
|
|
|
async def recent_ansible_transport_blockers(
|
|
*,
|
|
project_id: str = "awoooi",
|
|
cooldown_seconds: int | None = None,
|
|
include_repair_forced_command_blocker: bool | None = None,
|
|
) -> list[str]:
|
|
"""Return transport blockers observed from recent failed check-mode rows."""
|
|
|
|
cooldown = cooldown_seconds or settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_COOLDOWN_SECONDS
|
|
include_forced_blocker = (
|
|
_uses_repair_forced_command_transport()
|
|
if include_repair_forced_command_blocker is None
|
|
else include_repair_forced_command_blocker
|
|
)
|
|
async with get_db_context(project_id) as db:
|
|
result = await db.execute(
|
|
text("""
|
|
SELECT
|
|
coalesce(output::text, '') AS output_text,
|
|
coalesce(dry_run_result::text, '') AS dry_run_text,
|
|
coalesce(error, '') AS error_text,
|
|
coalesce(stderr_feed_back, '') AS stderr_text
|
|
FROM automation_operation_log
|
|
WHERE operation_type = 'ansible_check_mode_executed'
|
|
AND status = 'failed'
|
|
AND created_at >= NOW() - (:cooldown_seconds * INTERVAL '1 second')
|
|
ORDER BY created_at DESC
|
|
LIMIT 20
|
|
"""),
|
|
{"cooldown_seconds": max(60, cooldown)},
|
|
)
|
|
blockers: set[str] = set()
|
|
for row in result.mappings().all():
|
|
detected = detect_ansible_transport_blockers(
|
|
row.get("output_text"),
|
|
row.get("dry_run_text"),
|
|
row.get("error_text"),
|
|
row.get("stderr_text"),
|
|
)
|
|
blockers.update(
|
|
blocker
|
|
for blocker in detected
|
|
if blocker != FORCED_COMMAND_BLOCKER or include_forced_blocker
|
|
)
|
|
return sorted(blockers)
|
|
|
|
|
|
async def _insert_skipped_candidate(
|
|
db: Any,
|
|
*,
|
|
source_candidate_op_id: str,
|
|
candidate_input: dict[str, Any],
|
|
reason: str,
|
|
) -> None:
|
|
input_payload = {
|
|
"incident_id": _incident_id_from_payload(candidate_input),
|
|
"executor": "ansible",
|
|
"execution_backend": "ansible",
|
|
"execution_mode": "check_mode",
|
|
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
|
|
"check_mode": True,
|
|
"apply_enabled": False,
|
|
"source_candidate_op_id": source_candidate_op_id,
|
|
"not_used_reason": reason,
|
|
}
|
|
await db.execute(
|
|
text("""
|
|
INSERT INTO automation_operation_log (
|
|
operation_type, actor, status, incident_id,
|
|
input, output, dry_run_result,
|
|
parent_op_id, tags
|
|
) VALUES (
|
|
'ansible_execution_skipped',
|
|
'ansible_check_mode_worker',
|
|
'dry_run',
|
|
:incident_db_id,
|
|
CAST(:input AS jsonb),
|
|
CAST(:output AS jsonb),
|
|
CAST(:dry_run_result AS jsonb),
|
|
CAST(:parent_op_id AS uuid),
|
|
:tags
|
|
)
|
|
"""),
|
|
{
|
|
"incident_id": _incident_id_from_payload(input_payload),
|
|
"incident_db_id": _automation_operation_log_incident_id(
|
|
_incident_id_from_payload(input_payload)
|
|
),
|
|
"input": json.dumps(input_payload, ensure_ascii=False),
|
|
"output": json.dumps({
|
|
"not_used_reason": reason,
|
|
"decision_effect": "skipped_before_runtime",
|
|
}, ensure_ascii=False),
|
|
"dry_run_result": json.dumps({
|
|
"check_mode_executed": False,
|
|
"apply_executed": False,
|
|
"skipped": True,
|
|
"reason": reason,
|
|
}, ensure_ascii=False),
|
|
"parent_op_id": source_candidate_op_id,
|
|
"tags": ["ansible", "check_mode", "skipped", "apply_locked"],
|
|
},
|
|
)
|
|
|
|
|
|
async def finalize_check_mode_claim(
|
|
claim: AnsibleCheckModeClaim,
|
|
result: AnsibleRunResult,
|
|
*,
|
|
project_id: str = "awoooi",
|
|
) -> None:
|
|
status, output, dry_run_result, error = _build_result_payload(result)
|
|
async with get_db_context(project_id) as db:
|
|
await db.execute(
|
|
text("""
|
|
UPDATE automation_operation_log
|
|
SET status = :status,
|
|
output = CAST(:output AS jsonb),
|
|
dry_run_result = CAST(:dry_run_result AS jsonb),
|
|
error = :error,
|
|
duration_ms = :duration_ms,
|
|
stderr_feed_back = :stderr
|
|
WHERE op_id = CAST(:op_id AS uuid)
|
|
"""),
|
|
{
|
|
"status": status,
|
|
"output": json.dumps(output, ensure_ascii=False),
|
|
"dry_run_result": json.dumps(dry_run_result, ensure_ascii=False),
|
|
"error": _tail(error or "", 2000) or None,
|
|
"duration_ms": result.duration_ms,
|
|
"stderr": _tail(result.stderr, _STDERR_LIMIT),
|
|
"op_id": claim.op_id,
|
|
},
|
|
)
|
|
|
|
|
|
async def run_claimed_check_mode(
|
|
claim: AnsibleCheckModeClaim,
|
|
*,
|
|
timeout_seconds: int,
|
|
project_id: str = "awoooi",
|
|
) -> AnsibleRunResult:
|
|
try:
|
|
spec = build_ansible_check_mode_command(
|
|
playbook_path=claim.playbook_path,
|
|
inventory_hosts=claim.inventory_hosts,
|
|
)
|
|
result = await _run_ansible_command(spec, timeout_seconds=timeout_seconds)
|
|
except Exception as exc:
|
|
result = AnsibleRunResult(
|
|
returncode=1,
|
|
stdout="",
|
|
stderr=f"ansible_check_mode_runtime_error: {exc}",
|
|
duration_ms=0,
|
|
)
|
|
await finalize_check_mode_claim(claim, result, project_id=project_id)
|
|
logger.info(
|
|
"ansible_check_mode_claim_completed",
|
|
op_id=claim.op_id,
|
|
source_candidate_op_id=claim.source_candidate_op_id,
|
|
incident_id=claim.incident_id,
|
|
catalog_id=claim.catalog_id,
|
|
returncode=result.returncode,
|
|
timed_out=result.timed_out,
|
|
)
|
|
return result
|
|
|
|
|
|
async def run_pending_check_modes_once(
|
|
*,
|
|
project_id: str = "awoooi",
|
|
limit: int = 1,
|
|
timeout_seconds: int | None = None,
|
|
) -> dict[str, Any]:
|
|
blockers = _runtime_blockers()
|
|
if blockers:
|
|
logger.warning("ansible_check_mode_runtime_blocked", blockers=blockers)
|
|
return {"claimed": 0, "completed": 0, "failed": 0, "blockers": blockers}
|
|
transport_blockers = await recent_ansible_transport_blockers(project_id=project_id)
|
|
if transport_blockers:
|
|
logger.warning("ansible_check_mode_transport_blocked", blockers=transport_blockers)
|
|
return {"claimed": 0, "completed": 0, "failed": 0, "blockers": transport_blockers}
|
|
|
|
claims = await claim_pending_check_modes(
|
|
project_id=project_id,
|
|
limit=limit,
|
|
candidate_max_age_hours=settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS,
|
|
)
|
|
completed = 0
|
|
failed = 0
|
|
for claim in claims:
|
|
result = await run_claimed_check_mode(
|
|
claim,
|
|
timeout_seconds=timeout_seconds or settings.AWOOOP_ANSIBLE_CHECK_MODE_TIMEOUT_SECONDS,
|
|
project_id=project_id,
|
|
)
|
|
completed += 1
|
|
if result.returncode != 0:
|
|
failed += 1
|
|
return {
|
|
"claimed": len(claims),
|
|
"completed": completed,
|
|
"failed": failed,
|
|
"blockers": [],
|
|
}
|