wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 4 Packages Projects Releases Wiki Activity
Files
5ce3db6bb3a24fa6eab944fae0de3ba2ed2da858
awoooi/apps/api/src/services/awooop_ansible_check_mode_service.py
ogt e3acc321a7
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / build-and-deploy (push) Has been cancelled
Details
CD Pipeline / post-deploy-checks (push) Has been cancelled
Details
CD Pipeline / tests (push) Has been cancelled
Details
fix(agent): retain shadow policy on stale replay
2026-07-15 02:20:34 +08:00

9296 lines
380 KiB
Python
Raw Blame History

"""AwoooP Ansible controlled executor.
This service claims pending ``ansible_candidate_matched`` AOL rows, runs
``ansible-playbook --check --diff``, and for allowlisted low/medium/high risk
catalog rows performs controlled apply after the dry-run passes.
"""
from __future__ import annotations
import asyncio
import json
import os
import re
import shutil
import signal
import time
from collections.abc import Mapping
from dataclasses import dataclass, replace
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
import structlog
from sqlalchemy import select, text
from src.core.config import settings
from src.db.base import get_db_context
from src.db.models import ApprovalRecord, PlaybookRecord
from src.models.approval import ApprovalStatus
from src.services.ai_automation_runtime_contract import (
AI_AUTOMATION_STAGE_RECEIPT_SCHEMA_VERSION,
)
from src.services.awooop_ansible_audit_service import (
get_ansible_catalog_candidates,
get_ansible_catalog_item,
)
from src.services.awooop_ansible_learning_writeback import (
canonical_ansible_playbook_id,
ensure_ansible_rag_writeback,
record_ansible_playbook_trust_writeback,
)
from src.services.awooop_ansible_post_verifier import (
run_ansible_asset_post_verifier,
)
from src.services.playbook_evolver import TRUST_ARCHIVE_THRESHOLD
logger = structlog.get_logger(__name__)
_SAFE_HOST_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
_PLAYBOOK_PREFIX = Path("infra/ansible/playbooks")
_STDOUT_LIMIT = 20_000
_STDERR_LIMIT = 12_000
_CONTROLLED_RETRY_MAX_ATTEMPTS = 1
_CONTROLLED_RETRY_ERROR_BACKOFF_MIN_SECONDS = 15
_CONTROLLED_RETRY_ERROR_BACKOFF_MAX_SECONDS = 30
_STDIN_BOUNDARY_REPLAY_WINDOW_HOURS = 7 * 24
_CATALOG_REPAIR_REPLAY_WINDOW_HOURS = 72
_STALE_CLAIM_REPLAY_CONTROL_FIELDS = (
"bounded_execution_scope",
"catalog_drift_replay",
"catalog_repair_of_apply_op_id",
"catalog_replay_revision",
"previous_catalog_revision",
"previous_check_mode_playbook_path",
"replay_of_check_mode_op_id",
"semantic_repair_of_apply_op_id",
"semantic_route_reconciliation",
)
_HISTORICAL_SHADOW_REPLAY_TAGS = frozenset(
{"catalog_drift_replay", "semantic_catalog_reconciliation"}
)
_STDIN_BOUNDARY_TERMINAL_PROJECTION_BATCH_LIMIT = 5
_EXECUTION_CAPABILITY_MIN_TTL_SECONDS = 300
_EXECUTION_CAPABILITY_MAX_TTL_SECONDS = 1_800
_EXECUTION_CAPABILITY_SAFETY_MARGIN_SECONDS = 5
_WORKING_MEMORY_PROJECTION_WINDOW_HOURS = 8 * 24
_WORKING_MEMORY_PROJECTION_CURSOR: dict[str, tuple[datetime, str]] = {}
_EXECUTION_CAPABILITY_FALLBACK_OPERATION_TYPE = "remediation_executed"
_EXECUTION_CAPABILITY_OPERATION_TYPES = frozenset(
{
"ansible_executor_capability_issued",
"ansible_executor_capability_revoked",
"ansible_executor_capability_expired",
}
)
_VERIFIED_CLOSURE_REQUIRED_STAGE_IDS = frozenset(
{
"mcp_context",
"service_log_evidence",
"normalized_asset_identity",
"source_truth_diff",
"risk_policy_decision",
"executor_log_projection",
"retry_or_rollback",
"km_playbook_writeback",
"rag_writeback",
"playbook_trust",
"timeline_projection",
}
)
_NO_WRITE_REPLAY_REQUIRED_STAGE_IDS = _VERIFIED_CLOSURE_REQUIRED_STAGE_IDS
FORCED_COMMAND_BLOCKER = "ansible_repair_ssh_forced_command_denies_ansible_bootstrap"
REPAIR_FORCED_COMMAND_KEY_PATH = Path("/etc/repair-ssh/id_ed25519")
REPAIR_FORCED_COMMAND_KNOWN_HOSTS_PATH = Path("/etc/repair-known-hosts/known_hosts")
@dataclass(frozen=True)
class AnsibleCheckModeClaim:
op_id: str
source_candidate_op_id: str
incident_id: str
catalog_id: str
playbook_path: str
apply_playbook_path: str
inventory_hosts: tuple[str, ...]
risk_level: str
input_payload: dict[str, Any]
@dataclass(frozen=True)
class AnsibleCommandSpec:
command: list[str]
cwd: Path
env: dict[str, str]
playbook_abs_path: Path
inventory_abs_path: Path
@dataclass(frozen=True)
class AnsibleRunResult:
returncode: int
stdout: str
stderr: str
duration_ms: int
timed_out: bool = False
post_verifier_passed: bool | None = None
def _tail(text_value: str, limit: int) -> str:
if len(text_value) <= limit:
return text_value
return text_value[-limit:]
def _json_loads(value: Any) -> dict[str, Any]:
if isinstance(value, dict):
return value
if isinstance(value, str):
try:
parsed = json.loads(value)
except json.JSONDecodeError:
return {}
return parsed if isinstance(parsed, dict) else {}
return {}
def _incident_id_from_payload(payload: dict[str, Any]) -> str:
return str(payload.get("incident_id") or "").strip()
def _automation_operation_log_incident_id(value: str) -> int | None:
# ADR-090 keeps external INC-* ids in input JSON; the DB column is BIGINT.
normalized = str(value or "").strip()
return int(normalized) if normalized.isdigit() else None
def _check_mode_ssh_key_path() -> Path:
return Path(settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH)
def _check_mode_known_hosts_path() -> Path:
return Path(settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH)
def _uses_repair_forced_command_transport(key_path: Path | None = None) -> bool:
return (key_path or _check_mode_ssh_key_path()) == REPAIR_FORCED_COMMAND_KEY_PATH
def detect_ansible_transport_blockers(*values: Any) -> list[str]:
combined = " ".join(str(value or "") for value in values)
blockers: list[str] = []
if "REPAIR_DENIED:invalid_command" in combined:
blockers.append(FORCED_COMMAND_BLOCKER)
return blockers
def _playbook_roots(module_path: Path | None = None) -> list[Path]:
resolved_module_path = (module_path or Path(__file__)).resolve()
return [
Path("/app/infra/ansible"),
Path.cwd() / "infra" / "ansible",
*(parent / "infra" / "ansible" for parent in resolved_module_path.parents),
]
def _runtime_blockers(
*,
playbook_root: Path | None = None,
check_mode_ssh_key_path: Path | None = None,
check_mode_known_hosts_path: Path | None = None,
) -> list[str]:
root = playbook_root or next((path for path in _playbook_roots() if path.exists()), None)
ssh_key_path = check_mode_ssh_key_path or _check_mode_ssh_key_path()
known_hosts_path = check_mode_known_hosts_path or _check_mode_known_hosts_path()
blockers: list[str] = []
if shutil.which("ansible-playbook") is None:
blockers.append("ansible_playbook_binary_missing")
if root is None:
blockers.append("ansible_playbook_catalog_missing")
elif not (root / "inventory" / "hosts.yml").exists():
blockers.append("ansible_inventory_missing")
if not ssh_key_path.is_file() or not os.access(ssh_key_path, os.R_OK):
blockers.append("ansible_check_mode_ssh_key_missing")
if not known_hosts_path.is_file() or not os.access(known_hosts_path, os.R_OK):
blockers.append("ansible_check_mode_known_hosts_missing")
return blockers
def _safe_candidate(input_payload: dict[str, Any]) -> dict[str, Any]:
candidates = input_payload.get("executor_candidates")
if not isinstance(candidates, list) or not candidates:
raise ValueError("missing_executor_candidates")
for candidate in candidates:
if not isinstance(candidate, dict):
continue
catalog_id = str(candidate.get("catalog_id") or "")
catalog_item = get_ansible_catalog_item(catalog_id)
if not catalog_item:
continue
if catalog_item.get("supports_check_mode") is not True:
continue
catalog_playbook_path = str(catalog_item.get("playbook_path") or "")
candidate_playbook_path = str(candidate.get("playbook_path") or catalog_playbook_path)
check_mode_playbook_path = str(
candidate.get("check_mode_playbook_path")
or catalog_item.get("check_mode_playbook_path")
or catalog_playbook_path
)
if candidate_playbook_path not in {catalog_playbook_path, check_mode_playbook_path}:
continue
inventory_hosts = candidate.get("inventory_hosts") or catalog_item.get("inventory_hosts") or []
if (
isinstance(inventory_hosts, list)
and inventory_hosts
and all(isinstance(host, str) and _SAFE_HOST_RE.fullmatch(host) for host in inventory_hosts)
):
return {
"catalog_id": catalog_id,
"playbook_path": check_mode_playbook_path,
"catalog_playbook_path": catalog_playbook_path,
"source_candidate_playbook_path": candidate_playbook_path,
"check_mode_playbook_path": check_mode_playbook_path,
"inventory_hosts": tuple(inventory_hosts),
"risk_level": str(candidate.get("risk_level") or catalog_item.get("risk_level") or ""),
"catalog_revision": str(
candidate.get("catalog_revision")
or catalog_item.get("catalog_revision")
or ""
),
"auto_apply_enabled": bool(catalog_item.get("auto_apply_enabled") is True),
"approval_required": bool(catalog_item.get("approval_required") is True),
"break_glass_required": bool(catalog_item.get("break_glass_required") is True),
}
raise ValueError("no_safe_check_mode_candidate")
def _apply_ansible_playbook_trust_gate(
candidate_input: dict[str, Any],
trust_rows: list[Mapping[str, Any]],
) -> tuple[dict[str, Any], dict[str, Any]]:
"""Filter failed PlayBooks while allowing one repaired revision canary."""
candidates = candidate_input.get("executor_candidates")
if not isinstance(candidates, list):
candidates = []
trust_by_id = {
str(row.get("playbook_id") or ""): row
for row in trust_rows
if str(row.get("playbook_id") or "")
}
eligible: list[dict[str, Any]] = []
decisions: list[dict[str, Any]] = []
for candidate in candidates:
if not isinstance(candidate, dict):
continue
catalog_id = str(candidate.get("catalog_id") or "")
canonical_id = canonical_ansible_playbook_id(catalog_id)
row = trust_by_id.get(canonical_id)
status = str((row or {}).get("status") or "").strip().lower()
trust_score = float((row or {}).get("trust_score") or 0.0)
success_count = int((row or {}).get("success_count") or 0)
failure_count = int((row or {}).get("failure_count") or 0)
catalog_item = get_ansible_catalog_item(catalog_id) or {}
candidate_catalog_revision = str(
candidate.get("catalog_revision") or ""
).strip()
allowlisted_catalog_revision = str(
catalog_item.get("catalog_revision") or ""
).strip()
current_catalog_revision = str(
allowlisted_catalog_revision
or candidate_catalog_revision
or ""
).strip()
observed_pattern = _json_loads(
(row or {}).get("symptom_pattern")
)
observed_catalog_revision = str(
observed_pattern.get("catalog_revision") or ""
).strip()
retired = status in {"deprecated", "archived"}
learned_failure = bool(
row
and failure_count > success_count
and trust_score < TRUST_ARCHIVE_THRESHOLD
)
bounded_repair_canary = bool(
not retired
and learned_failure
and current_catalog_revision
and current_catalog_revision != observed_catalog_revision
)
circuit_open = retired or (
learned_failure and not bounded_repair_canary
)
reason = (
"playbook_status_retired"
if retired
else "repaired_catalog_revision_canary_allowed"
if bounded_repair_canary
else "playbook_trust_below_archive_threshold"
if learned_failure
else "playbook_trust_allows_controlled_canary"
if row
else "playbook_unobserved_canary_allowed"
)
decisions.append(
{
"catalog_id": catalog_id,
"canonical_playbook_id": canonical_id,
"trust_record_present": row is not None,
"status": status or None,
"trust_score": round(trust_score, 6) if row else None,
"success_count": success_count,
"failure_count": failure_count,
"review_required": bool((row or {}).get("review_required")),
"current_catalog_revision": current_catalog_revision or None,
"catalog_revision_source": (
"allowlisted_catalog"
if allowlisted_catalog_revision
else "candidate_fallback"
if candidate_catalog_revision
else None
),
"candidate_catalog_revision_mismatch": bool(
allowlisted_catalog_revision
and candidate_catalog_revision
and candidate_catalog_revision
!= allowlisted_catalog_revision
),
"observed_catalog_revision": observed_catalog_revision or None,
"bounded_repair_canary": bounded_repair_canary,
"circuit_open": circuit_open,
"reason": reason,
}
)
if not circuit_open:
eligible.append(candidate)
blocked_count = sum(1 for item in decisions if item["circuit_open"])
status = (
"circuit_open"
if decisions and not eligible
else "degraded_candidates_filtered"
if blocked_count
else "closed"
)
gate = {
"schema_version": "ansible_playbook_trust_claim_gate_v2",
"policy_source": "playbook_evolver.TRUST_ARCHIVE_THRESHOLD",
"trust_archive_threshold": TRUST_ARCHIVE_THRESHOLD,
"status": status,
"candidate_count": len(decisions),
"eligible_candidate_count": len(eligible),
"blocked_candidate_count": blocked_count,
"owner_review_used_as_terminal": False,
"next_controlled_action": (
"generate_or_verify_playbook_repair_candidate"
if status == "circuit_open"
else "continue_with_highest_ranked_eligible_candidate"
),
"decisions": decisions,
}
return (
{
**candidate_input,
"executor_candidates": eligible,
"playbook_trust_gate": gate,
},
gate,
)
async def _guard_candidate_input_by_playbook_trust(
db: Any,
candidate_input: dict[str, Any],
) -> tuple[dict[str, Any], dict[str, Any]]:
candidates = candidate_input.get("executor_candidates")
canonical_ids = sorted(
{
canonical_ansible_playbook_id(str(candidate.get("catalog_id") or ""))
for candidate in candidates or []
if isinstance(candidate, dict) and candidate.get("catalog_id")
}
)
trust_rows: list[Mapping[str, Any]] = []
if canonical_ids:
result = await db.execute(
select(
PlaybookRecord.playbook_id,
PlaybookRecord.status,
PlaybookRecord.trust_score,
PlaybookRecord.success_count,
PlaybookRecord.failure_count,
PlaybookRecord.review_required,
PlaybookRecord.symptom_pattern,
).where(PlaybookRecord.playbook_id.in_(canonical_ids))
)
trust_rows = list(result.mappings().all())
return _apply_ansible_playbook_trust_gate(candidate_input, trust_rows)
async def _revalidate_claim_playbook_trust(
db: Any,
claim: AnsibleCheckModeClaim,
) -> bool:
candidate_input = {
**claim.input_payload,
"executor_candidates": [
{
"catalog_id": claim.catalog_id,
"playbook_path": claim.apply_playbook_path,
"check_mode_playbook_path": claim.playbook_path,
"inventory_hosts": list(claim.inventory_hosts),
"risk_level": claim.risk_level,
"catalog_revision": str(
claim.input_payload.get("catalog_revision") or ""
),
}
],
}
_, gate = await _guard_candidate_input_by_playbook_trust(
db,
candidate_input,
)
claim.input_payload["playbook_trust_gate"] = gate
if gate.get("status") != "circuit_open":
return True
claim.input_payload["controlled_apply_allowed"] = False
claim.input_payload["controlled_apply_blocker"] = (
"playbook_trust_circuit_open"
)
await db.execute(
text("""
UPDATE automation_operation_log
SET input = coalesce(input, '{}'::jsonb) || jsonb_build_object(
'controlled_apply_allowed', false,
'controlled_apply_blocker', 'playbook_trust_circuit_open',
'playbook_trust_gate', CAST(:trust_gate AS jsonb)
),
dry_run_result = coalesce(
dry_run_result,
'{}'::jsonb
) || jsonb_build_object(
'controlled_apply_allowed', false,
'controlled_apply_blocker', 'playbook_trust_circuit_open',
'playbook_trust_gate', CAST(:trust_gate AS jsonb)
)
WHERE op_id = CAST(:op_id AS uuid)
AND operation_type = 'ansible_check_mode_executed'
"""),
{
"trust_gate": json.dumps(gate, ensure_ascii=False),
"op_id": claim.op_id,
},
)
return False
def _allowed_controlled_apply_risks() -> set[str]:
raw = str(settings.AWOOOP_ANSIBLE_CONTROLLED_APPLY_ALLOWED_RISK_LEVELS or "")
return {item.strip().lower() for item in raw.split(",") if item.strip()}
def _controlled_apply_allowed(candidate: dict[str, Any]) -> tuple[bool, str | None]:
risk = str(candidate.get("risk_level") or "").strip().lower()
if settings.ENABLE_AWOOOP_ANSIBLE_CONTROLLED_APPLY is not True:
return False, "controlled_apply_disabled_by_config"
if candidate.get("break_glass_required") is True:
return False, "break_glass_required"
if candidate.get("auto_apply_enabled") is not True:
return False, "catalog_auto_apply_disabled"
if candidate.get("approval_required") is True:
return False, "catalog_still_requires_approval"
if risk not in _allowed_controlled_apply_risks():
return False, f"risk_level_not_allowed:{risk or 'unknown'}"
return True, None
def build_ansible_check_mode_claim_input(
*,
source_candidate_op_id: str,
candidate_input: dict[str, Any],
) -> dict[str, Any]:
safe = _safe_candidate(candidate_input)
incident_id = _incident_id_from_payload(candidate_input)
controlled_apply_allowed, controlled_apply_blocker = _controlled_apply_allowed(safe)
proposal_risk_level = str(
candidate_input.get("proposal_risk_level") or ""
).strip().lower()
if proposal_risk_level == "critical":
controlled_apply_allowed = False
controlled_apply_blocker = "critical_break_glass_required"
return {
"automation_run_id": str(source_candidate_op_id),
"incident_id": incident_id,
"approval_id": str(candidate_input.get("approval_id") or ""),
"executor": "ansible",
"execution_backend": "ansible",
"execution_mode": "check_mode",
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
"check_mode": True,
"diff": True,
"apply_enabled": controlled_apply_allowed,
"approval_required_before_apply": not controlled_apply_allowed,
"controlled_apply_allowed": controlled_apply_allowed,
"controlled_apply_blocker": controlled_apply_blocker,
"candidate_idempotency_key": str(
candidate_input.get("idempotency_key") or ""
),
"apply_idempotency_key": (
f"ansible-apply:{source_candidate_op_id}:{safe['catalog_id']}"
),
"single_writer_executor": "awoooi-ansible-executor-broker",
"decision_path": str(candidate_input.get("decision_path") or ""),
"router_source_sha": str(candidate_input.get("router_source_sha") or "")
.strip()
.lower(),
"target_selector": candidate_input.get("target_selector") or {},
"source_truth_diff": candidate_input.get("source_truth_diff") or {},
"risk_policy_decision": candidate_input.get("risk_policy_decision") or {},
"playbook_trust_gate": candidate_input.get("playbook_trust_gate") or {},
"source_candidate_op_id": source_candidate_op_id,
"catalog_id": safe["catalog_id"],
"catalog_revision": safe["catalog_revision"],
"playbook_path": safe["playbook_path"],
"apply_playbook_path": safe["catalog_playbook_path"],
"catalog_playbook_path": safe["catalog_playbook_path"],
"source_candidate_playbook_path": safe["source_candidate_playbook_path"],
"check_mode_playbook_path": safe["check_mode_playbook_path"],
"inventory_hosts": list(safe["inventory_hosts"]),
"risk_level": safe["risk_level"],
}
def _resolve_playbook_path(playbook_root: Path, playbook_path: str) -> Path:
relative = Path(playbook_path)
if relative.is_absolute() or not str(relative).startswith(str(_PLAYBOOK_PREFIX)):
raise ValueError("unsafe_playbook_path")
repo_root = playbook_root.parent.parent
resolved = (repo_root / relative).resolve()
allowed_root = (repo_root / _PLAYBOOK_PREFIX).resolve()
if allowed_root not in resolved.parents:
raise ValueError("playbook_outside_catalog")
if resolved.suffix not in {".yml", ".yaml"} or not resolved.exists():
raise ValueError("playbook_not_found")
return resolved
def _ansible_command_env(playbook_root: Path) -> dict[str, str]:
roles_path = str((playbook_root / "roles").resolve())
existing_roles_path = os.environ.get("ANSIBLE_ROLES_PATH")
if existing_roles_path:
roles_path = os.pathsep.join([roles_path, existing_roles_path])
return {
**os.environ,
"ANSIBLE_HOST_KEY_CHECKING": "true",
"ANSIBLE_RETRY_FILES_ENABLED": "false",
"ANSIBLE_ROLES_PATH": roles_path,
}
def build_ansible_check_mode_command(
*,
playbook_path: str,
inventory_hosts: tuple[str, ...],
playbook_root: Path | None = None,
check_mode_ssh_key_path: Path | None = None,
check_mode_known_hosts_path: Path | None = None,
) -> AnsibleCommandSpec:
root = playbook_root or next((path for path in _playbook_roots() if path.exists()), None)
if root is None:
raise ValueError("ansible_playbook_catalog_missing")
inventory_path = (root / "inventory" / "hosts.yml").resolve()
if not inventory_path.exists():
raise ValueError("ansible_inventory_missing")
if not inventory_hosts or not all(_SAFE_HOST_RE.fullmatch(host) for host in inventory_hosts):
raise ValueError("unsafe_inventory_hosts")
playbook_abs = _resolve_playbook_path(root, playbook_path)
ssh_key_path = check_mode_ssh_key_path or _check_mode_ssh_key_path()
known_hosts_path = check_mode_known_hosts_path or _check_mode_known_hosts_path()
ssh_common_args = (
f"-o UserKnownHostsFile={known_hosts_path} "
"-o IdentitiesOnly=yes -o BatchMode=yes"
)
extra_vars = {
"ansible_ssh_private_key_file": str(ssh_key_path),
"ansible_ssh_common_args": ssh_common_args,
}
command = [
"ansible-playbook",
"-i",
str(inventory_path),
str(playbook_abs),
"--check",
"--diff",
"--limit",
",".join(inventory_hosts),
"--extra-vars",
json.dumps(extra_vars, ensure_ascii=False, separators=(",", ":")),
]
env = _ansible_command_env(root)
return AnsibleCommandSpec(
command=command,
cwd=root,
env=env,
playbook_abs_path=playbook_abs,
inventory_abs_path=inventory_path,
)
def build_ansible_apply_command(
*,
playbook_path: str,
inventory_hosts: tuple[str, ...],
playbook_root: Path | None = None,
check_mode_ssh_key_path: Path | None = None,
check_mode_known_hosts_path: Path | None = None,
) -> AnsibleCommandSpec:
"""Build the controlled apply command after check-mode has passed."""
root = playbook_root or next((path for path in _playbook_roots() if path.exists()), None)
if root is None:
raise ValueError("ansible_playbook_catalog_missing")
inventory_path = (root / "inventory" / "hosts.yml").resolve()
if not inventory_path.exists():
raise ValueError("ansible_inventory_missing")
if not inventory_hosts or not all(_SAFE_HOST_RE.fullmatch(host) for host in inventory_hosts):
raise ValueError("unsafe_inventory_hosts")
playbook_abs = _resolve_playbook_path(root, playbook_path)
ssh_key_path = check_mode_ssh_key_path or _check_mode_ssh_key_path()
known_hosts_path = check_mode_known_hosts_path or _check_mode_known_hosts_path()
ssh_common_args = (
f"-o UserKnownHostsFile={known_hosts_path} "
"-o IdentitiesOnly=yes -o BatchMode=yes"
)
extra_vars = {
"ansible_ssh_private_key_file": str(ssh_key_path),
"ansible_ssh_common_args": ssh_common_args,
}
command = [
"ansible-playbook",
"-i",
str(inventory_path),
str(playbook_abs),
"--diff",
"--limit",
",".join(inventory_hosts),
"--extra-vars",
json.dumps(extra_vars, ensure_ascii=False, separators=(",", ":")),
]
env = _ansible_command_env(root)
return AnsibleCommandSpec(
command=command,
cwd=root,
env=env,
playbook_abs_path=playbook_abs,
inventory_abs_path=inventory_path,
)
async def _run_ansible_command(spec: AnsibleCommandSpec, *, timeout_seconds: int) -> AnsibleRunResult:
started = time.monotonic()
process = await asyncio.create_subprocess_exec(
*spec.command,
cwd=str(spec.cwd),
env=spec.env,
start_new_session=True,
stdin=asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
def kill_process_group() -> None:
if process.returncode is not None:
return
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
return
except PermissionError:
process.kill()
timed_out = False
try:
stdout_bytes, stderr_bytes = await asyncio.wait_for(
process.communicate(),
timeout=timeout_seconds,
)
except TimeoutError:
timed_out = True
kill_process_group()
stdout_bytes, stderr_bytes = await process.communicate()
except asyncio.CancelledError:
kill_process_group()
await process.communicate()
raise
duration_ms = int((time.monotonic() - started) * 1000)
return AnsibleRunResult(
returncode=124 if timed_out else int(process.returncode or 0),
stdout=stdout_bytes.decode("utf-8", "replace"),
stderr=stderr_bytes.decode("utf-8", "replace"),
duration_ms=duration_ms,
timed_out=timed_out,
)
def _build_result_payload(
result: AnsibleRunResult,
*,
controlled_apply_allowed: bool = False,
controlled_apply_blocker: str | None = None,
) -> tuple[str, dict[str, Any], dict[str, Any], str | None]:
status = "success" if result.returncode == 0 else "failed"
stdout_tail = _tail(result.stdout, _STDOUT_LIMIT)
stderr_tail = _tail(result.stderr, _STDERR_LIMIT)
output = {
"executor": "ansible",
"execution_mode": "check_mode",
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
"ssh_key_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH,
"known_hosts_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH,
"check_mode": True,
"apply_enabled": controlled_apply_allowed,
"approval_required_before_apply": not controlled_apply_allowed,
"controlled_apply_allowed": controlled_apply_allowed,
"controlled_apply_blocker": controlled_apply_blocker,
"returncode": result.returncode,
"timed_out": result.timed_out,
"stdout_tail": stdout_tail,
"stderr_tail": stderr_tail,
"next_required_step": (
"controlled_apply_queued"
if result.returncode == 0 and controlled_apply_allowed
else "ai_playbook_or_transport_repair_required"
if result.returncode != 0
else "controlled_apply_blocked"
),
}
dry_run_result = {
"check_mode_executed": True,
"apply_executed": False,
"safe_to_apply_without_approval": controlled_apply_allowed,
"controlled_apply_allowed": controlled_apply_allowed,
"controlled_apply_blocker": controlled_apply_blocker,
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
"ssh_key_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH,
"known_hosts_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH,
"returncode": result.returncode,
"timed_out": result.timed_out,
"stdout_tail": stdout_tail,
"stderr_tail": stderr_tail,
}
error = None if result.returncode == 0 else (stderr_tail or f"ansible_check_mode_failed_rc_{result.returncode}")
return status, output, dry_run_result, error
def _build_apply_result_payload(result: AnsibleRunResult) -> tuple[str, dict[str, Any], dict[str, Any], str | None]:
status = "success" if result.returncode == 0 else "failed"
stdout_tail = _tail(result.stdout, _STDOUT_LIMIT)
stderr_tail = _tail(result.stderr, _STDERR_LIMIT)
output = {
"executor": "ansible",
"execution_mode": "controlled_apply",
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
"ssh_key_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH,
"known_hosts_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH,
"check_mode": False,
"apply_enabled": True,
"controlled_apply_executed": True,
"returncode": result.returncode,
"timed_out": result.timed_out,
"stdout_tail": stdout_tail,
"stderr_tail": stderr_tail,
"next_required_step": (
"post_apply_verifier_and_learning_writeback"
if result.returncode == 0
else "ai_rollback_or_playbook_repair_required"
),
}
dry_run_result = {
"check_mode_executed_before_apply": True,
"apply_executed": True,
"controlled_apply_executed": True,
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
"ssh_key_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH,
"known_hosts_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH,
"returncode": result.returncode,
"timed_out": result.timed_out,
"stdout_tail": stdout_tail,
"stderr_tail": stderr_tail,
}
error = None if result.returncode == 0 else (stderr_tail or f"ansible_apply_failed_rc_{result.returncode}")
return status, output, dry_run_result, error
def _approval_id_for_claim(claim: AnsibleCheckModeClaim) -> str:
raw_value = str(claim.input_payload.get("approval_id") or "").strip()
if not raw_value:
return ""
try:
return str(UUID(raw_value))
except ValueError:
return ""
async def _append_alert_lifecycle_receipt(
claim: AnsibleCheckModeClaim,
event_type: str,
*,
apply_op_id: str,
success: bool,
action_detail: str,
project_id: str,
error_message: str | None = None,
post_verifier_passed: bool | None = None,
) -> bool:
from src.repositories.alert_operation_log_repository import (
get_alert_operation_log_repository,
)
automation_run_id = _automation_run_id_for_claim(claim)
try:
async with get_db_context(project_id) as db:
existing = await db.execute(
text("""
SELECT EXISTS (
SELECT 1
FROM alert_operation_log
WHERE incident_id = :incident_id
AND event_type::text = :event_type
AND context ->> 'apply_op_id' = :apply_op_id
AND context ->> 'automation_run_id' = :automation_run_id
)
"""),
{
"incident_id": claim.incident_id,
"event_type": event_type,
"apply_op_id": apply_op_id,
"automation_run_id": automation_run_id,
},
)
if existing.scalar() is True:
return True
except Exception as exc:
logger.warning(
"ansible_alert_lifecycle_receipt_readback_failed",
incident_id=claim.incident_id,
apply_op_id=apply_op_id,
event_type=event_type,
error_type=type(exc).__name__,
)
record = await get_alert_operation_log_repository().append(
event_type,
incident_id=claim.incident_id,
approval_id=_approval_id_for_claim(claim) or None,
actor="awoooi-ansible-executor-broker",
action_detail=action_detail,
success=success,
error_message=error_message,
context={
"automation_run_id": automation_run_id,
"catalog_id": claim.catalog_id,
"check_mode_op_id": claim.op_id,
"apply_op_id": apply_op_id,
"single_writer_executor": "awoooi-ansible-executor-broker",
"post_verifier_passed": post_verifier_passed,
},
project_id=project_id,
)
return record is not None
async def _finalize_controlled_approval_projection(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
project_id: str,
) -> bool:
approval_id = _approval_id_for_claim(claim)
if not approval_id:
return False
success = bool(
result.returncode == 0 and result.post_verifier_passed is True
)
try:
async with get_db_context(project_id) as db:
selected = await db.execute(
select(ApprovalRecord).where(ApprovalRecord.id == approval_id)
)
approval = selected.scalar_one_or_none()
if approval is None:
return False
metadata = dict(approval.extra_metadata or {})
metadata.update({
"automation_run_id": _automation_run_id_for_claim(claim),
"single_writer_executor": "awoooi-ansible-executor-broker",
"execution_kind": "controlled_apply",
"repair_attempted": True,
"repair_executed": result.returncode == 0,
"post_verifier_passed": result.post_verifier_passed is True,
"apply_op_id": apply_op_id,
})
approval.extra_metadata = metadata
approval.status = (
ApprovalStatus.EXECUTION_SUCCESS
if success
else ApprovalStatus.EXECUTION_FAILED
)
approval.resolved_at = datetime.now(UTC)
if not success:
approval.rejection_reason = _tail(
result.stderr or "independent_post_verifier_failed",
2000,
)
except Exception as exc:
logger.warning(
"ansible_controlled_apply_approval_projection_failed",
incident_id=claim.incident_id,
approval_id=approval_id,
apply_op_id=apply_op_id,
error_type=type(exc).__name__,
)
return False
return True
def _build_auto_repair_execution_receipt(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
) -> dict[str, Any]:
success = bool(
result.returncode == 0 and result.post_verifier_passed is True
)
if success:
error_message = None
elif result.returncode != 0:
error_message = _tail(result.stderr or result.stdout, 2000)
else:
error_message = "independent_post_verifier_failed"
return {
"incident_id": claim.incident_id,
"playbook_id": str(claim.catalog_id or "")[:36] or "ansible",
"playbook_name": f"Ansible controlled apply: {claim.apply_playbook_path}"[:200],
"success": success,
"executed_steps": [
f"automation_run_id:{claim.input_payload.get('automation_run_id') or claim.source_candidate_op_id}",
f"candidate:{claim.source_candidate_op_id}",
f"check_mode:{claim.op_id}",
f"apply:{apply_op_id}",
f"catalog:{claim.catalog_id}",
f"returncode:{result.returncode}",
f"post_verifier_passed:{str(result.post_verifier_passed is True).lower()}",
],
"error_message": error_message,
"triggered_by": "ansible_controlled_apply",
"similarity_score": None,
"risk_level": str(claim.risk_level or ""),
"execution_time_ms": result.duration_ms,
}
def _int_from_value(value: Any, *, default: int = 1) -> int:
if value in (None, ""):
return default
try:
return int(value)
except (TypeError, ValueError):
return default
def _claim_from_apply_operation_row(row: dict[str, Any]) -> tuple[AnsibleCheckModeClaim, AnsibleRunResult] | None:
input_payload = _json_loads(row.get("input"))
output_payload = _json_loads(row.get("output"))
dry_run_result = _json_loads(row.get("dry_run_result"))
catalog_id = str(input_payload.get("catalog_id") or row.get("catalog_id") or "")
if not catalog_id:
return None
catalog_item = get_ansible_catalog_item(catalog_id) or {}
incident_id = str(row.get("incident_id") or input_payload.get("incident_id") or "")
if not incident_id:
return None
inventory_hosts = input_payload.get("inventory_hosts") or catalog_item.get("inventory_hosts") or []
if not isinstance(inventory_hosts, list) or not inventory_hosts:
return None
apply_playbook_path = str(
input_payload.get("apply_playbook_path")
or input_payload.get("playbook_path")
or row.get("playbook_path")
or catalog_item.get("playbook_path")
or ""
)
check_mode_playbook_path = str(
input_payload.get("check_mode_playbook_path")
or input_payload.get("playbook_path")
or row.get("playbook_path")
or apply_playbook_path
)
if not apply_playbook_path:
return None
returncode = _int_from_value(
output_payload.get("returncode", dry_run_result.get("returncode")),
default=0 if str(row.get("status") or "").lower() == "success" else 1,
)
claim = AnsibleCheckModeClaim(
op_id=str(input_payload.get("check_mode_op_id") or row.get("parent_op_id") or ""),
source_candidate_op_id=str(input_payload.get("source_candidate_op_id") or ""),
incident_id=incident_id,
catalog_id=catalog_id,
playbook_path=check_mode_playbook_path,
apply_playbook_path=apply_playbook_path,
inventory_hosts=tuple(str(host) for host in inventory_hosts),
risk_level=str(
input_payload.get("risk_level")
or row.get("risk_level")
or catalog_item.get("risk_level")
or ""
),
input_payload=input_payload,
)
result = AnsibleRunResult(
returncode=returncode,
stdout=str(output_payload.get("stdout_tail") or ""),
stderr=str(row.get("error") or output_payload.get("stderr_tail") or ""),
duration_ms=_int_from_value(row.get("duration_ms"), default=0),
timed_out=bool(output_payload.get("timed_out") or dry_run_result.get("timed_out") or False),
)
return claim, result
def _claim_from_stale_check_mode_row(
row: dict[str, Any],
) -> AnsibleCheckModeClaim | None:
input_payload = _json_loads(row.get("input"))
source_candidate_op_id = str(
input_payload.get("source_candidate_op_id")
or row.get("parent_op_id")
or ""
)
incident_id = str(
row.get("incident_id")
or input_payload.get("incident_id")
or ""
)
previous_catalog_id = str(input_payload.get("catalog_id") or "")
catalog_id = previous_catalog_id
incident_routing_evaluated = bool(
row.get("incident_alertname")
or row.get("incident_alert_category")
or row.get("incident_affected_services")
)
if incident_routing_evaluated:
candidates = get_ansible_catalog_candidates(
{
"alertname": row.get("incident_alertname"),
"alert_category": row.get("incident_alert_category"),
"affected_services": row.get("incident_affected_services") or [],
}
)
if not candidates:
return None
catalog_id = str(candidates[0].get("catalog_id") or "")
catalog_item = get_ansible_catalog_item(catalog_id) or {}
inventory_hosts = catalog_item.get("inventory_hosts") or input_payload.get(
"inventory_hosts"
)
source_playbook_path = str(
input_payload.get("source_candidate_playbook_path")
or input_payload.get("catalog_playbook_path")
or input_payload.get("apply_playbook_path")
or input_payload.get("playbook_path")
or ""
)
if (
not source_candidate_op_id
or not incident_id
or not catalog_id
or not source_playbook_path
or not isinstance(inventory_hosts, list)
or not inventory_hosts
):
return None
canonical_playbook_path = str(catalog_item.get("playbook_path") or "")
if not canonical_playbook_path:
return None
try:
claim_input = build_ansible_check_mode_claim_input(
source_candidate_op_id=source_candidate_op_id,
candidate_input={
"incident_id": incident_id,
"executor": "ansible",
"executor_candidates": [
{
"catalog_id": catalog_id,
# A stale row is evidence, never execution authority. Path
# migrations must resolve through the current allowlisted
# catalog so a repaired bounded playbook can replace an
# unsafe or retired predecessor without trusting DB input.
"playbook_path": canonical_playbook_path,
"inventory_hosts": inventory_hosts,
"risk_level": catalog_item.get("risk_level"),
}
],
},
)
except ValueError:
return None
claim_input.update(
{
"previous_catalog_id": previous_catalog_id or None,
"catalog_route_changed": catalog_id != previous_catalog_id,
"incident_semantic_route_rechecked": incident_routing_evaluated,
}
)
for field_name in _STALE_CLAIM_REPLAY_CONTROL_FIELDS:
if field_name in input_payload:
claim_input[field_name] = input_payload[field_name]
row_tags = {
str(tag)
for tag in (row.get("tags") or [])
if isinstance(tag, str)
}
historical_shadow_replay = bool(
row_tags & _HISTORICAL_SHADOW_REPLAY_TAGS
)
if (
historical_shadow_replay
or input_payload.get("historical_projection_backfill") is True
or input_payload.get("telegram_provider_delivery") == "shadow_only"
):
claim_input["historical_projection_backfill"] = True
claim_input["telegram_provider_delivery"] = "shadow_only"
return AnsibleCheckModeClaim(
op_id=str(row.get("op_id") or ""),
source_candidate_op_id=source_candidate_op_id,
incident_id=incident_id,
catalog_id=str(claim_input["catalog_id"]),
playbook_path=str(claim_input["playbook_path"]),
apply_playbook_path=str(claim_input["apply_playbook_path"]),
inventory_hosts=tuple(str(host) for host in claim_input["inventory_hosts"]),
risk_level=str(claim_input.get("risk_level") or ""),
input_payload=claim_input,
)
def _claim_from_failed_apply_catalog_repair_row(
row: dict[str, Any],
) -> AnsibleCheckModeClaim | None:
"""Build a new bounded claim when a failed apply's catalog was repaired."""
input_payload = _json_loads(row.get("input"))
apply_op_id = str(row.get("op_id") or "")
source_candidate_op_id = str(
input_payload.get("source_candidate_op_id") or ""
)
incident_id = str(
row.get("incident_id")
or input_payload.get("incident_id")
or ""
)
catalog_id = str(input_payload.get("catalog_id") or "")
catalog_item = get_ansible_catalog_item(catalog_id) or {}
current_revision = str(catalog_item.get("catalog_revision") or "")
previous_revision = str(input_payload.get("catalog_revision") or "")
current_playbook_path = str(catalog_item.get("playbook_path") or "")
inventory_hosts = catalog_item.get("inventory_hosts") or []
if not (
apply_op_id
and source_candidate_op_id
and incident_id
and catalog_id
and current_revision
and current_revision != previous_revision
and current_playbook_path
and isinstance(inventory_hosts, list)
and inventory_hosts
):
return None
try:
claim_input = build_ansible_check_mode_claim_input(
source_candidate_op_id=source_candidate_op_id,
candidate_input={
"incident_id": incident_id,
"approval_id": input_payload.get("approval_id"),
"proposal_risk_level": input_payload.get(
"proposal_risk_level"
),
"decision_path": "failed_apply_catalog_repair_replay",
"router_source_sha": input_payload.get("router_source_sha"),
"target_selector": input_payload.get("target_selector") or {},
"source_truth_diff": input_payload.get("source_truth_diff")
or {},
"risk_policy_decision": input_payload.get(
"risk_policy_decision"
)
or {},
"executor_candidates": [
{
"catalog_id": catalog_id,
"catalog_revision": current_revision,
"playbook_path": current_playbook_path,
"check_mode_playbook_path": str(
catalog_item.get("check_mode_playbook_path")
or current_playbook_path
),
"inventory_hosts": inventory_hosts,
"risk_level": catalog_item.get("risk_level"),
}
],
},
)
except ValueError:
return None
automation_run_id = str(
input_payload.get("automation_run_id")
or source_candidate_op_id
)
claim_input.update(
{
"automation_run_id": automation_run_id,
"catalog_repair_of_apply_op_id": apply_op_id,
"previous_catalog_revision": previous_revision or None,
"catalog_replay_revision": current_revision,
"apply_idempotency_key": (
f"ansible-repair-apply:{apply_op_id}:{current_revision}"
),
"bounded_execution_scope": "catalog_repair_single_apply",
}
)
return AnsibleCheckModeClaim(
op_id="",
source_candidate_op_id=source_candidate_op_id,
incident_id=incident_id,
catalog_id=str(claim_input["catalog_id"]),
playbook_path=str(claim_input["playbook_path"]),
apply_playbook_path=str(claim_input["apply_playbook_path"]),
inventory_hosts=tuple(
str(host) for host in claim_input["inventory_hosts"]
),
risk_level=str(claim_input.get("risk_level") or ""),
input_payload=claim_input,
)
async def _record_auto_repair_execution_receipt(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
project_id: str,
) -> bool:
receipt = _build_auto_repair_execution_receipt(
claim,
result,
apply_op_id=apply_op_id,
)
try:
async with get_db_context(project_id) as db:
inserted = await db.execute(
text("""
INSERT INTO auto_repair_executions (
incident_id,
playbook_id,
playbook_name,
success,
executed_steps,
error_message,
triggered_by,
similarity_score,
risk_level,
execution_time_ms
)
SELECT
CAST(:incident_id AS varchar(30)),
CAST(:playbook_id AS varchar(36)),
CAST(:playbook_name AS varchar(200)),
:success,
CAST(:executed_steps AS jsonb),
:error_message,
CAST(:triggered_by AS varchar(50)),
:similarity_score,
CAST(:risk_level AS varchar(20)),
:execution_time_ms
WHERE NOT EXISTS (
SELECT 1
FROM auto_repair_executions existing
WHERE existing.incident_id = CAST(:incident_id AS varchar(30))
AND existing.triggered_by = CAST(:triggered_by AS varchar(50))
AND existing.executed_steps::text LIKE CAST(:apply_op_id_needle AS text)
)
RETURNING id
"""),
{
**receipt,
"executed_steps": json.dumps(receipt["executed_steps"], ensure_ascii=False),
"apply_op_id_needle": f"%{apply_op_id}%",
},
)
return inserted.scalar() is not None
except Exception as exc:
logger.warning(
"ansible_auto_repair_execution_receipt_failed",
incident_id=claim.incident_id,
catalog_id=claim.catalog_id,
apply_op_id=apply_op_id,
error=str(exc),
)
return False
def _post_apply_verification_result(
result: AnsibleRunResult,
verifier_receipt: Mapping[str, Any] | None = None,
) -> str:
if result.timed_out:
return "timeout"
if result.returncode != 0:
return "failed"
if not isinstance(verifier_receipt, Mapping):
return "failed"
return (
"success"
if verifier_receipt.get("verification_result") == "success"
and verifier_receipt.get("all_postconditions_passed") is True
and verifier_receipt.get("executor_returncode_trusted") is False
else "failed"
)
def _post_apply_km_path_type(apply_op_id: str) -> str:
return f"ansible_apply_receipt:{str(apply_op_id or '')[:8]}"
def _post_apply_action_label(claim: AnsibleCheckModeClaim, *, apply_op_id: str) -> str:
return (
f"ansible_controlled_apply:{claim.catalog_id}:"
f"{claim.apply_playbook_path}:apply_op={apply_op_id}"
)
def _automation_run_id_for_claim(claim: AnsibleCheckModeClaim) -> str:
return str(
claim.input_payload.get("automation_run_id")
or claim.source_candidate_op_id
)
async def _resolve_execution_capability_operation_type(
db: Any,
*,
semantic_operation_type: str,
) -> str:
"""Use a schema-safe semantic receipt until the native type is allowed."""
if semantic_operation_type not in _EXECUTION_CAPABILITY_OPERATION_TYPES:
raise ValueError("unsupported_execution_capability_operation_type")
result = await db.execute(
text("""
SELECT CASE
WHEN EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'automation_operation_log_type_valid'
AND pg_get_constraintdef(oid)
LIKE '%' || :semantic_operation_type || '%'
)
THEN :semantic_operation_type
ELSE :fallback_operation_type
END
"""),
{
"semantic_operation_type": semantic_operation_type,
"fallback_operation_type": (
_EXECUTION_CAPABILITY_FALLBACK_OPERATION_TYPE
),
},
)
return str(
result.scalar_one()
or _EXECUTION_CAPABILITY_FALLBACK_OPERATION_TYPE
)
def _execution_capability_timeout_seconds(
claim: AnsibleCheckModeClaim,
*,
execution_mode: str,
requested_timeout_seconds: int,
now: datetime | None = None,
) -> int:
"""Validate the broker lease and keep execution inside its deadline."""
capability = claim.input_payload.get("execution_capability")
if not isinstance(capability, dict):
raise ValueError("execution_capability_missing")
capability_op_id = str(capability.get("capability_op_id") or "")
if not capability_op_id:
raise ValueError("execution_capability_id_missing")
if str(capability.get("automation_run_id") or "") != (
_automation_run_id_for_claim(claim)
):
raise ValueError("execution_capability_run_id_mismatch")
if str(capability.get("check_mode_op_id") or "") != claim.op_id:
raise ValueError("execution_capability_check_mode_mismatch")
scope = capability.get("scope")
if not isinstance(scope, dict):
raise ValueError("execution_capability_scope_missing")
expected_scope = {
"catalog_id": claim.catalog_id,
"playbook_path": claim.playbook_path,
"apply_playbook_path": claim.apply_playbook_path,
"inventory_hosts": list(claim.inventory_hosts),
"risk_level": claim.risk_level,
}
for key, expected_value in expected_scope.items():
if scope.get(key) != expected_value:
raise ValueError(f"execution_capability_scope_mismatch:{key}")
allowed_execution_modes = scope.get("allowed_execution_modes")
if (
not isinstance(allowed_execution_modes, list)
or execution_mode not in allowed_execution_modes
):
raise ValueError("execution_capability_mode_not_allowed")
try:
issued_at = datetime.fromisoformat(
str(capability.get("issued_at") or "").replace("Z", "+00:00")
)
expires_at = datetime.fromisoformat(
str(capability.get("expires_at") or "").replace("Z", "+00:00")
)
except ValueError as exc:
raise ValueError("execution_capability_timestamp_invalid") from exc
if issued_at.tzinfo is None or expires_at.tzinfo is None:
raise ValueError("execution_capability_timestamp_timezone_missing")
current_time = (now or datetime.now(UTC)).astimezone(UTC)
issued_at = issued_at.astimezone(UTC)
expires_at = expires_at.astimezone(UTC)
if current_time < issued_at:
raise ValueError("execution_capability_not_yet_valid")
remaining_seconds = int((expires_at - current_time).total_seconds())
bounded_timeout = min(
int(requested_timeout_seconds),
remaining_seconds - _EXECUTION_CAPABILITY_SAFETY_MARGIN_SECONDS,
)
if bounded_timeout < 1:
raise ValueError("execution_capability_expired")
return bounded_timeout
async def _issue_ansible_execution_capability(
claim: AnsibleCheckModeClaim,
*,
ttl_seconds: int,
project_id: str,
) -> tuple[AnsibleCheckModeClaim, str]:
"""Issue a bounded broker lease before check/apply can touch a target."""
catalog_item = get_ansible_catalog_item(claim.catalog_id)
if not catalog_item:
raise ValueError("execution_capability_catalog_not_allowlisted")
catalog_hosts = {
str(host) for host in catalog_item.get("inventory_hosts") or []
}
if not claim.inventory_hosts or not set(claim.inventory_hosts).issubset(
catalog_hosts
):
raise ValueError("execution_capability_inventory_outside_allowlist")
normalized_ttl_seconds = max(
_EXECUTION_CAPABILITY_MIN_TTL_SECONDS,
min(_EXECUTION_CAPABILITY_MAX_TTL_SECONDS, int(ttl_seconds)),
)
issued_at = datetime.now(UTC)
expires_at = issued_at + timedelta(seconds=normalized_ttl_seconds)
automation_run_id = _automation_run_id_for_claim(claim)
allowed_execution_modes = ["check_mode"]
if claim.input_payload.get("controlled_apply_allowed") is True:
allowed_execution_modes.append("controlled_apply")
capability_scope = {
"catalog_id": claim.catalog_id,
"playbook_path": claim.playbook_path,
"apply_playbook_path": claim.apply_playbook_path,
"inventory_hosts": list(claim.inventory_hosts),
"risk_level": claim.risk_level,
"allowed_execution_modes": allowed_execution_modes,
}
async with get_db_context(project_id) as db:
ledger_operation_type = await _resolve_execution_capability_operation_type(
db,
semantic_operation_type="ansible_executor_capability_issued",
)
inserted = await db.execute(
text("""
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, parent_op_id, tags
)
SELECT
:ledger_operation_type,
'ansible_execution_broker',
'success',
:incident_db_id,
CAST(:input AS jsonb),
CAST(:output AS jsonb),
CAST(:parent_op_id AS uuid),
:tags
WHERE NOT EXISTS (
SELECT 1
FROM automation_operation_log issued
WHERE issued.parent_op_id = CAST(:parent_op_id AS uuid)
AND coalesce(
issued.input ->> 'semantic_operation_type',
issued.operation_type
) = 'ansible_executor_capability_issued'
AND NULLIF(issued.input ->> 'expires_at', '')::timestamptz
> NOW()
AND NOT EXISTS (
SELECT 1
FROM automation_operation_log terminal
WHERE terminal.parent_op_id = issued.op_id
AND coalesce(
terminal.input ->> 'semantic_operation_type',
terminal.operation_type
) IN (
'ansible_executor_capability_revoked',
'ansible_executor_capability_expired'
)
)
)
RETURNING op_id
"""),
{
"incident_db_id": _automation_operation_log_incident_id(
claim.incident_id
),
"input": json.dumps(
{
"automation_run_id": automation_run_id,
"semantic_operation_type": (
"ansible_executor_capability_issued"
),
"source_candidate_op_id": claim.source_candidate_op_id,
"check_mode_op_id": claim.op_id,
"issued_at": issued_at.isoformat(),
"expires_at": expires_at.isoformat(),
"ttl_seconds": normalized_ttl_seconds,
"capability_scope": capability_scope,
},
ensure_ascii=False,
),
"output": json.dumps(
{
"issuance_status": "issued",
"revoke_required": True,
"raw_credential_exposed_to_requester": False,
"broker_enforced_allowlist": True,
},
ensure_ascii=False,
),
"parent_op_id": claim.op_id,
"ledger_operation_type": ledger_operation_type,
"tags": [
"ansible",
"execution_broker",
"capability_issued",
f"automation_run_id:{automation_run_id}",
],
},
)
inserted_op_id = inserted.scalar_one_or_none()
if inserted_op_id is None:
raise RuntimeError("execution_capability_active_lease_exists")
capability_op_id = str(inserted_op_id)
return (
replace(
claim,
input_payload={
**claim.input_payload,
"execution_capability": {
"capability_op_id": capability_op_id,
"automation_run_id": automation_run_id,
"check_mode_op_id": claim.op_id,
"issued_at": issued_at.isoformat(),
"expires_at": expires_at.isoformat(),
"scope": capability_scope,
},
},
),
capability_op_id,
)
async def _revoke_ansible_execution_capability(
claim: AnsibleCheckModeClaim,
*,
capability_op_id: str,
terminal_status: str,
project_id: str,
) -> bool:
automation_run_id = _automation_run_id_for_claim(claim)
async with get_db_context(project_id) as db:
ledger_operation_type = await _resolve_execution_capability_operation_type(
db,
semantic_operation_type="ansible_executor_capability_revoked",
)
result = await db.execute(
text("""
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, parent_op_id, tags
)
SELECT
:ledger_operation_type,
'ansible_execution_broker',
'success',
:incident_db_id,
CAST(:input AS jsonb),
CAST(:output AS jsonb),
CAST(:parent_op_id AS uuid),
:tags
WHERE NOT EXISTS (
SELECT 1
FROM automation_operation_log existing
WHERE existing.parent_op_id = CAST(:parent_op_id AS uuid)
AND coalesce(
existing.input ->> 'semantic_operation_type',
existing.operation_type
) IN (
'ansible_executor_capability_revoked',
'ansible_executor_capability_expired'
)
)
RETURNING op_id
"""),
{
"incident_db_id": _automation_operation_log_incident_id(
claim.incident_id
),
"input": json.dumps(
{
"automation_run_id": automation_run_id,
"semantic_operation_type": (
"ansible_executor_capability_revoked"
),
"capability_op_id": capability_op_id,
"check_mode_op_id": claim.op_id,
"terminal_status": terminal_status,
},
ensure_ascii=False,
),
"output": json.dumps(
{
"revocation_status": "revoked",
"raw_credential_persisted_in_receipt": False,
"further_execution_allowed": False,
},
ensure_ascii=False,
),
"parent_op_id": capability_op_id,
"ledger_operation_type": ledger_operation_type,
"tags": [
"ansible",
"execution_broker",
"capability_revoked",
f"automation_run_id:{automation_run_id}",
],
},
)
return result.scalar_one_or_none() is not None
async def _expire_stale_ansible_execution_capabilities(
*,
project_id: str,
limit: int = 20,
) -> int:
"""Close leases left open by a terminated broker process."""
async with get_db_context(project_id) as db:
ledger_operation_type = await _resolve_execution_capability_operation_type(
db,
semantic_operation_type="ansible_executor_capability_expired",
)
result = await db.execute(
text("""
WITH expired AS (
SELECT issued.op_id, issued.incident_id, issued.input
FROM automation_operation_log issued
WHERE coalesce(
issued.input ->> 'semantic_operation_type',
issued.operation_type
) = 'ansible_executor_capability_issued'
AND issued.status = 'success'
AND NULLIF(issued.input ->> 'expires_at', '')::timestamptz
<= NOW()
AND NOT EXISTS (
SELECT 1
FROM automation_operation_log terminal
WHERE terminal.parent_op_id = issued.op_id
AND coalesce(
terminal.input ->> 'semantic_operation_type',
terminal.operation_type
) IN (
'ansible_executor_capability_revoked',
'ansible_executor_capability_expired'
)
)
ORDER BY issued.created_at ASC
LIMIT :limit
FOR UPDATE SKIP LOCKED
)
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, parent_op_id, tags
)
SELECT
:ledger_operation_type,
'ansible_execution_broker',
'success',
expired.incident_id,
jsonb_build_object(
'automation_run_id', expired.input ->> 'automation_run_id',
'semantic_operation_type',
'ansible_executor_capability_expired',
'capability_op_id', expired.op_id::text,
'expired_at', NOW()
),
jsonb_build_object(
'expiry_status', 'expired',
'further_execution_allowed', false
),
expired.op_id,
ARRAY[
'ansible',
'execution_broker',
'capability_expired'
]::varchar[]
FROM expired
RETURNING op_id
"""),
{
"limit": max(1, limit),
"ledger_operation_type": ledger_operation_type,
},
)
return len(result.scalars().all())
def _runtime_stage_receipt(
claim: AnsibleCheckModeClaim,
*,
stage_id: str,
evidence_ref: str,
detail: dict[str, Any],
derived_from_durable_chain: bool = False,
) -> dict[str, Any]:
return {
"schema_version": AI_AUTOMATION_STAGE_RECEIPT_SCHEMA_VERSION,
"stage_id": stage_id,
"automation_run_id": _automation_run_id_for_claim(claim),
"incident_id": claim.incident_id,
"evidence_ref": evidence_ref,
"detail": detail,
"durable_receipt": True,
"derived_from_durable_chain": derived_from_durable_chain,
"raw_log_payload_stored": False,
"secret_value_stored": False,
}
def build_ansible_pre_apply_runtime_stage_receipts(
claim: AnsibleCheckModeClaim,
*,
derived_from_durable_chain: bool = False,
) -> list[dict[str, Any]]:
"""Build receipts proven before controlled apply from the claimed dry-run."""
return [
_runtime_stage_receipt(
claim,
stage_id="normalized_asset_identity",
evidence_ref=f"ansible_catalog:{claim.catalog_id}",
detail={
"asset_kind": "managed_host",
"canonical_asset_ids": list(claim.inventory_hosts),
"catalog_id": claim.catalog_id,
"playbook_path": claim.apply_playbook_path,
},
derived_from_durable_chain=derived_from_durable_chain,
),
_runtime_stage_receipt(
claim,
stage_id="source_truth_diff",
evidence_ref=(
"automation_operation_log:"
f"{claim.op_id}:dry_run_result"
),
detail={
"check_mode_op_id": claim.op_id,
"check_mode": True,
"diff": True,
"check_mode_passed_before_apply": True,
},
derived_from_durable_chain=derived_from_durable_chain,
),
_runtime_stage_receipt(
claim,
stage_id="risk_policy_decision",
evidence_ref=f"ansible_policy:{claim.catalog_id}",
detail={
"risk_level": str(claim.risk_level or "").lower(),
"controlled_apply_allowed": True,
"break_glass_required": False,
"policy": "global_product_governance_v2",
},
derived_from_durable_chain=derived_from_durable_chain,
),
]
def build_ansible_context_runtime_stage_receipts(
claim: AnsibleCheckModeClaim,
evidence: dict[str, Any],
*,
derived_from_durable_chain: bool = False,
) -> list[dict[str, Any]]:
"""Project public-safe pre-decision evidence into the current run."""
evidence_id = str(evidence.get("id") or "")
if not evidence_id:
return []
receipts: list[dict[str, Any]] = []
mcp_health = _json_loads(evidence.get("mcp_health"))
sensors_attempted = max(0, _int_from_value(evidence.get("sensors_attempted"), default=0))
sensors_succeeded = max(0, _int_from_value(evidence.get("sensors_succeeded"), default=0))
successful_tool_count = sum(
1 for ready in mcp_health.values() if ready is True
)
if sensors_succeeded > 0 and successful_tool_count > 0:
receipts.append(
_runtime_stage_receipt(
claim,
stage_id="mcp_context",
evidence_ref=f"incident_evidence:{evidence_id}:mcp_health",
detail={
"evidence_id": evidence_id,
"tool_status_count": len(mcp_health),
"successful_tool_count": successful_tool_count,
"sensors_attempted": sensors_attempted,
"sensors_succeeded": sensors_succeeded,
"context_collected_before_candidate": True,
},
derived_from_durable_chain=derived_from_durable_chain,
)
)
recent_logs = evidence.get("recent_logs")
if isinstance(recent_logs, str) and recent_logs.strip():
receipts.append(
_runtime_stage_receipt(
claim,
stage_id="service_log_evidence",
evidence_ref=f"incident_evidence:{evidence_id}:recent_logs",
detail={
"evidence_id": evidence_id,
"sanitized_character_count": len(recent_logs),
"content_in_receipt": False,
"redaction_contract": "incident_evidence.recent_logs_sanitized",
"context_collected_before_candidate": True,
},
derived_from_durable_chain=derived_from_durable_chain,
)
)
return receipts
def build_ansible_post_apply_runtime_stage_receipts(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
verifier_ready: bool,
derived_from_durable_chain: bool = False,
) -> list[dict[str, Any]]:
receipts = [
_runtime_stage_receipt(
claim,
stage_id="executor_log_projection",
evidence_ref=f"automation_operation_log:{apply_op_id}:output",
detail={
"apply_op_id": apply_op_id,
"returncode": result.returncode,
"timed_out": result.timed_out,
"duration_ms": result.duration_ms,
"projection_sanitized": True,
},
derived_from_durable_chain=derived_from_durable_chain,
)
]
if result.returncode == 0 and verifier_ready:
receipts.append(
_runtime_stage_receipt(
claim,
stage_id="retry_or_rollback",
evidence_ref=f"incident_evidence:{apply_op_id}:post_execution_state",
detail={
"apply_op_id": apply_op_id,
"terminal_type": "verified_success_no_retry_or_rollback_required",
"bounded_retry_performed": False,
"rollback_performed": False,
"post_apply_verifier_passed": True,
},
derived_from_durable_chain=derived_from_durable_chain,
)
)
return receipts
def build_ansible_timeline_runtime_stage_receipt(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
timeline_event_id: str,
derived_from_durable_chain: bool = False,
) -> dict[str, Any]:
verified_success = bool(
result.returncode == 0 and result.post_verifier_passed is True
)
return _runtime_stage_receipt(
claim,
stage_id="timeline_projection",
evidence_ref=f"timeline_events:{timeline_event_id}",
detail={
"timeline_event_id": timeline_event_id,
"apply_op_id": apply_op_id,
"returncode": result.returncode,
"post_verifier_passed": result.post_verifier_passed is True,
"projection_status": "success" if verified_success else "error",
},
derived_from_durable_chain=derived_from_durable_chain,
)
async def _load_pre_decision_context_runtime_stage_receipts(
claim: AnsibleCheckModeClaim,
*,
project_id: str,
derived_from_durable_chain: bool = False,
) -> list[dict[str, Any]]:
"""Load only evidence collected before the root candidate was created."""
try:
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT
evidence.id,
evidence.recent_logs,
evidence.mcp_health,
evidence.sensors_attempted,
evidence.sensors_succeeded,
evidence.collected_at
FROM incident_evidence evidence
JOIN automation_operation_log candidate
ON candidate.op_id = CAST(:candidate_op_id AS uuid)
WHERE evidence.incident_id = CAST(:incident_id AS varchar(30))
AND evidence.collected_at <= candidate.created_at
AND (
evidence.post_execution_state IS NULL
OR CAST(evidence.post_execution_state AS jsonb)
= 'null'::jsonb
)
AND (
NULLIF(evidence.recent_logs, '') IS NOT NULL
OR evidence.mcp_health IS NOT NULL
)
ORDER BY evidence.collected_at DESC
LIMIT 10
"""),
{
"candidate_op_id": claim.source_candidate_op_id,
"incident_id": claim.incident_id,
},
)
evidence_rows = [dict(row) for row in result.mappings().all()]
except Exception as exc:
logger.warning(
"ansible_pre_decision_context_receipt_read_failed",
incident_id=claim.incident_id,
candidate_op_id=claim.source_candidate_op_id,
error=str(exc),
)
return []
by_stage: dict[str, dict[str, Any]] = {}
for evidence in evidence_rows:
for receipt in build_ansible_context_runtime_stage_receipts(
claim,
evidence,
derived_from_durable_chain=derived_from_durable_chain,
):
by_stage.setdefault(str(receipt["stage_id"]), receipt)
return list(by_stage.values())
async def _record_timeline_projection_receipt(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
project_id: str,
derived_from_durable_chain: bool = False,
) -> dict[str, Any] | None:
"""Persist one idempotent operator timeline event and return its receipt."""
automation_run_id = _automation_run_id_for_claim(claim)
verified_success = bool(
result.returncode == 0 and result.post_verifier_passed is True
)
description = (
f"automation_run_id={automation_run_id};apply_op_id={apply_op_id};"
f"catalog_id={claim.catalog_id};returncode={result.returncode};"
f"post_verifier_passed={str(result.post_verifier_passed is True).lower()}"
)
try:
async with get_db_context(project_id) as db:
row = await db.execute(
text("""
WITH existing AS (
SELECT id
FROM timeline_events
WHERE incident_id = :incident_id
AND actor = 'ansible_controlled_apply_worker'
AND description = :description
ORDER BY created_at DESC
LIMIT 1
), inserted AS (
INSERT INTO timeline_events (
id, incident_id, event_type, status, title, description,
actor, actor_role, risk_level, created_at
)
SELECT
gen_random_uuid()::text,
:incident_id,
'exec',
:status,
:title,
:description,
'ansible_controlled_apply_worker',
'ai_agent',
:risk_level,
NOW()
WHERE NOT EXISTS (SELECT 1 FROM existing)
RETURNING id
)
SELECT id FROM inserted
UNION ALL
SELECT id FROM existing
LIMIT 1
"""),
{
"incident_id": claim.incident_id,
"status": "success" if verified_success else "error",
"title": f"AI controlled apply: {claim.catalog_id}"[:500],
"description": description,
"risk_level": str(claim.risk_level or "")[:20] or None,
},
)
timeline_event_id = str(row.scalar() or "")
except Exception as exc:
logger.warning(
"ansible_timeline_projection_write_failed",
incident_id=claim.incident_id,
apply_op_id=apply_op_id,
error=str(exc),
)
return None
if not timeline_event_id:
return None
return build_ansible_timeline_runtime_stage_receipt(
claim,
result,
apply_op_id=apply_op_id,
timeline_event_id=timeline_event_id,
derived_from_durable_chain=derived_from_durable_chain,
)
async def _record_runtime_stage_receipts(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
verifier_ready: bool,
project_id: str,
derived_from_durable_chain: bool = False,
extra_receipts: tuple[dict[str, Any], ...] = (),
) -> bool:
context_receipts = await _load_pre_decision_context_runtime_stage_receipts(
claim,
project_id=project_id,
derived_from_durable_chain=derived_from_durable_chain,
)
receipts = [
*context_receipts,
*build_ansible_pre_apply_runtime_stage_receipts(
claim,
derived_from_durable_chain=derived_from_durable_chain,
),
*build_ansible_post_apply_runtime_stage_receipts(
claim,
result,
apply_op_id=apply_op_id,
verifier_ready=verifier_ready,
derived_from_durable_chain=derived_from_durable_chain,
),
*extra_receipts,
]
try:
async with get_db_context(project_id) as db:
updated = await db.execute(
text("""
UPDATE automation_operation_log AS apply
SET input = jsonb_set(
coalesce(apply.input, '{}'::jsonb),
'{runtime_stage_receipts}',
(
SELECT coalesce(
jsonb_agg(
deduplicated.receipt
ORDER BY deduplicated.stage_id
),
'[]'::jsonb
)
FROM (
SELECT DISTINCT ON (
merged.receipt ->> 'stage_id'
)
merged.receipt,
merged.receipt ->> 'stage_id' AS stage_id
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
) || CAST(:receipts AS jsonb)
) WITH ORDINALITY AS merged(receipt, position)
WHERE merged.receipt ->> 'stage_id' IS NOT NULL
ORDER BY
merged.receipt ->> 'stage_id',
merged.position DESC
) deduplicated
),
true
)
WHERE apply.op_id = CAST(:apply_op_id AS uuid)
AND apply.operation_type = 'ansible_apply_executed'
"""),
{
"receipts": json.dumps(receipts, ensure_ascii=False),
"apply_op_id": apply_op_id,
},
)
return bool(updated.rowcount)
except Exception as exc:
logger.warning(
"ansible_runtime_stage_receipt_write_failed",
incident_id=claim.incident_id,
apply_op_id=apply_op_id,
error=str(exc),
)
return False
async def _append_runtime_stage_receipts_to_apply(
*,
apply_op_id: str,
receipts: tuple[dict[str, Any], ...],
project_id: str,
) -> bool:
return await _append_runtime_stage_receipts_to_operation(
operation_id=apply_op_id,
operation_type="ansible_apply_executed",
receipts=receipts,
project_id=project_id,
)
async def _append_runtime_stage_receipts_to_operation(
*,
operation_id: str,
operation_type: str,
receipts: tuple[dict[str, Any], ...],
project_id: str,
) -> bool:
if not receipts:
return False
try:
async with get_db_context(project_id) as db:
updated = await db.execute(
text("""
UPDATE automation_operation_log apply
SET input = jsonb_set(
coalesce(apply.input, '{}'::jsonb),
'{runtime_stage_receipts}',
coalesce(
(
SELECT jsonb_agg(
deduplicated.receipt
ORDER BY deduplicated.stage_id
)
FROM (
SELECT DISTINCT ON (
merged.receipt ->> 'stage_id'
)
merged.receipt,
merged.receipt ->> 'stage_id' AS stage_id
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
) || CAST(:receipts AS jsonb)
) WITH ORDINALITY AS merged(receipt, position)
WHERE merged.receipt ->> 'stage_id' IS NOT NULL
ORDER BY
merged.receipt ->> 'stage_id',
merged.position DESC
) deduplicated
),
'[]'::jsonb
),
true
)
WHERE apply.op_id = CAST(:operation_id AS uuid)
AND apply.operation_type = :operation_type
"""),
{
"receipts": json.dumps(receipts, ensure_ascii=False),
"operation_id": operation_id,
"operation_type": operation_type,
},
)
return bool(updated.rowcount)
except Exception as exc:
logger.warning(
"ansible_runtime_terminal_receipt_append_failed",
apply_op_id=operation_id,
operation_type=operation_type,
error_type=type(exc).__name__,
)
return False
def _runtime_stage_ids(value: Any) -> set[str]:
payload = _json_loads(value)
receipts = payload.get("runtime_stage_receipts")
if not isinstance(receipts, list):
return set()
return {
str(receipt.get("stage_id") or "")
for receipt in receipts
if isinstance(receipt, Mapping) and receipt.get("durable_receipt") is True
}
async def _read_verified_apply_closure_prerequisites(
claim: AnsibleCheckModeClaim,
*,
apply_op_id: str,
project_id: str,
) -> dict[str, Any]:
"""Read back every durable prerequisite before resolving an incident."""
automation_run_id = _automation_run_id_for_claim(claim)
approval_id = _approval_id_for_claim(claim)
canonical_playbook_id = canonical_ansible_playbook_id(claim.catalog_id)
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT
apply.status = 'success' AS apply_success,
apply.input AS apply_input,
coalesce(
apply.output ->> 'independent_post_verifier_passed',
'false'
) = 'true' AS apply_verifier_terminal,
check_mode.status = 'success' AS check_mode_success,
candidate.op_id IS NOT NULL AS candidate_present,
EXISTS (
SELECT 1
FROM auto_repair_executions receipt
WHERE receipt.incident_id = :incident_id
AND receipt.triggered_by = 'ansible_controlled_apply'
AND receipt.success IS TRUE
AND receipt.executed_steps::text LIKE
'%' || :apply_op_id || '%'
) AS auto_repair_receipt,
EXISTS (
SELECT 1
FROM incident_evidence evidence
WHERE evidence.incident_id = :incident_id
AND evidence.post_execution_state ->> 'apply_op_id'
= :apply_op_id
AND evidence.verification_result = 'success'
) AS post_verifier_receipt,
EXISTS (
SELECT 1
FROM knowledge_entries km
WHERE km.related_incident_id = :incident_id
AND km.path_type = :km_path_type
AND km.related_playbook_id = :canonical_playbook_id
AND km.updated_at IS NOT NULL
) AS km_writeback_receipt,
EXISTS (
SELECT 1
FROM automation_operation_log learning
WHERE learning.operation_type
= 'ansible_learning_writeback_recorded'
AND learning.parent_op_id = apply.op_id
AND learning.status = 'success'
AND learning.output ->> 'learning_recorded' = 'true'
AND learning.output ->> 'trust_updated' = 'true'
) AS playbook_trust_receipt,
EXISTS (
SELECT 1
FROM playbooks playbook
WHERE playbook.playbook_id = :canonical_playbook_id
AND playbook.success_count > 0
AND playbook.last_used_at IS NOT NULL
) AS playbook_readback,
EXISTS (
SELECT 1
FROM timeline_events timeline
WHERE timeline.incident_id = :incident_id
AND timeline.actor = 'ansible_controlled_apply_worker'
AND timeline.status = 'success'
AND timeline.description LIKE
'%apply_op_id=' || :apply_op_id || ';%'
) AS timeline_readback,
EXISTS (
SELECT 1
FROM awooop_outbound_message outbound
WHERE outbound.project_id = :project_id
AND outbound.channel_type = 'telegram'
AND outbound.send_status = 'sent'
AND outbound.provider_message_id IS NOT NULL
AND outbound.source_envelope #>>
'{callback_reply,action}'
= 'controlled_apply_result'
AND outbound.source_envelope
#>> '{callback_reply,action}'
= 'controlled_apply_result'
AND coalesce(
outbound.source_envelope
->> 'automation_run_id',
outbound.source_envelope
#>> '{callback_reply,automation_run_id}'
) = :automation_run_id
AND coalesce(
outbound.source_envelope
#>> '{callback_reply,incident_id}',
outbound.source_envelope
#>> '{source_refs,incident_ids,0}'
) = :incident_id
AND outbound.source_envelope #>>
'{callback_reply,apply_op_id}' = :apply_op_id
) AS telegram_receipt,
(
:approval_id = ''
OR EXISTS (
SELECT 1
FROM approval_records approval
WHERE approval.id = :approval_id
AND lower(approval.status::text)
= 'execution_success'
AND approval.extra_metadata ->> 'apply_op_id'
= :apply_op_id
AND approval.extra_metadata
->> 'post_verifier_passed' = 'true'
)
) AS approval_projection,
EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = :incident_id
AND lifecycle.event_type::text
= 'EXECUTION_COMPLETED'
AND lifecycle.success IS TRUE
AND lifecycle.context ->> 'apply_op_id'
= :apply_op_id
AND lifecycle.context ->> 'automation_run_id'
= :automation_run_id
) AS execution_lifecycle,
EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = :incident_id
AND lifecycle.event_type::text
= 'TELEGRAM_RESULT_SENT'
AND lifecycle.success IS TRUE
AND lifecycle.context ->> 'apply_op_id'
= :apply_op_id
AND lifecycle.context ->> 'automation_run_id'
= :automation_run_id
) AS telegram_lifecycle
FROM automation_operation_log apply
LEFT JOIN automation_operation_log check_mode
ON check_mode.op_id = apply.parent_op_id
AND check_mode.operation_type = 'ansible_check_mode_executed'
LEFT JOIN automation_operation_log candidate
ON candidate.op_id = check_mode.parent_op_id
AND candidate.operation_type = 'ansible_candidate_matched'
WHERE apply.op_id = CAST(:apply_op_id_uuid AS uuid)
AND apply.operation_type = 'ansible_apply_executed'
LIMIT 1
"""),
{
"apply_op_id": apply_op_id,
"apply_op_id_uuid": apply_op_id,
"incident_id": claim.incident_id,
"project_id": project_id,
"automation_run_id": automation_run_id,
"approval_id": approval_id,
"canonical_playbook_id": canonical_playbook_id,
"km_path_type": f"ansible_apply_receipt:{apply_op_id[:8]}",
},
)
row = result.mappings().one_or_none()
if row is None:
return {
"ready": False,
"missing": ["apply_operation_readback"],
"receipts": {},
"stage_ids": [],
}
stage_ids = _runtime_stage_ids(row.get("apply_input"))
receipts = {
"candidate": row.get("candidate_present") is True,
"check_mode": row.get("check_mode_success") is True,
"controlled_apply": row.get("apply_success") is True,
"post_apply_verifier": (
row.get("apply_verifier_terminal") is True
and row.get("post_verifier_receipt") is True
),
"auto_repair_execution_receipt": row.get("auto_repair_receipt") is True,
"km_playbook_writeback": row.get("km_writeback_receipt") is True,
"rag_writeback": "rag_writeback" in stage_ids,
"playbook_trust": (
row.get("playbook_trust_receipt") is True
and row.get("playbook_readback") is True
),
"timeline_projection": (
"timeline_projection" in stage_ids
and row.get("timeline_readback") is True
),
"telegram_receipt": row.get("telegram_receipt") is True,
"approval_projection": row.get("approval_projection") is True,
"execution_lifecycle": row.get("execution_lifecycle") is True,
"telegram_lifecycle": row.get("telegram_lifecycle") is True,
"required_runtime_stage_receipts": (
_VERIFIED_CLOSURE_REQUIRED_STAGE_IDS.issubset(stage_ids)
),
}
missing = sorted(name for name, ready in receipts.items() if not ready)
return {
"ready": not missing,
"missing": missing,
"receipts": receipts,
"stage_ids": sorted(stage_ids),
"required_stage_ids": sorted(_VERIFIED_CLOSURE_REQUIRED_STAGE_IDS),
"automation_run_id": automation_run_id,
"canonical_playbook_id": canonical_playbook_id,
}
async def _project_incident_terminal_to_working_memory(
*,
incident_id: str,
project_id: str,
incident_status: str,
updated_at: datetime,
resolved_at: datetime | None,
) -> str:
from src.models.incident import IncidentStatus
from src.services.incident_service import get_incident_service
normalized_status = str(incident_status or "").strip().lower()
if normalized_status not in {
IncidentStatus.RESOLVED.value,
IncidentStatus.CLOSED.value,
}:
return "failed"
return await get_incident_service().project_terminal_status_to_working_memory(
incident_id,
project_id=project_id,
status=IncidentStatus(normalized_status),
updated_at=updated_at,
resolved_at=resolved_at,
)
async def reconcile_verified_incident_working_memory_once(
*,
project_id: str = "awoooi",
window_hours: int = _WORKING_MEMORY_PROJECTION_WINDOW_HOURS,
limit: int = 10,
) -> dict[str, Any]:
"""Project pending terminal DB rows without starving older Redis repairs."""
stats: dict[str, Any] = {
"scanned": 0,
"updated": 0,
"current": 0,
"missing": 0,
"failed": 0,
"error": None,
}
try:
cursor = _WORKING_MEMORY_PROJECTION_CURSOR.get(project_id)
statement = text("""
SELECT
incident.incident_id,
incident.status::text AS incident_status,
incident.updated_at,
incident.resolved_at
FROM incidents incident
WHERE incident.project_id = :project_id
AND upper(incident.status::text) IN ('RESOLVED', 'CLOSED')
AND incident.resolved_at IS NOT NULL
AND cast(incident.outcome AS jsonb) #>>
'{automation_terminal,incident_resolved}' = 'true'
AND incident.updated_at >=
NOW() - (:window_hours * INTERVAL '1 hour')
AND (
coalesce(
cast(incident.outcome AS jsonb) #>>
'{working_memory_projection,ready}',
'false'
) <> 'true'
OR cast(incident.outcome AS jsonb) #>>
'{working_memory_projection,incident_status}'
IS DISTINCT FROM incident.status::text
OR cast(incident.outcome AS jsonb) #>>
'{working_memory_projection,source_updated_at_epoch_us}'
IS DISTINCT FROM CAST(
CAST(
round(
extract(epoch FROM incident.updated_at)
* 1000000
) AS bigint
) AS text
)
)
AND (
CAST(:cursor_updated_at AS timestamptz) IS NULL
OR incident.updated_at >
CAST(:cursor_updated_at AS timestamptz)
OR (
incident.updated_at =
CAST(:cursor_updated_at AS timestamptz)
AND incident.incident_id > :cursor_incident_id
)
)
ORDER BY incident.updated_at ASC, incident.incident_id ASC
LIMIT :limit
""")
def query_params(
active_cursor: tuple[datetime, str] | None,
) -> dict[str, Any]:
return {
"project_id": project_id,
"window_hours": max(1, int(window_hours)),
"limit": max(1, int(limit)),
"cursor_updated_at": active_cursor[0] if active_cursor else None,
"cursor_incident_id": active_cursor[1] if active_cursor else "",
}
async with get_db_context(project_id) as db:
result = await db.execute(statement, query_params(cursor))
rows = [dict(row) for row in result.mappings().all()]
if not rows and cursor is not None:
cursor = None
result = await db.execute(statement, query_params(None))
rows = [dict(row) for row in result.mappings().all()]
if rows:
last = rows[-1]
_WORKING_MEMORY_PROJECTION_CURSOR[project_id] = (
last["updated_at"],
str(last.get("incident_id") or ""),
)
else:
_WORKING_MEMORY_PROJECTION_CURSOR.pop(project_id, None)
stats["scanned"] = len(rows)
for row in rows:
try:
projection_status = (
await _project_incident_terminal_to_working_memory(
incident_id=str(row.get("incident_id") or ""),
project_id=project_id,
incident_status=str(row.get("incident_status") or ""),
updated_at=row["updated_at"],
resolved_at=row.get("resolved_at"),
)
)
except Exception as exc:
stats["failed"] += 1
logger.warning(
"ansible_incident_working_memory_projection_failed",
project_id=project_id,
incident_id=str(row.get("incident_id") or ""),
error_type=type(exc).__name__,
)
continue
if projection_status in stats:
stats[projection_status] += 1
else:
stats["failed"] += 1
except Exception as exc:
stats["error"] = f"{type(exc).__name__}: {exc}"[:500]
logger.warning(
"ansible_incident_working_memory_reconcile_failed",
project_id=project_id,
error_type=type(exc).__name__,
)
return stats
async def _read_incident_closure_readback(
claim: AnsibleCheckModeClaim,
*,
apply_op_id: str,
project_id: str,
) -> dict[str, Any]:
automation_run_id = _automation_run_id_for_claim(claim)
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT
incident.status::text AS incident_status,
incident.resolved_at,
incident.outcome,
EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) receipt(value)
WHERE receipt.value ->> 'stage_id' = 'incident_closure'
AND receipt.value ->> 'automation_run_id'
= :automation_run_id
AND receipt.value ->> 'durable_receipt' = 'true'
) AS closure_receipt,
EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = :incident_id
AND lifecycle.event_type::text = 'RESOLVED'
AND lifecycle.success IS TRUE
AND lifecycle.context ->> 'apply_op_id'
= :apply_op_id
AND lifecycle.context ->> 'automation_run_id'
= :automation_run_id
) AS resolved_lifecycle
FROM incidents incident
JOIN automation_operation_log apply
ON apply.op_id = CAST(:apply_op_id_uuid AS uuid)
WHERE incident.incident_id = :incident_id
AND incident.project_id = :project_id
LIMIT 1
"""),
{
"incident_id": claim.incident_id,
"project_id": project_id,
"apply_op_id": apply_op_id,
"apply_op_id_uuid": apply_op_id,
"automation_run_id": automation_run_id,
},
)
row = result.mappings().one_or_none()
if row is None:
return {"closed": False, "missing": ["incident_readback"]}
outcome = _json_loads(row.get("outcome"))
terminal = outcome.get("automation_terminal")
terminal_ready = bool(
isinstance(terminal, Mapping)
and terminal.get("automation_run_id") == automation_run_id
and terminal.get("apply_op_id") == apply_op_id
and terminal.get("incident_resolved") is True
)
checks = {
"incident_status": str(row.get("incident_status") or "").upper()
in {"RESOLVED", "CLOSED"},
"incident_resolved_at": row.get("resolved_at") is not None,
"incident_terminal_readback": terminal_ready,
"incident_closure_receipt": row.get("closure_receipt") is True,
"resolved_lifecycle": row.get("resolved_lifecycle") is True,
}
missing = sorted(name for name, ready in checks.items() if not ready)
return {
"closed": not missing,
"missing": missing,
"checks": checks,
"incident_status": str(row.get("incident_status") or ""),
"automation_run_id": automation_run_id,
"apply_op_id": apply_op_id,
}
def _incident_terminal_disposition_payload(
claim: AnsibleCheckModeClaim,
*,
apply_op_id: str,
terminal_type: str,
success_terminal: bool,
retry_op_id: str | None,
telegram_receipt_acknowledged: bool,
) -> dict[str, Any]:
return {
"schema_version": "ansible_incident_terminal_disposition_v1",
"automation_run_id": _automation_run_id_for_claim(claim),
"apply_op_id": apply_op_id,
"retry_op_id": retry_op_id,
"terminal_type": terminal_type,
"incident_status": "RESOLVED" if success_terminal else "MITIGATING",
"incident_resolved": success_terminal,
"no_write_terminal": not success_terminal,
"safe_next_action": (
"keep_verified_repair_and_monitor_recurrence"
if success_terminal
else "queue_ai_playbook_or_transport_repair_candidate"
),
"telegram_receipt_acknowledged": telegram_receipt_acknowledged,
"raw_log_payload_stored": False,
"secret_value_stored": False,
}
async def _commit_verified_incident_closure(
claim: AnsibleCheckModeClaim,
*,
apply_op_id: str,
project_id: str,
closure_prerequisite_count: int,
) -> dict[str, Any] | None:
"""Atomically persist the success terminal, receipt, and lifecycle row."""
terminal = {
**_incident_terminal_disposition_payload(
claim,
apply_op_id=apply_op_id,
terminal_type="controlled_apply_verified_closed",
success_terminal=True,
retry_op_id=None,
telegram_receipt_acknowledged=True,
),
"incident_id": claim.incident_id,
"proposal_executed": True,
"execution_success": True,
"repository_write_acknowledged": True,
"repository_readback_verified": True,
"durable_write_acknowledged": True,
"writer_source_sha": os.getenv(
"AWOOOI_BUILD_COMMIT_SHA",
"",
).strip().lower()
or None,
}
closure_receipt = _runtime_stage_receipt(
claim,
stage_id="incident_closure",
evidence_ref=f"incidents:{claim.incident_id}:automation_terminal",
detail={
**terminal,
"closure_prerequisite_count": closure_prerequisite_count,
"closure_prerequisites_verified": True,
},
)
automation_run_id = _automation_run_id_for_claim(claim)
lifecycle_context = {
"automation_run_id": automation_run_id,
"catalog_id": claim.catalog_id,
"check_mode_op_id": claim.op_id,
"apply_op_id": apply_op_id,
"single_writer_executor": "awoooi-ansible-executor-broker",
"post_verifier_passed": True,
}
try:
async with get_db_context(project_id) as db:
await db.execute(
text("""
SELECT pg_advisory_xact_lock(
hashtextextended(:closure_lock_key, 0)
)
"""),
{"closure_lock_key": f"ansible-closure:{apply_op_id}"},
)
committed = await db.execute(
text("""
WITH target_apply AS (
SELECT apply.op_id
FROM automation_operation_log apply
WHERE apply.op_id = CAST(:apply_op_id_uuid AS uuid)
AND apply.operation_type = 'ansible_apply_executed'
AND apply.status = 'success'
AND coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
) = :incident_id
AND apply.input ->> 'automation_run_id'
= :automation_run_id
AND coalesce(
apply.output
->> 'independent_post_verifier_passed',
'false'
) = 'true'
FOR UPDATE
),
lifecycle_existing AS (
SELECT lifecycle.id
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = :incident_id
AND lifecycle.event_type::text = 'RESOLVED'
AND lifecycle.success IS TRUE
AND lifecycle.context ->> 'apply_op_id'
= :apply_op_id
AND lifecycle.context ->> 'automation_run_id'
= :automation_run_id
),
lifecycle_inserted AS (
INSERT INTO alert_operation_log (
incident_id,
approval_id,
event_type,
actor,
action_detail,
success,
context
)
SELECT
CAST(:incident_id AS varchar(30)),
CAST(:approval_id AS varchar(36)),
'RESOLVED'::alert_event_type,
'awoooi-ansible-executor-broker',
'controlled_apply_verified_closed',
true,
CAST(:lifecycle_context AS jsonb)
FROM target_apply
WHERE NOT EXISTS (
SELECT 1 FROM lifecycle_existing
)
RETURNING id
),
lifecycle_ready AS (
SELECT id FROM lifecycle_existing
UNION ALL
SELECT id FROM lifecycle_inserted
),
apply_updated AS (
UPDATE automation_operation_log apply
SET input = jsonb_set(
coalesce(apply.input, '{}'::jsonb),
'{runtime_stage_receipts}',
coalesce(
(
SELECT jsonb_agg(
deduplicated.receipt
ORDER BY deduplicated.stage_id
)
FROM (
SELECT DISTINCT ON (
merged.receipt ->> 'stage_id'
)
merged.receipt,
merged.receipt ->> 'stage_id'
AS stage_id
FROM jsonb_array_elements(
coalesce(
apply.input
-> 'runtime_stage_receipts',
'[]'::jsonb
) || CAST(:receipts AS jsonb)
) WITH ORDINALITY
AS merged(receipt, position)
WHERE merged.receipt ->> 'stage_id'
IS NOT NULL
ORDER BY
merged.receipt ->> 'stage_id',
merged.position DESC
) deduplicated
),
'[]'::jsonb
),
true
)
FROM target_apply
WHERE apply.op_id = target_apply.op_id
AND EXISTS (SELECT 1 FROM lifecycle_ready)
RETURNING apply.op_id
),
incident_updated AS (
UPDATE incidents incident
SET status = CASE
WHEN upper(incident.status::text) = 'CLOSED'
THEN incident.status
ELSE 'RESOLVED'::incidentstatus
END,
outcome = CAST(
coalesce(
CASE
WHEN incident.outcome IS NULL
THEN '{}'::jsonb
WHEN jsonb_typeof(
CAST(incident.outcome AS jsonb)
) = 'object'
THEN CAST(incident.outcome AS jsonb)
ELSE jsonb_build_object(
'legacy_outcome',
CAST(incident.outcome AS jsonb)
)
END,
'{}'::jsonb
)
|| jsonb_build_object(
'automation_terminal',
CAST(:terminal_payload AS jsonb),
'proposal_executed',
true,
'execution_success',
true
)
AS json
),
resolved_at = coalesce(incident.resolved_at, NOW()),
updated_at = NOW()
WHERE incident.incident_id = :incident_id
AND incident.project_id = :project_id
AND EXISTS (SELECT 1 FROM apply_updated)
AND (
upper(incident.status::text)
NOT IN ('RESOLVED', 'CLOSED')
OR cast(incident.outcome AS jsonb) #>>
'{automation_terminal,automation_run_id}'
= :automation_run_id
)
RETURNING
incident.status::text AS incident_status,
incident.updated_at,
incident.resolved_at,
incident.outcome
)
SELECT
incident_status,
updated_at,
resolved_at,
outcome,
EXISTS (SELECT 1 FROM apply_updated)
AS closure_receipt_written,
EXISTS (SELECT 1 FROM lifecycle_ready)
AS resolved_lifecycle_written
FROM incident_updated
"""),
{
"apply_op_id": apply_op_id,
"apply_op_id_uuid": apply_op_id,
"incident_id": claim.incident_id,
"project_id": project_id,
"automation_run_id": automation_run_id,
"approval_id": _approval_id_for_claim(claim) or None,
"lifecycle_context": json.dumps(
lifecycle_context,
ensure_ascii=False,
),
"receipts": json.dumps(
[closure_receipt],
ensure_ascii=False,
),
"terminal_payload": json.dumps(
terminal,
ensure_ascii=False,
),
},
)
row = committed.mappings().one_or_none()
if row is None:
raise RuntimeError("verified_incident_closure_atomic_write_pending")
outcome = _json_loads(row.get("outcome"))
readback = outcome.get("automation_terminal")
if not (
isinstance(readback, Mapping)
and readback.get("automation_run_id") == automation_run_id
and readback.get("apply_op_id") == apply_op_id
and outcome.get("proposal_executed") is True
and outcome.get("execution_success") is True
and row.get("closure_receipt_written") is True
and row.get("resolved_lifecycle_written") is True
and row.get("resolved_at") is not None
):
raise RuntimeError("verified_incident_closure_atomic_readback_failed")
closure_result = {
**terminal,
"incident_status": str(row.get("incident_status") or ""),
"incident_row_version": (
row["updated_at"].isoformat()
if row.get("updated_at") is not None
else None
),
"closure_receipt_written": True,
"resolved_lifecycle_written": True,
}
try:
projection_status = await _project_incident_terminal_to_working_memory(
incident_id=claim.incident_id,
project_id=project_id,
incident_status=closure_result["incident_status"],
updated_at=row["updated_at"],
resolved_at=row.get("resolved_at"),
)
except Exception as projection_exc:
projection_status = "failed"
logger.warning(
"ansible_verified_incident_closure_projection_failed",
automation_run_id=automation_run_id,
incident_id=claim.incident_id,
apply_op_id=apply_op_id,
error_type=type(projection_exc).__name__,
durable_closure_preserved=True,
)
closure_result["working_memory_projection_status"] = projection_status
closure_result["working_memory_projection_ready"] = (
projection_status in {"updated", "current"}
)
if closure_result["working_memory_projection_ready"] is not True:
logger.warning(
"ansible_verified_incident_closure_working_memory_pending",
automation_run_id=automation_run_id,
incident_id=claim.incident_id,
apply_op_id=apply_op_id,
projection_status=projection_status,
)
return closure_result
except Exception as exc:
logger.warning(
"ansible_verified_incident_closure_atomic_write_failed",
automation_run_id=automation_run_id,
incident_id=claim.incident_id,
apply_op_id=apply_op_id,
error_type=type(exc).__name__,
)
return None
async def _finalize_verified_apply_closure(
claim: AnsibleCheckModeClaim,
*,
apply_op_id: str,
project_id: str,
) -> dict[str, Any]:
prerequisites = await _read_verified_apply_closure_prerequisites(
claim,
apply_op_id=apply_op_id,
project_id=project_id,
)
if prerequisites.get("ready") is not True:
return {
"status": "closure_receipts_pending",
"closed": False,
"missing": list(prerequisites.get("missing") or []),
}
terminal = await _commit_verified_incident_closure(
claim,
apply_op_id=apply_op_id,
project_id=project_id,
closure_prerequisite_count=len(prerequisites["receipts"]),
)
if terminal is None:
return {
"status": "atomic_closure_write_pending",
"closed": False,
"missing": [
"incident_terminal_readback",
"incident_closure_receipt",
"resolved_lifecycle",
],
}
readback = await _read_incident_closure_readback(
claim,
apply_op_id=apply_op_id,
project_id=project_id,
)
projection_ready = terminal.get("working_memory_projection_ready") is True
readback_closed = readback.get("closed") is True
missing = list(readback.get("missing") or [])
if not projection_ready:
missing = sorted({*missing, "working_memory_terminal_projection"})
return {
**readback,
"status": (
(
"controlled_apply_closed"
if projection_ready
else "controlled_apply_closed_projection_degraded"
)
if readback_closed
else "closure_readback_pending"
),
"closed": readback_closed,
"missing": missing,
"working_memory_projection_ready": projection_ready,
"read_model_degraded": not projection_ready,
}
async def _record_incident_terminal_disposition(
claim: AnsibleCheckModeClaim,
*,
apply_op_id: str,
project_id: str,
terminal_type: str,
success_terminal: bool,
retry_op_id: str | None = None,
telegram_receipt_acknowledged: bool,
) -> dict[str, Any] | None:
automation_run_id = _automation_run_id_for_claim(claim)
incident_status = "RESOLVED" if success_terminal else "MITIGATING"
terminal_payload = _incident_terminal_disposition_payload(
claim,
apply_op_id=apply_op_id,
terminal_type=terminal_type,
success_terminal=success_terminal,
retry_op_id=retry_op_id,
telegram_receipt_acknowledged=telegram_receipt_acknowledged,
)
try:
async with get_db_context(project_id) as db:
execution = await db.execute(
text("""
SELECT coalesce(
(apply.dry_run_result ->> 'apply_executed')::boolean,
(apply.output
->> 'controlled_apply_executed')::boolean,
false
) AS proposal_executed
FROM automation_operation_log apply
WHERE apply.op_id = CAST(:apply_op_id AS uuid)
AND apply.operation_type = 'ansible_apply_executed'
LIMIT 1
"""),
{"apply_op_id": apply_op_id},
)
execution_row = execution.mappings().one_or_none()
if execution_row is None:
return None
proposal_executed = execution_row.get("proposal_executed") is True
execution_success: bool | None = (
success_terminal if proposal_executed else None
)
if success_terminal and not proposal_executed:
return None
updated = await db.execute(
text("""
UPDATE incidents
SET status = CAST(:incident_status AS incidentstatus),
outcome = CAST(
coalesce(
CASE
WHEN outcome IS NULL THEN '{}'::jsonb
WHEN jsonb_typeof(CAST(outcome AS jsonb))
= 'object'
THEN CAST(outcome AS jsonb)
ELSE jsonb_build_object(
'legacy_outcome',
CAST(outcome AS jsonb)
)
END,
'{}'::jsonb
)
|| jsonb_build_object(
'automation_terminal',
CAST(:terminal_payload AS jsonb),
'proposal_executed',
CAST(:proposal_executed AS boolean),
'execution_success',
CAST(:execution_success AS boolean)
)
AS json
),
resolved_at = CASE
WHEN :success_terminal THEN NOW()
ELSE resolved_at
END,
updated_at = NOW()
WHERE incident_id = :incident_id
AND project_id = :project_id
AND (
:success_terminal
OR upper(status::text) NOT IN ('RESOLVED', 'CLOSED')
)
RETURNING
status::text AS incident_status,
updated_at,
resolved_at,
outcome
"""),
{
"incident_status": incident_status,
"terminal_payload": json.dumps(
terminal_payload,
ensure_ascii=False,
),
"success_terminal": success_terminal,
"proposal_executed": proposal_executed,
"execution_success": execution_success,
"incident_id": claim.incident_id,
"project_id": project_id,
},
)
row = updated.mappings().one_or_none()
if row is None:
return None
outcome = _json_loads(row.get("outcome"))
readback = outcome.get("automation_terminal")
if not isinstance(readback, Mapping):
return None
if (
outcome.get("proposal_executed") is not proposal_executed
or outcome.get("execution_success") is not execution_success
):
return None
if (
readback.get("automation_run_id") != automation_run_id
or readback.get("apply_op_id") != apply_op_id
or readback.get("terminal_type") != terminal_type
):
return None
return {
**terminal_payload,
"incident_id": claim.incident_id,
"incident_status": str(row.get("incident_status") or ""),
"proposal_executed": proposal_executed,
"execution_success": execution_success,
"incident_row_version": (
row["updated_at"].isoformat()
if row.get("updated_at") is not None
else None
),
"repository_write_acknowledged": True,
"repository_readback_verified": True,
"durable_write_acknowledged": True,
"writer_source_sha": os.getenv(
"AWOOOI_BUILD_COMMIT_SHA",
"",
).strip().lower()
or None,
}
except Exception as exc:
logger.warning(
"ansible_incident_terminal_disposition_write_failed",
automation_run_id=automation_run_id,
incident_id=claim.incident_id,
apply_op_id=apply_op_id,
error_type=type(exc).__name__,
)
return None
def _build_retry_runtime_stage_receipt(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
replay_op_id: str,
derived_from_durable_chain: bool = False,
check_mode_replay_performed: bool = True,
) -> dict[str, Any]:
terminal_type = (
"no_write_replay_passed_waiting_repair_candidate"
if result.returncode == 0
else "no_write_replay_failed_waiting_playbook_or_transport_repair"
)
return _runtime_stage_receipt(
claim,
stage_id="retry_or_rollback",
evidence_ref=f"automation_operation_log:{replay_op_id}:dry_run_result",
derived_from_durable_chain=derived_from_durable_chain,
detail={
"schema_version": "ansible_controlled_retry_terminal_v2",
"failed_apply_op_id": apply_op_id,
"retry_check_mode_op_id": replay_op_id,
"terminal_type": terminal_type,
"check_mode_replay_performed": check_mode_replay_performed,
"check_mode_replay_returncode": result.returncode,
"runtime_apply_executed": False,
"rollback_performed": False,
"verified_no_write_terminal": True,
"retry_operation_readback_verified": True,
"retry_attempt": 1,
"max_retry_attempts": _CONTROLLED_RETRY_MAX_ATTEMPTS,
"retry_idempotency_key": (
f"ansible-retry:{apply_op_id}:check-mode"
),
"retry_error_backoff_min_seconds": (
_CONTROLLED_RETRY_ERROR_BACKOFF_MIN_SECONDS
),
"retry_error_backoff_max_seconds": (
_CONTROLLED_RETRY_ERROR_BACKOFF_MAX_SECONDS
),
"bounded_retry": True,
"repair_candidate_required": True,
"safe_next_action": (
"queue_ai_playbook_or_transport_repair_candidate"
),
"repository_readback_verified": True,
"durable_write_acknowledged": True,
"writer_source_sha": os.getenv(
"AWOOOI_BUILD_COMMIT_SHA",
"",
).strip().lower()
or None,
"raw_log_payload_stored": False,
"secret_value_stored": False,
},
)
async def _retry_telegram_receipt_acknowledged(
claim: AnsibleCheckModeClaim,
*,
apply_op_id: str,
project_id: str,
) -> bool:
automation_run_id = _automation_run_id_for_claim(claim)
try:
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT EXISTS (
SELECT 1
FROM awooop_outbound_message outbound
WHERE outbound.project_id = :project_id
AND outbound.channel_type = 'telegram'
AND outbound.send_status = 'sent'
AND outbound.provider_message_id IS NOT NULL
AND outbound.source_envelope #>>
'{callback_reply,action}'
= 'controlled_apply_result'
AND coalesce(
outbound.source_envelope
->> 'automation_run_id',
outbound.source_envelope #>>
'{callback_reply,automation_run_id}',
outbound.source_envelope #>>
'{source_refs,automation_run_ids,0}'
) = :automation_run_id
AND coalesce(
outbound.source_envelope #>>
'{callback_reply,incident_id}',
outbound.source_envelope #>>
'{source_refs,incident_ids,0}'
) = :incident_id
AND outbound.source_envelope #>>
'{callback_reply,apply_op_id}' = :apply_op_id
)
"""),
{
"project_id": project_id,
"automation_run_id": automation_run_id,
"incident_id": claim.incident_id,
"apply_op_id": apply_op_id,
},
)
return result.scalar() is True
except Exception as exc:
logger.warning(
"ansible_retry_telegram_receipt_readback_failed",
automation_run_id=automation_run_id,
incident_id=claim.incident_id,
error_type=type(exc).__name__,
)
return False
async def _record_retry_runtime_stage_receipt(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
replay_op_id: str,
project_id: str,
check_mode_replay_performed: bool = True,
) -> bool:
"""Append the verified no-write retry terminal to the original apply run."""
automation_run_id = _automation_run_id_for_claim(claim)
terminal_type = (
"no_write_replay_passed_waiting_repair_candidate"
if result.returncode == 0
else "no_write_replay_failed_waiting_playbook_or_transport_repair"
)
replay_readback_verified = False
telegram_receipt_acknowledged = False
try:
async with get_db_context(project_id) as db:
replay_readback = await db.execute(
text("""
SELECT status, input, output, dry_run_result
FROM automation_operation_log
WHERE op_id = CAST(:replay_op_id AS uuid)
AND operation_type = 'ansible_execution_skipped'
AND parent_op_id = CAST(:apply_op_id AS uuid)
"""),
{
"replay_op_id": replay_op_id,
"apply_op_id": apply_op_id,
},
)
replay_row = replay_readback.mappings().one_or_none()
replay_input = _json_loads(
replay_row.get("input") if replay_row else None
)
replay_output = _json_loads(
replay_row.get("output") if replay_row else None
)
replay_readback_verified = bool(
replay_row
and replay_row.get("status")
== ("success" if result.returncode == 0 else "failed")
and replay_input.get("automation_run_id")
== automation_run_id
and replay_input.get("retry_of_apply_op_id") == apply_op_id
and replay_output.get("runtime_apply_executed") is False
and replay_output.get("terminal_disposition") == terminal_type
)
telegram_readback = await db.execute(
text("""
SELECT EXISTS (
SELECT 1
FROM awooop_outbound_message
WHERE project_id = :project_id
AND channel_type = 'telegram'
AND send_status = 'sent'
AND provider_message_id IS NOT NULL
AND source_envelope #>> '{callback_reply,action}'
= 'controlled_apply_result'
AND COALESCE(
source_envelope ->> 'automation_run_id',
source_envelope #>>
'{callback_reply,automation_run_id}',
source_envelope #>>
'{source_refs,automation_run_ids,0}'
) = :automation_run_id
AND COALESCE(
source_envelope #>>
'{callback_reply,incident_id}',
source_envelope #>>
'{source_refs,incident_ids,0}'
) = :incident_id
AND source_envelope #>>
'{callback_reply,apply_op_id}' = :apply_op_id
)
"""),
{
"project_id": project_id,
"automation_run_id": automation_run_id,
"incident_id": claim.incident_id,
"apply_op_id": apply_op_id,
},
)
telegram_receipt_acknowledged = telegram_readback.scalar() is True
except Exception as exc:
logger.warning(
"ansible_retry_terminal_readback_failed",
automation_run_id=automation_run_id,
apply_op_id=apply_op_id,
replay_op_id=replay_op_id,
error_type=type(exc).__name__,
)
if not replay_readback_verified:
logger.warning(
"ansible_retry_terminal_receipt_not_written_unverified",
automation_run_id=automation_run_id,
apply_op_id=apply_op_id,
replay_op_id=replay_op_id,
)
return False
retry_receipt = _build_retry_runtime_stage_receipt(
claim,
result,
apply_op_id=apply_op_id,
replay_op_id=replay_op_id,
check_mode_replay_performed=check_mode_replay_performed,
)
incident_detail = await _record_incident_terminal_disposition(
claim,
apply_op_id=apply_op_id,
project_id=project_id,
terminal_type=terminal_type,
success_terminal=False,
retry_op_id=replay_op_id,
telegram_receipt_acknowledged=telegram_receipt_acknowledged,
)
receipts = [retry_receipt]
if incident_detail is not None:
receipts.append(
_runtime_stage_receipt(
claim,
stage_id="incident_closure",
evidence_ref=(
f"incidents:{claim.incident_id}:automation_terminal"
),
detail=incident_detail,
)
)
return await _append_runtime_stage_receipts_to_apply(
apply_op_id=apply_op_id,
receipts=tuple(receipts),
project_id=project_id,
)
def _build_stdin_boundary_replay_stage_receipt(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
failed_check_mode_op_id: str,
terminal_op_id: str,
) -> dict[str, Any]:
terminal_type = (
"no_write_stdin_boundary_replay_passed_waiting_repair_candidate"
if result.returncode == 0
else "no_write_stdin_boundary_replay_failed_waiting_transport_repair"
)
return _runtime_stage_receipt(
claim,
stage_id="retry_or_rollback",
evidence_ref=(
f"automation_operation_log:{terminal_op_id}:dry_run_result"
),
detail={
"schema_version": (
"ansible_failed_check_stdin_boundary_retry_terminal_v1"
),
"failed_check_mode_op_id": failed_check_mode_op_id,
"retry_check_mode_op_id": claim.op_id,
"retry_terminal_op_id": terminal_op_id,
"terminal_type": terminal_type,
"check_mode_replay_performed": True,
"check_mode_replay_returncode": result.returncode,
"runtime_apply_executed": False,
"rollback_performed": False,
"verified_no_write_terminal": True,
"retry_operation_readback_verified": True,
"retry_attempt": 1,
"max_retry_attempts": 1,
"retry_idempotency_key": (
f"ansible-check-retry:{failed_check_mode_op_id}:stdin-boundary"
),
"bounded_retry": True,
"repair_candidate_required": True,
"safe_next_action": (
"queue_new_controlled_apply_candidate_after_no_write_replay"
if result.returncode == 0
else "queue_ai_transport_repair_candidate"
),
"repository_readback_verified": True,
"durable_write_acknowledged": True,
"writer_source_sha": os.getenv(
"AWOOOI_BUILD_COMMIT_SHA",
"",
).strip().lower()
or None,
"raw_log_payload_stored": False,
"secret_value_stored": False,
},
)
async def _record_stdin_boundary_replay_terminal(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
project_id: str,
) -> bool:
"""Persist one idempotent, public-safe terminal for a no-write replay."""
failed_check_mode_op_id = str(
claim.input_payload.get("replay_of_check_mode_op_id") or ""
)
if not failed_check_mode_op_id:
return False
automation_run_id = _automation_run_id_for_claim(claim)
expected_status = "success" if result.returncode == 0 else "failed"
terminal_type = (
"no_write_stdin_boundary_replay_passed_waiting_repair_candidate"
if result.returncode == 0
else "no_write_stdin_boundary_replay_failed_waiting_transport_repair"
)
try:
async with get_db_context(project_id) as db:
await db.execute(
text("""
SELECT pg_advisory_xact_lock(
hashtextextended(:retry_terminal_lock_key, 0)
)
"""),
{
"retry_terminal_lock_key": (
"ansible-check-retry-terminal:"
f"{failed_check_mode_op_id}"
)
},
)
selected = await db.execute(
text("""
SELECT op_id::text AS op_id
FROM automation_operation_log
WHERE operation_type = 'ansible_execution_skipped'
AND input ->> 'execution_mode'
= 'stdin_boundary_check_mode_replay_terminal'
AND input ->> 'retry_of_check_mode_op_id'
= :failed_check_mode_op_id
ORDER BY created_at DESC
LIMIT 1
"""),
{"failed_check_mode_op_id": failed_check_mode_op_id},
)
terminal_op_id = str(selected.scalar_one_or_none() or uuid4())
receipt = _build_stdin_boundary_replay_stage_receipt(
claim,
result,
failed_check_mode_op_id=failed_check_mode_op_id,
terminal_op_id=terminal_op_id,
)
terminal_input = {
"automation_run_id": automation_run_id,
"incident_id": claim.incident_id,
"catalog_id": claim.catalog_id,
"execution_mode": (
"stdin_boundary_check_mode_replay_terminal"
),
"retry_of_check_mode_op_id": failed_check_mode_op_id,
"retry_check_mode_op_id": claim.op_id,
"runtime_apply_executed": False,
"runtime_stage_receipts": [receipt],
}
terminal_output = {
"terminal_disposition": terminal_type,
"check_mode_replay_returncode": result.returncode,
"check_mode_replay_performed": True,
"runtime_apply_executed": False,
"verified_no_write_terminal": True,
"raw_log_payload_stored": False,
"secret_value_stored": False,
}
await db.execute(
text("""
INSERT INTO automation_operation_log (
op_id, operation_type, actor, status, incident_id,
input, output, dry_run_result, error, duration_ms,
parent_op_id, tags
)
SELECT
CAST(:op_id AS uuid),
'ansible_execution_skipped',
'ansible_stdin_boundary_replay_worker',
:status,
:incident_db_id,
CAST(:input AS jsonb),
CAST(:output AS jsonb),
CAST(:dry_run_result AS jsonb),
:error,
:duration_ms,
CAST(:parent_op_id AS uuid),
:tags
WHERE NOT EXISTS (
SELECT 1
FROM automation_operation_log existing
WHERE existing.operation_type
= 'ansible_execution_skipped'
AND existing.input ->> 'execution_mode'
= 'stdin_boundary_check_mode_replay_terminal'
AND existing.input ->> 'retry_of_check_mode_op_id'
= :failed_check_mode_op_id
)
"""),
{
"op_id": terminal_op_id,
"status": expected_status,
"incident_db_id": _automation_operation_log_incident_id(
claim.incident_id
),
"input": json.dumps(terminal_input, ensure_ascii=False),
"output": json.dumps(terminal_output, ensure_ascii=False),
"dry_run_result": json.dumps(
{
**terminal_output,
"check_mode": True,
"diff": True,
},
ensure_ascii=False,
),
"error": (
None
if result.returncode == 0
else (
"stdin_boundary_replay_check_mode_failed_rc_"
f"{result.returncode}"
)
),
"duration_ms": result.duration_ms,
"parent_op_id": claim.op_id,
"tags": [
"ansible",
"stdin_boundary_replay",
"verified_no_write_terminal",
f"automation_run_id:{automation_run_id}",
],
"failed_check_mode_op_id": failed_check_mode_op_id,
},
)
readback = await db.execute(
text("""
SELECT status, input, output
FROM automation_operation_log
WHERE op_id = CAST(:op_id AS uuid)
AND operation_type = 'ansible_execution_skipped'
LIMIT 1
"""),
{"op_id": terminal_op_id},
)
row = readback.mappings().one_or_none()
input_readback = _json_loads(row.get("input") if row else None)
output_readback = _json_loads(row.get("output") if row else None)
receipts = input_readback.get("runtime_stage_receipts")
return bool(
row
and row.get("status") == expected_status
and input_readback.get("automation_run_id")
== automation_run_id
and input_readback.get("retry_of_check_mode_op_id")
== failed_check_mode_op_id
and output_readback.get("runtime_apply_executed") is False
and output_readback.get("terminal_disposition")
== terminal_type
and isinstance(receipts, list)
and any(
isinstance(item, Mapping)
and item.get("stage_id") == "retry_or_rollback"
and item.get("durable_receipt") is True
for item in receipts
)
)
except Exception as exc:
logger.warning(
"ansible_stdin_boundary_replay_terminal_write_failed",
automation_run_id=automation_run_id,
failed_check_mode_op_id=failed_check_mode_op_id,
retry_check_mode_op_id=claim.op_id,
error_type=type(exc).__name__,
)
return False
def _claim_from_stdin_boundary_terminal_row(
row: dict[str, Any],
) -> AnsibleCheckModeClaim | None:
replay_input = _json_loads(row.get("replay_input"))
base = _claim_from_stale_check_mode_row(
{
"op_id": row.get("replay_op_id"),
"parent_op_id": row.get("source_candidate_op_id"),
"incident_id": row.get("incident_id"),
"input": replay_input,
}
)
if base is None:
return None
terminal_input = _json_loads(row.get("terminal_input"))
automation_run_id = str(
terminal_input.get("automation_run_id")
or replay_input.get("automation_run_id")
or base.source_candidate_op_id
)
return replace(
base,
input_payload={
**base.input_payload,
**replay_input,
"automation_run_id": automation_run_id,
"historical_stdin_boundary_replay": True,
"controlled_apply_allowed": False,
"apply_enabled": False,
},
)
async def _record_no_write_replay_timeline_receipt(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
terminal_op_id: str,
project_id: str,
) -> dict[str, Any] | None:
automation_run_id = _automation_run_id_for_claim(claim)
description = (
f"automation_run_id={automation_run_id};"
f"terminal_op_id={terminal_op_id};catalog_id={claim.catalog_id};"
f"returncode={result.returncode};runtime_apply_executed=false;"
"terminal=no_write_stdin_boundary_replay"
)
try:
async with get_db_context(project_id) as db:
selected = await db.execute(
text("""
WITH existing AS (
SELECT id
FROM timeline_events
WHERE incident_id = :incident_id
AND actor = 'ansible_stdin_boundary_replay_worker'
AND description = :description
ORDER BY created_at DESC
LIMIT 1
), inserted AS (
INSERT INTO timeline_events (
id, incident_id, event_type, status, title,
description, actor, actor_role, risk_level,
created_at
)
SELECT
gen_random_uuid()::text,
:incident_id,
'exec',
'error',
:title,
:description,
'ansible_stdin_boundary_replay_worker',
'ai_agent',
:risk_level,
NOW()
WHERE NOT EXISTS (SELECT 1 FROM existing)
RETURNING id
)
SELECT id FROM inserted
UNION ALL
SELECT id FROM existing
LIMIT 1
"""),
{
"incident_id": claim.incident_id,
"title": (
"AI no-write replay terminal: "
f"{claim.catalog_id}"
)[:500],
"description": description,
"risk_level": str(claim.risk_level or "")[:20] or None,
},
)
timeline_event_id = str(selected.scalar() or "")
except Exception as exc:
logger.warning(
"ansible_stdin_boundary_timeline_write_failed",
automation_run_id=automation_run_id,
terminal_op_id=terminal_op_id,
error_type=type(exc).__name__,
)
return None
if not timeline_event_id:
return None
return _runtime_stage_receipt(
claim,
stage_id="timeline_projection",
evidence_ref=f"timeline_events:{timeline_event_id}",
detail={
"timeline_event_id": timeline_event_id,
"terminal_op_id": terminal_op_id,
"returncode": result.returncode,
"runtime_apply_executed": False,
"projection_status": "repair_required",
"repository_readback_verified": True,
},
derived_from_durable_chain=True,
)
async def _record_no_write_replay_learning_receipts(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
terminal_op_id: str,
project_id: str,
) -> dict[str, Any]:
from src.models.knowledge import (
EntrySource,
EntryStatus,
EntryType,
KnowledgeEntryCreate,
)
from src.repositories.knowledge_repository import KnowledgeDBRepository
automation_run_id = _automation_run_id_for_claim(claim)
canonical_playbook_id = canonical_ansible_playbook_id(claim.catalog_id)
path_type = f"ansible_no_write_replay:{terminal_op_id[:8]}"
receipts = await _load_pre_decision_context_runtime_stage_receipts(
claim,
project_id=project_id,
derived_from_durable_chain=True,
)
failed_check_mode_op_id = str(
claim.input_payload.get("replay_of_check_mode_op_id") or ""
)
if failed_check_mode_op_id:
receipts.append(
_build_stdin_boundary_replay_stage_receipt(
claim,
result,
failed_check_mode_op_id=failed_check_mode_op_id,
terminal_op_id=terminal_op_id,
)
)
receipts.extend(
[
_runtime_stage_receipt(
claim,
stage_id="normalized_asset_identity",
evidence_ref=f"ansible_catalog:{claim.catalog_id}",
detail={
"asset_kind": "managed_host",
"canonical_asset_ids": list(claim.inventory_hosts),
"catalog_id": claim.catalog_id,
"playbook_path": claim.playbook_path,
},
derived_from_durable_chain=True,
),
_runtime_stage_receipt(
claim,
stage_id="source_truth_diff",
evidence_ref=(
f"automation_operation_log:{claim.op_id}:dry_run_result"
),
detail={
"check_mode_op_id": claim.op_id,
"check_mode_replay_performed": True,
"check_mode_replay_returncode": result.returncode,
"diff": True,
"runtime_apply_executed": False,
},
derived_from_durable_chain=True,
),
_runtime_stage_receipt(
claim,
stage_id="risk_policy_decision",
evidence_ref=f"ansible_policy:{claim.catalog_id}",
detail={
"risk_level": str(claim.risk_level or "").lower(),
"controlled_apply_allowed": (
str(claim.risk_level or "").lower()
in {"low", "medium", "high"}
),
"controlled_apply_executed": False,
"execution_scope": "historical_no_write_replay",
"apply_deferred_reason": (
"stdin_boundary_transport_or_playbook_repair_required"
),
"decision": (
"verified_no_write_terminal_pending_ai_repair"
),
"policy": "global_product_governance_v2",
},
derived_from_durable_chain=True,
),
_runtime_stage_receipt(
claim,
stage_id="executor_log_projection",
evidence_ref=(
f"automation_operation_log:{terminal_op_id}:output"
),
detail={
"terminal_op_id": terminal_op_id,
"returncode": result.returncode,
"duration_ms": result.duration_ms,
"runtime_apply_executed": False,
"projection_sanitized": True,
},
derived_from_durable_chain=True,
),
]
)
km_writeback: dict[str, Any] | None = None
try:
async with get_db_context(project_id) as db:
existing = await db.execute(
text("""
SELECT id
FROM knowledge_entries
WHERE related_incident_id = :incident_id
AND path_type = :path_type
LIMIT 1
"""),
{
"incident_id": claim.incident_id,
"path_type": path_type,
},
)
if existing.scalar() is None:
repo = KnowledgeDBRepository(db)
await repo.create(
KnowledgeEntryCreate(
title=f"AI 無寫入重試反模式：{claim.incident_id}",
content=(
"AI Agent 已完成歷史 Ansible stdin boundary "
"check-mode 重試，未執行 runtime apply。\n\n"
f"- Automation run: {automation_run_id}\n"
f"- Incident: {claim.incident_id}\n"
f"- Catalog: {claim.catalog_id}\n"
f"- PlayBook: {claim.playbook_path}\n"
f"- Terminal operation: {terminal_op_id}\n"
f"- Check-mode return code: {result.returncode}\n"
"- Runtime apply executed: false\n"
"- Next action: AI transport or PlayBook repair\n"
),
entry_type=EntryType.ANTI_PATTERN,
category="AI自動化/Ansible反模式",
tags=[
"ai_auto_repair",
"ansible_no_write_replay",
"stdin_boundary",
str(claim.catalog_id or ""),
f"automation_run_id:{automation_run_id}",
],
source=EntrySource.AI_EXTRACTED,
status=EntryStatus.PUBLISHED,
related_incident_id=claim.incident_id,
related_playbook_id=canonical_playbook_id,
path_type=path_type,
created_by="ai_agent_ansible_replay_worker",
)
)
readback = await db.execute(
text("""
SELECT id, related_playbook_id, path_type, status, updated_at
FROM knowledge_entries
WHERE related_incident_id = :incident_id
AND path_type = :path_type
LIMIT 1
"""),
{
"incident_id": claim.incident_id,
"path_type": path_type,
},
)
km_row = readback.mappings().first()
if (
km_row
and str(km_row.get("related_playbook_id") or "")
== canonical_playbook_id
and str(km_row.get("status") or "").lower() == "published"
and km_row.get("updated_at") is not None
):
km_writeback = {
"schema_version": "ansible_no_write_km_writeback_v1",
"knowledge_entry_id": str(km_row["id"]),
"canonical_playbook_id": canonical_playbook_id,
"path_type": path_type,
"status": str(km_row.get("status") or ""),
"repository_write_acknowledged": True,
"repository_readback_verified": True,
"durable_write_acknowledged": True,
"raw_log_payload_stored": False,
"secret_value_stored": False,
}
except Exception as exc:
logger.warning(
"ansible_stdin_boundary_km_writeback_failed",
automation_run_id=automation_run_id,
terminal_op_id=terminal_op_id,
error_type=type(exc).__name__,
)
if km_writeback is not None:
receipts.append(
_runtime_stage_receipt(
claim,
stage_id="km_playbook_writeback",
evidence_ref=(
"knowledge_entries:"
f"{km_writeback['knowledge_entry_id']}:row_version"
),
detail=km_writeback,
derived_from_durable_chain=True,
)
)
trust_writeback = await record_ansible_playbook_trust_writeback(
project_id=project_id,
automation_run_id=automation_run_id,
incident_id=claim.incident_id,
incident_db_id=_automation_operation_log_incident_id(
claim.incident_id
),
catalog_id=claim.catalog_id,
playbook_path=claim.playbook_path,
apply_op_id=terminal_op_id,
verification_result="not_applicable_no_runtime_apply",
trust_mutation_performed=False,
observation_kind="no_write_replay",
)
if trust_writeback is not None:
receipts.append(
_runtime_stage_receipt(
claim,
stage_id="playbook_trust",
evidence_ref=(
"playbooks:"
f"{trust_writeback['canonical_playbook_id']}:trust"
),
detail=trust_writeback,
derived_from_durable_chain=True,
)
)
rag_writeback = (
await ensure_ansible_rag_writeback(
project_id=project_id,
incident_id=claim.incident_id,
apply_op_id=terminal_op_id,
path_type=path_type,
)
if km_writeback is not None
else None
)
if rag_writeback is not None:
rag_evidence_ref = (
"knowledge_entries:"
f"{rag_writeback['knowledge_entry_id']}:embedding"
if rag_writeback.get("embedding_persisted") is True
else (
"rag_chunks:"
f"{rag_writeback.get('source_id', '')}:"
f"count={rag_writeback.get('rag_chunk_count', 0)}"
)
)
receipts.append(
_runtime_stage_receipt(
claim,
stage_id="rag_writeback",
evidence_ref=rag_evidence_ref,
detail=rag_writeback,
derived_from_durable_chain=True,
)
)
timeline_receipt = await _record_no_write_replay_timeline_receipt(
claim,
result,
terminal_op_id=terminal_op_id,
project_id=project_id,
)
if timeline_receipt is not None:
receipts.append(timeline_receipt)
by_stage = {
str(receipt.get("stage_id") or ""): receipt
for receipt in receipts
if isinstance(receipt, Mapping)
}
stage_ids = set(by_stage)
missing_stage_ids = sorted(
_NO_WRITE_REPLAY_REQUIRED_STAGE_IDS - stage_ids
)
return {
"ready": not missing_stage_ids,
"receipts": tuple(by_stage.values()),
"stage_ids": sorted(stage_ids),
"missing_stage_ids": missing_stage_ids,
"km_writeback": km_writeback is not None,
"rag_writeback": rag_writeback is not None,
"playbook_trust": trust_writeback is not None,
"mcp_context": "mcp_context" in stage_ids,
"runtime_apply_executed": False,
}
async def _load_missing_stdin_boundary_terminal_projection_rows(
*,
project_id: str,
window_hours: int,
limit: int,
) -> list[dict[str, Any]]:
async with get_db_context(project_id) as db:
await db.execute(text("SET LOCAL statement_timeout = '5000ms'"))
result = await db.execute(
text("""
SELECT
terminal.op_id::text AS terminal_op_id,
terminal.status AS terminal_status,
terminal.input AS terminal_input,
terminal.output AS terminal_output,
terminal.error AS terminal_error,
terminal.duration_ms AS terminal_duration_ms,
replay.op_id::text AS replay_op_id,
candidate.op_id::text AS source_candidate_op_id,
replay.input AS replay_input,
coalesce(
terminal.incident_id::text,
terminal.input ->> 'incident_id',
replay.incident_id::text,
replay.input ->> 'incident_id'
) AS incident_id,
CASE
WHEN lifecycle_projection.run_auto_repair_triggered
IS TRUE
AND (
lifecycle_projection.run_execution_started
IS NOT TRUE
OR lifecycle_projection.run_execution_completed
IS NOT TRUE
OR lifecycle_projection.run_telegram_result_acknowledged
IS NOT TRUE
)
THEN 0
ELSE 1
END AS lifecycle_open_priority
FROM automation_operation_log terminal
JOIN automation_operation_log replay
ON replay.op_id = terminal.parent_op_id
AND replay.operation_type = 'ansible_check_mode_executed'
AND replay.input ->> 'execution_mode'
= 'stdin_boundary_check_mode_replay'
AND replay.input ->> 'automation_run_id'
= terminal.input ->> 'automation_run_id'
JOIN automation_operation_log failed_check
ON failed_check.op_id = CAST(
NULLIF(
replay.input ->> 'replay_of_check_mode_op_id',
''
) AS uuid
)
AND failed_check.operation_type
= 'ansible_check_mode_executed'
AND failed_check.status = 'failed'
AND failed_check.parent_op_id = replay.parent_op_id
AND failed_check.input ->> 'automation_run_id'
= terminal.input ->> 'automation_run_id'
JOIN automation_operation_log candidate
ON candidate.op_id = replay.parent_op_id
AND candidate.operation_type = 'ansible_candidate_matched'
AND coalesce(
nullif(candidate.input ->> 'automation_run_id', ''),
candidate.op_id::text
) = terminal.input ->> 'automation_run_id'
JOIN incidents incident
ON incident.incident_id = coalesce(
terminal.incident_id::text,
terminal.input ->> 'incident_id',
replay.incident_id::text,
replay.input ->> 'incident_id'
)
AND incident.project_id = :project_id
JOIN LATERAL (
SELECT
coalesce(
bool_or(
lifecycle.event_type::text
= 'AUTO_REPAIR_TRIGGERED'
),
false
) AS run_auto_repair_triggered,
coalesce(
bool_or(
lifecycle.event_type::text
= 'EXECUTION_STARTED'
),
false
) AS run_execution_started,
coalesce(
bool_or(
lifecycle.event_type::text
= 'EXECUTION_COMPLETED'
),
false
) AS run_execution_completed,
coalesce(
bool_or(
lifecycle.event_type::text
= 'TELEGRAM_RESULT_SENT'
OR (
lifecycle.event_type::text
= 'NOTIFICATION_CLASSIFIED'
AND lifecycle.action_detail
= 'stdin_boundary_no_write_result_receipt_suppressed'
)
),
false
) AS run_telegram_result_acknowledged,
coalesce(
bool_or(
lifecycle.event_type::text
= 'AUTO_REPAIR_TRIGGERED'
AND lifecycle.context ->> 'apply_op_id'
= terminal.op_id::text
),
false
) AS terminal_auto_repair_triggered,
coalesce(
bool_or(
lifecycle.event_type::text
= 'EXECUTION_STARTED'
AND lifecycle.context ->> 'apply_op_id'
= terminal.op_id::text
),
false
) AS terminal_execution_started
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = incident.incident_id
AND lifecycle.context ->> 'automation_run_id'
= terminal.input ->> 'automation_run_id'
AND lifecycle.event_type::text IN (
'AUTO_REPAIR_TRIGGERED',
'EXECUTION_STARTED',
'EXECUTION_COMPLETED',
'TELEGRAM_RESULT_SENT',
'NOTIFICATION_CLASSIFIED'
)
) lifecycle_projection ON TRUE
WHERE terminal.operation_type = 'ansible_execution_skipped'
AND terminal.status IN ('success', 'failed')
AND terminal.input ->> 'execution_mode'
= 'stdin_boundary_check_mode_replay_terminal'
AND terminal.input ->> 'retry_of_check_mode_op_id'
= failed_check.op_id::text
AND terminal.output ->> 'runtime_apply_executed' = 'false'
AND terminal.created_at >= NOW() - (
:window_hours * INTERVAL '1 hour'
)
AND EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
terminal.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) receipt(value)
WHERE receipt.value ->> 'stage_id'
= 'retry_or_rollback'
AND receipt.value ->> 'automation_run_id'
= terminal.input ->> 'automation_run_id'
AND receipt.value #>>
'{detail,verified_no_write_terminal}' = 'true'
AND receipt.value #>>
'{detail,runtime_apply_executed}' = 'false'
AND receipt.value #>>
'{detail,repository_readback_verified}' = 'true'
AND receipt.value ->> 'durable_receipt' = 'true'
)
AND (
NOT EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
terminal.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) receipt(value)
WHERE receipt.value ->> 'stage_id'
= 'incident_closure'
AND receipt.value ->> 'automation_run_id'
= terminal.input ->> 'automation_run_id'
AND receipt.value #>>
'{detail,terminal_op_id}' = terminal.op_id::text
AND receipt.value #>>
'{detail,projection_readback_verified}' = 'true'
AND receipt.value ->> 'durable_receipt' = 'true'
)
OR lifecycle_projection.terminal_auto_repair_triggered
IS NOT TRUE
OR lifecycle_projection.terminal_execution_started
IS NOT TRUE
)
ORDER BY
lifecycle_open_priority ASC,
terminal.created_at ASC,
terminal.op_id ASC
LIMIT :limit
"""),
{
"project_id": project_id,
"window_hours": max(1, min(window_hours, 7 * 24)),
"limit": max(1, min(limit, 5)),
},
)
return [dict(row) for row in result.mappings().all()]
async def _stdin_boundary_replay_telegram_notification_disposition(
claim: AnsibleCheckModeClaim,
*,
terminal_op_id: str,
project_id: str,
) -> str:
automation_run_id = _automation_run_id_for_claim(claim)
try:
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT CASE
WHEN outbound.send_status = 'sent'
AND outbound.provider_message_id IS NOT NULL
THEN 'sent'
WHEN outbound.send_status = 'shadow'
AND outbound.provider_message_id IS NULL
AND outbound.source_envelope #>>
'{notification_policy,disposition}'
= 'suppressed'
THEN 'suppressed'
ELSE 'missing'
END AS disposition
FROM awooop_outbound_message outbound
WHERE outbound.project_id = :project_id
AND outbound.channel_type = 'telegram'
AND (
(
outbound.send_status = 'sent'
AND outbound.provider_message_id IS NOT NULL
)
OR (
outbound.send_status = 'shadow'
AND outbound.provider_message_id IS NULL
AND outbound.source_envelope #>>
'{notification_policy,disposition}'
= 'suppressed'
)
)
AND outbound.source_envelope #>>
'{callback_reply,action}'
= 'controlled_apply_result'
AND coalesce(
outbound.source_envelope #>>
'{callback_reply,execution_kind}',
outbound.source_envelope ->> 'execution_kind'
) = 'no_write_replay'
AND coalesce(
outbound.source_envelope
->> 'automation_run_id',
outbound.source_envelope #>>
'{callback_reply,automation_run_id}',
outbound.source_envelope #>>
'{source_refs,automation_run_ids,0}'
) = :automation_run_id
AND coalesce(
outbound.source_envelope #>>
'{callback_reply,incident_id}',
outbound.source_envelope #>>
'{source_refs,incident_ids,0}'
) = :incident_id
AND outbound.source_envelope #>>
'{callback_reply,apply_op_id}'
= :terminal_op_id
ORDER BY (outbound.send_status = 'sent') DESC
LIMIT 1
"""),
{
"project_id": project_id,
"automation_run_id": automation_run_id,
"incident_id": claim.incident_id,
"terminal_op_id": terminal_op_id,
},
)
return str(result.scalar_one_or_none() or "missing")
except Exception as exc:
logger.warning(
"ansible_stdin_boundary_telegram_readback_failed",
automation_run_id=automation_run_id,
terminal_op_id=terminal_op_id,
error_type=type(exc).__name__,
)
return "missing"
async def _stdin_boundary_replay_telegram_receipt_acknowledged(
claim: AnsibleCheckModeClaim,
*,
terminal_op_id: str,
project_id: str,
) -> bool:
disposition = (
await _stdin_boundary_replay_telegram_notification_disposition(
claim,
terminal_op_id=terminal_op_id,
project_id=project_id,
)
)
return disposition in {"sent", "suppressed"}
async def _record_stdin_boundary_incident_terminal_disposition(
claim: AnsibleCheckModeClaim,
*,
terminal_op_id: str,
learning_stage_ids: list[str],
project_id: str,
) -> dict[str, Any] | None:
automation_run_id = _automation_run_id_for_claim(claim)
terminal_payload = {
**_incident_terminal_disposition_payload(
claim,
apply_op_id=terminal_op_id,
terminal_type="no_write_stdin_boundary_replay_terminal",
success_terminal=False,
retry_op_id=claim.op_id,
telegram_receipt_acknowledged=True,
),
"schema_version": (
"ansible_stdin_boundary_incident_terminal_v1"
),
"terminal_op_id": terminal_op_id,
"proposal_executed": False,
"execution_success": None,
"learning_stage_ids": learning_stage_ids,
"learning_writeback_acknowledged": True,
"repository_readback_verified": True,
"runtime_apply_executed": False,
}
try:
async with get_db_context(project_id) as db:
selected = await db.execute(
text("""
SELECT status::text AS incident_status, outcome, updated_at
FROM incidents
WHERE incident_id = :incident_id
AND project_id = :project_id
LIMIT 1
FOR UPDATE
"""),
{
"incident_id": claim.incident_id,
"project_id": project_id,
},
)
current = selected.mappings().one_or_none()
if current is None:
return None
current_status = str(current.get("incident_status") or "")
projection_superseded = current_status.upper() in {
"RESOLVED",
"CLOSED",
}
incident_write_performed = False
readback = current
if not projection_superseded:
updated = await db.execute(
text("""
UPDATE incidents
SET status = 'MITIGATING'::incidentstatus,
outcome = CAST(
coalesce(
CASE
WHEN outcome IS NULL THEN '{}'::jsonb
WHEN jsonb_typeof(CAST(outcome AS jsonb))
= 'object'
THEN CAST(outcome AS jsonb)
ELSE jsonb_build_object(
'legacy_outcome',
CAST(outcome AS jsonb)
)
END,
'{}'::jsonb
) || jsonb_build_object(
'automation_terminal',
CAST(:terminal_payload AS jsonb),
'proposal_executed',
false,
'execution_success',
NULL
) AS json
),
updated_at = NOW()
WHERE incident_id = :incident_id
AND project_id = :project_id
AND upper(status::text)
NOT IN ('RESOLVED', 'CLOSED')
RETURNING
status::text AS incident_status,
outcome,
updated_at
"""),
{
"terminal_payload": json.dumps(
terminal_payload,
ensure_ascii=False,
),
"incident_id": claim.incident_id,
"project_id": project_id,
},
)
readback = updated.mappings().one_or_none()
incident_write_performed = readback is not None
if readback is None:
return None
outcome = _json_loads(readback.get("outcome"))
terminal_readback = outcome.get("automation_terminal")
if not (
isinstance(terminal_readback, Mapping)
and terminal_readback.get("automation_run_id")
== automation_run_id
and terminal_readback.get("terminal_op_id")
== terminal_op_id
and terminal_readback.get("runtime_apply_executed")
is False
):
return None
return {
**terminal_payload,
"incident_status": str(
readback.get("incident_status") or current_status
),
"incident_projection_write_performed": (
incident_write_performed
),
"incident_projection_superseded": projection_superseded,
"incident_projection_readback_verified": True,
"incident_row_version": (
readback["updated_at"].isoformat()
if readback.get("updated_at") is not None
else None
),
}
except Exception as exc:
logger.warning(
"ansible_stdin_boundary_incident_projection_failed",
automation_run_id=automation_run_id,
terminal_op_id=terminal_op_id,
error_type=type(exc).__name__,
)
return None
async def _verify_stdin_boundary_terminal_projection_readback(
claim: AnsibleCheckModeClaim,
*,
terminal_op_id: str,
project_id: str,
require_projection_flag: bool,
) -> bool:
automation_run_id = _automation_run_id_for_claim(claim)
try:
async with get_db_context(project_id) as db:
await db.execute(text("SET LOCAL statement_timeout = '5000ms'"))
result = await db.execute(
text("""
SELECT EXISTS (
SELECT 1
FROM automation_operation_log terminal
JOIN automation_operation_log replay
ON replay.op_id = terminal.parent_op_id
AND replay.operation_type
= 'ansible_check_mode_executed'
AND replay.input ->> 'execution_mode'
= 'stdin_boundary_check_mode_replay'
JOIN automation_operation_log failed_check
ON failed_check.op_id = CAST(
NULLIF(
replay.input
->> 'replay_of_check_mode_op_id',
''
) AS uuid
)
AND failed_check.operation_type
= 'ansible_check_mode_executed'
AND failed_check.status = 'failed'
AND failed_check.parent_op_id = replay.parent_op_id
AND failed_check.input ->> 'automation_run_id'
= :automation_run_id
JOIN automation_operation_log candidate
ON candidate.op_id = replay.parent_op_id
AND candidate.operation_type
= 'ansible_candidate_matched'
AND coalesce(
nullif(
candidate.input
->> 'automation_run_id',
''
),
candidate.op_id::text
) = :automation_run_id
WHERE terminal.op_id
= CAST(:terminal_op_id AS uuid)
AND terminal.operation_type
= 'ansible_execution_skipped'
AND terminal.status IN ('success', 'failed')
AND terminal.input ->> 'execution_mode'
= 'stdin_boundary_check_mode_replay_terminal'
AND terminal.input ->> 'automation_run_id'
= :automation_run_id
AND terminal.input
->> 'retry_of_check_mode_op_id'
= failed_check.op_id::text
AND replay.input ->> 'automation_run_id'
= :automation_run_id
AND coalesce(
terminal.incident_id::text,
terminal.input ->> 'incident_id'
) = :incident_id
AND terminal.output ->> 'runtime_apply_executed'
= 'false'
AND (
SELECT COUNT(DISTINCT receipt.value
->> 'stage_id')
FROM jsonb_array_elements(
coalesce(
terminal.input
-> 'runtime_stage_receipts',
'[]'::jsonb
)
) receipt(value)
WHERE receipt.value ->> 'stage_id' IN (
'mcp_context',
'service_log_evidence',
'normalized_asset_identity',
'source_truth_diff',
'risk_policy_decision',
'executor_log_projection',
'retry_or_rollback',
'km_playbook_writeback',
'rag_writeback',
'playbook_trust',
'timeline_projection'
)
AND receipt.value ->> 'automation_run_id'
= :automation_run_id
AND receipt.value ->> 'durable_receipt'
= 'true'
) = 11
AND EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
terminal.input
-> 'runtime_stage_receipts',
'[]'::jsonb
)
) retry_receipt(value)
WHERE retry_receipt.value ->> 'stage_id'
= 'retry_or_rollback'
AND retry_receipt.value #>>
'{detail,verified_no_write_terminal}'
= 'true'
AND retry_receipt.value #>>
'{detail,runtime_apply_executed}'
= 'false'
AND retry_receipt.value #>>
'{detail,repository_readback_verified}'
= 'true'
AND retry_receipt.value
->> 'durable_receipt' = 'true'
)
AND EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
terminal.input
-> 'runtime_stage_receipts',
'[]'::jsonb
)
) closure_receipt(value)
WHERE closure_receipt.value ->> 'stage_id'
= 'incident_closure'
AND closure_receipt.value
->> 'automation_run_id'
= :automation_run_id
AND closure_receipt.value #>>
'{detail,terminal_op_id}'
= CAST(:terminal_op_id AS text)
AND closure_receipt.value #>>
'{detail,no_write_terminal}' = 'true'
AND closure_receipt.value #>>
'{detail,telegram_receipt_acknowledged}'
= 'true'
AND closure_receipt.value #>>
'{detail,learning_writeback_acknowledged}'
= 'true'
AND closure_receipt.value #>>
'{detail,incident_projection_readback_verified}'
= 'true'
AND closure_receipt.value #>>
'{detail,repository_readback_verified}'
= 'true'
AND (
:require_projection_flag IS FALSE
OR closure_receipt.value #>>
'{detail,projection_readback_verified}'
= 'true'
)
AND closure_receipt.value
->> 'durable_receipt' = 'true'
)
AND EXISTS (
SELECT 1
FROM awooop_outbound_message outbound
WHERE outbound.project_id = :project_id
AND outbound.channel_type = 'telegram'
AND (
(
outbound.send_status = 'sent'
AND outbound.provider_message_id
IS NOT NULL
)
OR (
outbound.send_status = 'shadow'
AND outbound.provider_message_id
IS NULL
AND outbound.source_envelope #>>
'{notification_policy,disposition}'
= 'suppressed'
)
)
AND outbound.source_envelope #>>
'{callback_reply,action}'
= 'controlled_apply_result'
AND outbound.source_envelope #>>
'{callback_reply,execution_kind}'
= 'no_write_replay'
AND outbound.source_envelope #>>
'{callback_reply,automation_run_id}'
= :automation_run_id
AND outbound.source_envelope #>>
'{callback_reply,incident_id}'
= :incident_id
AND outbound.source_envelope #>>
'{callback_reply,apply_op_id}'
= CAST(:terminal_op_id AS text)
)
AND EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = :incident_id
AND lifecycle.event_type::text
= 'AUTO_REPAIR_TRIGGERED'
AND lifecycle.context
->> 'automation_run_id'
= :automation_run_id
AND lifecycle.context ->> 'apply_op_id'
= CAST(:terminal_op_id AS text)
)
AND EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = :incident_id
AND lifecycle.event_type::text
= 'EXECUTION_STARTED'
AND lifecycle.context
->> 'automation_run_id'
= :automation_run_id
AND lifecycle.context ->> 'apply_op_id'
= CAST(:terminal_op_id AS text)
)
AND EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = :incident_id
AND lifecycle.event_type::text
= 'EXECUTION_COMPLETED'
AND lifecycle.success IS FALSE
AND lifecycle.context
->> 'automation_run_id'
= :automation_run_id
AND lifecycle.context ->> 'apply_op_id'
= CAST(:terminal_op_id AS text)
)
AND EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = :incident_id
AND (
lifecycle.event_type::text
= 'TELEGRAM_RESULT_SENT'
OR (
lifecycle.event_type::text
= 'NOTIFICATION_CLASSIFIED'
AND lifecycle.action_detail
= 'stdin_boundary_no_write_result_receipt_suppressed'
)
)
AND lifecycle.context
->> 'automation_run_id'
= :automation_run_id
AND lifecycle.context ->> 'apply_op_id'
= CAST(:terminal_op_id AS text)
)
)
"""),
{
"terminal_op_id": terminal_op_id,
"automation_run_id": automation_run_id,
"incident_id": claim.incident_id,
"project_id": project_id,
"require_projection_flag": require_projection_flag,
},
)
return result.scalar() is True
except Exception as exc:
logger.warning(
"ansible_stdin_boundary_projection_readback_failed",
automation_run_id=automation_run_id,
terminal_op_id=terminal_op_id,
error_type=type(exc).__name__,
)
return False
async def backfill_missing_stdin_boundary_terminal_projections_once(
*,
project_id: str = "awoooi",
window_hours: int = _STDIN_BOUNDARY_REPLAY_WINDOW_HOURS,
limit: int = 1,
) -> dict[str, Any]:
"""Close no-write replay attempts without re-executing their apply."""
stats: dict[str, Any] = {
"scanned": 0,
"written": 0,
"learning_written": 0,
"telegram_receipt_acknowledged": 0,
"telegram_notification_suppressed": 0,
"incident_receipt_written": 0,
"lifecycle_written": 0,
"verified": 0,
"runtime_apply_executed": False,
"blockers": [],
"error": None,
}
try:
rows = await _load_missing_stdin_boundary_terminal_projection_rows(
project_id=project_id,
window_hours=window_hours,
limit=limit,
)
stats["scanned"] = len(rows)
for row in rows:
claim = _claim_from_stdin_boundary_terminal_row(row)
terminal_op_id = str(row.get("terminal_op_id") or "")
terminal_output = _json_loads(row.get("terminal_output"))
if claim is None or not terminal_op_id:
stats["blockers"].append("terminal_claim_reconstruction_failed")
continue
returncode = _int_from_value(
terminal_output.get("check_mode_replay_returncode"),
default=(
0
if str(row.get("terminal_status") or "") == "success"
else 1
),
)
replay_result = AnsibleRunResult(
returncode=returncode,
stdout="",
stderr=str(row.get("terminal_error") or ""),
duration_ms=_int_from_value(
row.get("terminal_duration_ms"),
default=0,
),
post_verifier_passed=False,
)
learning = await _record_no_write_replay_learning_receipts(
claim,
replay_result,
terminal_op_id=terminal_op_id,
project_id=project_id,
)
if learning.get("ready") is not True:
missing = ",".join(learning.get("missing_stage_ids") or [])
stats["blockers"].append(
f"learning_receipts_missing:{missing}"[:300]
)
continue
learning_written = (
await _append_runtime_stage_receipts_to_operation(
operation_id=terminal_op_id,
operation_type="ansible_execution_skipped",
receipts=tuple(learning.get("receipts") or ()),
project_id=project_id,
)
)
stats["learning_written"] += int(learning_written)
if not learning_written:
stats["blockers"].append("learning_receipt_append_failed")
continue
telegram_disposition = (
await _stdin_boundary_replay_telegram_notification_disposition(
claim,
terminal_op_id=terminal_op_id,
project_id=project_id,
)
)
if telegram_disposition == "missing":
receipt_sent = await _send_controlled_apply_telegram_receipt(
claim,
replay_result,
apply_op_id=terminal_op_id,
writeback={
"verification_result": "failed",
"verification": True,
"learning": True,
},
project_id=project_id,
execution_kind="no_write_replay",
provider_delivery="shadow_only",
)
if receipt_sent:
telegram_disposition = (
await _stdin_boundary_replay_telegram_notification_disposition(
claim,
terminal_op_id=terminal_op_id,
project_id=project_id,
)
)
if telegram_disposition not in {"sent", "suppressed"}:
stats["blockers"].append("telegram_receipt_pending")
continue
stats["telegram_receipt_acknowledged"] += 1
stats["telegram_notification_suppressed"] += int(
telegram_disposition == "suppressed"
)
trigger_lifecycle = await _append_alert_lifecycle_receipt(
claim,
"AUTO_REPAIR_TRIGGERED",
apply_op_id=terminal_op_id,
success=True,
action_detail=(
"stdin_boundary_no_write_replay_triggered"
),
project_id=project_id,
post_verifier_passed=None,
)
started_lifecycle = await _append_alert_lifecycle_receipt(
claim,
"EXECUTION_STARTED",
apply_op_id=terminal_op_id,
success=True,
action_detail=(
"stdin_boundary_no_write_replay_started"
),
project_id=project_id,
post_verifier_passed=None,
)
execution_lifecycle = await _append_alert_lifecycle_receipt(
claim,
"EXECUTION_COMPLETED",
apply_op_id=terminal_op_id,
success=False,
action_detail=(
"stdin_boundary_replay_verified_no_write_terminal"
),
project_id=project_id,
error_message="ai_transport_or_playbook_repair_required",
post_verifier_passed=False,
)
telegram_lifecycle = await _append_alert_lifecycle_receipt(
claim,
(
"NOTIFICATION_CLASSIFIED"
if telegram_disposition == "suppressed"
else "TELEGRAM_RESULT_SENT"
),
apply_op_id=terminal_op_id,
success=False,
action_detail=(
"stdin_boundary_no_write_result_receipt_suppressed"
if telegram_disposition == "suppressed"
else "stdin_boundary_no_write_result_receipt_sent"
),
project_id=project_id,
post_verifier_passed=False,
)
if not all((
trigger_lifecycle,
started_lifecycle,
execution_lifecycle,
telegram_lifecycle,
)):
stats["blockers"].append("lifecycle_projection_pending")
continue
stats["lifecycle_written"] += 1
incident_detail = (
await _record_stdin_boundary_incident_terminal_disposition(
claim,
terminal_op_id=terminal_op_id,
learning_stage_ids=list(learning["stage_ids"]),
project_id=project_id,
)
)
if incident_detail is None:
stats["blockers"].append("incident_projection_pending")
continue
closure_detail = {
**incident_detail,
"terminal_op_id": terminal_op_id,
"retry_check_mode_op_id": claim.op_id,
"check_mode_replay_returncode": replay_result.returncode,
"telegram_notification_disposition": telegram_disposition,
"runtime_apply_executed": False,
"verified_no_write_terminal": True,
"telegram_receipt_acknowledged": True,
"learning_writeback_acknowledged": True,
"required_learning_stage_ids": sorted(
_NO_WRITE_REPLAY_REQUIRED_STAGE_IDS
),
"projection_readback_verified": False,
"repository_readback_verified": True,
"safe_next_action": (
"queue_ai_transport_or_playbook_repair_candidate"
),
}
closure_receipt = _runtime_stage_receipt(
claim,
stage_id="incident_closure",
evidence_ref=(
f"automation_operation_log:{terminal_op_id}:"
"no_write_terminal"
),
detail=closure_detail,
derived_from_durable_chain=True,
)
incident_receipt_written = (
await _append_runtime_stage_receipts_to_operation(
operation_id=terminal_op_id,
operation_type="ansible_execution_skipped",
receipts=(closure_receipt,),
project_id=project_id,
)
)
stats["incident_receipt_written"] += int(
incident_receipt_written
)
if not incident_receipt_written:
stats["blockers"].append("incident_receipt_append_failed")
continue
verified = await _verify_stdin_boundary_terminal_projection_readback(
claim,
terminal_op_id=terminal_op_id,
project_id=project_id,
require_projection_flag=False,
)
if not verified:
stats["blockers"].append("projection_readback_pending")
continue
final_receipt = _runtime_stage_receipt(
claim,
stage_id="incident_closure",
evidence_ref=(
f"automation_operation_log:{terminal_op_id}:"
"no_write_terminal"
),
detail={
**closure_detail,
"projection_readback_verified": True,
},
derived_from_durable_chain=True,
)
finalized = await _append_runtime_stage_receipts_to_operation(
operation_id=terminal_op_id,
operation_type="ansible_execution_skipped",
receipts=(final_receipt,),
project_id=project_id,
)
if not finalized:
stats["blockers"].append("projection_finalization_pending")
continue
final_verified = (
await _verify_stdin_boundary_terminal_projection_readback(
claim,
terminal_op_id=terminal_op_id,
project_id=project_id,
require_projection_flag=True,
)
)
stats["verified"] += int(final_verified)
stats["written"] += int(final_verified)
if not final_verified:
stats["blockers"].append("final_projection_readback_pending")
except Exception as exc:
stats["error"] = f"{type(exc).__name__}: {exc}"[:500]
logger.warning(
"ansible_stdin_boundary_projection_backfill_failed",
project_id=project_id,
**stats,
)
stats["blockers"] = sorted(set(stats["blockers"]))
return stats
async def _record_learning_writeback_receipt(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
verification_result: str,
action_label: str,
project_id: str,
) -> dict[str, Any] | None:
"""Persist canonical PlayBook trust and its idempotent learning receipt."""
del result, action_label
return await record_ansible_playbook_trust_writeback(
project_id=project_id,
automation_run_id=_automation_run_id_for_claim(claim),
incident_id=claim.incident_id,
incident_db_id=_automation_operation_log_incident_id(claim.incident_id),
catalog_id=claim.catalog_id,
playbook_path=claim.apply_playbook_path,
apply_op_id=apply_op_id,
verification_result=verification_result,
)
async def _record_apply_post_verifier_terminal(
*,
apply_op_id: str,
verification_passed: bool,
verification_result: str,
verifier_receipt: Mapping[str, Any],
project_id: str,
) -> bool:
public_receipt = {
"schema_version": verifier_receipt.get("schema_version"),
"verifier": verifier_receipt.get("verifier"),
"independent_source": verifier_receipt.get("independent_source"),
"verifier_source_sha": verifier_receipt.get("verifier_source_sha"),
"verification_result": verification_result,
"all_postconditions_passed": (
verifier_receipt.get("all_postconditions_passed") is True
),
"required_postcondition_count": int(
verifier_receipt.get("required_postcondition_count") or 0
),
"passed_postcondition_count": int(
verifier_receipt.get("passed_postcondition_count") or 0
),
"executor_returncode_trusted": False,
"raw_output_stored": False,
}
try:
async with get_db_context(project_id) as db:
updated = await db.execute(
text("""
UPDATE automation_operation_log
SET status = CASE
WHEN :verification_passed THEN status
ELSE 'failed'
END,
output = coalesce(output, '{}'::jsonb) || CAST(
:verifier_terminal AS jsonb
),
error = CASE
WHEN :verification_passed THEN error
ELSE coalesce(error, 'independent_post_verifier_failed')
END
WHERE op_id = CAST(:apply_op_id AS uuid)
AND operation_type = 'ansible_apply_executed'
"""),
{
"verification_passed": verification_passed,
"verifier_terminal": json.dumps({
"independent_post_verifier_passed": verification_passed,
"independent_post_verifier": public_receipt,
}, ensure_ascii=False),
"apply_op_id": apply_op_id,
},
)
return bool(updated.rowcount)
except Exception as exc:
logger.warning(
"ansible_apply_post_verifier_terminal_write_failed",
apply_op_id=apply_op_id,
verification_result=verification_result,
error_type=type(exc).__name__,
)
return False
async def _record_post_apply_verifier_and_learning(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
project_id: str,
) -> dict[str, Any]:
"""Persist post-apply verifier evidence and KM learning for Ansible apply."""
action_label = _post_apply_action_label(claim, apply_op_id=apply_op_id)
automation_run_id = _automation_run_id_for_claim(claim)
try:
verifier_receipt = await run_ansible_asset_post_verifier(
catalog_id=claim.catalog_id,
automation_run_id=automation_run_id,
apply_op_id=apply_op_id,
inventory_hosts=claim.inventory_hosts,
executor_returncode=result.returncode,
)
except Exception as exc:
logger.warning(
"ansible_independent_post_verifier_failed",
incident_id=claim.incident_id,
catalog_id=claim.catalog_id,
apply_op_id=apply_op_id,
error_type=type(exc).__name__,
)
verifier_receipt = {
"schema_version": "awoooi_ansible_asset_post_verifier_v1",
"verifier": "asset_specific_read_only_host_postconditions",
"independent_source": "broker_ssh_host_runtime_readback",
"automation_run_id": automation_run_id,
"apply_op_id": apply_op_id,
"catalog_id": claim.catalog_id,
"verification_result": "failed",
"all_postconditions_passed": False,
"required_postcondition_count": 0,
"passed_postcondition_count": 0,
"postconditions": [],
"active_blockers": [
f"independent_post_verifier_error:{type(exc).__name__}"
],
"executor_returncode": result.returncode,
"executor_returncode_trusted": False,
"raw_output_stored": False,
"writes_on_verify": False,
}
verification_result = _post_apply_verification_result(
result,
verifier_receipt,
)
post_state = {
**verifier_receipt,
"playbook_path": claim.apply_playbook_path,
"timed_out": result.timed_out,
"action_taken": action_label,
"next_required_step": (
"km_playbook_trust_writeback"
if verification_result == "success"
else "automatic_retry_or_rollback"
),
}
evidence_summary = (
"AI Agent Ansible controlled apply verifier: "
f"incident={claim.incident_id}; catalog={claim.catalog_id}; "
f"result={verification_result}; returncode={result.returncode}; apply_op={apply_op_id}"
)
canonical_playbook_id = canonical_ansible_playbook_id(claim.catalog_id)
status: dict[str, Any] = {
"verification": False,
"verification_passed": False,
"verification_result": verification_result,
"independent_verifier": verifier_receipt,
"learning": False,
"km_writeback": None,
"trust_learning": False,
"rag_writeback": False,
"runtime_stage_receipts": [],
}
try:
async with get_db_context(project_id) as db:
inserted = await db.execute(
text("""
INSERT INTO incident_evidence (
id,
incident_id,
matched_playbook_id,
schema_version,
mcp_health,
collection_duration_ms,
sensors_attempted,
sensors_succeeded,
evidence_summary,
post_execution_state,
verification_result
)
SELECT
gen_random_uuid()::text,
CAST(:incident_id AS varchar(30)),
CAST(:matched_playbook_id AS varchar(36)),
'v1',
CAST(:mcp_health AS jsonb),
:collection_duration_ms,
:sensors_attempted,
:sensors_succeeded,
:evidence_summary,
CAST(:post_execution_state AS jsonb),
CAST(:verification_result AS varchar(20))
WHERE NOT EXISTS (
SELECT 1
FROM incident_evidence existing
WHERE existing.incident_id = CAST(:incident_id AS varchar(30))
AND existing.post_execution_state ->> 'apply_op_id' = CAST(:apply_op_id AS text)
)
RETURNING id
"""),
{
"incident_id": claim.incident_id,
"matched_playbook_id": canonical_playbook_id,
"mcp_health": json.dumps({
"asset_specific_read_only_host_postconditions": (
verification_result == "success"
)
}, ensure_ascii=False),
"collection_duration_ms": sum(
max(0, int(item.get("duration_ms") or 0))
for item in verifier_receipt.get("postconditions") or []
if isinstance(item, Mapping)
),
"sensors_attempted": int(
verifier_receipt.get("required_postcondition_count") or 0
),
"sensors_succeeded": int(
verifier_receipt.get("passed_postcondition_count") or 0
),
"evidence_summary": evidence_summary,
"post_execution_state": json.dumps(post_state, ensure_ascii=False),
"verification_result": verification_result,
"apply_op_id": apply_op_id,
},
)
verifier_evidence_id = inserted.scalar()
if verifier_evidence_id is None:
existing = await db.execute(
text("""
SELECT id
FROM incident_evidence
WHERE incident_id = CAST(:incident_id AS varchar(30))
AND post_execution_state ->> 'apply_op_id' = CAST(
:apply_op_id AS text
)
AND post_execution_state ->> 'schema_version'
= 'awoooi_ansible_asset_post_verifier_v1'
AND verification_result = :verification_result
LIMIT 1
"""),
{
"incident_id": claim.incident_id,
"apply_op_id": apply_op_id,
"verification_result": verification_result,
},
)
verifier_evidence_id = existing.scalar()
status["verification"] = verifier_evidence_id is not None
except Exception as exc:
logger.warning(
"ansible_post_apply_verifier_receipt_failed",
incident_id=claim.incident_id,
catalog_id=claim.catalog_id,
apply_op_id=apply_op_id,
error=str(exc),
)
verification_passed = bool(
verification_result == "success"
and status["verification"] is True
)
terminal_written = await _record_apply_post_verifier_terminal(
apply_op_id=apply_op_id,
verification_passed=verification_passed,
verification_result=verification_result,
verifier_receipt=verifier_receipt,
project_id=project_id,
)
verification_passed = bool(verification_passed and terminal_written)
if not verification_passed:
verification_result = (
"timeout" if verification_result == "timeout" else "failed"
)
status["verification_passed"] = verification_passed
status["verification_result"] = verification_result
status["apply_terminal_written"] = terminal_written
try:
from src.models.knowledge import (
EntrySource,
EntryStatus,
EntryType,
KnowledgeEntryCreate,
)
from src.repositories.knowledge_repository import KnowledgeDBRepository
async with get_db_context(project_id) as db:
path_type = _post_apply_km_path_type(apply_op_id)
existing = await db.execute(
text("""
SELECT id
FROM knowledge_entries
WHERE related_incident_id = :incident_id
AND path_type = :path_type
LIMIT 1
"""),
{
"incident_id": claim.incident_id,
"path_type": path_type,
},
)
if existing.scalar() is None:
repo = KnowledgeDBRepository(db)
await repo.create(
KnowledgeEntryCreate(
title=f"AI 自動修復沉澱：{claim.incident_id}",
content=(
"AI Agent 已執行 Ansible controlled apply 並寫入驗證摘要。\n\n"
"- Automation run: "
f"{claim.input_payload.get('automation_run_id') or claim.source_candidate_op_id}\n"
f"- Incident: {claim.incident_id}\n"
f"- Catalog: {claim.catalog_id}\n"
f"- PlayBook: {claim.apply_playbook_path}\n"
f"- Apply operation: {apply_op_id}\n"
f"- Verification result: {verification_result}\n"
f"- Return code: {result.returncode}\n"
f"- Next step: {post_state['next_required_step']}\n"
),
entry_type=EntryType.INCIDENT_CASE,
category="AI自動化/Ansible受控修復",
tags=[
"ai_auto_repair",
"ansible_controlled_apply",
verification_result,
str(claim.catalog_id or ""),
(
"automation_run_id:"
f"{claim.input_payload.get('automation_run_id') or claim.source_candidate_op_id}"
),
],
source=EntrySource.AI_EXTRACTED,
status=EntryStatus.REVIEW,
related_incident_id=claim.incident_id,
related_playbook_id=canonical_playbook_id,
path_type=path_type,
created_by="ai_agent_ansible_worker",
)
)
verified_km = await db.execute(
text("""
SELECT
id,
related_playbook_id,
path_type,
status,
updated_at
FROM knowledge_entries
WHERE related_incident_id = :incident_id
AND path_type = :path_type
LIMIT 1
"""),
{
"incident_id": claim.incident_id,
"path_type": path_type,
},
)
km_row = verified_km.mappings().first()
if (
km_row
and str(km_row.get("id") or "")
and str(km_row.get("related_playbook_id") or "")
== canonical_playbook_id
and str(km_row.get("path_type") or "") == path_type
and km_row.get("updated_at") is not None
):
status["learning"] = True
status["km_writeback"] = {
"schema_version": "ansible_km_writeback_v1",
"knowledge_entry_id": str(km_row["id"]),
"canonical_playbook_id": canonical_playbook_id,
"catalog_id": claim.catalog_id,
"path_type": path_type,
"status": str(km_row.get("status") or ""),
"km_row_version": km_row["updated_at"].isoformat(),
"repository_write_acknowledged": True,
"repository_readback_verified": True,
"durable_write_acknowledged": True,
"writer_source_sha": os.getenv(
"AWOOOI_BUILD_COMMIT_SHA",
"",
).strip().lower()
or None,
"raw_log_payload_stored": False,
"secret_value_stored": False,
}
except Exception as exc:
logger.warning(
"ansible_post_apply_learning_writeback_failed",
incident_id=claim.incident_id,
catalog_id=claim.catalog_id,
apply_op_id=apply_op_id,
error=str(exc),
)
km_writeback = status.get("km_writeback")
if isinstance(km_writeback, Mapping):
status["runtime_stage_receipts"].append(
_runtime_stage_receipt(
claim,
stage_id="km_playbook_writeback",
evidence_ref=(
"knowledge_entries:"
f"{km_writeback['knowledge_entry_id']}:row_version"
),
detail=dict(km_writeback),
)
)
trust_writeback = await _record_learning_writeback_receipt(
claim,
result,
apply_op_id=apply_op_id,
verification_result=verification_result,
action_label=action_label,
project_id=project_id,
)
status["trust_learning"] = trust_writeback is not None
if trust_writeback is not None:
status["runtime_stage_receipts"].append(
_runtime_stage_receipt(
claim,
stage_id="playbook_trust",
evidence_ref=(
"playbooks:"
f"{trust_writeback['canonical_playbook_id']}:trust"
),
detail=trust_writeback,
)
)
rag_writeback = (
await ensure_ansible_rag_writeback(
project_id=project_id,
incident_id=claim.incident_id,
apply_op_id=apply_op_id,
)
if status["learning"]
else None
)
status["rag_writeback"] = rag_writeback is not None
if rag_writeback is not None:
if rag_writeback.get("embedding_persisted") is True:
rag_evidence_ref = (
"knowledge_entries:"
f"{rag_writeback['knowledge_entry_id']}:embedding"
)
else:
rag_evidence_ref = (
"rag_chunks:"
f"{rag_writeback.get('source_id', '')}:"
f"count={rag_writeback.get('rag_chunk_count', 0)}"
)
status["runtime_stage_receipts"].append(
_runtime_stage_receipt(
claim,
stage_id="rag_writeback",
evidence_ref=rag_evidence_ref,
detail=rag_writeback,
)
)
return status
async def _send_controlled_apply_telegram_receipt(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
writeback: dict[str, Any],
project_id: str,
execution_kind: str = "controlled_apply",
provider_delivery: str = "digest",
) -> bool:
try:
from src.services.telegram_gateway import get_telegram_gateway
effective_provider_delivery = str(
claim.input_payload.get("telegram_provider_delivery")
or provider_delivery
)
response = await get_telegram_gateway().send_controlled_apply_result_receipt(
automation_run_id=str(
claim.input_payload.get("automation_run_id")
or claim.source_candidate_op_id
),
incident_id=claim.incident_id,
catalog_id=claim.catalog_id,
apply_op_id=apply_op_id,
playbook_path=claim.apply_playbook_path,
verification_result=str(
writeback.get("verification_result") or "failed"
),
returncode=result.returncode,
duration_ms=result.duration_ms,
verifier_written=bool(writeback.get("verification")),
learning_written=bool(writeback.get("learning")),
execution_kind=execution_kind,
provider_delivery=effective_provider_delivery,
project_id=project_id,
)
return bool(
isinstance(response, Mapping)
and response.get("ok") is True
and response.get("_awooop_outbound_mirror_acknowledged") is True
)
except Exception as exc:
logger.warning(
"ansible_controlled_apply_telegram_receipt_failed",
incident_id=claim.incident_id,
catalog_id=claim.catalog_id,
apply_op_id=apply_op_id,
error=str(exc),
)
return False
async def _reconcile_verified_apply_closure_projections(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
apply_op_id: str,
writeback: dict[str, Any],
project_id: str,
) -> dict[str, Any]:
"""Repair terminal projections without ever executing the apply again."""
verified_success = bool(
result.returncode == 0
and result.post_verifier_passed is True
and writeback.get("verification_passed") is True
)
if not verified_success:
telegram_receipt = await _retry_telegram_receipt_acknowledged(
claim,
apply_op_id=apply_op_id,
project_id=project_id,
)
if not telegram_receipt:
receipt_finalized = await _send_controlled_apply_telegram_receipt(
claim,
result,
apply_op_id=apply_op_id,
writeback=writeback,
project_id=project_id,
)
if receipt_finalized:
telegram_receipt = await _retry_telegram_receipt_acknowledged(
claim,
apply_op_id=apply_op_id,
project_id=project_id,
)
telegram_lifecycle = False
if telegram_receipt:
telegram_lifecycle = await _append_alert_lifecycle_receipt(
claim,
"TELEGRAM_RESULT_SENT",
apply_op_id=apply_op_id,
success=False,
action_detail="controlled_apply_result_receipt_sent",
project_id=project_id,
post_verifier_passed=result.post_verifier_passed,
)
return {
"status": "not_verified_success",
"closed": False,
"telegram_receipt_acknowledged": telegram_receipt,
"telegram_lifecycle_written": telegram_lifecycle,
"runtime_apply_executed": False,
}
approval_projection = await _finalize_controlled_approval_projection(
claim,
result,
apply_op_id=apply_op_id,
project_id=project_id,
)
execution_lifecycle = await _append_alert_lifecycle_receipt(
claim,
"EXECUTION_COMPLETED",
apply_op_id=apply_op_id,
success=True,
action_detail=f"controlled_apply_completed:{claim.catalog_id}",
project_id=project_id,
post_verifier_passed=True,
)
readback = await _read_verified_apply_closure_prerequisites(
claim,
apply_op_id=apply_op_id,
project_id=project_id,
)
telegram_receipt = bool(
(readback.get("receipts") or {}).get("telegram_receipt")
)
if not telegram_receipt:
await _send_controlled_apply_telegram_receipt(
claim,
result,
apply_op_id=apply_op_id,
writeback=writeback,
project_id=project_id,
)
readback = await _read_verified_apply_closure_prerequisites(
claim,
apply_op_id=apply_op_id,
project_id=project_id,
)
telegram_receipt = bool(
(readback.get("receipts") or {}).get("telegram_receipt")
)
telegram_lifecycle = False
if telegram_receipt:
telegram_lifecycle = await _append_alert_lifecycle_receipt(
claim,
"TELEGRAM_RESULT_SENT",
apply_op_id=apply_op_id,
success=True,
action_detail="controlled_apply_result_receipt_sent",
project_id=project_id,
post_verifier_passed=True,
)
closure = await _finalize_verified_apply_closure(
claim,
apply_op_id=apply_op_id,
project_id=project_id,
)
return {
**closure,
"approval_projection_written": (
approval_projection or not _approval_id_for_claim(claim)
),
"execution_lifecycle_written": execution_lifecycle,
"telegram_receipt_acknowledged": telegram_receipt,
"telegram_lifecycle_written": telegram_lifecycle,
"runtime_apply_executed": False,
}
async def backfill_missing_auto_repair_execution_receipts_once(
*,
project_id: str = "awoooi",
window_hours: int = 24,
limit: int = 10,
) -> dict[str, Any]:
"""Create auto_repair_executions receipts for existing Ansible apply rows."""
stats = {
"scanned": 0,
"written": 0,
"verification_written": 0,
"learning_written": 0,
"trust_learning_written": 0,
"rag_writeback_written": 0,
"playbook_trust_written": 0,
"timeline_projection_written": 0,
"runtime_stage_receipts_written": 0,
"incident_closure_written": 0,
"telegram_receipt_acknowledged": 0,
"retry_terminal_projection_scanned": 0,
"retry_terminal_projection_written": 0,
"retry_terminal_projection_retry_receipt_written": 0,
"retry_terminal_projection_telegram_receipt_acknowledged": 0,
"retry_terminal_projection_incident_receipt_written": 0,
"retry_terminal_projection_lifecycle_written": 0,
"retry_terminal_projection_verified": 0,
"retry_terminal_projection_runtime_apply_executed": False,
"skipped": 0,
"error": None,
}
try:
projection = await backfill_missing_retry_terminal_projections_once(
project_id=project_id,
window_hours=window_hours,
limit=max(1, min(limit, 5)),
)
stats["retry_terminal_projection_scanned"] = int(
projection.get("scanned") or 0
)
stats["retry_terminal_projection_written"] = int(
projection.get("written") or 0
)
stats["retry_terminal_projection_retry_receipt_written"] = int(
projection.get("retry_receipt_written") or 0
)
stats[
"retry_terminal_projection_telegram_receipt_acknowledged"
] = int(projection.get("telegram_receipt_acknowledged") or 0)
stats["retry_terminal_projection_incident_receipt_written"] = int(
projection.get("incident_receipt_written") or 0
)
stats["retry_terminal_projection_lifecycle_written"] = int(
projection.get("lifecycle_written") or 0
)
stats["retry_terminal_projection_verified"] = int(
projection.get("verified") or 0
)
projection_error = str(projection.get("error") or "")[:500]
stats["scanned"] = stats["retry_terminal_projection_scanned"]
stats["incident_closure_written"] = stats[
"retry_terminal_projection_written"
]
stats["error"] = projection_error or None
async with get_db_context(project_id) as db:
await db.execute(text("SET LOCAL statement_timeout = '5000ms'"))
result = await db.execute(
text("""
SELECT
apply.op_id::text AS op_id,
apply.parent_op_id::text AS parent_op_id,
coalesce(apply.incident_id::text, apply.input ->> 'incident_id') AS incident_id,
apply.input,
apply.output,
apply.dry_run_result,
apply.error,
apply.duration_ms,
apply.status,
EXISTS (
SELECT 1
FROM incident_evidence verifier
WHERE verifier.incident_id = coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
)
AND verifier.post_execution_state ->> 'apply_op_id' = apply.op_id::text
AND verifier.verification_result = 'success'
) AS verifier_ready,
apply.created_at
FROM automation_operation_log apply
WHERE apply.operation_type = 'ansible_apply_executed'
AND apply.created_at >= NOW() - (:window_hours * INTERVAL '1 hour')
AND (
NOT EXISTS (
SELECT 1
FROM auto_repair_executions existing
WHERE existing.created_at >= NOW() - (:window_hours * INTERVAL '1 hour')
AND existing.triggered_by = 'ansible_controlled_apply'
AND existing.executed_steps::text LIKE '%' || apply.op_id::text || '%'
)
OR NOT EXISTS (
SELECT 1
FROM incident_evidence evidence
WHERE evidence.incident_id = coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
)
AND evidence.post_execution_state ->> 'apply_op_id' = apply.op_id::text
)
OR NOT EXISTS (
SELECT 1
FROM knowledge_entries km
WHERE km.related_incident_id = coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
)
AND km.path_type = 'ansible_apply_receipt:' || left(apply.op_id::text, 8)
)
OR NOT EXISTS (
SELECT 1
FROM automation_operation_log learning
WHERE learning.operation_type = 'ansible_learning_writeback_recorded'
AND learning.parent_op_id = apply.op_id
)
OR NOT EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) AS receipt(value)
WHERE receipt.value ->> 'stage_id' = 'timeline_projection'
)
OR NOT EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) AS receipt(value)
WHERE receipt.value ->> 'stage_id' = 'playbook_trust'
)
OR (
NOT EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) AS receipt(value)
WHERE receipt.value ->> 'stage_id' = 'rag_writeback'
)
AND (
apply.input -> 'rag_writeback_attempt' IS NULL
OR NULLIF(
apply.input -> 'rag_writeback_attempt' ->> 'attempted_at',
''
)::timestamptz <= NOW() - INTERVAL '5 minutes'
)
)
OR (
apply.status = 'success'
AND (
NOT EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
apply.input
-> 'runtime_stage_receipts',
'[]'::jsonb
)
) AS receipt(value)
WHERE receipt.value ->> 'stage_id'
= 'incident_closure'
)
OR EXISTS (
SELECT 1
FROM incidents incident
WHERE incident.incident_id = coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
)
AND incident.project_id = :project_id
AND (
incident.resolved_at IS NULL
OR upper(incident.status::text)
NOT IN ('RESOLVED', 'CLOSED')
)
)
OR NOT EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
)
AND lifecycle.event_type::text
= 'RESOLVED'
AND lifecycle.context ->> 'apply_op_id'
= apply.op_id::text
)
)
)
)
ORDER BY apply.created_at DESC
LIMIT :limit
"""),
{
"project_id": project_id,
"window_hours": max(1, window_hours),
"limit": max(1, limit),
},
)
rows = [dict(row) for row in result.mappings().all()]
stats["scanned"] += len(rows)
for row in rows:
reconstructed = _claim_from_apply_operation_row(row)
if reconstructed is None:
stats["skipped"] += 1
continue
claim, result = reconstructed
writeback = await _record_post_apply_verifier_and_learning(
claim,
result,
apply_op_id=str(row.get("op_id") or ""),
project_id=project_id,
)
verified_result = replace(
result,
post_verifier_passed=(
writeback.get("verification_passed") is True
),
)
if await _record_auto_repair_execution_receipt(
claim,
verified_result,
apply_op_id=str(row.get("op_id") or ""),
project_id=project_id,
):
stats["written"] += 1
else:
stats["skipped"] += 1
if writeback.get("verification"):
stats["verification_written"] += 1
if writeback.get("learning"):
stats["learning_written"] += 1
if writeback.get("trust_learning"):
stats["trust_learning_written"] += 1
stats["playbook_trust_written"] += 1
if writeback.get("rag_writeback"):
stats["rag_writeback_written"] += 1
timeline_receipt = await _record_timeline_projection_receipt(
claim,
verified_result,
apply_op_id=str(row.get("op_id") or ""),
project_id=project_id,
derived_from_durable_chain=True,
)
if timeline_receipt is not None:
stats["timeline_projection_written"] += 1
learning_receipts = tuple(
receipt
for receipt in writeback.get("runtime_stage_receipts", [])
if isinstance(receipt, dict)
)
if await _record_runtime_stage_receipts(
claim,
result,
apply_op_id=str(row.get("op_id") or ""),
verifier_ready=bool(writeback.get("verification_passed")),
project_id=project_id,
derived_from_durable_chain=True,
extra_receipts=(
*((timeline_receipt,) if timeline_receipt else ()),
*learning_receipts,
),
):
stats["runtime_stage_receipts_written"] += 1
closure = await _reconcile_verified_apply_closure_projections(
claim,
verified_result,
apply_op_id=str(row.get("op_id") or ""),
writeback=writeback,
project_id=project_id,
)
if closure.get("telegram_receipt_acknowledged") is True:
stats["telegram_receipt_acknowledged"] += 1
if closure.get("closed") is True:
stats["incident_closure_written"] += 1
except Exception as exc:
stats["error"] = f"{type(exc).__name__}: {exc}"[:500]
logger.warning("ansible_auto_repair_execution_receipt_backfill_failed", **stats)
return stats
async def _load_open_failed_apply_retry_row(
*,
project_id: str,
window_hours: int,
stale_after_seconds: int | None = None,
retry_backoff_seconds: int = _CONTROLLED_RETRY_ERROR_BACKOFF_MIN_SECONDS,
) -> dict[str, Any] | None:
"""Read one durable failed-apply retry candidate before transport checks."""
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT
apply.op_id::text AS op_id,
apply.parent_op_id::text AS parent_op_id,
coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
) AS incident_id,
apply.input,
apply.output,
apply.dry_run_result,
apply.error,
apply.duration_ms,
apply.status,
apply.created_at,
replay.op_id::text AS stale_replay_op_id,
replay.input AS stale_replay_input,
replay.created_at AS stale_replay_created_at,
CASE
WHEN replay.input ->> 'retry_recovery_count'
~ '^[0-9]+$'
THEN (replay.input
->> 'retry_recovery_count')::integer
ELSE 0
END AS stale_replay_recovery_count
FROM automation_operation_log apply
LEFT JOIN LATERAL (
SELECT pending.op_id, pending.input, pending.status,
pending.created_at
FROM automation_operation_log pending
WHERE pending.operation_type
= 'ansible_execution_skipped'
AND pending.parent_op_id = apply.op_id
AND pending.input ->> 'execution_mode'
= 'controlled_retry_check_mode_replay'
ORDER BY pending.created_at DESC, pending.op_id DESC
LIMIT 1
) replay ON TRUE
WHERE apply.operation_type = 'ansible_apply_executed'
AND apply.status = 'failed'
AND apply.created_at >= NOW() - (
:window_hours * INTERVAL '1 hour'
)
AND apply.created_at <= NOW() - (
:retry_backoff_seconds * INTERVAL '1 second'
)
AND EXISTS (
SELECT 1
FROM incident_evidence verifier
WHERE verifier.incident_id = coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
)
AND verifier.post_execution_state ->> 'apply_op_id'
= apply.op_id::text
AND verifier.verification_result IN ('failed', 'timeout')
)
AND (
replay.op_id IS NULL
OR (
replay.status = 'pending'
AND coalesce(
nullif(
replay.input ->> 'retry_claimed_at',
''
)::timestamptz,
replay.created_at
) <= NOW() - (
:stale_after_seconds * INTERVAL '1 second'
)
)
)
ORDER BY apply.created_at DESC
LIMIT 1
FOR UPDATE OF apply SKIP LOCKED
"""),
{
"window_hours": max(1, window_hours),
"retry_backoff_seconds": max(
_CONTROLLED_RETRY_ERROR_BACKOFF_MIN_SECONDS,
min(
_CONTROLLED_RETRY_ERROR_BACKOFF_MAX_SECONDS,
retry_backoff_seconds,
),
),
"stale_after_seconds": max(
300,
stale_after_seconds
or (
settings.AWOOOP_ANSIBLE_CHECK_MODE_TIMEOUT_SECONDS
+ 120
),
),
},
)
row = result.mappings().one_or_none()
return dict(row) if row is not None else None
async def preflight_failed_apply_retry_queue_once(
*,
project_id: str = "awoooi",
window_hours: int = 24,
**_kwargs: Any,
) -> dict[str, Any]:
"""Query retry backlog without using executor transport or writing state."""
try:
row = await _load_open_failed_apply_retry_row(
project_id=project_id,
window_hours=window_hours,
)
return {
"scanned": 1 if row is not None else 0,
"replayed": 0,
"check_mode_passed": 0,
"check_mode_failed": 0,
"runtime_stage_receipt_written": 0,
"blockers": [],
"error": None,
"query_only": True,
"runtime_apply_executed": False,
"execution_owner": "awoooi-ansible-executor-broker",
}
except Exception as exc:
return {
"scanned": 0,
"replayed": 0,
"check_mode_passed": 0,
"check_mode_failed": 0,
"runtime_stage_receipt_written": 0,
"blockers": [],
"error": type(exc).__name__,
"query_only": True,
"runtime_apply_executed": False,
"execution_owner": "awoooi-ansible-executor-broker",
}
async def _load_missing_retry_terminal_projection_rows(
*,
project_id: str,
window_hours: int,
limit: int,
) -> list[dict[str, Any]]:
"""Load verified no-write retries whose incident projection is incomplete."""
async with get_db_context(project_id) as db:
await db.execute(text("SET LOCAL statement_timeout = '5000ms'"))
result = await db.execute(
text("""
SELECT
apply.op_id::text AS op_id,
apply.parent_op_id::text AS parent_op_id,
coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
) AS incident_id,
apply.input,
apply.output,
apply.dry_run_result,
apply.error,
apply.duration_ms,
apply.status,
apply.created_at,
replay.op_id::text AS replay_op_id,
replay.status AS replay_status,
replay.output AS replay_output,
replay.dry_run_result AS replay_dry_run_result,
replay.error AS replay_error,
replay.duration_ms AS replay_duration_ms,
retry_receipt.receipt IS NOT NULL
AS retry_receipt_present,
coalesce(
retry_receipt.receipt #>> '{detail,terminal_type}',
replay.output ->> 'terminal_disposition'
)
AS terminal_type,
EXISTS (
SELECT 1
FROM awooop_outbound_message outbound
WHERE outbound.project_id = :project_id
AND outbound.channel_type = 'telegram'
AND outbound.send_status = 'sent'
AND outbound.provider_message_id IS NOT NULL
AND outbound.source_envelope #>>
'{callback_reply,action}'
= 'controlled_apply_result'
AND coalesce(
outbound.source_envelope
->> 'automation_run_id',
outbound.source_envelope #>>
'{callback_reply,automation_run_id}',
outbound.source_envelope #>>
'{source_refs,automation_run_ids,0}'
) = apply.input ->> 'automation_run_id'
AND coalesce(
outbound.source_envelope #>>
'{callback_reply,incident_id}',
outbound.source_envelope #>>
'{source_refs,incident_ids,0}'
) = coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
)
AND outbound.source_envelope #>>
'{callback_reply,apply_op_id}' = apply.op_id::text
) AS telegram_receipt_acknowledged
FROM automation_operation_log apply
JOIN automation_operation_log replay
ON replay.parent_op_id = apply.op_id
AND replay.operation_type = 'ansible_execution_skipped'
AND replay.input ->> 'execution_mode'
= 'controlled_retry_check_mode_replay'
AND replay.status IN ('success', 'failed')
AND replay.output ->> 'runtime_apply_executed' = 'false'
AND replay.input ->> 'automation_run_id'
= apply.input ->> 'automation_run_id'
AND coalesce(
replay.incident_id::text,
replay.input ->> 'incident_id'
) = coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
)
AND replay.output ->> 'terminal_disposition' = CASE
WHEN replay.status = 'success'
THEN 'no_write_replay_passed_waiting_repair_candidate'
ELSE 'no_write_replay_failed_waiting_playbook_or_transport_repair'
END
JOIN incidents incident
ON incident.incident_id = coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
)
AND incident.project_id = :project_id
LEFT JOIN LATERAL (
SELECT receipt.value AS receipt
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) receipt(value)
WHERE receipt.value ->> 'stage_id' = 'retry_or_rollback'
AND receipt.value ->> 'automation_run_id'
= apply.input ->> 'automation_run_id'
AND receipt.value #>>
'{detail,retry_check_mode_op_id}' = replay.op_id::text
AND receipt.value #>>
'{detail,failed_apply_op_id}' = apply.op_id::text
AND receipt.value #>>
'{detail,verified_no_write_terminal}' = 'true'
AND receipt.value #>>
'{detail,runtime_apply_executed}' = 'false'
AND receipt.value #>>
'{detail,repository_readback_verified}' = 'true'
AND receipt.value #>> '{detail,terminal_type}'
= replay.output ->> 'terminal_disposition'
AND receipt.value ->> 'durable_receipt' = 'true'
LIMIT 1
) retry_receipt ON TRUE
WHERE apply.operation_type = 'ansible_apply_executed'
AND apply.status = 'failed'
AND apply.created_at >= NOW() - (
:window_hours * INTERVAL '1 hour'
)
AND EXISTS (
SELECT 1
FROM incident_evidence verifier
WHERE verifier.incident_id = incident.incident_id
AND verifier.post_execution_state ->> 'apply_op_id'
= apply.op_id::text
AND verifier.verification_result IN ('failed', 'timeout')
)
AND EXISTS (
SELECT 1
FROM automation_operation_log source_check
JOIN automation_operation_log source_candidate
ON source_candidate.op_id = source_check.parent_op_id
AND source_candidate.operation_type
= 'ansible_candidate_matched'
WHERE source_check.op_id = apply.parent_op_id
AND source_check.operation_type
= 'ansible_check_mode_executed'
)
AND NOT EXISTS (
SELECT 1
FROM automation_operation_log source_check
JOIN automation_operation_log source_candidate
ON source_candidate.op_id = source_check.parent_op_id
AND source_candidate.operation_type
= 'ansible_candidate_matched'
JOIN automation_operation_log newer_candidate
ON newer_candidate.operation_type
= 'ansible_candidate_matched'
AND newer_candidate.input ->> 'executor' = 'ansible'
AND newer_candidate.input ->> 'decision_path'
= 'repair_candidate_controlled_queue'
AND coalesce(
newer_candidate.incident_id::text,
newer_candidate.input ->> 'incident_id'
) = incident.incident_id
AND (
newer_candidate.created_at,
newer_candidate.op_id
) > (
source_candidate.created_at,
source_candidate.op_id
)
WHERE source_check.op_id = apply.parent_op_id
AND source_check.operation_type
= 'ansible_check_mode_executed'
)
AND (
retry_receipt.receipt IS NULL
OR NOT EXISTS (
SELECT 1
FROM awooop_outbound_message outbound
WHERE outbound.project_id = :project_id
AND outbound.channel_type = 'telegram'
AND outbound.send_status = 'sent'
AND outbound.provider_message_id IS NOT NULL
AND outbound.source_envelope #>>
'{callback_reply,action}'
= 'controlled_apply_result'
AND coalesce(
outbound.source_envelope
->> 'automation_run_id',
outbound.source_envelope #>>
'{callback_reply,automation_run_id}',
outbound.source_envelope #>>
'{source_refs,automation_run_ids,0}'
) = apply.input ->> 'automation_run_id'
AND coalesce(
outbound.source_envelope #>>
'{callback_reply,incident_id}',
outbound.source_envelope #>>
'{source_refs,incident_ids,0}'
) = incident.incident_id
AND outbound.source_envelope #>>
'{callback_reply,apply_op_id}'
= apply.op_id::text
)
OR NOT EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) closure_receipt(value)
WHERE closure_receipt.value ->> 'stage_id'
= 'incident_closure'
AND closure_receipt.value ->> 'automation_run_id'
= apply.input ->> 'automation_run_id'
AND closure_receipt.value #>>
'{detail,apply_op_id}' = apply.op_id::text
AND closure_receipt.value #>>
'{detail,retry_op_id}' = replay.op_id::text
AND closure_receipt.value #>>
'{detail,terminal_type}'
= replay.output ->> 'terminal_disposition'
AND closure_receipt.value #>>
'{detail,no_write_terminal}' = 'true'
AND closure_receipt.value #>>
'{detail,proposal_executed}' = 'true'
AND closure_receipt.value #>>
'{detail,execution_success}' = 'false'
AND closure_receipt.value #>>
'{detail,telegram_receipt_acknowledged}'
= 'true'
AND closure_receipt.value #>>
'{detail,repository_readback_verified}'
= 'true'
AND closure_receipt.value ->> 'durable_receipt'
= 'true'
)
OR NOT EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = incident.incident_id
AND lifecycle.event_type::text
= 'EXECUTION_COMPLETED'
AND lifecycle.success IS FALSE
AND lifecycle.context ->> 'apply_op_id'
= apply.op_id::text
AND lifecycle.context ->> 'automation_run_id'
= apply.input ->> 'automation_run_id'
)
OR (
upper(incident.status::text)
IS DISTINCT FROM 'MITIGATING'
OR cast(incident.outcome AS jsonb) #>>
'{automation_terminal,schema_version}'
IS DISTINCT FROM
'ansible_incident_terminal_disposition_v1'
OR cast(incident.outcome AS jsonb) ->>
'proposal_executed'
IS DISTINCT FROM 'true'
OR cast(incident.outcome AS jsonb) ->>
'execution_success'
IS DISTINCT FROM 'false'
OR
cast(incident.outcome AS jsonb) #>>
'{automation_terminal,automation_run_id}'
IS DISTINCT FROM apply.input
->> 'automation_run_id'
OR cast(incident.outcome AS jsonb) #>>
'{automation_terminal,apply_op_id}'
IS DISTINCT FROM apply.op_id::text
OR cast(incident.outcome AS jsonb) #>>
'{automation_terminal,retry_op_id}'
IS DISTINCT FROM replay.op_id::text
OR cast(incident.outcome AS jsonb) #>>
'{automation_terminal,terminal_type}'
IS DISTINCT FROM replay.output
->> 'terminal_disposition'
OR cast(incident.outcome AS jsonb) #>>
'{automation_terminal,no_write_terminal}'
IS DISTINCT FROM 'true'
OR cast(incident.outcome AS jsonb) #>>
'{automation_terminal,telegram_receipt_acknowledged}'
IS DISTINCT FROM 'true'
)
)
ORDER BY replay.created_at DESC
LIMIT :limit
"""),
{
"project_id": project_id,
"window_hours": max(1, window_hours),
"limit": max(1, min(limit, 5)),
},
)
return [dict(row) for row in result.mappings().all()]
async def _verify_retry_terminal_projection_readback(
claim: AnsibleCheckModeClaim,
*,
apply_op_id: str,
replay_op_id: str,
terminal_type: str,
project_id: str,
) -> bool:
automation_run_id = _automation_run_id_for_claim(claim)
async with get_db_context(project_id) as db:
await db.execute(text("SET LOCAL statement_timeout = '5000ms'"))
result = await db.execute(
text("""
SELECT
upper(incident.status::text) = 'MITIGATING'
AND cast(incident.outcome AS jsonb) #>>
'{automation_terminal,schema_version}'
= 'ansible_incident_terminal_disposition_v1'
AND cast(incident.outcome AS jsonb) #>>
'{automation_terminal,automation_run_id}'
= :automation_run_id
AND cast(incident.outcome AS jsonb) #>>
'{automation_terminal,apply_op_id}' = :apply_op_id
AND cast(incident.outcome AS jsonb) #>>
'{automation_terminal,retry_op_id}' = :replay_op_id
AND cast(incident.outcome AS jsonb) #>>
'{automation_terminal,terminal_type}' = :terminal_type
AND cast(incident.outcome AS jsonb) #>>
'{automation_terminal,no_write_terminal}' = 'true'
AND cast(incident.outcome AS jsonb) #>>
'{automation_terminal,telegram_receipt_acknowledged}'
= 'true'
AND cast(incident.outcome AS jsonb) ->>
'proposal_executed' = 'true'
AND cast(incident.outcome AS jsonb) ->>
'execution_success' = 'false'
AND EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) retry_receipt(value)
WHERE retry_receipt.value ->> 'stage_id'
= 'retry_or_rollback'
AND retry_receipt.value ->> 'automation_run_id'
= :automation_run_id
AND retry_receipt.value #>>
'{detail,failed_apply_op_id}' = :apply_op_id
AND retry_receipt.value #>>
'{detail,retry_check_mode_op_id}' = :replay_op_id
AND retry_receipt.value #>>
'{detail,terminal_type}' = :terminal_type
AND retry_receipt.value #>>
'{detail,verified_no_write_terminal}' = 'true'
AND retry_receipt.value #>>
'{detail,runtime_apply_executed}' = 'false'
AND retry_receipt.value #>>
'{detail,repository_readback_verified}' = 'true'
AND retry_receipt.value ->> 'durable_receipt' = 'true'
)
AND EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) receipt(value)
WHERE receipt.value ->> 'stage_id' = 'incident_closure'
AND receipt.value ->> 'automation_run_id'
= :automation_run_id
AND receipt.value #>> '{detail,apply_op_id}'
= :apply_op_id
AND receipt.value #>> '{detail,retry_op_id}'
= :replay_op_id
AND receipt.value #>>
'{detail,telegram_receipt_acknowledged}' = 'true'
AND receipt.value #>>
'{detail,proposal_executed}' = 'true'
AND receipt.value #>>
'{detail,execution_success}' = 'false'
AND receipt.value #>>
'{detail,repository_readback_verified}' = 'true'
AND receipt.value ->> 'durable_receipt' = 'true'
)
AND EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = incident.incident_id
AND lifecycle.event_type::text = 'EXECUTION_COMPLETED'
AND lifecycle.success IS FALSE
AND lifecycle.context ->> 'apply_op_id' = :apply_op_id
AND lifecycle.context ->> 'automation_run_id'
= :automation_run_id
) AS projection_verified
FROM incidents incident
JOIN automation_operation_log apply
ON apply.op_id = CAST(:apply_op_id_uuid AS uuid)
AND apply.operation_type = 'ansible_apply_executed'
AND apply.status = 'failed'
AND apply.input ->> 'automation_run_id'
= :automation_run_id
JOIN automation_operation_log replay
ON replay.op_id = CAST(:replay_op_id_uuid AS uuid)
AND replay.parent_op_id = apply.op_id
AND replay.operation_type = 'ansible_execution_skipped'
AND replay.status IN ('success', 'failed')
AND replay.input ->> 'execution_mode'
= 'controlled_retry_check_mode_replay'
AND replay.input ->> 'automation_run_id'
= :automation_run_id
AND coalesce(
replay.incident_id::text,
replay.input ->> 'incident_id'
) = incident.incident_id
AND replay.output ->> 'runtime_apply_executed' = 'false'
AND replay.output ->> 'terminal_disposition' = :terminal_type
WHERE incident.incident_id = :incident_id
AND incident.project_id = :project_id
LIMIT 1
"""),
{
"automation_run_id": automation_run_id,
"apply_op_id": apply_op_id,
"apply_op_id_uuid": apply_op_id,
"replay_op_id": replay_op_id,
"replay_op_id_uuid": replay_op_id,
"terminal_type": terminal_type,
"incident_id": claim.incident_id,
"project_id": project_id,
},
)
return result.scalar() is True
async def backfill_missing_retry_terminal_projections_once(
*,
project_id: str = "awoooi",
window_hours: int = 24,
limit: int = 1,
) -> dict[str, Any]:
"""Repair missing incident terminals from durable no-write retry receipts."""
stats: dict[str, Any] = {
"scanned": 0,
"written": 0,
"retry_receipt_written": 0,
"telegram_receipt_acknowledged": 0,
"incident_receipt_written": 0,
"lifecycle_written": 0,
"verified": 0,
"runtime_apply_executed": False,
"error": None,
}
try:
rows = await _load_missing_retry_terminal_projection_rows(
project_id=project_id,
window_hours=window_hours,
limit=limit,
)
stats["scanned"] = len(rows)
for row in rows:
reconstructed = _claim_from_apply_operation_row(row)
if reconstructed is None:
continue
claim, failed_apply_result = reconstructed
apply_op_id = str(row.get("op_id") or "")
replay_op_id = str(row.get("replay_op_id") or "")
terminal_type = str(row.get("terminal_type") or "")
replay_status = str(row.get("replay_status") or "")
expected_terminal_type = (
"no_write_replay_passed_waiting_repair_candidate"
if replay_status == "success"
else (
"no_write_replay_failed_waiting_playbook_or_transport_repair"
if replay_status == "failed"
else ""
)
)
if not (
apply_op_id
and replay_op_id
and terminal_type == expected_terminal_type
):
continue
telegram_acknowledged = (
row.get("telegram_receipt_acknowledged") is True
)
if not telegram_acknowledged:
apply_output = _json_loads(row.get("output"))
receipt_sent = await _send_controlled_apply_telegram_receipt(
claim,
failed_apply_result,
apply_op_id=apply_op_id,
writeback={
"verification_result": str(
apply_output.get(
"independent_post_verifier_result"
)
or "failed"
),
"verification": True,
"learning": False,
},
project_id=project_id,
)
if receipt_sent:
telegram_acknowledged = (
await _retry_telegram_receipt_acknowledged(
claim,
apply_op_id=apply_op_id,
project_id=project_id,
)
)
if not telegram_acknowledged:
continue
stats["telegram_receipt_acknowledged"] += 1
if row.get("retry_receipt_present") is not True:
replay_output = _json_loads(row.get("replay_output"))
replay_dry_run = _json_loads(
row.get("replay_dry_run_result")
)
raw_returncode = replay_output.get("returncode")
try:
replay_returncode = int(raw_returncode)
except (TypeError, ValueError):
replay_returncode = 0 if replay_status == "success" else 1
replay_result = AnsibleRunResult(
returncode=replay_returncode,
stdout="",
stderr=str(row.get("replay_error") or ""),
duration_ms=int(row.get("replay_duration_ms") or 0),
timed_out=replay_dry_run.get("timed_out") is True,
post_verifier_passed=False,
)
retry_receipt_written = (
await _append_runtime_stage_receipts_to_apply(
apply_op_id=apply_op_id,
receipts=(
_build_retry_runtime_stage_receipt(
claim,
replay_result,
apply_op_id=apply_op_id,
replay_op_id=replay_op_id,
derived_from_durable_chain=True,
check_mode_replay_performed=(
replay_output.get(
"check_mode_replay_performed"
)
is not False
),
),
),
project_id=project_id,
)
)
stats["retry_receipt_written"] += int(
retry_receipt_written
)
if not retry_receipt_written:
continue
terminal = await _record_incident_terminal_disposition(
claim,
apply_op_id=apply_op_id,
project_id=project_id,
terminal_type=terminal_type,
success_terminal=False,
retry_op_id=replay_op_id,
telegram_receipt_acknowledged=telegram_acknowledged,
)
if terminal is None:
continue
incident_receipt_written = (
await _append_runtime_stage_receipts_to_apply(
apply_op_id=apply_op_id,
receipts=(
_runtime_stage_receipt(
claim,
stage_id="incident_closure",
evidence_ref=(
f"incidents:{claim.incident_id}:automation_terminal"
),
detail=terminal,
derived_from_durable_chain=True,
),
),
project_id=project_id,
)
)
lifecycle_written = await _append_alert_lifecycle_receipt(
claim,
"EXECUTION_COMPLETED",
apply_op_id=apply_op_id,
success=False,
action_detail="controlled_retry_no_write_terminal_backfilled",
project_id=project_id,
error_message="failed_apply_retry_terminal",
post_verifier_passed=False,
)
stats["incident_receipt_written"] += int(
incident_receipt_written
)
stats["lifecycle_written"] += int(lifecycle_written)
if not incident_receipt_written or not lifecycle_written:
continue
verified = await _verify_retry_terminal_projection_readback(
claim,
apply_op_id=apply_op_id,
replay_op_id=replay_op_id,
terminal_type=terminal_type,
project_id=project_id,
)
stats["verified"] += int(verified)
stats["written"] += int(verified)
except Exception as exc:
stats["error"] = f"{type(exc).__name__}: {exc}"[:500]
logger.warning(
"ansible_retry_terminal_projection_backfill_failed",
project_id=project_id,
**stats,
)
return stats
async def run_failed_apply_check_mode_replay_once(
*,
project_id: str = "awoooi",
window_hours: int = 24,
timeout_seconds: int | None = None,
) -> dict[str, Any]:
"""Replay one failed apply in no-write check mode and persist its terminal receipt."""
stats: dict[str, Any] = {
"scanned": 0,
"replayed": 0,
"check_mode_passed": 0,
"check_mode_failed": 0,
"runtime_apply_executed": 0,
"runtime_stage_receipt_written": 0,
"stale_retry_reclaimed": 0,
"stale_retry_terminalized": 0,
"blockers": [],
"error": None,
}
try:
effective_timeout = (
timeout_seconds or settings.AWOOOP_ANSIBLE_CHECK_MODE_TIMEOUT_SECONDS
)
stale_after_seconds = max(300, effective_timeout + 120)
row = await _load_open_failed_apply_retry_row(
project_id=project_id,
window_hours=window_hours,
stale_after_seconds=stale_after_seconds,
)
if row is None:
return stats
stats["scanned"] = 1
reconstructed = _claim_from_apply_operation_row(row)
if reconstructed is None:
stats["error"] = "failed_apply_receipt_could_not_be_reconstructed"
return stats
claim, _failed_apply_result = reconstructed
apply_op_id = str(row.get("op_id") or "")
automation_run_id = str(
claim.input_payload.get("automation_run_id")
or claim.source_candidate_op_id
)
stale_replay_op_id = str(row.get("stale_replay_op_id") or "")
stale_recovery_count = int(
row.get("stale_replay_recovery_count") or 0
)
stale_retry_terminalized = bool(
stale_replay_op_id
and stale_recovery_count >= _CONTROLLED_RETRY_MAX_ATTEMPTS
)
stats["stale_retry_reclaimed"] = int(bool(stale_replay_op_id))
stats["stale_retry_terminalized"] = int(stale_retry_terminalized)
if not stale_retry_terminalized:
blockers = _runtime_blockers()
if blockers:
stats["blockers"] = blockers
return stats
transport_blockers = await recent_ansible_transport_blockers(
project_id=project_id
)
if transport_blockers:
stats["blockers"] = transport_blockers
return stats
retry_claim_id = str(uuid4())
retry_claimed_at = datetime.now(UTC).isoformat()
input_payload = {
**claim.input_payload,
"automation_run_id": automation_run_id,
"execution_mode": "controlled_retry_check_mode_replay",
"retry_of_apply_op_id": apply_op_id,
"check_mode": True,
"diff": True,
"apply_enabled": False,
"approval_required_before_apply": False,
"owner_review_required": False,
"controlled_apply_allowed": True,
"controlled_apply_blocker": None,
"bounded_execution_scope": "retry_check_mode_only",
"retry_claim_id": retry_claim_id,
"retry_claimed_at": retry_claimed_at,
"retry_recovery_count": 0,
"next_controlled_action": (
"queue_ai_playbook_or_transport_repair_candidate"
),
}
async with get_db_context(project_id) as db:
await db.execute(
text("""
SELECT pg_advisory_xact_lock(
hashtextextended(:retry_claim_lock_key, 0)
)
"""),
{
"retry_claim_lock_key": (
f"ansible-retry-claim:{apply_op_id}"
)
},
)
if stale_replay_op_id:
claimed = await db.execute(
text("""
UPDATE automation_operation_log replay
SET input = coalesce(replay.input, '{}'::jsonb)
|| jsonb_build_object(
'retry_claim_id',
CAST(:retry_claim_id AS text),
'retry_claimed_at',
CAST(:retry_claimed_at AS text),
'retry_claim_recovered',
true,
'retry_recovery_count',
CASE
WHEN replay.input
->> 'retry_recovery_count'
~ '^[0-9]+$'
THEN (replay.input
->> 'retry_recovery_count')::integer
+ 1
ELSE 1
END
),
dry_run_result = coalesce(
replay.dry_run_result,
'{}'::jsonb
) || jsonb_build_object(
'claim_state',
'reclaimed_after_stale_pending'
)
WHERE replay.op_id = CAST(:replay_op_id AS uuid)
AND replay.operation_type
= 'ansible_execution_skipped'
AND replay.status = 'pending'
AND replay.parent_op_id
= CAST(:parent_op_id AS uuid)
AND replay.input ->> 'automation_run_id'
= :automation_run_id
AND replay.input ->> 'execution_mode'
= 'controlled_retry_check_mode_replay'
AND coalesce(
nullif(
replay.input ->> 'retry_claimed_at',
''
)::timestamptz,
replay.created_at
) <= NOW() - (
:stale_after_seconds * INTERVAL '1 second'
)
RETURNING replay.op_id
"""),
{
"replay_op_id": stale_replay_op_id,
"retry_claim_id": retry_claim_id,
"retry_claimed_at": retry_claimed_at,
"stale_after_seconds": stale_after_seconds,
"parent_op_id": apply_op_id,
"automation_run_id": automation_run_id,
},
)
else:
claimed = await db.execute(
text("""
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, dry_run_result,
parent_op_id, tags
)
SELECT
'ansible_execution_skipped',
'ansible_controlled_retry_worker',
'pending',
:incident_db_id,
CAST(:input AS jsonb),
'{}'::jsonb,
CAST(:dry_run_result AS jsonb),
CAST(:parent_op_id AS uuid),
:tags
WHERE NOT EXISTS (
SELECT 1
FROM automation_operation_log replay
WHERE replay.operation_type = 'ansible_execution_skipped'
AND replay.parent_op_id = CAST(:parent_op_id AS uuid)
AND replay.input ->> 'execution_mode'
= 'controlled_retry_check_mode_replay'
)
RETURNING op_id
"""),
{
"incident_db_id": _automation_operation_log_incident_id(
claim.incident_id
),
"input": json.dumps(input_payload, ensure_ascii=False),
"dry_run_result": json.dumps(
{
"execution_mode": (
"controlled_retry_check_mode_replay"
),
"retry_of_apply_op_id": apply_op_id,
"check_mode_executed": False,
"runtime_apply_executed": False,
"claim_state": "claimed",
},
ensure_ascii=False,
),
"parent_op_id": apply_op_id,
"tags": [
"ansible",
"controlled_retry",
"check_mode_replay",
"no_runtime_apply",
f"automation_run_id:{automation_run_id}",
],
},
)
replay_op_id_value = claimed.scalar_one_or_none()
if replay_op_id_value is None:
stats["blockers"] = ["failed_apply_retry_claim_raced"]
return stats
replay_op_id = str(replay_op_id_value)
if stale_retry_terminalized:
replay_result = AnsibleRunResult(
returncode=1,
stdout="",
stderr="stale_pending_retry_recovery_budget_exhausted",
duration_ms=0,
)
else:
try:
spec = build_ansible_check_mode_command(
playbook_path=claim.playbook_path,
inventory_hosts=claim.inventory_hosts,
)
replay_result = await _run_ansible_command(
spec,
timeout_seconds=effective_timeout,
)
except Exception as exc:
replay_result = AnsibleRunResult(
returncode=1,
stdout="",
stderr=f"ansible_retry_check_mode_runtime_error: {exc}",
duration_ms=0,
)
status, output, dry_run_result, error = _build_result_payload(
replay_result,
controlled_apply_allowed=True,
controlled_apply_blocker=None,
)
output.update({
"execution_mode": "controlled_retry_check_mode_replay",
"retry_of_apply_op_id": apply_op_id,
"apply_enabled": False,
"approval_required_before_apply": False,
"owner_review_required": False,
"runtime_apply_executed": False,
"check_mode_executed": not stale_retry_terminalized,
"check_mode_replay_performed": not stale_retry_terminalized,
"stale_pending_retry_terminalized": stale_retry_terminalized,
"bounded_execution_scope": "retry_check_mode_only",
"next_required_step": (
"queue_ai_playbook_or_transport_repair_candidate"
),
"terminal_disposition": (
"no_write_replay_passed_waiting_repair_candidate"
if replay_result.returncode == 0
else "no_write_replay_failed_waiting_playbook_or_transport_repair"
),
})
dry_run_result.update({
"execution_mode": "controlled_retry_check_mode_replay",
"retry_of_apply_op_id": apply_op_id,
"apply_enabled": False,
"approval_required_before_apply": False,
"owner_review_required": False,
"runtime_apply_executed": False,
"check_mode_executed": not stale_retry_terminalized,
"check_mode_replay_performed": not stale_retry_terminalized,
"stale_pending_retry_terminalized": stale_retry_terminalized,
"bounded_execution_scope": "retry_check_mode_only",
"next_controlled_action": (
"queue_ai_playbook_or_transport_repair_candidate"
),
})
async with get_db_context(project_id) as db:
updated = await db.execute(
text("""
UPDATE automation_operation_log
SET status = :status,
output = CAST(:output AS jsonb),
dry_run_result = CAST(:dry_run_result AS jsonb),
error = :error,
duration_ms = :duration_ms,
stderr_feed_back = :stderr
WHERE op_id = CAST(:op_id AS uuid)
AND status = 'pending'
AND input ->> 'retry_claim_id' = :retry_claim_id
RETURNING op_id
"""),
{
"status": status,
"output": json.dumps(output, ensure_ascii=False),
"dry_run_result": json.dumps(dry_run_result, ensure_ascii=False),
"error": _tail(error or "", 2000) or None,
"duration_ms": replay_result.duration_ms,
"stderr": _tail(replay_result.stderr, _STDERR_LIMIT),
"op_id": replay_op_id,
"retry_claim_id": retry_claim_id,
},
)
if updated.scalar_one_or_none() is None:
stats["error"] = "failed_apply_retry_terminal_update_lost_lease"
return stats
stats["replayed"] = 0 if stale_retry_terminalized else 1
stats[
"check_mode_passed" if replay_result.returncode == 0 else "check_mode_failed"
] = 1
if await _record_retry_runtime_stage_receipt(
claim,
replay_result,
apply_op_id=apply_op_id,
replay_op_id=replay_op_id,
project_id=project_id,
check_mode_replay_performed=not stale_retry_terminalized,
):
stats["runtime_stage_receipt_written"] = 1
logger.info(
"ansible_failed_apply_check_mode_replay_completed",
automation_run_id=automation_run_id,
apply_op_id=apply_op_id,
replay_op_id=str(replay_op_id),
returncode=replay_result.returncode,
runtime_apply_executed=False,
runtime_stage_receipt_written=bool(
stats["runtime_stage_receipt_written"]
),
)
except Exception as exc:
stats["error"] = f"{type(exc).__name__}: {exc}"[:500]
logger.warning("ansible_failed_apply_check_mode_replay_failed", **stats)
return stats
async def claim_pending_check_modes(
*,
project_id: str = "awoooi",
limit: int = 1,
candidate_max_age_hours: int | None = None,
) -> list[AnsibleCheckModeClaim]:
"""Claim pending Ansible candidates by inserting pending check-mode rows."""
claims: list[AnsibleCheckModeClaim] = []
max_age_hours = candidate_max_age_hours or settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT
candidate.op_id,
candidate.input
FROM automation_operation_log candidate
WHERE candidate.operation_type = 'ansible_candidate_matched'
AND candidate.status = 'dry_run'
AND candidate.input ->> 'executor' = 'ansible'
AND candidate.created_at >= NOW() - (:candidate_max_age_hours * INTERVAL '1 hour')
AND COALESCE((candidate.dry_run_result ->> 'check_mode_executed')::boolean, false) = false
AND NOT EXISTS (
SELECT 1
FROM automation_operation_log existing
WHERE existing.parent_op_id = candidate.op_id
AND existing.operation_type IN (
'ansible_check_mode_executed',
'ansible_execution_skipped'
)
)
ORDER BY
CASE
WHEN candidate.input ->> 'work_item_id' LIKE 'P0-%'
THEN 0
WHEN candidate.input ->> 'proposal_source'
= 'alert_webhook_controlled_router'
THEN LEAST(
CASE
WHEN candidate.input ->> 'execution_priority'
~ '^[0-9]{1,3}$'
THEN (candidate.input ->> 'execution_priority')::int
ELSE 30
END,
30
)
WHEN candidate.input ->> 'execution_priority'
~ '^[0-9]{1,3}$'
THEN LEAST(
(candidate.input ->> 'execution_priority')::int,
100
)
ELSE 100
END ASC,
candidate.created_at ASC
LIMIT :limit
FOR UPDATE SKIP LOCKED
"""),
{
"limit": max(1, limit),
"candidate_max_age_hours": max(1, max_age_hours),
},
)
rows = result.mappings().all()
for row in rows:
source_op_id = str(row["op_id"])
candidate_input = _json_loads(row["input"])
candidate_input, trust_gate = await _guard_candidate_input_by_playbook_trust(
db,
candidate_input,
)
if trust_gate.get("status") == "circuit_open":
await _insert_skipped_candidate(
db,
source_candidate_op_id=source_op_id,
candidate_input=candidate_input,
reason="playbook_trust_circuit_open",
)
continue
try:
claim_input = build_ansible_check_mode_claim_input(
source_candidate_op_id=source_op_id,
candidate_input=candidate_input,
)
except ValueError as exc:
await _insert_skipped_candidate(
db,
source_candidate_op_id=source_op_id,
candidate_input=candidate_input,
reason=str(exc),
)
continue
inserted = await db.execute(
text("""
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, dry_run_result,
parent_op_id, tags
) VALUES (
'ansible_check_mode_executed',
'ansible_check_mode_worker',
'pending',
:incident_db_id,
CAST(:input AS jsonb),
'{}'::jsonb,
CAST(:dry_run_result AS jsonb),
CAST(:parent_op_id AS uuid),
:tags
)
RETURNING op_id
"""),
{
"incident_id": _incident_id_from_payload(claim_input),
"incident_db_id": _automation_operation_log_incident_id(
_incident_id_from_payload(claim_input)
),
"input": json.dumps(claim_input, ensure_ascii=False),
"dry_run_result": json.dumps({
"check_mode_executed": False,
"apply_executed": False,
"claim_state": "claimed",
"controlled_apply_allowed": bool(
claim_input.get("controlled_apply_allowed")
),
"controlled_apply_blocker": claim_input.get("controlled_apply_blocker"),
}, ensure_ascii=False),
"parent_op_id": source_op_id,
"tags": [
"ansible",
"check_mode",
"pending",
"controlled_apply_allowed"
if claim_input.get("controlled_apply_allowed")
else "controlled_apply_blocked",
],
},
)
op_id = str(inserted.scalar_one())
claims.append(
AnsibleCheckModeClaim(
op_id=op_id,
source_candidate_op_id=source_op_id,
incident_id=str(claim_input.get("incident_id") or ""),
catalog_id=str(claim_input["catalog_id"]),
playbook_path=str(claim_input["playbook_path"]),
apply_playbook_path=str(claim_input["apply_playbook_path"]),
inventory_hosts=tuple(str(host) for host in claim_input["inventory_hosts"]),
risk_level=str(claim_input.get("risk_level") or ""),
input_payload=claim_input,
)
)
return claims
async def claim_stale_pending_check_modes(
*,
project_id: str = "awoooi",
limit: int = 1,
stale_after_seconds: int,
) -> list[AnsibleCheckModeClaim]:
"""Lease and revalidate stale pending rows before replaying check-mode."""
claims: list[AnsibleCheckModeClaim] = []
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT
check_mode.op_id,
check_mode.parent_op_id,
check_mode.tags,
coalesce(
check_mode.incident_id::text,
check_mode.input ->> 'incident_id'
) AS incident_id,
check_mode.input
FROM automation_operation_log check_mode
WHERE check_mode.operation_type = 'ansible_check_mode_executed'
AND check_mode.status = 'pending'
AND coalesce(
nullif(
check_mode.dry_run_result ->> 'reclaimed_at',
''
)::timestamptz,
check_mode.created_at
) <= NOW() - (:stale_after_seconds * INTERVAL '1 second')
ORDER BY check_mode.created_at DESC
LIMIT :limit
FOR UPDATE SKIP LOCKED
"""),
{
"limit": max(1, limit),
"stale_after_seconds": max(300, stale_after_seconds),
},
)
for row_value in result.mappings().all():
row = dict(row_value)
claim = _claim_from_stale_check_mode_row(row)
if claim is None:
await db.execute(
text("""
UPDATE automation_operation_log
SET status = 'failed',
error = 'stale_pending_reclaim_rejected_by_current_policy',
dry_run_result = coalesce(
dry_run_result,
'{}'::jsonb
) || jsonb_build_object(
'claim_state', 'stale_reclaim_rejected',
'reclaimed_at', NOW(),
'check_mode_executed', false
)
WHERE op_id = CAST(:op_id AS uuid)
AND status = 'pending'
"""),
{"op_id": str(row.get("op_id") or "")},
)
continue
await db.execute(
text("""
UPDATE automation_operation_log
SET input = CAST(:input AS jsonb),
error = NULL,
dry_run_result = coalesce(
dry_run_result,
'{}'::jsonb
) || jsonb_build_object(
'claim_state', 'stale_reclaimed',
'reclaimed_at', NOW(),
'check_mode_executed', false,
'apply_executed', false
)
WHERE op_id = CAST(:op_id AS uuid)
AND status = 'pending'
"""),
{
"input": json.dumps(claim.input_payload, ensure_ascii=False),
"op_id": claim.op_id,
},
)
claims.append(claim)
return claims
async def claim_stdin_boundary_failed_check_modes(
*,
project_id: str = "awoooi",
limit: int = 1,
window_hours: int = _STDIN_BOUNDARY_REPLAY_WINDOW_HOURS,
) -> list[AnsibleCheckModeClaim]:
"""Replay historical nonblocking-stdin failures once, without apply."""
claims: list[AnsibleCheckModeClaim] = []
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT
check_mode.op_id,
check_mode.parent_op_id,
coalesce(
check_mode.input ->> 'incident_id',
check_mode.incident_id::text
) AS incident_id,
check_mode.input
FROM automation_operation_log check_mode
WHERE check_mode.operation_type = 'ansible_check_mode_executed'
AND check_mode.status = 'failed'
AND check_mode.created_at >= NOW() - (
:window_hours * INTERVAL '1 hour'
)
AND (
coalesce(check_mode.error, '')
LIKE '%Ansible requires blocking IO%'
OR coalesce(check_mode.error, '')
LIKE '%Non-blocking file handles detected%'
OR coalesce(check_mode.stderr_feed_back, '')
LIKE '%Ansible requires blocking IO%'
OR coalesce(check_mode.stderr_feed_back, '')
LIKE '%Non-blocking file handles detected%'
OR coalesce(check_mode.output::text, '')
LIKE '%Ansible requires blocking IO%'
OR coalesce(check_mode.output::text, '')
LIKE '%Non-blocking file handles detected%'
OR coalesce(check_mode.dry_run_result::text, '')
LIKE '%Ansible requires blocking IO%'
OR coalesce(check_mode.dry_run_result::text, '')
LIKE '%Non-blocking file handles detected%'
)
AND NOT EXISTS (
SELECT 1
FROM automation_operation_log replay
WHERE replay.operation_type
= 'ansible_check_mode_executed'
AND replay.input ->> 'replay_of_check_mode_op_id'
= check_mode.op_id::text
)
ORDER BY check_mode.created_at ASC
LIMIT :scan_limit
FOR UPDATE SKIP LOCKED
"""),
{
"window_hours": max(1, min(window_hours, 7 * 24)),
"scan_limit": min(100, max(10, limit * 10)),
},
)
for row_value in result.mappings().all():
row = dict(row_value)
original = _claim_from_stale_check_mode_row(row)
if original is None:
continue
failed_check_mode_op_id = str(row.get("op_id") or "")
replay_input = {
**original.input_payload,
"execution_mode": "stdin_boundary_check_mode_replay",
"historical_stdin_boundary_replay": True,
"replay_of_check_mode_op_id": failed_check_mode_op_id,
"check_mode": True,
"diff": True,
"apply_enabled": False,
"controlled_apply_allowed": False,
"controlled_apply_blocker": (
"historical_stdin_boundary_replay_no_write"
),
"approval_required_before_apply": False,
"owner_review_required": False,
"bounded_execution_scope": "check_mode_replay_only",
"safe_next_action": (
"queue_new_controlled_apply_candidate_after_no_write_replay"
),
}
inserted = await db.execute(
text("""
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, dry_run_result,
parent_op_id, tags
)
SELECT
'ansible_check_mode_executed',
'ansible_stdin_boundary_replay_worker',
'pending',
:incident_db_id,
CAST(:input AS jsonb),
'{}'::jsonb,
CAST(:dry_run_result AS jsonb),
CAST(:parent_op_id AS uuid),
:tags
WHERE NOT EXISTS (
SELECT 1
FROM automation_operation_log replay
WHERE replay.operation_type
= 'ansible_check_mode_executed'
AND replay.input ->> 'replay_of_check_mode_op_id'
= :failed_check_mode_op_id
)
RETURNING op_id
"""),
{
"incident_db_id": _automation_operation_log_incident_id(
original.incident_id
),
"input": json.dumps(replay_input, ensure_ascii=False),
"dry_run_result": json.dumps(
{
"execution_mode": (
"stdin_boundary_check_mode_replay"
),
"replay_of_check_mode_op_id": (
failed_check_mode_op_id
),
"check_mode_executed": False,
"runtime_apply_executed": False,
"claim_state": "claimed",
},
ensure_ascii=False,
),
"parent_op_id": original.source_candidate_op_id,
"tags": [
"ansible",
"check_mode",
"stdin_boundary_replay",
"no_runtime_apply",
],
"failed_check_mode_op_id": failed_check_mode_op_id,
},
)
op_id_value = inserted.scalar_one_or_none()
if op_id_value is None:
continue
claims.append(
AnsibleCheckModeClaim(
op_id=str(op_id_value),
source_candidate_op_id=original.source_candidate_op_id,
incident_id=original.incident_id,
catalog_id=original.catalog_id,
playbook_path=original.playbook_path,
apply_playbook_path=original.apply_playbook_path,
inventory_hosts=original.inventory_hosts,
risk_level=original.risk_level,
input_payload=replay_input,
)
)
if len(claims) >= max(1, limit):
break
return claims
async def claim_catalog_drift_failed_check_modes(
*,
project_id: str = "awoooi",
limit: int = 1,
candidate_max_age_hours: int | None = None,
) -> list[AnsibleCheckModeClaim]:
"""Replay a failed check once when its canonical check playbook changed."""
claims: list[AnsibleCheckModeClaim] = []
max_age_hours = max(
candidate_max_age_hours
or settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS,
_CATALOG_REPAIR_REPLAY_WINDOW_HOURS,
)
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT
check_mode.op_id,
check_mode.parent_op_id,
coalesce(
check_mode.input ->> 'incident_id',
check_mode.incident_id::text
) AS incident_id,
check_mode.input,
incident.alertname AS incident_alertname,
incident.alert_category AS incident_alert_category,
incident.affected_services AS incident_affected_services
FROM automation_operation_log check_mode
JOIN incidents incident
ON incident.incident_id = coalesce(
check_mode.input ->> 'incident_id',
check_mode.incident_id::text
)
AND incident.project_id = :project_id
WHERE check_mode.operation_type = 'ansible_check_mode_executed'
AND check_mode.status = 'failed'
AND upper(incident.status::text) NOT IN ('RESOLVED', 'CLOSED')
AND check_mode.created_at >= NOW() - (
:candidate_max_age_hours * INTERVAL '1 hour'
)
AND NOT EXISTS (
SELECT 1
FROM automation_operation_log replay
WHERE replay.input ->> 'replay_of_check_mode_op_id'
= check_mode.op_id::text
)
ORDER BY check_mode.created_at DESC
LIMIT :scan_limit
FOR UPDATE SKIP LOCKED
"""),
{
"project_id": project_id,
"candidate_max_age_hours": max(1, max_age_hours),
"scan_limit": 100,
},
)
ranked_rows: list[
tuple[
int,
dict[str, Any],
AnsibleCheckModeClaim,
str,
str,
]
] = []
for row_value in result.mappings().all():
row = dict(row_value)
previous_input = _json_loads(row.get("input"))
previous_check_path = str(
previous_input.get("check_mode_playbook_path")
or previous_input.get("playbook_path")
or ""
)
previous_catalog_id = str(
previous_input.get("catalog_id") or ""
)
previous_catalog_revision = str(
previous_input.get("catalog_revision") or ""
)
canonical_claim = _claim_from_stale_check_mode_row(row)
current_catalog_revision = str(
(canonical_claim.input_payload if canonical_claim else {}).get(
"catalog_revision"
)
or ""
)
path_changed = bool(
canonical_claim
and canonical_claim.playbook_path != previous_check_path
)
revision_changed = bool(
current_catalog_revision
and current_catalog_revision != previous_catalog_revision
)
if (
canonical_claim is None
or not previous_check_path
or not (path_changed or revision_changed)
):
continue
catalog_route_changed = bool(
previous_catalog_id
and canonical_claim.catalog_id != previous_catalog_id
)
replay_priority = (
0 if catalog_route_changed else 1 if path_changed else 2
)
ranked_rows.append(
(
replay_priority,
row,
canonical_claim,
previous_check_path,
previous_catalog_revision,
)
)
for (
_,
row,
canonical_claim,
previous_check_path,
previous_catalog_revision,
) in sorted(ranked_rows, key=lambda item: item[0]):
current_catalog_revision = str(
canonical_claim.input_payload.get("catalog_revision") or ""
)
catalog_replay_revision = (
current_catalog_revision
or f"path:{canonical_claim.playbook_path}"
)
replay_input = {
**canonical_claim.input_payload,
"catalog_drift_replay": True,
"catalog_replay_revision": catalog_replay_revision,
"replay_of_check_mode_op_id": str(row.get("op_id") or ""),
"previous_check_mode_playbook_path": previous_check_path,
"previous_catalog_revision": previous_catalog_revision or None,
"historical_projection_backfill": True,
"telegram_provider_delivery": "shadow_only",
}
inserted = await db.execute(
text("""
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, dry_run_result,
parent_op_id, tags
) SELECT
'ansible_check_mode_executed',
'ansible_check_mode_worker',
'pending',
:incident_db_id,
CAST(:input AS jsonb),
'{}'::jsonb,
CAST(:dry_run_result AS jsonb),
CAST(:parent_op_id AS uuid),
:tags
WHERE NOT EXISTS (
SELECT 1
FROM automation_operation_log existing_replay
WHERE existing_replay.operation_type
= 'ansible_check_mode_executed'
AND existing_replay.input ->> 'catalog_id'
= :catalog_id
AND existing_replay.input ->> 'catalog_replay_revision'
= :catalog_replay_revision
AND existing_replay.input ->> 'catalog_drift_replay'
= 'true'
AND existing_replay.input
->> 'replay_of_check_mode_op_id'
= :failed_check_mode_op_id
)
RETURNING op_id
"""),
{
"incident_db_id": _automation_operation_log_incident_id(
canonical_claim.incident_id
),
"input": json.dumps(replay_input, ensure_ascii=False),
"dry_run_result": json.dumps(
{
"check_mode_executed": False,
"apply_executed": False,
"claim_state": "catalog_drift_replay",
"controlled_apply_allowed": bool(
replay_input.get("controlled_apply_allowed")
),
"controlled_apply_blocker": replay_input.get(
"controlled_apply_blocker"
),
},
ensure_ascii=False,
),
"parent_op_id": canonical_claim.source_candidate_op_id,
"catalog_id": canonical_claim.catalog_id,
"catalog_replay_revision": catalog_replay_revision,
"failed_check_mode_op_id": str(row.get("op_id") or ""),
"tags": [
"ansible",
"check_mode",
"pending",
"catalog_drift_replay",
],
},
)
op_id_value = inserted.scalar_one_or_none()
if op_id_value is None:
continue
op_id = str(op_id_value)
claims.append(
AnsibleCheckModeClaim(
op_id=op_id,
source_candidate_op_id=canonical_claim.source_candidate_op_id,
incident_id=canonical_claim.incident_id,
catalog_id=canonical_claim.catalog_id,
playbook_path=canonical_claim.playbook_path,
apply_playbook_path=canonical_claim.apply_playbook_path,
inventory_hosts=canonical_claim.inventory_hosts,
risk_level=canonical_claim.risk_level,
input_payload=replay_input,
)
)
if len(claims) >= max(1, limit):
break
return claims
async def claim_catalog_drift_failed_applies(
*,
project_id: str = "awoooi",
limit: int = 1,
candidate_max_age_hours: int | None = None,
) -> list[AnsibleCheckModeClaim]:
"""Re-enter controlled execution once for a repaired failed apply."""
claims: list[AnsibleCheckModeClaim] = []
max_age_hours = (
candidate_max_age_hours
or settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS
)
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT
apply.op_id,
apply.parent_op_id,
coalesce(
apply.input ->> 'incident_id',
apply.incident_id::text
) AS incident_id,
apply.input
FROM automation_operation_log apply
WHERE apply.operation_type = 'ansible_apply_executed'
AND apply.status = 'failed'
AND apply.created_at >= NOW() - (
:candidate_max_age_hours * INTERVAL '1 hour'
)
ORDER BY apply.created_at DESC
LIMIT :scan_limit
FOR UPDATE OF apply SKIP LOCKED
"""),
{
"candidate_max_age_hours": max(1, max_age_hours),
"scan_limit": min(100, max(10, limit * 10)),
},
)
for row_value in result.mappings().all():
canonical_claim = _claim_from_failed_apply_catalog_repair_row(
dict(row_value)
)
if canonical_claim is None:
continue
replay_input = canonical_claim.input_payload
inserted = await db.execute(
text("""
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, dry_run_result,
parent_op_id, tags
) SELECT
'ansible_check_mode_executed',
'ansible_check_mode_worker',
'pending',
:incident_db_id,
CAST(:input AS jsonb),
'{}'::jsonb,
CAST(:dry_run_result AS jsonb),
CAST(:parent_op_id AS uuid),
:tags
WHERE NOT EXISTS (
SELECT 1
FROM automation_operation_log existing_replay
WHERE existing_replay.operation_type
= 'ansible_check_mode_executed'
AND existing_replay.input
->> 'catalog_repair_of_apply_op_id'
= :failed_apply_op_id
AND existing_replay.input
->> 'catalog_replay_revision'
= :catalog_replay_revision
)
RETURNING op_id
"""),
{
"incident_db_id": _automation_operation_log_incident_id(
canonical_claim.incident_id
),
"input": json.dumps(replay_input, ensure_ascii=False),
"dry_run_result": json.dumps(
{
"check_mode_executed": False,
"apply_executed": False,
"claim_state": "failed_apply_catalog_repair",
"controlled_apply_allowed": bool(
replay_input.get("controlled_apply_allowed")
),
"controlled_apply_blocker": replay_input.get(
"controlled_apply_blocker"
),
},
ensure_ascii=False,
),
"parent_op_id": (
canonical_claim.source_candidate_op_id
),
"failed_apply_op_id": replay_input[
"catalog_repair_of_apply_op_id"
],
"catalog_replay_revision": replay_input[
"catalog_replay_revision"
],
"tags": [
"ansible",
"check_mode",
"pending",
"failed_apply_catalog_repair",
],
},
)
op_id_value = inserted.scalar_one_or_none()
if op_id_value is None:
continue
claims.append(
replace(canonical_claim, op_id=str(op_id_value))
)
if len(claims) >= max(1, limit):
break
return claims
async def claim_semantically_misrouted_verified_applies(
*,
project_id: str = "awoooi",
limit: int = 1,
window_hours: int = _CATALOG_REPAIR_REPLAY_WINDOW_HOURS,
) -> list[AnsibleCheckModeClaim]:
"""Re-verify terminal applies whose catalog no longer matches the alert.
Resolved incidents remain eligible because this path corrects false terminal
attribution; it does not reopen incidents from an unresolved alert signal.
"""
claims: list[AnsibleCheckModeClaim] = []
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT
source_check.op_id,
source_check.parent_op_id,
incident.incident_id,
source_check.input,
incident.alertname AS incident_alertname,
incident.alert_category AS incident_alert_category,
incident.affected_services AS incident_affected_services,
apply.op_id::text AS previous_apply_op_id
FROM automation_operation_log apply
JOIN automation_operation_log source_check
ON source_check.op_id = apply.parent_op_id
AND source_check.operation_type
= 'ansible_check_mode_executed'
JOIN incidents incident
ON incident.incident_id = coalesce(
apply.input ->> 'incident_id',
apply.incident_id::text
)
AND incident.project_id = :project_id
WHERE apply.operation_type = 'ansible_apply_executed'
AND apply.status = 'success'
AND apply.output ->> 'independent_post_verifier_passed'
= 'true'
AND apply.created_at >= NOW() - (
:window_hours * INTERVAL '1 hour'
)
ORDER BY apply.created_at DESC
LIMIT :scan_limit
FOR UPDATE OF apply SKIP LOCKED
"""),
{
"project_id": project_id,
"window_hours": max(1, min(window_hours, 7 * 24)),
"scan_limit": min(100, max(10, limit * 10)),
},
)
seen_incident_ids: set[str] = set()
for row_value in result.mappings().all():
row = dict(row_value)
incident_id = str(row.get("incident_id") or "")
if not incident_id or incident_id in seen_incident_ids:
continue
seen_incident_ids.add(incident_id)
previous_input = _json_loads(row.get("input"))
previous_catalog_id = str(
previous_input.get("catalog_id") or ""
)
canonical_claim = _claim_from_stale_check_mode_row(row)
if (
canonical_claim is None
or not previous_catalog_id
or canonical_claim.catalog_id == previous_catalog_id
):
continue
current_revision = str(
canonical_claim.input_payload.get("catalog_revision") or ""
)
previous_apply_op_id = str(
row.get("previous_apply_op_id") or ""
)
if not current_revision or not previous_apply_op_id:
continue
semantic_repair_run_id = str(
uuid5(
NAMESPACE_URL,
(
"awoooi:ansible-semantic-repair:"
f"{project_id}:{previous_apply_op_id}:"
f"{current_revision}"
),
)
)
replay_input = {
**canonical_claim.input_payload,
"automation_run_id": semantic_repair_run_id,
"semantic_route_reconciliation": True,
"semantic_repair_of_apply_op_id": previous_apply_op_id,
"catalog_replay_revision": current_revision,
"historical_projection_backfill": True,
"telegram_provider_delivery": "shadow_only",
"apply_idempotency_key": (
"ansible-semantic-repair:"
f"{previous_apply_op_id}:{current_revision}"
),
"bounded_execution_scope": (
"semantic_catalog_reconciliation_single_apply"
),
}
inserted = await db.execute(
text("""
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, dry_run_result,
parent_op_id, tags
) SELECT
'ansible_check_mode_executed',
'ansible_check_mode_worker',
'pending',
:incident_db_id,
CAST(:input AS jsonb),
'{}'::jsonb,
CAST(:dry_run_result AS jsonb),
CAST(:parent_op_id AS uuid),
:tags
WHERE NOT EXISTS (
SELECT 1
FROM automation_operation_log existing_replay
WHERE existing_replay.operation_type
= 'ansible_check_mode_executed'
AND existing_replay.input
->> 'semantic_repair_of_apply_op_id'
= :previous_apply_op_id
AND existing_replay.input
->> 'catalog_replay_revision'
= :catalog_replay_revision
)
RETURNING op_id
"""),
{
"incident_db_id": _automation_operation_log_incident_id(
canonical_claim.incident_id
),
"input": json.dumps(replay_input, ensure_ascii=False),
"dry_run_result": json.dumps(
{
"check_mode_executed": False,
"apply_executed": False,
"claim_state": (
"semantic_catalog_reconciliation"
),
"controlled_apply_allowed": bool(
replay_input.get("controlled_apply_allowed")
),
},
ensure_ascii=False,
),
"parent_op_id": (
canonical_claim.source_candidate_op_id
),
"previous_apply_op_id": previous_apply_op_id,
"catalog_replay_revision": current_revision,
"tags": [
"ansible",
"check_mode",
"pending",
"semantic_catalog_reconciliation",
],
},
)
op_id = inserted.scalar_one_or_none()
if op_id is None:
continue
claims.append(
AnsibleCheckModeClaim(
op_id=str(op_id),
source_candidate_op_id=(
canonical_claim.source_candidate_op_id
),
incident_id=canonical_claim.incident_id,
catalog_id=canonical_claim.catalog_id,
playbook_path=canonical_claim.playbook_path,
apply_playbook_path=(
canonical_claim.apply_playbook_path
),
inventory_hosts=canonical_claim.inventory_hosts,
risk_level=canonical_claim.risk_level,
input_payload=replay_input,
)
)
if len(claims) >= max(1, limit):
break
return claims
async def recent_ansible_transport_blockers(
*,
project_id: str = "awoooi",
cooldown_seconds: int | None = None,
include_repair_forced_command_blocker: bool | None = None,
) -> list[str]:
"""Return transport blockers observed from recent failed check-mode rows."""
cooldown = cooldown_seconds or settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_COOLDOWN_SECONDS
include_forced_blocker = (
_uses_repair_forced_command_transport()
if include_repair_forced_command_blocker is None
else include_repair_forced_command_blocker
)
async with get_db_context(project_id) as db:
result = await db.execute(
text("""
SELECT
coalesce(output::text, '') AS output_text,
coalesce(dry_run_result::text, '') AS dry_run_text,
coalesce(error, '') AS error_text,
coalesce(stderr_feed_back, '') AS stderr_text
FROM automation_operation_log
WHERE operation_type = 'ansible_check_mode_executed'
AND status = 'failed'
AND created_at >= NOW() - (:cooldown_seconds * INTERVAL '1 second')
ORDER BY created_at DESC
LIMIT 20
"""),
{"cooldown_seconds": max(60, cooldown)},
)
blockers: set[str] = set()
for row in result.mappings().all():
detected = detect_ansible_transport_blockers(
row.get("output_text"),
row.get("dry_run_text"),
row.get("error_text"),
row.get("stderr_text"),
)
blockers.update(
blocker
for blocker in detected
if blocker != FORCED_COMMAND_BLOCKER or include_forced_blocker
)
return sorted(blockers)
async def _insert_skipped_candidate(
db: Any,
*,
source_candidate_op_id: str,
candidate_input: dict[str, Any],
reason: str,
) -> None:
trust_gate = candidate_input.get("playbook_trust_gate")
if not isinstance(trust_gate, dict):
trust_gate = {}
input_payload = {
"automation_run_id": str(source_candidate_op_id),
"incident_id": _incident_id_from_payload(candidate_input),
"executor": "ansible",
"execution_backend": "ansible",
"execution_mode": "check_mode",
"transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
"check_mode": True,
"apply_enabled": False,
"source_candidate_op_id": source_candidate_op_id,
"not_used_reason": reason,
"playbook_trust_gate": trust_gate,
}
await db.execute(
text("""
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, dry_run_result,
parent_op_id, tags
) VALUES (
'ansible_execution_skipped',
'ansible_check_mode_worker',
'dry_run',
:incident_db_id,
CAST(:input AS jsonb),
CAST(:output AS jsonb),
CAST(:dry_run_result AS jsonb),
CAST(:parent_op_id AS uuid),
:tags
)
"""),
{
"incident_id": _incident_id_from_payload(input_payload),
"incident_db_id": _automation_operation_log_incident_id(
_incident_id_from_payload(input_payload)
),
"input": json.dumps(input_payload, ensure_ascii=False),
"output": json.dumps({
"not_used_reason": reason,
"decision_effect": "skipped_before_runtime",
"next_controlled_action": (
"generate_or_verify_playbook_repair_candidate"
if reason == "playbook_trust_circuit_open"
else "repair_candidate_input_before_retry"
),
"playbook_trust_gate": trust_gate,
}, ensure_ascii=False),
"dry_run_result": json.dumps({
"check_mode_executed": False,
"apply_executed": False,
"skipped": True,
"reason": reason,
"playbook_trust_gate": trust_gate,
}, ensure_ascii=False),
"parent_op_id": source_candidate_op_id,
"tags": ["ansible", "check_mode", "skipped", "apply_locked"],
},
)
async def finalize_check_mode_claim(
claim: AnsibleCheckModeClaim,
result: AnsibleRunResult,
*,
project_id: str = "awoooi",
) -> None:
controlled_apply_allowed = bool(claim.input_payload.get("controlled_apply_allowed"))
controlled_apply_blocker = (
str(claim.input_payload.get("controlled_apply_blocker"))
if claim.input_payload.get("controlled_apply_blocker")
else None
)
status, output, dry_run_result, error = _build_result_payload(
result,
controlled_apply_allowed=controlled_apply_allowed,
controlled_apply_blocker=controlled_apply_blocker,
)
async with get_db_context(project_id) as db:
await db.execute(
text("""
UPDATE automation_operation_log
SET status = :status,
input = jsonb_set(
jsonb_set(
coalesce(input, '{}'::jsonb),
'{automation_run_id}',
to_jsonb(CAST(:automation_run_id AS text)),
true
),
'{execution_capability}',
CAST(:execution_capability AS jsonb),
true
),
output = CAST(:output AS jsonb),
dry_run_result = CAST(:dry_run_result AS jsonb),
error = :error,
duration_ms = :duration_ms,
stderr_feed_back = :stderr
WHERE op_id = CAST(:op_id AS uuid)
"""),
{
"status": status,
"automation_run_id": str(
claim.input_payload.get("automation_run_id")
or claim.source_candidate_op_id
),
"execution_capability": json.dumps(
claim.input_payload.get("execution_capability") or {},
ensure_ascii=False,
),
"output": json.dumps(output, ensure_ascii=False),
"dry_run_result": json.dumps(dry_run_result, ensure_ascii=False),
"error": _tail(error or "", 2000) or None,
"duration_ms": result.duration_ms,
"stderr": _tail(result.stderr, _STDERR_LIMIT),
"op_id": claim.op_id,
},
)
async def run_claimed_check_mode(
claim: AnsibleCheckModeClaim,
*,
timeout_seconds: int,
project_id: str = "awoooi",
) -> AnsibleRunResult:
try:
bounded_timeout_seconds = _execution_capability_timeout_seconds(
claim,
execution_mode="check_mode",
requested_timeout_seconds=timeout_seconds,
)
spec = build_ansible_check_mode_command(
playbook_path=claim.playbook_path,
inventory_hosts=claim.inventory_hosts,
)
result = await _run_ansible_command(
spec,
timeout_seconds=bounded_timeout_seconds,
)
except Exception as exc:
result = AnsibleRunResult(
returncode=1,
stdout="",
stderr=f"ansible_check_mode_runtime_error: {exc}",
duration_ms=0,
)
await finalize_check_mode_claim(claim, result, project_id=project_id)
logger.info(
"ansible_check_mode_claim_completed",
op_id=claim.op_id,
source_candidate_op_id=claim.source_candidate_op_id,
incident_id=claim.incident_id,
catalog_id=claim.catalog_id,
returncode=result.returncode,
timed_out=result.timed_out,
)
return result
async def run_controlled_apply_for_claim(
claim: AnsibleCheckModeClaim,
*,
timeout_seconds: int,
project_id: str = "awoooi",
) -> AnsibleRunResult | None:
"""Execute apply for an allowlisted claim after check-mode succeeds."""
if claim.input_payload.get("controlled_apply_allowed") is not True:
logger.info(
"ansible_controlled_apply_blocked",
op_id=claim.op_id,
source_candidate_op_id=claim.source_candidate_op_id,
incident_id=claim.incident_id,
catalog_id=claim.catalog_id,
blocker=claim.input_payload.get("controlled_apply_blocker"),
)
return None
bounded_timeout_seconds = _execution_capability_timeout_seconds(
claim,
execution_mode="controlled_apply",
requested_timeout_seconds=timeout_seconds,
)
apply_idempotency_key = str(
claim.input_payload.get("apply_idempotency_key")
or f"ansible-apply:{claim.source_candidate_op_id}:{claim.catalog_id}"
)
async with get_db_context(project_id) as db:
await db.execute(
text("""
SELECT pg_advisory_xact_lock(
hashtextextended(:idempotency_key, 0)
)
"""),
{"idempotency_key": apply_idempotency_key},
)
existing = await db.execute(
text("""
SELECT op_id, status
FROM automation_operation_log
WHERE operation_type = 'ansible_apply_executed'
AND parent_op_id = CAST(:parent_op_id AS uuid)
ORDER BY created_at ASC
LIMIT 1
"""),
{"parent_op_id": claim.op_id},
)
existing_row = existing.mappings().first()
if existing_row is not None:
claim.input_payload["apply_duplicate_suppressed"] = True
claim.input_payload["existing_apply_op_id"] = str(
existing_row.get("op_id") or ""
)
claim.input_payload["existing_apply_status"] = str(
existing_row.get("status") or ""
)
logger.warning(
"ansible_controlled_apply_duplicate_suppressed",
source_candidate_op_id=claim.source_candidate_op_id,
check_mode_op_id=claim.op_id,
existing_apply_op_id=claim.input_payload["existing_apply_op_id"],
existing_apply_status=claim.input_payload["existing_apply_status"],
idempotency_key=apply_idempotency_key,
)
return None
if not await _revalidate_claim_playbook_trust(db, claim):
logger.warning(
"ansible_controlled_apply_blocked_by_playbook_trust",
op_id=claim.op_id,
source_candidate_op_id=claim.source_candidate_op_id,
incident_id=claim.incident_id,
catalog_id=claim.catalog_id,
blocker="playbook_trust_circuit_open",
)
return None
inserted = await db.execute(
text("""
INSERT INTO automation_operation_log (
operation_type, actor, status, incident_id,
input, output, dry_run_result,
parent_op_id, tags
) VALUES (
'ansible_apply_executed',
'ansible_controlled_apply_worker',
'pending',
:incident_db_id,
CAST(:input AS jsonb),
'{}'::jsonb,
CAST(:dry_run_result AS jsonb),
CAST(:parent_op_id AS uuid),
:tags
)
RETURNING op_id
"""),
{
"incident_id": claim.incident_id,
"incident_db_id": _automation_operation_log_incident_id(claim.incident_id),
"input": json.dumps({
**claim.input_payload,
"automation_run_id": str(
claim.input_payload.get("automation_run_id")
or claim.source_candidate_op_id
),
"execution_mode": "controlled_apply",
"check_mode_op_id": claim.op_id,
"playbook_path": claim.apply_playbook_path,
"check_mode_playbook_path": claim.playbook_path,
"apply_playbook_path": claim.apply_playbook_path,
"check_mode": False,
"apply_enabled": True,
"approval_required_before_apply": False,
"controlled_apply_allowed": True,
"apply_idempotency_key": apply_idempotency_key,
"single_writer_executor": "awoooi-ansible-executor-broker",
"router_source_sha": str(
claim.input_payload.get("router_source_sha") or ""
)
.strip()
.lower(),
"runtime_stage_receipts": (
build_ansible_pre_apply_runtime_stage_receipts(claim)
),
}, ensure_ascii=False),
"dry_run_result": json.dumps({
"check_mode_executed_before_apply": True,
"apply_executed": False,
"claim_state": "claimed",
"source_check_mode_op_id": claim.op_id,
}, ensure_ascii=False),
"parent_op_id": claim.op_id,
"tags": [
"ansible",
"controlled_apply",
str(claim.risk_level or "unknown").lower(),
"ai_agent_auto_execution",
],
},
)
apply_op_id = str(inserted.scalar_one())
lifecycle_started = await _append_alert_lifecycle_receipt(
claim,
"EXECUTION_STARTED",
apply_op_id=apply_op_id,
success=True,
action_detail=f"controlled_apply_started:{claim.catalog_id}",
project_id=project_id,
)
cancelled = False
if not lifecycle_started:
result = AnsibleRunResult(
returncode=1,
stdout="",
stderr="alert_lifecycle_start_receipt_not_persisted",
duration_ms=0,
)
else:
try:
spec = build_ansible_apply_command(
playbook_path=claim.apply_playbook_path,
inventory_hosts=claim.inventory_hosts,
)
result = await _run_ansible_command(
spec,
timeout_seconds=bounded_timeout_seconds,
)
except asyncio.CancelledError:
cancelled = True
result = AnsibleRunResult(
returncode=130,
stdout="",
stderr="ansible_controlled_apply_interrupted_by_worker_shutdown",
duration_ms=0,
)
except Exception as exc:
result = AnsibleRunResult(
returncode=1,
stdout="",
stderr=f"ansible_controlled_apply_runtime_error: {exc}",
duration_ms=0,
)
status, output, dry_run_result, error = _build_apply_result_payload(result)
async with get_db_context(project_id) as db:
await db.execute(
text("""
UPDATE automation_operation_log
SET status = :status,
output = CAST(:output AS jsonb),
dry_run_result = CAST(:dry_run_result AS jsonb),
error = :error,
duration_ms = :duration_ms,
stderr_feed_back = :stderr
WHERE op_id = CAST(:op_id AS uuid)
"""),
{
"status": status,
"output": json.dumps(output, ensure_ascii=False),
"dry_run_result": json.dumps(dry_run_result, ensure_ascii=False),
"error": _tail(error or "", 2000) or None,
"duration_ms": result.duration_ms,
"stderr": _tail(result.stderr, _STDERR_LIMIT),
"op_id": apply_op_id,
},
)
if cancelled:
logger.warning(
"ansible_controlled_apply_interrupted",
op_id=apply_op_id,
check_mode_op_id=claim.op_id,
source_candidate_op_id=claim.source_candidate_op_id,
incident_id=claim.incident_id,
catalog_id=claim.catalog_id,
)
raise asyncio.CancelledError
writeback = await _record_post_apply_verifier_and_learning(
claim,
result,
apply_op_id=apply_op_id,
project_id=project_id,
)
verified_result = replace(
result,
post_verifier_passed=(writeback.get("verification_passed") is True),
)
receipt_written = await _record_auto_repair_execution_receipt(
claim,
verified_result,
apply_op_id=apply_op_id,
project_id=project_id,
)
timeline_receipt = await _record_timeline_projection_receipt(
claim,
verified_result,
apply_op_id=apply_op_id,
project_id=project_id,
)
learning_receipts = tuple(
receipt
for receipt in writeback.get("runtime_stage_receipts", [])
if isinstance(receipt, dict)
)
runtime_stage_receipts_written = await _record_runtime_stage_receipts(
claim,
result,
apply_op_id=apply_op_id,
verifier_ready=bool(writeback.get("verification_passed")),
project_id=project_id,
extra_receipts=(
*((timeline_receipt,) if timeline_receipt else ()),
*learning_receipts,
),
)
approval_projection_written = await _finalize_controlled_approval_projection(
claim,
verified_result,
apply_op_id=apply_op_id,
project_id=project_id,
)
execution_success = bool(
verified_result.returncode == 0
and verified_result.post_verifier_passed is True
)
lifecycle_completed = await _append_alert_lifecycle_receipt(
claim,
"EXECUTION_COMPLETED",
apply_op_id=apply_op_id,
success=execution_success,
action_detail=f"controlled_apply_completed:{claim.catalog_id}",
project_id=project_id,
error_message=(
None
if execution_success
else _tail(
verified_result.stderr or "independent_post_verifier_failed",
2000,
)
),
post_verifier_passed=verified_result.post_verifier_passed,
)
# Single dispatch entry: `_send_controlled_apply_telegram_receipt` and the
# `"TELEGRAM_RESULT_SENT"` lifecycle are owned by closure reconciliation.
try:
closure = await _reconcile_verified_apply_closure_projections(
claim,
verified_result,
apply_op_id=apply_op_id,
writeback=writeback,
project_id=project_id,
)
except Exception as exc:
closure = {
"status": "closure_reconcile_retry_required",
"closed": False,
"missing": [f"closure_reconcile:{type(exc).__name__}"],
}
logger.warning(
"ansible_verified_apply_closure_reconcile_failed",
automation_run_id=_automation_run_id_for_claim(claim),
incident_id=claim.incident_id,
apply_op_id=apply_op_id,
error_type=type(exc).__name__,
runtime_apply_replay_required=False,
)
telegram_receipt_sent = (
closure.get("telegram_receipt_acknowledged") is True
)
lifecycle_telegram_written = (
closure.get("telegram_lifecycle_written") is True
)
logger.info(
"ansible_controlled_apply_completed",
op_id=apply_op_id,
check_mode_op_id=claim.op_id,
source_candidate_op_id=claim.source_candidate_op_id,
incident_id=claim.incident_id,
catalog_id=claim.catalog_id,
returncode=result.returncode,
timed_out=result.timed_out,
auto_repair_receipt_written=receipt_written,
post_apply_verification_written=writeback.get("verification"),
post_apply_verification_passed=writeback.get("verification_passed"),
post_apply_verification_result=writeback.get("verification_result"),
post_apply_learning_written=writeback.get("learning"),
rag_writeback_written=writeback.get("rag_writeback"),
playbook_trust_written=writeback.get("trust_learning"),
timeline_projection_written=timeline_receipt is not None,
runtime_stage_receipts_written=runtime_stage_receipts_written,
approval_projection_written=approval_projection_written,
alert_lifecycle_started=lifecycle_started,
alert_lifecycle_completed=lifecycle_completed,
telegram_receipt_sent=telegram_receipt_sent,
alert_lifecycle_telegram_written=lifecycle_telegram_written,
incident_closure_status=closure.get("status"),
incident_closure_readback=closure.get("closed") is True,
incident_closure_missing=closure.get("missing") or [],
)
return verified_result
async def run_pending_check_modes_once(
*,
project_id: str = "awoooi",
limit: int = 1,
timeout_seconds: int | None = None,
) -> dict[str, Any]:
expired_capability_count = await _expire_stale_ansible_execution_capabilities(
project_id=project_id,
)
effective_timeout_seconds = (
timeout_seconds or settings.AWOOOP_ANSIBLE_CHECK_MODE_TIMEOUT_SECONDS
)
stdin_projection_stats = (
await backfill_missing_stdin_boundary_terminal_projections_once(
project_id=project_id,
window_hours=_STDIN_BOUNDARY_REPLAY_WINDOW_HOURS,
limit=_STDIN_BOUNDARY_TERMINAL_PROJECTION_BATCH_LIMIT,
)
)
stdin_projection_summary = {
"stdin_terminal_projection_scanned": int(
stdin_projection_stats.get("scanned") or 0
),
"stdin_terminal_projection_written": int(
stdin_projection_stats.get("written") or 0
),
"stdin_terminal_projection_learning_written": int(
stdin_projection_stats.get("learning_written") or 0
),
"stdin_terminal_projection_telegram_acknowledged": int(
stdin_projection_stats.get("telegram_receipt_acknowledged") or 0
),
"stdin_terminal_projection_incident_receipt_written": int(
stdin_projection_stats.get("incident_receipt_written") or 0
),
"stdin_terminal_projection_verified": int(
stdin_projection_stats.get("verified") or 0
),
"stdin_terminal_projection_blockers": [
str(item)[:300]
for item in stdin_projection_stats.get("blockers") or []
],
"stdin_terminal_projection_error": (
str(stdin_projection_stats.get("error") or "")[:500] or None
),
"stdin_terminal_projection_runtime_apply_executed": False,
}
if (
stdin_projection_summary["stdin_terminal_projection_scanned"]
or stdin_projection_summary["stdin_terminal_projection_blockers"]
or stdin_projection_summary["stdin_terminal_projection_error"]
):
logger.info(
"ansible_stdin_boundary_projection_priority_tick",
project_id=project_id,
**stdin_projection_summary,
)
receipt_stats = await backfill_missing_auto_repair_execution_receipts_once(
project_id=project_id,
window_hours=24,
limit=max(1, min(int(limit), 5)),
)
receipt_backfill_summary = {
"repair_receipt_backfill_scanned": int(
receipt_stats.get("scanned") or 0
),
"repair_receipt_backfill_written": int(
receipt_stats.get("written") or 0
),
"repair_receipt_closure_written": int(
receipt_stats.get("incident_closure_written") or 0
),
"repair_receipt_telegram_acknowledged": int(
receipt_stats.get("telegram_receipt_acknowledged") or 0
),
"retry_terminal_projection_scanned": int(
receipt_stats.get("retry_terminal_projection_scanned") or 0
),
"retry_terminal_projection_written": int(
receipt_stats.get("retry_terminal_projection_written") or 0
),
"retry_terminal_projection_retry_receipt_written": int(
receipt_stats.get(
"retry_terminal_projection_retry_receipt_written"
)
or 0
),
"retry_terminal_projection_telegram_receipt_acknowledged": int(
receipt_stats.get(
"retry_terminal_projection_telegram_receipt_acknowledged"
)
or 0
),
"retry_terminal_projection_verified": int(
receipt_stats.get("retry_terminal_projection_verified") or 0
),
"retry_terminal_projection_runtime_apply_executed": False,
"repair_receipt_backfill_error": (
str(receipt_stats.get("error") or "")[:500] or None
),
}
receipt_backfill_priority_tick = bool(
receipt_backfill_summary["repair_receipt_backfill_scanned"]
or receipt_backfill_summary["repair_receipt_backfill_error"]
)
receipt_backfill_summary["repair_receipt_backfill_priority_tick"] = (
receipt_backfill_priority_tick
)
if receipt_backfill_priority_tick:
logger.info(
"ansible_execution_broker_receipt_backfill_priority_tick",
project_id=project_id,
**receipt_backfill_summary,
)
retry_stats = await run_failed_apply_check_mode_replay_once(
project_id=project_id,
window_hours=24,
timeout_seconds=effective_timeout_seconds,
)
failed_apply_retry_blockers = [
str(item)[:200]
for item in retry_stats.get("blockers") or []
]
failed_apply_retry_error = (
str(retry_stats.get("error") or "")[:500] or None
)
failed_apply_retry_summary = {
"failed_apply_retry_scanned": int(retry_stats.get("scanned") or 0),
"failed_apply_retry_replayed": int(retry_stats.get("replayed") or 0),
"failed_apply_retry_check_mode_passed": int(
retry_stats.get("check_mode_passed") or 0
),
"failed_apply_retry_check_mode_failed": int(
retry_stats.get("check_mode_failed") or 0
),
"failed_apply_retry_stage_receipt_written": int(
retry_stats.get("runtime_stage_receipt_written") or 0
),
"failed_apply_retry_stale_reclaimed": int(
retry_stats.get("stale_retry_reclaimed") or 0
),
"failed_apply_retry_stale_terminalized": int(
retry_stats.get("stale_retry_terminalized") or 0
),
"failed_apply_retry_blockers": failed_apply_retry_blockers,
"failed_apply_retry_error": failed_apply_retry_error,
}
failed_apply_retry_priority_tick = bool(
failed_apply_retry_summary["failed_apply_retry_scanned"]
or failed_apply_retry_summary["failed_apply_retry_replayed"]
or failed_apply_retry_blockers
or failed_apply_retry_error
)
failed_apply_retry_summary["failed_apply_retry_priority_tick"] = (
failed_apply_retry_priority_tick
)
if failed_apply_retry_priority_tick:
logger.info(
"ansible_execution_broker_failed_apply_retry_priority_tick",
project_id=project_id,
**failed_apply_retry_summary,
)
blockers = _runtime_blockers()
if blockers:
logger.warning("ansible_check_mode_runtime_blocked", blockers=blockers)
return {
"claimed": 0,
"completed": 0,
"failed": 0,
"capability_expired": expired_capability_count,
"blockers": blockers,
**stdin_projection_summary,
**receipt_backfill_summary,
**failed_apply_retry_summary,
}
transport_blockers = await recent_ansible_transport_blockers(project_id=project_id)
if transport_blockers:
logger.warning("ansible_check_mode_transport_blocked", blockers=transport_blockers)
return {
"claimed": 0,
"completed": 0,
"failed": 0,
"capability_expired": expired_capability_count,
"blockers": transport_blockers,
**stdin_projection_summary,
**receipt_backfill_summary,
**failed_apply_retry_summary,
}
reclaimed_claims = await claim_stale_pending_check_modes(
project_id=project_id,
limit=limit,
stale_after_seconds=max(300, effective_timeout_seconds + 120),
)
remaining_limit = max(0, limit - len(reclaimed_claims))
stdin_boundary_replay_claims: list[AnsibleCheckModeClaim] = []
stdin_boundary_replay_error: str | None = None
if remaining_limit:
try:
stdin_boundary_replay_claims = (
await claim_stdin_boundary_failed_check_modes(
project_id=project_id,
limit=remaining_limit,
)
)
except Exception as exc:
stdin_boundary_replay_error = type(exc).__name__
logger.warning(
"ansible_stdin_boundary_replay_claim_failed",
project_id=project_id,
error_type=stdin_boundary_replay_error,
fresh_candidate_claim_continues=True,
)
remaining_limit = max(
0,
remaining_limit - len(stdin_boundary_replay_claims),
)
semantic_catalog_reconciliation_claims: list[
AnsibleCheckModeClaim
] = []
semantic_catalog_reconciliation_error: str | None = None
if remaining_limit:
try:
semantic_catalog_reconciliation_claims = (
await claim_semantically_misrouted_verified_applies(
project_id=project_id,
limit=remaining_limit,
)
)
except Exception as exc:
semantic_catalog_reconciliation_error = type(exc).__name__
logger.warning(
"ansible_semantic_catalog_reconciliation_claim_failed",
project_id=project_id,
error_type=semantic_catalog_reconciliation_error,
fresh_candidate_claim_continues=True,
)
remaining_limit = max(
0,
remaining_limit - len(semantic_catalog_reconciliation_claims),
)
failed_apply_catalog_replay_claims: list[AnsibleCheckModeClaim] = []
failed_apply_catalog_replay_error: str | None = None
if remaining_limit:
try:
failed_apply_catalog_replay_claims = (
await claim_catalog_drift_failed_applies(
project_id=project_id,
limit=remaining_limit,
candidate_max_age_hours=(
settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS
),
)
)
except Exception as exc:
failed_apply_catalog_replay_error = type(exc).__name__
logger.warning(
"ansible_failed_apply_catalog_repair_claim_failed",
project_id=project_id,
error_type=failed_apply_catalog_replay_error,
fresh_candidate_claim_continues=True,
)
remaining_limit = max(
0,
remaining_limit - len(failed_apply_catalog_replay_claims),
)
catalog_replay_claims: list[AnsibleCheckModeClaim] = []
catalog_replay_error: str | None = None
if remaining_limit:
try:
catalog_replay_claims = await claim_catalog_drift_failed_check_modes(
project_id=project_id,
limit=remaining_limit,
candidate_max_age_hours=(
settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS
),
)
except Exception as exc:
catalog_replay_error = type(exc).__name__
logger.warning(
"ansible_catalog_drift_replay_claim_failed",
project_id=project_id,
error_type=catalog_replay_error,
fresh_candidate_claim_continues=True,
)
remaining_limit = max(0, remaining_limit - len(catalog_replay_claims))
fresh_claims = (
await claim_pending_check_modes(
project_id=project_id,
limit=remaining_limit,
candidate_max_age_hours=(
settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS
),
)
if remaining_limit
else []
)
claims = [
*reclaimed_claims,
*stdin_boundary_replay_claims,
*semantic_catalog_reconciliation_claims,
*failed_apply_catalog_replay_claims,
*catalog_replay_claims,
*fresh_claims,
]
completed = 0
failed = 0
apply_completed = 0
apply_failed = 0
apply_blocked = 0
apply_duplicate_suppressed = 0
capability_issued = 0
capability_revoked = 0
capability_revoke_failed = 0
stdin_boundary_replay_terminal_written = 0
for claim in claims:
capability_op_id = ""
terminal_status = "broker_interrupted_before_execution"
try:
claim, capability_op_id = await _issue_ansible_execution_capability(
claim,
ttl_seconds=(
effective_timeout_seconds
+ settings.AWOOOP_ANSIBLE_CONTROLLED_APPLY_TIMEOUT_SECONDS
+ 120
),
project_id=project_id,
)
capability_issued += 1
result = await run_claimed_check_mode(
claim,
timeout_seconds=effective_timeout_seconds,
project_id=project_id,
)
completed += 1
if claim.input_payload.get(
"historical_stdin_boundary_replay"
) is True:
terminal_written = (
await _record_stdin_boundary_replay_terminal(
claim,
result,
project_id=project_id,
)
)
stdin_boundary_replay_terminal_written += int(
terminal_written
)
apply_blocked += 1
if result.returncode != 0:
failed += 1
terminal_status = (
"stdin_boundary_replay_failed_no_write_terminal"
if terminal_written
else "stdin_boundary_replay_failed_terminal_missing"
)
else:
terminal_status = (
"stdin_boundary_replay_passed_no_write_terminal"
if terminal_written
else "stdin_boundary_replay_passed_terminal_missing"
)
continue
if result.returncode != 0:
failed += 1
terminal_status = "check_mode_failed"
continue
apply_result = await run_controlled_apply_for_claim(
claim,
timeout_seconds=settings.AWOOOP_ANSIBLE_CONTROLLED_APPLY_TIMEOUT_SECONDS,
project_id=project_id,
)
if apply_result is None:
if claim.input_payload.get("apply_duplicate_suppressed") is True:
apply_duplicate_suppressed += 1
terminal_status = "controlled_apply_duplicate_suppressed"
else:
apply_blocked += 1
terminal_status = "check_mode_passed_apply_blocked_by_policy"
else:
apply_completed += 1
if apply_result.returncode != 0:
apply_failed += 1
terminal_status = "controlled_apply_failed"
elif apply_result.post_verifier_passed is not True:
apply_failed += 1
terminal_status = "controlled_apply_verifier_failed"
else:
terminal_status = "controlled_apply_verified"
except asyncio.CancelledError:
terminal_status = "broker_shutdown"
raise
except ValueError as exc:
failed += 1
terminal_status = f"broker_policy_error:{exc}"
if not capability_op_id:
await finalize_check_mode_claim(
claim,
AnsibleRunResult(
returncode=1,
stdout="",
stderr=f"ansible_execution_capability_rejected: {exc}",
duration_ms=0,
),
project_id=project_id,
)
completed += 1
logger.warning(
"ansible_execution_broker_claim_rejected",
op_id=claim.op_id,
automation_run_id=_automation_run_id_for_claim(claim),
blocker=str(exc),
)
except Exception as exc:
failed += 1
terminal_status = f"broker_error:{type(exc).__name__}"
logger.warning(
"ansible_execution_broker_claim_failed",
op_id=claim.op_id,
automation_run_id=_automation_run_id_for_claim(claim),
error_type=type(exc).__name__,
)
if not capability_op_id:
raise
finally:
if capability_op_id:
try:
revoked = await _revoke_ansible_execution_capability(
claim,
capability_op_id=capability_op_id,
terminal_status=terminal_status,
project_id=project_id,
)
capability_revoked += 1 if revoked else 0
except Exception as exc:
capability_revoke_failed += 1
logger.warning(
"ansible_execution_capability_revoke_failed",
capability_op_id=capability_op_id,
automation_run_id=_automation_run_id_for_claim(claim),
error_type=type(exc).__name__,
)
return {
"claimed": len(claims),
"reclaimed": len(reclaimed_claims),
"stdin_boundary_replayed": len(stdin_boundary_replay_claims),
"stdin_boundary_replay_terminal_written": (
stdin_boundary_replay_terminal_written
),
"stdin_boundary_replay_error": stdin_boundary_replay_error,
"semantic_catalog_reconciled": len(
semantic_catalog_reconciliation_claims
),
"semantic_catalog_reconciliation_error": (
semantic_catalog_reconciliation_error
),
"failed_apply_catalog_replayed": len(
failed_apply_catalog_replay_claims
),
"failed_apply_catalog_replay_error": (
failed_apply_catalog_replay_error
),
"catalog_replayed": len(catalog_replay_claims),
"completed": completed,
"failed": failed,
"apply_completed": apply_completed,
"apply_failed": apply_failed,
"apply_blocked": apply_blocked,
"apply_duplicate_suppressed": apply_duplicate_suppressed,
"capability_issued": capability_issued,
"capability_revoked": capability_revoked,
"capability_expired": expired_capability_count,
"capability_revoke_failed": capability_revoke_failed,
"catalog_replay_error": catalog_replay_error,
**stdin_projection_summary,
**receipt_backfill_summary,
**failed_apply_retry_summary,
"blockers": [],
}
Reference in New Issue View Git Blame Copy Permalink