fix(iwooos): retry incomplete wazuh closure

apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py

@@ -12,7 +12,7 @@ import asyncio
 import json
 import os
 import random
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable, Callable, Mapping
 from types import SimpleNamespace
 from typing import Any
 from uuid import NAMESPACE_URL, uuid4, uuid5
@@ -432,6 +432,8 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
                         candidate.input ->> 'automation_run_id' AS automation_run_id,
                         candidate.input ->> 'source_receipt_ref' AS source_receipt_ref,
                         check_mode.status AS latest_check_status,
+                        apply.status AS latest_apply_status,
+                        learning.status AS latest_learning_status,
                         EXISTS (
                             SELECT 1
                             FROM incident_evidence evidence
@@ -456,13 +458,29 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
                         ) AS predecision_evidence_ready
                     FROM automation_operation_log candidate
                     LEFT JOIN LATERAL (
-                        SELECT check_receipt.status
+                        SELECT check_receipt.op_id, check_receipt.status
                         FROM automation_operation_log check_receipt
                         WHERE check_receipt.parent_op_id = candidate.op_id
                           AND check_receipt.operation_type = 'ansible_check_mode_executed'
                         ORDER BY check_receipt.created_at DESC
                         LIMIT 1
                     ) check_mode ON TRUE
+                    LEFT JOIN LATERAL (
+                        SELECT apply_receipt.op_id, apply_receipt.status
+                        FROM automation_operation_log apply_receipt
+                        WHERE apply_receipt.parent_op_id = check_mode.op_id
+                          AND apply_receipt.operation_type = 'ansible_apply_executed'
+                        ORDER BY apply_receipt.created_at DESC
+                        LIMIT 1
+                    ) apply ON TRUE
+                    LEFT JOIN LATERAL (
+                        SELECT learning_receipt.status
+                        FROM automation_operation_log learning_receipt
+                        WHERE learning_receipt.parent_op_id = apply.op_id
+                          AND learning_receipt.operation_type = 'ansible_learning_writeback_recorded'
+                        ORDER BY learning_receipt.created_at DESC
+                        LIMIT 1
+                    ) learning ON TRUE
                     WHERE candidate.operation_type = 'ansible_candidate_matched'
                       AND candidate.input -> 'executor_candidates' @> CAST(:catalog_match AS jsonb)
                       AND candidate.created_at >= NOW() - (:freshness_hours * INTERVAL '1 hour')
@@ -485,9 +503,10 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
             existing_row.get("latest_check_status") or ""
         ) if existing_row else ""
         retry_attempt_ref: str | None = None
-        retry_reason: str | None = None
+        retry_reason = _wazuh_posture_retry_reason(existing_row)
         if (
             existing_row
+            and retry_reason is None
             and latest_check_status != "failed"
             and existing_evidence_ready
         ):
@@ -512,9 +531,12 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
                 and all(character in "0123456789abcdef" for character in source_sha)
                 else "runtime-retry"
             )
-            if latest_check_status == "failed":
+            if retry_reason == "check_mode_failed":
                 retry_attempt_ref = deploy_ref
-                retry_reason = "failed_check_mode"
+            elif retry_reason == "controlled_apply_failed":
+                retry_attempt_ref = f"{deploy_ref}-apply"
+            elif retry_reason == "post_apply_learning_incomplete":
+                retry_attempt_ref = f"{deploy_ref}-learning"
             elif latest_check_status == "success":
                 retry_attempt_ref = f"{deploy_ref}-context"
                 retry_reason = "predecision_context"
@@ -530,6 +552,7 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
                         or ""
                     ),
                     "work_item_id": _WAZUH_POSTURE_WORK_ITEM_ID,
+                    "retry_reason": retry_reason,
                     "active_blockers": [
                         "predecision_mcp_or_log_evidence_missing"
                     ],
@@ -541,8 +564,10 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
                 return {
                     "status": (
                         "failed_check_retry_already_attempted_for_deploy"
-                        if retry_reason == "failed_check_mode"
+                        if retry_reason == "check_mode_failed"
                         else "context_retry_already_attempted_for_deploy"
+                        if retry_reason == "predecision_context"
+                        else "incomplete_runtime_retry_already_attempted_for_deploy"
                     ),
                     "queued": 0,
                     "existing": 1,
@@ -553,15 +578,20 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
                         or ""
                     ),
                     "work_item_id": _WAZUH_POSTURE_WORK_ITEM_ID,
+                    "retry_reason": retry_reason,
                     "retry_of_failed_check_mode": (
-                        retry_reason == "failed_check_mode"
+                        retry_reason == "check_mode_failed"
                     ),
                     "retry_of_incomplete_predecision_context": (
                         retry_reason == "predecision_context"
                     ),
+                    "retry_of_incomplete_runtime_closure": retry_reason in {
+                        "controlled_apply_failed",
+                        "post_apply_learning_incomplete",
+                    },
                     "active_blockers": [
                         "bounded_retry_already_attempted_for_deploy"
-                        if retry_reason == "failed_check_mode"
+                        if retry_reason != "predecision_context"
                         else "bounded_context_retry_already_attempted_for_deploy"
                     ],
                 }
@@ -606,10 +636,15 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
                 "evidence_ready": 0,
                 "automation_run_id": automation_run_id,
                 "work_item_id": _WAZUH_POSTURE_WORK_ITEM_ID,
-                "retry_of_failed_check_mode": retry_reason == "failed_check_mode",
+                "retry_reason": retry_reason,
+                "retry_of_failed_check_mode": retry_reason == "check_mode_failed",
                 "retry_of_incomplete_predecision_context": (
                     retry_reason == "predecision_context"
                 ),
+                "retry_of_incomplete_runtime_closure": retry_reason in {
+                    "controlled_apply_failed",
+                    "post_apply_learning_incomplete",
+                },
                 "active_blockers": [
                     "predecision_mcp_or_log_evidence_missing"
                 ],
@@ -632,10 +667,10 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
         )
         return {
             "status": (
-                "queued_for_controlled_executor_retry"
-                if queued and retry_reason == "failed_check_mode"
-                else "queued_for_controlled_executor_context_retry"
+                "queued_for_controlled_executor_context_retry"
                 if queued and retry_reason == "predecision_context"
+                else "queued_for_controlled_executor_retry"
+                if queued and retry_reason
                 else "queued_for_controlled_executor"
                 if queued
                 else "candidate_race_deduplicated"
@@ -645,10 +680,15 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
             "evidence_ready": 1 if evidence_ready else 0,
             "automation_run_id": automation_run_id,
             "work_item_id": _WAZUH_POSTURE_WORK_ITEM_ID,
-            "retry_of_failed_check_mode": retry_reason == "failed_check_mode",
+            "retry_reason": retry_reason,
+            "retry_of_failed_check_mode": retry_reason == "check_mode_failed",
             "retry_of_incomplete_predecision_context": (
                 retry_reason == "predecision_context"
             ),
+            "retry_of_incomplete_runtime_closure": retry_reason in {
+                "controlled_apply_failed",
+                "post_apply_learning_incomplete",
+            },
             "active_blockers": [] if queued else ["candidate_write_not_acknowledged"],
         }
     except Exception as exc:
@@ -666,6 +706,23 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
         }
 
 
+def _wazuh_posture_retry_reason(
+    existing_row: Mapping[str, Any] | None,
+) -> str | None:
+    if not existing_row:
+        return None
+    check_status = str(existing_row.get("latest_check_status") or "")
+    apply_status = str(existing_row.get("latest_apply_status") or "")
+    learning_status = str(existing_row.get("latest_learning_status") or "")
+    if check_status == "failed":
+        return "check_mode_failed"
+    if apply_status == "failed":
+        return "controlled_apply_failed"
+    if apply_status == "success" and learning_status != "success":
+        return "post_apply_learning_incomplete"
+    return None
+
+
 async def enqueue_missing_ansible_candidates_once(
     *,
     project_id: str = "awoooi",

apps/api/src/services/iwooos_wazuh_controlled_executor_runtime.py

@@ -238,10 +238,10 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback(
     check_failed = data.get("check_status") == "failed"
     check_failure_class = _check_failure_class(data)
     apply_terminal = data.get("apply_status") in {"success", "failed"}
-    apply_passed = (
-        data.get("apply_status") == "success"
-        and str(data.get("apply_returncode") or "") == "0"
-    )
+    apply_passed = apply_terminal and str(
+        data.get("apply_returncode") or ""
+    ) == "0"
+    apply_failed = apply_terminal and not apply_passed
     verifier_passed = data.get("verification_result") == "success" and str(
         data.get("verifier_run_id") or ""
     ) == str(data.get("candidate_op_id") or "")
@@ -302,7 +302,7 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback(
         active_blockers.append("wazuh_manager_posture_check_mode_failed")
         if check_failure_class:
             active_blockers.append(check_failure_class)
-    if data.get("apply_status") == "failed":
+    if apply_failed:
         active_blockers.append("wazuh_manager_posture_controlled_probe_failed")
     if apply_passed and not verifier_passed:
         active_blockers.append("independent_post_verifier_missing_or_failed")