fix(iwooos): repair wazuh host trust and retry

apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 import asyncio
 import json
+import os
 import random
 from collections.abc import Awaitable, Callable
 from types import SimpleNamespace
@@ -350,10 +351,16 @@ async def enqueue_ai_decision_ansible_candidate(
 
 
 def _wazuh_posture_incident(
-    *, automation_run_id: str, bucket_ref: str
+    *,
+    automation_run_id: str,
+    bucket_ref: str,
+    attempt_ref: str | None = None,
 ) -> dict[str, Any]:
+    receipt_ref = (
+        f"{bucket_ref}-{attempt_ref.upper()}" if attempt_ref else bucket_ref
+    )
     return {
-        "incident_id": f"IWZ-POSTURE-{bucket_ref}",
+        "incident_id": f"IWZ-POSTURE-{receipt_ref}",
         "project_id": "awoooi",
         "status": "INVESTIGATING",
         "severity": "low",
@@ -363,7 +370,7 @@ def _wazuh_posture_incident(
         "trace_id": automation_run_id,
         "run_id": automation_run_id,
         "work_item_id": _WAZUH_POSTURE_WORK_ITEM_ID,
-        "source_receipt_ref": f"scheduled-wazuh-manager-posture:{bucket_ref}",
+        "source_receipt_ref": f"scheduled-wazuh-manager-posture:{receipt_ref}",
         "asset_scope_aliases": [_WAZUH_POSTURE_PUBLIC_ASSET_ALIAS],
         "affected_services": ["wazuh-manager", "siem"],
         "signals": [
@@ -411,14 +418,24 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
             existing = await db.execute(
                 text("""
                     SELECT
-                        op_id::text AS op_id,
-                        status,
-                        input ->> 'automation_run_id' AS automation_run_id
-                    FROM automation_operation_log
-                    WHERE operation_type = 'ansible_candidate_matched'
-                      AND input -> 'executor_candidates' @> CAST(:catalog_match AS jsonb)
-                      AND created_at >= NOW() - (:freshness_hours * INTERVAL '1 hour')
-                    ORDER BY created_at DESC
+                        candidate.op_id::text AS op_id,
+                        candidate.status AS candidate_status,
+                        candidate.input ->> 'automation_run_id' AS automation_run_id,
+                        candidate.input ->> 'source_receipt_ref' AS source_receipt_ref,
+                        check_mode.status AS latest_check_status
+                    FROM automation_operation_log candidate
+                    LEFT JOIN LATERAL (
+                        SELECT check_receipt.status
+                        FROM automation_operation_log check_receipt
+                        WHERE check_receipt.parent_op_id = candidate.op_id
+                          AND check_receipt.operation_type = 'ansible_check_mode_executed'
+                        ORDER BY check_receipt.created_at DESC
+                        LIMIT 1
+                    ) check_mode ON TRUE
+                    WHERE candidate.operation_type = 'ansible_candidate_matched'
+                      AND candidate.input -> 'executor_candidates' @> CAST(:catalog_match AS jsonb)
+                      AND candidate.created_at >= NOW() - (:freshness_hours * INTERVAL '1 hour')
+                    ORDER BY candidate.created_at DESC
                     LIMIT 1
                 """),
                 {
@@ -429,7 +446,8 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
                 },
             )
             existing_row = existing.mappings().first()
-        if existing_row:
+        retry_attempt_ref: str | None = None
+        if existing_row and existing_row.get("latest_check_status") != "failed":
             return {
                 "status": "fresh_candidate_already_present",
                 "queued": 0,
@@ -443,6 +461,34 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
                 "work_item_id": _WAZUH_POSTURE_WORK_ITEM_ID,
                 "active_blockers": [],
             }
+        if existing_row:
+            source_sha = os.getenv("AWOOOI_BUILD_COMMIT_SHA", "").strip().lower()
+            retry_attempt_ref = (
+                source_sha[:12]
+                if len(source_sha) >= 12
+                and all(character in "0123456789abcdef" for character in source_sha)
+                else "runtime-retry"
+            )
+            existing_source_ref = str(
+                existing_row.get("source_receipt_ref") or ""
+            ).lower()
+            if existing_source_ref.endswith(f"-{retry_attempt_ref.lower()}"):
+                return {
+                    "status": "failed_check_retry_already_attempted_for_deploy",
+                    "queued": 0,
+                    "existing": 1,
+                    "evidence_ready": 1,
+                    "automation_run_id": str(
+                        existing_row.get("automation_run_id")
+                        or existing_row.get("op_id")
+                        or ""
+                    ),
+                    "work_item_id": _WAZUH_POSTURE_WORK_ITEM_ID,
+                    "retry_of_failed_check_mode": True,
+                    "active_blockers": [
+                        "bounded_retry_already_attempted_for_deploy"
+                    ],
+                }
 
         now = now_taipei()
         bucket_hour = (now.hour // freshness_hours) * freshness_hours
@@ -450,12 +496,14 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
         automation_run_id = str(
             uuid5(
                 NAMESPACE_URL,
-                f"awoooi:iwooos:wazuh-manager-posture:{bucket.isoformat()}",
+                "awoooi:iwooos:wazuh-manager-posture:"
+                f"{bucket.isoformat()}:{retry_attempt_ref or 'initial'}",
             )
         )
         incident = _wazuh_posture_incident(
             automation_run_id=automation_run_id,
             bucket_ref=bucket.strftime("%Y%m%d%H"),
+            attempt_ref=retry_attempt_ref,
         )
         try:
             snapshot = await evidence_collector(
@@ -490,10 +538,14 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
         )
         return {
             "status": (
-                "queued_for_controlled_executor"
+                "queued_for_controlled_executor_retry"
+                if queued and evidence_ready and retry_attempt_ref
+                else "queued_for_controlled_executor"
                 if queued and evidence_ready
                 else (
-                    "queued_for_controlled_executor_with_degraded_context"
+                    "queued_for_controlled_executor_retry_with_degraded_context"
+                    if queued and retry_attempt_ref
+                    else "queued_for_controlled_executor_with_degraded_context"
                     if queued
                     else "candidate_race_deduplicated"
                 )
@@ -503,6 +555,7 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
             "evidence_ready": 1 if evidence_ready else 0,
             "automation_run_id": automation_run_id,
             "work_item_id": _WAZUH_POSTURE_WORK_ITEM_ID,
+            "retry_of_failed_check_mode": bool(retry_attempt_ref),
             "active_blockers": (
                 [] if evidence_ready else ["predecision_mcp_or_log_evidence_missing"]
             ),

apps/api/src/services/iwooos_wazuh_controlled_executor_runtime.py

@@ -54,6 +54,10 @@ _LIVE_RUNTIME_SQL = """
                 check_mode.output ->> 'returncode',
                 check_mode.dry_run_result ->> 'returncode'
             ) AS check_returncode,
+            coalesce(
+                check_mode.output ->> 'timed_out',
+                check_mode.dry_run_result ->> 'timed_out'
+            ) AS check_timed_out,
             check_mode.duration_ms AS check_duration_ms,
             check_mode.created_at AS check_created_at
         FROM automation_operation_log check_mode
@@ -231,6 +235,8 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback(
         data.get("check_status") == "success"
         and str(data.get("check_returncode") or "") == "0"
     )
+    check_failed = data.get("check_status") == "failed"
+    check_failure_class = _check_failure_class(data)
     apply_terminal = data.get("apply_status") in {"success", "failed"}
     apply_passed = (
         data.get("apply_status") == "success"
@@ -294,6 +300,8 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback(
         active_blockers.append("wazuh_manager_posture_catalog_missing")
     if candidate_present and data.get("check_status") == "failed":
         active_blockers.append("wazuh_manager_posture_check_mode_failed")
+        if check_failure_class:
+            active_blockers.append(check_failure_class)
     if data.get("apply_status") == "failed":
         active_blockers.append("wazuh_manager_posture_controlled_probe_failed")
     if apply_passed and not verifier_passed:
@@ -311,6 +319,7 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback(
             read_error_type=read_error_type,
             candidate_present=candidate_present,
             check_passed=check_passed,
+            check_failed=check_failed,
             apply_terminal=apply_terminal,
             apply_passed=apply_passed,
             runtime_closed=runtime_closed,
@@ -329,6 +338,7 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback(
             "executor_policy_enabled_count": 1 if enabled and catalog_ready else 0,
             "dispatch_candidate_count": 1 if candidate_present else 0,
             "check_mode_passed_count": 1 if check_passed else 0,
+            "check_mode_failed_count": 1 if check_failed else 0,
             "runtime_execution_performed_count": 1 if apply_terminal else 0,
             "bounded_posture_execution_passed_count": 1 if apply_passed else 0,
             "independent_post_verifier_passed_count": 1 if verifier_passed else 0,
@@ -359,9 +369,12 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback(
             for stage_id in required_stages
         ],
         "missing_stage_ids": missing_stage_ids,
+        "check_failure_class": check_failure_class,
         "next_safe_action": _next_action(
             candidate_present=candidate_present,
             check_passed=check_passed,
+            check_failed=check_failed,
+            check_failure_class=check_failure_class,
             apply_terminal=apply_terminal,
             apply_passed=apply_passed,
             verifier_passed=verifier_passed,
@@ -396,6 +409,7 @@ def _status(
     read_error_type: str | None,
     candidate_present: bool,
     check_passed: bool,
+    check_failed: bool,
     apply_terminal: bool,
     apply_passed: bool,
     runtime_closed: bool,
@@ -410,6 +424,8 @@ def _status(
         return "wazuh_manager_posture_executed_runtime_writeback_open"
     if check_passed:
         return "wazuh_manager_posture_check_passed_waiting_bounded_execution"
+    if check_failed:
+        return "wazuh_manager_posture_check_failed_waiting_bounded_retry"
     if candidate_present:
         return "wazuh_manager_posture_dispatched_waiting_check_mode"
     return "waiting_for_scheduled_wazuh_manager_posture_dispatch"
@@ -419,6 +435,8 @@ def _next_action(
     *,
     candidate_present: bool,
     check_passed: bool,
+    check_failed: bool,
+    check_failure_class: str | None,
     apply_terminal: bool,
     apply_passed: bool,
     verifier_passed: bool,
@@ -429,6 +447,11 @@ def _next_action(
         return "keep_scheduled_wazuh_manager_posture_runtime_receipts_fresh"
     if not candidate_present:
         return "internal_worker_enqueue_wazuh_manager_posture_candidate"
+    if check_failed:
+        return (
+            "repair_and_enqueue_bounded_retry:"
+            + (check_failure_class or "ansible_check_mode_failed")
+        )
     if not check_passed:
         return "worker_claim_and_run_allowlisted_ansible_check_mode"
     if not apply_terminal:
@@ -440,6 +463,19 @@ def _next_action(
     return "backfill_same_run_receipts:" + ",".join(missing_stage_ids[:6])
 
 
+def _check_failure_class(data: Mapping[str, Any]) -> str | None:
+    if data.get("check_status") != "failed":
+        return None
+    if str(data.get("check_timed_out") or "").lower() == "true":
+        return "ansible_check_mode_timeout"
+    returncode = str(data.get("check_returncode") or "").strip()
+    if returncode == "4":
+        return "ansible_target_unreachable"
+    if returncode.isdigit():
+        return f"ansible_check_mode_returncode_{returncode}"
+    return "ansible_check_mode_failed_unclassified"
+
+
 def _latest_receipt_at(data: Mapping[str, Any]) -> str | None:
     values = [
         value