fix(iwooos): repair wazuh host trust and retry

apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 import asyncio
 import json
+import os
 import random
 from collections.abc import Awaitable, Callable
 from types import SimpleNamespace
@@ -350,10 +351,16 @@ async def enqueue_ai_decision_ansible_candidate(
 
 
 def _wazuh_posture_incident(
-    *, automation_run_id: str, bucket_ref: str
+    *,
+    automation_run_id: str,
+    bucket_ref: str,
+    attempt_ref: str | None = None,
 ) -> dict[str, Any]:
+    receipt_ref = (
+        f"{bucket_ref}-{attempt_ref.upper()}" if attempt_ref else bucket_ref
+    )
     return {
-        "incident_id": f"IWZ-POSTURE-{bucket_ref}",
+        "incident_id": f"IWZ-POSTURE-{receipt_ref}",
         "project_id": "awoooi",
         "status": "INVESTIGATING",
         "severity": "low",
@@ -363,7 +370,7 @@ def _wazuh_posture_incident(
         "trace_id": automation_run_id,
         "run_id": automation_run_id,
         "work_item_id": _WAZUH_POSTURE_WORK_ITEM_ID,
-        "source_receipt_ref": f"scheduled-wazuh-manager-posture:{bucket_ref}",
+        "source_receipt_ref": f"scheduled-wazuh-manager-posture:{receipt_ref}",
         "asset_scope_aliases": [_WAZUH_POSTURE_PUBLIC_ASSET_ALIAS],
         "affected_services": ["wazuh-manager", "siem"],
         "signals": [
@@ -411,14 +418,24 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
             existing = await db.execute(
                 text("""
                     SELECT
-                        op_id::text AS op_id,
-                        status,
-                        input ->> 'automation_run_id' AS automation_run_id
-                    FROM automation_operation_log
-                    WHERE operation_type = 'ansible_candidate_matched'
-                      AND input -> 'executor_candidates' @> CAST(:catalog_match AS jsonb)
-                      AND created_at >= NOW() - (:freshness_hours * INTERVAL '1 hour')
-                    ORDER BY created_at DESC
+                        candidate.op_id::text AS op_id,
+                        candidate.status AS candidate_status,
+                        candidate.input ->> 'automation_run_id' AS automation_run_id,
+                        candidate.input ->> 'source_receipt_ref' AS source_receipt_ref,
+                        check_mode.status AS latest_check_status
+                    FROM automation_operation_log candidate
+                    LEFT JOIN LATERAL (
+                        SELECT check_receipt.status
+                        FROM automation_operation_log check_receipt
+                        WHERE check_receipt.parent_op_id = candidate.op_id
+                          AND check_receipt.operation_type = 'ansible_check_mode_executed'
+                        ORDER BY check_receipt.created_at DESC
+                        LIMIT 1
+                    ) check_mode ON TRUE
+                    WHERE candidate.operation_type = 'ansible_candidate_matched'
+                      AND candidate.input -> 'executor_candidates' @> CAST(:catalog_match AS jsonb)
+                      AND candidate.created_at >= NOW() - (:freshness_hours * INTERVAL '1 hour')
+                    ORDER BY candidate.created_at DESC
                     LIMIT 1
                 """),
                 {
@@ -429,7 +446,8 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
                 },
             )
             existing_row = existing.mappings().first()
-        if existing_row:
+        retry_attempt_ref: str | None = None
+        if existing_row and existing_row.get("latest_check_status") != "failed":
             return {
                 "status": "fresh_candidate_already_present",
                 "queued": 0,
@@ -443,6 +461,34 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
                 "work_item_id": _WAZUH_POSTURE_WORK_ITEM_ID,
                 "active_blockers": [],
             }
+        if existing_row:
+            source_sha = os.getenv("AWOOOI_BUILD_COMMIT_SHA", "").strip().lower()
+            retry_attempt_ref = (
+                source_sha[:12]
+                if len(source_sha) >= 12
+                and all(character in "0123456789abcdef" for character in source_sha)
+                else "runtime-retry"
+            )
+            existing_source_ref = str(
+                existing_row.get("source_receipt_ref") or ""
+            ).lower()
+            if existing_source_ref.endswith(f"-{retry_attempt_ref.lower()}"):
+                return {
+                    "status": "failed_check_retry_already_attempted_for_deploy",
+                    "queued": 0,
+                    "existing": 1,
+                    "evidence_ready": 1,
+                    "automation_run_id": str(
+                        existing_row.get("automation_run_id")
+                        or existing_row.get("op_id")
+                        or ""
+                    ),
+                    "work_item_id": _WAZUH_POSTURE_WORK_ITEM_ID,
+                    "retry_of_failed_check_mode": True,
+                    "active_blockers": [
+                        "bounded_retry_already_attempted_for_deploy"
+                    ],
+                }
 
         now = now_taipei()
         bucket_hour = (now.hour // freshness_hours) * freshness_hours
@@ -450,12 +496,14 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
         automation_run_id = str(
             uuid5(
                 NAMESPACE_URL,
-                f"awoooi:iwooos:wazuh-manager-posture:{bucket.isoformat()}",
+                "awoooi:iwooos:wazuh-manager-posture:"
+                f"{bucket.isoformat()}:{retry_attempt_ref or 'initial'}",
             )
         )
         incident = _wazuh_posture_incident(
             automation_run_id=automation_run_id,
             bucket_ref=bucket.strftime("%Y%m%d%H"),
+            attempt_ref=retry_attempt_ref,
         )
         try:
             snapshot = await evidence_collector(
@@ -490,10 +538,14 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
         )
         return {
             "status": (
-                "queued_for_controlled_executor"
+                "queued_for_controlled_executor_retry"
+                if queued and evidence_ready and retry_attempt_ref
+                else "queued_for_controlled_executor"
                 if queued and evidence_ready
                 else (
-                    "queued_for_controlled_executor_with_degraded_context"
+                    "queued_for_controlled_executor_retry_with_degraded_context"
+                    if queued and retry_attempt_ref
+                    else "queued_for_controlled_executor_with_degraded_context"
                     if queued
                     else "candidate_race_deduplicated"
                 )
@@ -503,6 +555,7 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once(
             "evidence_ready": 1 if evidence_ready else 0,
             "automation_run_id": automation_run_id,
             "work_item_id": _WAZUH_POSTURE_WORK_ITEM_ID,
+            "retry_of_failed_check_mode": bool(retry_attempt_ref),
             "active_blockers": (
                 [] if evidence_ready else ["predecision_mcp_or_log_evidence_missing"]
             ),

apps/api/src/services/iwooos_wazuh_controlled_executor_runtime.py

@@ -54,6 +54,10 @@ _LIVE_RUNTIME_SQL = """
                 check_mode.output ->> 'returncode',
                 check_mode.dry_run_result ->> 'returncode'
             ) AS check_returncode,
+            coalesce(
+                check_mode.output ->> 'timed_out',
+                check_mode.dry_run_result ->> 'timed_out'
+            ) AS check_timed_out,
             check_mode.duration_ms AS check_duration_ms,
             check_mode.created_at AS check_created_at
         FROM automation_operation_log check_mode
@@ -231,6 +235,8 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback(
         data.get("check_status") == "success"
         and str(data.get("check_returncode") or "") == "0"
     )
+    check_failed = data.get("check_status") == "failed"
+    check_failure_class = _check_failure_class(data)
     apply_terminal = data.get("apply_status") in {"success", "failed"}
     apply_passed = (
         data.get("apply_status") == "success"
@@ -294,6 +300,8 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback(
         active_blockers.append("wazuh_manager_posture_catalog_missing")
     if candidate_present and data.get("check_status") == "failed":
         active_blockers.append("wazuh_manager_posture_check_mode_failed")
+        if check_failure_class:
+            active_blockers.append(check_failure_class)
     if data.get("apply_status") == "failed":
         active_blockers.append("wazuh_manager_posture_controlled_probe_failed")
     if apply_passed and not verifier_passed:
@@ -311,6 +319,7 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback(
             read_error_type=read_error_type,
             candidate_present=candidate_present,
             check_passed=check_passed,
+            check_failed=check_failed,
             apply_terminal=apply_terminal,
             apply_passed=apply_passed,
             runtime_closed=runtime_closed,
@@ -329,6 +338,7 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback(
             "executor_policy_enabled_count": 1 if enabled and catalog_ready else 0,
             "dispatch_candidate_count": 1 if candidate_present else 0,
             "check_mode_passed_count": 1 if check_passed else 0,
+            "check_mode_failed_count": 1 if check_failed else 0,
             "runtime_execution_performed_count": 1 if apply_terminal else 0,
             "bounded_posture_execution_passed_count": 1 if apply_passed else 0,
             "independent_post_verifier_passed_count": 1 if verifier_passed else 0,
@@ -359,9 +369,12 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback(
             for stage_id in required_stages
         ],
         "missing_stage_ids": missing_stage_ids,
+        "check_failure_class": check_failure_class,
         "next_safe_action": _next_action(
             candidate_present=candidate_present,
             check_passed=check_passed,
+            check_failed=check_failed,
+            check_failure_class=check_failure_class,
             apply_terminal=apply_terminal,
             apply_passed=apply_passed,
             verifier_passed=verifier_passed,
@@ -396,6 +409,7 @@ def _status(
     read_error_type: str | None,
     candidate_present: bool,
     check_passed: bool,
+    check_failed: bool,
     apply_terminal: bool,
     apply_passed: bool,
     runtime_closed: bool,
@@ -410,6 +424,8 @@ def _status(
         return "wazuh_manager_posture_executed_runtime_writeback_open"
     if check_passed:
         return "wazuh_manager_posture_check_passed_waiting_bounded_execution"
+    if check_failed:
+        return "wazuh_manager_posture_check_failed_waiting_bounded_retry"
     if candidate_present:
         return "wazuh_manager_posture_dispatched_waiting_check_mode"
     return "waiting_for_scheduled_wazuh_manager_posture_dispatch"
@@ -419,6 +435,8 @@ def _next_action(
     *,
     candidate_present: bool,
     check_passed: bool,
+    check_failed: bool,
+    check_failure_class: str | None,
     apply_terminal: bool,
     apply_passed: bool,
     verifier_passed: bool,
@@ -429,6 +447,11 @@ def _next_action(
         return "keep_scheduled_wazuh_manager_posture_runtime_receipts_fresh"
     if not candidate_present:
         return "internal_worker_enqueue_wazuh_manager_posture_candidate"
+    if check_failed:
+        return (
+            "repair_and_enqueue_bounded_retry:"
+            + (check_failure_class or "ansible_check_mode_failed")
+        )
     if not check_passed:
         return "worker_claim_and_run_allowlisted_ansible_check_mode"
     if not apply_terminal:
@@ -440,6 +463,19 @@ def _next_action(
     return "backfill_same_run_receipts:" + ",".join(missing_stage_ids[:6])
 
 
+def _check_failure_class(data: Mapping[str, Any]) -> str | None:
+    if data.get("check_status") != "failed":
+        return None
+    if str(data.get("check_timed_out") or "").lower() == "true":
+        return "ansible_check_mode_timeout"
+    returncode = str(data.get("check_returncode") or "").strip()
+    if returncode == "4":
+        return "ansible_target_unreachable"
+    if returncode.isdigit():
+        return f"ansible_check_mode_returncode_{returncode}"
+    return "ansible_check_mode_failed_unclassified"
+
+
 def _latest_receipt_at(data: Mapping[str, Any]) -> str | None:
     values = [
         value

apps/api/tests/test_executor_network_policy_boundary.py

@@ -103,3 +103,12 @@ def test_cd_applies_rolls_back_and_verifies_network_boundary() -> None:
     assert "legacy_namespace_egress_allow_absent=true" in workflow
     assert "192.168.0.110/32 192.168.0.112/32 192.168.0.120/32" in workflow
     assert 'wc -l)" = "5"' in workflow
+    expected_hosts = (
+        "192.168.0.110 192.168.0.112 192.168.0.120 "
+        "192.168.0.121 192.168.0.188"
+    )
+    assert f"ssh-keyscan {expected_hosts}" in workflow
+    assert "EXPECTED_HOSTS=5" in workflow
+    assert workflow.count(expected_hosts) >= 4
+    assert "len(reachable) == len(sys.argv) - 1" in workflow
+    assert "ssh-mcp-key known_hosts 更新失敗，停止部署" in workflow

apps/api/tests/test_iwooos_wazuh_controlled_executor_runtime.py

@@ -113,6 +113,37 @@ def test_wazuh_runtime_readback_keeps_partial_execution_open() -> None:
     assert "same_run_mcp_context_missing" in payload["active_blockers"]
 
 
+def test_wazuh_runtime_readback_classifies_failed_check_for_bounded_retry() -> None:
+    row = {
+        "candidate_op_id": "00000000-0000-0000-0000-000000000311",
+        "automation_run_id": "00000000-0000-0000-0000-000000000311",
+        "trace_id": "00000000-0000-0000-0000-000000000311",
+        "run_id": "00000000-0000-0000-0000-000000000311",
+        "work_item_id": "P0-03-WAZUH-MANAGER-POSTURE",
+        "check_status": "failed",
+        "check_returncode": "4",
+        "check_timed_out": "false",
+        "stage_ids": [],
+    }
+
+    payload = build_iwooos_wazuh_controlled_executor_runtime_readback(
+        row,
+        executor_enabled=True,
+    )
+
+    assert payload["status"] == (
+        "wazuh_manager_posture_check_failed_waiting_bounded_retry"
+    )
+    assert payload["summary"]["check_mode_failed_count"] == 1
+    assert payload["summary"]["runtime_execution_performed_count"] == 0
+    assert payload["summary"]["runtime_closed_count"] == 0
+    assert payload["check_failure_class"] == "ansible_target_unreachable"
+    assert payload["next_safe_action"] == (
+        "repair_and_enqueue_bounded_retry:ansible_target_unreachable"
+    )
+    assert "ansible_target_unreachable" in payload["active_blockers"]
+
+
 def test_wazuh_posture_catalog_and_playbook_are_bounded_no_write() -> None:
     catalog = get_ansible_catalog_item("ansible:wazuh-manager-posture-readback")
     assert catalog is not None
@@ -224,6 +255,192 @@ async def test_wazuh_posture_scheduler_enqueues_after_mcp_and_log_evidence(
     posture_clock.assert_called_once_with()
 
 
+@pytest.mark.asyncio
+async def test_wazuh_posture_scheduler_retries_failed_check_once_per_deploy(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from src.jobs import awooop_ansible_candidate_backfill_job as job
+
+    existing_row = {
+        "op_id": "00000000-0000-0000-0000-000000000321",
+        "candidate_status": "dry_run",
+        "automation_run_id": "00000000-0000-0000-0000-000000000321",
+        "latest_check_status": "failed",
+    }
+
+    class _Mappings:
+        def first(self):
+            return existing_row
+
+    class _Result:
+        def mappings(self):
+            return _Mappings()
+
+    class _Db:
+        async def execute(self, _statement, _params):
+            return _Result()
+
+    @asynccontextmanager
+    async def fake_db_context(project_id: str):
+        assert project_id == "awoooi"
+        yield _Db()
+
+    recorder = AsyncMock(return_value=True)
+    monkeypatch.setattr(job, "get_db_context", fake_db_context)
+    monkeypatch.setattr(
+        job.settings,
+        "ENABLE_IWOOOS_WAZUH_MANAGER_POSTURE_EXECUTOR",
+        True,
+    )
+    monkeypatch.setattr(
+        job.settings,
+        "IWOOOS_WAZUH_MANAGER_POSTURE_FRESHNESS_HOURS",
+        6,
+    )
+    monkeypatch.setattr(
+        job,
+        "now_taipei",
+        MagicMock(return_value=datetime.fromisoformat("2026-07-11T05:59:59+08:00")),
+    )
+    monkeypatch.setenv(
+        "AWOOOI_BUILD_COMMIT_SHA",
+        "abcdef1234567890abcdef1234567890abcdef12",
+    )
+
+    result = await job.enqueue_iwooos_wazuh_manager_posture_candidate_once(
+        recorder=recorder,
+        evidence_collector=AsyncMock(return_value=MagicMock(snapshot_id="wazuh-2")),
+        evidence_verifier=AsyncMock(return_value=True),
+    )
+
+    assert result["status"] == "queued_for_controlled_executor_retry"
+    assert result["queued"] == 1
+    assert result["retry_of_failed_check_mode"] is True
+    recorded = recorder.await_args.kwargs
+    assert recorded["incident"]["incident_id"] == (
+        "IWZ-POSTURE-2026071100-ABCDEF123456"
+    )
+    assert recorded["incident"]["source_receipt_ref"] == (
+        "scheduled-wazuh-manager-posture:2026071100-ABCDEF123456"
+    )
+    assert recorded["automation_run_id"] == result["automation_run_id"]
+
+
+@pytest.mark.asyncio
+async def test_wazuh_posture_scheduler_does_not_duplicate_active_candidate(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from src.jobs import awooop_ansible_candidate_backfill_job as job
+
+    class _Mappings:
+        def first(self):
+            return {
+                "op_id": "00000000-0000-0000-0000-000000000331",
+                "candidate_status": "dry_run",
+                "automation_run_id": "00000000-0000-0000-0000-000000000331",
+                "latest_check_status": "pending",
+            }
+
+    class _Result:
+        def mappings(self):
+            return _Mappings()
+
+    class _Db:
+        async def execute(self, _statement, _params):
+            return _Result()
+
+    @asynccontextmanager
+    async def fake_db_context(_project_id: str):
+        yield _Db()
+
+    recorder = AsyncMock(return_value=True)
+    monkeypatch.setattr(job, "get_db_context", fake_db_context)
+    monkeypatch.setattr(
+        job.settings,
+        "ENABLE_IWOOOS_WAZUH_MANAGER_POSTURE_EXECUTOR",
+        True,
+    )
+    monkeypatch.setattr(
+        job.settings,
+        "IWOOOS_WAZUH_MANAGER_POSTURE_FRESHNESS_HOURS",
+        6,
+    )
+
+    result = await job.enqueue_iwooos_wazuh_manager_posture_candidate_once(
+        recorder=recorder,
+    )
+
+    assert result["status"] == "fresh_candidate_already_present"
+    assert result["queued"] == 0
+    recorder.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_wazuh_posture_scheduler_limits_failed_retry_to_once_per_deploy(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from src.jobs import awooop_ansible_candidate_backfill_job as job
+
+    class _Mappings:
+        def first(self):
+            return {
+                "op_id": "00000000-0000-0000-0000-000000000341",
+                "candidate_status": "failed",
+                "automation_run_id": "00000000-0000-0000-0000-000000000341",
+                "source_receipt_ref": (
+                    "scheduled-wazuh-manager-posture:"
+                    "2026071100-ABCDEF123456"
+                ),
+                "latest_check_status": "failed",
+            }
+
+    class _Result:
+        def mappings(self):
+            return _Mappings()
+
+    class _Db:
+        async def execute(self, _statement, _params):
+            return _Result()
+
+    @asynccontextmanager
+    async def fake_db_context(_project_id: str):
+        yield _Db()
+
+    recorder = AsyncMock(return_value=True)
+    collector = AsyncMock()
+    monkeypatch.setattr(job, "get_db_context", fake_db_context)
+    monkeypatch.setattr(
+        job.settings,
+        "ENABLE_IWOOOS_WAZUH_MANAGER_POSTURE_EXECUTOR",
+        True,
+    )
+    monkeypatch.setattr(
+        job.settings,
+        "IWOOOS_WAZUH_MANAGER_POSTURE_FRESHNESS_HOURS",
+        6,
+    )
+    monkeypatch.setenv(
+        "AWOOOI_BUILD_COMMIT_SHA",
+        "abcdef1234567890abcdef1234567890abcdef12",
+    )
+
+    result = await job.enqueue_iwooos_wazuh_manager_posture_candidate_once(
+        recorder=recorder,
+        evidence_collector=collector,
+    )
+
+    assert result["status"] == (
+        "failed_check_retry_already_attempted_for_deploy"
+    )
+    assert result["queued"] == 0
+    assert result["retry_of_failed_check_mode"] is True
+    assert result["active_blockers"] == [
+        "bounded_retry_already_attempted_for_deploy"
+    ]
+    recorder.assert_not_awaited()
+    collector.assert_not_awaited()
+
+
 @pytest.mark.asyncio
 async def test_wazuh_posture_scheduler_queues_bounded_probe_with_degraded_context(
     monkeypatch: pytest.MonkeyPatch,

apps/api/tests/test_signal_worker_ansible_executor_binding.py

@@ -91,6 +91,7 @@ def test_api_manifest_has_read_only_identity_and_no_ssh_executor_mounts() -> Non
     assert env["DATABASE_NULL_POOL"] == "true"
     assert env["ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER"] == "false"
     assert env["ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER"] == "false"
+    assert env["ENABLE_IWOOOS_WAZUH_MANAGER_POSTURE_EXECUTOR"] == "true"
     assert {"repair-ssh-key", "repair-known-hosts", "ssh-mcp-key"}.isdisjoint(
         mount_names | volume_names
     )