From 602ba003cf94bb0aebc226cfddeb837039e55aa9 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 22 Jul 2026 15:25:58 +0800 Subject: [PATCH] fix(iwooos): close Wazuh capability retry gate --- apps/api/src/api/v1/approvals.py | 54 +- apps/api/src/api/v1/telegram.py | 10 + .../awooop_ansible_candidate_backfill_job.py | 289 +++- apps/api/src/models/approval.py | 16 + apps/api/src/services/approval_db.py | 71 +- .../awooop_ansible_check_mode_service.py | 1454 ++++++++++++++++- ...wooos_wazuh_controlled_executor_runtime.py | 652 +++++++- apps/api/src/services/telegram_gateway.py | 10 + .../services/wazuh_break_glass_approval.py | 116 ++ ...wooos_wazuh_controlled_executor_runtime.py | 1113 ++++++++++++- ...test_telegram_webhook_execution_handoff.py | 19 +- .../test_wazuh_alertmanager_integration.py | 501 ++++++ ...est_wazuh_break_glass_approval_security.py | 275 ++++ 13 files changed, 4512 insertions(+), 68 deletions(-) create mode 100644 apps/api/src/services/wazuh_break_glass_approval.py create mode 100644 apps/api/tests/test_wazuh_break_glass_approval_security.py diff --git a/apps/api/src/api/v1/approvals.py b/apps/api/src/api/v1/approvals.py index 687dd9cef..da8b728e2 100644 --- a/apps/api/src/api/v1/approvals.py +++ b/apps/api/src/api/v1/approvals.py @@ -34,6 +34,9 @@ from fastapi import ( ) from fastapi.responses import StreamingResponse +from src.core.awooop_operator_auth import ( + authenticate_awooop_operator_headers, +) from src.core.config import settings from src.core.csrf import CSRFToken # Phase 20: CSRF Protection from src.core.logging import get_logger @@ -54,6 +57,11 @@ from src.services.approval_db import get_approval_service, get_timeline_service from src.services.approval_execution import get_execution_service from src.services.executor import get_executor from src.services.incident_approval_service import get_incident_approval_service +from src.services.wazuh_break_glass_approval import ( + CONTROLLED_OPERATOR_APPROVER_ROLE, + CONTROLLED_OPERATOR_AUTH_METHOD, + is_wazuh_break_glass_approval_action, +) router = APIRouter(prefix="/approvals", tags=["HITL Approvals"]) logger = get_logger("awoooi.approvals") @@ -267,6 +275,14 @@ async def sign_approval( request: SignRequest, background_tasks: BackgroundTasks, _csrf_token: CSRFToken, # Phase 20: CSRF Protection (驗證用，不需要使用值) + x_awooop_operator_id: str | None = Header( + None, + alias="X-AwoooP-Operator-Id", + ), + x_awooop_operator_key: str | None = Header( + None, + alias="X-AwoooP-Operator-Key", + ), ) -> SignResponse: """ 簽核授權請求 (Phase 5: Database + K8s Execution) @@ -289,11 +305,37 @@ async def sign_approval( service = get_approval_service() timeline = get_timeline_service() + effective_signer_id = request.signer_id + effective_signer_name = request.signer_name + trusted_signature: dict[str, str] = {} + existing_approval = await service.get_approval(approval_id) + if existing_approval is not None and is_wazuh_break_glass_approval_action( + existing_approval.action + ): + operator = authenticate_awooop_operator_headers( + operator_id=x_awooop_operator_id, + operator_key=x_awooop_operator_key, + ) + if operator.auth_method != "operator_api_key": + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Operator authentication required", + ) + effective_signer_id = operator.operator_id + effective_signer_name = operator.operator_id + trusted_signature = { + "authenticated_principal_id": operator.operator_id, + "authentication_method": CONTROLLED_OPERATOR_AUTH_METHOD, + "approver_role": CONTROLLED_OPERATOR_APPROVER_ROLE, + "signature_source": "api", + } + approval, message, execution_triggered = await service.sign_approval( approval_id=approval_id, - signer_id=request.signer_id, - signer_name=request.signer_name, + signer_id=effective_signer_id, + signer_name=effective_signer_name, comment=request.comment, + **trusted_signature, ) if approval is None: @@ -322,8 +364,8 @@ async def sign_approval( await timeline.add_event( event_type="human", status="success", - title=f"{request.signer_name} 簽核成功 ({approval.current_signatures}/{approval.required_signatures})", - actor=request.signer_name, + title=f"{effective_signer_name} 簽核成功 ({approval.current_signatures}/{approval.required_signatures})", + actor=effective_signer_name, actor_role="signer", risk_level=approval.risk_level.value, approval_id=str(approval_id), @@ -333,8 +375,8 @@ async def sign_approval( logger.info( "approval_signed_db", approval_id=str(approval_id), - signer_id=request.signer_id, - signer_name=request.signer_name, + signer_id=effective_signer_id, + signer_name=effective_signer_name, current_signatures=approval.current_signatures, required_signatures=approval.required_signatures, execution_triggered=execution_triggered, diff --git a/apps/api/src/api/v1/telegram.py b/apps/api/src/api/v1/telegram.py index e12d3e7fa..9b9bfa41c 100644 --- a/apps/api/src/api/v1/telegram.py +++ b/apps/api/src/api/v1/telegram.py @@ -41,6 +41,10 @@ from src.services.telegram_gateway import ( _telegram_send_delivery_succeeded, get_telegram_gateway, ) +from src.services.wazuh_break_glass_approval import ( + TELEGRAM_OWNER_APPROVER_ROLE, + TELEGRAM_OWNER_AUTH_METHOD, +) logger = get_logger("awoooi.telegram") router = APIRouter(prefix="/telegram", tags=["Telegram"]) @@ -407,6 +411,12 @@ async def telegram_webhook( signer_id=f"tg_{user_id}", signer_name=user.get("username") or user.get("first_name") or str(user_id), comment="Telegram 簽核", + authenticated_principal_id=f"tg_{user_id}", + authentication_method=TELEGRAM_OWNER_AUTH_METHOD, + approver_role=TELEGRAM_OWNER_APPROVER_ROLE, + signature_source="telegram", + telegram_user_id=int(user_id), + telegram_message_id=int(message_id), ) if approval: diff --git a/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py b/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py index c17cbe7b9..dd34d686f 100644 --- a/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py +++ b/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py @@ -31,6 +31,7 @@ from src.services.awooop_ansible_audit_service import ( verified_ansible_candidate_terminal_sql, ) from src.services.awooop_ansible_check_mode_service import ( + WAZUH_CONTROLLED_CAPABILITY_MAX_RETRIES, preflight_failed_apply_retry_queue_once, ) from src.services.evidence_snapshot import EvidenceSnapshot @@ -79,6 +80,12 @@ _WAZUH_CONTROLLED_CAPABILITY_PACKET_OPERATION_TYPE = ( _WAZUH_PRIVILEGED_CONVERGENCE_SECRET_REFERENCE_MISSING = ( "wazuh_privileged_convergence_secret_reference_missing" ) +_WAZUH_PRIVILEGED_CONVERGENCE_CAPABILITY_MISSING = ( + "wazuh_privileged_convergence_capability_missing" +) +_WAZUH_CRITICAL_BREAK_GLASS_RECEIPTS_MISSING = ( + "critical_secret_reference_owner_approval_and_broker_receipts_missing" +) _EXECUTION_CAPABILITY_FALLBACK_OPERATION_TYPE = "remediation_executed" _WAZUH_SOURCE_RECURRENCE_DRIFT_WORK_ITEM_ID = ( "DRIFT-WAZUH-SOURCE-RECURRENCE" @@ -1178,7 +1185,24 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once( candidate.status AS candidate_status, candidate.input ->> 'automation_run_id' AS automation_run_id, candidate.input ->> 'source_receipt_ref' AS source_receipt_ref, + candidate.input ->> 'incident_id' AS incident_id, check_mode.status AS latest_check_status, + check_mode.input ->> 'capability_retry_generation' + AS latest_check_capability_retry_generation, + check_mode.input ->> 'controlled_apply_blocker' + AS latest_check_controlled_apply_blocker, + check_mode.input ->> 'break_glass_approval_id' + AS latest_check_break_glass_approval_id, + coalesce( + ( + check_mode.input + ->> 'break_glass_apply_authorized' + )::boolean, + false + ) AS latest_check_break_glass_apply_authorized, + check_mode.input + ->> 'break_glass_apply_terminal_status' + AS latest_check_break_glass_apply_terminal_status, apply.status AS latest_apply_status, learning.status AS latest_learning_status, capability_packet.status @@ -1187,6 +1211,8 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once( AS latest_controlled_capability_queue_status, capability_packet.output ->> 'failure_class' AS latest_controlled_capability_failure_class, + capability_packet.input ->> 'capability_retry_generation' + AS latest_controlled_capability_retry_generation, capability_packet.input #>> '{gate_lifecycle,expires_at}' AS latest_controlled_capability_packet_expires_at, @@ -1217,7 +1243,10 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once( ) AS predecision_evidence_ready FROM automation_operation_log candidate LEFT JOIN LATERAL ( - SELECT check_receipt.op_id, check_receipt.status + SELECT + check_receipt.op_id, + check_receipt.status, + check_receipt.input FROM automation_operation_log check_receipt WHERE check_receipt.parent_op_id = candidate.op_id AND check_receipt.operation_type = 'ansible_check_mode_executed' @@ -1243,7 +1272,11 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once( LEFT JOIN LATERAL ( SELECT packet.status, packet.input, packet.output FROM automation_operation_log packet - WHERE packet.parent_op_id = check_mode.op_id + JOIN automation_operation_log packet_owner_check + ON packet.parent_op_id = packet_owner_check.op_id + WHERE packet_owner_check.parent_op_id = candidate.op_id + AND packet_owner_check.operation_type = + 'ansible_check_mode_executed' AND ( packet.operation_type = :capability_packet_operation_type @@ -1290,7 +1323,53 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once( AND candidate.input #>> '{source_recurrence,occurrence_id}' = :source_occurrence_id ) ) - AND candidate.created_at >= NOW() - (:freshness_hours * INTERVAL '1 hour') + AND ( + candidate.created_at >= NOW() - ( + :freshness_hours * INTERVAL '1 hour' + ) + OR ( + :retain_controlled_capability_owner + AND ( + ( + capability_packet.status = 'pending' + AND capability_packet.output + ->> 'queue_status' + IN ('queued', 'retry_exhausted') + ) + OR ( + check_mode.input + ->> 'capability_retry_generation' + ~ '^[1-9][0-9]*$' + AND ( + check_mode.status = 'pending' + OR check_mode.input + ->> 'controlled_apply_blocker' + IS NOT NULL + OR apply.status = 'pending' + OR ( + coalesce( + ( + check_mode.input + ->> 'break_glass_apply_authorized' + )::boolean, + false + ) + AND ( + apply.status IS NULL + OR apply.status = 'pending' + OR ( + apply.status = 'success' + AND learning.status + IS DISTINCT FROM + 'success' + ) + ) + ) + ) + ) + ) + ) + ) ORDER BY candidate.created_at DESC LIMIT 1 """), @@ -1305,6 +1384,9 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once( _EXECUTION_CAPABILITY_FALLBACK_OPERATION_TYPE ), "freshness_hours": freshness_hours, + "retain_controlled_capability_owner": ( + _catalog_id == _WAZUH_INGRESS_CATALOG_ID + ), "project_id": project_id, "work_item_id": _work_item_id, "proposal_source": _scheduler_source, @@ -1325,6 +1407,29 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once( latest_check_status = str( existing_row.get("latest_check_status") or "" ) if existing_row else "" + check_controlled_apply_blocker = str( + existing_row.get("latest_check_controlled_apply_blocker") or "" + ) if existing_row else "" + break_glass_approval_id = str( + existing_row.get("latest_check_break_glass_approval_id") or "" + ) if existing_row else "" + break_glass_apply_authorized = bool( + existing_row + and existing_row.get("latest_check_break_glass_apply_authorized") + is True + ) + break_glass_apply_terminal_status = str( + existing_row.get( + "latest_check_break_glass_apply_terminal_status" + ) + or "" + ) if existing_row else "" + latest_apply_status = str( + existing_row.get("latest_apply_status") or "" + ) if existing_row else "" + latest_learning_status = str( + existing_row.get("latest_learning_status") or "" + ) if existing_row else "" retry_attempt_ref: str | None = None retry_reason = _wazuh_posture_retry_reason( existing_row, @@ -1340,6 +1445,35 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once( existing_row.get("latest_controlled_capability_failure_class") or "" ) if existing_row else "" + capability_queue_status = str( + existing_row.get("latest_controlled_capability_queue_status") + or "" + ) if existing_row else "" + raw_capability_retry_generation = str( + existing_row.get("latest_controlled_capability_retry_generation") + or "0" + ) if existing_row else "0" + capability_retry_generation = ( + int(raw_capability_retry_generation) + if raw_capability_retry_generation.isdigit() + else 0 + ) + capability_packet_retry_owned = bool( + existing_row + and str( + existing_row.get( + "latest_controlled_capability_packet_status" + ) + or "" + ) + == "pending" + and capability_queue_status in {"queued", "retry_exhausted"} + and capability_failure_class + in { + _WAZUH_PRIVILEGED_CONVERGENCE_CAPABILITY_MISSING, + _WAZUH_PRIVILEGED_CONVERGENCE_SECRET_REFERENCE_MISSING, + } + ) critical_secret_reference_pending = bool( capability_failure_class == _WAZUH_PRIVILEGED_CONVERGENCE_SECRET_REFERENCE_MISSING @@ -1347,13 +1481,22 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once( if ( _catalog_id == _WAZUH_INGRESS_CATALOG_ID and retry_reason == "check_mode_failed" - and capability_packet_pending + and capability_packet_retry_owned ): + retry_exhausted = bool( + capability_queue_status == "retry_exhausted" + or capability_retry_generation + >= WAZUH_CONTROLLED_CAPABILITY_MAX_RETRIES + ) return { "status": ( - "critical_secret_reference_break_glass_already_queued" - if critical_secret_reference_pending - else "controlled_privilege_capability_repair_already_queued" + "controlled_capability_same_run_retry_exhausted" + if retry_exhausted + else ( + "critical_secret_reference_break_glass_already_queued" + if critical_secret_reference_pending + else "controlled_privilege_capability_repair_already_queued" + ) ), "queued": 0, "existing": 1, @@ -1369,14 +1512,140 @@ async def enqueue_iwooos_wazuh_manager_posture_candidate_once( ), "retry_reason": retry_reason, "retry_of_failed_check_mode": True, + "capability_packet_active": capability_packet_pending, + "capability_retry_generation": capability_retry_generation, "active_blockers": [ ( - "wazuh_critical_secret_reference_provisioning_pending" - if critical_secret_reference_pending - else "wazuh_controlled_privilege_capability_repair_pending" + "wazuh_same_run_capability_retry_exhausted" + if retry_exhausted + else ( + "wazuh_critical_secret_reference_provisioning_pending" + if critical_secret_reference_pending + else "wazuh_controlled_privilege_capability_repair_pending" + ) ) ], } + if ( + _catalog_id == _WAZUH_INGRESS_CATALOG_ID + and latest_check_status == "success" + and check_controlled_apply_blocker + == _WAZUH_CRITICAL_BREAK_GLASS_RECEIPTS_MISSING + ): + return { + "status": "critical_secret_reference_break_glass_receipts_pending", + "queued": 0, + "existing": 1, + "evidence_ready": 1 if existing_evidence_ready else 0, + "automation_run_id": str( + existing_row.get("automation_run_id") + or existing_row.get("op_id") + or "" + ), + "work_item_id": _work_item_id, + "controlled_apply_allowed": False, + "retry_reason": None, + "retry_of_failed_check_mode": True, + "capability_retry_generation": max( + capability_retry_generation, + int( + str( + existing_row.get( + "latest_check_capability_retry_generation" + ) + or "0" + ) + ) + if str( + existing_row.get( + "latest_check_capability_retry_generation" + ) + or "0" + ).isdigit() + else 0, + ), + "active_blockers": [check_controlled_apply_blocker], + "break_glass_approval_id": break_glass_approval_id, + "canonical_approval_url": ( + "/zh-TW/awooop/approvals" + f"?project_id={project_id}" + f"&incident_id={str(existing_row.get('incident_id') or '')}" + ), + } + if ( + _catalog_id == _WAZUH_INGRESS_CATALOG_ID + and latest_check_status == "success" + and break_glass_apply_authorized + and break_glass_apply_terminal_status + ): + return { + "status": "owner_authorized_break_glass_apply_blocked_terminal", + "queued": 0, + "existing": 1, + "evidence_ready": 1 if existing_evidence_ready else 0, + "automation_run_id": str( + existing_row.get("automation_run_id") + or existing_row.get("op_id") + or "" + ), + "work_item_id": _work_item_id, + "controlled_apply_allowed": False, + "retry_reason": None, + "retry_of_failed_check_mode": True, + "break_glass_approval_id": break_glass_approval_id, + "active_blockers": [break_glass_apply_terminal_status], + } + if ( + _catalog_id == _WAZUH_INGRESS_CATALOG_ID + and latest_check_status == "success" + and break_glass_apply_authorized + and latest_apply_status in {"", "pending"} + ): + return { + "status": "owner_authorized_break_glass_apply_in_progress", + "queued": 0, + "existing": 1, + "evidence_ready": 1 if existing_evidence_ready else 0, + "automation_run_id": str( + existing_row.get("automation_run_id") + or existing_row.get("op_id") + or "" + ), + "work_item_id": _work_item_id, + "controlled_apply_allowed": True, + "retry_reason": None, + "retry_of_failed_check_mode": True, + "break_glass_approval_id": break_glass_approval_id, + "active_blockers": [ + "owner_authorized_break_glass_apply_receipt_pending" + ], + } + if ( + _catalog_id == _WAZUH_INGRESS_CATALOG_ID + and latest_check_status == "success" + and break_glass_apply_authorized + and latest_apply_status == "success" + and latest_learning_status != "success" + ): + return { + "status": "same_run_break_glass_learning_writeback_pending", + "queued": 0, + "existing": 1, + "evidence_ready": 1 if existing_evidence_ready else 0, + "automation_run_id": str( + existing_row.get("automation_run_id") + or existing_row.get("op_id") + or "" + ), + "work_item_id": _work_item_id, + "controlled_apply_allowed": True, + "retry_reason": "post_apply_learning_incomplete", + "retry_of_failed_check_mode": True, + "break_glass_approval_id": break_glass_approval_id, + "active_blockers": [ + "same_run_break_glass_learning_writeback_pending" + ], + } if ( existing_row and retry_reason is None diff --git a/apps/api/src/models/approval.py b/apps/api/src/models/approval.py index 2761b35b8..89ff7d6cc 100644 --- a/apps/api/src/models/approval.py +++ b/apps/api/src/models/approval.py @@ -121,6 +121,22 @@ class Signature(BaseModel): default=None, description="Telegram 訊息 ID", ) + signature_schema_version: str | None = Field( + default=None, + description="Server-verified signature contract version", + ) + principal_id: str | None = Field( + default=None, + description="Authenticated principal bound by the server", + ) + auth_method: str | None = Field( + default=None, + description="Server-side authentication method", + ) + approver_role: str | None = Field( + default=None, + description="Authorized role used for separation of duties", + ) # [首席架構師] 移除 json_encoders (Pydantic v2 已 deprecated)，原生序列化輸出格式與 .isoformat() 一致 v1.1 2026-04-01 Asia/Taipei diff --git a/apps/api/src/services/approval_db.py b/apps/api/src/services/approval_db.py index 75402e956..3f0573479 100644 --- a/apps/api/src/services/approval_db.py +++ b/apps/api/src/services/approval_db.py @@ -38,6 +38,12 @@ from src.services.approval_action_classifier import ( is_executable_repair_approval_action, is_no_action_approval_action, ) +from src.services.wazuh_break_glass_approval import ( + WAZUH_BREAK_GLASS_REQUIRED_SIGNATURE_ROLES, + is_wazuh_break_glass_approval_action, + trusted_wazuh_signature_context, + wazuh_break_glass_signatures_authorized, +) logger = structlog.get_logger(__name__) @@ -89,6 +95,15 @@ def approval_record_to_request(record: ApprovalRecord) -> ApprovalRequest: if sig.get("timestamp") else datetime.now(UTC), comment=sig.get("comment"), + source=sig.get("source", "web"), + telegram_user_id=sig.get("telegram_user_id"), + telegram_message_id=sig.get("telegram_message_id"), + signature_schema_version=sig.get( + "signature_schema_version" + ), + principal_id=sig.get("principal_id"), + auth_method=sig.get("auth_method"), + approver_role=sig.get("approver_role"), ) ) @@ -718,6 +733,13 @@ class ApprovalDBService: signer_id: str, signer_name: str, comment: str | None = None, + *, + authenticated_principal_id: str | None = None, + authentication_method: str | None = None, + approver_role: str | None = None, + signature_source: str | None = None, + telegram_user_id: int | None = None, + telegram_message_id: int | None = None, ) -> tuple[ApprovalRequest | None, str, bool]: """ 簽核授權請求 @@ -756,6 +778,23 @@ class ApprovalDBService: False, ) + trusted_context: dict[str, str] | None = None + if is_wazuh_break_glass_approval_action(record.action): + trusted_context = trusted_wazuh_signature_context( + principal_id=str(authenticated_principal_id or ""), + auth_method=str(authentication_method or ""), + approver_role=str(approver_role or ""), + ) + if trusted_context is None: + return ( + approval_record_to_request(record), + "Cannot sign: authenticated Wazuh break-glass " + "principal and role are required", + False, + ) + signer_id = trusted_context["principal_id"] + signer_name = signer_id + # 檢查是否已簽核 signatures = record.signatures or [] for sig in signatures: @@ -765,6 +804,17 @@ class ApprovalDBService: f"User {signer_name} has already signed this approval", False, ) + if ( + trusted_context is not None + and sig.get("approver_role") + == trusted_context["approver_role"] + ): + return ( + approval_record_to_request(record), + "Cannot sign: Wazuh break-glass approver role " + "already signed", + False, + ) # Phase 5: 樂觀鎖 - 記錄更新前的簽名數 old_sig_count = record.current_signatures @@ -776,6 +826,14 @@ class ApprovalDBService: "timestamp": datetime.now(UTC).isoformat(), "comment": comment, } + if signature_source: + new_signature["source"] = signature_source + if telegram_user_id is not None: + new_signature["telegram_user_id"] = telegram_user_id + if telegram_message_id is not None: + new_signature["telegram_message_id"] = telegram_message_id + if trusted_context is not None: + new_signature.update(trusted_context) signatures.append(new_signature) new_sig_count = len(signatures) @@ -783,7 +841,13 @@ class ApprovalDBService: execution_triggered = False new_status = record.status resolved_at = None - if new_sig_count >= record.required_signatures: + threshold_met = new_sig_count >= record.required_signatures + if is_wazuh_break_glass_approval_action(record.action): + threshold_met = ( + threshold_met + and wazuh_break_glass_signatures_authorized(signatures) + ) + if threshold_met: new_status = ApprovalStatus.APPROVED resolved_at = datetime.now(UTC) execution_triggered = is_executable_repair_approval_action( @@ -793,6 +857,11 @@ class ApprovalDBService: # Phase 5: 樂觀鎖更新 - 使用 WHERE current_signatures = old_value # 如果其他人已更新，這個 UPDATE 會更新 0 行 metadata = dict(record.extra_metadata or {}) + if is_wazuh_break_glass_approval_action(record.action): + metadata["trusted_signature_roles"] = sorted( + WAZUH_BREAK_GLASS_REQUIRED_SIGNATURE_ROLES + ) + metadata["trusted_signature_roles_verified"] = threshold_met if is_no_action_approval_action(record.action): metadata["execution_kind"] = metadata.get("execution_kind") or "no_action" metadata["repair_executed"] = False diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index 4c17a46bf..d2cc812de 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -14,7 +14,7 @@ import re import shutil import signal import time -from collections.abc import Mapping +from collections.abc import Callable, Mapping from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta from pathlib import Path @@ -23,11 +23,12 @@ from uuid import NAMESPACE_URL, UUID, uuid4, uuid5 import structlog from sqlalchemy import select, text +from sqlalchemy.dialects.postgresql import insert as pg_insert from src.core.config import settings from src.db.base import get_db_context from src.db.models import ApprovalRecord, PlaybookRecord -from src.models.approval import ApprovalStatus +from src.models.approval import ApprovalStatus, RiskLevel from src.services.ai_automation_runtime_contract import ( AI_AUTOMATION_STAGE_RECEIPT_SCHEMA_VERSION, ) @@ -49,6 +50,15 @@ from src.services.awooop_ansible_post_verifier import ( run_ansible_asset_post_verifier, ) from src.services.playbook_evolver import TRUST_ARCHIVE_THRESHOLD +from src.services.wazuh_break_glass_approval import ( + CONTROLLED_OPERATOR_APPROVER_ROLE, + CONTROLLED_OPERATOR_AUTH_METHOD, + TELEGRAM_OWNER_APPROVER_ROLE, + TELEGRAM_OWNER_AUTH_METHOD, + WAZUH_BREAK_GLASS_APPROVAL_ACTION, + WAZUH_BREAK_GLASS_REQUIRED_SIGNATURE_ROLES, + WAZUH_BREAK_GLASS_SIGNATURE_SCHEMA_VERSION, +) logger = structlog.get_logger(__name__) @@ -109,6 +119,18 @@ _STALE_CLAIM_REPLAY_CONTROL_FIELDS = ( "semantic_repair_of_apply_op_id", "semantic_route_reconciliation", ) +_WAZUH_CAPABILITY_RETRY_CONTROL_FIELDS = ( + "bounded_execution_scope", + "break_glass_approval_id", + "capability_retry_failure_class", + "capability_retry_generation", + "capability_retry_of_packet_id", + "capability_retry_packet_receipt_id", + "execution_mode", + "replay_of_check_mode_op_id", + "same_run_correlation_verified", + "source_recurrence_replay_authority", +) _HISTORICAL_SHADOW_REPLAY_TAGS = frozenset( {"catalog_drift_replay", "semantic_catalog_reconciliation"} ) @@ -143,6 +165,15 @@ _WAZUH_CONTROLLED_CAPABILITY_PACKET_OPERATION_TYPE = ( _WAZUH_CONTROLLED_CAPABILITY_PACKET_SCHEMA_VERSION = ( "wazuh_controlled_privileged_capability_repair_v3" ) +WAZUH_CONTROLLED_CAPABILITY_MAX_RETRIES = 1 +_WAZUH_CONTROLLED_CAPABILITY_RETRY_WINDOW_HOURS = 7 * 24 +_WAZUH_CRITICAL_BREAK_GLASS_RECEIPTS_MISSING = ( + "critical_secret_reference_owner_approval_and_broker_receipts_missing" +) +_WAZUH_BREAK_GLASS_APPROVAL_ACTION = WAZUH_BREAK_GLASS_APPROVAL_ACTION +_WAZUH_BECOME_SECRET_REFERENCE = Path( + "/run/secrets/host112-become/password" +) _EXECUTION_CAPABILITY_OPERATION_TYPES = frozenset( { "ansible_executor_capability_issued", @@ -356,6 +387,135 @@ def _check_mode_known_hosts_path() -> Path: return Path(settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH) +def _wazuh_become_secret_reference_ready() -> bool: + """Check opaque projection metadata without reading secret content.""" + + reference = Path(settings.AWOOOP_ANSIBLE_BECOME_PASSWORD_FILE) + if reference != _WAZUH_BECOME_SECRET_REFERENCE: + return False + try: + return bool( + reference.is_file() + and os.access(reference, os.R_OK) + and reference.stat().st_size > 0 + ) + except OSError: + return False + + +def _wazuh_break_glass_approval_id(capability_packet_id: str) -> str: + return str( + uuid5( + NAMESPACE_URL, + "awoooi:wazuh-secret-reference-break-glass:" + f"{capability_packet_id}", + ) + ) + + +async def _ensure_wazuh_break_glass_approval( + db: Any, + *, + packet_receipt_id: str, + capability_packet_id: str, + incident_id: str, + source_candidate_op_id: str, + source_check_mode_op_id: str, + automation_run_id: str, +) -> str: + """Persist the owner-signable gate for one critical secret packet.""" + + approval_id = _wazuh_break_glass_approval_id(capability_packet_id) + metadata = { + "schema_version": "wazuh_secret_reference_break_glass_approval_v1", + "capability_packet_receipt_id": packet_receipt_id, + "capability_packet_id": capability_packet_id, + "source_candidate_op_id": source_candidate_op_id, + "source_check_mode_op_id": source_check_mode_op_id, + "automation_run_id": automation_run_id, + "catalog_id": _WAZUH_ALERTMANAGER_INTEGRATION_CATALOG_ID, + "canonical_asset_id": "service:wazuh-manager", + "secret_value_read": False, + "secret_value_collection_allowed": False, + "runtime_write_authorized": False, + "trusted_signature_schema_version": ( + WAZUH_BREAK_GLASS_SIGNATURE_SCHEMA_VERSION + ), + "trusted_signature_roles": sorted( + WAZUH_BREAK_GLASS_REQUIRED_SIGNATURE_ROLES + ), + "trusted_signature_auth_methods": sorted( + { + TELEGRAM_OWNER_AUTH_METHOD, + CONTROLLED_OPERATOR_AUTH_METHOD, + } + ), + } + await db.execute( + pg_insert(ApprovalRecord) + .values( + id=approval_id, + action=_WAZUH_BREAK_GLASS_APPROVAL_ACTION, + description=( + "Authorize the bounded Wazuh alert ingress apply only after " + "the broker persists opaque secret projection and same-run " + "check-mode receipts. No secret value is collected." + ), + status=ApprovalStatus.PENDING, + risk_level=RiskLevel.CRITICAL, + required_signatures=2, + current_signatures=0, + signatures=[], + blast_radius={ + "affected_pods": 1, + "estimated_downtime": "0", + "related_services": ["wazuh-manager"], + "data_impact": "write", + }, + dry_run_checks=[ + { + "name": "same_run_check_mode", + "passed": False, + "message": "awaiting broker no-write retry receipt", + } + ], + requested_by="ansible_controlled_capability_retry_broker", + extra_metadata=metadata, + fingerprint=( + f"wazuh-secret-reference-break-glass:{capability_packet_id}" + ), + hit_count=1, + approval_level="critical", + approval_votes=[], + required_votes=2, + incident_id=incident_id, + ) + .on_conflict_do_nothing(index_elements=[ApprovalRecord.id]) + ) + await db.execute( + text(""" + UPDATE automation_operation_log packet + SET input = coalesce(packet.input, '{}'::jsonb) + || jsonb_build_object( + 'break_glass_approval_id', + CAST(:approval_id AS text), + 'owner_approval_required', true, + 'owner_approval_schema_version', + 'wazuh_secret_reference_break_glass_approval_v1' + ) + WHERE packet.op_id = CAST(:packet_receipt_id AS uuid) + AND packet.input ->> 'capability_packet_id' = + :capability_packet_id + """), + { + "approval_id": approval_id, + "packet_receipt_id": packet_receipt_id, + "capability_packet_id": capability_packet_id, + }, + ) + return approval_id + + def _uses_repair_forced_command_transport(key_path: Path | None = None) -> bool: return (key_path or _check_mode_ssh_key_path()) == REPAIR_FORCED_COMMAND_KEY_PATH @@ -1866,10 +2026,155 @@ def _durable_post_verifier_receipt_for_backfill( return receipt if all(checks) else None +def _wazuh_capability_retry_replay_observed_now( + input_payload: Mapping[str, Any], + receipt_row: Mapping[str, Any], +) -> datetime | None: + """Validate the durable authority that permits one historical same-run replay.""" + + if ( + input_payload.get("execution_mode") + != "wazuh_controlled_capability_check_mode_retry" + or input_payload.get("same_run_correlation_verified") is not True + or _int_from_value( + input_payload.get("capability_retry_generation"), + default=0, + ) + != WAZUH_CONTROLLED_CAPABILITY_MAX_RETRIES + or input_payload.get("capability_retry_failure_class") + not in _WAZUH_PRIVILEGED_CONVERGENCE_FAILURE_MARKERS + or input_payload.get("bounded_execution_scope") + not in { + "same_run_check_mode_only_until_owner_authorized_" + "break_glass_receipts", + "same_run_check_mode_then_existing_controlled_apply_policy", + } + ): + return None + packet_id = str( + input_payload.get("capability_retry_of_packet_id") or "" + ) + packet_receipt_id = str( + input_payload.get("capability_retry_packet_receipt_id") or "" + ) + break_glass_approval_id = str( + input_payload.get("break_glass_approval_id") or "" + ) + source_check_mode_op_id = str( + input_payload.get("replay_of_check_mode_op_id") or "" + ) + source_candidate_op_id = str( + input_payload.get("source_candidate_op_id") or "" + ) + automation_run_id = str( + input_payload.get("automation_run_id") or "" + ) + if not ( + packet_id.startswith("IWZ-CC-") + and len(packet_id) == 23 + and packet_receipt_id + and ( + input_payload.get("capability_retry_failure_class") + != _WAZUH_PRIVILEGED_CONVERGENCE_SECRET_REFERENCE_MISSING + or break_glass_approval_id + == _wazuh_break_glass_approval_id(packet_id) + ) + and source_check_mode_op_id + and automation_run_id + == str(input_payload.get("trace_id") or "") + == str(input_payload.get("run_id") or "") + == source_candidate_op_id + ): + return None + try: + UUID(source_check_mode_op_id) + UUID(source_candidate_op_id) + UUID(packet_receipt_id) + except ValueError: + return None + authority = input_payload.get("source_recurrence_replay_authority") + source_recurrence = input_payload.get("source_recurrence") + if not isinstance(authority, Mapping) or not isinstance( + source_recurrence, Mapping + ): + return None + recurrence_observed_at = str( + source_recurrence.get("observed_at") or "" + ) + if not ( + authority.get("schema_version") + == "wazuh_capability_same_run_retry_authority_v1" + and authority.get("capability_packet_id") == packet_id + and authority.get("capability_packet_receipt_id") + == packet_receipt_id + and authority.get("break_glass_approval_id") + == break_glass_approval_id + and authority.get("source_check_mode_op_id") + == source_check_mode_op_id + and authority.get("source_candidate_op_id") + == source_candidate_op_id + and authority.get("automation_run_id") == automation_run_id + and authority.get("capability_retry_generation") + == WAZUH_CONTROLLED_CAPABILITY_MAX_RETRIES + and authority.get("source_recurrence_observed_at") + == recurrence_observed_at + and authority.get( + "secret_reference_presence_verified_without_read" + ) + is ( + input_payload.get("capability_retry_failure_class") + == _WAZUH_PRIVILEGED_CONVERGENCE_SECRET_REFERENCE_MISSING + ) + and authority.get("secret_value_read") is False + and str(receipt_row.get("retry_packet_receipt_id") or "") + == packet_receipt_id + and receipt_row.get("retry_packet_status") == "success" + and receipt_row.get("retry_packet_queue_status") == "retry_claimed" + and str(receipt_row.get("retry_packet_id") or "") == packet_id + and str(receipt_row.get("retry_packet_parent_op_id") or "") + == source_check_mode_op_id + and str(receipt_row.get("retry_packet_source_check_mode_op_id") or "") + == source_check_mode_op_id + and str(receipt_row.get("retry_packet_source_candidate_op_id") or "") + == source_candidate_op_id + and str(receipt_row.get("retry_packet_automation_run_id") or "") + == automation_run_id + and str(receipt_row.get("retry_packet_claimed_check_mode_op_id") or "") + == str(receipt_row.get("op_id") or "") + and receipt_row.get("retry_packet_claim_receipt_persisted") is True + and receipt_row.get("retry_packet_host_write_performed") is False + and receipt_row.get("retry_packet_secret_value_read") is False + and receipt_row.get( + "retry_packet_authorized_executor_dispatch_receipt_persisted" + ) + is True + and str(receipt_row.get("retry_packet_authorized_executor") or "") + == "awoooi-ansible-executor-broker" + and str(receipt_row.get("retry_packet_failure_class") or "") + == str(input_payload.get("capability_retry_failure_class") or "") + ): + return None + try: + observed_now = datetime.fromisoformat( + recurrence_observed_at.replace("Z", "+00:00") + ) + except ValueError: + return None + return observed_now if observed_now.tzinfo is not None else None + + def _claim_from_stale_check_mode_row( row: dict[str, Any], + *, + replay_observed_now: datetime | None = None, ) -> AnsibleCheckModeClaim | None: input_payload = _json_loads(row.get("input")) + capability_retry_replay_observed_now = ( + _wazuh_capability_retry_replay_observed_now(input_payload, row) + ) + effective_replay_observed_now = ( + replay_observed_now or capability_retry_replay_observed_now + ) source_candidate_op_id = str( input_payload.get("source_candidate_op_id") or row.get("parent_op_id") @@ -1969,6 +2274,7 @@ def _claim_from_stale_check_mode_row( } ], }, + observed_now=effective_replay_observed_now, ) except ValueError: return None @@ -1982,6 +2288,26 @@ def _claim_from_stale_check_mode_row( for field_name in _STALE_CLAIM_REPLAY_CONTROL_FIELDS: if field_name in input_payload: claim_input[field_name] = input_payload[field_name] + if capability_retry_replay_observed_now is not None: + for field_name in _WAZUH_CAPABILITY_RETRY_CONTROL_FIELDS: + if field_name in input_payload: + claim_input[field_name] = input_payload[field_name] + if ( + input_payload.get("capability_retry_failure_class") + == _WAZUH_PRIVILEGED_CONVERGENCE_SECRET_REFERENCE_MISSING + ): + claim_input.update( + { + "risk_level": "critical", + "apply_enabled": False, + "approval_required_before_apply": True, + "owner_review_required": True, + "controlled_apply_allowed": False, + "controlled_apply_blocker": ( + _WAZUH_CRITICAL_BREAK_GLASS_RECEIPTS_MISSING + ), + } + ) row_tags = { str(tag) for tag in (row.get("tags") or []) @@ -8449,6 +8775,778 @@ async def run_failed_apply_check_mode_replay_once( return stats +async def claim_expired_wazuh_controlled_capability_retries( + *, + project_id: str = "awoooi", + limit: int = 1, + window_hours: int = _WAZUH_CONTROLLED_CAPABILITY_RETRY_WINDOW_HOURS, + secret_reference_ready: Callable[ + [], bool + ] = _wazuh_become_secret_reference_ready, +) -> list[AnsibleCheckModeClaim]: + """Claim one bounded same-run retry per expired Wazuh capability packet.""" + + claims: list[AnsibleCheckModeClaim] = [] + async with get_db_context(project_id) as db: + result = await db.execute( + text(f""" + SELECT + check_mode.op_id, + check_mode.parent_op_id, + coalesce( + check_mode.input ->> 'incident_id', + check_mode.incident_id::text + ) AS incident_id, + check_mode.input, + packet.op_id::text AS capability_packet_receipt_id, + packet.input ->> 'capability_packet_id' + AS capability_packet_id, + packet.input ->> 'automation_run_id' + AS packet_automation_run_id, + packet.input ->> 'source_candidate_op_id' + AS packet_source_candidate_op_id, + packet.input ->> 'source_check_mode_op_id' + AS packet_source_check_mode_op_id, + packet.output ->> 'failure_class' AS failure_class + FROM automation_operation_log check_mode + JOIN automation_operation_log packet + ON packet.op_id = ( + SELECT capability.op_id + FROM automation_operation_log capability + WHERE capability.parent_op_id = check_mode.op_id + AND ( + capability.operation_type = + '{_WAZUH_CONTROLLED_CAPABILITY_PACKET_OPERATION_TYPE}' + OR ( + capability.operation_type = + '{_EXECUTION_CAPABILITY_FALLBACK_OPERATION_TYPE}' + AND capability.input ->> 'semantic_operation_type' = + '{_WAZUH_CONTROLLED_CAPABILITY_PACKET_OPERATION_TYPE}' + ) + ) + AND capability.status = 'pending' + ORDER BY capability.created_at DESC + LIMIT 1 + ) + WHERE check_mode.operation_type = 'ansible_check_mode_executed' + AND check_mode.status = 'failed' + AND check_mode.input ->> 'catalog_id' = + :wazuh_catalog_id + AND check_mode.created_at >= NOW() - ( + :window_hours * INTERVAL '1 hour' + ) + AND packet.output ->> 'queue_status' = 'queued' + AND packet.output ->> 'failure_class' IN ( + :capability_missing, + :secret_reference_missing + ) + AND nullif( + packet.input #>> '{{gate_lifecycle,expires_at}}', + '' + )::timestamptz <= NOW() + AND CASE + WHEN check_mode.input ->> 'capability_retry_generation' + ~ '^[0-9]+$' + THEN ( + check_mode.input ->> 'capability_retry_generation' + )::int + ELSE 0 + END < :max_retries + AND NOT EXISTS ( + SELECT 1 + FROM automation_operation_log replay + WHERE replay.operation_type = 'ansible_check_mode_executed' + AND replay.input ->> 'capability_retry_of_packet_id' = + packet.input ->> 'capability_packet_id' + ) + ORDER BY check_mode.created_at ASC + LIMIT :scan_limit + FOR UPDATE OF check_mode, packet SKIP LOCKED + """), + { + "wazuh_catalog_id": ( + _WAZUH_ALERTMANAGER_INTEGRATION_CATALOG_ID + ), + "window_hours": max( + 1, + min( + window_hours, + _WAZUH_CONTROLLED_CAPABILITY_RETRY_WINDOW_HOURS, + ), + ), + "capability_missing": ( + _WAZUH_PRIVILEGED_CONVERGENCE_CAPABILITY_MISSING + ), + "secret_reference_missing": ( + _WAZUH_PRIVILEGED_CONVERGENCE_SECRET_REFERENCE_MISSING + ), + "max_retries": WAZUH_CONTROLLED_CAPABILITY_MAX_RETRIES, + "scan_limit": min(100, max(10, limit * 10)), + }, + ) + for row_value in result.mappings().all(): + row = dict(row_value) + failed_check_mode_op_id = str(row.get("op_id") or "") + source_candidate_op_id = str(row.get("parent_op_id") or "") + packet_receipt_id = str( + row.get("capability_packet_receipt_id") or "" + ) + packet_id = str(row.get("capability_packet_id") or "") + failure_class = str(row.get("failure_class") or "") + original_input = _json_loads(row.get("input")) + original_generation = _int_from_value( + original_input.get("capability_retry_generation"), + default=0, + ) + if not ( + failed_check_mode_op_id + and source_candidate_op_id + and packet_receipt_id + and packet_id.startswith("IWZ-CC-") + and len(packet_id) == 23 + and failure_class + in _WAZUH_PRIVILEGED_CONVERGENCE_FAILURE_MARKERS + and 0 <= original_generation + < WAZUH_CONTROLLED_CAPABILITY_MAX_RETRIES + and str(row.get("packet_source_check_mode_op_id") or "") + == failed_check_mode_op_id + and str(row.get("packet_source_candidate_op_id") or "") + == source_candidate_op_id + ): + continue + secret_reference_retry = bool( + failure_class + == _WAZUH_PRIVILEGED_CONVERGENCE_SECRET_REFERENCE_MISSING + ) + break_glass_approval_id = "" + if secret_reference_retry: + break_glass_approval_id = ( + await _ensure_wazuh_break_glass_approval( + db, + packet_receipt_id=packet_receipt_id, + capability_packet_id=packet_id, + incident_id=str(row.get("incident_id") or ""), + source_candidate_op_id=source_candidate_op_id, + source_check_mode_op_id=failed_check_mode_op_id, + automation_run_id=str( + row.get("packet_automation_run_id") or "" + ), + ) + ) + if not secret_reference_ready(): + continue + source_recurrence = original_input.get("source_recurrence") + if not isinstance(source_recurrence, Mapping): + continue + try: + replay_observed_now = datetime.fromisoformat( + str(source_recurrence.get("observed_at") or "").replace( + "Z", "+00:00" + ) + ) + except ValueError: + continue + if replay_observed_now.tzinfo is None: + continue + original = _claim_from_stale_check_mode_row( + row, + replay_observed_now=replay_observed_now, + ) + if original is None: + continue + automation_run_id = str( + original.input_payload.get("automation_run_id") or "" + ) + same_run_correlation_verified = bool( + automation_run_id + == str(original.input_payload.get("trace_id") or "") + == str(original.input_payload.get("run_id") or "") + == original.source_candidate_op_id + == source_candidate_op_id + == str(row.get("packet_automation_run_id") or "") + ) + if not same_run_correlation_verified: + continue + retry_generation = original_generation + 1 + replay_input = { + **original.input_payload, + "execution_mode": ( + "wazuh_controlled_capability_check_mode_retry" + ), + "replay_of_check_mode_op_id": failed_check_mode_op_id, + "capability_retry_packet_receipt_id": packet_receipt_id, + "capability_retry_of_packet_id": packet_id, + "capability_retry_failure_class": failure_class, + "capability_retry_generation": retry_generation, + "break_glass_approval_id": break_glass_approval_id, + "same_run_correlation_verified": True, + "source_recurrence_replay_authority": { + "schema_version": ( + "wazuh_capability_same_run_retry_authority_v1" + ), + "capability_packet_id": packet_id, + "capability_packet_receipt_id": packet_receipt_id, + "break_glass_approval_id": break_glass_approval_id, + "source_check_mode_op_id": failed_check_mode_op_id, + "source_candidate_op_id": source_candidate_op_id, + "automation_run_id": automation_run_id, + "capability_retry_generation": retry_generation, + "source_recurrence_observed_at": str( + source_recurrence.get("observed_at") or "" + ), + "secret_reference_presence_verified_without_read": ( + secret_reference_retry + ), + "secret_value_read": False, + }, + "bounded_execution_scope": ( + "same_run_check_mode_only_until_owner_authorized_" + "break_glass_receipts" + if secret_reference_retry + else "same_run_check_mode_then_existing_controlled_apply_policy" + ), + } + if secret_reference_retry: + replay_input.update( + { + "risk_level": "critical", + "apply_enabled": False, + "approval_required_before_apply": True, + "owner_review_required": True, + "controlled_apply_allowed": False, + "controlled_apply_blocker": ( + _WAZUH_CRITICAL_BREAK_GLASS_RECEIPTS_MISSING + ), + } + ) + inserted = await db.execute( + text(""" + WITH claimed_packet AS ( + UPDATE automation_operation_log packet + SET status = 'success', + input = coalesce(packet.input, '{}'::jsonb) + || jsonb_build_object( + 'windows99_dispatch_required', false, + 'authorized_executor_dispatch_required', + true, + 'authorized_executors', jsonb_build_array( + 'Agent99', + 'awoooi-ansible-executor-broker' + ), + 'break_glass_approval_id', + CAST(:break_glass_approval_id AS text), + 'owner_approval_required', + :secret_projection_verified + ), + output = coalesce(packet.output, '{}'::jsonb) + || jsonb_build_object( + 'queue_status', 'retry_claiming', + 'claim_receipt_persisted', false, + 'claimed_at', NOW(), + 'opaque_secret_projection_presence_verified', + :secret_projection_verified, + 'authorized_executor', + 'awoooi-ansible-executor-broker', + 'authorized_executor_dispatch_receipt_persisted', + true, + 'secret_value_read', false, + 'host_write_performed', false + ) + WHERE packet.op_id = CAST( + :capability_packet_receipt_id AS uuid + ) + AND packet.status = 'pending' + AND packet.output ->> 'queue_status' = 'queued' + AND NOT EXISTS ( + SELECT 1 + FROM automation_operation_log existing_replay + WHERE existing_replay.operation_type = + 'ansible_check_mode_executed' + AND existing_replay.input + ->> 'capability_retry_of_packet_id' = + :capability_packet_id + ) + RETURNING packet.op_id + ), replay AS ( + INSERT INTO automation_operation_log ( + operation_type, actor, status, incident_id, + input, output, dry_run_result, + parent_op_id, tags + ) + SELECT + 'ansible_check_mode_executed', + 'ansible_controlled_capability_retry_broker', + 'pending', + :incident_db_id, + CAST(:input AS jsonb), + '{}'::jsonb, + CAST(:dry_run_result AS jsonb), + CAST(:parent_op_id AS uuid), + :tags + FROM claimed_packet + RETURNING op_id + ), finalized_packet AS ( + UPDATE automation_operation_log packet + SET output = coalesce(packet.output, '{}'::jsonb) + || jsonb_build_object( + 'queue_status', 'retry_claimed', + 'claimed_retry_check_mode_op_id', + replay.op_id::text, + 'claim_receipt_persisted', true, + 'capability_retry_generation', + :retry_generation, + 'automatic_retry_remaining', 0, + 'automatic_retry_exhausted', true + ) + FROM replay + WHERE packet.op_id = CAST( + :capability_packet_receipt_id AS uuid + ) + RETURNING replay.op_id + ) + SELECT op_id FROM finalized_packet + """), + { + "incident_db_id": _automation_operation_log_incident_id( + original.incident_id + ), + "input": json.dumps(replay_input, ensure_ascii=False), + "dry_run_result": json.dumps( + { + "execution_mode": ( + "wazuh_controlled_capability_check_mode_retry" + ), + "capability_retry_of_packet_id": packet_id, + "capability_retry_generation": retry_generation, + "check_mode_executed": False, + "apply_executed": False, + "runtime_apply_executed": False, + "claim_state": "claimed", + }, + ensure_ascii=False, + ), + "parent_op_id": original.source_candidate_op_id, + "capability_packet_receipt_id": packet_receipt_id, + "capability_packet_id": packet_id, + "break_glass_approval_id": break_glass_approval_id, + "retry_generation": retry_generation, + "secret_projection_verified": secret_reference_retry, + "tags": [ + "ansible", + "wazuh", + "check_mode", + "same_run_capability_retry", + "controlled_apply_policy_recheck", + ], + }, + ) + op_id_value = inserted.scalar_one_or_none() + if op_id_value is None: + continue + claims.append( + replace( + original, + op_id=str(op_id_value), + risk_level=( + "critical" + if secret_reference_retry + else original.risk_level + ), + input_payload=replay_input, + ) + ) + if len(claims) >= max(1, limit): + break + return claims + + +async def claim_owner_authorized_wazuh_secret_retry_applies( + *, + project_id: str = "awoooi", + limit: int = 1, + stale_after_seconds: int = 300, +) -> list[AnsibleCheckModeClaim]: + """Claim successful secret probes only after durable owner approval.""" + + claims: list[AnsibleCheckModeClaim] = [] + async with get_db_context(project_id) as db: + result = await db.execute( + text(""" + SELECT + check_mode.op_id, + check_mode.parent_op_id, + check_mode.tags, + coalesce( + check_mode.incident_id::text, + check_mode.input ->> 'incident_id' + ) AS incident_id, + check_mode.input, + packet.op_id::text AS retry_packet_receipt_id, + packet.status AS retry_packet_status, + packet.parent_op_id::text AS retry_packet_parent_op_id, + packet.input ->> 'capability_packet_id' + AS retry_packet_id, + packet.input ->> 'source_check_mode_op_id' + AS retry_packet_source_check_mode_op_id, + packet.input ->> 'source_candidate_op_id' + AS retry_packet_source_candidate_op_id, + packet.input ->> 'automation_run_id' + AS retry_packet_automation_run_id, + packet.output ->> 'queue_status' + AS retry_packet_queue_status, + packet.output ->> 'claimed_retry_check_mode_op_id' + AS retry_packet_claimed_check_mode_op_id, + packet.output ->> 'failure_class' + AS retry_packet_failure_class, + packet.output ->> 'claim_receipt_persisted' = 'true' + AS retry_packet_claim_receipt_persisted, + packet.output ->> 'host_write_performed' = 'true' + AS retry_packet_host_write_performed, + packet.output ->> 'secret_value_read' = 'true' + AS retry_packet_secret_value_read, + packet.output + ->> 'authorized_executor_dispatch_receipt_persisted' + = 'true' + AS retry_packet_authorized_executor_dispatch_receipt_persisted, + packet.output ->> 'authorized_executor' + AS retry_packet_authorized_executor, + packet.output + ->> 'opaque_secret_projection_presence_verified' + = 'true' + AS opaque_secret_projection_presence_verified, + approval.id::text AS break_glass_approval_id, + approval.resolved_at AS break_glass_approval_resolved_at + FROM automation_operation_log check_mode + JOIN automation_operation_log packet + ON packet.op_id::text = check_mode.input + ->> 'capability_retry_packet_receipt_id' + JOIN approval_records approval + ON approval.id = check_mode.input + ->> 'break_glass_approval_id' + WHERE check_mode.operation_type = 'ansible_check_mode_executed' + AND check_mode.status = 'success' + AND coalesce( + check_mode.output ->> 'returncode', + check_mode.dry_run_result ->> 'returncode' + ) = '0' + AND check_mode.input ->> 'catalog_id' = + :wazuh_catalog_id + AND check_mode.input ->> 'execution_mode' = + 'wazuh_controlled_capability_check_mode_retry' + AND check_mode.input ->> 'capability_retry_failure_class' = + :secret_reference_missing + AND check_mode.input ->> 'controlled_apply_blocker' = + :required_blocker + AND packet.status = 'success' + AND packet.output ->> 'queue_status' = 'retry_claimed' + AND packet.output ->> 'claim_receipt_persisted' = 'true' + AND packet.output + ->> 'opaque_secret_projection_presence_verified' + = 'true' + AND packet.output ->> 'secret_value_read' = 'false' + AND packet.output ->> 'host_write_performed' = 'false' + AND packet.output ->> 'claimed_retry_check_mode_op_id' = + check_mode.op_id::text + AND packet.input ->> 'break_glass_approval_id' = + approval.id + AND packet.input ->> 'capability_packet_id' = + check_mode.input ->> 'capability_retry_of_packet_id' + AND approval.action = :approval_action + AND lower(approval.status::text) = 'approved' + AND lower(approval.risk_level::text) = 'critical' + AND approval.required_signatures >= 2 + AND approval.current_signatures >= + approval.required_signatures + AND jsonb_typeof(CAST(approval.signatures AS jsonb)) = 'array' + AND jsonb_array_length(CAST(approval.signatures AS jsonb)) >= + approval.required_signatures + AND EXISTS ( + SELECT 1 + FROM jsonb_array_elements( + CASE + WHEN jsonb_typeof( + CAST(approval.signatures AS jsonb) + ) = 'array' + THEN CAST(approval.signatures AS jsonb) + ELSE '[]'::jsonb + END + ) trusted_signature + WHERE trusted_signature + ->> 'signature_schema_version' = + :trusted_signature_schema_version + AND trusted_signature ->> 'principal_id' = + trusted_signature ->> 'signer_id' + AND trusted_signature ->> 'auth_method' = + :telegram_owner_auth_method + AND trusted_signature ->> 'approver_role' = + :telegram_owner_approver_role + AND trusted_signature ->> 'source' = 'telegram' + AND trusted_signature ->> 'telegram_user_id' + ~ '^[0-9]+$' + AND trusted_signature ->> 'principal_id' = + 'tg_' || ( + trusted_signature + ->> 'telegram_user_id' + ) + ) + AND EXISTS ( + SELECT 1 + FROM jsonb_array_elements( + CASE + WHEN jsonb_typeof( + CAST(approval.signatures AS jsonb) + ) = 'array' + THEN CAST(approval.signatures AS jsonb) + ELSE '[]'::jsonb + END + ) trusted_signature + WHERE trusted_signature + ->> 'signature_schema_version' = + :trusted_signature_schema_version + AND trusted_signature ->> 'principal_id' = + trusted_signature ->> 'signer_id' + AND trusted_signature ->> 'auth_method' = + :controlled_operator_auth_method + AND trusted_signature ->> 'approver_role' = + :controlled_operator_approver_role + AND trusted_signature ->> 'source' = 'api' + ) + AND ( + SELECT count( + DISTINCT trusted_signature ->> 'principal_id' + ) + FROM jsonb_array_elements( + CASE + WHEN jsonb_typeof( + CAST(approval.signatures AS jsonb) + ) = 'array' + THEN CAST(approval.signatures AS jsonb) + ELSE '[]'::jsonb + END + ) trusted_signature + WHERE trusted_signature + ->> 'signature_schema_version' = + :trusted_signature_schema_version + AND trusted_signature ->> 'principal_id' = + trusted_signature ->> 'signer_id' + AND trusted_signature ->> 'approver_role' IN ( + :telegram_owner_approver_role, + :controlled_operator_approver_role + ) + ) >= 2 + AND approval.resolved_at IS NOT NULL + AND ( + approval.expires_at IS NULL + OR approval.expires_at > NOW() + ) + AND approval.incident_id = coalesce( + check_mode.incident_id::text, + check_mode.input ->> 'incident_id' + ) + AND CAST(approval.extra_metadata AS jsonb) + ->> 'schema_version' = + 'wazuh_secret_reference_break_glass_approval_v1' + AND CAST(approval.extra_metadata AS jsonb) + ->> 'capability_packet_receipt_id' = + packet.op_id::text + AND CAST(approval.extra_metadata AS jsonb) + ->> 'capability_packet_id' = + packet.input ->> 'capability_packet_id' + AND CAST(approval.extra_metadata AS jsonb) + ->> 'source_candidate_op_id' = + check_mode.parent_op_id::text + AND CAST(approval.extra_metadata AS jsonb) + ->> 'source_check_mode_op_id' = + packet.parent_op_id::text + AND CAST(approval.extra_metadata AS jsonb) + ->> 'automation_run_id' = + check_mode.input ->> 'automation_run_id' + AND NOT EXISTS ( + SELECT 1 + FROM automation_operation_log apply + WHERE apply.operation_type = 'ansible_apply_executed' + AND apply.parent_op_id = check_mode.op_id + ) + AND ( + check_mode.input ->> 'break_glass_apply_claimed_at' + IS NULL + OR nullif( + check_mode.input + ->> 'break_glass_apply_claimed_at', + '' + )::timestamptz <= NOW() - ( + :stale_after_seconds * INTERVAL '1 second' + ) + ) + AND coalesce( + check_mode.input + ->> 'break_glass_apply_terminal_status', + '' + ) = '' + ORDER BY approval.resolved_at ASC, check_mode.created_at ASC + LIMIT :limit + FOR UPDATE OF check_mode, packet, approval SKIP LOCKED + """), + { + "wazuh_catalog_id": ( + _WAZUH_ALERTMANAGER_INTEGRATION_CATALOG_ID + ), + "secret_reference_missing": ( + _WAZUH_PRIVILEGED_CONVERGENCE_SECRET_REFERENCE_MISSING + ), + "required_blocker": ( + _WAZUH_CRITICAL_BREAK_GLASS_RECEIPTS_MISSING + ), + "approval_action": _WAZUH_BREAK_GLASS_APPROVAL_ACTION, + "trusted_signature_schema_version": ( + WAZUH_BREAK_GLASS_SIGNATURE_SCHEMA_VERSION + ), + "telegram_owner_auth_method": TELEGRAM_OWNER_AUTH_METHOD, + "telegram_owner_approver_role": ( + TELEGRAM_OWNER_APPROVER_ROLE + ), + "controlled_operator_auth_method": ( + CONTROLLED_OPERATOR_AUTH_METHOD + ), + "controlled_operator_approver_role": ( + CONTROLLED_OPERATOR_APPROVER_ROLE + ), + "stale_after_seconds": max(300, stale_after_seconds), + "limit": max(1, limit), + }, + ) + for row_value in result.mappings().all(): + row = dict(row_value) + claim = _claim_from_stale_check_mode_row(row) + if ( + claim is None + or row.get("opaque_secret_projection_presence_verified") + is not True + ): + continue + approval_id = str(row.get("break_glass_approval_id") or "") + packet_receipt_id = str( + row.get("retry_packet_receipt_id") or "" + ) + if not ( + approval_id + == str(claim.input_payload.get("break_glass_approval_id") or "") + and packet_receipt_id + == str( + claim.input_payload.get( + "capability_retry_packet_receipt_id" + ) + or "" + ) + ): + continue + claim_id = str(uuid4()) + claimed_at = datetime.now(UTC).isoformat() + authorized_input = { + **claim.input_payload, + "approval_id": approval_id, + "risk_level": "critical", + "apply_enabled": True, + "approval_required_before_apply": False, + "owner_review_required": False, + "controlled_apply_allowed": True, + "controlled_apply_blocker": None, + "break_glass_apply_authorized": True, + "break_glass_apply_claim_id": claim_id, + "break_glass_apply_claimed_at": claimed_at, + "break_glass_receipt_bundle": { + "schema_version": ( + "wazuh_secret_reference_break_glass_receipts_v1" + ), + "owner_approval_receipt_id": approval_id, + "owner_approval_resolved_at": ( + row["break_glass_approval_resolved_at"].isoformat() + if isinstance( + row.get("break_glass_approval_resolved_at"), + datetime, + ) + else str( + row.get("break_glass_approval_resolved_at") or "" + ) + ), + "trusted_owner_approval_signatures_verified": True, + "opaque_secret_projection_receipt_id": packet_receipt_id, + "authorized_executor_dispatch_receipt_id": ( + packet_receipt_id + ), + "authorized_executor": ( + "awoooi-ansible-executor-broker" + ), + "secret_value_read": False, + "host_write_performed_before_apply": False, + }, + } + updated = await db.execute( + text(""" + UPDATE automation_operation_log check_mode + SET input = CAST(:input AS jsonb), + dry_run_result = coalesce( + check_mode.dry_run_result, + '{}'::jsonb + ) || jsonb_build_object( + 'break_glass_apply_claim_id', + CAST(:claim_id AS text), + 'break_glass_apply_claimed_at', + CAST(:claimed_at AS text), + 'break_glass_receipts_verified', true, + 'runtime_apply_executed', false + ) + WHERE check_mode.op_id = CAST(:op_id AS uuid) + AND check_mode.status = 'success' + AND NOT EXISTS ( + SELECT 1 + FROM automation_operation_log apply + WHERE apply.operation_type = 'ansible_apply_executed' + AND apply.parent_op_id = check_mode.op_id + ) + AND ( + check_mode.input + ->> 'break_glass_apply_claimed_at' IS NULL + OR nullif( + check_mode.input + ->> 'break_glass_apply_claimed_at', + '' + )::timestamptz <= NOW() - ( + :stale_after_seconds * INTERVAL '1 second' + ) + ) + AND coalesce( + check_mode.input + ->> 'break_glass_apply_terminal_status', + '' + ) = '' + RETURNING check_mode.op_id + """), + { + "input": json.dumps( + authorized_input, + ensure_ascii=False, + ), + "claim_id": claim_id, + "claimed_at": claimed_at, + "op_id": claim.op_id, + "stale_after_seconds": max(300, stale_after_seconds), + }, + ) + if updated.scalar_one_or_none() is None: + continue + claims.append( + replace( + claim, + risk_level="critical", + input_payload=authorized_input, + ) + ) + if len(claims) >= max(1, limit): + break + return claims + + async def claim_pending_check_modes( *, project_id: str = "awoooi", @@ -8614,7 +9712,7 @@ async def claim_stale_pending_check_modes( claims: list[AnsibleCheckModeClaim] = [] async with get_db_context(project_id) as db: result = await db.execute( - text(""" + text(f""" SELECT check_mode.op_id, check_mode.parent_op_id, @@ -8623,8 +9721,55 @@ async def claim_stale_pending_check_modes( check_mode.incident_id::text, check_mode.input ->> 'incident_id' ) AS incident_id, - check_mode.input + check_mode.input, + retry_packet.op_id::text AS retry_packet_receipt_id, + retry_packet.status AS retry_packet_status, + retry_packet.parent_op_id::text + AS retry_packet_parent_op_id, + retry_packet.input ->> 'capability_packet_id' + AS retry_packet_id, + retry_packet.input ->> 'source_check_mode_op_id' + AS retry_packet_source_check_mode_op_id, + retry_packet.input ->> 'source_candidate_op_id' + AS retry_packet_source_candidate_op_id, + retry_packet.input ->> 'automation_run_id' + AS retry_packet_automation_run_id, + retry_packet.output ->> 'queue_status' + AS retry_packet_queue_status, + retry_packet.output ->> 'claimed_retry_check_mode_op_id' + AS retry_packet_claimed_check_mode_op_id, + retry_packet.output ->> 'failure_class' + AS retry_packet_failure_class, + retry_packet.output ->> 'claim_receipt_persisted' = 'true' + AS retry_packet_claim_receipt_persisted, + retry_packet.output ->> 'host_write_performed' = 'true' + AS retry_packet_host_write_performed, + retry_packet.output ->> 'secret_value_read' = 'true' + AS retry_packet_secret_value_read, + retry_packet.output + ->> 'authorized_executor_dispatch_receipt_persisted' + = 'true' + AS retry_packet_authorized_executor_dispatch_receipt_persisted, + retry_packet.output ->> 'authorized_executor' + AS retry_packet_authorized_executor FROM automation_operation_log check_mode + LEFT JOIN LATERAL ( + SELECT packet.* + FROM automation_operation_log packet + WHERE packet.op_id::text = check_mode.input + ->> 'capability_retry_packet_receipt_id' + AND ( + packet.operation_type = + '{_WAZUH_CONTROLLED_CAPABILITY_PACKET_OPERATION_TYPE}' + OR ( + packet.operation_type = + '{_EXECUTION_CAPABILITY_FALLBACK_OPERATION_TYPE}' + AND packet.input ->> 'semantic_operation_type' = + '{_WAZUH_CONTROLLED_CAPABILITY_PACKET_OPERATION_TYPE}' + ) + ) + LIMIT 1 + ) retry_packet ON TRUE WHERE check_mode.operation_type = 'ansible_check_mode_executed' AND check_mode.status = 'pending' AND coalesce( @@ -9604,6 +10749,9 @@ def _build_wazuh_controlled_capability_packet( ), ) capability_packet_id = f"IWZ-CC-{packet_uuid.hex[:16].upper()}" + break_glass_approval_id = _wazuh_break_glass_approval_id( + capability_packet_id + ) secret_reference_missing = ( failure_class == _WAZUH_PRIVILEGED_CONVERGENCE_SECRET_REFERENCE_MISSING @@ -9619,19 +10767,32 @@ def _build_wazuh_controlled_capability_packet( if secret_reference_missing else "ansible_capability_repair_agent" ) + retry_generation = max( + 0, + min( + _int_from_value( + source_input.get("capability_retry_generation"), + default=0, + ), + WAZUH_CONTROLLED_CAPABILITY_MAX_RETRIES, + ), + ) + automatic_retry_exhausted = ( + retry_generation >= WAZUH_CONTROLLED_CAPABILITY_MAX_RETRIES + ) controlled_apply_allowed = not secret_reference_missing required_receipts = ( [ - "critical_secret_reference_provisioning_receipt", - "windows99_dispatch", + "owner_critical_break_glass_approval", "opaque_secret_projection_presence", + "ansible_executor_broker_dispatch", "bounded_same_run_check_mode_retry", "independent_post_verifier", "capability_revocation", ] if secret_reference_missing else [ - "windows99_dispatch", + "authorized_executor_dispatch", "capability_reference_presence", "no_secret_privilege_preflight", "bounded_same_run_check_mode_retry", @@ -9666,8 +10827,19 @@ def _build_wazuh_controlled_capability_packet( "risk_level": risk_level, "policy_route": policy_route, "queue_owner": queue_owner, + "capability_retry_generation": retry_generation, + "automatic_retry_exhausted": automatic_retry_exhausted, "controlled_apply_allowed": controlled_apply_allowed, - "windows99_dispatch_required": True, + "windows99_dispatch_required": False, + "authorized_executor_dispatch_required": True, + "authorized_executors": [ + "Agent99", + "awoooi-ansible-executor-broker", + ], + "break_glass_approval_id": ( + break_glass_approval_id if secret_reference_missing else "" + ), + "owner_approval_required": secret_reference_missing, "same_run_correlation_required": True, "execution_allowed": False, "execution_gate_reason": ( @@ -9698,7 +10870,7 @@ def _build_wazuh_controlled_capability_packet( "critical_secret_reference_provisioned_without_exposure_and_" "same_run_independent_post_verifier_pass" if secret_reference_missing - else "windows99_no_secret_privilege_preflight_and_same_run_" + else "authorized_executor_no_secret_privilege_preflight_and_same_run_" "independent_post_verifier_pass" ), "replacement_action": ( @@ -9712,16 +10884,31 @@ def _build_wazuh_controlled_capability_packet( "post_verifier": "wazuh_alert_ingress_controlled_post_verifier", }, } + retry_of_packet_id = str( + source_input.get("capability_retry_of_packet_id") or "" + ).strip() + if retry_of_packet_id.startswith("IWZ-CC-") and len(retry_of_packet_id) == 23: + packet_input["retry_of_capability_packet_id"] = retry_of_packet_id packet_output = { - "queue_status": "queued", + "queue_status": ( + "retry_exhausted" if automatic_retry_exhausted else "queued" + ), "failure_class": failure_class, + "automatic_retry_remaining": max( + 0, + WAZUH_CONTROLLED_CAPABILITY_MAX_RETRIES - retry_generation, + ), "required_receipts": required_receipts, "next_controlled_action": ( - "provision_opaque_secret_reference_via_break_glass_then_" - "windows99_verify_and_requeue_same_run_check_mode" - if secret_reference_missing - else "windows99_verify_controlled_privilege_capability_then_" - "requeue_same_run_check_mode" + "await_owner_authorized_capability_state_change_receipt" + if automatic_retry_exhausted + else ( + "provision_opaque_secret_reference_via_break_glass_then_" + "broker_verify_and_requeue_same_run_check_mode" + if secret_reference_missing + else "authorized_executor_verify_controlled_privilege_capability_then_" + "requeue_same_run_check_mode" + ) ), "telegram_notification_emitted": False, } @@ -9797,7 +10984,7 @@ async def _insert_wazuh_controlled_capability_packet( THEN 'critical_risk' ELSE 'high_risk' END, - 'capability_repair', 'windows99_dispatch', + 'capability_repair', 'authorized_executor_dispatch', 'no_host_write', 'no_secret' ]::varchar[] WHERE NOT EXISTS ( @@ -9835,7 +11022,49 @@ async def _insert_wazuh_controlled_capability_packet( ), }, ) - return inserted.scalar_one_or_none() is not None + packet_receipt_id_value = inserted.scalar_one_or_none() + packet_written = packet_receipt_id_value is not None + packet_receipt_id = str(packet_receipt_id_value or "") + if ( + packet_input["critical_secret_reference_required"] is True + and not packet_receipt_id + ): + existing = await db.execute( + text(f""" + SELECT packet.op_id::text + FROM automation_operation_log packet + WHERE ( + packet.operation_type = + '{_WAZUH_CONTROLLED_CAPABILITY_PACKET_OPERATION_TYPE}' + OR ( + packet.operation_type = + '{_EXECUTION_CAPABILITY_FALLBACK_OPERATION_TYPE}' + AND packet.input ->> 'semantic_operation_type' = + '{_WAZUH_CONTROLLED_CAPABILITY_PACKET_OPERATION_TYPE}' + ) + ) + AND packet.input ->> 'capability_packet_id' = + :capability_packet_id + ORDER BY packet.created_at DESC + LIMIT 1 + """), + {"capability_packet_id": packet_input["capability_packet_id"]}, + ) + packet_receipt_id = str(existing.scalar_one_or_none() or "") + if ( + packet_input["critical_secret_reference_required"] is True + and packet_receipt_id + ): + await _ensure_wazuh_break_glass_approval( + db, + packet_receipt_id=packet_receipt_id, + capability_packet_id=str(packet_input["capability_packet_id"]), + incident_id=incident_id, + source_candidate_op_id=source_candidate_op_id, + source_check_mode_op_id=check_op_id, + automation_run_id=str(packet_input["automation_run_id"]), + ) + return packet_written async def finalize_check_mode_claim( @@ -10188,6 +11417,59 @@ async def run_controlled_apply_for_claim( ) return None if not await _revalidate_claim_playbook_trust(db, claim): + if claim.input_payload.get("break_glass_apply_authorized") is True: + blocked_at = datetime.now(UTC).isoformat() + terminal_status = "blocked_by_playbook_trust_circuit_open" + claim.input_payload.update( + { + "break_glass_apply_terminal_status": terminal_status, + "break_glass_apply_blocked_at": blocked_at, + "break_glass_apply_reclaim_allowed": False, + } + ) + await db.execute( + text(""" + UPDATE automation_operation_log check_mode + SET input = CAST(:input AS jsonb), + dry_run_result = coalesce( + check_mode.dry_run_result, + '{}'::jsonb + ) || jsonb_build_object( + 'break_glass_apply_terminal_status', + CAST(:terminal_status AS text), + 'break_glass_apply_blocked_at', + CAST(:blocked_at AS text), + 'break_glass_apply_reclaim_allowed', false, + 'runtime_apply_executed', false + ) + WHERE check_mode.op_id = CAST(:op_id AS uuid) + AND check_mode.input + ->> 'break_glass_apply_claim_id' = + :claim_id + AND NOT EXISTS ( + SELECT 1 + FROM automation_operation_log apply + WHERE apply.operation_type = + 'ansible_apply_executed' + AND apply.parent_op_id = check_mode.op_id + ) + """), + { + "input": json.dumps( + claim.input_payload, + ensure_ascii=False, + ), + "terminal_status": terminal_status, + "blocked_at": blocked_at, + "op_id": claim.op_id, + "claim_id": str( + claim.input_payload.get( + "break_glass_apply_claim_id" + ) + or "" + ), + }, + ) logger.warning( "ansible_controlled_apply_blocked_by_playbook_trust", op_id=claim.op_id, @@ -10649,6 +11931,14 @@ async def run_pending_check_modes_once( project_id=project_id, **failed_apply_retry_summary, ) + wazuh_capability_retry_summary: dict[str, Any] = { + "wazuh_capability_retry_claimed": 0, + "wazuh_capability_retry_error": None, + } + wazuh_break_glass_apply_summary: dict[str, Any] = { + "wazuh_break_glass_apply_claimed": 0, + "wazuh_break_glass_apply_claim_error": None, + } blockers = _runtime_blockers() if blockers: logger.warning("ansible_check_mode_runtime_blocked", blockers=blockers) @@ -10661,6 +11951,8 @@ async def run_pending_check_modes_once( **stdin_projection_summary, **receipt_backfill_summary, **failed_apply_retry_summary, + **wazuh_capability_retry_summary, + **wazuh_break_glass_apply_summary, } transport_blockers = await recent_ansible_transport_blockers(project_id=project_id) if transport_blockers: @@ -10674,13 +11966,72 @@ async def run_pending_check_modes_once( **stdin_projection_summary, **receipt_backfill_summary, **failed_apply_retry_summary, + **wazuh_capability_retry_summary, + **wazuh_break_glass_apply_summary, } - reclaimed_claims = await claim_stale_pending_check_modes( - project_id=project_id, - limit=limit, - stale_after_seconds=max(300, effective_timeout_seconds + 120), + wazuh_break_glass_apply_claims: list[AnsibleCheckModeClaim] = [] + try: + wazuh_break_glass_apply_claims = ( + await claim_owner_authorized_wazuh_secret_retry_applies( + project_id=project_id, + limit=limit, + stale_after_seconds=max( + 300, + settings.AWOOOP_ANSIBLE_CONTROLLED_APPLY_TIMEOUT_SECONDS + + 120, + ), + ) + ) + except Exception as exc: + wazuh_break_glass_apply_summary[ + "wazuh_break_glass_apply_claim_error" + ] = type(exc).__name__ + logger.warning( + "wazuh_owner_authorized_break_glass_apply_claim_failed", + project_id=project_id, + error_type=type(exc).__name__, + fresh_candidate_claim_continues=True, + ) + wazuh_break_glass_apply_summary[ + "wazuh_break_glass_apply_claimed" + ] = len(wazuh_break_glass_apply_claims) + remaining_limit = max(0, limit - len(wazuh_break_glass_apply_claims)) + wazuh_capability_retry_claims: list[AnsibleCheckModeClaim] = [] + if remaining_limit: + try: + wazuh_capability_retry_claims = ( + await claim_expired_wazuh_controlled_capability_retries( + project_id=project_id, + limit=remaining_limit, + ) + ) + except Exception as exc: + wazuh_capability_retry_summary["wazuh_capability_retry_error"] = ( + type(exc).__name__ + ) + logger.warning( + "wazuh_controlled_capability_retry_claim_failed", + project_id=project_id, + error_type=type(exc).__name__, + fresh_candidate_claim_continues=True, + ) + wazuh_capability_retry_summary["wazuh_capability_retry_claimed"] = len( + wazuh_capability_retry_claims ) - remaining_limit = max(0, limit - len(reclaimed_claims)) + remaining_limit = max( + 0, + remaining_limit - len(wazuh_capability_retry_claims), + ) + reclaimed_claims = ( + await claim_stale_pending_check_modes( + project_id=project_id, + limit=remaining_limit, + stale_after_seconds=max(300, effective_timeout_seconds + 120), + ) + if remaining_limit + else [] + ) + remaining_limit = max(0, remaining_limit - len(reclaimed_claims)) stdin_boundary_replay_claims: list[AnsibleCheckModeClaim] = [] stdin_boundary_replay_error: str | None = None if remaining_limit: @@ -10784,6 +12135,7 @@ async def run_pending_check_modes_once( else [] ) claims = [ + *wazuh_capability_retry_claims, *reclaimed_claims, *stdin_boundary_replay_claims, *semantic_catalog_reconciliation_claims, @@ -10791,6 +12143,7 @@ async def run_pending_check_modes_once( *catalog_replay_claims, *fresh_claims, ] + execution_claims = [*wazuh_break_glass_apply_claims, *claims] completed = 0 failed = 0 apply_completed = 0 @@ -10802,7 +12155,10 @@ async def run_pending_check_modes_once( capability_revoked = 0 capability_revoke_failed = 0 stdin_boundary_replay_terminal_written = 0 - for claim in claims: + for claim in execution_claims: + apply_only = bool( + claim.input_payload.get("break_glass_apply_authorized") is True + ) capability_op_id = "" terminal_status = "broker_interrupted_before_execution" try: @@ -10816,6 +12172,51 @@ async def run_pending_check_modes_once( project_id=project_id, ) capability_issued += 1 + if apply_only: + apply_result = await run_controlled_apply_for_claim( + claim, + timeout_seconds=( + settings.AWOOOP_ANSIBLE_CONTROLLED_APPLY_TIMEOUT_SECONDS + ), + project_id=project_id, + ) + if apply_result is None: + if ( + claim.input_payload.get("apply_duplicate_suppressed") + is True + ): + apply_duplicate_suppressed += 1 + terminal_status = ( + "owner_authorized_apply_duplicate_suppressed" + ) + else: + apply_blocked += 1 + terminal_status = ( + "owner_authorized_apply_blocked_playbook_trust_" + "terminal" + if claim.input_payload.get( + "break_glass_apply_terminal_status" + ) + else "owner_authorized_apply_blocked_after_claim" + ) + else: + apply_completed += 1 + if apply_result.returncode != 0: + apply_failed += 1 + terminal_status = "owner_authorized_apply_failed" + elif apply_result.post_verifier_passed is None: + apply_verifier_pending += 1 + terminal_status = ( + "owner_authorized_apply_verifier_pending" + ) + elif apply_result.post_verifier_passed is not True: + apply_failed += 1 + terminal_status = ( + "owner_authorized_apply_verifier_failed" + ) + else: + terminal_status = "owner_authorized_apply_verified" + continue result = await run_claimed_check_mode( claim, timeout_seconds=effective_timeout_seconds, @@ -10885,7 +12286,7 @@ async def run_pending_check_modes_once( except ValueError as exc: failed += 1 terminal_status = f"broker_policy_error:{exc}" - if not capability_op_id: + if not capability_op_id and not apply_only: await finalize_check_mode_claim( claim, AnsibleRunResult( @@ -10933,7 +12334,8 @@ async def run_pending_check_modes_once( error_type=type(exc).__name__, ) return { - "claimed": len(claims), + "claimed": len(execution_claims), + "check_mode_claimed": len(claims), "reclaimed": len(reclaimed_claims), "stdin_boundary_replayed": len(stdin_boundary_replay_claims), "stdin_boundary_replay_terminal_written": ( @@ -10968,5 +12370,7 @@ async def run_pending_check_modes_once( **stdin_projection_summary, **receipt_backfill_summary, **failed_apply_retry_summary, + **wazuh_capability_retry_summary, + **wazuh_break_glass_apply_summary, "blockers": [], } diff --git a/apps/api/src/services/iwooos_wazuh_controlled_executor_runtime.py b/apps/api/src/services/iwooos_wazuh_controlled_executor_runtime.py index 00aba8e06..7a805000a 100644 --- a/apps/api/src/services/iwooos_wazuh_controlled_executor_runtime.py +++ b/apps/api/src/services/iwooos_wazuh_controlled_executor_runtime.py @@ -21,6 +21,17 @@ from src.services.ai_automation_runtime_contract import ( AI_AUTOMATION_REQUIRED_LOOP_STAGES, ) from src.services.awooop_ansible_audit_service import get_ansible_catalog_item +from src.services.awooop_ansible_check_mode_service import ( + WAZUH_CONTROLLED_CAPABILITY_MAX_RETRIES, +) +from src.services.wazuh_break_glass_approval import ( + CONTROLLED_OPERATOR_APPROVER_ROLE, + CONTROLLED_OPERATOR_AUTH_METHOD, + TELEGRAM_OWNER_APPROVER_ROLE, + TELEGRAM_OWNER_AUTH_METHOD, + WAZUH_BREAK_GLASS_APPROVAL_ACTION, + WAZUH_BREAK_GLASS_SIGNATURE_SCHEMA_VERSION, +) SCHEMA_VERSION = "iwooos_wazuh_controlled_executor_runtime_readback_v3" CATALOG_ID = "ansible:wazuh-manager-posture-readback" @@ -41,6 +52,10 @@ _PRIVILEGED_CONVERGENCE_SECRET_REFERENCE_MISSING = ( _CONTROLLED_CAPABILITY_PACKET_SCHEMA_VERSION = ( "wazuh_controlled_privileged_capability_repair_v3" ) +_CRITICAL_BREAK_GLASS_RECEIPTS_MISSING = ( + "critical_secret_reference_owner_approval_and_broker_receipts_missing" +) +_BREAK_GLASS_APPROVAL_ACTION = WAZUH_BREAK_GLASS_APPROVAL_ACTION _PUBLIC_SAFE_CHECK_FAILURE_MARKERS = frozenset( { _PRIVILEGED_CONVERGENCE_CAPABILITY_MISSING, @@ -107,6 +122,28 @@ _LIVE_RUNTIME_SQL = """ check_mode.input ->> 'run_id' AS check_run_id, check_mode.input ->> 'work_item_id' AS check_work_item_id, check_mode.input -> 'source_recurrence' AS check_source_recurrence, + coalesce( + (check_mode.input ->> 'capability_retry_generation')::int, + 0 + ) AS check_capability_retry_generation, + check_mode.input ->> 'capability_retry_of_packet_id' + AS check_capability_retry_of_packet_id, + check_mode.input ->> 'replay_of_check_mode_op_id' + AS check_replay_of_check_mode_op_id, + check_mode.input ->> 'capability_retry_packet_receipt_id' + AS check_capability_retry_packet_receipt_id, + check_mode.input ->> 'break_glass_approval_id' + AS check_break_glass_approval_id, + coalesce( + ( + check_mode.input ->> 'break_glass_apply_authorized' + )::boolean, + false + ) AS check_break_glass_apply_authorized, + check_mode.input ->> 'break_glass_apply_terminal_status' + AS check_break_glass_apply_terminal_status, + check_mode.input ->> 'controlled_apply_blocker' + AS check_controlled_apply_blocker, coalesce( check_mode.output ->> 'returncode', check_mode.dry_run_result ->> 'returncode' @@ -174,6 +211,10 @@ _LIVE_RUNTIME_SQL = """ packet.input ->> 'capability_packet_id' AS capability_packet_id, packet.input ->> 'automation_run_id' AS capability_automation_run_id, + packet.input ->> 'source_candidate_op_id' + AS capability_source_candidate_op_id, + packet.input ->> 'source_check_mode_op_id' + AS capability_source_check_mode_op_id, packet.input ->> 'trace_id' AS capability_trace_id, packet.input ->> 'run_id' AS capability_run_id, packet.input ->> 'work_item_id' AS capability_work_item_id, @@ -185,6 +226,28 @@ _LIVE_RUNTIME_SQL = """ packet.input ->> 'capability_type' AS capability_type, packet.input ->> 'risk_level' AS capability_risk_level, packet.input ->> 'policy_route' AS capability_policy_route, + packet.input ->> 'retry_of_capability_packet_id' + AS capability_retry_of_packet_id, + coalesce( + (packet.output ->> 'capability_retry_generation')::int, + (packet.input ->> 'capability_retry_generation')::int, + 0 + ) AS capability_retry_generation, + coalesce( + (packet.output ->> 'automatic_retry_exhausted')::boolean, + (packet.input ->> 'automatic_retry_exhausted')::boolean, + false + ) AS capability_automatic_retry_exhausted, + coalesce( + (packet.output ->> 'automatic_retry_remaining')::int, + greatest( + 0, + 1 - coalesce( + (packet.input ->> 'capability_retry_generation')::int, + 0 + ) + ) + ) AS capability_automatic_retry_remaining, coalesce( (packet.input ->> 'controlled_apply_allowed')::boolean, false @@ -193,6 +256,21 @@ _LIVE_RUNTIME_SQL = """ (packet.input ->> 'windows99_dispatch_required')::boolean, false ) AS capability_windows99_dispatch_required, + coalesce( + ( + packet.input + ->> 'authorized_executor_dispatch_required' + )::boolean, + false + ) AS capability_authorized_executor_dispatch_required, + packet.input -> 'authorized_executors' + AS capability_authorized_executors, + packet.input ->> 'break_glass_approval_id' + AS capability_break_glass_approval_id, + coalesce( + (packet.input ->> 'owner_approval_required')::boolean, + false + ) AS capability_owner_approval_required, coalesce( (packet.input ->> 'critical_secret_reference_required')::boolean, false @@ -212,10 +290,47 @@ _LIVE_RUNTIME_SQL = """ (packet.dry_run_result ->> 'host_write_performed')::boolean, false ) AS capability_packet_host_write_performed, + coalesce( + (packet.output ->> 'claim_receipt_persisted')::boolean, + false + ) AS capability_claim_receipt_persisted, + packet.output ->> 'claimed_retry_check_mode_op_id' + AS capability_claimed_retry_check_mode_op_id, + coalesce( + ( + packet.output + ->> 'opaque_secret_projection_presence_verified' + )::boolean, + false + ) AS capability_opaque_secret_projection_verified, + coalesce( + ( + packet.output + ->> 'authorized_executor_dispatch_receipt_persisted' + )::boolean, + false + ) AS capability_authorized_executor_dispatch_receipt_persisted, + packet.output ->> 'authorized_executor' + AS capability_authorized_executor, + coalesce( + (packet.output ->> 'secret_value_read')::boolean, + false + ) AS capability_secret_value_read, + coalesce( + (packet.output ->> 'host_write_performed')::boolean, + false + ) AS capability_claim_host_write_performed, packet.created_at AS controlled_capability_packet_created_at FROM automation_operation_log packet JOIN latest_check check_mode - ON packet.parent_op_id::text = check_mode.check_op_id + ON ( + packet.op_id::text = + check_mode.check_capability_retry_packet_receipt_id + OR ( + check_mode.check_capability_retry_packet_receipt_id IS NULL + AND packet.parent_op_id::text = check_mode.check_op_id + ) + ) WHERE ( packet.operation_type = 'ansible_controlled_capability_repair_queued' @@ -227,6 +342,122 @@ _LIVE_RUNTIME_SQL = """ ) ORDER BY packet.created_at DESC LIMIT 1 + ), latest_break_glass_approval AS ( + SELECT + approval.id AS break_glass_approval_id, + approval.action AS break_glass_approval_action, + lower(approval.status::text) AS break_glass_approval_status, + lower(approval.risk_level::text) + AS break_glass_approval_risk_level, + approval.required_signatures + AS break_glass_approval_required_signatures, + approval.current_signatures + AS break_glass_approval_current_signatures, + CASE + WHEN jsonb_typeof(CAST(approval.signatures AS jsonb)) = 'array' + THEN jsonb_array_length(CAST(approval.signatures AS jsonb)) + ELSE 0 + END AS break_glass_approval_signature_count, + EXISTS ( + SELECT 1 + FROM jsonb_array_elements( + CASE + WHEN jsonb_typeof( + CAST(approval.signatures AS jsonb) + ) = 'array' + THEN CAST(approval.signatures AS jsonb) + ELSE '[]'::jsonb + END + ) trusted_signature + WHERE trusted_signature ->> 'signature_schema_version' = + :trusted_signature_schema_version + AND trusted_signature ->> 'principal_id' = + trusted_signature ->> 'signer_id' + AND trusted_signature ->> 'auth_method' = + :telegram_owner_auth_method + AND trusted_signature ->> 'approver_role' = + :telegram_owner_approver_role + AND trusted_signature ->> 'source' = 'telegram' + AND trusted_signature ->> 'telegram_user_id' + ~ '^[0-9]+$' + AND trusted_signature ->> 'principal_id' = + 'tg_' || ( + trusted_signature ->> 'telegram_user_id' + ) + ) AS break_glass_trusted_owner_signature_verified, + EXISTS ( + SELECT 1 + FROM jsonb_array_elements( + CASE + WHEN jsonb_typeof( + CAST(approval.signatures AS jsonb) + ) = 'array' + THEN CAST(approval.signatures AS jsonb) + ELSE '[]'::jsonb + END + ) trusted_signature + WHERE trusted_signature ->> 'signature_schema_version' = + :trusted_signature_schema_version + AND trusted_signature ->> 'principal_id' = + trusted_signature ->> 'signer_id' + AND trusted_signature ->> 'auth_method' = + :controlled_operator_auth_method + AND trusted_signature ->> 'approver_role' = + :controlled_operator_approver_role + AND trusted_signature ->> 'source' = 'api' + ) AS break_glass_trusted_operator_signature_verified, + ( + SELECT count( + DISTINCT trusted_signature ->> 'principal_id' + ) + FROM jsonb_array_elements( + CASE + WHEN jsonb_typeof( + CAST(approval.signatures AS jsonb) + ) = 'array' + THEN CAST(approval.signatures AS jsonb) + ELSE '[]'::jsonb + END + ) trusted_signature + WHERE trusted_signature ->> 'signature_schema_version' = + :trusted_signature_schema_version + AND trusted_signature ->> 'principal_id' = + trusted_signature ->> 'signer_id' + AND trusted_signature ->> 'approver_role' IN ( + :telegram_owner_approver_role, + :controlled_operator_approver_role + ) + ) AS break_glass_trusted_distinct_principal_count, + approval.incident_id AS break_glass_approval_incident_id, + approval.resolved_at AS break_glass_approval_resolved_at, + approval.expires_at AS break_glass_approval_expires_at, + CAST(approval.extra_metadata AS jsonb) + ->> 'schema_version' + AS break_glass_approval_schema_version, + CAST(approval.extra_metadata AS jsonb) + ->> 'capability_packet_receipt_id' + AS break_glass_approval_packet_receipt_id, + CAST(approval.extra_metadata AS jsonb) + ->> 'capability_packet_id' + AS break_glass_approval_packet_id, + CAST(approval.extra_metadata AS jsonb) + ->> 'source_candidate_op_id' + AS break_glass_approval_source_candidate_op_id, + CAST(approval.extra_metadata AS jsonb) + ->> 'source_check_mode_op_id' + AS break_glass_approval_source_check_mode_op_id, + CAST(approval.extra_metadata AS jsonb) + ->> 'automation_run_id' + AS break_glass_approval_automation_run_id, + CAST(approval.extra_metadata AS jsonb) + ->> 'trusted_signature_schema_version' + AS break_glass_trusted_signature_schema_version + FROM approval_records approval + JOIN latest_check check_mode + ON approval.id = check_mode.check_break_glass_approval_id + JOIN latest_controlled_capability_packet packet + ON packet.capability_break_glass_approval_id = approval.id + LIMIT 1 ), runtime_stages AS ( SELECT coalesce( @@ -349,6 +580,7 @@ _LIVE_RUNTIME_SQL = """ candidate.*, check_mode.*, controlled_capability_packet.*, + break_glass_approval.*, apply.*, runtime_stages.stage_ids, runtime_stages.runtime_stage_identity_verified, @@ -397,6 +629,7 @@ _LIVE_RUNTIME_SQL = """ LEFT JOIN latest_check check_mode ON TRUE LEFT JOIN latest_controlled_capability_packet controlled_capability_packet ON TRUE + LEFT JOIN latest_break_glass_approval break_glass_approval ON TRUE LEFT JOIN latest_apply apply ON TRUE LEFT JOIN runtime_stages ON TRUE LEFT JOIN latest_verifier verifier ON TRUE @@ -449,6 +682,21 @@ async def load_iwooos_wazuh_controlled_executor_runtime_lanes( "work_item_id": work_item_id, "proposal_source": proposal_source, "run_namespace": run_namespace, + "trusted_signature_schema_version": ( + WAZUH_BREAK_GLASS_SIGNATURE_SCHEMA_VERSION + ), + "telegram_owner_auth_method": ( + TELEGRAM_OWNER_AUTH_METHOD + ), + "telegram_owner_approver_role": ( + TELEGRAM_OWNER_APPROVER_ROLE + ), + "controlled_operator_auth_method": ( + CONTROLLED_OPERATOR_AUTH_METHOD + ), + "controlled_operator_approver_role": ( + CONTROLLED_OPERATOR_APPROVER_ROLE + ), }, ) rows[lane_id] = result.mappings().first() @@ -501,6 +749,9 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( """Build a public-safe runtime summary from one projected DB row.""" data = dict(row or {}) + check_capability_retry_generation = _nonnegative_int( + data.get("check_capability_retry_generation") + ) enabled = ( settings.ENABLE_IWOOOS_WAZUH_MANAGER_POSTURE_EXECUTOR if executor_enabled is None @@ -626,9 +877,22 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( ) check_failed = data.get("check_status") == "failed" check_failure_class = _check_failure_class(data) + check_controlled_apply_blocker = str( + data.get("check_controlled_apply_blocker") or "" + ) + break_glass_apply_terminal_status = str( + data.get("check_break_glass_apply_terminal_status") or "" + ) + critical_break_glass_receipts_pending = bool( + check_controlled_apply_blocker + == _CRITICAL_BREAK_GLASS_RECEIPTS_MISSING + ) critical_secret_reference_required = bool( check_failure_class == _PRIVILEGED_CONVERGENCE_SECRET_REFERENCE_MISSING + or critical_break_glass_receipts_pending + or data.get("capability_critical_secret_reference_required") is True + or bool(data.get("check_break_glass_approval_id")) ) expected_capability_risk_level = ( "critical" if critical_secret_reference_required else "high" @@ -651,6 +915,13 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( and check_identity_verified and str(data.get("capability_automation_run_id") or "") == candidate_op_id + and str(data.get("capability_source_candidate_op_id") or "") + == candidate_op_id + and str(data.get("capability_source_check_mode_op_id") or "") + in { + str(data.get("check_op_id") or ""), + str(data.get("check_replay_of_check_mode_op_id") or ""), + } and str(data.get("capability_trace_id") or "") == candidate_op_id and str(data.get("capability_run_id") or "") == candidate_op_id and str(data.get("capability_work_item_id") or "") @@ -670,11 +941,33 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( capability_expires_at and capability_expires_at > datetime.now(UTC) ) - controlled_capability_packet_present = all( + capability_packet_state_verified = bool( + ( + data.get("controlled_capability_packet_status") == "pending" + and data.get("capability_queue_status") + in {"queued", "retry_exhausted"} + ) + or ( + data.get("controlled_capability_packet_status") == "success" + and data.get("capability_queue_status") == "retry_claimed" + and data.get("capability_claim_receipt_persisted") is True + and str( + data.get("capability_claimed_retry_check_mode_op_id") or "" + ) + == str(data.get("check_op_id") or "") + and str( + data.get("check_capability_retry_packet_receipt_id") or "" + ) + == str( + data.get("controlled_capability_packet_receipt_id") or "" + ) + ) + ) + controlled_capability_packet_receipt_ready = all( ( controlled_capability_packet_identity_verified, - controlled_capability_packet_active, - data.get("controlled_capability_packet_status") == "pending", + capability_expires_at is not None, + capability_packet_state_verified, data.get("capability_schema_version") == _CONTROLLED_CAPABILITY_PACKET_SCHEMA_VERSION, capability_packet_id.startswith("IWZ-CC-"), @@ -684,10 +977,14 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( == expected_capability_risk_level, data.get("capability_policy_route") == expected_capability_policy_route, - data.get("capability_queue_status") == "queued", data.get("capability_controlled_apply_allowed") is expected_controlled_apply_allowed, - data.get("capability_windows99_dispatch_required") is True, + data.get("capability_windows99_dispatch_required") is False, + data.get("capability_authorized_executor_dispatch_required") + is True, + isinstance(data.get("capability_authorized_executors"), list), + "awoooi-ansible-executor-broker" + in (data.get("capability_authorized_executors") or []), bool(data.get("capability_critical_secret_reference_required")) == critical_secret_reference_required, data.get("capability_execution_allowed") is False, @@ -695,6 +992,114 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( data.get("capability_packet_host_write_performed") is False, ) ) + owner_approval_expires_at = _utc_receipt_datetime( + data.get("break_glass_approval_expires_at") + ) + owner_approval_resolved_at = _utc_receipt_datetime( + data.get("break_glass_approval_resolved_at") + ) + owner_break_glass_approval_verified = bool( + critical_secret_reference_required + and controlled_capability_packet_receipt_ready + and str(data.get("break_glass_approval_id") or "") + == str(data.get("check_break_glass_approval_id") or "") + == str(data.get("capability_break_glass_approval_id") or "") + and data.get("break_glass_approval_action") + == _BREAK_GLASS_APPROVAL_ACTION + and data.get("break_glass_approval_status") == "approved" + and data.get("break_glass_approval_risk_level") == "critical" + and _nonnegative_int( + data.get("break_glass_approval_required_signatures") + ) + >= 2 + and _nonnegative_int( + data.get("break_glass_approval_current_signatures") + ) + >= _nonnegative_int( + data.get("break_glass_approval_required_signatures") + ) + and _nonnegative_int(data.get("break_glass_approval_signature_count")) + >= _nonnegative_int( + data.get("break_glass_approval_required_signatures") + ) + and data.get("break_glass_trusted_owner_signature_verified") is True + and data.get("break_glass_trusted_operator_signature_verified") is True + and _nonnegative_int( + data.get("break_glass_trusted_distinct_principal_count") + ) + >= 2 + and data.get("break_glass_trusted_signature_schema_version") + == WAZUH_BREAK_GLASS_SIGNATURE_SCHEMA_VERSION + and owner_approval_resolved_at is not None + and ( + owner_approval_expires_at is None + or owner_approval_expires_at > datetime.now(UTC) + ) + and str(data.get("break_glass_approval_incident_id") or "") + == str(data.get("incident_id") or "") + and data.get("break_glass_approval_schema_version") + == "wazuh_secret_reference_break_glass_approval_v1" + and str(data.get("break_glass_approval_packet_receipt_id") or "") + == str(data.get("controlled_capability_packet_receipt_id") or "") + and str(data.get("break_glass_approval_packet_id") or "") + == capability_packet_id + and str( + data.get("break_glass_approval_source_candidate_op_id") or "" + ) + == candidate_op_id + and str(data.get("break_glass_approval_source_check_mode_op_id") or "") + == str(data.get("capability_source_check_mode_op_id") or "") + and str(data.get("break_glass_approval_automation_run_id") or "") + == candidate_op_id + ) + opaque_secret_projection_receipt_verified = bool( + critical_secret_reference_required + and controlled_capability_packet_receipt_ready + and data.get("capability_queue_status") == "retry_claimed" + and data.get("capability_opaque_secret_projection_verified") is True + and data.get("capability_secret_value_read") is False + and data.get("capability_claim_host_write_performed") is False + ) + authorized_executor_dispatch_receipt_verified = bool( + controlled_capability_packet_receipt_ready + and data.get("capability_queue_status") == "retry_claimed" + and data.get( + "capability_authorized_executor_dispatch_receipt_persisted" + ) + is True + and data.get("capability_authorized_executor") + == "awoooi-ansible-executor-broker" + and data.get("capability_secret_value_read") is False + and data.get("capability_claim_host_write_performed") is False + ) + break_glass_receipt_bundle_verified = bool( + owner_break_glass_approval_verified + and opaque_secret_projection_receipt_verified + and authorized_executor_dispatch_receipt_verified + ) + capability_retry_generation = _nonnegative_int( + data.get("capability_retry_generation") + ) + capability_automatic_retry_exhausted = bool( + controlled_capability_packet_receipt_ready + and data.get("capability_queue_status") == "retry_exhausted" + and data.get("capability_automatic_retry_exhausted") is True + ) + capability_automatic_retry_remaining = ( + _nonnegative_int(data.get("capability_automatic_retry_remaining")) + if data.get("capability_automatic_retry_remaining") is not None + else max( + 0, + WAZUH_CONTROLLED_CAPABILITY_MAX_RETRIES + - capability_retry_generation, + ) + ) + controlled_capability_packet_present = bool( + controlled_capability_packet_receipt_ready + and controlled_capability_packet_active + and data.get("capability_queue_status") == "queued" + and not capability_automatic_retry_exhausted + ) apply_terminal = data.get("apply_status") in {"success", "failed"} apply_passed = apply_terminal and str( data.get("apply_returncode") or "" @@ -830,15 +1235,20 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( if check_failure_class in _PUBLIC_SAFE_CHECK_FAILURE_MARKERS: active_blockers.append( ( - "wazuh_critical_secret_reference_provisioning_pending" - if critical_secret_reference_required - else "controlled_capability_repair_packet_pending" + "wazuh_same_run_capability_retry_exhausted" + if capability_automatic_retry_exhausted + else ( + "wazuh_critical_secret_reference_provisioning_pending" + if critical_secret_reference_required + else "controlled_capability_repair_packet_pending" + ) ) if controlled_capability_packet_present + or capability_automatic_retry_exhausted else ( "controlled_capability_repair_packet_expired" if ( - controlled_capability_packet_identity_verified + controlled_capability_packet_receipt_ready and not controlled_capability_packet_active ) else "controlled_capability_repair_packet_identity_mismatch" @@ -846,6 +1256,29 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( else "controlled_capability_repair_packet_missing" ) ) + if ( + check_passed + and not apply_terminal + and check_controlled_apply_blocker + ): + active_blockers.append(check_controlled_apply_blocker) + if critical_secret_reference_required: + if not owner_break_glass_approval_verified: + active_blockers.append("owner_break_glass_approval_pending") + elif not opaque_secret_projection_receipt_verified: + active_blockers.append( + "opaque_secret_projection_receipt_pending" + ) + elif not authorized_executor_dispatch_receipt_verified: + active_blockers.append( + "authorized_executor_dispatch_receipt_pending" + ) + elif data.get("check_break_glass_apply_authorized") is not True: + active_blockers.append( + "owner_authorized_break_glass_apply_claim_pending" + ) + if break_glass_apply_terminal_status: + active_blockers.append(break_glass_apply_terminal_status) if apply_failed: active_blockers.append("wazuh_controlled_apply_failed") if apply_passed and not verifier_passed: @@ -869,6 +1302,13 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( check_failed=check_failed, apply_terminal=apply_terminal, apply_passed=apply_passed, + controlled_apply_blocker=check_controlled_apply_blocker, + break_glass_apply_terminal_status=( + break_glass_apply_terminal_status + ), + capability_retry_exhausted=( + capability_automatic_retry_exhausted + ), runtime_closed=runtime_closed, ), "mode": "live_durable_receipt_readback_no_raw_payload_no_secret", @@ -930,6 +1370,50 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( "controlled_capability_repair_packet_count": ( 1 if controlled_capability_packet_present else 0 ), + "controlled_capability_repair_packet_receipt_count": ( + 1 if controlled_capability_packet_receipt_ready else 0 + ), + "same_run_capability_retry_claimed_count": ( + 1 if check_capability_retry_generation > 0 else 0 + ), + "same_run_capability_retry_generation": ( + check_capability_retry_generation + ), + "same_run_capability_retry_exhausted_count": ( + 1 if capability_automatic_retry_exhausted else 0 + ), + "owner_break_glass_approval_verified_count": ( + 1 if owner_break_glass_approval_verified else 0 + ), + "trusted_owner_signature_verified_count": ( + 1 + if data.get("break_glass_trusted_owner_signature_verified") + is True + else 0 + ), + "trusted_operator_signature_verified_count": ( + 1 + if data.get("break_glass_trusted_operator_signature_verified") + is True + else 0 + ), + "break_glass_apply_terminal_blocked_count": ( + 1 if break_glass_apply_terminal_status else 0 + ), + "opaque_secret_projection_receipt_verified_count": ( + 1 if opaque_secret_projection_receipt_verified else 0 + ), + "authorized_executor_dispatch_receipt_verified_count": ( + 1 if authorized_executor_dispatch_receipt_verified else 0 + ), + "break_glass_receipt_bundle_verified_count": ( + 1 if break_glass_receipt_bundle_verified else 0 + ), + "owner_authorized_break_glass_apply_claimed_count": ( + 1 + if data.get("check_break_glass_apply_authorized") is True + else 0 + ), "runtime_execution_performed_count": 1 if apply_terminal else 0, "bounded_posture_execution_passed_count": ( 1 if apply_passed and not ingress_mode else 0 @@ -987,15 +1471,34 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( data.get("controlled_capability_packet_receipt_id") or "" ), "capability_packet_id": capability_packet_id, - "status": "pending", - "queue_status": "queued", + "status": str( + data.get("controlled_capability_packet_status") or "" + ), + "queue_status": str( + data.get("capability_queue_status") or "" + ), + "active": controlled_capability_packet_active, + "capability_retry_generation": capability_retry_generation, + "retry_of_capability_packet_id": str( + data.get("capability_retry_of_packet_id") or "" + ), + "automatic_retry_exhausted": ( + capability_automatic_retry_exhausted + ), + "automatic_retry_remaining": _nonnegative_int( + capability_automatic_retry_remaining + ), "capability_type": "bounded_privileged_convergence", "risk_level": expected_capability_risk_level, "policy_route": expected_capability_policy_route, "controlled_apply_allowed": ( expected_controlled_apply_allowed ), - "windows99_dispatch_required": True, + "windows99_dispatch_required": False, + "authorized_executor_dispatch_required": True, + "authorized_executor": str( + data.get("capability_authorized_executor") or "" + ), "critical_secret_reference_required": bool( data.get("capability_critical_secret_reference_required") is True @@ -1007,8 +1510,49 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( "secret_value_exposed": False, "secret_value_logged": False, "secret_value_read_back": False, + "owner_approval_receipt_verified": ( + owner_break_glass_approval_verified + ), + "opaque_secret_projection_receipt_verified": ( + opaque_secret_projection_receipt_verified + ), + "authorized_executor_dispatch_receipt_verified": ( + authorized_executor_dispatch_receipt_verified + ), } - if controlled_capability_packet_present + if controlled_capability_packet_receipt_ready + else None + ), + "break_glass_approval": ( + { + "approval_id": str(data.get("break_glass_approval_id") or ""), + "status": str( + data.get("break_glass_approval_status") or "pending" + ), + "required_signatures": _nonnegative_int( + data.get("break_glass_approval_required_signatures") + ), + "current_signatures": _nonnegative_int( + data.get("break_glass_approval_current_signatures") + ), + "receipt_verified": owner_break_glass_approval_verified, + "trusted_owner_signature_verified": ( + data.get("break_glass_trusted_owner_signature_verified") + is True + ), + "trusted_operator_signature_verified": ( + data.get( + "break_glass_trusted_operator_signature_verified" + ) + is True + ), + "canonical_url": ( + "/zh-TW/awooop/approvals" + f"?project_id={str(data.get('candidate_project_id') or 'awoooi')}" + f"&incident_id={str(data.get('incident_id') or '')}" + ), + } + if data.get("check_break_glass_approval_id") else None ), "next_safe_action": _next_action( @@ -1017,8 +1561,27 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( check_passed=check_passed, check_failed=check_failed, check_failure_class=check_failure_class, + controlled_apply_blocker=check_controlled_apply_blocker, + break_glass_apply_terminal_status=( + break_glass_apply_terminal_status + ), + owner_break_glass_approval_verified=( + owner_break_glass_approval_verified + ), + opaque_secret_projection_receipt_verified=( + opaque_secret_projection_receipt_verified + ), + authorized_executor_dispatch_receipt_verified=( + authorized_executor_dispatch_receipt_verified + ), + break_glass_apply_authorized=( + data.get("check_break_glass_apply_authorized") is True + ), controlled_capability_packet_present=( - controlled_capability_packet_present + controlled_capability_packet_receipt_ready + ), + capability_retry_exhausted=( + capability_automatic_retry_exhausted ), apply_terminal=apply_terminal, apply_passed=apply_passed, @@ -1055,9 +1618,24 @@ def build_iwooos_wazuh_controlled_executor_runtime_readback( "critical_secret_reference_provisioning_required": ( critical_secret_reference_required ), + "owner_break_glass_approval_verified": ( + owner_break_glass_approval_verified + ), + "opaque_secret_projection_receipt_verified": ( + opaque_secret_projection_receipt_verified + ), + "authorized_executor_dispatch_receipt_verified": ( + authorized_executor_dispatch_receipt_verified + ), + "break_glass_receipt_bundle_verified": ( + break_glass_receipt_bundle_verified + ), + "break_glass_apply_terminal_status": ( + break_glass_apply_terminal_status or None + ), "controlled_capability_execution_allowed": False, "controlled_capability_repair_packet_persisted": ( - controlled_capability_packet_present + controlled_capability_packet_receipt_ready ), "controlled_capability_repair_packet_identity_verified": ( controlled_capability_packet_identity_verified @@ -1090,6 +1668,9 @@ def _status( check_failed: bool, apply_terminal: bool, apply_passed: bool, + controlled_apply_blocker: str, + break_glass_apply_terminal_status: str, + capability_retry_exhausted: bool, runtime_closed: bool, ) -> str: prefix = "wazuh_alert_ingress" if ingress_mode else "wazuh_manager_posture" @@ -1101,8 +1682,14 @@ def _status( return f"{prefix}_execution_failed_waiting_retry_or_repair" if apply_passed: return f"{prefix}_executed_runtime_writeback_open" + if break_glass_apply_terminal_status: + return f"{prefix}_owner_authorized_apply_blocked_terminal" + if check_passed and controlled_apply_blocker: + return f"{prefix}_check_passed_apply_blocked_by_policy" if check_passed: return f"{prefix}_check_passed_waiting_bounded_execution" + if capability_retry_exhausted: + return f"{prefix}_same_run_capability_retry_exhausted" if check_failed: return f"{prefix}_check_failed_waiting_bounded_retry" if candidate_present: @@ -1117,7 +1704,14 @@ def _next_action( check_passed: bool, check_failed: bool, check_failure_class: str | None, + controlled_apply_blocker: str, + break_glass_apply_terminal_status: str, + owner_break_glass_approval_verified: bool, + opaque_secret_projection_receipt_verified: bool, + authorized_executor_dispatch_receipt_verified: bool, + break_glass_apply_authorized: bool, controlled_capability_packet_present: bool, + capability_retry_exhausted: bool, apply_terminal: bool, apply_passed: bool, verifier_passed: bool, @@ -1141,6 +1735,8 @@ def _next_action( ) if check_failed: if check_failure_class in _PUBLIC_SAFE_CHECK_FAILURE_MARKERS: + if capability_retry_exhausted: + return "await_owner_authorized_capability_state_change_receipt" if ( check_failure_class == _PRIVILEGED_CONVERGENCE_SECRET_REFERENCE_MISSING @@ -1148,12 +1744,12 @@ def _next_action( if controlled_capability_packet_present: return ( "provision_opaque_host112_secret_reference_then_" - "windows99_verify_and_requeue_same_run" + "broker_verify_and_requeue_same_run" ) return "queue_critical_secret_reference_break_glass_packet" if controlled_capability_packet_present: return ( - "windows99_verify_controlled_privilege_capability_then_" + "authorized_executor_verify_controlled_privilege_capability_then_" "requeue_same_run" ) return ( @@ -1165,6 +1761,19 @@ def _next_action( ) if not check_passed: return "worker_claim_and_run_allowlisted_ansible_check_mode" + if break_glass_apply_terminal_status: + return "repair_playbook_trust_before_new_break_glass_run" + if controlled_apply_blocker == _CRITICAL_BREAK_GLASS_RECEIPTS_MISSING: + if not owner_break_glass_approval_verified: + return "await_owner_break_glass_approval" + if not opaque_secret_projection_receipt_verified: + return "await_opaque_secret_projection_receipt" + if not authorized_executor_dispatch_receipt_verified: + return "await_authorized_executor_dispatch_receipt" + if not break_glass_apply_authorized: + return "broker_claim_owner_authorized_break_glass_apply" + if controlled_apply_blocker: + return "resolve_controlled_apply_policy_blocker_before_execution" if not apply_terminal: return ( "run_bounded_wazuh_alert_ingress_convergence" @@ -1212,6 +1821,13 @@ def _latest_receipt_at(data: Mapping[str, Any]) -> str | None: return max(str(value) for value in values) +def _nonnegative_int(value: Any) -> int: + try: + return max(0, int(value or 0)) + except (TypeError, ValueError): + return 0 + + def _utc_receipt_datetime(value: Any) -> datetime | None: if isinstance(value, datetime): parsed = value diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 72f24f582..df5b84f7b 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -84,6 +84,10 @@ from src.services.telegram_callback_correlation import ( controlled_callback_correlation_fingerprint, normalize_controlled_callback_correlation, ) +from src.services.wazuh_break_glass_approval import ( + TELEGRAM_OWNER_APPROVER_ROLE, + TELEGRAM_OWNER_AUTH_METHOD, +) # ============================================================================= # Snooze/Silence Redis Keys (2026-03-27 P1 優化) @@ -15448,6 +15452,12 @@ class TelegramGateway: signer_id=f"tg_{user_id}", signer_name=username, comment="Telegram 簽核 (Long Polling)", + authenticated_principal_id=f"tg_{user_id}", + authentication_method=TELEGRAM_OWNER_AUTH_METHOD, + approver_role=TELEGRAM_OWNER_APPROVER_ROLE, + signature_source="telegram", + telegram_user_id=int(user_id), + telegram_message_id=int(message_id), ) if approval: diff --git a/apps/api/src/services/wazuh_break_glass_approval.py b/apps/api/src/services/wazuh_break_glass_approval.py new file mode 100644 index 000000000..15c35dc9b --- /dev/null +++ b/apps/api/src/services/wazuh_break_glass_approval.py @@ -0,0 +1,116 @@ +"""Trusted signature contract for Wazuh secret-reference break glass. + +The generic approval API predates authenticated signer identities. Wazuh +controlled apply must therefore consume only signatures produced by two +independent, server-verified channels: a Telegram whitelist/nonce owner and an +AwoooP controlled operator authenticated by the server-side operator key. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +WAZUH_BREAK_GLASS_APPROVAL_ACTION = ( + "OWNER_REVIEW_REQUIRED:authorize_wazuh_secret_reference_controlled_apply" +) +WAZUH_BREAK_GLASS_SIGNATURE_SCHEMA_VERSION = ( + "wazuh_break_glass_trusted_signature_v1" +) + +TELEGRAM_OWNER_AUTH_METHOD = "telegram_whitelist_nonce" +TELEGRAM_OWNER_APPROVER_ROLE = "owner" +CONTROLLED_OPERATOR_AUTH_METHOD = "awooop_operator_api_key" +CONTROLLED_OPERATOR_APPROVER_ROLE = "controlled_operator" + +WAZUH_BREAK_GLASS_REQUIRED_SIGNATURE_ROLES = frozenset( + { + TELEGRAM_OWNER_APPROVER_ROLE, + CONTROLLED_OPERATOR_APPROVER_ROLE, + } +) +_ROLE_AUTH_METHOD = { + TELEGRAM_OWNER_APPROVER_ROLE: TELEGRAM_OWNER_AUTH_METHOD, + CONTROLLED_OPERATOR_APPROVER_ROLE: CONTROLLED_OPERATOR_AUTH_METHOD, +} + + +def is_wazuh_break_glass_approval_action(action: object) -> bool: + """Return whether an approval belongs to the Wazuh critical apply gate.""" + + return str(action or "").strip() == WAZUH_BREAK_GLASS_APPROVAL_ACTION + + +def trusted_wazuh_signature_context( + *, + principal_id: str, + auth_method: str, + approver_role: str, +) -> dict[str, str] | None: + """Validate and normalize one server-authenticated signature context.""" + + normalized_principal = str(principal_id or "").strip() + normalized_method = str(auth_method or "").strip() + normalized_role = str(approver_role or "").strip() + if ( + not normalized_principal + or _ROLE_AUTH_METHOD.get(normalized_role) != normalized_method + ): + return None + return { + "signature_schema_version": ( + WAZUH_BREAK_GLASS_SIGNATURE_SCHEMA_VERSION + ), + "principal_id": normalized_principal, + "auth_method": normalized_method, + "approver_role": normalized_role, + } + + +def wazuh_break_glass_signatures_authorized( + signatures: Sequence[Mapping[str, Any]] | object, +) -> bool: + """Require two distinct principals across the two trusted roles.""" + + if not isinstance(signatures, Sequence) or isinstance( + signatures, + (str, bytes, bytearray), + ): + return False + principals: set[str] = set() + roles: set[str] = set() + for signature in signatures: + if not isinstance(signature, Mapping): + continue + context = trusted_wazuh_signature_context( + principal_id=str(signature.get("principal_id") or ""), + auth_method=str(signature.get("auth_method") or ""), + approver_role=str(signature.get("approver_role") or ""), + ) + if context is None: + continue + if ( + str(signature.get("signature_schema_version") or "") + != WAZUH_BREAK_GLASS_SIGNATURE_SCHEMA_VERSION + or str(signature.get("signer_id") or "") + != context["principal_id"] + ): + continue + if context["approver_role"] == TELEGRAM_OWNER_APPROVER_ROLE: + telegram_user_id = str( + signature.get("telegram_user_id") or "" + ).strip() + if ( + str(signature.get("source") or "") != "telegram" + or not telegram_user_id.isdigit() + or context["principal_id"] != f"tg_{telegram_user_id}" + ): + continue + elif str(signature.get("source") or "") != "api": + continue + principals.add(context["principal_id"]) + roles.add(context["approver_role"]) + return ( + roles == WAZUH_BREAK_GLASS_REQUIRED_SIGNATURE_ROLES + and len(principals) >= 2 + ) diff --git a/apps/api/tests/test_iwooos_wazuh_controlled_executor_runtime.py b/apps/api/tests/test_iwooos_wazuh_controlled_executor_runtime.py index 7ea3be560..d5967fa1a 100644 --- a/apps/api/tests/test_iwooos_wazuh_controlled_executor_runtime.py +++ b/apps/api/tests/test_iwooos_wazuh_controlled_executor_runtime.py @@ -55,6 +55,14 @@ def test_wazuh_runtime_sql_reads_schema_safe_packet_fallback() -> None: assert "LEFT JOIN incidents incident" in _LIVE_RUNTIME_SQL assert "incident_terminal_automation_run_id" in _LIVE_RUNTIME_SQL assert "incident_terminal_durable_write_acknowledged" in _LIVE_RUNTIME_SQL + assert "approval_records approval" in _LIVE_RUNTIME_SQL + assert "check_capability_retry_packet_receipt_id" in _LIVE_RUNTIME_SQL + assert "authorized_executor_dispatch_receipt_persisted" in ( + _LIVE_RUNTIME_SQL + ) + assert "critical_secret_reference_provisioning_receipt_verified" not in ( + _LIVE_RUNTIME_SQL + ) def _durable_scheduled_wazuh_source_recurrence( @@ -571,6 +579,258 @@ def test_wazuh_runtime_readback_keeps_only_secret_reference_as_critical() -> Non ) +def test_wazuh_secret_retry_probe_cannot_apply_without_break_glass_receipts() -> None: + run_id = "00000000-0000-0000-0000-000000000344" + recurrence = _current_wazuh_source_recurrence( + "2026-07-17T10:00:00+08:00" + ) + blocker = ( + "critical_secret_reference_owner_approval_and_broker_receipts_missing" + ) + row = { + "candidate_op_id": run_id, + "automation_run_id": run_id, + "trace_id": run_id, + "run_id": run_id, + "candidate_project_id": "awoooi", + "work_item_id": "P0-03-WAZUH-ALERT-INGRESS", + "proposal_source": "iwooos_wazuh_alert_ingress_scheduler", + "run_namespace": "awoooi:iwooos:wazuh-alert-ingress", + "source_receipt_ref": "wazuh-alert:2026071710", + "candidate_source_recurrence": recurrence, + "candidate_source_recurrence_verified": True, + "catalog_id": "ansible:wazuh-alertmanager-integration", + "check_op_id": "00000000-0000-0000-0000-000000000345", + "check_status": "success", + "check_returncode": "0", + "check_automation_run_id": run_id, + "check_trace_id": run_id, + "check_run_id": run_id, + "check_work_item_id": "P0-03-WAZUH-ALERT-INGRESS", + "check_source_recurrence": recurrence, + "check_capability_retry_generation": 1, + "check_controlled_apply_blocker": blocker, + "stage_ids": [], + } + + payload = build_iwooos_wazuh_controlled_executor_runtime_readback( + row, + executor_enabled=True, + ) + + assert payload["status"] == ( + "wazuh_alert_ingress_check_passed_apply_blocked_by_policy" + ) + assert payload["next_safe_action"] == "await_owner_break_glass_approval" + assert payload["active_blockers"] == [ + blocker, + "owner_break_glass_approval_pending", + ] + assert payload["summary"]["runtime_execution_performed_count"] == 0 + assert payload["summary"][ + "owner_break_glass_approval_verified_count" + ] == 0 + assert payload["summary"][ + "authorized_executor_dispatch_receipt_verified_count" + ] == 0 + assert payload["boundaries"]["critical_break_glass_required"] is True + assert payload["boundaries"]["host_write_performed"] is False + + +def test_wazuh_secret_retry_readback_verifies_db_receipt_lineage() -> None: + from src.services import awooop_ansible_check_mode_service as service + + run_id = "00000000-0000-0000-0000-000000000346" + check_id = "00000000-0000-0000-0000-000000000347" + source_check_id = "00000000-0000-0000-0000-000000000348" + packet_receipt_id = "00000000-0000-0000-0000-000000000349" + packet_id = "IWZ-CC-0011223344556677" + approval_id = service._wazuh_break_glass_approval_id(packet_id) + incident_id = "IWZ-I-2026071512-RECEIPTS" + recurrence = _current_wazuh_source_recurrence( + "2026-07-17T10:00:00+08:00" + ) + blocker = ( + "critical_secret_reference_owner_approval_and_broker_receipts_missing" + ) + row = { + "candidate_op_id": run_id, + "automation_run_id": run_id, + "trace_id": run_id, + "run_id": run_id, + "candidate_project_id": "awoooi", + "work_item_id": "P0-03-WAZUH-ALERT-INGRESS", + "proposal_source": "iwooos_wazuh_alert_ingress_scheduler", + "run_namespace": "awoooi:iwooos:wazuh-alert-ingress", + "source_receipt_ref": "wazuh-alert:2026071710", + "incident_id": incident_id, + "candidate_source_recurrence": recurrence, + "candidate_source_recurrence_verified": True, + "catalog_id": "ansible:wazuh-alertmanager-integration", + "check_op_id": check_id, + "check_status": "success", + "check_returncode": "0", + "check_automation_run_id": run_id, + "check_trace_id": run_id, + "check_run_id": run_id, + "check_work_item_id": "P0-03-WAZUH-ALERT-INGRESS", + "check_source_recurrence": recurrence, + "check_capability_retry_generation": 1, + "check_capability_retry_packet_receipt_id": packet_receipt_id, + "check_replay_of_check_mode_op_id": source_check_id, + "check_break_glass_approval_id": approval_id, + "check_break_glass_apply_authorized": False, + "check_controlled_apply_blocker": blocker, + "controlled_capability_packet_receipt_id": packet_receipt_id, + "controlled_capability_packet_status": "success", + "capability_packet_id": packet_id, + "capability_automation_run_id": run_id, + "capability_source_candidate_op_id": run_id, + "capability_source_check_mode_op_id": source_check_id, + "capability_trace_id": run_id, + "capability_run_id": run_id, + "capability_work_item_id": "P0-03-WAZUH-ALERT-INGRESS", + "capability_project_id": "awoooi", + "capability_run_namespace": "awoooi:iwooos:wazuh-alert-ingress", + "capability_proposal_source": ( + "iwooos_wazuh_alert_ingress_scheduler" + ), + "capability_source_recurrence": recurrence, + "capability_schema_version": ( + "wazuh_controlled_privileged_capability_repair_v3" + ), + "capability_type": "bounded_privileged_convergence", + "capability_risk_level": "critical", + "capability_policy_route": "critical_secret_reference_break_glass", + "capability_retry_generation": 1, + "capability_automatic_retry_exhausted": True, + "capability_automatic_retry_remaining": 0, + "capability_controlled_apply_allowed": False, + "capability_windows99_dispatch_required": False, + "capability_authorized_executor_dispatch_required": True, + "capability_authorized_executors": [ + "Agent99", + "awoooi-ansible-executor-broker", + ], + "capability_break_glass_approval_id": approval_id, + "capability_owner_approval_required": True, + "capability_critical_secret_reference_required": True, + "capability_expires_at": "2030-07-17T02:30:00+00:00", + "capability_queue_status": "retry_claimed", + "capability_execution_allowed": False, + "capability_granted": False, + "capability_packet_host_write_performed": False, + "capability_claim_receipt_persisted": True, + "capability_claimed_retry_check_mode_op_id": check_id, + "capability_opaque_secret_projection_verified": True, + "capability_authorized_executor_dispatch_receipt_persisted": True, + "capability_authorized_executor": ( + "awoooi-ansible-executor-broker" + ), + "capability_secret_value_read": False, + "capability_claim_host_write_performed": False, + "break_glass_approval_id": approval_id, + "break_glass_approval_action": ( + "OWNER_REVIEW_REQUIRED:" + "authorize_wazuh_secret_reference_controlled_apply" + ), + "break_glass_approval_status": "approved", + "break_glass_approval_risk_level": "critical", + "break_glass_approval_required_signatures": 2, + "break_glass_approval_current_signatures": 2, + "break_glass_approval_signature_count": 2, + "break_glass_trusted_owner_signature_verified": True, + "break_glass_trusted_operator_signature_verified": True, + "break_glass_trusted_distinct_principal_count": 2, + "break_glass_approval_incident_id": incident_id, + "break_glass_approval_resolved_at": ( + "2026-07-22T01:00:00+00:00" + ), + "break_glass_approval_expires_at": "2030-07-22T01:00:00+00:00", + "break_glass_approval_schema_version": ( + "wazuh_secret_reference_break_glass_approval_v1" + ), + "break_glass_approval_packet_receipt_id": packet_receipt_id, + "break_glass_approval_packet_id": packet_id, + "break_glass_approval_source_candidate_op_id": run_id, + "break_glass_approval_source_check_mode_op_id": source_check_id, + "break_glass_approval_automation_run_id": run_id, + "break_glass_trusted_signature_schema_version": ( + "wazuh_break_glass_trusted_signature_v1" + ), + "stage_ids": [], + } + + payload = build_iwooos_wazuh_controlled_executor_runtime_readback( + row, + executor_enabled=True, + ) + + assert payload["summary"][ + "owner_break_glass_approval_verified_count" + ] == 1 + assert payload["summary"][ + "opaque_secret_projection_receipt_verified_count" + ] == 1 + assert payload["summary"][ + "authorized_executor_dispatch_receipt_verified_count" + ] == 1 + assert payload["summary"][ + "break_glass_receipt_bundle_verified_count" + ] == 1 + assert payload["next_safe_action"] == ( + "broker_claim_owner_authorized_break_glass_apply" + ) + assert payload["break_glass_approval"]["approval_id"] == approval_id + assert payload["break_glass_approval"]["receipt_verified"] is True + assert payload["break_glass_approval"][ + "trusted_owner_signature_verified" + ] is True + assert payload["break_glass_approval"][ + "trusted_operator_signature_verified" + ] is True + assert payload["active_blockers"] == [ + blocker, + "owner_authorized_break_glass_apply_claim_pending", + ] + + forged_signatures = { + **row, + "break_glass_trusted_owner_signature_verified": False, + "break_glass_trusted_operator_signature_verified": False, + "break_glass_trusted_distinct_principal_count": 0, + } + forged_signature_payload = ( + build_iwooos_wazuh_controlled_executor_runtime_readback( + forged_signatures, + executor_enabled=True, + ) + ) + assert forged_signature_payload["summary"][ + "owner_break_glass_approval_verified_count" + ] == 0 + assert forged_signature_payload["next_safe_action"] == ( + "await_owner_break_glass_approval" + ) + + forged = { + **row, + "break_glass_approval_id": None, + "critical_secret_reference_provisioning_receipt_verified": True, + "windows99_dispatch_receipt_verified": True, + } + forged_payload = build_iwooos_wazuh_controlled_executor_runtime_readback( + forged, + executor_enabled=True, + ) + assert forged_payload["summary"][ + "owner_break_glass_approval_verified_count" + ] == 0 + assert forged_payload["next_safe_action"] == ( + "await_owner_break_glass_approval" + ) + + def test_wazuh_runtime_readback_projects_controlled_capability_packet() -> None: run_id = "00000000-0000-0000-0000-000000000314" recurrence = _current_wazuh_source_recurrence( @@ -625,8 +885,20 @@ def test_wazuh_runtime_readback_projects_controlled_capability_packet() -> None: "high_risk_controlled_apply_capability_repair" ), "capability_queue_status": "queued", + "capability_retry_generation": 0, + "capability_automatic_retry_exhausted": False, + "capability_automatic_retry_remaining": 1, "capability_controlled_apply_allowed": True, - "capability_windows99_dispatch_required": True, + "capability_windows99_dispatch_required": False, + "capability_authorized_executor_dispatch_required": True, + "capability_authorized_executors": [ + "Agent99", + "awoooi-ansible-executor-broker", + ], + "capability_source_candidate_op_id": run_id, + "capability_source_check_mode_op_id": ( + "00000000-0000-0000-0000-000000000313" + ), "capability_critical_secret_reference_required": False, "capability_expires_at": "2030-07-17T02:30:00+00:00", "capability_execution_allowed": False, @@ -643,15 +915,22 @@ def test_wazuh_runtime_readback_projects_controlled_capability_packet() -> None: assert payload["summary"][ "controlled_capability_repair_packet_count" ] == 1 + assert payload["summary"][ + "controlled_capability_repair_packet_receipt_count" + ] == 1 assert payload["next_safe_action"] == ( - "windows99_verify_controlled_privilege_capability_then_" + "authorized_executor_verify_controlled_privilege_capability_then_" "requeue_same_run" ) packet = payload["controlled_capability_repair_packet"] assert packet["queue_status"] == "queued" + assert packet["capability_retry_generation"] == 0 + assert packet["automatic_retry_exhausted"] is False + assert packet["automatic_retry_remaining"] == 1 assert packet["execution_allowed"] is False assert packet["controlled_apply_allowed"] is True - assert packet["windows99_dispatch_required"] is True + assert packet["windows99_dispatch_required"] is False + assert packet["authorized_executor_dispatch_required"] is True assert packet["capability_granted"] is False assert packet["host_write_performed"] is False assert packet["secret_value_exposed"] is False @@ -662,6 +941,23 @@ def test_wazuh_runtime_readback_projects_controlled_capability_packet() -> None: "controlled_capability_repair_packet_persisted" ] is True + legacy_packet_row = { + key: value + for key, value in row.items() + if key + not in { + "capability_retry_generation", + "capability_automatic_retry_exhausted", + "capability_automatic_retry_remaining", + } + } + legacy_packet = build_iwooos_wazuh_controlled_executor_runtime_readback( + legacy_packet_row, + executor_enabled=True, + )["controlled_capability_repair_packet"] + assert legacy_packet["capability_retry_generation"] == 0 + assert legacy_packet["automatic_retry_remaining"] == 1 + critical_row = { **row, "check_failure_marker": ( @@ -687,7 +983,7 @@ def test_wazuh_runtime_readback_projects_controlled_capability_packet() -> None: assert critical_packet["execution_allowed"] is False assert critical["next_safe_action"] == ( "provision_opaque_host112_secret_reference_then_" - "windows99_verify_and_requeue_same_run" + "broker_verify_and_requeue_same_run" ) assert "wazuh_critical_secret_reference_provisioning_pending" in ( critical["active_blockers"] @@ -701,10 +997,53 @@ def test_wazuh_runtime_readback_projects_controlled_capability_packet() -> None: assert expired["summary"][ "controlled_capability_repair_packet_count" ] == 0 + assert expired["summary"][ + "controlled_capability_repair_packet_receipt_count" + ] == 1 + assert expired["controlled_capability_repair_packet"]["active"] is False assert "controlled_capability_repair_packet_expired" in ( expired["active_blockers"] ) + exhausted_row = { + **row, + "capability_expires_at": "2030-07-17T02:30:00+00:00", + "capability_queue_status": "retry_exhausted", + "capability_retry_generation": 1, + "capability_retry_of_packet_id": "IWZ-CC-FFEEDDCCBBAA9988", + "capability_automatic_retry_exhausted": True, + "capability_automatic_retry_remaining": 0, + "check_capability_retry_generation": 1, + "check_capability_retry_of_packet_id": ( + "IWZ-CC-FFEEDDCCBBAA9988" + ), + } + exhausted = build_iwooos_wazuh_controlled_executor_runtime_readback( + exhausted_row, + executor_enabled=True, + ) + assert exhausted["status"] == ( + "wazuh_alert_ingress_same_run_capability_retry_exhausted" + ) + assert exhausted["summary"][ + "same_run_capability_retry_claimed_count" + ] == 1 + assert exhausted["summary"][ + "same_run_capability_retry_exhausted_count" + ] == 1 + assert exhausted["controlled_capability_repair_packet"][ + "queue_status" + ] == "retry_exhausted" + assert exhausted["controlled_capability_repair_packet"][ + "retry_of_capability_packet_id" + ] == "IWZ-CC-FFEEDDCCBBAA9988" + assert exhausted["next_safe_action"] == ( + "await_owner_authorized_capability_state_change_receipt" + ) + assert "wazuh_same_run_capability_retry_exhausted" in ( + exhausted["active_blockers"] + ) + def test_wazuh_runtime_readback_rejects_untrusted_failure_marker() -> None: row = { @@ -765,9 +1104,16 @@ def test_wazuh_controlled_capability_packet_is_deterministic_and_public_safe() - "capability_packet_id" ] assert packet_input["risk_level"] == "high" + assert packet_input["capability_retry_generation"] == 0 + assert packet_input["automatic_retry_exhausted"] is False assert packet_input["execution_allowed"] is False assert packet_input["controlled_apply_allowed"] is True - assert packet_input["windows99_dispatch_required"] is True + assert packet_input["windows99_dispatch_required"] is False + assert packet_input["authorized_executor_dispatch_required"] is True + assert packet_input["authorized_executors"] == [ + "Agent99", + "awoooi-ansible-executor-broker", + ] assert packet_input["critical_secret_reference_required"] is False assert packet_input["gate_lifecycle"]["expires_at"] == ( "2026-07-17T02:30:00+00:00" @@ -776,6 +1122,8 @@ def test_wazuh_controlled_capability_packet_is_deterministic_and_public_safe() - assert packet_input["project_id"] == "awoooi" assert packet_input["source_recurrence"]["verified"] is True assert packet_output["queue_status"] == "queued" + assert packet_output["automatic_retry_remaining"] == 1 + assert packet_output["telegram_notification_emitted"] is False assert dry_run["capability_granted"] is False assert "must-not-be-projected" not in serialized assert "command_output" not in serialized @@ -798,9 +1146,114 @@ def test_wazuh_controlled_capability_packet_is_deterministic_and_public_safe() - assert critical_input["execution_allowed"] is False assert critical_input["critical_secret_reference_required"] is True assert critical_input["gate_lifecycle"]["risk_class"] == "critical" - assert "critical_secret_reference_provisioning_receipt" in ( + assert "owner_critical_break_glass_approval" in ( critical_output["required_receipts"] ) + assert "opaque_secret_projection_presence" in ( + critical_output["required_receipts"] + ) + assert "ansible_executor_broker_dispatch" in ( + critical_output["required_receipts"] + ) + + exhausted_input, exhausted_output, _ = ( + _build_wazuh_controlled_capability_packet( + **{ + **kwargs, + "check_op_id": "00000000-0000-0000-0000-000000000399", + "source_input": { + **kwargs["source_input"], + "capability_retry_generation": 1, + "capability_retry_of_packet_id": ( + "IWZ-CC-FFEEDDCCBBAA9988" + ), + }, + } + ) + ) + assert exhausted_input["capability_retry_generation"] == 1 + assert exhausted_input["automatic_retry_exhausted"] is True + assert exhausted_input["retry_of_capability_packet_id"] == ( + "IWZ-CC-FFEEDDCCBBAA9988" + ) + assert exhausted_output["queue_status"] == "retry_exhausted" + assert exhausted_output["automatic_retry_remaining"] == 0 + assert exhausted_output["telegram_notification_emitted"] is False + + +def test_wazuh_secret_projection_gate_checks_only_allowlisted_file_metadata( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + from src.services import awooop_ansible_check_mode_service as service + + reference = tmp_path / "password" + reference.write_text("opaque-test-value", encoding="utf-8") + monkeypatch.setattr( + service, + "_WAZUH_BECOME_SECRET_REFERENCE", + reference, + ) + monkeypatch.setattr( + service.settings, + "AWOOOP_ANSIBLE_BECOME_PASSWORD_FILE", + str(reference), + ) + + assert service._wazuh_become_secret_reference_ready() is True + reference.write_text("", encoding="utf-8") + assert service._wazuh_become_secret_reference_ready() is False + monkeypatch.setattr( + service.settings, + "AWOOOP_ANSIBLE_BECOME_PASSWORD_FILE", + str(tmp_path / "foreign-reference"), + ) + assert service._wazuh_become_secret_reference_ready() is False + + +@pytest.mark.asyncio +async def test_wazuh_secret_packet_persists_owner_signable_approval() -> None: + from src.services import awooop_ansible_check_mode_service as service + from src.services.approval_action_classifier import ( + is_executable_repair_approval_action, + ) + + packet_id = "IWZ-CC-0011223344556677" + packet_receipt_id = "00000000-0000-0000-0000-000000000390" + statements: list[tuple[str, dict[str, object]]] = [] + + class _Db: + async def execute(self, statement, params=None): + statements.append((str(statement), dict(params or {}))) + return MagicMock() + + approval_id = await service._ensure_wazuh_break_glass_approval( + _Db(), + packet_receipt_id=packet_receipt_id, + capability_packet_id=packet_id, + incident_id="IWZ-I-2026071512-APPROVAL", + source_candidate_op_id=( + "00000000-0000-0000-0000-000000000391" + ), + source_check_mode_op_id=( + "00000000-0000-0000-0000-000000000392" + ), + automation_run_id="00000000-0000-0000-0000-000000000391", + ) + + assert approval_id == service._wazuh_break_glass_approval_id(packet_id) + assert is_executable_repair_approval_action( + service._WAZUH_BREAK_GLASS_APPROVAL_ACTION + ) is False + assert len(statements) == 2 + assert "INSERT INTO approval_records" in statements[0][0] + assert "ON CONFLICT (id) DO NOTHING" in statements[0][0] + assert "UPDATE automation_operation_log packet" in statements[1][0] + assert statements[1][1] == { + "approval_id": approval_id, + "packet_receipt_id": packet_receipt_id, + "capability_packet_id": packet_id, + } @pytest.mark.asyncio @@ -887,6 +1340,654 @@ async def test_failed_wazuh_check_atomically_queues_deduplicated_packet( assert packet_input["secret_value_exposed"] is False +@pytest.mark.asyncio +async def test_expired_wazuh_packet_claims_one_same_run_retry_after_secret_projection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services import awooop_ansible_check_mode_service as service + + run_id = "00000000-0000-0000-0000-000000000400" + failed_check_id = "00000000-0000-0000-0000-000000000401" + packet_id = "IWZ-CC-0011223344556677" + source_recurrence = _durable_scheduled_wazuh_source_recurrence( + "2026-07-15T05:50:00+08:00" + ) + original_input = { + "automation_run_id": run_id, + "trace_id": run_id, + "run_id": run_id, + "work_item_id": "P0-03-WAZUH-ALERT-INGRESS", + "source_recurrence": source_recurrence, + } + rows = [{ + "op_id": failed_check_id, + "parent_op_id": run_id, + "incident_id": "IWZ-I-2026071512-TEST", + "input": json.dumps(original_input), + "capability_packet_receipt_id": ( + "00000000-0000-0000-0000-000000000403" + ), + "capability_packet_id": packet_id, + "packet_automation_run_id": run_id, + "packet_source_candidate_op_id": run_id, + "packet_source_check_mode_op_id": failed_check_id, + "failure_class": ( + "wazuh_privileged_convergence_secret_reference_missing" + ), + }] + insert_succeeds = True + + class _Result: + def __init__(self, *, selected: bool): + self.selected = selected + + def mappings(self): + return self + + def all(self): + return rows if self.selected else [] + + def scalar_one_or_none(self): + if self.selected: + return None + return ( + "00000000-0000-0000-0000-000000000402" + if insert_succeeds + else None + ) + + inserted: list[dict[str, object]] = [] + + class _Db: + async def execute(self, statement, params=None): + sql = str(statement) + selected = "FOR UPDATE OF check_mode, packet SKIP LOCKED" in sql + if selected: + assert "capability_retry_of_packet_id" in sql + assert "JOIN automation_operation_log packet" in sql + assert params["max_retries"] == 1 + assert params["window_hours"] == 7 * 24 + else: + inserted.append({ + "sql": sql, + "params": dict(params or {}), + }) + return _Result(selected=selected) + + @asynccontextmanager + async def fake_db_context(project_id: str): + assert project_id == "awoooi" + yield _Db() + + original = AnsibleCheckModeClaim( + op_id=failed_check_id, + source_candidate_op_id=run_id, + incident_id="IWZ-I-2026071512-TEST", + catalog_id="ansible:wazuh-alertmanager-integration", + playbook_path=( + "infra/ansible/playbooks/wazuh-alertmanager-integration.yml" + ), + apply_playbook_path=( + "infra/ansible/playbooks/wazuh-alertmanager-integration.yml" + ), + inventory_hosts=("host_112",), + risk_level="high", + input_payload={ + **original_input, + "project_id": "awoooi", + }, + ) + reconstruct = MagicMock(return_value=original) + monkeypatch.setattr(service, "get_db_context", fake_db_context) + monkeypatch.setattr( + service, + "_claim_from_stale_check_mode_row", + reconstruct, + ) + approval_id = service._wazuh_break_glass_approval_id(packet_id) + ensure_approval = AsyncMock(return_value=approval_id) + monkeypatch.setattr( + service, + "_ensure_wazuh_break_glass_approval", + ensure_approval, + ) + + claims = await service.claim_expired_wazuh_controlled_capability_retries( + secret_reference_ready=lambda: True, + ) + + assert len(claims) == 1 + claim = claims[0] + assert claim.source_candidate_op_id == run_id + assert claim.input_payload["automation_run_id"] == run_id + assert claim.input_payload["trace_id"] == run_id + assert claim.input_payload["run_id"] == run_id + assert claim.input_payload["replay_of_check_mode_op_id"] == failed_check_id + assert claim.input_payload["capability_retry_of_packet_id"] == packet_id + assert claim.input_payload["capability_retry_packet_receipt_id"] == ( + "00000000-0000-0000-0000-000000000403" + ) + assert claim.input_payload["capability_retry_generation"] == 1 + assert claim.input_payload["break_glass_approval_id"] == approval_id + authority = claim.input_payload["source_recurrence_replay_authority"] + assert authority["secret_reference_presence_verified_without_read"] is True + assert authority["secret_value_read"] is False + assert authority["source_candidate_op_id"] == run_id + assert authority["automation_run_id"] == run_id + assert authority["capability_retry_generation"] == 1 + assert authority["capability_packet_receipt_id"] == ( + "00000000-0000-0000-0000-000000000403" + ) + assert claim.input_payload["controlled_apply_allowed"] is False + assert claim.input_payload["apply_enabled"] is False + assert claim.input_payload["approval_required_before_apply"] is True + assert claim.input_payload["controlled_apply_blocker"] == ( + "critical_secret_reference_owner_approval_and_broker_receipts_missing" + ) + assert inserted[0]["params"]["parent_op_id"] == run_id + assert inserted[0]["params"]["capability_packet_receipt_id"] == ( + "00000000-0000-0000-0000-000000000403" + ) + assert inserted[0]["params"]["capability_packet_id"] == packet_id + assert "WITH claimed_packet AS" in str(inserted[0]["sql"]) + assert "FROM claimed_packet" in str(inserted[0]["sql"]) + assert "'queue_status', 'retry_claimed'" in str(inserted[0]["sql"]) + assert "SELECT op_id FROM finalized_packet" in str(inserted[0]["sql"]) + ensure_approval.assert_awaited_once() + reconstruct.assert_called_once() + assert reconstruct.call_args.kwargs["replay_observed_now"] == ( + datetime.fromisoformat("2026-07-15T05:50:00+08:00") + ) + + inserted.clear() + reconstruct.reset_mock() + insert_succeeds = False + raced = await service.claim_expired_wazuh_controlled_capability_retries( + secret_reference_ready=lambda: True, + ) + assert raced == [] + assert len(inserted) == 1 + assert "FROM claimed_packet" in str(inserted[0]["sql"]) + reconstruct.assert_called_once() + + inserted.clear() + reconstruct.reset_mock() + insert_succeeds = True + blocked = await service.claim_expired_wazuh_controlled_capability_retries( + secret_reference_ready=lambda: False, + ) + assert blocked == [] + assert inserted == [] + reconstruct.assert_not_called() + + rows[0]["input"] = json.dumps({ + **original_input, + "capability_retry_generation": 1, + }) + exhausted = await service.claim_expired_wazuh_controlled_capability_retries( + secret_reference_ready=lambda: True, + ) + assert exhausted == [] + assert inserted == [] + reconstruct.assert_not_called() + + +@pytest.mark.asyncio +async def test_execution_broker_prioritizes_same_run_wazuh_capability_retry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services import awooop_ansible_check_mode_service as service + + run_id = "00000000-0000-0000-0000-000000000410" + claim = AnsibleCheckModeClaim( + op_id="00000000-0000-0000-0000-000000000411", + source_candidate_op_id=run_id, + incident_id="IWZ-I-2026071512-RETRY", + catalog_id="ansible:wazuh-alertmanager-integration", + playbook_path=( + "infra/ansible/playbooks/wazuh-alertmanager-integration.yml" + ), + apply_playbook_path=( + "infra/ansible/playbooks/wazuh-alertmanager-integration.yml" + ), + inventory_hosts=("host_112",), + risk_level="high", + input_payload={ + "automation_run_id": run_id, + "trace_id": run_id, + "run_id": run_id, + "capability_retry_generation": 1, + "capability_retry_of_packet_id": "IWZ-CC-0011223344556677", + "controlled_apply_allowed": False, + "controlled_apply_blocker": ( + "critical_secret_reference_owner_approval_and_broker_" + "receipts_missing" + ), + }, + ) + capability_claimer = AsyncMock(return_value=[claim]) + break_glass_apply_claimer = AsyncMock(return_value=[]) + stale_claimer = AsyncMock(return_value=[]) + fresh_claimer = AsyncMock(return_value=[]) + issue_capability = AsyncMock( + return_value=( + claim, + "00000000-0000-0000-0000-000000000412", + ) + ) + check_runner = AsyncMock( + return_value=AnsibleRunResult( + returncode=0, + stdout="", + stderr="", + duration_ms=5, + ) + ) + monkeypatch.setattr( + service, + "_expire_stale_ansible_execution_capabilities", + AsyncMock(return_value=0), + ) + monkeypatch.setattr( + service, + "backfill_missing_stdin_boundary_terminal_projections_once", + AsyncMock(return_value={}), + ) + monkeypatch.setattr( + service, + "backfill_missing_auto_repair_execution_receipts_once", + AsyncMock(return_value={}), + ) + monkeypatch.setattr( + service, + "run_failed_apply_check_mode_replay_once", + AsyncMock(return_value={}), + ) + monkeypatch.setattr(service, "_runtime_blockers", lambda: []) + monkeypatch.setattr( + service, + "recent_ansible_transport_blockers", + AsyncMock(return_value=[]), + ) + monkeypatch.setattr( + service, + "claim_owner_authorized_wazuh_secret_retry_applies", + break_glass_apply_claimer, + ) + monkeypatch.setattr( + service, + "claim_expired_wazuh_controlled_capability_retries", + capability_claimer, + ) + monkeypatch.setattr( + service, + "claim_stale_pending_check_modes", + stale_claimer, + ) + monkeypatch.setattr(service, "claim_pending_check_modes", fresh_claimer) + monkeypatch.setattr( + service, + "_issue_ansible_execution_capability", + issue_capability, + ) + monkeypatch.setattr(service, "run_claimed_check_mode", check_runner) + monkeypatch.setattr( + service, + "_revoke_ansible_execution_capability", + AsyncMock(return_value=True), + ) + + result = await service.run_pending_check_modes_once(limit=1) + + assert result["claimed"] == 1 + assert result["wazuh_capability_retry_claimed"] == 1 + assert result["wazuh_break_glass_apply_claimed"] == 0 + assert result["wazuh_capability_retry_error"] is None + assert result["completed"] == 1 + assert result["failed"] == 0 + assert result["apply_completed"] == 0 + assert result["apply_blocked"] == 1 + capability_claimer.assert_awaited_once_with( + project_id="awoooi", + limit=1, + ) + break_glass_apply_claimer.assert_awaited_once() + stale_claimer.assert_not_awaited() + fresh_claimer.assert_not_awaited() + issue_capability.assert_awaited_once() + check_runner.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_owner_approved_secret_retry_claims_apply_from_db_receipts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services import awooop_ansible_check_mode_service as service + + run_id = "00000000-0000-0000-0000-000000000420" + check_id = "00000000-0000-0000-0000-000000000421" + packet_receipt_id = "00000000-0000-0000-0000-000000000422" + packet_id = "IWZ-CC-0011223344556677" + approval_id = service._wazuh_break_glass_approval_id(packet_id) + input_payload = { + "automation_run_id": run_id, + "trace_id": run_id, + "run_id": run_id, + "break_glass_approval_id": approval_id, + "capability_retry_packet_receipt_id": packet_receipt_id, + "controlled_apply_allowed": False, + "controlled_apply_blocker": ( + "critical_secret_reference_owner_approval_and_broker_" + "receipts_missing" + ), + } + row = { + "op_id": check_id, + "parent_op_id": run_id, + "incident_id": "IWZ-I-2026071512-APPROVED", + "input": input_payload, + "retry_packet_receipt_id": packet_receipt_id, + "retry_packet_status": "success", + "retry_packet_parent_op_id": ( + "00000000-0000-0000-0000-000000000419" + ), + "retry_packet_id": packet_id, + "retry_packet_queue_status": "retry_claimed", + "retry_packet_claim_receipt_persisted": True, + "retry_packet_host_write_performed": False, + "retry_packet_secret_value_read": False, + "retry_packet_authorized_executor_dispatch_receipt_persisted": True, + "retry_packet_authorized_executor": ( + "awoooi-ansible-executor-broker" + ), + "opaque_secret_projection_presence_verified": True, + "break_glass_approval_id": approval_id, + "break_glass_approval_resolved_at": datetime( + 2026, + 7, + 22, + 1, + 0, + tzinfo=UTC, + ), + } + source_claim = AnsibleCheckModeClaim( + op_id=check_id, + source_candidate_op_id=run_id, + incident_id="IWZ-I-2026071512-APPROVED", + catalog_id="ansible:wazuh-alertmanager-integration", + playbook_path=( + "infra/ansible/playbooks/wazuh-alertmanager-integration.yml" + ), + apply_playbook_path=( + "infra/ansible/playbooks/wazuh-alertmanager-integration.yml" + ), + inventory_hosts=("host_112",), + risk_level="critical", + input_payload=input_payload, + ) + executed_sql: list[str] = [] + update_params: list[dict[str, object]] = [] + + class _SelectResult: + def mappings(self): + return self + + def all(self): + return [row] + + class _UpdateResult: + def scalar_one_or_none(self): + return check_id + + class _Db: + async def execute(self, statement, params=None): + sql = str(statement) + executed_sql.append(sql) + if "FOR UPDATE OF check_mode, packet, approval" in sql: + assert params["approval_action"] == ( + service._WAZUH_BREAK_GLASS_APPROVAL_ACTION + ) + return _SelectResult() + update_params.append(dict(params or {})) + return _UpdateResult() + + @asynccontextmanager + async def fake_db_context(project_id: str): + assert project_id == "awoooi" + yield _Db() + + reconstruct = MagicMock(return_value=source_claim) + monkeypatch.setattr(service, "get_db_context", fake_db_context) + monkeypatch.setattr( + service, + "_claim_from_stale_check_mode_row", + reconstruct, + ) + + claims = await service.claim_owner_authorized_wazuh_secret_retry_applies() + + assert len(claims) == 1 + claim = claims[0] + assert claim.input_payload["controlled_apply_allowed"] is True + assert claim.input_payload["controlled_apply_blocker"] is None + assert claim.input_payload["break_glass_apply_authorized"] is True + assert claim.input_payload["break_glass_receipt_bundle"] == { + "schema_version": "wazuh_secret_reference_break_glass_receipts_v1", + "owner_approval_receipt_id": approval_id, + "owner_approval_resolved_at": "2026-07-22T01:00:00+00:00", + "trusted_owner_approval_signatures_verified": True, + "opaque_secret_projection_receipt_id": packet_receipt_id, + "authorized_executor_dispatch_receipt_id": packet_receipt_id, + "authorized_executor": "awoooi-ansible-executor-broker", + "secret_value_read": False, + "host_write_performed_before_apply": False, + } + select_sql = executed_sql[0] + assert "approval.required_signatures >= 2" in select_sql + assert "jsonb_array_length" in select_sql + assert "telegram_owner_auth_method" in select_sql + assert "controlled_operator_auth_method" in select_sql + assert "break_glass_apply_terminal_status" in select_sql + assert "opaque_secret_projection_presence_verified" in select_sql + assert "authorized_executor_dispatch_receipt_persisted" in select_sql + assert "FOR UPDATE OF check_mode, packet, approval SKIP LOCKED" in select_sql + persisted_input = json.loads(str(update_params[0]["input"])) + assert persisted_input["break_glass_approval_id"] == approval_id + assert persisted_input["break_glass_apply_authorized"] is True + + +@pytest.mark.asyncio +async def test_owner_authorized_trust_block_writes_terminal_no_reclaim_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services import awooop_ansible_check_mode_service as service + + claim = AnsibleCheckModeClaim( + op_id="00000000-0000-0000-0000-000000000451", + source_candidate_op_id="00000000-0000-0000-0000-000000000450", + incident_id="IWZ-I-2026071512-TRUST-BLOCK", + catalog_id="ansible:wazuh-alertmanager-integration", + playbook_path=( + "infra/ansible/playbooks/wazuh-alertmanager-integration.yml" + ), + apply_playbook_path=( + "infra/ansible/playbooks/wazuh-alertmanager-integration.yml" + ), + inventory_hosts=("host_112",), + risk_level="critical", + input_payload={ + "automation_run_id": ( + "00000000-0000-0000-0000-000000000450" + ), + "controlled_apply_allowed": True, + "break_glass_apply_authorized": True, + "break_glass_apply_claim_id": "claim-451", + }, + ) + statements: list[tuple[str, dict[str, object]]] = [] + + class _Mappings: + def first(self): + return None + + class _Result: + def mappings(self): + return _Mappings() + + class _Db: + async def execute(self, statement, params=None): + statements.append((str(statement), dict(params or {}))) + return _Result() + + @asynccontextmanager + async def fake_db_context(_project_id: str): + yield _Db() + + monkeypatch.setattr(service, "get_db_context", fake_db_context) + monkeypatch.setattr( + service, + "_execution_capability_timeout_seconds", + MagicMock(return_value=60), + ) + monkeypatch.setattr( + service, + "_revalidate_claim_playbook_trust", + AsyncMock(return_value=False), + ) + + result = await service.run_controlled_apply_for_claim( + claim, + timeout_seconds=60, + ) + + assert result is None + assert len(statements) == 3 + terminal_sql, terminal_params = statements[-1] + assert "break_glass_apply_terminal_status" in terminal_sql + assert "break_glass_apply_reclaim_allowed', false" in terminal_sql + assert terminal_params["claim_id"] == "claim-451" + persisted_input = json.loads(str(terminal_params["input"])) + assert persisted_input["break_glass_apply_terminal_status"] == ( + "blocked_by_playbook_trust_circuit_open" + ) + assert persisted_input["break_glass_apply_reclaim_allowed"] is False + + +@pytest.mark.asyncio +async def test_execution_broker_runs_owner_authorized_apply_without_rechecking( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.services import awooop_ansible_check_mode_service as service + + run_id = "00000000-0000-0000-0000-000000000430" + claim = AnsibleCheckModeClaim( + op_id="00000000-0000-0000-0000-000000000431", + source_candidate_op_id=run_id, + incident_id="IWZ-I-2026071512-APPLY", + catalog_id="ansible:wazuh-alertmanager-integration", + playbook_path=( + "infra/ansible/playbooks/wazuh-alertmanager-integration.yml" + ), + apply_playbook_path=( + "infra/ansible/playbooks/wazuh-alertmanager-integration.yml" + ), + inventory_hosts=("host_112",), + risk_level="critical", + input_payload={ + "automation_run_id": run_id, + "trace_id": run_id, + "run_id": run_id, + "controlled_apply_allowed": True, + "break_glass_apply_authorized": True, + }, + ) + break_glass_claimer = AsyncMock(return_value=[claim]) + retry_claimer = AsyncMock(return_value=[]) + check_runner = AsyncMock() + apply_runner = AsyncMock( + return_value=AnsibleRunResult( + returncode=0, + stdout="", + stderr="", + duration_ms=10, + post_verifier_passed=True, + ) + ) + monkeypatch.setattr( + service, + "_expire_stale_ansible_execution_capabilities", + AsyncMock(return_value=0), + ) + monkeypatch.setattr( + service, + "backfill_missing_stdin_boundary_terminal_projections_once", + AsyncMock(return_value={}), + ) + monkeypatch.setattr( + service, + "backfill_missing_auto_repair_execution_receipts_once", + AsyncMock(return_value={}), + ) + monkeypatch.setattr( + service, + "run_failed_apply_check_mode_replay_once", + AsyncMock(return_value={}), + ) + monkeypatch.setattr(service, "_runtime_blockers", lambda: []) + monkeypatch.setattr( + service, + "recent_ansible_transport_blockers", + AsyncMock(return_value=[]), + ) + monkeypatch.setattr( + service, + "claim_owner_authorized_wazuh_secret_retry_applies", + break_glass_claimer, + ) + monkeypatch.setattr( + service, + "claim_expired_wazuh_controlled_capability_retries", + retry_claimer, + ) + monkeypatch.setattr( + service, + "_issue_ansible_execution_capability", + AsyncMock( + return_value=( + claim, + "00000000-0000-0000-0000-000000000432", + ) + ), + ) + monkeypatch.setattr(service, "run_claimed_check_mode", check_runner) + monkeypatch.setattr( + service, + "run_controlled_apply_for_claim", + apply_runner, + ) + revoke = AsyncMock(return_value=True) + monkeypatch.setattr( + service, + "_revoke_ansible_execution_capability", + revoke, + ) + + result = await service.run_pending_check_modes_once(limit=1) + + assert result["claimed"] == 1 + assert result["check_mode_claimed"] == 0 + assert result["wazuh_break_glass_apply_claimed"] == 1 + assert result["completed"] == 0 + assert result["apply_completed"] == 1 + assert result["apply_failed"] == 0 + break_glass_claimer.assert_awaited_once() + retry_claimer.assert_not_awaited() + check_runner.assert_not_awaited() + apply_runner.assert_awaited_once() + revoke.assert_awaited_once() + + @pytest.mark.asyncio async def test_wazuh_controlled_capability_packet_backfill_closes_existing_gap( monkeypatch: pytest.MonkeyPatch, diff --git a/apps/api/tests/test_telegram_webhook_execution_handoff.py b/apps/api/tests/test_telegram_webhook_execution_handoff.py index 8412bf5b2..dec1ac4cc 100644 --- a/apps/api/tests/test_telegram_webhook_execution_handoff.py +++ b/apps/api/tests/test_telegram_webhook_execution_handoff.py @@ -29,8 +29,10 @@ class _FakeApprovalService: self.approval = approval self.execution_triggered = execution_triggered self.sign_message = sign_message + self.sign_calls: list[dict] = [] - async def sign_approval(self, **_kwargs): + async def sign_approval(self, **kwargs): + self.sign_calls.append(kwargs) return self.approval, self.sign_message, self.execution_triggered async def reject_approval(self, **_kwargs): @@ -83,10 +85,14 @@ async def test_telegram_approval_schedules_executor_after_required_signature(mon "user": {"id": 42, "username": "ops"}, }) monkeypatch.setattr(telegram_api, "get_telegram_gateway", lambda: fake_gateway) + approval_service = _FakeApprovalService( + approval, + execution_triggered=True, + ) monkeypatch.setattr( telegram_api, "get_approval_service", - lambda: _FakeApprovalService(approval, execution_triggered=True), + lambda: approval_service, ) monkeypatch.setattr(telegram_api, "_finalize_telegram_approval", fake_finalize) monkeypatch.setattr( @@ -115,6 +121,15 @@ async def test_telegram_approval_schedules_executor_after_required_signature(mon "execution_triggered": True, }] assert op_log_repo.rows[0]["kwargs"]["action_detail"] == "approve" + assert approval_service.sign_calls[0]["authenticated_principal_id"] == ( + "tg_42" + ) + assert approval_service.sign_calls[0]["authentication_method"] == ( + "telegram_whitelist_nonce" + ) + assert approval_service.sign_calls[0]["approver_role"] == "owner" + assert approval_service.sign_calls[0]["telegram_user_id"] == 42 + assert approval_service.sign_calls[0]["telegram_message_id"] == 99 @pytest.mark.asyncio diff --git a/apps/api/tests/test_wazuh_alertmanager_integration.py b/apps/api/tests/test_wazuh_alertmanager_integration.py index b17cf95d8..df6c8cadf 100644 --- a/apps/api/tests/test_wazuh_alertmanager_integration.py +++ b/apps/api/tests/test_wazuh_alertmanager_integration.py @@ -353,6 +353,7 @@ async def test_wazuh_ingress_scheduler_records_one_high_risk_controlled_candidat "wazuh-source-occurrence-5710" ) assert params["scheduled_recurrence"] is False + assert params["retain_controlled_capability_owner"] is True query = str(statement) assert "candidate.input ->> 'project_id' = :project_id" in query assert "candidate.input ->> 'work_item_id' = :work_item_id" in query @@ -366,6 +367,9 @@ async def test_wazuh_ingress_scheduler_records_one_high_risk_controlled_candidat "candidate.input #>> '{source_recurrence,occurrence_id}' = :source_occurrence_id" in query ) + assert ":retain_controlled_capability_owner" in query + assert "IN ('queued', 'retry_exhausted')" in query + assert "latest_check_controlled_apply_blocker" in query return _Result() @asynccontextmanager @@ -432,6 +436,132 @@ async def test_wazuh_ingress_scheduler_records_one_high_risk_controlled_candidat assert claim["work_item_id"] == "P0-03-WAZUH-ALERT-INGRESS" assert claim["run_namespace"] == "awoooi:iwooos:wazuh-alert-ingress" assert claim["source_recurrence"]["verified"] is True + from src.services import awooop_ansible_check_mode_service as check_service + + replay_claim = check_service._claim_from_stale_check_mode_row( + { + "op_id": "00000000-0000-0000-0000-000000000575", + "parent_op_id": result["automation_run_id"], + "incident_id": recorded["incident"]["incident_id"], + "input": claim, + }, + replay_observed_now=datetime.fromisoformat( + "2026-07-15T05:59:59+08:00" + ), + ) + assert replay_claim is not None + assert replay_claim.source_candidate_op_id == result["automation_run_id"] + assert replay_claim.input_payload["automation_run_id"] == ( + result["automation_run_id"] + ) + failed_check_mode_op_id = "00000000-0000-0000-0000-000000000576" + capability_packet_receipt_id = ( + "00000000-0000-0000-0000-000000000580" + ) + capability_packet_id = "IWZ-CC-0011223344556677" + break_glass_approval_id = ( + check_service._wazuh_break_glass_approval_id(capability_packet_id) + ) + retry_input = { + **replay_claim.input_payload, + "execution_mode": "wazuh_controlled_capability_check_mode_retry", + "replay_of_check_mode_op_id": failed_check_mode_op_id, + "capability_retry_packet_receipt_id": capability_packet_receipt_id, + "capability_retry_of_packet_id": capability_packet_id, + "capability_retry_failure_class": ( + "wazuh_privileged_convergence_secret_reference_missing" + ), + "capability_retry_generation": 1, + "break_glass_approval_id": break_glass_approval_id, + "same_run_correlation_verified": True, + "bounded_execution_scope": ( + "same_run_check_mode_only_until_owner_authorized_" + "break_glass_receipts" + ), + "source_recurrence_replay_authority": { + "schema_version": ( + "wazuh_capability_same_run_retry_authority_v1" + ), + "capability_packet_id": capability_packet_id, + "capability_packet_receipt_id": capability_packet_receipt_id, + "break_glass_approval_id": break_glass_approval_id, + "source_check_mode_op_id": failed_check_mode_op_id, + "source_candidate_op_id": result["automation_run_id"], + "automation_run_id": result["automation_run_id"], + "capability_retry_generation": 1, + "source_recurrence_observed_at": ( + claim["source_recurrence"]["observed_at"] + ), + "secret_reference_presence_verified_without_read": True, + "secret_value_read": False, + }, + } + crash_reclaimed_retry = check_service._claim_from_stale_check_mode_row( + { + "op_id": "00000000-0000-0000-0000-000000000577", + "parent_op_id": result["automation_run_id"], + "incident_id": recorded["incident"]["incident_id"], + "input": retry_input, + "retry_packet_receipt_id": capability_packet_receipt_id, + "retry_packet_status": "success", + "retry_packet_parent_op_id": failed_check_mode_op_id, + "retry_packet_id": capability_packet_id, + "retry_packet_source_check_mode_op_id": failed_check_mode_op_id, + "retry_packet_source_candidate_op_id": result["automation_run_id"], + "retry_packet_automation_run_id": result["automation_run_id"], + "retry_packet_queue_status": "retry_claimed", + "retry_packet_claimed_check_mode_op_id": ( + "00000000-0000-0000-0000-000000000577" + ), + "retry_packet_failure_class": ( + "wazuh_privileged_convergence_secret_reference_missing" + ), + "retry_packet_claim_receipt_persisted": True, + "retry_packet_host_write_performed": False, + "retry_packet_secret_value_read": False, + "retry_packet_authorized_executor_dispatch_receipt_persisted": ( + True + ), + "retry_packet_authorized_executor": ( + "awoooi-ansible-executor-broker" + ), + } + ) + assert crash_reclaimed_retry is not None + assert crash_reclaimed_retry.input_payload[ + "capability_retry_of_packet_id" + ] == capability_packet_id + assert crash_reclaimed_retry.input_payload[ + "capability_retry_generation" + ] == 1 + assert crash_reclaimed_retry.input_payload[ + "source_recurrence_replay_authority" + ] == retry_input["source_recurrence_replay_authority"] + assert crash_reclaimed_retry.input_payload[ + "controlled_apply_allowed" + ] is False + assert crash_reclaimed_retry.input_payload[ + "controlled_apply_blocker" + ] == ( + "critical_secret_reference_owner_approval_and_broker_receipts_missing" + ) + forged_packet_id = "IWZ-CC-FFEEDDCCBBAA9988" + tampered_retry_input = { + **retry_input, + "capability_retry_of_packet_id": forged_packet_id, + "source_recurrence_replay_authority": { + **retry_input["source_recurrence_replay_authority"], + "capability_packet_id": forged_packet_id, + }, + } + assert check_service._claim_from_stale_check_mode_row( + { + "op_id": "00000000-0000-0000-0000-000000000578", + "parent_op_id": result["automation_run_id"], + "incident_id": recorded["incident"]["incident_id"], + "input": tampered_retry_input, + } + ) is None mismatched_candidate = {**candidate["input"], "work_item_id": "other-lane"} with pytest.raises( ValueError, @@ -651,6 +781,377 @@ async def test_wazuh_ingress_scheduler_does_not_repeat_failed_check_while_capabi binder.assert_not_awaited() +@pytest.mark.asyncio +async def test_wazuh_ingress_scheduler_does_not_escape_exhausted_same_run_retry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.jobs import awooop_ansible_candidate_backfill_job as job + + run_id = "00000000-0000-0000-0000-000000000574" + + class _Mappings: + def first(self): + return { + "op_id": run_id, + "automation_run_id": run_id, + "source_receipt_ref": "wazuh-alert:2026071500", + "latest_check_status": "failed", + "latest_apply_status": None, + "latest_learning_status": None, + "latest_controlled_capability_packet_status": "pending", + "latest_controlled_capability_queue_status": ( + "retry_exhausted" + ), + "latest_controlled_capability_failure_class": ( + "wazuh_privileged_convergence_secret_reference_missing" + ), + "latest_controlled_capability_retry_generation": "1", + "latest_controlled_capability_packet_expires_at": ( + "2026-07-15T05:29:59+08:00" + ), + "predecision_evidence_ready": True, + } + + class _Result: + def mappings(self): + return _Mappings() + + class _Db: + async def execute(self, statement, params): + query = str(statement) + assert params["retain_controlled_capability_owner"] is True + assert ":retain_controlled_capability_owner" in query + assert "IN ('queued', 'retry_exhausted')" in query + return _Result() + + @asynccontextmanager + async def fake_db_context(_project_id: str): + yield _Db() + + recorder = AsyncMock(return_value=True) + collector = AsyncMock(return_value=MagicMock(snapshot_id="must-not-run")) + verifier = AsyncMock(return_value=True) + binder = AsyncMock(return_value=True) + monkeypatch.setattr(job, "get_db_context", fake_db_context) + monkeypatch.setattr( + job.settings, + "ENABLE_IWOOOS_WAZUH_MANAGER_POSTURE_EXECUTOR", + True, + ) + monkeypatch.setattr( + job, + "now_taipei", + MagicMock(return_value=datetime.fromisoformat("2026-07-15T05:59:59+08:00")), + ) + + result = await job.enqueue_iwooos_wazuh_alertmanager_integration_candidate_once( + recorder=recorder, + evidence_collector=collector, + evidence_verifier=verifier, + incident_binder=binder, + source_recurrence=_current_wazuh_source_recurrence( + "2026-07-15T05:59:58+08:00" + ), + ) + + assert result["status"] == ( + "controlled_capability_same_run_retry_exhausted" + ) + assert result["queued"] == 0 + assert result["automation_run_id"] == run_id + assert result["capability_packet_active"] is False + assert result["capability_retry_generation"] == 1 + assert result["active_blockers"] == [ + "wazuh_same_run_capability_retry_exhausted" + ] + recorder.assert_not_awaited() + collector.assert_not_awaited() + verifier.assert_not_awaited() + binder.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_wazuh_ingress_scheduler_retains_apply_blocked_retry_owner( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.jobs import awooop_ansible_candidate_backfill_job as job + + run_id = "00000000-0000-0000-0000-000000000579" + blocker = ( + "critical_secret_reference_owner_approval_and_broker_receipts_missing" + ) + + class _Mappings: + def first(self): + return { + "op_id": run_id, + "automation_run_id": run_id, + "latest_check_status": "success", + "latest_check_capability_retry_generation": "1", + "latest_check_controlled_apply_blocker": blocker, + "latest_apply_status": None, + "latest_learning_status": None, + "latest_controlled_capability_packet_status": "success", + "latest_controlled_capability_queue_status": "retry_claimed", + "latest_controlled_capability_retry_generation": "0", + "predecision_evidence_ready": True, + } + + class _Result: + def mappings(self): + return _Mappings() + + class _Db: + async def execute(self, statement, params): + query = str(statement) + assert params["freshness_hours"] == 6 + assert params["retain_controlled_capability_owner"] is True + assert "check_mode.input" in query + assert "controlled_apply_blocker" in query + return _Result() + + @asynccontextmanager + async def fake_db_context(_project_id: str): + yield _Db() + + recorder = AsyncMock(return_value=True) + collector = AsyncMock(return_value=MagicMock(snapshot_id="must-not-run")) + verifier = AsyncMock(return_value=True) + binder = AsyncMock(return_value=True) + monkeypatch.setattr(job, "get_db_context", fake_db_context) + monkeypatch.setattr( + job.settings, + "ENABLE_IWOOOS_WAZUH_MANAGER_POSTURE_EXECUTOR", + True, + ) + monkeypatch.setattr( + job.settings, + "IWOOOS_WAZUH_MANAGER_POSTURE_FRESHNESS_HOURS", + 6, + ) + monkeypatch.setattr( + job, + "now_taipei", + MagicMock( + return_value=datetime.fromisoformat("2026-07-15T12:59:59+08:00") + ), + ) + + result = await job.enqueue_iwooos_wazuh_alertmanager_integration_candidate_once( + recorder=recorder, + evidence_collector=collector, + evidence_verifier=verifier, + incident_binder=binder, + source_recurrence=_current_wazuh_source_recurrence( + "2026-07-15T12:59:58+08:00" + ), + ) + + assert result["status"] == ( + "critical_secret_reference_break_glass_receipts_pending" + ) + assert result["queued"] == 0 + assert result["automation_run_id"] == run_id + assert result["capability_retry_generation"] == 1 + assert result["controlled_apply_allowed"] is False + assert result["active_blockers"] == [blocker] + recorder.assert_not_awaited() + collector.assert_not_awaited() + verifier.assert_not_awaited() + binder.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("apply_status", "learning_status", "expected_status", "expected_blocker"), + [ + ( + None, + None, + "owner_authorized_break_glass_apply_in_progress", + "owner_authorized_break_glass_apply_receipt_pending", + ), + ( + "success", + None, + "same_run_break_glass_learning_writeback_pending", + "same_run_break_glass_learning_writeback_pending", + ), + ], +) +async def test_wazuh_ingress_scheduler_keeps_authorized_same_run_until_learning( + monkeypatch: pytest.MonkeyPatch, + apply_status: str | None, + learning_status: str | None, + expected_status: str, + expected_blocker: str, +) -> None: + from src.jobs import awooop_ansible_candidate_backfill_job as job + + run_id = "00000000-0000-0000-0000-000000000589" + approval_id = "00000000-0000-0000-0000-000000000590" + + class _Mappings: + def first(self): + return { + "op_id": run_id, + "automation_run_id": run_id, + "incident_id": "IWZ-I-2026071512-AUTHORIZED", + "latest_check_status": "success", + "latest_check_capability_retry_generation": "1", + "latest_check_controlled_apply_blocker": None, + "latest_check_break_glass_approval_id": approval_id, + "latest_check_break_glass_apply_authorized": True, + "latest_check_break_glass_apply_terminal_status": "", + "latest_apply_status": apply_status, + "latest_learning_status": learning_status, + "latest_controlled_capability_packet_status": "success", + "latest_controlled_capability_queue_status": "retry_claimed", + "latest_controlled_capability_retry_generation": "1", + "predecision_evidence_ready": True, + } + + class _Result: + def mappings(self): + return _Mappings() + + class _Db: + async def execute(self, statement, params): + query = str(statement) + assert params["retain_controlled_capability_owner"] is True + assert "break_glass_apply_authorized" in query + assert "learning.status" in query + return _Result() + + @asynccontextmanager + async def fake_db_context(_project_id: str): + yield _Db() + + recorder = AsyncMock(return_value=True) + collector = AsyncMock(return_value=MagicMock(snapshot_id="must-not-run")) + verifier = AsyncMock(return_value=True) + binder = AsyncMock(return_value=True) + monkeypatch.setattr(job, "get_db_context", fake_db_context) + monkeypatch.setattr( + job.settings, + "ENABLE_IWOOOS_WAZUH_MANAGER_POSTURE_EXECUTOR", + True, + ) + monkeypatch.setattr( + job.settings, + "IWOOOS_WAZUH_MANAGER_POSTURE_FRESHNESS_HOURS", + 6, + ) + monkeypatch.setattr( + job, + "now_taipei", + MagicMock( + return_value=datetime.fromisoformat("2026-07-15T12:59:59+08:00") + ), + ) + + result = await job.enqueue_iwooos_wazuh_alertmanager_integration_candidate_once( + recorder=recorder, + evidence_collector=collector, + evidence_verifier=verifier, + incident_binder=binder, + source_recurrence=_current_wazuh_source_recurrence( + "2026-07-15T12:59:58+08:00" + ), + ) + + assert result["status"] == expected_status + assert result["queued"] == 0 + assert result["automation_run_id"] == run_id + assert result["controlled_apply_allowed"] is True + assert result["break_glass_approval_id"] == approval_id + assert result["active_blockers"] == [expected_blocker] + recorder.assert_not_awaited() + collector.assert_not_awaited() + verifier.assert_not_awaited() + binder.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_wazuh_ingress_scheduler_surfaces_terminal_trust_block( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.jobs import awooop_ansible_candidate_backfill_job as job + + run_id = "00000000-0000-0000-0000-000000000591" + approval_id = "00000000-0000-0000-0000-000000000592" + terminal_status = "blocked_by_playbook_trust_circuit_open" + + class _Mappings: + def first(self): + return { + "op_id": run_id, + "automation_run_id": run_id, + "incident_id": "IWZ-I-2026071512-TRUST-BLOCK", + "latest_check_status": "success", + "latest_check_capability_retry_generation": "1", + "latest_check_controlled_apply_blocker": None, + "latest_check_break_glass_approval_id": approval_id, + "latest_check_break_glass_apply_authorized": True, + "latest_check_break_glass_apply_terminal_status": ( + terminal_status + ), + "latest_apply_status": None, + "latest_learning_status": None, + "latest_controlled_capability_packet_status": "success", + "latest_controlled_capability_queue_status": "retry_claimed", + "latest_controlled_capability_retry_generation": "1", + "predecision_evidence_ready": True, + } + + class _Result: + def mappings(self): + return _Mappings() + + class _Db: + async def execute(self, _statement, _params): + return _Result() + + @asynccontextmanager + async def fake_db_context(_project_id: str): + yield _Db() + + monkeypatch.setattr(job, "get_db_context", fake_db_context) + monkeypatch.setattr( + job.settings, + "ENABLE_IWOOOS_WAZUH_MANAGER_POSTURE_EXECUTOR", + True, + ) + monkeypatch.setattr( + job.settings, + "IWOOOS_WAZUH_MANAGER_POSTURE_FRESHNESS_HOURS", + 6, + ) + monkeypatch.setattr( + job, + "now_taipei", + MagicMock( + return_value=datetime.fromisoformat("2026-07-15T12:59:59+08:00") + ), + ) + + result = await job.enqueue_iwooos_wazuh_alertmanager_integration_candidate_once( + recorder=AsyncMock(), + evidence_collector=AsyncMock(), + evidence_verifier=AsyncMock(), + incident_binder=AsyncMock(), + source_recurrence=_current_wazuh_source_recurrence( + "2026-07-15T12:59:58+08:00" + ), + ) + + assert result["status"] == ( + "owner_authorized_break_glass_apply_blocked_terminal" + ) + assert result["controlled_apply_allowed"] is False + assert result["active_blockers"] == [terminal_status] + + def test_wazuh_controlled_capability_packet_cooldown_expires() -> None: from src.jobs import awooop_ansible_candidate_backfill_job as job diff --git a/apps/api/tests/test_wazuh_break_glass_approval_security.py b/apps/api/tests/test_wazuh_break_glass_approval_security.py new file mode 100644 index 000000000..e9c917e95 --- /dev/null +++ b/apps/api/tests/test_wazuh_break_glass_approval_security.py @@ -0,0 +1,275 @@ +from __future__ import annotations + +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from types import SimpleNamespace +from uuid import UUID + +import pytest +from fastapi import BackgroundTasks, HTTPException + +from src.api.v1 import approvals as approvals_api +from src.core.awooop_operator_auth import AwoooPOperatorPrincipal +from src.models.approval import SignRequest +from src.services import approval_db +from src.services.wazuh_break_glass_approval import ( + CONTROLLED_OPERATOR_APPROVER_ROLE, + CONTROLLED_OPERATOR_AUTH_METHOD, + TELEGRAM_OWNER_APPROVER_ROLE, + TELEGRAM_OWNER_AUTH_METHOD, + WAZUH_BREAK_GLASS_APPROVAL_ACTION, + WAZUH_BREAK_GLASS_SIGNATURE_SCHEMA_VERSION, + trusted_wazuh_signature_context, + wazuh_break_glass_signatures_authorized, +) + + +def _signature( + principal_id: str, + *, + auth_method: str, + approver_role: str, +) -> dict[str, str]: + signature = { + "signature_schema_version": ( + WAZUH_BREAK_GLASS_SIGNATURE_SCHEMA_VERSION + ), + "principal_id": principal_id, + "signer_id": principal_id, + "auth_method": auth_method, + "approver_role": approver_role, + "source": ( + "telegram" + if approver_role == TELEGRAM_OWNER_APPROVER_ROLE + else "api" + ), + } + if approver_role == TELEGRAM_OWNER_APPROVER_ROLE: + signature["telegram_user_id"] = principal_id.removeprefix("tg_") + return signature + + +def test_public_client_supplied_signers_cannot_authorize_wazuh_apply() -> None: + assert wazuh_break_glass_signatures_authorized( + [ + {"signer_id": "invented-owner"}, + {"signer_id": "invented-security"}, + ] + ) is False + + +def test_wazuh_apply_requires_two_trusted_channels_and_principals() -> None: + owner = _signature( + "tg_42", + auth_method=TELEGRAM_OWNER_AUTH_METHOD, + approver_role=TELEGRAM_OWNER_APPROVER_ROLE, + ) + operator = _signature( + "broker-operator", + auth_method=CONTROLLED_OPERATOR_AUTH_METHOD, + approver_role=CONTROLLED_OPERATOR_APPROVER_ROLE, + ) + + assert wazuh_break_glass_signatures_authorized([owner, operator]) is True + assert wazuh_break_glass_signatures_authorized([owner, owner]) is False + assert wazuh_break_glass_signatures_authorized( + [owner, {**operator, "principal_id": "tg_42", "signer_id": "tg_42"}] + ) is False + assert wazuh_break_glass_signatures_authorized( + [{**owner, "source": "web"}, operator] + ) is False + assert trusted_wazuh_signature_context( + principal_id="forged", + auth_method=TELEGRAM_OWNER_AUTH_METHOD, + approver_role=CONTROLLED_OPERATOR_APPROVER_ROLE, + ) is None + + +@pytest.mark.asyncio +async def test_approval_service_rejects_untrusted_wazuh_signature( + monkeypatch: pytest.MonkeyPatch, +) -> None: + approval_id = UUID("00000000-0000-0000-0000-000000000700") + now = datetime.now(UTC) + record = SimpleNamespace( + id=str(approval_id), + action=WAZUH_BREAK_GLASS_APPROVAL_ACTION, + description="bounded Wazuh apply", + status=SimpleNamespace(value="pending"), + risk_level=SimpleNamespace(value="critical"), + required_signatures=2, + current_signatures=0, + signatures=[], + blast_radius={ + "affected_pods": 1, + "estimated_downtime": "0", + "related_services": ["wazuh-manager"], + "data_impact": "write", + }, + dry_run_checks=[], + requested_by="test", + created_at=now, + expires_at=None, + resolved_at=None, + rejection_reason=None, + extra_metadata={}, + fingerprint="wazuh-break-glass-test", + hit_count=1, + last_seen_at=now, + telegram_message_id=None, + telegram_chat_id=None, + incident_id="IWZ-I-TEST", + matched_playbook_id=None, + ) + + class _Result: + def scalar_one_or_none(self): + return record + + class _Db: + async def execute(self, _statement): + return _Result() + + @asynccontextmanager + async def fake_db_context(): + yield _Db() + + monkeypatch.setattr(approval_db, "get_db_context", fake_db_context) + + approval, message, execution_triggered = await ( + approval_db.ApprovalDBService().sign_approval( + approval_id=approval_id, + signer_id="invented-browser-owner", + signer_name="Invented Browser Owner", + ) + ) + + assert approval is not None + assert message.startswith("Cannot sign: authenticated Wazuh") + assert execution_triggered is False + assert record.signatures == [] + + +class _ApprovalService: + def __init__(self) -> None: + self.sign_calls: list[dict[str, object]] = [] + + async def get_approval(self, _approval_id: UUID) -> SimpleNamespace: + return SimpleNamespace(action=WAZUH_BREAK_GLASS_APPROVAL_ACTION) + + async def sign_approval(self, **kwargs: object): + self.sign_calls.append(kwargs) + return None, "Approval not found", False + + +@pytest.mark.asyncio +async def test_public_sign_endpoint_fails_closed_without_operator_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = _ApprovalService() + monkeypatch.setattr( + approvals_api, + "get_approval_service", + lambda: service, + ) + + with pytest.raises(HTTPException) as exc_info: + await approvals_api.sign_approval( + UUID("00000000-0000-0000-0000-000000000701"), + SignRequest( + signer_id="invented-browser-owner", + signer_name="Invented Browser Owner", + ), + BackgroundTasks(), + "csrf-token", + None, + None, + ) + + assert exc_info.value.status_code == 401 + assert service.sign_calls == [] + + +@pytest.mark.asyncio +async def test_wazuh_sign_endpoint_rejects_dev_header_operator_fallback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = _ApprovalService() + monkeypatch.setattr( + approvals_api, + "get_approval_service", + lambda: service, + ) + monkeypatch.setattr( + approvals_api, + "authenticate_awooop_operator_headers", + lambda **_kwargs: AwoooPOperatorPrincipal( + operator_id="dev-header-operator", + auth_method="dev_header", + ), + ) + + with pytest.raises(HTTPException) as exc_info: + await approvals_api.sign_approval( + UUID("00000000-0000-0000-0000-000000000703"), + SignRequest( + signer_id="invented-browser-owner", + signer_name="Invented Browser Owner", + ), + BackgroundTasks(), + "csrf-token", + "dev-header-operator", + None, + ) + + assert exc_info.value.status_code == 401 + assert service.sign_calls == [] + + +@pytest.mark.asyncio +async def test_operator_sign_endpoint_ignores_client_supplied_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = _ApprovalService() + monkeypatch.setattr( + approvals_api, + "get_approval_service", + lambda: service, + ) + monkeypatch.setattr( + approvals_api, + "authenticate_awooop_operator_headers", + lambda **_kwargs: AwoooPOperatorPrincipal( + operator_id="broker-operator", + auth_method="operator_api_key", + ), + ) + + with pytest.raises(HTTPException) as exc_info: + await approvals_api.sign_approval( + UUID("00000000-0000-0000-0000-000000000702"), + SignRequest( + signer_id="invented-browser-owner", + signer_name="Invented Browser Owner", + ), + BackgroundTasks(), + "csrf-token", + "broker-operator", + "opaque-key-not-logged", + ) + + assert exc_info.value.status_code == 404 + assert service.sign_calls == [ + { + "approval_id": UUID( + "00000000-0000-0000-0000-000000000702" + ), + "signer_id": "broker-operator", + "signer_name": "broker-operator", + "comment": None, + "authenticated_principal_id": "broker-operator", + "authentication_method": CONTROLLED_OPERATOR_AUTH_METHOD, + "approver_role": CONTROLLED_OPERATOR_APPROVER_ROLE, + "signature_source": "api", + } + ]