diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py index 4d46c5430..bd2111f86 100644 --- a/apps/api/src/services/awooop_ansible_audit_service.py +++ b/apps/api/src/services/awooop_ansible_audit_service.py @@ -1118,7 +1118,83 @@ def _is_ansible_operation(row: dict[str, Any]) -> bool: return "infra/ansible/" in playbook_path or playbook_path.endswith(".yml") and "ansible" in playbook_path +_PUBLIC_ANSIBLE_DRY_RUN_KEYS = frozenset({ + "changed", + "check_mode", + "check_mode_executed", + "apply_enabled", + "apply_executed", + "controlled_apply_allowed", + "controlled_apply_executed", + "safe_to_apply_without_approval", + "returncode", + "timed_out", + "candidate_count", + "skipped", + "execution_mode", + "catalog_id", + "playbook_id", + "playbook_path", + "ansible_playbook_path", + "not_used_reason", + "failure_class", + "failure_reason", +}) + + +def public_ansible_dry_run_result(row: dict[str, Any]) -> dict[str, Any] | None: + """Return only stable, public-safe Ansible dry-run metadata.""" + + raw = _json_object(row, "dry_run_result") + projected = { + key: value + for key, value in raw.items() + if key in _PUBLIC_ANSIBLE_DRY_RUN_KEYS + } + failure_class = _first_present_with_json( + row, + ( + "output_failure_class", + "dry_run_failure_class", + "derived_failure_class", + ), + (("dry_run_result", "failure_class"),), + ) + failure_reason = _first_present_with_json( + row, + ( + "output_failure_reason", + "dry_run_failure_reason", + "derived_failure_reason", + ), + (("dry_run_result", "failure_reason"),), + ) + if failure_class not in (None, ""): + projected["failure_class"] = failure_class + if failure_reason not in (None, ""): + projected["failure_reason"] = failure_reason + return projected or None + + def _ansible_record(row: dict[str, Any]) -> dict[str, Any]: + failure_class = _first_present_with_json( + row, + ( + "output_failure_class", + "dry_run_failure_class", + "derived_failure_class", + ), + (("dry_run_result", "failure_class"),), + ) + failure_reason = _first_present_with_json( + row, + ( + "output_failure_reason", + "dry_run_failure_reason", + "derived_failure_reason", + ), + (("dry_run_result", "failure_reason"),), + ) return { "op_id": _get(row, "op_id"), "parent_op_id": _get(row, "parent_op_id"), @@ -1150,9 +1226,11 @@ def _ansible_record(row: dict[str, Any]) -> dict[str, Any]: ("output_returncode", "input_returncode", "dry_run_returncode"), (("dry_run_result", "returncode"),), ), + "failure_class": failure_class, + "failure_reason": failure_reason, "not_used_reason": _first_present(row, ("input_not_used_reason", "output_not_used_reason")), - "dry_run_result": _get(row, "dry_run_result"), - "error": _get(row, "error"), + "dry_run_result": public_ansible_dry_run_result(row), + "error": failure_reason, "duration_ms": _get(row, "duration_ms"), "tags": _get(row, "tags"), "created_at": _get(row, "created_at"), @@ -1226,6 +1304,8 @@ def summarize_ansible_execution(records: list[dict[str, Any]]) -> dict[str, Any] "latest_apply_enabled": _bool_or_none(focused.get("apply_enabled")), "latest_apply_executed": _bool_or_none(focused.get("apply_executed")), "latest_returncode": returncode if returncode is not None else focused.get("returncode"), + "latest_failure_class": focused.get("failure_class"), + "latest_failure_reason": focused.get("failure_reason"), "approval_source": approval_source, "latest_actor": focused.get("actor"), "latest_op_id": focused.get("op_id"), @@ -2033,13 +2113,20 @@ async def preflight_ansible_candidate_generation( latest_input.get("source_occurrence_generation") or "" ) == occurrence is_terminal = latest.get("is_terminal") is True - if same_generation and (not is_terminal or same_occurrence): + if same_generation and not is_terminal: return { "status": "existing_current_generation", "should_collect_evidence": False, "target_route_generation": generation, "existing_candidate_op_id": latest.get("op_id"), } + if same_generation and same_occurrence: + return { + "status": "existing_terminal_generation", + "should_collect_evidence": False, + "target_route_generation": generation, + "existing_candidate_op_id": latest.get("op_id"), + } if not is_terminal and latest.get("has_apply") is True: return { "status": "generation_reconciliation_blocked_has_apply", diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index 06052f764..cc18e539e 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -57,11 +57,36 @@ _PLAYBOOK_PREFIX = Path("infra/ansible/playbooks") _STDOUT_LIMIT = 20_000 _STDERR_LIMIT = 12_000 _SAFE_ANSIBLE_FAILURE_MARKER_RE = re.compile( - r"\bopenclaw_compose_(?:activation|rollback)_failed:" + r"\b(?:" + r"controlled_execution_correlation_missing_or_unsafe|" + r"openclaw_systemd_working_directory_not_allowlisted|" + r"clawbot_systemd_command_contract_mismatch|" + r"openclaw_payload_base_hash_mismatch|" + r"repo_owned_openclaw_payload_checksum_mismatch|" + r"repo_owned_openclaw_manifest_checksum_mismatch|" + r"clawbot_container_identity_not_unique(?:_after_(?:apply|rollback))?|" + r"clawbot_container_workdir_mismatch|" + r"openclaw_callback_target_mount_foreign_owner|" + r"openclaw_managed_directory_foreign_owner|" + r"openclaw_callback_deployment_identity_foreign_owner|" + r"openclaw_overlay_payload_foreign_owner|" + r"openclaw_managed_overlay_foreign_owner|" + r"openclaw_running_container_payload_base_incompatible|" + r"installed_openclaw_manifest_checksum_mismatch|" + r"installed_openclaw_payload_checksum_mismatch|" + r"clawbot_image_identity_changed_no_build_contract_failed|" + r"openclaw_callback_bind_mount_contract_failed|" + r"openclaw_container_overlay_hash_mismatch|" + r"openclaw_runtime_checkout_delta_outside_managed_override|" + r"openclaw_runtime_target_path_mutated|" + r"openclaw_callback_overlay_rollback_unverified|" + r"openclaw_callback_overlay_rollback_runtime_path_changed|" + r"openclaw_compose_(?:activation|rollback)_failed:" r"(?:bind_source_missing|bind_mount_contract_rejected|" r"port_binding_conflict|container_identity_conflict|" r"docker_permission_denied|image_identity_unavailable|" - r"dependency_health_failed|compose_rc_[0-9]{1,3})\b" + r"dependency_health_failed|compose_rc_[0-9]{1,3})" + r")\b" ) _CONTROLLED_RETRY_MAX_ATTEMPTS = 1 _CONTROLLED_RETRY_ERROR_BACKOFF_MIN_SECONDS = 15 @@ -209,6 +234,73 @@ def _safe_ansible_failure_error( return stderr_tail or fallback +def safe_ansible_failure_projection( + *, + returncode: int, + stdout_tail: str, + stderr_tail: str, + timed_out: bool = False, + fallback_prefix: str, +) -> tuple[str | None, str | None]: + """Return a public-safe failure class and stable reason code. + + Ansible normally writes task failures to stdout while stderr often contains + only interpreter warnings. Raw stdout is deliberately not projected to + public status surfaces; only allowlisted PlayBook markers or fixed transport + reason codes may leave this boundary. + """ + + if returncode == 0: + return None, None + stdout_tail = _tail(stdout_tail, _STDOUT_LIMIT) + stderr_tail = _tail(stderr_tail, _STDERR_LIMIT) + marker_matches = list(_SAFE_ANSIBLE_FAILURE_MARKER_RE.finditer(stdout_tail)) + if marker_matches: + return "playbook_contract", marker_matches[-1].group(0) + if timed_out or returncode == 124: + return "transport_timeout", "ansible_transport_timeout" + + combined = f"{stdout_tail}\n{stderr_tail}".lower() + transport_markers = ( + ( + ("repair_denied:invalid_command",), + "transport_forced_command_denied", + ), + ( + ("host key verification failed",), + "transport_host_key_rejected", + ), + ( + ("permission denied", "publickey"), + "transport_authentication_failed", + ), + ( + ("could not resolve hostname", "name or service not known"), + "transport_dns_failed", + ), + ( + ("no route to host",), + "transport_no_route", + ), + ( + ("connection refused",), + "transport_connection_refused", + ), + ( + ("connection timed out", "operation timed out"), + "transport_timeout", + ), + ( + ("unreachable!", "failed to connect to the host via ssh"), + "transport_unreachable", + ), + ) + for markers, reason in transport_markers: + if any(marker in combined for marker in markers): + return "transport", reason + return "execution_unknown", f"{fallback_prefix}_rc_{returncode}" + + def _json_loads(value: Any) -> dict[str, Any]: if isinstance(value, dict): return value @@ -1064,6 +1156,13 @@ def _build_result_payload( status = "success" if result.returncode == 0 else "failed" stdout_tail = _tail(result.stdout, _STDOUT_LIMIT) stderr_tail = _tail(result.stderr, _STDERR_LIMIT) + failure_class, failure_reason = safe_ansible_failure_projection( + returncode=result.returncode, + stdout_tail=stdout_tail, + stderr_tail=stderr_tail, + timed_out=result.timed_out, + fallback_prefix="ansible_check_mode_failed", + ) output = { "executor": "ansible", "execution_mode": "check_mode", @@ -1077,6 +1176,8 @@ def _build_result_payload( "controlled_apply_blocker": controlled_apply_blocker, "returncode": result.returncode, "timed_out": result.timed_out, + "failure_class": failure_class, + "failure_reason": failure_reason, "stdout_tail": stdout_tail, "stderr_tail": stderr_tail, "next_required_step": ( @@ -1098,10 +1199,20 @@ def _build_result_payload( "known_hosts_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH, "returncode": result.returncode, "timed_out": result.timed_out, + "failure_class": failure_class, + "failure_reason": failure_reason, "stdout_tail": stdout_tail, "stderr_tail": stderr_tail, } - error = None if result.returncode == 0 else (stderr_tail or f"ansible_check_mode_failed_rc_{result.returncode}") + error = ( + None + if result.returncode == 0 + else _safe_ansible_failure_error( + stdout_tail, + stderr_tail, + fallback=f"ansible_check_mode_failed_rc_{result.returncode}", + ) + ) return status, output, dry_run_result, error @@ -1109,6 +1220,13 @@ def _build_apply_result_payload(result: AnsibleRunResult) -> tuple[str, dict[str status = "success" if result.returncode == 0 else "failed" stdout_tail = _tail(result.stdout, _STDOUT_LIMIT) stderr_tail = _tail(result.stderr, _STDERR_LIMIT) + failure_class, failure_reason = safe_ansible_failure_projection( + returncode=result.returncode, + stdout_tail=stdout_tail, + stderr_tail=stderr_tail, + timed_out=result.timed_out, + fallback_prefix="ansible_apply_failed", + ) output = { "executor": "ansible", "execution_mode": "controlled_apply", @@ -1120,6 +1238,8 @@ def _build_apply_result_payload(result: AnsibleRunResult) -> tuple[str, dict[str "controlled_apply_executed": True, "returncode": result.returncode, "timed_out": result.timed_out, + "failure_class": failure_class, + "failure_reason": failure_reason, "stdout_tail": stdout_tail, "stderr_tail": stderr_tail, "next_required_step": ( @@ -1137,6 +1257,8 @@ def _build_apply_result_payload(result: AnsibleRunResult) -> tuple[str, dict[str "known_hosts_path": settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH, "returncode": result.returncode, "timed_out": result.timed_out, + "failure_class": failure_class, + "failure_reason": failure_reason, "stdout_tail": stdout_tail, "stderr_tail": stderr_tail, } diff --git a/apps/api/src/services/awooop_truth_chain_service.py b/apps/api/src/services/awooop_truth_chain_service.py index 8a6db6cb1..e124a1db8 100644 --- a/apps/api/src/services/awooop_truth_chain_service.py +++ b/apps/api/src/services/awooop_truth_chain_service.py @@ -22,10 +22,16 @@ from sqlalchemy import text from src.core.config import settings from src.db.base import get_db_context -from src.services.operator_outcome import build_operator_outcome -from src.services.awooop_ansible_check_mode_service import detect_ansible_transport_blockers -from src.services.awooop_ansible_audit_service import build_ansible_truth +from src.services.awooop_ansible_audit_service import ( + build_ansible_truth, + public_ansible_dry_run_result, +) +from src.services.awooop_ansible_check_mode_service import ( + detect_ansible_transport_blockers, + safe_ansible_failure_projection, +) from src.services.drift_repeat_state import build_drift_repeat_state +from src.services.operator_outcome import build_operator_outcome from src.services.operator_summary_cache import ( get_cached_operator_summary_async, store_operator_summary_async, @@ -116,7 +122,7 @@ def _clean(value: Any) -> Any: """Convert DB values into JSON-friendly primitives.""" if isinstance(value, UUID): return str(value) - if isinstance(value, (datetime, date)): + if isinstance(value, datetime | date): return value.isoformat() if isinstance(value, Decimal): return float(value) @@ -274,6 +280,150 @@ def _truth_chain_branch_limit(source_ids: list[str], limit: int) -> int: ) +def _derive_safe_ansible_failure_projection(row: dict[str, Any]) -> None: + """Backfill safe failure codes for rows written before the projection fields.""" + + if row.get("output_failure_reason") or row.get("dry_run_failure_reason"): + return + operation_type = str(row.get("operation_type") or "") + if operation_type not in { + "ansible_check_mode_executed", + "ansible_apply_executed", + }: + return + raw_returncode = ( + row.get("output_returncode") + or row.get("input_returncode") + or row.get("dry_run_returncode") + ) + try: + returncode = int(raw_returncode) + except (TypeError, ValueError): + return + timed_out = str( + row.get("output_timed_out") or row.get("dry_run_timed_out") or "" + ).lower() == "true" + failure_class, failure_reason = safe_ansible_failure_projection( + returncode=returncode, + stdout_tail=str( + row.get("output_stdout_tail") + or row.get("dry_run_stdout_tail") + or "" + ), + stderr_tail=str( + row.get("output_stderr_tail") + or row.get("dry_run_stderr_tail") + or "" + ), + timed_out=timed_out, + fallback_prefix=( + "ansible_apply_failed" + if operation_type == "ansible_apply_executed" + else "ansible_check_mode_failed" + ), + ) + row["derived_failure_class"] = failure_class + row["derived_failure_reason"] = failure_reason + + +_PUBLIC_AUTOMATION_OPERATION_KEYS = frozenset({ + "op_id", + "operation_type", + "status", + "incident_id", + "run_id", + "parent_op_id", + "actor", + "duration_ms", + "tags", + "created_at", + "input_action", + "input_executor", + "input_execution_backend", + "input_catalog_id", + "input_execution_mode", + "input_approval_source", + "input_apply_enabled", + "input_apply_executed", + "input_check_mode_executed", + "input_returncode", + "input_playbook_id", + "input_playbook_path", + "input_ansible_playbook_path", + "input_check_mode", + "input_not_used_reason", + "output_action", + "output_executor", + "output_execution_backend", + "output_catalog_id", + "output_execution_mode", + "output_approval_source", + "output_apply_enabled", + "output_apply_executed", + "output_check_mode_executed", + "output_returncode", + "output_failure_class", + "output_failure_reason", + "output_timed_out", + "output_playbook_id", + "output_playbook_path", + "output_ansible_playbook_path", + "output_check_mode", + "output_not_used_reason", + "dry_run_returncode", + "dry_run_failure_class", + "dry_run_failure_reason", + "dry_run_timed_out", + "dry_run_apply_executed", + "dry_run_check_mode_executed", + "derived_failure_class", + "derived_failure_reason", +}) + + +def _public_automation_operation_row(row: dict[str, Any]) -> dict[str, Any]: + """Project an operation row without raw command input, output, or errors.""" + + projected = { + key: value + for key, value in row.items() + if key in _PUBLIC_AUTOMATION_OPERATION_KEYS + } + dry_run_result = public_ansible_dry_run_result(row) + if dry_run_result is not None: + projected["dry_run_result"] = dry_run_result + failure_class = next( + ( + row.get(key) + for key in ( + "output_failure_class", + "dry_run_failure_class", + "derived_failure_class", + ) + if row.get(key) not in (None, "") + ), + None, + ) + failure_reason = next( + ( + row.get(key) + for key in ( + "output_failure_reason", + "dry_run_failure_reason", + "derived_failure_reason", + ) + if row.get(key) not in (None, "") + ), + None, + ) + if failure_class is not None: + projected["failure_class"] = failure_class + if failure_reason is not None: + projected["failure_reason"] = failure_reason + projected["error"] = failure_reason + return projected + + async def _fetch_automation_operation_rows_for_incidents( db: Any, *, @@ -325,12 +475,22 @@ async def _fetch_automation_operation_rows_for_incidents( output ->> 'apply_executed' AS output_apply_executed, output ->> 'check_mode_executed' AS output_check_mode_executed, output ->> 'returncode' AS output_returncode, + output ->> 'failure_class' AS output_failure_class, + output ->> 'failure_reason' AS output_failure_reason, + output ->> 'timed_out' AS output_timed_out, + output ->> 'stdout_tail' AS output_stdout_tail, + output ->> 'stderr_tail' AS output_stderr_tail, output ->> 'playbook_id' AS output_playbook_id, output ->> 'playbook_path' AS output_playbook_path, output ->> 'ansible_playbook_path' AS output_ansible_playbook_path, output ->> 'check_mode' AS output_check_mode, output ->> 'not_used_reason' AS output_not_used_reason, dry_run_result ->> 'returncode' AS dry_run_returncode, + dry_run_result ->> 'failure_class' AS dry_run_failure_class, + dry_run_result ->> 'failure_reason' AS dry_run_failure_reason, + dry_run_result ->> 'timed_out' AS dry_run_timed_out, + dry_run_result ->> 'stdout_tail' AS dry_run_stdout_tail, + dry_run_result ->> 'stderr_tail' AS dry_run_stderr_tail, dry_run_result ->> 'apply_executed' AS dry_run_apply_executed, dry_run_result ->> 'check_mode_executed' AS dry_run_check_mode_executed, @@ -373,6 +533,7 @@ async def _fetch_automation_operation_rows_for_incidents( }, ) for row in rows: + _derive_safe_ansible_failure_projection(row) op_id = str(row.get("op_id") or "") if op_id: rows_by_id.setdefault(op_id, row) @@ -2162,6 +2323,9 @@ async def fetch_truth_chain(source_id: str, project_id: str = "awoooi") -> dict[ automation_quality=automation_quality, source_id=source_id, ) + public_automation_ops = [ + _public_automation_operation_row(row) for row in automation_ops + ] result = { "project_id": project_id, @@ -2206,7 +2370,7 @@ async def fetch_truth_chain(source_id: str, project_id: str = "awoooi") -> dict[ }, }, "execution": { - "automation_operation_log": automation_ops, + "automation_operation_log": public_automation_ops, "auto_repair_executions": auto_repair_executions, "ansible": build_ansible_truth(automation_ops, incident=incident, drift=drift), }, @@ -2380,12 +2544,15 @@ def _build_summary_quality_records( automation_quality=automation_quality, source_id=incident_id, ) + public_automation_ops = [ + _public_automation_operation_row(row) for row in automation_ops + ] records.append({ "incident": incident, "truth_status": truth_status, "automation_quality": automation_quality, "execution": { - "automation_operation_log": automation_ops, + "automation_operation_log": public_automation_ops, "auto_repair_executions": auto_repair_executions, "ansible": build_ansible_truth(automation_ops, incident=incident, drift=None), }, diff --git a/apps/api/src/services/platform_operator_service.py b/apps/api/src/services/platform_operator_service.py index fa6a7c245..7edb8c3a1 100644 --- a/apps/api/src/services/platform_operator_service.py +++ b/apps/api/src/services/platform_operator_service.py @@ -6938,6 +6938,14 @@ def _status_chain_execution_section(truth_chain: dict[str, Any] | None) -> dict[ else latest_ansible.get("check_mode") ), "latest_returncode": ansible_summary.get("latest_returncode"), + "latest_failure_class": ( + ansible_summary.get("latest_failure_class") + or latest_ansible.get("failure_class") + ), + "latest_failure_reason": ( + ansible_summary.get("latest_failure_reason") + or latest_ansible.get("failure_reason") + ), "latest_apply_executed": ansible_summary.get("latest_apply_executed"), "approval_source": ansible_summary.get("approval_source"), "candidate_playbooks": [ diff --git a/apps/api/tests/test_ansible_verified_closure.py b/apps/api/tests/test_ansible_verified_closure.py index 8ad52fad1..f3339624f 100644 --- a/apps/api/tests/test_ansible_verified_closure.py +++ b/apps/api/tests/test_ansible_verified_closure.py @@ -144,6 +144,49 @@ def test_apply_result_prefers_safe_playbook_marker_over_stderr_warning() -> None assert dry_run_result["returncode"] == 2 +def test_check_mode_result_projects_safe_playbook_failure_contract() -> None: + result = service.AnsibleRunResult( + returncode=2, + stdout=( + "TASK [Fail closed when a payload base is not hash-compatible]\n" + "fatal: [host_188]: FAILED! => {\"msg\": " + "\"openclaw_payload_base_hash_mismatch; " + "drift_work_item=DRIFT-H188-OPENCLAW-SOURCE-001\"}" + ), + stderr="[WARNING]: interpreter discovery warning", + duration_ms=123, + ) + + status, output, dry_run_result, error = service._build_result_payload(result) + + assert status == "failed" + assert error == "openclaw_payload_base_hash_mismatch" + assert output["failure_class"] == "playbook_contract" + assert output["failure_reason"] == "openclaw_payload_base_hash_mismatch" + assert dry_run_result["failure_class"] == "playbook_contract" + assert dry_run_result["failure_reason"] == "openclaw_payload_base_hash_mismatch" + + +def test_check_mode_result_projects_fixed_transport_reason_without_raw_output() -> None: + result = service.AnsibleRunResult( + returncode=4, + stdout=( + "host_188 | UNREACHABLE! => {\"msg\": " + "\"Failed to connect to the host via ssh: private detail\"}" + ), + stderr="", + duration_ms=123, + ) + + _status, output, dry_run_result, error = service._build_result_payload(result) + + assert error == "ansible_check_mode_failed_rc_4" + assert output["failure_class"] == "transport" + assert output["failure_reason"] == "transport_unreachable" + assert dry_run_result["failure_reason"] == "transport_unreachable" + assert "private detail" not in output["failure_reason"] + + def test_apply_result_does_not_project_untrusted_stdout_as_error() -> None: result = service.AnsibleRunResult( returncode=2, diff --git a/apps/api/tests/test_awooop_operator_timeline_labels.py b/apps/api/tests/test_awooop_operator_timeline_labels.py index cd1242893..fb5067442 100644 --- a/apps/api/tests/test_awooop_operator_timeline_labels.py +++ b/apps/api/tests/test_awooop_operator_timeline_labels.py @@ -3517,6 +3517,72 @@ def test_awooop_status_chain_does_not_treat_ansible_check_mode_as_repair() -> No assert chain["execution"]["ansible"]["applied"] is False +def test_awooop_status_chain_surfaces_safe_ansible_failure_reason() -> None: + chain = _build_awooop_status_chain( + incident_ids=["INC-HOST188-CALLBACK"], + source_id="INC-HOST188-CALLBACK", + truth_chain={ + "truth_status": { + "current_stage": "execution_failed", + "stage_status": "error", + "needs_human": False, + "blockers": ["auto_repair_missing"], + }, + "automation_quality": { + "verdict": "execution_failed", + "facts": { + "auto_repair_execution_records": 0, + "automation_operation_records": 2, + "effective_execution_records": 1, + "verification_result": None, + "mcp_gateway_total": 8, + "knowledge_entries": 0, + }, + "blockers": ["auto_repair_missing"], + }, + "execution": { + "automation_operation_log": [ + { + "operation_type": "ansible_check_mode_executed", + "status": "failed", + "actor": "ansible_check_mode_worker", + "input_executor": "ansible", + } + ], + "ansible": { + "considered": True, + "records": [ + { + "operation_type": "ansible_check_mode_executed", + "status": "failed", + "actor": "ansible_check_mode_worker", + "catalog_id": "ansible:188-openclaw-callback-forwarder", + "playbook_path": ( + "infra/ansible/playbooks/" + "188-openclaw-callback-forwarder.yml" + ), + "execution_mode": "check_mode", + "check_mode": True, + "apply_executed": False, + "returncode": 2, + "failure_class": "playbook_contract", + "failure_reason": "openclaw_payload_base_hash_mismatch", + } + ], + "candidate_catalog": {"candidates": []}, + }, + }, + }, + remediation_history={"total": 0}, + ) + + ansible = chain["execution"]["ansible"] + assert ansible["latest_failure_class"] == "playbook_contract" + assert ansible["latest_failure_reason"] == ( + "openclaw_payload_base_hash_mismatch" + ) + + def test_awooop_status_chain_includes_source_provider_correlation() -> None: chain = _build_awooop_status_chain( incident_ids=["INC-20260520-4D1124"], diff --git a/apps/api/tests/test_awooop_truth_chain_service.py b/apps/api/tests/test_awooop_truth_chain_service.py index 846abedf1..97fddfe98 100644 --- a/apps/api/tests/test_awooop_truth_chain_service.py +++ b/apps/api/tests/test_awooop_truth_chain_service.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import inspect +import json import os import signal from datetime import UTC, datetime, timedelta @@ -1433,6 +1434,131 @@ def test_ansible_truth_surfaces_audited_check_mode_record() -> None: ] is True +def test_ansible_truth_surfaces_public_safe_failure_projection() -> None: + truth = build_ansible_truth( + [ + { + "op_id": "op-ansible-failed-1", + "operation_type": "ansible_check_mode_executed", + "status": "failed", + "actor": "ansible_check_mode_worker", + "input_catalog_id": "ansible:188-openclaw-callback-forwarder", + "input_execution_mode": "check_mode", + "input_playbook_path": ( + "infra/ansible/playbooks/188-openclaw-callback-forwarder.yml" + ), + "input_check_mode": "true", + "output_returncode": "2", + "output_failure_class": "playbook_contract", + "output_failure_reason": "openclaw_payload_base_hash_mismatch", + "error": "raw executor detail is not the public projection", + } + ], + incident={ + "incident_id": "INC-HOST188-CALLBACK", + "alertname": "TelegramCallbackIngressUnverified", + }, + drift=None, + ) + + assert truth["records"][0]["failure_class"] == "playbook_contract" + assert truth["records"][0]["failure_reason"] == ( + "openclaw_payload_base_hash_mismatch" + ) + assert truth["summary"]["latest_failure_class"] == "playbook_contract" + assert truth["summary"]["latest_failure_reason"] == ( + "openclaw_payload_base_hash_mismatch" + ) + + +def test_legacy_ansible_row_derives_safe_failure_without_public_raw_output() -> None: + row = { + "operation_type": "ansible_check_mode_executed", + "status": "failed", + "output_returncode": "2", + "output_stdout_tail": ( + "fatal: [host_188]: FAILED! => {\"msg\": " + "\"openclaw_payload_base_hash_mismatch; private drift detail\"}" + ), + "output_stderr_tail": "[WARNING]: interpreter discovery warning", + } + + truth_chain_service._derive_safe_ansible_failure_projection(row) + + assert row["derived_failure_class"] == "playbook_contract" + assert row["derived_failure_reason"] == "openclaw_payload_base_hash_mismatch" + projected = build_ansible_truth( + [row], + incident={"incident_id": "INC-HOST188-CALLBACK"}, + drift=None, + ) + assert projected["records"][0]["failure_reason"] == ( + "openclaw_payload_base_hash_mismatch" + ) + assert "private drift detail" not in projected["records"][0]["failure_reason"] + + +def test_truth_chain_public_operation_shape_redacts_raw_ansible_output() -> None: + private_marker = "PRIVATE_EXECUTOR_PAYLOAD_MUST_NOT_CROSS_API_BOUNDARY" + row = { + "op_id": "op-ansible-private-1", + "operation_type": "ansible_check_mode_executed", + "status": "failed", + "incident_id": "INC-HOST188-CALLBACK", + "actor": "ansible_check_mode_worker", + "output_returncode": "2", + "output_stdout_tail": private_marker, + "output_stderr_tail": f"stderr:{private_marker}", + "input_text": f"input:{private_marker}", + "output_text": f"output:{private_marker}", + "error": f"error:{private_marker}", + "dry_run_result": { + "changed": 0, + "returncode": 2, + "stdout_tail": private_marker, + "stderr_tail": private_marker, + "failure_class": "playbook_contract", + "failure_reason": "openclaw_payload_base_hash_mismatch", + }, + "derived_failure_class": "playbook_contract", + "derived_failure_reason": "openclaw_payload_base_hash_mismatch", + } + + public_row = truth_chain_service._public_automation_operation_row(row) + ansible_truth = build_ansible_truth( + [row], + incident={"incident_id": "INC-HOST188-CALLBACK"}, + drift=None, + ) + serialized = json.dumps( + {"operation": public_row, "ansible": ansible_truth}, + ensure_ascii=False, + ) + + assert private_marker not in serialized + assert "output_stdout_tail" not in public_row + assert "output_stderr_tail" not in public_row + assert "input_text" not in public_row + assert "output_text" not in public_row + assert public_row["failure_reason"] == "openclaw_payload_base_hash_mismatch" + assert public_row["error"] == "openclaw_payload_base_hash_mismatch" + assert public_row["dry_run_result"] == { + "changed": 0, + "returncode": 2, + "failure_class": "playbook_contract", + "failure_reason": "openclaw_payload_base_hash_mismatch", + } + assert ansible_truth["records"][0]["error"] == ( + "openclaw_payload_base_hash_mismatch" + ) + assert '"automation_operation_log": public_automation_ops' in inspect.getsource( + fetch_truth_chain + ) + assert '"automation_operation_log": public_automation_ops' in inspect.getsource( + truth_chain_service._build_summary_quality_records + ) + + def test_ansible_truth_marks_worker_apply_as_controlled_apply() -> None: truth = build_ansible_truth( [ diff --git a/apps/api/tests/test_sre_typed_domain_router.py b/apps/api/tests/test_sre_typed_domain_router.py index 64c23f982..32a4aa707 100644 --- a/apps/api/tests/test_sre_typed_domain_router.py +++ b/apps/api/tests/test_sre_typed_domain_router.py @@ -24,6 +24,9 @@ from src.services.awooop_ansible_audit_service import ( # noqa: E402 from src.services.awooop_ansible_check_mode_service import ( # noqa: E402 build_ansible_check_mode_claim_input, ) +from src.jobs.awooop_ansible_candidate_backfill_job import ( # noqa: E402 + enqueue_ai_decision_ansible_candidate, +) from src.services.awooop_ansible_post_verifier import ( # noqa: E402 postconditions_for_catalog, ) @@ -1074,6 +1077,89 @@ async def test_same_terminal_generation_accepts_one_fresh_occurrence( assert sum("'decision_manager'" in sql for sql in db.statements) == 1 +@pytest.mark.asyncio +async def test_same_terminal_occurrence_is_not_reported_as_fresh_queue( + monkeypatch: pytest.MonkeyPatch, +) -> None: + incident = _sentry_runtime_incident() + proposal = _sentry_runtime_proposal("alert-current-1") + payload = build_ansible_decision_audit_payload( + incident=incident, + proposal_data=proposal, + decision_path="repair_candidate_controlled_queue", + not_used_reason="test", + ) + assert payload is not None + db = _GenerationDb( + { + "op_id": "00000000-0000-0000-0000-000000471306", + "input": payload["input"], + "is_terminal": True, + "has_pending_child": False, + "has_apply": False, + } + ) + + @asynccontextmanager + async def fake_db_context(_project_id: str = "awoooi"): + yield db + + monkeypatch.setattr( + "src.services.awooop_ansible_audit_service.get_db_context", + fake_db_context, + ) + + result = await preflight_ansible_candidate_generation( + incident=incident, + proposal_data=proposal, + decision_path="repair_candidate_controlled_queue", + not_used_reason="test", + ) + + assert result["status"] == "existing_terminal_generation" + assert result["should_collect_evidence"] is False + assert result["existing_candidate_op_id"] == ( + "00000000-0000-0000-0000-000000471306" + ) + + +@pytest.mark.asyncio +async def test_terminal_generation_handoff_is_not_overclaimed_as_queued( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def terminal_preflight(**_kwargs): + return { + "status": "existing_terminal_generation", + "should_collect_evidence": False, + "existing_candidate_op_id": ( + "00000000-0000-0000-0000-000000471306" + ), + } + + async def must_not_collect(**_kwargs): + raise AssertionError("terminal generation must stop before evidence collection") + + monkeypatch.setattr( + "src.jobs.awooop_ansible_candidate_backfill_job." + "preflight_ansible_candidate_generation", + terminal_preflight, + ) + + handoff = await enqueue_ai_decision_ansible_candidate( + incident=_sentry_runtime_incident(), + proposal_data=_sentry_runtime_proposal("alert-current-1"), + evidence_collector=must_not_collect, + evidence_verifier=must_not_collect, + ) + + assert handoff["status"] == "existing_terminal_generation" + assert handoff["queued"] is False + assert handoff["active_blockers"] == ["existing_terminal_generation"] + assert handoff["automation_run_id"] == ( + "00000000-0000-0000-0000-000000471306" + ) + + @pytest.mark.asyncio async def test_webhook_generation_rejects_unverified_historical_recurrence( monkeypatch: pytest.MonkeyPatch,