From 36316ed7a9983cfeb1523bad43c79c0ce5a6e49b Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 18 Jul 2026 13:01:23 +0800 Subject: [PATCH] fix(sre): classify Host188 compose failures --- .../awooop_ansible_check_mode_service.py | 31 +++++- .../tests/test_ansible_verified_closure.py | 42 ++++++++ ...claw_callback_forwarder_ansible_catalog.py | 13 +++ .../188-openclaw-callback-forwarder.yml | 101 +++++++++++++++++- 4 files changed, 184 insertions(+), 3 deletions(-) diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index 762cf8e2a..b064b5790 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -56,6 +56,13 @@ _SAFE_CORRELATION_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,160}$") _PLAYBOOK_PREFIX = Path("infra/ansible/playbooks") _STDOUT_LIMIT = 20_000 _STDERR_LIMIT = 12_000 +_SAFE_ANSIBLE_FAILURE_MARKER_RE = re.compile( + r"\bopenclaw_compose_(?:activation|rollback)_failed:" + r"(?:bind_source_missing|bind_mount_contract_rejected|" + r"port_binding_conflict|container_identity_conflict|" + r"docker_permission_denied|image_identity_unavailable|" + r"dependency_health_failed|compose_rc_[0-9]{1,3})\b" +) _CONTROLLED_RETRY_MAX_ATTEMPTS = 1 _CONTROLLED_RETRY_ERROR_BACKOFF_MIN_SECONDS = 15 _CONTROLLED_RETRY_ERROR_BACKOFF_MAX_SECONDS = 30 @@ -188,6 +195,20 @@ def _tail(text_value: str, limit: int) -> str: return text_value[-limit:] +def _safe_ansible_failure_error( + stdout_tail: str, + stderr_tail: str, + *, + fallback: str, +) -> str: + """Prefer an allowlisted playbook marker over unrelated stderr warnings.""" + + matches = list(_SAFE_ANSIBLE_FAILURE_MARKER_RE.finditer(stdout_tail)) + if matches: + return matches[-1].group(0) + return stderr_tail or fallback + + def _json_loads(value: Any) -> dict[str, Any]: if isinstance(value, dict): return value @@ -1109,7 +1130,15 @@ def _build_apply_result_payload(result: AnsibleRunResult) -> tuple[str, dict[str "stdout_tail": stdout_tail, "stderr_tail": stderr_tail, } - error = None if result.returncode == 0 else (stderr_tail or f"ansible_apply_failed_rc_{result.returncode}") + error = ( + None + if result.returncode == 0 + else _safe_ansible_failure_error( + stdout_tail, + stderr_tail, + fallback=f"ansible_apply_failed_rc_{result.returncode}", + ) + ) return status, output, dry_run_result, error diff --git a/apps/api/tests/test_ansible_verified_closure.py b/apps/api/tests/test_ansible_verified_closure.py index 945484d01..11c7fdd1d 100644 --- a/apps/api/tests/test_ansible_verified_closure.py +++ b/apps/api/tests/test_ansible_verified_closure.py @@ -111,6 +111,48 @@ def _controlled_result_payload() -> dict: } +def test_apply_result_prefers_safe_playbook_marker_over_stderr_warning() -> None: + result = service.AnsibleRunResult( + returncode=2, + stdout=( + "TASK [Require successful bounded Compose activation]\n" + "fatal: [host_188]: FAILED! => {\"msg\": " + "\"openclaw_compose_activation_failed:bind_source_missing\"}" + ), + stderr=( + "[WARNING]: Platform linux on host host_188 is using the " + "discovered Python interpreter." + ), + duration_ms=123, + ) + + status, output, dry_run_result, error = ( + service._build_apply_result_payload(result) + ) + + assert status == "failed" + assert error == ( + "openclaw_compose_activation_failed:bind_source_missing" + ) + assert output["stderr_tail"].startswith("[WARNING]") + assert dry_run_result["returncode"] == 2 + + +def test_apply_result_does_not_project_untrusted_stdout_as_error() -> None: + result = service.AnsibleRunResult( + returncode=2, + stdout="fatal: [host_188]: FAILED! => secret-shaped-untrusted-output", + stderr="safe_transport_failure", + duration_ms=123, + ) + + _status, _output, _dry_run_result, error = ( + service._build_apply_result_payload(result) + ) + + assert error == "safe_transport_failure" + + def test_runtime_stage_ids_only_accepts_durable_receipts() -> None: stage_ids = service._runtime_stage_ids( { diff --git a/apps/api/tests/test_host188_openclaw_callback_forwarder_ansible_catalog.py b/apps/api/tests/test_host188_openclaw_callback_forwarder_ansible_catalog.py index 7aefd9c6b..ac064686f 100644 --- a/apps/api/tests/test_host188_openclaw_callback_forwarder_ansible_catalog.py +++ b/apps/api/tests/test_host188_openclaw_callback_forwarder_ansible_catalog.py @@ -331,6 +331,19 @@ def test_openclaw_playbook_is_check_first_bounded_and_rollback_capable() -> None ] assert tasks[name]["args"]["chdir"] == "{{ openclaw_workdir }}" assert tasks[name]["no_log"] is True + assert tasks[name]["failed_when"] is False + assert tasks[ + "Classify the bounded Compose activation result without secret output" + ]["no_log"] is True + assert tasks[ + "Classify the bounded Compose rollback result without secret output" + ]["no_log"] is True + assert tasks["Require successful bounded Compose activation"][ + "ansible.builtin.assert" + ]["fail_msg"].startswith("openclaw_compose_activation_failed:") + assert tasks["Require successful bounded Compose rollback activation"][ + "ansible.builtin.assert" + ]["fail_msg"].startswith("openclaw_compose_rollback_failed:") assert "Install COMPOSE_FILE-only systemd drop-in" not in tasks rendered_override = tasks["Define exact non-secret managed overlay documents"][ "ansible.builtin.set_fact" diff --git a/infra/ansible/playbooks/188-openclaw-callback-forwarder.yml b/infra/ansible/playbooks/188-openclaw-callback-forwarder.yml index 60d49175c..7134c0cf5 100644 --- a/infra/ansible/playbooks/188-openclaw-callback-forwarder.yml +++ b/infra/ansible/playbooks/188-openclaw-callback-forwarder.yml @@ -787,10 +787,58 @@ args: chdir: "{{ openclaw_workdir }}" register: compose_activation - changed_when: true + changed_when: compose_activation.rc | int == 0 + failed_when: false when: openclaw_apply_required | bool no_log: true + - name: Classify the bounded Compose activation result without secret output + ansible.builtin.set_fact: + compose_activation_failure_marker: >- + {%- set combined = + (((compose_activation.stdout | default('')) ~ '\n' ~ + (compose_activation.stderr | default(''))) | lower) -%} + {{ + 'none' + if compose_activation.rc | int == 0 + else 'bind_source_missing' + if ('bind source path does not exist' in combined + or 'no such file or directory' in combined) + else 'bind_mount_contract_rejected' + if ('invalid mount config' in combined + or 'not a directory' in combined) + else 'port_binding_conflict' + if ('port is already allocated' in combined + or 'address already in use' in combined) + else 'container_identity_conflict' + if ('container name' in combined and 'already in use' in combined) + else 'docker_permission_denied' + if 'permission denied' in combined + else 'image_identity_unavailable' + if ('no such image' in combined + or 'manifest unknown' in combined + or 'pull access denied' in combined) + else 'dependency_health_failed' + if ('is unhealthy' in combined + or 'health check' in combined + or 'dependency failed to start' in combined) + else 'compose_rc_' ~ (compose_activation.rc | string) + }} + changed_when: false + when: openclaw_apply_required | bool + no_log: true + + - name: Require successful bounded Compose activation + ansible.builtin.assert: + that: + - compose_activation.rc | int == 0 + fail_msg: >- + openclaw_compose_activation_failed:{{ + compose_activation_failure_marker + }} + changed_when: false + when: openclaw_apply_required | bool + - name: Wait for OpenClaw health after overlay activation become: false ansible.builtin.uri: @@ -1166,10 +1214,59 @@ - never args: chdir: "{{ openclaw_workdir }}" - changed_when: true + register: compose_rollback_activation + changed_when: compose_rollback_activation.rc | int == 0 + failed_when: false when: activation_attempted | default(false) | bool no_log: true + - name: Classify the bounded Compose rollback result without secret output + ansible.builtin.set_fact: + compose_rollback_failure_marker: >- + {%- set combined = + (((compose_rollback_activation.stdout | default('')) ~ '\n' ~ + (compose_rollback_activation.stderr | default(''))) | lower) -%} + {{ + 'none' + if compose_rollback_activation.rc | int == 0 + else 'bind_source_missing' + if ('bind source path does not exist' in combined + or 'no such file or directory' in combined) + else 'bind_mount_contract_rejected' + if ('invalid mount config' in combined + or 'not a directory' in combined) + else 'port_binding_conflict' + if ('port is already allocated' in combined + or 'address already in use' in combined) + else 'container_identity_conflict' + if ('container name' in combined and 'already in use' in combined) + else 'docker_permission_denied' + if 'permission denied' in combined + else 'image_identity_unavailable' + if ('no such image' in combined + or 'manifest unknown' in combined + or 'pull access denied' in combined) + else 'dependency_health_failed' + if ('is unhealthy' in combined + or 'health check' in combined + or 'dependency failed to start' in combined) + else 'compose_rc_' ~ (compose_rollback_activation.rc | string) + }} + changed_when: false + when: activation_attempted | default(false) | bool + no_log: true + + - name: Require successful bounded Compose rollback activation + ansible.builtin.assert: + that: + - compose_rollback_activation.rc | int == 0 + fail_msg: >- + openclaw_compose_rollback_failed:{{ + compose_rollback_failure_marker + }} + changed_when: false + when: activation_attempted | default(false) | bool + - name: Verify OpenClaw health after bounded rollback become: false ansible.builtin.uri: