fix(sre): classify Host188 compose failures
This commit is contained in:
@@ -56,6 +56,13 @@ _SAFE_CORRELATION_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,160}$")
|
||||
_PLAYBOOK_PREFIX = Path("infra/ansible/playbooks")
|
||||
_STDOUT_LIMIT = 20_000
|
||||
_STDERR_LIMIT = 12_000
|
||||
_SAFE_ANSIBLE_FAILURE_MARKER_RE = re.compile(
|
||||
r"\bopenclaw_compose_(?:activation|rollback)_failed:"
|
||||
r"(?:bind_source_missing|bind_mount_contract_rejected|"
|
||||
r"port_binding_conflict|container_identity_conflict|"
|
||||
r"docker_permission_denied|image_identity_unavailable|"
|
||||
r"dependency_health_failed|compose_rc_[0-9]{1,3})\b"
|
||||
)
|
||||
_CONTROLLED_RETRY_MAX_ATTEMPTS = 1
|
||||
_CONTROLLED_RETRY_ERROR_BACKOFF_MIN_SECONDS = 15
|
||||
_CONTROLLED_RETRY_ERROR_BACKOFF_MAX_SECONDS = 30
|
||||
@@ -188,6 +195,20 @@ def _tail(text_value: str, limit: int) -> str:
|
||||
return text_value[-limit:]
|
||||
|
||||
|
||||
def _safe_ansible_failure_error(
|
||||
stdout_tail: str,
|
||||
stderr_tail: str,
|
||||
*,
|
||||
fallback: str,
|
||||
) -> str:
|
||||
"""Prefer an allowlisted playbook marker over unrelated stderr warnings."""
|
||||
|
||||
matches = list(_SAFE_ANSIBLE_FAILURE_MARKER_RE.finditer(stdout_tail))
|
||||
if matches:
|
||||
return matches[-1].group(0)
|
||||
return stderr_tail or fallback
|
||||
|
||||
|
||||
def _json_loads(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
@@ -1109,7 +1130,15 @@ def _build_apply_result_payload(result: AnsibleRunResult) -> tuple[str, dict[str
|
||||
"stdout_tail": stdout_tail,
|
||||
"stderr_tail": stderr_tail,
|
||||
}
|
||||
error = None if result.returncode == 0 else (stderr_tail or f"ansible_apply_failed_rc_{result.returncode}")
|
||||
error = (
|
||||
None
|
||||
if result.returncode == 0
|
||||
else _safe_ansible_failure_error(
|
||||
stdout_tail,
|
||||
stderr_tail,
|
||||
fallback=f"ansible_apply_failed_rc_{result.returncode}",
|
||||
)
|
||||
)
|
||||
return status, output, dry_run_result, error
|
||||
|
||||
|
||||
|
||||
@@ -111,6 +111,48 @@ def _controlled_result_payload() -> dict:
|
||||
}
|
||||
|
||||
|
||||
def test_apply_result_prefers_safe_playbook_marker_over_stderr_warning() -> None:
|
||||
result = service.AnsibleRunResult(
|
||||
returncode=2,
|
||||
stdout=(
|
||||
"TASK [Require successful bounded Compose activation]\n"
|
||||
"fatal: [host_188]: FAILED! => {\"msg\": "
|
||||
"\"openclaw_compose_activation_failed:bind_source_missing\"}"
|
||||
),
|
||||
stderr=(
|
||||
"[WARNING]: Platform linux on host host_188 is using the "
|
||||
"discovered Python interpreter."
|
||||
),
|
||||
duration_ms=123,
|
||||
)
|
||||
|
||||
status, output, dry_run_result, error = (
|
||||
service._build_apply_result_payload(result)
|
||||
)
|
||||
|
||||
assert status == "failed"
|
||||
assert error == (
|
||||
"openclaw_compose_activation_failed:bind_source_missing"
|
||||
)
|
||||
assert output["stderr_tail"].startswith("[WARNING]")
|
||||
assert dry_run_result["returncode"] == 2
|
||||
|
||||
|
||||
def test_apply_result_does_not_project_untrusted_stdout_as_error() -> None:
|
||||
result = service.AnsibleRunResult(
|
||||
returncode=2,
|
||||
stdout="fatal: [host_188]: FAILED! => secret-shaped-untrusted-output",
|
||||
stderr="safe_transport_failure",
|
||||
duration_ms=123,
|
||||
)
|
||||
|
||||
_status, _output, _dry_run_result, error = (
|
||||
service._build_apply_result_payload(result)
|
||||
)
|
||||
|
||||
assert error == "safe_transport_failure"
|
||||
|
||||
|
||||
def test_runtime_stage_ids_only_accepts_durable_receipts() -> None:
|
||||
stage_ids = service._runtime_stage_ids(
|
||||
{
|
||||
|
||||
@@ -331,6 +331,19 @@ def test_openclaw_playbook_is_check_first_bounded_and_rollback_capable() -> None
|
||||
]
|
||||
assert tasks[name]["args"]["chdir"] == "{{ openclaw_workdir }}"
|
||||
assert tasks[name]["no_log"] is True
|
||||
assert tasks[name]["failed_when"] is False
|
||||
assert tasks[
|
||||
"Classify the bounded Compose activation result without secret output"
|
||||
]["no_log"] is True
|
||||
assert tasks[
|
||||
"Classify the bounded Compose rollback result without secret output"
|
||||
]["no_log"] is True
|
||||
assert tasks["Require successful bounded Compose activation"][
|
||||
"ansible.builtin.assert"
|
||||
]["fail_msg"].startswith("openclaw_compose_activation_failed:")
|
||||
assert tasks["Require successful bounded Compose rollback activation"][
|
||||
"ansible.builtin.assert"
|
||||
]["fail_msg"].startswith("openclaw_compose_rollback_failed:")
|
||||
assert "Install COMPOSE_FILE-only systemd drop-in" not in tasks
|
||||
rendered_override = tasks["Define exact non-secret managed overlay documents"][
|
||||
"ansible.builtin.set_fact"
|
||||
|
||||
@@ -787,10 +787,58 @@
|
||||
args:
|
||||
chdir: "{{ openclaw_workdir }}"
|
||||
register: compose_activation
|
||||
changed_when: true
|
||||
changed_when: compose_activation.rc | int == 0
|
||||
failed_when: false
|
||||
when: openclaw_apply_required | bool
|
||||
no_log: true
|
||||
|
||||
- name: Classify the bounded Compose activation result without secret output
|
||||
ansible.builtin.set_fact:
|
||||
compose_activation_failure_marker: >-
|
||||
{%- set combined =
|
||||
(((compose_activation.stdout | default('')) ~ '\n' ~
|
||||
(compose_activation.stderr | default(''))) | lower) -%}
|
||||
{{
|
||||
'none'
|
||||
if compose_activation.rc | int == 0
|
||||
else 'bind_source_missing'
|
||||
if ('bind source path does not exist' in combined
|
||||
or 'no such file or directory' in combined)
|
||||
else 'bind_mount_contract_rejected'
|
||||
if ('invalid mount config' in combined
|
||||
or 'not a directory' in combined)
|
||||
else 'port_binding_conflict'
|
||||
if ('port is already allocated' in combined
|
||||
or 'address already in use' in combined)
|
||||
else 'container_identity_conflict'
|
||||
if ('container name' in combined and 'already in use' in combined)
|
||||
else 'docker_permission_denied'
|
||||
if 'permission denied' in combined
|
||||
else 'image_identity_unavailable'
|
||||
if ('no such image' in combined
|
||||
or 'manifest unknown' in combined
|
||||
or 'pull access denied' in combined)
|
||||
else 'dependency_health_failed'
|
||||
if ('is unhealthy' in combined
|
||||
or 'health check' in combined
|
||||
or 'dependency failed to start' in combined)
|
||||
else 'compose_rc_' ~ (compose_activation.rc | string)
|
||||
}}
|
||||
changed_when: false
|
||||
when: openclaw_apply_required | bool
|
||||
no_log: true
|
||||
|
||||
- name: Require successful bounded Compose activation
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- compose_activation.rc | int == 0
|
||||
fail_msg: >-
|
||||
openclaw_compose_activation_failed:{{
|
||||
compose_activation_failure_marker
|
||||
}}
|
||||
changed_when: false
|
||||
when: openclaw_apply_required | bool
|
||||
|
||||
- name: Wait for OpenClaw health after overlay activation
|
||||
become: false
|
||||
ansible.builtin.uri:
|
||||
@@ -1166,10 +1214,59 @@
|
||||
- never
|
||||
args:
|
||||
chdir: "{{ openclaw_workdir }}"
|
||||
changed_when: true
|
||||
register: compose_rollback_activation
|
||||
changed_when: compose_rollback_activation.rc | int == 0
|
||||
failed_when: false
|
||||
when: activation_attempted | default(false) | bool
|
||||
no_log: true
|
||||
|
||||
- name: Classify the bounded Compose rollback result without secret output
|
||||
ansible.builtin.set_fact:
|
||||
compose_rollback_failure_marker: >-
|
||||
{%- set combined =
|
||||
(((compose_rollback_activation.stdout | default('')) ~ '\n' ~
|
||||
(compose_rollback_activation.stderr | default(''))) | lower) -%}
|
||||
{{
|
||||
'none'
|
||||
if compose_rollback_activation.rc | int == 0
|
||||
else 'bind_source_missing'
|
||||
if ('bind source path does not exist' in combined
|
||||
or 'no such file or directory' in combined)
|
||||
else 'bind_mount_contract_rejected'
|
||||
if ('invalid mount config' in combined
|
||||
or 'not a directory' in combined)
|
||||
else 'port_binding_conflict'
|
||||
if ('port is already allocated' in combined
|
||||
or 'address already in use' in combined)
|
||||
else 'container_identity_conflict'
|
||||
if ('container name' in combined and 'already in use' in combined)
|
||||
else 'docker_permission_denied'
|
||||
if 'permission denied' in combined
|
||||
else 'image_identity_unavailable'
|
||||
if ('no such image' in combined
|
||||
or 'manifest unknown' in combined
|
||||
or 'pull access denied' in combined)
|
||||
else 'dependency_health_failed'
|
||||
if ('is unhealthy' in combined
|
||||
or 'health check' in combined
|
||||
or 'dependency failed to start' in combined)
|
||||
else 'compose_rc_' ~ (compose_rollback_activation.rc | string)
|
||||
}}
|
||||
changed_when: false
|
||||
when: activation_attempted | default(false) | bool
|
||||
no_log: true
|
||||
|
||||
- name: Require successful bounded Compose rollback activation
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- compose_rollback_activation.rc | int == 0
|
||||
fail_msg: >-
|
||||
openclaw_compose_rollback_failed:{{
|
||||
compose_rollback_failure_marker
|
||||
}}
|
||||
changed_when: false
|
||||
when: activation_attempted | default(false) | bool
|
||||
|
||||
- name: Verify OpenClaw health after bounded rollback
|
||||
become: false
|
||||
ansible.builtin.uri:
|
||||
|
||||
Reference in New Issue
Block a user