fix(sre): classify Host188 compose failures

This commit is contained in:
ogt
2026-07-18 13:01:23 +08:00
parent 299e01bfda
commit 36316ed7a9
4 changed files with 184 additions and 3 deletions

View File

@@ -56,6 +56,13 @@ _SAFE_CORRELATION_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,160}$")
_PLAYBOOK_PREFIX = Path("infra/ansible/playbooks")
_STDOUT_LIMIT = 20_000
_STDERR_LIMIT = 12_000
_SAFE_ANSIBLE_FAILURE_MARKER_RE = re.compile(
r"\bopenclaw_compose_(?:activation|rollback)_failed:"
r"(?:bind_source_missing|bind_mount_contract_rejected|"
r"port_binding_conflict|container_identity_conflict|"
r"docker_permission_denied|image_identity_unavailable|"
r"dependency_health_failed|compose_rc_[0-9]{1,3})\b"
)
_CONTROLLED_RETRY_MAX_ATTEMPTS = 1
_CONTROLLED_RETRY_ERROR_BACKOFF_MIN_SECONDS = 15
_CONTROLLED_RETRY_ERROR_BACKOFF_MAX_SECONDS = 30
@@ -188,6 +195,20 @@ def _tail(text_value: str, limit: int) -> str:
return text_value[-limit:]
def _safe_ansible_failure_error(
stdout_tail: str,
stderr_tail: str,
*,
fallback: str,
) -> str:
"""Prefer an allowlisted playbook marker over unrelated stderr warnings."""
matches = list(_SAFE_ANSIBLE_FAILURE_MARKER_RE.finditer(stdout_tail))
if matches:
return matches[-1].group(0)
return stderr_tail or fallback
def _json_loads(value: Any) -> dict[str, Any]:
if isinstance(value, dict):
return value
@@ -1109,7 +1130,15 @@ def _build_apply_result_payload(result: AnsibleRunResult) -> tuple[str, dict[str
"stdout_tail": stdout_tail,
"stderr_tail": stderr_tail,
}
error = None if result.returncode == 0 else (stderr_tail or f"ansible_apply_failed_rc_{result.returncode}")
error = (
None
if result.returncode == 0
else _safe_ansible_failure_error(
stdout_tail,
stderr_tail,
fallback=f"ansible_apply_failed_rc_{result.returncode}",
)
)
return status, output, dry_run_result, error

View File

@@ -111,6 +111,48 @@ def _controlled_result_payload() -> dict:
}
def test_apply_result_prefers_safe_playbook_marker_over_stderr_warning() -> None:
result = service.AnsibleRunResult(
returncode=2,
stdout=(
"TASK [Require successful bounded Compose activation]\n"
"fatal: [host_188]: FAILED! => {\"msg\": "
"\"openclaw_compose_activation_failed:bind_source_missing\"}"
),
stderr=(
"[WARNING]: Platform linux on host host_188 is using the "
"discovered Python interpreter."
),
duration_ms=123,
)
status, output, dry_run_result, error = (
service._build_apply_result_payload(result)
)
assert status == "failed"
assert error == (
"openclaw_compose_activation_failed:bind_source_missing"
)
assert output["stderr_tail"].startswith("[WARNING]")
assert dry_run_result["returncode"] == 2
def test_apply_result_does_not_project_untrusted_stdout_as_error() -> None:
result = service.AnsibleRunResult(
returncode=2,
stdout="fatal: [host_188]: FAILED! => secret-shaped-untrusted-output",
stderr="safe_transport_failure",
duration_ms=123,
)
_status, _output, _dry_run_result, error = (
service._build_apply_result_payload(result)
)
assert error == "safe_transport_failure"
def test_runtime_stage_ids_only_accepts_durable_receipts() -> None:
stage_ids = service._runtime_stage_ids(
{

View File

@@ -331,6 +331,19 @@ def test_openclaw_playbook_is_check_first_bounded_and_rollback_capable() -> None
]
assert tasks[name]["args"]["chdir"] == "{{ openclaw_workdir }}"
assert tasks[name]["no_log"] is True
assert tasks[name]["failed_when"] is False
assert tasks[
"Classify the bounded Compose activation result without secret output"
]["no_log"] is True
assert tasks[
"Classify the bounded Compose rollback result without secret output"
]["no_log"] is True
assert tasks["Require successful bounded Compose activation"][
"ansible.builtin.assert"
]["fail_msg"].startswith("openclaw_compose_activation_failed:")
assert tasks["Require successful bounded Compose rollback activation"][
"ansible.builtin.assert"
]["fail_msg"].startswith("openclaw_compose_rollback_failed:")
assert "Install COMPOSE_FILE-only systemd drop-in" not in tasks
rendered_override = tasks["Define exact non-secret managed overlay documents"][
"ansible.builtin.set_fact"

View File

@@ -787,10 +787,58 @@
args:
chdir: "{{ openclaw_workdir }}"
register: compose_activation
changed_when: true
changed_when: compose_activation.rc | int == 0
failed_when: false
when: openclaw_apply_required | bool
no_log: true
- name: Classify the bounded Compose activation result without secret output
ansible.builtin.set_fact:
compose_activation_failure_marker: >-
{%- set combined =
(((compose_activation.stdout | default('')) ~ '\n' ~
(compose_activation.stderr | default(''))) | lower) -%}
{{
'none'
if compose_activation.rc | int == 0
else 'bind_source_missing'
if ('bind source path does not exist' in combined
or 'no such file or directory' in combined)
else 'bind_mount_contract_rejected'
if ('invalid mount config' in combined
or 'not a directory' in combined)
else 'port_binding_conflict'
if ('port is already allocated' in combined
or 'address already in use' in combined)
else 'container_identity_conflict'
if ('container name' in combined and 'already in use' in combined)
else 'docker_permission_denied'
if 'permission denied' in combined
else 'image_identity_unavailable'
if ('no such image' in combined
or 'manifest unknown' in combined
or 'pull access denied' in combined)
else 'dependency_health_failed'
if ('is unhealthy' in combined
or 'health check' in combined
or 'dependency failed to start' in combined)
else 'compose_rc_' ~ (compose_activation.rc | string)
}}
changed_when: false
when: openclaw_apply_required | bool
no_log: true
- name: Require successful bounded Compose activation
ansible.builtin.assert:
that:
- compose_activation.rc | int == 0
fail_msg: >-
openclaw_compose_activation_failed:{{
compose_activation_failure_marker
}}
changed_when: false
when: openclaw_apply_required | bool
- name: Wait for OpenClaw health after overlay activation
become: false
ansible.builtin.uri:
@@ -1166,10 +1214,59 @@
- never
args:
chdir: "{{ openclaw_workdir }}"
changed_when: true
register: compose_rollback_activation
changed_when: compose_rollback_activation.rc | int == 0
failed_when: false
when: activation_attempted | default(false) | bool
no_log: true
- name: Classify the bounded Compose rollback result without secret output
ansible.builtin.set_fact:
compose_rollback_failure_marker: >-
{%- set combined =
(((compose_rollback_activation.stdout | default('')) ~ '\n' ~
(compose_rollback_activation.stderr | default(''))) | lower) -%}
{{
'none'
if compose_rollback_activation.rc | int == 0
else 'bind_source_missing'
if ('bind source path does not exist' in combined
or 'no such file or directory' in combined)
else 'bind_mount_contract_rejected'
if ('invalid mount config' in combined
or 'not a directory' in combined)
else 'port_binding_conflict'
if ('port is already allocated' in combined
or 'address already in use' in combined)
else 'container_identity_conflict'
if ('container name' in combined and 'already in use' in combined)
else 'docker_permission_denied'
if 'permission denied' in combined
else 'image_identity_unavailable'
if ('no such image' in combined
or 'manifest unknown' in combined
or 'pull access denied' in combined)
else 'dependency_health_failed'
if ('is unhealthy' in combined
or 'health check' in combined
or 'dependency failed to start' in combined)
else 'compose_rc_' ~ (compose_rollback_activation.rc | string)
}}
changed_when: false
when: activation_attempted | default(false) | bool
no_log: true
- name: Require successful bounded Compose rollback activation
ansible.builtin.assert:
that:
- compose_rollback_activation.rc | int == 0
fail_msg: >-
openclaw_compose_rollback_failed:{{
compose_rollback_failure_marker
}}
changed_when: false
when: activation_attempted | default(false) | bool
- name: Verify OpenClaw health after bounded rollback
become: false
ansible.builtin.uri: