wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(sre): classify controlled executor failures safely

Browse Source
This commit is contained in:
Your Name
2026-07-18 20:00:49 +08:00
parent 6d1c0643e2
commit 39ce7f2f9b
8 changed files with 717 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
apps/api/tests/test_ansible_verified_closure.py
View File

@@ -144,6 +144,49 @@ def test_apply_result_prefers_safe_playbook_marker_over_stderr_warning() -> None
assert dry_run_result["returncode"] == 2
def test_check_mode_result_projects_safe_playbook_failure_contract() -> None:
result = service.AnsibleRunResult(
returncode=2,
stdout=(
"TASK [Fail closed when a payload base is not hash-compatible]\n"
"fatal: [host_188]: FAILED! => {\"msg\": "
"\"openclaw_payload_base_hash_mismatch; "
"drift_work_item=DRIFT-H188-OPENCLAW-SOURCE-001\"}"
),
stderr="[WARNING]: interpreter discovery warning",
duration_ms=123,
)
status, output, dry_run_result, error = service._build_result_payload(result)
assert status == "failed"
assert error == "openclaw_payload_base_hash_mismatch"
assert output["failure_class"] == "playbook_contract"
assert output["failure_reason"] == "openclaw_payload_base_hash_mismatch"
assert dry_run_result["failure_class"] == "playbook_contract"
assert dry_run_result["failure_reason"] == "openclaw_payload_base_hash_mismatch"
def test_check_mode_result_projects_fixed_transport_reason_without_raw_output() -> None:
result = service.AnsibleRunResult(
returncode=4,
stdout=(
"host_188 | UNREACHABLE! => {\"msg\": "
"\"Failed to connect to the host via ssh: private detail\"}"
),
stderr="",
duration_ms=123,
)
_status, output, dry_run_result, error = service._build_result_payload(result)
assert error == "ansible_check_mode_failed_rc_4"
assert output["failure_class"] == "transport"
assert output["failure_reason"] == "transport_unreachable"
assert dry_run_result["failure_reason"] == "transport_unreachable"
assert "private detail" not in output["failure_reason"]
def test_apply_result_does_not_project_untrusted_stdout_as_error() -> None:
result = service.AnsibleRunResult(
returncode=2,

66
apps/api/tests/test_awooop_operator_timeline_labels.py
View File

@@ -3517,6 +3517,72 @@ def test_awooop_status_chain_does_not_treat_ansible_check_mode_as_repair() -> No
assert chain["execution"]["ansible"]["applied"] is False
def test_awooop_status_chain_surfaces_safe_ansible_failure_reason() -> None:
chain = _build_awooop_status_chain(
incident_ids=["INC-HOST188-CALLBACK"],
source_id="INC-HOST188-CALLBACK",
truth_chain={
"truth_status": {
"current_stage": "execution_failed",
"stage_status": "error",
"needs_human": False,
"blockers": ["auto_repair_missing"],
},
"automation_quality": {
"verdict": "execution_failed",
"facts": {
"auto_repair_execution_records": 0,
"automation_operation_records": 2,
"effective_execution_records": 1,
"verification_result": None,
"mcp_gateway_total": 8,
"knowledge_entries": 0,
},
"blockers": ["auto_repair_missing"],
},
"execution": {
"automation_operation_log": [
{
"operation_type": "ansible_check_mode_executed",
"status": "failed",
"actor": "ansible_check_mode_worker",
"input_executor": "ansible",
}
],
"ansible": {
"considered": True,
"records": [
{
"operation_type": "ansible_check_mode_executed",
"status": "failed",
"actor": "ansible_check_mode_worker",
"catalog_id": "ansible:188-openclaw-callback-forwarder",
"playbook_path": (
"infra/ansible/playbooks/"
"188-openclaw-callback-forwarder.yml"
),
"execution_mode": "check_mode",
"check_mode": True,
"apply_executed": False,
"returncode": 2,
"failure_class": "playbook_contract",
"failure_reason": "openclaw_payload_base_hash_mismatch",
}
],
"candidate_catalog": {"candidates": []},
},
},
},
remediation_history={"total": 0},
)
ansible = chain["execution"]["ansible"]
assert ansible["latest_failure_class"] == "playbook_contract"
assert ansible["latest_failure_reason"] == (
"openclaw_payload_base_hash_mismatch"
)
def test_awooop_status_chain_includes_source_provider_correlation() -> None:
chain = _build_awooop_status_chain(
incident_ids=["INC-20260520-4D1124"],

126
apps/api/tests/test_awooop_truth_chain_service.py
View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import inspect
import json
import os
import signal
from datetime import UTC, datetime, timedelta
@@ -1433,6 +1434,131 @@ def test_ansible_truth_surfaces_audited_check_mode_record() -> None:
] is True
def test_ansible_truth_surfaces_public_safe_failure_projection() -> None:
truth = build_ansible_truth(
[
{
"op_id": "op-ansible-failed-1",
"operation_type": "ansible_check_mode_executed",
"status": "failed",
"actor": "ansible_check_mode_worker",
"input_catalog_id": "ansible:188-openclaw-callback-forwarder",
"input_execution_mode": "check_mode",
"input_playbook_path": (
"infra/ansible/playbooks/188-openclaw-callback-forwarder.yml"
),
"input_check_mode": "true",
"output_returncode": "2",
"output_failure_class": "playbook_contract",
"output_failure_reason": "openclaw_payload_base_hash_mismatch",
"error": "raw executor detail is not the public projection",
}
],
incident={
"incident_id": "INC-HOST188-CALLBACK",
"alertname": "TelegramCallbackIngressUnverified",
},
drift=None,
)
assert truth["records"][0]["failure_class"] == "playbook_contract"
assert truth["records"][0]["failure_reason"] == (
"openclaw_payload_base_hash_mismatch"
)
assert truth["summary"]["latest_failure_class"] == "playbook_contract"
assert truth["summary"]["latest_failure_reason"] == (
"openclaw_payload_base_hash_mismatch"
)
def test_legacy_ansible_row_derives_safe_failure_without_public_raw_output() -> None:
row = {
"operation_type": "ansible_check_mode_executed",
"status": "failed",
"output_returncode": "2",
"output_stdout_tail": (
"fatal: [host_188]: FAILED! => {\"msg\": "
"\"openclaw_payload_base_hash_mismatch; private drift detail\"}"
),
"output_stderr_tail": "[WARNING]: interpreter discovery warning",
}
truth_chain_service._derive_safe_ansible_failure_projection(row)
assert row["derived_failure_class"] == "playbook_contract"
assert row["derived_failure_reason"] == "openclaw_payload_base_hash_mismatch"
projected = build_ansible_truth(
[row],
incident={"incident_id": "INC-HOST188-CALLBACK"},
drift=None,
)
assert projected["records"][0]["failure_reason"] == (
"openclaw_payload_base_hash_mismatch"
)
assert "private drift detail" not in projected["records"][0]["failure_reason"]
def test_truth_chain_public_operation_shape_redacts_raw_ansible_output() -> None:
private_marker = "PRIVATE_EXECUTOR_PAYLOAD_MUST_NOT_CROSS_API_BOUNDARY"
row = {
"op_id": "op-ansible-private-1",
"operation_type": "ansible_check_mode_executed",
"status": "failed",
"incident_id": "INC-HOST188-CALLBACK",
"actor": "ansible_check_mode_worker",
"output_returncode": "2",
"output_stdout_tail": private_marker,
"output_stderr_tail": f"stderr:{private_marker}",
"input_text": f"input:{private_marker}",
"output_text": f"output:{private_marker}",
"error": f"error:{private_marker}",
"dry_run_result": {
"changed": 0,
"returncode": 2,
"stdout_tail": private_marker,
"stderr_tail": private_marker,
"failure_class": "playbook_contract",
"failure_reason": "openclaw_payload_base_hash_mismatch",
},
"derived_failure_class": "playbook_contract",
"derived_failure_reason": "openclaw_payload_base_hash_mismatch",
}
public_row = truth_chain_service._public_automation_operation_row(row)
ansible_truth = build_ansible_truth(
[row],
incident={"incident_id": "INC-HOST188-CALLBACK"},
drift=None,
)
serialized = json.dumps(
{"operation": public_row, "ansible": ansible_truth},
ensure_ascii=False,
)
assert private_marker not in serialized
assert "output_stdout_tail" not in public_row
assert "output_stderr_tail" not in public_row
assert "input_text" not in public_row
assert "output_text" not in public_row
assert public_row["failure_reason"] == "openclaw_payload_base_hash_mismatch"
assert public_row["error"] == "openclaw_payload_base_hash_mismatch"
assert public_row["dry_run_result"] == {
"changed": 0,
"returncode": 2,
"failure_class": "playbook_contract",
"failure_reason": "openclaw_payload_base_hash_mismatch",
}
assert ansible_truth["records"][0]["error"] == (
"openclaw_payload_base_hash_mismatch"
)
assert '"automation_operation_log": public_automation_ops' in inspect.getsource(
fetch_truth_chain
)
assert '"automation_operation_log": public_automation_ops' in inspect.getsource(
truth_chain_service._build_summary_quality_records
)
def test_ansible_truth_marks_worker_apply_as_controlled_apply() -> None:
truth = build_ansible_truth(
[

86
apps/api/tests/test_sre_typed_domain_router.py
View File

@@ -24,6 +24,9 @@ from src.services.awooop_ansible_audit_service import ( # noqa: E402
from src.services.awooop_ansible_check_mode_service import ( # noqa: E402
build_ansible_check_mode_claim_input,
)
from src.jobs.awooop_ansible_candidate_backfill_job import ( # noqa: E402
enqueue_ai_decision_ansible_candidate,
)
from src.services.awooop_ansible_post_verifier import ( # noqa: E402
postconditions_for_catalog,
)
@@ -1074,6 +1077,89 @@ async def test_same_terminal_generation_accepts_one_fresh_occurrence(
assert sum("'decision_manager'" in sql for sql in db.statements) == 1
@pytest.mark.asyncio
async def test_same_terminal_occurrence_is_not_reported_as_fresh_queue(
monkeypatch: pytest.MonkeyPatch,
) -> None:
incident = _sentry_runtime_incident()
proposal = _sentry_runtime_proposal("alert-current-1")
payload = build_ansible_decision_audit_payload(
incident=incident,
proposal_data=proposal,
decision_path="repair_candidate_controlled_queue",
not_used_reason="test",
)
assert payload is not None
db = _GenerationDb(
{
"op_id": "00000000-0000-0000-0000-000000471306",
"input": payload["input"],
"is_terminal": True,
"has_pending_child": False,
"has_apply": False,
}
)
@asynccontextmanager
async def fake_db_context(_project_id: str = "awoooi"):
yield db
monkeypatch.setattr(
"src.services.awooop_ansible_audit_service.get_db_context",
fake_db_context,
)
result = await preflight_ansible_candidate_generation(
incident=incident,
proposal_data=proposal,
decision_path="repair_candidate_controlled_queue",
not_used_reason="test",
)
assert result["status"] == "existing_terminal_generation"
assert result["should_collect_evidence"] is False
assert result["existing_candidate_op_id"] == (
"00000000-0000-0000-0000-000000471306"
)
@pytest.mark.asyncio
async def test_terminal_generation_handoff_is_not_overclaimed_as_queued(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def terminal_preflight(**_kwargs):
return {
"status": "existing_terminal_generation",
"should_collect_evidence": False,
"existing_candidate_op_id": (
"00000000-0000-0000-0000-000000471306"
),
}
async def must_not_collect(**_kwargs):
raise AssertionError("terminal generation must stop before evidence collection")
monkeypatch.setattr(
"src.jobs.awooop_ansible_candidate_backfill_job."
"preflight_ansible_candidate_generation",
terminal_preflight,
)
handoff = await enqueue_ai_decision_ansible_candidate(
incident=_sentry_runtime_incident(),
proposal_data=_sentry_runtime_proposal("alert-current-1"),
evidence_collector=must_not_collect,
evidence_verifier=must_not_collect,
)
assert handoff["status"] == "existing_terminal_generation"
assert handoff["queued"] is False
assert handoff["active_blockers"] == ["existing_terminal_generation"]
assert handoff["automation_run_id"] == (
"00000000-0000-0000-0000-000000471306"
)
@pytest.mark.asyncio
async def test_webhook_generation_rejects_unverified_historical_recurrence(
monkeypatch: pytest.MonkeyPatch,