fix(auto-repair): prefer exact playbooks and fail failed steps
This commit is contained in:
@@ -277,6 +277,49 @@ class TestAutoRepairService:
|
||||
assert decision.playbook.playbook_id == playbook.playbook_id
|
||||
assert decision.blocked_by is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exact_alert_match_wins_over_higher_fuzzy_similarity(
|
||||
self,
|
||||
service,
|
||||
mock_playbook_service,
|
||||
):
|
||||
"""Exact alert/service playbooks must outrank unrelated fuzzy matches."""
|
||||
fuzzy_playbook = create_high_quality_playbook(
|
||||
playbook_id="PB-FUZZY-HOST",
|
||||
risk_level=RiskLevel.LOW,
|
||||
)
|
||||
fuzzy_playbook.symptom_pattern = SymptomPattern(
|
||||
alert_names=["HostCPUHigh"],
|
||||
affected_services=["node-exporter"],
|
||||
severity_range=["P2"],
|
||||
)
|
||||
exact_playbook = create_high_quality_playbook(
|
||||
playbook_id="PB-EXACT-CANARY",
|
||||
risk_level=RiskLevel.LOW,
|
||||
)
|
||||
exact_playbook.symptom_pattern = SymptomPattern(
|
||||
alert_names=["AwoooPAutoRepairCanaryT16"],
|
||||
affected_services=["awoooi-auto-repair-canary"],
|
||||
severity_range=["P3"],
|
||||
)
|
||||
mock_playbook_service.add_playbook(fuzzy_playbook)
|
||||
mock_playbook_service.add_playbook(exact_playbook)
|
||||
mock_playbook_service.set_recommendations([
|
||||
MockPlaybookRecommendation(fuzzy_playbook, similarity_score=0.95),
|
||||
MockPlaybookRecommendation(exact_playbook, similarity_score=0.45),
|
||||
])
|
||||
|
||||
incident = create_test_incident(
|
||||
severity=Severity.P3,
|
||||
alert_name="AwoooPAutoRepairCanaryT16",
|
||||
)
|
||||
incident.affected_services = ["awoooi-auto-repair-canary"]
|
||||
decision = await service.evaluate_auto_repair(incident)
|
||||
|
||||
assert decision.can_auto_repair is True
|
||||
assert decision.playbook is not None
|
||||
assert decision.playbook.playbook_id == "PB-EXACT-CANARY"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backup_failure_blocks_k8s_playbook(self, service, mock_playbook_service):
|
||||
"""Backup/host incidents must not execute K8s rollout playbooks."""
|
||||
|
||||
@@ -298,9 +298,6 @@ async def test_auto_repair_failure_does_not_call_verifier(monkeypatch):
|
||||
pb_service = FailingPlaybookService()
|
||||
pb_service.add_playbook(playbook)
|
||||
|
||||
# 讓 _execute_step 拋例外以觸發失敗路徑
|
||||
original_execute_step = AutoRepairService._execute_step
|
||||
|
||||
async def _always_fail(self_inner, incident_arg, step_arg) -> str:
|
||||
raise RuntimeError("強制測試失敗")
|
||||
|
||||
@@ -323,6 +320,40 @@ async def test_auto_repair_failure_does_not_call_verifier(monkeypatch):
|
||||
assert len(stub_learning.verification_calls) == 0, "執行失敗時不應呼叫 record_verification_result"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_repair_failed_step_string_marks_execution_failure(monkeypatch):
|
||||
"""Executor returned FAILED text must not be stored as successful repair."""
|
||||
stub_verifier = StubVerifier(result="success")
|
||||
stub_learning = StubLearningService()
|
||||
|
||||
import src.services.post_execution_verifier as _pev_mod
|
||||
monkeypatch.setattr(_pev_mod, "_verifier", stub_verifier)
|
||||
|
||||
import src.services.learning_service as _ls_mod
|
||||
monkeypatch.setattr(_ls_mod, "_learning_service", stub_learning)
|
||||
|
||||
playbook = _make_playbook()
|
||||
pb_service = StubPlaybookService()
|
||||
pb_service.add_playbook(playbook)
|
||||
|
||||
async def _returns_failed(self_inner, incident_arg, step_arg) -> str:
|
||||
return "FAILED: simulated executor failure"
|
||||
|
||||
monkeypatch.setattr(AutoRepairService, "_execute_step", _returns_failed)
|
||||
|
||||
service = AutoRepairService(
|
||||
playbook_service=pb_service,
|
||||
cooldown_checker=_no_cooldown,
|
||||
)
|
||||
|
||||
result = await service.execute_auto_repair(_make_incident(), playbook)
|
||||
|
||||
assert result.success is False
|
||||
assert "simulated executor failure" in (result.error or "")
|
||||
assert len(stub_verifier.calls) == 0
|
||||
assert len(stub_learning.verification_calls) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_verification_result_no_playbook_id_does_not_crash():
|
||||
"""
|
||||
@@ -330,7 +361,6 @@ async def test_record_verification_result_no_playbook_id_does_not_crash():
|
||||
驗證 learning_service 對 None playbook_id 的防禦性。
|
||||
"""
|
||||
from src.services.learning_service import LearningService
|
||||
from src.repositories.interfaces import ILearningRepository, ITrustRepository
|
||||
|
||||
class NullLearningRepo:
|
||||
async def record_repair(self, **kwargs) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user