fix(auto-repair): prefer exact playbooks and fail failed steps
All checks were successful
Code Review / ai-code-review (push) Successful in 11s
CD Pipeline / tests (push) Successful in 1m3s
CD Pipeline / build-and-deploy (push) Successful in 3m31s
CD Pipeline / post-deploy-checks (push) Successful in 1m32s

This commit is contained in:
Your Name
2026-05-13 23:21:16 +08:00
parent ae643552e9
commit 7a8cbb3241
4 changed files with 155 additions and 11 deletions

View File

@@ -277,6 +277,49 @@ class TestAutoRepairService:
assert decision.playbook.playbook_id == playbook.playbook_id
assert decision.blocked_by is None
@pytest.mark.asyncio
async def test_exact_alert_match_wins_over_higher_fuzzy_similarity(
self,
service,
mock_playbook_service,
):
"""Exact alert/service playbooks must outrank unrelated fuzzy matches."""
fuzzy_playbook = create_high_quality_playbook(
playbook_id="PB-FUZZY-HOST",
risk_level=RiskLevel.LOW,
)
fuzzy_playbook.symptom_pattern = SymptomPattern(
alert_names=["HostCPUHigh"],
affected_services=["node-exporter"],
severity_range=["P2"],
)
exact_playbook = create_high_quality_playbook(
playbook_id="PB-EXACT-CANARY",
risk_level=RiskLevel.LOW,
)
exact_playbook.symptom_pattern = SymptomPattern(
alert_names=["AwoooPAutoRepairCanaryT16"],
affected_services=["awoooi-auto-repair-canary"],
severity_range=["P3"],
)
mock_playbook_service.add_playbook(fuzzy_playbook)
mock_playbook_service.add_playbook(exact_playbook)
mock_playbook_service.set_recommendations([
MockPlaybookRecommendation(fuzzy_playbook, similarity_score=0.95),
MockPlaybookRecommendation(exact_playbook, similarity_score=0.45),
])
incident = create_test_incident(
severity=Severity.P3,
alert_name="AwoooPAutoRepairCanaryT16",
)
incident.affected_services = ["awoooi-auto-repair-canary"]
decision = await service.evaluate_auto_repair(incident)
assert decision.can_auto_repair is True
assert decision.playbook is not None
assert decision.playbook.playbook_id == "PB-EXACT-CANARY"
@pytest.mark.asyncio
async def test_backup_failure_blocks_k8s_playbook(self, service, mock_playbook_service):
"""Backup/host incidents must not execute K8s rollout playbooks."""

View File

@@ -298,9 +298,6 @@ async def test_auto_repair_failure_does_not_call_verifier(monkeypatch):
pb_service = FailingPlaybookService()
pb_service.add_playbook(playbook)
# 讓 _execute_step 拋例外以觸發失敗路徑
original_execute_step = AutoRepairService._execute_step
async def _always_fail(self_inner, incident_arg, step_arg) -> str:
raise RuntimeError("強制測試失敗")
@@ -323,6 +320,40 @@ async def test_auto_repair_failure_does_not_call_verifier(monkeypatch):
assert len(stub_learning.verification_calls) == 0, "執行失敗時不應呼叫 record_verification_result"
@pytest.mark.asyncio
async def test_auto_repair_failed_step_string_marks_execution_failure(monkeypatch):
"""Executor returned FAILED text must not be stored as successful repair."""
stub_verifier = StubVerifier(result="success")
stub_learning = StubLearningService()
import src.services.post_execution_verifier as _pev_mod
monkeypatch.setattr(_pev_mod, "_verifier", stub_verifier)
import src.services.learning_service as _ls_mod
monkeypatch.setattr(_ls_mod, "_learning_service", stub_learning)
playbook = _make_playbook()
pb_service = StubPlaybookService()
pb_service.add_playbook(playbook)
async def _returns_failed(self_inner, incident_arg, step_arg) -> str:
return "FAILED: simulated executor failure"
monkeypatch.setattr(AutoRepairService, "_execute_step", _returns_failed)
service = AutoRepairService(
playbook_service=pb_service,
cooldown_checker=_no_cooldown,
)
result = await service.execute_auto_repair(_make_incident(), playbook)
assert result.success is False
assert "simulated executor failure" in (result.error or "")
assert len(stub_verifier.calls) == 0
assert len(stub_learning.verification_calls) == 0
@pytest.mark.asyncio
async def test_record_verification_result_no_playbook_id_does_not_crash():
"""
@@ -330,7 +361,6 @@ async def test_record_verification_result_no_playbook_id_does_not_crash():
驗證 learning_service 對 None playbook_id 的防禦性。
"""
from src.services.learning_service import LearningService
from src.repositories.interfaces import ILearningRepository, ITrustRepository
class NullLearningRepo:
async def record_repair(self, **kwargs) -> bool: