fix(ai): suppress sealed slo watchdog meta noise
This commit is contained in:
@@ -3,7 +3,10 @@ from __future__ import annotations
|
||||
from datetime import datetime, timedelta
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from src.jobs.ai_slo_watchdog_job import _format_slo_violation_for_alert
|
||||
from src.jobs.ai_slo_watchdog_job import (
|
||||
_format_slo_violation_for_alert,
|
||||
_is_observation_only_slo_violation,
|
||||
)
|
||||
from src.services.ai_slo_calculator import (
|
||||
SLO_AUTO_SUCCESS_MIN,
|
||||
SloMetric,
|
||||
@@ -176,3 +179,92 @@ def test_watchdog_formats_auto_execute_diagnostics_for_meta_alert():
|
||||
assert cause is not None
|
||||
assert "Top failure groups" in cause
|
||||
assert "不需要重啟服務或改寫歷史資料" in cause
|
||||
|
||||
|
||||
def test_watchdog_treats_sealed_auto_execute_slo_as_observation_only():
|
||||
report = SloReport(
|
||||
metrics=[
|
||||
SloMetric(
|
||||
name="auto_execute_success_rate",
|
||||
value=0.83,
|
||||
threshold=SLO_AUTO_SUCCESS_MIN,
|
||||
direction="above",
|
||||
sample_count=53,
|
||||
violated=True,
|
||||
)
|
||||
],
|
||||
any_violated=True,
|
||||
diagnostics={
|
||||
"auto_execute_success_rate": {
|
||||
"status": "sealed_waiting_window",
|
||||
"open_failure_group_count": 0,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert _is_observation_only_slo_violation(
|
||||
report,
|
||||
["auto_execute_success_rate"],
|
||||
)
|
||||
|
||||
|
||||
def test_watchdog_keeps_auto_execute_slo_alert_when_open_groups_remain():
|
||||
report = SloReport(
|
||||
metrics=[
|
||||
SloMetric(
|
||||
name="auto_execute_success_rate",
|
||||
value=0.83,
|
||||
threshold=SLO_AUTO_SUCCESS_MIN,
|
||||
direction="above",
|
||||
sample_count=53,
|
||||
violated=True,
|
||||
)
|
||||
],
|
||||
any_violated=True,
|
||||
diagnostics={
|
||||
"auto_execute_success_rate": {
|
||||
"status": "needs_investigation",
|
||||
"open_failure_group_count": 1,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert not _is_observation_only_slo_violation(
|
||||
report,
|
||||
["auto_execute_success_rate"],
|
||||
)
|
||||
|
||||
|
||||
def test_watchdog_keeps_slo_alert_when_other_metric_violates():
|
||||
report = SloReport(
|
||||
metrics=[
|
||||
SloMetric(
|
||||
name="auto_execute_success_rate",
|
||||
value=0.83,
|
||||
threshold=SLO_AUTO_SUCCESS_MIN,
|
||||
direction="above",
|
||||
sample_count=53,
|
||||
violated=True,
|
||||
),
|
||||
SloMetric(
|
||||
name="human_override_rate",
|
||||
value=0.25,
|
||||
threshold=0.20,
|
||||
direction="below",
|
||||
sample_count=10,
|
||||
violated=True,
|
||||
),
|
||||
],
|
||||
any_violated=True,
|
||||
diagnostics={
|
||||
"auto_execute_success_rate": {
|
||||
"status": "sealed_waiting_window",
|
||||
"open_failure_group_count": 0,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert not _is_observation_only_slo_violation(
|
||||
report,
|
||||
["auto_execute_success_rate", "human_override_rate"],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user