feat(ai): enforce single-writer automation router
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m21s
CD Pipeline / build-and-deploy (push) Successful in 6m53s
CD Pipeline / post-deploy-checks (push) Successful in 1m55s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 49s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m21s
CD Pipeline / build-and-deploy (push) Successful in 6m53s
CD Pipeline / post-deploy-checks (push) Successful in 1m55s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 49s
This commit is contained in:
@@ -1,43 +1,35 @@
|
||||
# apps/api/tests/test_cs1_auto_execute.py
|
||||
# 2026-04-27 ogt + Claude Sonnet 4.6 — CS1 LLM 高信心度自動執行邏輯單元測試
|
||||
"""
|
||||
測試覆蓋:
|
||||
1. confidence=0.90 + low risk + kubectl 有值 → execute_approved_action 被呼叫
|
||||
2. confidence=0.70 → 不執行(低信心度)
|
||||
3. confidence=0.85 + CRITICAL → 不執行
|
||||
4. confidence=0.90 + DESTRUCTIVE_PATTERN → 不執行
|
||||
5. confidence=0.90 + NO_ACTION → 不執行
|
||||
6. confidence=0.90 執行失敗(exception)→ 降級 PENDING 不 crash
|
||||
|
||||
測試分類:unit(mock ApprovalExecutionService / service,無 DB / Redis 依賴)
|
||||
"""
|
||||
"""Webhook AI decisions must route through the single-writer broker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import inspect
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from src.models.ai import AIBlastRadius, AIDataImpact, AIRiskLevel, OpenClawDecision, SuggestedAction
|
||||
from src.api.v1 import webhooks
|
||||
from src.models.ai import (
|
||||
AIBlastRadius,
|
||||
AIDataImpact,
|
||||
OpenClawDecision,
|
||||
SuggestedAction,
|
||||
)
|
||||
from src.models.approval import RiskLevel
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Helpers
|
||||
# =============================================================================
|
||||
from src.services.approval_execution import ApprovalExecutionService
|
||||
|
||||
|
||||
def _make_analysis(
|
||||
*,
|
||||
confidence: float = 0.90,
|
||||
kubectl_command: str = "kubectl rollout restart deployment/api -n prod",
|
||||
risk_level_str: str = "low",
|
||||
suggested_action: SuggestedAction = SuggestedAction.RESTART_DEPLOYMENT,
|
||||
) -> OpenClawDecision:
|
||||
return OpenClawDecision(
|
||||
action_title="Restart deployment",
|
||||
kubectl_command=kubectl_command,
|
||||
description="Auto restart",
|
||||
risk_level=risk_level_str,
|
||||
description="Controlled restart candidate",
|
||||
risk_level="low",
|
||||
suggested_action=suggested_action,
|
||||
confidence=confidence,
|
||||
blast_radius=AIBlastRadius(
|
||||
@@ -53,176 +45,124 @@ def _make_analysis(
|
||||
)
|
||||
|
||||
|
||||
def _run_cs1_block(
|
||||
analysis_result: OpenClawDecision | None,
|
||||
risk_level: RiskLevel,
|
||||
exec_side_effect=None,
|
||||
) -> tuple[MagicMock, MagicMock]:
|
||||
"""
|
||||
從 webhooks.py CS1 auto-execute 邏輯提取的同等邏輯,
|
||||
直接呼叫,驗證 execute_approved_action 的呼叫情況。
|
||||
|
||||
回傳 (mock_executor_class, mock_execute_method)
|
||||
"""
|
||||
def _can_route(analysis: OpenClawDecision, risk_level: RiskLevel) -> bool:
|
||||
from src.services.action_parser import is_safe_kubectl_action
|
||||
|
||||
mock_exec_instance = MagicMock()
|
||||
if exec_side_effect is not None:
|
||||
mock_exec_instance.execute_approved_action = AsyncMock(side_effect=exec_side_effect)
|
||||
else:
|
||||
mock_exec_instance.execute_approved_action = AsyncMock(return_value=True)
|
||||
non_actions = {"NO_ACTION", "INVESTIGATE", "OBSERVE"}
|
||||
action_name = analysis.suggested_action.value
|
||||
command = (analysis.kubectl_command or "").strip()
|
||||
return (
|
||||
bool(command)
|
||||
and analysis.confidence >= 0.85
|
||||
and risk_level != RiskLevel.CRITICAL
|
||||
and action_name not in non_actions
|
||||
and is_safe_kubectl_action(command)
|
||||
)
|
||||
|
||||
mock_executor_cls = MagicMock(return_value=mock_exec_instance)
|
||||
|
||||
with (
|
||||
patch("src.services.approval_execution.ApprovalExecutionService", mock_executor_cls),
|
||||
patch("src.models.approval.ApprovalRequest", MagicMock()),
|
||||
patch("src.models.approval.ApprovalStatus", MagicMock()),
|
||||
@pytest.mark.parametrize(
|
||||
("analysis", "risk_level", "expected"),
|
||||
[
|
||||
(_make_analysis(), RiskLevel.LOW, True),
|
||||
(_make_analysis(confidence=0.70), RiskLevel.LOW, False),
|
||||
(_make_analysis(confidence=0.85), RiskLevel.MEDIUM, True),
|
||||
(_make_analysis(), RiskLevel.CRITICAL, False),
|
||||
(
|
||||
_make_analysis(
|
||||
kubectl_command="kubectl delete pods --all -n prod",
|
||||
),
|
||||
RiskLevel.LOW,
|
||||
False,
|
||||
),
|
||||
(
|
||||
_make_analysis(suggested_action=SuggestedAction.NO_ACTION),
|
||||
RiskLevel.LOW,
|
||||
False,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_cs1_controlled_queue_eligibility(
|
||||
analysis: OpenClawDecision,
|
||||
risk_level: RiskLevel,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
assert _can_route(analysis, risk_level) is expected
|
||||
|
||||
|
||||
def test_active_webhook_paths_do_not_call_direct_executor() -> None:
|
||||
receive_source = inspect.getsource(webhooks.receive_alert)
|
||||
alertmanager_source = inspect.getsource(webhooks._process_new_alert_background)
|
||||
router_source = inspect.getsource(webhooks._try_auto_repair_background)
|
||||
legacy_source = inspect.getsource(
|
||||
webhooks._legacy_try_auto_repair_background_disabled
|
||||
)
|
||||
|
||||
for source in (receive_source, alertmanager_source, router_source):
|
||||
assert "execute_approved_action" not in source
|
||||
assert "ApprovalExecutionService" not in source
|
||||
assert "execute_auto_repair(" not in source
|
||||
assert "_try_auto_repair_background" in receive_source
|
||||
assert "_try_auto_repair_background" in alertmanager_source
|
||||
assert "enqueue_ai_decision_ansible_candidate" in router_source
|
||||
assert "webhook_direct_auto_repair_superseded" in legacy_source
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_webhook_router_writes_queue_receipt_without_side_effect(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
incident = SimpleNamespace(project_id="awoooi")
|
||||
incident_service = SimpleNamespace(
|
||||
get_from_working_memory=AsyncMock(return_value=incident)
|
||||
)
|
||||
append = AsyncMock()
|
||||
|
||||
async def queue(**_kwargs):
|
||||
return {
|
||||
"schema_version": "ai_decision_controlled_executor_handoff_v1",
|
||||
"status": "controlled_check_mode_queued",
|
||||
"automation_run_id": "00000000-0000-0000-0000-000000000101",
|
||||
"queued": True,
|
||||
"side_effect_performed": False,
|
||||
"single_writer_executor": "awoooi-ansible-executor-broker",
|
||||
"active_blockers": [],
|
||||
}
|
||||
|
||||
monkeypatch.setattr(webhooks, "get_incident_service", lambda: incident_service)
|
||||
monkeypatch.setattr(
|
||||
"src.jobs.awooop_ansible_candidate_backfill_job.enqueue_ai_decision_ansible_candidate",
|
||||
queue,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.repositories.alert_operation_log_repository.get_alert_operation_log_repository",
|
||||
lambda: SimpleNamespace(append=append),
|
||||
)
|
||||
|
||||
handoff = await webhooks._try_auto_repair_background(
|
||||
incident_id="INC-QUEUE",
|
||||
approval_id="00000000-0000-0000-0000-000000000201",
|
||||
alert_type="MomoPostgresBackupFailed",
|
||||
target_resource="momo-postgres",
|
||||
namespace="awoooi-prod",
|
||||
risk_level="low",
|
||||
)
|
||||
|
||||
assert handoff["queued"] is True
|
||||
assert handoff["side_effect_performed"] is False
|
||||
append.assert_awaited_once()
|
||||
assert append.await_args.kwargs["context"]["side_effect_performed"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_approval_executor_rejects_auto_approved_direct_caller() -> None:
|
||||
approval = SimpleNamespace(
|
||||
id="00000000-0000-0000-0000-000000000301",
|
||||
incident_id="INC-DIRECT-BLOCK",
|
||||
requested_by="auto_approve_rule_engine",
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="auto_approved_direct_execution_disabled_use_single_writer_broker",
|
||||
):
|
||||
# Replicate the exact condition logic from webhooks.py CS1 block
|
||||
_non_destructive_actions = {"NO_ACTION", "INVESTIGATE", "OBSERVE"}
|
||||
_sa_val = (
|
||||
analysis_result.suggested_action.value
|
||||
if analysis_result and hasattr(analysis_result.suggested_action, "value")
|
||||
else str(getattr(analysis_result, "suggested_action", ""))
|
||||
)
|
||||
|
||||
if analysis_result:
|
||||
_cs1_kubectl = analysis_result.kubectl_command.strip() if analysis_result.kubectl_command else ""
|
||||
_cs1_can_auto = (
|
||||
bool(_cs1_kubectl)
|
||||
and analysis_result.confidence >= 0.85
|
||||
and risk_level != RiskLevel.CRITICAL
|
||||
and _sa_val not in _non_destructive_actions
|
||||
and is_safe_kubectl_action(_cs1_kubectl)
|
||||
)
|
||||
if _cs1_can_auto:
|
||||
import asyncio
|
||||
|
||||
from src.models.approval import ApprovalRequest, ApprovalStatus
|
||||
from src.services.approval_execution import ApprovalExecutionService
|
||||
|
||||
_cs1_auto_approval = MagicMock()
|
||||
_cs1_executor = ApprovalExecutionService()
|
||||
asyncio.run(_cs1_executor.execute_approved_action(_cs1_auto_approval))
|
||||
|
||||
return mock_executor_cls, mock_exec_instance
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Tests
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestCS1AutoExecuteConditions:
|
||||
"""測試 CS1 自動執行的觸發條件"""
|
||||
|
||||
def test_high_confidence_low_risk_executes(self):
|
||||
"""confidence=0.90 + LOW risk + kubectl 有值 → execute 被呼叫"""
|
||||
analysis = _make_analysis(confidence=0.90)
|
||||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||||
mock_exec.execute_approved_action.assert_called_once()
|
||||
|
||||
def test_low_confidence_does_not_execute(self):
|
||||
"""confidence=0.70 → 不執行"""
|
||||
analysis = _make_analysis(confidence=0.70)
|
||||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||||
mock_exec.execute_approved_action.assert_not_called()
|
||||
|
||||
def test_boundary_confidence_085_executes(self):
|
||||
"""confidence=0.85 剛好等於門檻 → 執行"""
|
||||
analysis = _make_analysis(confidence=0.85)
|
||||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||||
mock_exec.execute_approved_action.assert_called_once()
|
||||
|
||||
def test_critical_risk_does_not_execute(self):
|
||||
"""confidence=0.90 + CRITICAL → 不執行"""
|
||||
analysis = _make_analysis(confidence=0.90, risk_level_str="critical")
|
||||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.CRITICAL)
|
||||
mock_exec.execute_approved_action.assert_not_called()
|
||||
|
||||
def test_destructive_pattern_does_not_execute(self):
|
||||
"""kubectl 含 destructive pattern → 不執行"""
|
||||
from src.services.auto_approve import _DESTRUCTIVE_PATTERNS
|
||||
# 取第一個 pattern 構造一個含危險詞的指令
|
||||
bad_pattern = _DESTRUCTIVE_PATTERNS[0]
|
||||
analysis = _make_analysis(
|
||||
confidence=0.90,
|
||||
kubectl_command=f"kubectl {bad_pattern} deployment/api -n prod",
|
||||
)
|
||||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||||
mock_exec.execute_approved_action.assert_not_called()
|
||||
|
||||
def test_single_delete_pod_executes(self):
|
||||
"""單一 Pod delete 是可恢復操作,parser 不應誤殺"""
|
||||
analysis = _make_analysis(
|
||||
confidence=0.90,
|
||||
kubectl_command="kubectl delete pod api-xxx-yyy -n prod",
|
||||
)
|
||||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||||
mock_exec.execute_approved_action.assert_called_once()
|
||||
|
||||
def test_no_action_does_not_execute(self):
|
||||
"""suggested_action=NO_ACTION → 不執行"""
|
||||
analysis = _make_analysis(
|
||||
confidence=0.90,
|
||||
suggested_action=SuggestedAction.NO_ACTION,
|
||||
)
|
||||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||||
mock_exec.execute_approved_action.assert_not_called()
|
||||
|
||||
def test_investigate_does_not_execute(self):
|
||||
"""suggested_action=INVESTIGATE → 不執行"""
|
||||
analysis = _make_analysis(
|
||||
confidence=0.90,
|
||||
suggested_action=SuggestedAction.INVESTIGATE,
|
||||
)
|
||||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||||
mock_exec.execute_approved_action.assert_not_called()
|
||||
|
||||
|
||||
class TestCS1AutoExecuteFailureDegradation:
|
||||
"""測試執行失敗時的降級行為"""
|
||||
|
||||
def test_execution_exception_does_not_crash(self):
|
||||
"""execute_approved_action 拋 exception → 捕捉後繼續,不 crash"""
|
||||
analysis = _make_analysis(confidence=0.90)
|
||||
|
||||
# 直接測試條件邏輯,確保例外被吞掉
|
||||
from src.services.action_parser import is_safe_kubectl_action
|
||||
|
||||
_non_destructive_actions = {"NO_ACTION", "INVESTIGATE", "OBSERVE"}
|
||||
_sa_val = analysis.suggested_action.value
|
||||
_cs1_kubectl = analysis.kubectl_command.strip()
|
||||
_cs1_can_auto = (
|
||||
bool(_cs1_kubectl)
|
||||
and analysis.confidence >= 0.85
|
||||
and RiskLevel.LOW != RiskLevel.CRITICAL
|
||||
and _sa_val not in _non_destructive_actions
|
||||
and is_safe_kubectl_action(_cs1_kubectl)
|
||||
)
|
||||
assert _cs1_can_auto, "前置條件必須為 True 才能測試降級"
|
||||
|
||||
raised = False
|
||||
try:
|
||||
import asyncio
|
||||
|
||||
async def _simulate():
|
||||
# 模擬整個 if _cs1_can_auto: try/except 區塊
|
||||
if _cs1_can_auto:
|
||||
try:
|
||||
raise RuntimeError("executor exploded")
|
||||
except Exception:
|
||||
pass # 降級:維持 PENDING
|
||||
|
||||
asyncio.run(_simulate())
|
||||
except Exception:
|
||||
raised = True
|
||||
|
||||
assert not raised, "例外應被 CS1 try/except 吞掉,不應傳播"
|
||||
|
||||
def test_empty_kubectl_does_not_execute(self):
|
||||
"""kubectl_command 為空字串 → 不執行"""
|
||||
analysis = _make_analysis(confidence=0.90, kubectl_command="")
|
||||
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
|
||||
mock_exec.execute_approved_action.assert_not_called()
|
||||
await ApprovalExecutionService().execute_approved_action(approval)
|
||||
|
||||
Reference in New Issue
Block a user