feat(ai): enforce single-writer automation router
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m21s
CD Pipeline / build-and-deploy (push) Successful in 6m53s
CD Pipeline / post-deploy-checks (push) Successful in 1m55s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 49s

This commit is contained in:
ogt
2026-07-11 05:45:48 +08:00
parent 01605b4947
commit f453359e34
20 changed files with 1674 additions and 570 deletions

View File

@@ -1075,6 +1075,18 @@ def test_runtime_receipt_auxiliary_sql_keeps_source_family_counts_schema_safe():
)
assert "automation_run_id" in _RUNTIME_OPERATION_LATEST_SQL
assert "automation_run_id" in runtime_control_module._RUNTIME_OPERATION_LATEST_DIRECT_SQL
for sql in (
_RUNTIME_OPERATION_LATEST_SQL,
_RUNTIME_OPERATION_LATEST_DIRECT_SQL,
runtime_control_module._RUNTIME_OPERATION_CHAIN_SQL,
runtime_control_module._RUNTIME_OPERATION_CHAIN_DIRECT_SQL,
):
assert "single_writer_executor" in sql
assert "candidate_idempotency_key" in sql
assert "apply_idempotency_key" in sql
assert "source_truth_diff_required" in sql
assert "target_selector_present" in sql
assert "router_source_sha" in sql
assert "automation_run_id" in runtime_control_module._RUNTIME_AUTO_REPAIR_LATEST_SQL
assert "automation_run_id" in runtime_control_module._RUNTIME_VERIFIER_LATEST_SQL
assert "automation_run_id" in runtime_control_module._RUNTIME_KM_LATEST_SQL
@@ -1102,6 +1114,99 @@ def test_runtime_receipt_auxiliary_sql_keeps_source_family_counts_schema_safe():
assert "channel_chat_id LIKE 'alert-group:%'" in _RUNTIME_GROUPED_ALERT_EVENT_COUNTS_SQL
def test_autonomous_single_writer_requires_same_run_runtime_and_deployed_source() -> None:
run_id = "11111111-1111-4111-8111-111111111111"
rows = [
{
"op_id": run_id,
"operation_type": "ansible_candidate_matched",
"status": "dry_run",
"automation_run_id": run_id,
"decision_path": "repair_candidate_controlled_queue",
"single_writer_executor": "awoooi-ansible-executor-broker",
"candidate_idempotency_key": "ansible-candidate:awoooi:101",
"target_selector_present": True,
"source_truth_diff_required": "true",
"router_source_sha": "source123",
},
{
"op_id": "22222222-2222-4222-8222-222222222222",
"operation_type": "ansible_check_mode_executed",
"status": "success",
"automation_run_id": run_id,
"single_writer_executor": "awoooi-ansible-executor-broker",
},
{
"op_id": "33333333-3333-4333-8333-333333333333",
"operation_type": "ansible_apply_executed",
"status": "success",
"automation_run_id": run_id,
"single_writer_executor": "awoooi-ansible-executor-broker",
"apply_idempotency_key": "ansible-apply:run:catalog",
},
]
runtime = runtime_control_module._build_autonomous_single_writer_runtime_readback(
operation_latest_rows=rows,
loop_ledger={
"automation_run_id": run_id,
"same_run_correlation": True,
},
)
assert runtime["runtime_evidence_ready"] is True
assert all(runtime["controls"].values())
payload = {
"strict_runtime_completion": {
"automation_run_id": run_id,
"closed": True,
},
"rollups": {},
}
source_controls = {
"decision_manager_queue_only": True,
"webhook_router_queue_only": True,
"failure_watcher_queue_only": True,
"auto_approved_direct_execution_blocked": True,
"candidate_idempotency_contract_verified": True,
"apply_idempotency_contract_verified": True,
"critical_break_glass_contract_verified": True,
}
runtime_control_module._attach_autonomous_single_writer_readback(
payload,
readback={"autonomous_single_writer_runtime": runtime},
boundary_readback={
"deployed_source_sha": "source123",
"autonomous_single_writer_source_boundary": {
"receipt_ref": "configmap:awoooi-prod/boundary",
"verifier": "cd_tests_and_production_source_sha_v1",
"controls": source_controls,
"active_blockers": [],
},
},
)
assert payload["autonomous_single_writer"]["closed"] is True
assert payload["autonomous_single_writer"]["active_blockers"] == []
assert payload["rollups"]["autonomous_single_writer_closed_count"] == 1
payload["strict_runtime_completion"]["automation_run_id"] = "other-run"
runtime_control_module._attach_autonomous_single_writer_readback(
payload,
readback={"autonomous_single_writer_runtime": runtime},
boundary_readback={
"deployed_source_sha": "source123",
"autonomous_single_writer_source_boundary": {
"controls": source_controls,
"active_blockers": [],
},
},
)
assert payload["autonomous_single_writer"]["closed"] is False
assert "strict_runtime_same_run_closed_not_verified" in payload[
"autonomous_single_writer"
]["active_blockers"]
def test_ai_agent_autonomous_runtime_control_uses_current_owner_directive():
data = build_ai_agent_autonomous_runtime_control()

View File

@@ -150,6 +150,30 @@ def _autonomous_runtime_control_trust_boundary_closed() -> dict:
return payload
def _autonomous_runtime_control_single_writer_closed() -> dict:
payload = _autonomous_runtime_control_trust_boundary_closed()
run_id = payload["strict_runtime_completion"]["automation_run_id"]
payload["autonomous_single_writer"] = {
"schema_version": "awoooi_autonomous_single_writer_readback_v1",
"closed": True,
"automation_run_id": run_id,
"router_source_sha": "source-p0-002",
"candidate_op_id": "candidate-p0-002",
"check_mode_op_id": "check-p0-002",
"apply_op_id": "apply-p0-002",
"source_receipt_ref": (
"configmap:awoooi-prod/awoooi-executor-boundary-verification"
),
"source_verifier": "cd_tests_and_production_source_sha_v1",
"controls": {
control_id: True
for control_id in priority_service._AI_SINGLE_WRITER_REQUIRED_CONTROL_IDS
},
"active_blockers": [],
}
return payload
async def _autonomous_runtime_control_closed_async() -> dict:
return _autonomous_runtime_control_closed()
@@ -2031,6 +2055,35 @@ def test_ai_automation_trust_boundary_requires_db_identity_and_budget_receipt()
assert program["summary"]["completion_percent"] == 5
def test_ai_automation_single_writer_runtime_receipt_advances_order() -> None:
payload = load_latest_awoooi_priority_work_order_readback()
apply_ai_automation_live_closure_readbacks(
payload,
autonomous_runtime_control=(
_autonomous_runtime_control_single_writer_closed()
),
)
program = payload["ai_automation_program_ledger"]
items = {item["id"]: item for item in program["work_items"]}
assert items["AIA-P0-001"]["status"] == "done"
assert items["AIA-P0-011"]["status"] == "done"
single_writer = items["AIA-P0-002"]
assert single_writer["status"] == "done"
assert single_writer["runtime_progress"]["runtime_closed"] is True
assert single_writer["runtime_progress"]["completion_percent"] == 100
assert single_writer["completion_evidence"]["automation_run_id"] == (
"run-p0-001-closed"
)
assert single_writer["completion_evidence"]["single_writer_executor"] == (
"awoooi-ansible-executor-broker"
)
assert program["summary"]["next_work_item_id"] == "AIA-P0-003"
assert program["summary"]["done_count"] == 3
assert program["summary"]["completion_percent"] == 15
def test_awoooi_priority_work_order_readback_endpoint_returns_snapshot(
monkeypatch: pytest.MonkeyPatch,
):

View File

@@ -9,6 +9,9 @@ from types import SimpleNamespace
import pytest
from src.core.config import settings
from src.jobs.awooop_ansible_candidate_backfill_job import (
enqueue_ai_decision_ansible_candidate,
)
from src.services.awooop_ansible_audit_service import (
build_ansible_decision_audit_payload,
build_ansible_truth,
@@ -1385,7 +1388,10 @@ def test_ansible_decision_audit_payload_marks_repair_candidate_queue_claimable()
assert payload["input"]["project_id"] == "awoooi"
def test_ansible_decision_audit_payload_exposes_check_mode_safety_flags() -> None:
def test_ansible_decision_audit_payload_exposes_check_mode_safety_flags(
monkeypatch,
) -> None:
monkeypatch.setenv("AWOOOI_BUILD_COMMIT_SHA", "source-sha-123")
incident = SimpleNamespace(
incident_id="INC-MOMO",
project_id="awoooi",
@@ -1417,6 +1423,7 @@ def test_ansible_decision_audit_payload_exposes_check_mode_safety_flags() -> Non
assert candidate["auto_apply_enabled"] is True
assert candidate["approval_required"] is False
assert candidate["risk_level"] == "low"
assert payload["input"]["router_source_sha"] == "source-sha-123"
def test_ansible_check_mode_claim_input_authorizes_controlled_apply() -> None:
@@ -1424,6 +1431,11 @@ def test_ansible_check_mode_claim_input_authorizes_controlled_apply() -> None:
"incident_id": "INC-MOMO",
"automation_run_id": "untrusted-upstream-run-id",
"executor": "ansible",
"decision_path": "repair_candidate_controlled_queue",
"idempotency_key": "ansible-candidate:awoooi:INC-MOMO",
"router_source_sha": "source-sha-123",
"target_selector": {"catalog_ids": ["ansible:188-ai-web"]},
"source_truth_diff": {"required_before_apply": True},
"executor_candidates": [
{
"catalog_id": "ansible:188-ai-web",
@@ -1448,6 +1460,19 @@ def test_ansible_check_mode_claim_input_authorizes_controlled_apply() -> None:
assert claim["approval_required_before_apply"] is False
assert claim["controlled_apply_allowed"] is True
assert claim["controlled_apply_blocker"] is None
assert claim["single_writer_executor"] == "awoooi-ansible-executor-broker"
assert claim["candidate_idempotency_key"] == (
"ansible-candidate:awoooi:INC-MOMO"
)
assert claim["apply_idempotency_key"] == (
"ansible-apply:00000000-0000-0000-0000-000000000001:"
"ansible:188-ai-web"
)
assert claim["router_source_sha"] == "source-sha-123"
assert claim["target_selector"] == {
"catalog_ids": ["ansible:188-ai-web"]
}
assert claim["source_truth_diff"]["required_before_apply"] is True
assert claim["catalog_playbook_path"] == "infra/ansible/playbooks/188-ai-web.yml"
assert claim["apply_playbook_path"] == "infra/ansible/playbooks/188-ai-web.yml"
assert claim["source_candidate_playbook_path"] == "infra/ansible/playbooks/188-ai-web.yml"
@@ -2231,6 +2256,165 @@ def test_ansible_check_and_apply_rows_persist_canonical_automation_run_id() -> N
assert '"automation_run_id": str(' in apply_source
@pytest.mark.asyncio
async def test_ai_decision_handoff_queues_evidence_backed_single_writer_candidate() -> None:
incident = SimpleNamespace(
incident_id="INC-AIA-P0-002",
project_id="awoooi",
severity=SimpleNamespace(value="low"),
alertname="MomoPostgresBackupFailed",
alert_category="backup_failure",
notification_type="TYPE-3",
affected_services=["momo-postgres"],
signals=[
SimpleNamespace(
alert_name="MomoPostgresBackupFailed",
labels={"alertname": "MomoPostgresBackupFailed", "instance": "188"},
annotations={},
)
],
)
recorded: list[dict] = []
async def collect(**_kwargs):
return SimpleNamespace(snapshot_id="evidence-1")
async def verify(**_kwargs):
return True
async def record(**kwargs):
recorded.append(kwargs)
return True
handoff = await enqueue_ai_decision_ansible_candidate(
incident=incident,
proposal_data={
"source": "decision_manager",
"risk_level": "low",
"action": "systemctl restart momo-backup",
},
recorder=record,
evidence_collector=collect,
evidence_verifier=verify,
)
assert handoff["status"] == "controlled_check_mode_queued"
assert handoff["queued"] is True
assert handoff["side_effect_performed"] is False
assert handoff["single_writer_executor"] == "awoooi-ansible-executor-broker"
assert handoff["source_truth_diff"]["required_before_apply"] is True
assert handoff["risk_policy_decision"]["risk_level"] == "low"
assert recorded[0]["decision_path"] == "repair_candidate_controlled_queue"
@pytest.mark.asyncio
async def test_ai_decision_handoff_blocks_critical_before_evidence_or_queue() -> None:
calls = 0
async def must_not_run(**_kwargs):
nonlocal calls
calls += 1
return True
handoff = await enqueue_ai_decision_ansible_candidate(
incident=SimpleNamespace(incident_id="INC-CRITICAL"),
proposal_data={"risk_level": "critical", "action": "restart"},
recorder=must_not_run,
evidence_collector=must_not_run,
evidence_verifier=must_not_run,
)
assert handoff["status"] == "critical_break_glass_required"
assert handoff["queued"] is False
assert handoff["side_effect_performed"] is False
assert calls == 0
def test_controlled_apply_uses_durable_idempotency_guard() -> None:
source = inspect.getsource(run_controlled_apply_for_claim)
broker_source = inspect.getsource(run_pending_check_modes_once)
assert "pg_advisory_xact_lock" in source
assert "apply_idempotency_key" in source
assert "ansible_controlled_apply_duplicate_suppressed" in source
assert "apply_duplicate_suppressed" in broker_source
@pytest.mark.asyncio
async def test_controlled_apply_duplicate_never_runs_subprocess(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from src.services import awooop_ansible_check_mode_service as service
class ExistingResult:
def mappings(self):
return self
def first(self):
return {
"op_id": "00000000-0000-0000-0000-000000000103",
"status": "success",
}
class FakeDb:
async def execute(self, *_args, **_kwargs):
return ExistingResult()
class FakeContext:
async def __aenter__(self):
return FakeDb()
async def __aexit__(self, *_args):
return False
subprocess_started = False
async def must_not_run(*_args, **_kwargs):
nonlocal subprocess_started
subprocess_started = True
raise AssertionError("duplicate apply must not execute")
monkeypatch.setattr(
service,
"get_db_context",
lambda _project_id: FakeContext(),
)
monkeypatch.setattr(
service,
"_execution_capability_timeout_seconds",
lambda *_args, **_kwargs: 30,
)
monkeypatch.setattr(service, "_run_ansible_command", must_not_run)
claim = AnsibleCheckModeClaim(
op_id="00000000-0000-0000-0000-000000000102",
source_candidate_op_id="00000000-0000-0000-0000-000000000101",
incident_id="INC-IDEMPOTENT",
catalog_id="ansible:188-momo-backup-user",
playbook_path="infra/ansible/playbooks/188-momo-backup-user.yml",
apply_playbook_path="infra/ansible/playbooks/188-momo-backup-user.yml",
inventory_hosts=("host_188",),
risk_level="low",
input_payload={
"controlled_apply_allowed": True,
"apply_idempotency_key": (
"ansible-apply:00000000-0000-0000-0000-000000000101:"
"ansible:188-momo-backup-user"
),
},
)
result = await service.run_controlled_apply_for_claim(
claim,
timeout_seconds=30,
)
assert result is None
assert subprocess_started is False
assert claim.input_payload["apply_duplicate_suppressed"] is True
assert claim.input_payload["existing_apply_status"] == "success"
def test_ansible_subprocess_is_terminated_when_worker_task_is_cancelled() -> None:
source = inspect.getsource(_run_ansible_command)

View File

@@ -1,43 +1,35 @@
# apps/api/tests/test_cs1_auto_execute.py
# 2026-04-27 ogt + Claude Sonnet 4.6 — CS1 LLM 高信心度自動執行邏輯單元測試
"""
測試覆蓋:
1. confidence=0.90 + low risk + kubectl 有值 → execute_approved_action 被呼叫
2. confidence=0.70 → 不執行(低信心度)
3. confidence=0.85 + CRITICAL → 不執行
4. confidence=0.90 + DESTRUCTIVE_PATTERN → 不執行
5. confidence=0.90 + NO_ACTION → 不執行
6. confidence=0.90 執行失敗exception→ 降級 PENDING 不 crash
測試分類unitmock ApprovalExecutionService / service無 DB / Redis 依賴)
"""
"""Webhook AI decisions must route through the single-writer broker."""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock, patch
import inspect
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from src.models.ai import AIBlastRadius, AIDataImpact, AIRiskLevel, OpenClawDecision, SuggestedAction
from src.api.v1 import webhooks
from src.models.ai import (
AIBlastRadius,
AIDataImpact,
OpenClawDecision,
SuggestedAction,
)
from src.models.approval import RiskLevel
# =============================================================================
# Helpers
# =============================================================================
from src.services.approval_execution import ApprovalExecutionService
def _make_analysis(
*,
confidence: float = 0.90,
kubectl_command: str = "kubectl rollout restart deployment/api -n prod",
risk_level_str: str = "low",
suggested_action: SuggestedAction = SuggestedAction.RESTART_DEPLOYMENT,
) -> OpenClawDecision:
return OpenClawDecision(
action_title="Restart deployment",
kubectl_command=kubectl_command,
description="Auto restart",
risk_level=risk_level_str,
description="Controlled restart candidate",
risk_level="low",
suggested_action=suggested_action,
confidence=confidence,
blast_radius=AIBlastRadius(
@@ -53,176 +45,124 @@ def _make_analysis(
)
def _run_cs1_block(
analysis_result: OpenClawDecision | None,
risk_level: RiskLevel,
exec_side_effect=None,
) -> tuple[MagicMock, MagicMock]:
"""
從 webhooks.py CS1 auto-execute 邏輯提取的同等邏輯,
直接呼叫,驗證 execute_approved_action 的呼叫情況。
回傳 (mock_executor_class, mock_execute_method)
"""
def _can_route(analysis: OpenClawDecision, risk_level: RiskLevel) -> bool:
from src.services.action_parser import is_safe_kubectl_action
mock_exec_instance = MagicMock()
if exec_side_effect is not None:
mock_exec_instance.execute_approved_action = AsyncMock(side_effect=exec_side_effect)
else:
mock_exec_instance.execute_approved_action = AsyncMock(return_value=True)
non_actions = {"NO_ACTION", "INVESTIGATE", "OBSERVE"}
action_name = analysis.suggested_action.value
command = (analysis.kubectl_command or "").strip()
return (
bool(command)
and analysis.confidence >= 0.85
and risk_level != RiskLevel.CRITICAL
and action_name not in non_actions
and is_safe_kubectl_action(command)
)
mock_executor_cls = MagicMock(return_value=mock_exec_instance)
with (
patch("src.services.approval_execution.ApprovalExecutionService", mock_executor_cls),
patch("src.models.approval.ApprovalRequest", MagicMock()),
patch("src.models.approval.ApprovalStatus", MagicMock()),
@pytest.mark.parametrize(
("analysis", "risk_level", "expected"),
[
(_make_analysis(), RiskLevel.LOW, True),
(_make_analysis(confidence=0.70), RiskLevel.LOW, False),
(_make_analysis(confidence=0.85), RiskLevel.MEDIUM, True),
(_make_analysis(), RiskLevel.CRITICAL, False),
(
_make_analysis(
kubectl_command="kubectl delete pods --all -n prod",
),
RiskLevel.LOW,
False,
),
(
_make_analysis(suggested_action=SuggestedAction.NO_ACTION),
RiskLevel.LOW,
False,
),
],
)
def test_cs1_controlled_queue_eligibility(
analysis: OpenClawDecision,
risk_level: RiskLevel,
expected: bool,
) -> None:
assert _can_route(analysis, risk_level) is expected
def test_active_webhook_paths_do_not_call_direct_executor() -> None:
receive_source = inspect.getsource(webhooks.receive_alert)
alertmanager_source = inspect.getsource(webhooks._process_new_alert_background)
router_source = inspect.getsource(webhooks._try_auto_repair_background)
legacy_source = inspect.getsource(
webhooks._legacy_try_auto_repair_background_disabled
)
for source in (receive_source, alertmanager_source, router_source):
assert "execute_approved_action" not in source
assert "ApprovalExecutionService" not in source
assert "execute_auto_repair(" not in source
assert "_try_auto_repair_background" in receive_source
assert "_try_auto_repair_background" in alertmanager_source
assert "enqueue_ai_decision_ansible_candidate" in router_source
assert "webhook_direct_auto_repair_superseded" in legacy_source
@pytest.mark.asyncio
async def test_webhook_router_writes_queue_receipt_without_side_effect(
monkeypatch: pytest.MonkeyPatch,
) -> None:
incident = SimpleNamespace(project_id="awoooi")
incident_service = SimpleNamespace(
get_from_working_memory=AsyncMock(return_value=incident)
)
append = AsyncMock()
async def queue(**_kwargs):
return {
"schema_version": "ai_decision_controlled_executor_handoff_v1",
"status": "controlled_check_mode_queued",
"automation_run_id": "00000000-0000-0000-0000-000000000101",
"queued": True,
"side_effect_performed": False,
"single_writer_executor": "awoooi-ansible-executor-broker",
"active_blockers": [],
}
monkeypatch.setattr(webhooks, "get_incident_service", lambda: incident_service)
monkeypatch.setattr(
"src.jobs.awooop_ansible_candidate_backfill_job.enqueue_ai_decision_ansible_candidate",
queue,
)
monkeypatch.setattr(
"src.repositories.alert_operation_log_repository.get_alert_operation_log_repository",
lambda: SimpleNamespace(append=append),
)
handoff = await webhooks._try_auto_repair_background(
incident_id="INC-QUEUE",
approval_id="00000000-0000-0000-0000-000000000201",
alert_type="MomoPostgresBackupFailed",
target_resource="momo-postgres",
namespace="awoooi-prod",
risk_level="low",
)
assert handoff["queued"] is True
assert handoff["side_effect_performed"] is False
append.assert_awaited_once()
assert append.await_args.kwargs["context"]["side_effect_performed"] is False
@pytest.mark.asyncio
async def test_approval_executor_rejects_auto_approved_direct_caller() -> None:
approval = SimpleNamespace(
id="00000000-0000-0000-0000-000000000301",
incident_id="INC-DIRECT-BLOCK",
requested_by="auto_approve_rule_engine",
)
with pytest.raises(
RuntimeError,
match="auto_approved_direct_execution_disabled_use_single_writer_broker",
):
# Replicate the exact condition logic from webhooks.py CS1 block
_non_destructive_actions = {"NO_ACTION", "INVESTIGATE", "OBSERVE"}
_sa_val = (
analysis_result.suggested_action.value
if analysis_result and hasattr(analysis_result.suggested_action, "value")
else str(getattr(analysis_result, "suggested_action", ""))
)
if analysis_result:
_cs1_kubectl = analysis_result.kubectl_command.strip() if analysis_result.kubectl_command else ""
_cs1_can_auto = (
bool(_cs1_kubectl)
and analysis_result.confidence >= 0.85
and risk_level != RiskLevel.CRITICAL
and _sa_val not in _non_destructive_actions
and is_safe_kubectl_action(_cs1_kubectl)
)
if _cs1_can_auto:
import asyncio
from src.models.approval import ApprovalRequest, ApprovalStatus
from src.services.approval_execution import ApprovalExecutionService
_cs1_auto_approval = MagicMock()
_cs1_executor = ApprovalExecutionService()
asyncio.run(_cs1_executor.execute_approved_action(_cs1_auto_approval))
return mock_executor_cls, mock_exec_instance
# =============================================================================
# Tests
# =============================================================================
class TestCS1AutoExecuteConditions:
"""測試 CS1 自動執行的觸發條件"""
def test_high_confidence_low_risk_executes(self):
"""confidence=0.90 + LOW risk + kubectl 有值 → execute 被呼叫"""
analysis = _make_analysis(confidence=0.90)
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
mock_exec.execute_approved_action.assert_called_once()
def test_low_confidence_does_not_execute(self):
"""confidence=0.70 → 不執行"""
analysis = _make_analysis(confidence=0.70)
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
mock_exec.execute_approved_action.assert_not_called()
def test_boundary_confidence_085_executes(self):
"""confidence=0.85 剛好等於門檻 → 執行"""
analysis = _make_analysis(confidence=0.85)
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
mock_exec.execute_approved_action.assert_called_once()
def test_critical_risk_does_not_execute(self):
"""confidence=0.90 + CRITICAL → 不執行"""
analysis = _make_analysis(confidence=0.90, risk_level_str="critical")
_, mock_exec = _run_cs1_block(analysis, RiskLevel.CRITICAL)
mock_exec.execute_approved_action.assert_not_called()
def test_destructive_pattern_does_not_execute(self):
"""kubectl 含 destructive pattern → 不執行"""
from src.services.auto_approve import _DESTRUCTIVE_PATTERNS
# 取第一個 pattern 構造一個含危險詞的指令
bad_pattern = _DESTRUCTIVE_PATTERNS[0]
analysis = _make_analysis(
confidence=0.90,
kubectl_command=f"kubectl {bad_pattern} deployment/api -n prod",
)
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
mock_exec.execute_approved_action.assert_not_called()
def test_single_delete_pod_executes(self):
"""單一 Pod delete 是可恢復操作parser 不應誤殺"""
analysis = _make_analysis(
confidence=0.90,
kubectl_command="kubectl delete pod api-xxx-yyy -n prod",
)
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
mock_exec.execute_approved_action.assert_called_once()
def test_no_action_does_not_execute(self):
"""suggested_action=NO_ACTION → 不執行"""
analysis = _make_analysis(
confidence=0.90,
suggested_action=SuggestedAction.NO_ACTION,
)
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
mock_exec.execute_approved_action.assert_not_called()
def test_investigate_does_not_execute(self):
"""suggested_action=INVESTIGATE → 不執行"""
analysis = _make_analysis(
confidence=0.90,
suggested_action=SuggestedAction.INVESTIGATE,
)
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
mock_exec.execute_approved_action.assert_not_called()
class TestCS1AutoExecuteFailureDegradation:
"""測試執行失敗時的降級行為"""
def test_execution_exception_does_not_crash(self):
"""execute_approved_action 拋 exception → 捕捉後繼續,不 crash"""
analysis = _make_analysis(confidence=0.90)
# 直接測試條件邏輯,確保例外被吞掉
from src.services.action_parser import is_safe_kubectl_action
_non_destructive_actions = {"NO_ACTION", "INVESTIGATE", "OBSERVE"}
_sa_val = analysis.suggested_action.value
_cs1_kubectl = analysis.kubectl_command.strip()
_cs1_can_auto = (
bool(_cs1_kubectl)
and analysis.confidence >= 0.85
and RiskLevel.LOW != RiskLevel.CRITICAL
and _sa_val not in _non_destructive_actions
and is_safe_kubectl_action(_cs1_kubectl)
)
assert _cs1_can_auto, "前置條件必須為 True 才能測試降級"
raised = False
try:
import asyncio
async def _simulate():
# 模擬整個 if _cs1_can_auto: try/except 區塊
if _cs1_can_auto:
try:
raise RuntimeError("executor exploded")
except Exception:
pass # 降級:維持 PENDING
asyncio.run(_simulate())
except Exception:
raised = True
assert not raised, "例外應被 CS1 try/except 吞掉,不應傳播"
def test_empty_kubectl_does_not_execute(self):
"""kubectl_command 為空字串 → 不執行"""
analysis = _make_analysis(confidence=0.90, kubectl_command="")
_, mock_exec = _run_cs1_block(analysis, RiskLevel.LOW)
mock_exec.execute_approved_action.assert_not_called()
await ApprovalExecutionService().execute_approved_action(approval)

View File

@@ -1,127 +1,69 @@
# apps/api/tests/test_cs3_auto_execute.py
# 2026-04-27 ogt + Claude Sonnet 4.6 — CS3 alertmanager AI path 高信心自動執行單元測試
"""
測試覆蓋:
1. confidence=0.90 + MEDIUM risk + kubectl 有值 → can_auto=True
2. confidence=0.70 → blocked
3. CRITICAL risk → blocked
4. kubectl="" → blocked
5. NO_ACTION title → blocked
6. destructive kubectl (delete) → blocked
7. destructive --force pattern → isinstance check
8. execute_approved_action 被呼叫
9. execute 拋例外不向上傳播
"""
"""Alertmanager high-confidence routing remains guarded and queue-only."""
from __future__ import annotations
import inspect
from types import SimpleNamespace
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from src.api.v1 import webhooks
from src.models.approval import RiskLevel
def _make_analysis(
confidence: float = 0.9,
def _analysis(
*,
confidence: float = 0.90,
action_title: str = "restart pod",
kubectl: str = "kubectl rollout restart deployment/foo",
):
a = MagicMock()
a.confidence = confidence
a.action_title = action_title
a.kubectl_command = kubectl
a.description = "test desc"
a.affected_services = []
a.primary_responsibility = "COLLAB"
return a
def _can_auto(analysis, risk_level, patterns):
from src.models.approval import RiskLevel
from src.services.action_parser import is_safe_kubectl_action
kubectl = (analysis.kubectl_command or "").strip()
return (
bool(kubectl)
and analysis.confidence >= 0.85
and risk_level != RiskLevel.CRITICAL
and "NO_ACTION" not in (analysis.action_title or "")
and is_safe_kubectl_action(kubectl)
) -> SimpleNamespace:
return SimpleNamespace(
confidence=confidence,
action_title=action_title,
kubectl_command=kubectl,
)
@pytest.fixture(scope="module")
def patterns():
from src.services.auto_approve import _DESTRUCTIVE_PATTERNS
return _DESTRUCTIVE_PATTERNS
def _can_route(analysis: SimpleNamespace, risk_level: RiskLevel) -> bool:
from src.services.action_parser import is_safe_kubectl_action
command = str(analysis.kubectl_command or "").strip()
return (
bool(command)
and analysis.confidence >= 0.85
and risk_level != RiskLevel.CRITICAL
and "NO_ACTION" not in str(analysis.action_title or "")
and is_safe_kubectl_action(command)
)
class TestCS3AutoExecute:
@pytest.mark.parametrize(
("analysis", "risk_level", "expected"),
[
(_analysis(), RiskLevel.MEDIUM, True),
(_analysis(confidence=0.70), RiskLevel.MEDIUM, False),
(_analysis(), RiskLevel.CRITICAL, False),
(_analysis(kubectl=""), RiskLevel.MEDIUM, False),
(_analysis(action_title="NO_ACTION: no fix"), RiskLevel.MEDIUM, False),
(
_analysis(kubectl="kubectl delete pods --all -n prod"),
RiskLevel.MEDIUM,
False,
),
],
)
def test_cs3_controlled_queue_eligibility(
analysis: SimpleNamespace,
risk_level: RiskLevel,
expected: bool,
) -> None:
assert _can_route(analysis, risk_level) is expected
def test_high_confidence_eligible(self, patterns):
from src.models.approval import RiskLevel
a = _make_analysis(confidence=0.9)
assert _can_auto(a, RiskLevel.MEDIUM, patterns) is True
def test_low_confidence_blocked(self, patterns):
from src.models.approval import RiskLevel
a = _make_analysis(confidence=0.7)
assert _can_auto(a, RiskLevel.MEDIUM, patterns) is False
def test_alertmanager_source_uses_queue_not_approval_executor() -> None:
source = inspect.getsource(webhooks._process_new_alert_background)
def test_critical_risk_blocked(self, patterns):
from src.models.approval import RiskLevel
a = _make_analysis(confidence=0.95)
assert _can_auto(a, RiskLevel.CRITICAL, patterns) is False
def test_empty_kubectl_blocked(self, patterns):
from src.models.approval import RiskLevel
a = _make_analysis(kubectl="")
assert _can_auto(a, RiskLevel.MEDIUM, patterns) is False
def test_no_action_blocked(self, patterns):
from src.models.approval import RiskLevel
a = _make_analysis(action_title="NO_ACTION: no fix needed")
assert _can_auto(a, RiskLevel.MEDIUM, patterns) is False
def test_single_delete_pod_eligible(self, patterns):
from src.models.approval import RiskLevel
a = _make_analysis(kubectl="kubectl delete pod foo-123")
assert _can_auto(a, RiskLevel.MEDIUM, patterns) is True
def test_delete_pods_all_blocked(self, patterns):
from src.models.approval import RiskLevel
a = _make_analysis(kubectl="kubectl delete pods --all -n prod")
assert _can_auto(a, RiskLevel.MEDIUM, patterns) is False
def test_destructive_force_check(self, patterns):
# --force 不一定在 pattern只驗 _can_auto 回傳 bool
from src.models.approval import RiskLevel
a = _make_analysis(kubectl="kubectl rollout restart --force deployment/bar")
result = _can_auto(a, RiskLevel.MEDIUM, patterns)
assert isinstance(result, bool)
@pytest.mark.asyncio
async def test_execute_called_when_eligible(self, patterns):
from src.models.approval import RiskLevel
a = _make_analysis(confidence=0.9)
risk_level = RiskLevel.MEDIUM
mock_svc = AsyncMock()
mock_svc.execute_approved_action = AsyncMock(return_value=True)
assert _can_auto(a, risk_level, patterns) is True
await mock_svc.execute_approved_action(MagicMock())
mock_svc.execute_approved_action.assert_called_once()
@pytest.mark.asyncio
async def test_execute_exception_does_not_propagate(self, patterns):
from src.models.approval import RiskLevel
a = _make_analysis(confidence=0.9)
risk_level = RiskLevel.MEDIUM
mock_svc = AsyncMock()
mock_svc.execute_approved_action = AsyncMock(side_effect=RuntimeError("boom"))
try:
if _can_auto(a, risk_level, patterns):
await mock_svc.execute_approved_action(MagicMock())
except Exception:
pass # prod code wraps in try/except; test confirms pattern
assert True
assert "_try_auto_repair_background" in source
assert "execute_approved_action" not in source
assert "ApprovalExecutionService" not in source
assert "execute_auto_repair(" not in source

View File

@@ -13,8 +13,9 @@ LLM 亂提「kubectl rollout restart awoooi-api」要被擋下。
from __future__ import annotations
import inspect
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import AsyncMock, patch
import pytest
@@ -59,6 +60,21 @@ def manager(monkeypatch):
"src.services.decision_manager._escalate_decision_auto_repair_unavailable",
lambda **k: None,
)
async def _queue_candidate(**_kwargs):
return {
"schema_version": "ai_decision_controlled_executor_handoff_v1",
"status": "controlled_check_mode_queued",
"automation_run_id": "00000000-0000-0000-0000-000000000101",
"queued": True,
"side_effect_performed": False,
"single_writer_executor": "awoooi-ansible-executor-broker",
"active_blockers": [],
}
monkeypatch.setattr(
"src.jobs.awooop_ansible_candidate_backfill_job.enqueue_ai_decision_ansible_candidate",
_queue_candidate,
)
return mgr
@@ -146,3 +162,27 @@ class TestBareMetalKubectlGuard:
pass
br = token.proposal_data.get("blocked_reason", "")
assert "host_type=bare_metal" not in br
@pytest.mark.asyncio
async def test_critical_decision_remains_break_glass(self, manager):
incident = _fake_incident(host_type="kubernetes")
token = _fake_token("kubectl rollout restart deployment awoooi-api")
token.proposal_data["risk_level"] = "critical"
await manager._auto_execute(incident, token)
assert token.state == DecisionState.READY
assert token.proposal_data["blocked_reason"] == "critical_break_glass_required"
assert token.proposal_data["direct_runtime_execution_performed"] is False
def test_active_auto_execute_only_routes_to_single_writer_queue() -> None:
source = inspect.getsource(DecisionManager._auto_execute)
legacy_source = inspect.getsource(DecisionManager._legacy_auto_execute_disabled)
assert "enqueue_ai_decision_ansible_candidate" in source
assert "ApprovalExecutionService" not in source
assert "execute_approved_action" not in source
assert "_ssh_execute" not in source
assert "critical_break_glass_required" in source
assert "decision_manager_direct_execution_superseded" in legacy_source

View File

@@ -25,6 +25,16 @@ def _verified_receipt(source_sha: str = "abc123") -> dict[str, str]:
"worker_ssh_egress_denied": "true",
"broker_ssh_egress_allowlisted": "true",
"legacy_namespace_egress_allow_absent": "true",
"single_writer_source_verifier": (
"cd_tests_and_production_source_sha_v1"
),
"decision_manager_queue_only": "true",
"webhook_router_queue_only": "true",
"failure_watcher_queue_only": "true",
"auto_approved_direct_execution_blocked": "true",
"candidate_idempotency_contract_verified": "true",
"apply_idempotency_contract_verified": "true",
"critical_break_glass_contract_verified": "true",
"db_identity_verifier": (
"runtime_role_fingerprint_secret_projection_and_"
"representative_preflight_v2"
@@ -92,6 +102,27 @@ def test_executor_trust_boundary_requires_live_controls_and_matching_source() ->
] is True
assert database_boundary["secret_values_read"] is False
assert database_boundary["role_names_exposed"] is False
source_boundary = readback["autonomous_single_writer_source_boundary"]
assert source_boundary["source_boundary_verified"] is True
assert source_boundary["active_blockers"] == []
assert all(source_boundary["controls"].values())
def test_executor_trust_boundary_single_writer_source_controls_fail_closed() -> None:
receipt = _verified_receipt()
receipt["failure_watcher_queue_only"] = "false"
readback = build_executor_trust_boundary_readback(
receipt,
deployed_source_sha="abc123",
)
source_boundary = readback["autonomous_single_writer_source_boundary"]
assert readback["production_boundary_verified"] is True
assert source_boundary["source_boundary_verified"] is False
assert "failure_watcher_queue_only_not_verified" in source_boundary[
"active_blockers"
]
def test_executor_trust_boundary_rejects_stale_or_incomplete_receipt() -> None:

View File

@@ -8,6 +8,8 @@ FailureWatcher Service Tests - Phase 18 失敗自動修復閉環
建立者: Claude Code (Phase 18.6 E2E 驗證)
"""
import inspect
import pytest
from src.services.failure_watcher import (
@@ -22,6 +24,34 @@ class TestFailureClassification:
def setup_method(self):
self.service = FailureWatcherService()
def test_active_failure_watcher_has_no_recursive_direct_executor(self):
process_source = inspect.getsource(self.service.process_failure)
legacy_source = inspect.getsource(
self.service._legacy_execute_auto_repair_disabled
)
assert "self.execute_auto_repair(" not in process_source
assert "single_writer_retry_or_repair_queued" in process_source
assert "get_executor" not in inspect.getsource(
self.service.execute_auto_repair
)
assert "failure_watcher_direct_execution_disabled" in inspect.getsource(
self.service.execute_auto_repair
)
assert "restart_deployment" in legacy_source
@pytest.mark.asyncio
async def test_legacy_direct_repair_entrypoint_is_fail_closed(self):
with pytest.raises(
RuntimeError,
match="failure_watcher_direct_execution_disabled_use_single_writer_broker",
):
await self.service.execute_auto_repair(
audit_log_id="audit-1",
repair_strategy="restart_deployment",
failure_data={"target_resource": "deployment/api"},
)
def test_classify_timeout(self):
"""測試超時錯誤分類"""
result = self.service._classify_by_rules("Connection timed out after 30s")

View File

@@ -156,9 +156,19 @@ def test_cd_prunes_and_verifies_api_executor_boundary_in_production() -> None:
assert "legacy cluster-wide executor binding still exists" in workflow
assert "awoooi-executor-boundary-verification" in workflow
assert "awoooi_executor_boundary_verification_v3" in workflow
assert "single_writer_source_verifier=cd_tests_and_production_source_sha_v1" in workflow
assert "decision_manager_queue_only=true" in workflow
assert "webhook_router_queue_only=true" in workflow
assert "failure_watcher_queue_only=true" in workflow
assert "auto_approved_direct_execution_blocked=true" in workflow
assert "candidate_idempotency_contract_verified=true" in workflow
assert "apply_idempotency_contract_verified=true" in workflow
assert "critical_break_glass_contract_verified=true" in workflow
assert "executor_boundary_public_readback=verified_ready" in workflow
assert 'boundary.get("full_workload_boundary_verified") is True' in workflow
assert 'database_boundary.get("db_identity_isolated") is True' in workflow
assert "autonomous_single_writer_source_boundary" in workflow
assert '"source_boundary_verified"' in workflow
assert 'database_boundary.get("connection_budget_verified") is True' in workflow
assert "verified_source_sha" in workflow
assert (