fix(api): 產生 MCP PlayBook 修復候選
Some checks failed
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
Code Review / ai-code-review (push) Has been cancelled

This commit is contained in:
Your Name
2026-06-11 15:42:11 +08:00
parent ab1ee29638
commit cc6140230d
4 changed files with 791 additions and 84 deletions

View File

@@ -81,6 +81,7 @@ from src.services.incident_service import (
# Phase 5: OpenClaw AI Engine
from src.services.openclaw import get_openclaw
from src.services.playbook_match_resolver import resolve_playbook_id_for_alert
from src.services.repair_candidate_service import get_repair_candidate_service
from src.services.security_interceptor import check_webhook_nonce # P0-06: nonce dedup via Service 層
from src.services.signal_producer import SignalData, get_signal_producer
@@ -2239,64 +2240,18 @@ async def _process_new_alert_background(
record_alert_chain_success("alertmanager")
else:
# LLM 失敗 - 使用預設值
# 2026-04-27 Claude Sonnet 4.6: shadow-run Step1 — 補 metadata kwarg讓 extra_metadata 可觀測
# LLM 失敗時,不再把 NO_ACTION 當成終點。
# 先用預配置 approval id 建立 incident讓後續 MCP evidence、
# PlayBook trust、approval 與 Telegram 都指向同一條真相鏈。
preallocated_approval_id = str(uuid.uuid4())
_matched_playbook_id_cs4 = await resolve_playbook_id_for_alert(
rule_id=str(rule_response.get("rule_id", "")),
alertname=alertname,
affected_services=[target_resource] if target_resource else [],
severity="medium",
)
_approval_metadata_cs4 = {
"source": "fallback",
"confidence_score": None,
"is_rule_based": False,
"playbook_id": _matched_playbook_id_cs4,
}
fallback_create = ApprovalRequestCreate(
action="NO_ACTION - REPAIR_CANDIDATE_MISSING: LLM 分析失敗,尚未產生可安全執行的修復指令",
description=f"[LLM Failed] {message}",
risk_level=RiskLevel.LOW,
blast_radius=BlastRadius(
affected_pods=1,
estimated_downtime="unknown",
related_services=[],
data_impact=DataImpact.NONE,
),
dry_run_checks=[],
requested_by="OpenClaw (fallback)",
metadata=_approval_metadata_cs4,
matched_playbook_id=_matched_playbook_id_cs4,
)
approval = await service.create_approval_with_fingerprint(
request=fallback_create,
fingerprint=fingerprint,
)
# 2026-04-27 Claude Sonnet 4.6: shadow-run Step2 — 只記 log不改執行決策
try:
_shadow_proposal_cs4 = {
"risk_level": "low",
"confidence": 0.0,
"action": fallback_create.action,
"kubectl_command": "",
"is_rule_based": False,
"source": "fallback",
}
_shadow_result_cs4 = get_auto_approve_policy().evaluate(_shadow_proposal_cs4)
logger.info(
"shadow_auto_approve_result",
approval_id=str(approval.id),
should_auto=_shadow_result_cs4.should_auto_approve,
reason=_shadow_result_cs4.reason.value,
source="fallback",
)
except Exception as _shadow_err_cs4:
logger.warning("shadow_auto_approve_failed", error=str(_shadow_err_cs4))
fallback_incident_id = await create_incident_for_approval(
approval_id=str(approval.id),
approval_id=preallocated_approval_id,
risk_level="medium",
target_resource=target_resource,
namespace=namespace,
@@ -2309,6 +2264,106 @@ async def _process_new_alert_background(
alert_category=alert_category,
)
fallback_action_text = (
"NO_ACTION - REPAIR_CANDIDATE_MISSING: "
"LLM 分析失敗MCP evidence / PlayBook trust 尚未產生可安全執行的修復指令"
)
repair_candidate_result = await get_repair_candidate_service().build_from_incident_id(
incident_id=fallback_incident_id,
alertname=alertname,
target_resource=target_resource,
namespace=namespace,
message=message,
fallback_action=fallback_action_text,
matched_playbook_id=_matched_playbook_id_cs4,
rule_id=str(rule_response.get("rule_id", "")),
severity="medium",
)
_approval_metadata_cs4 = {
"source": "llm_fallback_mcp_playbook_candidate",
"confidence_score": None,
"is_rule_based": False,
"playbook_id": _matched_playbook_id_cs4,
"preallocated_approval_id": preallocated_approval_id,
}
_approval_metadata_cs4.update(repair_candidate_result.metadata)
_approval_metadata_cs4["preallocated_approval_id"] = preallocated_approval_id
candidate_confidence = 0.0
if repair_candidate_result.candidate_found and repair_candidate_result.approval_request:
evidence = repair_candidate_result.evidence
playbook = repair_candidate_result.playbook
evidence_ratio = 0.0
if evidence and evidence.sensors_attempted:
evidence_ratio = evidence.sensors_succeeded / max(evidence.sensors_attempted, 1)
trust_score = float(playbook.trust_score) if playbook else 0.0
candidate_confidence = min(0.82, 0.45 + evidence_ratio * 0.2 + trust_score * 0.2)
fallback_create = repair_candidate_result.approval_request.model_copy(
update={
"incident_id": fallback_incident_id,
"metadata": _approval_metadata_cs4,
}
)
telegram_root_cause = (
"LLM fallback 後已由 MCP evidence + PlayBook trust 產生修復候選;"
"等待人工批准後進入 execution / verifier / KM 回寫。"
)
primary_responsibility = "OPENCLAW_PLAYBOOK"
else:
blockers = repair_candidate_result.blockers or ["repair_candidate_missing"]
blocker_text = ", ".join(blockers)
fallback_create = ApprovalRequestCreate(
action=f"NO_ACTION - REPAIR_CANDIDATE_MISSING: {blocker_text}",
description=f"[LLM Failed] {message}\n修復候選阻擋:{blocker_text}",
risk_level=RiskLevel.LOW,
blast_radius=BlastRadius(
affected_pods=1,
estimated_downtime="unknown",
related_services=[target_resource] if target_resource else [],
data_impact=DataImpact.NONE,
),
dry_run_checks=[
DryRunCheck(
name="MCP/PlayBook candidate gate",
passed=False,
message=blocker_text[:240],
)
],
requested_by="OpenClaw (fallback candidate gate)",
incident_id=fallback_incident_id,
metadata=_approval_metadata_cs4,
matched_playbook_id=_matched_playbook_id_cs4,
)
telegram_root_cause = f"LLM fallback 後未產生修復候選;阻擋:{blocker_text}"
primary_responsibility = "HUMAN"
approval = await service.create_approval_with_fingerprint(
request=fallback_create,
fingerprint=fingerprint,
)
# 2026-04-27 Claude Sonnet 4.6: shadow-run Step2 — 只記 log不改執行決策
try:
_shadow_proposal_cs4 = {
"risk_level": fallback_create.risk_level.value,
"confidence": candidate_confidence,
"action": fallback_create.action,
"kubectl_command": fallback_create.action if fallback_create.action.startswith("kubectl") else "",
"is_rule_based": False,
"source": _approval_metadata_cs4.get("source", "fallback"),
}
_shadow_result_cs4 = get_auto_approve_policy().evaluate(_shadow_proposal_cs4)
logger.info(
"shadow_auto_approve_result",
approval_id=str(approval.id),
should_auto=_shadow_result_cs4.should_auto_approve,
reason=_shadow_result_cs4.reason.value,
source="fallback_candidate",
)
except Exception as _shadow_err_cs4:
logger.warning("shadow_auto_approve_failed", error=str(_shadow_err_cs4))
try:
await service.update_incident_id(approval.id, fallback_incident_id)
approval.incident_id = fallback_incident_id
@@ -2339,46 +2394,61 @@ async def _process_new_alert_background(
)
_is_heartbeat = is_heartbeat_alertname(alertname)
if can_auto_repair and not _is_heartbeat:
await _try_auto_repair_background(
incident_id=fallback_incident_id,
approval_id=str(approval.id),
alert_type=alert_type,
target_resource=target_resource,
namespace=namespace,
)
elif not can_auto_repair and not _is_heartbeat:
if not _is_heartbeat:
from src.repositories.alert_operation_log_repository import get_alert_operation_log_repository
_op_log_fallback = get_alert_operation_log_repository()
await _op_log_fallback.append(
"GUARDRAIL_BLOCKED",
incident_id=fallback_incident_id,
approval_id=str(approval.id),
actor="prometheus-rule",
action_detail=f"Prometheus rule 設定 auto_repair=falsefallback 轉人工: {alertname}",
success=False,
context={"alertname": alertname, "auto_repair_flag": False},
)
await _escalate_auto_repair_unavailable(
incident_id=fallback_incident_id,
approval_id=str(approval.id),
alert_type=alert_type,
target_resource=target_resource,
namespace=namespace,
failure_reason="Prometheus rule auto_repair=falsefallback 未進入自動修復評估",
attempted_actions="llm_fallback -> guardrail:auto_repair_false -> emergency_intervention",
)
if repair_candidate_result.candidate_found:
await _op_log_fallback.append(
"REPAIR_CANDIDATE_READY",
incident_id=fallback_incident_id,
approval_id=str(approval.id),
actor="openclaw-repair-candidate",
action_detail=f"MCP evidence + PlayBook trust 產生候選,等待批准: {fallback_create.action[:220]}",
success=True,
context={
"alertname": alertname,
"auto_repair_flag": bool(can_auto_repair),
"playbook_id": fallback_create.matched_playbook_id,
"candidate_status": "ready_for_approval",
},
)
else:
await _op_log_fallback.append(
"REPAIR_CANDIDATE_BLOCKED",
incident_id=fallback_incident_id,
approval_id=str(approval.id),
actor="openclaw-repair-candidate",
action_detail=f"fallback 未產生候選: {fallback_create.action[:220]}",
success=False,
context={
"alertname": alertname,
"auto_repair_flag": bool(can_auto_repair),
"blockers": repair_candidate_result.blockers,
},
)
await _escalate_auto_repair_unavailable(
incident_id=fallback_incident_id,
approval_id=str(approval.id),
alert_type=alert_type,
target_resource=target_resource,
namespace=namespace,
failure_reason=telegram_root_cause,
attempted_actions=(
"llm_fallback -> mcp_evidence -> playbook_trust -> "
f"candidate_blocked:{','.join(repair_candidate_result.blockers or ['unknown'])}"
),
)
await _push_to_telegram_background(
approval_id=str(approval.id),
risk_level="low",
risk_level=fallback_create.risk_level.value,
resource_name=target_resource,
root_cause=message,
root_cause=telegram_root_cause,
suggested_action=fallback_create.action,
estimated_downtime="unknown",
hit_count=1,
primary_responsibility="HUMAN",
confidence=0.0,
primary_responsibility=primary_responsibility,
confidence=candidate_confidence,
namespace=namespace,
incident_id=fallback_incident_id,
notification_type=notification_type,

View File

@@ -130,6 +130,11 @@ def approval_request_to_record_data(
"""
Convert ApprovalRequestCreate to dict for ApprovalRecord creation
"""
metadata = dict(request.metadata or {})
preallocated_approval_id = str(metadata.pop("preallocated_approval_id", "") or "").strip()
if preallocated_approval_id:
UUID(preallocated_approval_id)
blast_radius_dict = None
if request.blast_radius:
blast_radius_dict = {
@@ -151,7 +156,7 @@ def approval_request_to_record_data(
})
now = datetime.now(UTC)
return {
record_data = {
"action": request.action,
"description": request.description,
"status": ApprovalStatus.APPROVED if risk_level == RiskLevel.LOW else ApprovalStatus.PENDING,
@@ -163,7 +168,7 @@ def approval_request_to_record_data(
"dry_run_checks": dry_run_checks_list,
"requested_by": request.requested_by,
"expires_at": request.expires_at,
"extra_metadata": request.metadata,
"extra_metadata": metadata or None,
"resolved_at": now if risk_level == RiskLevel.LOW else None,
# 戰略 B: 告警風暴收斂
"fingerprint": fingerprint,
@@ -177,6 +182,9 @@ def approval_request_to_record_data(
# 原本缺失 → Playbook 人工審核後 trust score 永遠不更新(學習閉環斷鏈)
"matched_playbook_id": getattr(request, "matched_playbook_id", None),
}
if preallocated_approval_id:
record_data["id"] = preallocated_approval_id
return record_data
def _record_value(value: Any) -> str:

View File

@@ -0,0 +1,427 @@
"""Repair candidate generation from MCP evidence and PlayBook trust.
This service is intentionally candidate-only. It never executes a repair; it
builds an ApprovalRequestCreate only when evidence, PlayBook status, trust and
command safety gates are all explicit enough for a human approval flow.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
import structlog
from src.models.approval import (
ApprovalRequestCreate,
BlastRadius,
DataImpact,
DryRunCheck,
)
from src.models.approval import RiskLevel as ApprovalRiskLevel
from src.models.incident import Incident
from src.models.playbook import (
ActionType,
Playbook,
PlaybookStatus,
)
from src.models.playbook import RiskLevel as PlaybookRiskLevel
from src.repositories.playbook_repository import get_playbook_repository
from src.services.action_parser import ActionKind, parse_kubectl_action
from src.services.auto_repair_service import AutoRepairService
from src.services.evidence_snapshot import EvidenceSnapshot
from src.services.incident_service import get_incident_service
from src.services.playbook_match_resolver import resolve_playbook_id_for_alert
from src.services.pre_decision_investigator import get_pre_decision_investigator
logger = structlog.get_logger(__name__)
MIN_REPAIR_CANDIDATE_TRUST = 0.30
@dataclass
class RepairCandidateResult:
"""Candidate generation result used by webhook and Telegram handoff paths."""
approval_request: ApprovalRequestCreate | None = None
evidence: EvidenceSnapshot | None = None
playbook: Playbook | None = None
blockers: list[str] = field(default_factory=list)
metadata: dict[str, Any] = field(default_factory=dict)
@property
def candidate_found(self) -> bool:
return self.approval_request is not None
class RepairCandidateService:
"""Build a repair approval candidate without executing it."""
def __init__(
self,
*,
incident_service: Any | None = None,
investigator: Any | None = None,
playbook_repository: Any | None = None,
auto_repair_service: AutoRepairService | None = None,
) -> None:
self._incident_service = incident_service or get_incident_service()
self._investigator = investigator or get_pre_decision_investigator()
self._playbook_repository = playbook_repository or get_playbook_repository()
self._auto_repair = auto_repair_service or AutoRepairService()
async def build_from_incident_id(
self,
*,
incident_id: str,
alertname: str,
target_resource: str,
namespace: str,
message: str,
fallback_action: str,
matched_playbook_id: str | None = None,
rule_id: str | None = None,
severity: str | None = None,
) -> RepairCandidateResult:
"""Load an incident and produce a repair candidate or explicit blockers."""
incident = await self._incident_service.get_from_working_memory(incident_id)
if incident is None:
return RepairCandidateResult(
blockers=["incident_not_found"],
metadata={"repair_candidate_status": "blocked", "blockers": ["incident_not_found"]},
)
return await self.build_from_incident(
incident=incident,
alertname=alertname,
target_resource=target_resource,
namespace=namespace,
message=message,
fallback_action=fallback_action,
matched_playbook_id=matched_playbook_id,
rule_id=rule_id,
severity=severity,
)
async def build_from_incident(
self,
*,
incident: Incident,
alertname: str,
target_resource: str,
namespace: str,
message: str,
fallback_action: str,
matched_playbook_id: str | None = None,
rule_id: str | None = None,
severity: str | None = None,
) -> RepairCandidateResult:
evidence = await self._collect_evidence(incident)
metadata = self._base_metadata(evidence=evidence)
blockers: list[str] = []
if not evidence or evidence.sensors_succeeded <= 0:
blockers.append("mcp_evidence_missing")
playbook_id = matched_playbook_id or await resolve_playbook_id_for_alert(
rule_id=rule_id,
alertname=alertname,
affected_services=[target_resource] if target_resource else [],
severity=severity,
)
if not playbook_id:
blockers.append("playbook_not_matched")
return self._blocked_result(
blockers=blockers,
metadata=metadata,
evidence=evidence,
fallback_action=fallback_action,
)
playbook = await self._playbook_repository.get_by_id(playbook_id)
if not playbook:
blockers.append("playbook_not_found")
return self._blocked_result(
blockers=blockers,
metadata=metadata,
evidence=evidence,
fallback_action=fallback_action,
)
metadata["playbook_trust"] = {
"playbook_id": playbook.playbook_id,
"name": playbook.name,
"status": playbook.status.value,
"trust_score": float(playbook.trust_score),
}
if playbook.status != PlaybookStatus.APPROVED:
blockers.append("playbook_not_approved")
if float(playbook.trust_score) < MIN_REPAIR_CANDIDATE_TRUST:
blockers.append("playbook_trust_below_gate")
step, step_blockers = self._select_executable_step(incident, playbook)
blockers.extend(step_blockers)
if blockers or step is None:
return self._blocked_result(
blockers=blockers,
metadata=metadata,
evidence=evidence,
playbook=playbook,
fallback_action=fallback_action,
)
metadata["repair_candidate"] = {
"source": "mcp_evidence_playbook_trust",
"playbook_id": playbook.playbook_id,
"playbook_name": playbook.name,
"step_number": step.step_number,
"command": step.command,
"action_type": step.action_type.value,
"risk_level": step.risk_level.value,
"expected_result": step.expected_result,
"rollback_command": step.rollback_command,
"requires_approval": True,
}
metadata["verifier_plan"] = self._build_verifier_plan(
command=step.command,
namespace=namespace,
target_resource=target_resource,
evidence=evidence,
)
metadata["repair_candidate_status"] = "candidate_ready_for_approval"
metadata["fallback_replaced"] = True
approval_request = ApprovalRequestCreate(
action=step.command,
description=self._build_description(
message=message,
playbook=playbook,
step_command=step.command,
evidence=evidence,
verifier_plan=metadata["verifier_plan"],
),
risk_level=self._approval_risk(step.risk_level),
blast_radius=BlastRadius(
affected_pods=1,
estimated_downtime=f"{playbook.estimated_duration_minutes} min",
related_services=[target_resource] if target_resource else [],
data_impact=DataImpact.WRITE,
),
dry_run_checks=self._build_dry_run_checks(
evidence=evidence,
playbook=playbook,
command=step.command,
),
requested_by="OpenClaw (MCP evidence + PlayBook trust)",
incident_id=incident.incident_id,
metadata=metadata,
matched_playbook_id=playbook.playbook_id,
)
logger.info(
"repair_candidate_generated",
incident_id=incident.incident_id,
playbook_id=playbook.playbook_id,
sensors_succeeded=evidence.sensors_succeeded if evidence else 0,
command=step.command[:160],
)
return RepairCandidateResult(
approval_request=approval_request,
evidence=evidence,
playbook=playbook,
metadata=metadata,
)
async def _collect_evidence(self, incident: Incident) -> EvidenceSnapshot | None:
try:
return await self._investigator.investigate(incident)
except Exception as exc:
logger.warning(
"repair_candidate_evidence_collect_failed",
incident_id=incident.incident_id,
error=str(exc),
)
return None
def _select_executable_step(
self,
incident: Incident,
playbook: Playbook,
) -> tuple[Any | None, list[str]]:
blockers: list[str] = []
if not playbook.repair_steps:
return None, ["playbook_has_no_repair_steps"]
rejected_readonly = False
rejected_unsafe = False
for step in playbook.repair_steps:
command = (step.command or "").strip()
if not command:
continue
if step.action_type == ActionType.MANUAL:
rejected_readonly = True
continue
if step.action_type == ActionType.KUBECTL or command.startswith("kubectl"):
parsed = parse_kubectl_action(command)
if not parsed.ok:
rejected_unsafe = True
continue
if parsed.kind == ActionKind.READONLY:
rejected_readonly = True
continue
return step, []
if step.action_type == ActionType.SSH_COMMAND or command.startswith("ssh "):
if self._auto_repair.preview_write_ssh_mcp_route(incident, command):
return step, []
rejected_unsafe = True
if rejected_unsafe:
blockers.append("playbook_command_not_safely_routable")
if rejected_readonly:
blockers.append("playbook_observe_only")
if not blockers:
blockers.append("playbook_has_no_executable_step")
return None, blockers
def _base_metadata(self, *, evidence: EvidenceSnapshot | None) -> dict[str, Any]:
if not evidence:
return {
"source": "mcp_playbook_repair_candidate",
"mcp_evidence": {"status": "missing"},
}
return {
"source": "mcp_playbook_repair_candidate",
"mcp_evidence": {
"status": "collected" if evidence.sensors_succeeded > 0 else "degraded",
"snapshot_id": evidence.snapshot_id,
"sensors_attempted": evidence.sensors_attempted,
"sensors_succeeded": evidence.sensors_succeeded,
"mcp_health": evidence.mcp_health,
"summary_excerpt": (evidence.evidence_summary or "")[:800],
},
}
def _blocked_result(
self,
*,
blockers: list[str],
metadata: dict[str, Any],
fallback_action: str,
evidence: EvidenceSnapshot | None = None,
playbook: Playbook | None = None,
) -> RepairCandidateResult:
metadata["repair_candidate_status"] = "blocked"
metadata["repair_candidate_blockers"] = list(dict.fromkeys(blockers))
metadata["fallback_action"] = fallback_action
return RepairCandidateResult(
evidence=evidence,
playbook=playbook,
blockers=metadata["repair_candidate_blockers"],
metadata=metadata,
)
def _build_description(
self,
*,
message: str,
playbook: Playbook,
step_command: str,
evidence: EvidenceSnapshot | None,
verifier_plan: list[str],
) -> str:
evidence_line = "MCP evidence missing"
if evidence:
evidence_line = (
f"MCP evidence {evidence.sensors_succeeded}/{evidence.sensors_attempted}; "
f"snapshot={evidence.snapshot_id}"
)
verifier_text = "; ".join(verifier_plan)
return (
"LLM fallback 後由 MCP evidence + PlayBook trust 產生修復候選。\n"
f"原始告警:{message[:500]}\n"
f"PlayBook{playbook.playbook_id} / {playbook.name} / trust={playbook.trust_score:.2f}\n"
f"證據:{evidence_line}\n"
f"候選命令:{step_command}\n"
f"Verifier plan{verifier_text}\n"
"注意:這只是 approval candidate未經批准不得執行。"
)
def _build_dry_run_checks(
self,
*,
evidence: EvidenceSnapshot | None,
playbook: Playbook,
command: str,
) -> list[DryRunCheck]:
sensors_attempted = evidence.sensors_attempted if evidence else 0
sensors_succeeded = evidence.sensors_succeeded if evidence else 0
return [
DryRunCheck(
name="MCP evidence collected",
passed=sensors_succeeded > 0,
message=f"{sensors_succeeded}/{sensors_attempted} sensors succeeded",
),
DryRunCheck(
name="PlayBook approved and trusted",
passed=(
playbook.status == PlaybookStatus.APPROVED
and float(playbook.trust_score) >= MIN_REPAIR_CANDIDATE_TRUST
),
message=f"{playbook.playbook_id} trust={playbook.trust_score:.2f}",
),
DryRunCheck(
name="Command safety gate",
passed=True,
message=command[:240],
),
DryRunCheck(
name="Post execution verifier planned",
passed=True,
message="verify health/evidence after execution before closing incident",
),
]
def _build_verifier_plan(
self,
*,
command: str,
namespace: str,
target_resource: str,
evidence: EvidenceSnapshot | None,
) -> list[str]:
plan = [
"rerun_pre_decision_evidence_after_execution",
"compare_mcp_sensor_success_before_after",
]
if command.startswith("kubectl"):
plan.append(f"kubectl rollout status/get events in {namespace or 'awoooi-prod'}")
if command.startswith("ssh "):
plan.append("ssh_mcp_readonly_health_check_after_write_route")
if target_resource:
plan.append(f"confirm target_resource={target_resource} alert stops recurring")
if evidence and evidence.snapshot_id:
plan.append(f"baseline_snapshot={evidence.snapshot_id}")
return plan
def _approval_risk(self, risk: PlaybookRiskLevel) -> ApprovalRiskLevel:
try:
mapped = ApprovalRiskLevel(risk.value.lower())
except Exception:
return ApprovalRiskLevel.MEDIUM
if mapped == ApprovalRiskLevel.LOW:
return ApprovalRiskLevel.MEDIUM
return mapped
_repair_candidate_service: RepairCandidateService | None = None
def get_repair_candidate_service() -> RepairCandidateService:
"""Return singleton candidate builder."""
global _repair_candidate_service
if _repair_candidate_service is None:
_repair_candidate_service = RepairCandidateService()
return _repair_candidate_service

View File

@@ -0,0 +1,202 @@
from datetime import datetime, timezone
from uuid import uuid4
import pytest
from src.models.approval import ApprovalRequestCreate, BlastRadius, DataImpact, RiskLevel
from src.models.incident import Incident, Severity, Signal
from src.models.playbook import (
ActionType,
Playbook,
PlaybookSource,
PlaybookStatus,
RepairStep,
)
from src.models.playbook import RiskLevel as PlaybookRiskLevel
from src.services.approval_db import approval_request_to_record_data
from src.services.evidence_snapshot import EvidenceSnapshot
from src.services.repair_candidate_service import RepairCandidateService
class FakeInvestigator:
def __init__(self, evidence: EvidenceSnapshot | None) -> None:
self.evidence = evidence
async def investigate(self, incident: Incident) -> EvidenceSnapshot | None:
return self.evidence
class FakePlaybookRepository:
def __init__(self, playbook: Playbook | None) -> None:
self.playbook = playbook
async def get_by_id(self, playbook_id: str) -> Playbook | None:
if self.playbook and self.playbook.playbook_id == playbook_id:
return self.playbook
return None
class FakeIncidentService:
async def get_from_working_memory(self, incident_id: str) -> None:
return None
class FakeAutoRepairService:
def preview_write_ssh_mcp_route(self, incident: Incident, command: str) -> bool:
return True
def _incident() -> Incident:
return Incident(
incident_id="INC-TEST-REPAIR",
severity=Severity.P2,
signals=[
Signal(
alert_name="NodeExporterDown",
severity=Severity.P2,
source="alertmanager",
fired_at=datetime.now(timezone.utc),
labels={"namespace": "awoooi-prod", "deployment": "awoooi-api"},
annotations={"summary": "node exporter down"},
)
],
affected_services=["awoooi-api"],
)
def _evidence(incident_id: str, *, sensors_succeeded: int = 2) -> EvidenceSnapshot:
return EvidenceSnapshot(
incident_id=incident_id,
sensors_attempted=3,
sensors_succeeded=sensors_succeeded,
mcp_health={"k8s": sensors_succeeded > 0, "prometheus": sensors_succeeded > 1},
evidence_summary="k8s events and metrics collected",
)
def _playbook(command: str, *, risk_level: PlaybookRiskLevel = PlaybookRiskLevel.LOW) -> Playbook:
return Playbook(
playbook_id="PB-REPAIR-001",
name="重啟 API deployment",
description="以 approved PlayBook 修復 API 工作負載",
status=PlaybookStatus.APPROVED,
source=PlaybookSource.MANUAL,
trust_score=0.72,
estimated_duration_minutes=4,
repair_steps=[
RepairStep(
step_number=1,
action_type=ActionType.KUBECTL,
command=command,
expected_result="deployment restarted and rollout verified",
risk_level=risk_level,
requires_approval=True,
)
],
)
@pytest.mark.asyncio
async def test_build_candidate_from_mcp_evidence_and_approved_playbook() -> None:
incident = _incident()
service = RepairCandidateService(
incident_service=FakeIncidentService(),
investigator=FakeInvestigator(_evidence(incident.incident_id)),
playbook_repository=FakePlaybookRepository(
_playbook("kubectl rollout restart deployment/awoooi-api -n awoooi-prod")
),
auto_repair_service=FakeAutoRepairService(),
)
result = await service.build_from_incident(
incident=incident,
alertname="NodeExporterDown",
target_resource="awoooi-api",
namespace="awoooi-prod",
message="node exporter is down",
fallback_action="NO_ACTION - REPAIR_CANDIDATE_MISSING",
matched_playbook_id="PB-REPAIR-001",
severity="medium",
)
assert result.candidate_found is True
assert result.approval_request is not None
assert result.approval_request.action == "kubectl rollout restart deployment/awoooi-api -n awoooi-prod"
assert result.approval_request.risk_level == RiskLevel.MEDIUM
assert result.approval_request.matched_playbook_id == "PB-REPAIR-001"
assert result.metadata["repair_candidate_status"] == "candidate_ready_for_approval"
assert result.metadata["mcp_evidence"]["sensors_succeeded"] == 2
assert result.metadata["playbook_trust"]["trust_score"] == 0.72
assert result.metadata["verifier_plan"]
@pytest.mark.asyncio
async def test_candidate_blocked_when_mcp_evidence_missing() -> None:
incident = _incident()
service = RepairCandidateService(
incident_service=FakeIncidentService(),
investigator=FakeInvestigator(_evidence(incident.incident_id, sensors_succeeded=0)),
playbook_repository=FakePlaybookRepository(
_playbook("kubectl rollout restart deployment/awoooi-api -n awoooi-prod")
),
auto_repair_service=FakeAutoRepairService(),
)
result = await service.build_from_incident(
incident=incident,
alertname="NodeExporterDown",
target_resource="awoooi-api",
namespace="awoooi-prod",
message="node exporter is down",
fallback_action="NO_ACTION - REPAIR_CANDIDATE_MISSING",
matched_playbook_id="PB-REPAIR-001",
severity="medium",
)
assert result.candidate_found is False
assert "mcp_evidence_missing" in result.blockers
assert result.metadata["repair_candidate_status"] == "blocked"
@pytest.mark.asyncio
async def test_candidate_blocked_when_playbook_is_observe_only() -> None:
incident = _incident()
service = RepairCandidateService(
incident_service=FakeIncidentService(),
investigator=FakeInvestigator(_evidence(incident.incident_id)),
playbook_repository=FakePlaybookRepository(
_playbook("kubectl get pods -n awoooi-prod")
),
auto_repair_service=FakeAutoRepairService(),
)
result = await service.build_from_incident(
incident=incident,
alertname="NodeExporterDown",
target_resource="awoooi-api",
namespace="awoooi-prod",
message="node exporter is down",
fallback_action="NO_ACTION - REPAIR_CANDIDATE_MISSING",
matched_playbook_id="PB-REPAIR-001",
severity="medium",
)
assert result.candidate_found is False
assert "playbook_observe_only" in result.blockers
def test_approval_record_data_uses_preallocated_id_without_leaking_metadata() -> None:
approval_id = str(uuid4())
request = ApprovalRequestCreate(
action="kubectl rollout restart deployment/awoooi-api -n awoooi-prod",
description="candidate",
risk_level=RiskLevel.MEDIUM,
requested_by="repair-candidate-test",
blast_radius=BlastRadius(data_impact=DataImpact.WRITE),
metadata={"preallocated_approval_id": approval_id, "source": "test"},
)
data = approval_request_to_record_data(request, RiskLevel.MEDIUM, required_sigs=1)
assert data["id"] == approval_id
assert data["extra_metadata"] == {"source": "test"}