fix(awooop): link auto approved execution evidence
This commit is contained in:
@@ -858,7 +858,7 @@ class ApprovalExecutionService:
|
||||
"""
|
||||
try:
|
||||
# 自動執行路徑 skip(避免與 _push_auto_repair_result 重複發訊息)
|
||||
if (approval.requested_by or "").lower() == "auto_approve":
|
||||
if self._is_auto_approved_request(approval):
|
||||
return
|
||||
|
||||
if not approval.incident_id:
|
||||
@@ -1106,6 +1106,186 @@ class ApprovalExecutionService:
|
||||
error=str(_e),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_auto_approved_request(approval: "ApprovalRequest") -> bool:
|
||||
requested_by = (getattr(approval, "requested_by", "") or "").lower()
|
||||
return requested_by.startswith("auto_approve")
|
||||
|
||||
@staticmethod
|
||||
def _is_observation_only_action(action: str | None) -> bool:
|
||||
action_upper = (action or "").strip().upper()
|
||||
return (
|
||||
not action_upper
|
||||
or "NO_ACTION" in action_upper
|
||||
or "NO-ACTION" in action_upper
|
||||
or "NOACTION" in action_upper
|
||||
or action_upper.startswith("OBSERVE")
|
||||
or action_upper.startswith("INVESTIGATE")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _approval_risk_value(approval: "ApprovalRequest") -> str | None:
|
||||
risk_level = getattr(approval, "risk_level", None)
|
||||
if risk_level is None:
|
||||
return None
|
||||
return getattr(risk_level, "value", str(risk_level))
|
||||
|
||||
async def finalize_auto_approved_execution(
|
||||
self,
|
||||
approval: "ApprovalRequest",
|
||||
*,
|
||||
success: bool,
|
||||
error_message: str | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
補齊「自動批准已執行」路徑的 incident-linked 證據鏈。
|
||||
|
||||
CS2/CS3 webhook 路徑為了快速執行,會先呼叫 execute_approved_action(),
|
||||
再建立 Incident。executor 當下沒有 incident_id,導致 verifier/KM/
|
||||
auto_repair_executions 都無法串回同一張告警卡。此方法只在 incident
|
||||
建立後補上 durable trace,不重新執行 action。
|
||||
"""
|
||||
if not self._is_auto_approved_request(approval):
|
||||
return
|
||||
|
||||
incident_id = getattr(approval, "incident_id", None)
|
||||
if not incident_id:
|
||||
logger.warning(
|
||||
"auto_approved_execution_finalize_skipped_no_incident",
|
||||
approval_id=str(getattr(approval, "id", "")),
|
||||
requested_by=getattr(approval, "requested_by", None),
|
||||
)
|
||||
return
|
||||
|
||||
if self._is_observation_only_action(getattr(approval, "action", None)):
|
||||
logger.info(
|
||||
"auto_approved_execution_finalize_skipped_observation_only",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=incident_id,
|
||||
action=(approval.action or "")[:120],
|
||||
)
|
||||
return
|
||||
|
||||
parsed = parse_operation_from_action(approval.action)
|
||||
operation_type = parsed.operation_type
|
||||
resource_name = parsed.resource_name or "unknown"
|
||||
namespace = parsed.namespace or "default"
|
||||
|
||||
playbook_id = str(getattr(approval, "matched_playbook_id", None) or approval.id)[:36]
|
||||
operation_label = operation_type.value if operation_type else "unknown"
|
||||
playbook_name = f"approval_auto_execute:{operation_label}:{resource_name}"[:200]
|
||||
triggered_by = (getattr(approval, "requested_by", None) or "auto_approve")[:50]
|
||||
action_taken = f"auto_repair_playbook:{playbook_id}:{operation_label}:{resource_name}"
|
||||
if not success:
|
||||
action_taken = f"{action_taken}:FAILED"
|
||||
error_message = error_message or "auto-approved executor returned failure; see approval/aol logs"
|
||||
|
||||
try:
|
||||
from src.repositories.audit_log_repository import get_auto_repair_execution_repository
|
||||
|
||||
repo = get_auto_repair_execution_repository()
|
||||
existing = await repo.list_by_incident(incident_id)
|
||||
already_recorded = any(
|
||||
str(getattr(row, "playbook_id", "")) == playbook_id
|
||||
and getattr(row, "triggered_by", "") == triggered_by
|
||||
and (approval.action or "") in list(getattr(row, "executed_steps", []) or [])
|
||||
for row in existing
|
||||
)
|
||||
if not already_recorded:
|
||||
await repo.create(
|
||||
incident_id=incident_id,
|
||||
playbook_id=playbook_id,
|
||||
playbook_name=playbook_name,
|
||||
success=success,
|
||||
executed_steps=[approval.action],
|
||||
error_message=error_message,
|
||||
triggered_by=triggered_by,
|
||||
risk_level=self._approval_risk_value(approval),
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"auto_approved_execution_record_already_exists",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=incident_id,
|
||||
playbook_id=playbook_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"auto_approved_execution_record_failed",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=incident_id,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
try:
|
||||
timeline = get_timeline_service()
|
||||
await timeline.add_event(
|
||||
event_type="exec",
|
||||
status="success" if success else "error",
|
||||
title=f"{'✅' if success else '❌'} 自動批准執行已補鏈: {operation_label}",
|
||||
description=(
|
||||
f"Target: {resource_name} @ {namespace}; "
|
||||
f"source={triggered_by}; action={approval.action[:160]}"
|
||||
),
|
||||
actor="leWOOOgo",
|
||||
actor_role="executor",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=incident_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"auto_approved_execution_timeline_failed",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=incident_id,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
try:
|
||||
await self.write_execution_result_to_km(approval, success, error_message)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"auto_approved_execution_km_failed",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=incident_id,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
from src.core.feature_flags import aiops_flags
|
||||
if aiops_flags.is_sub_flag_enabled("AIOPS_P1_POST_EXECUTION_VERIFIER"):
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._run_post_execution_verify(
|
||||
approval=approval,
|
||||
action_taken=action_taken,
|
||||
),
|
||||
timeout=_VERIFIER_AWAIT_TIMEOUT_SEC,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(
|
||||
"auto_approved_execution_post_verify_timeout",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=incident_id,
|
||||
timeout_sec=_VERIFIER_AWAIT_TIMEOUT_SEC,
|
||||
)
|
||||
|
||||
if success:
|
||||
try:
|
||||
from src.services.incident_service import get_incident_service
|
||||
|
||||
await get_incident_service().resolve_incident(incident_id)
|
||||
logger.info(
|
||||
"incident_resolved_after_auto_approved_execution_finalize",
|
||||
incident_id=incident_id,
|
||||
approval_id=str(approval.id),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"incident_resolve_after_auto_approved_execution_finalize_failed",
|
||||
incident_id=incident_id,
|
||||
approval_id=str(approval.id),
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
async def write_execution_result_to_km(
|
||||
self,
|
||||
approval: "ApprovalRequest",
|
||||
@@ -1124,7 +1304,7 @@ class ApprovalExecutionService:
|
||||
from src.services.km_writer import KMWritePayload, km_write_with_flag
|
||||
|
||||
# 來源辨識(B.1 精修)
|
||||
_is_auto = (approval.requested_by or "").lower() == "auto_approve"
|
||||
_is_auto = self._is_auto_approved_request(approval)
|
||||
_mode_prefix = "[自動修復]" if _is_auto else "[人工修復]"
|
||||
_mode_tag = "auto_executed" if _is_auto else "human_approved"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user