fix(awooop): link auto approved execution evidence
All checks were successful
Code Review / ai-code-review (push) Successful in 11s
CD Pipeline / tests (push) Successful in 1m17s
CD Pipeline / build-and-deploy (push) Successful in 3m42s
CD Pipeline / post-deploy-checks (push) Successful in 1m21s

This commit is contained in:
Your Name
2026-05-13 19:14:17 +08:00
parent c68cbd3139
commit 596f2f6820
3 changed files with 371 additions and 2 deletions

View File

@@ -858,7 +858,7 @@ class ApprovalExecutionService:
"""
try:
# 自動執行路徑 skip避免與 _push_auto_repair_result 重複發訊息)
if (approval.requested_by or "").lower() == "auto_approve":
if self._is_auto_approved_request(approval):
return
if not approval.incident_id:
@@ -1106,6 +1106,186 @@ class ApprovalExecutionService:
error=str(_e),
)
@staticmethod
def _is_auto_approved_request(approval: "ApprovalRequest") -> bool:
requested_by = (getattr(approval, "requested_by", "") or "").lower()
return requested_by.startswith("auto_approve")
@staticmethod
def _is_observation_only_action(action: str | None) -> bool:
action_upper = (action or "").strip().upper()
return (
not action_upper
or "NO_ACTION" in action_upper
or "NO-ACTION" in action_upper
or "NOACTION" in action_upper
or action_upper.startswith("OBSERVE")
or action_upper.startswith("INVESTIGATE")
)
@staticmethod
def _approval_risk_value(approval: "ApprovalRequest") -> str | None:
risk_level = getattr(approval, "risk_level", None)
if risk_level is None:
return None
return getattr(risk_level, "value", str(risk_level))
async def finalize_auto_approved_execution(
self,
approval: "ApprovalRequest",
*,
success: bool,
error_message: str | None = None,
) -> None:
"""
補齊「自動批准已執行」路徑的 incident-linked 證據鏈。
CS2/CS3 webhook 路徑為了快速執行,會先呼叫 execute_approved_action()
再建立 Incident。executor 當下沒有 incident_id導致 verifier/KM/
auto_repair_executions 都無法串回同一張告警卡。此方法只在 incident
建立後補上 durable trace不重新執行 action。
"""
if not self._is_auto_approved_request(approval):
return
incident_id = getattr(approval, "incident_id", None)
if not incident_id:
logger.warning(
"auto_approved_execution_finalize_skipped_no_incident",
approval_id=str(getattr(approval, "id", "")),
requested_by=getattr(approval, "requested_by", None),
)
return
if self._is_observation_only_action(getattr(approval, "action", None)):
logger.info(
"auto_approved_execution_finalize_skipped_observation_only",
approval_id=str(approval.id),
incident_id=incident_id,
action=(approval.action or "")[:120],
)
return
parsed = parse_operation_from_action(approval.action)
operation_type = parsed.operation_type
resource_name = parsed.resource_name or "unknown"
namespace = parsed.namespace or "default"
playbook_id = str(getattr(approval, "matched_playbook_id", None) or approval.id)[:36]
operation_label = operation_type.value if operation_type else "unknown"
playbook_name = f"approval_auto_execute:{operation_label}:{resource_name}"[:200]
triggered_by = (getattr(approval, "requested_by", None) or "auto_approve")[:50]
action_taken = f"auto_repair_playbook:{playbook_id}:{operation_label}:{resource_name}"
if not success:
action_taken = f"{action_taken}:FAILED"
error_message = error_message or "auto-approved executor returned failure; see approval/aol logs"
try:
from src.repositories.audit_log_repository import get_auto_repair_execution_repository
repo = get_auto_repair_execution_repository()
existing = await repo.list_by_incident(incident_id)
already_recorded = any(
str(getattr(row, "playbook_id", "")) == playbook_id
and getattr(row, "triggered_by", "") == triggered_by
and (approval.action or "") in list(getattr(row, "executed_steps", []) or [])
for row in existing
)
if not already_recorded:
await repo.create(
incident_id=incident_id,
playbook_id=playbook_id,
playbook_name=playbook_name,
success=success,
executed_steps=[approval.action],
error_message=error_message,
triggered_by=triggered_by,
risk_level=self._approval_risk_value(approval),
)
else:
logger.info(
"auto_approved_execution_record_already_exists",
approval_id=str(approval.id),
incident_id=incident_id,
playbook_id=playbook_id,
)
except Exception as exc:
logger.warning(
"auto_approved_execution_record_failed",
approval_id=str(approval.id),
incident_id=incident_id,
error=str(exc),
)
try:
timeline = get_timeline_service()
await timeline.add_event(
event_type="exec",
status="success" if success else "error",
title=f"{'' if success else ''} 自動批准執行已補鏈: {operation_label}",
description=(
f"Target: {resource_name} @ {namespace}; "
f"source={triggered_by}; action={approval.action[:160]}"
),
actor="leWOOOgo",
actor_role="executor",
approval_id=str(approval.id),
incident_id=incident_id,
)
except Exception as exc:
logger.warning(
"auto_approved_execution_timeline_failed",
approval_id=str(approval.id),
incident_id=incident_id,
error=str(exc),
)
try:
await self.write_execution_result_to_km(approval, success, error_message)
except Exception as exc:
logger.warning(
"auto_approved_execution_km_failed",
approval_id=str(approval.id),
incident_id=incident_id,
error=str(exc),
)
from src.core.feature_flags import aiops_flags
if aiops_flags.is_sub_flag_enabled("AIOPS_P1_POST_EXECUTION_VERIFIER"):
try:
await asyncio.wait_for(
self._run_post_execution_verify(
approval=approval,
action_taken=action_taken,
),
timeout=_VERIFIER_AWAIT_TIMEOUT_SEC,
)
except asyncio.TimeoutError:
logger.warning(
"auto_approved_execution_post_verify_timeout",
approval_id=str(approval.id),
incident_id=incident_id,
timeout_sec=_VERIFIER_AWAIT_TIMEOUT_SEC,
)
if success:
try:
from src.services.incident_service import get_incident_service
await get_incident_service().resolve_incident(incident_id)
logger.info(
"incident_resolved_after_auto_approved_execution_finalize",
incident_id=incident_id,
approval_id=str(approval.id),
)
except Exception as exc:
logger.warning(
"incident_resolve_after_auto_approved_execution_finalize_failed",
incident_id=incident_id,
approval_id=str(approval.id),
error=str(exc),
)
async def write_execution_result_to_km(
self,
approval: "ApprovalRequest",
@@ -1124,7 +1304,7 @@ class ApprovalExecutionService:
from src.services.km_writer import KMWritePayload, km_write_with_flag
# 來源辨識B.1 精修)
_is_auto = (approval.requested_by or "").lower() == "auto_approve"
_is_auto = self._is_auto_approved_request(approval)
_mode_prefix = "[自動修復]" if _is_auto else "[人工修復]"
_mode_tag = "auto_executed" if _is_auto else "human_approved"