fix(alerts): correct telegram execution truth
This commit is contained in:
@@ -36,6 +36,7 @@ from src.db.base import get_db_context
|
||||
from src.models.approval import ApprovalRequest
|
||||
from src.plugins.mcp.gateway import GatewayContext, McpGateway, McpGatewayError
|
||||
from src.plugins.mcp.interfaces import MCPToolResult
|
||||
from src.services.approval_action_classifier import is_no_action_approval_action
|
||||
from src.services.approval_db import get_approval_service, get_timeline_service
|
||||
from src.services.executor import ExecutionResult, OperationType, get_executor
|
||||
from src.services.operation_parser import parse_operation_from_action
|
||||
@@ -165,6 +166,7 @@ class ApprovalExecutionService:
|
||||
# ADR-090 § 自動化動作回灌 (2026-04-19): 主流程開始即在 aol 留痕,
|
||||
# 結束時 update。不依賴 fire-and-forget,確保 33 件/7d approval 全部可觀測。
|
||||
_aol_op_id = await self._log_aol_started(approval)
|
||||
await self._log_alert_execution_started(approval, aol_op_id=_aol_op_id)
|
||||
_aol_started_ms = time.time()
|
||||
|
||||
service = get_approval_service()
|
||||
@@ -228,15 +230,7 @@ class ApprovalExecutionService:
|
||||
# 2026-04-19 ogt + Claude Opus 4.7: 區分 NO_ACTION vs 真解析失敗
|
||||
# NO_ACTION 是 AI 刻意選的「純調查不破壞」,不該誤標 EXECUTION_FAILED
|
||||
# 污染 auto_execute 成功率 KPI (MASTER §7.1 #11)
|
||||
_action_upper = (approval.action or "").upper()
|
||||
_is_no_action = (
|
||||
"NO_ACTION" in _action_upper
|
||||
or "NO-ACTION" in _action_upper
|
||||
or "NOACTION" in _action_upper
|
||||
or "(未設)" in approval.action
|
||||
or _action_upper.startswith("OBSERVE")
|
||||
or _action_upper.startswith("INVESTIGATE")
|
||||
)
|
||||
_is_no_action = is_no_action_approval_action(approval.action)
|
||||
|
||||
if _is_no_action:
|
||||
logger.info(
|
||||
@@ -246,13 +240,17 @@ class ApprovalExecutionService:
|
||||
reason="NO_ACTION - 純調查/觀察類,不執行破壞動作",
|
||||
path="no_action",
|
||||
)
|
||||
# 標為 SUCCESS (觀察/調查本身就是成功完成)
|
||||
await service.update_execution_status(approval.id, success=True)
|
||||
# 仍以 terminal success 關閉簽核,但 metadata 明確標記未執行修復。
|
||||
await service.update_execution_status(
|
||||
approval.id,
|
||||
success=True,
|
||||
execution_kind="no_action",
|
||||
)
|
||||
await timeline.add_event(
|
||||
event_type="exec",
|
||||
status="success",
|
||||
title="✅ 純觀察類動作完成 (NO_ACTION)",
|
||||
description=f"Action: {approval.action[:120]}",
|
||||
title="ℹ️ 純觀察類動作已記錄(未執行修復)",
|
||||
description=f"Action: {(approval.action or '')[:120]}",
|
||||
actor="leWOOOgo",
|
||||
actor_role="executor",
|
||||
approval_id=str(approval.id),
|
||||
@@ -269,7 +267,22 @@ class ApprovalExecutionService:
|
||||
op_id=_aol_op_id,
|
||||
status="success",
|
||||
duration_ms=int((time.time() - _aol_started_ms) * 1000),
|
||||
output={"reason": "NO_ACTION", "action": approval.action[:200]},
|
||||
output={
|
||||
"reason": "NO_ACTION",
|
||||
"execution_kind": "no_action",
|
||||
"repair_executed": False,
|
||||
"action": (approval.action or "")[:200],
|
||||
},
|
||||
)
|
||||
await self._log_alert_execution_completed(
|
||||
approval,
|
||||
success=True,
|
||||
execution_kind="no_action",
|
||||
duration_ms=int((time.time() - _aol_started_ms) * 1000),
|
||||
output={
|
||||
"reason": "NO_ACTION",
|
||||
"repair_executed": False,
|
||||
},
|
||||
)
|
||||
# F2 (2026-05-07 ogt + Claude Sonnet 4.6 + Codex):
|
||||
# NO_ACTION 路徑要把 incident 推到 RESOLVED,否則 incident 永遠卡
|
||||
@@ -336,6 +349,13 @@ class ApprovalExecutionService:
|
||||
duration_ms=int((time.time() - _aol_started_ms) * 1000),
|
||||
error=f"parse_fail: {approval.action[:300]}",
|
||||
)
|
||||
await self._log_alert_execution_completed(
|
||||
approval,
|
||||
success=False,
|
||||
execution_kind="parse_failed",
|
||||
duration_ms=int((time.time() - _aol_started_ms) * 1000),
|
||||
error_message=f"Could not parse operation type from action: {approval.action[:150]}",
|
||||
)
|
||||
return False # 解析失敗 → 執行未發生
|
||||
|
||||
executor = get_executor()
|
||||
@@ -553,6 +573,20 @@ class ApprovalExecutionService:
|
||||
"total_attempts": total_attempts,
|
||||
},
|
||||
)
|
||||
await self._log_alert_execution_completed(
|
||||
approval,
|
||||
success=True,
|
||||
execution_kind=operation_type.value,
|
||||
duration_ms=int((time.time() - _aol_started_ms) * 1000),
|
||||
output={
|
||||
"operation_type": operation_type.value,
|
||||
"resource_name": resource_name,
|
||||
"namespace": namespace,
|
||||
"executor_duration_ms": result.duration_ms,
|
||||
"total_attempts": total_attempts,
|
||||
"repair_executed": True,
|
||||
},
|
||||
)
|
||||
return True # K8s 執行成功
|
||||
|
||||
else:
|
||||
@@ -654,6 +688,22 @@ class ApprovalExecutionService:
|
||||
error=result.error,
|
||||
stderr=result.error, # E6 stderr 回灌 — 給 retry/Playbook 負向強化用
|
||||
)
|
||||
await self._log_alert_execution_completed(
|
||||
approval,
|
||||
success=False,
|
||||
execution_kind=operation_type.value,
|
||||
duration_ms=int((time.time() - _aol_started_ms) * 1000),
|
||||
output={
|
||||
"operation_type": operation_type.value,
|
||||
"resource_name": resource_name,
|
||||
"namespace": namespace,
|
||||
"executor_duration_ms": result.duration_ms,
|
||||
"total_attempts": total_attempts,
|
||||
"repair_attempted": True,
|
||||
"repair_executed": False,
|
||||
},
|
||||
error_message=result.error,
|
||||
)
|
||||
return False # K8s 執行失敗
|
||||
|
||||
async def _execute_ssh_host_action(
|
||||
@@ -919,7 +969,14 @@ class ApprovalExecutionService:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if success:
|
||||
no_action = success and is_no_action_approval_action(approval.action)
|
||||
if no_action:
|
||||
text = (
|
||||
f"ℹ️ <b>已記錄觀察,未執行修復</b>\n"
|
||||
f"<code>{(approval.action or '')[:180]}</code>"
|
||||
f"{km_info}"
|
||||
)
|
||||
elif success:
|
||||
text = (
|
||||
f"✅ <b>執行成功</b>\n"
|
||||
f"<code>{(approval.action or '')[:180]}</code>"
|
||||
@@ -948,8 +1005,34 @@ class ApprovalExecutionService:
|
||||
incident_id=approval.incident_id,
|
||||
approval_id=str(approval.id),
|
||||
success=success,
|
||||
no_action=no_action,
|
||||
orig_msg_id=orig_msg_id,
|
||||
)
|
||||
try:
|
||||
from src.repositories.alert_operation_log_repository import (
|
||||
get_alert_operation_log_repository,
|
||||
)
|
||||
|
||||
await get_alert_operation_log_repository().append(
|
||||
"TELEGRAM_RESULT_SENT",
|
||||
incident_id=approval.incident_id,
|
||||
approval_id=str(approval.id),
|
||||
actor="approval_execution",
|
||||
action_detail="telegram_execution_result_sent",
|
||||
success=success,
|
||||
error_message=error,
|
||||
context={
|
||||
"reply_to_message_id": orig_msg_id,
|
||||
"execution_kind": "no_action" if no_action else "execution",
|
||||
"repair_executed": not no_action and success,
|
||||
},
|
||||
)
|
||||
except Exception as _log_e:
|
||||
logger.warning(
|
||||
"alert_op_telegram_result_write_failed",
|
||||
approval_id=str(approval.id),
|
||||
error=str(_log_e),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"push_execution_result_failed",
|
||||
@@ -1592,6 +1675,85 @@ class ApprovalExecutionService:
|
||||
# 22 筆 notification_formatted。修復後每次執行都留痕。
|
||||
# =========================================================================
|
||||
|
||||
async def _log_alert_execution_started(
|
||||
self,
|
||||
approval: ApprovalRequest,
|
||||
*,
|
||||
aol_op_id: str | None,
|
||||
) -> None:
|
||||
"""Append immutable alert_operation_log start event for manual execution."""
|
||||
try:
|
||||
from src.repositories.alert_operation_log_repository import (
|
||||
get_alert_operation_log_repository,
|
||||
)
|
||||
|
||||
await get_alert_operation_log_repository().append(
|
||||
"EXECUTION_STARTED",
|
||||
incident_id=approval.incident_id,
|
||||
approval_id=str(approval.id),
|
||||
actor="approval_execution",
|
||||
action_detail="approval_execution_started",
|
||||
success=None,
|
||||
context={
|
||||
"action": (approval.action or "")[:500],
|
||||
"automation_operation_id": aol_op_id,
|
||||
"execution_kind": (
|
||||
"no_action"
|
||||
if is_no_action_approval_action(approval.action)
|
||||
else "executable"
|
||||
),
|
||||
"repair_attempted": False,
|
||||
"repair_executed": False,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"alert_op_execution_started_write_failed",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=approval.incident_id,
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
async def _log_alert_execution_completed(
|
||||
self,
|
||||
approval: ApprovalRequest,
|
||||
*,
|
||||
success: bool,
|
||||
execution_kind: str,
|
||||
duration_ms: int,
|
||||
output: dict | None = None,
|
||||
error_message: str | None = None,
|
||||
) -> None:
|
||||
"""Append immutable alert_operation_log completion event for manual execution."""
|
||||
try:
|
||||
from src.repositories.alert_operation_log_repository import (
|
||||
get_alert_operation_log_repository,
|
||||
)
|
||||
|
||||
context = {
|
||||
"action": (approval.action or "")[:500],
|
||||
"duration_ms": duration_ms,
|
||||
"execution_kind": execution_kind,
|
||||
**(output or {}),
|
||||
}
|
||||
await get_alert_operation_log_repository().append(
|
||||
"EXECUTION_COMPLETED",
|
||||
incident_id=approval.incident_id,
|
||||
approval_id=str(approval.id),
|
||||
actor="approval_execution",
|
||||
action_detail=f"approval_execution_{execution_kind}",
|
||||
success=success,
|
||||
error_message=(error_message or "")[:2000] if error_message else None,
|
||||
context=context,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"alert_op_execution_completed_write_failed",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=approval.incident_id,
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
async def _log_aol_started(self, approval: ApprovalRequest) -> str | None:
|
||||
"""
|
||||
在 automation_operation_log 寫一筆 'pending' 紀錄,回傳 op_id 供 _log_aol_completed 更新。
|
||||
|
||||
Reference in New Issue
Block a user