fix(alerts): correct telegram execution truth
Some checks failed
CD Pipeline / tests (push) Failing after 52s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
Code Review / ai-code-review (push) Successful in 11s

This commit is contained in:
Your Name
2026-05-31 13:58:21 +08:00
parent 943a6feacf
commit e2ab879636
15 changed files with 624 additions and 49 deletions

View File

@@ -36,6 +36,7 @@ from src.db.base import get_db_context
from src.models.approval import ApprovalRequest
from src.plugins.mcp.gateway import GatewayContext, McpGateway, McpGatewayError
from src.plugins.mcp.interfaces import MCPToolResult
from src.services.approval_action_classifier import is_no_action_approval_action
from src.services.approval_db import get_approval_service, get_timeline_service
from src.services.executor import ExecutionResult, OperationType, get_executor
from src.services.operation_parser import parse_operation_from_action
@@ -165,6 +166,7 @@ class ApprovalExecutionService:
# ADR-090 § 自動化動作回灌 (2026-04-19): 主流程開始即在 aol 留痕,
# 結束時 update。不依賴 fire-and-forget,確保 33 件/7d approval 全部可觀測。
_aol_op_id = await self._log_aol_started(approval)
await self._log_alert_execution_started(approval, aol_op_id=_aol_op_id)
_aol_started_ms = time.time()
service = get_approval_service()
@@ -228,15 +230,7 @@ class ApprovalExecutionService:
# 2026-04-19 ogt + Claude Opus 4.7: 區分 NO_ACTION vs 真解析失敗
# NO_ACTION 是 AI 刻意選的「純調查不破壞」,不該誤標 EXECUTION_FAILED
# 污染 auto_execute 成功率 KPI (MASTER §7.1 #11)
_action_upper = (approval.action or "").upper()
_is_no_action = (
"NO_ACTION" in _action_upper
or "NO-ACTION" in _action_upper
or "NOACTION" in _action_upper
or "(未設)" in approval.action
or _action_upper.startswith("OBSERVE")
or _action_upper.startswith("INVESTIGATE")
)
_is_no_action = is_no_action_approval_action(approval.action)
if _is_no_action:
logger.info(
@@ -246,13 +240,17 @@ class ApprovalExecutionService:
reason="NO_ACTION - 純調查/觀察類,不執行破壞動作",
path="no_action",
)
# 標為 SUCCESS (觀察/調查本身就是成功完成)
await service.update_execution_status(approval.id, success=True)
# 仍以 terminal success 關閉簽核,但 metadata 明確標記未執行修復。
await service.update_execution_status(
approval.id,
success=True,
execution_kind="no_action",
)
await timeline.add_event(
event_type="exec",
status="success",
title=" 純觀察類動作完成 (NO_ACTION)",
description=f"Action: {approval.action[:120]}",
title=" 純觀察類動作已記錄(未執行修復)",
description=f"Action: {(approval.action or '')[:120]}",
actor="leWOOOgo",
actor_role="executor",
approval_id=str(approval.id),
@@ -269,7 +267,22 @@ class ApprovalExecutionService:
op_id=_aol_op_id,
status="success",
duration_ms=int((time.time() - _aol_started_ms) * 1000),
output={"reason": "NO_ACTION", "action": approval.action[:200]},
output={
"reason": "NO_ACTION",
"execution_kind": "no_action",
"repair_executed": False,
"action": (approval.action or "")[:200],
},
)
await self._log_alert_execution_completed(
approval,
success=True,
execution_kind="no_action",
duration_ms=int((time.time() - _aol_started_ms) * 1000),
output={
"reason": "NO_ACTION",
"repair_executed": False,
},
)
# F2 (2026-05-07 ogt + Claude Sonnet 4.6 + Codex):
# NO_ACTION 路徑要把 incident 推到 RESOLVED否則 incident 永遠卡
@@ -336,6 +349,13 @@ class ApprovalExecutionService:
duration_ms=int((time.time() - _aol_started_ms) * 1000),
error=f"parse_fail: {approval.action[:300]}",
)
await self._log_alert_execution_completed(
approval,
success=False,
execution_kind="parse_failed",
duration_ms=int((time.time() - _aol_started_ms) * 1000),
error_message=f"Could not parse operation type from action: {approval.action[:150]}",
)
return False # 解析失敗 → 執行未發生
executor = get_executor()
@@ -553,6 +573,20 @@ class ApprovalExecutionService:
"total_attempts": total_attempts,
},
)
await self._log_alert_execution_completed(
approval,
success=True,
execution_kind=operation_type.value,
duration_ms=int((time.time() - _aol_started_ms) * 1000),
output={
"operation_type": operation_type.value,
"resource_name": resource_name,
"namespace": namespace,
"executor_duration_ms": result.duration_ms,
"total_attempts": total_attempts,
"repair_executed": True,
},
)
return True # K8s 執行成功
else:
@@ -654,6 +688,22 @@ class ApprovalExecutionService:
error=result.error,
stderr=result.error, # E6 stderr 回灌 — 給 retry/Playbook 負向強化用
)
await self._log_alert_execution_completed(
approval,
success=False,
execution_kind=operation_type.value,
duration_ms=int((time.time() - _aol_started_ms) * 1000),
output={
"operation_type": operation_type.value,
"resource_name": resource_name,
"namespace": namespace,
"executor_duration_ms": result.duration_ms,
"total_attempts": total_attempts,
"repair_attempted": True,
"repair_executed": False,
},
error_message=result.error,
)
return False # K8s 執行失敗
async def _execute_ssh_host_action(
@@ -919,7 +969,14 @@ class ApprovalExecutionService:
except Exception:
pass
if success:
no_action = success and is_no_action_approval_action(approval.action)
if no_action:
text = (
f" <b>已記錄觀察,未執行修復</b>\n"
f"<code>{(approval.action or '')[:180]}</code>"
f"{km_info}"
)
elif success:
text = (
f"✅ <b>執行成功</b>\n"
f"<code>{(approval.action or '')[:180]}</code>"
@@ -948,8 +1005,34 @@ class ApprovalExecutionService:
incident_id=approval.incident_id,
approval_id=str(approval.id),
success=success,
no_action=no_action,
orig_msg_id=orig_msg_id,
)
try:
from src.repositories.alert_operation_log_repository import (
get_alert_operation_log_repository,
)
await get_alert_operation_log_repository().append(
"TELEGRAM_RESULT_SENT",
incident_id=approval.incident_id,
approval_id=str(approval.id),
actor="approval_execution",
action_detail="telegram_execution_result_sent",
success=success,
error_message=error,
context={
"reply_to_message_id": orig_msg_id,
"execution_kind": "no_action" if no_action else "execution",
"repair_executed": not no_action and success,
},
)
except Exception as _log_e:
logger.warning(
"alert_op_telegram_result_write_failed",
approval_id=str(approval.id),
error=str(_log_e),
)
except Exception as e:
logger.warning(
"push_execution_result_failed",
@@ -1592,6 +1675,85 @@ class ApprovalExecutionService:
# 22 筆 notification_formatted。修復後每次執行都留痕。
# =========================================================================
async def _log_alert_execution_started(
self,
approval: ApprovalRequest,
*,
aol_op_id: str | None,
) -> None:
"""Append immutable alert_operation_log start event for manual execution."""
try:
from src.repositories.alert_operation_log_repository import (
get_alert_operation_log_repository,
)
await get_alert_operation_log_repository().append(
"EXECUTION_STARTED",
incident_id=approval.incident_id,
approval_id=str(approval.id),
actor="approval_execution",
action_detail="approval_execution_started",
success=None,
context={
"action": (approval.action or "")[:500],
"automation_operation_id": aol_op_id,
"execution_kind": (
"no_action"
if is_no_action_approval_action(approval.action)
else "executable"
),
"repair_attempted": False,
"repair_executed": False,
},
)
except Exception as e:
logger.warning(
"alert_op_execution_started_write_failed",
approval_id=str(approval.id),
incident_id=approval.incident_id,
error=str(e),
)
async def _log_alert_execution_completed(
self,
approval: ApprovalRequest,
*,
success: bool,
execution_kind: str,
duration_ms: int,
output: dict | None = None,
error_message: str | None = None,
) -> None:
"""Append immutable alert_operation_log completion event for manual execution."""
try:
from src.repositories.alert_operation_log_repository import (
get_alert_operation_log_repository,
)
context = {
"action": (approval.action or "")[:500],
"duration_ms": duration_ms,
"execution_kind": execution_kind,
**(output or {}),
}
await get_alert_operation_log_repository().append(
"EXECUTION_COMPLETED",
incident_id=approval.incident_id,
approval_id=str(approval.id),
actor="approval_execution",
action_detail=f"approval_execution_{execution_kind}",
success=success,
error_message=(error_message or "")[:2000] if error_message else None,
context=context,
)
except Exception as e:
logger.warning(
"alert_op_execution_completed_write_failed",
approval_id=str(approval.id),
incident_id=approval.incident_id,
error=str(e),
)
async def _log_aol_started(self, approval: ApprovalRequest) -> str | None:
"""
在 automation_operation_log 寫一筆 'pending' 紀錄,回傳 op_id 供 _log_aol_completed 更新。