fix(aiops): route backup decisions through ssh
This commit is contained in:
@@ -22,7 +22,7 @@
|
||||
# description: <說明>
|
||||
|
||||
version: "1.0"
|
||||
last_updated: "2026-04-14"
|
||||
last_updated: "2026-05-01"
|
||||
|
||||
actions:
|
||||
# ==========================================================================
|
||||
@@ -188,6 +188,53 @@ actions:
|
||||
timeout_sec: 1
|
||||
description: "返回飛輪儀表板 URL"
|
||||
|
||||
backup_check_host_disk:
|
||||
label: "查主機磁碟"
|
||||
emoji: "💾"
|
||||
risk: low
|
||||
callback_format: info
|
||||
category: backup_failure
|
||||
mcp:
|
||||
provider: ssh
|
||||
tool: ssh_get_disk_usage
|
||||
params:
|
||||
host: "{labels.instance}"
|
||||
reply_format: code
|
||||
timeout_sec: 8
|
||||
description: "備份失敗時檢查主機磁碟容量與 Docker 目錄大小"
|
||||
|
||||
backup_check_jobs:
|
||||
label: "查備份 Job"
|
||||
emoji: "📦"
|
||||
risk: low
|
||||
callback_format: info
|
||||
category: backup_failure
|
||||
mcp:
|
||||
provider: k8s
|
||||
tool: kubectl_get
|
||||
params:
|
||||
namespace: "awoooi-prod"
|
||||
resource: "jobs"
|
||||
reply_format: truncated
|
||||
timeout_sec: 8
|
||||
description: "列出 awoooi-prod 內的備份相關 Job 狀態"
|
||||
|
||||
backup_check_velero:
|
||||
label: "查 Velero"
|
||||
emoji: "🧰"
|
||||
risk: low
|
||||
callback_format: info
|
||||
category: backup_failure
|
||||
mcp:
|
||||
provider: k8s
|
||||
tool: kubectl_get
|
||||
params:
|
||||
namespace: "velero"
|
||||
resource: "backups.velero.io"
|
||||
reply_format: truncated
|
||||
timeout_sec: 8
|
||||
description: "列出 Velero backup CR 狀態"
|
||||
|
||||
# ==========================================================================
|
||||
# 寫類按鈕(有副作用,4-part nonce callback)
|
||||
# ==========================================================================
|
||||
|
||||
@@ -35,6 +35,18 @@ import yaml
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
_PROVIDER_ALIASES = {
|
||||
"k8s": "kubernetes",
|
||||
"ssh": "ssh_host",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_provider_name(provider_name: str) -> str:
|
||||
"""Normalize legacy callback spec provider names to registered MCP providers."""
|
||||
|
||||
return _PROVIDER_ALIASES.get(provider_name, provider_name)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Data Types
|
||||
# =============================================================================
|
||||
@@ -262,14 +274,15 @@ async def dispatch_action(
|
||||
|
||||
# MCP registry dispatch
|
||||
from src.plugins.mcp.registry import get_provider
|
||||
provider = get_provider(spec.mcp_provider)
|
||||
provider_name = _resolve_provider_name(spec.mcp_provider)
|
||||
provider = get_provider(provider_name)
|
||||
if not provider:
|
||||
duration = (time.perf_counter() - start) * 1000
|
||||
return DispatchResult(
|
||||
success=False, action=action_name, incident_id=incident_id,
|
||||
user_id=user_id,
|
||||
result_text=f"{spec.emoji} {spec.label} 失敗:MCP provider '{spec.mcp_provider}' 未註冊",
|
||||
error=f"provider_not_found: {spec.mcp_provider}",
|
||||
result_text=f"{spec.emoji} {spec.label} 失敗:MCP provider '{provider_name}' 未註冊",
|
||||
error=f"provider_not_found: {provider_name}",
|
||||
duration_ms=duration,
|
||||
)
|
||||
|
||||
|
||||
@@ -85,6 +85,22 @@ def _should_escalate_auto_approve_rejection(reason: Any) -> bool:
|
||||
}
|
||||
|
||||
|
||||
_HOST_LAYER_SSH_CATEGORIES = {"infrastructure", "host_resource", "backup_failure"}
|
||||
_NON_K8S_HOST_CATEGORIES = {"host_resource", "backup_failure"}
|
||||
|
||||
|
||||
def _is_host_layer_ssh_category(category: str | None) -> bool:
|
||||
"""Return True when DecisionManager must route non-kubectl actions to SSH."""
|
||||
|
||||
return (category or "") in _HOST_LAYER_SSH_CATEGORIES
|
||||
|
||||
|
||||
def _is_non_k8s_host_category(category: str | None) -> bool:
|
||||
"""Return True for host/backup alerts that must not auto-run kubectl."""
|
||||
|
||||
return (category or "") in _NON_K8S_HOST_CATEGORIES
|
||||
|
||||
|
||||
async def _escalate_decision_auto_repair_unavailable(
|
||||
*,
|
||||
incident: Incident,
|
||||
@@ -1990,36 +2006,36 @@ class DecisionManager:
|
||||
except Exception as _rescue_err:
|
||||
logger.debug("target_rescue_skipped", error=str(_rescue_err))
|
||||
|
||||
# ADR-073 Phase 3-2: infrastructure 告警 (Docker/Host) → SSH MCP routing (2026-04-12 ogt)
|
||||
# alert_category = "infrastructure" 表示 Docker 告警,非 kubectl action → SSH
|
||||
# ADR-073 Phase 3-2: infrastructure/host/backup 告警 → SSH MCP routing.
|
||||
# alert_category = "backup_failure" uses the same host-layer path as AutoRepairService.
|
||||
# P1-1 fix 2026-04-12: 必須在 kubectl safety guard 之前 routing,否則 docker 指令被 _action_safe=False 攔截
|
||||
_alert_category = getattr(incident, "alert_category", None) or ""
|
||||
if _alert_category in {"infrastructure", "host_resource"} and action and not action.startswith("kubectl"):
|
||||
if _is_host_layer_ssh_category(_alert_category) and action and not action.startswith("kubectl"):
|
||||
await self._ssh_execute(incident, token, action, _target)
|
||||
return
|
||||
|
||||
# 2026-04-15 ogt: host_resource 告警(HostHighCpuLoad 等)不是 K8s workload 問題
|
||||
# 2026-04-15 ogt: host_resource/backup_failure 告警不是 K8s workload 問題
|
||||
# 不得執行 kubectl 操作,改降級人工審核
|
||||
# 根因:原本只擋了 infrastructure,忘記 host_resource 也不走 K8s
|
||||
if _alert_category == "host_resource" and action and action.startswith("kubectl"):
|
||||
if _is_non_k8s_host_category(_alert_category) and action and action.startswith("kubectl"):
|
||||
logger.warning(
|
||||
"auto_execute_blocked_host_resource_no_k8s",
|
||||
"auto_execute_blocked_host_layer_no_k8s",
|
||||
incident_id=incident.incident_id,
|
||||
alert_category=_alert_category,
|
||||
action=action[:80],
|
||||
reason="host_resource 告警不應執行 K8s kubectl 操作,降級人工審核",
|
||||
reason="host/backup 告警不應執行 K8s kubectl 操作,降級人工審核",
|
||||
)
|
||||
token.state = DecisionState.READY
|
||||
token.proposal_data["auto_executed"] = False
|
||||
token.proposal_data["mcp_all_failed"] = True
|
||||
token.proposal_data["blocked_reason"] = "host_resource 告警禁止 K8s kubectl,請人工排查主機"
|
||||
token.proposal_data["blocked_reason"] = f"{_alert_category} 告警禁止 K8s kubectl,請人工排查主機/備份"
|
||||
await self._save_token(token)
|
||||
_fire_and_forget(
|
||||
_escalate_decision_auto_repair_unavailable(
|
||||
incident=incident,
|
||||
token=token,
|
||||
failure_reason=token.proposal_data["blocked_reason"],
|
||||
attempted_actions="auto_execute -> host_resource_k8s_block -> emergency_intervention",
|
||||
attempted_actions=f"auto_execute -> {_alert_category}_k8s_block -> emergency_intervention",
|
||||
)
|
||||
)
|
||||
_fire_and_forget(_push_decision_to_telegram(incident, token.proposal_data))
|
||||
|
||||
Reference in New Issue
Block a user