fix(aiops): route backup decisions through ssh
This commit is contained in:
@@ -22,7 +22,7 @@
|
||||
# description: <說明>
|
||||
|
||||
version: "1.0"
|
||||
last_updated: "2026-04-14"
|
||||
last_updated: "2026-05-01"
|
||||
|
||||
actions:
|
||||
# ==========================================================================
|
||||
@@ -188,6 +188,53 @@ actions:
|
||||
timeout_sec: 1
|
||||
description: "返回飛輪儀表板 URL"
|
||||
|
||||
backup_check_host_disk:
|
||||
label: "查主機磁碟"
|
||||
emoji: "💾"
|
||||
risk: low
|
||||
callback_format: info
|
||||
category: backup_failure
|
||||
mcp:
|
||||
provider: ssh
|
||||
tool: ssh_get_disk_usage
|
||||
params:
|
||||
host: "{labels.instance}"
|
||||
reply_format: code
|
||||
timeout_sec: 8
|
||||
description: "備份失敗時檢查主機磁碟容量與 Docker 目錄大小"
|
||||
|
||||
backup_check_jobs:
|
||||
label: "查備份 Job"
|
||||
emoji: "📦"
|
||||
risk: low
|
||||
callback_format: info
|
||||
category: backup_failure
|
||||
mcp:
|
||||
provider: k8s
|
||||
tool: kubectl_get
|
||||
params:
|
||||
namespace: "awoooi-prod"
|
||||
resource: "jobs"
|
||||
reply_format: truncated
|
||||
timeout_sec: 8
|
||||
description: "列出 awoooi-prod 內的備份相關 Job 狀態"
|
||||
|
||||
backup_check_velero:
|
||||
label: "查 Velero"
|
||||
emoji: "🧰"
|
||||
risk: low
|
||||
callback_format: info
|
||||
category: backup_failure
|
||||
mcp:
|
||||
provider: k8s
|
||||
tool: kubectl_get
|
||||
params:
|
||||
namespace: "velero"
|
||||
resource: "backups.velero.io"
|
||||
reply_format: truncated
|
||||
timeout_sec: 8
|
||||
description: "列出 Velero backup CR 狀態"
|
||||
|
||||
# ==========================================================================
|
||||
# 寫類按鈕(有副作用,4-part nonce callback)
|
||||
# ==========================================================================
|
||||
|
||||
@@ -35,6 +35,18 @@ import yaml
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
_PROVIDER_ALIASES = {
|
||||
"k8s": "kubernetes",
|
||||
"ssh": "ssh_host",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_provider_name(provider_name: str) -> str:
|
||||
"""Normalize legacy callback spec provider names to registered MCP providers."""
|
||||
|
||||
return _PROVIDER_ALIASES.get(provider_name, provider_name)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Data Types
|
||||
# =============================================================================
|
||||
@@ -262,14 +274,15 @@ async def dispatch_action(
|
||||
|
||||
# MCP registry dispatch
|
||||
from src.plugins.mcp.registry import get_provider
|
||||
provider = get_provider(spec.mcp_provider)
|
||||
provider_name = _resolve_provider_name(spec.mcp_provider)
|
||||
provider = get_provider(provider_name)
|
||||
if not provider:
|
||||
duration = (time.perf_counter() - start) * 1000
|
||||
return DispatchResult(
|
||||
success=False, action=action_name, incident_id=incident_id,
|
||||
user_id=user_id,
|
||||
result_text=f"{spec.emoji} {spec.label} 失敗:MCP provider '{spec.mcp_provider}' 未註冊",
|
||||
error=f"provider_not_found: {spec.mcp_provider}",
|
||||
result_text=f"{spec.emoji} {spec.label} 失敗:MCP provider '{provider_name}' 未註冊",
|
||||
error=f"provider_not_found: {provider_name}",
|
||||
duration_ms=duration,
|
||||
)
|
||||
|
||||
|
||||
@@ -85,6 +85,22 @@ def _should_escalate_auto_approve_rejection(reason: Any) -> bool:
|
||||
}
|
||||
|
||||
|
||||
_HOST_LAYER_SSH_CATEGORIES = {"infrastructure", "host_resource", "backup_failure"}
|
||||
_NON_K8S_HOST_CATEGORIES = {"host_resource", "backup_failure"}
|
||||
|
||||
|
||||
def _is_host_layer_ssh_category(category: str | None) -> bool:
|
||||
"""Return True when DecisionManager must route non-kubectl actions to SSH."""
|
||||
|
||||
return (category or "") in _HOST_LAYER_SSH_CATEGORIES
|
||||
|
||||
|
||||
def _is_non_k8s_host_category(category: str | None) -> bool:
|
||||
"""Return True for host/backup alerts that must not auto-run kubectl."""
|
||||
|
||||
return (category or "") in _NON_K8S_HOST_CATEGORIES
|
||||
|
||||
|
||||
async def _escalate_decision_auto_repair_unavailable(
|
||||
*,
|
||||
incident: Incident,
|
||||
@@ -1990,36 +2006,36 @@ class DecisionManager:
|
||||
except Exception as _rescue_err:
|
||||
logger.debug("target_rescue_skipped", error=str(_rescue_err))
|
||||
|
||||
# ADR-073 Phase 3-2: infrastructure 告警 (Docker/Host) → SSH MCP routing (2026-04-12 ogt)
|
||||
# alert_category = "infrastructure" 表示 Docker 告警,非 kubectl action → SSH
|
||||
# ADR-073 Phase 3-2: infrastructure/host/backup 告警 → SSH MCP routing.
|
||||
# alert_category = "backup_failure" uses the same host-layer path as AutoRepairService.
|
||||
# P1-1 fix 2026-04-12: 必須在 kubectl safety guard 之前 routing,否則 docker 指令被 _action_safe=False 攔截
|
||||
_alert_category = getattr(incident, "alert_category", None) or ""
|
||||
if _alert_category in {"infrastructure", "host_resource"} and action and not action.startswith("kubectl"):
|
||||
if _is_host_layer_ssh_category(_alert_category) and action and not action.startswith("kubectl"):
|
||||
await self._ssh_execute(incident, token, action, _target)
|
||||
return
|
||||
|
||||
# 2026-04-15 ogt: host_resource 告警(HostHighCpuLoad 等)不是 K8s workload 問題
|
||||
# 2026-04-15 ogt: host_resource/backup_failure 告警不是 K8s workload 問題
|
||||
# 不得執行 kubectl 操作,改降級人工審核
|
||||
# 根因:原本只擋了 infrastructure,忘記 host_resource 也不走 K8s
|
||||
if _alert_category == "host_resource" and action and action.startswith("kubectl"):
|
||||
if _is_non_k8s_host_category(_alert_category) and action and action.startswith("kubectl"):
|
||||
logger.warning(
|
||||
"auto_execute_blocked_host_resource_no_k8s",
|
||||
"auto_execute_blocked_host_layer_no_k8s",
|
||||
incident_id=incident.incident_id,
|
||||
alert_category=_alert_category,
|
||||
action=action[:80],
|
||||
reason="host_resource 告警不應執行 K8s kubectl 操作,降級人工審核",
|
||||
reason="host/backup 告警不應執行 K8s kubectl 操作,降級人工審核",
|
||||
)
|
||||
token.state = DecisionState.READY
|
||||
token.proposal_data["auto_executed"] = False
|
||||
token.proposal_data["mcp_all_failed"] = True
|
||||
token.proposal_data["blocked_reason"] = "host_resource 告警禁止 K8s kubectl,請人工排查主機"
|
||||
token.proposal_data["blocked_reason"] = f"{_alert_category} 告警禁止 K8s kubectl,請人工排查主機/備份"
|
||||
await self._save_token(token)
|
||||
_fire_and_forget(
|
||||
_escalate_decision_auto_repair_unavailable(
|
||||
incident=incident,
|
||||
token=token,
|
||||
failure_reason=token.proposal_data["blocked_reason"],
|
||||
attempted_actions="auto_execute -> host_resource_k8s_block -> emergency_intervention",
|
||||
attempted_actions=f"auto_execute -> {_alert_category}_k8s_block -> emergency_intervention",
|
||||
)
|
||||
)
|
||||
_fire_and_forget(_push_decision_to_telegram(incident, token.proposal_data))
|
||||
|
||||
@@ -4,7 +4,11 @@ from src.api.v1.webhooks import (
|
||||
_should_bypass_alertmanager_llm,
|
||||
_should_use_alertmanager_rule_first,
|
||||
)
|
||||
from src.services.decision_manager import _should_escalate_auto_approve_rejection
|
||||
from src.services.decision_manager import (
|
||||
_is_host_layer_ssh_category,
|
||||
_is_non_k8s_host_category,
|
||||
_should_escalate_auto_approve_rejection,
|
||||
)
|
||||
from src.services.telegram_gateway import _format_resolved_guard_stamp
|
||||
|
||||
|
||||
@@ -84,6 +88,18 @@ def test_manual_gate_reasons_escalate_to_emergency_intervention():
|
||||
assert _should_escalate_auto_approve_rejection("critical_operation") is False
|
||||
|
||||
|
||||
def test_backup_failure_routes_to_decision_ssh_before_kubectl_parser():
|
||||
assert _is_host_layer_ssh_category("backup_failure") is True
|
||||
assert _is_host_layer_ssh_category("host_resource") is True
|
||||
assert _is_host_layer_ssh_category("kubernetes") is False
|
||||
|
||||
|
||||
def test_backup_failure_blocks_k8s_auto_execute():
|
||||
assert _is_non_k8s_host_category("backup_failure") is True
|
||||
assert _is_non_k8s_host_category("host_resource") is True
|
||||
assert _is_non_k8s_host_category("infrastructure") is False
|
||||
|
||||
|
||||
def test_resolved_guard_stamp_without_timestamp_is_clean():
|
||||
assert _format_resolved_guard_stamp(None) == "✅ 此事件已解決"
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ from src.services.callback_dispatcher import (
|
||||
list_actions_for_category,
|
||||
load_action_registry,
|
||||
_lookup_context,
|
||||
_resolve_provider_name,
|
||||
_resolve_template,
|
||||
)
|
||||
|
||||
@@ -68,6 +69,11 @@ class TestRegistryLoading:
|
||||
assert spec and spec.callback_format == "info", \
|
||||
f"{qa} should use info format"
|
||||
|
||||
def test_legacy_provider_aliases_resolve_to_registered_names(self):
|
||||
assert _resolve_provider_name("k8s") == "kubernetes"
|
||||
assert _resolve_provider_name("ssh") == "ssh_host"
|
||||
assert _resolve_provider_name("prometheus") == "prometheus"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Category filtering
|
||||
@@ -91,6 +97,16 @@ class TestCategoryFiltering:
|
||||
assert any(a.callback_format == "info" for a in actions), "需至少 1 個查類"
|
||||
assert any(a.callback_format == "nonce" for a in actions), "需至少 1 個寫類"
|
||||
|
||||
def test_backup_failure_has_read_only_diagnostics(self):
|
||||
actions = list_actions_for_category("backup_failure")
|
||||
names = {a.name for a in actions}
|
||||
assert {
|
||||
"backup_check_host_disk",
|
||||
"backup_check_jobs",
|
||||
"backup_check_velero",
|
||||
}.issubset(names)
|
||||
assert all(a.callback_format == "info" for a in actions)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Template variable resolution
|
||||
|
||||
Reference in New Issue
Block a user