fix(aiops): route backup decisions through ssh
Some checks failed
CD Pipeline / tests (push) Successful in 1m35s
Code Review / ai-code-review (push) Successful in 34s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled

This commit is contained in:
Your Name
2026-05-01 12:50:01 +08:00
parent 337bcb912e
commit 11673d80ea
11 changed files with 276 additions and 19 deletions

View File

@@ -22,7 +22,7 @@
# description: <說明>
version: "1.0"
last_updated: "2026-04-14"
last_updated: "2026-05-01"
actions:
# ==========================================================================
@@ -188,6 +188,53 @@ actions:
timeout_sec: 1
description: "返回飛輪儀表板 URL"
backup_check_host_disk:
label: "查主機磁碟"
emoji: "💾"
risk: low
callback_format: info
category: backup_failure
mcp:
provider: ssh
tool: ssh_get_disk_usage
params:
host: "{labels.instance}"
reply_format: code
timeout_sec: 8
description: "備份失敗時檢查主機磁碟容量與 Docker 目錄大小"
backup_check_jobs:
label: "查備份 Job"
emoji: "📦"
risk: low
callback_format: info
category: backup_failure
mcp:
provider: k8s
tool: kubectl_get
params:
namespace: "awoooi-prod"
resource: "jobs"
reply_format: truncated
timeout_sec: 8
description: "列出 awoooi-prod 內的備份相關 Job 狀態"
backup_check_velero:
label: "查 Velero"
emoji: "🧰"
risk: low
callback_format: info
category: backup_failure
mcp:
provider: k8s
tool: kubectl_get
params:
namespace: "velero"
resource: "backups.velero.io"
reply_format: truncated
timeout_sec: 8
description: "列出 Velero backup CR 狀態"
# ==========================================================================
# 寫類按鈕有副作用4-part nonce callback
# ==========================================================================

View File

@@ -35,6 +35,18 @@ import yaml
logger = structlog.get_logger(__name__)
_PROVIDER_ALIASES = {
"k8s": "kubernetes",
"ssh": "ssh_host",
}
def _resolve_provider_name(provider_name: str) -> str:
"""Normalize legacy callback spec provider names to registered MCP providers."""
return _PROVIDER_ALIASES.get(provider_name, provider_name)
# =============================================================================
# Data Types
# =============================================================================
@@ -262,14 +274,15 @@ async def dispatch_action(
# MCP registry dispatch
from src.plugins.mcp.registry import get_provider
provider = get_provider(spec.mcp_provider)
provider_name = _resolve_provider_name(spec.mcp_provider)
provider = get_provider(provider_name)
if not provider:
duration = (time.perf_counter() - start) * 1000
return DispatchResult(
success=False, action=action_name, incident_id=incident_id,
user_id=user_id,
result_text=f"{spec.emoji} {spec.label} 失敗MCP provider '{spec.mcp_provider}' 未註冊",
error=f"provider_not_found: {spec.mcp_provider}",
result_text=f"{spec.emoji} {spec.label} 失敗MCP provider '{provider_name}' 未註冊",
error=f"provider_not_found: {provider_name}",
duration_ms=duration,
)

View File

@@ -85,6 +85,22 @@ def _should_escalate_auto_approve_rejection(reason: Any) -> bool:
}
_HOST_LAYER_SSH_CATEGORIES = {"infrastructure", "host_resource", "backup_failure"}
_NON_K8S_HOST_CATEGORIES = {"host_resource", "backup_failure"}
def _is_host_layer_ssh_category(category: str | None) -> bool:
"""Return True when DecisionManager must route non-kubectl actions to SSH."""
return (category or "") in _HOST_LAYER_SSH_CATEGORIES
def _is_non_k8s_host_category(category: str | None) -> bool:
"""Return True for host/backup alerts that must not auto-run kubectl."""
return (category or "") in _NON_K8S_HOST_CATEGORIES
async def _escalate_decision_auto_repair_unavailable(
*,
incident: Incident,
@@ -1990,36 +2006,36 @@ class DecisionManager:
except Exception as _rescue_err:
logger.debug("target_rescue_skipped", error=str(_rescue_err))
# ADR-073 Phase 3-2: infrastructure 告警 (Docker/Host) → SSH MCP routing (2026-04-12 ogt)
# alert_category = "infrastructure" 表示 Docker 告警,非 kubectl action → SSH
# ADR-073 Phase 3-2: infrastructure/host/backup 告警 → SSH MCP routing.
# alert_category = "backup_failure" uses the same host-layer path as AutoRepairService.
# P1-1 fix 2026-04-12: 必須在 kubectl safety guard 之前 routing否則 docker 指令被 _action_safe=False 攔截
_alert_category = getattr(incident, "alert_category", None) or ""
if _alert_category in {"infrastructure", "host_resource"} and action and not action.startswith("kubectl"):
if _is_host_layer_ssh_category(_alert_category) and action and not action.startswith("kubectl"):
await self._ssh_execute(incident, token, action, _target)
return
# 2026-04-15 ogt: host_resource 告警HostHighCpuLoad 等)不是 K8s workload 問題
# 2026-04-15 ogt: host_resource/backup_failure 告警不是 K8s workload 問題
# 不得執行 kubectl 操作,改降級人工審核
# 根因:原本只擋了 infrastructure忘記 host_resource 也不走 K8s
if _alert_category == "host_resource" and action and action.startswith("kubectl"):
if _is_non_k8s_host_category(_alert_category) and action and action.startswith("kubectl"):
logger.warning(
"auto_execute_blocked_host_resource_no_k8s",
"auto_execute_blocked_host_layer_no_k8s",
incident_id=incident.incident_id,
alert_category=_alert_category,
action=action[:80],
reason="host_resource 告警不應執行 K8s kubectl 操作,降級人工審核",
reason="host/backup 告警不應執行 K8s kubectl 操作,降級人工審核",
)
token.state = DecisionState.READY
token.proposal_data["auto_executed"] = False
token.proposal_data["mcp_all_failed"] = True
token.proposal_data["blocked_reason"] = "host_resource 告警禁止 K8s kubectl請人工排查主機"
token.proposal_data["blocked_reason"] = f"{_alert_category} 告警禁止 K8s kubectl請人工排查主機/備份"
await self._save_token(token)
_fire_and_forget(
_escalate_decision_auto_repair_unavailable(
incident=incident,
token=token,
failure_reason=token.proposal_data["blocked_reason"],
attempted_actions="auto_execute -> host_resource_k8s_block -> emergency_intervention",
attempted_actions=f"auto_execute -> {_alert_category}_k8s_block -> emergency_intervention",
)
)
_fire_and_forget(_push_decision_to_telegram(incident, token.proposal_data))