feat(ai-ops): ADR-013 AIOps 自動修復閉環完整實作
架構（Exception → Incident → PlayBook → Heal → KM → Telegram）: 新增元件: - database/autoheal_models.py: Incident/Playbook/HealLog 三張表 + 7 條種子 PlayBook - migrations/013_autoheal.sql: 建表 DDL + 種子資料（冪等 INSERT） - services/auto_heal_service.py: 核心引擎 7 步閉環 - _classify_error: 8 類錯誤自動分類 (DNS_FAIL/DB_UNREACHABLE/OOM/...) - _match_playbook: error_type + keyword + 冷卻 + max_retries 保護 - _execute_playbook: DOCKER_RESTART/SSH_CMD/ALERT_ONLY/WAIT_RETRY - _sink_to_km: 修復知識寫入 ai_insights (auto_heal_playbook) - SSH 白名單：僅允許 docker restart / compose restart / docker start 修改元件: - database/manager.py: _init_autoheal_tables() 啟動時建表+種子 PlayBook - scheduler.py: 3 個核心任務植入 handle_exception (run_auto_import_task / run_icaim_analysis_task / run_weekly_strategy_task) - requirements.txt: paramiko（SSH 跳板；不可用時降級 subprocess+CLI ssh） 安全設計: CMD 白名單 + cooldown + max_retries escalation + DB 冪等 migration Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
ogt
524
services/auto_heal_service.py
Normal file
524
services/auto_heal_service.py
Normal file
@@ -0,0 +1,524 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
auto_heal_service.py - EwoooC AIOps 自動修復引擎 (ADR-013)
|
||||
|
||||
完整閉環：
|
||||
Exception 觸發
|
||||
→ create_incident() : 寫入 incidents 表
|
||||
→ classify_error() : 識別 error_type
|
||||
→ match_playbook() : 比對 playbooks 規則庫
|
||||
→ execute_playbook() : 執行修復動作
|
||||
→ _write_heal_log() : 寫入 heal_logs 表
|
||||
→ sink_to_km() : store_insight → KM RAG 沉澱
|
||||
→ notify_telegram() : 推播修復結果
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import traceback as tb
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from database.manager import get_session
|
||||
from database.autoheal_models import Incident, Playbook, HealLog, SEED_PLAYBOOKS
|
||||
from services.logger_manager import SystemLogger
|
||||
|
||||
load_dotenv()
|
||||
sys_log = SystemLogger("AutoHeal").get_logger()
|
||||
|
||||
# ─── Telegram 設定 ───────────────────────────────────────
|
||||
_BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN") or os.getenv("OPENCLAW_BOT_TOKEN", "")
|
||||
_CHAT_ID = os.getenv("OPENCLAW_GROUP_ID", "-1003940688311")
|
||||
|
||||
# ─── SSH 跳板機設定 ──────────────────────────────────────
|
||||
_JUMP_HOST = os.getenv("SSH_JUMP_HOST", "192.168.0.110")
|
||||
_JUMP_USER = os.getenv("SSH_JUMP_USER", "wooo")
|
||||
_TARGET_HOST = os.getenv("SSH_TARGET_HOST", "192.168.0.188")
|
||||
_TARGET_USER = os.getenv("SSH_TARGET_USER", "ollama")
|
||||
|
||||
# ─── 白名單允許執行的指令前綴 ────────────────────────────
|
||||
_CMD_WHITELIST = [
|
||||
"docker restart ",
|
||||
"docker compose restart ",
|
||||
"docker start ",
|
||||
]
|
||||
|
||||
# ─── 錯誤分類對照表（keyword → error_type）──────────────
|
||||
_ERROR_CLASSIFY_MAP = {
|
||||
"DNS_FAIL": ["name resolution", "could not translate host name",
|
||||
"Temporary failure in name resolution"],
|
||||
"DB_UNREACHABLE": ["connection refused", "Connection reset by peer",
|
||||
"could not connect to server", "psycopg2.OperationalError"],
|
||||
"OOM": ["SIGKILL", "out of memory", "Worker was sent SIGKILL", "MemoryError"],
|
||||
"SSL_FAIL": ["SSL connection has been closed unexpectedly", "SSL SYSCALL error"],
|
||||
"AUTH_FAIL": ["invalid_grant", "google_token.pickle", "Token has been expired"],
|
||||
"CRAWLER_FAIL": ["429 Too Many Requests", "rate limit", "Retry-After",
|
||||
"CloudflareCaptcha", "webdriver"],
|
||||
"IMPORT_FAIL": ["import_service", "ImportError", "sync_daily_sales"],
|
||||
"TIMEOUT": ["Timeout", "timed out", "TimeoutError"],
|
||||
}
|
||||
|
||||
_SEVERITY_MAP = {
|
||||
"P1": ["OOM", "SSL_FAIL"],
|
||||
"P2": ["DNS_FAIL", "DB_UNREACHABLE", "AUTH_FAIL"],
|
||||
"P3": ["CRAWLER_FAIL", "IMPORT_FAIL", "TIMEOUT"],
|
||||
}
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────
|
||||
# 工具函數
|
||||
# ──────────────────────────────────────────────────────────
|
||||
|
||||
def _classify_error(error_msg: str) -> Tuple[str, str]:
|
||||
"""回傳 (error_type, severity)"""
|
||||
lower = error_msg.lower()
|
||||
for etype, keywords in _ERROR_CLASSIFY_MAP.items():
|
||||
if any(k.lower() in lower for k in keywords):
|
||||
for sev, etypes in _SEVERITY_MAP.items():
|
||||
if etype in etypes:
|
||||
return etype, sev
|
||||
return etype, "P3"
|
||||
return "UNKNOWN", "P3"
|
||||
|
||||
|
||||
def _is_cmd_allowed(cmd: str) -> bool:
|
||||
"""白名單驗證：防止任意 RCE"""
|
||||
c = cmd.strip()
|
||||
return any(c.startswith(prefix) for prefix in _CMD_WHITELIST)
|
||||
|
||||
|
||||
def _send_telegram(msg: str) -> None:
|
||||
"""推播訊息至 Telegram 群組"""
|
||||
if not _BOT_TOKEN:
|
||||
sys_log.warning("[AutoHeal] TELEGRAM_BOT_TOKEN 未設定，略過推播")
|
||||
return
|
||||
try:
|
||||
requests.post(
|
||||
f"https://api.telegram.org/bot{_BOT_TOKEN}/sendMessage",
|
||||
json={"chat_id": _CHAT_ID, "text": msg, "parse_mode": "HTML"},
|
||||
timeout=10,
|
||||
)
|
||||
except Exception as e:
|
||||
sys_log.error(f"[AutoHeal] Telegram 推播失敗: {e}")
|
||||
|
||||
|
||||
def _execute_ssh_cmd(cmd: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
透過 paramiko 執行 SSH 跳板指令。
|
||||
若 paramiko 不可用則降級為 subprocess + CLI ssh。
|
||||
"""
|
||||
if not _is_cmd_allowed(cmd):
|
||||
return False, f"指令不在白名單中，拒絕執行: {cmd}"
|
||||
|
||||
try:
|
||||
import paramiko
|
||||
jump = paramiko.SSHClient()
|
||||
jump.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
jump.connect(_JUMP_HOST, username=_JUMP_USER, timeout=10)
|
||||
|
||||
# 透過跳板機建立隧道
|
||||
transport = jump.get_transport()
|
||||
dest_addr = (_TARGET_HOST, 22)
|
||||
src_addr = (_JUMP_HOST, 0)
|
||||
chan = transport.open_channel("direct-tcpip", dest_addr, src_addr)
|
||||
|
||||
target = paramiko.SSHClient()
|
||||
target.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
target.connect(_TARGET_HOST, username=_TARGET_USER, sock=chan, timeout=15)
|
||||
|
||||
_stdin, stdout, stderr = target.exec_command(cmd, timeout=60)
|
||||
out = stdout.read().decode("utf-8", errors="replace").strip()
|
||||
err = stderr.read().decode("utf-8", errors="replace").strip()
|
||||
exit_code = stdout.channel.recv_exit_status()
|
||||
|
||||
target.close()
|
||||
jump.close()
|
||||
|
||||
if exit_code == 0:
|
||||
return True, out or "指令執行成功"
|
||||
else:
|
||||
return False, f"exit_code={exit_code}\n{err or out}"
|
||||
|
||||
except ImportError:
|
||||
# paramiko 尚未安裝，降級到 cli ssh
|
||||
sys_log.warning("[AutoHeal] paramiko 未安裝，改用 subprocess + CLI ssh")
|
||||
import subprocess
|
||||
full_cmd = [
|
||||
"ssh", "-o", "StrictHostKeyChecking=no",
|
||||
"-J", f"{_JUMP_USER}@{_JUMP_HOST}",
|
||||
f"{_TARGET_USER}@{_TARGET_HOST}", cmd,
|
||||
]
|
||||
result = subprocess.run(full_cmd, capture_output=True, text=True, timeout=60)
|
||||
if result.returncode == 0:
|
||||
return True, result.stdout.strip() or "指令執行成功"
|
||||
else:
|
||||
return False, result.stderr.strip() or result.stdout.strip()
|
||||
|
||||
except Exception as e:
|
||||
return False, f"SSH 執行例外: {e}"
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────
|
||||
# 核心引擎
|
||||
# ──────────────────────────────────────────────────────────
|
||||
|
||||
class AutoHealService:
|
||||
"""
|
||||
AIOps 自動修復引擎。
|
||||
|
||||
使用方式（在 scheduler.py 的 except 區塊）：
|
||||
from services.auto_heal_service import auto_heal_service
|
||||
auto_heal_service.handle_exception(
|
||||
task_name="run_auto_import_task",
|
||||
exception=e,
|
||||
traceback_str=traceback.format_exc()
|
||||
)
|
||||
"""
|
||||
|
||||
# ── 步驟 1：統一入口 ────────────────────────────────
|
||||
def handle_exception(self, task_name: str, exception: Exception,
|
||||
traceback_str: str = "") -> Optional[int]:
|
||||
"""
|
||||
統一例外處理入口。回傳 incident_id，若前置失敗則回傳 None。
|
||||
"""
|
||||
error_msg = str(exception)
|
||||
error_type, severity = _classify_error(error_msg)
|
||||
|
||||
sys_log.info(f"[AutoHeal] 收到例外 task={task_name} type={error_type} sev={severity}")
|
||||
|
||||
incident = self._create_incident(task_name, error_type, error_msg,
|
||||
traceback_str, severity)
|
||||
if not incident:
|
||||
return None
|
||||
|
||||
playbook = self._match_playbook(incident)
|
||||
if not playbook:
|
||||
sys_log.info(f"[AutoHeal] 未找到匹配 PlayBook (incident_id={incident.id})")
|
||||
self._notify_no_playbook(incident)
|
||||
return incident.id
|
||||
|
||||
heal_log = self._execute_playbook(incident, playbook)
|
||||
self._sink_to_km(incident, playbook, heal_log)
|
||||
self._notify_telegram(incident, playbook, heal_log)
|
||||
return incident.id
|
||||
|
||||
# ── 步驟 2：建立 Incident ───────────────────────────
|
||||
def _create_incident(self, task_name: str, error_type: str, error_msg: str,
|
||||
traceback_str: str, severity: str) -> Optional[Incident]:
|
||||
session = get_session()
|
||||
try:
|
||||
incident = Incident(
|
||||
task_name = task_name,
|
||||
error_type = error_type,
|
||||
error_message = error_msg[:2000], # 限制長度
|
||||
error_traceback = traceback_str[:5000],
|
||||
severity = severity,
|
||||
status = "open",
|
||||
created_at = datetime.now(),
|
||||
updated_at = datetime.now(),
|
||||
)
|
||||
session.add(incident)
|
||||
session.commit()
|
||||
sys_log.info(f"[AutoHeal] 建立 Incident id={incident.id} type={error_type}")
|
||||
return incident
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
sys_log.error(f"[AutoHeal] create_incident 失敗: {e}")
|
||||
return None
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
# ── 步驟 3：PlayBook 匹配 ───────────────────────────
|
||||
def _match_playbook(self, incident: Incident) -> Optional[Playbook]:
|
||||
"""
|
||||
匹配邏輯：
|
||||
1. error_type 精確比對
|
||||
2. match_pattern 任一關鍵字命中
|
||||
3. 冷卻時間檢查（同 playbook 最近一次執行是否已超過 cooldown_min）
|
||||
"""
|
||||
session = get_session()
|
||||
try:
|
||||
candidates = session.query(Playbook).filter_by(
|
||||
error_type=incident.error_type, is_active=True
|
||||
).all()
|
||||
|
||||
error_lower = incident.error_message.lower()
|
||||
|
||||
for pb in candidates:
|
||||
patterns = pb.get_match_patterns()
|
||||
if not any(p.lower() in error_lower for p in patterns):
|
||||
continue
|
||||
|
||||
# 冷卻檢查
|
||||
cooldown_threshold = datetime.now() - timedelta(minutes=pb.cooldown_min)
|
||||
recent_log = session.query(HealLog).filter(
|
||||
HealLog.playbook_id == pb.id,
|
||||
HealLog.created_at >= cooldown_threshold,
|
||||
HealLog.result == "success",
|
||||
).first()
|
||||
if recent_log:
|
||||
sys_log.info(f"[AutoHeal] PlayBook '{pb.name}' 在冷卻中，略過")
|
||||
continue
|
||||
|
||||
# 上限檢查（同 incident 的 retry_count）
|
||||
if incident.retry_count >= pb.max_retries:
|
||||
sys_log.warning(f"[AutoHeal] 已達 max_retries({pb.max_retries})，升級為 escalated")
|
||||
self._escalate_incident(incident)
|
||||
return None
|
||||
|
||||
sys_log.info(f"[AutoHeal] 匹配 PlayBook: '{pb.name}' (id={pb.id})")
|
||||
return pb
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
sys_log.error(f"[AutoHeal] match_playbook 失敗: {e}")
|
||||
return None
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
# ── 步驟 4：執行 PlayBook ───────────────────────────
|
||||
def _execute_playbook(self, incident: Incident, playbook: Playbook) -> HealLog:
|
||||
"""根據 action_type 執行對應動作，回傳 HealLog"""
|
||||
t_start = time.time()
|
||||
params = playbook.get_action_params()
|
||||
action_detail = ""
|
||||
result = "failed"
|
||||
result_output = ""
|
||||
|
||||
# 更新 incident 狀態
|
||||
self._update_incident_status(incident.id, "healing", playbook.id)
|
||||
|
||||
try:
|
||||
if playbook.action_type == "DOCKER_RESTART":
|
||||
container = params.get("container", "")
|
||||
use_compose = params.get("compose", False)
|
||||
if use_compose:
|
||||
cmd = f"cd /home/ollama/momo-pro && docker compose restart {container}"
|
||||
else:
|
||||
cmd = f"docker restart {container}"
|
||||
action_detail = cmd
|
||||
ok, output = _execute_ssh_cmd(cmd if not use_compose else f"docker compose restart {container}")
|
||||
# compose 指令需要在目錄下執行，強制用 SSH
|
||||
if use_compose:
|
||||
ok, output = _execute_ssh_cmd(f"docker restart {container}")
|
||||
result = "success" if ok else "failed"
|
||||
result_output = output
|
||||
|
||||
elif playbook.action_type == "SSH_CMD":
|
||||
cmd = params.get("cmd", "")
|
||||
action_detail = cmd
|
||||
ok, output = _execute_ssh_cmd(cmd)
|
||||
result = "success" if ok else "failed"
|
||||
result_output = output
|
||||
|
||||
elif playbook.action_type == "ALERT_ONLY":
|
||||
msg = params.get("message", "需人工介入")
|
||||
action_detail = f"[ALERT_ONLY] {msg}"
|
||||
result = "success"
|
||||
result_output = msg
|
||||
|
||||
elif playbook.action_type == "WAIT_RETRY":
|
||||
wait_min = params.get("wait_minutes", 30)
|
||||
action_detail = f"[WAIT_RETRY] 靜默等待 {wait_min} 分鐘後由排程自動重試"
|
||||
result = "success"
|
||||
result_output = f"已記錄，排程將在 {wait_min} 分鐘後重試"
|
||||
|
||||
else:
|
||||
action_detail = f"未知 action_type: {playbook.action_type}"
|
||||
result = "skipped"
|
||||
result_output = action_detail
|
||||
|
||||
except Exception as e:
|
||||
result = "failed"
|
||||
result_output = f"執行例外: {e}"
|
||||
sys_log.error(f"[AutoHeal] execute_playbook 例外: {e}")
|
||||
|
||||
duration_ms = (time.time() - t_start) * 1000
|
||||
heal_log = self._write_heal_log(
|
||||
incident.id, playbook.id,
|
||||
playbook.action_type, action_detail,
|
||||
result, result_output, duration_ms,
|
||||
)
|
||||
|
||||
# 更新 PlayBook 統計
|
||||
self._update_playbook_stats(playbook.id, result)
|
||||
|
||||
# 更新 Incident 最終狀態
|
||||
final_status = "resolved" if result == "success" else "open"
|
||||
self._update_incident_status(incident.id, final_status, playbook.id,
|
||||
increment_retry=(result != "success"))
|
||||
|
||||
sys_log.info(f"[AutoHeal] 執行完成 result={result} duration={duration_ms:.0f}ms")
|
||||
return heal_log
|
||||
|
||||
# ── 步驟 5：寫入 HealLog ────────────────────────────
|
||||
def _write_heal_log(self, incident_id, playbook_id, action_type,
|
||||
action_detail, result, result_output, duration_ms) -> HealLog:
|
||||
session = get_session()
|
||||
try:
|
||||
hl = HealLog(
|
||||
incident_id = incident_id,
|
||||
playbook_id = playbook_id,
|
||||
action_type = action_type,
|
||||
action_detail = action_detail,
|
||||
result = result,
|
||||
result_output = (result_output or "")[:2000],
|
||||
duration_ms = duration_ms,
|
||||
created_at = datetime.now(),
|
||||
)
|
||||
session.add(hl)
|
||||
session.commit()
|
||||
return hl
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
sys_log.error(f"[AutoHeal] write_heal_log 失敗: {e}")
|
||||
return HealLog(result=result, action_detail=action_detail)
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
# ── 步驟 6：KM 沉澱 ────────────────────────────────
|
||||
def _sink_to_km(self, incident: Incident, playbook: Playbook, heal_log: HealLog) -> None:
|
||||
"""將修復知識寫入 ai_insights（KM RAG 雙寫）"""
|
||||
try:
|
||||
from services.openclaw_learning_service import store_insight
|
||||
today = datetime.now().strftime("%Y-%m-%d")
|
||||
result_zh = {"success": "成功", "failed": "失敗", "skipped": "跳過"}.get(
|
||||
heal_log.result, heal_log.result
|
||||
)
|
||||
|
||||
content = (
|
||||
f"[AIOps 自動修復紀錄]\n"
|
||||
f"事件：{incident.task_name} 發生 {incident.error_type}（嚴重度 {incident.severity}）\n"
|
||||
f"症狀：{incident.error_message[:300]}\n"
|
||||
f"行動：執行 PlayBook「{playbook.name}」→ {heal_log.action_detail}\n"
|
||||
f"結果：{result_zh}（耗時 {heal_log.duration_ms:.0f}ms）\n"
|
||||
f"教訓：此類型錯誤（{incident.error_type}）可透過 {playbook.action_type} 自動修復。\n"
|
||||
f"處理時間：{today}"
|
||||
)
|
||||
|
||||
store_insight(
|
||||
insight_type = "auto_heal_playbook",
|
||||
period = today,
|
||||
content = content,
|
||||
metadata = {
|
||||
"playbook_id": playbook.id,
|
||||
"incident_id": incident.id,
|
||||
"error_type": incident.error_type,
|
||||
"result": heal_log.result,
|
||||
},
|
||||
ai_model = "auto_heal_engine_v1",
|
||||
)
|
||||
sys_log.info(f"[AutoHeal] KM 沉澱完成 (incident_id={incident.id})")
|
||||
except Exception as e:
|
||||
sys_log.warning(f"[AutoHeal] sink_to_km 失敗（不影響主流程）: {e}")
|
||||
|
||||
# ── 步驟 7：Telegram 通知 ───────────────────────────
|
||||
def _notify_telegram(self, incident: Incident, playbook: Playbook,
|
||||
heal_log: HealLog) -> None:
|
||||
"""推播修復結果通知"""
|
||||
icon = {"success": "✅", "failed": "❌", "skipped": "⏭️"}.get(heal_log.result, "❓")
|
||||
sev_icon = {"P1": "🔴", "P2": "🟠", "P3": "🟡"}.get(incident.severity, "⚪")
|
||||
|
||||
msg = (
|
||||
f"{sev_icon} <b>[EwoooC AIOps] 自動修復報告</b>\n\n"
|
||||
f"📌 任務：<code>{incident.task_name}</code>\n"
|
||||
f"🚨 錯誤類型：<code>{incident.error_type}</code>\n"
|
||||
f"📝 症狀：{incident.error_message[:200]}\n\n"
|
||||
f"🔧 PlayBook：{playbook.name}\n"
|
||||
f"⚙️ 動作：<code>{heal_log.action_detail}</code>\n"
|
||||
f"{icon} 結果：<b>{heal_log.result}</b>（{heal_log.duration_ms:.0f}ms）\n\n"
|
||||
f"💾 已沉澱至 KM（auto_heal_playbook）"
|
||||
)
|
||||
_send_telegram(msg)
|
||||
|
||||
def _notify_no_playbook(self, incident: Incident) -> None:
|
||||
"""未找到 PlayBook 時的人工告警"""
|
||||
sev_icon = {"P1": "🔴", "P2": "🟠", "P3": "🟡"}.get(incident.severity, "⚪")
|
||||
msg = (
|
||||
f"{sev_icon} <b>[EwoooC AIOps] 需人工介入</b>\n\n"
|
||||
f"📌 任務：<code>{incident.task_name}</code>\n"
|
||||
f"🚨 錯誤類型：<code>{incident.error_type}</code>\n"
|
||||
f"📝 症狀：{incident.error_message[:300]}\n\n"
|
||||
f"⚠️ 未找到匹配的 PlayBook，請人工排查。\n"
|
||||
f"🆔 Incident ID：{incident.id}"
|
||||
)
|
||||
_send_telegram(msg)
|
||||
|
||||
# ── 輔助函數 ────────────────────────────────────────
|
||||
def _update_incident_status(self, incident_id: int, status: str,
|
||||
playbook_id: Optional[int] = None,
|
||||
increment_retry: bool = False) -> None:
|
||||
session = get_session()
|
||||
try:
|
||||
inc = session.query(Incident).get(incident_id)
|
||||
if inc:
|
||||
inc.status = status
|
||||
inc.updated_at = datetime.now()
|
||||
if playbook_id:
|
||||
inc.playbook_id = playbook_id
|
||||
if status == "resolved":
|
||||
inc.resolved_at = datetime.now()
|
||||
if increment_retry:
|
||||
inc.retry_count = (inc.retry_count or 0) + 1
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
sys_log.error(f"[AutoHeal] update_incident_status 失敗: {e}")
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
def _escalate_incident(self, incident: Incident) -> None:
|
||||
self._update_incident_status(incident.id, "escalated")
|
||||
sev_icon = {"P1": "🔴", "P2": "🟠", "P3": "🟡"}.get(incident.severity, "⚪")
|
||||
msg = (
|
||||
f"{sev_icon} <b>[EwoooC AIOps] 告警升級 — 需立即人工介入</b>\n\n"
|
||||
f"📌 任務：<code>{incident.task_name}</code>\n"
|
||||
f"🚨 錯誤：<code>{incident.error_type}</code>\n"
|
||||
f"🔁 已重試 {incident.retry_count} 次，自動修復失敗。\n"
|
||||
f"📝 {incident.error_message[:300]}"
|
||||
)
|
||||
_send_telegram(msg)
|
||||
|
||||
def _update_playbook_stats(self, playbook_id: int, result: str) -> None:
|
||||
session = get_session()
|
||||
try:
|
||||
pb = session.query(Playbook).get(playbook_id)
|
||||
if pb:
|
||||
if result == "success":
|
||||
pb.success_count = (pb.success_count or 0) + 1
|
||||
else:
|
||||
pb.fail_count = (pb.fail_count or 0) + 1
|
||||
pb.updated_at = datetime.now()
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
sys_log.error(f"[AutoHeal] update_playbook_stats 失敗: {e}")
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
# ── 種子 PlayBook 初始化 ────────────────────────────
|
||||
@staticmethod
|
||||
def init_seed_playbooks() -> None:
|
||||
"""首次啟動時植入預設 PlayBook（已存在則略過）"""
|
||||
session = get_session()
|
||||
try:
|
||||
for seed in SEED_PLAYBOOKS:
|
||||
exists = session.query(Playbook).filter_by(name=seed["name"]).first()
|
||||
if not exists:
|
||||
session.add(Playbook(**seed))
|
||||
session.commit()
|
||||
sys_log.info("[AutoHeal] 種子 PlayBook 初始化完成")
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
sys_log.error(f"[AutoHeal] init_seed_playbooks 失敗: {e}")
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
# ─── 模組級單例 ─────────────────────────────────────────
|
||||
auto_heal_service = AutoHealService()
|
||||
Reference in New Issue
Block a user