feat(ai-ops): ADR-013 AIOps 自動修復閉環完整實作
Some checks failed
CD Pipeline / deploy (push) Failing after 3m24s
Some checks failed
CD Pipeline / deploy (push) Failing after 3m24s
架構(Exception → Incident → PlayBook → Heal → KM → Telegram): 新增元件: - database/autoheal_models.py: Incident/Playbook/HealLog 三張表 + 7 條種子 PlayBook - migrations/013_autoheal.sql: 建表 DDL + 種子資料(冪等 INSERT) - services/auto_heal_service.py: 核心引擎 7 步閉環 - _classify_error: 8 類錯誤自動分類 (DNS_FAIL/DB_UNREACHABLE/OOM/...) - _match_playbook: error_type + keyword + 冷卻 + max_retries 保護 - _execute_playbook: DOCKER_RESTART/SSH_CMD/ALERT_ONLY/WAIT_RETRY - _sink_to_km: 修復知識寫入 ai_insights (auto_heal_playbook) - SSH 白名單:僅允許 docker restart / compose restart / docker start 修改元件: - database/manager.py: _init_autoheal_tables() 啟動時建表+種子 PlayBook - scheduler.py: 3 個核心任務植入 handle_exception (run_auto_import_task / run_icaim_analysis_task / run_weekly_strategy_task) - requirements.txt: paramiko(SSH 跳板;不可用時降級 subprocess+CLI ssh) 安全設計: CMD 白名單 + cooldown + max_retries escalation + DB 冪等 migration Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
33
scheduler.py
33
scheduler.py
@@ -1574,6 +1574,17 @@ def run_auto_import_task():
|
||||
except Exception as notify_error:
|
||||
logging.error(f"[Scheduler] [AutoImport] ❌ LINE 通知失敗 | Error: {notify_error}")
|
||||
|
||||
# ADR-013: AIOps 自動修復 — PlayBook 匹配 + KM 沉澱
|
||||
try:
|
||||
from services.auto_heal_service import auto_heal_service
|
||||
auto_heal_service.handle_exception(
|
||||
task_name="run_auto_import_task",
|
||||
exception=e,
|
||||
traceback_str=_tb.format_exc(),
|
||||
)
|
||||
except Exception as _heal_e:
|
||||
logging.error(f"[Scheduler] [AutoImport] auto_heal_service 失敗: {_heal_e}")
|
||||
|
||||
def run_competitor_price_feeder_task():
|
||||
"""
|
||||
競品價格補給線排程任務(每 4 小時執行一次)
|
||||
@@ -1679,8 +1690,19 @@ def run_icaim_analysis_task():
|
||||
_save_stats('icaim_dispatch', {**dispatch_result, "status": "Success"})
|
||||
|
||||
except Exception as e:
|
||||
import traceback as _tb
|
||||
logging.error(f"[Scheduler] [ICAIM] 🚨 任務異常 | Error: {e}")
|
||||
_save_stats('icaim_analysis', {"status": "Failed", "error": str(e)})
|
||||
# ADR-013: AIOps 自動修復
|
||||
try:
|
||||
from services.auto_heal_service import auto_heal_service
|
||||
auto_heal_service.handle_exception(
|
||||
task_name="run_icaim_analysis_task",
|
||||
exception=e,
|
||||
traceback_str=_tb.format_exc(),
|
||||
)
|
||||
except Exception as _heal_e:
|
||||
logging.error(f"[Scheduler] [ICAIM] auto_heal_service 失敗: {_heal_e}")
|
||||
|
||||
|
||||
def run_weekly_strategy_task():
|
||||
@@ -1694,8 +1716,19 @@ def run_weekly_strategy_task():
|
||||
generate_weekly_strategy_report(force_tg_alert=True)
|
||||
logging.info("[Scheduler] [Strategy] ✅ Gemini 策略師週報任務完成")
|
||||
except Exception as e:
|
||||
import traceback as _tb
|
||||
logging.error(f"[Scheduler] [Strategy] 🚨 任務異常 | Error: {e}")
|
||||
_save_stats('weekly_strategy', {"status": "Failed", "error": str(e)})
|
||||
# ADR-013: AIOps 自動修復
|
||||
try:
|
||||
from services.auto_heal_service import auto_heal_service
|
||||
auto_heal_service.handle_exception(
|
||||
task_name="run_weekly_strategy_task",
|
||||
exception=e,
|
||||
traceback_str=_tb.format_exc(),
|
||||
)
|
||||
except Exception as _heal_e:
|
||||
logging.error(f"[Scheduler] [Strategy] auto_heal_service 失敗: {_heal_e}")
|
||||
|
||||
|
||||
def run_db_backup_task():
|
||||
|
||||
Reference in New Issue
Block a user