""" Auto Repair Service - #8 自動升級決策 ===================================== 高品質 Playbook 自動修復執行 Phase 8: 自動化層實作 建立時間: 2026-03-26 17:30 (台北時區) 建立者: Claude Code (#8 自動升級決策) 遵循 leWOOOgo 積木化原則: - Service 層只依賴 Repository/Service Interface - 不直接存取 Redis/DB - 封裝所有自動修復邏輯 觸發條件 (AND): 1. 有匹配且 APPROVED 的 Playbook 2. 通過 Service Registry、anti-pattern、MCP/Executor guardrail 3. 不是 critical break-glass / secret / destructive / stateful data-destroying route 安全邊界: - LOW/MEDIUM/HIGH controlled actions 可由 AI Agent 自動處理 - CRITICAL / break-glass / 不可逆資料動作仍需明確 break-glass gate """ from collections.abc import Callable from dataclasses import dataclass import re from typing import Any, Protocol from uuid import NAMESPACE_URL, UUID, uuid5 import structlog from src.models.incident import Incident, Severity from src.models.playbook import ( ActionType, Playbook, PlaybookStatus, RiskLevel, SymptomPattern, ) from src.services.anomaly_counter import AnomalyFrequency, get_anomaly_counter from src.services.executor import get_executor from src.services.global_repair_cooldown import ( check_global_repair_cooldown, record_global_repair_action, ) # Sprint 5.1: Service Registry Guardrail (ADR-062) from src.services.service_registry import StatefulLevel, get_service_registry from src.services.playbook_service import IPlaybookService, get_playbook_service logger = structlog.get_logger(__name__) # ============================================================================= # Types # ============================================================================= @dataclass class AutoRepairDecision: """自動修復決策結果""" can_auto_repair: bool playbook: Playbook | None = None reason: str = "" risk_level: RiskLevel = RiskLevel.MEDIUM blocked_by: str | None = None # 阻擋原因 (如 HIGH_RISK, P1_SEVERITY) # 2026-04-07 Claude Code: Sprint 4 B2 — 追蹤首次信任 is_cold_start: bool = False # 2026-04-08 Claude Code: 傳入 execute_auto_repair 供 DB 記錄 similarity_score: float | None = None @dataclass class AutoRepairResult: """自動修復執行結果""" success: bool playbook_id: str incident_id: str executed_steps: list[str] error: str | None = None execution_time_ms: int = 0 @dataclass(frozen=True) class _SshMcpRoute: """Route a legacy SSH playbook command to a governed MCP tool.""" tool_name: str params: dict[str, Any] required_scope: str = "read" _SHORT_HOST_MAP: dict[str, str] = { "110": "192.168.0.110", "120": "192.168.0.120", "121": "192.168.0.121", "188": "192.168.0.188", } _SSH_DIAGNOSTIC_KEYWORDS = ( "ps aux", "docker stats", "docker inspect", "docker logs", "docker ps", "docker top", "df -h", "du -", "free -h", "journalctl", "systemctl show", "tail ", "top ", "uptime", ) _SSH_WRITE_KEYWORDS = ( "docker restart", "docker start", "docker stop", "docker rm", "docker prune", "systemctl restart", "systemctl stop", "systemctl start", "truncate ", " rm ", "rm -", "certbot renew", "bash ", ) _AUTO_REPAIR_GATEWAY_AGENT_ID = "auto_repair_executor" _AUTO_REPAIR_GATEWAY_PROJECT_ID = "awoooi" _AUTO_REPAIR_GATEWAY_APPROVAL_TTL_SECONDS = 600 _SAFE_DOCKER_CONTAINER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") _UNSAFE_LEGACY_WRITE_PATTERN = re.compile(r"[;|<>`\n]|(\$\{|\$\()") _EXTERNAL_SITE_ALERTNAMES = { "MoWoooWorkDown", "TsenyangWebsiteDown", "StockWoooWorkDown", "BitanWoooWorkDown", "ExternalSiteDown", "WebsiteDown", "BlackboxProbeFailed", } _K3S_NODE_ALERTNAMES = {"K3sNodeDown", "K3sVIPDown"} def _build_verification_action_taken(playbook_id: str, executed_steps: list[str]) -> str: """Build a compact verifier action string with the real executed step shape.""" base = f"auto_repair:{playbook_id}" if not executed_steps: return base joined = " | ".join(step.strip() for step in executed_steps if step.strip()) joined = re.sub(r"\s+", " ", joined) if not joined: return base return f"{base} steps={joined[:900]}" # ============================================================================= # Auto Repair Service Interface # ============================================================================= class IAutoRepairService(Protocol): """自動修復服務介面""" async def evaluate_auto_repair( self, incident: Incident, ) -> AutoRepairDecision: """ 評估是否可自動修復 Args: incident: 待處理的 Incident Returns: AutoRepairDecision: 決策結果 """ ... async def execute_auto_repair( self, incident: Incident, playbook: Playbook, run_post_verification: bool = True, ) -> AutoRepairResult: """ 執行自動修復 Args: incident: 待處理的 Incident playbook: 要執行的 Playbook Returns: AutoRepairResult: 執行結果 """ ... # ============================================================================= # Auto Repair Service Implementation # ============================================================================= class AutoRepairService: """ 自動修復服務實作 職責: - 評估 Incident 是否可自動修復 - 執行高品質 Playbook - 更新執行統計 """ # === 安全邊界常數 === # 2026-06-26 ogt + Codex: 統帥明確授權 low/medium/high 直接自動化處理。 # 風險門檻不再阻擋 LOW/MEDIUM/HIGH；CRITICAL 仍是 break-glass 硬閘。 MAX_AUTO_REPAIR_RISK = RiskLevel.MEDIUM # 保留供日後參考，不再用於阻擋 MAX_AUTO_REPAIR_SEVERITY = Severity.P0 # 保留供日後參考，不再用於阻擋 MIN_SIMILARITY_SCORE = 0.0 # 🔴 已取消門檻 COLD_START_TRUST_MAX_EXECUTIONS = 3 # 保留供參考 COLD_START_TRUST_DAILY_LIMIT = 5 # 保留供參考 def __init__( self, playbook_service: IPlaybookService | None = None, cooldown_checker: Callable | None = None, ): # 2026-04-01 ogt: 注入 cooldown_checker 支援測試隔離 (DI 原則) self._playbook_service = playbook_service or get_playbook_service() self._cooldown_checker = cooldown_checker or check_global_repair_cooldown # 2026-04-04 Claude Code: Phase 25 P1 — 持有 runbook_generator task 引用，防 GC 回收 import asyncio self._pending_tasks: set[asyncio.Task] = set() async def drain_pending_tasks(self, timeout: float = 60.0) -> dict: """K8s rolling restart 時優雅等待所有背景任務完成。 # 2026-04-27 Wave8-X3 by Claude — B25/B26 drain fix 在 lifespan shutdown 中呼叫，確保 _verify_and_learn / runbook_generator 等 fire-and-forget task 在 SIGTERM 後仍有機會寫入 trust_score / runbook。 """ import asyncio as _asyncio if not self._pending_tasks: return {"drained": 0, "timeout": False} pending_count = len(self._pending_tasks) logger.info( "auto_repair_draining_pending_tasks", count=pending_count, timeout=timeout, ) try: done, still_pending = await _asyncio.wait( self._pending_tasks, timeout=timeout, return_when=_asyncio.ALL_COMPLETED, ) return { "drained": len(done), "still_pending": len(still_pending), "timeout": len(still_pending) > 0, } except Exception as e: logger.exception("drain_pending_tasks_failed", error=str(e)) return {"drained": 0, "still_pending": pending_count, "error": str(e)} async def evaluate_auto_repair( self, incident: Incident, ) -> AutoRepairDecision: """ 評估是否可自動修復 決策流程: 1. 全域熔斷 / Service Registry / anti-pattern guardrail 2. 從 Playbook 找匹配項 3. 確認 Playbook APPROVED 且有真正 mutating steps 4. LOW / MEDIUM / HIGH 不再因風險或 P0/P1 severity 阻擋 """ logger.info( "auto_repair_evaluate_start", incident_id=incident.incident_id, severity=incident.severity.value if incident.severity else None, ) # 0. 全域熔斷檢查（ADR-039 最優先） can_repair, cooldown_reason = await self._cooldown_checker( incident_id=incident.incident_id, affected_services=incident.affected_services or [], ) if not can_repair: logger.warning( "auto_repair_blocked_global_cooldown", incident_id=incident.incident_id, reason=cooldown_reason, ) return AutoRepairDecision( can_auto_repair=False, reason=cooldown_reason, blocked_by="GLOBAL_GUARDRAIL", ) # 0.5 Sprint 5.1 Guardrail: Service Registry 服務分級檢查 # (2026-04-08 Claude Sonnet 4.6 Asia/Taipei，ADR-062) # 全域熔斷之後、嚴重度之前，BLOCK 等級直接拒絕 # 保守原則：Registry 讀取失敗也 block（優先安全，不放行） try: _registry = get_service_registry() _service_name = (incident.target_resource or "") if hasattr(incident, "target_resource") else "" if not _service_name and incident.affected_services: _service_name = incident.affected_services[0] _stateful_level = _registry.get_stateful_level(_service_name) if _stateful_level == StatefulLevel.BLOCK: logger.warning( "auto_repair_blocked_guardrail", incident_id=incident.incident_id, service_name=_service_name, stateful_level="BLOCK", ) return AutoRepairDecision( can_auto_repair=False, reason=f"GUARDRAIL_BLOCK: 服務 '{_service_name}' 屬於禁止自動修復清單（資料安全，見 service-registry.yaml）", blocked_by="SERVICE_REGISTRY_BLOCK", ) except Exception as _guardrail_err: # S1-3 修正: Registry 失敗時保守拒絕，不允許穿透（ADR-062 審查修正 2026-04-08） logger.error("guardrail_check_failed", error=str(_guardrail_err)) return AutoRepairDecision( can_auto_repair=False, reason="Guardrail Service Registry 讀取異常，保守拒絕自動修復", blocked_by="GUARDRAIL_ERROR", ) # 2. 提取症狀模式 symptoms = self._extract_symptoms(incident) # 2.1 2026-04-04 Claude Code: Phase 25 P1 — Anti-Pattern 閘門 # 根據確定性 hash 比對近 7 天失敗案例，避免 AI 在同一個坑重複摔倒 try: from src.services.knowledge_service import get_knowledge_service symptoms_hash = symptoms.compute_hash() anti_patterns = await get_knowledge_service().check_anti_pattern( symptoms_hash, days=7 ) if anti_patterns: ap = anti_patterns[0] logger.warning( "auto_repair_blocked_anti_pattern", incident_id=incident.incident_id, symptoms_hash=symptoms_hash, anti_pattern_id=ap.id, anti_pattern_title=ap.title, ) return AutoRepairDecision( can_auto_repair=False, reason=f"過去 7 天有失敗案例: {ap.title}", blocked_by="ANTI_PATTERN", ) except Exception as _ap_e: # Anti-Pattern 閘門失敗不阻塞主流程（僅記錄） logger.warning("anti_pattern_gate_error", error=str(_ap_e)) symptoms_hash = "" # 3. 找匹配的 Playbook recommendations = await self._playbook_service.get_recommendations( symptoms=symptoms, top_k=3, ) if not recommendations: logger.info( "auto_repair_no_playbook_match", incident_id=incident.incident_id, ) return AutoRepairDecision( can_auto_repair=False, reason="未找到匹配的 Playbook", blocked_by="NO_MATCH", ) # 4. 檢查最佳匹配 best_match = self._select_best_recommendation(recommendations, symptoms) if best_match is not recommendations[0]: logger.warning( "auto_repair_exact_match_prioritized", incident_id=incident.incident_id, selected_playbook_id=best_match.playbook.playbook_id, original_playbook_id=recommendations[0].playbook.playbook_id, selected_similarity=best_match.similarity_score, original_similarity=recommendations[0].similarity_score, ) # 2026-06-26 ogt + Codex: 統帥已放寬 low/medium/high 受控自動化。 # 移除: 相似度門檻、is_high_quality 門檻、冷啟動機制、P0/P1 severity gate。 # CRITICAL / break-glass / 不可逆資料動作仍由硬閘阻擋。 max_risk = self._get_max_risk_level(best_match.playbook) _is_cold_start = False if ( self._is_external_site_incident(incident) and self._playbook_is_k3s_node_repair(best_match.playbook) ): logger.warning( "auto_repair_blocked_external_site_k3s_playbook", incident_id=incident.incident_id, playbook_id=best_match.playbook.playbook_id, alert_category=getattr(incident, "alert_category", None), alert_names=symptoms.alert_names, ) return AutoRepairDecision( can_auto_repair=False, playbook=best_match.playbook, reason=( "外部網站探測告警不得執行 K3s node PlayBook；" "需使用外部站台規則、Docker 站台修復或 AI 受控接手" ), risk_level=max_risk, blocked_by="EXTERNAL_SITE_K3S_PLAYBOOK", similarity_score=best_match.similarity_score, ) # 只保留: Playbook 必須是 APPROVED 狀態 if best_match.playbook.status != PlaybookStatus.APPROVED: return AutoRepairDecision( can_auto_repair=False, playbook=best_match.playbook, reason=f"Playbook 狀態為 {best_match.playbook.status.value}，必須是 APPROVED", blocked_by="NOT_APPROVED", ) if max_risk == RiskLevel.CRITICAL: return AutoRepairDecision( can_auto_repair=False, playbook=best_match.playbook, reason=( "CRITICAL / break-glass PlayBook 不屬於 low / medium / high " "受控自動化授權範圍" ), risk_level=max_risk, blocked_by="CRITICAL_BREAK_GLASS", ) if self._is_host_or_backup_incident(incident) and self._playbook_has_k8s_steps(best_match.playbook): logger.warning( "auto_repair_blocked_host_backup_k8s_playbook", incident_id=incident.incident_id, playbook_id=best_match.playbook.playbook_id, alert_category=getattr(incident, "alert_category", None), ) return AutoRepairDecision( can_auto_repair=False, playbook=best_match.playbook, reason=( "主機/備份類告警禁止執行 K8s Playbook；" "需改走 SSH 診斷或緊急介入" ), blocked_by="HOST_BACKUP_K8S_PLAYBOOK", ) if not self._playbook_has_mutating_steps(best_match.playbook): logger.warning( "auto_repair_blocked_observe_only_playbook", incident_id=incident.incident_id, playbook_id=best_match.playbook.playbook_id, ) return AutoRepairDecision( can_auto_repair=False, playbook=best_match.playbook, reason=( "PlayBook 只有診斷 / 觀測步驟，不能宣稱自動修復；" "請轉成 Work Item 補真正修復步驟或 gated Ansible apply" ), risk_level=max_risk or RiskLevel.MEDIUM, blocked_by="OBSERVE_ONLY_PLAYBOOK", ) # 5. 可以自動修復 logger.info( "auto_repair_approved", incident_id=incident.incident_id, playbook_id=best_match.playbook.playbook_id, similarity=best_match.similarity_score, success_rate=best_match.playbook.success_rate, ) return AutoRepairDecision( can_auto_repair=True, playbook=best_match.playbook, reason=f"匹配 Playbook: {best_match.playbook.name} (相似度 {best_match.similarity_score:.0%})", risk_level=max_risk, is_cold_start=_is_cold_start, similarity_score=best_match.similarity_score, ) async def execute_auto_repair( self, incident: Incident, playbook: Playbook, is_cold_start: bool = False, similarity_score: float | None = None, run_post_verification: bool = True, ) -> AutoRepairResult: """ 執行自動修復 流程: 1. 依序執行 Playbook 中的 repair_steps 2. 記錄執行結果到 DB (auto_repair_executions) 3. 更新 Playbook 統計 4. 記錄處置類型 (Sprint 4 B1/B2) """ import time start_time = time.perf_counter() executed_steps: list[str] = [] logger.info( "auto_repair_execute_start", incident_id=incident.incident_id, playbook_id=playbook.playbook_id, steps_count=len(playbook.repair_steps), ) # ADR-039: 記錄全域修復計數（用於熔斷檢查） await record_global_repair_action() try: # 執行每個步驟 for step in playbook.repair_steps: # 2026-04-07 Claude Code: 統帥指令「直接全部跳成自動修復」 # 移除 step-level 風險門檻，所有步驟直接執行 # 執行步驟 step_result = await self._execute_step(incident, step) executed_steps.append( f"Step {step.step_number}: {step.command[:50]}... -> {step_result}" ) if self._is_step_failure_result(step_result): raise RuntimeError(f"Step {step.step_number} failed: {step_result}") # 更新 Playbook 統計 await self._playbook_service.record_execution( playbook_id=playbook.playbook_id, success=True, ) execution_time = int((time.perf_counter() - start_time) * 1000) logger.info( "auto_repair_execute_success", incident_id=incident.incident_id, playbook_id=playbook.playbook_id, executed_steps=len(executed_steps), execution_time_ms=execution_time, ) repair_result = AutoRepairResult( success=True, playbook_id=playbook.playbook_id, incident_id=incident.incident_id, executed_steps=executed_steps, execution_time_ms=execution_time, ) # 2026-04-08 Claude Code: 統帥指令「所有操作都必須被記錄，寫入資料庫」 try: from src.repositories.audit_log_repository import get_auto_repair_execution_repository max_risk = self._get_max_risk_level(playbook) await get_auto_repair_execution_repository().create( incident_id=incident.incident_id, playbook_id=playbook.playbook_id, playbook_name=playbook.name, success=True, executed_steps=executed_steps, triggered_by="cold_start_trust" if is_cold_start else "auto_repair", similarity_score=similarity_score, risk_level=max_risk.value if max_risk else None, execution_time_ms=execution_time, ) except Exception as _db_e: logger.error("auto_repair_db_write_failed", error=str(_db_e)) self._record_auto_repair_metric(playbook, success=True) # 2026-04-07 Claude Code: Sprint 4 B1/B2 — 記錄處置類型 # P0-1 Fix: 統一使用 AnomalyCounter.hash_signature() try: from src.services.anomaly_counter import get_anomaly_counter counter = get_anomaly_counter() anomaly_key = self._derive_anomaly_key(incident) if anomaly_key: disposition_type = "cold_start_trust" if is_cold_start else "auto_repair" await counter.record_disposition(anomaly_key, disposition_type) except Exception as _disp_e: logger.warning("disposition_record_failed", error=str(_disp_e)) # 2026-04-04 Claude Code: Phase 25 P1 — 成功修復後 fire-and-forget 生成 AUTO_RUNBOOK try: from src.services.runbook_generator import get_runbook_generator symptoms = self._extract_symptoms(incident) symptoms_hash = symptoms.compute_hash() gen = get_runbook_generator() import asyncio as _asyncio task = _asyncio.create_task( gen.generate_runbook(incident, playbook, repair_result, symptoms_hash) ) self._pending_tasks.add(task) if hasattr(self, "_pending_tasks") else None task.add_done_callback( lambda t: self._pending_tasks.discard(t) if hasattr(self, "_pending_tasks") else None ) except Exception as _rg_e: logger.warning("runbook_generator_task_failed", error=str(_rg_e)) # 2026-04-26 Wave4 P1.3+P1.4 by Claude Engineer-B3 — 飛輪閉環最後一哩 # 成功執行後，fire-and-forget 啟動後執行驗證 + EWMA 學習回饋 # verifier 有 10s warmup + 30s timeout，不能阻塞在主路徑 try: import asyncio as _asyncio from src.services.post_execution_verifier import get_post_execution_verifier from src.services.learning_service import get_learning_service _action_taken = _build_verification_action_taken( playbook.playbook_id, executed_steps, ) _verifier = get_post_execution_verifier() _learning = get_learning_service() async def _verify_and_learn() -> None: try: verification_result = await _verifier.verify( incident=incident, snapshot=None, action_taken=_action_taken, ) await _learning.record_verification_result( incident_id=incident.incident_id, action_taken=_action_taken, verification_result=verification_result, matched_playbook_id=playbook.playbook_id, ) logger.info( "auto_repair_verify_and_learn_done", incident_id=incident.incident_id, playbook_id=playbook.playbook_id, verification_result=verification_result, ) # 2026-04-27 P3.1-T1 by Claude — 三 Tier-1 服務整合 # PostExecutionVerifier 判斷失敗/降級 → 觸發自動 Rollback if verification_result in ("failed", "degraded"): if self._should_escalate_failed_verification(incident, playbook): await self._escalate_failed_verification( incident=incident, playbook=playbook, verification_result=verification_result, ) return try: from src.services.rollback_manager import get_rollback_manager from src.services.declarative_remediation import DeclarativeRemediation from src.core.metrics import ROLLBACK_EXECUTED_TOTAL # 從 Incident 推導 target / namespace / action _rb_target = (incident.affected_services or ["unknown"])[0] _rb_ns = "awoooi-prod" _rb_action = f"kubectl rollout restart deployment/{_rb_target} -n {_rb_ns}" _spec = DeclarativeRemediation().evaluate( action=_rb_action, target=_rb_target, namespace=_rb_ns, ) rollback_mgr = get_rollback_manager() rollback_result = await rollback_mgr.trigger( incident_id=incident.incident_id, spec=_spec, verification_result=verification_result, ) _rb_status = "success" if rollback_result.success else "failed" _rb_reason = "converged" if rollback_result.convergence_confirmed else ( "no_previous_revision" if rollback_result.error and "revision" in (rollback_result.error or "") else "error" ) ROLLBACK_EXECUTED_TOTAL.labels( status=_rb_status, reason=_rb_reason ).inc() logger.info( "auto_rollback_triggered", incident_id=incident.incident_id, rollback_success=rollback_result.success, convergence_confirmed=rollback_result.convergence_confirmed, rollback_error=rollback_result.error, ) except Exception as _rb_e: logger.exception( "auto_rollback_failed", incident_id=incident.incident_id, error=str(_rb_e), ) except Exception as _inner_e: logger.warning( "auto_repair_verify_and_learn_failed", incident_id=incident.incident_id, error=str(_inner_e), ) if run_post_verification: _vl_task = _asyncio.create_task(_verify_and_learn()) if hasattr(self, "_pending_tasks"): self._pending_tasks.add(_vl_task) _vl_task.add_done_callback(self._pending_tasks.discard) else: logger.info( "auto_repair_service_post_verify_delegated", incident_id=incident.incident_id, playbook_id=playbook.playbook_id, ) except Exception as _vl_e: logger.warning("auto_repair_verifier_setup_failed", error=str(_vl_e)) return repair_result except Exception as e: # 更新失敗統計 await self._playbook_service.record_execution( playbook_id=playbook.playbook_id, success=False, ) execution_time = int((time.perf_counter() - start_time) * 1000) logger.error( "auto_repair_execute_failed", incident_id=incident.incident_id, playbook_id=playbook.playbook_id, error=str(e), ) fail_result = AutoRepairResult( success=False, playbook_id=playbook.playbook_id, incident_id=incident.incident_id, executed_steps=executed_steps, error=str(e), execution_time_ms=execution_time, ) # 2026-04-08 Claude Code: 失敗也必須寫入 DB try: from src.repositories.audit_log_repository import get_auto_repair_execution_repository max_risk = self._get_max_risk_level(playbook) await get_auto_repair_execution_repository().create( incident_id=incident.incident_id, playbook_id=playbook.playbook_id, playbook_name=playbook.name, success=False, executed_steps=executed_steps, error_message=str(e), triggered_by="cold_start_trust" if is_cold_start else "auto_repair", similarity_score=similarity_score, risk_level=max_risk.value if max_risk else None, execution_time_ms=execution_time, ) except Exception as _db_e: logger.error("auto_repair_db_write_failed", error=str(_db_e)) self._record_auto_repair_metric(playbook, success=False) # 2026-04-04 Claude Code: Phase 25 P1 — 失敗修復後 fire-and-forget 生成 ANTI_PATTERN # 2026-04-05 Claude Code: I1 修正 — 補齊 _pending_tasks GC 防護（對稱化） try: from src.services.runbook_generator import get_runbook_generator import asyncio as _asyncio symptoms = self._extract_symptoms(incident) symptoms_hash = symptoms.compute_hash() gen = get_runbook_generator() _ap_task = _asyncio.create_task( gen.generate_anti_pattern(incident, playbook, fail_result, symptoms_hash) ) self._pending_tasks.add(_ap_task) _ap_task.add_done_callback(self._pending_tasks.discard) except Exception as _ap_e: logger.warning("anti_pattern_task_failed", error=str(_ap_e)) return fail_result # === Private Helpers === @staticmethod def _derive_anomaly_key(incident: Incident) -> str | None: """ 從 Incident 推導 anomaly_key。 2026-04-07 Claude Code: I1+S1 Fix — 委託 AnomalyCounter.derive_key_from_incident() """ from src.services.anomaly_counter import AnomalyCounter return AnomalyCounter.derive_key_from_incident(incident) def _extract_symptoms(self, incident: Incident) -> SymptomPattern: """從 Incident 提取症狀模式""" alert_names = [] keywords = [] if incident.signals: for signal in incident.signals: # 優先用 labels["alertname"]（原始 Prometheus alertname）， # fallback 到 signal.alert_name（可能是 "custom" 等類別值） # (2026-04-09 Claude Sonnet 4.6 Asia/Taipei, L7 E2E 修正) raw_alertname = signal.labels.get("alertname") if signal.labels else None alert_names.append(raw_alertname or signal.alert_name) # 從 annotations 提取關鍵字 if signal.annotations: for value in signal.annotations.values(): if isinstance(value, str) and len(value) < 50: keywords.append(value) return SymptomPattern( alert_names=alert_names, affected_services=incident.affected_services or [], severity_range=[incident.severity.value] if incident.severity else ["P2"], keywords=keywords[:10], ) def _select_best_recommendation( self, recommendations, symptoms: SymptomPattern, ): """Prefer deterministic alert/service matches over fuzzy similarity only. A higher fuzzy score must not outrank a playbook that explicitly names the firing alert or affected service. Live-fire T16 proved that this can route a safe K8s canary into an unrelated host diagnostic playbook. """ symptom_alerts = {str(name) for name in (symptoms.alert_names or []) if name} symptom_services = { str(service) for service in (symptoms.affected_services or []) if service } def _priority(recommendation) -> tuple[int, int, float]: pattern = recommendation.playbook.symptom_pattern playbook_alerts = { str(name) for name in (pattern.alert_names or []) if name } playbook_services = { str(service) for service in (pattern.affected_services or []) if service } alert_exact = int(bool(symptom_alerts & playbook_alerts)) service_exact = int(bool(symptom_services & playbook_services)) return (alert_exact, service_exact, float(recommendation.similarity_score or 0.0)) return max(recommendations, key=_priority) @staticmethod def _is_step_failure_result(step_result: str) -> bool: """Treat executor-declared failures as failed auto-repair executions.""" normalized = (step_result or "").strip().upper() return normalized.startswith("FAILED:") or normalized == "UNKNOWN_ACTION_TYPE" def _get_max_risk_level(self, playbook: Playbook) -> RiskLevel: """取得 Playbook 中最高的風險等級""" risk_order = { RiskLevel.LOW: 0, RiskLevel.MEDIUM: 1, RiskLevel.HIGH: 2, RiskLevel.CRITICAL: 3, } max_risk = RiskLevel.LOW for step in playbook.repair_steps: if risk_order.get(step.risk_level, 0) > risk_order.get(max_risk, 0): max_risk = step.risk_level return max_risk def _record_auto_repair_metric(self, playbook: Playbook, success: bool) -> None: """把實際 auto-repair 執行寫入 Prometheus 指標。 2026-05-06 ogt + Codex：DB 已有 auto_repair_executions，但 core.metrics.record_auto_repair() 長期零 caller，導致治理/心跳用 Prometheus 看起來像「飛輪沒做事」。label 使用 action_type，避免 playbook_id 造成高基數。 """ try: from src.core.metrics import record_auto_repair first_step = playbook.repair_steps[0] if playbook.repair_steps else None action = first_step.action_type.value if first_step else "unknown" max_risk = self._get_max_risk_level(playbook) tier = { RiskLevel.LOW: 1, RiskLevel.MEDIUM: 2, RiskLevel.HIGH: 3, RiskLevel.CRITICAL: 4, }.get(max_risk, 0) record_auto_repair(action=action, tier=tier, success=success) except Exception as e: logger.warning( "auto_repair_metric_record_failed", playbook_id=playbook.playbook_id, success=success, error=str(e), ) def _is_host_or_backup_incident(self, incident: Incident) -> bool: """主機/備份類事件只能走 SSH/只讀診斷，不允許 K8s rollout 類修復。""" category = (getattr(incident, "alert_category", None) or "").lower() if category in {"host_resource", "backup_failure"}: return True for signal in incident.signals or []: labels = signal.labels or {} alertname = str(labels.get("alertname") or signal.alert_name or "") if alertname.startswith("HostBackup") or alertname.startswith("Host"): return True return False def _is_external_site_incident(self, incident: Incident) -> bool: """外部網站探測告警不可被 K3s node 類 PlayBook fuzzy match 搶走。""" category = (getattr(incident, "alert_category", None) or "").lower() if category in {"external_site", "service_404"}: return True for signal in incident.signals or []: labels = signal.labels or {} alertname = str(labels.get("alertname") or signal.alert_name or "") layer = str(labels.get("layer") or "").lower() if alertname in _EXTERNAL_SITE_ALERTNAMES or layer == "external": return True return False def _playbook_has_k8s_steps(self, playbook: Playbook) -> bool: """檢查 Playbook 是否包含 K8s 指令，避免主機告警誤執行 deployment 操作。""" for step in playbook.repair_steps: command = (step.command or "").strip().lower() if step.action_type == ActionType.KUBECTL or command.startswith("kubectl "): return True return False def _playbook_has_mutating_steps(self, playbook: Playbook) -> bool: """Return true only when a PlayBook contains an actual repair action.""" for step in playbook.repair_steps: command = (step.command or "").strip().lower() if not command or step.action_type == ActionType.MANUAL: continue if step.action_type == ActionType.KUBECTL or command.startswith("kubectl "): if not command.startswith(( "kubectl get ", "kubectl describe ", "kubectl logs ", "kubectl top ", "kubectl explain ", )): return True continue if step.action_type == ActionType.SSH_COMMAND: if command.startswith(("openclaw://", "ansible://")): return True if any(token in command for token in _SSH_WRITE_KEYWORDS): return True continue if step.action_type == ActionType.SCRIPT: return True return False def _playbook_is_k3s_node_repair(self, playbook: Playbook) -> bool: """K3s node repair must only run for actual K3s node alerts.""" pattern_alerts = { str(alert) for alert in (playbook.symptom_pattern.alert_names or []) if alert } if pattern_alerts & _K3S_NODE_ALERTNAMES: return True for step in playbook.repair_steps: command = (step.command or "").strip().lower() if "kubectl get nodes" in command or "kubectl describe node" in command: return True return False def _should_escalate_failed_verification(self, incident: Incident, playbook: Playbook) -> bool: """非 K8s 修復或主機/備份事件驗證失敗時，禁止合成 K8s rollback。""" return self._is_host_or_backup_incident(incident) or not self._playbook_has_k8s_steps(playbook) async def _escalate_failed_verification( self, *, incident: Incident, playbook: Playbook, verification_result: str, ) -> None: """Post-verification failed but rollback is unsafe; notify emergency channel.""" target = (incident.affected_services or ["unknown"])[0] namespace = "awoooi-prod" alert_type = self._incident_alert_type(incident) reason = ( f"auto repair playbook {playbook.playbook_id} verification={verification_result}; " "rollback is unsafe for host/backup or non-K8s remediation" ) logger.warning( "auto_repair_verification_failed_emergency", incident_id=incident.incident_id, playbook_id=playbook.playbook_id, verification_result=verification_result, target=target, ) try: from src.services.emergency_escalation_service import ( escalate_auto_repair_unavailable, ) await escalate_auto_repair_unavailable( incident_id=incident.incident_id, approval_id=None, alert_type=alert_type, target_resource=target, namespace=namespace, failure_reason=reason, attempted_actions=( f"auto_repair:{playbook.playbook_id} -> verifier:{verification_result} " "-> emergency_intervention" ), ) except Exception as exc: logger.warning( "auto_repair_verification_emergency_failed", incident_id=incident.incident_id, playbook_id=playbook.playbook_id, error=str(exc), ) def _incident_alert_type(self, incident: Incident) -> str: """Best-effort alertname for emergency cards.""" for signal in incident.signals or []: labels = signal.labels or {} alertname = labels.get("alertname") or signal.alert_name if alertname: return str(alertname) return "AutoRepairVerificationFailed" def _risk_exceeds_threshold(self, risk: RiskLevel) -> bool: """檢查風險是否超過自動修復門檻""" high_risks = {RiskLevel.HIGH, RiskLevel.CRITICAL} return risk in high_risks async def _check_cold_start_daily_limit(self) -> bool: """ 檢查今日首次信任自動修復次數是否在限額內。 使用 Redis counter，key 含日期，自動過期。 2026-04-07 Claude Code: 方案 C — 冷啟動每日上限防護 """ try: from src.core.redis_client import get_redis redis = await get_redis() if redis is None: # Redis 不可用 → 保守拒絕 return False from src.utils.timezone import now_taipei today_key = f"cold_start_trust:{now_taipei().strftime('%Y-%m-%d')}" count = await redis.incr(today_key) # 首次建立 key 時設定過期 (25 小時，確保跨日清理) if count == 1: await redis.expire(today_key, 90000) if count > self.COLD_START_TRUST_DAILY_LIMIT: logger.warning( "cold_start_daily_limit_reached", today_key=today_key, count=count, limit=self.COLD_START_TRUST_DAILY_LIMIT, ) return False return True except Exception as e: logger.warning("cold_start_daily_limit_check_failed", error=str(e)) # 安全降級：檢查失敗 → 保守拒絕 return False def _route_legacy_ssh_command_to_mcp( self, incident: Incident, command: str, ) -> _SshMcpRoute | None: """Map read-only legacy ``ssh {host} '...'`` steps to MCP Gateway. YAML_RULE playbooks predate the URI executor and can contain compound shell diagnostics. Those commands should not bypass the newer scheme-based HostRepairAgent or loosen its shell safety guard; read-only diagnostics are instead routed to the governed SSH MCP provider. """ raw_command = (command or "").strip() lowered = raw_command.lower() if not lowered.startswith("ssh "): return None if any(token in lowered for token in _SSH_WRITE_KEYWORDS): return None if not any(token in lowered for token in _SSH_DIAGNOSTIC_KEYWORDS): return None host = self._resolve_ssh_host_for_incident(incident, raw_command) if not host: return None params: dict[str, Any] = {"host": host} container_name = self._resolve_container_name_for_incident(incident, raw_command) if container_name: params["container_name"] = container_name return _SshMcpRoute(tool_name="ssh_diagnose", params=params) def _route_legacy_ssh_write_command_to_mcp( self, incident: Incident, command: str, ) -> _SshMcpRoute | None: """Map safe legacy Docker restart steps to the write-scoped MCP tool. This intentionally supports only the historical pattern used by DockerContainerUnhealthy playbooks: ``ssh {host} 'docker inspect {container} ... && docker restart {container}'``. Broader shell, command substitution, exporter discovery, and systemd fallbacks stay blocked so they can move through controlled review. """ raw_command = (command or "").strip() lowered = raw_command.lower() if not lowered.startswith("ssh ") or "docker restart" not in lowered: return None if _UNSAFE_LEGACY_WRITE_PATTERN.search(raw_command): return None host = self._resolve_ssh_host_for_incident(incident, raw_command) if not host: return None container_name = ( self._resolve_container_name_for_incident(incident, raw_command) or self._extract_docker_restart_container(raw_command) ) if not self._is_safe_docker_container_name(container_name): return None if not self._has_simple_docker_restart_tail(raw_command, container_name): return None return _SshMcpRoute( tool_name="ssh_docker_restart", params={ "host": host, "container_name": container_name, "trust_score": 0.85, }, required_scope="write", ) def _upstream_repair_success_result( self, incident: Incident, command: str, ) -> str | None: """Treat trusted upstream host repair labels as a successful no-op. Some host monitors already perform the Docker restart before sending the alert. Re-running the restart from AwoooP would create duplicate disruption; the flywheel should record that the execute stage converged and leave evidence in the execution row instead. """ if "docker restart" not in (command or "").lower(): return None labels = self._incident_labels(incident) repair_result = str(labels.get("repair_result") or "").strip().lower() repair_action = str(labels.get("repair_action") or "").strip().lower() if repair_result not in {"success", "succeeded", "ok"}: return None if repair_action not in {"restart", "restarted", "docker_restart", "docker-restart"}: return None container_name = self._resolve_container_name_for_incident(incident, command) source = str(labels.get("source") or "upstream_monitor").strip() or "upstream_monitor" target = f" container={container_name}" if container_name else "" return ( "SUCCESS: upstream repair already completed " f"(source={source} action={repair_action} result={repair_result}{target})" ) def preview_read_only_ssh_mcp_route( self, incident: Incident, command: str, ) -> dict[str, Any] | None: """Preview whether a legacy SSH diagnostic can use the MCP Gateway. This is used by remediation dry-runs to prove the supported executor path without running the original PlayBook step or writing an execution result. """ route = self._route_legacy_ssh_command_to_mcp(incident, command) if route is None: return None return { "tool_name": route.tool_name, "params": route.params, "agent_id": "auto_repair_executor", "required_scope": "read", "flywheel_node": "execute", } def preview_write_ssh_mcp_route( self, incident: Incident, command: str, ) -> dict[str, Any] | None: """Preview whether a legacy SSH repair can use the write MCP Gateway. This mirrors the executor's narrow Docker restart routing without projecting a Gate 5 approval or mutating any runtime state. It lets ADR-100 Work Items show whether ``ready_for_replay`` is actually ready for a governed write executor. """ route = self._route_legacy_ssh_write_command_to_mcp(incident, command) if route is None: return None return { "tool_name": route.tool_name, "params": route.params, "agent_id": "auto_repair_executor", "required_scope": route.required_scope, "flywheel_node": "execute", } def _resolve_ssh_host_for_incident(self, incident: Incident, command: str) -> str: """Resolve ``{host}``, short host labels, and exporter instance ports.""" labels = self._incident_labels(incident) raw_host = "" match = re.match(r"ssh\s+([^\s'\"]+)", command.strip(), flags=re.IGNORECASE) if match: raw_host = match.group(1) if not raw_host or "{" in raw_host or "}" in raw_host: raw_host = ( str(labels.get("host") or "") or str(labels.get("instance") or "") or str(labels.get("node") or "") or str(labels.get("exported_instance") or "") ) return self._normalize_ssh_host(raw_host) @staticmethod def _normalize_ssh_host(raw_host: str) -> str: host = (raw_host or "").strip() if host.startswith("ssh://"): host = host.removeprefix("ssh://") if "@" in host: host = host.rsplit("@", 1)[1] if host.startswith("[") and "]" in host: host = host[1:host.index("]")] if host.count(":") == 1: maybe_host, maybe_port = host.rsplit(":", 1) if maybe_port.isdigit(): host = maybe_host if host in _SHORT_HOST_MAP: return _SHORT_HOST_MAP[host] match = re.fullmatch(r"(?:node-exporter-|host-)?(110|120|121|188)", host) if match: return _SHORT_HOST_MAP[match.group(1)] return host def _resolve_container_name_for_incident( self, incident: Incident, command: str, ) -> str: labels = self._incident_labels(incident) for key in ("container_name", "container", "name"): value = str(labels.get(key) or "").strip() if value and "{" not in value and "}" not in value: return value match = re.search( r"docker\s+(?:stats\s+--no-stream|inspect|logs|top|restart|ps\s+-a\s+--filter\s+name=)\s+([a-zA-Z0-9._-]+)", command, ) return match.group(1) if match else "" @staticmethod def _extract_docker_restart_container(command: str) -> str: match = re.search( r"docker\s+restart\s+([A-Za-z0-9][A-Za-z0-9_.-]{0,127})(?:\s*['\"])?\s*$", command, flags=re.IGNORECASE, ) return match.group(1) if match else "" @staticmethod def _is_safe_docker_container_name(container_name: str) -> bool: return bool(container_name and _SAFE_DOCKER_CONTAINER_RE.fullmatch(container_name)) @staticmethod def _has_simple_docker_restart_tail(command: str, container_name: str) -> bool: target = rf"(?:{re.escape(container_name)}|\{{container\}}|\{{target\}})" return bool( re.search( rf"docker\s+restart\s+{target}(?:\s*['\"])?\s*$", command, flags=re.IGNORECASE, ) ) @staticmethod def _incident_labels(incident: Incident) -> dict[str, Any]: for signal in incident.signals or []: labels = getattr(signal, "labels", None) if labels: return labels return {} async def _execute_ssh_mcp_route( self, incident: Incident, route: _SshMcpRoute, *, approved: bool = True, ) -> str: """Execute a routed SSH diagnostic through AwoooP MCP Gateway.""" try: from src.db.base import get_db_context from src.plugins.mcp.gateway import GatewayContext, McpGateway, McpGatewayError from src.services.mcp_audit_context import with_mcp_audit_context incident_id = incident.incident_id run_id: UUID | None = None if route.required_scope != "read": if not approved: return ( f"FAILED: mcp:{route.tool_name} approval required for " f"{route.required_scope} scope" ) run_id = self._mcp_auto_repair_run_id(incident_id, route) approval_projected = await self._project_auto_repair_mcp_approval( route=route, run_id=run_id, ) if not approval_projected: return f"FAILED: mcp:{route.tool_name} approval projection failed" params = with_mcp_audit_context( route.params, session_id=f"incident:{incident_id}:auto_repair_execute", incident_id=incident_id, flywheel_node="execute", agent_role=_AUTO_REPAIR_GATEWAY_AGENT_ID, ) async with get_db_context("awoooi") as db: ctx = GatewayContext( project_id=_AUTO_REPAIR_GATEWAY_PROJECT_ID, agent_id=_AUTO_REPAIR_GATEWAY_AGENT_ID, tool_name=route.tool_name, run_id=run_id, trace_id=incident_id, is_shadow=False, environment={"env": "prod"}, required_scope=route.required_scope, ) result = await McpGateway(db).call(ctx, params) except McpGatewayError as exc: return f"FAILED: mcp:{route.tool_name} {exc.error_code}: {exc}" except Exception as exc: logger.warning( "auto_repair_ssh_mcp_route_failed", incident_id=incident.incident_id, tool=route.tool_name, error=str(exc), ) return f"FAILED: mcp:{route.tool_name} {exc}" if result.success: preview = str(result.output or "")[:500] return f"SUCCESS: mcp:{route.tool_name} {preview}".strip() return f"FAILED: mcp:{route.tool_name} {result.error or 'execution failed'}" @staticmethod def _mcp_auto_repair_run_id(incident_id: str, route: _SshMcpRoute) -> UUID: stable_payload = ( f"{_AUTO_REPAIR_GATEWAY_PROJECT_ID}:" f"{_AUTO_REPAIR_GATEWAY_AGENT_ID}:" f"{route.tool_name}:" f"{incident_id}:" f"{route.params.get('host', '')}:" f"{route.params.get('container_name', '')}" ) return uuid5(NAMESPACE_URL, stable_payload) async def _project_auto_repair_mcp_approval( self, route: _SshMcpRoute, run_id: UUID, ) -> bool: """Project policy-approved auto repair into Gateway's Gate 5 key.""" try: from src.core.redis_client import get_redis redis = get_redis() approval_key = ( f"mcp_approval:{_AUTO_REPAIR_GATEWAY_PROJECT_ID}:" f"{_AUTO_REPAIR_GATEWAY_AGENT_ID}:{route.tool_name}:{run_id}" ) await redis.set( approval_key, "approved:auto_repair_policy", ex=_AUTO_REPAIR_GATEWAY_APPROVAL_TTL_SECONDS, ) logger.info( "auto_repair_mcp_approval_projected", tool=route.tool_name, required_scope=route.required_scope, run_id=str(run_id), ) return True except Exception as exc: logger.warning( "auto_repair_mcp_approval_projection_failed", tool=route.tool_name, required_scope=route.required_scope, run_id=str(run_id), error=str(exc), ) return False async def _execute_step(self, incident: Incident, step) -> str: """ 執行單一修復步驟 目前整合: - kubectl 命令: 透過 ActionExecutor - script: 透過 subprocess - manual: 跳過 (需人工) """ if step.action_type == ActionType.MANUAL: return "SKIPPED (manual step)" if step.action_type == ActionType.KUBECTL: # 整合 ActionExecutor try: executor = get_executor() # 替換 {target} 為實際目標 command = step.command if incident.affected_services: command = command.replace("{target}", incident.affected_services[0]) result = await executor.execute_kubectl_command(command) return "SUCCESS" if result.success else f"FAILED: {result.error}" except ImportError: logger.warning("action_executor_not_available") return "SKIPPED (executor not available)" # 2026-04-06 Claude Code: Sprint 3 — repair_by_uri (URI scheme 路由) if step.action_type == ActionType.SSH_COMMAND: upstream_success = self._upstream_repair_success_result(incident, step.command) if upstream_success is not None: return upstream_success route = self._route_legacy_ssh_command_to_mcp(incident, step.command) if route is not None: return await self._execute_ssh_mcp_route(incident, route) approved = not getattr(step, "requires_approval", False) route = self._route_legacy_ssh_write_command_to_mcp(incident, step.command) if route is not None: return await self._execute_ssh_mcp_route( incident, route, approved=approved, ) from src.services.host_repair_agent import HostRepairAgent agent = HostRepairAgent() result = await agent.repair_by_uri(step.command, approved=approved) if result.success: return f"SUCCESS: {result.output}" else: return f"FAILED: {result.error}" return "UNKNOWN_ACTION_TYPE" # === ADR-037: Tier-based Repair (2026-03-29) === # Tier 分級動作映射 TIER_ACTIONS = { 1: ["restart_pod", "restart_container"], # 臨時修復 2: ["scale_up", "increase_memory", "adjust_limits"], # 緩解修復 3: ["apply_hotfix", "update_config", "patch_deployment"], # 根因修復 4: ["create_issue", "notify_team", "schedule_fix"], # 架構修復 } async def determine_repair_tier( self, anomaly_key: str, frequency: AnomalyFrequency, ) -> int: """ 根據頻率決定修復 Tier (ADR-037) 統帥指示 (2026-03-29): - "重啟只是治標，不是治本！太常發生的異常必須徹底解決" - 根據異常頻率和修復歷史決定應該嘗試的修復層級 Returns: 1: 臨時修復 (重啟) 2: 緩解修復 (擴容) 3: 根因修復 (配置變更) 4: 架構修復 (需開發) """ # 取得修復歷史 counter = get_anomaly_counter() stats = await counter.get_all_repair_stats(anomaly_key) # 計算重啟次數 restart_count = stats.get("restart_pod", {}).get("total", 0) restart_count += stats.get("restart_container", {}).get("total", 0) # Tier 決策邏輯 if frequency.permanent_fix_applied: # 已有永久修復但仍出問題 → 需架構級修復 logger.info( "tier_decision", anomaly_key=anomaly_key, tier=4, reason="permanent_fix_still_failing", ) return 4 if frequency.escalation_level == "PERMANENT_FIX": # 24h 內 ≥10 次 → 根因修復 logger.info( "tier_decision", anomaly_key=anomaly_key, tier=3, reason="escalation_permanent_fix", ) return 3 if frequency.escalation_level == "ESCALATE": # 24h 內 ≥5 次 → 緩解修復 logger.info( "tier_decision", anomaly_key=anomaly_key, tier=2, reason="escalation_escalate", ) return 2 if restart_count >= 2: # 已重啟 2 次 → 升級到緩解 logger.info( "tier_decision", anomaly_key=anomaly_key, tier=2, reason=f"restart_count_{restart_count}", ) return 2 # 預設臨時修復 return 1 def get_tier_actions(self, tier: int) -> list[str]: """ 根據 Tier 返回可用修復動作 (ADR-037) """ return self.TIER_ACTIONS.get(tier, self.TIER_ACTIONS[1]) async def record_repair_result( self, anomaly_key: str, action: str, success: bool, tier: int = 1, ) -> None: """ 記錄修復結果到 AnomalyCounter (ADR-037) Args: anomaly_key: 異常 key action: 修復動作 success: 是否成功 tier: 修復 Tier """ counter = get_anomaly_counter() await counter.record_repair_attempt(anomaly_key, action, success) # 如果是 Tier 3 永久修復成功，標記已套用 if tier >= 3 and success: await counter.mark_permanent_fix_applied( anomaly_key=anomaly_key, fix_description=f"Tier {tier} repair: {action}", ) logger.info( "repair_result_recorded", anomaly_key=anomaly_key, action=action, success=success, tier=tier, ) # ============================================================================= # Singleton # ============================================================================= _service: AutoRepairService | None = None def get_auto_repair_service() -> IAutoRepairService: """取得 AutoRepairService 單例""" global _service if _service is None: _service = AutoRepairService() return _service def set_auto_repair_service(service: AutoRepairService | None) -> None: """注入 AutoRepairService 實例 (用於 DI 或測試)""" global _service _service = service