""" Incident Service - Phase 6.2 雙層記憶寫入 ========================================== 功能: - Working Memory (Redis): 活躍事件，7 天 TTL - Episodic Memory (PostgreSQL): 歷史事件，永久保留 設計原則: - 先寫 Redis (快)，再寫 PostgreSQL (持久) - 兩者都成功才算完成 - 失敗時記錄日誌但不中斷主流程 統帥鐵律: - 禁止硬編碼 IP 或密碼，嚴格讀取 .env - 所有寫入操作都必須有結構化日誌 C2 修正 (首席架構師審查 2026-04-10 Claude Sonnet 4.6 Asia/Taipei): create_incident_for_approval + _extract_affected_services 從 Router 層移入此 Service 層 原違規: 業務邏輯 (Severity 映射, Signal 建立, Incident 建立) 放在 api/v1/webhooks.py """ import json from datetime import UTC, datetime from typing import Any, Literal from uuid import UUID import structlog from src.core.redis_client import get_redis from src.db.base import get_db_context from src.db.models import IncidentRecord from src.models.incident import ( Incident, IncidentStatus, Severity, Signal, ) from src.utils.timezone import now_taipei logger = structlog.get_logger(__name__) # ============================================================================= # C2 修正: 從 webhooks.py 遷入的業務邏輯 # 2026-04-10 Claude Sonnet 4.6 Asia/Taipei # ============================================================================= # 風險等級 → 事件嚴重度映射 (原在 webhooks.py) _RISK_TO_SEVERITY = { "critical": Severity.P0, "high": Severity.P1, "medium": Severity.P2, "low": Severity.P3, } # I1 修正: 提升為 module-level frozenset，避免每次呼叫重建 (原在 webhooks.py 函數體內) _INFRA_JOB_NAMES: frozenset[str] = frozenset( j.lower().replace("-", "").replace("_", "") for j in {"node", "node-exporter", "pushgateway", "blackbox", "prometheus", "alertmanager", "cadvisor"} ) def extract_affected_services(labels: dict, target_resource: str) -> list[str]: """ 從告警 labels 提取真實服務名，防止 IP 或 alertname 污染 affected_services。 優先序: 1. component label（Docker-compose 層告警最可靠） 2. job label（排除 node-exporter / pushgateway 等基礎設施 job） 3. pod label（取 deployment name，去掉 hash suffix） 4. target_resource（不含冒號、不等於 alertname 時才採用） 5. 空列表（讓通用型 Playbook 透過空集合豁免規則匹配） Phase 1 飛輪修復 — 2026-04-10 Claude Sonnet 4.6 Asia/Taipei C2 修正: 從 api/v1/webhooks.py 移入 Service 層（純業務邏輯，無 I/O） """ alertname = labels.get("alertname", "") if comp := labels.get("component"): return [comp] if job := labels.get("job"): normalized = job.lower().replace("-", "").replace("_", "") if normalized not in _INFRA_JOB_NAMES: return [job] if pod := labels.get("pod"): parts = pod.rsplit("-", 2) if len(parts) >= 3 and len(parts[-1]) == 5 and len(parts[-2]) in (9, 10): return [parts[0]] elif len(parts) >= 2: return ["-".join(parts[:-1])] if (target_resource and ":" not in target_resource and target_resource != alertname and not target_resource[0].isdigit()): return [target_resource] return [] def classify_alert_early(alertname: str, severity: str, labels: dict | None = None) -> tuple[str, str]: """ ADR-073 Phase 2-2: 早期分診，在 LLM 分析前決定 alert_category + notification_type。 防止 HostBackupFailed 等告警被誤路由到 K8s executor。 規則優先順序（由高到低）: 1. ConfigurationDrift / KubeConfigDrift → TYPE-4D (Config Drift 卡片) 2. severity=info/none → TYPE-1 (純資訊，無按鈕) 3. backup/heartbeat 關鍵字 → TYPE-1 4. Docker/Host 前綴 → infrastructure TYPE-3 5. Kube/Pod/Deploy/Node/Velero/ArgoCD 前綴 → kubernetes TYPE-3 6. Postgres/Redis 前綴 → database TYPE-3 7. 預設 → general TYPE-3 C3 修正 (首席架構師 CR 2026-04-13): 從 Router 層 (webhooks.py) 移入 Service 層 原違規: 業務邏輯函數定義在 api/v1/webhooks.py Returns: tuple[str, str]: (alert_category, notification_type) """ # ADR-075 (2026-04-12): 完整重寫，修正 category 命名對齊 _build_inline_keyboard。 # 統帥決議：kubernetes（非 k8s_workload）、host_resource 從 infrastructure 分離。 alertname_lower = alertname.lower() # 1. Config Drift（最高優先，類型特殊不受 severity 影響） if alertname in ("ConfigurationDrift", "KubeConfigDrift"): return "config_drift", "TYPE-4D" # 2. 告警鏈路健康（meta-monitoring，優先於 severity 判斷） # 2026-04-12 ogt: 補入 NoAlertsReceived + PrometheusNotConnectedToAlertmanager if alertname in ( "AlertChainBroken_Alertmanager", "AlertChainBroken_Sentry", "NoAlertsReceived", "NoAlertsReceived2Hours", "AlertChainUnhealthy", "PrometheusNotConnectedToAlertmanager", ): return "alertchain_health", "TYPE-8M" # 3. 資安告警（高優先，防止被 severity/prefix 規則覆蓋） # ADR-075 TYPE-5S (2026-04-12 ogt) if any(alertname.startswith(p) for p in ( "UnauthorizedSSH", "KubeAudit", "CVECritical", "WAFAttack", "PodAbnormal", "SecurityBreach", )): return "secops", "TYPE-5S" # 4. 飛輪/AI 系統健康（優先於 severity 判斷） if alertname in ("AutoRepairLowSuccessRate", "PermanentFixRequired") or any( alertname.startswith(p) for p in ("Flywheel", "MCPProvider", "OllamaDown", "NemotronDown") ): return "flywheel_health", "TYPE-8M" # 4a. 業務/FinOps 告警（ADR-075 TYPE-6B） if any(alertname.startswith(p) for p in ( "AITokenCost", "GeminiAPIError", "SLOBurn", "APIErrorBudget", "MomoScraper", "ScraperSuccess", )): return "business", "TYPE-6B" # 5. 純資訊 if severity in ("info", "none"): return "info", "TYPE-1" # 5. Backup / Heartbeat — 純資訊，不進 LLM # HostBackupFailed 必須在 Host prefix 前攔截，否則被歸 host_resource/TYPE-3 # 2026-04-12 ogt: 只針對已知主機備份監控 alertname，不用寬泛關鍵字 # BackupJobFailed severity=warning 仍走 TYPE-3（見測試 test_backup_keyword_warning_not_type1） _BACKUP_TYPE1_NAMES = { "HostBackupFailed", "HostBackupStale", "HostBackupMissing", "BackupRestoreTestFailed", "BackupRestoreTestStale", } # 2026-04-12 ogt: 補入 DeadMansSwitch（HEARTBEAT_ALERT_NAMES 中但之前漏掉） if ( "watchdog" in alertname_lower or "deadmansswitch" in alertname_lower or alertname == "Heartbeat" or alertname in _BACKUP_TYPE1_NAMES or alertname.startswith("HostBackup") ): return "backup", "TYPE-1" # 6. 主機資源（從 infrastructure 分離，ADR-075 統帥決議） if alertname.startswith("Host"): return "host_resource", "TYPE-3" # 7. Docker 容器 if alertname.startswith("Docker"): return "infrastructure", "TYPE-3" # 8. K8s（統帥決議：統一用 kubernetes，不用 k8s_workload） if alertname.startswith(("Kube", "Pod", "Deploy", "Node", "Velero", "ArgoCD")): return "kubernetes", "TYPE-3" # 9. 資料庫 if alertname.startswith(("Postgres", "Redis")): return "database", "TYPE-3" # 10. 物件儲存 if alertname == "MinIODown": return "storage", "TYPE-3" # 11. DevOps 工具（ADR-075 修正：從 general 分離） if alertname in ( "OpenClawDown", "SignOzDown", "GiteaDown", "HarborDown", "SentryDown", "AlertmanagerDown", "KaliScannerDown", "GiteaCIPipelineFailed", ): return "devops_tool", "TYPE-3" # 12. 外部網站（ADR-075 修正：從 general 分離） if alertname in ( "MoWoooWorkDown", "TsenyangWebsiteDown", "StockWoooWorkDown", "BitanWoooWorkDown", ): return "external_site", "TYPE-3" # 13. SSL 憑證（ADR-075 修正：從 general 分離） # ≥14 天→TYPE-1（提醒，無需審核）；<14 天→TYPE-3（緊急審核） if alertname.startswith(("ExternalSiteSSL", "TLSCert")): days = int((labels or {}).get("days_remaining", 0)) if labels else 0 return "ssl_cert", ("TYPE-1" if days >= 14 else "TYPE-3") return "general", "TYPE-3" async def create_incident_for_approval( approval_id: str, risk_level: str, target_resource: str, namespace: str, alert_type: str, message: str, source: str = "alertmanager", alertname: str | None = None, alert_labels: dict | None = None, notification_type: str | None = None, # ADR-073 Phase 2-2 alert_category: str | None = None, # ADR-073 Phase 2-2 ) -> str: """ 為 Approval 創建對應的 Incident (活躍事件同步)。 設計原則: - Approval 和 Incident 必須同時存在 - Incident 存入 Redis (Working Memory) + PostgreSQL (Episodic Memory) - 7 天 TTL 自動過期 C2 修正: 從 api/v1/webhooks.py 移入 Service 層（業務邏輯不屬 Router 層） Returns: str: Incident ID """ incident_service = get_incident_service() severity = _RISK_TO_SEVERITY.get(risk_level.lower(), Severity.P2) _labels: dict = { "namespace": namespace, "resource": target_resource, "alertname": alertname or alert_type, **(alert_labels or {}), } signal = Signal( alert_name=alertname or alert_type, severity=severity, source=source, fired_at=now_taipei(), labels=_labels, annotations={"message": message}, ) _affected_services = extract_affected_services(_labels, target_resource) incident = Incident( status=IncidentStatus.INVESTIGATING, severity=severity, signals=[signal], affected_services=_affected_services, proposal_ids=[UUID(approval_id)], notification_type=notification_type, # ADR-073 Phase 2-2 alert_category=alert_category, # ADR-073 Phase 2-2 ) await incident_service.save_to_working_memory(incident) try: await incident_service.save_to_episodic_memory(incident) except Exception as _pg_err: logger.warning( "incident_episodic_memory_failed", incident_id=incident.incident_id, error=str(_pg_err), ) logger.info( "incident_created_for_approval", incident_id=incident.incident_id, approval_id=approval_id, severity=severity.value, target=target_resource, ) return incident.incident_id # ============================================================================= # Legacy Value Normalization (方案 C - 代碼相容舊格式) # ============================================================================= # 問題: Redis 有舊 Enum 值 (status='open', severity='critical') # 解法: 解析時正規化，不動 Redis 資料 # 回滾: git revert (秒級恢復) # ============================================================================= def normalize_status(value: str) -> str: """ 正規化 IncidentStatus 舊格式值 舊值 → 新值: - 'open' → 'investigating' """ legacy_map = { "open": "investigating", } return legacy_map.get(value, value) def normalize_severity(value: str) -> str: """ 正規化 Severity 舊格式值 舊值 → 新值: - 'critical' → 'P0' - 'high' → 'P1' - 'warning' → 'P2' - 'medium' → 'P2' - 'info' → 'P3' - 'low' → 'P3' - 'none' → 'P3' """ legacy_map = { "critical": "P0", "high": "P1", "warning": "P2", "medium": "P2", "info": "P3", "low": "P3", "none": "P3", } return legacy_map.get(value, value) # ============================================================================= # Constants # ============================================================================= # Redis Key Prefix INCIDENT_KEY_PREFIX = "incident:" # Working Memory TTL: 7 天 = 604800 秒 WORKING_MEMORY_TTL = 604800 # ============================================================================= # Incident Service # ============================================================================= class IncidentService: """ 雙層記憶服務 職責: 1. Working Memory (Redis): 活躍事件快取 2. Episodic Memory (PostgreSQL): 歷史事件持久化 使用方式: service = IncidentService() incident = await service.create_incident_from_signal(signal_data) """ # ========================================================================= # Working Memory (Redis) # ========================================================================= async def save_to_working_memory(self, incident: Incident) -> bool: """ 將 Incident 寫入 Working Memory (Redis) 使用 Redis Hash 儲存，Key 格式: incident:{incident_id} TTL: 7 天 (604800 秒) Returns: bool: 是否成功寫入 """ redis_client = get_redis() key = f"{INCIDENT_KEY_PREFIX}{incident.incident_id}" try: # 序列化為 JSON incident_json = incident.model_dump_json() # SET with TTL await redis_client.set( key, incident_json, ex=WORKING_MEMORY_TTL, ) logger.info( "working_memory_saved", incident_id=incident.incident_id, key=key, ttl_seconds=WORKING_MEMORY_TTL, ) return True except Exception as e: logger.exception( "working_memory_save_error", incident_id=incident.incident_id, error=str(e), ) return False async def get_from_working_memory(self, incident_id: str) -> Incident | None: """ 從 Working Memory 讀取 Incident 方案 C: 解析時正規化舊格式 Enum 值 Returns: Incident | None: 事件資料，若不存在則返回 None """ redis_client = get_redis() key = f"{INCIDENT_KEY_PREFIX}{incident_id}" try: data = await redis_client.get(key) if data is None: return None # 方案 C: 正規化舊格式 Enum 值 incident_dict = json.loads(data) if "status" in incident_dict: incident_dict["status"] = normalize_status(incident_dict["status"]) if "severity" in incident_dict: incident_dict["severity"] = normalize_severity(incident_dict["severity"]) # 同時正規化 signals 內的 severity for signal in incident_dict.get("signals", []): if "severity" in signal: signal["severity"] = normalize_severity(signal["severity"]) return Incident.model_validate(incident_dict) except Exception as e: logger.exception( "working_memory_get_error", incident_id=incident_id, error=str(e), ) return None async def get_active_incidents(self) -> list[Incident]: """ 列出所有活躍的 Incidents (從 Working Memory) 方案 C: 解析時正規化舊格式 Enum 值 Returns: list[Incident]: 活躍事件列表 (investigating 或 mitigating) """ redis_client = get_redis() incidents: list[Incident] = [] try: # SCAN 所有 incident:* keys async for key in redis_client.scan_iter( match=f"{INCIDENT_KEY_PREFIX}*", count=100, ): # 排除索引 keys if ":idx:" in key: continue data = await redis_client.get(key) if data is None: continue try: # 方案 C: 正規化舊格式 Enum 值 incident_dict = json.loads(data) if "status" in incident_dict: incident_dict["status"] = normalize_status(incident_dict["status"]) if "severity" in incident_dict: incident_dict["severity"] = normalize_severity(incident_dict["severity"]) # 正規化 signals 內的 severity for signal in incident_dict.get("signals", []): if "severity" in signal: signal["severity"] = normalize_severity(signal["severity"]) incident = Incident.model_validate(incident_dict) # 只返回活躍狀態的 Incident if incident.status in ( IncidentStatus.INVESTIGATING, IncidentStatus.MITIGATING, ): incidents.append(incident) except Exception as e: logger.warning( "incident_parse_error", key=key, error=str(e), ) continue logger.info( "get_active_incidents", count=len(incidents), ) return incidents except Exception as e: logger.exception( "get_active_incidents_error", error=str(e), ) return [] # ========================================================================= # Episodic Memory (PostgreSQL) # ========================================================================= async def save_to_episodic_memory(self, incident: Incident) -> bool: """ 將 Incident 寫入 Episodic Memory (PostgreSQL) 使用 SQLAlchemy async session 寫入 incidents 表。 Returns: bool: 是否成功寫入 """ try: async with get_db_context() as db: # 轉換為 SQLAlchemy model # 使用 model_dump(mode="json") 確保 datetime 正確序列化 # 從 signals 提取 alertname（ADR-073 Phase 2: incidents.alertname 欄位） _alertname = ( incident.signals[0].labels.get("alertname") or incident.signals[0].alert_name if incident.signals else None ) record = IncidentRecord( incident_id=incident.incident_id, status=incident.status.value, severity=incident.severity.value, signals=[ s.model_dump(mode="json") for s in incident.signals ], affected_services=incident.affected_services, decision_chain=( incident.decision_chain.model_dump(mode="json") if incident.decision_chain else None ), proposal_ids=[str(pid) for pid in incident.proposal_ids], outcome=( incident.outcome.model_dump(mode="json") if incident.outcome else None ), created_at=incident.created_at, updated_at=incident.updated_at, resolved_at=incident.resolved_at, closed_at=incident.closed_at, ttl_days=incident.ttl_days, vectorized=incident.vectorized, # ADR-073 Phase 2-2: 三個分類欄位（之前遺漏未寫入 DB） alertname=_alertname, notification_type=incident.notification_type, alert_category=incident.alert_category, ) db.add(record) # commit 由 get_db_context 自動處理 logger.info( "episodic_memory_saved", incident_id=incident.incident_id, table="incidents", ) return True except Exception as e: logger.exception( "episodic_memory_save_error", incident_id=incident.incident_id, error=str(e), ) return False async def get_from_episodic_memory(self, incident_id: str) -> Incident | None: """ 從 Episodic Memory 讀取 Incident Returns: Incident | None: 事件資料，若不存在則返回 None """ try: async with get_db_context() as db: from sqlalchemy import select stmt = select(IncidentRecord).where( IncidentRecord.incident_id == incident_id ) result = await db.execute(stmt) record = result.scalar_one_or_none() if record is None: return None # 轉換回 Pydantic model return self._record_to_incident(record) except Exception as e: logger.exception( "episodic_memory_get_error", incident_id=incident_id, error=str(e), ) return None def _record_to_incident(self, record: IncidentRecord) -> Incident: """ 將 SQLAlchemy record 轉換為 Pydantic Incident 方案 C: 解析時正規化舊格式 Enum 值 """ from src.models.incident import AIDecisionChain, IncidentOutcome # 方案 C: 正規化 signals 內的舊格式 severity signals = [] for s in (record.signals or []): signal_data = s.copy() if "severity" in signal_data: signal_data["severity"] = normalize_severity(signal_data["severity"]) signals.append(Signal(**signal_data)) decision_chain = ( AIDecisionChain(**record.decision_chain) if record.decision_chain else None ) outcome = ( IncidentOutcome(**record.outcome) if record.outcome else None ) # 方案 C: 正規化舊格式 Enum 值 normalized_status = normalize_status(record.status) normalized_severity = normalize_severity(record.severity) return Incident( incident_id=record.incident_id, status=IncidentStatus(normalized_status), severity=Severity(normalized_severity), signals=signals, affected_services=record.affected_services or [], decision_chain=decision_chain, proposal_ids=record.proposal_ids or [], outcome=outcome, created_at=record.created_at, updated_at=record.updated_at, resolved_at=record.resolved_at, closed_at=record.closed_at, ttl_days=record.ttl_days, persisted_to_pg=True, # 從 PG 讀取，必為 True vectorized=record.vectorized, ) # ========================================================================= # 雙層寫入核心邏輯 # ========================================================================= async def create_incident_from_signal( self, signal_data: dict[str, Any], frequency_stats: dict[str, Any] | None = None, ) -> Incident | None: """ 從 Signal 建立 Incident 並雙層寫入 Phase 6.2 核心邏輯: 1. 建立 Incident (含 Signal) 2. 寫入 Working Memory (Redis) - 7 天 TTL 3. 寫入 Episodic Memory (PostgreSQL) - 永久保留 4. 標記 persisted_to_pg = True Phase 21 (ADR-037) 擴展: 5. 含異常頻率統計 (用於 Tier 分級修復策略) Args: signal_data: 從 Redis Stream 收到的 Signal 資料 frequency_stats: ADR-037 異常頻率統計 (可選) Returns: Incident | None: 成功返回 Incident，失敗返回 None """ try: # 0. 去抖動 (Debounce) - 防止告警風暴 fingerprint = signal_data.get("fingerprint") if fingerprint: try: redis_client = get_redis() debounce_key = f"debounce:{fingerprint}" # SETNX 若成功表示是新的，給予 3 分鐘 TTL (180s) is_new = await redis_client.set(debounce_key, "1", ex=180, nx=True) if not is_new: logger.info( "incident_debounced", fingerprint=fingerprint, reason="Duplicate signal within 3 minutes", ) return None except Exception as e: logger.warning("incident_debounce_redis_error", error=str(e)) # 1. 解析 Signal signal = Signal( alert_name=signal_data.get("alert_name", "unknown"), severity=self._parse_severity(signal_data.get("severity", "warning")), source=self._parse_source(signal_data.get("source", "manual")), fired_at=datetime.now(UTC), labels=self._parse_dict(signal_data.get("labels", "{}")), annotations=self._parse_dict(signal_data.get("annotations", "{}")), fingerprint=signal_data.get("fingerprint"), ) # 2. 建立 Incident (含頻率統計) # ADR-037: 統帥指示「重啟只是治標，太常發生的異常必須徹底解決」 from src.models.incident import IncidentFrequencyStats freq_stats = None if frequency_stats: freq_stats = IncidentFrequencyStats( anomaly_key=frequency_stats.get("anomaly_key", "unknown"), count_1h=frequency_stats.get("count_1h", 0), count_24h=frequency_stats.get("count_24h", 0), count_7d=frequency_stats.get("count_7d", 0), count_30d=frequency_stats.get("count_30d", 0), escalation_level=frequency_stats.get("escalation_level"), auto_repair_count=frequency_stats.get("auto_repair_count", 0), ) incident = Incident( severity=signal.severity, signals=[signal], affected_services=[signal_data.get("target", "unknown")], frequency_stats=freq_stats, ) logger.info( "incident_created", incident_id=incident.incident_id, severity=incident.severity.value, signal_count=len(incident.signals), ) # 3. 寫入 Working Memory (Redis) redis_success = await self.save_to_working_memory(incident) # 4. 寫入 Episodic Memory (PostgreSQL) pg_success = await self.save_to_episodic_memory(incident) # 5. 更新狀態 if pg_success: incident.persisted_to_pg = True # 更新 Redis 中的狀態 if redis_success: await self.save_to_working_memory(incident) # 6. 記錄雙層寫入結果 logger.info( "dual_layer_memory_result", incident_id=incident.incident_id, redis_success=redis_success, pg_success=pg_success, persisted_to_pg=incident.persisted_to_pg, ) return incident except Exception as e: logger.exception( "create_incident_error", error=str(e), ) return None def _parse_source( self, source_str: str, ) -> Literal["prometheus", "signoz", "alertmanager", "manual", "telegram"]: """ 解析來源字串，映射到 Signal 允許的 Literal 值 不在白名單中的來源一律映射為 'manual' """ valid_sources = {"prometheus", "signoz", "alertmanager", "manual", "telegram"} if source_str.lower() in valid_sources: return source_str.lower() # type: ignore return "manual" def _parse_severity(self, severity_str: str) -> Severity: """解析嚴重度字串""" mapping = { "critical": Severity.P0, "high": Severity.P1, "warning": Severity.P2, "medium": Severity.P2, "low": Severity.P3, "info": Severity.P3, } return mapping.get(severity_str.lower(), Severity.P2) def _parse_dict(self, value: str | dict) -> dict[str, str]: """解析字典字串或字典""" if isinstance(value, dict): return {str(k): str(v) for k, v in value.items()} if isinstance(value, str): try: # 嘗試解析 JSON parsed = json.loads(value.replace("'", '"')) return {str(k): str(v) for k, v in parsed.items()} except (json.JSONDecodeError, TypeError): return {} return {} # ========================================================================= # Phase 17 P0: Router 層違規修復 - 新增方法 # ========================================================================= async def update_outcome( self, incident_id: str, effectiveness_score: int | None = None, human_feedback: str | None = None, learning_notes: str | None = None, should_remember: bool = True, ) -> Incident | None: """ 更新 Incident 的 outcome (人類回饋) Phase 17: 從 Router 層遷移至 Service 層 Args: incident_id: 事件 ID effectiveness_score: 有效性評分 (1-5) human_feedback: 文字回饋 learning_notes: 學習筆記 should_remember: 是否納入長期記憶 Returns: Incident | None: 更新後的事件，失敗返回 None """ from src.models.incident import IncidentOutcome from src.repositories.incident_repository import get_incident_repository from src.utils.timezone import now_taipei # 1. 從 Working Memory 讀取 incident = await self.get_from_working_memory(incident_id) if incident is None: logger.warning("incident_not_found_for_outcome", incident_id=incident_id) return None # 2. 更新 outcome if incident.outcome is None: incident.outcome = IncidentOutcome() if effectiveness_score is not None: incident.outcome.effectiveness_score = effectiveness_score if human_feedback is not None: incident.outcome.human_feedback = human_feedback if learning_notes is not None: incident.outcome.learning_notes = learning_notes incident.outcome.should_remember = should_remember incident.updated_at = now_taipei() # 3. 寫入 Working Memory redis_success = await self.save_to_working_memory(incident) if not redis_success: logger.error("outcome_redis_write_failed", incident_id=incident_id) return None # 4. 同步到 Episodic Memory (PostgreSQL) try: repo = get_incident_repository() await repo.update_outcome( incident_id=incident_id, outcome=incident.outcome.model_dump(mode="json"), updated_at=now_taipei(), ) logger.info("outcome_db_updated", incident_id=incident_id) except Exception as e: logger.warning( "outcome_db_update_failed", incident_id=incident_id, error=str(e), ) # DB 失敗不影響主流程 return incident async def resolve_incident(self, incident_id: str) -> Incident | None: """ 將 Incident 狀態更新為 RESOLVED Phase 17: 從 Router 層遷移至 Service 層 Args: incident_id: 事件 ID Returns: Incident | None: 更新後的事件，失敗返回 None """ from src.repositories.incident_repository import get_incident_repository from src.utils.timezone import now_taipei # 1. 從 Working Memory 讀取 incident = await self.get_from_working_memory(incident_id) if incident is None: logger.warning("incident_not_found_for_resolve", incident_id=incident_id) return None # 2. 更新狀態 incident.status = IncidentStatus.RESOLVED incident.resolved_at = now_taipei() incident.updated_at = now_taipei() # 3. 寫入 Working Memory redis_success = await self.save_to_working_memory(incident) if not redis_success: logger.error("resolve_redis_write_failed", incident_id=incident_id) return None # 4. 同步到 Episodic Memory try: repo = get_incident_repository() await repo.update_status( incident_id=incident_id, status="resolved", updated_at=now_taipei(), ) logger.info("resolve_db_updated", incident_id=incident_id) except Exception as e: logger.warning( "resolve_db_update_failed", incident_id=incident_id, error=str(e), ) # KB Phase 2-A: 自動萃取 KB 草稿 (fire-and-forget, 2026-04-03 ogt) try: import asyncio from src.services.knowledge_extractor_service import get_knowledge_extractor asyncio.create_task( get_knowledge_extractor().extract_from_incident(incident) ) except Exception: logger.exception("kb_extract_task_create_failed", incident_id=incident_id) # ADR-073 Phase 4-2: resolve 時觸發 KM conversion (2026-04-12 ogt) # 將已解決的 Incident 轉換為結構化 KM 條目，驅動飛輪學習固化節點 try: import asyncio from src.services.km_conversion_service import get_km_conversion_service asyncio.create_task( get_km_conversion_service().convert(incident) ) except Exception: logger.exception("km_conversion_task_create_failed", incident_id=incident_id) # 2026-04-07 Claude Code: Sprint 4 B4 — 手動處理推斷 # I1+S1 Fix: 委託 derive_key_from_incident() 統一推導 try: from src.services.anomaly_counter import AnomalyCounter, get_anomaly_counter counter = get_anomaly_counter() anomaly_key = AnomalyCounter.derive_key_from_incident(incident) if anomaly_key: disposition = await counter.get_disposition_stats(anomaly_key) has_system_resolution = ( disposition["auto_repair"] > 0 or disposition["human_approved"] > 0 or disposition["cold_start_trust"] > 0 ) if not has_system_resolution: await counter.record_disposition(anomaly_key, "manual_resolved") except Exception as _disp_e: logger.warning("disposition_manual_resolve_failed", error=str(_disp_e)) # MASTER Task 4.2 (2026-04-14 Claude Sonnet 4.6): Postmortem 自動組裝 # Incident duration > POSTMORTEM_MIN_DURATION_MINUTES(10min) 時自動生成 # 孤兒 report_generation_service.trigger_postmortem 本次接上 resolve 路徑 try: import asyncio from src.services.report_generation_service import get_report_generation_service alertname = ( incident.signals[0].labels.get("alertname", "UnknownAlert") if incident.signals else "UnknownAlert" ) title = f"{alertname} — {', '.join(incident.affected_services or ['N/A'])}" root_cause = None resolution_action = None ai_provider = None auto_repaired = False if incident.decision_chain: root_cause = incident.decision_chain.hypothesis ai_provider = incident.decision_chain.model_used if incident.outcome: resolution_action = (incident.outcome.learning_notes or None) auto_repaired = bool(incident.outcome.execution_success) asyncio.create_task( get_report_generation_service().trigger_postmortem( incident_id=incident.incident_id, title=title, created_at=incident.signals[0].fired_at if incident.signals else incident.resolved_at, resolved_at=incident.resolved_at, root_cause=root_cause, resolution_action=resolution_action, ai_provider=ai_provider, auto_repaired=auto_repaired, ) ) except Exception as _pm_e: logger.exception("postmortem_trigger_failed", incident_id=incident_id, error=str(_pm_e)) return incident async def find_by_proposal_id(self, proposal_id: str) -> Incident | None: """ 根據 proposal_id 查找關聯的 Incident Phase 17: 從 Router 層遷移至 Service 層 Args: proposal_id: 提案 ID (UUID 字串) Returns: Incident | None: 找到的事件，未找到返回 None """ from uuid import UUID redis_client = get_redis() try: target_uuid = UUID(proposal_id) async for key in redis_client.scan_iter( match=f"{INCIDENT_KEY_PREFIX}INC-*", count=100, ): data = await redis_client.get(key) if data is None: continue try: # 方案 C: 正規化舊格式 Enum 值 incident_dict = json.loads(data) if "status" in incident_dict: incident_dict["status"] = normalize_status(incident_dict["status"]) if "severity" in incident_dict: incident_dict["severity"] = normalize_severity(incident_dict["severity"]) for signal in incident_dict.get("signals", []): if "severity" in signal: signal["severity"] = normalize_severity(signal["severity"]) incident = Incident.model_validate(incident_dict) if target_uuid in incident.proposal_ids: return incident except Exception as e: logger.warning( "incident_parse_error_in_find", key=key, error=str(e), ) continue return None except Exception as e: logger.exception( "find_by_proposal_id_error", proposal_id=proposal_id, error=str(e), ) return None async def trigger_reanalysis(self, incident_id: str) -> dict: """ 觸發 Incident 重診 (ADR-050 P2: reanalyze button) 去重保護：同一 incident 10 分鐘內只觸發一次。 觸發後將 incident status 標記為 analyzing，等待 AI 自動接手。 Args: incident_id: Incident ID Returns: dict: { "triggered": bool, "message": str, "already_analyzing": bool, } 2026-04-01 Claude Code (ADR-050 P2): reanalyze button handler """ REANALYZE_TTL_SECONDS = 600 # 10 分鐘去重 TTL (ADR-050) dedup_key = f"reanalyze_dedup:{incident_id}" try: redis_client = get_redis() # 去重檢查 (SETNX: 只有第一次設定會成功) is_new = await redis_client.set(dedup_key, "1", ex=REANALYZE_TTL_SECONDS, nx=True) if not is_new: logger.info( "reanalyze_deduplicated", incident_id=incident_id, reason="Already triggered within 10 minutes", ) return { "triggered": False, "message": "重診已在進行中，請 10 分鐘後再試", "already_analyzing": True, } # 從 Working Memory 取得 Incident incident = await self.get_from_working_memory(incident_id) if not incident: incident = await self.get_from_episodic_memory(incident_id) if not incident: # 刪除剛設定的去重 key（讓下次能重試） await redis_client.delete(dedup_key) logger.warning("reanalyze_incident_not_found", incident_id=incident_id) return { "triggered": False, "message": f"找不到事件 {incident_id}", "already_analyzing": False, } # 標記 status 為 analyzing（讓 AI 引擎接手） # 使用延遲 import 避免循環依賴（同 create_incident_from_signal 模式） from src.models.incident import IncidentStatus # 使用 INVESTIGATING 若 ANALYZING 不存在 analyzing_status = getattr(IncidentStatus, "ANALYZING", None) or getattr(IncidentStatus, "INVESTIGATING", None) if analyzing_status: incident.status = analyzing_status await self.save_to_working_memory(incident) logger.info( "reanalyze_triggered", incident_id=incident_id, severity=incident.severity.value, ) return { "triggered": True, "message": "重診已排程，AI 正在分析中", "already_analyzing": False, } except Exception as e: logger.exception("reanalyze_failed", incident_id=incident_id, error=str(e)) return { "triggered": False, "message": f"重診觸發失敗: {str(e)[:80]}", "already_analyzing": False, } # ============================================================================= # Singleton # ============================================================================= _incident_service: IncidentService | None = None def get_incident_service() -> IncidentService: """取得 Incident Service 實例 (Singleton)""" global _incident_service if _incident_service is None: _incident_service = IncidentService() return _incident_service