diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index b98795a7a..420a3f699 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -208,6 +208,7 @@ async def _try_auto_repair_background( result = await repair_service.execute_auto_repair( incident=incident, playbook=decision.playbook, + is_cold_start=decision.is_cold_start, ) logger.info( diff --git a/apps/api/src/models/incident.py b/apps/api/src/models/incident.py index 98e5dbd67..fce5475ad 100644 --- a/apps/api/src/models/incident.py +++ b/apps/api/src/models/incident.py @@ -245,6 +245,28 @@ class IncidentFrequencyStats(BaseModel): description="最後一次修復是否成功", ) + # 2026-04-07 Claude Code: Sprint 4 — 告警處置統計 (A1) + human_approved_count: int = Field( + default=0, + ge=0, + description="人工按批准後執行次數", + ) + manual_resolved_count: int = Field( + default=0, + ge=0, + description="無系統修復紀錄但 resolved 次數", + ) + cold_start_trust_count: int = Field( + default=0, + ge=0, + description="首次信任自動放行次數", + ) + total_resolution_count: int = Field( + default=0, + ge=0, + description="總處置次數 (auto + human + manual + cold_start)", + ) + # ============================================================================= # Incident Outcome (CPO 要求：回饋循環) diff --git a/apps/api/src/services/anomaly_counter.py b/apps/api/src/services/anomaly_counter.py index e89246312..df6380740 100644 --- a/apps/api/src/services/anomaly_counter.py +++ b/apps/api/src/services/anomaly_counter.py @@ -50,6 +50,11 @@ class AnomalyFrequency: auto_repair_count: int permanent_fix_applied: bool escalation_level: str | None # None, REPEAT, ESCALATE, PERMANENT_FIX + # 2026-04-07 Claude Code: Sprint 4 A1 — 處置類型分佈 + human_approved_count: int = 0 + manual_resolved_count: int = 0 + cold_start_trust_count: int = 0 + total_resolution_count: int = 0 def to_dict(self) -> dict: """轉換為字典 (供 Telegram 告警使用)""" @@ -64,6 +69,11 @@ class AnomalyFrequency: "auto_repair_count": self.auto_repair_count, "permanent_fix_applied": self.permanent_fix_applied, "escalation_level": self.escalation_level, + # Sprint 4 處置統計 + "human_approved_count": self.human_approved_count, + "manual_resolved_count": self.manual_resolved_count, + "cold_start_trust_count": self.cold_start_trust_count, + "total_resolution_count": self.total_resolution_count, } @@ -99,6 +109,8 @@ class AnomalyCounter: PREFIX_PERMANENT_FIX = "anomaly:permanent_fix:" PREFIX_METADATA = "anomaly:metadata:" PREFIX_REPAIR_HISTORY = "anomaly:repair_history:" + # 2026-04-07 Claude Code: Sprint 4 A2 — 處置類型計數器 + PREFIX_DISPOSITION = "anomaly:disposition:" # TTL 設定 (35 天，比清理週期長一點) TTL_SECONDS = 35 * 24 * 3600 @@ -260,6 +272,9 @@ class AnomalyCounter: # 8. 判斷升級等級 escalation_level = self._get_escalation_level(count_24h) + # 9. 取得處置分佈 (Sprint 4 A3) + disposition = await self.get_disposition_stats(anomaly_key) + freq = AnomalyFrequency( anomaly_key=anomaly_key, count_1h=count_1h, @@ -271,6 +286,10 @@ class AnomalyCounter: auto_repair_count=auto_repair_count, permanent_fix_applied=permanent_fix, escalation_level=escalation_level, + human_approved_count=disposition["human_approved"], + manual_resolved_count=disposition["manual_resolved"], + cold_start_trust_count=disposition["cold_start_trust"], + total_resolution_count=disposition["total"], ) # 9. 記錄日誌 @@ -340,6 +359,76 @@ class AnomalyCounter: except Exception as e: logger.warning("record_repair_attempt_redis_error", error=str(e), anomaly_key=anomaly_key) + # ========================================================================== + # 2026-04-07 Claude Code: Sprint 4 A2/A3 — 處置類型統計 + # ========================================================================== + + VALID_DISPOSITION_TYPES = {"auto_repair", "human_approved", "manual_resolved", "cold_start_trust"} + + async def record_disposition( + self, + anomaly_key: str, + disposition_type: str, + ) -> None: + """ + 記錄告警處置類型 (原子 HINCRBY)。 + 2026-04-07 Claude Code: Sprint 4 A2 + + Args: + anomaly_key: 異常 key + disposition_type: "auto_repair" | "human_approved" | "manual_resolved" | "cold_start_trust" + """ + if disposition_type not in self.VALID_DISPOSITION_TYPES: + logger.warning( + "invalid_disposition_type", + anomaly_key=anomaly_key, + disposition_type=disposition_type, + ) + return + + try: + key = f"{self.PREFIX_DISPOSITION}{anomaly_key}" + await self.redis.hincrby(key, disposition_type, 1) + await self.redis.hincrby(key, "total", 1) + await self.redis.expire(key, self.TTL_SECONDS) + + logger.info( + "disposition_recorded", + anomaly_key=anomaly_key, + disposition_type=disposition_type, + ) + except Exception as e: + logger.warning("record_disposition_redis_error", error=str(e), anomaly_key=anomaly_key) + + async def get_disposition_stats(self, anomaly_key: str) -> dict: + """ + 取得處置類型分佈統計。 + 2026-04-07 Claude Code: Sprint 4 A3 + + Returns: + dict: {"auto_repair": N, "human_approved": N, "manual_resolved": N, + "cold_start_trust": N, "total": N} + """ + try: + key = f"{self.PREFIX_DISPOSITION}{anomaly_key}" + raw = await self.redis.hgetall(key) + return { + "auto_repair": int(raw.get(b"auto_repair", raw.get("auto_repair", 0))), + "human_approved": int(raw.get(b"human_approved", raw.get("human_approved", 0))), + "manual_resolved": int(raw.get(b"manual_resolved", raw.get("manual_resolved", 0))), + "cold_start_trust": int(raw.get(b"cold_start_trust", raw.get("cold_start_trust", 0))), + "total": int(raw.get(b"total", raw.get("total", 0))), + } + except Exception as e: + logger.warning("get_disposition_stats_redis_error", error=str(e), anomaly_key=anomaly_key) + return { + "auto_repair": 0, + "human_approved": 0, + "manual_resolved": 0, + "cold_start_trust": 0, + "total": 0, + } + async def mark_permanent_fix_applied( self, anomaly_key: str, diff --git a/apps/api/src/services/approval_execution.py b/apps/api/src/services/approval_execution.py index cd8c8d26e..b1780e67a 100644 --- a/apps/api/src/services/approval_execution.py +++ b/apps/api/src/services/approval_execution.py @@ -160,6 +160,17 @@ class ApprovalExecutionService: duration_seconds=result.duration_ms / 1000 if result.duration_ms else 0, ) ) + + # 2026-04-07 Claude Code: Sprint 4 B3 — 記錄人工批准處置類型 + try: + anomaly_key = await self._get_anomaly_key_from_approval(approval) + if anomaly_key: + from src.services.anomaly_counter import get_anomaly_counter + counter = get_anomaly_counter() + await counter.record_disposition(anomaly_key, "human_approved") + except Exception as _disp_e: + logger.warning("disposition_record_failed", error=str(_disp_e)) + else: logger.error( "background_execution_failed", @@ -206,6 +217,35 @@ class ApprovalExecutionService: ) ) + async def _get_anomaly_key_from_approval(self, approval: ApprovalRequest) -> str | None: + """ + 從 approval → incident → anomaly_signature → hash。 + 2026-04-07 Claude Code: Sprint 4 B3 + + Returns: + anomaly_key or None if not derivable + """ + try: + if not approval.incident_id: + return None + from src.services.incident_service import get_incident_service + incident_service = get_incident_service() + incident = await incident_service.get_from_working_memory(approval.incident_id) + if not incident or not incident.signals: + return None + # 從第一個 signal 建立 anomaly signature + signal = incident.signals[0] + signature = { + "alert_name": signal.alert_name, + "service": incident.affected_services[0] if incident.affected_services else "", + "namespace": getattr(signal, "namespace", ""), + } + from src.services.anomaly_counter import AnomalyCounter + return AnomalyCounter.hash_signature(signature) + except Exception as e: + logger.warning("get_anomaly_key_from_approval_failed", error=str(e)) + return None + async def _trigger_learning( self, approval: ApprovalRequest, diff --git a/apps/api/src/services/auto_repair_service.py b/apps/api/src/services/auto_repair_service.py index ad01297ee..6d59db0a0 100644 --- a/apps/api/src/services/auto_repair_service.py +++ b/apps/api/src/services/auto_repair_service.py @@ -61,6 +61,8 @@ class AutoRepairDecision: reason: str = "" risk_level: RiskLevel = RiskLevel.MEDIUM blocked_by: str | None = None # 阻擋原因 (如 HIGH_RISK, P1_SEVERITY) + # 2026-04-07 Claude Code: Sprint 4 B2 — 追蹤首次信任 + is_cold_start: bool = False @dataclass @@ -265,6 +267,7 @@ class AutoRepairService: # 高品質檢查 + 首次信任機制 # 2026-04-07 Claude Code: 方案 C — 打破冷啟動雞生蛋問題 max_risk = self._get_max_risk_level(best_match.playbook) + _is_cold_start = False # Sprint 4 B2: 預設非冷啟動 if not best_match.playbook.is_high_quality: # 首次信任: APPROVED + 全步驟 LOW risk + 執行次數 < N @@ -286,6 +289,7 @@ class AutoRepairService: total_executions=best_match.playbook.total_executions, max_risk=max_risk.value, ) + _is_cold_start = True # Sprint 4 B2: 標記冷啟動 # 跳過 is_high_quality 門檻，直接進入風險檢查 else: return AutoRepairDecision( @@ -327,12 +331,14 @@ class AutoRepairService: playbook=best_match.playbook, reason=f"匹配高品質 Playbook: {best_match.playbook.name} (成功率 {best_match.playbook.success_rate:.0%})", risk_level=max_risk, + is_cold_start=_is_cold_start, ) async def execute_auto_repair( self, incident: Incident, playbook: Playbook, + is_cold_start: bool = False, ) -> AutoRepairResult: """ 執行自動修復 @@ -341,6 +347,7 @@ class AutoRepairService: 1. 依序執行 Playbook 中的 repair_steps 2. 記錄執行結果 3. 更新 Playbook 統計 + 4. 記錄處置類型 (Sprint 4 B1/B2) """ import time @@ -399,6 +406,17 @@ class AutoRepairService: execution_time_ms=execution_time, ) + # 2026-04-07 Claude Code: Sprint 4 B1/B2 — 記錄處置類型 + try: + from src.services.anomaly_counter import get_anomaly_counter + counter = get_anomaly_counter() + symptoms = self._extract_symptoms(incident) + anomaly_key = symptoms.compute_hash() + disposition_type = "cold_start_trust" if is_cold_start else "auto_repair" + await counter.record_disposition(anomaly_key, disposition_type) + except Exception as _disp_e: + logger.warning("disposition_record_failed", error=str(_disp_e)) + # 2026-04-04 Claude Code: Phase 25 P1 — 成功修復後 fire-and-forget 生成 AUTO_RUNBOOK try: from src.services.runbook_generator import get_runbook_generator diff --git a/apps/api/src/services/incident_service.py b/apps/api/src/services/incident_service.py index 55beab355..9c357524a 100644 --- a/apps/api/src/services/incident_service.py +++ b/apps/api/src/services/incident_service.py @@ -692,6 +692,31 @@ class IncidentService: except Exception: logger.exception("kb_extract_task_create_failed", incident_id=incident_id) + # 2026-04-07 Claude Code: Sprint 4 B4 — 手動處理推斷 + # 若 resolved 但沒有系統修復紀錄 → manual_resolved + try: + from src.services.anomaly_counter import AnomalyCounter, get_anomaly_counter + counter = get_anomaly_counter() + if incident.signals: + signal = incident.signals[0] + signature = { + "alert_name": signal.alert_name, + "service": incident.affected_services[0] if incident.affected_services else "", + "namespace": getattr(signal, "namespace", ""), + } + anomaly_key = AnomalyCounter.hash_signature(signature) + disposition = await counter.get_disposition_stats(anomaly_key) + # 排除法: 如果已有 auto/human/cold_start 紀錄，代表系統已處理 + has_system_resolution = ( + disposition["auto_repair"] > 0 + or disposition["human_approved"] > 0 + or disposition["cold_start_trust"] > 0 + ) + if not has_system_resolution: + await counter.record_disposition(anomaly_key, "manual_resolved") + except Exception as _disp_e: + logger.warning("disposition_manual_resolve_failed", error=str(_disp_e)) + return incident async def find_by_proposal_id(self, proposal_id: str) -> Incident | None: