wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

feat(api): Sprint 4 Phase A+B — 告警處置統計資料層+寫入層

Browse Source 
Phase A: 資料層
- A1: IncidentFrequencyStats 新增 4 欄位 (human_approved/manual_resolved/cold_start_trust/total_resolution)
- A2: AnomalyCounter.record_disposition() — Redis HINCRBY 原子遞增
- A3: get_disposition_stats() — HGETALL 回傳處置分佈
- AnomalyFrequency dataclass 擴充 + to_dict() 同步
- _record_anomaly_impl() 整合 disposition stats

Phase B: 寫入層觸發點接線
- B1: 自動修復成功 → record_disposition("auto_repair")
- B2: 冷啟動信任成功 → record_disposition("cold_start_trust")
  - AutoRepairDecision 新增 is_cold_start flag
  - execute_auto_repair() 接收並區分處置類型
- B3: 人工批准執行成功 → record_disposition("human_approved")
  - 新增 _get_anomaly_key_from_approval() helper
- B4: 手動處理推斷 → resolve_incident() 排除法判定
  - 若 resolved 且無 auto/human/cold_start 紀錄 → manual_resolved

安全設計: 所有 disposition 記錄走 try/except，失敗不阻塞主流程

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
OG T
2026-04-07 11:54:46 +08:00
parent e82d3802c5
commit 9253281d46
6 changed files with 195 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
apps/api/src/services/anomaly_counter.py
View File

@@ -50,6 +50,11 @@ class AnomalyFrequency:
auto_repair_count: int
permanent_fix_applied: bool
escalation_level: str | None # None, REPEAT, ESCALATE, PERMANENT_FIX
# 2026-04-07 Claude Code: Sprint 4 A1 — 處置類型分佈
human_approved_count: int = 0
manual_resolved_count: int = 0
cold_start_trust_count: int = 0
total_resolution_count: int = 0
def to_dict(self) -> dict:
"""轉換為字典 (供 Telegram 告警使用)"""
@@ -64,6 +69,11 @@ class AnomalyFrequency:
"auto_repair_count": self.auto_repair_count,
"permanent_fix_applied": self.permanent_fix_applied,
"escalation_level": self.escalation_level,
# Sprint 4 處置統計
"human_approved_count": self.human_approved_count,
"manual_resolved_count": self.manual_resolved_count,
"cold_start_trust_count": self.cold_start_trust_count,
"total_resolution_count": self.total_resolution_count,
}
@@ -99,6 +109,8 @@ class AnomalyCounter:
PREFIX_PERMANENT_FIX = "anomaly:permanent_fix:"
PREFIX_METADATA = "anomaly:metadata:"
PREFIX_REPAIR_HISTORY = "anomaly:repair_history:"
# 2026-04-07 Claude Code: Sprint 4 A2 — 處置類型計數器
PREFIX_DISPOSITION = "anomaly:disposition:"
# TTL 設定 (35 天，比清理週期長一點)
TTL_SECONDS = 35 * 24 * 3600
@@ -260,6 +272,9 @@ class AnomalyCounter:
# 8. 判斷升級等級
escalation_level = self._get_escalation_level(count_24h)
# 9. 取得處置分佈 (Sprint 4 A3)
disposition = await self.get_disposition_stats(anomaly_key)
freq = AnomalyFrequency(
anomaly_key=anomaly_key,
count_1h=count_1h,
@@ -271,6 +286,10 @@ class AnomalyCounter:
auto_repair_count=auto_repair_count,
permanent_fix_applied=permanent_fix,
escalation_level=escalation_level,
human_approved_count=disposition["human_approved"],
manual_resolved_count=disposition["manual_resolved"],
cold_start_trust_count=disposition["cold_start_trust"],
total_resolution_count=disposition["total"],
)
# 9. 記錄日誌
@@ -340,6 +359,76 @@ class AnomalyCounter:
except Exception as e:
logger.warning("record_repair_attempt_redis_error", error=str(e), anomaly_key=anomaly_key)
# ==========================================================================
# 2026-04-07 Claude Code: Sprint 4 A2/A3 — 處置類型統計
# ==========================================================================
VALID_DISPOSITION_TYPES = {"auto_repair", "human_approved", "manual_resolved", "cold_start_trust"}
async def record_disposition(
self,
anomaly_key: str,
disposition_type: str,
) -> None:
"""
記錄告警處置類型 (原子 HINCRBY)。
2026-04-07 Claude Code: Sprint 4 A2
Args:
anomaly_key: 異常 key
disposition_type: "auto_repair" | "human_approved" | "manual_resolved" | "cold_start_trust"
"""
if disposition_type not in self.VALID_DISPOSITION_TYPES:
logger.warning(
"invalid_disposition_type",
anomaly_key=anomaly_key,
disposition_type=disposition_type,
)
return
try:
key = f"{self.PREFIX_DISPOSITION}{anomaly_key}"
await self.redis.hincrby(key, disposition_type, 1)
await self.redis.hincrby(key, "total", 1)
await self.redis.expire(key, self.TTL_SECONDS)
logger.info(
"disposition_recorded",
anomaly_key=anomaly_key,
disposition_type=disposition_type,
)
except Exception as e:
logger.warning("record_disposition_redis_error", error=str(e), anomaly_key=anomaly_key)
async def get_disposition_stats(self, anomaly_key: str) -> dict:
"""
取得處置類型分佈統計。
2026-04-07 Claude Code: Sprint 4 A3
Returns:
dict: {"auto_repair": N, "human_approved": N, "manual_resolved": N,
"cold_start_trust": N, "total": N}
"""
try:
key = f"{self.PREFIX_DISPOSITION}{anomaly_key}"
raw = await self.redis.hgetall(key)
return {
"auto_repair": int(raw.get(b"auto_repair", raw.get("auto_repair", 0))),
"human_approved": int(raw.get(b"human_approved", raw.get("human_approved", 0))),
"manual_resolved": int(raw.get(b"manual_resolved", raw.get("manual_resolved", 0))),
"cold_start_trust": int(raw.get(b"cold_start_trust", raw.get("cold_start_trust", 0))),
"total": int(raw.get(b"total", raw.get("total", 0))),
}
except Exception as e:
logger.warning("get_disposition_stats_redis_error", error=str(e), anomaly_key=anomaly_key)
return {
"auto_repair": 0,
"human_approved": 0,
"manual_resolved": 0,
"cold_start_trust": 0,
"total": 0,
}
async def mark_permanent_fix_applied(
self,
anomaly_key: str,

40
apps/api/src/services/approval_execution.py
View File

@@ -160,6 +160,17 @@ class ApprovalExecutionService:
duration_seconds=result.duration_ms / 1000 if result.duration_ms else 0,
)
)
# 2026-04-07 Claude Code: Sprint 4 B3 — 記錄人工批准處置類型
try:
anomaly_key = await self._get_anomaly_key_from_approval(approval)
if anomaly_key:
from src.services.anomaly_counter import get_anomaly_counter
counter = get_anomaly_counter()
await counter.record_disposition(anomaly_key, "human_approved")
except Exception as _disp_e:
logger.warning("disposition_record_failed", error=str(_disp_e))
else:
logger.error(
"background_execution_failed",
@@ -206,6 +217,35 @@ class ApprovalExecutionService:
)
)
async def _get_anomaly_key_from_approval(self, approval: ApprovalRequest) -> str | None:
"""
從 approval → incident → anomaly_signature → hash。
2026-04-07 Claude Code: Sprint 4 B3
Returns:
anomaly_key or None if not derivable
"""
try:
if not approval.incident_id:
return None
from src.services.incident_service import get_incident_service
incident_service = get_incident_service()
incident = await incident_service.get_from_working_memory(approval.incident_id)
if not incident or not incident.signals:
return None
# 從第一個 signal 建立 anomaly signature
signal = incident.signals[0]
signature = {
"alert_name": signal.alert_name,
"service": incident.affected_services[0] if incident.affected_services else "",
"namespace": getattr(signal, "namespace", ""),
}
from src.services.anomaly_counter import AnomalyCounter
return AnomalyCounter.hash_signature(signature)
except Exception as e:
logger.warning("get_anomaly_key_from_approval_failed", error=str(e))
return None
async def _trigger_learning(
self,
approval: ApprovalRequest,

18
apps/api/src/services/auto_repair_service.py
View File

@@ -61,6 +61,8 @@ class AutoRepairDecision:
reason: str = ""
risk_level: RiskLevel = RiskLevel.MEDIUM
blocked_by: str | None = None # 阻擋原因 (如 HIGH_RISK, P1_SEVERITY)
# 2026-04-07 Claude Code: Sprint 4 B2 — 追蹤首次信任
is_cold_start: bool = False
@dataclass
@@ -265,6 +267,7 @@ class AutoRepairService:
# 高品質檢查 + 首次信任機制
# 2026-04-07 Claude Code: 方案 C — 打破冷啟動雞生蛋問題
max_risk = self._get_max_risk_level(best_match.playbook)
_is_cold_start = False # Sprint 4 B2: 預設非冷啟動
if not best_match.playbook.is_high_quality:
# 首次信任: APPROVED + 全步驟 LOW risk + 執行次數 < N
@@ -286,6 +289,7 @@ class AutoRepairService:
total_executions=best_match.playbook.total_executions,
max_risk=max_risk.value,
)
_is_cold_start = True # Sprint 4 B2: 標記冷啟動
# 跳過 is_high_quality 門檻，直接進入風險檢查
else:
return AutoRepairDecision(
@@ -327,12 +331,14 @@ class AutoRepairService:
playbook=best_match.playbook,
reason=f"匹配高品質 Playbook: {best_match.playbook.name} (成功率 {best_match.playbook.success_rate:.0%})",
risk_level=max_risk,
is_cold_start=_is_cold_start,
)
async def execute_auto_repair(
self,
incident: Incident,
playbook: Playbook,
is_cold_start: bool = False,
) -> AutoRepairResult:
"""
執行自動修復
@@ -341,6 +347,7 @@ class AutoRepairService:
1. 依序執行 Playbook 中的 repair_steps
2. 記錄執行結果
3. 更新 Playbook 統計
4. 記錄處置類型 (Sprint 4 B1/B2)
"""
import time
@@ -399,6 +406,17 @@ class AutoRepairService:
execution_time_ms=execution_time,
)
# 2026-04-07 Claude Code: Sprint 4 B1/B2 — 記錄處置類型
try:
from src.services.anomaly_counter import get_anomaly_counter
counter = get_anomaly_counter()
symptoms = self._extract_symptoms(incident)
anomaly_key = symptoms.compute_hash()
disposition_type = "cold_start_trust" if is_cold_start else "auto_repair"
await counter.record_disposition(anomaly_key, disposition_type)
except Exception as _disp_e:
logger.warning("disposition_record_failed", error=str(_disp_e))
# 2026-04-04 Claude Code: Phase 25 P1 — 成功修復後 fire-and-forget 生成 AUTO_RUNBOOK
try:
from src.services.runbook_generator import get_runbook_generator

25
apps/api/src/services/incident_service.py
View File

@@ -692,6 +692,31 @@ class IncidentService:
except Exception:
logger.exception("kb_extract_task_create_failed", incident_id=incident_id)
# 2026-04-07 Claude Code: Sprint 4 B4 — 手動處理推斷
# 若 resolved 但沒有系統修復紀錄 → manual_resolved
try:
from src.services.anomaly_counter import AnomalyCounter, get_anomaly_counter
counter = get_anomaly_counter()
if incident.signals:
signal = incident.signals[0]
signature = {
"alert_name": signal.alert_name,
"service": incident.affected_services[0] if incident.affected_services else "",
"namespace": getattr(signal, "namespace", ""),
}
anomaly_key = AnomalyCounter.hash_signature(signature)
disposition = await counter.get_disposition_stats(anomaly_key)
# 排除法: 如果已有 auto/human/cold_start 紀錄，代表系統已處理
has_system_resolution = (
disposition["auto_repair"] > 0
or disposition["human_approved"] > 0
or disposition["cold_start_trust"] > 0
)
if not has_system_resolution:
await counter.record_disposition(anomaly_key, "manual_resolved")
except Exception as _disp_e:
logger.warning("disposition_manual_resolve_failed", error=str(_disp_e))
return incident
async def find_by_proposal_id(self, proposal_id: str) -> Incident | None: