fix(api): reconcile completed stuck incidents
This commit is contained in:
@@ -329,7 +329,7 @@ class FlywheelStatsService:
|
||||
# 卡住的 Incident(INVESTIGATING > 24h)
|
||||
stuck_q = await db.execute(
|
||||
select(func.count()).where(
|
||||
IncidentRecord.status == IncidentStatus.INVESTIGATING.value,
|
||||
IncidentRecord.status == IncidentStatus.INVESTIGATING,
|
||||
IncidentRecord.created_at <= stuck_threshold,
|
||||
)
|
||||
)
|
||||
@@ -340,7 +340,7 @@ class FlywheelStatsService:
|
||||
type4_q = await db.execute(
|
||||
select(func.count()).where(
|
||||
IncidentRecord.notification_type == "TYPE-4",
|
||||
IncidentRecord.status == IncidentStatus.INVESTIGATING.value,
|
||||
IncidentRecord.status == IncidentStatus.INVESTIGATING,
|
||||
)
|
||||
)
|
||||
type4_count = type4_q.scalar_one_or_none() or 0
|
||||
|
||||
@@ -445,20 +445,31 @@ async def create_incident_for_approval(
|
||||
# 回滾: git revert (秒級恢復)
|
||||
# =============================================================================
|
||||
|
||||
def normalize_status(value: str) -> str:
|
||||
def normalize_status(value: str | IncidentStatus) -> str:
|
||||
"""
|
||||
正規化 IncidentStatus 舊格式值
|
||||
|
||||
舊值 → 新值:
|
||||
- 'open' → 'investigating'
|
||||
"""
|
||||
if isinstance(value, IncidentStatus):
|
||||
return value.value
|
||||
|
||||
raw = str(value)
|
||||
if raw in IncidentStatus.__members__:
|
||||
return IncidentStatus[raw].value
|
||||
|
||||
normalized = raw.strip().lower()
|
||||
legacy_map = {
|
||||
"open": "investigating",
|
||||
}
|
||||
return legacy_map.get(value, value)
|
||||
valid_values = {status.value for status in IncidentStatus}
|
||||
if normalized in valid_values:
|
||||
return normalized
|
||||
return legacy_map.get(normalized, raw)
|
||||
|
||||
|
||||
def normalize_severity(value: str) -> str:
|
||||
def normalize_severity(value: str | Severity) -> str:
|
||||
"""
|
||||
正規化 Severity 舊格式值
|
||||
|
||||
@@ -471,6 +482,14 @@ def normalize_severity(value: str) -> str:
|
||||
- 'low' → 'P3'
|
||||
- 'none' → 'P3'
|
||||
"""
|
||||
if isinstance(value, Severity):
|
||||
return value.value
|
||||
|
||||
raw = str(value)
|
||||
if raw in Severity.__members__:
|
||||
return Severity[raw].value
|
||||
|
||||
normalized = raw.strip().lower()
|
||||
legacy_map = {
|
||||
"critical": "P0",
|
||||
"high": "P1",
|
||||
@@ -480,7 +499,7 @@ def normalize_severity(value: str) -> str:
|
||||
"low": "P3",
|
||||
"none": "P3",
|
||||
}
|
||||
return legacy_map.get(value, value)
|
||||
return legacy_map.get(normalized, raw)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -1097,17 +1116,24 @@ class IncidentService:
|
||||
from src.repositories.incident_repository import get_incident_repository
|
||||
from src.utils.timezone import now_taipei
|
||||
|
||||
# 1. 從 Working Memory 讀取
|
||||
# 1. 從 Working Memory 讀取;Redis TTL 過期時退回 PostgreSQL。
|
||||
incident = await self.get_from_working_memory(incident_id)
|
||||
if incident is None:
|
||||
logger.warning("incident_not_found_for_resolve", incident_id=incident_id)
|
||||
return None
|
||||
incident = await self.get_from_episodic_memory(incident_id)
|
||||
if incident is None:
|
||||
logger.warning("incident_not_found_for_resolve", incident_id=incident_id)
|
||||
return None
|
||||
logger.info(
|
||||
"incident_resolve_hydrated_from_episodic_memory",
|
||||
incident_id=incident_id,
|
||||
resolution_type=resolution_type,
|
||||
)
|
||||
|
||||
# 1.5 F2 (2026-05-07 ogt + Codex + Claude Sonnet 4.6) — 冪等保護:
|
||||
# 已經 RESOLVED 的 incident 直接 return existing,避免後續所有副作用
|
||||
# 已經 RESOLVED/CLOSED 的 incident 直接 return existing,避免後續所有副作用
|
||||
# 重複觸發(postmortem / KB extract / KM convert / disposition / Telegram)。
|
||||
# F2 NO_ACTION 路徑會頻繁呼叫 resolve_incident,必須擋在 status mutation 之前。
|
||||
if incident.status == IncidentStatus.RESOLVED:
|
||||
if incident.status in (IncidentStatus.RESOLVED, IncidentStatus.CLOSED):
|
||||
logger.info(
|
||||
"incident_resolve_skipped_already_resolved",
|
||||
incident_id=incident_id,
|
||||
@@ -1146,6 +1172,7 @@ class IncidentService:
|
||||
# KB Phase 2-A: 自動萃取 KB 草稿 (fire-and-forget, 2026-04-03 ogt)
|
||||
try:
|
||||
import asyncio
|
||||
|
||||
from src.services.knowledge_extractor_service import get_knowledge_extractor
|
||||
asyncio.create_task(
|
||||
get_knowledge_extractor().extract_from_incident(incident)
|
||||
@@ -1160,13 +1187,13 @@ class IncidentService:
|
||||
# 改為在呼叫點加統一契約保護(指數退避 3 次 + DLQ 失敗回收)。
|
||||
try:
|
||||
import asyncio
|
||||
|
||||
from src.core.config import settings
|
||||
from src.services.km_conversion_service import get_km_conversion_service
|
||||
from src.services.km_writer import (
|
||||
KMWritePayload,
|
||||
KMWriteResult,
|
||||
_RETRY_BASE_DELAY,
|
||||
_RETRY_MAX,
|
||||
KMWritePayload,
|
||||
_is_retriable,
|
||||
_write_to_dlq,
|
||||
)
|
||||
@@ -1253,7 +1280,10 @@ class IncidentService:
|
||||
# 孤兒 report_generation_service.trigger_postmortem 本次接上 resolve 路徑
|
||||
try:
|
||||
import asyncio
|
||||
from src.services.report_generation_service import get_report_generation_service
|
||||
|
||||
from src.services.report_generation_service import (
|
||||
get_report_generation_service,
|
||||
)
|
||||
|
||||
alertname = (
|
||||
incident.signals[0].labels.get("alertname", "UnknownAlert")
|
||||
|
||||
Reference in New Issue
Block a user