fix(api): reconcile completed stuck incidents
All checks were successful
Code Review / ai-code-review (push) Successful in 11s
CD Pipeline / tests (push) Successful in 1m2s
CD Pipeline / build-and-deploy (push) Successful in 3m34s
CD Pipeline / post-deploy-checks (push) Successful in 1m35s

This commit is contained in:
Your Name
2026-05-19 11:45:15 +08:00
parent 50833a0efb
commit d0835a7be1
7 changed files with 393 additions and 61 deletions

View File

@@ -329,7 +329,7 @@ class FlywheelStatsService:
# 卡住的 IncidentINVESTIGATING > 24h
stuck_q = await db.execute(
select(func.count()).where(
IncidentRecord.status == IncidentStatus.INVESTIGATING.value,
IncidentRecord.status == IncidentStatus.INVESTIGATING,
IncidentRecord.created_at <= stuck_threshold,
)
)
@@ -340,7 +340,7 @@ class FlywheelStatsService:
type4_q = await db.execute(
select(func.count()).where(
IncidentRecord.notification_type == "TYPE-4",
IncidentRecord.status == IncidentStatus.INVESTIGATING.value,
IncidentRecord.status == IncidentStatus.INVESTIGATING,
)
)
type4_count = type4_q.scalar_one_or_none() or 0

View File

@@ -445,20 +445,31 @@ async def create_incident_for_approval(
# 回滾: git revert (秒級恢復)
# =============================================================================
def normalize_status(value: str) -> str:
def normalize_status(value: str | IncidentStatus) -> str:
"""
正規化 IncidentStatus 舊格式值
舊值 → 新值:
- 'open''investigating'
"""
if isinstance(value, IncidentStatus):
return value.value
raw = str(value)
if raw in IncidentStatus.__members__:
return IncidentStatus[raw].value
normalized = raw.strip().lower()
legacy_map = {
"open": "investigating",
}
return legacy_map.get(value, value)
valid_values = {status.value for status in IncidentStatus}
if normalized in valid_values:
return normalized
return legacy_map.get(normalized, raw)
def normalize_severity(value: str) -> str:
def normalize_severity(value: str | Severity) -> str:
"""
正規化 Severity 舊格式值
@@ -471,6 +482,14 @@ def normalize_severity(value: str) -> str:
- 'low''P3'
- 'none''P3'
"""
if isinstance(value, Severity):
return value.value
raw = str(value)
if raw in Severity.__members__:
return Severity[raw].value
normalized = raw.strip().lower()
legacy_map = {
"critical": "P0",
"high": "P1",
@@ -480,7 +499,7 @@ def normalize_severity(value: str) -> str:
"low": "P3",
"none": "P3",
}
return legacy_map.get(value, value)
return legacy_map.get(normalized, raw)
# =============================================================================
@@ -1097,17 +1116,24 @@ class IncidentService:
from src.repositories.incident_repository import get_incident_repository
from src.utils.timezone import now_taipei
# 1. 從 Working Memory 讀取
# 1. 從 Working Memory 讀取Redis TTL 過期時退回 PostgreSQL。
incident = await self.get_from_working_memory(incident_id)
if incident is None:
logger.warning("incident_not_found_for_resolve", incident_id=incident_id)
return None
incident = await self.get_from_episodic_memory(incident_id)
if incident is None:
logger.warning("incident_not_found_for_resolve", incident_id=incident_id)
return None
logger.info(
"incident_resolve_hydrated_from_episodic_memory",
incident_id=incident_id,
resolution_type=resolution_type,
)
# 1.5 F2 (2026-05-07 ogt + Codex + Claude Sonnet 4.6) — 冪等保護:
# 已經 RESOLVED 的 incident 直接 return existing避免後續所有副作用
# 已經 RESOLVED/CLOSED 的 incident 直接 return existing避免後續所有副作用
# 重複觸發postmortem / KB extract / KM convert / disposition / Telegram
# F2 NO_ACTION 路徑會頻繁呼叫 resolve_incident必須擋在 status mutation 之前。
if incident.status == IncidentStatus.RESOLVED:
if incident.status in (IncidentStatus.RESOLVED, IncidentStatus.CLOSED):
logger.info(
"incident_resolve_skipped_already_resolved",
incident_id=incident_id,
@@ -1146,6 +1172,7 @@ class IncidentService:
# KB Phase 2-A: 自動萃取 KB 草稿 (fire-and-forget, 2026-04-03 ogt)
try:
import asyncio
from src.services.knowledge_extractor_service import get_knowledge_extractor
asyncio.create_task(
get_knowledge_extractor().extract_from_incident(incident)
@@ -1160,13 +1187,13 @@ class IncidentService:
# 改為在呼叫點加統一契約保護(指數退避 3 次 + DLQ 失敗回收)。
try:
import asyncio
from src.core.config import settings
from src.services.km_conversion_service import get_km_conversion_service
from src.services.km_writer import (
KMWritePayload,
KMWriteResult,
_RETRY_BASE_DELAY,
_RETRY_MAX,
KMWritePayload,
_is_retriable,
_write_to_dlq,
)
@@ -1253,7 +1280,10 @@ class IncidentService:
# 孤兒 report_generation_service.trigger_postmortem 本次接上 resolve 路徑
try:
import asyncio
from src.services.report_generation_service import get_report_generation_service
from src.services.report_generation_service import (
get_report_generation_service,
)
alertname = (
incident.signals[0].labels.get("alertname", "UnknownAlert")