feat(api): 重開機後自動 warm-up Redis Working Memory from PostgreSQL
- main.py lifespan: 啟動時從 DB restore INVESTIGATING/MITIGATING incidents - scripts/reboot-recovery: 188 + 110 自動化腳本 + systemd services - scripts/reboot-recovery: aiops-network 自動建立 (ClawBot 依賴) - docs/runbooks/REBOOT-RECOVERY-SOP.md: 完整改寫,含自動化腳本說明 Why: 重開機後 Redis 清空導致前端 incidents 顯示 0 筆(DB 完整保存) 統帥批准: 「所有數據必須被長久記錄下來」 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -229,6 +229,50 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
)
|
||||
logger.info("telegram_heartbeat_monitor_started")
|
||||
|
||||
# Reboot Recovery: Warm-up Redis Working Memory from PostgreSQL
|
||||
# 2026-04-05 ogt: 重開機後 Redis 清空,從 DB restore 未解決的 incidents
|
||||
# 統帥批准: 數據必須長久記錄,重開機後自動恢復 Working Memory
|
||||
try:
|
||||
from src.services.incident_service import get_incident_service
|
||||
from src.db.base import get_db_context
|
||||
from src.db.models import IncidentRecord
|
||||
from sqlalchemy import select
|
||||
|
||||
incident_service = get_incident_service()
|
||||
async with get_db_context() as db:
|
||||
result = await db.execute(
|
||||
select(IncidentRecord).where(
|
||||
IncidentRecord.status.in_(["investigating", "mitigating"])
|
||||
)
|
||||
)
|
||||
records = result.scalars().all()
|
||||
|
||||
restored = 0
|
||||
for record in records:
|
||||
from src.models.incident import Incident
|
||||
incident = Incident(
|
||||
incident_id=record.incident_id,
|
||||
status=record.status,
|
||||
severity=record.severity,
|
||||
signals=record.signals or [],
|
||||
affected_services=record.affected_services or [],
|
||||
decision_chain=record.decision_chain,
|
||||
proposal_ids=record.proposal_ids or [],
|
||||
outcome=record.outcome,
|
||||
created_at=record.created_at,
|
||||
updated_at=record.updated_at,
|
||||
resolved_at=record.resolved_at,
|
||||
closed_at=record.closed_at,
|
||||
ttl_days=record.ttl_days,
|
||||
vectorized=record.vectorized,
|
||||
)
|
||||
if await incident_service.save_to_working_memory(incident):
|
||||
restored += 1
|
||||
|
||||
logger.info("working_memory_warmed_up", restored=restored, total=len(records))
|
||||
except Exception as e:
|
||||
logger.warning("working_memory_warmup_failed", error=str(e))
|
||||
|
||||
# Phase 6.1: 啟動 Signal Worker (Redis Streams Consumer)
|
||||
# 統帥鐵律: Event Bus 解耦告警接收與處理
|
||||
await init_signal_worker()
|
||||
|
||||
Reference in New Issue
Block a user