fix(flywheel): 四階段系統性修復 AUTO_REPAIR NO_MATCH 斷層
Some checks failed
CD Pipeline / build-and-deploy (push) Has been cancelled
Some checks failed
CD Pipeline / build-and-deploy (push) Has been cancelled
Phase 1 — affected_services 污染根治
- webhooks.py: _extract_affected_services() 從 labels 精準萃取服務名
(component > job > pod deployment name > clean target_resource > [])
- create_incident_for_approval: alert_labels 完整保留進 Signal
- alert_name 從 alertname 取,不再用 "custom"
Phase 2 — Playbook alertname 變體擴充
- alert_rules.yaml: 5 條規則新增 HostHighCpuLoad、KubePodCrashLooping 等變體
- scripts/update_playbook_alert_variants.py: Redis index 已執行更新 ✅
Phase 3 — Jaccard 通用型 Playbook 豁免
- similarity.py: affected_services=[] → 1.0 豁免(基礎設施 Playbook 不針對特定服務)
- severity_range=[] → 1.0 豁免(適用所有嚴重度)
Phase 4 — Playbook Embedding 持久化(冷啟動修復)
- migrations/flywheel_playbook_embeddings.sql: pgvector 持久化表
- services/playbook_embedding_service.py: 啟動時重建 Redis 向量快取 + 同步 DB
- main.py: lifespan 啟動時 asyncio.create_task 非阻塞執行
2026-04-10 Asia/Taipei — Claude Sonnet 4.6
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
126
apps/api/src/services/playbook_embedding_service.py
Normal file
126
apps/api/src/services/playbook_embedding_service.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
Playbook Embedding Service — Phase 4 飛輪冷啟動修復
|
||||
====================================================
|
||||
ADR-067 延伸: Playbook 向量持久化到 PostgreSQL playbook_embeddings 表
|
||||
|
||||
職責:
|
||||
- 啟動時掃描 APPROVED Playbooks,重建 Redis 向量快取
|
||||
- 同步持久化到 playbook_embeddings (pgvector) 供跨重啟使用
|
||||
- 已索引且未變更的 Playbook 跳過 (updated_at 比對)
|
||||
|
||||
呼叫方: main.py lifespan (asyncio.create_task — 非阻塞)
|
||||
|
||||
2026-04-10 Claude Sonnet 4.6 Asia/Taipei
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import structlog
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
async def ensure_playbook_embeddings_indexed() -> None:
|
||||
"""
|
||||
確保所有 APPROVED Playbooks 都有向量索引。
|
||||
|
||||
執行步驟:
|
||||
1. 從 PlaybookService 取得所有 APPROVED Playbooks
|
||||
2. 呼叫 PlaybookRAGService.reindex_all_playbooks → 更新 Redis 向量快取
|
||||
3. 將向量持久化到 playbook_embeddings (pgvector) 表
|
||||
"""
|
||||
try:
|
||||
from src.models.playbook import PlaybookStatus
|
||||
from src.services.playbook_service import get_playbook_service
|
||||
from src.services.playbook_rag import get_playbook_rag_service
|
||||
|
||||
playbook_service = get_playbook_service()
|
||||
playbooks, total = await playbook_service.list_playbooks(
|
||||
status=PlaybookStatus.APPROVED, limit=500
|
||||
)
|
||||
|
||||
if not playbooks:
|
||||
logger.info("playbook_embedding_indexing_skipped", reason="no approved playbooks")
|
||||
return
|
||||
|
||||
logger.info("playbook_embedding_indexing_start", count=len(playbooks))
|
||||
|
||||
# Step 1: 重建 Redis 向量快取 (現有邏輯)
|
||||
rag_service = await get_playbook_rag_service()
|
||||
success, failed = await rag_service.reindex_all_playbooks(playbooks)
|
||||
|
||||
logger.info(
|
||||
"playbook_embedding_redis_done",
|
||||
success=success,
|
||||
failed=failed,
|
||||
)
|
||||
|
||||
# Step 2: 持久化到 PostgreSQL playbook_embeddings 表
|
||||
await _persist_embeddings_to_db(rag_service, playbooks)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("playbook_embedding_indexing_error", error=str(e))
|
||||
|
||||
|
||||
async def _persist_embeddings_to_db(rag_service, playbooks) -> None:
|
||||
"""將 Redis 向量快取同步寫入 playbook_embeddings DB 表 (持久化層)。"""
|
||||
try:
|
||||
from sqlalchemy import text
|
||||
from src.db.base import get_db_context
|
||||
|
||||
persisted = 0
|
||||
skipped = 0
|
||||
|
||||
async with get_db_context() as db:
|
||||
for playbook in playbooks:
|
||||
try:
|
||||
embedding = await rag_service.get_playbook_embedding(playbook.playbook_id)
|
||||
if not embedding:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
sp = playbook.symptom_pattern
|
||||
alert_names = list(sp.alert_names) if sp else []
|
||||
keywords = list(sp.keywords) if sp else []
|
||||
|
||||
# UPSERT: 已存在則更新向量快照
|
||||
await db.execute(
|
||||
text("""
|
||||
INSERT INTO playbook_embeddings
|
||||
(playbook_id, embedding, alert_names, keywords, indexed_at, updated_at)
|
||||
VALUES
|
||||
(:playbook_id, :embedding, :alert_names, :keywords,
|
||||
NOW(), NOW())
|
||||
ON CONFLICT (playbook_id) DO UPDATE SET
|
||||
embedding = EXCLUDED.embedding,
|
||||
alert_names = EXCLUDED.alert_names,
|
||||
keywords = EXCLUDED.keywords,
|
||||
updated_at = NOW()
|
||||
"""),
|
||||
{
|
||||
"playbook_id": playbook.playbook_id,
|
||||
"embedding": str(embedding), # pgvector accepts '[x,y,...]' string
|
||||
"alert_names": alert_names,
|
||||
"keywords": keywords,
|
||||
},
|
||||
)
|
||||
persisted += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"playbook_embedding_persist_error",
|
||||
playbook_id=playbook.playbook_id,
|
||||
error=str(e),
|
||||
)
|
||||
skipped += 1
|
||||
|
||||
await db.commit()
|
||||
|
||||
logger.info(
|
||||
"playbook_embedding_db_done",
|
||||
persisted=persisted,
|
||||
skipped=skipped,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("playbook_embedding_db_error", error=str(e))
|
||||
Reference in New Issue
Block a user