fix(ops): harden cold-start schedule recovery

This commit is contained in:
Your Name
2026-05-05 22:14:54 +08:00
parent 10cd9fc025
commit 894174da5b
13 changed files with 1073 additions and 43 deletions

View File

@@ -6,8 +6,9 @@
-- bge-m3 產生 1024 維向量,現有 schema vector(768) 不相容INSERT 會直接失敗
--
-- 影響範圍:
-- 1. rag_chunks.embedding vector(768) → vector(1024)
-- 2. playbook_embeddings.embedding vector(768) → vector(1024)
-- 1. knowledge_entries.embedding vector(768) → vector(1024)
-- 2. rag_chunks.embedding vector(768) → vector(1024)
-- 3. playbook_embeddings.embedding vector(768) → vector(1024)
--
-- 遷移策略:清空現有向量資料,切換維度後由 re-embed script 重新嵌入
-- 現有向量資料若要保留,需先 dump 用 nomic 格式備份(舊維度無法轉換)
@@ -21,7 +22,24 @@
BEGIN;
-- 1. rag_chunks清空向量資料,變更欄位維度
-- 1. knowledge_entries備份舊向量並清空,變更欄位維度
CREATE TABLE IF NOT EXISTS knowledge_entries_embedding_backup_20260505 AS
SELECT
id,
embedding::text AS embedding_768,
NOW() AS backed_up_at
FROM knowledge_entries
WHERE embedding IS NOT NULL;
ALTER TABLE knowledge_entries
ALTER COLUMN embedding TYPE vector(1024)
USING NULL; -- 清空現有 768 維向量(維度不可轉換)
COMMENT ON COLUMN knowledge_entries.embedding IS
'bge-m3:latest 1024 維向量 — 遷移自 nomic-embed-text 768 維 (2026-05-05 ADR-110 follow-up)';
-- 2. rag_chunks清空向量資料變更欄位維度
-- ivfflat index 必須先 DROP 才能 ALTER COLUMN
DROP INDEX IF EXISTS idx_rag_chunks_embedding;
@@ -39,7 +57,7 @@ COMMENT ON COLUMN rag_chunks.embedding IS
'bge-m3:latest 1024 維向量 — 遷移自 nomic-embed-text 768 維 (2026-05-04 ADR-110)';
-- 2. playbook_embeddings清空向量資料變更欄位維度
-- 3. playbook_embeddings清空向量資料變更欄位維度
DROP INDEX IF EXISTS ix_playbook_embeddings_vec;
ALTER TABLE playbook_embeddings
@@ -61,9 +79,15 @@ COMMENT ON TABLE playbook_embeddings IS
-- 3. 驗證遷移結果
DO $$
DECLARE
v_km_dim integer;
v_rag_dim integer;
v_pb_dim integer;
BEGIN
SELECT atttypmod INTO v_km_dim
FROM pg_attribute
JOIN pg_class ON attrelid = pg_class.oid
WHERE relname = 'knowledge_entries' AND attname = 'embedding';
SELECT atttypmod INTO v_rag_dim
FROM pg_attribute
JOIN pg_class ON attrelid = pg_class.oid
@@ -74,15 +98,18 @@ BEGIN
JOIN pg_class ON attrelid = pg_class.oid
WHERE relname = 'playbook_embeddings' AND attname = 'embedding';
-- atttypmod for vector(1024) = 1024 + 1 = 1025
IF v_rag_dim != 1025 THEN
RAISE EXCEPTION 'rag_chunks.embedding 維度驗證失敗expected 1025, got %', v_rag_dim;
-- pgvector atttypmod stores the configured dimension.
IF v_km_dim != 1024 THEN
RAISE EXCEPTION 'knowledge_entries.embedding 維度驗證失敗expected 1024, got %', v_km_dim;
END IF;
IF v_pb_dim != 1025 THEN
RAISE EXCEPTION 'playbook_embeddings.embedding 維度驗證失敗expected 1025, got %', v_pb_dim;
IF v_rag_dim != 1024 THEN
RAISE EXCEPTION 'rag_chunks.embedding 維度驗證失敗expected 1024, got %', v_rag_dim;
END IF;
IF v_pb_dim != 1024 THEN
RAISE EXCEPTION 'playbook_embeddings.embedding 維度驗證失敗expected 1024, got %', v_pb_dim;
END IF;
RAISE NOTICE '✅ embedding 遷移驗證通過rag_chunksplaybook_embeddings 均為 vector(1024)';
RAISE NOTICE '✅ embedding 遷移驗證通過:knowledge_entries、rag_chunksplaybook_embeddings 均為 vector(1024)';
END $$;
COMMIT;

View File

@@ -34,8 +34,12 @@ logger = structlog.get_logger(__name__)
# 台北時區
TZ_TAIPEI = ZoneInfo("Asia/Taipei")
# Prometheus 端點
PROMETHEUS_URL = "http://192.168.0.121:30090"
# Prometheus endpoint.
#
# 2026-05-05 Codex: do not pin this report job to a K3s worker NodePort.
# Production already injects PROMETHEUS_URL from ConfigMap, currently the
# Docker Prometheus on 110. This keeps reboot recovery independent of 121.
PROMETHEUS_URL = settings.PROMETHEUS_URL.rstrip("/")
# kube-state-metrics 查詢
PROM_QUERIES = {
@@ -215,7 +219,7 @@ class K3sMonitorService:
# 發送訊息
formatted = status.format()
result = await gateway.send_message(formatted)
result = await gateway.send_text(formatted)
if result:
logger.info("k3s_daily_report_sent", date=status.report_date)

View File

@@ -244,7 +244,7 @@ class WeeklyReportService:
# 發送訊息
formatted = report.format()
result = await gateway.send_message(formatted)
result = await gateway.send_text(formatted)
if result:
logger.info("weekly_report_sent", week=report.week_range)