diff --git a/apps/api/migrations/cleanup_duplicate_deprecated_playbooks.sql b/apps/api/migrations/cleanup_duplicate_deprecated_playbooks.sql
new file mode 100644
index 000000000..939b38a18
--- /dev/null
+++ b/apps/api/migrations/cleanup_duplicate_deprecated_playbooks.sql
@@ -0,0 +1,31 @@
+-- 清理重複的 deprecated yaml_rule Playbooks
+-- 根因:seeder 冪等 SQL 舊版排除 deprecated 記錄,導致每次啟動重建同名 Playbook
+-- C1 保護(evolver 不封存 yaml_rule)加入前已存在的 deprecated 歷史記錄
+-- 觸發無限重建迴圈(294 deprecated,25 approved)
+-- 修法:每個 name 只保留最新的一筆 deprecated,其餘刪除
+-- seeder 已同步修正(status 過濾移除),此腳本清理歷史垃圾
+-- 2026-04-24 ogt + Claude Sonnet 4.6(亞太)
+
+BEGIN;
+
+-- 診斷:執行前統計(可選,確認規模)
+-- SELECT source, status, COUNT(*) FROM playbooks GROUP BY source, status ORDER BY source, status;
+
+-- 找出每個 yaml_rule deprecated name 的最新 created_at(保留基準)
+-- 刪除同名同 source=yaml_rule + status=deprecated 中非最新的記錄
+DELETE FROM playbooks
+WHERE status = 'deprecated'
+ AND source = 'yaml_rule'
+ AND playbook_id NOT IN (
+ -- 每個 name 保留 created_at 最新的那一筆
+ SELECT DISTINCT ON (name) playbook_id
+ FROM playbooks
+ WHERE status = 'deprecated'
+ AND source = 'yaml_rule'
+ ORDER BY name, created_at DESC
+ );
+
+-- 執行後確認
+-- SELECT source, status, COUNT(*) FROM playbooks GROUP BY source, status ORDER BY source, status;
+
+COMMIT;
diff --git a/apps/api/src/api/v1/drift.py b/apps/api/src/api/v1/drift.py
index b5954f206..63a242a89 100644
--- a/apps/api/src/api/v1/drift.py
+++ b/apps/api/src/api/v1/drift.py
@@ -22,12 +22,15 @@ from src.models.drift import (
DriftReport,
DriftScanRequest,
DriftScanResponse,
+ DriftStatus,
)
from src.repositories.drift_repository import get_drift_repository
+from src.services.drift_adopt_service import get_drift_adopt_service
from src.services.drift_analyzer import get_drift_analyzer
from src.services.drift_detector import get_drift_detector
from src.services.drift_interpreter import get_drift_interpreter
from src.services.drift_remediator import get_drift_remediator
+from src.utils.timezone import now_taipei
router = APIRouter(prefix="/drift", tags=["drift"])
@@ -155,7 +158,17 @@ async def internal_scan(background_tasks: BackgroundTasks) -> dict:
# =============================================================================
async def _analyze_and_notify(report: DriftReport) -> None:
- """背景:Nemotron 意圖分析 + Telegram 推送 + Phase 30 AI 人話摘要"""
+ """
+ 背景:Nemotron 意圖分析 + 低風險自動採納嘗試 + Telegram 推送
+
+ 2026-04-24 ogt + Claude Sonnet 4.6: 新增低風險自動採納
+ 流程:
+ 1. Nemotron 意圖分析(同原先)
+ 2. 嘗試 auto_adopt_if_safe():
+ - 通過 → 發 TYPE-1 無按鈕通知(PR 已建立,請 SRE 複核),不再推送帶按鈕卡片
+ - 未通過(skipped=True)→ 走原有 narrator TYPE-4D 卡片流程
+ - 採納失敗(skipped=False, success=False)→ 同樣走 narrator 讓人工介入
+ """
import structlog as _structlog
_logger = _structlog.get_logger(__name__)
try:
@@ -165,6 +178,38 @@ async def _analyze_and_notify(report: DriftReport) -> None:
repo = get_drift_repository()
await repo.update_interpretation(report.report_id, interpretation)
+ # 2026-04-24: 嘗試低風險自動採納
+ auto_adopted = False
+ try:
+ adopt_svc = get_drift_adopt_service()
+ auto_result = await adopt_svc.auto_adopt_if_safe(report)
+ if auto_result.get("success"):
+ # 自動採納成功:更新狀態,跳過人工卡片
+ await repo.update_status(
+ report.report_id,
+ DriftStatus.ADOPTED,
+ resolved_at=now_taipei(),
+ )
+ auto_adopted = True
+ _logger.info(
+ "drift_auto_adopted",
+ report_id=report.report_id,
+ pr_url=auto_result.get("pr_url"),
+ )
+ else:
+ _logger.info(
+ "drift_auto_adopt_skipped",
+ report_id=report.report_id,
+ reason=auto_result.get("reason", ""),
+ skipped=auto_result.get("skipped", True),
+ )
+ except Exception as e:
+ _logger.warning("drift_auto_adopt_error", report_id=report.report_id, error=str(e))
+
+ if auto_adopted:
+ # 自動採納成功,Telegram 通知已在 auto_adopt_if_safe 內發出,不再推送按鈕卡片
+ return
+
# ADR-075: drift_narrator_service 負責發送 TYPE-4D 卡片(含按鈕)
# 舊的 send_text() 已移除,改由 narrate_and_notify() 統一處理
try:
diff --git a/apps/api/src/jobs/ai_slo_watchdog_job.py b/apps/api/src/jobs/ai_slo_watchdog_job.py
index 74978187a..d4aff21bf 100644
--- a/apps/api/src/jobs/ai_slo_watchdog_job.py
+++ b/apps/api/src/jobs/ai_slo_watchdog_job.py
@@ -3,11 +3,16 @@ AI SLO Watchdog Job — 系統自健診(每 15 分鐘)
=============================================
MASTER §1.1 AI 自主化方向:系統必須能感知自身故障。
ADR-092 (2026-04-20 ogt + Claude Opus 4.7 Asia/Taipei)
+ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
+ W-2 修復:改用 telegram_message_id IS NULL 判斷真正靜默,排除 tg_sent TTL 過期誤判
+ W-5 新增:Agent Debate 失敗導致告警卡在分析中(description='待分析')
檢查項目:
W-1 AI SLO 違反(決策品質,7d 滾動)
- W-2 Telegram 靜默偵測(PENDING 告警無 tg_sent 確認超過 30 分鐘)
+ W-2 Telegram 靜默偵測(PENDING 告警 telegram_message_id IS NULL 超過 30 分鐘)
W-3 飛輪 execution_success_rate 低落(< 30%)
+ W-4 無 APPROVED Playbook(自動修復鏈路斷裂)
+ W-5 Agent Debate 失敗(PENDING 告警 description='待分析' 超過 1 小時)
任一異常 → send_meta_alert(TYPE-8M,flywheel_health)
去重:Redis watchdog:alert:{dedup_hash} TTL 1h,避免每 15 分鐘重複洗版
@@ -19,7 +24,7 @@ import uuid
from datetime import UTC, datetime, timedelta
import structlog
-from sqlalchemy import and_, select
+from sqlalchemy import and_, func, select
from src.core.redis_client import get_redis
from src.db.base import get_db_context
@@ -31,8 +36,9 @@ logger = structlog.get_logger(__name__)
_INTERVAL_SEC = 900 # 每 15 分鐘
_DEDUP_TTL_SEC = 3600 # 同一告警 1 小時內不重複發送
-_TG_SILENCE_THRESHOLD = 2 # PENDING 無 tg_sent 確認數量告警門檻
+_TG_SILENCE_THRESHOLD = 2 # PENDING telegram_message_id IS NULL 告警門檻
_FLYWHEEL_SUCCESS_MIN = 0.30 # 執行成功率下限
+_STUCK_ANALYSIS_THRESHOLD = 3 # Agent Debate 失敗導致卡住的告警門檻
async def run_ai_slo_watchdog_loop() -> None:
@@ -62,11 +68,17 @@ async def _check_once() -> None:
except Exception as e:
logger.warning("watchdog_w1_slo_check_failed", error=str(e))
- # W-2: Telegram 靜默偵測(PENDING 無 tg_sent 確認 > 30 分鐘)
+ # W-2: Telegram 靜默偵測(PENDING 告警 telegram_message_id IS NULL 超過 30 分鐘)
+ # ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
+ # 修復誤判:改用 telegram_message_id IS NULL 判斷,排除 tg_sent Redis TTL 過期導致的誤報。
+ # 有 telegram_message_id = Telegram 已發過,tg_sent key 過期不算靜默;
+ # telegram_message_id IS NULL = Telegram 從未發過,才是真正靜默。
try:
silent_count = await _count_pending_no_tg_sent()
if silent_count >= _TG_SILENCE_THRESHOLD:
- violations.append(f"{silent_count} 個 PENDING 告警超 30 分鐘無 Telegram 確認(疑似靜默故障)")
+ violations.append(
+ f"{silent_count} 個 PENDING 告警超 30 分鐘未送達 Telegram(未曾發送,非 TTL 過期)"
+ )
except Exception as e:
logger.warning("watchdog_w2_tg_silence_check_failed", error=str(e))
@@ -87,8 +99,21 @@ async def _check_once() -> None:
except Exception as e:
logger.warning("watchdog_w4_playbook_check_failed", error=str(e))
+ # W-5: Agent Debate 失敗(PENDING 告警 description='待分析' 超過 1 小時)
+ # ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
+ # telegram_push_suppressed_no_analysis 那類案例在 DB 留下 PENDING + description='待分析',
+ # 查 DB 比查 structlog 更可靠。門檻 > 3(1h 內若偶有 1-2 筆超時可接受)。
+ try:
+ stuck_count = await _count_pending_stuck_analysis()
+ if stuck_count > _STUCK_ANALYSIS_THRESHOLD:
+ violations.append(
+ f"Agent Debate 失敗導致 {stuck_count} 個告警分析卡住(PENDING + description='待分析' 超過 1 小時)"
+ )
+ except Exception as e:
+ logger.warning("watchdog_w5_stuck_analysis_check_failed", error=str(e))
+
if not violations:
- logger.debug("ai_slo_watchdog_all_ok", checks=4)
+ logger.debug("ai_slo_watchdog_all_ok", checks=5)
return
# 去重:violations 相同內容 1 小時內不重複發
@@ -112,7 +137,7 @@ async def _check_once() -> None:
alert_category="flywheel_health",
diagnosis=diagnosis,
severity_level="critical",
- system_impact=f"{len(violations)} 項 KPI 異常,飛輪自動化能力可能降級",
+ system_impact=f"{len(violations)} 項 KPI 異常(W-1~W-5),飛輪自動化能力可能降級",
)
logger.warning(
"ai_slo_watchdog_alert_sent",
@@ -126,34 +151,34 @@ async def _check_once() -> None:
async def _count_pending_no_tg_sent() -> int:
"""
- 查詢 PENDING 超過 30 分鐘且 Redis tg_sent:{fingerprint} 無確認的告警數量。
- 與 ADR-092 B2 修復配合:B2 修復後新告警會標記 tg_sent;
- 此查詢偵測仍存在的靜默告警(B2 修復前殘留 + 未來潛在故障)。
+ 查詢真正靜默的 PENDING 告警:PENDING 超過 30 分鐘且 telegram_message_id IS NULL。
+
+ ADR-092 B3 修復(2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
+ 舊版只查 Redis tg_sent:{fp} key,但該 key TTL 僅 24h;
+ PENDING 超過 24h 後 key 自然過期 → 誤判為靜默故障(W-2 誤報根因)。
+
+ 修復方案:改用 PG 欄位 telegram_message_id IS NULL 作為判斷依據:
+ - telegram_message_id IS NOT NULL = Telegram 確實已發過(不算靜默)
+ - telegram_message_id IS NULL = Telegram 從未發過(才是真正靜默)
+ tg_sent Redis key 不再用於 W-2 判斷。
"""
cutoff = datetime.now(UTC) - timedelta(minutes=30)
- redis = get_redis()
- silent = 0
async with get_db_context() as db:
result = await db.execute(
- select(ApprovalRecord.id, ApprovalRecord.fingerprint)
+ select(ApprovalRecord.id)
.where(
and_(
ApprovalRecord.status == ApprovalStatus.PENDING,
ApprovalRecord.created_at <= cutoff,
- ApprovalRecord.fingerprint.isnot(None),
+ ApprovalRecord.telegram_message_id.is_(None),
)
)
.limit(20)
)
rows = result.all()
- for row in rows:
- fp = row.fingerprint
- if fp and not await redis.exists(f"tg_sent:{fp}"):
- silent += 1
-
- return silent
+ return len(rows)
async def _count_approved_playbooks() -> int:
@@ -164,3 +189,37 @@ async def _count_approved_playbooks() -> int:
sa_text("SELECT COUNT(*) FROM playbooks WHERE status = 'approved'")
)
return result.scalar() or 0
+
+
+async def _count_pending_stuck_analysis() -> int:
+ """
+ 查詢 Agent Debate 失敗導致卡住的 PENDING 告警數。
+
+ ADR-092 B3(2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
+ decision_manager.py 第 279-286 行:當 description='待分析' 且 action 屬於
+ 失敗狀態集合時,直接 return 不推送 Telegram,記錄 telegram_push_suppressed_no_analysis。
+ 這類告警在 DB 中留下 PENDING + description='待分析' 的特徵。
+
+ 查詢條件:
+ - status = PENDING
+ - description = '待分析'(Agent Debate 無有效輸出)
+ - created_at <= now - 1h(給一定緩衝,排除剛建立的正常案例)
+ """
+ cutoff = datetime.now(UTC) - timedelta(hours=1)
+
+ async with get_db_context() as db:
+ # 用 action IS NULL/空 比 description 字串比對更可靠:
+ # decision_manager 超時降級後 description 文字已改,但 action 仍為空
+ result = await db.execute(
+ select(func.count())
+ .select_from(ApprovalRecord)
+ .where(
+ and_(
+ ApprovalRecord.status == ApprovalStatus.PENDING,
+ ApprovalRecord.created_at <= cutoff,
+ ApprovalRecord.suggested_action.in_(["", "待分析", "NO_ACTION", None]),
+ ApprovalRecord.telegram_message_id.is_(None),
+ )
+ )
+ )
+ return result.scalar() or 0
diff --git a/apps/api/src/jobs/approval_timeout_resolver.py b/apps/api/src/jobs/approval_timeout_resolver.py
index a54cd98c5..5802f2317 100644
--- a/apps/api/src/jobs/approval_timeout_resolver.py
+++ b/apps/api/src/jobs/approval_timeout_resolver.py
@@ -22,6 +22,10 @@ disposition 記錄:
4. 每次執行記錄 resolved_count / error_count
2026-04-15 ogt + Claude Sonnet 4.6(亞太):P2 飛輪斷鏈修復
+ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
+ 執行間隔從 3600s(1h)縮短至 900s(15 分鐘),與 W-2 Watchdog 同頻,
+ 減少 PENDING 積壓導致 W-2 誤報的時間視窗;
+ BATCH_LIMIT 從 50 提升至 200,加快存量清理速度。
"""
from __future__ import annotations
@@ -40,13 +44,17 @@ from src.utils.timezone import now_taipei
logger = structlog.get_logger(__name__)
# 每次最多處理幾筆,避免單次執行阻塞過長
-BATCH_LIMIT = 50
+# ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):從 50 提升至 200,加快存量清理
+BATCH_LIMIT = 200
async def run_approval_timeout_resolver() -> None:
"""
- 無限迴圈:每小時執行一次逾期 Approval 結案掃描。
+ 無限迴圈:每 15 分鐘執行一次逾期 Approval 結案掃描。
在 main.py startup 以 asyncio.create_task 掛載。
+ ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
+ 從 3600s(1h)縮短至 900s(15 分鐘),與 ai_slo_watchdog_job 同頻,
+ 減少 W-2 因 PENDING 積壓誤報的時間視窗。
"""
while True:
try:
@@ -60,7 +68,7 @@ async def run_approval_timeout_resolver() -> None:
except Exception as e:
logger.error("approval_timeout_resolver_loop_error", error=str(e))
- await asyncio.sleep(3600) # 每小時執行一次
+ await asyncio.sleep(900) # ADR-092 B3: 每 15 分鐘執行(原 3600s)
async def _resolve_expired_approvals() -> tuple[int, int]:
diff --git a/apps/api/src/plugins/mcp/providers/k8s_provider.py b/apps/api/src/plugins/mcp/providers/k8s_provider.py
index 9cfb57b43..d904a426e 100644
--- a/apps/api/src/plugins/mcp/providers/k8s_provider.py
+++ b/apps/api/src/plugins/mcp/providers/k8s_provider.py
@@ -371,7 +371,12 @@ class K8sProvider(MCPToolProvider):
# =========================================================================
async def _k8s_get_pod_logs(self, executor, parameters: dict) -> dict:
- pod_name = _validate_name(parameters["pod_name"], "pod_name")
+ # Bug 根因:LLM 有時傳空字串 pod_name,_validate_name 直接 raise ValueError 炸整個 investigator
+ # 修復:pod_name 空時返回結構化 error,讓 Agent 可繼續分析其他工具(2026-04-24 Claude Sonnet 4.6)
+ raw_pod_name = parameters.get("pod_name", "")
+ if not raw_pod_name or not raw_pod_name.strip():
+ return {"error": "empty_pod_name", "message": "pod_name 為空,請提供具體 pod 名稱(如 awoooi-api-xxx-yyy)"}
+ pod_name = _validate_name(raw_pod_name, "pod_name")
namespace = _validate_namespace(parameters.get("namespace", DEFAULT_NAMESPACE))
container = parameters.get("container", "")
tail = max(1, min(int(parameters.get("tail", 100)), 500))
@@ -437,7 +442,11 @@ class K8sProvider(MCPToolProvider):
}
async def _k8s_describe_pod(self, executor, parameters: dict) -> dict:
- pod_name = _validate_name(parameters["pod_name"], "pod_name")
+ # Bug 根因:同 _k8s_get_pod_logs,空 pod_name 會 raise 炸 investigator(2026-04-24 Claude Sonnet 4.6)
+ raw_pod_name = parameters.get("pod_name", "")
+ if not raw_pod_name or not raw_pod_name.strip():
+ return {"error": "empty_pod_name", "message": "pod_name 為空,請提供具體 pod 名稱(如 awoooi-api-xxx-yyy)"}
+ pod_name = _validate_name(raw_pod_name, "pod_name")
namespace = _validate_namespace(parameters.get("namespace", DEFAULT_NAMESPACE))
cmd = f"kubectl describe pod {pod_name} -n {namespace}"
diff --git a/apps/api/src/plugins/mcp/providers/prometheus_provider.py b/apps/api/src/plugins/mcp/providers/prometheus_provider.py
index e2636a92e..ccb7fea26 100644
--- a/apps/api/src/plugins/mcp/providers/prometheus_provider.py
+++ b/apps/api/src/plugins/mcp/providers/prometheus_provider.py
@@ -191,7 +191,11 @@ class PrometheusProvider(MCPToolProvider):
# =========================================================================
async def _instant_query(self, params: dict) -> dict:
- query = params["query"]
+ # Bug 根因:LLM 產生的 payload 有時用 promql 而非 query,直接存取 ["query"] 會 KeyError
+ # 修復:優先取 query,fallback promql;兩者皆缺則返回結構化 error(2026-04-24 Claude Sonnet 4.6)
+ query = params.get("query", "") or params.get("promql", "")
+ if not query:
+ return {"error": "missing_query_parameter", "message": "必須提供 query 或 promql 參數"}
time_param = params.get("time", "")
url = f"{self._base_url()}/api/v1/query"
@@ -216,7 +220,10 @@ class PrometheusProvider(MCPToolProvider):
}
async def _range_query(self, params: dict) -> dict:
- query = params["query"]
+ # Bug 根因:同 _instant_query,LLM 可能用 promql 而非 query(2026-04-24 Claude Sonnet 4.6)
+ query = params.get("query", "") or params.get("promql", "")
+ if not query:
+ return {"error": "missing_query_parameter", "message": "必須提供 query 或 promql 參數"}
window_minutes = int(params.get("window_minutes", 15))
step_seconds = int(params.get("step_seconds", 60))
@@ -251,7 +258,10 @@ class PrometheusProvider(MCPToolProvider):
}
async def _alert_history(self, params: dict) -> dict:
- alertname = params["alertname"]
+ # Bug 根因:LLM 偶發 alertname 缺失,直接存取 ["alertname"] 會 KeyError(2026-04-24 Claude Sonnet 4.6)
+ alertname = params.get("alertname", "")
+ if not alertname:
+ return {"error": "missing_alertname_parameter", "message": "必須提供 alertname 參數"}
# P1 fix 2026-04-11: 白名單驗證防止 PromQL label injection
if not _RE_SAFE_ALERTNAME.match(alertname):
raise ValueError(f"Unsafe alertname: {alertname!r}")
diff --git a/apps/api/src/plugins/mcp/providers/ssh_provider.py b/apps/api/src/plugins/mcp/providers/ssh_provider.py
index 2bcab0e4f..2ddbd653f 100644
--- a/apps/api/src/plugins/mcp/providers/ssh_provider.py
+++ b/apps/api/src/plugins/mcp/providers/ssh_provider.py
@@ -544,5 +544,6 @@ class SSHProvider(MCPToolProvider):
known_hosts=known_hosts_path, # None = 跳過驗證(內網),或指定文件路徑
connect_timeout=timeout,
) as conn:
- result = await asyncssh.run(conn, cmd, timeout=timeout, check=False)
+ # Bug 根因:asyncssh 模組沒有頂層 run();應呼叫 conn.run()(2026-04-24 Claude Sonnet 4.6)
+ result = await conn.run(cmd, timeout=timeout, check=False)
return (result.stdout or ""), (result.stderr or "")
diff --git a/apps/api/src/services/alert_rule_engine.py b/apps/api/src/services/alert_rule_engine.py
index 5a2e6cd5b..be88925a2 100644
--- a/apps/api/src/services/alert_rule_engine.py
+++ b/apps/api/src/services/alert_rule_engine.py
@@ -703,25 +703,58 @@ async def auto_generate_rule(
# 1. 先試 Ollama
raw = await _call_ollama(prompt, ollama_url, model)
+ llm_source = "ollama" if raw else None
# 2. Ollama 失敗 → Gemini
if not raw and gemini_api_key:
raw = await _call_gemini(prompt, gemini_api_key)
+ llm_source = "gemini" if raw else None
if not raw:
- logger.warning("auto_rule_no_response", alertname=alertname_safe)
+ logger.warning(
+ "auto_rule_auto_generate_failed",
+ alertname=alertname_safe,
+ reason="llm_no_response",
+ )
return
yaml_block = _extract_yaml_block(raw)
success = _append_rule_to_yaml(yaml_block, alertname_safe)
if success:
- logger.info("auto_rule_success", alertname=alertname_safe, rule_id=rule_id)
- # 立即為新規則建立 APPROVED Playbook(不等下次重啟)
+ logger.info(
+ "alert_rule_auto_generated",
+ alertname=alertname_safe,
+ rule_id=rule_id,
+ source=llm_source,
+ )
+ # 成功後記錄今日新增規則數(供系統報告顯示)
import asyncio as _asyncio
+ try:
+ from src.core.redis_client import get_redis as _get_redis
+ from src.utils.timezone import now_taipei as _now_taipei
+ _today = _now_taipei().strftime("%Y%m%d")
+ _key = f"stats:auto_rule_generated:{_today}"
+ _redis = _get_redis()
+
+ async def _incr_rule_stat() -> None:
+ # pipeline 原子化 incr+expire,避免 race condition
+ async with _redis.pipeline() as _p:
+ _p.incr(_key)
+ _p.expire(_key, 86400 * 7)
+ await _p.execute()
+
+ _asyncio.create_task(_incr_rule_stat())
+ except Exception as _redis_err:
+ logger.debug("auto_rule_stats_redis_failed", error=str(_redis_err))
+ # 立即為新規則建立 APPROVED Playbook(不等下次重啟)
from src.services.playbook_seed_service import seed_playbooks_from_rules
_asyncio.create_task(seed_playbooks_from_rules())
else:
- logger.warning("auto_rule_failed_validation", alertname=alertname_safe)
+ logger.warning(
+ "auto_rule_auto_generate_failed",
+ alertname=alertname_safe,
+ reason="yaml_validation_failed",
+ )
except Exception as e:
logger.error("auto_rule_exception", alertname=alertname_safe, error=str(e))
diff --git a/apps/api/src/services/approval_db.py b/apps/api/src/services/approval_db.py
index f5739f57e..8b0b29db6 100644
--- a/apps/api/src/services/approval_db.py
+++ b/apps/api/src/services/approval_db.py
@@ -357,11 +357,15 @@ class ApprovalDBService:
return None
- async def mark_telegram_confirmed(self, fingerprint: str, ttl: int = 86400) -> None:
+ async def mark_telegram_confirmed(self, fingerprint: str, ttl: int = 108000) -> None:
"""
2026-04-20 ogt + Claude Opus 4.7: ADR-092
記錄 Telegram 已成功發送,防止 PENDING 誤收斂造成永久靜默。
- TTL 與 PENDING_TTL_HOURS 對齊(24h)。
+ ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
+ TTL 從 86400s(24h)延長至 108000s(30h = 24h + 6h buffer),
+ 避免 PENDING 存活期與 tg_sent key 同時到期時的邊緣誤判。
+ W-2 現已改用 telegram_message_id IS NULL 判斷(不再依賴此 key),
+ 此 TTL 保留供收斂去重邏輯(approval_db.py 第 342 行)使用。
"""
try:
r = get_redis()
diff --git a/apps/api/src/services/auto_approve.py b/apps/api/src/services/auto_approve.py
index fa4ade656..1a347447c 100644
--- a/apps/api/src/services/auto_approve.py
+++ b/apps/api/src/services/auto_approve.py
@@ -450,6 +450,28 @@ class AutoApprovePolicy:
trust_score=kwargs.get("trust_score"),
)
+ # 記錄拒絕原因計數(供系統報告分析人工審核積壓根因)
+ # 在 async context 中呼叫,用 create_task 不阻塞主流程
+ try:
+ import asyncio as _asyncio
+ from datetime import datetime as _dt
+ _today = _dt.now().strftime("%Y%m%d")
+ _reject_key = f"stats:auto_approve_rejected:{reason.value}:{_today}"
+
+ async def _incr_reject_stat() -> None:
+ try:
+ from src.core.redis_client import get_redis as _get_redis
+ _r = _get_redis()
+ await _r.incr(_reject_key)
+ await _r.expire(_reject_key, 86400 * 7)
+ except Exception:
+ pass # Redis 不可用時靜默降級,不影響核心流程
+
+ loop = _asyncio.get_running_loop()
+ loop.create_task(_incr_reject_stat())
+ except RuntimeError:
+ pass # 非 async context(如單元測試),靜默跳過
+
return decision
def _extract_action_pattern(self, action: str) -> str:
diff --git a/apps/api/src/services/decision_manager.py b/apps/api/src/services/decision_manager.py
index aa3bb4f2d..4ba8b8733 100644
--- a/apps/api/src/services/decision_manager.py
+++ b/apps/api/src/services/decision_manager.py
@@ -1114,7 +1114,23 @@ def _package_to_proposal_data(package: Any) -> dict[str, Any]:
# 2026-04-17 ogt + Claude Sonnet 4.6(亞太): 修復超時髒資料污染卡片
# 舊:blocked_reason → desc_parts → description → suggested_action 欄位顯示「備注:全局超時 > 90.0s」
# 新:blocked_reason 只寫入 proposal_data["blocked_reason"],供下游閘門邏輯用,禁止進卡片顯示
- description = ";".join(desc_parts) if desc_parts else (action[:200] if action else "待分析")
+ # 2026-04-24 ogt + Claude Sonnet 4.6: ADR-091 鐵閘 bypass 修復
+ # 問題:Agent Debate 超時 (TIMEOUT/FAILED) → desc_parts 空 + action 空 → description="待分析"
+ # → ADR-091 鐵閘攔截(description=待分析 AND action in FAILED_TOKENS)→ PENDING 積壓
+ # 修法:超時/全降級時使用明確降級文字,讓鐵閘放行 → Telegram 推送降級通知 → 人工可見
+ if desc_parts:
+ description = ";".join(desc_parts)
+ elif action:
+ description = action[:200]
+ else:
+ _status = getattr(package, "session_status", None)
+ _status_val = _status.value if _status and hasattr(_status, "value") else ""
+ if _status_val == "timeout":
+ description = "AI 分析超時(90s),降級至人工審核"
+ elif _status_val == "failed" or getattr(package, "all_agents_degraded", False):
+ description = "AI 分析降級,所有 Agent 無有效輸出,建議人工確認"
+ else:
+ description = "待分析"
return {
"action": action,
diff --git a/apps/api/src/services/drift_adopt_service.py b/apps/api/src/services/drift_adopt_service.py
index f049c87bb..7e30dac56 100644
--- a/apps/api/src/services/drift_adopt_service.py
+++ b/apps/api/src/services/drift_adopt_service.py
@@ -35,7 +35,7 @@ from src.core.config import get_settings
from src.utils.timezone import now_taipei
if TYPE_CHECKING:
- from src.models.drift import DriftReport
+ from src.models.drift import DriftInterpretation, DriftReport
logger = structlog.get_logger(__name__)
@@ -59,6 +59,165 @@ class DriftAdoptService:
self._repo = settings.GITEA_REPO_NAME
self._k8s_dir = pathlib.Path("k8s")
+ async def auto_adopt_if_safe(self, report: "DriftReport") -> dict:
+ """
+ 低風險 drift 自動採納(2026-04-24 ogt + Claude Sonnet 4.6)
+
+ 自動採納條件(全部滿足):
+ 1. high_count == 0(無 HIGH drift)
+ 2. medium_count <= 5(MEDIUM drift 不超過 5 項)
+ 3. actionable_count <= 10(非白名單非 trivial 項目不超過 10)
+ 4. interpretation.confidence >= 0.70(意圖確定性足夠)
+ 5. interpretation.intent NOT IN {unknown}(必須有明確意圖)
+ 6. interpretation.risk NOT IN {HIGH, CRITICAL}(非高風險)
+
+ 不滿足任一條件 → return {"success": False, "reason": "...", "skipped": True}
+ 自動採納 PR title 加 [AUTO] 前綴,通知標示「請 SRE 複核後 merge」
+
+ Returns:
+ {"success": bool, "skipped": bool, "reason": str, "pr_url": str | None}
+ skipped=True → 條件不符,人工卡片照舊推送
+ skipped=False → 嘗試了,但 Gitea API 失敗
+ """
+ interp = report.interpretation
+
+ # 條件 1: 無 HIGH drift
+ if report.high_count > 0:
+ return {
+ "success": False,
+ "skipped": True,
+ "reason": f"存在 {report.high_count} 項 HIGH drift,不自動採納",
+ "pr_url": None,
+ }
+
+ # 條件 2: MEDIUM 不超過 5 項
+ if report.medium_count > 5:
+ return {
+ "success": False,
+ "skipped": True,
+ "reason": f"MEDIUM drift {report.medium_count} 項超過上限 5,不自動採納",
+ "pr_url": None,
+ }
+
+ # 條件 3: actionable 不超過 10(過濾白名單 + trivial)
+ actionable_items = [
+ item for item in report.items
+ if not item.is_allowlisted
+ and not self._is_trivial_drift(item.git_value, item.actual_value)
+ ]
+ actionable_count = len(actionable_items)
+ if actionable_count > 10:
+ return {
+ "success": False,
+ "skipped": True,
+ "reason": f"可操作漂移 {actionable_count} 項超過上限 10,不自動採納",
+ "pr_url": None,
+ }
+
+ # 條件 4 / 5 / 6: 需要有 interpretation
+ if not interp:
+ return {
+ "success": False,
+ "skipped": True,
+ "reason": "尚無 Nemotron 意圖分析,不自動採納",
+ "pr_url": None,
+ }
+
+ if interp.confidence < 0.70:
+ return {
+ "success": False,
+ "skipped": True,
+ "reason": f"意圖信心 {interp.confidence:.0%} < 70%,不自動採納",
+ "pr_url": None,
+ }
+
+ if interp.intent.value == "unknown":
+ return {
+ "success": False,
+ "skipped": True,
+ "reason": "Nemotron 意圖為 unknown,不自動採納",
+ "pr_url": None,
+ }
+
+ if interp.risk.upper() in ("HIGH", "CRITICAL"):
+ return {
+ "success": False,
+ "skipped": True,
+ "reason": f"Nemotron 風險評估為 {interp.risk},不自動採納",
+ "pr_url": None,
+ }
+
+ logger.info(
+ "drift_auto_adopt_eligible",
+ report_id=report.report_id,
+ medium_count=report.medium_count,
+ actionable=actionable_count,
+ intent=interp.intent.value,
+ confidence=interp.confidence,
+ risk=interp.risk,
+ )
+
+ # 通過全部條件 → 執行採納(suppress_notification=True 避免與下方通知重複)
+ result = await self.adopt(report, field_description="[AUTO] 低風險自動採納", suppress_notification=True)
+ if result.get("success"):
+ # 通知 SRE:無按鈕,標示需複核
+ await self._notify_auto_adopt_telegram(result["pr_url"], report, actionable_count, interp)
+
+ return {
+ "success": result.get("success", False),
+ "skipped": False,
+ "reason": result.get("message", ""),
+ "pr_url": result.get("pr_url"),
+ }
+
+ @staticmethod
+ def _is_trivial_drift(git_val, actual_val) -> bool:
+ """
+ 判斷是否為 K8s controller 自動補齊的噪音
+ (與 DriftNarratorService._is_trivial_drift 邏輯一致)
+ """
+ def _is_empty(v) -> bool:
+ if v is None:
+ return True
+ s = str(v).strip()
+ return s in ("", "{}", "[]", "null", "None", "false", "False", "0")
+ return _is_empty(git_val) and _is_empty(actual_val)
+
+ async def _notify_auto_adopt_telegram(
+ self,
+ pr_url: str,
+ report: "DriftReport",
+ actionable_count: int,
+ interp: "DriftInterpretation",
+ ) -> None:
+ """
+ 自動採納成功後的無按鈕 Telegram 通知(TYPE-1 純資訊)
+ 標示「已自動建立 PR,請 SRE 複核後 merge」
+ """
+ try:
+ from src.services.telegram_gateway import get_telegram_gateway
+ tg = get_telegram_gateway()
+ message = (
+ f"Namespace: {report.namespace}\n"
+ f"漂移摘要: {report.summary}(可操作 {actionable_count} 項)\n"
+ f"Nemotron 意圖: {interp.intent.value} | 信心: {interp.confidence:.0%} | 風險: {interp.risk}\n\n"
+ f"PR: {pr_url}\n\n"
+ f"請 SRE 複核後 merge。"
+ )
+ await tg.send_info_notification(
+ incident_id=report.report_id,
+ title="Config Drift 已自動採納(低風險)",
+ message=message,
+ alertname="ConfigDriftAutoAdopt",
+ severity="info",
+ )
+ except Exception as e:
+ logger.warning("drift_auto_adopt_telegram_failed", error=str(e))
+
+ # =========================================================================
+ # Public: 人工按鈕觸發入口
+ # =========================================================================
+
async def adopt_drift(self, report_id: str) -> dict:
"""
2026-04-20 ogt + Claude Opus 4.7: Telegram 按鈕呼叫入口
@@ -71,7 +230,7 @@ class DriftAdoptService:
return {"success": False, "message": f"Report {report_id} not found"}
return await self.adopt(report)
- async def adopt(self, report: "DriftReport", field_description: str = "") -> dict:
+ async def adopt(self, report: "DriftReport", field_description: str = "", suppress_notification: bool = False) -> dict:
"""
將漂移寫回 Git:建立 branch + commit + PR
@@ -126,8 +285,9 @@ class DriftAdoptService:
if not pr_url:
return {"success": False, "message": "建立 PR 失敗", "pr_url": None}
- # Step 5: 推送 Telegram 通知
- await self._notify_telegram(pr_url, report, pr_title)
+ # Step 5: 推送 Telegram 通知(auto_adopt_if_safe 會自己發,避免重複)
+ if not suppress_notification:
+ await self._notify_telegram(pr_url, report, pr_title)
logger.info("drift_adopt_pr_created", report_id=report.report_id, pr_url=pr_url)
return {
diff --git a/apps/api/src/services/heartbeat_report_service.py b/apps/api/src/services/heartbeat_report_service.py
index 63686ee84..b89116dd5 100644
--- a/apps/api/src/services/heartbeat_report_service.py
+++ b/apps/api/src/services/heartbeat_report_service.py
@@ -87,6 +87,19 @@ class TelegramBotStats:
last_callback_ago_min: Optional[float] = None
+@dataclass
+class AutomationStats:
+ """自動化六大能力今日統計(2026-04-24 ogt: Task 3 — 告警自動化可觀測性)"""
+ # 自動規則生成
+ auto_rule_generated_today: int = 0
+ # 自動審核拒絕(按原因分組)
+ reject_counts: dict[str, int] = field(default_factory=dict)
+ # 今日 KM 新增
+ km_created_today: int = 0
+ # Config Drift 自動採納(今日)
+ drift_adopted_today: int = 0
+
+
@dataclass
class HeartbeatReport:
timestamp: datetime
@@ -102,6 +115,8 @@ class HeartbeatReport:
pods: list[PodInfo] = field(default_factory=list)
scanners: ScannerStats = field(default_factory=ScannerStats)
telegram_bot: TelegramBotStats = field(default_factory=TelegramBotStats)
+ # 2026-04-24 自動化統計
+ automation: AutomationStats = field(default_factory=AutomationStats)
@property
def has_warnings(self) -> bool:
@@ -139,6 +154,8 @@ class HeartbeatReportService:
self._get_pod_status(),
self._get_scanner_stats(),
self._probe_telegram_bot(),
+ # 2026-04-24 自動化統計
+ self._get_automation_stats(),
return_exceptions=True,
)
@@ -147,6 +164,7 @@ class HeartbeatReportService:
"_mcp_k8s", "_mcp_ssh", "_mcp_argocd", "_mcp_sentry",
"_argocd_sync", "_velero", "_flywheel",
"_alert_pipeline", "_db_redis", "_pods", "_scanners", "_tg_bot",
+ "_automation",
]
collected: dict = {}
for key, result in zip(keys, results):
@@ -189,6 +207,8 @@ class HeartbeatReportService:
report.scanners = collected["_scanners"]
if collected["_tg_bot"]:
report.telegram_bot = collected["_tg_bot"]
+ if collected["_automation"]:
+ report.automation = collected["_automation"]
# --- 彙整 warnings ---
report.warnings = self._build_warnings(report)
@@ -568,6 +588,61 @@ class HeartbeatReportService:
s.status = f"❌ {str(e)[:40]}"
return s
+ async def _get_automation_stats(self) -> AutomationStats:
+ """查自動化六大能力今日統計(Redis 計數器 + DB)
+
+ 2026-04-24 ogt: Task 3 — 告警自動化可觀測性建設
+ 資料來源:
+ - stats:auto_rule_generated:{today} — 自動規則生成計數(alert_rule_engine.py 寫入)
+ - stats:auto_approve_rejected:{reason}:{today} — 拒絕自動審核原因分布
+ - knowledge_entries 今日新增(DB)
+ - approval_records drift_adopted 今日(DB)
+ """
+ stats = AutomationStats()
+ try:
+ from src.core.redis_client import get_redis
+ redis = get_redis()
+ today = now_taipei().strftime("%Y%m%d")
+
+ # 自動規則生成今日計數
+ raw = await redis.get(f"stats:auto_rule_generated:{today}")
+ stats.auto_rule_generated_today = int(raw) if raw else 0
+
+ # 自動審核拒絕原因分布
+ reject_keys = await redis.keys(f"stats:auto_approve_rejected:*:{today}")
+ for key in reject_keys:
+ key_str = key.decode() if isinstance(key, bytes) else key
+ # key 格式:stats:auto_approve_rejected:{reason}:{today}
+ parts = key_str.split(":")
+ reason_part = parts[2] if len(parts) >= 4 else key_str
+ count_raw = await redis.get(key_str)
+ stats.reject_counts[reason_part] = int(count_raw) if count_raw else 0
+ except Exception as e:
+ logger.debug("heartbeat_automation_redis_failed", error=str(e))
+
+ try:
+ from src.db.base import get_db_context
+ from sqlalchemy import text as sa_text
+ async with get_db_context() as db:
+ # 今日新增 KM(timestamptz 直接比較,不需 AT TIME ZONE)
+ km_today = await db.scalar(sa_text(
+ "SELECT COUNT(*) FROM knowledge_entries "
+ "WHERE created_at >= NOW() - interval '24 hours'"
+ ))
+ stats.km_created_today = int(km_today or 0)
+
+ # 今日 Config Drift 自動採納(查 drift_reports,adopt() 在此表更新 status)
+ drift_today = await db.scalar(sa_text(
+ "SELECT COUNT(*) FROM drift_reports "
+ "WHERE status = 'adopted' "
+ "AND resolved_at >= NOW() - interval '24 hours'"
+ ))
+ stats.drift_adopted_today = int(drift_today or 0)
+ except Exception as e:
+ logger.debug("heartbeat_automation_db_failed", error=str(e))
+
+ return stats
+
# =========================================================================
# Warnings 彙整
# =========================================================================
@@ -755,6 +830,22 @@ def report_to_telegram_html(report: HeartbeatReport) -> str:
lines.append("🤖 Telegram Bot")
lines.append(f"└─ {tg.status}")
+ # --- 自動化統計(2026-04-24)---
+ auto = report.automation
+ lines.append("")
+ lines.append("⚙️ 自動化統計(今日)")
+ lines.append(f"├─ 新規則自動生成: {auto.auto_rule_generated_today} 條")
+ lines.append(f"├─ KM 新增: {auto.km_created_today} 條")
+ lines.append(f"├─ Drift 自動採納: {auto.drift_adopted_today} 筆")
+ if auto.reject_counts:
+ reject_total = sum(auto.reject_counts.values())
+ top_reason = max(auto.reject_counts, key=auto.reject_counts.get) # type: ignore[arg-type]
+ lines.append(
+ f"└─ 人工審核攔截: {reject_total} 次 主因: {html.escape(top_reason)}"
+ )
+ else:
+ lines.append("└─ 人工審核攔截: 0 次")
+
# --- Warnings / 總結 ---
lines.append("")
if report.warnings:
diff --git a/apps/api/src/services/intent_classifier.py b/apps/api/src/services/intent_classifier.py
index 23cdf75d8..12ee6e386 100644
--- a/apps/api/src/services/intent_classifier.py
+++ b/apps/api/src/services/intent_classifier.py
@@ -603,7 +603,16 @@ class IntentClassifier:
error=str(e),
response_preview=result_text[:100],
)
- return self._llm_fallback_result("JSON 解析失敗")
+ # 2026-04-24 ogt + Claude Sonnet 4.6: JSON 解析失敗也降級至關鍵字結果
+ _kw_result = self._keyword_classify(text)
+ return IntentResult(
+ intent=_kw_result.intent,
+ confidence=_kw_result.confidence,
+ method="llm_parse_failed_keyword",
+ matched_keywords=_kw_result.matched_keywords,
+ detected_resources=_kw_result.detected_resources,
+ reasoning=f"LLM JSON 解析失敗降級 → {_kw_result.reasoning}",
+ )
except httpx.TimeoutException:
elapsed_ms = (time.time() - start_time) * 1000
@@ -611,7 +620,20 @@ class IntentClassifier:
"intent_llm_timeout",
elapsed_ms=round(elapsed_ms, 1),
)
- return self._llm_fallback_result("LLM 超時")
+ # 2026-04-24 ogt + Claude Sonnet 4.6: LLM 超時直接降級至關鍵字結果
+ # 問題:_llm_fallback_result 返回 confidence=0.0/UNKNOWN,和 keyword 結果 confidence 相同
+ # classify() 比較 0.0 > 0.0 = False → 走 keyword(正確),但已浪費 5s 超時時間
+ # 若 Ollama 後端不通,每次都等 5s 才降級 → ai_router/ci_auto_repair 延遲累積
+ # 修法:超時直接回 keyword 結果,method 標記 "llm_timeout_keyword" 供可觀測性追蹤
+ _kw_result = self._keyword_classify(text)
+ return IntentResult(
+ intent=_kw_result.intent,
+ confidence=_kw_result.confidence,
+ method="llm_timeout_keyword",
+ matched_keywords=_kw_result.matched_keywords,
+ detected_resources=_kw_result.detected_resources,
+ reasoning=f"LLM 超時降級({round(elapsed_ms, 0):.0f}ms)→ {_kw_result.reasoning}",
+ )
except Exception as e:
logger.warning(
@@ -619,7 +641,16 @@ class IntentClassifier:
error=str(e),
error_type=type(e).__name__,
)
- return self._llm_fallback_result(f"LLM 錯誤: {type(e).__name__}")
+ # 2026-04-24 ogt + Claude Sonnet 4.6: LLM 錯誤同樣降級至關鍵字結果
+ _kw_result = self._keyword_classify(text)
+ return IntentResult(
+ intent=_kw_result.intent,
+ confidence=_kw_result.confidence,
+ method="llm_error_keyword",
+ matched_keywords=_kw_result.matched_keywords,
+ detected_resources=_kw_result.detected_resources,
+ reasoning=f"LLM 錯誤降級({type(e).__name__})→ {_kw_result.reasoning}",
+ )
def _parse_intent_type(self, intent_str: str) -> IntentType:
"""解析意圖字串為 IntentType"""
diff --git a/apps/api/src/services/playbook_evolver.py b/apps/api/src/services/playbook_evolver.py
index 3859a7bfc..b8bd48b2b 100644
--- a/apps/api/src/services/playbook_evolver.py
+++ b/apps/api/src/services/playbook_evolver.py
@@ -286,14 +286,26 @@ async def _merge_similar(playbooks: list[Playbook], report: EvolverReport) -> No
# ─────────────────────────────────────────────────────────────────────────────
async def _fetch_all_active_playbooks() -> list[Playbook]:
- """抓取所有非 DEPRECATED 的 Playbook(用於 Evolver 掃描)。"""
+ """
+ 抓取所有非 DEPRECATED 的 Playbook(用於 Evolver 掃描)。
+
+ 2026-04-24 ogt + Claude Sonnet 4.6: 改為分兩次 query(DRAFT + APPROVED)
+ 原本 list_playbooks(limit=500) 不傳 status → 把 294 筆 deprecated 全載入記憶體,
+ 隨 deprecated 累積效能惡化;改用兩次 status 過濾查詢,只拉需要的資料。
+ """
try:
+ from src.models.playbook import PlaybookStatus
from src.services.playbook_service import get_playbook_service
service = get_playbook_service()
- # 不過濾 status — Evolver 需要看到 DRAFT + APPROVED
- playbooks_page, total = await service.list_playbooks(limit=500)
- return playbooks_page
+
+ draft_playbooks, _ = await service.list_playbooks(
+ status=PlaybookStatus.DRAFT, limit=500
+ )
+ approved_playbooks, _ = await service.list_playbooks(
+ status=PlaybookStatus.APPROVED, limit=500
+ )
+ return draft_playbooks + approved_playbooks
except Exception as e:
logger.warning("evolver_fetch_playbooks_failed", error=str(e))
return []
diff --git a/apps/api/src/services/playbook_seed_service.py b/apps/api/src/services/playbook_seed_service.py
index 0417ce6c5..3fe58d0e1 100644
--- a/apps/api/src/services/playbook_seed_service.py
+++ b/apps/api/src/services/playbook_seed_service.py
@@ -45,13 +45,16 @@ async def seed_playbooks_from_rules() -> None:
# 取得現有 YAML_RULE playbook,依 name 去重避免重複建立
# 2026-04-15 ogt: 不再用 list_playbooks(舊格式 repair_steps 會 validation error)
# 改用 raw SQL 只撈 name 欄位,更穩健
+ # 2026-04-24 ogt + Claude Sonnet 4.6: 修復重複建立/封存迴圈
+ # 舊邏輯 status != 'deprecated' 導致:deprecated 歷史記錄永遠存在 →
+ # 每次啟動都重建同名 Playbook(C1 保護後不再封存,但 deprecated 不清除)
+ # 修:不管 status,同名 yaml_rule 任何狀態存在即視為已存在,不重建
from src.db.base import get_db_context
from sqlalchemy import text as sa_text
async with get_db_context() as db:
rows = await db.execute(
sa_text(
"SELECT name FROM playbooks WHERE source = 'yaml_rule'"
- " AND status != 'deprecated'"
)
)
existing_names = {r[0] for r in rows.fetchall()}
diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md
index 6079e6d80..a6d2168b6 100644
--- a/docs/LOGBOOK.md
+++ b/docs/LOGBOOK.md
@@ -6,6 +6,142 @@
---
+## 📍 2026-04-24 — 12-Agent 全景盤點 + 六大自動化飛輪修復
+
+### 根因(截圖告警分析)
+- META 告警(W-2)誤判:tg_sent: Redis TTL 24h 過期被誤報為「Telegram 靜默」,實際 Telegram 早已發送
+- 真正根因:MCP Provider 三個 Bug → Agent Debate 90s 超時 → description="待分析" → ADR-091 鐵閘不推 Telegram
+- Config Drift 全人工:無自動採納觸發鏈路
+- Playbook Evolver 迴圈:HTTP 5xx 重複建立/deprecated(294 deprecated / 25 approved)
+
+### 修復內容(706行+,17 檔)
+
+| 模組 | 修復 |
+|------|------|
+| `ssh_provider.py` | `asyncssh.run` → `conn.run()`(API 用錯) |
+| `prometheus_provider.py` | KeyError 'query' → `.get()` fallback + `promql` alias |
+| `k8s_provider.py` | 空 pod_name → early return error dict |
+| `ai_slo_watchdog_job.py` | W-2 改用 `telegram_message_id IS NULL`;W-5 新增(Agent Debate 卡住) |
+| `approval_timeout_resolver.py` | 1h → 15min;BATCH_LIMIT 50 → 200 |
+| `approval_db.py` | tg_sent TTL 24h → 30h(buffer 防邊緣誤判) |
+| `drift_adopt_service.py` | 新增 `auto_adopt_if_safe()`(6 條件自動採納 PR) |
+| `drift.py` | 背景任務加自動採納邏輯(低風險走 auto,高風險走人工) |
+| `playbook_seed_service.py` | 冪等 SQL 修復(去掉 `AND status != 'deprecated'` 防重複建立) |
+| `playbook_evolver.py` | `_fetch_all_active_playbooks` 只載 APPROVED+DRAFT,不載 deprecated |
+| `alert_rule_engine.py` | 自動規則生成加 telemetry + Redis pipeline 原子 incr/expire |
+| `auto_approve.py` | 拒絕原因 Redis 計數(供系統報告展示) |
+| `heartbeat_report_service.py` | 新增「自動化統計」區塊(今日規則/KM/Drift/Playbook) |
+| `decision_manager.py` | Agent Debate 超時降級文字(通過 ADR-091 鐵閘) |
+| `intent_classifier.py` | LLM 超時 → keyword fallback(不浪費 5s 等待) |
+| `migrations/cleanup_duplicate_deprecated_playbooks.sql` | 一次性清理 294 筆重複 deprecated |
+
+### Critic 審查後追加修復
+- `heartbeat_report_service.py` SQL:移除 `AT TIME ZONE`(timestamptz 直接比較);drift_today 改查 drift_reports 表
+- `drift_adopt_service.py` 雙重 Telegram 問題:`suppress_notification=True` 避免 auto_adopt 重複發
+- `alert_rule_engine.py` Redis race:pipeline 原子化 incr+expire
+- `ai_slo_watchdog_job.py` W-5:改用 `action IS NULL/空 + telegram_message_id IS NULL` 更可靠
+
+### 待手動執行
+```bash
+psql $DATABASE_URL -f apps/api/migrations/cleanup_duplicate_deprecated_playbooks.sql
+```
+
+---
+
+## 📍 2026-04-22 — 系統報告動態化:新增 5 大區塊(commit 9244c5e)
+
+### 需求
+統帥:「系統報告需要更全面、更完整,服務增加了很多,必須動態滾動增刪」
+
+### 實作
+| 新區塊 | 資料來源 | 說明 |
+|--------|---------|------|
+| 📊 告警流水線(24h) | `approval_records.status` | total/pending/success/failed |
+| 🗄️ DB & Redis | PG `SELECT 1` + Redis info | 連線狀態 + key 數 |
+| ☸️ K8s Pods | kubectl get pods | ready/restart count |
+| ⏱️ Scanner 狀態 | Redis daily lock TTL | 今日是否已執行 |
+| 🤖 Telegram Bot | Redis `telegram:polling_leader` | polling leader 是否存活 |
+
+- 5 個 probe 方法用 `asyncio.gather(return_exceptions=True)` 並行,任一失敗不影響其他
+- `_build_warnings()` 新增 DB/Redis/PENDING>10/Pod 未就緒 四種警告條件
+- 新增 5 個 dataclass:AlertPipelineStats / DbRedisStats / PodInfo / ScannerStats / TelegramBotStats
+
+### Commit
+- `9244c5e` feat(heartbeat): 系統報告新增 5 大動態區塊
+
+---
+
+## 📍 2026-04-22 早 — 日報重發 + 自動修復 0% 兩大根因修復(commits ef1353b + 88af639)
+
+### 問題
+統帥:「日報重複發送兩次」+「自動修復成功率 0.0%」
+
+### 根因
+
+| 問題 | 根因 | 修法 |
+|------|------|------|
+| 日報重發 | `run_daily_report_loop` 沒有呼叫 `try_acquire_daily_lock`(其他 3 個 scanner 都有),4 個 Pod 各自送一份 | sleep 後加 `try_acquire_daily_lock("daily_report")`,搶不到的 Pod 直接 `continue` |
+| 修復率 0% | `_collect_repair_stats` 查 `incidents.outcome->>'execution_success'`,但整條執行鏈路從未將 `execution_success` 寫入此 JSON 欄位 | 改查 `approval_records.status = 'EXECUTION_SUCCESS/FAILED'`(唯一可靠 source of truth) |
+| SQL 大小寫 | DB 以 SQLEnum 儲存 enum name(EXECUTION_FAILED 大寫),SQL 用小寫比對 | 加 `UPPER(status::text)` 保證命中 |
+
+### 驗證
+- Live DB 驗證:修正後 SQL → `success=0, failed=2`(之前永遠 0/0)
+- 明早 08:00 報告應顯示真實成功率(今日 0/2 = 0%,但數字正確)
+
+### Commits
+- `ef1353b` 主修復(leader lock + 改查 approval_records)
+- `88af639` SQL 大小寫修正
+
+---
+
+## 📍 2026-04-22 凌晨 — Telegram 按鈕靜默兩大根因修復(commit 1625e7b)
+
+### 問題
+統帥:「按鈕按下去,會產生的所有操作和結果,也都沒有回覆到 Telegram 群組上!」
+
+### 根因全景(Debugger 全景調查)
+
+| 陷阱 | 症狀 | 根因 | 修法 |
+|------|------|------|------|
+| T1 容量預測按鈕靜默 | "已處理"/"忽略 24h" 按下後群組無回覆 | `_handle_ai_advisory_action` 只呼叫 `_answer_callback`(toast 2-3 秒消失),從未 `sendMessage` 到群組 | 加 `message_id` 參數,toast 後發 `sendMessage reply` 到群組 |
+| T2 已解決告警批准靜默 | 再按「批准」→ 出現「⚡ 執行中...」但永遠沒結果 | `sign_approval` early-return(status != pending),但代碼仍呼叫 `_notify_approval_result` 發「執行中...」;`execute_approved_action` 因 status != APPROVED 跳過 → 永無結果 | 僅 `approval.status == APPROVED` 才發「執行中...」;否則發「ℹ️ 此告警已處理(狀態:...)」 |
+
+### 修復範圍
+- `telegram_gateway.py:_handle_ai_advisory_action` — 加 `message_id` + 群組 reply
+- `telegram_gateway.py:_execute_approval_action` — 非 PENDING 狀態正確通知
+
+### 部署
+- Commit: `1625e7b`(push gitea main)
+- Gitea pipeline build → Harbor push → K8s 滾動更新 → 02:13 完成
+- 新 image: `1625e7bd19017d9287fef55a5660ac125a413626`
+
+---
+
+## 📍 2026-04-21 晚 — 全流程三斷點修復(commit 4fc1f49)
+
+### 根因全景
+| 斷點 | 症狀 | 根因 | 修法 |
+|------|------|------|------|
+| D1 飛輪 SLO 公式 | 執行成功率永遠 0.0% | `execution_count` 欄位不存在於 Playbook 模型 → `total_exec` 恆為 0 | `flywheel_stats_service.py` 改讀 `success_count + failure_count` |
+| D2 幻覺降級風險未清 | NO_ACTION 仍等 TG 批准 | `_validate_deployment_inventory` 降級 NO_ACTION 後 `risk_level` 未重置 → 原 HIGH/CRITICAL 風險觸發 PENDING | `openclaw.py` 加 `result.risk_level = AIRiskLevel.LOW` |
+| D3 NO_ACTION PENDING 積壓 | 非破壞性動作等人工批准 | `webhooks.py` 未依 suggested_action 調整 risk_level → INVESTIGATE/OBSERVE/NO_ACTION 全走 Telegram | 兩處 alert path 加 `_non_destructive_actions` LOW risk 強制 |
+
+### 修復效果
+- 新告警若 LLM 返回 NO_ACTION/INVESTIGATE/OBSERVE → 立即 LOW risk → auto-approve → `approval_execution.py` NO_ACTION handler → EXECUTION_SUCCESS
+- `_validate_deployment_inventory` 幻覺降級後,後續批准路徑完全跳過 Telegram
+- 飛輪 SLO 指標有實際執行後將反映真實數字(有執行才有分母)
+
+### 待執行(Pod 更新後)
+```bash
+# 清理 35+ 筆舊 PENDING(無 tg_msg 且超 2h)
+kubectl -n awoooi-prod exec $POD -- python -c "..." # 見 LOGBOOK 說明
+```
+
+### Commit
+- `4fc1f49` (Gitea pipeline 部署中)
+
+---
+
## 📍 2026-04-21 下午 — BUTTON_DATA_INVALID 根治 + Gitea Code Review 修復
### 問題
@@ -1294,3 +1430,30 @@ CR 修補:
2. 重啟後 → 被封存 yaml_rule playbooks 復活(C2 seeder 修復)
3. AI 自動生成新規則 → 立即出現對應 APPROVED Playbook(C3 接線)
4. Watchdog W-4 → APPROVED 數量為 0 時 TYPE-8M 告警(C4 感知)
+
+---
+
+## 2026-04-24(台北)— Playbook 重複建立/封存迴圈根治
+
+**觸發**:DB 查詢顯示 `HTTP 5xx 錯誤率過高` Playbook 建立 7 次以上,294 筆 deprecated / 25 筆 approved。
+
+### 根因
+
+C1(evolver 加 YAML_RULE guard)+ C2(seeder SQL `AND status != 'deprecated'`)邏輯衝突:
+- C1 已讓 evolver 不再封存 YAML_RULE → deprecated 記錄只剩 C1 上線前的歷史垃圾
+- C2 讓 seeder「看到 deprecated 就重建」→ 歷史垃圾永遠在 DB → 每次重啟建一筆新 Playbook
+- C1 保護新建的 Playbook 不被封存,但 deprecated 歷史記錄不清除 → 無限重建
+
+### 修復
+
+| 檔案 | 修改 |
+|------|------|
+| `playbook_seed_service.py` | 移除 `AND status != 'deprecated'`,同名 yaml_rule 任何 status 都視為已存在 |
+| `playbook_evolver.py` | `_fetch_all_active_playbooks` 改分兩次 status 查詢(DRAFT + APPROVED),不載入 deprecated |
+| `migrations/cleanup_duplicate_deprecated_playbooks.sql` | 每個 name 保留最新一筆 deprecated,刪除其餘(需手動套用一次) |
+
+### 待手動執行
+
+```bash
+psql $DATABASE_URL -f apps/api/migrations/cleanup_duplicate_deprecated_playbooks.sql
+```