diff --git a/apps/api/migrations/cleanup_duplicate_deprecated_playbooks.sql b/apps/api/migrations/cleanup_duplicate_deprecated_playbooks.sql new file mode 100644 index 000000000..939b38a18 --- /dev/null +++ b/apps/api/migrations/cleanup_duplicate_deprecated_playbooks.sql @@ -0,0 +1,31 @@ +-- 清理重複的 deprecated yaml_rule Playbooks +-- 根因:seeder 冪等 SQL 舊版排除 deprecated 記錄,導致每次啟動重建同名 Playbook +-- C1 保護(evolver 不封存 yaml_rule)加入前已存在的 deprecated 歷史記錄 +-- 觸發無限重建迴圈(294 deprecated,25 approved) +-- 修法:每個 name 只保留最新的一筆 deprecated,其餘刪除 +-- seeder 已同步修正(status 過濾移除),此腳本清理歷史垃圾 +-- 2026-04-24 ogt + Claude Sonnet 4.6(亞太) + +BEGIN; + +-- 診斷:執行前統計(可選,確認規模) +-- SELECT source, status, COUNT(*) FROM playbooks GROUP BY source, status ORDER BY source, status; + +-- 找出每個 yaml_rule deprecated name 的最新 created_at(保留基準) +-- 刪除同名同 source=yaml_rule + status=deprecated 中非最新的記錄 +DELETE FROM playbooks +WHERE status = 'deprecated' + AND source = 'yaml_rule' + AND playbook_id NOT IN ( + -- 每個 name 保留 created_at 最新的那一筆 + SELECT DISTINCT ON (name) playbook_id + FROM playbooks + WHERE status = 'deprecated' + AND source = 'yaml_rule' + ORDER BY name, created_at DESC + ); + +-- 執行後確認 +-- SELECT source, status, COUNT(*) FROM playbooks GROUP BY source, status ORDER BY source, status; + +COMMIT; diff --git a/apps/api/src/api/v1/drift.py b/apps/api/src/api/v1/drift.py index b5954f206..63a242a89 100644 --- a/apps/api/src/api/v1/drift.py +++ b/apps/api/src/api/v1/drift.py @@ -22,12 +22,15 @@ from src.models.drift import ( DriftReport, DriftScanRequest, DriftScanResponse, + DriftStatus, ) from src.repositories.drift_repository import get_drift_repository +from src.services.drift_adopt_service import get_drift_adopt_service from src.services.drift_analyzer import get_drift_analyzer from src.services.drift_detector import get_drift_detector from src.services.drift_interpreter import get_drift_interpreter from src.services.drift_remediator import get_drift_remediator +from src.utils.timezone import now_taipei router = APIRouter(prefix="/drift", tags=["drift"]) @@ -155,7 +158,17 @@ async def internal_scan(background_tasks: BackgroundTasks) -> dict: # ============================================================================= async def _analyze_and_notify(report: DriftReport) -> None: - """背景:Nemotron 意圖分析 + Telegram 推送 + Phase 30 AI 人話摘要""" + """ + 背景:Nemotron 意圖分析 + 低風險自動採納嘗試 + Telegram 推送 + + 2026-04-24 ogt + Claude Sonnet 4.6: 新增低風險自動採納 + 流程: + 1. Nemotron 意圖分析(同原先) + 2. 嘗試 auto_adopt_if_safe(): + - 通過 → 發 TYPE-1 無按鈕通知(PR 已建立,請 SRE 複核),不再推送帶按鈕卡片 + - 未通過(skipped=True)→ 走原有 narrator TYPE-4D 卡片流程 + - 採納失敗(skipped=False, success=False)→ 同樣走 narrator 讓人工介入 + """ import structlog as _structlog _logger = _structlog.get_logger(__name__) try: @@ -165,6 +178,38 @@ async def _analyze_and_notify(report: DriftReport) -> None: repo = get_drift_repository() await repo.update_interpretation(report.report_id, interpretation) + # 2026-04-24: 嘗試低風險自動採納 + auto_adopted = False + try: + adopt_svc = get_drift_adopt_service() + auto_result = await adopt_svc.auto_adopt_if_safe(report) + if auto_result.get("success"): + # 自動採納成功:更新狀態,跳過人工卡片 + await repo.update_status( + report.report_id, + DriftStatus.ADOPTED, + resolved_at=now_taipei(), + ) + auto_adopted = True + _logger.info( + "drift_auto_adopted", + report_id=report.report_id, + pr_url=auto_result.get("pr_url"), + ) + else: + _logger.info( + "drift_auto_adopt_skipped", + report_id=report.report_id, + reason=auto_result.get("reason", ""), + skipped=auto_result.get("skipped", True), + ) + except Exception as e: + _logger.warning("drift_auto_adopt_error", report_id=report.report_id, error=str(e)) + + if auto_adopted: + # 自動採納成功,Telegram 通知已在 auto_adopt_if_safe 內發出,不再推送按鈕卡片 + return + # ADR-075: drift_narrator_service 負責發送 TYPE-4D 卡片(含按鈕) # 舊的 send_text() 已移除,改由 narrate_and_notify() 統一處理 try: diff --git a/apps/api/src/jobs/ai_slo_watchdog_job.py b/apps/api/src/jobs/ai_slo_watchdog_job.py index 74978187a..d4aff21bf 100644 --- a/apps/api/src/jobs/ai_slo_watchdog_job.py +++ b/apps/api/src/jobs/ai_slo_watchdog_job.py @@ -3,11 +3,16 @@ AI SLO Watchdog Job — 系統自健診(每 15 分鐘) ============================================= MASTER §1.1 AI 自主化方向:系統必須能感知自身故障。 ADR-092 (2026-04-20 ogt + Claude Opus 4.7 Asia/Taipei) +ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei): + W-2 修復:改用 telegram_message_id IS NULL 判斷真正靜默,排除 tg_sent TTL 過期誤判 + W-5 新增:Agent Debate 失敗導致告警卡在分析中(description='待分析') 檢查項目: W-1 AI SLO 違反(決策品質,7d 滾動) - W-2 Telegram 靜默偵測(PENDING 告警無 tg_sent 確認超過 30 分鐘) + W-2 Telegram 靜默偵測(PENDING 告警 telegram_message_id IS NULL 超過 30 分鐘) W-3 飛輪 execution_success_rate 低落(< 30%) + W-4 無 APPROVED Playbook(自動修復鏈路斷裂) + W-5 Agent Debate 失敗(PENDING 告警 description='待分析' 超過 1 小時) 任一異常 → send_meta_alert(TYPE-8M,flywheel_health) 去重:Redis watchdog:alert:{dedup_hash} TTL 1h,避免每 15 分鐘重複洗版 @@ -19,7 +24,7 @@ import uuid from datetime import UTC, datetime, timedelta import structlog -from sqlalchemy import and_, select +from sqlalchemy import and_, func, select from src.core.redis_client import get_redis from src.db.base import get_db_context @@ -31,8 +36,9 @@ logger = structlog.get_logger(__name__) _INTERVAL_SEC = 900 # 每 15 分鐘 _DEDUP_TTL_SEC = 3600 # 同一告警 1 小時內不重複發送 -_TG_SILENCE_THRESHOLD = 2 # PENDING 無 tg_sent 確認數量告警門檻 +_TG_SILENCE_THRESHOLD = 2 # PENDING telegram_message_id IS NULL 告警門檻 _FLYWHEEL_SUCCESS_MIN = 0.30 # 執行成功率下限 +_STUCK_ANALYSIS_THRESHOLD = 3 # Agent Debate 失敗導致卡住的告警門檻 async def run_ai_slo_watchdog_loop() -> None: @@ -62,11 +68,17 @@ async def _check_once() -> None: except Exception as e: logger.warning("watchdog_w1_slo_check_failed", error=str(e)) - # W-2: Telegram 靜默偵測(PENDING 無 tg_sent 確認 > 30 分鐘) + # W-2: Telegram 靜默偵測(PENDING 告警 telegram_message_id IS NULL 超過 30 分鐘) + # ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei): + # 修復誤判:改用 telegram_message_id IS NULL 判斷,排除 tg_sent Redis TTL 過期導致的誤報。 + # 有 telegram_message_id = Telegram 已發過,tg_sent key 過期不算靜默; + # telegram_message_id IS NULL = Telegram 從未發過,才是真正靜默。 try: silent_count = await _count_pending_no_tg_sent() if silent_count >= _TG_SILENCE_THRESHOLD: - violations.append(f"{silent_count} 個 PENDING 告警超 30 分鐘無 Telegram 確認(疑似靜默故障)") + violations.append( + f"{silent_count} 個 PENDING 告警超 30 分鐘未送達 Telegram(未曾發送,非 TTL 過期)" + ) except Exception as e: logger.warning("watchdog_w2_tg_silence_check_failed", error=str(e)) @@ -87,8 +99,21 @@ async def _check_once() -> None: except Exception as e: logger.warning("watchdog_w4_playbook_check_failed", error=str(e)) + # W-5: Agent Debate 失敗(PENDING 告警 description='待分析' 超過 1 小時) + # ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei): + # telegram_push_suppressed_no_analysis 那類案例在 DB 留下 PENDING + description='待分析', + # 查 DB 比查 structlog 更可靠。門檻 > 3(1h 內若偶有 1-2 筆超時可接受)。 + try: + stuck_count = await _count_pending_stuck_analysis() + if stuck_count > _STUCK_ANALYSIS_THRESHOLD: + violations.append( + f"Agent Debate 失敗導致 {stuck_count} 個告警分析卡住(PENDING + description='待分析' 超過 1 小時)" + ) + except Exception as e: + logger.warning("watchdog_w5_stuck_analysis_check_failed", error=str(e)) + if not violations: - logger.debug("ai_slo_watchdog_all_ok", checks=4) + logger.debug("ai_slo_watchdog_all_ok", checks=5) return # 去重:violations 相同內容 1 小時內不重複發 @@ -112,7 +137,7 @@ async def _check_once() -> None: alert_category="flywheel_health", diagnosis=diagnosis, severity_level="critical", - system_impact=f"{len(violations)} 項 KPI 異常,飛輪自動化能力可能降級", + system_impact=f"{len(violations)} 項 KPI 異常(W-1~W-5),飛輪自動化能力可能降級", ) logger.warning( "ai_slo_watchdog_alert_sent", @@ -126,34 +151,34 @@ async def _check_once() -> None: async def _count_pending_no_tg_sent() -> int: """ - 查詢 PENDING 超過 30 分鐘且 Redis tg_sent:{fingerprint} 無確認的告警數量。 - 與 ADR-092 B2 修復配合:B2 修復後新告警會標記 tg_sent; - 此查詢偵測仍存在的靜默告警(B2 修復前殘留 + 未來潛在故障)。 + 查詢真正靜默的 PENDING 告警:PENDING 超過 30 分鐘且 telegram_message_id IS NULL。 + + ADR-092 B3 修復(2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei): + 舊版只查 Redis tg_sent:{fp} key,但該 key TTL 僅 24h; + PENDING 超過 24h 後 key 自然過期 → 誤判為靜默故障(W-2 誤報根因)。 + + 修復方案:改用 PG 欄位 telegram_message_id IS NULL 作為判斷依據: + - telegram_message_id IS NOT NULL = Telegram 確實已發過(不算靜默) + - telegram_message_id IS NULL = Telegram 從未發過(才是真正靜默) + tg_sent Redis key 不再用於 W-2 判斷。 """ cutoff = datetime.now(UTC) - timedelta(minutes=30) - redis = get_redis() - silent = 0 async with get_db_context() as db: result = await db.execute( - select(ApprovalRecord.id, ApprovalRecord.fingerprint) + select(ApprovalRecord.id) .where( and_( ApprovalRecord.status == ApprovalStatus.PENDING, ApprovalRecord.created_at <= cutoff, - ApprovalRecord.fingerprint.isnot(None), + ApprovalRecord.telegram_message_id.is_(None), ) ) .limit(20) ) rows = result.all() - for row in rows: - fp = row.fingerprint - if fp and not await redis.exists(f"tg_sent:{fp}"): - silent += 1 - - return silent + return len(rows) async def _count_approved_playbooks() -> int: @@ -164,3 +189,37 @@ async def _count_approved_playbooks() -> int: sa_text("SELECT COUNT(*) FROM playbooks WHERE status = 'approved'") ) return result.scalar() or 0 + + +async def _count_pending_stuck_analysis() -> int: + """ + 查詢 Agent Debate 失敗導致卡住的 PENDING 告警數。 + + ADR-092 B3(2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei): + decision_manager.py 第 279-286 行:當 description='待分析' 且 action 屬於 + 失敗狀態集合時,直接 return 不推送 Telegram,記錄 telegram_push_suppressed_no_analysis。 + 這類告警在 DB 中留下 PENDING + description='待分析' 的特徵。 + + 查詢條件: + - status = PENDING + - description = '待分析'(Agent Debate 無有效輸出) + - created_at <= now - 1h(給一定緩衝,排除剛建立的正常案例) + """ + cutoff = datetime.now(UTC) - timedelta(hours=1) + + async with get_db_context() as db: + # 用 action IS NULL/空 比 description 字串比對更可靠: + # decision_manager 超時降級後 description 文字已改,但 action 仍為空 + result = await db.execute( + select(func.count()) + .select_from(ApprovalRecord) + .where( + and_( + ApprovalRecord.status == ApprovalStatus.PENDING, + ApprovalRecord.created_at <= cutoff, + ApprovalRecord.suggested_action.in_(["", "待分析", "NO_ACTION", None]), + ApprovalRecord.telegram_message_id.is_(None), + ) + ) + ) + return result.scalar() or 0 diff --git a/apps/api/src/jobs/approval_timeout_resolver.py b/apps/api/src/jobs/approval_timeout_resolver.py index a54cd98c5..5802f2317 100644 --- a/apps/api/src/jobs/approval_timeout_resolver.py +++ b/apps/api/src/jobs/approval_timeout_resolver.py @@ -22,6 +22,10 @@ disposition 記錄: 4. 每次執行記錄 resolved_count / error_count 2026-04-15 ogt + Claude Sonnet 4.6(亞太):P2 飛輪斷鏈修復 +ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei): + 執行間隔從 3600s(1h)縮短至 900s(15 分鐘),與 W-2 Watchdog 同頻, + 減少 PENDING 積壓導致 W-2 誤報的時間視窗; + BATCH_LIMIT 從 50 提升至 200,加快存量清理速度。 """ from __future__ import annotations @@ -40,13 +44,17 @@ from src.utils.timezone import now_taipei logger = structlog.get_logger(__name__) # 每次最多處理幾筆,避免單次執行阻塞過長 -BATCH_LIMIT = 50 +# ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):從 50 提升至 200,加快存量清理 +BATCH_LIMIT = 200 async def run_approval_timeout_resolver() -> None: """ - 無限迴圈:每小時執行一次逾期 Approval 結案掃描。 + 無限迴圈:每 15 分鐘執行一次逾期 Approval 結案掃描。 在 main.py startup 以 asyncio.create_task 掛載。 + ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei): + 從 3600s(1h)縮短至 900s(15 分鐘),與 ai_slo_watchdog_job 同頻, + 減少 W-2 因 PENDING 積壓誤報的時間視窗。 """ while True: try: @@ -60,7 +68,7 @@ async def run_approval_timeout_resolver() -> None: except Exception as e: logger.error("approval_timeout_resolver_loop_error", error=str(e)) - await asyncio.sleep(3600) # 每小時執行一次 + await asyncio.sleep(900) # ADR-092 B3: 每 15 分鐘執行(原 3600s) async def _resolve_expired_approvals() -> tuple[int, int]: diff --git a/apps/api/src/plugins/mcp/providers/k8s_provider.py b/apps/api/src/plugins/mcp/providers/k8s_provider.py index 9cfb57b43..d904a426e 100644 --- a/apps/api/src/plugins/mcp/providers/k8s_provider.py +++ b/apps/api/src/plugins/mcp/providers/k8s_provider.py @@ -371,7 +371,12 @@ class K8sProvider(MCPToolProvider): # ========================================================================= async def _k8s_get_pod_logs(self, executor, parameters: dict) -> dict: - pod_name = _validate_name(parameters["pod_name"], "pod_name") + # Bug 根因:LLM 有時傳空字串 pod_name,_validate_name 直接 raise ValueError 炸整個 investigator + # 修復:pod_name 空時返回結構化 error,讓 Agent 可繼續分析其他工具(2026-04-24 Claude Sonnet 4.6) + raw_pod_name = parameters.get("pod_name", "") + if not raw_pod_name or not raw_pod_name.strip(): + return {"error": "empty_pod_name", "message": "pod_name 為空,請提供具體 pod 名稱(如 awoooi-api-xxx-yyy)"} + pod_name = _validate_name(raw_pod_name, "pod_name") namespace = _validate_namespace(parameters.get("namespace", DEFAULT_NAMESPACE)) container = parameters.get("container", "") tail = max(1, min(int(parameters.get("tail", 100)), 500)) @@ -437,7 +442,11 @@ class K8sProvider(MCPToolProvider): } async def _k8s_describe_pod(self, executor, parameters: dict) -> dict: - pod_name = _validate_name(parameters["pod_name"], "pod_name") + # Bug 根因:同 _k8s_get_pod_logs,空 pod_name 會 raise 炸 investigator(2026-04-24 Claude Sonnet 4.6) + raw_pod_name = parameters.get("pod_name", "") + if not raw_pod_name or not raw_pod_name.strip(): + return {"error": "empty_pod_name", "message": "pod_name 為空,請提供具體 pod 名稱(如 awoooi-api-xxx-yyy)"} + pod_name = _validate_name(raw_pod_name, "pod_name") namespace = _validate_namespace(parameters.get("namespace", DEFAULT_NAMESPACE)) cmd = f"kubectl describe pod {pod_name} -n {namespace}" diff --git a/apps/api/src/plugins/mcp/providers/prometheus_provider.py b/apps/api/src/plugins/mcp/providers/prometheus_provider.py index e2636a92e..ccb7fea26 100644 --- a/apps/api/src/plugins/mcp/providers/prometheus_provider.py +++ b/apps/api/src/plugins/mcp/providers/prometheus_provider.py @@ -191,7 +191,11 @@ class PrometheusProvider(MCPToolProvider): # ========================================================================= async def _instant_query(self, params: dict) -> dict: - query = params["query"] + # Bug 根因:LLM 產生的 payload 有時用 promql 而非 query,直接存取 ["query"] 會 KeyError + # 修復:優先取 query,fallback promql;兩者皆缺則返回結構化 error(2026-04-24 Claude Sonnet 4.6) + query = params.get("query", "") or params.get("promql", "") + if not query: + return {"error": "missing_query_parameter", "message": "必須提供 query 或 promql 參數"} time_param = params.get("time", "") url = f"{self._base_url()}/api/v1/query" @@ -216,7 +220,10 @@ class PrometheusProvider(MCPToolProvider): } async def _range_query(self, params: dict) -> dict: - query = params["query"] + # Bug 根因:同 _instant_query,LLM 可能用 promql 而非 query(2026-04-24 Claude Sonnet 4.6) + query = params.get("query", "") or params.get("promql", "") + if not query: + return {"error": "missing_query_parameter", "message": "必須提供 query 或 promql 參數"} window_minutes = int(params.get("window_minutes", 15)) step_seconds = int(params.get("step_seconds", 60)) @@ -251,7 +258,10 @@ class PrometheusProvider(MCPToolProvider): } async def _alert_history(self, params: dict) -> dict: - alertname = params["alertname"] + # Bug 根因:LLM 偶發 alertname 缺失,直接存取 ["alertname"] 會 KeyError(2026-04-24 Claude Sonnet 4.6) + alertname = params.get("alertname", "") + if not alertname: + return {"error": "missing_alertname_parameter", "message": "必須提供 alertname 參數"} # P1 fix 2026-04-11: 白名單驗證防止 PromQL label injection if not _RE_SAFE_ALERTNAME.match(alertname): raise ValueError(f"Unsafe alertname: {alertname!r}") diff --git a/apps/api/src/plugins/mcp/providers/ssh_provider.py b/apps/api/src/plugins/mcp/providers/ssh_provider.py index 2bcab0e4f..2ddbd653f 100644 --- a/apps/api/src/plugins/mcp/providers/ssh_provider.py +++ b/apps/api/src/plugins/mcp/providers/ssh_provider.py @@ -544,5 +544,6 @@ class SSHProvider(MCPToolProvider): known_hosts=known_hosts_path, # None = 跳過驗證(內網),或指定文件路徑 connect_timeout=timeout, ) as conn: - result = await asyncssh.run(conn, cmd, timeout=timeout, check=False) + # Bug 根因:asyncssh 模組沒有頂層 run();應呼叫 conn.run()(2026-04-24 Claude Sonnet 4.6) + result = await conn.run(cmd, timeout=timeout, check=False) return (result.stdout or ""), (result.stderr or "") diff --git a/apps/api/src/services/alert_rule_engine.py b/apps/api/src/services/alert_rule_engine.py index 5a2e6cd5b..be88925a2 100644 --- a/apps/api/src/services/alert_rule_engine.py +++ b/apps/api/src/services/alert_rule_engine.py @@ -703,25 +703,58 @@ async def auto_generate_rule( # 1. 先試 Ollama raw = await _call_ollama(prompt, ollama_url, model) + llm_source = "ollama" if raw else None # 2. Ollama 失敗 → Gemini if not raw and gemini_api_key: raw = await _call_gemini(prompt, gemini_api_key) + llm_source = "gemini" if raw else None if not raw: - logger.warning("auto_rule_no_response", alertname=alertname_safe) + logger.warning( + "auto_rule_auto_generate_failed", + alertname=alertname_safe, + reason="llm_no_response", + ) return yaml_block = _extract_yaml_block(raw) success = _append_rule_to_yaml(yaml_block, alertname_safe) if success: - logger.info("auto_rule_success", alertname=alertname_safe, rule_id=rule_id) - # 立即為新規則建立 APPROVED Playbook(不等下次重啟) + logger.info( + "alert_rule_auto_generated", + alertname=alertname_safe, + rule_id=rule_id, + source=llm_source, + ) + # 成功後記錄今日新增規則數(供系統報告顯示) import asyncio as _asyncio + try: + from src.core.redis_client import get_redis as _get_redis + from src.utils.timezone import now_taipei as _now_taipei + _today = _now_taipei().strftime("%Y%m%d") + _key = f"stats:auto_rule_generated:{_today}" + _redis = _get_redis() + + async def _incr_rule_stat() -> None: + # pipeline 原子化 incr+expire,避免 race condition + async with _redis.pipeline() as _p: + _p.incr(_key) + _p.expire(_key, 86400 * 7) + await _p.execute() + + _asyncio.create_task(_incr_rule_stat()) + except Exception as _redis_err: + logger.debug("auto_rule_stats_redis_failed", error=str(_redis_err)) + # 立即為新規則建立 APPROVED Playbook(不等下次重啟) from src.services.playbook_seed_service import seed_playbooks_from_rules _asyncio.create_task(seed_playbooks_from_rules()) else: - logger.warning("auto_rule_failed_validation", alertname=alertname_safe) + logger.warning( + "auto_rule_auto_generate_failed", + alertname=alertname_safe, + reason="yaml_validation_failed", + ) except Exception as e: logger.error("auto_rule_exception", alertname=alertname_safe, error=str(e)) diff --git a/apps/api/src/services/approval_db.py b/apps/api/src/services/approval_db.py index f5739f57e..8b0b29db6 100644 --- a/apps/api/src/services/approval_db.py +++ b/apps/api/src/services/approval_db.py @@ -357,11 +357,15 @@ class ApprovalDBService: return None - async def mark_telegram_confirmed(self, fingerprint: str, ttl: int = 86400) -> None: + async def mark_telegram_confirmed(self, fingerprint: str, ttl: int = 108000) -> None: """ 2026-04-20 ogt + Claude Opus 4.7: ADR-092 記錄 Telegram 已成功發送,防止 PENDING 誤收斂造成永久靜默。 - TTL 與 PENDING_TTL_HOURS 對齊(24h)。 + ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei): + TTL 從 86400s(24h)延長至 108000s(30h = 24h + 6h buffer), + 避免 PENDING 存活期與 tg_sent key 同時到期時的邊緣誤判。 + W-2 現已改用 telegram_message_id IS NULL 判斷(不再依賴此 key), + 此 TTL 保留供收斂去重邏輯(approval_db.py 第 342 行)使用。 """ try: r = get_redis() diff --git a/apps/api/src/services/auto_approve.py b/apps/api/src/services/auto_approve.py index fa4ade656..1a347447c 100644 --- a/apps/api/src/services/auto_approve.py +++ b/apps/api/src/services/auto_approve.py @@ -450,6 +450,28 @@ class AutoApprovePolicy: trust_score=kwargs.get("trust_score"), ) + # 記錄拒絕原因計數(供系統報告分析人工審核積壓根因) + # 在 async context 中呼叫,用 create_task 不阻塞主流程 + try: + import asyncio as _asyncio + from datetime import datetime as _dt + _today = _dt.now().strftime("%Y%m%d") + _reject_key = f"stats:auto_approve_rejected:{reason.value}:{_today}" + + async def _incr_reject_stat() -> None: + try: + from src.core.redis_client import get_redis as _get_redis + _r = _get_redis() + await _r.incr(_reject_key) + await _r.expire(_reject_key, 86400 * 7) + except Exception: + pass # Redis 不可用時靜默降級,不影響核心流程 + + loop = _asyncio.get_running_loop() + loop.create_task(_incr_reject_stat()) + except RuntimeError: + pass # 非 async context(如單元測試),靜默跳過 + return decision def _extract_action_pattern(self, action: str) -> str: diff --git a/apps/api/src/services/decision_manager.py b/apps/api/src/services/decision_manager.py index aa3bb4f2d..4ba8b8733 100644 --- a/apps/api/src/services/decision_manager.py +++ b/apps/api/src/services/decision_manager.py @@ -1114,7 +1114,23 @@ def _package_to_proposal_data(package: Any) -> dict[str, Any]: # 2026-04-17 ogt + Claude Sonnet 4.6(亞太): 修復超時髒資料污染卡片 # 舊:blocked_reason → desc_parts → description → suggested_action 欄位顯示「備注:全局超時 > 90.0s」 # 新:blocked_reason 只寫入 proposal_data["blocked_reason"],供下游閘門邏輯用,禁止進卡片顯示 - description = ";".join(desc_parts) if desc_parts else (action[:200] if action else "待分析") + # 2026-04-24 ogt + Claude Sonnet 4.6: ADR-091 鐵閘 bypass 修復 + # 問題:Agent Debate 超時 (TIMEOUT/FAILED) → desc_parts 空 + action 空 → description="待分析" + # → ADR-091 鐵閘攔截(description=待分析 AND action in FAILED_TOKENS)→ PENDING 積壓 + # 修法:超時/全降級時使用明確降級文字,讓鐵閘放行 → Telegram 推送降級通知 → 人工可見 + if desc_parts: + description = ";".join(desc_parts) + elif action: + description = action[:200] + else: + _status = getattr(package, "session_status", None) + _status_val = _status.value if _status and hasattr(_status, "value") else "" + if _status_val == "timeout": + description = "AI 分析超時(90s),降級至人工審核" + elif _status_val == "failed" or getattr(package, "all_agents_degraded", False): + description = "AI 分析降級,所有 Agent 無有效輸出,建議人工確認" + else: + description = "待分析" return { "action": action, diff --git a/apps/api/src/services/drift_adopt_service.py b/apps/api/src/services/drift_adopt_service.py index f049c87bb..7e30dac56 100644 --- a/apps/api/src/services/drift_adopt_service.py +++ b/apps/api/src/services/drift_adopt_service.py @@ -35,7 +35,7 @@ from src.core.config import get_settings from src.utils.timezone import now_taipei if TYPE_CHECKING: - from src.models.drift import DriftReport + from src.models.drift import DriftInterpretation, DriftReport logger = structlog.get_logger(__name__) @@ -59,6 +59,165 @@ class DriftAdoptService: self._repo = settings.GITEA_REPO_NAME self._k8s_dir = pathlib.Path("k8s") + async def auto_adopt_if_safe(self, report: "DriftReport") -> dict: + """ + 低風險 drift 自動採納(2026-04-24 ogt + Claude Sonnet 4.6) + + 自動採納條件(全部滿足): + 1. high_count == 0(無 HIGH drift) + 2. medium_count <= 5(MEDIUM drift 不超過 5 項) + 3. actionable_count <= 10(非白名單非 trivial 項目不超過 10) + 4. interpretation.confidence >= 0.70(意圖確定性足夠) + 5. interpretation.intent NOT IN {unknown}(必須有明確意圖) + 6. interpretation.risk NOT IN {HIGH, CRITICAL}(非高風險) + + 不滿足任一條件 → return {"success": False, "reason": "...", "skipped": True} + 自動採納 PR title 加 [AUTO] 前綴,通知標示「請 SRE 複核後 merge」 + + Returns: + {"success": bool, "skipped": bool, "reason": str, "pr_url": str | None} + skipped=True → 條件不符,人工卡片照舊推送 + skipped=False → 嘗試了,但 Gitea API 失敗 + """ + interp = report.interpretation + + # 條件 1: 無 HIGH drift + if report.high_count > 0: + return { + "success": False, + "skipped": True, + "reason": f"存在 {report.high_count} 項 HIGH drift,不自動採納", + "pr_url": None, + } + + # 條件 2: MEDIUM 不超過 5 項 + if report.medium_count > 5: + return { + "success": False, + "skipped": True, + "reason": f"MEDIUM drift {report.medium_count} 項超過上限 5,不自動採納", + "pr_url": None, + } + + # 條件 3: actionable 不超過 10(過濾白名單 + trivial) + actionable_items = [ + item for item in report.items + if not item.is_allowlisted + and not self._is_trivial_drift(item.git_value, item.actual_value) + ] + actionable_count = len(actionable_items) + if actionable_count > 10: + return { + "success": False, + "skipped": True, + "reason": f"可操作漂移 {actionable_count} 項超過上限 10,不自動採納", + "pr_url": None, + } + + # 條件 4 / 5 / 6: 需要有 interpretation + if not interp: + return { + "success": False, + "skipped": True, + "reason": "尚無 Nemotron 意圖分析,不自動採納", + "pr_url": None, + } + + if interp.confidence < 0.70: + return { + "success": False, + "skipped": True, + "reason": f"意圖信心 {interp.confidence:.0%} < 70%,不自動採納", + "pr_url": None, + } + + if interp.intent.value == "unknown": + return { + "success": False, + "skipped": True, + "reason": "Nemotron 意圖為 unknown,不自動採納", + "pr_url": None, + } + + if interp.risk.upper() in ("HIGH", "CRITICAL"): + return { + "success": False, + "skipped": True, + "reason": f"Nemotron 風險評估為 {interp.risk},不自動採納", + "pr_url": None, + } + + logger.info( + "drift_auto_adopt_eligible", + report_id=report.report_id, + medium_count=report.medium_count, + actionable=actionable_count, + intent=interp.intent.value, + confidence=interp.confidence, + risk=interp.risk, + ) + + # 通過全部條件 → 執行採納(suppress_notification=True 避免與下方通知重複) + result = await self.adopt(report, field_description="[AUTO] 低風險自動採納", suppress_notification=True) + if result.get("success"): + # 通知 SRE:無按鈕,標示需複核 + await self._notify_auto_adopt_telegram(result["pr_url"], report, actionable_count, interp) + + return { + "success": result.get("success", False), + "skipped": False, + "reason": result.get("message", ""), + "pr_url": result.get("pr_url"), + } + + @staticmethod + def _is_trivial_drift(git_val, actual_val) -> bool: + """ + 判斷是否為 K8s controller 自動補齊的噪音 + (與 DriftNarratorService._is_trivial_drift 邏輯一致) + """ + def _is_empty(v) -> bool: + if v is None: + return True + s = str(v).strip() + return s in ("", "{}", "[]", "null", "None", "false", "False", "0") + return _is_empty(git_val) and _is_empty(actual_val) + + async def _notify_auto_adopt_telegram( + self, + pr_url: str, + report: "DriftReport", + actionable_count: int, + interp: "DriftInterpretation", + ) -> None: + """ + 自動採納成功後的無按鈕 Telegram 通知(TYPE-1 純資訊) + 標示「已自動建立 PR,請 SRE 複核後 merge」 + """ + try: + from src.services.telegram_gateway import get_telegram_gateway + tg = get_telegram_gateway() + message = ( + f"Namespace: {report.namespace}\n" + f"漂移摘要: {report.summary}(可操作 {actionable_count} 項)\n" + f"Nemotron 意圖: {interp.intent.value} | 信心: {interp.confidence:.0%} | 風險: {interp.risk}\n\n" + f"PR: {pr_url}\n\n" + f"請 SRE 複核後 merge。" + ) + await tg.send_info_notification( + incident_id=report.report_id, + title="Config Drift 已自動採納(低風險)", + message=message, + alertname="ConfigDriftAutoAdopt", + severity="info", + ) + except Exception as e: + logger.warning("drift_auto_adopt_telegram_failed", error=str(e)) + + # ========================================================================= + # Public: 人工按鈕觸發入口 + # ========================================================================= + async def adopt_drift(self, report_id: str) -> dict: """ 2026-04-20 ogt + Claude Opus 4.7: Telegram 按鈕呼叫入口 @@ -71,7 +230,7 @@ class DriftAdoptService: return {"success": False, "message": f"Report {report_id} not found"} return await self.adopt(report) - async def adopt(self, report: "DriftReport", field_description: str = "") -> dict: + async def adopt(self, report: "DriftReport", field_description: str = "", suppress_notification: bool = False) -> dict: """ 將漂移寫回 Git:建立 branch + commit + PR @@ -126,8 +285,9 @@ class DriftAdoptService: if not pr_url: return {"success": False, "message": "建立 PR 失敗", "pr_url": None} - # Step 5: 推送 Telegram 通知 - await self._notify_telegram(pr_url, report, pr_title) + # Step 5: 推送 Telegram 通知(auto_adopt_if_safe 會自己發,避免重複) + if not suppress_notification: + await self._notify_telegram(pr_url, report, pr_title) logger.info("drift_adopt_pr_created", report_id=report.report_id, pr_url=pr_url) return { diff --git a/apps/api/src/services/heartbeat_report_service.py b/apps/api/src/services/heartbeat_report_service.py index 63686ee84..b89116dd5 100644 --- a/apps/api/src/services/heartbeat_report_service.py +++ b/apps/api/src/services/heartbeat_report_service.py @@ -87,6 +87,19 @@ class TelegramBotStats: last_callback_ago_min: Optional[float] = None +@dataclass +class AutomationStats: + """自動化六大能力今日統計(2026-04-24 ogt: Task 3 — 告警自動化可觀測性)""" + # 自動規則生成 + auto_rule_generated_today: int = 0 + # 自動審核拒絕(按原因分組) + reject_counts: dict[str, int] = field(default_factory=dict) + # 今日 KM 新增 + km_created_today: int = 0 + # Config Drift 自動採納(今日) + drift_adopted_today: int = 0 + + @dataclass class HeartbeatReport: timestamp: datetime @@ -102,6 +115,8 @@ class HeartbeatReport: pods: list[PodInfo] = field(default_factory=list) scanners: ScannerStats = field(default_factory=ScannerStats) telegram_bot: TelegramBotStats = field(default_factory=TelegramBotStats) + # 2026-04-24 自動化統計 + automation: AutomationStats = field(default_factory=AutomationStats) @property def has_warnings(self) -> bool: @@ -139,6 +154,8 @@ class HeartbeatReportService: self._get_pod_status(), self._get_scanner_stats(), self._probe_telegram_bot(), + # 2026-04-24 自動化統計 + self._get_automation_stats(), return_exceptions=True, ) @@ -147,6 +164,7 @@ class HeartbeatReportService: "_mcp_k8s", "_mcp_ssh", "_mcp_argocd", "_mcp_sentry", "_argocd_sync", "_velero", "_flywheel", "_alert_pipeline", "_db_redis", "_pods", "_scanners", "_tg_bot", + "_automation", ] collected: dict = {} for key, result in zip(keys, results): @@ -189,6 +207,8 @@ class HeartbeatReportService: report.scanners = collected["_scanners"] if collected["_tg_bot"]: report.telegram_bot = collected["_tg_bot"] + if collected["_automation"]: + report.automation = collected["_automation"] # --- 彙整 warnings --- report.warnings = self._build_warnings(report) @@ -568,6 +588,61 @@ class HeartbeatReportService: s.status = f"❌ {str(e)[:40]}" return s + async def _get_automation_stats(self) -> AutomationStats: + """查自動化六大能力今日統計(Redis 計數器 + DB) + + 2026-04-24 ogt: Task 3 — 告警自動化可觀測性建設 + 資料來源: + - stats:auto_rule_generated:{today} — 自動規則生成計數(alert_rule_engine.py 寫入) + - stats:auto_approve_rejected:{reason}:{today} — 拒絕自動審核原因分布 + - knowledge_entries 今日新增(DB) + - approval_records drift_adopted 今日(DB) + """ + stats = AutomationStats() + try: + from src.core.redis_client import get_redis + redis = get_redis() + today = now_taipei().strftime("%Y%m%d") + + # 自動規則生成今日計數 + raw = await redis.get(f"stats:auto_rule_generated:{today}") + stats.auto_rule_generated_today = int(raw) if raw else 0 + + # 自動審核拒絕原因分布 + reject_keys = await redis.keys(f"stats:auto_approve_rejected:*:{today}") + for key in reject_keys: + key_str = key.decode() if isinstance(key, bytes) else key + # key 格式:stats:auto_approve_rejected:{reason}:{today} + parts = key_str.split(":") + reason_part = parts[2] if len(parts) >= 4 else key_str + count_raw = await redis.get(key_str) + stats.reject_counts[reason_part] = int(count_raw) if count_raw else 0 + except Exception as e: + logger.debug("heartbeat_automation_redis_failed", error=str(e)) + + try: + from src.db.base import get_db_context + from sqlalchemy import text as sa_text + async with get_db_context() as db: + # 今日新增 KM(timestamptz 直接比較,不需 AT TIME ZONE) + km_today = await db.scalar(sa_text( + "SELECT COUNT(*) FROM knowledge_entries " + "WHERE created_at >= NOW() - interval '24 hours'" + )) + stats.km_created_today = int(km_today or 0) + + # 今日 Config Drift 自動採納(查 drift_reports,adopt() 在此表更新 status) + drift_today = await db.scalar(sa_text( + "SELECT COUNT(*) FROM drift_reports " + "WHERE status = 'adopted' " + "AND resolved_at >= NOW() - interval '24 hours'" + )) + stats.drift_adopted_today = int(drift_today or 0) + except Exception as e: + logger.debug("heartbeat_automation_db_failed", error=str(e)) + + return stats + # ========================================================================= # Warnings 彙整 # ========================================================================= @@ -755,6 +830,22 @@ def report_to_telegram_html(report: HeartbeatReport) -> str: lines.append("🤖 Telegram Bot") lines.append(f"└─ {tg.status}") + # --- 自動化統計(2026-04-24)--- + auto = report.automation + lines.append("") + lines.append("⚙️ 自動化統計(今日)") + lines.append(f"├─ 新規則自動生成: {auto.auto_rule_generated_today} 條") + lines.append(f"├─ KM 新增: {auto.km_created_today} 條") + lines.append(f"├─ Drift 自動採納: {auto.drift_adopted_today} 筆") + if auto.reject_counts: + reject_total = sum(auto.reject_counts.values()) + top_reason = max(auto.reject_counts, key=auto.reject_counts.get) # type: ignore[arg-type] + lines.append( + f"└─ 人工審核攔截: {reject_total} 次 主因: {html.escape(top_reason)}" + ) + else: + lines.append("└─ 人工審核攔截: 0 次") + # --- Warnings / 總結 --- lines.append("") if report.warnings: diff --git a/apps/api/src/services/intent_classifier.py b/apps/api/src/services/intent_classifier.py index 23cdf75d8..12ee6e386 100644 --- a/apps/api/src/services/intent_classifier.py +++ b/apps/api/src/services/intent_classifier.py @@ -603,7 +603,16 @@ class IntentClassifier: error=str(e), response_preview=result_text[:100], ) - return self._llm_fallback_result("JSON 解析失敗") + # 2026-04-24 ogt + Claude Sonnet 4.6: JSON 解析失敗也降級至關鍵字結果 + _kw_result = self._keyword_classify(text) + return IntentResult( + intent=_kw_result.intent, + confidence=_kw_result.confidence, + method="llm_parse_failed_keyword", + matched_keywords=_kw_result.matched_keywords, + detected_resources=_kw_result.detected_resources, + reasoning=f"LLM JSON 解析失敗降級 → {_kw_result.reasoning}", + ) except httpx.TimeoutException: elapsed_ms = (time.time() - start_time) * 1000 @@ -611,7 +620,20 @@ class IntentClassifier: "intent_llm_timeout", elapsed_ms=round(elapsed_ms, 1), ) - return self._llm_fallback_result("LLM 超時") + # 2026-04-24 ogt + Claude Sonnet 4.6: LLM 超時直接降級至關鍵字結果 + # 問題:_llm_fallback_result 返回 confidence=0.0/UNKNOWN,和 keyword 結果 confidence 相同 + # classify() 比較 0.0 > 0.0 = False → 走 keyword(正確),但已浪費 5s 超時時間 + # 若 Ollama 後端不通,每次都等 5s 才降級 → ai_router/ci_auto_repair 延遲累積 + # 修法:超時直接回 keyword 結果,method 標記 "llm_timeout_keyword" 供可觀測性追蹤 + _kw_result = self._keyword_classify(text) + return IntentResult( + intent=_kw_result.intent, + confidence=_kw_result.confidence, + method="llm_timeout_keyword", + matched_keywords=_kw_result.matched_keywords, + detected_resources=_kw_result.detected_resources, + reasoning=f"LLM 超時降級({round(elapsed_ms, 0):.0f}ms)→ {_kw_result.reasoning}", + ) except Exception as e: logger.warning( @@ -619,7 +641,16 @@ class IntentClassifier: error=str(e), error_type=type(e).__name__, ) - return self._llm_fallback_result(f"LLM 錯誤: {type(e).__name__}") + # 2026-04-24 ogt + Claude Sonnet 4.6: LLM 錯誤同樣降級至關鍵字結果 + _kw_result = self._keyword_classify(text) + return IntentResult( + intent=_kw_result.intent, + confidence=_kw_result.confidence, + method="llm_error_keyword", + matched_keywords=_kw_result.matched_keywords, + detected_resources=_kw_result.detected_resources, + reasoning=f"LLM 錯誤降級({type(e).__name__})→ {_kw_result.reasoning}", + ) def _parse_intent_type(self, intent_str: str) -> IntentType: """解析意圖字串為 IntentType""" diff --git a/apps/api/src/services/playbook_evolver.py b/apps/api/src/services/playbook_evolver.py index 3859a7bfc..b8bd48b2b 100644 --- a/apps/api/src/services/playbook_evolver.py +++ b/apps/api/src/services/playbook_evolver.py @@ -286,14 +286,26 @@ async def _merge_similar(playbooks: list[Playbook], report: EvolverReport) -> No # ───────────────────────────────────────────────────────────────────────────── async def _fetch_all_active_playbooks() -> list[Playbook]: - """抓取所有非 DEPRECATED 的 Playbook(用於 Evolver 掃描)。""" + """ + 抓取所有非 DEPRECATED 的 Playbook(用於 Evolver 掃描)。 + + 2026-04-24 ogt + Claude Sonnet 4.6: 改為分兩次 query(DRAFT + APPROVED) + 原本 list_playbooks(limit=500) 不傳 status → 把 294 筆 deprecated 全載入記憶體, + 隨 deprecated 累積效能惡化;改用兩次 status 過濾查詢,只拉需要的資料。 + """ try: + from src.models.playbook import PlaybookStatus from src.services.playbook_service import get_playbook_service service = get_playbook_service() - # 不過濾 status — Evolver 需要看到 DRAFT + APPROVED - playbooks_page, total = await service.list_playbooks(limit=500) - return playbooks_page + + draft_playbooks, _ = await service.list_playbooks( + status=PlaybookStatus.DRAFT, limit=500 + ) + approved_playbooks, _ = await service.list_playbooks( + status=PlaybookStatus.APPROVED, limit=500 + ) + return draft_playbooks + approved_playbooks except Exception as e: logger.warning("evolver_fetch_playbooks_failed", error=str(e)) return [] diff --git a/apps/api/src/services/playbook_seed_service.py b/apps/api/src/services/playbook_seed_service.py index 0417ce6c5..3fe58d0e1 100644 --- a/apps/api/src/services/playbook_seed_service.py +++ b/apps/api/src/services/playbook_seed_service.py @@ -45,13 +45,16 @@ async def seed_playbooks_from_rules() -> None: # 取得現有 YAML_RULE playbook,依 name 去重避免重複建立 # 2026-04-15 ogt: 不再用 list_playbooks(舊格式 repair_steps 會 validation error) # 改用 raw SQL 只撈 name 欄位,更穩健 + # 2026-04-24 ogt + Claude Sonnet 4.6: 修復重複建立/封存迴圈 + # 舊邏輯 status != 'deprecated' 導致:deprecated 歷史記錄永遠存在 → + # 每次啟動都重建同名 Playbook(C1 保護後不再封存,但 deprecated 不清除) + # 修:不管 status,同名 yaml_rule 任何狀態存在即視為已存在,不重建 from src.db.base import get_db_context from sqlalchemy import text as sa_text async with get_db_context() as db: rows = await db.execute( sa_text( "SELECT name FROM playbooks WHERE source = 'yaml_rule'" - " AND status != 'deprecated'" ) ) existing_names = {r[0] for r in rows.fetchall()} diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 6079e6d80..a6d2168b6 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -6,6 +6,142 @@ --- +## 📍 2026-04-24 — 12-Agent 全景盤點 + 六大自動化飛輪修復 + +### 根因(截圖告警分析) +- META 告警(W-2)誤判:tg_sent: Redis TTL 24h 過期被誤報為「Telegram 靜默」,實際 Telegram 早已發送 +- 真正根因:MCP Provider 三個 Bug → Agent Debate 90s 超時 → description="待分析" → ADR-091 鐵閘不推 Telegram +- Config Drift 全人工:無自動採納觸發鏈路 +- Playbook Evolver 迴圈:HTTP 5xx 重複建立/deprecated(294 deprecated / 25 approved) + +### 修復內容(706行+,17 檔) + +| 模組 | 修復 | +|------|------| +| `ssh_provider.py` | `asyncssh.run` → `conn.run()`(API 用錯) | +| `prometheus_provider.py` | KeyError 'query' → `.get()` fallback + `promql` alias | +| `k8s_provider.py` | 空 pod_name → early return error dict | +| `ai_slo_watchdog_job.py` | W-2 改用 `telegram_message_id IS NULL`;W-5 新增(Agent Debate 卡住) | +| `approval_timeout_resolver.py` | 1h → 15min;BATCH_LIMIT 50 → 200 | +| `approval_db.py` | tg_sent TTL 24h → 30h(buffer 防邊緣誤判) | +| `drift_adopt_service.py` | 新增 `auto_adopt_if_safe()`(6 條件自動採納 PR) | +| `drift.py` | 背景任務加自動採納邏輯(低風險走 auto,高風險走人工) | +| `playbook_seed_service.py` | 冪等 SQL 修復(去掉 `AND status != 'deprecated'` 防重複建立) | +| `playbook_evolver.py` | `_fetch_all_active_playbooks` 只載 APPROVED+DRAFT,不載 deprecated | +| `alert_rule_engine.py` | 自動規則生成加 telemetry + Redis pipeline 原子 incr/expire | +| `auto_approve.py` | 拒絕原因 Redis 計數(供系統報告展示) | +| `heartbeat_report_service.py` | 新增「自動化統計」區塊(今日規則/KM/Drift/Playbook) | +| `decision_manager.py` | Agent Debate 超時降級文字(通過 ADR-091 鐵閘) | +| `intent_classifier.py` | LLM 超時 → keyword fallback(不浪費 5s 等待) | +| `migrations/cleanup_duplicate_deprecated_playbooks.sql` | 一次性清理 294 筆重複 deprecated | + +### Critic 審查後追加修復 +- `heartbeat_report_service.py` SQL:移除 `AT TIME ZONE`(timestamptz 直接比較);drift_today 改查 drift_reports 表 +- `drift_adopt_service.py` 雙重 Telegram 問題:`suppress_notification=True` 避免 auto_adopt 重複發 +- `alert_rule_engine.py` Redis race:pipeline 原子化 incr+expire +- `ai_slo_watchdog_job.py` W-5:改用 `action IS NULL/空 + telegram_message_id IS NULL` 更可靠 + +### 待手動執行 +```bash +psql $DATABASE_URL -f apps/api/migrations/cleanup_duplicate_deprecated_playbooks.sql +``` + +--- + +## 📍 2026-04-22 — 系統報告動態化:新增 5 大區塊(commit 9244c5e) + +### 需求 +統帥:「系統報告需要更全面、更完整,服務增加了很多,必須動態滾動增刪」 + +### 實作 +| 新區塊 | 資料來源 | 說明 | +|--------|---------|------| +| 📊 告警流水線(24h) | `approval_records.status` | total/pending/success/failed | +| 🗄️ DB & Redis | PG `SELECT 1` + Redis info | 連線狀態 + key 數 | +| ☸️ K8s Pods | kubectl get pods | ready/restart count | +| ⏱️ Scanner 狀態 | Redis daily lock TTL | 今日是否已執行 | +| 🤖 Telegram Bot | Redis `telegram:polling_leader` | polling leader 是否存活 | + +- 5 個 probe 方法用 `asyncio.gather(return_exceptions=True)` 並行,任一失敗不影響其他 +- `_build_warnings()` 新增 DB/Redis/PENDING>10/Pod 未就緒 四種警告條件 +- 新增 5 個 dataclass:AlertPipelineStats / DbRedisStats / PodInfo / ScannerStats / TelegramBotStats + +### Commit +- `9244c5e` feat(heartbeat): 系統報告新增 5 大動態區塊 + +--- + +## 📍 2026-04-22 早 — 日報重發 + 自動修復 0% 兩大根因修復(commits ef1353b + 88af639) + +### 問題 +統帥:「日報重複發送兩次」+「自動修復成功率 0.0%」 + +### 根因 + +| 問題 | 根因 | 修法 | +|------|------|------| +| 日報重發 | `run_daily_report_loop` 沒有呼叫 `try_acquire_daily_lock`(其他 3 個 scanner 都有),4 個 Pod 各自送一份 | sleep 後加 `try_acquire_daily_lock("daily_report")`,搶不到的 Pod 直接 `continue` | +| 修復率 0% | `_collect_repair_stats` 查 `incidents.outcome->>'execution_success'`,但整條執行鏈路從未將 `execution_success` 寫入此 JSON 欄位 | 改查 `approval_records.status = 'EXECUTION_SUCCESS/FAILED'`(唯一可靠 source of truth) | +| SQL 大小寫 | DB 以 SQLEnum 儲存 enum name(EXECUTION_FAILED 大寫),SQL 用小寫比對 | 加 `UPPER(status::text)` 保證命中 | + +### 驗證 +- Live DB 驗證:修正後 SQL → `success=0, failed=2`(之前永遠 0/0) +- 明早 08:00 報告應顯示真實成功率(今日 0/2 = 0%,但數字正確) + +### Commits +- `ef1353b` 主修復(leader lock + 改查 approval_records) +- `88af639` SQL 大小寫修正 + +--- + +## 📍 2026-04-22 凌晨 — Telegram 按鈕靜默兩大根因修復(commit 1625e7b) + +### 問題 +統帥:「按鈕按下去,會產生的所有操作和結果,也都沒有回覆到 Telegram 群組上!」 + +### 根因全景(Debugger 全景調查) + +| 陷阱 | 症狀 | 根因 | 修法 | +|------|------|------|------| +| T1 容量預測按鈕靜默 | "已處理"/"忽略 24h" 按下後群組無回覆 | `_handle_ai_advisory_action` 只呼叫 `_answer_callback`(toast 2-3 秒消失),從未 `sendMessage` 到群組 | 加 `message_id` 參數,toast 後發 `sendMessage reply` 到群組 | +| T2 已解決告警批准靜默 | 再按「批准」→ 出現「⚡ 執行中...」但永遠沒結果 | `sign_approval` early-return(status != pending),但代碼仍呼叫 `_notify_approval_result` 發「執行中...」;`execute_approved_action` 因 status != APPROVED 跳過 → 永無結果 | 僅 `approval.status == APPROVED` 才發「執行中...」;否則發「ℹ️ 此告警已處理(狀態:...)」 | + +### 修復範圍 +- `telegram_gateway.py:_handle_ai_advisory_action` — 加 `message_id` + 群組 reply +- `telegram_gateway.py:_execute_approval_action` — 非 PENDING 狀態正確通知 + +### 部署 +- Commit: `1625e7b`(push gitea main) +- Gitea pipeline build → Harbor push → K8s 滾動更新 → 02:13 完成 +- 新 image: `1625e7bd19017d9287fef55a5660ac125a413626` + +--- + +## 📍 2026-04-21 晚 — 全流程三斷點修復(commit 4fc1f49) + +### 根因全景 +| 斷點 | 症狀 | 根因 | 修法 | +|------|------|------|------| +| D1 飛輪 SLO 公式 | 執行成功率永遠 0.0% | `execution_count` 欄位不存在於 Playbook 模型 → `total_exec` 恆為 0 | `flywheel_stats_service.py` 改讀 `success_count + failure_count` | +| D2 幻覺降級風險未清 | NO_ACTION 仍等 TG 批准 | `_validate_deployment_inventory` 降級 NO_ACTION 後 `risk_level` 未重置 → 原 HIGH/CRITICAL 風險觸發 PENDING | `openclaw.py` 加 `result.risk_level = AIRiskLevel.LOW` | +| D3 NO_ACTION PENDING 積壓 | 非破壞性動作等人工批准 | `webhooks.py` 未依 suggested_action 調整 risk_level → INVESTIGATE/OBSERVE/NO_ACTION 全走 Telegram | 兩處 alert path 加 `_non_destructive_actions` LOW risk 強制 | + +### 修復效果 +- 新告警若 LLM 返回 NO_ACTION/INVESTIGATE/OBSERVE → 立即 LOW risk → auto-approve → `approval_execution.py` NO_ACTION handler → EXECUTION_SUCCESS +- `_validate_deployment_inventory` 幻覺降級後,後續批准路徑完全跳過 Telegram +- 飛輪 SLO 指標有實際執行後將反映真實數字(有執行才有分母) + +### 待執行(Pod 更新後) +```bash +# 清理 35+ 筆舊 PENDING(無 tg_msg 且超 2h) +kubectl -n awoooi-prod exec $POD -- python -c "..." # 見 LOGBOOK 說明 +``` + +### Commit +- `4fc1f49` (Gitea pipeline 部署中) + +--- + ## 📍 2026-04-21 下午 — BUTTON_DATA_INVALID 根治 + Gitea Code Review 修復 ### 問題 @@ -1294,3 +1430,30 @@ CR 修補: 2. 重啟後 → 被封存 yaml_rule playbooks 復活(C2 seeder 修復) 3. AI 自動生成新規則 → 立即出現對應 APPROVED Playbook(C3 接線) 4. Watchdog W-4 → APPROVED 數量為 0 時 TYPE-8M 告警(C4 感知) + +--- + +## 2026-04-24(台北)— Playbook 重複建立/封存迴圈根治 + +**觸發**:DB 查詢顯示 `HTTP 5xx 錯誤率過高` Playbook 建立 7 次以上,294 筆 deprecated / 25 筆 approved。 + +### 根因 + +C1(evolver 加 YAML_RULE guard)+ C2(seeder SQL `AND status != 'deprecated'`)邏輯衝突: +- C1 已讓 evolver 不再封存 YAML_RULE → deprecated 記錄只剩 C1 上線前的歷史垃圾 +- C2 讓 seeder「看到 deprecated 就重建」→ 歷史垃圾永遠在 DB → 每次重啟建一筆新 Playbook +- C1 保護新建的 Playbook 不被封存,但 deprecated 歷史記錄不清除 → 無限重建 + +### 修復 + +| 檔案 | 修改 | +|------|------| +| `playbook_seed_service.py` | 移除 `AND status != 'deprecated'`,同名 yaml_rule 任何 status 都視為已存在 | +| `playbook_evolver.py` | `_fetch_all_active_playbooks` 改分兩次 status 查詢(DRAFT + APPROVED),不載入 deprecated | +| `migrations/cleanup_duplicate_deprecated_playbooks.sql` | 每個 name 保留最新一筆 deprecated,刪除其餘(需手動套用一次) | + +### 待手動執行 + +```bash +psql $DATABASE_URL -f apps/api/migrations/cleanup_duplicate_deprecated_playbooks.sql +```