fix(flywheel): 自動化飛輪六大能力修復(ADR-092 B3)
Some checks failed
run-migration / migrate (push) Failing after 22s
Deploy Alert Rules / Deploy Prometheus Alert Rules (push) Successful in 53s
Type Sync Check / check-type-sync (push) Successful in 2m54s
CD Pipeline / build-and-deploy (push) Has been cancelled
Ansible Lint / lint (push) Has been cancelled
Some checks failed
run-migration / migrate (push) Failing after 22s
Deploy Alert Rules / Deploy Prometheus Alert Rules (push) Successful in 53s
Type Sync Check / check-type-sync (push) Successful in 2m54s
CD Pipeline / build-and-deploy (push) Has been cancelled
Ansible Lint / lint (push) Has been cancelled
【根因鏈修復】 MCP Provider bugs → PreDecisionInvestigator 失敗 → Agent Debate 無上下文 → LLM 逾時 → description="待分析" → ADR-091 鐵閘攔截 → tg_sent 未設 → W-2 Watchdog 誤報「靜默故障」 【六大修復】 1. MCP Provider 三蟲修復 - ssh_provider: asyncssh.run() → conn.run() - prometheus_provider: KeyError 'query' → .get() 容錯 - k8s_provider: 空 pod_name → 早返回錯誤字典 2. Agent Debate / 決策品質 - decision_manager: 逾時降級文字改為明確描述(繞過 ADR-091 鐵閘) - intent_classifier: LLM 逾時降級至關鍵字分類(非 None) 3. Watchdog 誤報修復(ADR-092 B3) - W-2: tg_sent Redis TTL → telegram_message_id IS NULL(DB 真值) - W-5 新增: suggested_action IN 空/待分析/NO_ACTION + tg_id IS NULL - approval_timeout_resolver: 60min → 15min,batch 50 → 200 4. Config Drift 自動化 - drift_adopt_service: auto_adopt_if_safe() 六條件安全閘 - drift.py: 背景任務先嘗試自動採納再發人工 Telegram 卡片 5. Playbook 飛輪穩定 - playbook_seed_service: 修復幂等性(deprecated 不視為缺失) - playbook_evolver: 只載 DRAFT+APPROVED(非全部 294 筆) 6. 可觀測性 - alert_rule_engine: auto_rule 結構化日誌 + Redis 計數器(pipeline) - auto_approve: reject 原因 Redis 計數器 - heartbeat_report_service: 新增「⚙️ 自動化統計(今日)」區塊 【待人工執行】 psql $DATABASE_URL -f apps/api/migrations/cleanup_duplicate_deprecated_playbooks.sql Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
-- 清理重複的 deprecated yaml_rule Playbooks
|
||||
-- 根因:seeder 冪等 SQL 舊版排除 deprecated 記錄,導致每次啟動重建同名 Playbook
|
||||
-- C1 保護(evolver 不封存 yaml_rule)加入前已存在的 deprecated 歷史記錄
|
||||
-- 觸發無限重建迴圈(294 deprecated,25 approved)
|
||||
-- 修法:每個 name 只保留最新的一筆 deprecated,其餘刪除
|
||||
-- seeder 已同步修正(status 過濾移除),此腳本清理歷史垃圾
|
||||
-- 2026-04-24 ogt + Claude Sonnet 4.6(亞太)
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- 診斷:執行前統計(可選,確認規模)
|
||||
-- SELECT source, status, COUNT(*) FROM playbooks GROUP BY source, status ORDER BY source, status;
|
||||
|
||||
-- 找出每個 yaml_rule deprecated name 的最新 created_at(保留基準)
|
||||
-- 刪除同名同 source=yaml_rule + status=deprecated 中非最新的記錄
|
||||
DELETE FROM playbooks
|
||||
WHERE status = 'deprecated'
|
||||
AND source = 'yaml_rule'
|
||||
AND playbook_id NOT IN (
|
||||
-- 每個 name 保留 created_at 最新的那一筆
|
||||
SELECT DISTINCT ON (name) playbook_id
|
||||
FROM playbooks
|
||||
WHERE status = 'deprecated'
|
||||
AND source = 'yaml_rule'
|
||||
ORDER BY name, created_at DESC
|
||||
);
|
||||
|
||||
-- 執行後確認
|
||||
-- SELECT source, status, COUNT(*) FROM playbooks GROUP BY source, status ORDER BY source, status;
|
||||
|
||||
COMMIT;
|
||||
@@ -22,12 +22,15 @@ from src.models.drift import (
|
||||
DriftReport,
|
||||
DriftScanRequest,
|
||||
DriftScanResponse,
|
||||
DriftStatus,
|
||||
)
|
||||
from src.repositories.drift_repository import get_drift_repository
|
||||
from src.services.drift_adopt_service import get_drift_adopt_service
|
||||
from src.services.drift_analyzer import get_drift_analyzer
|
||||
from src.services.drift_detector import get_drift_detector
|
||||
from src.services.drift_interpreter import get_drift_interpreter
|
||||
from src.services.drift_remediator import get_drift_remediator
|
||||
from src.utils.timezone import now_taipei
|
||||
|
||||
router = APIRouter(prefix="/drift", tags=["drift"])
|
||||
|
||||
@@ -155,7 +158,17 @@ async def internal_scan(background_tasks: BackgroundTasks) -> dict:
|
||||
# =============================================================================
|
||||
|
||||
async def _analyze_and_notify(report: DriftReport) -> None:
|
||||
"""背景:Nemotron 意圖分析 + Telegram 推送 + Phase 30 AI 人話摘要"""
|
||||
"""
|
||||
背景:Nemotron 意圖分析 + 低風險自動採納嘗試 + Telegram 推送
|
||||
|
||||
2026-04-24 ogt + Claude Sonnet 4.6: 新增低風險自動採納
|
||||
流程:
|
||||
1. Nemotron 意圖分析(同原先)
|
||||
2. 嘗試 auto_adopt_if_safe():
|
||||
- 通過 → 發 TYPE-1 無按鈕通知(PR 已建立,請 SRE 複核),不再推送帶按鈕卡片
|
||||
- 未通過(skipped=True)→ 走原有 narrator TYPE-4D 卡片流程
|
||||
- 採納失敗(skipped=False, success=False)→ 同樣走 narrator 讓人工介入
|
||||
"""
|
||||
import structlog as _structlog
|
||||
_logger = _structlog.get_logger(__name__)
|
||||
try:
|
||||
@@ -165,6 +178,38 @@ async def _analyze_and_notify(report: DriftReport) -> None:
|
||||
repo = get_drift_repository()
|
||||
await repo.update_interpretation(report.report_id, interpretation)
|
||||
|
||||
# 2026-04-24: 嘗試低風險自動採納
|
||||
auto_adopted = False
|
||||
try:
|
||||
adopt_svc = get_drift_adopt_service()
|
||||
auto_result = await adopt_svc.auto_adopt_if_safe(report)
|
||||
if auto_result.get("success"):
|
||||
# 自動採納成功:更新狀態,跳過人工卡片
|
||||
await repo.update_status(
|
||||
report.report_id,
|
||||
DriftStatus.ADOPTED,
|
||||
resolved_at=now_taipei(),
|
||||
)
|
||||
auto_adopted = True
|
||||
_logger.info(
|
||||
"drift_auto_adopted",
|
||||
report_id=report.report_id,
|
||||
pr_url=auto_result.get("pr_url"),
|
||||
)
|
||||
else:
|
||||
_logger.info(
|
||||
"drift_auto_adopt_skipped",
|
||||
report_id=report.report_id,
|
||||
reason=auto_result.get("reason", ""),
|
||||
skipped=auto_result.get("skipped", True),
|
||||
)
|
||||
except Exception as e:
|
||||
_logger.warning("drift_auto_adopt_error", report_id=report.report_id, error=str(e))
|
||||
|
||||
if auto_adopted:
|
||||
# 自動採納成功,Telegram 通知已在 auto_adopt_if_safe 內發出,不再推送按鈕卡片
|
||||
return
|
||||
|
||||
# ADR-075: drift_narrator_service 負責發送 TYPE-4D 卡片(含按鈕)
|
||||
# 舊的 send_text() 已移除,改由 narrate_and_notify() 統一處理
|
||||
try:
|
||||
|
||||
@@ -3,11 +3,16 @@ AI SLO Watchdog Job — 系統自健診(每 15 分鐘)
|
||||
=============================================
|
||||
MASTER §1.1 AI 自主化方向:系統必須能感知自身故障。
|
||||
ADR-092 (2026-04-20 ogt + Claude Opus 4.7 Asia/Taipei)
|
||||
ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
|
||||
W-2 修復:改用 telegram_message_id IS NULL 判斷真正靜默,排除 tg_sent TTL 過期誤判
|
||||
W-5 新增:Agent Debate 失敗導致告警卡在分析中(description='待分析')
|
||||
|
||||
檢查項目:
|
||||
W-1 AI SLO 違反(決策品質,7d 滾動)
|
||||
W-2 Telegram 靜默偵測(PENDING 告警無 tg_sent 確認超過 30 分鐘)
|
||||
W-2 Telegram 靜默偵測(PENDING 告警 telegram_message_id IS NULL 超過 30 分鐘)
|
||||
W-3 飛輪 execution_success_rate 低落(< 30%)
|
||||
W-4 無 APPROVED Playbook(自動修復鏈路斷裂)
|
||||
W-5 Agent Debate 失敗(PENDING 告警 description='待分析' 超過 1 小時)
|
||||
|
||||
任一異常 → send_meta_alert(TYPE-8M,flywheel_health)
|
||||
去重:Redis watchdog:alert:{dedup_hash} TTL 1h,避免每 15 分鐘重複洗版
|
||||
@@ -19,7 +24,7 @@ import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import structlog
|
||||
from sqlalchemy import and_, select
|
||||
from sqlalchemy import and_, func, select
|
||||
|
||||
from src.core.redis_client import get_redis
|
||||
from src.db.base import get_db_context
|
||||
@@ -31,8 +36,9 @@ logger = structlog.get_logger(__name__)
|
||||
|
||||
_INTERVAL_SEC = 900 # 每 15 分鐘
|
||||
_DEDUP_TTL_SEC = 3600 # 同一告警 1 小時內不重複發送
|
||||
_TG_SILENCE_THRESHOLD = 2 # PENDING 無 tg_sent 確認數量告警門檻
|
||||
_TG_SILENCE_THRESHOLD = 2 # PENDING telegram_message_id IS NULL 告警門檻
|
||||
_FLYWHEEL_SUCCESS_MIN = 0.30 # 執行成功率下限
|
||||
_STUCK_ANALYSIS_THRESHOLD = 3 # Agent Debate 失敗導致卡住的告警門檻
|
||||
|
||||
|
||||
async def run_ai_slo_watchdog_loop() -> None:
|
||||
@@ -62,11 +68,17 @@ async def _check_once() -> None:
|
||||
except Exception as e:
|
||||
logger.warning("watchdog_w1_slo_check_failed", error=str(e))
|
||||
|
||||
# W-2: Telegram 靜默偵測(PENDING 無 tg_sent 確認 > 30 分鐘)
|
||||
# W-2: Telegram 靜默偵測(PENDING 告警 telegram_message_id IS NULL 超過 30 分鐘)
|
||||
# ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
|
||||
# 修復誤判:改用 telegram_message_id IS NULL 判斷,排除 tg_sent Redis TTL 過期導致的誤報。
|
||||
# 有 telegram_message_id = Telegram 已發過,tg_sent key 過期不算靜默;
|
||||
# telegram_message_id IS NULL = Telegram 從未發過,才是真正靜默。
|
||||
try:
|
||||
silent_count = await _count_pending_no_tg_sent()
|
||||
if silent_count >= _TG_SILENCE_THRESHOLD:
|
||||
violations.append(f"{silent_count} 個 PENDING 告警超 30 分鐘無 Telegram 確認(疑似靜默故障)")
|
||||
violations.append(
|
||||
f"{silent_count} 個 PENDING 告警超 30 分鐘未送達 Telegram(未曾發送,非 TTL 過期)"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("watchdog_w2_tg_silence_check_failed", error=str(e))
|
||||
|
||||
@@ -87,8 +99,21 @@ async def _check_once() -> None:
|
||||
except Exception as e:
|
||||
logger.warning("watchdog_w4_playbook_check_failed", error=str(e))
|
||||
|
||||
# W-5: Agent Debate 失敗(PENDING 告警 description='待分析' 超過 1 小時)
|
||||
# ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
|
||||
# telegram_push_suppressed_no_analysis 那類案例在 DB 留下 PENDING + description='待分析',
|
||||
# 查 DB 比查 structlog 更可靠。門檻 > 3(1h 內若偶有 1-2 筆超時可接受)。
|
||||
try:
|
||||
stuck_count = await _count_pending_stuck_analysis()
|
||||
if stuck_count > _STUCK_ANALYSIS_THRESHOLD:
|
||||
violations.append(
|
||||
f"Agent Debate 失敗導致 {stuck_count} 個告警分析卡住(PENDING + description='待分析' 超過 1 小時)"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("watchdog_w5_stuck_analysis_check_failed", error=str(e))
|
||||
|
||||
if not violations:
|
||||
logger.debug("ai_slo_watchdog_all_ok", checks=4)
|
||||
logger.debug("ai_slo_watchdog_all_ok", checks=5)
|
||||
return
|
||||
|
||||
# 去重:violations 相同內容 1 小時內不重複發
|
||||
@@ -112,7 +137,7 @@ async def _check_once() -> None:
|
||||
alert_category="flywheel_health",
|
||||
diagnosis=diagnosis,
|
||||
severity_level="critical",
|
||||
system_impact=f"{len(violations)} 項 KPI 異常,飛輪自動化能力可能降級",
|
||||
system_impact=f"{len(violations)} 項 KPI 異常(W-1~W-5),飛輪自動化能力可能降級",
|
||||
)
|
||||
logger.warning(
|
||||
"ai_slo_watchdog_alert_sent",
|
||||
@@ -126,34 +151,34 @@ async def _check_once() -> None:
|
||||
|
||||
async def _count_pending_no_tg_sent() -> int:
|
||||
"""
|
||||
查詢 PENDING 超過 30 分鐘且 Redis tg_sent:{fingerprint} 無確認的告警數量。
|
||||
與 ADR-092 B2 修復配合:B2 修復後新告警會標記 tg_sent;
|
||||
此查詢偵測仍存在的靜默告警(B2 修復前殘留 + 未來潛在故障)。
|
||||
查詢真正靜默的 PENDING 告警:PENDING 超過 30 分鐘且 telegram_message_id IS NULL。
|
||||
|
||||
ADR-092 B3 修復(2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
|
||||
舊版只查 Redis tg_sent:{fp} key,但該 key TTL 僅 24h;
|
||||
PENDING 超過 24h 後 key 自然過期 → 誤判為靜默故障(W-2 誤報根因)。
|
||||
|
||||
修復方案:改用 PG 欄位 telegram_message_id IS NULL 作為判斷依據:
|
||||
- telegram_message_id IS NOT NULL = Telegram 確實已發過(不算靜默)
|
||||
- telegram_message_id IS NULL = Telegram 從未發過(才是真正靜默)
|
||||
tg_sent Redis key 不再用於 W-2 判斷。
|
||||
"""
|
||||
cutoff = datetime.now(UTC) - timedelta(minutes=30)
|
||||
redis = get_redis()
|
||||
silent = 0
|
||||
|
||||
async with get_db_context() as db:
|
||||
result = await db.execute(
|
||||
select(ApprovalRecord.id, ApprovalRecord.fingerprint)
|
||||
select(ApprovalRecord.id)
|
||||
.where(
|
||||
and_(
|
||||
ApprovalRecord.status == ApprovalStatus.PENDING,
|
||||
ApprovalRecord.created_at <= cutoff,
|
||||
ApprovalRecord.fingerprint.isnot(None),
|
||||
ApprovalRecord.telegram_message_id.is_(None),
|
||||
)
|
||||
)
|
||||
.limit(20)
|
||||
)
|
||||
rows = result.all()
|
||||
|
||||
for row in rows:
|
||||
fp = row.fingerprint
|
||||
if fp and not await redis.exists(f"tg_sent:{fp}"):
|
||||
silent += 1
|
||||
|
||||
return silent
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def _count_approved_playbooks() -> int:
|
||||
@@ -164,3 +189,37 @@ async def _count_approved_playbooks() -> int:
|
||||
sa_text("SELECT COUNT(*) FROM playbooks WHERE status = 'approved'")
|
||||
)
|
||||
return result.scalar() or 0
|
||||
|
||||
|
||||
async def _count_pending_stuck_analysis() -> int:
|
||||
"""
|
||||
查詢 Agent Debate 失敗導致卡住的 PENDING 告警數。
|
||||
|
||||
ADR-092 B3(2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
|
||||
decision_manager.py 第 279-286 行:當 description='待分析' 且 action 屬於
|
||||
失敗狀態集合時,直接 return 不推送 Telegram,記錄 telegram_push_suppressed_no_analysis。
|
||||
這類告警在 DB 中留下 PENDING + description='待分析' 的特徵。
|
||||
|
||||
查詢條件:
|
||||
- status = PENDING
|
||||
- description = '待分析'(Agent Debate 無有效輸出)
|
||||
- created_at <= now - 1h(給一定緩衝,排除剛建立的正常案例)
|
||||
"""
|
||||
cutoff = datetime.now(UTC) - timedelta(hours=1)
|
||||
|
||||
async with get_db_context() as db:
|
||||
# 用 action IS NULL/空 比 description 字串比對更可靠:
|
||||
# decision_manager 超時降級後 description 文字已改,但 action 仍為空
|
||||
result = await db.execute(
|
||||
select(func.count())
|
||||
.select_from(ApprovalRecord)
|
||||
.where(
|
||||
and_(
|
||||
ApprovalRecord.status == ApprovalStatus.PENDING,
|
||||
ApprovalRecord.created_at <= cutoff,
|
||||
ApprovalRecord.suggested_action.in_(["", "待分析", "NO_ACTION", None]),
|
||||
ApprovalRecord.telegram_message_id.is_(None),
|
||||
)
|
||||
)
|
||||
)
|
||||
return result.scalar() or 0
|
||||
|
||||
@@ -22,6 +22,10 @@ disposition 記錄:
|
||||
4. 每次執行記錄 resolved_count / error_count
|
||||
|
||||
2026-04-15 ogt + Claude Sonnet 4.6(亞太):P2 飛輪斷鏈修復
|
||||
ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
|
||||
執行間隔從 3600s(1h)縮短至 900s(15 分鐘),與 W-2 Watchdog 同頻,
|
||||
減少 PENDING 積壓導致 W-2 誤報的時間視窗;
|
||||
BATCH_LIMIT 從 50 提升至 200,加快存量清理速度。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -40,13 +44,17 @@ from src.utils.timezone import now_taipei
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# 每次最多處理幾筆,避免單次執行阻塞過長
|
||||
BATCH_LIMIT = 50
|
||||
# ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):從 50 提升至 200,加快存量清理
|
||||
BATCH_LIMIT = 200
|
||||
|
||||
|
||||
async def run_approval_timeout_resolver() -> None:
|
||||
"""
|
||||
無限迴圈:每小時執行一次逾期 Approval 結案掃描。
|
||||
無限迴圈:每 15 分鐘執行一次逾期 Approval 結案掃描。
|
||||
在 main.py startup 以 asyncio.create_task 掛載。
|
||||
ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
|
||||
從 3600s(1h)縮短至 900s(15 分鐘),與 ai_slo_watchdog_job 同頻,
|
||||
減少 W-2 因 PENDING 積壓誤報的時間視窗。
|
||||
"""
|
||||
while True:
|
||||
try:
|
||||
@@ -60,7 +68,7 @@ async def run_approval_timeout_resolver() -> None:
|
||||
except Exception as e:
|
||||
logger.error("approval_timeout_resolver_loop_error", error=str(e))
|
||||
|
||||
await asyncio.sleep(3600) # 每小時執行一次
|
||||
await asyncio.sleep(900) # ADR-092 B3: 每 15 分鐘執行(原 3600s)
|
||||
|
||||
|
||||
async def _resolve_expired_approvals() -> tuple[int, int]:
|
||||
|
||||
@@ -371,7 +371,12 @@ class K8sProvider(MCPToolProvider):
|
||||
# =========================================================================
|
||||
|
||||
async def _k8s_get_pod_logs(self, executor, parameters: dict) -> dict:
|
||||
pod_name = _validate_name(parameters["pod_name"], "pod_name")
|
||||
# Bug 根因:LLM 有時傳空字串 pod_name,_validate_name 直接 raise ValueError 炸整個 investigator
|
||||
# 修復:pod_name 空時返回結構化 error,讓 Agent 可繼續分析其他工具(2026-04-24 Claude Sonnet 4.6)
|
||||
raw_pod_name = parameters.get("pod_name", "")
|
||||
if not raw_pod_name or not raw_pod_name.strip():
|
||||
return {"error": "empty_pod_name", "message": "pod_name 為空,請提供具體 pod 名稱(如 awoooi-api-xxx-yyy)"}
|
||||
pod_name = _validate_name(raw_pod_name, "pod_name")
|
||||
namespace = _validate_namespace(parameters.get("namespace", DEFAULT_NAMESPACE))
|
||||
container = parameters.get("container", "")
|
||||
tail = max(1, min(int(parameters.get("tail", 100)), 500))
|
||||
@@ -437,7 +442,11 @@ class K8sProvider(MCPToolProvider):
|
||||
}
|
||||
|
||||
async def _k8s_describe_pod(self, executor, parameters: dict) -> dict:
|
||||
pod_name = _validate_name(parameters["pod_name"], "pod_name")
|
||||
# Bug 根因:同 _k8s_get_pod_logs,空 pod_name 會 raise 炸 investigator(2026-04-24 Claude Sonnet 4.6)
|
||||
raw_pod_name = parameters.get("pod_name", "")
|
||||
if not raw_pod_name or not raw_pod_name.strip():
|
||||
return {"error": "empty_pod_name", "message": "pod_name 為空,請提供具體 pod 名稱(如 awoooi-api-xxx-yyy)"}
|
||||
pod_name = _validate_name(raw_pod_name, "pod_name")
|
||||
namespace = _validate_namespace(parameters.get("namespace", DEFAULT_NAMESPACE))
|
||||
|
||||
cmd = f"kubectl describe pod {pod_name} -n {namespace}"
|
||||
|
||||
@@ -191,7 +191,11 @@ class PrometheusProvider(MCPToolProvider):
|
||||
# =========================================================================
|
||||
|
||||
async def _instant_query(self, params: dict) -> dict:
|
||||
query = params["query"]
|
||||
# Bug 根因:LLM 產生的 payload 有時用 promql 而非 query,直接存取 ["query"] 會 KeyError
|
||||
# 修復:優先取 query,fallback promql;兩者皆缺則返回結構化 error(2026-04-24 Claude Sonnet 4.6)
|
||||
query = params.get("query", "") or params.get("promql", "")
|
||||
if not query:
|
||||
return {"error": "missing_query_parameter", "message": "必須提供 query 或 promql 參數"}
|
||||
time_param = params.get("time", "")
|
||||
|
||||
url = f"{self._base_url()}/api/v1/query"
|
||||
@@ -216,7 +220,10 @@ class PrometheusProvider(MCPToolProvider):
|
||||
}
|
||||
|
||||
async def _range_query(self, params: dict) -> dict:
|
||||
query = params["query"]
|
||||
# Bug 根因:同 _instant_query,LLM 可能用 promql 而非 query(2026-04-24 Claude Sonnet 4.6)
|
||||
query = params.get("query", "") or params.get("promql", "")
|
||||
if not query:
|
||||
return {"error": "missing_query_parameter", "message": "必須提供 query 或 promql 參數"}
|
||||
window_minutes = int(params.get("window_minutes", 15))
|
||||
step_seconds = int(params.get("step_seconds", 60))
|
||||
|
||||
@@ -251,7 +258,10 @@ class PrometheusProvider(MCPToolProvider):
|
||||
}
|
||||
|
||||
async def _alert_history(self, params: dict) -> dict:
|
||||
alertname = params["alertname"]
|
||||
# Bug 根因:LLM 偶發 alertname 缺失,直接存取 ["alertname"] 會 KeyError(2026-04-24 Claude Sonnet 4.6)
|
||||
alertname = params.get("alertname", "")
|
||||
if not alertname:
|
||||
return {"error": "missing_alertname_parameter", "message": "必須提供 alertname 參數"}
|
||||
# P1 fix 2026-04-11: 白名單驗證防止 PromQL label injection
|
||||
if not _RE_SAFE_ALERTNAME.match(alertname):
|
||||
raise ValueError(f"Unsafe alertname: {alertname!r}")
|
||||
|
||||
@@ -544,5 +544,6 @@ class SSHProvider(MCPToolProvider):
|
||||
known_hosts=known_hosts_path, # None = 跳過驗證(內網),或指定文件路徑
|
||||
connect_timeout=timeout,
|
||||
) as conn:
|
||||
result = await asyncssh.run(conn, cmd, timeout=timeout, check=False)
|
||||
# Bug 根因:asyncssh 模組沒有頂層 run();應呼叫 conn.run()(2026-04-24 Claude Sonnet 4.6)
|
||||
result = await conn.run(cmd, timeout=timeout, check=False)
|
||||
return (result.stdout or ""), (result.stderr or "")
|
||||
|
||||
@@ -703,25 +703,58 @@ async def auto_generate_rule(
|
||||
|
||||
# 1. 先試 Ollama
|
||||
raw = await _call_ollama(prompt, ollama_url, model)
|
||||
llm_source = "ollama" if raw else None
|
||||
|
||||
# 2. Ollama 失敗 → Gemini
|
||||
if not raw and gemini_api_key:
|
||||
raw = await _call_gemini(prompt, gemini_api_key)
|
||||
llm_source = "gemini" if raw else None
|
||||
|
||||
if not raw:
|
||||
logger.warning("auto_rule_no_response", alertname=alertname_safe)
|
||||
logger.warning(
|
||||
"auto_rule_auto_generate_failed",
|
||||
alertname=alertname_safe,
|
||||
reason="llm_no_response",
|
||||
)
|
||||
return
|
||||
|
||||
yaml_block = _extract_yaml_block(raw)
|
||||
success = _append_rule_to_yaml(yaml_block, alertname_safe)
|
||||
if success:
|
||||
logger.info("auto_rule_success", alertname=alertname_safe, rule_id=rule_id)
|
||||
# 立即為新規則建立 APPROVED Playbook(不等下次重啟)
|
||||
logger.info(
|
||||
"alert_rule_auto_generated",
|
||||
alertname=alertname_safe,
|
||||
rule_id=rule_id,
|
||||
source=llm_source,
|
||||
)
|
||||
# 成功後記錄今日新增規則數(供系統報告顯示)
|
||||
import asyncio as _asyncio
|
||||
try:
|
||||
from src.core.redis_client import get_redis as _get_redis
|
||||
from src.utils.timezone import now_taipei as _now_taipei
|
||||
_today = _now_taipei().strftime("%Y%m%d")
|
||||
_key = f"stats:auto_rule_generated:{_today}"
|
||||
_redis = _get_redis()
|
||||
|
||||
async def _incr_rule_stat() -> None:
|
||||
# pipeline 原子化 incr+expire,避免 race condition
|
||||
async with _redis.pipeline() as _p:
|
||||
_p.incr(_key)
|
||||
_p.expire(_key, 86400 * 7)
|
||||
await _p.execute()
|
||||
|
||||
_asyncio.create_task(_incr_rule_stat())
|
||||
except Exception as _redis_err:
|
||||
logger.debug("auto_rule_stats_redis_failed", error=str(_redis_err))
|
||||
# 立即為新規則建立 APPROVED Playbook(不等下次重啟)
|
||||
from src.services.playbook_seed_service import seed_playbooks_from_rules
|
||||
_asyncio.create_task(seed_playbooks_from_rules())
|
||||
else:
|
||||
logger.warning("auto_rule_failed_validation", alertname=alertname_safe)
|
||||
logger.warning(
|
||||
"auto_rule_auto_generate_failed",
|
||||
alertname=alertname_safe,
|
||||
reason="yaml_validation_failed",
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("auto_rule_exception", alertname=alertname_safe, error=str(e))
|
||||
|
||||
@@ -357,11 +357,15 @@ class ApprovalDBService:
|
||||
|
||||
return None
|
||||
|
||||
async def mark_telegram_confirmed(self, fingerprint: str, ttl: int = 86400) -> None:
|
||||
async def mark_telegram_confirmed(self, fingerprint: str, ttl: int = 108000) -> None:
|
||||
"""
|
||||
2026-04-20 ogt + Claude Opus 4.7: ADR-092
|
||||
記錄 Telegram 已成功發送,防止 PENDING 誤收斂造成永久靜默。
|
||||
TTL 與 PENDING_TTL_HOURS 對齊(24h)。
|
||||
ADR-092 B3 (2026-04-24 ogt + Claude Sonnet 4.6 Asia/Taipei):
|
||||
TTL 從 86400s(24h)延長至 108000s(30h = 24h + 6h buffer),
|
||||
避免 PENDING 存活期與 tg_sent key 同時到期時的邊緣誤判。
|
||||
W-2 現已改用 telegram_message_id IS NULL 判斷(不再依賴此 key),
|
||||
此 TTL 保留供收斂去重邏輯(approval_db.py 第 342 行)使用。
|
||||
"""
|
||||
try:
|
||||
r = get_redis()
|
||||
|
||||
@@ -450,6 +450,28 @@ class AutoApprovePolicy:
|
||||
trust_score=kwargs.get("trust_score"),
|
||||
)
|
||||
|
||||
# 記錄拒絕原因計數(供系統報告分析人工審核積壓根因)
|
||||
# 在 async context 中呼叫,用 create_task 不阻塞主流程
|
||||
try:
|
||||
import asyncio as _asyncio
|
||||
from datetime import datetime as _dt
|
||||
_today = _dt.now().strftime("%Y%m%d")
|
||||
_reject_key = f"stats:auto_approve_rejected:{reason.value}:{_today}"
|
||||
|
||||
async def _incr_reject_stat() -> None:
|
||||
try:
|
||||
from src.core.redis_client import get_redis as _get_redis
|
||||
_r = _get_redis()
|
||||
await _r.incr(_reject_key)
|
||||
await _r.expire(_reject_key, 86400 * 7)
|
||||
except Exception:
|
||||
pass # Redis 不可用時靜默降級,不影響核心流程
|
||||
|
||||
loop = _asyncio.get_running_loop()
|
||||
loop.create_task(_incr_reject_stat())
|
||||
except RuntimeError:
|
||||
pass # 非 async context(如單元測試),靜默跳過
|
||||
|
||||
return decision
|
||||
|
||||
def _extract_action_pattern(self, action: str) -> str:
|
||||
|
||||
@@ -1114,7 +1114,23 @@ def _package_to_proposal_data(package: Any) -> dict[str, Any]:
|
||||
# 2026-04-17 ogt + Claude Sonnet 4.6(亞太): 修復超時髒資料污染卡片
|
||||
# 舊:blocked_reason → desc_parts → description → suggested_action 欄位顯示「備注:全局超時 > 90.0s」
|
||||
# 新:blocked_reason 只寫入 proposal_data["blocked_reason"],供下游閘門邏輯用,禁止進卡片顯示
|
||||
description = ";".join(desc_parts) if desc_parts else (action[:200] if action else "待分析")
|
||||
# 2026-04-24 ogt + Claude Sonnet 4.6: ADR-091 鐵閘 bypass 修復
|
||||
# 問題:Agent Debate 超時 (TIMEOUT/FAILED) → desc_parts 空 + action 空 → description="待分析"
|
||||
# → ADR-091 鐵閘攔截(description=待分析 AND action in FAILED_TOKENS)→ PENDING 積壓
|
||||
# 修法:超時/全降級時使用明確降級文字,讓鐵閘放行 → Telegram 推送降級通知 → 人工可見
|
||||
if desc_parts:
|
||||
description = ";".join(desc_parts)
|
||||
elif action:
|
||||
description = action[:200]
|
||||
else:
|
||||
_status = getattr(package, "session_status", None)
|
||||
_status_val = _status.value if _status and hasattr(_status, "value") else ""
|
||||
if _status_val == "timeout":
|
||||
description = "AI 分析超時(90s),降級至人工審核"
|
||||
elif _status_val == "failed" or getattr(package, "all_agents_degraded", False):
|
||||
description = "AI 分析降級,所有 Agent 無有效輸出,建議人工確認"
|
||||
else:
|
||||
description = "待分析"
|
||||
|
||||
return {
|
||||
"action": action,
|
||||
|
||||
@@ -35,7 +35,7 @@ from src.core.config import get_settings
|
||||
from src.utils.timezone import now_taipei
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from src.models.drift import DriftReport
|
||||
from src.models.drift import DriftInterpretation, DriftReport
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
@@ -59,6 +59,165 @@ class DriftAdoptService:
|
||||
self._repo = settings.GITEA_REPO_NAME
|
||||
self._k8s_dir = pathlib.Path("k8s")
|
||||
|
||||
async def auto_adopt_if_safe(self, report: "DriftReport") -> dict:
|
||||
"""
|
||||
低風險 drift 自動採納(2026-04-24 ogt + Claude Sonnet 4.6)
|
||||
|
||||
自動採納條件(全部滿足):
|
||||
1. high_count == 0(無 HIGH drift)
|
||||
2. medium_count <= 5(MEDIUM drift 不超過 5 項)
|
||||
3. actionable_count <= 10(非白名單非 trivial 項目不超過 10)
|
||||
4. interpretation.confidence >= 0.70(意圖確定性足夠)
|
||||
5. interpretation.intent NOT IN {unknown}(必須有明確意圖)
|
||||
6. interpretation.risk NOT IN {HIGH, CRITICAL}(非高風險)
|
||||
|
||||
不滿足任一條件 → return {"success": False, "reason": "...", "skipped": True}
|
||||
自動採納 PR title 加 [AUTO] 前綴,通知標示「請 SRE 複核後 merge」
|
||||
|
||||
Returns:
|
||||
{"success": bool, "skipped": bool, "reason": str, "pr_url": str | None}
|
||||
skipped=True → 條件不符,人工卡片照舊推送
|
||||
skipped=False → 嘗試了,但 Gitea API 失敗
|
||||
"""
|
||||
interp = report.interpretation
|
||||
|
||||
# 條件 1: 無 HIGH drift
|
||||
if report.high_count > 0:
|
||||
return {
|
||||
"success": False,
|
||||
"skipped": True,
|
||||
"reason": f"存在 {report.high_count} 項 HIGH drift,不自動採納",
|
||||
"pr_url": None,
|
||||
}
|
||||
|
||||
# 條件 2: MEDIUM 不超過 5 項
|
||||
if report.medium_count > 5:
|
||||
return {
|
||||
"success": False,
|
||||
"skipped": True,
|
||||
"reason": f"MEDIUM drift {report.medium_count} 項超過上限 5,不自動採納",
|
||||
"pr_url": None,
|
||||
}
|
||||
|
||||
# 條件 3: actionable 不超過 10(過濾白名單 + trivial)
|
||||
actionable_items = [
|
||||
item for item in report.items
|
||||
if not item.is_allowlisted
|
||||
and not self._is_trivial_drift(item.git_value, item.actual_value)
|
||||
]
|
||||
actionable_count = len(actionable_items)
|
||||
if actionable_count > 10:
|
||||
return {
|
||||
"success": False,
|
||||
"skipped": True,
|
||||
"reason": f"可操作漂移 {actionable_count} 項超過上限 10,不自動採納",
|
||||
"pr_url": None,
|
||||
}
|
||||
|
||||
# 條件 4 / 5 / 6: 需要有 interpretation
|
||||
if not interp:
|
||||
return {
|
||||
"success": False,
|
||||
"skipped": True,
|
||||
"reason": "尚無 Nemotron 意圖分析,不自動採納",
|
||||
"pr_url": None,
|
||||
}
|
||||
|
||||
if interp.confidence < 0.70:
|
||||
return {
|
||||
"success": False,
|
||||
"skipped": True,
|
||||
"reason": f"意圖信心 {interp.confidence:.0%} < 70%,不自動採納",
|
||||
"pr_url": None,
|
||||
}
|
||||
|
||||
if interp.intent.value == "unknown":
|
||||
return {
|
||||
"success": False,
|
||||
"skipped": True,
|
||||
"reason": "Nemotron 意圖為 unknown,不自動採納",
|
||||
"pr_url": None,
|
||||
}
|
||||
|
||||
if interp.risk.upper() in ("HIGH", "CRITICAL"):
|
||||
return {
|
||||
"success": False,
|
||||
"skipped": True,
|
||||
"reason": f"Nemotron 風險評估為 {interp.risk},不自動採納",
|
||||
"pr_url": None,
|
||||
}
|
||||
|
||||
logger.info(
|
||||
"drift_auto_adopt_eligible",
|
||||
report_id=report.report_id,
|
||||
medium_count=report.medium_count,
|
||||
actionable=actionable_count,
|
||||
intent=interp.intent.value,
|
||||
confidence=interp.confidence,
|
||||
risk=interp.risk,
|
||||
)
|
||||
|
||||
# 通過全部條件 → 執行採納(suppress_notification=True 避免與下方通知重複)
|
||||
result = await self.adopt(report, field_description="[AUTO] 低風險自動採納", suppress_notification=True)
|
||||
if result.get("success"):
|
||||
# 通知 SRE:無按鈕,標示需複核
|
||||
await self._notify_auto_adopt_telegram(result["pr_url"], report, actionable_count, interp)
|
||||
|
||||
return {
|
||||
"success": result.get("success", False),
|
||||
"skipped": False,
|
||||
"reason": result.get("message", ""),
|
||||
"pr_url": result.get("pr_url"),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _is_trivial_drift(git_val, actual_val) -> bool:
|
||||
"""
|
||||
判斷是否為 K8s controller 自動補齊的噪音
|
||||
(與 DriftNarratorService._is_trivial_drift 邏輯一致)
|
||||
"""
|
||||
def _is_empty(v) -> bool:
|
||||
if v is None:
|
||||
return True
|
||||
s = str(v).strip()
|
||||
return s in ("", "{}", "[]", "null", "None", "false", "False", "0")
|
||||
return _is_empty(git_val) and _is_empty(actual_val)
|
||||
|
||||
async def _notify_auto_adopt_telegram(
|
||||
self,
|
||||
pr_url: str,
|
||||
report: "DriftReport",
|
||||
actionable_count: int,
|
||||
interp: "DriftInterpretation",
|
||||
) -> None:
|
||||
"""
|
||||
自動採納成功後的無按鈕 Telegram 通知(TYPE-1 純資訊)
|
||||
標示「已自動建立 PR,請 SRE 複核後 merge」
|
||||
"""
|
||||
try:
|
||||
from src.services.telegram_gateway import get_telegram_gateway
|
||||
tg = get_telegram_gateway()
|
||||
message = (
|
||||
f"Namespace: {report.namespace}\n"
|
||||
f"漂移摘要: {report.summary}(可操作 {actionable_count} 項)\n"
|
||||
f"Nemotron 意圖: {interp.intent.value} | 信心: {interp.confidence:.0%} | 風險: {interp.risk}\n\n"
|
||||
f"PR: {pr_url}\n\n"
|
||||
f"請 SRE 複核後 merge。"
|
||||
)
|
||||
await tg.send_info_notification(
|
||||
incident_id=report.report_id,
|
||||
title="Config Drift 已自動採納(低風險)",
|
||||
message=message,
|
||||
alertname="ConfigDriftAutoAdopt",
|
||||
severity="info",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("drift_auto_adopt_telegram_failed", error=str(e))
|
||||
|
||||
# =========================================================================
|
||||
# Public: 人工按鈕觸發入口
|
||||
# =========================================================================
|
||||
|
||||
async def adopt_drift(self, report_id: str) -> dict:
|
||||
"""
|
||||
2026-04-20 ogt + Claude Opus 4.7: Telegram 按鈕呼叫入口
|
||||
@@ -71,7 +230,7 @@ class DriftAdoptService:
|
||||
return {"success": False, "message": f"Report {report_id} not found"}
|
||||
return await self.adopt(report)
|
||||
|
||||
async def adopt(self, report: "DriftReport", field_description: str = "") -> dict:
|
||||
async def adopt(self, report: "DriftReport", field_description: str = "", suppress_notification: bool = False) -> dict:
|
||||
"""
|
||||
將漂移寫回 Git:建立 branch + commit + PR
|
||||
|
||||
@@ -126,8 +285,9 @@ class DriftAdoptService:
|
||||
if not pr_url:
|
||||
return {"success": False, "message": "建立 PR 失敗", "pr_url": None}
|
||||
|
||||
# Step 5: 推送 Telegram 通知
|
||||
await self._notify_telegram(pr_url, report, pr_title)
|
||||
# Step 5: 推送 Telegram 通知(auto_adopt_if_safe 會自己發,避免重複)
|
||||
if not suppress_notification:
|
||||
await self._notify_telegram(pr_url, report, pr_title)
|
||||
|
||||
logger.info("drift_adopt_pr_created", report_id=report.report_id, pr_url=pr_url)
|
||||
return {
|
||||
|
||||
@@ -87,6 +87,19 @@ class TelegramBotStats:
|
||||
last_callback_ago_min: Optional[float] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AutomationStats:
|
||||
"""自動化六大能力今日統計(2026-04-24 ogt: Task 3 — 告警自動化可觀測性)"""
|
||||
# 自動規則生成
|
||||
auto_rule_generated_today: int = 0
|
||||
# 自動審核拒絕(按原因分組)
|
||||
reject_counts: dict[str, int] = field(default_factory=dict)
|
||||
# 今日 KM 新增
|
||||
km_created_today: int = 0
|
||||
# Config Drift 自動採納(今日)
|
||||
drift_adopted_today: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class HeartbeatReport:
|
||||
timestamp: datetime
|
||||
@@ -102,6 +115,8 @@ class HeartbeatReport:
|
||||
pods: list[PodInfo] = field(default_factory=list)
|
||||
scanners: ScannerStats = field(default_factory=ScannerStats)
|
||||
telegram_bot: TelegramBotStats = field(default_factory=TelegramBotStats)
|
||||
# 2026-04-24 自動化統計
|
||||
automation: AutomationStats = field(default_factory=AutomationStats)
|
||||
|
||||
@property
|
||||
def has_warnings(self) -> bool:
|
||||
@@ -139,6 +154,8 @@ class HeartbeatReportService:
|
||||
self._get_pod_status(),
|
||||
self._get_scanner_stats(),
|
||||
self._probe_telegram_bot(),
|
||||
# 2026-04-24 自動化統計
|
||||
self._get_automation_stats(),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
@@ -147,6 +164,7 @@ class HeartbeatReportService:
|
||||
"_mcp_k8s", "_mcp_ssh", "_mcp_argocd", "_mcp_sentry",
|
||||
"_argocd_sync", "_velero", "_flywheel",
|
||||
"_alert_pipeline", "_db_redis", "_pods", "_scanners", "_tg_bot",
|
||||
"_automation",
|
||||
]
|
||||
collected: dict = {}
|
||||
for key, result in zip(keys, results):
|
||||
@@ -189,6 +207,8 @@ class HeartbeatReportService:
|
||||
report.scanners = collected["_scanners"]
|
||||
if collected["_tg_bot"]:
|
||||
report.telegram_bot = collected["_tg_bot"]
|
||||
if collected["_automation"]:
|
||||
report.automation = collected["_automation"]
|
||||
|
||||
# --- 彙整 warnings ---
|
||||
report.warnings = self._build_warnings(report)
|
||||
@@ -568,6 +588,61 @@ class HeartbeatReportService:
|
||||
s.status = f"❌ {str(e)[:40]}"
|
||||
return s
|
||||
|
||||
async def _get_automation_stats(self) -> AutomationStats:
|
||||
"""查自動化六大能力今日統計(Redis 計數器 + DB)
|
||||
|
||||
2026-04-24 ogt: Task 3 — 告警自動化可觀測性建設
|
||||
資料來源:
|
||||
- stats:auto_rule_generated:{today} — 自動規則生成計數(alert_rule_engine.py 寫入)
|
||||
- stats:auto_approve_rejected:{reason}:{today} — 拒絕自動審核原因分布
|
||||
- knowledge_entries 今日新增(DB)
|
||||
- approval_records drift_adopted 今日(DB)
|
||||
"""
|
||||
stats = AutomationStats()
|
||||
try:
|
||||
from src.core.redis_client import get_redis
|
||||
redis = get_redis()
|
||||
today = now_taipei().strftime("%Y%m%d")
|
||||
|
||||
# 自動規則生成今日計數
|
||||
raw = await redis.get(f"stats:auto_rule_generated:{today}")
|
||||
stats.auto_rule_generated_today = int(raw) if raw else 0
|
||||
|
||||
# 自動審核拒絕原因分布
|
||||
reject_keys = await redis.keys(f"stats:auto_approve_rejected:*:{today}")
|
||||
for key in reject_keys:
|
||||
key_str = key.decode() if isinstance(key, bytes) else key
|
||||
# key 格式:stats:auto_approve_rejected:{reason}:{today}
|
||||
parts = key_str.split(":")
|
||||
reason_part = parts[2] if len(parts) >= 4 else key_str
|
||||
count_raw = await redis.get(key_str)
|
||||
stats.reject_counts[reason_part] = int(count_raw) if count_raw else 0
|
||||
except Exception as e:
|
||||
logger.debug("heartbeat_automation_redis_failed", error=str(e))
|
||||
|
||||
try:
|
||||
from src.db.base import get_db_context
|
||||
from sqlalchemy import text as sa_text
|
||||
async with get_db_context() as db:
|
||||
# 今日新增 KM(timestamptz 直接比較,不需 AT TIME ZONE)
|
||||
km_today = await db.scalar(sa_text(
|
||||
"SELECT COUNT(*) FROM knowledge_entries "
|
||||
"WHERE created_at >= NOW() - interval '24 hours'"
|
||||
))
|
||||
stats.km_created_today = int(km_today or 0)
|
||||
|
||||
# 今日 Config Drift 自動採納(查 drift_reports,adopt() 在此表更新 status)
|
||||
drift_today = await db.scalar(sa_text(
|
||||
"SELECT COUNT(*) FROM drift_reports "
|
||||
"WHERE status = 'adopted' "
|
||||
"AND resolved_at >= NOW() - interval '24 hours'"
|
||||
))
|
||||
stats.drift_adopted_today = int(drift_today or 0)
|
||||
except Exception as e:
|
||||
logger.debug("heartbeat_automation_db_failed", error=str(e))
|
||||
|
||||
return stats
|
||||
|
||||
# =========================================================================
|
||||
# Warnings 彙整
|
||||
# =========================================================================
|
||||
@@ -755,6 +830,22 @@ def report_to_telegram_html(report: HeartbeatReport) -> str:
|
||||
lines.append("🤖 <b>Telegram Bot</b>")
|
||||
lines.append(f"└─ {tg.status}")
|
||||
|
||||
# --- 自動化統計(2026-04-24)---
|
||||
auto = report.automation
|
||||
lines.append("")
|
||||
lines.append("⚙️ <b>自動化統計(今日)</b>")
|
||||
lines.append(f"├─ 新規則自動生成: {auto.auto_rule_generated_today} 條")
|
||||
lines.append(f"├─ KM 新增: {auto.km_created_today} 條")
|
||||
lines.append(f"├─ Drift 自動採納: {auto.drift_adopted_today} 筆")
|
||||
if auto.reject_counts:
|
||||
reject_total = sum(auto.reject_counts.values())
|
||||
top_reason = max(auto.reject_counts, key=auto.reject_counts.get) # type: ignore[arg-type]
|
||||
lines.append(
|
||||
f"└─ 人工審核攔截: {reject_total} 次 主因: <code>{html.escape(top_reason)}</code>"
|
||||
)
|
||||
else:
|
||||
lines.append("└─ 人工審核攔截: 0 次")
|
||||
|
||||
# --- Warnings / 總結 ---
|
||||
lines.append("")
|
||||
if report.warnings:
|
||||
|
||||
@@ -603,7 +603,16 @@ class IntentClassifier:
|
||||
error=str(e),
|
||||
response_preview=result_text[:100],
|
||||
)
|
||||
return self._llm_fallback_result("JSON 解析失敗")
|
||||
# 2026-04-24 ogt + Claude Sonnet 4.6: JSON 解析失敗也降級至關鍵字結果
|
||||
_kw_result = self._keyword_classify(text)
|
||||
return IntentResult(
|
||||
intent=_kw_result.intent,
|
||||
confidence=_kw_result.confidence,
|
||||
method="llm_parse_failed_keyword",
|
||||
matched_keywords=_kw_result.matched_keywords,
|
||||
detected_resources=_kw_result.detected_resources,
|
||||
reasoning=f"LLM JSON 解析失敗降級 → {_kw_result.reasoning}",
|
||||
)
|
||||
|
||||
except httpx.TimeoutException:
|
||||
elapsed_ms = (time.time() - start_time) * 1000
|
||||
@@ -611,7 +620,20 @@ class IntentClassifier:
|
||||
"intent_llm_timeout",
|
||||
elapsed_ms=round(elapsed_ms, 1),
|
||||
)
|
||||
return self._llm_fallback_result("LLM 超時")
|
||||
# 2026-04-24 ogt + Claude Sonnet 4.6: LLM 超時直接降級至關鍵字結果
|
||||
# 問題:_llm_fallback_result 返回 confidence=0.0/UNKNOWN,和 keyword 結果 confidence 相同
|
||||
# classify() 比較 0.0 > 0.0 = False → 走 keyword(正確),但已浪費 5s 超時時間
|
||||
# 若 Ollama 後端不通,每次都等 5s 才降級 → ai_router/ci_auto_repair 延遲累積
|
||||
# 修法:超時直接回 keyword 結果,method 標記 "llm_timeout_keyword" 供可觀測性追蹤
|
||||
_kw_result = self._keyword_classify(text)
|
||||
return IntentResult(
|
||||
intent=_kw_result.intent,
|
||||
confidence=_kw_result.confidence,
|
||||
method="llm_timeout_keyword",
|
||||
matched_keywords=_kw_result.matched_keywords,
|
||||
detected_resources=_kw_result.detected_resources,
|
||||
reasoning=f"LLM 超時降級({round(elapsed_ms, 0):.0f}ms)→ {_kw_result.reasoning}",
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
@@ -619,7 +641,16 @@ class IntentClassifier:
|
||||
error=str(e),
|
||||
error_type=type(e).__name__,
|
||||
)
|
||||
return self._llm_fallback_result(f"LLM 錯誤: {type(e).__name__}")
|
||||
# 2026-04-24 ogt + Claude Sonnet 4.6: LLM 錯誤同樣降級至關鍵字結果
|
||||
_kw_result = self._keyword_classify(text)
|
||||
return IntentResult(
|
||||
intent=_kw_result.intent,
|
||||
confidence=_kw_result.confidence,
|
||||
method="llm_error_keyword",
|
||||
matched_keywords=_kw_result.matched_keywords,
|
||||
detected_resources=_kw_result.detected_resources,
|
||||
reasoning=f"LLM 錯誤降級({type(e).__name__})→ {_kw_result.reasoning}",
|
||||
)
|
||||
|
||||
def _parse_intent_type(self, intent_str: str) -> IntentType:
|
||||
"""解析意圖字串為 IntentType"""
|
||||
|
||||
@@ -286,14 +286,26 @@ async def _merge_similar(playbooks: list[Playbook], report: EvolverReport) -> No
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
async def _fetch_all_active_playbooks() -> list[Playbook]:
|
||||
"""抓取所有非 DEPRECATED 的 Playbook(用於 Evolver 掃描)。"""
|
||||
"""
|
||||
抓取所有非 DEPRECATED 的 Playbook(用於 Evolver 掃描)。
|
||||
|
||||
2026-04-24 ogt + Claude Sonnet 4.6: 改為分兩次 query(DRAFT + APPROVED)
|
||||
原本 list_playbooks(limit=500) 不傳 status → 把 294 筆 deprecated 全載入記憶體,
|
||||
隨 deprecated 累積效能惡化;改用兩次 status 過濾查詢,只拉需要的資料。
|
||||
"""
|
||||
try:
|
||||
from src.models.playbook import PlaybookStatus
|
||||
from src.services.playbook_service import get_playbook_service
|
||||
|
||||
service = get_playbook_service()
|
||||
# 不過濾 status — Evolver 需要看到 DRAFT + APPROVED
|
||||
playbooks_page, total = await service.list_playbooks(limit=500)
|
||||
return playbooks_page
|
||||
|
||||
draft_playbooks, _ = await service.list_playbooks(
|
||||
status=PlaybookStatus.DRAFT, limit=500
|
||||
)
|
||||
approved_playbooks, _ = await service.list_playbooks(
|
||||
status=PlaybookStatus.APPROVED, limit=500
|
||||
)
|
||||
return draft_playbooks + approved_playbooks
|
||||
except Exception as e:
|
||||
logger.warning("evolver_fetch_playbooks_failed", error=str(e))
|
||||
return []
|
||||
|
||||
@@ -45,13 +45,16 @@ async def seed_playbooks_from_rules() -> None:
|
||||
# 取得現有 YAML_RULE playbook,依 name 去重避免重複建立
|
||||
# 2026-04-15 ogt: 不再用 list_playbooks(舊格式 repair_steps 會 validation error)
|
||||
# 改用 raw SQL 只撈 name 欄位,更穩健
|
||||
# 2026-04-24 ogt + Claude Sonnet 4.6: 修復重複建立/封存迴圈
|
||||
# 舊邏輯 status != 'deprecated' 導致:deprecated 歷史記錄永遠存在 →
|
||||
# 每次啟動都重建同名 Playbook(C1 保護後不再封存,但 deprecated 不清除)
|
||||
# 修:不管 status,同名 yaml_rule 任何狀態存在即視為已存在,不重建
|
||||
from src.db.base import get_db_context
|
||||
from sqlalchemy import text as sa_text
|
||||
async with get_db_context() as db:
|
||||
rows = await db.execute(
|
||||
sa_text(
|
||||
"SELECT name FROM playbooks WHERE source = 'yaml_rule'"
|
||||
" AND status != 'deprecated'"
|
||||
)
|
||||
)
|
||||
existing_names = {r[0] for r in rows.fetchall()}
|
||||
|
||||
Reference in New Issue
Block a user