feat(p3.1-t2-patha): DiagnosisAggregator 路徑 A + Solver F4 critical reject + 對齊測試
Some checks failed
CD Pipeline / build-and-deploy (push) Failing after 1m59s
Some checks failed
CD Pipeline / build-and-deploy (push) Failing after 1m59s
Wave 8 P3.1-T2 PathA 啟用 + Solver F4 安全強化 + test 對齊:
PathA — DiagnosisAggregator 信號分類層補 PDI:
- ENABLE_DIAGNOSIS_AGGREGATOR default=False → True
· PathA 純信號分類層(OOMKilled/CrashLoop 等業務邏輯)
· 不重複呼叫 K8s/SignOz API(只取 PDI 已收集的 raw 資料)
· 安全 default on — 純邏輯處理,無外部依賴重疊
- diagnosis_aggregator.py +155 行(PathA 實作)
- pre_decision_investigator.py 已接 (commit 3a2cd151)
F4 — Solver critical risk reject:
- solver_agent.py: _validate_recommended_action 拒絕 risk=critical
· 鐵律:critical 動作必須走人工審批,不可變 Telegram 按鈕
· log warning + return None(被 _extract 過濾掉)
- _extract_recommended_actions 改返回 (list, status_str) tuple
· status="ok"/"empty"/"all_invalid" 供呼叫端決策
- protocol.py +16 / metrics.py +9 / ai_router.py +18 — 配套 metric + protocol field
測試對齊:
- test_solver_recommended_actions.py 拆 test_all_valid → low/medium/high accepted +
test_critical_rejected
- result tuple unpack: result, _ = _extract_recommended_actions(...)
- test_diagnosis_aggregator_stub.py: feature flag default 改 True 對齊 PathA
Tests: 51 passed (solver 28 + aggregator 16 + router fallback 8)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-Authored-By: Multiple Engineers (Wave 8 P3.1-T2 PathA + F4) <noreply@anthropic.com>
This commit is contained in:
@@ -1094,6 +1094,10 @@ class AIRouterExecutor:
|
||||
for provider_name in provider_order:
|
||||
# 2026-04-27 Claude Sonnet 4.6: A2 — 若上一輪失敗且本輪開始,表示發生 fallback
|
||||
# 記錄 metric(DIAGNOSE intent 專屬;非 DIAGNOSE 不記,不影響其他路徑)
|
||||
# 2026-04-27 Claude Sonnet 4.6: F6 — fallback metric 只在真實 analyze() 失敗時觸發
|
||||
# _last_attempted_provider 僅在 provider.analyze() 執行後失敗才賦值(見下方兩處);
|
||||
# not_registered / privacy_skip / circuit_open / rate_limit 分支均不賦值,
|
||||
# 避免這些「被跳過的 provider」誤計入 from→to fallback 鏈,metric 不可信問題(F6)。
|
||||
if _is_diagnose_intent and _last_attempted_provider is not None:
|
||||
try:
|
||||
from src.core.metrics import record_diagnose_fallback
|
||||
@@ -1107,16 +1111,24 @@ class AIRouterExecutor:
|
||||
to_provider=provider_name,
|
||||
)
|
||||
except Exception as _metric_e:
|
||||
logger.debug("diagnose_fallback_metric_failed", error=str(_metric_e))
|
||||
# 2026-04-27 Claude Sonnet 4.6: F6 — 升 warning(原 debug 會 silent swallow)
|
||||
# + 計入 error counter 讓 metric 管道問題可被 Prometheus 偵測
|
||||
logger.warning("diagnose_fallback_metric_failed", error=str(_metric_e))
|
||||
try:
|
||||
from src.core.metrics import AIOPS_DIAGNOSE_FALLBACK_METRIC_ERROR_TOTAL
|
||||
AIOPS_DIAGNOSE_FALLBACK_METRIC_ERROR_TOTAL.inc()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
provider = self._registry.get(provider_name)
|
||||
if not provider:
|
||||
# 2026-04-14 Claude Sonnet 4.6: silent skip 改 errors 累積(觀測性)
|
||||
# 2026-04-27 Claude Sonnet 4.6: F6 — 不設 _last_attempted_provider(未真實執行 analyze)
|
||||
errors.append(f"{provider_name}: not_registered")
|
||||
_last_attempted_provider = provider_name
|
||||
continue
|
||||
|
||||
# 隱私過濾 (D7)
|
||||
# 2026-04-27 Claude Sonnet 4.6: F6 — privacy_skip 不設 _last_attempted_provider(未嘗試)
|
||||
if require_local and provider.privacy_level != "local":
|
||||
errors.append(f"{provider_name}: privacy_skip(non_local)")
|
||||
continue
|
||||
@@ -1126,7 +1138,7 @@ class AIRouterExecutor:
|
||||
if cb.is_open():
|
||||
errors.append(f"{provider_name}: circuit_open")
|
||||
logger.warning("ai_router_circuit_open", provider=provider_name)
|
||||
_last_attempted_provider = provider_name
|
||||
# 2026-04-27 Claude Sonnet 4.6: F6 — circuit_open 不設 _last_attempted_provider(未嘗試)
|
||||
continue
|
||||
|
||||
# 閘門 2: Rate Limiter
|
||||
|
||||
@@ -556,6 +556,161 @@ class DiagnosisAggregator:
|
||||
},
|
||||
))
|
||||
|
||||
def classify_signals_from_raw(
|
||||
self,
|
||||
k8s_data: dict | None = None,
|
||||
logs_data: str | None = None,
|
||||
metrics_data: dict | None = None,
|
||||
) -> list[DiagnosisSignal]:
|
||||
"""
|
||||
2026-04-27 P3.1-T2-PathA by Claude — DiagAggregator 信號分類層補 PDI
|
||||
|
||||
純邏輯信號分類:接受 PDI 已收集的 raw 資料做業務邏輯分類,
|
||||
不打外部 API(K8s/SignOz),不重複收集。
|
||||
|
||||
Args:
|
||||
k8s_data: EvidenceSnapshot.k8s_state(D1,PDI 已收集的 dict)
|
||||
logs_data: EvidenceSnapshot.recent_logs(D2,sanitized string)
|
||||
metrics_data: EvidenceSnapshot.metrics_snapshot(D3,PDI 已收集的 dict)
|
||||
|
||||
Returns:
|
||||
list[DiagnosisSignal]: 分類後的信號清單(空清單代表無異常)
|
||||
"""
|
||||
# 組裝暫時 context 供 _analyze_signals 使用(不觸發任何 IO)
|
||||
ctx = DiagnosisContext(target="_classify_only")
|
||||
|
||||
# D1: k8s_state dict → 嘗試映射為 K8sDiagnostics(只提取可分類欄位)
|
||||
if k8s_data and isinstance(k8s_data, dict):
|
||||
# 利用 K8sDiagnostics.from_dict(若存在)或直接從常見欄位提取信號
|
||||
# 不依賴 K8sDiagnostics.from_dict(避免 import coupling),
|
||||
# 改從 k8s_data 中提取已知信號模式
|
||||
self._classify_k8s_dict_signals(ctx, k8s_data)
|
||||
|
||||
# D3: metrics_snapshot → GoldMetrics-like 分析
|
||||
if metrics_data and isinstance(metrics_data, dict):
|
||||
self._classify_metrics_dict_signals(ctx, metrics_data)
|
||||
|
||||
# D2: logs string → 錯誤計數分類
|
||||
if logs_data and isinstance(logs_data, str):
|
||||
self._classify_log_string_signals(ctx, logs_data)
|
||||
|
||||
return ctx.signals
|
||||
|
||||
def _classify_k8s_dict_signals(
|
||||
self,
|
||||
context: DiagnosisContext,
|
||||
k8s_data: dict,
|
||||
) -> None:
|
||||
"""
|
||||
2026-04-27 P3.1-T2-PathA by Claude — 從 PDI k8s_state dict 提取信號
|
||||
不依賴 K8sDiagnostics 物件,直接從 dict 關鍵字段分類。
|
||||
"""
|
||||
phase = str(k8s_data.get("phase", "")).lower()
|
||||
reason = str(k8s_data.get("reason", "")).lower()
|
||||
restart_count = k8s_data.get("restart_count", 0) or 0
|
||||
|
||||
# CrashLoopBackOff
|
||||
if "crashloop" in phase or "crashloopbackoff" in reason:
|
||||
context.signals.append(DiagnosisSignal(
|
||||
source="k8s_state",
|
||||
signal_type="crash_loop",
|
||||
severity=DiagnosisSeverity.CRITICAL,
|
||||
message=f"CrashLoopBackOff detected (phase={k8s_data.get('phase', '?')})",
|
||||
evidence={"phase": k8s_data.get("phase"), "reason": k8s_data.get("reason")},
|
||||
))
|
||||
|
||||
# OOMKilled
|
||||
if "oomkilled" in phase or "oomkilled" in reason or "oom" in reason:
|
||||
context.signals.append(DiagnosisSignal(
|
||||
source="k8s_state",
|
||||
signal_type="oom_killed",
|
||||
severity=DiagnosisSeverity.CRITICAL,
|
||||
message=f"OOMKilled detected (reason={k8s_data.get('reason', '?')})",
|
||||
evidence={"phase": k8s_data.get("phase"), "reason": k8s_data.get("reason")},
|
||||
))
|
||||
|
||||
# Image pull error
|
||||
if "imagepullerr" in reason or "errimagepull" in reason or "imagepullbackoff" in reason:
|
||||
context.signals.append(DiagnosisSignal(
|
||||
source="k8s_state",
|
||||
signal_type="image_pull_error",
|
||||
severity=DiagnosisSeverity.HIGH,
|
||||
message=f"Image pull error (reason={k8s_data.get('reason', '?')})",
|
||||
evidence={"reason": k8s_data.get("reason")},
|
||||
))
|
||||
|
||||
# High restart count
|
||||
try:
|
||||
rc = int(restart_count)
|
||||
except (TypeError, ValueError):
|
||||
rc = 0
|
||||
if rc > 5:
|
||||
context.signals.append(DiagnosisSignal(
|
||||
source="k8s_state",
|
||||
signal_type="high_restart_count",
|
||||
severity=DiagnosisSeverity.MEDIUM,
|
||||
message=f"High restart count: {rc}",
|
||||
evidence={"restart_count": rc},
|
||||
))
|
||||
|
||||
def _classify_metrics_dict_signals(
|
||||
self,
|
||||
context: DiagnosisContext,
|
||||
metrics_data: dict,
|
||||
) -> None:
|
||||
"""
|
||||
2026-04-27 P3.1-T2-PathA by Claude — 從 PDI metrics_snapshot dict 提取信號
|
||||
"""
|
||||
try:
|
||||
error_rate = float(metrics_data.get("error_rate", 0) or 0)
|
||||
except (TypeError, ValueError):
|
||||
error_rate = 0.0
|
||||
|
||||
try:
|
||||
p99_ms = float(metrics_data.get("p99_latency_ms", 0) or 0)
|
||||
except (TypeError, ValueError):
|
||||
p99_ms = 0.0
|
||||
|
||||
if error_rate > 5:
|
||||
context.signals.append(DiagnosisSignal(
|
||||
source="metrics_snapshot",
|
||||
signal_type="high_error_rate",
|
||||
severity=DiagnosisSeverity.CRITICAL if error_rate > 20 else DiagnosisSeverity.HIGH,
|
||||
message=f"High error rate: {error_rate:.2f}%",
|
||||
evidence={"error_rate": error_rate},
|
||||
))
|
||||
|
||||
if p99_ms > 5000:
|
||||
context.signals.append(DiagnosisSignal(
|
||||
source="metrics_snapshot",
|
||||
signal_type="high_latency",
|
||||
severity=DiagnosisSeverity.HIGH if p99_ms >= 10000 else DiagnosisSeverity.MEDIUM,
|
||||
message=f"High P99 latency: {p99_ms:.0f}ms",
|
||||
evidence={"p99_latency_ms": p99_ms},
|
||||
))
|
||||
|
||||
def _classify_log_string_signals(
|
||||
self,
|
||||
context: DiagnosisContext,
|
||||
logs_data: str,
|
||||
) -> None:
|
||||
"""
|
||||
2026-04-27 P3.1-T2-PathA by Claude — 從 PDI recent_logs string 提取信號
|
||||
"""
|
||||
# 簡單計數 ERROR/FATAL 行
|
||||
error_lines = [
|
||||
line for line in logs_data.splitlines()
|
||||
if any(kw in line.upper() for kw in ("ERROR", "FATAL", "CRITICAL", "EXCEPTION", "TRACEBACK"))
|
||||
]
|
||||
if len(error_lines) > 10:
|
||||
context.signals.append(DiagnosisSignal(
|
||||
source="recent_logs",
|
||||
signal_type="frequent_errors",
|
||||
severity=DiagnosisSeverity.HIGH if len(error_lines) >= 50 else DiagnosisSeverity.MEDIUM,
|
||||
message=f"Frequent error lines in logs: {len(error_lines)}",
|
||||
evidence={"error_line_count": len(error_lines), "sample": error_lines[:3]},
|
||||
))
|
||||
|
||||
# =========================================================================
|
||||
# Utilities
|
||||
# =========================================================================
|
||||
|
||||
Reference in New Issue
Block a user