feat(p3.1-t2-patha): DiagnosisAggregator 路徑 A + Solver F4 critical reject + 對齊測試
Some checks failed
CD Pipeline / build-and-deploy (push) Failing after 1m59s

Wave 8 P3.1-T2 PathA 啟用 + Solver F4 安全強化 + test 對齊:

PathA — DiagnosisAggregator 信號分類層補 PDI:
- ENABLE_DIAGNOSIS_AGGREGATOR default=False → True
  · PathA 純信號分類層(OOMKilled/CrashLoop 等業務邏輯)
  · 不重複呼叫 K8s/SignOz API(只取 PDI 已收集的 raw 資料)
  · 安全 default on — 純邏輯處理,無外部依賴重疊
- diagnosis_aggregator.py +155 行(PathA 實作)
- pre_decision_investigator.py 已接 (commit 3a2cd151)

F4 — Solver critical risk reject:
- solver_agent.py: _validate_recommended_action 拒絕 risk=critical
  · 鐵律:critical 動作必須走人工審批,不可變 Telegram 按鈕
  · log warning + return None(被 _extract 過濾掉)
- _extract_recommended_actions 改返回 (list, status_str) tuple
  · status="ok"/"empty"/"all_invalid" 供呼叫端決策
- protocol.py +16 / metrics.py +9 / ai_router.py +18 — 配套 metric + protocol field

測試對齊:
- test_solver_recommended_actions.py 拆 test_all_valid → low/medium/high accepted +
  test_critical_rejected
- result tuple unpack: result, _ = _extract_recommended_actions(...)
- test_diagnosis_aggregator_stub.py: feature flag default 改 True 對齊 PathA

Tests: 51 passed (solver 28 + aggregator 16 + router fallback 8)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-Authored-By: Multiple Engineers (Wave 8 P3.1-T2 PathA + F4) <noreply@anthropic.com>
This commit is contained in:
Your Name
2026-04-27 14:42:29 +08:00
parent f09a8f56a9
commit 9908fdf50d
8 changed files with 566 additions and 126 deletions

View File

@@ -1094,6 +1094,10 @@ class AIRouterExecutor:
for provider_name in provider_order:
# 2026-04-27 Claude Sonnet 4.6: A2 — 若上一輪失敗且本輪開始,表示發生 fallback
# 記錄 metricDIAGNOSE intent 專屬;非 DIAGNOSE 不記,不影響其他路徑)
# 2026-04-27 Claude Sonnet 4.6: F6 — fallback metric 只在真實 analyze() 失敗時觸發
# _last_attempted_provider 僅在 provider.analyze() 執行後失敗才賦值(見下方兩處);
# not_registered / privacy_skip / circuit_open / rate_limit 分支均不賦值,
# 避免這些「被跳過的 provider」誤計入 from→to fallback 鏈metric 不可信問題F6
if _is_diagnose_intent and _last_attempted_provider is not None:
try:
from src.core.metrics import record_diagnose_fallback
@@ -1107,16 +1111,24 @@ class AIRouterExecutor:
to_provider=provider_name,
)
except Exception as _metric_e:
logger.debug("diagnose_fallback_metric_failed", error=str(_metric_e))
# 2026-04-27 Claude Sonnet 4.6: F6 — 升 warning原 debug 會 silent swallow
# + 計入 error counter 讓 metric 管道問題可被 Prometheus 偵測
logger.warning("diagnose_fallback_metric_failed", error=str(_metric_e))
try:
from src.core.metrics import AIOPS_DIAGNOSE_FALLBACK_METRIC_ERROR_TOTAL
AIOPS_DIAGNOSE_FALLBACK_METRIC_ERROR_TOTAL.inc()
except Exception:
pass
provider = self._registry.get(provider_name)
if not provider:
# 2026-04-14 Claude Sonnet 4.6: silent skip 改 errors 累積(觀測性)
# 2026-04-27 Claude Sonnet 4.6: F6 — 不設 _last_attempted_provider未真實執行 analyze
errors.append(f"{provider_name}: not_registered")
_last_attempted_provider = provider_name
continue
# 隱私過濾 (D7)
# 2026-04-27 Claude Sonnet 4.6: F6 — privacy_skip 不設 _last_attempted_provider未嘗試
if require_local and provider.privacy_level != "local":
errors.append(f"{provider_name}: privacy_skip(non_local)")
continue
@@ -1126,7 +1138,7 @@ class AIRouterExecutor:
if cb.is_open():
errors.append(f"{provider_name}: circuit_open")
logger.warning("ai_router_circuit_open", provider=provider_name)
_last_attempted_provider = provider_name
# 2026-04-27 Claude Sonnet 4.6: F6 — circuit_open 不設 _last_attempted_provider(未嘗試)
continue
# 閘門 2: Rate Limiter

View File

@@ -556,6 +556,161 @@ class DiagnosisAggregator:
},
))
def classify_signals_from_raw(
self,
k8s_data: dict | None = None,
logs_data: str | None = None,
metrics_data: dict | None = None,
) -> list[DiagnosisSignal]:
"""
2026-04-27 P3.1-T2-PathA by Claude — DiagAggregator 信號分類層補 PDI
純邏輯信號分類:接受 PDI 已收集的 raw 資料做業務邏輯分類,
不打外部 APIK8s/SignOz不重複收集。
Args:
k8s_data: EvidenceSnapshot.k8s_stateD1PDI 已收集的 dict
logs_data: EvidenceSnapshot.recent_logsD2sanitized string
metrics_data: EvidenceSnapshot.metrics_snapshotD3PDI 已收集的 dict
Returns:
list[DiagnosisSignal]: 分類後的信號清單(空清單代表無異常)
"""
# 組裝暫時 context 供 _analyze_signals 使用(不觸發任何 IO
ctx = DiagnosisContext(target="_classify_only")
# D1: k8s_state dict → 嘗試映射為 K8sDiagnostics只提取可分類欄位
if k8s_data and isinstance(k8s_data, dict):
# 利用 K8sDiagnostics.from_dict若存在或直接從常見欄位提取信號
# 不依賴 K8sDiagnostics.from_dict避免 import coupling
# 改從 k8s_data 中提取已知信號模式
self._classify_k8s_dict_signals(ctx, k8s_data)
# D3: metrics_snapshot → GoldMetrics-like 分析
if metrics_data and isinstance(metrics_data, dict):
self._classify_metrics_dict_signals(ctx, metrics_data)
# D2: logs string → 錯誤計數分類
if logs_data and isinstance(logs_data, str):
self._classify_log_string_signals(ctx, logs_data)
return ctx.signals
def _classify_k8s_dict_signals(
self,
context: DiagnosisContext,
k8s_data: dict,
) -> None:
"""
2026-04-27 P3.1-T2-PathA by Claude — 從 PDI k8s_state dict 提取信號
不依賴 K8sDiagnostics 物件,直接從 dict 關鍵字段分類。
"""
phase = str(k8s_data.get("phase", "")).lower()
reason = str(k8s_data.get("reason", "")).lower()
restart_count = k8s_data.get("restart_count", 0) or 0
# CrashLoopBackOff
if "crashloop" in phase or "crashloopbackoff" in reason:
context.signals.append(DiagnosisSignal(
source="k8s_state",
signal_type="crash_loop",
severity=DiagnosisSeverity.CRITICAL,
message=f"CrashLoopBackOff detected (phase={k8s_data.get('phase', '?')})",
evidence={"phase": k8s_data.get("phase"), "reason": k8s_data.get("reason")},
))
# OOMKilled
if "oomkilled" in phase or "oomkilled" in reason or "oom" in reason:
context.signals.append(DiagnosisSignal(
source="k8s_state",
signal_type="oom_killed",
severity=DiagnosisSeverity.CRITICAL,
message=f"OOMKilled detected (reason={k8s_data.get('reason', '?')})",
evidence={"phase": k8s_data.get("phase"), "reason": k8s_data.get("reason")},
))
# Image pull error
if "imagepullerr" in reason or "errimagepull" in reason or "imagepullbackoff" in reason:
context.signals.append(DiagnosisSignal(
source="k8s_state",
signal_type="image_pull_error",
severity=DiagnosisSeverity.HIGH,
message=f"Image pull error (reason={k8s_data.get('reason', '?')})",
evidence={"reason": k8s_data.get("reason")},
))
# High restart count
try:
rc = int(restart_count)
except (TypeError, ValueError):
rc = 0
if rc > 5:
context.signals.append(DiagnosisSignal(
source="k8s_state",
signal_type="high_restart_count",
severity=DiagnosisSeverity.MEDIUM,
message=f"High restart count: {rc}",
evidence={"restart_count": rc},
))
def _classify_metrics_dict_signals(
self,
context: DiagnosisContext,
metrics_data: dict,
) -> None:
"""
2026-04-27 P3.1-T2-PathA by Claude — 從 PDI metrics_snapshot dict 提取信號
"""
try:
error_rate = float(metrics_data.get("error_rate", 0) or 0)
except (TypeError, ValueError):
error_rate = 0.0
try:
p99_ms = float(metrics_data.get("p99_latency_ms", 0) or 0)
except (TypeError, ValueError):
p99_ms = 0.0
if error_rate > 5:
context.signals.append(DiagnosisSignal(
source="metrics_snapshot",
signal_type="high_error_rate",
severity=DiagnosisSeverity.CRITICAL if error_rate > 20 else DiagnosisSeverity.HIGH,
message=f"High error rate: {error_rate:.2f}%",
evidence={"error_rate": error_rate},
))
if p99_ms > 5000:
context.signals.append(DiagnosisSignal(
source="metrics_snapshot",
signal_type="high_latency",
severity=DiagnosisSeverity.HIGH if p99_ms >= 10000 else DiagnosisSeverity.MEDIUM,
message=f"High P99 latency: {p99_ms:.0f}ms",
evidence={"p99_latency_ms": p99_ms},
))
def _classify_log_string_signals(
self,
context: DiagnosisContext,
logs_data: str,
) -> None:
"""
2026-04-27 P3.1-T2-PathA by Claude — 從 PDI recent_logs string 提取信號
"""
# 簡單計數 ERROR/FATAL 行
error_lines = [
line for line in logs_data.splitlines()
if any(kw in line.upper() for kw in ("ERROR", "FATAL", "CRITICAL", "EXCEPTION", "TRACEBACK"))
]
if len(error_lines) > 10:
context.signals.append(DiagnosisSignal(
source="recent_logs",
signal_type="frequent_errors",
severity=DiagnosisSeverity.HIGH if len(error_lines) >= 50 else DiagnosisSeverity.MEDIUM,
message=f"Frequent error lines in logs: {len(error_lines)}",
evidence={"error_line_count": len(error_lines), "sample": error_lines[:3]},
))
# =========================================================================
# Utilities
# =========================================================================