fix(aiops-p0): 六大病根 P0 全面修復（ADR-092 B4）
【P0.1】knowledge_extractor_service.py:210 — AttributeError 修復 - Signal.description 欄位不存在（100% 失敗，KM 每天+5 根因） - 改用 alert_name + annotations.summary 拼接文字 【P0.2+P0.3】Gate 9+11 唯讀指令鬆綁 - blast_radius_calculator: kubectl get/top/describe/logs/version → score=1（非 50） - operation_parser: 增加 INVESTIGATE 類型識別（唯讀 kubectl 不回 None） - executor.py: OperationType 新增 INVESTIGATE enum - approval_execution.py: INVESTIGATE 路徑直接呼叫 execute_kubectl_command 【P0.4】MCP SSH/K8s Provider 修復 - decision_manager: params= → parameters=（符合 MCPToolProvider.execute 簽名） - decision_manager: MCPToolResult .get() → .success/.output（dataclass 用法） - decision_manager + ssh_provider: 補入 hosts 120/121（原 default 缺失） - auto_approve: phase2_agent_debate source bypass confidence 閾值 【P0.5】告警規則語義矛盾修復 - alert_rules.yaml: 8 條 kubectl 查詢規則 RESTART_DEPLOYMENT → NO_ACTION (CrashLoopBackOff/PostgreSQL 連線/慢查詢/MinIO 磁碟/K3s 節點/告警鏈路/SSL/CoreDNS 等) - incident_service.py: cAdvisor/CoreDNS 從 general 拆出獨立分類 【P0.6】proactive_inspector 動態基線 PromQL 全修 - 5 個 MONITORED_METRICS PromQL 全部修正（cadvisor label/datname/blackbox） - db_connection_pool: datname="awoooi" → "awoooi_prod" - http_error_rate: 無效 http_requests_total → blackbox probe_success - cpu/memory: namespace label → name=~"k8s_api_awoooi-api.*" Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Your Name
@@ -32,7 +32,7 @@ import structlog
|
||||
from src.core.config import settings
|
||||
from src.models.approval import ApprovalRequest
|
||||
from src.services.approval_db import get_approval_service, get_timeline_service
|
||||
from src.services.executor import get_executor
|
||||
from src.services.executor import OperationType, get_executor
|
||||
from src.services.operation_parser import parse_operation_from_action
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -242,52 +242,71 @@ class ApprovalExecutionService:
|
||||
)
|
||||
return False # 解析失敗 → 執行未發生
|
||||
|
||||
# ADR-076 Task 3: 執行失敗重試機制
|
||||
# 瞬態錯誤 (connection refused, timeout 等) 自動重試，最多 MAX_RETRY 次
|
||||
executor = get_executor()
|
||||
result = await executor.execute_with_audit(
|
||||
approval=approval,
|
||||
operation_type=operation_type,
|
||||
resource_name=resource_name,
|
||||
namespace=namespace,
|
||||
)
|
||||
attempt = 1 # 重試計數（INVESTIGATE 路徑不進入重試迴圈，保持 1）
|
||||
|
||||
attempt = 1
|
||||
while not result.success and attempt <= self.MAX_RETRY:
|
||||
if not self._is_transient_error(result.error):
|
||||
logger.info(
|
||||
"execution_retry_skipped_permanent_error",
|
||||
approval_id=str(approval.id),
|
||||
attempt=attempt,
|
||||
error=result.error,
|
||||
)
|
||||
break
|
||||
|
||||
logger.warning(
|
||||
"execution_retry_transient_error",
|
||||
approval_id=str(approval.id),
|
||||
attempt=attempt,
|
||||
max_retry=self.MAX_RETRY,
|
||||
error=result.error,
|
||||
delay_seconds=self.RETRY_DELAY_SECONDS,
|
||||
# 2026-04-24 ogt + Claude Sonnet 4.6: Gate 11 修復 — INVESTIGATE 唯讀查詢
|
||||
# 根因：INVESTIGATE 不在 executor.execute_with_audit 的 switch，走 else → success=False
|
||||
# 修法：偵測到 INVESTIGATE 類型，直接呼叫 execute_kubectl_command(approval.action)
|
||||
# 唯讀指令無需重試迴圈（失敗即失敗，不會有 transient error 改善空間）
|
||||
if operation_type == OperationType.INVESTIGATE:
|
||||
result = await executor.execute_kubectl_command(
|
||||
command=approval.action,
|
||||
timeout_sec=30,
|
||||
)
|
||||
await timeline.add_event(
|
||||
event_type="exec",
|
||||
status="warning",
|
||||
title=f"⚠️ 執行失敗，{self.RETRY_DELAY_SECONDS}s 後重試 ({attempt}/{self.MAX_RETRY})",
|
||||
description=f"Error: {result.error}",
|
||||
actor="leWOOOgo",
|
||||
actor_role="executor",
|
||||
logger.info(
|
||||
"background_execution_investigate",
|
||||
approval_id=str(approval.id),
|
||||
action=approval.action,
|
||||
success=result.success,
|
||||
message=result.message,
|
||||
)
|
||||
await asyncio.sleep(self.RETRY_DELAY_SECONDS)
|
||||
else:
|
||||
# ADR-076 Task 3: 執行失敗重試機制
|
||||
# 瞬態錯誤 (connection refused, timeout 等) 自動重試，最多 MAX_RETRY 次
|
||||
result = await executor.execute_with_audit(
|
||||
approval=approval,
|
||||
operation_type=operation_type,
|
||||
resource_name=resource_name,
|
||||
namespace=namespace,
|
||||
)
|
||||
attempt += 1
|
||||
|
||||
attempt = 1
|
||||
while not result.success and attempt <= self.MAX_RETRY:
|
||||
if not self._is_transient_error(result.error):
|
||||
logger.info(
|
||||
"execution_retry_skipped_permanent_error",
|
||||
approval_id=str(approval.id),
|
||||
attempt=attempt,
|
||||
error=result.error,
|
||||
)
|
||||
break
|
||||
|
||||
logger.warning(
|
||||
"execution_retry_transient_error",
|
||||
approval_id=str(approval.id),
|
||||
attempt=attempt,
|
||||
max_retry=self.MAX_RETRY,
|
||||
error=result.error,
|
||||
delay_seconds=self.RETRY_DELAY_SECONDS,
|
||||
)
|
||||
await timeline.add_event(
|
||||
event_type="exec",
|
||||
status="warning",
|
||||
title=f"⚠️ 執行失敗，{self.RETRY_DELAY_SECONDS}s 後重試 ({attempt}/{self.MAX_RETRY})",
|
||||
description=f"Error: {result.error}",
|
||||
actor="leWOOOgo",
|
||||
actor_role="executor",
|
||||
approval_id=str(approval.id),
|
||||
)
|
||||
await asyncio.sleep(self.RETRY_DELAY_SECONDS)
|
||||
result = await executor.execute_with_audit(
|
||||
approval=approval,
|
||||
operation_type=operation_type,
|
||||
resource_name=resource_name,
|
||||
namespace=namespace,
|
||||
)
|
||||
attempt += 1
|
||||
|
||||
# Phase 5: 更新資料庫狀態
|
||||
# 2026-04-18 ADR-090 L5 P0.2: 失敗時帶上 error_message,寫進 rejection_reason
|
||||
|
||||
Reference in New Issue
Block a user