refactor(api): Phase B P1 可靠性強化 (2 項)
Some checks failed
E2E Health Check / e2e-health (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled

1. send_cicd_progress 重試機制 (指數退避 1,2,4 秒)
2. K8s Repository 封裝 (IK8sRepository + K8sRepository)

首席架構師審查 P1 改進 - 模組化合規

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
OG T
2026-03-30 01:52:59 +08:00
parent 64e51bb4fd
commit 13bb1496b0
5 changed files with 395 additions and 116 deletions

View File

@@ -1290,11 +1290,16 @@ class TelegramGateway:
duration_seconds: int = 0,
message: str = "",
workflow_url: str = "",
max_retries: int = 3,
) -> dict:
"""
發送 CI/CD 進度通知 (簡潔版,不走 AI 仲裁)
2026-03-30 ogt: 新增,解決 CI/CD 告警被當成事件處理的問題
2026-03-30 P1: 新增重試機制 (指數退避)
Args:
max_retries: 最大重試次數 (預設 3)
Returns:
dict: Telegram API 回應
@@ -1318,10 +1323,37 @@ class TelegramGateway:
}
logger.info("telegram_cicd_progress_sending", job=job_name, status=status)
result = await self._send_request("sendMessage", payload)
logger.info("telegram_cicd_progress_sent", job=job_name, status=status)
return result
# 重試機制 (指數退避)
import asyncio
last_error = None
for attempt in range(max_retries):
try:
result = await self._send_request("sendMessage", payload)
logger.info("telegram_cicd_progress_sent", job=job_name, status=status, attempt=attempt + 1)
return result
except TelegramGatewayError as e:
last_error = e
if attempt < max_retries - 1:
delay = 2 ** attempt # 1, 2, 4 秒
logger.warning(
"telegram_cicd_progress_retry",
job=job_name,
attempt=attempt + 1,
delay=delay,
error=str(e),
)
await asyncio.sleep(delay)
# 所有重試都失敗
logger.error(
"telegram_cicd_progress_failed",
job=job_name,
status=status,
max_retries=max_retries,
error=str(last_error),
)
raise last_error
async def send_deploy_success(
self,

View File

@@ -34,6 +34,7 @@ from src.models.terminal import (
TerminalSession,
TerminalSessionStatus,
)
from src.repositories.k8s_repository import get_k8s_repository
from src.services.approval_db import get_approval_service
from src.services.openclaw import get_openclaw
@@ -649,6 +650,7 @@ class TerminalService:
處理狀態查詢
Phase 19.4: 整合真實 K8s API
P1 改進: 使用 K8sRepository (積木化)
@author Claude Code
@date 2026-03-30 (台北時間)
"""
@@ -656,54 +658,28 @@ class TerminalService:
await self._publish_tool_call(publisher, topic, "K8s-Status", "executing")
try:
# 使用 K8s Diagnostics Service 的 client
from src.services.k8s_diagnostics import _get_k8s_client
# P1 改進: 使用 K8sRepository 代替直接呼叫 K8s client
k8s_repo = get_k8s_repository()
client = await _get_k8s_client()
if not client:
raise RuntimeError("K8s client initialization failed")
# 檢查 K8s 可用性
if not await k8s_repo.is_available():
raise RuntimeError("K8s API unavailable")
v1 = client.CoreV1Api()
apps_v1 = client.AppsV1Api()
# 查詢 awoooi namespace 的 Pods
# 取得 Pod 狀態摘要
namespace = "awoooi"
pods = await v1.list_namespaced_pod(namespace=namespace)
pod_summary = await k8s_repo.get_pod_status_summary(namespace)
deployments = await k8s_repo.list_deployments(namespace)
# 統計 Pod 狀態
pods_total = len(pods.items)
pods_running = sum(
1 for p in pods.items
if p.status.phase == "Running"
)
pods_ready = sum(
1 for p in pods.items
if p.status.phase == "Running"
and all(
c.ready for c in (p.status.container_statuses or [])
)
)
# 統計
pods_total = pod_summary["total"]
pods_running = pod_summary["running"]
pods_ready = pods_running # running 已包含 ready 檢查
problem_pods = pod_summary["problem_pods"]
# 找出有問題的 Pods
problem_pods = []
for pod in pods.items:
if pod.status.phase != "Running":
problem_pods.append(f"{pod.metadata.name}: {pod.status.phase}")
elif pod.status.container_statuses:
for cs in pod.status.container_statuses:
if not cs.ready:
reason = ""
if cs.state and cs.state.waiting:
reason = cs.state.waiting.reason or ""
problem_pods.append(f"{pod.metadata.name}: {reason or 'NotReady'}")
break
# 查詢 Deployments
deployments = await apps_v1.list_namespaced_deployment(namespace=namespace)
deployments_total = len(deployments.items)
deployments_total = len(deployments)
deployments_healthy = sum(
1 for d in deployments.items
if d.status.ready_replicas == d.spec.replicas
1 for d in deployments
if d["ready_replicas"] == d["replicas"]
)
await self._publish_tool_call(
@@ -716,7 +692,7 @@ class TerminalService:
"deployments_total": deployments_total,
}
)
await asyncio.sleep(0.3)
await asyncio.sleep(SSE_DELAY_SECONDS)
# 組合狀態訊息
status_lines = [
@@ -727,7 +703,7 @@ class TerminalService:
if problem_pods:
status_lines.append(f"\n問題 Pods ({len(problem_pods)}):")
for pp in problem_pods[:3]: # 最多顯示 3 個
status_lines.append(f" - {pp}")
status_lines.append(f" - {pp['name']}: {pp['phase']}")
if len(problem_pods) > 3:
status_lines.append(f" ... 還有 {len(problem_pods) - 3}")
else:
@@ -742,11 +718,11 @@ class TerminalService:
logger.error("k8s_status_query_failed", error=str(e))
await self._publish_tool_call(
publisher, topic, "K8s-Status", "failed",
result={"error": str(e)[:100]}
result={"error": sanitize_error_message(str(e))}
)
await self._publish_thought(
publisher, topic, "System",
f"K8s 狀態查詢失敗: {str(e)[:50]}\n"
f"K8s 狀態查詢失敗: {sanitize_error_message(str(e), ERROR_MESSAGE_DISPLAY_LENGTH)}\n"
"請確認 K8s 連線配置。"
)