179e659f14
清理工作: - .gitignore 新增 playwright-report/ 和 test-results/ 排除 - 保留 phase19/ 參考截圖目錄 kube-state-metrics 告警擴充 (P3): - CronJobLastRunFailed: Job 執行失敗 - DaemonSetMissingPods: DaemonSet 缺少 Pod - StatefulSetReplicasMismatch: StatefulSet 副本不足 - ContainerWaiting: ImagePullBackOff/CrashLoopBackOff 偵測 - PDBViolation: PDB 健康 Pod 數不足 - NodeUnschedulable: 節點標記為不可排程 新增: - apps/api/scripts/test_nemotron_tool_calling.py (E2E 比較測試) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
303 lines
11 KiB
YAML
303 lines
11 KiB
YAML
# =============================================================================
|
||
# K3s Supplemental Alerts - 補充告警規則
|
||
# =============================================================================
|
||
# 首席架構師審查 2026-03-28: 補齊缺失的監控告警
|
||
# 部署位置: 192.168.0.188 Prometheus/Alertmanager
|
||
# =============================================================================
|
||
|
||
groups:
|
||
# ===== 基礎設施服務告警 =====
|
||
- name: infrastructure_services
|
||
rules:
|
||
# Redis 連線失敗
|
||
- alert: RedisDown
|
||
expr: probe_success{job="blackbox-tcp", instance="192.168.0.188:6380"} == 0
|
||
for: 1m
|
||
labels:
|
||
severity: critical
|
||
team: ops
|
||
component: redis
|
||
annotations:
|
||
summary: "🔴 Redis 服務離線"
|
||
description: "Redis Stack (192.168.0.188:6380) 已離線超過 1 分鐘"
|
||
|
||
# PostgreSQL 連線失敗
|
||
- alert: PostgreSQLDown
|
||
expr: probe_success{job="blackbox-tcp", instance="192.168.0.188:5432"} == 0
|
||
for: 1m
|
||
labels:
|
||
severity: critical
|
||
team: ops
|
||
component: postgresql
|
||
annotations:
|
||
summary: "🔴 PostgreSQL 服務離線"
|
||
description: "PostgreSQL (192.168.0.188:5432) 已離線超過 1 分鐘"
|
||
|
||
# Ollama AI 服務離線
|
||
- alert: OllamaDown
|
||
expr: probe_success{job="blackbox-tcp", instance="192.168.0.188:11434"} == 0
|
||
for: 2m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: ollama
|
||
annotations:
|
||
summary: "⚠️ Ollama AI 服務離線"
|
||
description: "Ollama (192.168.0.188:11434) 已離線超過 2 分鐘，AI 功能降級"
|
||
|
||
# OpenClaw 服務離線
|
||
- alert: OpenClawDown
|
||
expr: probe_success{job="blackbox-tcp", instance="192.168.0.188:8089"} == 0
|
||
for: 2m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: openclaw
|
||
annotations:
|
||
summary: "⚠️ OpenClaw 服務離線"
|
||
description: "OpenClaw (192.168.0.188:8089) 已離線超過 2 分鐘"
|
||
|
||
# Harbor Registry 離線
|
||
- alert: HarborDown
|
||
expr: probe_success{job="blackbox-tcp", instance="192.168.0.110:5000"} == 0
|
||
for: 2m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: harbor
|
||
annotations:
|
||
summary: "⚠️ Harbor Registry 離線"
|
||
description: "Harbor (192.168.0.110:5000) 已離線超過 2 分鐘，影響 CI/CD"
|
||
|
||
# Langfuse LLMOps 離線
|
||
- alert: LangfuseDown
|
||
expr: probe_success{job="blackbox-tcp", instance="192.168.0.110:3100"} == 0
|
||
for: 2m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: langfuse
|
||
annotations:
|
||
summary: "⚠️ Langfuse LLMOps 離線"
|
||
description: "Langfuse (192.168.0.110:3100) 已離線超過 2 分鐘"
|
||
|
||
# ===== HPA 告警 =====
|
||
- name: hpa_alerts
|
||
rules:
|
||
# HPA 達到最大副本數
|
||
- alert: HPAMaxedOut
|
||
expr: kube_horizontalpodautoscaler_status_current_replicas{namespace="awoooi-prod"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace="awoooi-prod"}
|
||
for: 10m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: hpa
|
||
annotations:
|
||
summary: "⚠️ HPA 達到最大副本數"
|
||
description: "{{ $labels.horizontalpodautoscaler }} 已達到最大 {{ $value }} 副本"
|
||
|
||
# HPA 無法獲取指標
|
||
- alert: HPAScalingDisabled
|
||
expr: kube_horizontalpodautoscaler_status_condition{condition="ScalingActive", status="false", namespace="awoooi-prod"} == 1
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: hpa
|
||
annotations:
|
||
summary: "⚠️ HPA 擴展停用"
|
||
description: "{{ $labels.horizontalpodautoscaler }} 無法擴展 (可能指標不可用)"
|
||
|
||
# ===== Worker 告警 =====
|
||
- name: worker_alerts
|
||
rules:
|
||
# Worker Pod 不可用
|
||
- alert: WorkerUnavailable
|
||
expr: kube_deployment_status_replicas_available{deployment="awoooi-worker", namespace="awoooi-prod"} == 0
|
||
for: 2m
|
||
labels:
|
||
severity: critical
|
||
team: ops
|
||
component: worker
|
||
annotations:
|
||
summary: "🔴 Signal Worker 不可用"
|
||
description: "awoooi-worker 沒有可用副本，Redis Streams 訊號無法處理"
|
||
|
||
# ===== ArgoCD 告警 =====
|
||
- name: argocd_alerts
|
||
rules:
|
||
# ArgoCD Application 同步失敗
|
||
- alert: ArgoCDSyncFailed
|
||
expr: argocd_app_info{sync_status!="Synced", namespace="argocd"} == 1
|
||
for: 10m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: argocd
|
||
annotations:
|
||
summary: "⚠️ ArgoCD 同步失敗"
|
||
description: "Application {{ $labels.name }} 同步狀態: {{ $labels.sync_status }}"
|
||
|
||
# ===== 資源使用告警 =====
|
||
- name: resource_alerts
|
||
rules:
|
||
# 節點 CPU 使用率過高
|
||
- alert: NodeHighCPU
|
||
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
|
||
for: 10m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: node
|
||
annotations:
|
||
summary: "⚠️ 節點 CPU 使用率過高"
|
||
description: "{{ $labels.instance }} CPU 使用率 {{ $value | printf \"%.1f\" }}%"
|
||
|
||
# 節點記憶體使用率過高
|
||
- alert: NodeHighMemory
|
||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
|
||
for: 10m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: node
|
||
annotations:
|
||
summary: "⚠️ 節點記憶體使用率過高"
|
||
description: "{{ $labels.instance }} 記憶體使用率 {{ $value | printf \"%.1f\" }}%"
|
||
|
||
# 容器 OOM Killed
|
||
- alert: ContainerOOMKilled
|
||
expr: increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled", namespace="awoooi-prod"}[1h]) > 0
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: container
|
||
annotations:
|
||
summary: "⚠️ 容器 OOM Killed"
|
||
description: "{{ $labels.pod }}/{{ $labels.container }} 過去 1 小時因 OOM 被終止"
|
||
|
||
# ===== TLS 證書告警 =====
|
||
- name: tls_certificate_alerts
|
||
rules:
|
||
# TLS 證書即將過期 (30 天內)
|
||
- alert: TLSCertExpiringIn30Days
|
||
expr: probe_ssl_earliest_cert_expiry{job="blackbox-https"} - time() < 86400 * 30
|
||
for: 1h
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: tls
|
||
annotations:
|
||
summary: "⚠️ TLS 證書即將過期"
|
||
description: "{{ $labels.instance }} 證書將在 {{ $value | humanizeDuration }} 後過期"
|
||
|
||
# TLS 證書即將過期 (7 天內)
|
||
- alert: TLSCertExpiringIn7Days
|
||
expr: probe_ssl_earliest_cert_expiry{job="blackbox-https"} - time() < 86400 * 7
|
||
for: 1h
|
||
labels:
|
||
severity: critical
|
||
team: ops
|
||
component: tls
|
||
annotations:
|
||
summary: "🔴 TLS 證書即將過期 (7 天內)"
|
||
description: "{{ $labels.instance }} 證書將在 {{ $value | humanizeDuration }} 後過期，請立即更新！"
|
||
|
||
# TLS 證書已過期
|
||
- alert: TLSCertExpired
|
||
expr: probe_ssl_earliest_cert_expiry{job="blackbox-https"} - time() < 0
|
||
labels:
|
||
severity: critical
|
||
team: ops
|
||
component: tls
|
||
annotations:
|
||
summary: "🔴🔴 TLS 證書已過期"
|
||
description: "{{ $labels.instance }} 證書已過期！HTTPS 服務可能無法正常運作"
|
||
|
||
# TLS 探測失敗
|
||
- alert: TLSProbeFailure
|
||
expr: probe_success{job="blackbox-https"} == 0
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: tls
|
||
annotations:
|
||
summary: "⚠️ TLS 探測失敗"
|
||
description: "無法連線到 {{ $labels.instance }} 進行 TLS 檢查"
|
||
|
||
# ===== kube-state-metrics 擴充告警 (P3 2026-03-29) =====
|
||
- name: kube_state_extended
|
||
rules:
|
||
# CronJob 上次執行失敗
|
||
- alert: CronJobLastRunFailed
|
||
expr: kube_job_status_failed{namespace="awoooi-prod"} > 0
|
||
for: 1m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: cronjob
|
||
annotations:
|
||
summary: "⚠️ CronJob 執行失敗"
|
||
description: "Job {{ $labels.job_name }} 執行失敗"
|
||
|
||
# DaemonSet 缺少 Pod
|
||
- alert: DaemonSetMissingPods
|
||
expr: kube_daemonset_status_number_unavailable{namespace=~"awoooi-prod|kube-system|velero"} > 0
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: daemonset
|
||
annotations:
|
||
summary: "⚠️ DaemonSet 缺少 Pod"
|
||
description: "{{ $labels.daemonset }} 缺少 {{ $value }} 個 Pod"
|
||
|
||
# StatefulSet 副本不足
|
||
- alert: StatefulSetReplicasMismatch
|
||
expr: kube_statefulset_status_replicas_ready{namespace="awoooi-prod"} != kube_statefulset_replicas{namespace="awoooi-prod"}
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: statefulset
|
||
annotations:
|
||
summary: "⚠️ StatefulSet 副本不足"
|
||
description: "{{ $labels.statefulset }} 預期副本數與就緒數不符"
|
||
|
||
# 容器長時間等待 (ImagePullBackOff/CrashLoopBackOff)
|
||
- alert: ContainerWaiting
|
||
expr: kube_pod_container_status_waiting_reason{namespace="awoooi-prod", reason=~"ImagePullBackOff|CrashLoopBackOff|ErrImagePull"} == 1
|
||
for: 10m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: container
|
||
annotations:
|
||
summary: "⚠️ 容器等待中"
|
||
description: "{{ $labels.pod }}/{{ $labels.container }} 處於 {{ $labels.reason }} 狀態"
|
||
|
||
# PDB 違規 (可用 Pod 數低於 minAvailable)
|
||
- alert: PDBViolation
|
||
expr: kube_poddisruptionbudget_status_current_healthy{namespace="awoooi-prod"} < kube_poddisruptionbudget_status_desired_healthy{namespace="awoooi-prod"}
|
||
for: 5m
|
||
labels:
|
||
severity: warning
|
||
team: ops
|
||
component: pdb
|
||
annotations:
|
||
summary: "⚠️ PDB 違規"
|
||
description: "{{ $labels.poddisruptionbudget }} 健康 Pod 數低於期望值"
|
||
|
||
# 節點 taint 未被容忍 (排程問題偵測)
|
||
- alert: NodeUnschedulable
|
||
expr: kube_node_spec_unschedulable == 1
|
||
for: 30m
|
||
labels:
|
||
severity: info
|
||
team: ops
|
||
component: node
|
||
annotations:
|
||
summary: "ℹ️ 節點標記為不可排程"
|
||
description: "節點 {{ $labels.node }} 已被標記為 cordon/unschedulable 超過 30 分鐘"
|