awoooi/k8s/observability/otel-collector-daemonset.yaml

# =============================================================================
# OTEL Collector DaemonSet - Phase O-2.1
# =============================================================================
# 建立者: Claude Code (首席架構師)
# 日期: 2026-04-02 (台北時間)
# 用途: 收集所有 K3s 節點的 Pod Log，統一送往 SigNoz ClickHouse
# 架構決策: SigNoz 統一派 (不另裝 Loki)
# =============================================================================
---
apiVersion: v1
kind: Namespace
metadata:
  name: observability
  labels:
    app.kubernetes.io/name: observability
    # Phase O-2.1: DaemonSet log collector 必須 root 讀取 /var/log/pods
    # 這是 OTEL Collector / Promtail / Fluentbit 的業界標準做法
    pod-security.kubernetes.io/enforce: privileged
    pod-security.kubernetes.io/audit: privileged
    pod-security.kubernetes.io/warn: privileged
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: otel-collector
  namespace: observability
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: otel-collector
rules:
  # filelog receiver 需要讀取 Pod 資訊來注入 k8s attributes
  - apiGroups: [""]
    resources: ["pods", "namespaces", "nodes"]
    verbs: ["get", "watch", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: otel-collector
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: otel-collector
subjects:
  - kind: ServiceAccount
    name: otel-collector
    namespace: observability
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: otel-collector-config
  namespace: observability
data:
  config.yaml: |
    receivers:
      # 讀取 K3s 節點上所有 Pod 的 stdout/stderr log
      filelog:
        include:
          - /var/log/pods/*/*/*.log
        exclude:
          # 排除 OTEL Collector 自身 (防遞迴)
          - /var/log/pods/observability_otel-collector*/**/*.log
        start_at: end
        include_file_path: true
        include_file_name: false
        operators:
          # K3s 使用 CRI log 格式: <timestamp> <stream> <flags> <log>
          - type: regex_parser
            id: cri_parser
            regex: '^(?P<time>[^ ]+) (?P<stream>stdout|stderr) (?P<flags>[^ ]*) (?P<log>.*)$'
            timestamp:
              parse_from: attributes.time
              # K3s CRI log: 2026-04-02T13:58:56.472854397+08:00
              # 使用 Go native layout (非 strptime)
              layout_type: gotime
              layout: '2006-01-02T15:04:05.999999999-07:00'
          # 從檔案路徑提取 namespace/pod/container 資訊
          - type: regex_parser
            id: extract_metadata
            parse_from: attributes["log.file.path"]
            regex: '^/var/log/pods/(?P<namespace>[^_]+)_(?P<pod>[^_]+)_[^/]+/(?P<container>[^/]+)/'
          # 移動到 resource attributes
          - type: move
            from: attributes.namespace
            to: resource["k8s.namespace.name"]
          - type: move
            from: attributes.pod
            to: resource["k8s.pod.name"]
          - type: move
            from: attributes.container
            to: resource["k8s.container.name"]
          # log body 使用解析出的 log 內容
          - type: move
            from: attributes.log
            to: body

    processors:
      # 批次發送: 降低網路開銷
      batch:
        send_batch_size: 200
        send_batch_max_size: 500
        timeout: 5s

      # 過濾: 捨棄低價值 log (降低 ClickHouse 儲存壓力)
      filter/drop_noise:
        logs:
          exclude:
            match_type: regexp
            # 排除 kube-system 中的高頻 debug log
            resource_attributes:
              - key: k8s.namespace.name
                value: "kube-system"
            bodies:
              - ".*level=debug.*"
              - ".*Listing and watching.*"
              - ".*reflector\\.go.*"

      # 記憶體限制: 防止 OOM
      memory_limiter:
        check_interval: 5s
        limit_mib: 100
        spike_limit_mib: 30

      # 注入節點資訊
      resource:
        attributes:
          - key: k8s.node.name
            value: "${env:K8S_NODE_NAME}"
            action: upsert
          - key: deployment.environment
            value: "prod"
            action: upsert

    exporters:
      # 發送到 SigNoz OTEL Collector (HTTP)
      otlphttp:
        endpoint: http://192.168.0.188:24318
        retry_on_failure:
          enabled: true
          initial_interval: 5s
          max_interval: 30s
        sending_queue:
          enabled: true
          num_consumers: 2
          queue_size: 100

    service:
      pipelines:
        logs:
          receivers: [filelog]
          processors: [memory_limiter, filter/drop_noise, resource, batch]
          exporters: [otlphttp]
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: otel-collector
  namespace: observability
  labels:
    app.kubernetes.io/name: otel-collector
    app.kubernetes.io/component: log-collector
    phase: o-2
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: otel-collector
  template:
    metadata:
      labels:
        app.kubernetes.io/name: otel-collector
        app.kubernetes.io/component: log-collector
    spec:
      serviceAccountName: otel-collector
      # observability namespace 設為 privileged PSA
      # 原因: /var/log/pods 是 root:root 0750，filelog receiver 必須 root 讀取
      securityContext:
        runAsUser: 0
        runAsGroup: 0
      containers:
        - name: otel-collector
          image: otel/opentelemetry-collector-contrib:0.96.0
          args:
            - --config=/etc/otel/config.yaml
          env:
            - name: K8S_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          securityContext:
            readOnlyRootFilesystem: true
          resources:
            requests:
              cpu: 50m
              memory: 64Mi
            limits:
              cpu: 200m
              memory: 128Mi
          volumeMounts:
            - name: config
              mountPath: /etc/otel
              readOnly: true
            - name: varlogpods
              mountPath: /var/log/pods
              readOnly: true
            # OTEL Collector 需要 tmp 來寫 checkpoint
            - name: tmp
              mountPath: /tmp
      volumes:
        - name: config
          configMap:
            name: otel-collector-config
        - name: varlogpods
          hostPath:
            path: /var/log/pods
            type: Directory
        - name: tmp
          emptyDir:
            sizeLimit: 50Mi
      tolerations:
        # 確保在所有節點上都運行 (包括 control-plane)
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
          effect: NoSchedule