Outcold Solutions LLC

Monitoring Kubernetes - Version 3

You are looking at documentation for an older release. See the current release documentation.

Collector configuration (Kubernetes version 1.5.x)

You can find configurations for other versions of Kubernetes by following this link Configuration Reference.

Download

collectorforkubernetes.yaml

curl

$ curl -O https://www.outcoldsolutions.com/docs/monitoring-kubernetes/v3/configuration/1.5/collectorforkubernetes.yaml

wget

$ wget https://www.outcoldsolutions.com/docs/monitoring-kubernetes/v3/configuration/1.5/collectorforkubernetes.yaml

collectorforkubernetes.yaml

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app: collectorforkubernetes
  name: collectorforkubernetes
  namespace: default

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: collectorforkubernetes
  labels:
    app: collectorforkubernetes
data:
  collector.conf: |
    # collector configuration file
    #
    # Run collector with flag -conf and specify location of the configuration file.
    #
    # You can override all the values using environment variables with the format like
    #   COLLECTOR__<ANYNAME>=<section>__<key>=<value>
    # As an example you can set dataPath in [general] section as
    #   COLLECTOR__DATAPATH=general__dataPath=C:\\some\\path\\data.db
    # This parameter can be configured using -env-override, set it to empty string to disable this feature

    [general]

    # Please review EULA https://www.outcoldsolutions.com/docs/license-agreement/
    # and accept eula by uncommenting this code and changing value to *true*
    ; acceptEULA = false

    # location for the database
    # is used to store position of the files and internal state
    ; dataPath = ./data/

    # log level (trace, debug, info, warn, error, fatal)
    ; logLevel = info

    # http server gives access to two endpoints
    # /healthz
    # /metrics
    ; httpServerBinding = :8080

    # telemetry report endpoint, set it to empty string to disable telemetry
    ; telemetryEndpoint = https://license.outcold.solutions/telemetry/

    # license check endpoint
    ; licenseEndpoint = https://license.outcold.solutions/license/

    # license server through proxy
    ; licenseServerProxyUrl =

    # license
    ; license =

    # docker daemon hostname is used by default as hostname
    # use this configuration to override
    ; hostname =

    # Include custom fields to attach to every event, in example below every event sent to Splunk will hav
    # indexed field my_environment=dev. Fields names should match to ^[a-z][_a-z0-9]*$
    # Better way to configure that is to specify labels for Kubernetes Nodes.
    # ; fields.my_environment = dev


    # connection to docker host
    [general.docker]

    # url for docker API, only unix socket is supported
    ; url = unix:///rootfs/var/run/docker.sock

    # path to docker root folder (can fallback to use folder structure to read docker metadata)
    ; dockerRootFolder = /rootfs/var/lib/docker/

    # In case if pod metadata was not retrievied. how often collector should retry to reload the pod metadata
    ; MetadataFetchRetry = 1s

    # In case if event is recent, how long pipeline should wait for the metadata to be available in Kubernetes API
    ; MetadataFetchWait = 5s

    # In case if collector does not see new events for specific container and with the last metadata refresh
    # We have not found this container - fow how long we should keep this metadata in cache.
    ; MetadataTTL = 5m


    # connection to kubernetes host
    [general.kubernetes]

    # name of kubernetes node (required)
    ; nodeName =

    ; tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token

    ; certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

    # In case if pod metadata was not retrievied. how often collector should retry to reload the pod metadata
    ; MetadataFetchRetry = 5s

    # In case if event is recent, how long pipeline should wait for the metadata to be available in Kubernetes API
    ; MetadataFetchWait = 30s

    # In case if collector does not see new events for specific container and with the last metadata refresh
    # We have not found this container - fow how long we should keep this metadata in cache.
    ; MetadataTTL = 5m


    # cgroup input
    [input.system_stats]

    # disable system level stats
    ; disabled = false

    # cgroups fs location
    ; pathCgroups = /rootfs/sys/fs/cgroup

    # proc location
    ; pathProc = /rootfs/proc

    # how often to collect cgroup stats
    ; statsInterval = 30s

    # override type
    ; type = kubernetes_stats

    # specify Splunk index
    ; index =


    # proc input
    [input.proc_stats]

    # disable proc level stats
    ; disabled = false

    # proc location
    ; pathProc = /rootfs/proc

    # how often to collect proc stats
    ; statsInterval = 30s

    # override type
    ; type = kubernetes_proc_stats

    # specify Splunk index
    ; index =


    # Log files
    [input.files]

    # disable container logs monitoring
    ; disabled = false

    # root location of docker files
    ; path = /rootfs/var/lib/docker/containers/

    # glob matching pattern for log files
    ; glob = */*-json.log*

    # files are read using polling schema, when reach the EOF how often to check if files got updated
    ; pollingInterval = 250ms

    # how often to look for the new files under logs path
    ; walkingInterval = 5s

    # include verbose fields in events (file offset)
    ; verboseFields = false

    # override type
    ; type = kubernetes_logs

    # specify Splunk index
    ; index =


    # Input syslog(.\d+)? files
    [input.files::syslog]

    # disable host level logs
    ; disabled = false

    # root location of docker files
    path = /rootfs/var/log/

    # regex matching pattern
    match = ^(syslog|messages)(.\d+)?$

    # limit search only on one level
    recursive = false

    # files are read using polling schema, when reach the EOF how often to check if files got updated
    pollingInterval = 250ms

    # how often o look for the new files under logs path
    walkingInterval = 5s

    # include verbose fields in events (file offset)
    verboseFields = false

    # override type
    type = kubernetes_host_logs

    # specify Splunk index
    ; index =

    # field extraction
    extraction = ^(?P<timestamp>[A-Za-z]+\s+\d+\s\d+:\d+:\d+)\s(?P<syslog_hostname>[^\s]+)\s(?P<syslog_component>[^:\[]+)(\[(?P<syslog_pid>\d+)\])?: (.+)$

    # timestamp field
    timestampField = timestamp

    # format for timestamp
    # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
    timestampFormat = Jan 2 15:04:05

    # Adjust date, if month/day aren't set in format
    ; timestampSetMonth = false
    ; timestampSetDay = false

    # timestamp location (if not defined by format)
    timestampLocation = Local


    # Input all *.log(.\d+)? files
    [input.files::logs]

    # disable host level logs
    ; disabled = false

    # root location of docker files
    path = /rootfs/var/log/

    # regex matching pattern
    match = ^[\w]+\.log(.\d+)?$

    # files are read using polling schema, when reach the EOF how often to check if files got updated
    pollingInterval = 250ms

    # how often o look for the new files under logs path
    walkingInterval = 5s

    # include verbose fields in events (file offset)
    verboseFields = false

    # override type
    type = kubernetes_host_logs

    # specify Splunk index
    ; index =

    # field extraction
    ; extraction =

    # timestamp field
    ; timestampField =

    # format for timestamp
    # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
    ; timestampFormat =

    # timestamp location (if not defined by format)
    ; timestampLocation =


    [input.kubernetes_events]

    # disable host level logs
    ; disabled = false

    # override type
    ; type = kubernetes_events

    # specify Splunk index
    ; index =

    # Set the timeout for how long request to watch events going to hang reading.
    ; eventsWatchTimeout = 30m

    # Ignore events last seen later that this duration.
    ; eventsTTL = 12h


    # Splunk output
    [output.splunk]

    # Splunk HTTP Event Collector url
    ; url =

    # Splunk HTTP Event Collector Token
    ; token =

    # Allow invalid SSL server certificate
    ; insecure = false

    # Path to CA cerificate
    ; caPath =

    # CA Name to verify
    ; caName =

    # Events are batched with the maximum size set by batchSize and staying in pipeline for not longer
    # than set by frequency
    ; frequency = 5s
    ; batchSize = 768K

    # Splunk through proxy
    ; proxyUrl =

    # Splunk acknowledgement url (.../services/collector/ack)
    ; ackUrl =

    # Enable index acknowledgment
    ; ackEnabled = false

    # Index acknowledgment timeout
    ; ackTimeout = 3m


    # Pipe to join events (container logs only)
    [pipe.join]

    # disable joining event
    ; disabled = false

    # Maximum interval of messages in pipeline
    ; maxInterval = 100ms

    # Maximum time to wait for the messages in pipeline
    ; maxWait = 1s

    # Maximum message size
    ; maxSize = 100K

    # Default pattern to indicate new message (should start not from space)
    ; patternRegex = ^[^\s]


    # Kube API Server has trace messages with multi line events
    [pipe.join::kube-apiserver]
    ; disabled = false
    matchRegex.kubernetes_container_image = ^gcr.io/google_containers/kube-apiserver-.*$
    matchRegex.docker_stream = stderr
    patternRegex = ^[IWEF]\d{4}\s\d{2}:\d{2}:\d{2}.\d{6}\s


    # Define special event join patterns for matched events
    # Section consist of [pipe.join::<name>]
    # [pipe.join::my_app]
    ## Set match pattern for the fields
    #; matchRegex.docker_container_image = my_app
    #; matchRegex.docker_stream = stdout
    ## All events start from '[<digits>'
    #; patternRegex = ^\[\d+
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: collectorforkubernetes
  labels:
    app: collectorforkubernetes
spec:
  # Default updateStrategy is OnDelete. For collector RollingUpdate is suitable
  # When you update configuration
  updateStrategy:
    type: RollingUpdate

  template:
    metadata:
      name: collectorforkubernetes
      labels:
        daemon: collectorforkubernetes
    spec:
      # Service account we use to connect to kubernetes proxy
      serviceAccountName: collectorforkubernetes
      # This allows to run DaemonSet pods on Master (or all nodes, which tained with NoSchedule)
      tolerations:
      - operator: "Exists"
        effect: "NoSchedule"
      - operator: "Exists"
        effect: "NoExecute"
      containers:
      - name: collectorforkubernetes
        # Collector version
        image: outcoldsolutions/collectorforkubernetes:3.0.87.180322
        securityContext:
          # Privileged only required to get access to IO in /proc file system.
          # You can disable privileged and still get most of the metrics and logs.
          #
          # Another option can be with enabling just SYS_PTRACE, that will allow access to most
          # processes io statistics (excluded processes running outside of containers)
          #
          #  capabilities:
          #    add: ["SYS_PTRACE"]
          #
          # To use SYS_PTRACE capability and collect IO metrics for all processes
          # you need to configure apparmour. See issue:
          #   https://github.com/moby/moby/issues/21051 (how to make it work in Docker)
          # Which requires `--security-opt apparmor:unconfined`
          # But following kubernetes security context documentation it is not possible
          # To set without manually creating apparmor profile
          #   https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
          #   https://kubernetes.io/docs/tutorials/clusters/apparmor/
          # `unconfined` will be available in future Kubernetes versions
          #   https://github.com/kubernetes/kubernetes/pull/52395
          privileged: true
        # Define your resources if you need. Defaults should be fine for most.
        # You can lower or increase based on your hosts.
        resources:
          limits:
            cpu: 2
            memory: 512Mi
          requests:
            cpu: 100m
            memory: 128Mi
        # This way we set Kubernetes Node name to be able to filter out
        # Only pods from current pod.
        env:
        - name: KUBERNETES_NODENAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        volumeMounts:
        # We store state in /data folder (file positions)
        - name: collectorforkubernetes-state
          mountPath: /data
        # Configuration file deployed with ConfigMap
        - name: collectorforkubernetes-config
          mountPath: /config/
          readOnly: true
        # Cgroup filesystem to get metrics
        - name: cgroup
          mountPath: /rootfs/sys/fs/cgroup
          readOnly: true
        # Proc filesystem to get metrics
        - name: proc
          mountPath: /rootfs/proc
          readOnly: true
        # Docker logs (JSON files)
        - name: docker-logs
          mountPath: /rootfs/var/lib/docker/containers/
          readOnly: true
        # Docker socket
        - name: docker-unix-socket
          mountPath: /rootfs/var/run/docker.sock
          readOnly: true
        # Host logs
        - name: logs
          mountPath: /rootfs/var/log/
          readOnly: true
      volumes:
      # We store state directly on host, change this location, if
      # your persistent volume is somewhere else
      - name: collectorforkubernetes-state
        hostPath:
          path: /var/lib/collectorforkubernetes/data/
      # Location of docker logs
      - name: docker-logs
        hostPath:
          path: /var/lib/docker/containers/
      # Location of cgroups file system
      - name: cgroup
        hostPath:
          path: /sys/fs/cgroup
      # Location of proc file system
      - name: proc
        hostPath:
          path: /proc
      # Host logs location
      - name: logs
        hostPath:
          path: /var/log
      # Docker socket
      - name: docker-unix-socket
        hostPath:
          path: /var/run/docker.sock
      # configuration from ConfigMap
      - name: collectorforkubernetes-config
        configMap:
          name: collectorforkubernetes

About Outcold Solutions

Outcold Solutions provides solutions for monitoring Kubernetes, OpenShift and Docker clusters in Splunk Enterprise and Splunk Cloud. We offer certified Splunk applications, which give you insights across all containers environments. We are helping businesses reduce complexity related to logging and monitoring by providing easy-to-use and deploy solutions for Linux and Windows containers. We deliver applications, which help developers monitor their applications and operators to keep their clusters healthy. With the power of Splunk Enterprise and Splunk Cloud, we offer one solution to help you keep all the metrics and logs in one place, allowing you to quickly address complex questions on container performance.