Download
Link
collectorforkubernetes-elasticsearch.yaml
CURL
1curl -O https://www.outcoldsolutions.com/docs/elasticsearch-kubernetes/collectorforkubernetes-elasticsearch.yamlWGET
1wget https://www.outcoldsolutions.com/docs/elasticsearch-kubernetes/collectorforkubernetes-elasticsearch.yamlcollectorforkubernetes.yaml
1apiVersion: v1
2kind: Namespace
3metadata:
4 labels:
5 app: collectorforkubernetes
6 name: collectorforkubernetes
7---
8apiVersion: apiextensions.k8s.io/v1
9kind: CustomResourceDefinition
10metadata:
11 name: configurations.collectord.io
12spec:
13 group: collectord.io
14 versions:
15 - name: v1
16 served: true
17 storage: true
18 schema:
19 openAPIV3Schema:
20 type: object
21 properties:
22 spec:
23 type: object
24 additionalProperties: true
25 force:
26 type: boolean
27 scope: Cluster
28 names:
29 plural: configurations
30 singular: configuration
31 kind: Configuration
32---
33apiVersion: v1
34kind: ServiceAccount
35metadata:
36 labels:
37 app: collectorforkubernetes
38 name: collectorforkubernetes
39 namespace: collectorforkubernetes
40---
41apiVersion: scheduling.k8s.io/v1
42kind: PriorityClass
43metadata:
44 name: collectorforkubernetes-critical
45value: 1000000000
46---
47apiVersion: rbac.authorization.k8s.io/v1
48kind: ClusterRole
49metadata:
50 labels:
51 app: collectorforkubernetes
52 name: collectorforkubernetes
53rules:
54- apiGroups: ['extensions']
55 resources: ['podsecuritypolicies']
56 verbs: ['use']
57 resourceNames:
58 - privileged
59- apiGroups:
60 - ""
61 - apps
62 - batch
63 - extensions
64 - rbac.authorization.k8s.io
65 - collectord.io
66 resources:
67 - alertmanagers
68 - cronjobs
69 - daemonsets
70 - deployments
71 - endpoints
72 - events
73 - jobs
74 - namespaces
75 - nodes
76 - nodes/metrics
77 - nodes/proxy
78 - pods
79 - replicasets
80 - replicationcontrollers
81 - scheduledjobs
82 - services
83 - statefulsets
84 - persistentvolumeclaims
85 - configurations
86 - resourcequotas
87 - clusterroles
88 - secrets
89 - configmaps
90 verbs:
91 - get
92 - list
93 - watch
94- nonResourceURLs:
95 - /metrics
96 verbs:
97 - get
98 apiGroups: []
99 resources: []
100---
101apiVersion: rbac.authorization.k8s.io/v1
102kind: ClusterRoleBinding
103metadata:
104 labels:
105 app: collectorforkubernetes
106 name: collectorforkubernetes
107 namespace: collectorforkubernetes
108roleRef:
109 apiGroup: rbac.authorization.k8s.io
110 kind: ClusterRole
111 name: collectorforkubernetes
112subjects:
113 - kind: ServiceAccount
114 name: collectorforkubernetes
115 namespace: collectorforkubernetes
116---
117apiVersion: v1
118kind: ConfigMap
119metadata:
120 name: collectorforkubernetes-elasticsearch
121 namespace: collectorforkubernetes
122 labels:
123 app: collectorforkubernetes-elasticsearch
124data:
125 001-general.conf: |
126 # The general configuration is used for all deployments
127 #
128 # Run collectord with the flag -conf and specify location of the configuration files.
129 #
130 # You can override all the values using environment variables with the format like
131 # COLLECTOR__<ANYNAME>=<section>__<key>=<value>
132 # As an example you can set dataPath in [general] section as
133 # COLLECTOR__DATAPATH=general__dataPath=C:\\some\\path\\data.db
134 # This parameter can be configured using -env-override, set it to empty string to disable this feature
135
136 [general]
137 # Please review license https://www.outcoldsolutions.com/docs/license-agreement/
138 # and accept license by changing the value to *true*
139 acceptLicense = false
140
141 # Location for the database
142 # Collectord stores positions of the files and internal state
143 dataPath = ./data/
144
145 # log level (accepted values are trace, debug, info, warn, error, fatal)
146 logLevel = info
147
148 # http server gives access to two endpoints
149 # /healthz
150 # /metrics/json
151 # /metrics/prometheus
152 httpServerBinding =
153
154 # telemetry report endpoint, set it to empty string to disable telemetry
155 telemetryEndpoint = https://license.outcold.solutions/telemetry/
156
157 # license check endpoint
158 licenseEndpoint = https://license.outcold.solutions/license/
159
160 # license server through proxy
161 licenseServerProxyUrl =
162
163 # authentication with basic authorization (user:password)
164 licenseServerProxyBasicAuth =
165
166 # license key
167 license =
168
169 # Environment variable $KUBERNETES_NODENAME is used by default to setup hostname
170 # Use value below to override specific name
171 # hostname = ${KUBERNETES_NODENAME}.second
172
173 # Default output for events, logs and metrics
174 # valid values: elasticsearch and devnull
175 # Use devnull by default if you don't want to redirect data
176 defaultOutput = elasticsearch
177
178 # Default buffer size for file input
179 fileInputBufferSize = 256b
180
181 # Maximum size of one line the file reader can read
182 fileInputLineMaxSize = 1mb
183
184 # Include custom fields to attach to every event, in example below every event sent to elasticsearch will have
185 # indexed field my_environment=dev. Fields names should match to ^[a-z][_a-z0-9\.]*$
186 # Better way to configure that is to specify labels for Kubernetes Nodes.
187 # ; fields.my_environment = dev
188 # Identify the cluster if you are planning to monitor multiple clusters
189 # For ElasticSearch look at the ECS (Elastic Common Schema) documentation https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html
190 fields.ecs.version = 8.0.0
191
192 fields.orchestrator.cluster.name = -
193 fields.orchestrator.type = kubernetes
194
195 fields.agent.type = collectord
196 fields.agent.version = ${COLLECTORD_VERSION}
197 fields.agent.ephemeral_id = ${COLLECTORD_INSTANCE_RUNTIME_ID}
198 fields.agent.id = ${COLLECTORD_INSTANCE_ID}
199 fields.agent.name = ${KUBERNETES_NODENAME}
200
201 fields.host.name = ${KUBERNETES_NODENAME}
202 fields.host.hostname = ${KUBERNETES_NODENAME}
203 fields.host.architecture = ${COLLECTORD_ARCH}
204
205 # Include EC2 Metadata (see list of possible fields https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
206 # Should be in format ec2Metadata.{desired_field_name} = {url path to read the value}
207 # ec2Metadata.cloud.instance.id = /latest/meta-data/instance-id
208 # ec2Metadata.cloud.machine.type = /latest/meta-data/instance-type
209
210 # subdomain for the annotations added to the pods, workloads, namespaces or containers, like elasticsearch.collectord.io/..
211 annotationsSubdomain = elasticsearch
212
213 # configure global thruput per second for forwarded logs (metrics are not included)
214 # for example if you set `thruputPerSecond = 512Kb`, that will limit amount of logs forwarded
215 # from the single Collectord instance to 512Kb per second.
216 # You can configure thruput individually for the logs (including specific for container logs) below
217 thruputPerSecond =
218 ; thruputPerSecond = 512Kb
219
220 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
221 # older than 7 days
222 tooOldEvents =
223
224 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
225 tooNewEvents =
226
227 # Multi-output async publishing. When enabled (default), events routed to
228 # non-default outputs are published asynchronously so that a slow or down
229 # output does not block events destined for other outputs.
230 ; multioutput.async = true
231 # Buffer size for the async proxy (default 100). Absorbs transient bursts.
232 # When this buffer and the output's own queue are both full, events are
233 # dropped immediately without blocking the pipeline.
234 ; multioutput.asyncBufferSize = 100
235
236 [license.client]
237 # point to the license located on the HTTP web server, or a hosted by the Collectord running as license server
238 url =
239 # basic authentication for the HTTP server
240 basicAuth =
241 # if SSL, ignore the certificate verification
242 insecure = false
243 # CA Path for the Server certificate
244 capath =
245 # CA Name fot the Server certificate
246 caname =
247 # license server through proxy
248 proxyUrl =
249 # authentication with basic authorization (user:password)
250 proxyBasicAuth =
251
252 # connection to kubernetes api
253 [general.kubernetes]
254
255 # Override service URL for Kubernetes (default is ${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT})
256 serviceURL =
257
258 # Environment variable $KUBERNETES_NODENAME is used by default to setup nodeName
259 # Use it only when you need to override it
260 nodeName =
261
262 # Configuration to access the API server,
263 # see https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod
264 # for details
265 tokenPath = /var/run/secrets/kubernetes.io/serviceaccount/token
266 certPath = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
267
268 # Default timeout for http responses. The streaming/watch requests depend on this timeout.
269 timeout = 30m
270
271 # How long to keep the cache for the recent calls to API server (to limit number of calls when collectord discovers new pods)
272 metadataTTL = 30s
273
274 # path to the kubelet root location (use it to discover application logs for emptyDir)
275 # the expected format is `pods/{pod-id}/volumes/kubernetes.io~empty-dir/{volume-name}/_data/`
276 volumesRootDir = /rootfs/var/lib/kubelet/
277
278 # You can attach annotations as a metadata, using the format
279 # includeAnnotations.{key} = {regexp}
280 # For example if you want to include all annotations that starts with `prometheus.io` or `example.com` you can include
281 # the following format:
282 # includeAnnotations.1 = ^prometheus\.io.*
283 # includeAnnotations.2 = ^example\.com.*
284
285 # watch for changes (annotations) in the objects
286 watch.namespaces = v1/namespace
287 watch.deployments = apps/v1/deployment
288 watch.configurations = collectord.io/v1/configuration
289
290 # Collectord can review the assigned ClusterRole and traverse metadata for the Pods only for the Owner objects
291 # that are defined in the ClusterRole, ignoring anything else, it does not have access to.
292 # This way Collectord does not generate 403 requests on API Server
293 clusterRole = collectorforkubernetes
294
295 # Alternative of telling Collectord about the ClusterRole is to manually list the objects.
296 # You can define which objects Collectord should traverse when it sees Owners.
297 ; traverseOwnership.namespaces = v1/namespace
298
299
300 # ElasticSearch output
301 [output.elasticsearch]
302
303 # Default data stream name
304 dataStream = logs-collectord-{{agent.version}}
305 dataStreamFailedEvents = logs-collectord-failed-{{agent.version}}
306
307 # ElasticSearch Scheme Host and Port
308 host =
309
310 # You can specify multiple hosts with
311 #
312 # hosts.0 = https://es0:9200
313 # hosts.1 = https://es1:9200
314 # hosts.2 = https://es2:9200
315
316 # Specify how Hosts should be picked up (in case if multiple is used)
317 # * random - choose random url on first selection and after each failure (connection or HTTP status code >= 500)
318 # * round-robin - choose url starting from first one and bump on each failure (connection or HTTP status code >= 500)
319 # * random-with-round-robin - choose random url on first selection and after that in round-robin on each
320 # failure (connection or HTTP status code >= 500)
321 hostSelection = random-with-round-robin
322
323 # Configuration for basic authorization
324 authorizationBasicUsername =
325 authorizationBasicPassword =
326
327 # additional headers
328 headers.Content-Type = application/json
329 headers.Accept = application/json
330
331 # Allow invalid SSL server certificate
332 insecure = false
333
334 # Path to CA certificate
335 caPath =
336
337 # CA Name to verify
338 caName =
339
340 # path for client certificate (if required)
341 clientCertPath =
342
343 # path for a client key (if required)
344 clientKeyPath =
345
346 # Events are batched with the maximum size set by batchSize and staying in pipeline for not longer
347 # than set by frequency
348 frequency = 5s
349 batchSize = 768K
350 # limit by the number of events (0 value has no limit on the number of events)
351 events = 50
352
353 # elasticsearch through proxy
354 proxyUrl =
355
356 # authentication with basic authorization (user:password)
357 proxyBasicAuth =
358
359 # Timeout specifies a time limit for requests made by collectord.
360 # The timeout includes connection time, any
361 # redirects, and reading the response body.
362 timeout = 30s
363
364 # gzip compression level (nocompression, default, 1...9)
365 compressionLevel = default
366
367 # number of dedicated elasticsearch output threads (to increase throughput above 4k events per second)
368 threads = 2
369 # Default algorithm between threads is roundrobin, but you can change it to weighted
370 threadsAlgorithm = roundrobin
371
372 # Submit objects to elasticsearch
373 put._ilm/policy/logs-collectord = /config/es-default-index-lifecycle-management-policy.json
374 put._index_template/logs-collectord-${COLLECTORD_VERSION} = /config/es-default-index-template.json
375 put._index_template/logs-collectord-failed-${COLLECTORD_VERSION} = /config/es-failed-index-template.json
376
377 # Dedicated queue size for the output, default is 1024, larger queue sizes will require more memory,
378 # but will allow to handle more events in case of network issues
379 queueSize = 1024
380
381 # Configure multiple outputs
382 # [output.elasticsearch::app1]
383 # host = http://esapp1:9200
384
385 002-daemonset.conf: |
386 # DaemonSet configuration is used for Nodes and Masters.
387
388 // connection to CRIO
389 [general.cri-o]
390
391 # url for CRIO API, only unix socket is supported
392 url = unix:///rootfs/var/run/crio/crio.sock
393
394 # Timeout for http responses to docker client. The streaming requests depend on this timeout.
395 timeout = 1m
396
397 [general.containerd]
398
399 runtimePath = /rootfs/var/run/containerd
400 namespace = k8s.io
401
402 # Container Log files
403 [input.files]
404
405 # disable container logs monitoring
406 disabled = false
407
408 # root location of docker log files
409 # logs are expected in standard docker format like {containerID}/{containerID}-json.log
410 # rotated files
411 path = /rootfs/var/lib/docker/containers/
412 # root location of CRI-O (including Containerd) files
413 # logs are expected in Kubernetes format, like {podID}/{containerName}/0.log
414 crioPath = /rootfs/var/log/pods/
415
416 # files are read using polling schema, when reach the EOF how often to check if files got updated
417 pollingInterval = 250ms
418
419 # how often to look for the new files under logs path
420 walkingInterval = 5s
421
422 # override type
423 type = container
424
425 # override datastream
426 elasticsearch.datastream =
427
428 # docker splits events when they are larger than 10-100k (depends on the docker version)
429 # we join them together by default and forward to elasticsearch as one event
430 joinPartialEvents = true
431
432 # In case if your containers report messages with terminal colors or other escape sequences
433 # you can enable strip for all the containers in one place.
434 # Better is to enable it only for required container with the label collectord.io/strip-terminal-escape-sequences=true
435 stripTerminalEscapeSequences = false
436 # Regexp used for stripping terminal colors, it does not stip all the escape sequences
437 # Read http://man7.org/linux/man-pages/man4/console_codes.4.html for more information
438 stripTerminalEscapeSequencesRegex = (\x1b\[\d{1,3}(;\d{1,3})*m)|(\x07)|(\x1b]\d+(\s\d)?;[^\x07]+\x07)|(.*\x1b\[K)
439
440 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
441 samplingPercent = -1
442
443 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
444 samplingKey =
445
446 # set output (elasticsearch or devnull, default is [general]defaultOutput)
447 output =
448
449 # configure default thruput per second for each container log
450 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
451 # from the single container to 128Kb per second.
452 thruputPerSecond =
453
454 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
455 # older than 7 days
456 tooOldEvents =
457
458 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
459 tooNewEvents =
460
461
462 # Application Logs
463 [input.app_logs]
464
465 # disable container application logs monitoring
466 disabled = false
467
468 # root location of mounts (applies to hostPath mounts only), if the hostPath differs inside container from the path on host
469 root = /rootfs/
470
471 # how often to review list of available volumes
472 syncInterval = 5s
473
474 # glob matching pattern for log files
475 glob = *.log*
476
477 # files are read using polling schema, when reach the EOF how often to check if files got updated
478 pollingInterval = 250ms
479
480 # how often to look for the new files under logs path
481 walkingInterval = 5s
482
483 # include verbose fields in events (file offset)
484 verboseFields = false
485
486 # override type
487 type = container.file
488
489 # override datastream
490 elasticsearch.datastream =
491
492 # we split files using new line character, with this configuration you can specify what defines the new event
493 # after new line
494 eventPatternRegex = ^[^\s]
495 # Maximum interval of messages in pipeline
496 eventPatternMaxInterval = 100ms
497 # Maximum time to wait for the messages in pipeline
498 eventPatternMaxWait = 1s
499 # Maximum message size
500 eventPatternMaxSize = 1MB
501
502 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
503 samplingPercent = -1
504
505 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
506 samplingKey =
507
508 # set output (elasticsearch or devnull, default is [general]defaultOutput)
509 output =
510
511 # configure default thruput per second for for each container log
512 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
513 # from the single container to 128Kb per second.
514 thruputPerSecond =
515
516 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
517 # older than 7 days
518 tooOldEvents =
519
520 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
521 tooNewEvents =
522
523 # Configure how long Collectord should keep the file descriptors open for files, that has not been forwarded yet
524 # When using PVC, and if pipeline is lagging behind, Collectord holding open fd for files, can cause long termination
525 # of pods, as kubelet cannot unmount the PVC volume from the system
526 maxHoldAfterClose = 1800s
527
528
529 # Host logs. Input syslog(.\d+)? files
530 [input.files::syslog]
531
532 # disable host level logs
533 disabled = false
534
535 # root location of docker files
536 path = /rootfs/var/log/
537
538 # regex matching pattern
539 match = ^(syslog|messages)(.\d+)?$
540
541 # limit search only on one level
542 recursive = false
543
544 # files are read using polling schema, when reach the EOF how often to check if files got updated
545 pollingInterval = 250ms
546
547 # how often o look for the new files under logs path
548 walkingInterval = 5s
549
550 # include verbose fields in events (file offset)
551 verboseFields = false
552
553 # override type
554 type = file
555
556 # override datastream
557 elasticsearch.datastream =
558
559 # field extraction
560 extraction = ^(?P<timestamp>[A-Za-z]+\s+\d+\s\d+:\d+:\d+)\s(?P<log__syslog__hostname>[^\s]+)\s(?P<log__syslog__appname>[^:\[]+)(\[(?P<log__syslog__procid>\d+)\])?: (.+)$
561 # extractionMessageField =
562
563 # timestamp field
564 timestampField = timestamp
565
566 # format for timestamp
567 # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
568 timestampFormat = Jan 2 15:04:05
569
570 # Adjust date, if month/day aren't set in format
571 timestampSetMonth = false
572 timestampSetDay = false
573
574 # timestamp location (if not defined by format)
575 timestampLocation = Local
576
577 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
578 samplingPercent = -1
579
580 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
581 samplingKey =
582
583 # set output (elasticsearch or devnull, default is [general]defaultOutput)
584 output =
585
586 # configure default thruput per second for this files group
587 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
588 # from the files in this group to 128Kb per second.
589 thruputPerSecond =
590
591 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
592 # older than 7 days
593 tooOldEvents =
594
595 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
596 tooNewEvents =
597
598 # by default every new event should start from not space symbol
599 eventPattern = ^[^\s]
600
601 # Blacklisting and whitelisting the logs
602 # whitelist.0 = ^regexp$
603 # blacklist.0 = ^regexp$
604
605
606 # Host logs. Input all *.log(.\d+)? files
607 [input.files::logs]
608
609 # disable host level logs
610 disabled = false
611
612 # root location of log files
613 path = /rootfs/var/log/
614
615 # regex matching pattern
616 match = ^(([\w\-.]+\.log(.[\d\-]+)?)|(docker))$
617
618 # files are read using polling schema, when reach the EOF how often to check if files got updated
619 pollingInterval = 250ms
620
621 # how often o look for the new files under logs path
622 walkingInterval = 5s
623
624 # include verbose fields in events (file offset)
625 verboseFields = false
626
627 # override type
628 type = file
629
630 # override datastream
631 elasticsearch.datastream =
632
633 # field extraction
634 extraction =
635
636 # timestamp field
637 timestampField =
638
639 # format for timestamp
640 # the layout defines the format by showing how the reference time, defined to be `Mon Jan 2 15:04:05 -0700 MST 2006`
641 timestampFormat =
642
643 # timestamp location (if not defined by format)
644 timestampLocation =
645
646 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
647 samplingPercent = -1
648
649 # sampling key for hash based sampling (should be regexp with the named match pattern `key`)
650 samplingKey =
651
652 # set output (elasticsearch or devnull, default is [general]defaultOutput)
653 output =
654
655 # configure default thruput per second for this files group
656 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
657 # from the files in this group to 128Kb per second.
658 thruputPerSecond =
659
660 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
661 # older than 7 days
662 tooOldEvents =
663
664 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
665 tooNewEvents =
666
667 # by default every new event should start from not space symbol
668 eventPattern = ^[^\s]
669
670 # Blacklisting and whitelisting the logs
671 # whitelist.0 = ^regexp$
672 # blacklist.0 = ^regexp$
673
674
675 [input.journald]
676
677 # disable host level logs
678 disabled = false
679
680 # root location of log files
681 path.persistent = /rootfs/var/log/journal/
682 path.volatile = /rootfs/run/log/journal/
683
684 # when reach end of journald, how often to pull
685 pollingInterval = 250ms
686
687 # if you don't want to forward journald from the beginning,
688 # set the oldest event in relative value, like -14h or -30m or -30s (h/m/s supported)
689 startFromRel =
690
691 # override type
692 type = journald
693
694 # override datastream
695 elasticsearch.datastream =
696
697 # sample output (-1 does not sample, 20 - only 20% of the logs should be forwarded)
698 samplingPercent = -1
699
700 # sampling key (should be regexp with the named match pattern `key`)
701 samplingKey =
702
703 # how often to reopen the journald to free old files
704 reopenInterval = 1h
705
706 # set output (elasticsearch or devnull, default is [general]defaultOutput)
707 output =
708
709 # configure default thruput per second for journald
710 # for example if you set `thruputPerSecond = 128Kb`, that will limit amount of logs forwarded
711 # from the journald to 128Kb per second.
712 thruputPerSecond =
713
714 # Configure events that are too old to be forwarded, for example 168h (7 days) - that will drop all events
715 # older than 7 days
716 tooOldEvents =
717
718 # Configure events that are too new to be forwarded, for example 1h - that will drop all events that are 1h in future
719 tooNewEvents =
720
721 # by default every new event should start from not space symbol
722 eventPattern = ^[^\s]
723
724 # Blacklisting and whitelisting the logs
725 # whitelist.0 = ^regexp$
726 # blacklist.0 = ^regexp$
727
728 # Pipe to join events (container logs only)
729 [pipe.join]
730
731 # disable joining event
732 disabled = false
733
734 # Maximum interval of messages in pipeline
735 maxInterval = 100ms
736
737 # Maximum time to wait for the messages in pipeline
738 maxWait = 1s
739
740 # Maximum message size
741 maxSize = 1MB
742
743 # Default pattern to indicate new message (should start not from space)
744 patternRegex = ^[^\s]
745
746 # You can configure global replace rules for the events, which can help to remove sensitive data
747 # from logs before they are sent to Splunk. Those rules will be applied to all pipelines for container logs, host logs,
748 # application logs and events.
749 # In the following example we replace password=TEST with password=********
750 ; [pipe.replace::name]
751 ; patternRegex = (password=)([^\s]+)
752 ; replace = $1********
753
754 # You can configure global hash rules for the events, which can help to hide sensitive data
755 # from logs before they are sent to outputs. Those rules will be applied to all pipelines for container logs, host logs,
756 # application logs and events.
757 # In the following example we hash IP addresses with fnv-1a-64
758 ; [pipe.hash::name]
759 ; match = (\d{1,3}\.){3}\d{1,3}'
760 ; function = fnv-1a-64
761
762 004-addon.conf: |
763 [ general ]
764
765 # addons can be run in parallel with agents
766 addon = true
767
768 [input.kubernetes_events]
769
770 # disable events
771 disabled = false
772
773 # override type
774 type = events
775
776 # override datastream
777 elasticsearch.datastream =
778
779 # set output (elasticsearch or devnull, default is [general]defaultOutput)
780 output =
781
782 # exclude managed fields from the metadata
783 excludeManagedFields = true
784
785 [input.kubernetes_watch::pods]
786
787 # disable events
788 disabled = false
789
790 # Set the timeout for how often watch request should refresh the whole list
791 refresh = 10m
792
793 apiVersion = v1
794 kind = Pod
795 namespace =
796 excludeManagedFields = true
797
798 # override type
799 type = objects
800
801 # override datastream
802 elasticsearch.datastream =
803
804 # set output (elasticsearch or devnull, default is [general]defaultOutput)
805 output =
806
807 # you can remove or hash some values in the events (after modifyValues you can define path in the JSON object,
808 # and the value can be hash:{hashFunction}, or remove to remove the object )
809 ; modifyValues.object.data.* = hash:sha256
810 ; modifyValues.object.metadata.annotations.* = remove
811
812 # You can exclude events by namespace with blacklist or whitelist only required namespaces
813 # blacklist.kubernetes_namespace = ^namespace0$
814 # whitelist.kubernetes_namespace = ^((namespace1)|(namespace2))$
815
816 [input.kubernetes_watch::deployments]
817
818 # disable events
819 disabled = false
820
821 # Set the timeout for how often watch request should refresh the whole list
822 refresh = 10m
823
824 apiVersion = apps/v1
825 kind = deployment
826 namespace =
827 excludeManagedFields = true
828
829 # override type
830 type = objects
831
832 # override datastream
833 elasticsearch.datastream =
834
835 # set output (elasticsearch or devnull, default is [general]defaultOutput)
836 output =
837
838 es-default-index-lifecycle-management-policy.json: |
839 {
840 "policy": {
841 "_meta": {
842 "description": "Default policy for Collectord indexes",
843 "version": 1
844 },
845 "phases": {
846 "hot": {
847 "min_age": "0ms",
848 "actions": {
849 "rollover": {
850 "max_primary_shard_size": "50gb",
851 "max_age": "30d"
852 }
853 }
854 }
855 }
856 }
857 }
858 es-default-index-template.json: |
859 {
860 "version": 1,
861 "data_stream": {},
862 "_meta": {
863 "description": "Default template for Collectord indexes"
864 },
865 "priority": 500,
866 "template": {
867 "settings": {
868 "index": {
869 "lifecycle": {
870 "name": "logs-collectord"
871 },
872 "refresh_interval": "5s",
873 "mapping": {
874 "total_fields": {
875 "limit": "10000"
876 }
877 },
878 "max_docvalue_fields_search": "200"
879 },
880 "query": {
881 "default_field": [
882 "message",
883 "container.name",
884 "container.image.name",
885 "host.name",
886 "orchestrator.namespace",
887 "orchestrator.cluster.name",
888 "container.name",
889 "kubernetes.pod.name",
890 "kubernetes.pod.id",
891 "kubernetes.pod.ip",
892 "kubernetes.namespace.name",
893 "kubernetes.node.name",
894 "kubernetes.cluster.name",
895 "kubernetes.replicaset.name",
896 "kubernetes.deployment.name",
897 "kubernetes.statefulset.name",
898 "kubernetes.daemonset.name",
899 "kubernetes.job.name",
900 "kubernetes.cronjob.name"
901 ]
902 }
903 },
904 "mappings": {
905 "dynamic_templates": [
906 {
907 "kubernetes.pod.labels.*": {
908 "path_match": "kubernetes.pod.labels.*",
909 "mapping": { "type": "keyword" },
910 "match_mapping_type": "*"
911 }
912 },
913 {
914 "kubernetes.namespace.labels.*": {
915 "path_match": "kubernetes.namespace.labels.*",
916 "mapping": { "type": "keyword" },
917 "match_mapping_type": "*"
918 }
919 },
920 {
921 "kubernetes.node.labels.*": {
922 "path_match": "kubernetes.node.labels.*",
923 "mapping": { "type": "keyword" },
924 "match_mapping_type": "*"
925 }
926 },
927 {
928 "kubernetes.daemonset.labels.*": {
929 "path_match": "kubernetes.daemonset.labels.*",
930 "mapping": { "type": "keyword" },
931 "match_mapping_type": "*"
932 }
933 },
934 {
935 "kubernetes.replicaset.labels.*": {
936 "path_match": "kubernetes.replicaset.labels.*",
937 "mapping": { "type": "keyword" },
938 "match_mapping_type": "*"
939 }
940 },
941 {
942 "kubernetes.deployment.labels.*": {
943 "path_match": "kubernetes.deployment.labels.*",
944 "mapping": { "type": "keyword" },
945 "match_mapping_type": "*"
946 }
947 },
948 {
949 "kubernetes.statefulset.labels.*": {
950 "path_match": "kubernetes.statefulset.labels.*",
951 "mapping": { "type": "keyword" },
952 "match_mapping_type": "*"
953 }
954 },
955 {
956 "kubernetes.job.labels.*": {
957 "path_match": "kubernetes.job.labels.*",
958 "mapping": { "type": "keyword" },
959 "match_mapping_type": "*"
960 }
961 },
962 {
963 "kubernetes.cronjob.labels.*": {
964 "path_match": "kubernetes.cronjob.labels.*",
965 "mapping": { "type": "keyword" },
966 "match_mapping_type": "*"
967 }
968 },
969 {
970 "strings_as_keyword": {
971 "mapping": { "ignore_above": 1024, "type": "keyword" },
972 "match_mapping_type": "string"
973 }
974 }
975 ],
976 "properties": {
977 "@timestamp": { "type": "date"},
978 "message":{"type":"match_only_text"},
979 "collectord_errors": {"type": "keyword","ignore_above": 1024},
980 "input":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},
981 "container": {"properties": {
982 "name": { "ignore_above": 1024, "type": "keyword"},
983 "id": { "ignore_above": 1024, "type": "keyword"},
984 "runtime": {"ignore_above": 1024, "type": "keyword"},
985 "image": {"properties": {
986 "name": { "ignore_above": 1024, "type": "keyword"}
987 }}
988 }
989 },
990 "event": {"properties": {
991 "id": {"ignore_above": 1024, "type": "keyword"}
992 }},
993 "host": {"properties": {
994 "name": {"ignore_above": 1024, "type": "keyword"},
995 "architecture": {"ignore_above": 1024, "type": "keyword"},
996 "hostname": {"ignore_above": 1024, "type": "keyword"}
997 }},
998 "log": { "properties": {
999 "file": { "properties": {
1000 "path": {"ignore_above": 1024, "type": "keyword"}
1001 }},
1002 "offset": {"type": "long"},
1003 "syslog": {"properties": {
1004 "appname": {"ignore_above": 1024, "type": "keyword"},
1005 "facility": {"properties": {
1006 "code": {"type": "long"}
1007 }},
1008 "priority": {"type": "long"},
1009 "procid": {"type": "long"},
1010 "hostname": {"ignore_above": 1024, "type": "keyword"}
1011 }}
1012 }},
1013 "orchestrator": {"properties": {
1014 "cluster": {"properties": {
1015 "name": {"ignore_above": 1024, "type": "keyword"}
1016 }},
1017 "namespace": {"properties": {
1018 "name":{"path":"kubernetes.namespace.name","type":"alias"}
1019 }},
1020 "type": {"ignore_above": 1024, "type": "keyword"}
1021 }},
1022 "stream": {"ignore_above": 1024, "type": "keyword"},
1023 "agent":{"properties":{
1024 "hostname":{"path":"agent.name","type":"alias"},
1025 "name":{"ignore_above":1024,"type":"keyword"},
1026 "id":{"ignore_above":1024,"type":"keyword"},
1027 "type":{"ignore_above":1024,"type":"keyword"},
1028 "ephemeral_id":{"ignore_above":1024,"type":"keyword"},
1029 "version":{"ignore_above":1024,"type":"keyword"}
1030 }},
1031 "ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},
1032 "kubernetes": { "properties": {
1033 "container": {"properties": {
1034 "imageid": {"ignore_above": 1024, "type": "keyword"}
1035 }},
1036 "host": {"properties": {
1037 "ip": {"type": "ip"}
1038 }},
1039 "namespace": {"properties": {
1040 "name": {"ignore_above": 1024, "type": "keyword"}
1041 }},
1042 "node": { "properties": {
1043 "id": {"ignore_above": 1024, "type": "keyword"},
1044 "name": {"ignore_above": 1024,"type": "keyword"}
1045 }},
1046 "pod": {"properties": {
1047 "id": {"ignore_above": 1024,"type": "keyword"},
1048 "ip": {"type": "ip"},
1049 "name": {"ignore_above": 1024,"type": "keyword"}
1050 }},
1051 "statefulset": { "properties": {
1052 "id": {"ignore_above": 1024,"type": "keyword"},
1053 "name": {"ignore_above": 1024,"type": "keyword"}
1054 }},
1055 "replicaset": {"properties": {
1056 "id": {"ignore_above": 1024,"type": "keyword"},
1057 "name": {"ignore_above": 1024, "type": "keyword"}
1058 }},
1059 "deployment": { "properties": {
1060 "id": {"ignore_above": 1024,"type": "keyword"},
1061 "name": {"ignore_above": 1024,"type": "keyword" }
1062 }},
1063 "cronjob": {"properties": {
1064 "id": {"ignore_above": 1024,"type": "keyword"},
1065 "name": {"ignore_above": 1024, "type": "keyword"}
1066 }},
1067 "job": {"properties": {
1068 "id": {"ignore_above": 1024,"type": "keyword"},
1069 "name": {"ignore_above": 1024, "type": "keyword"}
1070 }}
1071 }},
1072 "volume": {"properties": {
1073 "name": {"ignore_above": 1024,"type": "keyword"}
1074 }}
1075 }
1076 }
1077 },
1078 "index_patterns": ["logs-collectord-${COLLECTORD_VERSION}"]
1079 }
1080 es-failed-index-template.json: |
1081 {
1082 "version": 1,
1083 "data_stream": {},
1084 "_meta": {
1085 "description": "Default template for Collectord indexes for events that failed to be ingested"
1086 },
1087 "priority": 1000,
1088 "template": {
1089 "settings": {
1090 "index": {
1091 "lifecycle": {
1092 "name": "logs-collectord"
1093 },
1094 "refresh_interval": "5s",
1095 "mapping": {
1096 "total_fields": {
1097 "limit": "10000"
1098 }
1099 },
1100 "max_docvalue_fields_search": "200"
1101 },
1102 "query": {
1103 "default_field": [
1104 "message"
1105 ]
1106 }
1107 },
1108 "mappings": {
1109 "properties": {
1110 "@timestamp": {
1111 "type": "date"},
1112 "message": {
1113 "type": "match_only_text"}
1114 }
1115 }
1116 },
1117 "index_patterns": [
1118 "logs-collectord-failed-${COLLECTORD_VERSION}"]
1119 }
1120---
1121apiVersion: apps/v1
1122kind: DaemonSet
1123metadata:
1124 name: collectorforkubernetes-elasticsearch
1125 namespace: collectorforkubernetes
1126 labels:
1127 app: collectorforkubernetes-elasticsearch
1128spec:
1129 updateStrategy:
1130 type: RollingUpdate
1131
1132 selector:
1133 matchLabels:
1134 daemon: collectorforkubernetes-elasticsearch
1135
1136 template:
1137 metadata:
1138 name: collectorforkubernetes-elasticsearch
1139 labels:
1140 daemon: collectorforkubernetes-elasticsearch
1141 spec:
1142 priorityClassName: collectorforkubernetes-critical
1143 dnsPolicy: ClusterFirstWithHostNet
1144 hostNetwork: true
1145 serviceAccountName: collectorforkubernetes
1146 tolerations:
1147 - operator: "Exists"
1148 effect: "NoSchedule"
1149 - operator: "Exists"
1150 effect: "NoExecute"
1151 containers:
1152 - name: collectorforkubernetes
1153 image: docker.io/outcoldsolutions/collectorforkubernetes:26.04.1
1154 imagePullPolicy: Always
1155 securityContext:
1156 runAsUser: 0
1157 privileged: true
1158 # Define your resources if you need. Defaults should be fine for most.
1159 # You can lower or increase based on your hosts.
1160 resources:
1161 limits:
1162 cpu: 2000m
1163 memory: 512Mi
1164 requests:
1165 cpu: 200m
1166 memory: 192Mi
1167 env:
1168 - name: KUBERNETES_NODENAME
1169 valueFrom:
1170 fieldRef:
1171 fieldPath: spec.nodeName
1172 - name: POD_NAME
1173 valueFrom:
1174 fieldRef:
1175 fieldPath: metadata.name
1176 volumeMounts:
1177 # We store state in /data folder (file positions)
1178 - name: collectorforkubernetes-state
1179 mountPath: /data
1180 # Configuration file deployed with ConfigMap
1181 - name: collectorforkubernetes-config
1182 mountPath: /config/
1183 readOnly: true
1184 # Rootfs
1185 - name: rootfs
1186 mountPath: /rootfs/
1187 readOnly: false
1188 mountPropagation: HostToContainer
1189 # correct timezone
1190 - name: localtime
1191 mountPath: /etc/localtime
1192 readOnly: true
1193 volumes:
1194 # We store state directly on host, change this location, if
1195 # your persistent volume is somewhere else
1196 - name: collectorforkubernetes-state
1197 hostPath:
1198 path: /var/lib/collectorforkubernetes-elasticsearch/data/
1199 type: DirectoryOrCreate
1200 # Location of docker root (for container logs and metadata)
1201 - name: rootfs
1202 hostPath:
1203 path: /
1204 # correct timezone
1205 - name: localtime
1206 hostPath:
1207 path: /etc/localtime
1208 # configuration from ConfigMap
1209 - name: collectorforkubernetes-config
1210 configMap:
1211 name: collectorforkubernetes-elasticsearch
1212 items:
1213 - key: 001-general.conf
1214 path: 001-general.conf
1215 - key: 002-daemonset.conf
1216 path: 002-daemonset.conf
1217 - key: es-default-index-template.json
1218 path: es-default-index-template.json
1219 - key: es-default-index-lifecycle-management-policy.json
1220 path: es-default-index-lifecycle-management-policy.json
1221 - key: es-failed-index-template.json
1222 path: es-failed-index-template.json
1223---
1224apiVersion: apps/v1
1225kind: Deployment
1226metadata:
1227 name: collectorforkubernetes-elasticsearch-addon
1228 namespace: collectorforkubernetes
1229 labels:
1230 app: collectorforkubernetes-elasticsearch
1231spec:
1232 replicas: 1
1233
1234 selector:
1235 matchLabels:
1236 daemon: collectorforkubernetes-elasticsearch
1237
1238 template:
1239 metadata:
1240 name: collectorforkubernetes-elasticsearch-addon
1241 labels:
1242 daemon: collectorforkubernetes-elasticsearch
1243 spec:
1244 priorityClassName: collectorforkubernetes-critical
1245 serviceAccountName: collectorforkubernetes
1246 containers:
1247 - name: collectorforkubernetes
1248 image: docker.io/outcoldsolutions/collectorforkubernetes:26.04.1
1249 imagePullPolicy: Always
1250 securityContext:
1251 runAsUser: 0
1252 privileged: true
1253 resources:
1254 limits:
1255 cpu: 500m
1256 memory: 256Mi
1257 requests:
1258 cpu: 50m
1259 memory: 64Mi
1260 env:
1261 - name: KUBERNETES_NODENAME
1262 valueFrom:
1263 fieldRef:
1264 fieldPath: spec.nodeName
1265 - name: POD_NAME
1266 valueFrom:
1267 fieldRef:
1268 fieldPath: metadata.name
1269 volumeMounts:
1270 - name: collectorforkubernetes-state
1271 mountPath: /data
1272 - name: collectorforkubernetes-config
1273 mountPath: /config/
1274 readOnly: true
1275 volumes:
1276 - name: collectorforkubernetes-state
1277 hostPath:
1278 path: /var/lib/collectorforkubernetes-elasticsearch/data/
1279 type: Directory
1280 - name: collectorforkubernetes-config
1281 configMap:
1282 name: collectorforkubernetes-elasticsearch
1283 items:
1284 - key: 001-general.conf
1285 path: 001-general.conf
1286 - key: 004-addon.conf
1287 path: 004-addon.conf
1288 - key: es-default-index-template.json
1289 path: es-default-index-template.json
1290 - key: es-default-index-lifecycle-management-policy.json
1291 path: es-default-index-lifecycle-management-policy.json
1292 - key: es-failed-index-template.json
1293 path: es-failed-index-template.json